refactor(auth): standardize session management and security
- Centralize secret retrieval using `getAdminSessionSecret` - Replace manual session parsing with `verifyRequestAdminSession` utility - Tighten API validation and improve security headers handling - Enforce environment variable security for production deployments
This commit is contained in:
1 parent
34519c07df
commit
177ed4a844
78 files changed
+6748
-3157
No files matched your search
+123
-24
@@ -2,6 +2,15 @@ const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { Parser } = require('json2csv');
|
||||
|
||||
const LOCK_RETRY_MIN_MS = 5;
|
||||
const LOCK_RETRY_MAX_MS = 18;
|
||||
const LOCK_TIMEOUT_MS = 10000;
|
||||
const STALE_LOCK_AGE_MS = 30000;
|
||||
|
||||
function sleepMs(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/**
|
||||
* Background CSV Writer Worker
|
||||
* Flushes batches of real-time events to partitioned CSV files.
|
||||
@@ -27,14 +36,100 @@ class CsvWriterWorker {
|
||||
}
|
||||
}
|
||||
|
||||
getTargetFilePath(clientId, dateStr) {
|
||||
ensureClientDir(clientId) {
|
||||
const safeClientId = String(clientId || 'unknown').replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
const clientDir = path.join(this.baseDir, safeClientId);
|
||||
if (!fs.existsSync(clientDir)) {
|
||||
fs.mkdirSync(clientDir, { recursive: true });
|
||||
}
|
||||
return clientDir;
|
||||
}
|
||||
|
||||
const date = dateStr || new Date().toISOString().split('T')[0];
|
||||
async withPartitionLock(clientId, dateStr, fn) {
|
||||
const clientDir = this.ensureClientDir(clientId);
|
||||
const date = this.extractSafePartitionDate(dateStr);
|
||||
const lockPath = path.join(clientDir, `.${date}.csv.lock`);
|
||||
const deadline = Date.now() + LOCK_TIMEOUT_MS;
|
||||
|
||||
let acquired = false;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const fd = fs.openSync(lockPath, 'wx');
|
||||
try {
|
||||
fs.writeFileSync(fd, `${process.pid}:${Date.now()}`, 'utf8');
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
acquired = true;
|
||||
break;
|
||||
} catch (err) {
|
||||
if (!err || err.code !== 'EEXIST') {
|
||||
throw err;
|
||||
}
|
||||
try {
|
||||
const lockStat = fs.statSync(lockPath);
|
||||
if (Date.now() - lockStat.mtimeMs > STALE_LOCK_AGE_MS) {
|
||||
fs.unlinkSync(lockPath);
|
||||
continue;
|
||||
}
|
||||
} catch {
|
||||
// Released concurrently
|
||||
}
|
||||
const jitter =
|
||||
LOCK_RETRY_MIN_MS +
|
||||
Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS));
|
||||
await sleepMs(jitter);
|
||||
}
|
||||
}
|
||||
|
||||
if (!acquired) {
|
||||
throw new Error(`Timed out waiting for CSV partition lock: ${lockPath}`);
|
||||
}
|
||||
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
try {
|
||||
if (fs.existsSync(lockPath)) {
|
||||
fs.unlinkSync(lockPath);
|
||||
}
|
||||
} catch {
|
||||
// Ignore if already removed
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
extractSafePartitionDate(rawTimestamp) {
|
||||
const fallback = new Date().toISOString().slice(0, 10);
|
||||
if (!rawTimestamp || typeof rawTimestamp !== 'string') {
|
||||
return fallback;
|
||||
}
|
||||
const trimmed = rawTimestamp.trim();
|
||||
const prefix = trimmed.slice(0, 10);
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(prefix)) {
|
||||
const parsedPrefix = new Date(`${prefix}T00:00:00.000Z`);
|
||||
if (!Number.isNaN(parsedPrefix.getTime())) {
|
||||
return prefix;
|
||||
}
|
||||
}
|
||||
if (/^\d+(\.\d+)?$/.test(trimmed)) {
|
||||
const num = Number(trimmed);
|
||||
const ms = num < 1e11 ? num * 1000 : num;
|
||||
const d = new Date(ms);
|
||||
if (!Number.isNaN(d.getTime())) {
|
||||
return d.toISOString().slice(0, 10);
|
||||
}
|
||||
}
|
||||
const parsed = new Date(trimmed);
|
||||
if (!Number.isNaN(parsed.getTime())) {
|
||||
return parsed.toISOString().slice(0, 10);
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
getTargetFilePath(clientId, dateStr) {
|
||||
const clientDir = this.ensureClientDir(clientId);
|
||||
const date = this.extractSafePartitionDate(dateStr);
|
||||
let baseFileName = `${date}.csv`;
|
||||
let targetPath = path.join(clientDir, baseFileName);
|
||||
|
||||
@@ -62,7 +157,7 @@ class CsvWriterWorker {
|
||||
|
||||
const grouped = new Map();
|
||||
for (const evt of events) {
|
||||
const date = (evt.timestamp || new Date().toISOString()).split('T')[0];
|
||||
const date = this.extractSafePartitionDate(evt.timestamp);
|
||||
const key = `${evt.clientId}:::${date}`;
|
||||
if (!grouped.has(key)) grouped.set(key, []);
|
||||
grouped.get(key).push(evt);
|
||||
@@ -73,30 +168,34 @@ class CsvWriterWorker {
|
||||
|
||||
for (const [key, batch] of grouped.entries()) {
|
||||
const [clientId, date] = key.split(':::');
|
||||
const targetPath = this.getTargetFilePath(clientId, date);
|
||||
let hasContent = false;
|
||||
try {
|
||||
hasContent =
|
||||
fs.existsSync(targetPath) && fs.statSync(targetPath).size > 0;
|
||||
} catch {
|
||||
hasContent = false;
|
||||
}
|
||||
const targetPath = await this.withPartitionLock(clientId, date, () => {
|
||||
const resolvedPath = this.getTargetFilePath(clientId, date);
|
||||
let hasContent = false;
|
||||
try {
|
||||
hasContent =
|
||||
fs.existsSync(resolvedPath) && fs.statSync(resolvedPath).size > 0;
|
||||
} catch {
|
||||
hasContent = false;
|
||||
}
|
||||
|
||||
const cleanBatch = batch.map((item) => ({
|
||||
clientId: item.clientId,
|
||||
eventId: item.eventId,
|
||||
timestamp: item.timestamp,
|
||||
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
|
||||
}));
|
||||
const cleanBatch = batch.map((item) => ({
|
||||
clientId: item.clientId,
|
||||
eventId: item.eventId,
|
||||
timestamp: item.timestamp,
|
||||
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
|
||||
}));
|
||||
|
||||
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
|
||||
const csv = parser.parse(cleanBatch);
|
||||
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
|
||||
const csv = parser.parse(cleanBatch);
|
||||
|
||||
if (hasContent) {
|
||||
fs.appendFileSync(targetPath, '\n' + csv, 'utf8');
|
||||
} else {
|
||||
fs.writeFileSync(targetPath, csv, 'utf8');
|
||||
}
|
||||
if (hasContent) {
|
||||
fs.appendFileSync(resolvedPath, '\n' + csv, 'utf8');
|
||||
} else {
|
||||
fs.writeFileSync(resolvedPath, csv, 'utf8');
|
||||
}
|
||||
|
||||
return resolvedPath;
|
||||
});
|
||||
|
||||
files.push(targetPath);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user