Files
DataStack/server/workers/csvWriter.js
T
Team Dark Devil 177ed4a844 refactor(auth): standardize session management and security
- Centralize secret retrieval using `getAdminSessionSecret`
- Replace manual session parsing with `verifyRequestAdminSession` utility
- Tighten API validation and improve security headers handling
- Enforce environment variable security for production deployments
2026-11-18 05:18:55 +06:00

245 lines
7.0 KiB
JavaScript

const fs = require('fs');
const path = require('path');
const { Parser } = require('json2csv');
const LOCK_RETRY_MIN_MS = 5;
const LOCK_RETRY_MAX_MS = 18;
const LOCK_TIMEOUT_MS = 10000;
const STALE_LOCK_AGE_MS = 30000;
function sleepMs(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
/**
* Background CSV Writer Worker
* Flushes batches of real-time events to partitioned CSV files.
* Path convention: /storage/data/{clientId}/{YYYY-MM-DD}.csv
*/
class CsvWriterWorker {
constructor(baseDir = path.resolve(__dirname, '../../storage/data'), maxFileSize = 100 * 1024 * 1024) {
this.baseDir = baseDir;
this.maxFileSize = maxFileSize;
this.ensureBaseDir();
}
setMaxFileSizeMb(megabytes) {
const mb = Number(megabytes);
if (Number.isFinite(mb) && mb >= 1) {
this.maxFileSize = Math.floor(mb * 1024 * 1024);
}
}
ensureBaseDir() {
if (!fs.existsSync(this.baseDir)) {
fs.mkdirSync(this.baseDir, { recursive: true });
}
}
ensureClientDir(clientId) {
const safeClientId = String(clientId || 'unknown').replace(/[^a-zA-Z0-9_-]/g, '_');
const clientDir = path.join(this.baseDir, safeClientId);
if (!fs.existsSync(clientDir)) {
fs.mkdirSync(clientDir, { recursive: true });
}
return clientDir;
}
async withPartitionLock(clientId, dateStr, fn) {
const clientDir = this.ensureClientDir(clientId);
const date = this.extractSafePartitionDate(dateStr);
const lockPath = path.join(clientDir, `.${date}.csv.lock`);
const deadline = Date.now() + LOCK_TIMEOUT_MS;
let acquired = false;
while (Date.now() < deadline) {
try {
const fd = fs.openSync(lockPath, 'wx');
try {
fs.writeFileSync(fd, `${process.pid}:${Date.now()}`, 'utf8');
} finally {
fs.closeSync(fd);
}
acquired = true;
break;
} catch (err) {
if (!err || err.code !== 'EEXIST') {
throw err;
}
try {
const lockStat = fs.statSync(lockPath);
if (Date.now() - lockStat.mtimeMs > STALE_LOCK_AGE_MS) {
fs.unlinkSync(lockPath);
continue;
}
} catch {
// Released concurrently
}
const jitter =
LOCK_RETRY_MIN_MS +
Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS));
await sleepMs(jitter);
}
}
if (!acquired) {
throw new Error(`Timed out waiting for CSV partition lock: ${lockPath}`);
}
try {
return await fn();
} finally {
try {
if (fs.existsSync(lockPath)) {
fs.unlinkSync(lockPath);
}
} catch {
// Ignore if already removed
}
}
}
extractSafePartitionDate(rawTimestamp) {
const fallback = new Date().toISOString().slice(0, 10);
if (!rawTimestamp || typeof rawTimestamp !== 'string') {
return fallback;
}
const trimmed = rawTimestamp.trim();
const prefix = trimmed.slice(0, 10);
if (/^\d{4}-\d{2}-\d{2}$/.test(prefix)) {
const parsedPrefix = new Date(`${prefix}T00:00:00.000Z`);
if (!Number.isNaN(parsedPrefix.getTime())) {
return prefix;
}
}
if (/^\d+(\.\d+)?$/.test(trimmed)) {
const num = Number(trimmed);
const ms = num < 1e11 ? num * 1000 : num;
const d = new Date(ms);
if (!Number.isNaN(d.getTime())) {
return d.toISOString().slice(0, 10);
}
}
const parsed = new Date(trimmed);
if (!Number.isNaN(parsed.getTime())) {
return parsed.toISOString().slice(0, 10);
}
return fallback;
}
getTargetFilePath(clientId, dateStr) {
const clientDir = this.ensureClientDir(clientId);
const date = this.extractSafePartitionDate(dateStr);
let baseFileName = `${date}.csv`;
let targetPath = path.join(clientDir, baseFileName);
let partIndex = 1;
while (fs.existsSync(targetPath)) {
try {
const stats = fs.statSync(targetPath);
if (stats.size >= this.maxFileSize) {
partIndex += 1;
baseFileName = `${date}-part${partIndex}.csv`;
targetPath = path.join(clientDir, baseFileName);
} else {
break;
}
} catch {
break;
}
}
return targetPath;
}
async writeBatch(events) {
if (!events || events.length === 0) return { written: 0, files: [] };
const grouped = new Map();
for (const evt of events) {
const date = this.extractSafePartitionDate(evt.timestamp);
const key = `${evt.clientId}:::${date}`;
if (!grouped.has(key)) grouped.set(key, []);
grouped.get(key).push(evt);
}
const files = [];
const fields = ['clientId', 'eventId', 'timestamp', 'payload'];
for (const [key, batch] of grouped.entries()) {
const [clientId, date] = key.split(':::');
const targetPath = await this.withPartitionLock(clientId, date, () => {
const resolvedPath = this.getTargetFilePath(clientId, date);
let hasContent = false;
try {
hasContent =
fs.existsSync(resolvedPath) && fs.statSync(resolvedPath).size > 0;
} catch {
hasContent = false;
}
const cleanBatch = batch.map((item) => ({
clientId: item.clientId,
eventId: item.eventId,
timestamp: item.timestamp,
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
}));
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
const csv = parser.parse(cleanBatch);
if (hasContent) {
fs.appendFileSync(resolvedPath, '\n' + csv, 'utf8');
} else {
fs.writeFileSync(resolvedPath, csv, 'utf8');
}
return resolvedPath;
});
files.push(targetPath);
}
return { written: events.length, files };
}
pruneExpiredPartitions(autoPruneDays) {
const days = Number(autoPruneDays);
if (!Number.isFinite(days) || days <= 0 || !fs.existsSync(this.baseDir)) {
return 0;
}
const cutoffDate = new Date(Date.now() - Math.floor(days) * 86400000)
.toISOString()
.split('T')[0];
let prunedCount = 0;
try {
const clientDirs = fs.readdirSync(this.baseDir);
for (const clientDir of clientDirs) {
const fullClientPath = path.join(this.baseDir, clientDir);
if (!fs.statSync(fullClientPath).isDirectory()) continue;
const files = fs.readdirSync(fullClientPath);
for (const file of files) {
if (!file.endsWith('.csv')) continue;
const datePart = file.replace('.csv', '').split('-part')[0];
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
try {
fs.unlinkSync(path.join(fullClientPath, file));
prunedCount += 1;
} catch {
// Ignore individual file unlink failure
}
}
}
}
} catch {
// Ignore directory scan failure
}
return prunedCount;
}
}
module.exports = { CsvWriterWorker };