feat(auth): harden session management and security
- Implement server-side session revocation on logout - Add IP extraction utility and admin setup error handling - Improve error codes and response consistency for auth endpoints - Refine landing page redirect logic to handle transient DB outages - Update deployment documentation and configuration defaults
This commit is contained in:
1 parent
935f12a1e9
commit
89e4d68690
81 files changed
+11530
-6916
No files matched your search
+841
-244
File diff suppressed because it is too large.
Load diff
+274
-29
@@ -1,8 +1,30 @@
|
||||
const crypto = require("node:crypto");
|
||||
const jwt = require("jsonwebtoken");
|
||||
|
||||
const IDEMPOTENCY_TTL_MS = 5 * 60 * 1000; // 5 minutes
|
||||
const IDEMPOTENCY_MAX_ENTRIES = 20000;
|
||||
const CLEANUP_INTERVAL_MS = 60 * 1000;
|
||||
const REDIS_IDEMPOTENCY_PREFIX = "datastack:idem:";
|
||||
const REDIS_RATE_LIMIT_PREFIX = "datastack:rl:";
|
||||
const REDIS_RATE_LIMIT_LUA_SCRIPT = [
|
||||
"-- DATASTACK_RATE_LIMIT_INCR",
|
||||
'local c = redis.call("INCR", KEYS[1])',
|
||||
'if c == 1 or redis.call("PTTL", KEYS[1]) < 0 then',
|
||||
' redis.call("PEXPIRE", KEYS[1], tonumber(ARGV[1]))',
|
||||
"end",
|
||||
"return c",
|
||||
].join("\n");
|
||||
|
||||
function generateFallbackEventId() {
|
||||
return `evt_${Date.now()}_${crypto.randomBytes(6).toString("hex")}`;
|
||||
}
|
||||
|
||||
function normalizeClientId(clientId) {
|
||||
return String(clientId || "")
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9_-]/g, "_");
|
||||
}
|
||||
|
||||
function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
const getRuntimeConfig =
|
||||
@@ -36,6 +58,11 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
? options.onBufferPush
|
||||
: () => undefined;
|
||||
|
||||
const getRedisClient =
|
||||
typeof options.getRedisClient === "function"
|
||||
? options.getRedisClient
|
||||
: () => null;
|
||||
|
||||
const idempotencyMap = new Map();
|
||||
const rateLimitMap = new Map();
|
||||
let lastCleanupAt = Date.now();
|
||||
@@ -63,6 +90,25 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
}
|
||||
}
|
||||
|
||||
function trackLocalRateWindow(clientId, now) {
|
||||
const rl = rateLimitMap.get(clientId);
|
||||
if (!rl || now - rl.start >= 1000) {
|
||||
rateLimitMap.set(clientId, { count: 1, start: now });
|
||||
} else {
|
||||
rl.count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
function supportsSharedSecurityRedis(redis) {
|
||||
return Boolean(
|
||||
redis &&
|
||||
typeof redis.get === "function" &&
|
||||
typeof redis.incr === "function" &&
|
||||
typeof redis.pexpire === "function" &&
|
||||
typeof redis.set === "function",
|
||||
);
|
||||
}
|
||||
|
||||
function getSecurityStats() {
|
||||
cleanupMaps(Date.now(), true);
|
||||
const config = getRuntimeConfig();
|
||||
@@ -73,7 +119,13 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
function validateSingleEvent(clientId, eventId, rawData, clientRateLimit) {
|
||||
function validateSingleEventLocal(
|
||||
clientId,
|
||||
eventId,
|
||||
rawData,
|
||||
clientRateLimit,
|
||||
explicitByteLength,
|
||||
) {
|
||||
const config = getRuntimeConfig();
|
||||
const now = Date.now();
|
||||
cleanupMaps(now);
|
||||
@@ -93,9 +145,19 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
}
|
||||
|
||||
// 2. Payload byte size check (before consuming rate-limit quota)
|
||||
const payloadStr =
|
||||
typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {});
|
||||
const byteLength = Buffer.byteLength(payloadStr, "utf8");
|
||||
let payloadStr = "";
|
||||
let byteLength = 0;
|
||||
if (
|
||||
typeof explicitByteLength === "number" &&
|
||||
Number.isFinite(explicitByteLength) &&
|
||||
explicitByteLength >= 0
|
||||
) {
|
||||
byteLength = explicitByteLength;
|
||||
} else {
|
||||
payloadStr =
|
||||
typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {});
|
||||
byteLength = Buffer.byteLength(payloadStr, "utf8");
|
||||
}
|
||||
const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024;
|
||||
if (byteLength > maxBytes) {
|
||||
return {
|
||||
@@ -133,6 +195,126 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
return { allowed: true, payloadStr };
|
||||
}
|
||||
|
||||
async function validateSingleEventAsync(
|
||||
clientId,
|
||||
eventId,
|
||||
rawData,
|
||||
clientRateLimit,
|
||||
) {
|
||||
const redis = getRedisClient();
|
||||
if (!supportsSharedSecurityRedis(redis)) {
|
||||
return validateSingleEventLocal(
|
||||
clientId,
|
||||
eventId,
|
||||
rawData,
|
||||
clientRateLimit,
|
||||
);
|
||||
}
|
||||
|
||||
const config = getRuntimeConfig();
|
||||
const now = Date.now();
|
||||
cleanupMaps(now);
|
||||
const idempotencyKey = `${clientId}:${eventId}`;
|
||||
const redisIdemKey = `${REDIS_IDEMPOTENCY_PREFIX}${clientId}:${eventId}`;
|
||||
|
||||
try {
|
||||
// 1. Read-only idempotency check
|
||||
if (config.enableIdempotency && eventId) {
|
||||
const localTime = idempotencyMap.get(idempotencyKey);
|
||||
if (localTime && now - localTime < IDEMPOTENCY_TTL_MS) {
|
||||
return {
|
||||
allowed: false,
|
||||
duplicate: true,
|
||||
code: "DUPLICATE_EVENT",
|
||||
reason: "Duplicate event dropped",
|
||||
};
|
||||
}
|
||||
const existing = await redis.get(redisIdemKey);
|
||||
if (existing !== null) {
|
||||
idempotencyMap.set(idempotencyKey, now);
|
||||
return {
|
||||
allowed: false,
|
||||
duplicate: true,
|
||||
code: "DUPLICATE_EVENT",
|
||||
reason: "Duplicate event dropped",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Payload byte size check
|
||||
const payloadStr =
|
||||
typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {});
|
||||
const byteLength = Buffer.byteLength(payloadStr, "utf8");
|
||||
const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024;
|
||||
if (byteLength > maxBytes) {
|
||||
return {
|
||||
allowed: false,
|
||||
duplicate: false,
|
||||
code: "PAYLOAD_TOO_LARGE",
|
||||
reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`,
|
||||
};
|
||||
}
|
||||
|
||||
// 3. Shared atomic Redis rate limit check (single Lua round-trip)
|
||||
const effectiveLimit = Math.min(
|
||||
Number(clientRateLimit) || 100,
|
||||
Number(config.maxEventsPerSec) || 100,
|
||||
);
|
||||
const windowBucket = Math.floor(now / 1000);
|
||||
const rlKey = `${REDIS_RATE_LIMIT_PREFIX}${clientId}:${windowBucket}`;
|
||||
let count;
|
||||
if (typeof redis.eval === "function") {
|
||||
count = Number(
|
||||
await redis.eval(REDIS_RATE_LIMIT_LUA_SCRIPT, 1, rlKey, "2000"),
|
||||
);
|
||||
} else {
|
||||
count = Number(await redis.incr(rlKey));
|
||||
if (count === 1) {
|
||||
await redis.pexpire(rlKey, 2000);
|
||||
}
|
||||
}
|
||||
trackLocalRateWindow(clientId, now);
|
||||
|
||||
if (count > effectiveLimit) {
|
||||
return {
|
||||
allowed: false,
|
||||
duplicate: false,
|
||||
code: "RATE_LIMIT_EXCEEDED",
|
||||
reason: `RATE_LIMIT_EXCEEDED: Limit of ${effectiveLimit} events/sec exceeded`,
|
||||
};
|
||||
}
|
||||
|
||||
// 4. Atomic claim of idempotency key after all checks succeed
|
||||
if (config.enableIdempotency && eventId) {
|
||||
const claimed = await redis.set(
|
||||
redisIdemKey,
|
||||
"1",
|
||||
"PX",
|
||||
IDEMPOTENCY_TTL_MS,
|
||||
"NX",
|
||||
);
|
||||
idempotencyMap.set(idempotencyKey, now);
|
||||
if (claimed !== "OK") {
|
||||
return {
|
||||
allowed: false,
|
||||
duplicate: true,
|
||||
code: "DUPLICATE_EVENT",
|
||||
reason: "Duplicate event dropped",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return { allowed: true, payloadStr };
|
||||
} catch {
|
||||
return validateSingleEventLocal(
|
||||
clientId,
|
||||
eventId,
|
||||
rawData,
|
||||
clientRateLimit,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Middleware: Handshake Auth
|
||||
io.use(async (socket, next) => {
|
||||
const authHeader =
|
||||
@@ -157,9 +339,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
}
|
||||
const decoded = jwt.verify(cleanToken, config.jwtSecret);
|
||||
|
||||
const normalizedClientId = String(decoded.clientId || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const normalizedClientId = normalizeClientId(decoded.clientId);
|
||||
const dbCheck = await verifyClientInDb(
|
||||
normalizedClientId,
|
||||
cleanToken,
|
||||
@@ -181,18 +361,37 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
|
||||
const ENVELOPE_KEYS = new Set(["eventId", "timestamp", "data", "clientId"]);
|
||||
function extractBatchItemData(safeItem) {
|
||||
return extractSingleEventData(safeItem);
|
||||
}
|
||||
|
||||
function extractSingleEventData(rawPayload) {
|
||||
if (
|
||||
safeItem &&
|
||||
typeof safeItem === "object" &&
|
||||
!Array.isArray(safeItem) &&
|
||||
safeItem.data !== undefined
|
||||
rawPayload === null ||
|
||||
rawPayload === undefined ||
|
||||
typeof rawPayload !== "object" ||
|
||||
Array.isArray(rawPayload)
|
||||
) {
|
||||
const keys = Object.keys(safeItem);
|
||||
if (keys.every((k) => ENVELOPE_KEYS.has(k))) {
|
||||
return safeItem.data;
|
||||
}
|
||||
return rawPayload === undefined ? {} : rawPayload;
|
||||
}
|
||||
return safeItem;
|
||||
|
||||
if (rawPayload.data !== undefined) {
|
||||
const keys = Object.keys(rawPayload);
|
||||
if (keys.every((k) => ENVELOPE_KEYS.has(k))) {
|
||||
return rawPayload.data;
|
||||
}
|
||||
return rawPayload;
|
||||
}
|
||||
|
||||
const stripped = {};
|
||||
let hasNonEnvelopeKey = false;
|
||||
for (const [k, v] of Object.entries(rawPayload)) {
|
||||
if (k === "eventId" || k === "timestamp" || k === "clientId") {
|
||||
continue;
|
||||
}
|
||||
hasNonEnvelopeKey = true;
|
||||
stripped[k] = v;
|
||||
}
|
||||
return hasNonEnvelopeKey ? stripped : {};
|
||||
}
|
||||
|
||||
io.on("connection", (socket) => {
|
||||
@@ -221,16 +420,25 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
|
||||
const effectiveClientRate =
|
||||
dbCheck.rateLimit || socket.data.rateLimit || 100;
|
||||
const safePayload = payload && typeof payload === "object" ? payload : {};
|
||||
const safePayload =
|
||||
payload && typeof payload === "object" && !Array.isArray(payload)
|
||||
? payload
|
||||
: {};
|
||||
const eventId =
|
||||
safePayload.eventId ||
|
||||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||
const timestamp = safePayload.timestamp || new Date().toISOString();
|
||||
typeof safePayload.eventId === "string" && safePayload.eventId.trim()
|
||||
? safePayload.eventId.trim()
|
||||
: generateFallbackEventId();
|
||||
const timestamp =
|
||||
typeof safePayload.timestamp === "string" &&
|
||||
safePayload.timestamp.trim()
|
||||
? safePayload.timestamp.trim()
|
||||
: new Date().toISOString();
|
||||
const eventData = extractSingleEventData(payload);
|
||||
|
||||
const check = validateSingleEvent(
|
||||
const check = await validateSingleEventAsync(
|
||||
clientId,
|
||||
eventId,
|
||||
safePayload.data,
|
||||
eventData,
|
||||
effectiveClientRate,
|
||||
);
|
||||
if (!check.allowed) {
|
||||
@@ -260,7 +468,22 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
|
||||
bufferQueue.push(event);
|
||||
recordClientActivity(clientId, 1);
|
||||
onBufferPush();
|
||||
try {
|
||||
await onBufferPush();
|
||||
} catch (persistErr) {
|
||||
if (callback) {
|
||||
callback({
|
||||
success: false,
|
||||
eventId,
|
||||
code: "BUFFER_PERSIST_FAILED",
|
||||
error:
|
||||
persistErr instanceof Error
|
||||
? persistErr.message
|
||||
: "Failed to persist event to buffer",
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (callback) callback({ success: true, eventId });
|
||||
} catch (err) {
|
||||
@@ -309,12 +532,16 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
for (const item of batch) {
|
||||
const safeItem = item && typeof item === "object" ? item : {};
|
||||
const eventId =
|
||||
safeItem.eventId ||
|
||||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||
const timestamp = safeItem.timestamp || new Date().toISOString();
|
||||
typeof safeItem.eventId === "string" && safeItem.eventId.trim()
|
||||
? safeItem.eventId.trim()
|
||||
: generateFallbackEventId();
|
||||
const timestamp =
|
||||
typeof safeItem.timestamp === "string" && safeItem.timestamp.trim()
|
||||
? safeItem.timestamp.trim()
|
||||
: new Date().toISOString();
|
||||
const rawData = extractBatchItemData(safeItem);
|
||||
|
||||
const check = validateSingleEvent(
|
||||
const check = await validateSingleEventAsync(
|
||||
clientId,
|
||||
eventId,
|
||||
rawData,
|
||||
@@ -351,7 +578,24 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
|
||||
if (processed > 0) {
|
||||
recordClientActivity(clientId, processed);
|
||||
onBufferPush();
|
||||
try {
|
||||
await onBufferPush();
|
||||
} catch (persistErr) {
|
||||
if (callback) {
|
||||
callback({
|
||||
success: false,
|
||||
processed: 0,
|
||||
rejected: batch.length,
|
||||
duplicates,
|
||||
code: "BUFFER_PERSIST_FAILED",
|
||||
error:
|
||||
persistErr instanceof Error
|
||||
? persistErr.message
|
||||
: "Failed to persist batch to buffer",
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (processed === 0 && duplicates === 0 && rejected > 0) {
|
||||
@@ -443,7 +687,8 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
|
||||
return {
|
||||
getSecurityStats,
|
||||
validateSingleEventLocal,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { setupSocketHandlers };
|
||||
module.exports = { setupSocketHandlers, normalizeClientId };
|
||||
+267
-53
@@ -1,3 +1,4 @@
|
||||
const crypto = require('node:crypto');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { Parser } = require('json2csv');
|
||||
@@ -11,13 +12,21 @@ function sleepMs(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function resolveDefaultStorageDir() {
|
||||
const configured = (process.env.DATASTACK_STORAGE_DIR || '').trim();
|
||||
if (configured) {
|
||||
return path.resolve(configured);
|
||||
}
|
||||
return path.resolve(__dirname, '../../storage/data');
|
||||
}
|
||||
|
||||
/**
|
||||
* Background CSV Writer Worker
|
||||
* Flushes batches of real-time events to partitioned CSV files.
|
||||
* Path convention: /storage/data/{clientId}/{YYYY-MM-DD}.csv
|
||||
*/
|
||||
class CsvWriterWorker {
|
||||
constructor(baseDir = path.resolve(__dirname, '../../storage/data'), maxFileSize = 100 * 1024 * 1024) {
|
||||
constructor(baseDir = resolveDefaultStorageDir(), maxFileSize = 100 * 1024 * 1024) {
|
||||
this.baseDir = baseDir;
|
||||
this.maxFileSize = maxFileSize;
|
||||
this.ensureBaseDir();
|
||||
@@ -36,9 +45,85 @@ class CsvWriterWorker {
|
||||
}
|
||||
}
|
||||
|
||||
sanitizeClientId(clientId) {
|
||||
const raw = String(clientId || 'unknown').trim().replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
return raw || 'unknown';
|
||||
}
|
||||
|
||||
fileEndsWithNewline(filePath, sizeBytes) {
|
||||
if (sizeBytes <= 0) return false;
|
||||
const fd = fs.openSync(filePath, 'r');
|
||||
try {
|
||||
const lastByte = Buffer.allocUnsafe(1);
|
||||
const read = fs.readSync(fd, lastByte, 0, 1, sizeBytes - 1);
|
||||
return read === 1 && lastByte[0] === 0x0a;
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
}
|
||||
|
||||
removeStaleLockAtomically(lockPath) {
|
||||
try {
|
||||
const lockStat = fs.lstatSync(lockPath);
|
||||
if (Date.now() - lockStat.mtimeMs <= STALE_LOCK_AGE_MS) {
|
||||
return;
|
||||
}
|
||||
const staleTmp = `${lockPath}.stale.${crypto.randomUUID()}`;
|
||||
fs.renameSync(lockPath, staleTmp);
|
||||
try {
|
||||
fs.unlinkSync(staleTmp);
|
||||
} catch {
|
||||
// Ignore cleanup failure
|
||||
}
|
||||
} catch {
|
||||
// Lock was released or claimed concurrently
|
||||
}
|
||||
}
|
||||
|
||||
releaseLockIfOwned(lockPath, lockValue) {
|
||||
try {
|
||||
if (!fs.existsSync(lockPath)) return;
|
||||
const current = fs.readFileSync(lockPath, 'utf8');
|
||||
if (current === lockValue) {
|
||||
fs.unlinkSync(lockPath);
|
||||
}
|
||||
} catch {
|
||||
// Ignore if already removed
|
||||
}
|
||||
}
|
||||
|
||||
async removeStaleLockAtomicallyAsync(lockPath) {
|
||||
try {
|
||||
const lockStat = await fs.promises.lstat(lockPath);
|
||||
if (Date.now() - lockStat.mtimeMs <= STALE_LOCK_AGE_MS) {
|
||||
return;
|
||||
}
|
||||
const staleTmp = `${lockPath}.stale.${crypto.randomUUID()}`;
|
||||
await fs.promises.rename(lockPath, staleTmp);
|
||||
await fs.promises.unlink(staleTmp).catch(() => undefined);
|
||||
} catch {
|
||||
// Lock was released or claimed concurrently
|
||||
}
|
||||
}
|
||||
|
||||
async releaseLockIfOwnedAsync(lockPath, lockValue) {
|
||||
try {
|
||||
const current = await fs.promises.readFile(lockPath, 'utf8');
|
||||
if (current === lockValue) {
|
||||
await fs.promises.unlink(lockPath);
|
||||
}
|
||||
} catch {
|
||||
// Ignore if already removed
|
||||
}
|
||||
}
|
||||
|
||||
ensureClientDir(clientId) {
|
||||
const safeClientId = String(clientId || 'unknown').replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
const clientDir = path.join(this.baseDir, safeClientId);
|
||||
const safeClientId = this.sanitizeClientId(clientId);
|
||||
const resolvedBase = path.resolve(this.baseDir);
|
||||
const clientDir = path.resolve(resolvedBase, safeClientId);
|
||||
if (!clientDir.startsWith(resolvedBase + path.sep)) {
|
||||
throw new Error(`Invalid storage client directory: ${clientId}`);
|
||||
}
|
||||
if (!fs.existsSync(clientDir)) {
|
||||
fs.mkdirSync(clientDir, { recursive: true });
|
||||
}
|
||||
@@ -49,6 +134,7 @@ class CsvWriterWorker {
|
||||
const clientDir = this.ensureClientDir(clientId);
|
||||
const date = this.extractSafePartitionDate(dateStr);
|
||||
const lockPath = path.join(clientDir, `.${date}.csv.lock`);
|
||||
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
|
||||
const deadline = Date.now() + LOCK_TIMEOUT_MS;
|
||||
|
||||
let acquired = false;
|
||||
@@ -56,7 +142,7 @@ class CsvWriterWorker {
|
||||
try {
|
||||
const fd = fs.openSync(lockPath, 'wx');
|
||||
try {
|
||||
fs.writeFileSync(fd, `${process.pid}:${Date.now()}`, 'utf8');
|
||||
fs.writeFileSync(fd, lockValue, 'utf8');
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
@@ -66,15 +152,7 @@ class CsvWriterWorker {
|
||||
if (!err || err.code !== 'EEXIST') {
|
||||
throw err;
|
||||
}
|
||||
try {
|
||||
const lockStat = fs.statSync(lockPath);
|
||||
if (Date.now() - lockStat.mtimeMs > STALE_LOCK_AGE_MS) {
|
||||
fs.unlinkSync(lockPath);
|
||||
continue;
|
||||
}
|
||||
} catch {
|
||||
// Released concurrently
|
||||
}
|
||||
this.removeStaleLockAtomically(lockPath);
|
||||
const jitter =
|
||||
LOCK_RETRY_MIN_MS +
|
||||
Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS));
|
||||
@@ -89,13 +167,7 @@ class CsvWriterWorker {
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
try {
|
||||
if (fs.existsSync(lockPath)) {
|
||||
fs.unlinkSync(lockPath);
|
||||
}
|
||||
} catch {
|
||||
// Ignore if already removed
|
||||
}
|
||||
this.releaseLockIfOwned(lockPath, lockValue);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -127,7 +199,7 @@ class CsvWriterWorker {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
getTargetFilePath(clientId, dateStr) {
|
||||
getTargetFilePath(clientId, dateStr, incomingBytes = 0) {
|
||||
const clientDir = this.ensureClientDir(clientId);
|
||||
const date = this.extractSafePartitionDate(dateStr);
|
||||
let baseFileName = `${date}.csv`;
|
||||
@@ -135,16 +207,24 @@ class CsvWriterWorker {
|
||||
|
||||
let partIndex = 1;
|
||||
while (fs.existsSync(targetPath)) {
|
||||
let stats;
|
||||
try {
|
||||
const stats = fs.statSync(targetPath);
|
||||
if (stats.size >= this.maxFileSize) {
|
||||
partIndex += 1;
|
||||
baseFileName = `${date}-part${partIndex}.csv`;
|
||||
targetPath = path.join(clientDir, baseFileName);
|
||||
} else {
|
||||
stats = fs.statSync(targetPath);
|
||||
} catch (err) {
|
||||
if (err && err.code === 'ENOENT') {
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
throw err;
|
||||
}
|
||||
const overflowsLimit =
|
||||
stats.size > 0 &&
|
||||
incomingBytes > 0 &&
|
||||
stats.size + incomingBytes > this.maxFileSize;
|
||||
if (stats.size >= this.maxFileSize || overflowsLimit) {
|
||||
partIndex += 1;
|
||||
baseFileName = `${date}-part${partIndex}.csv`;
|
||||
targetPath = path.join(clientDir, baseFileName);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -158,40 +238,48 @@ class CsvWriterWorker {
|
||||
const grouped = new Map();
|
||||
for (const evt of events) {
|
||||
const date = this.extractSafePartitionDate(evt.timestamp);
|
||||
const key = `${evt.clientId}:::${date}`;
|
||||
if (!grouped.has(key)) grouped.set(key, []);
|
||||
grouped.get(key).push(evt);
|
||||
const clientId = String(evt.clientId || 'unknown');
|
||||
const key = JSON.stringify([clientId, date]);
|
||||
let bucket = grouped.get(key);
|
||||
if (!bucket) {
|
||||
bucket = { clientId, date, events: [] };
|
||||
grouped.set(key, bucket);
|
||||
}
|
||||
bucket.events.push(evt);
|
||||
}
|
||||
|
||||
const files = [];
|
||||
const fields = ['clientId', 'eventId', 'timestamp', 'payload'];
|
||||
|
||||
for (const [key, batch] of grouped.entries()) {
|
||||
const [clientId, date] = key.split(':::');
|
||||
for (const { clientId, date, events: batch } of grouped.values()) {
|
||||
const cleanBatch = batch.map((item) => ({
|
||||
clientId: item.clientId,
|
||||
eventId: item.eventId,
|
||||
timestamp: item.timestamp,
|
||||
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
|
||||
}));
|
||||
|
||||
const bodyCsvNoHeader = new Parser({ fields, header: false, eol: '\n' }).parse(cleanBatch);
|
||||
const incomingBytes = Buffer.byteLength(bodyCsvNoHeader, 'utf8') + 1;
|
||||
|
||||
const targetPath = await this.withPartitionLock(clientId, date, () => {
|
||||
const resolvedPath = this.getTargetFilePath(clientId, date);
|
||||
let hasContent = false;
|
||||
const resolvedPath = this.getTargetFilePath(clientId, date, incomingBytes);
|
||||
let sizeBytes = 0;
|
||||
try {
|
||||
hasContent =
|
||||
fs.existsSync(resolvedPath) && fs.statSync(resolvedPath).size > 0;
|
||||
if (fs.existsSync(resolvedPath)) {
|
||||
sizeBytes = fs.statSync(resolvedPath).size;
|
||||
}
|
||||
} catch {
|
||||
hasContent = false;
|
||||
sizeBytes = 0;
|
||||
}
|
||||
|
||||
const cleanBatch = batch.map((item) => ({
|
||||
clientId: item.clientId,
|
||||
eventId: item.eventId,
|
||||
timestamp: item.timestamp,
|
||||
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
|
||||
}));
|
||||
|
||||
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
|
||||
const csv = parser.parse(cleanBatch);
|
||||
const hasContent = sizeBytes > 0;
|
||||
|
||||
if (hasContent) {
|
||||
fs.appendFileSync(resolvedPath, '\n' + csv, 'utf8');
|
||||
const endsWithNl = this.fileEndsWithNewline(resolvedPath, sizeBytes);
|
||||
fs.appendFileSync(resolvedPath, (endsWithNl ? '' : '\n') + bodyCsvNoHeader, 'utf8');
|
||||
} else {
|
||||
fs.writeFileSync(resolvedPath, csv, 'utf8');
|
||||
const fullCsv = new Parser({ fields, header: true, eol: '\n' }).parse(cleanBatch);
|
||||
fs.writeFileSync(resolvedPath, fullCsv, 'utf8');
|
||||
}
|
||||
|
||||
return resolvedPath;
|
||||
@@ -218,18 +306,144 @@ class CsvWriterWorker {
|
||||
const clientDirs = fs.readdirSync(this.baseDir);
|
||||
for (const clientDir of clientDirs) {
|
||||
const fullClientPath = path.join(this.baseDir, clientDir);
|
||||
if (!fs.statSync(fullClientPath).isDirectory()) continue;
|
||||
let dirLstat;
|
||||
try {
|
||||
dirLstat = fs.lstatSync(fullClientPath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
|
||||
|
||||
const files = fs.readdirSync(fullClientPath);
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
|
||||
const fullPath = path.join(fullClientPath, file);
|
||||
let fileLstat;
|
||||
try {
|
||||
fs.unlinkSync(path.join(fullClientPath, file));
|
||||
fileLstat = fs.lstatSync(fullPath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (fileLstat.isSymbolicLink() || !fileLstat.isFile()) continue;
|
||||
|
||||
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
|
||||
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
|
||||
let acquired = false;
|
||||
try {
|
||||
const fd = fs.openSync(lockPath, 'wx');
|
||||
try {
|
||||
fs.writeFileSync(fd, lockValue, 'utf8');
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
acquired = true;
|
||||
} catch (err) {
|
||||
if (err && err.code === 'EEXIST') {
|
||||
this.removeStaleLockAtomically(lockPath);
|
||||
try {
|
||||
const fd = fs.openSync(lockPath, 'wx');
|
||||
try {
|
||||
fs.writeFileSync(fd, lockValue, 'utf8');
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
acquired = true;
|
||||
} catch {
|
||||
acquired = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!acquired) continue;
|
||||
|
||||
try {
|
||||
if (fs.existsSync(fullPath)) {
|
||||
fs.unlinkSync(fullPath);
|
||||
prunedCount += 1;
|
||||
}
|
||||
} catch {
|
||||
// Ignore individual file unlink failure
|
||||
} finally {
|
||||
this.releaseLockIfOwned(lockPath, lockValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore directory scan failure
|
||||
}
|
||||
return prunedCount;
|
||||
}
|
||||
|
||||
async pruneExpiredPartitionsAsync(autoPruneDays) {
|
||||
const days = Number(autoPruneDays);
|
||||
if (!Number.isFinite(days) || days <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const baseStat = await fs.promises.lstat(this.baseDir).catch(() => null);
|
||||
if (!baseStat || !baseStat.isDirectory()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const cutoffDate = new Date(Date.now() - Math.floor(days) * 86400000)
|
||||
.toISOString()
|
||||
.split('T')[0];
|
||||
|
||||
let prunedCount = 0;
|
||||
try {
|
||||
const clientDirs = await fs.promises.readdir(this.baseDir);
|
||||
for (const clientDir of clientDirs) {
|
||||
const fullClientPath = path.join(this.baseDir, clientDir);
|
||||
const dirLstat = await fs.promises.lstat(fullClientPath).catch(() => null);
|
||||
if (!dirLstat || dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
|
||||
|
||||
const files = await fs.promises.readdir(fullClientPath).catch(() => []);
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
|
||||
const fullPath = path.join(fullClientPath, file);
|
||||
const fileLstat = await fs.promises.lstat(fullPath).catch(() => null);
|
||||
if (!fileLstat || fileLstat.isSymbolicLink() || !fileLstat.isFile()) continue;
|
||||
|
||||
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
|
||||
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
|
||||
let acquired = false;
|
||||
const tryAcquireLock = async () => {
|
||||
const handle = await fs.promises.open(lockPath, 'wx');
|
||||
try {
|
||||
await handle.writeFile(lockValue, 'utf8');
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
try {
|
||||
acquired = await tryAcquireLock();
|
||||
} catch (err) {
|
||||
if (err && err.code === 'EEXIST') {
|
||||
await this.removeStaleLockAtomicallyAsync(lockPath);
|
||||
try {
|
||||
acquired = await tryAcquireLock();
|
||||
} catch {
|
||||
acquired = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!acquired) continue;
|
||||
|
||||
try {
|
||||
await fs.promises.unlink(fullPath);
|
||||
prunedCount += 1;
|
||||
} catch {
|
||||
// Ignore individual file unlink failure
|
||||
} finally {
|
||||
await this.releaseLockIfOwnedAsync(lockPath, lockValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -241,4 +455,4 @@ class CsvWriterWorker {
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { CsvWriterWorker };
|
||||
module.exports = { CsvWriterWorker, resolveDefaultStorageDir };
|
||||
Reference in new issue
Block a user