Files
DataStack/server/index.js
T
Team Dark Devil 89e4d68690 feat(auth): harden session management and security
- Implement server-side session revocation on logout
- Add IP extraction utility and admin setup error handling
- Improve error codes and response consistency for auth endpoints
- Refine landing page redirect logic to handle transient DB outages
- Update deployment documentation and configuration defaults
2026-11-19 20:22:01 +06:00

1598 lines
49 KiB
JavaScript

const crypto = require('node:crypto');
const express = require('express');
const http = require('http');
const cors = require('cors');
const { Server } = require('socket.io');
const { Pool } = require('pg');
const Redis = require('ioredis');
const { setupSocketHandlers, normalizeClientId } = require('./socket/handlers');
const { CsvWriterWorker } = require('./workers/csvWriter');
function parseAllowedCorsOrigins(raw = process.env.DATASTACK_CORS_ORIGINS) {
if (typeof raw !== 'string') return null;
const trimmed = raw.trim();
if (!trimmed || trimmed === '*') return null;
const origins = trimmed
.split(',')
.map((entry) => entry.trim().replace(/\/+$/, ''))
.filter(Boolean);
if (origins.length === 0 || origins.includes('*')) return null;
return new Set(origins);
}
function isOriginAllowedByCors(origin, raw = process.env.DATASTACK_CORS_ORIGINS) {
if (!origin) return true;
const allowlist = parseAllowedCorsOrigins(raw);
if (!allowlist) return true;
const normalized = String(origin).trim().replace(/\/+$/, '');
return allowlist.has(normalized);
}
function corsOriginDelegate(origin, callback) {
return callback(null, isOriginAllowedByCors(origin));
}
const PORT = Number(process.env.PORT_SOCKET) || 4000;
const app = express();
app.use(cors({ origin: corsOriginDelegate }));
app.use(express.json({ limit: '10mb' }));
const REDIS_BUFFER_KEY = 'datastack:buffer:events';
const REDIS_RECENT_KEY = 'datastack:recent:events';
const REDIS_FLUSH_LOCK_KEY = 'datastack:flush:lock';
const REDIS_CLIENTS_REGISTRY_KEY = 'datastack:clients:registry';
const REDIS_CLIENTS_DELETED_KEY = 'datastack:clients:deleted';
const DEFAULT_REDIS_RECENT_LIMIT = 500;
const MAX_REDIS_RECENT_LIMIT = 5000;
const FLUSH_LOCK_TTL_MS = 30000;
const FLUSH_LOCK_WAIT_MS = 400;
const FLUSH_LOCK_RETRY_MS = 25;
const UNLOCK_LUA_SCRIPT =
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end';
const RENEW_LOCK_LUA_SCRIPT =
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("pexpire", KEYS[1], ARGV[2]) else return 0 end';
function getRedisRecentLimit() {
const envRaw = Number(process.env.DATASTACK_RECENT_LIMIT);
if (Number.isFinite(envRaw) && envRaw >= 100) {
return Math.min(MAX_REDIS_RECENT_LIMIT, Math.floor(envRaw));
}
const threshold = Number(runtimeConfig && runtimeConfig.batchFlushThreshold) || DEFAULT_REDIS_RECENT_LIMIT;
return Math.min(
MAX_REDIS_RECENT_LIMIT,
Math.max(DEFAULT_REDIS_RECENT_LIMIT, Math.floor(threshold)),
);
}
function sleepMs(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
const bufferQueue = [];
const writer = new CsvWriterWorker();
let redisClient = undefined;
let redisUrlConfigured = '';
let redisConnected = false;
let redisTestOverride = undefined;
const pendingRedisWrites = new Set();
function getRedisClient() {
if (redisTestOverride !== undefined) {
return redisTestOverride;
}
const configuredUrl = (process.env.REDIS_URL || '').trim();
if (!configuredUrl) {
if (redisClient && typeof redisClient.disconnect === 'function') {
try {
redisClient.disconnect();
} catch {
// ignore
}
}
redisClient = null;
redisUrlConfigured = '';
redisConnected = false;
return null;
}
if (redisClient && redisUrlConfigured === configuredUrl) {
return redisClient;
}
if (redisClient && typeof redisClient.disconnect === 'function') {
try {
redisClient.disconnect();
} catch {
// ignore
}
}
try {
const client = new Redis(configuredUrl, {
maxRetriesPerRequest: 2,
enableReadyCheck: true,
retryStrategy: (times) => Math.min(times * 250, 3000),
});
client.on('ready', () => {
redisConnected = true;
});
client.on('connect', () => {
redisConnected = true;
});
client.on('close', () => {
redisConnected = false;
});
client.on('end', () => {
redisConnected = false;
});
client.on('error', () => {
redisConnected = false;
});
redisClient = client;
redisUrlConfigured = configuredUrl;
redisConnected = client.status === 'ready' || client.status === 'connect';
return redisClient;
} catch {
redisClient = null;
redisUrlConfigured = configuredUrl;
redisConnected = false;
return null;
}
}
function setRedisClientForTests(client) {
redisTestOverride = client;
redisConnected = Boolean(client);
}
async function pushEventsToRedisOrFallback(events) {
if (!events || events.length === 0) return bufferQueue.length;
const redis = getRedisClient();
if (redis) {
const serialized = events.map((e) => JSON.stringify(e));
let bufferLen = null;
try {
bufferLen = await redis.rpush(REDIS_BUFFER_KEY, ...serialized);
redisConnected = true;
} catch {
redisConnected = false;
}
if (bufferLen !== null) {
// Best-effort recent list mirror: never duplicate into RAM buffer once rpush succeeded
try {
const recentLimit = getRedisRecentLimit();
await redis.lpush(REDIS_RECENT_KEY, ...serialized);
await redis.ltrim(REDIS_RECENT_KEY, 0, recentLimit - 1);
} catch {
// Non-fatal: events are already durable in REDIS_BUFFER_KEY
}
return Number(bufferLen) + bufferQueue.length;
}
}
bufferQueue.push(...events);
return bufferQueue.length;
}
const IS_PRODUCTION = process.env.NODE_ENV === 'production';
const DEV_DEFAULT_JWT_SECRET = 'datastack-jwt-default-secret-key-32-chars-long';
function resolveInitialJwtSecret() {
const configured = (process.env.DATASTACK_JWT_SECRET || '').trim();
if (IS_PRODUCTION) {
return configured.length >= 32 ? configured : '';
}
if (configured.length > 0) {
return configured.length >= 32 ? configured : '';
}
return DEV_DEFAULT_JWT_SECRET;
}
const runtimeConfig = {
jwtSecret: resolveInitialJwtSecret(),
maxEventsPerSec: 100,
maxPayloadSizeBytes: 100 * 1024,
batchFlushThreshold: 500,
batchFlushIntervalMs: 2000,
fileRotationMb: 100,
enableIdempotency: true,
autoPruneDays: 30,
};
let previousJwtSecret = '';
let previousJwtSecretGraceUntil = 0;
const JWT_ROTATION_BRIDGE_GRACE_MS = 60 * 1000;
function resolveInternalSecret(jwtSecretOverride) {
const explicit = (process.env.DATASTACK_INTERNAL_SECRET || '').trim();
if (IS_PRODUCTION) {
if (explicit.length >= 32) return explicit;
if (explicit.length > 0) return '';
} else if (explicit.length > 0) {
return explicit;
}
const activeJwt = String(
jwtSecretOverride !== undefined ? jwtSecretOverride : runtimeConfig.jwtSecret || '',
).trim();
const minJwtLen = IS_PRODUCTION ? 32 : 1;
if (activeJwt.length < minJwtLen) {
return '';
}
return crypto
.createHash('sha256')
.update(`datastack-internal:${activeJwt}`)
.digest('hex');
}
function timingSafeSecretEqual(candidate, expected) {
if (!candidate || !expected) return false;
const a = crypto.createHash('sha256').update(String(candidate)).digest();
const b = crypto.createHash('sha256').update(String(expected)).digest();
return crypto.timingSafeEqual(a, b);
}
function requireInternalAuth(req, res, next) {
const expected = resolveInternalSecret();
if (!expected) {
return res.status(503).json({
success: false,
error: 'INTERNAL_AUTH_UNAVAILABLE: Internal bridge secret is not configured',
code: 'INTERNAL_AUTH_UNAVAILABLE',
});
}
const presented = req.get('x-datastack-internal-secret');
const matchesCurrent = Boolean(presented && timingSafeSecretEqual(presented, expected));
let matchesGrace = false;
if (!matchesCurrent && presented && previousJwtSecret && Date.now() <= previousJwtSecretGraceUntil) {
const graceExpected = resolveInternalSecret(previousJwtSecret);
matchesGrace = Boolean(graceExpected && timingSafeSecretEqual(presented, graceExpected));
}
if (!matchesCurrent && !matchesGrace) {
return res.status(401).json({
success: false,
error: 'UNAUTHORIZED: Missing or invalid internal authentication header',
code: 'UNAUTHORIZED',
});
}
return next();
}
let dbPool = null;
if (process.env.DATABASE_URL) {
try {
dbPool = new Pool({
connectionString: process.env.DATABASE_URL,
connectionTimeoutMillis: 3000,
idleTimeoutMillis: 30000,
max: 5,
});
dbPool.on('error', (err) => {
console.error(
'[DataStack Ingestion DB] Unexpected idle PostgreSQL pool error:',
err instanceof Error ? err.message : err,
);
});
} catch {
dbPool = null;
}
}
// Short-lived cache (2s TTL) for DB client status & token checks on high-frequency events
const clientAuthCache = new Map();
const deletedClientsSet = new Set();
const inMemoryClientRegistry = new Map();
const inMemoryClientActivity = new Map();
let hasSyncedInMemoryClients = false;
const CLIENT_CACHE_TTL_MS = 2000;
async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
const normalizedId = normalizeClientId(clientId);
if (!normalizedId) {
return {
allowed: false,
reason: 'CLIENT_REVOKED: Missing client identifier',
};
}
if (deletedClientsSet.has(normalizedId)) {
return {
allowed: false,
reason: 'CLIENT_REVOKED: Client access has been deleted or revoked by admin',
};
}
if (!dbPool) {
let memClient = inMemoryClientRegistry.get(normalizedId);
if (!memClient) {
const redis = getRedisClient();
if (redis && typeof redis.sismember === 'function' && typeof redis.hget === 'function') {
try {
const isDeleted = await redis.sismember(REDIS_CLIENTS_DELETED_KEY, normalizedId);
if (isDeleted) {
deletedClientsSet.add(normalizedId);
return {
allowed: false,
reason: 'CLIENT_REVOKED: Client access has been deleted by admin',
};
}
const rawClient = await redis.hget(REDIS_CLIENTS_REGISTRY_KEY, normalizedId);
if (rawClient) {
const parsed = JSON.parse(rawClient);
if (parsed && parsed.clientId) {
memClient = {
clientId: normalizedId,
name: String(parsed.name || normalizedId),
status: parsed.status === 'revoked' ? 'revoked' : 'active',
token: String(parsed.token || ''),
rateLimit: Math.max(1, Number(parsed.rateLimit) || 100),
};
inMemoryClientRegistry.set(normalizedId, memClient);
hasSyncedInMemoryClients = true;
}
}
} catch {
// Ignore Redis registry lookup error and check local state
}
}
}
if (memClient) {
if (memClient.status === 'revoked') {
return {
allowed: false,
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
};
}
if (memClient.token && presentedToken && memClient.token !== presentedToken) {
return {
allowed: false,
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
};
}
return { allowed: true, rateLimit: memClient.rateLimit || 100 };
}
return {
allowed: false,
reason: 'CLIENT_REVOKED: Client registration not found or has been deleted',
};
}
const now = Date.now();
const prevCached = clientAuthCache.get(normalizedId);
if (!bypassCache && prevCached && now - prevCached.checkedAt < CLIENT_CACHE_TTL_MS) {
if (prevCached.status === 'revoked') {
return {
allowed: false,
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
};
}
if (prevCached.token && presentedToken && prevCached.token !== presentedToken) {
return {
allowed: false,
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
};
}
return { allowed: true, rateLimit: prevCached.rateLimit };
}
try {
const deletedRes = await dbPool.query(
`SELECT 1 FROM deleted_clients WHERE client_id = $1`,
[normalizedId],
);
if (deletedRes.rowCount && deletedRes.rowCount > 0) {
clientAuthCache.delete(normalizedId);
deletedClientsSet.add(normalizedId);
return {
allowed: false,
reason: 'CLIENT_REVOKED: Client access has been deleted by admin',
};
}
const res = await dbPool.query(
`SELECT status, token, rate_limit FROM client_sdks WHERE client_id = $1`,
[normalizedId],
);
if (!res.rowCount || res.rowCount === 0) {
clientAuthCache.delete(normalizedId);
return {
allowed: false,
reason: 'CLIENT_REVOKED: Client registration not found or has been deleted',
};
}
const row = res.rows[0];
const entry = {
status: row.status,
token: row.token,
rateLimit: Number(row.rate_limit) || 100,
checkedAt: now,
};
clientAuthCache.set(normalizedId, entry);
if (entry.status === 'revoked') {
return {
allowed: false,
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
};
}
if (entry.token && presentedToken && entry.token !== presentedToken) {
return {
allowed: false,
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
};
}
return { allowed: true, rateLimit: entry.rateLimit };
} catch {
return {
allowed: false,
reason: 'AUTH_UNAVAILABLE: Unable to verify client credentials against database',
};
}
}
const pendingDbClientActivity = new Map();
const CLIENT_ACTIVITY_FLUSH_INTERVAL_MS = 1000;
let activityFlushTimer = null;
let activeActivityFlushPromise = null;
let schemaEnsurePromise = null;
let schemaEnsurePool = null;
function setDbPoolForTests(pool) {
dbPool = pool;
schemaEnsurePromise = null;
schemaEnsurePool = null;
}
async function flushPendingClientActivity() {
if (activeActivityFlushPromise) {
await activeActivityFlushPromise;
if (pendingDbClientActivity.size === 0) return 0;
}
if (!dbPool || pendingDbClientActivity.size === 0) return 0;
const snapshot = Array.from(pendingDbClientActivity.entries());
pendingDbClientActivity.clear();
const clientIds = [];
const deltas = [];
const lastActives = [];
for (const [cid, entry] of snapshot) {
clientIds.push(cid);
deltas.push(entry.count);
lastActives.push(entry.lastActive);
}
const run = (async () => {
try {
await dbPool.query(
`UPDATE client_sdks AS c
SET total_events = c.total_events + v.delta,
last_active = GREATEST(COALESCE(c.last_active, v.last_active), v.last_active),
updated_at = NOW()
FROM (
SELECT UNNEST($1::text[]) AS client_id,
UNNEST($2::bigint[]) AS delta,
UNNEST($3::timestamptz[]) AS last_active
) AS v
WHERE c.client_id = v.client_id`,
[clientIds, deltas, lastActives],
);
return clientIds.length;
} catch {
// Re-merge deltas so transient DB failures do not lose event counts
for (const [cid, entry] of snapshot) {
const current = pendingDbClientActivity.get(cid);
if (current) {
pendingDbClientActivity.set(cid, {
count: current.count + entry.count,
lastActive:
current.lastActive > entry.lastActive
? current.lastActive
: entry.lastActive,
});
} else {
pendingDbClientActivity.set(cid, { ...entry });
}
}
return 0;
} finally {
activeActivityFlushPromise = null;
}
})();
activeActivityFlushPromise = run;
return run;
}
function ensureActivityFlushTimer() {
if (activityFlushTimer) return;
activityFlushTimer = setInterval(() => {
void flushPendingClientActivity().catch(() => undefined);
}, CLIENT_ACTIVITY_FLUSH_INTERVAL_MS);
if (typeof activityFlushTimer.unref === 'function') {
activityFlushTimer.unref();
}
}
function recordClientActivity(clientId, count = 1) {
if (!clientId || count <= 0) return;
const normalizedId = normalizeClientId(clientId);
if (!normalizedId) return;
const nowIso = new Date().toISOString();
if (!dbPool) {
const prev = inMemoryClientActivity.get(normalizedId) || {
totalEvents: 0,
lastActive: nowIso,
};
inMemoryClientActivity.set(normalizedId, {
totalEvents: prev.totalEvents + count,
lastActive: nowIso,
});
return;
}
const prevPending = pendingDbClientActivity.get(normalizedId);
if (prevPending) {
pendingDbClientActivity.set(normalizedId, {
count: prevPending.count + count,
lastActive: nowIso > prevPending.lastActive ? nowIso : prevPending.lastActive,
});
} else {
pendingDbClientActivity.set(normalizedId, {
count,
lastActive: nowIso,
});
}
ensureActivityFlushTimer();
}
let isFlushing = false;
let activeFlushPromise = null;
let flushTimer = null;
function purgeClientFromLocalBuffer(normalizedId) {
if (!normalizedId || bufferQueue.length === 0) return 0;
let removed = 0;
let writeIdx = 0;
for (let i = 0; i < bufferQueue.length; i += 1) {
const evt = bufferQueue[i];
if (evt && normalizeClientId(evt.clientId) === normalizedId) {
removed += 1;
} else {
bufferQueue[writeIdx] = evt;
writeIdx += 1;
}
}
bufferQueue.length = writeIdx;
return removed;
}
async function isClientDeletedForFlush(clientId, redis, cache) {
const normalized = normalizeClientId(clientId);
if (!normalized) return false;
if (deletedClientsSet.has(normalized)) return true;
if (cache && cache.has(normalized)) return Boolean(cache.get(normalized));
if (redis && typeof redis.sismember === 'function') {
try {
const isDeleted =
Number(await redis.sismember(REDIS_CLIENTS_DELETED_KEY, normalized)) === 1;
if (cache) cache.set(normalized, isDeleted);
if (isDeleted) deletedClientsSet.add(normalized);
return isDeleted;
} catch {
// Ignore transient Redis tombstone lookup error
}
}
if (cache) cache.set(normalized, false);
return false;
}
async function flushBuffer(options = {}) {
if (pendingRedisWrites.size > 0) {
await Promise.allSettled([...pendingRedisWrites]);
}
if (activeFlushPromise) {
return activeFlushPromise;
}
const executeFlush = async () => {
isFlushing = true;
let totalWritten = 0;
let lockTimedOut = false;
const filesSet = new Set();
const deletedCheckCache = new Map();
try {
if (pendingRedisWrites.size > 0) {
await Promise.allSettled([...pendingRedisWrites]);
}
const redis = getRedisClient();
if (redis) {
const lockToken = crypto.randomUUID();
const lockTtlMs = Math.max(50, Number(options && options.lockTtlMs) || FLUSH_LOCK_TTL_MS);
const waitMs = Math.max(
0,
options && options.waitMs !== undefined ? Number(options.waitMs) : FLUSH_LOCK_WAIT_MS,
);
const deadline = Date.now() + waitMs;
let lockAcquired = false;
let redisError = false;
while (true) {
try {
const lockResult = await redis.set(
REDIS_FLUSH_LOCK_KEY,
lockToken,
'PX',
lockTtlMs,
'NX',
);
redisConnected = true;
if (lockResult === 'OK') {
lockAcquired = true;
break;
}
} catch {
redisConnected = false;
redisError = true;
break;
}
const remaining = deadline - Date.now();
if (remaining <= 0) {
break;
}
await sleepMs(Math.min(FLUSH_LOCK_RETRY_MS, remaining));
}
if (!lockAcquired && !redisError) {
lockTimedOut = true;
if (options && options.throwOnLockTimeout === true) {
throw new Error(
'FLUSH_LOCK_TIMEOUT: Timed out waiting for distributed Redis flush lock',
);
}
}
if (lockAcquired) {
const renewIntervalMs = Math.max(20, Math.floor(lockTtlMs / 3));
let lastRenewAt = Date.now();
let lockLost = false;
let renewInFlight = null;
const renewLockOnce = () => {
if (renewInFlight) return renewInFlight;
renewInFlight = (async () => {
try {
const renewed = await redis.eval(
RENEW_LOCK_LUA_SCRIPT,
1,
REDIS_FLUSH_LOCK_KEY,
lockToken,
String(lockTtlMs),
);
if (Number(renewed) === 1) {
lastRenewAt = Date.now();
return true;
}
lockLost = true;
return false;
} catch {
return false;
} finally {
renewInFlight = null;
}
})();
return renewInFlight;
};
const renewTimer = setInterval(() => {
void renewLockOnce();
}, renewIntervalMs);
if (typeof renewTimer.unref === 'function') {
renewTimer.unref();
}
try {
const chunkSize = Math.max(500, runtimeConfig.batchFlushThreshold);
while (true) {
if (lockLost) break;
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, chunkSize - 1);
if (!rawItems || rawItems.length === 0) {
break;
}
const parsedBatch = [];
let malformedCount = 0;
for (const raw of rawItems) {
try {
const evt = JSON.parse(raw);
if (evt && evt.eventId) {
if (await isClientDeletedForFlush(evt.clientId, redis, deletedCheckCache)) {
continue;
}
parsedBatch.push(evt);
} else {
malformedCount += 1;
}
} catch {
malformedCount += 1;
}
}
if (malformedCount > 0) {
console.warn(
`[CSV Worker] Skipped ${malformedCount} malformed buffered Redis event(s) during flush`,
);
}
if (Date.now() - lastRenewAt >= renewIntervalMs) {
await renewLockOnce();
}
if (lockLost) break;
if (parsedBatch.length > 0) {
const res = await writer.writeBatch(parsedBatch);
totalWritten += res.written;
for (const f of res.files) filesSet.add(f);
}
if (Date.now() - lastRenewAt >= renewIntervalMs) {
await renewLockOnce();
}
if (lockLost) break;
// Trim ONLY after CSV batch write succeeds and lock ownership holds
await redis.ltrim(REDIS_BUFFER_KEY, rawItems.length, -1);
if (rawItems.length < chunkSize) {
break;
}
}
} finally {
clearInterval(renewTimer);
if (renewInFlight) {
await renewInFlight.catch(() => false);
}
try {
await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
} catch {
// Lock expires automatically via PX TTL
}
}
}
}
if (bufferQueue.length > 0) {
const batch = bufferQueue.splice(0, bufferQueue.length);
const validBatch = [];
for (const evt of batch) {
if (await isClientDeletedForFlush(evt.clientId, redis, deletedCheckCache)) {
continue;
}
validBatch.push(evt);
}
if (validBatch.length > 0) {
try {
const res = await writer.writeBatch(validBatch);
totalWritten += res.written;
for (const f of res.files) filesSet.add(f);
} catch (err) {
bufferQueue.unshift(...validBatch);
throw err;
}
}
}
const files = Array.from(filesSet);
if (totalWritten > 0) {
console.log(
`[CSV Worker] Flushed ${totalWritten} events to ${files.length} file(s)`,
);
}
return {
written: totalWritten,
files,
...(lockTimedOut ? { lockTimedOut: true } : {}),
};
} catch (err) {
console.error('[CSV Worker] Flush error:', err);
throw err;
} finally {
isFlushing = false;
activeFlushPromise = null;
}
};
let flushDone = false;
const promise = (async () => {
try {
return await executeFlush();
} finally {
flushDone = true;
isFlushing = false;
if (activeFlushPromise === promise) {
activeFlushPromise = null;
}
}
})();
if (!flushDone) {
activeFlushPromise = promise;
}
return promise;
}
function scheduleFlushTimer(intervalMs) {
const safeInterval = Math.max(100, Number(intervalMs) || 2000);
runtimeConfig.batchFlushIntervalMs = safeInterval;
if (flushTimer) {
clearInterval(flushTimer);
}
flushTimer = setInterval(() => {
void flushBuffer().catch(() => undefined);
}, safeInterval);
}
scheduleFlushTimer(runtimeConfig.batchFlushIntervalMs);
const server = http.createServer(app);
const io = new Server(server, {
cors: { origin: corsOriginDelegate, methods: ['GET', 'POST'] },
});
function applyRuntimeSettingsObject(settingsObj) {
if (!settingsObj || typeof settingsObj !== 'object') {
return { jwtRotated: false };
}
if (settingsObj.maxEventsPerSec !== undefined) {
const val = Number(settingsObj.maxEventsPerSec);
if (Number.isFinite(val) && val >= 1) {
runtimeConfig.maxEventsPerSec = Math.floor(val);
}
}
if (settingsObj.maxPayloadSizeKb !== undefined) {
const kb = Number(settingsObj.maxPayloadSizeKb);
if (Number.isFinite(kb) && kb >= 1) {
runtimeConfig.maxPayloadSizeBytes = Math.floor(kb) * 1024;
}
}
if (settingsObj.batchFlushThreshold !== undefined) {
const threshold = Number(settingsObj.batchFlushThreshold);
if (Number.isFinite(threshold) && threshold >= 1) {
runtimeConfig.batchFlushThreshold = Math.floor(threshold);
}
}
if (settingsObj.enableIdempotency !== undefined) {
runtimeConfig.enableIdempotency = Boolean(settingsObj.enableIdempotency);
}
if (settingsObj.fileRotationMb !== undefined) {
const nextRotationMb = Number(settingsObj.fileRotationMb);
if (
Number.isFinite(nextRotationMb) &&
nextRotationMb >= 1 &&
nextRotationMb !== runtimeConfig.fileRotationMb
) {
runtimeConfig.fileRotationMb = nextRotationMb;
writer.setMaxFileSizeMb(nextRotationMb);
}
}
if (settingsObj.autoPruneDays !== undefined && settingsObj.autoPruneDays !== null) {
const parsedPrune = Number(settingsObj.autoPruneDays);
if (Number.isFinite(parsedPrune) && parsedPrune >= 0) {
const nextPruneDays = Math.floor(parsedPrune);
const changed = nextPruneDays !== runtimeConfig.autoPruneDays;
runtimeConfig.autoPruneDays = nextPruneDays;
if (changed && nextPruneDays > 0) {
void writer.pruneExpiredPartitionsAsync(nextPruneDays).catch(() => undefined);
}
}
}
if (settingsObj.batchFlushIntervalMs !== undefined) {
const nextIntervalMs = Math.max(
100,
Number(settingsObj.batchFlushIntervalMs) || 2000,
);
if (nextIntervalMs !== runtimeConfig.batchFlushIntervalMs) {
scheduleFlushTimer(nextIntervalMs);
}
}
let jwtRotated = false;
if (settingsObj.jwtSecret !== undefined && settingsObj.jwtSecret !== null) {
const candidateSecret = String(settingsObj.jwtSecret).trim();
const validSecret =
candidateSecret.length >= 32 ||
(!IS_PRODUCTION && candidateSecret.length > 0);
if (validSecret && candidateSecret !== runtimeConfig.jwtSecret) {
if (runtimeConfig.jwtSecret) {
previousJwtSecret = runtimeConfig.jwtSecret;
previousJwtSecretGraceUntil = Date.now() + JWT_ROTATION_BRIDGE_GRACE_MS;
}
runtimeConfig.jwtSecret = candidateSecret;
clientAuthCache.clear();
io.disconnectSockets(true);
jwtRotated = true;
}
}
return { jwtRotated };
}
async function ensureDatabaseSchema() {
const pool = dbPool;
if (!pool) return false;
if (schemaEnsurePromise && schemaEnsurePool === pool) {
return schemaEnsurePromise;
}
schemaEnsurePool = pool;
schemaEnsurePromise = (async () => {
try {
await pool.query(`
CREATE TABLE IF NOT EXISTS client_sdks (
client_id TEXT PRIMARY KEY,
name TEXT NOT NULL,
secret TEXT NOT NULL,
token TEXT NOT NULL,
rate_limit INTEGER NOT NULL DEFAULT 100,
status TEXT NOT NULL DEFAULT 'active',
total_events BIGINT NOT NULL DEFAULT 0,
last_active TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`);
await pool.query(`
CREATE TABLE IF NOT EXISTS admin_settings (
id TEXT PRIMARY KEY DEFAULT 'default',
server_port INTEGER NOT NULL DEFAULT 4000,
web_port INTEGER NOT NULL DEFAULT 3000,
max_events_per_sec INTEGER NOT NULL DEFAULT 100,
max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
file_rotation_mb INTEGER NOT NULL DEFAULT 100,
jwt_secret TEXT NOT NULL,
enable_idempotency BOOLEAN NOT NULL DEFAULT true,
auto_prune_days INTEGER NOT NULL DEFAULT 30,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`);
await pool.query(`
CREATE TABLE IF NOT EXISTS deleted_clients (
client_id TEXT PRIMARY KEY,
deleted_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`);
await pool.query(`
CREATE TABLE IF NOT EXISTS admin_users (
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
email TEXT NOT NULL UNIQUE,
password_salt BYTEA NOT NULL,
password_hash BYTEA NOT NULL,
token_version INTEGER NOT NULL DEFAULT 1,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`);
await pool.query(`
CREATE TABLE IF NOT EXISTS admin_login_attempts (
key_hash TEXT PRIMARY KEY,
window_started_at TIMESTAMPTZ NOT NULL,
failed_attempts INTEGER NOT NULL,
blocked_until TIMESTAMPTZ,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`);
await pool.query(`
ALTER TABLE client_sdks
ADD COLUMN IF NOT EXISTS rate_limit INTEGER NOT NULL DEFAULT 100,
ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'active',
ADD COLUMN IF NOT EXISTS total_events BIGINT NOT NULL DEFAULT 0,
ADD COLUMN IF NOT EXISTS last_active TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
`);
await pool.query(`
ALTER TABLE admin_settings
ADD COLUMN IF NOT EXISTS server_port INTEGER NOT NULL DEFAULT 4000,
ADD COLUMN IF NOT EXISTS web_port INTEGER NOT NULL DEFAULT 3000,
ADD COLUMN IF NOT EXISTS max_events_per_sec INTEGER NOT NULL DEFAULT 100,
ADD COLUMN IF NOT EXISTS max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
ADD COLUMN IF NOT EXISTS batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
ADD COLUMN IF NOT EXISTS batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
ADD COLUMN IF NOT EXISTS file_rotation_mb INTEGER NOT NULL DEFAULT 100,
ADD COLUMN IF NOT EXISTS enable_idempotency BOOLEAN NOT NULL DEFAULT true,
ADD COLUMN IF NOT EXISTS auto_prune_days INTEGER NOT NULL DEFAULT 30,
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
`);
await pool.query(`
ALTER TABLE admin_users
ADD COLUMN IF NOT EXISTS token_version INTEGER NOT NULL DEFAULT 1
`);
return true;
} catch {
if (schemaEnsurePool === pool) {
schemaEnsurePromise = null;
}
return false;
}
})();
return schemaEnsurePromise;
}
async function syncSettingsFromDatabase() {
if (!dbPool) return false;
try {
const schemaReady = await ensureDatabaseSchema();
if (!schemaReady) return false;
const res = await dbPool.query(
`SELECT max_events_per_sec, max_payload_size_kb, batch_flush_threshold,
batch_flush_interval_ms, file_rotation_mb, jwt_secret, enable_idempotency,
auto_prune_days
FROM admin_settings
WHERE id = 'default'`,
);
if (!res.rowCount || res.rowCount === 0) return true;
const row = res.rows[0];
const parsedPrune =
row.auto_prune_days !== undefined && row.auto_prune_days !== null
? Number(row.auto_prune_days)
: NaN;
applyRuntimeSettingsObject({
maxEventsPerSec: Number(row.max_events_per_sec) || 100,
maxPayloadSizeKb: Number(row.max_payload_size_kb) || 100,
batchFlushThreshold: Math.max(1, Number(row.batch_flush_threshold) || 500),
batchFlushIntervalMs: Math.max(
100,
Number(row.batch_flush_interval_ms) || 2000,
),
fileRotationMb: Number(row.file_rotation_mb) || 100,
enableIdempotency: Boolean(row.enable_idempotency),
autoPruneDays: Number.isFinite(parsedPrune) ? parsedPrune : 30,
jwtSecret: row.jwt_secret ? String(row.jwt_secret).trim() : undefined,
});
return true;
} catch {
return false;
}
}
const PRUNE_INTERVAL_MS = 60 * 60 * 1000;
let settingsSyncTimer = null;
let pruneTimer = null;
const socketHandlersApi = setupSocketHandlers(io, bufferQueue, writer, {
getRuntimeConfig: () => runtimeConfig,
getRedisClient,
verifyClientInDb,
recordClientActivity,
onBufferPush: async () => {
const redis = getRedisClient();
if (redis) {
if (bufferQueue.length > 0) {
const pushed = bufferQueue.splice(0, bufferQueue.length);
const op = pushEventsToRedisOrFallback(pushed);
pendingRedisWrites.add(op);
try {
const totalQueued = await op;
if (totalQueued >= runtimeConfig.batchFlushThreshold) {
void flushBuffer().catch(() => undefined);
}
} finally {
pendingRedisWrites.delete(op);
}
return;
}
if (pendingRedisWrites.size > 0) {
await Promise.all([...pendingRedisWrites]);
}
return;
}
if (bufferQueue.length >= runtimeConfig.batchFlushThreshold) {
void flushBuffer().catch(() => undefined);
}
},
});
async function getRedisBufferLen() {
const redis = getRedisClient();
if (!redis) return 0;
try {
const len = Number(await redis.llen(REDIS_BUFFER_KEY)) || 0;
redisConnected = true;
return len;
} catch {
redisConnected = false;
return 0;
}
}
// Public health check endpoint
app.get('/health', async (_req, res) => {
const redisBufferSize = await getRedisBufferLen();
res.json({
status: 'online',
service: 'DataStack Express Ingestion',
port: PORT,
serverPort: PORT,
webPort: Number(process.env.PORT) || 3000,
activeSockets: io.engine ? io.engine.clientsCount : 0,
bufferSize: bufferQueue.length + redisBufferSize,
localBufferSize: bufferQueue.length,
redisBufferSize,
redisConnected,
maxEventsPerSec: runtimeConfig.maxEventsPerSec,
maxPayloadSizeBytes: runtimeConfig.maxPayloadSizeBytes,
uptime: process.uptime(),
});
});
// Require internal shared authentication on all /internal/* bridge endpoints
app.use('/internal', requireInternalAuth);
// Internal metrics endpoint for Next.js dashboard & in-memory client activity aggregation
app.get('/internal/metrics', async (_req, res) => {
const redisBufferSize = await getRedisBufferLen();
res.json({
port: PORT,
serverPort: PORT,
webPort: Number(process.env.PORT) || 3000,
activeSockets: io.engine ? io.engine.clientsCount : 0,
bufferSize: bufferQueue.length + redisBufferSize,
localBufferSize: bufferQueue.length,
redisBufferSize,
redisConnected,
registrySize: inMemoryClientRegistry.size,
hasSyncedInMemoryClients,
inMemoryClients: Array.from(inMemoryClientRegistry.values()),
securityStats: socketHandlersApi.getSecurityStats(),
clientActivity: Object.fromEntries(inMemoryClientActivity.entries()),
});
});
// Internal shared security check for Port 3000 HTTP ingestion when Redis is unavailable (BUG-03)
app.post('/internal/security-check', (req, res) => {
const body = req.body && typeof req.body === 'object' ? req.body : {};
const clientId = normalizeClientId(body.clientId || '');
if (!clientId) {
return res.status(400).json({
allowed: false,
code: 'MALFORMED_PAYLOAD',
reason: 'Missing clientId',
});
}
const eventId = typeof body.eventId === 'string' ? body.eventId : '';
const clientRateLimit = Number(body.clientMaxRate) || 100;
const payloadBytes =
typeof body.payloadBytes === 'number' &&
Number.isFinite(body.payloadBytes) &&
body.payloadBytes >= 0
? body.payloadBytes
: undefined;
const result = socketHandlersApi.validateSingleEventLocal(
clientId,
eventId,
body.payload ?? {},
clientRateLimit,
payloadBytes,
);
return res.json({
allowed: Boolean(result.allowed),
duplicate: Boolean(result.duplicate),
...(result.code ? { code: result.code } : {}),
...(result.reason ? { reason: result.reason } : {}),
});
});
// Internal instant settings synchronization triggered by Next.js POST /api/settings
app.post('/internal/settings-sync', (req, res) => {
const result = applyRuntimeSettingsObject(req.body || {});
res.json({
success: true,
jwtRotated: result.jwtRotated,
});
});
// Internal flush endpoint triggered by Next.js POST /api/storage & GET /api/export
app.post('/internal/flush', async (req, res) => {
try {
const throwOnLockTimeout = Boolean(req.body && req.body.throwOnLockTimeout === true);
const waitMs =
req.body && Number.isFinite(Number(req.body.waitMs))
? Math.max(0, Number(req.body.waitMs))
: 2500;
const result = await flushBuffer({ waitMs, throwOnLockTimeout });
res.json({
success: true,
flushedCount: result.written,
filesUpdated: result.files,
...(result.lockTimedOut ? { lockTimedOut: true } : {}),
});
} catch (err) {
const message = err instanceof Error ? err.message : 'Flush failed';
if (message.includes('FLUSH_LOCK_TIMEOUT')) {
return res.status(503).json({
success: false,
code: 'FLUSH_LOCK_TIMEOUT',
error: message,
});
}
res.status(500).json({
success: false,
error: message,
});
}
});
// Internal endpoint to expose Port 4000 buffered events before CSV flush
app.get('/internal/recent', async (req, res) => {
const rawClientId = req.query.clientId ? String(req.query.clientId) : '';
const filterClientId =
rawClientId && rawClientId !== 'all' ? normalizeClientId(rawClientId) : '';
const recentLimit = getRedisRecentLimit();
const limit = Math.min(recentLimit, Math.max(1, Number(req.query.limit) || 50));
const redis = getRedisClient();
let redisEvents = [];
if (redis) {
try {
const readCap = Math.min(recentLimit, Math.max(limit * 2, limit));
const rawItems = await redis.lrange(REDIS_RECENT_KEY, 0, readCap - 1);
redisConnected = true;
for (const raw of rawItems) {
try {
const evt = JSON.parse(raw);
if (!evt || !evt.eventId) continue;
const normEvtClientId = normalizeClientId(evt.clientId);
if (deletedClientsSet.has(normEvtClientId)) continue;
if (filterClientId && normEvtClientId !== filterClientId) continue;
redisEvents.push(evt);
if (redisEvents.length >= limit) break;
} catch {
// ignore malformed item
}
}
} catch {
redisConnected = false;
redisEvents = [];
}
}
const localFiltered = filterClientId
? bufferQueue.filter((evt) => normalizeClientId(evt.clientId) === filterClientId)
: bufferQueue;
const localRecent = localFiltered.slice(-limit).reverse();
const merged = [];
const seen = new Set();
for (const evt of [...localRecent, ...redisEvents]) {
if (!evt || !evt.eventId) continue;
const key = `${evt.clientId}:${evt.eventId}`;
if (seen.has(key)) continue;
seen.add(key);
merged.push(evt);
if (merged.length >= limit) break;
}
res.json({ events: merged, count: merged.length });
});
// Internal endpoint triggered by Next.js SDKManager when a client is deleted, revoked, rotated, or synced
app.post('/internal/client-sync', (req, res) => {
const action = req.body && req.body.action ? String(req.body.action) : '';
if (action === 'bulk_sync' && Array.isArray(req.body.clients)) {
const forceReplace = req.body.forceReplace === true;
// BUG-02 Guard: Never wipe a populated Port 4000 in-memory registry when a cold-restarted
// Port 3000 (without DB) sends an empty bulk_sync without explicit forceReplace.
if (
req.body.clients.length === 0 &&
!forceReplace &&
inMemoryClientRegistry.size > 0
) {
hasSyncedInMemoryClients = true;
return res.json({
success: true,
count: inMemoryClientRegistry.size,
preserved: true,
clients: Array.from(inMemoryClientRegistry.values()),
});
}
hasSyncedInMemoryClients = true;
const prevRegistry = new Map(inMemoryClientRegistry);
inMemoryClientRegistry.clear();
clientAuthCache.clear();
const activeIds = new Set();
for (const item of req.body.clients) {
if (!item || !item.clientId) continue;
const cid = normalizeClientId(item.clientId);
if (!cid) continue;
const nextStatus = item.status === 'revoked' ? 'revoked' : 'active';
const nextToken = String(item.token || '');
const prev = prevRegistry.get(cid);
activeIds.add(cid);
deletedClientsSet.delete(cid);
inMemoryClientRegistry.set(cid, {
clientId: cid,
name: String(item.name || cid),
status: nextStatus,
token: nextToken,
rateLimit: Math.max(1, Number(item.rateLimit) || 100),
});
if (nextStatus === 'revoked' || (prev && nextToken && prev.token !== nextToken)) {
io.to(`client:${cid}`).disconnectSockets(true);
}
}
if (forceReplace) {
for (const prevCid of prevRegistry.keys()) {
if (!activeIds.has(prevCid)) {
io.to(`client:${prevCid}`).disconnectSockets(true);
}
}
}
return res.json({ success: true, count: activeIds.size, preserved: false });
}
const rawClientId = req.body && req.body.clientId ? String(req.body.clientId) : '';
const normalizedId = normalizeClientId(rawClientId);
const clientPayload =
req.body && req.body.client && typeof req.body.client === 'object'
? req.body.client
: null;
if (normalizedId) {
hasSyncedInMemoryClients = true;
clientAuthCache.delete(normalizedId);
if (action === 'deleted') {
deletedClientsSet.add(normalizedId);
inMemoryClientRegistry.delete(normalizedId);
inMemoryClientActivity.delete(normalizedId);
pendingDbClientActivity.delete(normalizedId);
purgeClientFromLocalBuffer(normalizedId);
io.to(`client:${normalizedId}`).disconnectSockets(true);
} else if (action === 'revoked') {
const existingMem = inMemoryClientRegistry.get(normalizedId);
if (clientPayload) {
inMemoryClientRegistry.set(normalizedId, {
clientId: normalizedId,
name: String(clientPayload.name || normalizedId),
status: 'revoked',
token: String(clientPayload.token || ''),
rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
});
} else if (existingMem) {
existingMem.status = 'revoked';
} else {
inMemoryClientRegistry.set(normalizedId, {
clientId: normalizedId,
name: normalizedId,
status: 'revoked',
token: '',
rateLimit: 100,
});
}
io.to(`client:${normalizedId}`).disconnectSockets(true);
} else if (action === 'registered') {
deletedClientsSet.delete(normalizedId);
if (clientPayload) {
inMemoryClientRegistry.set(normalizedId, {
clientId: normalizedId,
name: String(clientPayload.name || normalizedId),
status: clientPayload.status === 'revoked' ? 'revoked' : 'active',
token: String(clientPayload.token || ''),
rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
});
}
} else if (action === 'updated') {
if (clientPayload) {
inMemoryClientRegistry.set(normalizedId, {
clientId: normalizedId,
name: String(clientPayload.name || normalizedId),
status: clientPayload.status === 'revoked' ? 'revoked' : 'active',
token: String(clientPayload.token || ''),
rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
});
}
io.to(`client:${normalizedId}`).disconnectSockets(true);
}
}
res.json({ success: true });
});
function stopTimers() {
if (flushTimer) {
clearInterval(flushTimer);
flushTimer = null;
}
if (settingsSyncTimer) {
clearInterval(settingsSyncTimer);
settingsSyncTimer = null;
}
if (pruneTimer) {
clearInterval(pruneTimer);
pruneTimer = null;
}
if (activityFlushTimer) {
clearInterval(activityFlushTimer);
activityFlushTimer = null;
}
}
let isShuttingDown = false;
async function closeServerGracefully() {
try {
await Promise.race([
new Promise((resolve) => {
if (io && typeof io.close === 'function') {
io.close(() => resolve());
} else if (server && server.listening) {
server.close(() => resolve());
} else {
resolve();
}
}),
sleepMs(2000),
]);
if (server && typeof server.closeIdleConnections === 'function') {
server.closeIdleConnections();
}
} catch {
// Ignore close errors
}
}
async function gracefulShutdown(signal, options = {}) {
if (isShuttingDown) return;
isShuttingDown = true;
const shouldExit = options.exit !== false;
console.log(
`[DataStack Ingestion Engine] Received ${signal}, closing ingress and flushing buffer before exit...`,
);
try {
stopTimers();
// Close Socket.IO & HTTP listeners before final flush so late traffic cannot enter after drain starts
await closeServerGracefully();
try {
if (pendingRedisWrites.size > 0) {
await Promise.race([
Promise.allSettled([...pendingRedisWrites]),
sleepMs(2000),
]);
}
await Promise.race([
Promise.allSettled([flushBuffer(), flushPendingClientActivity()]),
sleepMs(6000),
]);
} catch {
// Best-effort flush on exit
}
try {
const activeRedis =
redisTestOverride !== undefined ? redisTestOverride : redisClient;
if (activeRedis && typeof activeRedis.quit === 'function') {
let quitFinished = false;
await Promise.race([
activeRedis
.quit()
.then(() => {
quitFinished = true;
})
.catch(() => {
quitFinished = true;
}),
sleepMs(2000),
]);
if (!quitFinished && typeof activeRedis.disconnect === 'function') {
activeRedis.disconnect();
}
}
if (dbPool && typeof dbPool.end === 'function') {
await Promise.race([dbPool.end().catch(() => undefined), sleepMs(2000)]);
}
} catch {
// Ignore close errors
}
if (shouldExit) {
process.exit(0);
}
} finally {
if (!shouldExit) {
isShuttingDown = false;
}
}
}
async function startIngestionServer() {
if (dbPool) {
await Promise.race([
syncSettingsFromDatabase().catch(() => false),
sleepMs(5000),
]);
settingsSyncTimer = setInterval(() => {
void syncSettingsFromDatabase();
}, 3000);
}
pruneTimer = setInterval(() => {
if (runtimeConfig.autoPruneDays > 0) {
void writer
.pruneExpiredPartitionsAsync(runtimeConfig.autoPruneDays)
.catch(() => undefined);
}
}, PRUNE_INTERVAL_MS);
if (isShuttingDown) return;
await new Promise((resolve) => {
server.listen(PORT, () => {
console.log(
`[DataStack Ingestion Engine] Express + Socket.IO listening on port ${PORT}`,
);
resolve();
});
});
}
if (process.env.DATASTACK_DISABLE_LISTEN !== '1') {
process.on('SIGTERM', () => {
void gracefulShutdown('SIGTERM');
});
process.on('SIGINT', () => {
void gracefulShutdown('SIGINT');
});
void startIngestionServer();
}
module.exports = {
app,
server,
io,
writer,
bufferQueue,
flushBuffer,
runtimeConfig,
verifyClientInDb,
recordClientActivity,
flushPendingClientActivity,
applyRuntimeSettingsObject,
ensureDatabaseSchema,
syncSettingsFromDatabase,
gracefulShutdown,
startIngestionServer,
setRedisClientForTests,
setDbPoolForTests,
stopTimers,
parseAllowedCorsOrigins,
isOriginAllowedByCors,
corsOriginDelegate,
};