2026-11-18 05:18:55 +06:00
|
|
|
const crypto = require('node:crypto');
|
2026-11-21 22:15:32 +06:00
|
|
|
const dnsPromises = require('node:dns/promises');
|
2026-11-21 00:06:55 +06:00
|
|
|
const net = require('node:net');
|
2026-10-23 11:43:49 -05:00
|
|
|
const express = require('express');
|
|
|
|
|
const http = require('http');
|
|
|
|
|
const cors = require('cors');
|
|
|
|
|
const { Server } = require('socket.io');
|
2026-11-17 20:05:00 +06:00
|
|
|
const { Pool } = require('pg');
|
2026-11-18 05:18:55 +06:00
|
|
|
const Redis = require('ioredis');
|
2026-11-19 20:22:01 +06:00
|
|
|
const { setupSocketHandlers, normalizeClientId } = require('./socket/handlers');
|
2026-10-23 11:43:49 -05:00
|
|
|
const { CsvWriterWorker } = require('./workers/csvWriter');
|
2026-11-21 22:15:32 +06:00
|
|
|
const {
|
|
|
|
|
mountFilesManager,
|
|
|
|
|
syncInMemoryAdminSessionState,
|
|
|
|
|
clearInMemoryAdminSessionStateForTests,
|
|
|
|
|
} = require('./filesManager');
|
2026-11-21 23:00:29 +06:00
|
|
|
const { installRuntimeLogHook } = require('./runtimeLogCapture');
|
2026-11-21 11:26:01 -08:00
|
|
|
const { createMcpEngine } = require('./mcp/mcpEngine');
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
function parseAllowedCorsOrigins(raw = process.env.DATASTACK_CORS_ORIGINS) {
|
|
|
|
|
if (typeof raw !== 'string') return null;
|
|
|
|
|
const trimmed = raw.trim();
|
|
|
|
|
if (!trimmed || trimmed === '*') return null;
|
|
|
|
|
const origins = trimmed
|
|
|
|
|
.split(',')
|
|
|
|
|
.map((entry) => entry.trim().replace(/\/+$/, ''))
|
|
|
|
|
.filter(Boolean);
|
|
|
|
|
if (origins.length === 0 || origins.includes('*')) return null;
|
|
|
|
|
return new Set(origins);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function isOriginAllowedByCors(origin, raw = process.env.DATASTACK_CORS_ORIGINS) {
|
|
|
|
|
if (!origin) return true;
|
|
|
|
|
const allowlist = parseAllowedCorsOrigins(raw);
|
|
|
|
|
if (!allowlist) return true;
|
|
|
|
|
const normalized = String(origin).trim().replace(/\/+$/, '');
|
|
|
|
|
return allowlist.has(normalized);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function corsOriginDelegate(origin, callback) {
|
|
|
|
|
return callback(null, isOriginAllowedByCors(origin));
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-17 20:05:00 +06:00
|
|
|
const PORT = Number(process.env.PORT_SOCKET) || 4000;
|
2026-10-23 11:43:49 -05:00
|
|
|
const app = express();
|
2026-11-19 20:22:01 +06:00
|
|
|
app.use(cors({ origin: corsOriginDelegate }));
|
2026-11-21 22:15:32 +06:00
|
|
|
const jsonBodyParser = express.json({ limit: '10mb' });
|
|
|
|
|
app.use((req, res, next) => {
|
|
|
|
|
if (req.path === '/vcon/files-engine' || req.path.startsWith('/vcon/files-engine/')) {
|
|
|
|
|
return next();
|
|
|
|
|
}
|
|
|
|
|
return jsonBodyParser(req, res, next);
|
|
|
|
|
});
|
|
|
|
|
let filesManagerHandle = null;
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
const REDIS_BUFFER_KEY = 'datastack:buffer:events';
|
|
|
|
|
const REDIS_RECENT_KEY = 'datastack:recent:events';
|
|
|
|
|
const REDIS_FLUSH_LOCK_KEY = 'datastack:flush:lock';
|
2026-11-19 20:22:01 +06:00
|
|
|
const REDIS_CLIENTS_REGISTRY_KEY = 'datastack:clients:registry';
|
|
|
|
|
const REDIS_CLIENTS_DELETED_KEY = 'datastack:clients:deleted';
|
|
|
|
|
const DEFAULT_REDIS_RECENT_LIMIT = 500;
|
|
|
|
|
const MAX_REDIS_RECENT_LIMIT = 5000;
|
|
|
|
|
const FLUSH_LOCK_TTL_MS = 30000;
|
|
|
|
|
const FLUSH_LOCK_WAIT_MS = 400;
|
|
|
|
|
const FLUSH_LOCK_RETRY_MS = 25;
|
2026-11-18 05:18:55 +06:00
|
|
|
const UNLOCK_LUA_SCRIPT =
|
|
|
|
|
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end';
|
2026-11-19 20:22:01 +06:00
|
|
|
const RENEW_LOCK_LUA_SCRIPT =
|
|
|
|
|
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("pexpire", KEYS[1], ARGV[2]) else return 0 end';
|
|
|
|
|
|
|
|
|
|
function getRedisRecentLimit() {
|
|
|
|
|
const envRaw = Number(process.env.DATASTACK_RECENT_LIMIT);
|
|
|
|
|
if (Number.isFinite(envRaw) && envRaw >= 100) {
|
|
|
|
|
return Math.min(MAX_REDIS_RECENT_LIMIT, Math.floor(envRaw));
|
|
|
|
|
}
|
|
|
|
|
const threshold = Number(runtimeConfig && runtimeConfig.batchFlushThreshold) || DEFAULT_REDIS_RECENT_LIMIT;
|
|
|
|
|
return Math.min(
|
|
|
|
|
MAX_REDIS_RECENT_LIMIT,
|
|
|
|
|
Math.max(DEFAULT_REDIS_RECENT_LIMIT, Math.floor(threshold)),
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function sleepMs(ms) {
|
|
|
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
|
2026-10-23 11:43:49 -05:00
|
|
|
const bufferQueue = [];
|
|
|
|
|
const writer = new CsvWriterWorker();
|
2026-11-18 05:18:55 +06:00
|
|
|
let redisClient = undefined;
|
|
|
|
|
let redisUrlConfigured = '';
|
|
|
|
|
let redisConnected = false;
|
|
|
|
|
let redisTestOverride = undefined;
|
|
|
|
|
const pendingRedisWrites = new Set();
|
|
|
|
|
|
|
|
|
|
function getRedisClient() {
|
|
|
|
|
if (redisTestOverride !== undefined) {
|
|
|
|
|
return redisTestOverride;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const configuredUrl = (process.env.REDIS_URL || '').trim();
|
|
|
|
|
if (!configuredUrl) {
|
|
|
|
|
if (redisClient && typeof redisClient.disconnect === 'function') {
|
|
|
|
|
try {
|
|
|
|
|
redisClient.disconnect();
|
|
|
|
|
} catch {
|
|
|
|
|
// ignore
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
redisClient = null;
|
|
|
|
|
redisUrlConfigured = '';
|
|
|
|
|
redisConnected = false;
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (redisClient && redisUrlConfigured === configuredUrl) {
|
|
|
|
|
return redisClient;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (redisClient && typeof redisClient.disconnect === 'function') {
|
|
|
|
|
try {
|
|
|
|
|
redisClient.disconnect();
|
|
|
|
|
} catch {
|
|
|
|
|
// ignore
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
const client = new Redis(configuredUrl, {
|
|
|
|
|
maxRetriesPerRequest: 2,
|
|
|
|
|
enableReadyCheck: true,
|
|
|
|
|
retryStrategy: (times) => Math.min(times * 250, 3000),
|
|
|
|
|
});
|
|
|
|
|
client.on('ready', () => {
|
|
|
|
|
redisConnected = true;
|
|
|
|
|
});
|
|
|
|
|
client.on('connect', () => {
|
|
|
|
|
redisConnected = true;
|
|
|
|
|
});
|
|
|
|
|
client.on('close', () => {
|
|
|
|
|
redisConnected = false;
|
|
|
|
|
});
|
|
|
|
|
client.on('end', () => {
|
|
|
|
|
redisConnected = false;
|
|
|
|
|
});
|
|
|
|
|
client.on('error', () => {
|
|
|
|
|
redisConnected = false;
|
|
|
|
|
});
|
|
|
|
|
redisClient = client;
|
|
|
|
|
redisUrlConfigured = configuredUrl;
|
|
|
|
|
redisConnected = client.status === 'ready' || client.status === 'connect';
|
|
|
|
|
return redisClient;
|
|
|
|
|
} catch {
|
|
|
|
|
redisClient = null;
|
|
|
|
|
redisUrlConfigured = configuredUrl;
|
|
|
|
|
redisConnected = false;
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function setRedisClientForTests(client) {
|
|
|
|
|
redisTestOverride = client;
|
|
|
|
|
redisConnected = Boolean(client);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function pushEventsToRedisOrFallback(events) {
|
|
|
|
|
if (!events || events.length === 0) return bufferQueue.length;
|
|
|
|
|
const redis = getRedisClient();
|
2026-11-22 15:33:14 +06:00
|
|
|
let persistFailed = false;
|
2026-11-18 05:18:55 +06:00
|
|
|
if (redis) {
|
|
|
|
|
const serialized = events.map((e) => JSON.stringify(e));
|
2026-11-19 20:22:01 +06:00
|
|
|
let bufferLen = null;
|
2026-11-18 05:18:55 +06:00
|
|
|
try {
|
2026-11-19 20:22:01 +06:00
|
|
|
bufferLen = await redis.rpush(REDIS_BUFFER_KEY, ...serialized);
|
2026-11-18 05:18:55 +06:00
|
|
|
redisConnected = true;
|
|
|
|
|
} catch {
|
|
|
|
|
redisConnected = false;
|
2026-11-22 15:33:14 +06:00
|
|
|
persistFailed = true;
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
|
|
|
|
|
if (bufferLen !== null) {
|
|
|
|
|
// Best-effort recent list mirror: never duplicate into RAM buffer once rpush succeeded
|
|
|
|
|
try {
|
|
|
|
|
const recentLimit = getRedisRecentLimit();
|
|
|
|
|
await redis.lpush(REDIS_RECENT_KEY, ...serialized);
|
|
|
|
|
await redis.ltrim(REDIS_RECENT_KEY, 0, recentLimit - 1);
|
|
|
|
|
} catch {
|
|
|
|
|
// Non-fatal: events are already durable in REDIS_BUFFER_KEY
|
|
|
|
|
}
|
|
|
|
|
return Number(bufferLen) + bufferQueue.length;
|
|
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bufferQueue.push(...events);
|
2026-11-22 15:33:14 +06:00
|
|
|
if (persistFailed) {
|
|
|
|
|
// Keep the volatile fallback for later recovery, but never ACK it as durable.
|
|
|
|
|
// Suppress sensitive Redis transport details in the client-facing error.
|
|
|
|
|
throw new Error('BUFFER_PERSIST_FAILED: Redis unavailable; event not durably queued');
|
|
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
return bufferQueue.length;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const IS_PRODUCTION = process.env.NODE_ENV === 'production';
|
|
|
|
|
const DEV_DEFAULT_JWT_SECRET = 'datastack-jwt-default-secret-key-32-chars-long';
|
|
|
|
|
|
|
|
|
|
function resolveInitialJwtSecret() {
|
|
|
|
|
const configured = (process.env.DATASTACK_JWT_SECRET || '').trim();
|
|
|
|
|
if (IS_PRODUCTION) {
|
|
|
|
|
return configured.length >= 32 ? configured : '';
|
|
|
|
|
}
|
|
|
|
|
if (configured.length > 0) {
|
|
|
|
|
return configured.length >= 32 ? configured : '';
|
|
|
|
|
}
|
|
|
|
|
return DEV_DEFAULT_JWT_SECRET;
|
|
|
|
|
}
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-11-17 20:05:00 +06:00
|
|
|
const runtimeConfig = {
|
2026-11-18 05:18:55 +06:00
|
|
|
jwtSecret: resolveInitialJwtSecret(),
|
2026-11-17 20:05:00 +06:00
|
|
|
maxEventsPerSec: 100,
|
|
|
|
|
maxPayloadSizeBytes: 100 * 1024,
|
|
|
|
|
batchFlushThreshold: 500,
|
|
|
|
|
batchFlushIntervalMs: 2000,
|
|
|
|
|
fileRotationMb: 100,
|
|
|
|
|
enableIdempotency: true,
|
2026-11-18 05:18:55 +06:00
|
|
|
autoPruneDays: 30,
|
2026-11-19 23:02:27 +06:00
|
|
|
compressAfterDays: 0,
|
|
|
|
|
enableS3Backup: false,
|
|
|
|
|
diskPauseThresholdPct: 95,
|
2026-11-21 00:06:55 +06:00
|
|
|
enableWebhookAlerts: false,
|
|
|
|
|
webhookUrl: '',
|
|
|
|
|
webhookSecret: '',
|
|
|
|
|
webhookCooldownSec: 300,
|
2026-11-17 20:05:00 +06:00
|
|
|
};
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
let previousJwtSecret = '';
|
|
|
|
|
let previousJwtSecretGraceUntil = 0;
|
|
|
|
|
const JWT_ROTATION_BRIDGE_GRACE_MS = 60 * 1000;
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
function resolveInternalSecret(jwtSecretOverride) {
|
|
|
|
|
const explicit = (process.env.DATASTACK_INTERNAL_SECRET || '').trim();
|
|
|
|
|
if (IS_PRODUCTION) {
|
|
|
|
|
if (explicit.length >= 32) return explicit;
|
|
|
|
|
if (explicit.length > 0) return '';
|
|
|
|
|
} else if (explicit.length > 0) {
|
|
|
|
|
return explicit;
|
|
|
|
|
}
|
|
|
|
|
const activeJwt = String(
|
|
|
|
|
jwtSecretOverride !== undefined ? jwtSecretOverride : runtimeConfig.jwtSecret || '',
|
|
|
|
|
).trim();
|
|
|
|
|
const minJwtLen = IS_PRODUCTION ? 32 : 1;
|
|
|
|
|
if (activeJwt.length < minJwtLen) {
|
|
|
|
|
return '';
|
|
|
|
|
}
|
|
|
|
|
return crypto
|
|
|
|
|
.createHash('sha256')
|
|
|
|
|
.update(`datastack-internal:${activeJwt}`)
|
|
|
|
|
.digest('hex');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function timingSafeSecretEqual(candidate, expected) {
|
|
|
|
|
if (!candidate || !expected) return false;
|
|
|
|
|
const a = crypto.createHash('sha256').update(String(candidate)).digest();
|
|
|
|
|
const b = crypto.createHash('sha256').update(String(expected)).digest();
|
|
|
|
|
return crypto.timingSafeEqual(a, b);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function requireInternalAuth(req, res, next) {
|
|
|
|
|
const expected = resolveInternalSecret();
|
|
|
|
|
if (!expected) {
|
|
|
|
|
return res.status(503).json({
|
|
|
|
|
success: false,
|
|
|
|
|
error: 'INTERNAL_AUTH_UNAVAILABLE: Internal bridge secret is not configured',
|
|
|
|
|
code: 'INTERNAL_AUTH_UNAVAILABLE',
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
const presented = req.get('x-datastack-internal-secret');
|
2026-11-19 20:22:01 +06:00
|
|
|
const matchesCurrent = Boolean(presented && timingSafeSecretEqual(presented, expected));
|
|
|
|
|
let matchesGrace = false;
|
|
|
|
|
if (!matchesCurrent && presented && previousJwtSecret && Date.now() <= previousJwtSecretGraceUntil) {
|
|
|
|
|
const graceExpected = resolveInternalSecret(previousJwtSecret);
|
|
|
|
|
matchesGrace = Boolean(graceExpected && timingSafeSecretEqual(presented, graceExpected));
|
|
|
|
|
}
|
|
|
|
|
if (!matchesCurrent && !matchesGrace) {
|
2026-11-18 05:18:55 +06:00
|
|
|
return res.status(401).json({
|
|
|
|
|
success: false,
|
|
|
|
|
error: 'UNAUTHORIZED: Missing or invalid internal authentication header',
|
|
|
|
|
code: 'UNAUTHORIZED',
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
return next();
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-17 20:05:00 +06:00
|
|
|
let dbPool = null;
|
|
|
|
|
if (process.env.DATABASE_URL) {
|
|
|
|
|
try {
|
|
|
|
|
dbPool = new Pool({
|
|
|
|
|
connectionString: process.env.DATABASE_URL,
|
|
|
|
|
connectionTimeoutMillis: 3000,
|
|
|
|
|
idleTimeoutMillis: 30000,
|
|
|
|
|
max: 5,
|
|
|
|
|
});
|
2026-11-19 20:22:01 +06:00
|
|
|
dbPool.on('error', (err) => {
|
|
|
|
|
console.error(
|
|
|
|
|
'[DataStack Ingestion DB] Unexpected idle PostgreSQL pool error:',
|
|
|
|
|
err instanceof Error ? err.message : err,
|
|
|
|
|
);
|
|
|
|
|
});
|
2026-11-17 20:05:00 +06:00
|
|
|
} catch {
|
|
|
|
|
dbPool = null;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
// Short-lived cache (2s TTL) for DB client status & token checks on high-frequency events
|
2026-11-17 20:05:00 +06:00
|
|
|
const clientAuthCache = new Map();
|
2026-11-18 05:18:55 +06:00
|
|
|
const deletedClientsSet = new Set();
|
|
|
|
|
const inMemoryClientRegistry = new Map();
|
|
|
|
|
const inMemoryClientActivity = new Map();
|
|
|
|
|
let hasSyncedInMemoryClients = false;
|
2026-11-17 20:05:00 +06:00
|
|
|
const CLIENT_CACHE_TTL_MS = 2000;
|
|
|
|
|
|
2026-11-19 23:02:27 +06:00
|
|
|
function stripIpv6MappedPrefix(rawIp) {
|
|
|
|
|
let ip = String(rawIp || '').trim();
|
|
|
|
|
if (!ip) return '';
|
|
|
|
|
if (ip.startsWith('[') && ip.includes(']')) {
|
|
|
|
|
ip = ip.slice(1, ip.indexOf(']'));
|
|
|
|
|
} else if (/^\d{1,3}(?:\.\d{1,3}){3}:\d+$/.test(ip)) {
|
|
|
|
|
ip = ip.slice(0, ip.lastIndexOf(':'));
|
|
|
|
|
}
|
|
|
|
|
const pctIndex = ip.indexOf('%');
|
|
|
|
|
if (pctIndex !== -1) {
|
|
|
|
|
ip = ip.slice(0, pctIndex);
|
|
|
|
|
}
|
|
|
|
|
if (/^::ffff:\d{1,3}(?:\.\d{1,3}){3}$/i.test(ip)) {
|
|
|
|
|
return ip.slice(7);
|
|
|
|
|
}
|
|
|
|
|
return ip.toLowerCase();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function ipv4ToUint32(ip) {
|
|
|
|
|
const parts = ip.split('.');
|
|
|
|
|
if (parts.length !== 4) return null;
|
|
|
|
|
let acc = 0;
|
|
|
|
|
for (const part of parts) {
|
|
|
|
|
if (!/^\d{1,3}$/.test(part)) return null;
|
|
|
|
|
if (part.length > 1 && part.startsWith('0')) return null;
|
|
|
|
|
const num = Number(part);
|
|
|
|
|
if (num < 0 || num > 255) return null;
|
|
|
|
|
acc = ((acc << 8) | num) >>> 0;
|
|
|
|
|
}
|
|
|
|
|
return acc >>> 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function matchesIpv4Cidr(clientIp, cidrPattern) {
|
|
|
|
|
const [networkStr, prefixStr] = cidrPattern.split('/');
|
|
|
|
|
if (!networkStr || prefixStr === undefined) return false;
|
|
|
|
|
const prefix = Number(prefixStr);
|
|
|
|
|
if (!Number.isInteger(prefix) || prefix < 0 || prefix > 32) return false;
|
|
|
|
|
const clientNum = ipv4ToUint32(clientIp);
|
|
|
|
|
const networkNum = ipv4ToUint32(networkStr);
|
|
|
|
|
if (clientNum === null || networkNum === null) return false;
|
|
|
|
|
if (prefix === 0) return true;
|
|
|
|
|
const mask = (0xffffffff << (32 - prefix)) >>> 0;
|
|
|
|
|
return (clientNum & mask) >>> 0 === (networkNum & mask) >>> 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function isClientIpAllowed(clientIp, rawAllowedIps) {
|
|
|
|
|
if (!rawAllowedIps || !String(rawAllowedIps).trim()) return true;
|
|
|
|
|
const entries = String(rawAllowedIps)
|
2026-11-21 00:06:55 +06:00
|
|
|
.split(/[\s,;]+/)
|
2026-11-19 23:02:27 +06:00
|
|
|
.map((entry) => entry.trim())
|
|
|
|
|
.filter(Boolean);
|
|
|
|
|
if (entries.length === 0) return true;
|
|
|
|
|
if (entries.includes('*') || entries.includes('0.0.0.0/0') || entries.includes('::/0')) {
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
const normalizedClientIp = stripIpv6MappedPrefix(clientIp);
|
|
|
|
|
if (!normalizedClientIp || normalizedClientIp === 'unknown') {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
2026-11-21 00:06:55 +06:00
|
|
|
const clientFamily = net.isIP(normalizedClientIp);
|
2026-11-19 23:02:27 +06:00
|
|
|
for (const entry of entries) {
|
|
|
|
|
const normalizedEntry = stripIpv6MappedPrefix(entry);
|
|
|
|
|
if (normalizedEntry.includes('/')) {
|
2026-11-21 00:06:55 +06:00
|
|
|
const [netAddrRaw, prefixStr] = normalizedEntry.split('/');
|
|
|
|
|
const netAddr = stripIpv6MappedPrefix(netAddrRaw);
|
|
|
|
|
const prefix = Number(prefixStr);
|
|
|
|
|
const ruleFamily = net.isIP(netAddr);
|
|
|
|
|
|
|
|
|
|
if (
|
|
|
|
|
clientFamily === 4 &&
|
|
|
|
|
ruleFamily === 4 &&
|
|
|
|
|
Number.isInteger(prefix) &&
|
|
|
|
|
prefix >= 0 &&
|
|
|
|
|
prefix <= 32
|
|
|
|
|
) {
|
|
|
|
|
if (prefix === 0) return true;
|
|
|
|
|
try {
|
|
|
|
|
const bl = new net.BlockList();
|
|
|
|
|
bl.addSubnet(netAddr, prefix, 'ipv4');
|
|
|
|
|
if (bl.check(normalizedClientIp, 'ipv4')) {
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
if (matchesIpv4Cidr(normalizedClientIp, `${netAddr}/${prefix}`)) {
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} else if (
|
|
|
|
|
clientFamily === 6 &&
|
|
|
|
|
ruleFamily === 6 &&
|
|
|
|
|
Number.isInteger(prefix) &&
|
|
|
|
|
prefix >= 0 &&
|
|
|
|
|
prefix <= 128
|
|
|
|
|
) {
|
|
|
|
|
if (prefix === 0) return true;
|
|
|
|
|
try {
|
|
|
|
|
const bl = new net.BlockList();
|
|
|
|
|
bl.addSubnet(netAddr, prefix, 'ipv6');
|
|
|
|
|
if (bl.check(normalizedClientIp, 'ipv6')) {
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
if (prefix === 128 && normalizedClientIp === netAddr) {
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-11-19 23:02:27 +06:00
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
if (normalizedClientIp === normalizedEntry) {
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
if (
|
|
|
|
|
(normalizedClientIp === '127.0.0.1' && normalizedEntry === '::1') ||
|
|
|
|
|
(normalizedClientIp === '::1' && normalizedEntry === '127.0.0.1')
|
|
|
|
|
) {
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function isClientTokenExpired(expiresAt, nowMs = Date.now()) {
|
|
|
|
|
if (!expiresAt) return false;
|
|
|
|
|
const ts =
|
|
|
|
|
expiresAt instanceof Date
|
|
|
|
|
? expiresAt.getTime()
|
|
|
|
|
: new Date(String(expiresAt)).getTime();
|
|
|
|
|
if (Number.isNaN(ts)) return false;
|
|
|
|
|
return ts <= nowMs;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function verifyClientInDb(clientId, presentedToken, bypassCache = false, clientIp = '') {
|
2026-11-19 20:22:01 +06:00
|
|
|
const normalizedId = normalizeClientId(clientId);
|
|
|
|
|
if (!normalizedId) {
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
2026-11-19 23:02:27 +06:00
|
|
|
code: 'CLIENT_REVOKED',
|
2026-11-19 20:22:01 +06:00
|
|
|
reason: 'CLIENT_REVOKED: Missing client identifier',
|
|
|
|
|
};
|
2026-11-17 20:05:00 +06:00
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
if (deletedClientsSet.has(normalizedId)) {
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
2026-11-19 23:02:27 +06:00
|
|
|
code: 'CLIENT_REVOKED',
|
2026-11-18 05:18:55 +06:00
|
|
|
reason: 'CLIENT_REVOKED: Client access has been deleted or revoked by admin',
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!dbPool) {
|
2026-11-19 20:22:01 +06:00
|
|
|
let memClient = inMemoryClientRegistry.get(normalizedId);
|
|
|
|
|
if (!memClient) {
|
|
|
|
|
const redis = getRedisClient();
|
|
|
|
|
if (redis && typeof redis.sismember === 'function' && typeof redis.hget === 'function') {
|
|
|
|
|
try {
|
|
|
|
|
const isDeleted = await redis.sismember(REDIS_CLIENTS_DELETED_KEY, normalizedId);
|
|
|
|
|
if (isDeleted) {
|
|
|
|
|
deletedClientsSet.add(normalizedId);
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
2026-11-19 23:02:27 +06:00
|
|
|
code: 'CLIENT_REVOKED',
|
2026-11-19 20:22:01 +06:00
|
|
|
reason: 'CLIENT_REVOKED: Client access has been deleted by admin',
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
const rawClient = await redis.hget(REDIS_CLIENTS_REGISTRY_KEY, normalizedId);
|
|
|
|
|
if (rawClient) {
|
|
|
|
|
const parsed = JSON.parse(rawClient);
|
|
|
|
|
if (parsed && parsed.clientId) {
|
|
|
|
|
memClient = {
|
|
|
|
|
clientId: normalizedId,
|
|
|
|
|
name: String(parsed.name || normalizedId),
|
|
|
|
|
status: parsed.status === 'revoked' ? 'revoked' : 'active',
|
|
|
|
|
token: String(parsed.token || ''),
|
|
|
|
|
rateLimit: Math.max(1, Number(parsed.rateLimit) || 100),
|
2026-11-19 23:02:27 +06:00
|
|
|
storageQuotaMb: Math.max(0, Math.floor(Number(parsed.storageQuotaMb) || 0)),
|
|
|
|
|
allowedIps: typeof parsed.allowedIps === 'string' ? parsed.allowedIps : '',
|
|
|
|
|
expiresAt: parsed.expiresAt ? String(parsed.expiresAt) : null,
|
2026-11-19 20:22:01 +06:00
|
|
|
};
|
|
|
|
|
inMemoryClientRegistry.set(normalizedId, memClient);
|
|
|
|
|
hasSyncedInMemoryClients = true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
// Ignore Redis registry lookup error and check local state
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
if (memClient) {
|
|
|
|
|
if (memClient.status === 'revoked') {
|
2026-11-17 20:05:00 +06:00
|
|
|
return {
|
|
|
|
|
allowed: false,
|
2026-11-19 23:02:27 +06:00
|
|
|
code: 'CLIENT_REVOKED',
|
2026-11-17 20:05:00 +06:00
|
|
|
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
|
|
|
|
|
};
|
|
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
if (memClient.token && presentedToken && memClient.token !== presentedToken) {
|
2026-11-17 20:05:00 +06:00
|
|
|
return {
|
|
|
|
|
allowed: false,
|
2026-11-19 23:02:27 +06:00
|
|
|
code: 'TOKEN_ROTATED',
|
2026-11-17 20:05:00 +06:00
|
|
|
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
|
|
|
|
|
};
|
|
|
|
|
}
|
2026-11-19 23:02:27 +06:00
|
|
|
if (isClientTokenExpired(memClient.expiresAt)) {
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
|
|
|
|
code: 'CLIENT_TOKEN_EXPIRED',
|
|
|
|
|
reason: `CLIENT_TOKEN_EXPIRED: Client '${normalizedId}' token has expired`,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
if (memClient.allowedIps && memClient.allowedIps.trim() && !isClientIpAllowed(clientIp, memClient.allowedIps)) {
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
|
|
|
|
code: 'CLIENT_IP_FORBIDDEN',
|
|
|
|
|
reason: `CLIENT_IP_FORBIDDEN: Client IP '${clientIp || 'unknown'}' is not authorized`,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
return {
|
|
|
|
|
allowed: true,
|
|
|
|
|
rateLimit: memClient.rateLimit || 100,
|
|
|
|
|
storageQuotaMb: memClient.storageQuotaMb || 0,
|
|
|
|
|
};
|
2026-11-17 20:05:00 +06:00
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
2026-11-19 23:02:27 +06:00
|
|
|
code: 'CLIENT_REVOKED',
|
2026-11-19 20:22:01 +06:00
|
|
|
reason: 'CLIENT_REVOKED: Client registration not found or has been deleted',
|
|
|
|
|
};
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const now = Date.now();
|
|
|
|
|
const prevCached = clientAuthCache.get(normalizedId);
|
|
|
|
|
if (!bypassCache && prevCached && now - prevCached.checkedAt < CLIENT_CACHE_TTL_MS) {
|
|
|
|
|
if (prevCached.status === 'revoked') {
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
2026-11-19 23:02:27 +06:00
|
|
|
code: 'CLIENT_REVOKED',
|
2026-11-18 05:18:55 +06:00
|
|
|
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
if (prevCached.token && presentedToken && prevCached.token !== presentedToken) {
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
2026-11-19 23:02:27 +06:00
|
|
|
code: 'TOKEN_ROTATED',
|
2026-11-18 05:18:55 +06:00
|
|
|
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
|
|
|
|
|
};
|
|
|
|
|
}
|
2026-11-19 23:02:27 +06:00
|
|
|
if (isClientTokenExpired(prevCached.expiresAt, now)) {
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
|
|
|
|
code: 'CLIENT_TOKEN_EXPIRED',
|
|
|
|
|
reason: `CLIENT_TOKEN_EXPIRED: Client '${normalizedId}' token has expired`,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
if (prevCached.allowedIps && prevCached.allowedIps.trim() && !isClientIpAllowed(clientIp, prevCached.allowedIps)) {
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
|
|
|
|
code: 'CLIENT_IP_FORBIDDEN',
|
|
|
|
|
reason: `CLIENT_IP_FORBIDDEN: Client IP '${clientIp || 'unknown'}' is not authorized`,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
return {
|
|
|
|
|
allowed: true,
|
|
|
|
|
rateLimit: prevCached.rateLimit,
|
|
|
|
|
storageQuotaMb: prevCached.storageQuotaMb || 0,
|
|
|
|
|
};
|
2026-11-17 20:05:00 +06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
try {
|
2026-11-18 05:18:55 +06:00
|
|
|
const deletedRes = await dbPool.query(
|
|
|
|
|
`SELECT 1 FROM deleted_clients WHERE client_id = $1`,
|
|
|
|
|
[normalizedId],
|
|
|
|
|
);
|
|
|
|
|
if (deletedRes.rowCount && deletedRes.rowCount > 0) {
|
|
|
|
|
clientAuthCache.delete(normalizedId);
|
|
|
|
|
deletedClientsSet.add(normalizedId);
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
2026-11-19 23:02:27 +06:00
|
|
|
code: 'CLIENT_REVOKED',
|
2026-11-18 05:18:55 +06:00
|
|
|
reason: 'CLIENT_REVOKED: Client access has been deleted by admin',
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-17 20:05:00 +06:00
|
|
|
const res = await dbPool.query(
|
2026-11-19 23:02:27 +06:00
|
|
|
`SELECT status, token, rate_limit, storage_quota_mb, allowed_ips, expires_at FROM client_sdks WHERE client_id = $1`,
|
2026-11-18 05:18:55 +06:00
|
|
|
[normalizedId],
|
2026-11-17 20:05:00 +06:00
|
|
|
);
|
|
|
|
|
if (!res.rowCount || res.rowCount === 0) {
|
2026-11-18 05:18:55 +06:00
|
|
|
clientAuthCache.delete(normalizedId);
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
2026-11-19 23:02:27 +06:00
|
|
|
code: 'CLIENT_REVOKED',
|
2026-11-18 05:18:55 +06:00
|
|
|
reason: 'CLIENT_REVOKED: Client registration not found or has been deleted',
|
|
|
|
|
};
|
2026-11-17 20:05:00 +06:00
|
|
|
}
|
|
|
|
|
const row = res.rows[0];
|
2026-11-19 23:02:27 +06:00
|
|
|
const parsedQuota = Number(row.storage_quota_mb);
|
|
|
|
|
let expiresAtIso = null;
|
|
|
|
|
if (row.expires_at) {
|
|
|
|
|
const d = new Date(row.expires_at);
|
|
|
|
|
if (!Number.isNaN(d.getTime())) {
|
|
|
|
|
expiresAtIso = d.toISOString();
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-11-17 20:05:00 +06:00
|
|
|
const entry = {
|
|
|
|
|
status: row.status,
|
|
|
|
|
token: row.token,
|
|
|
|
|
rateLimit: Number(row.rate_limit) || 100,
|
2026-11-19 23:02:27 +06:00
|
|
|
storageQuotaMb: Number.isFinite(parsedQuota) && parsedQuota >= 0 ? Math.floor(parsedQuota) : 0,
|
|
|
|
|
allowedIps: typeof row.allowed_ips === 'string' ? row.allowed_ips : '',
|
|
|
|
|
expiresAt: expiresAtIso,
|
2026-11-17 20:05:00 +06:00
|
|
|
checkedAt: now,
|
|
|
|
|
};
|
2026-11-18 05:18:55 +06:00
|
|
|
clientAuthCache.set(normalizedId, entry);
|
2026-11-17 20:05:00 +06:00
|
|
|
|
|
|
|
|
if (entry.status === 'revoked') {
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
2026-11-19 23:02:27 +06:00
|
|
|
code: 'CLIENT_REVOKED',
|
2026-11-17 20:05:00 +06:00
|
|
|
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
if (entry.token && presentedToken && entry.token !== presentedToken) {
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
2026-11-19 23:02:27 +06:00
|
|
|
code: 'TOKEN_ROTATED',
|
2026-11-17 20:05:00 +06:00
|
|
|
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
|
|
|
|
|
};
|
|
|
|
|
}
|
2026-11-19 23:02:27 +06:00
|
|
|
if (isClientTokenExpired(entry.expiresAt, now)) {
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
|
|
|
|
code: 'CLIENT_TOKEN_EXPIRED',
|
|
|
|
|
reason: `CLIENT_TOKEN_EXPIRED: Client '${normalizedId}' token has expired`,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
if (entry.allowedIps && entry.allowedIps.trim() && !isClientIpAllowed(clientIp, entry.allowedIps)) {
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
|
|
|
|
code: 'CLIENT_IP_FORBIDDEN',
|
|
|
|
|
reason: `CLIENT_IP_FORBIDDEN: Client IP '${clientIp || 'unknown'}' is not authorized`,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
return {
|
|
|
|
|
allowed: true,
|
|
|
|
|
rateLimit: entry.rateLimit,
|
|
|
|
|
storageQuotaMb: entry.storageQuotaMb,
|
|
|
|
|
};
|
2026-11-17 20:05:00 +06:00
|
|
|
} catch {
|
2026-11-18 05:18:55 +06:00
|
|
|
return {
|
|
|
|
|
allowed: false,
|
2026-11-19 23:02:27 +06:00
|
|
|
code: 'AUTH_UNAVAILABLE',
|
2026-11-18 05:18:55 +06:00
|
|
|
reason: 'AUTH_UNAVAILABLE: Unable to verify client credentials against database',
|
|
|
|
|
};
|
2026-11-17 20:05:00 +06:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
const pendingDbClientActivity = new Map();
|
|
|
|
|
const CLIENT_ACTIVITY_FLUSH_INTERVAL_MS = 1000;
|
|
|
|
|
let activityFlushTimer = null;
|
|
|
|
|
let activeActivityFlushPromise = null;
|
|
|
|
|
|
|
|
|
|
let schemaEnsurePromise = null;
|
|
|
|
|
let schemaEnsurePool = null;
|
2026-11-20 18:48:54 +06:00
|
|
|
let hasHydratedSettingsFromDb = false;
|
|
|
|
|
|
|
|
|
|
function isDbReadyForMaintenance() {
|
|
|
|
|
return !dbPool || hasHydratedSettingsFromDb === true;
|
|
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
|
|
|
|
|
function setDbPoolForTests(pool) {
|
|
|
|
|
dbPool = pool;
|
|
|
|
|
schemaEnsurePromise = null;
|
|
|
|
|
schemaEnsurePool = null;
|
2026-11-20 18:48:54 +06:00
|
|
|
hasHydratedSettingsFromDb = false;
|
2026-11-19 20:22:01 +06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function flushPendingClientActivity() {
|
|
|
|
|
if (activeActivityFlushPromise) {
|
|
|
|
|
await activeActivityFlushPromise;
|
|
|
|
|
if (pendingDbClientActivity.size === 0) return 0;
|
|
|
|
|
}
|
|
|
|
|
if (!dbPool || pendingDbClientActivity.size === 0) return 0;
|
|
|
|
|
|
|
|
|
|
const snapshot = Array.from(pendingDbClientActivity.entries());
|
|
|
|
|
pendingDbClientActivity.clear();
|
|
|
|
|
|
|
|
|
|
const clientIds = [];
|
|
|
|
|
const deltas = [];
|
|
|
|
|
const lastActives = [];
|
|
|
|
|
for (const [cid, entry] of snapshot) {
|
|
|
|
|
clientIds.push(cid);
|
|
|
|
|
deltas.push(entry.count);
|
|
|
|
|
lastActives.push(entry.lastActive);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const run = (async () => {
|
|
|
|
|
try {
|
|
|
|
|
await dbPool.query(
|
|
|
|
|
`UPDATE client_sdks AS c
|
|
|
|
|
SET total_events = c.total_events + v.delta,
|
|
|
|
|
last_active = GREATEST(COALESCE(c.last_active, v.last_active), v.last_active),
|
|
|
|
|
updated_at = NOW()
|
|
|
|
|
FROM (
|
|
|
|
|
SELECT UNNEST($1::text[]) AS client_id,
|
|
|
|
|
UNNEST($2::bigint[]) AS delta,
|
|
|
|
|
UNNEST($3::timestamptz[]) AS last_active
|
|
|
|
|
) AS v
|
|
|
|
|
WHERE c.client_id = v.client_id`,
|
|
|
|
|
[clientIds, deltas, lastActives],
|
|
|
|
|
);
|
|
|
|
|
return clientIds.length;
|
|
|
|
|
} catch {
|
|
|
|
|
// Re-merge deltas so transient DB failures do not lose event counts
|
|
|
|
|
for (const [cid, entry] of snapshot) {
|
|
|
|
|
const current = pendingDbClientActivity.get(cid);
|
|
|
|
|
if (current) {
|
|
|
|
|
pendingDbClientActivity.set(cid, {
|
|
|
|
|
count: current.count + entry.count,
|
|
|
|
|
lastActive:
|
|
|
|
|
current.lastActive > entry.lastActive
|
|
|
|
|
? current.lastActive
|
|
|
|
|
: entry.lastActive,
|
|
|
|
|
});
|
|
|
|
|
} else {
|
|
|
|
|
pendingDbClientActivity.set(cid, { ...entry });
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return 0;
|
|
|
|
|
} finally {
|
|
|
|
|
activeActivityFlushPromise = null;
|
|
|
|
|
}
|
|
|
|
|
})();
|
|
|
|
|
|
|
|
|
|
activeActivityFlushPromise = run;
|
|
|
|
|
return run;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function ensureActivityFlushTimer() {
|
|
|
|
|
if (activityFlushTimer) return;
|
|
|
|
|
activityFlushTimer = setInterval(() => {
|
|
|
|
|
void flushPendingClientActivity().catch(() => undefined);
|
|
|
|
|
}, CLIENT_ACTIVITY_FLUSH_INTERVAL_MS);
|
|
|
|
|
if (typeof activityFlushTimer.unref === 'function') {
|
|
|
|
|
activityFlushTimer.unref();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-17 20:05:00 +06:00
|
|
|
function recordClientActivity(clientId, count = 1) {
|
2026-11-18 05:18:55 +06:00
|
|
|
if (!clientId || count <= 0) return;
|
2026-11-19 20:22:01 +06:00
|
|
|
const normalizedId = normalizeClientId(clientId);
|
|
|
|
|
if (!normalizedId) return;
|
2026-11-17 20:05:00 +06:00
|
|
|
const nowIso = new Date().toISOString();
|
2026-11-18 05:18:55 +06:00
|
|
|
|
|
|
|
|
if (!dbPool) {
|
|
|
|
|
const prev = inMemoryClientActivity.get(normalizedId) || {
|
|
|
|
|
totalEvents: 0,
|
|
|
|
|
lastActive: nowIso,
|
|
|
|
|
};
|
|
|
|
|
inMemoryClientActivity.set(normalizedId, {
|
|
|
|
|
totalEvents: prev.totalEvents + count,
|
|
|
|
|
lastActive: nowIso,
|
|
|
|
|
});
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
const prevPending = pendingDbClientActivity.get(normalizedId);
|
|
|
|
|
if (prevPending) {
|
|
|
|
|
pendingDbClientActivity.set(normalizedId, {
|
|
|
|
|
count: prevPending.count + count,
|
|
|
|
|
lastActive: nowIso > prevPending.lastActive ? nowIso : prevPending.lastActive,
|
|
|
|
|
});
|
|
|
|
|
} else {
|
|
|
|
|
pendingDbClientActivity.set(normalizedId, {
|
|
|
|
|
count,
|
|
|
|
|
lastActive: nowIso,
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
ensureActivityFlushTimer();
|
2026-11-17 20:05:00 +06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let isFlushing = false;
|
2026-11-19 20:22:01 +06:00
|
|
|
let activeFlushPromise = null;
|
2026-11-17 20:05:00 +06:00
|
|
|
let flushTimer = null;
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
function purgeClientFromLocalBuffer(normalizedId) {
|
|
|
|
|
if (!normalizedId || bufferQueue.length === 0) return 0;
|
|
|
|
|
let removed = 0;
|
|
|
|
|
let writeIdx = 0;
|
|
|
|
|
for (let i = 0; i < bufferQueue.length; i += 1) {
|
|
|
|
|
const evt = bufferQueue[i];
|
|
|
|
|
if (evt && normalizeClientId(evt.clientId) === normalizedId) {
|
|
|
|
|
removed += 1;
|
|
|
|
|
} else {
|
|
|
|
|
bufferQueue[writeIdx] = evt;
|
|
|
|
|
writeIdx += 1;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
bufferQueue.length = writeIdx;
|
|
|
|
|
return removed;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function isClientDeletedForFlush(clientId, redis, cache) {
|
|
|
|
|
const normalized = normalizeClientId(clientId);
|
|
|
|
|
if (!normalized) return false;
|
|
|
|
|
if (deletedClientsSet.has(normalized)) return true;
|
|
|
|
|
if (cache && cache.has(normalized)) return Boolean(cache.get(normalized));
|
|
|
|
|
if (redis && typeof redis.sismember === 'function') {
|
|
|
|
|
try {
|
|
|
|
|
const isDeleted =
|
|
|
|
|
Number(await redis.sismember(REDIS_CLIENTS_DELETED_KEY, normalized)) === 1;
|
|
|
|
|
if (cache) cache.set(normalized, isDeleted);
|
|
|
|
|
if (isDeleted) deletedClientsSet.add(normalized);
|
|
|
|
|
return isDeleted;
|
|
|
|
|
} catch {
|
|
|
|
|
// Ignore transient Redis tombstone lookup error
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (cache) cache.set(normalized, false);
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function flushBuffer(options = {}) {
|
2026-11-22 15:33:14 +06:00
|
|
|
|
|
|
|
|
// P02 R-02: Redis and CSV cannot be committed atomically. Before replaying an
|
|
|
|
|
// untrimmed Redis batch, check actual CSV partitions for previously written IDs.
|
|
|
|
|
// Streaming parsing bounds memory and preserves multiline/quoted CSV fields.
|
|
|
|
|
const excludeAlreadyPersisted = async (events) => {
|
|
|
|
|
if (!events.length) return events;
|
|
|
|
|
const fs = require('node:fs');
|
|
|
|
|
const path = require('node:path');
|
|
|
|
|
const zlib = require('node:zlib');
|
|
|
|
|
const { pipeline } = require('node:stream/promises');
|
|
|
|
|
const { Writable } = require('node:stream');
|
|
|
|
|
const { parse } = require('csv-parse');
|
|
|
|
|
const groups = new Map();
|
|
|
|
|
for (const event of events) {
|
|
|
|
|
const clientId = String(event.clientId || 'unknown');
|
|
|
|
|
const date = writer.extractSafePartitionDate(event.timestamp);
|
|
|
|
|
const key = JSON.stringify([clientId, date]);
|
|
|
|
|
if (!groups.has(key)) groups.set(key, { clientId, date, ids: new Set() });
|
|
|
|
|
groups.get(key).ids.add(String(event.eventId));
|
|
|
|
|
}
|
|
|
|
|
const persisted = new Set();
|
|
|
|
|
for (const [groupKey, group] of groups) {
|
|
|
|
|
const dir = writer.ensureClientDir(group.clientId);
|
|
|
|
|
const entries = await fs.promises.readdir(dir, { withFileTypes: true });
|
|
|
|
|
for (const entry of entries) {
|
|
|
|
|
if (!entry.isFile() || !writer.isPartitionFilename(entry.name) ||
|
|
|
|
|
writer.extractDateFromFilename(entry.name) !== group.date) continue;
|
|
|
|
|
const reader = fs.createReadStream(path.join(dir, entry.name));
|
|
|
|
|
const streams = [reader];
|
|
|
|
|
if (entry.name.endsWith('.gz')) streams.push(zlib.createGunzip());
|
|
|
|
|
streams.push(parse({ columns: true, bom: true, skip_empty_lines: true }));
|
|
|
|
|
streams.push(new Writable({
|
|
|
|
|
objectMode: true,
|
|
|
|
|
write(row, _enc, done) {
|
|
|
|
|
if (row.clientId === group.clientId && group.ids.has(String(row.eventId))) {
|
|
|
|
|
persisted.add(JSON.stringify([groupKey, String(row.eventId)]));
|
|
|
|
|
}
|
|
|
|
|
done();
|
|
|
|
|
},
|
|
|
|
|
}));
|
|
|
|
|
// On read/parse failure, abort without LTRIM: never discard uncertain rows.
|
|
|
|
|
await pipeline(...streams);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
const batchSeen = new Set();
|
|
|
|
|
return events.filter((event) => {
|
|
|
|
|
const key = JSON.stringify([String(event.clientId || 'unknown'),
|
|
|
|
|
writer.extractSafePartitionDate(event.timestamp)]);
|
|
|
|
|
const rowKey = JSON.stringify([key, String(event.eventId)]);
|
|
|
|
|
if (persisted.has(rowKey) || batchSeen.has(rowKey)) return false;
|
|
|
|
|
batchSeen.add(rowKey);
|
|
|
|
|
return true;
|
|
|
|
|
});
|
|
|
|
|
};
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
if (pendingRedisWrites.size > 0) {
|
|
|
|
|
await Promise.allSettled([...pendingRedisWrites]);
|
|
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
if (activeFlushPromise) {
|
|
|
|
|
return activeFlushPromise;
|
2026-11-17 20:05:00 +06:00
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
const executeFlush = async () => {
|
|
|
|
|
isFlushing = true;
|
|
|
|
|
let totalWritten = 0;
|
|
|
|
|
let lockTimedOut = false;
|
|
|
|
|
const filesSet = new Set();
|
|
|
|
|
const deletedCheckCache = new Map();
|
2026-11-18 05:18:55 +06:00
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
try {
|
|
|
|
|
if (pendingRedisWrites.size > 0) {
|
|
|
|
|
await Promise.allSettled([...pendingRedisWrites]);
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
const redis = getRedisClient();
|
|
|
|
|
if (redis) {
|
|
|
|
|
const lockToken = crypto.randomUUID();
|
|
|
|
|
const lockTtlMs = Math.max(50, Number(options && options.lockTtlMs) || FLUSH_LOCK_TTL_MS);
|
|
|
|
|
const waitMs = Math.max(
|
|
|
|
|
0,
|
|
|
|
|
options && options.waitMs !== undefined ? Number(options.waitMs) : FLUSH_LOCK_WAIT_MS,
|
|
|
|
|
);
|
|
|
|
|
const deadline = Date.now() + waitMs;
|
|
|
|
|
let lockAcquired = false;
|
|
|
|
|
let redisError = false;
|
|
|
|
|
|
|
|
|
|
while (true) {
|
|
|
|
|
try {
|
|
|
|
|
const lockResult = await redis.set(
|
|
|
|
|
REDIS_FLUSH_LOCK_KEY,
|
|
|
|
|
lockToken,
|
|
|
|
|
'PX',
|
|
|
|
|
lockTtlMs,
|
|
|
|
|
'NX',
|
|
|
|
|
);
|
|
|
|
|
redisConnected = true;
|
|
|
|
|
if (lockResult === 'OK') {
|
|
|
|
|
lockAcquired = true;
|
2026-11-18 05:18:55 +06:00
|
|
|
break;
|
|
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
} catch {
|
|
|
|
|
redisConnected = false;
|
|
|
|
|
redisError = true;
|
|
|
|
|
break;
|
|
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
const remaining = deadline - Date.now();
|
|
|
|
|
if (remaining <= 0) {
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
await sleepMs(Math.min(FLUSH_LOCK_RETRY_MS, remaining));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!lockAcquired && !redisError) {
|
|
|
|
|
lockTimedOut = true;
|
|
|
|
|
if (options && options.throwOnLockTimeout === true) {
|
|
|
|
|
throw new Error(
|
|
|
|
|
'FLUSH_LOCK_TIMEOUT: Timed out waiting for distributed Redis flush lock',
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (lockAcquired) {
|
|
|
|
|
const renewIntervalMs = Math.max(20, Math.floor(lockTtlMs / 3));
|
|
|
|
|
let lastRenewAt = Date.now();
|
|
|
|
|
let lockLost = false;
|
|
|
|
|
let renewInFlight = null;
|
|
|
|
|
|
|
|
|
|
const renewLockOnce = () => {
|
|
|
|
|
if (renewInFlight) return renewInFlight;
|
|
|
|
|
renewInFlight = (async () => {
|
2026-11-18 05:18:55 +06:00
|
|
|
try {
|
2026-11-19 20:22:01 +06:00
|
|
|
const renewed = await redis.eval(
|
|
|
|
|
RENEW_LOCK_LUA_SCRIPT,
|
|
|
|
|
1,
|
|
|
|
|
REDIS_FLUSH_LOCK_KEY,
|
|
|
|
|
lockToken,
|
|
|
|
|
String(lockTtlMs),
|
|
|
|
|
);
|
|
|
|
|
if (Number(renewed) === 1) {
|
|
|
|
|
lastRenewAt = Date.now();
|
|
|
|
|
return true;
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
lockLost = true;
|
|
|
|
|
return false;
|
2026-11-18 05:18:55 +06:00
|
|
|
} catch {
|
2026-11-19 20:22:01 +06:00
|
|
|
return false;
|
|
|
|
|
} finally {
|
|
|
|
|
renewInFlight = null;
|
|
|
|
|
}
|
|
|
|
|
})();
|
|
|
|
|
return renewInFlight;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const renewTimer = setInterval(() => {
|
|
|
|
|
void renewLockOnce();
|
|
|
|
|
}, renewIntervalMs);
|
|
|
|
|
if (typeof renewTimer.unref === 'function') {
|
|
|
|
|
renewTimer.unref();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
const chunkSize = Math.max(500, runtimeConfig.batchFlushThreshold);
|
|
|
|
|
while (true) {
|
|
|
|
|
if (lockLost) break;
|
|
|
|
|
|
|
|
|
|
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, chunkSize - 1);
|
|
|
|
|
if (!rawItems || rawItems.length === 0) {
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const parsedBatch = [];
|
|
|
|
|
let malformedCount = 0;
|
|
|
|
|
for (const raw of rawItems) {
|
|
|
|
|
try {
|
|
|
|
|
const evt = JSON.parse(raw);
|
|
|
|
|
if (evt && evt.eventId) {
|
|
|
|
|
if (await isClientDeletedForFlush(evt.clientId, redis, deletedCheckCache)) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
parsedBatch.push(evt);
|
|
|
|
|
} else {
|
|
|
|
|
malformedCount += 1;
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
malformedCount += 1;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (malformedCount > 0) {
|
|
|
|
|
console.warn(
|
|
|
|
|
`[CSV Worker] Skipped ${malformedCount} malformed buffered Redis event(s) during flush`,
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (Date.now() - lastRenewAt >= renewIntervalMs) {
|
|
|
|
|
await renewLockOnce();
|
|
|
|
|
}
|
|
|
|
|
if (lockLost) break;
|
|
|
|
|
|
|
|
|
|
if (parsedBatch.length > 0) {
|
2026-11-22 15:33:14 +06:00
|
|
|
const unwritten = await excludeAlreadyPersisted(parsedBatch);
|
|
|
|
|
if (unwritten.length > 0) {
|
|
|
|
|
const res = await writer.writeBatch(unwritten);
|
|
|
|
|
totalWritten += res.written;
|
|
|
|
|
for (const f of res.files) filesSet.add(f);
|
|
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (Date.now() - lastRenewAt >= renewIntervalMs) {
|
|
|
|
|
await renewLockOnce();
|
|
|
|
|
}
|
|
|
|
|
if (lockLost) break;
|
|
|
|
|
|
|
|
|
|
// Trim ONLY after CSV batch write succeeds and lock ownership holds
|
|
|
|
|
await redis.ltrim(REDIS_BUFFER_KEY, rawItems.length, -1);
|
|
|
|
|
|
|
|
|
|
if (rawItems.length < chunkSize) {
|
|
|
|
|
break;
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
|
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
} finally {
|
|
|
|
|
clearInterval(renewTimer);
|
|
|
|
|
if (renewInFlight) {
|
|
|
|
|
await renewInFlight.catch(() => false);
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
try {
|
|
|
|
|
await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
|
|
|
|
|
} catch {
|
|
|
|
|
// Lock expires automatically via PX TTL
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
if (bufferQueue.length > 0) {
|
|
|
|
|
const batch = bufferQueue.splice(0, bufferQueue.length);
|
|
|
|
|
const validBatch = [];
|
|
|
|
|
for (const evt of batch) {
|
|
|
|
|
if (await isClientDeletedForFlush(evt.clientId, redis, deletedCheckCache)) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
validBatch.push(evt);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (validBatch.length > 0) {
|
|
|
|
|
try {
|
2026-11-22 15:33:14 +06:00
|
|
|
const unwritten = await excludeAlreadyPersisted(validBatch);
|
|
|
|
|
if (unwritten.length > 0) {
|
|
|
|
|
const res = await writer.writeBatch(unwritten);
|
|
|
|
|
totalWritten += res.written;
|
|
|
|
|
for (const f of res.files) filesSet.add(f);
|
|
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
} catch (err) {
|
|
|
|
|
bufferQueue.unshift(...validBatch);
|
|
|
|
|
throw err;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const files = Array.from(filesSet);
|
|
|
|
|
if (totalWritten > 0) {
|
|
|
|
|
console.log(
|
|
|
|
|
`[CSV Worker] Flushed ${totalWritten} events to ${files.length} file(s)`,
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
return {
|
|
|
|
|
written: totalWritten,
|
|
|
|
|
files,
|
|
|
|
|
...(lockTimedOut ? { lockTimedOut: true } : {}),
|
|
|
|
|
};
|
|
|
|
|
} catch (err) {
|
|
|
|
|
console.error('[CSV Worker] Flush error:', err);
|
|
|
|
|
throw err;
|
|
|
|
|
} finally {
|
|
|
|
|
isFlushing = false;
|
|
|
|
|
activeFlushPromise = null;
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
let flushDone = false;
|
|
|
|
|
const promise = (async () => {
|
|
|
|
|
try {
|
|
|
|
|
return await executeFlush();
|
|
|
|
|
} finally {
|
|
|
|
|
flushDone = true;
|
|
|
|
|
isFlushing = false;
|
|
|
|
|
if (activeFlushPromise === promise) {
|
|
|
|
|
activeFlushPromise = null;
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
|
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
})();
|
|
|
|
|
if (!flushDone) {
|
|
|
|
|
activeFlushPromise = promise;
|
2026-10-23 11:43:49 -05:00
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
return promise;
|
2026-10-23 11:43:49 -05:00
|
|
|
}
|
|
|
|
|
|
2026-11-17 20:05:00 +06:00
|
|
|
function scheduleFlushTimer(intervalMs) {
|
|
|
|
|
const safeInterval = Math.max(100, Number(intervalMs) || 2000);
|
|
|
|
|
runtimeConfig.batchFlushIntervalMs = safeInterval;
|
|
|
|
|
if (flushTimer) {
|
|
|
|
|
clearInterval(flushTimer);
|
|
|
|
|
}
|
|
|
|
|
flushTimer = setInterval(() => {
|
2026-11-19 20:22:01 +06:00
|
|
|
void flushBuffer().catch(() => undefined);
|
2026-11-17 20:05:00 +06:00
|
|
|
}, safeInterval);
|
|
|
|
|
}
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-11-17 20:05:00 +06:00
|
|
|
scheduleFlushTimer(runtimeConfig.batchFlushIntervalMs);
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
const server = http.createServer(app);
|
|
|
|
|
const io = new Server(server, {
|
2026-11-19 20:22:01 +06:00
|
|
|
cors: { origin: corsOriginDelegate, methods: ['GET', 'POST'] },
|
2026-11-18 05:18:55 +06:00
|
|
|
});
|
|
|
|
|
|
|
|
|
|
function applyRuntimeSettingsObject(settingsObj) {
|
|
|
|
|
if (!settingsObj || typeof settingsObj !== 'object') {
|
|
|
|
|
return { jwtRotated: false };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (settingsObj.maxEventsPerSec !== undefined) {
|
|
|
|
|
const val = Number(settingsObj.maxEventsPerSec);
|
|
|
|
|
if (Number.isFinite(val) && val >= 1) {
|
|
|
|
|
runtimeConfig.maxEventsPerSec = Math.floor(val);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (settingsObj.maxPayloadSizeKb !== undefined) {
|
|
|
|
|
const kb = Number(settingsObj.maxPayloadSizeKb);
|
|
|
|
|
if (Number.isFinite(kb) && kb >= 1) {
|
|
|
|
|
runtimeConfig.maxPayloadSizeBytes = Math.floor(kb) * 1024;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (settingsObj.batchFlushThreshold !== undefined) {
|
|
|
|
|
const threshold = Number(settingsObj.batchFlushThreshold);
|
|
|
|
|
if (Number.isFinite(threshold) && threshold >= 1) {
|
|
|
|
|
runtimeConfig.batchFlushThreshold = Math.floor(threshold);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (settingsObj.enableIdempotency !== undefined) {
|
|
|
|
|
runtimeConfig.enableIdempotency = Boolean(settingsObj.enableIdempotency);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (settingsObj.fileRotationMb !== undefined) {
|
|
|
|
|
const nextRotationMb = Number(settingsObj.fileRotationMb);
|
|
|
|
|
if (
|
|
|
|
|
Number.isFinite(nextRotationMb) &&
|
|
|
|
|
nextRotationMb >= 1 &&
|
|
|
|
|
nextRotationMb !== runtimeConfig.fileRotationMb
|
|
|
|
|
) {
|
|
|
|
|
runtimeConfig.fileRotationMb = nextRotationMb;
|
|
|
|
|
writer.setMaxFileSizeMb(nextRotationMb);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 23:02:27 +06:00
|
|
|
if (settingsObj.enableS3Backup !== undefined) {
|
|
|
|
|
runtimeConfig.enableS3Backup = Boolean(settingsObj.enableS3Backup);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (
|
|
|
|
|
settingsObj.diskPauseThresholdPct !== undefined &&
|
|
|
|
|
settingsObj.diskPauseThresholdPct !== null
|
|
|
|
|
) {
|
|
|
|
|
const pct = Number(settingsObj.diskPauseThresholdPct);
|
|
|
|
|
if (Number.isFinite(pct) && (pct === 0 || (pct >= 50 && pct <= 99))) {
|
|
|
|
|
runtimeConfig.diskPauseThresholdPct = Math.floor(pct);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (
|
|
|
|
|
settingsObj.compressAfterDays !== undefined &&
|
|
|
|
|
settingsObj.compressAfterDays !== null
|
|
|
|
|
) {
|
|
|
|
|
const parsedCompress = Number(settingsObj.compressAfterDays);
|
|
|
|
|
if (Number.isFinite(parsedCompress) && parsedCompress >= 0) {
|
|
|
|
|
const nextCompressDays = Math.floor(parsedCompress);
|
|
|
|
|
const changed = nextCompressDays !== runtimeConfig.compressAfterDays;
|
|
|
|
|
runtimeConfig.compressAfterDays = nextCompressDays;
|
2026-11-20 18:48:54 +06:00
|
|
|
if (changed && nextCompressDays > 0 && isDbReadyForMaintenance()) {
|
2026-11-19 23:02:27 +06:00
|
|
|
void writer.compressOldPartitionsAsync(nextCompressDays).catch(() => undefined);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
if (settingsObj.autoPruneDays !== undefined && settingsObj.autoPruneDays !== null) {
|
|
|
|
|
const parsedPrune = Number(settingsObj.autoPruneDays);
|
|
|
|
|
if (Number.isFinite(parsedPrune) && parsedPrune >= 0) {
|
|
|
|
|
const nextPruneDays = Math.floor(parsedPrune);
|
|
|
|
|
const changed = nextPruneDays !== runtimeConfig.autoPruneDays;
|
|
|
|
|
runtimeConfig.autoPruneDays = nextPruneDays;
|
2026-11-20 18:48:54 +06:00
|
|
|
if (
|
|
|
|
|
changed &&
|
|
|
|
|
nextPruneDays > 0 &&
|
|
|
|
|
!runtimeConfig.enableS3Backup &&
|
|
|
|
|
isDbReadyForMaintenance()
|
|
|
|
|
) {
|
2026-11-19 20:22:01 +06:00
|
|
|
void writer.pruneExpiredPartitionsAsync(nextPruneDays).catch(() => undefined);
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (settingsObj.batchFlushIntervalMs !== undefined) {
|
|
|
|
|
const nextIntervalMs = Math.max(
|
|
|
|
|
100,
|
|
|
|
|
Number(settingsObj.batchFlushIntervalMs) || 2000,
|
|
|
|
|
);
|
|
|
|
|
if (nextIntervalMs !== runtimeConfig.batchFlushIntervalMs) {
|
|
|
|
|
scheduleFlushTimer(nextIntervalMs);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-21 00:06:55 +06:00
|
|
|
if (settingsObj.enableWebhookAlerts !== undefined) {
|
|
|
|
|
runtimeConfig.enableWebhookAlerts = Boolean(settingsObj.enableWebhookAlerts);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (settingsObj.webhookUrl !== undefined && settingsObj.webhookUrl !== null) {
|
|
|
|
|
runtimeConfig.webhookUrl = String(settingsObj.webhookUrl).trim();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (settingsObj.webhookSecret !== undefined && settingsObj.webhookSecret !== null) {
|
|
|
|
|
runtimeConfig.webhookSecret = String(settingsObj.webhookSecret);
|
|
|
|
|
} else if (
|
|
|
|
|
settingsObj.webhookSecretEnc !== undefined &&
|
|
|
|
|
settingsObj.webhookSecretEnc !== null
|
|
|
|
|
) {
|
|
|
|
|
runtimeConfig.webhookSecret = decryptVaultSecret(String(settingsObj.webhookSecretEnc));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (
|
|
|
|
|
settingsObj.webhookCooldownSec !== undefined &&
|
|
|
|
|
settingsObj.webhookCooldownSec !== null
|
|
|
|
|
) {
|
|
|
|
|
const cooldown = Number(settingsObj.webhookCooldownSec);
|
|
|
|
|
if (Number.isFinite(cooldown) && cooldown >= 0) {
|
|
|
|
|
runtimeConfig.webhookCooldownSec = Math.floor(cooldown);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
let jwtRotated = false;
|
|
|
|
|
if (settingsObj.jwtSecret !== undefined && settingsObj.jwtSecret !== null) {
|
|
|
|
|
const candidateSecret = String(settingsObj.jwtSecret).trim();
|
|
|
|
|
const validSecret =
|
|
|
|
|
candidateSecret.length >= 32 ||
|
|
|
|
|
(!IS_PRODUCTION && candidateSecret.length > 0);
|
|
|
|
|
if (validSecret && candidateSecret !== runtimeConfig.jwtSecret) {
|
2026-11-19 20:22:01 +06:00
|
|
|
if (runtimeConfig.jwtSecret) {
|
|
|
|
|
previousJwtSecret = runtimeConfig.jwtSecret;
|
|
|
|
|
previousJwtSecretGraceUntil = Date.now() + JWT_ROTATION_BRIDGE_GRACE_MS;
|
|
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
runtimeConfig.jwtSecret = candidateSecret;
|
|
|
|
|
clientAuthCache.clear();
|
|
|
|
|
io.disconnectSockets(true);
|
|
|
|
|
jwtRotated = true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return { jwtRotated };
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-21 00:06:55 +06:00
|
|
|
const VAULT_PREFIX = 'v1';
|
|
|
|
|
const DEV_DEFAULT_SESSION_SECRET = 'datastack-admin-session-default-secret-32-chars';
|
|
|
|
|
const webhookCooldownMap = new Map();
|
|
|
|
|
|
|
|
|
|
function clearWebhookCooldownCacheForTests() {
|
|
|
|
|
webhookCooldownMap.clear();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function resolveVaultMasterSecret() {
|
|
|
|
|
const candidate = (process.env.DATASTACK_ADMIN_SESSION_SECRET || '').trim();
|
|
|
|
|
if (process.env.NODE_ENV === 'production') {
|
|
|
|
|
return candidate.length >= 32 ? candidate : null;
|
|
|
|
|
}
|
|
|
|
|
if (candidate.length > 0) {
|
|
|
|
|
return candidate.length >= 32 ? candidate : null;
|
|
|
|
|
}
|
|
|
|
|
return DEV_DEFAULT_SESSION_SECRET;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function decryptVaultSecret(ciphertext) {
|
|
|
|
|
if (!ciphertext || typeof ciphertext !== 'string') return '';
|
|
|
|
|
const trimmed = ciphertext.trim();
|
|
|
|
|
if (!trimmed) return '';
|
|
|
|
|
const parts = trimmed.split(':');
|
|
|
|
|
if (parts.length !== 4 || parts[0] !== VAULT_PREFIX) {
|
|
|
|
|
console.warn(
|
|
|
|
|
'[DataStack Vault] Ignoring unrecognized stored webhook secret format on Port 4000.',
|
|
|
|
|
);
|
|
|
|
|
return '';
|
|
|
|
|
}
|
|
|
|
|
const master = resolveVaultMasterSecret();
|
|
|
|
|
if (!master) {
|
|
|
|
|
console.warn(
|
|
|
|
|
'[DataStack Vault] DATASTACK_ADMIN_SESSION_SECRET (>= 32 chars) unavailable on Port 4000; skipping webhook secret decryption.',
|
|
|
|
|
);
|
|
|
|
|
return '';
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
const derived = crypto.hkdfSync(
|
|
|
|
|
'sha256',
|
|
|
|
|
Buffer.from(master, 'utf8'),
|
|
|
|
|
Buffer.from('datastack-vault-salt-v1', 'utf8'),
|
|
|
|
|
Buffer.from('datastack-aes-256-gcm-v1', 'utf8'),
|
|
|
|
|
32,
|
|
|
|
|
);
|
|
|
|
|
const key = Buffer.from(derived);
|
|
|
|
|
const iv = Buffer.from(parts[1], 'base64url');
|
|
|
|
|
const authTag = Buffer.from(parts[2], 'base64url');
|
|
|
|
|
const encrypted = Buffer.from(parts[3], 'base64url');
|
|
|
|
|
if (iv.length !== 12 || authTag.length !== 16) {
|
|
|
|
|
return '';
|
|
|
|
|
}
|
|
|
|
|
const decipher = crypto.createDecipheriv('aes-256-gcm', key, iv, {
|
|
|
|
|
authTagLength: 16,
|
|
|
|
|
});
|
|
|
|
|
decipher.setAuthTag(authTag);
|
|
|
|
|
const decrypted = Buffer.concat([decipher.update(encrypted), decipher.final()]);
|
|
|
|
|
return decrypted.toString('utf8');
|
|
|
|
|
} catch {
|
|
|
|
|
console.warn(
|
|
|
|
|
'[DataStack Vault] Failed to decrypt stored webhook secret on Port 4000. Verify DATASTACK_ADMIN_SESSION_SECRET matches Port 3000.',
|
|
|
|
|
);
|
|
|
|
|
return '';
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function formatOutboundWebhookBody(_url, payload, timestamp) {
|
|
|
|
|
const safeTimestamp = timestamp || payload.timestamp || new Date().toISOString();
|
|
|
|
|
const severity = payload.severity || 'warning';
|
|
|
|
|
const prefix =
|
|
|
|
|
severity === 'critical'
|
|
|
|
|
? '[CRITICAL]'
|
|
|
|
|
: severity === 'warning'
|
|
|
|
|
? '[WARNING]'
|
|
|
|
|
: '[INFO]';
|
|
|
|
|
const title = payload.title || String(payload.event || 'Alert');
|
|
|
|
|
const clientTag = payload.clientId ? ` (client: ${payload.clientId})` : '';
|
|
|
|
|
const summaryLine = `${prefix} DataStack ${payload.event}${clientTag} — ${title}: ${payload.message}`;
|
|
|
|
|
|
|
|
|
|
return JSON.stringify({
|
|
|
|
|
instance: 'DataStack',
|
|
|
|
|
event: payload.event,
|
|
|
|
|
severity,
|
|
|
|
|
title,
|
|
|
|
|
message: payload.message,
|
|
|
|
|
...(payload.clientId ? { clientId: payload.clientId } : {}),
|
|
|
|
|
...(payload.details ? { details: payload.details } : {}),
|
|
|
|
|
timestamp: safeTimestamp,
|
|
|
|
|
// Slack & Discord compatible top-level fields (1:1 parity with Port 3000)
|
|
|
|
|
text: summaryLine,
|
|
|
|
|
content: summaryLine,
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-21 22:15:32 +06:00
|
|
|
const WEBHOOK_ALWAYS_BLOCKED_HOSTNAMES = new Set([
|
|
|
|
|
'metadata.google.internal',
|
|
|
|
|
'metadata.goog',
|
|
|
|
|
'metadata',
|
|
|
|
|
'instance-data',
|
|
|
|
|
'instance-data.ec2.internal',
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
const WEBHOOK_PRIVATE_HOSTNAMES = new Set([
|
|
|
|
|
'localhost',
|
|
|
|
|
'localhost.localdomain',
|
|
|
|
|
'ip6-localhost',
|
|
|
|
|
'ip6-loopback',
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
const WEBHOOK_ALWAYS_BLOCKED_IPS = (() => {
|
|
|
|
|
const list = new net.BlockList();
|
|
|
|
|
list.addSubnet('0.0.0.0', 8, 'ipv4');
|
|
|
|
|
list.addSubnet('169.254.0.0', 16, 'ipv4');
|
|
|
|
|
list.addSubnet('224.0.0.0', 4, 'ipv4');
|
|
|
|
|
list.addSubnet('240.0.0.0', 4, 'ipv4');
|
|
|
|
|
list.addAddress('::', 'ipv6');
|
|
|
|
|
list.addSubnet('fe80::', 10, 'ipv6');
|
|
|
|
|
list.addSubnet('ff00::', 8, 'ipv6');
|
|
|
|
|
return list;
|
|
|
|
|
})();
|
|
|
|
|
|
|
|
|
|
const WEBHOOK_PRIVATE_IPS = (() => {
|
|
|
|
|
const list = new net.BlockList();
|
|
|
|
|
list.addSubnet('127.0.0.0', 8, 'ipv4');
|
|
|
|
|
list.addSubnet('10.0.0.0', 8, 'ipv4');
|
|
|
|
|
list.addSubnet('172.16.0.0', 12, 'ipv4');
|
|
|
|
|
list.addSubnet('192.168.0.0', 16, 'ipv4');
|
|
|
|
|
list.addSubnet('100.64.0.0', 10, 'ipv4');
|
|
|
|
|
list.addSubnet('198.18.0.0', 15, 'ipv4');
|
|
|
|
|
list.addAddress('::1', 'ipv6');
|
|
|
|
|
list.addSubnet('fc00::', 7, 'ipv6');
|
|
|
|
|
return list;
|
|
|
|
|
})();
|
|
|
|
|
|
|
|
|
|
function isPrivateWebhookUrlAllowed(env = process.env) {
|
|
|
|
|
const raw = String(env.DATASTACK_ALLOW_PRIVATE_URLS || '')
|
|
|
|
|
.trim()
|
|
|
|
|
.toLowerCase();
|
|
|
|
|
return raw === 'true' || raw === '1' || raw === 'yes';
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function normalizeWebhookIp(rawHost) {
|
|
|
|
|
let clean = String(rawHost || '').trim().toLowerCase();
|
|
|
|
|
if (clean.startsWith('[') && clean.endsWith(']')) {
|
|
|
|
|
clean = clean.slice(1, -1);
|
|
|
|
|
}
|
|
|
|
|
const zoneIdx = clean.indexOf('%');
|
|
|
|
|
if (zoneIdx >= 0) clean = clean.slice(0, zoneIdx);
|
|
|
|
|
clean = clean.replace(/\.+$/, '');
|
|
|
|
|
if (!clean) return null;
|
|
|
|
|
|
|
|
|
|
if (clean.startsWith('::ffff:')) {
|
|
|
|
|
const mapped = clean.slice('::ffff:'.length);
|
|
|
|
|
if (net.isIP(mapped) === 4) {
|
|
|
|
|
return { ip: mapped, family: 4 };
|
|
|
|
|
}
|
|
|
|
|
const hexParts = mapped.split(':');
|
|
|
|
|
if (hexParts.length === 2) {
|
|
|
|
|
const hi = Number.parseInt(hexParts[0], 16);
|
|
|
|
|
const lo = Number.parseInt(hexParts[1], 16);
|
|
|
|
|
if (
|
|
|
|
|
Number.isInteger(hi) &&
|
|
|
|
|
Number.isInteger(lo) &&
|
|
|
|
|
hi >= 0 &&
|
|
|
|
|
hi <= 0xffff &&
|
|
|
|
|
lo >= 0 &&
|
|
|
|
|
lo <= 0xffff
|
|
|
|
|
) {
|
|
|
|
|
const ipv4 = `${(hi >> 8) & 0xff}.${hi & 0xff}.${(lo >> 8) & 0xff}.${lo & 0xff}`;
|
|
|
|
|
if (net.isIP(ipv4) === 4) {
|
|
|
|
|
return { ip: ipv4, family: 4 };
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const family = net.isIP(clean);
|
|
|
|
|
if (family === 4 || family === 6) {
|
|
|
|
|
return { ip: clean, family };
|
|
|
|
|
}
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function checkWebhookIpAllowed(rawIp, allowPrivate) {
|
|
|
|
|
const parsed = normalizeWebhookIp(rawIp);
|
|
|
|
|
if (!parsed) return { allowed: false, reason: 'Invalid IP address' };
|
|
|
|
|
const ipType = parsed.family === 4 ? 'ipv4' : 'ipv6';
|
|
|
|
|
if (WEBHOOK_ALWAYS_BLOCKED_IPS.check(parsed.ip, ipType)) {
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
|
|
|
|
reason: `Address '${parsed.ip}' is in a restricted link-local/metadata/reserved range`,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
if (!allowPrivate && WEBHOOK_PRIVATE_IPS.check(parsed.ip, ipType)) {
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
|
|
|
|
reason: `Address '${parsed.ip}' is a private or loopback network address`,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
return { allowed: true };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function validateOutboundWebhookUrlAsync(rawUrl, options = {}) {
|
|
|
|
|
const allowPrivate =
|
|
|
|
|
options.allowPrivate !== undefined
|
|
|
|
|
? Boolean(options.allowPrivate)
|
|
|
|
|
: isPrivateWebhookUrlAllowed(options.env || process.env);
|
|
|
|
|
|
|
|
|
|
let parsed;
|
|
|
|
|
try {
|
|
|
|
|
parsed = new URL(String(rawUrl || '').trim());
|
|
|
|
|
} catch {
|
|
|
|
|
return { allowed: false, code: 'INVALID_URL', error: 'Webhook URL is not a valid URL.' };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
|
|
|
|
code: 'SSRF_BLOCKED_HOST',
|
|
|
|
|
error: 'Webhook URL must use http:// or https://.',
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
if (parsed.username || parsed.password) {
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
|
|
|
|
code: 'SSRF_BLOCKED_HOST',
|
|
|
|
|
error: 'Webhook URL must not embed credentials.',
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let host = String(parsed.hostname || '').trim().toLowerCase();
|
|
|
|
|
if (host.startsWith('[') && host.endsWith(']')) host = host.slice(1, -1);
|
|
|
|
|
host = host.replace(/\.+$/, '');
|
|
|
|
|
if (!host) {
|
|
|
|
|
return { allowed: false, code: 'INVALID_URL', error: 'Webhook URL hostname is empty.' };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (WEBHOOK_ALWAYS_BLOCKED_HOSTNAMES.has(host)) {
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
|
|
|
|
code: 'SSRF_BLOCKED_HOST',
|
|
|
|
|
error: `Webhook target '${host}' is blocked (cloud metadata/internal endpoint).`,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const literalIp = normalizeWebhookIp(host);
|
|
|
|
|
if (literalIp) {
|
|
|
|
|
const ipCheck = checkWebhookIpAllowed(host, allowPrivate);
|
|
|
|
|
if (!ipCheck.allowed) {
|
|
|
|
|
return { allowed: false, code: 'SSRF_BLOCKED_HOST', error: ipCheck.reason };
|
|
|
|
|
}
|
|
|
|
|
return { allowed: true, url: parsed.toString() };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!allowPrivate) {
|
|
|
|
|
if (
|
|
|
|
|
WEBHOOK_PRIVATE_HOSTNAMES.has(host) ||
|
|
|
|
|
host.endsWith('.localhost') ||
|
|
|
|
|
host.endsWith('.local') ||
|
|
|
|
|
host.endsWith('.internal') ||
|
|
|
|
|
!host.includes('.')
|
|
|
|
|
) {
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
|
|
|
|
code: 'SSRF_BLOCKED_HOST',
|
|
|
|
|
error: `Webhook target '${host}' is a private/internal hostname.`,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
const lookupFn =
|
|
|
|
|
options.dnsLookup ||
|
|
|
|
|
((h) => dnsPromises.lookup(h, { all: true, verbatim: true }));
|
|
|
|
|
const records = await lookupFn(host);
|
|
|
|
|
for (const rec of records) {
|
|
|
|
|
const check = checkWebhookIpAllowed(rec.address, allowPrivate);
|
|
|
|
|
if (!check.allowed) {
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
|
|
|
|
code: 'SSRF_BLOCKED_HOST',
|
|
|
|
|
error: `Webhook hostname '${host}' resolved to blocked address: ${check.reason}`,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
// Let fetch handle unresolved DNS names cleanly
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return { allowed: true, url: parsed.toString() };
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-21 00:06:55 +06:00
|
|
|
async function triggerWebhookAlert(payload, options = {}) {
|
|
|
|
|
try {
|
|
|
|
|
const enabled =
|
|
|
|
|
options.enableWebhookAlerts !== undefined
|
|
|
|
|
? Boolean(options.enableWebhookAlerts)
|
|
|
|
|
: Boolean(runtimeConfig.enableWebhookAlerts);
|
|
|
|
|
const url = (
|
|
|
|
|
options.webhookUrl !== undefined
|
|
|
|
|
? String(options.webhookUrl)
|
|
|
|
|
: String(runtimeConfig.webhookUrl || '')
|
|
|
|
|
).trim();
|
|
|
|
|
|
|
|
|
|
if (!enabled || !url) {
|
|
|
|
|
return { dispatched: false, reason: 'disabled_or_empty_url' };
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-21 22:15:32 +06:00
|
|
|
const ssrfCheck = await validateOutboundWebhookUrlAsync(url, {
|
|
|
|
|
allowPrivate: options.allowPrivate,
|
|
|
|
|
dnsLookup: options.dnsLookup,
|
|
|
|
|
});
|
|
|
|
|
if (!ssrfCheck.allowed) {
|
|
|
|
|
return {
|
|
|
|
|
dispatched: false,
|
|
|
|
|
code: ssrfCheck.code || 'SSRF_BLOCKED_HOST',
|
|
|
|
|
reason: ssrfCheck.code || 'SSRF_BLOCKED_HOST',
|
|
|
|
|
error: ssrfCheck.error,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-21 00:06:55 +06:00
|
|
|
const cooldownSec = Math.max(
|
|
|
|
|
0,
|
|
|
|
|
Number(
|
|
|
|
|
options.webhookCooldownSec !== undefined
|
|
|
|
|
? options.webhookCooldownSec
|
|
|
|
|
: runtimeConfig.webhookCooldownSec,
|
|
|
|
|
) || 0,
|
|
|
|
|
);
|
|
|
|
|
const dedupKey = `${payload.event}:${payload.clientId || 'global'}`;
|
|
|
|
|
const now = Date.now();
|
|
|
|
|
const lastFired = webhookCooldownMap.get(dedupKey) || 0;
|
|
|
|
|
|
|
|
|
|
if (!options.bypassCooldown && cooldownSec > 0 && now - lastFired < cooldownSec * 1000) {
|
|
|
|
|
return { dispatched: false, reason: 'cooldown_active' };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
webhookCooldownMap.set(dedupKey, now);
|
|
|
|
|
|
|
|
|
|
const timestamp = payload.timestamp || new Date(now).toISOString();
|
|
|
|
|
const rawBody = formatOutboundWebhookBody(url, payload, timestamp);
|
|
|
|
|
const secret =
|
|
|
|
|
options.webhookSecret !== undefined
|
|
|
|
|
? String(options.webhookSecret || '')
|
|
|
|
|
: String(runtimeConfig.webhookSecret || '');
|
|
|
|
|
|
|
|
|
|
const headers = {
|
|
|
|
|
'Content-Type': 'application/json',
|
|
|
|
|
'User-Agent': 'DataStack-Webhook-Notifier/2.0',
|
|
|
|
|
'X-DataStack-Event': String(payload.event || ''),
|
|
|
|
|
'X-DataStack-Timestamp': timestamp,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
if (secret) {
|
|
|
|
|
const signature = crypto
|
|
|
|
|
.createHmac('sha256', secret)
|
|
|
|
|
.update(rawBody)
|
|
|
|
|
.digest('hex');
|
|
|
|
|
headers['X-DataStack-Signature'] = `sha256=${signature}`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const controller = new AbortController();
|
|
|
|
|
const timeout = setTimeout(() => controller.abort(), 3500);
|
|
|
|
|
const fetchImpl = options.fetchImpl || global.fetch;
|
|
|
|
|
|
|
|
|
|
try {
|
2026-11-21 22:15:32 +06:00
|
|
|
const res = await fetchImpl(ssrfCheck.url, {
|
2026-11-21 00:06:55 +06:00
|
|
|
method: 'POST',
|
|
|
|
|
headers,
|
|
|
|
|
body: rawBody,
|
2026-11-21 22:15:32 +06:00
|
|
|
redirect: 'manual',
|
2026-11-21 00:06:55 +06:00
|
|
|
signal: controller.signal,
|
|
|
|
|
});
|
2026-11-21 22:15:32 +06:00
|
|
|
if (res.status >= 300 && res.status < 400) {
|
|
|
|
|
return {
|
|
|
|
|
dispatched: false,
|
|
|
|
|
code: 'SSRF_BLOCKED_HOST',
|
|
|
|
|
reason: 'SSRF_BLOCKED_HOST',
|
|
|
|
|
status: res.status,
|
|
|
|
|
error: `Webhook endpoint attempted an unsupported HTTP ${res.status} redirect.`,
|
|
|
|
|
};
|
|
|
|
|
}
|
2026-11-21 00:06:55 +06:00
|
|
|
return { dispatched: true, status: res.status };
|
|
|
|
|
} finally {
|
|
|
|
|
clearTimeout(timeout);
|
|
|
|
|
}
|
|
|
|
|
} catch (err) {
|
|
|
|
|
return {
|
|
|
|
|
dispatched: false,
|
|
|
|
|
reason: err && err.message ? err.message : 'webhook_dispatch_failed',
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-21 11:26:01 -08:00
|
|
|
const mcpEngine = createMcpEngine({
|
|
|
|
|
getStorageDataDir: () => writer.baseDir,
|
|
|
|
|
getDbPool: () => dbPool,
|
|
|
|
|
isClientIpAllowed,
|
|
|
|
|
resolveInternalSecret: () => resolveInternalSecret(),
|
|
|
|
|
decryptVaultSecret,
|
|
|
|
|
validateOutboundUrlAsync: validateOutboundWebhookUrlAsync,
|
|
|
|
|
flushBuffer,
|
|
|
|
|
});
|
|
|
|
|
|
2026-11-17 21:22:55 +06:00
|
|
|
async function ensureDatabaseSchema() {
|
2026-11-19 20:22:01 +06:00
|
|
|
const pool = dbPool;
|
|
|
|
|
if (!pool) return false;
|
|
|
|
|
if (schemaEnsurePromise && schemaEnsurePool === pool) {
|
|
|
|
|
return schemaEnsurePromise;
|
2026-11-17 21:22:55 +06:00
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
|
|
|
|
|
schemaEnsurePool = pool;
|
|
|
|
|
schemaEnsurePromise = (async () => {
|
|
|
|
|
try {
|
|
|
|
|
await pool.query(`
|
|
|
|
|
CREATE TABLE IF NOT EXISTS client_sdks (
|
|
|
|
|
client_id TEXT PRIMARY KEY,
|
|
|
|
|
name TEXT NOT NULL,
|
|
|
|
|
secret TEXT NOT NULL,
|
|
|
|
|
token TEXT NOT NULL,
|
|
|
|
|
rate_limit INTEGER NOT NULL DEFAULT 100,
|
2026-11-19 23:02:27 +06:00
|
|
|
storage_quota_mb INTEGER NOT NULL DEFAULT 0,
|
|
|
|
|
allowed_ips TEXT NOT NULL DEFAULT '',
|
|
|
|
|
expires_at TIMESTAMPTZ,
|
2026-11-19 20:22:01 +06:00
|
|
|
status TEXT NOT NULL DEFAULT 'active',
|
|
|
|
|
total_events BIGINT NOT NULL DEFAULT 0,
|
|
|
|
|
last_active TIMESTAMPTZ,
|
|
|
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
|
|
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
|
|
|
|
)
|
|
|
|
|
`);
|
|
|
|
|
await pool.query(`
|
|
|
|
|
CREATE TABLE IF NOT EXISTS admin_settings (
|
|
|
|
|
id TEXT PRIMARY KEY DEFAULT 'default',
|
|
|
|
|
server_port INTEGER NOT NULL DEFAULT 4000,
|
|
|
|
|
web_port INTEGER NOT NULL DEFAULT 3000,
|
|
|
|
|
max_events_per_sec INTEGER NOT NULL DEFAULT 100,
|
|
|
|
|
max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
|
|
|
|
|
batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
|
|
|
|
|
batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
|
|
|
|
|
file_rotation_mb INTEGER NOT NULL DEFAULT 100,
|
|
|
|
|
jwt_secret TEXT NOT NULL,
|
|
|
|
|
enable_idempotency BOOLEAN NOT NULL DEFAULT true,
|
|
|
|
|
auto_prune_days INTEGER NOT NULL DEFAULT 30,
|
2026-11-19 23:02:27 +06:00
|
|
|
compress_after_days INTEGER NOT NULL DEFAULT 0,
|
|
|
|
|
enable_s3_backup BOOLEAN NOT NULL DEFAULT false,
|
|
|
|
|
s3_endpoint TEXT NOT NULL DEFAULT '',
|
|
|
|
|
s3_region TEXT NOT NULL DEFAULT 'auto',
|
|
|
|
|
s3_bucket TEXT NOT NULL DEFAULT '',
|
|
|
|
|
s3_prefix TEXT NOT NULL DEFAULT 'datastack/',
|
|
|
|
|
s3_access_key_id TEXT NOT NULL DEFAULT '',
|
|
|
|
|
s3_secret_access_key_enc TEXT NOT NULL DEFAULT '',
|
|
|
|
|
disk_pause_threshold_pct INTEGER NOT NULL DEFAULT 95,
|
|
|
|
|
enable_webhook_alerts BOOLEAN NOT NULL DEFAULT false,
|
|
|
|
|
webhook_url TEXT NOT NULL DEFAULT '',
|
|
|
|
|
webhook_secret_enc TEXT NOT NULL DEFAULT '',
|
|
|
|
|
webhook_cooldown_sec INTEGER NOT NULL DEFAULT 300,
|
|
|
|
|
enable_prometheus_metrics BOOLEAN NOT NULL DEFAULT false,
|
|
|
|
|
prometheus_metrics_token_enc TEXT NOT NULL DEFAULT '',
|
2026-11-19 20:22:01 +06:00
|
|
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
|
|
|
|
)
|
|
|
|
|
`);
|
|
|
|
|
await pool.query(`
|
|
|
|
|
CREATE TABLE IF NOT EXISTS deleted_clients (
|
|
|
|
|
client_id TEXT PRIMARY KEY,
|
|
|
|
|
deleted_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
|
|
|
|
)
|
|
|
|
|
`);
|
|
|
|
|
await pool.query(`
|
|
|
|
|
CREATE TABLE IF NOT EXISTS admin_users (
|
|
|
|
|
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
|
|
|
|
|
email TEXT NOT NULL UNIQUE,
|
|
|
|
|
password_salt BYTEA NOT NULL,
|
|
|
|
|
password_hash BYTEA NOT NULL,
|
|
|
|
|
token_version INTEGER NOT NULL DEFAULT 1,
|
|
|
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
|
|
|
|
)
|
|
|
|
|
`);
|
|
|
|
|
await pool.query(`
|
|
|
|
|
CREATE TABLE IF NOT EXISTS admin_login_attempts (
|
|
|
|
|
key_hash TEXT PRIMARY KEY,
|
|
|
|
|
window_started_at TIMESTAMPTZ NOT NULL,
|
|
|
|
|
failed_attempts INTEGER NOT NULL,
|
|
|
|
|
blocked_until TIMESTAMPTZ,
|
|
|
|
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
|
|
|
|
)
|
|
|
|
|
`);
|
2026-11-21 22:15:32 +06:00
|
|
|
await pool.query(`
|
|
|
|
|
CREATE TABLE IF NOT EXISTS admin_team_members (
|
|
|
|
|
id TEXT PRIMARY KEY,
|
|
|
|
|
email TEXT NOT NULL UNIQUE,
|
|
|
|
|
role TEXT NOT NULL DEFAULT 'viewer',
|
|
|
|
|
status TEXT NOT NULL DEFAULT 'active',
|
|
|
|
|
password_salt BYTEA NOT NULL,
|
|
|
|
|
password_hash BYTEA NOT NULL,
|
|
|
|
|
token_version INTEGER NOT NULL DEFAULT 1,
|
|
|
|
|
invited_by TEXT NOT NULL DEFAULT 'owner',
|
|
|
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
|
|
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
|
|
|
|
)
|
|
|
|
|
`);
|
|
|
|
|
await pool.query(`
|
|
|
|
|
CREATE TABLE IF NOT EXISTS admin_audit_logs (
|
|
|
|
|
id TEXT PRIMARY KEY,
|
|
|
|
|
actor_email TEXT NOT NULL,
|
|
|
|
|
actor_role TEXT NOT NULL,
|
|
|
|
|
action TEXT NOT NULL,
|
|
|
|
|
target TEXT NOT NULL,
|
|
|
|
|
ip_address TEXT NOT NULL DEFAULT '',
|
|
|
|
|
metadata_json TEXT NOT NULL DEFAULT '{}',
|
|
|
|
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
|
|
|
|
)
|
|
|
|
|
`);
|
2026-11-21 23:00:29 +06:00
|
|
|
await pool.query(`
|
|
|
|
|
CREATE TABLE IF NOT EXISTS mcp_settings (
|
|
|
|
|
id TEXT PRIMARY KEY DEFAULT 'default',
|
|
|
|
|
enabled BOOLEAN NOT NULL DEFAULT false,
|
|
|
|
|
token_hash TEXT NOT NULL DEFAULT '',
|
|
|
|
|
token_enc TEXT NOT NULL DEFAULT '',
|
|
|
|
|
token_preview TEXT NOT NULL DEFAULT '',
|
|
|
|
|
allowed_ips TEXT NOT NULL DEFAULT '',
|
|
|
|
|
permissions_json TEXT NOT NULL DEFAULT '{}',
|
|
|
|
|
coolify_url TEXT NOT NULL DEFAULT '',
|
|
|
|
|
coolify_token_enc TEXT NOT NULL DEFAULT '',
|
|
|
|
|
coolify_app_uuid TEXT NOT NULL DEFAULT '',
|
|
|
|
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
|
|
|
|
)
|
|
|
|
|
`);
|
2026-11-19 20:22:01 +06:00
|
|
|
await pool.query(`
|
|
|
|
|
ALTER TABLE client_sdks
|
|
|
|
|
ADD COLUMN IF NOT EXISTS rate_limit INTEGER NOT NULL DEFAULT 100,
|
2026-11-19 23:02:27 +06:00
|
|
|
ADD COLUMN IF NOT EXISTS storage_quota_mb INTEGER NOT NULL DEFAULT 0,
|
|
|
|
|
ADD COLUMN IF NOT EXISTS allowed_ips TEXT NOT NULL DEFAULT '',
|
|
|
|
|
ADD COLUMN IF NOT EXISTS expires_at TIMESTAMPTZ,
|
2026-11-19 20:22:01 +06:00
|
|
|
ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'active',
|
|
|
|
|
ADD COLUMN IF NOT EXISTS total_events BIGINT NOT NULL DEFAULT 0,
|
|
|
|
|
ADD COLUMN IF NOT EXISTS last_active TIMESTAMPTZ,
|
|
|
|
|
ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
|
|
|
|
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
|
|
|
|
`);
|
|
|
|
|
await pool.query(`
|
|
|
|
|
ALTER TABLE admin_settings
|
|
|
|
|
ADD COLUMN IF NOT EXISTS server_port INTEGER NOT NULL DEFAULT 4000,
|
|
|
|
|
ADD COLUMN IF NOT EXISTS web_port INTEGER NOT NULL DEFAULT 3000,
|
|
|
|
|
ADD COLUMN IF NOT EXISTS max_events_per_sec INTEGER NOT NULL DEFAULT 100,
|
|
|
|
|
ADD COLUMN IF NOT EXISTS max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
|
|
|
|
|
ADD COLUMN IF NOT EXISTS batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
|
|
|
|
|
ADD COLUMN IF NOT EXISTS batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
|
|
|
|
|
ADD COLUMN IF NOT EXISTS file_rotation_mb INTEGER NOT NULL DEFAULT 100,
|
|
|
|
|
ADD COLUMN IF NOT EXISTS enable_idempotency BOOLEAN NOT NULL DEFAULT true,
|
|
|
|
|
ADD COLUMN IF NOT EXISTS auto_prune_days INTEGER NOT NULL DEFAULT 30,
|
2026-11-19 23:02:27 +06:00
|
|
|
ADD COLUMN IF NOT EXISTS compress_after_days INTEGER NOT NULL DEFAULT 0,
|
|
|
|
|
ADD COLUMN IF NOT EXISTS enable_s3_backup BOOLEAN NOT NULL DEFAULT false,
|
|
|
|
|
ADD COLUMN IF NOT EXISTS s3_endpoint TEXT NOT NULL DEFAULT '',
|
|
|
|
|
ADD COLUMN IF NOT EXISTS s3_region TEXT NOT NULL DEFAULT 'auto',
|
|
|
|
|
ADD COLUMN IF NOT EXISTS s3_bucket TEXT NOT NULL DEFAULT '',
|
|
|
|
|
ADD COLUMN IF NOT EXISTS s3_prefix TEXT NOT NULL DEFAULT 'datastack/',
|
|
|
|
|
ADD COLUMN IF NOT EXISTS s3_access_key_id TEXT NOT NULL DEFAULT '',
|
|
|
|
|
ADD COLUMN IF NOT EXISTS s3_secret_access_key_enc TEXT NOT NULL DEFAULT '',
|
|
|
|
|
ADD COLUMN IF NOT EXISTS disk_pause_threshold_pct INTEGER NOT NULL DEFAULT 95,
|
|
|
|
|
ADD COLUMN IF NOT EXISTS enable_webhook_alerts BOOLEAN NOT NULL DEFAULT false,
|
|
|
|
|
ADD COLUMN IF NOT EXISTS webhook_url TEXT NOT NULL DEFAULT '',
|
|
|
|
|
ADD COLUMN IF NOT EXISTS webhook_secret_enc TEXT NOT NULL DEFAULT '',
|
|
|
|
|
ADD COLUMN IF NOT EXISTS webhook_cooldown_sec INTEGER NOT NULL DEFAULT 300,
|
|
|
|
|
ADD COLUMN IF NOT EXISTS enable_prometheus_metrics BOOLEAN NOT NULL DEFAULT false,
|
|
|
|
|
ADD COLUMN IF NOT EXISTS prometheus_metrics_token_enc TEXT NOT NULL DEFAULT '',
|
2026-11-19 20:22:01 +06:00
|
|
|
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
|
|
|
|
`);
|
|
|
|
|
await pool.query(`
|
|
|
|
|
ALTER TABLE admin_users
|
|
|
|
|
ADD COLUMN IF NOT EXISTS token_version INTEGER NOT NULL DEFAULT 1
|
|
|
|
|
`);
|
|
|
|
|
return true;
|
|
|
|
|
} catch {
|
|
|
|
|
if (schemaEnsurePool === pool) {
|
|
|
|
|
schemaEnsurePromise = null;
|
|
|
|
|
}
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
})();
|
|
|
|
|
|
|
|
|
|
return schemaEnsurePromise;
|
2026-11-17 21:22:55 +06:00
|
|
|
}
|
|
|
|
|
|
2026-11-17 20:05:00 +06:00
|
|
|
async function syncSettingsFromDatabase() {
|
2026-11-19 20:22:01 +06:00
|
|
|
if (!dbPool) return false;
|
2026-11-17 20:05:00 +06:00
|
|
|
try {
|
2026-11-19 20:22:01 +06:00
|
|
|
const schemaReady = await ensureDatabaseSchema();
|
|
|
|
|
if (!schemaReady) return false;
|
|
|
|
|
|
2026-11-17 20:05:00 +06:00
|
|
|
const res = await dbPool.query(
|
|
|
|
|
`SELECT max_events_per_sec, max_payload_size_kb, batch_flush_threshold,
|
2026-11-18 02:51:04 +06:00
|
|
|
batch_flush_interval_ms, file_rotation_mb, jwt_secret, enable_idempotency,
|
2026-11-21 00:06:55 +06:00
|
|
|
auto_prune_days, compress_after_days, enable_s3_backup, disk_pause_threshold_pct,
|
|
|
|
|
enable_webhook_alerts, webhook_url, webhook_secret_enc, webhook_cooldown_sec
|
2026-11-17 20:05:00 +06:00
|
|
|
FROM admin_settings
|
|
|
|
|
WHERE id = 'default'`,
|
|
|
|
|
);
|
2026-11-20 18:48:54 +06:00
|
|
|
if (!res.rowCount || res.rowCount === 0) {
|
|
|
|
|
hasHydratedSettingsFromDb = true;
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
hasHydratedSettingsFromDb = true;
|
2026-11-17 20:05:00 +06:00
|
|
|
const row = res.rows[0];
|
2026-11-18 05:18:55 +06:00
|
|
|
const parsedPrune =
|
|
|
|
|
row.auto_prune_days !== undefined && row.auto_prune_days !== null
|
|
|
|
|
? Number(row.auto_prune_days)
|
|
|
|
|
: NaN;
|
2026-11-19 23:02:27 +06:00
|
|
|
const parsedCompress =
|
|
|
|
|
row.compress_after_days !== undefined && row.compress_after_days !== null
|
|
|
|
|
? Number(row.compress_after_days)
|
|
|
|
|
: NaN;
|
|
|
|
|
const parsedDiskPct =
|
|
|
|
|
row.disk_pause_threshold_pct !== undefined && row.disk_pause_threshold_pct !== null
|
|
|
|
|
? Number(row.disk_pause_threshold_pct)
|
|
|
|
|
: NaN;
|
2026-11-21 00:06:55 +06:00
|
|
|
const parsedWebhookCooldown =
|
|
|
|
|
row.webhook_cooldown_sec !== undefined && row.webhook_cooldown_sec !== null
|
|
|
|
|
? Number(row.webhook_cooldown_sec)
|
|
|
|
|
: NaN;
|
2026-11-17 20:05:00 +06:00
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
applyRuntimeSettingsObject({
|
|
|
|
|
maxEventsPerSec: Number(row.max_events_per_sec) || 100,
|
|
|
|
|
maxPayloadSizeKb: Number(row.max_payload_size_kb) || 100,
|
|
|
|
|
batchFlushThreshold: Math.max(1, Number(row.batch_flush_threshold) || 500),
|
|
|
|
|
batchFlushIntervalMs: Math.max(
|
|
|
|
|
100,
|
|
|
|
|
Number(row.batch_flush_interval_ms) || 2000,
|
|
|
|
|
),
|
|
|
|
|
fileRotationMb: Number(row.file_rotation_mb) || 100,
|
|
|
|
|
enableIdempotency: Boolean(row.enable_idempotency),
|
|
|
|
|
autoPruneDays: Number.isFinite(parsedPrune) ? parsedPrune : 30,
|
2026-11-19 23:02:27 +06:00
|
|
|
compressAfterDays: Number.isFinite(parsedCompress) ? Math.max(0, parsedCompress) : 0,
|
|
|
|
|
enableS3Backup: Boolean(row.enable_s3_backup),
|
|
|
|
|
diskPauseThresholdPct: Number.isFinite(parsedDiskPct) ? parsedDiskPct : 95,
|
2026-11-21 00:06:55 +06:00
|
|
|
enableWebhookAlerts: Boolean(row.enable_webhook_alerts),
|
|
|
|
|
webhookUrl: row.webhook_url ? String(row.webhook_url).trim() : '',
|
|
|
|
|
webhookSecretEnc: row.webhook_secret_enc ? String(row.webhook_secret_enc) : '',
|
|
|
|
|
webhookCooldownSec: Number.isFinite(parsedWebhookCooldown)
|
|
|
|
|
? Math.max(0, parsedWebhookCooldown)
|
|
|
|
|
: 300,
|
2026-11-18 05:18:55 +06:00
|
|
|
jwtSecret: row.jwt_secret ? String(row.jwt_secret).trim() : undefined,
|
|
|
|
|
});
|
2026-11-21 11:26:01 -08:00
|
|
|
await mcpEngine.hydrateFromDatabase(dbPool);
|
2026-11-19 20:22:01 +06:00
|
|
|
return true;
|
2026-11-17 20:05:00 +06:00
|
|
|
} catch {
|
2026-11-19 20:22:01 +06:00
|
|
|
return false;
|
2026-11-17 20:05:00 +06:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
const PRUNE_INTERVAL_MS = 60 * 60 * 1000;
|
2026-11-17 21:22:55 +06:00
|
|
|
let settingsSyncTimer = null;
|
2026-11-18 05:18:55 +06:00
|
|
|
let pruneTimer = null;
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
const socketHandlersApi = setupSocketHandlers(io, bufferQueue, writer, {
|
|
|
|
|
getRuntimeConfig: () => runtimeConfig,
|
2026-11-19 20:22:01 +06:00
|
|
|
getRedisClient,
|
2026-11-18 05:18:55 +06:00
|
|
|
verifyClientInDb,
|
|
|
|
|
recordClientActivity,
|
2026-11-21 00:06:55 +06:00
|
|
|
triggerWebhookAlert,
|
2026-11-19 20:22:01 +06:00
|
|
|
onBufferPush: async () => {
|
2026-11-18 05:18:55 +06:00
|
|
|
const redis = getRedisClient();
|
2026-11-19 20:22:01 +06:00
|
|
|
if (redis) {
|
|
|
|
|
if (bufferQueue.length > 0) {
|
|
|
|
|
const pushed = bufferQueue.splice(0, bufferQueue.length);
|
|
|
|
|
const op = pushEventsToRedisOrFallback(pushed);
|
|
|
|
|
pendingRedisWrites.add(op);
|
|
|
|
|
try {
|
|
|
|
|
const totalQueued = await op;
|
|
|
|
|
if (totalQueued >= runtimeConfig.batchFlushThreshold) {
|
|
|
|
|
void flushBuffer().catch(() => undefined);
|
|
|
|
|
}
|
|
|
|
|
} finally {
|
|
|
|
|
pendingRedisWrites.delete(op);
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
if (pendingRedisWrites.size > 0) {
|
|
|
|
|
await Promise.all([...pendingRedisWrites]);
|
|
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (bufferQueue.length >= runtimeConfig.batchFlushThreshold) {
|
2026-11-19 20:22:01 +06:00
|
|
|
void flushBuffer().catch(() => undefined);
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
|
|
|
|
},
|
2026-10-23 11:43:49 -05:00
|
|
|
});
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
async function getRedisBufferLen() {
|
|
|
|
|
const redis = getRedisClient();
|
|
|
|
|
if (!redis) return 0;
|
|
|
|
|
try {
|
|
|
|
|
const len = Number(await redis.llen(REDIS_BUFFER_KEY)) || 0;
|
|
|
|
|
redisConnected = true;
|
|
|
|
|
return len;
|
|
|
|
|
} catch {
|
|
|
|
|
redisConnected = false;
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Public health check endpoint
|
|
|
|
|
app.get('/health', async (_req, res) => {
|
|
|
|
|
const redisBufferSize = await getRedisBufferLen();
|
2026-11-17 20:05:00 +06:00
|
|
|
res.json({
|
|
|
|
|
status: 'online',
|
|
|
|
|
service: 'DataStack Express Ingestion',
|
|
|
|
|
port: PORT,
|
2026-11-19 20:22:01 +06:00
|
|
|
serverPort: PORT,
|
|
|
|
|
webPort: Number(process.env.PORT) || 3000,
|
2026-11-17 20:05:00 +06:00
|
|
|
activeSockets: io.engine ? io.engine.clientsCount : 0,
|
2026-11-18 05:18:55 +06:00
|
|
|
bufferSize: bufferQueue.length + redisBufferSize,
|
|
|
|
|
localBufferSize: bufferQueue.length,
|
|
|
|
|
redisBufferSize,
|
|
|
|
|
redisConnected,
|
2026-11-18 02:23:01 +06:00
|
|
|
maxEventsPerSec: runtimeConfig.maxEventsPerSec,
|
|
|
|
|
maxPayloadSizeBytes: runtimeConfig.maxPayloadSizeBytes,
|
2026-11-17 20:05:00 +06:00
|
|
|
uptime: process.uptime(),
|
|
|
|
|
});
|
2026-10-23 11:43:49 -05:00
|
|
|
});
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
// Require internal shared authentication on all /internal/* bridge endpoints
|
|
|
|
|
app.use('/internal', requireInternalAuth);
|
|
|
|
|
|
|
|
|
|
// Internal metrics endpoint for Next.js dashboard & in-memory client activity aggregation
|
|
|
|
|
app.get('/internal/metrics', async (_req, res) => {
|
|
|
|
|
const redisBufferSize = await getRedisBufferLen();
|
|
|
|
|
res.json({
|
2026-11-19 20:22:01 +06:00
|
|
|
port: PORT,
|
|
|
|
|
serverPort: PORT,
|
|
|
|
|
webPort: Number(process.env.PORT) || 3000,
|
2026-11-18 05:18:55 +06:00
|
|
|
activeSockets: io.engine ? io.engine.clientsCount : 0,
|
|
|
|
|
bufferSize: bufferQueue.length + redisBufferSize,
|
|
|
|
|
localBufferSize: bufferQueue.length,
|
|
|
|
|
redisBufferSize,
|
|
|
|
|
redisConnected,
|
|
|
|
|
registrySize: inMemoryClientRegistry.size,
|
|
|
|
|
hasSyncedInMemoryClients,
|
2026-11-19 20:22:01 +06:00
|
|
|
inMemoryClients: Array.from(inMemoryClientRegistry.values()),
|
2026-11-18 05:18:55 +06:00
|
|
|
securityStats: socketHandlersApi.getSecurityStats(),
|
|
|
|
|
clientActivity: Object.fromEntries(inMemoryClientActivity.entries()),
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
// Internal shared security check for Port 3000 HTTP ingestion when Redis is unavailable (BUG-03)
|
|
|
|
|
app.post('/internal/security-check', (req, res) => {
|
|
|
|
|
const body = req.body && typeof req.body === 'object' ? req.body : {};
|
|
|
|
|
const clientId = normalizeClientId(body.clientId || '');
|
|
|
|
|
if (!clientId) {
|
|
|
|
|
return res.status(400).json({
|
|
|
|
|
allowed: false,
|
|
|
|
|
code: 'MALFORMED_PAYLOAD',
|
|
|
|
|
reason: 'Missing clientId',
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
const eventId = typeof body.eventId === 'string' ? body.eventId : '';
|
|
|
|
|
const clientRateLimit = Number(body.clientMaxRate) || 100;
|
|
|
|
|
const payloadBytes =
|
|
|
|
|
typeof body.payloadBytes === 'number' &&
|
|
|
|
|
Number.isFinite(body.payloadBytes) &&
|
|
|
|
|
body.payloadBytes >= 0
|
|
|
|
|
? body.payloadBytes
|
|
|
|
|
: undefined;
|
|
|
|
|
|
|
|
|
|
const result = socketHandlersApi.validateSingleEventLocal(
|
|
|
|
|
clientId,
|
|
|
|
|
eventId,
|
|
|
|
|
body.payload ?? {},
|
|
|
|
|
clientRateLimit,
|
|
|
|
|
payloadBytes,
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
return res.json({
|
|
|
|
|
allowed: Boolean(result.allowed),
|
|
|
|
|
duplicate: Boolean(result.duplicate),
|
|
|
|
|
...(result.code ? { code: result.code } : {}),
|
|
|
|
|
...(result.reason ? { reason: result.reason } : {}),
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
// Internal instant settings synchronization triggered by Next.js POST /api/settings
|
|
|
|
|
app.post('/internal/settings-sync', (req, res) => {
|
2026-11-20 18:48:54 +06:00
|
|
|
hasHydratedSettingsFromDb = true;
|
2026-11-18 05:18:55 +06:00
|
|
|
const result = applyRuntimeSettingsObject(req.body || {});
|
|
|
|
|
res.json({
|
|
|
|
|
success: true,
|
|
|
|
|
jwtRotated: result.jwtRotated,
|
|
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
// Internal flush endpoint triggered by Next.js POST /api/storage & GET /api/export
|
2026-11-19 20:22:01 +06:00
|
|
|
app.post('/internal/flush', async (req, res) => {
|
2026-11-17 20:05:00 +06:00
|
|
|
try {
|
2026-11-19 20:22:01 +06:00
|
|
|
const throwOnLockTimeout = Boolean(req.body && req.body.throwOnLockTimeout === true);
|
|
|
|
|
const waitMs =
|
|
|
|
|
req.body && Number.isFinite(Number(req.body.waitMs))
|
|
|
|
|
? Math.max(0, Number(req.body.waitMs))
|
|
|
|
|
: 2500;
|
|
|
|
|
const result = await flushBuffer({ waitMs, throwOnLockTimeout });
|
2026-11-17 20:05:00 +06:00
|
|
|
res.json({
|
|
|
|
|
success: true,
|
|
|
|
|
flushedCount: result.written,
|
|
|
|
|
filesUpdated: result.files,
|
2026-11-19 20:22:01 +06:00
|
|
|
...(result.lockTimedOut ? { lockTimedOut: true } : {}),
|
2026-11-17 20:05:00 +06:00
|
|
|
});
|
|
|
|
|
} catch (err) {
|
2026-11-19 20:22:01 +06:00
|
|
|
const message = err instanceof Error ? err.message : 'Flush failed';
|
|
|
|
|
if (message.includes('FLUSH_LOCK_TIMEOUT')) {
|
|
|
|
|
return res.status(503).json({
|
|
|
|
|
success: false,
|
|
|
|
|
code: 'FLUSH_LOCK_TIMEOUT',
|
|
|
|
|
error: message,
|
|
|
|
|
});
|
|
|
|
|
}
|
2026-11-17 20:05:00 +06:00
|
|
|
res.status(500).json({
|
|
|
|
|
success: false,
|
2026-11-19 20:22:01 +06:00
|
|
|
error: message,
|
2026-11-17 20:05:00 +06:00
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
2026-11-18 03:20:00 +06:00
|
|
|
// Internal endpoint to expose Port 4000 buffered events before CSV flush
|
2026-11-18 05:18:55 +06:00
|
|
|
app.get('/internal/recent', async (req, res) => {
|
2026-11-19 20:22:01 +06:00
|
|
|
const rawClientId = req.query.clientId ? String(req.query.clientId) : '';
|
|
|
|
|
const filterClientId =
|
|
|
|
|
rawClientId && rawClientId !== 'all' ? normalizeClientId(rawClientId) : '';
|
|
|
|
|
const recentLimit = getRedisRecentLimit();
|
|
|
|
|
const limit = Math.min(recentLimit, Math.max(1, Number(req.query.limit) || 50));
|
2026-11-18 05:18:55 +06:00
|
|
|
|
|
|
|
|
const redis = getRedisClient();
|
|
|
|
|
let redisEvents = [];
|
|
|
|
|
if (redis) {
|
|
|
|
|
try {
|
2026-11-19 20:22:01 +06:00
|
|
|
const readCap = Math.min(recentLimit, Math.max(limit * 2, limit));
|
2026-11-18 05:18:55 +06:00
|
|
|
const rawItems = await redis.lrange(REDIS_RECENT_KEY, 0, readCap - 1);
|
|
|
|
|
redisConnected = true;
|
|
|
|
|
for (const raw of rawItems) {
|
|
|
|
|
try {
|
|
|
|
|
const evt = JSON.parse(raw);
|
|
|
|
|
if (!evt || !evt.eventId) continue;
|
2026-11-19 20:22:01 +06:00
|
|
|
const normEvtClientId = normalizeClientId(evt.clientId);
|
|
|
|
|
if (deletedClientsSet.has(normEvtClientId)) continue;
|
|
|
|
|
if (filterClientId && normEvtClientId !== filterClientId) continue;
|
2026-11-18 05:18:55 +06:00
|
|
|
redisEvents.push(evt);
|
|
|
|
|
if (redisEvents.length >= limit) break;
|
|
|
|
|
} catch {
|
|
|
|
|
// ignore malformed item
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
redisConnected = false;
|
|
|
|
|
redisEvents = [];
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
const localFiltered = filterClientId
|
|
|
|
|
? bufferQueue.filter((evt) => normalizeClientId(evt.clientId) === filterClientId)
|
|
|
|
|
: bufferQueue;
|
2026-11-18 05:18:55 +06:00
|
|
|
const localRecent = localFiltered.slice(-limit).reverse();
|
|
|
|
|
|
|
|
|
|
const merged = [];
|
|
|
|
|
const seen = new Set();
|
|
|
|
|
for (const evt of [...localRecent, ...redisEvents]) {
|
|
|
|
|
if (!evt || !evt.eventId) continue;
|
|
|
|
|
const key = `${evt.clientId}:${evt.eventId}`;
|
|
|
|
|
if (seen.has(key)) continue;
|
|
|
|
|
seen.add(key);
|
|
|
|
|
merged.push(evt);
|
|
|
|
|
if (merged.length >= limit) break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
res.json({ events: merged, count: merged.length });
|
2026-11-18 03:20:00 +06:00
|
|
|
});
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
// Internal endpoint triggered by Next.js SDKManager when a client is deleted, revoked, rotated, or synced
|
2026-11-21 22:15:32 +06:00
|
|
|
app.post('/internal/client-sync', async (req, res) => {
|
2026-11-18 05:18:55 +06:00
|
|
|
const action = req.body && req.body.action ? String(req.body.action) : '';
|
|
|
|
|
|
|
|
|
|
if (action === 'bulk_sync' && Array.isArray(req.body.clients)) {
|
2026-11-19 20:22:01 +06:00
|
|
|
const forceReplace = req.body.forceReplace === true;
|
|
|
|
|
// BUG-02 Guard: Never wipe a populated Port 4000 in-memory registry when a cold-restarted
|
|
|
|
|
// Port 3000 (without DB) sends an empty bulk_sync without explicit forceReplace.
|
|
|
|
|
if (
|
|
|
|
|
req.body.clients.length === 0 &&
|
|
|
|
|
!forceReplace &&
|
|
|
|
|
inMemoryClientRegistry.size > 0
|
|
|
|
|
) {
|
|
|
|
|
hasSyncedInMemoryClients = true;
|
|
|
|
|
return res.json({
|
|
|
|
|
success: true,
|
|
|
|
|
count: inMemoryClientRegistry.size,
|
|
|
|
|
preserved: true,
|
|
|
|
|
clients: Array.from(inMemoryClientRegistry.values()),
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
hasSyncedInMemoryClients = true;
|
2026-11-19 20:22:01 +06:00
|
|
|
const prevRegistry = new Map(inMemoryClientRegistry);
|
2026-11-18 05:18:55 +06:00
|
|
|
inMemoryClientRegistry.clear();
|
|
|
|
|
clientAuthCache.clear();
|
|
|
|
|
const activeIds = new Set();
|
|
|
|
|
for (const item of req.body.clients) {
|
|
|
|
|
if (!item || !item.clientId) continue;
|
2026-11-19 20:22:01 +06:00
|
|
|
const cid = normalizeClientId(item.clientId);
|
|
|
|
|
if (!cid) continue;
|
|
|
|
|
const nextStatus = item.status === 'revoked' ? 'revoked' : 'active';
|
|
|
|
|
const nextToken = String(item.token || '');
|
|
|
|
|
const prev = prevRegistry.get(cid);
|
2026-11-18 05:18:55 +06:00
|
|
|
activeIds.add(cid);
|
|
|
|
|
deletedClientsSet.delete(cid);
|
|
|
|
|
inMemoryClientRegistry.set(cid, {
|
|
|
|
|
clientId: cid,
|
|
|
|
|
name: String(item.name || cid),
|
2026-11-19 20:22:01 +06:00
|
|
|
status: nextStatus,
|
|
|
|
|
token: nextToken,
|
2026-11-18 05:18:55 +06:00
|
|
|
rateLimit: Math.max(1, Number(item.rateLimit) || 100),
|
2026-11-19 23:02:27 +06:00
|
|
|
storageQuotaMb: Math.max(0, Math.floor(Number(item.storageQuotaMb) || 0)),
|
|
|
|
|
allowedIps: typeof item.allowedIps === 'string' ? item.allowedIps : '',
|
|
|
|
|
expiresAt: item.expiresAt ? String(item.expiresAt) : null,
|
2026-11-18 05:18:55 +06:00
|
|
|
});
|
2026-11-19 20:22:01 +06:00
|
|
|
if (nextStatus === 'revoked' || (prev && nextToken && prev.token !== nextToken)) {
|
2026-11-18 05:18:55 +06:00
|
|
|
io.to(`client:${cid}`).disconnectSockets(true);
|
|
|
|
|
}
|
2026-11-17 20:05:00 +06:00
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
if (forceReplace) {
|
|
|
|
|
for (const prevCid of prevRegistry.keys()) {
|
|
|
|
|
if (!activeIds.has(prevCid)) {
|
|
|
|
|
io.to(`client:${prevCid}`).disconnectSockets(true);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return res.json({ success: true, count: activeIds.size, preserved: false });
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const rawClientId = req.body && req.body.clientId ? String(req.body.clientId) : '';
|
2026-11-19 20:22:01 +06:00
|
|
|
const normalizedId = normalizeClientId(rawClientId);
|
2026-11-18 05:18:55 +06:00
|
|
|
const clientPayload =
|
|
|
|
|
req.body && req.body.client && typeof req.body.client === 'object'
|
|
|
|
|
? req.body.client
|
|
|
|
|
: null;
|
|
|
|
|
|
|
|
|
|
if (normalizedId) {
|
|
|
|
|
hasSyncedInMemoryClients = true;
|
|
|
|
|
clientAuthCache.delete(normalizedId);
|
|
|
|
|
|
|
|
|
|
if (action === 'deleted') {
|
|
|
|
|
deletedClientsSet.add(normalizedId);
|
|
|
|
|
inMemoryClientRegistry.delete(normalizedId);
|
|
|
|
|
inMemoryClientActivity.delete(normalizedId);
|
2026-11-19 20:22:01 +06:00
|
|
|
pendingDbClientActivity.delete(normalizedId);
|
|
|
|
|
purgeClientFromLocalBuffer(normalizedId);
|
2026-11-18 05:18:55 +06:00
|
|
|
io.to(`client:${normalizedId}`).disconnectSockets(true);
|
|
|
|
|
} else if (action === 'revoked') {
|
|
|
|
|
const existingMem = inMemoryClientRegistry.get(normalizedId);
|
|
|
|
|
if (clientPayload) {
|
|
|
|
|
inMemoryClientRegistry.set(normalizedId, {
|
|
|
|
|
clientId: normalizedId,
|
|
|
|
|
name: String(clientPayload.name || normalizedId),
|
|
|
|
|
status: 'revoked',
|
|
|
|
|
token: String(clientPayload.token || ''),
|
|
|
|
|
rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
|
2026-11-19 23:02:27 +06:00
|
|
|
storageQuotaMb: Math.max(0, Math.floor(Number(clientPayload.storageQuotaMb) || 0)),
|
|
|
|
|
allowedIps: typeof clientPayload.allowedIps === 'string' ? clientPayload.allowedIps : '',
|
|
|
|
|
expiresAt: clientPayload.expiresAt ? String(clientPayload.expiresAt) : null,
|
2026-11-18 05:18:55 +06:00
|
|
|
});
|
|
|
|
|
} else if (existingMem) {
|
|
|
|
|
existingMem.status = 'revoked';
|
|
|
|
|
} else {
|
|
|
|
|
inMemoryClientRegistry.set(normalizedId, {
|
|
|
|
|
clientId: normalizedId,
|
|
|
|
|
name: normalizedId,
|
|
|
|
|
status: 'revoked',
|
|
|
|
|
token: '',
|
|
|
|
|
rateLimit: 100,
|
2026-11-19 23:02:27 +06:00
|
|
|
storageQuotaMb: 0,
|
|
|
|
|
allowedIps: '',
|
|
|
|
|
expiresAt: null,
|
2026-11-18 05:18:55 +06:00
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
io.to(`client:${normalizedId}`).disconnectSockets(true);
|
|
|
|
|
} else if (action === 'registered') {
|
|
|
|
|
deletedClientsSet.delete(normalizedId);
|
|
|
|
|
if (clientPayload) {
|
|
|
|
|
inMemoryClientRegistry.set(normalizedId, {
|
|
|
|
|
clientId: normalizedId,
|
|
|
|
|
name: String(clientPayload.name || normalizedId),
|
|
|
|
|
status: clientPayload.status === 'revoked' ? 'revoked' : 'active',
|
|
|
|
|
token: String(clientPayload.token || ''),
|
|
|
|
|
rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
|
2026-11-19 23:02:27 +06:00
|
|
|
storageQuotaMb: Math.max(0, Math.floor(Number(clientPayload.storageQuotaMb) || 0)),
|
|
|
|
|
allowedIps: typeof clientPayload.allowedIps === 'string' ? clientPayload.allowedIps : '',
|
|
|
|
|
expiresAt: clientPayload.expiresAt ? String(clientPayload.expiresAt) : null,
|
2026-11-18 05:18:55 +06:00
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
} else if (action === 'updated') {
|
2026-11-21 22:15:32 +06:00
|
|
|
const prevClient = inMemoryClientRegistry.get(normalizedId);
|
|
|
|
|
const nextStatus =
|
|
|
|
|
clientPayload && clientPayload.status === 'revoked' ? 'revoked' : 'active';
|
|
|
|
|
const nextToken = clientPayload ? String(clientPayload.token || '') : '';
|
|
|
|
|
const nextAllowedIps =
|
|
|
|
|
clientPayload && typeof clientPayload.allowedIps === 'string'
|
|
|
|
|
? clientPayload.allowedIps.trim()
|
|
|
|
|
: '';
|
|
|
|
|
const nextExpiresAt =
|
|
|
|
|
clientPayload && clientPayload.expiresAt
|
|
|
|
|
? String(clientPayload.expiresAt)
|
|
|
|
|
: null;
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
if (clientPayload) {
|
|
|
|
|
inMemoryClientRegistry.set(normalizedId, {
|
|
|
|
|
clientId: normalizedId,
|
|
|
|
|
name: String(clientPayload.name || normalizedId),
|
2026-11-21 22:15:32 +06:00
|
|
|
status: nextStatus,
|
|
|
|
|
token: nextToken,
|
2026-11-18 05:18:55 +06:00
|
|
|
rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
|
2026-11-19 23:02:27 +06:00
|
|
|
storageQuotaMb: Math.max(0, Math.floor(Number(clientPayload.storageQuotaMb) || 0)),
|
2026-11-21 22:15:32 +06:00
|
|
|
allowedIps: nextAllowedIps,
|
|
|
|
|
expiresAt: nextExpiresAt,
|
2026-11-18 05:18:55 +06:00
|
|
|
});
|
|
|
|
|
}
|
2026-11-21 22:15:32 +06:00
|
|
|
|
|
|
|
|
const tokenExpired = Boolean(
|
|
|
|
|
nextExpiresAt && isClientTokenExpired(nextExpiresAt),
|
|
|
|
|
);
|
|
|
|
|
const shouldDisconnectRoom =
|
|
|
|
|
!clientPayload ||
|
|
|
|
|
nextStatus === 'revoked' ||
|
|
|
|
|
tokenExpired ||
|
|
|
|
|
Boolean(
|
|
|
|
|
prevClient &&
|
|
|
|
|
(prevClient.status !== nextStatus ||
|
|
|
|
|
(nextToken && prevClient.token !== nextToken) ||
|
|
|
|
|
(prevClient.allowedIps || '').trim() !== nextAllowedIps ||
|
|
|
|
|
(prevClient.expiresAt || null) !== nextExpiresAt),
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
let disconnected = false;
|
|
|
|
|
if (shouldDisconnectRoom) {
|
|
|
|
|
io.to(`client:${normalizedId}`).disconnectSockets(true);
|
|
|
|
|
disconnected = true;
|
|
|
|
|
} else if (!prevClient) {
|
|
|
|
|
const liveSockets = await io.in(`client:${normalizedId}`).fetchSockets();
|
|
|
|
|
for (const sock of liveSockets) {
|
|
|
|
|
const sockToken =
|
|
|
|
|
sock.data && sock.data.token ? String(sock.data.token) : '';
|
|
|
|
|
const sockIp =
|
|
|
|
|
sock.data && sock.data.clientIp ? String(sock.data.clientIp) : '';
|
|
|
|
|
const tokenMismatch = Boolean(
|
|
|
|
|
nextToken && sockToken && sockToken !== nextToken,
|
|
|
|
|
);
|
|
|
|
|
const ipForbidden = Boolean(
|
|
|
|
|
nextAllowedIps && (!sockIp || !isClientIpAllowed(sockIp, nextAllowedIps)),
|
|
|
|
|
);
|
|
|
|
|
if (tokenMismatch || ipForbidden) {
|
|
|
|
|
sock.disconnect(true);
|
|
|
|
|
disconnected = true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return res.json({ success: true, disconnected });
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
res.json({ success: true });
|
2026-11-17 20:05:00 +06:00
|
|
|
});
|
|
|
|
|
|
2026-11-21 22:15:32 +06:00
|
|
|
app.post('/internal/admin-session-sync', requireInternalAuth, (req, res) => {
|
|
|
|
|
const synced = syncInMemoryAdminSessionState(req.body || {});
|
|
|
|
|
res.json({ success: true, synced });
|
|
|
|
|
});
|
|
|
|
|
|
2026-11-21 11:26:01 -08:00
|
|
|
app.post('/internal/mcp-sync', requireInternalAuth, (req, res) => {
|
|
|
|
|
mcpEngine.syncState(req.body || {});
|
|
|
|
|
res.json({ success: true, state: mcpEngine.getStateSnapshot() });
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
app.all('/mcp', (req, res) => {
|
|
|
|
|
void mcpEngine.handleHttpRequest(req, res);
|
|
|
|
|
});
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
function stopTimers() {
|
2026-11-17 21:22:55 +06:00
|
|
|
if (flushTimer) {
|
|
|
|
|
clearInterval(flushTimer);
|
|
|
|
|
flushTimer = null;
|
|
|
|
|
}
|
|
|
|
|
if (settingsSyncTimer) {
|
|
|
|
|
clearInterval(settingsSyncTimer);
|
|
|
|
|
settingsSyncTimer = null;
|
|
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
if (pruneTimer) {
|
|
|
|
|
clearInterval(pruneTimer);
|
|
|
|
|
pruneTimer = null;
|
|
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
if (activityFlushTimer) {
|
|
|
|
|
clearInterval(activityFlushTimer);
|
|
|
|
|
activityFlushTimer = null;
|
|
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let isShuttingDown = false;
|
2026-11-17 21:22:55 +06:00
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
async function closeServerGracefully() {
|
2026-11-17 21:22:55 +06:00
|
|
|
try {
|
2026-11-19 20:22:01 +06:00
|
|
|
await Promise.race([
|
|
|
|
|
new Promise((resolve) => {
|
|
|
|
|
if (io && typeof io.close === 'function') {
|
|
|
|
|
io.close(() => resolve());
|
|
|
|
|
} else if (server && server.listening) {
|
|
|
|
|
server.close(() => resolve());
|
|
|
|
|
} else {
|
|
|
|
|
resolve();
|
|
|
|
|
}
|
|
|
|
|
}),
|
|
|
|
|
sleepMs(2000),
|
|
|
|
|
]);
|
|
|
|
|
if (server && typeof server.closeIdleConnections === 'function') {
|
|
|
|
|
server.closeIdleConnections();
|
2026-11-17 21:22:55 +06:00
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
// Ignore close errors
|
|
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
}
|
2026-11-17 21:22:55 +06:00
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
async function gracefulShutdown(signal, options = {}) {
|
|
|
|
|
if (isShuttingDown) return;
|
|
|
|
|
isShuttingDown = true;
|
|
|
|
|
const shouldExit = options.exit !== false;
|
|
|
|
|
console.log(
|
|
|
|
|
`[DataStack Ingestion Engine] Received ${signal}, closing ingress and flushing buffer before exit...`,
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
stopTimers();
|
|
|
|
|
|
|
|
|
|
// Close Socket.IO & HTTP listeners before final flush so late traffic cannot enter after drain starts
|
2026-11-21 22:15:32 +06:00
|
|
|
if (filesManagerHandle && typeof filesManagerHandle.close === 'function') {
|
|
|
|
|
await Promise.race([filesManagerHandle.close().catch(() => undefined), sleepMs(2000)]);
|
|
|
|
|
filesManagerHandle = null;
|
|
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
await closeServerGracefully();
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
if (pendingRedisWrites.size > 0) {
|
|
|
|
|
await Promise.race([
|
|
|
|
|
Promise.allSettled([...pendingRedisWrites]),
|
|
|
|
|
sleepMs(2000),
|
|
|
|
|
]);
|
|
|
|
|
}
|
|
|
|
|
await Promise.race([
|
|
|
|
|
Promise.allSettled([flushBuffer(), flushPendingClientActivity()]),
|
|
|
|
|
sleepMs(6000),
|
|
|
|
|
]);
|
|
|
|
|
} catch {
|
|
|
|
|
// Best-effort flush on exit
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
const activeRedis =
|
|
|
|
|
redisTestOverride !== undefined ? redisTestOverride : redisClient;
|
|
|
|
|
if (activeRedis && typeof activeRedis.quit === 'function') {
|
|
|
|
|
let quitFinished = false;
|
|
|
|
|
await Promise.race([
|
|
|
|
|
activeRedis
|
|
|
|
|
.quit()
|
|
|
|
|
.then(() => {
|
|
|
|
|
quitFinished = true;
|
|
|
|
|
})
|
|
|
|
|
.catch(() => {
|
|
|
|
|
quitFinished = true;
|
|
|
|
|
}),
|
|
|
|
|
sleepMs(2000),
|
|
|
|
|
]);
|
|
|
|
|
if (!quitFinished && typeof activeRedis.disconnect === 'function') {
|
|
|
|
|
activeRedis.disconnect();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (dbPool && typeof dbPool.end === 'function') {
|
|
|
|
|
await Promise.race([dbPool.end().catch(() => undefined), sleepMs(2000)]);
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
// Ignore close errors
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (shouldExit) {
|
|
|
|
|
process.exit(0);
|
|
|
|
|
}
|
|
|
|
|
} finally {
|
|
|
|
|
if (!shouldExit) {
|
|
|
|
|
isShuttingDown = false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function startIngestionServer() {
|
2026-11-21 23:00:29 +06:00
|
|
|
if (process.env.DATASTACK_DISABLE_LISTEN !== '1') {
|
|
|
|
|
installRuntimeLogHook('ingestion');
|
|
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
if (dbPool) {
|
|
|
|
|
await Promise.race([
|
|
|
|
|
syncSettingsFromDatabase().catch(() => false),
|
|
|
|
|
sleepMs(5000),
|
|
|
|
|
]);
|
|
|
|
|
settingsSyncTimer = setInterval(() => {
|
|
|
|
|
void syncSettingsFromDatabase();
|
|
|
|
|
}, 3000);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pruneTimer = setInterval(() => {
|
2026-11-20 18:48:54 +06:00
|
|
|
if (!isDbReadyForMaintenance()) return;
|
2026-11-19 23:02:27 +06:00
|
|
|
if (runtimeConfig.compressAfterDays > 0) {
|
|
|
|
|
void writer
|
|
|
|
|
.compressOldPartitionsAsync(runtimeConfig.compressAfterDays)
|
|
|
|
|
.catch(() => undefined);
|
|
|
|
|
}
|
|
|
|
|
if (runtimeConfig.autoPruneDays > 0 && !runtimeConfig.enableS3Backup) {
|
2026-11-19 20:22:01 +06:00
|
|
|
void writer
|
|
|
|
|
.pruneExpiredPartitionsAsync(runtimeConfig.autoPruneDays)
|
|
|
|
|
.catch(() => undefined);
|
|
|
|
|
}
|
|
|
|
|
}, PRUNE_INTERVAL_MS);
|
|
|
|
|
|
|
|
|
|
if (isShuttingDown) return;
|
|
|
|
|
|
2026-11-21 22:15:32 +06:00
|
|
|
if (!filesManagerHandle) {
|
|
|
|
|
filesManagerHandle = await mountFilesManager({
|
|
|
|
|
app,
|
|
|
|
|
server,
|
|
|
|
|
rootDir: writer.baseDir,
|
|
|
|
|
getDbPool: () => dbPool,
|
|
|
|
|
corsOriginDelegate,
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
await new Promise((resolve) => {
|
|
|
|
|
server.listen(PORT, () => {
|
|
|
|
|
console.log(
|
|
|
|
|
`[DataStack Ingestion Engine] Express + Socket.IO listening on port ${PORT}`,
|
|
|
|
|
);
|
|
|
|
|
resolve();
|
|
|
|
|
});
|
|
|
|
|
});
|
2026-11-17 21:22:55 +06:00
|
|
|
}
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
if (process.env.DATASTACK_DISABLE_LISTEN !== '1') {
|
|
|
|
|
process.on('SIGTERM', () => {
|
|
|
|
|
void gracefulShutdown('SIGTERM');
|
|
|
|
|
});
|
|
|
|
|
process.on('SIGINT', () => {
|
|
|
|
|
void gracefulShutdown('SIGINT');
|
|
|
|
|
});
|
2026-11-17 21:22:55 +06:00
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
void startIngestionServer();
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
2026-11-17 20:05:00 +06:00
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
module.exports = {
|
|
|
|
|
app,
|
|
|
|
|
server,
|
|
|
|
|
io,
|
|
|
|
|
writer,
|
|
|
|
|
bufferQueue,
|
|
|
|
|
flushBuffer,
|
|
|
|
|
runtimeConfig,
|
|
|
|
|
verifyClientInDb,
|
|
|
|
|
recordClientActivity,
|
2026-11-19 20:22:01 +06:00
|
|
|
flushPendingClientActivity,
|
2026-11-18 05:18:55 +06:00
|
|
|
applyRuntimeSettingsObject,
|
2026-11-19 20:22:01 +06:00
|
|
|
ensureDatabaseSchema,
|
|
|
|
|
syncSettingsFromDatabase,
|
2026-11-20 18:48:54 +06:00
|
|
|
isDbReadyForMaintenance,
|
2026-11-19 20:22:01 +06:00
|
|
|
gracefulShutdown,
|
|
|
|
|
startIngestionServer,
|
2026-11-18 05:18:55 +06:00
|
|
|
setRedisClientForTests,
|
2026-11-19 20:22:01 +06:00
|
|
|
setDbPoolForTests,
|
2026-11-18 05:18:55 +06:00
|
|
|
stopTimers,
|
2026-11-19 20:22:01 +06:00
|
|
|
parseAllowedCorsOrigins,
|
|
|
|
|
isOriginAllowedByCors,
|
|
|
|
|
corsOriginDelegate,
|
2026-11-19 23:02:27 +06:00
|
|
|
isClientIpAllowed,
|
|
|
|
|
isClientTokenExpired,
|
2026-11-21 00:06:55 +06:00
|
|
|
triggerWebhookAlert,
|
2026-11-21 22:15:32 +06:00
|
|
|
validateOutboundWebhookUrlAsync,
|
2026-11-21 00:06:55 +06:00
|
|
|
clearWebhookCooldownCacheForTests,
|
|
|
|
|
decryptVaultSecret,
|
|
|
|
|
formatOutboundWebhookBody,
|
2026-11-21 22:15:32 +06:00
|
|
|
mountFilesManager,
|
|
|
|
|
syncInMemoryAdminSessionState,
|
|
|
|
|
clearInMemoryAdminSessionStateForTests,
|
|
|
|
|
getFilesManagerHandle: () => filesManagerHandle,
|
2026-11-21 11:26:01 -08:00
|
|
|
mcpEngine,
|
2026-11-18 05:18:55 +06:00
|
|
|
};
|