Files
DataStack/server/socket/handlers.js
T

1012 lines
32 KiB
JavaScript
Raw Normal View History

const crypto = require("node:crypto");
2026-10-23 19:56:38 -07:00
const jwt = require("jsonwebtoken");
2026-10-23 11:43:49 -05:00
const IDEMPOTENCY_TTL_MS = 5 * 60 * 1000; // 5 minutes
const IDEMPOTENCY_MAX_ENTRIES = 20000;
const CLEANUP_INTERVAL_MS = 60 * 1000;
const REDIS_IDEMPOTENCY_PREFIX = "datastack:idem:";
const REDIS_RATE_LIMIT_PREFIX = "datastack:rl:";
const REDIS_RATE_LIMIT_LUA_SCRIPT = [
"-- DATASTACK_RATE_LIMIT_INCR",
'local c = redis.call("INCR", KEYS[1])',
'if c == 1 or redis.call("PTTL", KEYS[1]) < 0 then',
' redis.call("PEXPIRE", KEYS[1], tonumber(ARGV[1]))',
"end",
"return c",
].join("\n");
2026-11-21 22:15:32 +06:00
const DEFAULT_MAX_BATCH_ITEMS = 1000;
const MIN_BATCH_ITEMS = 1;
const MAX_BATCH_ITEMS_CEILING = 10000;
function resolveMaxBatchItems() {
const raw = process.env.DATASTACK_MAX_BATCH_ITEMS;
if (raw === undefined || String(raw).trim() === "") {
return DEFAULT_MAX_BATCH_ITEMS;
}
const parsed = Math.floor(Number(raw));
if (!Number.isFinite(parsed)) {
return DEFAULT_MAX_BATCH_ITEMS;
}
return Math.min(MAX_BATCH_ITEMS_CEILING, Math.max(MIN_BATCH_ITEMS, parsed));
}
function generateFallbackEventId() {
return `evt_${Date.now()}_${crypto.randomBytes(6).toString("hex")}`;
}
function normalizeClientId(clientId) {
return String(clientId || "")
.trim()
.toLowerCase()
.replace(/[^a-z0-9_-]/g, "_");
}
function extractSocketClientIp(socket) {
const headers = (socket && socket.handshake && socket.handshake.headers) || {};
const realIp =
typeof headers["x-real-ip"] === "string" ? headers["x-real-ip"].trim() : "";
if (realIp) {
return realIp.slice(0, 128);
}
const forwardedFor =
typeof headers["x-forwarded-for"] === "string"
? headers["x-forwarded-for"]
: "";
if (forwardedFor) {
const hops = forwardedFor
.split(",")
.map((part) => part.trim())
.filter(Boolean);
const rightmost = hops[hops.length - 1];
if (rightmost) {
return rightmost.slice(0, 128);
}
}
const addr =
(socket && socket.handshake && socket.handshake.address) ||
(socket && socket.conn && socket.conn.remoteAddress) ||
"";
return String(addr).trim() || "127.0.0.1";
}
2026-10-23 11:43:49 -05:00
function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
const getRuntimeConfig =
typeof options.getRuntimeConfig === "function"
? options.getRuntimeConfig
: () => ({
jwtSecret:
options.jwtSecret ||
(process.env.DATASTACK_JWT_SECRET || "").trim() ||
(process.env.NODE_ENV === "production"
? ""
: "datastack-jwt-default-secret-key-32-chars-long"),
maxEventsPerSec: 100,
maxPayloadSizeBytes: 100 * 1024,
batchFlushThreshold: 500,
enableIdempotency: true,
});
const verifyClientInDb =
typeof options.verifyClientInDb === "function"
? options.verifyClientInDb
: async () => ({ allowed: true });
const recordClientActivity =
typeof options.recordClientActivity === "function"
? options.recordClientActivity
: () => undefined;
const onBufferPush =
typeof options.onBufferPush === "function"
? options.onBufferPush
: () => undefined;
const getRedisClient =
typeof options.getRedisClient === "function"
? options.getRedisClient
: () => null;
const triggerWebhookAlert =
typeof options.triggerWebhookAlert === "function"
? options.triggerWebhookAlert
: null;
const WEBHOOK_REJECTION_CODES = new Set([
"CLIENT_TOKEN_EXPIRED",
"CLIENT_IP_FORBIDDEN",
"DISK_SPACE_LOW",
"STORAGE_QUOTA_EXCEEDED",
]);
function dispatchRejectionWebhook(code, reason, clientId, details) {
if (!triggerWebhookAlert || !WEBHOOK_REJECTION_CODES.has(code)) return;
const severity = code === "DISK_SPACE_LOW" ? "critical" : "warning";
try {
Promise.resolve(
triggerWebhookAlert({
event: code,
severity,
clientId,
message:
reason ||
`${code}: Ingestion request rejected on Port 4000 Socket.IO.`,
details: details || { transport: "socketio" },
}),
).catch(() => undefined);
} catch {
// Never block socket handlers on webhook alert dispatch
}
}
2026-10-23 11:43:49 -05:00
const idempotencyMap = new Map();
const rateLimitMap = new Map();
let lastCleanupAt = Date.now();
function cleanupMaps(now, force = false) {
if (
!force &&
now - lastCleanupAt < CLEANUP_INTERVAL_MS &&
idempotencyMap.size <= IDEMPOTENCY_MAX_ENTRIES
) {
return;
}
lastCleanupAt = now;
for (const [key, timestamp] of idempotencyMap.entries()) {
if (now - timestamp > IDEMPOTENCY_TTL_MS) {
idempotencyMap.delete(key);
}
}
for (const [clientId, rl] of rateLimitMap.entries()) {
if (now - rl.start > 10000) {
rateLimitMap.delete(clientId);
}
}
}
function trackLocalRateWindow(clientId, now) {
const rl = rateLimitMap.get(clientId);
if (!rl || now - rl.start >= 1000) {
rateLimitMap.set(clientId, { count: 1, start: now });
} else {
rl.count += 1;
}
}
function supportsSharedSecurityRedis(redis) {
return Boolean(
redis &&
typeof redis.get === "function" &&
typeof redis.incr === "function" &&
typeof redis.pexpire === "function" &&
typeof redis.set === "function",
);
}
function getSecurityStats() {
cleanupMaps(Date.now(), true);
const config = getRuntimeConfig();
return {
activeIdempotencyKeys: idempotencyMap.size,
rateLimitedClientsCount: rateLimitMap.size,
maxPayloadSizeBytes: Number(config.maxPayloadSizeBytes) || 100 * 1024,
};
}
function validateSingleEventLocal(
clientId,
eventId,
rawData,
clientRateLimit,
explicitByteLength,
) {
const config = getRuntimeConfig();
const now = Date.now();
cleanupMaps(now);
// 1. Idempotency check (read-only check first; record only after rate-limit & payload checks pass)
const idempotencyKey = `${clientId}:${eventId}`;
if (config.enableIdempotency && eventId) {
const existingTime = idempotencyMap.get(idempotencyKey);
if (existingTime && now - existingTime < IDEMPOTENCY_TTL_MS) {
return {
allowed: false,
duplicate: true,
code: "DUPLICATE_EVENT",
reason: "Duplicate event dropped",
};
}
}
// 2. Payload byte size check (before consuming rate-limit quota)
let payloadStr = "";
let byteLength = 0;
if (
typeof explicitByteLength === "number" &&
Number.isFinite(explicitByteLength) &&
explicitByteLength >= 0
) {
byteLength = explicitByteLength;
} else {
payloadStr =
typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {});
byteLength = Buffer.byteLength(payloadStr, "utf8");
}
const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024;
if (byteLength > maxBytes) {
return {
allowed: false,
duplicate: false,
code: "PAYLOAD_TOO_LARGE",
reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`,
};
}
// 3. Rate limit check
const effectiveLimit = Math.min(
Number(clientRateLimit) || 100,
Number(config.maxEventsPerSec) || 100,
);
const rl = rateLimitMap.get(clientId);
if (!rl || now - rl.start >= 1000) {
rateLimitMap.set(clientId, { count: 1, start: now });
} else if (rl.count >= effectiveLimit) {
return {
allowed: false,
duplicate: false,
code: "RATE_LIMIT_EXCEEDED",
reason: `RATE_LIMIT_EXCEEDED: Limit of ${effectiveLimit} events/sec exceeded`,
};
} else {
rl.count += 1;
}
// 4. Record idempotency key ONLY after event passes all checks
if (config.enableIdempotency && eventId) {
idempotencyMap.set(idempotencyKey, now);
}
return { allowed: true, payloadStr };
}
2026-10-23 11:43:49 -05:00
async function validateSingleEventAsync(
clientId,
eventId,
rawData,
clientRateLimit,
) {
const redis = getRedisClient();
if (!supportsSharedSecurityRedis(redis)) {
return validateSingleEventLocal(
clientId,
eventId,
rawData,
clientRateLimit,
);
}
const config = getRuntimeConfig();
const now = Date.now();
cleanupMaps(now);
const idempotencyKey = `${clientId}:${eventId}`;
const redisIdemKey = `${REDIS_IDEMPOTENCY_PREFIX}${clientId}:${eventId}`;
try {
// 1. Read-only idempotency check
if (config.enableIdempotency && eventId) {
const localTime = idempotencyMap.get(idempotencyKey);
if (localTime && now - localTime < IDEMPOTENCY_TTL_MS) {
return {
allowed: false,
duplicate: true,
code: "DUPLICATE_EVENT",
reason: "Duplicate event dropped",
};
}
const existing = await redis.get(redisIdemKey);
if (existing !== null) {
idempotencyMap.set(idempotencyKey, now);
return {
allowed: false,
duplicate: true,
code: "DUPLICATE_EVENT",
reason: "Duplicate event dropped",
};
}
}
// 2. Payload byte size check
const payloadStr =
typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {});
const byteLength = Buffer.byteLength(payloadStr, "utf8");
const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024;
if (byteLength > maxBytes) {
return {
allowed: false,
duplicate: false,
code: "PAYLOAD_TOO_LARGE",
reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`,
};
}
// 3. Shared atomic Redis rate limit check (single Lua round-trip)
const effectiveLimit = Math.min(
Number(clientRateLimit) || 100,
Number(config.maxEventsPerSec) || 100,
);
const windowBucket = Math.floor(now / 1000);
const rlKey = `${REDIS_RATE_LIMIT_PREFIX}${clientId}:${windowBucket}`;
let count;
if (typeof redis.eval === "function") {
count = Number(
await redis.eval(REDIS_RATE_LIMIT_LUA_SCRIPT, 1, rlKey, "2000"),
);
} else {
count = Number(await redis.incr(rlKey));
if (count === 1) {
await redis.pexpire(rlKey, 2000);
}
}
trackLocalRateWindow(clientId, now);
if (count > effectiveLimit) {
return {
allowed: false,
duplicate: false,
code: "RATE_LIMIT_EXCEEDED",
reason: `RATE_LIMIT_EXCEEDED: Limit of ${effectiveLimit} events/sec exceeded`,
};
}
// 4. Atomic claim of idempotency key after all checks succeed
if (config.enableIdempotency && eventId) {
const claimed = await redis.set(
redisIdemKey,
"1",
"PX",
IDEMPOTENCY_TTL_MS,
"NX",
);
idempotencyMap.set(idempotencyKey, now);
if (claimed !== "OK") {
return {
allowed: false,
duplicate: true,
code: "DUPLICATE_EVENT",
reason: "Duplicate event dropped",
};
}
}
return { allowed: true, payloadStr };
} catch {
return validateSingleEventLocal(
clientId,
eventId,
rawData,
clientRateLimit,
);
}
}
2026-10-23 11:43:49 -05:00
// Middleware: Handshake Auth
io.use(async (socket, next) => {
2026-10-23 11:43:49 -05:00
const authHeader =
socket.handshake.auth?.token ||
socket.handshake.headers?.authorization ||
socket.handshake.query?.token;
if (!authHeader) {
2026-10-23 19:56:38 -07:00
return next(new Error("AUTH_REQUIRED: Handshake missing JWT"));
2026-10-23 11:43:49 -05:00
}
try {
const cleanToken = String(authHeader).startsWith("Bearer ")
? String(authHeader).slice(7).trim()
: String(authHeader).trim();
const config = getRuntimeConfig();
const minSecretLen = process.env.NODE_ENV === "production" ? 32 : 1;
if (!config.jwtSecret || config.jwtSecret.length < minSecretLen) {
return next(
new Error("AUTH_UNAVAILABLE: DATASTACK_JWT_SECRET is not configured"),
);
}
const decoded = jwt.verify(cleanToken, config.jwtSecret);
const normalizedClientId = normalizeClientId(decoded.clientId);
const clientIp = extractSocketClientIp(socket);
const dbCheck = await verifyClientInDb(
normalizedClientId,
cleanToken,
true,
clientIp,
);
if (!dbCheck.allowed) {
const errCode = dbCheck.code || "CLIENT_REVOKED";
dispatchRejectionWebhook(errCode, dbCheck.reason, normalizedClientId, {
transport: "socketio_handshake",
clientIp,
});
return next(new Error(dbCheck.reason || errCode));
}
socket.data.clientId = normalizedClientId;
socket.data.token = cleanToken;
socket.data.clientIp = clientIp;
socket.data.rateLimit =
dbCheck.rateLimit || decoded.rateLimit || config.maxEventsPerSec || 100;
2026-10-23 11:43:49 -05:00
next();
} catch (err) {
next(new Error(`AUTH_INVALID: ${err.message}`));
}
});
const ENVELOPE_KEYS = new Set(["eventId", "timestamp", "data", "clientId"]);
function extractBatchItemData(safeItem) {
return extractSingleEventData(safeItem);
}
function extractSingleEventData(rawPayload) {
if (
rawPayload === null ||
rawPayload === undefined ||
typeof rawPayload !== "object" ||
Array.isArray(rawPayload)
) {
return rawPayload === undefined ? {} : rawPayload;
}
if (rawPayload.data !== undefined) {
const keys = Object.keys(rawPayload);
if (keys.every((k) => ENVELOPE_KEYS.has(k))) {
return rawPayload.data;
}
return rawPayload;
}
const stripped = {};
let hasNonEnvelopeKey = false;
for (const [k, v] of Object.entries(rawPayload)) {
if (k === "eventId" || k === "timestamp" || k === "clientId") {
continue;
}
hasNonEnvelopeKey = true;
stripped[k] = v;
}
return hasNonEnvelopeKey ? stripped : {};
}
// P02 R-01: an idempotency claim is created before persistence. After a
// crash that claim may survive without its event. Prove the event reached
// Redis's buffer or a CSV partition before issuing a duplicate-success ACK.
async function hasDurableEvent(clientId, eventId, timestamp) {
const ackKey = `\u0000durably-acked:${clientId}:${eventId}`;
const previousAck = idempotencyMap.get(ackKey);
if (typeof previousAck === 'number' && Date.now() - previousAck < IDEMPOTENCY_TTL_MS) {
return true;
}
const redis = getRedisClient();
if (redis && typeof redis.lrange === 'function') {
try {
// Bounded read: reject uncertain old claims instead of loading a huge Redis queue.
const queued = await redis.lrange('datastack:buffer:events', -512, -1);
if (Array.isArray(queued) && queued.some((raw) => {
try {
const evt = JSON.parse(raw);
return evt && evt.clientId === clientId && evt.eventId === eventId;
} catch { return false; }
})) {
idempotencyMap.set(ackKey, Date.now());
return true;
}
} catch {
// Redis cannot prove durability; fall through to read-only CSV proof.
}
}
if (!worker || typeof worker.sanitizeClientId !== 'function' ||
typeof worker.extractSafePartitionDate !== 'function' ||
typeof worker.isPartitionFilename !== 'function' ||
typeof worker.extractDateFromFilename !== 'function') return false;
const fs = require('node:fs');
const path = require('node:path');
const { pipeline } = require('node:stream/promises');
const { Writable } = require('node:stream');
const { parse } = require('csv-parse');
const zlib = require('node:zlib');
const aborter = new AbortController();
const deadline = setTimeout(() => aborter.abort(), 1800);
deadline.unref?.();
try {
const base = await fs.promises.realpath(worker.baseDir);
const directory = path.resolve(base, worker.sanitizeClientId(clientId));
const realDir = await fs.promises.realpath(directory);
if (!realDir.startsWith(base + path.sep)) return false;
const date = worker.extractSafePartitionDate(timestamp);
const files = await fs.promises.readdir(realDir, { withFileTypes: true });
let scannedBytes = 0;
for (const item of files) {
if (!item.isFile() || !worker.isPartitionFilename(item.name) ||
worker.extractDateFromFilename(item.name) !== date) continue;
const filename = path.join(realDir, item.name);
scannedBytes += (await fs.promises.stat(filename)).size;
if (scannedBytes > 256 * 1024 * 1024) return false;
let found = false;
const stages = [fs.createReadStream(filename)];
if (item.name.endsWith('.gz')) stages.push(zlib.createGunzip());
stages.push(parse({ columns: true, bom: true, skip_empty_lines: true }));
stages.push(new Writable({
objectMode: true,
write(row, _encoding, done) {
if (row.clientId === clientId && row.eventId === eventId) found = true;
done();
},
}));
await pipeline(...stages, { signal: aborter.signal });
if (found) {
idempotencyMap.set(ackKey, Date.now());
return true;
}
}
} catch {
// Missing, corrupt, unreadable or slow CSV cannot justify positive ACK.
} finally {
clearTimeout(deadline);
}
return false;
}
2026-10-23 19:56:38 -07:00
io.on("connection", (socket) => {
2026-10-23 11:43:49 -05:00
const clientId = socket.data.clientId;
socket.join(`client:${clientId}`);
// Single event
socket.on("sync_data", async (payload, callback) => {
2026-10-23 11:43:49 -05:00
try {
const clientIp = socket.data.clientIp || extractSocketClientIp(socket);
const dbCheck = await verifyClientInDb(
clientId,
socket.data.token,
false,
clientIp,
);
if (!dbCheck.allowed) {
const errCode = dbCheck.code || "CLIENT_REVOKED";
dispatchRejectionWebhook(errCode, dbCheck.reason, clientId, {
transport: "socketio_sync_data",
clientIp,
});
if (callback) {
callback({
success: false,
code: errCode,
error: dbCheck.reason || "CLIENT_REVOKED",
});
}
socket.disconnect(true);
2026-10-23 11:43:49 -05:00
return;
}
if (worker && typeof worker.checkStorageAdmission === "function") {
const config = getRuntimeConfig();
const admission = await worker.checkStorageAdmission(
clientId,
dbCheck.storageQuotaMb || 0,
config.diskPauseThresholdPct ?? 95,
);
if (!admission.allowed) {
dispatchRejectionWebhook(admission.code, admission.reason, clientId, {
transport: "socketio_sync_data",
clientIp,
});
if (callback) {
callback({
success: false,
code: admission.code,
error: admission.reason,
});
}
return;
}
}
const effectiveClientRate =
dbCheck.rateLimit || socket.data.rateLimit || 100;
const safePayload =
payload && typeof payload === "object" && !Array.isArray(payload)
? payload
: {};
const eventId =
typeof safePayload.eventId === "string" && safePayload.eventId.trim()
? safePayload.eventId.trim()
: generateFallbackEventId();
const timestamp =
typeof safePayload.timestamp === "string" &&
safePayload.timestamp.trim()
? safePayload.timestamp.trim()
: new Date().toISOString();
const eventData = extractSingleEventData(payload);
// A previous Redis write failure can leave an uncommitted idempotency
// claim. Never report that claim as a successful durable duplicate.
const check = await validateSingleEventAsync(
clientId,
eventId,
eventData,
effectiveClientRate,
);
if (!check.allowed) {
if (check.duplicate) {
if (await hasDurableEvent(clientId, eventId, timestamp)) {
idempotencyMap.delete(`\u0000failed-persist:${clientId}:${eventId}`);
if (callback) callback({ success: true, eventId, note: "DUPLICATE_DROPPED" });
} else {
// A stale local claim must not block a valid same-ID SDK retry
// after Redis recovers; the Redis claim still guards other workers.
idempotencyMap.delete(`${clientId}:${eventId}`);
if (callback) callback({
success: false, eventId, code: "BUFFER_PERSIST_FAILED",
error: "BUFFER_PERSIST_FAILED: Idempotency claim has no durable event",
});
}
return;
}
if (callback) {
callback({
success: false,
eventId,
code: check.code,
error: check.reason,
});
}
2026-10-23 11:43:49 -05:00
return;
}
const event = {
clientId,
eventId,
timestamp,
payload: check.payloadStr,
2026-10-23 11:43:49 -05:00
};
bufferQueue.push(event);
recordClientActivity(clientId, 1);
try {
await onBufferPush();
} catch (persistErr) {
// Invalidate the pre-persistence idempotency claim so SDK retry can
// resubmit after Redis recovers. Keep the volatile RAM event pending.
const claimKey = `${clientId}:${eventId}`;
idempotencyMap.delete(claimKey);
idempotencyMap.set(`\u0000failed-persist:${claimKey}`, Date.now());
const claimRedis = getRedisClient();
if (claimRedis && typeof claimRedis.del === "function") {
try {
await claimRedis.del(`${REDIS_IDEMPOTENCY_PREFIX}${claimKey}`);
idempotencyMap.delete(claimKey);
} catch {
// Claim may remain: negative ACK must be retained for retries.
}
}
if (callback) {
callback({
success: false,
eventId,
code: "BUFFER_PERSIST_FAILED",
error:
persistErr instanceof Error
? persistErr.message
: "Failed to persist event to buffer",
});
}
return;
}
2026-10-23 11:43:49 -05:00
// Clear any failed local persistence marker only after a durable write.
idempotencyMap.delete(`\u0000failed-persist:${clientId}:${eventId}`);
idempotencyMap.set(`\u0000durably-acked:${clientId}:${eventId}`, Date.now());
2026-10-23 11:43:49 -05:00
if (callback) callback({ success: true, eventId });
} catch (err) {
if (callback) callback({ success: false, error: err.message });
}
});
// Batch event
socket.on("batch_sync", async (batch, callback) => {
try {
if (!Array.isArray(batch)) {
if (callback) callback({ success: false, error: "Expected array" });
return;
}
2026-10-23 11:43:49 -05:00
2026-11-21 22:15:32 +06:00
const maxBatchItems = resolveMaxBatchItems();
if (batch.length > maxBatchItems) {
if (callback) {
callback({
success: false,
processed: 0,
rejected: batch.length,
code: "BATCH_LIMIT_EXCEEDED",
error: `BATCH_LIMIT_EXCEEDED: Batch size (${batch.length}) exceeds maximum allowed ${maxBatchItems} items per request.`,
maxBatchItems,
});
}
return;
}
const clientIp = socket.data.clientIp || extractSocketClientIp(socket);
const dbCheck = await verifyClientInDb(
2026-10-23 11:43:49 -05:00
clientId,
socket.data.token,
false,
clientIp,
);
if (!dbCheck.allowed) {
const errCode = dbCheck.code || "CLIENT_REVOKED";
dispatchRejectionWebhook(errCode, dbCheck.reason, clientId, {
transport: "socketio_batch_sync",
clientIp,
});
if (callback) {
callback({
success: false,
processed: 0,
rejected: batch.length,
code: errCode,
error: dbCheck.reason || "CLIENT_REVOKED",
});
}
socket.disconnect(true);
return;
}
2026-10-23 11:43:49 -05:00
if (worker && typeof worker.checkStorageAdmission === "function") {
const config = getRuntimeConfig();
const admission = await worker.checkStorageAdmission(
clientId,
dbCheck.storageQuotaMb || 0,
config.diskPauseThresholdPct ?? 95,
);
if (!admission.allowed) {
dispatchRejectionWebhook(admission.code, admission.reason, clientId, {
transport: "socketio_batch_sync",
clientIp,
});
if (callback) {
callback({
success: false,
processed: 0,
rejected: batch.length,
code: admission.code,
error: admission.reason,
});
}
return;
}
}
const effectiveClientRate =
dbCheck.rateLimit || socket.data.rateLimit || 100;
let processed = 0;
let rejected = 0;
let duplicates = 0;
let hitRateLimit = false;
let hitPayloadLimit = false;
const rateLimitedEventIds = [];
const oversizedEventIds = [];
const errors = [];
const acceptedEventIds = [];
let volatileDuplicate = false;
for (const item of batch) {
const safeItem = item && typeof item === "object" ? item : {};
const eventId =
typeof safeItem.eventId === "string" && safeItem.eventId.trim()
? safeItem.eventId.trim()
: generateFallbackEventId();
const timestamp =
typeof safeItem.timestamp === "string" && safeItem.timestamp.trim()
? safeItem.timestamp.trim()
: new Date().toISOString();
const rawData = extractBatchItemData(safeItem);
const check = await validateSingleEventAsync(
clientId,
eventId,
rawData,
effectiveClientRate,
);
if (!check.allowed) {
if (check.duplicate) {
if (await hasDurableEvent(clientId, eventId, timestamp)) {
idempotencyMap.delete(`\u0000failed-persist:${clientId}:${eventId}`);
duplicates += 1;
} else {
idempotencyMap.delete(`${clientId}:${eventId}`);
rejected += 1;
volatileDuplicate = true;
if (errors.length < 5) errors.push(
`${eventId}: BUFFER_PERSIST_FAILED: Idempotency claim has no durable event`);
}
continue;
}
rejected += 1;
if (check.code === "RATE_LIMIT_EXCEEDED") {
hitRateLimit = true;
rateLimitedEventIds.push(eventId);
}
if (check.code === "PAYLOAD_TOO_LARGE") {
hitPayloadLimit = true;
oversizedEventIds.push(eventId);
}
if (errors.length < 5) {
errors.push(`${eventId}: ${check.reason}`);
}
continue;
}
bufferQueue.push({
clientId,
eventId,
timestamp,
payload: check.payloadStr,
});
processed += 1;
acceptedEventIds.push(eventId);
}
if (processed > 0) {
recordClientActivity(clientId, processed);
try {
await onBufferPush();
} catch (persistErr) {
const claimRedis = getRedisClient();
for (const eventId of acceptedEventIds) {
const claimKey = `${clientId}:${eventId}`;
idempotencyMap.delete(claimKey);
idempotencyMap.set(`\u0000failed-persist:${claimKey}`, Date.now());
if (claimRedis && typeof claimRedis.del === "function") {
try {
await claimRedis.del(`${REDIS_IDEMPOTENCY_PREFIX}${claimKey}`);
idempotencyMap.delete(claimKey);
} catch {
// Preserve negative-ACK retry guard if claim removal fails.
}
}
}
if (callback) {
callback({
success: false,
processed: 0,
rejected: batch.length,
duplicates,
code: "BUFFER_PERSIST_FAILED",
error:
persistErr instanceof Error
? persistErr.message
: "Failed to persist batch to buffer",
});
}
return;
}
}
for (const persistedId of acceptedEventIds) {
idempotencyMap.delete(`\u0000failed-persist:${clientId}:${persistedId}`);
idempotencyMap.set(`\u0000durably-acked:${clientId}:${persistedId}`, Date.now());
}
// Never ACK a mixed batch as successful while any event remains volatile.
if (volatileDuplicate) {
if (callback) callback({
success: false, processed, rejected, duplicates,
code: "BUFFER_PERSIST_FAILED",
error: "BUFFER_PERSIST_FAILED: Batch contains non-durable event",
errors,
});
return;
}
if (processed === 0 && duplicates === 0 && rejected > 0) {
const code = volatileDuplicate
? "BUFFER_PERSIST_FAILED"
: hitPayloadLimit
? "PAYLOAD_TOO_LARGE"
: hitRateLimit
? "RATE_LIMIT_EXCEEDED"
: "BATCH_REJECTED";
if (callback) {
callback({
success: false,
processed,
rejected,
duplicates,
code,
error: errors[0] || "Batch validation failed",
errors,
...(rateLimitedEventIds.length > 0 ? { rateLimitedEventIds } : {}),
...(oversizedEventIds.length > 0 ? { oversizedEventIds } : {}),
});
}
return;
}
if (callback) {
callback({
success: true,
processed,
rejected,
duplicates,
...(rateLimitedEventIds.length > 0 ? { rateLimitedEventIds } : {}),
...(oversizedEventIds.length > 0 ? { oversizedEventIds } : {}),
...(errors.length > 0 ? { errors } : {}),
});
}
} catch (err) {
if (callback) {
callback({
success: false,
processed: 0,
rejected: Array.isArray(batch) ? batch.length : 0,
error: err.message,
});
}
}
2026-10-23 11:43:49 -05:00
});
// Heartbeat & Reconnection Verification Event
socket.on("ping_verify", async (_payload, callback) => {
try {
const clientIp = socket.data.clientIp || extractSocketClientIp(socket);
const dbCheck = await verifyClientInDb(
clientId,
socket.data.token,
true,
clientIp,
);
if (!dbCheck.allowed) {
const errCode = dbCheck.code || "CLIENT_REVOKED";
dispatchRejectionWebhook(errCode, dbCheck.reason, clientId, {
transport: "socketio_ping_verify",
clientIp,
});
if (callback) {
callback({
success: false,
status: "REVOKED",
code: errCode,
error: dbCheck.reason || "CLIENT_REVOKED",
});
}
socket.disconnect(true);
return;
}
const config = getRuntimeConfig();
if (callback) {
callback({
success: true,
status: "VERIFIED",
clientId,
rateLimit: dbCheck.rateLimit || socket.data.rateLimit || 100,
maxPayloadSizeBytes: config.maxPayloadSizeBytes,
serverTime: new Date().toISOString(),
});
}
} catch (err) {
if (callback) {
callback({
success: false,
status: "ERROR",
error: err.message,
});
}
}
});
2026-10-23 11:43:49 -05:00
});
return {
getSecurityStats,
validateSingleEventLocal,
};
2026-10-23 11:43:49 -05:00
}
module.exports = { setupSocketHandlers, normalizeClientId, extractSocketClientIp };