feat: implement R4 production hardening
- Add configurable ingestion batch item limits - Improve client serialization and sanitization - Add graceful handling for corrupt GZIP partitions in data API - Expand admin session token scope - Update project documentation for R4 status
This commit is contained in:
1 parent
8132087085
commit
ae9b554189
59 files changed
+16879
-1074
No files matched your search
@@ -15,6 +15,22 @@ const REDIS_RATE_LIMIT_LUA_SCRIPT = [
|
||||
"return c",
|
||||
].join("\n");
|
||||
|
||||
const DEFAULT_MAX_BATCH_ITEMS = 1000;
|
||||
const MIN_BATCH_ITEMS = 1;
|
||||
const MAX_BATCH_ITEMS_CEILING = 10000;
|
||||
|
||||
function resolveMaxBatchItems() {
|
||||
const raw = process.env.DATASTACK_MAX_BATCH_ITEMS;
|
||||
if (raw === undefined || String(raw).trim() === "") {
|
||||
return DEFAULT_MAX_BATCH_ITEMS;
|
||||
}
|
||||
const parsed = Math.floor(Number(raw));
|
||||
if (!Number.isFinite(parsed)) {
|
||||
return DEFAULT_MAX_BATCH_ITEMS;
|
||||
}
|
||||
return Math.min(MAX_BATCH_ITEMS_CEILING, Math.max(MIN_BATCH_ITEMS, parsed));
|
||||
}
|
||||
|
||||
function generateFallbackEventId() {
|
||||
return `evt_${Date.now()}_${crypto.randomBytes(6).toString("hex")}`;
|
||||
}
|
||||
@@ -597,6 +613,21 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
return;
|
||||
}
|
||||
|
||||
const maxBatchItems = resolveMaxBatchItems();
|
||||
if (batch.length > maxBatchItems) {
|
||||
if (callback) {
|
||||
callback({
|
||||
success: false,
|
||||
processed: 0,
|
||||
rejected: batch.length,
|
||||
code: "BATCH_LIMIT_EXCEEDED",
|
||||
error: `BATCH_LIMIT_EXCEEDED: Batch size (${batch.length}) exceeds maximum allowed ${maxBatchItems} items per request.`,
|
||||
maxBatchItems,
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const clientIp = socket.data.clientIp || extractSocketClientIp(socket);
|
||||
const dbCheck = await verifyClientInDb(
|
||||
clientId,
|
||||
|
||||
Reference in new issue
Block a user