feat: implement R4 production hardening

- Add configurable ingestion batch item limits
- Improve client serialization and sanitization
- Add graceful handling for corrupt GZIP partitions in data API
- Expand admin session token scope
- Update project documentation for R4 status
This commit is contained in:
Team Dark Devil committed 2026-11-21 22:15:32 +06:00
1 parent 8132087085
commit ae9b554189
59 files changed
+16879 -1074

No files matched your search

+61 -17
View File
@@ -645,8 +645,7 @@ export class DataStackClient {
const msg = String(ack.error || 'Socket event rejected');
const authErr = classifyAuthRejection(code, msg);
if (authErr) {
this.heartbeatStopped = true;
this.stopHeartbeatMonitor();
this.disconnect();
reject(
new DataStackAuthError(msg, authErr.code, authErr.statusCode, ack)
);
@@ -902,8 +901,7 @@ export class DataStackClient {
const msg = String(ack.error || 'Batch rejected');
const authErr = classifyAuthRejection(code, msg);
if (authErr) {
this.heartbeatStopped = true;
this.stopHeartbeatMonitor();
this.disconnect();
reject(
new DataStackAuthError(
msg,
@@ -1055,7 +1053,7 @@ export class DataStackClient {
private async deliverChunkWithSplitAndRetry(
chunk: DataStackEnvelope[],
allowSpoolFallback = true
): Promise<DataStackSendResult> {
): Promise<DataStackSendResult & { unprocessedEnvelopes?: DataStackEnvelope[] }> {
let pending = [...chunk];
let totalAccepted = 0;
let totalDuplicates = 0;
@@ -1084,6 +1082,18 @@ export class DataStackClient {
pending = pending.filter((item) => rateLimitedSet.has(item.eventId));
if (attempt >= this.config.maxRetries) {
if (!allowSpoolFallback) {
return {
success: false,
transport: lastTransport,
accepted: totalAccepted,
duplicates: totalDuplicates,
spooled: totalSpooled,
rateLimitedEventIds: pending.map((p) => p.eventId),
unprocessedItems: pending.map((p) => p.data),
unprocessedEnvelopes: pending,
};
}
throw new DataStackRateLimitError(
`Rate limit exceeded for ${pending.length} event(s) after ${this.config.maxRetries} retries.`,
pending.map((p) => p.eventId)
@@ -1105,34 +1115,57 @@ export class DataStackClient {
pending.slice(mid),
allowSpoolFallback
);
const combinedEnvelopes = [
...(left.unprocessedEnvelopes || []),
...(right.unprocessedEnvelopes || []),
];
const combinedItems = [
...(left.unprocessedItems || []),
...(right.unprocessedItems || []),
];
return {
success: left.success && right.success,
transport: right.transport,
accepted: (left.accepted || 0) + (right.accepted || 0),
duplicates: (left.duplicates || 0) + (right.duplicates || 0),
spooled: (left.spooled || 0) + (right.spooled || 0),
...(combinedItems.length > 0 ? { unprocessedItems: combinedItems } : {}),
...(combinedEnvelopes.length > 0
? { unprocessedEnvelopes: combinedEnvelopes }
: {}),
};
}
if (
err instanceof DataStackRateLimitError &&
attempt < this.config.maxRetries
) {
if (err instanceof DataStackRateLimitError) {
const rlSet = new Set(err.rateLimitedEventIds);
if (rlSet.size > 0 && rlSet.size < pending.length) {
totalAccepted += pending.length - rlSet.size;
pending = pending.filter((p) => rlSet.has(p.eventId));
}
attempt += 1;
await new Promise((r) => setTimeout(r, 50 * attempt));
continue;
if (attempt < this.config.maxRetries) {
attempt += 1;
await new Promise((r) => setTimeout(r, 50 * attempt));
continue;
}
if (!allowSpoolFallback) {
return {
success: false,
transport: lastTransport,
accepted: totalAccepted,
duplicates: totalDuplicates,
spooled: totalSpooled,
rateLimitedEventIds: pending.map((p) => p.eventId),
unprocessedItems: pending.map((p) => p.data),
unprocessedEnvelopes: pending,
};
}
throw err;
}
if (
err instanceof DataStackAuthError ||
err instanceof DataStackQuotaExceededError ||
err instanceof DataStackPayloadTooLargeError ||
err instanceof DataStackRateLimitError ||
err instanceof DataStackConfigError
) {
throw err;
@@ -1160,6 +1193,7 @@ export class DataStackClient {
duplicates: totalDuplicates,
spooled: totalSpooled,
unprocessedItems: pending.map((p) => p.data),
...(pending.length > 0 ? { unprocessedEnvelopes: pending } : {}),
};
}
@@ -1365,12 +1399,22 @@ export class DataStackClient {
const batch = this.spooler.peekBatch(this.config.batchChunkSize);
if (batch.length === 0) break;
const res = await this.deliverChunkWithSplitAndRetry(batch, false);
if (res.success) {
this.spooler.removeFirst(batch.length);
drained += batch.length;
} else {
const processedNow = res.accepted || 0;
const duplicatesNow = res.duplicates || 0;
const remaining = res.unprocessedEnvelopes ?? [];
if (!res.success && processedNow === 0 && duplicatesNow === 0) {
break;
}
if (remaining.length > 0) {
const acked = this.spooler.ackAndRequeueFront(batch.length, remaining);
drained += acked;
break;
}
this.spooler.ackAndRequeueFront(batch.length, []);
drained += batch.length;
}
} finally {
this.isDrainingSpool = false;
+5 -1
View File
@@ -11,7 +11,8 @@
}
},
"scripts": {
"build": "tsc -p tsconfig.json"
"build": "tsc -p tsconfig.json",
"prepare": "npm run build"
},
"files": [
"dist",
@@ -28,6 +29,9 @@
"csv-parse": "^7.0.3",
"socket.io-client": "^4.8.4"
},
"devDependencies": {
"typescript": "^5.9.3"
},
"keywords": [
"datastack",
"telemetry",
+58
View File
@@ -138,6 +138,64 @@ export class DiskSpooler {
}
}
/**
* Atomically removes `ackedCount` head lines from `spool.jsonl` while
* preserving any `unprocessed` envelopes at the FRONT of the queue in a
* single atomic temp-file + `fsyncSync` + `renameSync` operation (FIFO safe).
*/
public ackAndRequeueFront(
ackedCount: number,
unprocessed: DataStackEnvelope[] = []
): number {
if (!this.enabled || ackedCount <= 0 || !fs.existsSync(this.spoolFile)) {
return 0;
}
const raw = fs.readFileSync(this.spoolFile, 'utf8');
const allLines = raw
.split('\n')
.map((l) => l.trim())
.filter(Boolean);
const consumed = Math.min(Math.floor(ackedCount), allLines.length);
const tailLines = allLines.slice(consumed);
const headLines = unprocessed.map((evt) => JSON.stringify(evt));
const combined = [...headLines, ...tailLines];
if (combined.length === 0) {
try {
fs.unlinkSync(this.spoolFile);
} catch {
// Ignore if already removed
}
return consumed;
}
this.ensureDir();
const tmpFile = `${this.spoolFile}.${process.pid}.${Date.now()}.tmp`;
const payload = combined.join('\n') + '\n';
try {
const fd = fs.openSync(tmpFile, 'w');
try {
fs.writeFileSync(fd, payload, 'utf8');
fs.fsyncSync(fd);
} finally {
fs.closeSync(fd);
}
fs.renameSync(tmpFile, this.spoolFile);
} catch (err) {
try {
if (fs.existsSync(tmpFile)) {
fs.unlinkSync(tmpFile);
}
} catch {
// Ignore cleanup failure
}
throw err;
}
return Math.max(0, consumed - headLines.length);
}
public count(): number {
if (!this.enabled || !fs.existsSync(this.spoolFile)) {
return 0;