feat: implement R4 production hardening
- Add configurable ingestion batch item limits - Improve client serialization and sanitization - Add graceful handling for corrupt GZIP partitions in data API - Expand admin session token scope - Update project documentation for R4 status
This commit is contained in:
1 parent
8132087085
commit
ae9b554189
59 files changed
+16879
-1074
No files matched your search
+61
-17
@@ -645,8 +645,7 @@ export class DataStackClient {
|
||||
const msg = String(ack.error || 'Socket event rejected');
|
||||
const authErr = classifyAuthRejection(code, msg);
|
||||
if (authErr) {
|
||||
this.heartbeatStopped = true;
|
||||
this.stopHeartbeatMonitor();
|
||||
this.disconnect();
|
||||
reject(
|
||||
new DataStackAuthError(msg, authErr.code, authErr.statusCode, ack)
|
||||
);
|
||||
@@ -902,8 +901,7 @@ export class DataStackClient {
|
||||
const msg = String(ack.error || 'Batch rejected');
|
||||
const authErr = classifyAuthRejection(code, msg);
|
||||
if (authErr) {
|
||||
this.heartbeatStopped = true;
|
||||
this.stopHeartbeatMonitor();
|
||||
this.disconnect();
|
||||
reject(
|
||||
new DataStackAuthError(
|
||||
msg,
|
||||
@@ -1055,7 +1053,7 @@ export class DataStackClient {
|
||||
private async deliverChunkWithSplitAndRetry(
|
||||
chunk: DataStackEnvelope[],
|
||||
allowSpoolFallback = true
|
||||
): Promise<DataStackSendResult> {
|
||||
): Promise<DataStackSendResult & { unprocessedEnvelopes?: DataStackEnvelope[] }> {
|
||||
let pending = [...chunk];
|
||||
let totalAccepted = 0;
|
||||
let totalDuplicates = 0;
|
||||
@@ -1084,6 +1082,18 @@ export class DataStackClient {
|
||||
pending = pending.filter((item) => rateLimitedSet.has(item.eventId));
|
||||
|
||||
if (attempt >= this.config.maxRetries) {
|
||||
if (!allowSpoolFallback) {
|
||||
return {
|
||||
success: false,
|
||||
transport: lastTransport,
|
||||
accepted: totalAccepted,
|
||||
duplicates: totalDuplicates,
|
||||
spooled: totalSpooled,
|
||||
rateLimitedEventIds: pending.map((p) => p.eventId),
|
||||
unprocessedItems: pending.map((p) => p.data),
|
||||
unprocessedEnvelopes: pending,
|
||||
};
|
||||
}
|
||||
throw new DataStackRateLimitError(
|
||||
`Rate limit exceeded for ${pending.length} event(s) after ${this.config.maxRetries} retries.`,
|
||||
pending.map((p) => p.eventId)
|
||||
@@ -1105,34 +1115,57 @@ export class DataStackClient {
|
||||
pending.slice(mid),
|
||||
allowSpoolFallback
|
||||
);
|
||||
const combinedEnvelopes = [
|
||||
...(left.unprocessedEnvelopes || []),
|
||||
...(right.unprocessedEnvelopes || []),
|
||||
];
|
||||
const combinedItems = [
|
||||
...(left.unprocessedItems || []),
|
||||
...(right.unprocessedItems || []),
|
||||
];
|
||||
return {
|
||||
success: left.success && right.success,
|
||||
transport: right.transport,
|
||||
accepted: (left.accepted || 0) + (right.accepted || 0),
|
||||
duplicates: (left.duplicates || 0) + (right.duplicates || 0),
|
||||
spooled: (left.spooled || 0) + (right.spooled || 0),
|
||||
...(combinedItems.length > 0 ? { unprocessedItems: combinedItems } : {}),
|
||||
...(combinedEnvelopes.length > 0
|
||||
? { unprocessedEnvelopes: combinedEnvelopes }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
err instanceof DataStackRateLimitError &&
|
||||
attempt < this.config.maxRetries
|
||||
) {
|
||||
if (err instanceof DataStackRateLimitError) {
|
||||
const rlSet = new Set(err.rateLimitedEventIds);
|
||||
if (rlSet.size > 0 && rlSet.size < pending.length) {
|
||||
totalAccepted += pending.length - rlSet.size;
|
||||
pending = pending.filter((p) => rlSet.has(p.eventId));
|
||||
}
|
||||
attempt += 1;
|
||||
await new Promise((r) => setTimeout(r, 50 * attempt));
|
||||
continue;
|
||||
if (attempt < this.config.maxRetries) {
|
||||
attempt += 1;
|
||||
await new Promise((r) => setTimeout(r, 50 * attempt));
|
||||
continue;
|
||||
}
|
||||
if (!allowSpoolFallback) {
|
||||
return {
|
||||
success: false,
|
||||
transport: lastTransport,
|
||||
accepted: totalAccepted,
|
||||
duplicates: totalDuplicates,
|
||||
spooled: totalSpooled,
|
||||
rateLimitedEventIds: pending.map((p) => p.eventId),
|
||||
unprocessedItems: pending.map((p) => p.data),
|
||||
unprocessedEnvelopes: pending,
|
||||
};
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
if (
|
||||
err instanceof DataStackAuthError ||
|
||||
err instanceof DataStackQuotaExceededError ||
|
||||
err instanceof DataStackPayloadTooLargeError ||
|
||||
err instanceof DataStackRateLimitError ||
|
||||
err instanceof DataStackConfigError
|
||||
) {
|
||||
throw err;
|
||||
@@ -1160,6 +1193,7 @@ export class DataStackClient {
|
||||
duplicates: totalDuplicates,
|
||||
spooled: totalSpooled,
|
||||
unprocessedItems: pending.map((p) => p.data),
|
||||
...(pending.length > 0 ? { unprocessedEnvelopes: pending } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1365,12 +1399,22 @@ export class DataStackClient {
|
||||
const batch = this.spooler.peekBatch(this.config.batchChunkSize);
|
||||
if (batch.length === 0) break;
|
||||
const res = await this.deliverChunkWithSplitAndRetry(batch, false);
|
||||
if (res.success) {
|
||||
this.spooler.removeFirst(batch.length);
|
||||
drained += batch.length;
|
||||
} else {
|
||||
const processedNow = res.accepted || 0;
|
||||
const duplicatesNow = res.duplicates || 0;
|
||||
const remaining = res.unprocessedEnvelopes ?? [];
|
||||
|
||||
if (!res.success && processedNow === 0 && duplicatesNow === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (remaining.length > 0) {
|
||||
const acked = this.spooler.ackAndRequeueFront(batch.length, remaining);
|
||||
drained += acked;
|
||||
break;
|
||||
}
|
||||
|
||||
this.spooler.ackAndRequeueFront(batch.length, []);
|
||||
drained += batch.length;
|
||||
}
|
||||
} finally {
|
||||
this.isDrainingSpool = false;
|
||||
|
||||
@@ -11,7 +11,8 @@
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc -p tsconfig.json"
|
||||
"build": "tsc -p tsconfig.json",
|
||||
"prepare": "npm run build"
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
@@ -28,6 +29,9 @@
|
||||
"csv-parse": "^7.0.3",
|
||||
"socket.io-client": "^4.8.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"typescript": "^5.9.3"
|
||||
},
|
||||
"keywords": [
|
||||
"datastack",
|
||||
"telemetry",
|
||||
|
||||
@@ -138,6 +138,64 @@ export class DiskSpooler {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Atomically removes `ackedCount` head lines from `spool.jsonl` while
|
||||
* preserving any `unprocessed` envelopes at the FRONT of the queue in a
|
||||
* single atomic temp-file + `fsyncSync` + `renameSync` operation (FIFO safe).
|
||||
*/
|
||||
public ackAndRequeueFront(
|
||||
ackedCount: number,
|
||||
unprocessed: DataStackEnvelope[] = []
|
||||
): number {
|
||||
if (!this.enabled || ackedCount <= 0 || !fs.existsSync(this.spoolFile)) {
|
||||
return 0;
|
||||
}
|
||||
const raw = fs.readFileSync(this.spoolFile, 'utf8');
|
||||
const allLines = raw
|
||||
.split('\n')
|
||||
.map((l) => l.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
const consumed = Math.min(Math.floor(ackedCount), allLines.length);
|
||||
const tailLines = allLines.slice(consumed);
|
||||
const headLines = unprocessed.map((evt) => JSON.stringify(evt));
|
||||
const combined = [...headLines, ...tailLines];
|
||||
|
||||
if (combined.length === 0) {
|
||||
try {
|
||||
fs.unlinkSync(this.spoolFile);
|
||||
} catch {
|
||||
// Ignore if already removed
|
||||
}
|
||||
return consumed;
|
||||
}
|
||||
|
||||
this.ensureDir();
|
||||
const tmpFile = `${this.spoolFile}.${process.pid}.${Date.now()}.tmp`;
|
||||
const payload = combined.join('\n') + '\n';
|
||||
try {
|
||||
const fd = fs.openSync(tmpFile, 'w');
|
||||
try {
|
||||
fs.writeFileSync(fd, payload, 'utf8');
|
||||
fs.fsyncSync(fd);
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
fs.renameSync(tmpFile, this.spoolFile);
|
||||
} catch (err) {
|
||||
try {
|
||||
if (fs.existsSync(tmpFile)) {
|
||||
fs.unlinkSync(tmpFile);
|
||||
}
|
||||
} catch {
|
||||
// Ignore cleanup failure
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
return Math.max(0, consumed - headLines.length);
|
||||
}
|
||||
|
||||
public count(): number {
|
||||
if (!this.enabled || !fs.existsSync(this.spoolFile)) {
|
||||
return 0;
|
||||
|
||||
Reference in new issue
Block a user