fix(p02): guard Redis ACK and dedupe CSV crash-window replays [skip ci]

This commit is contained in:
victorsteele428 committed 2026-11-22 15:44:57 +06:00
1 parent fd57a3518d
commit 76cd972bde
11 files changed
+375 -9

No files matched your search

+68 -2
View File
@@ -650,6 +650,66 @@ export class DataBufferEngine {
}
const executeFlush = async (): Promise<FlushResult> => {
// P02 R-02: examine CSV partitions before replay of an untrimmed Redis
// batch. Streaming parsing limits memory and handles escaped/newline CSV.
const excludeAlreadyPersisted = async (events: DataEvent[]): Promise<DataEvent[]> => {
if (!events.length) return events;
const fs = await import('node:fs');
const path = await import('node:path');
const zlib = await import('node:zlib');
const { pipeline } = await import('node:stream/promises');
const { Writable } = await import('node:stream');
const { parse } = await import('csv-parse');
const groups = new Map<string, {
clientId: string; date: string; ids: Set<string>;
}>();
for (const event of events) {
const clientId = String(event.clientId || 'unknown');
const date = CSVStorageManager.extractSafePartitionDate(event.timestamp);
const key = JSON.stringify([clientId, date]);
if (!groups.has(key)) groups.set(key, { clientId, date, ids: new Set() });
groups.get(key)!.ids.add(String(event.eventId));
}
const persisted = new Set<string>();
for (const [groupKey, group] of groups) {
const dir = CSVStorageManager.ensureClientDir(group.clientId);
const entries = await fs.promises.readdir(dir, { withFileTypes: true });
for (const entry of entries) {
if (!entry.isFile() ||
!/\.csv(?:\.gz)?$/.test(entry.name) ||
CSVStorageManager.extractSafePartitionDate(
entry.name.replace(/\.csv(?:\.gz)?$/, '').replace(/(?:-(?:part|gz)\d+)+$/, ''),
) !== group.date) continue;
const reader = fs.createReadStream(path.join(dir, entry.name));
const parser = parse({ columns: true, bom: true, skip_empty_lines: true });
const sink = new Writable({
objectMode: true,
write(row: { clientId?: string; eventId?: string }, _enc, done) {
if (row.clientId === group.clientId && group.ids.has(String(row.eventId))) {
persisted.add(JSON.stringify([groupKey, String(row.eventId)]));
}
done();
},
});
if (entry.name.endsWith('.gz')) {
await pipeline(reader, zlib.createGunzip(), parser, sink);
} else {
await pipeline(reader, parser, sink);
}
}
}
const batchSeen = new Set<string>();
return events.filter((event) => {
const key = JSON.stringify([String(event.clientId || 'unknown'),
CSVStorageManager.extractSafePartitionDate(event.timestamp)]);
const rowKey = JSON.stringify([key, String(event.eventId)]);
if (persisted.has(rowKey) || batchSeen.has(rowKey)) return false;
batchSeen.add(rowKey);
return true;
});
};
state.isFlushing = true;
let totalWritten = 0;
let lockTimedOut = false;
@@ -793,10 +853,13 @@ export class DataBufferEngine {
if (lockLost) break;
if (parsedBatch.length > 0) {
const res = await CSVStorageManager.appendBatch(parsedBatch);
const unwritten = await excludeAlreadyPersisted(parsedBatch);
if (unwritten.length > 0) {
const res = await CSVStorageManager.appendBatch(unwritten);
totalWritten += res.written;
for (const f of res.filesUpdated) filesSet.add(f);
}
}
if (Date.now() - lastRenewAt >= renewIntervalMs) {
await renewLockOnce();
@@ -845,9 +908,12 @@ export class DataBufferEngine {
if (validRamBatch.length > 0) {
try {
const res = await CSVStorageManager.appendBatch(validRamBatch);
const unwritten = await excludeAlreadyPersisted(validRamBatch);
if (unwritten.length > 0) {
const res = await CSVStorageManager.appendBatch(unwritten);
totalWritten += res.written;
for (const f of res.filesUpdated) filesSet.add(f);
}
} catch (err) {
state.buffer.unshift(...validRamBatch);
throw err;
+11
View File
@@ -89,3 +89,14 @@ Source freeze for P02: Gitea/GitHub main `667a140c4e9e75382bfdcdbe1d3e9eb96f6e80
- **R-03 health:** `server/index.js:1926-1957` returns HTTP 200 `status:online` when Redis LLEN fails, while surfacing `redisConnected:false`; the historical route is **liveness**, not dependency readiness. `app/api/health/route.ts:95-111` likewise reports web liveness/buffer and does not prove database/Redis write readiness. A new readiness endpoint or altered public response requires separate approval.
Do not auto-fix source based solely on tests. Runtime code is READ-ONLY under the current approval. Any proven R-01/R-02 risk requiring behavior change needs a new exact method/file approval before application changes. P03 remains independent and unstarted.
## P02 Runtime Remediation Candidate — 2026-11-22 (NOT YET VERIFIED)
Authorized follow-up baseline: Gitea/GitHub `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898` (GitLab same tree). Local isolated branch `p02-runtime-remediation-20261122`, implementation commit `44eee8877e40d1683f44502c2a0ba3d437a03b88`. Scope: **server/index.js** (`pushEventsToRedisOrFallback`, `flushBuffer`), **server/socket/handlers.js** (sync_data/batch_sync ACK and claim rollback), **core/data/buffer.ts** (`flush`), **tests/phase02-sync.test.ts** and **tests/phase03-storage-buffer.test.ts** only, plus notes/001–006. All other source is unchanged.
**Candidate R-01 behavior:** when a configured Redis client returns an RPUSH error, `pushEventsToRedisOrFallback` retains RAM fallback but throws a sanitized `BUFFER_PERSIST_FAILED` instead of returning a durable-looking buffer count. Socket single/batch ACK returns `success:false`; best-effort Redis idempotency-key removal and bounded in-process negative-ACK marker handle ordinary retries. This is a **candidate** pending tests. When Redis is intentionally absent, historical RAM fallback is deliberately unchanged; it is not a universal durable-ACK guarantee. If Redis claim deletion fails and the server restarts, stale Redis idempotency claims could still misclassify a retry; a full crash-atomic claim/queue protocol is **not established** by this patch.
**Candidate R-02 behavior:** Port 4000 `flushBuffer` and Port 3000 `DataBufferEngine.flush` stream-parse existing CSV/CSV.gz partitions for same-client/date/event IDs before replaying any buffered Redis or RAM batch, dropping already-persisted IDs while retaining Redis queue until the write/check completes. Scan/read/parse exceptions fail closed before LTRIM. In-scope test adds a direct Port 4000 writer + Redis LTRIM-failure/recovery test and updates the existing web-buffer replay fixture. **Not proven globally exactly-once:** cross-process CSV writers outside the Redis flush lock and unknown archival/partial-corruption cases remain; scanning partitions on each flush also has real disk-I/O cost, requiring throughput evaluation before production promotion.
**Candidate acceptance:** one disposable Gitea CI with 24/24 suites, strict TS, lint, Linux/Windows clean npm CI and production build; direct Port4000 replay tests; exact allowlist/diff audit and three-remote tree parity. The Gitea MCP/host was unavailable on the implementation attempt, so this is **NOT PASSED** and the P02 phase remains **BLOCKED / NOT CLOSED**. No live deployment or runtime production mutation.
+9
View File
@@ -81,3 +81,12 @@ P02 approved as **diagnostic-only**. Published P01 baseline: `667a140c4e9e75382b
`APPROVED — DataStack P02-RUNTIME-REMEDIATION. Baseline: use the verified P02 diagnostic commit/remote tree. Fix reproduced R-01 and R-02 only, following exact notes/001–006 proof and scoped server/index.js pushEventsToRedisOrFallback/onBufferPush and flushBuffer, server/socket/handlers.js sync_data/batch_sync ACK, and their three existing P02 test files. First document safe ACK/retry semantics, exactly-once versus at-least-once CSV contract and crash recovery. Add disposable fault tests proving no positive ACK for non-durable data and no unbounded duplicate replay. Runtime sources core/data/buffer.ts and server/workers/csvWriter.js remain READ-ONLY unless a separately justified exact file/line amendment is approved. Preserve all UI/auth/SDK/public health behavior and existing error codes, production data, security, other features and Docker. One focused CI, diff/security audit, notes update, commit and 3-remote parity. No deployment, migrations, tags/releases or scope expansion. If a safe fix cannot be achieved within these exact methods, stop and report the required additional Scope Lock rather than broadening automatically.`
After separate approved remediation and verified P02 closure, **P03 plan** remains CI-06 Linux Python runner provisioning, CI-07 isolated Docker/BuildKit failure remediation and 3-image build, CI-08 supply-chain advisory triage, and non-mutating readiness evidence. P03 should not modify unrelated UI/runtime/API code by default.
## P02 Runtime Remediation Execution Handoff — 2026-11-22
The user explicitly approved **only** R-01 Redis failure ACK and R-02 Redis-to-CSV replay corrections in these methods: `server/index.js` `pushEventsToRedisOrFallback` / `flushBuffer` / existing `onBufferPush` integration; `server/socket/handlers.js` `sync_data` / `batch_sync` ACK handling; `core/data/buffer.ts` `flush` if needed; the three earlier P02 test files; `notes/001–006`. No other application method/UI/API/auth/SDK/Docker/CI-workflow edit. Public R-03 `/health` semantics remain unchanged.
Local candidate `44eee8877e40d1683f44502c2a0ba3d437a03b88` is **not release-ready** until Gitea acceptance. Before closure, prove: negative ACK on Redis failure with no sensitive error leakage; ordinary successful Redis ACK preserved; both single and batch retries don't convert volatile failures to success; Redis queued event replays after crash without writing duplicate CSV rows in the Port 4000 worker and Port 3000 web buffer; CSV read/parse failure never causes premature Redis trim; validate bounded scan overhead and edge cases for rotated/gzipped partitions; all historical 401/403/429/507/503 tests unchanged.
**Stop rule:** if required fixes need touching `validateSingleEventAsync`, `csvWriter.js` or separate journal/storage scheme, do not broaden the implementation scope; record the residual and request new explicit approval. Existing unrelated files including `MEMORY.md` remain unchanged due exact Scope Lock; this note is the cross-IDE handoff. P03 must wait for P02 CLOSED, not merely implementation committed.
+16
View File
@@ -91,3 +91,19 @@ Historical baseline code stayed frozen through these CI workflow changes. Curren
- **BLOCKING RELEASE FINDINGS:** R-01 success ACK issued for non-durable RAM fallback when Redis RPUSH fails; R-02 identical eventId can append twice on post-CSV Redis-trim failure (web-side representative test). R-03 health remains online on Redis loss; liveness-only behavior is backward-compatible, protected dependency-readiness verification remains missing.
- **P02 PHASE = BLOCKED**, diagnostic implementation done, release reliability acceptance NOT satisfied. Runtime remediation requires separate exact approval. 1/3 phases still CLOSED (P01 only); P02 and P03 remain unclosed. No P03 start authorization from this diagnostic completion.
- Next approval must explicitly authorize exact narrow server/core patch for ACK durability and CSV replay deduplication, plus concrete no-loss/no-duplicate tests. Any readiness endpoint change needs separate explicit approval. User's initial P02 approval is insufficient for runtime mutation.
## P02-RUNTIME-REMEDIATION — Candidate implementation checkpoint (2026-11-22)
- **Current P02: BLOCKED / NOT CLOSED.** One previous diagnostic Gitea CI #63 reproduced R-01/R-02/R-03; runtime remediation was separately approved after that. Verified completed implementation phases remain **1/3** (P01 only). Do not approve P03 yet.
- Published P02 diagnosis baseline `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`; isolated local candidate commit `44eee8877e40d1683f44502c2a0ba3d437a03b88` on `p02-runtime-remediation-20261122`.
- Exactly five code/test files changed in this candidate: `server/index.js`, `server/socket/handlers.js`, `core/data/buffer.ts`, `tests/phase02-sync.test.ts`, `tests/phase03-storage-buffer.test.ts`. No `server/workers/csvWriter.js`, `app/**`, `components/**`, SDK, public R-03 health, CI workflow, database, migration, production data, secrets or deployment touched.
- JS `node --check` on two changed JS files PASS; `git diff --check` PASS and allowlist audit PASS. Targeted TS/Node regression and Gitea CI **NOT YET RUN TO TERMINAL PASS**. The Gitea connector/host was unavailable when dispatch/push was attempted. Local temporary dependency installation also not yet complete.
- The original user PC root `D:\VibTools_Workspace\01_Products\14_DataStack` still has 16 pre-existing staged P01 paths on older `d8ac96e`; deliberately preserved without reset/stash/discard. All P02 implementation is in a separate temporary worktree. Do not assume the old local root is up to date.
- R-01 and R-02 implementations are candidates; practical reliability limitations and I/O cost recorded in `notes/001` and `notes/004`. No false claim of globally exactly-once CSV or universally durable ACK.
- **Next gates:** restore Gitea access; run one focused disposable acceptance; fix any in-scope test/TS defects; record exact CI terminal results; only after required acceptance and security audit, squash single phase main commit and verify GitHub/GitLab/Gitea tree parity; then, and only then, decide P02 CLOSED versus residual-risk blocker.
### Verification environment blocker — local dependency install
The authorized Windows PC attempted a **local disposable** `npm.cmd ci --ignore-scripts --no-audit --no-fund --prefer-offline` in the isolated P02 worktree only; it failed with `npm ERR! ETARGET No matching version found for yaml@2.9.1` plus cleanup EPERM warnings in the temp worktree's `node_modules`. Exact output captured in `%TEMP%\DataStack-P02-npm-ci.log` on the authorized PC. This is **not a passed code test nor proof of a repository package defect**, and `package-lock.json` / `package.json` cannot be edited under P02 scope. Gitea MCP remained UNAVAILABLE and the Git remote/HTTP service was not responding when CI was attempted, so the mandatory P02 focused CI was **not dispatched**. Phase remains BLOCKED pending actual CI evidence; no runtime change was pushed to production/main.
+12
View File
@@ -64,3 +64,15 @@ The previous #58 package-lock error was resolved by adding the exact nested opti
| R-03 availability | Redis `llen` rejection leaves Express `/health` online with `redisConnected:false`. Recovery flips the flag while HTTP 200 remains unchanged. | Public 200 is liveness by existing contract; a protected dependency readiness probe/503 is **not implemented** and needs explicit approval. |
Security invariants: do not expose secrets in ACK/errors, keep JWT/auth and 401/403/429/503/507 contracts unchanged, and never run failure-injection against actual Redis, Postgres, production API, live events, or user CSV. Disposable fake Redis and temporary files only. CI #63: https://dev.ivib.cc/vibtools/DataStack/actions/runs/63; results tracked in notes/003.
## P02 Runtime Candidate — Error-path change inventory (pending CI)
| Path | Candidate result | Explicit limitation |
|---|---|---|
| Redis `RPUSH` rejects with configured Redis client | Retains event in volatile RAM as preexisting fallback but returns `BUFFER_PERSIST_FAILED` to Socket.IO via thrown error; **no successful ACK** for that failure; error text does not expose connection details | No-Redis intentional legacy RAM mode unchanged, still not a durable guarantee |
| `sync_data` and `batch_sync` failures | `success:false`, `code:BUFFER_PERSIST_FAILED` and best-effort rollback of previously claimed idempotency keys, negative failure marker for in-process retries | If claim rollback cannot reach Redis and process crashes, durable claim/queue atomicity remains unsolved |
| Redis buffer `LTRIM` rejects after successful CSV append | Subsequent flush checks physically stored CSV IDs and avoids appending matching rows again; Redis is trimmed only after successful check/write | Concurrent independent CSV appenders, compressed archived CSV, corrupt partial rows, and per-flush CSV scan cost require separate proof |
| Legacy public health | **UNCHANGED** HTTP 200 liveness contract and Redis-connected diagnostic field; R-03 readiness is not introduced in this approval | Does not prove full DB/Redis transactional readiness |
Security/API invariants: leave token/JWT/auth/rate limits/quotas/webhooks/MCP/sandbox/UI/SDK unchanged. No production Redis/Postgres/API/CSV mutation. No lockfile/dependency versions or CI workflow changes. CI acceptance remains pending; these are code-level observations, not release certification.
+8
View File
@@ -52,3 +52,11 @@ Do not mark "actual working" simply because a patch compiled. Distinguish source
- R-02: existing CSV-before-Redis-trim order audited, *not corrected*. Exactly-once replay safety is **NOT VERIFIED**.
- R-03: current public Express health 200 plus `redisConnected:false` is liveness, not dependency readiness. Separate readiness remains **NOT IMPLEMENTED**.
- Gitea disposable CI #63: https://dev.ivib.cc/vibtools/DataStack/actions/runs/63. The previous P01 working results remain valid; no live read/write test or production deploy is included in P02.
## P02 Runtime Candidate Feature Delta — 2026-11-22 (NOT RELEASED)
- **Intended change:** failure cases on socket ingestion report `BUFFER_PERSIST_FAILED` instead of positive ACK when Redis RPUSH rejects; single and batch code paths tested by new disposable assertions. Ordinary success ACK, auth, limits and SDK public interfaces intended unchanged.
- **Intended change:** Redis replay flush on both ingestion/Next paths inspects existing client CSV partitions and filters already-stored event IDs before reappend; safe failure leaves Redis queue untrimmed. This prevents the tested post-append/LTRIM replay duplication but does **not** prove transactionally exactly-once delivery or zero performance impact.
- **No changes:** application UI/UX, auth, SDK, database schema, Docker, environment, public R-03 health, other features and production runtime. No deployment or live verification.
- **Status:** P02 code candidate locally committed, acceptance CI blocked by Gitea unavailability; **NOT CLOSED**, not production-ready. P01 remains only completed phase.
+22
View File
@@ -237,3 +237,25 @@ return bufferQueue.length;
- `P02 R-03 diagnostic: Redis query failure leaves public ingestion health 200/online` — **ok 16**, same suite exit 0, Redis flag false while liveness stays online.
- `P02 R-02 diagnostic: post-CSV Redis trim failure replays identical event on retry` — **ok 18**, `phase03-storage-buffer` suite exit 0; duplicate append after retry reproduced on representative web buffer path. **Direct ingestion Port 4000 CSV worker crash coverage remains outstanding** and should be requested in remediation approval before guarantee changes.
- These three diagnostic tests **prove findings, do not close production-readiness**. See Gitea https://dev.ivib.cc/vibtools/DataStack/actions/runs/63; never call run successful until terminal conclusion is confirmed. No server-side source fix applied.
## F. P02 Remediation Candidate Patch Proof — 2026-11-22
Base: Gitea/GitHub published P02 diagnostic tree `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`; GitLab distinct commit, same previous tree. Local candidate commit `44eee8877e40d1683f44502c2a0ba3d437a03b88` on isolated worktree. Initial exact delta: 5 approved source/test paths, 310 insertions and 22 deletions, no application UI, Python SDK, auth, public health route, CI or deployment edit. `git diff --check` and `node --check` on JS files PASS.
- `server/index.js:pushEventsToRedisOrFallback`: catch RPUSH failure, keep previous RAM buffering, throw sanitized `BUFFER_PERSIST_FAILED`; no positive ACK for that branch.
- `server/socket/handlers.js:sync_data/batch_sync`: negative ACK and best-effort claimed idempotency-key rollback with negative retry guard, no change to legitimate duplicate success path or existing auth/limits/error codes.
- `server/index.js:flushBuffer` and `core/data/buffer.ts:flush`: scoped streaming CSV/CSV.gz read-before-reappend (parsing quoted/multiline rows), exact client/date/event ID match; failures abort before Redis LTRIM. No changes to `server/workers/csvWriter.js`.
- `tests/phase02-sync.test.ts`: R-01 single/batch outage and duplicate claim ACK negative, successful recovery ACK, and unchanged R-03 liveness.
- `tests/phase03-storage-buffer.test.ts`: web-side replay verifies exactly one physically stored row; new direct Port 4000 writer Redis-trim failure/recovery proof. All tests on fake Redis and disposable temp CSV paths.
**Unverified:** Gitea P02 implementation acceptance was unavailable due inaccessible host/connector. Local `npm ci` not confirmed successful. Do not invent a CI run ID or claim full PASS. Full cross-process transactional exactly-once and all Redis claim crash windows are out of proof scope and may require exact additional approval. The streaming dedupe scan changes I/O cost per flush; performance validation before production is required.
### Blocked acceptance evidence — actual tool outcomes
- JS syntax: local `node --check server/index.js` and `node --check server/socket/handlers.js` PASS.
- Source scope: local `git diff --check` PASS; only the five allowed P02 runtime/test paths changed before notes updates.
- Gitea: connection returned `UNAVAILABLE: McpServerError: Connection failed`; Git remote `dev.ivib.cc` unresponsive. No Gitea CI run ID exists for candidate `44eee8877e40d1683f44502c2a0ba3d437a03b88`.
- Local Windows npm: `npm ci` failed `ETARGET yaml@2.9.1` and cleanup `EPERM` in the isolated worktree, **not** a TypeScript/test result. Dependency changes are out of P02 scope.
- Therefore neither TS typecheck nor regression-suite remediation acceptance is established; keep P02 BLOCKED and do not advance P03 or alter any protected runtime/source beyond this approval.
+70 -2
View File
@@ -165,6 +165,7 @@ function setRedisClientForTests(client) {
async function pushEventsToRedisOrFallback(events) {
if (!events || events.length === 0) return bufferQueue.length;
const redis = getRedisClient();
let persistFailed = false;
if (redis) {
const serialized = events.map((e) => JSON.stringify(e));
let bufferLen = null;
@@ -173,6 +174,7 @@ async function pushEventsToRedisOrFallback(events) {
redisConnected = true;
} catch {
redisConnected = false;
persistFailed = true;
}
if (bufferLen !== null) {
@@ -189,6 +191,11 @@ async function pushEventsToRedisOrFallback(events) {
}
bufferQueue.push(...events);
if (persistFailed) {
// Keep the volatile fallback for later recovery, but never ACK it as durable.
// Suppress sensitive Redis transport details in the client-facing error.
throw new Error('BUFFER_PERSIST_FAILED: Redis unavailable; event not durably queued');
}
return bufferQueue.length;
}
@@ -834,6 +841,61 @@ async function isClientDeletedForFlush(clientId, redis, cache) {
}
async function flushBuffer(options = {}) {
// P02 R-02: Redis and CSV cannot be committed atomically. Before replaying an
// untrimmed Redis batch, check actual CSV partitions for previously written IDs.
// Streaming parsing bounds memory and preserves multiline/quoted CSV fields.
const excludeAlreadyPersisted = async (events) => {
if (!events.length) return events;
const fs = require('node:fs');
const path = require('node:path');
const zlib = require('node:zlib');
const { pipeline } = require('node:stream/promises');
const { Writable } = require('node:stream');
const { parse } = require('csv-parse');
const groups = new Map();
for (const event of events) {
const clientId = String(event.clientId || 'unknown');
const date = writer.extractSafePartitionDate(event.timestamp);
const key = JSON.stringify([clientId, date]);
if (!groups.has(key)) groups.set(key, { clientId, date, ids: new Set() });
groups.get(key).ids.add(String(event.eventId));
}
const persisted = new Set();
for (const [groupKey, group] of groups) {
const dir = writer.ensureClientDir(group.clientId);
const entries = await fs.promises.readdir(dir, { withFileTypes: true });
for (const entry of entries) {
if (!entry.isFile() || !writer.isPartitionFilename(entry.name) ||
writer.extractDateFromFilename(entry.name) !== group.date) continue;
const reader = fs.createReadStream(path.join(dir, entry.name));
const streams = [reader];
if (entry.name.endsWith('.gz')) streams.push(zlib.createGunzip());
streams.push(parse({ columns: true, bom: true, skip_empty_lines: true }));
streams.push(new Writable({
objectMode: true,
write(row, _enc, done) {
if (row.clientId === group.clientId && group.ids.has(String(row.eventId))) {
persisted.add(JSON.stringify([groupKey, String(row.eventId)]));
}
done();
},
}));
// On read/parse failure, abort without LTRIM: never discard uncertain rows.
await pipeline(...streams);
}
}
const batchSeen = new Set();
return events.filter((event) => {
const key = JSON.stringify([String(event.clientId || 'unknown'),
writer.extractSafePartitionDate(event.timestamp)]);
const rowKey = JSON.stringify([key, String(event.eventId)]);
if (persisted.has(rowKey) || batchSeen.has(rowKey)) return false;
batchSeen.add(rowKey);
return true;
});
};
if (pendingRedisWrites.size > 0) {
await Promise.allSettled([...pendingRedisWrites]);
}
@@ -979,10 +1041,13 @@ async function flushBuffer(options = {}) {
if (lockLost) break;
if (parsedBatch.length > 0) {
const res = await writer.writeBatch(parsedBatch);
const unwritten = await excludeAlreadyPersisted(parsedBatch);
if (unwritten.length > 0) {
const res = await writer.writeBatch(unwritten);
totalWritten += res.written;
for (const f of res.files) filesSet.add(f);
}
}
if (Date.now() - lastRenewAt >= renewIntervalMs) {
await renewLockOnce();
@@ -1022,9 +1087,12 @@ async function flushBuffer(options = {}) {
if (validBatch.length > 0) {
try {
const res = await writer.writeBatch(validBatch);
const unwritten = await excludeAlreadyPersisted(validBatch);
if (unwritten.length > 0) {
const res = await writer.writeBatch(unwritten);
totalWritten += res.written;
for (const f of res.files) filesSet.add(f);
}
} catch (err) {
bufferQueue.unshift(...validBatch);
throw err;
+51 -2
View File
@@ -549,6 +549,9 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
: new Date().toISOString();
const eventData = extractSingleEventData(payload);
// A previous Redis write failure can leave an uncommitted idempotency
// claim. Never report that claim as a successful durable duplicate.
const failedAt = idempotencyMap.get(`${clientId}:${eventId}`);
const check = await validateSingleEventAsync(
clientId,
eventId,
@@ -557,7 +560,13 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
);
if (!check.allowed) {
if (check.duplicate) {
if (callback) {
if (typeof failedAt === "number" && failedAt < 0 &&
Date.now() + failedAt < IDEMPOTENCY_TTL_MS) {
if (callback) callback({
success: false, eventId, code: "BUFFER_PERSIST_FAILED",
error: "BUFFER_PERSIST_FAILED: Event not durably queued",
});
} else if (callback) {
callback({ success: true, eventId, note: "DUPLICATE_DROPPED" });
}
return;
@@ -585,6 +594,19 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
try {
await onBufferPush();
} catch (persistErr) {
// Invalidate the pre-persistence idempotency claim so SDK retry can
// resubmit after Redis recovers. Keep the volatile RAM event pending.
const claimKey = `${clientId}:${eventId}`;
idempotencyMap.set(claimKey, -Date.now());
const claimRedis = getRedisClient();
if (claimRedis && typeof claimRedis.del === "function") {
try {
await claimRedis.del(`${REDIS_IDEMPOTENCY_PREFIX}${claimKey}`);
idempotencyMap.delete(claimKey);
} catch {
// Claim may remain: negative ACK must be retained for retries.
}
}
if (callback) {
callback({
success: false,
@@ -689,6 +711,8 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
const rateLimitedEventIds = [];
const oversizedEventIds = [];
const errors = [];
const acceptedEventIds = [];
let volatileDuplicate = false;
for (const item of batch) {
const safeItem = item && typeof item === "object" ? item : {};
@@ -702,6 +726,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
: new Date().toISOString();
const rawData = extractBatchItemData(safeItem);
const failedAt = idempotencyMap.get(`${clientId}:${eventId}`);
const check = await validateSingleEventAsync(
clientId,
eventId,
@@ -710,7 +735,15 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
);
if (!check.allowed) {
if (check.duplicate) {
if (typeof failedAt === "number" && failedAt < 0 &&
Date.now() + failedAt < IDEMPOTENCY_TTL_MS) {
rejected += 1;
volatileDuplicate = true;
if (errors.length < 5) errors.push(
`${eventId}: BUFFER_PERSIST_FAILED: Event not durably queued`);
} else {
duplicates += 1;
}
continue;
}
rejected += 1;
@@ -735,6 +768,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
payload: check.payloadStr,
});
processed += 1;
acceptedEventIds.push(eventId);
}
if (processed > 0) {
@@ -742,6 +776,19 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
try {
await onBufferPush();
} catch (persistErr) {
const claimRedis = getRedisClient();
for (const eventId of acceptedEventIds) {
const claimKey = `${clientId}:${eventId}`;
idempotencyMap.set(claimKey, -Date.now());
if (claimRedis && typeof claimRedis.del === "function") {
try {
await claimRedis.del(`${REDIS_IDEMPOTENCY_PREFIX}${claimKey}`);
idempotencyMap.delete(claimKey);
} catch {
// Preserve negative-ACK retry guard if claim removal fails.
}
}
}
if (callback) {
callback({
success: false,
@@ -760,7 +807,9 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
}
if (processed === 0 && duplicates === 0 && rejected > 0) {
const code = hitPayloadLimit
const code = volatileDuplicate
? "BUFFER_PERSIST_FAILED"
: hitPayloadLimit
? "PAYLOAD_TOO_LARGE"
: hitRateLimit
? "RATE_LIMIT_EXCEEDED"
+58 -1
View File
@@ -1246,12 +1246,29 @@ test("P02 R-01 diagnostic: Redis RPUSH failure yields positive WS ACK for volati
);
const committedRows = await failingRedis.lrange("datastack:buffer:events", 0, -1);
const volatileRows = serverMod.bufferQueue.filter((e) => e.eventId === "p02-ack-before-crash-001");
assert.equal(ack.success, true, "Existing handler currently sends a positive ACK after Redis failure");
assert.equal(ack.success, false, "P02 remediation must reject non-durable ACK");
assert.equal(ack.code, "BUFFER_PERSIST_FAILED");
const sameIdRetry = await emitWithAck<{ success: boolean; code?: string }>(
socket, "sync_data",
{ eventId: "p02-ack-before-crash-001", data: { value: 42 } },
);
assert.equal(sameIdRetry.success, false,
"Pending idempotency claims must not return false positive duplicate ACK");
assert.equal(sameIdRetry.code, "BUFFER_PERSIST_FAILED");
assert.equal(committedRows.length, 0, "Fake Redis has no durable event");
assert.equal(volatileRows.length, 1, "Acknowledged event is RAM-only");
// Crash model: restart loses RAM but fake Redis survives. No actual process is killed.
serverMod.bufferQueue.length = 0;
assert.equal((await failingRedis.lrange("datastack:buffer:events", 0, -1)).length, 0);
// A new event may ACK only after Redis actually resumes persistence.
const healthyRedis = createSharedMockRedis();
serverMod.setRedisClientForTests(healthyRedis);
const recovered = await emitWithAck<{ success: boolean; eventId?: string }>(
socket, "sync_data",
{ eventId: "p02-ack-recovered-002", data: { value: 43 } },
);
assert.equal(recovered.success, true);
assert.equal((await healthyRedis.lrange("datastack:buffer:events", 0, -1)).length, 1);
} finally {
socket?.disconnect();
serverMod.setRedisClientForTests(undefined);
@@ -1281,3 +1298,43 @@ test("P02 R-03 diagnostic: Redis query failure leaves public ingestion health 20
serverMod.setRedisClientForTests(undefined);
}
});
test("P02 R-01: batch_sync rejects non-durable Redis writes and pending duplicate claims", async () => {
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
const previousQueue = serverMod.bufferQueue.slice();
const failingRedis = createSharedMockRedis();
failingRedis.rpush = async () => { throw new Error("P02 injected batch Redis outage"); };
serverMod.setRedisClientForTests(failingRedis);
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
serverMod.bufferQueue.length = 0;
let socket: Socket | undefined;
const payload = [
{ eventId: "p02-batch-volatile-1", data: { value: 1 } },
{ eventId: "p02-batch-volatile-2", data: { value: 2 } },
];
try {
const client = await SDKManager.registerClientAsync({
clientId: "p02_redis_batch_failure",
name: "P02 Redis Batch Failure",
rateLimit: 100,
});
socket = await connectSocket(client.token);
const ack = await emitWithAck<{ success: boolean; code?: string; processed?: number }>(
socket, "batch_sync", payload,
);
assert.equal(ack.success, false);
assert.equal(ack.code, "BUFFER_PERSIST_FAILED");
assert.equal(ack.processed, 0);
assert.equal((await failingRedis.lrange("datastack:buffer:events", 0, -1)).length, 0);
const retry = await emitWithAck<{ success: boolean; code?: string }>(
socket, "batch_sync", payload,
);
assert.equal(retry.success, false);
assert.equal(retry.code, "BUFFER_PERSIST_FAILED");
} finally {
socket?.disconnect();
serverMod.setRedisClientForTests(undefined);
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
}
});
+50 -2
View File
@@ -1584,10 +1584,58 @@ test("P02 R-02 diagnostic: post-CSV Redis trim failure replays identical event o
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 1, "Redis still queues previously appended CSV event");
await DataBufferEngine.flush();
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0);
assert.equal(writtenIds.filter((id) => id === event.eventId).length, 2,
"A second flush appends the same eventId again after failed trim: at-least-once, not exactly-once");
assert.equal(writtenIds.filter((id) => id === event.eventId).length, 1,
"Replay must detect the persisted CSV eventId before attempting another append");
const csvPath = path.join(tmpStorageDir, event.clientId, "2026-11-22.csv");
const csvContents = fs.readFileSync(csvPath, "utf8");
assert.equal(csvContents.split(event.eventId).length - 1, 1,
"Exactly one physical CSV row must exist after a failed LTRIM and retry");
} finally {
CSVStorageManager.appendBatch = originalAppend;
DataBufferEngine.setRedisClientForTests(undefined);
}
});
test("P02 R-02: Port 4000 CSV worker suppresses a crash-window replay after Redis LTRIM error", async () => {
const mockRedis = new FakeRedisClient();
const originalTrim = mockRedis.ltrim.bind(mockRedis);
let failedOnce = false;
mockRedis.ltrim = async (key: string, start: number, stop: number): Promise<"OK"> => {
if (key === REDIS_BUFFER_KEY && !failedOnce) {
failedOnce = true;
throw new Error("P02 Port4000 injected post-CSV trim failure");
}
return originalTrim(key, start, stop);
};
const ingestion = require("../server/index.js") as IngestionServerModule;
const previousDir = ingestion.writer.baseDir;
const previousQueue = ingestion.bufferQueue.slice();
ingestion.writer.baseDir = tmpStorageDir;
ingestion.bufferQueue.length = 0;
ingestion.setRedisClientForTests(mockRedis);
const evt: DataEvent = {
clientId: "p02_ingestion_replay",
eventId: "p02-port4000-001",
timestamp: "2026-11-22T13:00:00.000Z",
payload: JSON.stringify({ stage: "before-ltrim" }),
};
try {
await mockRedis.rpush(REDIS_BUFFER_KEY, JSON.stringify(evt));
await assert.rejects(
() => ingestion.flushBuffer(),
/P02 Port4000 injected post-CSV trim failure/,
);
assert.equal(failedOnce, true);
assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 1);
const csvPath = path.join(tmpStorageDir, evt.clientId, "2026-11-22.csv");
assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1);
const recovered = await ingestion.flushBuffer();
assert.equal(recovered.written, 0, "Recovered replay should not append the event again");
assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 0);
assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1);
} finally {
ingestion.writer.baseDir = previousDir;
ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...previousQueue);
ingestion.setRedisClientForTests(undefined);
}
});