fix(p02): guard Redis ACK and dedupe CSV crash-window replays [skip ci]
This commit is contained in:
1 parent
fd57a3518d
commit
76cd972bde
11 files changed
+388
-22
No files matched your search
+72
-6
@@ -650,6 +650,66 @@ export class DataBufferEngine {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const executeFlush = async (): Promise<FlushResult> => {
|
const executeFlush = async (): Promise<FlushResult> => {
|
||||||
|
|
||||||
|
// P02 R-02: examine CSV partitions before replay of an untrimmed Redis
|
||||||
|
// batch. Streaming parsing limits memory and handles escaped/newline CSV.
|
||||||
|
const excludeAlreadyPersisted = async (events: DataEvent[]): Promise<DataEvent[]> => {
|
||||||
|
if (!events.length) return events;
|
||||||
|
const fs = await import('node:fs');
|
||||||
|
const path = await import('node:path');
|
||||||
|
const zlib = await import('node:zlib');
|
||||||
|
const { pipeline } = await import('node:stream/promises');
|
||||||
|
const { Writable } = await import('node:stream');
|
||||||
|
const { parse } = await import('csv-parse');
|
||||||
|
const groups = new Map<string, {
|
||||||
|
clientId: string; date: string; ids: Set<string>;
|
||||||
|
}>();
|
||||||
|
for (const event of events) {
|
||||||
|
const clientId = String(event.clientId || 'unknown');
|
||||||
|
const date = CSVStorageManager.extractSafePartitionDate(event.timestamp);
|
||||||
|
const key = JSON.stringify([clientId, date]);
|
||||||
|
if (!groups.has(key)) groups.set(key, { clientId, date, ids: new Set() });
|
||||||
|
groups.get(key)!.ids.add(String(event.eventId));
|
||||||
|
}
|
||||||
|
const persisted = new Set<string>();
|
||||||
|
for (const [groupKey, group] of groups) {
|
||||||
|
const dir = CSVStorageManager.ensureClientDir(group.clientId);
|
||||||
|
const entries = await fs.promises.readdir(dir, { withFileTypes: true });
|
||||||
|
for (const entry of entries) {
|
||||||
|
if (!entry.isFile() ||
|
||||||
|
!/\.csv(?:\.gz)?$/.test(entry.name) ||
|
||||||
|
CSVStorageManager.extractSafePartitionDate(
|
||||||
|
entry.name.replace(/\.csv(?:\.gz)?$/, '').replace(/(?:-(?:part|gz)\d+)+$/, ''),
|
||||||
|
) !== group.date) continue;
|
||||||
|
const reader = fs.createReadStream(path.join(dir, entry.name));
|
||||||
|
const parser = parse({ columns: true, bom: true, skip_empty_lines: true });
|
||||||
|
const sink = new Writable({
|
||||||
|
objectMode: true,
|
||||||
|
write(row: { clientId?: string; eventId?: string }, _enc, done) {
|
||||||
|
if (row.clientId === group.clientId && group.ids.has(String(row.eventId))) {
|
||||||
|
persisted.add(JSON.stringify([groupKey, String(row.eventId)]));
|
||||||
|
}
|
||||||
|
done();
|
||||||
|
},
|
||||||
|
});
|
||||||
|
if (entry.name.endsWith('.gz')) {
|
||||||
|
await pipeline(reader, zlib.createGunzip(), parser, sink);
|
||||||
|
} else {
|
||||||
|
await pipeline(reader, parser, sink);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const batchSeen = new Set<string>();
|
||||||
|
return events.filter((event) => {
|
||||||
|
const key = JSON.stringify([String(event.clientId || 'unknown'),
|
||||||
|
CSVStorageManager.extractSafePartitionDate(event.timestamp)]);
|
||||||
|
const rowKey = JSON.stringify([key, String(event.eventId)]);
|
||||||
|
if (persisted.has(rowKey) || batchSeen.has(rowKey)) return false;
|
||||||
|
batchSeen.add(rowKey);
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
state.isFlushing = true;
|
state.isFlushing = true;
|
||||||
let totalWritten = 0;
|
let totalWritten = 0;
|
||||||
let lockTimedOut = false;
|
let lockTimedOut = false;
|
||||||
@@ -793,9 +853,12 @@ export class DataBufferEngine {
|
|||||||
if (lockLost) break;
|
if (lockLost) break;
|
||||||
|
|
||||||
if (parsedBatch.length > 0) {
|
if (parsedBatch.length > 0) {
|
||||||
const res = await CSVStorageManager.appendBatch(parsedBatch);
|
const unwritten = await excludeAlreadyPersisted(parsedBatch);
|
||||||
totalWritten += res.written;
|
if (unwritten.length > 0) {
|
||||||
for (const f of res.filesUpdated) filesSet.add(f);
|
const res = await CSVStorageManager.appendBatch(unwritten);
|
||||||
|
totalWritten += res.written;
|
||||||
|
for (const f of res.filesUpdated) filesSet.add(f);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (Date.now() - lastRenewAt >= renewIntervalMs) {
|
if (Date.now() - lastRenewAt >= renewIntervalMs) {
|
||||||
@@ -845,9 +908,12 @@ export class DataBufferEngine {
|
|||||||
|
|
||||||
if (validRamBatch.length > 0) {
|
if (validRamBatch.length > 0) {
|
||||||
try {
|
try {
|
||||||
const res = await CSVStorageManager.appendBatch(validRamBatch);
|
const unwritten = await excludeAlreadyPersisted(validRamBatch);
|
||||||
totalWritten += res.written;
|
if (unwritten.length > 0) {
|
||||||
for (const f of res.filesUpdated) filesSet.add(f);
|
const res = await CSVStorageManager.appendBatch(unwritten);
|
||||||
|
totalWritten += res.written;
|
||||||
|
for (const f of res.filesUpdated) filesSet.add(f);
|
||||||
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
state.buffer.unshift(...validRamBatch);
|
state.buffer.unshift(...validRamBatch);
|
||||||
throw err;
|
throw err;
|
||||||
|
|||||||
@@ -89,3 +89,14 @@ Source freeze for P02: Gitea/GitHub main `667a140c4e9e75382bfdcdbe1d3e9eb96f6e80
|
|||||||
- **R-03 health:** `server/index.js:1926-1957` returns HTTP 200 `status:online` when Redis LLEN fails, while surfacing `redisConnected:false`; the historical route is **liveness**, not dependency readiness. `app/api/health/route.ts:95-111` likewise reports web liveness/buffer and does not prove database/Redis write readiness. A new readiness endpoint or altered public response requires separate approval.
|
- **R-03 health:** `server/index.js:1926-1957` returns HTTP 200 `status:online` when Redis LLEN fails, while surfacing `redisConnected:false`; the historical route is **liveness**, not dependency readiness. `app/api/health/route.ts:95-111` likewise reports web liveness/buffer and does not prove database/Redis write readiness. A new readiness endpoint or altered public response requires separate approval.
|
||||||
|
|
||||||
Do not auto-fix source based solely on tests. Runtime code is READ-ONLY under the current approval. Any proven R-01/R-02 risk requiring behavior change needs a new exact method/file approval before application changes. P03 remains independent and unstarted.
|
Do not auto-fix source based solely on tests. Runtime code is READ-ONLY under the current approval. Any proven R-01/R-02 risk requiring behavior change needs a new exact method/file approval before application changes. P03 remains independent and unstarted.
|
||||||
|
|
||||||
|
|
||||||
|
## P02 Runtime Remediation Candidate — 2026-11-22 (NOT YET VERIFIED)
|
||||||
|
|
||||||
|
Authorized follow-up baseline: Gitea/GitHub `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898` (GitLab same tree). Local isolated branch `p02-runtime-remediation-20261122`, implementation commit `44eee8877e40d1683f44502c2a0ba3d437a03b88`. Scope: **server/index.js** (`pushEventsToRedisOrFallback`, `flushBuffer`), **server/socket/handlers.js** (sync_data/batch_sync ACK and claim rollback), **core/data/buffer.ts** (`flush`), **tests/phase02-sync.test.ts** and **tests/phase03-storage-buffer.test.ts** only, plus notes/001–006. All other source is unchanged.
|
||||||
|
|
||||||
|
**Candidate R-01 behavior:** when a configured Redis client returns an RPUSH error, `pushEventsToRedisOrFallback` retains RAM fallback but throws a sanitized `BUFFER_PERSIST_FAILED` instead of returning a durable-looking buffer count. Socket single/batch ACK returns `success:false`; best-effort Redis idempotency-key removal and bounded in-process negative-ACK marker handle ordinary retries. This is a **candidate** pending tests. When Redis is intentionally absent, historical RAM fallback is deliberately unchanged; it is not a universal durable-ACK guarantee. If Redis claim deletion fails and the server restarts, stale Redis idempotency claims could still misclassify a retry; a full crash-atomic claim/queue protocol is **not established** by this patch.
|
||||||
|
|
||||||
|
**Candidate R-02 behavior:** Port 4000 `flushBuffer` and Port 3000 `DataBufferEngine.flush` stream-parse existing CSV/CSV.gz partitions for same-client/date/event IDs before replaying any buffered Redis or RAM batch, dropping already-persisted IDs while retaining Redis queue until the write/check completes. Scan/read/parse exceptions fail closed before LTRIM. In-scope test adds a direct Port 4000 writer + Redis LTRIM-failure/recovery test and updates the existing web-buffer replay fixture. **Not proven globally exactly-once:** cross-process CSV writers outside the Redis flush lock and unknown archival/partial-corruption cases remain; scanning partitions on each flush also has real disk-I/O cost, requiring throughput evaluation before production promotion.
|
||||||
|
|
||||||
|
**Candidate acceptance:** one disposable Gitea CI with 24/24 suites, strict TS, lint, Linux/Windows clean npm CI and production build; direct Port4000 replay tests; exact allowlist/diff audit and three-remote tree parity. The Gitea MCP/host was unavailable on the implementation attempt, so this is **NOT PASSED** and the P02 phase remains **BLOCKED / NOT CLOSED**. No live deployment or runtime production mutation.
|
||||||
@@ -81,3 +81,12 @@ P02 approved as **diagnostic-only**. Published P01 baseline: `667a140c4e9e75382b
|
|||||||
`APPROVED — DataStack P02-RUNTIME-REMEDIATION. Baseline: use the verified P02 diagnostic commit/remote tree. Fix reproduced R-01 and R-02 only, following exact notes/001–006 proof and scoped server/index.js pushEventsToRedisOrFallback/onBufferPush and flushBuffer, server/socket/handlers.js sync_data/batch_sync ACK, and their three existing P02 test files. First document safe ACK/retry semantics, exactly-once versus at-least-once CSV contract and crash recovery. Add disposable fault tests proving no positive ACK for non-durable data and no unbounded duplicate replay. Runtime sources core/data/buffer.ts and server/workers/csvWriter.js remain READ-ONLY unless a separately justified exact file/line amendment is approved. Preserve all UI/auth/SDK/public health behavior and existing error codes, production data, security, other features and Docker. One focused CI, diff/security audit, notes update, commit and 3-remote parity. No deployment, migrations, tags/releases or scope expansion. If a safe fix cannot be achieved within these exact methods, stop and report the required additional Scope Lock rather than broadening automatically.`
|
`APPROVED — DataStack P02-RUNTIME-REMEDIATION. Baseline: use the verified P02 diagnostic commit/remote tree. Fix reproduced R-01 and R-02 only, following exact notes/001–006 proof and scoped server/index.js pushEventsToRedisOrFallback/onBufferPush and flushBuffer, server/socket/handlers.js sync_data/batch_sync ACK, and their three existing P02 test files. First document safe ACK/retry semantics, exactly-once versus at-least-once CSV contract and crash recovery. Add disposable fault tests proving no positive ACK for non-durable data and no unbounded duplicate replay. Runtime sources core/data/buffer.ts and server/workers/csvWriter.js remain READ-ONLY unless a separately justified exact file/line amendment is approved. Preserve all UI/auth/SDK/public health behavior and existing error codes, production data, security, other features and Docker. One focused CI, diff/security audit, notes update, commit and 3-remote parity. No deployment, migrations, tags/releases or scope expansion. If a safe fix cannot be achieved within these exact methods, stop and report the required additional Scope Lock rather than broadening automatically.`
|
||||||
|
|
||||||
After separate approved remediation and verified P02 closure, **P03 plan** remains CI-06 Linux Python runner provisioning, CI-07 isolated Docker/BuildKit failure remediation and 3-image build, CI-08 supply-chain advisory triage, and non-mutating readiness evidence. P03 should not modify unrelated UI/runtime/API code by default.
|
After separate approved remediation and verified P02 closure, **P03 plan** remains CI-06 Linux Python runner provisioning, CI-07 isolated Docker/BuildKit failure remediation and 3-image build, CI-08 supply-chain advisory triage, and non-mutating readiness evidence. P03 should not modify unrelated UI/runtime/API code by default.
|
||||||
|
|
||||||
|
|
||||||
|
## P02 Runtime Remediation Execution Handoff — 2026-11-22
|
||||||
|
|
||||||
|
The user explicitly approved **only** R-01 Redis failure ACK and R-02 Redis-to-CSV replay corrections in these methods: `server/index.js` `pushEventsToRedisOrFallback` / `flushBuffer` / existing `onBufferPush` integration; `server/socket/handlers.js` `sync_data` / `batch_sync` ACK handling; `core/data/buffer.ts` `flush` if needed; the three earlier P02 test files; `notes/001–006`. No other application method/UI/API/auth/SDK/Docker/CI-workflow edit. Public R-03 `/health` semantics remain unchanged.
|
||||||
|
|
||||||
|
Local candidate `44eee8877e40d1683f44502c2a0ba3d437a03b88` is **not release-ready** until Gitea acceptance. Before closure, prove: negative ACK on Redis failure with no sensitive error leakage; ordinary successful Redis ACK preserved; both single and batch retries don't convert volatile failures to success; Redis queued event replays after crash without writing duplicate CSV rows in the Port 4000 worker and Port 3000 web buffer; CSV read/parse failure never causes premature Redis trim; validate bounded scan overhead and edge cases for rotated/gzipped partitions; all historical 401/403/429/507/503 tests unchanged.
|
||||||
|
|
||||||
|
**Stop rule:** if required fixes need touching `validateSingleEventAsync`, `csvWriter.js` or separate journal/storage scheme, do not broaden the implementation scope; record the residual and request new explicit approval. Existing unrelated files including `MEMORY.md` remain unchanged due exact Scope Lock; this note is the cross-IDE handoff. P03 must wait for P02 CLOSED, not merely implementation committed.
|
||||||
@@ -91,3 +91,19 @@ Historical baseline code stayed frozen through these CI workflow changes. Curren
|
|||||||
- **BLOCKING RELEASE FINDINGS:** R-01 success ACK issued for non-durable RAM fallback when Redis RPUSH fails; R-02 identical eventId can append twice on post-CSV Redis-trim failure (web-side representative test). R-03 health remains online on Redis loss; liveness-only behavior is backward-compatible, protected dependency-readiness verification remains missing.
|
- **BLOCKING RELEASE FINDINGS:** R-01 success ACK issued for non-durable RAM fallback when Redis RPUSH fails; R-02 identical eventId can append twice on post-CSV Redis-trim failure (web-side representative test). R-03 health remains online on Redis loss; liveness-only behavior is backward-compatible, protected dependency-readiness verification remains missing.
|
||||||
- **P02 PHASE = BLOCKED**, diagnostic implementation done, release reliability acceptance NOT satisfied. Runtime remediation requires separate exact approval. 1/3 phases still CLOSED (P01 only); P02 and P03 remain unclosed. No P03 start authorization from this diagnostic completion.
|
- **P02 PHASE = BLOCKED**, diagnostic implementation done, release reliability acceptance NOT satisfied. Runtime remediation requires separate exact approval. 1/3 phases still CLOSED (P01 only); P02 and P03 remain unclosed. No P03 start authorization from this diagnostic completion.
|
||||||
- Next approval must explicitly authorize exact narrow server/core patch for ACK durability and CSV replay deduplication, plus concrete no-loss/no-duplicate tests. Any readiness endpoint change needs separate explicit approval. User's initial P02 approval is insufficient for runtime mutation.
|
- Next approval must explicitly authorize exact narrow server/core patch for ACK durability and CSV replay deduplication, plus concrete no-loss/no-duplicate tests. Any readiness endpoint change needs separate explicit approval. User's initial P02 approval is insufficient for runtime mutation.
|
||||||
|
|
||||||
|
|
||||||
|
## P02-RUNTIME-REMEDIATION — Candidate implementation checkpoint (2026-11-22)
|
||||||
|
|
||||||
|
- **Current P02: BLOCKED / NOT CLOSED.** One previous diagnostic Gitea CI #63 reproduced R-01/R-02/R-03; runtime remediation was separately approved after that. Verified completed implementation phases remain **1/3** (P01 only). Do not approve P03 yet.
|
||||||
|
- Published P02 diagnosis baseline `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`; isolated local candidate commit `44eee8877e40d1683f44502c2a0ba3d437a03b88` on `p02-runtime-remediation-20261122`.
|
||||||
|
- Exactly five code/test files changed in this candidate: `server/index.js`, `server/socket/handlers.js`, `core/data/buffer.ts`, `tests/phase02-sync.test.ts`, `tests/phase03-storage-buffer.test.ts`. No `server/workers/csvWriter.js`, `app/**`, `components/**`, SDK, public R-03 health, CI workflow, database, migration, production data, secrets or deployment touched.
|
||||||
|
- JS `node --check` on two changed JS files PASS; `git diff --check` PASS and allowlist audit PASS. Targeted TS/Node regression and Gitea CI **NOT YET RUN TO TERMINAL PASS**. The Gitea connector/host was unavailable when dispatch/push was attempted. Local temporary dependency installation also not yet complete.
|
||||||
|
- The original user PC root `D:\VibTools_Workspace\01_Products\14_DataStack` still has 16 pre-existing staged P01 paths on older `d8ac96e`; deliberately preserved without reset/stash/discard. All P02 implementation is in a separate temporary worktree. Do not assume the old local root is up to date.
|
||||||
|
- R-01 and R-02 implementations are candidates; practical reliability limitations and I/O cost recorded in `notes/001` and `notes/004`. No false claim of globally exactly-once CSV or universally durable ACK.
|
||||||
|
- **Next gates:** restore Gitea access; run one focused disposable acceptance; fix any in-scope test/TS defects; record exact CI terminal results; only after required acceptance and security audit, squash single phase main commit and verify GitHub/GitLab/Gitea tree parity; then, and only then, decide P02 CLOSED versus residual-risk blocker.
|
||||||
|
|
||||||
|
|
||||||
|
### Verification environment blocker — local dependency install
|
||||||
|
|
||||||
|
The authorized Windows PC attempted a **local disposable** `npm.cmd ci --ignore-scripts --no-audit --no-fund --prefer-offline` in the isolated P02 worktree only; it failed with `npm ERR! ETARGET No matching version found for yaml@2.9.1` plus cleanup EPERM warnings in the temp worktree's `node_modules`. Exact output captured in `%TEMP%\DataStack-P02-npm-ci.log` on the authorized PC. This is **not a passed code test nor proof of a repository package defect**, and `package-lock.json` / `package.json` cannot be edited under P02 scope. Gitea MCP remained UNAVAILABLE and the Git remote/HTTP service was not responding when CI was attempted, so the mandatory P02 focused CI was **not dispatched**. Phase remains BLOCKED pending actual CI evidence; no runtime change was pushed to production/main.
|
||||||
@@ -64,3 +64,15 @@ The previous #58 package-lock error was resolved by adding the exact nested opti
|
|||||||
| R-03 availability | Redis `llen` rejection leaves Express `/health` online with `redisConnected:false`. Recovery flips the flag while HTTP 200 remains unchanged. | Public 200 is liveness by existing contract; a protected dependency readiness probe/503 is **not implemented** and needs explicit approval. |
|
| R-03 availability | Redis `llen` rejection leaves Express `/health` online with `redisConnected:false`. Recovery flips the flag while HTTP 200 remains unchanged. | Public 200 is liveness by existing contract; a protected dependency readiness probe/503 is **not implemented** and needs explicit approval. |
|
||||||
|
|
||||||
Security invariants: do not expose secrets in ACK/errors, keep JWT/auth and 401/403/429/503/507 contracts unchanged, and never run failure-injection against actual Redis, Postgres, production API, live events, or user CSV. Disposable fake Redis and temporary files only. CI #63: https://dev.ivib.cc/vibtools/DataStack/actions/runs/63; results tracked in notes/003.
|
Security invariants: do not expose secrets in ACK/errors, keep JWT/auth and 401/403/429/503/507 contracts unchanged, and never run failure-injection against actual Redis, Postgres, production API, live events, or user CSV. Disposable fake Redis and temporary files only. CI #63: https://dev.ivib.cc/vibtools/DataStack/actions/runs/63; results tracked in notes/003.
|
||||||
|
|
||||||
|
|
||||||
|
## P02 Runtime Candidate — Error-path change inventory (pending CI)
|
||||||
|
|
||||||
|
| Path | Candidate result | Explicit limitation |
|
||||||
|
|---|---|---|
|
||||||
|
| Redis `RPUSH` rejects with configured Redis client | Retains event in volatile RAM as preexisting fallback but returns `BUFFER_PERSIST_FAILED` to Socket.IO via thrown error; **no successful ACK** for that failure; error text does not expose connection details | No-Redis intentional legacy RAM mode unchanged, still not a durable guarantee |
|
||||||
|
| `sync_data` and `batch_sync` failures | `success:false`, `code:BUFFER_PERSIST_FAILED` and best-effort rollback of previously claimed idempotency keys, negative failure marker for in-process retries | If claim rollback cannot reach Redis and process crashes, durable claim/queue atomicity remains unsolved |
|
||||||
|
| Redis buffer `LTRIM` rejects after successful CSV append | Subsequent flush checks physically stored CSV IDs and avoids appending matching rows again; Redis is trimmed only after successful check/write | Concurrent independent CSV appenders, compressed archived CSV, corrupt partial rows, and per-flush CSV scan cost require separate proof |
|
||||||
|
| Legacy public health | **UNCHANGED** HTTP 200 liveness contract and Redis-connected diagnostic field; R-03 readiness is not introduced in this approval | Does not prove full DB/Redis transactional readiness |
|
||||||
|
|
||||||
|
Security/API invariants: leave token/JWT/auth/rate limits/quotas/webhooks/MCP/sandbox/UI/SDK unchanged. No production Redis/Postgres/API/CSV mutation. No lockfile/dependency versions or CI workflow changes. CI acceptance remains pending; these are code-level observations, not release certification.
|
||||||
@@ -52,3 +52,11 @@ Do not mark "actual working" simply because a patch compiled. Distinguish source
|
|||||||
- R-02: existing CSV-before-Redis-trim order audited, *not corrected*. Exactly-once replay safety is **NOT VERIFIED**.
|
- R-02: existing CSV-before-Redis-trim order audited, *not corrected*. Exactly-once replay safety is **NOT VERIFIED**.
|
||||||
- R-03: current public Express health 200 plus `redisConnected:false` is liveness, not dependency readiness. Separate readiness remains **NOT IMPLEMENTED**.
|
- R-03: current public Express health 200 plus `redisConnected:false` is liveness, not dependency readiness. Separate readiness remains **NOT IMPLEMENTED**.
|
||||||
- Gitea disposable CI #63: https://dev.ivib.cc/vibtools/DataStack/actions/runs/63. The previous P01 working results remain valid; no live read/write test or production deploy is included in P02.
|
- Gitea disposable CI #63: https://dev.ivib.cc/vibtools/DataStack/actions/runs/63. The previous P01 working results remain valid; no live read/write test or production deploy is included in P02.
|
||||||
|
|
||||||
|
|
||||||
|
## P02 Runtime Candidate Feature Delta — 2026-11-22 (NOT RELEASED)
|
||||||
|
|
||||||
|
- **Intended change:** failure cases on socket ingestion report `BUFFER_PERSIST_FAILED` instead of positive ACK when Redis RPUSH rejects; single and batch code paths tested by new disposable assertions. Ordinary success ACK, auth, limits and SDK public interfaces intended unchanged.
|
||||||
|
- **Intended change:** Redis replay flush on both ingestion/Next paths inspects existing client CSV partitions and filters already-stored event IDs before reappend; safe failure leaves Redis queue untrimmed. This prevents the tested post-append/LTRIM replay duplication but does **not** prove transactionally exactly-once delivery or zero performance impact.
|
||||||
|
- **No changes:** application UI/UX, auth, SDK, database schema, Docker, environment, public R-03 health, other features and production runtime. No deployment or live verification.
|
||||||
|
- **Status:** P02 code candidate locally committed, acceptance CI blocked by Gitea unavailability; **NOT CLOSED**, not production-ready. P01 remains only completed phase.
|
||||||
@@ -237,3 +237,25 @@ return bufferQueue.length;
|
|||||||
- `P02 R-03 diagnostic: Redis query failure leaves public ingestion health 200/online` — **ok 16**, same suite exit 0, Redis flag false while liveness stays online.
|
- `P02 R-03 diagnostic: Redis query failure leaves public ingestion health 200/online` — **ok 16**, same suite exit 0, Redis flag false while liveness stays online.
|
||||||
- `P02 R-02 diagnostic: post-CSV Redis trim failure replays identical event on retry` — **ok 18**, `phase03-storage-buffer` suite exit 0; duplicate append after retry reproduced on representative web buffer path. **Direct ingestion Port 4000 CSV worker crash coverage remains outstanding** and should be requested in remediation approval before guarantee changes.
|
- `P02 R-02 diagnostic: post-CSV Redis trim failure replays identical event on retry` — **ok 18**, `phase03-storage-buffer` suite exit 0; duplicate append after retry reproduced on representative web buffer path. **Direct ingestion Port 4000 CSV worker crash coverage remains outstanding** and should be requested in remediation approval before guarantee changes.
|
||||||
- These three diagnostic tests **prove findings, do not close production-readiness**. See Gitea https://dev.ivib.cc/vibtools/DataStack/actions/runs/63; never call run successful until terminal conclusion is confirmed. No server-side source fix applied.
|
- These three diagnostic tests **prove findings, do not close production-readiness**. See Gitea https://dev.ivib.cc/vibtools/DataStack/actions/runs/63; never call run successful until terminal conclusion is confirmed. No server-side source fix applied.
|
||||||
|
|
||||||
|
|
||||||
|
## F. P02 Remediation Candidate Patch Proof — 2026-11-22
|
||||||
|
|
||||||
|
Base: Gitea/GitHub published P02 diagnostic tree `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`; GitLab distinct commit, same previous tree. Local candidate commit `44eee8877e40d1683f44502c2a0ba3d437a03b88` on isolated worktree. Initial exact delta: 5 approved source/test paths, 310 insertions and 22 deletions, no application UI, Python SDK, auth, public health route, CI or deployment edit. `git diff --check` and `node --check` on JS files PASS.
|
||||||
|
|
||||||
|
- `server/index.js:pushEventsToRedisOrFallback`: catch RPUSH failure, keep previous RAM buffering, throw sanitized `BUFFER_PERSIST_FAILED`; no positive ACK for that branch.
|
||||||
|
- `server/socket/handlers.js:sync_data/batch_sync`: negative ACK and best-effort claimed idempotency-key rollback with negative retry guard, no change to legitimate duplicate success path or existing auth/limits/error codes.
|
||||||
|
- `server/index.js:flushBuffer` and `core/data/buffer.ts:flush`: scoped streaming CSV/CSV.gz read-before-reappend (parsing quoted/multiline rows), exact client/date/event ID match; failures abort before Redis LTRIM. No changes to `server/workers/csvWriter.js`.
|
||||||
|
- `tests/phase02-sync.test.ts`: R-01 single/batch outage and duplicate claim ACK negative, successful recovery ACK, and unchanged R-03 liveness.
|
||||||
|
- `tests/phase03-storage-buffer.test.ts`: web-side replay verifies exactly one physically stored row; new direct Port 4000 writer Redis-trim failure/recovery proof. All tests on fake Redis and disposable temp CSV paths.
|
||||||
|
|
||||||
|
**Unverified:** Gitea P02 implementation acceptance was unavailable due inaccessible host/connector. Local `npm ci` not confirmed successful. Do not invent a CI run ID or claim full PASS. Full cross-process transactional exactly-once and all Redis claim crash windows are out of proof scope and may require exact additional approval. The streaming dedupe scan changes I/O cost per flush; performance validation before production is required.
|
||||||
|
|
||||||
|
|
||||||
|
### Blocked acceptance evidence — actual tool outcomes
|
||||||
|
|
||||||
|
- JS syntax: local `node --check server/index.js` and `node --check server/socket/handlers.js` PASS.
|
||||||
|
- Source scope: local `git diff --check` PASS; only the five allowed P02 runtime/test paths changed before notes updates.
|
||||||
|
- Gitea: connection returned `UNAVAILABLE: McpServerError: Connection failed`; Git remote `dev.ivib.cc` unresponsive. No Gitea CI run ID exists for candidate `44eee8877e40d1683f44502c2a0ba3d437a03b88`.
|
||||||
|
- Local Windows npm: `npm ci` failed `ETARGET yaml@2.9.1` and cleanup `EPERM` in the isolated worktree, **not** a TypeScript/test result. Dependency changes are out of P02 scope.
|
||||||
|
- Therefore neither TS typecheck nor regression-suite remediation acceptance is established; keep P02 BLOCKED and do not advance P03 or alter any protected runtime/source beyond this approval.
|
||||||
+74
-6
@@ -165,6 +165,7 @@ function setRedisClientForTests(client) {
|
|||||||
async function pushEventsToRedisOrFallback(events) {
|
async function pushEventsToRedisOrFallback(events) {
|
||||||
if (!events || events.length === 0) return bufferQueue.length;
|
if (!events || events.length === 0) return bufferQueue.length;
|
||||||
const redis = getRedisClient();
|
const redis = getRedisClient();
|
||||||
|
let persistFailed = false;
|
||||||
if (redis) {
|
if (redis) {
|
||||||
const serialized = events.map((e) => JSON.stringify(e));
|
const serialized = events.map((e) => JSON.stringify(e));
|
||||||
let bufferLen = null;
|
let bufferLen = null;
|
||||||
@@ -173,6 +174,7 @@ async function pushEventsToRedisOrFallback(events) {
|
|||||||
redisConnected = true;
|
redisConnected = true;
|
||||||
} catch {
|
} catch {
|
||||||
redisConnected = false;
|
redisConnected = false;
|
||||||
|
persistFailed = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (bufferLen !== null) {
|
if (bufferLen !== null) {
|
||||||
@@ -189,6 +191,11 @@ async function pushEventsToRedisOrFallback(events) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
bufferQueue.push(...events);
|
bufferQueue.push(...events);
|
||||||
|
if (persistFailed) {
|
||||||
|
// Keep the volatile fallback for later recovery, but never ACK it as durable.
|
||||||
|
// Suppress sensitive Redis transport details in the client-facing error.
|
||||||
|
throw new Error('BUFFER_PERSIST_FAILED: Redis unavailable; event not durably queued');
|
||||||
|
}
|
||||||
return bufferQueue.length;
|
return bufferQueue.length;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -834,6 +841,61 @@ async function isClientDeletedForFlush(clientId, redis, cache) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function flushBuffer(options = {}) {
|
async function flushBuffer(options = {}) {
|
||||||
|
|
||||||
|
// P02 R-02: Redis and CSV cannot be committed atomically. Before replaying an
|
||||||
|
// untrimmed Redis batch, check actual CSV partitions for previously written IDs.
|
||||||
|
// Streaming parsing bounds memory and preserves multiline/quoted CSV fields.
|
||||||
|
const excludeAlreadyPersisted = async (events) => {
|
||||||
|
if (!events.length) return events;
|
||||||
|
const fs = require('node:fs');
|
||||||
|
const path = require('node:path');
|
||||||
|
const zlib = require('node:zlib');
|
||||||
|
const { pipeline } = require('node:stream/promises');
|
||||||
|
const { Writable } = require('node:stream');
|
||||||
|
const { parse } = require('csv-parse');
|
||||||
|
const groups = new Map();
|
||||||
|
for (const event of events) {
|
||||||
|
const clientId = String(event.clientId || 'unknown');
|
||||||
|
const date = writer.extractSafePartitionDate(event.timestamp);
|
||||||
|
const key = JSON.stringify([clientId, date]);
|
||||||
|
if (!groups.has(key)) groups.set(key, { clientId, date, ids: new Set() });
|
||||||
|
groups.get(key).ids.add(String(event.eventId));
|
||||||
|
}
|
||||||
|
const persisted = new Set();
|
||||||
|
for (const [groupKey, group] of groups) {
|
||||||
|
const dir = writer.ensureClientDir(group.clientId);
|
||||||
|
const entries = await fs.promises.readdir(dir, { withFileTypes: true });
|
||||||
|
for (const entry of entries) {
|
||||||
|
if (!entry.isFile() || !writer.isPartitionFilename(entry.name) ||
|
||||||
|
writer.extractDateFromFilename(entry.name) !== group.date) continue;
|
||||||
|
const reader = fs.createReadStream(path.join(dir, entry.name));
|
||||||
|
const streams = [reader];
|
||||||
|
if (entry.name.endsWith('.gz')) streams.push(zlib.createGunzip());
|
||||||
|
streams.push(parse({ columns: true, bom: true, skip_empty_lines: true }));
|
||||||
|
streams.push(new Writable({
|
||||||
|
objectMode: true,
|
||||||
|
write(row, _enc, done) {
|
||||||
|
if (row.clientId === group.clientId && group.ids.has(String(row.eventId))) {
|
||||||
|
persisted.add(JSON.stringify([groupKey, String(row.eventId)]));
|
||||||
|
}
|
||||||
|
done();
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
// On read/parse failure, abort without LTRIM: never discard uncertain rows.
|
||||||
|
await pipeline(...streams);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const batchSeen = new Set();
|
||||||
|
return events.filter((event) => {
|
||||||
|
const key = JSON.stringify([String(event.clientId || 'unknown'),
|
||||||
|
writer.extractSafePartitionDate(event.timestamp)]);
|
||||||
|
const rowKey = JSON.stringify([key, String(event.eventId)]);
|
||||||
|
if (persisted.has(rowKey) || batchSeen.has(rowKey)) return false;
|
||||||
|
batchSeen.add(rowKey);
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
if (pendingRedisWrites.size > 0) {
|
if (pendingRedisWrites.size > 0) {
|
||||||
await Promise.allSettled([...pendingRedisWrites]);
|
await Promise.allSettled([...pendingRedisWrites]);
|
||||||
}
|
}
|
||||||
@@ -979,9 +1041,12 @@ async function flushBuffer(options = {}) {
|
|||||||
if (lockLost) break;
|
if (lockLost) break;
|
||||||
|
|
||||||
if (parsedBatch.length > 0) {
|
if (parsedBatch.length > 0) {
|
||||||
const res = await writer.writeBatch(parsedBatch);
|
const unwritten = await excludeAlreadyPersisted(parsedBatch);
|
||||||
totalWritten += res.written;
|
if (unwritten.length > 0) {
|
||||||
for (const f of res.files) filesSet.add(f);
|
const res = await writer.writeBatch(unwritten);
|
||||||
|
totalWritten += res.written;
|
||||||
|
for (const f of res.files) filesSet.add(f);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (Date.now() - lastRenewAt >= renewIntervalMs) {
|
if (Date.now() - lastRenewAt >= renewIntervalMs) {
|
||||||
@@ -1022,9 +1087,12 @@ async function flushBuffer(options = {}) {
|
|||||||
|
|
||||||
if (validBatch.length > 0) {
|
if (validBatch.length > 0) {
|
||||||
try {
|
try {
|
||||||
const res = await writer.writeBatch(validBatch);
|
const unwritten = await excludeAlreadyPersisted(validBatch);
|
||||||
totalWritten += res.written;
|
if (unwritten.length > 0) {
|
||||||
for (const f of res.files) filesSet.add(f);
|
const res = await writer.writeBatch(unwritten);
|
||||||
|
totalWritten += res.written;
|
||||||
|
for (const f of res.files) filesSet.add(f);
|
||||||
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
bufferQueue.unshift(...validBatch);
|
bufferQueue.unshift(...validBatch);
|
||||||
throw err;
|
throw err;
|
||||||
|
|||||||
@@ -549,6 +549,9 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
: new Date().toISOString();
|
: new Date().toISOString();
|
||||||
const eventData = extractSingleEventData(payload);
|
const eventData = extractSingleEventData(payload);
|
||||||
|
|
||||||
|
// A previous Redis write failure can leave an uncommitted idempotency
|
||||||
|
// claim. Never report that claim as a successful durable duplicate.
|
||||||
|
const failedAt = idempotencyMap.get(`${clientId}:${eventId}`);
|
||||||
const check = await validateSingleEventAsync(
|
const check = await validateSingleEventAsync(
|
||||||
clientId,
|
clientId,
|
||||||
eventId,
|
eventId,
|
||||||
@@ -557,7 +560,13 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
);
|
);
|
||||||
if (!check.allowed) {
|
if (!check.allowed) {
|
||||||
if (check.duplicate) {
|
if (check.duplicate) {
|
||||||
if (callback) {
|
if (typeof failedAt === "number" && failedAt < 0 &&
|
||||||
|
Date.now() + failedAt < IDEMPOTENCY_TTL_MS) {
|
||||||
|
if (callback) callback({
|
||||||
|
success: false, eventId, code: "BUFFER_PERSIST_FAILED",
|
||||||
|
error: "BUFFER_PERSIST_FAILED: Event not durably queued",
|
||||||
|
});
|
||||||
|
} else if (callback) {
|
||||||
callback({ success: true, eventId, note: "DUPLICATE_DROPPED" });
|
callback({ success: true, eventId, note: "DUPLICATE_DROPPED" });
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
@@ -585,6 +594,19 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
try {
|
try {
|
||||||
await onBufferPush();
|
await onBufferPush();
|
||||||
} catch (persistErr) {
|
} catch (persistErr) {
|
||||||
|
// Invalidate the pre-persistence idempotency claim so SDK retry can
|
||||||
|
// resubmit after Redis recovers. Keep the volatile RAM event pending.
|
||||||
|
const claimKey = `${clientId}:${eventId}`;
|
||||||
|
idempotencyMap.set(claimKey, -Date.now());
|
||||||
|
const claimRedis = getRedisClient();
|
||||||
|
if (claimRedis && typeof claimRedis.del === "function") {
|
||||||
|
try {
|
||||||
|
await claimRedis.del(`${REDIS_IDEMPOTENCY_PREFIX}${claimKey}`);
|
||||||
|
idempotencyMap.delete(claimKey);
|
||||||
|
} catch {
|
||||||
|
// Claim may remain: negative ACK must be retained for retries.
|
||||||
|
}
|
||||||
|
}
|
||||||
if (callback) {
|
if (callback) {
|
||||||
callback({
|
callback({
|
||||||
success: false,
|
success: false,
|
||||||
@@ -689,6 +711,8 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
const rateLimitedEventIds = [];
|
const rateLimitedEventIds = [];
|
||||||
const oversizedEventIds = [];
|
const oversizedEventIds = [];
|
||||||
const errors = [];
|
const errors = [];
|
||||||
|
const acceptedEventIds = [];
|
||||||
|
let volatileDuplicate = false;
|
||||||
|
|
||||||
for (const item of batch) {
|
for (const item of batch) {
|
||||||
const safeItem = item && typeof item === "object" ? item : {};
|
const safeItem = item && typeof item === "object" ? item : {};
|
||||||
@@ -702,6 +726,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
: new Date().toISOString();
|
: new Date().toISOString();
|
||||||
const rawData = extractBatchItemData(safeItem);
|
const rawData = extractBatchItemData(safeItem);
|
||||||
|
|
||||||
|
const failedAt = idempotencyMap.get(`${clientId}:${eventId}`);
|
||||||
const check = await validateSingleEventAsync(
|
const check = await validateSingleEventAsync(
|
||||||
clientId,
|
clientId,
|
||||||
eventId,
|
eventId,
|
||||||
@@ -710,7 +735,15 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
);
|
);
|
||||||
if (!check.allowed) {
|
if (!check.allowed) {
|
||||||
if (check.duplicate) {
|
if (check.duplicate) {
|
||||||
duplicates += 1;
|
if (typeof failedAt === "number" && failedAt < 0 &&
|
||||||
|
Date.now() + failedAt < IDEMPOTENCY_TTL_MS) {
|
||||||
|
rejected += 1;
|
||||||
|
volatileDuplicate = true;
|
||||||
|
if (errors.length < 5) errors.push(
|
||||||
|
`${eventId}: BUFFER_PERSIST_FAILED: Event not durably queued`);
|
||||||
|
} else {
|
||||||
|
duplicates += 1;
|
||||||
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
rejected += 1;
|
rejected += 1;
|
||||||
@@ -735,6 +768,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
payload: check.payloadStr,
|
payload: check.payloadStr,
|
||||||
});
|
});
|
||||||
processed += 1;
|
processed += 1;
|
||||||
|
acceptedEventIds.push(eventId);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (processed > 0) {
|
if (processed > 0) {
|
||||||
@@ -742,6 +776,19 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
try {
|
try {
|
||||||
await onBufferPush();
|
await onBufferPush();
|
||||||
} catch (persistErr) {
|
} catch (persistErr) {
|
||||||
|
const claimRedis = getRedisClient();
|
||||||
|
for (const eventId of acceptedEventIds) {
|
||||||
|
const claimKey = `${clientId}:${eventId}`;
|
||||||
|
idempotencyMap.set(claimKey, -Date.now());
|
||||||
|
if (claimRedis && typeof claimRedis.del === "function") {
|
||||||
|
try {
|
||||||
|
await claimRedis.del(`${REDIS_IDEMPOTENCY_PREFIX}${claimKey}`);
|
||||||
|
idempotencyMap.delete(claimKey);
|
||||||
|
} catch {
|
||||||
|
// Preserve negative-ACK retry guard if claim removal fails.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
if (callback) {
|
if (callback) {
|
||||||
callback({
|
callback({
|
||||||
success: false,
|
success: false,
|
||||||
@@ -760,11 +807,13 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (processed === 0 && duplicates === 0 && rejected > 0) {
|
if (processed === 0 && duplicates === 0 && rejected > 0) {
|
||||||
const code = hitPayloadLimit
|
const code = volatileDuplicate
|
||||||
? "PAYLOAD_TOO_LARGE"
|
? "BUFFER_PERSIST_FAILED"
|
||||||
: hitRateLimit
|
: hitPayloadLimit
|
||||||
? "RATE_LIMIT_EXCEEDED"
|
? "PAYLOAD_TOO_LARGE"
|
||||||
: "BATCH_REJECTED";
|
: hitRateLimit
|
||||||
|
? "RATE_LIMIT_EXCEEDED"
|
||||||
|
: "BATCH_REJECTED";
|
||||||
if (callback) {
|
if (callback) {
|
||||||
callback({
|
callback({
|
||||||
success: false,
|
success: false,
|
||||||
|
|||||||
@@ -1246,12 +1246,29 @@ test("P02 R-01 diagnostic: Redis RPUSH failure yields positive WS ACK for volati
|
|||||||
);
|
);
|
||||||
const committedRows = await failingRedis.lrange("datastack:buffer:events", 0, -1);
|
const committedRows = await failingRedis.lrange("datastack:buffer:events", 0, -1);
|
||||||
const volatileRows = serverMod.bufferQueue.filter((e) => e.eventId === "p02-ack-before-crash-001");
|
const volatileRows = serverMod.bufferQueue.filter((e) => e.eventId === "p02-ack-before-crash-001");
|
||||||
assert.equal(ack.success, true, "Existing handler currently sends a positive ACK after Redis failure");
|
assert.equal(ack.success, false, "P02 remediation must reject non-durable ACK");
|
||||||
|
assert.equal(ack.code, "BUFFER_PERSIST_FAILED");
|
||||||
|
const sameIdRetry = await emitWithAck<{ success: boolean; code?: string }>(
|
||||||
|
socket, "sync_data",
|
||||||
|
{ eventId: "p02-ack-before-crash-001", data: { value: 42 } },
|
||||||
|
);
|
||||||
|
assert.equal(sameIdRetry.success, false,
|
||||||
|
"Pending idempotency claims must not return false positive duplicate ACK");
|
||||||
|
assert.equal(sameIdRetry.code, "BUFFER_PERSIST_FAILED");
|
||||||
assert.equal(committedRows.length, 0, "Fake Redis has no durable event");
|
assert.equal(committedRows.length, 0, "Fake Redis has no durable event");
|
||||||
assert.equal(volatileRows.length, 1, "Acknowledged event is RAM-only");
|
assert.equal(volatileRows.length, 1, "Acknowledged event is RAM-only");
|
||||||
// Crash model: restart loses RAM but fake Redis survives. No actual process is killed.
|
// Crash model: restart loses RAM but fake Redis survives. No actual process is killed.
|
||||||
serverMod.bufferQueue.length = 0;
|
serverMod.bufferQueue.length = 0;
|
||||||
assert.equal((await failingRedis.lrange("datastack:buffer:events", 0, -1)).length, 0);
|
assert.equal((await failingRedis.lrange("datastack:buffer:events", 0, -1)).length, 0);
|
||||||
|
// A new event may ACK only after Redis actually resumes persistence.
|
||||||
|
const healthyRedis = createSharedMockRedis();
|
||||||
|
serverMod.setRedisClientForTests(healthyRedis);
|
||||||
|
const recovered = await emitWithAck<{ success: boolean; eventId?: string }>(
|
||||||
|
socket, "sync_data",
|
||||||
|
{ eventId: "p02-ack-recovered-002", data: { value: 43 } },
|
||||||
|
);
|
||||||
|
assert.equal(recovered.success, true);
|
||||||
|
assert.equal((await healthyRedis.lrange("datastack:buffer:events", 0, -1)).length, 1);
|
||||||
} finally {
|
} finally {
|
||||||
socket?.disconnect();
|
socket?.disconnect();
|
||||||
serverMod.setRedisClientForTests(undefined);
|
serverMod.setRedisClientForTests(undefined);
|
||||||
@@ -1281,3 +1298,43 @@ test("P02 R-03 diagnostic: Redis query failure leaves public ingestion health 20
|
|||||||
serverMod.setRedisClientForTests(undefined);
|
serverMod.setRedisClientForTests(undefined);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("P02 R-01: batch_sync rejects non-durable Redis writes and pending duplicate claims", async () => {
|
||||||
|
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
|
||||||
|
const previousQueue = serverMod.bufferQueue.slice();
|
||||||
|
const failingRedis = createSharedMockRedis();
|
||||||
|
failingRedis.rpush = async () => { throw new Error("P02 injected batch Redis outage"); };
|
||||||
|
serverMod.setRedisClientForTests(failingRedis);
|
||||||
|
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
|
||||||
|
serverMod.bufferQueue.length = 0;
|
||||||
|
let socket: Socket | undefined;
|
||||||
|
const payload = [
|
||||||
|
{ eventId: "p02-batch-volatile-1", data: { value: 1 } },
|
||||||
|
{ eventId: "p02-batch-volatile-2", data: { value: 2 } },
|
||||||
|
];
|
||||||
|
try {
|
||||||
|
const client = await SDKManager.registerClientAsync({
|
||||||
|
clientId: "p02_redis_batch_failure",
|
||||||
|
name: "P02 Redis Batch Failure",
|
||||||
|
rateLimit: 100,
|
||||||
|
});
|
||||||
|
socket = await connectSocket(client.token);
|
||||||
|
const ack = await emitWithAck<{ success: boolean; code?: string; processed?: number }>(
|
||||||
|
socket, "batch_sync", payload,
|
||||||
|
);
|
||||||
|
assert.equal(ack.success, false);
|
||||||
|
assert.equal(ack.code, "BUFFER_PERSIST_FAILED");
|
||||||
|
assert.equal(ack.processed, 0);
|
||||||
|
assert.equal((await failingRedis.lrange("datastack:buffer:events", 0, -1)).length, 0);
|
||||||
|
const retry = await emitWithAck<{ success: boolean; code?: string }>(
|
||||||
|
socket, "batch_sync", payload,
|
||||||
|
);
|
||||||
|
assert.equal(retry.success, false);
|
||||||
|
assert.equal(retry.code, "BUFFER_PERSIST_FAILED");
|
||||||
|
} finally {
|
||||||
|
socket?.disconnect();
|
||||||
|
serverMod.setRedisClientForTests(undefined);
|
||||||
|
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
|
||||||
|
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
|
||||||
|
}
|
||||||
|
});
|
||||||
@@ -1584,10 +1584,58 @@ test("P02 R-02 diagnostic: post-CSV Redis trim failure replays identical event o
|
|||||||
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 1, "Redis still queues previously appended CSV event");
|
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 1, "Redis still queues previously appended CSV event");
|
||||||
await DataBufferEngine.flush();
|
await DataBufferEngine.flush();
|
||||||
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0);
|
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0);
|
||||||
assert.equal(writtenIds.filter((id) => id === event.eventId).length, 2,
|
assert.equal(writtenIds.filter((id) => id === event.eventId).length, 1,
|
||||||
"A second flush appends the same eventId again after failed trim: at-least-once, not exactly-once");
|
"Replay must detect the persisted CSV eventId before attempting another append");
|
||||||
|
const csvPath = path.join(tmpStorageDir, event.clientId, "2026-11-22.csv");
|
||||||
|
const csvContents = fs.readFileSync(csvPath, "utf8");
|
||||||
|
assert.equal(csvContents.split(event.eventId).length - 1, 1,
|
||||||
|
"Exactly one physical CSV row must exist after a failed LTRIM and retry");
|
||||||
} finally {
|
} finally {
|
||||||
CSVStorageManager.appendBatch = originalAppend;
|
CSVStorageManager.appendBatch = originalAppend;
|
||||||
DataBufferEngine.setRedisClientForTests(undefined);
|
DataBufferEngine.setRedisClientForTests(undefined);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("P02 R-02: Port 4000 CSV worker suppresses a crash-window replay after Redis LTRIM error", async () => {
|
||||||
|
const mockRedis = new FakeRedisClient();
|
||||||
|
const originalTrim = mockRedis.ltrim.bind(mockRedis);
|
||||||
|
let failedOnce = false;
|
||||||
|
mockRedis.ltrim = async (key: string, start: number, stop: number): Promise<"OK"> => {
|
||||||
|
if (key === REDIS_BUFFER_KEY && !failedOnce) {
|
||||||
|
failedOnce = true;
|
||||||
|
throw new Error("P02 Port4000 injected post-CSV trim failure");
|
||||||
|
}
|
||||||
|
return originalTrim(key, start, stop);
|
||||||
|
};
|
||||||
|
const ingestion = require("../server/index.js") as IngestionServerModule;
|
||||||
|
const previousDir = ingestion.writer.baseDir;
|
||||||
|
const previousQueue = ingestion.bufferQueue.slice();
|
||||||
|
ingestion.writer.baseDir = tmpStorageDir;
|
||||||
|
ingestion.bufferQueue.length = 0;
|
||||||
|
ingestion.setRedisClientForTests(mockRedis);
|
||||||
|
const evt: DataEvent = {
|
||||||
|
clientId: "p02_ingestion_replay",
|
||||||
|
eventId: "p02-port4000-001",
|
||||||
|
timestamp: "2026-11-22T13:00:00.000Z",
|
||||||
|
payload: JSON.stringify({ stage: "before-ltrim" }),
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
await mockRedis.rpush(REDIS_BUFFER_KEY, JSON.stringify(evt));
|
||||||
|
await assert.rejects(
|
||||||
|
() => ingestion.flushBuffer(),
|
||||||
|
/P02 Port4000 injected post-CSV trim failure/,
|
||||||
|
);
|
||||||
|
assert.equal(failedOnce, true);
|
||||||
|
assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 1);
|
||||||
|
const csvPath = path.join(tmpStorageDir, evt.clientId, "2026-11-22.csv");
|
||||||
|
assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1);
|
||||||
|
const recovered = await ingestion.flushBuffer();
|
||||||
|
assert.equal(recovered.written, 0, "Recovered replay should not append the event again");
|
||||||
|
assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 0);
|
||||||
|
assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1);
|
||||||
|
} finally {
|
||||||
|
ingestion.writer.baseDir = previousDir;
|
||||||
|
ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...previousQueue);
|
||||||
|
ingestion.setRedisClientForTests(undefined);
|
||||||
|
}
|
||||||
|
});
|
||||||
Reference in new issue
Block a user