diff --git a/core/data/buffer.ts b/core/data/buffer.ts index 661f9e0..9e37623 100644 --- a/core/data/buffer.ts +++ b/core/data/buffer.ts @@ -650,6 +650,66 @@ export class DataBufferEngine { } const executeFlush = async (): Promise => { + + // P02 R-02: examine CSV partitions before replay of an untrimmed Redis + // batch. Streaming parsing limits memory and handles escaped/newline CSV. + const excludeAlreadyPersisted = async (events: DataEvent[]): Promise => { + if (!events.length) return events; + const fs = await import('node:fs'); + const path = await import('node:path'); + const zlib = await import('node:zlib'); + const { pipeline } = await import('node:stream/promises'); + const { Writable } = await import('node:stream'); + const { parse } = await import('csv-parse'); + const groups = new Map; + }>(); + for (const event of events) { + const clientId = String(event.clientId || 'unknown'); + const date = CSVStorageManager.extractSafePartitionDate(event.timestamp); + const key = JSON.stringify([clientId, date]); + if (!groups.has(key)) groups.set(key, { clientId, date, ids: new Set() }); + groups.get(key)!.ids.add(String(event.eventId)); + } + const persisted = new Set(); + for (const [groupKey, group] of groups) { + const dir = CSVStorageManager.ensureClientDir(group.clientId); + const entries = await fs.promises.readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + if (!entry.isFile() || + !/\.csv(?:\.gz)?$/.test(entry.name) || + CSVStorageManager.extractSafePartitionDate( + entry.name.replace(/\.csv(?:\.gz)?$/, '').replace(/(?:-(?:part|gz)\d+)+$/, ''), + ) !== group.date) continue; + const reader = fs.createReadStream(path.join(dir, entry.name)); + const parser = parse({ columns: true, bom: true, skip_empty_lines: true }); + const sink = new Writable({ + objectMode: true, + write(row: { clientId?: string; eventId?: string }, _enc, done) { + if (row.clientId === group.clientId && group.ids.has(String(row.eventId))) { + persisted.add(JSON.stringify([groupKey, String(row.eventId)])); + } + done(); + }, + }); + if (entry.name.endsWith('.gz')) { + await pipeline(reader, zlib.createGunzip(), parser, sink); + } else { + await pipeline(reader, parser, sink); + } + } + } + const batchSeen = new Set(); + return events.filter((event) => { + const key = JSON.stringify([String(event.clientId || 'unknown'), + CSVStorageManager.extractSafePartitionDate(event.timestamp)]); + const rowKey = JSON.stringify([key, String(event.eventId)]); + if (persisted.has(rowKey) || batchSeen.has(rowKey)) return false; + batchSeen.add(rowKey); + return true; + }); + }; + state.isFlushing = true; let totalWritten = 0; let lockTimedOut = false; @@ -793,9 +853,12 @@ export class DataBufferEngine { if (lockLost) break; if (parsedBatch.length > 0) { - const res = await CSVStorageManager.appendBatch(parsedBatch); - totalWritten += res.written; - for (const f of res.filesUpdated) filesSet.add(f); + const unwritten = await excludeAlreadyPersisted(parsedBatch); + if (unwritten.length > 0) { + const res = await CSVStorageManager.appendBatch(unwritten); + totalWritten += res.written; + for (const f of res.filesUpdated) filesSet.add(f); + } } if (Date.now() - lastRenewAt >= renewIntervalMs) { @@ -845,9 +908,12 @@ export class DataBufferEngine { if (validRamBatch.length > 0) { try { - const res = await CSVStorageManager.appendBatch(validRamBatch); - totalWritten += res.written; - for (const f of res.filesUpdated) filesSet.add(f); + const unwritten = await excludeAlreadyPersisted(validRamBatch); + if (unwritten.length > 0) { + const res = await CSVStorageManager.appendBatch(unwritten); + totalWritten += res.written; + for (const f of res.filesUpdated) filesSet.add(f); + } } catch (err) { state.buffer.unshift(...validRamBatch); throw err; diff --git a/notes/001-PRODUCTION-FORENSIC-AUDIT.md b/notes/001-PRODUCTION-FORENSIC-AUDIT.md index 52b75d1..a2ae8f8 100644 --- a/notes/001-PRODUCTION-FORENSIC-AUDIT.md +++ b/notes/001-PRODUCTION-FORENSIC-AUDIT.md @@ -89,3 +89,31 @@ Source freeze for P02: Gitea/GitHub main `667a140c4e9e75382bfdcdbe1d3e9eb96f6e80 - **R-03 health:** `server/index.js:1926-1957` returns HTTP 200 `status:online` when Redis LLEN fails, while surfacing `redisConnected:false`; the historical route is **liveness**, not dependency readiness. `app/api/health/route.ts:95-111` likewise reports web liveness/buffer and does not prove database/Redis write readiness. A new readiness endpoint or altered public response requires separate approval. Do not auto-fix source based solely on tests. Runtime code is READ-ONLY under the current approval. Any proven R-01/R-02 risk requiring behavior change needs a new exact method/file approval before application changes. P03 remains independent and unstarted. + + +## P02 Runtime Remediation Candidate — 2026-11-22 (NOT YET VERIFIED) + +Authorized follow-up baseline: Gitea/GitHub `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898` (GitLab same tree). Local isolated branch `p02-runtime-remediation-20261122`, implementation commit `44eee8877e40d1683f44502c2a0ba3d437a03b88`. Scope: **server/index.js** (`pushEventsToRedisOrFallback`, `flushBuffer`), **server/socket/handlers.js** (sync_data/batch_sync ACK and claim rollback), **core/data/buffer.ts** (`flush`), **tests/phase02-sync.test.ts** and **tests/phase03-storage-buffer.test.ts** only, plus notes/001–006. All other source is unchanged. + +**Candidate R-01 behavior:** when a configured Redis client returns an RPUSH error, `pushEventsToRedisOrFallback` retains RAM fallback but throws a sanitized `BUFFER_PERSIST_FAILED` instead of returning a durable-looking buffer count. Socket single/batch ACK returns `success:false`; best-effort Redis idempotency-key removal and bounded in-process negative-ACK marker handle ordinary retries. This is a **candidate** pending tests. When Redis is intentionally absent, historical RAM fallback is deliberately unchanged; it is not a universal durable-ACK guarantee. If Redis claim deletion fails and the server restarts, stale Redis idempotency claims could still misclassify a retry; a full crash-atomic claim/queue protocol is **not established** by this patch. + +**Candidate R-02 behavior:** Port 4000 `flushBuffer` and Port 3000 `DataBufferEngine.flush` stream-parse existing CSV/CSV.gz partitions for same-client/date/event IDs before replaying any buffered Redis or RAM batch, dropping already-persisted IDs while retaining Redis queue until the write/check completes. Scan/read/parse exceptions fail closed before LTRIM. In-scope test adds a direct Port 4000 writer + Redis LTRIM-failure/recovery test and updates the existing web-buffer replay fixture. **Not proven globally exactly-once:** cross-process CSV writers outside the Redis flush lock and unknown archival/partial-corruption cases remain; scanning partitions on each flush also has real disk-I/O cost, requiring throughput evaluation before production promotion. + +**Candidate acceptance:** one disposable Gitea CI with 24/24 suites, strict TS, lint, Linux/Windows clean npm CI and production build; direct Port4000 replay tests; exact allowlist/diff audit and three-remote tree parity. The Gitea MCP/host was unavailable on the implementation attempt, so this is **NOT PASSED** and the P02 phase remains **BLOCKED / NOT CLOSED**. No live deployment or runtime production mutation. + +## P02-RUNTIME-REMEDIATION-02 — Scope-locked post-patch forensic audit (2026-11-22) + +**Source baseline:** P02 diagnostic `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`. **Tested patch branch:** `p02-runtime-remediation-20261122`, final remediation source snapshot `5c880885378ae0f4fa8dd3a145f9c7055dada635`. This appendix supersedes the earlier "Gitea unavailable" candidate note. **Do not claim P02 CLOSED**: a separate HTTP fallback code path remains in scope-lock-excluded files. + +**R-01 Port 4000 improvements implemented (not live-deployed):** +- `server/index.js:pushEventsToRedisOrFallback` fails closed with sanitized `BUFFER_PERSIST_FAILED` on Redis RPUSH rejection while retaining RAM for recovery; also deduplicates same-client/eventId retries inside a single push. +- The `onBufferPush` integration synchronously flushes to CSV before a successful ACK when Redis is intentionally absent; failed CSV append returns a negative ACK. No-Redis mode is therefore not an automatic RAM-only success ACK. +- `server/socket/handlers.js` single/batch ACK logic clears failed local idempotency claims, never calls an uncertain Redis claim "durable duplicate", verifies a claimed duplicate against a bounded Redis buffer read or read-only CSV/.csv.gz scan, and denies uncertain mixed-batch ACKs. The proof uses a 512-item Redis window and a bounded CSV scan (256 MiB, 1.8 seconds); lack of proof yields a negative ACK, **not proof of data loss**. + +**R-02 Port 4000 + web-buffer improvements implemented:** Both allowed flush methods check same-client/partition/eventId in physically persisted CSV/CSV.gz (including rotation) before retrying after Redis trim failure, and only trim when the disk append/check succeeds. Direct disposable Port 4000 LTRIM failure and gzip-partition replay tests exist. **Not a formal globally atomic/exactly-once guarantee:** concurrent independent writers, data corruption and repeated full-partition scans may introduce performance or recovery limitations. + +**Critical residual R-01 HTTP path (outside this approval):** `core/data/buffer.ts:412-442` silently falls back to RAM on Redis RPUSH error; `app/api/events/route.ts:744-754` returns HTTP success after `pushAsync`. `sdk/node/datastack/index.ts:643-666,916-942` routes non-auth WebSocket rejections into HTTP fallback. Thus a negative Socket.IO ACK does **not establish end-to-end SDK/HTTP durability**. A disposable HTTP fault diagnostic was added to the P02 test allowlist; it is not a runtime fix. Fixing HTTP `POST` and the buffer push/idempotency contract needs separate explicit approval, including `core/Security/security.ts:230-375` only if reproduced claim rollback demands it. + +**R-03:** public `/health` remains HTTP 200 liveness; no readiness endpoint, UI or auth change. + +**CI facts:** Gitea [#66](https://dev.ivib.cc/vibtools/DataStack/actions/runs/66) terminal SUCCESS for initial patch; [#68](https://dev.ivib.cc/vibtools/DataStack/actions/runs/68) diagnosed a same-ID retry assertion regression; the exact local idempotency-claim cleanup was fixed in scope. [#70](https://dev.ivib.cc/vibtools/DataStack/actions/runs/70) is the latest disposable source acceptance (check terminal outcome in notes/003). **No code was deployed to production.** diff --git a/notes/002-PRODUCTION-UPDATE-ROADMAP.md b/notes/002-PRODUCTION-UPDATE-ROADMAP.md index 7b6df8d..9e2fdb9 100644 --- a/notes/002-PRODUCTION-UPDATE-ROADMAP.md +++ b/notes/002-PRODUCTION-UPDATE-ROADMAP.md @@ -81,3 +81,21 @@ P02 approved as **diagnostic-only**. Published P01 baseline: `667a140c4e9e75382b `APPROVED — DataStack P02-RUNTIME-REMEDIATION. Baseline: use the verified P02 diagnostic commit/remote tree. Fix reproduced R-01 and R-02 only, following exact notes/001–006 proof and scoped server/index.js pushEventsToRedisOrFallback/onBufferPush and flushBuffer, server/socket/handlers.js sync_data/batch_sync ACK, and their three existing P02 test files. First document safe ACK/retry semantics, exactly-once versus at-least-once CSV contract and crash recovery. Add disposable fault tests proving no positive ACK for non-durable data and no unbounded duplicate replay. Runtime sources core/data/buffer.ts and server/workers/csvWriter.js remain READ-ONLY unless a separately justified exact file/line amendment is approved. Preserve all UI/auth/SDK/public health behavior and existing error codes, production data, security, other features and Docker. One focused CI, diff/security audit, notes update, commit and 3-remote parity. No deployment, migrations, tags/releases or scope expansion. If a safe fix cannot be achieved within these exact methods, stop and report the required additional Scope Lock rather than broadening automatically.` After separate approved remediation and verified P02 closure, **P03 plan** remains CI-06 Linux Python runner provisioning, CI-07 isolated Docker/BuildKit failure remediation and 3-image build, CI-08 supply-chain advisory triage, and non-mutating readiness evidence. P03 should not modify unrelated UI/runtime/API code by default. + + +## P02 Runtime Remediation Execution Handoff — 2026-11-22 + +The user explicitly approved **only** R-01 Redis failure ACK and R-02 Redis-to-CSV replay corrections in these methods: `server/index.js` `pushEventsToRedisOrFallback` / `flushBuffer` / existing `onBufferPush` integration; `server/socket/handlers.js` `sync_data` / `batch_sync` ACK handling; `core/data/buffer.ts` `flush` if needed; the three earlier P02 test files; `notes/001–006`. No other application method/UI/API/auth/SDK/Docker/CI-workflow edit. Public R-03 `/health` semantics remain unchanged. + +Local candidate `44eee8877e40d1683f44502c2a0ba3d437a03b88` is **not release-ready** until Gitea acceptance. Before closure, prove: negative ACK on Redis failure with no sensitive error leakage; ordinary successful Redis ACK preserved; both single and batch retries don't convert volatile failures to success; Redis queued event replays after crash without writing duplicate CSV rows in the Port 4000 worker and Port 3000 web buffer; CSV read/parse failure never causes premature Redis trim; validate bounded scan overhead and edge cases for rotated/gzipped partitions; all historical 401/403/429/507/503 tests unchanged. + +**Stop rule:** if required fixes need touching `validateSingleEventAsync`, `csvWriter.js` or separate journal/storage scheme, do not broaden the implementation scope; record the residual and request new explicit approval. Existing unrelated files including `MEMORY.md` remain unchanged due exact Scope Lock; this note is the cross-IDE handoff. P03 must wait for P02 CLOSED, not merely implementation committed. + +## P02 Post-implementation approval boundary — 2026-11-22 + +The separately approved R-01/R-02 **Socket.IO and CSV flush** hardening is implemented on an isolated branch, with all allowed runtime sources/tests only. Existing candidate approval text above is historic; the newest evidence is the appendix in notes/001 and notes/003. P02 must **NOT** be closed solely on P02 branch CI success: the HTTP `POST /api/events` RAM-fallback-success path is source-observed outside the current runtime method allowlist. P03 remains NOT STARTED until P02 acceptance and a new explicit approval. + +**Proposed separate Scope Lock approval — P02-HTTP-DURABILITY-01 (NOT YET APPROVED):** +`APPROVED — DataStack P02-HTTP-DURABILITY-01. Use latest verified P02 runtime branch and notes/001–006. Default WRITE only app/api/events/route.ts POST; core/data/buffer.ts writeEventsToRedisOrFallback, pushAsync and pushBatchAsync; tests/phase02-sync.test.ts, tests/phase03-storage-buffer.test.ts, tests/r4-phase02-ingestion-audit-streaming.test.ts; notes/001–006. Allow core/Security/security.ts validateEventAsync/idempotency rollback ONLY if a focused failing test proves stale claims cannot be handled within listed methods, and record exact lines before any edit. Fix HTTP success on volatile RAM after Redis write failure; preserve legitimate Redis-free CSV durable fallback, cross-transport duplicate correctness, current 401/403/429/507 contracts, SDK wire format, auth and user data. Use no production Redis/DB, no live requests, no UI, SDK, Docker, migrations, releases, R-03 health change or unrelated files. One focused disposable Gitea CI, exact patch/security audit, notes update, one Git commit, three-remote content parity. Then close P02 only if the entire acceptance matrix is demonstrated; provide P03 approval command.` + +**P03 outline once P02 is truly CLOSED:** CI-06 restore reliable Python CI runner provisioning; CI-07 diagnose Docker/containerd Builder snapshot error and build all three images in disposable CI; CI-08 trace exact npm advisories and apply only proven compatible security upgrades. Preserve UI, auth, backend feature semantics and live data. No automatic deployment. diff --git a/notes/003-PHASE-STATUS-LOG.md b/notes/003-PHASE-STATUS-LOG.md index 32a61ca..f6dbec7 100644 --- a/notes/003-PHASE-STATUS-LOG.md +++ b/notes/003-PHASE-STATUS-LOG.md @@ -91,3 +91,29 @@ Historical baseline code stayed frozen through these CI workflow changes. Curren - **BLOCKING RELEASE FINDINGS:** R-01 success ACK issued for non-durable RAM fallback when Redis RPUSH fails; R-02 identical eventId can append twice on post-CSV Redis-trim failure (web-side representative test). R-03 health remains online on Redis loss; liveness-only behavior is backward-compatible, protected dependency-readiness verification remains missing. - **P02 PHASE = BLOCKED**, diagnostic implementation done, release reliability acceptance NOT satisfied. Runtime remediation requires separate exact approval. 1/3 phases still CLOSED (P01 only); P02 and P03 remain unclosed. No P03 start authorization from this diagnostic completion. - Next approval must explicitly authorize exact narrow server/core patch for ACK durability and CSV replay deduplication, plus concrete no-loss/no-duplicate tests. Any readiness endpoint change needs separate explicit approval. User's initial P02 approval is insufficient for runtime mutation. + + +## P02-RUNTIME-REMEDIATION — Candidate implementation checkpoint (2026-11-22) + +- **Current P02: BLOCKED / NOT CLOSED.** One previous diagnostic Gitea CI #63 reproduced R-01/R-02/R-03; runtime remediation was separately approved after that. Verified completed implementation phases remain **1/3** (P01 only). Do not approve P03 yet. +- Published P02 diagnosis baseline `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`; isolated local candidate commit `44eee8877e40d1683f44502c2a0ba3d437a03b88` on `p02-runtime-remediation-20261122`. +- Exactly five code/test files changed in this candidate: `server/index.js`, `server/socket/handlers.js`, `core/data/buffer.ts`, `tests/phase02-sync.test.ts`, `tests/phase03-storage-buffer.test.ts`. No `server/workers/csvWriter.js`, `app/**`, `components/**`, SDK, public R-03 health, CI workflow, database, migration, production data, secrets or deployment touched. +- JS `node --check` on two changed JS files PASS; `git diff --check` PASS and allowlist audit PASS. Targeted TS/Node regression and Gitea CI **NOT YET RUN TO TERMINAL PASS**. The Gitea connector/host was unavailable when dispatch/push was attempted. Local temporary dependency installation also not yet complete. +- The original user PC root `D:\VibTools_Workspace\01_Products\14_DataStack` still has 16 pre-existing staged P01 paths on older `d8ac96e`; deliberately preserved without reset/stash/discard. All P02 implementation is in a separate temporary worktree. Do not assume the old local root is up to date. +- R-01 and R-02 implementations are candidates; practical reliability limitations and I/O cost recorded in `notes/001` and `notes/004`. No false claim of globally exactly-once CSV or universally durable ACK. +- **Next gates:** restore Gitea access; run one focused disposable acceptance; fix any in-scope test/TS defects; record exact CI terminal results; only after required acceptance and security audit, squash single phase main commit and verify GitHub/GitLab/Gitea tree parity; then, and only then, decide P02 CLOSED versus residual-risk blocker. + + +### Verification environment blocker — local dependency install + +The authorized Windows PC attempted a **local disposable** `npm.cmd ci --ignore-scripts --no-audit --no-fund --prefer-offline` in the isolated P02 worktree only; it failed with `npm ERR! ETARGET No matching version found for yaml@2.9.1` plus cleanup EPERM warnings in the temp worktree's `node_modules`. Exact output captured in `%TEMP%\DataStack-P02-npm-ci.log` on the authorized PC. This is **not a passed code test nor proof of a repository package defect**, and `package-lock.json` / `package.json` cannot be edited under P02 scope. Gitea MCP remained UNAVAILABLE and the Git remote/HTTP service was not responding when CI was attempted, so the mandatory P02 focused CI was **not dispatched**. Phase remains BLOCKED pending actual CI evidence; no runtime change was pushed to production/main. + +## P02-RUNTIME-REMEDIATION-02 — Post-patch evidence/status (2026-11-22) + +- **P02 current status: BLOCKED / NOT CLOSED** by a separate Port 3000 HTTP RAM-ACK path (and until terminal CI evidence). Completed implementation phases remain **1/3** (P01 only); P03 not authorized. +- Baseline `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`; isolated Gitea branch `p02-runtime-remediation-20261122`; inspected code commit `5c880885378ae0f4fa8dd3a145f9c7055dada635`. Original PC root `main` at earlier docs SHA with 16 user-staged P01 paths remains untouched. No force-reset, stash, production write or deployment. +- **Verified earlier focused CI #66:** 24/24 TS suites, strict TypeScript, ESLint, Windows/npm SDK and Next.js standalone production build success; Python/Docker jobs skipped as separate P03 work. +- **CI #68:** new same-ID retry test failed because an unproven duplicate had left a local claim set; this was corrected in handlers on the subsequent scoped code commit. #68 was cancelled after failure (logs preserved). **CI #70:** revised focused acceptance and added HTTP failure diagnostic; confirm actual terminal job conclusions before claiming PASS. The HTTP diagnostic is intentionally expected to expose an unresolved behavior, not fix it. +- P02 code changes confined to `server/index.js` selected ACK/buffer/flush methods, `server/socket/handlers.js` persistence ACK/duplicate proof methods, `core/data/buffer.ts` flush, and three approved test files; notes/001–006 only. R-03 public health unchanged. All other app/UI/auth/SDK/CI workflow/lockfile/Docker/runtime source unchanged. +- **Residual blocking source evidence:** `app/api/events/route.ts:744-754` returns success following `core/data/buffer.ts:412-442` silent Redis-error RAM fallback; Node SDK may switch to HTTP after negative WS ACK. A separate exact scope exception is required for end-to-end no-positive-ACK-on-volatile-data acceptance. Do not merge the blocked candidate into three `main` refs or assert P02 production ready. +- Performance boundary: replay checking can stream entire CSV partitions per flush (not throughput benchmarked); duplicate proof is capped at last 512 Redis queue items and 256 MiB/1.8s CSV proof (negative ACK on uncertainty). This is safer than a false positive, but compatibility under heavy ingestion must be reviewed before release. diff --git a/notes/004-ERROR-HANDLING-REGISTER.md b/notes/004-ERROR-HANDLING-REGISTER.md index 6ea9755..1af5be2 100644 --- a/notes/004-ERROR-HANDLING-REGISTER.md +++ b/notes/004-ERROR-HANDLING-REGISTER.md @@ -64,3 +64,29 @@ The previous #58 package-lock error was resolved by adding the exact nested opti | R-03 availability | Redis `llen` rejection leaves Express `/health` online with `redisConnected:false`. Recovery flips the flag while HTTP 200 remains unchanged. | Public 200 is liveness by existing contract; a protected dependency readiness probe/503 is **not implemented** and needs explicit approval. | Security invariants: do not expose secrets in ACK/errors, keep JWT/auth and 401/403/429/503/507 contracts unchanged, and never run failure-injection against actual Redis, Postgres, production API, live events, or user CSV. Disposable fake Redis and temporary files only. CI #63: https://dev.ivib.cc/vibtools/DataStack/actions/runs/63; results tracked in notes/003. + + +## P02 Runtime Candidate — Error-path change inventory (pending CI) + +| Path | Candidate result | Explicit limitation | +|---|---|---| +| Redis `RPUSH` rejects with configured Redis client | Retains event in volatile RAM as preexisting fallback but returns `BUFFER_PERSIST_FAILED` to Socket.IO via thrown error; **no successful ACK** for that failure; error text does not expose connection details | No-Redis intentional legacy RAM mode unchanged, still not a durable guarantee | +| `sync_data` and `batch_sync` failures | `success:false`, `code:BUFFER_PERSIST_FAILED` and best-effort rollback of previously claimed idempotency keys, negative failure marker for in-process retries | If claim rollback cannot reach Redis and process crashes, durable claim/queue atomicity remains unsolved | +| Redis buffer `LTRIM` rejects after successful CSV append | Subsequent flush checks physically stored CSV IDs and avoids appending matching rows again; Redis is trimmed only after successful check/write | Concurrent independent CSV appenders, compressed archived CSV, corrupt partial rows, and per-flush CSV scan cost require separate proof | +| Legacy public health | **UNCHANGED** HTTP 200 liveness contract and Redis-connected diagnostic field; R-03 readiness is not introduced in this approval | Does not prove full DB/Redis transactional readiness | + +Security/API invariants: leave token/JWT/auth/rate limits/quotas/webhooks/MCP/sandbox/UI/SDK unchanged. No production Redis/Postgres/API/CSV mutation. No lockfile/dependency versions or CI workflow changes. CI acceptance remains pending; these are code-level observations, not release certification. + +## P02-RUNTIME-REMEDIATION-02 — Updated safety contract evidence + +| Scenario | In-scope behavior | Residual | +|---|---|---| +| Port 4000 Redis `RPUSH` failure | `BUFFER_PERSIST_FAILED`, `success:false`; no Redis transport diagnostics disclosed in client ACK | SDK can retry via HTTP; Port 3000 volatile RAM success path remains unpatched | +| Port 4000 without Redis | Await actual CSV flush before positive ACK; disk error yields negative ACK and queue retained | Disk API does not provide fsync-level power-loss guarantees | +| Stale idempotency after restart | Never treat a claim alone as persisted; only accept a duplicate after in-memory acknowledged proof, Redis queued ID or read-only CSV partition proof | Bounded scans may cause temporary negative ACK for otherwise durable old events, rather than unsafe success | +| Mixed `batch_sync` with uncertain claim | Overall negative `BUFFER_PERSIST_FAILED`; newer items may already be persisted, repeat dedupe resolves them | Preserve existing response fields and rate/quota/auth validations | +| Redis `LTRIM` fails after CSV write | On retry, scan CSV/CSV.gz by client/date/event ID and skip already written event; trim Redis only after safe check | Cross-process exact-once not proven; I/O cost needs throughput validation | +| Port 3000 HTTP `pushAsync` / `pushBatchAsync` | **UNCHANGED** (excluded from current approval); Redis error can return HTTP success with data only in volatile RAM | **OPEN R-01 blocker**, requires separate approval; `core/Security/security.ts` precommit idempotency claims may also need scoped rollback | +| Public `/health` | Unchanged 200 liveness, `redisConnected` diagnostic | No dependency readiness probe under P02-R01/R02 authorization | + +Do not silently remap 401/403/429/507 errors; no SDK edits or live fault injection. Tests use mock Redis and temp CSV only. Relevant CI #66/#68/#70 in notes/003. diff --git a/notes/005-ACTUAL-FEATURE-REGISTER.md b/notes/005-ACTUAL-FEATURE-REGISTER.md index c04fb43..5ab2717 100644 --- a/notes/005-ACTUAL-FEATURE-REGISTER.md +++ b/notes/005-ACTUAL-FEATURE-REGISTER.md @@ -52,3 +52,19 @@ Do not mark "actual working" simply because a patch compiled. Distinguish source - R-02: existing CSV-before-Redis-trim order audited, *not corrected*. Exactly-once replay safety is **NOT VERIFIED**. - R-03: current public Express health 200 plus `redisConnected:false` is liveness, not dependency readiness. Separate readiness remains **NOT IMPLEMENTED**. - Gitea disposable CI #63: https://dev.ivib.cc/vibtools/DataStack/actions/runs/63. The previous P01 working results remain valid; no live read/write test or production deploy is included in P02. + + +## P02 Runtime Candidate Feature Delta — 2026-11-22 (NOT RELEASED) + +- **Intended change:** failure cases on socket ingestion report `BUFFER_PERSIST_FAILED` instead of positive ACK when Redis RPUSH rejects; single and batch code paths tested by new disposable assertions. Ordinary success ACK, auth, limits and SDK public interfaces intended unchanged. +- **Intended change:** Redis replay flush on both ingestion/Next paths inspects existing client CSV partitions and filters already-stored event IDs before reappend; safe failure leaves Redis queue untrimmed. This prevents the tested post-append/LTRIM replay duplication but does **not** prove transactionally exactly-once delivery or zero performance impact. +- **No changes:** application UI/UX, auth, SDK, database schema, Docker, environment, public R-03 health, other features and production runtime. No deployment or live verification. +- **Status:** P02 code candidate locally committed, acceptance CI blocked by Gitea unavailability; **NOT CLOSED**, not production-ready. P01 remains only completed phase. + +## P02 Remediation Hardening Register — 2026-11-22 + +- **Changed in source, not deployed:** Port 4000 Socket.IO error ACK behavior on Redis write failure; no-Redis mode now waits for CSV flush before success; duplicate ACK checks actual Redis/CSV durability; mixed batch uncertainty gives negative ACK. EventId retry handling is made fail-closed with in-scope handler logic. +- **Changed in source, not deployed:** Port 4000 and web-buffer CSV replay after Redis LTRIM failure now checks persisted CSV/CSV.gz event IDs. Existing file rotation/compression and disk errors need focused tests; scanner I/O impacts high-volume performance and does not certify global exactly-once semantics. +- **Unchanged:** UI/UX/pages, public R-03 health shape/status, auth/JWT/SDK implementation, database/Prisma, Docker/Compose, environment, unrelated features, production data, live server and deployment. +- **Still NOT working as end-to-end guarantee:** HTTP `POST /api/events` currently may send success after Redis RPUSH fails while only RAM contains the event. This is an additional source-observed release blocker, not something fixed by WS error ACK. **P02 NOT CLOSED; P03 NOT STARTED.** +- **Evidence:** Gitea #66 initial remediation PASS; #68 exposed same-ID retry bug fixed in current branch; #70 final focused run awaiting terminal confirmation. The original PC main workspace's 16 staged P01 paths remain untouched. diff --git a/notes/006-BASELINE-AND-PATCH-EVIDENCE.md b/notes/006-BASELINE-AND-PATCH-EVIDENCE.md index f8470de..636c342 100644 --- a/notes/006-BASELINE-AND-PATCH-EVIDENCE.md +++ b/notes/006-BASELINE-AND-PATCH-EVIDENCE.md @@ -237,3 +237,37 @@ return bufferQueue.length; - `P02 R-03 diagnostic: Redis query failure leaves public ingestion health 200/online` — **ok 16**, same suite exit 0, Redis flag false while liveness stays online. - `P02 R-02 diagnostic: post-CSV Redis trim failure replays identical event on retry` — **ok 18**, `phase03-storage-buffer` suite exit 0; duplicate append after retry reproduced on representative web buffer path. **Direct ingestion Port 4000 CSV worker crash coverage remains outstanding** and should be requested in remediation approval before guarantee changes. - These three diagnostic tests **prove findings, do not close production-readiness**. See Gitea https://dev.ivib.cc/vibtools/DataStack/actions/runs/63; never call run successful until terminal conclusion is confirmed. No server-side source fix applied. + + +## F. P02 Remediation Candidate Patch Proof — 2026-11-22 + +Base: Gitea/GitHub published P02 diagnostic tree `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`; GitLab distinct commit, same previous tree. Local candidate commit `44eee8877e40d1683f44502c2a0ba3d437a03b88` on isolated worktree. Initial exact delta: 5 approved source/test paths, 310 insertions and 22 deletions, no application UI, Python SDK, auth, public health route, CI or deployment edit. `git diff --check` and `node --check` on JS files PASS. + +- `server/index.js:pushEventsToRedisOrFallback`: catch RPUSH failure, keep previous RAM buffering, throw sanitized `BUFFER_PERSIST_FAILED`; no positive ACK for that branch. +- `server/socket/handlers.js:sync_data/batch_sync`: negative ACK and best-effort claimed idempotency-key rollback with negative retry guard, no change to legitimate duplicate success path or existing auth/limits/error codes. +- `server/index.js:flushBuffer` and `core/data/buffer.ts:flush`: scoped streaming CSV/CSV.gz read-before-reappend (parsing quoted/multiline rows), exact client/date/event ID match; failures abort before Redis LTRIM. No changes to `server/workers/csvWriter.js`. +- `tests/phase02-sync.test.ts`: R-01 single/batch outage and duplicate claim ACK negative, successful recovery ACK, and unchanged R-03 liveness. +- `tests/phase03-storage-buffer.test.ts`: web-side replay verifies exactly one physically stored row; new direct Port 4000 writer Redis-trim failure/recovery proof. All tests on fake Redis and disposable temp CSV paths. + +**Unverified:** Gitea P02 implementation acceptance was unavailable due inaccessible host/connector. Local `npm ci` not confirmed successful. Do not invent a CI run ID or claim full PASS. Full cross-process transactional exactly-once and all Redis claim crash windows are out of proof scope and may require exact additional approval. The streaming dedupe scan changes I/O cost per flush; performance validation before production is required. + + +### Blocked acceptance evidence — actual tool outcomes + +- JS syntax: local `node --check server/index.js` and `node --check server/socket/handlers.js` PASS. +- Source scope: local `git diff --check` PASS; only the five allowed P02 runtime/test paths changed before notes updates. +- Gitea: connection returned `UNAVAILABLE: McpServerError: Connection failed`; Git remote `dev.ivib.cc` unresponsive. No Gitea CI run ID exists for candidate `44eee8877e40d1683f44502c2a0ba3d437a03b88`. +- Local Windows npm: `npm ci` failed `ETARGET yaml@2.9.1` and cleanup `EPERM` in the isolated worktree, **not** a TypeScript/test result. Dependency changes are out of P02 scope. +- Therefore neither TS typecheck nor regression-suite remediation acceptance is established; keep P02 BLOCKED and do not advance P03 or alter any protected runtime/source beyond this approval. + +## F. P02-RUNTIME-REMEDIATION-02 — Exact edge-case proof and scope audit (2026-11-22) + +**Baseline** `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`; isolated branch `p02-runtime-remediation-20261122`; patch revision `5c880885378ae0f4fa8dd3a145f9c7055dada635`. Scope: `server/index.js` (RPUSH fallback/onBufferPush/flush), `server/socket/handlers.js` (single/batch ACK + durability verification helper), `core/data/buffer.ts` (flush), `tests/{phase02-sync,phase03-storage-buffer,r4-phase02-ingestion-audit-streaming}.test.ts` and `notes/001–006` only. No other source file/CI workflow/dependency/SDK/UI change. + +**Runtime contract and tests:** configured Redis RPUSH rejection -> negative `BUFFER_PERSIST_FAILED`; Redis-free Socket.IO ACK -> synchronous local CSV flush, with negative ACK on disk error; fail closed on previous Redis idempotency claim unless persisted to Redis buffer or matching physical CSV; recover same eventId after Redis reconnection; mixed batch uncertain duplicate remains negative until persistence proven. R-02 test injects failure after Port 4000 CSV append and before Redis trim, verifies subsequent retry does not add another row, including moved/rotated `.csv.gz`. R-03 `/health` remains unchanged. + +**Test-only HTTP residual diagnostic:** `tests/phase02-sync.test.ts` adds Redis RPUSH failure against `POST /api/events` on a fake Redis. Expected prior behavior: HTTP 200 `success:true` while `DataBufferEngine.getLocalBufferSnapshot()` has the event and Redis buffered list is empty, then simulated RAM loss. This *diagnostic PASS* would confirm a remaining risk, not a completed fix. Source: `app/api/events/route.ts:744-754`, `core/data/buffer.ts:412-442,481-491`, `core/Security/security.ts:230-375`, Node SDK `sdk/node/datastack/index.ts:643-666,916-942`. Those runtime methods are **READ ONLY under current approval**. + +**CI ledger:** [#66](https://dev.ivib.cc/vibtools/DataStack/actions/runs/66) initial in-scope implementation overall SUCCESS; [#68](https://dev.ivib.cc/vibtools/DataStack/actions/runs/68) discovered same-ID retry local claim leak, corrected in subsequent branch commit; [#70](https://dev.ivib.cc/vibtools/DataStack/actions/runs/70) revised source/test acceptance queued/running as of this addendum. Do not assert it PASS without terminal proof. + +**Security/limitations:** never expose Redis backend errors or credentials in ACKs; preserve auth, quotas, rate limits and SDK response fields. Duplicate proof reads no more than 512 recent Redis buffered records and scans up to 256 MiB of CSV partition data with an abort deadline of 1800 ms, refusing uncertain duplicate-success. R-02 streaming CSV scan per flush has unmeasured throughput cost; no globally atomic multi-process exactly-once claim. Unmodified production data, no migrations, no live events, no releases/deploys; original local root staged changes untouched. P02 remains BLOCKED until a separate narrow HTTP durability approval or an equivalent tested safe contract. diff --git a/server/index.js b/server/index.js index f0c60f4..6f8b7df 100644 --- a/server/index.js +++ b/server/index.js @@ -164,7 +164,18 @@ function setRedisClientForTests(client) { async function pushEventsToRedisOrFallback(events) { if (!events || events.length === 0) return bufferQueue.length; + // Failed persistence may leave an unacknowledged RAM event beside an SDK + // retry with the same ID. Persist each client/event ID at most once per push. + const seen = new Set(); + events = events.filter((evt) => { + if (!evt || !evt.clientId || !evt.eventId) return true; + const key = JSON.stringify([evt.clientId, evt.eventId]); + if (seen.has(key)) return false; + seen.add(key); + return true; + }); const redis = getRedisClient(); + let persistFailed = false; if (redis) { const serialized = events.map((e) => JSON.stringify(e)); let bufferLen = null; @@ -173,6 +184,7 @@ async function pushEventsToRedisOrFallback(events) { redisConnected = true; } catch { redisConnected = false; + persistFailed = true; } if (bufferLen !== null) { @@ -189,6 +201,11 @@ async function pushEventsToRedisOrFallback(events) { } bufferQueue.push(...events); + if (persistFailed) { + // Keep the volatile fallback for later recovery, but never ACK it as durable. + // Suppress sensitive Redis transport details in the client-facing error. + throw new Error('BUFFER_PERSIST_FAILED: Redis unavailable; event not durably queued'); + } return bufferQueue.length; } @@ -834,6 +851,61 @@ async function isClientDeletedForFlush(clientId, redis, cache) { } async function flushBuffer(options = {}) { + + // P02 R-02: Redis and CSV cannot be committed atomically. Before replaying an + // untrimmed Redis batch, check actual CSV partitions for previously written IDs. + // Streaming parsing bounds memory and preserves multiline/quoted CSV fields. + const excludeAlreadyPersisted = async (events) => { + if (!events.length) return events; + const fs = require('node:fs'); + const path = require('node:path'); + const zlib = require('node:zlib'); + const { pipeline } = require('node:stream/promises'); + const { Writable } = require('node:stream'); + const { parse } = require('csv-parse'); + const groups = new Map(); + for (const event of events) { + const clientId = String(event.clientId || 'unknown'); + const date = writer.extractSafePartitionDate(event.timestamp); + const key = JSON.stringify([clientId, date]); + if (!groups.has(key)) groups.set(key, { clientId, date, ids: new Set() }); + groups.get(key).ids.add(String(event.eventId)); + } + const persisted = new Set(); + for (const [groupKey, group] of groups) { + const dir = writer.ensureClientDir(group.clientId); + const entries = await fs.promises.readdir(dir, { withFileTypes: true }); + for (const entry of entries) { + if (!entry.isFile() || !writer.isPartitionFilename(entry.name) || + writer.extractDateFromFilename(entry.name) !== group.date) continue; + const reader = fs.createReadStream(path.join(dir, entry.name)); + const streams = [reader]; + if (entry.name.endsWith('.gz')) streams.push(zlib.createGunzip()); + streams.push(parse({ columns: true, bom: true, skip_empty_lines: true })); + streams.push(new Writable({ + objectMode: true, + write(row, _enc, done) { + if (row.clientId === group.clientId && group.ids.has(String(row.eventId))) { + persisted.add(JSON.stringify([groupKey, String(row.eventId)])); + } + done(); + }, + })); + // On read/parse failure, abort without LTRIM: never discard uncertain rows. + await pipeline(...streams); + } + } + const batchSeen = new Set(); + return events.filter((event) => { + const key = JSON.stringify([String(event.clientId || 'unknown'), + writer.extractSafePartitionDate(event.timestamp)]); + const rowKey = JSON.stringify([key, String(event.eventId)]); + if (persisted.has(rowKey) || batchSeen.has(rowKey)) return false; + batchSeen.add(rowKey); + return true; + }); + }; + if (pendingRedisWrites.size > 0) { await Promise.allSettled([...pendingRedisWrites]); } @@ -979,9 +1051,12 @@ async function flushBuffer(options = {}) { if (lockLost) break; if (parsedBatch.length > 0) { - const res = await writer.writeBatch(parsedBatch); - totalWritten += res.written; - for (const f of res.files) filesSet.add(f); + const unwritten = await excludeAlreadyPersisted(parsedBatch); + if (unwritten.length > 0) { + const res = await writer.writeBatch(unwritten); + totalWritten += res.written; + for (const f of res.files) filesSet.add(f); + } } if (Date.now() - lastRenewAt >= renewIntervalMs) { @@ -1022,9 +1097,12 @@ async function flushBuffer(options = {}) { if (validBatch.length > 0) { try { - const res = await writer.writeBatch(validBatch); - totalWritten += res.written; - for (const f of res.files) filesSet.add(f); + const unwritten = await excludeAlreadyPersisted(validBatch); + if (unwritten.length > 0) { + const res = await writer.writeBatch(unwritten); + totalWritten += res.written; + for (const f of res.files) filesSet.add(f); + } } catch (err) { bufferQueue.unshift(...validBatch); throw err; @@ -1917,8 +1995,14 @@ const socketHandlersApi = setupSocketHandlers(io, bufferQueue, writer, { return; } - if (bufferQueue.length >= runtimeConfig.batchFlushThreshold) { - void flushBuffer().catch(() => undefined); + // Redis is optional. Without it, a positive Socket.IO ACK requires a + // completed CSV write, not merely an event sitting in volatile RAM. + // Keep existing RAM retry state if the disk write fails. + if (bufferQueue.length > 0) { + const saved = await flushBuffer(); + if (saved.lockTimedOut || bufferQueue.length > 0) { + throw new Error('BUFFER_PERSIST_FAILED: Event not durably queued'); + } } }, }); diff --git a/server/socket/handlers.js b/server/socket/handlers.js index 917088c..5edf8d7 100644 --- a/server/socket/handlers.js +++ b/server/socket/handlers.js @@ -478,6 +478,84 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { return hasNonEnvelopeKey ? stripped : {}; } + // P02 R-01: an idempotency claim is created before persistence. After a + // crash that claim may survive without its event. Prove the event reached + // Redis's buffer or a CSV partition before issuing a duplicate-success ACK. + async function hasDurableEvent(clientId, eventId, timestamp) { + const ackKey = `\u0000durably-acked:${clientId}:${eventId}`; + const previousAck = idempotencyMap.get(ackKey); + if (typeof previousAck === 'number' && Date.now() - previousAck < IDEMPOTENCY_TTL_MS) { + return true; + } + const redis = getRedisClient(); + if (redis && typeof redis.lrange === 'function') { + try { + // Bounded read: reject uncertain old claims instead of loading a huge Redis queue. + const queued = await redis.lrange('datastack:buffer:events', -512, -1); + if (Array.isArray(queued) && queued.some((raw) => { + try { + const evt = JSON.parse(raw); + return evt && evt.clientId === clientId && evt.eventId === eventId; + } catch { return false; } + })) { + idempotencyMap.set(ackKey, Date.now()); + return true; + } + } catch { + // Redis cannot prove durability; fall through to read-only CSV proof. + } + } + if (!worker || typeof worker.sanitizeClientId !== 'function' || + typeof worker.extractSafePartitionDate !== 'function' || + typeof worker.isPartitionFilename !== 'function' || + typeof worker.extractDateFromFilename !== 'function') return false; + const fs = require('node:fs'); + const path = require('node:path'); + const { pipeline } = require('node:stream/promises'); + const { Writable } = require('node:stream'); + const { parse } = require('csv-parse'); + const zlib = require('node:zlib'); + const aborter = new AbortController(); + const deadline = setTimeout(() => aborter.abort(), 1800); + deadline.unref?.(); + try { + const base = await fs.promises.realpath(worker.baseDir); + const directory = path.resolve(base, worker.sanitizeClientId(clientId)); + const realDir = await fs.promises.realpath(directory); + if (!realDir.startsWith(base + path.sep)) return false; + const date = worker.extractSafePartitionDate(timestamp); + const files = await fs.promises.readdir(realDir, { withFileTypes: true }); + let scannedBytes = 0; + for (const item of files) { + if (!item.isFile() || !worker.isPartitionFilename(item.name) || + worker.extractDateFromFilename(item.name) !== date) continue; + const filename = path.join(realDir, item.name); + scannedBytes += (await fs.promises.stat(filename)).size; + if (scannedBytes > 256 * 1024 * 1024) return false; + let found = false; + const stages = [fs.createReadStream(filename)]; + if (item.name.endsWith('.gz')) stages.push(zlib.createGunzip()); + stages.push(parse({ columns: true, bom: true, skip_empty_lines: true })); + stages.push(new Writable({ + objectMode: true, + write(row, _encoding, done) { + if (row.clientId === clientId && row.eventId === eventId) found = true; + done(); + }, + })); + await pipeline(...stages, { signal: aborter.signal }); + if (found) { + idempotencyMap.set(ackKey, Date.now()); + return true; + } + } + } catch { + // Missing, corrupt, unreadable or slow CSV cannot justify positive ACK. + } finally { + clearTimeout(deadline); + } + return false; + } io.on("connection", (socket) => { const clientId = socket.data.clientId; socket.join(`client:${clientId}`); @@ -549,6 +627,8 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { : new Date().toISOString(); const eventData = extractSingleEventData(payload); + // A previous Redis write failure can leave an uncommitted idempotency + // claim. Never report that claim as a successful durable duplicate. const check = await validateSingleEventAsync( clientId, eventId, @@ -557,8 +637,17 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { ); if (!check.allowed) { if (check.duplicate) { - if (callback) { - callback({ success: true, eventId, note: "DUPLICATE_DROPPED" }); + if (await hasDurableEvent(clientId, eventId, timestamp)) { + idempotencyMap.delete(`\u0000failed-persist:${clientId}:${eventId}`); + if (callback) callback({ success: true, eventId, note: "DUPLICATE_DROPPED" }); + } else { + // A stale local claim must not block a valid same-ID SDK retry + // after Redis recovers; the Redis claim still guards other workers. + idempotencyMap.delete(`${clientId}:${eventId}`); + if (callback) callback({ + success: false, eventId, code: "BUFFER_PERSIST_FAILED", + error: "BUFFER_PERSIST_FAILED: Idempotency claim has no durable event", + }); } return; } @@ -585,6 +674,20 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { try { await onBufferPush(); } catch (persistErr) { + // Invalidate the pre-persistence idempotency claim so SDK retry can + // resubmit after Redis recovers. Keep the volatile RAM event pending. + const claimKey = `${clientId}:${eventId}`; + idempotencyMap.delete(claimKey); + idempotencyMap.set(`\u0000failed-persist:${claimKey}`, Date.now()); + const claimRedis = getRedisClient(); + if (claimRedis && typeof claimRedis.del === "function") { + try { + await claimRedis.del(`${REDIS_IDEMPOTENCY_PREFIX}${claimKey}`); + idempotencyMap.delete(claimKey); + } catch { + // Claim may remain: negative ACK must be retained for retries. + } + } if (callback) { callback({ success: false, @@ -599,6 +702,9 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { return; } + // Clear any failed local persistence marker only after a durable write. + idempotencyMap.delete(`\u0000failed-persist:${clientId}:${eventId}`); + idempotencyMap.set(`\u0000durably-acked:${clientId}:${eventId}`, Date.now()); if (callback) callback({ success: true, eventId }); } catch (err) { if (callback) callback({ success: false, error: err.message }); @@ -689,6 +795,8 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { const rateLimitedEventIds = []; const oversizedEventIds = []; const errors = []; + const acceptedEventIds = []; + let volatileDuplicate = false; for (const item of batch) { const safeItem = item && typeof item === "object" ? item : {}; @@ -702,7 +810,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { : new Date().toISOString(); const rawData = extractBatchItemData(safeItem); - const check = await validateSingleEventAsync( + const check = await validateSingleEventAsync( clientId, eventId, rawData, @@ -710,7 +818,16 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { ); if (!check.allowed) { if (check.duplicate) { - duplicates += 1; + if (await hasDurableEvent(clientId, eventId, timestamp)) { + idempotencyMap.delete(`\u0000failed-persist:${clientId}:${eventId}`); + duplicates += 1; + } else { + idempotencyMap.delete(`${clientId}:${eventId}`); + rejected += 1; + volatileDuplicate = true; + if (errors.length < 5) errors.push( + `${eventId}: BUFFER_PERSIST_FAILED: Idempotency claim has no durable event`); + } continue; } rejected += 1; @@ -735,6 +852,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { payload: check.payloadStr, }); processed += 1; + acceptedEventIds.push(eventId); } if (processed > 0) { @@ -742,6 +860,20 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { try { await onBufferPush(); } catch (persistErr) { + const claimRedis = getRedisClient(); + for (const eventId of acceptedEventIds) { + const claimKey = `${clientId}:${eventId}`; + idempotencyMap.delete(claimKey); + idempotencyMap.set(`\u0000failed-persist:${claimKey}`, Date.now()); + if (claimRedis && typeof claimRedis.del === "function") { + try { + await claimRedis.del(`${REDIS_IDEMPOTENCY_PREFIX}${claimKey}`); + idempotencyMap.delete(claimKey); + } catch { + // Preserve negative-ACK retry guard if claim removal fails. + } + } + } if (callback) { callback({ success: false, @@ -759,12 +891,29 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { } } + for (const persistedId of acceptedEventIds) { + idempotencyMap.delete(`\u0000failed-persist:${clientId}:${persistedId}`); + idempotencyMap.set(`\u0000durably-acked:${clientId}:${persistedId}`, Date.now()); + } + // Never ACK a mixed batch as successful while any event remains volatile. + if (volatileDuplicate) { + if (callback) callback({ + success: false, processed, rejected, duplicates, + code: "BUFFER_PERSIST_FAILED", + error: "BUFFER_PERSIST_FAILED: Batch contains non-durable event", + errors, + }); + return; + } + if (processed === 0 && duplicates === 0 && rejected > 0) { - const code = hitPayloadLimit - ? "PAYLOAD_TOO_LARGE" - : hitRateLimit - ? "RATE_LIMIT_EXCEEDED" - : "BATCH_REJECTED"; + const code = volatileDuplicate + ? "BUFFER_PERSIST_FAILED" + : hitPayloadLimit + ? "PAYLOAD_TOO_LARGE" + : hitRateLimit + ? "RATE_LIMIT_EXCEEDED" + : "BATCH_REJECTED"; if (callback) { callback({ success: false, diff --git a/tests/phase02-sync.test.ts b/tests/phase02-sync.test.ts index 8d65f9d..9f18709 100644 --- a/tests/phase02-sync.test.ts +++ b/tests/phase02-sync.test.ts @@ -36,6 +36,7 @@ import { const require = createRequire(import.meta.url); interface IngestionServerModule { + writer: { baseDir: string }; server: import("node:http").Server; io: import("socket.io").Server; bufferQueue: Array<{ clientId: string; eventId: string; timestamp: string; payload: string }>; @@ -1221,7 +1222,7 @@ test("R2 PHASE-02 BUG-10: GET, PATCH (regenerate_token), and DELETE /api/clients } }); -test("P02 R-01 diagnostic: Redis RPUSH failure yields positive WS ACK for volatile RAM-only event", async () => { +test("P02 R-01 remediation: Redis RPUSH failure rejects non-durable WS ACK", async () => { // Disposable test-only fixture: no production Redis, service, or storage state. const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold; const previousRam = serverMod.bufferQueue.slice(); @@ -1246,12 +1247,40 @@ test("P02 R-01 diagnostic: Redis RPUSH failure yields positive WS ACK for volati ); const committedRows = await failingRedis.lrange("datastack:buffer:events", 0, -1); const volatileRows = serverMod.bufferQueue.filter((e) => e.eventId === "p02-ack-before-crash-001"); - assert.equal(ack.success, true, "Existing handler currently sends a positive ACK after Redis failure"); + assert.equal(ack.success, false, "P02 remediation must reject non-durable ACK"); + assert.equal(ack.code, "BUFFER_PERSIST_FAILED"); + const sameIdRetry = await emitWithAck<{ success: boolean; code?: string }>( + socket, "sync_data", + { eventId: "p02-ack-before-crash-001", data: { value: 42 } }, + ); + assert.equal(sameIdRetry.success, false, + "Pending idempotency claims must not return false positive duplicate ACK"); + assert.equal(sameIdRetry.code, "BUFFER_PERSIST_FAILED"); + const secondRetry = await emitWithAck<{ success: boolean; code?: string }>( + socket, "sync_data", + { eventId: "p02-ack-before-crash-001", data: { value: 42 } }, + ); + assert.equal(secondRetry.success, false, "A repeated pending claim must remain negative"); + assert.equal(secondRetry.code, "BUFFER_PERSIST_FAILED"); assert.equal(committedRows.length, 0, "Fake Redis has no durable event"); assert.equal(volatileRows.length, 1, "Acknowledged event is RAM-only"); // Crash model: restart loses RAM but fake Redis survives. No actual process is killed. serverMod.bufferQueue.length = 0; assert.equal((await failingRedis.lrange("datastack:buffer:events", 0, -1)).length, 0); + // A new event may ACK only after Redis actually resumes persistence. + const healthyRedis = createSharedMockRedis(); + serverMod.setRedisClientForTests(healthyRedis); + const recovered = await emitWithAck<{ success: boolean; eventId?: string }>( + socket, "sync_data", + { eventId: "p02-ack-recovered-002", data: { value: 43 } }, + ); + assert.equal(recovered.success, true); + const recoveredSameId = await emitWithAck<{ success: boolean; code?: string }>( + socket, "sync_data", + { eventId: "p02-ack-before-crash-001", data: { value: 42 } }, + ); + assert.equal(recoveredSameId.success, true, "Original eventId must be retryable after Redis recovery"); + assert.equal((await healthyRedis.lrange("datastack:buffer:events", 0, -1)).length, 2); } finally { socket?.disconnect(); serverMod.setRedisClientForTests(undefined); @@ -1281,3 +1310,143 @@ test("P02 R-03 diagnostic: Redis query failure leaves public ingestion health 20 serverMod.setRedisClientForTests(undefined); } }); + +test("P02 R-01: batch_sync rejects non-durable Redis writes and pending duplicate claims", async () => { + const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold; + const previousQueue = serverMod.bufferQueue.slice(); + const failingRedis = createSharedMockRedis(); + const recoveredRpush = failingRedis.rpush.bind(failingRedis); + failingRedis.rpush = async () => { throw new Error("P02 injected batch Redis outage"); }; + serverMod.setRedisClientForTests(failingRedis); + serverMod.runtimeConfig.batchFlushThreshold = 100_000; + serverMod.bufferQueue.length = 0; + let socket: Socket | undefined; + const payload = [ + { eventId: "p02-batch-volatile-1", data: { value: 1 } }, + { eventId: "p02-batch-volatile-2", data: { value: 2 } }, + ]; + try { + const client = await SDKManager.registerClientAsync({ + clientId: "p02_redis_batch_failure", + name: "P02 Redis Batch Failure", + rateLimit: 100, + }); + socket = await connectSocket(client.token); + const ack = await emitWithAck<{ success: boolean; code?: string; processed?: number }>( + socket, "batch_sync", payload, + ); + assert.equal(ack.success, false); + assert.equal(ack.code, "BUFFER_PERSIST_FAILED"); + assert.equal(ack.processed, 0); + assert.equal((await failingRedis.lrange("datastack:buffer:events", 0, -1)).length, 0); + const retry = await emitWithAck<{ success: boolean; code?: string }>( + socket, "batch_sync", payload, + ); + assert.equal(retry.success, false); + assert.equal(retry.code, "BUFFER_PERSIST_FAILED"); + failingRedis.rpush = recoveredRpush; + const mixed = await emitWithAck<{ success: boolean; code?: string; processed?: number }>( + socket, "batch_sync", [payload[0], { eventId: "p02-batch-recovered-3", data: { value: 3 } }], + ); + assert.equal(mixed.success, false, "A mixed recovered/volatile batch cannot ACK success"); + assert.equal(mixed.code, "BUFFER_PERSIST_FAILED"); + assert.equal(mixed.processed, 1, "New item is durably queued, old idempotency claim is uncertain"); + const confirmed = await emitWithAck<{ success: boolean; duplicates?: number }>( + socket, "batch_sync", payload, + ); + assert.equal(confirmed.success, true, "Once old events are persisted, duplicates are safely acknowledged"); + assert.equal(confirmed.duplicates, 2); + } finally { + socket?.disconnect(); + serverMod.setRedisClientForTests(undefined); + serverMod.runtimeConfig.batchFlushThreshold = previousThreshold; + serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue); + } +}); + +test("P02 R-01: Redis-free Socket.IO ACK waits for CSV durability and denies disk errors", async () => { + const fs = await import("node:fs"); + const os = await import("node:os"); + const path = await import("node:path"); + const baseDir = fs.mkdtempSync(path.join(os.tmpdir(), "datastack-p02-ack-disk-")); + const previousDir = serverMod.writer.baseDir; + const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold; + const previousQueue = serverMod.bufferQueue.slice(); + serverMod.writer.baseDir = baseDir; + serverMod.runtimeConfig.batchFlushThreshold = 100_000; + serverMod.setRedisClientForTests(null); + serverMod.bufferQueue.length = 0; + let socket: Socket | undefined; + try { + const client = await SDKManager.registerClientAsync({ + clientId: "p02_disk_durability", name: "P02 Disk Durability", rateLimit: 100, + }); + socket = await connectSocket(client.token); + const accepted = await emitWithAck<{ success: boolean; code?: string }>( + socket, "sync_data", { eventId: "p02-disk-ack-001", data: { value: 11 } }, + ); + assert.equal(accepted.success, true); + const csvPath = path.join(baseDir, client.clientId, new Date().toISOString().slice(0, 10) + ".csv"); + assert.equal(fs.readFileSync(csvPath, "utf8").includes("p02-disk-ack-001"), true, + "A successful ACK requires a completed CSV append when Redis is absent"); + const blockedBase = path.join(baseDir, "not-a-directory"); + fs.writeFileSync(blockedBase, "fixture"); + serverMod.writer.baseDir = blockedBase; + const failed = await emitWithAck<{ success: boolean; code?: string }>( + socket, "sync_data", { eventId: "p02-disk-ack-002", data: { value: 12 } }, + ); + assert.equal(failed.success, false, "CSV failure must never receive a positive ACK"); + assert.equal(failed.code, "BUFFER_PERSIST_FAILED"); + serverMod.writer.baseDir = baseDir; + const retry = await emitWithAck<{ success: boolean; code?: string }>( + socket, "sync_data", { eventId: "p02-disk-ack-002", data: { value: 12 } }, + ); + assert.equal(retry.success, true, "The same eventId must recover after disk restoration"); + const csv = fs.readFileSync(csvPath, "utf8"); + assert.equal(csv.split("p02-disk-ack-002").length - 1, 1); + } finally { + socket?.disconnect(); + serverMod.setRedisClientForTests(undefined); + serverMod.writer.baseDir = previousDir; + serverMod.runtimeConfig.batchFlushThreshold = previousThreshold; + serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue); + fs.rmSync(baseDir, { recursive: true, force: true }); + } +}); + +test("P02 R-01 diagnostic: HTTP fallback still ACKs RAM-only data after Redis outage (scope blocker)", async () => { + // P02 remediation only authorizes Socket.IO ACK and buffer *flush* changes. + // Record the separate Port 3000 write/ACK risk; do NOT alter HTTP runtime here. + const redis = createSharedMockRedis(); + redis.rpush = async () => { throw new Error("P02 injected HTTP Redis RPUSH outage"); }; + const client = await SDKManager.registerClientAsync({ + clientId: "p02_http_redis_outage", name: "P02 HTTP Redis Outage", rateLimit: 100, + }); + const eventId = "p02-http-ram-only-001"; + DataBufferEngine.getLocalBufferSnapshot(1); + const state = (globalThis as unknown as { + __dataStackBufferState?: { buffer: Array<{ eventId: string }>; batchFlushThreshold: number }; + }).__dataStackBufferState; + const previousThreshold = state?.batchFlushThreshold; + if (state) state.batchFlushThreshold = 100_000; + DataBufferEngine.setRedisClientForTests(redis); + try { + const response = await postEventsRoute(new NextRequest("http://localhost:3000/api/events", { + method: "POST", + headers: { "content-type": "application/json", authorization: `Bearer ${client.token}` }, + body: JSON.stringify({ eventId, data: { source: "p02-http-diagnostic" } }), + })); + const body = await response.json() as { success?: boolean; code?: string }; + assert.equal(response.status, 200, "Existing HTTP endpoint reports accepted despite failed Redis write"); + assert.equal(body.success, true); + const queued = DataBufferEngine.getLocalBufferSnapshot(1000).filter((evt) => evt.eventId === eventId); + assert.equal(queued.length, 1, "Accepted HTTP event is volatile RAM-only"); + assert.equal((await redis.lrange("datastack:buffer:events", 0, -1)).length, 0); + // Simulated process-loss proof: dropping RAM removes the accepted event. + if (state) state.buffer = state.buffer.filter((evt) => evt.eventId !== eventId); + assert.equal(DataBufferEngine.getLocalBufferSnapshot(1000).some((evt) => evt.eventId === eventId), false); + } finally { + if (state && previousThreshold !== undefined) state.batchFlushThreshold = previousThreshold; + DataBufferEngine.setRedisClientForTests(undefined); + } +}); diff --git a/tests/phase03-storage-buffer.test.ts b/tests/phase03-storage-buffer.test.ts index 6e971cd..16a3036 100644 --- a/tests/phase03-storage-buffer.test.ts +++ b/tests/phase03-storage-buffer.test.ts @@ -1584,10 +1584,66 @@ test("P02 R-02 diagnostic: post-CSV Redis trim failure replays identical event o assert.equal(await redis.llen(REDIS_BUFFER_KEY), 1, "Redis still queues previously appended CSV event"); await DataBufferEngine.flush(); assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0); - assert.equal(writtenIds.filter((id) => id === event.eventId).length, 2, - "A second flush appends the same eventId again after failed trim: at-least-once, not exactly-once"); + assert.equal(writtenIds.filter((id) => id === event.eventId).length, 1, + "Replay must detect the persisted CSV eventId before attempting another append"); + const csvPath = path.join(tmpStorageDir, event.clientId, "2026-11-22.csv"); + const csvContents = fs.readFileSync(csvPath, "utf8"); + assert.equal(csvContents.split(event.eventId).length - 1, 1, + "Exactly one physical CSV row must exist after a failed LTRIM and retry"); } finally { CSVStorageManager.appendBatch = originalAppend; DataBufferEngine.setRedisClientForTests(undefined); } }); + +test("P02 R-02: Port 4000 CSV worker suppresses a crash-window replay after Redis LTRIM error", async () => { + const mockRedis = new FakeRedisClient(); + const originalTrim = mockRedis.ltrim.bind(mockRedis); + let failedOnce = false; + mockRedis.ltrim = async (key: string, start: number, stop: number): Promise<"OK"> => { + if (key === REDIS_BUFFER_KEY && !failedOnce) { + failedOnce = true; + throw new Error("P02 Port4000 injected post-CSV trim failure"); + } + return originalTrim(key, start, stop); + }; + const ingestion = require("../server/index.js") as IngestionServerModule; + const previousDir = ingestion.writer.baseDir; + const previousQueue = ingestion.bufferQueue.slice(); + ingestion.writer.baseDir = tmpStorageDir; + ingestion.bufferQueue.length = 0; + ingestion.setRedisClientForTests(mockRedis); + const evt: DataEvent = { + clientId: "p02_ingestion_replay", + eventId: "p02-port4000-001", + timestamp: "2026-11-22T13:00:00.000Z", + payload: JSON.stringify({ stage: "before-ltrim" }), + }; + try { + await mockRedis.rpush(REDIS_BUFFER_KEY, JSON.stringify(evt)); + await assert.rejects( + () => ingestion.flushBuffer(), + /P02 Port4000 injected post-CSV trim failure/, + ); + assert.equal(failedOnce, true); + assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 1); + const csvPath = path.join(tmpStorageDir, evt.clientId, "2026-11-22.csv"); + assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1); + // Crash/restart may occur after daily part rotation and archive compression. + // Verify replay detection against an actual rotated .csv.gz partition. + const zlib = await import("node:zlib"); + const archivedPath = path.join(tmpStorageDir, evt.clientId, "2026-11-22-part2.csv.gz"); + fs.writeFileSync(archivedPath, zlib.gzipSync(fs.readFileSync(csvPath))); + fs.unlinkSync(csvPath); + const recovered = await ingestion.flushBuffer(); + assert.equal(recovered.written, 0, "Recovered compressed replay must not append again"); + assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 0); + assert.equal(fs.existsSync(csvPath), false, "No duplicate active CSV partition should be created"); + const archivedRows = zlib.gunzipSync(fs.readFileSync(archivedPath)).toString("utf8"); + assert.equal(archivedRows.split(evt.eventId).length - 1, 1); + } finally { + ingestion.writer.baseDir = previousDir; + ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...previousQueue); + ingestion.setRedisClientForTests(undefined); + } +}); diff --git a/tests/r4-phase02-ingestion-audit-streaming.test.ts b/tests/r4-phase02-ingestion-audit-streaming.test.ts index e4dd1ed..baecb68 100644 --- a/tests/r4-phase02-ingestion-audit-streaming.test.ts +++ b/tests/r4-phase02-ingestion-audit-streaming.test.ts @@ -776,3 +776,60 @@ test("R4-Phase-02 AUDIT-12 / ERR-R4-08: Streaming S3 upload & streaming .csv.gz assert.equal(fs.existsSync(tmpDir), false); } }); + +test("P02 R-01: stale pre-persistence Redis claim after restart never yields a false duplicate ACK", async () => { + // A fresh Socket.IO handler has no in-process ACK cache, but Redis retained + // the idempotency claim from a hypothetical crash before Redis RPUSH. + const client = await SDKManager.registerClientAsync({ + clientId: "p02_stale_claim", name: "P02 Stale Claim", rateLimit: 200, + }); + const eventId = "p02-not-yet-persisted-001"; + const stored = [] as string[]; + const redis = { + status: "ready", + async get(key: string) { + return key === `datastack:idem:${client.clientId}:${eventId}` ? "1" : null; + }, + async set() { return null; }, + async incr() { return 1; }, + async pexpire() { return 1; }, + async lrange() { return stored; }, + }; + const httpServer = http.createServer(); + const io = new Server(httpServer); + setupSocketHandlers(io, [], null, { + getRuntimeConfig: () => ({ + jwtSecret: "datastack-jwt-default-secret-key-32-chars-long", + maxEventsPerSec: 200, maxPayloadSizeBytes: 100 * 1024, enableIdempotency: true, + }), + verifyClientInDb: async () => ({ allowed: true, rateLimit: 200 }), + getRedisClient: () => redis, + onBufferPush: async () => undefined, + }); + let socket: ReturnType | null = null; + try { + await new Promise((resolve) => httpServer.listen(0, "127.0.0.1", resolve)); + const port = (httpServer.address() as AddressInfo).port; + socket = ioClient(`http://127.0.0.1:${port}`, { auth: { token: client.token }, transports: ["websocket"] }); + await new Promise((resolve, reject) => { + socket!.once("connect", resolve); + socket!.once("connect_error", reject); + }); + const sendDuplicate = () => new Promise<{ success: boolean; code?: string; note?: string }>((resolve, reject) => { + socket!.timeout(3000).emit("sync_data", { eventId, data: { seq: 1 } }, + (err: Error | null, ack: { success: boolean; code?: string; note?: string }) => { + if (err) reject(err); else resolve(ack); + }); + }); + const uncertain = await sendDuplicate(); + assert.equal(uncertain.success, false, "Redis idempotency claim alone is not a persistence proof"); + assert.equal(uncertain.code, "BUFFER_PERSIST_FAILED"); + stored.push(JSON.stringify({ clientId: client.clientId, eventId })); + const durable = await sendDuplicate(); + assert.equal(durable.success, true, "Same claim is a valid duplicate only after its event is durable"); + assert.equal(durable.note, "DUPLICATE_DROPPED"); + } finally { + socket?.disconnect(); + await new Promise((resolve) => io.close(() => resolve())); + } +});