From b0835fd43b52f0bd66bcf755771fa47afe98e972 Mon Sep 17 00:00:00 2001 From: victorsteele428 Date: Sun, 22 Nov 2026 16:44:21 +0600 Subject: [PATCH] fix(p02): fail closed on uncertain ACKs and verify restart durability [skip ci] --- server/index.js | 20 ++- server/socket/handlers.js | 130 +++++++++++++++--- tests/phase02-sync.test.ts | 79 ++++++++++- tests/phase03-storage-buffer.test.ts | 12 +- ...-phase02-ingestion-audit-streaming.test.ts | 57 ++++++++ 5 files changed, 275 insertions(+), 23 deletions(-) diff --git a/server/index.js b/server/index.js index a605320..6f8b7df 100644 --- a/server/index.js +++ b/server/index.js @@ -164,6 +164,16 @@ function setRedisClientForTests(client) { async function pushEventsToRedisOrFallback(events) { if (!events || events.length === 0) return bufferQueue.length; + // Failed persistence may leave an unacknowledged RAM event beside an SDK + // retry with the same ID. Persist each client/event ID at most once per push. + const seen = new Set(); + events = events.filter((evt) => { + if (!evt || !evt.clientId || !evt.eventId) return true; + const key = JSON.stringify([evt.clientId, evt.eventId]); + if (seen.has(key)) return false; + seen.add(key); + return true; + }); const redis = getRedisClient(); let persistFailed = false; if (redis) { @@ -1985,8 +1995,14 @@ const socketHandlersApi = setupSocketHandlers(io, bufferQueue, writer, { return; } - if (bufferQueue.length >= runtimeConfig.batchFlushThreshold) { - void flushBuffer().catch(() => undefined); + // Redis is optional. Without it, a positive Socket.IO ACK requires a + // completed CSV write, not merely an event sitting in volatile RAM. + // Keep existing RAM retry state if the disk write fails. + if (bufferQueue.length > 0) { + const saved = await flushBuffer(); + if (saved.lockTimedOut || bufferQueue.length > 0) { + throw new Error('BUFFER_PERSIST_FAILED: Event not durably queued'); + } } }, }); diff --git a/server/socket/handlers.js b/server/socket/handlers.js index fa406d3..17c371a 100644 --- a/server/socket/handlers.js +++ b/server/socket/handlers.js @@ -478,6 +478,84 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { return hasNonEnvelopeKey ? stripped : {}; } + // P02 R-01: an idempotency claim is created before persistence. After a + // crash that claim may survive without its event. Prove the event reached + // Redis's buffer or a CSV partition before issuing a duplicate-success ACK. + async function hasDurableEvent(clientId, eventId, timestamp) { + const ackKey = `\u0000durably-acked:${clientId}:${eventId}`; + const previousAck = idempotencyMap.get(ackKey); + if (typeof previousAck === 'number' && Date.now() - previousAck < IDEMPOTENCY_TTL_MS) { + return true; + } + const redis = getRedisClient(); + if (redis && typeof redis.lrange === 'function') { + try { + // Bounded read: reject uncertain old claims instead of loading a huge Redis queue. + const queued = await redis.lrange('datastack:buffer:events', -512, -1); + if (Array.isArray(queued) && queued.some((raw) => { + try { + const evt = JSON.parse(raw); + return evt && evt.clientId === clientId && evt.eventId === eventId; + } catch { return false; } + })) { + idempotencyMap.set(ackKey, Date.now()); + return true; + } + } catch { + // Redis cannot prove durability; fall through to read-only CSV proof. + } + } + if (!worker || typeof worker.sanitizeClientId !== 'function' || + typeof worker.extractSafePartitionDate !== 'function' || + typeof worker.isPartitionFilename !== 'function' || + typeof worker.extractDateFromFilename !== 'function') return false; + const fs = require('node:fs'); + const path = require('node:path'); + const { pipeline } = require('node:stream/promises'); + const { Writable } = require('node:stream'); + const { parse } = require('csv-parse'); + const zlib = require('node:zlib'); + const aborter = new AbortController(); + const deadline = setTimeout(() => aborter.abort(), 1800); + deadline.unref?.(); + try { + const base = await fs.promises.realpath(worker.baseDir); + const directory = path.resolve(base, worker.sanitizeClientId(clientId)); + const realDir = await fs.promises.realpath(directory); + if (!realDir.startsWith(base + path.sep)) return false; + const date = worker.extractSafePartitionDate(timestamp); + const files = await fs.promises.readdir(realDir, { withFileTypes: true }); + let scannedBytes = 0; + for (const item of files) { + if (!item.isFile() || !worker.isPartitionFilename(item.name) || + worker.extractDateFromFilename(item.name) !== date) continue; + const filename = path.join(realDir, item.name); + scannedBytes += (await fs.promises.stat(filename)).size; + if (scannedBytes > 256 * 1024 * 1024) return false; + let found = false; + const stages = [fs.createReadStream(filename)]; + if (item.name.endsWith('.gz')) stages.push(zlib.createGunzip()); + stages.push(parse({ columns: true, bom: true, skip_empty_lines: true })); + stages.push(new Writable({ + objectMode: true, + write(row, _encoding, done) { + if (row.clientId === clientId && row.eventId === eventId) found = true; + done(); + }, + })); + await pipeline(...stages, { signal: aborter.signal }); + if (found) { + idempotencyMap.set(ackKey, Date.now()); + return true; + } + } + } catch { + // Missing, corrupt, unreadable or slow CSV cannot justify positive ACK. + } finally { + clearTimeout(deadline); + } + return false; + } io.on("connection", (socket) => { const clientId = socket.data.clientId; socket.join(`client:${clientId}`); @@ -551,7 +629,6 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { // A previous Redis write failure can leave an uncommitted idempotency // claim. Never report that claim as a successful durable duplicate. - const failedAt = idempotencyMap.get(`${clientId}:${eventId}`); const check = await validateSingleEventAsync( clientId, eventId, @@ -560,14 +637,14 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { ); if (!check.allowed) { if (check.duplicate) { - if (typeof failedAt === "number" && failedAt < 0 && - Date.now() + failedAt < IDEMPOTENCY_TTL_MS) { - if (callback) callback({ - success: false, eventId, code: "BUFFER_PERSIST_FAILED", - error: "BUFFER_PERSIST_FAILED: Event not durably queued", - }); + if (await hasDurableEvent(clientId, eventId, timestamp)) { + idempotencyMap.delete(`\u0000failed-persist:${clientId}:${eventId}`); + if (callback) callback({ success: true, eventId, note: "DUPLICATE_DROPPED" }); } else if (callback) { - callback({ success: true, eventId, note: "DUPLICATE_DROPPED" }); + callback({ + success: false, eventId, code: "BUFFER_PERSIST_FAILED", + error: "BUFFER_PERSIST_FAILED: Idempotency claim has no durable event", + }); } return; } @@ -597,7 +674,8 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { // Invalidate the pre-persistence idempotency claim so SDK retry can // resubmit after Redis recovers. Keep the volatile RAM event pending. const claimKey = `${clientId}:${eventId}`; - idempotencyMap.set(claimKey, -Date.now()); + idempotencyMap.delete(claimKey); + idempotencyMap.set(`\u0000failed-persist:${claimKey}`, Date.now()); const claimRedis = getRedisClient(); if (claimRedis && typeof claimRedis.del === "function") { try { @@ -621,6 +699,9 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { return; } + // Clear any failed local persistence marker only after a durable write. + idempotencyMap.delete(`\u0000failed-persist:${clientId}:${eventId}`); + idempotencyMap.set(`\u0000durably-acked:${clientId}:${eventId}`, Date.now()); if (callback) callback({ success: true, eventId }); } catch (err) { if (callback) callback({ success: false, error: err.message }); @@ -726,8 +807,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { : new Date().toISOString(); const rawData = extractBatchItemData(safeItem); - const failedAt = idempotencyMap.get(`${clientId}:${eventId}`); - const check = await validateSingleEventAsync( + const check = await validateSingleEventAsync( clientId, eventId, rawData, @@ -735,14 +815,14 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { ); if (!check.allowed) { if (check.duplicate) { - if (typeof failedAt === "number" && failedAt < 0 && - Date.now() + failedAt < IDEMPOTENCY_TTL_MS) { + if (await hasDurableEvent(clientId, eventId, timestamp)) { + idempotencyMap.delete(`\u0000failed-persist:${clientId}:${eventId}`); + duplicates += 1; + } else { rejected += 1; volatileDuplicate = true; if (errors.length < 5) errors.push( - `${eventId}: BUFFER_PERSIST_FAILED: Event not durably queued`); - } else { - duplicates += 1; + `${eventId}: BUFFER_PERSIST_FAILED: Idempotency claim has no durable event`); } continue; } @@ -779,7 +859,8 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { const claimRedis = getRedisClient(); for (const eventId of acceptedEventIds) { const claimKey = `${clientId}:${eventId}`; - idempotencyMap.set(claimKey, -Date.now()); + idempotencyMap.delete(claimKey); + idempotencyMap.set(`\u0000failed-persist:${claimKey}`, Date.now()); if (claimRedis && typeof claimRedis.del === "function") { try { await claimRedis.del(`${REDIS_IDEMPOTENCY_PREFIX}${claimKey}`); @@ -806,6 +887,21 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { } } + for (const persistedId of acceptedEventIds) { + idempotencyMap.delete(`\u0000failed-persist:${clientId}:${persistedId}`); + idempotencyMap.set(`\u0000durably-acked:${clientId}:${persistedId}`, Date.now()); + } + // Never ACK a mixed batch as successful while any event remains volatile. + if (volatileDuplicate) { + if (callback) callback({ + success: false, processed, rejected, duplicates, + code: "BUFFER_PERSIST_FAILED", + error: "BUFFER_PERSIST_FAILED: Batch contains non-durable event", + errors, + }); + return; + } + if (processed === 0 && duplicates === 0 && rejected > 0) { const code = volatileDuplicate ? "BUFFER_PERSIST_FAILED" diff --git a/tests/phase02-sync.test.ts b/tests/phase02-sync.test.ts index 6067ab5..e4382e4 100644 --- a/tests/phase02-sync.test.ts +++ b/tests/phase02-sync.test.ts @@ -36,6 +36,7 @@ import { const require = createRequire(import.meta.url); interface IngestionServerModule { + writer: { baseDir: string }; server: import("node:http").Server; io: import("socket.io").Server; bufferQueue: Array<{ clientId: string; eventId: string; timestamp: string; payload: string }>; @@ -1221,7 +1222,7 @@ test("R2 PHASE-02 BUG-10: GET, PATCH (regenerate_token), and DELETE /api/clients } }); -test("P02 R-01 diagnostic: Redis RPUSH failure yields positive WS ACK for volatile RAM-only event", async () => { +test("P02 R-01 remediation: Redis RPUSH failure rejects non-durable WS ACK", async () => { // Disposable test-only fixture: no production Redis, service, or storage state. const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold; const previousRam = serverMod.bufferQueue.slice(); @@ -1255,6 +1256,12 @@ test("P02 R-01 diagnostic: Redis RPUSH failure yields positive WS ACK for volati assert.equal(sameIdRetry.success, false, "Pending idempotency claims must not return false positive duplicate ACK"); assert.equal(sameIdRetry.code, "BUFFER_PERSIST_FAILED"); + const secondRetry = await emitWithAck<{ success: boolean; code?: string }>( + socket, "sync_data", + { eventId: "p02-ack-before-crash-001", data: { value: 42 } }, + ); + assert.equal(secondRetry.success, false, "A repeated pending claim must remain negative"); + assert.equal(secondRetry.code, "BUFFER_PERSIST_FAILED"); assert.equal(committedRows.length, 0, "Fake Redis has no durable event"); assert.equal(volatileRows.length, 1, "Acknowledged event is RAM-only"); // Crash model: restart loses RAM but fake Redis survives. No actual process is killed. @@ -1268,7 +1275,12 @@ test("P02 R-01 diagnostic: Redis RPUSH failure yields positive WS ACK for volati { eventId: "p02-ack-recovered-002", data: { value: 43 } }, ); assert.equal(recovered.success, true); - assert.equal((await healthyRedis.lrange("datastack:buffer:events", 0, -1)).length, 1); + const recoveredSameId = await emitWithAck<{ success: boolean; code?: string }>( + socket, "sync_data", + { eventId: "p02-ack-before-crash-001", data: { value: 42 } }, + ); + assert.equal(recoveredSameId.success, true, "Original eventId must be retryable after Redis recovery"); + assert.equal((await healthyRedis.lrange("datastack:buffer:events", 0, -1)).length, 2); } finally { socket?.disconnect(); serverMod.setRedisClientForTests(undefined); @@ -1303,6 +1315,7 @@ test("P02 R-01: batch_sync rejects non-durable Redis writes and pending duplicat const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold; const previousQueue = serverMod.bufferQueue.slice(); const failingRedis = createSharedMockRedis(); + const recoveredRpush = failingRedis.rpush.bind(failingRedis); failingRedis.rpush = async () => { throw new Error("P02 injected batch Redis outage"); }; serverMod.setRedisClientForTests(failingRedis); serverMod.runtimeConfig.batchFlushThreshold = 100_000; @@ -1331,6 +1344,18 @@ test("P02 R-01: batch_sync rejects non-durable Redis writes and pending duplicat ); assert.equal(retry.success, false); assert.equal(retry.code, "BUFFER_PERSIST_FAILED"); + failingRedis.rpush = recoveredRpush; + const mixed = await emitWithAck<{ success: boolean; code?: string; processed?: number }>( + socket, "batch_sync", [payload[0], { eventId: "p02-batch-recovered-3", data: { value: 3 } }], + ); + assert.equal(mixed.success, false, "A mixed recovered/volatile batch cannot ACK success"); + assert.equal(mixed.code, "BUFFER_PERSIST_FAILED"); + assert.equal(mixed.processed, 1, "New item is durably queued, old idempotency claim is uncertain"); + const confirmed = await emitWithAck<{ success: boolean; duplicates?: number }>( + socket, "batch_sync", payload, + ); + assert.equal(confirmed.success, true, "Once old events are persisted, duplicates are safely acknowledged"); + assert.equal(confirmed.duplicates, 2); } finally { socket?.disconnect(); serverMod.setRedisClientForTests(undefined); @@ -1338,3 +1363,53 @@ test("P02 R-01: batch_sync rejects non-durable Redis writes and pending duplicat serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue); } }); + +test("P02 R-01: Redis-free Socket.IO ACK waits for CSV durability and denies disk errors", async () => { + const fs = await import("node:fs"); + const os = await import("node:os"); + const path = await import("node:path"); + const baseDir = fs.mkdtempSync(path.join(os.tmpdir(), "datastack-p02-ack-disk-")); + const previousDir = serverMod.writer.baseDir; + const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold; + const previousQueue = serverMod.bufferQueue.slice(); + serverMod.writer.baseDir = baseDir; + serverMod.runtimeConfig.batchFlushThreshold = 100_000; + serverMod.setRedisClientForTests(null); + serverMod.bufferQueue.length = 0; + let socket: Socket | undefined; + try { + const client = await SDKManager.registerClientAsync({ + clientId: "p02_disk_durability", name: "P02 Disk Durability", rateLimit: 100, + }); + socket = await connectSocket(client.token); + const accepted = await emitWithAck<{ success: boolean; code?: string }>( + socket, "sync_data", { eventId: "p02-disk-ack-001", data: { value: 11 } }, + ); + assert.equal(accepted.success, true); + const csvPath = path.join(baseDir, client.clientId, new Date().toISOString().slice(0, 10) + ".csv"); + assert.equal(fs.readFileSync(csvPath, "utf8").includes("p02-disk-ack-001"), true, + "A successful ACK requires a completed CSV append when Redis is absent"); + const blockedBase = path.join(baseDir, "not-a-directory"); + fs.writeFileSync(blockedBase, "fixture"); + serverMod.writer.baseDir = blockedBase; + const failed = await emitWithAck<{ success: boolean; code?: string }>( + socket, "sync_data", { eventId: "p02-disk-ack-002", data: { value: 12 } }, + ); + assert.equal(failed.success, false, "CSV failure must never receive a positive ACK"); + assert.equal(failed.code, "BUFFER_PERSIST_FAILED"); + serverMod.writer.baseDir = baseDir; + const retry = await emitWithAck<{ success: boolean; code?: string }>( + socket, "sync_data", { eventId: "p02-disk-ack-002", data: { value: 12 } }, + ); + assert.equal(retry.success, true, "The same eventId must recover after disk restoration"); + const csv = fs.readFileSync(csvPath, "utf8"); + assert.equal(csv.split("p02-disk-ack-002").length - 1, 1); + } finally { + socket?.disconnect(); + serverMod.setRedisClientForTests(undefined); + serverMod.writer.baseDir = previousDir; + serverMod.runtimeConfig.batchFlushThreshold = previousThreshold; + serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue); + fs.rmSync(baseDir, { recursive: true, force: true }); + } +}); diff --git a/tests/phase03-storage-buffer.test.ts b/tests/phase03-storage-buffer.test.ts index a75652f..16a3036 100644 --- a/tests/phase03-storage-buffer.test.ts +++ b/tests/phase03-storage-buffer.test.ts @@ -1629,10 +1629,18 @@ test("P02 R-02: Port 4000 CSV worker suppresses a crash-window replay after Redi assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 1); const csvPath = path.join(tmpStorageDir, evt.clientId, "2026-11-22.csv"); assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1); + // Crash/restart may occur after daily part rotation and archive compression. + // Verify replay detection against an actual rotated .csv.gz partition. + const zlib = await import("node:zlib"); + const archivedPath = path.join(tmpStorageDir, evt.clientId, "2026-11-22-part2.csv.gz"); + fs.writeFileSync(archivedPath, zlib.gzipSync(fs.readFileSync(csvPath))); + fs.unlinkSync(csvPath); const recovered = await ingestion.flushBuffer(); - assert.equal(recovered.written, 0, "Recovered replay should not append the event again"); + assert.equal(recovered.written, 0, "Recovered compressed replay must not append again"); assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 0); - assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1); + assert.equal(fs.existsSync(csvPath), false, "No duplicate active CSV partition should be created"); + const archivedRows = zlib.gunzipSync(fs.readFileSync(archivedPath)).toString("utf8"); + assert.equal(archivedRows.split(evt.eventId).length - 1, 1); } finally { ingestion.writer.baseDir = previousDir; ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...previousQueue); diff --git a/tests/r4-phase02-ingestion-audit-streaming.test.ts b/tests/r4-phase02-ingestion-audit-streaming.test.ts index e4dd1ed..baecb68 100644 --- a/tests/r4-phase02-ingestion-audit-streaming.test.ts +++ b/tests/r4-phase02-ingestion-audit-streaming.test.ts @@ -776,3 +776,60 @@ test("R4-Phase-02 AUDIT-12 / ERR-R4-08: Streaming S3 upload & streaming .csv.gz assert.equal(fs.existsSync(tmpDir), false); } }); + +test("P02 R-01: stale pre-persistence Redis claim after restart never yields a false duplicate ACK", async () => { + // A fresh Socket.IO handler has no in-process ACK cache, but Redis retained + // the idempotency claim from a hypothetical crash before Redis RPUSH. + const client = await SDKManager.registerClientAsync({ + clientId: "p02_stale_claim", name: "P02 Stale Claim", rateLimit: 200, + }); + const eventId = "p02-not-yet-persisted-001"; + const stored = [] as string[]; + const redis = { + status: "ready", + async get(key: string) { + return key === `datastack:idem:${client.clientId}:${eventId}` ? "1" : null; + }, + async set() { return null; }, + async incr() { return 1; }, + async pexpire() { return 1; }, + async lrange() { return stored; }, + }; + const httpServer = http.createServer(); + const io = new Server(httpServer); + setupSocketHandlers(io, [], null, { + getRuntimeConfig: () => ({ + jwtSecret: "datastack-jwt-default-secret-key-32-chars-long", + maxEventsPerSec: 200, maxPayloadSizeBytes: 100 * 1024, enableIdempotency: true, + }), + verifyClientInDb: async () => ({ allowed: true, rateLimit: 200 }), + getRedisClient: () => redis, + onBufferPush: async () => undefined, + }); + let socket: ReturnType | null = null; + try { + await new Promise((resolve) => httpServer.listen(0, "127.0.0.1", resolve)); + const port = (httpServer.address() as AddressInfo).port; + socket = ioClient(`http://127.0.0.1:${port}`, { auth: { token: client.token }, transports: ["websocket"] }); + await new Promise((resolve, reject) => { + socket!.once("connect", resolve); + socket!.once("connect_error", reject); + }); + const sendDuplicate = () => new Promise<{ success: boolean; code?: string; note?: string }>((resolve, reject) => { + socket!.timeout(3000).emit("sync_data", { eventId, data: { seq: 1 } }, + (err: Error | null, ack: { success: boolean; code?: string; note?: string }) => { + if (err) reject(err); else resolve(ack); + }); + }); + const uncertain = await sendDuplicate(); + assert.equal(uncertain.success, false, "Redis idempotency claim alone is not a persistence proof"); + assert.equal(uncertain.code, "BUFFER_PERSIST_FAILED"); + stored.push(JSON.stringify({ clientId: client.clientId, eventId })); + const durable = await sendDuplicate(); + assert.equal(durable.success, true, "Same claim is a valid duplicate only after its event is durable"); + assert.equal(durable.note, "DUPLICATE_DROPPED"); + } finally { + socket?.disconnect(); + await new Promise((resolve) => io.close(() => resolve())); + } +});