diff --git a/server/socket/handlers.js b/server/socket/handlers.js index fa406d3..2e109f2 100644 --- a/server/socket/handlers.js +++ b/server/socket/handlers.js @@ -551,7 +551,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { // A previous Redis write failure can leave an uncommitted idempotency // claim. Never report that claim as a successful durable duplicate. - const failedAt = idempotencyMap.get(`${clientId}:${eventId}`); + const failedAt = idempotencyMap.get(`\u0000failed-persist:${clientId}:${eventId}`); const check = await validateSingleEventAsync( clientId, eventId, @@ -560,8 +560,8 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { ); if (!check.allowed) { if (check.duplicate) { - if (typeof failedAt === "number" && failedAt < 0 && - Date.now() + failedAt < IDEMPOTENCY_TTL_MS) { + if (typeof failedAt === "number" && + Date.now() - failedAt < IDEMPOTENCY_TTL_MS) { if (callback) callback({ success: false, eventId, code: "BUFFER_PERSIST_FAILED", error: "BUFFER_PERSIST_FAILED: Event not durably queued", @@ -597,7 +597,8 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { // Invalidate the pre-persistence idempotency claim so SDK retry can // resubmit after Redis recovers. Keep the volatile RAM event pending. const claimKey = `${clientId}:${eventId}`; - idempotencyMap.set(claimKey, -Date.now()); + idempotencyMap.delete(claimKey); + idempotencyMap.set(`\u0000failed-persist:${claimKey}`, Date.now()); const claimRedis = getRedisClient(); if (claimRedis && typeof claimRedis.del === "function") { try { @@ -621,6 +622,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { return; } + idempotencyMap.delete(`\u0000failed-persist:${clientId}:${eventId}`); if (callback) callback({ success: true, eventId }); } catch (err) { if (callback) callback({ success: false, error: err.message }); @@ -726,7 +728,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { : new Date().toISOString(); const rawData = extractBatchItemData(safeItem); - const failedAt = idempotencyMap.get(`${clientId}:${eventId}`); + const failedAt = idempotencyMap.get(`\u0000failed-persist:${clientId}:${eventId}`); const check = await validateSingleEventAsync( clientId, eventId, @@ -735,8 +737,8 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { ); if (!check.allowed) { if (check.duplicate) { - if (typeof failedAt === "number" && failedAt < 0 && - Date.now() + failedAt < IDEMPOTENCY_TTL_MS) { + if (typeof failedAt === "number" && + Date.now() - failedAt < IDEMPOTENCY_TTL_MS) { rejected += 1; volatileDuplicate = true; if (errors.length < 5) errors.push( @@ -779,7 +781,8 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { const claimRedis = getRedisClient(); for (const eventId of acceptedEventIds) { const claimKey = `${clientId}:${eventId}`; - idempotencyMap.set(claimKey, -Date.now()); + idempotencyMap.delete(claimKey); + idempotencyMap.set(`\u0000failed-persist:${claimKey}`, Date.now()); if (claimRedis && typeof claimRedis.del === "function") { try { await claimRedis.del(`${REDIS_IDEMPOTENCY_PREFIX}${claimKey}`); @@ -806,6 +809,22 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { } } + // Only durable new writes clear pending failure markers. + for (const persistedId of acceptedEventIds) { + idempotencyMap.delete(`\u0000failed-persist:${clientId}:${persistedId}`); + } + // A mixed batch must never receive success while one event still has + // an uncommitted Redis claim, even if other items were persisted. + if (volatileDuplicate && processed > 0) { + if (callback) callback({ + success: false, processed, rejected, duplicates, + code: "BUFFER_PERSIST_FAILED", + error: "BUFFER_PERSIST_FAILED: Batch contains non-durable event", + errors, + }); + return; + } + if (processed === 0 && duplicates === 0 && rejected > 0) { const code = volatileDuplicate ? "BUFFER_PERSIST_FAILED" diff --git a/tests/phase02-sync.test.ts b/tests/phase02-sync.test.ts index 6067ab5..44a3993 100644 --- a/tests/phase02-sync.test.ts +++ b/tests/phase02-sync.test.ts @@ -1338,3 +1338,111 @@ test("P02 R-01: batch_sync rejects non-durable Redis writes and pending duplicat serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue); } }); + +test("P02 FINAL R-01: stale Redis claim never becomes positive ACK after cleanup window", async () => { + // Source proof at candidate 76cd972: a failure marker is -Date.now(); + // cleanupMaps compares now - negativeTimestamp > TTL, which expires immediately + // on the first scheduled cleanup even though a Redis idempotency claim remains. + const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold; + const previousQueue = serverMod.bufferQueue.slice(); + const redis = createSharedMockRedis(); // intentionally no DEL: simulate stuck Redis claim + redis.rpush = async () => { throw new Error("Injected Redis queue outage"); }; + serverMod.setRedisClientForTests(redis); + serverMod.runtimeConfig.batchFlushThreshold = 100_000; + serverMod.bufferQueue.length = 0; + const originalNow = Date.now; + let socket: Socket | undefined; + try { + const client = await SDKManager.registerClientAsync({ + clientId: "p02_stale_claim_cleanup", name: "P02 Stale Claim", rateLimit: 100, + }); + socket = await connectSocket(client.token); + const payload = { eventId: "p02-stale-claim-1", data: { value: 1 } }; + const first = await emitWithAck<{ success: boolean; code?: string }>(socket, "sync_data", payload); + assert.equal(first.success, false); + assert.equal(first.code, "BUFFER_PERSIST_FAILED"); + + // Advance only the fixture clock beyond cleanup interval, but within + // five-minute idempotency TTL. Two retries expose the old negative-marker + // cleanup defect; no actual timer wait or production timestamp is changed. + Date.now = () => originalNow() + 61_000; + const retry1 = await emitWithAck<{ success: boolean; code?: string }>(socket, "sync_data", payload); + const retry2 = await emitWithAck<{ success: boolean; code?: string }>(socket, "sync_data", payload); + assert.equal(retry1.success, false); + assert.equal(retry1.code, "BUFFER_PERSIST_FAILED"); + assert.equal(retry2.success, false, "Cleanup must not convert a volatile claim into DUPLICATE_DROPPED success"); + assert.equal(retry2.code, "BUFFER_PERSIST_FAILED"); + assert.equal((await redis.lrange("datastack:buffer:events", 0, -1)).length, 0); + } finally { + Date.now = originalNow; + socket?.disconnect(); + serverMod.setRedisClientForTests(undefined); + serverMod.runtimeConfig.batchFlushThreshold = previousThreshold; + serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue); + } +}); + +test("P02 FINAL R-01: mixed batch with stale volatile claim never ACKs success", async () => { + const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold; + const previousQueue = serverMod.bufferQueue.slice(); + const redis = createSharedMockRedis(); + const originalRpush = redis.rpush.bind(redis); + redis.rpush = async () => { throw new Error("Injected Redis outage before batch recovery"); }; + serverMod.setRedisClientForTests(redis); + serverMod.runtimeConfig.batchFlushThreshold = 100_000; + serverMod.bufferQueue.length = 0; + let socket: Socket | undefined; + try { + const client = await SDKManager.registerClientAsync({ + clientId: "p02_mixed_batch_claim", name: "P02 Mixed Batch Claim", rateLimit: 100, + }); + socket = await connectSocket(client.token); + const pending = { eventId: "p02-mixed-pending-01", data: { v: 1 } }; + const first = await emitWithAck<{ success: boolean; code?: string }>( + socket, "sync_data", pending, + ); + assert.equal(first.success, false); + // Simulated process RAM loss: Redis idempotency key survives; queue does not. + serverMod.bufferQueue.length = 0; + redis.rpush = originalRpush; + const mixed = await emitWithAck<{ success: boolean; code?: string; processed?: number }>( + socket, "batch_sync", [pending, { eventId: "p02-mixed-durable-02", data: { v: 2 } }], + ); + assert.equal(mixed.success, false, "Mixed batch cannot falsely ACK the volatile duplicate"); + assert.equal(mixed.code, "BUFFER_PERSIST_FAILED"); + assert.equal(mixed.processed, 1, "A distinct new event can commit without masking the failed claim"); + const rows = await redis.lrange("datastack:buffer:events", 0, -1); + assert.equal(rows.length, 1); + assert.equal((JSON.parse(rows[0]) as { eventId: string }).eventId, "p02-mixed-durable-02"); + } finally { + socket?.disconnect(); + serverMod.setRedisClientForTests(undefined); + serverMod.runtimeConfig.batchFlushThreshold = previousThreshold; + serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue); + } +}); + +test("P02 FINAL R-01 compatibility: deliberately Redis-free RAM mode remains unchanged", async () => { + const threshold = serverMod.runtimeConfig.batchFlushThreshold; + const queue = serverMod.bufferQueue.slice(); + serverMod.setRedisClientForTests(null); + serverMod.runtimeConfig.batchFlushThreshold = 100_000; + serverMod.bufferQueue.length = 0; + let socket: Socket | undefined; + try { + const client = await SDKManager.registerClientAsync({ + clientId: "p02_redis_free_compat", name: "P02 Redis-free Compatibility", rateLimit: 100, + }); + socket = await connectSocket(client.token); + const ack = await emitWithAck<{ success: boolean; eventId?: string }>( + socket, "sync_data", { eventId: "p02-ram-compat-001", data: { value: 1 } }, + ); + assert.equal(ack.success, true, "Preserve historical RAM-mode acceptance contract"); + assert.equal(serverMod.bufferQueue.some((e) => e.eventId === "p02-ram-compat-001"), true); + } finally { + socket?.disconnect(); + serverMod.setRedisClientForTests(undefined); + serverMod.runtimeConfig.batchFlushThreshold = threshold; + serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...queue); + } +}); diff --git a/tests/phase03-storage-buffer.test.ts b/tests/phase03-storage-buffer.test.ts index a75652f..e3849a3 100644 --- a/tests/phase03-storage-buffer.test.ts +++ b/tests/phase03-storage-buffer.test.ts @@ -1639,3 +1639,79 @@ test("P02 R-02: Port 4000 CSV worker suppresses a crash-window replay after Redi ingestion.setRedisClientForTests(undefined); } }); + +test("P02 FINAL R-02: Port 4000 scans rotated compressed CSV with quoted multiline payload", async () => { + const zlib = await import("node:zlib"); + const redis = new FakeRedisClient(); + const ingestion = require("../server/index.js") as IngestionServerModule; + const oldDir = ingestion.writer.baseDir; + const oldQueue = ingestion.bufferQueue.slice(); + ingestion.writer.baseDir = tmpStorageDir; + ingestion.bufferQueue.length = 0; + ingestion.setRedisClientForTests(redis); + const event: DataEvent = { + clientId: "p02_rotated_gzip", + eventId: "p02-gzip-quoted-001", + timestamp: "2026-11-22T12:45:00.000Z", + payload: JSON.stringify({ body: "Line one, \"quoted\"\nLine two" }), + }; + try { + const diskWriter = new CsvWriterWorker(tmpStorageDir); + await diskWriter.writeBatch([event]); + const dir = path.join(tmpStorageDir, event.clientId); + const csvPath = path.join(dir, "2026-11-22.csv"); + const rotatedPath = path.join(dir, "2026-11-22-part2.csv"); + const gzPath = rotatedPath + ".gz"; + fs.renameSync(csvPath, rotatedPath); + fs.writeFileSync(gzPath, zlib.gzipSync(fs.readFileSync(rotatedPath))); + fs.unlinkSync(rotatedPath); + // A different date partition must not be parsed as part of this replay. + fs.writeFileSync(path.join(dir, "2026-11-21.csv"), 'invalid,"unterminated'); + await redis.rpush(REDIS_BUFFER_KEY, JSON.stringify(event)); + const replay = await ingestion.flushBuffer(); + assert.equal(replay.written, 0, "Already written rotated/gzipped event must not append again"); + assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0, "Successful dedupe permits Redis trim"); + assert.equal(fs.existsSync(csvPath), false, "Do not recreate a duplicate plain CSV"); + assert.equal(zlib.gunzipSync(fs.readFileSync(gzPath)).toString("utf8").split(event.eventId).length - 1, 1); + } finally { + ingestion.writer.baseDir = oldDir; + ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...oldQueue); + ingestion.setRedisClientForTests(undefined); + } +}); + +test("P02 FINAL R-02: partial corrupted CSV blocks Redis trim and recovers after fixture repair", async () => { + const redis = new FakeRedisClient(); + const ingestion = require("../server/index.js") as IngestionServerModule; + const oldDir = ingestion.writer.baseDir; + const oldQueue = ingestion.bufferQueue.slice(); + ingestion.writer.baseDir = tmpStorageDir; + ingestion.bufferQueue.length = 0; + ingestion.setRedisClientForTests(redis); + const event: DataEvent = { + clientId: "p02_partial_corrupt", + eventId: "p02-corrupt-recovery-001", + timestamp: "2026-11-22T13:15:00.000Z", + payload: JSON.stringify({ value: "safe" }), + }; + try { + const dir = path.join(tmpStorageDir, event.clientId); + fs.mkdirSync(dir, { recursive: true }); + const badPath = path.join(dir, "2026-11-22.csv"); + fs.writeFileSync(badPath, 'clientId,eventId,timestamp,payload\n"unterminated'); + await redis.rpush(REDIS_BUFFER_KEY, JSON.stringify(event)); + await assert.rejects(() => ingestion.flushBuffer(), + /quote|CSV_|invalid|record|unterminated/i, "Malformed existing CSV must fail closed"); + assert.equal(await redis.llen(REDIS_BUFFER_KEY), 1, "Never trim after failed CSV replay check"); + // Disposable fixture repair only; does not alter user or production files. + fs.unlinkSync(badPath); + const recovered = await ingestion.flushBuffer(); + assert.equal(recovered.written, 1); + assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0); + assert.equal(fs.readFileSync(badPath, "utf8").split(event.eventId).length - 1, 1); + } finally { + ingestion.writer.baseDir = oldDir; + ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...oldQueue); + ingestion.setRedisClientForTests(undefined); + } +});