fix(p02): fail closed on uncertain ACKs and verify restart durability [skip ci]
This commit is contained in:
1 parent
76cd972bde
commit
b0835fd43b
5 files changed
+274
-22
No files matched your search
+18
-2
@@ -164,6 +164,16 @@ function setRedisClientForTests(client) {
|
|||||||
|
|
||||||
async function pushEventsToRedisOrFallback(events) {
|
async function pushEventsToRedisOrFallback(events) {
|
||||||
if (!events || events.length === 0) return bufferQueue.length;
|
if (!events || events.length === 0) return bufferQueue.length;
|
||||||
|
// Failed persistence may leave an unacknowledged RAM event beside an SDK
|
||||||
|
// retry with the same ID. Persist each client/event ID at most once per push.
|
||||||
|
const seen = new Set();
|
||||||
|
events = events.filter((evt) => {
|
||||||
|
if (!evt || !evt.clientId || !evt.eventId) return true;
|
||||||
|
const key = JSON.stringify([evt.clientId, evt.eventId]);
|
||||||
|
if (seen.has(key)) return false;
|
||||||
|
seen.add(key);
|
||||||
|
return true;
|
||||||
|
});
|
||||||
const redis = getRedisClient();
|
const redis = getRedisClient();
|
||||||
let persistFailed = false;
|
let persistFailed = false;
|
||||||
if (redis) {
|
if (redis) {
|
||||||
@@ -1985,8 +1995,14 @@ const socketHandlersApi = setupSocketHandlers(io, bufferQueue, writer, {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (bufferQueue.length >= runtimeConfig.batchFlushThreshold) {
|
// Redis is optional. Without it, a positive Socket.IO ACK requires a
|
||||||
void flushBuffer().catch(() => undefined);
|
// completed CSV write, not merely an event sitting in volatile RAM.
|
||||||
|
// Keep existing RAM retry state if the disk write fails.
|
||||||
|
if (bufferQueue.length > 0) {
|
||||||
|
const saved = await flushBuffer();
|
||||||
|
if (saved.lockTimedOut || bufferQueue.length > 0) {
|
||||||
|
throw new Error('BUFFER_PERSIST_FAILED: Event not durably queued');
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|||||||
+112
-16
@@ -478,6 +478,84 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
return hasNonEnvelopeKey ? stripped : {};
|
return hasNonEnvelopeKey ? stripped : {};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// P02 R-01: an idempotency claim is created before persistence. After a
|
||||||
|
// crash that claim may survive without its event. Prove the event reached
|
||||||
|
// Redis's buffer or a CSV partition before issuing a duplicate-success ACK.
|
||||||
|
async function hasDurableEvent(clientId, eventId, timestamp) {
|
||||||
|
const ackKey = `\u0000durably-acked:${clientId}:${eventId}`;
|
||||||
|
const previousAck = idempotencyMap.get(ackKey);
|
||||||
|
if (typeof previousAck === 'number' && Date.now() - previousAck < IDEMPOTENCY_TTL_MS) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
const redis = getRedisClient();
|
||||||
|
if (redis && typeof redis.lrange === 'function') {
|
||||||
|
try {
|
||||||
|
// Bounded read: reject uncertain old claims instead of loading a huge Redis queue.
|
||||||
|
const queued = await redis.lrange('datastack:buffer:events', -512, -1);
|
||||||
|
if (Array.isArray(queued) && queued.some((raw) => {
|
||||||
|
try {
|
||||||
|
const evt = JSON.parse(raw);
|
||||||
|
return evt && evt.clientId === clientId && evt.eventId === eventId;
|
||||||
|
} catch { return false; }
|
||||||
|
})) {
|
||||||
|
idempotencyMap.set(ackKey, Date.now());
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Redis cannot prove durability; fall through to read-only CSV proof.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!worker || typeof worker.sanitizeClientId !== 'function' ||
|
||||||
|
typeof worker.extractSafePartitionDate !== 'function' ||
|
||||||
|
typeof worker.isPartitionFilename !== 'function' ||
|
||||||
|
typeof worker.extractDateFromFilename !== 'function') return false;
|
||||||
|
const fs = require('node:fs');
|
||||||
|
const path = require('node:path');
|
||||||
|
const { pipeline } = require('node:stream/promises');
|
||||||
|
const { Writable } = require('node:stream');
|
||||||
|
const { parse } = require('csv-parse');
|
||||||
|
const zlib = require('node:zlib');
|
||||||
|
const aborter = new AbortController();
|
||||||
|
const deadline = setTimeout(() => aborter.abort(), 1800);
|
||||||
|
deadline.unref?.();
|
||||||
|
try {
|
||||||
|
const base = await fs.promises.realpath(worker.baseDir);
|
||||||
|
const directory = path.resolve(base, worker.sanitizeClientId(clientId));
|
||||||
|
const realDir = await fs.promises.realpath(directory);
|
||||||
|
if (!realDir.startsWith(base + path.sep)) return false;
|
||||||
|
const date = worker.extractSafePartitionDate(timestamp);
|
||||||
|
const files = await fs.promises.readdir(realDir, { withFileTypes: true });
|
||||||
|
let scannedBytes = 0;
|
||||||
|
for (const item of files) {
|
||||||
|
if (!item.isFile() || !worker.isPartitionFilename(item.name) ||
|
||||||
|
worker.extractDateFromFilename(item.name) !== date) continue;
|
||||||
|
const filename = path.join(realDir, item.name);
|
||||||
|
scannedBytes += (await fs.promises.stat(filename)).size;
|
||||||
|
if (scannedBytes > 256 * 1024 * 1024) return false;
|
||||||
|
let found = false;
|
||||||
|
const stages = [fs.createReadStream(filename)];
|
||||||
|
if (item.name.endsWith('.gz')) stages.push(zlib.createGunzip());
|
||||||
|
stages.push(parse({ columns: true, bom: true, skip_empty_lines: true }));
|
||||||
|
stages.push(new Writable({
|
||||||
|
objectMode: true,
|
||||||
|
write(row, _encoding, done) {
|
||||||
|
if (row.clientId === clientId && row.eventId === eventId) found = true;
|
||||||
|
done();
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
await pipeline(...stages, { signal: aborter.signal });
|
||||||
|
if (found) {
|
||||||
|
idempotencyMap.set(ackKey, Date.now());
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Missing, corrupt, unreadable or slow CSV cannot justify positive ACK.
|
||||||
|
} finally {
|
||||||
|
clearTimeout(deadline);
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
io.on("connection", (socket) => {
|
io.on("connection", (socket) => {
|
||||||
const clientId = socket.data.clientId;
|
const clientId = socket.data.clientId;
|
||||||
socket.join(`client:${clientId}`);
|
socket.join(`client:${clientId}`);
|
||||||
@@ -551,7 +629,6 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
|
|
||||||
// A previous Redis write failure can leave an uncommitted idempotency
|
// A previous Redis write failure can leave an uncommitted idempotency
|
||||||
// claim. Never report that claim as a successful durable duplicate.
|
// claim. Never report that claim as a successful durable duplicate.
|
||||||
const failedAt = idempotencyMap.get(`${clientId}:${eventId}`);
|
|
||||||
const check = await validateSingleEventAsync(
|
const check = await validateSingleEventAsync(
|
||||||
clientId,
|
clientId,
|
||||||
eventId,
|
eventId,
|
||||||
@@ -560,14 +637,14 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
);
|
);
|
||||||
if (!check.allowed) {
|
if (!check.allowed) {
|
||||||
if (check.duplicate) {
|
if (check.duplicate) {
|
||||||
if (typeof failedAt === "number" && failedAt < 0 &&
|
if (await hasDurableEvent(clientId, eventId, timestamp)) {
|
||||||
Date.now() + failedAt < IDEMPOTENCY_TTL_MS) {
|
idempotencyMap.delete(`\u0000failed-persist:${clientId}:${eventId}`);
|
||||||
if (callback) callback({
|
if (callback) callback({ success: true, eventId, note: "DUPLICATE_DROPPED" });
|
||||||
success: false, eventId, code: "BUFFER_PERSIST_FAILED",
|
|
||||||
error: "BUFFER_PERSIST_FAILED: Event not durably queued",
|
|
||||||
});
|
|
||||||
} else if (callback) {
|
} else if (callback) {
|
||||||
callback({ success: true, eventId, note: "DUPLICATE_DROPPED" });
|
callback({
|
||||||
|
success: false, eventId, code: "BUFFER_PERSIST_FAILED",
|
||||||
|
error: "BUFFER_PERSIST_FAILED: Idempotency claim has no durable event",
|
||||||
|
});
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -597,7 +674,8 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
// Invalidate the pre-persistence idempotency claim so SDK retry can
|
// Invalidate the pre-persistence idempotency claim so SDK retry can
|
||||||
// resubmit after Redis recovers. Keep the volatile RAM event pending.
|
// resubmit after Redis recovers. Keep the volatile RAM event pending.
|
||||||
const claimKey = `${clientId}:${eventId}`;
|
const claimKey = `${clientId}:${eventId}`;
|
||||||
idempotencyMap.set(claimKey, -Date.now());
|
idempotencyMap.delete(claimKey);
|
||||||
|
idempotencyMap.set(`\u0000failed-persist:${claimKey}`, Date.now());
|
||||||
const claimRedis = getRedisClient();
|
const claimRedis = getRedisClient();
|
||||||
if (claimRedis && typeof claimRedis.del === "function") {
|
if (claimRedis && typeof claimRedis.del === "function") {
|
||||||
try {
|
try {
|
||||||
@@ -621,6 +699,9 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Clear any failed local persistence marker only after a durable write.
|
||||||
|
idempotencyMap.delete(`\u0000failed-persist:${clientId}:${eventId}`);
|
||||||
|
idempotencyMap.set(`\u0000durably-acked:${clientId}:${eventId}`, Date.now());
|
||||||
if (callback) callback({ success: true, eventId });
|
if (callback) callback({ success: true, eventId });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (callback) callback({ success: false, error: err.message });
|
if (callback) callback({ success: false, error: err.message });
|
||||||
@@ -726,7 +807,6 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
: new Date().toISOString();
|
: new Date().toISOString();
|
||||||
const rawData = extractBatchItemData(safeItem);
|
const rawData = extractBatchItemData(safeItem);
|
||||||
|
|
||||||
const failedAt = idempotencyMap.get(`${clientId}:${eventId}`);
|
|
||||||
const check = await validateSingleEventAsync(
|
const check = await validateSingleEventAsync(
|
||||||
clientId,
|
clientId,
|
||||||
eventId,
|
eventId,
|
||||||
@@ -735,14 +815,14 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
);
|
);
|
||||||
if (!check.allowed) {
|
if (!check.allowed) {
|
||||||
if (check.duplicate) {
|
if (check.duplicate) {
|
||||||
if (typeof failedAt === "number" && failedAt < 0 &&
|
if (await hasDurableEvent(clientId, eventId, timestamp)) {
|
||||||
Date.now() + failedAt < IDEMPOTENCY_TTL_MS) {
|
idempotencyMap.delete(`\u0000failed-persist:${clientId}:${eventId}`);
|
||||||
|
duplicates += 1;
|
||||||
|
} else {
|
||||||
rejected += 1;
|
rejected += 1;
|
||||||
volatileDuplicate = true;
|
volatileDuplicate = true;
|
||||||
if (errors.length < 5) errors.push(
|
if (errors.length < 5) errors.push(
|
||||||
`${eventId}: BUFFER_PERSIST_FAILED: Event not durably queued`);
|
`${eventId}: BUFFER_PERSIST_FAILED: Idempotency claim has no durable event`);
|
||||||
} else {
|
|
||||||
duplicates += 1;
|
|
||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -779,7 +859,8 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
const claimRedis = getRedisClient();
|
const claimRedis = getRedisClient();
|
||||||
for (const eventId of acceptedEventIds) {
|
for (const eventId of acceptedEventIds) {
|
||||||
const claimKey = `${clientId}:${eventId}`;
|
const claimKey = `${clientId}:${eventId}`;
|
||||||
idempotencyMap.set(claimKey, -Date.now());
|
idempotencyMap.delete(claimKey);
|
||||||
|
idempotencyMap.set(`\u0000failed-persist:${claimKey}`, Date.now());
|
||||||
if (claimRedis && typeof claimRedis.del === "function") {
|
if (claimRedis && typeof claimRedis.del === "function") {
|
||||||
try {
|
try {
|
||||||
await claimRedis.del(`${REDIS_IDEMPOTENCY_PREFIX}${claimKey}`);
|
await claimRedis.del(`${REDIS_IDEMPOTENCY_PREFIX}${claimKey}`);
|
||||||
@@ -806,6 +887,21 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for (const persistedId of acceptedEventIds) {
|
||||||
|
idempotencyMap.delete(`\u0000failed-persist:${clientId}:${persistedId}`);
|
||||||
|
idempotencyMap.set(`\u0000durably-acked:${clientId}:${persistedId}`, Date.now());
|
||||||
|
}
|
||||||
|
// Never ACK a mixed batch as successful while any event remains volatile.
|
||||||
|
if (volatileDuplicate) {
|
||||||
|
if (callback) callback({
|
||||||
|
success: false, processed, rejected, duplicates,
|
||||||
|
code: "BUFFER_PERSIST_FAILED",
|
||||||
|
error: "BUFFER_PERSIST_FAILED: Batch contains non-durable event",
|
||||||
|
errors,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (processed === 0 && duplicates === 0 && rejected > 0) {
|
if (processed === 0 && duplicates === 0 && rejected > 0) {
|
||||||
const code = volatileDuplicate
|
const code = volatileDuplicate
|
||||||
? "BUFFER_PERSIST_FAILED"
|
? "BUFFER_PERSIST_FAILED"
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ import {
|
|||||||
const require = createRequire(import.meta.url);
|
const require = createRequire(import.meta.url);
|
||||||
|
|
||||||
interface IngestionServerModule {
|
interface IngestionServerModule {
|
||||||
|
writer: { baseDir: string };
|
||||||
server: import("node:http").Server;
|
server: import("node:http").Server;
|
||||||
io: import("socket.io").Server;
|
io: import("socket.io").Server;
|
||||||
bufferQueue: Array<{ clientId: string; eventId: string; timestamp: string; payload: string }>;
|
bufferQueue: Array<{ clientId: string; eventId: string; timestamp: string; payload: string }>;
|
||||||
@@ -1221,7 +1222,7 @@ test("R2 PHASE-02 BUG-10: GET, PATCH (regenerate_token), and DELETE /api/clients
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
test("P02 R-01 diagnostic: Redis RPUSH failure yields positive WS ACK for volatile RAM-only event", async () => {
|
test("P02 R-01 remediation: Redis RPUSH failure rejects non-durable WS ACK", async () => {
|
||||||
// Disposable test-only fixture: no production Redis, service, or storage state.
|
// Disposable test-only fixture: no production Redis, service, or storage state.
|
||||||
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
|
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
|
||||||
const previousRam = serverMod.bufferQueue.slice();
|
const previousRam = serverMod.bufferQueue.slice();
|
||||||
@@ -1255,6 +1256,12 @@ test("P02 R-01 diagnostic: Redis RPUSH failure yields positive WS ACK for volati
|
|||||||
assert.equal(sameIdRetry.success, false,
|
assert.equal(sameIdRetry.success, false,
|
||||||
"Pending idempotency claims must not return false positive duplicate ACK");
|
"Pending idempotency claims must not return false positive duplicate ACK");
|
||||||
assert.equal(sameIdRetry.code, "BUFFER_PERSIST_FAILED");
|
assert.equal(sameIdRetry.code, "BUFFER_PERSIST_FAILED");
|
||||||
|
const secondRetry = await emitWithAck<{ success: boolean; code?: string }>(
|
||||||
|
socket, "sync_data",
|
||||||
|
{ eventId: "p02-ack-before-crash-001", data: { value: 42 } },
|
||||||
|
);
|
||||||
|
assert.equal(secondRetry.success, false, "A repeated pending claim must remain negative");
|
||||||
|
assert.equal(secondRetry.code, "BUFFER_PERSIST_FAILED");
|
||||||
assert.equal(committedRows.length, 0, "Fake Redis has no durable event");
|
assert.equal(committedRows.length, 0, "Fake Redis has no durable event");
|
||||||
assert.equal(volatileRows.length, 1, "Acknowledged event is RAM-only");
|
assert.equal(volatileRows.length, 1, "Acknowledged event is RAM-only");
|
||||||
// Crash model: restart loses RAM but fake Redis survives. No actual process is killed.
|
// Crash model: restart loses RAM but fake Redis survives. No actual process is killed.
|
||||||
@@ -1268,7 +1275,12 @@ test("P02 R-01 diagnostic: Redis RPUSH failure yields positive WS ACK for volati
|
|||||||
{ eventId: "p02-ack-recovered-002", data: { value: 43 } },
|
{ eventId: "p02-ack-recovered-002", data: { value: 43 } },
|
||||||
);
|
);
|
||||||
assert.equal(recovered.success, true);
|
assert.equal(recovered.success, true);
|
||||||
assert.equal((await healthyRedis.lrange("datastack:buffer:events", 0, -1)).length, 1);
|
const recoveredSameId = await emitWithAck<{ success: boolean; code?: string }>(
|
||||||
|
socket, "sync_data",
|
||||||
|
{ eventId: "p02-ack-before-crash-001", data: { value: 42 } },
|
||||||
|
);
|
||||||
|
assert.equal(recoveredSameId.success, true, "Original eventId must be retryable after Redis recovery");
|
||||||
|
assert.equal((await healthyRedis.lrange("datastack:buffer:events", 0, -1)).length, 2);
|
||||||
} finally {
|
} finally {
|
||||||
socket?.disconnect();
|
socket?.disconnect();
|
||||||
serverMod.setRedisClientForTests(undefined);
|
serverMod.setRedisClientForTests(undefined);
|
||||||
@@ -1303,6 +1315,7 @@ test("P02 R-01: batch_sync rejects non-durable Redis writes and pending duplicat
|
|||||||
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
|
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
|
||||||
const previousQueue = serverMod.bufferQueue.slice();
|
const previousQueue = serverMod.bufferQueue.slice();
|
||||||
const failingRedis = createSharedMockRedis();
|
const failingRedis = createSharedMockRedis();
|
||||||
|
const recoveredRpush = failingRedis.rpush.bind(failingRedis);
|
||||||
failingRedis.rpush = async () => { throw new Error("P02 injected batch Redis outage"); };
|
failingRedis.rpush = async () => { throw new Error("P02 injected batch Redis outage"); };
|
||||||
serverMod.setRedisClientForTests(failingRedis);
|
serverMod.setRedisClientForTests(failingRedis);
|
||||||
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
|
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
|
||||||
@@ -1331,6 +1344,18 @@ test("P02 R-01: batch_sync rejects non-durable Redis writes and pending duplicat
|
|||||||
);
|
);
|
||||||
assert.equal(retry.success, false);
|
assert.equal(retry.success, false);
|
||||||
assert.equal(retry.code, "BUFFER_PERSIST_FAILED");
|
assert.equal(retry.code, "BUFFER_PERSIST_FAILED");
|
||||||
|
failingRedis.rpush = recoveredRpush;
|
||||||
|
const mixed = await emitWithAck<{ success: boolean; code?: string; processed?: number }>(
|
||||||
|
socket, "batch_sync", [payload[0], { eventId: "p02-batch-recovered-3", data: { value: 3 } }],
|
||||||
|
);
|
||||||
|
assert.equal(mixed.success, false, "A mixed recovered/volatile batch cannot ACK success");
|
||||||
|
assert.equal(mixed.code, "BUFFER_PERSIST_FAILED");
|
||||||
|
assert.equal(mixed.processed, 1, "New item is durably queued, old idempotency claim is uncertain");
|
||||||
|
const confirmed = await emitWithAck<{ success: boolean; duplicates?: number }>(
|
||||||
|
socket, "batch_sync", payload,
|
||||||
|
);
|
||||||
|
assert.equal(confirmed.success, true, "Once old events are persisted, duplicates are safely acknowledged");
|
||||||
|
assert.equal(confirmed.duplicates, 2);
|
||||||
} finally {
|
} finally {
|
||||||
socket?.disconnect();
|
socket?.disconnect();
|
||||||
serverMod.setRedisClientForTests(undefined);
|
serverMod.setRedisClientForTests(undefined);
|
||||||
@@ -1338,3 +1363,53 @@ test("P02 R-01: batch_sync rejects non-durable Redis writes and pending duplicat
|
|||||||
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
|
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("P02 R-01: Redis-free Socket.IO ACK waits for CSV durability and denies disk errors", async () => {
|
||||||
|
const fs = await import("node:fs");
|
||||||
|
const os = await import("node:os");
|
||||||
|
const path = await import("node:path");
|
||||||
|
const baseDir = fs.mkdtempSync(path.join(os.tmpdir(), "datastack-p02-ack-disk-"));
|
||||||
|
const previousDir = serverMod.writer.baseDir;
|
||||||
|
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
|
||||||
|
const previousQueue = serverMod.bufferQueue.slice();
|
||||||
|
serverMod.writer.baseDir = baseDir;
|
||||||
|
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
|
||||||
|
serverMod.setRedisClientForTests(null);
|
||||||
|
serverMod.bufferQueue.length = 0;
|
||||||
|
let socket: Socket | undefined;
|
||||||
|
try {
|
||||||
|
const client = await SDKManager.registerClientAsync({
|
||||||
|
clientId: "p02_disk_durability", name: "P02 Disk Durability", rateLimit: 100,
|
||||||
|
});
|
||||||
|
socket = await connectSocket(client.token);
|
||||||
|
const accepted = await emitWithAck<{ success: boolean; code?: string }>(
|
||||||
|
socket, "sync_data", { eventId: "p02-disk-ack-001", data: { value: 11 } },
|
||||||
|
);
|
||||||
|
assert.equal(accepted.success, true);
|
||||||
|
const csvPath = path.join(baseDir, client.clientId, new Date().toISOString().slice(0, 10) + ".csv");
|
||||||
|
assert.equal(fs.readFileSync(csvPath, "utf8").includes("p02-disk-ack-001"), true,
|
||||||
|
"A successful ACK requires a completed CSV append when Redis is absent");
|
||||||
|
const blockedBase = path.join(baseDir, "not-a-directory");
|
||||||
|
fs.writeFileSync(blockedBase, "fixture");
|
||||||
|
serverMod.writer.baseDir = blockedBase;
|
||||||
|
const failed = await emitWithAck<{ success: boolean; code?: string }>(
|
||||||
|
socket, "sync_data", { eventId: "p02-disk-ack-002", data: { value: 12 } },
|
||||||
|
);
|
||||||
|
assert.equal(failed.success, false, "CSV failure must never receive a positive ACK");
|
||||||
|
assert.equal(failed.code, "BUFFER_PERSIST_FAILED");
|
||||||
|
serverMod.writer.baseDir = baseDir;
|
||||||
|
const retry = await emitWithAck<{ success: boolean; code?: string }>(
|
||||||
|
socket, "sync_data", { eventId: "p02-disk-ack-002", data: { value: 12 } },
|
||||||
|
);
|
||||||
|
assert.equal(retry.success, true, "The same eventId must recover after disk restoration");
|
||||||
|
const csv = fs.readFileSync(csvPath, "utf8");
|
||||||
|
assert.equal(csv.split("p02-disk-ack-002").length - 1, 1);
|
||||||
|
} finally {
|
||||||
|
socket?.disconnect();
|
||||||
|
serverMod.setRedisClientForTests(undefined);
|
||||||
|
serverMod.writer.baseDir = previousDir;
|
||||||
|
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
|
||||||
|
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
|
||||||
|
fs.rmSync(baseDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
@@ -1629,10 +1629,18 @@ test("P02 R-02: Port 4000 CSV worker suppresses a crash-window replay after Redi
|
|||||||
assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 1);
|
assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 1);
|
||||||
const csvPath = path.join(tmpStorageDir, evt.clientId, "2026-11-22.csv");
|
const csvPath = path.join(tmpStorageDir, evt.clientId, "2026-11-22.csv");
|
||||||
assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1);
|
assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1);
|
||||||
|
// Crash/restart may occur after daily part rotation and archive compression.
|
||||||
|
// Verify replay detection against an actual rotated .csv.gz partition.
|
||||||
|
const zlib = await import("node:zlib");
|
||||||
|
const archivedPath = path.join(tmpStorageDir, evt.clientId, "2026-11-22-part2.csv.gz");
|
||||||
|
fs.writeFileSync(archivedPath, zlib.gzipSync(fs.readFileSync(csvPath)));
|
||||||
|
fs.unlinkSync(csvPath);
|
||||||
const recovered = await ingestion.flushBuffer();
|
const recovered = await ingestion.flushBuffer();
|
||||||
assert.equal(recovered.written, 0, "Recovered replay should not append the event again");
|
assert.equal(recovered.written, 0, "Recovered compressed replay must not append again");
|
||||||
assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 0);
|
assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 0);
|
||||||
assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1);
|
assert.equal(fs.existsSync(csvPath), false, "No duplicate active CSV partition should be created");
|
||||||
|
const archivedRows = zlib.gunzipSync(fs.readFileSync(archivedPath)).toString("utf8");
|
||||||
|
assert.equal(archivedRows.split(evt.eventId).length - 1, 1);
|
||||||
} finally {
|
} finally {
|
||||||
ingestion.writer.baseDir = previousDir;
|
ingestion.writer.baseDir = previousDir;
|
||||||
ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...previousQueue);
|
ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...previousQueue);
|
||||||
|
|||||||
@@ -776,3 +776,60 @@ test("R4-Phase-02 AUDIT-12 / ERR-R4-08: Streaming S3 upload & streaming .csv.gz
|
|||||||
assert.equal(fs.existsSync(tmpDir), false);
|
assert.equal(fs.existsSync(tmpDir), false);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("P02 R-01: stale pre-persistence Redis claim after restart never yields a false duplicate ACK", async () => {
|
||||||
|
// A fresh Socket.IO handler has no in-process ACK cache, but Redis retained
|
||||||
|
// the idempotency claim from a hypothetical crash before Redis RPUSH.
|
||||||
|
const client = await SDKManager.registerClientAsync({
|
||||||
|
clientId: "p02_stale_claim", name: "P02 Stale Claim", rateLimit: 200,
|
||||||
|
});
|
||||||
|
const eventId = "p02-not-yet-persisted-001";
|
||||||
|
const stored = [] as string[];
|
||||||
|
const redis = {
|
||||||
|
status: "ready",
|
||||||
|
async get(key: string) {
|
||||||
|
return key === `datastack:idem:${client.clientId}:${eventId}` ? "1" : null;
|
||||||
|
},
|
||||||
|
async set() { return null; },
|
||||||
|
async incr() { return 1; },
|
||||||
|
async pexpire() { return 1; },
|
||||||
|
async lrange() { return stored; },
|
||||||
|
};
|
||||||
|
const httpServer = http.createServer();
|
||||||
|
const io = new Server(httpServer);
|
||||||
|
setupSocketHandlers(io, [], null, {
|
||||||
|
getRuntimeConfig: () => ({
|
||||||
|
jwtSecret: "datastack-jwt-default-secret-key-32-chars-long",
|
||||||
|
maxEventsPerSec: 200, maxPayloadSizeBytes: 100 * 1024, enableIdempotency: true,
|
||||||
|
}),
|
||||||
|
verifyClientInDb: async () => ({ allowed: true, rateLimit: 200 }),
|
||||||
|
getRedisClient: () => redis,
|
||||||
|
onBufferPush: async () => undefined,
|
||||||
|
});
|
||||||
|
let socket: ReturnType<typeof ioClient> | null = null;
|
||||||
|
try {
|
||||||
|
await new Promise<void>((resolve) => httpServer.listen(0, "127.0.0.1", resolve));
|
||||||
|
const port = (httpServer.address() as AddressInfo).port;
|
||||||
|
socket = ioClient(`http://127.0.0.1:${port}`, { auth: { token: client.token }, transports: ["websocket"] });
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
socket!.once("connect", resolve);
|
||||||
|
socket!.once("connect_error", reject);
|
||||||
|
});
|
||||||
|
const sendDuplicate = () => new Promise<{ success: boolean; code?: string; note?: string }>((resolve, reject) => {
|
||||||
|
socket!.timeout(3000).emit("sync_data", { eventId, data: { seq: 1 } },
|
||||||
|
(err: Error | null, ack: { success: boolean; code?: string; note?: string }) => {
|
||||||
|
if (err) reject(err); else resolve(ack);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
const uncertain = await sendDuplicate();
|
||||||
|
assert.equal(uncertain.success, false, "Redis idempotency claim alone is not a persistence proof");
|
||||||
|
assert.equal(uncertain.code, "BUFFER_PERSIST_FAILED");
|
||||||
|
stored.push(JSON.stringify({ clientId: client.clientId, eventId }));
|
||||||
|
const durable = await sendDuplicate();
|
||||||
|
assert.equal(durable.success, true, "Same claim is a valid duplicate only after its event is durable");
|
||||||
|
assert.equal(durable.note, "DUPLICATE_DROPPED");
|
||||||
|
} finally {
|
||||||
|
socket?.disconnect();
|
||||||
|
await new Promise<void>((resolve) => io.close(() => resolve()));
|
||||||
|
}
|
||||||
|
});
|
||||||
Reference in new issue
Block a user