fix(p02): guard Redis ACK and dedupe CSV crash-window replays [skip ci]
This commit is contained in:
1 parent
fd57a3518d
commit
76cd972bde
11 files changed
+388
-22
No files matched your search
@@ -1246,12 +1246,29 @@ test("P02 R-01 diagnostic: Redis RPUSH failure yields positive WS ACK for volati
|
||||
);
|
||||
const committedRows = await failingRedis.lrange("datastack:buffer:events", 0, -1);
|
||||
const volatileRows = serverMod.bufferQueue.filter((e) => e.eventId === "p02-ack-before-crash-001");
|
||||
assert.equal(ack.success, true, "Existing handler currently sends a positive ACK after Redis failure");
|
||||
assert.equal(ack.success, false, "P02 remediation must reject non-durable ACK");
|
||||
assert.equal(ack.code, "BUFFER_PERSIST_FAILED");
|
||||
const sameIdRetry = await emitWithAck<{ success: boolean; code?: string }>(
|
||||
socket, "sync_data",
|
||||
{ eventId: "p02-ack-before-crash-001", data: { value: 42 } },
|
||||
);
|
||||
assert.equal(sameIdRetry.success, false,
|
||||
"Pending idempotency claims must not return false positive duplicate ACK");
|
||||
assert.equal(sameIdRetry.code, "BUFFER_PERSIST_FAILED");
|
||||
assert.equal(committedRows.length, 0, "Fake Redis has no durable event");
|
||||
assert.equal(volatileRows.length, 1, "Acknowledged event is RAM-only");
|
||||
// Crash model: restart loses RAM but fake Redis survives. No actual process is killed.
|
||||
serverMod.bufferQueue.length = 0;
|
||||
assert.equal((await failingRedis.lrange("datastack:buffer:events", 0, -1)).length, 0);
|
||||
// A new event may ACK only after Redis actually resumes persistence.
|
||||
const healthyRedis = createSharedMockRedis();
|
||||
serverMod.setRedisClientForTests(healthyRedis);
|
||||
const recovered = await emitWithAck<{ success: boolean; eventId?: string }>(
|
||||
socket, "sync_data",
|
||||
{ eventId: "p02-ack-recovered-002", data: { value: 43 } },
|
||||
);
|
||||
assert.equal(recovered.success, true);
|
||||
assert.equal((await healthyRedis.lrange("datastack:buffer:events", 0, -1)).length, 1);
|
||||
} finally {
|
||||
socket?.disconnect();
|
||||
serverMod.setRedisClientForTests(undefined);
|
||||
@@ -1281,3 +1298,43 @@ test("P02 R-03 diagnostic: Redis query failure leaves public ingestion health 20
|
||||
serverMod.setRedisClientForTests(undefined);
|
||||
}
|
||||
});
|
||||
|
||||
test("P02 R-01: batch_sync rejects non-durable Redis writes and pending duplicate claims", async () => {
|
||||
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
|
||||
const previousQueue = serverMod.bufferQueue.slice();
|
||||
const failingRedis = createSharedMockRedis();
|
||||
failingRedis.rpush = async () => { throw new Error("P02 injected batch Redis outage"); };
|
||||
serverMod.setRedisClientForTests(failingRedis);
|
||||
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
|
||||
serverMod.bufferQueue.length = 0;
|
||||
let socket: Socket | undefined;
|
||||
const payload = [
|
||||
{ eventId: "p02-batch-volatile-1", data: { value: 1 } },
|
||||
{ eventId: "p02-batch-volatile-2", data: { value: 2 } },
|
||||
];
|
||||
try {
|
||||
const client = await SDKManager.registerClientAsync({
|
||||
clientId: "p02_redis_batch_failure",
|
||||
name: "P02 Redis Batch Failure",
|
||||
rateLimit: 100,
|
||||
});
|
||||
socket = await connectSocket(client.token);
|
||||
const ack = await emitWithAck<{ success: boolean; code?: string; processed?: number }>(
|
||||
socket, "batch_sync", payload,
|
||||
);
|
||||
assert.equal(ack.success, false);
|
||||
assert.equal(ack.code, "BUFFER_PERSIST_FAILED");
|
||||
assert.equal(ack.processed, 0);
|
||||
assert.equal((await failingRedis.lrange("datastack:buffer:events", 0, -1)).length, 0);
|
||||
const retry = await emitWithAck<{ success: boolean; code?: string }>(
|
||||
socket, "batch_sync", payload,
|
||||
);
|
||||
assert.equal(retry.success, false);
|
||||
assert.equal(retry.code, "BUFFER_PERSIST_FAILED");
|
||||
} finally {
|
||||
socket?.disconnect();
|
||||
serverMod.setRedisClientForTests(undefined);
|
||||
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
|
||||
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
|
||||
}
|
||||
});
|
||||
@@ -1584,10 +1584,58 @@ test("P02 R-02 diagnostic: post-CSV Redis trim failure replays identical event o
|
||||
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 1, "Redis still queues previously appended CSV event");
|
||||
await DataBufferEngine.flush();
|
||||
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0);
|
||||
assert.equal(writtenIds.filter((id) => id === event.eventId).length, 2,
|
||||
"A second flush appends the same eventId again after failed trim: at-least-once, not exactly-once");
|
||||
assert.equal(writtenIds.filter((id) => id === event.eventId).length, 1,
|
||||
"Replay must detect the persisted CSV eventId before attempting another append");
|
||||
const csvPath = path.join(tmpStorageDir, event.clientId, "2026-11-22.csv");
|
||||
const csvContents = fs.readFileSync(csvPath, "utf8");
|
||||
assert.equal(csvContents.split(event.eventId).length - 1, 1,
|
||||
"Exactly one physical CSV row must exist after a failed LTRIM and retry");
|
||||
} finally {
|
||||
CSVStorageManager.appendBatch = originalAppend;
|
||||
DataBufferEngine.setRedisClientForTests(undefined);
|
||||
}
|
||||
});
|
||||
|
||||
test("P02 R-02: Port 4000 CSV worker suppresses a crash-window replay after Redis LTRIM error", async () => {
|
||||
const mockRedis = new FakeRedisClient();
|
||||
const originalTrim = mockRedis.ltrim.bind(mockRedis);
|
||||
let failedOnce = false;
|
||||
mockRedis.ltrim = async (key: string, start: number, stop: number): Promise<"OK"> => {
|
||||
if (key === REDIS_BUFFER_KEY && !failedOnce) {
|
||||
failedOnce = true;
|
||||
throw new Error("P02 Port4000 injected post-CSV trim failure");
|
||||
}
|
||||
return originalTrim(key, start, stop);
|
||||
};
|
||||
const ingestion = require("../server/index.js") as IngestionServerModule;
|
||||
const previousDir = ingestion.writer.baseDir;
|
||||
const previousQueue = ingestion.bufferQueue.slice();
|
||||
ingestion.writer.baseDir = tmpStorageDir;
|
||||
ingestion.bufferQueue.length = 0;
|
||||
ingestion.setRedisClientForTests(mockRedis);
|
||||
const evt: DataEvent = {
|
||||
clientId: "p02_ingestion_replay",
|
||||
eventId: "p02-port4000-001",
|
||||
timestamp: "2026-11-22T13:00:00.000Z",
|
||||
payload: JSON.stringify({ stage: "before-ltrim" }),
|
||||
};
|
||||
try {
|
||||
await mockRedis.rpush(REDIS_BUFFER_KEY, JSON.stringify(evt));
|
||||
await assert.rejects(
|
||||
() => ingestion.flushBuffer(),
|
||||
/P02 Port4000 injected post-CSV trim failure/,
|
||||
);
|
||||
assert.equal(failedOnce, true);
|
||||
assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 1);
|
||||
const csvPath = path.join(tmpStorageDir, evt.clientId, "2026-11-22.csv");
|
||||
assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1);
|
||||
const recovered = await ingestion.flushBuffer();
|
||||
assert.equal(recovered.written, 0, "Recovered replay should not append the event again");
|
||||
assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 0);
|
||||
assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1);
|
||||
} finally {
|
||||
ingestion.writer.baseDir = previousDir;
|
||||
ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...previousQueue);
|
||||
ingestion.setRedisClientForTests(undefined);
|
||||
}
|
||||
});
|
||||
Reference in new issue
Block a user