fix(p02): fail closed on uncertain ACKs and verify restart durability [skip ci]
This commit is contained in:
1 parent
76cd972bde
commit
b0835fd43b
5 files changed
+275
-23
No files matched your search
@@ -36,6 +36,7 @@ import {
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
interface IngestionServerModule {
|
||||
writer: { baseDir: string };
|
||||
server: import("node:http").Server;
|
||||
io: import("socket.io").Server;
|
||||
bufferQueue: Array<{ clientId: string; eventId: string; timestamp: string; payload: string }>;
|
||||
@@ -1221,7 +1222,7 @@ test("R2 PHASE-02 BUG-10: GET, PATCH (regenerate_token), and DELETE /api/clients
|
||||
}
|
||||
});
|
||||
|
||||
test("P02 R-01 diagnostic: Redis RPUSH failure yields positive WS ACK for volatile RAM-only event", async () => {
|
||||
test("P02 R-01 remediation: Redis RPUSH failure rejects non-durable WS ACK", async () => {
|
||||
// Disposable test-only fixture: no production Redis, service, or storage state.
|
||||
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
|
||||
const previousRam = serverMod.bufferQueue.slice();
|
||||
@@ -1255,6 +1256,12 @@ test("P02 R-01 diagnostic: Redis RPUSH failure yields positive WS ACK for volati
|
||||
assert.equal(sameIdRetry.success, false,
|
||||
"Pending idempotency claims must not return false positive duplicate ACK");
|
||||
assert.equal(sameIdRetry.code, "BUFFER_PERSIST_FAILED");
|
||||
const secondRetry = await emitWithAck<{ success: boolean; code?: string }>(
|
||||
socket, "sync_data",
|
||||
{ eventId: "p02-ack-before-crash-001", data: { value: 42 } },
|
||||
);
|
||||
assert.equal(secondRetry.success, false, "A repeated pending claim must remain negative");
|
||||
assert.equal(secondRetry.code, "BUFFER_PERSIST_FAILED");
|
||||
assert.equal(committedRows.length, 0, "Fake Redis has no durable event");
|
||||
assert.equal(volatileRows.length, 1, "Acknowledged event is RAM-only");
|
||||
// Crash model: restart loses RAM but fake Redis survives. No actual process is killed.
|
||||
@@ -1268,7 +1275,12 @@ test("P02 R-01 diagnostic: Redis RPUSH failure yields positive WS ACK for volati
|
||||
{ eventId: "p02-ack-recovered-002", data: { value: 43 } },
|
||||
);
|
||||
assert.equal(recovered.success, true);
|
||||
assert.equal((await healthyRedis.lrange("datastack:buffer:events", 0, -1)).length, 1);
|
||||
const recoveredSameId = await emitWithAck<{ success: boolean; code?: string }>(
|
||||
socket, "sync_data",
|
||||
{ eventId: "p02-ack-before-crash-001", data: { value: 42 } },
|
||||
);
|
||||
assert.equal(recoveredSameId.success, true, "Original eventId must be retryable after Redis recovery");
|
||||
assert.equal((await healthyRedis.lrange("datastack:buffer:events", 0, -1)).length, 2);
|
||||
} finally {
|
||||
socket?.disconnect();
|
||||
serverMod.setRedisClientForTests(undefined);
|
||||
@@ -1303,6 +1315,7 @@ test("P02 R-01: batch_sync rejects non-durable Redis writes and pending duplicat
|
||||
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
|
||||
const previousQueue = serverMod.bufferQueue.slice();
|
||||
const failingRedis = createSharedMockRedis();
|
||||
const recoveredRpush = failingRedis.rpush.bind(failingRedis);
|
||||
failingRedis.rpush = async () => { throw new Error("P02 injected batch Redis outage"); };
|
||||
serverMod.setRedisClientForTests(failingRedis);
|
||||
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
|
||||
@@ -1331,6 +1344,18 @@ test("P02 R-01: batch_sync rejects non-durable Redis writes and pending duplicat
|
||||
);
|
||||
assert.equal(retry.success, false);
|
||||
assert.equal(retry.code, "BUFFER_PERSIST_FAILED");
|
||||
failingRedis.rpush = recoveredRpush;
|
||||
const mixed = await emitWithAck<{ success: boolean; code?: string; processed?: number }>(
|
||||
socket, "batch_sync", [payload[0], { eventId: "p02-batch-recovered-3", data: { value: 3 } }],
|
||||
);
|
||||
assert.equal(mixed.success, false, "A mixed recovered/volatile batch cannot ACK success");
|
||||
assert.equal(mixed.code, "BUFFER_PERSIST_FAILED");
|
||||
assert.equal(mixed.processed, 1, "New item is durably queued, old idempotency claim is uncertain");
|
||||
const confirmed = await emitWithAck<{ success: boolean; duplicates?: number }>(
|
||||
socket, "batch_sync", payload,
|
||||
);
|
||||
assert.equal(confirmed.success, true, "Once old events are persisted, duplicates are safely acknowledged");
|
||||
assert.equal(confirmed.duplicates, 2);
|
||||
} finally {
|
||||
socket?.disconnect();
|
||||
serverMod.setRedisClientForTests(undefined);
|
||||
@@ -1338,3 +1363,53 @@ test("P02 R-01: batch_sync rejects non-durable Redis writes and pending duplicat
|
||||
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
|
||||
}
|
||||
});
|
||||
|
||||
test("P02 R-01: Redis-free Socket.IO ACK waits for CSV durability and denies disk errors", async () => {
|
||||
const fs = await import("node:fs");
|
||||
const os = await import("node:os");
|
||||
const path = await import("node:path");
|
||||
const baseDir = fs.mkdtempSync(path.join(os.tmpdir(), "datastack-p02-ack-disk-"));
|
||||
const previousDir = serverMod.writer.baseDir;
|
||||
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
|
||||
const previousQueue = serverMod.bufferQueue.slice();
|
||||
serverMod.writer.baseDir = baseDir;
|
||||
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
|
||||
serverMod.setRedisClientForTests(null);
|
||||
serverMod.bufferQueue.length = 0;
|
||||
let socket: Socket | undefined;
|
||||
try {
|
||||
const client = await SDKManager.registerClientAsync({
|
||||
clientId: "p02_disk_durability", name: "P02 Disk Durability", rateLimit: 100,
|
||||
});
|
||||
socket = await connectSocket(client.token);
|
||||
const accepted = await emitWithAck<{ success: boolean; code?: string }>(
|
||||
socket, "sync_data", { eventId: "p02-disk-ack-001", data: { value: 11 } },
|
||||
);
|
||||
assert.equal(accepted.success, true);
|
||||
const csvPath = path.join(baseDir, client.clientId, new Date().toISOString().slice(0, 10) + ".csv");
|
||||
assert.equal(fs.readFileSync(csvPath, "utf8").includes("p02-disk-ack-001"), true,
|
||||
"A successful ACK requires a completed CSV append when Redis is absent");
|
||||
const blockedBase = path.join(baseDir, "not-a-directory");
|
||||
fs.writeFileSync(blockedBase, "fixture");
|
||||
serverMod.writer.baseDir = blockedBase;
|
||||
const failed = await emitWithAck<{ success: boolean; code?: string }>(
|
||||
socket, "sync_data", { eventId: "p02-disk-ack-002", data: { value: 12 } },
|
||||
);
|
||||
assert.equal(failed.success, false, "CSV failure must never receive a positive ACK");
|
||||
assert.equal(failed.code, "BUFFER_PERSIST_FAILED");
|
||||
serverMod.writer.baseDir = baseDir;
|
||||
const retry = await emitWithAck<{ success: boolean; code?: string }>(
|
||||
socket, "sync_data", { eventId: "p02-disk-ack-002", data: { value: 12 } },
|
||||
);
|
||||
assert.equal(retry.success, true, "The same eventId must recover after disk restoration");
|
||||
const csv = fs.readFileSync(csvPath, "utf8");
|
||||
assert.equal(csv.split("p02-disk-ack-002").length - 1, 1);
|
||||
} finally {
|
||||
socket?.disconnect();
|
||||
serverMod.setRedisClientForTests(undefined);
|
||||
serverMod.writer.baseDir = previousDir;
|
||||
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
|
||||
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
|
||||
fs.rmSync(baseDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -1629,10 +1629,18 @@ test("P02 R-02: Port 4000 CSV worker suppresses a crash-window replay after Redi
|
||||
assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 1);
|
||||
const csvPath = path.join(tmpStorageDir, evt.clientId, "2026-11-22.csv");
|
||||
assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1);
|
||||
// Crash/restart may occur after daily part rotation and archive compression.
|
||||
// Verify replay detection against an actual rotated .csv.gz partition.
|
||||
const zlib = await import("node:zlib");
|
||||
const archivedPath = path.join(tmpStorageDir, evt.clientId, "2026-11-22-part2.csv.gz");
|
||||
fs.writeFileSync(archivedPath, zlib.gzipSync(fs.readFileSync(csvPath)));
|
||||
fs.unlinkSync(csvPath);
|
||||
const recovered = await ingestion.flushBuffer();
|
||||
assert.equal(recovered.written, 0, "Recovered replay should not append the event again");
|
||||
assert.equal(recovered.written, 0, "Recovered compressed replay must not append again");
|
||||
assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 0);
|
||||
assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1);
|
||||
assert.equal(fs.existsSync(csvPath), false, "No duplicate active CSV partition should be created");
|
||||
const archivedRows = zlib.gunzipSync(fs.readFileSync(archivedPath)).toString("utf8");
|
||||
assert.equal(archivedRows.split(evt.eventId).length - 1, 1);
|
||||
} finally {
|
||||
ingestion.writer.baseDir = previousDir;
|
||||
ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...previousQueue);
|
||||
|
||||
@@ -776,3 +776,60 @@ test("R4-Phase-02 AUDIT-12 / ERR-R4-08: Streaming S3 upload & streaming .csv.gz
|
||||
assert.equal(fs.existsSync(tmpDir), false);
|
||||
}
|
||||
});
|
||||
|
||||
test("P02 R-01: stale pre-persistence Redis claim after restart never yields a false duplicate ACK", async () => {
|
||||
// A fresh Socket.IO handler has no in-process ACK cache, but Redis retained
|
||||
// the idempotency claim from a hypothetical crash before Redis RPUSH.
|
||||
const client = await SDKManager.registerClientAsync({
|
||||
clientId: "p02_stale_claim", name: "P02 Stale Claim", rateLimit: 200,
|
||||
});
|
||||
const eventId = "p02-not-yet-persisted-001";
|
||||
const stored = [] as string[];
|
||||
const redis = {
|
||||
status: "ready",
|
||||
async get(key: string) {
|
||||
return key === `datastack:idem:${client.clientId}:${eventId}` ? "1" : null;
|
||||
},
|
||||
async set() { return null; },
|
||||
async incr() { return 1; },
|
||||
async pexpire() { return 1; },
|
||||
async lrange() { return stored; },
|
||||
};
|
||||
const httpServer = http.createServer();
|
||||
const io = new Server(httpServer);
|
||||
setupSocketHandlers(io, [], null, {
|
||||
getRuntimeConfig: () => ({
|
||||
jwtSecret: "datastack-jwt-default-secret-key-32-chars-long",
|
||||
maxEventsPerSec: 200, maxPayloadSizeBytes: 100 * 1024, enableIdempotency: true,
|
||||
}),
|
||||
verifyClientInDb: async () => ({ allowed: true, rateLimit: 200 }),
|
||||
getRedisClient: () => redis,
|
||||
onBufferPush: async () => undefined,
|
||||
});
|
||||
let socket: ReturnType<typeof ioClient> | null = null;
|
||||
try {
|
||||
await new Promise<void>((resolve) => httpServer.listen(0, "127.0.0.1", resolve));
|
||||
const port = (httpServer.address() as AddressInfo).port;
|
||||
socket = ioClient(`http://127.0.0.1:${port}`, { auth: { token: client.token }, transports: ["websocket"] });
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
socket!.once("connect", resolve);
|
||||
socket!.once("connect_error", reject);
|
||||
});
|
||||
const sendDuplicate = () => new Promise<{ success: boolean; code?: string; note?: string }>((resolve, reject) => {
|
||||
socket!.timeout(3000).emit("sync_data", { eventId, data: { seq: 1 } },
|
||||
(err: Error | null, ack: { success: boolean; code?: string; note?: string }) => {
|
||||
if (err) reject(err); else resolve(ack);
|
||||
});
|
||||
});
|
||||
const uncertain = await sendDuplicate();
|
||||
assert.equal(uncertain.success, false, "Redis idempotency claim alone is not a persistence proof");
|
||||
assert.equal(uncertain.code, "BUFFER_PERSIST_FAILED");
|
||||
stored.push(JSON.stringify({ clientId: client.clientId, eventId }));
|
||||
const durable = await sendDuplicate();
|
||||
assert.equal(durable.success, true, "Same claim is a valid duplicate only after its event is durable");
|
||||
assert.equal(durable.note, "DUPLICATE_DROPPED");
|
||||
} finally {
|
||||
socket?.disconnect();
|
||||
await new Promise<void>((resolve) => io.close(() => resolve()));
|
||||
}
|
||||
});
|
||||
Reference in new issue
Block a user