test(p02): guard stale Redis claims and verify CSV replay edge cases [skip ci]
This commit is contained in:
1 parent
76cd972bde
commit
bf525f0d4e
3 files changed
+211
-8
No files matched your search
@@ -551,7 +551,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
|
|
||||||
// A previous Redis write failure can leave an uncommitted idempotency
|
// A previous Redis write failure can leave an uncommitted idempotency
|
||||||
// claim. Never report that claim as a successful durable duplicate.
|
// claim. Never report that claim as a successful durable duplicate.
|
||||||
const failedAt = idempotencyMap.get(`${clientId}:${eventId}`);
|
const failedAt = idempotencyMap.get(`\u0000failed-persist:${clientId}:${eventId}`);
|
||||||
const check = await validateSingleEventAsync(
|
const check = await validateSingleEventAsync(
|
||||||
clientId,
|
clientId,
|
||||||
eventId,
|
eventId,
|
||||||
@@ -560,8 +560,8 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
);
|
);
|
||||||
if (!check.allowed) {
|
if (!check.allowed) {
|
||||||
if (check.duplicate) {
|
if (check.duplicate) {
|
||||||
if (typeof failedAt === "number" && failedAt < 0 &&
|
if (typeof failedAt === "number" &&
|
||||||
Date.now() + failedAt < IDEMPOTENCY_TTL_MS) {
|
Date.now() - failedAt < IDEMPOTENCY_TTL_MS) {
|
||||||
if (callback) callback({
|
if (callback) callback({
|
||||||
success: false, eventId, code: "BUFFER_PERSIST_FAILED",
|
success: false, eventId, code: "BUFFER_PERSIST_FAILED",
|
||||||
error: "BUFFER_PERSIST_FAILED: Event not durably queued",
|
error: "BUFFER_PERSIST_FAILED: Event not durably queued",
|
||||||
@@ -597,7 +597,8 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
// Invalidate the pre-persistence idempotency claim so SDK retry can
|
// Invalidate the pre-persistence idempotency claim so SDK retry can
|
||||||
// resubmit after Redis recovers. Keep the volatile RAM event pending.
|
// resubmit after Redis recovers. Keep the volatile RAM event pending.
|
||||||
const claimKey = `${clientId}:${eventId}`;
|
const claimKey = `${clientId}:${eventId}`;
|
||||||
idempotencyMap.set(claimKey, -Date.now());
|
idempotencyMap.delete(claimKey);
|
||||||
|
idempotencyMap.set(`\u0000failed-persist:${claimKey}`, Date.now());
|
||||||
const claimRedis = getRedisClient();
|
const claimRedis = getRedisClient();
|
||||||
if (claimRedis && typeof claimRedis.del === "function") {
|
if (claimRedis && typeof claimRedis.del === "function") {
|
||||||
try {
|
try {
|
||||||
@@ -621,6 +622,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
idempotencyMap.delete(`\u0000failed-persist:${clientId}:${eventId}`);
|
||||||
if (callback) callback({ success: true, eventId });
|
if (callback) callback({ success: true, eventId });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (callback) callback({ success: false, error: err.message });
|
if (callback) callback({ success: false, error: err.message });
|
||||||
@@ -726,7 +728,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
: new Date().toISOString();
|
: new Date().toISOString();
|
||||||
const rawData = extractBatchItemData(safeItem);
|
const rawData = extractBatchItemData(safeItem);
|
||||||
|
|
||||||
const failedAt = idempotencyMap.get(`${clientId}:${eventId}`);
|
const failedAt = idempotencyMap.get(`\u0000failed-persist:${clientId}:${eventId}`);
|
||||||
const check = await validateSingleEventAsync(
|
const check = await validateSingleEventAsync(
|
||||||
clientId,
|
clientId,
|
||||||
eventId,
|
eventId,
|
||||||
@@ -735,8 +737,8 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
);
|
);
|
||||||
if (!check.allowed) {
|
if (!check.allowed) {
|
||||||
if (check.duplicate) {
|
if (check.duplicate) {
|
||||||
if (typeof failedAt === "number" && failedAt < 0 &&
|
if (typeof failedAt === "number" &&
|
||||||
Date.now() + failedAt < IDEMPOTENCY_TTL_MS) {
|
Date.now() - failedAt < IDEMPOTENCY_TTL_MS) {
|
||||||
rejected += 1;
|
rejected += 1;
|
||||||
volatileDuplicate = true;
|
volatileDuplicate = true;
|
||||||
if (errors.length < 5) errors.push(
|
if (errors.length < 5) errors.push(
|
||||||
@@ -779,7 +781,8 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
const claimRedis = getRedisClient();
|
const claimRedis = getRedisClient();
|
||||||
for (const eventId of acceptedEventIds) {
|
for (const eventId of acceptedEventIds) {
|
||||||
const claimKey = `${clientId}:${eventId}`;
|
const claimKey = `${clientId}:${eventId}`;
|
||||||
idempotencyMap.set(claimKey, -Date.now());
|
idempotencyMap.delete(claimKey);
|
||||||
|
idempotencyMap.set(`\u0000failed-persist:${claimKey}`, Date.now());
|
||||||
if (claimRedis && typeof claimRedis.del === "function") {
|
if (claimRedis && typeof claimRedis.del === "function") {
|
||||||
try {
|
try {
|
||||||
await claimRedis.del(`${REDIS_IDEMPOTENCY_PREFIX}${claimKey}`);
|
await claimRedis.del(`${REDIS_IDEMPOTENCY_PREFIX}${claimKey}`);
|
||||||
@@ -806,6 +809,22 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Only durable new writes clear pending failure markers.
|
||||||
|
for (const persistedId of acceptedEventIds) {
|
||||||
|
idempotencyMap.delete(`\u0000failed-persist:${clientId}:${persistedId}`);
|
||||||
|
}
|
||||||
|
// A mixed batch must never receive success while one event still has
|
||||||
|
// an uncommitted Redis claim, even if other items were persisted.
|
||||||
|
if (volatileDuplicate && processed > 0) {
|
||||||
|
if (callback) callback({
|
||||||
|
success: false, processed, rejected, duplicates,
|
||||||
|
code: "BUFFER_PERSIST_FAILED",
|
||||||
|
error: "BUFFER_PERSIST_FAILED: Batch contains non-durable event",
|
||||||
|
errors,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (processed === 0 && duplicates === 0 && rejected > 0) {
|
if (processed === 0 && duplicates === 0 && rejected > 0) {
|
||||||
const code = volatileDuplicate
|
const code = volatileDuplicate
|
||||||
? "BUFFER_PERSIST_FAILED"
|
? "BUFFER_PERSIST_FAILED"
|
||||||
|
|||||||
@@ -1338,3 +1338,111 @@ test("P02 R-01: batch_sync rejects non-durable Redis writes and pending duplicat
|
|||||||
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
|
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("P02 FINAL R-01: stale Redis claim never becomes positive ACK after cleanup window", async () => {
|
||||||
|
// Source proof at candidate 76cd972: a failure marker is -Date.now();
|
||||||
|
// cleanupMaps compares now - negativeTimestamp > TTL, which expires immediately
|
||||||
|
// on the first scheduled cleanup even though a Redis idempotency claim remains.
|
||||||
|
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
|
||||||
|
const previousQueue = serverMod.bufferQueue.slice();
|
||||||
|
const redis = createSharedMockRedis(); // intentionally no DEL: simulate stuck Redis claim
|
||||||
|
redis.rpush = async () => { throw new Error("Injected Redis queue outage"); };
|
||||||
|
serverMod.setRedisClientForTests(redis);
|
||||||
|
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
|
||||||
|
serverMod.bufferQueue.length = 0;
|
||||||
|
const originalNow = Date.now;
|
||||||
|
let socket: Socket | undefined;
|
||||||
|
try {
|
||||||
|
const client = await SDKManager.registerClientAsync({
|
||||||
|
clientId: "p02_stale_claim_cleanup", name: "P02 Stale Claim", rateLimit: 100,
|
||||||
|
});
|
||||||
|
socket = await connectSocket(client.token);
|
||||||
|
const payload = { eventId: "p02-stale-claim-1", data: { value: 1 } };
|
||||||
|
const first = await emitWithAck<{ success: boolean; code?: string }>(socket, "sync_data", payload);
|
||||||
|
assert.equal(first.success, false);
|
||||||
|
assert.equal(first.code, "BUFFER_PERSIST_FAILED");
|
||||||
|
|
||||||
|
// Advance only the fixture clock beyond cleanup interval, but within
|
||||||
|
// five-minute idempotency TTL. Two retries expose the old negative-marker
|
||||||
|
// cleanup defect; no actual timer wait or production timestamp is changed.
|
||||||
|
Date.now = () => originalNow() + 61_000;
|
||||||
|
const retry1 = await emitWithAck<{ success: boolean; code?: string }>(socket, "sync_data", payload);
|
||||||
|
const retry2 = await emitWithAck<{ success: boolean; code?: string }>(socket, "sync_data", payload);
|
||||||
|
assert.equal(retry1.success, false);
|
||||||
|
assert.equal(retry1.code, "BUFFER_PERSIST_FAILED");
|
||||||
|
assert.equal(retry2.success, false, "Cleanup must not convert a volatile claim into DUPLICATE_DROPPED success");
|
||||||
|
assert.equal(retry2.code, "BUFFER_PERSIST_FAILED");
|
||||||
|
assert.equal((await redis.lrange("datastack:buffer:events", 0, -1)).length, 0);
|
||||||
|
} finally {
|
||||||
|
Date.now = originalNow;
|
||||||
|
socket?.disconnect();
|
||||||
|
serverMod.setRedisClientForTests(undefined);
|
||||||
|
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
|
||||||
|
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("P02 FINAL R-01: mixed batch with stale volatile claim never ACKs success", async () => {
|
||||||
|
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
|
||||||
|
const previousQueue = serverMod.bufferQueue.slice();
|
||||||
|
const redis = createSharedMockRedis();
|
||||||
|
const originalRpush = redis.rpush.bind(redis);
|
||||||
|
redis.rpush = async () => { throw new Error("Injected Redis outage before batch recovery"); };
|
||||||
|
serverMod.setRedisClientForTests(redis);
|
||||||
|
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
|
||||||
|
serverMod.bufferQueue.length = 0;
|
||||||
|
let socket: Socket | undefined;
|
||||||
|
try {
|
||||||
|
const client = await SDKManager.registerClientAsync({
|
||||||
|
clientId: "p02_mixed_batch_claim", name: "P02 Mixed Batch Claim", rateLimit: 100,
|
||||||
|
});
|
||||||
|
socket = await connectSocket(client.token);
|
||||||
|
const pending = { eventId: "p02-mixed-pending-01", data: { v: 1 } };
|
||||||
|
const first = await emitWithAck<{ success: boolean; code?: string }>(
|
||||||
|
socket, "sync_data", pending,
|
||||||
|
);
|
||||||
|
assert.equal(first.success, false);
|
||||||
|
// Simulated process RAM loss: Redis idempotency key survives; queue does not.
|
||||||
|
serverMod.bufferQueue.length = 0;
|
||||||
|
redis.rpush = originalRpush;
|
||||||
|
const mixed = await emitWithAck<{ success: boolean; code?: string; processed?: number }>(
|
||||||
|
socket, "batch_sync", [pending, { eventId: "p02-mixed-durable-02", data: { v: 2 } }],
|
||||||
|
);
|
||||||
|
assert.equal(mixed.success, false, "Mixed batch cannot falsely ACK the volatile duplicate");
|
||||||
|
assert.equal(mixed.code, "BUFFER_PERSIST_FAILED");
|
||||||
|
assert.equal(mixed.processed, 1, "A distinct new event can commit without masking the failed claim");
|
||||||
|
const rows = await redis.lrange("datastack:buffer:events", 0, -1);
|
||||||
|
assert.equal(rows.length, 1);
|
||||||
|
assert.equal((JSON.parse(rows[0]) as { eventId: string }).eventId, "p02-mixed-durable-02");
|
||||||
|
} finally {
|
||||||
|
socket?.disconnect();
|
||||||
|
serverMod.setRedisClientForTests(undefined);
|
||||||
|
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
|
||||||
|
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("P02 FINAL R-01 compatibility: deliberately Redis-free RAM mode remains unchanged", async () => {
|
||||||
|
const threshold = serverMod.runtimeConfig.batchFlushThreshold;
|
||||||
|
const queue = serverMod.bufferQueue.slice();
|
||||||
|
serverMod.setRedisClientForTests(null);
|
||||||
|
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
|
||||||
|
serverMod.bufferQueue.length = 0;
|
||||||
|
let socket: Socket | undefined;
|
||||||
|
try {
|
||||||
|
const client = await SDKManager.registerClientAsync({
|
||||||
|
clientId: "p02_redis_free_compat", name: "P02 Redis-free Compatibility", rateLimit: 100,
|
||||||
|
});
|
||||||
|
socket = await connectSocket(client.token);
|
||||||
|
const ack = await emitWithAck<{ success: boolean; eventId?: string }>(
|
||||||
|
socket, "sync_data", { eventId: "p02-ram-compat-001", data: { value: 1 } },
|
||||||
|
);
|
||||||
|
assert.equal(ack.success, true, "Preserve historical RAM-mode acceptance contract");
|
||||||
|
assert.equal(serverMod.bufferQueue.some((e) => e.eventId === "p02-ram-compat-001"), true);
|
||||||
|
} finally {
|
||||||
|
socket?.disconnect();
|
||||||
|
serverMod.setRedisClientForTests(undefined);
|
||||||
|
serverMod.runtimeConfig.batchFlushThreshold = threshold;
|
||||||
|
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...queue);
|
||||||
|
}
|
||||||
|
});
|
||||||
@@ -1639,3 +1639,79 @@ test("P02 R-02: Port 4000 CSV worker suppresses a crash-window replay after Redi
|
|||||||
ingestion.setRedisClientForTests(undefined);
|
ingestion.setRedisClientForTests(undefined);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("P02 FINAL R-02: Port 4000 scans rotated compressed CSV with quoted multiline payload", async () => {
|
||||||
|
const zlib = await import("node:zlib");
|
||||||
|
const redis = new FakeRedisClient();
|
||||||
|
const ingestion = require("../server/index.js") as IngestionServerModule;
|
||||||
|
const oldDir = ingestion.writer.baseDir;
|
||||||
|
const oldQueue = ingestion.bufferQueue.slice();
|
||||||
|
ingestion.writer.baseDir = tmpStorageDir;
|
||||||
|
ingestion.bufferQueue.length = 0;
|
||||||
|
ingestion.setRedisClientForTests(redis);
|
||||||
|
const event: DataEvent = {
|
||||||
|
clientId: "p02_rotated_gzip",
|
||||||
|
eventId: "p02-gzip-quoted-001",
|
||||||
|
timestamp: "2026-11-22T12:45:00.000Z",
|
||||||
|
payload: JSON.stringify({ body: "Line one, \"quoted\"\nLine two" }),
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
const diskWriter = new CsvWriterWorker(tmpStorageDir);
|
||||||
|
await diskWriter.writeBatch([event]);
|
||||||
|
const dir = path.join(tmpStorageDir, event.clientId);
|
||||||
|
const csvPath = path.join(dir, "2026-11-22.csv");
|
||||||
|
const rotatedPath = path.join(dir, "2026-11-22-part2.csv");
|
||||||
|
const gzPath = rotatedPath + ".gz";
|
||||||
|
fs.renameSync(csvPath, rotatedPath);
|
||||||
|
fs.writeFileSync(gzPath, zlib.gzipSync(fs.readFileSync(rotatedPath)));
|
||||||
|
fs.unlinkSync(rotatedPath);
|
||||||
|
// A different date partition must not be parsed as part of this replay.
|
||||||
|
fs.writeFileSync(path.join(dir, "2026-11-21.csv"), 'invalid,"unterminated');
|
||||||
|
await redis.rpush(REDIS_BUFFER_KEY, JSON.stringify(event));
|
||||||
|
const replay = await ingestion.flushBuffer();
|
||||||
|
assert.equal(replay.written, 0, "Already written rotated/gzipped event must not append again");
|
||||||
|
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0, "Successful dedupe permits Redis trim");
|
||||||
|
assert.equal(fs.existsSync(csvPath), false, "Do not recreate a duplicate plain CSV");
|
||||||
|
assert.equal(zlib.gunzipSync(fs.readFileSync(gzPath)).toString("utf8").split(event.eventId).length - 1, 1);
|
||||||
|
} finally {
|
||||||
|
ingestion.writer.baseDir = oldDir;
|
||||||
|
ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...oldQueue);
|
||||||
|
ingestion.setRedisClientForTests(undefined);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("P02 FINAL R-02: partial corrupted CSV blocks Redis trim and recovers after fixture repair", async () => {
|
||||||
|
const redis = new FakeRedisClient();
|
||||||
|
const ingestion = require("../server/index.js") as IngestionServerModule;
|
||||||
|
const oldDir = ingestion.writer.baseDir;
|
||||||
|
const oldQueue = ingestion.bufferQueue.slice();
|
||||||
|
ingestion.writer.baseDir = tmpStorageDir;
|
||||||
|
ingestion.bufferQueue.length = 0;
|
||||||
|
ingestion.setRedisClientForTests(redis);
|
||||||
|
const event: DataEvent = {
|
||||||
|
clientId: "p02_partial_corrupt",
|
||||||
|
eventId: "p02-corrupt-recovery-001",
|
||||||
|
timestamp: "2026-11-22T13:15:00.000Z",
|
||||||
|
payload: JSON.stringify({ value: "safe" }),
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
const dir = path.join(tmpStorageDir, event.clientId);
|
||||||
|
fs.mkdirSync(dir, { recursive: true });
|
||||||
|
const badPath = path.join(dir, "2026-11-22.csv");
|
||||||
|
fs.writeFileSync(badPath, 'clientId,eventId,timestamp,payload\n"unterminated');
|
||||||
|
await redis.rpush(REDIS_BUFFER_KEY, JSON.stringify(event));
|
||||||
|
await assert.rejects(() => ingestion.flushBuffer(),
|
||||||
|
/quote|CSV_|invalid|record|unterminated/i, "Malformed existing CSV must fail closed");
|
||||||
|
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 1, "Never trim after failed CSV replay check");
|
||||||
|
// Disposable fixture repair only; does not alter user or production files.
|
||||||
|
fs.unlinkSync(badPath);
|
||||||
|
const recovered = await ingestion.flushBuffer();
|
||||||
|
assert.equal(recovered.written, 1);
|
||||||
|
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0);
|
||||||
|
assert.equal(fs.readFileSync(badPath, "utf8").split(event.eventId).length - 1, 1);
|
||||||
|
} finally {
|
||||||
|
ingestion.writer.baseDir = oldDir;
|
||||||
|
ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...oldQueue);
|
||||||
|
ingestion.setRedisClientForTests(undefined);
|
||||||
|
}
|
||||||
|
});
|
||||||
Reference in new issue
Block a user