test(p02): guard stale Redis claims and verify CSV replay edge cases [skip ci]

This commit is contained in:
victorsteele428 committed 2026-11-22 16:45:45 +06:00
1 parent 76cd972bde
commit bf525f0d4e
3 files changed
+211 -8

No files matched your search

+76
View File
@@ -1639,3 +1639,79 @@ test("P02 R-02: Port 4000 CSV worker suppresses a crash-window replay after Redi
ingestion.setRedisClientForTests(undefined);
}
});
test("P02 FINAL R-02: Port 4000 scans rotated compressed CSV with quoted multiline payload", async () => {
const zlib = await import("node:zlib");
const redis = new FakeRedisClient();
const ingestion = require("../server/index.js") as IngestionServerModule;
const oldDir = ingestion.writer.baseDir;
const oldQueue = ingestion.bufferQueue.slice();
ingestion.writer.baseDir = tmpStorageDir;
ingestion.bufferQueue.length = 0;
ingestion.setRedisClientForTests(redis);
const event: DataEvent = {
clientId: "p02_rotated_gzip",
eventId: "p02-gzip-quoted-001",
timestamp: "2026-11-22T12:45:00.000Z",
payload: JSON.stringify({ body: "Line one, \"quoted\"\nLine two" }),
};
try {
const diskWriter = new CsvWriterWorker(tmpStorageDir);
await diskWriter.writeBatch([event]);
const dir = path.join(tmpStorageDir, event.clientId);
const csvPath = path.join(dir, "2026-11-22.csv");
const rotatedPath = path.join(dir, "2026-11-22-part2.csv");
const gzPath = rotatedPath + ".gz";
fs.renameSync(csvPath, rotatedPath);
fs.writeFileSync(gzPath, zlib.gzipSync(fs.readFileSync(rotatedPath)));
fs.unlinkSync(rotatedPath);
// A different date partition must not be parsed as part of this replay.
fs.writeFileSync(path.join(dir, "2026-11-21.csv"), 'invalid,"unterminated');
await redis.rpush(REDIS_BUFFER_KEY, JSON.stringify(event));
const replay = await ingestion.flushBuffer();
assert.equal(replay.written, 0, "Already written rotated/gzipped event must not append again");
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0, "Successful dedupe permits Redis trim");
assert.equal(fs.existsSync(csvPath), false, "Do not recreate a duplicate plain CSV");
assert.equal(zlib.gunzipSync(fs.readFileSync(gzPath)).toString("utf8").split(event.eventId).length - 1, 1);
} finally {
ingestion.writer.baseDir = oldDir;
ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...oldQueue);
ingestion.setRedisClientForTests(undefined);
}
});
test("P02 FINAL R-02: partial corrupted CSV blocks Redis trim and recovers after fixture repair", async () => {
const redis = new FakeRedisClient();
const ingestion = require("../server/index.js") as IngestionServerModule;
const oldDir = ingestion.writer.baseDir;
const oldQueue = ingestion.bufferQueue.slice();
ingestion.writer.baseDir = tmpStorageDir;
ingestion.bufferQueue.length = 0;
ingestion.setRedisClientForTests(redis);
const event: DataEvent = {
clientId: "p02_partial_corrupt",
eventId: "p02-corrupt-recovery-001",
timestamp: "2026-11-22T13:15:00.000Z",
payload: JSON.stringify({ value: "safe" }),
};
try {
const dir = path.join(tmpStorageDir, event.clientId);
fs.mkdirSync(dir, { recursive: true });
const badPath = path.join(dir, "2026-11-22.csv");
fs.writeFileSync(badPath, 'clientId,eventId,timestamp,payload\n"unterminated');
await redis.rpush(REDIS_BUFFER_KEY, JSON.stringify(event));
await assert.rejects(() => ingestion.flushBuffer(),
/quote|CSV_|invalid|record|unterminated/i, "Malformed existing CSV must fail closed");
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 1, "Never trim after failed CSV replay check");
// Disposable fixture repair only; does not alter user or production files.
fs.unlinkSync(badPath);
const recovered = await ingestion.flushBuffer();
assert.equal(recovered.written, 1);
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0);
assert.equal(fs.readFileSync(badPath, "utf8").split(event.eventId).length - 1, 1);
} finally {
ingestion.writer.baseDir = oldDir;
ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...oldQueue);
ingestion.setRedisClientForTests(undefined);
}
});