test(p02): probe Redis durability, CSV replay and degraded health [skip ci]
This commit is contained in:
1 parent
667a140c4e
commit
bbe73d9166
2 files changed
+103
No files matched your search
@@ -1220,3 +1220,64 @@ test("R2 PHASE-02 BUG-10: GET, PATCH (regenerate_token), and DELETE /api/clients
|
|||||||
SDKManager.deleteClientWithStatsAsync = origDeleteWithStats;
|
SDKManager.deleteClientWithStatsAsync = origDeleteWithStats;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("P02 R-01 diagnostic: Redis RPUSH failure yields positive WS ACK for volatile RAM-only event", async () => {
|
||||||
|
// Disposable test-only fixture: no production Redis, service, or storage state.
|
||||||
|
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
|
||||||
|
const previousRam = serverMod.bufferQueue.slice();
|
||||||
|
const failingRedis = createSharedMockRedis();
|
||||||
|
failingRedis.rpush = async () => {
|
||||||
|
throw new Error("P02 injected Redis RPUSH outage");
|
||||||
|
};
|
||||||
|
serverMod.setRedisClientForTests(failingRedis);
|
||||||
|
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
|
||||||
|
serverMod.bufferQueue.length = 0;
|
||||||
|
let socket: Socket | undefined;
|
||||||
|
try {
|
||||||
|
const client = await SDKManager.registerClientAsync({
|
||||||
|
clientId: "p02_redis_rpush_failure",
|
||||||
|
name: "P02 Redis RPUSH Failure",
|
||||||
|
rateLimit: 100,
|
||||||
|
});
|
||||||
|
socket = await connectSocket(client.token);
|
||||||
|
const ack = await emitWithAck<{ success: boolean; eventId?: string; code?: string }>(
|
||||||
|
socket, "sync_data",
|
||||||
|
{ eventId: "p02-ack-before-crash-001", data: { value: 42 } },
|
||||||
|
);
|
||||||
|
const committedRows = await failingRedis.lrange("datastack:buffer:events", 0, -1);
|
||||||
|
const volatileRows = serverMod.bufferQueue.filter((e) => e.eventId === "p02-ack-before-crash-001");
|
||||||
|
assert.equal(ack.success, true, "Existing handler currently sends a positive ACK after Redis failure");
|
||||||
|
assert.equal(committedRows.length, 0, "Fake Redis has no durable event");
|
||||||
|
assert.equal(volatileRows.length, 1, "Acknowledged event is RAM-only");
|
||||||
|
// Crash model: restart loses RAM but fake Redis survives. No actual process is killed.
|
||||||
|
serverMod.bufferQueue.length = 0;
|
||||||
|
assert.equal((await failingRedis.lrange("datastack:buffer:events", 0, -1)).length, 0);
|
||||||
|
} finally {
|
||||||
|
socket?.disconnect();
|
||||||
|
serverMod.setRedisClientForTests(undefined);
|
||||||
|
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
|
||||||
|
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousRam);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("P02 R-03 diagnostic: Redis query failure leaves public ingestion health 200/online", async () => {
|
||||||
|
const redis = createSharedMockRedis();
|
||||||
|
redis.llen = async () => { throw new Error("P02 simulated Redis health outage"); };
|
||||||
|
serverMod.setRedisClientForTests(redis);
|
||||||
|
try {
|
||||||
|
const res = await fetch(`${ingestionUrl}/health`);
|
||||||
|
assert.equal(res.status, 200, "Legacy liveness status remains backwards-compatible");
|
||||||
|
const body = await res.json() as { status: string; redisConnected: boolean; redisBufferSize: number };
|
||||||
|
assert.equal(body.status, "online");
|
||||||
|
assert.equal(body.redisConnected, false, "Redis failure is represented only in the payload");
|
||||||
|
assert.equal(body.redisBufferSize, 0);
|
||||||
|
const healthyRedis = createSharedMockRedis();
|
||||||
|
serverMod.setRedisClientForTests(healthyRedis);
|
||||||
|
const recovered = await fetch(`${ingestionUrl}/health`);
|
||||||
|
const recoveredBody = await recovered.json() as { redisConnected: boolean };
|
||||||
|
assert.equal(recovered.status, 200);
|
||||||
|
assert.equal(recoveredBody.redisConnected, true);
|
||||||
|
} finally {
|
||||||
|
serverMod.setRedisClientForTests(undefined);
|
||||||
|
}
|
||||||
|
});
|
||||||
@@ -1549,3 +1549,45 @@ test("R2 PHASE-03 BUG-15: GET /api/export rejects inverted date range with 400 a
|
|||||||
assert.match(serverIndexSrc, /const filterClientId =\s*rawClientId && rawClientId !== 'all' \? normalizeClientId\(rawClientId\) : ''/);
|
assert.match(serverIndexSrc, /const filterClientId =\s*rawClientId && rawClientId !== 'all' \? normalizeClientId\(rawClientId\) : ''/);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
||||||
|
test("P02 R-02 diagnostic: post-CSV Redis trim failure replays identical event on retry", async () => {
|
||||||
|
const redis = new FakeRedisClient();
|
||||||
|
const realTrim = redis.ltrim.bind(redis);
|
||||||
|
let injected = false;
|
||||||
|
redis.ltrim = async (key: string, start: number, stop: number): Promise<"OK"> => {
|
||||||
|
if (key === REDIS_BUFFER_KEY && !injected) {
|
||||||
|
injected = true;
|
||||||
|
throw new Error("P02 simulated crash boundary after CSV append, before Redis trim");
|
||||||
|
}
|
||||||
|
return realTrim(key, start, stop);
|
||||||
|
};
|
||||||
|
const event: DataEvent = {
|
||||||
|
clientId: "p02_csv_replay",
|
||||||
|
eventId: "p02-replay-001",
|
||||||
|
timestamp: "2026-11-22T01:00:00.000Z",
|
||||||
|
payload: JSON.stringify({ marker: "p02" }),
|
||||||
|
};
|
||||||
|
DataBufferEngine.setRedisClientForTests(redis);
|
||||||
|
const originalAppend = CSVStorageManager.appendBatch;
|
||||||
|
const writtenIds: string[] = [];
|
||||||
|
CSVStorageManager.appendBatch = async (events) => {
|
||||||
|
const result = await originalAppend.call(CSVStorageManager, events);
|
||||||
|
writtenIds.push(...events.map((row) => row.eventId));
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
await DataBufferEngine.pushBatchAsync([event]);
|
||||||
|
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 1);
|
||||||
|
await assert.rejects(() => DataBufferEngine.flush(), /P02 simulated crash boundary/);
|
||||||
|
assert.equal(injected, true);
|
||||||
|
assert.equal(writtenIds.filter((id) => id === event.eventId).length, 1);
|
||||||
|
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 1, "Redis still queues previously appended CSV event");
|
||||||
|
await DataBufferEngine.flush();
|
||||||
|
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0);
|
||||||
|
assert.equal(writtenIds.filter((id) => id === event.eventId).length, 2,
|
||||||
|
"A second flush appends the same eventId again after failed trim: at-least-once, not exactly-once");
|
||||||
|
} finally {
|
||||||
|
CSVStorageManager.appendBatch = originalAppend;
|
||||||
|
DataBufferEngine.setRedisClientForTests(undefined);
|
||||||
|
}
|
||||||
|
});
|
||||||
Reference in new issue
Block a user