fix(p02): close verified Redis ACK and CSV replay acceptance [skip ci]
This commit was merged in pull request #3.
This commit is contained in:
1 parent
fd57a3518d
commit
48fb277079
11 files changed
+799
-25
No files matched your search
+74
-6
@@ -165,6 +165,7 @@ function setRedisClientForTests(client) {
|
||||
async function pushEventsToRedisOrFallback(events) {
|
||||
if (!events || events.length === 0) return bufferQueue.length;
|
||||
const redis = getRedisClient();
|
||||
let persistFailed = false;
|
||||
if (redis) {
|
||||
const serialized = events.map((e) => JSON.stringify(e));
|
||||
let bufferLen = null;
|
||||
@@ -173,6 +174,7 @@ async function pushEventsToRedisOrFallback(events) {
|
||||
redisConnected = true;
|
||||
} catch {
|
||||
redisConnected = false;
|
||||
persistFailed = true;
|
||||
}
|
||||
|
||||
if (bufferLen !== null) {
|
||||
@@ -189,6 +191,11 @@ async function pushEventsToRedisOrFallback(events) {
|
||||
}
|
||||
|
||||
bufferQueue.push(...events);
|
||||
if (persistFailed) {
|
||||
// Keep the volatile fallback for later recovery, but never ACK it as durable.
|
||||
// Suppress sensitive Redis transport details in the client-facing error.
|
||||
throw new Error('BUFFER_PERSIST_FAILED: Redis unavailable; event not durably queued');
|
||||
}
|
||||
return bufferQueue.length;
|
||||
}
|
||||
|
||||
@@ -834,6 +841,61 @@ async function isClientDeletedForFlush(clientId, redis, cache) {
|
||||
}
|
||||
|
||||
async function flushBuffer(options = {}) {
|
||||
|
||||
// P02 R-02: Redis and CSV cannot be committed atomically. Before replaying an
|
||||
// untrimmed Redis batch, check actual CSV partitions for previously written IDs.
|
||||
// Streaming parsing bounds memory and preserves multiline/quoted CSV fields.
|
||||
const excludeAlreadyPersisted = async (events) => {
|
||||
if (!events.length) return events;
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const zlib = require('node:zlib');
|
||||
const { pipeline } = require('node:stream/promises');
|
||||
const { Writable } = require('node:stream');
|
||||
const { parse } = require('csv-parse');
|
||||
const groups = new Map();
|
||||
for (const event of events) {
|
||||
const clientId = String(event.clientId || 'unknown');
|
||||
const date = writer.extractSafePartitionDate(event.timestamp);
|
||||
const key = JSON.stringify([clientId, date]);
|
||||
if (!groups.has(key)) groups.set(key, { clientId, date, ids: new Set() });
|
||||
groups.get(key).ids.add(String(event.eventId));
|
||||
}
|
||||
const persisted = new Set();
|
||||
for (const [groupKey, group] of groups) {
|
||||
const dir = writer.ensureClientDir(group.clientId);
|
||||
const entries = await fs.promises.readdir(dir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
if (!entry.isFile() || !writer.isPartitionFilename(entry.name) ||
|
||||
writer.extractDateFromFilename(entry.name) !== group.date) continue;
|
||||
const reader = fs.createReadStream(path.join(dir, entry.name));
|
||||
const streams = [reader];
|
||||
if (entry.name.endsWith('.gz')) streams.push(zlib.createGunzip());
|
||||
streams.push(parse({ columns: true, bom: true, skip_empty_lines: true }));
|
||||
streams.push(new Writable({
|
||||
objectMode: true,
|
||||
write(row, _enc, done) {
|
||||
if (row.clientId === group.clientId && group.ids.has(String(row.eventId))) {
|
||||
persisted.add(JSON.stringify([groupKey, String(row.eventId)]));
|
||||
}
|
||||
done();
|
||||
},
|
||||
}));
|
||||
// On read/parse failure, abort without LTRIM: never discard uncertain rows.
|
||||
await pipeline(...streams);
|
||||
}
|
||||
}
|
||||
const batchSeen = new Set();
|
||||
return events.filter((event) => {
|
||||
const key = JSON.stringify([String(event.clientId || 'unknown'),
|
||||
writer.extractSafePartitionDate(event.timestamp)]);
|
||||
const rowKey = JSON.stringify([key, String(event.eventId)]);
|
||||
if (persisted.has(rowKey) || batchSeen.has(rowKey)) return false;
|
||||
batchSeen.add(rowKey);
|
||||
return true;
|
||||
});
|
||||
};
|
||||
|
||||
if (pendingRedisWrites.size > 0) {
|
||||
await Promise.allSettled([...pendingRedisWrites]);
|
||||
}
|
||||
@@ -979,9 +1041,12 @@ async function flushBuffer(options = {}) {
|
||||
if (lockLost) break;
|
||||
|
||||
if (parsedBatch.length > 0) {
|
||||
const res = await writer.writeBatch(parsedBatch);
|
||||
totalWritten += res.written;
|
||||
for (const f of res.files) filesSet.add(f);
|
||||
const unwritten = await excludeAlreadyPersisted(parsedBatch);
|
||||
if (unwritten.length > 0) {
|
||||
const res = await writer.writeBatch(unwritten);
|
||||
totalWritten += res.written;
|
||||
for (const f of res.files) filesSet.add(f);
|
||||
}
|
||||
}
|
||||
|
||||
if (Date.now() - lastRenewAt >= renewIntervalMs) {
|
||||
@@ -1022,9 +1087,12 @@ async function flushBuffer(options = {}) {
|
||||
|
||||
if (validBatch.length > 0) {
|
||||
try {
|
||||
const res = await writer.writeBatch(validBatch);
|
||||
totalWritten += res.written;
|
||||
for (const f of res.files) filesSet.add(f);
|
||||
const unwritten = await excludeAlreadyPersisted(validBatch);
|
||||
if (unwritten.length > 0) {
|
||||
const res = await writer.writeBatch(unwritten);
|
||||
totalWritten += res.written;
|
||||
for (const f of res.files) filesSet.add(f);
|
||||
}
|
||||
} catch (err) {
|
||||
bufferQueue.unshift(...validBatch);
|
||||
throw err;
|
||||
|
||||
@@ -549,6 +549,9 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
: new Date().toISOString();
|
||||
const eventData = extractSingleEventData(payload);
|
||||
|
||||
// A previous Redis write failure can leave an uncommitted idempotency
|
||||
// claim. Never report that claim as a successful durable duplicate.
|
||||
const failedAt = idempotencyMap.get(`\u0000failed-persist:${clientId}:${eventId}`);
|
||||
const check = await validateSingleEventAsync(
|
||||
clientId,
|
||||
eventId,
|
||||
@@ -557,7 +560,13 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
);
|
||||
if (!check.allowed) {
|
||||
if (check.duplicate) {
|
||||
if (callback) {
|
||||
if (typeof failedAt === "number" &&
|
||||
Date.now() - failedAt < IDEMPOTENCY_TTL_MS) {
|
||||
if (callback) callback({
|
||||
success: false, eventId, code: "BUFFER_PERSIST_FAILED",
|
||||
error: "BUFFER_PERSIST_FAILED: Event not durably queued",
|
||||
});
|
||||
} else if (callback) {
|
||||
callback({ success: true, eventId, note: "DUPLICATE_DROPPED" });
|
||||
}
|
||||
return;
|
||||
@@ -585,6 +594,20 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
try {
|
||||
await onBufferPush();
|
||||
} catch (persistErr) {
|
||||
// Invalidate the pre-persistence idempotency claim so SDK retry can
|
||||
// resubmit after Redis recovers. Keep the volatile RAM event pending.
|
||||
const claimKey = `${clientId}:${eventId}`;
|
||||
idempotencyMap.delete(claimKey);
|
||||
idempotencyMap.set(`\u0000failed-persist:${claimKey}`, Date.now());
|
||||
const claimRedis = getRedisClient();
|
||||
if (claimRedis && typeof claimRedis.del === "function") {
|
||||
try {
|
||||
await claimRedis.del(`${REDIS_IDEMPOTENCY_PREFIX}${claimKey}`);
|
||||
idempotencyMap.delete(claimKey);
|
||||
} catch {
|
||||
// Claim may remain: negative ACK must be retained for retries.
|
||||
}
|
||||
}
|
||||
if (callback) {
|
||||
callback({
|
||||
success: false,
|
||||
@@ -599,6 +622,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
return;
|
||||
}
|
||||
|
||||
idempotencyMap.delete(`\u0000failed-persist:${clientId}:${eventId}`);
|
||||
if (callback) callback({ success: true, eventId });
|
||||
} catch (err) {
|
||||
if (callback) callback({ success: false, error: err.message });
|
||||
@@ -689,6 +713,8 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
const rateLimitedEventIds = [];
|
||||
const oversizedEventIds = [];
|
||||
const errors = [];
|
||||
const acceptedEventIds = [];
|
||||
let volatileDuplicate = false;
|
||||
|
||||
for (const item of batch) {
|
||||
const safeItem = item && typeof item === "object" ? item : {};
|
||||
@@ -702,6 +728,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
: new Date().toISOString();
|
||||
const rawData = extractBatchItemData(safeItem);
|
||||
|
||||
const failedAt = idempotencyMap.get(`\u0000failed-persist:${clientId}:${eventId}`);
|
||||
const check = await validateSingleEventAsync(
|
||||
clientId,
|
||||
eventId,
|
||||
@@ -710,7 +737,15 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
);
|
||||
if (!check.allowed) {
|
||||
if (check.duplicate) {
|
||||
duplicates += 1;
|
||||
if (typeof failedAt === "number" &&
|
||||
Date.now() - failedAt < IDEMPOTENCY_TTL_MS) {
|
||||
rejected += 1;
|
||||
volatileDuplicate = true;
|
||||
if (errors.length < 5) errors.push(
|
||||
`${eventId}: BUFFER_PERSIST_FAILED: Event not durably queued`);
|
||||
} else {
|
||||
duplicates += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
rejected += 1;
|
||||
@@ -735,6 +770,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
payload: check.payloadStr,
|
||||
});
|
||||
processed += 1;
|
||||
acceptedEventIds.push(eventId);
|
||||
}
|
||||
|
||||
if (processed > 0) {
|
||||
@@ -742,6 +778,20 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
try {
|
||||
await onBufferPush();
|
||||
} catch (persistErr) {
|
||||
const claimRedis = getRedisClient();
|
||||
for (const eventId of acceptedEventIds) {
|
||||
const claimKey = `${clientId}:${eventId}`;
|
||||
idempotencyMap.delete(claimKey);
|
||||
idempotencyMap.set(`\u0000failed-persist:${claimKey}`, Date.now());
|
||||
if (claimRedis && typeof claimRedis.del === "function") {
|
||||
try {
|
||||
await claimRedis.del(`${REDIS_IDEMPOTENCY_PREFIX}${claimKey}`);
|
||||
idempotencyMap.delete(claimKey);
|
||||
} catch {
|
||||
// Preserve negative-ACK retry guard if claim removal fails.
|
||||
}
|
||||
}
|
||||
}
|
||||
if (callback) {
|
||||
callback({
|
||||
success: false,
|
||||
@@ -759,12 +809,30 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
}
|
||||
}
|
||||
|
||||
// Only durable new writes clear pending failure markers.
|
||||
for (const persistedId of acceptedEventIds) {
|
||||
idempotencyMap.delete(`\u0000failed-persist:${clientId}:${persistedId}`);
|
||||
}
|
||||
// A mixed batch must never receive success while one event still has
|
||||
// an uncommitted Redis claim, even if other items were persisted.
|
||||
if (volatileDuplicate) {
|
||||
if (callback) callback({
|
||||
success: false, processed, rejected, duplicates,
|
||||
code: "BUFFER_PERSIST_FAILED",
|
||||
error: "BUFFER_PERSIST_FAILED: Batch contains non-durable event",
|
||||
errors,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (processed === 0 && duplicates === 0 && rejected > 0) {
|
||||
const code = hitPayloadLimit
|
||||
? "PAYLOAD_TOO_LARGE"
|
||||
: hitRateLimit
|
||||
? "RATE_LIMIT_EXCEEDED"
|
||||
: "BATCH_REJECTED";
|
||||
const code = volatileDuplicate
|
||||
? "BUFFER_PERSIST_FAILED"
|
||||
: hitPayloadLimit
|
||||
? "PAYLOAD_TOO_LARGE"
|
||||
: hitRateLimit
|
||||
? "RATE_LIMIT_EXCEEDED"
|
||||
: "BATCH_REJECTED";
|
||||
if (callback) {
|
||||
callback({
|
||||
success: false,
|
||||
|
||||
Reference in new issue
Block a user