2026-11-18 05:18:55 +06:00
|
|
|
import crypto from 'node:crypto';
|
|
|
|
|
import Redis from 'ioredis';
|
2026-10-23 11:43:49 -05:00
|
|
|
import { CSVStorageManager, DataEvent } from '../Storage/csvStorage';
|
|
|
|
|
|
|
|
|
|
export type EventSubscriber = (event: DataEvent) => void;
|
|
|
|
|
export type FlushCallback = (count: number, files: string[]) => void;
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
export interface RedisBufferClient {
|
|
|
|
|
status?: string;
|
|
|
|
|
rpush(key: string, ...values: string[]): Promise<number>;
|
|
|
|
|
lpush(key: string, ...values: string[]): Promise<number>;
|
|
|
|
|
ltrim(key: string, start: number, stop: number): Promise<unknown>;
|
|
|
|
|
lrange(key: string, start: number, stop: number): Promise<string[]>;
|
|
|
|
|
llen(key: string): Promise<number>;
|
|
|
|
|
set(
|
|
|
|
|
key: string,
|
|
|
|
|
value: string,
|
|
|
|
|
pxMode: 'PX',
|
|
|
|
|
ttlMs: number,
|
|
|
|
|
nxMode: 'NX',
|
|
|
|
|
): Promise<string | null>;
|
2026-11-19 20:22:01 +06:00
|
|
|
get?(key: string): Promise<string | null>;
|
|
|
|
|
incr?(key: string): Promise<number>;
|
|
|
|
|
pexpire?(key: string, ttlMs: number): Promise<number>;
|
|
|
|
|
hget?(key: string, field: string): Promise<string | null>;
|
|
|
|
|
hset?(key: string, field: string, value: string): Promise<number>;
|
|
|
|
|
hdel?(key: string, ...fields: string[]): Promise<number>;
|
|
|
|
|
sismember?(key: string, member: string): Promise<number>;
|
|
|
|
|
sadd?(key: string, ...members: string[]): Promise<number>;
|
|
|
|
|
srem?(key: string, ...members: string[]): Promise<number>;
|
2026-11-18 05:18:55 +06:00
|
|
|
eval(script: string, numKeys: number, ...args: string[]): Promise<unknown>;
|
|
|
|
|
quit?: () => Promise<unknown>;
|
|
|
|
|
disconnect?: () => void;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export const REDIS_BUFFER_KEY = 'datastack:buffer:events';
|
|
|
|
|
export const REDIS_RECENT_KEY = 'datastack:recent:events';
|
|
|
|
|
export const REDIS_FLUSH_LOCK_KEY = 'datastack:flush:lock';
|
2026-11-19 20:22:01 +06:00
|
|
|
const REDIS_CLIENTS_DELETED_KEY = 'datastack:clients:deleted';
|
|
|
|
|
export const DEFAULT_REDIS_RECENT_LIMIT = 500;
|
|
|
|
|
export const MAX_REDIS_RECENT_LIMIT = 5000;
|
|
|
|
|
const FLUSH_LOCK_TTL_MS = 30000;
|
|
|
|
|
const FLUSH_LOCK_WAIT_MS = 400;
|
|
|
|
|
const FLUSH_LOCK_RETRY_MS = 25;
|
2026-11-18 05:18:55 +06:00
|
|
|
const UNLOCK_LUA_SCRIPT =
|
|
|
|
|
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end';
|
2026-11-19 20:22:01 +06:00
|
|
|
const RENEW_LOCK_LUA_SCRIPT =
|
|
|
|
|
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("pexpire", KEYS[1], ARGV[2]) else return 0 end';
|
|
|
|
|
export const PURGE_CLIENT_LIST_LUA_SCRIPT = `
|
|
|
|
|
-- DATASTACK_PURGE_CLIENT_LIST
|
|
|
|
|
local key = KEYS[1]
|
|
|
|
|
local target = ARGV[1]
|
|
|
|
|
local items = redis.call("LRANGE", key, 0, -1)
|
|
|
|
|
local removed = 0
|
|
|
|
|
local kept = {}
|
|
|
|
|
for i = 1, #items do
|
|
|
|
|
local raw = items[i]
|
|
|
|
|
local ok, obj = pcall(cjson.decode, raw)
|
|
|
|
|
local cid = ""
|
|
|
|
|
if ok and type(obj) == "table" and obj.clientId ~= nil then
|
|
|
|
|
cid = string.lower(tostring(obj.clientId))
|
|
|
|
|
cid = string.gsub(cid, "^%s*(.-)%s*$", "%1")
|
|
|
|
|
cid = string.gsub(cid, "[^a-z0-9_-]", "_")
|
|
|
|
|
end
|
|
|
|
|
if cid == target then
|
|
|
|
|
removed = removed + 1
|
|
|
|
|
else
|
|
|
|
|
kept[#kept + 1] = raw
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
if removed > 0 then
|
|
|
|
|
redis.call("DEL", key)
|
|
|
|
|
local chunk = 500
|
|
|
|
|
for i = 1, #kept, chunk do
|
|
|
|
|
local stop = math.min(i + chunk - 1, #kept)
|
|
|
|
|
redis.call("RPUSH", key, unpack(kept, i, stop))
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
return removed
|
|
|
|
|
`.trim();
|
|
|
|
|
|
|
|
|
|
export function resolveRedisRecentLimit(batchFlushThreshold?: number): number {
|
|
|
|
|
const envRaw = Number(process.env.DATASTACK_RECENT_LIMIT);
|
|
|
|
|
if (Number.isFinite(envRaw) && envRaw >= 100) {
|
|
|
|
|
return Math.min(MAX_REDIS_RECENT_LIMIT, Math.floor(envRaw));
|
|
|
|
|
}
|
|
|
|
|
const threshold = Number(batchFlushThreshold) || DEFAULT_REDIS_RECENT_LIMIT;
|
|
|
|
|
return Math.min(
|
|
|
|
|
MAX_REDIS_RECENT_LIMIT,
|
|
|
|
|
Math.max(DEFAULT_REDIS_RECENT_LIMIT, Math.floor(threshold)),
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export interface FlushOptions {
|
|
|
|
|
waitMs?: number;
|
|
|
|
|
lockTtlMs?: number;
|
|
|
|
|
throwOnLockTimeout?: boolean;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export interface FlushResult {
|
|
|
|
|
count: number;
|
|
|
|
|
files: string[];
|
|
|
|
|
lockTimedOut?: boolean;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function sleepMs(ms: number): Promise<void> {
|
|
|
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function normalizeBufferClientId(clientId: string): string {
|
|
|
|
|
return String(clientId || '')
|
|
|
|
|
.trim()
|
|
|
|
|
.toLowerCase()
|
|
|
|
|
.replace(/[^a-z0-9_-]/g, '_');
|
|
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
|
2026-11-17 20:05:00 +06:00
|
|
|
interface BufferGlobalState {
|
|
|
|
|
buffer: DataEvent[];
|
|
|
|
|
recentEvents: DataEvent[];
|
|
|
|
|
batchFlushThreshold: number;
|
|
|
|
|
batchFlushIntervalMs: number;
|
|
|
|
|
flushTimer: NodeJS.Timeout | null;
|
|
|
|
|
subscribers: Set<EventSubscriber>;
|
|
|
|
|
flushListeners: Set<FlushCallback>;
|
|
|
|
|
totalIngested: number;
|
|
|
|
|
totalFlushed: number;
|
|
|
|
|
isFlushing: boolean;
|
2026-11-19 20:22:01 +06:00
|
|
|
activeFlushPromise: Promise<FlushResult> | null;
|
2026-11-18 05:18:55 +06:00
|
|
|
redisClient: RedisBufferClient | null | undefined;
|
|
|
|
|
redisUrlConfigured: string;
|
|
|
|
|
redisConnected: boolean;
|
|
|
|
|
redisTestOverride: RedisBufferClient | null | undefined;
|
|
|
|
|
pendingRedisWrites: Set<Promise<void>>;
|
2026-11-19 20:22:01 +06:00
|
|
|
deletedClientIds: Set<string>;
|
|
|
|
|
shutdownHooks: Set<() => Promise<unknown>>;
|
2026-11-17 20:05:00 +06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
type BufferGlobals = typeof globalThis & {
|
|
|
|
|
__dataStackBufferState?: BufferGlobalState;
|
2026-11-17 21:22:55 +06:00
|
|
|
__dataStackBufferShutdownRegistered?: boolean;
|
2026-11-18 05:42:19 +06:00
|
|
|
__dataStackBufferShuttingDown?: Promise<void> | null;
|
2026-11-17 20:05:00 +06:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const globals = globalThis as BufferGlobals;
|
|
|
|
|
|
|
|
|
|
function getState(): BufferGlobalState {
|
|
|
|
|
if (!globals.__dataStackBufferState) {
|
|
|
|
|
globals.__dataStackBufferState = {
|
|
|
|
|
buffer: [],
|
|
|
|
|
recentEvents: [],
|
|
|
|
|
batchFlushThreshold: 500,
|
|
|
|
|
batchFlushIntervalMs: 2000,
|
|
|
|
|
flushTimer: null,
|
|
|
|
|
subscribers: new Set(),
|
|
|
|
|
flushListeners: new Set(),
|
|
|
|
|
totalIngested: 0,
|
|
|
|
|
totalFlushed: 0,
|
|
|
|
|
isFlushing: false,
|
2026-11-19 20:22:01 +06:00
|
|
|
activeFlushPromise: null,
|
2026-11-18 05:18:55 +06:00
|
|
|
redisClient: undefined,
|
|
|
|
|
redisUrlConfigured: '',
|
|
|
|
|
redisConnected: false,
|
|
|
|
|
redisTestOverride: undefined,
|
|
|
|
|
pendingRedisWrites: new Set(),
|
2026-11-19 20:22:01 +06:00
|
|
|
deletedClientIds: new Set(),
|
|
|
|
|
shutdownHooks: new Set(),
|
2026-11-17 20:05:00 +06:00
|
|
|
};
|
|
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
if (!globals.__dataStackBufferState.deletedClientIds) {
|
|
|
|
|
globals.__dataStackBufferState.deletedClientIds = new Set();
|
|
|
|
|
}
|
|
|
|
|
if (!globals.__dataStackBufferState.shutdownHooks) {
|
|
|
|
|
globals.__dataStackBufferState.shutdownHooks = new Set();
|
|
|
|
|
}
|
2026-11-17 20:05:00 +06:00
|
|
|
return globals.__dataStackBufferState;
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
function getRedisClient(): RedisBufferClient | null {
|
|
|
|
|
const state = getState();
|
|
|
|
|
if (state.redisTestOverride !== undefined) {
|
|
|
|
|
return state.redisTestOverride;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const configuredUrl = (process.env.REDIS_URL || '').trim();
|
|
|
|
|
if (!configuredUrl) {
|
|
|
|
|
if (state.redisClient && typeof state.redisClient.disconnect === 'function') {
|
|
|
|
|
try {
|
|
|
|
|
state.redisClient.disconnect();
|
|
|
|
|
} catch {
|
|
|
|
|
// ignore
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
state.redisClient = null;
|
|
|
|
|
state.redisUrlConfigured = '';
|
|
|
|
|
state.redisConnected = false;
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (state.redisClient && state.redisUrlConfigured === configuredUrl) {
|
|
|
|
|
return state.redisClient;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (state.redisClient && typeof state.redisClient.disconnect === 'function') {
|
|
|
|
|
try {
|
|
|
|
|
state.redisClient.disconnect();
|
|
|
|
|
} catch {
|
|
|
|
|
// ignore
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
const client = new Redis(configuredUrl, {
|
|
|
|
|
maxRetriesPerRequest: 2,
|
|
|
|
|
enableReadyCheck: true,
|
|
|
|
|
retryStrategy: (times: number) => Math.min(times * 250, 3000),
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
client.on('ready', () => {
|
|
|
|
|
state.redisConnected = true;
|
|
|
|
|
});
|
|
|
|
|
client.on('connect', () => {
|
|
|
|
|
state.redisConnected = true;
|
|
|
|
|
});
|
|
|
|
|
client.on('close', () => {
|
|
|
|
|
state.redisConnected = false;
|
|
|
|
|
});
|
|
|
|
|
client.on('end', () => {
|
|
|
|
|
state.redisConnected = false;
|
|
|
|
|
});
|
|
|
|
|
client.on('error', () => {
|
|
|
|
|
state.redisConnected = false;
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
state.redisClient = client as unknown as RedisBufferClient;
|
|
|
|
|
state.redisUrlConfigured = configuredUrl;
|
|
|
|
|
state.redisConnected = client.status === 'ready' || client.status === 'connect';
|
|
|
|
|
return state.redisClient;
|
|
|
|
|
} catch {
|
|
|
|
|
state.redisClient = null;
|
|
|
|
|
state.redisUrlConfigured = configuredUrl;
|
|
|
|
|
state.redisConnected = false;
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-10-23 11:43:49 -05:00
|
|
|
export class DataBufferEngine {
|
2026-11-19 20:22:01 +06:00
|
|
|
public static getRecentLimit(): number {
|
|
|
|
|
return resolveRedisRecentLimit(getState().batchFlushThreshold);
|
|
|
|
|
}
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-11-17 20:05:00 +06:00
|
|
|
public static get batchFlushThreshold(): number {
|
|
|
|
|
return getState().batchFlushThreshold;
|
|
|
|
|
}
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-11-17 20:05:00 +06:00
|
|
|
public static set batchFlushThreshold(value: number) {
|
|
|
|
|
getState().batchFlushThreshold = Math.max(1, Number(value) || 500);
|
|
|
|
|
}
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-11-17 20:05:00 +06:00
|
|
|
public static get batchFlushIntervalMs(): number {
|
|
|
|
|
return getState().batchFlushIntervalMs;
|
|
|
|
|
}
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-11-17 20:05:00 +06:00
|
|
|
public static set batchFlushIntervalMs(value: number) {
|
|
|
|
|
this.setFlushInterval(value);
|
2026-10-23 11:43:49 -05:00
|
|
|
}
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
public static setRedisClientForTests(client: RedisBufferClient | null | undefined): void {
|
|
|
|
|
const state = getState();
|
|
|
|
|
state.redisTestOverride = client;
|
|
|
|
|
state.redisConnected = Boolean(client);
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
public static getRedisClient(): RedisBufferClient | null {
|
|
|
|
|
return getRedisClient();
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
public static async shutdown(): Promise<void> {
|
|
|
|
|
const state = getState();
|
|
|
|
|
if (state.flushTimer) {
|
|
|
|
|
clearInterval(state.flushTimer);
|
|
|
|
|
state.flushTimer = null;
|
|
|
|
|
}
|
|
|
|
|
if (state.pendingRedisWrites.size > 0) {
|
|
|
|
|
await Promise.allSettled([...state.pendingRedisWrites]);
|
|
|
|
|
}
|
|
|
|
|
const client = state.redisClient;
|
|
|
|
|
state.redisClient = undefined;
|
|
|
|
|
state.redisUrlConfigured = '';
|
|
|
|
|
state.redisConnected = false;
|
|
|
|
|
state.redisTestOverride = undefined;
|
2026-11-19 20:22:01 +06:00
|
|
|
state.deletedClientIds.clear();
|
2026-11-18 05:18:55 +06:00
|
|
|
if (client) {
|
|
|
|
|
try {
|
|
|
|
|
if (typeof client.quit === 'function') {
|
|
|
|
|
await client.quit();
|
|
|
|
|
} else if (typeof client.disconnect === 'function') {
|
|
|
|
|
client.disconnect();
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
if (typeof client.disconnect === 'function') {
|
|
|
|
|
client.disconnect();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
public static registerShutdownHook(hook: () => Promise<unknown>): () => void {
|
|
|
|
|
const state = getState();
|
|
|
|
|
state.shutdownHooks.add(hook);
|
|
|
|
|
return () => {
|
|
|
|
|
state.shutdownHooks.delete(hook);
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-18 05:42:19 +06:00
|
|
|
public static async flushAndShutdownOnSignal(): Promise<void> {
|
|
|
|
|
if (globals.__dataStackBufferShuttingDown) {
|
|
|
|
|
return globals.__dataStackBufferShuttingDown;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const run = (async () => {
|
|
|
|
|
try {
|
2026-11-19 20:22:01 +06:00
|
|
|
const hooks = [...getState().shutdownHooks];
|
2026-11-18 05:42:19 +06:00
|
|
|
await Promise.race([
|
2026-11-19 20:22:01 +06:00
|
|
|
Promise.allSettled([
|
|
|
|
|
this.flush().catch(() => undefined),
|
|
|
|
|
...hooks.map((hook) =>
|
|
|
|
|
Promise.resolve()
|
|
|
|
|
.then(() => hook())
|
|
|
|
|
.catch(() => undefined),
|
|
|
|
|
),
|
|
|
|
|
]),
|
2026-11-18 05:42:19 +06:00
|
|
|
new Promise<void>((resolve) => setTimeout(resolve, 4000)),
|
|
|
|
|
]);
|
|
|
|
|
} finally {
|
|
|
|
|
await Promise.race([
|
|
|
|
|
this.shutdown().catch(() => undefined),
|
|
|
|
|
new Promise<void>((resolve) => setTimeout(resolve, 1500)),
|
|
|
|
|
]);
|
|
|
|
|
globals.__dataStackBufferShuttingDown = null;
|
|
|
|
|
}
|
|
|
|
|
})();
|
|
|
|
|
|
|
|
|
|
globals.__dataStackBufferShuttingDown = run;
|
|
|
|
|
return run;
|
|
|
|
|
}
|
|
|
|
|
|
2026-10-23 11:43:49 -05:00
|
|
|
public static init() {
|
2026-11-17 20:05:00 +06:00
|
|
|
const state = getState();
|
|
|
|
|
if (!state.flushTimer) {
|
|
|
|
|
state.flushTimer = setInterval(() => {
|
2026-11-18 05:18:55 +06:00
|
|
|
void this.flush().catch(() => undefined);
|
2026-11-17 20:05:00 +06:00
|
|
|
}, state.batchFlushIntervalMs);
|
2026-10-23 11:43:49 -05:00
|
|
|
}
|
2026-11-17 21:22:55 +06:00
|
|
|
|
|
|
|
|
if (
|
|
|
|
|
typeof process !== 'undefined' &&
|
2026-11-18 05:42:19 +06:00
|
|
|
typeof process.once === 'function' &&
|
2026-11-17 21:22:55 +06:00
|
|
|
!globals.__dataStackBufferShutdownRegistered
|
|
|
|
|
) {
|
|
|
|
|
globals.__dataStackBufferShutdownRegistered = true;
|
|
|
|
|
const flushOnExit = () => {
|
2026-11-18 05:42:19 +06:00
|
|
|
void this.flushAndShutdownOnSignal().finally(() => {
|
|
|
|
|
if (
|
|
|
|
|
process.env.NODE_ENV === 'production' &&
|
|
|
|
|
!process.execArgv.includes('--test')
|
|
|
|
|
) {
|
|
|
|
|
process.exit(0);
|
|
|
|
|
}
|
|
|
|
|
});
|
2026-11-17 21:22:55 +06:00
|
|
|
};
|
2026-11-18 05:42:19 +06:00
|
|
|
process.once('SIGTERM', flushOnExit);
|
|
|
|
|
process.once('SIGINT', flushOnExit);
|
2026-11-17 21:22:55 +06:00
|
|
|
}
|
2026-10-23 11:43:49 -05:00
|
|
|
}
|
|
|
|
|
|
2026-11-17 20:05:00 +06:00
|
|
|
/**
|
|
|
|
|
* Updates flush interval and immediately restarts the active interval timer.
|
|
|
|
|
*/
|
|
|
|
|
public static setFlushInterval(intervalMs: number) {
|
|
|
|
|
const state = getState();
|
|
|
|
|
const nextInterval = Math.max(100, Number(intervalMs) || 2000);
|
|
|
|
|
state.batchFlushIntervalMs = nextInterval;
|
|
|
|
|
|
|
|
|
|
if (state.flushTimer) {
|
|
|
|
|
clearInterval(state.flushTimer);
|
|
|
|
|
state.flushTimer = null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
state.flushTimer = setInterval(() => {
|
2026-11-18 05:18:55 +06:00
|
|
|
void this.flush().catch(() => undefined);
|
2026-11-17 20:05:00 +06:00
|
|
|
}, state.batchFlushIntervalMs);
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
private static recordLocalMetaAndNotify(event: DataEvent): void {
|
2026-11-17 20:05:00 +06:00
|
|
|
const state = getState();
|
|
|
|
|
state.totalIngested += 1;
|
|
|
|
|
state.recentEvents.unshift(event);
|
2026-11-19 20:22:01 +06:00
|
|
|
const recentLimit = this.getRecentLimit();
|
|
|
|
|
while (state.recentEvents.length > recentLimit) {
|
2026-11-17 20:05:00 +06:00
|
|
|
state.recentEvents.pop();
|
2026-10-23 11:43:49 -05:00
|
|
|
}
|
|
|
|
|
|
2026-11-17 20:05:00 +06:00
|
|
|
for (const sub of state.subscribers) {
|
2026-10-23 11:43:49 -05:00
|
|
|
try {
|
|
|
|
|
sub(event);
|
|
|
|
|
} catch {
|
2026-11-17 20:05:00 +06:00
|
|
|
// Ignore subscriber error
|
2026-10-23 11:43:49 -05:00
|
|
|
}
|
|
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
private static async writeEventsToRedisOrFallback(events: DataEvent[]): Promise<number> {
|
|
|
|
|
if (events.length === 0) return 0;
|
|
|
|
|
const state = getState();
|
|
|
|
|
const redis = getRedisClient();
|
|
|
|
|
|
|
|
|
|
if (redis) {
|
|
|
|
|
const serialized = events.map((e) => JSON.stringify(e));
|
2026-11-19 20:22:01 +06:00
|
|
|
let bufferLen: number | null = null;
|
2026-11-18 05:18:55 +06:00
|
|
|
try {
|
2026-11-19 20:22:01 +06:00
|
|
|
bufferLen = await redis.rpush(REDIS_BUFFER_KEY, ...serialized);
|
2026-11-18 05:18:55 +06:00
|
|
|
state.redisConnected = true;
|
|
|
|
|
} catch {
|
|
|
|
|
state.redisConnected = false;
|
|
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
|
|
|
|
|
if (bufferLen !== null) {
|
|
|
|
|
// Best-effort recent list mirror: never duplicate into RAM buffer once rpush succeeded
|
|
|
|
|
try {
|
|
|
|
|
const recentLimit = this.getRecentLimit();
|
|
|
|
|
await redis.lpush(REDIS_RECENT_KEY, ...serialized);
|
|
|
|
|
await redis.ltrim(REDIS_RECENT_KEY, 0, recentLimit - 1);
|
|
|
|
|
} catch {
|
|
|
|
|
// Non-fatal: events are already committed in REDIS_BUFFER_KEY and state.recentEvents
|
|
|
|
|
}
|
|
|
|
|
return Number(bufferLen) + state.buffer.length;
|
|
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
state.buffer.push(...events);
|
|
|
|
|
return state.buffer.length;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Synchronous-signature push (queues to Redis asynchronously when configured, with automatic RAM fallback).
|
|
|
|
|
*/
|
|
|
|
|
public static push(event: DataEvent): void {
|
|
|
|
|
const state = getState();
|
|
|
|
|
this.recordLocalMetaAndNotify(event);
|
|
|
|
|
|
|
|
|
|
const redis = getRedisClient();
|
|
|
|
|
if (redis) {
|
|
|
|
|
const op = this.writeEventsToRedisOrFallback([event]).then((queuedCount) => {
|
|
|
|
|
if (queuedCount >= state.batchFlushThreshold) {
|
|
|
|
|
void this.flush().catch(() => undefined);
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
state.pendingRedisWrites.add(op);
|
|
|
|
|
void op.finally(() => {
|
|
|
|
|
state.pendingRedisWrites.delete(op);
|
|
|
|
|
});
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
state.buffer.push(event);
|
2026-11-17 20:05:00 +06:00
|
|
|
if (state.buffer.length >= state.batchFlushThreshold) {
|
2026-11-18 05:18:55 +06:00
|
|
|
void this.flush().catch(() => undefined);
|
2026-10-23 11:43:49 -05:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
2026-11-18 05:18:55 +06:00
|
|
|
* Async single-event push (awaited by API routes so Redis persistence is guaranteed before HTTP response).
|
|
|
|
|
*/
|
|
|
|
|
public static async pushAsync(event: DataEvent): Promise<void> {
|
|
|
|
|
await this.pushBatchAsync([event]);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Async batch push with shared Redis durability and RAM fallback.
|
|
|
|
|
*/
|
|
|
|
|
public static async pushBatchAsync(events: DataEvent[]): Promise<void> {
|
|
|
|
|
if (!events || events.length === 0) return;
|
|
|
|
|
const state = getState();
|
|
|
|
|
for (const evt of events) {
|
|
|
|
|
this.recordLocalMetaAndNotify(evt);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const queuedCount = await this.writeEventsToRedisOrFallback(events);
|
|
|
|
|
if (queuedCount >= state.batchFlushThreshold) {
|
|
|
|
|
void this.flush().catch(() => undefined);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
private static async isClientDeletedForBuffer(
|
|
|
|
|
clientId: string,
|
|
|
|
|
redis: RedisBufferClient | null,
|
|
|
|
|
cache?: Map<string, boolean>,
|
|
|
|
|
): Promise<boolean> {
|
|
|
|
|
const normalized = normalizeBufferClientId(clientId);
|
|
|
|
|
if (!normalized) return false;
|
2026-11-17 20:05:00 +06:00
|
|
|
const state = getState();
|
2026-11-19 20:22:01 +06:00
|
|
|
if (state.deletedClientIds.has(normalized)) return true;
|
|
|
|
|
if (cache && cache.has(normalized)) return Boolean(cache.get(normalized));
|
|
|
|
|
if (redis && typeof redis.sismember === 'function') {
|
|
|
|
|
try {
|
|
|
|
|
const isDeleted =
|
|
|
|
|
Number(await redis.sismember(REDIS_CLIENTS_DELETED_KEY, normalized)) === 1;
|
|
|
|
|
if (cache) cache.set(normalized, isDeleted);
|
|
|
|
|
if (isDeleted) state.deletedClientIds.add(normalized);
|
|
|
|
|
return isDeleted;
|
|
|
|
|
} catch {
|
|
|
|
|
// Ignore transient Redis tombstone lookup error
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (cache) cache.set(normalized, false);
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Marks a client deleted in the buffer layer and immediately strips queued local events.
|
|
|
|
|
*/
|
|
|
|
|
public static markClientDeleted(clientId: string): void {
|
|
|
|
|
const normalized = normalizeBufferClientId(clientId);
|
|
|
|
|
if (!normalized) return;
|
|
|
|
|
const state = getState();
|
|
|
|
|
state.deletedClientIds.add(normalized);
|
|
|
|
|
state.buffer = state.buffer.filter(
|
|
|
|
|
(evt) => normalizeBufferClientId(evt.clientId) !== normalized,
|
|
|
|
|
);
|
|
|
|
|
state.recentEvents = state.recentEvents.filter(
|
|
|
|
|
(evt) => normalizeBufferClientId(evt.clientId) !== normalized,
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Clears deleted tombstone when a client ID is explicitly re-registered.
|
|
|
|
|
*/
|
|
|
|
|
public static unmarkClientDeleted(clientId: string): void {
|
|
|
|
|
const normalized = normalizeBufferClientId(clientId);
|
|
|
|
|
if (!normalized) return;
|
|
|
|
|
getState().deletedClientIds.delete(normalized);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static async filterRedisListExcludingClient(
|
|
|
|
|
redis: RedisBufferClient,
|
|
|
|
|
listKey: string,
|
|
|
|
|
normalizedClientId: string,
|
|
|
|
|
): Promise<number> {
|
|
|
|
|
const removed = await redis.eval(
|
|
|
|
|
PURGE_CLIENT_LIST_LUA_SCRIPT,
|
|
|
|
|
1,
|
|
|
|
|
listKey,
|
|
|
|
|
normalizedClientId,
|
|
|
|
|
);
|
|
|
|
|
return Number(removed) || 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Purges all queued events for a deleted client from both local RAM and shared Redis queues (BUG-19).
|
|
|
|
|
*/
|
|
|
|
|
public static async purgeClientEventsAsync(
|
|
|
|
|
clientId: string,
|
|
|
|
|
): Promise<{ removedLocal: number; removedRedis: number }> {
|
|
|
|
|
const normalized = normalizeBufferClientId(clientId);
|
|
|
|
|
if (!normalized) return { removedLocal: 0, removedRedis: 0 };
|
|
|
|
|
|
|
|
|
|
const state = getState();
|
|
|
|
|
state.deletedClientIds.add(normalized);
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
if (state.pendingRedisWrites.size > 0) {
|
|
|
|
|
await Promise.allSettled([...state.pendingRedisWrites]);
|
|
|
|
|
}
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
const beforeLocal = state.buffer.length;
|
|
|
|
|
state.buffer = state.buffer.filter(
|
|
|
|
|
(evt) => normalizeBufferClientId(evt.clientId) !== normalized,
|
|
|
|
|
);
|
|
|
|
|
const removedLocal = beforeLocal - state.buffer.length;
|
|
|
|
|
state.recentEvents = state.recentEvents.filter(
|
|
|
|
|
(evt) => normalizeBufferClientId(evt.clientId) !== normalized,
|
|
|
|
|
);
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
let removedRedis = 0;
|
|
|
|
|
const redis = getRedisClient();
|
|
|
|
|
if (redis) {
|
|
|
|
|
const lockToken = crypto.randomUUID();
|
|
|
|
|
let lockAcquired = false;
|
|
|
|
|
const deadline = Date.now() + 1500;
|
|
|
|
|
while (true) {
|
2026-10-23 11:43:49 -05:00
|
|
|
try {
|
2026-11-19 20:22:01 +06:00
|
|
|
const lockRes = await redis.set(
|
2026-11-18 05:18:55 +06:00
|
|
|
REDIS_FLUSH_LOCK_KEY,
|
|
|
|
|
lockToken,
|
|
|
|
|
'PX',
|
|
|
|
|
FLUSH_LOCK_TTL_MS,
|
|
|
|
|
'NX',
|
|
|
|
|
);
|
|
|
|
|
state.redisConnected = true;
|
2026-11-19 20:22:01 +06:00
|
|
|
if (lockRes === 'OK') {
|
|
|
|
|
lockAcquired = true;
|
|
|
|
|
break;
|
|
|
|
|
}
|
2026-10-23 11:43:49 -05:00
|
|
|
} catch {
|
2026-11-18 05:18:55 +06:00
|
|
|
state.redisConnected = false;
|
2026-11-19 20:22:01 +06:00
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
const remaining = deadline - Date.now();
|
|
|
|
|
if (remaining <= 0) break;
|
|
|
|
|
await sleepMs(Math.min(FLUSH_LOCK_RETRY_MS, remaining));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (lockAcquired) {
|
|
|
|
|
try {
|
|
|
|
|
removedRedis += await this.filterRedisListExcludingClient(
|
|
|
|
|
redis,
|
|
|
|
|
REDIS_BUFFER_KEY,
|
|
|
|
|
normalized,
|
|
|
|
|
);
|
|
|
|
|
await this.filterRedisListExcludingClient(
|
|
|
|
|
redis,
|
|
|
|
|
REDIS_RECENT_KEY,
|
|
|
|
|
normalized,
|
|
|
|
|
).catch(() => 0);
|
|
|
|
|
} catch {
|
|
|
|
|
// Flush-time deletedClientIds/tombstone filter remains active as secondary guard
|
|
|
|
|
} finally {
|
|
|
|
|
try {
|
|
|
|
|
await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
|
|
|
|
|
} catch {
|
|
|
|
|
// Lock expires via PX TTL
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return { removedLocal, removedRedis };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Flushes queued events (shared Redis list under distributed lock + local RAM fallback) to partitioned CSV files.
|
|
|
|
|
* Redis LTRIM runs ONLY after CSVStorageManager.appendBatch succeeds and lock ownership is verified.
|
|
|
|
|
*/
|
|
|
|
|
public static async flush(options?: FlushOptions): Promise<FlushResult> {
|
|
|
|
|
const state = getState();
|
|
|
|
|
if (state.pendingRedisWrites.size > 0) {
|
|
|
|
|
await Promise.allSettled([...state.pendingRedisWrites]);
|
|
|
|
|
}
|
|
|
|
|
if (state.activeFlushPromise) {
|
|
|
|
|
return state.activeFlushPromise;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const executeFlush = async (): Promise<FlushResult> => {
|
2026-11-22 15:33:14 +06:00
|
|
|
|
|
|
|
|
// P02 R-02: examine CSV partitions before replay of an untrimmed Redis
|
|
|
|
|
// batch. Streaming parsing limits memory and handles escaped/newline CSV.
|
|
|
|
|
const excludeAlreadyPersisted = async (events: DataEvent[]): Promise<DataEvent[]> => {
|
|
|
|
|
if (!events.length) return events;
|
|
|
|
|
const fs = await import('node:fs');
|
|
|
|
|
const path = await import('node:path');
|
|
|
|
|
const zlib = await import('node:zlib');
|
|
|
|
|
const { pipeline } = await import('node:stream/promises');
|
|
|
|
|
const { Writable } = await import('node:stream');
|
|
|
|
|
const { parse } = await import('csv-parse');
|
|
|
|
|
const groups = new Map<string, {
|
|
|
|
|
clientId: string; date: string; ids: Set<string>;
|
|
|
|
|
}>();
|
|
|
|
|
for (const event of events) {
|
|
|
|
|
const clientId = String(event.clientId || 'unknown');
|
|
|
|
|
const date = CSVStorageManager.extractSafePartitionDate(event.timestamp);
|
|
|
|
|
const key = JSON.stringify([clientId, date]);
|
|
|
|
|
if (!groups.has(key)) groups.set(key, { clientId, date, ids: new Set() });
|
|
|
|
|
groups.get(key)!.ids.add(String(event.eventId));
|
|
|
|
|
}
|
|
|
|
|
const persisted = new Set<string>();
|
|
|
|
|
for (const [groupKey, group] of groups) {
|
|
|
|
|
const dir = CSVStorageManager.ensureClientDir(group.clientId);
|
|
|
|
|
const entries = await fs.promises.readdir(dir, { withFileTypes: true });
|
|
|
|
|
for (const entry of entries) {
|
|
|
|
|
if (!entry.isFile() ||
|
|
|
|
|
!/\.csv(?:\.gz)?$/.test(entry.name) ||
|
|
|
|
|
CSVStorageManager.extractSafePartitionDate(
|
|
|
|
|
entry.name.replace(/\.csv(?:\.gz)?$/, '').replace(/(?:-(?:part|gz)\d+)+$/, ''),
|
|
|
|
|
) !== group.date) continue;
|
|
|
|
|
const reader = fs.createReadStream(path.join(dir, entry.name));
|
|
|
|
|
const parser = parse({ columns: true, bom: true, skip_empty_lines: true });
|
|
|
|
|
const sink = new Writable({
|
|
|
|
|
objectMode: true,
|
|
|
|
|
write(row: { clientId?: string; eventId?: string }, _enc, done) {
|
|
|
|
|
if (row.clientId === group.clientId && group.ids.has(String(row.eventId))) {
|
|
|
|
|
persisted.add(JSON.stringify([groupKey, String(row.eventId)]));
|
|
|
|
|
}
|
|
|
|
|
done();
|
|
|
|
|
},
|
|
|
|
|
});
|
|
|
|
|
if (entry.name.endsWith('.gz')) {
|
|
|
|
|
await pipeline(reader, zlib.createGunzip(), parser, sink);
|
|
|
|
|
} else {
|
|
|
|
|
await pipeline(reader, parser, sink);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
const batchSeen = new Set<string>();
|
|
|
|
|
return events.filter((event) => {
|
|
|
|
|
const key = JSON.stringify([String(event.clientId || 'unknown'),
|
|
|
|
|
CSVStorageManager.extractSafePartitionDate(event.timestamp)]);
|
|
|
|
|
const rowKey = JSON.stringify([key, String(event.eventId)]);
|
|
|
|
|
if (persisted.has(rowKey) || batchSeen.has(rowKey)) return false;
|
|
|
|
|
batchSeen.add(rowKey);
|
|
|
|
|
return true;
|
|
|
|
|
});
|
|
|
|
|
};
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
state.isFlushing = true;
|
|
|
|
|
let totalWritten = 0;
|
|
|
|
|
let lockTimedOut = false;
|
|
|
|
|
const filesSet = new Set<string>();
|
|
|
|
|
const deletedCheckCache = new Map<string, boolean>();
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
if (state.pendingRedisWrites.size > 0) {
|
|
|
|
|
await Promise.allSettled([...state.pendingRedisWrites]);
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
const redis = getRedisClient();
|
|
|
|
|
if (redis) {
|
|
|
|
|
const lockToken = crypto.randomUUID();
|
|
|
|
|
const lockTtlMs = Math.max(50, Number(options?.lockTtlMs) || FLUSH_LOCK_TTL_MS);
|
|
|
|
|
const waitMs = Math.max(
|
|
|
|
|
0,
|
|
|
|
|
options?.waitMs !== undefined ? Number(options.waitMs) : FLUSH_LOCK_WAIT_MS,
|
|
|
|
|
);
|
|
|
|
|
const deadline = Date.now() + waitMs;
|
|
|
|
|
let lockAcquired = false;
|
|
|
|
|
let redisError = false;
|
|
|
|
|
|
|
|
|
|
while (true) {
|
|
|
|
|
try {
|
|
|
|
|
const lockResult = await redis.set(
|
|
|
|
|
REDIS_FLUSH_LOCK_KEY,
|
|
|
|
|
lockToken,
|
|
|
|
|
'PX',
|
|
|
|
|
lockTtlMs,
|
|
|
|
|
'NX',
|
|
|
|
|
);
|
|
|
|
|
state.redisConnected = true;
|
|
|
|
|
if (lockResult === 'OK') {
|
|
|
|
|
lockAcquired = true;
|
2026-11-18 05:18:55 +06:00
|
|
|
break;
|
|
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
} catch {
|
|
|
|
|
state.redisConnected = false;
|
|
|
|
|
redisError = true;
|
|
|
|
|
break;
|
|
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
const remaining = deadline - Date.now();
|
|
|
|
|
if (remaining <= 0) {
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
await sleepMs(Math.min(FLUSH_LOCK_RETRY_MS, remaining));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!lockAcquired && !redisError) {
|
|
|
|
|
lockTimedOut = true;
|
|
|
|
|
if (options?.throwOnLockTimeout === true) {
|
|
|
|
|
throw new Error(
|
|
|
|
|
'FLUSH_LOCK_TIMEOUT: Timed out waiting for distributed Redis flush lock',
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (lockAcquired) {
|
|
|
|
|
const renewIntervalMs = Math.max(20, Math.floor(lockTtlMs / 3));
|
|
|
|
|
let lastRenewAt = Date.now();
|
|
|
|
|
let lockLost = false;
|
|
|
|
|
let renewInFlight: Promise<boolean> | null = null;
|
|
|
|
|
|
|
|
|
|
const renewLockOnce = (): Promise<boolean> => {
|
|
|
|
|
if (renewInFlight) return renewInFlight;
|
|
|
|
|
renewInFlight = (async () => {
|
2026-11-18 05:18:55 +06:00
|
|
|
try {
|
2026-11-19 20:22:01 +06:00
|
|
|
const renewed = await redis.eval(
|
|
|
|
|
RENEW_LOCK_LUA_SCRIPT,
|
|
|
|
|
1,
|
|
|
|
|
REDIS_FLUSH_LOCK_KEY,
|
|
|
|
|
lockToken,
|
|
|
|
|
String(lockTtlMs),
|
|
|
|
|
);
|
|
|
|
|
if (Number(renewed) === 1) {
|
|
|
|
|
lastRenewAt = Date.now();
|
|
|
|
|
return true;
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
lockLost = true;
|
|
|
|
|
return false;
|
2026-11-18 05:18:55 +06:00
|
|
|
} catch {
|
2026-11-19 20:22:01 +06:00
|
|
|
return false;
|
|
|
|
|
} finally {
|
|
|
|
|
renewInFlight = null;
|
|
|
|
|
}
|
|
|
|
|
})();
|
|
|
|
|
return renewInFlight;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const renewTimer = setInterval(() => {
|
|
|
|
|
void renewLockOnce();
|
|
|
|
|
}, renewIntervalMs);
|
|
|
|
|
if (typeof renewTimer.unref === 'function') {
|
|
|
|
|
renewTimer.unref();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
const chunkSize = Math.max(500, state.batchFlushThreshold);
|
|
|
|
|
while (true) {
|
|
|
|
|
if (lockLost) break;
|
|
|
|
|
|
|
|
|
|
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, chunkSize - 1);
|
|
|
|
|
if (!rawItems || rawItems.length === 0) {
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const parsedBatch: DataEvent[] = [];
|
|
|
|
|
let malformedCount = 0;
|
|
|
|
|
for (const raw of rawItems) {
|
|
|
|
|
try {
|
|
|
|
|
const evt = JSON.parse(raw) as DataEvent;
|
|
|
|
|
if (evt && evt.eventId) {
|
|
|
|
|
if (
|
|
|
|
|
await this.isClientDeletedForBuffer(
|
|
|
|
|
evt.clientId,
|
|
|
|
|
redis,
|
|
|
|
|
deletedCheckCache,
|
|
|
|
|
)
|
|
|
|
|
) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
parsedBatch.push(evt);
|
|
|
|
|
} else {
|
|
|
|
|
malformedCount += 1;
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
malformedCount += 1;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (malformedCount > 0) {
|
|
|
|
|
console.warn(
|
|
|
|
|
`[DataBufferEngine] Skipped ${malformedCount} malformed buffered Redis event(s) during flush`,
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (Date.now() - lastRenewAt >= renewIntervalMs) {
|
|
|
|
|
await renewLockOnce();
|
|
|
|
|
}
|
|
|
|
|
if (lockLost) break;
|
|
|
|
|
|
|
|
|
|
if (parsedBatch.length > 0) {
|
2026-11-22 15:33:14 +06:00
|
|
|
const unwritten = await excludeAlreadyPersisted(parsedBatch);
|
|
|
|
|
if (unwritten.length > 0) {
|
|
|
|
|
const res = await CSVStorageManager.appendBatch(unwritten);
|
|
|
|
|
totalWritten += res.written;
|
|
|
|
|
for (const f of res.filesUpdated) filesSet.add(f);
|
|
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (Date.now() - lastRenewAt >= renewIntervalMs) {
|
|
|
|
|
await renewLockOnce();
|
|
|
|
|
}
|
|
|
|
|
if (lockLost) {
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Trim strictly AFTER CSVStorageManager.appendBatch resolves and lock ownership holds
|
|
|
|
|
await redis.ltrim(REDIS_BUFFER_KEY, rawItems.length, -1);
|
|
|
|
|
|
|
|
|
|
if (rawItems.length < chunkSize) {
|
|
|
|
|
break;
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
|
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
} finally {
|
|
|
|
|
clearInterval(renewTimer);
|
|
|
|
|
if (renewInFlight) {
|
|
|
|
|
await renewInFlight.catch(() => false);
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
try {
|
|
|
|
|
await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
|
|
|
|
|
} catch {
|
|
|
|
|
// Lock auto-expires after lockTtlMs
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
|
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (state.buffer.length > 0) {
|
|
|
|
|
const ramBatch = [...state.buffer];
|
|
|
|
|
state.buffer = [];
|
|
|
|
|
const validRamBatch: DataEvent[] = [];
|
|
|
|
|
for (const evt of ramBatch) {
|
|
|
|
|
if (
|
|
|
|
|
await this.isClientDeletedForBuffer(
|
|
|
|
|
evt.clientId,
|
|
|
|
|
redis,
|
|
|
|
|
deletedCheckCache,
|
|
|
|
|
)
|
|
|
|
|
) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
validRamBatch.push(evt);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (validRamBatch.length > 0) {
|
2026-11-18 05:18:55 +06:00
|
|
|
try {
|
2026-11-22 15:33:14 +06:00
|
|
|
const unwritten = await excludeAlreadyPersisted(validRamBatch);
|
|
|
|
|
if (unwritten.length > 0) {
|
|
|
|
|
const res = await CSVStorageManager.appendBatch(unwritten);
|
|
|
|
|
totalWritten += res.written;
|
|
|
|
|
for (const f of res.filesUpdated) filesSet.add(f);
|
|
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
} catch (err) {
|
|
|
|
|
state.buffer.unshift(...validRamBatch);
|
|
|
|
|
throw err;
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
|
|
|
|
}
|
2026-10-23 11:43:49 -05:00
|
|
|
}
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
if (totalWritten > 0) {
|
|
|
|
|
state.totalFlushed += totalWritten;
|
|
|
|
|
const filesUpdated = Array.from(filesSet);
|
|
|
|
|
for (const listener of state.flushListeners) {
|
|
|
|
|
try {
|
|
|
|
|
listener(totalWritten, filesUpdated);
|
|
|
|
|
} catch {
|
|
|
|
|
// ignore
|
|
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
return {
|
|
|
|
|
count: totalWritten,
|
|
|
|
|
files: Array.from(filesSet),
|
|
|
|
|
...(lockTimedOut ? { lockTimedOut: true } : {}),
|
|
|
|
|
};
|
|
|
|
|
} finally {
|
|
|
|
|
state.isFlushing = false;
|
|
|
|
|
state.activeFlushPromise = null;
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
let flushDone = false;
|
|
|
|
|
let promise: Promise<FlushResult> | null = null;
|
|
|
|
|
promise = (async () => {
|
|
|
|
|
try {
|
|
|
|
|
return await executeFlush();
|
|
|
|
|
} finally {
|
|
|
|
|
flushDone = true;
|
|
|
|
|
state.isFlushing = false;
|
|
|
|
|
if (state.activeFlushPromise === promise) {
|
|
|
|
|
state.activeFlushPromise = null;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
})();
|
|
|
|
|
if (!flushDone) {
|
|
|
|
|
state.activeFlushPromise = promise;
|
2026-10-23 11:43:49 -05:00
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
return promise;
|
2026-10-23 11:43:49 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Subscribe to real-time events (for UI monitoring)
|
|
|
|
|
*/
|
|
|
|
|
public static subscribe(subscriber: EventSubscriber): () => void {
|
2026-11-17 20:05:00 +06:00
|
|
|
const state = getState();
|
|
|
|
|
state.subscribers.add(subscriber);
|
2026-10-23 11:43:49 -05:00
|
|
|
return () => {
|
2026-11-17 20:05:00 +06:00
|
|
|
state.subscribers.delete(subscriber);
|
2026-10-23 11:43:49 -05:00
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static onFlush(callback: FlushCallback): () => void {
|
2026-11-17 20:05:00 +06:00
|
|
|
const state = getState();
|
|
|
|
|
state.flushListeners.add(callback);
|
2026-10-23 11:43:49 -05:00
|
|
|
return () => {
|
2026-11-17 20:05:00 +06:00
|
|
|
state.flushListeners.delete(callback);
|
2026-10-23 11:43:49 -05:00
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
public static getLocalBufferSnapshot(limit?: number): DataEvent[] {
|
|
|
|
|
const state = getState();
|
|
|
|
|
if (typeof limit === 'number' && Number.isFinite(limit) && limit >= 0) {
|
|
|
|
|
const cap = Math.floor(limit);
|
|
|
|
|
return cap === 0 ? [] : state.buffer.slice(-cap);
|
|
|
|
|
}
|
|
|
|
|
return [...state.buffer];
|
|
|
|
|
}
|
|
|
|
|
|
2026-10-23 11:43:49 -05:00
|
|
|
public static getRecentEvents(limit = 50, filterClientId?: string): DataEvent[] {
|
2026-11-17 20:05:00 +06:00
|
|
|
const state = getState();
|
2026-10-23 11:43:49 -05:00
|
|
|
if (filterClientId && filterClientId !== 'all') {
|
2026-11-17 20:05:00 +06:00
|
|
|
return state.recentEvents
|
|
|
|
|
.filter((e) => e.clientId === filterClientId)
|
|
|
|
|
.slice(0, limit);
|
2026-10-23 11:43:49 -05:00
|
|
|
}
|
2026-11-17 20:05:00 +06:00
|
|
|
return state.recentEvents.slice(0, limit);
|
2026-10-23 11:43:49 -05:00
|
|
|
}
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
public static async getRecentEventsAsync(
|
|
|
|
|
limit = 50,
|
|
|
|
|
filterClientId?: string,
|
|
|
|
|
): Promise<DataEvent[]> {
|
|
|
|
|
const state = getState();
|
|
|
|
|
const redis = getRedisClient();
|
|
|
|
|
|
|
|
|
|
if (redis) {
|
|
|
|
|
try {
|
2026-11-19 20:22:01 +06:00
|
|
|
const recentLimit = this.getRecentLimit();
|
|
|
|
|
const readCap = Math.min(recentLimit, Math.max(limit * 4, limit));
|
2026-11-18 05:18:55 +06:00
|
|
|
const rawItems = await redis.lrange(REDIS_RECENT_KEY, 0, readCap - 1);
|
|
|
|
|
state.redisConnected = true;
|
|
|
|
|
|
|
|
|
|
const redisEvents: DataEvent[] = [];
|
|
|
|
|
for (const raw of rawItems) {
|
|
|
|
|
try {
|
|
|
|
|
const parsed = JSON.parse(raw) as DataEvent;
|
|
|
|
|
if (!parsed || !parsed.eventId) continue;
|
2026-11-19 20:22:01 +06:00
|
|
|
if (state.deletedClientIds.has(normalizeBufferClientId(parsed.clientId))) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
if (filterClientId && filterClientId !== 'all' && parsed.clientId !== filterClientId) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
redisEvents.push(parsed);
|
|
|
|
|
if (redisEvents.length >= limit) break;
|
|
|
|
|
} catch {
|
|
|
|
|
// skip malformed JSON entry
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (redisEvents.length > 0 || state.recentEvents.length === 0) {
|
|
|
|
|
return redisEvents;
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
state.redisConnected = false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return this.getRecentEvents(limit, filterClientId);
|
|
|
|
|
}
|
|
|
|
|
|
2026-10-23 11:43:49 -05:00
|
|
|
public static getBufferStats() {
|
2026-11-17 20:05:00 +06:00
|
|
|
const state = getState();
|
2026-10-23 11:43:49 -05:00
|
|
|
return {
|
2026-11-17 20:05:00 +06:00
|
|
|
currentBufferSize: state.buffer.length,
|
2026-11-18 05:18:55 +06:00
|
|
|
localBufferSize: state.buffer.length,
|
|
|
|
|
redisBufferSize: 0,
|
|
|
|
|
redisEnabled: Boolean(getRedisClient()),
|
|
|
|
|
redisConnected: state.redisConnected,
|
|
|
|
|
batchFlushThreshold: state.batchFlushThreshold,
|
|
|
|
|
batchFlushIntervalMs: state.batchFlushIntervalMs,
|
|
|
|
|
totalIngested: state.totalIngested,
|
|
|
|
|
totalFlushed: state.totalFlushed,
|
|
|
|
|
recentEventsCached: state.recentEvents.length,
|
|
|
|
|
activeSubscribers: state.subscribers.size,
|
|
|
|
|
isFlushing: state.isFlushing,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static async getBufferStatsAsync() {
|
|
|
|
|
const state = getState();
|
|
|
|
|
const redis = getRedisClient();
|
|
|
|
|
let redisBufferSize = 0;
|
|
|
|
|
|
|
|
|
|
if (redis) {
|
|
|
|
|
try {
|
|
|
|
|
redisBufferSize = Number(await redis.llen(REDIS_BUFFER_KEY)) || 0;
|
|
|
|
|
state.redisConnected = true;
|
|
|
|
|
} catch {
|
|
|
|
|
state.redisConnected = false;
|
|
|
|
|
redisBufferSize = 0;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return {
|
|
|
|
|
currentBufferSize: state.buffer.length + redisBufferSize,
|
|
|
|
|
localBufferSize: state.buffer.length,
|
|
|
|
|
redisBufferSize,
|
|
|
|
|
redisEnabled: Boolean(redis),
|
|
|
|
|
redisConnected: state.redisConnected,
|
2026-11-17 20:05:00 +06:00
|
|
|
batchFlushThreshold: state.batchFlushThreshold,
|
|
|
|
|
batchFlushIntervalMs: state.batchFlushIntervalMs,
|
|
|
|
|
totalIngested: state.totalIngested,
|
|
|
|
|
totalFlushed: state.totalFlushed,
|
|
|
|
|
recentEventsCached: state.recentEvents.length,
|
|
|
|
|
activeSubscribers: state.subscribers.size,
|
|
|
|
|
isFlushing: state.isFlushing,
|
2026-10-23 11:43:49 -05:00
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Auto-start timer
|
|
|
|
|
DataBufferEngine.init();
|
|
|
|
|
|
|
|
|
|
export default DataBufferEngine;
|