Files
DataStack/core/data/buffer.ts
T

215 lines
5.7 KiB
TypeScript
Raw Normal View History

2026-10-23 11:43:49 -05:00
import { CSVStorageManager, DataEvent } from '../Storage/csvStorage';
export type EventSubscriber = (event: DataEvent) => void;
export type FlushCallback = (count: number, files: string[]) => void;
interface BufferGlobalState {
buffer: DataEvent[];
recentEvents: DataEvent[];
batchFlushThreshold: number;
batchFlushIntervalMs: number;
flushTimer: NodeJS.Timeout | null;
subscribers: Set<EventSubscriber>;
flushListeners: Set<FlushCallback>;
totalIngested: number;
totalFlushed: number;
isFlushing: boolean;
}
type BufferGlobals = typeof globalThis & {
__dataStackBufferState?: BufferGlobalState;
__dataStackBufferShutdownRegistered?: boolean;
};
const globals = globalThis as BufferGlobals;
function getState(): BufferGlobalState {
if (!globals.__dataStackBufferState) {
globals.__dataStackBufferState = {
buffer: [],
recentEvents: [],
batchFlushThreshold: 500,
batchFlushIntervalMs: 2000,
flushTimer: null,
subscribers: new Set(),
flushListeners: new Set(),
totalIngested: 0,
totalFlushed: 0,
isFlushing: false,
};
}
return globals.__dataStackBufferState;
}
2026-10-23 11:43:49 -05:00
export class DataBufferEngine {
private static maxRecentEvents = 200;
public static get batchFlushThreshold(): number {
return getState().batchFlushThreshold;
}
2026-10-23 11:43:49 -05:00
public static set batchFlushThreshold(value: number) {
getState().batchFlushThreshold = Math.max(1, Number(value) || 500);
}
2026-10-23 11:43:49 -05:00
public static get batchFlushIntervalMs(): number {
return getState().batchFlushIntervalMs;
}
2026-10-23 11:43:49 -05:00
public static set batchFlushIntervalMs(value: number) {
this.setFlushInterval(value);
2026-10-23 11:43:49 -05:00
}
public static init() {
const state = getState();
if (!state.flushTimer) {
state.flushTimer = setInterval(() => {
void this.flush();
}, state.batchFlushIntervalMs);
2026-10-23 11:43:49 -05:00
}
if (
typeof process !== 'undefined' &&
typeof process.on === 'function' &&
!globals.__dataStackBufferShutdownRegistered
) {
globals.__dataStackBufferShutdownRegistered = true;
const flushOnExit = () => {
void this.flush().catch(() => undefined);
};
process.on('SIGTERM', flushOnExit);
process.on('SIGINT', flushOnExit);
}
2026-10-23 11:43:49 -05:00
}
/**
* Updates flush interval and immediately restarts the active interval timer.
*/
public static setFlushInterval(intervalMs: number) {
const state = getState();
const nextInterval = Math.max(100, Number(intervalMs) || 2000);
state.batchFlushIntervalMs = nextInterval;
if (state.flushTimer) {
clearInterval(state.flushTimer);
state.flushTimer = null;
}
state.flushTimer = setInterval(() => {
void this.flush();
}, state.batchFlushIntervalMs);
}
2026-10-23 11:43:49 -05:00
/**
* Push incoming event to in-memory buffer.
* If buffer hits threshold (e.g. 500 rows), triggers immediate batch flush.
*/
public static push(event: DataEvent): void {
const state = getState();
state.buffer.push(event);
state.totalIngested += 1;
2026-10-23 11:43:49 -05:00
// Cache in recent events for live dashboard viewer
state.recentEvents.unshift(event);
if (state.recentEvents.length > this.maxRecentEvents) {
state.recentEvents.pop();
2026-10-23 11:43:49 -05:00
}
// Notify live subscribers
for (const sub of state.subscribers) {
2026-10-23 11:43:49 -05:00
try {
sub(event);
} catch {
// Ignore subscriber error
2026-10-23 11:43:49 -05:00
}
}
// Check if buffer threshold reached
if (state.buffer.length >= state.batchFlushThreshold) {
void this.flush();
2026-10-23 11:43:49 -05:00
}
}
/**
* Flushes current buffer to CSV files via CSVStorageManager
*/
public static async flush(): Promise<{ count: number; files: string[] }> {
const state = getState();
if (state.buffer.length === 0 || state.isFlushing) {
2026-10-23 11:43:49 -05:00
return { count: 0, files: [] };
}
state.isFlushing = true;
const batchToFlush = [...state.buffer];
state.buffer = [];
2026-10-23 11:43:49 -05:00
try {
const result = await CSVStorageManager.appendBatch(batchToFlush);
state.totalFlushed += result.written;
2026-10-23 11:43:49 -05:00
for (const listener of state.flushListeners) {
2026-10-23 11:43:49 -05:00
try {
listener(result.written, result.filesUpdated);
} catch {
// ignore
}
}
return { count: result.written, files: result.filesUpdated };
} catch (err) {
// On failure, re-queue to buffer front for retry
state.buffer.unshift(...batchToFlush);
2026-10-23 11:43:49 -05:00
throw err;
} finally {
state.isFlushing = false;
2026-10-23 11:43:49 -05:00
}
}
/**
* Subscribe to real-time events (for UI monitoring)
*/
public static subscribe(subscriber: EventSubscriber): () => void {
const state = getState();
state.subscribers.add(subscriber);
2026-10-23 11:43:49 -05:00
return () => {
state.subscribers.delete(subscriber);
2026-10-23 11:43:49 -05:00
};
}
public static onFlush(callback: FlushCallback): () => void {
const state = getState();
state.flushListeners.add(callback);
2026-10-23 11:43:49 -05:00
return () => {
state.flushListeners.delete(callback);
2026-10-23 11:43:49 -05:00
};
}
public static getRecentEvents(limit = 50, filterClientId?: string): DataEvent[] {
const state = getState();
2026-10-23 11:43:49 -05:00
if (filterClientId && filterClientId !== 'all') {
return state.recentEvents
.filter((e) => e.clientId === filterClientId)
.slice(0, limit);
2026-10-23 11:43:49 -05:00
}
return state.recentEvents.slice(0, limit);
2026-10-23 11:43:49 -05:00
}
public static getBufferStats() {
const state = getState();
2026-10-23 11:43:49 -05:00
return {
currentBufferSize: state.buffer.length,
batchFlushThreshold: state.batchFlushThreshold,
batchFlushIntervalMs: state.batchFlushIntervalMs,
totalIngested: state.totalIngested,
totalFlushed: state.totalFlushed,
recentEventsCached: state.recentEvents.length,
activeSubscribers: state.subscribers.size,
isFlushing: state.isFlushing,
2026-10-23 11:43:49 -05:00
};
}
}
// Auto-start timer
DataBufferEngine.init();
export default DataBufferEngine;