Files
Team Dark Devil ae9b554189 feat: implement R4 production hardening
- Add configurable ingestion batch item limits
- Improve client serialization and sanitization
- Add graceful handling for corrupt GZIP partitions in data API
- Expand admin session token scope
- Update project documentation for R4 status
2026-11-21 22:15:32 +06:00

235 lines
6.3 KiB
TypeScript

import fs from 'node:fs';
import path from 'node:path';
import type { DataStackEnvelope } from './types';
export interface DiskSpoolerOptions {
spoolDir?: string;
maxSizeMb?: number;
enabled?: boolean;
}
/**
* Durable newline-delimited JSON (`.datastack_spool/spool.jsonl`) local disk spooler
* for offline telemetry persistence when both Port 4000 WebSocket and Port 3000 HTTP
* endpoints are temporarily unreachable.
*/
export class DiskSpooler {
public readonly spoolDir: string;
public readonly spoolFile: string;
public readonly maxBytes: number;
public readonly enabled: boolean;
constructor(options: DiskSpoolerOptions = {}) {
this.spoolDir = path.resolve(options.spoolDir || '.datastack_spool');
this.spoolFile = path.join(this.spoolDir, 'spool.jsonl');
const maxSizeMb = Number(options.maxSizeMb ?? 50);
this.maxBytes = Math.max(1, Number.isFinite(maxSizeMb) ? maxSizeMb : 50) * 1024 * 1024;
this.enabled = options.enabled !== false;
}
private ensureDir(): void {
if (!this.enabled) return;
if (!fs.existsSync(this.spoolDir)) {
fs.mkdirSync(this.spoolDir, { recursive: true });
}
}
/**
* Appends one or more event envelopes to `spool.jsonl`.
* Returns the number of events persisted to disk.
*/
public enqueue(items: DataStackEnvelope | DataStackEnvelope[]): number {
if (!this.enabled) return 0;
const list = Array.isArray(items) ? items : [items];
if (list.length === 0) return 0;
this.ensureDir();
const currentSize = this.sizeBytes();
const lines = list.map((item) => JSON.stringify(item)).join('\n') + '\n';
const byteLength = Buffer.byteLength(lines, 'utf8');
if (currentSize + byteLength > this.maxBytes) {
return 0;
}
const fd = fs.openSync(this.spoolFile, 'a');
try {
fs.writeFileSync(fd, lines, 'utf8');
fs.fsyncSync(fd);
} finally {
fs.closeSync(fd);
}
return list.length;
}
/**
* Reads up to `limit` spooled event envelopes from the head of `spool.jsonl`
* without removing them until `removeFirst(count)` is called after server ACK.
*/
public peekBatch(limit = 250): DataStackEnvelope[] {
if (!this.enabled || !fs.existsSync(this.spoolFile)) {
return [];
}
const raw = fs.readFileSync(this.spoolFile, 'utf8');
const lines = raw
.split('\n')
.map((l) => l.trim())
.filter(Boolean);
const results: DataStackEnvelope[] = [];
for (const line of lines.slice(0, Math.max(1, limit))) {
try {
const parsed = JSON.parse(line) as DataStackEnvelope;
if (parsed && typeof parsed === 'object' && parsed.eventId && parsed.data) {
results.push(parsed);
}
} catch {
// Skip malformed line
}
}
return results;
}
/**
* Atomically removes the first `removeCount` lines from `spool.jsonl`
* via a temporary file + `fsync` + `renameSync`.
*/
public removeFirst(removeCount: number): void {
if (!this.enabled || removeCount <= 0 || !fs.existsSync(this.spoolFile)) {
return;
}
const raw = fs.readFileSync(this.spoolFile, 'utf8');
const lines = raw
.split('\n')
.map((l) => l.trim())
.filter(Boolean);
const remaining = lines.slice(removeCount);
if (remaining.length === 0) {
try {
fs.unlinkSync(this.spoolFile);
} catch {
// Ignore if already removed
}
return;
}
this.ensureDir();
const tmpFile = `${this.spoolFile}.${process.pid}.${Date.now()}.tmp`;
const payload = remaining.join('\n') + '\n';
try {
const fd = fs.openSync(tmpFile, 'w');
try {
fs.writeFileSync(fd, payload, 'utf8');
fs.fsyncSync(fd);
} finally {
fs.closeSync(fd);
}
fs.renameSync(tmpFile, this.spoolFile);
} catch (err) {
try {
if (fs.existsSync(tmpFile)) {
fs.unlinkSync(tmpFile);
}
} catch {
// Ignore cleanup failure
}
throw err;
}
}
/**
* Atomically removes `ackedCount` head lines from `spool.jsonl` while
* preserving any `unprocessed` envelopes at the FRONT of the queue in a
* single atomic temp-file + `fsyncSync` + `renameSync` operation (FIFO safe).
*/
public ackAndRequeueFront(
ackedCount: number,
unprocessed: DataStackEnvelope[] = []
): number {
if (!this.enabled || ackedCount <= 0 || !fs.existsSync(this.spoolFile)) {
return 0;
}
const raw = fs.readFileSync(this.spoolFile, 'utf8');
const allLines = raw
.split('\n')
.map((l) => l.trim())
.filter(Boolean);
const consumed = Math.min(Math.floor(ackedCount), allLines.length);
const tailLines = allLines.slice(consumed);
const headLines = unprocessed.map((evt) => JSON.stringify(evt));
const combined = [...headLines, ...tailLines];
if (combined.length === 0) {
try {
fs.unlinkSync(this.spoolFile);
} catch {
// Ignore if already removed
}
return consumed;
}
this.ensureDir();
const tmpFile = `${this.spoolFile}.${process.pid}.${Date.now()}.tmp`;
const payload = combined.join('\n') + '\n';
try {
const fd = fs.openSync(tmpFile, 'w');
try {
fs.writeFileSync(fd, payload, 'utf8');
fs.fsyncSync(fd);
} finally {
fs.closeSync(fd);
}
fs.renameSync(tmpFile, this.spoolFile);
} catch (err) {
try {
if (fs.existsSync(tmpFile)) {
fs.unlinkSync(tmpFile);
}
} catch {
// Ignore cleanup failure
}
throw err;
}
return Math.max(0, consumed - headLines.length);
}
public count(): number {
if (!this.enabled || !fs.existsSync(this.spoolFile)) {
return 0;
}
try {
const raw = fs.readFileSync(this.spoolFile, 'utf8');
return raw
.split('\n')
.map((l) => l.trim())
.filter(Boolean).length;
} catch {
return 0;
}
}
public sizeBytes(): number {
if (!this.enabled || !fs.existsSync(this.spoolFile)) {
return 0;
}
try {
return fs.statSync(this.spoolFile).size;
} catch {
return 0;
}
}
public clear(): void {
if (fs.existsSync(this.spoolFile)) {
try {
fs.unlinkSync(this.spoolFile);
} catch {
// Ignore
}
}
}
}