import assert from "node:assert/strict"; import fs from "node:fs"; import { createServer, type Server } from "node:http"; import type { AddressInfo } from "node:net"; import { createRequire } from "node:module"; import os from "node:os"; import path from "node:path"; import test, { after, before, beforeEach } from "node:test"; import { NextRequest } from "next/server"; import { io as ioClient } from "socket.io-client"; import { GET as getDataRoute } from "../app/api/data/route"; import { GET as getEventsRoute } from "../app/api/events/route"; import { GET as getExportRoute } from "../app/api/export/route"; import { GET as getStorageRoute, POST as postStorageRoute, } from "../app/api/storage/route"; import { ApiEngine } from "../core/apiEngine/apiService"; import { DataBufferEngine, REDIS_BUFFER_KEY, REDIS_FLUSH_LOCK_KEY, REDIS_RECENT_KEY, type RedisBufferClient, } from "../core/data/buffer"; import { SDKManager } from "../core/sdk/sdkManager"; import { CSVStorageManager, resolveDefaultStorageDir as resolveWebStorageDir, type DataEvent, } from "../core/Storage/csvStorage"; import { createAdminSessionToken, getAdminSessionSecret } from "../core/auth/adminSession"; import { createInitialAdmin, resetAdminStateForTests } from "../core/auth/adminUsers"; const require = createRequire(import.meta.url); const { CsvWriterWorker, resolveDefaultStorageDir: resolveWorkerStorageDir, } = require("../server/workers/csvWriter") as { CsvWriterWorker: new (baseDir?: string, maxFileSize?: number) => { baseDir: string; writeBatch: (events: DataEvent[]) => Promise<{ written: number; files: string[] }>; pruneExpiredPartitions: (days: number) => number; pruneExpiredPartitionsAsync: (days: number) => Promise; }; resolveDefaultStorageDir: () => string; }; interface IngestionServerModule { server: Server; io: import("socket.io").Server; writer: { baseDir: string }; bufferQueue: DataEvent[]; runtimeConfig: { batchFlushThreshold: number; autoPruneDays: number; }; applyRuntimeSettingsObject: (obj: Record) => { jwtRotated: boolean }; flushBuffer: (options?: { waitMs?: number; lockTtlMs?: number; throwOnLockTimeout?: boolean; }) => Promise<{ written: number; files: string[]; lockTimedOut?: boolean }>; setRedisClientForTests: (client: RedisBufferClient | null | undefined) => void; stopTimers: () => void; } class FakeRedisClient implements RedisBufferClient { public status = "ready"; public lists = new Map(); public strings = new Map(); public sets = new Map>(); public renewCalls = 0; public purgeEvalCalls = 0; public ltrimCalls = 0; public lpushCalls = 0; public async rpush(key: string, ...values: string[]): Promise { const list = this.lists.get(key) || []; list.push(...values); this.lists.set(key, list); return list.length; } public async lpush(key: string, ...values: string[]): Promise { this.lpushCalls += 1; const list = this.lists.get(key) || []; for (const v of values) { list.unshift(v); } this.lists.set(key, list); return list.length; } public async ltrim(key: string, start: number, stop: number): Promise<"OK"> { this.ltrimCalls += 1; const list = this.lists.get(key) || []; const len = list.length; const normStart = start < 0 ? Math.max(0, len + start) : start; const normStop = stop < 0 ? len + stop : stop; if (normStart > normStop || normStart >= len) { this.lists.set(key, []); } else { this.lists.set(key, list.slice(normStart, normStop + 1)); } return "OK"; } public async lrange(key: string, start: number, stop: number): Promise { const list = this.lists.get(key) || []; const len = list.length; if (len === 0) return []; const normStart = start < 0 ? Math.max(0, len + start) : start; const normStop = stop < 0 ? len + stop : stop; if (normStart > normStop || normStart >= len) return []; return list.slice(normStart, normStop + 1); } public async llen(key: string): Promise { return (this.lists.get(key) || []).length; } public async set( key: string, value: string, _pxMode: "PX", ttlMs: number, _nxMode: "NX", ): Promise { const existing = this.strings.get(key); if (existing && existing.expiresAt > Date.now()) { return null; } this.strings.set(key, { value, expiresAt: Date.now() + ttlMs }); return "OK"; } public async sadd(key: string, ...members: string[]): Promise { const set = this.sets.get(key) || new Set(); let added = 0; for (const member of members) { if (!set.has(member)) { set.add(member); added += 1; } } this.sets.set(key, set); return added; } public async srem(key: string, ...members: string[]): Promise { const set = this.sets.get(key); if (!set) return 0; let removed = 0; for (const member of members) { if (set.delete(member)) removed += 1; } return removed; } public async sismember(key: string, member: string): Promise { return this.sets.get(key)?.has(member) ? 1 : 0; } public async eval( script: string, _numKeys: number, key: string, token: string, ttlArg?: string, ): Promise { if (script.includes("DATASTACK_PURGE_CLIENT_LIST")) { this.purgeEvalCalls += 1; const list = this.lists.get(key) || []; const kept: string[] = []; let removed = 0; for (const raw of list) { try { const parsed = JSON.parse(raw) as { clientId?: string }; const cid = String(parsed?.clientId || "") .trim() .toLowerCase() .replace(/[^a-z0-9_-]/g, "_"); if (cid === token) { removed += 1; continue; } } catch { // keep unparseable items for standard flush cleanup } kept.push(raw); } if (removed > 0) { this.lists.set(key, kept); } return removed; } const existing = this.strings.get(key); if (script.includes("pexpire")) { if (existing && existing.value === token && existing.expiresAt > Date.now()) { const ttlMs = Number(ttlArg); existing.expiresAt = Date.now() + (Number.isFinite(ttlMs) && ttlMs > 0 ? ttlMs : 30000); this.renewCalls += 1; return 1; } return 0; } if (existing && existing.value === token) { this.strings.delete(key); return 1; } return 0; } } const g = globalThis as typeof globalThis & { __dataStackBufferState?: { buffer: DataEvent[]; recentEvents: DataEvent[]; deletedClientIds?: Set; flushTimer: ReturnType | null; }; __dataStackAdminSettings?: unknown; __dataStackClientsMap?: Map; __dataStackDeletedClients?: Set; }; let tmpStorageDir = ""; let adminCookieHeader = ""; const origBaseStorageDir = CSVStorageManager.baseStorageDir; const prevDisableListen = process.env.DATASTACK_DISABLE_LISTEN; const prevIngestionUrl = process.env.INGESTION_INTERNAL_URL; const prevDbUrl = process.env.DATABASE_URL; const prevRedisUrl = process.env.REDIS_URL; before(async () => { process.env.DATASTACK_DISABLE_LISTEN = "1"; delete process.env.DATABASE_URL; delete process.env.REDIS_URL; delete process.env.INGESTION_INTERNAL_URL; resetAdminStateForTests(); await createInitialAdmin("admin@datastack.dev", "Phase03Pass123!"); const secret = getAdminSessionSecret()!; const token = await createAdminSessionToken("admin@datastack.dev", secret, Date.now() + 3600_000, 1); adminCookieHeader = `datastack_admin_session=${token}`; tmpStorageDir = fs.mkdtempSync(path.join(os.tmpdir(), "datastack-phase03-")); CSVStorageManager.baseStorageDir = tmpStorageDir; CSVStorageManager.clearRowCountCache(); }); beforeEach(async () => { delete process.env.INGESTION_INTERNAL_URL; DataBufferEngine.setRedisClientForTests(undefined); if (g.__dataStackBufferState) { g.__dataStackBufferState.buffer = []; g.__dataStackBufferState.recentEvents = []; g.__dataStackBufferState.deletedClientIds?.clear(); } CSVStorageManager.clearRowCountCache(); fs.rmSync(tmpStorageDir, { recursive: true, force: true }); fs.mkdirSync(tmpStorageDir, { recursive: true }); delete g.__dataStackAdminSettings; g.__dataStackClientsMap?.clear(); g.__dataStackDeletedClients?.clear(); }); after(async () => { await DataBufferEngine.shutdown(); CSVStorageManager.baseStorageDir = origBaseStorageDir; CSVStorageManager.clearRowCountCache(); if (tmpStorageDir) { fs.rmSync(tmpStorageDir, { recursive: true, force: true }); } if (prevDisableListen === undefined) delete process.env.DATASTACK_DISABLE_LISTEN; else process.env.DATASTACK_DISABLE_LISTEN = prevDisableListen; if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL; else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl; if (prevDbUrl === undefined) delete process.env.DATABASE_URL; else process.env.DATABASE_URL = prevDbUrl; if (prevRedisUrl === undefined) delete process.env.REDIS_URL; else process.env.REDIS_URL = prevRedisUrl; }); test("BUG-11: Shared Redis buffer preserves events on CSV error, honors distributed flush lock, avoids double-counting, and falls back to RAM cleanly", async () => { const fakeRedis = new FakeRedisClient(); DataBufferEngine.setRedisClientForTests(fakeRedis); const sampleEvents: DataEvent[] = [ { clientId: "phase03_redis", eventId: "r-evt-1", timestamp: "2026-11-17T10:00:00.000Z", payload: JSON.stringify({ seq: 1 }), }, { clientId: "phase03_redis", eventId: "r-evt-2", timestamp: "2026-11-17T10:00:01.000Z", payload: JSON.stringify({ seq: 2 }), }, ]; await DataBufferEngine.pushBatchAsync(sampleEvents); assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 2); const recentFromRedis = await DataBufferEngine.getRecentEventsAsync(10, "phase03_redis"); assert.equal(recentFromRedis.length, 2); assert.equal(recentFromRedis[0].eventId, "r-evt-2"); // Lock contention must skip concurrent Redis flush without corrupting queue await fakeRedis.set(REDIS_FLUSH_LOCK_KEY, "other-worker-token", "PX", 15000, "NX"); const skipped = await DataBufferEngine.flush(); assert.equal(skipped.count, 0); assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 2); await fakeRedis.eval("", 1, REDIS_FLUSH_LOCK_KEY, "other-worker-token"); // Crash/error safety: if appendBatch throws before finishing, Redis LTRIM must NOT discard events const originalAppend = CSVStorageManager.appendBatch; CSVStorageManager.appendBatch = async () => { throw new Error("Simulated disk write failure"); }; await assert.rejects(() => DataBufferEngine.flush(), /Simulated disk write failure/); CSVStorageManager.appendBatch = originalAppend; assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 2); // Subsequent healthy flush drains Redis list, releases distributed lock, and writes CSV const flushed = await DataBufferEngine.flush(); assert.equal(flushed.count, 2); assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 0); assert.equal(fakeRedis.strings.has(REDIS_FLUSH_LOCK_KEY), false); // Verify Port 4000 /internal/recent & /internal/metrics read shared Redis without double-counting const serverMod = require("../server/index.js") as IngestionServerModule; const origWriterDir = serverMod.writer.baseDir; serverMod.writer.baseDir = tmpStorageDir; serverMod.setRedisClientForTests(fakeRedis); await new Promise((resolve) => serverMod.server.listen(0, "127.0.0.1", () => resolve())); const addr = serverMod.server.address() as AddressInfo; process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${addr.port}`; try { await DataBufferEngine.pushAsync({ clientId: "phase03_redis", eventId: "r-evt-3", timestamp: "2026-11-17T10:00:02.000Z", payload: JSON.stringify({ seq: 3 }), }); const stats = await DataBufferEngine.getBufferStatsAsync(); assert.equal(stats.redisEnabled, true); assert.equal(stats.redisBufferSize, 1); const metrics = await ApiEngine.getDashboardMetrics(); // Shared Redis count (1) must NOT be counted twice as 2 across Port 3000 + Port 4000 assert.equal(metrics.bufferStats.currentBufferSize, 1); // Clear recent list & buffer via Port 4000 flush const flushRes = await serverMod.flushBuffer(); assert.equal(flushRes.written, 1); assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 0); } finally { serverMod.writer.baseDir = origWriterDir; serverMod.setRedisClientForTests(undefined); serverMod.stopTimers(); delete process.env.INGESTION_INTERNAL_URL; await new Promise((resolve) => serverMod.io.close(() => serverMod.server.close(() => resolve()))); } // Fallback to local RAM buffer when Redis is disabled (null) DataBufferEngine.setRedisClientForTests(null); await DataBufferEngine.pushAsync({ clientId: "phase03_ram", eventId: "ram-evt-1", timestamp: "2026-11-17T10:01:00.000Z", payload: JSON.stringify({ mode: "ram" }), }); const ramStats = await DataBufferEngine.getBufferStatsAsync(); assert.equal(ramStats.redisEnabled, false); assert.equal(ramStats.localBufferSize, 1); const ramFlushed = await DataBufferEngine.flush(); assert.equal(ramFlushed.count, 1); }); test("BUG-09: Cooperative CSV partition locking serializes concurrent Port 3000 + Port 4000 writers, recovers stale locks, and surfaces Port 4000 flush errors", async () => { const worker = new CsvWriterWorker(tmpStorageDir); const partitionDate = "2026-11-17"; const clientId = "phase03_concurrent"; const batchA: DataEvent[] = Array.from({ length: 15 }, (_, idx) => ({ clientId, eventId: `a-${idx}`, timestamp: `${partitionDate}T12:00:00.000Z`, payload: JSON.stringify({ writer: "A", idx }), })); const batchB: DataEvent[] = Array.from({ length: 15 }, (_, idx) => ({ clientId, eventId: `b-${idx}`, timestamp: `${partitionDate}T12:00:01.000Z`, payload: JSON.stringify({ writer: "B", idx }), })); await Promise.all([ CSVStorageManager.appendBatch(batchA), worker.writeBatch(batchB), ]); const csvPath = path.join(tmpStorageDir, clientId, `${partitionDate}.csv`); const lockPath = path.join(tmpStorageDir, clientId, `.${partitionDate}.csv.lock`); assert.equal(fs.existsSync(lockPath), false, "Lock file must be cleaned up after writes"); const lines = fs.readFileSync(csvPath, "utf8").trim().split("\n"); assert.equal(lines[0], '"clientId","eventId","timestamp","payload"'); assert.equal(lines.length, 31, "Must contain exactly 1 header + 30 complete rows without interleaving"); // Stale lock recovery (>30s old mtime) fs.writeFileSync(lockPath, "999999:0", "utf8"); const pastTime = new Date(Date.now() - 45_000); fs.utimesSync(lockPath, pastTime, pastTime); await CSVStorageManager.appendBatch([ { clientId, eventId: "after-stale-lock", timestamp: `${partitionDate}T12:00:02.000Z`, payload: JSON.stringify({ staleRecovered: true }), }, ]); assert.equal(fs.existsSync(lockPath), false); const files = await CSVStorageManager.listAllFilesAsync(); assert.equal(files[0].linesCount, 31); // Explicit flush error & warning propagation in POST /api/storage const failingBridge = createServer((_req, res) => { res.statusCode = 500; res.setHeader("Content-Type", "application/json"); res.end(JSON.stringify({ success: false, error: "Disk permission denied on Port 4000" })); }); await new Promise((resolve) => failingBridge.listen(0, "127.0.0.1", () => resolve())); const bridgeAddr = failingBridge.address() as AddressInfo; process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${bridgeAddr.port}`; try { const errRes = await postStorageRoute(new NextRequest("http://localhost:3000/api/storage", { method: "POST", headers: { cookie: adminCookieHeader } })); assert.equal(errRes.status, 502); const errJson = await errRes.json(); assert.equal(errJson.success, false); assert.equal(errJson.code, "INGESTION_FLUSH_FAILED"); assert.equal(errJson.ingestionFlushed, false); } finally { delete process.env.INGESTION_INTERNAL_URL; await new Promise((resolve) => failingBridge.close(() => resolve())); } // When Port 4000 is down and INGESTION_INTERNAL_URL is unset, standalone dev surfaces explicit warning + ingestionFlushed: false const origFetch = globalThis.fetch; globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; if (url.startsWith("http://127.0.0.1:4000/internal/flush")) { throw new TypeError("fetch failed (ECONNREFUSED simulated)"); } return origFetch(input, init); }) as typeof fetch; try { const standaloneRes = await postStorageRoute(new NextRequest("http://localhost:3000/api/storage", { method: "POST", headers: { cookie: adminCookieHeader } })); assert.equal(standaloneRes.status, 200); const standaloneJson = await standaloneRes.json(); assert.equal(standaloneJson.ingestionFlushed, false); assert.match(String(standaloneJson.warning || ""), /unreachable/i); } finally { globalThis.fetch = origFetch; } }); test("BUG-10: Cached streaming row counter counts multiline RFC 4180 payloads accurately and avoids repeated disk reads", async () => { await CSVStorageManager.appendBatch([ { clientId: "phase03_perf", eventId: "multiline-1", timestamp: "2026-11-17T08:00:00.000Z", payload: JSON.stringify({ note: "line1\nline2\n\"quoted\"\nline4" }), }, { clientId: "phase03_perf", eventId: "multiline-2", timestamp: "2026-11-17T08:00:01.000Z", payload: JSON.stringify({ note: "single" }), }, ]); const firstStats = await CSVStorageManager.getStorageStatsAsync(); assert.equal(firstStats.totalRows, 2); // Spy on fs.createReadStream to verify repeated dashboard polls hit mtime+size cache with 0 file scans const origCreateReadStream = fs.createReadStream; let streamReads = 0; (fs as unknown as { createReadStream: typeof fs.createReadStream }).createReadStream = (( ...args: Parameters ) => { streamReads += 1; return origCreateReadStream(...args); }) as typeof fs.createReadStream; try { const secondStats = await CSVStorageManager.getStorageStatsAsync(); assert.equal(secondStats.totalRows, 2); assert.equal(streamReads, 0, "Unchanged CSV files must use cached row count"); } finally { (fs as unknown as { createReadStream: typeof fs.createReadStream }).createReadStream = origCreateReadStream; } // Server 3s settings sync must not repeatedly prune when autoPruneDays is unchanged const serverMod = require("../server/index.js") as IngestionServerModule; const oldPartitionDir = path.join(tmpStorageDir, "phase03_prune_check"); fs.mkdirSync(oldPartitionDir, { recursive: true }); fs.writeFileSync(path.join(oldPartitionDir, "2020-01-01.csv"), "h\nr\n", "utf8"); const prunedCount = CSVStorageManager.pruneExpiredPartitions(30); assert.equal(prunedCount, 1); serverMod.applyRuntimeSettingsObject({ autoPruneDays: 30 }); fs.writeFileSync(path.join(oldPartitionDir, "2020-01-02.csv"), "h\nr\n", "utf8"); // Same autoPruneDays=30 applied again (simulating 3s DB poll) must NOT prune on every tick serverMod.applyRuntimeSettingsObject({ autoPruneDays: 30 }); assert.equal(fs.existsSync(path.join(oldPartitionDir, "2020-01-02.csv")), true); serverMod.stopTimers(); }); test("BUG-12: Prisma schema models and mappings align cleanly with runtime SQL DDL and Docker Compose Redis wiring", () => { const schemaPath = path.resolve(process.cwd(), "prisma/schema.prisma"); const schemaText = fs.readFileSync(schemaPath, "utf8"); assert.match(schemaText, /model ClientSDK[\s\S]*clientId\s+String\s+@id\s+@map\("client_id"\)/); assert.match(schemaText, /token\s+String/); assert.match(schemaText, /model DeletedClient[\s\S]*@@map\("deleted_clients"\)/); assert.match(schemaText, /model AdminUser[\s\S]*tokenVersion\s+Int\s+@default\(1\)\s+@map\("token_version"\)/); assert.match(schemaText, /model AdminLoginAttempt[\s\S]*@@map\("admin_login_attempts"\)/); assert.match(schemaText, /model AdminSettings[\s\S]*maxEventsPerSec\s+Int\s+@default\(100\)\s+@map\("max_events_per_sec"\)/); assert.equal(/model ApiKey\b/.test(schemaText), false); assert.equal(/model AuditLog\b/.test(schemaText), false); const composeText = fs.readFileSync(path.resolve(process.cwd(), "docker/docker-compose.yml"), "utf8"); assert.equal((composeText.match(/REDIS_URL:\s*"redis:\/\/redis:6379"/g) || []).length, 2); assert.match(composeText, /redis:\s*\n\s*condition:\s*service_healthy/); }); test("BUG-14: GET /api/data searches eventId & clientId + UTC dates, GET /api/events caps CSV tail, and GET /api/export rejects ambiguous filename & honors ?flush=false", async () => { DataBufferEngine.setRedisClientForTests(null); await SDKManager.registerClientAsync({ clientId: "alpha_cli", name: "Alpha", rateLimit: 100 }); await SDKManager.registerClientAsync({ clientId: "beta_cli", name: "Beta", rateLimit: 100 }); const tsMillis = String(Date.UTC(2026, 10, 17, 15, 0, 0)); // 2026-11-17 await DataBufferEngine.pushBatchAsync([ { clientId: "alpha_cli", eventId: "unique-findme-id-99", timestamp: tsMillis, payload: JSON.stringify({ temp: 22 }), }, { clientId: "beta_cli", eventId: "beta-id-1", timestamp: "2026-11-17T15:01:00.000Z", payload: JSON.stringify({ temp: 33 }), }, ]); // 1. Search by eventId with from/to UTC normalization while still buffered const dataByEvent = await getDataRoute( new NextRequest("http://localhost:3000/api/data?search=unique-findme-id-99&from=2026-11-17&to=2026-11-17", { headers: { cookie: adminCookieHeader } }), ); assert.equal(dataByEvent.status, 200); const dataEventJson = await dataByEvent.json(); assert.equal(dataEventJson.total, 1); assert.equal(dataEventJson.events[0].eventId, "unique-findme-id-99"); // 2. Search by clientId substring const dataByClient = await getDataRoute( new NextRequest("http://localhost:3000/api/data?search=beta_cli", { headers: { cookie: adminCookieHeader } }), ); const dataClientJson = await dataByClient.json(); assert.equal(dataClientJson.total, 1); assert.equal(dataClientJson.events[0].clientId, "beta_cli"); // 3. Export with ?flush=false must NOT drain active buffer const noFlushRes = await getExportRoute( new NextRequest("http://localhost:3000/api/export?flush=false&format=json", { headers: { cookie: adminCookieHeader } }), ); assert.equal(noFlushRes.status, 200); assert.equal((await noFlushRes.text()).trim(), "[]"); assert.equal((await DataBufferEngine.getBufferStatsAsync()).localBufferSize, 2); // 4. Default flush writes both 2026-11-17.csv files -> clientId=all + filename must 400 AMBIGUOUS_FILENAME const flushExportRes = await getExportRoute( new NextRequest("http://localhost:3000/api/export?format=json", { headers: { cookie: adminCookieHeader } }), ); assert.equal(flushExportRes.status, 200); const flushedPayload = await flushExportRes.json(); assert.equal(flushedPayload.length, 2); const ambiguousRes = await getExportRoute( new NextRequest("http://localhost:3000/api/export?clientId=all&filename=2026-11-17.csv", { headers: { cookie: adminCookieHeader } }), ); assert.equal(ambiguousRes.status, 400); const ambiguousJson = await ambiguousRes.json(); assert.equal(ambiguousJson.code, "AMBIGUOUS_FILENAME"); // Disambiguated download with explicit clientId succeeds cleanly const specificRes = await getExportRoute( new NextRequest("http://localhost:3000/api/export?clientId=alpha_cli&filename=2026-11-17.csv", { headers: { cookie: adminCookieHeader } }), ); assert.equal(specificRes.status, 200); const csvBody = await specificRes.text(); assert.match(csvBody, /unique-findme-id-99/); // 5. GET /api/events with limit=1 returns latest single stored event cleanly const eventsRes = await getEventsRoute( new NextRequest("http://localhost:3000/api/events?clientId=alpha_cli&limit=1", { headers: { cookie: adminCookieHeader } }), ); assert.equal(eventsRes.status, 200); const eventsJson = await eventsRes.json(); assert.equal(eventsJson.events.length, 1); assert.equal(eventsJson.events[0].eventId, "unique-findme-id-99"); // 6. SPEC-07: extractSafePartitionDate returns null on corrupt timestamp with null fallback assert.equal(CSVStorageManager.extractSafePartitionDate("not-a-timestamp", null), null); assert.equal(CSVStorageManager.extractSafePartitionDate("", null), null); assert.equal(CSVStorageManager.extractSafePartitionDate("2026-11-17T12:00:00.000Z", null), "2026-11-17"); // Corrupt timestamp in buffer must NOT leak into today's date range query await DataBufferEngine.pushBatchAsync([ { clientId: "alpha_cli", eventId: "corrupt-ts-event", timestamp: "corrupted_raw_value", payload: JSON.stringify({ broken: true }), }, ]); const today = new Date().toISOString().slice(0, 10); const dateRangeRes = await getDataRoute( new NextRequest(`http://localhost:3000/api/data?search=corrupt-ts-event&from=${today}&to=${today}`, { headers: { cookie: adminCookieHeader } }), ); assert.equal(dateRangeRes.status, 200); const dateRangeJson = await dateRangeRes.json(); assert.equal(dateRangeJson.total, 0, "Corrupt timestamp must NOT match date range"); // But without from/to date filters, corrupt event is still visible for admin inspection const allDataRes = await getDataRoute( new NextRequest("http://localhost:3000/api/data?search=corrupt-ts-event", { headers: { cookie: adminCookieHeader } }), ); assert.equal(allDataRes.status, 200); const allDataJson = await allDataRes.json(); assert.equal(allDataJson.total, 1, "Corrupt timestamp event must be discoverable in unfiltered search"); }); test("BUG-C01 & BUG-C02: Concurrent in-flight flush coalesces cleanly, Redis recent errors do not duplicate RAM buffer, and lock ownership prevents foreign unlock", async () => { const failingRecentRedis = new FakeRedisClient(); failingRecentRedis.lpush = async () => { throw new Error("Transient Redis recent cache write failure"); }; DataBufferEngine.setRedisClientForTests(failingRecentRedis); await DataBufferEngine.pushAsync({ clientId: "phase03_coalesce", eventId: "coal-1", timestamp: "2026-11-18T10:00:00.000Z", payload: JSON.stringify({ step: 1 }), }); // rpush succeeded, so local RAM buffer MUST stay empty despite recent cache throw const statsAfterPush = await DataBufferEngine.getBufferStatsAsync(); assert.equal(statsAfterPush.redisBufferSize, 1); assert.equal(statsAfterPush.localBufferSize, 0); // Coalesce concurrent in-process flushes while appendBatch is in flight DataBufferEngine.setRedisClientForTests(null); await DataBufferEngine.pushAsync({ clientId: "phase03_coalesce", eventId: "coal-2", timestamp: "2026-11-18T10:00:01.000Z", payload: JSON.stringify({ step: 2 }), }); const origAppend = CSVStorageManager.appendBatch.bind(CSVStorageManager); let releaseGate!: () => void; const gatePromise = new Promise((resolve) => { releaseGate = resolve; }); let firstEnterGate!: () => void; const enteredPromise = new Promise((resolve) => { firstEnterGate = resolve; }); CSVStorageManager.appendBatch = async (evts: DataEvent[]) => { firstEnterGate(); await gatePromise; return origAppend(evts); }; try { const flushA = DataBufferEngine.flush(); await enteredPromise; const flushB = DataBufferEngine.flush(); releaseGate(); const [resA, resB] = await Promise.all([flushA, flushB]); assert.equal(resA.count, 1); assert.equal(resB.count, 1, "Second concurrent caller must await active flush instead of returning 0"); } finally { CSVStorageManager.appendBatch = origAppend; } // Partition lock ownership: a new lock file written after initial acquisition must not be unlinked by foreign content const partDir = CSVStorageManager.ensureClientDir("phase03_lock_owner"); const lockPath = path.join(partDir, ".2026-11-18.csv.lock"); let insideLockChecked = false; await CSVStorageManager.withPartitionLock("phase03_lock_owner", "2026-11-18", async () => { const ownToken = fs.readFileSync(lockPath, "utf8"); assert.match(ownToken, /^\d+:\d+:/); // Simulate stale takeover replacing lock file with another owner's token fs.writeFileSync(lockPath, "other:999:foreign-uuid", "utf8"); insideLockChecked = true; }); assert.equal(insideLockChecked, true); assert.equal(fs.existsSync(lockPath), true, "Original holder must NOT unlink a lock now owned by another token"); fs.unlinkSync(lockPath); }); test("BUG-C03 & BUG-C04: Safe newline appends, size rotation overflow, and locked partition pruning with empty directory cleanup", async () => { const origMax = CSVStorageManager.maxFileSizeBytes; CSVStorageManager.maxFileSizeBytes = 220; try { // Seed existing partition ending cleanly with trailing newline const clientDir = CSVStorageManager.ensureClientDir("phase03_rotate"); const firstFile = path.join(clientDir, "2026-11-18.csv"); fs.writeFileSync( firstFile, '"clientId","eventId","timestamp","payload"\n"phase03_rotate","seed-1","2026-11-18T00:00:00.000Z","{}"\n', "utf8", ); // Append batch that stays or overflows limit safely without blank lines await CSVStorageManager.appendBatch([ { clientId: "phase03_rotate", eventId: "rot-small", timestamp: "2026-11-18T00:01:00.000Z", payload: JSON.stringify({ a: 1 }), }, { clientId: "phase03_rotate", eventId: "rot-overflow-with-long-payload-padding-1234567890", timestamp: "2026-11-18T00:02:00.000Z", payload: JSON.stringify({ data: "x".repeat(160) }), }, ]); const files = await CSVStorageManager.listAllFilesAsync(); const clientFiles = files.filter((f) => f.clientId === "phase03_rotate"); assert.ok(clientFiles.length >= 2, "Overflowing batch should rotate to -part2.csv"); for (const f of clientFiles) { const lines = fs.readFileSync(f.filePath, "utf8").trimEnd().split("\n"); assert.equal(lines.some((line) => line === ""), false, "CSV must never contain blank lines from double newline appends"); } // Prune respects active partition lock and deletes unlocked expired partitions const lockedOldDir = CSVStorageManager.ensureClientDir("phase03_locked_old"); fs.writeFileSync(path.join(lockedOldDir, "2020-01-01.csv"), '"h"\n"r"\n', "utf8"); fs.writeFileSync(path.join(lockedOldDir, ".2020-01-01.csv.lock"), `1:${Date.now()}:active`, "utf8"); const unlockedOldDir = CSVStorageManager.ensureClientDir("phase03_unlocked_old"); const unlockedOldCsv = path.join(unlockedOldDir, "2020-01-01.csv"); fs.writeFileSync(unlockedOldCsv, '"h"\n"r"\n', "utf8"); const pruned = CSVStorageManager.pruneExpiredPartitions(30); assert.equal(pruned, 1, "Only the unlocked expired partition should be pruned"); assert.equal(fs.existsSync(path.join(lockedOldDir, "2020-01-01.csv")), true); assert.equal(fs.existsSync(unlockedOldCsv), false, "Unlocked expired partition should be pruned"); } finally { CSVStorageManager.maxFileSizeBytes = origMax; } }); test("BUG-C05 & BUG-C06: Export honors ?format=json on ?filename=, aborts on corrupt CSV, and /api/data + /api/events sort mixed timestamps & normalize clientId", async () => { DataBufferEngine.setRedisClientForTests(null); await CSVStorageManager.appendBatch([ { clientId: "phase03_mix", eventId: "evt-iso", timestamp: "2026-11-18T10:00:00.000Z", payload: JSON.stringify({ kind: "iso" }), }, { clientId: "phase03_mix", eventId: "evt-epoch-newer", timestamp: String(Date.parse("2026-11-18T12:00:00.000Z")), payload: JSON.stringify({ kind: "epoch" }), }, ]); // Normalized clientId query with surrounding spaces matches stored dir and orders epoch-ms newer first const dataRes = await getDataRoute( new NextRequest("http://localhost:3000/api/data?clientId=%20phase03_mix%20", { headers: { cookie: adminCookieHeader }, }), ); assert.equal(dataRes.status, 200); const dataJson = await dataRes.json(); assert.equal(dataJson.total, 2); assert.equal(dataJson.events[0].eventId, "evt-epoch-newer"); assert.equal(dataJson.events[1].eventId, "evt-iso"); // GET /api/events orders epoch-ms newer first too const eventsRes = await getEventsRoute( new NextRequest("http://localhost:3000/api/events?clientId=%20phase03_mix%20&limit=5", { headers: { cookie: adminCookieHeader }, }), ); assert.equal(eventsRes.status, 200); const eventsJson = await eventsRes.json(); assert.equal(eventsJson.events[0].eventId, "evt-epoch-newer"); // Export ?filename=2026-11-18.csv&format=json returns valid JSON records const jsonFileExport = await getExportRoute( new NextRequest( "http://localhost:3000/api/export?clientId=phase03_mix&filename=2026-11-18.csv&format=json&flush=false", { headers: { cookie: adminCookieHeader } }, ), ); assert.equal(jsonFileExport.status, 200); assert.match(jsonFileExport.headers.get("content-type") || "", /application\/json/); const parsedExport = await jsonFileExport.json(); assert.equal(parsedExport.length, 2); // Corrupt CSV aborts export stream rather than returning truncated OK JSON const corruptDir = CSVStorageManager.ensureClientDir("phase03_corrupt"); fs.writeFileSync( path.join(corruptDir, "2026-11-18.csv"), '"clientId","eventId","timestamp","payload"\n"phase03_corrupt","c1","2026-11-18T00:00:00Z","unclosed quote', "utf8", ); const corruptExportRes = await getExportRoute( new NextRequest( "http://localhost:3000/api/export?clientId=phase03_corrupt&format=json&flush=false", { headers: { cookie: adminCookieHeader } }, ), ); await assert.rejects(async () => corruptExportRes.text()); }); test("FORENSIC BUG-18: Redis flush lock waits and retries on contention, throws FLUSH_LOCK_TIMEOUT when requested, and renews TTL via Lua during long flushes", async () => { const fakeRedis = new FakeRedisClient(); DataBufferEngine.setRedisClientForTests(fakeRedis); await DataBufferEngine.pushAsync({ clientId: "phase03_lock", eventId: "lock-evt-1", timestamp: "2026-11-19T08:00:00.000Z", payload: JSON.stringify({ step: "wait-retry" }), }); // 1. Lock held briefly (70ms) -> flush() waits and succeeds instead of silently returning 0 await fakeRedis.set(REDIS_FLUSH_LOCK_KEY, "other-worker", "PX", 5000, "NX"); const releaseTimer = setTimeout(() => { void fakeRedis.eval("del", 1, REDIS_FLUSH_LOCK_KEY, "other-worker"); }, 70); try { const waitedFlush = await DataBufferEngine.flush({ waitMs: 500 }); assert.equal(waitedFlush.count, 1); assert.equal(waitedFlush.lockTimedOut, undefined); assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 0); } finally { clearTimeout(releaseTimer); } // 2. Lock stays held past waitMs -> throwOnLockTimeout throws explicit FLUSH_LOCK_TIMEOUT await DataBufferEngine.pushAsync({ clientId: "phase03_lock", eventId: "lock-evt-2", timestamp: "2026-11-19T08:01:00.000Z", payload: JSON.stringify({ step: "timeout" }), }); await fakeRedis.set(REDIS_FLUSH_LOCK_KEY, "stuck-worker", "PX", 5000, "NX"); await assert.rejects( () => DataBufferEngine.flush({ waitMs: 60, throwOnLockTimeout: true }), /FLUSH_LOCK_TIMEOUT/, ); const nonThrowing = await DataBufferEngine.flush({ waitMs: 40 }); assert.equal(nonThrowing.count, 0); assert.equal(nonThrowing.lockTimedOut, true); await fakeRedis.eval("del", 1, REDIS_FLUSH_LOCK_KEY, "stuck-worker"); // 3. Long flush exceeding lockTtlMs (90ms) renews TTL via Lua pexpire and completes cleanly const originalAppend = CSVStorageManager.appendBatch; CSVStorageManager.appendBatch = async (events) => { await new Promise((resolve) => setTimeout(resolve, 140)); return originalAppend.call(CSVStorageManager, events); }; fakeRedis.renewCalls = 0; try { const longFlush = await DataBufferEngine.flush({ lockTtlMs: 90, waitMs: 200 }); assert.equal(longFlush.count, 1); assert.ok( fakeRedis.renewCalls >= 1, `Expected Lua pexpire TTL renewal calls >= 1, got ${fakeRedis.renewCalls}`, ); assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 0); assert.equal(fakeRedis.strings.has(REDIS_FLUSH_LOCK_KEY), false); } finally { CSVStorageManager.appendBatch = originalAppend; } // 4. Verify Port 4000 POST /internal/flush returns 503 FLUSH_LOCK_TIMEOUT when requested & renews TTL const serverMod = require("../server/index.js") as IngestionServerModule & { writer: { baseDir: string; writeBatch: (events: DataEvent[]) => Promise<{ written: number; files: string[] }> }; }; const { getInternalIngestionHeaders } = await import("../core/Security/security"); const origWriterDir = serverMod.writer.baseDir; serverMod.writer.baseDir = tmpStorageDir; serverMod.setRedisClientForTests(fakeRedis); await new Promise((resolve) => serverMod.server.listen(0, "127.0.0.1", () => resolve())); const addr = serverMod.server.address() as AddressInfo; try { await fakeRedis.set(REDIS_FLUSH_LOCK_KEY, "port4000-blocker", "PX", 5000, "NX"); const timeoutRes = await fetch(`http://127.0.0.1:${addr.port}/internal/flush`, { method: "POST", headers: { "content-type": "application/json", ...getInternalIngestionHeaders(), }, body: JSON.stringify({ waitMs: 50, throwOnLockTimeout: true }), }); assert.equal(timeoutRes.status, 503); const timeoutJson = await timeoutRes.json(); assert.equal(timeoutJson.code, "FLUSH_LOCK_TIMEOUT"); await fakeRedis.eval("del", 1, REDIS_FLUSH_LOCK_KEY, "port4000-blocker"); await DataBufferEngine.pushAsync({ clientId: "phase03_lock", eventId: "lock-evt-p4000", timestamp: "2026-11-19T08:02:00.000Z", payload: JSON.stringify({ step: "p4000-renew" }), }); const origWriteBatch = serverMod.writer.writeBatch.bind(serverMod.writer); serverMod.writer.writeBatch = async (events: DataEvent[]) => { await new Promise((resolve) => setTimeout(resolve, 140)); return origWriteBatch(events); }; fakeRedis.renewCalls = 0; try { const p4000Flush = await serverMod.flushBuffer({ lockTtlMs: 90, waitMs: 200 }); assert.equal(p4000Flush.written, 1); assert.ok( fakeRedis.renewCalls >= 1, `Expected Port 4000 Lua pexpire TTL renewal calls >= 1, got ${fakeRedis.renewCalls}`, ); } finally { serverMod.writer.writeBatch = origWriteBatch; } } finally { serverMod.writer.baseDir = origWriterDir; serverMod.setRedisClientForTests(undefined); serverMod.bufferQueue.length = 0; serverMod.stopTimers(); await new Promise((resolve) => serverMod.server.close(() => resolve())); } }); test("FORENSIC BUG-19: Deleting a client purges RAM, Redis, and Port 4000 buffers so subsequent flushes never resurrect deleted CSV directories", async () => { const fakeRedis = new FakeRedisClient(); DataBufferEngine.setRedisClientForTests(fakeRedis); const serverMod = require("../server/index.js") as IngestionServerModule; const origWriterDir = serverMod.writer.baseDir; serverMod.writer.baseDir = tmpStorageDir; serverMod.setRedisClientForTests(fakeRedis); await new Promise((resolve) => serverMod.server.listen(0, "127.0.0.1", () => resolve())); const addr = serverMod.server.address() as AddressInfo; process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${addr.port}`; try { const victim = await SDKManager.registerClientAsync({ clientId: "phase03_victim", name: "Victim Client", rateLimit: 100, }); const survivor = await SDKManager.registerClientAsync({ clientId: "phase03_survivor", name: "Survivor Client", rateLimit: 100, }); // Write one initial CSV batch so victim dir exists on disk before delete await CSVStorageManager.appendBatch([ { clientId: victim.clientId, eventId: "v-disk-1", timestamp: "2026-11-19T09:00:00.000Z", payload: JSON.stringify({ phase: "initial" }), }, ]); const victimDir = path.join(tmpStorageDir, victim.clientId); assert.equal(fs.existsSync(victimDir), true); // Queue victim + survivor events across Redis, Port 3000 RAM, and Port 4000 local bufferQueue await DataBufferEngine.pushBatchAsync([ { clientId: victim.clientId, eventId: "v-redis-1", timestamp: "2026-11-19T09:01:00.000Z", payload: JSON.stringify({ src: "redis" }), }, { clientId: survivor.clientId, eventId: "s-redis-1", timestamp: "2026-11-19T09:01:01.000Z", payload: JSON.stringify({ src: "redis" }), }, ]); g.__dataStackBufferState?.buffer.push({ clientId: victim.clientId, eventId: "v-ram-1", timestamp: "2026-11-19T09:01:02.000Z", payload: JSON.stringify({ src: "ram" }), }); serverMod.bufferQueue.push( { clientId: victim.clientId, eventId: "v-p4000-1", timestamp: "2026-11-19T09:01:03.000Z", payload: JSON.stringify({ src: "port4000" }), }, { clientId: survivor.clientId, eventId: "s-p4000-1", timestamp: "2026-11-19T09:01:04.000Z", payload: JSON.stringify({ src: "port4000" }), }, ); // Delete victim with purgeFiles: true const deleted = await SDKManager.deleteClientAsync(victim.clientId, { purgeFiles: true }); assert.equal(deleted, true); assert.equal(fs.existsSync(victimDir), false); // Recent events in Redis must also be stripped of victim const recentAfterDelete = await fakeRedis.lrange(REDIS_RECENT_KEY, 0, -1); assert.equal( recentAfterDelete.some((raw) => raw.includes("phase03_victim")), false, ); // Flush both Port 3000 and Port 4000 buffers -> victim dir must NOT resurrect, survivor MUST flush const webFlush = await DataBufferEngine.flush(); const workerFlush = await serverMod.flushBuffer(); assert.equal( fs.existsSync(victimDir), false, "Deleted client directory must never be resurrected by queued buffer flushes", ); assert.equal(webFlush.count + workerFlush.written, 2); const survivorFiles = CSVStorageManager.listAllFiles().filter( (f) => f.clientId === survivor.clientId, ); assert.equal(survivorFiles.length, 1); assert.equal(survivorFiles[0].linesCount, 2); } finally { serverMod.writer.baseDir = origWriterDir; serverMod.setRedisClientForTests(undefined); serverMod.bufferQueue.length = 0; serverMod.stopTimers(); await new Promise((resolve) => serverMod.server.close(() => resolve())); delete process.env.INGESTION_INTERNAL_URL; } }); test("FORENSIC BUG-20: Socket.IO sync_data and batch_sync ACKs wait until async Redis RPUSH completes", async () => { const fakeRedis = new FakeRedisClient(); const rpushGateControl: { resolve?: () => void } = {}; let rpushEnteredResolve: (() => void) | null = null; let rpushGate = new Promise((resolve) => { rpushGateControl.resolve = resolve; }); let rpushEntered = new Promise((resolve) => { rpushEnteredResolve = resolve; }); const baseRpush = fakeRedis.rpush.bind(fakeRedis); fakeRedis.rpush = async (key: string, ...values: string[]) => { if (key === REDIS_BUFFER_KEY) { rpushEnteredResolve?.(); await rpushGate; } return baseRpush(key, ...values); }; const serverMod = require("../server/index.js") as IngestionServerModule; const origWriterDir = serverMod.writer.baseDir; serverMod.writer.baseDir = tmpStorageDir; serverMod.setRedisClientForTests(fakeRedis); await new Promise((resolve) => serverMod.server.listen(0, "127.0.0.1", () => resolve())); const addr = serverMod.server.address() as AddressInfo; process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${addr.port}`; const client = await SDKManager.registerClientAsync({ clientId: "phase03_sock_ack", name: "Socket Ack Durability Client", rateLimit: 100, }); const socket = ioClient(`http://127.0.0.1:${addr.port}`, { transports: ["websocket"], auth: { token: client.token }, forceNew: true, reconnection: false, }); try { await new Promise((resolve, reject) => { socket.once("connect", () => resolve()); socket.once("connect_error", (err) => reject(err)); }); // 1. sync_data must NOT ACK while rpush is still pending let syncAcked = false; const syncAckPromise = new Promise<{ success: boolean; eventId?: string }>((resolve) => { socket.emit( "sync_data", { eventId: "ack-durability-1", timestamp: "2026-11-19T10:00:00.000Z", data: { ok: true }, }, (ack: { success: boolean; eventId?: string }) => { syncAcked = true; resolve(ack); }, ); }); await rpushEntered; await new Promise((resolve) => setTimeout(resolve, 40)); assert.equal( syncAcked, false, "sync_data ACK must NOT fire before Redis RPUSH finishes", ); rpushGateControl.resolve?.(); const syncAck = await syncAckPromise; assert.equal(syncAck.success, true); assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 1); // 2. batch_sync must also wait for Redis RPUSH before ACK rpushGate = new Promise((resolve) => { rpushGateControl.resolve = resolve; }); rpushEntered = new Promise((resolve) => { rpushEnteredResolve = resolve; }); let batchAcked = false; const batchAckPromise = new Promise<{ success: boolean; processed?: number }>((resolve) => { socket.emit( "batch_sync", [ { eventId: "ack-durability-b1", timestamp: "2026-11-19T10:00:01.000Z", data: { seq: 1 }, }, { eventId: "ack-durability-b2", timestamp: "2026-11-19T10:00:02.000Z", data: { seq: 2 }, }, ], (ack: { success: boolean; processed?: number }) => { batchAcked = true; resolve(ack); }, ); }); await rpushEntered; await new Promise((resolve) => setTimeout(resolve, 40)); assert.equal( batchAcked, false, "batch_sync ACK must NOT fire before Redis RPUSH finishes", ); rpushGateControl.resolve?.(); const batchAck = await batchAckPromise; assert.equal(batchAck.success, true); assert.equal(batchAck.processed, 2); assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 3); } finally { socket.disconnect(); serverMod.writer.baseDir = origWriterDir; serverMod.setRedisClientForTests(undefined); serverMod.bufferQueue.length = 0; serverMod.stopTimers(); await new Promise((resolve) => serverMod.server.close(() => resolve())); delete process.env.INGESTION_INTERNAL_URL; } }); test("FORENSIC BUG-29: DATASTACK_STORAGE_DIR resolves canonically across CSVStorageManager, CsvWriterWorker, and SDKManager.purgeClientStorageDir", async () => { const prevStorageEnv = process.env.DATASTACK_STORAGE_DIR; const customRoot = fs.mkdtempSync(path.join(os.tmpdir(), "datastack-canonical-root-")); const prevBaseDir = CSVStorageManager.baseStorageDir; try { process.env.DATASTACK_STORAGE_DIR = customRoot; const webResolved = resolveWebStorageDir(); const workerResolved = resolveWorkerStorageDir(); assert.equal(webResolved, path.resolve(customRoot)); assert.equal(workerResolved, path.resolve(customRoot)); assert.equal(webResolved, workerResolved); const worker = new CsvWriterWorker(); assert.equal(worker.baseDir, path.resolve(customRoot)); CSVStorageManager.baseStorageDir = webResolved; const client = await SDKManager.registerClientAsync({ clientId: "phase03_canonical", name: "Canonical Storage Client", rateLimit: 100, }); await worker.writeBatch([ { clientId: client.clientId, eventId: "canon-evt-1", timestamp: "2026-11-19T11:00:00.000Z", payload: JSON.stringify({ canonical: true }), }, ]); const filesBefore = CSVStorageManager.listAllFiles().filter( (f) => f.clientId === client.clientId, ); assert.equal(filesBefore.length, 1); assert.ok(filesBefore[0].filePath.startsWith(path.resolve(customRoot))); const deleted = await SDKManager.deleteClientAsync(client.clientId, { purgeFiles: true }); assert.equal(deleted, true); assert.equal(fs.existsSync(path.join(customRoot, client.clientId)), false); } finally { CSVStorageManager.baseStorageDir = prevBaseDir; if (prevStorageEnv === undefined) delete process.env.DATASTACK_STORAGE_DIR; else process.env.DATASTACK_STORAGE_DIR = prevStorageEnv; fs.rmSync(customRoot, { recursive: true, force: true }); } }); test("R2 PHASE-02 BUG-06: purgeClientEventsAsync uses atomic Redis Lua EVAL instead of non-atomic LTRIM+LPUSH", async () => { const fakeRedis = new FakeRedisClient(); DataBufferEngine.setRedisClientForTests(fakeRedis); try { await DataBufferEngine.pushBatchAsync([ { clientId: "keep_alpha", eventId: "keep-1", timestamp: "2026-11-19T12:00:00.000Z", payload: JSON.stringify({ seq: 1 }), }, { clientId: "delete_me", eventId: "del-1", timestamp: "2026-11-19T12:00:01.000Z", payload: JSON.stringify({ seq: 2 }), }, { clientId: "keep_beta", eventId: "keep-2", timestamp: "2026-11-19T12:00:02.000Z", payload: JSON.stringify({ seq: 3 }), }, ]); const ltrimBefore = fakeRedis.ltrimCalls; const lpushBefore = fakeRedis.lpushCalls; const purgeRes = await DataBufferEngine.purgeClientEventsAsync("delete_me"); assert.equal(purgeRes.removedRedis, 1); assert.equal( fakeRedis.purgeEvalCalls, 2, "Must purge both buffer & recent lists atomically via Lua EVAL", ); assert.equal( fakeRedis.ltrimCalls, ltrimBefore, "Purge must NOT execute separate non-atomic LTRIM", ); assert.equal( fakeRedis.lpushCalls, lpushBefore, "Purge must NOT execute separate non-atomic LPUSH", ); const remaining = (await fakeRedis.lrange(REDIS_BUFFER_KEY, 0, -1)).map( (raw) => (JSON.parse(raw) as DataEvent).eventId, ); assert.deepEqual(remaining, ["keep-1", "keep-2"]); } finally { DataBufferEngine.setRedisClientForTests(null); } }); test("R2 PHASE-02 BUG-07: GET /api/export and POST /api/storage propagate local and remote FLUSH_LOCK_TIMEOUT as HTTP 503", async () => { const origFlush = DataBufferEngine.flush; const origIngestionUrl = process.env.INGESTION_INTERNAL_URL; try { // 1. Local DataBufferEngine.flush() throws FLUSH_LOCK_TIMEOUT -> both routes return 503 FLUSH_LOCK_TIMEOUT DataBufferEngine.flush = async (opts) => { assert.equal(opts?.throwOnLockTimeout, true); throw new Error("FLUSH_LOCK_TIMEOUT: Timed out waiting for distributed Redis flush lock"); }; const exportLocalTimeout = await getExportRoute( new NextRequest("http://localhost:3000/api/export?format=json", { headers: { cookie: adminCookieHeader }, }), ); assert.equal(exportLocalTimeout.status, 503); assert.equal((await exportLocalTimeout.json()).code, "FLUSH_LOCK_TIMEOUT"); const storageLocalTimeout = await postStorageRoute( new NextRequest("http://localhost:3000/api/storage", { method: "POST", headers: { cookie: adminCookieHeader }, }), ); assert.equal(storageLocalTimeout.status, 503); assert.equal((await storageLocalTimeout.json()).code, "FLUSH_LOCK_TIMEOUT"); // 2. Local flush succeeds, remote Port 4000 /internal/flush receives throwOnLockTimeout: true and returns 503 DataBufferEngine.flush = async () => ({ count: 0, files: [] }); const receivedBodies: Array> = []; const mockBridge = createServer((req, res) => { let body = ""; req.on("data", (chunk) => { body += chunk; }); req.on("end", () => { try { receivedBodies.push(JSON.parse(body || "{}")); } catch { receivedBodies.push({}); } res.writeHead(503, { "Content-Type": "application/json" }); res.end( JSON.stringify({ success: false, code: "FLUSH_LOCK_TIMEOUT", error: "FLUSH_LOCK_TIMEOUT: Port 4000 lock busy", }), ); }); }); await new Promise((resolve) => mockBridge.listen(0, "127.0.0.1", () => resolve())); const { port } = mockBridge.address() as AddressInfo; process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${port}`; try { const exportRemoteTimeout = await getExportRoute( new NextRequest("http://localhost:3000/api/export?format=json", { headers: { cookie: adminCookieHeader }, }), ); assert.equal(exportRemoteTimeout.status, 503); assert.equal((await exportRemoteTimeout.json()).code, "FLUSH_LOCK_TIMEOUT"); const storageRemoteTimeout = await postStorageRoute( new NextRequest("http://localhost:3000/api/storage", { method: "POST", headers: { cookie: adminCookieHeader }, }), ); assert.equal(storageRemoteTimeout.status, 503); assert.equal((await storageRemoteTimeout.json()).code, "FLUSH_LOCK_TIMEOUT"); assert.equal(receivedBodies.length, 2); assert.equal(receivedBodies[0].throwOnLockTimeout, true); assert.equal(receivedBodies[1].throwOnLockTimeout, true); } finally { await new Promise((resolve) => mockBridge.close(() => resolve())); } } finally { DataBufferEngine.flush = origFlush; if (origIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL; else process.env.INGESTION_INTERNAL_URL = origIngestionUrl; } }); test("R2 PHASE-02 BUG-08: Port 3000 flushAndShutdownOnSignal flushes pending SDKManager activity deltas before exit", async () => { const origFlushActivity = SDKManager.flushPendingClientActivityAsync; let activityFlushCount = 0; SDKManager.flushPendingClientActivityAsync = async () => { activityFlushCount += 1; return 1; }; try { DataBufferEngine.setRedisClientForTests(null); await DataBufferEngine.pushAsync({ clientId: "phase02_shutdown_cli", eventId: "shutdown-evt-1", timestamp: "2026-11-19T12:30:00.000Z", payload: JSON.stringify({ shutdown: true }), }); await DataBufferEngine.flushAndShutdownOnSignal(); assert.equal( activityFlushCount, 1, "flushAndShutdownOnSignal must flush coalesced SDKManager client activity deltas", ); assert.equal( (await DataBufferEngine.getBufferStatsAsync()).localBufferSize, 0, "flushAndShutdownOnSignal must also flush buffered CSV events", ); } finally { SDKManager.flushPendingClientActivityAsync = origFlushActivity; } }); test("R2 PHASE-03 BUG-13: GET /api/storage performs a single disk scan with client-scoped stats and async pruners remove expired unlocked files", async () => { await CSVStorageManager.appendBatch([ { clientId: "alpha_store", eventId: "st-a1", timestamp: "2026-11-18T10:00:00.000Z", payload: JSON.stringify({ n: 1 }), }, { clientId: "alpha_store", eventId: "st-a2", timestamp: "2026-11-19T10:00:00.000Z", payload: JSON.stringify({ n: 2 }), }, { clientId: "beta_store", eventId: "st-b1", timestamp: "2026-11-19T11:00:00.000Z", payload: JSON.stringify({ n: 3 }), }, ]); let listCalls = 0; const origListAsync = CSVStorageManager.listAllFilesAsync.bind(CSVStorageManager); CSVStorageManager.listAllFilesAsync = async () => { listCalls += 1; return origListAsync(); }; try { const scopedRes = await getStorageRoute( new NextRequest("http://localhost:3000/api/storage?clientId=ALPHA_STORE", { headers: { cookie: adminCookieHeader }, }), ); assert.equal(scopedRes.status, 200); assert.equal(listCalls, 1, "GET /api/storage must execute listAllFilesAsync strictly once"); const scopedJson = await scopedRes.json(); assert.equal(scopedJson.files.length, 2); assert.equal(scopedJson.stats.totalFiles, 2); assert.equal(scopedJson.stats.totalRows, 2); assert.equal(scopedJson.stats.uniqueClients, 1); } finally { CSVStorageManager.listAllFilesAsync = origListAsync; } // Non-blocking async pruning on CSVStorageManager (respects active lock) const lockedDir = CSVStorageManager.ensureClientDir("async_prune_locked"); fs.writeFileSync(path.join(lockedDir, "2020-01-01.csv"), '"h"\n"r"\n', "utf8"); fs.writeFileSync(path.join(lockedDir, ".2020-01-01.csv.lock"), `1:${Date.now()}:active`, "utf8"); const unlockedDir = CSVStorageManager.ensureClientDir("async_prune_unlocked"); fs.writeFileSync(path.join(unlockedDir, "2020-01-01.csv"), '"h"\n"r"\n', "utf8"); const prunedWeb = await CSVStorageManager.pruneExpiredPartitionsAsync(30); assert.equal(prunedWeb, 1); assert.equal(fs.existsSync(path.join(unlockedDir, "2020-01-01.csv")), false); assert.equal(fs.existsSync(path.join(lockedDir, "2020-01-01.csv")), true); fs.unlinkSync(path.join(lockedDir, ".2020-01-01.csv.lock")); // Non-blocking async pruning on Port 4000 CsvWriterWorker const worker = new CsvWriterWorker(tmpStorageDir); const workerOldDir = CSVStorageManager.ensureClientDir("worker_async_prune"); fs.writeFileSync(path.join(workerOldDir, "2019-06-01.csv"), '"h"\n"r"\n', "utf8"); const prunedWorker = await worker.pruneExpiredPartitionsAsync(30); assert.equal(prunedWorker, 2, "Should prune both worker_async_prune and previously locked 2020-01-01 partition"); assert.equal(fs.existsSync(path.join(workerOldDir, "2019-06-01.csv")), false); }); test("R2 PHASE-03 BUG-15: GET /api/export rejects inverted date range with 400 and export/recent clientId filters are case-insensitive", async () => { await CSVStorageManager.appendBatch([ { clientId: "case_client_01", eventId: "case-evt-1", timestamp: "2026-11-15T12:00:00.000Z", payload: JSON.stringify({ ok: true }), }, ]); // 1. Inverted date range returns 400 INVALID_DATE_RANGE const badRangeRes = await getExportRoute( new NextRequest( "http://localhost:3000/api/export?from=2026-11-20&to=2026-11-10&flush=false", { headers: { cookie: adminCookieHeader } }, ), ); assert.equal(badRangeRes.status, 400); const badRangeJson = await badRangeRes.json(); assert.equal(badRangeJson.code, "INVALID_DATE_RANGE"); // 2. ApiEngine.getExportFiles & getExportFilesAsync match case-insensitively const syncMatched = ApiEngine.getExportFiles("CASE_CLIENT_01"); assert.equal(syncMatched.length, 1); assert.equal(syncMatched[0].clientId, "case_client_01"); const asyncMatched = await ApiEngine.getExportFilesAsync(" CASE_CLIENT_01 "); assert.equal(asyncMatched.length, 1); assert.equal(asyncMatched[0].clientId, "case_client_01"); // 3. Port 4000 /internal/recent normalizes clientId filter case-insensitively const serverIndexSrc = fs.readFileSync( path.resolve(process.cwd(), "server/index.js"), "utf8", ); assert.match(serverIndexSrc, /const filterClientId =\s*rawClientId && rawClientId !== 'all' \? normalizeClientId\(rawClientId\) : ''/); }); test("P02 R-02 diagnostic: post-CSV Redis trim failure replays identical event on retry", async () => { const redis = new FakeRedisClient(); const realTrim = redis.ltrim.bind(redis); let injected = false; redis.ltrim = async (key: string, start: number, stop: number): Promise<"OK"> => { if (key === REDIS_BUFFER_KEY && !injected) { injected = true; throw new Error("P02 simulated crash boundary after CSV append, before Redis trim"); } return realTrim(key, start, stop); }; const event: DataEvent = { clientId: "p02_csv_replay", eventId: "p02-replay-001", timestamp: "2026-11-22T01:00:00.000Z", payload: JSON.stringify({ marker: "p02" }), }; DataBufferEngine.setRedisClientForTests(redis); const originalAppend = CSVStorageManager.appendBatch; const writtenIds: string[] = []; CSVStorageManager.appendBatch = async (events) => { const result = await originalAppend.call(CSVStorageManager, events); writtenIds.push(...events.map((row) => row.eventId)); return result; }; try { await DataBufferEngine.pushBatchAsync([event]); assert.equal(await redis.llen(REDIS_BUFFER_KEY), 1); await assert.rejects(() => DataBufferEngine.flush(), /P02 simulated crash boundary/); assert.equal(injected, true); assert.equal(writtenIds.filter((id) => id === event.eventId).length, 1); assert.equal(await redis.llen(REDIS_BUFFER_KEY), 1, "Redis still queues previously appended CSV event"); await DataBufferEngine.flush(); assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0); assert.equal(writtenIds.filter((id) => id === event.eventId).length, 1, "Replay must detect the persisted CSV eventId before attempting another append"); const csvPath = path.join(tmpStorageDir, event.clientId, "2026-11-22.csv"); const csvContents = fs.readFileSync(csvPath, "utf8"); assert.equal(csvContents.split(event.eventId).length - 1, 1, "Exactly one physical CSV row must exist after a failed LTRIM and retry"); } finally { CSVStorageManager.appendBatch = originalAppend; DataBufferEngine.setRedisClientForTests(undefined); } }); test("P02 R-02: Port 4000 CSV worker suppresses a crash-window replay after Redis LTRIM error", async () => { const mockRedis = new FakeRedisClient(); const originalTrim = mockRedis.ltrim.bind(mockRedis); let failedOnce = false; mockRedis.ltrim = async (key: string, start: number, stop: number): Promise<"OK"> => { if (key === REDIS_BUFFER_KEY && !failedOnce) { failedOnce = true; throw new Error("P02 Port4000 injected post-CSV trim failure"); } return originalTrim(key, start, stop); }; const ingestion = require("../server/index.js") as IngestionServerModule; const previousDir = ingestion.writer.baseDir; const previousQueue = ingestion.bufferQueue.slice(); ingestion.writer.baseDir = tmpStorageDir; ingestion.bufferQueue.length = 0; ingestion.setRedisClientForTests(mockRedis); const evt: DataEvent = { clientId: "p02_ingestion_replay", eventId: "p02-port4000-001", timestamp: "2026-11-22T13:00:00.000Z", payload: JSON.stringify({ stage: "before-ltrim" }), }; try { await mockRedis.rpush(REDIS_BUFFER_KEY, JSON.stringify(evt)); await assert.rejects( () => ingestion.flushBuffer(), /P02 Port4000 injected post-CSV trim failure/, ); assert.equal(failedOnce, true); assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 1); const csvPath = path.join(tmpStorageDir, evt.clientId, "2026-11-22.csv"); assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1); // Crash/restart may occur after daily part rotation and archive compression. // Verify replay detection against an actual rotated .csv.gz partition. const zlib = await import("node:zlib"); const archivedPath = path.join(tmpStorageDir, evt.clientId, "2026-11-22-part2.csv.gz"); fs.writeFileSync(archivedPath, zlib.gzipSync(fs.readFileSync(csvPath))); fs.unlinkSync(csvPath); const recovered = await ingestion.flushBuffer(); assert.equal(recovered.written, 0, "Recovered compressed replay must not append again"); assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 0); assert.equal(fs.existsSync(csvPath), false, "No duplicate active CSV partition should be created"); const archivedRows = zlib.gunzipSync(fs.readFileSync(archivedPath)).toString("utf8"); assert.equal(archivedRows.split(evt.eventId).length - 1, 1); } finally { ingestion.writer.baseDir = previousDir; ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...previousQueue); ingestion.setRedisClientForTests(undefined); } });