import assert from "node:assert/strict"; import fs from "node:fs"; import { createServer, type Server } from "node:http"; import type { AddressInfo } from "node:net"; import { createRequire } from "node:module"; import os from "node:os"; import path from "node:path"; import test, { after, before, beforeEach } from "node:test"; import { NextRequest } from "next/server"; import { GET as getDataRoute } from "../app/api/data/route"; import { GET as getEventsRoute } from "../app/api/events/route"; import { GET as getExportRoute } from "../app/api/export/route"; import { POST as postStorageRoute } from "../app/api/storage/route"; import { ApiEngine } from "../core/apiEngine/apiService"; import { DataBufferEngine, REDIS_BUFFER_KEY, REDIS_FLUSH_LOCK_KEY, type RedisBufferClient, } from "../core/data/buffer"; import { SDKManager } from "../core/sdk/sdkManager"; import { CSVStorageManager, type DataEvent } from "../core/Storage/csvStorage"; const require = createRequire(import.meta.url); const { CsvWriterWorker } = require("../server/workers/csvWriter") as { CsvWriterWorker: new (baseDir?: string, maxFileSize?: number) => { baseDir: string; writeBatch: (events: DataEvent[]) => Promise<{ written: number; files: string[] }>; pruneExpiredPartitions: (days: number) => number; }; }; interface IngestionServerModule { server: Server; io: import("socket.io").Server; writer: { baseDir: string }; bufferQueue: DataEvent[]; runtimeConfig: { batchFlushThreshold: number; autoPruneDays: number; }; applyRuntimeSettingsObject: (obj: Record) => { jwtRotated: boolean }; flushBuffer: () => Promise<{ written: number; files: string[] }>; setRedisClientForTests: (client: RedisBufferClient | null | undefined) => void; stopTimers: () => void; } class FakeRedisClient implements RedisBufferClient { public status = "ready"; public lists = new Map(); public strings = new Map(); public async rpush(key: string, ...values: string[]): Promise { const list = this.lists.get(key) || []; list.push(...values); this.lists.set(key, list); return list.length; } public async lpush(key: string, ...values: string[]): Promise { const list = this.lists.get(key) || []; for (const v of values) { list.unshift(v); } this.lists.set(key, list); return list.length; } public async ltrim(key: string, start: number, stop: number): Promise<"OK"> { const list = this.lists.get(key) || []; const len = list.length; const normStart = start < 0 ? Math.max(0, len + start) : start; const normStop = stop < 0 ? len + stop : stop; if (normStart > normStop || normStart >= len) { this.lists.set(key, []); } else { this.lists.set(key, list.slice(normStart, normStop + 1)); } return "OK"; } public async lrange(key: string, start: number, stop: number): Promise { const list = this.lists.get(key) || []; const len = list.length; if (len === 0) return []; const normStart = start < 0 ? Math.max(0, len + start) : start; const normStop = stop < 0 ? len + stop : stop; if (normStart > normStop || normStart >= len) return []; return list.slice(normStart, normStop + 1); } public async llen(key: string): Promise { return (this.lists.get(key) || []).length; } public async set( key: string, value: string, _pxMode: "PX", ttlMs: number, _nxMode: "NX", ): Promise { const existing = this.strings.get(key); if (existing && existing.expiresAt > Date.now()) { return null; } this.strings.set(key, { value, expiresAt: Date.now() + ttlMs }); return "OK"; } public async eval(_script: string, _numKeys: number, key: string, token: string): Promise { const existing = this.strings.get(key); if (existing && existing.value === token) { this.strings.delete(key); return 1; } return 0; } } const g = globalThis as typeof globalThis & { __dataStackBufferState?: { buffer: DataEvent[]; recentEvents: DataEvent[]; flushTimer: ReturnType | null; }; __dataStackAdminSettings?: unknown; __dataStackClientsMap?: Map; __dataStackDeletedClients?: Set; }; let tmpStorageDir = ""; const origBaseStorageDir = CSVStorageManager.baseStorageDir; const prevDisableListen = process.env.DATASTACK_DISABLE_LISTEN; const prevIngestionUrl = process.env.INGESTION_INTERNAL_URL; const prevDbUrl = process.env.DATABASE_URL; const prevRedisUrl = process.env.REDIS_URL; before(async () => { process.env.DATASTACK_DISABLE_LISTEN = "1"; delete process.env.DATABASE_URL; delete process.env.REDIS_URL; delete process.env.INGESTION_INTERNAL_URL; tmpStorageDir = fs.mkdtempSync(path.join(os.tmpdir(), "datastack-phase03-")); CSVStorageManager.baseStorageDir = tmpStorageDir; CSVStorageManager.clearRowCountCache(); }); beforeEach(async () => { DataBufferEngine.setRedisClientForTests(undefined); if (g.__dataStackBufferState) { g.__dataStackBufferState.buffer = []; g.__dataStackBufferState.recentEvents = []; } CSVStorageManager.clearRowCountCache(); fs.rmSync(tmpStorageDir, { recursive: true, force: true }); fs.mkdirSync(tmpStorageDir, { recursive: true }); delete g.__dataStackAdminSettings; g.__dataStackClientsMap?.clear(); g.__dataStackDeletedClients?.clear(); }); after(async () => { await DataBufferEngine.shutdown(); CSVStorageManager.baseStorageDir = origBaseStorageDir; CSVStorageManager.clearRowCountCache(); if (tmpStorageDir) { fs.rmSync(tmpStorageDir, { recursive: true, force: true }); } if (prevDisableListen === undefined) delete process.env.DATASTACK_DISABLE_LISTEN; else process.env.DATASTACK_DISABLE_LISTEN = prevDisableListen; if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL; else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl; if (prevDbUrl === undefined) delete process.env.DATABASE_URL; else process.env.DATABASE_URL = prevDbUrl; if (prevRedisUrl === undefined) delete process.env.REDIS_URL; else process.env.REDIS_URL = prevRedisUrl; }); test("BUG-11: Shared Redis buffer preserves events on CSV error, honors distributed flush lock, avoids double-counting, and falls back to RAM cleanly", async () => { const fakeRedis = new FakeRedisClient(); DataBufferEngine.setRedisClientForTests(fakeRedis); const sampleEvents: DataEvent[] = [ { clientId: "phase03_redis", eventId: "r-evt-1", timestamp: "2026-11-17T10:00:00.000Z", payload: JSON.stringify({ seq: 1 }), }, { clientId: "phase03_redis", eventId: "r-evt-2", timestamp: "2026-11-17T10:00:01.000Z", payload: JSON.stringify({ seq: 2 }), }, ]; await DataBufferEngine.pushBatchAsync(sampleEvents); assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 2); const recentFromRedis = await DataBufferEngine.getRecentEventsAsync(10, "phase03_redis"); assert.equal(recentFromRedis.length, 2); assert.equal(recentFromRedis[0].eventId, "r-evt-2"); // Lock contention must skip concurrent Redis flush without corrupting queue await fakeRedis.set(REDIS_FLUSH_LOCK_KEY, "other-worker-token", "PX", 15000, "NX"); const skipped = await DataBufferEngine.flush(); assert.equal(skipped.count, 0); assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 2); await fakeRedis.eval("", 1, REDIS_FLUSH_LOCK_KEY, "other-worker-token"); // Crash/error safety: if appendBatch throws before finishing, Redis LTRIM must NOT discard events const originalAppend = CSVStorageManager.appendBatch; CSVStorageManager.appendBatch = async () => { throw new Error("Simulated disk write failure"); }; await assert.rejects(() => DataBufferEngine.flush(), /Simulated disk write failure/); CSVStorageManager.appendBatch = originalAppend; assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 2); // Subsequent healthy flush drains Redis list, releases distributed lock, and writes CSV const flushed = await DataBufferEngine.flush(); assert.equal(flushed.count, 2); assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 0); assert.equal(fakeRedis.strings.has(REDIS_FLUSH_LOCK_KEY), false); // Verify Port 4000 /internal/recent & /internal/metrics read shared Redis without double-counting const serverMod = require("../server/index.js") as IngestionServerModule; const origWriterDir = serverMod.writer.baseDir; serverMod.writer.baseDir = tmpStorageDir; serverMod.setRedisClientForTests(fakeRedis); await new Promise((resolve) => serverMod.server.listen(0, "127.0.0.1", () => resolve())); const addr = serverMod.server.address() as AddressInfo; process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${addr.port}`; try { await DataBufferEngine.pushAsync({ clientId: "phase03_redis", eventId: "r-evt-3", timestamp: "2026-11-17T10:00:02.000Z", payload: JSON.stringify({ seq: 3 }), }); const stats = await DataBufferEngine.getBufferStatsAsync(); assert.equal(stats.redisEnabled, true); assert.equal(stats.redisBufferSize, 1); const metrics = await ApiEngine.getDashboardMetrics(); // Shared Redis count (1) must NOT be counted twice as 2 across Port 3000 + Port 4000 assert.equal(metrics.bufferStats.currentBufferSize, 1); // Clear recent list & buffer via Port 4000 flush const flushRes = await serverMod.flushBuffer(); assert.equal(flushRes.written, 1); assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 0); } finally { serverMod.writer.baseDir = origWriterDir; serverMod.setRedisClientForTests(undefined); serverMod.stopTimers(); delete process.env.INGESTION_INTERNAL_URL; await new Promise((resolve) => serverMod.io.close(() => serverMod.server.close(() => resolve()))); } // Fallback to local RAM buffer when Redis is disabled (null) DataBufferEngine.setRedisClientForTests(null); await DataBufferEngine.pushAsync({ clientId: "phase03_ram", eventId: "ram-evt-1", timestamp: "2026-11-17T10:01:00.000Z", payload: JSON.stringify({ mode: "ram" }), }); const ramStats = await DataBufferEngine.getBufferStatsAsync(); assert.equal(ramStats.redisEnabled, false); assert.equal(ramStats.localBufferSize, 1); const ramFlushed = await DataBufferEngine.flush(); assert.equal(ramFlushed.count, 1); }); test("BUG-09: Cooperative CSV partition locking serializes concurrent Port 3000 + Port 4000 writers, recovers stale locks, and surfaces Port 4000 flush errors", async () => { const worker = new CsvWriterWorker(tmpStorageDir); const partitionDate = "2026-11-17"; const clientId = "phase03_concurrent"; const batchA: DataEvent[] = Array.from({ length: 15 }, (_, idx) => ({ clientId, eventId: `a-${idx}`, timestamp: `${partitionDate}T12:00:00.000Z`, payload: JSON.stringify({ writer: "A", idx }), })); const batchB: DataEvent[] = Array.from({ length: 15 }, (_, idx) => ({ clientId, eventId: `b-${idx}`, timestamp: `${partitionDate}T12:00:01.000Z`, payload: JSON.stringify({ writer: "B", idx }), })); await Promise.all([ CSVStorageManager.appendBatch(batchA), worker.writeBatch(batchB), ]); const csvPath = path.join(tmpStorageDir, clientId, `${partitionDate}.csv`); const lockPath = path.join(tmpStorageDir, clientId, `.${partitionDate}.csv.lock`); assert.equal(fs.existsSync(lockPath), false, "Lock file must be cleaned up after writes"); const lines = fs.readFileSync(csvPath, "utf8").trim().split("\n"); assert.equal(lines[0], '"clientId","eventId","timestamp","payload"'); assert.equal(lines.length, 31, "Must contain exactly 1 header + 30 complete rows without interleaving"); // Stale lock recovery (>30s old mtime) fs.writeFileSync(lockPath, "999999:0", "utf8"); const pastTime = new Date(Date.now() - 45_000); fs.utimesSync(lockPath, pastTime, pastTime); await CSVStorageManager.appendBatch([ { clientId, eventId: "after-stale-lock", timestamp: `${partitionDate}T12:00:02.000Z`, payload: JSON.stringify({ staleRecovered: true }), }, ]); assert.equal(fs.existsSync(lockPath), false); const files = await CSVStorageManager.listAllFilesAsync(); assert.equal(files[0].linesCount, 31); // Explicit flush error & warning propagation in POST /api/storage const failingBridge = createServer((_req, res) => { res.statusCode = 500; res.setHeader("Content-Type", "application/json"); res.end(JSON.stringify({ success: false, error: "Disk permission denied on Port 4000" })); }); await new Promise((resolve) => failingBridge.listen(0, "127.0.0.1", () => resolve())); const bridgeAddr = failingBridge.address() as AddressInfo; process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${bridgeAddr.port}`; try { const errRes = await postStorageRoute(new NextRequest("http://localhost:3000/api/storage", { method: "POST" })); assert.equal(errRes.status, 502); const errJson = await errRes.json(); assert.equal(errJson.success, false); assert.equal(errJson.code, "INGESTION_FLUSH_FAILED"); assert.equal(errJson.ingestionFlushed, false); } finally { delete process.env.INGESTION_INTERNAL_URL; await new Promise((resolve) => failingBridge.close(() => resolve())); } // When Port 4000 is down and INGESTION_INTERNAL_URL is unset, standalone dev surfaces explicit warning + ingestionFlushed: false const standaloneRes = await postStorageRoute(new NextRequest("http://localhost:3000/api/storage", { method: "POST" })); assert.equal(standaloneRes.status, 200); const standaloneJson = await standaloneRes.json(); assert.equal(standaloneJson.ingestionFlushed, false); assert.match(String(standaloneJson.warning || ""), /unreachable/i); }); test("BUG-10: Cached streaming row counter counts multiline RFC 4180 payloads accurately and avoids repeated disk reads", async () => { await CSVStorageManager.appendBatch([ { clientId: "phase03_perf", eventId: "multiline-1", timestamp: "2026-11-17T08:00:00.000Z", payload: JSON.stringify({ note: "line1\nline2\n\"quoted\"\nline4" }), }, { clientId: "phase03_perf", eventId: "multiline-2", timestamp: "2026-11-17T08:00:01.000Z", payload: JSON.stringify({ note: "single" }), }, ]); const firstStats = await CSVStorageManager.getStorageStatsAsync(); assert.equal(firstStats.totalRows, 2); // Spy on fs.createReadStream to verify repeated dashboard polls hit mtime+size cache with 0 file scans const origCreateReadStream = fs.createReadStream; let streamReads = 0; (fs as unknown as { createReadStream: typeof fs.createReadStream }).createReadStream = (( ...args: Parameters ) => { streamReads += 1; return origCreateReadStream(...args); }) as typeof fs.createReadStream; try { const secondStats = await CSVStorageManager.getStorageStatsAsync(); assert.equal(secondStats.totalRows, 2); assert.equal(streamReads, 0, "Unchanged CSV files must use cached row count"); } finally { (fs as unknown as { createReadStream: typeof fs.createReadStream }).createReadStream = origCreateReadStream; } // Server 3s settings sync must not repeatedly prune when autoPruneDays is unchanged const serverMod = require("../server/index.js") as IngestionServerModule; const oldPartitionDir = path.join(tmpStorageDir, "phase03_prune_check"); fs.mkdirSync(oldPartitionDir, { recursive: true }); fs.writeFileSync(path.join(oldPartitionDir, "2020-01-01.csv"), "h\nr\n", "utf8"); const prunedCount = CSVStorageManager.pruneExpiredPartitions(30); assert.equal(prunedCount, 1); serverMod.applyRuntimeSettingsObject({ autoPruneDays: 30 }); fs.writeFileSync(path.join(oldPartitionDir, "2020-01-02.csv"), "h\nr\n", "utf8"); // Same autoPruneDays=30 applied again (simulating 3s DB poll) must NOT prune on every tick serverMod.applyRuntimeSettingsObject({ autoPruneDays: 30 }); assert.equal(fs.existsSync(path.join(oldPartitionDir, "2020-01-02.csv")), true); serverMod.stopTimers(); }); test("BUG-12: Prisma schema models and mappings align cleanly with runtime SQL DDL and Docker Compose Redis wiring", () => { const schemaPath = path.resolve(process.cwd(), "prisma/schema.prisma"); const schemaText = fs.readFileSync(schemaPath, "utf8"); assert.match(schemaText, /model ClientSDK[\s\S]*clientId\s+String\s+@id\s+@map\("client_id"\)/); assert.match(schemaText, /token\s+String/); assert.match(schemaText, /model DeletedClient[\s\S]*@@map\("deleted_clients"\)/); assert.match(schemaText, /model AdminUser[\s\S]*tokenVersion\s+Int\s+@default\(1\)\s+@map\("token_version"\)/); assert.match(schemaText, /model AdminLoginAttempt[\s\S]*@@map\("admin_login_attempts"\)/); assert.match(schemaText, /model AdminSettings[\s\S]*maxEventsPerSec\s+Int\s+@default\(100\)\s+@map\("max_events_per_sec"\)/); assert.equal(/model ApiKey\b/.test(schemaText), false); assert.equal(/model AuditLog\b/.test(schemaText), false); const composeText = fs.readFileSync(path.resolve(process.cwd(), "docker/docker-compose.yml"), "utf8"); assert.equal((composeText.match(/REDIS_URL:\s*"redis:\/\/redis:6379"/g) || []).length, 2); assert.match(composeText, /redis:\s*\n\s*condition:\s*service_healthy/); }); test("BUG-14: GET /api/data searches eventId & clientId + UTC dates, GET /api/events caps CSV tail, and GET /api/export rejects ambiguous filename & honors ?flush=false", async () => { DataBufferEngine.setRedisClientForTests(null); await SDKManager.registerClientAsync({ clientId: "alpha_cli", name: "Alpha", rateLimit: 100 }); await SDKManager.registerClientAsync({ clientId: "beta_cli", name: "Beta", rateLimit: 100 }); const tsMillis = String(Date.UTC(2026, 10, 17, 15, 0, 0)); // 2026-11-17 await DataBufferEngine.pushBatchAsync([ { clientId: "alpha_cli", eventId: "unique-findme-id-99", timestamp: tsMillis, payload: JSON.stringify({ temp: 22 }), }, { clientId: "beta_cli", eventId: "beta-id-1", timestamp: "2026-11-17T15:01:00.000Z", payload: JSON.stringify({ temp: 33 }), }, ]); // 1. Search by eventId with from/to UTC normalization while still buffered const dataByEvent = await getDataRoute( new NextRequest("http://localhost:3000/api/data?search=unique-findme-id-99&from=2026-11-17&to=2026-11-17"), ); assert.equal(dataByEvent.status, 200); const dataEventJson = await dataByEvent.json(); assert.equal(dataEventJson.total, 1); assert.equal(dataEventJson.events[0].eventId, "unique-findme-id-99"); // 2. Search by clientId substring const dataByClient = await getDataRoute( new NextRequest("http://localhost:3000/api/data?search=beta_cli"), ); const dataClientJson = await dataByClient.json(); assert.equal(dataClientJson.total, 1); assert.equal(dataClientJson.events[0].clientId, "beta_cli"); // 3. Export with ?flush=false must NOT drain active buffer const noFlushRes = await getExportRoute( new NextRequest("http://localhost:3000/api/export?flush=false&format=json"), ); assert.equal(noFlushRes.status, 200); assert.equal((await noFlushRes.text()).trim(), "[]"); assert.equal((await DataBufferEngine.getBufferStatsAsync()).localBufferSize, 2); // 4. Default flush writes both 2026-11-17.csv files -> clientId=all + filename must 400 AMBIGUOUS_FILENAME const flushExportRes = await getExportRoute( new NextRequest("http://localhost:3000/api/export?format=json"), ); assert.equal(flushExportRes.status, 200); const flushedPayload = await flushExportRes.json(); assert.equal(flushedPayload.length, 2); const ambiguousRes = await getExportRoute( new NextRequest("http://localhost:3000/api/export?clientId=all&filename=2026-11-17.csv"), ); assert.equal(ambiguousRes.status, 400); const ambiguousJson = await ambiguousRes.json(); assert.equal(ambiguousJson.code, "AMBIGUOUS_FILENAME"); // Disambiguated download with explicit clientId succeeds cleanly const specificRes = await getExportRoute( new NextRequest("http://localhost:3000/api/export?clientId=alpha_cli&filename=2026-11-17.csv"), ); assert.equal(specificRes.status, 200); const csvBody = await specificRes.text(); assert.match(csvBody, /unique-findme-id-99/); // 5. GET /api/events with limit=1 returns latest single stored event cleanly const eventsRes = await getEventsRoute( new NextRequest("http://localhost:3000/api/events?clientId=alpha_cli&limit=1"), ); assert.equal(eventsRes.status, 200); const eventsJson = await eventsRes.json(); assert.equal(eventsJson.events.length, 1); assert.equal(eventsJson.events[0].eventId, "unique-findme-id-99"); });