import { SettingsManager } from "@/core/adminSettings/settingsManager"; import { DataBufferEngine } from "@/core/data/buffer"; import { getInternalIngestionHeaders } from "@/core/Security/security"; import { CSVStorageManager } from "@/core/Storage/csvStorage"; import { parse } from "csv-parse"; import { NextRequest, NextResponse } from "next/server"; import { createReadStream, readdirSync, statSync } from "node:fs"; import path from "node:path"; interface DataEvent { clientId: string; eventId: string; timestamp: string; payload: string; status: "stored" | "buffered"; } export async function GET(request: NextRequest) { try { const { searchParams } = new URL(request.url); const clientId = searchParams.get("clientId") || "all"; const from = searchParams.get("from") || ""; const to = searchParams.get("to") || ""; const search = (searchParams.get("search") || "").toLowerCase(); const page = Math.max(1, Number(searchParams.get("page")) || 1); const pageSize = Math.min( 100, Math.max(1, Number(searchParams.get("pageSize")) || 12), ); const offset = (page - 1) * pageSize; const selectedRows: DataEvent[] = []; const eventIds = new Set(); let total = 0; const addIfMatching = (event: DataEvent) => { const eventDate = CSVStorageManager.extractSafePartitionDate(event.timestamp, null); const dedupeKey = `${event.clientId}:${event.eventId}`; const matchesSearch = !search || event.payload.toLowerCase().includes(search) || event.eventId.toLowerCase().includes(search) || event.clientId.toLowerCase().includes(search); if ( (clientId !== "all" && event.clientId !== clientId) || ((from || to) && (!eventDate || (from && eventDate < from) || (to && eventDate > to))) || !matchesSearch || eventIds.has(dedupeKey) ) { return; } eventIds.add(dedupeKey); total += 1; let low = 0; let high = selectedRows.length; while (low < high) { const middle = (low + high) >>> 1; if (selectedRows[middle].timestamp >= event.timestamp) low = middle + 1; else high = middle; } if (low < offset + pageSize) { selectedRows.splice(low, 0, event); if (selectedRows.length > offset + pageSize) selectedRows.pop(); } }; const storageDir = CSVStorageManager.baseStorageDir; let clientDirs: string[] = []; try { clientDirs = readdirSync(storageDir); } catch { clientDirs = []; } for (const storedClientId of clientDirs) { if (clientId !== "all" && storedClientId !== clientId) continue; const clientDir = path.join(storageDir, storedClientId); if (!statSync(clientDir).isDirectory()) continue; let filenames: string[]; try { filenames = readdirSync(clientDir).filter((filename) => /^\d{4}-\d{2}-\d{2}(?:-part\d+)?\.csv$/.test(filename), ); } catch { continue; } for (const filename of filenames) { const fileDate = filename.slice(0, 10); if ((from && fileDate < from) || (to && fileDate > to)) continue; const parser = createReadStream(path.join(clientDir, filename)).pipe( parse({ columns: true, skip_empty_lines: true, bom: true }), ); try { for await (const record of parser as AsyncIterable< Record >) { const event = { clientId: record.clientId || storedClientId, eventId: record.eventId || "", timestamp: record.timestamp || `${fileDate}T00:00:00.000Z`, payload: record.payload || "", status: "stored" as const, }; if (event.eventId) addIfMatching(event); } } catch { parser.destroy(); } } } const settings = await SettingsManager.getSettingsAsync(); const recentWindowLimit = Math.min( 2000, Math.max(200, settings.batchFlushThreshold || 500), ); for (const event of await DataBufferEngine.getRecentEventsAsync( recentWindowLimit, clientId, )) { addIfMatching({ ...event, status: "buffered" }); } const ingestionBaseUrl = ( process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000" ).replace(/\/$/, ""); try { const qs = new URLSearchParams({ limit: String(recentWindowLimit) }); if (clientId && clientId !== "all") { qs.set("clientId", clientId); } const res = await fetch( `${ingestionBaseUrl}/internal/recent?${qs.toString()}`, { headers: getInternalIngestionHeaders(), cache: "no-store", signal: AbortSignal.timeout(1000), }, ); if (res.ok) { const data = await res.json(); if (Array.isArray(data.events)) { for (const evt of data.events) { if (evt && evt.eventId) { addIfMatching({ clientId: String(evt.clientId || ""), eventId: String(evt.eventId), timestamp: String(evt.timestamp || new Date().toISOString()), payload: typeof evt.payload === "string" ? evt.payload : JSON.stringify(evt.payload ?? {}), status: "buffered", }); } } } } } catch { // Port 4000 may not be running in isolated single-process dev mode } return NextResponse.json({ events: selectedRows.slice(offset), total, page, pageSize, pageCount: Math.max(1, Math.ceil(total / pageSize)), }); } catch (error) { const message = error instanceof Error ? error.message : "Data retrieval failed"; return NextResponse.json({ error: message }, { status: 500 }); } }