2026-11-18 05:18:55 +06:00
|
|
|
import { SettingsManager } from "@/core/adminSettings/settingsManager";
|
2026-10-23 19:56:38 -07:00
|
|
|
import { DataBufferEngine } from "@/core/data/buffer";
|
2026-11-19 20:22:01 +06:00
|
|
|
import { SDKManager } from "@/core/sdk/sdkManager";
|
2026-11-18 05:18:55 +06:00
|
|
|
import { getInternalIngestionHeaders } from "@/core/Security/security";
|
2026-10-23 19:56:38 -07:00
|
|
|
import { CSVStorageManager } from "@/core/Storage/csvStorage";
|
2026-11-19 20:22:01 +06:00
|
|
|
import { requireAdminApiSession } from "@/lib/adminAuth";
|
2026-10-23 19:56:38 -07:00
|
|
|
import { parse } from "csv-parse";
|
|
|
|
|
import { NextRequest, NextResponse } from "next/server";
|
2026-11-19 20:22:01 +06:00
|
|
|
import { createReadStream } from "node:fs";
|
|
|
|
|
import { lstat, readdir } from "node:fs/promises";
|
2026-10-23 19:56:38 -07:00
|
|
|
import path from "node:path";
|
|
|
|
|
|
|
|
|
|
interface DataEvent {
|
|
|
|
|
clientId: string;
|
|
|
|
|
eventId: string;
|
|
|
|
|
timestamp: string;
|
|
|
|
|
payload: string;
|
|
|
|
|
status: "stored" | "buffered";
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
function parseTimestampMs(raw: string): number {
|
|
|
|
|
const trimmed = String(raw || "").trim();
|
|
|
|
|
if (/^\d+(\.\d+)?$/.test(trimmed)) {
|
|
|
|
|
const num = Number(trimmed);
|
|
|
|
|
return num < 1e11 ? num * 1000 : num;
|
|
|
|
|
}
|
|
|
|
|
const parsed = Date.parse(trimmed);
|
|
|
|
|
return Number.isNaN(parsed) ? Number.NEGATIVE_INFINITY : parsed;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function compareEventsDesc(a: DataEvent, b: DataEvent): number {
|
|
|
|
|
const aMs = parseTimestampMs(a.timestamp);
|
|
|
|
|
const bMs = parseTimestampMs(b.timestamp);
|
|
|
|
|
if (aMs !== bMs) {
|
|
|
|
|
return bMs - aMs;
|
|
|
|
|
}
|
|
|
|
|
return b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0;
|
|
|
|
|
}
|
|
|
|
|
|
2026-10-23 19:56:38 -07:00
|
|
|
export async function GET(request: NextRequest) {
|
2026-11-19 20:22:01 +06:00
|
|
|
const auth = await requireAdminApiSession(request);
|
|
|
|
|
if (!auth.valid) {
|
|
|
|
|
return auth.response;
|
|
|
|
|
}
|
|
|
|
|
|
2026-10-23 19:56:38 -07:00
|
|
|
try {
|
|
|
|
|
const { searchParams } = new URL(request.url);
|
2026-11-19 20:22:01 +06:00
|
|
|
const rawClientId = searchParams.get("clientId") || "all";
|
|
|
|
|
const clientId =
|
|
|
|
|
rawClientId === "all"
|
|
|
|
|
? "all"
|
|
|
|
|
: SDKManager.normalizeClientId(rawClientId) || "all";
|
|
|
|
|
const sanitizedClientId =
|
|
|
|
|
clientId === "all" ? "all" : CSVStorageManager.sanitizeClientId(clientId);
|
2026-10-23 19:56:38 -07:00
|
|
|
const from = searchParams.get("from") || "";
|
|
|
|
|
const to = searchParams.get("to") || "";
|
|
|
|
|
const search = (searchParams.get("search") || "").toLowerCase();
|
|
|
|
|
const page = Math.max(1, Number(searchParams.get("page")) || 1);
|
|
|
|
|
const pageSize = Math.min(
|
|
|
|
|
100,
|
|
|
|
|
Math.max(1, Number(searchParams.get("pageSize")) || 12),
|
|
|
|
|
);
|
|
|
|
|
const offset = (page - 1) * pageSize;
|
|
|
|
|
const selectedRows: DataEvent[] = [];
|
|
|
|
|
const eventIds = new Set<string>();
|
|
|
|
|
let total = 0;
|
|
|
|
|
|
|
|
|
|
const addIfMatching = (event: DataEvent) => {
|
2026-11-19 00:42:38 +06:00
|
|
|
const eventDate = CSVStorageManager.extractSafePartitionDate(event.timestamp, null);
|
2026-11-19 20:22:01 +06:00
|
|
|
const normEventClient = SDKManager.normalizeClientId(event.clientId) || event.clientId;
|
|
|
|
|
const dedupeKey = `${normEventClient}:${event.eventId}`;
|
|
|
|
|
const matchesClient =
|
|
|
|
|
clientId === "all" ||
|
|
|
|
|
normEventClient === clientId ||
|
|
|
|
|
CSVStorageManager.sanitizeClientId(normEventClient) === sanitizedClientId;
|
2026-11-18 05:18:55 +06:00
|
|
|
const matchesSearch =
|
|
|
|
|
!search ||
|
|
|
|
|
event.payload.toLowerCase().includes(search) ||
|
|
|
|
|
event.eventId.toLowerCase().includes(search) ||
|
|
|
|
|
event.clientId.toLowerCase().includes(search);
|
|
|
|
|
|
2026-10-23 19:56:38 -07:00
|
|
|
if (
|
2026-11-19 20:22:01 +06:00
|
|
|
!matchesClient ||
|
2026-11-19 00:42:38 +06:00
|
|
|
((from || to) && (!eventDate || (from && eventDate < from) || (to && eventDate > to))) ||
|
2026-11-18 05:18:55 +06:00
|
|
|
!matchesSearch ||
|
|
|
|
|
eventIds.has(dedupeKey)
|
2026-10-23 19:56:38 -07:00
|
|
|
) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
eventIds.add(dedupeKey);
|
2026-10-23 19:56:38 -07:00
|
|
|
total += 1;
|
|
|
|
|
|
|
|
|
|
let low = 0;
|
|
|
|
|
let high = selectedRows.length;
|
|
|
|
|
while (low < high) {
|
|
|
|
|
const middle = (low + high) >>> 1;
|
2026-11-19 20:22:01 +06:00
|
|
|
if (compareEventsDesc(selectedRows[middle], event) <= 0) low = middle + 1;
|
2026-10-23 19:56:38 -07:00
|
|
|
else high = middle;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (low < offset + pageSize) {
|
|
|
|
|
selectedRows.splice(low, 0, event);
|
|
|
|
|
if (selectedRows.length > offset + pageSize) selectedRows.pop();
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const storageDir = CSVStorageManager.baseStorageDir;
|
|
|
|
|
let clientDirs: string[] = [];
|
|
|
|
|
try {
|
2026-11-19 20:22:01 +06:00
|
|
|
clientDirs = await readdir(storageDir);
|
2026-10-23 19:56:38 -07:00
|
|
|
} catch {
|
|
|
|
|
clientDirs = [];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (const storedClientId of clientDirs) {
|
2026-11-19 20:22:01 +06:00
|
|
|
if (
|
|
|
|
|
clientId !== "all" &&
|
|
|
|
|
storedClientId !== clientId &&
|
|
|
|
|
storedClientId !== sanitizedClientId
|
|
|
|
|
) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
2026-10-23 19:56:38 -07:00
|
|
|
|
|
|
|
|
const clientDir = path.join(storageDir, storedClientId);
|
2026-11-19 20:22:01 +06:00
|
|
|
let dirLstat;
|
|
|
|
|
try {
|
|
|
|
|
dirLstat = await lstat(clientDir);
|
|
|
|
|
} catch {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
if (dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
|
2026-10-23 19:56:38 -07:00
|
|
|
|
|
|
|
|
let filenames: string[];
|
|
|
|
|
try {
|
2026-11-19 20:22:01 +06:00
|
|
|
filenames = (await readdir(clientDir))
|
|
|
|
|
.filter((filename) => {
|
|
|
|
|
if (!/^\d{4}-\d{2}-\d{2}(?:-part\d+)?\.csv$/.test(filename)) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
const fileDate = filename.slice(0, 10);
|
|
|
|
|
if ((from && fileDate < from) || (to && fileDate > to)) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
return true;
|
|
|
|
|
})
|
|
|
|
|
.sort((a, b) => b.localeCompare(a));
|
2026-10-23 19:56:38 -07:00
|
|
|
} catch {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (const filename of filenames) {
|
|
|
|
|
const fileDate = filename.slice(0, 10);
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
const filePath = path.join(clientDir, filename);
|
|
|
|
|
let fileLstat;
|
|
|
|
|
try {
|
|
|
|
|
fileLstat = await lstat(filePath);
|
|
|
|
|
} catch {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
if (fileLstat.isSymbolicLink() || !fileLstat.isFile()) continue;
|
|
|
|
|
|
|
|
|
|
const fileStream = createReadStream(filePath);
|
|
|
|
|
const parser = fileStream.pipe(
|
2026-10-23 19:56:38 -07:00
|
|
|
parse({ columns: true, skip_empty_lines: true, bom: true }),
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
for await (const record of parser as AsyncIterable<
|
|
|
|
|
Record<string, string>
|
|
|
|
|
>) {
|
|
|
|
|
const event = {
|
|
|
|
|
clientId: record.clientId || storedClientId,
|
|
|
|
|
eventId: record.eventId || "",
|
|
|
|
|
timestamp: record.timestamp || `${fileDate}T00:00:00.000Z`,
|
|
|
|
|
payload: record.payload || "",
|
|
|
|
|
status: "stored" as const,
|
|
|
|
|
};
|
|
|
|
|
if (event.eventId) addIfMatching(event);
|
|
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
} catch (err) {
|
|
|
|
|
console.warn(`[API /data] Failed reading CSV partition ${filePath}:`, err);
|
|
|
|
|
} finally {
|
|
|
|
|
if (!parser.destroyed) parser.destroy();
|
|
|
|
|
if (!fileStream.destroyed) fileStream.destroy();
|
2026-10-23 19:56:38 -07:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
await SettingsManager.getSettingsAsync();
|
|
|
|
|
const recentWindowLimit = DataBufferEngine.getRecentLimit();
|
2026-11-18 05:18:55 +06:00
|
|
|
|
|
|
|
|
for (const event of await DataBufferEngine.getRecentEventsAsync(
|
|
|
|
|
recentWindowLimit,
|
|
|
|
|
clientId,
|
|
|
|
|
)) {
|
2026-10-23 19:56:38 -07:00
|
|
|
addIfMatching({ ...event, status: "buffered" });
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-18 03:20:00 +06:00
|
|
|
const ingestionBaseUrl = (
|
|
|
|
|
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
|
|
|
|
).replace(/\/$/, "");
|
|
|
|
|
try {
|
2026-11-18 05:18:55 +06:00
|
|
|
const qs = new URLSearchParams({ limit: String(recentWindowLimit) });
|
|
|
|
|
if (clientId && clientId !== "all") {
|
|
|
|
|
qs.set("clientId", clientId);
|
|
|
|
|
}
|
|
|
|
|
const res = await fetch(
|
|
|
|
|
`${ingestionBaseUrl}/internal/recent?${qs.toString()}`,
|
|
|
|
|
{
|
|
|
|
|
headers: getInternalIngestionHeaders(),
|
|
|
|
|
cache: "no-store",
|
|
|
|
|
signal: AbortSignal.timeout(1000),
|
|
|
|
|
},
|
|
|
|
|
);
|
2026-11-18 03:20:00 +06:00
|
|
|
if (res.ok) {
|
|
|
|
|
const data = await res.json();
|
|
|
|
|
if (Array.isArray(data.events)) {
|
|
|
|
|
for (const evt of data.events) {
|
|
|
|
|
if (evt && evt.eventId) {
|
|
|
|
|
addIfMatching({
|
|
|
|
|
clientId: String(evt.clientId || ""),
|
|
|
|
|
eventId: String(evt.eventId),
|
|
|
|
|
timestamp: String(evt.timestamp || new Date().toISOString()),
|
|
|
|
|
payload:
|
|
|
|
|
typeof evt.payload === "string"
|
|
|
|
|
? evt.payload
|
|
|
|
|
: JSON.stringify(evt.payload ?? {}),
|
|
|
|
|
status: "buffered",
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
// Port 4000 may not be running in isolated single-process dev mode
|
|
|
|
|
}
|
|
|
|
|
|
2026-10-23 19:56:38 -07:00
|
|
|
return NextResponse.json({
|
|
|
|
|
events: selectedRows.slice(offset),
|
|
|
|
|
total,
|
|
|
|
|
page,
|
|
|
|
|
pageSize,
|
|
|
|
|
pageCount: Math.max(1, Math.ceil(total / pageSize)),
|
|
|
|
|
});
|
|
|
|
|
} catch (error) {
|
|
|
|
|
const message =
|
|
|
|
|
error instanceof Error ? error.message : "Data retrieval failed";
|
|
|
|
|
return NextResponse.json({ error: message }, { status: 500 });
|
|
|
|
|
}
|
|
|
|
|
}
|