2026-11-18 05:18:55 +06:00
|
|
|
import { SettingsManager } from "@/core/adminSettings/settingsManager";
|
2026-11-21 22:15:32 +06:00
|
|
|
import { recordAdminAuditLogAsync } from "@/core/auth/auditLog";
|
2026-11-17 20:05:00 +06:00
|
|
|
import { DataBufferEngine } from "@/core/data/buffer";
|
2026-11-19 20:22:01 +06:00
|
|
|
import { SDKManager } from "@/core/sdk/sdkManager";
|
2026-11-18 05:18:55 +06:00
|
|
|
import { getInternalIngestionHeaders } from "@/core/Security/security";
|
2026-10-23 19:56:38 -07:00
|
|
|
import {
|
|
|
|
|
CSVStorageManager,
|
|
|
|
|
type StorageFileMeta,
|
|
|
|
|
} from "@/core/Storage/csvStorage";
|
2026-11-21 22:15:32 +06:00
|
|
|
import { extractClientIp, requireAdminApiSession } from "@/lib/adminAuth";
|
2026-10-23 19:56:38 -07:00
|
|
|
import { parse } from "csv-parse";
|
|
|
|
|
import { NextRequest, NextResponse } from "next/server";
|
|
|
|
|
import { createReadStream } from "node:fs";
|
2026-11-18 05:18:55 +06:00
|
|
|
import path from "node:path";
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-11-19 23:02:27 +06:00
|
|
|
type ExportFormat = "csv" | "json" | "ndjson";
|
2026-10-23 19:56:38 -07:00
|
|
|
type CsvRecord = Record<string, string>;
|
|
|
|
|
type StreamChunk = string | Uint8Array;
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-10-23 19:56:38 -07:00
|
|
|
function toByteStream(chunks: AsyncIterable<StreamChunk>) {
|
|
|
|
|
const iterator = chunks[Symbol.asyncIterator]();
|
|
|
|
|
const encoder = new TextEncoder();
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-10-23 19:56:38 -07:00
|
|
|
return new ReadableStream<Uint8Array>({
|
|
|
|
|
async pull(controller) {
|
2026-10-23 11:43:49 -05:00
|
|
|
try {
|
2026-10-23 19:56:38 -07:00
|
|
|
const result = await iterator.next();
|
|
|
|
|
if (result.done) {
|
|
|
|
|
controller.close();
|
|
|
|
|
} else {
|
|
|
|
|
controller.enqueue(
|
|
|
|
|
typeof result.value === "string"
|
|
|
|
|
? encoder.encode(result.value)
|
|
|
|
|
: result.value,
|
|
|
|
|
);
|
2026-10-23 11:43:49 -05:00
|
|
|
}
|
2026-10-23 19:56:38 -07:00
|
|
|
} catch (error) {
|
|
|
|
|
controller.error(error);
|
|
|
|
|
}
|
|
|
|
|
},
|
|
|
|
|
async cancel() {
|
|
|
|
|
await iterator.return?.();
|
|
|
|
|
},
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function* readFileRecords(
|
|
|
|
|
file: StorageFileMeta,
|
|
|
|
|
): AsyncGenerator<CsvRecord> {
|
2026-11-19 23:02:27 +06:00
|
|
|
const fileStream = CSVStorageManager.openPartitionTextStream(file.filePath);
|
2026-11-19 20:22:01 +06:00
|
|
|
const parser = fileStream.pipe(
|
2026-10-23 19:56:38 -07:00
|
|
|
parse({ columns: true, skip_empty_lines: true, bom: true }),
|
|
|
|
|
);
|
2026-11-19 23:02:27 +06:00
|
|
|
fileStream.on("error", (err) => {
|
|
|
|
|
if (!parser.destroyed) parser.destroy(err);
|
|
|
|
|
});
|
2026-10-23 19:56:38 -07:00
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
for await (const record of parser as AsyncIterable<CsvRecord>) {
|
|
|
|
|
yield record;
|
|
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
} finally {
|
|
|
|
|
if (!parser.destroyed) parser.destroy();
|
|
|
|
|
if (!fileStream.destroyed) fileStream.destroy();
|
2026-10-23 19:56:38 -07:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function makeExportStream(
|
|
|
|
|
files: StorageFileMeta[],
|
|
|
|
|
format: ExportFormat,
|
|
|
|
|
includeHeader: boolean,
|
|
|
|
|
) {
|
|
|
|
|
const fields = ["clientId", "eventId", "timestamp", "payload"];
|
|
|
|
|
const chunks = (async function* () {
|
|
|
|
|
let isFirst = true;
|
|
|
|
|
|
|
|
|
|
if (format === "json") {
|
|
|
|
|
yield "[";
|
2026-11-19 23:02:27 +06:00
|
|
|
} else if (format === "csv" && includeHeader) {
|
2026-10-23 19:56:38 -07:00
|
|
|
yield `${fields.join(",")}\n`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (const file of files) {
|
|
|
|
|
for await (const record of readFileRecords(file)) {
|
2026-11-19 23:02:27 +06:00
|
|
|
if (format === "json" || format === "ndjson") {
|
2026-10-23 19:56:38 -07:00
|
|
|
const payload = record.payload ?? "";
|
|
|
|
|
let parsedPayload: unknown = payload;
|
|
|
|
|
try {
|
|
|
|
|
parsedPayload = JSON.parse(payload);
|
|
|
|
|
} catch {
|
|
|
|
|
// Keep non-JSON payloads as strings.
|
|
|
|
|
}
|
2026-11-19 23:02:27 +06:00
|
|
|
const serialized = JSON.stringify({
|
|
|
|
|
...record,
|
|
|
|
|
payload: parsedPayload,
|
|
|
|
|
});
|
|
|
|
|
if (format === "ndjson") {
|
|
|
|
|
yield `${serialized}\n`;
|
|
|
|
|
} else {
|
|
|
|
|
yield `${isFirst ? "" : ","}${serialized}`;
|
|
|
|
|
}
|
2026-10-23 19:56:38 -07:00
|
|
|
} else {
|
|
|
|
|
const line = fields
|
2026-11-19 20:22:01 +06:00
|
|
|
.map((field) => `"${String(record[field] ?? "").replaceAll('"', '""')}"`)
|
2026-10-23 19:56:38 -07:00
|
|
|
.join(",");
|
|
|
|
|
yield `${line}\n`;
|
|
|
|
|
}
|
|
|
|
|
isFirst = false;
|
2026-10-23 11:43:49 -05:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-10-23 19:56:38 -07:00
|
|
|
if (format === "json") yield "]";
|
|
|
|
|
})();
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-10-23 19:56:38 -07:00
|
|
|
return toByteStream(chunks);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function streamFile(file: StorageFileMeta) {
|
|
|
|
|
async function* chunks() {
|
2026-11-19 20:22:01 +06:00
|
|
|
const fileStream = createReadStream(file.filePath);
|
|
|
|
|
try {
|
|
|
|
|
for await (const chunk of fileStream) {
|
|
|
|
|
yield new Uint8Array(chunk as Buffer);
|
|
|
|
|
}
|
|
|
|
|
} finally {
|
|
|
|
|
if (!fileStream.destroyed) fileStream.destroy();
|
2026-10-23 19:56:38 -07:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return toByteStream(chunks());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export async function GET(request: NextRequest) {
|
2026-11-19 20:22:01 +06:00
|
|
|
const auth = await requireAdminApiSession(request);
|
|
|
|
|
if (!auth.valid) {
|
|
|
|
|
return auth.response;
|
|
|
|
|
}
|
|
|
|
|
|
2026-10-23 19:56:38 -07:00
|
|
|
try {
|
2026-11-18 05:18:55 +06:00
|
|
|
const { searchParams } = new URL(request.url);
|
2026-11-19 20:22:01 +06:00
|
|
|
const from = (searchParams.get("from") || "").trim();
|
|
|
|
|
const to = (searchParams.get("to") || "").trim();
|
|
|
|
|
if (from && to && from > to) {
|
|
|
|
|
return NextResponse.json(
|
|
|
|
|
{
|
|
|
|
|
error: 'Start date ("from") must be on or before end date ("to").',
|
|
|
|
|
code: "INVALID_DATE_RANGE",
|
|
|
|
|
},
|
|
|
|
|
{ status: 400 },
|
|
|
|
|
);
|
|
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
const shouldFlush = searchParams.get("flush") !== "false";
|
|
|
|
|
|
|
|
|
|
await SettingsManager.getSettingsAsync();
|
|
|
|
|
if (shouldFlush) {
|
2026-11-19 20:22:01 +06:00
|
|
|
try {
|
|
|
|
|
await DataBufferEngine.flush({ waitMs: 2500, throwOnLockTimeout: true });
|
|
|
|
|
} catch (err: unknown) {
|
|
|
|
|
const message =
|
|
|
|
|
err instanceof Error ? err.message : "Pre-export flush failed";
|
|
|
|
|
if (message.includes("FLUSH_LOCK_TIMEOUT")) {
|
|
|
|
|
return NextResponse.json(
|
|
|
|
|
{
|
|
|
|
|
error: message,
|
|
|
|
|
code: "FLUSH_LOCK_TIMEOUT",
|
|
|
|
|
},
|
|
|
|
|
{ status: 503 },
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
throw err;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const explicitIngestionUrl = Boolean(process.env.INGESTION_INTERNAL_URL?.trim());
|
2026-11-18 05:18:55 +06:00
|
|
|
const ingestionBaseUrl = (
|
|
|
|
|
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
|
|
|
|
).replace(/\/$/, "");
|
|
|
|
|
try {
|
2026-11-19 20:22:01 +06:00
|
|
|
const flushRes = await fetch(`${ingestionBaseUrl}/internal/flush`, {
|
2026-11-18 05:18:55 +06:00
|
|
|
method: "POST",
|
2026-11-19 20:22:01 +06:00
|
|
|
headers: {
|
|
|
|
|
"Content-Type": "application/json",
|
|
|
|
|
...getInternalIngestionHeaders(),
|
|
|
|
|
},
|
|
|
|
|
body: JSON.stringify({ waitMs: 2500, throwOnLockTimeout: true }),
|
|
|
|
|
cache: "no-store",
|
|
|
|
|
signal: AbortSignal.timeout(4000),
|
2026-11-18 05:18:55 +06:00
|
|
|
});
|
2026-11-19 20:22:01 +06:00
|
|
|
if (!flushRes.ok) {
|
|
|
|
|
const errPayload = await flushRes.json().catch(() => ({}));
|
|
|
|
|
const remoteErr =
|
|
|
|
|
typeof errPayload?.error === "string"
|
|
|
|
|
? errPayload.error
|
|
|
|
|
: `Ingestion flush returned HTTP ${flushRes.status}`;
|
|
|
|
|
if (
|
|
|
|
|
flushRes.status === 503 &&
|
|
|
|
|
errPayload?.code === "FLUSH_LOCK_TIMEOUT"
|
|
|
|
|
) {
|
|
|
|
|
return NextResponse.json(
|
|
|
|
|
{
|
|
|
|
|
error: remoteErr,
|
|
|
|
|
code: "FLUSH_LOCK_TIMEOUT",
|
|
|
|
|
ingestionStatus: flushRes.status,
|
|
|
|
|
},
|
|
|
|
|
{ status: 503 },
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
return NextResponse.json(
|
|
|
|
|
{
|
|
|
|
|
error: remoteErr,
|
|
|
|
|
code: "INGESTION_FLUSH_FAILED",
|
|
|
|
|
ingestionStatus: flushRes.status,
|
|
|
|
|
},
|
|
|
|
|
{ status: 502 },
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
} catch (err: unknown) {
|
|
|
|
|
if (explicitIngestionUrl) {
|
|
|
|
|
const errMsg =
|
|
|
|
|
err instanceof Error ? err.message : "Ingestion server unreachable";
|
|
|
|
|
return NextResponse.json(
|
|
|
|
|
{
|
|
|
|
|
error: `Ingestion server flush failed before export: ${errMsg}`,
|
|
|
|
|
code: "INGESTION_UNREACHABLE",
|
|
|
|
|
},
|
|
|
|
|
{ status: 502 },
|
|
|
|
|
);
|
|
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
// Port 4000 may not be running in isolated single-process dev mode
|
|
|
|
|
}
|
2026-11-17 20:05:00 +06:00
|
|
|
}
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
const rawClientId = searchParams.get("clientId") || "all";
|
|
|
|
|
const clientId =
|
|
|
|
|
rawClientId === "all"
|
|
|
|
|
? "all"
|
|
|
|
|
: SDKManager.normalizeClientId(rawClientId) || "all";
|
|
|
|
|
const sanitizedClientId =
|
|
|
|
|
clientId === "all" ? "all" : CSVStorageManager.sanitizeClientId(clientId);
|
2026-11-19 23:02:27 +06:00
|
|
|
const filename = searchParams.get("filename") || searchParams.get("file");
|
|
|
|
|
const rawFormat = (searchParams.get("format") || "csv").toLowerCase();
|
|
|
|
|
const format: ExportFormat =
|
|
|
|
|
rawFormat === "json"
|
|
|
|
|
? "json"
|
|
|
|
|
: rawFormat === "ndjson"
|
|
|
|
|
? "ndjson"
|
|
|
|
|
: "csv";
|
2026-10-23 19:56:38 -07:00
|
|
|
const includeHeader = searchParams.get("includeHeader") !== "false";
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
let files = await CSVStorageManager.listAllFilesAsync();
|
2026-11-19 20:22:01 +06:00
|
|
|
if (clientId !== "all") {
|
|
|
|
|
files = files.filter(
|
|
|
|
|
(file) => file.clientId === clientId || file.clientId === sanitizedClientId,
|
|
|
|
|
);
|
|
|
|
|
}
|
2026-10-23 19:56:38 -07:00
|
|
|
|
|
|
|
|
if (filename) {
|
2026-11-18 05:18:55 +06:00
|
|
|
const safeFilename = path.basename(filename);
|
|
|
|
|
const matches = files.filter(
|
|
|
|
|
(candidate) => candidate.filename === safeFilename,
|
|
|
|
|
);
|
|
|
|
|
if (matches.length === 0) {
|
2026-10-23 19:56:38 -07:00
|
|
|
return NextResponse.json(
|
|
|
|
|
{ error: "CSV file not found" },
|
|
|
|
|
{ status: 404 },
|
|
|
|
|
);
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
|
|
|
|
if (matches.length > 1) {
|
|
|
|
|
return NextResponse.json(
|
|
|
|
|
{
|
|
|
|
|
error:
|
|
|
|
|
"Ambiguous filename matches multiple clients. Specify clientId.",
|
|
|
|
|
code: "AMBIGUOUS_FILENAME",
|
|
|
|
|
matchingClients: matches.map((m) => m.clientId),
|
|
|
|
|
},
|
|
|
|
|
{ status: 400 },
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
const file = matches[0];
|
2026-11-19 23:02:27 +06:00
|
|
|
const baseStem = file.filename.replace(/\.csv(?:\.gz)?$/i, "");
|
2026-11-19 20:22:01 +06:00
|
|
|
const downloadName =
|
|
|
|
|
format === "json"
|
2026-11-19 23:02:27 +06:00
|
|
|
? `${baseStem}.json`
|
|
|
|
|
: format === "ndjson"
|
|
|
|
|
? `${baseStem}.ndjson`
|
|
|
|
|
: !includeHeader && file.compressed
|
|
|
|
|
? `${baseStem}.csv`
|
|
|
|
|
: file.filename;
|
2026-11-19 20:22:01 +06:00
|
|
|
const bodyStream =
|
2026-11-19 23:02:27 +06:00
|
|
|
format === "json" || format === "ndjson" || !includeHeader
|
2026-11-19 20:22:01 +06:00
|
|
|
? makeExportStream([file], format, includeHeader)
|
|
|
|
|
: streamFile(file);
|
2026-11-19 23:02:27 +06:00
|
|
|
const contentType =
|
|
|
|
|
format === "json"
|
|
|
|
|
? "application/json; charset=utf-8"
|
|
|
|
|
: format === "ndjson"
|
|
|
|
|
? "application/x-ndjson; charset=utf-8"
|
|
|
|
|
: file.compressed && includeHeader
|
|
|
|
|
? "application/gzip"
|
|
|
|
|
: "text/csv; charset=utf-8";
|
2026-11-21 22:15:32 +06:00
|
|
|
|
|
|
|
|
await recordAdminAuditLogAsync({
|
|
|
|
|
actorEmail: auth.auth.session.email,
|
|
|
|
|
actorRole: auth.auth.session.role || "owner",
|
|
|
|
|
action: "DATA_EXPORT_DOWNLOADED",
|
|
|
|
|
target: file.clientId,
|
|
|
|
|
ipAddress: extractClientIp(request),
|
|
|
|
|
metadata: {
|
|
|
|
|
mode: "single_file",
|
|
|
|
|
clientId: file.clientId,
|
|
|
|
|
filename: file.filename,
|
|
|
|
|
downloadName,
|
|
|
|
|
format,
|
|
|
|
|
includeHeader,
|
|
|
|
|
fileCount: 1,
|
|
|
|
|
},
|
|
|
|
|
});
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
return new NextResponse(bodyStream, {
|
2026-10-23 19:56:38 -07:00
|
|
|
status: 200,
|
|
|
|
|
headers: {
|
2026-11-19 23:02:27 +06:00
|
|
|
"Content-Type": contentType,
|
2026-11-19 20:22:01 +06:00
|
|
|
"Content-Disposition": `attachment; filename="${downloadName.replace(/"/g, "")}"`,
|
|
|
|
|
"Cache-Control": "no-store",
|
2026-10-23 19:56:38 -07:00
|
|
|
},
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
files = files.filter(
|
|
|
|
|
(file) => (!from || file.date >= from) && (!to || file.date <= to),
|
|
|
|
|
);
|
|
|
|
|
const datePart = new Date().toISOString().slice(0, 10);
|
2026-11-19 20:22:01 +06:00
|
|
|
const safeClientId = sanitizedClientId === "all" ? "all" : sanitizedClientId;
|
2026-10-23 19:56:38 -07:00
|
|
|
const baseName =
|
|
|
|
|
clientId === "all" ? "datastack_export" : `${safeClientId}_export`;
|
2026-11-21 22:15:32 +06:00
|
|
|
const downloadName = `${baseName}_${datePart}.${format}`;
|
|
|
|
|
|
|
|
|
|
await recordAdminAuditLogAsync({
|
|
|
|
|
actorEmail: auth.auth.session.email,
|
|
|
|
|
actorRole: auth.auth.session.role || "owner",
|
|
|
|
|
action: "DATA_EXPORT_DOWNLOADED",
|
|
|
|
|
target: clientId,
|
|
|
|
|
ipAddress: extractClientIp(request),
|
|
|
|
|
metadata: {
|
|
|
|
|
mode: "filtered_range",
|
|
|
|
|
clientId,
|
|
|
|
|
from: from || null,
|
|
|
|
|
to: to || null,
|
|
|
|
|
format,
|
|
|
|
|
includeHeader,
|
|
|
|
|
fileCount: files.length,
|
|
|
|
|
filename: downloadName,
|
|
|
|
|
},
|
|
|
|
|
});
|
2026-10-23 19:56:38 -07:00
|
|
|
|
|
|
|
|
return new NextResponse(makeExportStream(files, format, includeHeader), {
|
2026-10-23 11:43:49 -05:00
|
|
|
status: 200,
|
|
|
|
|
headers: {
|
2026-10-23 19:56:38 -07:00
|
|
|
"Content-Type":
|
|
|
|
|
format === "json"
|
|
|
|
|
? "application/json; charset=utf-8"
|
2026-11-19 23:02:27 +06:00
|
|
|
: format === "ndjson"
|
|
|
|
|
? "application/x-ndjson; charset=utf-8"
|
|
|
|
|
: "text/csv; charset=utf-8",
|
2026-11-21 22:15:32 +06:00
|
|
|
"Content-Disposition": `attachment; filename="${downloadName}"`,
|
2026-10-23 19:56:38 -07:00
|
|
|
"Cache-Control": "no-store",
|
2026-10-23 11:43:49 -05:00
|
|
|
},
|
|
|
|
|
});
|
2026-10-23 19:56:38 -07:00
|
|
|
} catch (error) {
|
|
|
|
|
const message = error instanceof Error ? error.message : "Export failed";
|
2026-10-23 11:43:49 -05:00
|
|
|
return NextResponse.json({ error: message }, { status: 500 });
|
|
|
|
|
}
|
|
|
|
|
}
|