2026-10-23 19:56:38 -07:00
|
|
|
import {
|
|
|
|
|
CSVStorageManager,
|
|
|
|
|
type StorageFileMeta,
|
|
|
|
|
} from "@/core/Storage/csvStorage";
|
|
|
|
|
import { parse } from "csv-parse";
|
|
|
|
|
import { NextRequest, NextResponse } from "next/server";
|
|
|
|
|
import { createReadStream } from "node:fs";
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-10-23 19:56:38 -07:00
|
|
|
type ExportFormat = "csv" | "json";
|
|
|
|
|
type CsvRecord = Record<string, string>;
|
|
|
|
|
type StreamChunk = string | Uint8Array;
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-10-23 19:56:38 -07:00
|
|
|
function toByteStream(chunks: AsyncIterable<StreamChunk>) {
|
|
|
|
|
const iterator = chunks[Symbol.asyncIterator]();
|
|
|
|
|
const encoder = new TextEncoder();
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-10-23 19:56:38 -07:00
|
|
|
return new ReadableStream<Uint8Array>({
|
|
|
|
|
async pull(controller) {
|
2026-10-23 11:43:49 -05:00
|
|
|
try {
|
2026-10-23 19:56:38 -07:00
|
|
|
const result = await iterator.next();
|
|
|
|
|
if (result.done) {
|
|
|
|
|
controller.close();
|
|
|
|
|
} else {
|
|
|
|
|
controller.enqueue(
|
|
|
|
|
typeof result.value === "string"
|
|
|
|
|
? encoder.encode(result.value)
|
|
|
|
|
: result.value,
|
|
|
|
|
);
|
2026-10-23 11:43:49 -05:00
|
|
|
}
|
2026-10-23 19:56:38 -07:00
|
|
|
} catch (error) {
|
|
|
|
|
controller.error(error);
|
|
|
|
|
}
|
|
|
|
|
},
|
|
|
|
|
async cancel() {
|
|
|
|
|
await iterator.return?.();
|
|
|
|
|
},
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function* readFileRecords(
|
|
|
|
|
file: StorageFileMeta,
|
|
|
|
|
): AsyncGenerator<CsvRecord> {
|
|
|
|
|
const parser = createReadStream(file.filePath).pipe(
|
|
|
|
|
parse({ columns: true, skip_empty_lines: true, bom: true }),
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
for await (const record of parser as AsyncIterable<CsvRecord>) {
|
|
|
|
|
yield record;
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
parser.destroy();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function makeExportStream(
|
|
|
|
|
files: StorageFileMeta[],
|
|
|
|
|
format: ExportFormat,
|
|
|
|
|
includeHeader: boolean,
|
|
|
|
|
) {
|
|
|
|
|
const fields = ["clientId", "eventId", "timestamp", "payload"];
|
|
|
|
|
const chunks = (async function* () {
|
|
|
|
|
let isFirst = true;
|
|
|
|
|
|
|
|
|
|
if (format === "json") {
|
|
|
|
|
yield "[";
|
|
|
|
|
} else if (includeHeader) {
|
|
|
|
|
yield `${fields.join(",")}\n`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (const file of files) {
|
|
|
|
|
for await (const record of readFileRecords(file)) {
|
|
|
|
|
if (format === "json") {
|
|
|
|
|
const payload = record.payload ?? "";
|
|
|
|
|
let parsedPayload: unknown = payload;
|
|
|
|
|
try {
|
|
|
|
|
parsedPayload = JSON.parse(payload);
|
|
|
|
|
} catch {
|
|
|
|
|
// Keep non-JSON payloads as strings.
|
|
|
|
|
}
|
|
|
|
|
yield `${isFirst ? "" : ","}${JSON.stringify({ ...record, payload: parsedPayload })}`;
|
|
|
|
|
} else {
|
|
|
|
|
const line = fields
|
|
|
|
|
.map((field) => `"${(record[field] ?? "").replaceAll('"', '""')}"`)
|
|
|
|
|
.join(",");
|
|
|
|
|
yield `${line}\n`;
|
|
|
|
|
}
|
|
|
|
|
isFirst = false;
|
2026-10-23 11:43:49 -05:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-10-23 19:56:38 -07:00
|
|
|
if (format === "json") yield "]";
|
|
|
|
|
})();
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-10-23 19:56:38 -07:00
|
|
|
return toByteStream(chunks);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function streamFile(file: StorageFileMeta) {
|
|
|
|
|
async function* chunks() {
|
|
|
|
|
for await (const chunk of createReadStream(file.filePath)) {
|
|
|
|
|
yield new Uint8Array(chunk);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return toByteStream(chunks());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export async function GET(request: NextRequest) {
|
|
|
|
|
try {
|
|
|
|
|
const { searchParams } = new URL(request.url);
|
|
|
|
|
const clientId = searchParams.get("clientId") || "all";
|
|
|
|
|
const filename = searchParams.get("filename");
|
|
|
|
|
const from = searchParams.get("from") || "";
|
|
|
|
|
const to = searchParams.get("to") || "";
|
|
|
|
|
const format: ExportFormat =
|
|
|
|
|
searchParams.get("format") === "json" ? "json" : "csv";
|
|
|
|
|
const includeHeader = searchParams.get("includeHeader") !== "false";
|
|
|
|
|
|
|
|
|
|
let files = CSVStorageManager.listAllFiles();
|
|
|
|
|
if (clientId !== "all")
|
|
|
|
|
files = files.filter((file) => file.clientId === clientId);
|
|
|
|
|
|
|
|
|
|
if (filename) {
|
|
|
|
|
const file = files.find((candidate) => candidate.filename === filename);
|
|
|
|
|
if (!file)
|
|
|
|
|
return NextResponse.json(
|
|
|
|
|
{ error: "CSV file not found" },
|
|
|
|
|
{ status: 404 },
|
|
|
|
|
);
|
|
|
|
|
return new NextResponse(streamFile(file), {
|
|
|
|
|
status: 200,
|
|
|
|
|
headers: {
|
|
|
|
|
"Content-Type": "text/csv; charset=utf-8",
|
|
|
|
|
"Content-Disposition": `attachment; filename="${file.filename}"`,
|
|
|
|
|
},
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
files = files.filter(
|
|
|
|
|
(file) => (!from || file.date >= from) && (!to || file.date <= to),
|
|
|
|
|
);
|
|
|
|
|
const datePart = new Date().toISOString().slice(0, 10);
|
|
|
|
|
const safeClientId = clientId.replace(/[^a-zA-Z0-9_-]/g, "_");
|
|
|
|
|
const baseName =
|
|
|
|
|
clientId === "all" ? "datastack_export" : `${safeClientId}_export`;
|
|
|
|
|
|
|
|
|
|
return new NextResponse(makeExportStream(files, format, includeHeader), {
|
2026-10-23 11:43:49 -05:00
|
|
|
status: 200,
|
|
|
|
|
headers: {
|
2026-10-23 19:56:38 -07:00
|
|
|
"Content-Type":
|
|
|
|
|
format === "json"
|
|
|
|
|
? "application/json; charset=utf-8"
|
|
|
|
|
: "text/csv; charset=utf-8",
|
|
|
|
|
"Content-Disposition": `attachment; filename="${baseName}_${datePart}.${format}"`,
|
|
|
|
|
"Cache-Control": "no-store",
|
2026-10-23 11:43:49 -05:00
|
|
|
},
|
|
|
|
|
});
|
2026-10-23 19:56:38 -07:00
|
|
|
} catch (error) {
|
|
|
|
|
const message = error instanceof Error ? error.message : "Export failed";
|
2026-10-23 11:43:49 -05:00
|
|
|
return NextResponse.json({ error: message }, { status: 500 });
|
|
|
|
|
}
|
|
|
|
|
}
|