chore: publish current DataStack updates
Publish the current application, design, and Phase 1 deployment updates. Excludes ignored personal notes, local MCP config, and generated TypeScript build info.
This commit is contained in:
1 parent
d39d9f92f9
commit
fcc4f4db2d
88 files changed
+21248
-2213
No files matched your search
@@ -0,0 +1,76 @@
|
||||
import { createAdminSessionToken } from "@/core/auth/adminSession";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { createHash, timingSafeEqual } from "node:crypto";
|
||||
|
||||
const SESSION_COOKIE = "datastack_admin_session";
|
||||
const SESSION_SECONDS = 8 * 60 * 60;
|
||||
const REMEMBERED_SESSION_SECONDS = 30 * 24 * 60 * 60;
|
||||
|
||||
function constantTimeEqual(left: string, right: string) {
|
||||
const leftHash = createHash("sha256").update(left).digest();
|
||||
const rightHash = createHash("sha256").update(right).digest();
|
||||
return timingSafeEqual(leftHash, rightHash);
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
const adminEmail = process.env.DATASTACK_ADMIN_EMAIL;
|
||||
const adminPassword = process.env.DATASTACK_ADMIN_PASSWORD;
|
||||
const sessionSecret = process.env.DATASTACK_ADMIN_SESSION_SECRET;
|
||||
|
||||
if (
|
||||
!adminEmail ||
|
||||
!adminPassword ||
|
||||
!sessionSecret ||
|
||||
sessionSecret.length < 32
|
||||
) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error:
|
||||
"Admin login is not configured. Set the required server environment variables.",
|
||||
},
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const body = await request.json();
|
||||
const email =
|
||||
typeof body.email === "string" ? body.email.trim().toLowerCase() : "";
|
||||
const password = typeof body.password === "string" ? body.password : "";
|
||||
|
||||
if (
|
||||
!constantTimeEqual(email, adminEmail.trim().toLowerCase()) ||
|
||||
!constantTimeEqual(password, adminPassword)
|
||||
) {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid email or password." },
|
||||
{ status: 401 },
|
||||
);
|
||||
}
|
||||
|
||||
const remember = body.remember === true;
|
||||
const maxAge = remember ? REMEMBERED_SESSION_SECONDS : SESSION_SECONDS;
|
||||
const expiresAt = Date.now() + maxAge * 1000;
|
||||
const token = await createAdminSessionToken(
|
||||
email,
|
||||
sessionSecret,
|
||||
expiresAt,
|
||||
);
|
||||
const response = NextResponse.json({ success: true });
|
||||
|
||||
response.cookies.set(SESSION_COOKIE, token, {
|
||||
httpOnly: true,
|
||||
secure: process.env.NODE_ENV === "production",
|
||||
sameSite: "strict",
|
||||
path: "/",
|
||||
maxAge,
|
||||
});
|
||||
|
||||
return response;
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid login request." },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,54 +1,58 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { SDKManager } from '@/core/sdk/sdkManager';
|
||||
import { SDKManager } from "@/core/sdk/sdkManager";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
export async function GET(
|
||||
_req: NextRequest,
|
||||
{ params }: { params: Promise<{ id: string }> }
|
||||
{ params }: { params: Promise<{ id: string }> },
|
||||
) {
|
||||
const { id } = await params;
|
||||
const client = SDKManager.getClient(id);
|
||||
if (!client) {
|
||||
return NextResponse.json({ error: 'Client not found' }, { status: 404 });
|
||||
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
||||
}
|
||||
return NextResponse.json({ client });
|
||||
const { secret, token, ...clientSummary } = client;
|
||||
return NextResponse.json({ client: clientSummary });
|
||||
}
|
||||
|
||||
export async function PATCH(
|
||||
req: NextRequest,
|
||||
{ params }: { params: Promise<{ id: string }> }
|
||||
{ params }: { params: Promise<{ id: string }> },
|
||||
) {
|
||||
const { id } = await params;
|
||||
try {
|
||||
const body = await req.json();
|
||||
|
||||
// Special action: regenerate token
|
||||
if (body.action === 'regenerate_token') {
|
||||
if (body.action === "regenerate_token") {
|
||||
const refreshed = SDKManager.regenerateToken(id);
|
||||
if (!refreshed) {
|
||||
return NextResponse.json({ error: 'Client not found' }, { status: 404 });
|
||||
return NextResponse.json(
|
||||
{ error: "Client not found" },
|
||||
{ status: 404 },
|
||||
);
|
||||
}
|
||||
return NextResponse.json({ client: refreshed });
|
||||
}
|
||||
|
||||
const updated = SDKManager.updateClient(id, body);
|
||||
if (!updated) {
|
||||
return NextResponse.json({ error: 'Client not found' }, { status: 404 });
|
||||
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
||||
}
|
||||
return NextResponse.json({ client: updated });
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : 'Update failed';
|
||||
const message = err instanceof Error ? err.message : "Update failed";
|
||||
return NextResponse.json({ error: message }, { status: 500 });
|
||||
}
|
||||
}
|
||||
|
||||
export async function DELETE(
|
||||
_req: NextRequest,
|
||||
{ params }: { params: Promise<{ id: string }> }
|
||||
{ params }: { params: Promise<{ id: string }> },
|
||||
) {
|
||||
const { id } = await params;
|
||||
const deleted = SDKManager.deleteClient(id);
|
||||
if (!deleted) {
|
||||
return NextResponse.json({ error: 'Client not found' }, { status: 404 });
|
||||
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
||||
}
|
||||
return NextResponse.json({ success: true, message: `Client ${id} deleted` });
|
||||
}
|
||||
@@ -1,12 +1,15 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { SDKManager } from '@/core/sdk/sdkManager';
|
||||
import { SDKManager } from "@/core/sdk/sdkManager";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
export async function GET() {
|
||||
try {
|
||||
const clients = SDKManager.listClients();
|
||||
const clients = SDKManager.listClients().map(
|
||||
({ secret, token, ...client }) => client,
|
||||
);
|
||||
return NextResponse.json({ clients, count: clients.length });
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : 'Failed to list clients';
|
||||
const message =
|
||||
err instanceof Error ? err.message : "Failed to list clients";
|
||||
return NextResponse.json({ error: message }, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -18,8 +21,8 @@ export async function POST(req: NextRequest) {
|
||||
|
||||
if (!clientId || !name) {
|
||||
return NextResponse.json(
|
||||
{ error: 'clientId and name are required' },
|
||||
{ status: 400 }
|
||||
{ error: "clientId and name are required" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -27,7 +30,7 @@ export async function POST(req: NextRequest) {
|
||||
if (existing) {
|
||||
return NextResponse.json(
|
||||
{ error: `Client '${clientId}' already exists` },
|
||||
{ status: 409 }
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -39,7 +42,8 @@ export async function POST(req: NextRequest) {
|
||||
|
||||
return NextResponse.json({ client: newClient }, { status: 201 });
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : 'Client creation failed';
|
||||
const message =
|
||||
err instanceof Error ? err.message : "Client creation failed";
|
||||
return NextResponse.json({ error: message }, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
import { DataBufferEngine } from "@/core/data/buffer";
|
||||
import { CSVStorageManager } from "@/core/Storage/csvStorage";
|
||||
import { parse } from "csv-parse";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { createReadStream, readdirSync, statSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
interface DataEvent {
|
||||
clientId: string;
|
||||
eventId: string;
|
||||
timestamp: string;
|
||||
payload: string;
|
||||
status: "stored" | "buffered";
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const clientId = searchParams.get("clientId") || "all";
|
||||
const from = searchParams.get("from") || "";
|
||||
const to = searchParams.get("to") || "";
|
||||
const search = (searchParams.get("search") || "").toLowerCase();
|
||||
const page = Math.max(1, Number(searchParams.get("page")) || 1);
|
||||
const pageSize = Math.min(
|
||||
100,
|
||||
Math.max(1, Number(searchParams.get("pageSize")) || 12),
|
||||
);
|
||||
const offset = (page - 1) * pageSize;
|
||||
const selectedRows: DataEvent[] = [];
|
||||
const eventIds = new Set<string>();
|
||||
let total = 0;
|
||||
|
||||
const addIfMatching = (event: DataEvent) => {
|
||||
const eventDate = event.timestamp.slice(0, 10);
|
||||
if (
|
||||
(clientId !== "all" && event.clientId !== clientId) ||
|
||||
(from && eventDate < from) ||
|
||||
(to && eventDate > to) ||
|
||||
(search && !event.payload.toLowerCase().includes(search)) ||
|
||||
eventIds.has(event.eventId)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
eventIds.add(event.eventId);
|
||||
total += 1;
|
||||
|
||||
let low = 0;
|
||||
let high = selectedRows.length;
|
||||
while (low < high) {
|
||||
const middle = (low + high) >>> 1;
|
||||
if (selectedRows[middle].timestamp >= event.timestamp) low = middle + 1;
|
||||
else high = middle;
|
||||
}
|
||||
|
||||
if (low < offset + pageSize) {
|
||||
selectedRows.splice(low, 0, event);
|
||||
if (selectedRows.length > offset + pageSize) selectedRows.pop();
|
||||
}
|
||||
};
|
||||
|
||||
const storageDir = CSVStorageManager.baseStorageDir;
|
||||
let clientDirs: string[] = [];
|
||||
try {
|
||||
clientDirs = readdirSync(storageDir);
|
||||
} catch {
|
||||
clientDirs = [];
|
||||
}
|
||||
|
||||
for (const storedClientId of clientDirs) {
|
||||
if (clientId !== "all" && storedClientId !== clientId) continue;
|
||||
|
||||
const clientDir = path.join(storageDir, storedClientId);
|
||||
if (!statSync(clientDir).isDirectory()) continue;
|
||||
|
||||
let filenames: string[];
|
||||
try {
|
||||
filenames = readdirSync(clientDir).filter((filename) =>
|
||||
/^\d{4}-\d{2}-\d{2}(?:-part\d+)?\.csv$/.test(filename),
|
||||
);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const filename of filenames) {
|
||||
const fileDate = filename.slice(0, 10);
|
||||
if ((from && fileDate < from) || (to && fileDate > to)) continue;
|
||||
|
||||
const parser = createReadStream(path.join(clientDir, filename)).pipe(
|
||||
parse({ columns: true, skip_empty_lines: true, bom: true }),
|
||||
);
|
||||
|
||||
try {
|
||||
for await (const record of parser as AsyncIterable<
|
||||
Record<string, string>
|
||||
>) {
|
||||
const event = {
|
||||
clientId: record.clientId || storedClientId,
|
||||
eventId: record.eventId || "",
|
||||
timestamp: record.timestamp || `${fileDate}T00:00:00.000Z`,
|
||||
payload: record.payload || "",
|
||||
status: "stored" as const,
|
||||
};
|
||||
if (event.eventId) addIfMatching(event);
|
||||
}
|
||||
} catch {
|
||||
parser.destroy();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const event of DataBufferEngine.getRecentEvents(200)) {
|
||||
addIfMatching({ ...event, status: "buffered" });
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
events: selectedRows.slice(offset),
|
||||
total,
|
||||
page,
|
||||
pageSize,
|
||||
pageCount: Math.max(1, Math.ceil(total / pageSize)),
|
||||
});
|
||||
} catch (error) {
|
||||
const message =
|
||||
error instanceof Error ? error.message : "Data retrieval failed";
|
||||
return NextResponse.json({ error: message }, { status: 500 });
|
||||
}
|
||||
}
|
||||
+146
-60
@@ -1,76 +1,162 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { CSVStorageManager } from '@/core/Storage/csvStorage';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import {
|
||||
CSVStorageManager,
|
||||
type StorageFileMeta,
|
||||
} from "@/core/Storage/csvStorage";
|
||||
import { parse } from "csv-parse";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { createReadStream } from "node:fs";
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
try {
|
||||
const { searchParams } = new URL(req.url);
|
||||
const clientId = searchParams.get('clientId');
|
||||
const filename = searchParams.get('filename');
|
||||
type ExportFormat = "csv" | "json";
|
||||
type CsvRecord = Record<string, string>;
|
||||
type StreamChunk = string | Uint8Array;
|
||||
|
||||
// Single specific file export
|
||||
if (clientId && filename) {
|
||||
const content = CSVStorageManager.getFileContent(clientId, filename);
|
||||
if (!content) {
|
||||
return NextResponse.json({ error: 'CSV file not found' }, { status: 404 });
|
||||
}
|
||||
function toByteStream(chunks: AsyncIterable<StreamChunk>) {
|
||||
const iterator = chunks[Symbol.asyncIterator]();
|
||||
const encoder = new TextEncoder();
|
||||
|
||||
return new NextResponse(content, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/csv; charset=utf-8',
|
||||
'Content-Disposition': `attachment; filename="${filename}"`,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Stream aggregate or list files
|
||||
const allFiles = CSVStorageManager.listAllFiles();
|
||||
const filtered = clientId && clientId !== 'all'
|
||||
? allFiles.filter((f) => f.clientId === clientId)
|
||||
: allFiles;
|
||||
|
||||
if (filtered.length === 0) {
|
||||
const emptyCsv = 'clientId,eventId,timestamp,payload\n';
|
||||
return new NextResponse(emptyCsv, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/csv; charset=utf-8',
|
||||
'Content-Disposition': `attachment; filename="datastack_export_${Date.now()}.csv"`,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Combine matching files with a single header
|
||||
let combined = 'clientId,eventId,timestamp,payload\n';
|
||||
for (const f of filtered) {
|
||||
return new ReadableStream<Uint8Array>({
|
||||
async pull(controller) {
|
||||
try {
|
||||
const raw = fs.readFileSync(f.filePath, 'utf8');
|
||||
const lines = raw.split('\n');
|
||||
// skip header on subsequent lines
|
||||
const dataLines = lines.slice(1).filter((l) => l.trim().length > 0);
|
||||
if (dataLines.length > 0) {
|
||||
combined += dataLines.join('\n') + '\n';
|
||||
const result = await iterator.next();
|
||||
if (result.done) {
|
||||
controller.close();
|
||||
} else {
|
||||
controller.enqueue(
|
||||
typeof result.value === "string"
|
||||
? encoder.encode(result.value)
|
||||
: result.value,
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
// continue
|
||||
} catch (error) {
|
||||
controller.error(error);
|
||||
}
|
||||
},
|
||||
async cancel() {
|
||||
await iterator.return?.();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function* readFileRecords(
|
||||
file: StorageFileMeta,
|
||||
): AsyncGenerator<CsvRecord> {
|
||||
const parser = createReadStream(file.filePath).pipe(
|
||||
parse({ columns: true, skip_empty_lines: true, bom: true }),
|
||||
);
|
||||
|
||||
try {
|
||||
for await (const record of parser as AsyncIterable<CsvRecord>) {
|
||||
yield record;
|
||||
}
|
||||
} catch {
|
||||
parser.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
function makeExportStream(
|
||||
files: StorageFileMeta[],
|
||||
format: ExportFormat,
|
||||
includeHeader: boolean,
|
||||
) {
|
||||
const fields = ["clientId", "eventId", "timestamp", "payload"];
|
||||
const chunks = (async function* () {
|
||||
let isFirst = true;
|
||||
|
||||
if (format === "json") {
|
||||
yield "[";
|
||||
} else if (includeHeader) {
|
||||
yield `${fields.join(",")}\n`;
|
||||
}
|
||||
|
||||
for (const file of files) {
|
||||
for await (const record of readFileRecords(file)) {
|
||||
if (format === "json") {
|
||||
const payload = record.payload ?? "";
|
||||
let parsedPayload: unknown = payload;
|
||||
try {
|
||||
parsedPayload = JSON.parse(payload);
|
||||
} catch {
|
||||
// Keep non-JSON payloads as strings.
|
||||
}
|
||||
yield `${isFirst ? "" : ","}${JSON.stringify({ ...record, payload: parsedPayload })}`;
|
||||
} else {
|
||||
const line = fields
|
||||
.map((field) => `"${(record[field] ?? "").replaceAll('"', '""')}"`)
|
||||
.join(",");
|
||||
yield `${line}\n`;
|
||||
}
|
||||
isFirst = false;
|
||||
}
|
||||
}
|
||||
|
||||
const exportFileName = clientId && clientId !== 'all'
|
||||
? `${clientId}_export_${new Date().toISOString().split('T')[0]}.csv`
|
||||
: `datastack_universal_export_${new Date().toISOString().split('T')[0]}.csv`;
|
||||
if (format === "json") yield "]";
|
||||
})();
|
||||
|
||||
return new NextResponse(combined, {
|
||||
return toByteStream(chunks);
|
||||
}
|
||||
|
||||
function streamFile(file: StorageFileMeta) {
|
||||
async function* chunks() {
|
||||
for await (const chunk of createReadStream(file.filePath)) {
|
||||
yield new Uint8Array(chunk);
|
||||
}
|
||||
}
|
||||
|
||||
return toByteStream(chunks());
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const clientId = searchParams.get("clientId") || "all";
|
||||
const filename = searchParams.get("filename");
|
||||
const from = searchParams.get("from") || "";
|
||||
const to = searchParams.get("to") || "";
|
||||
const format: ExportFormat =
|
||||
searchParams.get("format") === "json" ? "json" : "csv";
|
||||
const includeHeader = searchParams.get("includeHeader") !== "false";
|
||||
|
||||
let files = CSVStorageManager.listAllFiles();
|
||||
if (clientId !== "all")
|
||||
files = files.filter((file) => file.clientId === clientId);
|
||||
|
||||
if (filename) {
|
||||
const file = files.find((candidate) => candidate.filename === filename);
|
||||
if (!file)
|
||||
return NextResponse.json(
|
||||
{ error: "CSV file not found" },
|
||||
{ status: 404 },
|
||||
);
|
||||
return new NextResponse(streamFile(file), {
|
||||
status: 200,
|
||||
headers: {
|
||||
"Content-Type": "text/csv; charset=utf-8",
|
||||
"Content-Disposition": `attachment; filename="${file.filename}"`,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
files = files.filter(
|
||||
(file) => (!from || file.date >= from) && (!to || file.date <= to),
|
||||
);
|
||||
const datePart = new Date().toISOString().slice(0, 10);
|
||||
const safeClientId = clientId.replace(/[^a-zA-Z0-9_-]/g, "_");
|
||||
const baseName =
|
||||
clientId === "all" ? "datastack_export" : `${safeClientId}_export`;
|
||||
|
||||
return new NextResponse(makeExportStream(files, format, includeHeader), {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'text/csv; charset=utf-8',
|
||||
'Content-Disposition': `attachment; filename="${exportFileName}"`,
|
||||
"Content-Type":
|
||||
format === "json"
|
||||
? "application/json; charset=utf-8"
|
||||
: "text/csv; charset=utf-8",
|
||||
"Content-Disposition": `attachment; filename="${baseName}_${datePart}.${format}"`,
|
||||
"Cache-Control": "no-store",
|
||||
},
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : 'Export failed';
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : "Export failed";
|
||||
return NextResponse.json({ error: message }, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,12 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { SettingsManager } from '@/core/adminSettings/settingsManager';
|
||||
import { SettingsManager } from "@/core/adminSettings/settingsManager";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
export async function GET() {
|
||||
try {
|
||||
const settings = SettingsManager.getSettings();
|
||||
const settings = SettingsManager.getPublicSettings();
|
||||
return NextResponse.json({ settings });
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : 'Settings read failed';
|
||||
const message = err instanceof Error ? err.message : "Settings read failed";
|
||||
return NextResponse.json({ error: message }, { status: 500 });
|
||||
}
|
||||
}
|
||||
@@ -14,10 +14,14 @@ export async function GET() {
|
||||
export async function POST(req: NextRequest) {
|
||||
try {
|
||||
const body = await req.json();
|
||||
const updated = SettingsManager.updateSettings(body);
|
||||
return NextResponse.json({ settings: updated, success: true });
|
||||
SettingsManager.updateSettings(body);
|
||||
return NextResponse.json({
|
||||
settings: SettingsManager.getPublicSettings(),
|
||||
success: true,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : 'Settings update failed';
|
||||
const message =
|
||||
err instanceof Error ? err.message : "Settings update failed";
|
||||
return NextResponse.json({ error: message }, { status: 500 });
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user