chore: publish current DataStack updates

Publish the current application, design, and Phase 1 deployment updates. Excludes ignored personal notes, local MCP config, and generated TypeScript build info.
This commit is contained in:
i vib committed 2026-10-23 19:56:38 -07:00
1 parent d39d9f92f9
commit fcc4f4db2d
88 files changed
+21248 -2213

No files matched your search

+76
View File
@@ -0,0 +1,76 @@
import { createAdminSessionToken } from "@/core/auth/adminSession";
import { NextRequest, NextResponse } from "next/server";
import { createHash, timingSafeEqual } from "node:crypto";
const SESSION_COOKIE = "datastack_admin_session";
const SESSION_SECONDS = 8 * 60 * 60;
const REMEMBERED_SESSION_SECONDS = 30 * 24 * 60 * 60;
function constantTimeEqual(left: string, right: string) {
const leftHash = createHash("sha256").update(left).digest();
const rightHash = createHash("sha256").update(right).digest();
return timingSafeEqual(leftHash, rightHash);
}
export async function POST(request: NextRequest) {
const adminEmail = process.env.DATASTACK_ADMIN_EMAIL;
const adminPassword = process.env.DATASTACK_ADMIN_PASSWORD;
const sessionSecret = process.env.DATASTACK_ADMIN_SESSION_SECRET;
if (
!adminEmail ||
!adminPassword ||
!sessionSecret ||
sessionSecret.length < 32
) {
return NextResponse.json(
{
error:
"Admin login is not configured. Set the required server environment variables.",
},
{ status: 503 },
);
}
try {
const body = await request.json();
const email =
typeof body.email === "string" ? body.email.trim().toLowerCase() : "";
const password = typeof body.password === "string" ? body.password : "";
if (
!constantTimeEqual(email, adminEmail.trim().toLowerCase()) ||
!constantTimeEqual(password, adminPassword)
) {
return NextResponse.json(
{ error: "Invalid email or password." },
{ status: 401 },
);
}
const remember = body.remember === true;
const maxAge = remember ? REMEMBERED_SESSION_SECONDS : SESSION_SECONDS;
const expiresAt = Date.now() + maxAge * 1000;
const token = await createAdminSessionToken(
email,
sessionSecret,
expiresAt,
);
const response = NextResponse.json({ success: true });
response.cookies.set(SESSION_COOKIE, token, {
httpOnly: true,
secure: process.env.NODE_ENV === "production",
sameSite: "strict",
path: "/",
maxAge,
});
return response;
} catch {
return NextResponse.json(
{ error: "Invalid login request." },
{ status: 400 },
);
}
}
+16 -12
View File
@@ -1,54 +1,58 @@
import { NextRequest, NextResponse } from 'next/server';
import { SDKManager } from '@/core/sdk/sdkManager';
import { SDKManager } from "@/core/sdk/sdkManager";
import { NextRequest, NextResponse } from "next/server";
export async function GET(
_req: NextRequest,
{ params }: { params: Promise<{ id: string }> }
{ params }: { params: Promise<{ id: string }> },
) {
const { id } = await params;
const client = SDKManager.getClient(id);
if (!client) {
return NextResponse.json({ error: 'Client not found' }, { status: 404 });
return NextResponse.json({ error: "Client not found" }, { status: 404 });
}
return NextResponse.json({ client });
const { secret, token, ...clientSummary } = client;
return NextResponse.json({ client: clientSummary });
}
export async function PATCH(
req: NextRequest,
{ params }: { params: Promise<{ id: string }> }
{ params }: { params: Promise<{ id: string }> },
) {
const { id } = await params;
try {
const body = await req.json();
// Special action: regenerate token
if (body.action === 'regenerate_token') {
if (body.action === "regenerate_token") {
const refreshed = SDKManager.regenerateToken(id);
if (!refreshed) {
return NextResponse.json({ error: 'Client not found' }, { status: 404 });
return NextResponse.json(
{ error: "Client not found" },
{ status: 404 },
);
}
return NextResponse.json({ client: refreshed });
}
const updated = SDKManager.updateClient(id, body);
if (!updated) {
return NextResponse.json({ error: 'Client not found' }, { status: 404 });
return NextResponse.json({ error: "Client not found" }, { status: 404 });
}
return NextResponse.json({ client: updated });
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Update failed';
const message = err instanceof Error ? err.message : "Update failed";
return NextResponse.json({ error: message }, { status: 500 });
}
}
export async function DELETE(
_req: NextRequest,
{ params }: { params: Promise<{ id: string }> }
{ params }: { params: Promise<{ id: string }> },
) {
const { id } = await params;
const deleted = SDKManager.deleteClient(id);
if (!deleted) {
return NextResponse.json({ error: 'Client not found' }, { status: 404 });
return NextResponse.json({ error: "Client not found" }, { status: 404 });
}
return NextResponse.json({ success: true, message: `Client ${id} deleted` });
}
+12 -8
View File
@@ -1,12 +1,15 @@
import { NextRequest, NextResponse } from 'next/server';
import { SDKManager } from '@/core/sdk/sdkManager';
import { SDKManager } from "@/core/sdk/sdkManager";
import { NextRequest, NextResponse } from "next/server";
export async function GET() {
try {
const clients = SDKManager.listClients();
const clients = SDKManager.listClients().map(
({ secret, token, ...client }) => client,
);
return NextResponse.json({ clients, count: clients.length });
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Failed to list clients';
const message =
err instanceof Error ? err.message : "Failed to list clients";
return NextResponse.json({ error: message }, { status: 500 });
}
}
@@ -18,8 +21,8 @@ export async function POST(req: NextRequest) {
if (!clientId || !name) {
return NextResponse.json(
{ error: 'clientId and name are required' },
{ status: 400 }
{ error: "clientId and name are required" },
{ status: 400 },
);
}
@@ -27,7 +30,7 @@ export async function POST(req: NextRequest) {
if (existing) {
return NextResponse.json(
{ error: `Client '${clientId}' already exists` },
{ status: 409 }
{ status: 409 },
);
}
@@ -39,7 +42,8 @@ export async function POST(req: NextRequest) {
return NextResponse.json({ client: newClient }, { status: 201 });
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Client creation failed';
const message =
err instanceof Error ? err.message : "Client creation failed";
return NextResponse.json({ error: message }, { status: 500 });
}
}
+128
View File
@@ -0,0 +1,128 @@
import { DataBufferEngine } from "@/core/data/buffer";
import { CSVStorageManager } from "@/core/Storage/csvStorage";
import { parse } from "csv-parse";
import { NextRequest, NextResponse } from "next/server";
import { createReadStream, readdirSync, statSync } from "node:fs";
import path from "node:path";
interface DataEvent {
clientId: string;
eventId: string;
timestamp: string;
payload: string;
status: "stored" | "buffered";
}
export async function GET(request: NextRequest) {
try {
const { searchParams } = new URL(request.url);
const clientId = searchParams.get("clientId") || "all";
const from = searchParams.get("from") || "";
const to = searchParams.get("to") || "";
const search = (searchParams.get("search") || "").toLowerCase();
const page = Math.max(1, Number(searchParams.get("page")) || 1);
const pageSize = Math.min(
100,
Math.max(1, Number(searchParams.get("pageSize")) || 12),
);
const offset = (page - 1) * pageSize;
const selectedRows: DataEvent[] = [];
const eventIds = new Set<string>();
let total = 0;
const addIfMatching = (event: DataEvent) => {
const eventDate = event.timestamp.slice(0, 10);
if (
(clientId !== "all" && event.clientId !== clientId) ||
(from && eventDate < from) ||
(to && eventDate > to) ||
(search && !event.payload.toLowerCase().includes(search)) ||
eventIds.has(event.eventId)
) {
return;
}
eventIds.add(event.eventId);
total += 1;
let low = 0;
let high = selectedRows.length;
while (low < high) {
const middle = (low + high) >>> 1;
if (selectedRows[middle].timestamp >= event.timestamp) low = middle + 1;
else high = middle;
}
if (low < offset + pageSize) {
selectedRows.splice(low, 0, event);
if (selectedRows.length > offset + pageSize) selectedRows.pop();
}
};
const storageDir = CSVStorageManager.baseStorageDir;
let clientDirs: string[] = [];
try {
clientDirs = readdirSync(storageDir);
} catch {
clientDirs = [];
}
for (const storedClientId of clientDirs) {
if (clientId !== "all" && storedClientId !== clientId) continue;
const clientDir = path.join(storageDir, storedClientId);
if (!statSync(clientDir).isDirectory()) continue;
let filenames: string[];
try {
filenames = readdirSync(clientDir).filter((filename) =>
/^\d{4}-\d{2}-\d{2}(?:-part\d+)?\.csv$/.test(filename),
);
} catch {
continue;
}
for (const filename of filenames) {
const fileDate = filename.slice(0, 10);
if ((from && fileDate < from) || (to && fileDate > to)) continue;
const parser = createReadStream(path.join(clientDir, filename)).pipe(
parse({ columns: true, skip_empty_lines: true, bom: true }),
);
try {
for await (const record of parser as AsyncIterable<
Record<string, string>
>) {
const event = {
clientId: record.clientId || storedClientId,
eventId: record.eventId || "",
timestamp: record.timestamp || `${fileDate}T00:00:00.000Z`,
payload: record.payload || "",
status: "stored" as const,
};
if (event.eventId) addIfMatching(event);
}
} catch {
parser.destroy();
}
}
}
for (const event of DataBufferEngine.getRecentEvents(200)) {
addIfMatching({ ...event, status: "buffered" });
}
return NextResponse.json({
events: selectedRows.slice(offset),
total,
page,
pageSize,
pageCount: Math.max(1, Math.ceil(total / pageSize)),
});
} catch (error) {
const message =
error instanceof Error ? error.message : "Data retrieval failed";
return NextResponse.json({ error: message }, { status: 500 });
}
}
+146 -60
View File
@@ -1,76 +1,162 @@
import { NextRequest, NextResponse } from 'next/server';
import { CSVStorageManager } from '@/core/Storage/csvStorage';
import fs from 'fs';
import path from 'path';
import {
CSVStorageManager,
type StorageFileMeta,
} from "@/core/Storage/csvStorage";
import { parse } from "csv-parse";
import { NextRequest, NextResponse } from "next/server";
import { createReadStream } from "node:fs";
export async function GET(req: NextRequest) {
try {
const { searchParams } = new URL(req.url);
const clientId = searchParams.get('clientId');
const filename = searchParams.get('filename');
type ExportFormat = "csv" | "json";
type CsvRecord = Record<string, string>;
type StreamChunk = string | Uint8Array;
// Single specific file export
if (clientId && filename) {
const content = CSVStorageManager.getFileContent(clientId, filename);
if (!content) {
return NextResponse.json({ error: 'CSV file not found' }, { status: 404 });
}
function toByteStream(chunks: AsyncIterable<StreamChunk>) {
const iterator = chunks[Symbol.asyncIterator]();
const encoder = new TextEncoder();
return new NextResponse(content, {
status: 200,
headers: {
'Content-Type': 'text/csv; charset=utf-8',
'Content-Disposition': `attachment; filename="${filename}"`,
},
});
}
// Stream aggregate or list files
const allFiles = CSVStorageManager.listAllFiles();
const filtered = clientId && clientId !== 'all'
? allFiles.filter((f) => f.clientId === clientId)
: allFiles;
if (filtered.length === 0) {
const emptyCsv = 'clientId,eventId,timestamp,payload\n';
return new NextResponse(emptyCsv, {
status: 200,
headers: {
'Content-Type': 'text/csv; charset=utf-8',
'Content-Disposition': `attachment; filename="datastack_export_${Date.now()}.csv"`,
},
});
}
// Combine matching files with a single header
let combined = 'clientId,eventId,timestamp,payload\n';
for (const f of filtered) {
return new ReadableStream<Uint8Array>({
async pull(controller) {
try {
const raw = fs.readFileSync(f.filePath, 'utf8');
const lines = raw.split('\n');
// skip header on subsequent lines
const dataLines = lines.slice(1).filter((l) => l.trim().length > 0);
if (dataLines.length > 0) {
combined += dataLines.join('\n') + '\n';
const result = await iterator.next();
if (result.done) {
controller.close();
} else {
controller.enqueue(
typeof result.value === "string"
? encoder.encode(result.value)
: result.value,
);
}
} catch {
// continue
} catch (error) {
controller.error(error);
}
},
async cancel() {
await iterator.return?.();
},
});
}
async function* readFileRecords(
file: StorageFileMeta,
): AsyncGenerator<CsvRecord> {
const parser = createReadStream(file.filePath).pipe(
parse({ columns: true, skip_empty_lines: true, bom: true }),
);
try {
for await (const record of parser as AsyncIterable<CsvRecord>) {
yield record;
}
} catch {
parser.destroy();
}
}
function makeExportStream(
files: StorageFileMeta[],
format: ExportFormat,
includeHeader: boolean,
) {
const fields = ["clientId", "eventId", "timestamp", "payload"];
const chunks = (async function* () {
let isFirst = true;
if (format === "json") {
yield "[";
} else if (includeHeader) {
yield `${fields.join(",")}\n`;
}
for (const file of files) {
for await (const record of readFileRecords(file)) {
if (format === "json") {
const payload = record.payload ?? "";
let parsedPayload: unknown = payload;
try {
parsedPayload = JSON.parse(payload);
} catch {
// Keep non-JSON payloads as strings.
}
yield `${isFirst ? "" : ","}${JSON.stringify({ ...record, payload: parsedPayload })}`;
} else {
const line = fields
.map((field) => `"${(record[field] ?? "").replaceAll('"', '""')}"`)
.join(",");
yield `${line}\n`;
}
isFirst = false;
}
}
const exportFileName = clientId && clientId !== 'all'
? `${clientId}_export_${new Date().toISOString().split('T')[0]}.csv`
: `datastack_universal_export_${new Date().toISOString().split('T')[0]}.csv`;
if (format === "json") yield "]";
})();
return new NextResponse(combined, {
return toByteStream(chunks);
}
function streamFile(file: StorageFileMeta) {
async function* chunks() {
for await (const chunk of createReadStream(file.filePath)) {
yield new Uint8Array(chunk);
}
}
return toByteStream(chunks());
}
export async function GET(request: NextRequest) {
try {
const { searchParams } = new URL(request.url);
const clientId = searchParams.get("clientId") || "all";
const filename = searchParams.get("filename");
const from = searchParams.get("from") || "";
const to = searchParams.get("to") || "";
const format: ExportFormat =
searchParams.get("format") === "json" ? "json" : "csv";
const includeHeader = searchParams.get("includeHeader") !== "false";
let files = CSVStorageManager.listAllFiles();
if (clientId !== "all")
files = files.filter((file) => file.clientId === clientId);
if (filename) {
const file = files.find((candidate) => candidate.filename === filename);
if (!file)
return NextResponse.json(
{ error: "CSV file not found" },
{ status: 404 },
);
return new NextResponse(streamFile(file), {
status: 200,
headers: {
"Content-Type": "text/csv; charset=utf-8",
"Content-Disposition": `attachment; filename="${file.filename}"`,
},
});
}
files = files.filter(
(file) => (!from || file.date >= from) && (!to || file.date <= to),
);
const datePart = new Date().toISOString().slice(0, 10);
const safeClientId = clientId.replace(/[^a-zA-Z0-9_-]/g, "_");
const baseName =
clientId === "all" ? "datastack_export" : `${safeClientId}_export`;
return new NextResponse(makeExportStream(files, format, includeHeader), {
status: 200,
headers: {
'Content-Type': 'text/csv; charset=utf-8',
'Content-Disposition': `attachment; filename="${exportFileName}"`,
"Content-Type":
format === "json"
? "application/json; charset=utf-8"
: "text/csv; charset=utf-8",
"Content-Disposition": `attachment; filename="${baseName}_${datePart}.${format}"`,
"Cache-Control": "no-store",
},
});
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Export failed';
} catch (error) {
const message = error instanceof Error ? error.message : "Export failed";
return NextResponse.json({ error: message }, { status: 500 });
}
}
+11 -7
View File
@@ -1,12 +1,12 @@
import { NextRequest, NextResponse } from 'next/server';
import { SettingsManager } from '@/core/adminSettings/settingsManager';
import { SettingsManager } from "@/core/adminSettings/settingsManager";
import { NextRequest, NextResponse } from "next/server";
export async function GET() {
try {
const settings = SettingsManager.getSettings();
const settings = SettingsManager.getPublicSettings();
return NextResponse.json({ settings });
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Settings read failed';
const message = err instanceof Error ? err.message : "Settings read failed";
return NextResponse.json({ error: message }, { status: 500 });
}
}
@@ -14,10 +14,14 @@ export async function GET() {
export async function POST(req: NextRequest) {
try {
const body = await req.json();
const updated = SettingsManager.updateSettings(body);
return NextResponse.json({ settings: updated, success: true });
SettingsManager.updateSettings(body);
return NextResponse.json({
settings: SettingsManager.getPublicSettings(),
success: true,
});
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Settings update failed';
const message =
err instanceof Error ? err.message : "Settings update failed";
return NextResponse.json({ error: message }, { status: 500 });
}
}