feat(auth): harden session management and security

- Implement server-side session revocation on logout
- Add IP extraction utility and admin setup error handling
- Improve error codes and response consistency for auth endpoints
- Refine landing page redirect logic to handle transient DB outages
- Update deployment documentation and configuration defaults
This commit is contained in:
Team Dark Devil committed 2026-11-19 20:22:01 +06:00
1 parent 935f12a1e9
commit 89e4d68690
81 files changed
+11530 -6916

No files matched your search

+14 -2
View File
@@ -9,8 +9,20 @@ import { redirect } from 'next/navigation';
export const dynamic = 'force-dynamic';
export default async function LandingPage() {
const isConfigured = await isAdminSetupComplete().catch(() => false);
if (!isConfigured) redirect('/vcon/setup');
try {
const isConfigured = await isAdminSetupComplete();
if (!isConfigured) redirect('/vcon/setup');
} catch (error) {
if (
error &&
typeof error === 'object' &&
'digest' in error &&
String((error as { digest?: unknown }).digest).startsWith('NEXT_REDIRECT')
) {
throw error;
}
// Keep public landing accessible on transient DB outage instead of redirecting to setup
}
return (
<MainLayout>
+12 -12
View File
@@ -4,27 +4,18 @@ import {
getSessionCookieOptions,
} from "@/core/auth/adminSession";
import {
AdminSetupRequiredError,
authenticateAdminCredentials,
clearAdminLoginAttempts,
consumeAdminLoginAttempt,
} from "@/core/auth/adminUsers";
import { extractClientIp } from "@/lib/adminAuth";
import { NextRequest, NextResponse } from "next/server";
const SESSION_COOKIE = "datastack_admin_session";
const SESSION_SECONDS = 8 * 60 * 60;
const REMEMBERED_SESSION_SECONDS = 30 * 24 * 60 * 60;
function extractClientIp(request: NextRequest) {
const forwardedFor = request.headers.get("x-forwarded-for");
if (forwardedFor) {
const first = forwardedFor.split(",")[0]?.trim();
if (first) return first;
}
const realIp = request.headers.get("x-real-ip")?.trim();
if (realIp) return realIp;
return "unknown";
}
export async function POST(request: NextRequest) {
const sessionSecret = getAdminSessionSecret();
@@ -91,7 +82,16 @@ export async function POST(request: NextRequest) {
let authResult: { authenticated: boolean; tokenVersion: number };
try {
authResult = await authenticateAdminCredentials(email, password);
} catch {
} catch (err) {
if (err instanceof AdminSetupRequiredError) {
return NextResponse.json(
{
error: "Initial admin setup is required before signing in.",
code: err.code,
},
{ status: 409 },
);
}
return NextResponse.json(
{ error: "Admin authentication is temporarily unavailable." },
{ status: 503 },
+16
View File
@@ -1,9 +1,25 @@
import { getSessionCookieOptions } from "@/core/auth/adminSession";
import {
revokeAdminSessions,
verifyRequestAdminSession,
} from "@/core/auth/adminUsers";
import { NextRequest, NextResponse } from "next/server";
const SESSION_COOKIE = "datastack_admin_session";
export async function POST(request: NextRequest) {
const token = request.cookies.get(SESSION_COOKIE)?.value;
if (token) {
try {
const check = await verifyRequestAdminSession(token);
if (check.valid) {
await revokeAdminSessions();
}
} catch {
// Still clear client cookie even if server revocation encounters transient DB error
}
}
const response = NextResponse.json({ success: true });
response.cookies.set(SESSION_COOKIE, "", {
...getSessionCookieOptions(request, 0),
+10 -3
View File
@@ -22,11 +22,13 @@ export async function GET(request: NextRequest) {
if (!auth.valid) {
const status = auth.reason === "UNAUTHORIZED" ? 401 : 503;
const code =
auth.reason === "UNAUTHORIZED" ? "SESSION_EXPIRED" : "AUTH_UNAVAILABLE";
const message =
auth.reason === "UNAUTHORIZED"
? "Authentication required."
: "Admin authentication is temporarily unavailable.";
return NextResponse.json({ error: message }, { status });
return NextResponse.json({ error: message, code }, { status });
}
try {
@@ -54,11 +56,13 @@ export async function PUT(request: NextRequest) {
if (!auth.valid) {
const status = auth.reason === "UNAUTHORIZED" ? 401 : 503;
const code =
auth.reason === "UNAUTHORIZED" ? "SESSION_EXPIRED" : "AUTH_UNAVAILABLE";
const message =
auth.reason === "UNAUTHORIZED"
? "Authentication required."
: "Admin authentication is temporarily unavailable.";
return NextResponse.json({ error: message }, { status });
return NextResponse.json({ error: message, code }, { status });
}
let body: Record<string, unknown>;
@@ -115,7 +119,10 @@ export async function PUT(request: NextRequest) {
});
if (!result.ok) {
return NextResponse.json({ error: result.error }, { status: 401 });
return NextResponse.json(
{ error: result.error, code: "INVALID_PASSWORD" },
{ status: 401 },
);
}
// Preserve remaining session duration (e.g., 30-day "Remember me" session)
+59 -7
View File
@@ -4,14 +4,20 @@ import {
getSessionCookieOptions,
} from "@/core/auth/adminSession";
import {
AdminDatabaseRequiredError,
clearAdminLoginAttempts,
consumeAdminLoginAttempt,
createInitialAdmin,
isAdminSetupComplete,
requiresPersistentAdminDatabase,
verifyAdminSetupKey,
} from "@/core/auth/adminUsers";
import { extractClientIp } from "@/lib/adminAuth";
import { NextRequest, NextResponse } from "next/server";
const SESSION_COOKIE = "datastack_admin_session";
const SESSION_SECONDS = 8 * 60 * 60;
const SETUP_THROTTLE_BUCKET = "__admin_setup__";
export const runtime = "nodejs";
export const dynamic = "force-dynamic";
@@ -29,6 +35,17 @@ export async function POST(request: NextRequest) {
);
}
if (requiresPersistentAdminDatabase()) {
return NextResponse.json(
{
error:
"PostgreSQL DATABASE_URL must be configured in production before initial admin setup.",
code: "DATABASE_NOT_CONFIGURED",
},
{ status: 503 },
);
}
let body: Record<string, unknown>;
try {
const parsed = await request.json();
@@ -61,12 +78,7 @@ export async function POST(request: NextRequest) {
{ status: 400 },
);
}
if (!verifyAdminSetupKey(setupKey)) {
return NextResponse.json(
{ error: "The installation key is invalid." },
{ status: 403 },
);
}
if (await isAdminSetupComplete()) {
return NextResponse.json(
{ error: "Initial admin setup has already been completed." },
@@ -74,6 +86,34 @@ export async function POST(request: NextRequest) {
);
}
const clientIp = extractClientIp(request);
const throttleDecision = await consumeAdminLoginAttempt(
SETUP_THROTTLE_BUCKET,
sessionSecret,
clientIp,
);
if (!throttleDecision.allowed) {
return NextResponse.json(
{
error: "Too many setup attempts. Try again later.",
code: "TOO_MANY_ATTEMPTS",
},
{
status: 429,
headers: {
"Retry-After": String(throttleDecision.retryAfterSeconds),
},
},
);
}
if (!verifyAdminSetupKey(setupKey)) {
return NextResponse.json(
{ error: "The installation key is invalid." },
{ status: 403 },
);
}
const created = await createInitialAdmin(email, password);
if (!created) {
return NextResponse.json(
@@ -82,6 +122,12 @@ export async function POST(request: NextRequest) {
);
}
await clearAdminLoginAttempts(
SETUP_THROTTLE_BUCKET,
sessionSecret,
clientIp,
).catch(() => undefined);
const token = await createAdminSessionToken(
email,
sessionSecret,
@@ -95,7 +141,13 @@ export async function POST(request: NextRequest) {
getSessionCookieOptions(request, SESSION_SECONDS),
);
return response;
} catch {
} catch (err) {
if (err instanceof AdminDatabaseRequiredError) {
return NextResponse.json(
{ error: err.message, code: err.code },
{ status: 503 },
);
}
return NextResponse.json(
{ error: "Admin setup is temporarily unavailable." },
{ status: 503 },
+160 -28
View File
@@ -3,6 +3,7 @@ import { SDKManager } from "@/core/sdk/sdkManager";
import { NextRequest, NextResponse } from "next/server";
const SESSION_COOKIE = "datastack_admin_session";
const ALLOWED_PATCH_KEYS = new Set(["action", "name", "rateLimit", "status"]);
export async function GET(
req: NextRequest,
@@ -19,20 +20,33 @@ export async function GET(
}
const { id } = await params;
const client = await SDKManager.getClientAsync(id);
if (!client) {
return NextResponse.json({ error: "Client not found" }, { status: 404 });
const normalizedId = SDKManager.normalizeClientId(id);
try {
const client = await SDKManager.getClientAsync(normalizedId);
if (!client) {
return NextResponse.json({ error: "Client not found" }, { status: 404 });
}
const { secret, token, ...clientSummary } = client;
void secret;
const includeToken = req.nextUrl.searchParams.get("includeToken") === "true";
return NextResponse.json({
client: {
...clientSummary,
tokenPreview: token ? `${token.slice(0, 12)}...` : "",
...(includeToken ? { token } : {}),
},
});
} catch (err: unknown) {
const message = err instanceof Error ? err.message : "Failed to load client";
const isDbError = message.includes("DATABASE_");
return NextResponse.json(
{
error: message,
code: isDbError ? "DATABASE_READ_FAILED" : "CLIENT_FETCH_FAILED",
},
{ status: isDbError ? 503 : 500 },
);
}
const { secret, token, ...clientSummary } = client;
void secret;
const includeToken = req.nextUrl.searchParams.get("includeToken") === "true";
return NextResponse.json({
client: {
...clientSummary,
tokenPreview: token ? `${token.slice(0, 12)}...` : "",
...(includeToken ? { token } : {}),
},
});
}
export async function PATCH(
@@ -50,29 +64,128 @@ export async function PATCH(
}
const { id } = await params;
try {
const body = await req.json();
const normalizedId = SDKManager.normalizeClientId(id);
// Special action: regenerate token
if (body.action === "regenerate_token") {
const refreshed = await SDKManager.regenerateTokenAsync(id);
let rawBody: unknown;
try {
rawBody = await req.json();
} catch {
return NextResponse.json(
{ error: "Invalid JSON request payload.", code: "INVALID_JSON" },
{ status: 400 },
);
}
if (!rawBody || typeof rawBody !== "object" || Array.isArray(rawBody)) {
return NextResponse.json(
{ error: "Update payload must be a JSON object.", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
const body = rawBody as Record<string, unknown>;
const unknownKeys = Object.keys(body).filter(
(key) => !ALLOWED_PATCH_KEYS.has(key),
);
if (unknownKeys.length > 0) {
return NextResponse.json(
{
error: `Unsupported client field(s): ${unknownKeys.join(", ")}`,
code: "VALIDATION_ERROR",
},
{ status: 400 },
);
}
// Special action: regenerate token
if (body.action !== undefined) {
if (body.action !== "regenerate_token") {
return NextResponse.json(
{ error: "Unsupported client action.", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
try {
const refreshed = await SDKManager.regenerateTokenAsync(normalizedId);
if (!refreshed) {
return NextResponse.json(
{ error: "Client not found" },
{ status: 404 },
);
return NextResponse.json({ error: "Client not found" }, { status: 404 });
}
return NextResponse.json({ client: refreshed });
} catch (err: unknown) {
const message =
err instanceof Error ? err.message : "Token regeneration failed";
const isDbWrite = message.includes("DATABASE_WRITE_FAILED");
return NextResponse.json(
{
error: message,
code: isDbWrite ? "DATABASE_WRITE_FAILED" : "TOKEN_REGENERATION_FAILED",
},
{ status: isDbWrite ? 503 : 500 },
);
}
}
const updated = await SDKManager.updateClientAsync(id, body);
const updates: { name?: string; rateLimit?: number; status?: "active" | "revoked" } = {};
if (body.name !== undefined) {
if (
typeof body.name !== "string" ||
body.name.trim().length < 1 ||
body.name.trim().length > 120
) {
return NextResponse.json(
{ error: "Client name must be a non-empty string (max 120 characters).", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
updates.name = body.name.trim();
}
if (body.rateLimit !== undefined) {
if (
typeof body.rateLimit !== "number" ||
!Number.isInteger(body.rateLimit) ||
body.rateLimit < 1 ||
body.rateLimit > 100000
) {
return NextResponse.json(
{ error: "rateLimit must be an integer between 1 and 100000.", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
updates.rateLimit = body.rateLimit;
}
if (body.status !== undefined) {
if (body.status !== "active" && body.status !== "revoked") {
return NextResponse.json(
{ error: "status must be 'active' or 'revoked'.", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
updates.status = body.status;
}
if (Object.keys(updates).length === 0) {
return NextResponse.json(
{ error: "No valid fields provided for update.", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
try {
const updated = await SDKManager.updateClientAsync(normalizedId, updates);
if (!updated) {
return NextResponse.json({ error: "Client not found" }, { status: 404 });
}
return NextResponse.json({ client: updated });
} catch (err: unknown) {
const message = err instanceof Error ? err.message : "Update failed";
return NextResponse.json({ error: message }, { status: 500 });
const isDbWrite = message.includes("DATABASE_WRITE_FAILED");
return NextResponse.json(
{
error: message,
code: isDbWrite ? "DATABASE_WRITE_FAILED" : "CLIENT_UPDATE_FAILED",
},
{ status: isDbWrite ? 503 : 500 },
);
}
}
@@ -91,10 +204,29 @@ export async function DELETE(
}
const { id } = await params;
const normalizedId = SDKManager.normalizeClientId(id);
const purgeFiles = req.nextUrl.searchParams.get("purgeFiles") === "true";
const deleted = await SDKManager.deleteClientAsync(id, { purgeFiles });
if (!deleted) {
return NextResponse.json({ error: "Client not found" }, { status: 404 });
try {
const result = await SDKManager.deleteClientWithStatsAsync(normalizedId, {
purgeFiles,
});
if (!result.deleted) {
return NextResponse.json({ error: "Client not found" }, { status: 404 });
}
return NextResponse.json({
success: true,
message: `Client ${normalizedId} deleted`,
purgedFiles: result.purgedFiles,
});
} catch (err: unknown) {
const message = err instanceof Error ? err.message : "Client deletion failed";
const isDbWrite = message.includes("DATABASE_WRITE_FAILED");
return NextResponse.json(
{
error: message,
code: isDbWrite ? "DATABASE_WRITE_FAILED" : "CLIENT_DELETE_FAILED",
},
{ status: isDbWrite ? 503 : 500 },
);
}
return NextResponse.json({ success: true, message: `Client ${id} deleted` });
}
+20 -8
View File
@@ -60,17 +60,29 @@ export async function POST(req: NextRequest) {
try {
const { clientId, name, rateLimit } = body;
if (!clientId || !name) {
if (typeof clientId !== "string" || typeof name !== "string" || !clientId.trim() || !name.trim()) {
return NextResponse.json(
{ error: "clientId and name are required" },
{ error: "clientId and name are required strings", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
const safeClientId = SDKManager.normalizeClientId(clientId);
if (!safeClientId) {
return NextResponse.json(
{ error: "clientId is invalid", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
const parsedRateLimit = rateLimit === undefined ? 100 : Number(rateLimit);
if (!Number.isInteger(parsedRateLimit) || parsedRateLimit < 1 || parsedRateLimit > 100000) {
return NextResponse.json(
{ error: "rateLimit must be an integer between 1 and 100000.", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
const safeClientId = String(clientId)
.trim()
.toLowerCase()
.replace(/[^a-z0-9_-]/g, "_");
const existing = await SDKManager.getClientAsync(safeClientId);
if (existing) {
return NextResponse.json(
@@ -81,8 +93,8 @@ export async function POST(req: NextRequest) {
const newClient = await SDKManager.registerClientAsync({
clientId: safeClientId,
name: String(name),
rateLimit: rateLimit ? Number(rateLimit) : 100,
name: name.trim().slice(0, 120),
rateLimit: parsedRateLimit,
});
return NextResponse.json({ client: newClient }, { status: 201 });
+88 -20
View File
@@ -1,10 +1,13 @@
import { SettingsManager } from "@/core/adminSettings/settingsManager";
import { DataBufferEngine } from "@/core/data/buffer";
import { SDKManager } from "@/core/sdk/sdkManager";
import { getInternalIngestionHeaders } from "@/core/Security/security";
import { CSVStorageManager } from "@/core/Storage/csvStorage";
import { requireAdminApiSession } from "@/lib/adminAuth";
import { parse } from "csv-parse";
import { NextRequest, NextResponse } from "next/server";
import { createReadStream, readdirSync, statSync } from "node:fs";
import { createReadStream } from "node:fs";
import { lstat, readdir } from "node:fs/promises";
import path from "node:path";
interface DataEvent {
@@ -15,10 +18,40 @@ interface DataEvent {
status: "stored" | "buffered";
}
function parseTimestampMs(raw: string): number {
const trimmed = String(raw || "").trim();
if (/^\d+(\.\d+)?$/.test(trimmed)) {
const num = Number(trimmed);
return num < 1e11 ? num * 1000 : num;
}
const parsed = Date.parse(trimmed);
return Number.isNaN(parsed) ? Number.NEGATIVE_INFINITY : parsed;
}
function compareEventsDesc(a: DataEvent, b: DataEvent): number {
const aMs = parseTimestampMs(a.timestamp);
const bMs = parseTimestampMs(b.timestamp);
if (aMs !== bMs) {
return bMs - aMs;
}
return b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0;
}
export async function GET(request: NextRequest) {
const auth = await requireAdminApiSession(request);
if (!auth.valid) {
return auth.response;
}
try {
const { searchParams } = new URL(request.url);
const clientId = searchParams.get("clientId") || "all";
const rawClientId = searchParams.get("clientId") || "all";
const clientId =
rawClientId === "all"
? "all"
: SDKManager.normalizeClientId(rawClientId) || "all";
const sanitizedClientId =
clientId === "all" ? "all" : CSVStorageManager.sanitizeClientId(clientId);
const from = searchParams.get("from") || "";
const to = searchParams.get("to") || "";
const search = (searchParams.get("search") || "").toLowerCase();
@@ -34,7 +67,12 @@ export async function GET(request: NextRequest) {
const addIfMatching = (event: DataEvent) => {
const eventDate = CSVStorageManager.extractSafePartitionDate(event.timestamp, null);
const dedupeKey = `${event.clientId}:${event.eventId}`;
const normEventClient = SDKManager.normalizeClientId(event.clientId) || event.clientId;
const dedupeKey = `${normEventClient}:${event.eventId}`;
const matchesClient =
clientId === "all" ||
normEventClient === clientId ||
CSVStorageManager.sanitizeClientId(normEventClient) === sanitizedClientId;
const matchesSearch =
!search ||
event.payload.toLowerCase().includes(search) ||
@@ -42,7 +80,7 @@ export async function GET(request: NextRequest) {
event.clientId.toLowerCase().includes(search);
if (
(clientId !== "all" && event.clientId !== clientId) ||
!matchesClient ||
((from || to) && (!eventDate || (from && eventDate < from) || (to && eventDate > to))) ||
!matchesSearch ||
eventIds.has(dedupeKey)
@@ -57,7 +95,7 @@ export async function GET(request: NextRequest) {
let high = selectedRows.length;
while (low < high) {
const middle = (low + high) >>> 1;
if (selectedRows[middle].timestamp >= event.timestamp) low = middle + 1;
if (compareEventsDesc(selectedRows[middle], event) <= 0) low = middle + 1;
else high = middle;
}
@@ -70,31 +108,61 @@ export async function GET(request: NextRequest) {
const storageDir = CSVStorageManager.baseStorageDir;
let clientDirs: string[] = [];
try {
clientDirs = readdirSync(storageDir);
clientDirs = await readdir(storageDir);
} catch {
clientDirs = [];
}
for (const storedClientId of clientDirs) {
if (clientId !== "all" && storedClientId !== clientId) continue;
if (
clientId !== "all" &&
storedClientId !== clientId &&
storedClientId !== sanitizedClientId
) {
continue;
}
const clientDir = path.join(storageDir, storedClientId);
if (!statSync(clientDir).isDirectory()) continue;
let dirLstat;
try {
dirLstat = await lstat(clientDir);
} catch {
continue;
}
if (dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
let filenames: string[];
try {
filenames = readdirSync(clientDir).filter((filename) =>
/^\d{4}-\d{2}-\d{2}(?:-part\d+)?\.csv$/.test(filename),
);
filenames = (await readdir(clientDir))
.filter((filename) => {
if (!/^\d{4}-\d{2}-\d{2}(?:-part\d+)?\.csv$/.test(filename)) {
return false;
}
const fileDate = filename.slice(0, 10);
if ((from && fileDate < from) || (to && fileDate > to)) {
return false;
}
return true;
})
.sort((a, b) => b.localeCompare(a));
} catch {
continue;
}
for (const filename of filenames) {
const fileDate = filename.slice(0, 10);
if ((from && fileDate < from) || (to && fileDate > to)) continue;
const parser = createReadStream(path.join(clientDir, filename)).pipe(
const filePath = path.join(clientDir, filename);
let fileLstat;
try {
fileLstat = await lstat(filePath);
} catch {
continue;
}
if (fileLstat.isSymbolicLink() || !fileLstat.isFile()) continue;
const fileStream = createReadStream(filePath);
const parser = fileStream.pipe(
parse({ columns: true, skip_empty_lines: true, bom: true }),
);
@@ -111,17 +179,17 @@ export async function GET(request: NextRequest) {
};
if (event.eventId) addIfMatching(event);
}
} catch {
parser.destroy();
} catch (err) {
console.warn(`[API /data] Failed reading CSV partition ${filePath}:`, err);
} finally {
if (!parser.destroyed) parser.destroy();
if (!fileStream.destroyed) fileStream.destroy();
}
}
}
const settings = await SettingsManager.getSettingsAsync();
const recentWindowLimit = Math.min(
2000,
Math.max(200, settings.batchFlushThreshold || 500),
);
await SettingsManager.getSettingsAsync();
const recentWindowLimit = DataBufferEngine.getRecentLimit();
for (const event of await DataBufferEngine.getRecentEventsAsync(
recentWindowLimit,
+296 -41
View File
@@ -1,33 +1,240 @@
import { SettingsManager } from '@/core/adminSettings/settingsManager';
import { verifyRequestAdminSession } from '@/core/auth/adminUsers';
import { AuthService } from '@/core/auth/jwt';
import { DataBufferEngine } from '@/core/data/buffer';
import { DataBufferEngine, REDIS_BUFFER_KEY } from '@/core/data/buffer';
import { SDKManager } from '@/core/sdk/sdkManager';
import {
SecurityEngine,
getInternalIngestionHeaders,
} from '@/core/Security/security';
import { CSVStorageManager, type DataEvent } from '@/core/Storage/csvStorage';
import {
CSVStorageManager,
type DataEvent,
type StorageFileMeta,
} from '@/core/Storage/csvStorage';
import { parse } from 'csv-parse';
import { NextRequest, NextResponse } from 'next/server';
import crypto from 'node:crypto';
import { createReadStream } from 'node:fs';
import { requireAdminApiSession } from '@/lib/adminAuth';
import type { BufferedBucket, BufferedSummary } from '@/lib/vconUiHelpers';
const SESSION_COOKIE = 'datastack_admin_session';
const SINGLE_EVENT_ENVELOPE_KEYS = new Set(['eventId', 'timestamp', 'data', 'clientId']);
const MAX_SINGLE_CLIENT_SCAN_FILES = 10;
const MAX_ALL_CLIENTS_PER_CLIENT_FILES = 2;
const MAX_ALL_CLIENTS_TOTAL_SCAN_FILES = 50;
function generateFallbackEventId(): string {
return `evt_${Date.now()}_${crypto.randomBytes(6).toString('hex')}`;
}
function extractSingleHttpEventData(rawBody: Record<string, unknown>): unknown {
if (rawBody.data !== undefined) {
const keys = Object.keys(rawBody);
if (keys.every((k) => SINGLE_EVENT_ENVELOPE_KEYS.has(k))) {
return rawBody.data;
}
return rawBody;
}
const stripped: Record<string, unknown> = {};
let hasNonEnvelopeKey = false;
for (const [k, v] of Object.entries(rawBody)) {
if (k === 'eventId' || k === 'timestamp' || k === 'clientId') {
continue;
}
hasNonEnvelopeKey = true;
stripped[k] = v;
}
return hasNonEnvelopeKey ? stripped : {};
}
function parseTimestampMs(raw: string): number {
const trimmed = String(raw || '').trim();
if (/^\d+(\.\d+)?$/.test(trimmed)) {
const num = Number(trimmed);
return num < 1e11 ? num * 1000 : num;
}
const parsed = Date.parse(trimmed);
return Number.isNaN(parsed) ? Number.NEGATIVE_INFINITY : parsed;
}
function resolveEventUtcDate(rawTimestamp: string): string {
const ms = parseTimestampMs(rawTimestamp);
if (Number.isFinite(ms)) {
const parsed = new Date(ms);
if (!Number.isNaN(parsed.getTime())) {
return parsed.toISOString().slice(0, 10);
}
}
return new Date().toISOString().slice(0, 10);
}
function selectRecentPartitionFiles(
allFiles: StorageFileMeta[],
normClientId?: string,
sanitizedDir?: string,
): StorageFileMeta[] {
const matching = allFiles
.filter(
(f) =>
!normClientId ||
f.clientId === normClientId ||
f.clientId === sanitizedDir,
)
.sort((a, b) => {
if (a.date !== b.date) return b.date.localeCompare(a.date);
return new Date(b.modifiedAt).getTime() - new Date(a.modifiedAt).getTime();
});
if (normClientId) {
return matching.slice(0, MAX_SINGLE_CLIENT_SCAN_FILES);
}
// Multi-client fair scan: take up to 2 newest partitions per active client (capped at 50 files total)
// so clients beyond the 5th most-recently-modified file are never silently hidden.
const perClientCounts = new Map<string, number>();
const selected: StorageFileMeta[] = [];
for (const file of matching) {
const currentCount = perClientCounts.get(file.clientId) ?? 0;
if (currentCount >= MAX_ALL_CLIENTS_PER_CLIENT_FILES) continue;
perClientCounts.set(file.clientId, currentCount + 1);
selected.push(file);
if (selected.length >= MAX_ALL_CLIENTS_TOTAL_SCAN_FILES) break;
}
return selected;
}
async function getPendingBufferedSummary(
ingestionBaseUrl: string,
filterClientId?: string,
): Promise<{ events: DataEvent[]; summary: BufferedSummary }> {
const bufferStats = await DataBufferEngine.getBufferStatsAsync();
const recentLimit = DataBufferEngine.getRecentLimit();
const pending: DataEvent[] = [];
let totalQueued = bufferStats.localBufferSize + bufferStats.redisBufferSize;
// 1. Read exact unflushed Redis queue (when Redis is enabled & connected)
const redis = DataBufferEngine.getRedisClient();
if (redis && bufferStats.redisBufferSize > 0) {
try {
const cap = Math.min(recentLimit, bufferStats.redisBufferSize);
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, cap - 1);
for (const raw of rawItems) {
try {
const parsed = JSON.parse(raw) as DataEvent;
if (!parsed || !parsed.eventId) continue;
pending.push(parsed);
} catch {
// Skip malformed queued item
}
}
} catch {
// Fall back to local buffer telemetry
}
}
// 2. Read Port 3000 exact unflushed RAM queue snapshot
if (bufferStats.localBufferSize > 0) {
const localPending = DataBufferEngine.getLocalBufferSnapshot(recentLimit);
pending.push(...localPending);
}
// 3. When Redis is not enabled or temporarily disconnected, Port 4000 `/internal/recent` serves its unflushed RAM `bufferQueue`
if (!bufferStats.redisEnabled || !bufferStats.redisConnected) {
try {
const qs = new URLSearchParams({ limit: String(recentLimit) });
const res = await fetch(`${ingestionBaseUrl}/internal/recent?${qs.toString()}`, {
headers: getInternalIngestionHeaders(),
cache: 'no-store',
signal: AbortSignal.timeout(1000),
});
if (res.ok) {
const data = await res.json();
if (Array.isArray(data.events) && data.events.length > 0) {
pending.push(...(data.events as DataEvent[]));
totalQueued += data.events.length;
}
}
} catch {
// Port 4000 may not be running in isolated single-process dev mode
}
}
const seen = new Set<string>();
const dedupedAll: DataEvent[] = [];
const filteredEvents: DataEvent[] = [];
const bucketMap = new Map<string, BufferedBucket>();
const sanitizedFilter = filterClientId
? CSVStorageManager.sanitizeClientId(filterClientId)
: undefined;
for (const evt of pending) {
if (!evt || !evt.eventId) continue;
const normEvtClient = SDKManager.normalizeClientId(evt.clientId) || evt.clientId;
const dedupeKey = `${normEvtClient}:${evt.eventId}`;
if (seen.has(dedupeKey)) continue;
seen.add(dedupeKey);
dedupedAll.push(evt);
const matchesFilter =
!filterClientId ||
normEvtClient === filterClientId ||
evt.clientId === filterClientId ||
evt.clientId === sanitizedFilter;
if (matchesFilter) {
filteredEvents.push(evt);
const utcDate = resolveEventUtcDate(evt.timestamp);
const bucketKey = `${normEvtClient}|${utcDate}`;
const existing = bucketMap.get(bucketKey);
if (existing) {
existing.count += 1;
} else {
bucketMap.set(bucketKey, {
clientId: normEvtClient,
date: utcDate,
count: 1,
});
}
}
}
const effectiveTotalQueued = Math.max(totalQueued, dedupedAll.length);
const unattributedCount = Math.max(0, effectiveTotalQueued - dedupedAll.length);
return {
events: filteredEvents,
summary: {
total: filteredEvents.length,
totalQueued: filterClientId
? filteredEvents.length + unattributedCount
: effectiveTotalQueued,
unattributedCount,
truncated: unattributedCount > 0,
buckets: Array.from(bucketMap.values()),
},
};
}
async function getRecentStoredEvents(
limit: number,
clientId?: string,
): Promise<DataEvent[]> {
const normClientId = clientId && clientId !== 'all' ? SDKManager.normalizeClientId(clientId) : undefined;
const sanitizedDir = normClientId ? CSVStorageManager.sanitizeClientId(normClientId) : undefined;
const allFiles = await CSVStorageManager.listAllFilesAsync();
const files = allFiles
.filter((f) => !clientId || clientId === 'all' || f.clientId === clientId)
.slice(0, 5);
const files = selectRecentPartitionFiles(allFiles, normClientId, sanitizedDir);
const collected: DataEvent[] = [];
for (const file of files) {
const fileTail: DataEvent[] = [];
const parser = createReadStream(file.filePath).pipe(
const fileStream = createReadStream(file.filePath);
const parser = fileStream.pipe(
parse({ columns: true, skip_empty_lines: true, bom: true }),
);
@@ -46,8 +253,11 @@ async function getRecentStoredEvents(
payload: record.payload || '',
});
}
} catch {
parser.destroy();
} catch (err) {
console.warn(`[API /events] Failed reading CSV partition ${file.filePath}:`, err);
} finally {
if (!parser.destroyed) parser.destroy();
if (!fileStream.destroyed) fileStream.destroy();
}
if (fileTail.length > 0) {
@@ -59,18 +269,47 @@ async function getRecentStoredEvents(
}
export async function GET(req: NextRequest) {
const auth = await requireAdminApiSession(req);
if (!auth.valid) {
return auth.response;
}
try {
const { searchParams } = new URL(req.url);
const clientId = searchParams.get('clientId') || undefined;
const rawClientId = searchParams.get('clientId');
const clientId =
rawClientId && rawClientId !== 'all'
? SDKManager.normalizeClientId(rawClientId) || undefined
: undefined;
const limit = Math.min(2000, Math.max(1, Number(searchParams.get('limit')) || 50));
const bufferedOnly = searchParams.get('bufferedOnly') === 'true';
const includeBufferedSummary =
bufferedOnly || searchParams.get('includeBufferedSummary') === 'true';
await SettingsManager.getSettingsAsync();
const bufferedEvents = await DataBufferEngine.getRecentEventsAsync(limit, clientId);
let socketBufferedEvents: DataEvent[] = [];
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
).replace(/\/$/, '');
if (bufferedOnly) {
const pendingResult = await getPendingBufferedSummary(ingestionBaseUrl, clientId);
const sortedPending = [...pendingResult.events].sort((a, b) => {
const aMs = parseTimestampMs(a.timestamp);
const bMs = parseTimestampMs(b.timestamp);
if (aMs !== bMs) return bMs - aMs;
return b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0;
});
const events = sortedPending.slice(0, limit);
return NextResponse.json({
events,
count: events.length,
bufferedSummary: pendingResult.summary,
});
}
const bufferedEvents = await DataBufferEngine.getRecentEventsAsync(limit, clientId);
let socketBufferedEvents: DataEvent[] = [];
try {
const qs = new URLSearchParams({ limit: String(limit) });
if (clientId) qs.set('clientId', clientId);
@@ -89,23 +328,38 @@ export async function GET(req: NextRequest) {
// Port 4000 may not be running in isolated single-process dev mode
}
const storedEvents = await getRecentStoredEvents(limit, clientId);
const [storedEvents, pendingResult] = await Promise.all([
getRecentStoredEvents(limit, clientId),
includeBufferedSummary
? getPendingBufferedSummary(ingestionBaseUrl, clientId)
: Promise.resolve(null),
]);
const seenIds = new Set<string>();
const merged: DataEvent[] = [];
for (const evt of [...bufferedEvents, ...socketBufferedEvents, ...storedEvents]) {
if (!evt.eventId) continue;
const dedupeKey = `${evt.clientId}:${evt.eventId}`;
const normEvtClient = SDKManager.normalizeClientId(evt.clientId) || evt.clientId;
const dedupeKey = `${normEvtClient}:${evt.eventId}`;
if (seenIds.has(dedupeKey)) continue;
seenIds.add(dedupeKey);
merged.push(evt);
}
merged.sort((a, b) => (b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0));
merged.sort((a, b) => {
const aMs = parseTimestampMs(a.timestamp);
const bMs = parseTimestampMs(b.timestamp);
if (aMs !== bMs) return bMs - aMs;
return b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0;
});
const events = merged.slice(0, limit);
return NextResponse.json({ events, count: events.length });
return NextResponse.json({
events,
count: events.length,
...(pendingResult ? { bufferedSummary: pendingResult.summary } : {}),
});
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Events retrieval failed';
return NextResponse.json({ error: message }, { status: 500 });
@@ -165,12 +419,18 @@ export async function POST(req: NextRequest) {
? (body as Record<string, unknown>)
: {};
const clientId = String(
tokenClientId || bodyObj.clientId || 'py_telemetry_agent_01',
)
.trim()
.toLowerCase()
.replace(/[^a-z0-9_-]/g, '_');
const clientId = SDKManager.normalizeClientId(
String(tokenClientId || bodyObj.clientId || ''),
);
if (!clientId) {
return NextResponse.json(
{
error: 'clientId is required and must belong to a registered client.',
code: 'VALIDATION_ERROR',
},
{ status: 400 },
);
}
if (await SDKManager.isClientDeletedAsync(clientId)) {
return NextResponse.json(
@@ -183,7 +443,7 @@ export async function POST(req: NextRequest) {
}
const registeredClient = await SDKManager.getClientAsync(clientId);
if (rawBearerToken && !registeredClient) {
if (!registeredClient) {
return NextResponse.json(
{
error: `CLIENT_REVOKED: Client '${clientId}' registration not found or has been deleted.`,
@@ -231,10 +491,11 @@ export async function POST(req: NextRequest) {
? (bodyObj.batch as unknown[])
: Array.isArray(bodyObj.records)
? (bodyObj.records as unknown[])
: null;
: bodyObj.data === undefined && Array.isArray(bodyObj.events)
? (bodyObj.events as unknown[])
: null;
if (batchItems !== null) {
const envelopeKeys = new Set(['eventId', 'timestamp', 'data', 'clientId']);
let processed = 0;
let rejected = 0;
let duplicates = 0;
@@ -255,19 +516,16 @@ export async function POST(req: NextRequest) {
const itemEventId =
typeof safeItem.eventId === 'string' && safeItem.eventId.trim()
? safeItem.eventId.trim()
: `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
: generateFallbackEventId();
const itemTimestamp =
typeof safeItem.timestamp === 'string' && safeItem.timestamp.trim()
? safeItem.timestamp.trim()
: new Date().toISOString();
const isEnvelope =
safeItem.data !== undefined &&
Object.keys(safeItem).every((k) => envelopeKeys.has(k));
const itemData = isEnvelope ? safeItem.data : safeItem;
const itemData = extractSingleHttpEventData(safeItem);
const val = SecurityEngine.validateEvent(
const val = await SecurityEngine.validateEventAsync(
clientId,
itemEventId,
itemData,
@@ -309,7 +567,7 @@ export async function POST(req: NextRequest) {
}
if (processed > 0) {
SDKManager.incrementEventCount(clientId, processed);
await SDKManager.incrementEventCountAsync(clientId, processed);
}
// If zero items were processed or deduplicated and all failed due to payload or rate limit, return corresponding status
@@ -323,6 +581,7 @@ export async function POST(req: NextRequest) {
return NextResponse.json(
{
success: false,
accepted: processed,
processed,
rejected,
duplicates,
@@ -338,6 +597,7 @@ export async function POST(req: NextRequest) {
return NextResponse.json({
success: true,
accepted: processed,
processed,
rejected,
duplicates,
@@ -350,24 +610,19 @@ export async function POST(req: NextRequest) {
});
}
// Single Event Ingestion
// Single Event Ingestion (exact envelope parity with Port 4000 WS sync_data)
const eventId =
typeof bodyObj.eventId === 'string' && bodyObj.eventId.trim()
? bodyObj.eventId.trim()
: `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
: generateFallbackEventId();
const timestamp =
typeof bodyObj.timestamp === 'string' && bodyObj.timestamp.trim()
? bodyObj.timestamp.trim()
: new Date().toISOString();
const data =
bodyObj.data !== undefined
? bodyObj.data
: Object.keys(bodyObj).length > 0
? bodyObj
: { sample: 'Realtime event via DataStack API', rand: Math.random() };
const data = extractSingleHttpEventData(bodyObj);
// Validate security (idempotency, rate limit, payload size)
const val = SecurityEngine.validateEvent(
const val = await SecurityEngine.validateEventAsync(
clientId,
eventId,
data,
@@ -393,7 +648,7 @@ export async function POST(req: NextRequest) {
};
await DataBufferEngine.pushAsync(event);
SDKManager.incrementEventCount(clientId, 1);
await SDKManager.incrementEventCountAsync(clientId, 1);
return NextResponse.json({ success: true, eventId, event });
} catch (err: unknown) {
+127 -22
View File
@@ -1,10 +1,12 @@
import { SettingsManager } from "@/core/adminSettings/settingsManager";
import { DataBufferEngine } from "@/core/data/buffer";
import { SDKManager } from "@/core/sdk/sdkManager";
import { getInternalIngestionHeaders } from "@/core/Security/security";
import {
CSVStorageManager,
type StorageFileMeta,
} from "@/core/Storage/csvStorage";
import { requireAdminApiSession } from "@/lib/adminAuth";
import { parse } from "csv-parse";
import { NextRequest, NextResponse } from "next/server";
import { createReadStream } from "node:fs";
@@ -44,7 +46,8 @@ function toByteStream(chunks: AsyncIterable<StreamChunk>) {
async function* readFileRecords(
file: StorageFileMeta,
): AsyncGenerator<CsvRecord> {
const parser = createReadStream(file.filePath).pipe(
const fileStream = createReadStream(file.filePath);
const parser = fileStream.pipe(
parse({ columns: true, skip_empty_lines: true, bom: true }),
);
@@ -52,8 +55,9 @@ async function* readFileRecords(
for await (const record of parser as AsyncIterable<CsvRecord>) {
yield record;
}
} catch {
parser.destroy();
} finally {
if (!parser.destroyed) parser.destroy();
if (!fileStream.destroyed) fileStream.destroy();
}
}
@@ -85,7 +89,7 @@ function makeExportStream(
yield `${isFirst ? "" : ","}${JSON.stringify({ ...record, payload: parsedPayload })}`;
} else {
const line = fields
.map((field) => `"${(record[field] ?? "").replaceAll('"', '""')}"`)
.map((field) => `"${String(record[field] ?? "").replaceAll('"', '""')}"`)
.join(",");
yield `${line}\n`;
}
@@ -101,8 +105,13 @@ function makeExportStream(
function streamFile(file: StorageFileMeta) {
async function* chunks() {
for await (const chunk of createReadStream(file.filePath)) {
yield new Uint8Array(chunk);
const fileStream = createReadStream(file.filePath);
try {
for await (const chunk of fileStream) {
yield new Uint8Array(chunk as Buffer);
}
} finally {
if (!fileStream.destroyed) fileStream.destroy();
}
}
@@ -110,38 +119,122 @@ function streamFile(file: StorageFileMeta) {
}
export async function GET(request: NextRequest) {
const auth = await requireAdminApiSession(request);
if (!auth.valid) {
return auth.response;
}
try {
const { searchParams } = new URL(request.url);
const from = (searchParams.get("from") || "").trim();
const to = (searchParams.get("to") || "").trim();
if (from && to && from > to) {
return NextResponse.json(
{
error: 'Start date ("from") must be on or before end date ("to").',
code: "INVALID_DATE_RANGE",
},
{ status: 400 },
);
}
const shouldFlush = searchParams.get("flush") !== "false";
await SettingsManager.getSettingsAsync();
if (shouldFlush) {
await DataBufferEngine.flush();
try {
await DataBufferEngine.flush({ waitMs: 2500, throwOnLockTimeout: true });
} catch (err: unknown) {
const message =
err instanceof Error ? err.message : "Pre-export flush failed";
if (message.includes("FLUSH_LOCK_TIMEOUT")) {
return NextResponse.json(
{
error: message,
code: "FLUSH_LOCK_TIMEOUT",
},
{ status: 503 },
);
}
throw err;
}
const explicitIngestionUrl = Boolean(process.env.INGESTION_INTERNAL_URL?.trim());
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, "");
try {
await fetch(`${ingestionBaseUrl}/internal/flush`, {
const flushRes = await fetch(`${ingestionBaseUrl}/internal/flush`, {
method: "POST",
headers: getInternalIngestionHeaders(),
signal: AbortSignal.timeout(1500),
headers: {
"Content-Type": "application/json",
...getInternalIngestionHeaders(),
},
body: JSON.stringify({ waitMs: 2500, throwOnLockTimeout: true }),
cache: "no-store",
signal: AbortSignal.timeout(4000),
});
} catch {
if (!flushRes.ok) {
const errPayload = await flushRes.json().catch(() => ({}));
const remoteErr =
typeof errPayload?.error === "string"
? errPayload.error
: `Ingestion flush returned HTTP ${flushRes.status}`;
if (
flushRes.status === 503 &&
errPayload?.code === "FLUSH_LOCK_TIMEOUT"
) {
return NextResponse.json(
{
error: remoteErr,
code: "FLUSH_LOCK_TIMEOUT",
ingestionStatus: flushRes.status,
},
{ status: 503 },
);
}
return NextResponse.json(
{
error: remoteErr,
code: "INGESTION_FLUSH_FAILED",
ingestionStatus: flushRes.status,
},
{ status: 502 },
);
}
} catch (err: unknown) {
if (explicitIngestionUrl) {
const errMsg =
err instanceof Error ? err.message : "Ingestion server unreachable";
return NextResponse.json(
{
error: `Ingestion server flush failed before export: ${errMsg}`,
code: "INGESTION_UNREACHABLE",
},
{ status: 502 },
);
}
// Port 4000 may not be running in isolated single-process dev mode
}
}
const clientId = searchParams.get("clientId") || "all";
const rawClientId = searchParams.get("clientId") || "all";
const clientId =
rawClientId === "all"
? "all"
: SDKManager.normalizeClientId(rawClientId) || "all";
const sanitizedClientId =
clientId === "all" ? "all" : CSVStorageManager.sanitizeClientId(clientId);
const filename = searchParams.get("filename");
const from = searchParams.get("from") || "";
const to = searchParams.get("to") || "";
const format: ExportFormat =
searchParams.get("format") === "json" ? "json" : "csv";
const rawFormat = searchParams.get("format");
const format: ExportFormat = rawFormat === "json" ? "json" : "csv";
const includeHeader = searchParams.get("includeHeader") !== "false";
let files = await CSVStorageManager.listAllFilesAsync();
if (clientId !== "all")
files = files.filter((file) => file.clientId === clientId);
if (clientId !== "all") {
files = files.filter(
(file) => file.clientId === clientId || file.clientId === sanitizedClientId,
);
}
if (filename) {
const safeFilename = path.basename(filename);
@@ -166,11 +259,23 @@ export async function GET(request: NextRequest) {
);
}
const file = matches[0];
return new NextResponse(streamFile(file), {
const downloadName =
format === "json"
? `${file.filename.replace(/\.csv$/i, "")}.json`
: file.filename;
const bodyStream =
format === "json" || !includeHeader
? makeExportStream([file], format, includeHeader)
: streamFile(file);
return new NextResponse(bodyStream, {
status: 200,
headers: {
"Content-Type": "text/csv; charset=utf-8",
"Content-Disposition": `attachment; filename="${file.filename}"`,
"Content-Type":
format === "json"
? "application/json; charset=utf-8"
: "text/csv; charset=utf-8",
"Content-Disposition": `attachment; filename="${downloadName.replace(/"/g, "")}"`,
"Cache-Control": "no-store",
},
});
}
@@ -179,7 +284,7 @@ export async function GET(request: NextRequest) {
(file) => (!from || file.date >= from) && (!to || file.date <= to),
);
const datePart = new Date().toISOString().slice(0, 10);
const safeClientId = clientId.replace(/[^a-zA-Z0-9_-]/g, "_");
const safeClientId = sanitizedClientId === "all" ? "all" : sanitizedClientId;
const baseName =
clientId === "all" ? "datastack_export" : `${safeClientId}_export`;
+33 -25
View File
@@ -23,10 +23,10 @@ export async function GET(req: NextRequest) {
);
}
const clientId = String(verified.payload.clientId || '')
.trim()
.toLowerCase();
if (SDKManager.isClientDeleted(clientId)) {
const clientId = SDKManager.normalizeClientId(
String(verified.payload.clientId || ''),
);
if (await SDKManager.isClientDeletedAsync(clientId)) {
return NextResponse.json(
{
status: 'error',
@@ -37,27 +37,35 @@ export async function GET(req: NextRequest) {
);
}
const registeredClient = await SDKManager.getClientAsync(clientId);
if (registeredClient) {
if (registeredClient.status === 'revoked') {
return NextResponse.json(
{
status: 'error',
code: 'CLIENT_REVOKED',
error: `CLIENT_REVOKED: Client '${clientId}' access has been revoked by administrator.`,
},
{ status: 403 },
);
}
if (registeredClient.token && registeredClient.token !== rawToken) {
return NextResponse.json(
{
status: 'error',
code: 'TOKEN_ROTATED',
error: 'TOKEN_ROTATED: Client credentials have been rotated. Use the latest token.',
},
{ status: 401 },
);
}
if (!registeredClient) {
return NextResponse.json(
{
status: 'error',
code: 'CLIENT_REVOKED',
error: `CLIENT_REVOKED: Client '${clientId}' registration not found or has been deleted.`,
},
{ status: 403 },
);
}
if (registeredClient.status === 'revoked') {
return NextResponse.json(
{
status: 'error',
code: 'CLIENT_REVOKED',
error: `CLIENT_REVOKED: Client '${clientId}' access has been revoked by administrator.`,
},
{ status: 403 },
);
}
if (registeredClient.token && registeredClient.token !== rawToken) {
return NextResponse.json(
{
status: 'error',
code: 'TOKEN_ROTATED',
error: 'TOKEN_ROTATED: Client credentials have been rotated. Use the latest token.',
},
{ status: 401 },
);
}
}
+8 -2
View File
@@ -1,7 +1,13 @@
import { NextResponse } from 'next/server';
import { NextRequest, NextResponse } from 'next/server';
import { ApiEngine } from '@/core/apiEngine/apiService';
import { requireAdminApiSession } from '@/lib/adminAuth';
export async function GET(req: NextRequest) {
const auth = await requireAdminApiSession(req);
if (!auth.valid) {
return auth.response;
}
export async function GET() {
try {
const metrics = await ApiEngine.getDashboardMetrics();
return NextResponse.json(metrics);
+15
View File
@@ -188,9 +188,24 @@ export async function POST(req: NextRequest) {
await SettingsManager.updateSettingsAsync(validated);
const settings = await SettingsManager.getPublicSettingsAsync();
const bridgeSync = SettingsManager.getLastBridgeSyncStatus();
return NextResponse.json({
settings,
success: true,
...(bridgeSync
? {
bridgeSync: {
attempted: bridgeSync.attempted ?? true,
ok: bridgeSync.ok ?? bridgeSync.synced,
synced: bridgeSync.synced,
...(bridgeSync.status !== undefined
? { status: bridgeSync.status }
: {}),
...(bridgeSync.error ? { error: bridgeSync.error } : {}),
checkedAt: bridgeSync.checkedAt,
},
}
: {}),
});
} catch (err: unknown) {
const message =
+70 -11
View File
@@ -1,24 +1,44 @@
import { NextRequest, NextResponse } from 'next/server';
import { SettingsManager } from '@/core/adminSettings/settingsManager';
import { SDKManager } from '@/core/sdk/sdkManager';
import { getInternalIngestionHeaders } from '@/core/Security/security';
import { CSVStorageManager } from '@/core/Storage/csvStorage';
import { DataBufferEngine } from '@/core/data/buffer';
import { requireAdminApiSession } from '@/lib/adminAuth';
export async function GET(req: NextRequest) {
const auth = await requireAdminApiSession(req);
if (!auth.valid) {
return auth.response;
}
try {
const { searchParams } = new URL(req.url);
const clientId = searchParams.get('clientId') || undefined;
const rawClientId = searchParams.get('clientId');
const clientId =
rawClientId && rawClientId !== 'all'
? SDKManager.normalizeClientId(rawClientId)
: undefined;
const sanitizedClientId = clientId
? CSVStorageManager.sanitizeClientId(clientId)
: undefined;
const [stats, allFiles] = await Promise.all([
CSVStorageManager.getStorageStatsAsync(),
CSVStorageManager.listAllFilesAsync(),
]);
const allFiles = await CSVStorageManager.listAllFilesAsync();
let files = allFiles;
if (clientId && clientId !== 'all') {
files = files.filter((f) => f.clientId === clientId);
if (clientId) {
files = allFiles.filter((f) => {
const normFolder = SDKManager.normalizeClientId(f.clientId);
return (
normFolder === clientId ||
f.clientId === clientId ||
f.clientId === sanitizedClientId
);
});
}
const stats = CSVStorageManager.formatStorageStats(files);
return NextResponse.json({ stats, files });
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Storage stats failed';
@@ -26,12 +46,20 @@ export async function GET(req: NextRequest) {
}
}
export async function POST(_req: NextRequest) {
export async function POST(req: NextRequest) {
const auth = await requireAdminApiSession(req);
if (!auth.valid) {
return auth.response;
}
try {
await SettingsManager.getSettingsAsync();
// 1. Flush local / shared Next.js DataBufferEngine
const localRes = await DataBufferEngine.flush();
const localRes = await DataBufferEngine.flush({
waitMs: 2500,
throwOnLockTimeout: true,
});
// 2. Also flush Port 4000 Socket.IO ingestion buffer
let remoteCount = 0;
@@ -46,9 +74,13 @@ export async function POST(_req: NextRequest) {
try {
const ingestRes = await fetch(`${ingestionBaseUrl}/internal/flush`, {
method: 'POST',
headers: getInternalIngestionHeaders(),
headers: {
'Content-Type': 'application/json',
...getInternalIngestionHeaders(),
},
body: JSON.stringify({ waitMs: 2500, throwOnLockTimeout: true }),
cache: 'no-store',
signal: AbortSignal.timeout(2500),
signal: AbortSignal.timeout(4000),
});
if (!ingestRes.ok) {
const errPayload = await ingestRes.json().catch(() => ({}));
@@ -56,6 +88,23 @@ export async function POST(_req: NextRequest) {
typeof errPayload?.error === 'string'
? errPayload.error
: `Ingestion flush returned HTTP ${ingestRes.status}`;
if (
ingestRes.status === 503 &&
errPayload?.code === 'FLUSH_LOCK_TIMEOUT'
) {
return NextResponse.json(
{
success: false,
error: remoteErr,
code: 'FLUSH_LOCK_TIMEOUT',
flushedCount: localRes.count,
filesUpdated: localRes.files,
ingestionFlushed: false,
ingestionStatus: ingestRes.status,
},
{ status: 503 },
);
}
return NextResponse.json(
{
success: false,
@@ -108,6 +157,16 @@ export async function POST(_req: NextRequest) {
});
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Manual flush failed';
if (message.includes('FLUSH_LOCK_TIMEOUT')) {
return NextResponse.json(
{
success: false,
error: message,
code: 'FLUSH_LOCK_TIMEOUT',
},
{ status: 503 },
);
}
return NextResponse.json({ error: message }, { status: 500 });
}
}
+6 -1
View File
@@ -38,9 +38,14 @@ export const metadata: Metadata = {
},
};
const THEME_BOOTSTRAP_SCRIPT = `(function(){try{var t=localStorage.getItem('datastack-theme');var r=document.documentElement;if(t==='light'){r.classList.remove('dark');r.classList.add('light');}else{r.classList.remove('light');r.classList.add('dark');}}catch(e){}})();`;
export default function RootLayout({ children }: { children: React.ReactNode }) {
return (
<html lang="en" className="dark">
<html lang="en" className="dark" suppressHydrationWarning>
<head>
<script dangerouslySetInnerHTML={{ __html: THEME_BOOTSTRAP_SCRIPT }} />
</head>
<body suppressHydrationWarning className="min-h-screen bg-[var(--bg-main)] text-[var(--text)] antialiased">
<ToastProvider>
{children}
+46 -8
View File
@@ -1,12 +1,50 @@
'use client';
import { MainLayout } from '@/components/layout/MainLayout';
import { usePathname } from 'next/navigation';
import { VconShell } from '@/components/layout/VconShell';
import { verifyRequestAdminSession } from '@/core/auth/adminUsers';
import { cookies, headers } from 'next/headers';
import { redirect } from 'next/navigation';
import React from 'react';
export default function VconLayout({ children }: { children: React.ReactNode }) {
const pathname = usePathname();
if (pathname === '/vcon/login' || pathname === '/vcon/setup') return <>{children}</>;
export const dynamic = 'force-dynamic';
return <MainLayout showSidebar={true}>{children}</MainLayout>;
export default async function VconLayout({
children,
}: {
children: React.ReactNode;
}) {
const headerList = await headers();
const pathname = headerList.get('x-datastack-pathname') || '';
if (pathname === '/vcon/login' || pathname === '/vcon/setup') {
return <>{children}</>;
}
const cookieStore = await cookies();
const token = cookieStore.get('datastack_admin_session')?.value;
const auth = await verifyRequestAdminSession(token);
if (!auth.valid) {
if (auth.reason === 'UNAUTHORIZED') {
const loginTarget = pathname
? `/vcon/login?next=${encodeURIComponent(pathname)}`
: '/vcon/login';
redirect(loginTarget);
}
return (
<main className="grid min-h-dvh place-items-center bg-[var(--bg)] p-4 text-[var(--text)]">
<section className="w-full max-w-sm border border-[var(--border)] bg-[var(--surface)] p-5">
<h1 className="text-[13px] font-medium text-[var(--text)]">
Console authentication unavailable
</h1>
<p className="mt-2 text-[12px] text-[var(--text-muted)]">
{auth.reason === 'UNCONFIGURED'
? 'Admin session secret is not configured.'
: 'Database authentication check is temporarily unavailable. Please retry shortly.'}
</p>
</section>
</main>
);
}
return <VconShell>{children}</VconShell>;
}
+32 -4
View File
@@ -1,18 +1,46 @@
import { AdminLogin } from '@/components/sections/AdminLogin';
import { isAdminSetupComplete } from '@/core/auth/adminUsers';
import {
isAdminSetupComplete,
verifyRequestAdminSession,
} from '@/core/auth/adminUsers';
import { cookies } from 'next/headers';
import { redirect } from 'next/navigation';
export const runtime = 'nodejs';
export const dynamic = 'force-dynamic';
export default async function LoginPage() {
const isConfigured = await isAdminSetupComplete().catch(() => false);
if (!isConfigured) redirect('/vcon/setup');
let databaseReady = true;
try {
const isConfigured = await isAdminSetupComplete();
if (!isConfigured) redirect('/vcon/setup');
} catch (error) {
if (
error &&
typeof error === 'object' &&
'digest' in error &&
String((error as { digest?: unknown }).digest).startsWith('NEXT_REDIRECT')
) {
throw error;
}
databaseReady = false;
}
if (databaseReady) {
const cookieStore = await cookies();
const token = cookieStore.get('datastack_admin_session')?.value;
if (token) {
const auth = await verifyRequestAdminSession(token);
if (auth.valid) {
redirect('/vcon/dashboard');
}
}
}
const showDemoHint =
!process.env.DATABASE_URL &&
process.env.NODE_ENV !== 'production' &&
process.env.DATASTACK_ALLOW_DEMO_ADMIN !== 'false';
return <AdminLogin showDemoHint={showDemoHint} />;
return <AdminLogin showDemoHint={showDemoHint} databaseReady={databaseReady} />;
}
+62 -29
View File
@@ -9,6 +9,7 @@ import { Panel } from '@/components/ui/Panel';
import { StatStrip } from '@/components/ui/StatStrip';
import { useToast } from '@/components/ui/Toast';
import { getCached, setCached } from '@/lib/vconCache';
import { resolvePublicEndpoints } from '@/lib/vconUiHelpers';
import { useEffect, useState } from 'react';
interface SdkClientItem {
@@ -34,23 +35,23 @@ export default function SdkGuidePage() {
const [apiUrl, setApiUrl] = useState<string>('http://localhost:3000');
useEffect(() => {
const isLocalDev = window.location.port === '3000';
setServerUrl(
isLocalDev
? `${window.location.protocol}//${window.location.hostname}:4000`
: window.location.origin
);
setApiUrl(window.location.origin);
const initialPort = getCached<number>('vcon:server-port', 4000);
const initialEndpoints = resolvePublicEndpoints(window.location, initialPort);
setServerUrl(initialEndpoints.serverUrl);
setApiUrl(initialEndpoints.apiUrl);
async function loadClients() {
async function loadSdkContext() {
try {
const res = await fetch('/api/clients', { cache: 'no-store' });
if (res.status === 401) {
const [clientsRes, metricsRes] = await Promise.all([
fetch('/api/clients', { cache: 'no-store' }),
fetch('/api/metrics', { cache: 'no-store' }).catch(() => null),
]);
if (clientsRes.status === 401 || metricsRes?.status === 401) {
window.location.assign('/vcon/login');
return;
}
if (res.ok) {
const data = await res.json();
if (clientsRes.ok) {
const data = await clientsRes.json();
const list = setCached<SdkClientItem[]>('vcon:clients', data.clients || []);
setClients(list);
if (list.length > 0) {
@@ -61,11 +62,21 @@ export default function SdkGuidePage() {
);
}
}
if (metricsRes && metricsRes.ok) {
const metricsData = await metricsRes.json().catch(() => null);
const runtimePort = Number(metricsData?.runtime?.serverPort);
if (Number.isFinite(runtimePort) && runtimePort > 0) {
setCached('vcon:server-port', runtimePort);
const resolved = resolvePublicEndpoints(window.location, runtimePort);
setServerUrl(resolved.serverUrl);
setApiUrl(resolved.apiUrl);
}
}
} catch {
// Retain cached list on transient network error
}
}
void loadClients();
void loadSdkContext();
}, []);
useEffect(() => {
@@ -75,6 +86,7 @@ export default function SdkGuidePage() {
}
let cancelled = false;
setSelectedClientToken('');
setLoadingToken(true);
async function fetchToken() {
@@ -110,8 +122,12 @@ export default function SdkGuidePage() {
const activeClient =
clients.find((c) => c.clientId === selectedClientId) || clients[0];
const resolvedToken = selectedClientToken || activeClient?.token || '';
const sampleToken = resolvedToken || 'YOUR_SIGNED_CLIENT_JWT_TOKEN';
const resolvedToken = loadingToken
? ''
: selectedClientToken || activeClient?.token || '';
const sampleToken = loadingToken
? 'LOADING_CLIENT_JWT_TOKEN...'
: resolvedToken || 'YOUR_SIGNED_CLIENT_JWT_TOKEN';
const configSnippet = `# config.datastack.py — Place in your Python project root
DATASTACK_CONFIG = {
@@ -194,6 +210,7 @@ client.disconnect()`;
size="xs"
variant="secondary"
icon="FileCode"
disabled={loadingToken}
onClick={() => void copySnippet(configSnippet, 'config.datastack.py')}
>
Copy Config
@@ -216,27 +233,42 @@ client.disconnect()`;
{ label: 'Server endpoint', value: serverUrl },
{
label: 'Active client',
value: activeClient?.clientId || 'None selected',
value: activeClient
? activeClient.status === 'revoked'
? `${activeClient.clientId} (revoked)`
: activeClient.clientId
: 'None selected',
dot: activeClient?.status === 'revoked' ? 'off' : activeClient ? 'ok' : undefined,
},
]}
/>
<Card
compact
className="flex w-full items-center justify-between bg-[var(--surface)] px-3 py-2 font-mono text-[11.5px]"
className="flex w-full flex-col gap-1.5 bg-[var(--surface)] px-3 py-2.5"
>
<span className="text-[var(--accent)]">
pip install ./sdk/python/datastack
</span>
<Button
size="xs"
variant="ghost"
icon="Copy"
aria-label="Copy pip install command"
onClick={() =>
void copySnippet('pip install ./sdk/python/datastack', 'Pip command')
}
/>
<div className="flex items-center justify-between gap-2 font-mono text-[11.5px]">
<span className="truncate text-[var(--accent)]">
pip install /path/to/datastack/sdk/python/datastack
</span>
<Button
size="xs"
variant="ghost"
icon="Copy"
aria-label="Copy pip install command"
onClick={() =>
void copySnippet(
'pip install /path/to/datastack/sdk/python/datastack',
'Pip command'
)
}
/>
</div>
<p className="text-[11px] text-[var(--text-faint)]">
Self-hosted package bundled in <code className="font-mono text-[var(--text-muted)]">sdk/python/datastack</code>. From inside the repo run{' '}
<code className="font-mono text-[var(--text-muted)]">pip install ./sdk/python/datastack</code>, or build a portable wheel with{' '}
<code className="font-mono text-[var(--text-muted)]">pip wheel ./sdk/python/datastack -w dist/</code> for external projects.
</p>
</Card>
<Panel
@@ -265,6 +297,7 @@ client.disconnect()`;
variant="ghost"
icon="Copy"
aria-label="Copy config.datastack.py"
disabled={loadingToken}
onClick={() => void copySnippet(configSnippet, 'config.datastack.py')}
/>
</div>
+22 -4
View File
@@ -1,5 +1,8 @@
import { AdminSetup } from '@/components/sections/AdminSetup';
import { isAdminSetupComplete } from '@/core/auth/adminUsers';
import {
isAdminSetupComplete,
requiresPersistentAdminDatabase,
} from '@/core/auth/adminUsers';
import { redirect } from 'next/navigation';
export const runtime = 'nodejs';
@@ -7,15 +10,30 @@ export const dynamic = 'force-dynamic';
export default async function SetupPage() {
let databaseReady = true;
try {
if (await isAdminSetupComplete()) redirect('/vcon/login');
} catch {
let databaseError = '';
let setupComplete = false;
if (requiresPersistentAdminDatabase()) {
databaseReady = false;
databaseError =
'PostgreSQL DATABASE_URL is required in production before creating an administrator account.';
} else {
try {
setupComplete = await isAdminSetupComplete();
} catch {
databaseReady = false;
databaseError = 'Database connection unavailable.';
}
}
if (databaseReady && setupComplete) {
redirect('/vcon/login');
}
return (
<AdminSetup
databaseReady={databaseReady}
databaseError={databaseError}
initialEmail={process.env.DATASTACK_ADMIN_EMAIL?.trim().toLowerCase() ?? ''}
/>
);
+62 -56
View File
@@ -9,6 +9,10 @@ import { Pill } from '@/components/ui/Pill';
import { StatStrip } from '@/components/ui/StatStrip';
import { useToast } from '@/components/ui/Toast';
import { getCached, setCached } from '@/lib/vconCache';
import {
resolvePublicEndpoints,
summarizeHttpBatchResult,
} from '@/lib/vconUiHelpers';
import Link from 'next/link';
import { useEffect, useState } from 'react';
import { io } from 'socket.io-client';
@@ -22,11 +26,8 @@ interface SimulatorClient {
type TransportMode = 'http' | 'websocket';
type DispatchStatus = 'Ready' | 'Accepted' | 'Rejected' | 'Error';
function getSocketOrigin(): string {
const isLocalDev = window.location.port === '3000';
const protocol = window.location.protocol === 'https:' ? 'https:' : 'http:';
const host = isLocalDev ? `${window.location.hostname}:4000` : window.location.host;
return `${protocol}//${host}`;
function getSocketOrigin(serverPort: number): string {
return resolvePublicEndpoints(window.location, serverPort).serverUrl;
}
export default function SimulatorPage() {
@@ -52,20 +53,26 @@ export default function SimulatorPage() {
2
)
);
const [serverPort, setServerPort] = useState<number>(() =>
getCached<number>('vcon:server-port', 4000)
);
const [sending, setSending] = useState(false);
const [dispatchStatus, setDispatchStatus] = useState<DispatchStatus>('Ready');
const [lastResponse, setLastResponse] = useState<Record<string, unknown> | null>(null);
useEffect(() => {
async function loadClients() {
async function loadSimulatorContext() {
try {
const res = await fetch('/api/clients', { cache: 'no-store' });
if (res.status === 401) {
const [clientsRes, metricsRes] = await Promise.all([
fetch('/api/clients', { cache: 'no-store' }),
fetch('/api/metrics', { cache: 'no-store' }).catch(() => null),
]);
if (clientsRes.status === 401 || metricsRes?.status === 401) {
window.location.assign('/vcon/login');
return;
}
if (res.ok) {
const data = await res.json();
if (clientsRes.ok) {
const data = await clientsRes.json();
const list = setCached<SimulatorClient[]>('vcon:clients', data.clients || []);
setClients(list);
setSelectedClient((prev) => {
@@ -73,11 +80,18 @@ export default function SimulatorPage() {
return list.find((c) => c.status === 'active')?.clientId ?? list[0]?.clientId ?? '';
});
}
if (metricsRes && metricsRes.ok) {
const metricsData = await metricsRes.json().catch(() => null);
const runtimePort = Number(metricsData?.runtime?.serverPort);
if (Number.isFinite(runtimePort) && runtimePort > 0) {
setServerPort(setCached('vcon:server-port', runtimePort));
}
}
} catch {
// Ignore transient failure and retain cached list
}
}
void loadClients();
void loadSimulatorContext();
}, []);
const fetchClientJwt = async (clientId: string): Promise<string> => {
@@ -100,7 +114,7 @@ export default function SimulatorPage() {
items: Array<Record<string, unknown>>
): Promise<{ accepted: number; rejected: number; responses: Record<string, unknown>[] }> => {
const token = await fetchClientJwt(clientId);
const socketUrl = getSocketOrigin();
const socketUrl = getSocketOrigin(serverPort);
return new Promise((resolve, reject) => {
const socket = io(socketUrl, {
@@ -112,7 +126,7 @@ export default function SimulatorPage() {
const timer = window.setTimeout(() => {
socket.disconnect();
reject(new Error('WebSocket timeout (is Port 4000 ingestion server running?)'));
reject(new Error(`WebSocket timeout (is Port ${serverPort} ingestion server running?)`));
}, 5000);
socket.on('connect_error', (connectErr) => {
@@ -271,55 +285,47 @@ export default function SimulatorPage() {
return;
}
let accepted = 0;
let rejected = 0;
const errors: string[] = [];
for (const item of items) {
const res = await fetch('/api/events', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
clientId: selectedClient,
data: item,
}),
});
if (res.status === 401) {
window.location.assign('/vcon/login');
return;
}
if (res.ok) {
accepted++;
} else {
rejected++;
try {
const errJson = await res.json();
if (errJson?.error && errors.length < 3) {
errors.push(String(errJson.error));
}
} catch {
// ignore non-json error body
}
}
const batchPayload = items.map((item, idx) => ({
eventId: `sim_http_${Date.now()}_${idx}`,
timestamp: item.timestamp,
data: item,
}));
const res = await fetch('/api/events', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
clientId: selectedClient,
batch: batchPayload,
}),
});
if (res.status === 401) {
window.location.assign('/vcon/login');
return;
}
const data = await res.json().catch(() => ({}));
const summary = summarizeHttpBatchResult(
res.ok,
data && typeof data === 'object' ? (data as Record<string, unknown>) : null,
count,
);
setLastResponse({
transport: 'http',
status: res.status,
batchSent: count,
accepted,
rejected,
...(errors.length > 0 ? { errors } : {}),
accepted: summary.accepted,
rejected: summary.rejected,
duplicates: summary.duplicates,
...(data && typeof data === 'object' ? data : {}),
});
if (accepted === 0) {
setDispatchStatus('Rejected');
error('Batch Rejected', `0/${count} ingested${errors[0] ? ` (${errors[0]})` : ''}`);
} else if (accepted < count) {
setDispatchStatus('Rejected');
warning('Batch Partial', `${accepted}/${count} ingested (${rejected} rejected)`);
setDispatchStatus(summary.status);
if (summary.level === 'error') {
error(summary.title, summary.message);
} else if (summary.level === 'warning') {
warning(summary.title, summary.message);
} else {
setDispatchStatus('Accepted');
success('Batch Streamed', `${accepted}/${count} ingested`);
success(summary.title, summary.message);
}
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Batch dispatch error';
@@ -379,7 +385,7 @@ export default function SimulatorPage() {
},
{
label: 'Ingest transport',
value: transport === 'http' ? 'POST /api/events' : 'WS Port 4000',
value: transport === 'http' ? 'POST /api/events' : `WS Port ${serverPort}`,
},
{ label: 'Payload format', value: 'JSON' },
{
@@ -412,7 +418,7 @@ export default function SimulatorPage() {
active={transport === 'websocket'}
onClick={() => setTransport('websocket')}
>
WebSocket Port 4000
WebSocket Port {serverPort}
</Pill>
</div>
</div>
+71 -24
View File
@@ -18,52 +18,96 @@ import { useToast } from '@/components/ui/Toast';
import { getCached, setCached } from '@/lib/vconCache';
import { useCallback, useEffect, useState } from 'react';
interface StorageStatsSummary {
totalMB?: string;
totalFiles?: number;
totalRows?: number;
rotationRule?: string;
rotationMb?: number;
}
interface StoragePartitionFile {
clientId: string;
filename: string;
sizeBytes: number;
linesCount: number;
modifiedAt: string;
}
export default function StoragePage() {
const { success, warning, error } = useToast();
const [stats, setStats] = useState<any>(() =>
getCached<any>('vcon:storage-stats', null)
const [stats, setStats] = useState<StorageStatsSummary | null>(() =>
getCached<StorageStatsSummary | null>('vcon:storage-stats', null)
);
const [files, setFiles] = useState<any[]>(() =>
getCached<any[]>('vcon:storage-files', [])
const [files, setFiles] = useState<StoragePartitionFile[]>(() =>
getCached<StoragePartitionFile[]>('vcon:storage-files', [])
);
const [refreshing, setRefreshing] = useState(false);
const [flushing, setFlushing] = useState(false);
const fetchStorageData = useCallback(async () => {
setRefreshing(true);
try {
const res = await fetch('/api/storage');
if (!res.ok) throw new Error('Unable to load storage partitions');
const data = await res.json();
setStats(setCached('vcon:storage-stats', data.stats));
const res = await fetch('/api/storage', { cache: 'no-store' });
if (res.status === 401) {
window.location.assign('/vcon/login');
return;
}
const data = await res.json().catch(() => ({}));
if (!res.ok) {
throw new Error(data.error || 'Unable to load storage partitions');
}
setStats(setCached('vcon:storage-stats', data.stats ?? null));
setFiles(setCached('vcon:storage-files', data.files || []));
} catch {
error('Storage load failed', 'Could not load storage partitions.');
} catch (loadErr) {
error(
'Storage load failed',
loadErr instanceof Error
? loadErr.message
: 'Could not load storage partitions.'
);
} finally {
setRefreshing(false);
}
}, [error]);
useEffect(() => {
fetchStorageData();
void fetchStorageData();
}, [fetchStorageData]);
const handleManualFlush = async () => {
setFlushing(true);
try {
const res = await fetch('/api/storage', { method: 'POST' });
const res = await fetch('/api/storage', {
method: 'POST',
cache: 'no-store',
});
if (res.status === 401) {
window.location.assign('/vcon/login');
return;
}
const data = await res.json().catch(() => ({}));
if (res.ok && data.success) {
const count = Number(data.flushedCount) || 0;
success(
'Buffer Flushed',
`Flushed ${count} event${count === 1 ? '' : 's'} to CSV${data.warning ? ` (${data.warning})` : ''}`
);
fetchStorageData();
} else if (data.flushedCount !== undefined && data.flushedCount > 0) {
// Partial flush: local events persisted, remote Port 4000 flush returned error
if (data.warning) {
warning(
'Partial Flush (Web Only)',
`Flushed ${count} event${count === 1 ? '' : 's'}. ${String(data.warning)}`
);
} else {
success(
'Buffer Flushed',
`Flushed ${count} event${count === 1 ? '' : 's'} to CSV.`
);
}
await fetchStorageData();
} else if (data.flushedCount !== undefined && Number(data.flushedCount) > 0) {
warning(
'Partial Flush',
`Flushed ${data.flushedCount} local events. Ingestion server: ${data.error || 'flush failed'}`
);
fetchStorageData();
await fetchStorageData();
} else {
error('Flush Failed', data.error || 'Could not flush buffer');
}
@@ -84,7 +128,9 @@ export default function StoragePage() {
size="xs"
variant="secondary"
icon="RotateCw"
onClick={fetchStorageData}
loading={refreshing}
disabled={refreshing || flushing}
onClick={() => void fetchStorageData()}
>
Refresh
</Button>
@@ -93,7 +139,8 @@ export default function StoragePage() {
variant="primary"
icon="HardDrive"
loading={flushing}
onClick={handleManualFlush}
disabled={flushing}
onClick={() => void handleManualFlush()}
>
Flush Buffer
</Button>
@@ -131,8 +178,8 @@ export default function StoragePage() {
{files.length === 0 ? (
<TableEmpty colSpan={5} message="No CSV files" />
) : (
files.map((file, idx) => (
<TableRow key={idx}>
files.map((file) => (
<TableRow key={`${file.clientId}/${file.filename}`}>
<TableCell className="font-mono text-[var(--accent)]">
{file.clientId}
</TableCell>