refactor(auth): standardize session management and security

- Centralize secret retrieval using `getAdminSessionSecret`
- Replace manual session parsing with `verifyRequestAdminSession` utility
- Tighten API validation and improve security headers handling
- Enforce environment variable security for production deployments
This commit is contained in:
Team Dark Devil committed 2026-11-18 05:18:55 +06:00
1 parent 34519c07df
commit 177ed4a844
78 files changed
+6748 -3157

No files matched your search

+17 -10
View File
@@ -1,16 +1,23 @@
# DataStack Environment Variables Configuration # DataStack Environment Variables Configuration
# NOTE: In production (NODE_ENV=production), replace all placeholder values below with strong 32+ char secrets.
# Administrator Account (defaults to admin@datastack.dev / admin12345678 if omitted) # Security & Session Secrets (REQUIRED in production, min 32 characters)
DATASTACK_ADMIN_EMAIL=admin@datastack.dev DATASTACK_ADMIN_SESSION_SECRET=replace-with-32-plus-char-admin-session-secret
DATASTACK_ADMIN_PASSWORD=admin12345678 DATASTACK_JWT_SECRET=replace-with-32-plus-char-client-jwt-secret
DATASTACK_ADMIN_SETUP_KEY=replace-with-32-plus-char-admin-setup-key
DATASTACK_INTERNAL_SECRET=replace-with-32-plus-char-internal-service-secret
# Security & Session Secrets (min 32 characters) # Administrator Account (development/in-memory bootstrap; production uses /vcon/setup)
DATASTACK_ADMIN_SESSION_SECRET=datastack-admin-session-default-secret-32-chars DATASTACK_ADMIN_EMAIL=admin@example.com
DATASTACK_JWT_SECRET=datastack-jwt-default-secret-key-32-chars-long # DATASTACK_ADMIN_PASSWORD=replace-with-strong-password
DATASTACK_ADMIN_SETUP_KEY=datastack-default-setup-key-32-chars # DATASTACK_ALLOW_DEMO_ADMIN=false
# PostgreSQL Database (optional - in-memory fallback active when not configured) # PostgreSQL Database (recommended for production persistence)
# DATABASE_URL=postgresql://datastack:password@localhost:5432/datastack_db # DATABASE_URL=postgresql://datastack:password@localhost:5432/datastack_db
# Ingestion Engine Port (defaults to 4000) # Shared Redis Buffer (optional; enables cross-process buffering & distributed flush locks)
PORT_SOCKET=4000 # REDIS_URL=redis://127.0.0.1:6379
# Service Networking
PORT_SOCKET=4000
INGESTION_INTERNAL_URL=http://127.0.0.1:4000
+2 -4
View File
@@ -9,10 +9,8 @@ import { redirect } from 'next/navigation';
export const dynamic = 'force-dynamic'; export const dynamic = 'force-dynamic';
export default async function LandingPage() { export default async function LandingPage() {
if (process.env.DATABASE_URL) { const isConfigured = await isAdminSetupComplete().catch(() => false);
const isConfigured = await isAdminSetupComplete().catch(() => false); if (!isConfigured) redirect('/vcon/setup');
if (!isConfigured) redirect('/vcon/setup');
}
return ( return (
<MainLayout> <MainLayout>
+102 -79
View File
@@ -1,11 +1,12 @@
import { import {
createAdminSessionToken, createAdminSessionToken,
getAdminSessionSecret,
getSessionCookieOptions, getSessionCookieOptions,
} from "@/core/auth/adminSession"; } from "@/core/auth/adminSession";
import { import {
authenticateAdminCredentials,
clearAdminLoginAttempts, clearAdminLoginAttempts,
consumeAdminLoginAttempt, consumeAdminLoginAttempt,
verifyAdminCredentials,
} from "@/core/auth/adminUsers"; } from "@/core/auth/adminUsers";
import { NextRequest, NextResponse } from "next/server"; import { NextRequest, NextResponse } from "next/server";
@@ -13,10 +14,19 @@ const SESSION_COOKIE = "datastack_admin_session";
const SESSION_SECONDS = 8 * 60 * 60; const SESSION_SECONDS = 8 * 60 * 60;
const REMEMBERED_SESSION_SECONDS = 30 * 24 * 60 * 60; const REMEMBERED_SESSION_SECONDS = 30 * 24 * 60 * 60;
function extractClientIp(request: NextRequest) {
const forwardedFor = request.headers.get("x-forwarded-for");
if (forwardedFor) {
const first = forwardedFor.split(",")[0]?.trim();
if (first) return first;
}
const realIp = request.headers.get("x-real-ip")?.trim();
if (realIp) return realIp;
return "unknown";
}
export async function POST(request: NextRequest) { export async function POST(request: NextRequest) {
const sessionSecret = const sessionSecret = getAdminSessionSecret();
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
"datastack-admin-session-default-secret-32-chars";
if (!sessionSecret || sessionSecret.length < 32) { if (!sessionSecret || sessionSecret.length < 32) {
return NextResponse.json( return NextResponse.json(
@@ -27,86 +37,99 @@ export async function POST(request: NextRequest) {
); );
} }
let body: Record<string, unknown>;
try { try {
const body = await request.json(); const parsed = await request.json();
const email = body =
typeof body.email === "string" ? body.email.trim().toLowerCase() : ""; parsed && typeof parsed === "object" && !Array.isArray(parsed)
const password = typeof body.password === "string" ? body.password : ""; ? (parsed as Record<string, unknown>)
if (!email || password.length === 0 || password.length > 1024) { : {};
return NextResponse.json(
{ error: "Invalid email or password." },
{ status: 401 },
);
}
let throttleDecision;
try {
throttleDecision = await consumeAdminLoginAttempt(email, sessionSecret);
} catch {
return NextResponse.json(
{ error: "Admin authentication is temporarily unavailable." },
{ status: 503 },
);
}
if (!throttleDecision.allowed) {
return NextResponse.json(
{ error: "Too many sign-in attempts. Try again later." },
{
status: 429,
headers: {
"Retry-After": String(throttleDecision.retryAfterSeconds),
},
},
);
}
let authenticated = false;
try {
authenticated = await verifyAdminCredentials(email, password);
} catch {
return NextResponse.json(
{ error: "Admin authentication is temporarily unavailable." },
{ status: 503 },
);
}
if (!authenticated) {
return NextResponse.json(
{ error: "Invalid email or password." },
{ status: 401 },
);
}
try {
await clearAdminLoginAttempts(email, sessionSecret);
} catch {
return NextResponse.json(
{ error: "Admin authentication is temporarily unavailable." },
{ status: 503 },
);
}
const remember = body.remember === true;
const maxAge = remember ? REMEMBERED_SESSION_SECONDS : SESSION_SECONDS;
const expiresAt = Date.now() + maxAge * 1000;
const token = await createAdminSessionToken(
email,
sessionSecret,
expiresAt,
);
const response = NextResponse.json({ success: true });
response.cookies.set(
SESSION_COOKIE,
token,
getSessionCookieOptions(request, maxAge),
);
return response;
} catch { } catch {
return NextResponse.json( return NextResponse.json(
{ error: "Invalid login request." }, { error: "Invalid login request." },
{ status: 400 }, { status: 400 },
); );
} }
const email =
typeof body.email === "string" ? body.email.trim().toLowerCase() : "";
const password = typeof body.password === "string" ? body.password : "";
if (!email || password.length === 0 || password.length > 1024) {
return NextResponse.json(
{ error: "Invalid email or password." },
{ status: 401 },
);
}
const clientIp = extractClientIp(request);
let throttleDecision;
try {
throttleDecision = await consumeAdminLoginAttempt(
email,
sessionSecret,
clientIp,
);
} catch {
return NextResponse.json(
{ error: "Admin authentication is temporarily unavailable." },
{ status: 503 },
);
}
if (!throttleDecision.allowed) {
return NextResponse.json(
{ error: "Too many sign-in attempts. Try again later." },
{
status: 429,
headers: {
"Retry-After": String(throttleDecision.retryAfterSeconds),
},
},
);
}
let authResult: { authenticated: boolean; tokenVersion: number };
try {
authResult = await authenticateAdminCredentials(email, password);
} catch {
return NextResponse.json(
{ error: "Admin authentication is temporarily unavailable." },
{ status: 503 },
);
}
if (!authResult.authenticated) {
return NextResponse.json(
{ error: "Invalid email or password." },
{ status: 401 },
);
}
try {
await clearAdminLoginAttempts(email, sessionSecret, clientIp);
} catch {
return NextResponse.json(
{ error: "Admin authentication is temporarily unavailable." },
{ status: 503 },
);
}
const remember = body.remember === true;
const maxAge = remember ? REMEMBERED_SESSION_SECONDS : SESSION_SECONDS;
const expiresAt = Date.now() + maxAge * 1000;
const token = await createAdminSessionToken(
email,
sessionSecret,
expiresAt,
authResult.tokenVersion,
);
const response = NextResponse.json({ success: true });
response.cookies.set(
SESSION_COOKIE,
token,
getSessionCookieOptions(request, maxAge),
);
return response;
} }
+54 -35
View File
@@ -1,11 +1,11 @@
import { import {
createAdminSessionToken, createAdminSessionToken,
getAdminSessionPayload,
getSessionCookieOptions, getSessionCookieOptions,
} from "@/core/auth/adminSession"; } from "@/core/auth/adminSession";
import { import {
getAdminAccountInfo, getAdminAccountInfo,
updateAdminAccount, updateAdminAccount,
verifyRequestAdminSession,
} from "@/core/auth/adminUsers"; } from "@/core/auth/adminUsers";
import { NextRequest, NextResponse } from "next/server"; import { NextRequest, NextResponse } from "next/server";
@@ -17,51 +17,69 @@ function validEmail(email: string) {
} }
export async function GET(request: NextRequest) { export async function GET(request: NextRequest) {
const sessionSecret = const token = request.cookies.get(SESSION_COOKIE)?.value;
process.env.DATASTACK_ADMIN_SESSION_SECRET || const auth = await verifyRequestAdminSession(token);
"datastack-admin-session-default-secret-32-chars";
const token = request.cookies.get(SESSION_COOKIE)?.value || "";
const session = await getAdminSessionPayload(token, sessionSecret);
if (!session) { if (!auth.valid) {
return NextResponse.json( const status = auth.reason === "UNAUTHORIZED" ? 401 : 503;
{ error: "Authentication required." }, const message =
{ status: 401 }, auth.reason === "UNAUTHORIZED"
); ? "Authentication required."
: "Admin authentication is temporarily unavailable.";
return NextResponse.json({ error: message }, { status });
} }
const account = await getAdminAccountInfo(session.email); try {
return NextResponse.json({ const account = await getAdminAccountInfo(auth.session.email);
profile: { return NextResponse.json({
email: account.email, profile: {
role: "Owner", email: account.email,
storageMode: account.storageMode, role: "Owner",
createdAt: account.createdAt, storageMode: account.storageMode,
sessionExpiresAt: new Date(session.expiresAt).toISOString(), createdAt: account.createdAt,
}, sessionExpiresAt: new Date(auth.session.expiresAt).toISOString(),
}); },
});
} catch {
return NextResponse.json(
{ error: "Unable to load admin profile." },
{ status: 503 },
);
}
} }
export async function PUT(request: NextRequest) { export async function PUT(request: NextRequest) {
const sessionSecret = const token = request.cookies.get(SESSION_COOKIE)?.value;
process.env.DATASTACK_ADMIN_SESSION_SECRET || const auth = await verifyRequestAdminSession(token);
"datastack-admin-session-default-secret-32-chars";
const token = request.cookies.get(SESSION_COOKIE)?.value || "";
const session = await getAdminSessionPayload(token, sessionSecret);
if (!session) { if (!auth.valid) {
const status = auth.reason === "UNAUTHORIZED" ? 401 : 503;
const message =
auth.reason === "UNAUTHORIZED"
? "Authentication required."
: "Admin authentication is temporarily unavailable.";
return NextResponse.json({ error: message }, { status });
}
let body: Record<string, unknown>;
try {
const parsed = await request.json();
body =
parsed && typeof parsed === "object" && !Array.isArray(parsed)
? (parsed as Record<string, unknown>)
: {};
} catch {
return NextResponse.json( return NextResponse.json(
{ error: "Authentication required." }, { error: "Invalid profile payload." },
{ status: 401 }, { status: 400 },
); );
} }
try { try {
const body = await request.json();
const nextEmail = const nextEmail =
typeof body.email === "string" typeof body.email === "string"
? body.email.trim().toLowerCase() ? body.email.trim().toLowerCase()
: session.email; : auth.session.email;
const currentPassword = const currentPassword =
typeof body.currentPassword === "string" ? body.currentPassword : ""; typeof body.currentPassword === "string" ? body.currentPassword : "";
const newPassword = const newPassword =
@@ -88,7 +106,7 @@ export async function PUT(request: NextRequest) {
); );
} }
const account = await getAdminAccountInfo(session.email); const account = await getAdminAccountInfo(auth.session.email);
const result = await updateAdminAccount({ const result = await updateAdminAccount({
currentEmail: account.email, currentEmail: account.email,
currentPassword, currentPassword,
@@ -103,13 +121,14 @@ export async function PUT(request: NextRequest) {
// Preserve remaining session duration (e.g., 30-day "Remember me" session) // Preserve remaining session duration (e.g., 30-day "Remember me" session)
const remainingSeconds = Math.max( const remainingSeconds = Math.max(
SESSION_SECONDS, SESSION_SECONDS,
Math.ceil((session.expiresAt - Date.now()) / 1000), Math.ceil((auth.session.expiresAt - Date.now()) / 1000),
); );
const expiresAt = Date.now() + remainingSeconds * 1000; const expiresAt = Date.now() + remainingSeconds * 1000;
const nextToken = await createAdminSessionToken( const nextToken = await createAdminSessionToken(
result.email, result.email,
sessionSecret, auth.sessionSecret,
expiresAt, expiresAt,
result.tokenVersion,
); );
const response = NextResponse.json({ const response = NextResponse.json({
@@ -133,7 +152,7 @@ export async function PUT(request: NextRequest) {
} catch { } catch {
return NextResponse.json( return NextResponse.json(
{ error: "Unable to update profile." }, { error: "Unable to update profile." },
{ status: 400 }, { status: 500 },
); );
} }
} }
+17 -4
View File
@@ -1,5 +1,6 @@
import { import {
createAdminSessionToken, createAdminSessionToken,
getAdminSessionSecret,
getSessionCookieOptions, getSessionCookieOptions,
} from "@/core/auth/adminSession"; } from "@/core/auth/adminSession";
import { import {
@@ -20,9 +21,7 @@ function validEmail(email: string) {
} }
export async function POST(request: NextRequest) { export async function POST(request: NextRequest) {
const sessionSecret = const sessionSecret = getAdminSessionSecret();
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
"datastack-admin-session-default-secret-32-chars";
if (!sessionSecret || sessionSecret.length < 32) { if (!sessionSecret || sessionSecret.length < 32) {
return NextResponse.json( return NextResponse.json(
{ error: "Admin session configuration is unavailable." }, { error: "Admin session configuration is unavailable." },
@@ -30,8 +29,21 @@ export async function POST(request: NextRequest) {
); );
} }
let body: Record<string, unknown>;
try {
const parsed = await request.json();
body =
parsed && typeof parsed === "object" && !Array.isArray(parsed)
? (parsed as Record<string, unknown>)
: {};
} catch {
return NextResponse.json(
{ error: "Invalid setup request payload." },
{ status: 400 },
);
}
try { try {
const body = await request.json();
const email = const email =
typeof body.email === "string" ? body.email.trim().toLowerCase() : ""; typeof body.email === "string" ? body.email.trim().toLowerCase() : "";
const password = typeof body.password === "string" ? body.password : ""; const password = typeof body.password === "string" ? body.password : "";
@@ -74,6 +86,7 @@ export async function POST(request: NextRequest) {
email, email,
sessionSecret, sessionSecret,
Date.now() + SESSION_SECONDS * 1000, Date.now() + SESSION_SECONDS * 1000,
1,
); );
const response = NextResponse.json({ success: true }); const response = NextResponse.json({ success: true });
response.cookies.set( response.cookies.set(
+46 -4
View File
@@ -1,23 +1,54 @@
import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
import { SDKManager } from "@/core/sdk/sdkManager"; import { SDKManager } from "@/core/sdk/sdkManager";
import { NextRequest, NextResponse } from "next/server"; import { NextRequest, NextResponse } from "next/server";
const SESSION_COOKIE = "datastack_admin_session";
export async function GET( export async function GET(
_req: NextRequest, req: NextRequest,
{ params }: { params: Promise<{ id: string }> }, { params }: { params: Promise<{ id: string }> },
) { ) {
const auth = await verifyRequestAdminSession(
req.cookies.get(SESSION_COOKIE)?.value,
);
if (!auth.valid) {
return NextResponse.json(
{ error: "Authentication required." },
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
);
}
const { id } = await params; const { id } = await params;
const client = await SDKManager.getClientAsync(id); const client = await SDKManager.getClientAsync(id);
if (!client) { if (!client) {
return NextResponse.json({ error: "Client not found" }, { status: 404 }); return NextResponse.json({ error: "Client not found" }, { status: 404 });
} }
const { secret, token, ...clientSummary } = client; const { secret, token, ...clientSummary } = client;
return NextResponse.json({ client: clientSummary }); void secret;
const includeToken = req.nextUrl.searchParams.get("includeToken") === "true";
return NextResponse.json({
client: {
...clientSummary,
tokenPreview: token ? `${token.slice(0, 12)}...` : "",
...(includeToken ? { token } : {}),
},
});
} }
export async function PATCH( export async function PATCH(
req: NextRequest, req: NextRequest,
{ params }: { params: Promise<{ id: string }> }, { params }: { params: Promise<{ id: string }> },
) { ) {
const auth = await verifyRequestAdminSession(
req.cookies.get(SESSION_COOKIE)?.value,
);
if (!auth.valid) {
return NextResponse.json(
{ error: "Authentication required." },
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
);
}
const { id } = await params; const { id } = await params;
try { try {
const body = await req.json(); const body = await req.json();
@@ -46,11 +77,22 @@ export async function PATCH(
} }
export async function DELETE( export async function DELETE(
_req: NextRequest, req: NextRequest,
{ params }: { params: Promise<{ id: string }> }, { params }: { params: Promise<{ id: string }> },
) { ) {
const auth = await verifyRequestAdminSession(
req.cookies.get(SESSION_COOKIE)?.value,
);
if (!auth.valid) {
return NextResponse.json(
{ error: "Authentication required." },
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
);
}
const { id } = await params; const { id } = await params;
const deleted = await SDKManager.deleteClientAsync(id); const purgeFiles = req.nextUrl.searchParams.get("purgeFiles") === "true";
const deleted = await SDKManager.deleteClientAsync(id, { purgeFiles });
if (!deleted) { if (!deleted) {
return NextResponse.json({ error: "Client not found" }, { status: 404 }); return NextResponse.json({ error: "Client not found" }, { status: 404 });
} }
+43 -4
View File
@@ -1,12 +1,28 @@
import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
import { SDKManager } from "@/core/sdk/sdkManager"; import { SDKManager } from "@/core/sdk/sdkManager";
import { NextRequest, NextResponse } from "next/server"; import { NextRequest, NextResponse } from "next/server";
export async function GET() { const SESSION_COOKIE = "datastack_admin_session";
export async function GET(req: NextRequest) {
const auth = await verifyRequestAdminSession(
req.cookies.get(SESSION_COOKIE)?.value,
);
if (!auth.valid) {
return NextResponse.json(
{ error: "Authentication required." },
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
);
}
try { try {
const allClients = await SDKManager.listClientsAsync(); const allClients = await SDKManager.listClientsAsync();
const clients = allClients.map(({ secret, ...client }) => { const clients = allClients.map(({ secret, token, ...client }) => {
void secret; void secret;
return client; return {
...client,
tokenPreview: token ? `${token.slice(0, 12)}...` : "",
};
}); });
return NextResponse.json({ clients, count: clients.length }); return NextResponse.json({ clients, count: clients.length });
} catch (err: unknown) { } catch (err: unknown) {
@@ -17,8 +33,31 @@ export async function GET() {
} }
export async function POST(req: NextRequest) { export async function POST(req: NextRequest) {
const auth = await verifyRequestAdminSession(
req.cookies.get(SESSION_COOKIE)?.value,
);
if (!auth.valid) {
return NextResponse.json(
{ error: "Authentication required." },
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
);
}
let body: Record<string, unknown>;
try {
const parsed = await req.json();
body =
parsed && typeof parsed === "object" && !Array.isArray(parsed)
? (parsed as Record<string, unknown>)
: {};
} catch {
return NextResponse.json(
{ error: "Invalid request payload." },
{ status: 400 },
);
}
try { try {
const body = await req.json();
const { clientId, name, rateLimit } = body; const { clientId, name, rateLimit } = body;
if (!clientId || !name) { if (!clientId || !name) {
+35 -9
View File
@@ -1,4 +1,6 @@
import { SettingsManager } from "@/core/adminSettings/settingsManager";
import { DataBufferEngine } from "@/core/data/buffer"; import { DataBufferEngine } from "@/core/data/buffer";
import { getInternalIngestionHeaders } from "@/core/Security/security";
import { CSVStorageManager } from "@/core/Storage/csvStorage"; import { CSVStorageManager } from "@/core/Storage/csvStorage";
import { parse } from "csv-parse"; import { parse } from "csv-parse";
import { NextRequest, NextResponse } from "next/server"; import { NextRequest, NextResponse } from "next/server";
@@ -31,18 +33,25 @@ export async function GET(request: NextRequest) {
let total = 0; let total = 0;
const addIfMatching = (event: DataEvent) => { const addIfMatching = (event: DataEvent) => {
const eventDate = event.timestamp.slice(0, 10); const eventDate = CSVStorageManager.extractSafePartitionDate(event.timestamp);
const dedupeKey = `${event.clientId}:${event.eventId}`;
const matchesSearch =
!search ||
event.payload.toLowerCase().includes(search) ||
event.eventId.toLowerCase().includes(search) ||
event.clientId.toLowerCase().includes(search);
if ( if (
(clientId !== "all" && event.clientId !== clientId) || (clientId !== "all" && event.clientId !== clientId) ||
(from && eventDate < from) || (from && eventDate < from) ||
(to && eventDate > to) || (to && eventDate > to) ||
(search && !event.payload.toLowerCase().includes(search)) || !matchesSearch ||
eventIds.has(event.eventId) eventIds.has(dedupeKey)
) { ) {
return; return;
} }
eventIds.add(event.eventId); eventIds.add(dedupeKey);
total += 1; total += 1;
let low = 0; let low = 0;
@@ -109,7 +118,16 @@ export async function GET(request: NextRequest) {
} }
} }
for (const event of DataBufferEngine.getRecentEvents(200)) { const settings = await SettingsManager.getSettingsAsync();
const recentWindowLimit = Math.min(
2000,
Math.max(200, settings.batchFlushThreshold || 500),
);
for (const event of await DataBufferEngine.getRecentEventsAsync(
recentWindowLimit,
clientId,
)) {
addIfMatching({ ...event, status: "buffered" }); addIfMatching({ ...event, status: "buffered" });
} }
@@ -117,10 +135,18 @@ export async function GET(request: NextRequest) {
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000" process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, ""); ).replace(/\/$/, "");
try { try {
const res = await fetch(`${ingestionBaseUrl}/internal/recent?limit=200`, { const qs = new URLSearchParams({ limit: String(recentWindowLimit) });
cache: "no-store", if (clientId && clientId !== "all") {
signal: AbortSignal.timeout(1000), qs.set("clientId", clientId);
}); }
const res = await fetch(
`${ingestionBaseUrl}/internal/recent?${qs.toString()}`,
{
headers: getInternalIngestionHeaders(),
cache: "no-store",
signal: AbortSignal.timeout(1000),
},
);
if (res.ok) { if (res.ok) {
const data = await res.json(); const data = await res.json();
if (Array.isArray(data.events)) { if (Array.isArray(data.events)) {
+75 -30
View File
@@ -1,9 +1,12 @@
import { SettingsManager } from '@/core/adminSettings/settingsManager'; import { SettingsManager } from '@/core/adminSettings/settingsManager';
import { verifyAdminSessionToken } from '@/core/auth/adminSession'; import { verifyRequestAdminSession } from '@/core/auth/adminUsers';
import { AuthService } from '@/core/auth/jwt'; import { AuthService } from '@/core/auth/jwt';
import { DataBufferEngine } from '@/core/data/buffer'; import { DataBufferEngine } from '@/core/data/buffer';
import { SDKManager } from '@/core/sdk/sdkManager'; import { SDKManager } from '@/core/sdk/sdkManager';
import { SecurityEngine } from '@/core/Security/security'; import {
SecurityEngine,
getInternalIngestionHeaders,
} from '@/core/Security/security';
import { CSVStorageManager, type DataEvent } from '@/core/Storage/csvStorage'; import { CSVStorageManager, type DataEvent } from '@/core/Storage/csvStorage';
import { parse } from 'csv-parse'; import { parse } from 'csv-parse';
import { NextRequest, NextResponse } from 'next/server'; import { NextRequest, NextResponse } from 'next/server';
@@ -15,14 +18,15 @@ async function getRecentStoredEvents(
limit: number, limit: number,
clientId?: string, clientId?: string,
): Promise<DataEvent[]> { ): Promise<DataEvent[]> {
const files = CSVStorageManager.listAllFiles() const allFiles = await CSVStorageManager.listAllFilesAsync();
const files = allFiles
.filter((f) => !clientId || clientId === 'all' || f.clientId === clientId) .filter((f) => !clientId || clientId === 'all' || f.clientId === clientId)
.slice(0, 5); .slice(0, 5);
const collected: DataEvent[] = []; const collected: DataEvent[] = [];
for (const file of files) { for (const file of files) {
const fileEvents: DataEvent[] = []; const fileTail: DataEvent[] = [];
const parser = createReadStream(file.filePath).pipe( const parser = createReadStream(file.filePath).pipe(
parse({ columns: true, skip_empty_lines: true, bom: true }), parse({ columns: true, skip_empty_lines: true, bom: true }),
); );
@@ -32,7 +36,10 @@ async function getRecentStoredEvents(
Record<string, string> Record<string, string>
>) { >) {
if (!record.eventId) continue; if (!record.eventId) continue;
fileEvents.push({ if (fileTail.length >= limit) {
fileTail.shift();
}
fileTail.push({
clientId: record.clientId || file.clientId, clientId: record.clientId || file.clientId,
eventId: record.eventId, eventId: record.eventId,
timestamp: record.timestamp || `${file.date}T00:00:00.000Z`, timestamp: record.timestamp || `${file.date}T00:00:00.000Z`,
@@ -43,8 +50,8 @@ async function getRecentStoredEvents(
parser.destroy(); parser.destroy();
} }
if (fileEvents.length > 0) { if (fileTail.length > 0) {
collected.push(...fileEvents.slice(-limit)); collected.push(...fileTail);
} }
} }
@@ -55,9 +62,10 @@ export async function GET(req: NextRequest) {
try { try {
const { searchParams } = new URL(req.url); const { searchParams } = new URL(req.url);
const clientId = searchParams.get('clientId') || undefined; const clientId = searchParams.get('clientId') || undefined;
const limit = Math.min(200, Math.max(1, Number(searchParams.get('limit')) || 50)); const limit = Math.min(2000, Math.max(1, Number(searchParams.get('limit')) || 50));
const bufferedEvents = DataBufferEngine.getRecentEvents(limit, clientId); await SettingsManager.getSettingsAsync();
const bufferedEvents = await DataBufferEngine.getRecentEventsAsync(limit, clientId);
let socketBufferedEvents: DataEvent[] = []; let socketBufferedEvents: DataEvent[] = [];
const ingestionBaseUrl = ( const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000' process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
@@ -67,6 +75,7 @@ export async function GET(req: NextRequest) {
const qs = new URLSearchParams({ limit: String(limit) }); const qs = new URLSearchParams({ limit: String(limit) });
if (clientId) qs.set('clientId', clientId); if (clientId) qs.set('clientId', clientId);
const res = await fetch(`${ingestionBaseUrl}/internal/recent?${qs.toString()}`, { const res = await fetch(`${ingestionBaseUrl}/internal/recent?${qs.toString()}`, {
headers: getInternalIngestionHeaders(),
cache: 'no-store', cache: 'no-store',
signal: AbortSignal.timeout(1000), signal: AbortSignal.timeout(1000),
}); });
@@ -86,8 +95,10 @@ export async function GET(req: NextRequest) {
const merged: DataEvent[] = []; const merged: DataEvent[] = [];
for (const evt of [...bufferedEvents, ...socketBufferedEvents, ...storedEvents]) { for (const evt of [...bufferedEvents, ...socketBufferedEvents, ...storedEvents]) {
if (!evt.eventId || seenIds.has(evt.eventId)) continue; if (!evt.eventId) continue;
seenIds.add(evt.eventId); const dedupeKey = `${evt.clientId}:${evt.eventId}`;
if (seenIds.has(dedupeKey)) continue;
seenIds.add(dedupeKey);
merged.push(evt); merged.push(evt);
} }
@@ -107,9 +118,6 @@ export async function POST(req: NextRequest) {
// Verify either Client Bearer JWT or Admin Session Cookie // Verify either Client Bearer JWT or Admin Session Cookie
const authHeader = req.headers.get('authorization'); const authHeader = req.headers.get('authorization');
const sessionSecret =
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
'datastack-admin-session-default-secret-32-chars';
const sessionCookie = req.cookies.get(SESSION_COOKIE)?.value; const sessionCookie = req.cookies.get(SESSION_COOKIE)?.value;
let tokenClientId: string | null = null; let tokenClientId: string | null = null;
@@ -128,20 +136,30 @@ export async function POST(req: NextRequest) {
tokenClientId = verified.payload.clientId; tokenClientId = verified.payload.clientId;
tokenRateLimit = verified.payload.rateLimit; tokenRateLimit = verified.payload.rateLimit;
} else { } else {
const isAdmin = Boolean( const auth = await verifyRequestAdminSession(sessionCookie);
sessionSecret && if (!auth.valid) {
sessionCookie &&
(await verifyAdminSessionToken(sessionCookie, sessionSecret)),
);
if (!isAdmin) {
return NextResponse.json( return NextResponse.json(
{ error: 'Authentication required. Provide a valid Client Bearer JWT or Admin Session.' }, {
{ status: 401 }, error:
auth.reason === 'UNAUTHORIZED'
? 'Authentication required. Provide a valid Client Bearer JWT or Admin Session.'
: 'Admin authentication is temporarily unavailable.',
},
{ status: auth.reason === 'UNAUTHORIZED' ? 401 : 503 },
); );
} }
} }
const body = await req.json(); let body: unknown;
try {
body = await req.json();
} catch {
return NextResponse.json(
{ error: 'Invalid JSON request payload.', code: 'INVALID_JSON' },
{ status: 400 },
);
}
const bodyObj = const bodyObj =
body && typeof body === 'object' && !Array.isArray(body) body && typeof body === 'object' && !Array.isArray(body)
? (body as Record<string, unknown>) ? (body as Record<string, unknown>)
@@ -154,7 +172,27 @@ export async function POST(req: NextRequest) {
.toLowerCase() .toLowerCase()
.replace(/[^a-z0-9_-]/g, '_'); .replace(/[^a-z0-9_-]/g, '_');
if (await SDKManager.isClientDeletedAsync(clientId)) {
return NextResponse.json(
{
error: `CLIENT_REVOKED: Client '${clientId}' has been deleted by administrator.`,
code: 'CLIENT_REVOKED',
},
{ status: 403 },
);
}
const registeredClient = await SDKManager.getClientAsync(clientId); const registeredClient = await SDKManager.getClientAsync(clientId);
if (rawBearerToken && !registeredClient) {
return NextResponse.json(
{
error: `CLIENT_REVOKED: Client '${clientId}' registration not found or has been deleted.`,
code: 'CLIENT_REVOKED',
},
{ status: 403 },
);
}
if (registeredClient) { if (registeredClient) {
if (registeredClient.status === 'revoked') { if (registeredClient.status === 'revoked') {
return NextResponse.json( return NextResponse.json(
@@ -196,6 +234,7 @@ export async function POST(req: NextRequest) {
: null; : null;
if (batchItems !== null) { if (batchItems !== null) {
const envelopeKeys = new Set(['eventId', 'timestamp', 'data', 'clientId']);
let processed = 0; let processed = 0;
let rejected = 0; let rejected = 0;
let duplicates = 0; let duplicates = 0;
@@ -205,6 +244,8 @@ export async function POST(req: NextRequest) {
const oversizedEventIds: string[] = []; const oversizedEventIds: string[] = [];
const errors: string[] = []; const errors: string[] = [];
const batchEvents: DataEvent[] = [];
for (const item of batchItems) { for (const item of batchItems) {
const safeItem = const safeItem =
item && typeof item === 'object' && !Array.isArray(item) item && typeof item === 'object' && !Array.isArray(item)
@@ -221,8 +262,10 @@ export async function POST(req: NextRequest) {
? safeItem.timestamp.trim() ? safeItem.timestamp.trim()
: new Date().toISOString(); : new Date().toISOString();
const itemData = const isEnvelope =
safeItem.data !== undefined ? safeItem.data : safeItem; safeItem.data !== undefined &&
Object.keys(safeItem).every((k) => envelopeKeys.has(k));
const itemData = isEnvelope ? safeItem.data : safeItem;
const val = SecurityEngine.validateEvent( const val = SecurityEngine.validateEvent(
clientId, clientId,
@@ -251,18 +294,20 @@ export async function POST(req: NextRequest) {
continue; continue;
} }
const event: DataEvent = { batchEvents.push({
clientId, clientId,
eventId: itemEventId, eventId: itemEventId,
timestamp: itemTimestamp, timestamp: itemTimestamp,
payload: payload:
typeof itemData === 'string' ? itemData : JSON.stringify(itemData), typeof itemData === 'string' ? itemData : JSON.stringify(itemData),
}; });
DataBufferEngine.push(event);
processed += 1; processed += 1;
} }
if (batchEvents.length > 0) {
await DataBufferEngine.pushBatchAsync(batchEvents);
}
if (processed > 0) { if (processed > 0) {
SDKManager.incrementEventCount(clientId, processed); SDKManager.incrementEventCount(clientId, processed);
} }
@@ -347,7 +392,7 @@ export async function POST(req: NextRequest) {
payload: typeof data === 'string' ? data : JSON.stringify(data), payload: typeof data === 'string' ? data : JSON.stringify(data),
}; };
DataBufferEngine.push(event); await DataBufferEngine.pushAsync(event);
SDKManager.incrementEventCount(clientId, 1); SDKManager.incrementEventCount(clientId, 1);
return NextResponse.json({ success: true, eventId, event }); return NextResponse.json({ success: true, eventId, event });
+40 -14
View File
@@ -1,4 +1,6 @@
import { SettingsManager } from "@/core/adminSettings/settingsManager";
import { DataBufferEngine } from "@/core/data/buffer"; import { DataBufferEngine } from "@/core/data/buffer";
import { getInternalIngestionHeaders } from "@/core/Security/security";
import { import {
CSVStorageManager, CSVStorageManager,
type StorageFileMeta, type StorageFileMeta,
@@ -6,6 +8,7 @@ import {
import { parse } from "csv-parse"; import { parse } from "csv-parse";
import { NextRequest, NextResponse } from "next/server"; import { NextRequest, NextResponse } from "next/server";
import { createReadStream } from "node:fs"; import { createReadStream } from "node:fs";
import path from "node:path";
type ExportFormat = "csv" | "json"; type ExportFormat = "csv" | "json";
type CsvRecord = Record<string, string>; type CsvRecord = Record<string, string>;
@@ -108,19 +111,26 @@ function streamFile(file: StorageFileMeta) {
export async function GET(request: NextRequest) { export async function GET(request: NextRequest) {
try { try {
await DataBufferEngine.flush(); const { searchParams } = new URL(request.url);
const ingestionBaseUrl = const shouldFlush = searchParams.get("flush") !== "false";
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000";
try { await SettingsManager.getSettingsAsync();
await fetch(`${ingestionBaseUrl}/internal/flush`, { if (shouldFlush) {
method: "POST", await DataBufferEngine.flush();
signal: AbortSignal.timeout(1500), const ingestionBaseUrl = (
}); process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
} catch { ).replace(/\/$/, "");
// Port 4000 may not be running in isolated single-process dev mode try {
await fetch(`${ingestionBaseUrl}/internal/flush`, {
method: "POST",
headers: getInternalIngestionHeaders(),
signal: AbortSignal.timeout(1500),
});
} catch {
// Port 4000 may not be running in isolated single-process dev mode
}
} }
const { searchParams } = new URL(request.url);
const clientId = searchParams.get("clientId") || "all"; const clientId = searchParams.get("clientId") || "all";
const filename = searchParams.get("filename"); const filename = searchParams.get("filename");
const from = searchParams.get("from") || ""; const from = searchParams.get("from") || "";
@@ -129,17 +139,33 @@ export async function GET(request: NextRequest) {
searchParams.get("format") === "json" ? "json" : "csv"; searchParams.get("format") === "json" ? "json" : "csv";
const includeHeader = searchParams.get("includeHeader") !== "false"; const includeHeader = searchParams.get("includeHeader") !== "false";
let files = CSVStorageManager.listAllFiles(); let files = await CSVStorageManager.listAllFilesAsync();
if (clientId !== "all") if (clientId !== "all")
files = files.filter((file) => file.clientId === clientId); files = files.filter((file) => file.clientId === clientId);
if (filename) { if (filename) {
const file = files.find((candidate) => candidate.filename === filename); const safeFilename = path.basename(filename);
if (!file) const matches = files.filter(
(candidate) => candidate.filename === safeFilename,
);
if (matches.length === 0) {
return NextResponse.json( return NextResponse.json(
{ error: "CSV file not found" }, { error: "CSV file not found" },
{ status: 404 }, { status: 404 },
); );
}
if (matches.length > 1) {
return NextResponse.json(
{
error:
"Ambiguous filename matches multiple clients. Specify clientId.",
code: "AMBIGUOUS_FILENAME",
matchingClients: matches.map((m) => m.clientId),
},
{ status: 400 },
);
}
const file = matches[0];
return new NextResponse(streamFile(file), { return new NextResponse(streamFile(file), {
status: 200, status: 200,
headers: { headers: {
+13 -1
View File
@@ -23,7 +23,19 @@ export async function GET(req: NextRequest) {
); );
} }
const clientId = verified.payload.clientId; const clientId = String(verified.payload.clientId || '')
.trim()
.toLowerCase();
if (SDKManager.isClientDeleted(clientId)) {
return NextResponse.json(
{
status: 'error',
code: 'CLIENT_REVOKED',
error: `CLIENT_REVOKED: Client '${clientId}' has been deleted by administrator.`,
},
{ status: 403 },
);
}
const registeredClient = await SDKManager.getClientAsync(clientId); const registeredClient = await SDKManager.getClientAsync(clientId);
if (registeredClient) { if (registeredClient) {
if (registeredClient.status === 'revoked') { if (registeredClient.status === 'revoked') {
+176 -4
View File
@@ -1,8 +1,55 @@
import { SettingsManager } from "@/core/adminSettings/settingsManager"; import {
SettingsManager,
type AdminSettings,
} from "@/core/adminSettings/settingsManager";
import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
import { NextRequest, NextResponse } from "next/server"; import { NextRequest, NextResponse } from "next/server";
export async function GET() { const SESSION_COOKIE = "datastack_admin_session";
const ALLOWED_SETTING_KEYS = new Set([
"maxEventsPerSec",
"maxPayloadSizeKb",
"batchFlushThreshold",
"batchFlushIntervalMs",
"fileRotationMb",
"enableIdempotency",
"autoPruneDays",
"jwtSecret",
]);
function parseBoundedInt(
raw: unknown,
field: string,
min: number,
max: number,
): number {
if (typeof raw !== "number" || !Number.isInteger(raw)) {
throw new Error(`${field} must be an integer between ${min} and ${max}.`);
}
if (raw < min || raw > max) {
throw new Error(`${field} must be between ${min} and ${max}.`);
}
return raw;
}
export async function GET(req: NextRequest) {
try { try {
const auth = await verifyRequestAdminSession(
req.cookies.get(SESSION_COOKIE)?.value,
);
if (!auth.valid) {
return NextResponse.json(
{
error:
auth.reason === "UNAUTHORIZED"
? "Authentication required."
: "Admin authentication is temporarily unavailable.",
},
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
);
}
const settings = await SettingsManager.getPublicSettingsAsync(); const settings = await SettingsManager.getPublicSettingsAsync();
return NextResponse.json({ settings }); return NextResponse.json({ settings });
} catch (err: unknown) { } catch (err: unknown) {
@@ -13,8 +60,133 @@ export async function GET() {
export async function POST(req: NextRequest) { export async function POST(req: NextRequest) {
try { try {
const body = await req.json(); const auth = await verifyRequestAdminSession(
await SettingsManager.updateSettingsAsync(body); req.cookies.get(SESSION_COOKIE)?.value,
);
if (!auth.valid) {
return NextResponse.json(
{
error:
auth.reason === "UNAUTHORIZED"
? "Authentication required."
: "Admin authentication is temporarily unavailable.",
},
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
);
}
let rawBody: unknown;
try {
rawBody = await req.json();
} catch {
return NextResponse.json(
{ error: "Invalid JSON request payload.", code: "INVALID_JSON" },
{ status: 400 },
);
}
if (!rawBody || typeof rawBody !== "object" || Array.isArray(rawBody)) {
return NextResponse.json(
{
error: "Settings payload must be a JSON object.",
code: "VALIDATION_ERROR",
},
{ status: 400 },
);
}
const body = rawBody as Record<string, unknown>;
const unknownKeys = Object.keys(body).filter(
(key) => !ALLOWED_SETTING_KEYS.has(key),
);
if (unknownKeys.length > 0) {
return NextResponse.json(
{
error: `Unsupported settings field(s): ${unknownKeys.join(", ")}`,
code: "VALIDATION_ERROR",
},
{ status: 400 },
);
}
const validated: Partial<AdminSettings> = {};
try {
if (body.maxEventsPerSec !== undefined) {
validated.maxEventsPerSec = parseBoundedInt(
body.maxEventsPerSec,
"maxEventsPerSec",
1,
100000,
);
}
if (body.maxPayloadSizeKb !== undefined) {
validated.maxPayloadSizeKb = parseBoundedInt(
body.maxPayloadSizeKb,
"maxPayloadSizeKb",
1,
10240,
);
}
if (body.batchFlushThreshold !== undefined) {
validated.batchFlushThreshold = parseBoundedInt(
body.batchFlushThreshold,
"batchFlushThreshold",
1,
50000,
);
}
if (body.batchFlushIntervalMs !== undefined) {
validated.batchFlushIntervalMs = parseBoundedInt(
body.batchFlushIntervalMs,
"batchFlushIntervalMs",
100,
60000,
);
}
if (body.fileRotationMb !== undefined) {
validated.fileRotationMb = parseBoundedInt(
body.fileRotationMb,
"fileRotationMb",
1,
2048,
);
}
if (body.autoPruneDays !== undefined) {
validated.autoPruneDays = parseBoundedInt(
body.autoPruneDays,
"autoPruneDays",
0,
3650,
);
}
if (body.enableIdempotency !== undefined) {
if (typeof body.enableIdempotency !== "boolean") {
throw new Error("enableIdempotency must be a boolean.");
}
validated.enableIdempotency = body.enableIdempotency;
}
if (body.jwtSecret !== undefined) {
if (typeof body.jwtSecret !== "string" || body.jwtSecret.trim().length < 32) {
throw new Error(
"jwtSecret must be a string with at least 32 characters.",
);
}
validated.jwtSecret = body.jwtSecret.trim();
}
} catch (validationErr: unknown) {
return NextResponse.json(
{
error:
validationErr instanceof Error
? validationErr.message
: "Invalid settings value.",
code: "VALIDATION_ERROR",
},
{ status: 400 },
);
}
await SettingsManager.updateSettingsAsync(validated);
const settings = await SettingsManager.getPublicSettingsAsync(); const settings = await SettingsManager.getPublicSettingsAsync();
return NextResponse.json({ return NextResponse.json({
settings, settings,
+59 -11
View File
@@ -1,4 +1,6 @@
import { NextRequest, NextResponse } from 'next/server'; import { NextRequest, NextResponse } from 'next/server';
import { SettingsManager } from '@/core/adminSettings/settingsManager';
import { getInternalIngestionHeaders } from '@/core/Security/security';
import { CSVStorageManager } from '@/core/Storage/csvStorage'; import { CSVStorageManager } from '@/core/Storage/csvStorage';
import { DataBufferEngine } from '@/core/data/buffer'; import { DataBufferEngine } from '@/core/data/buffer';
@@ -7,8 +9,11 @@ export async function GET(req: NextRequest) {
const { searchParams } = new URL(req.url); const { searchParams } = new URL(req.url);
const clientId = searchParams.get('clientId') || undefined; const clientId = searchParams.get('clientId') || undefined;
const stats = CSVStorageManager.getStorageStats(); const [stats, allFiles] = await Promise.all([
let files = CSVStorageManager.listAllFiles(); CSVStorageManager.getStorageStatsAsync(),
CSVStorageManager.listAllFilesAsync(),
]);
let files = allFiles;
if (clientId && clientId !== 'all') { if (clientId && clientId !== 'all') {
files = files.filter((f) => f.clientId === clientId); files = files.filter((f) => f.clientId === clientId);
@@ -23,12 +28,17 @@ export async function GET(req: NextRequest) {
export async function POST(_req: NextRequest) { export async function POST(_req: NextRequest) {
try { try {
// 1. Flush local Next.js DataBufferEngine await SettingsManager.getSettingsAsync();
// 1. Flush local / shared Next.js DataBufferEngine
const localRes = await DataBufferEngine.flush(); const localRes = await DataBufferEngine.flush();
// 2. Also flush Port 4000 Socket.IO ingestion buffer // 2. Also flush Port 4000 Socket.IO ingestion buffer
let remoteCount = 0; let remoteCount = 0;
let remoteFiles: string[] = []; let remoteFiles: string[] = [];
let ingestionFlushed = false;
let warning: string | undefined;
const explicitIngestionUrl = Boolean(process.env.INGESTION_INTERNAL_URL?.trim());
const ingestionBaseUrl = ( const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000' process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
).replace(/\/$/, ''); ).replace(/\/$/, '');
@@ -36,17 +46,53 @@ export async function POST(_req: NextRequest) {
try { try {
const ingestRes = await fetch(`${ingestionBaseUrl}/internal/flush`, { const ingestRes = await fetch(`${ingestionBaseUrl}/internal/flush`, {
method: 'POST', method: 'POST',
headers: getInternalIngestionHeaders(),
cache: 'no-store', cache: 'no-store',
signal: AbortSignal.timeout(2500),
}); });
if (ingestRes.ok) { if (!ingestRes.ok) {
const ingestData = await ingestRes.json(); const errPayload = await ingestRes.json().catch(() => ({}));
remoteCount = Number(ingestData.flushedCount) || 0; const remoteErr =
if (Array.isArray(ingestData.filesUpdated)) { typeof errPayload?.error === 'string'
remoteFiles = ingestData.filesUpdated; ? errPayload.error
} : `Ingestion flush returned HTTP ${ingestRes.status}`;
return NextResponse.json(
{
success: false,
error: remoteErr,
code: 'INGESTION_FLUSH_FAILED',
flushedCount: localRes.count,
filesUpdated: localRes.files,
ingestionFlushed: false,
ingestionStatus: ingestRes.status,
},
{ status: 502 },
);
} }
} catch {
// Port 4000 server may be offline in standalone Next.js dev mode ingestionFlushed = true;
const ingestData = await ingestRes.json();
remoteCount = Number(ingestData.flushedCount) || 0;
if (Array.isArray(ingestData.filesUpdated)) {
remoteFiles = ingestData.filesUpdated;
}
} catch (err: unknown) {
const errMsg =
err instanceof Error ? err.message : 'Ingestion server unreachable';
if (explicitIngestionUrl) {
return NextResponse.json(
{
success: false,
error: `Ingestion server flush failed: ${errMsg}`,
code: 'INGESTION_UNREACHABLE',
flushedCount: localRes.count,
filesUpdated: localRes.files,
ingestionFlushed: false,
},
{ status: 502 },
);
}
warning = `Port 4000 ingestion server is unreachable (${errMsg}); flushed Next.js buffer only.`;
} }
const combinedFiles = Array.from( const combinedFiles = Array.from(
@@ -57,6 +103,8 @@ export async function POST(_req: NextRequest) {
success: true, success: true,
flushedCount: localRes.count + remoteCount, flushedCount: localRes.count + remoteCount,
filesUpdated: combinedFiles, filesUpdated: combinedFiles,
ingestionFlushed,
...(warning ? { warning } : {}),
}); });
} catch (err: unknown) { } catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Manual flush failed'; const message = err instanceof Error ? err.message : 'Manual flush failed';
+50
View File
@@ -0,0 +1,50 @@
'use client';
import { Button } from '@/components/ui/Button';
import { AlertTriangle, RotateCcw } from 'lucide-react';
import Link from 'next/link';
import { useEffect } from 'react';
export default function RootErrorBoundary({
error,
reset,
}: {
error: Error & { digest?: string };
reset: () => void;
}) {
useEffect(() => {
console.error('Application error boundary:', error);
}, [error]);
return (
<main className="flex min-h-screen items-center justify-center bg-[var(--bg-main)] px-4 py-8 text-[var(--text)]">
<section className="w-full max-w-[440px] border border-[var(--border-strong)] bg-[var(--surface)] p-5">
<div className="mb-3 flex items-center gap-2 border-b border-[var(--border)] pb-3">
<AlertTriangle className="h-4 w-4 shrink-0 text-[var(--danger)]" aria-hidden="true" />
<h1 className="text-[13px] font-medium uppercase tracking-wider text-[var(--text)]">
Runtime error
</h1>
</div>
<p className="mb-3 text-[12px] leading-relaxed text-[var(--text-muted)]">
An unexpected rendering error occurred. You can retry the view or return to the overview page.
</p>
{error?.message && (
<div className="mb-4 border border-[var(--border)] bg-[var(--bg-main)] px-2.5 py-2 font-mono text-[11px] text-[var(--danger)] break-words">
{error.message}
</div>
)}
<div className="flex flex-wrap items-center justify-end gap-2 border-t border-[var(--border)] pt-3">
<Link href="/">
<Button type="button" size="xs" variant="ghost">
Return home
</Button>
</Link>
<Button type="button" size="xs" variant="primary" onClick={() => reset()}>
<RotateCcw className="h-3 w-3" aria-hidden="true" />
Try again
</Button>
</div>
</section>
</main>
);
}
+25 -6
View File
@@ -1,11 +1,31 @@
import { PwaLifecycle } from '@/components/pwa/PwaLifecycle';
import { ToastProvider } from '@/components/ui/Toast'; import { ToastProvider } from '@/components/ui/Toast';
import type { Metadata } from 'next'; import type { Metadata, Viewport } from 'next';
import './globals.css'; import './globals.css';
export const viewport: Viewport = {
themeColor: '#0a0a0b',
width: 'device-width',
initialScale: 1,
};
export const metadata: Metadata = { export const metadata: Metadata = {
title: 'DataStack - Realtime Data Collection Platform', title: 'DataStack - Realtime Data Collection Platform',
description: 'Self-hosted universal realtime data collection engine with Socket.IO ingestion, Redis buffer, JWT client auth, CSV batch rotation, and /vcon/ admin panel.', description: 'Self-hosted universal realtime data collection engine with Socket.IO ingestion, Redis buffer, JWT client auth, CSV batch rotation, and /vcon/ admin panel.',
manifest: '/manifest.json', manifest: '/manifest.json',
appleWebApp: {
capable: true,
statusBarStyle: 'black-translucent',
title: 'DataStack',
},
icons: {
icon: [
{ url: '/icon.svg', type: 'image/svg+xml' },
{ url: '/icon-192.png', sizes: '192x192', type: 'image/png' },
{ url: '/icon-512.png', sizes: '512x512', type: 'image/png' },
],
apple: [{ url: '/apple-touch-icon.png', sizes: '180x180', type: 'image/png' }],
},
openGraph: { openGraph: {
title: 'DataStack - Realtime Data Collection Platform', title: 'DataStack - Realtime Data Collection Platform',
description: 'Self-hosted universal realtime data collection engine with Socket.IO ingestion, Redis buffer, JWT client auth, CSV batch rotation, and /vcon/ admin panel.', description: 'Self-hosted universal realtime data collection engine with Socket.IO ingestion, Redis buffer, JWT client auth, CSV batch rotation, and /vcon/ admin panel.',
@@ -21,12 +41,11 @@ export const metadata: Metadata = {
export default function RootLayout({ children }: { children: React.ReactNode }) { export default function RootLayout({ children }: { children: React.ReactNode }) {
return ( return (
<html lang="en" className="dark"> <html lang="en" className="dark">
<head>
<link rel="manifest" href="/manifest.json" />
<meta name="theme-color" content="#0a0a0b" />
</head>
<body suppressHydrationWarning className="min-h-screen bg-[var(--bg-main)] text-[var(--text)] antialiased"> <body suppressHydrationWarning className="min-h-screen bg-[var(--bg-main)] text-[var(--text)] antialiased">
<ToastProvider>{children}</ToastProvider> <ToastProvider>
{children}
<PwaLifecycle />
</ToastProvider>
</body> </body>
</html> </html>
); );
+6 -5
View File
@@ -1,5 +1,5 @@
import { MainLayout } from '@/components/layout/MainLayout'; import { MainLayout } from '@/components/layout/MainLayout';
import { Button } from '@/components/ui/Button'; import { buttonClassNames } from '@/components/ui/Button';
import Link from 'next/link'; import Link from 'next/link';
export default function NotFound() { export default function NotFound() {
@@ -11,10 +11,11 @@ export default function NotFound() {
The requested page or route does not exist on DataStack. The requested page or route does not exist on DataStack.
</p> </p>
<div className="flex items-center gap-2 pt-2"> <div className="flex items-center gap-2 pt-2">
<Link href="/"> <Link href="/" className={buttonClassNames({ variant: 'primary', size: 'sm' })}>
<Button variant="secondary" size="sm"> Return Home
Return Home </Link>
</Button> <Link href="/vcon/dashboard" className={buttonClassNames({ variant: 'secondary', size: 'sm' })}>
Open Console
</Link> </Link>
</div> </div>
</div> </div>
+17 -4
View File
@@ -23,7 +23,11 @@ export default function ClientsPage() {
const [now, setNow] = useState(() => Date.now()); const [now, setNow] = useState(() => Date.now());
const fetchClients = useCallback(async () => { const fetchClients = useCallback(async () => {
const response = await fetch('/api/clients'); const response = await fetch('/api/clients', { cache: 'no-store' });
if (response.status === 401) {
window.location.assign('/vcon/login');
return;
}
if (!response.ok) throw new Error('Unable to load clients'); if (!response.ok) throw new Error('Unable to load clients');
const data = await response.json(); const data = await response.json();
setClients(setCached('vcon:clients', data.clients || [])); setClients(setCached('vcon:clients', data.clients || []));
@@ -49,16 +53,19 @@ export default function ClientsPage() {
!query || !query ||
client.clientId.toLowerCase().includes(query) || client.clientId.toLowerCase().includes(query) ||
client.name.toLowerCase().includes(query); client.name.toLowerCase().includes(query);
const revoked = client.status === 'revoked';
const online = const online =
client.status === 'active' && client.status === 'active' &&
Boolean(client.lastActive) && Boolean(client.lastActive) &&
now - new Date(client.lastActive!).getTime() < 60_000; now - new Date(client.lastActive!).getTime() < 60_000;
const pending = client.status === 'active' && !client.lastActive; const pending = client.status === 'active' && !client.lastActive;
const offline = client.status === 'active' && !online && !pending;
const matchesStatus = const matchesStatus =
statusFilter === 'all' || statusFilter === 'all' ||
(statusFilter === 'online' && online) || (statusFilter === 'online' && online) ||
(statusFilter === 'offline' && !online && !pending) || (statusFilter === 'offline' && offline) ||
(statusFilter === 'pending' && pending); (statusFilter === 'pending' && pending) ||
(statusFilter === 'revoked' && revoked);
return matchesSearch && matchesStatus; return matchesSearch && matchesStatus;
}); });
}, [clients, now, search, statusFilter]); }, [clients, now, search, statusFilter]);
@@ -72,7 +79,10 @@ export default function ClientsPage() {
const pendingCount = clients.filter( const pendingCount = clients.filter(
(client) => client.status === 'active' && !client.lastActive (client) => client.status === 'active' && !client.lastActive
).length; ).length;
const offlineCount = clients.length - onlineCount - pendingCount; const revokedCount = clients.filter(
(client) => client.status === 'revoked'
).length;
const offlineCount = clients.length - onlineCount - pendingCount - revokedCount;
return ( return (
<PageShell <PageShell
@@ -90,11 +100,13 @@ export default function ClientsPage() {
} }
> >
<StatStrip <StatStrip
className="md:grid-cols-5"
items={[ items={[
{ label: 'Total', value: clients.length }, { label: 'Total', value: clients.length },
{ label: 'Online', value: onlineCount, dot: 'ok' }, { label: 'Online', value: onlineCount, dot: 'ok' },
{ label: 'Offline', value: offlineCount, dot: 'off' }, { label: 'Offline', value: offlineCount, dot: 'off' },
{ label: 'Pending', value: pendingCount, dot: 'pending' }, { label: 'Pending', value: pendingCount, dot: 'pending' },
{ label: 'Revoked', value: revokedCount, dot: 'off' },
]} ]}
/> />
@@ -109,6 +121,7 @@ export default function ClientsPage() {
online: onlineCount, online: onlineCount,
offline: offlineCount, offline: offlineCount,
pending: pendingCount, pending: pendingCount,
revoked: revokedCount,
}} }}
/> />
<ClientsTable <ClientsTable
+50
View File
@@ -0,0 +1,50 @@
'use client';
import { Button } from '@/components/ui/Button';
import { AlertTriangle, RotateCcw } from 'lucide-react';
import Link from 'next/link';
import { useEffect } from 'react';
export default function VconErrorBoundary({
error,
reset,
}: {
error: Error & { digest?: string };
reset: () => void;
}) {
useEffect(() => {
console.error('Admin console error boundary:', error);
}, [error]);
return (
<div className="flex min-h-[60vh] items-center justify-center p-4 text-[var(--text)]">
<section className="w-full max-w-[440px] border border-[var(--border-strong)] bg-[var(--surface)] p-5">
<div className="mb-3 flex items-center gap-2 border-b border-[var(--border)] pb-3">
<AlertTriangle className="h-4 w-4 shrink-0 text-[var(--danger)]" aria-hidden="true" />
<h1 className="text-[13px] font-medium uppercase tracking-wider text-[var(--text)]">
Console panel error
</h1>
</div>
<p className="mb-3 text-[12px] leading-relaxed text-[var(--text-muted)]">
This console view encountered an unexpected failure. Retry rendering or navigate back to the dashboard.
</p>
{error?.message && (
<div className="mb-4 border border-[var(--border)] bg-[var(--bg-main)] px-2.5 py-2 font-mono text-[11px] text-[var(--danger)] break-words">
{error.message}
</div>
)}
<div className="flex flex-wrap items-center justify-end gap-2 border-t border-[var(--border)] pt-3">
<Link href="/vcon/dashboard">
<Button type="button" size="xs" variant="ghost">
Dashboard
</Button>
</Link>
<Button type="button" size="xs" variant="primary" onClick={() => reset()}>
<RotateCcw className="h-3 w-3" aria-hidden="true" />
Retry panel
</Button>
</div>
</section>
</div>
);
}
+6 -5
View File
@@ -6,12 +6,13 @@ export const runtime = 'nodejs';
export const dynamic = 'force-dynamic'; export const dynamic = 'force-dynamic';
export default async function LoginPage() { export default async function LoginPage() {
if (process.env.DATABASE_URL) { const isConfigured = await isAdminSetupComplete().catch(() => false);
const isConfigured = await isAdminSetupComplete().catch(() => false); if (!isConfigured) redirect('/vcon/setup');
if (!isConfigured) redirect('/vcon/setup');
}
const showDemoHint = !process.env.DATABASE_URL && process.env.NODE_ENV !== 'production'; const showDemoHint =
!process.env.DATABASE_URL &&
process.env.NODE_ENV !== 'production' &&
process.env.DATASTACK_ALLOW_DEMO_ADMIN !== 'false';
return <AdminLogin showDemoHint={showDemoHint} />; return <AdminLogin showDemoHint={showDemoHint} />;
} }
+96 -21
View File
@@ -11,15 +11,25 @@ import { useToast } from '@/components/ui/Toast';
import { getCached, setCached } from '@/lib/vconCache'; import { getCached, setCached } from '@/lib/vconCache';
import { useEffect, useState } from 'react'; import { useEffect, useState } from 'react';
interface SdkClientItem {
clientId: string;
name: string;
status?: 'active' | 'revoked';
tokenPreview?: string;
token?: string;
}
export default function SdkGuidePage() { export default function SdkGuidePage() {
const { success } = useToast(); const { success, error } = useToast();
const [clients, setClients] = useState<any[]>(() => const [clients, setClients] = useState<SdkClientItem[]>(() =>
getCached<any[]>('vcon:clients', []) getCached<SdkClientItem[]>('vcon:clients', [])
); );
const [selectedClientId, setSelectedClientId] = useState<string>(() => { const [selectedClientId, setSelectedClientId] = useState<string>(() => {
const cached = getCached<any[]>('vcon:clients', []); const cached = getCached<SdkClientItem[]>('vcon:clients', []);
return cached[0]?.clientId ?? ''; return cached.find((c) => c.status !== 'revoked')?.clientId ?? cached[0]?.clientId ?? '';
}); });
const [selectedClientToken, setSelectedClientToken] = useState<string>('');
const [loadingToken, setLoadingToken] = useState<boolean>(false);
const [serverUrl, setServerUrl] = useState<string>('http://localhost:4000'); const [serverUrl, setServerUrl] = useState<string>('http://localhost:4000');
const [apiUrl, setApiUrl] = useState<string>('http://localhost:3000'); const [apiUrl, setApiUrl] = useState<string>('http://localhost:3000');
@@ -34,33 +44,81 @@ export default function SdkGuidePage() {
async function loadClients() { async function loadClients() {
try { try {
const res = await fetch('/api/clients'); const res = await fetch('/api/clients', { cache: 'no-store' });
if (res.status === 401) {
window.location.assign('/vcon/login');
return;
}
if (res.ok) { if (res.ok) {
const data = await res.json(); const data = await res.json();
const list = setCached('vcon:clients', data.clients || []); const list = setCached<SdkClientItem[]>('vcon:clients', data.clients || []);
setClients(list); setClients(list);
if (list.length > 0) { if (list.length > 0) {
setSelectedClientId((prev) => setSelectedClientId((prev) =>
list.some((c: any) => c.clientId === prev) ? prev : list[0].clientId list.some((c) => c.clientId === prev)
? prev
: list.find((c) => c.status !== 'revoked')?.clientId ?? list[0].clientId
); );
} }
} }
} catch { } catch {
// ignore // Retain cached list on transient network error
} }
} }
loadClients(); void loadClients();
}, []); }, []);
useEffect(() => {
if (!selectedClientId) {
setSelectedClientToken('');
return;
}
let cancelled = false;
setLoadingToken(true);
async function fetchToken() {
try {
const res = await fetch(
`/api/clients/${encodeURIComponent(selectedClientId)}?includeToken=true`,
{ cache: 'no-store' }
);
if (res.status === 401) {
window.location.assign('/vcon/login');
return;
}
if (res.ok) {
const data = await res.json();
if (!cancelled) {
setSelectedClientToken(String(data?.client?.token || ''));
}
} else if (!cancelled) {
setSelectedClientToken('');
}
} catch {
if (!cancelled) setSelectedClientToken('');
} finally {
if (!cancelled) setLoadingToken(false);
}
}
void fetchToken();
return () => {
cancelled = true;
};
}, [selectedClientId]);
const activeClient = const activeClient =
clients.find((c) => c.clientId === selectedClientId) || clients[0]; clients.find((c) => c.clientId === selectedClientId) || clients[0];
const sampleToken = activeClient?.token || 'YOUR_SIGNED_CLIENT_JWT_TOKEN'; const resolvedToken = selectedClientToken || activeClient?.token || '';
const sampleToken = resolvedToken || 'YOUR_SIGNED_CLIENT_JWT_TOKEN';
const configSnippet = `# config.datastack.py — Place in your Python project root const configSnippet = `# config.datastack.py — Place in your Python project root
DATASTACK_CONFIG = { DATASTACK_CONFIG = {
"SERVER_URL": "${serverUrl}", "SERVER_URL": "${serverUrl}",
"API_URL": "${apiUrl}", "API_URL": "${apiUrl}",
"CLIENT_TOKEN": "${sampleToken}", "CLIENT_TOKEN": "${sampleToken}",
"CLIENT_ID": "${activeClient?.clientId || ''}",
"AUTO_CONNECT": True, "AUTO_CONNECT": True,
"ENABLE_HTTP_FALLBACK": True, "ENABLE_HTTP_FALLBACK": True,
"HEARTBEAT_INTERVAL_SEC": 5.0, "HEARTBEAT_INTERVAL_SEC": 5.0,
@@ -99,9 +157,21 @@ print("SDK Status:", client.status())
client.disconnect()`; client.disconnect()`;
const copySnippet = (text: string, label: string) => { const copySnippet = async (text: string, label: string) => {
navigator.clipboard.writeText(text); try {
success('Copied', `${label} copied`); if (!navigator?.clipboard?.writeText) {
throw new Error('Clipboard API unavailable in this browser context');
}
await navigator.clipboard.writeText(text);
success('Copied', `${label} copied`);
} catch (copyErr) {
error(
'Copy failed',
copyErr instanceof Error
? copyErr.message
: 'Unable to copy to clipboard. Copy manually from the code block.'
);
}
}; };
return ( return (
@@ -109,12 +179,13 @@ client.disconnect()`;
title="Python SDK" title="Python SDK"
actions={ actions={
<> <>
{activeClient?.token && ( {resolvedToken && (
<Button <Button
size="xs" size="xs"
variant="secondary" variant="secondary"
icon="Key" icon="Key"
onClick={() => copySnippet(activeClient.token, 'Client JWT token')} disabled={loadingToken}
onClick={() => void copySnippet(resolvedToken, 'Client JWT token')}
> >
Copy Token Copy Token
</Button> </Button>
@@ -123,7 +194,7 @@ client.disconnect()`;
size="xs" size="xs"
variant="secondary" variant="secondary"
icon="FileCode" icon="FileCode"
onClick={() => copySnippet(configSnippet, 'config.datastack.py')} onClick={() => void copySnippet(configSnippet, 'config.datastack.py')}
> >
Copy Config Copy Config
</Button> </Button>
@@ -131,7 +202,7 @@ client.disconnect()`;
size="xs" size="xs"
variant="secondary" variant="secondary"
icon="Copy" icon="Copy"
onClick={() => copySnippet(pythonSnippet, 'Python SDK snippet')} onClick={() => void copySnippet(pythonSnippet, 'Python SDK snippet')}
> >
Copy Code Copy Code
</Button> </Button>
@@ -161,8 +232,9 @@ client.disconnect()`;
size="xs" size="xs"
variant="ghost" variant="ghost"
icon="Copy" icon="Copy"
aria-label="Copy pip install command"
onClick={() => onClick={() =>
copySnippet('pip install ./sdk/python/datastack', 'Pip command') void copySnippet('pip install ./sdk/python/datastack', 'Pip command')
} }
/> />
</Card> </Card>
@@ -176,6 +248,7 @@ client.disconnect()`;
<div className="w-[190px]"> <div className="w-[190px]">
<Select <Select
compact compact
aria-label="Select SDK client"
value={selectedClientId} value={selectedClientId}
onChange={(e) => setSelectedClientId(e.target.value)} onChange={(e) => setSelectedClientId(e.target.value)}
> >
@@ -191,7 +264,8 @@ client.disconnect()`;
size="xs" size="xs"
variant="ghost" variant="ghost"
icon="Copy" icon="Copy"
onClick={() => copySnippet(configSnippet, 'config.datastack.py')} aria-label="Copy config.datastack.py"
onClick={() => void copySnippet(configSnippet, 'config.datastack.py')}
/> />
</div> </div>
} }
@@ -207,7 +281,8 @@ client.disconnect()`;
size="xs" size="xs"
variant="ghost" variant="ghost"
icon="Copy" icon="Copy"
onClick={() => copySnippet(pythonSnippet, 'Python SDK snippet')} aria-label="Copy Python SDK snippet"
onClick={() => void copySnippet(pythonSnippet, 'Python SDK snippet')}
/> />
} }
> >
+310 -43
View File
@@ -5,20 +5,40 @@ import { Button } from '@/components/ui/Button';
import { CodeBlock } from '@/components/ui/CodeBlock'; import { CodeBlock } from '@/components/ui/CodeBlock';
import { Select, Textarea } from '@/components/ui/Input'; import { Select, Textarea } from '@/components/ui/Input';
import { Panel } from '@/components/ui/Panel'; import { Panel } from '@/components/ui/Panel';
import { Pill } from '@/components/ui/Pill';
import { StatStrip } from '@/components/ui/StatStrip'; import { StatStrip } from '@/components/ui/StatStrip';
import { useToast } from '@/components/ui/Toast'; import { useToast } from '@/components/ui/Toast';
import { getCached, setCached } from '@/lib/vconCache'; import { getCached, setCached } from '@/lib/vconCache';
import Link from 'next/link';
import { useEffect, useState } from 'react'; import { useEffect, useState } from 'react';
import { io } from 'socket.io-client';
interface SimulatorClient {
clientId: string;
name: string;
status: 'active' | 'revoked';
}
type TransportMode = 'http' | 'websocket';
type DispatchStatus = 'Ready' | 'Accepted' | 'Rejected' | 'Error';
function getSocketOrigin(): string {
const isLocalDev = window.location.port === '3000';
const protocol = window.location.protocol === 'https:' ? 'wss' : 'ws';
const host = isLocalDev ? `${window.location.hostname}:4000` : window.location.host;
return `${protocol}://${host}`;
}
export default function SimulatorPage() { export default function SimulatorPage() {
const { success, error } = useToast(); const { success, warning, error } = useToast();
const [clients, setClients] = useState<any[]>(() => const [clients, setClients] = useState<SimulatorClient[]>(() =>
getCached<any[]>('vcon:clients', []) getCached<SimulatorClient[]>('vcon:clients', [])
); );
const [selectedClient, setSelectedClient] = useState<string>(() => { const [selectedClient, setSelectedClient] = useState<string>(() => {
const cached = getCached<any[]>('vcon:clients', []); const cached = getCached<SimulatorClient[]>('vcon:clients', []);
return cached[0]?.clientId ?? ''; return cached.find((c) => c.status === 'active')?.clientId ?? cached[0]?.clientId ?? '';
}); });
const [transport, setTransport] = useState<TransportMode>('http');
const [payloadText, setPayloadText] = useState<string>( const [payloadText, setPayloadText] = useState<string>(
JSON.stringify( JSON.stringify(
{ {
@@ -33,30 +53,136 @@ export default function SimulatorPage() {
) )
); );
const [sending, setSending] = useState(false); const [sending, setSending] = useState(false);
const [lastResponse, setLastResponse] = useState<any>(null); const [dispatchStatus, setDispatchStatus] = useState<DispatchStatus>('Ready');
const [lastResponse, setLastResponse] = useState<Record<string, unknown> | null>(null);
useEffect(() => { useEffect(() => {
async function loadClients() { async function loadClients() {
try { try {
const res = await fetch('/api/clients'); const res = await fetch('/api/clients', { cache: 'no-store' });
if (res.status === 401) {
window.location.assign('/vcon/login');
return;
}
if (res.ok) { if (res.ok) {
const data = await res.json(); const data = await res.json();
const list = setCached('vcon:clients', data.clients || []); const list = setCached<SimulatorClient[]>('vcon:clients', data.clients || []);
setClients(list); setClients(list);
if (list.length > 0) setSelectedClient((prev) => {
setSelectedClient((prev) => prev || list[0].clientId); if (prev && list.some((c) => c.clientId === prev)) return prev;
return list.find((c) => c.status === 'active')?.clientId ?? list[0]?.clientId ?? '';
});
} }
} catch { } catch {
// ignore // Ignore transient failure and retain cached list
} }
} }
loadClients(); void loadClients();
}, []); }, []);
const fetchClientJwt = async (clientId: string): Promise<string> => {
const res = await fetch(`/api/clients/${encodeURIComponent(clientId)}?includeToken=true`, {
cache: 'no-store',
});
if (res.status === 401) {
window.location.assign('/vcon/login');
throw new Error('Authentication required');
}
const data = await res.json();
if (!res.ok || !data?.client?.token) {
throw new Error(data?.error || 'Unable to retrieve client JWT token');
}
return String(data.client.token);
};
const emitSocketEvents = async (
clientId: string,
items: Array<Record<string, unknown>>
): Promise<{ accepted: number; rejected: number; responses: Record<string, unknown>[] }> => {
const token = await fetchClientJwt(clientId);
const socketUrl = getSocketOrigin();
return new Promise((resolve, reject) => {
const socket = io(socketUrl, {
auth: { token },
transports: ['websocket', 'polling'],
reconnection: false,
timeout: 4000,
});
const timer = window.setTimeout(() => {
socket.disconnect();
reject(new Error('WebSocket handshake timed out (is Port 4000 ingestion server running?)'));
}, 4500);
socket.on('connect_error', (connectErr) => {
window.clearTimeout(timer);
socket.disconnect();
reject(new Error(connectErr.message || 'WebSocket authentication failed'));
});
socket.on('connect', () => {
const responses: Record<string, unknown>[] = [];
let accepted = 0;
let rejected = 0;
let settled = 0;
const finishIfDone = () => {
if (settled >= items.length) {
window.clearTimeout(timer);
socket.disconnect();
resolve({ accepted, rejected, responses });
}
};
const onAck = (ack: Record<string, unknown>) => {
responses.push(ack);
if (ack?.status === 'ok' || ack?.status === 'duplicate') {
accepted++;
} else {
rejected++;
}
settled++;
finishIfDone();
};
const onErrorEvent = (errPayload: Record<string, unknown>) => {
responses.push(errPayload);
rejected++;
settled++;
finishIfDone();
};
socket.on('ack', onAck);
socket.on('error_event', onErrorEvent);
items.forEach((data, index) => {
const eventId = `sim_ws_${Date.now()}_${index}`;
socket.emit('telemetry_event', {
clientId,
eventId,
timestamp: new Date().toISOString(),
data,
});
});
});
});
};
const handleSendSingle = async () => { const handleSendSingle = async () => {
let parsedData: any; if (!selectedClient) {
error('No target client', 'Register a client in /vcon/clients first.');
return;
}
let parsedData: Record<string, unknown>;
try { try {
parsedData = JSON.parse(payloadText); const raw = JSON.parse(payloadText);
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
error('Invalid payload', 'Payload must be a JSON object.');
return;
}
parsedData = raw as Record<string, unknown>;
} catch { } catch {
error('Invalid JSON', 'Fix payload syntax'); error('Invalid JSON', 'Fix payload syntax');
return; return;
@@ -64,54 +190,145 @@ export default function SimulatorPage() {
setSending(true); setSending(true);
try { try {
if (transport === 'websocket') {
const result = await emitSocketEvents(selectedClient, [parsedData]);
const first = result.responses[0] || {};
setLastResponse({ transport: 'websocket', ...first });
if (result.accepted > 0) {
setDispatchStatus('Accepted');
success('WebSocket Event Accepted', `${String(first.eventId || selectedClient)} acknowledged`);
} else {
setDispatchStatus('Rejected');
error('WebSocket Event Rejected', String(first.message || first.error || 'Rejected by Port 4000'));
}
return;
}
const res = await fetch('/api/events', { const res = await fetch('/api/events', {
method: 'POST', method: 'POST',
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ body: JSON.stringify({
clientId: selectedClient || 'py_telemetry_agent_01', clientId: selectedClient,
data: parsedData, data: parsedData,
}), }),
}); });
if (res.status === 401) {
window.location.assign('/vcon/login');
return;
}
const data = await res.json(); const data = await res.json();
setLastResponse(data); setLastResponse({ transport: 'http', status: res.status, ...data });
if (res.ok) { if (res.ok) {
success('Event Accepted', `${data.event.eventId} buffered`); setDispatchStatus('Accepted');
success('Event Accepted', `${data.event?.eventId || selectedClient} buffered`);
} else { } else {
setDispatchStatus('Rejected');
error('Event Rejected', data.error || 'Server rejected'); error('Event Rejected', data.error || 'Server rejected');
} }
} catch (err: any) { } catch (err: unknown) {
error('Error', err.message); const message = err instanceof Error ? err.message : 'Dispatch error';
setDispatchStatus('Error');
setLastResponse({ transport, error: message });
error('Dispatch failed', message);
} finally { } finally {
setSending(false); setSending(false);
} }
}; };
const handleSendBatch = async (count = 10) => { const handleSendBatch = async (count = 10) => {
if (!selectedClient) {
error('No target client', 'Register a client in /vcon/clients first.');
return;
}
const items = Array.from({ length: count }, (_, i) => ({
batch_index: i + 1,
sensor: `sensor_${(i % 3) + 1}`,
reading: +(Math.random() * 100).toFixed(2),
timestamp: new Date().toISOString(),
}));
setSending(true); setSending(true);
try { try {
if (transport === 'websocket') {
const result = await emitSocketEvents(selectedClient, items);
setLastResponse({
transport: 'websocket',
batchSent: count,
accepted: result.accepted,
rejected: result.rejected,
sample: result.responses.slice(0, 3),
});
if (result.accepted === 0) {
setDispatchStatus('Rejected');
error('WebSocket Batch Rejected', `0/${count} events ingested`);
} else if (result.accepted < count) {
setDispatchStatus('Rejected');
warning('WebSocket Batch Partial', `${result.accepted}/${count} ingested (${result.rejected} rejected)`);
} else {
setDispatchStatus('Accepted');
success('WebSocket Batch Streamed', `${result.accepted}/${count} ingested`);
}
return;
}
let accepted = 0; let accepted = 0;
for (let i = 0; i < count; i++) { let rejected = 0;
const errors: string[] = [];
for (const item of items) {
const res = await fetch('/api/events', { const res = await fetch('/api/events', {
method: 'POST', method: 'POST',
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ body: JSON.stringify({
clientId: selectedClient || 'py_telemetry_agent_01', clientId: selectedClient,
data: { data: item,
batch_index: i + 1,
sensor: `sensor_${(i % 3) + 1}`,
reading: +(Math.random() * 100).toFixed(2),
timestamp: new Date().toISOString(),
},
}), }),
}); });
if (res.ok) accepted++; if (res.status === 401) {
window.location.assign('/vcon/login');
return;
}
if (res.ok) {
accepted++;
} else {
rejected++;
try {
const errJson = await res.json();
if (errJson?.error && errors.length < 3) {
errors.push(String(errJson.error));
}
} catch {
// ignore non-json error body
}
}
} }
success('Batch Streamed', `${accepted}/${count} ingested`);
setLastResponse({ success: true, batchSent: count, accepted }); setLastResponse({
} catch (err: any) { transport: 'http',
error('Batch Error', err.message); batchSent: count,
accepted,
rejected,
...(errors.length > 0 ? { errors } : {}),
});
if (accepted === 0) {
setDispatchStatus('Rejected');
error('Batch Rejected', `0/${count} ingested${errors[0] ? ` (${errors[0]})` : ''}`);
} else if (accepted < count) {
setDispatchStatus('Rejected');
warning('Batch Partial', `${accepted}/${count} ingested (${rejected} rejected)`);
} else {
setDispatchStatus('Accepted');
success('Batch Streamed', `${accepted}/${count} ingested`);
}
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Batch dispatch error';
setDispatchStatus('Error');
setLastResponse({ transport, error: message });
error('Batch Error', message);
} finally { } finally {
setSending(false); setSending(false);
} }
@@ -126,7 +343,8 @@ export default function SimulatorPage() {
size="xs" size="xs"
variant="secondary" variant="secondary"
loading={sending} loading={sending}
onClick={() => handleSendBatch(10)} disabled={sending || !selectedClient}
onClick={() => void handleSendBatch(10)}
> >
Burst 10 Burst 10
</Button> </Button>
@@ -135,44 +353,93 @@ export default function SimulatorPage() {
variant="primary" variant="primary"
icon="Send" icon="Send"
loading={sending} loading={sending}
onClick={handleSendSingle} disabled={sending || !selectedClient}
onClick={() => void handleSendSingle()}
> >
Send Event Send Event
</Button> </Button>
</> </>
} }
> >
{clients.length === 0 && (
<div
role="alert"
className="mb-3 flex flex-wrap items-center justify-between gap-2 border border-[var(--warning)] bg-[var(--warning-soft)] px-3 py-2 text-[11.5px] text-[var(--text)]"
>
<span>No registered SDK clients found. Register a client before dispatching events.</span>
<Link href="/vcon/clients" className="font-medium text-[var(--accent)] underline">
Go to Clients →
</Link>
</div>
)}
<StatStrip <StatStrip
items={[ items={[
{ {
label: 'Target client', label: 'Target client',
value: selectedClient || 'py_telemetry_agent_01', value: selectedClient || 'None selected',
dot: 'ok', dot: selectedClient ? 'ok' : 'off',
},
{
label: 'Ingest transport',
value: transport === 'http' ? 'POST /api/events' : 'WS Port 4000',
}, },
{ label: 'Ingest route', value: 'POST /api/events' },
{ label: 'Payload format', value: 'JSON' }, { label: 'Payload format', value: 'JSON' },
{ {
label: 'Last dispatch', label: 'Last dispatch',
value: lastResponse ? 'Accepted' : 'Ready', value: dispatchStatus,
dot:
dispatchStatus === 'Accepted'
? 'ok'
: dispatchStatus === 'Ready'
? undefined
: 'off',
}, },
]} ]}
/> />
<div className="grid grid-cols-1 gap-3 lg:grid-cols-2"> <div className="grid grid-cols-1 gap-3 lg:grid-cols-2">
<Panel title="Payload" bodyClassName="flex flex-col gap-2.5 p-3"> <Panel title="Payload" bodyClassName="flex flex-col gap-2.5 p-3">
<div className="flex flex-wrap items-center justify-between gap-2">
<span className="text-[10px] uppercase tracking-wider text-[var(--text-faint)]">
Transport
</span>
<div className="flex gap-1" role="group" aria-label="Ingest transport">
<Pill
active={transport === 'http'}
onClick={() => setTransport('http')}
>
HTTP /api/events
</Pill>
<Pill
active={transport === 'websocket'}
onClick={() => setTransport('websocket')}
>
WebSocket Port 4000
</Pill>
</div>
</div>
<Select <Select
compact compact
label="Target client"
value={selectedClient} value={selectedClient}
disabled={clients.length === 0}
onChange={(e) => setSelectedClient(e.target.value)} onChange={(e) => setSelectedClient(e.target.value)}
> >
{clients.map((c) => ( {clients.length === 0 ? (
<option key={c.clientId} value={c.clientId}> <option value="">No registered clients</option>
{c.clientId} ) : (
</option> clients.map((c) => (
))} <option key={c.clientId} value={c.clientId}>
{c.clientId} ({c.status})
</option>
))
)}
</Select> </Select>
<Textarea <Textarea
label="Event JSON"
rows={7} rows={7}
value={payloadText} value={payloadText}
onChange={(e) => setPayloadText(e.target.value)} onChange={(e) => setPayloadText(e.target.value)}
+208 -234
View File
File diff suppressed because it is too large. Load diff
+3 -3
View File
@@ -12,9 +12,9 @@ export const Footer: React.FC<FooterProps> = ({ isAdmin = false }) => {
<footer className="shrink-0 min-h-9 bg-[var(--bg-activity)] border-t border-[var(--border)] px-4 sm:px-6 py-2 flex flex-wrap items-center justify-between gap-2 text-[11px] text-[var(--text-dim)] select-none"> <footer className="shrink-0 min-h-9 bg-[var(--bg-activity)] border-t border-[var(--border)] px-4 sm:px-6 py-2 flex flex-wrap items-center justify-between gap-2 text-[11px] text-[var(--text-dim)] select-none">
<span>DataStack</span> <span>DataStack</span>
<nav aria-label="Footer navigation" className="flex items-center gap-5"> <nav aria-label="Footer navigation" className="flex items-center gap-5">
<Link href="#features" className="hover:text-[var(--text-muted)]">Features</Link> <Link href="/#features" className="hover:text-[var(--text-muted)]">Features</Link>
<Link href="#sdk" className="hover:text-[var(--text-muted)]">SDK</Link> <Link href="/#sdk" className="hover:text-[var(--text-muted)]">SDK</Link>
<Link href="#about" className="hover:text-[var(--text-muted)]">About</Link> <Link href="/#about" className="hover:text-[var(--text-muted)]">About</Link>
</nav> </nav>
</footer> </footer>
); );
+12 -8
View File
@@ -1,6 +1,7 @@
'use client'; 'use client';
import { Button } from '@/components/ui/Button'; import { PWAInstallButton } from '@/components/pwa/PwaLifecycle';
import { Button, buttonClassNames } from '@/components/ui/Button';
import Link from 'next/link'; import Link from 'next/link';
import { usePathname } from 'next/navigation'; import { usePathname } from 'next/navigation';
import React, { useEffect, useState } from 'react'; import React, { useEffect, useState } from 'react';
@@ -95,12 +96,14 @@ export const Navbar: React.FC<NavbarProps> = ({
</div> </div>
<nav className="hidden sm:flex items-center gap-5 text-[12px]"> <nav className="hidden sm:flex items-center gap-5 text-[12px]">
<Link href="#features" className="text-[var(--text-muted)] hover:text-[var(--text)] whitespace-nowrap shrink-0">Features</Link> <Link href="/#features" className="text-[var(--text-muted)] hover:text-[var(--text)] whitespace-nowrap shrink-0">Features</Link>
<Link href="#sdk" className="text-[var(--text-muted)] hover:text-[var(--text)] whitespace-nowrap shrink-0">SDK</Link> <Link href="/#sdk" className="text-[var(--text-muted)] hover:text-[var(--text)] whitespace-nowrap shrink-0">SDK</Link>
<Link href="#about" className="text-[var(--text-muted)] hover:text-[var(--text)] whitespace-nowrap shrink-0">About</Link> <Link href="/#about" className="text-[var(--text-muted)] hover:text-[var(--text)] whitespace-nowrap shrink-0">About</Link>
</nav> </nav>
<div className="flex items-center gap-2 shrink-0"> <div className="flex items-center gap-2 shrink-0">
<PWAInstallButton />
<Button <Button
type="button" type="button"
variant="icon" variant="icon"
@@ -112,10 +115,11 @@ export const Navbar: React.FC<NavbarProps> = ({
className="h-7 w-7 border border-[var(--border)] bg-[var(--bg-elevated)]" className="h-7 w-7 border border-[var(--border)] bg-[var(--bg-elevated)]"
/> />
<Link href="/vcon/dashboard"> <Link
<Button size="xs" variant="primary"> href="/vcon/dashboard"
Console className={buttonClassNames({ size: 'xs', variant: 'primary' })}
</Button> >
Console
</Link> </Link>
</div> </div>
</header> </header>
+175
View File
@@ -0,0 +1,175 @@
'use client';
import { Button } from '@/components/ui/Button';
import { Icon } from '@/components/ui/Icon';
import React, { useEffect, useState } from 'react';
interface BeforeInstallPromptEvent extends Event {
prompt: () => Promise<void>;
userChoice: Promise<{ outcome: 'accepted' | 'dismissed'; platform: string }>;
}
export function usePWAInstall() {
const [deferredPrompt, setDeferredPrompt] = useState<BeforeInstallPromptEvent | null>(null);
const [isInstalled, setIsInstalled] = useState(false);
const [isIOS, setIsIOS] = useState(false);
useEffect(() => {
const isStandalone =
window.matchMedia('(display-mode: standalone)').matches ||
(window.navigator as unknown as { standalone?: boolean }).standalone === true;
setIsInstalled(isStandalone);
const ua = window.navigator.userAgent.toLowerCase();
setIsIOS(/iphone|ipad|ipod/.test(ua));
const handleBeforeInstallPrompt = (e: Event) => {
e.preventDefault();
setDeferredPrompt(e as BeforeInstallPromptEvent);
};
const handleAppInstalled = () => {
setIsInstalled(true);
setDeferredPrompt(null);
};
window.addEventListener('beforeinstallprompt', handleBeforeInstallPrompt);
window.addEventListener('appinstalled', handleAppInstalled);
return () => {
window.removeEventListener('beforeinstallprompt', handleBeforeInstallPrompt);
window.removeEventListener('appinstalled', handleAppInstalled);
};
}, []);
const install = async () => {
if (!deferredPrompt) return false;
await deferredPrompt.prompt();
const { outcome } = await deferredPrompt.userChoice;
if (outcome === 'accepted') {
setIsInstalled(true);
setDeferredPrompt(null);
return true;
}
return false;
};
return {
isInstallable: Boolean(deferredPrompt),
isInstalled,
isIOS,
install,
};
}
export const PWAInstallButton: React.FC = () => {
const { isInstallable, isInstalled, isIOS, install } = usePWAInstall();
const [showIOSGuide, setShowIOSGuide] = useState(false);
if (isInstalled) return null;
if (isInstallable) {
return (
<Button
type="button"
size="xs"
variant="secondary"
icon="Download"
onClick={() => void install()}
>
Install
</Button>
);
}
if (isIOS) {
return (
<>
<Button
type="button"
size="xs"
variant="secondary"
icon="Download"
onClick={() => setShowIOSGuide(true)}
>
Install
</Button>
{showIOSGuide && (
<div className="fixed inset-0 z-50 flex items-center justify-center bg-black/60 p-4">
<div className="w-full max-w-sm border border-[var(--border-strong)] bg-[var(--surface)] p-5">
<div className="mb-2 flex items-center justify-between gap-2">
<h3 className="text-[13px] font-medium text-[var(--text)]">
Install DataStack on iOS
</h3>
<button
type="button"
aria-label="Close install instructions"
onClick={() => setShowIOSGuide(false)}
className="text-[var(--text-muted)] hover:text-[var(--text)]"
>
<Icon name="X" size={14} />
</button>
</div>
<p className="text-[12px] leading-relaxed text-[var(--text-muted)]">
1. Tap the <strong className="text-[var(--text)]">Share</strong> icon in Safari.
<br />
2. Select <strong className="text-[var(--text)]">Add to Home Screen</strong>.
</p>
<div className="mt-4 flex justify-end">
<Button
type="button"
size="xs"
variant="primary"
onClick={() => setShowIOSGuide(false)}
>
Done
</Button>
</div>
</div>
</div>
)}
</>
);
}
return null;
};
export const PwaLifecycle: React.FC = () => {
const [isOnline, setIsOnline] = useState(true);
useEffect(() => {
setIsOnline(typeof navigator !== 'undefined' ? navigator.onLine : true);
const onOnline = () => setIsOnline(true);
const onOffline = () => setIsOnline(false);
window.addEventListener('online', onOnline);
window.addEventListener('offline', onOffline);
if (typeof navigator !== 'undefined' && 'serviceWorker' in navigator) {
navigator.serviceWorker.register('/sw.js', { scope: '/' }).catch(() => {
// Non-fatal in environments without SW origin permissions
});
}
return () => {
window.removeEventListener('online', onOnline);
window.removeEventListener('offline', onOffline);
};
}, []);
if (isOnline) return null;
return (
<div
role="status"
className="fixed bottom-4 left-4 z-50 flex items-center gap-2 border border-[var(--warning)] bg-[var(--surface)] px-3 py-1.5 text-[11px] font-mono text-[var(--text)]"
>
<Icon name="WifiOff" size={12} className="text-[var(--warning)]" />
<span>Offline mode — cached shell active</span>
</div>
);
};
export default PwaLifecycle;
+2 -2
View File
@@ -19,8 +19,8 @@ export function AdminLogin({
}: { }: {
showDemoHint?: boolean; showDemoHint?: boolean;
}) { }) {
const [email, setEmail] = useState(showDemoHint ? 'admin@datastack.dev' : ''); const [email, setEmail] = useState('');
const [password, setPassword] = useState(showDemoHint ? 'admin12345678' : ''); const [password, setPassword] = useState('');
const [remember, setRemember] = useState(true); const [remember, setRemember] = useState(true);
const [showPassword, setShowPassword] = useState(false); const [showPassword, setShowPassword] = useState(false);
const [isSubmitting, setIsSubmitting] = useState(false); const [isSubmitting, setIsSubmitting] = useState(false);
+1
View File
@@ -17,6 +17,7 @@ const FILTERS: { key: ClientStatusFilter; label: string }[] = [
{ key: 'online', label: 'Online' }, { key: 'online', label: 'Online' },
{ key: 'offline', label: 'Offline' }, { key: 'offline', label: 'Offline' },
{ key: 'pending', label: 'Pending' }, { key: 'pending', label: 'Pending' },
{ key: 'revoked', label: 'Revoked' },
]; ];
export function ClientsFilters({ export function ClientsFilters({
+149 -10
View File
@@ -2,7 +2,7 @@
import { Button } from '@/components/ui/Button'; import { Button } from '@/components/ui/Button';
import { CopyButton } from '@/components/ui/CopyButton'; import { CopyButton } from '@/components/ui/CopyButton';
import { Textarea } from '@/components/ui/Input'; import { Input, Textarea } from '@/components/ui/Input';
import { Modal } from '@/components/ui/Modal'; import { Modal } from '@/components/ui/Modal';
import { import {
Table, Table,
@@ -43,8 +43,61 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
type: 'rotate' | 'revoke' | 'delete'; type: 'rotate' | 'revoke' | 'delete';
client: ClientItem; client: ClientItem;
} | null>(null); } | null>(null);
const [editClient, setEditClient] = useState<ClientItem | null>(null);
const [editName, setEditName] = useState('');
const [editRateLimit, setEditRateLimit] = useState('100');
const [editError, setEditError] = useState('');
const [processing, setProcessing] = useState(false); const [processing, setProcessing] = useState(false);
const openEditModal = (client: ClientItem) => {
setEditClient(client);
setEditName(client.name);
setEditRateLimit(String(client.rateLimit || 100));
setEditError('');
};
const executeEditClient = async (event: React.FormEvent<HTMLFormElement>) => {
event.preventDefault();
if (!editClient) return;
const trimmedName = editName.trim();
const numericRateLimit = Number(editRateLimit);
if (!trimmedName) {
setEditError('Display name is required.');
return;
}
if (!Number.isFinite(numericRateLimit) || numericRateLimit < 1 || numericRateLimit > 10000) {
setEditError('Rate limit must be between 1 and 10,000 events/s.');
return;
}
setEditError('');
setProcessing(true);
try {
const response = await fetch(`/api/clients/${encodeURIComponent(editClient.clientId)}`, {
method: 'PATCH',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
name: trimmedName,
rateLimit: Math.floor(numericRateLimit),
}),
});
if (response.status === 401) {
window.location.assign('/vcon/login');
return;
}
const data = await response.json();
if (!response.ok) throw new Error(data.error || 'Unable to update client');
success('Client updated', `${editClient.clientId} settings saved.`);
setEditClient(null);
await onStatusChange();
} catch (updateError) {
error('Client update failed', updateError instanceof Error ? updateError.message : 'Unknown error');
} finally {
setProcessing(false);
}
};
const executeRotateKey = async (client: ClientItem) => { const executeRotateKey = async (client: ClientItem) => {
setProcessing(true); setProcessing(true);
try { try {
@@ -53,6 +106,10 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ action: 'regenerate_token' }), body: JSON.stringify({ action: 'regenerate_token' }),
}); });
if (response.status === 401) {
window.location.assign('/vcon/login');
return;
}
const data = await response.json(); const data = await response.json();
if (!response.ok) throw new Error(data.error || 'Unable to rotate credentials'); if (!response.ok) throw new Error(data.error || 'Unable to rotate credentials');
setConfirmAction(null); setConfirmAction(null);
@@ -74,6 +131,10 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ status }), body: JSON.stringify({ status }),
}); });
if (response.status === 401) {
window.location.assign('/vcon/login');
return;
}
const data = await response.json(); const data = await response.json();
if (!response.ok) throw new Error(data.error || 'Unable to update client status'); if (!response.ok) throw new Error(data.error || 'Unable to update client status');
setConfirmAction(null); setConfirmAction(null);
@@ -92,6 +153,10 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
const response = await fetch(`/api/clients/${encodeURIComponent(client.clientId)}`, { const response = await fetch(`/api/clients/${encodeURIComponent(client.clientId)}`, {
method: 'DELETE', method: 'DELETE',
}); });
if (response.status === 401) {
window.location.assign('/vcon/login');
return;
}
const data = await response.json(); const data = await response.json();
if (!response.ok) throw new Error(data.error || 'Unable to delete client'); if (!response.ok) throw new Error(data.error || 'Unable to delete client');
setConfirmAction(null); setConfirmAction(null);
@@ -130,7 +195,7 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
<TableHeader> <TableHeader>
<TableRow> <TableRow>
<TableHead>Client</TableHead> <TableHead>Client</TableHead>
<TableHead>API key</TableHead> <TableHead>Credentials</TableHead>
<TableHead className="text-right">Events</TableHead> <TableHead className="text-right">Events</TableHead>
<TableHead>Last seen</TableHead> <TableHead>Last seen</TableHead>
<TableHead className="text-right">Actions</TableHead> <TableHead className="text-right">Actions</TableHead>
@@ -141,15 +206,18 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
<TableEmpty colSpan={5} message="No matching clients" /> <TableEmpty colSpan={5} message="No matching clients" />
) : ( ) : (
clients.map((client) => { clients.map((client) => {
const revoked = client.status === 'revoked';
const online = client.status === 'active' && Boolean(client.lastActive) && const online = client.status === 'active' && Boolean(client.lastActive) &&
now - new Date(client.lastActive!).getTime() < 60_000; now - new Date(client.lastActive!).getTime() < 60_000;
const pending = client.status === 'active' && !client.lastActive; const pending = client.status === 'active' && !client.lastActive;
const label = pending ? 'pending' : online ? 'online' : 'offline'; const label = revoked ? 'revoked' : pending ? 'pending' : online ? 'online' : 'offline';
const dotClass = online const dotClass = revoked
? 'bg-[var(--accent)]' ? 'bg-[var(--danger)]'
: pending : online
? 'bg-[var(--warning)]' ? 'bg-[var(--accent)]'
: 'bg-[var(--text-faint)]'; : pending
? 'bg-[var(--warning)]'
: 'bg-[var(--text-faint)]';
return ( return (
<TableRow key={client.clientId}> <TableRow key={client.clientId}>
@@ -157,13 +225,25 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
<div className="flex min-w-0 items-center gap-2"> <div className="flex min-w-0 items-center gap-2">
<span className={`h-1.5 w-1.5 shrink-0 ${dotClass}`} title={label} /> <span className={`h-1.5 w-1.5 shrink-0 ${dotClass}`} title={label} />
<div className="min-w-0 leading-tight"> <div className="min-w-0 leading-tight">
<div className="truncate font-medium text-[var(--text)]">{client.name}</div> <div className="flex items-center gap-1.5">
<span className="truncate font-medium text-[var(--text)]">{client.name}</span>
{revoked && (
<span className="border border-[var(--danger)] bg-[var(--danger-soft)] px-1 py-0 text-[9px] uppercase tracking-wider text-[var(--danger)]">
revoked
</span>
)}
</div>
<div className="mt-0.5 truncate font-mono text-[10.5px] text-[var(--text-faint)]">{client.clientId}</div> <div className="mt-0.5 truncate font-mono text-[10.5px] text-[var(--text-faint)]">{client.clientId}</div>
</div> </div>
</div> </div>
</TableCell> </TableCell>
<TableCell className="font-mono"> <TableCell className="font-mono">
sec_•••••••• <div className="truncate text-[11px] text-[var(--text)]">
{client.tokenPreview || 'JWT Active'}
</div>
<div className="mt-0.5 text-[10px] text-[var(--text-faint)]">
{client.rateLimit}/s limit
</div>
</TableCell> </TableCell>
<TableCell className="text-right font-mono tabular-nums"> <TableCell className="text-right font-mono tabular-nums">
{client.totalEvents.toLocaleString()} {client.totalEvents.toLocaleString()}
@@ -184,6 +264,16 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
aria-label={`View events for ${client.clientId}`} aria-label={`View events for ${client.clientId}`}
onClick={() => router.push(`/vcon/data?clientId=${encodeURIComponent(client.clientId)}`)} onClick={() => router.push(`/vcon/data?clientId=${encodeURIComponent(client.clientId)}`)}
/> />
<Button
type="button"
variant="icon"
size="icon"
icon="Pencil"
className="h-6 w-6 border border-transparent hover:border-[var(--border)] hover:bg-[var(--surface-2)]"
title={`Edit ${client.clientId}`}
aria-label={`Edit ${client.clientId}`}
onClick={() => openEditModal(client)}
/>
<Button <Button
type="button" type="button"
variant="icon" variant="icon"
@@ -224,6 +314,55 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
</Table> </Table>
</TableWrap> </TableWrap>
<Modal
isOpen={Boolean(editClient)}
onClose={() => !processing && setEditClient(null)}
title={`Edit client: ${editClient?.clientId || ''}`}
maxWidth="xs"
footer={
<>
<Button type="button" variant="ghost" size="xs" disabled={processing} onClick={() => setEditClient(null)}>
Cancel
</Button>
<Button
type="submit"
form="edit-client-form"
variant="primary"
size="xs"
loading={processing}
>
Save changes
</Button>
</>
}
>
<form id="edit-client-form" onSubmit={executeEditClient} className="flex flex-col gap-2.5">
<Input
label="Display name"
compact
value={editName}
onChange={(event) => setEditName(event.target.value)}
placeholder="Service display name"
required
/>
<Input
label="Rate limit (events/s)"
compact
type="number"
min="1"
max="10000"
value={editRateLimit}
onChange={(event) => setEditRateLimit(event.target.value)}
required
/>
{editError && (
<p role="alert" className="text-[11px] text-[var(--danger)]">
{editError}
</p>
)}
</form>
</Modal>
<Modal <Modal
isOpen={Boolean(confirmAction)} isOpen={Boolean(confirmAction)}
onClose={() => !processing && setConfirmAction(null)} onClose={() => !processing && setConfirmAction(null)}
+84 -28
View File
@@ -9,6 +9,7 @@ import {
Table, Table,
TableBody, TableBody,
TableCell, TableCell,
TableEmpty,
TableHead, TableHead,
TableHeader, TableHeader,
TableRow, TableRow,
@@ -26,6 +27,7 @@ type EventItem = {
}; };
type DashboardMetrics = { type DashboardMetrics = {
activeSockets?: number;
activeClients: number; activeClients: number;
totalClients: number; totalClients: number;
bufferStats: { bufferStats: {
@@ -62,6 +64,7 @@ export const DashboardOverview: React.FC = () => {
getCached<StorageFile[]>('vcon:storage-files', []) getCached<StorageFile[]>('vcon:storage-files', [])
); );
const [isRefreshing, setIsRefreshing] = useState(false); const [isRefreshing, setIsRefreshing] = useState(false);
const [fetchError, setFetchError] = useState('');
const refresh = useCallback(async () => { const refresh = useCallback(async () => {
setIsRefreshing(true); setIsRefreshing(true);
@@ -74,25 +77,58 @@ export const DashboardOverview: React.FC = () => {
fetch('/api/clients', { cache: 'no-store' }), fetch('/api/clients', { cache: 'no-store' }),
]); ]);
if (
metricsResponse.status === 401 ||
eventsResponse.status === 401 ||
storageResponse.status === 401 ||
clientsResponse.status === 401
) {
window.location.assign('/vcon/login');
return;
}
const errors: string[] = [];
if (metricsResponse.ok) { if (metricsResponse.ok) {
const nextMetrics = await metricsResponse.json(); const nextMetrics = await metricsResponse.json();
setMetrics(setCached('vcon:metrics', nextMetrics)); setMetrics(setCached('vcon:metrics', nextMetrics));
} else {
errors.push('metrics');
} }
if (eventsResponse.ok) { if (eventsResponse.ok) {
const data = await eventsResponse.json(); const data = await eventsResponse.json();
setEvents(setCached('vcon:dashboard-events', data.events ?? [])); setEvents(setCached('vcon:dashboard-events', data.events ?? []));
} else {
errors.push('events');
} }
if (storageResponse.ok) { if (storageResponse.ok) {
const data = await storageResponse.json(); const data = await storageResponse.json();
setFiles(setCached('vcon:storage-files', data.files ?? [])); setFiles(setCached('vcon:storage-files', data.files ?? []));
if (data.stats) setCached('vcon:storage-stats', data.stats); if (data.stats) setCached('vcon:storage-stats', data.stats);
} else {
errors.push('storage');
} }
if (clientsResponse.ok) { if (clientsResponse.ok) {
const data = await clientsResponse.json(); const data = await clientsResponse.json();
setClients(setCached('vcon:clients', data.clients ?? [])); setClients(setCached('vcon:clients', data.clients ?? []));
} else {
errors.push('clients');
}
if (errors.length > 0) {
setFetchError(
`Partial telemetry refresh failed (${errors.join(', ')}). Showing last available snapshot.`
);
} else {
setFetchError('');
} }
} catch { } catch {
// Keep the last successful dashboard snapshot when one request fails. setFetchError(
'Unable to reach telemetry endpoints. Showing last available snapshot.'
);
} finally { } finally {
setIsRefreshing(false); setIsRefreshing(false);
} }
@@ -108,18 +144,21 @@ export const DashboardOverview: React.FC = () => {
const storedRecordsToday = files const storedRecordsToday = files
.filter((file) => file.date === today) .filter((file) => file.date === today)
.reduce((total, file) => total + file.linesCount, 0); .reduce((total, file) => total + file.linesCount, 0);
const eventsToday = const bufferedCount = metrics?.bufferStats.currentBufferSize ?? 0;
storedRecordsToday + (metrics?.bufferStats.currentBufferSize ?? 0); const eventsToday = storedRecordsToday + bufferedCount;
const activeClients = const activeClients =
metrics?.activeClients ?? metrics?.activeClients ??
clients.filter((client) => client.status === 'active').length; clients.filter((client) => client.status === 'active').length;
const totalClients = metrics?.totalClients ?? clients.length; const totalClients = metrics?.totalClients ?? clients.length;
const activeSockets = metrics?.activeSockets ?? 0;
const totalStorage = metrics?.storageStats.totalMB ?? '0.00'; const totalStorage = metrics?.storageStats.totalMB ?? '0.00';
const totalStorageBytes = metrics?.storageStats.totalBytes ?? 0; const totalStorageBytes = metrics?.storageStats.totalBytes ?? 0;
const storageValue = const storageValue =
totalStorageBytes < 1024 * 1024 totalStorageBytes === 0
? String(Math.max(1, Math.round(totalStorageBytes / 1024))) ? '0'
: totalStorage; : totalStorageBytes < 1024 * 1024
? String(Math.max(1, Math.round(totalStorageBytes / 1024)))
: totalStorage;
const storageUnit = totalStorageBytes < 1024 * 1024 ? 'KB' : 'MB'; const storageUnit = totalStorageBytes < 1024 * 1024 ? 'KB' : 'MB';
const totalFiles = metrics?.storageStats.totalFiles ?? files.length; const totalFiles = metrics?.storageStats.totalFiles ?? files.length;
const totalRows = const totalRows =
@@ -153,13 +192,22 @@ export const DashboardOverview: React.FC = () => {
</> </>
} }
> >
{fetchError && (
<div
role="alert"
className="mb-3 border border-[var(--danger)] bg-[var(--danger-soft)] px-3 py-2 text-[11.5px] text-[var(--text)]"
>
{fetchError}
</div>
)}
<StatStrip <StatStrip
items={[ items={[
{ label: 'Events today', value: eventsToday.toLocaleString() }, { label: 'Events today', value: eventsToday.toLocaleString() },
{ {
label: 'Active clients', label: 'Active clients · Sockets',
value: `${activeClients} / ${totalClients}`, value: `${activeClients} / ${totalClients} · ${activeSockets} ws`,
dot: activeClients > 0 ? 'ok' : 'off', dot: activeClients > 0 || activeSockets > 0 ? 'ok' : 'off',
}, },
{ label: 'Storage used', value: `${storageValue} ${storageUnit}` }, { label: 'Storage used', value: `${storageValue} ${storageUnit}` },
{ label: 'Stored rows', value: totalRows.toLocaleString() }, { label: 'Stored rows', value: totalRows.toLocaleString() },
@@ -169,7 +217,11 @@ export const DashboardOverview: React.FC = () => {
<div className="grid min-w-0 grid-cols-1 gap-3 lg:grid-cols-[minmax(0,1.85fr)_minmax(0,1fr)]"> <div className="grid min-w-0 grid-cols-1 gap-3 lg:grid-cols-[minmax(0,1.85fr)_minmax(0,1fr)]">
<Panel <Panel
title="Recent events" title="Recent events"
meta={`${events.length}`} meta={
bufferedCount > 0
? `${events.length} · ${bufferedCount} buffered`
: `${events.length}`
}
action={ action={
<Link <Link
href="/vcon/data" href="/vcon/data"
@@ -190,24 +242,28 @@ export const DashboardOverview: React.FC = () => {
</TableRow> </TableRow>
</TableHeader> </TableHeader>
<TableBody> <TableBody>
{events.map((event) => ( {events.length === 0 ? (
<TableRow key={event.eventId}> <TableEmpty colSpan={4} message="No recent events" />
<TableCell className="font-mono tabular-nums text-[var(--text-faint)]"> ) : (
{event.timestamp.includes('T') events.map((event) => (
? new Date(event.timestamp).toLocaleTimeString() <TableRow key={event.eventId}>
: event.timestamp} <TableCell className="font-mono tabular-nums text-[var(--text-faint)]">
</TableCell> {event.timestamp.includes('T')
<TableCell className="font-mono text-[var(--accent)]"> ? new Date(event.timestamp).toLocaleTimeString()
{event.clientId} : event.timestamp}
</TableCell> </TableCell>
<TableCell className="max-w-[380px] truncate font-mono text-[var(--text)]"> <TableCell className="font-mono text-[var(--accent)]">
{event.payload} {event.clientId}
</TableCell> </TableCell>
<TableCell className="text-right font-mono tabular-nums text-[var(--text-faint)]"> <TableCell className="max-w-[380px] truncate font-mono text-[var(--text)]">
{new TextEncoder().encode(event.payload).length} B {event.payload}
</TableCell> </TableCell>
</TableRow> <TableCell className="text-right font-mono tabular-nums text-[var(--text-faint)]">
))} {new TextEncoder().encode(event.payload).length} B
</TableCell>
</TableRow>
))
)}
</TableBody> </TableBody>
</Table> </Table>
</TableWrap> </TableWrap>
+127 -9
View File
@@ -3,7 +3,10 @@
import { PageShell } from '@/components/layout/PageShell'; import { PageShell } from '@/components/layout/PageShell';
import { Button } from '@/components/ui/Button'; import { Button } from '@/components/ui/Button';
import { Card } from '@/components/ui/Card'; import { Card } from '@/components/ui/Card';
import { CodeBlock } from '@/components/ui/CodeBlock';
import { CopyButton } from '@/components/ui/CopyButton';
import { Input, Select } from '@/components/ui/Input'; import { Input, Select } from '@/components/ui/Input';
import { Modal } from '@/components/ui/Modal';
import { Panel } from '@/components/ui/Panel'; import { Panel } from '@/components/ui/Panel';
import { Pill } from '@/components/ui/Pill'; import { Pill } from '@/components/ui/Pill';
import { import {
@@ -34,6 +37,18 @@ interface Client {
const PAGE_SIZE = 12; const PAGE_SIZE = 12;
function formatEventPayload(raw: string): string {
try {
return JSON.stringify(JSON.parse(raw), null, 2);
} catch {
return raw;
}
}
function getPayloadBytes(raw: string): number {
return new TextEncoder().encode(raw).length;
}
export function DataBrowser() { export function DataBrowser() {
const router = useRouter(); const router = useRouter();
const [events, setEvents] = useState<StreamEvent[]>([]); const [events, setEvents] = useState<StreamEvent[]>([]);
@@ -54,6 +69,7 @@ export function DataBrowser() {
const [filtersReady, setFiltersReady] = useState(false); const [filtersReady, setFiltersReady] = useState(false);
const [refreshing, setRefreshing] = useState(false); const [refreshing, setRefreshing] = useState(false);
const [errorMessage, setErrorMessage] = useState(''); const [errorMessage, setErrorMessage] = useState('');
const [selectedEvent, setSelectedEvent] = useState<StreamEvent | null>(null);
const fetchEvents = useCallback(async () => { const fetchEvents = useCallback(async () => {
try { try {
@@ -65,7 +81,11 @@ export function DataBrowser() {
page: String(page), page: String(page),
pageSize: String(PAGE_SIZE), pageSize: String(PAGE_SIZE),
}); });
const response = await fetch(`/api/data?${params}`); const response = await fetch(`/api/data?${params}`, { cache: 'no-store' });
if (response.status === 401) {
window.location.assign('/vcon/login');
return;
}
if (!response.ok) throw new Error('Unable to load events'); if (!response.ok) throw new Error('Unable to load events');
const data = await response.json(); const data = await response.json();
setEvents(data.events || []); setEvents(data.events || []);
@@ -78,8 +98,14 @@ export function DataBrowser() {
}, [appliedFilters, page]); }, [appliedFilters, page]);
useEffect(() => { useEffect(() => {
fetch('/api/clients') fetch('/api/clients', { cache: 'no-store' })
.then((response) => (response.ok ? response.json() : { clients: [] })) .then((response) => {
if (response.status === 401) {
window.location.assign('/vcon/login');
return { clients: [] };
}
return response.ok ? response.json() : { clients: [] };
})
.then((data) => setClients(data.clients || [])) .then((data) => setClients(data.clients || []))
.catch(() => setClients([])); .catch(() => setClients([]));
}, []); }, []);
@@ -131,6 +157,10 @@ export function DataBrowser() {
setRefreshing(false); setRefreshing(false);
}; };
const formattedSelectedPayload = selectedEvent
? formatEventPayload(selectedEvent.payload)
: '';
return ( return (
<PageShell <PageShell
title="Events" title="Events"
@@ -238,7 +268,11 @@ export function DataBrowser() {
<TableEmpty colSpan={6} message="No matching events" /> <TableEmpty colSpan={6} message="No matching events" />
) : ( ) : (
events.map((event) => ( events.map((event) => (
<TableRow key={event.eventId}> <TableRow
key={`${event.clientId}:${event.eventId}`}
className="cursor-pointer"
onClick={() => setSelectedEvent(event)}
>
<TableCell className="font-mono tabular-nums text-[var(--text)]"> <TableCell className="font-mono tabular-nums text-[var(--text)]">
{new Date(event.timestamp).toLocaleTimeString([], { {new Date(event.timestamp).toLocaleTimeString([], {
hour: '2-digit', hour: '2-digit',
@@ -257,13 +291,34 @@ export function DataBrowser() {
{event.payload} {event.payload}
</TableCell> </TableCell>
<TableCell className="hidden font-mono tabular-nums md:table-cell"> <TableCell className="hidden font-mono tabular-nums md:table-cell">
{new TextEncoder().encode(event.payload).length} B {getPayloadBytes(event.payload)} B
</TableCell> </TableCell>
<TableCell className="text-right"> <TableCell className="text-right">
<span className="inline-flex items-center gap-1.5 text-[var(--text-muted)]"> <div className="inline-flex items-center gap-2">
<span className="h-1.5 w-1.5 bg-[var(--accent)]" /> <span className="inline-flex items-center gap-1.5 text-[var(--text-muted)]">
{event.status} <span
</span> className={`h-1.5 w-1.5 ${
event.status === 'buffered'
? 'bg-[var(--warning)]'
: 'bg-[var(--accent)]'
}`}
/>
{event.status}
</span>
<Button
type="button"
variant="icon"
size="icon"
icon="Eye"
className="h-5 w-5 border border-transparent hover:border-[var(--border)] hover:bg-[var(--surface-2)]"
title={`Inspect event ${event.eventId}`}
aria-label={`Inspect event ${event.eventId}`}
onClick={(e) => {
e.stopPropagation();
setSelectedEvent(event);
}}
/>
</div>
</TableCell> </TableCell>
</TableRow> </TableRow>
)) ))
@@ -338,6 +393,69 @@ export function DataBrowser() {
</nav> </nav>
</footer> </footer>
</Panel> </Panel>
<Modal
isOpen={Boolean(selectedEvent)}
onClose={() => setSelectedEvent(null)}
title={`Event payload: ${selectedEvent?.eventId || ''}`}
maxWidth="lg"
footer={
<Button type="button" size="xs" onClick={() => setSelectedEvent(null)}>
Close
</Button>
}
>
{selectedEvent && (
<div className="flex flex-col gap-3">
<div className="grid grid-cols-2 gap-px border border-[var(--border)] bg-[var(--border)] sm:grid-cols-4">
<div className="bg-[var(--surface)] p-2">
<div className="text-[9.5px] uppercase tracking-wider text-[var(--text-faint)]">
Client
</div>
<div className="truncate font-mono text-[11.5px] text-[var(--accent)]">
{selectedEvent.clientId}
</div>
</div>
<div className="bg-[var(--surface)] p-2">
<div className="text-[9.5px] uppercase tracking-wider text-[var(--text-faint)]">
Status
</div>
<div className="font-mono text-[11.5px] text-[var(--text)]">
{selectedEvent.status}
</div>
</div>
<div className="bg-[var(--surface)] p-2">
<div className="text-[9.5px] uppercase tracking-wider text-[var(--text-faint)]">
Size
</div>
<div className="font-mono text-[11.5px] tabular-nums text-[var(--text)]">
{getPayloadBytes(selectedEvent.payload)} B
</div>
</div>
<div className="bg-[var(--surface)] p-2">
<div className="text-[9.5px] uppercase tracking-wider text-[var(--text-faint)]">
Timestamp
</div>
<div className="truncate font-mono text-[11px] tabular-nums text-[var(--text-muted)]">
{selectedEvent.timestamp}
</div>
</div>
</div>
<div className="flex flex-col gap-1">
<div className="flex items-center justify-between gap-2">
<span className="text-[10px] uppercase tracking-wider text-[var(--text-faint)]">
JSON / Raw Payload
</span>
<CopyButton value={formattedSelectedPayload} />
</div>
<CodeBlock className="max-h-[340px] border border-[var(--border)] text-[11px]">
{formattedSelectedPayload}
</CodeBlock>
</div>
</div>
)}
</Modal>
</PageShell> </PageShell>
); );
} }
+46 -9
View File
@@ -56,6 +56,7 @@ export function ExportWorkspace() {
const [files, setFiles] = useState<StorageFile[]>(() => const [files, setFiles] = useState<StorageFile[]>(() =>
getCached<StorageFile[]>('vcon:storage-files', []) getCached<StorageFile[]>('vcon:storage-files', [])
); );
const [bufferedCount, setBufferedCount] = useState<number>(0);
const [selectedClient, setSelectedClient] = useState('all'); const [selectedClient, setSelectedClient] = useState('all');
const [fromDate, setFromDate] = useState(''); const [fromDate, setFromDate] = useState('');
const [toDate, setToDate] = useState(''); const [toDate, setToDate] = useState('');
@@ -69,19 +70,33 @@ export function ExportWorkspace() {
useEffect(() => { useEffect(() => {
Promise.all([ Promise.all([
fetch('/api/clients').then(async (response) => { fetch('/api/clients', { cache: 'no-store' }).then(async (response) => {
if (response.status === 401) {
window.location.assign('/vcon/login');
return { clients: [] };
}
if (!response.ok) throw new Error('Unable to load clients'); if (!response.ok) throw new Error('Unable to load clients');
return response.json(); return response.json();
}), }),
fetch('/api/storage').then(async (response) => { fetch('/api/storage', { cache: 'no-store' }).then(async (response) => {
if (response.status === 401) {
window.location.assign('/vcon/login');
return { files: [] };
}
if (!response.ok) throw new Error('Unable to load stored files'); if (!response.ok) throw new Error('Unable to load stored files');
return response.json(); return response.json();
}), }),
fetch('/api/metrics', { cache: 'no-store' })
.then(async (response) => (response.ok ? response.json() : null))
.catch(() => null),
]) ])
.then(([clientData, storageData]) => { .then(([clientData, storageData, metricsData]) => {
setClients(setCached('vcon:clients', clientData.clients || [])); setClients(setCached('vcon:clients', clientData.clients || []));
setFiles(setCached('vcon:storage-files', storageData.files || [])); setFiles(setCached('vcon:storage-files', storageData.files || []));
if (storageData.stats) setCached('vcon:storage-stats', storageData.stats); if (storageData.stats) setCached('vcon:storage-stats', storageData.stats);
if (metricsData?.bufferStats) {
setBufferedCount(Number(metricsData.bufferStats.currentBufferSize) || 0);
}
setLoadError(''); setLoadError('');
}) })
.catch((fetchError) => { .catch((fetchError) => {
@@ -162,15 +177,24 @@ export function ExportWorkspace() {
return `/api/export?${params.toString()}`; return `/api/export?${params.toString()}`;
}, [format, fromDate, selectedClient, toDate]); }, [format, fromDate, selectedClient, toDate]);
const invalidRange = Boolean(fromDate && toDate && fromDate > toDate);
const nothingToExport = preview.rows === 0 && bufferedCount === 0;
const downloadExport = () => { const downloadExport = () => {
if (fromDate && toDate && fromDate > toDate) { if (invalidRange) {
error('Invalid date range', 'Start date must be on or before end date.'); error('Invalid date range', 'Start date must be on or before end date.');
return; return;
} }
if (nothingToExport) {
error('Nothing to export', 'No stored rows or buffered events match the selected filters.');
return;
}
window.location.assign(exportUrl); window.location.assign(exportUrl);
info( info(
'Export started', 'Export started',
`${format.toUpperCase()} download for ${preview.rows.toLocaleString()} rows.` bufferedCount > 0
? `${format.toUpperCase()} download (${preview.rows.toLocaleString()} stored + ${bufferedCount} buffered auto-flushed).`
: `${format.toUpperCase()} download for ${preview.rows.toLocaleString()} rows.`
); );
}; };
@@ -307,17 +331,30 @@ export function ExportWorkspace() {
</strong>{' '} </strong>{' '}
files files
</span> </span>
{bufferedCount > 0 && (
<>
<span aria-hidden="true">·</span>
<span className="text-[var(--warning)]">
+{bufferedCount} buffered (flushed on export)
</span>
</>
)}
</div> </div>
{loadError && ( {loadError && (
<p role="alert" className="text-[11px] text-[var(--text-muted)]"> <p role="alert" className="text-[11px] text-[var(--danger)]">
{loadError} {loadError}
</p> </p>
)} )}
{fromDate && toDate && fromDate > toDate && ( {invalidRange && (
<p role="alert" className="text-[11px] text-[var(--text-muted)]"> <p role="alert" className="text-[11px] text-[var(--danger)]">
Start date must be on or before end date. Start date must be on or before end date.
</p> </p>
)} )}
{!loading && !invalidRange && nothingToExport && (
<p role="status" className="text-[11px] text-[var(--text-faint)]">
No stored rows or buffered events match the selected filters.
</p>
)}
</div> </div>
<div className="flex flex-wrap items-center justify-end gap-1.5 border-t border-[var(--border)] pt-2.5"> <div className="flex flex-wrap items-center justify-end gap-1.5 border-t border-[var(--border)] pt-2.5">
<Button type="button" size="xs" onClick={resetExport}> <Button type="button" size="xs" onClick={resetExport}>
@@ -328,7 +365,7 @@ export function ExportWorkspace() {
variant="primary" variant="primary"
size="xs" size="xs"
icon="Download" icon="Download"
disabled={loading || Boolean(fromDate && toDate && fromDate > toDate)} disabled={loading || invalidRange || nothingToExport}
onClick={downloadExport} onClick={downloadExport}
> >
Download {format.toUpperCase()} Download {format.toUpperCase()}
+5 -5
View File
@@ -1,5 +1,5 @@
import { ActivityPreview } from '@/components/ui/ActivityPreview'; import { ActivityPreview } from '@/components/ui/ActivityPreview';
import { Button } from '@/components/ui/Button'; import { buttonClassNames } from '@/components/ui/Button';
import Link from 'next/link'; import Link from 'next/link';
import React from 'react'; import React from 'react';
@@ -17,11 +17,11 @@ export const Hero: React.FC = () => (
Keep activity organized, review the records that matter, and export them when you need them. Keep activity organized, review the records that matter, and export them when you need them.
</p> </p>
<div className="flex flex-wrap gap-2"> <div className="flex flex-wrap gap-2">
<Link href="#features"> <Link href="/#features" className={buttonClassNames({ variant: 'primary', size: 'sm' })}>
<Button variant="primary" size="sm">Explore features</Button> Explore features
</Link> </Link>
<Link href="#sdk"> <Link href="/#sdk" className={buttonClassNames({ variant: 'secondary', size: 'sm' })}>
<Button variant="secondary" size="sm">Python SDK</Button> Python SDK
</Link> </Link>
</div> </div>
<ActivityPreview /> <ActivityPreview />
+3 -3
View File
@@ -1,4 +1,4 @@
import { Button } from '@/components/ui/Button'; import { buttonClassNames } from '@/components/ui/Button';
import Link from 'next/link'; import Link from 'next/link';
import React from 'react'; import React from 'react';
@@ -8,8 +8,8 @@ export const LandingAbout: React.FC = () => (
<h2 className="mb-1 text-[20px] font-medium text-[var(--text)]">Ready to simplify your data workflow?</h2> <h2 className="mb-1 text-[20px] font-medium text-[var(--text)]">Ready to simplify your data workflow?</h2>
<p className="text-[13px] text-[var(--text-muted)]">Start with the SDK guide and make the workspace your own.</p> <p className="text-[13px] text-[var(--text-muted)]">Start with the SDK guide and make the workspace your own.</p>
</div> </div>
<Link href="#sdk"> <Link href="/#sdk" className={buttonClassNames({ variant: 'primary', size: 'sm' })}>
<Button variant="primary" size="sm">View Python SDK</Button> View Python SDK
</Link> </Link>
</section> </section>
); );
+181 -28
View File
@@ -4,6 +4,7 @@ import { PageShell } from '@/components/layout/PageShell';
import { Button } from '@/components/ui/Button'; import { Button } from '@/components/ui/Button';
import { FormSwitch } from '@/components/ui/Form'; import { FormSwitch } from '@/components/ui/Form';
import { Input } from '@/components/ui/Input'; import { Input } from '@/components/ui/Input';
import { Modal } from '@/components/ui/Modal';
import { import {
SettingsField, SettingsField,
SettingsFooter, SettingsFooter,
@@ -14,7 +15,7 @@ import {
import { StatStrip } from '@/components/ui/StatStrip'; import { StatStrip } from '@/components/ui/StatStrip';
import { useToast } from '@/components/ui/Toast'; import { useToast } from '@/components/ui/Toast';
import { getCached, setCached } from '@/lib/vconCache'; import { getCached, setCached } from '@/lib/vconCache';
import { useEffect, useMemo, useState } from 'react'; import { useCallback, useEffect, useMemo, useState } from 'react';
interface EditableSettings { interface EditableSettings {
maxEventsPerSec: number; maxEventsPerSec: number;
@@ -50,6 +51,7 @@ const TABS = [
]; ];
type TabId = (typeof TABS)[number]['id']; type TabId = (typeof TABS)[number]['id'];
type SocketHealthState = 'idle' | 'checking' | 'online' | 'offline';
export function SettingsWorkspace() { export function SettingsWorkspace() {
const { success, error } = useToast(); const { success, error } = useToast();
@@ -61,11 +63,15 @@ export function SettingsWorkspace() {
); );
const [activeTab, setActiveTab] = useState<TabId>('general'); const [activeTab, setActiveTab] = useState<TabId>('general');
const [jwtSecret, setJwtSecret] = useState(''); const [jwtSecret, setJwtSecret] = useState('');
const [jwtError, setJwtError] = useState('');
const [confirmRotateOpen, setConfirmRotateOpen] = useState(false);
const [storagePath, setStoragePath] = useState(() => const [storagePath, setStoragePath] = useState(() =>
getCached<string>('vcon:storage-path', '') getCached<string>('vcon:storage-path', '')
); );
const [baseUrl, setBaseUrl] = useState(''); const [baseUrl, setBaseUrl] = useState('');
const [webSocketUrl, setWebSocketUrl] = useState(''); const [webSocketUrl, setWebSocketUrl] = useState('');
const [socketStatus, setSocketStatus] = useState<SocketHealthState>('idle');
const [activeSockets, setActiveSockets] = useState<number>(0);
const [serverPort, setServerPort] = useState(() => const [serverPort, setServerPort] = useState(() =>
getCached<number>('vcon:server-port', 4000) getCached<number>('vcon:server-port', 4000)
); );
@@ -74,26 +80,60 @@ export function SettingsWorkspace() {
); );
const [saving, setSaving] = useState(false); const [saving, setSaving] = useState(false);
const probeSocketServer = useCallback(async (port: number) => {
setSocketStatus('checking');
const isLocalDev = window.location.port === '3000';
const httpHost = isLocalDev
? `${window.location.hostname}:${port}`
: window.location.host;
const healthUrl = `${window.location.protocol}//${httpHost}/health`;
try {
const response = await fetch(healthUrl, {
cache: 'no-store',
signal: AbortSignal.timeout(1500),
});
setSocketStatus(response.ok ? 'online' : 'offline');
} catch {
setSocketStatus('offline');
}
}, []);
useEffect(() => { useEffect(() => {
setBaseUrl(window.location.origin); setBaseUrl(window.location.origin);
const isLocalDev = window.location.port === '3000'; const isLocalDev = window.location.port === '3000';
const socketProtocol = window.location.protocol === 'https:' ? 'wss' : 'ws'; const socketProtocol = window.location.protocol === 'https:' ? 'wss' : 'ws';
const socketHost = isLocalDev const socketHost = isLocalDev
? `${window.location.hostname}:4000` ? `${window.location.hostname}:${serverPort}`
: window.location.host; : window.location.host;
setWebSocketUrl(`${socketProtocol}://${socketHost}`); setWebSocketUrl(`${socketProtocol}://${socketHost}`);
Promise.all([ Promise.all([
fetch('/api/settings').then(async (response) => { fetch('/api/settings', { cache: 'no-store' }).then(async (response) => {
if (response.status === 401) {
window.location.assign('/vcon/login');
return null;
}
if (!response.ok) throw new Error('Unable to load settings'); if (!response.ok) throw new Error('Unable to load settings');
return response.json(); return response.json();
}), }),
fetch('/api/storage').then(async (response) => { fetch('/api/storage', { cache: 'no-store' }).then(async (response) => {
if (response.status === 401) {
window.location.assign('/vcon/login');
return null;
}
if (!response.ok) throw new Error('Unable to load storage information'); if (!response.ok) throw new Error('Unable to load storage information');
return response.json(); return response.json();
}), }),
fetch('/api/metrics', { cache: 'no-store' })
.then(async (response) => (response.ok ? response.json() : null))
.catch(() => null),
]) ])
.then(([settingsData, storageData]) => { .then(([settingsData, storageData, metricsData]) => {
if (!settingsData || !storageData) return;
const loadedPort = Number(
(settingsData.settings as PublicSettings).serverPort ?? 4000
);
const loaded: EditableSettings = { const loaded: EditableSettings = {
maxEventsPerSec: Number( maxEventsPerSec: Number(
settingsData.settings.maxEventsPerSec ?? settingsData.settings.maxEventsPerSec ??
@@ -126,25 +166,26 @@ export function SettingsWorkspace() {
}; };
setSettings(setCached('vcon:settings', loaded)); setSettings(setCached('vcon:settings', loaded));
setSavedSettings(loaded); setSavedSettings(loaded);
setServerPort( setServerPort(setCached('vcon:server-port', loadedPort));
setCached(
'vcon:server-port',
Number((settingsData.settings as PublicSettings).serverPort ?? 4000)
)
);
setStoragePath( setStoragePath(
setCached( setCached(
'vcon:storage-path', 'vcon:storage-path',
String(storageData.stats?.storageDirectory ?? '') String(storageData.stats?.storageDirectory ?? '')
) )
); );
if (metricsData && typeof metricsData.activeSockets === 'number') {
setActiveSockets(metricsData.activeSockets);
if (metricsData.activeSockets > 0) {
setSocketStatus('online');
}
}
setCached('vcon:settings-loaded', true); setCached('vcon:settings-loaded', true);
}) })
.catch(() => { .catch(() => {
error('Settings load failed', 'Using default runtime settings.'); error('Settings load failed', 'Using default runtime settings.');
}) })
.finally(() => setLoading(false)); .finally(() => setLoading(false));
}, [error]); }, [error, serverPort]);
const isDirty = useMemo( const isDirty = useMemo(
() => () =>
@@ -157,9 +198,9 @@ export function SettingsWorkspace() {
setSettings((current) => ({ ...current, [key]: Number(value) || 0 })); setSettings((current) => ({ ...current, [key]: Number(value) || 0 }));
}; };
const saveSettings = async (event: React.FormEvent<HTMLFormElement>) => { const buildNormalizedSettings = (): EditableSettings => {
event.preventDefault(); const parsedPrune = Number(settings.autoPruneDays);
const normalized: EditableSettings = { return {
maxEventsPerSec: Math.max( maxEventsPerSec: Math.max(
1, 1,
Math.min( Math.min(
@@ -187,11 +228,16 @@ export function SettingsWorkspace() {
settings.fileRotationMb || DEFAULT_SETTINGS.fileRotationMb settings.fileRotationMb || DEFAULT_SETTINGS.fileRotationMb
), ),
enableIdempotency: settings.enableIdempotency, enableIdempotency: settings.enableIdempotency,
autoPruneDays: Math.max( autoPruneDays: Number.isFinite(parsedPrune)
1, ? Math.max(0, Math.min(365, Math.floor(parsedPrune)))
Math.min(365, settings.autoPruneDays || DEFAULT_SETTINGS.autoPruneDays) : DEFAULT_SETTINGS.autoPruneDays,
),
}; };
};
const executeSave = async () => {
const normalized = buildNormalizedSettings();
const trimmedSecret = jwtSecret.trim();
setConfirmRotateOpen(false);
setSaving(true); setSaving(true);
try { try {
const response = await fetch('/api/settings', { const response = await fetch('/api/settings', {
@@ -199,15 +245,25 @@ export function SettingsWorkspace() {
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ body: JSON.stringify({
...normalized, ...normalized,
...(jwtSecret.trim() ? { jwtSecret: jwtSecret.trim() } : {}), ...(trimmedSecret ? { jwtSecret: trimmedSecret } : {}),
}), }),
}); });
if (response.status === 401) {
window.location.assign('/vcon/login');
return;
}
const data = await response.json(); const data = await response.json();
if (!response.ok) throw new Error(data.error || 'Settings update failed'); if (!response.ok) throw new Error(data.error || 'Settings update failed');
setSettings(setCached('vcon:settings', normalized)); setSettings(setCached('vcon:settings', normalized));
setSavedSettings(normalized); setSavedSettings(normalized);
setJwtSecret(''); setJwtSecret('');
success('Settings saved', 'Runtime settings updated.'); setJwtError('');
success(
'Settings saved',
trimmedSecret
? 'Runtime settings saved and client JWT credentials rotated.'
: 'Runtime settings updated.'
);
} catch (saveError) { } catch (saveError) {
error( error(
'Save failed', 'Save failed',
@@ -220,9 +276,30 @@ export function SettingsWorkspace() {
} }
}; };
const handleFormSubmit = async (event: React.FormEvent<HTMLFormElement>) => {
event.preventDefault();
const trimmedSecret = jwtSecret.trim();
if (trimmedSecret.length > 0 && trimmedSecret.length < 32) {
const message = 'JWT secret must be at least 32 characters.';
setJwtError(message);
setActiveTab('security');
error('Invalid JWT secret', message);
return;
}
setJwtError('');
if (trimmedSecret.length >= 32) {
setConfirmRotateOpen(true);
return;
}
await executeSave();
};
const discardChanges = () => { const discardChanges = () => {
setSettings(savedSettings); setSettings(savedSettings);
setJwtSecret(''); setJwtSecret('');
setJwtError('');
}; };
return ( return (
@@ -237,7 +314,9 @@ export function SettingsWorkspace() {
}, },
{ {
label: 'CSV rotation', label: 'CSV rotation',
value: `${settings.fileRotationMb}MB · ${settings.autoPruneDays}d`, value: `${settings.fileRotationMb}MB · ${
settings.autoPruneDays === 0 ? 'Keep all' : `${settings.autoPruneDays}d`
}`,
}, },
]} ]}
/> />
@@ -249,7 +328,7 @@ export function SettingsWorkspace() {
onChange={(tab) => setActiveTab(tab as TabId)} onChange={(tab) => setActiveTab(tab as TabId)}
/> />
<form onSubmit={saveSettings} className="w-full"> <form onSubmit={handleFormSubmit} className="w-full">
<SettingsPanel <SettingsPanel
id={`settings-panel-${activeTab}`} id={`settings-panel-${activeTab}`}
role="tabpanel" role="tabpanel"
@@ -307,6 +386,40 @@ export function SettingsWorkspace() {
{activeTab === 'connections' && ( {activeTab === 'connections' && (
<> <>
<SettingsGroup title="WebSocket server"> <SettingsGroup title="WebSocket server">
<SettingsField label="Live health">
<div className="flex items-center justify-between gap-2 border border-[var(--border)] bg-[var(--surface-2)] px-2.5 py-1.5">
<span className="inline-flex items-center gap-1.5 font-mono text-[11.5px] text-[var(--text)]">
<span
aria-hidden="true"
className={`h-1.5 w-1.5 ${
socketStatus === 'online'
? 'bg-[var(--accent)]'
: socketStatus === 'checking'
? 'bg-[var(--warning)]'
: socketStatus === 'offline'
? 'bg-[var(--danger)]'
: 'bg-[var(--text-faint)]'
}`}
/>
{socketStatus === 'online'
? `Online (Port ${serverPort} · ${activeSockets} active)`
: socketStatus === 'checking'
? 'Probing ingestion server...'
: socketStatus === 'offline'
? `Offline (Port ${serverPort} unreachable)`
: `Port ${serverPort} · ${activeSockets} active socket(s)`}
</span>
<Button
type="button"
size="xs"
variant="ghost"
icon="RefreshCw"
onClick={() => void probeSocketServer(serverPort)}
>
Probe
</Button>
</div>
</SettingsField>
<SettingsField label="WebSocket URL"> <SettingsField label="WebSocket URL">
<Input <Input
compact compact
@@ -378,11 +491,11 @@ export function SettingsWorkspace() {
className="font-mono" className="font-mono"
/> />
</SettingsField> </SettingsField>
<SettingsField label="Retention"> <SettingsField label="Retention (0 = never prune)">
<Input <Input
compact compact
type="number" type="number"
min="1" min="0"
max="365" max="365"
suffix="days" suffix="days"
value={settings.autoPruneDays} value={settings.autoPruneDays}
@@ -408,14 +521,21 @@ export function SettingsWorkspace() {
{activeTab === 'security' && ( {activeTab === 'security' && (
<SettingsGroup title="Authentication"> <SettingsGroup title="Authentication">
<SettingsField label="JWT secret"> <SettingsField label="JWT secret (min 32 chars)">
<Input <Input
compact compact
type="password" type="password"
autoComplete="new-password" autoComplete="new-password"
value={jwtSecret} value={jwtSecret}
onChange={(event) => setJwtSecret(event.target.value)} error={jwtError}
placeholder="New secret..." onChange={(event) => {
const val = event.target.value;
setJwtSecret(val);
if (!val.trim() || val.trim().length >= 32) {
setJwtError('');
}
}}
placeholder="Minimum 32 characters to rotate..."
/> />
</SettingsField> </SettingsField>
<SettingsField label="Payload size limit"> <SettingsField label="Payload size limit">
@@ -471,6 +591,39 @@ export function SettingsWorkspace() {
</SettingsPanel> </SettingsPanel>
</form> </form>
</div> </div>
<Modal
isOpen={confirmRotateOpen}
onClose={() => !saving && setConfirmRotateOpen(false)}
title="Confirm JWT secret rotation"
maxWidth="xs"
footer={
<>
<Button
type="button"
variant="ghost"
size="xs"
disabled={saving}
onClick={() => setConfirmRotateOpen(false)}
>
Cancel
</Button>
<Button
type="button"
variant="danger"
size="xs"
loading={saving}
onClick={() => void executeSave()}
>
Rotate & Save
</Button>
</>
}
>
<p className="text-[12px] text-[var(--text-muted)]">
Rotating the signing JWT secret will immediately re-sign registered client tokens and invalidate all older SDK JWT tokens and active WebSocket sessions. Continue?
</p>
</Modal>
</PageShell> </PageShell>
); );
} }
+7 -1
View File
@@ -6,6 +6,7 @@ export interface ClientItem {
totalEvents: number; totalEvents: number;
lastActive?: string; lastActive?: string;
createdAt: string; createdAt: string;
tokenPreview?: string;
} }
export interface ClientCredentials extends ClientItem { export interface ClientCredentials extends ClientItem {
@@ -13,4 +14,9 @@ export interface ClientCredentials extends ClientItem {
token: string; token: string;
} }
export type ClientStatusFilter = "all" | "online" | "offline" | "pending"; export type ClientStatusFilter =
| "all"
| "online"
| "offline"
| "pending"
| "revoked";
+41 -29
View File
@@ -10,6 +10,46 @@ export interface ButtonProps extends React.ButtonHTMLAttributes<HTMLButtonElemen
loading?: boolean; loading?: boolean;
} }
const BUTTON_BASE_STYLES =
'inline-flex items-center justify-center font-normal transition-colors select-none cursor-pointer disabled:opacity-50 disabled:cursor-not-allowed border border-transparent whitespace-nowrap outline-none';
const BUTTON_VARIANT_STYLES = {
primary:
'bg-[var(--text)] text-[var(--bg)] hover:opacity-90 border-[var(--text)]',
secondary:
'bg-[var(--bg-elevated)] text-[var(--text)] border-[var(--border-strong)] hover:bg-[var(--bg-hover)]',
ghost:
'bg-transparent text-[var(--text-muted)] hover:text-[var(--text)] hover:bg-[var(--bg-hover)] border-transparent',
danger:
'bg-transparent text-[var(--danger)] border-[var(--danger)] hover:bg-[var(--danger-soft)]',
icon:
'bg-transparent text-[var(--text-muted)] hover:text-[var(--text)] hover:bg-[var(--bg-hover)] border-transparent p-0',
} as const;
const BUTTON_SIZE_STYLES = {
default: 'min-h-8.5 px-3 text-[12px] gap-1.5',
sm: 'h-7.5 px-2.5 text-[11.5px] gap-1.5',
xs: 'h-7 px-2 text-[11px] gap-1',
icon: 'w-7.5 h-7.5 p-0',
} as const;
export function buttonClassNames({
variant = 'secondary',
size = 'default',
className,
}: {
variant?: keyof typeof BUTTON_VARIANT_STYLES;
size?: keyof typeof BUTTON_SIZE_STYLES;
className?: string;
} = {}): string {
return cn(
BUTTON_BASE_STYLES,
BUTTON_VARIANT_STYLES[variant],
size === 'icon' ? BUTTON_SIZE_STYLES.icon : BUTTON_SIZE_STYLES[size],
className
);
}
export const Button = React.forwardRef<HTMLButtonElement, ButtonProps>( export const Button = React.forwardRef<HTMLButtonElement, ButtonProps>(
( (
{ {
@@ -25,39 +65,11 @@ export const Button = React.forwardRef<HTMLButtonElement, ButtonProps>(
}, },
ref ref
) => { ) => {
const baseStyles =
'inline-flex items-center justify-center font-normal transition-colors select-none cursor-pointer disabled:opacity-50 disabled:cursor-not-allowed border border-transparent whitespace-nowrap outline-none';
const variantStyles = {
primary:
'bg-[var(--text)] text-[var(--bg)] hover:opacity-90 border-[var(--text)]',
secondary:
'bg-[var(--bg-elevated)] text-[var(--text)] border-[var(--border-strong)] hover:bg-[var(--bg-hover)]',
ghost:
'bg-transparent text-[var(--text-muted)] hover:text-[var(--text)] hover:bg-[var(--bg-hover)] border-transparent',
danger:
'bg-transparent text-[var(--danger)] border-[var(--danger)] hover:bg-[var(--danger-soft)]',
icon:
'bg-transparent text-[var(--text-muted)] hover:text-[var(--text)] hover:bg-[var(--bg-hover)] border-transparent p-0',
};
const sizeStyles = {
default: 'min-h-8.5 px-3 text-[12px] gap-1.5',
sm: 'h-7.5 px-2.5 text-[11.5px] gap-1.5',
xs: 'h-7 px-2 text-[11px] gap-1',
icon: 'w-7.5 h-7.5 p-0',
};
return ( return (
<button <button
ref={ref} ref={ref}
disabled={disabled || loading} disabled={disabled || loading}
className={cn( className={buttonClassNames({ variant, size, className })}
baseStyles,
variantStyles[variant],
size === 'icon' ? sizeStyles.icon : sizeStyles[size],
className
)}
{...props} {...props}
> >
{loading ? ( {loading ? (
+25
View File
@@ -73,6 +73,31 @@ export class DatabaseEngine {
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
) )
`), `),
pool.query(`
CREATE TABLE IF NOT EXISTS deleted_clients (
client_id TEXT PRIMARY KEY,
deleted_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`),
pool.query(`
CREATE TABLE IF NOT EXISTS admin_users (
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
email TEXT NOT NULL UNIQUE,
password_salt BYTEA NOT NULL,
password_hash BYTEA NOT NULL,
token_version INTEGER NOT NULL DEFAULT 1,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`),
pool.query(`
CREATE TABLE IF NOT EXISTS admin_login_attempts (
key_hash TEXT PRIMARY KEY,
window_started_at TIMESTAMPTZ NOT NULL,
failed_attempts INTEGER NOT NULL,
blocked_until TIMESTAMPTZ,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`),
]).then(() => undefined); ]).then(() => undefined);
try { try {
+48 -1
View File
@@ -1,3 +1,38 @@
import { createHash } from "node:crypto";
import { AuthService } from "../auth/jwt";
export function resolveInternalIngestionSecret(
jwtSecretOverride?: string,
): string {
const explicit = (process.env.DATASTACK_INTERNAL_SECRET || "").trim();
const isProd = process.env.NODE_ENV === "production";
if (isProd) {
if (explicit.length >= 32) return explicit;
if (explicit.length > 0) return "";
} else if (explicit.length > 0) {
return explicit;
}
const activeJwt = String(
jwtSecretOverride ?? AuthService.getSecret() ?? "",
).trim();
const minJwtLen = isProd ? 32 : 1;
if (activeJwt.length < minJwtLen) {
return "";
}
return createHash("sha256")
.update(`datastack-internal:${activeJwt}`)
.digest("hex");
}
export function getInternalIngestionHeaders(
jwtSecretOverride?: string,
): Record<string, string> {
const secret = resolveInternalIngestionSecret(jwtSecretOverride);
return secret ? { "x-datastack-internal-secret": secret } : {};
}
export interface ValidationResult { export interface ValidationResult {
allowed: boolean; allowed: boolean;
reason?: string; reason?: string;
@@ -130,7 +165,7 @@ export class SecurityEngine {
if (!withinLimit) { if (!withinLimit) {
return { return {
allowed: false, allowed: false,
reason: `Rate limit of ${clientMaxRate} events/sec exceeded for client ${clientId}`, reason: `Rate limit of ${effectiveRateLimit} events/sec exceeded for client ${clientId}`,
code: "RATE_LIMIT_EXCEEDED", code: "RATE_LIMIT_EXCEEDED",
}; };
} }
@@ -144,6 +179,18 @@ export class SecurityEngine {
} }
public static getStats() { public static getStats() {
const now = Date.now();
for (const [k, timestamp] of this.idempotencySet.entries()) {
if (now - timestamp > this.IDEMPOTENCY_TTL_MS) {
this.idempotencySet.delete(k);
}
}
for (const [clientId, entry] of this.rateLimitMap.entries()) {
if (now - entry.windowStart > 10000) {
this.rateLimitMap.delete(clientId);
}
}
return { return {
activeIdempotencyKeys: this.idempotencySet.size, activeIdempotencyKeys: this.idempotencySet.size,
rateLimitedClientsCount: this.rateLimitMap.size, rateLimitedClientsCount: this.rateLimitMap.size,
+392 -53
View File
@@ -19,10 +19,30 @@ export interface StorageFileMeta {
modifiedAt: Date; modifiedAt: Date;
} }
interface RowCountCacheEntry {
sizeBytes: number;
mtimeMs: number;
linesCount: number;
}
const LOCK_RETRY_MIN_MS = 5;
const LOCK_RETRY_MAX_MS = 18;
const LOCK_TIMEOUT_MS = 10_000;
const STALE_LOCK_AGE_MS = 30_000;
function sleepMs(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
export class CSVStorageManager { export class CSVStorageManager {
// Base storage directory: /storage/data // Base storage directory: /storage/data
public static baseStorageDir = path.resolve(process.cwd(), 'storage', 'data'); public static baseStorageDir = path.resolve(process.cwd(), 'storage', 'data');
public static maxFileSizeBytes = 100 * 1024 * 1024; // 100MB rotation limit public static maxFileSizeBytes = 100 * 1024 * 1024; // 100MB rotation limit
private static rowCountCache = new Map<string, RowCountCacheEntry>();
public static clearRowCountCache(): void {
this.rowCountCache.clear();
}
/** /**
* Ensures client directory exists * Ensures client directory exists
@@ -37,15 +57,194 @@ export class CSVStorageManager {
} }
/** /**
* Resolves target CSV file path with daily and 100MB rotation rules * Acquires an exclusive cooperative lock file for a client+date partition across Port 3000 & Port 4000
*/
public static async withPartitionLock<T>(
clientId: string,
dateStr: string,
fn: () => Promise<T> | T,
): Promise<T> {
const clientDir = this.ensureClientDir(clientId);
const date = this.extractSafePartitionDate(dateStr);
const lockPath = path.join(clientDir, `.${date}.csv.lock`);
const deadline = Date.now() + LOCK_TIMEOUT_MS;
let acquired = false;
while (Date.now() < deadline) {
try {
const fd = fs.openSync(lockPath, 'wx');
try {
fs.writeFileSync(fd, `${process.pid}:${Date.now()}`, 'utf8');
} finally {
fs.closeSync(fd);
}
acquired = true;
break;
} catch (err: unknown) {
const code = (err as NodeJS.ErrnoException)?.code;
if (code !== 'EEXIST') {
throw err;
}
try {
const lockStat = fs.statSync(lockPath);
if (Date.now() - lockStat.mtimeMs > STALE_LOCK_AGE_MS) {
fs.unlinkSync(lockPath);
continue;
}
} catch {
// Lock was released concurrently
}
const jitter =
LOCK_RETRY_MIN_MS +
Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS));
await sleepMs(jitter);
}
}
if (!acquired) {
throw new Error(`Timed out waiting for CSV partition lock: ${lockPath}`);
}
try {
return await fn();
} finally {
try {
if (fs.existsSync(lockPath)) {
fs.unlinkSync(lockPath);
}
} catch {
// Ignore cleanup failure if already unlinked
}
}
}
/**
* Normalizes and validates any timestamp or date string into a strict YYYY-MM-DD partition key
*/
public static extractSafePartitionDate(rawTimestamp?: string): string {
const fallback = new Date().toISOString().slice(0, 10);
if (!rawTimestamp || typeof rawTimestamp !== 'string') {
return fallback;
}
const trimmed = rawTimestamp.trim();
const prefix = trimmed.slice(0, 10);
if (/^\d{4}-\d{2}-\d{2}$/.test(prefix)) {
const parsedPrefix = new Date(`${prefix}T00:00:00.000Z`);
if (!Number.isNaN(parsedPrefix.getTime())) {
return prefix;
}
}
if (/^\d+(\.\d+)?$/.test(trimmed)) {
const num = Number(trimmed);
const ms = num < 1e11 ? num * 1000 : num;
const d = new Date(ms);
if (!Number.isNaN(d.getTime())) {
return d.toISOString().slice(0, 10);
}
}
const parsed = new Date(trimmed);
if (!Number.isNaN(parsed.getTime())) {
return parsed.toISOString().slice(0, 10);
}
return fallback;
}
/**
* Stateful RFC 4180 quote-aware chunked row counter so multi-100MB CSVs never load wholly into RAM
*/
private static createChunkRowCounter() {
let inQuotes = false;
let pendingQuote = false;
let currentLineHasChars = false;
let totalRows = 0;
return {
pushChunk(chunk: Buffer | string) {
const str = typeof chunk === 'string' ? chunk : chunk.toString('utf8');
for (let i = 0; i < str.length; i += 1) {
const ch = str[i];
if (pendingQuote) {
pendingQuote = false;
if (ch === '"') {
// Escaped quote "" across chunk boundary or inline
continue;
}
inQuotes = !inQuotes;
}
if (ch === '"') {
currentLineHasChars = true;
if (inQuotes) {
if (i + 1 < str.length) {
if (str[i + 1] === '"') {
i += 1;
} else {
inQuotes = false;
}
} else {
pendingQuote = true;
}
} else {
inQuotes = true;
}
} else if (ch === '\n' && !inQuotes) {
if (currentLineHasChars) {
totalRows += 1;
currentLineHasChars = false;
}
} else if (ch !== '\r' && ch !== ' ' && ch !== '\t') {
currentLineHasChars = true;
}
}
},
finish(): number {
if (currentLineHasChars) {
totalRows += 1;
}
return Math.max(0, totalRows - 1);
},
};
}
private static countCsvDataRowsChunkedSync(filePath: string): number {
const counter = this.createChunkRowCounter();
const buf = Buffer.allocUnsafe(64 * 1024);
const fd = fs.openSync(filePath, 'r');
try {
let bytesRead = 0;
while ((bytesRead = fs.readSync(fd, buf, 0, buf.length, null)) > 0) {
counter.pushChunk(buf.subarray(0, bytesRead));
}
return counter.finish();
} finally {
fs.closeSync(fd);
}
}
private static async countCsvDataRowsStreamAsync(
filePath: string,
): Promise<number> {
const counter = this.createChunkRowCounter();
return new Promise<number>((resolve, reject) => {
const stream = fs.createReadStream(filePath, {
highWaterMark: 64 * 1024,
});
stream.on('data', (chunk) => counter.pushChunk(chunk));
stream.on('end', () => resolve(counter.finish()));
stream.on('error', (err) => reject(err));
});
}
/**
* Resolves target CSV file path with daily and configured MB rotation rules
*/ */
public static getTargetFilePath(clientId: string, dateStr?: string): string { public static getTargetFilePath(clientId: string, dateStr?: string): string {
const clientDir = this.ensureClientDir(clientId); const clientDir = this.ensureClientDir(clientId);
const date = dateStr || new Date().toISOString().split('T')[0]; const date = this.extractSafePartitionDate(dateStr);
let baseFileName = `${date}.csv`; let baseFileName = `${date}.csv`;
let targetPath = path.join(clientDir, baseFileName); let targetPath = path.join(clientDir, baseFileName);
// Check for 100MB rotation // Check for file rotation
let partIndex = 1; let partIndex = 1;
while (fs.existsSync(targetPath)) { while (fs.existsSync(targetPath)) {
try { try {
@@ -66,10 +265,12 @@ export class CSVStorageManager {
} }
/** /**
* Batch append events into target CSV file. * Batch append events into target CSV file under cooperative partition lock.
* If file doesn't exist, writes header: clientId,eventId,timestamp,payload * If file doesn't exist, writes header: clientId,eventId,timestamp,payload
*/ */
public static async appendBatch(events: DataEvent[]): Promise<{ written: number; filesUpdated: string[] }> { public static async appendBatch(
events: DataEvent[],
): Promise<{ written: number; filesUpdated: string[] }> {
if (!events || events.length === 0) { if (!events || events.length === 0) {
return { written: 0, filesUpdated: [] }; return { written: 0, filesUpdated: [] };
} }
@@ -77,7 +278,7 @@ export class CSVStorageManager {
// Group events by client + date // Group events by client + date
const grouped = new Map<string, DataEvent[]>(); const grouped = new Map<string, DataEvent[]>();
for (const evt of events) { for (const evt of events) {
const date = (evt.timestamp || new Date().toISOString()).split('T')[0]; const date = this.extractSafePartitionDate(evt.timestamp);
const key = `${evt.clientId}:::${date}`; const key = `${evt.clientId}:::${date}`;
if (!grouped.has(key)) { if (!grouped.has(key)) {
grouped.set(key, []); grouped.set(key, []);
@@ -90,31 +291,39 @@ export class CSVStorageManager {
for (const [key, batch] of grouped.entries()) { for (const [key, batch] of grouped.entries()) {
const [clientId, date] = key.split(':::'); const [clientId, date] = key.split(':::');
const targetPath = this.getTargetFilePath(clientId, date); const targetPath = await this.withPartitionLock(clientId, date, () => {
let hasContent = false; const resolvedPath = this.getTargetFilePath(clientId, date);
try { let hasContent = false;
hasContent = try {
fs.existsSync(targetPath) && fs.statSync(targetPath).size > 0; hasContent =
} catch { fs.existsSync(resolvedPath) && fs.statSync(resolvedPath).size > 0;
hasContent = false; } catch {
} hasContent = false;
}
// Format payload as JSON string if not already const cleanBatch = batch.map((item) => ({
const cleanBatch = batch.map((item) => ({ clientId: item.clientId,
clientId: item.clientId, eventId: item.eventId,
eventId: item.eventId, timestamp: item.timestamp,
timestamp: item.timestamp, payload:
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload), typeof item.payload === 'string'
})); ? item.payload
: JSON.stringify(item.payload),
}));
const parser = new Parser({ fields, header: !hasContent, eol: '\n' }); const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
const csvData = parser.parse(cleanBatch); const csvData = parser.parse(cleanBatch);
if (hasContent) { if (hasContent) {
fs.appendFileSync(targetPath, '\n' + csvData, 'utf8'); fs.appendFileSync(resolvedPath, '\n' + csvData, 'utf8');
} else { } else {
fs.writeFileSync(targetPath, csvData, 'utf8'); fs.writeFileSync(resolvedPath, csvData, 'utf8');
} }
// Invalidate row cache for modified file immediately
this.rowCountCache.delete(resolvedPath);
return resolvedPath;
});
filesUpdated.push(targetPath); filesUpdated.push(targetPath);
} }
@@ -123,14 +332,16 @@ export class CSVStorageManager {
} }
/** /**
* List all stored CSV files with sizes and dates * List all stored CSV files with cached row counts (invalidated on mtime/size changes)
*/ */
public static listAllFiles(): StorageFileMeta[] { public static listAllFiles(): StorageFileMeta[] {
const results: StorageFileMeta[] = []; const results: StorageFileMeta[] = [];
if (!fs.existsSync(this.baseStorageDir)) { if (!fs.existsSync(this.baseStorageDir)) {
this.rowCountCache.clear();
return results; return results;
} }
const activePaths = new Set<string>();
try { try {
const clientDirs = fs.readdirSync(this.baseStorageDir); const clientDirs = fs.readdirSync(this.baseStorageDir);
for (const clientDir of clientDirs) { for (const clientDir of clientDirs) {
@@ -142,15 +353,27 @@ export class CSVStorageManager {
if (!file.endsWith('.csv')) continue; if (!file.endsWith('.csv')) continue;
const fullFilePath = path.join(fullClientPath, file); const fullFilePath = path.join(fullClientPath, file);
const stats = fs.statSync(fullFilePath); const stats = fs.statSync(fullFilePath);
activePaths.add(fullFilePath);
// Count lines approximately (or sample) const cached = this.rowCountCache.get(fullFilePath);
let linesCount = 0; let linesCount = 0;
try { if (
const content = fs.readFileSync(fullFilePath, 'utf8'); cached &&
linesCount = content.split('\n').filter((l) => l.trim().length > 0).length - 1; // subtract header cached.sizeBytes === stats.size &&
if (linesCount < 0) linesCount = 0; cached.mtimeMs === stats.mtimeMs
} catch { ) {
linesCount = 0; linesCount = cached.linesCount;
} else {
try {
linesCount = this.countCsvDataRowsChunkedSync(fullFilePath);
this.rowCountCache.set(fullFilePath, {
sizeBytes: stats.size,
mtimeMs: stats.mtimeMs,
linesCount,
});
} catch {
linesCount = 0;
}
} }
const datePart = file.replace('.csv', '').split('-part')[0]; const datePart = file.replace('.csv', '').split('-part')[0];
@@ -166,6 +389,12 @@ export class CSVStorageManager {
}); });
} }
} }
for (const key of this.rowCountCache.keys()) {
if (!activePaths.has(key)) {
this.rowCountCache.delete(key);
}
}
} catch { } catch {
// Return whatever collected // Return whatever collected
} }
@@ -174,12 +403,88 @@ export class CSVStorageManager {
} }
/** /**
* Prune CSV partitions older than autoPruneDays * Non-blocking async listing using fs.promises + streaming row counting on cache misses
*/
public static async listAllFilesAsync(): Promise<StorageFileMeta[]> {
const results: StorageFileMeta[] = [];
if (!fs.existsSync(this.baseStorageDir)) {
this.rowCountCache.clear();
return results;
}
const activePaths = new Set<string>();
try {
const clientDirs = await fs.promises.readdir(this.baseStorageDir);
for (const clientDir of clientDirs) {
const fullClientPath = path.join(this.baseStorageDir, clientDir);
const dirStat = await fs.promises.stat(fullClientPath).catch(() => null);
if (!dirStat || !dirStat.isDirectory()) continue;
const files = await fs.promises.readdir(fullClientPath);
for (const file of files) {
if (!file.endsWith('.csv')) continue;
const fullFilePath = path.join(fullClientPath, file);
const stats = await fs.promises.stat(fullFilePath).catch(() => null);
if (!stats) continue;
activePaths.add(fullFilePath);
const cached = this.rowCountCache.get(fullFilePath);
let linesCount = 0;
if (
cached &&
cached.sizeBytes === stats.size &&
cached.mtimeMs === stats.mtimeMs
) {
linesCount = cached.linesCount;
} else {
try {
linesCount = await this.countCsvDataRowsStreamAsync(fullFilePath);
this.rowCountCache.set(fullFilePath, {
sizeBytes: stats.size,
mtimeMs: stats.mtimeMs,
linesCount,
});
} catch {
linesCount = 0;
}
}
const datePart = file.replace('.csv', '').split('-part')[0];
results.push({
clientId: clientDir,
date: datePart,
filename: file,
filePath: fullFilePath,
sizeBytes: stats.size,
linesCount,
modifiedAt: stats.mtime,
});
}
}
for (const key of this.rowCountCache.keys()) {
if (!activePaths.has(key)) {
this.rowCountCache.delete(key);
}
}
} catch {
// Return whatever collected
}
return results.sort((a, b) => b.modifiedAt.getTime() - a.modifiedAt.getTime());
}
/**
* Prune CSV partitions older than autoPruneDays without counting row contents
*/ */
public static pruneExpiredPartitions(autoPruneDays: number): number { public static pruneExpiredPartitions(autoPruneDays: number): number {
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) { if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
return 0; return 0;
} }
if (!fs.existsSync(this.baseStorageDir)) {
return 0;
}
const cutoffDate = new Date( const cutoffDate = new Date(
Date.now() - Math.floor(autoPruneDays) * 86_400_000, Date.now() - Math.floor(autoPruneDays) * 86_400_000,
@@ -188,28 +493,38 @@ export class CSVStorageManager {
.split('T')[0]; .split('T')[0];
let prunedCount = 0; let prunedCount = 0;
const files = this.listAllFiles(); try {
for (const file of files) { const clientDirs = fs.readdirSync(this.baseStorageDir);
if (/^\d{4}-\d{2}-\d{2}$/.test(file.date) && file.date < cutoffDate) { for (const clientDir of clientDirs) {
try { const fullClientPath = path.join(this.baseStorageDir, clientDir);
if (fs.existsSync(file.filePath)) { if (!fs.statSync(fullClientPath).isDirectory()) continue;
fs.unlinkSync(file.filePath);
prunedCount += 1; const files = fs.readdirSync(fullClientPath);
for (const file of files) {
if (!file.endsWith('.csv')) continue;
const datePart = file.replace('.csv', '').split('-part')[0];
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
const fullPath = path.join(fullClientPath, file);
try {
if (fs.existsSync(fullPath)) {
fs.unlinkSync(fullPath);
this.rowCountCache.delete(fullPath);
prunedCount += 1;
}
} catch {
// Ignore file removal error
}
} }
} catch {
// Ignore file removal error
} }
} }
} catch {
// Ignore directory scan failure
} }
return prunedCount; return prunedCount;
} }
/** private static formatStorageStats(files: StorageFileMeta[]) {
* Get aggregate storage statistics
*/
public static getStorageStats() {
const files = this.listAllFiles();
const totalBytes = files.reduce((acc, f) => acc + f.sizeBytes, 0); const totalBytes = files.reduce((acc, f) => acc + f.sizeBytes, 0);
const totalRows = files.reduce((acc, f) => acc + f.linesCount, 0); const totalRows = files.reduce((acc, f) => acc + f.linesCount, 0);
const uniqueClients = new Set(files.map((f) => f.clientId)).size; const uniqueClients = new Set(files.map((f) => f.clientId)).size;
@@ -231,10 +546,34 @@ export class CSVStorageManager {
} }
/** /**
* Reads file content for export / download * Get aggregate storage statistics (sync cached)
*/
public static getStorageStats() {
return this.formatStorageStats(this.listAllFiles());
}
/**
* Get aggregate storage statistics (non-blocking async)
*/
public static async getStorageStatsAsync() {
const files = await this.listAllFilesAsync();
return this.formatStorageStats(files);
}
/**
* Reads file content for export / download with path traversal protection
*/ */
public static getFileContent(clientId: string, filename: string): string | null { public static getFileContent(clientId: string, filename: string): string | null {
const fullPath = path.join(this.baseStorageDir, clientId, filename); const safeClientId = String(clientId || '').replace(/[^a-zA-Z0-9_-]/g, '_');
const safeFilename = path.basename(String(filename || ''));
if (!safeClientId || !safeFilename || !safeFilename.endsWith('.csv')) {
return null;
}
const fullPath = path.resolve(this.baseStorageDir, safeClientId, safeFilename);
const resolvedBase = path.resolve(this.baseStorageDir);
if (!fullPath.startsWith(resolvedBase + path.sep)) {
return null;
}
if (fs.existsSync(fullPath)) { if (fs.existsSync(fullPath)) {
return fs.readFileSync(fullPath, 'utf8'); return fs.readFileSync(fullPath, 'utf8');
} }
+69 -11
View File
@@ -1,8 +1,11 @@
import { AuthService } from "../auth/jwt"; import { AuthService, resolveDefaultJwtSecret } from "../auth/jwt";
import { DataBufferEngine } from "../data/buffer"; import { DataBufferEngine } from "../data/buffer";
import { DatabaseEngine } from "../Database/database"; import { DatabaseEngine } from "../Database/database";
import { SDKManager } from "../sdk/sdkManager"; import { SDKManager } from "../sdk/sdkManager";
import { SecurityEngine } from "../Security/security"; import {
SecurityEngine,
getInternalIngestionHeaders,
} from "../Security/security";
import { CSVStorageManager } from "../Storage/csvStorage"; import { CSVStorageManager } from "../Storage/csvStorage";
export interface AdminSettings { export interface AdminSettings {
@@ -35,9 +38,7 @@ function getDefaultSettings(): AdminSettings {
batchFlushThreshold: 500, batchFlushThreshold: 500,
batchFlushIntervalMs: 2000, batchFlushIntervalMs: 2000,
fileRotationMb: 100, fileRotationMb: 100,
jwtSecret: jwtSecret: resolveDefaultJwtSecret(),
process.env.DATASTACK_JWT_SECRET ||
"datastack-jwt-default-secret-key-32-chars-long",
enableIdempotency: true, enableIdempotency: true,
autoPruneDays: 30, autoPruneDays: 30,
updatedAt: new Date().toISOString(), updatedAt: new Date().toISOString(),
@@ -62,7 +63,9 @@ export class SettingsManager {
AuthService.setSecret(settings.jwtSecret); AuthService.setSecret(settings.jwtSecret);
} }
CSVStorageManager.maxFileSizeBytes = settings.fileRotationMb * 1024 * 1024; CSVStorageManager.maxFileSizeBytes = settings.fileRotationMb * 1024 * 1024;
CSVStorageManager.pruneExpiredPartitions(settings.autoPruneDays); if (settings.autoPruneDays > 0) {
CSVStorageManager.pruneExpiredPartitions(settings.autoPruneDays);
}
} }
private static async syncFromDatabase(): Promise<AdminSettings | null> { private static async syncFromDatabase(): Promise<AdminSettings | null> {
@@ -83,6 +86,10 @@ export class SettingsManager {
if (result.rowCount && result.rowCount > 0) { if (result.rowCount && result.rowCount > 0) {
const row = result.rows[0]; const row = result.rows[0];
const parsedPruneDays =
row.auto_prune_days !== undefined && row.auto_prune_days !== null
? Number(row.auto_prune_days)
: NaN;
const loaded: AdminSettings = { const loaded: AdminSettings = {
serverPort: Number(row.server_port) || 4000, serverPort: Number(row.server_port) || 4000,
webPort: Number(row.web_port) || 3000, webPort: Number(row.web_port) || 3000,
@@ -91,12 +98,9 @@ export class SettingsManager {
batchFlushThreshold: Number(row.batch_flush_threshold) || 500, batchFlushThreshold: Number(row.batch_flush_threshold) || 500,
batchFlushIntervalMs: Number(row.batch_flush_interval_ms) || 2000, batchFlushIntervalMs: Number(row.batch_flush_interval_ms) || 2000,
fileRotationMb: Number(row.file_rotation_mb) || 100, fileRotationMb: Number(row.file_rotation_mb) || 100,
jwtSecret: jwtSecret: String(row.jwt_secret || "").trim() || resolveDefaultJwtSecret(),
String(row.jwt_secret || "") ||
process.env.DATASTACK_JWT_SECRET ||
"datastack-jwt-default-secret-key-32-chars-long",
enableIdempotency: Boolean(row.enable_idempotency), enableIdempotency: Boolean(row.enable_idempotency),
autoPruneDays: Number(row.auto_prune_days) || 30, autoPruneDays: Number.isFinite(parsedPruneDays) ? parsedPruneDays : 30,
updatedAt: row.updated_at updatedAt: row.updated_at
? new Date(row.updated_at as string | Date).toISOString() ? new Date(row.updated_at as string | Date).toISOString()
: new Date().toISOString(), : new Date().toISOString(),
@@ -214,8 +218,46 @@ export class SettingsManager {
}; };
} }
private static async notifyIngestionSettingsSync(
settings: AdminSettings,
authJwtSecret: string,
jwtSecretChanged: boolean,
): Promise<void> {
const headers = getInternalIngestionHeaders(authJwtSecret);
if (!headers["x-datastack-internal-secret"]) return;
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, "");
try {
await fetch(`${ingestionBaseUrl}/internal/settings-sync`, {
method: "POST",
headers: {
"Content-Type": "application/json",
...headers,
},
body: JSON.stringify({
maxEventsPerSec: settings.maxEventsPerSec,
maxPayloadSizeKb: settings.maxPayloadSizeKb,
batchFlushThreshold: settings.batchFlushThreshold,
batchFlushIntervalMs: settings.batchFlushIntervalMs,
fileRotationMb: settings.fileRotationMb,
enableIdempotency: settings.enableIdempotency,
autoPruneDays: settings.autoPruneDays,
...(jwtSecretChanged ? { jwtSecret: settings.jwtSecret } : {}),
}),
cache: "no-store",
signal: AbortSignal.timeout(1500),
});
} catch {
// Port 4000 may not be running in isolated single-process dev mode
}
}
public static updateSettings(updates: Partial<AdminSettings>): AdminSettings { public static updateSettings(updates: Partial<AdminSettings>): AdminSettings {
const current = getSettingsState(); const current = getSettingsState();
const previousJwtSecret = current.jwtSecret;
const jwtSecretChanged = const jwtSecretChanged =
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret; Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
const nextSettings: AdminSettings = { const nextSettings: AdminSettings = {
@@ -229,6 +271,13 @@ export class SettingsManager {
SDKManager.resignAllClientTokens(); SDKManager.resignAllClientTokens();
} }
void this.persistSettingsToDb(nextSettings); void this.persistSettingsToDb(nextSettings);
void this.notifyIngestionSettingsSync(
nextSettings,
previousJwtSecret,
jwtSecretChanged,
).then(() =>
jwtSecretChanged ? SDKManager.syncAllClientsToIngestionAsync() : undefined,
);
return { ...nextSettings }; return { ...nextSettings };
} }
@@ -236,6 +285,7 @@ export class SettingsManager {
updates: Partial<AdminSettings>, updates: Partial<AdminSettings>,
): Promise<AdminSettings> { ): Promise<AdminSettings> {
const current = await this.getSettingsAsync(); const current = await this.getSettingsAsync();
const previousJwtSecret = current.jwtSecret;
const jwtSecretChanged = const jwtSecretChanged =
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret; Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
const nextSettings: AdminSettings = { const nextSettings: AdminSettings = {
@@ -249,6 +299,14 @@ export class SettingsManager {
await SDKManager.resignAllClientTokensAsync(); await SDKManager.resignAllClientTokensAsync();
} }
await this.persistSettingsToDb(nextSettings); await this.persistSettingsToDb(nextSettings);
await this.notifyIngestionSettingsSync(
nextSettings,
previousJwtSecret,
jwtSecretChanged,
);
if (jwtSecretChanged) {
await SDKManager.syncAllClientsToIngestionAsync();
}
return { ...nextSettings }; return { ...nextSettings };
} }
} }
+70 -13
View File
@@ -1,44 +1,93 @@
import { SettingsManager } from '../adminSettings/settingsManager';
import { SDKManager } from '../sdk/sdkManager'; import { SDKManager } from '../sdk/sdkManager';
import { CSVStorageManager } from '../Storage/csvStorage'; import { CSVStorageManager } from '../Storage/csvStorage';
import { DataBufferEngine } from '../data/buffer'; import { DataBufferEngine } from '../data/buffer';
import { SecurityEngine } from '../Security/security'; import {
SecurityEngine,
getInternalIngestionHeaders,
} from '../Security/security';
export class ApiEngine { export class ApiEngine {
/** /**
* Retrieves overall telemetry and health metrics for the dashboard * Retrieves overall telemetry and health metrics for the dashboard
*/ */
public static async getDashboardMetrics() { public static async getDashboardMetrics() {
const storageStats = CSVStorageManager.getStorageStats(); const settings = await SettingsManager.getSettingsAsync();
const localBufferStats = DataBufferEngine.getBufferStats(); const [storageStats, localBufferStats, clients] = await Promise.all([
const clients = await SDKManager.listClientsAsync(); CSVStorageManager.getStorageStatsAsync(),
DataBufferEngine.getBufferStatsAsync(),
SDKManager.listClientsAsync(),
]);
const activeClients = clients.filter((c) => c.status === 'active').length; const activeClients = clients.filter((c) => c.status === 'active').length;
let activeSockets = 0; let activeSockets = 0;
let ingestionBufferSize = 0; let ingestionLocalBufferSize = 0;
let ingestionRedisBufferSize = 0;
let ingestionRedisConnected = false;
let remoteSecurityStats: {
activeIdempotencyKeys?: number;
rateLimitedClientsCount?: number;
maxPayloadSizeBytes?: number;
} | null = null;
const ingestionBaseUrl = ( const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000' process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
).replace(/\/$/, ''); ).replace(/\/$/, '');
const internalHeaders = getInternalIngestionHeaders();
try { try {
const res = await fetch(`${ingestionBaseUrl}/health`, { const res = await fetch(`${ingestionBaseUrl}/internal/metrics`, {
headers: internalHeaders,
cache: 'no-store', cache: 'no-store',
signal: AbortSignal.timeout(1000),
}); });
if (res.ok) { if (res.ok) {
const data = await res.json(); const data = await res.json();
activeSockets = Number(data.activeSockets) || 0; activeSockets = Number(data.activeSockets) || 0;
ingestionBufferSize = Number(data.bufferSize) || 0; ingestionRedisConnected = Boolean(data.redisConnected);
if (data.localBufferSize !== undefined) {
ingestionLocalBufferSize = Number(data.localBufferSize) || 0;
ingestionRedisBufferSize = Number(data.redisBufferSize) || 0;
} else {
ingestionLocalBufferSize = Number(data.bufferSize) || 0;
}
if (data.securityStats && typeof data.securityStats === 'object') {
remoteSecurityStats = data.securityStats;
}
} }
} catch { } catch {
// Ingestion server may not be running in single-process dev mode // Ingestion server may not be running in single-process dev mode
activeSockets = 0; activeSockets = 0;
ingestionBufferSize = 0; ingestionLocalBufferSize = 0;
ingestionRedisBufferSize = 0;
} }
const securityStats = SecurityEngine.getStats(); const localSecurityStats = SecurityEngine.getStats();
const securityStats = {
activeIdempotencyKeys:
localSecurityStats.activeIdempotencyKeys +
(Number(remoteSecurityStats?.activeIdempotencyKeys) || 0),
rateLimitedClientsCount:
localSecurityStats.rateLimitedClientsCount +
(Number(remoteSecurityStats?.rateLimitedClientsCount) || 0),
maxPayloadSizeBytes: Math.min(
localSecurityStats.maxPayloadSizeBytes,
Number(remoteSecurityStats?.maxPayloadSizeBytes) ||
localSecurityStats.maxPayloadSizeBytes,
),
};
const effectiveRedisBufferSize = Math.max(
localBufferStats.redisBufferSize || 0,
ingestionRedisBufferSize,
);
const bufferStats = { const bufferStats = {
...localBufferStats, ...localBufferStats,
redisConnected: localBufferStats.redisConnected || ingestionRedisConnected,
redisBufferSize: effectiveRedisBufferSize,
currentBufferSize: currentBufferSize:
localBufferStats.currentBufferSize + ingestionBufferSize, localBufferStats.localBufferSize +
ingestionLocalBufferSize +
effectiveRedisBufferSize,
}; };
return { return {
@@ -49,8 +98,8 @@ export class ApiEngine {
storageStats, storageStats,
securityStats, securityStats,
runtime: { runtime: {
serverPort: 4000, serverPort: settings.serverPort || 4000,
webPort: 3000, webPort: settings.webPort || 3000,
uptimeSeconds: Math.floor(process.uptime()), uptimeSeconds: Math.floor(process.uptime()),
memoryUsageMb: (process.memoryUsage().rss / (1024 * 1024)).toFixed(1), memoryUsageMb: (process.memoryUsage().rss / (1024 * 1024)).toFixed(1),
nodeVersion: process.version, nodeVersion: process.version,
@@ -59,7 +108,7 @@ export class ApiEngine {
} }
/** /**
* Filters and streams stored CSV data for export * Filters and lists stored CSV files for export
*/ */
public static getExportFiles(clientId?: string) { public static getExportFiles(clientId?: string) {
const allFiles = CSVStorageManager.listAllFiles(); const allFiles = CSVStorageManager.listAllFiles();
@@ -68,6 +117,14 @@ export class ApiEngine {
} }
return allFiles; return allFiles;
} }
public static async getExportFilesAsync(clientId?: string) {
const allFiles = await CSVStorageManager.listAllFilesAsync();
if (clientId && clientId !== 'all') {
return allFiles.filter((f) => f.clientId === clientId);
}
return allFiles;
}
} }
export default ApiEngine; export default ApiEngine;
+27 -3
View File
@@ -1,8 +1,23 @@
const encoder = new TextEncoder(); const encoder = new TextEncoder();
interface AdminSessionPayload { const DEV_DEFAULT_SESSION_SECRET =
"datastack-admin-session-default-secret-32-chars";
export interface AdminSessionPayload {
email: string; email: string;
expiresAt: number; expiresAt: number;
tokenVersion: number;
}
export function getAdminSessionSecret(): string | null {
const configured = process.env.DATASTACK_ADMIN_SESSION_SECRET?.trim() || "";
if (process.env.NODE_ENV === "production") {
return configured.length >= 32 ? configured : null;
}
if (configured.length > 0) {
return configured.length >= 32 ? configured : null;
}
return DEV_DEFAULT_SESSION_SECRET;
} }
function encodeBase64Url(bytes: Uint8Array) { function encodeBase64Url(bytes: Uint8Array) {
@@ -34,9 +49,14 @@ export async function createAdminSessionToken(
email: string, email: string,
secret: string, secret: string,
expiresAt: number, expiresAt: number,
tokenVersion = 1,
) { ) {
const safeVersion =
Number.isInteger(tokenVersion) && tokenVersion >= 1 ? tokenVersion : 1;
const payload = encodeBase64Url( const payload = encodeBase64Url(
encoder.encode(JSON.stringify({ email, expiresAt })), encoder.encode(
JSON.stringify({ email, expiresAt, tokenVersion: safeVersion }),
),
); );
const signature = await crypto.subtle.sign( const signature = await crypto.subtle.sign(
"HMAC", "HMAC",
@@ -50,6 +70,7 @@ export async function getAdminSessionPayload(
token: string, token: string,
secret: string, secret: string,
): Promise<AdminSessionPayload | null> { ): Promise<AdminSessionPayload | null> {
if (!secret || secret.length < 32) return null;
const [payload, signature, extra] = token.split("."); const [payload, signature, extra] = token.split(".");
if (!payload || !signature || extra) return null; if (!payload || !signature || extra) return null;
@@ -68,7 +89,10 @@ export async function getAdminSessionPayload(
if ( if (
typeof session.email === "string" && typeof session.email === "string" &&
typeof session.expiresAt === "number" && typeof session.expiresAt === "number" &&
session.expiresAt > Date.now() session.expiresAt > Date.now() &&
typeof session.tokenVersion === "number" &&
Number.isInteger(session.tokenVersion) &&
session.tokenVersion >= 1
) { ) {
return session; return session;
} }
+279 -163
View File
@@ -6,15 +6,22 @@ import {
timingSafeEqual, timingSafeEqual,
} from "node:crypto"; } from "node:crypto";
import { Pool, type PoolClient } from "pg"; import { Pool, type PoolClient } from "pg";
import {
type AdminSessionPayload,
getAdminSessionPayload,
getAdminSessionSecret,
} from "./adminSession";
const PASSWORD_HASH_BYTES = 64; const PASSWORD_HASH_BYTES = 64;
const PASSWORD_SALT_BYTES = 16; const PASSWORD_SALT_BYTES = 16;
const DATABASE_LOCK_ID = "72025102501"; const DATABASE_LOCK_ID = "72025102501";
const LOGIN_ATTEMPT_LIMIT = 5; const LOGIN_ATTEMPT_LIMIT = 5;
const IP_LOGIN_ATTEMPT_LIMIT = 25;
const LOGIN_WINDOW_MS = 15 * 60 * 1000; const LOGIN_WINDOW_MS = 15 * 60 * 1000;
const LOGIN_BLOCK_MS = 15 * 60 * 1000; const LOGIN_BLOCK_MS = 15 * 60 * 1000;
const LOCAL_THROTTLE_MAX_ENTRIES = 5000; const LOCAL_THROTTLE_MAX_ENTRIES = 5000;
const LOCAL_THROTTLE_CLEANUP_MS = 60 * 1000; const LOCAL_THROTTLE_CLEANUP_MS = 60 * 1000;
const DEV_DEFAULT_SETUP_KEY = "datastack-default-setup-key-32-chars";
interface LoginThrottleEntry { interface LoginThrottleEntry {
windowStartedAt: number; windowStartedAt: number;
@@ -28,11 +35,17 @@ interface LoginThrottleDecision {
retryAfterSeconds: number; retryAfterSeconds: number;
} }
export interface LoginThrottleBucket {
keyHash: string;
limit: number;
}
interface InMemoryAdmin { interface InMemoryAdmin {
id: number; id: number;
email: string; email: string;
passwordSalt: Buffer; passwordSalt: Buffer;
passwordHash: Buffer; passwordHash: Buffer;
tokenVersion: number;
createdAt: Date; createdAt: Date;
} }
@@ -47,8 +60,12 @@ const localLoginThrottle = new Map<string, LoginThrottleEntry>();
let lastLocalThrottleCleanup = 0; let lastLocalThrottleCleanup = 0;
let lastDatabaseThrottleCleanup = 0; let lastDatabaseThrottleCleanup = 0;
function isProductionEnv() {
return process.env.NODE_ENV === "production";
}
function getPool(): Pool | null { function getPool(): Pool | null {
const connectionString = process.env.DATABASE_URL; const connectionString = process.env.DATABASE_URL?.trim();
if (!connectionString) { if (!connectionString) {
return null; return null;
} }
@@ -74,17 +91,24 @@ async function ensureAdminTable() {
if (!pool) return; if (!pool) return;
globals.__dataStackAdminSchema ??= Promise.all([ globals.__dataStackAdminSchema ??= Promise.all([
pool.query( pool
` .query(
`
CREATE TABLE IF NOT EXISTS admin_users ( CREATE TABLE IF NOT EXISTS admin_users (
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1), id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
email TEXT NOT NULL UNIQUE, email TEXT NOT NULL UNIQUE,
password_salt BYTEA NOT NULL, password_salt BYTEA NOT NULL,
password_hash BYTEA NOT NULL, password_hash BYTEA NOT NULL,
token_version INTEGER NOT NULL DEFAULT 1,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
) )
`, `,
), )
.then(() =>
pool.query(
`ALTER TABLE admin_users ADD COLUMN IF NOT EXISTS token_version INTEGER NOT NULL DEFAULT 1`,
),
),
pool.query( pool.query(
` `
CREATE TABLE IF NOT EXISTS admin_login_attempts ( CREATE TABLE IF NOT EXISTS admin_login_attempts (
@@ -121,8 +145,41 @@ function matchesSecret(candidate: string, expected: string) {
return timingSafeEqual(candidateHash, expectedHash); return timingSafeEqual(candidateHash, expectedHash);
} }
function createLoginThrottleKeys(email: string, secret: string) { export function getAdminSetupKey(): string | null {
return [createHmac("sha256", secret).update(`email:${email}`).digest("hex")]; const configured = process.env.DATASTACK_ADMIN_SETUP_KEY?.trim() || "";
if (isProductionEnv()) {
return configured.length >= 32 ? configured : null;
}
if (configured.length > 0) {
return configured.length >= 32 ? configured : null;
}
return DEV_DEFAULT_SETUP_KEY;
}
export function createLoginThrottleBuckets(
email: string,
secret: string,
ip = "unknown",
): LoginThrottleBucket[] {
const cleanEmail = email.trim().toLowerCase();
const cleanIp = ip.trim().toLowerCase() || "unknown";
const buckets: LoginThrottleBucket[] = [
{
keyHash: createHmac("sha256", secret)
.update(`ip_email:${cleanIp}:${cleanEmail}`)
.digest("hex"),
limit: LOGIN_ATTEMPT_LIMIT,
},
];
if (cleanIp !== "unknown") {
buckets.push({
keyHash: createHmac("sha256", secret)
.update(`ip:${cleanIp}`)
.digest("hex"),
limit: IP_LOGIN_ATTEMPT_LIMIT,
});
}
return buckets;
} }
function getAdvisoryLockId(keyHash: string) { function getAdvisoryLockId(keyHash: string) {
@@ -157,14 +214,14 @@ function cleanupLocalLoginThrottle(now: number) {
} }
function consumeLocalLoginAttempt( function consumeLocalLoginAttempt(
keyHashes: string[], buckets: LoginThrottleBucket[],
now: number, now: number,
): LoginThrottleDecision { ): LoginThrottleDecision {
cleanupLocalLoginThrottle(now); cleanupLocalLoginThrottle(now);
const blockedUntil = keyHashes.reduce( const blockedUntil = buckets.reduce(
(latest, keyHash) => (latest, bucket) =>
Math.max(latest, localLoginThrottle.get(keyHash)?.blockedUntil ?? 0), Math.max(latest, localLoginThrottle.get(bucket.keyHash)?.blockedUntil ?? 0),
0, 0,
); );
if (blockedUntil > now) { if (blockedUntil > now) {
@@ -174,16 +231,16 @@ function consumeLocalLoginAttempt(
}; };
} }
for (const keyHash of keyHashes) { for (const bucket of buckets) {
const previous = localLoginThrottle.get(keyHash); const previous = localLoginThrottle.get(bucket.keyHash);
const windowExpired = const windowExpired =
!previous || now - previous.windowStartedAt >= LOGIN_WINDOW_MS; !previous || now - previous.windowStartedAt >= LOGIN_WINDOW_MS;
const failedAttempts = windowExpired ? 1 : previous.failedAttempts + 1; const failedAttempts = windowExpired ? 1 : previous.failedAttempts + 1;
localLoginThrottle.set(keyHash, { localLoginThrottle.set(bucket.keyHash, {
windowStartedAt: windowExpired ? now : previous.windowStartedAt, windowStartedAt: windowExpired ? now : previous.windowStartedAt,
failedAttempts, failedAttempts,
blockedUntil: blockedUntil:
failedAttempts >= LOGIN_ATTEMPT_LIMIT ? now + LOGIN_BLOCK_MS : 0, failedAttempts >= bucket.limit ? now + LOGIN_BLOCK_MS : 0,
updatedAt: now, updatedAt: now,
}); });
} }
@@ -192,12 +249,13 @@ function consumeLocalLoginAttempt(
} }
async function consumeDatabaseLoginAttempt( async function consumeDatabaseLoginAttempt(
keyHashes: string[], buckets: LoginThrottleBucket[],
now: number, now: number,
): Promise<LoginThrottleDecision> { ): Promise<LoginThrottleDecision> {
const pool = getPool(); const pool = getPool();
if (!pool) return consumeLocalLoginAttempt(keyHashes, now); if (!pool) return consumeLocalLoginAttempt(buckets, now);
await ensureAdminTable(); await ensureAdminTable();
const keyHashes = buckets.map((b) => b.keyHash);
const client = await pool.connect(); const client = await pool.connect();
try { try {
@@ -229,8 +287,8 @@ async function consumeDatabaseLoginAttempt(
}; };
} }
for (const keyHash of keyHashes) { for (const bucket of buckets) {
const previous = entries.get(keyHash); const previous = entries.get(bucket.keyHash);
const windowStartedAt = previous const windowStartedAt = previous
? new Date(previous.window_started_at as string | Date).getTime() ? new Date(previous.window_started_at as string | Date).getTime()
: 0; : 0;
@@ -243,7 +301,7 @@ async function consumeDatabaseLoginAttempt(
? new Date(now) ? new Date(now)
: new Date(windowStartedAt); : new Date(windowStartedAt);
const nextBlockedUntil = const nextBlockedUntil =
failedAttempts >= LOGIN_ATTEMPT_LIMIT failedAttempts >= bucket.limit
? new Date(now + LOGIN_BLOCK_MS) ? new Date(now + LOGIN_BLOCK_MS)
: null; : null;
@@ -256,7 +314,7 @@ async function consumeDatabaseLoginAttempt(
failed_attempts = EXCLUDED.failed_attempts, failed_attempts = EXCLUDED.failed_attempts,
blocked_until = EXCLUDED.blocked_until, blocked_until = EXCLUDED.blocked_until,
updated_at = NOW()`, updated_at = NOW()`,
[keyHash, nextWindowStartedAt, failedAttempts, nextBlockedUntil], [bucket.keyHash, nextWindowStartedAt, failedAttempts, nextBlockedUntil],
); );
} }
@@ -279,13 +337,20 @@ async function consumeDatabaseLoginAttempt(
async function getOrInitInMemoryAdmin(): Promise<InMemoryAdmin> { async function getOrInitInMemoryAdmin(): Promise<InMemoryAdmin> {
if (!globals.__dataStackInMemoryAdmin) { if (!globals.__dataStackInMemoryAdmin) {
const envPassword = process.env.DATASTACK_ADMIN_PASSWORD;
const allowDemo =
!isProductionEnv() && process.env.DATASTACK_ALLOW_DEMO_ADMIN !== "false";
if (!envPassword && !allowDemo) {
throw new Error("Admin account has not completed initial setup.");
}
const defaultEmail = ( const defaultEmail = (
process.env.DATASTACK_ADMIN_EMAIL || "admin@datastack.dev" process.env.DATASTACK_ADMIN_EMAIL || "admin@datastack.dev"
) )
.trim() .trim()
.toLowerCase(); .toLowerCase();
const defaultPassword = const defaultPassword = envPassword || "admin12345678";
process.env.DATASTACK_ADMIN_PASSWORD || "admin12345678";
const salt = randomBytes(PASSWORD_SALT_BYTES); const salt = randomBytes(PASSWORD_SALT_BYTES);
const hash = await derivePasswordHash(defaultPassword, salt); const hash = await derivePasswordHash(defaultPassword, salt);
globals.__dataStackInMemoryAdmin = { globals.__dataStackInMemoryAdmin = {
@@ -293,6 +358,7 @@ async function getOrInitInMemoryAdmin(): Promise<InMemoryAdmin> {
email: defaultEmail, email: defaultEmail,
passwordSalt: salt, passwordSalt: salt,
passwordHash: hash, passwordHash: hash,
tokenVersion: 1,
createdAt: new Date(), createdAt: new Date(),
}; };
} }
@@ -302,24 +368,26 @@ async function getOrInitInMemoryAdmin(): Promise<InMemoryAdmin> {
export async function consumeAdminLoginAttempt( export async function consumeAdminLoginAttempt(
email: string, email: string,
secret: string, secret: string,
ip = "unknown",
): Promise<LoginThrottleDecision> { ): Promise<LoginThrottleDecision> {
const keyHashes = createLoginThrottleKeys(email, secret); const buckets = createLoginThrottleBuckets(email, secret, ip);
const now = Date.now(); const now = Date.now();
const pool = getPool(); const pool = getPool();
if (!pool) { if (!pool) {
return consumeLocalLoginAttempt(keyHashes, now); return consumeLocalLoginAttempt(buckets, now);
} }
try { return await consumeDatabaseLoginAttempt(buckets, now);
return await consumeDatabaseLoginAttempt(keyHashes, now);
} catch {
return consumeLocalLoginAttempt(keyHashes, now);
}
} }
export async function clearAdminLoginAttempts(email: string, secret: string) { export async function clearAdminLoginAttempts(
const keyHashes = createLoginThrottleKeys(email, secret); email: string,
secret: string,
ip = "unknown",
) {
const buckets = createLoginThrottleBuckets(email, secret, ip);
const keyHashes = buckets.map((b) => b.keyHash);
const pool = getPool(); const pool = getPool();
if (!pool) { if (!pool) {
@@ -327,50 +395,45 @@ export async function clearAdminLoginAttempts(email: string, secret: string) {
return; return;
} }
await ensureAdminTable();
const client = await pool.connect();
try { try {
await ensureAdminTable(); await client.query("BEGIN");
const client = await pool.connect(); await lockLoginThrottleKeys(client, keyHashes);
try { await client.query(
await client.query("BEGIN"); "DELETE FROM admin_login_attempts WHERE key_hash = ANY($1::text[])",
await lockLoginThrottleKeys(client, keyHashes); [keyHashes],
await client.query( );
"DELETE FROM admin_login_attempts WHERE key_hash = ANY($1::text[])", await client.query("COMMIT");
[keyHashes], } catch (error) {
); await client.query("ROLLBACK").catch(() => undefined);
await client.query("COMMIT"); throw error;
} catch (error) { } finally {
await client.query("ROLLBACK").catch(() => undefined); client.release();
throw error;
} finally {
client.release();
}
} catch {
keyHashes.forEach((keyHash) => localLoginThrottle.delete(keyHash));
} }
} }
export function verifyAdminSetupKey(candidate: string) { export function verifyAdminSetupKey(candidate: string) {
const expected = const expected = getAdminSetupKey();
process.env.DATASTACK_ADMIN_SETUP_KEY || if (!expected || expected.length < 32 || !candidate) return false;
"datastack-default-setup-key-32-chars";
if (!expected || expected.length < 32) return false;
return matchesSecret(candidate, expected); return matchesSecret(candidate, expected);
} }
export async function isAdminSetupComplete() { export async function isAdminSetupComplete() {
const pool = getPool(); const pool = getPool();
if (!pool) { if (!pool) {
return true; if (globals.__dataStackInMemoryAdmin) {
} return true;
try { }
await ensureAdminTable(); if (isProductionEnv()) {
const result = await pool.query( return Boolean(process.env.DATASTACK_ADMIN_PASSWORD);
"SELECT 1 FROM admin_users WHERE id = 1", }
); return process.env.DATASTACK_ALLOW_DEMO_ADMIN !== "false";
return result.rowCount === 1;
} catch {
return true;
} }
await ensureAdminTable();
const result = await pool.query("SELECT 1 FROM admin_users WHERE id = 1");
return result.rowCount === 1;
} }
export async function createInitialAdmin(email: string, password: string) { export async function createInitialAdmin(email: string, password: string) {
@@ -380,136 +443,176 @@ export async function createInitialAdmin(email: string, password: string) {
const pool = getPool(); const pool = getPool();
if (!pool) { if (!pool) {
if (globals.__dataStackInMemoryAdmin) {
return false;
}
globals.__dataStackInMemoryAdmin = { globals.__dataStackInMemoryAdmin = {
id: 1, id: 1,
email: cleanEmail, email: cleanEmail,
passwordSalt: salt, passwordSalt: salt,
passwordHash, passwordHash,
tokenVersion: 1,
createdAt: new Date(), createdAt: new Date(),
}; };
return true; return true;
} }
await ensureAdminTable();
const client = await pool.connect();
try { try {
await ensureAdminTable(); await client.query("BEGIN");
const client = await pool.connect(); await client.query("SELECT pg_advisory_xact_lock($1::bigint)", [
DATABASE_LOCK_ID,
]);
try { const existing = await client.query(
await client.query("BEGIN"); "SELECT 1 FROM admin_users WHERE id = 1",
await client.query("SELECT pg_advisory_xact_lock($1::bigint)", [ );
DATABASE_LOCK_ID, if (existing.rowCount) {
]); await client.query("ROLLBACK");
return false;
const existing = await client.query(
"SELECT 1 FROM admin_users WHERE id = 1",
);
if (existing.rowCount) {
await client.query("ROLLBACK");
return false;
}
await client.query(
`INSERT INTO admin_users (id, email, password_salt, password_hash)
VALUES (1, $1, $2, $3)`,
[cleanEmail, salt, passwordHash],
);
await client.query("COMMIT");
return true;
} catch (error) {
await client.query("ROLLBACK").catch(() => undefined);
throw error;
} finally {
client.release();
} }
} catch {
globals.__dataStackInMemoryAdmin = { await client.query(
id: 1, `INSERT INTO admin_users (id, email, password_salt, password_hash, token_version)
email: cleanEmail, VALUES (1, $1, $2, $3, 1)`,
passwordSalt: salt, [cleanEmail, salt, passwordHash],
passwordHash, );
createdAt: new Date(), await client.query("COMMIT");
};
return true; return true;
} catch (error) {
await client.query("ROLLBACK").catch(() => undefined);
throw error;
} finally {
client.release();
} }
} }
export async function verifyAdminCredentials(email: string, password: string) { export async function authenticateAdminCredentials(
email: string,
password: string,
): Promise<{ authenticated: boolean; tokenVersion: number }> {
const cleanEmail = email.trim().toLowerCase(); const cleanEmail = email.trim().toLowerCase();
const pool = getPool(); const pool = getPool();
if (pool) { if (pool) {
try { await ensureAdminTable();
await ensureAdminTable(); const result = await pool.query(
const result = await pool.query( `SELECT email, password_salt, password_hash, COALESCE(token_version, 1) AS token_version
`SELECT email, password_salt, password_hash FROM admin_users
FROM admin_users WHERE id = 1`,
WHERE id = 1`, );
); if (!result.rowCount || result.rowCount === 0) {
if (result.rowCount && result.rowCount > 0) { return { authenticated: false, tokenVersion: 0 };
const admin = result.rows[0] as {
email: string;
password_salt: Buffer;
password_hash: Buffer;
};
const emailMatches = matchesSecret(
cleanEmail,
admin.email.trim().toLowerCase(),
);
const actualHash = await derivePasswordHash(
password,
admin.password_salt,
);
const passwordMatches = timingSafeEqual(
actualHash,
admin.password_hash,
);
return emailMatches && passwordMatches;
}
if (!process.env.DATASTACK_ADMIN_PASSWORD) {
return false;
}
} catch {
// Database connection error, fall back to in-memory admin
} }
const admin = result.rows[0] as {
email: string;
password_salt: Buffer;
password_hash: Buffer;
token_version: number;
};
const emailMatches = matchesSecret(
cleanEmail,
admin.email.trim().toLowerCase(),
);
const actualHash = await derivePasswordHash(password, admin.password_salt);
const passwordMatches = timingSafeEqual(actualHash, admin.password_hash);
return {
authenticated: emailMatches && passwordMatches,
tokenVersion: Number(admin.token_version) || 1,
};
} }
const inMem = await getOrInitInMemoryAdmin(); const inMem = await getOrInitInMemoryAdmin();
if (!matchesSecret(cleanEmail, inMem.email)) { if (!matchesSecret(cleanEmail, inMem.email)) {
return false; return { authenticated: false, tokenVersion: 0 };
} }
const actualHash = await derivePasswordHash(password, inMem.passwordSalt); const actualHash = await derivePasswordHash(password, inMem.passwordSalt);
return timingSafeEqual(actualHash, inMem.passwordHash); const authenticated = timingSafeEqual(actualHash, inMem.passwordHash);
return {
authenticated,
tokenVersion: authenticated ? inMem.tokenVersion : 0,
};
}
export async function verifyAdminCredentials(email: string, password: string) {
const result = await authenticateAdminCredentials(email, password);
return result.authenticated;
} }
export async function getAdminAccountInfo(fallbackEmail?: string) { export async function getAdminAccountInfo(fallbackEmail?: string) {
const pool = getPool(); const pool = getPool();
if (pool) { if (pool) {
try { await ensureAdminTable();
await ensureAdminTable(); const result = await pool.query(
const result = await pool.query( `SELECT email, created_at, COALESCE(token_version, 1) AS token_version
`SELECT email, created_at FROM admin_users WHERE id = 1`, FROM admin_users
); WHERE id = 1`,
if (result.rowCount && result.rowCount > 0) { );
const row = result.rows[0] as { email: string; created_at: Date | string }; if (result.rowCount && result.rowCount > 0) {
return { const row = result.rows[0] as {
email: row.email, email: string;
createdAt: new Date(row.created_at).toISOString(), created_at: Date | string;
storageMode: "PostgreSQL" as const, token_version: number;
}; };
} return {
} catch { email: row.email,
// Fall back to in-memory admin createdAt: new Date(row.created_at).toISOString(),
tokenVersion: Number(row.token_version) || 1,
storageMode: "PostgreSQL" as const,
};
} }
throw new Error("Admin account is not initialized.");
} }
const inMem = await getOrInitInMemoryAdmin(); const inMem = await getOrInitInMemoryAdmin();
return { return {
email: fallbackEmail || inMem.email, email: fallbackEmail || inMem.email,
createdAt: inMem.createdAt.toISOString(), createdAt: inMem.createdAt.toISOString(),
tokenVersion: inMem.tokenVersion,
storageMode: "In-Memory" as const, storageMode: "In-Memory" as const,
}; };
} }
export async function verifyRequestAdminSession(
cookieToken: string | undefined,
): Promise<
| { valid: true; session: AdminSessionPayload; sessionSecret: string }
| { valid: false; reason: "UNCONFIGURED" | "UNAUTHORIZED" | "UNAVAILABLE" }
> {
const sessionSecret = getAdminSessionSecret();
if (!sessionSecret) {
return { valid: false, reason: "UNCONFIGURED" };
}
if (!cookieToken) {
return { valid: false, reason: "UNAUTHORIZED" };
}
const session = await getAdminSessionPayload(cookieToken, sessionSecret);
if (!session) {
return { valid: false, reason: "UNAUTHORIZED" };
}
try {
const account = await getAdminAccountInfo();
const emailMatches = matchesSecret(
session.email.trim().toLowerCase(),
account.email.trim().toLowerCase(),
);
if (!emailMatches || session.tokenVersion !== account.tokenVersion) {
return { valid: false, reason: "UNAUTHORIZED" };
}
return { valid: true, session, sessionSecret };
} catch (error) {
if (getPool()) {
return { valid: false, reason: "UNAVAILABLE" };
}
void error;
return { valid: false, reason: "UNAUTHORIZED" };
}
}
export async function updateAdminAccount({ export async function updateAdminAccount({
currentEmail, currentEmail,
currentPassword, currentPassword,
@@ -534,29 +637,42 @@ export async function updateAdminAccount({
const pool = getPool(); const pool = getPool();
if (pool) { if (pool) {
try { await ensureAdminTable();
await ensureAdminTable(); const updated = await pool.query(
const updated = await pool.query( `UPDATE admin_users
`UPDATE admin_users SET email = $1,
SET email = $1, password_salt = $2, password_hash = $3 password_salt = $2,
WHERE id = 1`, password_hash = $3,
[cleanEmail, salt, passwordHash], token_version = COALESCE(token_version, 1) + 1
); WHERE id = 1
if (updated.rowCount && updated.rowCount > 0) { RETURNING token_version`,
return { ok: true as const, email: cleanEmail }; [cleanEmail, salt, passwordHash],
} );
} catch { if (!updated.rowCount || updated.rowCount === 0) {
// Fall back to in-memory update return { ok: false as const, error: "Admin account not found." };
} }
return {
ok: true as const,
email: cleanEmail,
tokenVersion: Number(updated.rows[0].token_version) || 1,
};
} }
const existing = await getOrInitInMemoryAdmin(); const existing = await getOrInitInMemoryAdmin();
const nextVersion = existing.tokenVersion + 1;
globals.__dataStackInMemoryAdmin = { globals.__dataStackInMemoryAdmin = {
id: 1, id: 1,
email: cleanEmail, email: cleanEmail,
passwordSalt: salt, passwordSalt: salt,
passwordHash, passwordHash,
tokenVersion: nextVersion,
createdAt: existing.createdAt, createdAt: existing.createdAt,
}; };
return { ok: true as const, email: cleanEmail }; return { ok: true as const, email: cleanEmail, tokenVersion: nextVersion };
}
export function resetAdminStateForTests() {
globals.__dataStackInMemoryAdmin = undefined;
localLoginThrottle.clear();
lastLocalThrottleCleanup = 0;
} }
+33 -10
View File
@@ -8,16 +8,31 @@ export interface ClientTokenPayload {
exp?: number; exp?: number;
} }
const DEFAULT_SECRET = const DEV_DEFAULT_JWT_SECRET =
process.env.DATASTACK_JWT_SECRET ||
"datastack-jwt-default-secret-key-32-chars-long"; "datastack-jwt-default-secret-key-32-chars-long";
export function resolveDefaultJwtSecret(): string {
const configured = process.env.DATASTACK_JWT_SECRET?.trim() || "";
if (process.env.NODE_ENV === "production") {
return configured.length >= 32 ? configured : "";
}
if (configured.length > 0) {
return configured.length >= 32 ? configured : "";
}
return DEV_DEFAULT_JWT_SECRET;
}
export class AuthService { export class AuthService {
private static secret: string = DEFAULT_SECRET; private static secret: string = resolveDefaultJwtSecret();
public static setSecret(newSecret: string) { public static setSecret(newSecret: string) {
if (newSecret && newSecret.trim().length > 0) { const trimmed = newSecret ? newSecret.trim() : "";
this.secret = newSecret.trim(); if (trimmed.length >= 32) {
this.secret = trimmed;
} else if (process.env.NODE_ENV !== "production" && trimmed.length > 0) {
this.secret = trimmed;
} else if (process.env.NODE_ENV === "production") {
this.secret = "";
} }
} }
@@ -25,6 +40,11 @@ export class AuthService {
return this.secret; return this.secret;
} }
public static isConfigured(): boolean {
const minLen = process.env.NODE_ENV === "production" ? 32 : 1;
return this.secret.length >= minLen;
}
/** /**
* Generates a signed JWT for client SDK authentication. * Generates a signed JWT for client SDK authentication.
*/ */
@@ -32,9 +52,9 @@ export class AuthService {
payload: { clientId: string; name?: string; rateLimit?: number }, payload: { clientId: string; name?: string; rateLimit?: number },
expiresIn = "30d", expiresIn = "30d",
): string { ): string {
if (!this.secret) { if (!this.isConfigured()) {
throw new Error( throw new Error(
"DATASTACK_JWT_SECRET must be configured before generating client tokens.", "DATASTACK_JWT_SECRET (>= 32 characters) must be configured before generating client tokens.",
); );
} }
@@ -50,15 +70,18 @@ export class AuthService {
} }
/** /**
* Verifies and decodes incoming token from Socket.IO handshake. * Verifies and decodes incoming token from Socket.IO handshake or HTTP ingestion.
*/ */
public static verifyClientToken(token: string): { public static verifyClientToken(token: string): {
valid: boolean; valid: boolean;
payload?: ClientTokenPayload; payload?: ClientTokenPayload;
error?: string; error?: string;
} { } {
if (!this.secret) { if (!this.isConfigured()) {
return { valid: false, error: "DATASTACK_JWT_SECRET is not configured." }; return {
valid: false,
error: "DATASTACK_JWT_SECRET (>= 32 characters) is not configured.",
};
} }
try { try {
+380 -28
View File
@@ -1,8 +1,37 @@
import crypto from 'node:crypto';
import Redis from 'ioredis';
import { CSVStorageManager, DataEvent } from '../Storage/csvStorage'; import { CSVStorageManager, DataEvent } from '../Storage/csvStorage';
export type EventSubscriber = (event: DataEvent) => void; export type EventSubscriber = (event: DataEvent) => void;
export type FlushCallback = (count: number, files: string[]) => void; export type FlushCallback = (count: number, files: string[]) => void;
export interface RedisBufferClient {
status?: string;
rpush(key: string, ...values: string[]): Promise<number>;
lpush(key: string, ...values: string[]): Promise<number>;
ltrim(key: string, start: number, stop: number): Promise<unknown>;
lrange(key: string, start: number, stop: number): Promise<string[]>;
llen(key: string): Promise<number>;
set(
key: string,
value: string,
pxMode: 'PX',
ttlMs: number,
nxMode: 'NX',
): Promise<string | null>;
eval(script: string, numKeys: number, ...args: string[]): Promise<unknown>;
quit?: () => Promise<unknown>;
disconnect?: () => void;
}
export const REDIS_BUFFER_KEY = 'datastack:buffer:events';
export const REDIS_RECENT_KEY = 'datastack:recent:events';
export const REDIS_FLUSH_LOCK_KEY = 'datastack:flush:lock';
const REDIS_RECENT_LIMIT = 500;
const FLUSH_LOCK_TTL_MS = 15000;
const UNLOCK_LUA_SCRIPT =
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end';
interface BufferGlobalState { interface BufferGlobalState {
buffer: DataEvent[]; buffer: DataEvent[];
recentEvents: DataEvent[]; recentEvents: DataEvent[];
@@ -14,6 +43,11 @@ interface BufferGlobalState {
totalIngested: number; totalIngested: number;
totalFlushed: number; totalFlushed: number;
isFlushing: boolean; isFlushing: boolean;
redisClient: RedisBufferClient | null | undefined;
redisUrlConfigured: string;
redisConnected: boolean;
redisTestOverride: RedisBufferClient | null | undefined;
pendingRedisWrites: Set<Promise<void>>;
} }
type BufferGlobals = typeof globalThis & { type BufferGlobals = typeof globalThis & {
@@ -36,13 +70,86 @@ function getState(): BufferGlobalState {
totalIngested: 0, totalIngested: 0,
totalFlushed: 0, totalFlushed: 0,
isFlushing: false, isFlushing: false,
redisClient: undefined,
redisUrlConfigured: '',
redisConnected: false,
redisTestOverride: undefined,
pendingRedisWrites: new Set(),
}; };
} }
return globals.__dataStackBufferState; return globals.__dataStackBufferState;
} }
function getRedisClient(): RedisBufferClient | null {
const state = getState();
if (state.redisTestOverride !== undefined) {
return state.redisTestOverride;
}
const configuredUrl = (process.env.REDIS_URL || '').trim();
if (!configuredUrl) {
if (state.redisClient && typeof state.redisClient.disconnect === 'function') {
try {
state.redisClient.disconnect();
} catch {
// ignore
}
}
state.redisClient = null;
state.redisUrlConfigured = '';
state.redisConnected = false;
return null;
}
if (state.redisClient && state.redisUrlConfigured === configuredUrl) {
return state.redisClient;
}
if (state.redisClient && typeof state.redisClient.disconnect === 'function') {
try {
state.redisClient.disconnect();
} catch {
// ignore
}
}
try {
const client = new Redis(configuredUrl, {
maxRetriesPerRequest: 2,
enableReadyCheck: true,
retryStrategy: (times: number) => Math.min(times * 250, 3000),
});
client.on('ready', () => {
state.redisConnected = true;
});
client.on('connect', () => {
state.redisConnected = true;
});
client.on('close', () => {
state.redisConnected = false;
});
client.on('end', () => {
state.redisConnected = false;
});
client.on('error', () => {
state.redisConnected = false;
});
state.redisClient = client as unknown as RedisBufferClient;
state.redisUrlConfigured = configuredUrl;
state.redisConnected = client.status === 'ready' || client.status === 'connect';
return state.redisClient;
} catch {
state.redisClient = null;
state.redisUrlConfigured = configuredUrl;
state.redisConnected = false;
return null;
}
}
export class DataBufferEngine { export class DataBufferEngine {
private static maxRecentEvents = 200; private static maxRecentEvents = REDIS_RECENT_LIMIT;
public static get batchFlushThreshold(): number { public static get batchFlushThreshold(): number {
return getState().batchFlushThreshold; return getState().batchFlushThreshold;
@@ -60,11 +167,46 @@ export class DataBufferEngine {
this.setFlushInterval(value); this.setFlushInterval(value);
} }
public static setRedisClientForTests(client: RedisBufferClient | null | undefined): void {
const state = getState();
state.redisTestOverride = client;
state.redisConnected = Boolean(client);
}
public static async shutdown(): Promise<void> {
const state = getState();
if (state.flushTimer) {
clearInterval(state.flushTimer);
state.flushTimer = null;
}
if (state.pendingRedisWrites.size > 0) {
await Promise.allSettled([...state.pendingRedisWrites]);
}
const client = state.redisClient;
state.redisClient = undefined;
state.redisUrlConfigured = '';
state.redisConnected = false;
state.redisTestOverride = undefined;
if (client) {
try {
if (typeof client.quit === 'function') {
await client.quit();
} else if (typeof client.disconnect === 'function') {
client.disconnect();
}
} catch {
if (typeof client.disconnect === 'function') {
client.disconnect();
}
}
}
}
public static init() { public static init() {
const state = getState(); const state = getState();
if (!state.flushTimer) { if (!state.flushTimer) {
state.flushTimer = setInterval(() => { state.flushTimer = setInterval(() => {
void this.flush(); void this.flush().catch(() => undefined);
}, state.batchFlushIntervalMs); }, state.batchFlushIntervalMs);
} }
@@ -96,26 +238,18 @@ export class DataBufferEngine {
} }
state.flushTimer = setInterval(() => { state.flushTimer = setInterval(() => {
void this.flush(); void this.flush().catch(() => undefined);
}, state.batchFlushIntervalMs); }, state.batchFlushIntervalMs);
} }
/** private static recordLocalMetaAndNotify(event: DataEvent): void {
* Push incoming event to in-memory buffer.
* If buffer hits threshold (e.g. 500 rows), triggers immediate batch flush.
*/
public static push(event: DataEvent): void {
const state = getState(); const state = getState();
state.buffer.push(event);
state.totalIngested += 1; state.totalIngested += 1;
// Cache in recent events for live dashboard viewer
state.recentEvents.unshift(event); state.recentEvents.unshift(event);
if (state.recentEvents.length > this.maxRecentEvents) { if (state.recentEvents.length > this.maxRecentEvents) {
state.recentEvents.pop(); state.recentEvents.pop();
} }
// Notify live subscribers
for (const sub of state.subscribers) { for (const sub of state.subscribers) {
try { try {
sub(event); sub(event);
@@ -123,43 +257,188 @@ export class DataBufferEngine {
// Ignore subscriber error // Ignore subscriber error
} }
} }
}
// Check if buffer threshold reached private static async writeEventsToRedisOrFallback(events: DataEvent[]): Promise<number> {
if (events.length === 0) return 0;
const state = getState();
const redis = getRedisClient();
if (redis) {
const serialized = events.map((e) => JSON.stringify(e));
try {
const bufferLen = await redis.rpush(REDIS_BUFFER_KEY, ...serialized);
await redis.lpush(REDIS_RECENT_KEY, ...serialized);
await redis.ltrim(REDIS_RECENT_KEY, 0, REDIS_RECENT_LIMIT - 1);
state.redisConnected = true;
return Number(bufferLen) + state.buffer.length;
} catch {
state.redisConnected = false;
}
}
state.buffer.push(...events);
return state.buffer.length;
}
/**
* Synchronous-signature push (queues to Redis asynchronously when configured, with automatic RAM fallback).
*/
public static push(event: DataEvent): void {
const state = getState();
this.recordLocalMetaAndNotify(event);
const redis = getRedisClient();
if (redis) {
const op = this.writeEventsToRedisOrFallback([event]).then((queuedCount) => {
if (queuedCount >= state.batchFlushThreshold) {
void this.flush().catch(() => undefined);
}
});
state.pendingRedisWrites.add(op);
void op.finally(() => {
state.pendingRedisWrites.delete(op);
});
return;
}
state.buffer.push(event);
if (state.buffer.length >= state.batchFlushThreshold) { if (state.buffer.length >= state.batchFlushThreshold) {
void this.flush(); void this.flush().catch(() => undefined);
} }
} }
/** /**
* Flushes current buffer to CSV files via CSVStorageManager * Async single-event push (awaited by API routes so Redis persistence is guaranteed before HTTP response).
*/
public static async pushAsync(event: DataEvent): Promise<void> {
await this.pushBatchAsync([event]);
}
/**
* Async batch push with shared Redis durability and RAM fallback.
*/
public static async pushBatchAsync(events: DataEvent[]): Promise<void> {
if (!events || events.length === 0) return;
const state = getState();
for (const evt of events) {
this.recordLocalMetaAndNotify(evt);
}
const queuedCount = await this.writeEventsToRedisOrFallback(events);
if (queuedCount >= state.batchFlushThreshold) {
void this.flush().catch(() => undefined);
}
}
/**
* Flushes queued events (shared Redis list under distributed lock + local RAM fallback) to partitioned CSV files.
* Redis LTRIM runs ONLY after CSVStorageManager.appendBatch succeeds so crashes never lose data.
*/ */
public static async flush(): Promise<{ count: number; files: string[] }> { public static async flush(): Promise<{ count: number; files: string[] }> {
const state = getState(); const state = getState();
if (state.buffer.length === 0 || state.isFlushing) { if (state.pendingRedisWrites.size > 0) {
await Promise.allSettled([...state.pendingRedisWrites]);
}
if (state.isFlushing) {
return { count: 0, files: [] }; return { count: 0, files: [] };
} }
state.isFlushing = true; state.isFlushing = true;
const batchToFlush = [...state.buffer]; let totalWritten = 0;
state.buffer = []; const filesSet = new Set<string>();
try { try {
const result = await CSVStorageManager.appendBatch(batchToFlush); const redis = getRedisClient();
state.totalFlushed += result.written; if (redis) {
const lockToken = crypto.randomUUID();
let lockAcquired = false;
for (const listener of state.flushListeners) {
try { try {
listener(result.written, result.filesUpdated); const lockResult = await redis.set(
REDIS_FLUSH_LOCK_KEY,
lockToken,
'PX',
FLUSH_LOCK_TTL_MS,
'NX',
);
lockAcquired = lockResult === 'OK';
state.redisConnected = true;
} catch { } catch {
// ignore state.redisConnected = false;
lockAcquired = false;
}
if (lockAcquired) {
try {
const chunkSize = Math.max(500, state.batchFlushThreshold);
while (true) {
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, chunkSize - 1);
if (!rawItems || rawItems.length === 0) {
break;
}
const parsedBatch: DataEvent[] = [];
for (const raw of rawItems) {
try {
const evt = JSON.parse(raw) as DataEvent;
if (evt && evt.eventId) {
parsedBatch.push(evt);
}
} catch {
// Ignore malformed legacy entry while still trimming it
}
}
if (parsedBatch.length > 0) {
const res = await CSVStorageManager.appendBatch(parsedBatch);
totalWritten += res.written;
for (const f of res.filesUpdated) filesSet.add(f);
}
// Trim strictly AFTER CSVStorageManager.appendBatch resolves safely
await redis.ltrim(REDIS_BUFFER_KEY, rawItems.length, -1);
if (rawItems.length < chunkSize) {
break;
}
}
} finally {
try {
await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
} catch {
// Lock auto-expires after FLUSH_LOCK_TTL_MS
}
}
} }
} }
return { count: result.written, files: result.filesUpdated }; if (state.buffer.length > 0) {
} catch (err) { const ramBatch = [...state.buffer];
// On failure, re-queue to buffer front for retry state.buffer = [];
state.buffer.unshift(...batchToFlush); try {
throw err; const res = await CSVStorageManager.appendBatch(ramBatch);
totalWritten += res.written;
for (const f of res.filesUpdated) filesSet.add(f);
} catch (err) {
state.buffer.unshift(...ramBatch);
throw err;
}
}
if (totalWritten > 0) {
state.totalFlushed += totalWritten;
const filesUpdated = Array.from(filesSet);
for (const listener of state.flushListeners) {
try {
listener(totalWritten, filesUpdated);
} catch {
// ignore
}
}
}
return { count: totalWritten, files: Array.from(filesSet) };
} finally { } finally {
state.isFlushing = false; state.isFlushing = false;
} }
@@ -194,10 +473,83 @@ export class DataBufferEngine {
return state.recentEvents.slice(0, limit); return state.recentEvents.slice(0, limit);
} }
public static async getRecentEventsAsync(
limit = 50,
filterClientId?: string,
): Promise<DataEvent[]> {
const state = getState();
const redis = getRedisClient();
if (redis) {
try {
const readCap = Math.min(REDIS_RECENT_LIMIT, Math.max(limit * 4, limit));
const rawItems = await redis.lrange(REDIS_RECENT_KEY, 0, readCap - 1);
state.redisConnected = true;
const redisEvents: DataEvent[] = [];
for (const raw of rawItems) {
try {
const parsed = JSON.parse(raw) as DataEvent;
if (!parsed || !parsed.eventId) continue;
if (filterClientId && filterClientId !== 'all' && parsed.clientId !== filterClientId) {
continue;
}
redisEvents.push(parsed);
if (redisEvents.length >= limit) break;
} catch {
// skip malformed JSON entry
}
}
if (redisEvents.length > 0 || state.recentEvents.length === 0) {
return redisEvents;
}
} catch {
state.redisConnected = false;
}
}
return this.getRecentEvents(limit, filterClientId);
}
public static getBufferStats() { public static getBufferStats() {
const state = getState(); const state = getState();
return { return {
currentBufferSize: state.buffer.length, currentBufferSize: state.buffer.length,
localBufferSize: state.buffer.length,
redisBufferSize: 0,
redisEnabled: Boolean(getRedisClient()),
redisConnected: state.redisConnected,
batchFlushThreshold: state.batchFlushThreshold,
batchFlushIntervalMs: state.batchFlushIntervalMs,
totalIngested: state.totalIngested,
totalFlushed: state.totalFlushed,
recentEventsCached: state.recentEvents.length,
activeSubscribers: state.subscribers.size,
isFlushing: state.isFlushing,
};
}
public static async getBufferStatsAsync() {
const state = getState();
const redis = getRedisClient();
let redisBufferSize = 0;
if (redis) {
try {
redisBufferSize = Number(await redis.llen(REDIS_BUFFER_KEY)) || 0;
state.redisConnected = true;
} catch {
state.redisConnected = false;
redisBufferSize = 0;
}
}
return {
currentBufferSize: state.buffer.length + redisBufferSize,
localBufferSize: state.buffer.length,
redisBufferSize,
redisEnabled: Boolean(redis),
redisConnected: state.redisConnected,
batchFlushThreshold: state.batchFlushThreshold, batchFlushThreshold: state.batchFlushThreshold,
batchFlushIntervalMs: state.batchFlushIntervalMs, batchFlushIntervalMs: state.batchFlushIntervalMs,
totalIngested: state.totalIngested, totalIngested: state.totalIngested,
+315 -47
View File
@@ -1,5 +1,8 @@
import { existsSync, rmSync } from "node:fs";
import { resolve, sep } from "node:path";
import { AuthService } from "../auth/jwt"; import { AuthService } from "../auth/jwt";
import { DatabaseEngine } from "../Database/database"; import { DatabaseEngine } from "../Database/database";
import { getInternalIngestionHeaders } from "../Security/security";
export interface ClientSDK { export interface ClientSDK {
clientId: string; clientId: string;
@@ -15,7 +18,9 @@ export interface ClientSDK {
type SDKDatabaseGlobals = typeof globalThis & { type SDKDatabaseGlobals = typeof globalThis & {
__dataStackClientsMap?: Map<string, ClientSDK>; __dataStackClientsMap?: Map<string, ClientSDK>;
__dataStackDeletedClients?: Set<string>;
__dataStackClientsHydrated?: boolean; __dataStackClientsHydrated?: boolean;
__dataStackSyncedSocketEvents?: Map<string, number>;
}; };
const globals = globalThis as SDKDatabaseGlobals; const globals = globalThis as SDKDatabaseGlobals;
@@ -27,6 +32,67 @@ function getClientsMap(): Map<string, ClientSDK> {
return globals.__dataStackClientsMap; return globals.__dataStackClientsMap;
} }
function getDeletedClientsSet(): Set<string> {
if (!globals.__dataStackDeletedClients) {
globals.__dataStackDeletedClients = new Set<string>();
}
return globals.__dataStackDeletedClients;
}
function getSyncedSocketEventsMap(): Map<string, number> {
if (!globals.__dataStackSyncedSocketEvents) {
globals.__dataStackSyncedSocketEvents = new Map<string, number>();
}
return globals.__dataStackSyncedSocketEvents;
}
function serializeSyncClient(client: ClientSDK) {
return {
clientId: client.clientId,
name: client.name,
status: client.status,
token: client.token,
rateLimit: client.rateLimit,
};
}
async function notifyIngestionClientSyncAsync(
clientId: string,
action: "deleted" | "revoked" | "updated" | "registered",
client?: ClientSDK,
): Promise<void> {
const headers = getInternalIngestionHeaders();
if (!headers["x-datastack-internal-secret"]) return;
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, "");
try {
await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
method: "POST",
headers: {
"Content-Type": "application/json",
...headers,
},
body: JSON.stringify({
clientId,
action,
...(client ? { client: serializeSyncClient(client) } : {}),
}),
signal: AbortSignal.timeout(1000),
});
} catch {
// Port 4000 may not be running in isolated single-process dev mode
}
}
function notifyIngestionClientSync(
clientId: string,
action: "deleted" | "revoked" | "updated" | "registered",
client?: ClientSDK,
): void {
void notifyIngestionClientSyncAsync(clientId, action, client);
}
function mapRowToClient(row: Record<string, unknown>): ClientSDK { function mapRowToClient(row: Record<string, unknown>): ClientSDK {
return { return {
clientId: String(row.client_id), clientId: String(row.client_id),
@@ -54,11 +120,20 @@ export class SDKManager {
const ready = await DatabaseEngine.ensureSchema(); const ready = await DatabaseEngine.ensureSchema();
if (!ready) return null; if (!ready) return null;
const result = await pool.query( const [result, deletedResult] = await Promise.all([
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at pool.query(
FROM client_sdks `SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
ORDER BY created_at DESC`, FROM client_sdks
); ORDER BY created_at DESC`,
),
pool.query(`SELECT client_id FROM deleted_clients`),
]);
const deletedSet = getDeletedClientsSet();
deletedSet.clear();
for (const row of deletedResult.rows) {
deletedSet.add(String(row.client_id).trim().toLowerCase());
}
const map = getClientsMap(); const map = getClientsMap();
map.clear(); map.clear();
@@ -84,12 +159,91 @@ export class SDKManager {
return Array.from(getClientsMap().values()); return Array.from(getClientsMap().values());
} }
public static async syncAllClientsToIngestionAsync(): Promise<void> {
const headers = getInternalIngestionHeaders();
if (!headers["x-datastack-internal-secret"]) return;
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, "");
const clients = Array.from(getClientsMap().values()).map(serializeSyncClient);
try {
await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
method: "POST",
headers: {
"Content-Type": "application/json",
...headers,
},
body: JSON.stringify({
action: "bulk_sync",
clients,
}),
signal: AbortSignal.timeout(1500),
});
} catch {
// Port 4000 may not be running in isolated single-process dev mode
}
}
public static async listClientsAsync(): Promise<ClientSDK[]> { public static async listClientsAsync(): Promise<ClientSDK[]> {
const fromDb = await this.syncFromDatabase(); const fromDb = await this.syncFromDatabase();
if (fromDb !== null) { if (fromDb !== null) {
return fromDb; return fromDb;
} }
return Array.from(getClientsMap().values());
const map = getClientsMap();
const headers = getInternalIngestionHeaders();
if (headers["x-datastack-internal-secret"]) {
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, "");
try {
const res = await fetch(`${ingestionBaseUrl}/internal/metrics`, {
headers,
cache: "no-store",
signal: AbortSignal.timeout(1000),
});
if (res.ok) {
const data = (await res.json()) as {
hasSyncedInMemoryClients?: boolean;
clientActivity?: Record<
string,
{ totalEvents?: number; lastActive?: string }
>;
};
if (data.hasSyncedInMemoryClients === false && map.size > 0) {
await this.syncAllClientsToIngestionAsync();
}
if (data.clientActivity && typeof data.clientActivity === "object") {
const syncedMap = getSyncedSocketEventsMap();
for (const [cid, activity] of Object.entries(data.clientActivity)) {
const client = map.get(cid);
if (!client) continue;
const remoteTotal = Math.max(0, Number(activity?.totalEvents) || 0);
const prevSynced = syncedMap.get(cid) || 0;
const delta =
remoteTotal >= prevSynced
? remoteTotal - prevSynced
: remoteTotal;
if (delta > 0) {
client.totalEvents += delta;
}
syncedMap.set(cid, remoteTotal);
if (
activity?.lastActive &&
(!client.lastActive || activity.lastActive > client.lastActive)
) {
client.lastActive = activity.lastActive;
}
map.set(cid, client);
}
}
}
} catch {
// Port 4000 may not be running in isolated single-process dev mode
}
}
return Array.from(map.values());
} }
public static getClient(clientId: string): ClientSDK | undefined { public static getClient(clientId: string): ClientSDK | undefined {
@@ -152,8 +306,10 @@ export class SDKManager {
createdAt: new Date().toISOString(), createdAt: new Date().toISOString(),
}; };
getDeletedClientsSet().delete(safeClientId);
getClientsMap().set(safeClientId, newClient); getClientsMap().set(safeClientId, newClient);
void this.persistNewClient(newClient); void this.persistNewClient(newClient);
notifyIngestionClientSync(safeClientId, "registered", newClient);
return newClient; return newClient;
} }
@@ -163,6 +319,10 @@ export class SDKManager {
try { try {
await DatabaseEngine.ensureSchema(); await DatabaseEngine.ensureSchema();
await pool.query(
`DELETE FROM deleted_clients WHERE client_id = $1`,
[client.clientId],
);
await pool.query( await pool.query(
`INSERT INTO client_sdks (client_id, name, secret, token, rate_limit, status, total_events, created_at, updated_at) `INSERT INTO client_sdks (client_id, name, secret, token, rate_limit, status, total_events, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW()) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW())
@@ -218,8 +378,10 @@ export class SDKManager {
createdAt: new Date().toISOString(), createdAt: new Date().toISOString(),
}; };
getDeletedClientsSet().delete(safeClientId);
getClientsMap().set(safeClientId, newClient); getClientsMap().set(safeClientId, newClient);
await this.persistNewClient(newClient); await this.persistNewClient(newClient);
await notifyIngestionClientSyncAsync(safeClientId, "registered", newClient);
return newClient; return newClient;
} }
@@ -227,16 +389,20 @@ export class SDKManager {
const client = getClientsMap().get(clientId); const client = getClientsMap().get(clientId);
if (!client) return null; if (!client) return null;
client.secret = `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`; const updated: ClientSDK = {
client.token = AuthService.generateClientToken({ ...client,
clientId: client.clientId, secret: `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`,
name: client.name, token: AuthService.generateClientToken({
rateLimit: client.rateLimit, clientId: client.clientId,
}); name: client.name,
rateLimit: client.rateLimit,
}),
};
getClientsMap().set(clientId, client); getClientsMap().set(clientId, updated);
void this.persistClientCredentials(client); void this.persistClientCredentials(updated);
return client; notifyIngestionClientSync(clientId, "updated", updated);
return updated;
} }
private static async persistClientCredentials( private static async persistClientCredentials(
@@ -264,16 +430,20 @@ export class SDKManager {
const existing = await this.getClientAsync(clientId); const existing = await this.getClientAsync(clientId);
if (!existing) return null; if (!existing) return null;
existing.secret = `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`; const updated: ClientSDK = {
existing.token = AuthService.generateClientToken({ ...existing,
clientId: existing.clientId, secret: `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`,
name: existing.name, token: AuthService.generateClientToken({
rateLimit: existing.rateLimit, clientId: existing.clientId,
}); name: existing.name,
rateLimit: existing.rateLimit,
}),
};
getClientsMap().set(clientId, existing); getClientsMap().set(clientId, updated);
await this.persistClientCredentials(existing); await this.persistClientCredentials(updated);
return existing; await notifyIngestionClientSyncAsync(clientId, "updated", updated);
return updated;
} }
public static updateClient( public static updateClient(
@@ -283,14 +453,23 @@ export class SDKManager {
const client = getClientsMap().get(clientId); const client = getClientsMap().get(clientId);
if (!client) return null; if (!client) return null;
if (updates.name !== undefined) client.name = updates.name.trim(); const updated: ClientSDK = {
if (updates.rateLimit !== undefined) ...client,
client.rateLimit = Math.max(1, Number(updates.rateLimit) || 100); ...(updates.name !== undefined ? { name: updates.name.trim() } : {}),
if (updates.status !== undefined) client.status = updates.status; ...(updates.rateLimit !== undefined
? { rateLimit: Math.max(1, Number(updates.rateLimit) || 100) }
: {}),
...(updates.status !== undefined ? { status: updates.status } : {}),
};
getClientsMap().set(clientId, client); getClientsMap().set(clientId, updated);
void this.persistClientUpdates(client); void this.persistClientUpdates(updated);
return client; notifyIngestionClientSync(
clientId,
updated.status === "revoked" ? "revoked" : "updated",
updated,
);
return updated;
} }
private static async persistClientUpdates(client: ClientSDK): Promise<void> { private static async persistClientUpdates(client: ClientSDK): Promise<void> {
@@ -317,19 +496,67 @@ export class SDKManager {
const client = await this.getClientAsync(clientId); const client = await this.getClientAsync(clientId);
if (!client) return null; if (!client) return null;
if (updates.name !== undefined) client.name = updates.name.trim(); const updated: ClientSDK = {
if (updates.rateLimit !== undefined) ...client,
client.rateLimit = Math.max(1, Number(updates.rateLimit) || 100); ...(updates.name !== undefined ? { name: updates.name.trim() } : {}),
if (updates.status !== undefined) client.status = updates.status; ...(updates.rateLimit !== undefined
? { rateLimit: Math.max(1, Number(updates.rateLimit) || 100) }
: {}),
...(updates.status !== undefined ? { status: updates.status } : {}),
};
getClientsMap().set(clientId, client); getClientsMap().set(clientId, updated);
await this.persistClientUpdates(client); await this.persistClientUpdates(updated);
return client; await notifyIngestionClientSyncAsync(
clientId,
updated.status === "revoked" ? "revoked" : "updated",
updated,
);
return updated;
}
public static isClientDeleted(clientId: string): boolean {
const normalized = String(clientId || "")
.trim()
.toLowerCase();
return getDeletedClientsSet().has(normalized);
}
public static async isClientDeletedAsync(clientId: string): Promise<boolean> {
const normalized = String(clientId || "")
.trim()
.toLowerCase();
if (!normalized) return false;
if (getDeletedClientsSet().has(normalized)) {
return true;
}
const pool = DatabaseEngine.getPool();
if (!pool) return false;
await DatabaseEngine.ensureSchema();
const res = await pool.query(
`SELECT 1 FROM deleted_clients WHERE client_id = $1`,
[normalized],
);
if (res.rowCount && res.rowCount > 0) {
getDeletedClientsSet().add(normalized);
return true;
}
return false;
} }
public static deleteClient(clientId: string): boolean { public static deleteClient(clientId: string): boolean {
const deleted = getClientsMap().delete(clientId); const normalized = String(clientId || "")
void this.deleteClientFromDb(clientId); .trim()
.toLowerCase();
const deleted = getClientsMap().delete(normalized);
getSyncedSocketEventsMap().delete(normalized);
if (deleted) {
getDeletedClientsSet().add(normalized);
}
void this.deleteClientFromDb(normalized);
notifyIngestionClientSync(normalized, "deleted");
return deleted; return deleted;
} }
@@ -337,21 +564,62 @@ export class SDKManager {
const pool = DatabaseEngine.getPool(); const pool = DatabaseEngine.getPool();
if (!pool) return false; if (!pool) return false;
await DatabaseEngine.ensureSchema();
const dbClient = await pool.connect();
try { try {
await DatabaseEngine.ensureSchema(); await dbClient.query("BEGIN");
const res = await pool.query( const res = await dbClient.query(
`DELETE FROM client_sdks WHERE client_id = $1`, `DELETE FROM client_sdks WHERE client_id = $1`,
[clientId], [clientId],
); );
return Boolean(res.rowCount && res.rowCount > 0); if (res.rowCount && res.rowCount > 0) {
} catch { await dbClient.query(
`INSERT INTO deleted_clients (client_id, deleted_at)
VALUES ($1, NOW())
ON CONFLICT (client_id) DO UPDATE SET deleted_at = NOW()`,
[clientId],
);
await dbClient.query("COMMIT");
return true;
}
await dbClient.query("ROLLBACK");
return false; return false;
} catch {
await dbClient.query("ROLLBACK").catch(() => undefined);
return false;
} finally {
dbClient.release();
} }
} }
public static async deleteClientAsync(clientId: string): Promise<boolean> { private static purgeClientStorageDir(clientId: string): void {
const existedInMem = getClientsMap().delete(clientId); const safeId = clientId.replace(/[^a-z0-9_-]/gi, "_");
const deletedFromDb = await this.deleteClientFromDb(clientId); if (!safeId) return;
const storageRoot = resolve(process.cwd(), "storage", "data");
const targetDir = resolve(storageRoot, safeId);
if (!targetDir.startsWith(`${storageRoot}${sep}`)) return;
if (existsSync(targetDir)) {
rmSync(targetDir, { recursive: true, force: true });
}
}
public static async deleteClientAsync(
clientId: string,
options?: { purgeFiles?: boolean },
): Promise<boolean> {
const normalized = String(clientId || "")
.trim()
.toLowerCase();
const existedInMem = getClientsMap().delete(normalized);
getSyncedSocketEventsMap().delete(normalized);
const deletedFromDb = await this.deleteClientFromDb(normalized);
if (existedInMem || deletedFromDb) {
getDeletedClientsSet().add(normalized);
if (options?.purgeFiles) {
this.purgeClientStorageDir(normalized);
}
await notifyIngestionClientSyncAsync(normalized, "deleted");
}
return existedInMem || deletedFromDb; return existedInMem || deletedFromDb;
} }
+7 -1
View File
@@ -17,9 +17,12 @@ services:
DATASTACK_ADMIN_SESSION_SECRET: "${SERVICE_REALBASE64_64_ADMINSESSION:?Coolify must generate SERVICE_REALBASE64_64_ADMINSESSION}" DATASTACK_ADMIN_SESSION_SECRET: "${SERVICE_REALBASE64_64_ADMINSESSION:?Coolify must generate SERVICE_REALBASE64_64_ADMINSESSION}"
DATASTACK_ADMIN_SETUP_KEY: "${SERVICE_REALBASE64_64_ADMININSTALL:?Coolify must generate SERVICE_REALBASE64_64_ADMININSTALL}" DATASTACK_ADMIN_SETUP_KEY: "${SERVICE_REALBASE64_64_ADMININSTALL:?Coolify must generate SERVICE_REALBASE64_64_ADMININSTALL}"
DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}" DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}"
REDIS_URL: "redis://redis:6379"
depends_on: depends_on:
postgres: postgres:
condition: service_healthy condition: service_healthy
redis:
condition: service_healthy
restart: unless-stopped restart: unless-stopped
healthcheck: healthcheck:
test: test:
@@ -48,9 +51,12 @@ services:
PORT_SOCKET: "4000" PORT_SOCKET: "4000"
DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}" DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}"
DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}" DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}"
REDIS_URL: "redis://redis:6379"
depends_on: depends_on:
postgres: postgres:
condition: service_healthy condition: service_healthy
redis:
condition: service_healthy
restart: unless-stopped restart: unless-stopped
healthcheck: healthcheck:
test: test:
@@ -85,7 +91,7 @@ services:
retries: 3 retries: 3
start_period: 10s start_period: 10s
# Reserved for the planned shared buffer integration; not consumed by the current runtime. # Shared Redis ingestion buffer & distributed flush coordination
redis: redis:
image: redis:7-alpine image: redis:7-alpine
restart: unless-stopped restart: unless-stopped
+33 -16
View File
@@ -1,4 +1,7 @@
import { verifyAdminSessionToken } from "@/core/auth/adminSession"; import {
getAdminSessionSecret,
verifyAdminSessionToken,
} from "@/core/auth/adminSession";
import { NextRequest, NextResponse } from "next/server"; import { NextRequest, NextResponse } from "next/server";
const SESSION_COOKIE = "datastack_admin_session"; const SESSION_COOKIE = "datastack_admin_session";
@@ -13,28 +16,37 @@ function isProtectedApi(pathname: string, method: string) {
return pathname.startsWith("/api/"); return pathname.startsWith("/api/");
} }
function isSameOriginRequest(request: NextRequest) { function matchesRequestHost(candidateUrl: string, request: NextRequest) {
const originHeader = request.headers.get("origin");
const requestHost = request.headers.get("host"); const requestHost = request.headers.get("host");
const forwardedHost = request.headers const forwardedHost = request.headers
.get("x-forwarded-host") .get("x-forwarded-host")
?.split(",")[0] ?.split(",")[0]
?.trim(); ?.trim();
if (!originHeader || (!requestHost && !forwardedHost)) return false; if (!requestHost && !forwardedHost) return false;
try { try {
const origin = new URL(originHeader); const parsedHost = new URL(candidateUrl).host.toLowerCase();
const originHost = origin.host.toLowerCase(); return Boolean(
const matchesHost = (requestHost && parsedHost === requestHost.toLowerCase()) ||
(requestHost && originHost === requestHost.toLowerCase()) || (forwardedHost && parsedHost === forwardedHost.toLowerCase()),
(forwardedHost && originHost === forwardedHost.toLowerCase()); );
return Boolean(matchesHost);
} catch { } catch {
return false; return false;
} }
} }
export function isSameOriginRequest(request: NextRequest) {
const originHeader = request.headers.get("origin");
if (originHeader) {
return matchesRequestHost(originHeader, request);
}
const refererHeader = request.headers.get("referer");
if (refererHeader) {
return matchesRequestHost(refererHeader, request);
}
return false;
}
export async function middleware(request: NextRequest) { export async function middleware(request: NextRequest) {
const { pathname } = request.nextUrl; const { pathname } = request.nextUrl;
const isLoginPage = pathname === "/vcon/login"; const isLoginPage = pathname === "/vcon/login";
@@ -73,14 +85,19 @@ export async function middleware(request: NextRequest) {
return NextResponse.next(); return NextResponse.next();
} }
const sessionSecret = const sessionSecret = getAdminSessionSecret();
process.env.DATASTACK_ADMIN_SESSION_SECRET || if (!sessionSecret && isAdminApi) {
"datastack-admin-session-default-secret-32-chars"; return NextResponse.json(
{ error: "Admin authentication is not configured." },
{ status: 503 },
);
}
const token = request.cookies.get(SESSION_COOKIE)?.value; const token = request.cookies.get(SESSION_COOKIE)?.value;
const authenticated = Boolean( const authenticated = Boolean(
sessionSecret && sessionSecret &&
token && token &&
(await verifyAdminSessionToken(token, sessionSecret)), (await verifyAdminSessionToken(token, sessionSecret)),
); );
if (isLoginPage) { if (isLoginPage) {
+20 -1
View File
@@ -1,3 +1,18 @@
map $http_upgrade $connection_upgrade {
default upgrade;
'' close;
}
map $http_x_forwarded_proto $forwarded_proto {
default $http_x_forwarded_proto;
'' $scheme;
}
map $http_x_forwarded_host $forwarded_host {
default $http_x_forwarded_host;
'' $host;
}
upstream nextjs_upstream { upstream nextjs_upstream {
server 127.0.0.1:3000; server 127.0.0.1:3000;
} }
@@ -17,10 +32,12 @@ server {
proxy_pass http://socketio_upstream; proxy_pass http://socketio_upstream;
proxy_http_version 1.1; proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade; proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade"; proxy_set_header Connection $connection_upgrade;
proxy_set_header Host $host; proxy_set_header Host $host;
proxy_set_header X-Forwarded-Host $forwarded_host;
proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $forwarded_proto;
proxy_read_timeout 86400; proxy_read_timeout 86400;
} }
@@ -29,7 +46,9 @@ server {
proxy_pass http://nextjs_upstream; proxy_pass http://nextjs_upstream;
proxy_http_version 1.1; proxy_http_version 1.1;
proxy_set_header Host $host; proxy_set_header Host $host;
proxy_set_header X-Forwarded-Host $forwarded_host;
proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $forwarded_proto;
} }
} }
+1
View File
@@ -19,6 +19,7 @@
"cors": "^2.8.6", "cors": "^2.8.6",
"csv-parse": "^7.0.3", "csv-parse": "^7.0.3",
"express": "^5.3.0", "express": "^5.3.0",
"ioredis": "^6.0.0",
"json2csv": "^6.0.0-alpha.2", "json2csv": "^6.0.0-alpha.2",
"jsonwebtoken": "^9.0.3", "jsonwebtoken": "^9.0.3",
"lucide-react": "^0.553.0", "lucide-react": "^0.553.0",
+36 -42
View File
@@ -14,31 +14,25 @@ datasource db {
// All collected data is strictly written to CSV files in /storage/data/{clientId}/{YYYY-MM-DD}.csv // All collected data is strictly written to CSV files in /storage/data/{clientId}/{YYYY-MM-DD}.csv
model ClientSDK { model ClientSDK {
id String @id @default(cuid()) clientId String @id @map("client_id")
clientId String @unique
name String name String
secret String secret String
rateLimit Int @default(100) // Max events per second token String
status String @default("active") // active | revoked rateLimit Int @default(100) @map("rate_limit") // Max events per second
totalEvents BigInt @default(0) status String @default("active") // active | revoked
lastActive DateTime? totalEvents BigInt @default(0) @map("total_events")
createdAt DateTime @default(now()) lastActive DateTime? @map("last_active") @db.Timestamptz(6)
updatedAt DateTime @updatedAt createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(6)
updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(6)
apiKeys ApiKey[]
@@map("client_sdks") @@map("client_sdks")
} }
model ApiKey { model DeletedClient {
id String @id @default(cuid()) clientId String @id @map("client_id")
clientId String deletedAt DateTime @default(now()) @map("deleted_at") @db.Timestamptz(6)
client ClientSDK @relation(fields: [clientId], references: [clientId], onDelete: Cascade)
keyHash String
expiresAt DateTime?
createdAt DateTime @default(now())
@@map("api_keys") @@map("deleted_clients")
} }
model AdminUser { model AdminUser {
@@ -46,35 +40,35 @@ model AdminUser {
email String @unique email String @unique
passwordSalt Bytes @map("password_salt") passwordSalt Bytes @map("password_salt")
passwordHash Bytes @map("password_hash") passwordHash Bytes @map("password_hash")
createdAt DateTime @default(now()) @map("created_at") tokenVersion Int @default(1) @map("token_version")
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(6)
@@map("admin_users") @@map("admin_users")
} }
model AdminLoginAttempt {
keyHash String @id @map("key_hash")
windowStartedAt DateTime @map("window_started_at") @db.Timestamptz(6)
failedAttempts Int @map("failed_attempts")
blockedUntil DateTime? @map("blocked_until") @db.Timestamptz(6)
updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(6)
@@map("admin_login_attempts")
}
model AdminSettings { model AdminSettings {
id String @id @default("default") id String @id @default("default")
serverPort Int @default(4000) serverPort Int @default(4000) @map("server_port")
webPort Int @default(3000) webPort Int @default(3000) @map("web_port")
maxEventsPerSec Int @default(100) maxEventsPerSec Int @default(100) @map("max_events_per_sec")
maxPayloadSizeKb Int @default(100) maxPayloadSizeKb Int @default(100) @map("max_payload_size_kb")
batchFlushThreshold Int @default(500) batchFlushThreshold Int @default(500) @map("batch_flush_threshold")
batchFlushIntervalMs Int @default(2000) batchFlushIntervalMs Int @default(2000) @map("batch_flush_interval_ms")
fileRotationMb Int @default(100) fileRotationMb Int @default(100) @map("file_rotation_mb")
jwtSecret String jwtSecret String @map("jwt_secret")
enableIdempotency Boolean @default(true) enableIdempotency Boolean @default(true) @map("enable_idempotency")
autoPruneDays Int @default(30) autoPruneDays Int @default(30) @map("auto_prune_days")
updatedAt DateTime @updatedAt updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(6)
@@map("admin_settings") @@map("admin_settings")
} }
model AuditLog {
id String @id @default(cuid())
action String
actor String
details String? // JSON string
ipAddress String?
createdAt DateTime @default(now())
@@map("audit_logs")
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 632 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 718 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.7 KiB

+7
View File
@@ -0,0 +1,7 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512" width="512" height="512">
<rect width="512" height="512" fill="#0a0a0b" />
<rect x="24" y="24" width="464" height="464" fill="#111113" stroke="#2e2e33" stroke-width="8" />
<rect x="112" y="112" width="288" height="288" fill="#a3e635" />
<rect x="192" y="192" width="128" height="128" fill="#0a0a0b" />
<rect x="232" y="232" width="48" height="48" fill="#e8e8ea" />
</svg>

After

Width:  |  Height:  |  Size: 444 B

+16 -6
View File
@@ -1,21 +1,31 @@
{ {
"name": "DataStack - Realtime Data Platform", "id": "/",
"name": "DataStack - Realtime Data Collection Platform",
"short_name": "DataStack", "short_name": "DataStack",
"description": "Self-hosted universal realtime data collection engine", "description": "Self-hosted universal realtime data collection engine with Socket.IO ingestion, Redis buffer, JWT client auth, CSV batch rotation, and /vcon/ admin panel.",
"start_url": "/vcon/dashboard", "start_url": "/vcon/dashboard",
"scope": "/",
"display": "standalone", "display": "standalone",
"background_color": "#0b0f17", "background_color": "#0a0a0b",
"theme_color": "#2563eb", "theme_color": "#0a0a0b",
"icons": [ "icons": [
{ {
"src": "/icon-192.png", "src": "/icon-192.png",
"sizes": "192x192", "sizes": "192x192",
"type": "image/png" "type": "image/png",
"purpose": "any"
}, },
{ {
"src": "/icon-512.png", "src": "/icon-512.png",
"sizes": "512x512", "sizes": "512x512",
"type": "image/png" "type": "image/png",
"purpose": "any"
},
{
"src": "/icon-maskable-512.png",
"sizes": "512x512",
"type": "image/png",
"purpose": "maskable"
} }
] ]
} }
+45 -7
View File
@@ -1,5 +1,13 @@
const CACHE_NAME = 'datastack-cache-v1'; const CACHE_NAME = 'datastack-cache-v2';
const STATIC_ASSETS = ['/', '/vcon/dashboard', '/manifest.json']; const STATIC_ASSETS = [
'/',
'/manifest.json',
'/icon.svg',
'/icon-192.png',
'/icon-512.png',
'/icon-maskable-512.png',
'/apple-touch-icon.png',
];
self.addEventListener('install', (event) => { self.addEventListener('install', (event) => {
event.waitUntil( event.waitUntil(
@@ -22,13 +30,43 @@ self.addEventListener('activate', (event) => {
}); });
self.addEventListener('fetch', (event) => { self.addEventListener('fetch', (event) => {
// Let WebSocket and API requests pass through const { request } = event;
if (event.request.url.includes('/socket.io/') || event.request.url.includes('/api/')) { if (request.method !== 'GET') return;
let url;
try {
url = new URL(request.url);
} catch {
return; return;
} }
// Only handle same-origin GET requests; never intercept WebSocket, HMR, or API routes
if (url.origin !== self.location.origin) return;
if (
url.pathname.startsWith('/socket.io/') ||
url.pathname.startsWith('/api/') ||
url.pathname.startsWith('/_next/webpack-hmr')
) {
return;
}
event.respondWith( event.respondWith(
fetch(event.request).catch(() => { fetch(request)
return caches.match(event.request); .then((response) => {
}) if (response && response.ok && response.type === 'basic') {
const clone = response.clone();
caches.open(CACHE_NAME).then((cache) => cache.put(request, clone)).catch(() => {});
}
return response;
})
.catch(async () => {
const cached = await caches.match(request);
if (cached) return cached;
return new Response('Offline — network unavailable.', {
status: 503,
statusText: 'Service Unavailable',
headers: { 'Content-Type': 'text/plain; charset=utf-8' },
});
})
); );
}); });
+11 -9
View File
@@ -6,24 +6,24 @@
Official Python client library for **DataStack — Self-Hosted Real-Time Telemetry & Event Logging Platform**, developed by **[Vib Tools](https://vib.tools/)**. Official Python client library for **DataStack — Self-Hosted Real-Time Telemetry & Event Logging Platform**, developed by **[Vib Tools](https://vib.tools/)**.
`datastack-sdk` enables Python services, IoT sensor nodes, microservice workers, and research pipelines to stream structured JSON telemetry events to a DataStack server over Socket.IO with deterministic `threading.Event` acknowledgments. `datastack-sdk` is a compatibility alias for the canonical **`datastack`** package located at [`../sdk/python/datastack`](../sdk/python/datastack). Both `from datastack import DataStack` and `from datastack_sdk import DataStackClient` share a single source of truth with `config.datastack.py` auto-discovery, `CLIENT_ID` wiring, Socket.IO streaming, automatic HTTP fallback, offline disk spooling, and direct `.csv`/`.json`/`.jsonl` file streaming.
--- ---
## Installation ## Installation
Install directly from the `sdk-python` directory: Install the canonical package directly from `sdk/python/datastack`:
```bash
pip install ./sdk/python/datastack
```
Or install the compatibility alias from `sdk-python`:
```bash ```bash
pip install ./sdk-python pip install ./sdk-python
``` ```
For editable development mode:
```bash
cd sdk-python && pip install -e .
```
--- ---
## Quick Start ## Quick Start
@@ -34,10 +34,12 @@ from datastack_sdk import DataStackClient
# 1. Initialize the client with a signed JWT token from /vcon/clients or /vcon/sdk # 1. Initialize the client with a signed JWT token from /vcon/clients or /vcon/sdk
client = DataStackClient( client = DataStackClient(
server_url="http://localhost:4000", server_url="http://localhost:4000",
api_url="http://localhost:3000",
token="YOUR_CLIENT_JWT_TOKEN", token="YOUR_CLIENT_JWT_TOKEN",
client_id="YOUR_CLIENT_ID", # optional (auto-derived from JWT sub/clientId)
) )
# 2. Connect to the DataStack Socket.IO server # 2. Connect to the DataStack Socket.IO server (or auto HTTP fallback)
if client.connect(): if client.connect():
# 3. Stream a single structured event and wait for server acknowledgment # 3. Stream a single structured event and wait for server acknowledgment
res = client.sync( res = client.sync(
+24 -6
View File
@@ -1,10 +1,29 @@
""" """
DataStack Python SDK (`datastack_sdk`) DataStack Python SDK Compatibility Alias (`datastack_sdk`).
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Re-exports all canonical symbols from `sdk/python/datastack` (`datastack`)
Official Python client library for DataStack — Self-Hosted Real-Time Telemetry & Event Logging Platform. so both `from datastack import DataStack` and legacy
`from datastack_sdk import DataStackClient` share a single source of truth.
""" """
from .client import ( import sys
from pathlib import Path
def _ensure_canonical_datastack_on_path() -> None:
try:
import datastack # noqa: F401
return
except ImportError:
repo_canonical = (
Path(__file__).resolve().parents[2] / "sdk" / "python" / "datastack"
)
if repo_canonical.is_dir() and str(repo_canonical) not in sys.path:
sys.path.insert(0, str(repo_canonical))
_ensure_canonical_datastack_on_path()
from datastack import ( # noqa: E402, F401
ConnectionState, ConnectionState,
DataStack, DataStack,
DataStackAuthError, DataStackAuthError,
@@ -19,11 +38,10 @@ from .client import (
DataStackRateLimitError, DataStackRateLimitError,
DiskSpooler, DiskSpooler,
HeartbeatMonitor, HeartbeatMonitor,
__version__,
load_config, load_config,
) )
__version__ = "2.2.0"
__all__ = [ __all__ = [
"DataStack", "DataStack",
"DataStackClient", "DataStackClient",
File diff suppressed because it is too large. Load diff
+4 -318
View File
@@ -1,321 +1,7 @@
""" """Compatibility re-export of canonical `datastack.config`."""
Auto-Discovery Configuration Loader for `config.datastack.py`.
"""
import importlib.util from . import _ensure_canonical_datastack_on_path
import os
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Dict, List, Optional
from urllib.parse import urlparse
from .exceptions import DataStackConfigError _ensure_canonical_datastack_on_path()
CONFIG_FILENAME = "config.datastack.py" from datastack.config import * # noqa: E402, F401, F403
@dataclass
class DataStackConfig:
server_url: str = "http://localhost:4000"
api_url: str = "http://localhost:3000"
client_token: str = ""
client_id: str = ""
auto_connect: bool = True
enable_http_fallback: bool = True
heartbeat_interval_sec: float = 5.0
request_timeout_sec: float = 8.0
max_retries: int = 3
retry_backoff_base_sec: float = 0.5
batch_chunk_size: int = 250
offline_spool_enabled: bool = True
offline_spool_dir: str = ".datastack_spool"
max_spool_size_mb: int = 50
config_source_path: Optional[str] = None
def _derive_default_api_url(server_url: str) -> str:
"""
If API_URL is not explicitly set, infer Port 3000 on the same host when SERVER_URL
uses Port 4000, or fall back to http://localhost:3000.
"""
try:
parsed = urlparse(server_url)
if parsed.scheme and parsed.hostname:
if parsed.port == 4000:
return f"{parsed.scheme}://{parsed.hostname}:3000"
return f"{parsed.scheme}://{parsed.netloc}"
except Exception:
pass
return "http://localhost:3000"
def _candidate_config_paths(explicit_path: Optional[str] = None) -> List[Path]:
candidates: List[Path] = []
if explicit_path:
p = Path(explicit_path).expanduser().resolve()
if p.is_dir():
candidates.append(p / CONFIG_FILENAME)
else:
candidates.append(p)
env_path = os.environ.get("DATASTACK_CONFIG_PATH", "").strip()
if env_path:
ep = Path(env_path).expanduser().resolve()
if ep.is_dir():
candidates.append(ep / CONFIG_FILENAME)
else:
candidates.append(ep)
# Search current working directory and up to 3 parent levels
try:
cwd = Path.cwd().resolve()
candidates.append(cwd / CONFIG_FILENAME)
for parent in list(cwd.parents)[:3]:
candidates.append(parent / CONFIG_FILENAME)
except Exception:
pass
# Search main entry script directory
try:
if sys.argv and sys.argv[0]:
script_dir = Path(sys.argv[0]).resolve().parent
candidates.append(script_dir / CONFIG_FILENAME)
for parent in list(script_dir.parents)[:2]:
candidates.append(parent / CONFIG_FILENAME)
except Exception:
pass
# Deduplicate while preserving order
seen = set()
unique: List[Path] = []
for c in candidates:
key = str(c)
if key not in seen:
seen.add(key)
unique.append(c)
return unique
def _load_python_config_file(file_path: Path) -> Dict[str, Any]:
try:
spec = importlib.util.spec_from_file_location(
"datastack_user_config", str(file_path)
)
if spec is None or spec.loader is None:
raise DataStackConfigError(
f"Unable to load configuration specification from {file_path}"
)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
raw: Dict[str, Any] = {}
if hasattr(module, "DATASTACK_CONFIG") and isinstance(
getattr(module, "DATASTACK_CONFIG"), dict
):
raw.update(getattr(module, "DATASTACK_CONFIG"))
# Also support top-level module variables (e.g., SERVER_URL = "...", CLIENT_TOKEN = "...")
for attr in dir(module):
if attr.isupper() and attr != "DATASTACK_CONFIG":
raw[attr] = getattr(module, attr)
return raw
except DataStackConfigError:
raise
except Exception as exc:
raise DataStackConfigError(
f"Failed to parse {file_path}: {exc}"
) from exc
def load_config(
config_path: Optional[str] = None,
require_token: bool = False,
**overrides: Any,
) -> DataStackConfig:
"""
Discovers and loads `config.datastack.py` from the client application's
directory, merges environment variables (`DATASTACK_*`), and applies
any explicit keyword overrides.
"""
file_values: Dict[str, Any] = {}
resolved_source: Optional[str] = None
for candidate in _candidate_config_paths(config_path):
if candidate.is_file():
file_values = _load_python_config_file(candidate)
resolved_source = str(candidate)
break
if config_path and not resolved_source:
raise DataStackConfigError(
f"Explicitly requested config file was not found: {config_path}"
)
# Normalize keys to uppercase for lookup
normalized_file = {str(k).upper(): v for k, v in file_values.items()}
normalized_overrides = {
str(k).upper(): v for k, v in overrides.items() if v is not None
}
def pick(keys: List[str], env_keys: List[str], default: Any) -> Any:
for k in keys:
if k in normalized_overrides:
return normalized_overrides[k]
for ek in env_keys:
env_val = os.environ.get(ek)
if env_val is not None and str(env_val).strip() != "":
return env_val
for k in keys:
if k in normalized_file and normalized_file[k] is not None:
return normalized_file[k]
return default
def to_bool(val: Any, default: bool) -> bool:
if isinstance(val, bool):
return val
if isinstance(val, str):
return val.strip().lower() in ("1", "true", "yes", "on")
if val is None:
return default
return bool(val)
server_url = str(
pick(
["SERVER_URL", "SOCKET_URL", "WS_URL"],
["DATASTACK_SERVER_URL", "DATASTACK_SOCKET_URL"],
"http://localhost:4000",
)
).rstrip("/")
default_api = _derive_default_api_url(server_url)
api_url = str(
pick(
["API_URL", "HTTP_URL", "WEB_URL"],
["DATASTACK_API_URL", "DATASTACK_HTTP_URL"],
default_api,
)
).rstrip("/")
client_token = str(
pick(
["CLIENT_TOKEN", "TOKEN", "JWT_TOKEN", "API_TOKEN"],
["DATASTACK_CLIENT_TOKEN", "DATASTACK_TOKEN"],
"",
)
).strip()
client_id = str(
pick(
["CLIENT_ID"],
["DATASTACK_CLIENT_ID"],
"",
)
).strip()
cfg = DataStackConfig(
server_url=server_url,
api_url=api_url,
client_token=client_token,
client_id=client_id,
auto_connect=to_bool(
pick(["AUTO_CONNECT"], ["DATASTACK_AUTO_CONNECT"], True), True
),
enable_http_fallback=to_bool(
pick(
["ENABLE_HTTP_FALLBACK", "HTTP_FALLBACK"],
["DATASTACK_ENABLE_HTTP_FALLBACK"],
True,
),
True,
),
heartbeat_interval_sec=max(
1.0,
float(
pick(
["HEARTBEAT_INTERVAL_SEC", "HEARTBEAT_INTERVAL"],
["DATASTACK_HEARTBEAT_INTERVAL_SEC"],
5.0,
)
),
),
request_timeout_sec=max(
1.0,
float(
pick(
["REQUEST_TIMEOUT_SEC", "TIMEOUT"],
["DATASTACK_REQUEST_TIMEOUT_SEC"],
8.0,
)
),
),
max_retries=max(
0,
int(
pick(
["MAX_RETRIES", "RETRIES"],
["DATASTACK_MAX_RETRIES"],
3,
)
),
),
retry_backoff_base_sec=max(
0.1,
float(
pick(
["RETRY_BACKOFF_BASE_SEC"],
["DATASTACK_RETRY_BACKOFF_BASE_SEC"],
0.5,
)
),
),
batch_chunk_size=max(
1,
int(
pick(
["BATCH_CHUNK_SIZE", "CHUNK_SIZE"],
["DATASTACK_BATCH_CHUNK_SIZE"],
250,
)
),
),
offline_spool_enabled=to_bool(
pick(
["OFFLINE_SPOOL_ENABLED"],
["DATASTACK_OFFLINE_SPOOL_ENABLED"],
True,
),
True,
),
offline_spool_dir=str(
pick(
["OFFLINE_SPOOL_DIR", "SPOOL_DIR"],
["DATASTACK_OFFLINE_SPOOL_DIR"],
".datastack_spool",
)
),
max_spool_size_mb=max(
1,
int(
pick(
["MAX_SPOOL_SIZE_MB"],
["DATASTACK_MAX_SPOOL_SIZE_MB"],
50,
)
),
),
config_source_path=resolved_source,
)
if require_token and (
not cfg.client_token
or cfg.client_token == "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN"
):
raise DataStackConfigError(
"DataStack CLIENT_TOKEN is missing. Add your token to `config.datastack.py`, "
"set DATASTACK_TOKEN in your environment, or pass token='...' to DataStack()."
)
return cfg
+4 -69
View File
@@ -1,72 +1,7 @@
""" """Compatibility re-export of canonical `datastack.exceptions`."""
Structured Exception Hierarchy for the DataStack Python SDK.
"""
from typing import Optional from . import _ensure_canonical_datastack_on_path
_ensure_canonical_datastack_on_path()
class DataStackError(Exception): from datastack.exceptions import * # noqa: E402, F401, F403
"""Base exception for all DataStack SDK errors."""
def __init__(
self,
message: str,
code: Optional[str] = None,
retryable: bool = False,
) -> None:
super().__init__(message)
self.message = message
self.code = code or "DATASTACK_ERROR"
self.retryable = retryable
class DataStackConfigError(DataStackError):
"""Raised when `config.datastack.py` or required credentials are missing or invalid."""
def __init__(self, message: str) -> None:
super().__init__(message, code="CONFIG_ERROR", retryable=False)
class DataStackAuthError(DataStackError):
"""
Raised when authentication fails (`AUTH_REQUIRED`, `AUTH_INVALID`,
`CLIENT_REVOKED`, or `TOKEN_ROTATED`). Non-retryable until credentials are updated.
"""
def __init__(self, message: str, code: str = "AUTH_ERROR") -> None:
super().__init__(message, code=code, retryable=False)
class DataStackRateLimitError(DataStackError):
"""Raised when `RATE_LIMIT_EXCEEDED` persists after all backoff retries."""
def __init__(self, message: str) -> None:
super().__init__(message, code="RATE_LIMIT_EXCEEDED", retryable=True)
class DataStackPayloadTooLargeError(DataStackError):
"""Raised when a single event exceeds the server's `maxPayloadSizeKb` limit."""
def __init__(self, message: str) -> None:
super().__init__(message, code="PAYLOAD_TOO_LARGE", retryable=False)
class DataStackConnectionError(DataStackError):
"""Raised when both WebSocket and HTTP fallback are unreachable and offline spooling is disabled."""
def __init__(self, message: str) -> None:
super().__init__(message, code="CONNECTION_ERROR", retryable=True)
class DataStackFileError(DataStackError):
"""Raised when a file passed to `send_file()` is missing, unsupported, or malformed."""
def __init__(self, message: str) -> None:
super().__init__(message, code="FILE_ERROR", retryable=False)
class DataStackOfflineSpoolError(DataStackError):
"""Raised when the local offline disk spool directory is full or unwriteable."""
def __init__(self, message: str) -> None:
super().__init__(message, code="OFFLINE_SPOOL_ERROR", retryable=False)
+4 -183
View File
@@ -1,186 +1,7 @@
""" """Compatibility re-export of canonical `datastack.heartbeat`."""
Smart Heartbeat Monitor & Connection State Machine (`HeartbeatMonitor`).
Lifecycle States: from . import _ensure_canonical_datastack_on_path
- `CONNECTED_IDLE`: WebSocket (Port 4000) is connected and verified. Heartbeat sleeps in IDLE mode.
- `FALLBACK_HTTP`: WebSocket disconnected. Client automatically sends events via HTTP (`POST /api/events`)
while Heartbeat actively probes to reconnect and verify WebSocket, then returns to `CONNECTED_IDLE`.
- `OFFLINE_SPOOL`: Device is completely offline. Events spool to disk (`.datastack_spool/spool.jsonl`).
Heartbeat monitors network restoration, reconnects, replays spooled events, and returns to `CONNECTED_IDLE`.
- `RECONNECTING_VERIFY`: Heartbeat is verifying handshake (`ping_verify`) and draining spooled events.
- `STOPPED`: Client was closed intentionally or halted due to non-retryable auth revocation.
"""
import logging _ensure_canonical_datastack_on_path()
import threading
import time
from typing import Any, Callable, Dict, Optional
logger = logging.getLogger("datastack.heartbeat") from datastack.heartbeat import * # noqa: E402, F401, F403
class ConnectionState:
CONNECTED_IDLE = "CONNECTED_IDLE"
FALLBACK_HTTP = "FALLBACK_HTTP"
OFFLINE_SPOOL = "OFFLINE_SPOOL"
RECONNECTING_VERIFY = "RECONNECTING_VERIFY"
STOPPED = "STOPPED"
class HeartbeatMonitor:
"""
Background recovery thread that sleeps in zero-CPU IDLE mode while WebSocket is
healthy (`CONNECTED_IDLE`) and wakes up automatically when WebSocket disconnects
or when the client device goes offline.
"""
def __init__(
self,
interval_sec: float,
recovery_callback: Callable[[], bool],
) -> None:
self.interval_sec = max(0.2, float(interval_sec))
self._recovery_callback = recovery_callback
self._state = ConnectionState.CONNECTED_IDLE
self._closed = False
self._last_error: Optional[str] = None
self._recovery_attempts = 0
self._last_verified_at: Optional[float] = None
self._lock = threading.RLock()
self._wake_event = threading.Event()
self._stop_event = threading.Event()
self._thread: Optional[threading.Thread] = None
@property
def state(self) -> str:
with self._lock:
return self._state
@property
def is_idle(self) -> bool:
with self._lock:
return self._state == ConnectionState.CONNECTED_IDLE
def set_idle_connected(self) -> None:
"""
Marks the connection as healthy (`CONNECTED_IDLE`) and puts the Heartbeat
Monitor into IDLE mode (`_wake_event.clear()`).
"""
with self._lock:
self._closed = False
self._state = ConnectionState.CONNECTED_IDLE
self._recovery_attempts = 0
self._last_error = None
self._last_verified_at = time.time()
self._wake_event.clear()
def trigger_fallback(self, reason: str = "WebSocket disconnected") -> None:
"""
Transitions to `FALLBACK_HTTP` and wakes up the Heartbeat Monitor thread
to repair the WebSocket connection in the background.
"""
with self._lock:
if self._closed:
return
self._state = ConnectionState.FALLBACK_HTTP
self._last_error = reason
self._ensure_thread_started()
self._wake_event.set()
def trigger_offline(self, reason: str = "Device offline") -> None:
"""
Transitions to `OFFLINE_SPOOL` when both WebSocket and HTTP fallback are
unreachable, waking the Heartbeat Monitor to watch for network return.
"""
with self._lock:
if self._closed:
return
self._state = ConnectionState.OFFLINE_SPOOL
self._last_error = reason
self._ensure_thread_started()
self._wake_event.set()
def set_reconnecting(self) -> None:
with self._lock:
if not self._closed:
self._state = ConnectionState.RECONNECTING_VERIFY
def start(self) -> None:
with self._lock:
self._closed = False
self._stop_event.clear()
self._ensure_thread_started()
def stop(self, reason: Optional[str] = None) -> None:
with self._lock:
self._closed = True
self._state = ConnectionState.STOPPED
if reason:
self._last_error = reason
self._stop_event.set()
self._wake_event.set()
def status_snapshot(self) -> Dict[str, Any]:
with self._lock:
return {
"state": self._state,
"heartbeatMode": (
"IDLE"
if self._state == ConnectionState.CONNECTED_IDLE
else "ACTIVE_RECOVERY"
if self._state
in (
ConnectionState.FALLBACK_HTTP,
ConnectionState.OFFLINE_SPOOL,
ConnectionState.RECONNECTING_VERIFY,
)
else "STOPPED"
),
"recoveryAttempts": self._recovery_attempts,
"lastError": self._last_error,
"lastVerifiedAt": self._last_verified_at,
"intervalSec": self.interval_sec,
}
def _ensure_thread_started(self) -> None:
if self._thread is not None and self._thread.is_alive():
return
self._stop_event.clear()
self._thread = threading.Thread(
target=self._run_loop,
name="DataStack-HeartbeatMonitor",
daemon=True,
)
self._thread.start()
def _run_loop(self) -> None:
while not self._stop_event.is_set():
# Wait in zero-CPU IDLE mode until woken by a disconnect or offline trigger
self._wake_event.wait()
if self._stop_event.is_set():
break
# Wait for the heartbeat interval before probing
if self._stop_event.wait(self.interval_sec):
break
if not self._wake_event.is_set():
continue
with self._lock:
if self._closed or self._state == ConnectionState.STOPPED:
break
self._recovery_attempts += 1
try:
recovered_to_idle = self._recovery_callback()
if recovered_to_idle:
self.set_idle_connected()
logger.info(
"[DataStack Heartbeat] WebSocket reconnected & verified. Returning to IDLE mode."
)
except Exception as exc:
with self._lock:
self._last_error = str(exc)
logger.debug("[DataStack Heartbeat] Recovery probe failed: %s", exc)
+4 -161
View File
@@ -1,164 +1,7 @@
""" """Compatibility re-export of canonical `datastack.spool`."""
Thread-Safe Offline Disk Spooler (`DiskSpooler`).
Persists unsent telemetry events to `.datastack_spool/spool.jsonl` when a client
device loses network connectivity completely, and drains them in deterministic
batches once the Heartbeat Monitor restores connectivity.
"""
import json from . import _ensure_canonical_datastack_on_path
import os
import threading
from pathlib import Path
from typing import Any, Dict, List
from .exceptions import DataStackOfflineSpoolError _ensure_canonical_datastack_on_path()
from datastack.spool import * # noqa: E402, F401, F403
class DiskSpooler:
"""
Append-only JSONL disk queue with atomic batch drainage and size cap protection.
"""
def __init__(
self,
spool_dir: str = ".datastack_spool",
max_size_mb: int = 50,
enabled: bool = True,
) -> None:
self.enabled = enabled
self.spool_dir = Path(spool_dir)
self.spool_file = self.spool_dir / "spool.jsonl"
self.max_bytes = max(1, int(max_size_mb)) * 1024 * 1024
self._lock = threading.RLock()
def _ensure_dir(self) -> None:
try:
self.spool_dir.mkdir(parents=True, exist_ok=True)
except Exception as exc:
raise DataStackOfflineSpoolError(
f"Cannot create offline spool directory '{self.spool_dir}': {exc}"
) from exc
def enqueue(self, events: List[Dict[str, Any]]) -> int:
"""
Appends one or more formatted events (`{eventId, timestamp, data}`)
to `.datastack_spool/spool.jsonl`. Returns count of spooled events.
"""
if not self.enabled or not events:
return 0
with self._lock:
self._ensure_dir()
current_size = (
self.spool_file.stat().st_size if self.spool_file.exists() else 0
)
lines: List[str] = []
added_bytes = 0
for evt in events:
serialized = json.dumps(evt, ensure_ascii=False) + "\n"
encoded_len = len(serialized.encode("utf-8"))
if current_size + added_bytes + encoded_len > self.max_bytes:
raise DataStackOfflineSpoolError(
f"Offline spool limit ({self.max_bytes // (1024 * 1024)} MB) "
f"reached at '{self.spool_file}'."
)
lines.append(serialized)
added_bytes += encoded_len
try:
with self.spool_file.open("a", encoding="utf-8") as fh:
fh.writelines(lines)
fh.flush()
except Exception as exc:
raise DataStackOfflineSpoolError(
f"Failed writing to offline spool file '{self.spool_file}': {exc}"
) from exc
return len(lines)
def peek_batch(self, limit: int = 250) -> List[Dict[str, Any]]:
"""
Reads up to `limit` spooled events from the front of the spool file
without removing them until `ack_batch()` confirms delivery.
"""
if not self.enabled:
return []
with self._lock:
if not self.spool_file.exists():
return []
batch: List[Dict[str, Any]] = []
try:
with self.spool_file.open("r", encoding="utf-8") as fh:
for line in fh:
if len(batch) >= limit:
break
stripped = line.strip()
if not stripped:
continue
try:
parsed = json.loads(stripped)
if isinstance(parsed, dict):
batch.append(parsed)
except json.JSONDecodeError:
continue
except Exception:
return []
return batch
def ack_batch(self, count: int) -> int:
"""
Removes the first `count` non-empty lines from the spool file atomically
after the Heartbeat Monitor confirms they have been ingested by DataStack.
"""
if not self.enabled or count <= 0:
return 0
with self._lock:
if not self.spool_file.exists():
return 0
try:
with self.spool_file.open("r", encoding="utf-8") as fh:
all_lines = [ln for ln in fh if ln.strip()]
if count >= len(all_lines):
self.spool_file.unlink(missing_ok=True)
return len(all_lines)
remaining = all_lines[count:]
tmp_file = self.spool_dir / "spool.jsonl.tmp"
with tmp_file.open("w", encoding="utf-8") as fh:
fh.writelines(remaining)
fh.flush()
os.replace(str(tmp_file), str(self.spool_file))
return count
except Exception:
return 0
def pending_count(self) -> int:
"""Returns the number of queued events waiting in the disk spool."""
if not self.enabled:
return 0
with self._lock:
if not self.spool_file.exists():
return 0
try:
with self.spool_file.open("r", encoding="utf-8") as fh:
return sum(1 for ln in fh if ln.strip())
except Exception:
return 0
def size_bytes(self) -> int:
"""Returns current disk usage of the spool file in bytes."""
with self._lock:
if not self.spool_file.exists():
return 0
try:
return self.spool_file.stat().st_size
except Exception:
return 0
+1
View File
@@ -33,6 +33,7 @@ DATASTACK_CONFIG = {
"SERVER_URL": "http://localhost:4000", # Port 4000 Socket.IO Ingestion Service "SERVER_URL": "http://localhost:4000", # Port 4000 Socket.IO Ingestion Service
"API_URL": "http://localhost:3000", # Port 3000 HTTP Fallback & File Ingestion API "API_URL": "http://localhost:3000", # Port 3000 HTTP Fallback & File Ingestion API
"CLIENT_TOKEN": "YOUR_CLIENT_JWT_TOKEN", # Provisioned in /vcon/clients "CLIENT_TOKEN": "YOUR_CLIENT_JWT_TOKEN", # Provisioned in /vcon/clients
"CLIENT_ID": "", # Optional override (defaults to JWT sub/clientId)
"AUTO_CONNECT": True, "AUTO_CONNECT": True,
"ENABLE_HTTP_FALLBACK": True, "ENABLE_HTTP_FALLBACK": True,
"HEARTBEAT_INTERVAL_SEC": 5.0, "HEARTBEAT_INTERVAL_SEC": 5.0,
+4 -6
View File
@@ -5,6 +5,7 @@ Socket.IO ingestion (Port 4000), automatic Heartbeat HTTP fallback (Port 3000),
offline disk spooling, and direct `.csv` / `.json` / `.jsonl` file streaming. offline disk spooling, and direct `.csv` / `.json` / `.jsonl` file streaming.
""" """
from .client import DataStack, DataStackClient
from .config import DataStackConfig, load_config from .config import DataStackConfig, load_config
from .exceptions import ( from .exceptions import (
DataStackAuthError, DataStackAuthError,
@@ -16,14 +17,9 @@ from .exceptions import (
DataStackPayloadTooLargeError, DataStackPayloadTooLargeError,
DataStackRateLimitError, DataStackRateLimitError,
) )
from .heartbeat import ConnectionState, HeartbeatMonitor
from .spool import DiskSpooler from .spool import DiskSpooler
try:
from .client import DataStack, DataStackClient
except ImportError:
DataStack = None # type: ignore
DataStackClient = None # type: ignore
__version__ = "2.2.0" __version__ = "2.2.0"
__all__ = [ __all__ = [
@@ -31,6 +27,8 @@ __all__ = [
"DataStackClient", "DataStackClient",
"DataStackConfig", "DataStackConfig",
"load_config", "load_config",
"ConnectionState",
"HeartbeatMonitor",
"DiskSpooler", "DiskSpooler",
"DataStackError", "DataStackError",
"DataStackConfigError", "DataStackConfigError",
+95 -21
View File
@@ -13,6 +13,7 @@ Features:
and lossless partial-batch rate-limit retry (`429 RATE_LIMIT_EXCEEDED`). and lossless partial-batch rate-limit retry (`429 RATE_LIMIT_EXCEEDED`).
""" """
import base64
import csv import csv
import hashlib import hashlib
import json import json
@@ -44,6 +45,7 @@ from .exceptions import (
DataStackConnectionError, DataStackConnectionError,
DataStackError, DataStackError,
DataStackFileError, DataStackFileError,
DataStackOfflineSpoolError,
DataStackPayloadTooLargeError, DataStackPayloadTooLargeError,
DataStackRateLimitError, DataStackRateLimitError,
) )
@@ -72,6 +74,23 @@ def _make_deterministic_file_event_id(
return f"file_{safe_stem}_{row_index}_{digest}" return f"file_{safe_stem}_{row_index}_{digest}"
def _decode_jwt_client_id(token: str) -> str:
"""Safely extracts `clientId` or `sub` claim from an unverified JWT payload string."""
try:
parts = (token or "").split(".")
if len(parts) != 3:
return ""
payload_b64 = parts[1]
padding = "=" * (-len(payload_b64) % 4)
raw = base64.urlsafe_b64decode(payload_b64 + padding).decode("utf-8")
claims = json.loads(raw)
if isinstance(claims, dict):
return str(claims.get("clientId") or claims.get("sub") or "").strip()
except Exception:
pass
return ""
class DataStackClient: class DataStackClient:
""" """
Production-grade DataStack Python Client with `config.datastack.py` auto-loading, Production-grade DataStack Python Client with `config.datastack.py` auto-loading,
@@ -87,6 +106,7 @@ class DataStackClient:
auto_connect: Optional[bool] = None, auto_connect: Optional[bool] = None,
reconnection_attempts: Optional[int] = None, reconnection_attempts: Optional[int] = None,
reconnection_delay: Optional[float] = None, reconnection_delay: Optional[float] = None,
client_id: Optional[str] = None,
**kwargs: Any, **kwargs: Any,
) -> None: ) -> None:
if reconnection_attempts is not None and "MAX_RETRIES" not in kwargs: if reconnection_attempts is not None and "MAX_RETRIES" not in kwargs:
@@ -105,6 +125,7 @@ class DataStackClient:
SERVER_URL=server_url, SERVER_URL=server_url,
CLIENT_TOKEN=token, CLIENT_TOKEN=token,
API_URL=api_url, API_URL=api_url,
CLIENT_ID=client_id,
AUTO_CONNECT=auto_connect, AUTO_CONNECT=auto_connect,
**kwargs, **kwargs,
) )
@@ -112,6 +133,20 @@ class DataStackClient:
self.server_url = self.config.server_url self.server_url = self.config.server_url
self.api_url = self.config.api_url self.api_url = self.config.api_url
self.token = self.config.client_token self.token = self.config.client_token
configured_client_id = (self.config.client_id or "").strip()
token_client_id = _decode_jwt_client_id(self.token)
if (
configured_client_id
and token_client_id
and configured_client_id != token_client_id
):
raise DataStackConfigError(
f"Configured CLIENT_ID '{configured_client_id}' does not match "
f"CLIENT_TOKEN subject '{token_client_id}'."
)
self.client_id: str = configured_client_id or token_client_id
self.is_connected = False self.is_connected = False
self._lock = threading.RLock() self._lock = threading.RLock()
self._is_draining_spool = False self._is_draining_spool = False
@@ -311,8 +346,11 @@ class DataStackClient:
raise DataStackAuthError(msg) from exc raise DataStackAuthError(msg) from exc
return False return False
# Verify handshake & PostgreSQL client status via `ping_verify` # Verify handshake & PostgreSQL client status via `ping_verify` while still serialized under RLock
return self._verify_websocket_handshake(timeout=effective_timeout) verified = self._verify_websocket_handshake(timeout=effective_timeout)
if not verified:
self.is_connected = False
return verified
def _verify_websocket_handshake( def _verify_websocket_handshake(
self, timeout: Optional[float] = None self, timeout: Optional[float] = None
@@ -424,6 +462,8 @@ class DataStackClient:
"Content-Type": "application/json", "Content-Type": "application/json",
"Accept": "application/json", "Accept": "application/json",
} }
if self.client_id:
headers["X-DataStack-Client-Id"] = self.client_id
if requests is not None: if requests is not None:
resp = requests.request( resp = requests.request(
@@ -735,10 +775,14 @@ class DataStackClient:
): ):
break break
self.spooler.ack_batch(len(batch)) unprocessed_list = (
if isinstance(unprocessed, list) and len(unprocessed) > 0: unprocessed if isinstance(unprocessed, list) else []
self.spooler.enqueue(unprocessed) )
total_flushed += max(0, len(batch) - len(unprocessed)) self.spooler.ack_and_requeue_front(
len(batch), unprocessed_list
)
if unprocessed_list:
total_flushed += max(0, len(batch) - len(unprocessed_list))
break break
total_flushed += len(batch) total_flushed += len(batch)
@@ -767,11 +811,13 @@ class DataStackClient:
""" """
resolved_id = (event_id or "").strip() or _make_event_id() resolved_id = (event_id or "").strip() or _make_event_id()
resolved_ts = (timestamp or "").strip() or _utc_now_iso() resolved_ts = (timestamp or "").strip() or _utc_now_iso()
envelope = { envelope: Dict[str, Any] = {
"eventId": resolved_id, "eventId": resolved_id,
"timestamp": resolved_ts, "timestamp": resolved_ts,
"data": data, "data": data,
} }
if self.client_id:
envelope["clientId"] = self.client_id
return self._execute_with_retry_and_fallback( return self._execute_with_retry_and_fallback(
ws_event="sync_data", ws_event="sync_data",
ws_payload=envelope, ws_payload=envelope,
@@ -810,6 +856,7 @@ class DataStackClient:
if not isinstance(source_list, list) or len(source_list) == 0: if not isinstance(source_list, list) or len(source_list) == 0:
return {"success": True, "processed": 0, "rejected": 0} return {"success": True, "processed": 0, "rejected": 0}
envelope_keys = {"eventId", "timestamp", "data", "clientId"}
formatted: List[Dict[str, Any]] = [] formatted: List[Dict[str, Any]] = []
for item in source_list: for item in source_list:
safe_item = item if isinstance(item, dict) else {"value": item} safe_item = item if isinstance(item, dict) else {"value": item}
@@ -819,14 +866,21 @@ class DataStackClient:
evt_ts = ( evt_ts = (
str(safe_item.get("timestamp") or "").strip() or _utc_now_iso() str(safe_item.get("timestamp") or "").strip() or _utc_now_iso()
) )
evt_data = safe_item.get("data", safe_item) is_envelope = "data" in safe_item and set(safe_item.keys()).issubset(
formatted.append( envelope_keys
{
"eventId": evt_id,
"timestamp": evt_ts,
"data": evt_data,
}
) )
evt_data = safe_item["data"] if is_envelope else safe_item
evt_client_id = (
str(safe_item.get("clientId") or "").strip() or self.client_id
)
entry: Dict[str, Any] = {
"eventId": evt_id,
"timestamp": evt_ts,
"data": evt_data,
}
if evt_client_id:
entry["clientId"] = evt_client_id
formatted.append(entry)
return self._send_formatted_batch_recursive( return self._send_formatted_batch_recursive(
formatted, formatted,
@@ -1057,13 +1111,14 @@ class DataStackClient:
) )
evt_ts = existing_ts or _utc_now_iso() evt_ts = existing_ts or _utc_now_iso()
buffer.append( file_entry: Dict[str, Any] = {
{ "eventId": evt_id,
"eventId": evt_id, "timestamp": evt_ts,
"timestamp": evt_ts, "data": enriched,
"data": enriched, }
} if self.client_id:
) file_entry["clientId"] = self.client_id
buffer.append(file_entry)
if len(buffer) >= effective_chunk: if len(buffer) >= effective_chunk:
res = self.send_batch(buffer, file_name=path.name) res = self.send_batch(buffer, file_name=path.name)
total_processed += int(res.get("processed", 0)) total_processed += int(res.get("processed", 0))
@@ -1162,6 +1217,7 @@ class DataStackClient:
hb = self.heartbeat.status_snapshot() hb = self.heartbeat.status_snapshot()
return { return {
"connectedWebSocket": self.is_connected, "connectedWebSocket": self.is_connected,
"clientId": self.client_id or None,
"serverUrl": self.server_url, "serverUrl": self.server_url,
"apiUrl": self.api_url, "apiUrl": self.api_url,
"configSource": self.config.config_source_path, "configSource": self.config.config_source_path,
@@ -1176,3 +1232,21 @@ class DataStackClient:
# Convenient alias so developers can write `from datastack import DataStack` # Convenient alias so developers can write `from datastack import DataStack`
DataStack = DataStackClient DataStack = DataStackClient
__all__ = [
"DataStack",
"DataStackClient",
"DataStackConfig",
"load_config",
"ConnectionState",
"HeartbeatMonitor",
"DiskSpooler",
"DataStackError",
"DataStackConfigError",
"DataStackAuthError",
"DataStackConnectionError",
"DataStackRateLimitError",
"DataStackPayloadTooLargeError",
"DataStackFileError",
"DataStackOfflineSpoolError",
]
@@ -171,6 +171,11 @@ class HeartbeatMonitor:
with self._lock: with self._lock:
if self._closed or self._state == ConnectionState.STOPPED: if self._closed or self._state == ConnectionState.STOPPED:
break break
if (
self._state == ConnectionState.CONNECTED_IDLE
or not self._wake_event.is_set()
):
continue
self._recovery_attempts += 1 self._recovery_attempts += 1
try: try:
+44 -1
View File
@@ -9,7 +9,7 @@ import json
import os import os
import threading import threading
from pathlib import Path from pathlib import Path
from typing import Any, Dict, List from typing import Any, Dict, List, Optional
from .exceptions import DataStackOfflineSpoolError from .exceptions import DataStackOfflineSpoolError
@@ -139,6 +139,49 @@ class DiskSpooler:
except Exception: except Exception:
return 0 return 0
def ack_and_requeue_front(
self,
acked_count: int,
unprocessed: Optional[List[Dict[str, Any]]] = None,
) -> int:
"""
Atomically removes `acked_count` head lines from `spool.jsonl` while
preserving any `unprocessed` events at the FRONT of the queue in a single
atomic `os.replace()` write (FIFO safe, zero crash-window data loss).
"""
if not self.enabled or acked_count <= 0:
return 0
with self._lock:
if not self.spool_file.exists():
return 0
try:
with self.spool_file.open("r", encoding="utf-8") as fh:
all_lines = [ln for ln in fh if ln.strip()]
consumed = min(acked_count, len(all_lines))
tail_lines = all_lines[consumed:]
head_lines = [
json.dumps(evt, ensure_ascii=False) + "\n"
for evt in (unprocessed or [])
]
combined = head_lines + tail_lines
if not combined:
self.spool_file.unlink(missing_ok=True)
return consumed
self._ensure_dir()
tmp_file = self.spool_dir / "spool.jsonl.tmp"
with tmp_file.open("w", encoding="utf-8") as fh:
fh.writelines(combined)
fh.flush()
os.replace(str(tmp_file), str(self.spool_file))
return max(0, consumed - len(head_lines))
except Exception:
return 0
def pending_count(self) -> int: def pending_count(self) -> int:
"""Returns the number of queued events waiting in the disk spool.""" """Returns the number of queued events waiting in the disk spool."""
if not self.enabled: if not self.enabled:
+714 -98
View File
@@ -1,8 +1,10 @@
const crypto = require('node:crypto');
const express = require('express'); const express = require('express');
const http = require('http'); const http = require('http');
const cors = require('cors'); const cors = require('cors');
const { Server } = require('socket.io'); const { Server } = require('socket.io');
const { Pool } = require('pg'); const { Pool } = require('pg');
const Redis = require('ioredis');
const { setupSocketHandlers } = require('./socket/handlers'); const { setupSocketHandlers } = require('./socket/handlers');
const { CsvWriterWorker } = require('./workers/csvWriter'); const { CsvWriterWorker } = require('./workers/csvWriter');
@@ -11,21 +13,185 @@ const app = express();
app.use(cors({ origin: '*' })); app.use(cors({ origin: '*' }));
app.use(express.json({ limit: '10mb' })); app.use(express.json({ limit: '10mb' }));
const REDIS_BUFFER_KEY = 'datastack:buffer:events';
const REDIS_RECENT_KEY = 'datastack:recent:events';
const REDIS_FLUSH_LOCK_KEY = 'datastack:flush:lock';
const REDIS_RECENT_LIMIT = 500;
const FLUSH_LOCK_TTL_MS = 15000;
const UNLOCK_LUA_SCRIPT =
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end';
const bufferQueue = []; const bufferQueue = [];
const writer = new CsvWriterWorker(); const writer = new CsvWriterWorker();
let redisClient = undefined;
let redisUrlConfigured = '';
let redisConnected = false;
let redisTestOverride = undefined;
const pendingRedisWrites = new Set();
function getRedisClient() {
if (redisTestOverride !== undefined) {
return redisTestOverride;
}
const configuredUrl = (process.env.REDIS_URL || '').trim();
if (!configuredUrl) {
if (redisClient && typeof redisClient.disconnect === 'function') {
try {
redisClient.disconnect();
} catch {
// ignore
}
}
redisClient = null;
redisUrlConfigured = '';
redisConnected = false;
return null;
}
if (redisClient && redisUrlConfigured === configuredUrl) {
return redisClient;
}
if (redisClient && typeof redisClient.disconnect === 'function') {
try {
redisClient.disconnect();
} catch {
// ignore
}
}
try {
const client = new Redis(configuredUrl, {
maxRetriesPerRequest: 2,
enableReadyCheck: true,
retryStrategy: (times) => Math.min(times * 250, 3000),
});
client.on('ready', () => {
redisConnected = true;
});
client.on('connect', () => {
redisConnected = true;
});
client.on('close', () => {
redisConnected = false;
});
client.on('end', () => {
redisConnected = false;
});
client.on('error', () => {
redisConnected = false;
});
redisClient = client;
redisUrlConfigured = configuredUrl;
redisConnected = client.status === 'ready' || client.status === 'connect';
return redisClient;
} catch {
redisClient = null;
redisUrlConfigured = configuredUrl;
redisConnected = false;
return null;
}
}
function setRedisClientForTests(client) {
redisTestOverride = client;
redisConnected = Boolean(client);
}
async function pushEventsToRedisOrFallback(events) {
if (!events || events.length === 0) return bufferQueue.length;
const redis = getRedisClient();
if (redis) {
const serialized = events.map((e) => JSON.stringify(e));
try {
const len = await redis.rpush(REDIS_BUFFER_KEY, ...serialized);
await redis.lpush(REDIS_RECENT_KEY, ...serialized);
await redis.ltrim(REDIS_RECENT_KEY, 0, REDIS_RECENT_LIMIT - 1);
redisConnected = true;
return Number(len) + bufferQueue.length;
} catch {
redisConnected = false;
}
}
bufferQueue.push(...events);
return bufferQueue.length;
}
const IS_PRODUCTION = process.env.NODE_ENV === 'production';
const DEV_DEFAULT_JWT_SECRET = 'datastack-jwt-default-secret-key-32-chars-long';
function resolveInitialJwtSecret() {
const configured = (process.env.DATASTACK_JWT_SECRET || '').trim();
if (IS_PRODUCTION) {
return configured.length >= 32 ? configured : '';
}
if (configured.length > 0) {
return configured.length >= 32 ? configured : '';
}
return DEV_DEFAULT_JWT_SECRET;
}
const runtimeConfig = { const runtimeConfig = {
jwtSecret: jwtSecret: resolveInitialJwtSecret(),
process.env.DATASTACK_JWT_SECRET ||
'datastack-jwt-default-secret-key-32-chars-long',
maxEventsPerSec: 100, maxEventsPerSec: 100,
maxPayloadSizeBytes: 100 * 1024, maxPayloadSizeBytes: 100 * 1024,
batchFlushThreshold: 500, batchFlushThreshold: 500,
batchFlushIntervalMs: 2000, batchFlushIntervalMs: 2000,
fileRotationMb: 100, fileRotationMb: 100,
enableIdempotency: true, enableIdempotency: true,
autoPruneDays: 30,
}; };
function resolveInternalSecret(jwtSecretOverride) {
const explicit = (process.env.DATASTACK_INTERNAL_SECRET || '').trim();
if (IS_PRODUCTION) {
if (explicit.length >= 32) return explicit;
if (explicit.length > 0) return '';
} else if (explicit.length > 0) {
return explicit;
}
const activeJwt = String(
jwtSecretOverride !== undefined ? jwtSecretOverride : runtimeConfig.jwtSecret || '',
).trim();
const minJwtLen = IS_PRODUCTION ? 32 : 1;
if (activeJwt.length < minJwtLen) {
return '';
}
return crypto
.createHash('sha256')
.update(`datastack-internal:${activeJwt}`)
.digest('hex');
}
function timingSafeSecretEqual(candidate, expected) {
if (!candidate || !expected) return false;
const a = crypto.createHash('sha256').update(String(candidate)).digest();
const b = crypto.createHash('sha256').update(String(expected)).digest();
return crypto.timingSafeEqual(a, b);
}
function requireInternalAuth(req, res, next) {
const expected = resolveInternalSecret();
if (!expected) {
return res.status(503).json({
success: false,
error: 'INTERNAL_AUTH_UNAVAILABLE: Internal bridge secret is not configured',
code: 'INTERNAL_AUTH_UNAVAILABLE',
});
}
const presented = req.get('x-datastack-internal-secret');
if (!presented || !timingSafeSecretEqual(presented, expected)) {
return res.status(401).json({
success: false,
error: 'UNAUTHORIZED: Missing or invalid internal authentication header',
code: 'UNAUTHORIZED',
});
}
return next();
}
let dbPool = null; let dbPool = null;
if (process.env.DATABASE_URL) { if (process.env.DATABASE_URL) {
try { try {
@@ -40,43 +206,95 @@ if (process.env.DATABASE_URL) {
} }
} }
// Short-lived cache (2s TTL) for client status & token checks on high-frequency events // Short-lived cache (2s TTL) for DB client status & token checks on high-frequency events
const clientAuthCache = new Map(); const clientAuthCache = new Map();
const deletedClientsSet = new Set();
const inMemoryClientRegistry = new Map();
const inMemoryClientActivity = new Map();
let hasSyncedInMemoryClients = false;
const CLIENT_CACHE_TTL_MS = 2000; const CLIENT_CACHE_TTL_MS = 2000;
async function verifyClientInDb(clientId, presentedToken, bypassCache = false) { async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
if (!dbPool || !clientId) { if (!clientId) {
return { allowed: true }; return { allowed: true };
} }
const now = Date.now(); const normalizedId = String(clientId).trim().toLowerCase();
if (!bypassCache) { if (deletedClientsSet.has(normalizedId)) {
const cached = clientAuthCache.get(clientId); return {
if (cached && now - cached.checkedAt < CLIENT_CACHE_TTL_MS) { allowed: false,
if (cached.status === 'revoked') { reason: 'CLIENT_REVOKED: Client access has been deleted or revoked by admin',
};
}
if (!dbPool) {
const memClient = inMemoryClientRegistry.get(normalizedId);
if (memClient) {
if (memClient.status === 'revoked') {
return { return {
allowed: false, allowed: false,
reason: 'CLIENT_REVOKED: Client access has been revoked by admin', reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
}; };
} }
if (cached.token && presentedToken && cached.token !== presentedToken) { if (memClient.token && presentedToken && memClient.token !== presentedToken) {
return { return {
allowed: false, allowed: false,
reason: 'TOKEN_ROTATED: Client credentials have been rotated', reason: 'TOKEN_ROTATED: Client credentials have been rotated',
}; };
} }
return { allowed: true, rateLimit: cached.rateLimit }; return { allowed: true, rateLimit: memClient.rateLimit || 100 };
} }
if (hasSyncedInMemoryClients) {
return {
allowed: false,
reason: 'CLIENT_REVOKED: Client registration not found or has been deleted',
};
}
return { allowed: true };
}
const now = Date.now();
const prevCached = clientAuthCache.get(normalizedId);
if (!bypassCache && prevCached && now - prevCached.checkedAt < CLIENT_CACHE_TTL_MS) {
if (prevCached.status === 'revoked') {
return {
allowed: false,
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
};
}
if (prevCached.token && presentedToken && prevCached.token !== presentedToken) {
return {
allowed: false,
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
};
}
return { allowed: true, rateLimit: prevCached.rateLimit };
} }
try { try {
const deletedRes = await dbPool.query(
`SELECT 1 FROM deleted_clients WHERE client_id = $1`,
[normalizedId],
);
if (deletedRes.rowCount && deletedRes.rowCount > 0) {
clientAuthCache.delete(normalizedId);
deletedClientsSet.add(normalizedId);
return {
allowed: false,
reason: 'CLIENT_REVOKED: Client access has been deleted by admin',
};
}
const res = await dbPool.query( const res = await dbPool.query(
`SELECT status, token, rate_limit FROM client_sdks WHERE client_id = $1`, `SELECT status, token, rate_limit FROM client_sdks WHERE client_id = $1`,
[clientId], [normalizedId],
); );
if (!res.rowCount || res.rowCount === 0) { if (!res.rowCount || res.rowCount === 0) {
// Allow token-authenticated client if not explicitly revoked in table clientAuthCache.delete(normalizedId);
return { allowed: true }; return {
allowed: false,
reason: 'CLIENT_REVOKED: Client registration not found or has been deleted',
};
} }
const row = res.rows[0]; const row = res.rows[0];
const entry = { const entry = {
@@ -85,7 +303,7 @@ async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
rateLimit: Number(row.rate_limit) || 100, rateLimit: Number(row.rate_limit) || 100,
checkedAt: now, checkedAt: now,
}; };
clientAuthCache.set(clientId, entry); clientAuthCache.set(normalizedId, entry);
if (entry.status === 'revoked') { if (entry.status === 'revoked') {
return { return {
@@ -101,13 +319,30 @@ async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
} }
return { allowed: true, rateLimit: entry.rateLimit }; return { allowed: true, rateLimit: entry.rateLimit };
} catch { } catch {
return { allowed: true }; return {
allowed: false,
reason: 'AUTH_UNAVAILABLE: Unable to verify client credentials against database',
};
} }
} }
function recordClientActivity(clientId, count = 1) { function recordClientActivity(clientId, count = 1) {
if (!dbPool || !clientId || count <= 0) return; if (!clientId || count <= 0) return;
const normalizedId = String(clientId).trim().toLowerCase();
const nowIso = new Date().toISOString(); const nowIso = new Date().toISOString();
if (!dbPool) {
const prev = inMemoryClientActivity.get(normalizedId) || {
totalEvents: 0,
lastActive: nowIso,
};
inMemoryClientActivity.set(normalizedId, {
totalEvents: prev.totalEvents + count,
lastActive: nowIso,
});
return;
}
dbPool dbPool
.query( .query(
`UPDATE client_sdks `UPDATE client_sdks
@@ -115,7 +350,7 @@ function recordClientActivity(clientId, count = 1) {
last_active = $3, last_active = $3,
updated_at = NOW() updated_at = NOW()
WHERE client_id = $1`, WHERE client_id = $1`,
[clientId, count, nowIso], [normalizedId, count, nowIso],
) )
.catch(() => undefined); .catch(() => undefined);
} }
@@ -124,23 +359,103 @@ let isFlushing = false;
let flushTimer = null; let flushTimer = null;
async function flushBuffer() { async function flushBuffer() {
if (bufferQueue.length === 0 || isFlushing) { if (pendingRedisWrites.size > 0) {
await Promise.allSettled([...pendingRedisWrites]);
}
if (isFlushing) {
return { written: 0, files: [] }; return { written: 0, files: [] };
} }
isFlushing = true; isFlushing = true;
const batch = bufferQueue.splice(0, bufferQueue.length); let totalWritten = 0;
const filesSet = new Set();
try { try {
const result = await writer.writeBatch(batch); const redis = getRedisClient();
if (result.written > 0) { if (redis) {
const lockToken = crypto.randomUUID();
let lockAcquired = false;
try {
const lockResult = await redis.set(
REDIS_FLUSH_LOCK_KEY,
lockToken,
'PX',
FLUSH_LOCK_TTL_MS,
'NX',
);
lockAcquired = lockResult === 'OK';
redisConnected = true;
} catch {
redisConnected = false;
lockAcquired = false;
}
if (lockAcquired) {
try {
const chunkSize = Math.max(500, runtimeConfig.batchFlushThreshold);
while (true) {
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, chunkSize - 1);
if (!rawItems || rawItems.length === 0) {
break;
}
const parsedBatch = [];
for (const raw of rawItems) {
try {
const evt = JSON.parse(raw);
if (evt && evt.eventId) {
parsedBatch.push(evt);
}
} catch {
// Skip malformed entry while still trimming consumed range
}
}
if (parsedBatch.length > 0) {
const res = await writer.writeBatch(parsedBatch);
totalWritten += res.written;
for (const f of res.files) filesSet.add(f);
}
// Trim ONLY after CSV batch write succeeds
await redis.ltrim(REDIS_BUFFER_KEY, rawItems.length, -1);
if (rawItems.length < chunkSize) {
break;
}
}
} finally {
try {
await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
} catch {
// Lock expires automatically via PX TTL
}
}
}
}
if (bufferQueue.length > 0) {
const batch = bufferQueue.splice(0, bufferQueue.length);
try {
const res = await writer.writeBatch(batch);
totalWritten += res.written;
for (const f of res.files) filesSet.add(f);
} catch (err) {
bufferQueue.unshift(...batch);
throw err;
}
}
const files = Array.from(filesSet);
if (totalWritten > 0) {
console.log( console.log(
`[CSV Worker] Flushed ${result.written} events to ${result.files.length} file(s)`, `[CSV Worker] Flushed ${totalWritten} events to ${files.length} file(s)`,
); );
} }
return result; return { written: totalWritten, files };
} catch (err) { } catch (err) {
console.error('[CSV Worker] Flush error:', err); console.error('[CSV Worker] Flush error:', err);
bufferQueue.unshift(...batch); return { written: totalWritten, files: Array.from(filesSet) };
return { written: 0, files: [] };
} finally { } finally {
isFlushing = false; isFlushing = false;
} }
@@ -159,6 +474,92 @@ function scheduleFlushTimer(intervalMs) {
scheduleFlushTimer(runtimeConfig.batchFlushIntervalMs); scheduleFlushTimer(runtimeConfig.batchFlushIntervalMs);
const server = http.createServer(app);
const io = new Server(server, {
cors: { origin: '*', methods: ['GET', 'POST'] },
});
function applyRuntimeSettingsObject(settingsObj) {
if (!settingsObj || typeof settingsObj !== 'object') {
return { jwtRotated: false };
}
if (settingsObj.maxEventsPerSec !== undefined) {
const val = Number(settingsObj.maxEventsPerSec);
if (Number.isFinite(val) && val >= 1) {
runtimeConfig.maxEventsPerSec = Math.floor(val);
}
}
if (settingsObj.maxPayloadSizeKb !== undefined) {
const kb = Number(settingsObj.maxPayloadSizeKb);
if (Number.isFinite(kb) && kb >= 1) {
runtimeConfig.maxPayloadSizeBytes = Math.floor(kb) * 1024;
}
}
if (settingsObj.batchFlushThreshold !== undefined) {
const threshold = Number(settingsObj.batchFlushThreshold);
if (Number.isFinite(threshold) && threshold >= 1) {
runtimeConfig.batchFlushThreshold = Math.floor(threshold);
}
}
if (settingsObj.enableIdempotency !== undefined) {
runtimeConfig.enableIdempotency = Boolean(settingsObj.enableIdempotency);
}
if (settingsObj.fileRotationMb !== undefined) {
const nextRotationMb = Number(settingsObj.fileRotationMb);
if (
Number.isFinite(nextRotationMb) &&
nextRotationMb >= 1 &&
nextRotationMb !== runtimeConfig.fileRotationMb
) {
runtimeConfig.fileRotationMb = nextRotationMb;
writer.setMaxFileSizeMb(nextRotationMb);
}
}
if (settingsObj.autoPruneDays !== undefined && settingsObj.autoPruneDays !== null) {
const parsedPrune = Number(settingsObj.autoPruneDays);
if (Number.isFinite(parsedPrune) && parsedPrune >= 0) {
const nextPruneDays = Math.floor(parsedPrune);
const changed = nextPruneDays !== runtimeConfig.autoPruneDays;
runtimeConfig.autoPruneDays = nextPruneDays;
if (changed && nextPruneDays > 0) {
writer.pruneExpiredPartitions(nextPruneDays);
}
}
}
if (settingsObj.batchFlushIntervalMs !== undefined) {
const nextIntervalMs = Math.max(
100,
Number(settingsObj.batchFlushIntervalMs) || 2000,
);
if (nextIntervalMs !== runtimeConfig.batchFlushIntervalMs) {
scheduleFlushTimer(nextIntervalMs);
}
}
let jwtRotated = false;
if (settingsObj.jwtSecret !== undefined && settingsObj.jwtSecret !== null) {
const candidateSecret = String(settingsObj.jwtSecret).trim();
const validSecret =
candidateSecret.length >= 32 ||
(!IS_PRODUCTION && candidateSecret.length > 0);
if (validSecret && candidateSecret !== runtimeConfig.jwtSecret) {
runtimeConfig.jwtSecret = candidateSecret;
clientAuthCache.clear();
io.disconnectSockets(true);
jwtRotated = true;
}
}
return { jwtRotated };
}
async function ensureDatabaseSchema() { async function ensureDatabaseSchema() {
if (!dbPool) return; if (!dbPool) return;
try { try {
@@ -192,6 +593,12 @@ async function ensureDatabaseSchema() {
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
) )
`); `);
await dbPool.query(`
CREATE TABLE IF NOT EXISTS deleted_clients (
client_id TEXT PRIMARY KEY,
deleted_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`);
} catch { } catch {
// Ignore transient startup DB error; retried on next sync cycle // Ignore transient startup DB error; retried on next sync cycle
} }
@@ -210,69 +617,132 @@ async function syncSettingsFromDatabase() {
); );
if (!res.rowCount || res.rowCount === 0) return; if (!res.rowCount || res.rowCount === 0) return;
const row = res.rows[0]; const row = res.rows[0];
const parsedPrune =
row.auto_prune_days !== undefined && row.auto_prune_days !== null
? Number(row.auto_prune_days)
: NaN;
runtimeConfig.maxEventsPerSec = Number(row.max_events_per_sec) || 100; applyRuntimeSettingsObject({
runtimeConfig.maxPayloadSizeBytes = maxEventsPerSec: Number(row.max_events_per_sec) || 100,
(Number(row.max_payload_size_kb) || 100) * 1024; maxPayloadSizeKb: Number(row.max_payload_size_kb) || 100,
runtimeConfig.batchFlushThreshold = batchFlushThreshold: Math.max(1, Number(row.batch_flush_threshold) || 500),
Math.max(1, Number(row.batch_flush_threshold) || 500); batchFlushIntervalMs: Math.max(
runtimeConfig.enableIdempotency = Boolean(row.enable_idempotency); 100,
Number(row.batch_flush_interval_ms) || 2000,
if (row.jwt_secret && String(row.jwt_secret).trim().length > 0) { ),
runtimeConfig.jwtSecret = String(row.jwt_secret).trim(); fileRotationMb: Number(row.file_rotation_mb) || 100,
} enableIdempotency: Boolean(row.enable_idempotency),
autoPruneDays: Number.isFinite(parsedPrune) ? parsedPrune : 30,
const nextRotationMb = Number(row.file_rotation_mb) || 100; jwtSecret: row.jwt_secret ? String(row.jwt_secret).trim() : undefined,
if (nextRotationMb !== runtimeConfig.fileRotationMb) { });
runtimeConfig.fileRotationMb = nextRotationMb;
writer.setMaxFileSizeMb(nextRotationMb);
}
const nextPruneDays = Number(row.auto_prune_days) || 30;
if (nextPruneDays > 0) {
writer.pruneExpiredPartitions(nextPruneDays);
}
const nextIntervalMs = Math.max(
100,
Number(row.batch_flush_interval_ms) || 2000,
);
if (nextIntervalMs !== runtimeConfig.batchFlushIntervalMs) {
scheduleFlushTimer(nextIntervalMs);
}
} catch { } catch {
// Keep existing runtimeConfig if table is not ready yet // Keep existing runtimeConfig if table is not ready yet
} }
} }
const PRUNE_INTERVAL_MS = 60 * 60 * 1000;
let settingsSyncTimer = null; let settingsSyncTimer = null;
if (dbPool) { let pruneTimer = null;
void syncSettingsFromDatabase(); if (process.env.DATASTACK_DISABLE_LISTEN !== '1') {
settingsSyncTimer = setInterval(() => { if (dbPool) {
void syncSettingsFromDatabase(); void syncSettingsFromDatabase();
}, 3000); settingsSyncTimer = setInterval(() => {
void syncSettingsFromDatabase();
}, 3000);
}
pruneTimer = setInterval(() => {
if (runtimeConfig.autoPruneDays > 0) {
writer.pruneExpiredPartitions(runtimeConfig.autoPruneDays);
}
}, PRUNE_INTERVAL_MS);
} }
const server = http.createServer(app); const socketHandlersApi = setupSocketHandlers(io, bufferQueue, writer, {
const io = new Server(server, { getRuntimeConfig: () => runtimeConfig,
cors: { origin: '*', methods: ['GET', 'POST'] }, verifyClientInDb,
recordClientActivity,
onBufferPush: () => {
const redis = getRedisClient();
if (redis && bufferQueue.length > 0) {
const pushed = bufferQueue.splice(0, bufferQueue.length);
const op = pushEventsToRedisOrFallback(pushed).then((totalQueued) => {
if (totalQueued >= runtimeConfig.batchFlushThreshold) {
void flushBuffer();
}
});
pendingRedisWrites.add(op);
void op.finally(() => {
pendingRedisWrites.delete(op);
});
return;
}
if (bufferQueue.length >= runtimeConfig.batchFlushThreshold) {
void flushBuffer();
}
},
}); });
// Health check endpoint with activeSockets count and runtime capabilities async function getRedisBufferLen() {
app.get('/health', (_req, res) => { const redis = getRedisClient();
if (!redis) return 0;
try {
const len = Number(await redis.llen(REDIS_BUFFER_KEY)) || 0;
redisConnected = true;
return len;
} catch {
redisConnected = false;
return 0;
}
}
// Public health check endpoint
app.get('/health', async (_req, res) => {
const redisBufferSize = await getRedisBufferLen();
res.json({ res.json({
status: 'online', status: 'online',
service: 'DataStack Express Ingestion', service: 'DataStack Express Ingestion',
port: PORT, port: PORT,
activeSockets: io.engine ? io.engine.clientsCount : 0, activeSockets: io.engine ? io.engine.clientsCount : 0,
bufferSize: bufferQueue.length, bufferSize: bufferQueue.length + redisBufferSize,
localBufferSize: bufferQueue.length,
redisBufferSize,
redisConnected,
maxEventsPerSec: runtimeConfig.maxEventsPerSec, maxEventsPerSec: runtimeConfig.maxEventsPerSec,
maxPayloadSizeBytes: runtimeConfig.maxPayloadSizeBytes, maxPayloadSizeBytes: runtimeConfig.maxPayloadSizeBytes,
uptime: process.uptime(), uptime: process.uptime(),
}); });
}); });
// Internal flush endpoint triggered by Next.js POST /api/storage // Require internal shared authentication on all /internal/* bridge endpoints
app.use('/internal', requireInternalAuth);
// Internal metrics endpoint for Next.js dashboard & in-memory client activity aggregation
app.get('/internal/metrics', async (_req, res) => {
const redisBufferSize = await getRedisBufferLen();
res.json({
activeSockets: io.engine ? io.engine.clientsCount : 0,
bufferSize: bufferQueue.length + redisBufferSize,
localBufferSize: bufferQueue.length,
redisBufferSize,
redisConnected,
registrySize: inMemoryClientRegistry.size,
hasSyncedInMemoryClients,
securityStats: socketHandlersApi.getSecurityStats(),
clientActivity: Object.fromEntries(inMemoryClientActivity.entries()),
});
});
// Internal instant settings synchronization triggered by Next.js POST /api/settings
app.post('/internal/settings-sync', (req, res) => {
const result = applyRuntimeSettingsObject(req.body || {});
res.json({
success: true,
jwtRotated: result.jwtRotated,
});
});
// Internal flush endpoint triggered by Next.js POST /api/storage & GET /api/export
app.post('/internal/flush', async (_req, res) => { app.post('/internal/flush', async (_req, res) => {
try { try {
const result = await flushBuffer(); const result = await flushBuffer();
@@ -290,34 +760,149 @@ app.post('/internal/flush', async (_req, res) => {
}); });
// Internal endpoint to expose Port 4000 buffered events before CSV flush // Internal endpoint to expose Port 4000 buffered events before CSV flush
app.get('/internal/recent', (req, res) => { app.get('/internal/recent', async (req, res) => {
const clientId = req.query.clientId ? String(req.query.clientId) : ''; const clientId = req.query.clientId ? String(req.query.clientId) : '';
const limit = Math.min(200, Math.max(1, Number(req.query.limit) || 50)); const limit = Math.min(2000, Math.max(1, Number(req.query.limit) || 50));
const filtered =
const redis = getRedisClient();
let redisEvents = [];
if (redis) {
try {
const readCap = Math.min(REDIS_RECENT_LIMIT, Math.max(limit * 2, limit));
const rawItems = await redis.lrange(REDIS_RECENT_KEY, 0, readCap - 1);
redisConnected = true;
for (const raw of rawItems) {
try {
const evt = JSON.parse(raw);
if (!evt || !evt.eventId) continue;
if (clientId && clientId !== 'all' && evt.clientId !== clientId) continue;
redisEvents.push(evt);
if (redisEvents.length >= limit) break;
} catch {
// ignore malformed item
}
}
} catch {
redisConnected = false;
redisEvents = [];
}
}
const localFiltered =
clientId && clientId !== 'all' clientId && clientId !== 'all'
? bufferQueue.filter((evt) => evt.clientId === clientId) ? bufferQueue.filter((evt) => evt.clientId === clientId)
: bufferQueue; : bufferQueue;
const recent = filtered.slice(-limit).reverse(); const localRecent = localFiltered.slice(-limit).reverse();
res.json({ events: recent, count: recent.length });
const merged = [];
const seen = new Set();
for (const evt of [...localRecent, ...redisEvents]) {
if (!evt || !evt.eventId) continue;
const key = `${evt.clientId}:${evt.eventId}`;
if (seen.has(key)) continue;
seen.add(key);
merged.push(evt);
if (merged.length >= limit) break;
}
res.json({ events: merged, count: merged.length });
}); });
setupSocketHandlers(io, bufferQueue, writer, { // Internal endpoint triggered by Next.js SDKManager when a client is deleted, revoked, rotated, or synced
getRuntimeConfig: () => runtimeConfig, app.post('/internal/client-sync', (req, res) => {
verifyClientInDb, const action = req.body && req.body.action ? String(req.body.action) : '';
recordClientActivity,
onBufferPush: () => { if (action === 'bulk_sync' && Array.isArray(req.body.clients)) {
if (bufferQueue.length >= runtimeConfig.batchFlushThreshold) { hasSyncedInMemoryClients = true;
void flushBuffer(); inMemoryClientRegistry.clear();
clientAuthCache.clear();
const activeIds = new Set();
for (const item of req.body.clients) {
if (!item || !item.clientId) continue;
const cid = String(item.clientId).trim().toLowerCase();
activeIds.add(cid);
deletedClientsSet.delete(cid);
inMemoryClientRegistry.set(cid, {
clientId: cid,
name: String(item.name || cid),
status: item.status === 'revoked' ? 'revoked' : 'active',
token: String(item.token || ''),
rateLimit: Math.max(1, Number(item.rateLimit) || 100),
});
if (item.status === 'revoked') {
io.to(`client:${cid}`).disconnectSockets(true);
}
} }
}, return res.json({ success: true, count: activeIds.size });
}
const rawClientId = req.body && req.body.clientId ? String(req.body.clientId) : '';
const normalizedId = rawClientId.trim().toLowerCase();
const clientPayload =
req.body && req.body.client && typeof req.body.client === 'object'
? req.body.client
: null;
if (normalizedId) {
hasSyncedInMemoryClients = true;
clientAuthCache.delete(normalizedId);
if (action === 'deleted') {
deletedClientsSet.add(normalizedId);
inMemoryClientRegistry.delete(normalizedId);
inMemoryClientActivity.delete(normalizedId);
io.to(`client:${normalizedId}`).disconnectSockets(true);
} else if (action === 'revoked') {
const existingMem = inMemoryClientRegistry.get(normalizedId);
if (clientPayload) {
inMemoryClientRegistry.set(normalizedId, {
clientId: normalizedId,
name: String(clientPayload.name || normalizedId),
status: 'revoked',
token: String(clientPayload.token || ''),
rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
});
} else if (existingMem) {
existingMem.status = 'revoked';
} else {
inMemoryClientRegistry.set(normalizedId, {
clientId: normalizedId,
name: normalizedId,
status: 'revoked',
token: '',
rateLimit: 100,
});
}
io.to(`client:${normalizedId}`).disconnectSockets(true);
} else if (action === 'registered') {
deletedClientsSet.delete(normalizedId);
if (clientPayload) {
inMemoryClientRegistry.set(normalizedId, {
clientId: normalizedId,
name: String(clientPayload.name || normalizedId),
status: clientPayload.status === 'revoked' ? 'revoked' : 'active',
token: String(clientPayload.token || ''),
rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
});
}
} else if (action === 'updated') {
if (clientPayload) {
inMemoryClientRegistry.set(normalizedId, {
clientId: normalizedId,
name: String(clientPayload.name || normalizedId),
status: clientPayload.status === 'revoked' ? 'revoked' : 'active',
token: String(clientPayload.token || ''),
rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
});
}
io.to(`client:${normalizedId}`).disconnectSockets(true);
}
}
res.json({ success: true });
}); });
let isShuttingDown = false; function stopTimers() {
async function gracefulShutdown(signal) {
if (isShuttingDown) return;
isShuttingDown = true;
console.log(`[DataStack Ingestion Engine] Received ${signal}, flushing buffer before exit...`);
if (flushTimer) { if (flushTimer) {
clearInterval(flushTimer); clearInterval(flushTimer);
flushTimer = null; flushTimer = null;
@@ -326,6 +911,19 @@ async function gracefulShutdown(signal) {
clearInterval(settingsSyncTimer); clearInterval(settingsSyncTimer);
settingsSyncTimer = null; settingsSyncTimer = null;
} }
if (pruneTimer) {
clearInterval(pruneTimer);
pruneTimer = null;
}
}
let isShuttingDown = false;
async function gracefulShutdown(signal) {
if (isShuttingDown) return;
isShuttingDown = true;
console.log(`[DataStack Ingestion Engine] Received ${signal}, flushing buffer before exit...`);
stopTimers();
try { try {
await flushBuffer(); await flushBuffer();
@@ -336,6 +934,9 @@ async function gracefulShutdown(signal) {
try { try {
io.close(); io.close();
server.close(); server.close();
if (redisClient && typeof redisClient.quit === 'function') {
await redisClient.quit().catch(() => undefined);
}
if (dbPool) { if (dbPool) {
await dbPool.end(); await dbPool.end();
} }
@@ -346,17 +947,32 @@ async function gracefulShutdown(signal) {
process.exit(0); process.exit(0);
} }
process.on('SIGTERM', () => { if (process.env.DATASTACK_DISABLE_LISTEN !== '1') {
void gracefulShutdown('SIGTERM'); process.on('SIGTERM', () => {
}); void gracefulShutdown('SIGTERM');
process.on('SIGINT', () => { });
void gracefulShutdown('SIGINT'); process.on('SIGINT', () => {
}); void gracefulShutdown('SIGINT');
});
server.listen(PORT, () => { server.listen(PORT, () => {
console.log( console.log(
`[DataStack Ingestion Engine] Express + Socket.IO listening on port ${PORT}`, `[DataStack Ingestion Engine] Express + Socket.IO listening on port ${PORT}`,
); );
}); });
}
module.exports = { app, server, io, bufferQueue, flushBuffer }; module.exports = {
app,
server,
io,
writer,
bufferQueue,
flushBuffer,
runtimeConfig,
verifyClientInDb,
recordClientActivity,
applyRuntimeSettingsObject,
setRedisClientForTests,
stopTimers,
};
+48 -7
View File
@@ -11,8 +11,10 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
: () => ({ : () => ({
jwtSecret: jwtSecret:
options.jwtSecret || options.jwtSecret ||
process.env.DATASTACK_JWT_SECRET || (process.env.DATASTACK_JWT_SECRET || "").trim() ||
"datastack-jwt-default-secret-key-32-chars-long", (process.env.NODE_ENV === "production"
? ""
: "datastack-jwt-default-secret-key-32-chars-long"),
maxEventsPerSec: 100, maxEventsPerSec: 100,
maxPayloadSizeBytes: 100 * 1024, maxPayloadSizeBytes: 100 * 1024,
batchFlushThreshold: 500, batchFlushThreshold: 500,
@@ -38,8 +40,9 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
const rateLimitMap = new Map(); const rateLimitMap = new Map();
let lastCleanupAt = Date.now(); let lastCleanupAt = Date.now();
function cleanupMaps(now) { function cleanupMaps(now, force = false) {
if ( if (
!force &&
now - lastCleanupAt < CLEANUP_INTERVAL_MS && now - lastCleanupAt < CLEANUP_INTERVAL_MS &&
idempotencyMap.size <= IDEMPOTENCY_MAX_ENTRIES idempotencyMap.size <= IDEMPOTENCY_MAX_ENTRIES
) { ) {
@@ -60,6 +63,16 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
} }
} }
function getSecurityStats() {
cleanupMaps(Date.now(), true);
const config = getRuntimeConfig();
return {
activeIdempotencyKeys: idempotencyMap.size,
rateLimitedClientsCount: rateLimitMap.size,
maxPayloadSizeBytes: Number(config.maxPayloadSizeBytes) || 100 * 1024,
};
}
function validateSingleEvent(clientId, eventId, rawData, clientRateLimit) { function validateSingleEvent(clientId, eventId, rawData, clientRateLimit) {
const config = getRuntimeConfig(); const config = getRuntimeConfig();
const now = Date.now(); const now = Date.now();
@@ -136,10 +149,19 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
? String(authHeader).slice(7).trim() ? String(authHeader).slice(7).trim()
: String(authHeader).trim(); : String(authHeader).trim();
const config = getRuntimeConfig(); const config = getRuntimeConfig();
const minSecretLen = process.env.NODE_ENV === "production" ? 32 : 1;
if (!config.jwtSecret || config.jwtSecret.length < minSecretLen) {
return next(
new Error("AUTH_UNAVAILABLE: DATASTACK_JWT_SECRET is not configured"),
);
}
const decoded = jwt.verify(cleanToken, config.jwtSecret); const decoded = jwt.verify(cleanToken, config.jwtSecret);
const normalizedClientId = String(decoded.clientId || "")
.trim()
.toLowerCase();
const dbCheck = await verifyClientInDb( const dbCheck = await verifyClientInDb(
decoded.clientId, normalizedClientId,
cleanToken, cleanToken,
true, true,
); );
@@ -147,7 +169,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
return next(new Error(dbCheck.reason || "CLIENT_REVOKED")); return next(new Error(dbCheck.reason || "CLIENT_REVOKED"));
} }
socket.data.clientId = decoded.clientId; socket.data.clientId = normalizedClientId;
socket.data.token = cleanToken; socket.data.token = cleanToken;
socket.data.rateLimit = socket.data.rateLimit =
dbCheck.rateLimit || decoded.rateLimit || config.maxEventsPerSec || 100; dbCheck.rateLimit || decoded.rateLimit || config.maxEventsPerSec || 100;
@@ -157,6 +179,22 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
} }
}); });
const ENVELOPE_KEYS = new Set(["eventId", "timestamp", "data", "clientId"]);
function extractBatchItemData(safeItem) {
if (
safeItem &&
typeof safeItem === "object" &&
!Array.isArray(safeItem) &&
safeItem.data !== undefined
) {
const keys = Object.keys(safeItem);
if (keys.every((k) => ENVELOPE_KEYS.has(k))) {
return safeItem.data;
}
}
return safeItem;
}
io.on("connection", (socket) => { io.on("connection", (socket) => {
const clientId = socket.data.clientId; const clientId = socket.data.clientId;
socket.join(`client:${clientId}`); socket.join(`client:${clientId}`);
@@ -274,8 +312,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
safeItem.eventId || safeItem.eventId ||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`; `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
const timestamp = safeItem.timestamp || new Date().toISOString(); const timestamp = safeItem.timestamp || new Date().toISOString();
const rawData = const rawData = extractBatchItemData(safeItem);
safeItem.data !== undefined ? safeItem.data : safeItem;
const check = validateSingleEvent( const check = validateSingleEvent(
clientId, clientId,
@@ -403,6 +440,10 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
} }
}); });
}); });
return {
getSecurityStats,
};
} }
module.exports = { setupSocketHandlers }; module.exports = { setupSocketHandlers };
+123 -24
View File
@@ -2,6 +2,15 @@ const fs = require('fs');
const path = require('path'); const path = require('path');
const { Parser } = require('json2csv'); const { Parser } = require('json2csv');
const LOCK_RETRY_MIN_MS = 5;
const LOCK_RETRY_MAX_MS = 18;
const LOCK_TIMEOUT_MS = 10000;
const STALE_LOCK_AGE_MS = 30000;
function sleepMs(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
/** /**
* Background CSV Writer Worker * Background CSV Writer Worker
* Flushes batches of real-time events to partitioned CSV files. * Flushes batches of real-time events to partitioned CSV files.
@@ -27,14 +36,100 @@ class CsvWriterWorker {
} }
} }
getTargetFilePath(clientId, dateStr) { ensureClientDir(clientId) {
const safeClientId = String(clientId || 'unknown').replace(/[^a-zA-Z0-9_-]/g, '_'); const safeClientId = String(clientId || 'unknown').replace(/[^a-zA-Z0-9_-]/g, '_');
const clientDir = path.join(this.baseDir, safeClientId); const clientDir = path.join(this.baseDir, safeClientId);
if (!fs.existsSync(clientDir)) { if (!fs.existsSync(clientDir)) {
fs.mkdirSync(clientDir, { recursive: true }); fs.mkdirSync(clientDir, { recursive: true });
} }
return clientDir;
}
const date = dateStr || new Date().toISOString().split('T')[0]; async withPartitionLock(clientId, dateStr, fn) {
const clientDir = this.ensureClientDir(clientId);
const date = this.extractSafePartitionDate(dateStr);
const lockPath = path.join(clientDir, `.${date}.csv.lock`);
const deadline = Date.now() + LOCK_TIMEOUT_MS;
let acquired = false;
while (Date.now() < deadline) {
try {
const fd = fs.openSync(lockPath, 'wx');
try {
fs.writeFileSync(fd, `${process.pid}:${Date.now()}`, 'utf8');
} finally {
fs.closeSync(fd);
}
acquired = true;
break;
} catch (err) {
if (!err || err.code !== 'EEXIST') {
throw err;
}
try {
const lockStat = fs.statSync(lockPath);
if (Date.now() - lockStat.mtimeMs > STALE_LOCK_AGE_MS) {
fs.unlinkSync(lockPath);
continue;
}
} catch {
// Released concurrently
}
const jitter =
LOCK_RETRY_MIN_MS +
Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS));
await sleepMs(jitter);
}
}
if (!acquired) {
throw new Error(`Timed out waiting for CSV partition lock: ${lockPath}`);
}
try {
return await fn();
} finally {
try {
if (fs.existsSync(lockPath)) {
fs.unlinkSync(lockPath);
}
} catch {
// Ignore if already removed
}
}
}
extractSafePartitionDate(rawTimestamp) {
const fallback = new Date().toISOString().slice(0, 10);
if (!rawTimestamp || typeof rawTimestamp !== 'string') {
return fallback;
}
const trimmed = rawTimestamp.trim();
const prefix = trimmed.slice(0, 10);
if (/^\d{4}-\d{2}-\d{2}$/.test(prefix)) {
const parsedPrefix = new Date(`${prefix}T00:00:00.000Z`);
if (!Number.isNaN(parsedPrefix.getTime())) {
return prefix;
}
}
if (/^\d+(\.\d+)?$/.test(trimmed)) {
const num = Number(trimmed);
const ms = num < 1e11 ? num * 1000 : num;
const d = new Date(ms);
if (!Number.isNaN(d.getTime())) {
return d.toISOString().slice(0, 10);
}
}
const parsed = new Date(trimmed);
if (!Number.isNaN(parsed.getTime())) {
return parsed.toISOString().slice(0, 10);
}
return fallback;
}
getTargetFilePath(clientId, dateStr) {
const clientDir = this.ensureClientDir(clientId);
const date = this.extractSafePartitionDate(dateStr);
let baseFileName = `${date}.csv`; let baseFileName = `${date}.csv`;
let targetPath = path.join(clientDir, baseFileName); let targetPath = path.join(clientDir, baseFileName);
@@ -62,7 +157,7 @@ class CsvWriterWorker {
const grouped = new Map(); const grouped = new Map();
for (const evt of events) { for (const evt of events) {
const date = (evt.timestamp || new Date().toISOString()).split('T')[0]; const date = this.extractSafePartitionDate(evt.timestamp);
const key = `${evt.clientId}:::${date}`; const key = `${evt.clientId}:::${date}`;
if (!grouped.has(key)) grouped.set(key, []); if (!grouped.has(key)) grouped.set(key, []);
grouped.get(key).push(evt); grouped.get(key).push(evt);
@@ -73,30 +168,34 @@ class CsvWriterWorker {
for (const [key, batch] of grouped.entries()) { for (const [key, batch] of grouped.entries()) {
const [clientId, date] = key.split(':::'); const [clientId, date] = key.split(':::');
const targetPath = this.getTargetFilePath(clientId, date); const targetPath = await this.withPartitionLock(clientId, date, () => {
let hasContent = false; const resolvedPath = this.getTargetFilePath(clientId, date);
try { let hasContent = false;
hasContent = try {
fs.existsSync(targetPath) && fs.statSync(targetPath).size > 0; hasContent =
} catch { fs.existsSync(resolvedPath) && fs.statSync(resolvedPath).size > 0;
hasContent = false; } catch {
} hasContent = false;
}
const cleanBatch = batch.map((item) => ({ const cleanBatch = batch.map((item) => ({
clientId: item.clientId, clientId: item.clientId,
eventId: item.eventId, eventId: item.eventId,
timestamp: item.timestamp, timestamp: item.timestamp,
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload), payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
})); }));
const parser = new Parser({ fields, header: !hasContent, eol: '\n' }); const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
const csv = parser.parse(cleanBatch); const csv = parser.parse(cleanBatch);
if (hasContent) { if (hasContent) {
fs.appendFileSync(targetPath, '\n' + csv, 'utf8'); fs.appendFileSync(resolvedPath, '\n' + csv, 'utf8');
} else { } else {
fs.writeFileSync(targetPath, csv, 'utf8'); fs.writeFileSync(resolvedPath, csv, 'utf8');
} }
return resolvedPath;
});
files.push(targetPath); files.push(targetPath);
} }
+340
View File
@@ -0,0 +1,340 @@
import assert from "node:assert/strict";
import { after, afterEach, beforeEach, describe, it } from "node:test";
import { NextRequest } from "next/server";
import { GET as profileGet, PUT as profilePut } from "@/app/api/auth/profile/route";
import { GET as clientsGet, POST as clientsPost } from "@/app/api/clients/route";
import {
DELETE as clientDelete,
GET as clientGet,
} from "@/app/api/clients/[id]/route";
import { POST as eventsPost } from "@/app/api/events/route";
import {
createAdminSessionToken,
getAdminSessionPayload,
getAdminSessionSecret,
} from "@/core/auth/adminSession";
import {
consumeAdminLoginAttempt,
createInitialAdmin,
getAdminSetupKey,
isAdminSetupComplete,
resetAdminStateForTests,
verifyAdminCredentials,
verifyAdminSetupKey,
verifyRequestAdminSession,
} from "@/core/auth/adminUsers";
import { AuthService, resolveDefaultJwtSecret } from "@/core/auth/jwt";
import { isSameOriginRequest } from "@/middleware";
const ORIGINAL_ENV = { ...process.env };
function restoreEnv() {
for (const key of Object.keys(process.env)) {
if (!(key in ORIGINAL_ENV)) {
delete process.env[key];
}
}
Object.assign(process.env, ORIGINAL_ENV);
AuthService.setSecret(resolveDefaultJwtSecret() || "");
}
describe("Phase-01 Auth, Secrets, Revocation & Proxy Hardening", () => {
beforeEach(() => {
restoreEnv();
resetAdminStateForTests();
});
afterEach(() => {
restoreEnv();
resetAdminStateForTests();
});
after(() => {
const bufState = (
globalThis as {
__dataStackBufferState?: { flushTimer?: ReturnType<typeof setInterval> | null };
}
).__dataStackBufferState;
if (bufState?.flushTimer) {
clearInterval(bufState.flushTimer);
bufState.flushTimer = null;
}
});
it("BUG-01 & BUG-02: fails closed in production when secrets or DB/setup are missing", async () => {
process.env.NODE_ENV = "production";
delete process.env.DATABASE_URL;
delete process.env.DATASTACK_ADMIN_SESSION_SECRET;
delete process.env.DATASTACK_ADMIN_SETUP_KEY;
delete process.env.DATASTACK_JWT_SECRET;
delete process.env.DATASTACK_ADMIN_PASSWORD;
assert.equal(getAdminSessionSecret(), null);
assert.equal(getAdminSetupKey(), null);
assert.equal(resolveDefaultJwtSecret(), "");
assert.equal(
verifyAdminSetupKey("datastack-default-setup-key-32-chars"),
false,
);
assert.equal(await isAdminSetupComplete(), false);
await assert.rejects(
() => verifyAdminCredentials("admin@datastack.dev", "admin12345678"),
/setup/i,
);
AuthService.setSecret(resolveDefaultJwtSecret());
assert.equal(AuthService.isConfigured(), false);
assert.throws(() => AuthService.generateClientToken({ clientId: "c1" }));
});
it("BUG-03: bumps tokenVersion on credential update and rejects old session cookies", async () => {
delete process.env.DATABASE_URL;
process.env.DATASTACK_ALLOW_DEMO_ADMIN = "false";
const created = await createInitialAdmin("owner@datastack.dev", "InitialPass123!");
assert.equal(created, true);
assert.equal(await isAdminSetupComplete(), true);
const secret = getAdminSessionSecret()!;
const v1Token = await createAdminSessionToken(
"owner@datastack.dev",
secret,
Date.now() + 3600_000,
1,
);
const checkV1 = await verifyRequestAdminSession(v1Token);
assert.equal(checkV1.valid, true);
const updateReq = new NextRequest("http://localhost:3000/api/auth/profile", {
method: "PUT",
headers: {
"content-type": "application/json",
cookie: `datastack_admin_session=${v1Token}`,
},
body: JSON.stringify({
email: "owner2@datastack.dev",
currentPassword: "InitialPass123!",
newPassword: "UpdatedPass456!",
}),
});
const updateRes = await profilePut(updateReq);
assert.equal(updateRes.status, 200);
const setCookie = updateRes.cookies.get("datastack_admin_session")?.value;
assert.ok(setCookie);
const newPayload = await getAdminSessionPayload(setCookie!, secret);
assert.ok(newPayload);
assert.equal(newPayload!.tokenVersion, 2);
// Old v1 session must now return 401
const staleProfileReq = new NextRequest("http://localhost:3000/api/auth/profile", {
method: "GET",
headers: { cookie: `datastack_admin_session=${v1Token}` },
});
const staleRes = await profileGet(staleProfileReq);
assert.equal(staleRes.status, 401);
// New v2 session succeeds
const freshProfileReq = new NextRequest("http://localhost:3000/api/auth/profile", {
method: "GET",
headers: { cookie: `datastack_admin_session=${setCookie}` },
});
const freshRes = await profileGet(freshProfileReq);
assert.equal(freshRes.status, 200);
});
it("BUG-04: throttles by composite IP+email without locking out other admin IPs", async () => {
const secret = "test-throttle-secret-at-least-32-chars-long";
const targetEmail = "admin@datastack.dev";
for (let i = 0; i < 5; i++) {
const dec = await consumeAdminLoginAttempt(targetEmail, secret, "198.51.100.10");
assert.equal(dec.allowed, true);
}
// 6th attempt from attacker IP is blocked
const blocked = await consumeAdminLoginAttempt(targetEmail, secret, "198.51.100.10");
assert.equal(blocked.allowed, false);
assert.ok(blocked.retryAfterSeconds >= 1);
// Real admin from different IP is still allowed
const otherIpAllowed = await consumeAdminLoginAttempt(
targetEmail,
secret,
"203.0.113.55",
);
assert.equal(otherIpAllowed.allowed, true);
// Same IP spraying 25 distinct emails triggers IP bucket block on 26th
for (let i = 0; i < 25; i++) {
const spray = await consumeAdminLoginAttempt(
`spray${i}@example.com`,
secret,
"198.51.100.99",
);
assert.equal(spray.allowed, true);
}
const sprayBlocked = await consumeAdminLoginAttempt(
"spray25@example.com",
secret,
"198.51.100.99",
);
assert.equal(sprayBlocked.allowed, false);
});
it("BUG-05 & BUG-15: redacts tokens in client list, returns 400 on malformed JSON, and blocks deleted/unregistered Bearer tokens", async () => {
delete process.env.DATABASE_URL;
await createInitialAdmin("admin@datastack.dev", "AdminPass123!");
const secret = getAdminSessionSecret()!;
const adminCookie = await createAdminSessionToken(
"admin@datastack.dev",
secret,
Date.now() + 3600_000,
1,
);
const createReq = new NextRequest("http://localhost:3000/api/clients", {
method: "POST",
headers: {
"content-type": "application/json",
cookie: `datastack_admin_session=${adminCookie}`,
},
body: JSON.stringify({
clientId: "phase01_probe_client",
name: "Phase01 Probe",
rateLimit: 50,
}),
});
const createRes = await clientsPost(createReq);
assert.equal(createRes.status, 201);
const createdJson = await createRes.json();
const clientToken = createdJson.client.token as string;
assert.ok(clientToken.length > 20);
// List endpoint must redact full token & secret
const listReq = new NextRequest("http://localhost:3000/api/clients", {
method: "GET",
headers: { cookie: `datastack_admin_session=${adminCookie}` },
});
const listRes = await clientsGet(listReq);
assert.equal(listRes.status, 200);
const listJson = await listRes.json();
const listed = listJson.clients.find(
(c: { clientId: string }) => c.clientId === "phase01_probe_client",
);
assert.ok(listed);
assert.equal("token" in listed, false);
assert.equal("secret" in listed, false);
assert.equal(listed.tokenPreview, `${clientToken.slice(0, 12)}...`);
// Single client GET omits token by default, reveals on ?includeToken=true
const getDefaultRes = await clientGet(
new NextRequest("http://localhost:3000/api/clients/phase01_probe_client", {
headers: { cookie: `datastack_admin_session=${adminCookie}` },
}),
{ params: Promise.resolve({ id: "phase01_probe_client" }) },
);
const defaultClient = (await getDefaultRes.json()).client;
assert.equal("token" in defaultClient, false);
const getIncludeRes = await clientGet(
new NextRequest(
"http://localhost:3000/api/clients/phase01_probe_client?includeToken=true",
{ headers: { cookie: `datastack_admin_session=${adminCookie}` } },
),
{ params: Promise.resolve({ id: "phase01_probe_client" }) },
);
const includeClient = (await getIncludeRes.json()).client;
assert.equal(includeClient.token, clientToken);
// Malformed JSON on POST /api/events returns 400 (not 500)
const malformedRes = await eventsPost(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
authorization: `Bearer ${clientToken}`,
"content-type": "application/json",
},
body: "{bad-json",
}),
);
assert.equal(malformedRes.status, 400);
// Delete client -> subsequent Bearer ingestion fails closed with 403 CLIENT_REVOKED
const delRes = await clientDelete(
new NextRequest("http://localhost:3000/api/clients/phase01_probe_client", {
method: "DELETE",
headers: { cookie: `datastack_admin_session=${adminCookie}` },
}),
{ params: Promise.resolve({ id: "phase01_probe_client" }) },
);
assert.equal(delRes.status, 200);
const rejectedRes = await eventsPost(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
authorization: `Bearer ${clientToken}`,
"content-type": "application/json",
},
body: JSON.stringify({ eventId: "evt_after_delete", data: { ok: 1 } }),
}),
);
assert.equal(rejectedRes.status, 403);
// Unregistered signed Bearer token also rejected with 403 CLIENT_REVOKED
const ghostToken = AuthService.generateClientToken({
clientId: "unregistered_ghost_client",
});
const ghostRes = await eventsPost(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
authorization: `Bearer ${ghostToken}`,
"content-type": "application/json",
},
body: JSON.stringify({ eventId: "evt_ghost", data: { ok: 1 } }),
}),
);
assert.equal(ghostRes.status, 403);
});
it("BUG-26: validates Origin or Referer against Host / X-Forwarded-Host", () => {
const reqOriginOk = new NextRequest("http://localhost:3000/api/clients", {
method: "POST",
headers: {
origin: "https://console.example.com",
"x-forwarded-host": "console.example.com",
},
});
assert.equal(isSameOriginRequest(reqOriginOk), true);
const reqRefererOk = new NextRequest("http://localhost:3000/api/clients", {
method: "POST",
headers: {
referer: "https://console.example.com/vcon/clients",
host: "console.example.com",
},
});
assert.equal(isSameOriginRequest(reqRefererOk), true);
const reqCrossOrigin = new NextRequest("http://localhost:3000/api/clients", {
method: "POST",
headers: {
origin: "https://evil.example.com",
host: "console.example.com",
},
});
assert.equal(isSameOriginRequest(reqCrossOrigin), false);
const reqMissingBoth = new NextRequest("http://localhost:3000/api/clients", {
method: "POST",
headers: {
host: "console.example.com",
},
});
assert.equal(isSameOriginRequest(reqMissingBoth), false);
});
});
+375
View File
@@ -0,0 +1,375 @@
import assert from "node:assert/strict";
import { createRequire } from "node:module";
import type { AddressInfo } from "node:net";
import test, { after, before } from "node:test";
import { NextRequest } from "next/server";
import { io as ioClient, type Socket } from "socket.io-client";
import { GET as getSettingsRoute, POST as postSettingsRoute } from "../app/api/settings/route";
import { SettingsManager } from "../core/adminSettings/settingsManager";
import {
createAdminSessionToken,
getAdminSessionSecret,
} from "../core/auth/adminSession";
import {
createInitialAdmin,
resetAdminStateForTests,
} from "../core/auth/adminUsers";
import { ApiEngine } from "../core/apiEngine/apiService";
import { SDKManager } from "../core/sdk/sdkManager";
import {
SecurityEngine,
getInternalIngestionHeaders,
} from "../core/Security/security";
const require = createRequire(import.meta.url);
interface IngestionServerModule {
server: import("node:http").Server;
io: import("socket.io").Server;
bufferQueue: Array<{ clientId: string; eventId: string; timestamp: string; payload: string }>;
runtimeConfig: {
jwtSecret: string;
maxEventsPerSec: number;
maxPayloadSizeBytes: number;
batchFlushThreshold: number;
batchFlushIntervalMs: number;
fileRotationMb: number;
enableIdempotency: boolean;
autoPruneDays: number;
};
stopTimers: () => void;
}
const g = globalThis as typeof globalThis & {
__dataStackBufferState?: {
flushTimer: ReturnType<typeof setInterval> | null;
};
__dataStackAdminSettings?: unknown;
__dataStackClientsMap?: Map<string, unknown>;
__dataStackDeletedClients?: Set<string>;
__dataStackSyncedSocketEvents?: Map<string, number>;
};
let serverMod: IngestionServerModule;
let ingestionUrl = "";
const prevIngestionUrl = process.env.INGESTION_INTERNAL_URL;
const prevDisableListen = process.env.DATASTACK_DISABLE_LISTEN;
const prevDbUrl = process.env.DATABASE_URL;
const prevInternalSecret = process.env.DATASTACK_INTERNAL_SECRET;
const openSockets: Socket[] = [];
function connectSocket(token: string): Promise<Socket> {
return new Promise((resolve, reject) => {
const client = ioClient(ingestionUrl, {
auth: { token },
transports: ["websocket"],
reconnection: false,
timeout: 3000,
});
openSockets.push(client);
const onConnect = () => {
cleanup();
resolve(client);
};
const onError = (err: Error) => {
cleanup();
client.disconnect();
reject(err);
};
function cleanup() {
client.off("connect", onConnect);
client.off("connect_error", onError);
}
client.on("connect", onConnect);
client.on("connect_error", onError);
});
}
function emitWithAck<T = unknown>(
socket: Socket,
event: string,
payload: unknown,
): Promise<T> {
return new Promise((resolve, reject) => {
socket.timeout(3000).emit(event, payload, (err: unknown, res: T) => {
if (err) reject(err);
else resolve(res);
});
});
}
async function createAdminCookieHeader(): Promise<string> {
resetAdminStateForTests();
await createInitialAdmin("admin@datastack.dev", "InitialAdminPass123!");
const secret = getAdminSessionSecret();
assert.ok(secret, "Expected admin session secret in test environment");
const token = await createAdminSessionToken(
"admin@datastack.dev",
secret,
Date.now() + 3600_000,
1,
);
return `datastack_admin_session=${token}`;
}
before(async () => {
process.env.DATASTACK_DISABLE_LISTEN = "1";
delete process.env.DATABASE_URL;
delete process.env.DATASTACK_INTERNAL_SECRET;
delete g.__dataStackAdminSettings;
g.__dataStackClientsMap?.clear();
g.__dataStackDeletedClients?.clear();
g.__dataStackSyncedSocketEvents?.clear();
resetAdminStateForTests();
serverMod = require("../server/index.js") as IngestionServerModule;
await new Promise<void>((resolve) => {
serverMod.server.listen(0, "127.0.0.1", () => resolve());
});
const addr = serverMod.server.address() as AddressInfo;
ingestionUrl = `http://127.0.0.1:${addr.port}`;
process.env.INGESTION_INTERNAL_URL = ingestionUrl;
});
after(async () => {
for (const sock of openSockets) {
try {
sock.disconnect();
} catch {
// ignore
}
}
if (serverMod) {
serverMod.stopTimers();
await new Promise<void>((resolve) => {
serverMod.io.close(() => {
serverMod.server.close(() => resolve());
});
});
}
if (g.__dataStackBufferState?.flushTimer) {
clearInterval(g.__dataStackBufferState.flushTimer);
g.__dataStackBufferState.flushTimer = null;
}
if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL;
else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl;
if (prevDisableListen === undefined) delete process.env.DATASTACK_DISABLE_LISTEN;
else process.env.DATASTACK_DISABLE_LISTEN = prevDisableListen;
if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
else process.env.DATABASE_URL = prevDbUrl;
if (prevInternalSecret === undefined) delete process.env.DATASTACK_INTERNAL_SECRET;
else process.env.DATASTACK_INTERNAL_SECRET = prevInternalSecret;
});
test("BUG-13: Port 4000 /internal/* rejects unauthenticated callers and accepts valid internal secret while /health remains public", async () => {
const healthRes = await fetch(`${ingestionUrl}/health`);
assert.equal(healthRes.status, 200);
for (const [method, path, body] of [
["GET", "/internal/metrics", undefined],
["POST", "/internal/flush", {}],
["GET", "/internal/recent", undefined],
["POST", "/internal/client-sync", { clientId: "x", action: "registered" }],
["POST", "/internal/settings-sync", { maxEventsPerSec: 120 }],
] as const) {
const noAuth = await fetch(`${ingestionUrl}${path}`, {
method,
headers: body ? { "Content-Type": "application/json" } : undefined,
body: body ? JSON.stringify(body) : undefined,
});
assert.equal(noAuth.status, 401, `${method} ${path} must reject missing secret`);
const wrongAuth = await fetch(`${ingestionUrl}${path}`, {
method,
headers: {
...(body ? { "Content-Type": "application/json" } : {}),
"x-datastack-internal-secret": "wrong-secret-value",
},
body: body ? JSON.stringify(body) : undefined,
});
assert.equal(wrongAuth.status, 401, `${method} ${path} must reject wrong secret`);
const validAuth = await fetch(`${ingestionUrl}${path}`, {
method,
headers: {
...(body ? { "Content-Type": "application/json" } : {}),
...getInternalIngestionHeaders(),
},
body: body ? JSON.stringify(body) : undefined,
});
assert.equal(validAuth.status, 200, `${method} ${path} must accept valid internal secret`);
}
});
test("BUG-06: POST /api/settings validates allowlisted bounds, syncs Port 4000 immediately, preserves autoPruneDays=0, and disconnects sockets on JWT rotation", async () => {
const unauthGet = await getSettingsRoute(new NextRequest("http://localhost:3000/api/settings"));
assert.equal(unauthGet.status, 401);
const cookieHeader = await createAdminCookieHeader();
const badJsonReq = new NextRequest("http://localhost:3000/api/settings", {
method: "POST",
headers: { "Content-Type": "application/json", cookie: cookieHeader },
body: "{bad-json",
});
const badJsonRes = await postSettingsRoute(badJsonReq);
assert.equal(badJsonRes.status, 400);
const unsupportedReq = new NextRequest("http://localhost:3000/api/settings", {
method: "POST",
headers: { "Content-Type": "application/json", cookie: cookieHeader },
body: JSON.stringify({ serverPort: 9999 }),
});
assert.equal((await postSettingsRoute(unsupportedReq)).status, 400);
const weakSecretReq = new NextRequest("http://localhost:3000/api/settings", {
method: "POST",
headers: { "Content-Type": "application/json", cookie: cookieHeader },
body: JSON.stringify({ jwtSecret: "too-short" }),
});
assert.equal((await postSettingsRoute(weakSecretReq)).status, 400);
const validUpdateReq = new NextRequest("http://localhost:3000/api/settings", {
method: "POST",
headers: { "Content-Type": "application/json", cookie: cookieHeader },
body: JSON.stringify({
maxEventsPerSec: 42,
autoPruneDays: 0,
enableIdempotency: true,
}),
});
const validUpdateRes = await postSettingsRoute(validUpdateReq);
assert.equal(validUpdateRes.status, 200);
const updatedJson = await validUpdateRes.json();
assert.equal(updatedJson.settings.maxEventsPerSec, 42);
assert.equal(updatedJson.settings.autoPruneDays, 0);
assert.equal(serverMod.runtimeConfig.maxEventsPerSec, 42);
assert.equal(serverMod.runtimeConfig.autoPruneDays, 0);
// Register client & open WebSocket before JWT rotation
const client = await SDKManager.registerClientAsync({
clientId: "phase02_jwt_rot",
name: "Phase02 Rotation",
rateLimit: 50,
});
const oldToken = client.token;
const socket = await connectSocket(oldToken);
assert.equal(socket.connected, true);
const disconnectedPromise = new Promise<string>((resolve) => {
socket.once("disconnect", (reason) => resolve(reason));
});
const newSecret = "phase02-rotated-production-secret-key-minimum-32-chars";
const rotateReq = new NextRequest("http://localhost:3000/api/settings", {
method: "POST",
headers: { "Content-Type": "application/json", cookie: cookieHeader },
body: JSON.stringify({ jwtSecret: newSecret }),
});
const rotateRes = await postSettingsRoute(rotateReq);
assert.equal(rotateRes.status, 200);
assert.equal(serverMod.runtimeConfig.jwtSecret, newSecret);
const disconnectReason = await Promise.race([
disconnectedPromise,
new Promise<string>((_, reject) =>
setTimeout(() => reject(new Error("Expected socket disconnect on JWT rotation")), 2000),
),
]);
assert.ok(disconnectReason.length > 0);
// Old JWT token must fail handshake after rotation
await assert.rejects(() => connectSocket(oldToken), /AUTH_INVALID|jwt|signature/i);
// New resigned client token must succeed cleanly
const refreshedClient = SDKManager.getClient("phase02_jwt_rot");
assert.ok(refreshedClient && refreshedClient.token !== oldToken);
const reconnected = await connectSocket(refreshedClient.token);
assert.equal(reconnected.connected, true);
reconnected.disconnect();
});
test("BUG-07: In-memory client registry, socket event counting, token rotation & revocation enforce cleanly on Port 4000", async () => {
const client = await SDKManager.registerClientAsync({
clientId: "phase02_mem_client",
name: "Phase02 Memory Sync",
rateLimit: 25,
});
const socket = await connectSocket(client.token);
const ack = await emitWithAck<{ success: boolean; eventId: string }>(
socket,
"sync_data",
{ eventId: "evt-mem-1", data: { ok: true } },
);
assert.equal(ack.success, true);
const listFirst = await SDKManager.listClientsAsync();
const syncedFirst = listFirst.find((c) => c.clientId === "phase02_mem_client");
assert.ok(syncedFirst);
assert.equal(syncedFirst.totalEvents, 1);
assert.ok(syncedFirst.lastActive);
// Polling again must NOT double-count socket activity
const listSecond = await SDKManager.listClientsAsync();
const syncedSecond = listSecond.find((c) => c.clientId === "phase02_mem_client");
assert.equal(syncedSecond?.totalEvents, 1);
// Advance across JWT iat 1-second boundary before regenerating client token
await new Promise((r) => setTimeout(r, 1100));
// Regenerating client token must kick active socket and reject old token
const rotatedClient = await SDKManager.regenerateTokenAsync("phase02_mem_client");
assert.ok(rotatedClient && rotatedClient.token !== client.token);
await new Promise((r) => setTimeout(r, 50));
assert.equal(socket.connected, false);
await assert.rejects(() => connectSocket(client.token), /TOKEN_ROTATED/);
// New token works; revoking kicks and rejects
const socket2 = await connectSocket(rotatedClient.token);
assert.equal(socket2.connected, true);
await SDKManager.updateClientAsync("phase02_mem_client", { status: "revoked" });
await new Promise((r) => setTimeout(r, 50));
assert.equal(socket2.connected, false);
await assert.rejects(() => connectSocket(rotatedClient.token), /CLIENT_REVOKED/);
});
test("BUG-08: Rate-limit error reports effectiveRateLimit and ApiEngine aggregates Port 3000 + Port 4000 security metrics", async () => {
await SettingsManager.updateSettingsAsync({ maxEventsPerSec: 3, enableIdempotency: true });
const rlClientId = "phase02_rl_msg";
let lastError = "";
for (let i = 0; i < 5; i += 1) {
const res = SecurityEngine.validateEvent(rlClientId, `rl-evt-${i}`, { i }, 100);
if (!res.allowed && res.code === "RATE_LIMIT_EXCEEDED") {
lastError = res.reason || "";
break;
}
}
assert.match(lastError, /Rate limit of 3 events\/sec exceeded/);
const client = await SDKManager.registerClientAsync({
clientId: "phase02_stats_client",
name: "Phase02 Stats Client",
rateLimit: 50,
});
await new Promise((r) => setTimeout(r, 80));
const socket = await connectSocket(client.token);
await emitWithAck(socket, "sync_data", { eventId: "evt-stats-ws-1", data: { a: 1 } });
socket.disconnect();
const metrics = await ApiEngine.getDashboardMetrics();
assert.ok(
metrics.securityStats.activeIdempotencyKeys >= 2,
`Expected aggregated idempotency keys >= 2 (HTTP + WS), got ${metrics.securityStats.activeIdempotencyKeys}`,
);
assert.ok(
metrics.securityStats.rateLimitedClientsCount >= 2,
`Expected aggregated rateLimitedClientsCount >= 2, got ${metrics.securityStats.rateLimitedClientsCount}`,
);
});
+511
View File
@@ -0,0 +1,511 @@
import assert from "node:assert/strict";
import fs from "node:fs";
import { createServer, type Server } from "node:http";
import type { AddressInfo } from "node:net";
import { createRequire } from "node:module";
import os from "node:os";
import path from "node:path";
import test, { after, before, beforeEach } from "node:test";
import { NextRequest } from "next/server";
import { GET as getDataRoute } from "../app/api/data/route";
import { GET as getEventsRoute } from "../app/api/events/route";
import { GET as getExportRoute } from "../app/api/export/route";
import { POST as postStorageRoute } from "../app/api/storage/route";
import { ApiEngine } from "../core/apiEngine/apiService";
import {
DataBufferEngine,
REDIS_BUFFER_KEY,
REDIS_FLUSH_LOCK_KEY,
type RedisBufferClient,
} from "../core/data/buffer";
import { SDKManager } from "../core/sdk/sdkManager";
import { CSVStorageManager, type DataEvent } from "../core/Storage/csvStorage";
const require = createRequire(import.meta.url);
const { CsvWriterWorker } = require("../server/workers/csvWriter") as {
CsvWriterWorker: new (baseDir?: string, maxFileSize?: number) => {
baseDir: string;
writeBatch: (events: DataEvent[]) => Promise<{ written: number; files: string[] }>;
pruneExpiredPartitions: (days: number) => number;
};
};
interface IngestionServerModule {
server: Server;
io: import("socket.io").Server;
writer: { baseDir: string };
bufferQueue: DataEvent[];
runtimeConfig: {
batchFlushThreshold: number;
autoPruneDays: number;
};
applyRuntimeSettingsObject: (obj: Record<string, unknown>) => { jwtRotated: boolean };
flushBuffer: () => Promise<{ written: number; files: string[] }>;
setRedisClientForTests: (client: RedisBufferClient | null | undefined) => void;
stopTimers: () => void;
}
class FakeRedisClient implements RedisBufferClient {
public status = "ready";
public lists = new Map<string, string[]>();
public strings = new Map<string, { value: string; expiresAt: number }>();
public async rpush(key: string, ...values: string[]): Promise<number> {
const list = this.lists.get(key) || [];
list.push(...values);
this.lists.set(key, list);
return list.length;
}
public async lpush(key: string, ...values: string[]): Promise<number> {
const list = this.lists.get(key) || [];
for (const v of values) {
list.unshift(v);
}
this.lists.set(key, list);
return list.length;
}
public async ltrim(key: string, start: number, stop: number): Promise<"OK"> {
const list = this.lists.get(key) || [];
const len = list.length;
const normStart = start < 0 ? Math.max(0, len + start) : start;
const normStop = stop < 0 ? len + stop : stop;
if (normStart > normStop || normStart >= len) {
this.lists.set(key, []);
} else {
this.lists.set(key, list.slice(normStart, normStop + 1));
}
return "OK";
}
public async lrange(key: string, start: number, stop: number): Promise<string[]> {
const list = this.lists.get(key) || [];
const len = list.length;
if (len === 0) return [];
const normStart = start < 0 ? Math.max(0, len + start) : start;
const normStop = stop < 0 ? len + stop : stop;
if (normStart > normStop || normStart >= len) return [];
return list.slice(normStart, normStop + 1);
}
public async llen(key: string): Promise<number> {
return (this.lists.get(key) || []).length;
}
public async set(
key: string,
value: string,
_pxMode: "PX",
ttlMs: number,
_nxMode: "NX",
): Promise<string | null> {
const existing = this.strings.get(key);
if (existing && existing.expiresAt > Date.now()) {
return null;
}
this.strings.set(key, { value, expiresAt: Date.now() + ttlMs });
return "OK";
}
public async eval(_script: string, _numKeys: number, key: string, token: string): Promise<number> {
const existing = this.strings.get(key);
if (existing && existing.value === token) {
this.strings.delete(key);
return 1;
}
return 0;
}
}
const g = globalThis as typeof globalThis & {
__dataStackBufferState?: {
buffer: DataEvent[];
recentEvents: DataEvent[];
flushTimer: ReturnType<typeof setInterval> | null;
};
__dataStackAdminSettings?: unknown;
__dataStackClientsMap?: Map<string, unknown>;
__dataStackDeletedClients?: Set<string>;
};
let tmpStorageDir = "";
const origBaseStorageDir = CSVStorageManager.baseStorageDir;
const prevDisableListen = process.env.DATASTACK_DISABLE_LISTEN;
const prevIngestionUrl = process.env.INGESTION_INTERNAL_URL;
const prevDbUrl = process.env.DATABASE_URL;
const prevRedisUrl = process.env.REDIS_URL;
before(async () => {
process.env.DATASTACK_DISABLE_LISTEN = "1";
delete process.env.DATABASE_URL;
delete process.env.REDIS_URL;
delete process.env.INGESTION_INTERNAL_URL;
tmpStorageDir = fs.mkdtempSync(path.join(os.tmpdir(), "datastack-phase03-"));
CSVStorageManager.baseStorageDir = tmpStorageDir;
CSVStorageManager.clearRowCountCache();
});
beforeEach(async () => {
DataBufferEngine.setRedisClientForTests(undefined);
if (g.__dataStackBufferState) {
g.__dataStackBufferState.buffer = [];
g.__dataStackBufferState.recentEvents = [];
}
CSVStorageManager.clearRowCountCache();
fs.rmSync(tmpStorageDir, { recursive: true, force: true });
fs.mkdirSync(tmpStorageDir, { recursive: true });
delete g.__dataStackAdminSettings;
g.__dataStackClientsMap?.clear();
g.__dataStackDeletedClients?.clear();
});
after(async () => {
await DataBufferEngine.shutdown();
CSVStorageManager.baseStorageDir = origBaseStorageDir;
CSVStorageManager.clearRowCountCache();
if (tmpStorageDir) {
fs.rmSync(tmpStorageDir, { recursive: true, force: true });
}
if (prevDisableListen === undefined) delete process.env.DATASTACK_DISABLE_LISTEN;
else process.env.DATASTACK_DISABLE_LISTEN = prevDisableListen;
if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL;
else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl;
if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
else process.env.DATABASE_URL = prevDbUrl;
if (prevRedisUrl === undefined) delete process.env.REDIS_URL;
else process.env.REDIS_URL = prevRedisUrl;
});
test("BUG-11: Shared Redis buffer preserves events on CSV error, honors distributed flush lock, avoids double-counting, and falls back to RAM cleanly", async () => {
const fakeRedis = new FakeRedisClient();
DataBufferEngine.setRedisClientForTests(fakeRedis);
const sampleEvents: DataEvent[] = [
{
clientId: "phase03_redis",
eventId: "r-evt-1",
timestamp: "2026-11-17T10:00:00.000Z",
payload: JSON.stringify({ seq: 1 }),
},
{
clientId: "phase03_redis",
eventId: "r-evt-2",
timestamp: "2026-11-17T10:00:01.000Z",
payload: JSON.stringify({ seq: 2 }),
},
];
await DataBufferEngine.pushBatchAsync(sampleEvents);
assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 2);
const recentFromRedis = await DataBufferEngine.getRecentEventsAsync(10, "phase03_redis");
assert.equal(recentFromRedis.length, 2);
assert.equal(recentFromRedis[0].eventId, "r-evt-2");
// Lock contention must skip concurrent Redis flush without corrupting queue
await fakeRedis.set(REDIS_FLUSH_LOCK_KEY, "other-worker-token", "PX", 15000, "NX");
const skipped = await DataBufferEngine.flush();
assert.equal(skipped.count, 0);
assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 2);
await fakeRedis.eval("", 1, REDIS_FLUSH_LOCK_KEY, "other-worker-token");
// Crash/error safety: if appendBatch throws before finishing, Redis LTRIM must NOT discard events
const originalAppend = CSVStorageManager.appendBatch;
CSVStorageManager.appendBatch = async () => {
throw new Error("Simulated disk write failure");
};
await assert.rejects(() => DataBufferEngine.flush(), /Simulated disk write failure/);
CSVStorageManager.appendBatch = originalAppend;
assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 2);
// Subsequent healthy flush drains Redis list, releases distributed lock, and writes CSV
const flushed = await DataBufferEngine.flush();
assert.equal(flushed.count, 2);
assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 0);
assert.equal(fakeRedis.strings.has(REDIS_FLUSH_LOCK_KEY), false);
// Verify Port 4000 /internal/recent & /internal/metrics read shared Redis without double-counting
const serverMod = require("../server/index.js") as IngestionServerModule;
const origWriterDir = serverMod.writer.baseDir;
serverMod.writer.baseDir = tmpStorageDir;
serverMod.setRedisClientForTests(fakeRedis);
await new Promise<void>((resolve) => serverMod.server.listen(0, "127.0.0.1", () => resolve()));
const addr = serverMod.server.address() as AddressInfo;
process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${addr.port}`;
try {
await DataBufferEngine.pushAsync({
clientId: "phase03_redis",
eventId: "r-evt-3",
timestamp: "2026-11-17T10:00:02.000Z",
payload: JSON.stringify({ seq: 3 }),
});
const stats = await DataBufferEngine.getBufferStatsAsync();
assert.equal(stats.redisEnabled, true);
assert.equal(stats.redisBufferSize, 1);
const metrics = await ApiEngine.getDashboardMetrics();
// Shared Redis count (1) must NOT be counted twice as 2 across Port 3000 + Port 4000
assert.equal(metrics.bufferStats.currentBufferSize, 1);
// Clear recent list & buffer via Port 4000 flush
const flushRes = await serverMod.flushBuffer();
assert.equal(flushRes.written, 1);
assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 0);
} finally {
serverMod.writer.baseDir = origWriterDir;
serverMod.setRedisClientForTests(undefined);
serverMod.stopTimers();
delete process.env.INGESTION_INTERNAL_URL;
await new Promise<void>((resolve) => serverMod.io.close(() => serverMod.server.close(() => resolve())));
}
// Fallback to local RAM buffer when Redis is disabled (null)
DataBufferEngine.setRedisClientForTests(null);
await DataBufferEngine.pushAsync({
clientId: "phase03_ram",
eventId: "ram-evt-1",
timestamp: "2026-11-17T10:01:00.000Z",
payload: JSON.stringify({ mode: "ram" }),
});
const ramStats = await DataBufferEngine.getBufferStatsAsync();
assert.equal(ramStats.redisEnabled, false);
assert.equal(ramStats.localBufferSize, 1);
const ramFlushed = await DataBufferEngine.flush();
assert.equal(ramFlushed.count, 1);
});
test("BUG-09: Cooperative CSV partition locking serializes concurrent Port 3000 + Port 4000 writers, recovers stale locks, and surfaces Port 4000 flush errors", async () => {
const worker = new CsvWriterWorker(tmpStorageDir);
const partitionDate = "2026-11-17";
const clientId = "phase03_concurrent";
const batchA: DataEvent[] = Array.from({ length: 15 }, (_, idx) => ({
clientId,
eventId: `a-${idx}`,
timestamp: `${partitionDate}T12:00:00.000Z`,
payload: JSON.stringify({ writer: "A", idx }),
}));
const batchB: DataEvent[] = Array.from({ length: 15 }, (_, idx) => ({
clientId,
eventId: `b-${idx}`,
timestamp: `${partitionDate}T12:00:01.000Z`,
payload: JSON.stringify({ writer: "B", idx }),
}));
await Promise.all([
CSVStorageManager.appendBatch(batchA),
worker.writeBatch(batchB),
]);
const csvPath = path.join(tmpStorageDir, clientId, `${partitionDate}.csv`);
const lockPath = path.join(tmpStorageDir, clientId, `.${partitionDate}.csv.lock`);
assert.equal(fs.existsSync(lockPath), false, "Lock file must be cleaned up after writes");
const lines = fs.readFileSync(csvPath, "utf8").trim().split("\n");
assert.equal(lines[0], '"clientId","eventId","timestamp","payload"');
assert.equal(lines.length, 31, "Must contain exactly 1 header + 30 complete rows without interleaving");
// Stale lock recovery (>30s old mtime)
fs.writeFileSync(lockPath, "999999:0", "utf8");
const pastTime = new Date(Date.now() - 45_000);
fs.utimesSync(lockPath, pastTime, pastTime);
await CSVStorageManager.appendBatch([
{
clientId,
eventId: "after-stale-lock",
timestamp: `${partitionDate}T12:00:02.000Z`,
payload: JSON.stringify({ staleRecovered: true }),
},
]);
assert.equal(fs.existsSync(lockPath), false);
const files = await CSVStorageManager.listAllFilesAsync();
assert.equal(files[0].linesCount, 31);
// Explicit flush error & warning propagation in POST /api/storage
const failingBridge = createServer((_req, res) => {
res.statusCode = 500;
res.setHeader("Content-Type", "application/json");
res.end(JSON.stringify({ success: false, error: "Disk permission denied on Port 4000" }));
});
await new Promise<void>((resolve) => failingBridge.listen(0, "127.0.0.1", () => resolve()));
const bridgeAddr = failingBridge.address() as AddressInfo;
process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${bridgeAddr.port}`;
try {
const errRes = await postStorageRoute(new NextRequest("http://localhost:3000/api/storage", { method: "POST" }));
assert.equal(errRes.status, 502);
const errJson = await errRes.json();
assert.equal(errJson.success, false);
assert.equal(errJson.code, "INGESTION_FLUSH_FAILED");
assert.equal(errJson.ingestionFlushed, false);
} finally {
delete process.env.INGESTION_INTERNAL_URL;
await new Promise<void>((resolve) => failingBridge.close(() => resolve()));
}
// When Port 4000 is down and INGESTION_INTERNAL_URL is unset, standalone dev surfaces explicit warning + ingestionFlushed: false
const standaloneRes = await postStorageRoute(new NextRequest("http://localhost:3000/api/storage", { method: "POST" }));
assert.equal(standaloneRes.status, 200);
const standaloneJson = await standaloneRes.json();
assert.equal(standaloneJson.ingestionFlushed, false);
assert.match(String(standaloneJson.warning || ""), /unreachable/i);
});
test("BUG-10: Cached streaming row counter counts multiline RFC 4180 payloads accurately and avoids repeated disk reads", async () => {
await CSVStorageManager.appendBatch([
{
clientId: "phase03_perf",
eventId: "multiline-1",
timestamp: "2026-11-17T08:00:00.000Z",
payload: JSON.stringify({ note: "line1\nline2\n\"quoted\"\nline4" }),
},
{
clientId: "phase03_perf",
eventId: "multiline-2",
timestamp: "2026-11-17T08:00:01.000Z",
payload: JSON.stringify({ note: "single" }),
},
]);
const firstStats = await CSVStorageManager.getStorageStatsAsync();
assert.equal(firstStats.totalRows, 2);
// Spy on fs.createReadStream to verify repeated dashboard polls hit mtime+size cache with 0 file scans
const origCreateReadStream = fs.createReadStream;
let streamReads = 0;
(fs as unknown as { createReadStream: typeof fs.createReadStream }).createReadStream = ((
...args: Parameters<typeof fs.createReadStream>
) => {
streamReads += 1;
return origCreateReadStream(...args);
}) as typeof fs.createReadStream;
try {
const secondStats = await CSVStorageManager.getStorageStatsAsync();
assert.equal(secondStats.totalRows, 2);
assert.equal(streamReads, 0, "Unchanged CSV files must use cached row count");
} finally {
(fs as unknown as { createReadStream: typeof fs.createReadStream }).createReadStream =
origCreateReadStream;
}
// Server 3s settings sync must not repeatedly prune when autoPruneDays is unchanged
const serverMod = require("../server/index.js") as IngestionServerModule;
const oldPartitionDir = path.join(tmpStorageDir, "phase03_prune_check");
fs.mkdirSync(oldPartitionDir, { recursive: true });
fs.writeFileSync(path.join(oldPartitionDir, "2020-01-01.csv"), "h\nr\n", "utf8");
const prunedCount = CSVStorageManager.pruneExpiredPartitions(30);
assert.equal(prunedCount, 1);
serverMod.applyRuntimeSettingsObject({ autoPruneDays: 30 });
fs.writeFileSync(path.join(oldPartitionDir, "2020-01-02.csv"), "h\nr\n", "utf8");
// Same autoPruneDays=30 applied again (simulating 3s DB poll) must NOT prune on every tick
serverMod.applyRuntimeSettingsObject({ autoPruneDays: 30 });
assert.equal(fs.existsSync(path.join(oldPartitionDir, "2020-01-02.csv")), true);
serverMod.stopTimers();
});
test("BUG-12: Prisma schema models and mappings align cleanly with runtime SQL DDL and Docker Compose Redis wiring", () => {
const schemaPath = path.resolve(process.cwd(), "prisma/schema.prisma");
const schemaText = fs.readFileSync(schemaPath, "utf8");
assert.match(schemaText, /model ClientSDK[\s\S]*clientId\s+String\s+@id\s+@map\("client_id"\)/);
assert.match(schemaText, /token\s+String/);
assert.match(schemaText, /model DeletedClient[\s\S]*@@map\("deleted_clients"\)/);
assert.match(schemaText, /model AdminUser[\s\S]*tokenVersion\s+Int\s+@default\(1\)\s+@map\("token_version"\)/);
assert.match(schemaText, /model AdminLoginAttempt[\s\S]*@@map\("admin_login_attempts"\)/);
assert.match(schemaText, /model AdminSettings[\s\S]*maxEventsPerSec\s+Int\s+@default\(100\)\s+@map\("max_events_per_sec"\)/);
assert.equal(/model ApiKey\b/.test(schemaText), false);
assert.equal(/model AuditLog\b/.test(schemaText), false);
const composeText = fs.readFileSync(path.resolve(process.cwd(), "docker/docker-compose.yml"), "utf8");
assert.equal((composeText.match(/REDIS_URL:\s*"redis:\/\/redis:6379"/g) || []).length, 2);
assert.match(composeText, /redis:\s*\n\s*condition:\s*service_healthy/);
});
test("BUG-14: GET /api/data searches eventId & clientId + UTC dates, GET /api/events caps CSV tail, and GET /api/export rejects ambiguous filename & honors ?flush=false", async () => {
DataBufferEngine.setRedisClientForTests(null);
await SDKManager.registerClientAsync({ clientId: "alpha_cli", name: "Alpha", rateLimit: 100 });
await SDKManager.registerClientAsync({ clientId: "beta_cli", name: "Beta", rateLimit: 100 });
const tsMillis = String(Date.UTC(2026, 10, 17, 15, 0, 0)); // 2026-11-17
await DataBufferEngine.pushBatchAsync([
{
clientId: "alpha_cli",
eventId: "unique-findme-id-99",
timestamp: tsMillis,
payload: JSON.stringify({ temp: 22 }),
},
{
clientId: "beta_cli",
eventId: "beta-id-1",
timestamp: "2026-11-17T15:01:00.000Z",
payload: JSON.stringify({ temp: 33 }),
},
]);
// 1. Search by eventId with from/to UTC normalization while still buffered
const dataByEvent = await getDataRoute(
new NextRequest("http://localhost:3000/api/data?search=unique-findme-id-99&from=2026-11-17&to=2026-11-17"),
);
assert.equal(dataByEvent.status, 200);
const dataEventJson = await dataByEvent.json();
assert.equal(dataEventJson.total, 1);
assert.equal(dataEventJson.events[0].eventId, "unique-findme-id-99");
// 2. Search by clientId substring
const dataByClient = await getDataRoute(
new NextRequest("http://localhost:3000/api/data?search=beta_cli"),
);
const dataClientJson = await dataByClient.json();
assert.equal(dataClientJson.total, 1);
assert.equal(dataClientJson.events[0].clientId, "beta_cli");
// 3. Export with ?flush=false must NOT drain active buffer
const noFlushRes = await getExportRoute(
new NextRequest("http://localhost:3000/api/export?flush=false&format=json"),
);
assert.equal(noFlushRes.status, 200);
assert.equal((await noFlushRes.text()).trim(), "[]");
assert.equal((await DataBufferEngine.getBufferStatsAsync()).localBufferSize, 2);
// 4. Default flush writes both 2026-11-17.csv files -> clientId=all + filename must 400 AMBIGUOUS_FILENAME
const flushExportRes = await getExportRoute(
new NextRequest("http://localhost:3000/api/export?format=json"),
);
assert.equal(flushExportRes.status, 200);
const flushedPayload = await flushExportRes.json();
assert.equal(flushedPayload.length, 2);
const ambiguousRes = await getExportRoute(
new NextRequest("http://localhost:3000/api/export?clientId=all&filename=2026-11-17.csv"),
);
assert.equal(ambiguousRes.status, 400);
const ambiguousJson = await ambiguousRes.json();
assert.equal(ambiguousJson.code, "AMBIGUOUS_FILENAME");
// Disambiguated download with explicit clientId succeeds cleanly
const specificRes = await getExportRoute(
new NextRequest("http://localhost:3000/api/export?clientId=alpha_cli&filename=2026-11-17.csv"),
);
assert.equal(specificRes.status, 200);
const csvBody = await specificRes.text();
assert.match(csvBody, /unique-findme-id-99/);
// 5. GET /api/events with limit=1 returns latest single stored event cleanly
const eventsRes = await getEventsRoute(
new NextRequest("http://localhost:3000/api/events?clientId=alpha_cli&limit=1"),
);
assert.equal(eventsRes.status, 200);
const eventsJson = await eventsRes.json();
assert.equal(eventsJson.events.length, 1);
assert.equal(eventsJson.events[0].eventId, "unique-findme-id-99");
});
+266
View File
@@ -0,0 +1,266 @@
"""
Phase-05 Verification Tests (BUG-18):
- Single Source of Truth between canonical `datastack` (`sdk/python/datastack`)
and compatibility alias `datastack_sdk` (`sdk-python`).
- `CLIENT_ID` config loading (`config.datastack.py`, `DATASTACK_CLIENT_ID`, kwarg),
JWT `clientId`/`sub` fallback extraction, and mismatch validation.
- End-to-end propagation of `clientId` & `X-DataStack-Client-Id` into HTTP
fallback ingestion (`send`, `send_batch`, `send_file`), status snapshot, and
offline disk spool replay against a local HTTP test server.
"""
import base64
import json
import os
import sys
import tempfile
import threading
import unittest
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import Any, Dict, List
REPO_ROOT = Path(__file__).resolve().parents[1]
CANONICAL_PKG_ROOT = REPO_ROOT / "sdk" / "python" / "datastack"
COMPAT_PKG_ROOT = REPO_ROOT / "sdk-python"
for p in (str(CANONICAL_PKG_ROOT), str(COMPAT_PKG_ROOT)):
if p not in sys.path:
sys.path.insert(0, p)
import datastack # noqa: E402
import datastack_sdk # noqa: E402
from datastack import ( # noqa: E402
DataStack,
DataStackClient,
DataStackConfig,
DataStackConfigError,
DataStackOfflineSpoolError,
DiskSpooler,
HeartbeatMonitor,
load_config,
)
def _make_jwt(payload: Dict[str, Any]) -> str:
header = (
base64.urlsafe_b64encode(json.dumps({"alg": "HS256", "typ": "JWT"}).encode())
.decode()
.rstrip("=")
)
body = (
base64.urlsafe_b64encode(json.dumps(payload).encode())
.decode()
.rstrip("=")
)
return f"{header}.{body}.testsig"
class Phase05PythonSdkTests(unittest.TestCase):
def test_single_source_of_truth_identity(self) -> None:
self.assertIs(datastack_sdk.DataStack, datastack.DataStack)
self.assertIs(datastack_sdk.DataStackClient, datastack.DataStackClient)
self.assertIs(datastack_sdk.DataStackConfig, datastack.DataStackConfig)
self.assertIs(datastack_sdk.load_config, datastack.load_config)
self.assertIs(
datastack_sdk.DataStackConfigError, datastack.DataStackConfigError
)
self.assertIs(
datastack_sdk.DataStackOfflineSpoolError,
datastack.DataStackOfflineSpoolError,
)
self.assertIs(datastack_sdk.DiskSpooler, datastack.DiskSpooler)
self.assertIs(datastack_sdk.HeartbeatMonitor, datastack.HeartbeatMonitor)
self.assertEqual(datastack_sdk.__version__, datastack.__version__)
# Submodule shims also resolve to canonical classes
from datastack_sdk.client import DataStackClient as ShimClient
from datastack_sdk.config import load_config as shim_load_config
from datastack_sdk.exceptions import (
DataStackOfflineSpoolError as ShimSpoolErr,
)
self.assertIs(ShimClient, DataStackClient)
self.assertIs(shim_load_config, load_config)
self.assertIs(ShimSpoolErr, DataStackOfflineSpoolError)
def test_client_id_config_env_and_jwt_validation(self) -> None:
tok_alpha = _make_jwt({"clientId": "alpha_svc", "sub": "alpha_svc"})
with tempfile.TemporaryDirectory() as tmpdir:
cfg_file = Path(tmpdir) / "config.datastack.py"
cfg_file.write_text(
f'DATASTACK_CONFIG = {{\n'
f' "SERVER_URL": "http://127.0.0.1:49999",\n'
f' "API_URL": "http://127.0.0.1:39999",\n'
f' "CLIENT_TOKEN": "{tok_alpha}",\n'
f' "CLIENT_ID": "alpha_svc",\n'
f' "AUTO_CONNECT": False,\n'
f' "OFFLINE_SPOOL_DIR": "{Path(tmpdir) / "spool"}",\n'
f'}}\n',
encoding="utf-8",
)
cfg = load_config(config_path=str(cfg_file))
self.assertIsInstance(cfg, DataStackConfig)
self.assertEqual(cfg.client_id, "alpha_svc")
# Matching explicit client_id succeeds
client = DataStack(config_path=str(cfg_file), auto_connect=False)
self.assertEqual(client.client_id, "alpha_svc")
self.assertEqual(client.status()["clientId"], "alpha_svc")
client.disconnect()
# Env override matching JWT succeeds; mismatching raises DataStackConfigError
old_env = os.environ.get("DATASTACK_CLIENT_ID")
try:
os.environ["DATASTACK_CLIENT_ID"] = "wrong_svc"
with self.assertRaises(DataStackConfigError):
DataStack(config_path=str(cfg_file), auto_connect=False)
finally:
if old_env is None:
os.environ.pop("DATASTACK_CLIENT_ID", None)
else:
os.environ["DATASTACK_CLIENT_ID"] = old_env
# Mismatched kwarg client_id raises DataStackConfigError
with self.assertRaises(DataStackConfigError):
DataStack(
config_path=str(cfg_file),
client_id="mismatch_svc",
auto_connect=False,
)
# Auto-derives client_id from JWT when CLIENT_ID is omitted
tok_derived = _make_jwt({"sub": "derived_worker"})
derived_client = DataStack(
server_url="http://127.0.0.1:49999",
api_url="http://127.0.0.1:39999",
token=tok_derived,
auto_connect=False,
)
self.assertEqual(derived_client.client_id, "derived_worker")
self.assertEqual(derived_client.status()["clientId"], "derived_worker")
derived_client.disconnect()
def test_http_fallback_and_offline_spool_carry_client_id(self) -> None:
recorded_requests: List[Dict[str, Any]] = []
tok = _make_jwt({"clientId": "sensor_01", "sub": "sensor_01"})
class MockHandler(BaseHTTPRequestHandler):
def do_GET(self) -> None: # noqa: N802
if self.path == "/api/health":
body = json.dumps({"status": "online"}).encode("utf-8")
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
return
self.send_response(404)
self.end_headers()
def do_POST(self) -> None: # noqa: N802
length = int(self.headers.get("Content-Length", "0"))
raw = self.rfile.read(length).decode("utf-8")
parsed = json.loads(raw) if raw else {}
recorded_requests.append(
{
"path": self.path,
"client_id_header": self.headers.get("X-DataStack-Client-Id"),
"auth": self.headers.get("Authorization"),
"body": parsed,
}
)
resp_payload: Dict[str, Any] = {"success": True}
if isinstance(parsed, dict) and "batch" in parsed:
resp_payload["processed"] = len(parsed["batch"])
resp_payload["rejected"] = 0
elif isinstance(parsed, dict) and "records" in parsed:
resp_payload["processed"] = len(parsed["records"])
resp_payload["rejected"] = 0
body = json.dumps(resp_payload).encode("utf-8")
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, format: str, *args: Any) -> None:
return
server = ThreadingHTTPServer(("127.0.0.1", 0), MockHandler)
port = server.server_address[1]
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
with tempfile.TemporaryDirectory() as tmpdir:
spool_dir = Path(tmpdir) / "spool"
csv_file = Path(tmpdir) / "sample.csv"
csv_file.write_text("temp,unit\n21.5,C\n22.0,C\n", encoding="utf-8")
client = DataStack(
server_url="http://127.0.0.1:1", # unreachable WS -> HTTP fallback
api_url=f"http://127.0.0.1:{port}",
token=tok,
client_id="sensor_01",
auto_connect=False,
request_timeout_sec=1.0,
max_retries=0,
offline_spool_dir=str(spool_dir),
)
# Single event via HTTP fallback
res_single = client.send({"reading": 42})
self.assertTrue(res_single.get("success"))
self.assertEqual(res_single.get("transport"), "HTTP_FALLBACK")
# Batch events via HTTP fallback
res_batch = client.send_batch([{"k": "a"}, {"k": "b"}])
self.assertEqual(res_batch.get("processed"), 2)
# File ingestion via HTTP fallback
res_file = client.send_file(csv_file)
self.assertEqual(res_file.get("processed"), 2)
client.disconnect()
self.assertEqual(len(recorded_requests), 3)
for req in recorded_requests:
self.assertEqual(req["path"], "/api/events")
self.assertEqual(req["client_id_header"], "sensor_01")
self.assertEqual(req["auth"], f"Bearer {tok}")
self.assertEqual(recorded_requests[0]["body"]["clientId"], "sensor_01")
self.assertEqual(
[r["clientId"] for r in recorded_requests[1]["body"]["batch"]],
["sensor_01", "sensor_01"],
)
self.assertEqual(
[r["clientId"] for r in recorded_requests[2]["body"]["records"]],
["sensor_01", "sensor_01"],
)
# Offline spool check when HTTP is also unreachable
offline_client = datastack_sdk.DataStackClient(
server_url="http://127.0.0.1:1",
api_url="http://127.0.0.1:1",
token=tok,
auto_connect=False,
request_timeout_sec=1.0,
max_retries=0,
offline_spool_dir=str(spool_dir),
)
spool_res = offline_client.send({"offline_metric": 99})
offline_client.disconnect()
self.assertTrue(spool_res.get("spooled"))
self.assertEqual(spool_res.get("transport"), "OFFLINE_SPOOL")
self.assertGreaterEqual(spool_res.get("spooledCount", 0), 1)
finally:
server.shutdown()
server.server_close()
if __name__ == "__main__":
unittest.main(verbosity=2)
+81
View File
@@ -0,0 +1,81 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import test from 'node:test';
const ROOT = process.cwd();
function readPngDimensions(filePath: string): { width: number; height: number } {
const buf = fs.readFileSync(filePath);
const pngSig = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
assert.ok(buf.subarray(0, 8).equals(pngSig), `${path.basename(filePath)} must be a valid PNG`);
assert.equal(buf.subarray(12, 16).toString('ascii'), 'IHDR');
return {
width: buf.readUInt32BE(16),
height: buf.readUInt32BE(20),
};
}
test('BUG-25: PWA manifest, icons (192/512/maskable/apple-touch), and Service Worker are complete and valid', () => {
const manifestPath = path.join(ROOT, 'public/manifest.json');
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
assert.equal(manifest.id, '/');
assert.equal(manifest.scope, '/');
assert.equal(manifest.display, 'standalone');
assert.ok(manifest.short_name && manifest.short_name.length <= 12);
assert.ok(Array.isArray(manifest.icons) && manifest.icons.length >= 3);
for (const icon of manifest.icons) {
assert.ok(!String(icon.purpose || '').includes('any maskable'), 'Separate any and maskable purposes');
const rel = String(icon.src).replace(/^\//, '');
const full = path.join(ROOT, 'public', rel);
assert.ok(fs.existsSync(full), `Missing manifest icon ${icon.src}`);
const [expW, expH] = String(icon.sizes).split('x').map(Number);
const dims = readPngDimensions(full);
assert.equal(dims.width, expW);
assert.equal(dims.height, expH);
}
const appleDims = readPngDimensions(path.join(ROOT, 'public/apple-touch-icon.png'));
assert.deepEqual(appleDims, { width: 180, height: 180 });
assert.ok(fs.existsSync(path.join(ROOT, 'public/icon.svg')));
const swContent = fs.readFileSync(path.join(ROOT, 'public/sw.js'), 'utf8');
assert.match(swContent, /request\.method !== ['"]GET['"]/);
assert.match(swContent, /\/api\//);
assert.match(swContent, /\/socket\.io\//);
assert.match(swContent, /new Response\(/);
const layoutContent = fs.readFileSync(path.join(ROOT, 'app/layout.tsx'), 'utf8');
assert.match(layoutContent, /PwaLifecycle/);
assert.match(layoutContent, /export const viewport: Viewport/);
assert.match(layoutContent, /apple-touch-icon\.png/);
const pwaClient = fs.readFileSync(path.join(ROOT, 'components/pwa/PwaLifecycle.tsx'), 'utf8');
assert.match(pwaClient, /serviceWorker\.register\(['"]\/sw\.js['"]/);
assert.match(pwaClient, /beforeinstallprompt/);
});
test('BUG-17: Landing/Navbar/Footer/404 navigation uses root-relative anchors and valid Link semantics', () => {
const files = [
'components/layout/Navbar.tsx',
'components/layout/Footer.tsx',
'components/sections/Hero.tsx',
'components/sections/LandingAbout.tsx',
'app/not-found.tsx',
];
for (const rel of files) {
const content = fs.readFileSync(path.join(ROOT, rel), 'utf8');
assert.equal(/href=["']#/.test(content), false, `${rel} should use /#... root anchors instead of bare #...`);
const linkBlocks = content.match(/<Link\b[\s\S]*?<\/Link>/g) || [];
for (const block of linkBlocks) {
assert.equal(/<Button\b/.test(block), false, `${rel} should not nest <Button> inside <Link>: ${block}`);
}
}
const notFoundContent = fs.readFileSync(path.join(ROOT, 'app/not-found.tsx'), 'utf8');
assert.match(notFoundContent, /href=["']\/["']/);
assert.match(notFoundContent, /href=["']\/vcon\/dashboard["']/);
});