From 177ed4a844118117603de65cdf8653fba2a768a0 Mon Sep 17 00:00:00 2001
From: Team Dark Devil
Date: Wed, 18 Nov 2026 05:18:55 +0600
Subject: [PATCH] refactor(auth): standardize session management and security
- Centralize secret retrieval using `getAdminSessionSecret`
- Replace manual session parsing with `verifyRequestAdminSession` utility
- Tighten API validation and improve security headers handling
- Enforce environment variable security for production deployments
---
.env.example | 27 +-
app/(landing)/page.tsx | 6 +-
app/api/auth/login/route.ts | 181 +--
app/api/auth/profile/route.ts | 89 +-
app/api/auth/setup/route.ts | 21 +-
app/api/clients/[id]/route.ts | 50 +-
app/api/clients/route.ts | 47 +-
app/api/data/route.ts | 44 +-
app/api/events/route.ts | 105 +-
app/api/export/route.ts | 54 +-
app/api/health/route.ts | 14 +-
app/api/settings/route.ts | 180 ++-
app/api/storage/route.ts | 70 +-
app/error.tsx | 50 +
app/layout.tsx | 31 +-
app/not-found.tsx | 11 +-
app/vcon/clients/page.tsx | 21 +-
app/vcon/error.tsx | 50 +
app/vcon/login/page.tsx | 11 +-
app/vcon/sdk/page.tsx | 117 +-
app/vcon/simulator/page.tsx | 353 +++++-
bun.lock | 442 ++++---
components/layout/Footer.tsx | 6 +-
components/layout/Navbar.tsx | 20 +-
components/pwa/PwaLifecycle.tsx | 175 +++
components/sections/AdminLogin.tsx | 4 +-
components/sections/ClientsFilters.tsx | 1 +
components/sections/ClientsTable.tsx | 159 ++-
components/sections/DashboardOverview.tsx | 112 +-
components/sections/DataBrowser.tsx | 136 ++-
components/sections/ExportWorkspace.tsx | 55 +-
components/sections/Hero.tsx | 10 +-
components/sections/LandingAbout.tsx | 6 +-
components/sections/SettingsWorkspace.tsx | 209 +++-
components/sections/clients/types.ts | 8 +-
components/ui/Button.tsx | 70 +-
core/Database/database.ts | 25 +
core/Security/security.ts | 49 +-
core/Storage/csvStorage.ts | 445 ++++++-
core/adminSettings/settingsManager.ts | 80 +-
core/apiEngine/apiService.ts | 83 +-
core/auth/adminSession.ts | 30 +-
core/auth/adminUsers.ts | 442 ++++---
core/auth/jwt.ts | 43 +-
core/data/buffer.ts | 408 ++++++-
core/sdk/sdkManager.ts | 362 +++++-
docker/docker-compose.yml | 8 +-
middleware.ts | 49 +-
nginx/default.conf | 21 +-
package.json | 1 +
prisma/schema.prisma | 78 +-
public/apple-touch-icon.png | Bin 0 -> 632 bytes
public/icon-192.png | Bin 0 -> 718 bytes
public/icon-512.png | Bin 0 -> 3695 bytes
public/icon-maskable-512.png | Bin 0 -> 2802 bytes
public/icon.svg | 7 +
public/manifest.json | 22 +-
public/sw.js | 52 +-
sdk-python/README.md | 20 +-
sdk-python/datastack_sdk/__init__.py | 30 +-
sdk-python/datastack_sdk/client.py | 1194 +------------------
sdk-python/datastack_sdk/config.py | 322 +----
sdk-python/datastack_sdk/exceptions.py | 73 +-
sdk-python/datastack_sdk/heartbeat.py | 187 +--
sdk-python/datastack_sdk/spool.py | 165 +--
sdk/python/datastack/README.md | 1 +
sdk/python/datastack/datastack/__init__.py | 10 +-
sdk/python/datastack/datastack/client.py | 116 +-
sdk/python/datastack/datastack/heartbeat.py | 5 +
sdk/python/datastack/datastack/spool.py | 45 +-
server/index.js | 812 +++++++++++--
server/socket/handlers.js | 55 +-
server/workers/csvWriter.js | 147 ++-
tests/phase01-auth.test.ts | 340 ++++++
tests/phase02-sync.test.ts | 375 ++++++
tests/phase03-storage-buffer.test.ts | 511 ++++++++
tests/phase05_sdk_test.py | 266 +++++
tests/phase06-pwa-landing.test.ts | 81 ++
78 files changed, 6748 insertions(+), 3157 deletions(-)
create mode 100644 app/error.tsx
create mode 100644 app/vcon/error.tsx
create mode 100644 components/pwa/PwaLifecycle.tsx
create mode 100644 public/apple-touch-icon.png
create mode 100644 public/icon-192.png
create mode 100644 public/icon-512.png
create mode 100644 public/icon-maskable-512.png
create mode 100644 public/icon.svg
create mode 100644 tests/phase01-auth.test.ts
create mode 100644 tests/phase02-sync.test.ts
create mode 100644 tests/phase03-storage-buffer.test.ts
create mode 100644 tests/phase05_sdk_test.py
create mode 100644 tests/phase06-pwa-landing.test.ts
diff --git a/.env.example b/.env.example
index 7e70749..8f114b4 100644
--- a/.env.example
+++ b/.env.example
@@ -1,16 +1,23 @@
# DataStack Environment Variables Configuration
+# NOTE: In production (NODE_ENV=production), replace all placeholder values below with strong 32+ char secrets.
-# Administrator Account (defaults to admin@datastack.dev / admin12345678 if omitted)
-DATASTACK_ADMIN_EMAIL=admin@datastack.dev
-DATASTACK_ADMIN_PASSWORD=admin12345678
+# Security & Session Secrets (REQUIRED in production, min 32 characters)
+DATASTACK_ADMIN_SESSION_SECRET=replace-with-32-plus-char-admin-session-secret
+DATASTACK_JWT_SECRET=replace-with-32-plus-char-client-jwt-secret
+DATASTACK_ADMIN_SETUP_KEY=replace-with-32-plus-char-admin-setup-key
+DATASTACK_INTERNAL_SECRET=replace-with-32-plus-char-internal-service-secret
-# Security & Session Secrets (min 32 characters)
-DATASTACK_ADMIN_SESSION_SECRET=datastack-admin-session-default-secret-32-chars
-DATASTACK_JWT_SECRET=datastack-jwt-default-secret-key-32-chars-long
-DATASTACK_ADMIN_SETUP_KEY=datastack-default-setup-key-32-chars
+# Administrator Account (development/in-memory bootstrap; production uses /vcon/setup)
+DATASTACK_ADMIN_EMAIL=admin@example.com
+# DATASTACK_ADMIN_PASSWORD=replace-with-strong-password
+# DATASTACK_ALLOW_DEMO_ADMIN=false
-# PostgreSQL Database (optional - in-memory fallback active when not configured)
+# PostgreSQL Database (recommended for production persistence)
# DATABASE_URL=postgresql://datastack:password@localhost:5432/datastack_db
-# Ingestion Engine Port (defaults to 4000)
-PORT_SOCKET=4000
\ No newline at end of file
+# Shared Redis Buffer (optional; enables cross-process buffering & distributed flush locks)
+# REDIS_URL=redis://127.0.0.1:6379
+
+# Service Networking
+PORT_SOCKET=4000
+INGESTION_INTERNAL_URL=http://127.0.0.1:4000
\ No newline at end of file
diff --git a/app/(landing)/page.tsx b/app/(landing)/page.tsx
index 5be914c..7545dd7 100644
--- a/app/(landing)/page.tsx
+++ b/app/(landing)/page.tsx
@@ -9,10 +9,8 @@ import { redirect } from 'next/navigation';
export const dynamic = 'force-dynamic';
export default async function LandingPage() {
- if (process.env.DATABASE_URL) {
- const isConfigured = await isAdminSetupComplete().catch(() => false);
- if (!isConfigured) redirect('/vcon/setup');
- }
+ const isConfigured = await isAdminSetupComplete().catch(() => false);
+ if (!isConfigured) redirect('/vcon/setup');
return (
diff --git a/app/api/auth/login/route.ts b/app/api/auth/login/route.ts
index 96d7f02..afd12a6 100644
--- a/app/api/auth/login/route.ts
+++ b/app/api/auth/login/route.ts
@@ -1,11 +1,12 @@
import {
createAdminSessionToken,
+ getAdminSessionSecret,
getSessionCookieOptions,
} from "@/core/auth/adminSession";
import {
+ authenticateAdminCredentials,
clearAdminLoginAttempts,
consumeAdminLoginAttempt,
- verifyAdminCredentials,
} from "@/core/auth/adminUsers";
import { NextRequest, NextResponse } from "next/server";
@@ -13,10 +14,19 @@ const SESSION_COOKIE = "datastack_admin_session";
const SESSION_SECONDS = 8 * 60 * 60;
const REMEMBERED_SESSION_SECONDS = 30 * 24 * 60 * 60;
+function extractClientIp(request: NextRequest) {
+ const forwardedFor = request.headers.get("x-forwarded-for");
+ if (forwardedFor) {
+ const first = forwardedFor.split(",")[0]?.trim();
+ if (first) return first;
+ }
+ const realIp = request.headers.get("x-real-ip")?.trim();
+ if (realIp) return realIp;
+ return "unknown";
+}
+
export async function POST(request: NextRequest) {
- const sessionSecret =
- process.env.DATASTACK_ADMIN_SESSION_SECRET ||
- "datastack-admin-session-default-secret-32-chars";
+ const sessionSecret = getAdminSessionSecret();
if (!sessionSecret || sessionSecret.length < 32) {
return NextResponse.json(
@@ -27,86 +37,99 @@ export async function POST(request: NextRequest) {
);
}
+ let body: Record;
try {
- const body = await request.json();
- const email =
- typeof body.email === "string" ? body.email.trim().toLowerCase() : "";
- const password = typeof body.password === "string" ? body.password : "";
- if (!email || password.length === 0 || password.length > 1024) {
- return NextResponse.json(
- { error: "Invalid email or password." },
- { status: 401 },
- );
- }
-
- let throttleDecision;
- try {
- throttleDecision = await consumeAdminLoginAttempt(email, sessionSecret);
- } catch {
- return NextResponse.json(
- { error: "Admin authentication is temporarily unavailable." },
- { status: 503 },
- );
- }
- if (!throttleDecision.allowed) {
- return NextResponse.json(
- { error: "Too many sign-in attempts. Try again later." },
- {
- status: 429,
- headers: {
- "Retry-After": String(throttleDecision.retryAfterSeconds),
- },
- },
- );
- }
-
- let authenticated = false;
- try {
- authenticated = await verifyAdminCredentials(email, password);
- } catch {
- return NextResponse.json(
- { error: "Admin authentication is temporarily unavailable." },
- { status: 503 },
- );
- }
-
- if (!authenticated) {
- return NextResponse.json(
- { error: "Invalid email or password." },
- { status: 401 },
- );
- }
-
- try {
- await clearAdminLoginAttempts(email, sessionSecret);
- } catch {
- return NextResponse.json(
- { error: "Admin authentication is temporarily unavailable." },
- { status: 503 },
- );
- }
-
- const remember = body.remember === true;
- const maxAge = remember ? REMEMBERED_SESSION_SECONDS : SESSION_SECONDS;
- const expiresAt = Date.now() + maxAge * 1000;
- const token = await createAdminSessionToken(
- email,
- sessionSecret,
- expiresAt,
- );
- const response = NextResponse.json({ success: true });
-
- response.cookies.set(
- SESSION_COOKIE,
- token,
- getSessionCookieOptions(request, maxAge),
- );
-
- return response;
+ const parsed = await request.json();
+ body =
+ parsed && typeof parsed === "object" && !Array.isArray(parsed)
+ ? (parsed as Record)
+ : {};
} catch {
return NextResponse.json(
{ error: "Invalid login request." },
{ status: 400 },
);
}
+
+ const email =
+ typeof body.email === "string" ? body.email.trim().toLowerCase() : "";
+ const password = typeof body.password === "string" ? body.password : "";
+ if (!email || password.length === 0 || password.length > 1024) {
+ return NextResponse.json(
+ { error: "Invalid email or password." },
+ { status: 401 },
+ );
+ }
+
+ const clientIp = extractClientIp(request);
+
+ let throttleDecision;
+ try {
+ throttleDecision = await consumeAdminLoginAttempt(
+ email,
+ sessionSecret,
+ clientIp,
+ );
+ } catch {
+ return NextResponse.json(
+ { error: "Admin authentication is temporarily unavailable." },
+ { status: 503 },
+ );
+ }
+ if (!throttleDecision.allowed) {
+ return NextResponse.json(
+ { error: "Too many sign-in attempts. Try again later." },
+ {
+ status: 429,
+ headers: {
+ "Retry-After": String(throttleDecision.retryAfterSeconds),
+ },
+ },
+ );
+ }
+
+ let authResult: { authenticated: boolean; tokenVersion: number };
+ try {
+ authResult = await authenticateAdminCredentials(email, password);
+ } catch {
+ return NextResponse.json(
+ { error: "Admin authentication is temporarily unavailable." },
+ { status: 503 },
+ );
+ }
+
+ if (!authResult.authenticated) {
+ return NextResponse.json(
+ { error: "Invalid email or password." },
+ { status: 401 },
+ );
+ }
+
+ try {
+ await clearAdminLoginAttempts(email, sessionSecret, clientIp);
+ } catch {
+ return NextResponse.json(
+ { error: "Admin authentication is temporarily unavailable." },
+ { status: 503 },
+ );
+ }
+
+ const remember = body.remember === true;
+ const maxAge = remember ? REMEMBERED_SESSION_SECONDS : SESSION_SECONDS;
+ const expiresAt = Date.now() + maxAge * 1000;
+ const token = await createAdminSessionToken(
+ email,
+ sessionSecret,
+ expiresAt,
+ authResult.tokenVersion,
+ );
+ const response = NextResponse.json({ success: true });
+
+ response.cookies.set(
+ SESSION_COOKIE,
+ token,
+ getSessionCookieOptions(request, maxAge),
+ );
+
+ return response;
}
diff --git a/app/api/auth/profile/route.ts b/app/api/auth/profile/route.ts
index 8baee38..9cdee9c 100644
--- a/app/api/auth/profile/route.ts
+++ b/app/api/auth/profile/route.ts
@@ -1,11 +1,11 @@
import {
createAdminSessionToken,
- getAdminSessionPayload,
getSessionCookieOptions,
} from "@/core/auth/adminSession";
import {
getAdminAccountInfo,
updateAdminAccount,
+ verifyRequestAdminSession,
} from "@/core/auth/adminUsers";
import { NextRequest, NextResponse } from "next/server";
@@ -17,51 +17,69 @@ function validEmail(email: string) {
}
export async function GET(request: NextRequest) {
- const sessionSecret =
- process.env.DATASTACK_ADMIN_SESSION_SECRET ||
- "datastack-admin-session-default-secret-32-chars";
- const token = request.cookies.get(SESSION_COOKIE)?.value || "";
- const session = await getAdminSessionPayload(token, sessionSecret);
+ const token = request.cookies.get(SESSION_COOKIE)?.value;
+ const auth = await verifyRequestAdminSession(token);
- if (!session) {
- return NextResponse.json(
- { error: "Authentication required." },
- { status: 401 },
- );
+ if (!auth.valid) {
+ const status = auth.reason === "UNAUTHORIZED" ? 401 : 503;
+ const message =
+ auth.reason === "UNAUTHORIZED"
+ ? "Authentication required."
+ : "Admin authentication is temporarily unavailable.";
+ return NextResponse.json({ error: message }, { status });
}
- const account = await getAdminAccountInfo(session.email);
- return NextResponse.json({
- profile: {
- email: account.email,
- role: "Owner",
- storageMode: account.storageMode,
- createdAt: account.createdAt,
- sessionExpiresAt: new Date(session.expiresAt).toISOString(),
- },
- });
+ try {
+ const account = await getAdminAccountInfo(auth.session.email);
+ return NextResponse.json({
+ profile: {
+ email: account.email,
+ role: "Owner",
+ storageMode: account.storageMode,
+ createdAt: account.createdAt,
+ sessionExpiresAt: new Date(auth.session.expiresAt).toISOString(),
+ },
+ });
+ } catch {
+ return NextResponse.json(
+ { error: "Unable to load admin profile." },
+ { status: 503 },
+ );
+ }
}
export async function PUT(request: NextRequest) {
- const sessionSecret =
- process.env.DATASTACK_ADMIN_SESSION_SECRET ||
- "datastack-admin-session-default-secret-32-chars";
- const token = request.cookies.get(SESSION_COOKIE)?.value || "";
- const session = await getAdminSessionPayload(token, sessionSecret);
+ const token = request.cookies.get(SESSION_COOKIE)?.value;
+ const auth = await verifyRequestAdminSession(token);
- if (!session) {
+ if (!auth.valid) {
+ const status = auth.reason === "UNAUTHORIZED" ? 401 : 503;
+ const message =
+ auth.reason === "UNAUTHORIZED"
+ ? "Authentication required."
+ : "Admin authentication is temporarily unavailable.";
+ return NextResponse.json({ error: message }, { status });
+ }
+
+ let body: Record;
+ try {
+ const parsed = await request.json();
+ body =
+ parsed && typeof parsed === "object" && !Array.isArray(parsed)
+ ? (parsed as Record)
+ : {};
+ } catch {
return NextResponse.json(
- { error: "Authentication required." },
- { status: 401 },
+ { error: "Invalid profile payload." },
+ { status: 400 },
);
}
try {
- const body = await request.json();
const nextEmail =
typeof body.email === "string"
? body.email.trim().toLowerCase()
- : session.email;
+ : auth.session.email;
const currentPassword =
typeof body.currentPassword === "string" ? body.currentPassword : "";
const newPassword =
@@ -88,7 +106,7 @@ export async function PUT(request: NextRequest) {
);
}
- const account = await getAdminAccountInfo(session.email);
+ const account = await getAdminAccountInfo(auth.session.email);
const result = await updateAdminAccount({
currentEmail: account.email,
currentPassword,
@@ -103,13 +121,14 @@ export async function PUT(request: NextRequest) {
// Preserve remaining session duration (e.g., 30-day "Remember me" session)
const remainingSeconds = Math.max(
SESSION_SECONDS,
- Math.ceil((session.expiresAt - Date.now()) / 1000),
+ Math.ceil((auth.session.expiresAt - Date.now()) / 1000),
);
const expiresAt = Date.now() + remainingSeconds * 1000;
const nextToken = await createAdminSessionToken(
result.email,
- sessionSecret,
+ auth.sessionSecret,
expiresAt,
+ result.tokenVersion,
);
const response = NextResponse.json({
@@ -133,7 +152,7 @@ export async function PUT(request: NextRequest) {
} catch {
return NextResponse.json(
{ error: "Unable to update profile." },
- { status: 400 },
+ { status: 500 },
);
}
}
diff --git a/app/api/auth/setup/route.ts b/app/api/auth/setup/route.ts
index 2881166..331a20b 100644
--- a/app/api/auth/setup/route.ts
+++ b/app/api/auth/setup/route.ts
@@ -1,5 +1,6 @@
import {
createAdminSessionToken,
+ getAdminSessionSecret,
getSessionCookieOptions,
} from "@/core/auth/adminSession";
import {
@@ -20,9 +21,7 @@ function validEmail(email: string) {
}
export async function POST(request: NextRequest) {
- const sessionSecret =
- process.env.DATASTACK_ADMIN_SESSION_SECRET ||
- "datastack-admin-session-default-secret-32-chars";
+ const sessionSecret = getAdminSessionSecret();
if (!sessionSecret || sessionSecret.length < 32) {
return NextResponse.json(
{ error: "Admin session configuration is unavailable." },
@@ -30,8 +29,21 @@ export async function POST(request: NextRequest) {
);
}
+ let body: Record;
+ try {
+ const parsed = await request.json();
+ body =
+ parsed && typeof parsed === "object" && !Array.isArray(parsed)
+ ? (parsed as Record)
+ : {};
+ } catch {
+ return NextResponse.json(
+ { error: "Invalid setup request payload." },
+ { status: 400 },
+ );
+ }
+
try {
- const body = await request.json();
const email =
typeof body.email === "string" ? body.email.trim().toLowerCase() : "";
const password = typeof body.password === "string" ? body.password : "";
@@ -74,6 +86,7 @@ export async function POST(request: NextRequest) {
email,
sessionSecret,
Date.now() + SESSION_SECONDS * 1000,
+ 1,
);
const response = NextResponse.json({ success: true });
response.cookies.set(
diff --git a/app/api/clients/[id]/route.ts b/app/api/clients/[id]/route.ts
index ce37e00..8523234 100644
--- a/app/api/clients/[id]/route.ts
+++ b/app/api/clients/[id]/route.ts
@@ -1,23 +1,54 @@
+import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
import { SDKManager } from "@/core/sdk/sdkManager";
import { NextRequest, NextResponse } from "next/server";
+const SESSION_COOKIE = "datastack_admin_session";
+
export async function GET(
- _req: NextRequest,
+ req: NextRequest,
{ params }: { params: Promise<{ id: string }> },
) {
+ const auth = await verifyRequestAdminSession(
+ req.cookies.get(SESSION_COOKIE)?.value,
+ );
+ if (!auth.valid) {
+ return NextResponse.json(
+ { error: "Authentication required." },
+ { status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
+ );
+ }
+
const { id } = await params;
const client = await SDKManager.getClientAsync(id);
if (!client) {
return NextResponse.json({ error: "Client not found" }, { status: 404 });
}
const { secret, token, ...clientSummary } = client;
- return NextResponse.json({ client: clientSummary });
+ void secret;
+ const includeToken = req.nextUrl.searchParams.get("includeToken") === "true";
+ return NextResponse.json({
+ client: {
+ ...clientSummary,
+ tokenPreview: token ? `${token.slice(0, 12)}...` : "",
+ ...(includeToken ? { token } : {}),
+ },
+ });
}
export async function PATCH(
req: NextRequest,
{ params }: { params: Promise<{ id: string }> },
) {
+ const auth = await verifyRequestAdminSession(
+ req.cookies.get(SESSION_COOKIE)?.value,
+ );
+ if (!auth.valid) {
+ return NextResponse.json(
+ { error: "Authentication required." },
+ { status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
+ );
+ }
+
const { id } = await params;
try {
const body = await req.json();
@@ -46,11 +77,22 @@ export async function PATCH(
}
export async function DELETE(
- _req: NextRequest,
+ req: NextRequest,
{ params }: { params: Promise<{ id: string }> },
) {
+ const auth = await verifyRequestAdminSession(
+ req.cookies.get(SESSION_COOKIE)?.value,
+ );
+ if (!auth.valid) {
+ return NextResponse.json(
+ { error: "Authentication required." },
+ { status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
+ );
+ }
+
const { id } = await params;
- const deleted = await SDKManager.deleteClientAsync(id);
+ const purgeFiles = req.nextUrl.searchParams.get("purgeFiles") === "true";
+ const deleted = await SDKManager.deleteClientAsync(id, { purgeFiles });
if (!deleted) {
return NextResponse.json({ error: "Client not found" }, { status: 404 });
}
diff --git a/app/api/clients/route.ts b/app/api/clients/route.ts
index 2b7250f..000932f 100644
--- a/app/api/clients/route.ts
+++ b/app/api/clients/route.ts
@@ -1,12 +1,28 @@
+import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
import { SDKManager } from "@/core/sdk/sdkManager";
import { NextRequest, NextResponse } from "next/server";
-export async function GET() {
+const SESSION_COOKIE = "datastack_admin_session";
+
+export async function GET(req: NextRequest) {
+ const auth = await verifyRequestAdminSession(
+ req.cookies.get(SESSION_COOKIE)?.value,
+ );
+ if (!auth.valid) {
+ return NextResponse.json(
+ { error: "Authentication required." },
+ { status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
+ );
+ }
+
try {
const allClients = await SDKManager.listClientsAsync();
- const clients = allClients.map(({ secret, ...client }) => {
+ const clients = allClients.map(({ secret, token, ...client }) => {
void secret;
- return client;
+ return {
+ ...client,
+ tokenPreview: token ? `${token.slice(0, 12)}...` : "",
+ };
});
return NextResponse.json({ clients, count: clients.length });
} catch (err: unknown) {
@@ -17,8 +33,31 @@ export async function GET() {
}
export async function POST(req: NextRequest) {
+ const auth = await verifyRequestAdminSession(
+ req.cookies.get(SESSION_COOKIE)?.value,
+ );
+ if (!auth.valid) {
+ return NextResponse.json(
+ { error: "Authentication required." },
+ { status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
+ );
+ }
+
+ let body: Record;
+ try {
+ const parsed = await req.json();
+ body =
+ parsed && typeof parsed === "object" && !Array.isArray(parsed)
+ ? (parsed as Record)
+ : {};
+ } catch {
+ return NextResponse.json(
+ { error: "Invalid request payload." },
+ { status: 400 },
+ );
+ }
+
try {
- const body = await req.json();
const { clientId, name, rateLimit } = body;
if (!clientId || !name) {
diff --git a/app/api/data/route.ts b/app/api/data/route.ts
index bdb6f32..934c437 100644
--- a/app/api/data/route.ts
+++ b/app/api/data/route.ts
@@ -1,4 +1,6 @@
+import { SettingsManager } from "@/core/adminSettings/settingsManager";
import { DataBufferEngine } from "@/core/data/buffer";
+import { getInternalIngestionHeaders } from "@/core/Security/security";
import { CSVStorageManager } from "@/core/Storage/csvStorage";
import { parse } from "csv-parse";
import { NextRequest, NextResponse } from "next/server";
@@ -31,18 +33,25 @@ export async function GET(request: NextRequest) {
let total = 0;
const addIfMatching = (event: DataEvent) => {
- const eventDate = event.timestamp.slice(0, 10);
+ const eventDate = CSVStorageManager.extractSafePartitionDate(event.timestamp);
+ const dedupeKey = `${event.clientId}:${event.eventId}`;
+ const matchesSearch =
+ !search ||
+ event.payload.toLowerCase().includes(search) ||
+ event.eventId.toLowerCase().includes(search) ||
+ event.clientId.toLowerCase().includes(search);
+
if (
(clientId !== "all" && event.clientId !== clientId) ||
(from && eventDate < from) ||
(to && eventDate > to) ||
- (search && !event.payload.toLowerCase().includes(search)) ||
- eventIds.has(event.eventId)
+ !matchesSearch ||
+ eventIds.has(dedupeKey)
) {
return;
}
- eventIds.add(event.eventId);
+ eventIds.add(dedupeKey);
total += 1;
let low = 0;
@@ -109,7 +118,16 @@ export async function GET(request: NextRequest) {
}
}
- for (const event of DataBufferEngine.getRecentEvents(200)) {
+ const settings = await SettingsManager.getSettingsAsync();
+ const recentWindowLimit = Math.min(
+ 2000,
+ Math.max(200, settings.batchFlushThreshold || 500),
+ );
+
+ for (const event of await DataBufferEngine.getRecentEventsAsync(
+ recentWindowLimit,
+ clientId,
+ )) {
addIfMatching({ ...event, status: "buffered" });
}
@@ -117,10 +135,18 @@ export async function GET(request: NextRequest) {
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, "");
try {
- const res = await fetch(`${ingestionBaseUrl}/internal/recent?limit=200`, {
- cache: "no-store",
- signal: AbortSignal.timeout(1000),
- });
+ const qs = new URLSearchParams({ limit: String(recentWindowLimit) });
+ if (clientId && clientId !== "all") {
+ qs.set("clientId", clientId);
+ }
+ const res = await fetch(
+ `${ingestionBaseUrl}/internal/recent?${qs.toString()}`,
+ {
+ headers: getInternalIngestionHeaders(),
+ cache: "no-store",
+ signal: AbortSignal.timeout(1000),
+ },
+ );
if (res.ok) {
const data = await res.json();
if (Array.isArray(data.events)) {
diff --git a/app/api/events/route.ts b/app/api/events/route.ts
index a93d99f..bf607f1 100644
--- a/app/api/events/route.ts
+++ b/app/api/events/route.ts
@@ -1,9 +1,12 @@
import { SettingsManager } from '@/core/adminSettings/settingsManager';
-import { verifyAdminSessionToken } from '@/core/auth/adminSession';
+import { verifyRequestAdminSession } from '@/core/auth/adminUsers';
import { AuthService } from '@/core/auth/jwt';
import { DataBufferEngine } from '@/core/data/buffer';
import { SDKManager } from '@/core/sdk/sdkManager';
-import { SecurityEngine } from '@/core/Security/security';
+import {
+ SecurityEngine,
+ getInternalIngestionHeaders,
+} from '@/core/Security/security';
import { CSVStorageManager, type DataEvent } from '@/core/Storage/csvStorage';
import { parse } from 'csv-parse';
import { NextRequest, NextResponse } from 'next/server';
@@ -15,14 +18,15 @@ async function getRecentStoredEvents(
limit: number,
clientId?: string,
): Promise {
- const files = CSVStorageManager.listAllFiles()
+ const allFiles = await CSVStorageManager.listAllFilesAsync();
+ const files = allFiles
.filter((f) => !clientId || clientId === 'all' || f.clientId === clientId)
.slice(0, 5);
const collected: DataEvent[] = [];
for (const file of files) {
- const fileEvents: DataEvent[] = [];
+ const fileTail: DataEvent[] = [];
const parser = createReadStream(file.filePath).pipe(
parse({ columns: true, skip_empty_lines: true, bom: true }),
);
@@ -32,7 +36,10 @@ async function getRecentStoredEvents(
Record
>) {
if (!record.eventId) continue;
- fileEvents.push({
+ if (fileTail.length >= limit) {
+ fileTail.shift();
+ }
+ fileTail.push({
clientId: record.clientId || file.clientId,
eventId: record.eventId,
timestamp: record.timestamp || `${file.date}T00:00:00.000Z`,
@@ -43,8 +50,8 @@ async function getRecentStoredEvents(
parser.destroy();
}
- if (fileEvents.length > 0) {
- collected.push(...fileEvents.slice(-limit));
+ if (fileTail.length > 0) {
+ collected.push(...fileTail);
}
}
@@ -55,9 +62,10 @@ export async function GET(req: NextRequest) {
try {
const { searchParams } = new URL(req.url);
const clientId = searchParams.get('clientId') || undefined;
- const limit = Math.min(200, Math.max(1, Number(searchParams.get('limit')) || 50));
+ const limit = Math.min(2000, Math.max(1, Number(searchParams.get('limit')) || 50));
- const bufferedEvents = DataBufferEngine.getRecentEvents(limit, clientId);
+ await SettingsManager.getSettingsAsync();
+ const bufferedEvents = await DataBufferEngine.getRecentEventsAsync(limit, clientId);
let socketBufferedEvents: DataEvent[] = [];
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
@@ -67,6 +75,7 @@ export async function GET(req: NextRequest) {
const qs = new URLSearchParams({ limit: String(limit) });
if (clientId) qs.set('clientId', clientId);
const res = await fetch(`${ingestionBaseUrl}/internal/recent?${qs.toString()}`, {
+ headers: getInternalIngestionHeaders(),
cache: 'no-store',
signal: AbortSignal.timeout(1000),
});
@@ -86,8 +95,10 @@ export async function GET(req: NextRequest) {
const merged: DataEvent[] = [];
for (const evt of [...bufferedEvents, ...socketBufferedEvents, ...storedEvents]) {
- if (!evt.eventId || seenIds.has(evt.eventId)) continue;
- seenIds.add(evt.eventId);
+ if (!evt.eventId) continue;
+ const dedupeKey = `${evt.clientId}:${evt.eventId}`;
+ if (seenIds.has(dedupeKey)) continue;
+ seenIds.add(dedupeKey);
merged.push(evt);
}
@@ -107,9 +118,6 @@ export async function POST(req: NextRequest) {
// Verify either Client Bearer JWT or Admin Session Cookie
const authHeader = req.headers.get('authorization');
- const sessionSecret =
- process.env.DATASTACK_ADMIN_SESSION_SECRET ||
- 'datastack-admin-session-default-secret-32-chars';
const sessionCookie = req.cookies.get(SESSION_COOKIE)?.value;
let tokenClientId: string | null = null;
@@ -128,20 +136,30 @@ export async function POST(req: NextRequest) {
tokenClientId = verified.payload.clientId;
tokenRateLimit = verified.payload.rateLimit;
} else {
- const isAdmin = Boolean(
- sessionSecret &&
- sessionCookie &&
- (await verifyAdminSessionToken(sessionCookie, sessionSecret)),
- );
- if (!isAdmin) {
+ const auth = await verifyRequestAdminSession(sessionCookie);
+ if (!auth.valid) {
return NextResponse.json(
- { error: 'Authentication required. Provide a valid Client Bearer JWT or Admin Session.' },
- { status: 401 },
+ {
+ error:
+ auth.reason === 'UNAUTHORIZED'
+ ? 'Authentication required. Provide a valid Client Bearer JWT or Admin Session.'
+ : 'Admin authentication is temporarily unavailable.',
+ },
+ { status: auth.reason === 'UNAUTHORIZED' ? 401 : 503 },
);
}
}
- const body = await req.json();
+ let body: unknown;
+ try {
+ body = await req.json();
+ } catch {
+ return NextResponse.json(
+ { error: 'Invalid JSON request payload.', code: 'INVALID_JSON' },
+ { status: 400 },
+ );
+ }
+
const bodyObj =
body && typeof body === 'object' && !Array.isArray(body)
? (body as Record)
@@ -154,7 +172,27 @@ export async function POST(req: NextRequest) {
.toLowerCase()
.replace(/[^a-z0-9_-]/g, '_');
+ if (await SDKManager.isClientDeletedAsync(clientId)) {
+ return NextResponse.json(
+ {
+ error: `CLIENT_REVOKED: Client '${clientId}' has been deleted by administrator.`,
+ code: 'CLIENT_REVOKED',
+ },
+ { status: 403 },
+ );
+ }
+
const registeredClient = await SDKManager.getClientAsync(clientId);
+ if (rawBearerToken && !registeredClient) {
+ return NextResponse.json(
+ {
+ error: `CLIENT_REVOKED: Client '${clientId}' registration not found or has been deleted.`,
+ code: 'CLIENT_REVOKED',
+ },
+ { status: 403 },
+ );
+ }
+
if (registeredClient) {
if (registeredClient.status === 'revoked') {
return NextResponse.json(
@@ -196,6 +234,7 @@ export async function POST(req: NextRequest) {
: null;
if (batchItems !== null) {
+ const envelopeKeys = new Set(['eventId', 'timestamp', 'data', 'clientId']);
let processed = 0;
let rejected = 0;
let duplicates = 0;
@@ -205,6 +244,8 @@ export async function POST(req: NextRequest) {
const oversizedEventIds: string[] = [];
const errors: string[] = [];
+ const batchEvents: DataEvent[] = [];
+
for (const item of batchItems) {
const safeItem =
item && typeof item === 'object' && !Array.isArray(item)
@@ -221,8 +262,10 @@ export async function POST(req: NextRequest) {
? safeItem.timestamp.trim()
: new Date().toISOString();
- const itemData =
- safeItem.data !== undefined ? safeItem.data : safeItem;
+ const isEnvelope =
+ safeItem.data !== undefined &&
+ Object.keys(safeItem).every((k) => envelopeKeys.has(k));
+ const itemData = isEnvelope ? safeItem.data : safeItem;
const val = SecurityEngine.validateEvent(
clientId,
@@ -251,18 +294,20 @@ export async function POST(req: NextRequest) {
continue;
}
- const event: DataEvent = {
+ batchEvents.push({
clientId,
eventId: itemEventId,
timestamp: itemTimestamp,
payload:
typeof itemData === 'string' ? itemData : JSON.stringify(itemData),
- };
-
- DataBufferEngine.push(event);
+ });
processed += 1;
}
+ if (batchEvents.length > 0) {
+ await DataBufferEngine.pushBatchAsync(batchEvents);
+ }
+
if (processed > 0) {
SDKManager.incrementEventCount(clientId, processed);
}
@@ -347,7 +392,7 @@ export async function POST(req: NextRequest) {
payload: typeof data === 'string' ? data : JSON.stringify(data),
};
- DataBufferEngine.push(event);
+ await DataBufferEngine.pushAsync(event);
SDKManager.incrementEventCount(clientId, 1);
return NextResponse.json({ success: true, eventId, event });
diff --git a/app/api/export/route.ts b/app/api/export/route.ts
index 7d31b8e..27eff34 100644
--- a/app/api/export/route.ts
+++ b/app/api/export/route.ts
@@ -1,4 +1,6 @@
+import { SettingsManager } from "@/core/adminSettings/settingsManager";
import { DataBufferEngine } from "@/core/data/buffer";
+import { getInternalIngestionHeaders } from "@/core/Security/security";
import {
CSVStorageManager,
type StorageFileMeta,
@@ -6,6 +8,7 @@ import {
import { parse } from "csv-parse";
import { NextRequest, NextResponse } from "next/server";
import { createReadStream } from "node:fs";
+import path from "node:path";
type ExportFormat = "csv" | "json";
type CsvRecord = Record;
@@ -108,19 +111,26 @@ function streamFile(file: StorageFileMeta) {
export async function GET(request: NextRequest) {
try {
- await DataBufferEngine.flush();
- const ingestionBaseUrl =
- process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000";
- try {
- await fetch(`${ingestionBaseUrl}/internal/flush`, {
- method: "POST",
- signal: AbortSignal.timeout(1500),
- });
- } catch {
- // Port 4000 may not be running in isolated single-process dev mode
+ const { searchParams } = new URL(request.url);
+ const shouldFlush = searchParams.get("flush") !== "false";
+
+ await SettingsManager.getSettingsAsync();
+ if (shouldFlush) {
+ await DataBufferEngine.flush();
+ const ingestionBaseUrl = (
+ process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
+ ).replace(/\/$/, "");
+ try {
+ await fetch(`${ingestionBaseUrl}/internal/flush`, {
+ method: "POST",
+ headers: getInternalIngestionHeaders(),
+ signal: AbortSignal.timeout(1500),
+ });
+ } catch {
+ // Port 4000 may not be running in isolated single-process dev mode
+ }
}
- const { searchParams } = new URL(request.url);
const clientId = searchParams.get("clientId") || "all";
const filename = searchParams.get("filename");
const from = searchParams.get("from") || "";
@@ -129,17 +139,33 @@ export async function GET(request: NextRequest) {
searchParams.get("format") === "json" ? "json" : "csv";
const includeHeader = searchParams.get("includeHeader") !== "false";
- let files = CSVStorageManager.listAllFiles();
+ let files = await CSVStorageManager.listAllFilesAsync();
if (clientId !== "all")
files = files.filter((file) => file.clientId === clientId);
if (filename) {
- const file = files.find((candidate) => candidate.filename === filename);
- if (!file)
+ const safeFilename = path.basename(filename);
+ const matches = files.filter(
+ (candidate) => candidate.filename === safeFilename,
+ );
+ if (matches.length === 0) {
return NextResponse.json(
{ error: "CSV file not found" },
{ status: 404 },
);
+ }
+ if (matches.length > 1) {
+ return NextResponse.json(
+ {
+ error:
+ "Ambiguous filename matches multiple clients. Specify clientId.",
+ code: "AMBIGUOUS_FILENAME",
+ matchingClients: matches.map((m) => m.clientId),
+ },
+ { status: 400 },
+ );
+ }
+ const file = matches[0];
return new NextResponse(streamFile(file), {
status: 200,
headers: {
diff --git a/app/api/health/route.ts b/app/api/health/route.ts
index 8654d60..4359f46 100644
--- a/app/api/health/route.ts
+++ b/app/api/health/route.ts
@@ -23,7 +23,19 @@ export async function GET(req: NextRequest) {
);
}
- const clientId = verified.payload.clientId;
+ const clientId = String(verified.payload.clientId || '')
+ .trim()
+ .toLowerCase();
+ if (SDKManager.isClientDeleted(clientId)) {
+ return NextResponse.json(
+ {
+ status: 'error',
+ code: 'CLIENT_REVOKED',
+ error: `CLIENT_REVOKED: Client '${clientId}' has been deleted by administrator.`,
+ },
+ { status: 403 },
+ );
+ }
const registeredClient = await SDKManager.getClientAsync(clientId);
if (registeredClient) {
if (registeredClient.status === 'revoked') {
diff --git a/app/api/settings/route.ts b/app/api/settings/route.ts
index 4cb6bbe..5d8c3b3 100644
--- a/app/api/settings/route.ts
+++ b/app/api/settings/route.ts
@@ -1,8 +1,55 @@
-import { SettingsManager } from "@/core/adminSettings/settingsManager";
+import {
+ SettingsManager,
+ type AdminSettings,
+} from "@/core/adminSettings/settingsManager";
+import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
import { NextRequest, NextResponse } from "next/server";
-export async function GET() {
+const SESSION_COOKIE = "datastack_admin_session";
+
+const ALLOWED_SETTING_KEYS = new Set([
+ "maxEventsPerSec",
+ "maxPayloadSizeKb",
+ "batchFlushThreshold",
+ "batchFlushIntervalMs",
+ "fileRotationMb",
+ "enableIdempotency",
+ "autoPruneDays",
+ "jwtSecret",
+]);
+
+function parseBoundedInt(
+ raw: unknown,
+ field: string,
+ min: number,
+ max: number,
+): number {
+ if (typeof raw !== "number" || !Number.isInteger(raw)) {
+ throw new Error(`${field} must be an integer between ${min} and ${max}.`);
+ }
+ if (raw < min || raw > max) {
+ throw new Error(`${field} must be between ${min} and ${max}.`);
+ }
+ return raw;
+}
+
+export async function GET(req: NextRequest) {
try {
+ const auth = await verifyRequestAdminSession(
+ req.cookies.get(SESSION_COOKIE)?.value,
+ );
+ if (!auth.valid) {
+ return NextResponse.json(
+ {
+ error:
+ auth.reason === "UNAUTHORIZED"
+ ? "Authentication required."
+ : "Admin authentication is temporarily unavailable.",
+ },
+ { status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
+ );
+ }
+
const settings = await SettingsManager.getPublicSettingsAsync();
return NextResponse.json({ settings });
} catch (err: unknown) {
@@ -13,8 +60,133 @@ export async function GET() {
export async function POST(req: NextRequest) {
try {
- const body = await req.json();
- await SettingsManager.updateSettingsAsync(body);
+ const auth = await verifyRequestAdminSession(
+ req.cookies.get(SESSION_COOKIE)?.value,
+ );
+ if (!auth.valid) {
+ return NextResponse.json(
+ {
+ error:
+ auth.reason === "UNAUTHORIZED"
+ ? "Authentication required."
+ : "Admin authentication is temporarily unavailable.",
+ },
+ { status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
+ );
+ }
+
+ let rawBody: unknown;
+ try {
+ rawBody = await req.json();
+ } catch {
+ return NextResponse.json(
+ { error: "Invalid JSON request payload.", code: "INVALID_JSON" },
+ { status: 400 },
+ );
+ }
+
+ if (!rawBody || typeof rawBody !== "object" || Array.isArray(rawBody)) {
+ return NextResponse.json(
+ {
+ error: "Settings payload must be a JSON object.",
+ code: "VALIDATION_ERROR",
+ },
+ { status: 400 },
+ );
+ }
+
+ const body = rawBody as Record;
+ const unknownKeys = Object.keys(body).filter(
+ (key) => !ALLOWED_SETTING_KEYS.has(key),
+ );
+ if (unknownKeys.length > 0) {
+ return NextResponse.json(
+ {
+ error: `Unsupported settings field(s): ${unknownKeys.join(", ")}`,
+ code: "VALIDATION_ERROR",
+ },
+ { status: 400 },
+ );
+ }
+
+ const validated: Partial = {};
+ try {
+ if (body.maxEventsPerSec !== undefined) {
+ validated.maxEventsPerSec = parseBoundedInt(
+ body.maxEventsPerSec,
+ "maxEventsPerSec",
+ 1,
+ 100000,
+ );
+ }
+ if (body.maxPayloadSizeKb !== undefined) {
+ validated.maxPayloadSizeKb = parseBoundedInt(
+ body.maxPayloadSizeKb,
+ "maxPayloadSizeKb",
+ 1,
+ 10240,
+ );
+ }
+ if (body.batchFlushThreshold !== undefined) {
+ validated.batchFlushThreshold = parseBoundedInt(
+ body.batchFlushThreshold,
+ "batchFlushThreshold",
+ 1,
+ 50000,
+ );
+ }
+ if (body.batchFlushIntervalMs !== undefined) {
+ validated.batchFlushIntervalMs = parseBoundedInt(
+ body.batchFlushIntervalMs,
+ "batchFlushIntervalMs",
+ 100,
+ 60000,
+ );
+ }
+ if (body.fileRotationMb !== undefined) {
+ validated.fileRotationMb = parseBoundedInt(
+ body.fileRotationMb,
+ "fileRotationMb",
+ 1,
+ 2048,
+ );
+ }
+ if (body.autoPruneDays !== undefined) {
+ validated.autoPruneDays = parseBoundedInt(
+ body.autoPruneDays,
+ "autoPruneDays",
+ 0,
+ 3650,
+ );
+ }
+ if (body.enableIdempotency !== undefined) {
+ if (typeof body.enableIdempotency !== "boolean") {
+ throw new Error("enableIdempotency must be a boolean.");
+ }
+ validated.enableIdempotency = body.enableIdempotency;
+ }
+ if (body.jwtSecret !== undefined) {
+ if (typeof body.jwtSecret !== "string" || body.jwtSecret.trim().length < 32) {
+ throw new Error(
+ "jwtSecret must be a string with at least 32 characters.",
+ );
+ }
+ validated.jwtSecret = body.jwtSecret.trim();
+ }
+ } catch (validationErr: unknown) {
+ return NextResponse.json(
+ {
+ error:
+ validationErr instanceof Error
+ ? validationErr.message
+ : "Invalid settings value.",
+ code: "VALIDATION_ERROR",
+ },
+ { status: 400 },
+ );
+ }
+
+ await SettingsManager.updateSettingsAsync(validated);
const settings = await SettingsManager.getPublicSettingsAsync();
return NextResponse.json({
settings,
diff --git a/app/api/storage/route.ts b/app/api/storage/route.ts
index 9f098ea..9739161 100644
--- a/app/api/storage/route.ts
+++ b/app/api/storage/route.ts
@@ -1,4 +1,6 @@
import { NextRequest, NextResponse } from 'next/server';
+import { SettingsManager } from '@/core/adminSettings/settingsManager';
+import { getInternalIngestionHeaders } from '@/core/Security/security';
import { CSVStorageManager } from '@/core/Storage/csvStorage';
import { DataBufferEngine } from '@/core/data/buffer';
@@ -7,8 +9,11 @@ export async function GET(req: NextRequest) {
const { searchParams } = new URL(req.url);
const clientId = searchParams.get('clientId') || undefined;
- const stats = CSVStorageManager.getStorageStats();
- let files = CSVStorageManager.listAllFiles();
+ const [stats, allFiles] = await Promise.all([
+ CSVStorageManager.getStorageStatsAsync(),
+ CSVStorageManager.listAllFilesAsync(),
+ ]);
+ let files = allFiles;
if (clientId && clientId !== 'all') {
files = files.filter((f) => f.clientId === clientId);
@@ -23,12 +28,17 @@ export async function GET(req: NextRequest) {
export async function POST(_req: NextRequest) {
try {
- // 1. Flush local Next.js DataBufferEngine
+ await SettingsManager.getSettingsAsync();
+
+ // 1. Flush local / shared Next.js DataBufferEngine
const localRes = await DataBufferEngine.flush();
// 2. Also flush Port 4000 Socket.IO ingestion buffer
let remoteCount = 0;
let remoteFiles: string[] = [];
+ let ingestionFlushed = false;
+ let warning: string | undefined;
+ const explicitIngestionUrl = Boolean(process.env.INGESTION_INTERNAL_URL?.trim());
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
).replace(/\/$/, '');
@@ -36,17 +46,53 @@ export async function POST(_req: NextRequest) {
try {
const ingestRes = await fetch(`${ingestionBaseUrl}/internal/flush`, {
method: 'POST',
+ headers: getInternalIngestionHeaders(),
cache: 'no-store',
+ signal: AbortSignal.timeout(2500),
});
- if (ingestRes.ok) {
- const ingestData = await ingestRes.json();
- remoteCount = Number(ingestData.flushedCount) || 0;
- if (Array.isArray(ingestData.filesUpdated)) {
- remoteFiles = ingestData.filesUpdated;
- }
+ if (!ingestRes.ok) {
+ const errPayload = await ingestRes.json().catch(() => ({}));
+ const remoteErr =
+ typeof errPayload?.error === 'string'
+ ? errPayload.error
+ : `Ingestion flush returned HTTP ${ingestRes.status}`;
+ return NextResponse.json(
+ {
+ success: false,
+ error: remoteErr,
+ code: 'INGESTION_FLUSH_FAILED',
+ flushedCount: localRes.count,
+ filesUpdated: localRes.files,
+ ingestionFlushed: false,
+ ingestionStatus: ingestRes.status,
+ },
+ { status: 502 },
+ );
}
- } catch {
- // Port 4000 server may be offline in standalone Next.js dev mode
+
+ ingestionFlushed = true;
+ const ingestData = await ingestRes.json();
+ remoteCount = Number(ingestData.flushedCount) || 0;
+ if (Array.isArray(ingestData.filesUpdated)) {
+ remoteFiles = ingestData.filesUpdated;
+ }
+ } catch (err: unknown) {
+ const errMsg =
+ err instanceof Error ? err.message : 'Ingestion server unreachable';
+ if (explicitIngestionUrl) {
+ return NextResponse.json(
+ {
+ success: false,
+ error: `Ingestion server flush failed: ${errMsg}`,
+ code: 'INGESTION_UNREACHABLE',
+ flushedCount: localRes.count,
+ filesUpdated: localRes.files,
+ ingestionFlushed: false,
+ },
+ { status: 502 },
+ );
+ }
+ warning = `Port 4000 ingestion server is unreachable (${errMsg}); flushed Next.js buffer only.`;
}
const combinedFiles = Array.from(
@@ -57,6 +103,8 @@ export async function POST(_req: NextRequest) {
success: true,
flushedCount: localRes.count + remoteCount,
filesUpdated: combinedFiles,
+ ingestionFlushed,
+ ...(warning ? { warning } : {}),
});
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Manual flush failed';
diff --git a/app/error.tsx b/app/error.tsx
new file mode 100644
index 0000000..0273cb2
--- /dev/null
+++ b/app/error.tsx
@@ -0,0 +1,50 @@
+'use client';
+
+import { Button } from '@/components/ui/Button';
+import { AlertTriangle, RotateCcw } from 'lucide-react';
+import Link from 'next/link';
+import { useEffect } from 'react';
+
+export default function RootErrorBoundary({
+ error,
+ reset,
+}: {
+ error: Error & { digest?: string };
+ reset: () => void;
+}) {
+ useEffect(() => {
+ console.error('Application error boundary:', error);
+ }, [error]);
+
+ return (
+
+
+
+
+ An unexpected rendering error occurred. You can retry the view or return to the overview page.
+
+ {error?.message && (
+
+ {error.message}
+
+ )}
+
+
+
+ Return home
+
+
+ reset()}>
+
+ Try again
+
+
+
+
+ );
+}
diff --git a/app/layout.tsx b/app/layout.tsx
index 37377a6..b0c4e73 100644
--- a/app/layout.tsx
+++ b/app/layout.tsx
@@ -1,11 +1,31 @@
+import { PwaLifecycle } from '@/components/pwa/PwaLifecycle';
import { ToastProvider } from '@/components/ui/Toast';
-import type { Metadata } from 'next';
+import type { Metadata, Viewport } from 'next';
import './globals.css';
+export const viewport: Viewport = {
+ themeColor: '#0a0a0b',
+ width: 'device-width',
+ initialScale: 1,
+};
+
export const metadata: Metadata = {
title: 'DataStack - Realtime Data Collection Platform',
description: 'Self-hosted universal realtime data collection engine with Socket.IO ingestion, Redis buffer, JWT client auth, CSV batch rotation, and /vcon/ admin panel.',
manifest: '/manifest.json',
+ appleWebApp: {
+ capable: true,
+ statusBarStyle: 'black-translucent',
+ title: 'DataStack',
+ },
+ icons: {
+ icon: [
+ { url: '/icon.svg', type: 'image/svg+xml' },
+ { url: '/icon-192.png', sizes: '192x192', type: 'image/png' },
+ { url: '/icon-512.png', sizes: '512x512', type: 'image/png' },
+ ],
+ apple: [{ url: '/apple-touch-icon.png', sizes: '180x180', type: 'image/png' }],
+ },
openGraph: {
title: 'DataStack - Realtime Data Collection Platform',
description: 'Self-hosted universal realtime data collection engine with Socket.IO ingestion, Redis buffer, JWT client auth, CSV batch rotation, and /vcon/ admin panel.',
@@ -21,12 +41,11 @@ export const metadata: Metadata = {
export default function RootLayout({ children }: { children: React.ReactNode }) {
return (
-
-
-
-
- {children}
+
+ {children}
+
+
);
diff --git a/app/not-found.tsx b/app/not-found.tsx
index 01fe544..3ce7075 100644
--- a/app/not-found.tsx
+++ b/app/not-found.tsx
@@ -1,5 +1,5 @@
import { MainLayout } from '@/components/layout/MainLayout';
-import { Button } from '@/components/ui/Button';
+import { buttonClassNames } from '@/components/ui/Button';
import Link from 'next/link';
export default function NotFound() {
@@ -11,10 +11,11 @@ export default function NotFound() {
The requested page or route does not exist on DataStack.
-
-
- Return Home
-
+
+ Return Home
+
+
+ Open Console
diff --git a/app/vcon/clients/page.tsx b/app/vcon/clients/page.tsx
index bac8ae2..da7b501 100644
--- a/app/vcon/clients/page.tsx
+++ b/app/vcon/clients/page.tsx
@@ -23,7 +23,11 @@ export default function ClientsPage() {
const [now, setNow] = useState(() => Date.now());
const fetchClients = useCallback(async () => {
- const response = await fetch('/api/clients');
+ const response = await fetch('/api/clients', { cache: 'no-store' });
+ if (response.status === 401) {
+ window.location.assign('/vcon/login');
+ return;
+ }
if (!response.ok) throw new Error('Unable to load clients');
const data = await response.json();
setClients(setCached('vcon:clients', data.clients || []));
@@ -49,16 +53,19 @@ export default function ClientsPage() {
!query ||
client.clientId.toLowerCase().includes(query) ||
client.name.toLowerCase().includes(query);
+ const revoked = client.status === 'revoked';
const online =
client.status === 'active' &&
Boolean(client.lastActive) &&
now - new Date(client.lastActive!).getTime() < 60_000;
const pending = client.status === 'active' && !client.lastActive;
+ const offline = client.status === 'active' && !online && !pending;
const matchesStatus =
statusFilter === 'all' ||
(statusFilter === 'online' && online) ||
- (statusFilter === 'offline' && !online && !pending) ||
- (statusFilter === 'pending' && pending);
+ (statusFilter === 'offline' && offline) ||
+ (statusFilter === 'pending' && pending) ||
+ (statusFilter === 'revoked' && revoked);
return matchesSearch && matchesStatus;
});
}, [clients, now, search, statusFilter]);
@@ -72,7 +79,10 @@ export default function ClientsPage() {
const pendingCount = clients.filter(
(client) => client.status === 'active' && !client.lastActive
).length;
- const offlineCount = clients.length - onlineCount - pendingCount;
+ const revokedCount = clients.filter(
+ (client) => client.status === 'revoked'
+ ).length;
+ const offlineCount = clients.length - onlineCount - pendingCount - revokedCount;
return (
@@ -109,6 +121,7 @@ export default function ClientsPage() {
online: onlineCount,
offline: offlineCount,
pending: pendingCount,
+ revoked: revokedCount,
}}
/>
void;
+}) {
+ useEffect(() => {
+ console.error('Admin console error boundary:', error);
+ }, [error]);
+
+ return (
+
+
+
+
+
+ Console panel error
+
+
+
+ This console view encountered an unexpected failure. Retry rendering or navigate back to the dashboard.
+
+ {error?.message && (
+
+ {error.message}
+
+ )}
+
+
+
+ Dashboard
+
+
+ reset()}>
+
+ Retry panel
+
+
+
+
+ );
+}
diff --git a/app/vcon/login/page.tsx b/app/vcon/login/page.tsx
index 30578fd..3d14267 100644
--- a/app/vcon/login/page.tsx
+++ b/app/vcon/login/page.tsx
@@ -6,12 +6,13 @@ export const runtime = 'nodejs';
export const dynamic = 'force-dynamic';
export default async function LoginPage() {
- if (process.env.DATABASE_URL) {
- const isConfigured = await isAdminSetupComplete().catch(() => false);
- if (!isConfigured) redirect('/vcon/setup');
- }
+ const isConfigured = await isAdminSetupComplete().catch(() => false);
+ if (!isConfigured) redirect('/vcon/setup');
- const showDemoHint = !process.env.DATABASE_URL && process.env.NODE_ENV !== 'production';
+ const showDemoHint =
+ !process.env.DATABASE_URL &&
+ process.env.NODE_ENV !== 'production' &&
+ process.env.DATASTACK_ALLOW_DEMO_ADMIN !== 'false';
return ;
}
diff --git a/app/vcon/sdk/page.tsx b/app/vcon/sdk/page.tsx
index dce2a0c..f580043 100644
--- a/app/vcon/sdk/page.tsx
+++ b/app/vcon/sdk/page.tsx
@@ -11,15 +11,25 @@ import { useToast } from '@/components/ui/Toast';
import { getCached, setCached } from '@/lib/vconCache';
import { useEffect, useState } from 'react';
+interface SdkClientItem {
+ clientId: string;
+ name: string;
+ status?: 'active' | 'revoked';
+ tokenPreview?: string;
+ token?: string;
+}
+
export default function SdkGuidePage() {
- const { success } = useToast();
- const [clients, setClients] = useState(() =>
- getCached('vcon:clients', [])
+ const { success, error } = useToast();
+ const [clients, setClients] = useState(() =>
+ getCached('vcon:clients', [])
);
const [selectedClientId, setSelectedClientId] = useState(() => {
- const cached = getCached('vcon:clients', []);
- return cached[0]?.clientId ?? '';
+ const cached = getCached('vcon:clients', []);
+ return cached.find((c) => c.status !== 'revoked')?.clientId ?? cached[0]?.clientId ?? '';
});
+ const [selectedClientToken, setSelectedClientToken] = useState('');
+ const [loadingToken, setLoadingToken] = useState(false);
const [serverUrl, setServerUrl] = useState('http://localhost:4000');
const [apiUrl, setApiUrl] = useState('http://localhost:3000');
@@ -34,33 +44,81 @@ export default function SdkGuidePage() {
async function loadClients() {
try {
- const res = await fetch('/api/clients');
+ const res = await fetch('/api/clients', { cache: 'no-store' });
+ if (res.status === 401) {
+ window.location.assign('/vcon/login');
+ return;
+ }
if (res.ok) {
const data = await res.json();
- const list = setCached('vcon:clients', data.clients || []);
+ const list = setCached('vcon:clients', data.clients || []);
setClients(list);
if (list.length > 0) {
setSelectedClientId((prev) =>
- list.some((c: any) => c.clientId === prev) ? prev : list[0].clientId
+ list.some((c) => c.clientId === prev)
+ ? prev
+ : list.find((c) => c.status !== 'revoked')?.clientId ?? list[0].clientId
);
}
}
} catch {
- // ignore
+ // Retain cached list on transient network error
}
}
- loadClients();
+ void loadClients();
}, []);
+ useEffect(() => {
+ if (!selectedClientId) {
+ setSelectedClientToken('');
+ return;
+ }
+
+ let cancelled = false;
+ setLoadingToken(true);
+
+ async function fetchToken() {
+ try {
+ const res = await fetch(
+ `/api/clients/${encodeURIComponent(selectedClientId)}?includeToken=true`,
+ { cache: 'no-store' }
+ );
+ if (res.status === 401) {
+ window.location.assign('/vcon/login');
+ return;
+ }
+ if (res.ok) {
+ const data = await res.json();
+ if (!cancelled) {
+ setSelectedClientToken(String(data?.client?.token || ''));
+ }
+ } else if (!cancelled) {
+ setSelectedClientToken('');
+ }
+ } catch {
+ if (!cancelled) setSelectedClientToken('');
+ } finally {
+ if (!cancelled) setLoadingToken(false);
+ }
+ }
+
+ void fetchToken();
+ return () => {
+ cancelled = true;
+ };
+ }, [selectedClientId]);
+
const activeClient =
clients.find((c) => c.clientId === selectedClientId) || clients[0];
- const sampleToken = activeClient?.token || 'YOUR_SIGNED_CLIENT_JWT_TOKEN';
+ const resolvedToken = selectedClientToken || activeClient?.token || '';
+ const sampleToken = resolvedToken || 'YOUR_SIGNED_CLIENT_JWT_TOKEN';
const configSnippet = `# config.datastack.py — Place in your Python project root
DATASTACK_CONFIG = {
"SERVER_URL": "${serverUrl}",
"API_URL": "${apiUrl}",
"CLIENT_TOKEN": "${sampleToken}",
+ "CLIENT_ID": "${activeClient?.clientId || ''}",
"AUTO_CONNECT": True,
"ENABLE_HTTP_FALLBACK": True,
"HEARTBEAT_INTERVAL_SEC": 5.0,
@@ -99,9 +157,21 @@ print("SDK Status:", client.status())
client.disconnect()`;
- const copySnippet = (text: string, label: string) => {
- navigator.clipboard.writeText(text);
- success('Copied', `${label} copied`);
+ const copySnippet = async (text: string, label: string) => {
+ try {
+ if (!navigator?.clipboard?.writeText) {
+ throw new Error('Clipboard API unavailable in this browser context');
+ }
+ await navigator.clipboard.writeText(text);
+ success('Copied', `${label} copied`);
+ } catch (copyErr) {
+ error(
+ 'Copy failed',
+ copyErr instanceof Error
+ ? copyErr.message
+ : 'Unable to copy to clipboard. Copy manually from the code block.'
+ );
+ }
};
return (
@@ -109,12 +179,13 @@ client.disconnect()`;
title="Python SDK"
actions={
<>
- {activeClient?.token && (
+ {resolvedToken && (
copySnippet(activeClient.token, 'Client JWT token')}
+ disabled={loadingToken}
+ onClick={() => void copySnippet(resolvedToken, 'Client JWT token')}
>
Copy Token
@@ -123,7 +194,7 @@ client.disconnect()`;
size="xs"
variant="secondary"
icon="FileCode"
- onClick={() => copySnippet(configSnippet, 'config.datastack.py')}
+ onClick={() => void copySnippet(configSnippet, 'config.datastack.py')}
>
Copy Config
@@ -131,7 +202,7 @@ client.disconnect()`;
size="xs"
variant="secondary"
icon="Copy"
- onClick={() => copySnippet(pythonSnippet, 'Python SDK snippet')}
+ onClick={() => void copySnippet(pythonSnippet, 'Python SDK snippet')}
>
Copy Code
@@ -161,8 +232,9 @@ client.disconnect()`;
size="xs"
variant="ghost"
icon="Copy"
+ aria-label="Copy pip install command"
onClick={() =>
- copySnippet('pip install ./sdk/python/datastack', 'Pip command')
+ void copySnippet('pip install ./sdk/python/datastack', 'Pip command')
}
/>
@@ -176,6 +248,7 @@ client.disconnect()`;
setSelectedClientId(e.target.value)}
>
@@ -191,7 +264,8 @@ client.disconnect()`;
size="xs"
variant="ghost"
icon="Copy"
- onClick={() => copySnippet(configSnippet, 'config.datastack.py')}
+ aria-label="Copy config.datastack.py"
+ onClick={() => void copySnippet(configSnippet, 'config.datastack.py')}
/>
}
@@ -207,7 +281,8 @@ client.disconnect()`;
size="xs"
variant="ghost"
icon="Copy"
- onClick={() => copySnippet(pythonSnippet, 'Python SDK snippet')}
+ aria-label="Copy Python SDK snippet"
+ onClick={() => void copySnippet(pythonSnippet, 'Python SDK snippet')}
/>
}
>
diff --git a/app/vcon/simulator/page.tsx b/app/vcon/simulator/page.tsx
index c789c80..8d0efa9 100644
--- a/app/vcon/simulator/page.tsx
+++ b/app/vcon/simulator/page.tsx
@@ -5,20 +5,40 @@ import { Button } from '@/components/ui/Button';
import { CodeBlock } from '@/components/ui/CodeBlock';
import { Select, Textarea } from '@/components/ui/Input';
import { Panel } from '@/components/ui/Panel';
+import { Pill } from '@/components/ui/Pill';
import { StatStrip } from '@/components/ui/StatStrip';
import { useToast } from '@/components/ui/Toast';
import { getCached, setCached } from '@/lib/vconCache';
+import Link from 'next/link';
import { useEffect, useState } from 'react';
+import { io } from 'socket.io-client';
+
+interface SimulatorClient {
+ clientId: string;
+ name: string;
+ status: 'active' | 'revoked';
+}
+
+type TransportMode = 'http' | 'websocket';
+type DispatchStatus = 'Ready' | 'Accepted' | 'Rejected' | 'Error';
+
+function getSocketOrigin(): string {
+ const isLocalDev = window.location.port === '3000';
+ const protocol = window.location.protocol === 'https:' ? 'wss' : 'ws';
+ const host = isLocalDev ? `${window.location.hostname}:4000` : window.location.host;
+ return `${protocol}://${host}`;
+}
export default function SimulatorPage() {
- const { success, error } = useToast();
- const [clients, setClients] = useState(() =>
- getCached('vcon:clients', [])
+ const { success, warning, error } = useToast();
+ const [clients, setClients] = useState(() =>
+ getCached('vcon:clients', [])
);
const [selectedClient, setSelectedClient] = useState(() => {
- const cached = getCached('vcon:clients', []);
- return cached[0]?.clientId ?? '';
+ const cached = getCached('vcon:clients', []);
+ return cached.find((c) => c.status === 'active')?.clientId ?? cached[0]?.clientId ?? '';
});
+ const [transport, setTransport] = useState('http');
const [payloadText, setPayloadText] = useState(
JSON.stringify(
{
@@ -33,30 +53,136 @@ export default function SimulatorPage() {
)
);
const [sending, setSending] = useState(false);
- const [lastResponse, setLastResponse] = useState(null);
+ const [dispatchStatus, setDispatchStatus] = useState('Ready');
+ const [lastResponse, setLastResponse] = useState | null>(null);
useEffect(() => {
async function loadClients() {
try {
- const res = await fetch('/api/clients');
+ const res = await fetch('/api/clients', { cache: 'no-store' });
+ if (res.status === 401) {
+ window.location.assign('/vcon/login');
+ return;
+ }
if (res.ok) {
const data = await res.json();
- const list = setCached('vcon:clients', data.clients || []);
+ const list = setCached('vcon:clients', data.clients || []);
setClients(list);
- if (list.length > 0)
- setSelectedClient((prev) => prev || list[0].clientId);
+ setSelectedClient((prev) => {
+ if (prev && list.some((c) => c.clientId === prev)) return prev;
+ return list.find((c) => c.status === 'active')?.clientId ?? list[0]?.clientId ?? '';
+ });
}
} catch {
- // ignore
+ // Ignore transient failure and retain cached list
}
}
- loadClients();
+ void loadClients();
}, []);
+ const fetchClientJwt = async (clientId: string): Promise => {
+ const res = await fetch(`/api/clients/${encodeURIComponent(clientId)}?includeToken=true`, {
+ cache: 'no-store',
+ });
+ if (res.status === 401) {
+ window.location.assign('/vcon/login');
+ throw new Error('Authentication required');
+ }
+ const data = await res.json();
+ if (!res.ok || !data?.client?.token) {
+ throw new Error(data?.error || 'Unable to retrieve client JWT token');
+ }
+ return String(data.client.token);
+ };
+
+ const emitSocketEvents = async (
+ clientId: string,
+ items: Array>
+ ): Promise<{ accepted: number; rejected: number; responses: Record[] }> => {
+ const token = await fetchClientJwt(clientId);
+ const socketUrl = getSocketOrigin();
+
+ return new Promise((resolve, reject) => {
+ const socket = io(socketUrl, {
+ auth: { token },
+ transports: ['websocket', 'polling'],
+ reconnection: false,
+ timeout: 4000,
+ });
+
+ const timer = window.setTimeout(() => {
+ socket.disconnect();
+ reject(new Error('WebSocket handshake timed out (is Port 4000 ingestion server running?)'));
+ }, 4500);
+
+ socket.on('connect_error', (connectErr) => {
+ window.clearTimeout(timer);
+ socket.disconnect();
+ reject(new Error(connectErr.message || 'WebSocket authentication failed'));
+ });
+
+ socket.on('connect', () => {
+ const responses: Record[] = [];
+ let accepted = 0;
+ let rejected = 0;
+ let settled = 0;
+
+ const finishIfDone = () => {
+ if (settled >= items.length) {
+ window.clearTimeout(timer);
+ socket.disconnect();
+ resolve({ accepted, rejected, responses });
+ }
+ };
+
+ const onAck = (ack: Record) => {
+ responses.push(ack);
+ if (ack?.status === 'ok' || ack?.status === 'duplicate') {
+ accepted++;
+ } else {
+ rejected++;
+ }
+ settled++;
+ finishIfDone();
+ };
+
+ const onErrorEvent = (errPayload: Record) => {
+ responses.push(errPayload);
+ rejected++;
+ settled++;
+ finishIfDone();
+ };
+
+ socket.on('ack', onAck);
+ socket.on('error_event', onErrorEvent);
+
+ items.forEach((data, index) => {
+ const eventId = `sim_ws_${Date.now()}_${index}`;
+ socket.emit('telemetry_event', {
+ clientId,
+ eventId,
+ timestamp: new Date().toISOString(),
+ data,
+ });
+ });
+ });
+ });
+ };
+
const handleSendSingle = async () => {
- let parsedData: any;
+ if (!selectedClient) {
+ error('No target client', 'Register a client in /vcon/clients first.');
+ return;
+ }
+
+ let parsedData: Record;
try {
- parsedData = JSON.parse(payloadText);
+ const raw = JSON.parse(payloadText);
+ if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
+ error('Invalid payload', 'Payload must be a JSON object.');
+ return;
+ }
+ parsedData = raw as Record;
} catch {
error('Invalid JSON', 'Fix payload syntax');
return;
@@ -64,54 +190,145 @@ export default function SimulatorPage() {
setSending(true);
try {
+ if (transport === 'websocket') {
+ const result = await emitSocketEvents(selectedClient, [parsedData]);
+ const first = result.responses[0] || {};
+ setLastResponse({ transport: 'websocket', ...first });
+ if (result.accepted > 0) {
+ setDispatchStatus('Accepted');
+ success('WebSocket Event Accepted', `${String(first.eventId || selectedClient)} acknowledged`);
+ } else {
+ setDispatchStatus('Rejected');
+ error('WebSocket Event Rejected', String(first.message || first.error || 'Rejected by Port 4000'));
+ }
+ return;
+ }
+
const res = await fetch('/api/events', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
- clientId: selectedClient || 'py_telemetry_agent_01',
+ clientId: selectedClient,
data: parsedData,
}),
});
+ if (res.status === 401) {
+ window.location.assign('/vcon/login');
+ return;
+ }
const data = await res.json();
- setLastResponse(data);
+ setLastResponse({ transport: 'http', status: res.status, ...data });
if (res.ok) {
- success('Event Accepted', `${data.event.eventId} buffered`);
+ setDispatchStatus('Accepted');
+ success('Event Accepted', `${data.event?.eventId || selectedClient} buffered`);
} else {
+ setDispatchStatus('Rejected');
error('Event Rejected', data.error || 'Server rejected');
}
- } catch (err: any) {
- error('Error', err.message);
+ } catch (err: unknown) {
+ const message = err instanceof Error ? err.message : 'Dispatch error';
+ setDispatchStatus('Error');
+ setLastResponse({ transport, error: message });
+ error('Dispatch failed', message);
} finally {
setSending(false);
}
};
const handleSendBatch = async (count = 10) => {
+ if (!selectedClient) {
+ error('No target client', 'Register a client in /vcon/clients first.');
+ return;
+ }
+
+ const items = Array.from({ length: count }, (_, i) => ({
+ batch_index: i + 1,
+ sensor: `sensor_${(i % 3) + 1}`,
+ reading: +(Math.random() * 100).toFixed(2),
+ timestamp: new Date().toISOString(),
+ }));
+
setSending(true);
try {
+ if (transport === 'websocket') {
+ const result = await emitSocketEvents(selectedClient, items);
+ setLastResponse({
+ transport: 'websocket',
+ batchSent: count,
+ accepted: result.accepted,
+ rejected: result.rejected,
+ sample: result.responses.slice(0, 3),
+ });
+ if (result.accepted === 0) {
+ setDispatchStatus('Rejected');
+ error('WebSocket Batch Rejected', `0/${count} events ingested`);
+ } else if (result.accepted < count) {
+ setDispatchStatus('Rejected');
+ warning('WebSocket Batch Partial', `${result.accepted}/${count} ingested (${result.rejected} rejected)`);
+ } else {
+ setDispatchStatus('Accepted');
+ success('WebSocket Batch Streamed', `${result.accepted}/${count} ingested`);
+ }
+ return;
+ }
+
let accepted = 0;
- for (let i = 0; i < count; i++) {
+ let rejected = 0;
+ const errors: string[] = [];
+
+ for (const item of items) {
const res = await fetch('/api/events', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
- clientId: selectedClient || 'py_telemetry_agent_01',
- data: {
- batch_index: i + 1,
- sensor: `sensor_${(i % 3) + 1}`,
- reading: +(Math.random() * 100).toFixed(2),
- timestamp: new Date().toISOString(),
- },
+ clientId: selectedClient,
+ data: item,
}),
});
- if (res.ok) accepted++;
+ if (res.status === 401) {
+ window.location.assign('/vcon/login');
+ return;
+ }
+ if (res.ok) {
+ accepted++;
+ } else {
+ rejected++;
+ try {
+ const errJson = await res.json();
+ if (errJson?.error && errors.length < 3) {
+ errors.push(String(errJson.error));
+ }
+ } catch {
+ // ignore non-json error body
+ }
+ }
}
- success('Batch Streamed', `${accepted}/${count} ingested`);
- setLastResponse({ success: true, batchSent: count, accepted });
- } catch (err: any) {
- error('Batch Error', err.message);
+
+ setLastResponse({
+ transport: 'http',
+ batchSent: count,
+ accepted,
+ rejected,
+ ...(errors.length > 0 ? { errors } : {}),
+ });
+
+ if (accepted === 0) {
+ setDispatchStatus('Rejected');
+ error('Batch Rejected', `0/${count} ingested${errors[0] ? ` (${errors[0]})` : ''}`);
+ } else if (accepted < count) {
+ setDispatchStatus('Rejected');
+ warning('Batch Partial', `${accepted}/${count} ingested (${rejected} rejected)`);
+ } else {
+ setDispatchStatus('Accepted');
+ success('Batch Streamed', `${accepted}/${count} ingested`);
+ }
+ } catch (err: unknown) {
+ const message = err instanceof Error ? err.message : 'Batch dispatch error';
+ setDispatchStatus('Error');
+ setLastResponse({ transport, error: message });
+ error('Batch Error', message);
} finally {
setSending(false);
}
@@ -126,7 +343,8 @@ export default function SimulatorPage() {
size="xs"
variant="secondary"
loading={sending}
- onClick={() => handleSendBatch(10)}
+ disabled={sending || !selectedClient}
+ onClick={() => void handleSendBatch(10)}
>
Burst 10
@@ -135,44 +353,93 @@ export default function SimulatorPage() {
variant="primary"
icon="Send"
loading={sending}
- onClick={handleSendSingle}
+ disabled={sending || !selectedClient}
+ onClick={() => void handleSendSingle()}
>
Send Event
>
}
>
+ {clients.length === 0 && (
+
+ No registered SDK clients found. Register a client before dispatching events.
+
+ Go to Clients →
+
+
+ )}
+
+
+
+ Transport
+
+
+
setTransport('http')}
+ >
+ HTTP /api/events
+
+
setTransport('websocket')}
+ >
+ WebSocket Port 4000
+
+
+
+
setSelectedClient(e.target.value)}
>
- {clients.map((c) => (
-
- {c.clientId}
-
- ))}
+ {clients.length === 0 ? (
+ No registered clients
+ ) : (
+ clients.map((c) => (
+
+ {c.clientId} ({c.status})
+
+ ))
+ )}
- Features
- SDK
- About
+ Features
+ SDK
+ About
+
+
= ({
className="h-7 w-7 border border-[var(--border)] bg-[var(--bg-elevated)]"
/>
-
-
- Console
-
+
+ Console
diff --git a/components/pwa/PwaLifecycle.tsx b/components/pwa/PwaLifecycle.tsx
new file mode 100644
index 0000000..0a46866
--- /dev/null
+++ b/components/pwa/PwaLifecycle.tsx
@@ -0,0 +1,175 @@
+'use client';
+
+import { Button } from '@/components/ui/Button';
+import { Icon } from '@/components/ui/Icon';
+import React, { useEffect, useState } from 'react';
+
+interface BeforeInstallPromptEvent extends Event {
+ prompt: () => Promise;
+ userChoice: Promise<{ outcome: 'accepted' | 'dismissed'; platform: string }>;
+}
+
+export function usePWAInstall() {
+ const [deferredPrompt, setDeferredPrompt] = useState(null);
+ const [isInstalled, setIsInstalled] = useState(false);
+ const [isIOS, setIsIOS] = useState(false);
+
+ useEffect(() => {
+ const isStandalone =
+ window.matchMedia('(display-mode: standalone)').matches ||
+ (window.navigator as unknown as { standalone?: boolean }).standalone === true;
+ setIsInstalled(isStandalone);
+
+ const ua = window.navigator.userAgent.toLowerCase();
+ setIsIOS(/iphone|ipad|ipod/.test(ua));
+
+ const handleBeforeInstallPrompt = (e: Event) => {
+ e.preventDefault();
+ setDeferredPrompt(e as BeforeInstallPromptEvent);
+ };
+
+ const handleAppInstalled = () => {
+ setIsInstalled(true);
+ setDeferredPrompt(null);
+ };
+
+ window.addEventListener('beforeinstallprompt', handleBeforeInstallPrompt);
+ window.addEventListener('appinstalled', handleAppInstalled);
+
+ return () => {
+ window.removeEventListener('beforeinstallprompt', handleBeforeInstallPrompt);
+ window.removeEventListener('appinstalled', handleAppInstalled);
+ };
+ }, []);
+
+ const install = async () => {
+ if (!deferredPrompt) return false;
+ await deferredPrompt.prompt();
+ const { outcome } = await deferredPrompt.userChoice;
+ if (outcome === 'accepted') {
+ setIsInstalled(true);
+ setDeferredPrompt(null);
+ return true;
+ }
+ return false;
+ };
+
+ return {
+ isInstallable: Boolean(deferredPrompt),
+ isInstalled,
+ isIOS,
+ install,
+ };
+}
+
+export const PWAInstallButton: React.FC = () => {
+ const { isInstallable, isInstalled, isIOS, install } = usePWAInstall();
+ const [showIOSGuide, setShowIOSGuide] = useState(false);
+
+ if (isInstalled) return null;
+
+ if (isInstallable) {
+ return (
+ void install()}
+ >
+ Install
+
+ );
+ }
+
+ if (isIOS) {
+ return (
+ <>
+ setShowIOSGuide(true)}
+ >
+ Install
+
+
+ {showIOSGuide && (
+
+
+
+
+ Install DataStack on iOS
+
+ setShowIOSGuide(false)}
+ className="text-[var(--text-muted)] hover:text-[var(--text)]"
+ >
+
+
+
+
+ 1. Tap the Share icon in Safari.
+
+ 2. Select Add to Home Screen .
+
+
+ setShowIOSGuide(false)}
+ >
+ Done
+
+
+
+
+ )}
+ >
+ );
+ }
+
+ return null;
+};
+
+export const PwaLifecycle: React.FC = () => {
+ const [isOnline, setIsOnline] = useState(true);
+
+ useEffect(() => {
+ setIsOnline(typeof navigator !== 'undefined' ? navigator.onLine : true);
+
+ const onOnline = () => setIsOnline(true);
+ const onOffline = () => setIsOnline(false);
+ window.addEventListener('online', onOnline);
+ window.addEventListener('offline', onOffline);
+
+ if (typeof navigator !== 'undefined' && 'serviceWorker' in navigator) {
+ navigator.serviceWorker.register('/sw.js', { scope: '/' }).catch(() => {
+ // Non-fatal in environments without SW origin permissions
+ });
+ }
+
+ return () => {
+ window.removeEventListener('online', onOnline);
+ window.removeEventListener('offline', onOffline);
+ };
+ }, []);
+
+ if (isOnline) return null;
+
+ return (
+
+
+ Offline mode — cached shell active
+
+ );
+};
+
+export default PwaLifecycle;
diff --git a/components/sections/AdminLogin.tsx b/components/sections/AdminLogin.tsx
index 1b614e1..b01d0e6 100644
--- a/components/sections/AdminLogin.tsx
+++ b/components/sections/AdminLogin.tsx
@@ -19,8 +19,8 @@ export function AdminLogin({
}: {
showDemoHint?: boolean;
}) {
- const [email, setEmail] = useState(showDemoHint ? 'admin@datastack.dev' : '');
- const [password, setPassword] = useState(showDemoHint ? 'admin12345678' : '');
+ const [email, setEmail] = useState('');
+ const [password, setPassword] = useState('');
const [remember, setRemember] = useState(true);
const [showPassword, setShowPassword] = useState(false);
const [isSubmitting, setIsSubmitting] = useState(false);
diff --git a/components/sections/ClientsFilters.tsx b/components/sections/ClientsFilters.tsx
index cb8d00d..813f00b 100644
--- a/components/sections/ClientsFilters.tsx
+++ b/components/sections/ClientsFilters.tsx
@@ -17,6 +17,7 @@ const FILTERS: { key: ClientStatusFilter; label: string }[] = [
{ key: 'online', label: 'Online' },
{ key: 'offline', label: 'Offline' },
{ key: 'pending', label: 'Pending' },
+ { key: 'revoked', label: 'Revoked' },
];
export function ClientsFilters({
diff --git a/components/sections/ClientsTable.tsx b/components/sections/ClientsTable.tsx
index 63e4f54..d36e198 100644
--- a/components/sections/ClientsTable.tsx
+++ b/components/sections/ClientsTable.tsx
@@ -2,7 +2,7 @@
import { Button } from '@/components/ui/Button';
import { CopyButton } from '@/components/ui/CopyButton';
-import { Textarea } from '@/components/ui/Input';
+import { Input, Textarea } from '@/components/ui/Input';
import { Modal } from '@/components/ui/Modal';
import {
Table,
@@ -43,8 +43,61 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
type: 'rotate' | 'revoke' | 'delete';
client: ClientItem;
} | null>(null);
+ const [editClient, setEditClient] = useState(null);
+ const [editName, setEditName] = useState('');
+ const [editRateLimit, setEditRateLimit] = useState('100');
+ const [editError, setEditError] = useState('');
const [processing, setProcessing] = useState(false);
+ const openEditModal = (client: ClientItem) => {
+ setEditClient(client);
+ setEditName(client.name);
+ setEditRateLimit(String(client.rateLimit || 100));
+ setEditError('');
+ };
+
+ const executeEditClient = async (event: React.FormEvent) => {
+ event.preventDefault();
+ if (!editClient) return;
+
+ const trimmedName = editName.trim();
+ const numericRateLimit = Number(editRateLimit);
+ if (!trimmedName) {
+ setEditError('Display name is required.');
+ return;
+ }
+ if (!Number.isFinite(numericRateLimit) || numericRateLimit < 1 || numericRateLimit > 10000) {
+ setEditError('Rate limit must be between 1 and 10,000 events/s.');
+ return;
+ }
+
+ setEditError('');
+ setProcessing(true);
+ try {
+ const response = await fetch(`/api/clients/${encodeURIComponent(editClient.clientId)}`, {
+ method: 'PATCH',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({
+ name: trimmedName,
+ rateLimit: Math.floor(numericRateLimit),
+ }),
+ });
+ if (response.status === 401) {
+ window.location.assign('/vcon/login');
+ return;
+ }
+ const data = await response.json();
+ if (!response.ok) throw new Error(data.error || 'Unable to update client');
+ success('Client updated', `${editClient.clientId} settings saved.`);
+ setEditClient(null);
+ await onStatusChange();
+ } catch (updateError) {
+ error('Client update failed', updateError instanceof Error ? updateError.message : 'Unknown error');
+ } finally {
+ setProcessing(false);
+ }
+ };
+
const executeRotateKey = async (client: ClientItem) => {
setProcessing(true);
try {
@@ -53,6 +106,10 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ action: 'regenerate_token' }),
});
+ if (response.status === 401) {
+ window.location.assign('/vcon/login');
+ return;
+ }
const data = await response.json();
if (!response.ok) throw new Error(data.error || 'Unable to rotate credentials');
setConfirmAction(null);
@@ -74,6 +131,10 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ status }),
});
+ if (response.status === 401) {
+ window.location.assign('/vcon/login');
+ return;
+ }
const data = await response.json();
if (!response.ok) throw new Error(data.error || 'Unable to update client status');
setConfirmAction(null);
@@ -92,6 +153,10 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
const response = await fetch(`/api/clients/${encodeURIComponent(client.clientId)}`, {
method: 'DELETE',
});
+ if (response.status === 401) {
+ window.location.assign('/vcon/login');
+ return;
+ }
const data = await response.json();
if (!response.ok) throw new Error(data.error || 'Unable to delete client');
setConfirmAction(null);
@@ -130,7 +195,7 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
Client
- API key
+ Credentials
Events
Last seen
Actions
@@ -141,15 +206,18 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
) : (
clients.map((client) => {
+ const revoked = client.status === 'revoked';
const online = client.status === 'active' && Boolean(client.lastActive) &&
now - new Date(client.lastActive!).getTime() < 60_000;
const pending = client.status === 'active' && !client.lastActive;
- const label = pending ? 'pending' : online ? 'online' : 'offline';
- const dotClass = online
- ? 'bg-[var(--accent)]'
- : pending
- ? 'bg-[var(--warning)]'
- : 'bg-[var(--text-faint)]';
+ const label = revoked ? 'revoked' : pending ? 'pending' : online ? 'online' : 'offline';
+ const dotClass = revoked
+ ? 'bg-[var(--danger)]'
+ : online
+ ? 'bg-[var(--accent)]'
+ : pending
+ ? 'bg-[var(--warning)]'
+ : 'bg-[var(--text-faint)]';
return (
@@ -157,13 +225,25 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
-
{client.name}
+
+ {client.name}
+ {revoked && (
+
+ revoked
+
+ )}
+
{client.clientId}
- sec_••••••••
+
+ {client.tokenPreview || 'JWT Active'}
+
+
+ {client.rateLimit}/s limit
+
{client.totalEvents.toLocaleString()}
@@ -184,6 +264,16 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
aria-label={`View events for ${client.clientId}`}
onClick={() => router.push(`/vcon/data?clientId=${encodeURIComponent(client.clientId)}`)}
/>
+ openEditModal(client)}
+ />
+ !processing && setEditClient(null)}
+ title={`Edit client: ${editClient?.clientId || ''}`}
+ maxWidth="xs"
+ footer={
+ <>
+ setEditClient(null)}>
+ Cancel
+
+
+ Save changes
+
+ >
+ }
+ >
+
+
+
!processing && setConfirmAction(null)}
diff --git a/components/sections/DashboardOverview.tsx b/components/sections/DashboardOverview.tsx
index ff9ba9a..bed2f22 100644
--- a/components/sections/DashboardOverview.tsx
+++ b/components/sections/DashboardOverview.tsx
@@ -9,6 +9,7 @@ import {
Table,
TableBody,
TableCell,
+ TableEmpty,
TableHead,
TableHeader,
TableRow,
@@ -26,6 +27,7 @@ type EventItem = {
};
type DashboardMetrics = {
+ activeSockets?: number;
activeClients: number;
totalClients: number;
bufferStats: {
@@ -62,6 +64,7 @@ export const DashboardOverview: React.FC = () => {
getCached('vcon:storage-files', [])
);
const [isRefreshing, setIsRefreshing] = useState(false);
+ const [fetchError, setFetchError] = useState('');
const refresh = useCallback(async () => {
setIsRefreshing(true);
@@ -74,25 +77,58 @@ export const DashboardOverview: React.FC = () => {
fetch('/api/clients', { cache: 'no-store' }),
]);
+ if (
+ metricsResponse.status === 401 ||
+ eventsResponse.status === 401 ||
+ storageResponse.status === 401 ||
+ clientsResponse.status === 401
+ ) {
+ window.location.assign('/vcon/login');
+ return;
+ }
+
+ const errors: string[] = [];
+
if (metricsResponse.ok) {
const nextMetrics = await metricsResponse.json();
setMetrics(setCached('vcon:metrics', nextMetrics));
+ } else {
+ errors.push('metrics');
}
+
if (eventsResponse.ok) {
const data = await eventsResponse.json();
setEvents(setCached('vcon:dashboard-events', data.events ?? []));
+ } else {
+ errors.push('events');
}
+
if (storageResponse.ok) {
const data = await storageResponse.json();
setFiles(setCached('vcon:storage-files', data.files ?? []));
if (data.stats) setCached('vcon:storage-stats', data.stats);
+ } else {
+ errors.push('storage');
}
+
if (clientsResponse.ok) {
const data = await clientsResponse.json();
setClients(setCached('vcon:clients', data.clients ?? []));
+ } else {
+ errors.push('clients');
+ }
+
+ if (errors.length > 0) {
+ setFetchError(
+ `Partial telemetry refresh failed (${errors.join(', ')}). Showing last available snapshot.`
+ );
+ } else {
+ setFetchError('');
}
} catch {
- // Keep the last successful dashboard snapshot when one request fails.
+ setFetchError(
+ 'Unable to reach telemetry endpoints. Showing last available snapshot.'
+ );
} finally {
setIsRefreshing(false);
}
@@ -108,18 +144,21 @@ export const DashboardOverview: React.FC = () => {
const storedRecordsToday = files
.filter((file) => file.date === today)
.reduce((total, file) => total + file.linesCount, 0);
- const eventsToday =
- storedRecordsToday + (metrics?.bufferStats.currentBufferSize ?? 0);
+ const bufferedCount = metrics?.bufferStats.currentBufferSize ?? 0;
+ const eventsToday = storedRecordsToday + bufferedCount;
const activeClients =
metrics?.activeClients ??
clients.filter((client) => client.status === 'active').length;
const totalClients = metrics?.totalClients ?? clients.length;
+ const activeSockets = metrics?.activeSockets ?? 0;
const totalStorage = metrics?.storageStats.totalMB ?? '0.00';
const totalStorageBytes = metrics?.storageStats.totalBytes ?? 0;
const storageValue =
- totalStorageBytes < 1024 * 1024
- ? String(Math.max(1, Math.round(totalStorageBytes / 1024)))
- : totalStorage;
+ totalStorageBytes === 0
+ ? '0'
+ : totalStorageBytes < 1024 * 1024
+ ? String(Math.max(1, Math.round(totalStorageBytes / 1024)))
+ : totalStorage;
const storageUnit = totalStorageBytes < 1024 * 1024 ? 'KB' : 'MB';
const totalFiles = metrics?.storageStats.totalFiles ?? files.length;
const totalRows =
@@ -153,13 +192,22 @@ export const DashboardOverview: React.FC = () => {
>
}
>
+ {fetchError && (
+
+ {fetchError}
+
+ )}
+
0 ? 'ok' : 'off',
+ label: 'Active clients · Sockets',
+ value: `${activeClients} / ${totalClients} · ${activeSockets} ws`,
+ dot: activeClients > 0 || activeSockets > 0 ? 'ok' : 'off',
},
{ label: 'Storage used', value: `${storageValue} ${storageUnit}` },
{ label: 'Stored rows', value: totalRows.toLocaleString() },
@@ -169,7 +217,11 @@ export const DashboardOverview: React.FC = () => {
0
+ ? `${events.length} · ${bufferedCount} buffered`
+ : `${events.length}`
+ }
action={
{
- {events.map((event) => (
-
-
- {event.timestamp.includes('T')
- ? new Date(event.timestamp).toLocaleTimeString()
- : event.timestamp}
-
-
- {event.clientId}
-
-
- {event.payload}
-
-
- {new TextEncoder().encode(event.payload).length} B
-
-
- ))}
+ {events.length === 0 ? (
+
+ ) : (
+ events.map((event) => (
+
+
+ {event.timestamp.includes('T')
+ ? new Date(event.timestamp).toLocaleTimeString()
+ : event.timestamp}
+
+
+ {event.clientId}
+
+
+ {event.payload}
+
+
+ {new TextEncoder().encode(event.payload).length} B
+
+
+ ))
+ )}
diff --git a/components/sections/DataBrowser.tsx b/components/sections/DataBrowser.tsx
index 12a9eed..e9105d2 100644
--- a/components/sections/DataBrowser.tsx
+++ b/components/sections/DataBrowser.tsx
@@ -3,7 +3,10 @@
import { PageShell } from '@/components/layout/PageShell';
import { Button } from '@/components/ui/Button';
import { Card } from '@/components/ui/Card';
+import { CodeBlock } from '@/components/ui/CodeBlock';
+import { CopyButton } from '@/components/ui/CopyButton';
import { Input, Select } from '@/components/ui/Input';
+import { Modal } from '@/components/ui/Modal';
import { Panel } from '@/components/ui/Panel';
import { Pill } from '@/components/ui/Pill';
import {
@@ -34,6 +37,18 @@ interface Client {
const PAGE_SIZE = 12;
+function formatEventPayload(raw: string): string {
+ try {
+ return JSON.stringify(JSON.parse(raw), null, 2);
+ } catch {
+ return raw;
+ }
+}
+
+function getPayloadBytes(raw: string): number {
+ return new TextEncoder().encode(raw).length;
+}
+
export function DataBrowser() {
const router = useRouter();
const [events, setEvents] = useState([]);
@@ -54,6 +69,7 @@ export function DataBrowser() {
const [filtersReady, setFiltersReady] = useState(false);
const [refreshing, setRefreshing] = useState(false);
const [errorMessage, setErrorMessage] = useState('');
+ const [selectedEvent, setSelectedEvent] = useState(null);
const fetchEvents = useCallback(async () => {
try {
@@ -65,7 +81,11 @@ export function DataBrowser() {
page: String(page),
pageSize: String(PAGE_SIZE),
});
- const response = await fetch(`/api/data?${params}`);
+ const response = await fetch(`/api/data?${params}`, { cache: 'no-store' });
+ if (response.status === 401) {
+ window.location.assign('/vcon/login');
+ return;
+ }
if (!response.ok) throw new Error('Unable to load events');
const data = await response.json();
setEvents(data.events || []);
@@ -78,8 +98,14 @@ export function DataBrowser() {
}, [appliedFilters, page]);
useEffect(() => {
- fetch('/api/clients')
- .then((response) => (response.ok ? response.json() : { clients: [] }))
+ fetch('/api/clients', { cache: 'no-store' })
+ .then((response) => {
+ if (response.status === 401) {
+ window.location.assign('/vcon/login');
+ return { clients: [] };
+ }
+ return response.ok ? response.json() : { clients: [] };
+ })
.then((data) => setClients(data.clients || []))
.catch(() => setClients([]));
}, []);
@@ -131,6 +157,10 @@ export function DataBrowser() {
setRefreshing(false);
};
+ const formattedSelectedPayload = selectedEvent
+ ? formatEventPayload(selectedEvent.payload)
+ : '';
+
return (
) : (
events.map((event) => (
-
+ setSelectedEvent(event)}
+ >
{new Date(event.timestamp).toLocaleTimeString([], {
hour: '2-digit',
@@ -257,13 +291,34 @@ export function DataBrowser() {
{event.payload}
- {new TextEncoder().encode(event.payload).length} B
+ {getPayloadBytes(event.payload)} B
-
-
- {event.status}
-
+
+
+
+ {event.status}
+
+ {
+ e.stopPropagation();
+ setSelectedEvent(event);
+ }}
+ />
+
))
@@ -338,6 +393,69 @@ export function DataBrowser() {
+
+
setSelectedEvent(null)}
+ title={`Event payload: ${selectedEvent?.eventId || ''}`}
+ maxWidth="lg"
+ footer={
+ setSelectedEvent(null)}>
+ Close
+
+ }
+ >
+ {selectedEvent && (
+
+
+
+
+ Client
+
+
+ {selectedEvent.clientId}
+
+
+
+
+ Status
+
+
+ {selectedEvent.status}
+
+
+
+
+ Size
+
+
+ {getPayloadBytes(selectedEvent.payload)} B
+
+
+
+
+ Timestamp
+
+
+ {selectedEvent.timestamp}
+
+
+
+
+
+
+
+ JSON / Raw Payload
+
+
+
+
+ {formattedSelectedPayload}
+
+
+
+ )}
+
);
}
diff --git a/components/sections/ExportWorkspace.tsx b/components/sections/ExportWorkspace.tsx
index 9c7f768..4364a7e 100644
--- a/components/sections/ExportWorkspace.tsx
+++ b/components/sections/ExportWorkspace.tsx
@@ -56,6 +56,7 @@ export function ExportWorkspace() {
const [files, setFiles] = useState
(() =>
getCached('vcon:storage-files', [])
);
+ const [bufferedCount, setBufferedCount] = useState(0);
const [selectedClient, setSelectedClient] = useState('all');
const [fromDate, setFromDate] = useState('');
const [toDate, setToDate] = useState('');
@@ -69,19 +70,33 @@ export function ExportWorkspace() {
useEffect(() => {
Promise.all([
- fetch('/api/clients').then(async (response) => {
+ fetch('/api/clients', { cache: 'no-store' }).then(async (response) => {
+ if (response.status === 401) {
+ window.location.assign('/vcon/login');
+ return { clients: [] };
+ }
if (!response.ok) throw new Error('Unable to load clients');
return response.json();
}),
- fetch('/api/storage').then(async (response) => {
+ fetch('/api/storage', { cache: 'no-store' }).then(async (response) => {
+ if (response.status === 401) {
+ window.location.assign('/vcon/login');
+ return { files: [] };
+ }
if (!response.ok) throw new Error('Unable to load stored files');
return response.json();
}),
+ fetch('/api/metrics', { cache: 'no-store' })
+ .then(async (response) => (response.ok ? response.json() : null))
+ .catch(() => null),
])
- .then(([clientData, storageData]) => {
+ .then(([clientData, storageData, metricsData]) => {
setClients(setCached('vcon:clients', clientData.clients || []));
setFiles(setCached('vcon:storage-files', storageData.files || []));
if (storageData.stats) setCached('vcon:storage-stats', storageData.stats);
+ if (metricsData?.bufferStats) {
+ setBufferedCount(Number(metricsData.bufferStats.currentBufferSize) || 0);
+ }
setLoadError('');
})
.catch((fetchError) => {
@@ -162,15 +177,24 @@ export function ExportWorkspace() {
return `/api/export?${params.toString()}`;
}, [format, fromDate, selectedClient, toDate]);
+ const invalidRange = Boolean(fromDate && toDate && fromDate > toDate);
+ const nothingToExport = preview.rows === 0 && bufferedCount === 0;
+
const downloadExport = () => {
- if (fromDate && toDate && fromDate > toDate) {
+ if (invalidRange) {
error('Invalid date range', 'Start date must be on or before end date.');
return;
}
+ if (nothingToExport) {
+ error('Nothing to export', 'No stored rows or buffered events match the selected filters.');
+ return;
+ }
window.location.assign(exportUrl);
info(
'Export started',
- `${format.toUpperCase()} download for ${preview.rows.toLocaleString()} rows.`
+ bufferedCount > 0
+ ? `${format.toUpperCase()} download (${preview.rows.toLocaleString()} stored + ${bufferedCount} buffered auto-flushed).`
+ : `${format.toUpperCase()} download for ${preview.rows.toLocaleString()} rows.`
);
};
@@ -307,17 +331,30 @@ export function ExportWorkspace() {
{' '}
files
+ {bufferedCount > 0 && (
+ <>
+ ·
+
+ +{bufferedCount} buffered (flushed on export)
+
+ >
+ )}
{loadError && (
-
+
{loadError}
)}
- {fromDate && toDate && fromDate > toDate && (
-
+ {invalidRange && (
+
Start date must be on or before end date.
)}
+ {!loading && !invalidRange && nothingToExport && (
+
+ No stored rows or buffered events match the selected filters.
+
+ )}
@@ -328,7 +365,7 @@ export function ExportWorkspace() {
variant="primary"
size="xs"
icon="Download"
- disabled={loading || Boolean(fromDate && toDate && fromDate > toDate)}
+ disabled={loading || invalidRange || nothingToExport}
onClick={downloadExport}
>
Download {format.toUpperCase()}
diff --git a/components/sections/Hero.tsx b/components/sections/Hero.tsx
index ef34d58..ba35414 100644
--- a/components/sections/Hero.tsx
+++ b/components/sections/Hero.tsx
@@ -1,5 +1,5 @@
import { ActivityPreview } from '@/components/ui/ActivityPreview';
-import { Button } from '@/components/ui/Button';
+import { buttonClassNames } from '@/components/ui/Button';
import Link from 'next/link';
import React from 'react';
@@ -17,11 +17,11 @@ export const Hero: React.FC = () => (
Keep activity organized, review the records that matter, and export them when you need them.
-
- Explore features
+
+ Explore features
-
- Python SDK
+
+ Python SDK
diff --git a/components/sections/LandingAbout.tsx b/components/sections/LandingAbout.tsx
index 6ba857a..268f8c5 100644
--- a/components/sections/LandingAbout.tsx
+++ b/components/sections/LandingAbout.tsx
@@ -1,4 +1,4 @@
-import { Button } from '@/components/ui/Button';
+import { buttonClassNames } from '@/components/ui/Button';
import Link from 'next/link';
import React from 'react';
@@ -8,8 +8,8 @@ export const LandingAbout: React.FC = () => (
Ready to simplify your data workflow?
Start with the SDK guide and make the workspace your own.
-
- View Python SDK
+
+ View Python SDK
);
diff --git a/components/sections/SettingsWorkspace.tsx b/components/sections/SettingsWorkspace.tsx
index 786e419..d55cb76 100644
--- a/components/sections/SettingsWorkspace.tsx
+++ b/components/sections/SettingsWorkspace.tsx
@@ -4,6 +4,7 @@ import { PageShell } from '@/components/layout/PageShell';
import { Button } from '@/components/ui/Button';
import { FormSwitch } from '@/components/ui/Form';
import { Input } from '@/components/ui/Input';
+import { Modal } from '@/components/ui/Modal';
import {
SettingsField,
SettingsFooter,
@@ -14,7 +15,7 @@ import {
import { StatStrip } from '@/components/ui/StatStrip';
import { useToast } from '@/components/ui/Toast';
import { getCached, setCached } from '@/lib/vconCache';
-import { useEffect, useMemo, useState } from 'react';
+import { useCallback, useEffect, useMemo, useState } from 'react';
interface EditableSettings {
maxEventsPerSec: number;
@@ -50,6 +51,7 @@ const TABS = [
];
type TabId = (typeof TABS)[number]['id'];
+type SocketHealthState = 'idle' | 'checking' | 'online' | 'offline';
export function SettingsWorkspace() {
const { success, error } = useToast();
@@ -61,11 +63,15 @@ export function SettingsWorkspace() {
);
const [activeTab, setActiveTab] = useState('general');
const [jwtSecret, setJwtSecret] = useState('');
+ const [jwtError, setJwtError] = useState('');
+ const [confirmRotateOpen, setConfirmRotateOpen] = useState(false);
const [storagePath, setStoragePath] = useState(() =>
getCached('vcon:storage-path', '')
);
const [baseUrl, setBaseUrl] = useState('');
const [webSocketUrl, setWebSocketUrl] = useState('');
+ const [socketStatus, setSocketStatus] = useState('idle');
+ const [activeSockets, setActiveSockets] = useState(0);
const [serverPort, setServerPort] = useState(() =>
getCached('vcon:server-port', 4000)
);
@@ -74,26 +80,60 @@ export function SettingsWorkspace() {
);
const [saving, setSaving] = useState(false);
+ const probeSocketServer = useCallback(async (port: number) => {
+ setSocketStatus('checking');
+ const isLocalDev = window.location.port === '3000';
+ const httpHost = isLocalDev
+ ? `${window.location.hostname}:${port}`
+ : window.location.host;
+ const healthUrl = `${window.location.protocol}//${httpHost}/health`;
+
+ try {
+ const response = await fetch(healthUrl, {
+ cache: 'no-store',
+ signal: AbortSignal.timeout(1500),
+ });
+ setSocketStatus(response.ok ? 'online' : 'offline');
+ } catch {
+ setSocketStatus('offline');
+ }
+ }, []);
+
useEffect(() => {
setBaseUrl(window.location.origin);
const isLocalDev = window.location.port === '3000';
const socketProtocol = window.location.protocol === 'https:' ? 'wss' : 'ws';
const socketHost = isLocalDev
- ? `${window.location.hostname}:4000`
+ ? `${window.location.hostname}:${serverPort}`
: window.location.host;
setWebSocketUrl(`${socketProtocol}://${socketHost}`);
Promise.all([
- fetch('/api/settings').then(async (response) => {
+ fetch('/api/settings', { cache: 'no-store' }).then(async (response) => {
+ if (response.status === 401) {
+ window.location.assign('/vcon/login');
+ return null;
+ }
if (!response.ok) throw new Error('Unable to load settings');
return response.json();
}),
- fetch('/api/storage').then(async (response) => {
+ fetch('/api/storage', { cache: 'no-store' }).then(async (response) => {
+ if (response.status === 401) {
+ window.location.assign('/vcon/login');
+ return null;
+ }
if (!response.ok) throw new Error('Unable to load storage information');
return response.json();
}),
+ fetch('/api/metrics', { cache: 'no-store' })
+ .then(async (response) => (response.ok ? response.json() : null))
+ .catch(() => null),
])
- .then(([settingsData, storageData]) => {
+ .then(([settingsData, storageData, metricsData]) => {
+ if (!settingsData || !storageData) return;
+ const loadedPort = Number(
+ (settingsData.settings as PublicSettings).serverPort ?? 4000
+ );
const loaded: EditableSettings = {
maxEventsPerSec: Number(
settingsData.settings.maxEventsPerSec ??
@@ -126,25 +166,26 @@ export function SettingsWorkspace() {
};
setSettings(setCached('vcon:settings', loaded));
setSavedSettings(loaded);
- setServerPort(
- setCached(
- 'vcon:server-port',
- Number((settingsData.settings as PublicSettings).serverPort ?? 4000)
- )
- );
+ setServerPort(setCached('vcon:server-port', loadedPort));
setStoragePath(
setCached(
'vcon:storage-path',
String(storageData.stats?.storageDirectory ?? '')
)
);
+ if (metricsData && typeof metricsData.activeSockets === 'number') {
+ setActiveSockets(metricsData.activeSockets);
+ if (metricsData.activeSockets > 0) {
+ setSocketStatus('online');
+ }
+ }
setCached('vcon:settings-loaded', true);
})
.catch(() => {
error('Settings load failed', 'Using default runtime settings.');
})
.finally(() => setLoading(false));
- }, [error]);
+ }, [error, serverPort]);
const isDirty = useMemo(
() =>
@@ -157,9 +198,9 @@ export function SettingsWorkspace() {
setSettings((current) => ({ ...current, [key]: Number(value) || 0 }));
};
- const saveSettings = async (event: React.FormEvent) => {
- event.preventDefault();
- const normalized: EditableSettings = {
+ const buildNormalizedSettings = (): EditableSettings => {
+ const parsedPrune = Number(settings.autoPruneDays);
+ return {
maxEventsPerSec: Math.max(
1,
Math.min(
@@ -187,11 +228,16 @@ export function SettingsWorkspace() {
settings.fileRotationMb || DEFAULT_SETTINGS.fileRotationMb
),
enableIdempotency: settings.enableIdempotency,
- autoPruneDays: Math.max(
- 1,
- Math.min(365, settings.autoPruneDays || DEFAULT_SETTINGS.autoPruneDays)
- ),
+ autoPruneDays: Number.isFinite(parsedPrune)
+ ? Math.max(0, Math.min(365, Math.floor(parsedPrune)))
+ : DEFAULT_SETTINGS.autoPruneDays,
};
+ };
+
+ const executeSave = async () => {
+ const normalized = buildNormalizedSettings();
+ const trimmedSecret = jwtSecret.trim();
+ setConfirmRotateOpen(false);
setSaving(true);
try {
const response = await fetch('/api/settings', {
@@ -199,15 +245,25 @@ export function SettingsWorkspace() {
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
...normalized,
- ...(jwtSecret.trim() ? { jwtSecret: jwtSecret.trim() } : {}),
+ ...(trimmedSecret ? { jwtSecret: trimmedSecret } : {}),
}),
});
+ if (response.status === 401) {
+ window.location.assign('/vcon/login');
+ return;
+ }
const data = await response.json();
if (!response.ok) throw new Error(data.error || 'Settings update failed');
setSettings(setCached('vcon:settings', normalized));
setSavedSettings(normalized);
setJwtSecret('');
- success('Settings saved', 'Runtime settings updated.');
+ setJwtError('');
+ success(
+ 'Settings saved',
+ trimmedSecret
+ ? 'Runtime settings saved and client JWT credentials rotated.'
+ : 'Runtime settings updated.'
+ );
} catch (saveError) {
error(
'Save failed',
@@ -220,9 +276,30 @@ export function SettingsWorkspace() {
}
};
+ const handleFormSubmit = async (event: React.FormEvent) => {
+ event.preventDefault();
+ const trimmedSecret = jwtSecret.trim();
+ if (trimmedSecret.length > 0 && trimmedSecret.length < 32) {
+ const message = 'JWT secret must be at least 32 characters.';
+ setJwtError(message);
+ setActiveTab('security');
+ error('Invalid JWT secret', message);
+ return;
+ }
+
+ setJwtError('');
+ if (trimmedSecret.length >= 32) {
+ setConfirmRotateOpen(true);
+ return;
+ }
+
+ await executeSave();
+ };
+
const discardChanges = () => {
setSettings(savedSettings);
setJwtSecret('');
+ setJwtError('');
};
return (
@@ -237,7 +314,9 @@ export function SettingsWorkspace() {
},
{
label: 'CSV rotation',
- value: `${settings.fileRotationMb}MB · ${settings.autoPruneDays}d`,
+ value: `${settings.fileRotationMb}MB · ${
+ settings.autoPruneDays === 0 ? 'Keep all' : `${settings.autoPruneDays}d`
+ }`,
},
]}
/>
@@ -249,7 +328,7 @@ export function SettingsWorkspace() {
onChange={(tab) => setActiveTab(tab as TabId)}
/>
-
+
+ !saving && setConfirmRotateOpen(false)}
+ title="Confirm JWT secret rotation"
+ maxWidth="xs"
+ footer={
+ <>
+ setConfirmRotateOpen(false)}
+ >
+ Cancel
+
+ void executeSave()}
+ >
+ Rotate & Save
+
+ >
+ }
+ >
+
+ Rotating the signing JWT secret will immediately re-sign registered client tokens and invalidate all older SDK JWT tokens and active WebSocket sessions. Continue?
+
+
);
}
diff --git a/components/sections/clients/types.ts b/components/sections/clients/types.ts
index 3e61337..a63cbdd 100644
--- a/components/sections/clients/types.ts
+++ b/components/sections/clients/types.ts
@@ -6,6 +6,7 @@ export interface ClientItem {
totalEvents: number;
lastActive?: string;
createdAt: string;
+ tokenPreview?: string;
}
export interface ClientCredentials extends ClientItem {
@@ -13,4 +14,9 @@ export interface ClientCredentials extends ClientItem {
token: string;
}
-export type ClientStatusFilter = "all" | "online" | "offline" | "pending";
+export type ClientStatusFilter =
+ | "all"
+ | "online"
+ | "offline"
+ | "pending"
+ | "revoked";
diff --git a/components/ui/Button.tsx b/components/ui/Button.tsx
index a33ef98..8131cd1 100644
--- a/components/ui/Button.tsx
+++ b/components/ui/Button.tsx
@@ -10,6 +10,46 @@ export interface ButtonProps extends React.ButtonHTMLAttributes(
(
{
@@ -25,39 +65,11 @@ export const Button = React.forwardRef(
},
ref
) => {
- const baseStyles =
- 'inline-flex items-center justify-center font-normal transition-colors select-none cursor-pointer disabled:opacity-50 disabled:cursor-not-allowed border border-transparent whitespace-nowrap outline-none';
-
- const variantStyles = {
- primary:
- 'bg-[var(--text)] text-[var(--bg)] hover:opacity-90 border-[var(--text)]',
- secondary:
- 'bg-[var(--bg-elevated)] text-[var(--text)] border-[var(--border-strong)] hover:bg-[var(--bg-hover)]',
- ghost:
- 'bg-transparent text-[var(--text-muted)] hover:text-[var(--text)] hover:bg-[var(--bg-hover)] border-transparent',
- danger:
- 'bg-transparent text-[var(--danger)] border-[var(--danger)] hover:bg-[var(--danger-soft)]',
- icon:
- 'bg-transparent text-[var(--text-muted)] hover:text-[var(--text)] hover:bg-[var(--bg-hover)] border-transparent p-0',
- };
-
- const sizeStyles = {
- default: 'min-h-8.5 px-3 text-[12px] gap-1.5',
- sm: 'h-7.5 px-2.5 text-[11.5px] gap-1.5',
- xs: 'h-7 px-2 text-[11px] gap-1',
- icon: 'w-7.5 h-7.5 p-0',
- };
-
return (
{loading ? (
diff --git a/core/Database/database.ts b/core/Database/database.ts
index 21ea2b5..0adc05f 100644
--- a/core/Database/database.ts
+++ b/core/Database/database.ts
@@ -73,6 +73,31 @@ export class DatabaseEngine {
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`),
+ pool.query(`
+ CREATE TABLE IF NOT EXISTS deleted_clients (
+ client_id TEXT PRIMARY KEY,
+ deleted_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+ )
+ `),
+ pool.query(`
+ CREATE TABLE IF NOT EXISTS admin_users (
+ id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
+ email TEXT NOT NULL UNIQUE,
+ password_salt BYTEA NOT NULL,
+ password_hash BYTEA NOT NULL,
+ token_version INTEGER NOT NULL DEFAULT 1,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+ )
+ `),
+ pool.query(`
+ CREATE TABLE IF NOT EXISTS admin_login_attempts (
+ key_hash TEXT PRIMARY KEY,
+ window_started_at TIMESTAMPTZ NOT NULL,
+ failed_attempts INTEGER NOT NULL,
+ blocked_until TIMESTAMPTZ,
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+ )
+ `),
]).then(() => undefined);
try {
diff --git a/core/Security/security.ts b/core/Security/security.ts
index 452f24c..483d702 100644
--- a/core/Security/security.ts
+++ b/core/Security/security.ts
@@ -1,3 +1,38 @@
+import { createHash } from "node:crypto";
+import { AuthService } from "../auth/jwt";
+
+export function resolveInternalIngestionSecret(
+ jwtSecretOverride?: string,
+): string {
+ const explicit = (process.env.DATASTACK_INTERNAL_SECRET || "").trim();
+ const isProd = process.env.NODE_ENV === "production";
+ if (isProd) {
+ if (explicit.length >= 32) return explicit;
+ if (explicit.length > 0) return "";
+ } else if (explicit.length > 0) {
+ return explicit;
+ }
+
+ const activeJwt = String(
+ jwtSecretOverride ?? AuthService.getSecret() ?? "",
+ ).trim();
+ const minJwtLen = isProd ? 32 : 1;
+ if (activeJwt.length < minJwtLen) {
+ return "";
+ }
+
+ return createHash("sha256")
+ .update(`datastack-internal:${activeJwt}`)
+ .digest("hex");
+}
+
+export function getInternalIngestionHeaders(
+ jwtSecretOverride?: string,
+): Record {
+ const secret = resolveInternalIngestionSecret(jwtSecretOverride);
+ return secret ? { "x-datastack-internal-secret": secret } : {};
+}
+
export interface ValidationResult {
allowed: boolean;
reason?: string;
@@ -130,7 +165,7 @@ export class SecurityEngine {
if (!withinLimit) {
return {
allowed: false,
- reason: `Rate limit of ${clientMaxRate} events/sec exceeded for client ${clientId}`,
+ reason: `Rate limit of ${effectiveRateLimit} events/sec exceeded for client ${clientId}`,
code: "RATE_LIMIT_EXCEEDED",
};
}
@@ -144,6 +179,18 @@ export class SecurityEngine {
}
public static getStats() {
+ const now = Date.now();
+ for (const [k, timestamp] of this.idempotencySet.entries()) {
+ if (now - timestamp > this.IDEMPOTENCY_TTL_MS) {
+ this.idempotencySet.delete(k);
+ }
+ }
+ for (const [clientId, entry] of this.rateLimitMap.entries()) {
+ if (now - entry.windowStart > 10000) {
+ this.rateLimitMap.delete(clientId);
+ }
+ }
+
return {
activeIdempotencyKeys: this.idempotencySet.size,
rateLimitedClientsCount: this.rateLimitMap.size,
diff --git a/core/Storage/csvStorage.ts b/core/Storage/csvStorage.ts
index badf20b..e6dd983 100644
--- a/core/Storage/csvStorage.ts
+++ b/core/Storage/csvStorage.ts
@@ -19,10 +19,30 @@ export interface StorageFileMeta {
modifiedAt: Date;
}
+interface RowCountCacheEntry {
+ sizeBytes: number;
+ mtimeMs: number;
+ linesCount: number;
+}
+
+const LOCK_RETRY_MIN_MS = 5;
+const LOCK_RETRY_MAX_MS = 18;
+const LOCK_TIMEOUT_MS = 10_000;
+const STALE_LOCK_AGE_MS = 30_000;
+
+function sleepMs(ms: number): Promise {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
export class CSVStorageManager {
// Base storage directory: /storage/data
public static baseStorageDir = path.resolve(process.cwd(), 'storage', 'data');
public static maxFileSizeBytes = 100 * 1024 * 1024; // 100MB rotation limit
+ private static rowCountCache = new Map();
+
+ public static clearRowCountCache(): void {
+ this.rowCountCache.clear();
+ }
/**
* Ensures client directory exists
@@ -37,15 +57,194 @@ export class CSVStorageManager {
}
/**
- * Resolves target CSV file path with daily and 100MB rotation rules
+ * Acquires an exclusive cooperative lock file for a client+date partition across Port 3000 & Port 4000
+ */
+ public static async withPartitionLock(
+ clientId: string,
+ dateStr: string,
+ fn: () => Promise | T,
+ ): Promise {
+ const clientDir = this.ensureClientDir(clientId);
+ const date = this.extractSafePartitionDate(dateStr);
+ const lockPath = path.join(clientDir, `.${date}.csv.lock`);
+ const deadline = Date.now() + LOCK_TIMEOUT_MS;
+
+ let acquired = false;
+ while (Date.now() < deadline) {
+ try {
+ const fd = fs.openSync(lockPath, 'wx');
+ try {
+ fs.writeFileSync(fd, `${process.pid}:${Date.now()}`, 'utf8');
+ } finally {
+ fs.closeSync(fd);
+ }
+ acquired = true;
+ break;
+ } catch (err: unknown) {
+ const code = (err as NodeJS.ErrnoException)?.code;
+ if (code !== 'EEXIST') {
+ throw err;
+ }
+ try {
+ const lockStat = fs.statSync(lockPath);
+ if (Date.now() - lockStat.mtimeMs > STALE_LOCK_AGE_MS) {
+ fs.unlinkSync(lockPath);
+ continue;
+ }
+ } catch {
+ // Lock was released concurrently
+ }
+ const jitter =
+ LOCK_RETRY_MIN_MS +
+ Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS));
+ await sleepMs(jitter);
+ }
+ }
+
+ if (!acquired) {
+ throw new Error(`Timed out waiting for CSV partition lock: ${lockPath}`);
+ }
+
+ try {
+ return await fn();
+ } finally {
+ try {
+ if (fs.existsSync(lockPath)) {
+ fs.unlinkSync(lockPath);
+ }
+ } catch {
+ // Ignore cleanup failure if already unlinked
+ }
+ }
+ }
+
+ /**
+ * Normalizes and validates any timestamp or date string into a strict YYYY-MM-DD partition key
+ */
+ public static extractSafePartitionDate(rawTimestamp?: string): string {
+ const fallback = new Date().toISOString().slice(0, 10);
+ if (!rawTimestamp || typeof rawTimestamp !== 'string') {
+ return fallback;
+ }
+ const trimmed = rawTimestamp.trim();
+ const prefix = trimmed.slice(0, 10);
+ if (/^\d{4}-\d{2}-\d{2}$/.test(prefix)) {
+ const parsedPrefix = new Date(`${prefix}T00:00:00.000Z`);
+ if (!Number.isNaN(parsedPrefix.getTime())) {
+ return prefix;
+ }
+ }
+ if (/^\d+(\.\d+)?$/.test(trimmed)) {
+ const num = Number(trimmed);
+ const ms = num < 1e11 ? num * 1000 : num;
+ const d = new Date(ms);
+ if (!Number.isNaN(d.getTime())) {
+ return d.toISOString().slice(0, 10);
+ }
+ }
+ const parsed = new Date(trimmed);
+ if (!Number.isNaN(parsed.getTime())) {
+ return parsed.toISOString().slice(0, 10);
+ }
+ return fallback;
+ }
+
+ /**
+ * Stateful RFC 4180 quote-aware chunked row counter so multi-100MB CSVs never load wholly into RAM
+ */
+ private static createChunkRowCounter() {
+ let inQuotes = false;
+ let pendingQuote = false;
+ let currentLineHasChars = false;
+ let totalRows = 0;
+
+ return {
+ pushChunk(chunk: Buffer | string) {
+ const str = typeof chunk === 'string' ? chunk : chunk.toString('utf8');
+ for (let i = 0; i < str.length; i += 1) {
+ const ch = str[i];
+ if (pendingQuote) {
+ pendingQuote = false;
+ if (ch === '"') {
+ // Escaped quote "" across chunk boundary or inline
+ continue;
+ }
+ inQuotes = !inQuotes;
+ }
+
+ if (ch === '"') {
+ currentLineHasChars = true;
+ if (inQuotes) {
+ if (i + 1 < str.length) {
+ if (str[i + 1] === '"') {
+ i += 1;
+ } else {
+ inQuotes = false;
+ }
+ } else {
+ pendingQuote = true;
+ }
+ } else {
+ inQuotes = true;
+ }
+ } else if (ch === '\n' && !inQuotes) {
+ if (currentLineHasChars) {
+ totalRows += 1;
+ currentLineHasChars = false;
+ }
+ } else if (ch !== '\r' && ch !== ' ' && ch !== '\t') {
+ currentLineHasChars = true;
+ }
+ }
+ },
+ finish(): number {
+ if (currentLineHasChars) {
+ totalRows += 1;
+ }
+ return Math.max(0, totalRows - 1);
+ },
+ };
+ }
+
+ private static countCsvDataRowsChunkedSync(filePath: string): number {
+ const counter = this.createChunkRowCounter();
+ const buf = Buffer.allocUnsafe(64 * 1024);
+ const fd = fs.openSync(filePath, 'r');
+ try {
+ let bytesRead = 0;
+ while ((bytesRead = fs.readSync(fd, buf, 0, buf.length, null)) > 0) {
+ counter.pushChunk(buf.subarray(0, bytesRead));
+ }
+ return counter.finish();
+ } finally {
+ fs.closeSync(fd);
+ }
+ }
+
+ private static async countCsvDataRowsStreamAsync(
+ filePath: string,
+ ): Promise {
+ const counter = this.createChunkRowCounter();
+ return new Promise((resolve, reject) => {
+ const stream = fs.createReadStream(filePath, {
+ highWaterMark: 64 * 1024,
+ });
+ stream.on('data', (chunk) => counter.pushChunk(chunk));
+ stream.on('end', () => resolve(counter.finish()));
+ stream.on('error', (err) => reject(err));
+ });
+ }
+
+ /**
+ * Resolves target CSV file path with daily and configured MB rotation rules
*/
public static getTargetFilePath(clientId: string, dateStr?: string): string {
const clientDir = this.ensureClientDir(clientId);
- const date = dateStr || new Date().toISOString().split('T')[0];
+ const date = this.extractSafePartitionDate(dateStr);
let baseFileName = `${date}.csv`;
let targetPath = path.join(clientDir, baseFileName);
- // Check for 100MB rotation
+ // Check for file rotation
let partIndex = 1;
while (fs.existsSync(targetPath)) {
try {
@@ -66,10 +265,12 @@ export class CSVStorageManager {
}
/**
- * Batch append events into target CSV file.
+ * Batch append events into target CSV file under cooperative partition lock.
* If file doesn't exist, writes header: clientId,eventId,timestamp,payload
*/
- public static async appendBatch(events: DataEvent[]): Promise<{ written: number; filesUpdated: string[] }> {
+ public static async appendBatch(
+ events: DataEvent[],
+ ): Promise<{ written: number; filesUpdated: string[] }> {
if (!events || events.length === 0) {
return { written: 0, filesUpdated: [] };
}
@@ -77,7 +278,7 @@ export class CSVStorageManager {
// Group events by client + date
const grouped = new Map();
for (const evt of events) {
- const date = (evt.timestamp || new Date().toISOString()).split('T')[0];
+ const date = this.extractSafePartitionDate(evt.timestamp);
const key = `${evt.clientId}:::${date}`;
if (!grouped.has(key)) {
grouped.set(key, []);
@@ -90,31 +291,39 @@ export class CSVStorageManager {
for (const [key, batch] of grouped.entries()) {
const [clientId, date] = key.split(':::');
- const targetPath = this.getTargetFilePath(clientId, date);
- let hasContent = false;
- try {
- hasContent =
- fs.existsSync(targetPath) && fs.statSync(targetPath).size > 0;
- } catch {
- hasContent = false;
- }
+ const targetPath = await this.withPartitionLock(clientId, date, () => {
+ const resolvedPath = this.getTargetFilePath(clientId, date);
+ let hasContent = false;
+ try {
+ hasContent =
+ fs.existsSync(resolvedPath) && fs.statSync(resolvedPath).size > 0;
+ } catch {
+ hasContent = false;
+ }
- // Format payload as JSON string if not already
- const cleanBatch = batch.map((item) => ({
- clientId: item.clientId,
- eventId: item.eventId,
- timestamp: item.timestamp,
- payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
- }));
+ const cleanBatch = batch.map((item) => ({
+ clientId: item.clientId,
+ eventId: item.eventId,
+ timestamp: item.timestamp,
+ payload:
+ typeof item.payload === 'string'
+ ? item.payload
+ : JSON.stringify(item.payload),
+ }));
- const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
- const csvData = parser.parse(cleanBatch);
+ const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
+ const csvData = parser.parse(cleanBatch);
- if (hasContent) {
- fs.appendFileSync(targetPath, '\n' + csvData, 'utf8');
- } else {
- fs.writeFileSync(targetPath, csvData, 'utf8');
- }
+ if (hasContent) {
+ fs.appendFileSync(resolvedPath, '\n' + csvData, 'utf8');
+ } else {
+ fs.writeFileSync(resolvedPath, csvData, 'utf8');
+ }
+
+ // Invalidate row cache for modified file immediately
+ this.rowCountCache.delete(resolvedPath);
+ return resolvedPath;
+ });
filesUpdated.push(targetPath);
}
@@ -123,14 +332,16 @@ export class CSVStorageManager {
}
/**
- * List all stored CSV files with sizes and dates
+ * List all stored CSV files with cached row counts (invalidated on mtime/size changes)
*/
public static listAllFiles(): StorageFileMeta[] {
const results: StorageFileMeta[] = [];
if (!fs.existsSync(this.baseStorageDir)) {
+ this.rowCountCache.clear();
return results;
}
+ const activePaths = new Set();
try {
const clientDirs = fs.readdirSync(this.baseStorageDir);
for (const clientDir of clientDirs) {
@@ -142,15 +353,27 @@ export class CSVStorageManager {
if (!file.endsWith('.csv')) continue;
const fullFilePath = path.join(fullClientPath, file);
const stats = fs.statSync(fullFilePath);
+ activePaths.add(fullFilePath);
- // Count lines approximately (or sample)
+ const cached = this.rowCountCache.get(fullFilePath);
let linesCount = 0;
- try {
- const content = fs.readFileSync(fullFilePath, 'utf8');
- linesCount = content.split('\n').filter((l) => l.trim().length > 0).length - 1; // subtract header
- if (linesCount < 0) linesCount = 0;
- } catch {
- linesCount = 0;
+ if (
+ cached &&
+ cached.sizeBytes === stats.size &&
+ cached.mtimeMs === stats.mtimeMs
+ ) {
+ linesCount = cached.linesCount;
+ } else {
+ try {
+ linesCount = this.countCsvDataRowsChunkedSync(fullFilePath);
+ this.rowCountCache.set(fullFilePath, {
+ sizeBytes: stats.size,
+ mtimeMs: stats.mtimeMs,
+ linesCount,
+ });
+ } catch {
+ linesCount = 0;
+ }
}
const datePart = file.replace('.csv', '').split('-part')[0];
@@ -166,6 +389,12 @@ export class CSVStorageManager {
});
}
}
+
+ for (const key of this.rowCountCache.keys()) {
+ if (!activePaths.has(key)) {
+ this.rowCountCache.delete(key);
+ }
+ }
} catch {
// Return whatever collected
}
@@ -174,12 +403,88 @@ export class CSVStorageManager {
}
/**
- * Prune CSV partitions older than autoPruneDays
+ * Non-blocking async listing using fs.promises + streaming row counting on cache misses
+ */
+ public static async listAllFilesAsync(): Promise {
+ const results: StorageFileMeta[] = [];
+ if (!fs.existsSync(this.baseStorageDir)) {
+ this.rowCountCache.clear();
+ return results;
+ }
+
+ const activePaths = new Set();
+ try {
+ const clientDirs = await fs.promises.readdir(this.baseStorageDir);
+ for (const clientDir of clientDirs) {
+ const fullClientPath = path.join(this.baseStorageDir, clientDir);
+ const dirStat = await fs.promises.stat(fullClientPath).catch(() => null);
+ if (!dirStat || !dirStat.isDirectory()) continue;
+
+ const files = await fs.promises.readdir(fullClientPath);
+ for (const file of files) {
+ if (!file.endsWith('.csv')) continue;
+ const fullFilePath = path.join(fullClientPath, file);
+ const stats = await fs.promises.stat(fullFilePath).catch(() => null);
+ if (!stats) continue;
+ activePaths.add(fullFilePath);
+
+ const cached = this.rowCountCache.get(fullFilePath);
+ let linesCount = 0;
+ if (
+ cached &&
+ cached.sizeBytes === stats.size &&
+ cached.mtimeMs === stats.mtimeMs
+ ) {
+ linesCount = cached.linesCount;
+ } else {
+ try {
+ linesCount = await this.countCsvDataRowsStreamAsync(fullFilePath);
+ this.rowCountCache.set(fullFilePath, {
+ sizeBytes: stats.size,
+ mtimeMs: stats.mtimeMs,
+ linesCount,
+ });
+ } catch {
+ linesCount = 0;
+ }
+ }
+
+ const datePart = file.replace('.csv', '').split('-part')[0];
+
+ results.push({
+ clientId: clientDir,
+ date: datePart,
+ filename: file,
+ filePath: fullFilePath,
+ sizeBytes: stats.size,
+ linesCount,
+ modifiedAt: stats.mtime,
+ });
+ }
+ }
+
+ for (const key of this.rowCountCache.keys()) {
+ if (!activePaths.has(key)) {
+ this.rowCountCache.delete(key);
+ }
+ }
+ } catch {
+ // Return whatever collected
+ }
+
+ return results.sort((a, b) => b.modifiedAt.getTime() - a.modifiedAt.getTime());
+ }
+
+ /**
+ * Prune CSV partitions older than autoPruneDays without counting row contents
*/
public static pruneExpiredPartitions(autoPruneDays: number): number {
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
return 0;
}
+ if (!fs.existsSync(this.baseStorageDir)) {
+ return 0;
+ }
const cutoffDate = new Date(
Date.now() - Math.floor(autoPruneDays) * 86_400_000,
@@ -188,28 +493,38 @@ export class CSVStorageManager {
.split('T')[0];
let prunedCount = 0;
- const files = this.listAllFiles();
- for (const file of files) {
- if (/^\d{4}-\d{2}-\d{2}$/.test(file.date) && file.date < cutoffDate) {
- try {
- if (fs.existsSync(file.filePath)) {
- fs.unlinkSync(file.filePath);
- prunedCount += 1;
+ try {
+ const clientDirs = fs.readdirSync(this.baseStorageDir);
+ for (const clientDir of clientDirs) {
+ const fullClientPath = path.join(this.baseStorageDir, clientDir);
+ if (!fs.statSync(fullClientPath).isDirectory()) continue;
+
+ const files = fs.readdirSync(fullClientPath);
+ for (const file of files) {
+ if (!file.endsWith('.csv')) continue;
+ const datePart = file.replace('.csv', '').split('-part')[0];
+ if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
+ const fullPath = path.join(fullClientPath, file);
+ try {
+ if (fs.existsSync(fullPath)) {
+ fs.unlinkSync(fullPath);
+ this.rowCountCache.delete(fullPath);
+ prunedCount += 1;
+ }
+ } catch {
+ // Ignore file removal error
+ }
}
- } catch {
- // Ignore file removal error
}
}
+ } catch {
+ // Ignore directory scan failure
}
return prunedCount;
}
- /**
- * Get aggregate storage statistics
- */
- public static getStorageStats() {
- const files = this.listAllFiles();
+ private static formatStorageStats(files: StorageFileMeta[]) {
const totalBytes = files.reduce((acc, f) => acc + f.sizeBytes, 0);
const totalRows = files.reduce((acc, f) => acc + f.linesCount, 0);
const uniqueClients = new Set(files.map((f) => f.clientId)).size;
@@ -231,10 +546,34 @@ export class CSVStorageManager {
}
/**
- * Reads file content for export / download
+ * Get aggregate storage statistics (sync cached)
+ */
+ public static getStorageStats() {
+ return this.formatStorageStats(this.listAllFiles());
+ }
+
+ /**
+ * Get aggregate storage statistics (non-blocking async)
+ */
+ public static async getStorageStatsAsync() {
+ const files = await this.listAllFilesAsync();
+ return this.formatStorageStats(files);
+ }
+
+ /**
+ * Reads file content for export / download with path traversal protection
*/
public static getFileContent(clientId: string, filename: string): string | null {
- const fullPath = path.join(this.baseStorageDir, clientId, filename);
+ const safeClientId = String(clientId || '').replace(/[^a-zA-Z0-9_-]/g, '_');
+ const safeFilename = path.basename(String(filename || ''));
+ if (!safeClientId || !safeFilename || !safeFilename.endsWith('.csv')) {
+ return null;
+ }
+ const fullPath = path.resolve(this.baseStorageDir, safeClientId, safeFilename);
+ const resolvedBase = path.resolve(this.baseStorageDir);
+ if (!fullPath.startsWith(resolvedBase + path.sep)) {
+ return null;
+ }
if (fs.existsSync(fullPath)) {
return fs.readFileSync(fullPath, 'utf8');
}
diff --git a/core/adminSettings/settingsManager.ts b/core/adminSettings/settingsManager.ts
index d5eb3a6..f03a5d5 100644
--- a/core/adminSettings/settingsManager.ts
+++ b/core/adminSettings/settingsManager.ts
@@ -1,8 +1,11 @@
-import { AuthService } from "../auth/jwt";
+import { AuthService, resolveDefaultJwtSecret } from "../auth/jwt";
import { DataBufferEngine } from "../data/buffer";
import { DatabaseEngine } from "../Database/database";
import { SDKManager } from "../sdk/sdkManager";
-import { SecurityEngine } from "../Security/security";
+import {
+ SecurityEngine,
+ getInternalIngestionHeaders,
+} from "../Security/security";
import { CSVStorageManager } from "../Storage/csvStorage";
export interface AdminSettings {
@@ -35,9 +38,7 @@ function getDefaultSettings(): AdminSettings {
batchFlushThreshold: 500,
batchFlushIntervalMs: 2000,
fileRotationMb: 100,
- jwtSecret:
- process.env.DATASTACK_JWT_SECRET ||
- "datastack-jwt-default-secret-key-32-chars-long",
+ jwtSecret: resolveDefaultJwtSecret(),
enableIdempotency: true,
autoPruneDays: 30,
updatedAt: new Date().toISOString(),
@@ -62,7 +63,9 @@ export class SettingsManager {
AuthService.setSecret(settings.jwtSecret);
}
CSVStorageManager.maxFileSizeBytes = settings.fileRotationMb * 1024 * 1024;
- CSVStorageManager.pruneExpiredPartitions(settings.autoPruneDays);
+ if (settings.autoPruneDays > 0) {
+ CSVStorageManager.pruneExpiredPartitions(settings.autoPruneDays);
+ }
}
private static async syncFromDatabase(): Promise {
@@ -83,6 +86,10 @@ export class SettingsManager {
if (result.rowCount && result.rowCount > 0) {
const row = result.rows[0];
+ const parsedPruneDays =
+ row.auto_prune_days !== undefined && row.auto_prune_days !== null
+ ? Number(row.auto_prune_days)
+ : NaN;
const loaded: AdminSettings = {
serverPort: Number(row.server_port) || 4000,
webPort: Number(row.web_port) || 3000,
@@ -91,12 +98,9 @@ export class SettingsManager {
batchFlushThreshold: Number(row.batch_flush_threshold) || 500,
batchFlushIntervalMs: Number(row.batch_flush_interval_ms) || 2000,
fileRotationMb: Number(row.file_rotation_mb) || 100,
- jwtSecret:
- String(row.jwt_secret || "") ||
- process.env.DATASTACK_JWT_SECRET ||
- "datastack-jwt-default-secret-key-32-chars-long",
+ jwtSecret: String(row.jwt_secret || "").trim() || resolveDefaultJwtSecret(),
enableIdempotency: Boolean(row.enable_idempotency),
- autoPruneDays: Number(row.auto_prune_days) || 30,
+ autoPruneDays: Number.isFinite(parsedPruneDays) ? parsedPruneDays : 30,
updatedAt: row.updated_at
? new Date(row.updated_at as string | Date).toISOString()
: new Date().toISOString(),
@@ -214,8 +218,46 @@ export class SettingsManager {
};
}
+ private static async notifyIngestionSettingsSync(
+ settings: AdminSettings,
+ authJwtSecret: string,
+ jwtSecretChanged: boolean,
+ ): Promise {
+ const headers = getInternalIngestionHeaders(authJwtSecret);
+ if (!headers["x-datastack-internal-secret"]) return;
+
+ const ingestionBaseUrl = (
+ process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
+ ).replace(/\/$/, "");
+
+ try {
+ await fetch(`${ingestionBaseUrl}/internal/settings-sync`, {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ ...headers,
+ },
+ body: JSON.stringify({
+ maxEventsPerSec: settings.maxEventsPerSec,
+ maxPayloadSizeKb: settings.maxPayloadSizeKb,
+ batchFlushThreshold: settings.batchFlushThreshold,
+ batchFlushIntervalMs: settings.batchFlushIntervalMs,
+ fileRotationMb: settings.fileRotationMb,
+ enableIdempotency: settings.enableIdempotency,
+ autoPruneDays: settings.autoPruneDays,
+ ...(jwtSecretChanged ? { jwtSecret: settings.jwtSecret } : {}),
+ }),
+ cache: "no-store",
+ signal: AbortSignal.timeout(1500),
+ });
+ } catch {
+ // Port 4000 may not be running in isolated single-process dev mode
+ }
+ }
+
public static updateSettings(updates: Partial): AdminSettings {
const current = getSettingsState();
+ const previousJwtSecret = current.jwtSecret;
const jwtSecretChanged =
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
const nextSettings: AdminSettings = {
@@ -229,6 +271,13 @@ export class SettingsManager {
SDKManager.resignAllClientTokens();
}
void this.persistSettingsToDb(nextSettings);
+ void this.notifyIngestionSettingsSync(
+ nextSettings,
+ previousJwtSecret,
+ jwtSecretChanged,
+ ).then(() =>
+ jwtSecretChanged ? SDKManager.syncAllClientsToIngestionAsync() : undefined,
+ );
return { ...nextSettings };
}
@@ -236,6 +285,7 @@ export class SettingsManager {
updates: Partial,
): Promise {
const current = await this.getSettingsAsync();
+ const previousJwtSecret = current.jwtSecret;
const jwtSecretChanged =
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
const nextSettings: AdminSettings = {
@@ -249,6 +299,14 @@ export class SettingsManager {
await SDKManager.resignAllClientTokensAsync();
}
await this.persistSettingsToDb(nextSettings);
+ await this.notifyIngestionSettingsSync(
+ nextSettings,
+ previousJwtSecret,
+ jwtSecretChanged,
+ );
+ if (jwtSecretChanged) {
+ await SDKManager.syncAllClientsToIngestionAsync();
+ }
return { ...nextSettings };
}
}
diff --git a/core/apiEngine/apiService.ts b/core/apiEngine/apiService.ts
index 8deae18..1090f1a 100644
--- a/core/apiEngine/apiService.ts
+++ b/core/apiEngine/apiService.ts
@@ -1,44 +1,93 @@
+import { SettingsManager } from '../adminSettings/settingsManager';
import { SDKManager } from '../sdk/sdkManager';
import { CSVStorageManager } from '../Storage/csvStorage';
import { DataBufferEngine } from '../data/buffer';
-import { SecurityEngine } from '../Security/security';
+import {
+ SecurityEngine,
+ getInternalIngestionHeaders,
+} from '../Security/security';
export class ApiEngine {
/**
* Retrieves overall telemetry and health metrics for the dashboard
*/
public static async getDashboardMetrics() {
- const storageStats = CSVStorageManager.getStorageStats();
- const localBufferStats = DataBufferEngine.getBufferStats();
- const clients = await SDKManager.listClientsAsync();
+ const settings = await SettingsManager.getSettingsAsync();
+ const [storageStats, localBufferStats, clients] = await Promise.all([
+ CSVStorageManager.getStorageStatsAsync(),
+ DataBufferEngine.getBufferStatsAsync(),
+ SDKManager.listClientsAsync(),
+ ]);
const activeClients = clients.filter((c) => c.status === 'active').length;
let activeSockets = 0;
- let ingestionBufferSize = 0;
+ let ingestionLocalBufferSize = 0;
+ let ingestionRedisBufferSize = 0;
+ let ingestionRedisConnected = false;
+ let remoteSecurityStats: {
+ activeIdempotencyKeys?: number;
+ rateLimitedClientsCount?: number;
+ maxPayloadSizeBytes?: number;
+ } | null = null;
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
).replace(/\/$/, '');
+ const internalHeaders = getInternalIngestionHeaders();
try {
- const res = await fetch(`${ingestionBaseUrl}/health`, {
+ const res = await fetch(`${ingestionBaseUrl}/internal/metrics`, {
+ headers: internalHeaders,
cache: 'no-store',
+ signal: AbortSignal.timeout(1000),
});
if (res.ok) {
const data = await res.json();
activeSockets = Number(data.activeSockets) || 0;
- ingestionBufferSize = Number(data.bufferSize) || 0;
+ ingestionRedisConnected = Boolean(data.redisConnected);
+ if (data.localBufferSize !== undefined) {
+ ingestionLocalBufferSize = Number(data.localBufferSize) || 0;
+ ingestionRedisBufferSize = Number(data.redisBufferSize) || 0;
+ } else {
+ ingestionLocalBufferSize = Number(data.bufferSize) || 0;
+ }
+ if (data.securityStats && typeof data.securityStats === 'object') {
+ remoteSecurityStats = data.securityStats;
+ }
}
} catch {
// Ingestion server may not be running in single-process dev mode
activeSockets = 0;
- ingestionBufferSize = 0;
+ ingestionLocalBufferSize = 0;
+ ingestionRedisBufferSize = 0;
}
- const securityStats = SecurityEngine.getStats();
+ const localSecurityStats = SecurityEngine.getStats();
+ const securityStats = {
+ activeIdempotencyKeys:
+ localSecurityStats.activeIdempotencyKeys +
+ (Number(remoteSecurityStats?.activeIdempotencyKeys) || 0),
+ rateLimitedClientsCount:
+ localSecurityStats.rateLimitedClientsCount +
+ (Number(remoteSecurityStats?.rateLimitedClientsCount) || 0),
+ maxPayloadSizeBytes: Math.min(
+ localSecurityStats.maxPayloadSizeBytes,
+ Number(remoteSecurityStats?.maxPayloadSizeBytes) ||
+ localSecurityStats.maxPayloadSizeBytes,
+ ),
+ };
+
+ const effectiveRedisBufferSize = Math.max(
+ localBufferStats.redisBufferSize || 0,
+ ingestionRedisBufferSize,
+ );
const bufferStats = {
...localBufferStats,
+ redisConnected: localBufferStats.redisConnected || ingestionRedisConnected,
+ redisBufferSize: effectiveRedisBufferSize,
currentBufferSize:
- localBufferStats.currentBufferSize + ingestionBufferSize,
+ localBufferStats.localBufferSize +
+ ingestionLocalBufferSize +
+ effectiveRedisBufferSize,
};
return {
@@ -49,8 +98,8 @@ export class ApiEngine {
storageStats,
securityStats,
runtime: {
- serverPort: 4000,
- webPort: 3000,
+ serverPort: settings.serverPort || 4000,
+ webPort: settings.webPort || 3000,
uptimeSeconds: Math.floor(process.uptime()),
memoryUsageMb: (process.memoryUsage().rss / (1024 * 1024)).toFixed(1),
nodeVersion: process.version,
@@ -59,7 +108,7 @@ export class ApiEngine {
}
/**
- * Filters and streams stored CSV data for export
+ * Filters and lists stored CSV files for export
*/
public static getExportFiles(clientId?: string) {
const allFiles = CSVStorageManager.listAllFiles();
@@ -68,6 +117,14 @@ export class ApiEngine {
}
return allFiles;
}
+
+ public static async getExportFilesAsync(clientId?: string) {
+ const allFiles = await CSVStorageManager.listAllFilesAsync();
+ if (clientId && clientId !== 'all') {
+ return allFiles.filter((f) => f.clientId === clientId);
+ }
+ return allFiles;
+ }
}
export default ApiEngine;
diff --git a/core/auth/adminSession.ts b/core/auth/adminSession.ts
index 2acc35e..8fac921 100644
--- a/core/auth/adminSession.ts
+++ b/core/auth/adminSession.ts
@@ -1,8 +1,23 @@
const encoder = new TextEncoder();
-interface AdminSessionPayload {
+const DEV_DEFAULT_SESSION_SECRET =
+ "datastack-admin-session-default-secret-32-chars";
+
+export interface AdminSessionPayload {
email: string;
expiresAt: number;
+ tokenVersion: number;
+}
+
+export function getAdminSessionSecret(): string | null {
+ const configured = process.env.DATASTACK_ADMIN_SESSION_SECRET?.trim() || "";
+ if (process.env.NODE_ENV === "production") {
+ return configured.length >= 32 ? configured : null;
+ }
+ if (configured.length > 0) {
+ return configured.length >= 32 ? configured : null;
+ }
+ return DEV_DEFAULT_SESSION_SECRET;
}
function encodeBase64Url(bytes: Uint8Array) {
@@ -34,9 +49,14 @@ export async function createAdminSessionToken(
email: string,
secret: string,
expiresAt: number,
+ tokenVersion = 1,
) {
+ const safeVersion =
+ Number.isInteger(tokenVersion) && tokenVersion >= 1 ? tokenVersion : 1;
const payload = encodeBase64Url(
- encoder.encode(JSON.stringify({ email, expiresAt })),
+ encoder.encode(
+ JSON.stringify({ email, expiresAt, tokenVersion: safeVersion }),
+ ),
);
const signature = await crypto.subtle.sign(
"HMAC",
@@ -50,6 +70,7 @@ export async function getAdminSessionPayload(
token: string,
secret: string,
): Promise {
+ if (!secret || secret.length < 32) return null;
const [payload, signature, extra] = token.split(".");
if (!payload || !signature || extra) return null;
@@ -68,7 +89,10 @@ export async function getAdminSessionPayload(
if (
typeof session.email === "string" &&
typeof session.expiresAt === "number" &&
- session.expiresAt > Date.now()
+ session.expiresAt > Date.now() &&
+ typeof session.tokenVersion === "number" &&
+ Number.isInteger(session.tokenVersion) &&
+ session.tokenVersion >= 1
) {
return session;
}
diff --git a/core/auth/adminUsers.ts b/core/auth/adminUsers.ts
index 60f2641..478d3fc 100644
--- a/core/auth/adminUsers.ts
+++ b/core/auth/adminUsers.ts
@@ -6,15 +6,22 @@ import {
timingSafeEqual,
} from "node:crypto";
import { Pool, type PoolClient } from "pg";
+import {
+ type AdminSessionPayload,
+ getAdminSessionPayload,
+ getAdminSessionSecret,
+} from "./adminSession";
const PASSWORD_HASH_BYTES = 64;
const PASSWORD_SALT_BYTES = 16;
const DATABASE_LOCK_ID = "72025102501";
const LOGIN_ATTEMPT_LIMIT = 5;
+const IP_LOGIN_ATTEMPT_LIMIT = 25;
const LOGIN_WINDOW_MS = 15 * 60 * 1000;
const LOGIN_BLOCK_MS = 15 * 60 * 1000;
const LOCAL_THROTTLE_MAX_ENTRIES = 5000;
const LOCAL_THROTTLE_CLEANUP_MS = 60 * 1000;
+const DEV_DEFAULT_SETUP_KEY = "datastack-default-setup-key-32-chars";
interface LoginThrottleEntry {
windowStartedAt: number;
@@ -28,11 +35,17 @@ interface LoginThrottleDecision {
retryAfterSeconds: number;
}
+export interface LoginThrottleBucket {
+ keyHash: string;
+ limit: number;
+}
+
interface InMemoryAdmin {
id: number;
email: string;
passwordSalt: Buffer;
passwordHash: Buffer;
+ tokenVersion: number;
createdAt: Date;
}
@@ -47,8 +60,12 @@ const localLoginThrottle = new Map();
let lastLocalThrottleCleanup = 0;
let lastDatabaseThrottleCleanup = 0;
+function isProductionEnv() {
+ return process.env.NODE_ENV === "production";
+}
+
function getPool(): Pool | null {
- const connectionString = process.env.DATABASE_URL;
+ const connectionString = process.env.DATABASE_URL?.trim();
if (!connectionString) {
return null;
}
@@ -74,17 +91,24 @@ async function ensureAdminTable() {
if (!pool) return;
globals.__dataStackAdminSchema ??= Promise.all([
- pool.query(
- `
+ pool
+ .query(
+ `
CREATE TABLE IF NOT EXISTS admin_users (
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
email TEXT NOT NULL UNIQUE,
password_salt BYTEA NOT NULL,
password_hash BYTEA NOT NULL,
+ token_version INTEGER NOT NULL DEFAULT 1,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`,
- ),
+ )
+ .then(() =>
+ pool.query(
+ `ALTER TABLE admin_users ADD COLUMN IF NOT EXISTS token_version INTEGER NOT NULL DEFAULT 1`,
+ ),
+ ),
pool.query(
`
CREATE TABLE IF NOT EXISTS admin_login_attempts (
@@ -121,8 +145,41 @@ function matchesSecret(candidate: string, expected: string) {
return timingSafeEqual(candidateHash, expectedHash);
}
-function createLoginThrottleKeys(email: string, secret: string) {
- return [createHmac("sha256", secret).update(`email:${email}`).digest("hex")];
+export function getAdminSetupKey(): string | null {
+ const configured = process.env.DATASTACK_ADMIN_SETUP_KEY?.trim() || "";
+ if (isProductionEnv()) {
+ return configured.length >= 32 ? configured : null;
+ }
+ if (configured.length > 0) {
+ return configured.length >= 32 ? configured : null;
+ }
+ return DEV_DEFAULT_SETUP_KEY;
+}
+
+export function createLoginThrottleBuckets(
+ email: string,
+ secret: string,
+ ip = "unknown",
+): LoginThrottleBucket[] {
+ const cleanEmail = email.trim().toLowerCase();
+ const cleanIp = ip.trim().toLowerCase() || "unknown";
+ const buckets: LoginThrottleBucket[] = [
+ {
+ keyHash: createHmac("sha256", secret)
+ .update(`ip_email:${cleanIp}:${cleanEmail}`)
+ .digest("hex"),
+ limit: LOGIN_ATTEMPT_LIMIT,
+ },
+ ];
+ if (cleanIp !== "unknown") {
+ buckets.push({
+ keyHash: createHmac("sha256", secret)
+ .update(`ip:${cleanIp}`)
+ .digest("hex"),
+ limit: IP_LOGIN_ATTEMPT_LIMIT,
+ });
+ }
+ return buckets;
}
function getAdvisoryLockId(keyHash: string) {
@@ -157,14 +214,14 @@ function cleanupLocalLoginThrottle(now: number) {
}
function consumeLocalLoginAttempt(
- keyHashes: string[],
+ buckets: LoginThrottleBucket[],
now: number,
): LoginThrottleDecision {
cleanupLocalLoginThrottle(now);
- const blockedUntil = keyHashes.reduce(
- (latest, keyHash) =>
- Math.max(latest, localLoginThrottle.get(keyHash)?.blockedUntil ?? 0),
+ const blockedUntil = buckets.reduce(
+ (latest, bucket) =>
+ Math.max(latest, localLoginThrottle.get(bucket.keyHash)?.blockedUntil ?? 0),
0,
);
if (blockedUntil > now) {
@@ -174,16 +231,16 @@ function consumeLocalLoginAttempt(
};
}
- for (const keyHash of keyHashes) {
- const previous = localLoginThrottle.get(keyHash);
+ for (const bucket of buckets) {
+ const previous = localLoginThrottle.get(bucket.keyHash);
const windowExpired =
!previous || now - previous.windowStartedAt >= LOGIN_WINDOW_MS;
const failedAttempts = windowExpired ? 1 : previous.failedAttempts + 1;
- localLoginThrottle.set(keyHash, {
+ localLoginThrottle.set(bucket.keyHash, {
windowStartedAt: windowExpired ? now : previous.windowStartedAt,
failedAttempts,
blockedUntil:
- failedAttempts >= LOGIN_ATTEMPT_LIMIT ? now + LOGIN_BLOCK_MS : 0,
+ failedAttempts >= bucket.limit ? now + LOGIN_BLOCK_MS : 0,
updatedAt: now,
});
}
@@ -192,12 +249,13 @@ function consumeLocalLoginAttempt(
}
async function consumeDatabaseLoginAttempt(
- keyHashes: string[],
+ buckets: LoginThrottleBucket[],
now: number,
): Promise {
const pool = getPool();
- if (!pool) return consumeLocalLoginAttempt(keyHashes, now);
+ if (!pool) return consumeLocalLoginAttempt(buckets, now);
await ensureAdminTable();
+ const keyHashes = buckets.map((b) => b.keyHash);
const client = await pool.connect();
try {
@@ -229,8 +287,8 @@ async function consumeDatabaseLoginAttempt(
};
}
- for (const keyHash of keyHashes) {
- const previous = entries.get(keyHash);
+ for (const bucket of buckets) {
+ const previous = entries.get(bucket.keyHash);
const windowStartedAt = previous
? new Date(previous.window_started_at as string | Date).getTime()
: 0;
@@ -243,7 +301,7 @@ async function consumeDatabaseLoginAttempt(
? new Date(now)
: new Date(windowStartedAt);
const nextBlockedUntil =
- failedAttempts >= LOGIN_ATTEMPT_LIMIT
+ failedAttempts >= bucket.limit
? new Date(now + LOGIN_BLOCK_MS)
: null;
@@ -256,7 +314,7 @@ async function consumeDatabaseLoginAttempt(
failed_attempts = EXCLUDED.failed_attempts,
blocked_until = EXCLUDED.blocked_until,
updated_at = NOW()`,
- [keyHash, nextWindowStartedAt, failedAttempts, nextBlockedUntil],
+ [bucket.keyHash, nextWindowStartedAt, failedAttempts, nextBlockedUntil],
);
}
@@ -279,13 +337,20 @@ async function consumeDatabaseLoginAttempt(
async function getOrInitInMemoryAdmin(): Promise {
if (!globals.__dataStackInMemoryAdmin) {
+ const envPassword = process.env.DATASTACK_ADMIN_PASSWORD;
+ const allowDemo =
+ !isProductionEnv() && process.env.DATASTACK_ALLOW_DEMO_ADMIN !== "false";
+
+ if (!envPassword && !allowDemo) {
+ throw new Error("Admin account has not completed initial setup.");
+ }
+
const defaultEmail = (
process.env.DATASTACK_ADMIN_EMAIL || "admin@datastack.dev"
)
.trim()
.toLowerCase();
- const defaultPassword =
- process.env.DATASTACK_ADMIN_PASSWORD || "admin12345678";
+ const defaultPassword = envPassword || "admin12345678";
const salt = randomBytes(PASSWORD_SALT_BYTES);
const hash = await derivePasswordHash(defaultPassword, salt);
globals.__dataStackInMemoryAdmin = {
@@ -293,6 +358,7 @@ async function getOrInitInMemoryAdmin(): Promise {
email: defaultEmail,
passwordSalt: salt,
passwordHash: hash,
+ tokenVersion: 1,
createdAt: new Date(),
};
}
@@ -302,24 +368,26 @@ async function getOrInitInMemoryAdmin(): Promise {
export async function consumeAdminLoginAttempt(
email: string,
secret: string,
+ ip = "unknown",
): Promise {
- const keyHashes = createLoginThrottleKeys(email, secret);
+ const buckets = createLoginThrottleBuckets(email, secret, ip);
const now = Date.now();
const pool = getPool();
if (!pool) {
- return consumeLocalLoginAttempt(keyHashes, now);
+ return consumeLocalLoginAttempt(buckets, now);
}
- try {
- return await consumeDatabaseLoginAttempt(keyHashes, now);
- } catch {
- return consumeLocalLoginAttempt(keyHashes, now);
- }
+ return await consumeDatabaseLoginAttempt(buckets, now);
}
-export async function clearAdminLoginAttempts(email: string, secret: string) {
- const keyHashes = createLoginThrottleKeys(email, secret);
+export async function clearAdminLoginAttempts(
+ email: string,
+ secret: string,
+ ip = "unknown",
+) {
+ const buckets = createLoginThrottleBuckets(email, secret, ip);
+ const keyHashes = buckets.map((b) => b.keyHash);
const pool = getPool();
if (!pool) {
@@ -327,50 +395,45 @@ export async function clearAdminLoginAttempts(email: string, secret: string) {
return;
}
+ await ensureAdminTable();
+ const client = await pool.connect();
try {
- await ensureAdminTable();
- const client = await pool.connect();
- try {
- await client.query("BEGIN");
- await lockLoginThrottleKeys(client, keyHashes);
- await client.query(
- "DELETE FROM admin_login_attempts WHERE key_hash = ANY($1::text[])",
- [keyHashes],
- );
- await client.query("COMMIT");
- } catch (error) {
- await client.query("ROLLBACK").catch(() => undefined);
- throw error;
- } finally {
- client.release();
- }
- } catch {
- keyHashes.forEach((keyHash) => localLoginThrottle.delete(keyHash));
+ await client.query("BEGIN");
+ await lockLoginThrottleKeys(client, keyHashes);
+ await client.query(
+ "DELETE FROM admin_login_attempts WHERE key_hash = ANY($1::text[])",
+ [keyHashes],
+ );
+ await client.query("COMMIT");
+ } catch (error) {
+ await client.query("ROLLBACK").catch(() => undefined);
+ throw error;
+ } finally {
+ client.release();
}
}
export function verifyAdminSetupKey(candidate: string) {
- const expected =
- process.env.DATASTACK_ADMIN_SETUP_KEY ||
- "datastack-default-setup-key-32-chars";
- if (!expected || expected.length < 32) return false;
+ const expected = getAdminSetupKey();
+ if (!expected || expected.length < 32 || !candidate) return false;
return matchesSecret(candidate, expected);
}
export async function isAdminSetupComplete() {
const pool = getPool();
if (!pool) {
- return true;
- }
- try {
- await ensureAdminTable();
- const result = await pool.query(
- "SELECT 1 FROM admin_users WHERE id = 1",
- );
- return result.rowCount === 1;
- } catch {
- return true;
+ if (globals.__dataStackInMemoryAdmin) {
+ return true;
+ }
+ if (isProductionEnv()) {
+ return Boolean(process.env.DATASTACK_ADMIN_PASSWORD);
+ }
+ return process.env.DATASTACK_ALLOW_DEMO_ADMIN !== "false";
}
+
+ await ensureAdminTable();
+ const result = await pool.query("SELECT 1 FROM admin_users WHERE id = 1");
+ return result.rowCount === 1;
}
export async function createInitialAdmin(email: string, password: string) {
@@ -380,136 +443,176 @@ export async function createInitialAdmin(email: string, password: string) {
const pool = getPool();
if (!pool) {
+ if (globals.__dataStackInMemoryAdmin) {
+ return false;
+ }
globals.__dataStackInMemoryAdmin = {
id: 1,
email: cleanEmail,
passwordSalt: salt,
passwordHash,
+ tokenVersion: 1,
createdAt: new Date(),
};
return true;
}
+ await ensureAdminTable();
+ const client = await pool.connect();
+
try {
- await ensureAdminTable();
- const client = await pool.connect();
+ await client.query("BEGIN");
+ await client.query("SELECT pg_advisory_xact_lock($1::bigint)", [
+ DATABASE_LOCK_ID,
+ ]);
- try {
- await client.query("BEGIN");
- await client.query("SELECT pg_advisory_xact_lock($1::bigint)", [
- DATABASE_LOCK_ID,
- ]);
-
- const existing = await client.query(
- "SELECT 1 FROM admin_users WHERE id = 1",
- );
- if (existing.rowCount) {
- await client.query("ROLLBACK");
- return false;
- }
-
- await client.query(
- `INSERT INTO admin_users (id, email, password_salt, password_hash)
- VALUES (1, $1, $2, $3)`,
- [cleanEmail, salt, passwordHash],
- );
- await client.query("COMMIT");
- return true;
- } catch (error) {
- await client.query("ROLLBACK").catch(() => undefined);
- throw error;
- } finally {
- client.release();
+ const existing = await client.query(
+ "SELECT 1 FROM admin_users WHERE id = 1",
+ );
+ if (existing.rowCount) {
+ await client.query("ROLLBACK");
+ return false;
}
- } catch {
- globals.__dataStackInMemoryAdmin = {
- id: 1,
- email: cleanEmail,
- passwordSalt: salt,
- passwordHash,
- createdAt: new Date(),
- };
+
+ await client.query(
+ `INSERT INTO admin_users (id, email, password_salt, password_hash, token_version)
+ VALUES (1, $1, $2, $3, 1)`,
+ [cleanEmail, salt, passwordHash],
+ );
+ await client.query("COMMIT");
return true;
+ } catch (error) {
+ await client.query("ROLLBACK").catch(() => undefined);
+ throw error;
+ } finally {
+ client.release();
}
}
-export async function verifyAdminCredentials(email: string, password: string) {
+export async function authenticateAdminCredentials(
+ email: string,
+ password: string,
+): Promise<{ authenticated: boolean; tokenVersion: number }> {
const cleanEmail = email.trim().toLowerCase();
const pool = getPool();
if (pool) {
- try {
- await ensureAdminTable();
- const result = await pool.query(
- `SELECT email, password_salt, password_hash
- FROM admin_users
- WHERE id = 1`,
- );
- if (result.rowCount && result.rowCount > 0) {
- const admin = result.rows[0] as {
- email: string;
- password_salt: Buffer;
- password_hash: Buffer;
- };
- const emailMatches = matchesSecret(
- cleanEmail,
- admin.email.trim().toLowerCase(),
- );
- const actualHash = await derivePasswordHash(
- password,
- admin.password_salt,
- );
- const passwordMatches = timingSafeEqual(
- actualHash,
- admin.password_hash,
- );
- return emailMatches && passwordMatches;
- }
- if (!process.env.DATASTACK_ADMIN_PASSWORD) {
- return false;
- }
- } catch {
- // Database connection error, fall back to in-memory admin
+ await ensureAdminTable();
+ const result = await pool.query(
+ `SELECT email, password_salt, password_hash, COALESCE(token_version, 1) AS token_version
+ FROM admin_users
+ WHERE id = 1`,
+ );
+ if (!result.rowCount || result.rowCount === 0) {
+ return { authenticated: false, tokenVersion: 0 };
}
+ const admin = result.rows[0] as {
+ email: string;
+ password_salt: Buffer;
+ password_hash: Buffer;
+ token_version: number;
+ };
+ const emailMatches = matchesSecret(
+ cleanEmail,
+ admin.email.trim().toLowerCase(),
+ );
+ const actualHash = await derivePasswordHash(password, admin.password_salt);
+ const passwordMatches = timingSafeEqual(actualHash, admin.password_hash);
+ return {
+ authenticated: emailMatches && passwordMatches,
+ tokenVersion: Number(admin.token_version) || 1,
+ };
}
const inMem = await getOrInitInMemoryAdmin();
if (!matchesSecret(cleanEmail, inMem.email)) {
- return false;
+ return { authenticated: false, tokenVersion: 0 };
}
const actualHash = await derivePasswordHash(password, inMem.passwordSalt);
- return timingSafeEqual(actualHash, inMem.passwordHash);
+ const authenticated = timingSafeEqual(actualHash, inMem.passwordHash);
+ return {
+ authenticated,
+ tokenVersion: authenticated ? inMem.tokenVersion : 0,
+ };
+}
+
+export async function verifyAdminCredentials(email: string, password: string) {
+ const result = await authenticateAdminCredentials(email, password);
+ return result.authenticated;
}
export async function getAdminAccountInfo(fallbackEmail?: string) {
const pool = getPool();
if (pool) {
- try {
- await ensureAdminTable();
- const result = await pool.query(
- `SELECT email, created_at FROM admin_users WHERE id = 1`,
- );
- if (result.rowCount && result.rowCount > 0) {
- const row = result.rows[0] as { email: string; created_at: Date | string };
- return {
- email: row.email,
- createdAt: new Date(row.created_at).toISOString(),
- storageMode: "PostgreSQL" as const,
- };
- }
- } catch {
- // Fall back to in-memory admin
+ await ensureAdminTable();
+ const result = await pool.query(
+ `SELECT email, created_at, COALESCE(token_version, 1) AS token_version
+ FROM admin_users
+ WHERE id = 1`,
+ );
+ if (result.rowCount && result.rowCount > 0) {
+ const row = result.rows[0] as {
+ email: string;
+ created_at: Date | string;
+ token_version: number;
+ };
+ return {
+ email: row.email,
+ createdAt: new Date(row.created_at).toISOString(),
+ tokenVersion: Number(row.token_version) || 1,
+ storageMode: "PostgreSQL" as const,
+ };
}
+ throw new Error("Admin account is not initialized.");
}
const inMem = await getOrInitInMemoryAdmin();
return {
email: fallbackEmail || inMem.email,
createdAt: inMem.createdAt.toISOString(),
+ tokenVersion: inMem.tokenVersion,
storageMode: "In-Memory" as const,
};
}
+export async function verifyRequestAdminSession(
+ cookieToken: string | undefined,
+): Promise<
+ | { valid: true; session: AdminSessionPayload; sessionSecret: string }
+ | { valid: false; reason: "UNCONFIGURED" | "UNAUTHORIZED" | "UNAVAILABLE" }
+> {
+ const sessionSecret = getAdminSessionSecret();
+ if (!sessionSecret) {
+ return { valid: false, reason: "UNCONFIGURED" };
+ }
+ if (!cookieToken) {
+ return { valid: false, reason: "UNAUTHORIZED" };
+ }
+
+ const session = await getAdminSessionPayload(cookieToken, sessionSecret);
+ if (!session) {
+ return { valid: false, reason: "UNAUTHORIZED" };
+ }
+
+ try {
+ const account = await getAdminAccountInfo();
+ const emailMatches = matchesSecret(
+ session.email.trim().toLowerCase(),
+ account.email.trim().toLowerCase(),
+ );
+ if (!emailMatches || session.tokenVersion !== account.tokenVersion) {
+ return { valid: false, reason: "UNAUTHORIZED" };
+ }
+ return { valid: true, session, sessionSecret };
+ } catch (error) {
+ if (getPool()) {
+ return { valid: false, reason: "UNAVAILABLE" };
+ }
+ void error;
+ return { valid: false, reason: "UNAUTHORIZED" };
+ }
+}
+
export async function updateAdminAccount({
currentEmail,
currentPassword,
@@ -534,29 +637,42 @@ export async function updateAdminAccount({
const pool = getPool();
if (pool) {
- try {
- await ensureAdminTable();
- const updated = await pool.query(
- `UPDATE admin_users
- SET email = $1, password_salt = $2, password_hash = $3
- WHERE id = 1`,
- [cleanEmail, salt, passwordHash],
- );
- if (updated.rowCount && updated.rowCount > 0) {
- return { ok: true as const, email: cleanEmail };
- }
- } catch {
- // Fall back to in-memory update
+ await ensureAdminTable();
+ const updated = await pool.query(
+ `UPDATE admin_users
+ SET email = $1,
+ password_salt = $2,
+ password_hash = $3,
+ token_version = COALESCE(token_version, 1) + 1
+ WHERE id = 1
+ RETURNING token_version`,
+ [cleanEmail, salt, passwordHash],
+ );
+ if (!updated.rowCount || updated.rowCount === 0) {
+ return { ok: false as const, error: "Admin account not found." };
}
+ return {
+ ok: true as const,
+ email: cleanEmail,
+ tokenVersion: Number(updated.rows[0].token_version) || 1,
+ };
}
const existing = await getOrInitInMemoryAdmin();
+ const nextVersion = existing.tokenVersion + 1;
globals.__dataStackInMemoryAdmin = {
id: 1,
email: cleanEmail,
passwordSalt: salt,
passwordHash,
+ tokenVersion: nextVersion,
createdAt: existing.createdAt,
};
- return { ok: true as const, email: cleanEmail };
+ return { ok: true as const, email: cleanEmail, tokenVersion: nextVersion };
+}
+
+export function resetAdminStateForTests() {
+ globals.__dataStackInMemoryAdmin = undefined;
+ localLoginThrottle.clear();
+ lastLocalThrottleCleanup = 0;
}
diff --git a/core/auth/jwt.ts b/core/auth/jwt.ts
index 0c76628..2b1131a 100644
--- a/core/auth/jwt.ts
+++ b/core/auth/jwt.ts
@@ -8,16 +8,31 @@ export interface ClientTokenPayload {
exp?: number;
}
-const DEFAULT_SECRET =
- process.env.DATASTACK_JWT_SECRET ||
+const DEV_DEFAULT_JWT_SECRET =
"datastack-jwt-default-secret-key-32-chars-long";
+export function resolveDefaultJwtSecret(): string {
+ const configured = process.env.DATASTACK_JWT_SECRET?.trim() || "";
+ if (process.env.NODE_ENV === "production") {
+ return configured.length >= 32 ? configured : "";
+ }
+ if (configured.length > 0) {
+ return configured.length >= 32 ? configured : "";
+ }
+ return DEV_DEFAULT_JWT_SECRET;
+}
+
export class AuthService {
- private static secret: string = DEFAULT_SECRET;
+ private static secret: string = resolveDefaultJwtSecret();
public static setSecret(newSecret: string) {
- if (newSecret && newSecret.trim().length > 0) {
- this.secret = newSecret.trim();
+ const trimmed = newSecret ? newSecret.trim() : "";
+ if (trimmed.length >= 32) {
+ this.secret = trimmed;
+ } else if (process.env.NODE_ENV !== "production" && trimmed.length > 0) {
+ this.secret = trimmed;
+ } else if (process.env.NODE_ENV === "production") {
+ this.secret = "";
}
}
@@ -25,6 +40,11 @@ export class AuthService {
return this.secret;
}
+ public static isConfigured(): boolean {
+ const minLen = process.env.NODE_ENV === "production" ? 32 : 1;
+ return this.secret.length >= minLen;
+ }
+
/**
* Generates a signed JWT for client SDK authentication.
*/
@@ -32,9 +52,9 @@ export class AuthService {
payload: { clientId: string; name?: string; rateLimit?: number },
expiresIn = "30d",
): string {
- if (!this.secret) {
+ if (!this.isConfigured()) {
throw new Error(
- "DATASTACK_JWT_SECRET must be configured before generating client tokens.",
+ "DATASTACK_JWT_SECRET (>= 32 characters) must be configured before generating client tokens.",
);
}
@@ -50,15 +70,18 @@ export class AuthService {
}
/**
- * Verifies and decodes incoming token from Socket.IO handshake.
+ * Verifies and decodes incoming token from Socket.IO handshake or HTTP ingestion.
*/
public static verifyClientToken(token: string): {
valid: boolean;
payload?: ClientTokenPayload;
error?: string;
} {
- if (!this.secret) {
- return { valid: false, error: "DATASTACK_JWT_SECRET is not configured." };
+ if (!this.isConfigured()) {
+ return {
+ valid: false,
+ error: "DATASTACK_JWT_SECRET (>= 32 characters) is not configured.",
+ };
}
try {
diff --git a/core/data/buffer.ts b/core/data/buffer.ts
index 9c499ed..6a5d2d7 100644
--- a/core/data/buffer.ts
+++ b/core/data/buffer.ts
@@ -1,8 +1,37 @@
+import crypto from 'node:crypto';
+import Redis from 'ioredis';
import { CSVStorageManager, DataEvent } from '../Storage/csvStorage';
export type EventSubscriber = (event: DataEvent) => void;
export type FlushCallback = (count: number, files: string[]) => void;
+export interface RedisBufferClient {
+ status?: string;
+ rpush(key: string, ...values: string[]): Promise;
+ lpush(key: string, ...values: string[]): Promise;
+ ltrim(key: string, start: number, stop: number): Promise;
+ lrange(key: string, start: number, stop: number): Promise;
+ llen(key: string): Promise;
+ set(
+ key: string,
+ value: string,
+ pxMode: 'PX',
+ ttlMs: number,
+ nxMode: 'NX',
+ ): Promise;
+ eval(script: string, numKeys: number, ...args: string[]): Promise;
+ quit?: () => Promise;
+ disconnect?: () => void;
+}
+
+export const REDIS_BUFFER_KEY = 'datastack:buffer:events';
+export const REDIS_RECENT_KEY = 'datastack:recent:events';
+export const REDIS_FLUSH_LOCK_KEY = 'datastack:flush:lock';
+const REDIS_RECENT_LIMIT = 500;
+const FLUSH_LOCK_TTL_MS = 15000;
+const UNLOCK_LUA_SCRIPT =
+ 'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end';
+
interface BufferGlobalState {
buffer: DataEvent[];
recentEvents: DataEvent[];
@@ -14,6 +43,11 @@ interface BufferGlobalState {
totalIngested: number;
totalFlushed: number;
isFlushing: boolean;
+ redisClient: RedisBufferClient | null | undefined;
+ redisUrlConfigured: string;
+ redisConnected: boolean;
+ redisTestOverride: RedisBufferClient | null | undefined;
+ pendingRedisWrites: Set>;
}
type BufferGlobals = typeof globalThis & {
@@ -36,13 +70,86 @@ function getState(): BufferGlobalState {
totalIngested: 0,
totalFlushed: 0,
isFlushing: false,
+ redisClient: undefined,
+ redisUrlConfigured: '',
+ redisConnected: false,
+ redisTestOverride: undefined,
+ pendingRedisWrites: new Set(),
};
}
return globals.__dataStackBufferState;
}
+function getRedisClient(): RedisBufferClient | null {
+ const state = getState();
+ if (state.redisTestOverride !== undefined) {
+ return state.redisTestOverride;
+ }
+
+ const configuredUrl = (process.env.REDIS_URL || '').trim();
+ if (!configuredUrl) {
+ if (state.redisClient && typeof state.redisClient.disconnect === 'function') {
+ try {
+ state.redisClient.disconnect();
+ } catch {
+ // ignore
+ }
+ }
+ state.redisClient = null;
+ state.redisUrlConfigured = '';
+ state.redisConnected = false;
+ return null;
+ }
+
+ if (state.redisClient && state.redisUrlConfigured === configuredUrl) {
+ return state.redisClient;
+ }
+
+ if (state.redisClient && typeof state.redisClient.disconnect === 'function') {
+ try {
+ state.redisClient.disconnect();
+ } catch {
+ // ignore
+ }
+ }
+
+ try {
+ const client = new Redis(configuredUrl, {
+ maxRetriesPerRequest: 2,
+ enableReadyCheck: true,
+ retryStrategy: (times: number) => Math.min(times * 250, 3000),
+ });
+
+ client.on('ready', () => {
+ state.redisConnected = true;
+ });
+ client.on('connect', () => {
+ state.redisConnected = true;
+ });
+ client.on('close', () => {
+ state.redisConnected = false;
+ });
+ client.on('end', () => {
+ state.redisConnected = false;
+ });
+ client.on('error', () => {
+ state.redisConnected = false;
+ });
+
+ state.redisClient = client as unknown as RedisBufferClient;
+ state.redisUrlConfigured = configuredUrl;
+ state.redisConnected = client.status === 'ready' || client.status === 'connect';
+ return state.redisClient;
+ } catch {
+ state.redisClient = null;
+ state.redisUrlConfigured = configuredUrl;
+ state.redisConnected = false;
+ return null;
+ }
+}
+
export class DataBufferEngine {
- private static maxRecentEvents = 200;
+ private static maxRecentEvents = REDIS_RECENT_LIMIT;
public static get batchFlushThreshold(): number {
return getState().batchFlushThreshold;
@@ -60,11 +167,46 @@ export class DataBufferEngine {
this.setFlushInterval(value);
}
+ public static setRedisClientForTests(client: RedisBufferClient | null | undefined): void {
+ const state = getState();
+ state.redisTestOverride = client;
+ state.redisConnected = Boolean(client);
+ }
+
+ public static async shutdown(): Promise {
+ const state = getState();
+ if (state.flushTimer) {
+ clearInterval(state.flushTimer);
+ state.flushTimer = null;
+ }
+ if (state.pendingRedisWrites.size > 0) {
+ await Promise.allSettled([...state.pendingRedisWrites]);
+ }
+ const client = state.redisClient;
+ state.redisClient = undefined;
+ state.redisUrlConfigured = '';
+ state.redisConnected = false;
+ state.redisTestOverride = undefined;
+ if (client) {
+ try {
+ if (typeof client.quit === 'function') {
+ await client.quit();
+ } else if (typeof client.disconnect === 'function') {
+ client.disconnect();
+ }
+ } catch {
+ if (typeof client.disconnect === 'function') {
+ client.disconnect();
+ }
+ }
+ }
+ }
+
public static init() {
const state = getState();
if (!state.flushTimer) {
state.flushTimer = setInterval(() => {
- void this.flush();
+ void this.flush().catch(() => undefined);
}, state.batchFlushIntervalMs);
}
@@ -96,26 +238,18 @@ export class DataBufferEngine {
}
state.flushTimer = setInterval(() => {
- void this.flush();
+ void this.flush().catch(() => undefined);
}, state.batchFlushIntervalMs);
}
- /**
- * Push incoming event to in-memory buffer.
- * If buffer hits threshold (e.g. 500 rows), triggers immediate batch flush.
- */
- public static push(event: DataEvent): void {
+ private static recordLocalMetaAndNotify(event: DataEvent): void {
const state = getState();
- state.buffer.push(event);
state.totalIngested += 1;
-
- // Cache in recent events for live dashboard viewer
state.recentEvents.unshift(event);
if (state.recentEvents.length > this.maxRecentEvents) {
state.recentEvents.pop();
}
- // Notify live subscribers
for (const sub of state.subscribers) {
try {
sub(event);
@@ -123,43 +257,188 @@ export class DataBufferEngine {
// Ignore subscriber error
}
}
+ }
- // Check if buffer threshold reached
+ private static async writeEventsToRedisOrFallback(events: DataEvent[]): Promise {
+ if (events.length === 0) return 0;
+ const state = getState();
+ const redis = getRedisClient();
+
+ if (redis) {
+ const serialized = events.map((e) => JSON.stringify(e));
+ try {
+ const bufferLen = await redis.rpush(REDIS_BUFFER_KEY, ...serialized);
+ await redis.lpush(REDIS_RECENT_KEY, ...serialized);
+ await redis.ltrim(REDIS_RECENT_KEY, 0, REDIS_RECENT_LIMIT - 1);
+ state.redisConnected = true;
+ return Number(bufferLen) + state.buffer.length;
+ } catch {
+ state.redisConnected = false;
+ }
+ }
+
+ state.buffer.push(...events);
+ return state.buffer.length;
+ }
+
+ /**
+ * Synchronous-signature push (queues to Redis asynchronously when configured, with automatic RAM fallback).
+ */
+ public static push(event: DataEvent): void {
+ const state = getState();
+ this.recordLocalMetaAndNotify(event);
+
+ const redis = getRedisClient();
+ if (redis) {
+ const op = this.writeEventsToRedisOrFallback([event]).then((queuedCount) => {
+ if (queuedCount >= state.batchFlushThreshold) {
+ void this.flush().catch(() => undefined);
+ }
+ });
+ state.pendingRedisWrites.add(op);
+ void op.finally(() => {
+ state.pendingRedisWrites.delete(op);
+ });
+ return;
+ }
+
+ state.buffer.push(event);
if (state.buffer.length >= state.batchFlushThreshold) {
- void this.flush();
+ void this.flush().catch(() => undefined);
}
}
/**
- * Flushes current buffer to CSV files via CSVStorageManager
+ * Async single-event push (awaited by API routes so Redis persistence is guaranteed before HTTP response).
+ */
+ public static async pushAsync(event: DataEvent): Promise {
+ await this.pushBatchAsync([event]);
+ }
+
+ /**
+ * Async batch push with shared Redis durability and RAM fallback.
+ */
+ public static async pushBatchAsync(events: DataEvent[]): Promise {
+ if (!events || events.length === 0) return;
+ const state = getState();
+ for (const evt of events) {
+ this.recordLocalMetaAndNotify(evt);
+ }
+
+ const queuedCount = await this.writeEventsToRedisOrFallback(events);
+ if (queuedCount >= state.batchFlushThreshold) {
+ void this.flush().catch(() => undefined);
+ }
+ }
+
+ /**
+ * Flushes queued events (shared Redis list under distributed lock + local RAM fallback) to partitioned CSV files.
+ * Redis LTRIM runs ONLY after CSVStorageManager.appendBatch succeeds so crashes never lose data.
*/
public static async flush(): Promise<{ count: number; files: string[] }> {
const state = getState();
- if (state.buffer.length === 0 || state.isFlushing) {
+ if (state.pendingRedisWrites.size > 0) {
+ await Promise.allSettled([...state.pendingRedisWrites]);
+ }
+ if (state.isFlushing) {
return { count: 0, files: [] };
}
state.isFlushing = true;
- const batchToFlush = [...state.buffer];
- state.buffer = [];
+ let totalWritten = 0;
+ const filesSet = new Set();
try {
- const result = await CSVStorageManager.appendBatch(batchToFlush);
- state.totalFlushed += result.written;
+ const redis = getRedisClient();
+ if (redis) {
+ const lockToken = crypto.randomUUID();
+ let lockAcquired = false;
- for (const listener of state.flushListeners) {
try {
- listener(result.written, result.filesUpdated);
+ const lockResult = await redis.set(
+ REDIS_FLUSH_LOCK_KEY,
+ lockToken,
+ 'PX',
+ FLUSH_LOCK_TTL_MS,
+ 'NX',
+ );
+ lockAcquired = lockResult === 'OK';
+ state.redisConnected = true;
} catch {
- // ignore
+ state.redisConnected = false;
+ lockAcquired = false;
+ }
+
+ if (lockAcquired) {
+ try {
+ const chunkSize = Math.max(500, state.batchFlushThreshold);
+ while (true) {
+ const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, chunkSize - 1);
+ if (!rawItems || rawItems.length === 0) {
+ break;
+ }
+
+ const parsedBatch: DataEvent[] = [];
+ for (const raw of rawItems) {
+ try {
+ const evt = JSON.parse(raw) as DataEvent;
+ if (evt && evt.eventId) {
+ parsedBatch.push(evt);
+ }
+ } catch {
+ // Ignore malformed legacy entry while still trimming it
+ }
+ }
+
+ if (parsedBatch.length > 0) {
+ const res = await CSVStorageManager.appendBatch(parsedBatch);
+ totalWritten += res.written;
+ for (const f of res.filesUpdated) filesSet.add(f);
+ }
+
+ // Trim strictly AFTER CSVStorageManager.appendBatch resolves safely
+ await redis.ltrim(REDIS_BUFFER_KEY, rawItems.length, -1);
+
+ if (rawItems.length < chunkSize) {
+ break;
+ }
+ }
+ } finally {
+ try {
+ await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
+ } catch {
+ // Lock auto-expires after FLUSH_LOCK_TTL_MS
+ }
+ }
}
}
- return { count: result.written, files: result.filesUpdated };
- } catch (err) {
- // On failure, re-queue to buffer front for retry
- state.buffer.unshift(...batchToFlush);
- throw err;
+ if (state.buffer.length > 0) {
+ const ramBatch = [...state.buffer];
+ state.buffer = [];
+ try {
+ const res = await CSVStorageManager.appendBatch(ramBatch);
+ totalWritten += res.written;
+ for (const f of res.filesUpdated) filesSet.add(f);
+ } catch (err) {
+ state.buffer.unshift(...ramBatch);
+ throw err;
+ }
+ }
+
+ if (totalWritten > 0) {
+ state.totalFlushed += totalWritten;
+ const filesUpdated = Array.from(filesSet);
+ for (const listener of state.flushListeners) {
+ try {
+ listener(totalWritten, filesUpdated);
+ } catch {
+ // ignore
+ }
+ }
+ }
+
+ return { count: totalWritten, files: Array.from(filesSet) };
} finally {
state.isFlushing = false;
}
@@ -194,10 +473,83 @@ export class DataBufferEngine {
return state.recentEvents.slice(0, limit);
}
+ public static async getRecentEventsAsync(
+ limit = 50,
+ filterClientId?: string,
+ ): Promise {
+ const state = getState();
+ const redis = getRedisClient();
+
+ if (redis) {
+ try {
+ const readCap = Math.min(REDIS_RECENT_LIMIT, Math.max(limit * 4, limit));
+ const rawItems = await redis.lrange(REDIS_RECENT_KEY, 0, readCap - 1);
+ state.redisConnected = true;
+
+ const redisEvents: DataEvent[] = [];
+ for (const raw of rawItems) {
+ try {
+ const parsed = JSON.parse(raw) as DataEvent;
+ if (!parsed || !parsed.eventId) continue;
+ if (filterClientId && filterClientId !== 'all' && parsed.clientId !== filterClientId) {
+ continue;
+ }
+ redisEvents.push(parsed);
+ if (redisEvents.length >= limit) break;
+ } catch {
+ // skip malformed JSON entry
+ }
+ }
+ if (redisEvents.length > 0 || state.recentEvents.length === 0) {
+ return redisEvents;
+ }
+ } catch {
+ state.redisConnected = false;
+ }
+ }
+
+ return this.getRecentEvents(limit, filterClientId);
+ }
+
public static getBufferStats() {
const state = getState();
return {
currentBufferSize: state.buffer.length,
+ localBufferSize: state.buffer.length,
+ redisBufferSize: 0,
+ redisEnabled: Boolean(getRedisClient()),
+ redisConnected: state.redisConnected,
+ batchFlushThreshold: state.batchFlushThreshold,
+ batchFlushIntervalMs: state.batchFlushIntervalMs,
+ totalIngested: state.totalIngested,
+ totalFlushed: state.totalFlushed,
+ recentEventsCached: state.recentEvents.length,
+ activeSubscribers: state.subscribers.size,
+ isFlushing: state.isFlushing,
+ };
+ }
+
+ public static async getBufferStatsAsync() {
+ const state = getState();
+ const redis = getRedisClient();
+ let redisBufferSize = 0;
+
+ if (redis) {
+ try {
+ redisBufferSize = Number(await redis.llen(REDIS_BUFFER_KEY)) || 0;
+ state.redisConnected = true;
+ } catch {
+ state.redisConnected = false;
+ redisBufferSize = 0;
+ }
+ }
+
+ return {
+ currentBufferSize: state.buffer.length + redisBufferSize,
+ localBufferSize: state.buffer.length,
+ redisBufferSize,
+ redisEnabled: Boolean(redis),
+ redisConnected: state.redisConnected,
batchFlushThreshold: state.batchFlushThreshold,
batchFlushIntervalMs: state.batchFlushIntervalMs,
totalIngested: state.totalIngested,
diff --git a/core/sdk/sdkManager.ts b/core/sdk/sdkManager.ts
index 6903ee8..674ce1a 100644
--- a/core/sdk/sdkManager.ts
+++ b/core/sdk/sdkManager.ts
@@ -1,5 +1,8 @@
+import { existsSync, rmSync } from "node:fs";
+import { resolve, sep } from "node:path";
import { AuthService } from "../auth/jwt";
import { DatabaseEngine } from "../Database/database";
+import { getInternalIngestionHeaders } from "../Security/security";
export interface ClientSDK {
clientId: string;
@@ -15,7 +18,9 @@ export interface ClientSDK {
type SDKDatabaseGlobals = typeof globalThis & {
__dataStackClientsMap?: Map;
+ __dataStackDeletedClients?: Set;
__dataStackClientsHydrated?: boolean;
+ __dataStackSyncedSocketEvents?: Map;
};
const globals = globalThis as SDKDatabaseGlobals;
@@ -27,6 +32,67 @@ function getClientsMap(): Map {
return globals.__dataStackClientsMap;
}
+function getDeletedClientsSet(): Set {
+ if (!globals.__dataStackDeletedClients) {
+ globals.__dataStackDeletedClients = new Set();
+ }
+ return globals.__dataStackDeletedClients;
+}
+
+function getSyncedSocketEventsMap(): Map {
+ if (!globals.__dataStackSyncedSocketEvents) {
+ globals.__dataStackSyncedSocketEvents = new Map();
+ }
+ return globals.__dataStackSyncedSocketEvents;
+}
+
+function serializeSyncClient(client: ClientSDK) {
+ return {
+ clientId: client.clientId,
+ name: client.name,
+ status: client.status,
+ token: client.token,
+ rateLimit: client.rateLimit,
+ };
+}
+
+async function notifyIngestionClientSyncAsync(
+ clientId: string,
+ action: "deleted" | "revoked" | "updated" | "registered",
+ client?: ClientSDK,
+): Promise {
+ const headers = getInternalIngestionHeaders();
+ if (!headers["x-datastack-internal-secret"]) return;
+ const ingestionBaseUrl = (
+ process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
+ ).replace(/\/$/, "");
+ try {
+ await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ ...headers,
+ },
+ body: JSON.stringify({
+ clientId,
+ action,
+ ...(client ? { client: serializeSyncClient(client) } : {}),
+ }),
+ signal: AbortSignal.timeout(1000),
+ });
+ } catch {
+ // Port 4000 may not be running in isolated single-process dev mode
+ }
+}
+
+function notifyIngestionClientSync(
+ clientId: string,
+ action: "deleted" | "revoked" | "updated" | "registered",
+ client?: ClientSDK,
+): void {
+ void notifyIngestionClientSyncAsync(clientId, action, client);
+}
+
function mapRowToClient(row: Record): ClientSDK {
return {
clientId: String(row.client_id),
@@ -54,11 +120,20 @@ export class SDKManager {
const ready = await DatabaseEngine.ensureSchema();
if (!ready) return null;
- const result = await pool.query(
- `SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
- FROM client_sdks
- ORDER BY created_at DESC`,
- );
+ const [result, deletedResult] = await Promise.all([
+ pool.query(
+ `SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
+ FROM client_sdks
+ ORDER BY created_at DESC`,
+ ),
+ pool.query(`SELECT client_id FROM deleted_clients`),
+ ]);
+
+ const deletedSet = getDeletedClientsSet();
+ deletedSet.clear();
+ for (const row of deletedResult.rows) {
+ deletedSet.add(String(row.client_id).trim().toLowerCase());
+ }
const map = getClientsMap();
map.clear();
@@ -84,12 +159,91 @@ export class SDKManager {
return Array.from(getClientsMap().values());
}
+ public static async syncAllClientsToIngestionAsync(): Promise {
+ const headers = getInternalIngestionHeaders();
+ if (!headers["x-datastack-internal-secret"]) return;
+ const ingestionBaseUrl = (
+ process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
+ ).replace(/\/$/, "");
+ const clients = Array.from(getClientsMap().values()).map(serializeSyncClient);
+ try {
+ await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ ...headers,
+ },
+ body: JSON.stringify({
+ action: "bulk_sync",
+ clients,
+ }),
+ signal: AbortSignal.timeout(1500),
+ });
+ } catch {
+ // Port 4000 may not be running in isolated single-process dev mode
+ }
+ }
+
public static async listClientsAsync(): Promise {
const fromDb = await this.syncFromDatabase();
if (fromDb !== null) {
return fromDb;
}
- return Array.from(getClientsMap().values());
+
+ const map = getClientsMap();
+ const headers = getInternalIngestionHeaders();
+ if (headers["x-datastack-internal-secret"]) {
+ const ingestionBaseUrl = (
+ process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
+ ).replace(/\/$/, "");
+ try {
+ const res = await fetch(`${ingestionBaseUrl}/internal/metrics`, {
+ headers,
+ cache: "no-store",
+ signal: AbortSignal.timeout(1000),
+ });
+ if (res.ok) {
+ const data = (await res.json()) as {
+ hasSyncedInMemoryClients?: boolean;
+ clientActivity?: Record<
+ string,
+ { totalEvents?: number; lastActive?: string }
+ >;
+ };
+ if (data.hasSyncedInMemoryClients === false && map.size > 0) {
+ await this.syncAllClientsToIngestionAsync();
+ }
+ if (data.clientActivity && typeof data.clientActivity === "object") {
+ const syncedMap = getSyncedSocketEventsMap();
+ for (const [cid, activity] of Object.entries(data.clientActivity)) {
+ const client = map.get(cid);
+ if (!client) continue;
+ const remoteTotal = Math.max(0, Number(activity?.totalEvents) || 0);
+ const prevSynced = syncedMap.get(cid) || 0;
+ const delta =
+ remoteTotal >= prevSynced
+ ? remoteTotal - prevSynced
+ : remoteTotal;
+ if (delta > 0) {
+ client.totalEvents += delta;
+ }
+ syncedMap.set(cid, remoteTotal);
+ if (
+ activity?.lastActive &&
+ (!client.lastActive || activity.lastActive > client.lastActive)
+ ) {
+ client.lastActive = activity.lastActive;
+ }
+ map.set(cid, client);
+ }
+ }
+ }
+ } catch {
+ // Port 4000 may not be running in isolated single-process dev mode
+ }
+ }
+
+ return Array.from(map.values());
}
public static getClient(clientId: string): ClientSDK | undefined {
@@ -152,8 +306,10 @@ export class SDKManager {
createdAt: new Date().toISOString(),
};
+ getDeletedClientsSet().delete(safeClientId);
getClientsMap().set(safeClientId, newClient);
void this.persistNewClient(newClient);
+ notifyIngestionClientSync(safeClientId, "registered", newClient);
return newClient;
}
@@ -163,6 +319,10 @@ export class SDKManager {
try {
await DatabaseEngine.ensureSchema();
+ await pool.query(
+ `DELETE FROM deleted_clients WHERE client_id = $1`,
+ [client.clientId],
+ );
await pool.query(
`INSERT INTO client_sdks (client_id, name, secret, token, rate_limit, status, total_events, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW())
@@ -218,8 +378,10 @@ export class SDKManager {
createdAt: new Date().toISOString(),
};
+ getDeletedClientsSet().delete(safeClientId);
getClientsMap().set(safeClientId, newClient);
await this.persistNewClient(newClient);
+ await notifyIngestionClientSyncAsync(safeClientId, "registered", newClient);
return newClient;
}
@@ -227,16 +389,20 @@ export class SDKManager {
const client = getClientsMap().get(clientId);
if (!client) return null;
- client.secret = `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`;
- client.token = AuthService.generateClientToken({
- clientId: client.clientId,
- name: client.name,
- rateLimit: client.rateLimit,
- });
+ const updated: ClientSDK = {
+ ...client,
+ secret: `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`,
+ token: AuthService.generateClientToken({
+ clientId: client.clientId,
+ name: client.name,
+ rateLimit: client.rateLimit,
+ }),
+ };
- getClientsMap().set(clientId, client);
- void this.persistClientCredentials(client);
- return client;
+ getClientsMap().set(clientId, updated);
+ void this.persistClientCredentials(updated);
+ notifyIngestionClientSync(clientId, "updated", updated);
+ return updated;
}
private static async persistClientCredentials(
@@ -264,16 +430,20 @@ export class SDKManager {
const existing = await this.getClientAsync(clientId);
if (!existing) return null;
- existing.secret = `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`;
- existing.token = AuthService.generateClientToken({
- clientId: existing.clientId,
- name: existing.name,
- rateLimit: existing.rateLimit,
- });
+ const updated: ClientSDK = {
+ ...existing,
+ secret: `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`,
+ token: AuthService.generateClientToken({
+ clientId: existing.clientId,
+ name: existing.name,
+ rateLimit: existing.rateLimit,
+ }),
+ };
- getClientsMap().set(clientId, existing);
- await this.persistClientCredentials(existing);
- return existing;
+ getClientsMap().set(clientId, updated);
+ await this.persistClientCredentials(updated);
+ await notifyIngestionClientSyncAsync(clientId, "updated", updated);
+ return updated;
}
public static updateClient(
@@ -283,14 +453,23 @@ export class SDKManager {
const client = getClientsMap().get(clientId);
if (!client) return null;
- if (updates.name !== undefined) client.name = updates.name.trim();
- if (updates.rateLimit !== undefined)
- client.rateLimit = Math.max(1, Number(updates.rateLimit) || 100);
- if (updates.status !== undefined) client.status = updates.status;
+ const updated: ClientSDK = {
+ ...client,
+ ...(updates.name !== undefined ? { name: updates.name.trim() } : {}),
+ ...(updates.rateLimit !== undefined
+ ? { rateLimit: Math.max(1, Number(updates.rateLimit) || 100) }
+ : {}),
+ ...(updates.status !== undefined ? { status: updates.status } : {}),
+ };
- getClientsMap().set(clientId, client);
- void this.persistClientUpdates(client);
- return client;
+ getClientsMap().set(clientId, updated);
+ void this.persistClientUpdates(updated);
+ notifyIngestionClientSync(
+ clientId,
+ updated.status === "revoked" ? "revoked" : "updated",
+ updated,
+ );
+ return updated;
}
private static async persistClientUpdates(client: ClientSDK): Promise {
@@ -317,19 +496,67 @@ export class SDKManager {
const client = await this.getClientAsync(clientId);
if (!client) return null;
- if (updates.name !== undefined) client.name = updates.name.trim();
- if (updates.rateLimit !== undefined)
- client.rateLimit = Math.max(1, Number(updates.rateLimit) || 100);
- if (updates.status !== undefined) client.status = updates.status;
+ const updated: ClientSDK = {
+ ...client,
+ ...(updates.name !== undefined ? { name: updates.name.trim() } : {}),
+ ...(updates.rateLimit !== undefined
+ ? { rateLimit: Math.max(1, Number(updates.rateLimit) || 100) }
+ : {}),
+ ...(updates.status !== undefined ? { status: updates.status } : {}),
+ };
- getClientsMap().set(clientId, client);
- await this.persistClientUpdates(client);
- return client;
+ getClientsMap().set(clientId, updated);
+ await this.persistClientUpdates(updated);
+ await notifyIngestionClientSyncAsync(
+ clientId,
+ updated.status === "revoked" ? "revoked" : "updated",
+ updated,
+ );
+ return updated;
+ }
+
+ public static isClientDeleted(clientId: string): boolean {
+ const normalized = String(clientId || "")
+ .trim()
+ .toLowerCase();
+ return getDeletedClientsSet().has(normalized);
+ }
+
+ public static async isClientDeletedAsync(clientId: string): Promise {
+ const normalized = String(clientId || "")
+ .trim()
+ .toLowerCase();
+ if (!normalized) return false;
+ if (getDeletedClientsSet().has(normalized)) {
+ return true;
+ }
+
+ const pool = DatabaseEngine.getPool();
+ if (!pool) return false;
+
+ await DatabaseEngine.ensureSchema();
+ const res = await pool.query(
+ `SELECT 1 FROM deleted_clients WHERE client_id = $1`,
+ [normalized],
+ );
+ if (res.rowCount && res.rowCount > 0) {
+ getDeletedClientsSet().add(normalized);
+ return true;
+ }
+ return false;
}
public static deleteClient(clientId: string): boolean {
- const deleted = getClientsMap().delete(clientId);
- void this.deleteClientFromDb(clientId);
+ const normalized = String(clientId || "")
+ .trim()
+ .toLowerCase();
+ const deleted = getClientsMap().delete(normalized);
+ getSyncedSocketEventsMap().delete(normalized);
+ if (deleted) {
+ getDeletedClientsSet().add(normalized);
+ }
+ void this.deleteClientFromDb(normalized);
+ notifyIngestionClientSync(normalized, "deleted");
return deleted;
}
@@ -337,21 +564,62 @@ export class SDKManager {
const pool = DatabaseEngine.getPool();
if (!pool) return false;
+ await DatabaseEngine.ensureSchema();
+ const dbClient = await pool.connect();
try {
- await DatabaseEngine.ensureSchema();
- const res = await pool.query(
+ await dbClient.query("BEGIN");
+ const res = await dbClient.query(
`DELETE FROM client_sdks WHERE client_id = $1`,
[clientId],
);
- return Boolean(res.rowCount && res.rowCount > 0);
- } catch {
+ if (res.rowCount && res.rowCount > 0) {
+ await dbClient.query(
+ `INSERT INTO deleted_clients (client_id, deleted_at)
+ VALUES ($1, NOW())
+ ON CONFLICT (client_id) DO UPDATE SET deleted_at = NOW()`,
+ [clientId],
+ );
+ await dbClient.query("COMMIT");
+ return true;
+ }
+ await dbClient.query("ROLLBACK");
return false;
+ } catch {
+ await dbClient.query("ROLLBACK").catch(() => undefined);
+ return false;
+ } finally {
+ dbClient.release();
}
}
- public static async deleteClientAsync(clientId: string): Promise {
- const existedInMem = getClientsMap().delete(clientId);
- const deletedFromDb = await this.deleteClientFromDb(clientId);
+ private static purgeClientStorageDir(clientId: string): void {
+ const safeId = clientId.replace(/[^a-z0-9_-]/gi, "_");
+ if (!safeId) return;
+ const storageRoot = resolve(process.cwd(), "storage", "data");
+ const targetDir = resolve(storageRoot, safeId);
+ if (!targetDir.startsWith(`${storageRoot}${sep}`)) return;
+ if (existsSync(targetDir)) {
+ rmSync(targetDir, { recursive: true, force: true });
+ }
+ }
+
+ public static async deleteClientAsync(
+ clientId: string,
+ options?: { purgeFiles?: boolean },
+ ): Promise {
+ const normalized = String(clientId || "")
+ .trim()
+ .toLowerCase();
+ const existedInMem = getClientsMap().delete(normalized);
+ getSyncedSocketEventsMap().delete(normalized);
+ const deletedFromDb = await this.deleteClientFromDb(normalized);
+ if (existedInMem || deletedFromDb) {
+ getDeletedClientsSet().add(normalized);
+ if (options?.purgeFiles) {
+ this.purgeClientStorageDir(normalized);
+ }
+ await notifyIngestionClientSyncAsync(normalized, "deleted");
+ }
return existedInMem || deletedFromDb;
}
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index 9508a8c..3b017dc 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -17,9 +17,12 @@ services:
DATASTACK_ADMIN_SESSION_SECRET: "${SERVICE_REALBASE64_64_ADMINSESSION:?Coolify must generate SERVICE_REALBASE64_64_ADMINSESSION}"
DATASTACK_ADMIN_SETUP_KEY: "${SERVICE_REALBASE64_64_ADMININSTALL:?Coolify must generate SERVICE_REALBASE64_64_ADMININSTALL}"
DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}"
+ REDIS_URL: "redis://redis:6379"
depends_on:
postgres:
condition: service_healthy
+ redis:
+ condition: service_healthy
restart: unless-stopped
healthcheck:
test:
@@ -48,9 +51,12 @@ services:
PORT_SOCKET: "4000"
DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}"
DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}"
+ REDIS_URL: "redis://redis:6379"
depends_on:
postgres:
condition: service_healthy
+ redis:
+ condition: service_healthy
restart: unless-stopped
healthcheck:
test:
@@ -85,7 +91,7 @@ services:
retries: 3
start_period: 10s
- # Reserved for the planned shared buffer integration; not consumed by the current runtime.
+ # Shared Redis ingestion buffer & distributed flush coordination
redis:
image: redis:7-alpine
restart: unless-stopped
diff --git a/middleware.ts b/middleware.ts
index fd6c6d4..420e9e5 100644
--- a/middleware.ts
+++ b/middleware.ts
@@ -1,4 +1,7 @@
-import { verifyAdminSessionToken } from "@/core/auth/adminSession";
+import {
+ getAdminSessionSecret,
+ verifyAdminSessionToken,
+} from "@/core/auth/adminSession";
import { NextRequest, NextResponse } from "next/server";
const SESSION_COOKIE = "datastack_admin_session";
@@ -13,28 +16,37 @@ function isProtectedApi(pathname: string, method: string) {
return pathname.startsWith("/api/");
}
-function isSameOriginRequest(request: NextRequest) {
- const originHeader = request.headers.get("origin");
+function matchesRequestHost(candidateUrl: string, request: NextRequest) {
const requestHost = request.headers.get("host");
const forwardedHost = request.headers
.get("x-forwarded-host")
?.split(",")[0]
?.trim();
- if (!originHeader || (!requestHost && !forwardedHost)) return false;
+ if (!requestHost && !forwardedHost) return false;
try {
- const origin = new URL(originHeader);
- const originHost = origin.host.toLowerCase();
- const matchesHost =
- (requestHost && originHost === requestHost.toLowerCase()) ||
- (forwardedHost && originHost === forwardedHost.toLowerCase());
-
- return Boolean(matchesHost);
+ const parsedHost = new URL(candidateUrl).host.toLowerCase();
+ return Boolean(
+ (requestHost && parsedHost === requestHost.toLowerCase()) ||
+ (forwardedHost && parsedHost === forwardedHost.toLowerCase()),
+ );
} catch {
return false;
}
}
+export function isSameOriginRequest(request: NextRequest) {
+ const originHeader = request.headers.get("origin");
+ if (originHeader) {
+ return matchesRequestHost(originHeader, request);
+ }
+ const refererHeader = request.headers.get("referer");
+ if (refererHeader) {
+ return matchesRequestHost(refererHeader, request);
+ }
+ return false;
+}
+
export async function middleware(request: NextRequest) {
const { pathname } = request.nextUrl;
const isLoginPage = pathname === "/vcon/login";
@@ -73,14 +85,19 @@ export async function middleware(request: NextRequest) {
return NextResponse.next();
}
- const sessionSecret =
- process.env.DATASTACK_ADMIN_SESSION_SECRET ||
- "datastack-admin-session-default-secret-32-chars";
+ const sessionSecret = getAdminSessionSecret();
+ if (!sessionSecret && isAdminApi) {
+ return NextResponse.json(
+ { error: "Admin authentication is not configured." },
+ { status: 503 },
+ );
+ }
+
const token = request.cookies.get(SESSION_COOKIE)?.value;
const authenticated = Boolean(
sessionSecret &&
- token &&
- (await verifyAdminSessionToken(token, sessionSecret)),
+ token &&
+ (await verifyAdminSessionToken(token, sessionSecret)),
);
if (isLoginPage) {
diff --git a/nginx/default.conf b/nginx/default.conf
index 22fa2fc..dc8c131 100644
--- a/nginx/default.conf
+++ b/nginx/default.conf
@@ -1,3 +1,18 @@
+map $http_upgrade $connection_upgrade {
+ default upgrade;
+ '' close;
+}
+
+map $http_x_forwarded_proto $forwarded_proto {
+ default $http_x_forwarded_proto;
+ '' $scheme;
+}
+
+map $http_x_forwarded_host $forwarded_host {
+ default $http_x_forwarded_host;
+ '' $host;
+}
+
upstream nextjs_upstream {
server 127.0.0.1:3000;
}
@@ -17,10 +32,12 @@ server {
proxy_pass http://socketio_upstream;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
- proxy_set_header Connection "upgrade";
+ proxy_set_header Connection $connection_upgrade;
proxy_set_header Host $host;
+ proxy_set_header X-Forwarded-Host $forwarded_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto $forwarded_proto;
proxy_read_timeout 86400;
}
@@ -29,7 +46,9 @@ server {
proxy_pass http://nextjs_upstream;
proxy_http_version 1.1;
proxy_set_header Host $host;
+ proxy_set_header X-Forwarded-Host $forwarded_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto $forwarded_proto;
}
}
diff --git a/package.json b/package.json
index 640bbc4..294be01 100644
--- a/package.json
+++ b/package.json
@@ -19,6 +19,7 @@
"cors": "^2.8.6",
"csv-parse": "^7.0.3",
"express": "^5.3.0",
+ "ioredis": "^6.0.0",
"json2csv": "^6.0.0-alpha.2",
"jsonwebtoken": "^9.0.3",
"lucide-react": "^0.553.0",
diff --git a/prisma/schema.prisma b/prisma/schema.prisma
index 9eed0ec..bfbc1a6 100644
--- a/prisma/schema.prisma
+++ b/prisma/schema.prisma
@@ -14,31 +14,25 @@ datasource db {
// All collected data is strictly written to CSV files in /storage/data/{clientId}/{YYYY-MM-DD}.csv
model ClientSDK {
- id String @id @default(cuid())
- clientId String @unique
+ clientId String @id @map("client_id")
name String
secret String
- rateLimit Int @default(100) // Max events per second
- status String @default("active") // active | revoked
- totalEvents BigInt @default(0)
- lastActive DateTime?
- createdAt DateTime @default(now())
- updatedAt DateTime @updatedAt
-
- apiKeys ApiKey[]
+ token String
+ rateLimit Int @default(100) @map("rate_limit") // Max events per second
+ status String @default("active") // active | revoked
+ totalEvents BigInt @default(0) @map("total_events")
+ lastActive DateTime? @map("last_active") @db.Timestamptz(6)
+ createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(6)
+ updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(6)
@@map("client_sdks")
}
-model ApiKey {
- id String @id @default(cuid())
- clientId String
- client ClientSDK @relation(fields: [clientId], references: [clientId], onDelete: Cascade)
- keyHash String
- expiresAt DateTime?
- createdAt DateTime @default(now())
+model DeletedClient {
+ clientId String @id @map("client_id")
+ deletedAt DateTime @default(now()) @map("deleted_at") @db.Timestamptz(6)
- @@map("api_keys")
+ @@map("deleted_clients")
}
model AdminUser {
@@ -46,35 +40,35 @@ model AdminUser {
email String @unique
passwordSalt Bytes @map("password_salt")
passwordHash Bytes @map("password_hash")
- createdAt DateTime @default(now()) @map("created_at")
+ tokenVersion Int @default(1) @map("token_version")
+ createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(6)
@@map("admin_users")
}
+model AdminLoginAttempt {
+ keyHash String @id @map("key_hash")
+ windowStartedAt DateTime @map("window_started_at") @db.Timestamptz(6)
+ failedAttempts Int @map("failed_attempts")
+ blockedUntil DateTime? @map("blocked_until") @db.Timestamptz(6)
+ updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(6)
+
+ @@map("admin_login_attempts")
+}
+
model AdminSettings {
- id String @id @default("default")
- serverPort Int @default(4000)
- webPort Int @default(3000)
- maxEventsPerSec Int @default(100)
- maxPayloadSizeKb Int @default(100)
- batchFlushThreshold Int @default(500)
- batchFlushIntervalMs Int @default(2000)
- fileRotationMb Int @default(100)
- jwtSecret String
- enableIdempotency Boolean @default(true)
- autoPruneDays Int @default(30)
- updatedAt DateTime @updatedAt
+ id String @id @default("default")
+ serverPort Int @default(4000) @map("server_port")
+ webPort Int @default(3000) @map("web_port")
+ maxEventsPerSec Int @default(100) @map("max_events_per_sec")
+ maxPayloadSizeKb Int @default(100) @map("max_payload_size_kb")
+ batchFlushThreshold Int @default(500) @map("batch_flush_threshold")
+ batchFlushIntervalMs Int @default(2000) @map("batch_flush_interval_ms")
+ fileRotationMb Int @default(100) @map("file_rotation_mb")
+ jwtSecret String @map("jwt_secret")
+ enableIdempotency Boolean @default(true) @map("enable_idempotency")
+ autoPruneDays Int @default(30) @map("auto_prune_days")
+ updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(6)
@@map("admin_settings")
}
-
-model AuditLog {
- id String @id @default(cuid())
- action String
- actor String
- details String? // JSON string
- ipAddress String?
- createdAt DateTime @default(now())
-
- @@map("audit_logs")
-}
diff --git a/public/apple-touch-icon.png b/public/apple-touch-icon.png
new file mode 100644
index 0000000000000000000000000000000000000000..4257521646624859381a7906c656648f1472d8a6
GIT binary patch
literal 632
zcmeAS@N?(olHy`uVBq!ia0vp^TR@nD4M^IaWiw)6V6yjgaSW-r_4f8g&ISV!*TB~i
zoz+VO)yvE|^%hEZEXep{c{KJx;jQaSue=YMVx`QH!3l1X9k8a1+k(nm+|s6C=RtvGG6KfjRM6TF!q3v84}Ch=d)zvT$$-lfGLW>)78&q
Iol`;+0Pc^)&Hw-a
literal 0
HcmV?d00001
diff --git a/public/icon-192.png b/public/icon-192.png
new file mode 100644
index 0000000000000000000000000000000000000000..580dd57adb6b8b71c9675fa34fe0fa9da1cf06c0
GIT binary patch
literal 718
zcmeAS@N?(olHy`uVBq!ia0vp^2SAvE4M+yv$zf+;V4CXb;uunK>+S8GybT5du8v8<
zkzXFN1RCyD>{eUAnj-UK#skw2TZP{V9s1?5d7FeIg9kD!X7;8g(*!3Il4lz0cG0b-&QTLMT(VN^KF>F`Pw
noFX2mGh|)?rVt?{cxlJp1G_)eOkX%_Iml(6u6{1-oD!M<>=8}S
literal 0
HcmV?d00001
diff --git a/public/icon-512.png b/public/icon-512.png
new file mode 100644
index 0000000000000000000000000000000000000000..37e7e1ee637dd9a1aaf5e2f2ac9b2866913f0557
GIT binary patch
literal 3695
zcmeHJziSjh6#ix$WeJ{Kz#_2-+u10Hg((zV!$rL&Xs{5HB9c-N8}-m;q7YO>3tI~t
z(LxllvXT4)LJAR-*l2AhB8uXBZ)R@pZg2PQRt1q%`+mRoeQ#!WZ=9GrRvX(j2B6kF
z-k1k8i5kf^dVFrZ{s!zg&}S3m+}BvH{iU`Z-><-^zZKA1$S$;nh9<@|&T>#WOU=T-fq
z!TWI>9T3?kd621J!$CQPlTu$A!AGhnm99?+HQEPWMzCS$oQvh%g&!|en
zij56y^WGhB31qvlA$FXHqjC$cKtXl2ZQUYiR~~sQ+tPd+W=O
zuQdS`;Kj9=Hl0rLSD}k+VNK#9e?f|za)2x_Uoz}9BtZwMFny>w!eZO2R9=efrB1=d
zJxwQ!j&x957(Pl}fW$mv`Y>~JmZb3tx6sb31b7$21UHNo_0o;;7-Fluf;u>y$F|1ZXRZ^geD
zsa*UQxu1997&?O-m!7C9IZRAXnP@8L69ahT
zMbq?++NWVGlmUt=cr5B+W$yOR-5sHy&JOqr_-f}A*mL(?}S^|Rf
zAm||n3zc9`R=nh5e?cpvm(pBHwV+~c39b0OnO)bCY`UxV;9;3%=Y8+{-kWb`cJ9Vh
zuFn`Wh^TM!+QdyF6|XAkXEA)6U2hW^uO=tPr)S^qzFj#-=K1XS$FbUl*N-1u-_|RS
zwvsO`{pn)*t42Q_UQ|dUGJ#ZxbRr9HNEat^@YZ`OVz`Nfk|b1iKimRPYU=jQ!EA4x
zKv^`VMJ>-IQ=5rMU2$QuISz#o*R}Q0m&4Ob-)BaeXFerWQ?S(NTt?H@n|U(Q=IbUhP_^^P;+klD$NQ>nPHEI^};wG9Cp
zD71E5&>eTKYr-#eyI&+c$Y-h|;o(v$%Lzw7ktQTef+9gk*h|-ttwhrWHG2`b??2x!
zU!Mb!-jO;gWFK{4#y63p0*3)Mm|KOEy;x3JxqM5g`KCZov{FaG;==bU4H
zEf;eJT0TsggMK&?mqv#SQZ~^KEVcu39=y4@#EC6}oSV5ATeeh;IjN+GN1#0|HVFt0
z@yv=E+p=
zTAmwOzHnol%_H6DtvZ{!&5mZ*C>
+
+
+
+
+
+
diff --git a/public/manifest.json b/public/manifest.json
index 7e8b269..c7e7d89 100644
--- a/public/manifest.json
+++ b/public/manifest.json
@@ -1,21 +1,31 @@
{
- "name": "DataStack - Realtime Data Platform",
+ "id": "/",
+ "name": "DataStack - Realtime Data Collection Platform",
"short_name": "DataStack",
- "description": "Self-hosted universal realtime data collection engine",
+ "description": "Self-hosted universal realtime data collection engine with Socket.IO ingestion, Redis buffer, JWT client auth, CSV batch rotation, and /vcon/ admin panel.",
"start_url": "/vcon/dashboard",
+ "scope": "/",
"display": "standalone",
- "background_color": "#0b0f17",
- "theme_color": "#2563eb",
+ "background_color": "#0a0a0b",
+ "theme_color": "#0a0a0b",
"icons": [
{
"src": "/icon-192.png",
"sizes": "192x192",
- "type": "image/png"
+ "type": "image/png",
+ "purpose": "any"
},
{
"src": "/icon-512.png",
"sizes": "512x512",
- "type": "image/png"
+ "type": "image/png",
+ "purpose": "any"
+ },
+ {
+ "src": "/icon-maskable-512.png",
+ "sizes": "512x512",
+ "type": "image/png",
+ "purpose": "maskable"
}
]
}
diff --git a/public/sw.js b/public/sw.js
index c6f4ff1..b7ac75c 100644
--- a/public/sw.js
+++ b/public/sw.js
@@ -1,5 +1,13 @@
-const CACHE_NAME = 'datastack-cache-v1';
-const STATIC_ASSETS = ['/', '/vcon/dashboard', '/manifest.json'];
+const CACHE_NAME = 'datastack-cache-v2';
+const STATIC_ASSETS = [
+ '/',
+ '/manifest.json',
+ '/icon.svg',
+ '/icon-192.png',
+ '/icon-512.png',
+ '/icon-maskable-512.png',
+ '/apple-touch-icon.png',
+];
self.addEventListener('install', (event) => {
event.waitUntil(
@@ -22,13 +30,43 @@ self.addEventListener('activate', (event) => {
});
self.addEventListener('fetch', (event) => {
- // Let WebSocket and API requests pass through
- if (event.request.url.includes('/socket.io/') || event.request.url.includes('/api/')) {
+ const { request } = event;
+ if (request.method !== 'GET') return;
+
+ let url;
+ try {
+ url = new URL(request.url);
+ } catch {
return;
}
+
+ // Only handle same-origin GET requests; never intercept WebSocket, HMR, or API routes
+ if (url.origin !== self.location.origin) return;
+ if (
+ url.pathname.startsWith('/socket.io/') ||
+ url.pathname.startsWith('/api/') ||
+ url.pathname.startsWith('/_next/webpack-hmr')
+ ) {
+ return;
+ }
+
event.respondWith(
- fetch(event.request).catch(() => {
- return caches.match(event.request);
- })
+ fetch(request)
+ .then((response) => {
+ if (response && response.ok && response.type === 'basic') {
+ const clone = response.clone();
+ caches.open(CACHE_NAME).then((cache) => cache.put(request, clone)).catch(() => {});
+ }
+ return response;
+ })
+ .catch(async () => {
+ const cached = await caches.match(request);
+ if (cached) return cached;
+ return new Response('Offline — network unavailable.', {
+ status: 503,
+ statusText: 'Service Unavailable',
+ headers: { 'Content-Type': 'text/plain; charset=utf-8' },
+ });
+ })
);
});
diff --git a/sdk-python/README.md b/sdk-python/README.md
index 193e68f..b75bfd4 100644
--- a/sdk-python/README.md
+++ b/sdk-python/README.md
@@ -6,24 +6,24 @@
Official Python client library for **DataStack — Self-Hosted Real-Time Telemetry & Event Logging Platform**, developed by **[Vib Tools](https://vib.tools/)**.
-`datastack-sdk` enables Python services, IoT sensor nodes, microservice workers, and research pipelines to stream structured JSON telemetry events to a DataStack server over Socket.IO with deterministic `threading.Event` acknowledgments.
+`datastack-sdk` is a compatibility alias for the canonical **`datastack`** package located at [`../sdk/python/datastack`](../sdk/python/datastack). Both `from datastack import DataStack` and `from datastack_sdk import DataStackClient` share a single source of truth with `config.datastack.py` auto-discovery, `CLIENT_ID` wiring, Socket.IO streaming, automatic HTTP fallback, offline disk spooling, and direct `.csv`/`.json`/`.jsonl` file streaming.
---
## Installation
-Install directly from the `sdk-python` directory:
+Install the canonical package directly from `sdk/python/datastack`:
+
+```bash
+pip install ./sdk/python/datastack
+```
+
+Or install the compatibility alias from `sdk-python`:
```bash
pip install ./sdk-python
```
-For editable development mode:
-
-```bash
-cd sdk-python && pip install -e .
-```
-
---
## Quick Start
@@ -34,10 +34,12 @@ from datastack_sdk import DataStackClient
# 1. Initialize the client with a signed JWT token from /vcon/clients or /vcon/sdk
client = DataStackClient(
server_url="http://localhost:4000",
+ api_url="http://localhost:3000",
token="YOUR_CLIENT_JWT_TOKEN",
+ client_id="YOUR_CLIENT_ID", # optional (auto-derived from JWT sub/clientId)
)
-# 2. Connect to the DataStack Socket.IO server
+# 2. Connect to the DataStack Socket.IO server (or auto HTTP fallback)
if client.connect():
# 3. Stream a single structured event and wait for server acknowledgment
res = client.sync(
diff --git a/sdk-python/datastack_sdk/__init__.py b/sdk-python/datastack_sdk/__init__.py
index ceae469..1ec1c4c 100644
--- a/sdk-python/datastack_sdk/__init__.py
+++ b/sdk-python/datastack_sdk/__init__.py
@@ -1,10 +1,29 @@
"""
-DataStack Python SDK (`datastack_sdk`)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Official Python client library for DataStack — Self-Hosted Real-Time Telemetry & Event Logging Platform.
+DataStack Python SDK Compatibility Alias (`datastack_sdk`).
+Re-exports all canonical symbols from `sdk/python/datastack` (`datastack`)
+so both `from datastack import DataStack` and legacy
+`from datastack_sdk import DataStackClient` share a single source of truth.
"""
-from .client import (
+import sys
+from pathlib import Path
+
+
+def _ensure_canonical_datastack_on_path() -> None:
+ try:
+ import datastack # noqa: F401
+ return
+ except ImportError:
+ repo_canonical = (
+ Path(__file__).resolve().parents[2] / "sdk" / "python" / "datastack"
+ )
+ if repo_canonical.is_dir() and str(repo_canonical) not in sys.path:
+ sys.path.insert(0, str(repo_canonical))
+
+
+_ensure_canonical_datastack_on_path()
+
+from datastack import ( # noqa: E402, F401
ConnectionState,
DataStack,
DataStackAuthError,
@@ -19,11 +38,10 @@ from .client import (
DataStackRateLimitError,
DiskSpooler,
HeartbeatMonitor,
+ __version__,
load_config,
)
-__version__ = "2.2.0"
-
__all__ = [
"DataStack",
"DataStackClient",
diff --git a/sdk-python/datastack_sdk/client.py b/sdk-python/datastack_sdk/client.py
index 2758060..e904b56 100644
--- a/sdk-python/datastack_sdk/client.py
+++ b/sdk-python/datastack_sdk/client.py
@@ -1,1193 +1,7 @@
-"""
-Official DataStack Python SDK Client (`DataStack` / `DataStackClient`) for `datastack_sdk`.
+"""Compatibility re-export of canonical `datastack.client`."""
-Self-contained implementation with 100% feature parity across both `datastack`
-and `datastack_sdk` packages:
-- Automatic discovery of `config.datastack.py` in the client app's source/working directory.
-- Primary real-time Socket.IO telemetry streaming (`sync_data`, `batch_sync`, `ping_verify`).
-- Automatic Heartbeat HTTP Fallback (`POST /api/events`) when WebSocket closes or disconnects.
-- Automatic WebSocket reconnection, handshake verification (`ping_verify`), and return to IDLE mode.
-- Offline Disk Spooling (`.datastack_spool/spool.jsonl`) when the client device loses internet,
- with automatic paced replay when connectivity returns.
-- Direct `.csv`, `.json`, and `.jsonl` file ingestion (`send_file()`) with deterministic SHA-256
- `eventId` deduplication, automatic batch chunking (`413 PAYLOAD_TOO_LARGE` auto-splitting),
- and lossless partial-batch rate-limit retry (`429 RATE_LIMIT_EXCEEDED`).
-"""
+from . import _ensure_canonical_datastack_on_path
-import csv
-import hashlib
-import json
-import logging
-import random
-import threading
-import time
-import urllib.error
-import urllib.request
-import uuid
-from datetime import datetime, timezone
-from pathlib import Path
-from typing import Any, Dict, Iterable, List, Optional, Union
+_ensure_canonical_datastack_on_path()
-try:
- import socketio # type: ignore
-except ImportError:
- socketio = None # type: ignore
-
-try:
- import requests # type: ignore
-except ImportError:
- requests = None # type: ignore
-
-from .config import DataStackConfig, load_config
-from .exceptions import (
- DataStackAuthError,
- DataStackConfigError,
- DataStackConnectionError,
- DataStackError,
- DataStackFileError,
- DataStackOfflineSpoolError,
- DataStackPayloadTooLargeError,
- DataStackRateLimitError,
-)
-from .heartbeat import ConnectionState, HeartbeatMonitor
-from .spool import DiskSpooler
-
-logger = logging.getLogger("datastack.client")
-
-
-def _utc_now_iso() -> str:
- return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
-
-
-def _make_event_id() -> str:
- return f"evt_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}"
-
-
-def _make_deterministic_file_event_id(
- file_stem: str, row_index: int, record: Any
-) -> str:
- raw = f"{file_stem}:{row_index}:{json.dumps(record, sort_keys=True, default=str, ensure_ascii=False)}"
- digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:12]
- safe_stem = "".join(
- ch if ch.isalnum() or ch in ("_", "-") else "_" for ch in file_stem
- )[:32]
- return f"file_{safe_stem}_{row_index}_{digest}"
-
-
-class DataStackClient:
- """
- Production-grade DataStack Python Client with `config.datastack.py` auto-loading,
- WebSocket + Heartbeat HTTP Fallback + Offline Spooling + Direct File Streaming.
- """
-
- def __init__(
- self,
- server_url: Optional[str] = None,
- token: Optional[str] = None,
- api_url: Optional[str] = None,
- config_path: Optional[str] = None,
- auto_connect: Optional[bool] = None,
- reconnection_attempts: Optional[int] = None,
- reconnection_delay: Optional[float] = None,
- **kwargs: Any,
- ) -> None:
- if reconnection_attempts is not None and "MAX_RETRIES" not in kwargs:
- kwargs["MAX_RETRIES"] = reconnection_attempts
- if (
- reconnection_delay is not None
- and "HEARTBEAT_INTERVAL_SEC" not in kwargs
- ):
- kwargs["HEARTBEAT_INTERVAL_SEC"] = reconnection_delay
-
- self.config: DataStackConfig = load_config(
- config_path=config_path,
- require_token=False,
- SERVER_URL=server_url,
- CLIENT_TOKEN=token,
- API_URL=api_url,
- AUTO_CONNECT=auto_connect,
- **kwargs,
- )
-
- self.server_url = self.config.server_url
- self.api_url = self.config.api_url
- self.token = self.config.client_token
- self.is_connected = False
- self._lock = threading.RLock()
- self._is_draining_spool = False
-
- # Offline Disk Spooler
- self.spooler = DiskSpooler(
- spool_dir=self.config.offline_spool_dir,
- max_size_mb=self.config.max_spool_size_mb,
- enabled=self.config.offline_spool_enabled,
- )
-
- # Heartbeat Monitor (starts in IDLE once connected, wakes on disconnect/offline)
- self.heartbeat = HeartbeatMonitor(
- interval_sec=self.config.heartbeat_interval_sec,
- recovery_callback=self._heartbeat_recovery_step,
- )
-
- # Socket.IO Client (disable built-in infinite loop so HeartbeatMonitor coordinates fallback + IDLE)
- self.sio = (
- socketio.Client(
- reconnection=False,
- logger=False,
- engineio_logger=False,
- )
- if socketio is not None
- else None
- )
- if self.sio is not None:
- self._register_socket_events()
-
- if (
- self.config.auto_connect
- and self.token
- and self.token != "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN"
- ):
- try:
- self.connect()
- except DataStackAuthError:
- raise
- except Exception as exc:
- logger.warning(
- "[DataStack] Initial connect entered fallback/recovery mode: %s",
- exc,
- )
-
- def __enter__(self) -> "DataStackClient":
- if (
- not self.is_connected
- and self.token
- and self.token != "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN"
- ):
- try:
- self.connect()
- except DataStackAuthError:
- raise
- except Exception:
- pass
- return self
-
- def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
- self.disconnect()
-
- def _register_socket_events(self) -> None:
- if self.sio is None:
- return
-
- @self.sio.event
- def connect() -> None:
- with self._lock:
- self.is_connected = True
- logger.info("[DataStack] WebSocket connected to %s", self.server_url)
-
- @self.sio.event
- def connect_error(data: Any) -> None:
- with self._lock:
- self.is_connected = False
- msg = str(data)
- if self._is_non_retryable_auth_error(msg):
- self.heartbeat.stop(reason=msg)
- else:
- self.heartbeat.trigger_fallback(reason=f"Socket connect_error: {msg}")
-
- @self.sio.event
- def disconnect() -> None:
- with self._lock:
- was_connected = self.is_connected
- self.is_connected = False
- if was_connected and self.heartbeat.state != ConnectionState.STOPPED:
- logger.warning(
- "[DataStack] WebSocket disconnected. Activating Heartbeat HTTP Fallback."
- )
- self.heartbeat.trigger_fallback(reason="WebSocket connection closed")
-
- @staticmethod
- def _is_non_retryable_auth_error(message: str) -> bool:
- upper = (message or "").upper()
- return any(
- code in upper
- for code in (
- "AUTH_REQUIRED",
- "AUTH_INVALID",
- "CLIENT_REVOKED",
- "TOKEN_ROTATED",
- )
- )
-
- def _ensure_token_present(self) -> None:
- if not self.token or self.token == "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN":
- raise DataStackConfigError(
- "Missing CLIENT_TOKEN. Set CLIENT_TOKEN in `config.datastack.py` or pass token='...'."
- )
-
- def connect(self, wait_timeout: Optional[float] = None) -> bool:
- """
- Establishes the primary WebSocket connection on Port 4000 and verifies the
- handshake via `ping_verify`. If WebSocket is unavailable and `ENABLE_HTTP_FALLBACK`
- is True, automatically enters `FALLBACK_HTTP` (or `OFFLINE_SPOOL` if offline)
- and starts the background Heartbeat Monitor.
- """
- self._ensure_token_present()
- self.heartbeat.start()
-
- try:
- if self._connect_and_verify_websocket(wait_timeout=wait_timeout):
- self._drain_offline_spool()
- self.heartbeat.set_idle_connected()
- return True
- except DataStackAuthError as auth_err:
- self.heartbeat.stop(reason=str(auth_err))
- raise
- except Exception as exc:
- logger.warning("[DataStack] WebSocket connect failed: %s", exc)
-
- # WebSocket was not reachable; check if HTTP fallback API is reachable
- if self.config.enable_http_fallback:
- if self._check_http_health():
- self.heartbeat.trigger_fallback(
- reason="WebSocket unreachable on connect; using HTTP fallback"
- )
- self._drain_offline_spool()
- return True
-
- if self.config.offline_spool_enabled:
- self.heartbeat.trigger_offline(
- reason="Both WebSocket and HTTP endpoints unreachable on connect"
- )
- return False
-
- raise DataStackConnectionError(
- f"Unable to connect to DataStack WebSocket ({self.server_url}) or HTTP API ({self.api_url})."
- )
-
- def disconnect(self) -> None:
- """Gracefully stops the Heartbeat Monitor and disconnects the WebSocket."""
- self.heartbeat.stop(reason="Client closed by application")
- with self._lock:
- self.is_connected = False
- if self.sio is not None and self.sio.connected:
- try:
- self.sio.disconnect()
- except Exception:
- pass
-
- def _connect_and_verify_websocket(
- self, wait_timeout: Optional[float] = None
- ) -> bool:
- if self.sio is None:
- return False
-
- effective_timeout = (
- float(wait_timeout)
- if wait_timeout is not None
- else self.config.request_timeout_sec
- )
-
- with self._lock:
- self.is_connected = False
- if self.sio.connected:
- try:
- self.sio.disconnect()
- except Exception:
- pass
-
- try:
- self.sio.connect(
- self.server_url,
- auth={"token": self.token},
- headers={"Authorization": f"Bearer {self.token}"},
- transports=["websocket", "polling"],
- wait_timeout=effective_timeout,
- )
- self.is_connected = True
- except Exception as exc:
- self.is_connected = False
- msg = str(exc)
- if self._is_non_retryable_auth_error(msg):
- raise DataStackAuthError(msg) from exc
- return False
-
- # Verify handshake & PostgreSQL client status via `ping_verify`
- return self._verify_websocket_handshake(timeout=effective_timeout)
-
- def _verify_websocket_handshake(
- self, timeout: Optional[float] = None
- ) -> bool:
- if self.sio is None or not self.sio.connected:
- return False
-
- ack_event = threading.Event()
- result_holder: Dict[str, Any] = {}
-
- def on_verify(resp: Dict[str, Any]) -> None:
- if isinstance(resp, dict):
- result_holder.update(resp)
- ack_event.set()
-
- wait_sec = timeout or self.config.request_timeout_sec
- try:
- self.sio.emit("ping_verify", {}, callback=on_verify)
- if not ack_event.wait(timeout=wait_sec):
- return False
- except Exception:
- return False
-
- if not result_holder.get("success"):
- err_msg = str(
- result_holder.get("error") or "Handshake verification failed"
- )
- if self._is_non_retryable_auth_error(err_msg):
- raise DataStackAuthError(err_msg)
- return False
-
- return result_holder.get("status") == "VERIFIED"
-
- def _heartbeat_recovery_step(self) -> bool:
- """
- Invoked periodically by `HeartbeatMonitor` ONLY when not in `CONNECTED_IDLE` mode.
- 1. Attempts to reconnect and verify (`ping_verify`) the primary WebSocket.
- 2. If WebSocket is verified, drains any offline spooled events and returns True
- so `HeartbeatMonitor` transitions back to `CONNECTED_IDLE` (IDLE mode).
- 3. If WebSocket is still down, checks if HTTP fallback (`Port 3000`) is online:
- - If HTTP is online, drains offline spooled events over HTTP and remains in `FALLBACK_HTTP`.
- - If HTTP returns `401/403` auth revocation, stops Heartbeat immediately.
- - If HTTP is also unreachable, sets state to `OFFLINE_SPOOL`.
- """
- self.heartbeat.set_reconnecting()
- try:
- if self._connect_and_verify_websocket():
- self._drain_offline_spool()
- return True
- except DataStackAuthError as auth_err:
- self.heartbeat.stop(reason=str(auth_err))
- return False
- except Exception:
- pass
-
- # WebSocket still unavailable; check HTTP availability
- try:
- if self.config.enable_http_fallback and self._check_http_health():
- self.heartbeat.trigger_fallback(
- reason="WebSocket still unavailable; HTTP fallback active"
- )
- self._drain_offline_spool()
- return False
- except DataStackAuthError as auth_err:
- self.heartbeat.stop(reason=str(auth_err))
- return False
-
- self.heartbeat.trigger_offline(
- reason="Device offline: WebSocket and HTTP endpoints unreachable"
- )
- return False
-
- def _check_http_health(self) -> bool:
- """
- Probes `GET {API_URL}/api/health` with the client's Bearer JWT token.
- Raises `DataStackAuthError` and halts Heartbeat if client credentials were
- revoked or rotated (`401` / `403`). Returns `True` ONLY on `2xx` with
- `status == 'online'`.
- """
- url = f"{self.api_url}/api/health"
- try:
- status_code, body = self._http_request(
- "GET", url, payload=None, timeout=3.0
- )
- except Exception:
- return False
-
- err_code = str(body.get("code") or "")
- err_msg = str(body.get("error") or f"HTTP {status_code}")
-
- if status_code in (401, 403) or self._is_non_retryable_auth_error(
- f"{err_code} {err_msg}"
- ):
- self.heartbeat.stop(reason=err_msg)
- raise DataStackAuthError(err_msg, code=err_code or "AUTH_ERROR")
-
- return 200 <= status_code < 300 and body.get("status") == "online"
-
- def _http_request(
- self,
- method: str,
- url: str,
- payload: Optional[Dict[str, Any]] = None,
- timeout: Optional[float] = None,
- ) -> tuple[int, Dict[str, Any]]:
- req_timeout = timeout or self.config.request_timeout_sec
- headers = {
- "Authorization": f"Bearer {self.token}",
- "Content-Type": "application/json",
- "Accept": "application/json",
- }
-
- if requests is not None:
- resp = requests.request(
- method=method,
- url=url,
- json=payload,
- headers=headers,
- timeout=req_timeout,
- )
- try:
- body = resp.json() if resp.text else {}
- except Exception:
- body = {"error": resp.text}
- return resp.status_code, (
- body if isinstance(body, dict) else {"data": body}
- )
-
- # Standard library urllib fallback when `requests` is not installed
- data_bytes = (
- json.dumps(payload, ensure_ascii=False).encode("utf-8")
- if payload is not None
- else None
- )
- req = urllib.request.Request(
- url=url, data=data_bytes, headers=headers, method=method
- )
- try:
- with urllib.request.urlopen(req, timeout=req_timeout) as response:
- raw = response.read().decode("utf-8")
- parsed = json.loads(raw) if raw else {}
- return (
- response.status,
- parsed if isinstance(parsed, dict) else {"data": parsed},
- )
- except urllib.error.HTTPError as http_err:
- raw_err = http_err.read().decode("utf-8", errors="replace")
- try:
- parsed_err = json.loads(raw_err) if raw_err else {}
- except Exception:
- parsed_err = {"error": raw_err or str(http_err)}
- return (
- http_err.code,
- parsed_err
- if isinstance(parsed_err, dict)
- else {"error": str(parsed_err)},
- )
-
- def _send_via_http_fallback(
- self, payload: Dict[str, Any]
- ) -> Dict[str, Any]:
- """
- Sends a single event, batch array (`{ batch: [...] }`), or file records
- (`{ fileName, records: [...] }`) over HTTP `POST {API_URL}/api/events`.
- """
- url = f"{self.api_url}/api/events"
- status_code, body = self._http_request("POST", url, payload=payload)
-
- if 200 <= status_code < 300 and body.get("success", True):
- return {
- **body,
- "success": True,
- "transport": "HTTP_FALLBACK",
- }
-
- errors_list = body.get("errors")
- first_err = (
- errors_list[0]
- if isinstance(errors_list, list) and len(errors_list) > 0
- else None
- )
- err_msg = str(body.get("error") or first_err or f"HTTP {status_code}")
- err_code = str(body.get("code") or "")
- combined = f"{err_code} {err_msg}"
-
- if status_code in (401, 403) or self._is_non_retryable_auth_error(
- combined
- ):
- self.heartbeat.stop(reason=err_msg)
- raise DataStackAuthError(err_msg, code=err_code or "AUTH_ERROR")
-
- if status_code == 413 or "PAYLOAD_TOO_LARGE" in combined:
- raise DataStackPayloadTooLargeError(err_msg)
-
- if status_code == 429 or "RATE_LIMIT_EXCEEDED" in combined:
- raise DataStackRateLimitError(err_msg)
-
- raise DataStackConnectionError(
- f"HTTP fallback error ({status_code}): {err_msg}"
- )
-
- def _emit_via_websocket(
- self,
- event_name: str,
- payload: Any,
- timeout: Optional[float] = None,
- ) -> Dict[str, Any]:
- if self.sio is None or not self.is_connected or not self.sio.connected:
- raise DataStackConnectionError(
- "WebSocket is not currently connected"
- )
-
- ack_event = threading.Event()
- response_data: Dict[str, Any] = {}
-
- def ack_callback(res: Dict[str, Any]) -> None:
- if isinstance(res, dict):
- response_data.update(res)
- ack_event.set()
-
- try:
- self.sio.emit(event_name, payload, callback=ack_callback)
- except Exception as exc:
- self.is_connected = False
- self.heartbeat.trigger_fallback(
- reason=f"WebSocket emit error: {exc}"
- )
- raise DataStackConnectionError(
- f"WebSocket emit failed: {exc}"
- ) from exc
-
- wait_sec = timeout or self.config.request_timeout_sec
- if not ack_event.wait(timeout=wait_sec):
- self.is_connected = False
- self.heartbeat.trigger_fallback(
- reason=f"WebSocket ACK timeout after {wait_sec}s"
- )
- raise DataStackConnectionError(
- f"WebSocket ACK timed out after {wait_sec}s"
- )
-
- if not response_data.get("success", False):
- errors_list = response_data.get("errors")
- first_err = (
- errors_list[0]
- if isinstance(errors_list, list) and len(errors_list) > 0
- else None
- )
- err_msg = str(
- response_data.get("error")
- or first_err
- or "WebSocket event rejected"
- )
- err_code = str(response_data.get("code") or "")
- combined = f"{err_code} {err_msg}"
-
- if self._is_non_retryable_auth_error(combined):
- self.heartbeat.stop(reason=err_msg)
- raise DataStackAuthError(err_msg, code=err_code or "AUTH_ERROR")
- if "PAYLOAD_TOO_LARGE" in combined:
- raise DataStackPayloadTooLargeError(err_msg)
- if "RATE_LIMIT_EXCEEDED" in combined:
- raise DataStackRateLimitError(err_msg)
- raise DataStackError(err_msg, code=err_code or "WS_REJECTED")
-
- return {
- **response_data,
- "transport": "WEBSOCKET",
- }
-
- def _execute_with_retry_and_fallback(
- self,
- ws_event: str,
- ws_payload: Any,
- http_payload: Dict[str, Any],
- spool_items: List[Dict[str, Any]],
- timeout: Optional[float] = None,
- allow_spool: bool = True,
- ) -> Dict[str, Any]:
- """
- Core resilient dispatcher:
- 1. Attempts WebSocket if connected (`CONNECTED_IDLE`).
- 2. On WebSocket failure/disconnect, triggers `HeartbeatMonitor` (`FALLBACK_HTTP`)
- and automatically sends via HTTP `POST /api/events`.
- 3. Applies exponential backoff with jitter on `RATE_LIMIT_EXCEEDED` or transient errors.
- 4. If both WebSocket and HTTP are unreachable (device offline) and `allow_spool` is True,
- spools events to `.datastack_spool/spool.jsonl` and triggers `OFFLINE_SPOOL`.
- """
- self._ensure_token_present()
- max_attempts = max(1, self.config.max_retries + 1)
- last_exception: Optional[Exception] = None
-
- for attempt in range(max_attempts):
- if attempt > 0:
- base_delay = (
- max(1.0, self.config.retry_backoff_base_sec)
- if isinstance(last_exception, DataStackRateLimitError)
- else self.config.retry_backoff_base_sec
- )
- delay = base_delay * (2 ** (attempt - 1))
- jitter = random.uniform(0.0, 0.2 * delay)
- time.sleep(delay + jitter)
-
- # 1. Try Primary WebSocket if connected
- if self.is_connected and self.sio is not None and self.sio.connected:
- try:
- res = self._emit_via_websocket(
- ws_event, ws_payload, timeout=timeout
- )
- if (
- allow_spool
- and not self._is_draining_spool
- and self.spooler.pending_count() > 0
- ):
- self._drain_offline_spool()
- return res
- except (DataStackAuthError, DataStackPayloadTooLargeError):
- raise
- except DataStackRateLimitError as rl_err:
- last_exception = rl_err
- continue
- except Exception as ws_err:
- last_exception = ws_err
- self.heartbeat.trigger_fallback(reason=str(ws_err))
-
- # 2. Try HTTP Fallback (`POST /api/events`)
- if self.config.enable_http_fallback:
- try:
- res = self._send_via_http_fallback(http_payload)
- if self.heartbeat.state in (
- ConnectionState.CONNECTED_IDLE,
- ConnectionState.OFFLINE_SPOOL,
- ):
- self.heartbeat.trigger_fallback(
- reason="WebSocket not connected; routing via HTTP fallback"
- )
- if (
- allow_spool
- and not self._is_draining_spool
- and self.spooler.pending_count() > 0
- ):
- self._drain_offline_spool()
- return res
- except (DataStackAuthError, DataStackPayloadTooLargeError):
- raise
- except DataStackRateLimitError as rl_err:
- last_exception = rl_err
- continue
- except Exception as http_err:
- last_exception = http_err
-
- # 3. If rate-limited or payload-rejected after all retries, raise directly
- if isinstance(
- last_exception,
- (DataStackRateLimitError, DataStackPayloadTooLargeError),
- ):
- raise last_exception
-
- # 4. Device is offline (both WebSocket and HTTP failed): Spool to local disk
- if (
- allow_spool
- and self.config.offline_spool_enabled
- and spool_items
- ):
- spooled_count = self.spooler.enqueue(spool_items)
- self.heartbeat.trigger_offline(
- reason=str(last_exception or "Network unreachable")
- )
- return {
- "success": True,
- "spooled": True,
- "spooledCount": spooled_count,
- "pendingSpoolTotal": self.spooler.pending_count(),
- "transport": "OFFLINE_SPOOL",
- }
-
- raise DataStackConnectionError(
- f"DataStack delivery failed after {max_attempts} attempts: {last_exception}"
- )
-
- def _drain_offline_spool(self) -> int:
- """
- Replays any offline spooled events from `.datastack_spool/spool.jsonl`
- once connectivity (WebSocket or HTTP fallback) is restored.
- Routes batches through `_send_formatted_batch_recursive(..., allow_spool=False)`
- and preserves any `unprocessedItems` so zero spooled events are ever lost.
- """
- if not self.config.offline_spool_enabled:
- return 0
-
- with self._lock:
- if self._is_draining_spool:
- return 0
- self._is_draining_spool = True
-
- total_flushed = 0
- try:
- while True:
- batch = self.spooler.peek_batch(
- limit=self.config.batch_chunk_size
- )
- if not batch:
- break
-
- res = self._send_formatted_batch_recursive(
- batch,
- timeout=self.config.request_timeout_sec,
- file_name=None,
- allow_spool=False,
- )
- processed_now = int(res.get("processed", 0))
- duplicates_now = int(res.get("duplicates", 0))
- unprocessed = res.get("unprocessedItems")
-
- if (
- not res.get("success", False)
- and processed_now == 0
- and duplicates_now == 0
- ):
- break
-
- self.spooler.ack_batch(len(batch))
- if isinstance(unprocessed, list) and len(unprocessed) > 0:
- self.spooler.enqueue(unprocessed)
- total_flushed += max(0, len(batch) - len(unprocessed))
- break
-
- total_flushed += len(batch)
- except Exception as exc:
- logger.debug("[DataStack] Spool drain paused: %s", exc)
- finally:
- with self._lock:
- self._is_draining_spool = False
-
- return total_flushed
-
- # =========================================================================
- # Public Telemetry Ingestion API
- # =========================================================================
-
- def send(
- self,
- data: Dict[str, Any],
- event_id: Optional[str] = None,
- timestamp: Optional[str] = None,
- timeout: Optional[float] = None,
- ) -> Dict[str, Any]:
- """
- Sends a single telemetry event. Automatically falls back to HTTP if WebSocket
- is disconnected, or spools to disk if the device is offline.
- """
- resolved_id = (event_id or "").strip() or _make_event_id()
- resolved_ts = (timestamp or "").strip() or _utc_now_iso()
- envelope = {
- "eventId": resolved_id,
- "timestamp": resolved_ts,
- "data": data,
- }
- return self._execute_with_retry_and_fallback(
- ws_event="sync_data",
- ws_payload=envelope,
- http_payload=envelope,
- spool_items=[envelope],
- timeout=timeout,
- allow_spool=True,
- )
-
- def sync(
- self,
- data: Dict[str, Any],
- event_id: Optional[str] = None,
- timeout: float = 5.0,
- ) -> Dict[str, Any]:
- """Alias for `send()` preserving backward compatibility."""
- return self.send(data=data, event_id=event_id, timeout=timeout)
-
- def send_batch(
- self,
- events: Optional[List[Dict[str, Any]]] = None,
- timeout: Optional[float] = None,
- file_name: Optional[str] = None,
- *,
- records: Optional[List[Dict[str, Any]]] = None,
- ) -> Dict[str, Any]:
- """
- Sends a batch of events (accepts either `events=[...]` or `records=[...]`).
- - If the batch triggers `413 PAYLOAD_TOO_LARGE`, automatically splits the batch
- in half recursively and retries.
- - If a subset of events in the batch hits `429 RATE_LIMIT_EXCEEDED`
- (`rateLimitedEventIds`), automatically backs off and retries the remaining
- rate-limited events so zero records are dropped.
- """
- source_list = events if events is not None else records
- if not isinstance(source_list, list) or len(source_list) == 0:
- return {"success": True, "processed": 0, "rejected": 0}
-
- formatted: List[Dict[str, Any]] = []
- for item in source_list:
- safe_item = item if isinstance(item, dict) else {"value": item}
- evt_id = (
- str(safe_item.get("eventId") or "").strip() or _make_event_id()
- )
- evt_ts = (
- str(safe_item.get("timestamp") or "").strip() or _utc_now_iso()
- )
- evt_data = safe_item.get("data", safe_item)
- formatted.append(
- {
- "eventId": evt_id,
- "timestamp": evt_ts,
- "data": evt_data,
- }
- )
-
- return self._send_formatted_batch_recursive(
- formatted,
- timeout=timeout,
- file_name=file_name,
- allow_spool=True,
- )
-
- def batch_sync(
- self,
- events: Optional[List[Dict[str, Any]]] = None,
- timeout: float = 10.0,
- *,
- records: Optional[List[Dict[str, Any]]] = None,
- ) -> Dict[str, Any]:
- """Alias for `send_batch()` supporting both `events` and `records` kwargs."""
- return self.send_batch(events=events, records=records, timeout=timeout)
-
- def _send_formatted_batch_recursive(
- self,
- formatted_batch: List[Dict[str, Any]],
- timeout: Optional[float] = None,
- file_name: Optional[str] = None,
- allow_spool: bool = True,
- ) -> Dict[str, Any]:
- http_payload: Dict[str, Any] = (
- {"fileName": file_name, "records": formatted_batch}
- if file_name
- else {"batch": formatted_batch}
- )
- try:
- res = self._execute_with_retry_and_fallback(
- ws_event="batch_sync",
- ws_payload=formatted_batch,
- http_payload=http_payload,
- spool_items=formatted_batch,
- timeout=timeout,
- allow_spool=allow_spool,
- )
- except DataStackPayloadTooLargeError:
- if len(formatted_batch) <= 1:
- raise
- mid = len(formatted_batch) // 2
- left_res = self._send_formatted_batch_recursive(
- formatted_batch[:mid],
- timeout=timeout,
- file_name=file_name,
- allow_spool=allow_spool,
- )
- right_res = self._send_formatted_batch_recursive(
- formatted_batch[mid:],
- timeout=timeout,
- file_name=file_name,
- allow_spool=allow_spool,
- )
- left_unprocessed = left_res.get("unprocessedItems") or []
- right_unprocessed = right_res.get("unprocessedItems") or []
- combined_unprocessed = [
- *(left_unprocessed if isinstance(left_unprocessed, list) else []),
- *(right_unprocessed if isinstance(right_unprocessed, list) else []),
- ]
- return {
- "success": bool(
- left_res.get("success", True)
- and right_res.get("success", True)
- ),
- "processed": int(left_res.get("processed", 0))
- + int(right_res.get("processed", 0)),
- "rejected": int(left_res.get("rejected", 0))
- + int(right_res.get("rejected", 0)),
- "duplicates": int(left_res.get("duplicates", 0))
- + int(right_res.get("duplicates", 0)),
- "spooledCount": int(left_res.get("spooledCount", 0))
- + int(right_res.get("spooledCount", 0)),
- "transport": right_res.get("transport")
- or left_res.get("transport"),
- **(
- {"unprocessedItems": combined_unprocessed}
- if len(combined_unprocessed) > 0
- else {}
- ),
- }
-
- # Handle partial-batch rate limiting (`rateLimitedEventIds`) so zero rows are dropped
- rate_limited_ids = res.get("rateLimitedEventIds")
- if isinstance(rate_limited_ids, list) and len(rate_limited_ids) > 0:
- pending_id_set = {str(eid) for eid in rate_limited_ids}
- remaining_items = [
- item
- for item in formatted_batch
- if str(item.get("eventId")) in pending_id_set
- ]
- total_processed = int(res.get("processed", 0))
- total_duplicates = int(res.get("duplicates", 0))
- total_spooled = int(res.get("spooledCount", 0))
- last_transport = res.get("transport")
-
- if self.config.max_retries > 0:
- for _retry_round in range(1, self.config.max_retries + 2):
- if not remaining_items:
- break
- wait_delay = max(
- 1.05, self.config.retry_backoff_base_sec
- ) + random.uniform(0.0, 0.15)
- time.sleep(wait_delay)
-
- sub_http_payload: Dict[str, Any] = (
- {"fileName": file_name, "records": remaining_items}
- if file_name
- else {"batch": remaining_items}
- )
- sub_res = self._execute_with_retry_and_fallback(
- ws_event="batch_sync",
- ws_payload=remaining_items,
- http_payload=sub_http_payload,
- spool_items=remaining_items,
- timeout=timeout,
- allow_spool=allow_spool,
- )
- total_processed += int(sub_res.get("processed", 0))
- total_duplicates += int(sub_res.get("duplicates", 0))
- total_spooled += int(sub_res.get("spooledCount", 0))
- last_transport = sub_res.get("transport") or last_transport
-
- if sub_res.get("spooled"):
- remaining_items = []
- break
-
- next_rl_ids = sub_res.get("rateLimitedEventIds")
- if (
- not isinstance(next_rl_ids, list)
- or len(next_rl_ids) == 0
- ):
- remaining_items = []
- break
- next_set = {str(eid) for eid in next_rl_ids}
- remaining_items = [
- item
- for item in remaining_items
- if str(item.get("eventId")) in next_set
- ]
-
- # If any items still remain rate-limited after retries and spooling is allowed, spool them so zero data is lost
- if (
- remaining_items
- and allow_spool
- and self.config.offline_spool_enabled
- ):
- spooled_now = self.spooler.enqueue(remaining_items)
- total_spooled += spooled_now
- remaining_items = []
-
- oversized_ids = res.get("oversizedEventIds")
- final_rejected = len(remaining_items) + (
- len(oversized_ids) if isinstance(oversized_ids, list) else 0
- )
- out: Dict[str, Any] = {
- "success": final_rejected == 0
- or total_processed > 0
- or total_spooled > 0,
- "processed": total_processed,
- "rejected": final_rejected,
- "duplicates": total_duplicates,
- "spooledCount": total_spooled,
- "transport": last_transport,
- }
- if remaining_items:
- out["unprocessedItems"] = remaining_items
- return out
-
- return res
-
- # =========================================================================
- # Direct File Streaming (.csv, .json, .jsonl / .ndjson)
- # =========================================================================
-
- def send_file(
- self,
- file_path: Union[str, Path],
- chunk_size: Optional[int] = None,
- extra_metadata: Optional[Dict[str, Any]] = None,
- ) -> Dict[str, Any]:
- """
- Streams a `.csv`, `.json`, or `.jsonl` (`.ndjson`) file directly to DataStack
- in safe chunks with deterministic SHA-256 idempotency IDs, automatic HTTP fallback,
- partial-batch rate-limit pacing, and offline spooling support.
- """
- path = Path(file_path).expanduser().resolve()
- if not path.exists() or not path.is_file():
- raise DataStackFileError(f"File not found: {path}")
-
- ext = path.suffix.lower()
- if ext not in (".csv", ".json", ".jsonl", ".ndjson"):
- raise DataStackFileError(
- f"Unsupported file extension '{ext}'. Supported formats: .csv, .json, .jsonl, .ndjson"
- )
-
- effective_chunk = max(
- 1, int(chunk_size or self.config.batch_chunk_size)
- )
- total_processed = 0
- total_rejected = 0
- total_duplicates = 0
- total_spooled = 0
- chunks_sent = 0
- last_transport = "WEBSOCKET"
-
- buffer: List[Dict[str, Any]] = []
- for idx, record in enumerate(self._iter_file_records(path, ext), start=1):
- enriched = (
- {**extra_metadata, **record}
- if extra_metadata and isinstance(record, dict)
- else record
- )
- existing_id = (
- str(enriched.get("eventId") or "").strip()
- if isinstance(enriched, dict)
- else ""
- )
- evt_id = existing_id or _make_deterministic_file_event_id(
- path.stem, idx, enriched
- )
- existing_ts = (
- str(enriched.get("timestamp") or "").strip()
- if isinstance(enriched, dict)
- else ""
- )
- evt_ts = existing_ts or _utc_now_iso()
-
- buffer.append(
- {
- "eventId": evt_id,
- "timestamp": evt_ts,
- "data": enriched,
- }
- )
- if len(buffer) >= effective_chunk:
- res = self.send_batch(buffer, file_name=path.name)
- total_processed += int(res.get("processed", 0))
- total_rejected += int(res.get("rejected", 0))
- total_duplicates += int(res.get("duplicates", 0))
- total_spooled += int(res.get("spooledCount", 0))
- last_transport = str(res.get("transport", last_transport))
- chunks_sent += 1
- buffer = []
-
- if buffer:
- res = self.send_batch(buffer, file_name=path.name)
- total_processed += int(res.get("processed", 0))
- total_rejected += int(res.get("rejected", 0))
- total_duplicates += int(res.get("duplicates", 0))
- total_spooled += int(res.get("spooledCount", 0))
- last_transport = str(res.get("transport", last_transport))
- chunks_sent += 1
-
- return {
- "success": True,
- "fileName": path.name,
- "fileType": ext.lstrip("."),
- "chunksSent": chunks_sent,
- "processed": total_processed,
- "rejected": total_rejected,
- "duplicates": total_duplicates,
- "spooledCount": total_spooled,
- "transport": last_transport,
- }
-
- @staticmethod
- def _iter_file_records(path: Path, ext: str) -> Iterable[Dict[str, Any]]:
- try:
- if ext == ".csv":
- with path.open("r", encoding="utf-8-sig", newline="") as fh:
- reader = csv.DictReader(fh)
- for row in reader:
- yield dict(row)
- elif ext in (".jsonl", ".ndjson"):
- with path.open("r", encoding="utf-8-sig") as fh:
- for line_no, line in enumerate(fh, start=1):
- stripped = line.strip()
- if not stripped:
- continue
- try:
- parsed = json.loads(stripped)
- except json.JSONDecodeError as exc:
- raise DataStackFileError(
- f"Malformed JSONL at line {line_no} in {path.name}: {exc}"
- ) from exc
- yield (
- parsed
- if isinstance(parsed, dict)
- else {"value": parsed}
- )
- elif ext == ".json":
- with path.open("r", encoding="utf-8-sig") as fh:
- parsed = json.load(fh)
- if isinstance(parsed, list):
- for item in parsed:
- yield (
- item if isinstance(item, dict) else {"value": item}
- )
- elif isinstance(parsed, dict):
- if isinstance(parsed.get("records"), list):
- for item in parsed["records"]:
- yield (
- item
- if isinstance(item, dict)
- else {"value": item}
- )
- elif isinstance(parsed.get("data"), list):
- for item in parsed["data"]:
- yield (
- item
- if isinstance(item, dict)
- else {"value": item}
- )
- else:
- yield parsed
- else:
- yield {"value": parsed}
- except DataStackFileError:
- raise
- except Exception as exc:
- raise DataStackFileError(
- f"Failed reading file '{path.name}': {exc}"
- ) from exc
-
- def status(self) -> Dict[str, Any]:
- """
- Returns a complete diagnostic snapshot of the SDK connection state,
- Heartbeat Monitor mode (`IDLE` vs `ACTIVE_RECOVERY`), and offline spool queue.
- """
- hb = self.heartbeat.status_snapshot()
- return {
- "connectedWebSocket": self.is_connected,
- "serverUrl": self.server_url,
- "apiUrl": self.api_url,
- "configSource": self.config.config_source_path,
- "connectionState": hb["state"],
- "heartbeatMode": hb["heartbeatMode"],
- "recoveryAttempts": hb["recoveryAttempts"],
- "lastError": hb["lastError"],
- "offlineSpoolPending": self.spooler.pending_count(),
- "offlineSpoolBytes": self.spooler.size_bytes(),
- }
-
-
-DataStack = DataStackClient
-
-__all__ = [
- "DataStack",
- "DataStackClient",
- "DataStackConfig",
- "load_config",
- "ConnectionState",
- "HeartbeatMonitor",
- "DiskSpooler",
- "DataStackError",
- "DataStackConfigError",
- "DataStackAuthError",
- "DataStackConnectionError",
- "DataStackRateLimitError",
- "DataStackPayloadTooLargeError",
- "DataStackFileError",
- "DataStackOfflineSpoolError",
-]
+from datastack.client import * # noqa: E402, F401, F403
diff --git a/sdk-python/datastack_sdk/config.py b/sdk-python/datastack_sdk/config.py
index 76c272f..d083d3d 100644
--- a/sdk-python/datastack_sdk/config.py
+++ b/sdk-python/datastack_sdk/config.py
@@ -1,321 +1,7 @@
-"""
-Auto-Discovery Configuration Loader for `config.datastack.py`.
-"""
+"""Compatibility re-export of canonical `datastack.config`."""
-import importlib.util
-import os
-import sys
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-from urllib.parse import urlparse
+from . import _ensure_canonical_datastack_on_path
-from .exceptions import DataStackConfigError
+_ensure_canonical_datastack_on_path()
-CONFIG_FILENAME = "config.datastack.py"
-
-
-@dataclass
-class DataStackConfig:
- server_url: str = "http://localhost:4000"
- api_url: str = "http://localhost:3000"
- client_token: str = ""
- client_id: str = ""
- auto_connect: bool = True
- enable_http_fallback: bool = True
- heartbeat_interval_sec: float = 5.0
- request_timeout_sec: float = 8.0
- max_retries: int = 3
- retry_backoff_base_sec: float = 0.5
- batch_chunk_size: int = 250
- offline_spool_enabled: bool = True
- offline_spool_dir: str = ".datastack_spool"
- max_spool_size_mb: int = 50
- config_source_path: Optional[str] = None
-
-
-def _derive_default_api_url(server_url: str) -> str:
- """
- If API_URL is not explicitly set, infer Port 3000 on the same host when SERVER_URL
- uses Port 4000, or fall back to http://localhost:3000.
- """
- try:
- parsed = urlparse(server_url)
- if parsed.scheme and parsed.hostname:
- if parsed.port == 4000:
- return f"{parsed.scheme}://{parsed.hostname}:3000"
- return f"{parsed.scheme}://{parsed.netloc}"
- except Exception:
- pass
- return "http://localhost:3000"
-
-
-def _candidate_config_paths(explicit_path: Optional[str] = None) -> List[Path]:
- candidates: List[Path] = []
-
- if explicit_path:
- p = Path(explicit_path).expanduser().resolve()
- if p.is_dir():
- candidates.append(p / CONFIG_FILENAME)
- else:
- candidates.append(p)
-
- env_path = os.environ.get("DATASTACK_CONFIG_PATH", "").strip()
- if env_path:
- ep = Path(env_path).expanduser().resolve()
- if ep.is_dir():
- candidates.append(ep / CONFIG_FILENAME)
- else:
- candidates.append(ep)
-
- # Search current working directory and up to 3 parent levels
- try:
- cwd = Path.cwd().resolve()
- candidates.append(cwd / CONFIG_FILENAME)
- for parent in list(cwd.parents)[:3]:
- candidates.append(parent / CONFIG_FILENAME)
- except Exception:
- pass
-
- # Search main entry script directory
- try:
- if sys.argv and sys.argv[0]:
- script_dir = Path(sys.argv[0]).resolve().parent
- candidates.append(script_dir / CONFIG_FILENAME)
- for parent in list(script_dir.parents)[:2]:
- candidates.append(parent / CONFIG_FILENAME)
- except Exception:
- pass
-
- # Deduplicate while preserving order
- seen = set()
- unique: List[Path] = []
- for c in candidates:
- key = str(c)
- if key not in seen:
- seen.add(key)
- unique.append(c)
- return unique
-
-
-def _load_python_config_file(file_path: Path) -> Dict[str, Any]:
- try:
- spec = importlib.util.spec_from_file_location(
- "datastack_user_config", str(file_path)
- )
- if spec is None or spec.loader is None:
- raise DataStackConfigError(
- f"Unable to load configuration specification from {file_path}"
- )
- module = importlib.util.module_from_spec(spec)
- spec.loader.exec_module(module)
-
- raw: Dict[str, Any] = {}
- if hasattr(module, "DATASTACK_CONFIG") and isinstance(
- getattr(module, "DATASTACK_CONFIG"), dict
- ):
- raw.update(getattr(module, "DATASTACK_CONFIG"))
-
- # Also support top-level module variables (e.g., SERVER_URL = "...", CLIENT_TOKEN = "...")
- for attr in dir(module):
- if attr.isupper() and attr != "DATASTACK_CONFIG":
- raw[attr] = getattr(module, attr)
-
- return raw
- except DataStackConfigError:
- raise
- except Exception as exc:
- raise DataStackConfigError(
- f"Failed to parse {file_path}: {exc}"
- ) from exc
-
-
-def load_config(
- config_path: Optional[str] = None,
- require_token: bool = False,
- **overrides: Any,
-) -> DataStackConfig:
- """
- Discovers and loads `config.datastack.py` from the client application's
- directory, merges environment variables (`DATASTACK_*`), and applies
- any explicit keyword overrides.
- """
- file_values: Dict[str, Any] = {}
- resolved_source: Optional[str] = None
-
- for candidate in _candidate_config_paths(config_path):
- if candidate.is_file():
- file_values = _load_python_config_file(candidate)
- resolved_source = str(candidate)
- break
-
- if config_path and not resolved_source:
- raise DataStackConfigError(
- f"Explicitly requested config file was not found: {config_path}"
- )
-
- # Normalize keys to uppercase for lookup
- normalized_file = {str(k).upper(): v for k, v in file_values.items()}
- normalized_overrides = {
- str(k).upper(): v for k, v in overrides.items() if v is not None
- }
-
- def pick(keys: List[str], env_keys: List[str], default: Any) -> Any:
- for k in keys:
- if k in normalized_overrides:
- return normalized_overrides[k]
- for ek in env_keys:
- env_val = os.environ.get(ek)
- if env_val is not None and str(env_val).strip() != "":
- return env_val
- for k in keys:
- if k in normalized_file and normalized_file[k] is not None:
- return normalized_file[k]
- return default
-
- def to_bool(val: Any, default: bool) -> bool:
- if isinstance(val, bool):
- return val
- if isinstance(val, str):
- return val.strip().lower() in ("1", "true", "yes", "on")
- if val is None:
- return default
- return bool(val)
-
- server_url = str(
- pick(
- ["SERVER_URL", "SOCKET_URL", "WS_URL"],
- ["DATASTACK_SERVER_URL", "DATASTACK_SOCKET_URL"],
- "http://localhost:4000",
- )
- ).rstrip("/")
-
- default_api = _derive_default_api_url(server_url)
- api_url = str(
- pick(
- ["API_URL", "HTTP_URL", "WEB_URL"],
- ["DATASTACK_API_URL", "DATASTACK_HTTP_URL"],
- default_api,
- )
- ).rstrip("/")
-
- client_token = str(
- pick(
- ["CLIENT_TOKEN", "TOKEN", "JWT_TOKEN", "API_TOKEN"],
- ["DATASTACK_CLIENT_TOKEN", "DATASTACK_TOKEN"],
- "",
- )
- ).strip()
-
- client_id = str(
- pick(
- ["CLIENT_ID"],
- ["DATASTACK_CLIENT_ID"],
- "",
- )
- ).strip()
-
- cfg = DataStackConfig(
- server_url=server_url,
- api_url=api_url,
- client_token=client_token,
- client_id=client_id,
- auto_connect=to_bool(
- pick(["AUTO_CONNECT"], ["DATASTACK_AUTO_CONNECT"], True), True
- ),
- enable_http_fallback=to_bool(
- pick(
- ["ENABLE_HTTP_FALLBACK", "HTTP_FALLBACK"],
- ["DATASTACK_ENABLE_HTTP_FALLBACK"],
- True,
- ),
- True,
- ),
- heartbeat_interval_sec=max(
- 1.0,
- float(
- pick(
- ["HEARTBEAT_INTERVAL_SEC", "HEARTBEAT_INTERVAL"],
- ["DATASTACK_HEARTBEAT_INTERVAL_SEC"],
- 5.0,
- )
- ),
- ),
- request_timeout_sec=max(
- 1.0,
- float(
- pick(
- ["REQUEST_TIMEOUT_SEC", "TIMEOUT"],
- ["DATASTACK_REQUEST_TIMEOUT_SEC"],
- 8.0,
- )
- ),
- ),
- max_retries=max(
- 0,
- int(
- pick(
- ["MAX_RETRIES", "RETRIES"],
- ["DATASTACK_MAX_RETRIES"],
- 3,
- )
- ),
- ),
- retry_backoff_base_sec=max(
- 0.1,
- float(
- pick(
- ["RETRY_BACKOFF_BASE_SEC"],
- ["DATASTACK_RETRY_BACKOFF_BASE_SEC"],
- 0.5,
- )
- ),
- ),
- batch_chunk_size=max(
- 1,
- int(
- pick(
- ["BATCH_CHUNK_SIZE", "CHUNK_SIZE"],
- ["DATASTACK_BATCH_CHUNK_SIZE"],
- 250,
- )
- ),
- ),
- offline_spool_enabled=to_bool(
- pick(
- ["OFFLINE_SPOOL_ENABLED"],
- ["DATASTACK_OFFLINE_SPOOL_ENABLED"],
- True,
- ),
- True,
- ),
- offline_spool_dir=str(
- pick(
- ["OFFLINE_SPOOL_DIR", "SPOOL_DIR"],
- ["DATASTACK_OFFLINE_SPOOL_DIR"],
- ".datastack_spool",
- )
- ),
- max_spool_size_mb=max(
- 1,
- int(
- pick(
- ["MAX_SPOOL_SIZE_MB"],
- ["DATASTACK_MAX_SPOOL_SIZE_MB"],
- 50,
- )
- ),
- ),
- config_source_path=resolved_source,
- )
-
- if require_token and (
- not cfg.client_token
- or cfg.client_token == "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN"
- ):
- raise DataStackConfigError(
- "DataStack CLIENT_TOKEN is missing. Add your token to `config.datastack.py`, "
- "set DATASTACK_TOKEN in your environment, or pass token='...' to DataStack()."
- )
-
- return cfg
+from datastack.config import * # noqa: E402, F401, F403
diff --git a/sdk-python/datastack_sdk/exceptions.py b/sdk-python/datastack_sdk/exceptions.py
index 1aa255d..ebb6217 100644
--- a/sdk-python/datastack_sdk/exceptions.py
+++ b/sdk-python/datastack_sdk/exceptions.py
@@ -1,72 +1,7 @@
-"""
-Structured Exception Hierarchy for the DataStack Python SDK.
-"""
+"""Compatibility re-export of canonical `datastack.exceptions`."""
-from typing import Optional
+from . import _ensure_canonical_datastack_on_path
+_ensure_canonical_datastack_on_path()
-class DataStackError(Exception):
- """Base exception for all DataStack SDK errors."""
-
- def __init__(
- self,
- message: str,
- code: Optional[str] = None,
- retryable: bool = False,
- ) -> None:
- super().__init__(message)
- self.message = message
- self.code = code or "DATASTACK_ERROR"
- self.retryable = retryable
-
-
-class DataStackConfigError(DataStackError):
- """Raised when `config.datastack.py` or required credentials are missing or invalid."""
-
- def __init__(self, message: str) -> None:
- super().__init__(message, code="CONFIG_ERROR", retryable=False)
-
-
-class DataStackAuthError(DataStackError):
- """
- Raised when authentication fails (`AUTH_REQUIRED`, `AUTH_INVALID`,
- `CLIENT_REVOKED`, or `TOKEN_ROTATED`). Non-retryable until credentials are updated.
- """
-
- def __init__(self, message: str, code: str = "AUTH_ERROR") -> None:
- super().__init__(message, code=code, retryable=False)
-
-
-class DataStackRateLimitError(DataStackError):
- """Raised when `RATE_LIMIT_EXCEEDED` persists after all backoff retries."""
-
- def __init__(self, message: str) -> None:
- super().__init__(message, code="RATE_LIMIT_EXCEEDED", retryable=True)
-
-
-class DataStackPayloadTooLargeError(DataStackError):
- """Raised when a single event exceeds the server's `maxPayloadSizeKb` limit."""
-
- def __init__(self, message: str) -> None:
- super().__init__(message, code="PAYLOAD_TOO_LARGE", retryable=False)
-
-
-class DataStackConnectionError(DataStackError):
- """Raised when both WebSocket and HTTP fallback are unreachable and offline spooling is disabled."""
-
- def __init__(self, message: str) -> None:
- super().__init__(message, code="CONNECTION_ERROR", retryable=True)
-
-
-class DataStackFileError(DataStackError):
- """Raised when a file passed to `send_file()` is missing, unsupported, or malformed."""
-
- def __init__(self, message: str) -> None:
- super().__init__(message, code="FILE_ERROR", retryable=False)
-
-
-class DataStackOfflineSpoolError(DataStackError):
- """Raised when the local offline disk spool directory is full or unwriteable."""
-
- def __init__(self, message: str) -> None:
- super().__init__(message, code="OFFLINE_SPOOL_ERROR", retryable=False)
+from datastack.exceptions import * # noqa: E402, F401, F403
diff --git a/sdk-python/datastack_sdk/heartbeat.py b/sdk-python/datastack_sdk/heartbeat.py
index 34e9893..8aa64ce 100644
--- a/sdk-python/datastack_sdk/heartbeat.py
+++ b/sdk-python/datastack_sdk/heartbeat.py
@@ -1,186 +1,7 @@
-"""
-Smart Heartbeat Monitor & Connection State Machine (`HeartbeatMonitor`).
+"""Compatibility re-export of canonical `datastack.heartbeat`."""
-Lifecycle States:
-- `CONNECTED_IDLE`: WebSocket (Port 4000) is connected and verified. Heartbeat sleeps in IDLE mode.
-- `FALLBACK_HTTP`: WebSocket disconnected. Client automatically sends events via HTTP (`POST /api/events`)
- while Heartbeat actively probes to reconnect and verify WebSocket, then returns to `CONNECTED_IDLE`.
-- `OFFLINE_SPOOL`: Device is completely offline. Events spool to disk (`.datastack_spool/spool.jsonl`).
- Heartbeat monitors network restoration, reconnects, replays spooled events, and returns to `CONNECTED_IDLE`.
-- `RECONNECTING_VERIFY`: Heartbeat is verifying handshake (`ping_verify`) and draining spooled events.
-- `STOPPED`: Client was closed intentionally or halted due to non-retryable auth revocation.
-"""
+from . import _ensure_canonical_datastack_on_path
-import logging
-import threading
-import time
-from typing import Any, Callable, Dict, Optional
+_ensure_canonical_datastack_on_path()
-logger = logging.getLogger("datastack.heartbeat")
-
-
-class ConnectionState:
- CONNECTED_IDLE = "CONNECTED_IDLE"
- FALLBACK_HTTP = "FALLBACK_HTTP"
- OFFLINE_SPOOL = "OFFLINE_SPOOL"
- RECONNECTING_VERIFY = "RECONNECTING_VERIFY"
- STOPPED = "STOPPED"
-
-
-class HeartbeatMonitor:
- """
- Background recovery thread that sleeps in zero-CPU IDLE mode while WebSocket is
- healthy (`CONNECTED_IDLE`) and wakes up automatically when WebSocket disconnects
- or when the client device goes offline.
- """
-
- def __init__(
- self,
- interval_sec: float,
- recovery_callback: Callable[[], bool],
- ) -> None:
- self.interval_sec = max(0.2, float(interval_sec))
- self._recovery_callback = recovery_callback
- self._state = ConnectionState.CONNECTED_IDLE
- self._closed = False
- self._last_error: Optional[str] = None
- self._recovery_attempts = 0
- self._last_verified_at: Optional[float] = None
-
- self._lock = threading.RLock()
- self._wake_event = threading.Event()
- self._stop_event = threading.Event()
- self._thread: Optional[threading.Thread] = None
-
- @property
- def state(self) -> str:
- with self._lock:
- return self._state
-
- @property
- def is_idle(self) -> bool:
- with self._lock:
- return self._state == ConnectionState.CONNECTED_IDLE
-
- def set_idle_connected(self) -> None:
- """
- Marks the connection as healthy (`CONNECTED_IDLE`) and puts the Heartbeat
- Monitor into IDLE mode (`_wake_event.clear()`).
- """
- with self._lock:
- self._closed = False
- self._state = ConnectionState.CONNECTED_IDLE
- self._recovery_attempts = 0
- self._last_error = None
- self._last_verified_at = time.time()
- self._wake_event.clear()
-
- def trigger_fallback(self, reason: str = "WebSocket disconnected") -> None:
- """
- Transitions to `FALLBACK_HTTP` and wakes up the Heartbeat Monitor thread
- to repair the WebSocket connection in the background.
- """
- with self._lock:
- if self._closed:
- return
- self._state = ConnectionState.FALLBACK_HTTP
- self._last_error = reason
- self._ensure_thread_started()
- self._wake_event.set()
-
- def trigger_offline(self, reason: str = "Device offline") -> None:
- """
- Transitions to `OFFLINE_SPOOL` when both WebSocket and HTTP fallback are
- unreachable, waking the Heartbeat Monitor to watch for network return.
- """
- with self._lock:
- if self._closed:
- return
- self._state = ConnectionState.OFFLINE_SPOOL
- self._last_error = reason
- self._ensure_thread_started()
- self._wake_event.set()
-
- def set_reconnecting(self) -> None:
- with self._lock:
- if not self._closed:
- self._state = ConnectionState.RECONNECTING_VERIFY
-
- def start(self) -> None:
- with self._lock:
- self._closed = False
- self._stop_event.clear()
- self._ensure_thread_started()
-
- def stop(self, reason: Optional[str] = None) -> None:
- with self._lock:
- self._closed = True
- self._state = ConnectionState.STOPPED
- if reason:
- self._last_error = reason
- self._stop_event.set()
- self._wake_event.set()
-
- def status_snapshot(self) -> Dict[str, Any]:
- with self._lock:
- return {
- "state": self._state,
- "heartbeatMode": (
- "IDLE"
- if self._state == ConnectionState.CONNECTED_IDLE
- else "ACTIVE_RECOVERY"
- if self._state
- in (
- ConnectionState.FALLBACK_HTTP,
- ConnectionState.OFFLINE_SPOOL,
- ConnectionState.RECONNECTING_VERIFY,
- )
- else "STOPPED"
- ),
- "recoveryAttempts": self._recovery_attempts,
- "lastError": self._last_error,
- "lastVerifiedAt": self._last_verified_at,
- "intervalSec": self.interval_sec,
- }
-
- def _ensure_thread_started(self) -> None:
- if self._thread is not None and self._thread.is_alive():
- return
- self._stop_event.clear()
- self._thread = threading.Thread(
- target=self._run_loop,
- name="DataStack-HeartbeatMonitor",
- daemon=True,
- )
- self._thread.start()
-
- def _run_loop(self) -> None:
- while not self._stop_event.is_set():
- # Wait in zero-CPU IDLE mode until woken by a disconnect or offline trigger
- self._wake_event.wait()
- if self._stop_event.is_set():
- break
-
- # Wait for the heartbeat interval before probing
- if self._stop_event.wait(self.interval_sec):
- break
-
- if not self._wake_event.is_set():
- continue
-
- with self._lock:
- if self._closed or self._state == ConnectionState.STOPPED:
- break
- self._recovery_attempts += 1
-
- try:
- recovered_to_idle = self._recovery_callback()
- if recovered_to_idle:
- self.set_idle_connected()
- logger.info(
- "[DataStack Heartbeat] WebSocket reconnected & verified. Returning to IDLE mode."
- )
- except Exception as exc:
- with self._lock:
- self._last_error = str(exc)
- logger.debug("[DataStack Heartbeat] Recovery probe failed: %s", exc)
+from datastack.heartbeat import * # noqa: E402, F401, F403
diff --git a/sdk-python/datastack_sdk/spool.py b/sdk-python/datastack_sdk/spool.py
index e98bdad..95908f5 100644
--- a/sdk-python/datastack_sdk/spool.py
+++ b/sdk-python/datastack_sdk/spool.py
@@ -1,164 +1,7 @@
-"""
-Thread-Safe Offline Disk Spooler (`DiskSpooler`).
-Persists unsent telemetry events to `.datastack_spool/spool.jsonl` when a client
-device loses network connectivity completely, and drains them in deterministic
-batches once the Heartbeat Monitor restores connectivity.
-"""
+"""Compatibility re-export of canonical `datastack.spool`."""
-import json
-import os
-import threading
-from pathlib import Path
-from typing import Any, Dict, List
+from . import _ensure_canonical_datastack_on_path
-from .exceptions import DataStackOfflineSpoolError
+_ensure_canonical_datastack_on_path()
-
-class DiskSpooler:
- """
- Append-only JSONL disk queue with atomic batch drainage and size cap protection.
- """
-
- def __init__(
- self,
- spool_dir: str = ".datastack_spool",
- max_size_mb: int = 50,
- enabled: bool = True,
- ) -> None:
- self.enabled = enabled
- self.spool_dir = Path(spool_dir)
- self.spool_file = self.spool_dir / "spool.jsonl"
- self.max_bytes = max(1, int(max_size_mb)) * 1024 * 1024
- self._lock = threading.RLock()
-
- def _ensure_dir(self) -> None:
- try:
- self.spool_dir.mkdir(parents=True, exist_ok=True)
- except Exception as exc:
- raise DataStackOfflineSpoolError(
- f"Cannot create offline spool directory '{self.spool_dir}': {exc}"
- ) from exc
-
- def enqueue(self, events: List[Dict[str, Any]]) -> int:
- """
- Appends one or more formatted events (`{eventId, timestamp, data}`)
- to `.datastack_spool/spool.jsonl`. Returns count of spooled events.
- """
- if not self.enabled or not events:
- return 0
-
- with self._lock:
- self._ensure_dir()
- current_size = (
- self.spool_file.stat().st_size if self.spool_file.exists() else 0
- )
-
- lines: List[str] = []
- added_bytes = 0
- for evt in events:
- serialized = json.dumps(evt, ensure_ascii=False) + "\n"
- encoded_len = len(serialized.encode("utf-8"))
- if current_size + added_bytes + encoded_len > self.max_bytes:
- raise DataStackOfflineSpoolError(
- f"Offline spool limit ({self.max_bytes // (1024 * 1024)} MB) "
- f"reached at '{self.spool_file}'."
- )
- lines.append(serialized)
- added_bytes += encoded_len
-
- try:
- with self.spool_file.open("a", encoding="utf-8") as fh:
- fh.writelines(lines)
- fh.flush()
- except Exception as exc:
- raise DataStackOfflineSpoolError(
- f"Failed writing to offline spool file '{self.spool_file}': {exc}"
- ) from exc
-
- return len(lines)
-
- def peek_batch(self, limit: int = 250) -> List[Dict[str, Any]]:
- """
- Reads up to `limit` spooled events from the front of the spool file
- without removing them until `ack_batch()` confirms delivery.
- """
- if not self.enabled:
- return []
-
- with self._lock:
- if not self.spool_file.exists():
- return []
-
- batch: List[Dict[str, Any]] = []
- try:
- with self.spool_file.open("r", encoding="utf-8") as fh:
- for line in fh:
- if len(batch) >= limit:
- break
- stripped = line.strip()
- if not stripped:
- continue
- try:
- parsed = json.loads(stripped)
- if isinstance(parsed, dict):
- batch.append(parsed)
- except json.JSONDecodeError:
- continue
- except Exception:
- return []
-
- return batch
-
- def ack_batch(self, count: int) -> int:
- """
- Removes the first `count` non-empty lines from the spool file atomically
- after the Heartbeat Monitor confirms they have been ingested by DataStack.
- """
- if not self.enabled or count <= 0:
- return 0
-
- with self._lock:
- if not self.spool_file.exists():
- return 0
-
- try:
- with self.spool_file.open("r", encoding="utf-8") as fh:
- all_lines = [ln for ln in fh if ln.strip()]
-
- if count >= len(all_lines):
- self.spool_file.unlink(missing_ok=True)
- return len(all_lines)
-
- remaining = all_lines[count:]
- tmp_file = self.spool_dir / "spool.jsonl.tmp"
- with tmp_file.open("w", encoding="utf-8") as fh:
- fh.writelines(remaining)
- fh.flush()
- os.replace(str(tmp_file), str(self.spool_file))
- return count
- except Exception:
- return 0
-
- def pending_count(self) -> int:
- """Returns the number of queued events waiting in the disk spool."""
- if not self.enabled:
- return 0
-
- with self._lock:
- if not self.spool_file.exists():
- return 0
- try:
- with self.spool_file.open("r", encoding="utf-8") as fh:
- return sum(1 for ln in fh if ln.strip())
- except Exception:
- return 0
-
- def size_bytes(self) -> int:
- """Returns current disk usage of the spool file in bytes."""
- with self._lock:
- if not self.spool_file.exists():
- return 0
- try:
- return self.spool_file.stat().st_size
- except Exception:
- return 0
+from datastack.spool import * # noqa: E402, F401, F403
diff --git a/sdk/python/datastack/README.md b/sdk/python/datastack/README.md
index 23984c5..e0f8efc 100644
--- a/sdk/python/datastack/README.md
+++ b/sdk/python/datastack/README.md
@@ -33,6 +33,7 @@ DATASTACK_CONFIG = {
"SERVER_URL": "http://localhost:4000", # Port 4000 Socket.IO Ingestion Service
"API_URL": "http://localhost:3000", # Port 3000 HTTP Fallback & File Ingestion API
"CLIENT_TOKEN": "YOUR_CLIENT_JWT_TOKEN", # Provisioned in /vcon/clients
+ "CLIENT_ID": "", # Optional override (defaults to JWT sub/clientId)
"AUTO_CONNECT": True,
"ENABLE_HTTP_FALLBACK": True,
"HEARTBEAT_INTERVAL_SEC": 5.0,
diff --git a/sdk/python/datastack/datastack/__init__.py b/sdk/python/datastack/datastack/__init__.py
index d648233..913d13c 100644
--- a/sdk/python/datastack/datastack/__init__.py
+++ b/sdk/python/datastack/datastack/__init__.py
@@ -5,6 +5,7 @@ Socket.IO ingestion (Port 4000), automatic Heartbeat HTTP fallback (Port 3000),
offline disk spooling, and direct `.csv` / `.json` / `.jsonl` file streaming.
"""
+from .client import DataStack, DataStackClient
from .config import DataStackConfig, load_config
from .exceptions import (
DataStackAuthError,
@@ -16,14 +17,9 @@ from .exceptions import (
DataStackPayloadTooLargeError,
DataStackRateLimitError,
)
+from .heartbeat import ConnectionState, HeartbeatMonitor
from .spool import DiskSpooler
-try:
- from .client import DataStack, DataStackClient
-except ImportError:
- DataStack = None # type: ignore
- DataStackClient = None # type: ignore
-
__version__ = "2.2.0"
__all__ = [
@@ -31,6 +27,8 @@ __all__ = [
"DataStackClient",
"DataStackConfig",
"load_config",
+ "ConnectionState",
+ "HeartbeatMonitor",
"DiskSpooler",
"DataStackError",
"DataStackConfigError",
diff --git a/sdk/python/datastack/datastack/client.py b/sdk/python/datastack/datastack/client.py
index 959fed0..e75be3c 100644
--- a/sdk/python/datastack/datastack/client.py
+++ b/sdk/python/datastack/datastack/client.py
@@ -13,6 +13,7 @@ Features:
and lossless partial-batch rate-limit retry (`429 RATE_LIMIT_EXCEEDED`).
"""
+import base64
import csv
import hashlib
import json
@@ -44,6 +45,7 @@ from .exceptions import (
DataStackConnectionError,
DataStackError,
DataStackFileError,
+ DataStackOfflineSpoolError,
DataStackPayloadTooLargeError,
DataStackRateLimitError,
)
@@ -72,6 +74,23 @@ def _make_deterministic_file_event_id(
return f"file_{safe_stem}_{row_index}_{digest}"
+def _decode_jwt_client_id(token: str) -> str:
+ """Safely extracts `clientId` or `sub` claim from an unverified JWT payload string."""
+ try:
+ parts = (token or "").split(".")
+ if len(parts) != 3:
+ return ""
+ payload_b64 = parts[1]
+ padding = "=" * (-len(payload_b64) % 4)
+ raw = base64.urlsafe_b64decode(payload_b64 + padding).decode("utf-8")
+ claims = json.loads(raw)
+ if isinstance(claims, dict):
+ return str(claims.get("clientId") or claims.get("sub") or "").strip()
+ except Exception:
+ pass
+ return ""
+
+
class DataStackClient:
"""
Production-grade DataStack Python Client with `config.datastack.py` auto-loading,
@@ -87,6 +106,7 @@ class DataStackClient:
auto_connect: Optional[bool] = None,
reconnection_attempts: Optional[int] = None,
reconnection_delay: Optional[float] = None,
+ client_id: Optional[str] = None,
**kwargs: Any,
) -> None:
if reconnection_attempts is not None and "MAX_RETRIES" not in kwargs:
@@ -105,6 +125,7 @@ class DataStackClient:
SERVER_URL=server_url,
CLIENT_TOKEN=token,
API_URL=api_url,
+ CLIENT_ID=client_id,
AUTO_CONNECT=auto_connect,
**kwargs,
)
@@ -112,6 +133,20 @@ class DataStackClient:
self.server_url = self.config.server_url
self.api_url = self.config.api_url
self.token = self.config.client_token
+
+ configured_client_id = (self.config.client_id or "").strip()
+ token_client_id = _decode_jwt_client_id(self.token)
+ if (
+ configured_client_id
+ and token_client_id
+ and configured_client_id != token_client_id
+ ):
+ raise DataStackConfigError(
+ f"Configured CLIENT_ID '{configured_client_id}' does not match "
+ f"CLIENT_TOKEN subject '{token_client_id}'."
+ )
+ self.client_id: str = configured_client_id or token_client_id
+
self.is_connected = False
self._lock = threading.RLock()
self._is_draining_spool = False
@@ -311,8 +346,11 @@ class DataStackClient:
raise DataStackAuthError(msg) from exc
return False
- # Verify handshake & PostgreSQL client status via `ping_verify`
- return self._verify_websocket_handshake(timeout=effective_timeout)
+ # Verify handshake & PostgreSQL client status via `ping_verify` while still serialized under RLock
+ verified = self._verify_websocket_handshake(timeout=effective_timeout)
+ if not verified:
+ self.is_connected = False
+ return verified
def _verify_websocket_handshake(
self, timeout: Optional[float] = None
@@ -424,6 +462,8 @@ class DataStackClient:
"Content-Type": "application/json",
"Accept": "application/json",
}
+ if self.client_id:
+ headers["X-DataStack-Client-Id"] = self.client_id
if requests is not None:
resp = requests.request(
@@ -735,10 +775,14 @@ class DataStackClient:
):
break
- self.spooler.ack_batch(len(batch))
- if isinstance(unprocessed, list) and len(unprocessed) > 0:
- self.spooler.enqueue(unprocessed)
- total_flushed += max(0, len(batch) - len(unprocessed))
+ unprocessed_list = (
+ unprocessed if isinstance(unprocessed, list) else []
+ )
+ self.spooler.ack_and_requeue_front(
+ len(batch), unprocessed_list
+ )
+ if unprocessed_list:
+ total_flushed += max(0, len(batch) - len(unprocessed_list))
break
total_flushed += len(batch)
@@ -767,11 +811,13 @@ class DataStackClient:
"""
resolved_id = (event_id or "").strip() or _make_event_id()
resolved_ts = (timestamp or "").strip() or _utc_now_iso()
- envelope = {
+ envelope: Dict[str, Any] = {
"eventId": resolved_id,
"timestamp": resolved_ts,
"data": data,
}
+ if self.client_id:
+ envelope["clientId"] = self.client_id
return self._execute_with_retry_and_fallback(
ws_event="sync_data",
ws_payload=envelope,
@@ -810,6 +856,7 @@ class DataStackClient:
if not isinstance(source_list, list) or len(source_list) == 0:
return {"success": True, "processed": 0, "rejected": 0}
+ envelope_keys = {"eventId", "timestamp", "data", "clientId"}
formatted: List[Dict[str, Any]] = []
for item in source_list:
safe_item = item if isinstance(item, dict) else {"value": item}
@@ -819,14 +866,21 @@ class DataStackClient:
evt_ts = (
str(safe_item.get("timestamp") or "").strip() or _utc_now_iso()
)
- evt_data = safe_item.get("data", safe_item)
- formatted.append(
- {
- "eventId": evt_id,
- "timestamp": evt_ts,
- "data": evt_data,
- }
+ is_envelope = "data" in safe_item and set(safe_item.keys()).issubset(
+ envelope_keys
)
+ evt_data = safe_item["data"] if is_envelope else safe_item
+ evt_client_id = (
+ str(safe_item.get("clientId") or "").strip() or self.client_id
+ )
+ entry: Dict[str, Any] = {
+ "eventId": evt_id,
+ "timestamp": evt_ts,
+ "data": evt_data,
+ }
+ if evt_client_id:
+ entry["clientId"] = evt_client_id
+ formatted.append(entry)
return self._send_formatted_batch_recursive(
formatted,
@@ -1057,13 +1111,14 @@ class DataStackClient:
)
evt_ts = existing_ts or _utc_now_iso()
- buffer.append(
- {
- "eventId": evt_id,
- "timestamp": evt_ts,
- "data": enriched,
- }
- )
+ file_entry: Dict[str, Any] = {
+ "eventId": evt_id,
+ "timestamp": evt_ts,
+ "data": enriched,
+ }
+ if self.client_id:
+ file_entry["clientId"] = self.client_id
+ buffer.append(file_entry)
if len(buffer) >= effective_chunk:
res = self.send_batch(buffer, file_name=path.name)
total_processed += int(res.get("processed", 0))
@@ -1162,6 +1217,7 @@ class DataStackClient:
hb = self.heartbeat.status_snapshot()
return {
"connectedWebSocket": self.is_connected,
+ "clientId": self.client_id or None,
"serverUrl": self.server_url,
"apiUrl": self.api_url,
"configSource": self.config.config_source_path,
@@ -1176,3 +1232,21 @@ class DataStackClient:
# Convenient alias so developers can write `from datastack import DataStack`
DataStack = DataStackClient
+
+__all__ = [
+ "DataStack",
+ "DataStackClient",
+ "DataStackConfig",
+ "load_config",
+ "ConnectionState",
+ "HeartbeatMonitor",
+ "DiskSpooler",
+ "DataStackError",
+ "DataStackConfigError",
+ "DataStackAuthError",
+ "DataStackConnectionError",
+ "DataStackRateLimitError",
+ "DataStackPayloadTooLargeError",
+ "DataStackFileError",
+ "DataStackOfflineSpoolError",
+]
diff --git a/sdk/python/datastack/datastack/heartbeat.py b/sdk/python/datastack/datastack/heartbeat.py
index 34e9893..49c7dd9 100644
--- a/sdk/python/datastack/datastack/heartbeat.py
+++ b/sdk/python/datastack/datastack/heartbeat.py
@@ -171,6 +171,11 @@ class HeartbeatMonitor:
with self._lock:
if self._closed or self._state == ConnectionState.STOPPED:
break
+ if (
+ self._state == ConnectionState.CONNECTED_IDLE
+ or not self._wake_event.is_set()
+ ):
+ continue
self._recovery_attempts += 1
try:
diff --git a/sdk/python/datastack/datastack/spool.py b/sdk/python/datastack/datastack/spool.py
index e98bdad..3009642 100644
--- a/sdk/python/datastack/datastack/spool.py
+++ b/sdk/python/datastack/datastack/spool.py
@@ -9,7 +9,7 @@ import json
import os
import threading
from pathlib import Path
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
from .exceptions import DataStackOfflineSpoolError
@@ -139,6 +139,49 @@ class DiskSpooler:
except Exception:
return 0
+ def ack_and_requeue_front(
+ self,
+ acked_count: int,
+ unprocessed: Optional[List[Dict[str, Any]]] = None,
+ ) -> int:
+ """
+ Atomically removes `acked_count` head lines from `spool.jsonl` while
+ preserving any `unprocessed` events at the FRONT of the queue in a single
+ atomic `os.replace()` write (FIFO safe, zero crash-window data loss).
+ """
+ if not self.enabled or acked_count <= 0:
+ return 0
+
+ with self._lock:
+ if not self.spool_file.exists():
+ return 0
+
+ try:
+ with self.spool_file.open("r", encoding="utf-8") as fh:
+ all_lines = [ln for ln in fh if ln.strip()]
+
+ consumed = min(acked_count, len(all_lines))
+ tail_lines = all_lines[consumed:]
+ head_lines = [
+ json.dumps(evt, ensure_ascii=False) + "\n"
+ for evt in (unprocessed or [])
+ ]
+ combined = head_lines + tail_lines
+
+ if not combined:
+ self.spool_file.unlink(missing_ok=True)
+ return consumed
+
+ self._ensure_dir()
+ tmp_file = self.spool_dir / "spool.jsonl.tmp"
+ with tmp_file.open("w", encoding="utf-8") as fh:
+ fh.writelines(combined)
+ fh.flush()
+ os.replace(str(tmp_file), str(self.spool_file))
+ return max(0, consumed - len(head_lines))
+ except Exception:
+ return 0
+
def pending_count(self) -> int:
"""Returns the number of queued events waiting in the disk spool."""
if not self.enabled:
diff --git a/server/index.js b/server/index.js
index bad4e29..c144839 100644
--- a/server/index.js
+++ b/server/index.js
@@ -1,8 +1,10 @@
+const crypto = require('node:crypto');
const express = require('express');
const http = require('http');
const cors = require('cors');
const { Server } = require('socket.io');
const { Pool } = require('pg');
+const Redis = require('ioredis');
const { setupSocketHandlers } = require('./socket/handlers');
const { CsvWriterWorker } = require('./workers/csvWriter');
@@ -11,21 +13,185 @@ const app = express();
app.use(cors({ origin: '*' }));
app.use(express.json({ limit: '10mb' }));
+const REDIS_BUFFER_KEY = 'datastack:buffer:events';
+const REDIS_RECENT_KEY = 'datastack:recent:events';
+const REDIS_FLUSH_LOCK_KEY = 'datastack:flush:lock';
+const REDIS_RECENT_LIMIT = 500;
+const FLUSH_LOCK_TTL_MS = 15000;
+const UNLOCK_LUA_SCRIPT =
+ 'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end';
+
const bufferQueue = [];
const writer = new CsvWriterWorker();
+let redisClient = undefined;
+let redisUrlConfigured = '';
+let redisConnected = false;
+let redisTestOverride = undefined;
+const pendingRedisWrites = new Set();
+
+function getRedisClient() {
+ if (redisTestOverride !== undefined) {
+ return redisTestOverride;
+ }
+
+ const configuredUrl = (process.env.REDIS_URL || '').trim();
+ if (!configuredUrl) {
+ if (redisClient && typeof redisClient.disconnect === 'function') {
+ try {
+ redisClient.disconnect();
+ } catch {
+ // ignore
+ }
+ }
+ redisClient = null;
+ redisUrlConfigured = '';
+ redisConnected = false;
+ return null;
+ }
+
+ if (redisClient && redisUrlConfigured === configuredUrl) {
+ return redisClient;
+ }
+
+ if (redisClient && typeof redisClient.disconnect === 'function') {
+ try {
+ redisClient.disconnect();
+ } catch {
+ // ignore
+ }
+ }
+
+ try {
+ const client = new Redis(configuredUrl, {
+ maxRetriesPerRequest: 2,
+ enableReadyCheck: true,
+ retryStrategy: (times) => Math.min(times * 250, 3000),
+ });
+ client.on('ready', () => {
+ redisConnected = true;
+ });
+ client.on('connect', () => {
+ redisConnected = true;
+ });
+ client.on('close', () => {
+ redisConnected = false;
+ });
+ client.on('end', () => {
+ redisConnected = false;
+ });
+ client.on('error', () => {
+ redisConnected = false;
+ });
+ redisClient = client;
+ redisUrlConfigured = configuredUrl;
+ redisConnected = client.status === 'ready' || client.status === 'connect';
+ return redisClient;
+ } catch {
+ redisClient = null;
+ redisUrlConfigured = configuredUrl;
+ redisConnected = false;
+ return null;
+ }
+}
+
+function setRedisClientForTests(client) {
+ redisTestOverride = client;
+ redisConnected = Boolean(client);
+}
+
+async function pushEventsToRedisOrFallback(events) {
+ if (!events || events.length === 0) return bufferQueue.length;
+ const redis = getRedisClient();
+ if (redis) {
+ const serialized = events.map((e) => JSON.stringify(e));
+ try {
+ const len = await redis.rpush(REDIS_BUFFER_KEY, ...serialized);
+ await redis.lpush(REDIS_RECENT_KEY, ...serialized);
+ await redis.ltrim(REDIS_RECENT_KEY, 0, REDIS_RECENT_LIMIT - 1);
+ redisConnected = true;
+ return Number(len) + bufferQueue.length;
+ } catch {
+ redisConnected = false;
+ }
+ }
+
+ bufferQueue.push(...events);
+ return bufferQueue.length;
+}
+
+const IS_PRODUCTION = process.env.NODE_ENV === 'production';
+const DEV_DEFAULT_JWT_SECRET = 'datastack-jwt-default-secret-key-32-chars-long';
+
+function resolveInitialJwtSecret() {
+ const configured = (process.env.DATASTACK_JWT_SECRET || '').trim();
+ if (IS_PRODUCTION) {
+ return configured.length >= 32 ? configured : '';
+ }
+ if (configured.length > 0) {
+ return configured.length >= 32 ? configured : '';
+ }
+ return DEV_DEFAULT_JWT_SECRET;
+}
const runtimeConfig = {
- jwtSecret:
- process.env.DATASTACK_JWT_SECRET ||
- 'datastack-jwt-default-secret-key-32-chars-long',
+ jwtSecret: resolveInitialJwtSecret(),
maxEventsPerSec: 100,
maxPayloadSizeBytes: 100 * 1024,
batchFlushThreshold: 500,
batchFlushIntervalMs: 2000,
fileRotationMb: 100,
enableIdempotency: true,
+ autoPruneDays: 30,
};
+function resolveInternalSecret(jwtSecretOverride) {
+ const explicit = (process.env.DATASTACK_INTERNAL_SECRET || '').trim();
+ if (IS_PRODUCTION) {
+ if (explicit.length >= 32) return explicit;
+ if (explicit.length > 0) return '';
+ } else if (explicit.length > 0) {
+ return explicit;
+ }
+ const activeJwt = String(
+ jwtSecretOverride !== undefined ? jwtSecretOverride : runtimeConfig.jwtSecret || '',
+ ).trim();
+ const minJwtLen = IS_PRODUCTION ? 32 : 1;
+ if (activeJwt.length < minJwtLen) {
+ return '';
+ }
+ return crypto
+ .createHash('sha256')
+ .update(`datastack-internal:${activeJwt}`)
+ .digest('hex');
+}
+
+function timingSafeSecretEqual(candidate, expected) {
+ if (!candidate || !expected) return false;
+ const a = crypto.createHash('sha256').update(String(candidate)).digest();
+ const b = crypto.createHash('sha256').update(String(expected)).digest();
+ return crypto.timingSafeEqual(a, b);
+}
+
+function requireInternalAuth(req, res, next) {
+ const expected = resolveInternalSecret();
+ if (!expected) {
+ return res.status(503).json({
+ success: false,
+ error: 'INTERNAL_AUTH_UNAVAILABLE: Internal bridge secret is not configured',
+ code: 'INTERNAL_AUTH_UNAVAILABLE',
+ });
+ }
+ const presented = req.get('x-datastack-internal-secret');
+ if (!presented || !timingSafeSecretEqual(presented, expected)) {
+ return res.status(401).json({
+ success: false,
+ error: 'UNAUTHORIZED: Missing or invalid internal authentication header',
+ code: 'UNAUTHORIZED',
+ });
+ }
+ return next();
+}
+
let dbPool = null;
if (process.env.DATABASE_URL) {
try {
@@ -40,43 +206,95 @@ if (process.env.DATABASE_URL) {
}
}
-// Short-lived cache (2s TTL) for client status & token checks on high-frequency events
+// Short-lived cache (2s TTL) for DB client status & token checks on high-frequency events
const clientAuthCache = new Map();
+const deletedClientsSet = new Set();
+const inMemoryClientRegistry = new Map();
+const inMemoryClientActivity = new Map();
+let hasSyncedInMemoryClients = false;
const CLIENT_CACHE_TTL_MS = 2000;
async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
- if (!dbPool || !clientId) {
+ if (!clientId) {
return { allowed: true };
}
- const now = Date.now();
- if (!bypassCache) {
- const cached = clientAuthCache.get(clientId);
- if (cached && now - cached.checkedAt < CLIENT_CACHE_TTL_MS) {
- if (cached.status === 'revoked') {
+ const normalizedId = String(clientId).trim().toLowerCase();
+ if (deletedClientsSet.has(normalizedId)) {
+ return {
+ allowed: false,
+ reason: 'CLIENT_REVOKED: Client access has been deleted or revoked by admin',
+ };
+ }
+
+ if (!dbPool) {
+ const memClient = inMemoryClientRegistry.get(normalizedId);
+ if (memClient) {
+ if (memClient.status === 'revoked') {
return {
allowed: false,
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
};
}
- if (cached.token && presentedToken && cached.token !== presentedToken) {
+ if (memClient.token && presentedToken && memClient.token !== presentedToken) {
return {
allowed: false,
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
};
}
- return { allowed: true, rateLimit: cached.rateLimit };
+ return { allowed: true, rateLimit: memClient.rateLimit || 100 };
}
+ if (hasSyncedInMemoryClients) {
+ return {
+ allowed: false,
+ reason: 'CLIENT_REVOKED: Client registration not found or has been deleted',
+ };
+ }
+ return { allowed: true };
+ }
+
+ const now = Date.now();
+ const prevCached = clientAuthCache.get(normalizedId);
+ if (!bypassCache && prevCached && now - prevCached.checkedAt < CLIENT_CACHE_TTL_MS) {
+ if (prevCached.status === 'revoked') {
+ return {
+ allowed: false,
+ reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
+ };
+ }
+ if (prevCached.token && presentedToken && prevCached.token !== presentedToken) {
+ return {
+ allowed: false,
+ reason: 'TOKEN_ROTATED: Client credentials have been rotated',
+ };
+ }
+ return { allowed: true, rateLimit: prevCached.rateLimit };
}
try {
+ const deletedRes = await dbPool.query(
+ `SELECT 1 FROM deleted_clients WHERE client_id = $1`,
+ [normalizedId],
+ );
+ if (deletedRes.rowCount && deletedRes.rowCount > 0) {
+ clientAuthCache.delete(normalizedId);
+ deletedClientsSet.add(normalizedId);
+ return {
+ allowed: false,
+ reason: 'CLIENT_REVOKED: Client access has been deleted by admin',
+ };
+ }
+
const res = await dbPool.query(
`SELECT status, token, rate_limit FROM client_sdks WHERE client_id = $1`,
- [clientId],
+ [normalizedId],
);
if (!res.rowCount || res.rowCount === 0) {
- // Allow token-authenticated client if not explicitly revoked in table
- return { allowed: true };
+ clientAuthCache.delete(normalizedId);
+ return {
+ allowed: false,
+ reason: 'CLIENT_REVOKED: Client registration not found or has been deleted',
+ };
}
const row = res.rows[0];
const entry = {
@@ -85,7 +303,7 @@ async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
rateLimit: Number(row.rate_limit) || 100,
checkedAt: now,
};
- clientAuthCache.set(clientId, entry);
+ clientAuthCache.set(normalizedId, entry);
if (entry.status === 'revoked') {
return {
@@ -101,13 +319,30 @@ async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
}
return { allowed: true, rateLimit: entry.rateLimit };
} catch {
- return { allowed: true };
+ return {
+ allowed: false,
+ reason: 'AUTH_UNAVAILABLE: Unable to verify client credentials against database',
+ };
}
}
function recordClientActivity(clientId, count = 1) {
- if (!dbPool || !clientId || count <= 0) return;
+ if (!clientId || count <= 0) return;
+ const normalizedId = String(clientId).trim().toLowerCase();
const nowIso = new Date().toISOString();
+
+ if (!dbPool) {
+ const prev = inMemoryClientActivity.get(normalizedId) || {
+ totalEvents: 0,
+ lastActive: nowIso,
+ };
+ inMemoryClientActivity.set(normalizedId, {
+ totalEvents: prev.totalEvents + count,
+ lastActive: nowIso,
+ });
+ return;
+ }
+
dbPool
.query(
`UPDATE client_sdks
@@ -115,7 +350,7 @@ function recordClientActivity(clientId, count = 1) {
last_active = $3,
updated_at = NOW()
WHERE client_id = $1`,
- [clientId, count, nowIso],
+ [normalizedId, count, nowIso],
)
.catch(() => undefined);
}
@@ -124,23 +359,103 @@ let isFlushing = false;
let flushTimer = null;
async function flushBuffer() {
- if (bufferQueue.length === 0 || isFlushing) {
+ if (pendingRedisWrites.size > 0) {
+ await Promise.allSettled([...pendingRedisWrites]);
+ }
+ if (isFlushing) {
return { written: 0, files: [] };
}
isFlushing = true;
- const batch = bufferQueue.splice(0, bufferQueue.length);
+ let totalWritten = 0;
+ const filesSet = new Set();
+
try {
- const result = await writer.writeBatch(batch);
- if (result.written > 0) {
+ const redis = getRedisClient();
+ if (redis) {
+ const lockToken = crypto.randomUUID();
+ let lockAcquired = false;
+
+ try {
+ const lockResult = await redis.set(
+ REDIS_FLUSH_LOCK_KEY,
+ lockToken,
+ 'PX',
+ FLUSH_LOCK_TTL_MS,
+ 'NX',
+ );
+ lockAcquired = lockResult === 'OK';
+ redisConnected = true;
+ } catch {
+ redisConnected = false;
+ lockAcquired = false;
+ }
+
+ if (lockAcquired) {
+ try {
+ const chunkSize = Math.max(500, runtimeConfig.batchFlushThreshold);
+ while (true) {
+ const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, chunkSize - 1);
+ if (!rawItems || rawItems.length === 0) {
+ break;
+ }
+
+ const parsedBatch = [];
+ for (const raw of rawItems) {
+ try {
+ const evt = JSON.parse(raw);
+ if (evt && evt.eventId) {
+ parsedBatch.push(evt);
+ }
+ } catch {
+ // Skip malformed entry while still trimming consumed range
+ }
+ }
+
+ if (parsedBatch.length > 0) {
+ const res = await writer.writeBatch(parsedBatch);
+ totalWritten += res.written;
+ for (const f of res.files) filesSet.add(f);
+ }
+
+ // Trim ONLY after CSV batch write succeeds
+ await redis.ltrim(REDIS_BUFFER_KEY, rawItems.length, -1);
+
+ if (rawItems.length < chunkSize) {
+ break;
+ }
+ }
+ } finally {
+ try {
+ await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
+ } catch {
+ // Lock expires automatically via PX TTL
+ }
+ }
+ }
+ }
+
+ if (bufferQueue.length > 0) {
+ const batch = bufferQueue.splice(0, bufferQueue.length);
+ try {
+ const res = await writer.writeBatch(batch);
+ totalWritten += res.written;
+ for (const f of res.files) filesSet.add(f);
+ } catch (err) {
+ bufferQueue.unshift(...batch);
+ throw err;
+ }
+ }
+
+ const files = Array.from(filesSet);
+ if (totalWritten > 0) {
console.log(
- `[CSV Worker] Flushed ${result.written} events to ${result.files.length} file(s)`,
+ `[CSV Worker] Flushed ${totalWritten} events to ${files.length} file(s)`,
);
}
- return result;
+ return { written: totalWritten, files };
} catch (err) {
console.error('[CSV Worker] Flush error:', err);
- bufferQueue.unshift(...batch);
- return { written: 0, files: [] };
+ return { written: totalWritten, files: Array.from(filesSet) };
} finally {
isFlushing = false;
}
@@ -159,6 +474,92 @@ function scheduleFlushTimer(intervalMs) {
scheduleFlushTimer(runtimeConfig.batchFlushIntervalMs);
+const server = http.createServer(app);
+const io = new Server(server, {
+ cors: { origin: '*', methods: ['GET', 'POST'] },
+});
+
+function applyRuntimeSettingsObject(settingsObj) {
+ if (!settingsObj || typeof settingsObj !== 'object') {
+ return { jwtRotated: false };
+ }
+
+ if (settingsObj.maxEventsPerSec !== undefined) {
+ const val = Number(settingsObj.maxEventsPerSec);
+ if (Number.isFinite(val) && val >= 1) {
+ runtimeConfig.maxEventsPerSec = Math.floor(val);
+ }
+ }
+
+ if (settingsObj.maxPayloadSizeKb !== undefined) {
+ const kb = Number(settingsObj.maxPayloadSizeKb);
+ if (Number.isFinite(kb) && kb >= 1) {
+ runtimeConfig.maxPayloadSizeBytes = Math.floor(kb) * 1024;
+ }
+ }
+
+ if (settingsObj.batchFlushThreshold !== undefined) {
+ const threshold = Number(settingsObj.batchFlushThreshold);
+ if (Number.isFinite(threshold) && threshold >= 1) {
+ runtimeConfig.batchFlushThreshold = Math.floor(threshold);
+ }
+ }
+
+ if (settingsObj.enableIdempotency !== undefined) {
+ runtimeConfig.enableIdempotency = Boolean(settingsObj.enableIdempotency);
+ }
+
+ if (settingsObj.fileRotationMb !== undefined) {
+ const nextRotationMb = Number(settingsObj.fileRotationMb);
+ if (
+ Number.isFinite(nextRotationMb) &&
+ nextRotationMb >= 1 &&
+ nextRotationMb !== runtimeConfig.fileRotationMb
+ ) {
+ runtimeConfig.fileRotationMb = nextRotationMb;
+ writer.setMaxFileSizeMb(nextRotationMb);
+ }
+ }
+
+ if (settingsObj.autoPruneDays !== undefined && settingsObj.autoPruneDays !== null) {
+ const parsedPrune = Number(settingsObj.autoPruneDays);
+ if (Number.isFinite(parsedPrune) && parsedPrune >= 0) {
+ const nextPruneDays = Math.floor(parsedPrune);
+ const changed = nextPruneDays !== runtimeConfig.autoPruneDays;
+ runtimeConfig.autoPruneDays = nextPruneDays;
+ if (changed && nextPruneDays > 0) {
+ writer.pruneExpiredPartitions(nextPruneDays);
+ }
+ }
+ }
+
+ if (settingsObj.batchFlushIntervalMs !== undefined) {
+ const nextIntervalMs = Math.max(
+ 100,
+ Number(settingsObj.batchFlushIntervalMs) || 2000,
+ );
+ if (nextIntervalMs !== runtimeConfig.batchFlushIntervalMs) {
+ scheduleFlushTimer(nextIntervalMs);
+ }
+ }
+
+ let jwtRotated = false;
+ if (settingsObj.jwtSecret !== undefined && settingsObj.jwtSecret !== null) {
+ const candidateSecret = String(settingsObj.jwtSecret).trim();
+ const validSecret =
+ candidateSecret.length >= 32 ||
+ (!IS_PRODUCTION && candidateSecret.length > 0);
+ if (validSecret && candidateSecret !== runtimeConfig.jwtSecret) {
+ runtimeConfig.jwtSecret = candidateSecret;
+ clientAuthCache.clear();
+ io.disconnectSockets(true);
+ jwtRotated = true;
+ }
+ }
+
+ return { jwtRotated };
+}
+
async function ensureDatabaseSchema() {
if (!dbPool) return;
try {
@@ -192,6 +593,12 @@ async function ensureDatabaseSchema() {
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`);
+ await dbPool.query(`
+ CREATE TABLE IF NOT EXISTS deleted_clients (
+ client_id TEXT PRIMARY KEY,
+ deleted_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+ )
+ `);
} catch {
// Ignore transient startup DB error; retried on next sync cycle
}
@@ -210,69 +617,132 @@ async function syncSettingsFromDatabase() {
);
if (!res.rowCount || res.rowCount === 0) return;
const row = res.rows[0];
+ const parsedPrune =
+ row.auto_prune_days !== undefined && row.auto_prune_days !== null
+ ? Number(row.auto_prune_days)
+ : NaN;
- runtimeConfig.maxEventsPerSec = Number(row.max_events_per_sec) || 100;
- runtimeConfig.maxPayloadSizeBytes =
- (Number(row.max_payload_size_kb) || 100) * 1024;
- runtimeConfig.batchFlushThreshold =
- Math.max(1, Number(row.batch_flush_threshold) || 500);
- runtimeConfig.enableIdempotency = Boolean(row.enable_idempotency);
-
- if (row.jwt_secret && String(row.jwt_secret).trim().length > 0) {
- runtimeConfig.jwtSecret = String(row.jwt_secret).trim();
- }
-
- const nextRotationMb = Number(row.file_rotation_mb) || 100;
- if (nextRotationMb !== runtimeConfig.fileRotationMb) {
- runtimeConfig.fileRotationMb = nextRotationMb;
- writer.setMaxFileSizeMb(nextRotationMb);
- }
-
- const nextPruneDays = Number(row.auto_prune_days) || 30;
- if (nextPruneDays > 0) {
- writer.pruneExpiredPartitions(nextPruneDays);
- }
-
- const nextIntervalMs = Math.max(
- 100,
- Number(row.batch_flush_interval_ms) || 2000,
- );
- if (nextIntervalMs !== runtimeConfig.batchFlushIntervalMs) {
- scheduleFlushTimer(nextIntervalMs);
- }
+ applyRuntimeSettingsObject({
+ maxEventsPerSec: Number(row.max_events_per_sec) || 100,
+ maxPayloadSizeKb: Number(row.max_payload_size_kb) || 100,
+ batchFlushThreshold: Math.max(1, Number(row.batch_flush_threshold) || 500),
+ batchFlushIntervalMs: Math.max(
+ 100,
+ Number(row.batch_flush_interval_ms) || 2000,
+ ),
+ fileRotationMb: Number(row.file_rotation_mb) || 100,
+ enableIdempotency: Boolean(row.enable_idempotency),
+ autoPruneDays: Number.isFinite(parsedPrune) ? parsedPrune : 30,
+ jwtSecret: row.jwt_secret ? String(row.jwt_secret).trim() : undefined,
+ });
} catch {
// Keep existing runtimeConfig if table is not ready yet
}
}
+const PRUNE_INTERVAL_MS = 60 * 60 * 1000;
let settingsSyncTimer = null;
-if (dbPool) {
- void syncSettingsFromDatabase();
- settingsSyncTimer = setInterval(() => {
+let pruneTimer = null;
+if (process.env.DATASTACK_DISABLE_LISTEN !== '1') {
+ if (dbPool) {
void syncSettingsFromDatabase();
- }, 3000);
+ settingsSyncTimer = setInterval(() => {
+ void syncSettingsFromDatabase();
+ }, 3000);
+ }
+ pruneTimer = setInterval(() => {
+ if (runtimeConfig.autoPruneDays > 0) {
+ writer.pruneExpiredPartitions(runtimeConfig.autoPruneDays);
+ }
+ }, PRUNE_INTERVAL_MS);
}
-const server = http.createServer(app);
-const io = new Server(server, {
- cors: { origin: '*', methods: ['GET', 'POST'] },
+const socketHandlersApi = setupSocketHandlers(io, bufferQueue, writer, {
+ getRuntimeConfig: () => runtimeConfig,
+ verifyClientInDb,
+ recordClientActivity,
+ onBufferPush: () => {
+ const redis = getRedisClient();
+ if (redis && bufferQueue.length > 0) {
+ const pushed = bufferQueue.splice(0, bufferQueue.length);
+ const op = pushEventsToRedisOrFallback(pushed).then((totalQueued) => {
+ if (totalQueued >= runtimeConfig.batchFlushThreshold) {
+ void flushBuffer();
+ }
+ });
+ pendingRedisWrites.add(op);
+ void op.finally(() => {
+ pendingRedisWrites.delete(op);
+ });
+ return;
+ }
+
+ if (bufferQueue.length >= runtimeConfig.batchFlushThreshold) {
+ void flushBuffer();
+ }
+ },
});
-// Health check endpoint with activeSockets count and runtime capabilities
-app.get('/health', (_req, res) => {
+async function getRedisBufferLen() {
+ const redis = getRedisClient();
+ if (!redis) return 0;
+ try {
+ const len = Number(await redis.llen(REDIS_BUFFER_KEY)) || 0;
+ redisConnected = true;
+ return len;
+ } catch {
+ redisConnected = false;
+ return 0;
+ }
+}
+
+// Public health check endpoint
+app.get('/health', async (_req, res) => {
+ const redisBufferSize = await getRedisBufferLen();
res.json({
status: 'online',
service: 'DataStack Express Ingestion',
port: PORT,
activeSockets: io.engine ? io.engine.clientsCount : 0,
- bufferSize: bufferQueue.length,
+ bufferSize: bufferQueue.length + redisBufferSize,
+ localBufferSize: bufferQueue.length,
+ redisBufferSize,
+ redisConnected,
maxEventsPerSec: runtimeConfig.maxEventsPerSec,
maxPayloadSizeBytes: runtimeConfig.maxPayloadSizeBytes,
uptime: process.uptime(),
});
});
-// Internal flush endpoint triggered by Next.js POST /api/storage
+// Require internal shared authentication on all /internal/* bridge endpoints
+app.use('/internal', requireInternalAuth);
+
+// Internal metrics endpoint for Next.js dashboard & in-memory client activity aggregation
+app.get('/internal/metrics', async (_req, res) => {
+ const redisBufferSize = await getRedisBufferLen();
+ res.json({
+ activeSockets: io.engine ? io.engine.clientsCount : 0,
+ bufferSize: bufferQueue.length + redisBufferSize,
+ localBufferSize: bufferQueue.length,
+ redisBufferSize,
+ redisConnected,
+ registrySize: inMemoryClientRegistry.size,
+ hasSyncedInMemoryClients,
+ securityStats: socketHandlersApi.getSecurityStats(),
+ clientActivity: Object.fromEntries(inMemoryClientActivity.entries()),
+ });
+});
+
+// Internal instant settings synchronization triggered by Next.js POST /api/settings
+app.post('/internal/settings-sync', (req, res) => {
+ const result = applyRuntimeSettingsObject(req.body || {});
+ res.json({
+ success: true,
+ jwtRotated: result.jwtRotated,
+ });
+});
+
+// Internal flush endpoint triggered by Next.js POST /api/storage & GET /api/export
app.post('/internal/flush', async (_req, res) => {
try {
const result = await flushBuffer();
@@ -290,34 +760,149 @@ app.post('/internal/flush', async (_req, res) => {
});
// Internal endpoint to expose Port 4000 buffered events before CSV flush
-app.get('/internal/recent', (req, res) => {
+app.get('/internal/recent', async (req, res) => {
const clientId = req.query.clientId ? String(req.query.clientId) : '';
- const limit = Math.min(200, Math.max(1, Number(req.query.limit) || 50));
- const filtered =
+ const limit = Math.min(2000, Math.max(1, Number(req.query.limit) || 50));
+
+ const redis = getRedisClient();
+ let redisEvents = [];
+ if (redis) {
+ try {
+ const readCap = Math.min(REDIS_RECENT_LIMIT, Math.max(limit * 2, limit));
+ const rawItems = await redis.lrange(REDIS_RECENT_KEY, 0, readCap - 1);
+ redisConnected = true;
+ for (const raw of rawItems) {
+ try {
+ const evt = JSON.parse(raw);
+ if (!evt || !evt.eventId) continue;
+ if (clientId && clientId !== 'all' && evt.clientId !== clientId) continue;
+ redisEvents.push(evt);
+ if (redisEvents.length >= limit) break;
+ } catch {
+ // ignore malformed item
+ }
+ }
+ } catch {
+ redisConnected = false;
+ redisEvents = [];
+ }
+ }
+
+ const localFiltered =
clientId && clientId !== 'all'
? bufferQueue.filter((evt) => evt.clientId === clientId)
: bufferQueue;
- const recent = filtered.slice(-limit).reverse();
- res.json({ events: recent, count: recent.length });
+ const localRecent = localFiltered.slice(-limit).reverse();
+
+ const merged = [];
+ const seen = new Set();
+ for (const evt of [...localRecent, ...redisEvents]) {
+ if (!evt || !evt.eventId) continue;
+ const key = `${evt.clientId}:${evt.eventId}`;
+ if (seen.has(key)) continue;
+ seen.add(key);
+ merged.push(evt);
+ if (merged.length >= limit) break;
+ }
+
+ res.json({ events: merged, count: merged.length });
});
-setupSocketHandlers(io, bufferQueue, writer, {
- getRuntimeConfig: () => runtimeConfig,
- verifyClientInDb,
- recordClientActivity,
- onBufferPush: () => {
- if (bufferQueue.length >= runtimeConfig.batchFlushThreshold) {
- void flushBuffer();
+// Internal endpoint triggered by Next.js SDKManager when a client is deleted, revoked, rotated, or synced
+app.post('/internal/client-sync', (req, res) => {
+ const action = req.body && req.body.action ? String(req.body.action) : '';
+
+ if (action === 'bulk_sync' && Array.isArray(req.body.clients)) {
+ hasSyncedInMemoryClients = true;
+ inMemoryClientRegistry.clear();
+ clientAuthCache.clear();
+ const activeIds = new Set();
+ for (const item of req.body.clients) {
+ if (!item || !item.clientId) continue;
+ const cid = String(item.clientId).trim().toLowerCase();
+ activeIds.add(cid);
+ deletedClientsSet.delete(cid);
+ inMemoryClientRegistry.set(cid, {
+ clientId: cid,
+ name: String(item.name || cid),
+ status: item.status === 'revoked' ? 'revoked' : 'active',
+ token: String(item.token || ''),
+ rateLimit: Math.max(1, Number(item.rateLimit) || 100),
+ });
+ if (item.status === 'revoked') {
+ io.to(`client:${cid}`).disconnectSockets(true);
+ }
}
- },
+ return res.json({ success: true, count: activeIds.size });
+ }
+
+ const rawClientId = req.body && req.body.clientId ? String(req.body.clientId) : '';
+ const normalizedId = rawClientId.trim().toLowerCase();
+ const clientPayload =
+ req.body && req.body.client && typeof req.body.client === 'object'
+ ? req.body.client
+ : null;
+
+ if (normalizedId) {
+ hasSyncedInMemoryClients = true;
+ clientAuthCache.delete(normalizedId);
+
+ if (action === 'deleted') {
+ deletedClientsSet.add(normalizedId);
+ inMemoryClientRegistry.delete(normalizedId);
+ inMemoryClientActivity.delete(normalizedId);
+ io.to(`client:${normalizedId}`).disconnectSockets(true);
+ } else if (action === 'revoked') {
+ const existingMem = inMemoryClientRegistry.get(normalizedId);
+ if (clientPayload) {
+ inMemoryClientRegistry.set(normalizedId, {
+ clientId: normalizedId,
+ name: String(clientPayload.name || normalizedId),
+ status: 'revoked',
+ token: String(clientPayload.token || ''),
+ rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
+ });
+ } else if (existingMem) {
+ existingMem.status = 'revoked';
+ } else {
+ inMemoryClientRegistry.set(normalizedId, {
+ clientId: normalizedId,
+ name: normalizedId,
+ status: 'revoked',
+ token: '',
+ rateLimit: 100,
+ });
+ }
+ io.to(`client:${normalizedId}`).disconnectSockets(true);
+ } else if (action === 'registered') {
+ deletedClientsSet.delete(normalizedId);
+ if (clientPayload) {
+ inMemoryClientRegistry.set(normalizedId, {
+ clientId: normalizedId,
+ name: String(clientPayload.name || normalizedId),
+ status: clientPayload.status === 'revoked' ? 'revoked' : 'active',
+ token: String(clientPayload.token || ''),
+ rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
+ });
+ }
+ } else if (action === 'updated') {
+ if (clientPayload) {
+ inMemoryClientRegistry.set(normalizedId, {
+ clientId: normalizedId,
+ name: String(clientPayload.name || normalizedId),
+ status: clientPayload.status === 'revoked' ? 'revoked' : 'active',
+ token: String(clientPayload.token || ''),
+ rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
+ });
+ }
+ io.to(`client:${normalizedId}`).disconnectSockets(true);
+ }
+ }
+
+ res.json({ success: true });
});
-let isShuttingDown = false;
-async function gracefulShutdown(signal) {
- if (isShuttingDown) return;
- isShuttingDown = true;
- console.log(`[DataStack Ingestion Engine] Received ${signal}, flushing buffer before exit...`);
-
+function stopTimers() {
if (flushTimer) {
clearInterval(flushTimer);
flushTimer = null;
@@ -326,6 +911,19 @@ async function gracefulShutdown(signal) {
clearInterval(settingsSyncTimer);
settingsSyncTimer = null;
}
+ if (pruneTimer) {
+ clearInterval(pruneTimer);
+ pruneTimer = null;
+ }
+}
+
+let isShuttingDown = false;
+async function gracefulShutdown(signal) {
+ if (isShuttingDown) return;
+ isShuttingDown = true;
+ console.log(`[DataStack Ingestion Engine] Received ${signal}, flushing buffer before exit...`);
+
+ stopTimers();
try {
await flushBuffer();
@@ -336,6 +934,9 @@ async function gracefulShutdown(signal) {
try {
io.close();
server.close();
+ if (redisClient && typeof redisClient.quit === 'function') {
+ await redisClient.quit().catch(() => undefined);
+ }
if (dbPool) {
await dbPool.end();
}
@@ -346,17 +947,32 @@ async function gracefulShutdown(signal) {
process.exit(0);
}
-process.on('SIGTERM', () => {
- void gracefulShutdown('SIGTERM');
-});
-process.on('SIGINT', () => {
- void gracefulShutdown('SIGINT');
-});
+if (process.env.DATASTACK_DISABLE_LISTEN !== '1') {
+ process.on('SIGTERM', () => {
+ void gracefulShutdown('SIGTERM');
+ });
+ process.on('SIGINT', () => {
+ void gracefulShutdown('SIGINT');
+ });
-server.listen(PORT, () => {
- console.log(
- `[DataStack Ingestion Engine] Express + Socket.IO listening on port ${PORT}`,
- );
-});
+ server.listen(PORT, () => {
+ console.log(
+ `[DataStack Ingestion Engine] Express + Socket.IO listening on port ${PORT}`,
+ );
+ });
+}
-module.exports = { app, server, io, bufferQueue, flushBuffer };
+module.exports = {
+ app,
+ server,
+ io,
+ writer,
+ bufferQueue,
+ flushBuffer,
+ runtimeConfig,
+ verifyClientInDb,
+ recordClientActivity,
+ applyRuntimeSettingsObject,
+ setRedisClientForTests,
+ stopTimers,
+};
diff --git a/server/socket/handlers.js b/server/socket/handlers.js
index 9f1922b..860ab3b 100644
--- a/server/socket/handlers.js
+++ b/server/socket/handlers.js
@@ -11,8 +11,10 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
: () => ({
jwtSecret:
options.jwtSecret ||
- process.env.DATASTACK_JWT_SECRET ||
- "datastack-jwt-default-secret-key-32-chars-long",
+ (process.env.DATASTACK_JWT_SECRET || "").trim() ||
+ (process.env.NODE_ENV === "production"
+ ? ""
+ : "datastack-jwt-default-secret-key-32-chars-long"),
maxEventsPerSec: 100,
maxPayloadSizeBytes: 100 * 1024,
batchFlushThreshold: 500,
@@ -38,8 +40,9 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
const rateLimitMap = new Map();
let lastCleanupAt = Date.now();
- function cleanupMaps(now) {
+ function cleanupMaps(now, force = false) {
if (
+ !force &&
now - lastCleanupAt < CLEANUP_INTERVAL_MS &&
idempotencyMap.size <= IDEMPOTENCY_MAX_ENTRIES
) {
@@ -60,6 +63,16 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
}
}
+ function getSecurityStats() {
+ cleanupMaps(Date.now(), true);
+ const config = getRuntimeConfig();
+ return {
+ activeIdempotencyKeys: idempotencyMap.size,
+ rateLimitedClientsCount: rateLimitMap.size,
+ maxPayloadSizeBytes: Number(config.maxPayloadSizeBytes) || 100 * 1024,
+ };
+ }
+
function validateSingleEvent(clientId, eventId, rawData, clientRateLimit) {
const config = getRuntimeConfig();
const now = Date.now();
@@ -136,10 +149,19 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
? String(authHeader).slice(7).trim()
: String(authHeader).trim();
const config = getRuntimeConfig();
+ const minSecretLen = process.env.NODE_ENV === "production" ? 32 : 1;
+ if (!config.jwtSecret || config.jwtSecret.length < minSecretLen) {
+ return next(
+ new Error("AUTH_UNAVAILABLE: DATASTACK_JWT_SECRET is not configured"),
+ );
+ }
const decoded = jwt.verify(cleanToken, config.jwtSecret);
+ const normalizedClientId = String(decoded.clientId || "")
+ .trim()
+ .toLowerCase();
const dbCheck = await verifyClientInDb(
- decoded.clientId,
+ normalizedClientId,
cleanToken,
true,
);
@@ -147,7 +169,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
return next(new Error(dbCheck.reason || "CLIENT_REVOKED"));
}
- socket.data.clientId = decoded.clientId;
+ socket.data.clientId = normalizedClientId;
socket.data.token = cleanToken;
socket.data.rateLimit =
dbCheck.rateLimit || decoded.rateLimit || config.maxEventsPerSec || 100;
@@ -157,6 +179,22 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
}
});
+ const ENVELOPE_KEYS = new Set(["eventId", "timestamp", "data", "clientId"]);
+ function extractBatchItemData(safeItem) {
+ if (
+ safeItem &&
+ typeof safeItem === "object" &&
+ !Array.isArray(safeItem) &&
+ safeItem.data !== undefined
+ ) {
+ const keys = Object.keys(safeItem);
+ if (keys.every((k) => ENVELOPE_KEYS.has(k))) {
+ return safeItem.data;
+ }
+ }
+ return safeItem;
+ }
+
io.on("connection", (socket) => {
const clientId = socket.data.clientId;
socket.join(`client:${clientId}`);
@@ -274,8 +312,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
safeItem.eventId ||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
const timestamp = safeItem.timestamp || new Date().toISOString();
- const rawData =
- safeItem.data !== undefined ? safeItem.data : safeItem;
+ const rawData = extractBatchItemData(safeItem);
const check = validateSingleEvent(
clientId,
@@ -403,6 +440,10 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
}
});
});
+
+ return {
+ getSecurityStats,
+ };
}
module.exports = { setupSocketHandlers };
diff --git a/server/workers/csvWriter.js b/server/workers/csvWriter.js
index 4cb5359..79102c0 100644
--- a/server/workers/csvWriter.js
+++ b/server/workers/csvWriter.js
@@ -2,6 +2,15 @@ const fs = require('fs');
const path = require('path');
const { Parser } = require('json2csv');
+const LOCK_RETRY_MIN_MS = 5;
+const LOCK_RETRY_MAX_MS = 18;
+const LOCK_TIMEOUT_MS = 10000;
+const STALE_LOCK_AGE_MS = 30000;
+
+function sleepMs(ms) {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
/**
* Background CSV Writer Worker
* Flushes batches of real-time events to partitioned CSV files.
@@ -27,14 +36,100 @@ class CsvWriterWorker {
}
}
- getTargetFilePath(clientId, dateStr) {
+ ensureClientDir(clientId) {
const safeClientId = String(clientId || 'unknown').replace(/[^a-zA-Z0-9_-]/g, '_');
const clientDir = path.join(this.baseDir, safeClientId);
if (!fs.existsSync(clientDir)) {
fs.mkdirSync(clientDir, { recursive: true });
}
+ return clientDir;
+ }
- const date = dateStr || new Date().toISOString().split('T')[0];
+ async withPartitionLock(clientId, dateStr, fn) {
+ const clientDir = this.ensureClientDir(clientId);
+ const date = this.extractSafePartitionDate(dateStr);
+ const lockPath = path.join(clientDir, `.${date}.csv.lock`);
+ const deadline = Date.now() + LOCK_TIMEOUT_MS;
+
+ let acquired = false;
+ while (Date.now() < deadline) {
+ try {
+ const fd = fs.openSync(lockPath, 'wx');
+ try {
+ fs.writeFileSync(fd, `${process.pid}:${Date.now()}`, 'utf8');
+ } finally {
+ fs.closeSync(fd);
+ }
+ acquired = true;
+ break;
+ } catch (err) {
+ if (!err || err.code !== 'EEXIST') {
+ throw err;
+ }
+ try {
+ const lockStat = fs.statSync(lockPath);
+ if (Date.now() - lockStat.mtimeMs > STALE_LOCK_AGE_MS) {
+ fs.unlinkSync(lockPath);
+ continue;
+ }
+ } catch {
+ // Released concurrently
+ }
+ const jitter =
+ LOCK_RETRY_MIN_MS +
+ Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS));
+ await sleepMs(jitter);
+ }
+ }
+
+ if (!acquired) {
+ throw new Error(`Timed out waiting for CSV partition lock: ${lockPath}`);
+ }
+
+ try {
+ return await fn();
+ } finally {
+ try {
+ if (fs.existsSync(lockPath)) {
+ fs.unlinkSync(lockPath);
+ }
+ } catch {
+ // Ignore if already removed
+ }
+ }
+ }
+
+ extractSafePartitionDate(rawTimestamp) {
+ const fallback = new Date().toISOString().slice(0, 10);
+ if (!rawTimestamp || typeof rawTimestamp !== 'string') {
+ return fallback;
+ }
+ const trimmed = rawTimestamp.trim();
+ const prefix = trimmed.slice(0, 10);
+ if (/^\d{4}-\d{2}-\d{2}$/.test(prefix)) {
+ const parsedPrefix = new Date(`${prefix}T00:00:00.000Z`);
+ if (!Number.isNaN(parsedPrefix.getTime())) {
+ return prefix;
+ }
+ }
+ if (/^\d+(\.\d+)?$/.test(trimmed)) {
+ const num = Number(trimmed);
+ const ms = num < 1e11 ? num * 1000 : num;
+ const d = new Date(ms);
+ if (!Number.isNaN(d.getTime())) {
+ return d.toISOString().slice(0, 10);
+ }
+ }
+ const parsed = new Date(trimmed);
+ if (!Number.isNaN(parsed.getTime())) {
+ return parsed.toISOString().slice(0, 10);
+ }
+ return fallback;
+ }
+
+ getTargetFilePath(clientId, dateStr) {
+ const clientDir = this.ensureClientDir(clientId);
+ const date = this.extractSafePartitionDate(dateStr);
let baseFileName = `${date}.csv`;
let targetPath = path.join(clientDir, baseFileName);
@@ -62,7 +157,7 @@ class CsvWriterWorker {
const grouped = new Map();
for (const evt of events) {
- const date = (evt.timestamp || new Date().toISOString()).split('T')[0];
+ const date = this.extractSafePartitionDate(evt.timestamp);
const key = `${evt.clientId}:::${date}`;
if (!grouped.has(key)) grouped.set(key, []);
grouped.get(key).push(evt);
@@ -73,30 +168,34 @@ class CsvWriterWorker {
for (const [key, batch] of grouped.entries()) {
const [clientId, date] = key.split(':::');
- const targetPath = this.getTargetFilePath(clientId, date);
- let hasContent = false;
- try {
- hasContent =
- fs.existsSync(targetPath) && fs.statSync(targetPath).size > 0;
- } catch {
- hasContent = false;
- }
+ const targetPath = await this.withPartitionLock(clientId, date, () => {
+ const resolvedPath = this.getTargetFilePath(clientId, date);
+ let hasContent = false;
+ try {
+ hasContent =
+ fs.existsSync(resolvedPath) && fs.statSync(resolvedPath).size > 0;
+ } catch {
+ hasContent = false;
+ }
- const cleanBatch = batch.map((item) => ({
- clientId: item.clientId,
- eventId: item.eventId,
- timestamp: item.timestamp,
- payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
- }));
+ const cleanBatch = batch.map((item) => ({
+ clientId: item.clientId,
+ eventId: item.eventId,
+ timestamp: item.timestamp,
+ payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
+ }));
- const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
- const csv = parser.parse(cleanBatch);
+ const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
+ const csv = parser.parse(cleanBatch);
- if (hasContent) {
- fs.appendFileSync(targetPath, '\n' + csv, 'utf8');
- } else {
- fs.writeFileSync(targetPath, csv, 'utf8');
- }
+ if (hasContent) {
+ fs.appendFileSync(resolvedPath, '\n' + csv, 'utf8');
+ } else {
+ fs.writeFileSync(resolvedPath, csv, 'utf8');
+ }
+
+ return resolvedPath;
+ });
files.push(targetPath);
}
diff --git a/tests/phase01-auth.test.ts b/tests/phase01-auth.test.ts
new file mode 100644
index 0000000..4a73804
--- /dev/null
+++ b/tests/phase01-auth.test.ts
@@ -0,0 +1,340 @@
+import assert from "node:assert/strict";
+import { after, afterEach, beforeEach, describe, it } from "node:test";
+import { NextRequest } from "next/server";
+import { GET as profileGet, PUT as profilePut } from "@/app/api/auth/profile/route";
+import { GET as clientsGet, POST as clientsPost } from "@/app/api/clients/route";
+import {
+ DELETE as clientDelete,
+ GET as clientGet,
+} from "@/app/api/clients/[id]/route";
+import { POST as eventsPost } from "@/app/api/events/route";
+import {
+ createAdminSessionToken,
+ getAdminSessionPayload,
+ getAdminSessionSecret,
+} from "@/core/auth/adminSession";
+import {
+ consumeAdminLoginAttempt,
+ createInitialAdmin,
+ getAdminSetupKey,
+ isAdminSetupComplete,
+ resetAdminStateForTests,
+ verifyAdminCredentials,
+ verifyAdminSetupKey,
+ verifyRequestAdminSession,
+} from "@/core/auth/adminUsers";
+import { AuthService, resolveDefaultJwtSecret } from "@/core/auth/jwt";
+import { isSameOriginRequest } from "@/middleware";
+
+const ORIGINAL_ENV = { ...process.env };
+
+function restoreEnv() {
+ for (const key of Object.keys(process.env)) {
+ if (!(key in ORIGINAL_ENV)) {
+ delete process.env[key];
+ }
+ }
+ Object.assign(process.env, ORIGINAL_ENV);
+ AuthService.setSecret(resolveDefaultJwtSecret() || "");
+}
+
+describe("Phase-01 Auth, Secrets, Revocation & Proxy Hardening", () => {
+ beforeEach(() => {
+ restoreEnv();
+ resetAdminStateForTests();
+ });
+
+ afterEach(() => {
+ restoreEnv();
+ resetAdminStateForTests();
+ });
+
+ after(() => {
+ const bufState = (
+ globalThis as {
+ __dataStackBufferState?: { flushTimer?: ReturnType | null };
+ }
+ ).__dataStackBufferState;
+ if (bufState?.flushTimer) {
+ clearInterval(bufState.flushTimer);
+ bufState.flushTimer = null;
+ }
+ });
+
+ it("BUG-01 & BUG-02: fails closed in production when secrets or DB/setup are missing", async () => {
+ process.env.NODE_ENV = "production";
+ delete process.env.DATABASE_URL;
+ delete process.env.DATASTACK_ADMIN_SESSION_SECRET;
+ delete process.env.DATASTACK_ADMIN_SETUP_KEY;
+ delete process.env.DATASTACK_JWT_SECRET;
+ delete process.env.DATASTACK_ADMIN_PASSWORD;
+
+ assert.equal(getAdminSessionSecret(), null);
+ assert.equal(getAdminSetupKey(), null);
+ assert.equal(resolveDefaultJwtSecret(), "");
+ assert.equal(
+ verifyAdminSetupKey("datastack-default-setup-key-32-chars"),
+ false,
+ );
+ assert.equal(await isAdminSetupComplete(), false);
+
+ await assert.rejects(
+ () => verifyAdminCredentials("admin@datastack.dev", "admin12345678"),
+ /setup/i,
+ );
+
+ AuthService.setSecret(resolveDefaultJwtSecret());
+ assert.equal(AuthService.isConfigured(), false);
+ assert.throws(() => AuthService.generateClientToken({ clientId: "c1" }));
+ });
+
+ it("BUG-03: bumps tokenVersion on credential update and rejects old session cookies", async () => {
+ delete process.env.DATABASE_URL;
+ process.env.DATASTACK_ALLOW_DEMO_ADMIN = "false";
+ const created = await createInitialAdmin("owner@datastack.dev", "InitialPass123!");
+ assert.equal(created, true);
+ assert.equal(await isAdminSetupComplete(), true);
+
+ const secret = getAdminSessionSecret()!;
+ const v1Token = await createAdminSessionToken(
+ "owner@datastack.dev",
+ secret,
+ Date.now() + 3600_000,
+ 1,
+ );
+
+ const checkV1 = await verifyRequestAdminSession(v1Token);
+ assert.equal(checkV1.valid, true);
+
+ const updateReq = new NextRequest("http://localhost:3000/api/auth/profile", {
+ method: "PUT",
+ headers: {
+ "content-type": "application/json",
+ cookie: `datastack_admin_session=${v1Token}`,
+ },
+ body: JSON.stringify({
+ email: "owner2@datastack.dev",
+ currentPassword: "InitialPass123!",
+ newPassword: "UpdatedPass456!",
+ }),
+ });
+
+ const updateRes = await profilePut(updateReq);
+ assert.equal(updateRes.status, 200);
+ const setCookie = updateRes.cookies.get("datastack_admin_session")?.value;
+ assert.ok(setCookie);
+
+ const newPayload = await getAdminSessionPayload(setCookie!, secret);
+ assert.ok(newPayload);
+ assert.equal(newPayload!.tokenVersion, 2);
+
+ // Old v1 session must now return 401
+ const staleProfileReq = new NextRequest("http://localhost:3000/api/auth/profile", {
+ method: "GET",
+ headers: { cookie: `datastack_admin_session=${v1Token}` },
+ });
+ const staleRes = await profileGet(staleProfileReq);
+ assert.equal(staleRes.status, 401);
+
+ // New v2 session succeeds
+ const freshProfileReq = new NextRequest("http://localhost:3000/api/auth/profile", {
+ method: "GET",
+ headers: { cookie: `datastack_admin_session=${setCookie}` },
+ });
+ const freshRes = await profileGet(freshProfileReq);
+ assert.equal(freshRes.status, 200);
+ });
+
+ it("BUG-04: throttles by composite IP+email without locking out other admin IPs", async () => {
+ const secret = "test-throttle-secret-at-least-32-chars-long";
+ const targetEmail = "admin@datastack.dev";
+
+ for (let i = 0; i < 5; i++) {
+ const dec = await consumeAdminLoginAttempt(targetEmail, secret, "198.51.100.10");
+ assert.equal(dec.allowed, true);
+ }
+
+ // 6th attempt from attacker IP is blocked
+ const blocked = await consumeAdminLoginAttempt(targetEmail, secret, "198.51.100.10");
+ assert.equal(blocked.allowed, false);
+ assert.ok(blocked.retryAfterSeconds >= 1);
+
+ // Real admin from different IP is still allowed
+ const otherIpAllowed = await consumeAdminLoginAttempt(
+ targetEmail,
+ secret,
+ "203.0.113.55",
+ );
+ assert.equal(otherIpAllowed.allowed, true);
+
+ // Same IP spraying 25 distinct emails triggers IP bucket block on 26th
+ for (let i = 0; i < 25; i++) {
+ const spray = await consumeAdminLoginAttempt(
+ `spray${i}@example.com`,
+ secret,
+ "198.51.100.99",
+ );
+ assert.equal(spray.allowed, true);
+ }
+ const sprayBlocked = await consumeAdminLoginAttempt(
+ "spray25@example.com",
+ secret,
+ "198.51.100.99",
+ );
+ assert.equal(sprayBlocked.allowed, false);
+ });
+
+ it("BUG-05 & BUG-15: redacts tokens in client list, returns 400 on malformed JSON, and blocks deleted/unregistered Bearer tokens", async () => {
+ delete process.env.DATABASE_URL;
+ await createInitialAdmin("admin@datastack.dev", "AdminPass123!");
+ const secret = getAdminSessionSecret()!;
+ const adminCookie = await createAdminSessionToken(
+ "admin@datastack.dev",
+ secret,
+ Date.now() + 3600_000,
+ 1,
+ );
+
+ const createReq = new NextRequest("http://localhost:3000/api/clients", {
+ method: "POST",
+ headers: {
+ "content-type": "application/json",
+ cookie: `datastack_admin_session=${adminCookie}`,
+ },
+ body: JSON.stringify({
+ clientId: "phase01_probe_client",
+ name: "Phase01 Probe",
+ rateLimit: 50,
+ }),
+ });
+ const createRes = await clientsPost(createReq);
+ assert.equal(createRes.status, 201);
+ const createdJson = await createRes.json();
+ const clientToken = createdJson.client.token as string;
+ assert.ok(clientToken.length > 20);
+
+ // List endpoint must redact full token & secret
+ const listReq = new NextRequest("http://localhost:3000/api/clients", {
+ method: "GET",
+ headers: { cookie: `datastack_admin_session=${adminCookie}` },
+ });
+ const listRes = await clientsGet(listReq);
+ assert.equal(listRes.status, 200);
+ const listJson = await listRes.json();
+ const listed = listJson.clients.find(
+ (c: { clientId: string }) => c.clientId === "phase01_probe_client",
+ );
+ assert.ok(listed);
+ assert.equal("token" in listed, false);
+ assert.equal("secret" in listed, false);
+ assert.equal(listed.tokenPreview, `${clientToken.slice(0, 12)}...`);
+
+ // Single client GET omits token by default, reveals on ?includeToken=true
+ const getDefaultRes = await clientGet(
+ new NextRequest("http://localhost:3000/api/clients/phase01_probe_client", {
+ headers: { cookie: `datastack_admin_session=${adminCookie}` },
+ }),
+ { params: Promise.resolve({ id: "phase01_probe_client" }) },
+ );
+ const defaultClient = (await getDefaultRes.json()).client;
+ assert.equal("token" in defaultClient, false);
+
+ const getIncludeRes = await clientGet(
+ new NextRequest(
+ "http://localhost:3000/api/clients/phase01_probe_client?includeToken=true",
+ { headers: { cookie: `datastack_admin_session=${adminCookie}` } },
+ ),
+ { params: Promise.resolve({ id: "phase01_probe_client" }) },
+ );
+ const includeClient = (await getIncludeRes.json()).client;
+ assert.equal(includeClient.token, clientToken);
+
+ // Malformed JSON on POST /api/events returns 400 (not 500)
+ const malformedRes = await eventsPost(
+ new NextRequest("http://localhost:3000/api/events", {
+ method: "POST",
+ headers: {
+ authorization: `Bearer ${clientToken}`,
+ "content-type": "application/json",
+ },
+ body: "{bad-json",
+ }),
+ );
+ assert.equal(malformedRes.status, 400);
+
+ // Delete client -> subsequent Bearer ingestion fails closed with 403 CLIENT_REVOKED
+ const delRes = await clientDelete(
+ new NextRequest("http://localhost:3000/api/clients/phase01_probe_client", {
+ method: "DELETE",
+ headers: { cookie: `datastack_admin_session=${adminCookie}` },
+ }),
+ { params: Promise.resolve({ id: "phase01_probe_client" }) },
+ );
+ assert.equal(delRes.status, 200);
+
+ const rejectedRes = await eventsPost(
+ new NextRequest("http://localhost:3000/api/events", {
+ method: "POST",
+ headers: {
+ authorization: `Bearer ${clientToken}`,
+ "content-type": "application/json",
+ },
+ body: JSON.stringify({ eventId: "evt_after_delete", data: { ok: 1 } }),
+ }),
+ );
+ assert.equal(rejectedRes.status, 403);
+
+ // Unregistered signed Bearer token also rejected with 403 CLIENT_REVOKED
+ const ghostToken = AuthService.generateClientToken({
+ clientId: "unregistered_ghost_client",
+ });
+ const ghostRes = await eventsPost(
+ new NextRequest("http://localhost:3000/api/events", {
+ method: "POST",
+ headers: {
+ authorization: `Bearer ${ghostToken}`,
+ "content-type": "application/json",
+ },
+ body: JSON.stringify({ eventId: "evt_ghost", data: { ok: 1 } }),
+ }),
+ );
+ assert.equal(ghostRes.status, 403);
+ });
+
+ it("BUG-26: validates Origin or Referer against Host / X-Forwarded-Host", () => {
+ const reqOriginOk = new NextRequest("http://localhost:3000/api/clients", {
+ method: "POST",
+ headers: {
+ origin: "https://console.example.com",
+ "x-forwarded-host": "console.example.com",
+ },
+ });
+ assert.equal(isSameOriginRequest(reqOriginOk), true);
+
+ const reqRefererOk = new NextRequest("http://localhost:3000/api/clients", {
+ method: "POST",
+ headers: {
+ referer: "https://console.example.com/vcon/clients",
+ host: "console.example.com",
+ },
+ });
+ assert.equal(isSameOriginRequest(reqRefererOk), true);
+
+ const reqCrossOrigin = new NextRequest("http://localhost:3000/api/clients", {
+ method: "POST",
+ headers: {
+ origin: "https://evil.example.com",
+ host: "console.example.com",
+ },
+ });
+ assert.equal(isSameOriginRequest(reqCrossOrigin), false);
+
+ const reqMissingBoth = new NextRequest("http://localhost:3000/api/clients", {
+ method: "POST",
+ headers: {
+ host: "console.example.com",
+ },
+ });
+ assert.equal(isSameOriginRequest(reqMissingBoth), false);
+ });
+});
diff --git a/tests/phase02-sync.test.ts b/tests/phase02-sync.test.ts
new file mode 100644
index 0000000..eb13154
--- /dev/null
+++ b/tests/phase02-sync.test.ts
@@ -0,0 +1,375 @@
+import assert from "node:assert/strict";
+import { createRequire } from "node:module";
+import type { AddressInfo } from "node:net";
+import test, { after, before } from "node:test";
+import { NextRequest } from "next/server";
+import { io as ioClient, type Socket } from "socket.io-client";
+import { GET as getSettingsRoute, POST as postSettingsRoute } from "../app/api/settings/route";
+import { SettingsManager } from "../core/adminSettings/settingsManager";
+import {
+ createAdminSessionToken,
+ getAdminSessionSecret,
+} from "../core/auth/adminSession";
+import {
+ createInitialAdmin,
+ resetAdminStateForTests,
+} from "../core/auth/adminUsers";
+import { ApiEngine } from "../core/apiEngine/apiService";
+import { SDKManager } from "../core/sdk/sdkManager";
+import {
+ SecurityEngine,
+ getInternalIngestionHeaders,
+} from "../core/Security/security";
+
+const require = createRequire(import.meta.url);
+
+interface IngestionServerModule {
+ server: import("node:http").Server;
+ io: import("socket.io").Server;
+ bufferQueue: Array<{ clientId: string; eventId: string; timestamp: string; payload: string }>;
+ runtimeConfig: {
+ jwtSecret: string;
+ maxEventsPerSec: number;
+ maxPayloadSizeBytes: number;
+ batchFlushThreshold: number;
+ batchFlushIntervalMs: number;
+ fileRotationMb: number;
+ enableIdempotency: boolean;
+ autoPruneDays: number;
+ };
+ stopTimers: () => void;
+}
+
+const g = globalThis as typeof globalThis & {
+ __dataStackBufferState?: {
+ flushTimer: ReturnType | null;
+ };
+ __dataStackAdminSettings?: unknown;
+ __dataStackClientsMap?: Map;
+ __dataStackDeletedClients?: Set;
+ __dataStackSyncedSocketEvents?: Map;
+};
+
+let serverMod: IngestionServerModule;
+let ingestionUrl = "";
+const prevIngestionUrl = process.env.INGESTION_INTERNAL_URL;
+const prevDisableListen = process.env.DATASTACK_DISABLE_LISTEN;
+const prevDbUrl = process.env.DATABASE_URL;
+const prevInternalSecret = process.env.DATASTACK_INTERNAL_SECRET;
+const openSockets: Socket[] = [];
+
+function connectSocket(token: string): Promise {
+ return new Promise((resolve, reject) => {
+ const client = ioClient(ingestionUrl, {
+ auth: { token },
+ transports: ["websocket"],
+ reconnection: false,
+ timeout: 3000,
+ });
+ openSockets.push(client);
+ const onConnect = () => {
+ cleanup();
+ resolve(client);
+ };
+ const onError = (err: Error) => {
+ cleanup();
+ client.disconnect();
+ reject(err);
+ };
+ function cleanup() {
+ client.off("connect", onConnect);
+ client.off("connect_error", onError);
+ }
+ client.on("connect", onConnect);
+ client.on("connect_error", onError);
+ });
+}
+
+function emitWithAck(
+ socket: Socket,
+ event: string,
+ payload: unknown,
+): Promise {
+ return new Promise((resolve, reject) => {
+ socket.timeout(3000).emit(event, payload, (err: unknown, res: T) => {
+ if (err) reject(err);
+ else resolve(res);
+ });
+ });
+}
+
+async function createAdminCookieHeader(): Promise {
+ resetAdminStateForTests();
+ await createInitialAdmin("admin@datastack.dev", "InitialAdminPass123!");
+ const secret = getAdminSessionSecret();
+ assert.ok(secret, "Expected admin session secret in test environment");
+ const token = await createAdminSessionToken(
+ "admin@datastack.dev",
+ secret,
+ Date.now() + 3600_000,
+ 1,
+ );
+ return `datastack_admin_session=${token}`;
+}
+
+before(async () => {
+ process.env.DATASTACK_DISABLE_LISTEN = "1";
+ delete process.env.DATABASE_URL;
+ delete process.env.DATASTACK_INTERNAL_SECRET;
+ delete g.__dataStackAdminSettings;
+ g.__dataStackClientsMap?.clear();
+ g.__dataStackDeletedClients?.clear();
+ g.__dataStackSyncedSocketEvents?.clear();
+ resetAdminStateForTests();
+
+ serverMod = require("../server/index.js") as IngestionServerModule;
+ await new Promise((resolve) => {
+ serverMod.server.listen(0, "127.0.0.1", () => resolve());
+ });
+ const addr = serverMod.server.address() as AddressInfo;
+ ingestionUrl = `http://127.0.0.1:${addr.port}`;
+ process.env.INGESTION_INTERNAL_URL = ingestionUrl;
+});
+
+after(async () => {
+ for (const sock of openSockets) {
+ try {
+ sock.disconnect();
+ } catch {
+ // ignore
+ }
+ }
+ if (serverMod) {
+ serverMod.stopTimers();
+ await new Promise((resolve) => {
+ serverMod.io.close(() => {
+ serverMod.server.close(() => resolve());
+ });
+ });
+ }
+ if (g.__dataStackBufferState?.flushTimer) {
+ clearInterval(g.__dataStackBufferState.flushTimer);
+ g.__dataStackBufferState.flushTimer = null;
+ }
+
+ if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL;
+ else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl;
+
+ if (prevDisableListen === undefined) delete process.env.DATASTACK_DISABLE_LISTEN;
+ else process.env.DATASTACK_DISABLE_LISTEN = prevDisableListen;
+
+ if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
+ else process.env.DATABASE_URL = prevDbUrl;
+
+ if (prevInternalSecret === undefined) delete process.env.DATASTACK_INTERNAL_SECRET;
+ else process.env.DATASTACK_INTERNAL_SECRET = prevInternalSecret;
+});
+
+test("BUG-13: Port 4000 /internal/* rejects unauthenticated callers and accepts valid internal secret while /health remains public", async () => {
+ const healthRes = await fetch(`${ingestionUrl}/health`);
+ assert.equal(healthRes.status, 200);
+
+ for (const [method, path, body] of [
+ ["GET", "/internal/metrics", undefined],
+ ["POST", "/internal/flush", {}],
+ ["GET", "/internal/recent", undefined],
+ ["POST", "/internal/client-sync", { clientId: "x", action: "registered" }],
+ ["POST", "/internal/settings-sync", { maxEventsPerSec: 120 }],
+ ] as const) {
+ const noAuth = await fetch(`${ingestionUrl}${path}`, {
+ method,
+ headers: body ? { "Content-Type": "application/json" } : undefined,
+ body: body ? JSON.stringify(body) : undefined,
+ });
+ assert.equal(noAuth.status, 401, `${method} ${path} must reject missing secret`);
+
+ const wrongAuth = await fetch(`${ingestionUrl}${path}`, {
+ method,
+ headers: {
+ ...(body ? { "Content-Type": "application/json" } : {}),
+ "x-datastack-internal-secret": "wrong-secret-value",
+ },
+ body: body ? JSON.stringify(body) : undefined,
+ });
+ assert.equal(wrongAuth.status, 401, `${method} ${path} must reject wrong secret`);
+
+ const validAuth = await fetch(`${ingestionUrl}${path}`, {
+ method,
+ headers: {
+ ...(body ? { "Content-Type": "application/json" } : {}),
+ ...getInternalIngestionHeaders(),
+ },
+ body: body ? JSON.stringify(body) : undefined,
+ });
+ assert.equal(validAuth.status, 200, `${method} ${path} must accept valid internal secret`);
+ }
+});
+
+test("BUG-06: POST /api/settings validates allowlisted bounds, syncs Port 4000 immediately, preserves autoPruneDays=0, and disconnects sockets on JWT rotation", async () => {
+ const unauthGet = await getSettingsRoute(new NextRequest("http://localhost:3000/api/settings"));
+ assert.equal(unauthGet.status, 401);
+
+ const cookieHeader = await createAdminCookieHeader();
+
+ const badJsonReq = new NextRequest("http://localhost:3000/api/settings", {
+ method: "POST",
+ headers: { "Content-Type": "application/json", cookie: cookieHeader },
+ body: "{bad-json",
+ });
+ const badJsonRes = await postSettingsRoute(badJsonReq);
+ assert.equal(badJsonRes.status, 400);
+
+ const unsupportedReq = new NextRequest("http://localhost:3000/api/settings", {
+ method: "POST",
+ headers: { "Content-Type": "application/json", cookie: cookieHeader },
+ body: JSON.stringify({ serverPort: 9999 }),
+ });
+ assert.equal((await postSettingsRoute(unsupportedReq)).status, 400);
+
+ const weakSecretReq = new NextRequest("http://localhost:3000/api/settings", {
+ method: "POST",
+ headers: { "Content-Type": "application/json", cookie: cookieHeader },
+ body: JSON.stringify({ jwtSecret: "too-short" }),
+ });
+ assert.equal((await postSettingsRoute(weakSecretReq)).status, 400);
+
+ const validUpdateReq = new NextRequest("http://localhost:3000/api/settings", {
+ method: "POST",
+ headers: { "Content-Type": "application/json", cookie: cookieHeader },
+ body: JSON.stringify({
+ maxEventsPerSec: 42,
+ autoPruneDays: 0,
+ enableIdempotency: true,
+ }),
+ });
+ const validUpdateRes = await postSettingsRoute(validUpdateReq);
+ assert.equal(validUpdateRes.status, 200);
+ const updatedJson = await validUpdateRes.json();
+ assert.equal(updatedJson.settings.maxEventsPerSec, 42);
+ assert.equal(updatedJson.settings.autoPruneDays, 0);
+ assert.equal(serverMod.runtimeConfig.maxEventsPerSec, 42);
+ assert.equal(serverMod.runtimeConfig.autoPruneDays, 0);
+
+ // Register client & open WebSocket before JWT rotation
+ const client = await SDKManager.registerClientAsync({
+ clientId: "phase02_jwt_rot",
+ name: "Phase02 Rotation",
+ rateLimit: 50,
+ });
+ const oldToken = client.token;
+ const socket = await connectSocket(oldToken);
+ assert.equal(socket.connected, true);
+
+ const disconnectedPromise = new Promise((resolve) => {
+ socket.once("disconnect", (reason) => resolve(reason));
+ });
+
+ const newSecret = "phase02-rotated-production-secret-key-minimum-32-chars";
+ const rotateReq = new NextRequest("http://localhost:3000/api/settings", {
+ method: "POST",
+ headers: { "Content-Type": "application/json", cookie: cookieHeader },
+ body: JSON.stringify({ jwtSecret: newSecret }),
+ });
+ const rotateRes = await postSettingsRoute(rotateReq);
+ assert.equal(rotateRes.status, 200);
+ assert.equal(serverMod.runtimeConfig.jwtSecret, newSecret);
+
+ const disconnectReason = await Promise.race([
+ disconnectedPromise,
+ new Promise((_, reject) =>
+ setTimeout(() => reject(new Error("Expected socket disconnect on JWT rotation")), 2000),
+ ),
+ ]);
+ assert.ok(disconnectReason.length > 0);
+
+ // Old JWT token must fail handshake after rotation
+ await assert.rejects(() => connectSocket(oldToken), /AUTH_INVALID|jwt|signature/i);
+
+ // New resigned client token must succeed cleanly
+ const refreshedClient = SDKManager.getClient("phase02_jwt_rot");
+ assert.ok(refreshedClient && refreshedClient.token !== oldToken);
+ const reconnected = await connectSocket(refreshedClient.token);
+ assert.equal(reconnected.connected, true);
+ reconnected.disconnect();
+});
+
+test("BUG-07: In-memory client registry, socket event counting, token rotation & revocation enforce cleanly on Port 4000", async () => {
+ const client = await SDKManager.registerClientAsync({
+ clientId: "phase02_mem_client",
+ name: "Phase02 Memory Sync",
+ rateLimit: 25,
+ });
+
+ const socket = await connectSocket(client.token);
+ const ack = await emitWithAck<{ success: boolean; eventId: string }>(
+ socket,
+ "sync_data",
+ { eventId: "evt-mem-1", data: { ok: true } },
+ );
+ assert.equal(ack.success, true);
+
+ const listFirst = await SDKManager.listClientsAsync();
+ const syncedFirst = listFirst.find((c) => c.clientId === "phase02_mem_client");
+ assert.ok(syncedFirst);
+ assert.equal(syncedFirst.totalEvents, 1);
+ assert.ok(syncedFirst.lastActive);
+
+ // Polling again must NOT double-count socket activity
+ const listSecond = await SDKManager.listClientsAsync();
+ const syncedSecond = listSecond.find((c) => c.clientId === "phase02_mem_client");
+ assert.equal(syncedSecond?.totalEvents, 1);
+
+ // Advance across JWT iat 1-second boundary before regenerating client token
+ await new Promise((r) => setTimeout(r, 1100));
+
+ // Regenerating client token must kick active socket and reject old token
+ const rotatedClient = await SDKManager.regenerateTokenAsync("phase02_mem_client");
+ assert.ok(rotatedClient && rotatedClient.token !== client.token);
+ await new Promise((r) => setTimeout(r, 50));
+ assert.equal(socket.connected, false);
+ await assert.rejects(() => connectSocket(client.token), /TOKEN_ROTATED/);
+
+ // New token works; revoking kicks and rejects
+ const socket2 = await connectSocket(rotatedClient.token);
+ assert.equal(socket2.connected, true);
+ await SDKManager.updateClientAsync("phase02_mem_client", { status: "revoked" });
+ await new Promise((r) => setTimeout(r, 50));
+ assert.equal(socket2.connected, false);
+ await assert.rejects(() => connectSocket(rotatedClient.token), /CLIENT_REVOKED/);
+});
+
+test("BUG-08: Rate-limit error reports effectiveRateLimit and ApiEngine aggregates Port 3000 + Port 4000 security metrics", async () => {
+ await SettingsManager.updateSettingsAsync({ maxEventsPerSec: 3, enableIdempotency: true });
+
+ const rlClientId = "phase02_rl_msg";
+ let lastError = "";
+ for (let i = 0; i < 5; i += 1) {
+ const res = SecurityEngine.validateEvent(rlClientId, `rl-evt-${i}`, { i }, 100);
+ if (!res.allowed && res.code === "RATE_LIMIT_EXCEEDED") {
+ lastError = res.reason || "";
+ break;
+ }
+ }
+ assert.match(lastError, /Rate limit of 3 events\/sec exceeded/);
+
+ const client = await SDKManager.registerClientAsync({
+ clientId: "phase02_stats_client",
+ name: "Phase02 Stats Client",
+ rateLimit: 50,
+ });
+ await new Promise((r) => setTimeout(r, 80));
+
+ const socket = await connectSocket(client.token);
+ await emitWithAck(socket, "sync_data", { eventId: "evt-stats-ws-1", data: { a: 1 } });
+ socket.disconnect();
+
+ const metrics = await ApiEngine.getDashboardMetrics();
+ assert.ok(
+ metrics.securityStats.activeIdempotencyKeys >= 2,
+ `Expected aggregated idempotency keys >= 2 (HTTP + WS), got ${metrics.securityStats.activeIdempotencyKeys}`,
+ );
+ assert.ok(
+ metrics.securityStats.rateLimitedClientsCount >= 2,
+ `Expected aggregated rateLimitedClientsCount >= 2, got ${metrics.securityStats.rateLimitedClientsCount}`,
+ );
+});
diff --git a/tests/phase03-storage-buffer.test.ts b/tests/phase03-storage-buffer.test.ts
new file mode 100644
index 0000000..50bebd9
--- /dev/null
+++ b/tests/phase03-storage-buffer.test.ts
@@ -0,0 +1,511 @@
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import { createServer, type Server } from "node:http";
+import type { AddressInfo } from "node:net";
+import { createRequire } from "node:module";
+import os from "node:os";
+import path from "node:path";
+import test, { after, before, beforeEach } from "node:test";
+import { NextRequest } from "next/server";
+import { GET as getDataRoute } from "../app/api/data/route";
+import { GET as getEventsRoute } from "../app/api/events/route";
+import { GET as getExportRoute } from "../app/api/export/route";
+import { POST as postStorageRoute } from "../app/api/storage/route";
+import { ApiEngine } from "../core/apiEngine/apiService";
+import {
+ DataBufferEngine,
+ REDIS_BUFFER_KEY,
+ REDIS_FLUSH_LOCK_KEY,
+ type RedisBufferClient,
+} from "../core/data/buffer";
+import { SDKManager } from "../core/sdk/sdkManager";
+import { CSVStorageManager, type DataEvent } from "../core/Storage/csvStorage";
+
+const require = createRequire(import.meta.url);
+const { CsvWriterWorker } = require("../server/workers/csvWriter") as {
+ CsvWriterWorker: new (baseDir?: string, maxFileSize?: number) => {
+ baseDir: string;
+ writeBatch: (events: DataEvent[]) => Promise<{ written: number; files: string[] }>;
+ pruneExpiredPartitions: (days: number) => number;
+ };
+};
+
+interface IngestionServerModule {
+ server: Server;
+ io: import("socket.io").Server;
+ writer: { baseDir: string };
+ bufferQueue: DataEvent[];
+ runtimeConfig: {
+ batchFlushThreshold: number;
+ autoPruneDays: number;
+ };
+ applyRuntimeSettingsObject: (obj: Record) => { jwtRotated: boolean };
+ flushBuffer: () => Promise<{ written: number; files: string[] }>;
+ setRedisClientForTests: (client: RedisBufferClient | null | undefined) => void;
+ stopTimers: () => void;
+}
+
+class FakeRedisClient implements RedisBufferClient {
+ public status = "ready";
+ public lists = new Map();
+ public strings = new Map();
+
+ public async rpush(key: string, ...values: string[]): Promise {
+ const list = this.lists.get(key) || [];
+ list.push(...values);
+ this.lists.set(key, list);
+ return list.length;
+ }
+
+ public async lpush(key: string, ...values: string[]): Promise {
+ const list = this.lists.get(key) || [];
+ for (const v of values) {
+ list.unshift(v);
+ }
+ this.lists.set(key, list);
+ return list.length;
+ }
+
+ public async ltrim(key: string, start: number, stop: number): Promise<"OK"> {
+ const list = this.lists.get(key) || [];
+ const len = list.length;
+ const normStart = start < 0 ? Math.max(0, len + start) : start;
+ const normStop = stop < 0 ? len + stop : stop;
+ if (normStart > normStop || normStart >= len) {
+ this.lists.set(key, []);
+ } else {
+ this.lists.set(key, list.slice(normStart, normStop + 1));
+ }
+ return "OK";
+ }
+
+ public async lrange(key: string, start: number, stop: number): Promise {
+ const list = this.lists.get(key) || [];
+ const len = list.length;
+ if (len === 0) return [];
+ const normStart = start < 0 ? Math.max(0, len + start) : start;
+ const normStop = stop < 0 ? len + stop : stop;
+ if (normStart > normStop || normStart >= len) return [];
+ return list.slice(normStart, normStop + 1);
+ }
+
+ public async llen(key: string): Promise {
+ return (this.lists.get(key) || []).length;
+ }
+
+ public async set(
+ key: string,
+ value: string,
+ _pxMode: "PX",
+ ttlMs: number,
+ _nxMode: "NX",
+ ): Promise {
+ const existing = this.strings.get(key);
+ if (existing && existing.expiresAt > Date.now()) {
+ return null;
+ }
+ this.strings.set(key, { value, expiresAt: Date.now() + ttlMs });
+ return "OK";
+ }
+
+ public async eval(_script: string, _numKeys: number, key: string, token: string): Promise {
+ const existing = this.strings.get(key);
+ if (existing && existing.value === token) {
+ this.strings.delete(key);
+ return 1;
+ }
+ return 0;
+ }
+}
+
+const g = globalThis as typeof globalThis & {
+ __dataStackBufferState?: {
+ buffer: DataEvent[];
+ recentEvents: DataEvent[];
+ flushTimer: ReturnType | null;
+ };
+ __dataStackAdminSettings?: unknown;
+ __dataStackClientsMap?: Map;
+ __dataStackDeletedClients?: Set;
+};
+
+let tmpStorageDir = "";
+const origBaseStorageDir = CSVStorageManager.baseStorageDir;
+const prevDisableListen = process.env.DATASTACK_DISABLE_LISTEN;
+const prevIngestionUrl = process.env.INGESTION_INTERNAL_URL;
+const prevDbUrl = process.env.DATABASE_URL;
+const prevRedisUrl = process.env.REDIS_URL;
+
+before(async () => {
+ process.env.DATASTACK_DISABLE_LISTEN = "1";
+ delete process.env.DATABASE_URL;
+ delete process.env.REDIS_URL;
+ delete process.env.INGESTION_INTERNAL_URL;
+
+ tmpStorageDir = fs.mkdtempSync(path.join(os.tmpdir(), "datastack-phase03-"));
+ CSVStorageManager.baseStorageDir = tmpStorageDir;
+ CSVStorageManager.clearRowCountCache();
+});
+
+beforeEach(async () => {
+ DataBufferEngine.setRedisClientForTests(undefined);
+ if (g.__dataStackBufferState) {
+ g.__dataStackBufferState.buffer = [];
+ g.__dataStackBufferState.recentEvents = [];
+ }
+ CSVStorageManager.clearRowCountCache();
+ fs.rmSync(tmpStorageDir, { recursive: true, force: true });
+ fs.mkdirSync(tmpStorageDir, { recursive: true });
+ delete g.__dataStackAdminSettings;
+ g.__dataStackClientsMap?.clear();
+ g.__dataStackDeletedClients?.clear();
+});
+
+after(async () => {
+ await DataBufferEngine.shutdown();
+ CSVStorageManager.baseStorageDir = origBaseStorageDir;
+ CSVStorageManager.clearRowCountCache();
+ if (tmpStorageDir) {
+ fs.rmSync(tmpStorageDir, { recursive: true, force: true });
+ }
+
+ if (prevDisableListen === undefined) delete process.env.DATASTACK_DISABLE_LISTEN;
+ else process.env.DATASTACK_DISABLE_LISTEN = prevDisableListen;
+
+ if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL;
+ else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl;
+
+ if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
+ else process.env.DATABASE_URL = prevDbUrl;
+
+ if (prevRedisUrl === undefined) delete process.env.REDIS_URL;
+ else process.env.REDIS_URL = prevRedisUrl;
+});
+
+test("BUG-11: Shared Redis buffer preserves events on CSV error, honors distributed flush lock, avoids double-counting, and falls back to RAM cleanly", async () => {
+ const fakeRedis = new FakeRedisClient();
+ DataBufferEngine.setRedisClientForTests(fakeRedis);
+
+ const sampleEvents: DataEvent[] = [
+ {
+ clientId: "phase03_redis",
+ eventId: "r-evt-1",
+ timestamp: "2026-11-17T10:00:00.000Z",
+ payload: JSON.stringify({ seq: 1 }),
+ },
+ {
+ clientId: "phase03_redis",
+ eventId: "r-evt-2",
+ timestamp: "2026-11-17T10:00:01.000Z",
+ payload: JSON.stringify({ seq: 2 }),
+ },
+ ];
+
+ await DataBufferEngine.pushBatchAsync(sampleEvents);
+
+ assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 2);
+ const recentFromRedis = await DataBufferEngine.getRecentEventsAsync(10, "phase03_redis");
+ assert.equal(recentFromRedis.length, 2);
+ assert.equal(recentFromRedis[0].eventId, "r-evt-2");
+
+ // Lock contention must skip concurrent Redis flush without corrupting queue
+ await fakeRedis.set(REDIS_FLUSH_LOCK_KEY, "other-worker-token", "PX", 15000, "NX");
+ const skipped = await DataBufferEngine.flush();
+ assert.equal(skipped.count, 0);
+ assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 2);
+ await fakeRedis.eval("", 1, REDIS_FLUSH_LOCK_KEY, "other-worker-token");
+
+ // Crash/error safety: if appendBatch throws before finishing, Redis LTRIM must NOT discard events
+ const originalAppend = CSVStorageManager.appendBatch;
+ CSVStorageManager.appendBatch = async () => {
+ throw new Error("Simulated disk write failure");
+ };
+ await assert.rejects(() => DataBufferEngine.flush(), /Simulated disk write failure/);
+ CSVStorageManager.appendBatch = originalAppend;
+ assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 2);
+
+ // Subsequent healthy flush drains Redis list, releases distributed lock, and writes CSV
+ const flushed = await DataBufferEngine.flush();
+ assert.equal(flushed.count, 2);
+ assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 0);
+ assert.equal(fakeRedis.strings.has(REDIS_FLUSH_LOCK_KEY), false);
+
+ // Verify Port 4000 /internal/recent & /internal/metrics read shared Redis without double-counting
+ const serverMod = require("../server/index.js") as IngestionServerModule;
+ const origWriterDir = serverMod.writer.baseDir;
+ serverMod.writer.baseDir = tmpStorageDir;
+ serverMod.setRedisClientForTests(fakeRedis);
+ await new Promise((resolve) => serverMod.server.listen(0, "127.0.0.1", () => resolve()));
+ const addr = serverMod.server.address() as AddressInfo;
+ process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${addr.port}`;
+
+ try {
+ await DataBufferEngine.pushAsync({
+ clientId: "phase03_redis",
+ eventId: "r-evt-3",
+ timestamp: "2026-11-17T10:00:02.000Z",
+ payload: JSON.stringify({ seq: 3 }),
+ });
+ const stats = await DataBufferEngine.getBufferStatsAsync();
+ assert.equal(stats.redisEnabled, true);
+ assert.equal(stats.redisBufferSize, 1);
+
+ const metrics = await ApiEngine.getDashboardMetrics();
+ // Shared Redis count (1) must NOT be counted twice as 2 across Port 3000 + Port 4000
+ assert.equal(metrics.bufferStats.currentBufferSize, 1);
+
+ // Clear recent list & buffer via Port 4000 flush
+ const flushRes = await serverMod.flushBuffer();
+ assert.equal(flushRes.written, 1);
+ assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 0);
+ } finally {
+ serverMod.writer.baseDir = origWriterDir;
+ serverMod.setRedisClientForTests(undefined);
+ serverMod.stopTimers();
+ delete process.env.INGESTION_INTERNAL_URL;
+ await new Promise((resolve) => serverMod.io.close(() => serverMod.server.close(() => resolve())));
+ }
+
+ // Fallback to local RAM buffer when Redis is disabled (null)
+ DataBufferEngine.setRedisClientForTests(null);
+ await DataBufferEngine.pushAsync({
+ clientId: "phase03_ram",
+ eventId: "ram-evt-1",
+ timestamp: "2026-11-17T10:01:00.000Z",
+ payload: JSON.stringify({ mode: "ram" }),
+ });
+ const ramStats = await DataBufferEngine.getBufferStatsAsync();
+ assert.equal(ramStats.redisEnabled, false);
+ assert.equal(ramStats.localBufferSize, 1);
+ const ramFlushed = await DataBufferEngine.flush();
+ assert.equal(ramFlushed.count, 1);
+});
+
+test("BUG-09: Cooperative CSV partition locking serializes concurrent Port 3000 + Port 4000 writers, recovers stale locks, and surfaces Port 4000 flush errors", async () => {
+ const worker = new CsvWriterWorker(tmpStorageDir);
+ const partitionDate = "2026-11-17";
+ const clientId = "phase03_concurrent";
+
+ const batchA: DataEvent[] = Array.from({ length: 15 }, (_, idx) => ({
+ clientId,
+ eventId: `a-${idx}`,
+ timestamp: `${partitionDate}T12:00:00.000Z`,
+ payload: JSON.stringify({ writer: "A", idx }),
+ }));
+ const batchB: DataEvent[] = Array.from({ length: 15 }, (_, idx) => ({
+ clientId,
+ eventId: `b-${idx}`,
+ timestamp: `${partitionDate}T12:00:01.000Z`,
+ payload: JSON.stringify({ writer: "B", idx }),
+ }));
+
+ await Promise.all([
+ CSVStorageManager.appendBatch(batchA),
+ worker.writeBatch(batchB),
+ ]);
+
+ const csvPath = path.join(tmpStorageDir, clientId, `${partitionDate}.csv`);
+ const lockPath = path.join(tmpStorageDir, clientId, `.${partitionDate}.csv.lock`);
+ assert.equal(fs.existsSync(lockPath), false, "Lock file must be cleaned up after writes");
+
+ const lines = fs.readFileSync(csvPath, "utf8").trim().split("\n");
+ assert.equal(lines[0], '"clientId","eventId","timestamp","payload"');
+ assert.equal(lines.length, 31, "Must contain exactly 1 header + 30 complete rows without interleaving");
+
+ // Stale lock recovery (>30s old mtime)
+ fs.writeFileSync(lockPath, "999999:0", "utf8");
+ const pastTime = new Date(Date.now() - 45_000);
+ fs.utimesSync(lockPath, pastTime, pastTime);
+ await CSVStorageManager.appendBatch([
+ {
+ clientId,
+ eventId: "after-stale-lock",
+ timestamp: `${partitionDate}T12:00:02.000Z`,
+ payload: JSON.stringify({ staleRecovered: true }),
+ },
+ ]);
+ assert.equal(fs.existsSync(lockPath), false);
+ const files = await CSVStorageManager.listAllFilesAsync();
+ assert.equal(files[0].linesCount, 31);
+
+ // Explicit flush error & warning propagation in POST /api/storage
+ const failingBridge = createServer((_req, res) => {
+ res.statusCode = 500;
+ res.setHeader("Content-Type", "application/json");
+ res.end(JSON.stringify({ success: false, error: "Disk permission denied on Port 4000" }));
+ });
+ await new Promise((resolve) => failingBridge.listen(0, "127.0.0.1", () => resolve()));
+ const bridgeAddr = failingBridge.address() as AddressInfo;
+ process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${bridgeAddr.port}`;
+
+ try {
+ const errRes = await postStorageRoute(new NextRequest("http://localhost:3000/api/storage", { method: "POST" }));
+ assert.equal(errRes.status, 502);
+ const errJson = await errRes.json();
+ assert.equal(errJson.success, false);
+ assert.equal(errJson.code, "INGESTION_FLUSH_FAILED");
+ assert.equal(errJson.ingestionFlushed, false);
+ } finally {
+ delete process.env.INGESTION_INTERNAL_URL;
+ await new Promise((resolve) => failingBridge.close(() => resolve()));
+ }
+
+ // When Port 4000 is down and INGESTION_INTERNAL_URL is unset, standalone dev surfaces explicit warning + ingestionFlushed: false
+ const standaloneRes = await postStorageRoute(new NextRequest("http://localhost:3000/api/storage", { method: "POST" }));
+ assert.equal(standaloneRes.status, 200);
+ const standaloneJson = await standaloneRes.json();
+ assert.equal(standaloneJson.ingestionFlushed, false);
+ assert.match(String(standaloneJson.warning || ""), /unreachable/i);
+});
+
+test("BUG-10: Cached streaming row counter counts multiline RFC 4180 payloads accurately and avoids repeated disk reads", async () => {
+ await CSVStorageManager.appendBatch([
+ {
+ clientId: "phase03_perf",
+ eventId: "multiline-1",
+ timestamp: "2026-11-17T08:00:00.000Z",
+ payload: JSON.stringify({ note: "line1\nline2\n\"quoted\"\nline4" }),
+ },
+ {
+ clientId: "phase03_perf",
+ eventId: "multiline-2",
+ timestamp: "2026-11-17T08:00:01.000Z",
+ payload: JSON.stringify({ note: "single" }),
+ },
+ ]);
+
+ const firstStats = await CSVStorageManager.getStorageStatsAsync();
+ assert.equal(firstStats.totalRows, 2);
+
+ // Spy on fs.createReadStream to verify repeated dashboard polls hit mtime+size cache with 0 file scans
+ const origCreateReadStream = fs.createReadStream;
+ let streamReads = 0;
+ (fs as unknown as { createReadStream: typeof fs.createReadStream }).createReadStream = ((
+ ...args: Parameters
+ ) => {
+ streamReads += 1;
+ return origCreateReadStream(...args);
+ }) as typeof fs.createReadStream;
+
+ try {
+ const secondStats = await CSVStorageManager.getStorageStatsAsync();
+ assert.equal(secondStats.totalRows, 2);
+ assert.equal(streamReads, 0, "Unchanged CSV files must use cached row count");
+ } finally {
+ (fs as unknown as { createReadStream: typeof fs.createReadStream }).createReadStream =
+ origCreateReadStream;
+ }
+
+ // Server 3s settings sync must not repeatedly prune when autoPruneDays is unchanged
+ const serverMod = require("../server/index.js") as IngestionServerModule;
+ const oldPartitionDir = path.join(tmpStorageDir, "phase03_prune_check");
+ fs.mkdirSync(oldPartitionDir, { recursive: true });
+ fs.writeFileSync(path.join(oldPartitionDir, "2020-01-01.csv"), "h\nr\n", "utf8");
+ const prunedCount = CSVStorageManager.pruneExpiredPartitions(30);
+ assert.equal(prunedCount, 1);
+
+ serverMod.applyRuntimeSettingsObject({ autoPruneDays: 30 });
+ fs.writeFileSync(path.join(oldPartitionDir, "2020-01-02.csv"), "h\nr\n", "utf8");
+ // Same autoPruneDays=30 applied again (simulating 3s DB poll) must NOT prune on every tick
+ serverMod.applyRuntimeSettingsObject({ autoPruneDays: 30 });
+ assert.equal(fs.existsSync(path.join(oldPartitionDir, "2020-01-02.csv")), true);
+ serverMod.stopTimers();
+});
+
+test("BUG-12: Prisma schema models and mappings align cleanly with runtime SQL DDL and Docker Compose Redis wiring", () => {
+ const schemaPath = path.resolve(process.cwd(), "prisma/schema.prisma");
+ const schemaText = fs.readFileSync(schemaPath, "utf8");
+
+ assert.match(schemaText, /model ClientSDK[\s\S]*clientId\s+String\s+@id\s+@map\("client_id"\)/);
+ assert.match(schemaText, /token\s+String/);
+ assert.match(schemaText, /model DeletedClient[\s\S]*@@map\("deleted_clients"\)/);
+ assert.match(schemaText, /model AdminUser[\s\S]*tokenVersion\s+Int\s+@default\(1\)\s+@map\("token_version"\)/);
+ assert.match(schemaText, /model AdminLoginAttempt[\s\S]*@@map\("admin_login_attempts"\)/);
+ assert.match(schemaText, /model AdminSettings[\s\S]*maxEventsPerSec\s+Int\s+@default\(100\)\s+@map\("max_events_per_sec"\)/);
+ assert.equal(/model ApiKey\b/.test(schemaText), false);
+ assert.equal(/model AuditLog\b/.test(schemaText), false);
+
+ const composeText = fs.readFileSync(path.resolve(process.cwd(), "docker/docker-compose.yml"), "utf8");
+ assert.equal((composeText.match(/REDIS_URL:\s*"redis:\/\/redis:6379"/g) || []).length, 2);
+ assert.match(composeText, /redis:\s*\n\s*condition:\s*service_healthy/);
+});
+
+test("BUG-14: GET /api/data searches eventId & clientId + UTC dates, GET /api/events caps CSV tail, and GET /api/export rejects ambiguous filename & honors ?flush=false", async () => {
+ DataBufferEngine.setRedisClientForTests(null);
+
+ await SDKManager.registerClientAsync({ clientId: "alpha_cli", name: "Alpha", rateLimit: 100 });
+ await SDKManager.registerClientAsync({ clientId: "beta_cli", name: "Beta", rateLimit: 100 });
+
+ const tsMillis = String(Date.UTC(2026, 10, 17, 15, 0, 0)); // 2026-11-17
+ await DataBufferEngine.pushBatchAsync([
+ {
+ clientId: "alpha_cli",
+ eventId: "unique-findme-id-99",
+ timestamp: tsMillis,
+ payload: JSON.stringify({ temp: 22 }),
+ },
+ {
+ clientId: "beta_cli",
+ eventId: "beta-id-1",
+ timestamp: "2026-11-17T15:01:00.000Z",
+ payload: JSON.stringify({ temp: 33 }),
+ },
+ ]);
+
+ // 1. Search by eventId with from/to UTC normalization while still buffered
+ const dataByEvent = await getDataRoute(
+ new NextRequest("http://localhost:3000/api/data?search=unique-findme-id-99&from=2026-11-17&to=2026-11-17"),
+ );
+ assert.equal(dataByEvent.status, 200);
+ const dataEventJson = await dataByEvent.json();
+ assert.equal(dataEventJson.total, 1);
+ assert.equal(dataEventJson.events[0].eventId, "unique-findme-id-99");
+
+ // 2. Search by clientId substring
+ const dataByClient = await getDataRoute(
+ new NextRequest("http://localhost:3000/api/data?search=beta_cli"),
+ );
+ const dataClientJson = await dataByClient.json();
+ assert.equal(dataClientJson.total, 1);
+ assert.equal(dataClientJson.events[0].clientId, "beta_cli");
+
+ // 3. Export with ?flush=false must NOT drain active buffer
+ const noFlushRes = await getExportRoute(
+ new NextRequest("http://localhost:3000/api/export?flush=false&format=json"),
+ );
+ assert.equal(noFlushRes.status, 200);
+ assert.equal((await noFlushRes.text()).trim(), "[]");
+ assert.equal((await DataBufferEngine.getBufferStatsAsync()).localBufferSize, 2);
+
+ // 4. Default flush writes both 2026-11-17.csv files -> clientId=all + filename must 400 AMBIGUOUS_FILENAME
+ const flushExportRes = await getExportRoute(
+ new NextRequest("http://localhost:3000/api/export?format=json"),
+ );
+ assert.equal(flushExportRes.status, 200);
+ const flushedPayload = await flushExportRes.json();
+ assert.equal(flushedPayload.length, 2);
+
+ const ambiguousRes = await getExportRoute(
+ new NextRequest("http://localhost:3000/api/export?clientId=all&filename=2026-11-17.csv"),
+ );
+ assert.equal(ambiguousRes.status, 400);
+ const ambiguousJson = await ambiguousRes.json();
+ assert.equal(ambiguousJson.code, "AMBIGUOUS_FILENAME");
+
+ // Disambiguated download with explicit clientId succeeds cleanly
+ const specificRes = await getExportRoute(
+ new NextRequest("http://localhost:3000/api/export?clientId=alpha_cli&filename=2026-11-17.csv"),
+ );
+ assert.equal(specificRes.status, 200);
+ const csvBody = await specificRes.text();
+ assert.match(csvBody, /unique-findme-id-99/);
+
+ // 5. GET /api/events with limit=1 returns latest single stored event cleanly
+ const eventsRes = await getEventsRoute(
+ new NextRequest("http://localhost:3000/api/events?clientId=alpha_cli&limit=1"),
+ );
+ assert.equal(eventsRes.status, 200);
+ const eventsJson = await eventsRes.json();
+ assert.equal(eventsJson.events.length, 1);
+ assert.equal(eventsJson.events[0].eventId, "unique-findme-id-99");
+});
diff --git a/tests/phase05_sdk_test.py b/tests/phase05_sdk_test.py
new file mode 100644
index 0000000..ada6b1e
--- /dev/null
+++ b/tests/phase05_sdk_test.py
@@ -0,0 +1,266 @@
+"""
+Phase-05 Verification Tests (BUG-18):
+- Single Source of Truth between canonical `datastack` (`sdk/python/datastack`)
+ and compatibility alias `datastack_sdk` (`sdk-python`).
+- `CLIENT_ID` config loading (`config.datastack.py`, `DATASTACK_CLIENT_ID`, kwarg),
+ JWT `clientId`/`sub` fallback extraction, and mismatch validation.
+- End-to-end propagation of `clientId` & `X-DataStack-Client-Id` into HTTP
+ fallback ingestion (`send`, `send_batch`, `send_file`), status snapshot, and
+ offline disk spool replay against a local HTTP test server.
+"""
+
+import base64
+import json
+import os
+import sys
+import tempfile
+import threading
+import unittest
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+from typing import Any, Dict, List
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+CANONICAL_PKG_ROOT = REPO_ROOT / "sdk" / "python" / "datastack"
+COMPAT_PKG_ROOT = REPO_ROOT / "sdk-python"
+
+for p in (str(CANONICAL_PKG_ROOT), str(COMPAT_PKG_ROOT)):
+ if p not in sys.path:
+ sys.path.insert(0, p)
+
+import datastack # noqa: E402
+import datastack_sdk # noqa: E402
+from datastack import ( # noqa: E402
+ DataStack,
+ DataStackClient,
+ DataStackConfig,
+ DataStackConfigError,
+ DataStackOfflineSpoolError,
+ DiskSpooler,
+ HeartbeatMonitor,
+ load_config,
+)
+
+
+def _make_jwt(payload: Dict[str, Any]) -> str:
+ header = (
+ base64.urlsafe_b64encode(json.dumps({"alg": "HS256", "typ": "JWT"}).encode())
+ .decode()
+ .rstrip("=")
+ )
+ body = (
+ base64.urlsafe_b64encode(json.dumps(payload).encode())
+ .decode()
+ .rstrip("=")
+ )
+ return f"{header}.{body}.testsig"
+
+
+class Phase05PythonSdkTests(unittest.TestCase):
+ def test_single_source_of_truth_identity(self) -> None:
+ self.assertIs(datastack_sdk.DataStack, datastack.DataStack)
+ self.assertIs(datastack_sdk.DataStackClient, datastack.DataStackClient)
+ self.assertIs(datastack_sdk.DataStackConfig, datastack.DataStackConfig)
+ self.assertIs(datastack_sdk.load_config, datastack.load_config)
+ self.assertIs(
+ datastack_sdk.DataStackConfigError, datastack.DataStackConfigError
+ )
+ self.assertIs(
+ datastack_sdk.DataStackOfflineSpoolError,
+ datastack.DataStackOfflineSpoolError,
+ )
+ self.assertIs(datastack_sdk.DiskSpooler, datastack.DiskSpooler)
+ self.assertIs(datastack_sdk.HeartbeatMonitor, datastack.HeartbeatMonitor)
+ self.assertEqual(datastack_sdk.__version__, datastack.__version__)
+
+ # Submodule shims also resolve to canonical classes
+ from datastack_sdk.client import DataStackClient as ShimClient
+ from datastack_sdk.config import load_config as shim_load_config
+ from datastack_sdk.exceptions import (
+ DataStackOfflineSpoolError as ShimSpoolErr,
+ )
+
+ self.assertIs(ShimClient, DataStackClient)
+ self.assertIs(shim_load_config, load_config)
+ self.assertIs(ShimSpoolErr, DataStackOfflineSpoolError)
+
+ def test_client_id_config_env_and_jwt_validation(self) -> None:
+ tok_alpha = _make_jwt({"clientId": "alpha_svc", "sub": "alpha_svc"})
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ cfg_file = Path(tmpdir) / "config.datastack.py"
+ cfg_file.write_text(
+ f'DATASTACK_CONFIG = {{\n'
+ f' "SERVER_URL": "http://127.0.0.1:49999",\n'
+ f' "API_URL": "http://127.0.0.1:39999",\n'
+ f' "CLIENT_TOKEN": "{tok_alpha}",\n'
+ f' "CLIENT_ID": "alpha_svc",\n'
+ f' "AUTO_CONNECT": False,\n'
+ f' "OFFLINE_SPOOL_DIR": "{Path(tmpdir) / "spool"}",\n'
+ f'}}\n',
+ encoding="utf-8",
+ )
+
+ cfg = load_config(config_path=str(cfg_file))
+ self.assertIsInstance(cfg, DataStackConfig)
+ self.assertEqual(cfg.client_id, "alpha_svc")
+
+ # Matching explicit client_id succeeds
+ client = DataStack(config_path=str(cfg_file), auto_connect=False)
+ self.assertEqual(client.client_id, "alpha_svc")
+ self.assertEqual(client.status()["clientId"], "alpha_svc")
+ client.disconnect()
+
+ # Env override matching JWT succeeds; mismatching raises DataStackConfigError
+ old_env = os.environ.get("DATASTACK_CLIENT_ID")
+ try:
+ os.environ["DATASTACK_CLIENT_ID"] = "wrong_svc"
+ with self.assertRaises(DataStackConfigError):
+ DataStack(config_path=str(cfg_file), auto_connect=False)
+ finally:
+ if old_env is None:
+ os.environ.pop("DATASTACK_CLIENT_ID", None)
+ else:
+ os.environ["DATASTACK_CLIENT_ID"] = old_env
+
+ # Mismatched kwarg client_id raises DataStackConfigError
+ with self.assertRaises(DataStackConfigError):
+ DataStack(
+ config_path=str(cfg_file),
+ client_id="mismatch_svc",
+ auto_connect=False,
+ )
+
+ # Auto-derives client_id from JWT when CLIENT_ID is omitted
+ tok_derived = _make_jwt({"sub": "derived_worker"})
+ derived_client = DataStack(
+ server_url="http://127.0.0.1:49999",
+ api_url="http://127.0.0.1:39999",
+ token=tok_derived,
+ auto_connect=False,
+ )
+ self.assertEqual(derived_client.client_id, "derived_worker")
+ self.assertEqual(derived_client.status()["clientId"], "derived_worker")
+ derived_client.disconnect()
+
+ def test_http_fallback_and_offline_spool_carry_client_id(self) -> None:
+ recorded_requests: List[Dict[str, Any]] = []
+ tok = _make_jwt({"clientId": "sensor_01", "sub": "sensor_01"})
+
+ class MockHandler(BaseHTTPRequestHandler):
+ def do_GET(self) -> None: # noqa: N802
+ if self.path == "/api/health":
+ body = json.dumps({"status": "online"}).encode("utf-8")
+ self.send_response(200)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+ return
+ self.send_response(404)
+ self.end_headers()
+
+ def do_POST(self) -> None: # noqa: N802
+ length = int(self.headers.get("Content-Length", "0"))
+ raw = self.rfile.read(length).decode("utf-8")
+ parsed = json.loads(raw) if raw else {}
+ recorded_requests.append(
+ {
+ "path": self.path,
+ "client_id_header": self.headers.get("X-DataStack-Client-Id"),
+ "auth": self.headers.get("Authorization"),
+ "body": parsed,
+ }
+ )
+ resp_payload: Dict[str, Any] = {"success": True}
+ if isinstance(parsed, dict) and "batch" in parsed:
+ resp_payload["processed"] = len(parsed["batch"])
+ resp_payload["rejected"] = 0
+ elif isinstance(parsed, dict) and "records" in parsed:
+ resp_payload["processed"] = len(parsed["records"])
+ resp_payload["rejected"] = 0
+ body = json.dumps(resp_payload).encode("utf-8")
+ self.send_response(200)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+ def log_message(self, format: str, *args: Any) -> None:
+ return
+
+ server = ThreadingHTTPServer(("127.0.0.1", 0), MockHandler)
+ port = server.server_address[1]
+ thread = threading.Thread(target=server.serve_forever, daemon=True)
+ thread.start()
+
+ try:
+ with tempfile.TemporaryDirectory() as tmpdir:
+ spool_dir = Path(tmpdir) / "spool"
+ csv_file = Path(tmpdir) / "sample.csv"
+ csv_file.write_text("temp,unit\n21.5,C\n22.0,C\n", encoding="utf-8")
+
+ client = DataStack(
+ server_url="http://127.0.0.1:1", # unreachable WS -> HTTP fallback
+ api_url=f"http://127.0.0.1:{port}",
+ token=tok,
+ client_id="sensor_01",
+ auto_connect=False,
+ request_timeout_sec=1.0,
+ max_retries=0,
+ offline_spool_dir=str(spool_dir),
+ )
+
+ # Single event via HTTP fallback
+ res_single = client.send({"reading": 42})
+ self.assertTrue(res_single.get("success"))
+ self.assertEqual(res_single.get("transport"), "HTTP_FALLBACK")
+
+ # Batch events via HTTP fallback
+ res_batch = client.send_batch([{"k": "a"}, {"k": "b"}])
+ self.assertEqual(res_batch.get("processed"), 2)
+
+ # File ingestion via HTTP fallback
+ res_file = client.send_file(csv_file)
+ self.assertEqual(res_file.get("processed"), 2)
+
+ client.disconnect()
+
+ self.assertEqual(len(recorded_requests), 3)
+ for req in recorded_requests:
+ self.assertEqual(req["path"], "/api/events")
+ self.assertEqual(req["client_id_header"], "sensor_01")
+ self.assertEqual(req["auth"], f"Bearer {tok}")
+
+ self.assertEqual(recorded_requests[0]["body"]["clientId"], "sensor_01")
+ self.assertEqual(
+ [r["clientId"] for r in recorded_requests[1]["body"]["batch"]],
+ ["sensor_01", "sensor_01"],
+ )
+ self.assertEqual(
+ [r["clientId"] for r in recorded_requests[2]["body"]["records"]],
+ ["sensor_01", "sensor_01"],
+ )
+
+ # Offline spool check when HTTP is also unreachable
+ offline_client = datastack_sdk.DataStackClient(
+ server_url="http://127.0.0.1:1",
+ api_url="http://127.0.0.1:1",
+ token=tok,
+ auto_connect=False,
+ request_timeout_sec=1.0,
+ max_retries=0,
+ offline_spool_dir=str(spool_dir),
+ )
+ spool_res = offline_client.send({"offline_metric": 99})
+ offline_client.disconnect()
+ self.assertTrue(spool_res.get("spooled"))
+ self.assertEqual(spool_res.get("transport"), "OFFLINE_SPOOL")
+ self.assertGreaterEqual(spool_res.get("spooledCount", 0), 1)
+ finally:
+ server.shutdown()
+ server.server_close()
+
+
+if __name__ == "__main__":
+ unittest.main(verbosity=2)
diff --git a/tests/phase06-pwa-landing.test.ts b/tests/phase06-pwa-landing.test.ts
new file mode 100644
index 0000000..9bfdec6
--- /dev/null
+++ b/tests/phase06-pwa-landing.test.ts
@@ -0,0 +1,81 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import path from 'node:path';
+import test from 'node:test';
+
+const ROOT = process.cwd();
+
+function readPngDimensions(filePath: string): { width: number; height: number } {
+ const buf = fs.readFileSync(filePath);
+ const pngSig = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
+ assert.ok(buf.subarray(0, 8).equals(pngSig), `${path.basename(filePath)} must be a valid PNG`);
+ assert.equal(buf.subarray(12, 16).toString('ascii'), 'IHDR');
+ return {
+ width: buf.readUInt32BE(16),
+ height: buf.readUInt32BE(20),
+ };
+}
+
+test('BUG-25: PWA manifest, icons (192/512/maskable/apple-touch), and Service Worker are complete and valid', () => {
+ const manifestPath = path.join(ROOT, 'public/manifest.json');
+ const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
+
+ assert.equal(manifest.id, '/');
+ assert.equal(manifest.scope, '/');
+ assert.equal(manifest.display, 'standalone');
+ assert.ok(manifest.short_name && manifest.short_name.length <= 12);
+ assert.ok(Array.isArray(manifest.icons) && manifest.icons.length >= 3);
+
+ for (const icon of manifest.icons) {
+ assert.ok(!String(icon.purpose || '').includes('any maskable'), 'Separate any and maskable purposes');
+ const rel = String(icon.src).replace(/^\//, '');
+ const full = path.join(ROOT, 'public', rel);
+ assert.ok(fs.existsSync(full), `Missing manifest icon ${icon.src}`);
+ const [expW, expH] = String(icon.sizes).split('x').map(Number);
+ const dims = readPngDimensions(full);
+ assert.equal(dims.width, expW);
+ assert.equal(dims.height, expH);
+ }
+
+ const appleDims = readPngDimensions(path.join(ROOT, 'public/apple-touch-icon.png'));
+ assert.deepEqual(appleDims, { width: 180, height: 180 });
+ assert.ok(fs.existsSync(path.join(ROOT, 'public/icon.svg')));
+
+ const swContent = fs.readFileSync(path.join(ROOT, 'public/sw.js'), 'utf8');
+ assert.match(swContent, /request\.method !== ['"]GET['"]/);
+ assert.match(swContent, /\/api\//);
+ assert.match(swContent, /\/socket\.io\//);
+ assert.match(swContent, /new Response\(/);
+
+ const layoutContent = fs.readFileSync(path.join(ROOT, 'app/layout.tsx'), 'utf8');
+ assert.match(layoutContent, /PwaLifecycle/);
+ assert.match(layoutContent, /export const viewport: Viewport/);
+ assert.match(layoutContent, /apple-touch-icon\.png/);
+
+ const pwaClient = fs.readFileSync(path.join(ROOT, 'components/pwa/PwaLifecycle.tsx'), 'utf8');
+ assert.match(pwaClient, /serviceWorker\.register\(['"]\/sw\.js['"]/);
+ assert.match(pwaClient, /beforeinstallprompt/);
+});
+
+test('BUG-17: Landing/Navbar/Footer/404 navigation uses root-relative anchors and valid Link semantics', () => {
+ const files = [
+ 'components/layout/Navbar.tsx',
+ 'components/layout/Footer.tsx',
+ 'components/sections/Hero.tsx',
+ 'components/sections/LandingAbout.tsx',
+ 'app/not-found.tsx',
+ ];
+
+ for (const rel of files) {
+ const content = fs.readFileSync(path.join(ROOT, rel), 'utf8');
+ assert.equal(/href=["']#/.test(content), false, `${rel} should use /#... root anchors instead of bare #...`);
+ const linkBlocks = content.match(/ /g) || [];
+ for (const block of linkBlocks) {
+ assert.equal(/ inside : ${block}`);
+ }
+ }
+
+ const notFoundContent = fs.readFileSync(path.join(ROOT, 'app/not-found.tsx'), 'utf8');
+ assert.match(notFoundContent, /href=["']\/["']/);
+ assert.match(notFoundContent, /href=["']\/vcon\/dashboard["']/);
+});