refactor(auth): standardize session management and security
- Centralize secret retrieval using `getAdminSessionSecret` - Replace manual session parsing with `verifyRequestAdminSession` utility - Tighten API validation and improve security headers handling - Enforce environment variable security for production deployments
This commit is contained in:
1 parent
34519c07df
commit
177ed4a844
78 files changed
+6748
-3157
No files matched your search
+17
-10
@@ -1,16 +1,23 @@
|
||||
# DataStack Environment Variables Configuration
|
||||
# NOTE: In production (NODE_ENV=production), replace all placeholder values below with strong 32+ char secrets.
|
||||
|
||||
# Administrator Account (defaults to admin@datastack.dev / admin12345678 if omitted)
|
||||
DATASTACK_ADMIN_EMAIL=admin@datastack.dev
|
||||
DATASTACK_ADMIN_PASSWORD=admin12345678
|
||||
# Security & Session Secrets (REQUIRED in production, min 32 characters)
|
||||
DATASTACK_ADMIN_SESSION_SECRET=replace-with-32-plus-char-admin-session-secret
|
||||
DATASTACK_JWT_SECRET=replace-with-32-plus-char-client-jwt-secret
|
||||
DATASTACK_ADMIN_SETUP_KEY=replace-with-32-plus-char-admin-setup-key
|
||||
DATASTACK_INTERNAL_SECRET=replace-with-32-plus-char-internal-service-secret
|
||||
|
||||
# Security & Session Secrets (min 32 characters)
|
||||
DATASTACK_ADMIN_SESSION_SECRET=datastack-admin-session-default-secret-32-chars
|
||||
DATASTACK_JWT_SECRET=datastack-jwt-default-secret-key-32-chars-long
|
||||
DATASTACK_ADMIN_SETUP_KEY=datastack-default-setup-key-32-chars
|
||||
# Administrator Account (development/in-memory bootstrap; production uses /vcon/setup)
|
||||
DATASTACK_ADMIN_EMAIL=admin@example.com
|
||||
# DATASTACK_ADMIN_PASSWORD=replace-with-strong-password
|
||||
# DATASTACK_ALLOW_DEMO_ADMIN=false
|
||||
|
||||
# PostgreSQL Database (optional - in-memory fallback active when not configured)
|
||||
# PostgreSQL Database (recommended for production persistence)
|
||||
# DATABASE_URL=postgresql://datastack:password@localhost:5432/datastack_db
|
||||
|
||||
# Ingestion Engine Port (defaults to 4000)
|
||||
PORT_SOCKET=4000
|
||||
# Shared Redis Buffer (optional; enables cross-process buffering & distributed flush locks)
|
||||
# REDIS_URL=redis://127.0.0.1:6379
|
||||
|
||||
# Service Networking
|
||||
PORT_SOCKET=4000
|
||||
INGESTION_INTERNAL_URL=http://127.0.0.1:4000
|
||||
@@ -9,10 +9,8 @@ import { redirect } from 'next/navigation';
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
export default async function LandingPage() {
|
||||
if (process.env.DATABASE_URL) {
|
||||
const isConfigured = await isAdminSetupComplete().catch(() => false);
|
||||
if (!isConfigured) redirect('/vcon/setup');
|
||||
}
|
||||
const isConfigured = await isAdminSetupComplete().catch(() => false);
|
||||
if (!isConfigured) redirect('/vcon/setup');
|
||||
|
||||
return (
|
||||
<MainLayout>
|
||||
|
||||
+102
-79
@@ -1,11 +1,12 @@
|
||||
import {
|
||||
createAdminSessionToken,
|
||||
getAdminSessionSecret,
|
||||
getSessionCookieOptions,
|
||||
} from "@/core/auth/adminSession";
|
||||
import {
|
||||
authenticateAdminCredentials,
|
||||
clearAdminLoginAttempts,
|
||||
consumeAdminLoginAttempt,
|
||||
verifyAdminCredentials,
|
||||
} from "@/core/auth/adminUsers";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
@@ -13,10 +14,19 @@ const SESSION_COOKIE = "datastack_admin_session";
|
||||
const SESSION_SECONDS = 8 * 60 * 60;
|
||||
const REMEMBERED_SESSION_SECONDS = 30 * 24 * 60 * 60;
|
||||
|
||||
function extractClientIp(request: NextRequest) {
|
||||
const forwardedFor = request.headers.get("x-forwarded-for");
|
||||
if (forwardedFor) {
|
||||
const first = forwardedFor.split(",")[0]?.trim();
|
||||
if (first) return first;
|
||||
}
|
||||
const realIp = request.headers.get("x-real-ip")?.trim();
|
||||
if (realIp) return realIp;
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
const sessionSecret =
|
||||
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
|
||||
"datastack-admin-session-default-secret-32-chars";
|
||||
const sessionSecret = getAdminSessionSecret();
|
||||
|
||||
if (!sessionSecret || sessionSecret.length < 32) {
|
||||
return NextResponse.json(
|
||||
@@ -27,86 +37,99 @@ export async function POST(request: NextRequest) {
|
||||
);
|
||||
}
|
||||
|
||||
let body: Record<string, unknown>;
|
||||
try {
|
||||
const body = await request.json();
|
||||
const email =
|
||||
typeof body.email === "string" ? body.email.trim().toLowerCase() : "";
|
||||
const password = typeof body.password === "string" ? body.password : "";
|
||||
if (!email || password.length === 0 || password.length > 1024) {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid email or password." },
|
||||
{ status: 401 },
|
||||
);
|
||||
}
|
||||
|
||||
let throttleDecision;
|
||||
try {
|
||||
throttleDecision = await consumeAdminLoginAttempt(email, sessionSecret);
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Admin authentication is temporarily unavailable." },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
if (!throttleDecision.allowed) {
|
||||
return NextResponse.json(
|
||||
{ error: "Too many sign-in attempts. Try again later." },
|
||||
{
|
||||
status: 429,
|
||||
headers: {
|
||||
"Retry-After": String(throttleDecision.retryAfterSeconds),
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let authenticated = false;
|
||||
try {
|
||||
authenticated = await verifyAdminCredentials(email, password);
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Admin authentication is temporarily unavailable." },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
if (!authenticated) {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid email or password." },
|
||||
{ status: 401 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
await clearAdminLoginAttempts(email, sessionSecret);
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Admin authentication is temporarily unavailable." },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
const remember = body.remember === true;
|
||||
const maxAge = remember ? REMEMBERED_SESSION_SECONDS : SESSION_SECONDS;
|
||||
const expiresAt = Date.now() + maxAge * 1000;
|
||||
const token = await createAdminSessionToken(
|
||||
email,
|
||||
sessionSecret,
|
||||
expiresAt,
|
||||
);
|
||||
const response = NextResponse.json({ success: true });
|
||||
|
||||
response.cookies.set(
|
||||
SESSION_COOKIE,
|
||||
token,
|
||||
getSessionCookieOptions(request, maxAge),
|
||||
);
|
||||
|
||||
return response;
|
||||
const parsed = await request.json();
|
||||
body =
|
||||
parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
||||
? (parsed as Record<string, unknown>)
|
||||
: {};
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid login request." },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
const email =
|
||||
typeof body.email === "string" ? body.email.trim().toLowerCase() : "";
|
||||
const password = typeof body.password === "string" ? body.password : "";
|
||||
if (!email || password.length === 0 || password.length > 1024) {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid email or password." },
|
||||
{ status: 401 },
|
||||
);
|
||||
}
|
||||
|
||||
const clientIp = extractClientIp(request);
|
||||
|
||||
let throttleDecision;
|
||||
try {
|
||||
throttleDecision = await consumeAdminLoginAttempt(
|
||||
email,
|
||||
sessionSecret,
|
||||
clientIp,
|
||||
);
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Admin authentication is temporarily unavailable." },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
if (!throttleDecision.allowed) {
|
||||
return NextResponse.json(
|
||||
{ error: "Too many sign-in attempts. Try again later." },
|
||||
{
|
||||
status: 429,
|
||||
headers: {
|
||||
"Retry-After": String(throttleDecision.retryAfterSeconds),
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let authResult: { authenticated: boolean; tokenVersion: number };
|
||||
try {
|
||||
authResult = await authenticateAdminCredentials(email, password);
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Admin authentication is temporarily unavailable." },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
if (!authResult.authenticated) {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid email or password." },
|
||||
{ status: 401 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
await clearAdminLoginAttempts(email, sessionSecret, clientIp);
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Admin authentication is temporarily unavailable." },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
const remember = body.remember === true;
|
||||
const maxAge = remember ? REMEMBERED_SESSION_SECONDS : SESSION_SECONDS;
|
||||
const expiresAt = Date.now() + maxAge * 1000;
|
||||
const token = await createAdminSessionToken(
|
||||
email,
|
||||
sessionSecret,
|
||||
expiresAt,
|
||||
authResult.tokenVersion,
|
||||
);
|
||||
const response = NextResponse.json({ success: true });
|
||||
|
||||
response.cookies.set(
|
||||
SESSION_COOKIE,
|
||||
token,
|
||||
getSessionCookieOptions(request, maxAge),
|
||||
);
|
||||
|
||||
return response;
|
||||
}
|
||||
@@ -1,11 +1,11 @@
|
||||
import {
|
||||
createAdminSessionToken,
|
||||
getAdminSessionPayload,
|
||||
getSessionCookieOptions,
|
||||
} from "@/core/auth/adminSession";
|
||||
import {
|
||||
getAdminAccountInfo,
|
||||
updateAdminAccount,
|
||||
verifyRequestAdminSession,
|
||||
} from "@/core/auth/adminUsers";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
@@ -17,51 +17,69 @@ function validEmail(email: string) {
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const sessionSecret =
|
||||
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
|
||||
"datastack-admin-session-default-secret-32-chars";
|
||||
const token = request.cookies.get(SESSION_COOKIE)?.value || "";
|
||||
const session = await getAdminSessionPayload(token, sessionSecret);
|
||||
const token = request.cookies.get(SESSION_COOKIE)?.value;
|
||||
const auth = await verifyRequestAdminSession(token);
|
||||
|
||||
if (!session) {
|
||||
return NextResponse.json(
|
||||
{ error: "Authentication required." },
|
||||
{ status: 401 },
|
||||
);
|
||||
if (!auth.valid) {
|
||||
const status = auth.reason === "UNAUTHORIZED" ? 401 : 503;
|
||||
const message =
|
||||
auth.reason === "UNAUTHORIZED"
|
||||
? "Authentication required."
|
||||
: "Admin authentication is temporarily unavailable.";
|
||||
return NextResponse.json({ error: message }, { status });
|
||||
}
|
||||
|
||||
const account = await getAdminAccountInfo(session.email);
|
||||
return NextResponse.json({
|
||||
profile: {
|
||||
email: account.email,
|
||||
role: "Owner",
|
||||
storageMode: account.storageMode,
|
||||
createdAt: account.createdAt,
|
||||
sessionExpiresAt: new Date(session.expiresAt).toISOString(),
|
||||
},
|
||||
});
|
||||
try {
|
||||
const account = await getAdminAccountInfo(auth.session.email);
|
||||
return NextResponse.json({
|
||||
profile: {
|
||||
email: account.email,
|
||||
role: "Owner",
|
||||
storageMode: account.storageMode,
|
||||
createdAt: account.createdAt,
|
||||
sessionExpiresAt: new Date(auth.session.expiresAt).toISOString(),
|
||||
},
|
||||
});
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Unable to load admin profile." },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function PUT(request: NextRequest) {
|
||||
const sessionSecret =
|
||||
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
|
||||
"datastack-admin-session-default-secret-32-chars";
|
||||
const token = request.cookies.get(SESSION_COOKIE)?.value || "";
|
||||
const session = await getAdminSessionPayload(token, sessionSecret);
|
||||
const token = request.cookies.get(SESSION_COOKIE)?.value;
|
||||
const auth = await verifyRequestAdminSession(token);
|
||||
|
||||
if (!session) {
|
||||
if (!auth.valid) {
|
||||
const status = auth.reason === "UNAUTHORIZED" ? 401 : 503;
|
||||
const message =
|
||||
auth.reason === "UNAUTHORIZED"
|
||||
? "Authentication required."
|
||||
: "Admin authentication is temporarily unavailable.";
|
||||
return NextResponse.json({ error: message }, { status });
|
||||
}
|
||||
|
||||
let body: Record<string, unknown>;
|
||||
try {
|
||||
const parsed = await request.json();
|
||||
body =
|
||||
parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
||||
? (parsed as Record<string, unknown>)
|
||||
: {};
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Authentication required." },
|
||||
{ status: 401 },
|
||||
{ error: "Invalid profile payload." },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const body = await request.json();
|
||||
const nextEmail =
|
||||
typeof body.email === "string"
|
||||
? body.email.trim().toLowerCase()
|
||||
: session.email;
|
||||
: auth.session.email;
|
||||
const currentPassword =
|
||||
typeof body.currentPassword === "string" ? body.currentPassword : "";
|
||||
const newPassword =
|
||||
@@ -88,7 +106,7 @@ export async function PUT(request: NextRequest) {
|
||||
);
|
||||
}
|
||||
|
||||
const account = await getAdminAccountInfo(session.email);
|
||||
const account = await getAdminAccountInfo(auth.session.email);
|
||||
const result = await updateAdminAccount({
|
||||
currentEmail: account.email,
|
||||
currentPassword,
|
||||
@@ -103,13 +121,14 @@ export async function PUT(request: NextRequest) {
|
||||
// Preserve remaining session duration (e.g., 30-day "Remember me" session)
|
||||
const remainingSeconds = Math.max(
|
||||
SESSION_SECONDS,
|
||||
Math.ceil((session.expiresAt - Date.now()) / 1000),
|
||||
Math.ceil((auth.session.expiresAt - Date.now()) / 1000),
|
||||
);
|
||||
const expiresAt = Date.now() + remainingSeconds * 1000;
|
||||
const nextToken = await createAdminSessionToken(
|
||||
result.email,
|
||||
sessionSecret,
|
||||
auth.sessionSecret,
|
||||
expiresAt,
|
||||
result.tokenVersion,
|
||||
);
|
||||
|
||||
const response = NextResponse.json({
|
||||
@@ -133,7 +152,7 @@ export async function PUT(request: NextRequest) {
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Unable to update profile." },
|
||||
{ status: 400 },
|
||||
{ status: 500 },
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import {
|
||||
createAdminSessionToken,
|
||||
getAdminSessionSecret,
|
||||
getSessionCookieOptions,
|
||||
} from "@/core/auth/adminSession";
|
||||
import {
|
||||
@@ -20,9 +21,7 @@ function validEmail(email: string) {
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
const sessionSecret =
|
||||
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
|
||||
"datastack-admin-session-default-secret-32-chars";
|
||||
const sessionSecret = getAdminSessionSecret();
|
||||
if (!sessionSecret || sessionSecret.length < 32) {
|
||||
return NextResponse.json(
|
||||
{ error: "Admin session configuration is unavailable." },
|
||||
@@ -30,8 +29,21 @@ export async function POST(request: NextRequest) {
|
||||
);
|
||||
}
|
||||
|
||||
let body: Record<string, unknown>;
|
||||
try {
|
||||
const parsed = await request.json();
|
||||
body =
|
||||
parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
||||
? (parsed as Record<string, unknown>)
|
||||
: {};
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid setup request payload." },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const body = await request.json();
|
||||
const email =
|
||||
typeof body.email === "string" ? body.email.trim().toLowerCase() : "";
|
||||
const password = typeof body.password === "string" ? body.password : "";
|
||||
@@ -74,6 +86,7 @@ export async function POST(request: NextRequest) {
|
||||
email,
|
||||
sessionSecret,
|
||||
Date.now() + SESSION_SECONDS * 1000,
|
||||
1,
|
||||
);
|
||||
const response = NextResponse.json({ success: true });
|
||||
response.cookies.set(
|
||||
|
||||
@@ -1,23 +1,54 @@
|
||||
import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
|
||||
import { SDKManager } from "@/core/sdk/sdkManager";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
const SESSION_COOKIE = "datastack_admin_session";
|
||||
|
||||
export async function GET(
|
||||
_req: NextRequest,
|
||||
req: NextRequest,
|
||||
{ params }: { params: Promise<{ id: string }> },
|
||||
) {
|
||||
const auth = await verifyRequestAdminSession(
|
||||
req.cookies.get(SESSION_COOKIE)?.value,
|
||||
);
|
||||
if (!auth.valid) {
|
||||
return NextResponse.json(
|
||||
{ error: "Authentication required." },
|
||||
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
|
||||
);
|
||||
}
|
||||
|
||||
const { id } = await params;
|
||||
const client = await SDKManager.getClientAsync(id);
|
||||
if (!client) {
|
||||
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
||||
}
|
||||
const { secret, token, ...clientSummary } = client;
|
||||
return NextResponse.json({ client: clientSummary });
|
||||
void secret;
|
||||
const includeToken = req.nextUrl.searchParams.get("includeToken") === "true";
|
||||
return NextResponse.json({
|
||||
client: {
|
||||
...clientSummary,
|
||||
tokenPreview: token ? `${token.slice(0, 12)}...` : "",
|
||||
...(includeToken ? { token } : {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function PATCH(
|
||||
req: NextRequest,
|
||||
{ params }: { params: Promise<{ id: string }> },
|
||||
) {
|
||||
const auth = await verifyRequestAdminSession(
|
||||
req.cookies.get(SESSION_COOKIE)?.value,
|
||||
);
|
||||
if (!auth.valid) {
|
||||
return NextResponse.json(
|
||||
{ error: "Authentication required." },
|
||||
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
|
||||
);
|
||||
}
|
||||
|
||||
const { id } = await params;
|
||||
try {
|
||||
const body = await req.json();
|
||||
@@ -46,11 +77,22 @@ export async function PATCH(
|
||||
}
|
||||
|
||||
export async function DELETE(
|
||||
_req: NextRequest,
|
||||
req: NextRequest,
|
||||
{ params }: { params: Promise<{ id: string }> },
|
||||
) {
|
||||
const auth = await verifyRequestAdminSession(
|
||||
req.cookies.get(SESSION_COOKIE)?.value,
|
||||
);
|
||||
if (!auth.valid) {
|
||||
return NextResponse.json(
|
||||
{ error: "Authentication required." },
|
||||
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
|
||||
);
|
||||
}
|
||||
|
||||
const { id } = await params;
|
||||
const deleted = await SDKManager.deleteClientAsync(id);
|
||||
const purgeFiles = req.nextUrl.searchParams.get("purgeFiles") === "true";
|
||||
const deleted = await SDKManager.deleteClientAsync(id, { purgeFiles });
|
||||
if (!deleted) {
|
||||
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
||||
}
|
||||
|
||||
@@ -1,12 +1,28 @@
|
||||
import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
|
||||
import { SDKManager } from "@/core/sdk/sdkManager";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
export async function GET() {
|
||||
const SESSION_COOKIE = "datastack_admin_session";
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
const auth = await verifyRequestAdminSession(
|
||||
req.cookies.get(SESSION_COOKIE)?.value,
|
||||
);
|
||||
if (!auth.valid) {
|
||||
return NextResponse.json(
|
||||
{ error: "Authentication required." },
|
||||
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const allClients = await SDKManager.listClientsAsync();
|
||||
const clients = allClients.map(({ secret, ...client }) => {
|
||||
const clients = allClients.map(({ secret, token, ...client }) => {
|
||||
void secret;
|
||||
return client;
|
||||
return {
|
||||
...client,
|
||||
tokenPreview: token ? `${token.slice(0, 12)}...` : "",
|
||||
};
|
||||
});
|
||||
return NextResponse.json({ clients, count: clients.length });
|
||||
} catch (err: unknown) {
|
||||
@@ -17,8 +33,31 @@ export async function GET() {
|
||||
}
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
const auth = await verifyRequestAdminSession(
|
||||
req.cookies.get(SESSION_COOKIE)?.value,
|
||||
);
|
||||
if (!auth.valid) {
|
||||
return NextResponse.json(
|
||||
{ error: "Authentication required." },
|
||||
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
|
||||
);
|
||||
}
|
||||
|
||||
let body: Record<string, unknown>;
|
||||
try {
|
||||
const parsed = await req.json();
|
||||
body =
|
||||
parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
||||
? (parsed as Record<string, unknown>)
|
||||
: {};
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid request payload." },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const body = await req.json();
|
||||
const { clientId, name, rateLimit } = body;
|
||||
|
||||
if (!clientId || !name) {
|
||||
|
||||
+35
-9
@@ -1,4 +1,6 @@
|
||||
import { SettingsManager } from "@/core/adminSettings/settingsManager";
|
||||
import { DataBufferEngine } from "@/core/data/buffer";
|
||||
import { getInternalIngestionHeaders } from "@/core/Security/security";
|
||||
import { CSVStorageManager } from "@/core/Storage/csvStorage";
|
||||
import { parse } from "csv-parse";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
@@ -31,18 +33,25 @@ export async function GET(request: NextRequest) {
|
||||
let total = 0;
|
||||
|
||||
const addIfMatching = (event: DataEvent) => {
|
||||
const eventDate = event.timestamp.slice(0, 10);
|
||||
const eventDate = CSVStorageManager.extractSafePartitionDate(event.timestamp);
|
||||
const dedupeKey = `${event.clientId}:${event.eventId}`;
|
||||
const matchesSearch =
|
||||
!search ||
|
||||
event.payload.toLowerCase().includes(search) ||
|
||||
event.eventId.toLowerCase().includes(search) ||
|
||||
event.clientId.toLowerCase().includes(search);
|
||||
|
||||
if (
|
||||
(clientId !== "all" && event.clientId !== clientId) ||
|
||||
(from && eventDate < from) ||
|
||||
(to && eventDate > to) ||
|
||||
(search && !event.payload.toLowerCase().includes(search)) ||
|
||||
eventIds.has(event.eventId)
|
||||
!matchesSearch ||
|
||||
eventIds.has(dedupeKey)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
eventIds.add(event.eventId);
|
||||
eventIds.add(dedupeKey);
|
||||
total += 1;
|
||||
|
||||
let low = 0;
|
||||
@@ -109,7 +118,16 @@ export async function GET(request: NextRequest) {
|
||||
}
|
||||
}
|
||||
|
||||
for (const event of DataBufferEngine.getRecentEvents(200)) {
|
||||
const settings = await SettingsManager.getSettingsAsync();
|
||||
const recentWindowLimit = Math.min(
|
||||
2000,
|
||||
Math.max(200, settings.batchFlushThreshold || 500),
|
||||
);
|
||||
|
||||
for (const event of await DataBufferEngine.getRecentEventsAsync(
|
||||
recentWindowLimit,
|
||||
clientId,
|
||||
)) {
|
||||
addIfMatching({ ...event, status: "buffered" });
|
||||
}
|
||||
|
||||
@@ -117,10 +135,18 @@ export async function GET(request: NextRequest) {
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||
).replace(/\/$/, "");
|
||||
try {
|
||||
const res = await fetch(`${ingestionBaseUrl}/internal/recent?limit=200`, {
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(1000),
|
||||
});
|
||||
const qs = new URLSearchParams({ limit: String(recentWindowLimit) });
|
||||
if (clientId && clientId !== "all") {
|
||||
qs.set("clientId", clientId);
|
||||
}
|
||||
const res = await fetch(
|
||||
`${ingestionBaseUrl}/internal/recent?${qs.toString()}`,
|
||||
{
|
||||
headers: getInternalIngestionHeaders(),
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(1000),
|
||||
},
|
||||
);
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
if (Array.isArray(data.events)) {
|
||||
|
||||
+75
-30
@@ -1,9 +1,12 @@
|
||||
import { SettingsManager } from '@/core/adminSettings/settingsManager';
|
||||
import { verifyAdminSessionToken } from '@/core/auth/adminSession';
|
||||
import { verifyRequestAdminSession } from '@/core/auth/adminUsers';
|
||||
import { AuthService } from '@/core/auth/jwt';
|
||||
import { DataBufferEngine } from '@/core/data/buffer';
|
||||
import { SDKManager } from '@/core/sdk/sdkManager';
|
||||
import { SecurityEngine } from '@/core/Security/security';
|
||||
import {
|
||||
SecurityEngine,
|
||||
getInternalIngestionHeaders,
|
||||
} from '@/core/Security/security';
|
||||
import { CSVStorageManager, type DataEvent } from '@/core/Storage/csvStorage';
|
||||
import { parse } from 'csv-parse';
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
@@ -15,14 +18,15 @@ async function getRecentStoredEvents(
|
||||
limit: number,
|
||||
clientId?: string,
|
||||
): Promise<DataEvent[]> {
|
||||
const files = CSVStorageManager.listAllFiles()
|
||||
const allFiles = await CSVStorageManager.listAllFilesAsync();
|
||||
const files = allFiles
|
||||
.filter((f) => !clientId || clientId === 'all' || f.clientId === clientId)
|
||||
.slice(0, 5);
|
||||
|
||||
const collected: DataEvent[] = [];
|
||||
|
||||
for (const file of files) {
|
||||
const fileEvents: DataEvent[] = [];
|
||||
const fileTail: DataEvent[] = [];
|
||||
const parser = createReadStream(file.filePath).pipe(
|
||||
parse({ columns: true, skip_empty_lines: true, bom: true }),
|
||||
);
|
||||
@@ -32,7 +36,10 @@ async function getRecentStoredEvents(
|
||||
Record<string, string>
|
||||
>) {
|
||||
if (!record.eventId) continue;
|
||||
fileEvents.push({
|
||||
if (fileTail.length >= limit) {
|
||||
fileTail.shift();
|
||||
}
|
||||
fileTail.push({
|
||||
clientId: record.clientId || file.clientId,
|
||||
eventId: record.eventId,
|
||||
timestamp: record.timestamp || `${file.date}T00:00:00.000Z`,
|
||||
@@ -43,8 +50,8 @@ async function getRecentStoredEvents(
|
||||
parser.destroy();
|
||||
}
|
||||
|
||||
if (fileEvents.length > 0) {
|
||||
collected.push(...fileEvents.slice(-limit));
|
||||
if (fileTail.length > 0) {
|
||||
collected.push(...fileTail);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,9 +62,10 @@ export async function GET(req: NextRequest) {
|
||||
try {
|
||||
const { searchParams } = new URL(req.url);
|
||||
const clientId = searchParams.get('clientId') || undefined;
|
||||
const limit = Math.min(200, Math.max(1, Number(searchParams.get('limit')) || 50));
|
||||
const limit = Math.min(2000, Math.max(1, Number(searchParams.get('limit')) || 50));
|
||||
|
||||
const bufferedEvents = DataBufferEngine.getRecentEvents(limit, clientId);
|
||||
await SettingsManager.getSettingsAsync();
|
||||
const bufferedEvents = await DataBufferEngine.getRecentEventsAsync(limit, clientId);
|
||||
let socketBufferedEvents: DataEvent[] = [];
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
|
||||
@@ -67,6 +75,7 @@ export async function GET(req: NextRequest) {
|
||||
const qs = new URLSearchParams({ limit: String(limit) });
|
||||
if (clientId) qs.set('clientId', clientId);
|
||||
const res = await fetch(`${ingestionBaseUrl}/internal/recent?${qs.toString()}`, {
|
||||
headers: getInternalIngestionHeaders(),
|
||||
cache: 'no-store',
|
||||
signal: AbortSignal.timeout(1000),
|
||||
});
|
||||
@@ -86,8 +95,10 @@ export async function GET(req: NextRequest) {
|
||||
const merged: DataEvent[] = [];
|
||||
|
||||
for (const evt of [...bufferedEvents, ...socketBufferedEvents, ...storedEvents]) {
|
||||
if (!evt.eventId || seenIds.has(evt.eventId)) continue;
|
||||
seenIds.add(evt.eventId);
|
||||
if (!evt.eventId) continue;
|
||||
const dedupeKey = `${evt.clientId}:${evt.eventId}`;
|
||||
if (seenIds.has(dedupeKey)) continue;
|
||||
seenIds.add(dedupeKey);
|
||||
merged.push(evt);
|
||||
}
|
||||
|
||||
@@ -107,9 +118,6 @@ export async function POST(req: NextRequest) {
|
||||
|
||||
// Verify either Client Bearer JWT or Admin Session Cookie
|
||||
const authHeader = req.headers.get('authorization');
|
||||
const sessionSecret =
|
||||
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
|
||||
'datastack-admin-session-default-secret-32-chars';
|
||||
const sessionCookie = req.cookies.get(SESSION_COOKIE)?.value;
|
||||
|
||||
let tokenClientId: string | null = null;
|
||||
@@ -128,20 +136,30 @@ export async function POST(req: NextRequest) {
|
||||
tokenClientId = verified.payload.clientId;
|
||||
tokenRateLimit = verified.payload.rateLimit;
|
||||
} else {
|
||||
const isAdmin = Boolean(
|
||||
sessionSecret &&
|
||||
sessionCookie &&
|
||||
(await verifyAdminSessionToken(sessionCookie, sessionSecret)),
|
||||
);
|
||||
if (!isAdmin) {
|
||||
const auth = await verifyRequestAdminSession(sessionCookie);
|
||||
if (!auth.valid) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Authentication required. Provide a valid Client Bearer JWT or Admin Session.' },
|
||||
{ status: 401 },
|
||||
{
|
||||
error:
|
||||
auth.reason === 'UNAUTHORIZED'
|
||||
? 'Authentication required. Provide a valid Client Bearer JWT or Admin Session.'
|
||||
: 'Admin authentication is temporarily unavailable.',
|
||||
},
|
||||
{ status: auth.reason === 'UNAUTHORIZED' ? 401 : 503 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const body = await req.json();
|
||||
let body: unknown;
|
||||
try {
|
||||
body = await req.json();
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: 'Invalid JSON request payload.', code: 'INVALID_JSON' },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
const bodyObj =
|
||||
body && typeof body === 'object' && !Array.isArray(body)
|
||||
? (body as Record<string, unknown>)
|
||||
@@ -154,7 +172,27 @@ export async function POST(req: NextRequest) {
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9_-]/g, '_');
|
||||
|
||||
if (await SDKManager.isClientDeletedAsync(clientId)) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: `CLIENT_REVOKED: Client '${clientId}' has been deleted by administrator.`,
|
||||
code: 'CLIENT_REVOKED',
|
||||
},
|
||||
{ status: 403 },
|
||||
);
|
||||
}
|
||||
|
||||
const registeredClient = await SDKManager.getClientAsync(clientId);
|
||||
if (rawBearerToken && !registeredClient) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: `CLIENT_REVOKED: Client '${clientId}' registration not found or has been deleted.`,
|
||||
code: 'CLIENT_REVOKED',
|
||||
},
|
||||
{ status: 403 },
|
||||
);
|
||||
}
|
||||
|
||||
if (registeredClient) {
|
||||
if (registeredClient.status === 'revoked') {
|
||||
return NextResponse.json(
|
||||
@@ -196,6 +234,7 @@ export async function POST(req: NextRequest) {
|
||||
: null;
|
||||
|
||||
if (batchItems !== null) {
|
||||
const envelopeKeys = new Set(['eventId', 'timestamp', 'data', 'clientId']);
|
||||
let processed = 0;
|
||||
let rejected = 0;
|
||||
let duplicates = 0;
|
||||
@@ -205,6 +244,8 @@ export async function POST(req: NextRequest) {
|
||||
const oversizedEventIds: string[] = [];
|
||||
const errors: string[] = [];
|
||||
|
||||
const batchEvents: DataEvent[] = [];
|
||||
|
||||
for (const item of batchItems) {
|
||||
const safeItem =
|
||||
item && typeof item === 'object' && !Array.isArray(item)
|
||||
@@ -221,8 +262,10 @@ export async function POST(req: NextRequest) {
|
||||
? safeItem.timestamp.trim()
|
||||
: new Date().toISOString();
|
||||
|
||||
const itemData =
|
||||
safeItem.data !== undefined ? safeItem.data : safeItem;
|
||||
const isEnvelope =
|
||||
safeItem.data !== undefined &&
|
||||
Object.keys(safeItem).every((k) => envelopeKeys.has(k));
|
||||
const itemData = isEnvelope ? safeItem.data : safeItem;
|
||||
|
||||
const val = SecurityEngine.validateEvent(
|
||||
clientId,
|
||||
@@ -251,18 +294,20 @@ export async function POST(req: NextRequest) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const event: DataEvent = {
|
||||
batchEvents.push({
|
||||
clientId,
|
||||
eventId: itemEventId,
|
||||
timestamp: itemTimestamp,
|
||||
payload:
|
||||
typeof itemData === 'string' ? itemData : JSON.stringify(itemData),
|
||||
};
|
||||
|
||||
DataBufferEngine.push(event);
|
||||
});
|
||||
processed += 1;
|
||||
}
|
||||
|
||||
if (batchEvents.length > 0) {
|
||||
await DataBufferEngine.pushBatchAsync(batchEvents);
|
||||
}
|
||||
|
||||
if (processed > 0) {
|
||||
SDKManager.incrementEventCount(clientId, processed);
|
||||
}
|
||||
@@ -347,7 +392,7 @@ export async function POST(req: NextRequest) {
|
||||
payload: typeof data === 'string' ? data : JSON.stringify(data),
|
||||
};
|
||||
|
||||
DataBufferEngine.push(event);
|
||||
await DataBufferEngine.pushAsync(event);
|
||||
SDKManager.incrementEventCount(clientId, 1);
|
||||
|
||||
return NextResponse.json({ success: true, eventId, event });
|
||||
|
||||
+40
-14
@@ -1,4 +1,6 @@
|
||||
import { SettingsManager } from "@/core/adminSettings/settingsManager";
|
||||
import { DataBufferEngine } from "@/core/data/buffer";
|
||||
import { getInternalIngestionHeaders } from "@/core/Security/security";
|
||||
import {
|
||||
CSVStorageManager,
|
||||
type StorageFileMeta,
|
||||
@@ -6,6 +8,7 @@ import {
|
||||
import { parse } from "csv-parse";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { createReadStream } from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
type ExportFormat = "csv" | "json";
|
||||
type CsvRecord = Record<string, string>;
|
||||
@@ -108,19 +111,26 @@ function streamFile(file: StorageFileMeta) {
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
await DataBufferEngine.flush();
|
||||
const ingestionBaseUrl =
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000";
|
||||
try {
|
||||
await fetch(`${ingestionBaseUrl}/internal/flush`, {
|
||||
method: "POST",
|
||||
signal: AbortSignal.timeout(1500),
|
||||
});
|
||||
} catch {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
const { searchParams } = new URL(request.url);
|
||||
const shouldFlush = searchParams.get("flush") !== "false";
|
||||
|
||||
await SettingsManager.getSettingsAsync();
|
||||
if (shouldFlush) {
|
||||
await DataBufferEngine.flush();
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||
).replace(/\/$/, "");
|
||||
try {
|
||||
await fetch(`${ingestionBaseUrl}/internal/flush`, {
|
||||
method: "POST",
|
||||
headers: getInternalIngestionHeaders(),
|
||||
signal: AbortSignal.timeout(1500),
|
||||
});
|
||||
} catch {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
}
|
||||
}
|
||||
|
||||
const { searchParams } = new URL(request.url);
|
||||
const clientId = searchParams.get("clientId") || "all";
|
||||
const filename = searchParams.get("filename");
|
||||
const from = searchParams.get("from") || "";
|
||||
@@ -129,17 +139,33 @@ export async function GET(request: NextRequest) {
|
||||
searchParams.get("format") === "json" ? "json" : "csv";
|
||||
const includeHeader = searchParams.get("includeHeader") !== "false";
|
||||
|
||||
let files = CSVStorageManager.listAllFiles();
|
||||
let files = await CSVStorageManager.listAllFilesAsync();
|
||||
if (clientId !== "all")
|
||||
files = files.filter((file) => file.clientId === clientId);
|
||||
|
||||
if (filename) {
|
||||
const file = files.find((candidate) => candidate.filename === filename);
|
||||
if (!file)
|
||||
const safeFilename = path.basename(filename);
|
||||
const matches = files.filter(
|
||||
(candidate) => candidate.filename === safeFilename,
|
||||
);
|
||||
if (matches.length === 0) {
|
||||
return NextResponse.json(
|
||||
{ error: "CSV file not found" },
|
||||
{ status: 404 },
|
||||
);
|
||||
}
|
||||
if (matches.length > 1) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error:
|
||||
"Ambiguous filename matches multiple clients. Specify clientId.",
|
||||
code: "AMBIGUOUS_FILENAME",
|
||||
matchingClients: matches.map((m) => m.clientId),
|
||||
},
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
const file = matches[0];
|
||||
return new NextResponse(streamFile(file), {
|
||||
status: 200,
|
||||
headers: {
|
||||
|
||||
+13
-1
@@ -23,7 +23,19 @@ export async function GET(req: NextRequest) {
|
||||
);
|
||||
}
|
||||
|
||||
const clientId = verified.payload.clientId;
|
||||
const clientId = String(verified.payload.clientId || '')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
if (SDKManager.isClientDeleted(clientId)) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
status: 'error',
|
||||
code: 'CLIENT_REVOKED',
|
||||
error: `CLIENT_REVOKED: Client '${clientId}' has been deleted by administrator.`,
|
||||
},
|
||||
{ status: 403 },
|
||||
);
|
||||
}
|
||||
const registeredClient = await SDKManager.getClientAsync(clientId);
|
||||
if (registeredClient) {
|
||||
if (registeredClient.status === 'revoked') {
|
||||
|
||||
+176
-4
@@ -1,8 +1,55 @@
|
||||
import { SettingsManager } from "@/core/adminSettings/settingsManager";
|
||||
import {
|
||||
SettingsManager,
|
||||
type AdminSettings,
|
||||
} from "@/core/adminSettings/settingsManager";
|
||||
import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
export async function GET() {
|
||||
const SESSION_COOKIE = "datastack_admin_session";
|
||||
|
||||
const ALLOWED_SETTING_KEYS = new Set([
|
||||
"maxEventsPerSec",
|
||||
"maxPayloadSizeKb",
|
||||
"batchFlushThreshold",
|
||||
"batchFlushIntervalMs",
|
||||
"fileRotationMb",
|
||||
"enableIdempotency",
|
||||
"autoPruneDays",
|
||||
"jwtSecret",
|
||||
]);
|
||||
|
||||
function parseBoundedInt(
|
||||
raw: unknown,
|
||||
field: string,
|
||||
min: number,
|
||||
max: number,
|
||||
): number {
|
||||
if (typeof raw !== "number" || !Number.isInteger(raw)) {
|
||||
throw new Error(`${field} must be an integer between ${min} and ${max}.`);
|
||||
}
|
||||
if (raw < min || raw > max) {
|
||||
throw new Error(`${field} must be between ${min} and ${max}.`);
|
||||
}
|
||||
return raw;
|
||||
}
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
try {
|
||||
const auth = await verifyRequestAdminSession(
|
||||
req.cookies.get(SESSION_COOKIE)?.value,
|
||||
);
|
||||
if (!auth.valid) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error:
|
||||
auth.reason === "UNAUTHORIZED"
|
||||
? "Authentication required."
|
||||
: "Admin authentication is temporarily unavailable.",
|
||||
},
|
||||
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
|
||||
);
|
||||
}
|
||||
|
||||
const settings = await SettingsManager.getPublicSettingsAsync();
|
||||
return NextResponse.json({ settings });
|
||||
} catch (err: unknown) {
|
||||
@@ -13,8 +60,133 @@ export async function GET() {
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
try {
|
||||
const body = await req.json();
|
||||
await SettingsManager.updateSettingsAsync(body);
|
||||
const auth = await verifyRequestAdminSession(
|
||||
req.cookies.get(SESSION_COOKIE)?.value,
|
||||
);
|
||||
if (!auth.valid) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error:
|
||||
auth.reason === "UNAUTHORIZED"
|
||||
? "Authentication required."
|
||||
: "Admin authentication is temporarily unavailable.",
|
||||
},
|
||||
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
|
||||
);
|
||||
}
|
||||
|
||||
let rawBody: unknown;
|
||||
try {
|
||||
rawBody = await req.json();
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid JSON request payload.", code: "INVALID_JSON" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
if (!rawBody || typeof rawBody !== "object" || Array.isArray(rawBody)) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: "Settings payload must be a JSON object.",
|
||||
code: "VALIDATION_ERROR",
|
||||
},
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
const body = rawBody as Record<string, unknown>;
|
||||
const unknownKeys = Object.keys(body).filter(
|
||||
(key) => !ALLOWED_SETTING_KEYS.has(key),
|
||||
);
|
||||
if (unknownKeys.length > 0) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: `Unsupported settings field(s): ${unknownKeys.join(", ")}`,
|
||||
code: "VALIDATION_ERROR",
|
||||
},
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
const validated: Partial<AdminSettings> = {};
|
||||
try {
|
||||
if (body.maxEventsPerSec !== undefined) {
|
||||
validated.maxEventsPerSec = parseBoundedInt(
|
||||
body.maxEventsPerSec,
|
||||
"maxEventsPerSec",
|
||||
1,
|
||||
100000,
|
||||
);
|
||||
}
|
||||
if (body.maxPayloadSizeKb !== undefined) {
|
||||
validated.maxPayloadSizeKb = parseBoundedInt(
|
||||
body.maxPayloadSizeKb,
|
||||
"maxPayloadSizeKb",
|
||||
1,
|
||||
10240,
|
||||
);
|
||||
}
|
||||
if (body.batchFlushThreshold !== undefined) {
|
||||
validated.batchFlushThreshold = parseBoundedInt(
|
||||
body.batchFlushThreshold,
|
||||
"batchFlushThreshold",
|
||||
1,
|
||||
50000,
|
||||
);
|
||||
}
|
||||
if (body.batchFlushIntervalMs !== undefined) {
|
||||
validated.batchFlushIntervalMs = parseBoundedInt(
|
||||
body.batchFlushIntervalMs,
|
||||
"batchFlushIntervalMs",
|
||||
100,
|
||||
60000,
|
||||
);
|
||||
}
|
||||
if (body.fileRotationMb !== undefined) {
|
||||
validated.fileRotationMb = parseBoundedInt(
|
||||
body.fileRotationMb,
|
||||
"fileRotationMb",
|
||||
1,
|
||||
2048,
|
||||
);
|
||||
}
|
||||
if (body.autoPruneDays !== undefined) {
|
||||
validated.autoPruneDays = parseBoundedInt(
|
||||
body.autoPruneDays,
|
||||
"autoPruneDays",
|
||||
0,
|
||||
3650,
|
||||
);
|
||||
}
|
||||
if (body.enableIdempotency !== undefined) {
|
||||
if (typeof body.enableIdempotency !== "boolean") {
|
||||
throw new Error("enableIdempotency must be a boolean.");
|
||||
}
|
||||
validated.enableIdempotency = body.enableIdempotency;
|
||||
}
|
||||
if (body.jwtSecret !== undefined) {
|
||||
if (typeof body.jwtSecret !== "string" || body.jwtSecret.trim().length < 32) {
|
||||
throw new Error(
|
||||
"jwtSecret must be a string with at least 32 characters.",
|
||||
);
|
||||
}
|
||||
validated.jwtSecret = body.jwtSecret.trim();
|
||||
}
|
||||
} catch (validationErr: unknown) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error:
|
||||
validationErr instanceof Error
|
||||
? validationErr.message
|
||||
: "Invalid settings value.",
|
||||
code: "VALIDATION_ERROR",
|
||||
},
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
await SettingsManager.updateSettingsAsync(validated);
|
||||
const settings = await SettingsManager.getPublicSettingsAsync();
|
||||
return NextResponse.json({
|
||||
settings,
|
||||
|
||||
+59
-11
@@ -1,4 +1,6 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { SettingsManager } from '@/core/adminSettings/settingsManager';
|
||||
import { getInternalIngestionHeaders } from '@/core/Security/security';
|
||||
import { CSVStorageManager } from '@/core/Storage/csvStorage';
|
||||
import { DataBufferEngine } from '@/core/data/buffer';
|
||||
|
||||
@@ -7,8 +9,11 @@ export async function GET(req: NextRequest) {
|
||||
const { searchParams } = new URL(req.url);
|
||||
const clientId = searchParams.get('clientId') || undefined;
|
||||
|
||||
const stats = CSVStorageManager.getStorageStats();
|
||||
let files = CSVStorageManager.listAllFiles();
|
||||
const [stats, allFiles] = await Promise.all([
|
||||
CSVStorageManager.getStorageStatsAsync(),
|
||||
CSVStorageManager.listAllFilesAsync(),
|
||||
]);
|
||||
let files = allFiles;
|
||||
|
||||
if (clientId && clientId !== 'all') {
|
||||
files = files.filter((f) => f.clientId === clientId);
|
||||
@@ -23,12 +28,17 @@ export async function GET(req: NextRequest) {
|
||||
|
||||
export async function POST(_req: NextRequest) {
|
||||
try {
|
||||
// 1. Flush local Next.js DataBufferEngine
|
||||
await SettingsManager.getSettingsAsync();
|
||||
|
||||
// 1. Flush local / shared Next.js DataBufferEngine
|
||||
const localRes = await DataBufferEngine.flush();
|
||||
|
||||
// 2. Also flush Port 4000 Socket.IO ingestion buffer
|
||||
let remoteCount = 0;
|
||||
let remoteFiles: string[] = [];
|
||||
let ingestionFlushed = false;
|
||||
let warning: string | undefined;
|
||||
const explicitIngestionUrl = Boolean(process.env.INGESTION_INTERNAL_URL?.trim());
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
|
||||
).replace(/\/$/, '');
|
||||
@@ -36,17 +46,53 @@ export async function POST(_req: NextRequest) {
|
||||
try {
|
||||
const ingestRes = await fetch(`${ingestionBaseUrl}/internal/flush`, {
|
||||
method: 'POST',
|
||||
headers: getInternalIngestionHeaders(),
|
||||
cache: 'no-store',
|
||||
signal: AbortSignal.timeout(2500),
|
||||
});
|
||||
if (ingestRes.ok) {
|
||||
const ingestData = await ingestRes.json();
|
||||
remoteCount = Number(ingestData.flushedCount) || 0;
|
||||
if (Array.isArray(ingestData.filesUpdated)) {
|
||||
remoteFiles = ingestData.filesUpdated;
|
||||
}
|
||||
if (!ingestRes.ok) {
|
||||
const errPayload = await ingestRes.json().catch(() => ({}));
|
||||
const remoteErr =
|
||||
typeof errPayload?.error === 'string'
|
||||
? errPayload.error
|
||||
: `Ingestion flush returned HTTP ${ingestRes.status}`;
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: remoteErr,
|
||||
code: 'INGESTION_FLUSH_FAILED',
|
||||
flushedCount: localRes.count,
|
||||
filesUpdated: localRes.files,
|
||||
ingestionFlushed: false,
|
||||
ingestionStatus: ingestRes.status,
|
||||
},
|
||||
{ status: 502 },
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
// Port 4000 server may be offline in standalone Next.js dev mode
|
||||
|
||||
ingestionFlushed = true;
|
||||
const ingestData = await ingestRes.json();
|
||||
remoteCount = Number(ingestData.flushedCount) || 0;
|
||||
if (Array.isArray(ingestData.filesUpdated)) {
|
||||
remoteFiles = ingestData.filesUpdated;
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
const errMsg =
|
||||
err instanceof Error ? err.message : 'Ingestion server unreachable';
|
||||
if (explicitIngestionUrl) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: `Ingestion server flush failed: ${errMsg}`,
|
||||
code: 'INGESTION_UNREACHABLE',
|
||||
flushedCount: localRes.count,
|
||||
filesUpdated: localRes.files,
|
||||
ingestionFlushed: false,
|
||||
},
|
||||
{ status: 502 },
|
||||
);
|
||||
}
|
||||
warning = `Port 4000 ingestion server is unreachable (${errMsg}); flushed Next.js buffer only.`;
|
||||
}
|
||||
|
||||
const combinedFiles = Array.from(
|
||||
@@ -57,6 +103,8 @@ export async function POST(_req: NextRequest) {
|
||||
success: true,
|
||||
flushedCount: localRes.count + remoteCount,
|
||||
filesUpdated: combinedFiles,
|
||||
ingestionFlushed,
|
||||
...(warning ? { warning } : {}),
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : 'Manual flush failed';
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
'use client';
|
||||
|
||||
import { Button } from '@/components/ui/Button';
|
||||
import { AlertTriangle, RotateCcw } from 'lucide-react';
|
||||
import Link from 'next/link';
|
||||
import { useEffect } from 'react';
|
||||
|
||||
export default function RootErrorBoundary({
|
||||
error,
|
||||
reset,
|
||||
}: {
|
||||
error: Error & { digest?: string };
|
||||
reset: () => void;
|
||||
}) {
|
||||
useEffect(() => {
|
||||
console.error('Application error boundary:', error);
|
||||
}, [error]);
|
||||
|
||||
return (
|
||||
<main className="flex min-h-screen items-center justify-center bg-[var(--bg-main)] px-4 py-8 text-[var(--text)]">
|
||||
<section className="w-full max-w-[440px] border border-[var(--border-strong)] bg-[var(--surface)] p-5">
|
||||
<div className="mb-3 flex items-center gap-2 border-b border-[var(--border)] pb-3">
|
||||
<AlertTriangle className="h-4 w-4 shrink-0 text-[var(--danger)]" aria-hidden="true" />
|
||||
<h1 className="text-[13px] font-medium uppercase tracking-wider text-[var(--text)]">
|
||||
Runtime error
|
||||
</h1>
|
||||
</div>
|
||||
<p className="mb-3 text-[12px] leading-relaxed text-[var(--text-muted)]">
|
||||
An unexpected rendering error occurred. You can retry the view or return to the overview page.
|
||||
</p>
|
||||
{error?.message && (
|
||||
<div className="mb-4 border border-[var(--border)] bg-[var(--bg-main)] px-2.5 py-2 font-mono text-[11px] text-[var(--danger)] break-words">
|
||||
{error.message}
|
||||
</div>
|
||||
)}
|
||||
<div className="flex flex-wrap items-center justify-end gap-2 border-t border-[var(--border)] pt-3">
|
||||
<Link href="/">
|
||||
<Button type="button" size="xs" variant="ghost">
|
||||
Return home
|
||||
</Button>
|
||||
</Link>
|
||||
<Button type="button" size="xs" variant="primary" onClick={() => reset()}>
|
||||
<RotateCcw className="h-3 w-3" aria-hidden="true" />
|
||||
Try again
|
||||
</Button>
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
+25
-6
@@ -1,11 +1,31 @@
|
||||
import { PwaLifecycle } from '@/components/pwa/PwaLifecycle';
|
||||
import { ToastProvider } from '@/components/ui/Toast';
|
||||
import type { Metadata } from 'next';
|
||||
import type { Metadata, Viewport } from 'next';
|
||||
import './globals.css';
|
||||
|
||||
export const viewport: Viewport = {
|
||||
themeColor: '#0a0a0b',
|
||||
width: 'device-width',
|
||||
initialScale: 1,
|
||||
};
|
||||
|
||||
export const metadata: Metadata = {
|
||||
title: 'DataStack - Realtime Data Collection Platform',
|
||||
description: 'Self-hosted universal realtime data collection engine with Socket.IO ingestion, Redis buffer, JWT client auth, CSV batch rotation, and /vcon/ admin panel.',
|
||||
manifest: '/manifest.json',
|
||||
appleWebApp: {
|
||||
capable: true,
|
||||
statusBarStyle: 'black-translucent',
|
||||
title: 'DataStack',
|
||||
},
|
||||
icons: {
|
||||
icon: [
|
||||
{ url: '/icon.svg', type: 'image/svg+xml' },
|
||||
{ url: '/icon-192.png', sizes: '192x192', type: 'image/png' },
|
||||
{ url: '/icon-512.png', sizes: '512x512', type: 'image/png' },
|
||||
],
|
||||
apple: [{ url: '/apple-touch-icon.png', sizes: '180x180', type: 'image/png' }],
|
||||
},
|
||||
openGraph: {
|
||||
title: 'DataStack - Realtime Data Collection Platform',
|
||||
description: 'Self-hosted universal realtime data collection engine with Socket.IO ingestion, Redis buffer, JWT client auth, CSV batch rotation, and /vcon/ admin panel.',
|
||||
@@ -21,12 +41,11 @@ export const metadata: Metadata = {
|
||||
export default function RootLayout({ children }: { children: React.ReactNode }) {
|
||||
return (
|
||||
<html lang="en" className="dark">
|
||||
<head>
|
||||
<link rel="manifest" href="/manifest.json" />
|
||||
<meta name="theme-color" content="#0a0a0b" />
|
||||
</head>
|
||||
<body suppressHydrationWarning className="min-h-screen bg-[var(--bg-main)] text-[var(--text)] antialiased">
|
||||
<ToastProvider>{children}</ToastProvider>
|
||||
<ToastProvider>
|
||||
{children}
|
||||
<PwaLifecycle />
|
||||
</ToastProvider>
|
||||
</body>
|
||||
</html>
|
||||
);
|
||||
|
||||
+6
-5
@@ -1,5 +1,5 @@
|
||||
import { MainLayout } from '@/components/layout/MainLayout';
|
||||
import { Button } from '@/components/ui/Button';
|
||||
import { buttonClassNames } from '@/components/ui/Button';
|
||||
import Link from 'next/link';
|
||||
|
||||
export default function NotFound() {
|
||||
@@ -11,10 +11,11 @@ export default function NotFound() {
|
||||
The requested page or route does not exist on DataStack.
|
||||
</p>
|
||||
<div className="flex items-center gap-2 pt-2">
|
||||
<Link href="/">
|
||||
<Button variant="secondary" size="sm">
|
||||
Return Home
|
||||
</Button>
|
||||
<Link href="/" className={buttonClassNames({ variant: 'primary', size: 'sm' })}>
|
||||
Return Home
|
||||
</Link>
|
||||
<Link href="/vcon/dashboard" className={buttonClassNames({ variant: 'secondary', size: 'sm' })}>
|
||||
Open Console
|
||||
</Link>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -23,7 +23,11 @@ export default function ClientsPage() {
|
||||
const [now, setNow] = useState(() => Date.now());
|
||||
|
||||
const fetchClients = useCallback(async () => {
|
||||
const response = await fetch('/api/clients');
|
||||
const response = await fetch('/api/clients', { cache: 'no-store' });
|
||||
if (response.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
if (!response.ok) throw new Error('Unable to load clients');
|
||||
const data = await response.json();
|
||||
setClients(setCached('vcon:clients', data.clients || []));
|
||||
@@ -49,16 +53,19 @@ export default function ClientsPage() {
|
||||
!query ||
|
||||
client.clientId.toLowerCase().includes(query) ||
|
||||
client.name.toLowerCase().includes(query);
|
||||
const revoked = client.status === 'revoked';
|
||||
const online =
|
||||
client.status === 'active' &&
|
||||
Boolean(client.lastActive) &&
|
||||
now - new Date(client.lastActive!).getTime() < 60_000;
|
||||
const pending = client.status === 'active' && !client.lastActive;
|
||||
const offline = client.status === 'active' && !online && !pending;
|
||||
const matchesStatus =
|
||||
statusFilter === 'all' ||
|
||||
(statusFilter === 'online' && online) ||
|
||||
(statusFilter === 'offline' && !online && !pending) ||
|
||||
(statusFilter === 'pending' && pending);
|
||||
(statusFilter === 'offline' && offline) ||
|
||||
(statusFilter === 'pending' && pending) ||
|
||||
(statusFilter === 'revoked' && revoked);
|
||||
return matchesSearch && matchesStatus;
|
||||
});
|
||||
}, [clients, now, search, statusFilter]);
|
||||
@@ -72,7 +79,10 @@ export default function ClientsPage() {
|
||||
const pendingCount = clients.filter(
|
||||
(client) => client.status === 'active' && !client.lastActive
|
||||
).length;
|
||||
const offlineCount = clients.length - onlineCount - pendingCount;
|
||||
const revokedCount = clients.filter(
|
||||
(client) => client.status === 'revoked'
|
||||
).length;
|
||||
const offlineCount = clients.length - onlineCount - pendingCount - revokedCount;
|
||||
|
||||
return (
|
||||
<PageShell
|
||||
@@ -90,11 +100,13 @@ export default function ClientsPage() {
|
||||
}
|
||||
>
|
||||
<StatStrip
|
||||
className="md:grid-cols-5"
|
||||
items={[
|
||||
{ label: 'Total', value: clients.length },
|
||||
{ label: 'Online', value: onlineCount, dot: 'ok' },
|
||||
{ label: 'Offline', value: offlineCount, dot: 'off' },
|
||||
{ label: 'Pending', value: pendingCount, dot: 'pending' },
|
||||
{ label: 'Revoked', value: revokedCount, dot: 'off' },
|
||||
]}
|
||||
/>
|
||||
|
||||
@@ -109,6 +121,7 @@ export default function ClientsPage() {
|
||||
online: onlineCount,
|
||||
offline: offlineCount,
|
||||
pending: pendingCount,
|
||||
revoked: revokedCount,
|
||||
}}
|
||||
/>
|
||||
<ClientsTable
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
'use client';
|
||||
|
||||
import { Button } from '@/components/ui/Button';
|
||||
import { AlertTriangle, RotateCcw } from 'lucide-react';
|
||||
import Link from 'next/link';
|
||||
import { useEffect } from 'react';
|
||||
|
||||
export default function VconErrorBoundary({
|
||||
error,
|
||||
reset,
|
||||
}: {
|
||||
error: Error & { digest?: string };
|
||||
reset: () => void;
|
||||
}) {
|
||||
useEffect(() => {
|
||||
console.error('Admin console error boundary:', error);
|
||||
}, [error]);
|
||||
|
||||
return (
|
||||
<div className="flex min-h-[60vh] items-center justify-center p-4 text-[var(--text)]">
|
||||
<section className="w-full max-w-[440px] border border-[var(--border-strong)] bg-[var(--surface)] p-5">
|
||||
<div className="mb-3 flex items-center gap-2 border-b border-[var(--border)] pb-3">
|
||||
<AlertTriangle className="h-4 w-4 shrink-0 text-[var(--danger)]" aria-hidden="true" />
|
||||
<h1 className="text-[13px] font-medium uppercase tracking-wider text-[var(--text)]">
|
||||
Console panel error
|
||||
</h1>
|
||||
</div>
|
||||
<p className="mb-3 text-[12px] leading-relaxed text-[var(--text-muted)]">
|
||||
This console view encountered an unexpected failure. Retry rendering or navigate back to the dashboard.
|
||||
</p>
|
||||
{error?.message && (
|
||||
<div className="mb-4 border border-[var(--border)] bg-[var(--bg-main)] px-2.5 py-2 font-mono text-[11px] text-[var(--danger)] break-words">
|
||||
{error.message}
|
||||
</div>
|
||||
)}
|
||||
<div className="flex flex-wrap items-center justify-end gap-2 border-t border-[var(--border)] pt-3">
|
||||
<Link href="/vcon/dashboard">
|
||||
<Button type="button" size="xs" variant="ghost">
|
||||
Dashboard
|
||||
</Button>
|
||||
</Link>
|
||||
<Button type="button" size="xs" variant="primary" onClick={() => reset()}>
|
||||
<RotateCcw className="h-3 w-3" aria-hidden="true" />
|
||||
Retry panel
|
||||
</Button>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -6,12 +6,13 @@ export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
export default async function LoginPage() {
|
||||
if (process.env.DATABASE_URL) {
|
||||
const isConfigured = await isAdminSetupComplete().catch(() => false);
|
||||
if (!isConfigured) redirect('/vcon/setup');
|
||||
}
|
||||
const isConfigured = await isAdminSetupComplete().catch(() => false);
|
||||
if (!isConfigured) redirect('/vcon/setup');
|
||||
|
||||
const showDemoHint = !process.env.DATABASE_URL && process.env.NODE_ENV !== 'production';
|
||||
const showDemoHint =
|
||||
!process.env.DATABASE_URL &&
|
||||
process.env.NODE_ENV !== 'production' &&
|
||||
process.env.DATASTACK_ALLOW_DEMO_ADMIN !== 'false';
|
||||
|
||||
return <AdminLogin showDemoHint={showDemoHint} />;
|
||||
}
|
||||
+96
-21
@@ -11,15 +11,25 @@ import { useToast } from '@/components/ui/Toast';
|
||||
import { getCached, setCached } from '@/lib/vconCache';
|
||||
import { useEffect, useState } from 'react';
|
||||
|
||||
interface SdkClientItem {
|
||||
clientId: string;
|
||||
name: string;
|
||||
status?: 'active' | 'revoked';
|
||||
tokenPreview?: string;
|
||||
token?: string;
|
||||
}
|
||||
|
||||
export default function SdkGuidePage() {
|
||||
const { success } = useToast();
|
||||
const [clients, setClients] = useState<any[]>(() =>
|
||||
getCached<any[]>('vcon:clients', [])
|
||||
const { success, error } = useToast();
|
||||
const [clients, setClients] = useState<SdkClientItem[]>(() =>
|
||||
getCached<SdkClientItem[]>('vcon:clients', [])
|
||||
);
|
||||
const [selectedClientId, setSelectedClientId] = useState<string>(() => {
|
||||
const cached = getCached<any[]>('vcon:clients', []);
|
||||
return cached[0]?.clientId ?? '';
|
||||
const cached = getCached<SdkClientItem[]>('vcon:clients', []);
|
||||
return cached.find((c) => c.status !== 'revoked')?.clientId ?? cached[0]?.clientId ?? '';
|
||||
});
|
||||
const [selectedClientToken, setSelectedClientToken] = useState<string>('');
|
||||
const [loadingToken, setLoadingToken] = useState<boolean>(false);
|
||||
const [serverUrl, setServerUrl] = useState<string>('http://localhost:4000');
|
||||
const [apiUrl, setApiUrl] = useState<string>('http://localhost:3000');
|
||||
|
||||
@@ -34,33 +44,81 @@ export default function SdkGuidePage() {
|
||||
|
||||
async function loadClients() {
|
||||
try {
|
||||
const res = await fetch('/api/clients');
|
||||
const res = await fetch('/api/clients', { cache: 'no-store' });
|
||||
if (res.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
const list = setCached('vcon:clients', data.clients || []);
|
||||
const list = setCached<SdkClientItem[]>('vcon:clients', data.clients || []);
|
||||
setClients(list);
|
||||
if (list.length > 0) {
|
||||
setSelectedClientId((prev) =>
|
||||
list.some((c: any) => c.clientId === prev) ? prev : list[0].clientId
|
||||
list.some((c) => c.clientId === prev)
|
||||
? prev
|
||||
: list.find((c) => c.status !== 'revoked')?.clientId ?? list[0].clientId
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
// Retain cached list on transient network error
|
||||
}
|
||||
}
|
||||
loadClients();
|
||||
void loadClients();
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (!selectedClientId) {
|
||||
setSelectedClientToken('');
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
setLoadingToken(true);
|
||||
|
||||
async function fetchToken() {
|
||||
try {
|
||||
const res = await fetch(
|
||||
`/api/clients/${encodeURIComponent(selectedClientId)}?includeToken=true`,
|
||||
{ cache: 'no-store' }
|
||||
);
|
||||
if (res.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
if (!cancelled) {
|
||||
setSelectedClientToken(String(data?.client?.token || ''));
|
||||
}
|
||||
} else if (!cancelled) {
|
||||
setSelectedClientToken('');
|
||||
}
|
||||
} catch {
|
||||
if (!cancelled) setSelectedClientToken('');
|
||||
} finally {
|
||||
if (!cancelled) setLoadingToken(false);
|
||||
}
|
||||
}
|
||||
|
||||
void fetchToken();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [selectedClientId]);
|
||||
|
||||
const activeClient =
|
||||
clients.find((c) => c.clientId === selectedClientId) || clients[0];
|
||||
const sampleToken = activeClient?.token || 'YOUR_SIGNED_CLIENT_JWT_TOKEN';
|
||||
const resolvedToken = selectedClientToken || activeClient?.token || '';
|
||||
const sampleToken = resolvedToken || 'YOUR_SIGNED_CLIENT_JWT_TOKEN';
|
||||
|
||||
const configSnippet = `# config.datastack.py — Place in your Python project root
|
||||
DATASTACK_CONFIG = {
|
||||
"SERVER_URL": "${serverUrl}",
|
||||
"API_URL": "${apiUrl}",
|
||||
"CLIENT_TOKEN": "${sampleToken}",
|
||||
"CLIENT_ID": "${activeClient?.clientId || ''}",
|
||||
"AUTO_CONNECT": True,
|
||||
"ENABLE_HTTP_FALLBACK": True,
|
||||
"HEARTBEAT_INTERVAL_SEC": 5.0,
|
||||
@@ -99,9 +157,21 @@ print("SDK Status:", client.status())
|
||||
|
||||
client.disconnect()`;
|
||||
|
||||
const copySnippet = (text: string, label: string) => {
|
||||
navigator.clipboard.writeText(text);
|
||||
success('Copied', `${label} copied`);
|
||||
const copySnippet = async (text: string, label: string) => {
|
||||
try {
|
||||
if (!navigator?.clipboard?.writeText) {
|
||||
throw new Error('Clipboard API unavailable in this browser context');
|
||||
}
|
||||
await navigator.clipboard.writeText(text);
|
||||
success('Copied', `${label} copied`);
|
||||
} catch (copyErr) {
|
||||
error(
|
||||
'Copy failed',
|
||||
copyErr instanceof Error
|
||||
? copyErr.message
|
||||
: 'Unable to copy to clipboard. Copy manually from the code block.'
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
@@ -109,12 +179,13 @@ client.disconnect()`;
|
||||
title="Python SDK"
|
||||
actions={
|
||||
<>
|
||||
{activeClient?.token && (
|
||||
{resolvedToken && (
|
||||
<Button
|
||||
size="xs"
|
||||
variant="secondary"
|
||||
icon="Key"
|
||||
onClick={() => copySnippet(activeClient.token, 'Client JWT token')}
|
||||
disabled={loadingToken}
|
||||
onClick={() => void copySnippet(resolvedToken, 'Client JWT token')}
|
||||
>
|
||||
Copy Token
|
||||
</Button>
|
||||
@@ -123,7 +194,7 @@ client.disconnect()`;
|
||||
size="xs"
|
||||
variant="secondary"
|
||||
icon="FileCode"
|
||||
onClick={() => copySnippet(configSnippet, 'config.datastack.py')}
|
||||
onClick={() => void copySnippet(configSnippet, 'config.datastack.py')}
|
||||
>
|
||||
Copy Config
|
||||
</Button>
|
||||
@@ -131,7 +202,7 @@ client.disconnect()`;
|
||||
size="xs"
|
||||
variant="secondary"
|
||||
icon="Copy"
|
||||
onClick={() => copySnippet(pythonSnippet, 'Python SDK snippet')}
|
||||
onClick={() => void copySnippet(pythonSnippet, 'Python SDK snippet')}
|
||||
>
|
||||
Copy Code
|
||||
</Button>
|
||||
@@ -161,8 +232,9 @@ client.disconnect()`;
|
||||
size="xs"
|
||||
variant="ghost"
|
||||
icon="Copy"
|
||||
aria-label="Copy pip install command"
|
||||
onClick={() =>
|
||||
copySnippet('pip install ./sdk/python/datastack', 'Pip command')
|
||||
void copySnippet('pip install ./sdk/python/datastack', 'Pip command')
|
||||
}
|
||||
/>
|
||||
</Card>
|
||||
@@ -176,6 +248,7 @@ client.disconnect()`;
|
||||
<div className="w-[190px]">
|
||||
<Select
|
||||
compact
|
||||
aria-label="Select SDK client"
|
||||
value={selectedClientId}
|
||||
onChange={(e) => setSelectedClientId(e.target.value)}
|
||||
>
|
||||
@@ -191,7 +264,8 @@ client.disconnect()`;
|
||||
size="xs"
|
||||
variant="ghost"
|
||||
icon="Copy"
|
||||
onClick={() => copySnippet(configSnippet, 'config.datastack.py')}
|
||||
aria-label="Copy config.datastack.py"
|
||||
onClick={() => void copySnippet(configSnippet, 'config.datastack.py')}
|
||||
/>
|
||||
</div>
|
||||
}
|
||||
@@ -207,7 +281,8 @@ client.disconnect()`;
|
||||
size="xs"
|
||||
variant="ghost"
|
||||
icon="Copy"
|
||||
onClick={() => copySnippet(pythonSnippet, 'Python SDK snippet')}
|
||||
aria-label="Copy Python SDK snippet"
|
||||
onClick={() => void copySnippet(pythonSnippet, 'Python SDK snippet')}
|
||||
/>
|
||||
}
|
||||
>
|
||||
|
||||
+310
-43
@@ -5,20 +5,40 @@ import { Button } from '@/components/ui/Button';
|
||||
import { CodeBlock } from '@/components/ui/CodeBlock';
|
||||
import { Select, Textarea } from '@/components/ui/Input';
|
||||
import { Panel } from '@/components/ui/Panel';
|
||||
import { Pill } from '@/components/ui/Pill';
|
||||
import { StatStrip } from '@/components/ui/StatStrip';
|
||||
import { useToast } from '@/components/ui/Toast';
|
||||
import { getCached, setCached } from '@/lib/vconCache';
|
||||
import Link from 'next/link';
|
||||
import { useEffect, useState } from 'react';
|
||||
import { io } from 'socket.io-client';
|
||||
|
||||
interface SimulatorClient {
|
||||
clientId: string;
|
||||
name: string;
|
||||
status: 'active' | 'revoked';
|
||||
}
|
||||
|
||||
type TransportMode = 'http' | 'websocket';
|
||||
type DispatchStatus = 'Ready' | 'Accepted' | 'Rejected' | 'Error';
|
||||
|
||||
function getSocketOrigin(): string {
|
||||
const isLocalDev = window.location.port === '3000';
|
||||
const protocol = window.location.protocol === 'https:' ? 'wss' : 'ws';
|
||||
const host = isLocalDev ? `${window.location.hostname}:4000` : window.location.host;
|
||||
return `${protocol}://${host}`;
|
||||
}
|
||||
|
||||
export default function SimulatorPage() {
|
||||
const { success, error } = useToast();
|
||||
const [clients, setClients] = useState<any[]>(() =>
|
||||
getCached<any[]>('vcon:clients', [])
|
||||
const { success, warning, error } = useToast();
|
||||
const [clients, setClients] = useState<SimulatorClient[]>(() =>
|
||||
getCached<SimulatorClient[]>('vcon:clients', [])
|
||||
);
|
||||
const [selectedClient, setSelectedClient] = useState<string>(() => {
|
||||
const cached = getCached<any[]>('vcon:clients', []);
|
||||
return cached[0]?.clientId ?? '';
|
||||
const cached = getCached<SimulatorClient[]>('vcon:clients', []);
|
||||
return cached.find((c) => c.status === 'active')?.clientId ?? cached[0]?.clientId ?? '';
|
||||
});
|
||||
const [transport, setTransport] = useState<TransportMode>('http');
|
||||
const [payloadText, setPayloadText] = useState<string>(
|
||||
JSON.stringify(
|
||||
{
|
||||
@@ -33,30 +53,136 @@ export default function SimulatorPage() {
|
||||
)
|
||||
);
|
||||
const [sending, setSending] = useState(false);
|
||||
const [lastResponse, setLastResponse] = useState<any>(null);
|
||||
const [dispatchStatus, setDispatchStatus] = useState<DispatchStatus>('Ready');
|
||||
const [lastResponse, setLastResponse] = useState<Record<string, unknown> | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
async function loadClients() {
|
||||
try {
|
||||
const res = await fetch('/api/clients');
|
||||
const res = await fetch('/api/clients', { cache: 'no-store' });
|
||||
if (res.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
const list = setCached('vcon:clients', data.clients || []);
|
||||
const list = setCached<SimulatorClient[]>('vcon:clients', data.clients || []);
|
||||
setClients(list);
|
||||
if (list.length > 0)
|
||||
setSelectedClient((prev) => prev || list[0].clientId);
|
||||
setSelectedClient((prev) => {
|
||||
if (prev && list.some((c) => c.clientId === prev)) return prev;
|
||||
return list.find((c) => c.status === 'active')?.clientId ?? list[0]?.clientId ?? '';
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
// Ignore transient failure and retain cached list
|
||||
}
|
||||
}
|
||||
loadClients();
|
||||
void loadClients();
|
||||
}, []);
|
||||
|
||||
const fetchClientJwt = async (clientId: string): Promise<string> => {
|
||||
const res = await fetch(`/api/clients/${encodeURIComponent(clientId)}?includeToken=true`, {
|
||||
cache: 'no-store',
|
||||
});
|
||||
if (res.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
throw new Error('Authentication required');
|
||||
}
|
||||
const data = await res.json();
|
||||
if (!res.ok || !data?.client?.token) {
|
||||
throw new Error(data?.error || 'Unable to retrieve client JWT token');
|
||||
}
|
||||
return String(data.client.token);
|
||||
};
|
||||
|
||||
const emitSocketEvents = async (
|
||||
clientId: string,
|
||||
items: Array<Record<string, unknown>>
|
||||
): Promise<{ accepted: number; rejected: number; responses: Record<string, unknown>[] }> => {
|
||||
const token = await fetchClientJwt(clientId);
|
||||
const socketUrl = getSocketOrigin();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const socket = io(socketUrl, {
|
||||
auth: { token },
|
||||
transports: ['websocket', 'polling'],
|
||||
reconnection: false,
|
||||
timeout: 4000,
|
||||
});
|
||||
|
||||
const timer = window.setTimeout(() => {
|
||||
socket.disconnect();
|
||||
reject(new Error('WebSocket handshake timed out (is Port 4000 ingestion server running?)'));
|
||||
}, 4500);
|
||||
|
||||
socket.on('connect_error', (connectErr) => {
|
||||
window.clearTimeout(timer);
|
||||
socket.disconnect();
|
||||
reject(new Error(connectErr.message || 'WebSocket authentication failed'));
|
||||
});
|
||||
|
||||
socket.on('connect', () => {
|
||||
const responses: Record<string, unknown>[] = [];
|
||||
let accepted = 0;
|
||||
let rejected = 0;
|
||||
let settled = 0;
|
||||
|
||||
const finishIfDone = () => {
|
||||
if (settled >= items.length) {
|
||||
window.clearTimeout(timer);
|
||||
socket.disconnect();
|
||||
resolve({ accepted, rejected, responses });
|
||||
}
|
||||
};
|
||||
|
||||
const onAck = (ack: Record<string, unknown>) => {
|
||||
responses.push(ack);
|
||||
if (ack?.status === 'ok' || ack?.status === 'duplicate') {
|
||||
accepted++;
|
||||
} else {
|
||||
rejected++;
|
||||
}
|
||||
settled++;
|
||||
finishIfDone();
|
||||
};
|
||||
|
||||
const onErrorEvent = (errPayload: Record<string, unknown>) => {
|
||||
responses.push(errPayload);
|
||||
rejected++;
|
||||
settled++;
|
||||
finishIfDone();
|
||||
};
|
||||
|
||||
socket.on('ack', onAck);
|
||||
socket.on('error_event', onErrorEvent);
|
||||
|
||||
items.forEach((data, index) => {
|
||||
const eventId = `sim_ws_${Date.now()}_${index}`;
|
||||
socket.emit('telemetry_event', {
|
||||
clientId,
|
||||
eventId,
|
||||
timestamp: new Date().toISOString(),
|
||||
data,
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
const handleSendSingle = async () => {
|
||||
let parsedData: any;
|
||||
if (!selectedClient) {
|
||||
error('No target client', 'Register a client in /vcon/clients first.');
|
||||
return;
|
||||
}
|
||||
|
||||
let parsedData: Record<string, unknown>;
|
||||
try {
|
||||
parsedData = JSON.parse(payloadText);
|
||||
const raw = JSON.parse(payloadText);
|
||||
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
|
||||
error('Invalid payload', 'Payload must be a JSON object.');
|
||||
return;
|
||||
}
|
||||
parsedData = raw as Record<string, unknown>;
|
||||
} catch {
|
||||
error('Invalid JSON', 'Fix payload syntax');
|
||||
return;
|
||||
@@ -64,54 +190,145 @@ export default function SimulatorPage() {
|
||||
|
||||
setSending(true);
|
||||
try {
|
||||
if (transport === 'websocket') {
|
||||
const result = await emitSocketEvents(selectedClient, [parsedData]);
|
||||
const first = result.responses[0] || {};
|
||||
setLastResponse({ transport: 'websocket', ...first });
|
||||
if (result.accepted > 0) {
|
||||
setDispatchStatus('Accepted');
|
||||
success('WebSocket Event Accepted', `${String(first.eventId || selectedClient)} acknowledged`);
|
||||
} else {
|
||||
setDispatchStatus('Rejected');
|
||||
error('WebSocket Event Rejected', String(first.message || first.error || 'Rejected by Port 4000'));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const res = await fetch('/api/events', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
clientId: selectedClient || 'py_telemetry_agent_01',
|
||||
clientId: selectedClient,
|
||||
data: parsedData,
|
||||
}),
|
||||
});
|
||||
if (res.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
setLastResponse(data);
|
||||
setLastResponse({ transport: 'http', status: res.status, ...data });
|
||||
|
||||
if (res.ok) {
|
||||
success('Event Accepted', `${data.event.eventId} buffered`);
|
||||
setDispatchStatus('Accepted');
|
||||
success('Event Accepted', `${data.event?.eventId || selectedClient} buffered`);
|
||||
} else {
|
||||
setDispatchStatus('Rejected');
|
||||
error('Event Rejected', data.error || 'Server rejected');
|
||||
}
|
||||
} catch (err: any) {
|
||||
error('Error', err.message);
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : 'Dispatch error';
|
||||
setDispatchStatus('Error');
|
||||
setLastResponse({ transport, error: message });
|
||||
error('Dispatch failed', message);
|
||||
} finally {
|
||||
setSending(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleSendBatch = async (count = 10) => {
|
||||
if (!selectedClient) {
|
||||
error('No target client', 'Register a client in /vcon/clients first.');
|
||||
return;
|
||||
}
|
||||
|
||||
const items = Array.from({ length: count }, (_, i) => ({
|
||||
batch_index: i + 1,
|
||||
sensor: `sensor_${(i % 3) + 1}`,
|
||||
reading: +(Math.random() * 100).toFixed(2),
|
||||
timestamp: new Date().toISOString(),
|
||||
}));
|
||||
|
||||
setSending(true);
|
||||
try {
|
||||
if (transport === 'websocket') {
|
||||
const result = await emitSocketEvents(selectedClient, items);
|
||||
setLastResponse({
|
||||
transport: 'websocket',
|
||||
batchSent: count,
|
||||
accepted: result.accepted,
|
||||
rejected: result.rejected,
|
||||
sample: result.responses.slice(0, 3),
|
||||
});
|
||||
if (result.accepted === 0) {
|
||||
setDispatchStatus('Rejected');
|
||||
error('WebSocket Batch Rejected', `0/${count} events ingested`);
|
||||
} else if (result.accepted < count) {
|
||||
setDispatchStatus('Rejected');
|
||||
warning('WebSocket Batch Partial', `${result.accepted}/${count} ingested (${result.rejected} rejected)`);
|
||||
} else {
|
||||
setDispatchStatus('Accepted');
|
||||
success('WebSocket Batch Streamed', `${result.accepted}/${count} ingested`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let accepted = 0;
|
||||
for (let i = 0; i < count; i++) {
|
||||
let rejected = 0;
|
||||
const errors: string[] = [];
|
||||
|
||||
for (const item of items) {
|
||||
const res = await fetch('/api/events', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
clientId: selectedClient || 'py_telemetry_agent_01',
|
||||
data: {
|
||||
batch_index: i + 1,
|
||||
sensor: `sensor_${(i % 3) + 1}`,
|
||||
reading: +(Math.random() * 100).toFixed(2),
|
||||
timestamp: new Date().toISOString(),
|
||||
},
|
||||
clientId: selectedClient,
|
||||
data: item,
|
||||
}),
|
||||
});
|
||||
if (res.ok) accepted++;
|
||||
if (res.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
if (res.ok) {
|
||||
accepted++;
|
||||
} else {
|
||||
rejected++;
|
||||
try {
|
||||
const errJson = await res.json();
|
||||
if (errJson?.error && errors.length < 3) {
|
||||
errors.push(String(errJson.error));
|
||||
}
|
||||
} catch {
|
||||
// ignore non-json error body
|
||||
}
|
||||
}
|
||||
}
|
||||
success('Batch Streamed', `${accepted}/${count} ingested`);
|
||||
setLastResponse({ success: true, batchSent: count, accepted });
|
||||
} catch (err: any) {
|
||||
error('Batch Error', err.message);
|
||||
|
||||
setLastResponse({
|
||||
transport: 'http',
|
||||
batchSent: count,
|
||||
accepted,
|
||||
rejected,
|
||||
...(errors.length > 0 ? { errors } : {}),
|
||||
});
|
||||
|
||||
if (accepted === 0) {
|
||||
setDispatchStatus('Rejected');
|
||||
error('Batch Rejected', `0/${count} ingested${errors[0] ? ` (${errors[0]})` : ''}`);
|
||||
} else if (accepted < count) {
|
||||
setDispatchStatus('Rejected');
|
||||
warning('Batch Partial', `${accepted}/${count} ingested (${rejected} rejected)`);
|
||||
} else {
|
||||
setDispatchStatus('Accepted');
|
||||
success('Batch Streamed', `${accepted}/${count} ingested`);
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : 'Batch dispatch error';
|
||||
setDispatchStatus('Error');
|
||||
setLastResponse({ transport, error: message });
|
||||
error('Batch Error', message);
|
||||
} finally {
|
||||
setSending(false);
|
||||
}
|
||||
@@ -126,7 +343,8 @@ export default function SimulatorPage() {
|
||||
size="xs"
|
||||
variant="secondary"
|
||||
loading={sending}
|
||||
onClick={() => handleSendBatch(10)}
|
||||
disabled={sending || !selectedClient}
|
||||
onClick={() => void handleSendBatch(10)}
|
||||
>
|
||||
Burst 10
|
||||
</Button>
|
||||
@@ -135,44 +353,93 @@ export default function SimulatorPage() {
|
||||
variant="primary"
|
||||
icon="Send"
|
||||
loading={sending}
|
||||
onClick={handleSendSingle}
|
||||
disabled={sending || !selectedClient}
|
||||
onClick={() => void handleSendSingle()}
|
||||
>
|
||||
Send Event
|
||||
</Button>
|
||||
</>
|
||||
}
|
||||
>
|
||||
{clients.length === 0 && (
|
||||
<div
|
||||
role="alert"
|
||||
className="mb-3 flex flex-wrap items-center justify-between gap-2 border border-[var(--warning)] bg-[var(--warning-soft)] px-3 py-2 text-[11.5px] text-[var(--text)]"
|
||||
>
|
||||
<span>No registered SDK clients found. Register a client before dispatching events.</span>
|
||||
<Link href="/vcon/clients" className="font-medium text-[var(--accent)] underline">
|
||||
Go to Clients →
|
||||
</Link>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<StatStrip
|
||||
items={[
|
||||
{
|
||||
label: 'Target client',
|
||||
value: selectedClient || 'py_telemetry_agent_01',
|
||||
dot: 'ok',
|
||||
value: selectedClient || 'None selected',
|
||||
dot: selectedClient ? 'ok' : 'off',
|
||||
},
|
||||
{
|
||||
label: 'Ingest transport',
|
||||
value: transport === 'http' ? 'POST /api/events' : 'WS Port 4000',
|
||||
},
|
||||
{ label: 'Ingest route', value: 'POST /api/events' },
|
||||
{ label: 'Payload format', value: 'JSON' },
|
||||
{
|
||||
label: 'Last dispatch',
|
||||
value: lastResponse ? 'Accepted' : 'Ready',
|
||||
value: dispatchStatus,
|
||||
dot:
|
||||
dispatchStatus === 'Accepted'
|
||||
? 'ok'
|
||||
: dispatchStatus === 'Ready'
|
||||
? undefined
|
||||
: 'off',
|
||||
},
|
||||
]}
|
||||
/>
|
||||
|
||||
<div className="grid grid-cols-1 gap-3 lg:grid-cols-2">
|
||||
<Panel title="Payload" bodyClassName="flex flex-col gap-2.5 p-3">
|
||||
<div className="flex flex-wrap items-center justify-between gap-2">
|
||||
<span className="text-[10px] uppercase tracking-wider text-[var(--text-faint)]">
|
||||
Transport
|
||||
</span>
|
||||
<div className="flex gap-1" role="group" aria-label="Ingest transport">
|
||||
<Pill
|
||||
active={transport === 'http'}
|
||||
onClick={() => setTransport('http')}
|
||||
>
|
||||
HTTP /api/events
|
||||
</Pill>
|
||||
<Pill
|
||||
active={transport === 'websocket'}
|
||||
onClick={() => setTransport('websocket')}
|
||||
>
|
||||
WebSocket Port 4000
|
||||
</Pill>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<Select
|
||||
compact
|
||||
label="Target client"
|
||||
value={selectedClient}
|
||||
disabled={clients.length === 0}
|
||||
onChange={(e) => setSelectedClient(e.target.value)}
|
||||
>
|
||||
{clients.map((c) => (
|
||||
<option key={c.clientId} value={c.clientId}>
|
||||
{c.clientId}
|
||||
</option>
|
||||
))}
|
||||
{clients.length === 0 ? (
|
||||
<option value="">No registered clients</option>
|
||||
) : (
|
||||
clients.map((c) => (
|
||||
<option key={c.clientId} value={c.clientId}>
|
||||
{c.clientId} ({c.status})
|
||||
</option>
|
||||
))
|
||||
)}
|
||||
</Select>
|
||||
|
||||
<Textarea
|
||||
label="Event JSON"
|
||||
rows={7}
|
||||
value={payloadText}
|
||||
onChange={(e) => setPayloadText(e.target.value)}
|
||||
|
||||
@@ -12,9 +12,9 @@ export const Footer: React.FC<FooterProps> = ({ isAdmin = false }) => {
|
||||
<footer className="shrink-0 min-h-9 bg-[var(--bg-activity)] border-t border-[var(--border)] px-4 sm:px-6 py-2 flex flex-wrap items-center justify-between gap-2 text-[11px] text-[var(--text-dim)] select-none">
|
||||
<span>DataStack</span>
|
||||
<nav aria-label="Footer navigation" className="flex items-center gap-5">
|
||||
<Link href="#features" className="hover:text-[var(--text-muted)]">Features</Link>
|
||||
<Link href="#sdk" className="hover:text-[var(--text-muted)]">SDK</Link>
|
||||
<Link href="#about" className="hover:text-[var(--text-muted)]">About</Link>
|
||||
<Link href="/#features" className="hover:text-[var(--text-muted)]">Features</Link>
|
||||
<Link href="/#sdk" className="hover:text-[var(--text-muted)]">SDK</Link>
|
||||
<Link href="/#about" className="hover:text-[var(--text-muted)]">About</Link>
|
||||
</nav>
|
||||
</footer>
|
||||
);
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
'use client';
|
||||
|
||||
import { Button } from '@/components/ui/Button';
|
||||
import { PWAInstallButton } from '@/components/pwa/PwaLifecycle';
|
||||
import { Button, buttonClassNames } from '@/components/ui/Button';
|
||||
import Link from 'next/link';
|
||||
import { usePathname } from 'next/navigation';
|
||||
import React, { useEffect, useState } from 'react';
|
||||
@@ -95,12 +96,14 @@ export const Navbar: React.FC<NavbarProps> = ({
|
||||
</div>
|
||||
|
||||
<nav className="hidden sm:flex items-center gap-5 text-[12px]">
|
||||
<Link href="#features" className="text-[var(--text-muted)] hover:text-[var(--text)] whitespace-nowrap shrink-0">Features</Link>
|
||||
<Link href="#sdk" className="text-[var(--text-muted)] hover:text-[var(--text)] whitespace-nowrap shrink-0">SDK</Link>
|
||||
<Link href="#about" className="text-[var(--text-muted)] hover:text-[var(--text)] whitespace-nowrap shrink-0">About</Link>
|
||||
<Link href="/#features" className="text-[var(--text-muted)] hover:text-[var(--text)] whitespace-nowrap shrink-0">Features</Link>
|
||||
<Link href="/#sdk" className="text-[var(--text-muted)] hover:text-[var(--text)] whitespace-nowrap shrink-0">SDK</Link>
|
||||
<Link href="/#about" className="text-[var(--text-muted)] hover:text-[var(--text)] whitespace-nowrap shrink-0">About</Link>
|
||||
</nav>
|
||||
|
||||
<div className="flex items-center gap-2 shrink-0">
|
||||
<PWAInstallButton />
|
||||
|
||||
<Button
|
||||
type="button"
|
||||
variant="icon"
|
||||
@@ -112,10 +115,11 @@ export const Navbar: React.FC<NavbarProps> = ({
|
||||
className="h-7 w-7 border border-[var(--border)] bg-[var(--bg-elevated)]"
|
||||
/>
|
||||
|
||||
<Link href="/vcon/dashboard">
|
||||
<Button size="xs" variant="primary">
|
||||
Console
|
||||
</Button>
|
||||
<Link
|
||||
href="/vcon/dashboard"
|
||||
className={buttonClassNames({ size: 'xs', variant: 'primary' })}
|
||||
>
|
||||
Console
|
||||
</Link>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
'use client';
|
||||
|
||||
import { Button } from '@/components/ui/Button';
|
||||
import { Icon } from '@/components/ui/Icon';
|
||||
import React, { useEffect, useState } from 'react';
|
||||
|
||||
interface BeforeInstallPromptEvent extends Event {
|
||||
prompt: () => Promise<void>;
|
||||
userChoice: Promise<{ outcome: 'accepted' | 'dismissed'; platform: string }>;
|
||||
}
|
||||
|
||||
export function usePWAInstall() {
|
||||
const [deferredPrompt, setDeferredPrompt] = useState<BeforeInstallPromptEvent | null>(null);
|
||||
const [isInstalled, setIsInstalled] = useState(false);
|
||||
const [isIOS, setIsIOS] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const isStandalone =
|
||||
window.matchMedia('(display-mode: standalone)').matches ||
|
||||
(window.navigator as unknown as { standalone?: boolean }).standalone === true;
|
||||
setIsInstalled(isStandalone);
|
||||
|
||||
const ua = window.navigator.userAgent.toLowerCase();
|
||||
setIsIOS(/iphone|ipad|ipod/.test(ua));
|
||||
|
||||
const handleBeforeInstallPrompt = (e: Event) => {
|
||||
e.preventDefault();
|
||||
setDeferredPrompt(e as BeforeInstallPromptEvent);
|
||||
};
|
||||
|
||||
const handleAppInstalled = () => {
|
||||
setIsInstalled(true);
|
||||
setDeferredPrompt(null);
|
||||
};
|
||||
|
||||
window.addEventListener('beforeinstallprompt', handleBeforeInstallPrompt);
|
||||
window.addEventListener('appinstalled', handleAppInstalled);
|
||||
|
||||
return () => {
|
||||
window.removeEventListener('beforeinstallprompt', handleBeforeInstallPrompt);
|
||||
window.removeEventListener('appinstalled', handleAppInstalled);
|
||||
};
|
||||
}, []);
|
||||
|
||||
const install = async () => {
|
||||
if (!deferredPrompt) return false;
|
||||
await deferredPrompt.prompt();
|
||||
const { outcome } = await deferredPrompt.userChoice;
|
||||
if (outcome === 'accepted') {
|
||||
setIsInstalled(true);
|
||||
setDeferredPrompt(null);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
return {
|
||||
isInstallable: Boolean(deferredPrompt),
|
||||
isInstalled,
|
||||
isIOS,
|
||||
install,
|
||||
};
|
||||
}
|
||||
|
||||
export const PWAInstallButton: React.FC = () => {
|
||||
const { isInstallable, isInstalled, isIOS, install } = usePWAInstall();
|
||||
const [showIOSGuide, setShowIOSGuide] = useState(false);
|
||||
|
||||
if (isInstalled) return null;
|
||||
|
||||
if (isInstallable) {
|
||||
return (
|
||||
<Button
|
||||
type="button"
|
||||
size="xs"
|
||||
variant="secondary"
|
||||
icon="Download"
|
||||
onClick={() => void install()}
|
||||
>
|
||||
Install
|
||||
</Button>
|
||||
);
|
||||
}
|
||||
|
||||
if (isIOS) {
|
||||
return (
|
||||
<>
|
||||
<Button
|
||||
type="button"
|
||||
size="xs"
|
||||
variant="secondary"
|
||||
icon="Download"
|
||||
onClick={() => setShowIOSGuide(true)}
|
||||
>
|
||||
Install
|
||||
</Button>
|
||||
|
||||
{showIOSGuide && (
|
||||
<div className="fixed inset-0 z-50 flex items-center justify-center bg-black/60 p-4">
|
||||
<div className="w-full max-w-sm border border-[var(--border-strong)] bg-[var(--surface)] p-5">
|
||||
<div className="mb-2 flex items-center justify-between gap-2">
|
||||
<h3 className="text-[13px] font-medium text-[var(--text)]">
|
||||
Install DataStack on iOS
|
||||
</h3>
|
||||
<button
|
||||
type="button"
|
||||
aria-label="Close install instructions"
|
||||
onClick={() => setShowIOSGuide(false)}
|
||||
className="text-[var(--text-muted)] hover:text-[var(--text)]"
|
||||
>
|
||||
<Icon name="X" size={14} />
|
||||
</button>
|
||||
</div>
|
||||
<p className="text-[12px] leading-relaxed text-[var(--text-muted)]">
|
||||
1. Tap the <strong className="text-[var(--text)]">Share</strong> icon in Safari.
|
||||
<br />
|
||||
2. Select <strong className="text-[var(--text)]">Add to Home Screen</strong>.
|
||||
</p>
|
||||
<div className="mt-4 flex justify-end">
|
||||
<Button
|
||||
type="button"
|
||||
size="xs"
|
||||
variant="primary"
|
||||
onClick={() => setShowIOSGuide(false)}
|
||||
>
|
||||
Done
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
|
||||
export const PwaLifecycle: React.FC = () => {
|
||||
const [isOnline, setIsOnline] = useState(true);
|
||||
|
||||
useEffect(() => {
|
||||
setIsOnline(typeof navigator !== 'undefined' ? navigator.onLine : true);
|
||||
|
||||
const onOnline = () => setIsOnline(true);
|
||||
const onOffline = () => setIsOnline(false);
|
||||
window.addEventListener('online', onOnline);
|
||||
window.addEventListener('offline', onOffline);
|
||||
|
||||
if (typeof navigator !== 'undefined' && 'serviceWorker' in navigator) {
|
||||
navigator.serviceWorker.register('/sw.js', { scope: '/' }).catch(() => {
|
||||
// Non-fatal in environments without SW origin permissions
|
||||
});
|
||||
}
|
||||
|
||||
return () => {
|
||||
window.removeEventListener('online', onOnline);
|
||||
window.removeEventListener('offline', onOffline);
|
||||
};
|
||||
}, []);
|
||||
|
||||
if (isOnline) return null;
|
||||
|
||||
return (
|
||||
<div
|
||||
role="status"
|
||||
className="fixed bottom-4 left-4 z-50 flex items-center gap-2 border border-[var(--warning)] bg-[var(--surface)] px-3 py-1.5 text-[11px] font-mono text-[var(--text)]"
|
||||
>
|
||||
<Icon name="WifiOff" size={12} className="text-[var(--warning)]" />
|
||||
<span>Offline mode — cached shell active</span>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default PwaLifecycle;
|
||||
@@ -19,8 +19,8 @@ export function AdminLogin({
|
||||
}: {
|
||||
showDemoHint?: boolean;
|
||||
}) {
|
||||
const [email, setEmail] = useState(showDemoHint ? 'admin@datastack.dev' : '');
|
||||
const [password, setPassword] = useState(showDemoHint ? 'admin12345678' : '');
|
||||
const [email, setEmail] = useState('');
|
||||
const [password, setPassword] = useState('');
|
||||
const [remember, setRemember] = useState(true);
|
||||
const [showPassword, setShowPassword] = useState(false);
|
||||
const [isSubmitting, setIsSubmitting] = useState(false);
|
||||
|
||||
@@ -17,6 +17,7 @@ const FILTERS: { key: ClientStatusFilter; label: string }[] = [
|
||||
{ key: 'online', label: 'Online' },
|
||||
{ key: 'offline', label: 'Offline' },
|
||||
{ key: 'pending', label: 'Pending' },
|
||||
{ key: 'revoked', label: 'Revoked' },
|
||||
];
|
||||
|
||||
export function ClientsFilters({
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
import { Button } from '@/components/ui/Button';
|
||||
import { CopyButton } from '@/components/ui/CopyButton';
|
||||
import { Textarea } from '@/components/ui/Input';
|
||||
import { Input, Textarea } from '@/components/ui/Input';
|
||||
import { Modal } from '@/components/ui/Modal';
|
||||
import {
|
||||
Table,
|
||||
@@ -43,8 +43,61 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
||||
type: 'rotate' | 'revoke' | 'delete';
|
||||
client: ClientItem;
|
||||
} | null>(null);
|
||||
const [editClient, setEditClient] = useState<ClientItem | null>(null);
|
||||
const [editName, setEditName] = useState('');
|
||||
const [editRateLimit, setEditRateLimit] = useState('100');
|
||||
const [editError, setEditError] = useState('');
|
||||
const [processing, setProcessing] = useState(false);
|
||||
|
||||
const openEditModal = (client: ClientItem) => {
|
||||
setEditClient(client);
|
||||
setEditName(client.name);
|
||||
setEditRateLimit(String(client.rateLimit || 100));
|
||||
setEditError('');
|
||||
};
|
||||
|
||||
const executeEditClient = async (event: React.FormEvent<HTMLFormElement>) => {
|
||||
event.preventDefault();
|
||||
if (!editClient) return;
|
||||
|
||||
const trimmedName = editName.trim();
|
||||
const numericRateLimit = Number(editRateLimit);
|
||||
if (!trimmedName) {
|
||||
setEditError('Display name is required.');
|
||||
return;
|
||||
}
|
||||
if (!Number.isFinite(numericRateLimit) || numericRateLimit < 1 || numericRateLimit > 10000) {
|
||||
setEditError('Rate limit must be between 1 and 10,000 events/s.');
|
||||
return;
|
||||
}
|
||||
|
||||
setEditError('');
|
||||
setProcessing(true);
|
||||
try {
|
||||
const response = await fetch(`/api/clients/${encodeURIComponent(editClient.clientId)}`, {
|
||||
method: 'PATCH',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
name: trimmedName,
|
||||
rateLimit: Math.floor(numericRateLimit),
|
||||
}),
|
||||
});
|
||||
if (response.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
const data = await response.json();
|
||||
if (!response.ok) throw new Error(data.error || 'Unable to update client');
|
||||
success('Client updated', `${editClient.clientId} settings saved.`);
|
||||
setEditClient(null);
|
||||
await onStatusChange();
|
||||
} catch (updateError) {
|
||||
error('Client update failed', updateError instanceof Error ? updateError.message : 'Unknown error');
|
||||
} finally {
|
||||
setProcessing(false);
|
||||
}
|
||||
};
|
||||
|
||||
const executeRotateKey = async (client: ClientItem) => {
|
||||
setProcessing(true);
|
||||
try {
|
||||
@@ -53,6 +106,10 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ action: 'regenerate_token' }),
|
||||
});
|
||||
if (response.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
const data = await response.json();
|
||||
if (!response.ok) throw new Error(data.error || 'Unable to rotate credentials');
|
||||
setConfirmAction(null);
|
||||
@@ -74,6 +131,10 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ status }),
|
||||
});
|
||||
if (response.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
const data = await response.json();
|
||||
if (!response.ok) throw new Error(data.error || 'Unable to update client status');
|
||||
setConfirmAction(null);
|
||||
@@ -92,6 +153,10 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
||||
const response = await fetch(`/api/clients/${encodeURIComponent(client.clientId)}`, {
|
||||
method: 'DELETE',
|
||||
});
|
||||
if (response.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
const data = await response.json();
|
||||
if (!response.ok) throw new Error(data.error || 'Unable to delete client');
|
||||
setConfirmAction(null);
|
||||
@@ -130,7 +195,7 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>Client</TableHead>
|
||||
<TableHead>API key</TableHead>
|
||||
<TableHead>Credentials</TableHead>
|
||||
<TableHead className="text-right">Events</TableHead>
|
||||
<TableHead>Last seen</TableHead>
|
||||
<TableHead className="text-right">Actions</TableHead>
|
||||
@@ -141,15 +206,18 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
||||
<TableEmpty colSpan={5} message="No matching clients" />
|
||||
) : (
|
||||
clients.map((client) => {
|
||||
const revoked = client.status === 'revoked';
|
||||
const online = client.status === 'active' && Boolean(client.lastActive) &&
|
||||
now - new Date(client.lastActive!).getTime() < 60_000;
|
||||
const pending = client.status === 'active' && !client.lastActive;
|
||||
const label = pending ? 'pending' : online ? 'online' : 'offline';
|
||||
const dotClass = online
|
||||
? 'bg-[var(--accent)]'
|
||||
: pending
|
||||
? 'bg-[var(--warning)]'
|
||||
: 'bg-[var(--text-faint)]';
|
||||
const label = revoked ? 'revoked' : pending ? 'pending' : online ? 'online' : 'offline';
|
||||
const dotClass = revoked
|
||||
? 'bg-[var(--danger)]'
|
||||
: online
|
||||
? 'bg-[var(--accent)]'
|
||||
: pending
|
||||
? 'bg-[var(--warning)]'
|
||||
: 'bg-[var(--text-faint)]';
|
||||
|
||||
return (
|
||||
<TableRow key={client.clientId}>
|
||||
@@ -157,13 +225,25 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
||||
<div className="flex min-w-0 items-center gap-2">
|
||||
<span className={`h-1.5 w-1.5 shrink-0 ${dotClass}`} title={label} />
|
||||
<div className="min-w-0 leading-tight">
|
||||
<div className="truncate font-medium text-[var(--text)]">{client.name}</div>
|
||||
<div className="flex items-center gap-1.5">
|
||||
<span className="truncate font-medium text-[var(--text)]">{client.name}</span>
|
||||
{revoked && (
|
||||
<span className="border border-[var(--danger)] bg-[var(--danger-soft)] px-1 py-0 text-[9px] uppercase tracking-wider text-[var(--danger)]">
|
||||
revoked
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="mt-0.5 truncate font-mono text-[10.5px] text-[var(--text-faint)]">{client.clientId}</div>
|
||||
</div>
|
||||
</div>
|
||||
</TableCell>
|
||||
<TableCell className="font-mono">
|
||||
sec_••••••••
|
||||
<div className="truncate text-[11px] text-[var(--text)]">
|
||||
{client.tokenPreview || 'JWT Active'}
|
||||
</div>
|
||||
<div className="mt-0.5 text-[10px] text-[var(--text-faint)]">
|
||||
{client.rateLimit}/s limit
|
||||
</div>
|
||||
</TableCell>
|
||||
<TableCell className="text-right font-mono tabular-nums">
|
||||
{client.totalEvents.toLocaleString()}
|
||||
@@ -184,6 +264,16 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
||||
aria-label={`View events for ${client.clientId}`}
|
||||
onClick={() => router.push(`/vcon/data?clientId=${encodeURIComponent(client.clientId)}`)}
|
||||
/>
|
||||
<Button
|
||||
type="button"
|
||||
variant="icon"
|
||||
size="icon"
|
||||
icon="Pencil"
|
||||
className="h-6 w-6 border border-transparent hover:border-[var(--border)] hover:bg-[var(--surface-2)]"
|
||||
title={`Edit ${client.clientId}`}
|
||||
aria-label={`Edit ${client.clientId}`}
|
||||
onClick={() => openEditModal(client)}
|
||||
/>
|
||||
<Button
|
||||
type="button"
|
||||
variant="icon"
|
||||
@@ -224,6 +314,55 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
||||
</Table>
|
||||
</TableWrap>
|
||||
|
||||
<Modal
|
||||
isOpen={Boolean(editClient)}
|
||||
onClose={() => !processing && setEditClient(null)}
|
||||
title={`Edit client: ${editClient?.clientId || ''}`}
|
||||
maxWidth="xs"
|
||||
footer={
|
||||
<>
|
||||
<Button type="button" variant="ghost" size="xs" disabled={processing} onClick={() => setEditClient(null)}>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button
|
||||
type="submit"
|
||||
form="edit-client-form"
|
||||
variant="primary"
|
||||
size="xs"
|
||||
loading={processing}
|
||||
>
|
||||
Save changes
|
||||
</Button>
|
||||
</>
|
||||
}
|
||||
>
|
||||
<form id="edit-client-form" onSubmit={executeEditClient} className="flex flex-col gap-2.5">
|
||||
<Input
|
||||
label="Display name"
|
||||
compact
|
||||
value={editName}
|
||||
onChange={(event) => setEditName(event.target.value)}
|
||||
placeholder="Service display name"
|
||||
required
|
||||
/>
|
||||
<Input
|
||||
label="Rate limit (events/s)"
|
||||
compact
|
||||
type="number"
|
||||
min="1"
|
||||
max="10000"
|
||||
value={editRateLimit}
|
||||
onChange={(event) => setEditRateLimit(event.target.value)}
|
||||
required
|
||||
/>
|
||||
{editError && (
|
||||
<p role="alert" className="text-[11px] text-[var(--danger)]">
|
||||
{editError}
|
||||
</p>
|
||||
)}
|
||||
</form>
|
||||
</Modal>
|
||||
|
||||
<Modal
|
||||
isOpen={Boolean(confirmAction)}
|
||||
onClose={() => !processing && setConfirmAction(null)}
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
Table,
|
||||
TableBody,
|
||||
TableCell,
|
||||
TableEmpty,
|
||||
TableHead,
|
||||
TableHeader,
|
||||
TableRow,
|
||||
@@ -26,6 +27,7 @@ type EventItem = {
|
||||
};
|
||||
|
||||
type DashboardMetrics = {
|
||||
activeSockets?: number;
|
||||
activeClients: number;
|
||||
totalClients: number;
|
||||
bufferStats: {
|
||||
@@ -62,6 +64,7 @@ export const DashboardOverview: React.FC = () => {
|
||||
getCached<StorageFile[]>('vcon:storage-files', [])
|
||||
);
|
||||
const [isRefreshing, setIsRefreshing] = useState(false);
|
||||
const [fetchError, setFetchError] = useState('');
|
||||
|
||||
const refresh = useCallback(async () => {
|
||||
setIsRefreshing(true);
|
||||
@@ -74,25 +77,58 @@ export const DashboardOverview: React.FC = () => {
|
||||
fetch('/api/clients', { cache: 'no-store' }),
|
||||
]);
|
||||
|
||||
if (
|
||||
metricsResponse.status === 401 ||
|
||||
eventsResponse.status === 401 ||
|
||||
storageResponse.status === 401 ||
|
||||
clientsResponse.status === 401
|
||||
) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
|
||||
const errors: string[] = [];
|
||||
|
||||
if (metricsResponse.ok) {
|
||||
const nextMetrics = await metricsResponse.json();
|
||||
setMetrics(setCached('vcon:metrics', nextMetrics));
|
||||
} else {
|
||||
errors.push('metrics');
|
||||
}
|
||||
|
||||
if (eventsResponse.ok) {
|
||||
const data = await eventsResponse.json();
|
||||
setEvents(setCached('vcon:dashboard-events', data.events ?? []));
|
||||
} else {
|
||||
errors.push('events');
|
||||
}
|
||||
|
||||
if (storageResponse.ok) {
|
||||
const data = await storageResponse.json();
|
||||
setFiles(setCached('vcon:storage-files', data.files ?? []));
|
||||
if (data.stats) setCached('vcon:storage-stats', data.stats);
|
||||
} else {
|
||||
errors.push('storage');
|
||||
}
|
||||
|
||||
if (clientsResponse.ok) {
|
||||
const data = await clientsResponse.json();
|
||||
setClients(setCached('vcon:clients', data.clients ?? []));
|
||||
} else {
|
||||
errors.push('clients');
|
||||
}
|
||||
|
||||
if (errors.length > 0) {
|
||||
setFetchError(
|
||||
`Partial telemetry refresh failed (${errors.join(', ')}). Showing last available snapshot.`
|
||||
);
|
||||
} else {
|
||||
setFetchError('');
|
||||
}
|
||||
} catch {
|
||||
// Keep the last successful dashboard snapshot when one request fails.
|
||||
setFetchError(
|
||||
'Unable to reach telemetry endpoints. Showing last available snapshot.'
|
||||
);
|
||||
} finally {
|
||||
setIsRefreshing(false);
|
||||
}
|
||||
@@ -108,18 +144,21 @@ export const DashboardOverview: React.FC = () => {
|
||||
const storedRecordsToday = files
|
||||
.filter((file) => file.date === today)
|
||||
.reduce((total, file) => total + file.linesCount, 0);
|
||||
const eventsToday =
|
||||
storedRecordsToday + (metrics?.bufferStats.currentBufferSize ?? 0);
|
||||
const bufferedCount = metrics?.bufferStats.currentBufferSize ?? 0;
|
||||
const eventsToday = storedRecordsToday + bufferedCount;
|
||||
const activeClients =
|
||||
metrics?.activeClients ??
|
||||
clients.filter((client) => client.status === 'active').length;
|
||||
const totalClients = metrics?.totalClients ?? clients.length;
|
||||
const activeSockets = metrics?.activeSockets ?? 0;
|
||||
const totalStorage = metrics?.storageStats.totalMB ?? '0.00';
|
||||
const totalStorageBytes = metrics?.storageStats.totalBytes ?? 0;
|
||||
const storageValue =
|
||||
totalStorageBytes < 1024 * 1024
|
||||
? String(Math.max(1, Math.round(totalStorageBytes / 1024)))
|
||||
: totalStorage;
|
||||
totalStorageBytes === 0
|
||||
? '0'
|
||||
: totalStorageBytes < 1024 * 1024
|
||||
? String(Math.max(1, Math.round(totalStorageBytes / 1024)))
|
||||
: totalStorage;
|
||||
const storageUnit = totalStorageBytes < 1024 * 1024 ? 'KB' : 'MB';
|
||||
const totalFiles = metrics?.storageStats.totalFiles ?? files.length;
|
||||
const totalRows =
|
||||
@@ -153,13 +192,22 @@ export const DashboardOverview: React.FC = () => {
|
||||
</>
|
||||
}
|
||||
>
|
||||
{fetchError && (
|
||||
<div
|
||||
role="alert"
|
||||
className="mb-3 border border-[var(--danger)] bg-[var(--danger-soft)] px-3 py-2 text-[11.5px] text-[var(--text)]"
|
||||
>
|
||||
{fetchError}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<StatStrip
|
||||
items={[
|
||||
{ label: 'Events today', value: eventsToday.toLocaleString() },
|
||||
{
|
||||
label: 'Active clients',
|
||||
value: `${activeClients} / ${totalClients}`,
|
||||
dot: activeClients > 0 ? 'ok' : 'off',
|
||||
label: 'Active clients · Sockets',
|
||||
value: `${activeClients} / ${totalClients} · ${activeSockets} ws`,
|
||||
dot: activeClients > 0 || activeSockets > 0 ? 'ok' : 'off',
|
||||
},
|
||||
{ label: 'Storage used', value: `${storageValue} ${storageUnit}` },
|
||||
{ label: 'Stored rows', value: totalRows.toLocaleString() },
|
||||
@@ -169,7 +217,11 @@ export const DashboardOverview: React.FC = () => {
|
||||
<div className="grid min-w-0 grid-cols-1 gap-3 lg:grid-cols-[minmax(0,1.85fr)_minmax(0,1fr)]">
|
||||
<Panel
|
||||
title="Recent events"
|
||||
meta={`${events.length}`}
|
||||
meta={
|
||||
bufferedCount > 0
|
||||
? `${events.length} · ${bufferedCount} buffered`
|
||||
: `${events.length}`
|
||||
}
|
||||
action={
|
||||
<Link
|
||||
href="/vcon/data"
|
||||
@@ -190,24 +242,28 @@ export const DashboardOverview: React.FC = () => {
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{events.map((event) => (
|
||||
<TableRow key={event.eventId}>
|
||||
<TableCell className="font-mono tabular-nums text-[var(--text-faint)]">
|
||||
{event.timestamp.includes('T')
|
||||
? new Date(event.timestamp).toLocaleTimeString()
|
||||
: event.timestamp}
|
||||
</TableCell>
|
||||
<TableCell className="font-mono text-[var(--accent)]">
|
||||
{event.clientId}
|
||||
</TableCell>
|
||||
<TableCell className="max-w-[380px] truncate font-mono text-[var(--text)]">
|
||||
{event.payload}
|
||||
</TableCell>
|
||||
<TableCell className="text-right font-mono tabular-nums text-[var(--text-faint)]">
|
||||
{new TextEncoder().encode(event.payload).length} B
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
{events.length === 0 ? (
|
||||
<TableEmpty colSpan={4} message="No recent events" />
|
||||
) : (
|
||||
events.map((event) => (
|
||||
<TableRow key={event.eventId}>
|
||||
<TableCell className="font-mono tabular-nums text-[var(--text-faint)]">
|
||||
{event.timestamp.includes('T')
|
||||
? new Date(event.timestamp).toLocaleTimeString()
|
||||
: event.timestamp}
|
||||
</TableCell>
|
||||
<TableCell className="font-mono text-[var(--accent)]">
|
||||
{event.clientId}
|
||||
</TableCell>
|
||||
<TableCell className="max-w-[380px] truncate font-mono text-[var(--text)]">
|
||||
{event.payload}
|
||||
</TableCell>
|
||||
<TableCell className="text-right font-mono tabular-nums text-[var(--text-faint)]">
|
||||
{new TextEncoder().encode(event.payload).length} B
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))
|
||||
)}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</TableWrap>
|
||||
|
||||
@@ -3,7 +3,10 @@
|
||||
import { PageShell } from '@/components/layout/PageShell';
|
||||
import { Button } from '@/components/ui/Button';
|
||||
import { Card } from '@/components/ui/Card';
|
||||
import { CodeBlock } from '@/components/ui/CodeBlock';
|
||||
import { CopyButton } from '@/components/ui/CopyButton';
|
||||
import { Input, Select } from '@/components/ui/Input';
|
||||
import { Modal } from '@/components/ui/Modal';
|
||||
import { Panel } from '@/components/ui/Panel';
|
||||
import { Pill } from '@/components/ui/Pill';
|
||||
import {
|
||||
@@ -34,6 +37,18 @@ interface Client {
|
||||
|
||||
const PAGE_SIZE = 12;
|
||||
|
||||
function formatEventPayload(raw: string): string {
|
||||
try {
|
||||
return JSON.stringify(JSON.parse(raw), null, 2);
|
||||
} catch {
|
||||
return raw;
|
||||
}
|
||||
}
|
||||
|
||||
function getPayloadBytes(raw: string): number {
|
||||
return new TextEncoder().encode(raw).length;
|
||||
}
|
||||
|
||||
export function DataBrowser() {
|
||||
const router = useRouter();
|
||||
const [events, setEvents] = useState<StreamEvent[]>([]);
|
||||
@@ -54,6 +69,7 @@ export function DataBrowser() {
|
||||
const [filtersReady, setFiltersReady] = useState(false);
|
||||
const [refreshing, setRefreshing] = useState(false);
|
||||
const [errorMessage, setErrorMessage] = useState('');
|
||||
const [selectedEvent, setSelectedEvent] = useState<StreamEvent | null>(null);
|
||||
|
||||
const fetchEvents = useCallback(async () => {
|
||||
try {
|
||||
@@ -65,7 +81,11 @@ export function DataBrowser() {
|
||||
page: String(page),
|
||||
pageSize: String(PAGE_SIZE),
|
||||
});
|
||||
const response = await fetch(`/api/data?${params}`);
|
||||
const response = await fetch(`/api/data?${params}`, { cache: 'no-store' });
|
||||
if (response.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
if (!response.ok) throw new Error('Unable to load events');
|
||||
const data = await response.json();
|
||||
setEvents(data.events || []);
|
||||
@@ -78,8 +98,14 @@ export function DataBrowser() {
|
||||
}, [appliedFilters, page]);
|
||||
|
||||
useEffect(() => {
|
||||
fetch('/api/clients')
|
||||
.then((response) => (response.ok ? response.json() : { clients: [] }))
|
||||
fetch('/api/clients', { cache: 'no-store' })
|
||||
.then((response) => {
|
||||
if (response.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return { clients: [] };
|
||||
}
|
||||
return response.ok ? response.json() : { clients: [] };
|
||||
})
|
||||
.then((data) => setClients(data.clients || []))
|
||||
.catch(() => setClients([]));
|
||||
}, []);
|
||||
@@ -131,6 +157,10 @@ export function DataBrowser() {
|
||||
setRefreshing(false);
|
||||
};
|
||||
|
||||
const formattedSelectedPayload = selectedEvent
|
||||
? formatEventPayload(selectedEvent.payload)
|
||||
: '';
|
||||
|
||||
return (
|
||||
<PageShell
|
||||
title="Events"
|
||||
@@ -238,7 +268,11 @@ export function DataBrowser() {
|
||||
<TableEmpty colSpan={6} message="No matching events" />
|
||||
) : (
|
||||
events.map((event) => (
|
||||
<TableRow key={event.eventId}>
|
||||
<TableRow
|
||||
key={`${event.clientId}:${event.eventId}`}
|
||||
className="cursor-pointer"
|
||||
onClick={() => setSelectedEvent(event)}
|
||||
>
|
||||
<TableCell className="font-mono tabular-nums text-[var(--text)]">
|
||||
{new Date(event.timestamp).toLocaleTimeString([], {
|
||||
hour: '2-digit',
|
||||
@@ -257,13 +291,34 @@ export function DataBrowser() {
|
||||
{event.payload}
|
||||
</TableCell>
|
||||
<TableCell className="hidden font-mono tabular-nums md:table-cell">
|
||||
{new TextEncoder().encode(event.payload).length} B
|
||||
{getPayloadBytes(event.payload)} B
|
||||
</TableCell>
|
||||
<TableCell className="text-right">
|
||||
<span className="inline-flex items-center gap-1.5 text-[var(--text-muted)]">
|
||||
<span className="h-1.5 w-1.5 bg-[var(--accent)]" />
|
||||
{event.status}
|
||||
</span>
|
||||
<div className="inline-flex items-center gap-2">
|
||||
<span className="inline-flex items-center gap-1.5 text-[var(--text-muted)]">
|
||||
<span
|
||||
className={`h-1.5 w-1.5 ${
|
||||
event.status === 'buffered'
|
||||
? 'bg-[var(--warning)]'
|
||||
: 'bg-[var(--accent)]'
|
||||
}`}
|
||||
/>
|
||||
{event.status}
|
||||
</span>
|
||||
<Button
|
||||
type="button"
|
||||
variant="icon"
|
||||
size="icon"
|
||||
icon="Eye"
|
||||
className="h-5 w-5 border border-transparent hover:border-[var(--border)] hover:bg-[var(--surface-2)]"
|
||||
title={`Inspect event ${event.eventId}`}
|
||||
aria-label={`Inspect event ${event.eventId}`}
|
||||
onClick={(e) => {
|
||||
e.stopPropagation();
|
||||
setSelectedEvent(event);
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))
|
||||
@@ -338,6 +393,69 @@ export function DataBrowser() {
|
||||
</nav>
|
||||
</footer>
|
||||
</Panel>
|
||||
|
||||
<Modal
|
||||
isOpen={Boolean(selectedEvent)}
|
||||
onClose={() => setSelectedEvent(null)}
|
||||
title={`Event payload: ${selectedEvent?.eventId || ''}`}
|
||||
maxWidth="lg"
|
||||
footer={
|
||||
<Button type="button" size="xs" onClick={() => setSelectedEvent(null)}>
|
||||
Close
|
||||
</Button>
|
||||
}
|
||||
>
|
||||
{selectedEvent && (
|
||||
<div className="flex flex-col gap-3">
|
||||
<div className="grid grid-cols-2 gap-px border border-[var(--border)] bg-[var(--border)] sm:grid-cols-4">
|
||||
<div className="bg-[var(--surface)] p-2">
|
||||
<div className="text-[9.5px] uppercase tracking-wider text-[var(--text-faint)]">
|
||||
Client
|
||||
</div>
|
||||
<div className="truncate font-mono text-[11.5px] text-[var(--accent)]">
|
||||
{selectedEvent.clientId}
|
||||
</div>
|
||||
</div>
|
||||
<div className="bg-[var(--surface)] p-2">
|
||||
<div className="text-[9.5px] uppercase tracking-wider text-[var(--text-faint)]">
|
||||
Status
|
||||
</div>
|
||||
<div className="font-mono text-[11.5px] text-[var(--text)]">
|
||||
{selectedEvent.status}
|
||||
</div>
|
||||
</div>
|
||||
<div className="bg-[var(--surface)] p-2">
|
||||
<div className="text-[9.5px] uppercase tracking-wider text-[var(--text-faint)]">
|
||||
Size
|
||||
</div>
|
||||
<div className="font-mono text-[11.5px] tabular-nums text-[var(--text)]">
|
||||
{getPayloadBytes(selectedEvent.payload)} B
|
||||
</div>
|
||||
</div>
|
||||
<div className="bg-[var(--surface)] p-2">
|
||||
<div className="text-[9.5px] uppercase tracking-wider text-[var(--text-faint)]">
|
||||
Timestamp
|
||||
</div>
|
||||
<div className="truncate font-mono text-[11px] tabular-nums text-[var(--text-muted)]">
|
||||
{selectedEvent.timestamp}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-1">
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<span className="text-[10px] uppercase tracking-wider text-[var(--text-faint)]">
|
||||
JSON / Raw Payload
|
||||
</span>
|
||||
<CopyButton value={formattedSelectedPayload} />
|
||||
</div>
|
||||
<CodeBlock className="max-h-[340px] border border-[var(--border)] text-[11px]">
|
||||
{formattedSelectedPayload}
|
||||
</CodeBlock>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</Modal>
|
||||
</PageShell>
|
||||
);
|
||||
}
|
||||
@@ -56,6 +56,7 @@ export function ExportWorkspace() {
|
||||
const [files, setFiles] = useState<StorageFile[]>(() =>
|
||||
getCached<StorageFile[]>('vcon:storage-files', [])
|
||||
);
|
||||
const [bufferedCount, setBufferedCount] = useState<number>(0);
|
||||
const [selectedClient, setSelectedClient] = useState('all');
|
||||
const [fromDate, setFromDate] = useState('');
|
||||
const [toDate, setToDate] = useState('');
|
||||
@@ -69,19 +70,33 @@ export function ExportWorkspace() {
|
||||
|
||||
useEffect(() => {
|
||||
Promise.all([
|
||||
fetch('/api/clients').then(async (response) => {
|
||||
fetch('/api/clients', { cache: 'no-store' }).then(async (response) => {
|
||||
if (response.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return { clients: [] };
|
||||
}
|
||||
if (!response.ok) throw new Error('Unable to load clients');
|
||||
return response.json();
|
||||
}),
|
||||
fetch('/api/storage').then(async (response) => {
|
||||
fetch('/api/storage', { cache: 'no-store' }).then(async (response) => {
|
||||
if (response.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return { files: [] };
|
||||
}
|
||||
if (!response.ok) throw new Error('Unable to load stored files');
|
||||
return response.json();
|
||||
}),
|
||||
fetch('/api/metrics', { cache: 'no-store' })
|
||||
.then(async (response) => (response.ok ? response.json() : null))
|
||||
.catch(() => null),
|
||||
])
|
||||
.then(([clientData, storageData]) => {
|
||||
.then(([clientData, storageData, metricsData]) => {
|
||||
setClients(setCached('vcon:clients', clientData.clients || []));
|
||||
setFiles(setCached('vcon:storage-files', storageData.files || []));
|
||||
if (storageData.stats) setCached('vcon:storage-stats', storageData.stats);
|
||||
if (metricsData?.bufferStats) {
|
||||
setBufferedCount(Number(metricsData.bufferStats.currentBufferSize) || 0);
|
||||
}
|
||||
setLoadError('');
|
||||
})
|
||||
.catch((fetchError) => {
|
||||
@@ -162,15 +177,24 @@ export function ExportWorkspace() {
|
||||
return `/api/export?${params.toString()}`;
|
||||
}, [format, fromDate, selectedClient, toDate]);
|
||||
|
||||
const invalidRange = Boolean(fromDate && toDate && fromDate > toDate);
|
||||
const nothingToExport = preview.rows === 0 && bufferedCount === 0;
|
||||
|
||||
const downloadExport = () => {
|
||||
if (fromDate && toDate && fromDate > toDate) {
|
||||
if (invalidRange) {
|
||||
error('Invalid date range', 'Start date must be on or before end date.');
|
||||
return;
|
||||
}
|
||||
if (nothingToExport) {
|
||||
error('Nothing to export', 'No stored rows or buffered events match the selected filters.');
|
||||
return;
|
||||
}
|
||||
window.location.assign(exportUrl);
|
||||
info(
|
||||
'Export started',
|
||||
`${format.toUpperCase()} download for ${preview.rows.toLocaleString()} rows.`
|
||||
bufferedCount > 0
|
||||
? `${format.toUpperCase()} download (${preview.rows.toLocaleString()} stored + ${bufferedCount} buffered auto-flushed).`
|
||||
: `${format.toUpperCase()} download for ${preview.rows.toLocaleString()} rows.`
|
||||
);
|
||||
};
|
||||
|
||||
@@ -307,17 +331,30 @@ export function ExportWorkspace() {
|
||||
</strong>{' '}
|
||||
files
|
||||
</span>
|
||||
{bufferedCount > 0 && (
|
||||
<>
|
||||
<span aria-hidden="true">·</span>
|
||||
<span className="text-[var(--warning)]">
|
||||
+{bufferedCount} buffered (flushed on export)
|
||||
</span>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
{loadError && (
|
||||
<p role="alert" className="text-[11px] text-[var(--text-muted)]">
|
||||
<p role="alert" className="text-[11px] text-[var(--danger)]">
|
||||
{loadError}
|
||||
</p>
|
||||
)}
|
||||
{fromDate && toDate && fromDate > toDate && (
|
||||
<p role="alert" className="text-[11px] text-[var(--text-muted)]">
|
||||
{invalidRange && (
|
||||
<p role="alert" className="text-[11px] text-[var(--danger)]">
|
||||
Start date must be on or before end date.
|
||||
</p>
|
||||
)}
|
||||
{!loading && !invalidRange && nothingToExport && (
|
||||
<p role="status" className="text-[11px] text-[var(--text-faint)]">
|
||||
No stored rows or buffered events match the selected filters.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex flex-wrap items-center justify-end gap-1.5 border-t border-[var(--border)] pt-2.5">
|
||||
<Button type="button" size="xs" onClick={resetExport}>
|
||||
@@ -328,7 +365,7 @@ export function ExportWorkspace() {
|
||||
variant="primary"
|
||||
size="xs"
|
||||
icon="Download"
|
||||
disabled={loading || Boolean(fromDate && toDate && fromDate > toDate)}
|
||||
disabled={loading || invalidRange || nothingToExport}
|
||||
onClick={downloadExport}
|
||||
>
|
||||
Download {format.toUpperCase()}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { ActivityPreview } from '@/components/ui/ActivityPreview';
|
||||
import { Button } from '@/components/ui/Button';
|
||||
import { buttonClassNames } from '@/components/ui/Button';
|
||||
import Link from 'next/link';
|
||||
import React from 'react';
|
||||
|
||||
@@ -17,11 +17,11 @@ export const Hero: React.FC = () => (
|
||||
Keep activity organized, review the records that matter, and export them when you need them.
|
||||
</p>
|
||||
<div className="flex flex-wrap gap-2">
|
||||
<Link href="#features">
|
||||
<Button variant="primary" size="sm">Explore features</Button>
|
||||
<Link href="/#features" className={buttonClassNames({ variant: 'primary', size: 'sm' })}>
|
||||
Explore features
|
||||
</Link>
|
||||
<Link href="#sdk">
|
||||
<Button variant="secondary" size="sm">Python SDK</Button>
|
||||
<Link href="/#sdk" className={buttonClassNames({ variant: 'secondary', size: 'sm' })}>
|
||||
Python SDK
|
||||
</Link>
|
||||
</div>
|
||||
<ActivityPreview />
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Button } from '@/components/ui/Button';
|
||||
import { buttonClassNames } from '@/components/ui/Button';
|
||||
import Link from 'next/link';
|
||||
import React from 'react';
|
||||
|
||||
@@ -8,8 +8,8 @@ export const LandingAbout: React.FC = () => (
|
||||
<h2 className="mb-1 text-[20px] font-medium text-[var(--text)]">Ready to simplify your data workflow?</h2>
|
||||
<p className="text-[13px] text-[var(--text-muted)]">Start with the SDK guide and make the workspace your own.</p>
|
||||
</div>
|
||||
<Link href="#sdk">
|
||||
<Button variant="primary" size="sm">View Python SDK</Button>
|
||||
<Link href="/#sdk" className={buttonClassNames({ variant: 'primary', size: 'sm' })}>
|
||||
View Python SDK
|
||||
</Link>
|
||||
</section>
|
||||
);
|
||||
|
||||
@@ -4,6 +4,7 @@ import { PageShell } from '@/components/layout/PageShell';
|
||||
import { Button } from '@/components/ui/Button';
|
||||
import { FormSwitch } from '@/components/ui/Form';
|
||||
import { Input } from '@/components/ui/Input';
|
||||
import { Modal } from '@/components/ui/Modal';
|
||||
import {
|
||||
SettingsField,
|
||||
SettingsFooter,
|
||||
@@ -14,7 +15,7 @@ import {
|
||||
import { StatStrip } from '@/components/ui/StatStrip';
|
||||
import { useToast } from '@/components/ui/Toast';
|
||||
import { getCached, setCached } from '@/lib/vconCache';
|
||||
import { useEffect, useMemo, useState } from 'react';
|
||||
import { useCallback, useEffect, useMemo, useState } from 'react';
|
||||
|
||||
interface EditableSettings {
|
||||
maxEventsPerSec: number;
|
||||
@@ -50,6 +51,7 @@ const TABS = [
|
||||
];
|
||||
|
||||
type TabId = (typeof TABS)[number]['id'];
|
||||
type SocketHealthState = 'idle' | 'checking' | 'online' | 'offline';
|
||||
|
||||
export function SettingsWorkspace() {
|
||||
const { success, error } = useToast();
|
||||
@@ -61,11 +63,15 @@ export function SettingsWorkspace() {
|
||||
);
|
||||
const [activeTab, setActiveTab] = useState<TabId>('general');
|
||||
const [jwtSecret, setJwtSecret] = useState('');
|
||||
const [jwtError, setJwtError] = useState('');
|
||||
const [confirmRotateOpen, setConfirmRotateOpen] = useState(false);
|
||||
const [storagePath, setStoragePath] = useState(() =>
|
||||
getCached<string>('vcon:storage-path', '')
|
||||
);
|
||||
const [baseUrl, setBaseUrl] = useState('');
|
||||
const [webSocketUrl, setWebSocketUrl] = useState('');
|
||||
const [socketStatus, setSocketStatus] = useState<SocketHealthState>('idle');
|
||||
const [activeSockets, setActiveSockets] = useState<number>(0);
|
||||
const [serverPort, setServerPort] = useState(() =>
|
||||
getCached<number>('vcon:server-port', 4000)
|
||||
);
|
||||
@@ -74,26 +80,60 @@ export function SettingsWorkspace() {
|
||||
);
|
||||
const [saving, setSaving] = useState(false);
|
||||
|
||||
const probeSocketServer = useCallback(async (port: number) => {
|
||||
setSocketStatus('checking');
|
||||
const isLocalDev = window.location.port === '3000';
|
||||
const httpHost = isLocalDev
|
||||
? `${window.location.hostname}:${port}`
|
||||
: window.location.host;
|
||||
const healthUrl = `${window.location.protocol}//${httpHost}/health`;
|
||||
|
||||
try {
|
||||
const response = await fetch(healthUrl, {
|
||||
cache: 'no-store',
|
||||
signal: AbortSignal.timeout(1500),
|
||||
});
|
||||
setSocketStatus(response.ok ? 'online' : 'offline');
|
||||
} catch {
|
||||
setSocketStatus('offline');
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
setBaseUrl(window.location.origin);
|
||||
const isLocalDev = window.location.port === '3000';
|
||||
const socketProtocol = window.location.protocol === 'https:' ? 'wss' : 'ws';
|
||||
const socketHost = isLocalDev
|
||||
? `${window.location.hostname}:4000`
|
||||
? `${window.location.hostname}:${serverPort}`
|
||||
: window.location.host;
|
||||
setWebSocketUrl(`${socketProtocol}://${socketHost}`);
|
||||
|
||||
Promise.all([
|
||||
fetch('/api/settings').then(async (response) => {
|
||||
fetch('/api/settings', { cache: 'no-store' }).then(async (response) => {
|
||||
if (response.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return null;
|
||||
}
|
||||
if (!response.ok) throw new Error('Unable to load settings');
|
||||
return response.json();
|
||||
}),
|
||||
fetch('/api/storage').then(async (response) => {
|
||||
fetch('/api/storage', { cache: 'no-store' }).then(async (response) => {
|
||||
if (response.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return null;
|
||||
}
|
||||
if (!response.ok) throw new Error('Unable to load storage information');
|
||||
return response.json();
|
||||
}),
|
||||
fetch('/api/metrics', { cache: 'no-store' })
|
||||
.then(async (response) => (response.ok ? response.json() : null))
|
||||
.catch(() => null),
|
||||
])
|
||||
.then(([settingsData, storageData]) => {
|
||||
.then(([settingsData, storageData, metricsData]) => {
|
||||
if (!settingsData || !storageData) return;
|
||||
const loadedPort = Number(
|
||||
(settingsData.settings as PublicSettings).serverPort ?? 4000
|
||||
);
|
||||
const loaded: EditableSettings = {
|
||||
maxEventsPerSec: Number(
|
||||
settingsData.settings.maxEventsPerSec ??
|
||||
@@ -126,25 +166,26 @@ export function SettingsWorkspace() {
|
||||
};
|
||||
setSettings(setCached('vcon:settings', loaded));
|
||||
setSavedSettings(loaded);
|
||||
setServerPort(
|
||||
setCached(
|
||||
'vcon:server-port',
|
||||
Number((settingsData.settings as PublicSettings).serverPort ?? 4000)
|
||||
)
|
||||
);
|
||||
setServerPort(setCached('vcon:server-port', loadedPort));
|
||||
setStoragePath(
|
||||
setCached(
|
||||
'vcon:storage-path',
|
||||
String(storageData.stats?.storageDirectory ?? '')
|
||||
)
|
||||
);
|
||||
if (metricsData && typeof metricsData.activeSockets === 'number') {
|
||||
setActiveSockets(metricsData.activeSockets);
|
||||
if (metricsData.activeSockets > 0) {
|
||||
setSocketStatus('online');
|
||||
}
|
||||
}
|
||||
setCached('vcon:settings-loaded', true);
|
||||
})
|
||||
.catch(() => {
|
||||
error('Settings load failed', 'Using default runtime settings.');
|
||||
})
|
||||
.finally(() => setLoading(false));
|
||||
}, [error]);
|
||||
}, [error, serverPort]);
|
||||
|
||||
const isDirty = useMemo(
|
||||
() =>
|
||||
@@ -157,9 +198,9 @@ export function SettingsWorkspace() {
|
||||
setSettings((current) => ({ ...current, [key]: Number(value) || 0 }));
|
||||
};
|
||||
|
||||
const saveSettings = async (event: React.FormEvent<HTMLFormElement>) => {
|
||||
event.preventDefault();
|
||||
const normalized: EditableSettings = {
|
||||
const buildNormalizedSettings = (): EditableSettings => {
|
||||
const parsedPrune = Number(settings.autoPruneDays);
|
||||
return {
|
||||
maxEventsPerSec: Math.max(
|
||||
1,
|
||||
Math.min(
|
||||
@@ -187,11 +228,16 @@ export function SettingsWorkspace() {
|
||||
settings.fileRotationMb || DEFAULT_SETTINGS.fileRotationMb
|
||||
),
|
||||
enableIdempotency: settings.enableIdempotency,
|
||||
autoPruneDays: Math.max(
|
||||
1,
|
||||
Math.min(365, settings.autoPruneDays || DEFAULT_SETTINGS.autoPruneDays)
|
||||
),
|
||||
autoPruneDays: Number.isFinite(parsedPrune)
|
||||
? Math.max(0, Math.min(365, Math.floor(parsedPrune)))
|
||||
: DEFAULT_SETTINGS.autoPruneDays,
|
||||
};
|
||||
};
|
||||
|
||||
const executeSave = async () => {
|
||||
const normalized = buildNormalizedSettings();
|
||||
const trimmedSecret = jwtSecret.trim();
|
||||
setConfirmRotateOpen(false);
|
||||
setSaving(true);
|
||||
try {
|
||||
const response = await fetch('/api/settings', {
|
||||
@@ -199,15 +245,25 @@ export function SettingsWorkspace() {
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
...normalized,
|
||||
...(jwtSecret.trim() ? { jwtSecret: jwtSecret.trim() } : {}),
|
||||
...(trimmedSecret ? { jwtSecret: trimmedSecret } : {}),
|
||||
}),
|
||||
});
|
||||
if (response.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
const data = await response.json();
|
||||
if (!response.ok) throw new Error(data.error || 'Settings update failed');
|
||||
setSettings(setCached('vcon:settings', normalized));
|
||||
setSavedSettings(normalized);
|
||||
setJwtSecret('');
|
||||
success('Settings saved', 'Runtime settings updated.');
|
||||
setJwtError('');
|
||||
success(
|
||||
'Settings saved',
|
||||
trimmedSecret
|
||||
? 'Runtime settings saved and client JWT credentials rotated.'
|
||||
: 'Runtime settings updated.'
|
||||
);
|
||||
} catch (saveError) {
|
||||
error(
|
||||
'Save failed',
|
||||
@@ -220,9 +276,30 @@ export function SettingsWorkspace() {
|
||||
}
|
||||
};
|
||||
|
||||
const handleFormSubmit = async (event: React.FormEvent<HTMLFormElement>) => {
|
||||
event.preventDefault();
|
||||
const trimmedSecret = jwtSecret.trim();
|
||||
if (trimmedSecret.length > 0 && trimmedSecret.length < 32) {
|
||||
const message = 'JWT secret must be at least 32 characters.';
|
||||
setJwtError(message);
|
||||
setActiveTab('security');
|
||||
error('Invalid JWT secret', message);
|
||||
return;
|
||||
}
|
||||
|
||||
setJwtError('');
|
||||
if (trimmedSecret.length >= 32) {
|
||||
setConfirmRotateOpen(true);
|
||||
return;
|
||||
}
|
||||
|
||||
await executeSave();
|
||||
};
|
||||
|
||||
const discardChanges = () => {
|
||||
setSettings(savedSettings);
|
||||
setJwtSecret('');
|
||||
setJwtError('');
|
||||
};
|
||||
|
||||
return (
|
||||
@@ -237,7 +314,9 @@ export function SettingsWorkspace() {
|
||||
},
|
||||
{
|
||||
label: 'CSV rotation',
|
||||
value: `${settings.fileRotationMb}MB · ${settings.autoPruneDays}d`,
|
||||
value: `${settings.fileRotationMb}MB · ${
|
||||
settings.autoPruneDays === 0 ? 'Keep all' : `${settings.autoPruneDays}d`
|
||||
}`,
|
||||
},
|
||||
]}
|
||||
/>
|
||||
@@ -249,7 +328,7 @@ export function SettingsWorkspace() {
|
||||
onChange={(tab) => setActiveTab(tab as TabId)}
|
||||
/>
|
||||
|
||||
<form onSubmit={saveSettings} className="w-full">
|
||||
<form onSubmit={handleFormSubmit} className="w-full">
|
||||
<SettingsPanel
|
||||
id={`settings-panel-${activeTab}`}
|
||||
role="tabpanel"
|
||||
@@ -307,6 +386,40 @@ export function SettingsWorkspace() {
|
||||
{activeTab === 'connections' && (
|
||||
<>
|
||||
<SettingsGroup title="WebSocket server">
|
||||
<SettingsField label="Live health">
|
||||
<div className="flex items-center justify-between gap-2 border border-[var(--border)] bg-[var(--surface-2)] px-2.5 py-1.5">
|
||||
<span className="inline-flex items-center gap-1.5 font-mono text-[11.5px] text-[var(--text)]">
|
||||
<span
|
||||
aria-hidden="true"
|
||||
className={`h-1.5 w-1.5 ${
|
||||
socketStatus === 'online'
|
||||
? 'bg-[var(--accent)]'
|
||||
: socketStatus === 'checking'
|
||||
? 'bg-[var(--warning)]'
|
||||
: socketStatus === 'offline'
|
||||
? 'bg-[var(--danger)]'
|
||||
: 'bg-[var(--text-faint)]'
|
||||
}`}
|
||||
/>
|
||||
{socketStatus === 'online'
|
||||
? `Online (Port ${serverPort} · ${activeSockets} active)`
|
||||
: socketStatus === 'checking'
|
||||
? 'Probing ingestion server...'
|
||||
: socketStatus === 'offline'
|
||||
? `Offline (Port ${serverPort} unreachable)`
|
||||
: `Port ${serverPort} · ${activeSockets} active socket(s)`}
|
||||
</span>
|
||||
<Button
|
||||
type="button"
|
||||
size="xs"
|
||||
variant="ghost"
|
||||
icon="RefreshCw"
|
||||
onClick={() => void probeSocketServer(serverPort)}
|
||||
>
|
||||
Probe
|
||||
</Button>
|
||||
</div>
|
||||
</SettingsField>
|
||||
<SettingsField label="WebSocket URL">
|
||||
<Input
|
||||
compact
|
||||
@@ -378,11 +491,11 @@ export function SettingsWorkspace() {
|
||||
className="font-mono"
|
||||
/>
|
||||
</SettingsField>
|
||||
<SettingsField label="Retention">
|
||||
<SettingsField label="Retention (0 = never prune)">
|
||||
<Input
|
||||
compact
|
||||
type="number"
|
||||
min="1"
|
||||
min="0"
|
||||
max="365"
|
||||
suffix="days"
|
||||
value={settings.autoPruneDays}
|
||||
@@ -408,14 +521,21 @@ export function SettingsWorkspace() {
|
||||
|
||||
{activeTab === 'security' && (
|
||||
<SettingsGroup title="Authentication">
|
||||
<SettingsField label="JWT secret">
|
||||
<SettingsField label="JWT secret (min 32 chars)">
|
||||
<Input
|
||||
compact
|
||||
type="password"
|
||||
autoComplete="new-password"
|
||||
value={jwtSecret}
|
||||
onChange={(event) => setJwtSecret(event.target.value)}
|
||||
placeholder="New secret..."
|
||||
error={jwtError}
|
||||
onChange={(event) => {
|
||||
const val = event.target.value;
|
||||
setJwtSecret(val);
|
||||
if (!val.trim() || val.trim().length >= 32) {
|
||||
setJwtError('');
|
||||
}
|
||||
}}
|
||||
placeholder="Minimum 32 characters to rotate..."
|
||||
/>
|
||||
</SettingsField>
|
||||
<SettingsField label="Payload size limit">
|
||||
@@ -471,6 +591,39 @@ export function SettingsWorkspace() {
|
||||
</SettingsPanel>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<Modal
|
||||
isOpen={confirmRotateOpen}
|
||||
onClose={() => !saving && setConfirmRotateOpen(false)}
|
||||
title="Confirm JWT secret rotation"
|
||||
maxWidth="xs"
|
||||
footer={
|
||||
<>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="xs"
|
||||
disabled={saving}
|
||||
onClick={() => setConfirmRotateOpen(false)}
|
||||
>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="danger"
|
||||
size="xs"
|
||||
loading={saving}
|
||||
onClick={() => void executeSave()}
|
||||
>
|
||||
Rotate & Save
|
||||
</Button>
|
||||
</>
|
||||
}
|
||||
>
|
||||
<p className="text-[12px] text-[var(--text-muted)]">
|
||||
Rotating the signing JWT secret will immediately re-sign registered client tokens and invalidate all older SDK JWT tokens and active WebSocket sessions. Continue?
|
||||
</p>
|
||||
</Modal>
|
||||
</PageShell>
|
||||
);
|
||||
}
|
||||
@@ -6,6 +6,7 @@ export interface ClientItem {
|
||||
totalEvents: number;
|
||||
lastActive?: string;
|
||||
createdAt: string;
|
||||
tokenPreview?: string;
|
||||
}
|
||||
|
||||
export interface ClientCredentials extends ClientItem {
|
||||
@@ -13,4 +14,9 @@ export interface ClientCredentials extends ClientItem {
|
||||
token: string;
|
||||
}
|
||||
|
||||
export type ClientStatusFilter = "all" | "online" | "offline" | "pending";
|
||||
export type ClientStatusFilter =
|
||||
| "all"
|
||||
| "online"
|
||||
| "offline"
|
||||
| "pending"
|
||||
| "revoked";
|
||||
+41
-29
@@ -10,6 +10,46 @@ export interface ButtonProps extends React.ButtonHTMLAttributes<HTMLButtonElemen
|
||||
loading?: boolean;
|
||||
}
|
||||
|
||||
const BUTTON_BASE_STYLES =
|
||||
'inline-flex items-center justify-center font-normal transition-colors select-none cursor-pointer disabled:opacity-50 disabled:cursor-not-allowed border border-transparent whitespace-nowrap outline-none';
|
||||
|
||||
const BUTTON_VARIANT_STYLES = {
|
||||
primary:
|
||||
'bg-[var(--text)] text-[var(--bg)] hover:opacity-90 border-[var(--text)]',
|
||||
secondary:
|
||||
'bg-[var(--bg-elevated)] text-[var(--text)] border-[var(--border-strong)] hover:bg-[var(--bg-hover)]',
|
||||
ghost:
|
||||
'bg-transparent text-[var(--text-muted)] hover:text-[var(--text)] hover:bg-[var(--bg-hover)] border-transparent',
|
||||
danger:
|
||||
'bg-transparent text-[var(--danger)] border-[var(--danger)] hover:bg-[var(--danger-soft)]',
|
||||
icon:
|
||||
'bg-transparent text-[var(--text-muted)] hover:text-[var(--text)] hover:bg-[var(--bg-hover)] border-transparent p-0',
|
||||
} as const;
|
||||
|
||||
const BUTTON_SIZE_STYLES = {
|
||||
default: 'min-h-8.5 px-3 text-[12px] gap-1.5',
|
||||
sm: 'h-7.5 px-2.5 text-[11.5px] gap-1.5',
|
||||
xs: 'h-7 px-2 text-[11px] gap-1',
|
||||
icon: 'w-7.5 h-7.5 p-0',
|
||||
} as const;
|
||||
|
||||
export function buttonClassNames({
|
||||
variant = 'secondary',
|
||||
size = 'default',
|
||||
className,
|
||||
}: {
|
||||
variant?: keyof typeof BUTTON_VARIANT_STYLES;
|
||||
size?: keyof typeof BUTTON_SIZE_STYLES;
|
||||
className?: string;
|
||||
} = {}): string {
|
||||
return cn(
|
||||
BUTTON_BASE_STYLES,
|
||||
BUTTON_VARIANT_STYLES[variant],
|
||||
size === 'icon' ? BUTTON_SIZE_STYLES.icon : BUTTON_SIZE_STYLES[size],
|
||||
className
|
||||
);
|
||||
}
|
||||
|
||||
export const Button = React.forwardRef<HTMLButtonElement, ButtonProps>(
|
||||
(
|
||||
{
|
||||
@@ -25,39 +65,11 @@ export const Button = React.forwardRef<HTMLButtonElement, ButtonProps>(
|
||||
},
|
||||
ref
|
||||
) => {
|
||||
const baseStyles =
|
||||
'inline-flex items-center justify-center font-normal transition-colors select-none cursor-pointer disabled:opacity-50 disabled:cursor-not-allowed border border-transparent whitespace-nowrap outline-none';
|
||||
|
||||
const variantStyles = {
|
||||
primary:
|
||||
'bg-[var(--text)] text-[var(--bg)] hover:opacity-90 border-[var(--text)]',
|
||||
secondary:
|
||||
'bg-[var(--bg-elevated)] text-[var(--text)] border-[var(--border-strong)] hover:bg-[var(--bg-hover)]',
|
||||
ghost:
|
||||
'bg-transparent text-[var(--text-muted)] hover:text-[var(--text)] hover:bg-[var(--bg-hover)] border-transparent',
|
||||
danger:
|
||||
'bg-transparent text-[var(--danger)] border-[var(--danger)] hover:bg-[var(--danger-soft)]',
|
||||
icon:
|
||||
'bg-transparent text-[var(--text-muted)] hover:text-[var(--text)] hover:bg-[var(--bg-hover)] border-transparent p-0',
|
||||
};
|
||||
|
||||
const sizeStyles = {
|
||||
default: 'min-h-8.5 px-3 text-[12px] gap-1.5',
|
||||
sm: 'h-7.5 px-2.5 text-[11.5px] gap-1.5',
|
||||
xs: 'h-7 px-2 text-[11px] gap-1',
|
||||
icon: 'w-7.5 h-7.5 p-0',
|
||||
};
|
||||
|
||||
return (
|
||||
<button
|
||||
ref={ref}
|
||||
disabled={disabled || loading}
|
||||
className={cn(
|
||||
baseStyles,
|
||||
variantStyles[variant],
|
||||
size === 'icon' ? sizeStyles.icon : sizeStyles[size],
|
||||
className
|
||||
)}
|
||||
className={buttonClassNames({ variant, size, className })}
|
||||
{...props}
|
||||
>
|
||||
{loading ? (
|
||||
|
||||
@@ -73,6 +73,31 @@ export class DatabaseEngine {
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`),
|
||||
pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS deleted_clients (
|
||||
client_id TEXT PRIMARY KEY,
|
||||
deleted_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`),
|
||||
pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS admin_users (
|
||||
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
|
||||
email TEXT NOT NULL UNIQUE,
|
||||
password_salt BYTEA NOT NULL,
|
||||
password_hash BYTEA NOT NULL,
|
||||
token_version INTEGER NOT NULL DEFAULT 1,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`),
|
||||
pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS admin_login_attempts (
|
||||
key_hash TEXT PRIMARY KEY,
|
||||
window_started_at TIMESTAMPTZ NOT NULL,
|
||||
failed_attempts INTEGER NOT NULL,
|
||||
blocked_until TIMESTAMPTZ,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`),
|
||||
]).then(() => undefined);
|
||||
|
||||
try {
|
||||
|
||||
@@ -1,3 +1,38 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { AuthService } from "../auth/jwt";
|
||||
|
||||
export function resolveInternalIngestionSecret(
|
||||
jwtSecretOverride?: string,
|
||||
): string {
|
||||
const explicit = (process.env.DATASTACK_INTERNAL_SECRET || "").trim();
|
||||
const isProd = process.env.NODE_ENV === "production";
|
||||
if (isProd) {
|
||||
if (explicit.length >= 32) return explicit;
|
||||
if (explicit.length > 0) return "";
|
||||
} else if (explicit.length > 0) {
|
||||
return explicit;
|
||||
}
|
||||
|
||||
const activeJwt = String(
|
||||
jwtSecretOverride ?? AuthService.getSecret() ?? "",
|
||||
).trim();
|
||||
const minJwtLen = isProd ? 32 : 1;
|
||||
if (activeJwt.length < minJwtLen) {
|
||||
return "";
|
||||
}
|
||||
|
||||
return createHash("sha256")
|
||||
.update(`datastack-internal:${activeJwt}`)
|
||||
.digest("hex");
|
||||
}
|
||||
|
||||
export function getInternalIngestionHeaders(
|
||||
jwtSecretOverride?: string,
|
||||
): Record<string, string> {
|
||||
const secret = resolveInternalIngestionSecret(jwtSecretOverride);
|
||||
return secret ? { "x-datastack-internal-secret": secret } : {};
|
||||
}
|
||||
|
||||
export interface ValidationResult {
|
||||
allowed: boolean;
|
||||
reason?: string;
|
||||
@@ -130,7 +165,7 @@ export class SecurityEngine {
|
||||
if (!withinLimit) {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: `Rate limit of ${clientMaxRate} events/sec exceeded for client ${clientId}`,
|
||||
reason: `Rate limit of ${effectiveRateLimit} events/sec exceeded for client ${clientId}`,
|
||||
code: "RATE_LIMIT_EXCEEDED",
|
||||
};
|
||||
}
|
||||
@@ -144,6 +179,18 @@ export class SecurityEngine {
|
||||
}
|
||||
|
||||
public static getStats() {
|
||||
const now = Date.now();
|
||||
for (const [k, timestamp] of this.idempotencySet.entries()) {
|
||||
if (now - timestamp > this.IDEMPOTENCY_TTL_MS) {
|
||||
this.idempotencySet.delete(k);
|
||||
}
|
||||
}
|
||||
for (const [clientId, entry] of this.rateLimitMap.entries()) {
|
||||
if (now - entry.windowStart > 10000) {
|
||||
this.rateLimitMap.delete(clientId);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
activeIdempotencyKeys: this.idempotencySet.size,
|
||||
rateLimitedClientsCount: this.rateLimitMap.size,
|
||||
|
||||
+392
-53
@@ -19,10 +19,30 @@ export interface StorageFileMeta {
|
||||
modifiedAt: Date;
|
||||
}
|
||||
|
||||
interface RowCountCacheEntry {
|
||||
sizeBytes: number;
|
||||
mtimeMs: number;
|
||||
linesCount: number;
|
||||
}
|
||||
|
||||
const LOCK_RETRY_MIN_MS = 5;
|
||||
const LOCK_RETRY_MAX_MS = 18;
|
||||
const LOCK_TIMEOUT_MS = 10_000;
|
||||
const STALE_LOCK_AGE_MS = 30_000;
|
||||
|
||||
function sleepMs(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
export class CSVStorageManager {
|
||||
// Base storage directory: /storage/data
|
||||
public static baseStorageDir = path.resolve(process.cwd(), 'storage', 'data');
|
||||
public static maxFileSizeBytes = 100 * 1024 * 1024; // 100MB rotation limit
|
||||
private static rowCountCache = new Map<string, RowCountCacheEntry>();
|
||||
|
||||
public static clearRowCountCache(): void {
|
||||
this.rowCountCache.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures client directory exists
|
||||
@@ -37,15 +57,194 @@ export class CSVStorageManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves target CSV file path with daily and 100MB rotation rules
|
||||
* Acquires an exclusive cooperative lock file for a client+date partition across Port 3000 & Port 4000
|
||||
*/
|
||||
public static async withPartitionLock<T>(
|
||||
clientId: string,
|
||||
dateStr: string,
|
||||
fn: () => Promise<T> | T,
|
||||
): Promise<T> {
|
||||
const clientDir = this.ensureClientDir(clientId);
|
||||
const date = this.extractSafePartitionDate(dateStr);
|
||||
const lockPath = path.join(clientDir, `.${date}.csv.lock`);
|
||||
const deadline = Date.now() + LOCK_TIMEOUT_MS;
|
||||
|
||||
let acquired = false;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const fd = fs.openSync(lockPath, 'wx');
|
||||
try {
|
||||
fs.writeFileSync(fd, `${process.pid}:${Date.now()}`, 'utf8');
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
acquired = true;
|
||||
break;
|
||||
} catch (err: unknown) {
|
||||
const code = (err as NodeJS.ErrnoException)?.code;
|
||||
if (code !== 'EEXIST') {
|
||||
throw err;
|
||||
}
|
||||
try {
|
||||
const lockStat = fs.statSync(lockPath);
|
||||
if (Date.now() - lockStat.mtimeMs > STALE_LOCK_AGE_MS) {
|
||||
fs.unlinkSync(lockPath);
|
||||
continue;
|
||||
}
|
||||
} catch {
|
||||
// Lock was released concurrently
|
||||
}
|
||||
const jitter =
|
||||
LOCK_RETRY_MIN_MS +
|
||||
Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS));
|
||||
await sleepMs(jitter);
|
||||
}
|
||||
}
|
||||
|
||||
if (!acquired) {
|
||||
throw new Error(`Timed out waiting for CSV partition lock: ${lockPath}`);
|
||||
}
|
||||
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
try {
|
||||
if (fs.existsSync(lockPath)) {
|
||||
fs.unlinkSync(lockPath);
|
||||
}
|
||||
} catch {
|
||||
// Ignore cleanup failure if already unlinked
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes and validates any timestamp or date string into a strict YYYY-MM-DD partition key
|
||||
*/
|
||||
public static extractSafePartitionDate(rawTimestamp?: string): string {
|
||||
const fallback = new Date().toISOString().slice(0, 10);
|
||||
if (!rawTimestamp || typeof rawTimestamp !== 'string') {
|
||||
return fallback;
|
||||
}
|
||||
const trimmed = rawTimestamp.trim();
|
||||
const prefix = trimmed.slice(0, 10);
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(prefix)) {
|
||||
const parsedPrefix = new Date(`${prefix}T00:00:00.000Z`);
|
||||
if (!Number.isNaN(parsedPrefix.getTime())) {
|
||||
return prefix;
|
||||
}
|
||||
}
|
||||
if (/^\d+(\.\d+)?$/.test(trimmed)) {
|
||||
const num = Number(trimmed);
|
||||
const ms = num < 1e11 ? num * 1000 : num;
|
||||
const d = new Date(ms);
|
||||
if (!Number.isNaN(d.getTime())) {
|
||||
return d.toISOString().slice(0, 10);
|
||||
}
|
||||
}
|
||||
const parsed = new Date(trimmed);
|
||||
if (!Number.isNaN(parsed.getTime())) {
|
||||
return parsed.toISOString().slice(0, 10);
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stateful RFC 4180 quote-aware chunked row counter so multi-100MB CSVs never load wholly into RAM
|
||||
*/
|
||||
private static createChunkRowCounter() {
|
||||
let inQuotes = false;
|
||||
let pendingQuote = false;
|
||||
let currentLineHasChars = false;
|
||||
let totalRows = 0;
|
||||
|
||||
return {
|
||||
pushChunk(chunk: Buffer | string) {
|
||||
const str = typeof chunk === 'string' ? chunk : chunk.toString('utf8');
|
||||
for (let i = 0; i < str.length; i += 1) {
|
||||
const ch = str[i];
|
||||
if (pendingQuote) {
|
||||
pendingQuote = false;
|
||||
if (ch === '"') {
|
||||
// Escaped quote "" across chunk boundary or inline
|
||||
continue;
|
||||
}
|
||||
inQuotes = !inQuotes;
|
||||
}
|
||||
|
||||
if (ch === '"') {
|
||||
currentLineHasChars = true;
|
||||
if (inQuotes) {
|
||||
if (i + 1 < str.length) {
|
||||
if (str[i + 1] === '"') {
|
||||
i += 1;
|
||||
} else {
|
||||
inQuotes = false;
|
||||
}
|
||||
} else {
|
||||
pendingQuote = true;
|
||||
}
|
||||
} else {
|
||||
inQuotes = true;
|
||||
}
|
||||
} else if (ch === '\n' && !inQuotes) {
|
||||
if (currentLineHasChars) {
|
||||
totalRows += 1;
|
||||
currentLineHasChars = false;
|
||||
}
|
||||
} else if (ch !== '\r' && ch !== ' ' && ch !== '\t') {
|
||||
currentLineHasChars = true;
|
||||
}
|
||||
}
|
||||
},
|
||||
finish(): number {
|
||||
if (currentLineHasChars) {
|
||||
totalRows += 1;
|
||||
}
|
||||
return Math.max(0, totalRows - 1);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private static countCsvDataRowsChunkedSync(filePath: string): number {
|
||||
const counter = this.createChunkRowCounter();
|
||||
const buf = Buffer.allocUnsafe(64 * 1024);
|
||||
const fd = fs.openSync(filePath, 'r');
|
||||
try {
|
||||
let bytesRead = 0;
|
||||
while ((bytesRead = fs.readSync(fd, buf, 0, buf.length, null)) > 0) {
|
||||
counter.pushChunk(buf.subarray(0, bytesRead));
|
||||
}
|
||||
return counter.finish();
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
}
|
||||
|
||||
private static async countCsvDataRowsStreamAsync(
|
||||
filePath: string,
|
||||
): Promise<number> {
|
||||
const counter = this.createChunkRowCounter();
|
||||
return new Promise<number>((resolve, reject) => {
|
||||
const stream = fs.createReadStream(filePath, {
|
||||
highWaterMark: 64 * 1024,
|
||||
});
|
||||
stream.on('data', (chunk) => counter.pushChunk(chunk));
|
||||
stream.on('end', () => resolve(counter.finish()));
|
||||
stream.on('error', (err) => reject(err));
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves target CSV file path with daily and configured MB rotation rules
|
||||
*/
|
||||
public static getTargetFilePath(clientId: string, dateStr?: string): string {
|
||||
const clientDir = this.ensureClientDir(clientId);
|
||||
const date = dateStr || new Date().toISOString().split('T')[0];
|
||||
const date = this.extractSafePartitionDate(dateStr);
|
||||
let baseFileName = `${date}.csv`;
|
||||
let targetPath = path.join(clientDir, baseFileName);
|
||||
|
||||
// Check for 100MB rotation
|
||||
// Check for file rotation
|
||||
let partIndex = 1;
|
||||
while (fs.existsSync(targetPath)) {
|
||||
try {
|
||||
@@ -66,10 +265,12 @@ export class CSVStorageManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch append events into target CSV file.
|
||||
* Batch append events into target CSV file under cooperative partition lock.
|
||||
* If file doesn't exist, writes header: clientId,eventId,timestamp,payload
|
||||
*/
|
||||
public static async appendBatch(events: DataEvent[]): Promise<{ written: number; filesUpdated: string[] }> {
|
||||
public static async appendBatch(
|
||||
events: DataEvent[],
|
||||
): Promise<{ written: number; filesUpdated: string[] }> {
|
||||
if (!events || events.length === 0) {
|
||||
return { written: 0, filesUpdated: [] };
|
||||
}
|
||||
@@ -77,7 +278,7 @@ export class CSVStorageManager {
|
||||
// Group events by client + date
|
||||
const grouped = new Map<string, DataEvent[]>();
|
||||
for (const evt of events) {
|
||||
const date = (evt.timestamp || new Date().toISOString()).split('T')[0];
|
||||
const date = this.extractSafePartitionDate(evt.timestamp);
|
||||
const key = `${evt.clientId}:::${date}`;
|
||||
if (!grouped.has(key)) {
|
||||
grouped.set(key, []);
|
||||
@@ -90,31 +291,39 @@ export class CSVStorageManager {
|
||||
|
||||
for (const [key, batch] of grouped.entries()) {
|
||||
const [clientId, date] = key.split(':::');
|
||||
const targetPath = this.getTargetFilePath(clientId, date);
|
||||
let hasContent = false;
|
||||
try {
|
||||
hasContent =
|
||||
fs.existsSync(targetPath) && fs.statSync(targetPath).size > 0;
|
||||
} catch {
|
||||
hasContent = false;
|
||||
}
|
||||
const targetPath = await this.withPartitionLock(clientId, date, () => {
|
||||
const resolvedPath = this.getTargetFilePath(clientId, date);
|
||||
let hasContent = false;
|
||||
try {
|
||||
hasContent =
|
||||
fs.existsSync(resolvedPath) && fs.statSync(resolvedPath).size > 0;
|
||||
} catch {
|
||||
hasContent = false;
|
||||
}
|
||||
|
||||
// Format payload as JSON string if not already
|
||||
const cleanBatch = batch.map((item) => ({
|
||||
clientId: item.clientId,
|
||||
eventId: item.eventId,
|
||||
timestamp: item.timestamp,
|
||||
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
|
||||
}));
|
||||
const cleanBatch = batch.map((item) => ({
|
||||
clientId: item.clientId,
|
||||
eventId: item.eventId,
|
||||
timestamp: item.timestamp,
|
||||
payload:
|
||||
typeof item.payload === 'string'
|
||||
? item.payload
|
||||
: JSON.stringify(item.payload),
|
||||
}));
|
||||
|
||||
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
|
||||
const csvData = parser.parse(cleanBatch);
|
||||
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
|
||||
const csvData = parser.parse(cleanBatch);
|
||||
|
||||
if (hasContent) {
|
||||
fs.appendFileSync(targetPath, '\n' + csvData, 'utf8');
|
||||
} else {
|
||||
fs.writeFileSync(targetPath, csvData, 'utf8');
|
||||
}
|
||||
if (hasContent) {
|
||||
fs.appendFileSync(resolvedPath, '\n' + csvData, 'utf8');
|
||||
} else {
|
||||
fs.writeFileSync(resolvedPath, csvData, 'utf8');
|
||||
}
|
||||
|
||||
// Invalidate row cache for modified file immediately
|
||||
this.rowCountCache.delete(resolvedPath);
|
||||
return resolvedPath;
|
||||
});
|
||||
|
||||
filesUpdated.push(targetPath);
|
||||
}
|
||||
@@ -123,14 +332,16 @@ export class CSVStorageManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* List all stored CSV files with sizes and dates
|
||||
* List all stored CSV files with cached row counts (invalidated on mtime/size changes)
|
||||
*/
|
||||
public static listAllFiles(): StorageFileMeta[] {
|
||||
const results: StorageFileMeta[] = [];
|
||||
if (!fs.existsSync(this.baseStorageDir)) {
|
||||
this.rowCountCache.clear();
|
||||
return results;
|
||||
}
|
||||
|
||||
const activePaths = new Set<string>();
|
||||
try {
|
||||
const clientDirs = fs.readdirSync(this.baseStorageDir);
|
||||
for (const clientDir of clientDirs) {
|
||||
@@ -142,15 +353,27 @@ export class CSVStorageManager {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
const fullFilePath = path.join(fullClientPath, file);
|
||||
const stats = fs.statSync(fullFilePath);
|
||||
activePaths.add(fullFilePath);
|
||||
|
||||
// Count lines approximately (or sample)
|
||||
const cached = this.rowCountCache.get(fullFilePath);
|
||||
let linesCount = 0;
|
||||
try {
|
||||
const content = fs.readFileSync(fullFilePath, 'utf8');
|
||||
linesCount = content.split('\n').filter((l) => l.trim().length > 0).length - 1; // subtract header
|
||||
if (linesCount < 0) linesCount = 0;
|
||||
} catch {
|
||||
linesCount = 0;
|
||||
if (
|
||||
cached &&
|
||||
cached.sizeBytes === stats.size &&
|
||||
cached.mtimeMs === stats.mtimeMs
|
||||
) {
|
||||
linesCount = cached.linesCount;
|
||||
} else {
|
||||
try {
|
||||
linesCount = this.countCsvDataRowsChunkedSync(fullFilePath);
|
||||
this.rowCountCache.set(fullFilePath, {
|
||||
sizeBytes: stats.size,
|
||||
mtimeMs: stats.mtimeMs,
|
||||
linesCount,
|
||||
});
|
||||
} catch {
|
||||
linesCount = 0;
|
||||
}
|
||||
}
|
||||
|
||||
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||
@@ -166,6 +389,12 @@ export class CSVStorageManager {
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const key of this.rowCountCache.keys()) {
|
||||
if (!activePaths.has(key)) {
|
||||
this.rowCountCache.delete(key);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Return whatever collected
|
||||
}
|
||||
@@ -174,12 +403,88 @@ export class CSVStorageManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Prune CSV partitions older than autoPruneDays
|
||||
* Non-blocking async listing using fs.promises + streaming row counting on cache misses
|
||||
*/
|
||||
public static async listAllFilesAsync(): Promise<StorageFileMeta[]> {
|
||||
const results: StorageFileMeta[] = [];
|
||||
if (!fs.existsSync(this.baseStorageDir)) {
|
||||
this.rowCountCache.clear();
|
||||
return results;
|
||||
}
|
||||
|
||||
const activePaths = new Set<string>();
|
||||
try {
|
||||
const clientDirs = await fs.promises.readdir(this.baseStorageDir);
|
||||
for (const clientDir of clientDirs) {
|
||||
const fullClientPath = path.join(this.baseStorageDir, clientDir);
|
||||
const dirStat = await fs.promises.stat(fullClientPath).catch(() => null);
|
||||
if (!dirStat || !dirStat.isDirectory()) continue;
|
||||
|
||||
const files = await fs.promises.readdir(fullClientPath);
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
const fullFilePath = path.join(fullClientPath, file);
|
||||
const stats = await fs.promises.stat(fullFilePath).catch(() => null);
|
||||
if (!stats) continue;
|
||||
activePaths.add(fullFilePath);
|
||||
|
||||
const cached = this.rowCountCache.get(fullFilePath);
|
||||
let linesCount = 0;
|
||||
if (
|
||||
cached &&
|
||||
cached.sizeBytes === stats.size &&
|
||||
cached.mtimeMs === stats.mtimeMs
|
||||
) {
|
||||
linesCount = cached.linesCount;
|
||||
} else {
|
||||
try {
|
||||
linesCount = await this.countCsvDataRowsStreamAsync(fullFilePath);
|
||||
this.rowCountCache.set(fullFilePath, {
|
||||
sizeBytes: stats.size,
|
||||
mtimeMs: stats.mtimeMs,
|
||||
linesCount,
|
||||
});
|
||||
} catch {
|
||||
linesCount = 0;
|
||||
}
|
||||
}
|
||||
|
||||
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||
|
||||
results.push({
|
||||
clientId: clientDir,
|
||||
date: datePart,
|
||||
filename: file,
|
||||
filePath: fullFilePath,
|
||||
sizeBytes: stats.size,
|
||||
linesCount,
|
||||
modifiedAt: stats.mtime,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const key of this.rowCountCache.keys()) {
|
||||
if (!activePaths.has(key)) {
|
||||
this.rowCountCache.delete(key);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Return whatever collected
|
||||
}
|
||||
|
||||
return results.sort((a, b) => b.modifiedAt.getTime() - a.modifiedAt.getTime());
|
||||
}
|
||||
|
||||
/**
|
||||
* Prune CSV partitions older than autoPruneDays without counting row contents
|
||||
*/
|
||||
public static pruneExpiredPartitions(autoPruneDays: number): number {
|
||||
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
|
||||
return 0;
|
||||
}
|
||||
if (!fs.existsSync(this.baseStorageDir)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const cutoffDate = new Date(
|
||||
Date.now() - Math.floor(autoPruneDays) * 86_400_000,
|
||||
@@ -188,28 +493,38 @@ export class CSVStorageManager {
|
||||
.split('T')[0];
|
||||
|
||||
let prunedCount = 0;
|
||||
const files = this.listAllFiles();
|
||||
for (const file of files) {
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(file.date) && file.date < cutoffDate) {
|
||||
try {
|
||||
if (fs.existsSync(file.filePath)) {
|
||||
fs.unlinkSync(file.filePath);
|
||||
prunedCount += 1;
|
||||
try {
|
||||
const clientDirs = fs.readdirSync(this.baseStorageDir);
|
||||
for (const clientDir of clientDirs) {
|
||||
const fullClientPath = path.join(this.baseStorageDir, clientDir);
|
||||
if (!fs.statSync(fullClientPath).isDirectory()) continue;
|
||||
|
||||
const files = fs.readdirSync(fullClientPath);
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
|
||||
const fullPath = path.join(fullClientPath, file);
|
||||
try {
|
||||
if (fs.existsSync(fullPath)) {
|
||||
fs.unlinkSync(fullPath);
|
||||
this.rowCountCache.delete(fullPath);
|
||||
prunedCount += 1;
|
||||
}
|
||||
} catch {
|
||||
// Ignore file removal error
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore file removal error
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore directory scan failure
|
||||
}
|
||||
|
||||
return prunedCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get aggregate storage statistics
|
||||
*/
|
||||
public static getStorageStats() {
|
||||
const files = this.listAllFiles();
|
||||
private static formatStorageStats(files: StorageFileMeta[]) {
|
||||
const totalBytes = files.reduce((acc, f) => acc + f.sizeBytes, 0);
|
||||
const totalRows = files.reduce((acc, f) => acc + f.linesCount, 0);
|
||||
const uniqueClients = new Set(files.map((f) => f.clientId)).size;
|
||||
@@ -231,10 +546,34 @@ export class CSVStorageManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads file content for export / download
|
||||
* Get aggregate storage statistics (sync cached)
|
||||
*/
|
||||
public static getStorageStats() {
|
||||
return this.formatStorageStats(this.listAllFiles());
|
||||
}
|
||||
|
||||
/**
|
||||
* Get aggregate storage statistics (non-blocking async)
|
||||
*/
|
||||
public static async getStorageStatsAsync() {
|
||||
const files = await this.listAllFilesAsync();
|
||||
return this.formatStorageStats(files);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads file content for export / download with path traversal protection
|
||||
*/
|
||||
public static getFileContent(clientId: string, filename: string): string | null {
|
||||
const fullPath = path.join(this.baseStorageDir, clientId, filename);
|
||||
const safeClientId = String(clientId || '').replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
const safeFilename = path.basename(String(filename || ''));
|
||||
if (!safeClientId || !safeFilename || !safeFilename.endsWith('.csv')) {
|
||||
return null;
|
||||
}
|
||||
const fullPath = path.resolve(this.baseStorageDir, safeClientId, safeFilename);
|
||||
const resolvedBase = path.resolve(this.baseStorageDir);
|
||||
if (!fullPath.startsWith(resolvedBase + path.sep)) {
|
||||
return null;
|
||||
}
|
||||
if (fs.existsSync(fullPath)) {
|
||||
return fs.readFileSync(fullPath, 'utf8');
|
||||
}
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
import { AuthService } from "../auth/jwt";
|
||||
import { AuthService, resolveDefaultJwtSecret } from "../auth/jwt";
|
||||
import { DataBufferEngine } from "../data/buffer";
|
||||
import { DatabaseEngine } from "../Database/database";
|
||||
import { SDKManager } from "../sdk/sdkManager";
|
||||
import { SecurityEngine } from "../Security/security";
|
||||
import {
|
||||
SecurityEngine,
|
||||
getInternalIngestionHeaders,
|
||||
} from "../Security/security";
|
||||
import { CSVStorageManager } from "../Storage/csvStorage";
|
||||
|
||||
export interface AdminSettings {
|
||||
@@ -35,9 +38,7 @@ function getDefaultSettings(): AdminSettings {
|
||||
batchFlushThreshold: 500,
|
||||
batchFlushIntervalMs: 2000,
|
||||
fileRotationMb: 100,
|
||||
jwtSecret:
|
||||
process.env.DATASTACK_JWT_SECRET ||
|
||||
"datastack-jwt-default-secret-key-32-chars-long",
|
||||
jwtSecret: resolveDefaultJwtSecret(),
|
||||
enableIdempotency: true,
|
||||
autoPruneDays: 30,
|
||||
updatedAt: new Date().toISOString(),
|
||||
@@ -62,7 +63,9 @@ export class SettingsManager {
|
||||
AuthService.setSecret(settings.jwtSecret);
|
||||
}
|
||||
CSVStorageManager.maxFileSizeBytes = settings.fileRotationMb * 1024 * 1024;
|
||||
CSVStorageManager.pruneExpiredPartitions(settings.autoPruneDays);
|
||||
if (settings.autoPruneDays > 0) {
|
||||
CSVStorageManager.pruneExpiredPartitions(settings.autoPruneDays);
|
||||
}
|
||||
}
|
||||
|
||||
private static async syncFromDatabase(): Promise<AdminSettings | null> {
|
||||
@@ -83,6 +86,10 @@ export class SettingsManager {
|
||||
|
||||
if (result.rowCount && result.rowCount > 0) {
|
||||
const row = result.rows[0];
|
||||
const parsedPruneDays =
|
||||
row.auto_prune_days !== undefined && row.auto_prune_days !== null
|
||||
? Number(row.auto_prune_days)
|
||||
: NaN;
|
||||
const loaded: AdminSettings = {
|
||||
serverPort: Number(row.server_port) || 4000,
|
||||
webPort: Number(row.web_port) || 3000,
|
||||
@@ -91,12 +98,9 @@ export class SettingsManager {
|
||||
batchFlushThreshold: Number(row.batch_flush_threshold) || 500,
|
||||
batchFlushIntervalMs: Number(row.batch_flush_interval_ms) || 2000,
|
||||
fileRotationMb: Number(row.file_rotation_mb) || 100,
|
||||
jwtSecret:
|
||||
String(row.jwt_secret || "") ||
|
||||
process.env.DATASTACK_JWT_SECRET ||
|
||||
"datastack-jwt-default-secret-key-32-chars-long",
|
||||
jwtSecret: String(row.jwt_secret || "").trim() || resolveDefaultJwtSecret(),
|
||||
enableIdempotency: Boolean(row.enable_idempotency),
|
||||
autoPruneDays: Number(row.auto_prune_days) || 30,
|
||||
autoPruneDays: Number.isFinite(parsedPruneDays) ? parsedPruneDays : 30,
|
||||
updatedAt: row.updated_at
|
||||
? new Date(row.updated_at as string | Date).toISOString()
|
||||
: new Date().toISOString(),
|
||||
@@ -214,8 +218,46 @@ export class SettingsManager {
|
||||
};
|
||||
}
|
||||
|
||||
private static async notifyIngestionSettingsSync(
|
||||
settings: AdminSettings,
|
||||
authJwtSecret: string,
|
||||
jwtSecretChanged: boolean,
|
||||
): Promise<void> {
|
||||
const headers = getInternalIngestionHeaders(authJwtSecret);
|
||||
if (!headers["x-datastack-internal-secret"]) return;
|
||||
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||
).replace(/\/$/, "");
|
||||
|
||||
try {
|
||||
await fetch(`${ingestionBaseUrl}/internal/settings-sync`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...headers,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
maxEventsPerSec: settings.maxEventsPerSec,
|
||||
maxPayloadSizeKb: settings.maxPayloadSizeKb,
|
||||
batchFlushThreshold: settings.batchFlushThreshold,
|
||||
batchFlushIntervalMs: settings.batchFlushIntervalMs,
|
||||
fileRotationMb: settings.fileRotationMb,
|
||||
enableIdempotency: settings.enableIdempotency,
|
||||
autoPruneDays: settings.autoPruneDays,
|
||||
...(jwtSecretChanged ? { jwtSecret: settings.jwtSecret } : {}),
|
||||
}),
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(1500),
|
||||
});
|
||||
} catch {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
}
|
||||
}
|
||||
|
||||
public static updateSettings(updates: Partial<AdminSettings>): AdminSettings {
|
||||
const current = getSettingsState();
|
||||
const previousJwtSecret = current.jwtSecret;
|
||||
const jwtSecretChanged =
|
||||
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
|
||||
const nextSettings: AdminSettings = {
|
||||
@@ -229,6 +271,13 @@ export class SettingsManager {
|
||||
SDKManager.resignAllClientTokens();
|
||||
}
|
||||
void this.persistSettingsToDb(nextSettings);
|
||||
void this.notifyIngestionSettingsSync(
|
||||
nextSettings,
|
||||
previousJwtSecret,
|
||||
jwtSecretChanged,
|
||||
).then(() =>
|
||||
jwtSecretChanged ? SDKManager.syncAllClientsToIngestionAsync() : undefined,
|
||||
);
|
||||
return { ...nextSettings };
|
||||
}
|
||||
|
||||
@@ -236,6 +285,7 @@ export class SettingsManager {
|
||||
updates: Partial<AdminSettings>,
|
||||
): Promise<AdminSettings> {
|
||||
const current = await this.getSettingsAsync();
|
||||
const previousJwtSecret = current.jwtSecret;
|
||||
const jwtSecretChanged =
|
||||
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
|
||||
const nextSettings: AdminSettings = {
|
||||
@@ -249,6 +299,14 @@ export class SettingsManager {
|
||||
await SDKManager.resignAllClientTokensAsync();
|
||||
}
|
||||
await this.persistSettingsToDb(nextSettings);
|
||||
await this.notifyIngestionSettingsSync(
|
||||
nextSettings,
|
||||
previousJwtSecret,
|
||||
jwtSecretChanged,
|
||||
);
|
||||
if (jwtSecretChanged) {
|
||||
await SDKManager.syncAllClientsToIngestionAsync();
|
||||
}
|
||||
return { ...nextSettings };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,44 +1,93 @@
|
||||
import { SettingsManager } from '../adminSettings/settingsManager';
|
||||
import { SDKManager } from '../sdk/sdkManager';
|
||||
import { CSVStorageManager } from '../Storage/csvStorage';
|
||||
import { DataBufferEngine } from '../data/buffer';
|
||||
import { SecurityEngine } from '../Security/security';
|
||||
import {
|
||||
SecurityEngine,
|
||||
getInternalIngestionHeaders,
|
||||
} from '../Security/security';
|
||||
|
||||
export class ApiEngine {
|
||||
/**
|
||||
* Retrieves overall telemetry and health metrics for the dashboard
|
||||
*/
|
||||
public static async getDashboardMetrics() {
|
||||
const storageStats = CSVStorageManager.getStorageStats();
|
||||
const localBufferStats = DataBufferEngine.getBufferStats();
|
||||
const clients = await SDKManager.listClientsAsync();
|
||||
const settings = await SettingsManager.getSettingsAsync();
|
||||
const [storageStats, localBufferStats, clients] = await Promise.all([
|
||||
CSVStorageManager.getStorageStatsAsync(),
|
||||
DataBufferEngine.getBufferStatsAsync(),
|
||||
SDKManager.listClientsAsync(),
|
||||
]);
|
||||
const activeClients = clients.filter((c) => c.status === 'active').length;
|
||||
let activeSockets = 0;
|
||||
let ingestionBufferSize = 0;
|
||||
let ingestionLocalBufferSize = 0;
|
||||
let ingestionRedisBufferSize = 0;
|
||||
let ingestionRedisConnected = false;
|
||||
let remoteSecurityStats: {
|
||||
activeIdempotencyKeys?: number;
|
||||
rateLimitedClientsCount?: number;
|
||||
maxPayloadSizeBytes?: number;
|
||||
} | null = null;
|
||||
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
|
||||
).replace(/\/$/, '');
|
||||
const internalHeaders = getInternalIngestionHeaders();
|
||||
|
||||
try {
|
||||
const res = await fetch(`${ingestionBaseUrl}/health`, {
|
||||
const res = await fetch(`${ingestionBaseUrl}/internal/metrics`, {
|
||||
headers: internalHeaders,
|
||||
cache: 'no-store',
|
||||
signal: AbortSignal.timeout(1000),
|
||||
});
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
activeSockets = Number(data.activeSockets) || 0;
|
||||
ingestionBufferSize = Number(data.bufferSize) || 0;
|
||||
ingestionRedisConnected = Boolean(data.redisConnected);
|
||||
if (data.localBufferSize !== undefined) {
|
||||
ingestionLocalBufferSize = Number(data.localBufferSize) || 0;
|
||||
ingestionRedisBufferSize = Number(data.redisBufferSize) || 0;
|
||||
} else {
|
||||
ingestionLocalBufferSize = Number(data.bufferSize) || 0;
|
||||
}
|
||||
if (data.securityStats && typeof data.securityStats === 'object') {
|
||||
remoteSecurityStats = data.securityStats;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ingestion server may not be running in single-process dev mode
|
||||
activeSockets = 0;
|
||||
ingestionBufferSize = 0;
|
||||
ingestionLocalBufferSize = 0;
|
||||
ingestionRedisBufferSize = 0;
|
||||
}
|
||||
|
||||
const securityStats = SecurityEngine.getStats();
|
||||
const localSecurityStats = SecurityEngine.getStats();
|
||||
const securityStats = {
|
||||
activeIdempotencyKeys:
|
||||
localSecurityStats.activeIdempotencyKeys +
|
||||
(Number(remoteSecurityStats?.activeIdempotencyKeys) || 0),
|
||||
rateLimitedClientsCount:
|
||||
localSecurityStats.rateLimitedClientsCount +
|
||||
(Number(remoteSecurityStats?.rateLimitedClientsCount) || 0),
|
||||
maxPayloadSizeBytes: Math.min(
|
||||
localSecurityStats.maxPayloadSizeBytes,
|
||||
Number(remoteSecurityStats?.maxPayloadSizeBytes) ||
|
||||
localSecurityStats.maxPayloadSizeBytes,
|
||||
),
|
||||
};
|
||||
|
||||
const effectiveRedisBufferSize = Math.max(
|
||||
localBufferStats.redisBufferSize || 0,
|
||||
ingestionRedisBufferSize,
|
||||
);
|
||||
const bufferStats = {
|
||||
...localBufferStats,
|
||||
redisConnected: localBufferStats.redisConnected || ingestionRedisConnected,
|
||||
redisBufferSize: effectiveRedisBufferSize,
|
||||
currentBufferSize:
|
||||
localBufferStats.currentBufferSize + ingestionBufferSize,
|
||||
localBufferStats.localBufferSize +
|
||||
ingestionLocalBufferSize +
|
||||
effectiveRedisBufferSize,
|
||||
};
|
||||
|
||||
return {
|
||||
@@ -49,8 +98,8 @@ export class ApiEngine {
|
||||
storageStats,
|
||||
securityStats,
|
||||
runtime: {
|
||||
serverPort: 4000,
|
||||
webPort: 3000,
|
||||
serverPort: settings.serverPort || 4000,
|
||||
webPort: settings.webPort || 3000,
|
||||
uptimeSeconds: Math.floor(process.uptime()),
|
||||
memoryUsageMb: (process.memoryUsage().rss / (1024 * 1024)).toFixed(1),
|
||||
nodeVersion: process.version,
|
||||
@@ -59,7 +108,7 @@ export class ApiEngine {
|
||||
}
|
||||
|
||||
/**
|
||||
* Filters and streams stored CSV data for export
|
||||
* Filters and lists stored CSV files for export
|
||||
*/
|
||||
public static getExportFiles(clientId?: string) {
|
||||
const allFiles = CSVStorageManager.listAllFiles();
|
||||
@@ -68,6 +117,14 @@ export class ApiEngine {
|
||||
}
|
||||
return allFiles;
|
||||
}
|
||||
|
||||
public static async getExportFilesAsync(clientId?: string) {
|
||||
const allFiles = await CSVStorageManager.listAllFilesAsync();
|
||||
if (clientId && clientId !== 'all') {
|
||||
return allFiles.filter((f) => f.clientId === clientId);
|
||||
}
|
||||
return allFiles;
|
||||
}
|
||||
}
|
||||
|
||||
export default ApiEngine;
|
||||
@@ -1,8 +1,23 @@
|
||||
const encoder = new TextEncoder();
|
||||
|
||||
interface AdminSessionPayload {
|
||||
const DEV_DEFAULT_SESSION_SECRET =
|
||||
"datastack-admin-session-default-secret-32-chars";
|
||||
|
||||
export interface AdminSessionPayload {
|
||||
email: string;
|
||||
expiresAt: number;
|
||||
tokenVersion: number;
|
||||
}
|
||||
|
||||
export function getAdminSessionSecret(): string | null {
|
||||
const configured = process.env.DATASTACK_ADMIN_SESSION_SECRET?.trim() || "";
|
||||
if (process.env.NODE_ENV === "production") {
|
||||
return configured.length >= 32 ? configured : null;
|
||||
}
|
||||
if (configured.length > 0) {
|
||||
return configured.length >= 32 ? configured : null;
|
||||
}
|
||||
return DEV_DEFAULT_SESSION_SECRET;
|
||||
}
|
||||
|
||||
function encodeBase64Url(bytes: Uint8Array) {
|
||||
@@ -34,9 +49,14 @@ export async function createAdminSessionToken(
|
||||
email: string,
|
||||
secret: string,
|
||||
expiresAt: number,
|
||||
tokenVersion = 1,
|
||||
) {
|
||||
const safeVersion =
|
||||
Number.isInteger(tokenVersion) && tokenVersion >= 1 ? tokenVersion : 1;
|
||||
const payload = encodeBase64Url(
|
||||
encoder.encode(JSON.stringify({ email, expiresAt })),
|
||||
encoder.encode(
|
||||
JSON.stringify({ email, expiresAt, tokenVersion: safeVersion }),
|
||||
),
|
||||
);
|
||||
const signature = await crypto.subtle.sign(
|
||||
"HMAC",
|
||||
@@ -50,6 +70,7 @@ export async function getAdminSessionPayload(
|
||||
token: string,
|
||||
secret: string,
|
||||
): Promise<AdminSessionPayload | null> {
|
||||
if (!secret || secret.length < 32) return null;
|
||||
const [payload, signature, extra] = token.split(".");
|
||||
if (!payload || !signature || extra) return null;
|
||||
|
||||
@@ -68,7 +89,10 @@ export async function getAdminSessionPayload(
|
||||
if (
|
||||
typeof session.email === "string" &&
|
||||
typeof session.expiresAt === "number" &&
|
||||
session.expiresAt > Date.now()
|
||||
session.expiresAt > Date.now() &&
|
||||
typeof session.tokenVersion === "number" &&
|
||||
Number.isInteger(session.tokenVersion) &&
|
||||
session.tokenVersion >= 1
|
||||
) {
|
||||
return session;
|
||||
}
|
||||
|
||||
+279
-163
@@ -6,15 +6,22 @@ import {
|
||||
timingSafeEqual,
|
||||
} from "node:crypto";
|
||||
import { Pool, type PoolClient } from "pg";
|
||||
import {
|
||||
type AdminSessionPayload,
|
||||
getAdminSessionPayload,
|
||||
getAdminSessionSecret,
|
||||
} from "./adminSession";
|
||||
|
||||
const PASSWORD_HASH_BYTES = 64;
|
||||
const PASSWORD_SALT_BYTES = 16;
|
||||
const DATABASE_LOCK_ID = "72025102501";
|
||||
const LOGIN_ATTEMPT_LIMIT = 5;
|
||||
const IP_LOGIN_ATTEMPT_LIMIT = 25;
|
||||
const LOGIN_WINDOW_MS = 15 * 60 * 1000;
|
||||
const LOGIN_BLOCK_MS = 15 * 60 * 1000;
|
||||
const LOCAL_THROTTLE_MAX_ENTRIES = 5000;
|
||||
const LOCAL_THROTTLE_CLEANUP_MS = 60 * 1000;
|
||||
const DEV_DEFAULT_SETUP_KEY = "datastack-default-setup-key-32-chars";
|
||||
|
||||
interface LoginThrottleEntry {
|
||||
windowStartedAt: number;
|
||||
@@ -28,11 +35,17 @@ interface LoginThrottleDecision {
|
||||
retryAfterSeconds: number;
|
||||
}
|
||||
|
||||
export interface LoginThrottleBucket {
|
||||
keyHash: string;
|
||||
limit: number;
|
||||
}
|
||||
|
||||
interface InMemoryAdmin {
|
||||
id: number;
|
||||
email: string;
|
||||
passwordSalt: Buffer;
|
||||
passwordHash: Buffer;
|
||||
tokenVersion: number;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
@@ -47,8 +60,12 @@ const localLoginThrottle = new Map<string, LoginThrottleEntry>();
|
||||
let lastLocalThrottleCleanup = 0;
|
||||
let lastDatabaseThrottleCleanup = 0;
|
||||
|
||||
function isProductionEnv() {
|
||||
return process.env.NODE_ENV === "production";
|
||||
}
|
||||
|
||||
function getPool(): Pool | null {
|
||||
const connectionString = process.env.DATABASE_URL;
|
||||
const connectionString = process.env.DATABASE_URL?.trim();
|
||||
if (!connectionString) {
|
||||
return null;
|
||||
}
|
||||
@@ -74,17 +91,24 @@ async function ensureAdminTable() {
|
||||
if (!pool) return;
|
||||
|
||||
globals.__dataStackAdminSchema ??= Promise.all([
|
||||
pool.query(
|
||||
`
|
||||
pool
|
||||
.query(
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS admin_users (
|
||||
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
|
||||
email TEXT NOT NULL UNIQUE,
|
||||
password_salt BYTEA NOT NULL,
|
||||
password_hash BYTEA NOT NULL,
|
||||
token_version INTEGER NOT NULL DEFAULT 1,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`,
|
||||
),
|
||||
)
|
||||
.then(() =>
|
||||
pool.query(
|
||||
`ALTER TABLE admin_users ADD COLUMN IF NOT EXISTS token_version INTEGER NOT NULL DEFAULT 1`,
|
||||
),
|
||||
),
|
||||
pool.query(
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS admin_login_attempts (
|
||||
@@ -121,8 +145,41 @@ function matchesSecret(candidate: string, expected: string) {
|
||||
return timingSafeEqual(candidateHash, expectedHash);
|
||||
}
|
||||
|
||||
function createLoginThrottleKeys(email: string, secret: string) {
|
||||
return [createHmac("sha256", secret).update(`email:${email}`).digest("hex")];
|
||||
export function getAdminSetupKey(): string | null {
|
||||
const configured = process.env.DATASTACK_ADMIN_SETUP_KEY?.trim() || "";
|
||||
if (isProductionEnv()) {
|
||||
return configured.length >= 32 ? configured : null;
|
||||
}
|
||||
if (configured.length > 0) {
|
||||
return configured.length >= 32 ? configured : null;
|
||||
}
|
||||
return DEV_DEFAULT_SETUP_KEY;
|
||||
}
|
||||
|
||||
export function createLoginThrottleBuckets(
|
||||
email: string,
|
||||
secret: string,
|
||||
ip = "unknown",
|
||||
): LoginThrottleBucket[] {
|
||||
const cleanEmail = email.trim().toLowerCase();
|
||||
const cleanIp = ip.trim().toLowerCase() || "unknown";
|
||||
const buckets: LoginThrottleBucket[] = [
|
||||
{
|
||||
keyHash: createHmac("sha256", secret)
|
||||
.update(`ip_email:${cleanIp}:${cleanEmail}`)
|
||||
.digest("hex"),
|
||||
limit: LOGIN_ATTEMPT_LIMIT,
|
||||
},
|
||||
];
|
||||
if (cleanIp !== "unknown") {
|
||||
buckets.push({
|
||||
keyHash: createHmac("sha256", secret)
|
||||
.update(`ip:${cleanIp}`)
|
||||
.digest("hex"),
|
||||
limit: IP_LOGIN_ATTEMPT_LIMIT,
|
||||
});
|
||||
}
|
||||
return buckets;
|
||||
}
|
||||
|
||||
function getAdvisoryLockId(keyHash: string) {
|
||||
@@ -157,14 +214,14 @@ function cleanupLocalLoginThrottle(now: number) {
|
||||
}
|
||||
|
||||
function consumeLocalLoginAttempt(
|
||||
keyHashes: string[],
|
||||
buckets: LoginThrottleBucket[],
|
||||
now: number,
|
||||
): LoginThrottleDecision {
|
||||
cleanupLocalLoginThrottle(now);
|
||||
|
||||
const blockedUntil = keyHashes.reduce(
|
||||
(latest, keyHash) =>
|
||||
Math.max(latest, localLoginThrottle.get(keyHash)?.blockedUntil ?? 0),
|
||||
const blockedUntil = buckets.reduce(
|
||||
(latest, bucket) =>
|
||||
Math.max(latest, localLoginThrottle.get(bucket.keyHash)?.blockedUntil ?? 0),
|
||||
0,
|
||||
);
|
||||
if (blockedUntil > now) {
|
||||
@@ -174,16 +231,16 @@ function consumeLocalLoginAttempt(
|
||||
};
|
||||
}
|
||||
|
||||
for (const keyHash of keyHashes) {
|
||||
const previous = localLoginThrottle.get(keyHash);
|
||||
for (const bucket of buckets) {
|
||||
const previous = localLoginThrottle.get(bucket.keyHash);
|
||||
const windowExpired =
|
||||
!previous || now - previous.windowStartedAt >= LOGIN_WINDOW_MS;
|
||||
const failedAttempts = windowExpired ? 1 : previous.failedAttempts + 1;
|
||||
localLoginThrottle.set(keyHash, {
|
||||
localLoginThrottle.set(bucket.keyHash, {
|
||||
windowStartedAt: windowExpired ? now : previous.windowStartedAt,
|
||||
failedAttempts,
|
||||
blockedUntil:
|
||||
failedAttempts >= LOGIN_ATTEMPT_LIMIT ? now + LOGIN_BLOCK_MS : 0,
|
||||
failedAttempts >= bucket.limit ? now + LOGIN_BLOCK_MS : 0,
|
||||
updatedAt: now,
|
||||
});
|
||||
}
|
||||
@@ -192,12 +249,13 @@ function consumeLocalLoginAttempt(
|
||||
}
|
||||
|
||||
async function consumeDatabaseLoginAttempt(
|
||||
keyHashes: string[],
|
||||
buckets: LoginThrottleBucket[],
|
||||
now: number,
|
||||
): Promise<LoginThrottleDecision> {
|
||||
const pool = getPool();
|
||||
if (!pool) return consumeLocalLoginAttempt(keyHashes, now);
|
||||
if (!pool) return consumeLocalLoginAttempt(buckets, now);
|
||||
await ensureAdminTable();
|
||||
const keyHashes = buckets.map((b) => b.keyHash);
|
||||
const client = await pool.connect();
|
||||
|
||||
try {
|
||||
@@ -229,8 +287,8 @@ async function consumeDatabaseLoginAttempt(
|
||||
};
|
||||
}
|
||||
|
||||
for (const keyHash of keyHashes) {
|
||||
const previous = entries.get(keyHash);
|
||||
for (const bucket of buckets) {
|
||||
const previous = entries.get(bucket.keyHash);
|
||||
const windowStartedAt = previous
|
||||
? new Date(previous.window_started_at as string | Date).getTime()
|
||||
: 0;
|
||||
@@ -243,7 +301,7 @@ async function consumeDatabaseLoginAttempt(
|
||||
? new Date(now)
|
||||
: new Date(windowStartedAt);
|
||||
const nextBlockedUntil =
|
||||
failedAttempts >= LOGIN_ATTEMPT_LIMIT
|
||||
failedAttempts >= bucket.limit
|
||||
? new Date(now + LOGIN_BLOCK_MS)
|
||||
: null;
|
||||
|
||||
@@ -256,7 +314,7 @@ async function consumeDatabaseLoginAttempt(
|
||||
failed_attempts = EXCLUDED.failed_attempts,
|
||||
blocked_until = EXCLUDED.blocked_until,
|
||||
updated_at = NOW()`,
|
||||
[keyHash, nextWindowStartedAt, failedAttempts, nextBlockedUntil],
|
||||
[bucket.keyHash, nextWindowStartedAt, failedAttempts, nextBlockedUntil],
|
||||
);
|
||||
}
|
||||
|
||||
@@ -279,13 +337,20 @@ async function consumeDatabaseLoginAttempt(
|
||||
|
||||
async function getOrInitInMemoryAdmin(): Promise<InMemoryAdmin> {
|
||||
if (!globals.__dataStackInMemoryAdmin) {
|
||||
const envPassword = process.env.DATASTACK_ADMIN_PASSWORD;
|
||||
const allowDemo =
|
||||
!isProductionEnv() && process.env.DATASTACK_ALLOW_DEMO_ADMIN !== "false";
|
||||
|
||||
if (!envPassword && !allowDemo) {
|
||||
throw new Error("Admin account has not completed initial setup.");
|
||||
}
|
||||
|
||||
const defaultEmail = (
|
||||
process.env.DATASTACK_ADMIN_EMAIL || "admin@datastack.dev"
|
||||
)
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const defaultPassword =
|
||||
process.env.DATASTACK_ADMIN_PASSWORD || "admin12345678";
|
||||
const defaultPassword = envPassword || "admin12345678";
|
||||
const salt = randomBytes(PASSWORD_SALT_BYTES);
|
||||
const hash = await derivePasswordHash(defaultPassword, salt);
|
||||
globals.__dataStackInMemoryAdmin = {
|
||||
@@ -293,6 +358,7 @@ async function getOrInitInMemoryAdmin(): Promise<InMemoryAdmin> {
|
||||
email: defaultEmail,
|
||||
passwordSalt: salt,
|
||||
passwordHash: hash,
|
||||
tokenVersion: 1,
|
||||
createdAt: new Date(),
|
||||
};
|
||||
}
|
||||
@@ -302,24 +368,26 @@ async function getOrInitInMemoryAdmin(): Promise<InMemoryAdmin> {
|
||||
export async function consumeAdminLoginAttempt(
|
||||
email: string,
|
||||
secret: string,
|
||||
ip = "unknown",
|
||||
): Promise<LoginThrottleDecision> {
|
||||
const keyHashes = createLoginThrottleKeys(email, secret);
|
||||
const buckets = createLoginThrottleBuckets(email, secret, ip);
|
||||
const now = Date.now();
|
||||
|
||||
const pool = getPool();
|
||||
if (!pool) {
|
||||
return consumeLocalLoginAttempt(keyHashes, now);
|
||||
return consumeLocalLoginAttempt(buckets, now);
|
||||
}
|
||||
|
||||
try {
|
||||
return await consumeDatabaseLoginAttempt(keyHashes, now);
|
||||
} catch {
|
||||
return consumeLocalLoginAttempt(keyHashes, now);
|
||||
}
|
||||
return await consumeDatabaseLoginAttempt(buckets, now);
|
||||
}
|
||||
|
||||
export async function clearAdminLoginAttempts(email: string, secret: string) {
|
||||
const keyHashes = createLoginThrottleKeys(email, secret);
|
||||
export async function clearAdminLoginAttempts(
|
||||
email: string,
|
||||
secret: string,
|
||||
ip = "unknown",
|
||||
) {
|
||||
const buckets = createLoginThrottleBuckets(email, secret, ip);
|
||||
const keyHashes = buckets.map((b) => b.keyHash);
|
||||
|
||||
const pool = getPool();
|
||||
if (!pool) {
|
||||
@@ -327,50 +395,45 @@ export async function clearAdminLoginAttempts(email: string, secret: string) {
|
||||
return;
|
||||
}
|
||||
|
||||
await ensureAdminTable();
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await ensureAdminTable();
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
await lockLoginThrottleKeys(client, keyHashes);
|
||||
await client.query(
|
||||
"DELETE FROM admin_login_attempts WHERE key_hash = ANY($1::text[])",
|
||||
[keyHashes],
|
||||
);
|
||||
await client.query("COMMIT");
|
||||
} catch (error) {
|
||||
await client.query("ROLLBACK").catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
} catch {
|
||||
keyHashes.forEach((keyHash) => localLoginThrottle.delete(keyHash));
|
||||
await client.query("BEGIN");
|
||||
await lockLoginThrottleKeys(client, keyHashes);
|
||||
await client.query(
|
||||
"DELETE FROM admin_login_attempts WHERE key_hash = ANY($1::text[])",
|
||||
[keyHashes],
|
||||
);
|
||||
await client.query("COMMIT");
|
||||
} catch (error) {
|
||||
await client.query("ROLLBACK").catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
export function verifyAdminSetupKey(candidate: string) {
|
||||
const expected =
|
||||
process.env.DATASTACK_ADMIN_SETUP_KEY ||
|
||||
"datastack-default-setup-key-32-chars";
|
||||
if (!expected || expected.length < 32) return false;
|
||||
const expected = getAdminSetupKey();
|
||||
if (!expected || expected.length < 32 || !candidate) return false;
|
||||
return matchesSecret(candidate, expected);
|
||||
}
|
||||
|
||||
export async function isAdminSetupComplete() {
|
||||
const pool = getPool();
|
||||
if (!pool) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
await ensureAdminTable();
|
||||
const result = await pool.query(
|
||||
"SELECT 1 FROM admin_users WHERE id = 1",
|
||||
);
|
||||
return result.rowCount === 1;
|
||||
} catch {
|
||||
return true;
|
||||
if (globals.__dataStackInMemoryAdmin) {
|
||||
return true;
|
||||
}
|
||||
if (isProductionEnv()) {
|
||||
return Boolean(process.env.DATASTACK_ADMIN_PASSWORD);
|
||||
}
|
||||
return process.env.DATASTACK_ALLOW_DEMO_ADMIN !== "false";
|
||||
}
|
||||
|
||||
await ensureAdminTable();
|
||||
const result = await pool.query("SELECT 1 FROM admin_users WHERE id = 1");
|
||||
return result.rowCount === 1;
|
||||
}
|
||||
|
||||
export async function createInitialAdmin(email: string, password: string) {
|
||||
@@ -380,136 +443,176 @@ export async function createInitialAdmin(email: string, password: string) {
|
||||
const pool = getPool();
|
||||
|
||||
if (!pool) {
|
||||
if (globals.__dataStackInMemoryAdmin) {
|
||||
return false;
|
||||
}
|
||||
globals.__dataStackInMemoryAdmin = {
|
||||
id: 1,
|
||||
email: cleanEmail,
|
||||
passwordSalt: salt,
|
||||
passwordHash,
|
||||
tokenVersion: 1,
|
||||
createdAt: new Date(),
|
||||
};
|
||||
return true;
|
||||
}
|
||||
|
||||
await ensureAdminTable();
|
||||
const client = await pool.connect();
|
||||
|
||||
try {
|
||||
await ensureAdminTable();
|
||||
const client = await pool.connect();
|
||||
await client.query("BEGIN");
|
||||
await client.query("SELECT pg_advisory_xact_lock($1::bigint)", [
|
||||
DATABASE_LOCK_ID,
|
||||
]);
|
||||
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
await client.query("SELECT pg_advisory_xact_lock($1::bigint)", [
|
||||
DATABASE_LOCK_ID,
|
||||
]);
|
||||
|
||||
const existing = await client.query(
|
||||
"SELECT 1 FROM admin_users WHERE id = 1",
|
||||
);
|
||||
if (existing.rowCount) {
|
||||
await client.query("ROLLBACK");
|
||||
return false;
|
||||
}
|
||||
|
||||
await client.query(
|
||||
`INSERT INTO admin_users (id, email, password_salt, password_hash)
|
||||
VALUES (1, $1, $2, $3)`,
|
||||
[cleanEmail, salt, passwordHash],
|
||||
);
|
||||
await client.query("COMMIT");
|
||||
return true;
|
||||
} catch (error) {
|
||||
await client.query("ROLLBACK").catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
const existing = await client.query(
|
||||
"SELECT 1 FROM admin_users WHERE id = 1",
|
||||
);
|
||||
if (existing.rowCount) {
|
||||
await client.query("ROLLBACK");
|
||||
return false;
|
||||
}
|
||||
} catch {
|
||||
globals.__dataStackInMemoryAdmin = {
|
||||
id: 1,
|
||||
email: cleanEmail,
|
||||
passwordSalt: salt,
|
||||
passwordHash,
|
||||
createdAt: new Date(),
|
||||
};
|
||||
|
||||
await client.query(
|
||||
`INSERT INTO admin_users (id, email, password_salt, password_hash, token_version)
|
||||
VALUES (1, $1, $2, $3, 1)`,
|
||||
[cleanEmail, salt, passwordHash],
|
||||
);
|
||||
await client.query("COMMIT");
|
||||
return true;
|
||||
} catch (error) {
|
||||
await client.query("ROLLBACK").catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
export async function verifyAdminCredentials(email: string, password: string) {
|
||||
export async function authenticateAdminCredentials(
|
||||
email: string,
|
||||
password: string,
|
||||
): Promise<{ authenticated: boolean; tokenVersion: number }> {
|
||||
const cleanEmail = email.trim().toLowerCase();
|
||||
const pool = getPool();
|
||||
|
||||
if (pool) {
|
||||
try {
|
||||
await ensureAdminTable();
|
||||
const result = await pool.query(
|
||||
`SELECT email, password_salt, password_hash
|
||||
FROM admin_users
|
||||
WHERE id = 1`,
|
||||
);
|
||||
if (result.rowCount && result.rowCount > 0) {
|
||||
const admin = result.rows[0] as {
|
||||
email: string;
|
||||
password_salt: Buffer;
|
||||
password_hash: Buffer;
|
||||
};
|
||||
const emailMatches = matchesSecret(
|
||||
cleanEmail,
|
||||
admin.email.trim().toLowerCase(),
|
||||
);
|
||||
const actualHash = await derivePasswordHash(
|
||||
password,
|
||||
admin.password_salt,
|
||||
);
|
||||
const passwordMatches = timingSafeEqual(
|
||||
actualHash,
|
||||
admin.password_hash,
|
||||
);
|
||||
return emailMatches && passwordMatches;
|
||||
}
|
||||
if (!process.env.DATASTACK_ADMIN_PASSWORD) {
|
||||
return false;
|
||||
}
|
||||
} catch {
|
||||
// Database connection error, fall back to in-memory admin
|
||||
await ensureAdminTable();
|
||||
const result = await pool.query(
|
||||
`SELECT email, password_salt, password_hash, COALESCE(token_version, 1) AS token_version
|
||||
FROM admin_users
|
||||
WHERE id = 1`,
|
||||
);
|
||||
if (!result.rowCount || result.rowCount === 0) {
|
||||
return { authenticated: false, tokenVersion: 0 };
|
||||
}
|
||||
const admin = result.rows[0] as {
|
||||
email: string;
|
||||
password_salt: Buffer;
|
||||
password_hash: Buffer;
|
||||
token_version: number;
|
||||
};
|
||||
const emailMatches = matchesSecret(
|
||||
cleanEmail,
|
||||
admin.email.trim().toLowerCase(),
|
||||
);
|
||||
const actualHash = await derivePasswordHash(password, admin.password_salt);
|
||||
const passwordMatches = timingSafeEqual(actualHash, admin.password_hash);
|
||||
return {
|
||||
authenticated: emailMatches && passwordMatches,
|
||||
tokenVersion: Number(admin.token_version) || 1,
|
||||
};
|
||||
}
|
||||
|
||||
const inMem = await getOrInitInMemoryAdmin();
|
||||
if (!matchesSecret(cleanEmail, inMem.email)) {
|
||||
return false;
|
||||
return { authenticated: false, tokenVersion: 0 };
|
||||
}
|
||||
const actualHash = await derivePasswordHash(password, inMem.passwordSalt);
|
||||
return timingSafeEqual(actualHash, inMem.passwordHash);
|
||||
const authenticated = timingSafeEqual(actualHash, inMem.passwordHash);
|
||||
return {
|
||||
authenticated,
|
||||
tokenVersion: authenticated ? inMem.tokenVersion : 0,
|
||||
};
|
||||
}
|
||||
|
||||
export async function verifyAdminCredentials(email: string, password: string) {
|
||||
const result = await authenticateAdminCredentials(email, password);
|
||||
return result.authenticated;
|
||||
}
|
||||
|
||||
export async function getAdminAccountInfo(fallbackEmail?: string) {
|
||||
const pool = getPool();
|
||||
if (pool) {
|
||||
try {
|
||||
await ensureAdminTable();
|
||||
const result = await pool.query(
|
||||
`SELECT email, created_at FROM admin_users WHERE id = 1`,
|
||||
);
|
||||
if (result.rowCount && result.rowCount > 0) {
|
||||
const row = result.rows[0] as { email: string; created_at: Date | string };
|
||||
return {
|
||||
email: row.email,
|
||||
createdAt: new Date(row.created_at).toISOString(),
|
||||
storageMode: "PostgreSQL" as const,
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
// Fall back to in-memory admin
|
||||
await ensureAdminTable();
|
||||
const result = await pool.query(
|
||||
`SELECT email, created_at, COALESCE(token_version, 1) AS token_version
|
||||
FROM admin_users
|
||||
WHERE id = 1`,
|
||||
);
|
||||
if (result.rowCount && result.rowCount > 0) {
|
||||
const row = result.rows[0] as {
|
||||
email: string;
|
||||
created_at: Date | string;
|
||||
token_version: number;
|
||||
};
|
||||
return {
|
||||
email: row.email,
|
||||
createdAt: new Date(row.created_at).toISOString(),
|
||||
tokenVersion: Number(row.token_version) || 1,
|
||||
storageMode: "PostgreSQL" as const,
|
||||
};
|
||||
}
|
||||
throw new Error("Admin account is not initialized.");
|
||||
}
|
||||
|
||||
const inMem = await getOrInitInMemoryAdmin();
|
||||
return {
|
||||
email: fallbackEmail || inMem.email,
|
||||
createdAt: inMem.createdAt.toISOString(),
|
||||
tokenVersion: inMem.tokenVersion,
|
||||
storageMode: "In-Memory" as const,
|
||||
};
|
||||
}
|
||||
|
||||
export async function verifyRequestAdminSession(
|
||||
cookieToken: string | undefined,
|
||||
): Promise<
|
||||
| { valid: true; session: AdminSessionPayload; sessionSecret: string }
|
||||
| { valid: false; reason: "UNCONFIGURED" | "UNAUTHORIZED" | "UNAVAILABLE" }
|
||||
> {
|
||||
const sessionSecret = getAdminSessionSecret();
|
||||
if (!sessionSecret) {
|
||||
return { valid: false, reason: "UNCONFIGURED" };
|
||||
}
|
||||
if (!cookieToken) {
|
||||
return { valid: false, reason: "UNAUTHORIZED" };
|
||||
}
|
||||
|
||||
const session = await getAdminSessionPayload(cookieToken, sessionSecret);
|
||||
if (!session) {
|
||||
return { valid: false, reason: "UNAUTHORIZED" };
|
||||
}
|
||||
|
||||
try {
|
||||
const account = await getAdminAccountInfo();
|
||||
const emailMatches = matchesSecret(
|
||||
session.email.trim().toLowerCase(),
|
||||
account.email.trim().toLowerCase(),
|
||||
);
|
||||
if (!emailMatches || session.tokenVersion !== account.tokenVersion) {
|
||||
return { valid: false, reason: "UNAUTHORIZED" };
|
||||
}
|
||||
return { valid: true, session, sessionSecret };
|
||||
} catch (error) {
|
||||
if (getPool()) {
|
||||
return { valid: false, reason: "UNAVAILABLE" };
|
||||
}
|
||||
void error;
|
||||
return { valid: false, reason: "UNAUTHORIZED" };
|
||||
}
|
||||
}
|
||||
|
||||
export async function updateAdminAccount({
|
||||
currentEmail,
|
||||
currentPassword,
|
||||
@@ -534,29 +637,42 @@ export async function updateAdminAccount({
|
||||
|
||||
const pool = getPool();
|
||||
if (pool) {
|
||||
try {
|
||||
await ensureAdminTable();
|
||||
const updated = await pool.query(
|
||||
`UPDATE admin_users
|
||||
SET email = $1, password_salt = $2, password_hash = $3
|
||||
WHERE id = 1`,
|
||||
[cleanEmail, salt, passwordHash],
|
||||
);
|
||||
if (updated.rowCount && updated.rowCount > 0) {
|
||||
return { ok: true as const, email: cleanEmail };
|
||||
}
|
||||
} catch {
|
||||
// Fall back to in-memory update
|
||||
await ensureAdminTable();
|
||||
const updated = await pool.query(
|
||||
`UPDATE admin_users
|
||||
SET email = $1,
|
||||
password_salt = $2,
|
||||
password_hash = $3,
|
||||
token_version = COALESCE(token_version, 1) + 1
|
||||
WHERE id = 1
|
||||
RETURNING token_version`,
|
||||
[cleanEmail, salt, passwordHash],
|
||||
);
|
||||
if (!updated.rowCount || updated.rowCount === 0) {
|
||||
return { ok: false as const, error: "Admin account not found." };
|
||||
}
|
||||
return {
|
||||
ok: true as const,
|
||||
email: cleanEmail,
|
||||
tokenVersion: Number(updated.rows[0].token_version) || 1,
|
||||
};
|
||||
}
|
||||
|
||||
const existing = await getOrInitInMemoryAdmin();
|
||||
const nextVersion = existing.tokenVersion + 1;
|
||||
globals.__dataStackInMemoryAdmin = {
|
||||
id: 1,
|
||||
email: cleanEmail,
|
||||
passwordSalt: salt,
|
||||
passwordHash,
|
||||
tokenVersion: nextVersion,
|
||||
createdAt: existing.createdAt,
|
||||
};
|
||||
return { ok: true as const, email: cleanEmail };
|
||||
return { ok: true as const, email: cleanEmail, tokenVersion: nextVersion };
|
||||
}
|
||||
|
||||
export function resetAdminStateForTests() {
|
||||
globals.__dataStackInMemoryAdmin = undefined;
|
||||
localLoginThrottle.clear();
|
||||
lastLocalThrottleCleanup = 0;
|
||||
}
|
||||
+33
-10
@@ -8,16 +8,31 @@ export interface ClientTokenPayload {
|
||||
exp?: number;
|
||||
}
|
||||
|
||||
const DEFAULT_SECRET =
|
||||
process.env.DATASTACK_JWT_SECRET ||
|
||||
const DEV_DEFAULT_JWT_SECRET =
|
||||
"datastack-jwt-default-secret-key-32-chars-long";
|
||||
|
||||
export function resolveDefaultJwtSecret(): string {
|
||||
const configured = process.env.DATASTACK_JWT_SECRET?.trim() || "";
|
||||
if (process.env.NODE_ENV === "production") {
|
||||
return configured.length >= 32 ? configured : "";
|
||||
}
|
||||
if (configured.length > 0) {
|
||||
return configured.length >= 32 ? configured : "";
|
||||
}
|
||||
return DEV_DEFAULT_JWT_SECRET;
|
||||
}
|
||||
|
||||
export class AuthService {
|
||||
private static secret: string = DEFAULT_SECRET;
|
||||
private static secret: string = resolveDefaultJwtSecret();
|
||||
|
||||
public static setSecret(newSecret: string) {
|
||||
if (newSecret && newSecret.trim().length > 0) {
|
||||
this.secret = newSecret.trim();
|
||||
const trimmed = newSecret ? newSecret.trim() : "";
|
||||
if (trimmed.length >= 32) {
|
||||
this.secret = trimmed;
|
||||
} else if (process.env.NODE_ENV !== "production" && trimmed.length > 0) {
|
||||
this.secret = trimmed;
|
||||
} else if (process.env.NODE_ENV === "production") {
|
||||
this.secret = "";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +40,11 @@ export class AuthService {
|
||||
return this.secret;
|
||||
}
|
||||
|
||||
public static isConfigured(): boolean {
|
||||
const minLen = process.env.NODE_ENV === "production" ? 32 : 1;
|
||||
return this.secret.length >= minLen;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates a signed JWT for client SDK authentication.
|
||||
*/
|
||||
@@ -32,9 +52,9 @@ export class AuthService {
|
||||
payload: { clientId: string; name?: string; rateLimit?: number },
|
||||
expiresIn = "30d",
|
||||
): string {
|
||||
if (!this.secret) {
|
||||
if (!this.isConfigured()) {
|
||||
throw new Error(
|
||||
"DATASTACK_JWT_SECRET must be configured before generating client tokens.",
|
||||
"DATASTACK_JWT_SECRET (>= 32 characters) must be configured before generating client tokens.",
|
||||
);
|
||||
}
|
||||
|
||||
@@ -50,15 +70,18 @@ export class AuthService {
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies and decodes incoming token from Socket.IO handshake.
|
||||
* Verifies and decodes incoming token from Socket.IO handshake or HTTP ingestion.
|
||||
*/
|
||||
public static verifyClientToken(token: string): {
|
||||
valid: boolean;
|
||||
payload?: ClientTokenPayload;
|
||||
error?: string;
|
||||
} {
|
||||
if (!this.secret) {
|
||||
return { valid: false, error: "DATASTACK_JWT_SECRET is not configured." };
|
||||
if (!this.isConfigured()) {
|
||||
return {
|
||||
valid: false,
|
||||
error: "DATASTACK_JWT_SECRET (>= 32 characters) is not configured.",
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
|
||||
+380
-28
@@ -1,8 +1,37 @@
|
||||
import crypto from 'node:crypto';
|
||||
import Redis from 'ioredis';
|
||||
import { CSVStorageManager, DataEvent } from '../Storage/csvStorage';
|
||||
|
||||
export type EventSubscriber = (event: DataEvent) => void;
|
||||
export type FlushCallback = (count: number, files: string[]) => void;
|
||||
|
||||
export interface RedisBufferClient {
|
||||
status?: string;
|
||||
rpush(key: string, ...values: string[]): Promise<number>;
|
||||
lpush(key: string, ...values: string[]): Promise<number>;
|
||||
ltrim(key: string, start: number, stop: number): Promise<unknown>;
|
||||
lrange(key: string, start: number, stop: number): Promise<string[]>;
|
||||
llen(key: string): Promise<number>;
|
||||
set(
|
||||
key: string,
|
||||
value: string,
|
||||
pxMode: 'PX',
|
||||
ttlMs: number,
|
||||
nxMode: 'NX',
|
||||
): Promise<string | null>;
|
||||
eval(script: string, numKeys: number, ...args: string[]): Promise<unknown>;
|
||||
quit?: () => Promise<unknown>;
|
||||
disconnect?: () => void;
|
||||
}
|
||||
|
||||
export const REDIS_BUFFER_KEY = 'datastack:buffer:events';
|
||||
export const REDIS_RECENT_KEY = 'datastack:recent:events';
|
||||
export const REDIS_FLUSH_LOCK_KEY = 'datastack:flush:lock';
|
||||
const REDIS_RECENT_LIMIT = 500;
|
||||
const FLUSH_LOCK_TTL_MS = 15000;
|
||||
const UNLOCK_LUA_SCRIPT =
|
||||
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end';
|
||||
|
||||
interface BufferGlobalState {
|
||||
buffer: DataEvent[];
|
||||
recentEvents: DataEvent[];
|
||||
@@ -14,6 +43,11 @@ interface BufferGlobalState {
|
||||
totalIngested: number;
|
||||
totalFlushed: number;
|
||||
isFlushing: boolean;
|
||||
redisClient: RedisBufferClient | null | undefined;
|
||||
redisUrlConfigured: string;
|
||||
redisConnected: boolean;
|
||||
redisTestOverride: RedisBufferClient | null | undefined;
|
||||
pendingRedisWrites: Set<Promise<void>>;
|
||||
}
|
||||
|
||||
type BufferGlobals = typeof globalThis & {
|
||||
@@ -36,13 +70,86 @@ function getState(): BufferGlobalState {
|
||||
totalIngested: 0,
|
||||
totalFlushed: 0,
|
||||
isFlushing: false,
|
||||
redisClient: undefined,
|
||||
redisUrlConfigured: '',
|
||||
redisConnected: false,
|
||||
redisTestOverride: undefined,
|
||||
pendingRedisWrites: new Set(),
|
||||
};
|
||||
}
|
||||
return globals.__dataStackBufferState;
|
||||
}
|
||||
|
||||
function getRedisClient(): RedisBufferClient | null {
|
||||
const state = getState();
|
||||
if (state.redisTestOverride !== undefined) {
|
||||
return state.redisTestOverride;
|
||||
}
|
||||
|
||||
const configuredUrl = (process.env.REDIS_URL || '').trim();
|
||||
if (!configuredUrl) {
|
||||
if (state.redisClient && typeof state.redisClient.disconnect === 'function') {
|
||||
try {
|
||||
state.redisClient.disconnect();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
state.redisClient = null;
|
||||
state.redisUrlConfigured = '';
|
||||
state.redisConnected = false;
|
||||
return null;
|
||||
}
|
||||
|
||||
if (state.redisClient && state.redisUrlConfigured === configuredUrl) {
|
||||
return state.redisClient;
|
||||
}
|
||||
|
||||
if (state.redisClient && typeof state.redisClient.disconnect === 'function') {
|
||||
try {
|
||||
state.redisClient.disconnect();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const client = new Redis(configuredUrl, {
|
||||
maxRetriesPerRequest: 2,
|
||||
enableReadyCheck: true,
|
||||
retryStrategy: (times: number) => Math.min(times * 250, 3000),
|
||||
});
|
||||
|
||||
client.on('ready', () => {
|
||||
state.redisConnected = true;
|
||||
});
|
||||
client.on('connect', () => {
|
||||
state.redisConnected = true;
|
||||
});
|
||||
client.on('close', () => {
|
||||
state.redisConnected = false;
|
||||
});
|
||||
client.on('end', () => {
|
||||
state.redisConnected = false;
|
||||
});
|
||||
client.on('error', () => {
|
||||
state.redisConnected = false;
|
||||
});
|
||||
|
||||
state.redisClient = client as unknown as RedisBufferClient;
|
||||
state.redisUrlConfigured = configuredUrl;
|
||||
state.redisConnected = client.status === 'ready' || client.status === 'connect';
|
||||
return state.redisClient;
|
||||
} catch {
|
||||
state.redisClient = null;
|
||||
state.redisUrlConfigured = configuredUrl;
|
||||
state.redisConnected = false;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export class DataBufferEngine {
|
||||
private static maxRecentEvents = 200;
|
||||
private static maxRecentEvents = REDIS_RECENT_LIMIT;
|
||||
|
||||
public static get batchFlushThreshold(): number {
|
||||
return getState().batchFlushThreshold;
|
||||
@@ -60,11 +167,46 @@ export class DataBufferEngine {
|
||||
this.setFlushInterval(value);
|
||||
}
|
||||
|
||||
public static setRedisClientForTests(client: RedisBufferClient | null | undefined): void {
|
||||
const state = getState();
|
||||
state.redisTestOverride = client;
|
||||
state.redisConnected = Boolean(client);
|
||||
}
|
||||
|
||||
public static async shutdown(): Promise<void> {
|
||||
const state = getState();
|
||||
if (state.flushTimer) {
|
||||
clearInterval(state.flushTimer);
|
||||
state.flushTimer = null;
|
||||
}
|
||||
if (state.pendingRedisWrites.size > 0) {
|
||||
await Promise.allSettled([...state.pendingRedisWrites]);
|
||||
}
|
||||
const client = state.redisClient;
|
||||
state.redisClient = undefined;
|
||||
state.redisUrlConfigured = '';
|
||||
state.redisConnected = false;
|
||||
state.redisTestOverride = undefined;
|
||||
if (client) {
|
||||
try {
|
||||
if (typeof client.quit === 'function') {
|
||||
await client.quit();
|
||||
} else if (typeof client.disconnect === 'function') {
|
||||
client.disconnect();
|
||||
}
|
||||
} catch {
|
||||
if (typeof client.disconnect === 'function') {
|
||||
client.disconnect();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static init() {
|
||||
const state = getState();
|
||||
if (!state.flushTimer) {
|
||||
state.flushTimer = setInterval(() => {
|
||||
void this.flush();
|
||||
void this.flush().catch(() => undefined);
|
||||
}, state.batchFlushIntervalMs);
|
||||
}
|
||||
|
||||
@@ -96,26 +238,18 @@ export class DataBufferEngine {
|
||||
}
|
||||
|
||||
state.flushTimer = setInterval(() => {
|
||||
void this.flush();
|
||||
void this.flush().catch(() => undefined);
|
||||
}, state.batchFlushIntervalMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Push incoming event to in-memory buffer.
|
||||
* If buffer hits threshold (e.g. 500 rows), triggers immediate batch flush.
|
||||
*/
|
||||
public static push(event: DataEvent): void {
|
||||
private static recordLocalMetaAndNotify(event: DataEvent): void {
|
||||
const state = getState();
|
||||
state.buffer.push(event);
|
||||
state.totalIngested += 1;
|
||||
|
||||
// Cache in recent events for live dashboard viewer
|
||||
state.recentEvents.unshift(event);
|
||||
if (state.recentEvents.length > this.maxRecentEvents) {
|
||||
state.recentEvents.pop();
|
||||
}
|
||||
|
||||
// Notify live subscribers
|
||||
for (const sub of state.subscribers) {
|
||||
try {
|
||||
sub(event);
|
||||
@@ -123,43 +257,188 @@ export class DataBufferEngine {
|
||||
// Ignore subscriber error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if buffer threshold reached
|
||||
private static async writeEventsToRedisOrFallback(events: DataEvent[]): Promise<number> {
|
||||
if (events.length === 0) return 0;
|
||||
const state = getState();
|
||||
const redis = getRedisClient();
|
||||
|
||||
if (redis) {
|
||||
const serialized = events.map((e) => JSON.stringify(e));
|
||||
try {
|
||||
const bufferLen = await redis.rpush(REDIS_BUFFER_KEY, ...serialized);
|
||||
await redis.lpush(REDIS_RECENT_KEY, ...serialized);
|
||||
await redis.ltrim(REDIS_RECENT_KEY, 0, REDIS_RECENT_LIMIT - 1);
|
||||
state.redisConnected = true;
|
||||
return Number(bufferLen) + state.buffer.length;
|
||||
} catch {
|
||||
state.redisConnected = false;
|
||||
}
|
||||
}
|
||||
|
||||
state.buffer.push(...events);
|
||||
return state.buffer.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Synchronous-signature push (queues to Redis asynchronously when configured, with automatic RAM fallback).
|
||||
*/
|
||||
public static push(event: DataEvent): void {
|
||||
const state = getState();
|
||||
this.recordLocalMetaAndNotify(event);
|
||||
|
||||
const redis = getRedisClient();
|
||||
if (redis) {
|
||||
const op = this.writeEventsToRedisOrFallback([event]).then((queuedCount) => {
|
||||
if (queuedCount >= state.batchFlushThreshold) {
|
||||
void this.flush().catch(() => undefined);
|
||||
}
|
||||
});
|
||||
state.pendingRedisWrites.add(op);
|
||||
void op.finally(() => {
|
||||
state.pendingRedisWrites.delete(op);
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
state.buffer.push(event);
|
||||
if (state.buffer.length >= state.batchFlushThreshold) {
|
||||
void this.flush();
|
||||
void this.flush().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Flushes current buffer to CSV files via CSVStorageManager
|
||||
* Async single-event push (awaited by API routes so Redis persistence is guaranteed before HTTP response).
|
||||
*/
|
||||
public static async pushAsync(event: DataEvent): Promise<void> {
|
||||
await this.pushBatchAsync([event]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Async batch push with shared Redis durability and RAM fallback.
|
||||
*/
|
||||
public static async pushBatchAsync(events: DataEvent[]): Promise<void> {
|
||||
if (!events || events.length === 0) return;
|
||||
const state = getState();
|
||||
for (const evt of events) {
|
||||
this.recordLocalMetaAndNotify(evt);
|
||||
}
|
||||
|
||||
const queuedCount = await this.writeEventsToRedisOrFallback(events);
|
||||
if (queuedCount >= state.batchFlushThreshold) {
|
||||
void this.flush().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Flushes queued events (shared Redis list under distributed lock + local RAM fallback) to partitioned CSV files.
|
||||
* Redis LTRIM runs ONLY after CSVStorageManager.appendBatch succeeds so crashes never lose data.
|
||||
*/
|
||||
public static async flush(): Promise<{ count: number; files: string[] }> {
|
||||
const state = getState();
|
||||
if (state.buffer.length === 0 || state.isFlushing) {
|
||||
if (state.pendingRedisWrites.size > 0) {
|
||||
await Promise.allSettled([...state.pendingRedisWrites]);
|
||||
}
|
||||
if (state.isFlushing) {
|
||||
return { count: 0, files: [] };
|
||||
}
|
||||
|
||||
state.isFlushing = true;
|
||||
const batchToFlush = [...state.buffer];
|
||||
state.buffer = [];
|
||||
let totalWritten = 0;
|
||||
const filesSet = new Set<string>();
|
||||
|
||||
try {
|
||||
const result = await CSVStorageManager.appendBatch(batchToFlush);
|
||||
state.totalFlushed += result.written;
|
||||
const redis = getRedisClient();
|
||||
if (redis) {
|
||||
const lockToken = crypto.randomUUID();
|
||||
let lockAcquired = false;
|
||||
|
||||
for (const listener of state.flushListeners) {
|
||||
try {
|
||||
listener(result.written, result.filesUpdated);
|
||||
const lockResult = await redis.set(
|
||||
REDIS_FLUSH_LOCK_KEY,
|
||||
lockToken,
|
||||
'PX',
|
||||
FLUSH_LOCK_TTL_MS,
|
||||
'NX',
|
||||
);
|
||||
lockAcquired = lockResult === 'OK';
|
||||
state.redisConnected = true;
|
||||
} catch {
|
||||
// ignore
|
||||
state.redisConnected = false;
|
||||
lockAcquired = false;
|
||||
}
|
||||
|
||||
if (lockAcquired) {
|
||||
try {
|
||||
const chunkSize = Math.max(500, state.batchFlushThreshold);
|
||||
while (true) {
|
||||
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, chunkSize - 1);
|
||||
if (!rawItems || rawItems.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
const parsedBatch: DataEvent[] = [];
|
||||
for (const raw of rawItems) {
|
||||
try {
|
||||
const evt = JSON.parse(raw) as DataEvent;
|
||||
if (evt && evt.eventId) {
|
||||
parsedBatch.push(evt);
|
||||
}
|
||||
} catch {
|
||||
// Ignore malformed legacy entry while still trimming it
|
||||
}
|
||||
}
|
||||
|
||||
if (parsedBatch.length > 0) {
|
||||
const res = await CSVStorageManager.appendBatch(parsedBatch);
|
||||
totalWritten += res.written;
|
||||
for (const f of res.filesUpdated) filesSet.add(f);
|
||||
}
|
||||
|
||||
// Trim strictly AFTER CSVStorageManager.appendBatch resolves safely
|
||||
await redis.ltrim(REDIS_BUFFER_KEY, rawItems.length, -1);
|
||||
|
||||
if (rawItems.length < chunkSize) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
|
||||
} catch {
|
||||
// Lock auto-expires after FLUSH_LOCK_TTL_MS
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { count: result.written, files: result.filesUpdated };
|
||||
} catch (err) {
|
||||
// On failure, re-queue to buffer front for retry
|
||||
state.buffer.unshift(...batchToFlush);
|
||||
throw err;
|
||||
if (state.buffer.length > 0) {
|
||||
const ramBatch = [...state.buffer];
|
||||
state.buffer = [];
|
||||
try {
|
||||
const res = await CSVStorageManager.appendBatch(ramBatch);
|
||||
totalWritten += res.written;
|
||||
for (const f of res.filesUpdated) filesSet.add(f);
|
||||
} catch (err) {
|
||||
state.buffer.unshift(...ramBatch);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
if (totalWritten > 0) {
|
||||
state.totalFlushed += totalWritten;
|
||||
const filesUpdated = Array.from(filesSet);
|
||||
for (const listener of state.flushListeners) {
|
||||
try {
|
||||
listener(totalWritten, filesUpdated);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { count: totalWritten, files: Array.from(filesSet) };
|
||||
} finally {
|
||||
state.isFlushing = false;
|
||||
}
|
||||
@@ -194,10 +473,83 @@ export class DataBufferEngine {
|
||||
return state.recentEvents.slice(0, limit);
|
||||
}
|
||||
|
||||
public static async getRecentEventsAsync(
|
||||
limit = 50,
|
||||
filterClientId?: string,
|
||||
): Promise<DataEvent[]> {
|
||||
const state = getState();
|
||||
const redis = getRedisClient();
|
||||
|
||||
if (redis) {
|
||||
try {
|
||||
const readCap = Math.min(REDIS_RECENT_LIMIT, Math.max(limit * 4, limit));
|
||||
const rawItems = await redis.lrange(REDIS_RECENT_KEY, 0, readCap - 1);
|
||||
state.redisConnected = true;
|
||||
|
||||
const redisEvents: DataEvent[] = [];
|
||||
for (const raw of rawItems) {
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as DataEvent;
|
||||
if (!parsed || !parsed.eventId) continue;
|
||||
if (filterClientId && filterClientId !== 'all' && parsed.clientId !== filterClientId) {
|
||||
continue;
|
||||
}
|
||||
redisEvents.push(parsed);
|
||||
if (redisEvents.length >= limit) break;
|
||||
} catch {
|
||||
// skip malformed JSON entry
|
||||
}
|
||||
}
|
||||
if (redisEvents.length > 0 || state.recentEvents.length === 0) {
|
||||
return redisEvents;
|
||||
}
|
||||
} catch {
|
||||
state.redisConnected = false;
|
||||
}
|
||||
}
|
||||
|
||||
return this.getRecentEvents(limit, filterClientId);
|
||||
}
|
||||
|
||||
public static getBufferStats() {
|
||||
const state = getState();
|
||||
return {
|
||||
currentBufferSize: state.buffer.length,
|
||||
localBufferSize: state.buffer.length,
|
||||
redisBufferSize: 0,
|
||||
redisEnabled: Boolean(getRedisClient()),
|
||||
redisConnected: state.redisConnected,
|
||||
batchFlushThreshold: state.batchFlushThreshold,
|
||||
batchFlushIntervalMs: state.batchFlushIntervalMs,
|
||||
totalIngested: state.totalIngested,
|
||||
totalFlushed: state.totalFlushed,
|
||||
recentEventsCached: state.recentEvents.length,
|
||||
activeSubscribers: state.subscribers.size,
|
||||
isFlushing: state.isFlushing,
|
||||
};
|
||||
}
|
||||
|
||||
public static async getBufferStatsAsync() {
|
||||
const state = getState();
|
||||
const redis = getRedisClient();
|
||||
let redisBufferSize = 0;
|
||||
|
||||
if (redis) {
|
||||
try {
|
||||
redisBufferSize = Number(await redis.llen(REDIS_BUFFER_KEY)) || 0;
|
||||
state.redisConnected = true;
|
||||
} catch {
|
||||
state.redisConnected = false;
|
||||
redisBufferSize = 0;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
currentBufferSize: state.buffer.length + redisBufferSize,
|
||||
localBufferSize: state.buffer.length,
|
||||
redisBufferSize,
|
||||
redisEnabled: Boolean(redis),
|
||||
redisConnected: state.redisConnected,
|
||||
batchFlushThreshold: state.batchFlushThreshold,
|
||||
batchFlushIntervalMs: state.batchFlushIntervalMs,
|
||||
totalIngested: state.totalIngested,
|
||||
|
||||
+315
-47
@@ -1,5 +1,8 @@
|
||||
import { existsSync, rmSync } from "node:fs";
|
||||
import { resolve, sep } from "node:path";
|
||||
import { AuthService } from "../auth/jwt";
|
||||
import { DatabaseEngine } from "../Database/database";
|
||||
import { getInternalIngestionHeaders } from "../Security/security";
|
||||
|
||||
export interface ClientSDK {
|
||||
clientId: string;
|
||||
@@ -15,7 +18,9 @@ export interface ClientSDK {
|
||||
|
||||
type SDKDatabaseGlobals = typeof globalThis & {
|
||||
__dataStackClientsMap?: Map<string, ClientSDK>;
|
||||
__dataStackDeletedClients?: Set<string>;
|
||||
__dataStackClientsHydrated?: boolean;
|
||||
__dataStackSyncedSocketEvents?: Map<string, number>;
|
||||
};
|
||||
|
||||
const globals = globalThis as SDKDatabaseGlobals;
|
||||
@@ -27,6 +32,67 @@ function getClientsMap(): Map<string, ClientSDK> {
|
||||
return globals.__dataStackClientsMap;
|
||||
}
|
||||
|
||||
function getDeletedClientsSet(): Set<string> {
|
||||
if (!globals.__dataStackDeletedClients) {
|
||||
globals.__dataStackDeletedClients = new Set<string>();
|
||||
}
|
||||
return globals.__dataStackDeletedClients;
|
||||
}
|
||||
|
||||
function getSyncedSocketEventsMap(): Map<string, number> {
|
||||
if (!globals.__dataStackSyncedSocketEvents) {
|
||||
globals.__dataStackSyncedSocketEvents = new Map<string, number>();
|
||||
}
|
||||
return globals.__dataStackSyncedSocketEvents;
|
||||
}
|
||||
|
||||
function serializeSyncClient(client: ClientSDK) {
|
||||
return {
|
||||
clientId: client.clientId,
|
||||
name: client.name,
|
||||
status: client.status,
|
||||
token: client.token,
|
||||
rateLimit: client.rateLimit,
|
||||
};
|
||||
}
|
||||
|
||||
async function notifyIngestionClientSyncAsync(
|
||||
clientId: string,
|
||||
action: "deleted" | "revoked" | "updated" | "registered",
|
||||
client?: ClientSDK,
|
||||
): Promise<void> {
|
||||
const headers = getInternalIngestionHeaders();
|
||||
if (!headers["x-datastack-internal-secret"]) return;
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||
).replace(/\/$/, "");
|
||||
try {
|
||||
await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...headers,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
clientId,
|
||||
action,
|
||||
...(client ? { client: serializeSyncClient(client) } : {}),
|
||||
}),
|
||||
signal: AbortSignal.timeout(1000),
|
||||
});
|
||||
} catch {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
}
|
||||
}
|
||||
|
||||
function notifyIngestionClientSync(
|
||||
clientId: string,
|
||||
action: "deleted" | "revoked" | "updated" | "registered",
|
||||
client?: ClientSDK,
|
||||
): void {
|
||||
void notifyIngestionClientSyncAsync(clientId, action, client);
|
||||
}
|
||||
|
||||
function mapRowToClient(row: Record<string, unknown>): ClientSDK {
|
||||
return {
|
||||
clientId: String(row.client_id),
|
||||
@@ -54,11 +120,20 @@ export class SDKManager {
|
||||
const ready = await DatabaseEngine.ensureSchema();
|
||||
if (!ready) return null;
|
||||
|
||||
const result = await pool.query(
|
||||
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
|
||||
FROM client_sdks
|
||||
ORDER BY created_at DESC`,
|
||||
);
|
||||
const [result, deletedResult] = await Promise.all([
|
||||
pool.query(
|
||||
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
|
||||
FROM client_sdks
|
||||
ORDER BY created_at DESC`,
|
||||
),
|
||||
pool.query(`SELECT client_id FROM deleted_clients`),
|
||||
]);
|
||||
|
||||
const deletedSet = getDeletedClientsSet();
|
||||
deletedSet.clear();
|
||||
for (const row of deletedResult.rows) {
|
||||
deletedSet.add(String(row.client_id).trim().toLowerCase());
|
||||
}
|
||||
|
||||
const map = getClientsMap();
|
||||
map.clear();
|
||||
@@ -84,12 +159,91 @@ export class SDKManager {
|
||||
return Array.from(getClientsMap().values());
|
||||
}
|
||||
|
||||
public static async syncAllClientsToIngestionAsync(): Promise<void> {
|
||||
const headers = getInternalIngestionHeaders();
|
||||
if (!headers["x-datastack-internal-secret"]) return;
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||
).replace(/\/$/, "");
|
||||
const clients = Array.from(getClientsMap().values()).map(serializeSyncClient);
|
||||
try {
|
||||
await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...headers,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
action: "bulk_sync",
|
||||
clients,
|
||||
}),
|
||||
signal: AbortSignal.timeout(1500),
|
||||
});
|
||||
} catch {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
}
|
||||
}
|
||||
|
||||
public static async listClientsAsync(): Promise<ClientSDK[]> {
|
||||
const fromDb = await this.syncFromDatabase();
|
||||
if (fromDb !== null) {
|
||||
return fromDb;
|
||||
}
|
||||
return Array.from(getClientsMap().values());
|
||||
|
||||
const map = getClientsMap();
|
||||
const headers = getInternalIngestionHeaders();
|
||||
if (headers["x-datastack-internal-secret"]) {
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||
).replace(/\/$/, "");
|
||||
try {
|
||||
const res = await fetch(`${ingestionBaseUrl}/internal/metrics`, {
|
||||
headers,
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(1000),
|
||||
});
|
||||
if (res.ok) {
|
||||
const data = (await res.json()) as {
|
||||
hasSyncedInMemoryClients?: boolean;
|
||||
clientActivity?: Record<
|
||||
string,
|
||||
{ totalEvents?: number; lastActive?: string }
|
||||
>;
|
||||
};
|
||||
if (data.hasSyncedInMemoryClients === false && map.size > 0) {
|
||||
await this.syncAllClientsToIngestionAsync();
|
||||
}
|
||||
if (data.clientActivity && typeof data.clientActivity === "object") {
|
||||
const syncedMap = getSyncedSocketEventsMap();
|
||||
for (const [cid, activity] of Object.entries(data.clientActivity)) {
|
||||
const client = map.get(cid);
|
||||
if (!client) continue;
|
||||
const remoteTotal = Math.max(0, Number(activity?.totalEvents) || 0);
|
||||
const prevSynced = syncedMap.get(cid) || 0;
|
||||
const delta =
|
||||
remoteTotal >= prevSynced
|
||||
? remoteTotal - prevSynced
|
||||
: remoteTotal;
|
||||
if (delta > 0) {
|
||||
client.totalEvents += delta;
|
||||
}
|
||||
syncedMap.set(cid, remoteTotal);
|
||||
if (
|
||||
activity?.lastActive &&
|
||||
(!client.lastActive || activity.lastActive > client.lastActive)
|
||||
) {
|
||||
client.lastActive = activity.lastActive;
|
||||
}
|
||||
map.set(cid, client);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(map.values());
|
||||
}
|
||||
|
||||
public static getClient(clientId: string): ClientSDK | undefined {
|
||||
@@ -152,8 +306,10 @@ export class SDKManager {
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
getDeletedClientsSet().delete(safeClientId);
|
||||
getClientsMap().set(safeClientId, newClient);
|
||||
void this.persistNewClient(newClient);
|
||||
notifyIngestionClientSync(safeClientId, "registered", newClient);
|
||||
return newClient;
|
||||
}
|
||||
|
||||
@@ -163,6 +319,10 @@ export class SDKManager {
|
||||
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
await pool.query(
|
||||
`DELETE FROM deleted_clients WHERE client_id = $1`,
|
||||
[client.clientId],
|
||||
);
|
||||
await pool.query(
|
||||
`INSERT INTO client_sdks (client_id, name, secret, token, rate_limit, status, total_events, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW())
|
||||
@@ -218,8 +378,10 @@ export class SDKManager {
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
getDeletedClientsSet().delete(safeClientId);
|
||||
getClientsMap().set(safeClientId, newClient);
|
||||
await this.persistNewClient(newClient);
|
||||
await notifyIngestionClientSyncAsync(safeClientId, "registered", newClient);
|
||||
return newClient;
|
||||
}
|
||||
|
||||
@@ -227,16 +389,20 @@ export class SDKManager {
|
||||
const client = getClientsMap().get(clientId);
|
||||
if (!client) return null;
|
||||
|
||||
client.secret = `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`;
|
||||
client.token = AuthService.generateClientToken({
|
||||
clientId: client.clientId,
|
||||
name: client.name,
|
||||
rateLimit: client.rateLimit,
|
||||
});
|
||||
const updated: ClientSDK = {
|
||||
...client,
|
||||
secret: `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`,
|
||||
token: AuthService.generateClientToken({
|
||||
clientId: client.clientId,
|
||||
name: client.name,
|
||||
rateLimit: client.rateLimit,
|
||||
}),
|
||||
};
|
||||
|
||||
getClientsMap().set(clientId, client);
|
||||
void this.persistClientCredentials(client);
|
||||
return client;
|
||||
getClientsMap().set(clientId, updated);
|
||||
void this.persistClientCredentials(updated);
|
||||
notifyIngestionClientSync(clientId, "updated", updated);
|
||||
return updated;
|
||||
}
|
||||
|
||||
private static async persistClientCredentials(
|
||||
@@ -264,16 +430,20 @@ export class SDKManager {
|
||||
const existing = await this.getClientAsync(clientId);
|
||||
if (!existing) return null;
|
||||
|
||||
existing.secret = `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`;
|
||||
existing.token = AuthService.generateClientToken({
|
||||
clientId: existing.clientId,
|
||||
name: existing.name,
|
||||
rateLimit: existing.rateLimit,
|
||||
});
|
||||
const updated: ClientSDK = {
|
||||
...existing,
|
||||
secret: `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`,
|
||||
token: AuthService.generateClientToken({
|
||||
clientId: existing.clientId,
|
||||
name: existing.name,
|
||||
rateLimit: existing.rateLimit,
|
||||
}),
|
||||
};
|
||||
|
||||
getClientsMap().set(clientId, existing);
|
||||
await this.persistClientCredentials(existing);
|
||||
return existing;
|
||||
getClientsMap().set(clientId, updated);
|
||||
await this.persistClientCredentials(updated);
|
||||
await notifyIngestionClientSyncAsync(clientId, "updated", updated);
|
||||
return updated;
|
||||
}
|
||||
|
||||
public static updateClient(
|
||||
@@ -283,14 +453,23 @@ export class SDKManager {
|
||||
const client = getClientsMap().get(clientId);
|
||||
if (!client) return null;
|
||||
|
||||
if (updates.name !== undefined) client.name = updates.name.trim();
|
||||
if (updates.rateLimit !== undefined)
|
||||
client.rateLimit = Math.max(1, Number(updates.rateLimit) || 100);
|
||||
if (updates.status !== undefined) client.status = updates.status;
|
||||
const updated: ClientSDK = {
|
||||
...client,
|
||||
...(updates.name !== undefined ? { name: updates.name.trim() } : {}),
|
||||
...(updates.rateLimit !== undefined
|
||||
? { rateLimit: Math.max(1, Number(updates.rateLimit) || 100) }
|
||||
: {}),
|
||||
...(updates.status !== undefined ? { status: updates.status } : {}),
|
||||
};
|
||||
|
||||
getClientsMap().set(clientId, client);
|
||||
void this.persistClientUpdates(client);
|
||||
return client;
|
||||
getClientsMap().set(clientId, updated);
|
||||
void this.persistClientUpdates(updated);
|
||||
notifyIngestionClientSync(
|
||||
clientId,
|
||||
updated.status === "revoked" ? "revoked" : "updated",
|
||||
updated,
|
||||
);
|
||||
return updated;
|
||||
}
|
||||
|
||||
private static async persistClientUpdates(client: ClientSDK): Promise<void> {
|
||||
@@ -317,19 +496,67 @@ export class SDKManager {
|
||||
const client = await this.getClientAsync(clientId);
|
||||
if (!client) return null;
|
||||
|
||||
if (updates.name !== undefined) client.name = updates.name.trim();
|
||||
if (updates.rateLimit !== undefined)
|
||||
client.rateLimit = Math.max(1, Number(updates.rateLimit) || 100);
|
||||
if (updates.status !== undefined) client.status = updates.status;
|
||||
const updated: ClientSDK = {
|
||||
...client,
|
||||
...(updates.name !== undefined ? { name: updates.name.trim() } : {}),
|
||||
...(updates.rateLimit !== undefined
|
||||
? { rateLimit: Math.max(1, Number(updates.rateLimit) || 100) }
|
||||
: {}),
|
||||
...(updates.status !== undefined ? { status: updates.status } : {}),
|
||||
};
|
||||
|
||||
getClientsMap().set(clientId, client);
|
||||
await this.persistClientUpdates(client);
|
||||
return client;
|
||||
getClientsMap().set(clientId, updated);
|
||||
await this.persistClientUpdates(updated);
|
||||
await notifyIngestionClientSyncAsync(
|
||||
clientId,
|
||||
updated.status === "revoked" ? "revoked" : "updated",
|
||||
updated,
|
||||
);
|
||||
return updated;
|
||||
}
|
||||
|
||||
public static isClientDeleted(clientId: string): boolean {
|
||||
const normalized = String(clientId || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
return getDeletedClientsSet().has(normalized);
|
||||
}
|
||||
|
||||
public static async isClientDeletedAsync(clientId: string): Promise<boolean> {
|
||||
const normalized = String(clientId || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
if (!normalized) return false;
|
||||
if (getDeletedClientsSet().has(normalized)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (!pool) return false;
|
||||
|
||||
await DatabaseEngine.ensureSchema();
|
||||
const res = await pool.query(
|
||||
`SELECT 1 FROM deleted_clients WHERE client_id = $1`,
|
||||
[normalized],
|
||||
);
|
||||
if (res.rowCount && res.rowCount > 0) {
|
||||
getDeletedClientsSet().add(normalized);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public static deleteClient(clientId: string): boolean {
|
||||
const deleted = getClientsMap().delete(clientId);
|
||||
void this.deleteClientFromDb(clientId);
|
||||
const normalized = String(clientId || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const deleted = getClientsMap().delete(normalized);
|
||||
getSyncedSocketEventsMap().delete(normalized);
|
||||
if (deleted) {
|
||||
getDeletedClientsSet().add(normalized);
|
||||
}
|
||||
void this.deleteClientFromDb(normalized);
|
||||
notifyIngestionClientSync(normalized, "deleted");
|
||||
return deleted;
|
||||
}
|
||||
|
||||
@@ -337,21 +564,62 @@ export class SDKManager {
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (!pool) return false;
|
||||
|
||||
await DatabaseEngine.ensureSchema();
|
||||
const dbClient = await pool.connect();
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
const res = await pool.query(
|
||||
await dbClient.query("BEGIN");
|
||||
const res = await dbClient.query(
|
||||
`DELETE FROM client_sdks WHERE client_id = $1`,
|
||||
[clientId],
|
||||
);
|
||||
return Boolean(res.rowCount && res.rowCount > 0);
|
||||
} catch {
|
||||
if (res.rowCount && res.rowCount > 0) {
|
||||
await dbClient.query(
|
||||
`INSERT INTO deleted_clients (client_id, deleted_at)
|
||||
VALUES ($1, NOW())
|
||||
ON CONFLICT (client_id) DO UPDATE SET deleted_at = NOW()`,
|
||||
[clientId],
|
||||
);
|
||||
await dbClient.query("COMMIT");
|
||||
return true;
|
||||
}
|
||||
await dbClient.query("ROLLBACK");
|
||||
return false;
|
||||
} catch {
|
||||
await dbClient.query("ROLLBACK").catch(() => undefined);
|
||||
return false;
|
||||
} finally {
|
||||
dbClient.release();
|
||||
}
|
||||
}
|
||||
|
||||
public static async deleteClientAsync(clientId: string): Promise<boolean> {
|
||||
const existedInMem = getClientsMap().delete(clientId);
|
||||
const deletedFromDb = await this.deleteClientFromDb(clientId);
|
||||
private static purgeClientStorageDir(clientId: string): void {
|
||||
const safeId = clientId.replace(/[^a-z0-9_-]/gi, "_");
|
||||
if (!safeId) return;
|
||||
const storageRoot = resolve(process.cwd(), "storage", "data");
|
||||
const targetDir = resolve(storageRoot, safeId);
|
||||
if (!targetDir.startsWith(`${storageRoot}${sep}`)) return;
|
||||
if (existsSync(targetDir)) {
|
||||
rmSync(targetDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
public static async deleteClientAsync(
|
||||
clientId: string,
|
||||
options?: { purgeFiles?: boolean },
|
||||
): Promise<boolean> {
|
||||
const normalized = String(clientId || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const existedInMem = getClientsMap().delete(normalized);
|
||||
getSyncedSocketEventsMap().delete(normalized);
|
||||
const deletedFromDb = await this.deleteClientFromDb(normalized);
|
||||
if (existedInMem || deletedFromDb) {
|
||||
getDeletedClientsSet().add(normalized);
|
||||
if (options?.purgeFiles) {
|
||||
this.purgeClientStorageDir(normalized);
|
||||
}
|
||||
await notifyIngestionClientSyncAsync(normalized, "deleted");
|
||||
}
|
||||
return existedInMem || deletedFromDb;
|
||||
}
|
||||
|
||||
|
||||
@@ -17,9 +17,12 @@ services:
|
||||
DATASTACK_ADMIN_SESSION_SECRET: "${SERVICE_REALBASE64_64_ADMINSESSION:?Coolify must generate SERVICE_REALBASE64_64_ADMINSESSION}"
|
||||
DATASTACK_ADMIN_SETUP_KEY: "${SERVICE_REALBASE64_64_ADMININSTALL:?Coolify must generate SERVICE_REALBASE64_64_ADMININSTALL}"
|
||||
DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}"
|
||||
REDIS_URL: "redis://redis:6379"
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test:
|
||||
@@ -48,9 +51,12 @@ services:
|
||||
PORT_SOCKET: "4000"
|
||||
DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}"
|
||||
DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}"
|
||||
REDIS_URL: "redis://redis:6379"
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test:
|
||||
@@ -85,7 +91,7 @@ services:
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
|
||||
# Reserved for the planned shared buffer integration; not consumed by the current runtime.
|
||||
# Shared Redis ingestion buffer & distributed flush coordination
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
restart: unless-stopped
|
||||
|
||||
+33
-16
@@ -1,4 +1,7 @@
|
||||
import { verifyAdminSessionToken } from "@/core/auth/adminSession";
|
||||
import {
|
||||
getAdminSessionSecret,
|
||||
verifyAdminSessionToken,
|
||||
} from "@/core/auth/adminSession";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
const SESSION_COOKIE = "datastack_admin_session";
|
||||
@@ -13,28 +16,37 @@ function isProtectedApi(pathname: string, method: string) {
|
||||
return pathname.startsWith("/api/");
|
||||
}
|
||||
|
||||
function isSameOriginRequest(request: NextRequest) {
|
||||
const originHeader = request.headers.get("origin");
|
||||
function matchesRequestHost(candidateUrl: string, request: NextRequest) {
|
||||
const requestHost = request.headers.get("host");
|
||||
const forwardedHost = request.headers
|
||||
.get("x-forwarded-host")
|
||||
?.split(",")[0]
|
||||
?.trim();
|
||||
if (!originHeader || (!requestHost && !forwardedHost)) return false;
|
||||
if (!requestHost && !forwardedHost) return false;
|
||||
|
||||
try {
|
||||
const origin = new URL(originHeader);
|
||||
const originHost = origin.host.toLowerCase();
|
||||
const matchesHost =
|
||||
(requestHost && originHost === requestHost.toLowerCase()) ||
|
||||
(forwardedHost && originHost === forwardedHost.toLowerCase());
|
||||
|
||||
return Boolean(matchesHost);
|
||||
const parsedHost = new URL(candidateUrl).host.toLowerCase();
|
||||
return Boolean(
|
||||
(requestHost && parsedHost === requestHost.toLowerCase()) ||
|
||||
(forwardedHost && parsedHost === forwardedHost.toLowerCase()),
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function isSameOriginRequest(request: NextRequest) {
|
||||
const originHeader = request.headers.get("origin");
|
||||
if (originHeader) {
|
||||
return matchesRequestHost(originHeader, request);
|
||||
}
|
||||
const refererHeader = request.headers.get("referer");
|
||||
if (refererHeader) {
|
||||
return matchesRequestHost(refererHeader, request);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function middleware(request: NextRequest) {
|
||||
const { pathname } = request.nextUrl;
|
||||
const isLoginPage = pathname === "/vcon/login";
|
||||
@@ -73,14 +85,19 @@ export async function middleware(request: NextRequest) {
|
||||
return NextResponse.next();
|
||||
}
|
||||
|
||||
const sessionSecret =
|
||||
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
|
||||
"datastack-admin-session-default-secret-32-chars";
|
||||
const sessionSecret = getAdminSessionSecret();
|
||||
if (!sessionSecret && isAdminApi) {
|
||||
return NextResponse.json(
|
||||
{ error: "Admin authentication is not configured." },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
const token = request.cookies.get(SESSION_COOKIE)?.value;
|
||||
const authenticated = Boolean(
|
||||
sessionSecret &&
|
||||
token &&
|
||||
(await verifyAdminSessionToken(token, sessionSecret)),
|
||||
token &&
|
||||
(await verifyAdminSessionToken(token, sessionSecret)),
|
||||
);
|
||||
|
||||
if (isLoginPage) {
|
||||
|
||||
+20
-1
@@ -1,3 +1,18 @@
|
||||
map $http_upgrade $connection_upgrade {
|
||||
default upgrade;
|
||||
'' close;
|
||||
}
|
||||
|
||||
map $http_x_forwarded_proto $forwarded_proto {
|
||||
default $http_x_forwarded_proto;
|
||||
'' $scheme;
|
||||
}
|
||||
|
||||
map $http_x_forwarded_host $forwarded_host {
|
||||
default $http_x_forwarded_host;
|
||||
'' $host;
|
||||
}
|
||||
|
||||
upstream nextjs_upstream {
|
||||
server 127.0.0.1:3000;
|
||||
}
|
||||
@@ -17,10 +32,12 @@ server {
|
||||
proxy_pass http://socketio_upstream;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection "upgrade";
|
||||
proxy_set_header Connection $connection_upgrade;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-Host $forwarded_host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $forwarded_proto;
|
||||
proxy_read_timeout 86400;
|
||||
}
|
||||
|
||||
@@ -29,7 +46,9 @@ server {
|
||||
proxy_pass http://nextjs_upstream;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-Host $forwarded_host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $forwarded_proto;
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,7 @@
|
||||
"cors": "^2.8.6",
|
||||
"csv-parse": "^7.0.3",
|
||||
"express": "^5.3.0",
|
||||
"ioredis": "^6.0.0",
|
||||
"json2csv": "^6.0.0-alpha.2",
|
||||
"jsonwebtoken": "^9.0.3",
|
||||
"lucide-react": "^0.553.0",
|
||||
|
||||
+36
-42
@@ -14,31 +14,25 @@ datasource db {
|
||||
// All collected data is strictly written to CSV files in /storage/data/{clientId}/{YYYY-MM-DD}.csv
|
||||
|
||||
model ClientSDK {
|
||||
id String @id @default(cuid())
|
||||
clientId String @unique
|
||||
clientId String @id @map("client_id")
|
||||
name String
|
||||
secret String
|
||||
rateLimit Int @default(100) // Max events per second
|
||||
status String @default("active") // active | revoked
|
||||
totalEvents BigInt @default(0)
|
||||
lastActive DateTime?
|
||||
createdAt DateTime @default(now())
|
||||
updatedAt DateTime @updatedAt
|
||||
|
||||
apiKeys ApiKey[]
|
||||
token String
|
||||
rateLimit Int @default(100) @map("rate_limit") // Max events per second
|
||||
status String @default("active") // active | revoked
|
||||
totalEvents BigInt @default(0) @map("total_events")
|
||||
lastActive DateTime? @map("last_active") @db.Timestamptz(6)
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(6)
|
||||
updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(6)
|
||||
|
||||
@@map("client_sdks")
|
||||
}
|
||||
|
||||
model ApiKey {
|
||||
id String @id @default(cuid())
|
||||
clientId String
|
||||
client ClientSDK @relation(fields: [clientId], references: [clientId], onDelete: Cascade)
|
||||
keyHash String
|
||||
expiresAt DateTime?
|
||||
createdAt DateTime @default(now())
|
||||
model DeletedClient {
|
||||
clientId String @id @map("client_id")
|
||||
deletedAt DateTime @default(now()) @map("deleted_at") @db.Timestamptz(6)
|
||||
|
||||
@@map("api_keys")
|
||||
@@map("deleted_clients")
|
||||
}
|
||||
|
||||
model AdminUser {
|
||||
@@ -46,35 +40,35 @@ model AdminUser {
|
||||
email String @unique
|
||||
passwordSalt Bytes @map("password_salt")
|
||||
passwordHash Bytes @map("password_hash")
|
||||
createdAt DateTime @default(now()) @map("created_at")
|
||||
tokenVersion Int @default(1) @map("token_version")
|
||||
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(6)
|
||||
|
||||
@@map("admin_users")
|
||||
}
|
||||
|
||||
model AdminLoginAttempt {
|
||||
keyHash String @id @map("key_hash")
|
||||
windowStartedAt DateTime @map("window_started_at") @db.Timestamptz(6)
|
||||
failedAttempts Int @map("failed_attempts")
|
||||
blockedUntil DateTime? @map("blocked_until") @db.Timestamptz(6)
|
||||
updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(6)
|
||||
|
||||
@@map("admin_login_attempts")
|
||||
}
|
||||
|
||||
model AdminSettings {
|
||||
id String @id @default("default")
|
||||
serverPort Int @default(4000)
|
||||
webPort Int @default(3000)
|
||||
maxEventsPerSec Int @default(100)
|
||||
maxPayloadSizeKb Int @default(100)
|
||||
batchFlushThreshold Int @default(500)
|
||||
batchFlushIntervalMs Int @default(2000)
|
||||
fileRotationMb Int @default(100)
|
||||
jwtSecret String
|
||||
enableIdempotency Boolean @default(true)
|
||||
autoPruneDays Int @default(30)
|
||||
updatedAt DateTime @updatedAt
|
||||
id String @id @default("default")
|
||||
serverPort Int @default(4000) @map("server_port")
|
||||
webPort Int @default(3000) @map("web_port")
|
||||
maxEventsPerSec Int @default(100) @map("max_events_per_sec")
|
||||
maxPayloadSizeKb Int @default(100) @map("max_payload_size_kb")
|
||||
batchFlushThreshold Int @default(500) @map("batch_flush_threshold")
|
||||
batchFlushIntervalMs Int @default(2000) @map("batch_flush_interval_ms")
|
||||
fileRotationMb Int @default(100) @map("file_rotation_mb")
|
||||
jwtSecret String @map("jwt_secret")
|
||||
enableIdempotency Boolean @default(true) @map("enable_idempotency")
|
||||
autoPruneDays Int @default(30) @map("auto_prune_days")
|
||||
updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(6)
|
||||
|
||||
@@map("admin_settings")
|
||||
}
|
||||
|
||||
model AuditLog {
|
||||
id String @id @default(cuid())
|
||||
action String
|
||||
actor String
|
||||
details String? // JSON string
|
||||
ipAddress String?
|
||||
createdAt DateTime @default(now())
|
||||
|
||||
@@map("audit_logs")
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 632 B |
Binary file not shown.
|
After Width: | Height: | Size: 718 B |
Binary file not shown.
|
After Width: | Height: | Size: 3.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.7 KiB |
@@ -0,0 +1,7 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512" width="512" height="512">
|
||||
<rect width="512" height="512" fill="#0a0a0b" />
|
||||
<rect x="24" y="24" width="464" height="464" fill="#111113" stroke="#2e2e33" stroke-width="8" />
|
||||
<rect x="112" y="112" width="288" height="288" fill="#a3e635" />
|
||||
<rect x="192" y="192" width="128" height="128" fill="#0a0a0b" />
|
||||
<rect x="232" y="232" width="48" height="48" fill="#e8e8ea" />
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 444 B |
+16
-6
@@ -1,21 +1,31 @@
|
||||
{
|
||||
"name": "DataStack - Realtime Data Platform",
|
||||
"id": "/",
|
||||
"name": "DataStack - Realtime Data Collection Platform",
|
||||
"short_name": "DataStack",
|
||||
"description": "Self-hosted universal realtime data collection engine",
|
||||
"description": "Self-hosted universal realtime data collection engine with Socket.IO ingestion, Redis buffer, JWT client auth, CSV batch rotation, and /vcon/ admin panel.",
|
||||
"start_url": "/vcon/dashboard",
|
||||
"scope": "/",
|
||||
"display": "standalone",
|
||||
"background_color": "#0b0f17",
|
||||
"theme_color": "#2563eb",
|
||||
"background_color": "#0a0a0b",
|
||||
"theme_color": "#0a0a0b",
|
||||
"icons": [
|
||||
{
|
||||
"src": "/icon-192.png",
|
||||
"sizes": "192x192",
|
||||
"type": "image/png"
|
||||
"type": "image/png",
|
||||
"purpose": "any"
|
||||
},
|
||||
{
|
||||
"src": "/icon-512.png",
|
||||
"sizes": "512x512",
|
||||
"type": "image/png"
|
||||
"type": "image/png",
|
||||
"purpose": "any"
|
||||
},
|
||||
{
|
||||
"src": "/icon-maskable-512.png",
|
||||
"sizes": "512x512",
|
||||
"type": "image/png",
|
||||
"purpose": "maskable"
|
||||
}
|
||||
]
|
||||
}
|
||||
+45
-7
@@ -1,5 +1,13 @@
|
||||
const CACHE_NAME = 'datastack-cache-v1';
|
||||
const STATIC_ASSETS = ['/', '/vcon/dashboard', '/manifest.json'];
|
||||
const CACHE_NAME = 'datastack-cache-v2';
|
||||
const STATIC_ASSETS = [
|
||||
'/',
|
||||
'/manifest.json',
|
||||
'/icon.svg',
|
||||
'/icon-192.png',
|
||||
'/icon-512.png',
|
||||
'/icon-maskable-512.png',
|
||||
'/apple-touch-icon.png',
|
||||
];
|
||||
|
||||
self.addEventListener('install', (event) => {
|
||||
event.waitUntil(
|
||||
@@ -22,13 +30,43 @@ self.addEventListener('activate', (event) => {
|
||||
});
|
||||
|
||||
self.addEventListener('fetch', (event) => {
|
||||
// Let WebSocket and API requests pass through
|
||||
if (event.request.url.includes('/socket.io/') || event.request.url.includes('/api/')) {
|
||||
const { request } = event;
|
||||
if (request.method !== 'GET') return;
|
||||
|
||||
let url;
|
||||
try {
|
||||
url = new URL(request.url);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
// Only handle same-origin GET requests; never intercept WebSocket, HMR, or API routes
|
||||
if (url.origin !== self.location.origin) return;
|
||||
if (
|
||||
url.pathname.startsWith('/socket.io/') ||
|
||||
url.pathname.startsWith('/api/') ||
|
||||
url.pathname.startsWith('/_next/webpack-hmr')
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
event.respondWith(
|
||||
fetch(event.request).catch(() => {
|
||||
return caches.match(event.request);
|
||||
})
|
||||
fetch(request)
|
||||
.then((response) => {
|
||||
if (response && response.ok && response.type === 'basic') {
|
||||
const clone = response.clone();
|
||||
caches.open(CACHE_NAME).then((cache) => cache.put(request, clone)).catch(() => {});
|
||||
}
|
||||
return response;
|
||||
})
|
||||
.catch(async () => {
|
||||
const cached = await caches.match(request);
|
||||
if (cached) return cached;
|
||||
return new Response('Offline — network unavailable.', {
|
||||
status: 503,
|
||||
statusText: 'Service Unavailable',
|
||||
headers: { 'Content-Type': 'text/plain; charset=utf-8' },
|
||||
});
|
||||
})
|
||||
);
|
||||
});
|
||||
+11
-9
@@ -6,24 +6,24 @@
|
||||
|
||||
Official Python client library for **DataStack — Self-Hosted Real-Time Telemetry & Event Logging Platform**, developed by **[Vib Tools](https://vib.tools/)**.
|
||||
|
||||
`datastack-sdk` enables Python services, IoT sensor nodes, microservice workers, and research pipelines to stream structured JSON telemetry events to a DataStack server over Socket.IO with deterministic `threading.Event` acknowledgments.
|
||||
`datastack-sdk` is a compatibility alias for the canonical **`datastack`** package located at [`../sdk/python/datastack`](../sdk/python/datastack). Both `from datastack import DataStack` and `from datastack_sdk import DataStackClient` share a single source of truth with `config.datastack.py` auto-discovery, `CLIENT_ID` wiring, Socket.IO streaming, automatic HTTP fallback, offline disk spooling, and direct `.csv`/`.json`/`.jsonl` file streaming.
|
||||
|
||||
---
|
||||
|
||||
## Installation
|
||||
|
||||
Install directly from the `sdk-python` directory:
|
||||
Install the canonical package directly from `sdk/python/datastack`:
|
||||
|
||||
```bash
|
||||
pip install ./sdk/python/datastack
|
||||
```
|
||||
|
||||
Or install the compatibility alias from `sdk-python`:
|
||||
|
||||
```bash
|
||||
pip install ./sdk-python
|
||||
```
|
||||
|
||||
For editable development mode:
|
||||
|
||||
```bash
|
||||
cd sdk-python && pip install -e .
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Quick Start
|
||||
@@ -34,10 +34,12 @@ from datastack_sdk import DataStackClient
|
||||
# 1. Initialize the client with a signed JWT token from /vcon/clients or /vcon/sdk
|
||||
client = DataStackClient(
|
||||
server_url="http://localhost:4000",
|
||||
api_url="http://localhost:3000",
|
||||
token="YOUR_CLIENT_JWT_TOKEN",
|
||||
client_id="YOUR_CLIENT_ID", # optional (auto-derived from JWT sub/clientId)
|
||||
)
|
||||
|
||||
# 2. Connect to the DataStack Socket.IO server
|
||||
# 2. Connect to the DataStack Socket.IO server (or auto HTTP fallback)
|
||||
if client.connect():
|
||||
# 3. Stream a single structured event and wait for server acknowledgment
|
||||
res = client.sync(
|
||||
|
||||
@@ -1,10 +1,29 @@
|
||||
"""
|
||||
DataStack Python SDK (`datastack_sdk`)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
Official Python client library for DataStack — Self-Hosted Real-Time Telemetry & Event Logging Platform.
|
||||
DataStack Python SDK Compatibility Alias (`datastack_sdk`).
|
||||
Re-exports all canonical symbols from `sdk/python/datastack` (`datastack`)
|
||||
so both `from datastack import DataStack` and legacy
|
||||
`from datastack_sdk import DataStackClient` share a single source of truth.
|
||||
"""
|
||||
|
||||
from .client import (
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _ensure_canonical_datastack_on_path() -> None:
|
||||
try:
|
||||
import datastack # noqa: F401
|
||||
return
|
||||
except ImportError:
|
||||
repo_canonical = (
|
||||
Path(__file__).resolve().parents[2] / "sdk" / "python" / "datastack"
|
||||
)
|
||||
if repo_canonical.is_dir() and str(repo_canonical) not in sys.path:
|
||||
sys.path.insert(0, str(repo_canonical))
|
||||
|
||||
|
||||
_ensure_canonical_datastack_on_path()
|
||||
|
||||
from datastack import ( # noqa: E402, F401
|
||||
ConnectionState,
|
||||
DataStack,
|
||||
DataStackAuthError,
|
||||
@@ -19,11 +38,10 @@ from .client import (
|
||||
DataStackRateLimitError,
|
||||
DiskSpooler,
|
||||
HeartbeatMonitor,
|
||||
__version__,
|
||||
load_config,
|
||||
)
|
||||
|
||||
__version__ = "2.2.0"
|
||||
|
||||
__all__ = [
|
||||
"DataStack",
|
||||
"DataStackClient",
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,321 +1,7 @@
|
||||
"""
|
||||
Auto-Discovery Configuration Loader for `config.datastack.py`.
|
||||
"""
|
||||
"""Compatibility re-export of canonical `datastack.config`."""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
from urllib.parse import urlparse
|
||||
from . import _ensure_canonical_datastack_on_path
|
||||
|
||||
from .exceptions import DataStackConfigError
|
||||
_ensure_canonical_datastack_on_path()
|
||||
|
||||
CONFIG_FILENAME = "config.datastack.py"
|
||||
|
||||
|
||||
@dataclass
|
||||
class DataStackConfig:
|
||||
server_url: str = "http://localhost:4000"
|
||||
api_url: str = "http://localhost:3000"
|
||||
client_token: str = ""
|
||||
client_id: str = ""
|
||||
auto_connect: bool = True
|
||||
enable_http_fallback: bool = True
|
||||
heartbeat_interval_sec: float = 5.0
|
||||
request_timeout_sec: float = 8.0
|
||||
max_retries: int = 3
|
||||
retry_backoff_base_sec: float = 0.5
|
||||
batch_chunk_size: int = 250
|
||||
offline_spool_enabled: bool = True
|
||||
offline_spool_dir: str = ".datastack_spool"
|
||||
max_spool_size_mb: int = 50
|
||||
config_source_path: Optional[str] = None
|
||||
|
||||
|
||||
def _derive_default_api_url(server_url: str) -> str:
|
||||
"""
|
||||
If API_URL is not explicitly set, infer Port 3000 on the same host when SERVER_URL
|
||||
uses Port 4000, or fall back to http://localhost:3000.
|
||||
"""
|
||||
try:
|
||||
parsed = urlparse(server_url)
|
||||
if parsed.scheme and parsed.hostname:
|
||||
if parsed.port == 4000:
|
||||
return f"{parsed.scheme}://{parsed.hostname}:3000"
|
||||
return f"{parsed.scheme}://{parsed.netloc}"
|
||||
except Exception:
|
||||
pass
|
||||
return "http://localhost:3000"
|
||||
|
||||
|
||||
def _candidate_config_paths(explicit_path: Optional[str] = None) -> List[Path]:
|
||||
candidates: List[Path] = []
|
||||
|
||||
if explicit_path:
|
||||
p = Path(explicit_path).expanduser().resolve()
|
||||
if p.is_dir():
|
||||
candidates.append(p / CONFIG_FILENAME)
|
||||
else:
|
||||
candidates.append(p)
|
||||
|
||||
env_path = os.environ.get("DATASTACK_CONFIG_PATH", "").strip()
|
||||
if env_path:
|
||||
ep = Path(env_path).expanduser().resolve()
|
||||
if ep.is_dir():
|
||||
candidates.append(ep / CONFIG_FILENAME)
|
||||
else:
|
||||
candidates.append(ep)
|
||||
|
||||
# Search current working directory and up to 3 parent levels
|
||||
try:
|
||||
cwd = Path.cwd().resolve()
|
||||
candidates.append(cwd / CONFIG_FILENAME)
|
||||
for parent in list(cwd.parents)[:3]:
|
||||
candidates.append(parent / CONFIG_FILENAME)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Search main entry script directory
|
||||
try:
|
||||
if sys.argv and sys.argv[0]:
|
||||
script_dir = Path(sys.argv[0]).resolve().parent
|
||||
candidates.append(script_dir / CONFIG_FILENAME)
|
||||
for parent in list(script_dir.parents)[:2]:
|
||||
candidates.append(parent / CONFIG_FILENAME)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Deduplicate while preserving order
|
||||
seen = set()
|
||||
unique: List[Path] = []
|
||||
for c in candidates:
|
||||
key = str(c)
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
unique.append(c)
|
||||
return unique
|
||||
|
||||
|
||||
def _load_python_config_file(file_path: Path) -> Dict[str, Any]:
|
||||
try:
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"datastack_user_config", str(file_path)
|
||||
)
|
||||
if spec is None or spec.loader is None:
|
||||
raise DataStackConfigError(
|
||||
f"Unable to load configuration specification from {file_path}"
|
||||
)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
raw: Dict[str, Any] = {}
|
||||
if hasattr(module, "DATASTACK_CONFIG") and isinstance(
|
||||
getattr(module, "DATASTACK_CONFIG"), dict
|
||||
):
|
||||
raw.update(getattr(module, "DATASTACK_CONFIG"))
|
||||
|
||||
# Also support top-level module variables (e.g., SERVER_URL = "...", CLIENT_TOKEN = "...")
|
||||
for attr in dir(module):
|
||||
if attr.isupper() and attr != "DATASTACK_CONFIG":
|
||||
raw[attr] = getattr(module, attr)
|
||||
|
||||
return raw
|
||||
except DataStackConfigError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise DataStackConfigError(
|
||||
f"Failed to parse {file_path}: {exc}"
|
||||
) from exc
|
||||
|
||||
|
||||
def load_config(
|
||||
config_path: Optional[str] = None,
|
||||
require_token: bool = False,
|
||||
**overrides: Any,
|
||||
) -> DataStackConfig:
|
||||
"""
|
||||
Discovers and loads `config.datastack.py` from the client application's
|
||||
directory, merges environment variables (`DATASTACK_*`), and applies
|
||||
any explicit keyword overrides.
|
||||
"""
|
||||
file_values: Dict[str, Any] = {}
|
||||
resolved_source: Optional[str] = None
|
||||
|
||||
for candidate in _candidate_config_paths(config_path):
|
||||
if candidate.is_file():
|
||||
file_values = _load_python_config_file(candidate)
|
||||
resolved_source = str(candidate)
|
||||
break
|
||||
|
||||
if config_path and not resolved_source:
|
||||
raise DataStackConfigError(
|
||||
f"Explicitly requested config file was not found: {config_path}"
|
||||
)
|
||||
|
||||
# Normalize keys to uppercase for lookup
|
||||
normalized_file = {str(k).upper(): v for k, v in file_values.items()}
|
||||
normalized_overrides = {
|
||||
str(k).upper(): v for k, v in overrides.items() if v is not None
|
||||
}
|
||||
|
||||
def pick(keys: List[str], env_keys: List[str], default: Any) -> Any:
|
||||
for k in keys:
|
||||
if k in normalized_overrides:
|
||||
return normalized_overrides[k]
|
||||
for ek in env_keys:
|
||||
env_val = os.environ.get(ek)
|
||||
if env_val is not None and str(env_val).strip() != "":
|
||||
return env_val
|
||||
for k in keys:
|
||||
if k in normalized_file and normalized_file[k] is not None:
|
||||
return normalized_file[k]
|
||||
return default
|
||||
|
||||
def to_bool(val: Any, default: bool) -> bool:
|
||||
if isinstance(val, bool):
|
||||
return val
|
||||
if isinstance(val, str):
|
||||
return val.strip().lower() in ("1", "true", "yes", "on")
|
||||
if val is None:
|
||||
return default
|
||||
return bool(val)
|
||||
|
||||
server_url = str(
|
||||
pick(
|
||||
["SERVER_URL", "SOCKET_URL", "WS_URL"],
|
||||
["DATASTACK_SERVER_URL", "DATASTACK_SOCKET_URL"],
|
||||
"http://localhost:4000",
|
||||
)
|
||||
).rstrip("/")
|
||||
|
||||
default_api = _derive_default_api_url(server_url)
|
||||
api_url = str(
|
||||
pick(
|
||||
["API_URL", "HTTP_URL", "WEB_URL"],
|
||||
["DATASTACK_API_URL", "DATASTACK_HTTP_URL"],
|
||||
default_api,
|
||||
)
|
||||
).rstrip("/")
|
||||
|
||||
client_token = str(
|
||||
pick(
|
||||
["CLIENT_TOKEN", "TOKEN", "JWT_TOKEN", "API_TOKEN"],
|
||||
["DATASTACK_CLIENT_TOKEN", "DATASTACK_TOKEN"],
|
||||
"",
|
||||
)
|
||||
).strip()
|
||||
|
||||
client_id = str(
|
||||
pick(
|
||||
["CLIENT_ID"],
|
||||
["DATASTACK_CLIENT_ID"],
|
||||
"",
|
||||
)
|
||||
).strip()
|
||||
|
||||
cfg = DataStackConfig(
|
||||
server_url=server_url,
|
||||
api_url=api_url,
|
||||
client_token=client_token,
|
||||
client_id=client_id,
|
||||
auto_connect=to_bool(
|
||||
pick(["AUTO_CONNECT"], ["DATASTACK_AUTO_CONNECT"], True), True
|
||||
),
|
||||
enable_http_fallback=to_bool(
|
||||
pick(
|
||||
["ENABLE_HTTP_FALLBACK", "HTTP_FALLBACK"],
|
||||
["DATASTACK_ENABLE_HTTP_FALLBACK"],
|
||||
True,
|
||||
),
|
||||
True,
|
||||
),
|
||||
heartbeat_interval_sec=max(
|
||||
1.0,
|
||||
float(
|
||||
pick(
|
||||
["HEARTBEAT_INTERVAL_SEC", "HEARTBEAT_INTERVAL"],
|
||||
["DATASTACK_HEARTBEAT_INTERVAL_SEC"],
|
||||
5.0,
|
||||
)
|
||||
),
|
||||
),
|
||||
request_timeout_sec=max(
|
||||
1.0,
|
||||
float(
|
||||
pick(
|
||||
["REQUEST_TIMEOUT_SEC", "TIMEOUT"],
|
||||
["DATASTACK_REQUEST_TIMEOUT_SEC"],
|
||||
8.0,
|
||||
)
|
||||
),
|
||||
),
|
||||
max_retries=max(
|
||||
0,
|
||||
int(
|
||||
pick(
|
||||
["MAX_RETRIES", "RETRIES"],
|
||||
["DATASTACK_MAX_RETRIES"],
|
||||
3,
|
||||
)
|
||||
),
|
||||
),
|
||||
retry_backoff_base_sec=max(
|
||||
0.1,
|
||||
float(
|
||||
pick(
|
||||
["RETRY_BACKOFF_BASE_SEC"],
|
||||
["DATASTACK_RETRY_BACKOFF_BASE_SEC"],
|
||||
0.5,
|
||||
)
|
||||
),
|
||||
),
|
||||
batch_chunk_size=max(
|
||||
1,
|
||||
int(
|
||||
pick(
|
||||
["BATCH_CHUNK_SIZE", "CHUNK_SIZE"],
|
||||
["DATASTACK_BATCH_CHUNK_SIZE"],
|
||||
250,
|
||||
)
|
||||
),
|
||||
),
|
||||
offline_spool_enabled=to_bool(
|
||||
pick(
|
||||
["OFFLINE_SPOOL_ENABLED"],
|
||||
["DATASTACK_OFFLINE_SPOOL_ENABLED"],
|
||||
True,
|
||||
),
|
||||
True,
|
||||
),
|
||||
offline_spool_dir=str(
|
||||
pick(
|
||||
["OFFLINE_SPOOL_DIR", "SPOOL_DIR"],
|
||||
["DATASTACK_OFFLINE_SPOOL_DIR"],
|
||||
".datastack_spool",
|
||||
)
|
||||
),
|
||||
max_spool_size_mb=max(
|
||||
1,
|
||||
int(
|
||||
pick(
|
||||
["MAX_SPOOL_SIZE_MB"],
|
||||
["DATASTACK_MAX_SPOOL_SIZE_MB"],
|
||||
50,
|
||||
)
|
||||
),
|
||||
),
|
||||
config_source_path=resolved_source,
|
||||
)
|
||||
|
||||
if require_token and (
|
||||
not cfg.client_token
|
||||
or cfg.client_token == "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN"
|
||||
):
|
||||
raise DataStackConfigError(
|
||||
"DataStack CLIENT_TOKEN is missing. Add your token to `config.datastack.py`, "
|
||||
"set DATASTACK_TOKEN in your environment, or pass token='...' to DataStack()."
|
||||
)
|
||||
|
||||
return cfg
|
||||
from datastack.config import * # noqa: E402, F401, F403
|
||||
@@ -1,72 +1,7 @@
|
||||
"""
|
||||
Structured Exception Hierarchy for the DataStack Python SDK.
|
||||
"""
|
||||
"""Compatibility re-export of canonical `datastack.exceptions`."""
|
||||
|
||||
from typing import Optional
|
||||
from . import _ensure_canonical_datastack_on_path
|
||||
|
||||
_ensure_canonical_datastack_on_path()
|
||||
|
||||
class DataStackError(Exception):
|
||||
"""Base exception for all DataStack SDK errors."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
code: Optional[str] = None,
|
||||
retryable: bool = False,
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
self.code = code or "DATASTACK_ERROR"
|
||||
self.retryable = retryable
|
||||
|
||||
|
||||
class DataStackConfigError(DataStackError):
|
||||
"""Raised when `config.datastack.py` or required credentials are missing or invalid."""
|
||||
|
||||
def __init__(self, message: str) -> None:
|
||||
super().__init__(message, code="CONFIG_ERROR", retryable=False)
|
||||
|
||||
|
||||
class DataStackAuthError(DataStackError):
|
||||
"""
|
||||
Raised when authentication fails (`AUTH_REQUIRED`, `AUTH_INVALID`,
|
||||
`CLIENT_REVOKED`, or `TOKEN_ROTATED`). Non-retryable until credentials are updated.
|
||||
"""
|
||||
|
||||
def __init__(self, message: str, code: str = "AUTH_ERROR") -> None:
|
||||
super().__init__(message, code=code, retryable=False)
|
||||
|
||||
|
||||
class DataStackRateLimitError(DataStackError):
|
||||
"""Raised when `RATE_LIMIT_EXCEEDED` persists after all backoff retries."""
|
||||
|
||||
def __init__(self, message: str) -> None:
|
||||
super().__init__(message, code="RATE_LIMIT_EXCEEDED", retryable=True)
|
||||
|
||||
|
||||
class DataStackPayloadTooLargeError(DataStackError):
|
||||
"""Raised when a single event exceeds the server's `maxPayloadSizeKb` limit."""
|
||||
|
||||
def __init__(self, message: str) -> None:
|
||||
super().__init__(message, code="PAYLOAD_TOO_LARGE", retryable=False)
|
||||
|
||||
|
||||
class DataStackConnectionError(DataStackError):
|
||||
"""Raised when both WebSocket and HTTP fallback are unreachable and offline spooling is disabled."""
|
||||
|
||||
def __init__(self, message: str) -> None:
|
||||
super().__init__(message, code="CONNECTION_ERROR", retryable=True)
|
||||
|
||||
|
||||
class DataStackFileError(DataStackError):
|
||||
"""Raised when a file passed to `send_file()` is missing, unsupported, or malformed."""
|
||||
|
||||
def __init__(self, message: str) -> None:
|
||||
super().__init__(message, code="FILE_ERROR", retryable=False)
|
||||
|
||||
|
||||
class DataStackOfflineSpoolError(DataStackError):
|
||||
"""Raised when the local offline disk spool directory is full or unwriteable."""
|
||||
|
||||
def __init__(self, message: str) -> None:
|
||||
super().__init__(message, code="OFFLINE_SPOOL_ERROR", retryable=False)
|
||||
from datastack.exceptions import * # noqa: E402, F401, F403
|
||||
@@ -1,186 +1,7 @@
|
||||
"""
|
||||
Smart Heartbeat Monitor & Connection State Machine (`HeartbeatMonitor`).
|
||||
"""Compatibility re-export of canonical `datastack.heartbeat`."""
|
||||
|
||||
Lifecycle States:
|
||||
- `CONNECTED_IDLE`: WebSocket (Port 4000) is connected and verified. Heartbeat sleeps in IDLE mode.
|
||||
- `FALLBACK_HTTP`: WebSocket disconnected. Client automatically sends events via HTTP (`POST /api/events`)
|
||||
while Heartbeat actively probes to reconnect and verify WebSocket, then returns to `CONNECTED_IDLE`.
|
||||
- `OFFLINE_SPOOL`: Device is completely offline. Events spool to disk (`.datastack_spool/spool.jsonl`).
|
||||
Heartbeat monitors network restoration, reconnects, replays spooled events, and returns to `CONNECTED_IDLE`.
|
||||
- `RECONNECTING_VERIFY`: Heartbeat is verifying handshake (`ping_verify`) and draining spooled events.
|
||||
- `STOPPED`: Client was closed intentionally or halted due to non-retryable auth revocation.
|
||||
"""
|
||||
from . import _ensure_canonical_datastack_on_path
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Callable, Dict, Optional
|
||||
_ensure_canonical_datastack_on_path()
|
||||
|
||||
logger = logging.getLogger("datastack.heartbeat")
|
||||
|
||||
|
||||
class ConnectionState:
|
||||
CONNECTED_IDLE = "CONNECTED_IDLE"
|
||||
FALLBACK_HTTP = "FALLBACK_HTTP"
|
||||
OFFLINE_SPOOL = "OFFLINE_SPOOL"
|
||||
RECONNECTING_VERIFY = "RECONNECTING_VERIFY"
|
||||
STOPPED = "STOPPED"
|
||||
|
||||
|
||||
class HeartbeatMonitor:
|
||||
"""
|
||||
Background recovery thread that sleeps in zero-CPU IDLE mode while WebSocket is
|
||||
healthy (`CONNECTED_IDLE`) and wakes up automatically when WebSocket disconnects
|
||||
or when the client device goes offline.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
interval_sec: float,
|
||||
recovery_callback: Callable[[], bool],
|
||||
) -> None:
|
||||
self.interval_sec = max(0.2, float(interval_sec))
|
||||
self._recovery_callback = recovery_callback
|
||||
self._state = ConnectionState.CONNECTED_IDLE
|
||||
self._closed = False
|
||||
self._last_error: Optional[str] = None
|
||||
self._recovery_attempts = 0
|
||||
self._last_verified_at: Optional[float] = None
|
||||
|
||||
self._lock = threading.RLock()
|
||||
self._wake_event = threading.Event()
|
||||
self._stop_event = threading.Event()
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
|
||||
@property
|
||||
def state(self) -> str:
|
||||
with self._lock:
|
||||
return self._state
|
||||
|
||||
@property
|
||||
def is_idle(self) -> bool:
|
||||
with self._lock:
|
||||
return self._state == ConnectionState.CONNECTED_IDLE
|
||||
|
||||
def set_idle_connected(self) -> None:
|
||||
"""
|
||||
Marks the connection as healthy (`CONNECTED_IDLE`) and puts the Heartbeat
|
||||
Monitor into IDLE mode (`_wake_event.clear()`).
|
||||
"""
|
||||
with self._lock:
|
||||
self._closed = False
|
||||
self._state = ConnectionState.CONNECTED_IDLE
|
||||
self._recovery_attempts = 0
|
||||
self._last_error = None
|
||||
self._last_verified_at = time.time()
|
||||
self._wake_event.clear()
|
||||
|
||||
def trigger_fallback(self, reason: str = "WebSocket disconnected") -> None:
|
||||
"""
|
||||
Transitions to `FALLBACK_HTTP` and wakes up the Heartbeat Monitor thread
|
||||
to repair the WebSocket connection in the background.
|
||||
"""
|
||||
with self._lock:
|
||||
if self._closed:
|
||||
return
|
||||
self._state = ConnectionState.FALLBACK_HTTP
|
||||
self._last_error = reason
|
||||
self._ensure_thread_started()
|
||||
self._wake_event.set()
|
||||
|
||||
def trigger_offline(self, reason: str = "Device offline") -> None:
|
||||
"""
|
||||
Transitions to `OFFLINE_SPOOL` when both WebSocket and HTTP fallback are
|
||||
unreachable, waking the Heartbeat Monitor to watch for network return.
|
||||
"""
|
||||
with self._lock:
|
||||
if self._closed:
|
||||
return
|
||||
self._state = ConnectionState.OFFLINE_SPOOL
|
||||
self._last_error = reason
|
||||
self._ensure_thread_started()
|
||||
self._wake_event.set()
|
||||
|
||||
def set_reconnecting(self) -> None:
|
||||
with self._lock:
|
||||
if not self._closed:
|
||||
self._state = ConnectionState.RECONNECTING_VERIFY
|
||||
|
||||
def start(self) -> None:
|
||||
with self._lock:
|
||||
self._closed = False
|
||||
self._stop_event.clear()
|
||||
self._ensure_thread_started()
|
||||
|
||||
def stop(self, reason: Optional[str] = None) -> None:
|
||||
with self._lock:
|
||||
self._closed = True
|
||||
self._state = ConnectionState.STOPPED
|
||||
if reason:
|
||||
self._last_error = reason
|
||||
self._stop_event.set()
|
||||
self._wake_event.set()
|
||||
|
||||
def status_snapshot(self) -> Dict[str, Any]:
|
||||
with self._lock:
|
||||
return {
|
||||
"state": self._state,
|
||||
"heartbeatMode": (
|
||||
"IDLE"
|
||||
if self._state == ConnectionState.CONNECTED_IDLE
|
||||
else "ACTIVE_RECOVERY"
|
||||
if self._state
|
||||
in (
|
||||
ConnectionState.FALLBACK_HTTP,
|
||||
ConnectionState.OFFLINE_SPOOL,
|
||||
ConnectionState.RECONNECTING_VERIFY,
|
||||
)
|
||||
else "STOPPED"
|
||||
),
|
||||
"recoveryAttempts": self._recovery_attempts,
|
||||
"lastError": self._last_error,
|
||||
"lastVerifiedAt": self._last_verified_at,
|
||||
"intervalSec": self.interval_sec,
|
||||
}
|
||||
|
||||
def _ensure_thread_started(self) -> None:
|
||||
if self._thread is not None and self._thread.is_alive():
|
||||
return
|
||||
self._stop_event.clear()
|
||||
self._thread = threading.Thread(
|
||||
target=self._run_loop,
|
||||
name="DataStack-HeartbeatMonitor",
|
||||
daemon=True,
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
def _run_loop(self) -> None:
|
||||
while not self._stop_event.is_set():
|
||||
# Wait in zero-CPU IDLE mode until woken by a disconnect or offline trigger
|
||||
self._wake_event.wait()
|
||||
if self._stop_event.is_set():
|
||||
break
|
||||
|
||||
# Wait for the heartbeat interval before probing
|
||||
if self._stop_event.wait(self.interval_sec):
|
||||
break
|
||||
|
||||
if not self._wake_event.is_set():
|
||||
continue
|
||||
|
||||
with self._lock:
|
||||
if self._closed or self._state == ConnectionState.STOPPED:
|
||||
break
|
||||
self._recovery_attempts += 1
|
||||
|
||||
try:
|
||||
recovered_to_idle = self._recovery_callback()
|
||||
if recovered_to_idle:
|
||||
self.set_idle_connected()
|
||||
logger.info(
|
||||
"[DataStack Heartbeat] WebSocket reconnected & verified. Returning to IDLE mode."
|
||||
)
|
||||
except Exception as exc:
|
||||
with self._lock:
|
||||
self._last_error = str(exc)
|
||||
logger.debug("[DataStack Heartbeat] Recovery probe failed: %s", exc)
|
||||
from datastack.heartbeat import * # noqa: E402, F401, F403
|
||||
@@ -1,164 +1,7 @@
|
||||
"""
|
||||
Thread-Safe Offline Disk Spooler (`DiskSpooler`).
|
||||
Persists unsent telemetry events to `.datastack_spool/spool.jsonl` when a client
|
||||
device loses network connectivity completely, and drains them in deterministic
|
||||
batches once the Heartbeat Monitor restores connectivity.
|
||||
"""
|
||||
"""Compatibility re-export of canonical `datastack.spool`."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
from . import _ensure_canonical_datastack_on_path
|
||||
|
||||
from .exceptions import DataStackOfflineSpoolError
|
||||
_ensure_canonical_datastack_on_path()
|
||||
|
||||
|
||||
class DiskSpooler:
|
||||
"""
|
||||
Append-only JSONL disk queue with atomic batch drainage and size cap protection.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
spool_dir: str = ".datastack_spool",
|
||||
max_size_mb: int = 50,
|
||||
enabled: bool = True,
|
||||
) -> None:
|
||||
self.enabled = enabled
|
||||
self.spool_dir = Path(spool_dir)
|
||||
self.spool_file = self.spool_dir / "spool.jsonl"
|
||||
self.max_bytes = max(1, int(max_size_mb)) * 1024 * 1024
|
||||
self._lock = threading.RLock()
|
||||
|
||||
def _ensure_dir(self) -> None:
|
||||
try:
|
||||
self.spool_dir.mkdir(parents=True, exist_ok=True)
|
||||
except Exception as exc:
|
||||
raise DataStackOfflineSpoolError(
|
||||
f"Cannot create offline spool directory '{self.spool_dir}': {exc}"
|
||||
) from exc
|
||||
|
||||
def enqueue(self, events: List[Dict[str, Any]]) -> int:
|
||||
"""
|
||||
Appends one or more formatted events (`{eventId, timestamp, data}`)
|
||||
to `.datastack_spool/spool.jsonl`. Returns count of spooled events.
|
||||
"""
|
||||
if not self.enabled or not events:
|
||||
return 0
|
||||
|
||||
with self._lock:
|
||||
self._ensure_dir()
|
||||
current_size = (
|
||||
self.spool_file.stat().st_size if self.spool_file.exists() else 0
|
||||
)
|
||||
|
||||
lines: List[str] = []
|
||||
added_bytes = 0
|
||||
for evt in events:
|
||||
serialized = json.dumps(evt, ensure_ascii=False) + "\n"
|
||||
encoded_len = len(serialized.encode("utf-8"))
|
||||
if current_size + added_bytes + encoded_len > self.max_bytes:
|
||||
raise DataStackOfflineSpoolError(
|
||||
f"Offline spool limit ({self.max_bytes // (1024 * 1024)} MB) "
|
||||
f"reached at '{self.spool_file}'."
|
||||
)
|
||||
lines.append(serialized)
|
||||
added_bytes += encoded_len
|
||||
|
||||
try:
|
||||
with self.spool_file.open("a", encoding="utf-8") as fh:
|
||||
fh.writelines(lines)
|
||||
fh.flush()
|
||||
except Exception as exc:
|
||||
raise DataStackOfflineSpoolError(
|
||||
f"Failed writing to offline spool file '{self.spool_file}': {exc}"
|
||||
) from exc
|
||||
|
||||
return len(lines)
|
||||
|
||||
def peek_batch(self, limit: int = 250) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Reads up to `limit` spooled events from the front of the spool file
|
||||
without removing them until `ack_batch()` confirms delivery.
|
||||
"""
|
||||
if not self.enabled:
|
||||
return []
|
||||
|
||||
with self._lock:
|
||||
if not self.spool_file.exists():
|
||||
return []
|
||||
|
||||
batch: List[Dict[str, Any]] = []
|
||||
try:
|
||||
with self.spool_file.open("r", encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
if len(batch) >= limit:
|
||||
break
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
try:
|
||||
parsed = json.loads(stripped)
|
||||
if isinstance(parsed, dict):
|
||||
batch.append(parsed)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
return batch
|
||||
|
||||
def ack_batch(self, count: int) -> int:
|
||||
"""
|
||||
Removes the first `count` non-empty lines from the spool file atomically
|
||||
after the Heartbeat Monitor confirms they have been ingested by DataStack.
|
||||
"""
|
||||
if not self.enabled or count <= 0:
|
||||
return 0
|
||||
|
||||
with self._lock:
|
||||
if not self.spool_file.exists():
|
||||
return 0
|
||||
|
||||
try:
|
||||
with self.spool_file.open("r", encoding="utf-8") as fh:
|
||||
all_lines = [ln for ln in fh if ln.strip()]
|
||||
|
||||
if count >= len(all_lines):
|
||||
self.spool_file.unlink(missing_ok=True)
|
||||
return len(all_lines)
|
||||
|
||||
remaining = all_lines[count:]
|
||||
tmp_file = self.spool_dir / "spool.jsonl.tmp"
|
||||
with tmp_file.open("w", encoding="utf-8") as fh:
|
||||
fh.writelines(remaining)
|
||||
fh.flush()
|
||||
os.replace(str(tmp_file), str(self.spool_file))
|
||||
return count
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
def pending_count(self) -> int:
|
||||
"""Returns the number of queued events waiting in the disk spool."""
|
||||
if not self.enabled:
|
||||
return 0
|
||||
|
||||
with self._lock:
|
||||
if not self.spool_file.exists():
|
||||
return 0
|
||||
try:
|
||||
with self.spool_file.open("r", encoding="utf-8") as fh:
|
||||
return sum(1 for ln in fh if ln.strip())
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
def size_bytes(self) -> int:
|
||||
"""Returns current disk usage of the spool file in bytes."""
|
||||
with self._lock:
|
||||
if not self.spool_file.exists():
|
||||
return 0
|
||||
try:
|
||||
return self.spool_file.stat().st_size
|
||||
except Exception:
|
||||
return 0
|
||||
from datastack.spool import * # noqa: E402, F401, F403
|
||||
@@ -33,6 +33,7 @@ DATASTACK_CONFIG = {
|
||||
"SERVER_URL": "http://localhost:4000", # Port 4000 Socket.IO Ingestion Service
|
||||
"API_URL": "http://localhost:3000", # Port 3000 HTTP Fallback & File Ingestion API
|
||||
"CLIENT_TOKEN": "YOUR_CLIENT_JWT_TOKEN", # Provisioned in /vcon/clients
|
||||
"CLIENT_ID": "", # Optional override (defaults to JWT sub/clientId)
|
||||
"AUTO_CONNECT": True,
|
||||
"ENABLE_HTTP_FALLBACK": True,
|
||||
"HEARTBEAT_INTERVAL_SEC": 5.0,
|
||||
|
||||
@@ -5,6 +5,7 @@ Socket.IO ingestion (Port 4000), automatic Heartbeat HTTP fallback (Port 3000),
|
||||
offline disk spooling, and direct `.csv` / `.json` / `.jsonl` file streaming.
|
||||
"""
|
||||
|
||||
from .client import DataStack, DataStackClient
|
||||
from .config import DataStackConfig, load_config
|
||||
from .exceptions import (
|
||||
DataStackAuthError,
|
||||
@@ -16,14 +17,9 @@ from .exceptions import (
|
||||
DataStackPayloadTooLargeError,
|
||||
DataStackRateLimitError,
|
||||
)
|
||||
from .heartbeat import ConnectionState, HeartbeatMonitor
|
||||
from .spool import DiskSpooler
|
||||
|
||||
try:
|
||||
from .client import DataStack, DataStackClient
|
||||
except ImportError:
|
||||
DataStack = None # type: ignore
|
||||
DataStackClient = None # type: ignore
|
||||
|
||||
__version__ = "2.2.0"
|
||||
|
||||
__all__ = [
|
||||
@@ -31,6 +27,8 @@ __all__ = [
|
||||
"DataStackClient",
|
||||
"DataStackConfig",
|
||||
"load_config",
|
||||
"ConnectionState",
|
||||
"HeartbeatMonitor",
|
||||
"DiskSpooler",
|
||||
"DataStackError",
|
||||
"DataStackConfigError",
|
||||
|
||||
@@ -13,6 +13,7 @@ Features:
|
||||
and lossless partial-batch rate-limit retry (`429 RATE_LIMIT_EXCEEDED`).
|
||||
"""
|
||||
|
||||
import base64
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
@@ -44,6 +45,7 @@ from .exceptions import (
|
||||
DataStackConnectionError,
|
||||
DataStackError,
|
||||
DataStackFileError,
|
||||
DataStackOfflineSpoolError,
|
||||
DataStackPayloadTooLargeError,
|
||||
DataStackRateLimitError,
|
||||
)
|
||||
@@ -72,6 +74,23 @@ def _make_deterministic_file_event_id(
|
||||
return f"file_{safe_stem}_{row_index}_{digest}"
|
||||
|
||||
|
||||
def _decode_jwt_client_id(token: str) -> str:
|
||||
"""Safely extracts `clientId` or `sub` claim from an unverified JWT payload string."""
|
||||
try:
|
||||
parts = (token or "").split(".")
|
||||
if len(parts) != 3:
|
||||
return ""
|
||||
payload_b64 = parts[1]
|
||||
padding = "=" * (-len(payload_b64) % 4)
|
||||
raw = base64.urlsafe_b64decode(payload_b64 + padding).decode("utf-8")
|
||||
claims = json.loads(raw)
|
||||
if isinstance(claims, dict):
|
||||
return str(claims.get("clientId") or claims.get("sub") or "").strip()
|
||||
except Exception:
|
||||
pass
|
||||
return ""
|
||||
|
||||
|
||||
class DataStackClient:
|
||||
"""
|
||||
Production-grade DataStack Python Client with `config.datastack.py` auto-loading,
|
||||
@@ -87,6 +106,7 @@ class DataStackClient:
|
||||
auto_connect: Optional[bool] = None,
|
||||
reconnection_attempts: Optional[int] = None,
|
||||
reconnection_delay: Optional[float] = None,
|
||||
client_id: Optional[str] = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if reconnection_attempts is not None and "MAX_RETRIES" not in kwargs:
|
||||
@@ -105,6 +125,7 @@ class DataStackClient:
|
||||
SERVER_URL=server_url,
|
||||
CLIENT_TOKEN=token,
|
||||
API_URL=api_url,
|
||||
CLIENT_ID=client_id,
|
||||
AUTO_CONNECT=auto_connect,
|
||||
**kwargs,
|
||||
)
|
||||
@@ -112,6 +133,20 @@ class DataStackClient:
|
||||
self.server_url = self.config.server_url
|
||||
self.api_url = self.config.api_url
|
||||
self.token = self.config.client_token
|
||||
|
||||
configured_client_id = (self.config.client_id or "").strip()
|
||||
token_client_id = _decode_jwt_client_id(self.token)
|
||||
if (
|
||||
configured_client_id
|
||||
and token_client_id
|
||||
and configured_client_id != token_client_id
|
||||
):
|
||||
raise DataStackConfigError(
|
||||
f"Configured CLIENT_ID '{configured_client_id}' does not match "
|
||||
f"CLIENT_TOKEN subject '{token_client_id}'."
|
||||
)
|
||||
self.client_id: str = configured_client_id or token_client_id
|
||||
|
||||
self.is_connected = False
|
||||
self._lock = threading.RLock()
|
||||
self._is_draining_spool = False
|
||||
@@ -311,8 +346,11 @@ class DataStackClient:
|
||||
raise DataStackAuthError(msg) from exc
|
||||
return False
|
||||
|
||||
# Verify handshake & PostgreSQL client status via `ping_verify`
|
||||
return self._verify_websocket_handshake(timeout=effective_timeout)
|
||||
# Verify handshake & PostgreSQL client status via `ping_verify` while still serialized under RLock
|
||||
verified = self._verify_websocket_handshake(timeout=effective_timeout)
|
||||
if not verified:
|
||||
self.is_connected = False
|
||||
return verified
|
||||
|
||||
def _verify_websocket_handshake(
|
||||
self, timeout: Optional[float] = None
|
||||
@@ -424,6 +462,8 @@ class DataStackClient:
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
if self.client_id:
|
||||
headers["X-DataStack-Client-Id"] = self.client_id
|
||||
|
||||
if requests is not None:
|
||||
resp = requests.request(
|
||||
@@ -735,10 +775,14 @@ class DataStackClient:
|
||||
):
|
||||
break
|
||||
|
||||
self.spooler.ack_batch(len(batch))
|
||||
if isinstance(unprocessed, list) and len(unprocessed) > 0:
|
||||
self.spooler.enqueue(unprocessed)
|
||||
total_flushed += max(0, len(batch) - len(unprocessed))
|
||||
unprocessed_list = (
|
||||
unprocessed if isinstance(unprocessed, list) else []
|
||||
)
|
||||
self.spooler.ack_and_requeue_front(
|
||||
len(batch), unprocessed_list
|
||||
)
|
||||
if unprocessed_list:
|
||||
total_flushed += max(0, len(batch) - len(unprocessed_list))
|
||||
break
|
||||
|
||||
total_flushed += len(batch)
|
||||
@@ -767,11 +811,13 @@ class DataStackClient:
|
||||
"""
|
||||
resolved_id = (event_id or "").strip() or _make_event_id()
|
||||
resolved_ts = (timestamp or "").strip() or _utc_now_iso()
|
||||
envelope = {
|
||||
envelope: Dict[str, Any] = {
|
||||
"eventId": resolved_id,
|
||||
"timestamp": resolved_ts,
|
||||
"data": data,
|
||||
}
|
||||
if self.client_id:
|
||||
envelope["clientId"] = self.client_id
|
||||
return self._execute_with_retry_and_fallback(
|
||||
ws_event="sync_data",
|
||||
ws_payload=envelope,
|
||||
@@ -810,6 +856,7 @@ class DataStackClient:
|
||||
if not isinstance(source_list, list) or len(source_list) == 0:
|
||||
return {"success": True, "processed": 0, "rejected": 0}
|
||||
|
||||
envelope_keys = {"eventId", "timestamp", "data", "clientId"}
|
||||
formatted: List[Dict[str, Any]] = []
|
||||
for item in source_list:
|
||||
safe_item = item if isinstance(item, dict) else {"value": item}
|
||||
@@ -819,14 +866,21 @@ class DataStackClient:
|
||||
evt_ts = (
|
||||
str(safe_item.get("timestamp") or "").strip() or _utc_now_iso()
|
||||
)
|
||||
evt_data = safe_item.get("data", safe_item)
|
||||
formatted.append(
|
||||
{
|
||||
"eventId": evt_id,
|
||||
"timestamp": evt_ts,
|
||||
"data": evt_data,
|
||||
}
|
||||
is_envelope = "data" in safe_item and set(safe_item.keys()).issubset(
|
||||
envelope_keys
|
||||
)
|
||||
evt_data = safe_item["data"] if is_envelope else safe_item
|
||||
evt_client_id = (
|
||||
str(safe_item.get("clientId") or "").strip() or self.client_id
|
||||
)
|
||||
entry: Dict[str, Any] = {
|
||||
"eventId": evt_id,
|
||||
"timestamp": evt_ts,
|
||||
"data": evt_data,
|
||||
}
|
||||
if evt_client_id:
|
||||
entry["clientId"] = evt_client_id
|
||||
formatted.append(entry)
|
||||
|
||||
return self._send_formatted_batch_recursive(
|
||||
formatted,
|
||||
@@ -1057,13 +1111,14 @@ class DataStackClient:
|
||||
)
|
||||
evt_ts = existing_ts or _utc_now_iso()
|
||||
|
||||
buffer.append(
|
||||
{
|
||||
"eventId": evt_id,
|
||||
"timestamp": evt_ts,
|
||||
"data": enriched,
|
||||
}
|
||||
)
|
||||
file_entry: Dict[str, Any] = {
|
||||
"eventId": evt_id,
|
||||
"timestamp": evt_ts,
|
||||
"data": enriched,
|
||||
}
|
||||
if self.client_id:
|
||||
file_entry["clientId"] = self.client_id
|
||||
buffer.append(file_entry)
|
||||
if len(buffer) >= effective_chunk:
|
||||
res = self.send_batch(buffer, file_name=path.name)
|
||||
total_processed += int(res.get("processed", 0))
|
||||
@@ -1162,6 +1217,7 @@ class DataStackClient:
|
||||
hb = self.heartbeat.status_snapshot()
|
||||
return {
|
||||
"connectedWebSocket": self.is_connected,
|
||||
"clientId": self.client_id or None,
|
||||
"serverUrl": self.server_url,
|
||||
"apiUrl": self.api_url,
|
||||
"configSource": self.config.config_source_path,
|
||||
@@ -1176,3 +1232,21 @@ class DataStackClient:
|
||||
|
||||
# Convenient alias so developers can write `from datastack import DataStack`
|
||||
DataStack = DataStackClient
|
||||
|
||||
__all__ = [
|
||||
"DataStack",
|
||||
"DataStackClient",
|
||||
"DataStackConfig",
|
||||
"load_config",
|
||||
"ConnectionState",
|
||||
"HeartbeatMonitor",
|
||||
"DiskSpooler",
|
||||
"DataStackError",
|
||||
"DataStackConfigError",
|
||||
"DataStackAuthError",
|
||||
"DataStackConnectionError",
|
||||
"DataStackRateLimitError",
|
||||
"DataStackPayloadTooLargeError",
|
||||
"DataStackFileError",
|
||||
"DataStackOfflineSpoolError",
|
||||
]
|
||||
@@ -171,6 +171,11 @@ class HeartbeatMonitor:
|
||||
with self._lock:
|
||||
if self._closed or self._state == ConnectionState.STOPPED:
|
||||
break
|
||||
if (
|
||||
self._state == ConnectionState.CONNECTED_IDLE
|
||||
or not self._wake_event.is_set()
|
||||
):
|
||||
continue
|
||||
self._recovery_attempts += 1
|
||||
|
||||
try:
|
||||
|
||||
@@ -9,7 +9,7 @@ import json
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .exceptions import DataStackOfflineSpoolError
|
||||
|
||||
@@ -139,6 +139,49 @@ class DiskSpooler:
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
def ack_and_requeue_front(
|
||||
self,
|
||||
acked_count: int,
|
||||
unprocessed: Optional[List[Dict[str, Any]]] = None,
|
||||
) -> int:
|
||||
"""
|
||||
Atomically removes `acked_count` head lines from `spool.jsonl` while
|
||||
preserving any `unprocessed` events at the FRONT of the queue in a single
|
||||
atomic `os.replace()` write (FIFO safe, zero crash-window data loss).
|
||||
"""
|
||||
if not self.enabled or acked_count <= 0:
|
||||
return 0
|
||||
|
||||
with self._lock:
|
||||
if not self.spool_file.exists():
|
||||
return 0
|
||||
|
||||
try:
|
||||
with self.spool_file.open("r", encoding="utf-8") as fh:
|
||||
all_lines = [ln for ln in fh if ln.strip()]
|
||||
|
||||
consumed = min(acked_count, len(all_lines))
|
||||
tail_lines = all_lines[consumed:]
|
||||
head_lines = [
|
||||
json.dumps(evt, ensure_ascii=False) + "\n"
|
||||
for evt in (unprocessed or [])
|
||||
]
|
||||
combined = head_lines + tail_lines
|
||||
|
||||
if not combined:
|
||||
self.spool_file.unlink(missing_ok=True)
|
||||
return consumed
|
||||
|
||||
self._ensure_dir()
|
||||
tmp_file = self.spool_dir / "spool.jsonl.tmp"
|
||||
with tmp_file.open("w", encoding="utf-8") as fh:
|
||||
fh.writelines(combined)
|
||||
fh.flush()
|
||||
os.replace(str(tmp_file), str(self.spool_file))
|
||||
return max(0, consumed - len(head_lines))
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
def pending_count(self) -> int:
|
||||
"""Returns the number of queued events waiting in the disk spool."""
|
||||
if not self.enabled:
|
||||
|
||||
+714
-98
@@ -1,8 +1,10 @@
|
||||
const crypto = require('node:crypto');
|
||||
const express = require('express');
|
||||
const http = require('http');
|
||||
const cors = require('cors');
|
||||
const { Server } = require('socket.io');
|
||||
const { Pool } = require('pg');
|
||||
const Redis = require('ioredis');
|
||||
const { setupSocketHandlers } = require('./socket/handlers');
|
||||
const { CsvWriterWorker } = require('./workers/csvWriter');
|
||||
|
||||
@@ -11,21 +13,185 @@ const app = express();
|
||||
app.use(cors({ origin: '*' }));
|
||||
app.use(express.json({ limit: '10mb' }));
|
||||
|
||||
const REDIS_BUFFER_KEY = 'datastack:buffer:events';
|
||||
const REDIS_RECENT_KEY = 'datastack:recent:events';
|
||||
const REDIS_FLUSH_LOCK_KEY = 'datastack:flush:lock';
|
||||
const REDIS_RECENT_LIMIT = 500;
|
||||
const FLUSH_LOCK_TTL_MS = 15000;
|
||||
const UNLOCK_LUA_SCRIPT =
|
||||
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end';
|
||||
|
||||
const bufferQueue = [];
|
||||
const writer = new CsvWriterWorker();
|
||||
let redisClient = undefined;
|
||||
let redisUrlConfigured = '';
|
||||
let redisConnected = false;
|
||||
let redisTestOverride = undefined;
|
||||
const pendingRedisWrites = new Set();
|
||||
|
||||
function getRedisClient() {
|
||||
if (redisTestOverride !== undefined) {
|
||||
return redisTestOverride;
|
||||
}
|
||||
|
||||
const configuredUrl = (process.env.REDIS_URL || '').trim();
|
||||
if (!configuredUrl) {
|
||||
if (redisClient && typeof redisClient.disconnect === 'function') {
|
||||
try {
|
||||
redisClient.disconnect();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
redisClient = null;
|
||||
redisUrlConfigured = '';
|
||||
redisConnected = false;
|
||||
return null;
|
||||
}
|
||||
|
||||
if (redisClient && redisUrlConfigured === configuredUrl) {
|
||||
return redisClient;
|
||||
}
|
||||
|
||||
if (redisClient && typeof redisClient.disconnect === 'function') {
|
||||
try {
|
||||
redisClient.disconnect();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const client = new Redis(configuredUrl, {
|
||||
maxRetriesPerRequest: 2,
|
||||
enableReadyCheck: true,
|
||||
retryStrategy: (times) => Math.min(times * 250, 3000),
|
||||
});
|
||||
client.on('ready', () => {
|
||||
redisConnected = true;
|
||||
});
|
||||
client.on('connect', () => {
|
||||
redisConnected = true;
|
||||
});
|
||||
client.on('close', () => {
|
||||
redisConnected = false;
|
||||
});
|
||||
client.on('end', () => {
|
||||
redisConnected = false;
|
||||
});
|
||||
client.on('error', () => {
|
||||
redisConnected = false;
|
||||
});
|
||||
redisClient = client;
|
||||
redisUrlConfigured = configuredUrl;
|
||||
redisConnected = client.status === 'ready' || client.status === 'connect';
|
||||
return redisClient;
|
||||
} catch {
|
||||
redisClient = null;
|
||||
redisUrlConfigured = configuredUrl;
|
||||
redisConnected = false;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function setRedisClientForTests(client) {
|
||||
redisTestOverride = client;
|
||||
redisConnected = Boolean(client);
|
||||
}
|
||||
|
||||
async function pushEventsToRedisOrFallback(events) {
|
||||
if (!events || events.length === 0) return bufferQueue.length;
|
||||
const redis = getRedisClient();
|
||||
if (redis) {
|
||||
const serialized = events.map((e) => JSON.stringify(e));
|
||||
try {
|
||||
const len = await redis.rpush(REDIS_BUFFER_KEY, ...serialized);
|
||||
await redis.lpush(REDIS_RECENT_KEY, ...serialized);
|
||||
await redis.ltrim(REDIS_RECENT_KEY, 0, REDIS_RECENT_LIMIT - 1);
|
||||
redisConnected = true;
|
||||
return Number(len) + bufferQueue.length;
|
||||
} catch {
|
||||
redisConnected = false;
|
||||
}
|
||||
}
|
||||
|
||||
bufferQueue.push(...events);
|
||||
return bufferQueue.length;
|
||||
}
|
||||
|
||||
const IS_PRODUCTION = process.env.NODE_ENV === 'production';
|
||||
const DEV_DEFAULT_JWT_SECRET = 'datastack-jwt-default-secret-key-32-chars-long';
|
||||
|
||||
function resolveInitialJwtSecret() {
|
||||
const configured = (process.env.DATASTACK_JWT_SECRET || '').trim();
|
||||
if (IS_PRODUCTION) {
|
||||
return configured.length >= 32 ? configured : '';
|
||||
}
|
||||
if (configured.length > 0) {
|
||||
return configured.length >= 32 ? configured : '';
|
||||
}
|
||||
return DEV_DEFAULT_JWT_SECRET;
|
||||
}
|
||||
|
||||
const runtimeConfig = {
|
||||
jwtSecret:
|
||||
process.env.DATASTACK_JWT_SECRET ||
|
||||
'datastack-jwt-default-secret-key-32-chars-long',
|
||||
jwtSecret: resolveInitialJwtSecret(),
|
||||
maxEventsPerSec: 100,
|
||||
maxPayloadSizeBytes: 100 * 1024,
|
||||
batchFlushThreshold: 500,
|
||||
batchFlushIntervalMs: 2000,
|
||||
fileRotationMb: 100,
|
||||
enableIdempotency: true,
|
||||
autoPruneDays: 30,
|
||||
};
|
||||
|
||||
function resolveInternalSecret(jwtSecretOverride) {
|
||||
const explicit = (process.env.DATASTACK_INTERNAL_SECRET || '').trim();
|
||||
if (IS_PRODUCTION) {
|
||||
if (explicit.length >= 32) return explicit;
|
||||
if (explicit.length > 0) return '';
|
||||
} else if (explicit.length > 0) {
|
||||
return explicit;
|
||||
}
|
||||
const activeJwt = String(
|
||||
jwtSecretOverride !== undefined ? jwtSecretOverride : runtimeConfig.jwtSecret || '',
|
||||
).trim();
|
||||
const minJwtLen = IS_PRODUCTION ? 32 : 1;
|
||||
if (activeJwt.length < minJwtLen) {
|
||||
return '';
|
||||
}
|
||||
return crypto
|
||||
.createHash('sha256')
|
||||
.update(`datastack-internal:${activeJwt}`)
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
function timingSafeSecretEqual(candidate, expected) {
|
||||
if (!candidate || !expected) return false;
|
||||
const a = crypto.createHash('sha256').update(String(candidate)).digest();
|
||||
const b = crypto.createHash('sha256').update(String(expected)).digest();
|
||||
return crypto.timingSafeEqual(a, b);
|
||||
}
|
||||
|
||||
function requireInternalAuth(req, res, next) {
|
||||
const expected = resolveInternalSecret();
|
||||
if (!expected) {
|
||||
return res.status(503).json({
|
||||
success: false,
|
||||
error: 'INTERNAL_AUTH_UNAVAILABLE: Internal bridge secret is not configured',
|
||||
code: 'INTERNAL_AUTH_UNAVAILABLE',
|
||||
});
|
||||
}
|
||||
const presented = req.get('x-datastack-internal-secret');
|
||||
if (!presented || !timingSafeSecretEqual(presented, expected)) {
|
||||
return res.status(401).json({
|
||||
success: false,
|
||||
error: 'UNAUTHORIZED: Missing or invalid internal authentication header',
|
||||
code: 'UNAUTHORIZED',
|
||||
});
|
||||
}
|
||||
return next();
|
||||
}
|
||||
|
||||
let dbPool = null;
|
||||
if (process.env.DATABASE_URL) {
|
||||
try {
|
||||
@@ -40,43 +206,95 @@ if (process.env.DATABASE_URL) {
|
||||
}
|
||||
}
|
||||
|
||||
// Short-lived cache (2s TTL) for client status & token checks on high-frequency events
|
||||
// Short-lived cache (2s TTL) for DB client status & token checks on high-frequency events
|
||||
const clientAuthCache = new Map();
|
||||
const deletedClientsSet = new Set();
|
||||
const inMemoryClientRegistry = new Map();
|
||||
const inMemoryClientActivity = new Map();
|
||||
let hasSyncedInMemoryClients = false;
|
||||
const CLIENT_CACHE_TTL_MS = 2000;
|
||||
|
||||
async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
|
||||
if (!dbPool || !clientId) {
|
||||
if (!clientId) {
|
||||
return { allowed: true };
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
if (!bypassCache) {
|
||||
const cached = clientAuthCache.get(clientId);
|
||||
if (cached && now - cached.checkedAt < CLIENT_CACHE_TTL_MS) {
|
||||
if (cached.status === 'revoked') {
|
||||
const normalizedId = String(clientId).trim().toLowerCase();
|
||||
if (deletedClientsSet.has(normalizedId)) {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: 'CLIENT_REVOKED: Client access has been deleted or revoked by admin',
|
||||
};
|
||||
}
|
||||
|
||||
if (!dbPool) {
|
||||
const memClient = inMemoryClientRegistry.get(normalizedId);
|
||||
if (memClient) {
|
||||
if (memClient.status === 'revoked') {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
|
||||
};
|
||||
}
|
||||
if (cached.token && presentedToken && cached.token !== presentedToken) {
|
||||
if (memClient.token && presentedToken && memClient.token !== presentedToken) {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
|
||||
};
|
||||
}
|
||||
return { allowed: true, rateLimit: cached.rateLimit };
|
||||
return { allowed: true, rateLimit: memClient.rateLimit || 100 };
|
||||
}
|
||||
if (hasSyncedInMemoryClients) {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: 'CLIENT_REVOKED: Client registration not found or has been deleted',
|
||||
};
|
||||
}
|
||||
return { allowed: true };
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
const prevCached = clientAuthCache.get(normalizedId);
|
||||
if (!bypassCache && prevCached && now - prevCached.checkedAt < CLIENT_CACHE_TTL_MS) {
|
||||
if (prevCached.status === 'revoked') {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
|
||||
};
|
||||
}
|
||||
if (prevCached.token && presentedToken && prevCached.token !== presentedToken) {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
|
||||
};
|
||||
}
|
||||
return { allowed: true, rateLimit: prevCached.rateLimit };
|
||||
}
|
||||
|
||||
try {
|
||||
const deletedRes = await dbPool.query(
|
||||
`SELECT 1 FROM deleted_clients WHERE client_id = $1`,
|
||||
[normalizedId],
|
||||
);
|
||||
if (deletedRes.rowCount && deletedRes.rowCount > 0) {
|
||||
clientAuthCache.delete(normalizedId);
|
||||
deletedClientsSet.add(normalizedId);
|
||||
return {
|
||||
allowed: false,
|
||||
reason: 'CLIENT_REVOKED: Client access has been deleted by admin',
|
||||
};
|
||||
}
|
||||
|
||||
const res = await dbPool.query(
|
||||
`SELECT status, token, rate_limit FROM client_sdks WHERE client_id = $1`,
|
||||
[clientId],
|
||||
[normalizedId],
|
||||
);
|
||||
if (!res.rowCount || res.rowCount === 0) {
|
||||
// Allow token-authenticated client if not explicitly revoked in table
|
||||
return { allowed: true };
|
||||
clientAuthCache.delete(normalizedId);
|
||||
return {
|
||||
allowed: false,
|
||||
reason: 'CLIENT_REVOKED: Client registration not found or has been deleted',
|
||||
};
|
||||
}
|
||||
const row = res.rows[0];
|
||||
const entry = {
|
||||
@@ -85,7 +303,7 @@ async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
|
||||
rateLimit: Number(row.rate_limit) || 100,
|
||||
checkedAt: now,
|
||||
};
|
||||
clientAuthCache.set(clientId, entry);
|
||||
clientAuthCache.set(normalizedId, entry);
|
||||
|
||||
if (entry.status === 'revoked') {
|
||||
return {
|
||||
@@ -101,13 +319,30 @@ async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
|
||||
}
|
||||
return { allowed: true, rateLimit: entry.rateLimit };
|
||||
} catch {
|
||||
return { allowed: true };
|
||||
return {
|
||||
allowed: false,
|
||||
reason: 'AUTH_UNAVAILABLE: Unable to verify client credentials against database',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function recordClientActivity(clientId, count = 1) {
|
||||
if (!dbPool || !clientId || count <= 0) return;
|
||||
if (!clientId || count <= 0) return;
|
||||
const normalizedId = String(clientId).trim().toLowerCase();
|
||||
const nowIso = new Date().toISOString();
|
||||
|
||||
if (!dbPool) {
|
||||
const prev = inMemoryClientActivity.get(normalizedId) || {
|
||||
totalEvents: 0,
|
||||
lastActive: nowIso,
|
||||
};
|
||||
inMemoryClientActivity.set(normalizedId, {
|
||||
totalEvents: prev.totalEvents + count,
|
||||
lastActive: nowIso,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
dbPool
|
||||
.query(
|
||||
`UPDATE client_sdks
|
||||
@@ -115,7 +350,7 @@ function recordClientActivity(clientId, count = 1) {
|
||||
last_active = $3,
|
||||
updated_at = NOW()
|
||||
WHERE client_id = $1`,
|
||||
[clientId, count, nowIso],
|
||||
[normalizedId, count, nowIso],
|
||||
)
|
||||
.catch(() => undefined);
|
||||
}
|
||||
@@ -124,23 +359,103 @@ let isFlushing = false;
|
||||
let flushTimer = null;
|
||||
|
||||
async function flushBuffer() {
|
||||
if (bufferQueue.length === 0 || isFlushing) {
|
||||
if (pendingRedisWrites.size > 0) {
|
||||
await Promise.allSettled([...pendingRedisWrites]);
|
||||
}
|
||||
if (isFlushing) {
|
||||
return { written: 0, files: [] };
|
||||
}
|
||||
isFlushing = true;
|
||||
const batch = bufferQueue.splice(0, bufferQueue.length);
|
||||
let totalWritten = 0;
|
||||
const filesSet = new Set();
|
||||
|
||||
try {
|
||||
const result = await writer.writeBatch(batch);
|
||||
if (result.written > 0) {
|
||||
const redis = getRedisClient();
|
||||
if (redis) {
|
||||
const lockToken = crypto.randomUUID();
|
||||
let lockAcquired = false;
|
||||
|
||||
try {
|
||||
const lockResult = await redis.set(
|
||||
REDIS_FLUSH_LOCK_KEY,
|
||||
lockToken,
|
||||
'PX',
|
||||
FLUSH_LOCK_TTL_MS,
|
||||
'NX',
|
||||
);
|
||||
lockAcquired = lockResult === 'OK';
|
||||
redisConnected = true;
|
||||
} catch {
|
||||
redisConnected = false;
|
||||
lockAcquired = false;
|
||||
}
|
||||
|
||||
if (lockAcquired) {
|
||||
try {
|
||||
const chunkSize = Math.max(500, runtimeConfig.batchFlushThreshold);
|
||||
while (true) {
|
||||
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, chunkSize - 1);
|
||||
if (!rawItems || rawItems.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
const parsedBatch = [];
|
||||
for (const raw of rawItems) {
|
||||
try {
|
||||
const evt = JSON.parse(raw);
|
||||
if (evt && evt.eventId) {
|
||||
parsedBatch.push(evt);
|
||||
}
|
||||
} catch {
|
||||
// Skip malformed entry while still trimming consumed range
|
||||
}
|
||||
}
|
||||
|
||||
if (parsedBatch.length > 0) {
|
||||
const res = await writer.writeBatch(parsedBatch);
|
||||
totalWritten += res.written;
|
||||
for (const f of res.files) filesSet.add(f);
|
||||
}
|
||||
|
||||
// Trim ONLY after CSV batch write succeeds
|
||||
await redis.ltrim(REDIS_BUFFER_KEY, rawItems.length, -1);
|
||||
|
||||
if (rawItems.length < chunkSize) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
|
||||
} catch {
|
||||
// Lock expires automatically via PX TTL
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (bufferQueue.length > 0) {
|
||||
const batch = bufferQueue.splice(0, bufferQueue.length);
|
||||
try {
|
||||
const res = await writer.writeBatch(batch);
|
||||
totalWritten += res.written;
|
||||
for (const f of res.files) filesSet.add(f);
|
||||
} catch (err) {
|
||||
bufferQueue.unshift(...batch);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
const files = Array.from(filesSet);
|
||||
if (totalWritten > 0) {
|
||||
console.log(
|
||||
`[CSV Worker] Flushed ${result.written} events to ${result.files.length} file(s)`,
|
||||
`[CSV Worker] Flushed ${totalWritten} events to ${files.length} file(s)`,
|
||||
);
|
||||
}
|
||||
return result;
|
||||
return { written: totalWritten, files };
|
||||
} catch (err) {
|
||||
console.error('[CSV Worker] Flush error:', err);
|
||||
bufferQueue.unshift(...batch);
|
||||
return { written: 0, files: [] };
|
||||
return { written: totalWritten, files: Array.from(filesSet) };
|
||||
} finally {
|
||||
isFlushing = false;
|
||||
}
|
||||
@@ -159,6 +474,92 @@ function scheduleFlushTimer(intervalMs) {
|
||||
|
||||
scheduleFlushTimer(runtimeConfig.batchFlushIntervalMs);
|
||||
|
||||
const server = http.createServer(app);
|
||||
const io = new Server(server, {
|
||||
cors: { origin: '*', methods: ['GET', 'POST'] },
|
||||
});
|
||||
|
||||
function applyRuntimeSettingsObject(settingsObj) {
|
||||
if (!settingsObj || typeof settingsObj !== 'object') {
|
||||
return { jwtRotated: false };
|
||||
}
|
||||
|
||||
if (settingsObj.maxEventsPerSec !== undefined) {
|
||||
const val = Number(settingsObj.maxEventsPerSec);
|
||||
if (Number.isFinite(val) && val >= 1) {
|
||||
runtimeConfig.maxEventsPerSec = Math.floor(val);
|
||||
}
|
||||
}
|
||||
|
||||
if (settingsObj.maxPayloadSizeKb !== undefined) {
|
||||
const kb = Number(settingsObj.maxPayloadSizeKb);
|
||||
if (Number.isFinite(kb) && kb >= 1) {
|
||||
runtimeConfig.maxPayloadSizeBytes = Math.floor(kb) * 1024;
|
||||
}
|
||||
}
|
||||
|
||||
if (settingsObj.batchFlushThreshold !== undefined) {
|
||||
const threshold = Number(settingsObj.batchFlushThreshold);
|
||||
if (Number.isFinite(threshold) && threshold >= 1) {
|
||||
runtimeConfig.batchFlushThreshold = Math.floor(threshold);
|
||||
}
|
||||
}
|
||||
|
||||
if (settingsObj.enableIdempotency !== undefined) {
|
||||
runtimeConfig.enableIdempotency = Boolean(settingsObj.enableIdempotency);
|
||||
}
|
||||
|
||||
if (settingsObj.fileRotationMb !== undefined) {
|
||||
const nextRotationMb = Number(settingsObj.fileRotationMb);
|
||||
if (
|
||||
Number.isFinite(nextRotationMb) &&
|
||||
nextRotationMb >= 1 &&
|
||||
nextRotationMb !== runtimeConfig.fileRotationMb
|
||||
) {
|
||||
runtimeConfig.fileRotationMb = nextRotationMb;
|
||||
writer.setMaxFileSizeMb(nextRotationMb);
|
||||
}
|
||||
}
|
||||
|
||||
if (settingsObj.autoPruneDays !== undefined && settingsObj.autoPruneDays !== null) {
|
||||
const parsedPrune = Number(settingsObj.autoPruneDays);
|
||||
if (Number.isFinite(parsedPrune) && parsedPrune >= 0) {
|
||||
const nextPruneDays = Math.floor(parsedPrune);
|
||||
const changed = nextPruneDays !== runtimeConfig.autoPruneDays;
|
||||
runtimeConfig.autoPruneDays = nextPruneDays;
|
||||
if (changed && nextPruneDays > 0) {
|
||||
writer.pruneExpiredPartitions(nextPruneDays);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (settingsObj.batchFlushIntervalMs !== undefined) {
|
||||
const nextIntervalMs = Math.max(
|
||||
100,
|
||||
Number(settingsObj.batchFlushIntervalMs) || 2000,
|
||||
);
|
||||
if (nextIntervalMs !== runtimeConfig.batchFlushIntervalMs) {
|
||||
scheduleFlushTimer(nextIntervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
let jwtRotated = false;
|
||||
if (settingsObj.jwtSecret !== undefined && settingsObj.jwtSecret !== null) {
|
||||
const candidateSecret = String(settingsObj.jwtSecret).trim();
|
||||
const validSecret =
|
||||
candidateSecret.length >= 32 ||
|
||||
(!IS_PRODUCTION && candidateSecret.length > 0);
|
||||
if (validSecret && candidateSecret !== runtimeConfig.jwtSecret) {
|
||||
runtimeConfig.jwtSecret = candidateSecret;
|
||||
clientAuthCache.clear();
|
||||
io.disconnectSockets(true);
|
||||
jwtRotated = true;
|
||||
}
|
||||
}
|
||||
|
||||
return { jwtRotated };
|
||||
}
|
||||
|
||||
async function ensureDatabaseSchema() {
|
||||
if (!dbPool) return;
|
||||
try {
|
||||
@@ -192,6 +593,12 @@ async function ensureDatabaseSchema() {
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`);
|
||||
await dbPool.query(`
|
||||
CREATE TABLE IF NOT EXISTS deleted_clients (
|
||||
client_id TEXT PRIMARY KEY,
|
||||
deleted_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`);
|
||||
} catch {
|
||||
// Ignore transient startup DB error; retried on next sync cycle
|
||||
}
|
||||
@@ -210,69 +617,132 @@ async function syncSettingsFromDatabase() {
|
||||
);
|
||||
if (!res.rowCount || res.rowCount === 0) return;
|
||||
const row = res.rows[0];
|
||||
const parsedPrune =
|
||||
row.auto_prune_days !== undefined && row.auto_prune_days !== null
|
||||
? Number(row.auto_prune_days)
|
||||
: NaN;
|
||||
|
||||
runtimeConfig.maxEventsPerSec = Number(row.max_events_per_sec) || 100;
|
||||
runtimeConfig.maxPayloadSizeBytes =
|
||||
(Number(row.max_payload_size_kb) || 100) * 1024;
|
||||
runtimeConfig.batchFlushThreshold =
|
||||
Math.max(1, Number(row.batch_flush_threshold) || 500);
|
||||
runtimeConfig.enableIdempotency = Boolean(row.enable_idempotency);
|
||||
|
||||
if (row.jwt_secret && String(row.jwt_secret).trim().length > 0) {
|
||||
runtimeConfig.jwtSecret = String(row.jwt_secret).trim();
|
||||
}
|
||||
|
||||
const nextRotationMb = Number(row.file_rotation_mb) || 100;
|
||||
if (nextRotationMb !== runtimeConfig.fileRotationMb) {
|
||||
runtimeConfig.fileRotationMb = nextRotationMb;
|
||||
writer.setMaxFileSizeMb(nextRotationMb);
|
||||
}
|
||||
|
||||
const nextPruneDays = Number(row.auto_prune_days) || 30;
|
||||
if (nextPruneDays > 0) {
|
||||
writer.pruneExpiredPartitions(nextPruneDays);
|
||||
}
|
||||
|
||||
const nextIntervalMs = Math.max(
|
||||
100,
|
||||
Number(row.batch_flush_interval_ms) || 2000,
|
||||
);
|
||||
if (nextIntervalMs !== runtimeConfig.batchFlushIntervalMs) {
|
||||
scheduleFlushTimer(nextIntervalMs);
|
||||
}
|
||||
applyRuntimeSettingsObject({
|
||||
maxEventsPerSec: Number(row.max_events_per_sec) || 100,
|
||||
maxPayloadSizeKb: Number(row.max_payload_size_kb) || 100,
|
||||
batchFlushThreshold: Math.max(1, Number(row.batch_flush_threshold) || 500),
|
||||
batchFlushIntervalMs: Math.max(
|
||||
100,
|
||||
Number(row.batch_flush_interval_ms) || 2000,
|
||||
),
|
||||
fileRotationMb: Number(row.file_rotation_mb) || 100,
|
||||
enableIdempotency: Boolean(row.enable_idempotency),
|
||||
autoPruneDays: Number.isFinite(parsedPrune) ? parsedPrune : 30,
|
||||
jwtSecret: row.jwt_secret ? String(row.jwt_secret).trim() : undefined,
|
||||
});
|
||||
} catch {
|
||||
// Keep existing runtimeConfig if table is not ready yet
|
||||
}
|
||||
}
|
||||
|
||||
const PRUNE_INTERVAL_MS = 60 * 60 * 1000;
|
||||
let settingsSyncTimer = null;
|
||||
if (dbPool) {
|
||||
void syncSettingsFromDatabase();
|
||||
settingsSyncTimer = setInterval(() => {
|
||||
let pruneTimer = null;
|
||||
if (process.env.DATASTACK_DISABLE_LISTEN !== '1') {
|
||||
if (dbPool) {
|
||||
void syncSettingsFromDatabase();
|
||||
}, 3000);
|
||||
settingsSyncTimer = setInterval(() => {
|
||||
void syncSettingsFromDatabase();
|
||||
}, 3000);
|
||||
}
|
||||
pruneTimer = setInterval(() => {
|
||||
if (runtimeConfig.autoPruneDays > 0) {
|
||||
writer.pruneExpiredPartitions(runtimeConfig.autoPruneDays);
|
||||
}
|
||||
}, PRUNE_INTERVAL_MS);
|
||||
}
|
||||
|
||||
const server = http.createServer(app);
|
||||
const io = new Server(server, {
|
||||
cors: { origin: '*', methods: ['GET', 'POST'] },
|
||||
const socketHandlersApi = setupSocketHandlers(io, bufferQueue, writer, {
|
||||
getRuntimeConfig: () => runtimeConfig,
|
||||
verifyClientInDb,
|
||||
recordClientActivity,
|
||||
onBufferPush: () => {
|
||||
const redis = getRedisClient();
|
||||
if (redis && bufferQueue.length > 0) {
|
||||
const pushed = bufferQueue.splice(0, bufferQueue.length);
|
||||
const op = pushEventsToRedisOrFallback(pushed).then((totalQueued) => {
|
||||
if (totalQueued >= runtimeConfig.batchFlushThreshold) {
|
||||
void flushBuffer();
|
||||
}
|
||||
});
|
||||
pendingRedisWrites.add(op);
|
||||
void op.finally(() => {
|
||||
pendingRedisWrites.delete(op);
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (bufferQueue.length >= runtimeConfig.batchFlushThreshold) {
|
||||
void flushBuffer();
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
// Health check endpoint with activeSockets count and runtime capabilities
|
||||
app.get('/health', (_req, res) => {
|
||||
async function getRedisBufferLen() {
|
||||
const redis = getRedisClient();
|
||||
if (!redis) return 0;
|
||||
try {
|
||||
const len = Number(await redis.llen(REDIS_BUFFER_KEY)) || 0;
|
||||
redisConnected = true;
|
||||
return len;
|
||||
} catch {
|
||||
redisConnected = false;
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Public health check endpoint
|
||||
app.get('/health', async (_req, res) => {
|
||||
const redisBufferSize = await getRedisBufferLen();
|
||||
res.json({
|
||||
status: 'online',
|
||||
service: 'DataStack Express Ingestion',
|
||||
port: PORT,
|
||||
activeSockets: io.engine ? io.engine.clientsCount : 0,
|
||||
bufferSize: bufferQueue.length,
|
||||
bufferSize: bufferQueue.length + redisBufferSize,
|
||||
localBufferSize: bufferQueue.length,
|
||||
redisBufferSize,
|
||||
redisConnected,
|
||||
maxEventsPerSec: runtimeConfig.maxEventsPerSec,
|
||||
maxPayloadSizeBytes: runtimeConfig.maxPayloadSizeBytes,
|
||||
uptime: process.uptime(),
|
||||
});
|
||||
});
|
||||
|
||||
// Internal flush endpoint triggered by Next.js POST /api/storage
|
||||
// Require internal shared authentication on all /internal/* bridge endpoints
|
||||
app.use('/internal', requireInternalAuth);
|
||||
|
||||
// Internal metrics endpoint for Next.js dashboard & in-memory client activity aggregation
|
||||
app.get('/internal/metrics', async (_req, res) => {
|
||||
const redisBufferSize = await getRedisBufferLen();
|
||||
res.json({
|
||||
activeSockets: io.engine ? io.engine.clientsCount : 0,
|
||||
bufferSize: bufferQueue.length + redisBufferSize,
|
||||
localBufferSize: bufferQueue.length,
|
||||
redisBufferSize,
|
||||
redisConnected,
|
||||
registrySize: inMemoryClientRegistry.size,
|
||||
hasSyncedInMemoryClients,
|
||||
securityStats: socketHandlersApi.getSecurityStats(),
|
||||
clientActivity: Object.fromEntries(inMemoryClientActivity.entries()),
|
||||
});
|
||||
});
|
||||
|
||||
// Internal instant settings synchronization triggered by Next.js POST /api/settings
|
||||
app.post('/internal/settings-sync', (req, res) => {
|
||||
const result = applyRuntimeSettingsObject(req.body || {});
|
||||
res.json({
|
||||
success: true,
|
||||
jwtRotated: result.jwtRotated,
|
||||
});
|
||||
});
|
||||
|
||||
// Internal flush endpoint triggered by Next.js POST /api/storage & GET /api/export
|
||||
app.post('/internal/flush', async (_req, res) => {
|
||||
try {
|
||||
const result = await flushBuffer();
|
||||
@@ -290,34 +760,149 @@ app.post('/internal/flush', async (_req, res) => {
|
||||
});
|
||||
|
||||
// Internal endpoint to expose Port 4000 buffered events before CSV flush
|
||||
app.get('/internal/recent', (req, res) => {
|
||||
app.get('/internal/recent', async (req, res) => {
|
||||
const clientId = req.query.clientId ? String(req.query.clientId) : '';
|
||||
const limit = Math.min(200, Math.max(1, Number(req.query.limit) || 50));
|
||||
const filtered =
|
||||
const limit = Math.min(2000, Math.max(1, Number(req.query.limit) || 50));
|
||||
|
||||
const redis = getRedisClient();
|
||||
let redisEvents = [];
|
||||
if (redis) {
|
||||
try {
|
||||
const readCap = Math.min(REDIS_RECENT_LIMIT, Math.max(limit * 2, limit));
|
||||
const rawItems = await redis.lrange(REDIS_RECENT_KEY, 0, readCap - 1);
|
||||
redisConnected = true;
|
||||
for (const raw of rawItems) {
|
||||
try {
|
||||
const evt = JSON.parse(raw);
|
||||
if (!evt || !evt.eventId) continue;
|
||||
if (clientId && clientId !== 'all' && evt.clientId !== clientId) continue;
|
||||
redisEvents.push(evt);
|
||||
if (redisEvents.length >= limit) break;
|
||||
} catch {
|
||||
// ignore malformed item
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
redisConnected = false;
|
||||
redisEvents = [];
|
||||
}
|
||||
}
|
||||
|
||||
const localFiltered =
|
||||
clientId && clientId !== 'all'
|
||||
? bufferQueue.filter((evt) => evt.clientId === clientId)
|
||||
: bufferQueue;
|
||||
const recent = filtered.slice(-limit).reverse();
|
||||
res.json({ events: recent, count: recent.length });
|
||||
const localRecent = localFiltered.slice(-limit).reverse();
|
||||
|
||||
const merged = [];
|
||||
const seen = new Set();
|
||||
for (const evt of [...localRecent, ...redisEvents]) {
|
||||
if (!evt || !evt.eventId) continue;
|
||||
const key = `${evt.clientId}:${evt.eventId}`;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
merged.push(evt);
|
||||
if (merged.length >= limit) break;
|
||||
}
|
||||
|
||||
res.json({ events: merged, count: merged.length });
|
||||
});
|
||||
|
||||
setupSocketHandlers(io, bufferQueue, writer, {
|
||||
getRuntimeConfig: () => runtimeConfig,
|
||||
verifyClientInDb,
|
||||
recordClientActivity,
|
||||
onBufferPush: () => {
|
||||
if (bufferQueue.length >= runtimeConfig.batchFlushThreshold) {
|
||||
void flushBuffer();
|
||||
// Internal endpoint triggered by Next.js SDKManager when a client is deleted, revoked, rotated, or synced
|
||||
app.post('/internal/client-sync', (req, res) => {
|
||||
const action = req.body && req.body.action ? String(req.body.action) : '';
|
||||
|
||||
if (action === 'bulk_sync' && Array.isArray(req.body.clients)) {
|
||||
hasSyncedInMemoryClients = true;
|
||||
inMemoryClientRegistry.clear();
|
||||
clientAuthCache.clear();
|
||||
const activeIds = new Set();
|
||||
for (const item of req.body.clients) {
|
||||
if (!item || !item.clientId) continue;
|
||||
const cid = String(item.clientId).trim().toLowerCase();
|
||||
activeIds.add(cid);
|
||||
deletedClientsSet.delete(cid);
|
||||
inMemoryClientRegistry.set(cid, {
|
||||
clientId: cid,
|
||||
name: String(item.name || cid),
|
||||
status: item.status === 'revoked' ? 'revoked' : 'active',
|
||||
token: String(item.token || ''),
|
||||
rateLimit: Math.max(1, Number(item.rateLimit) || 100),
|
||||
});
|
||||
if (item.status === 'revoked') {
|
||||
io.to(`client:${cid}`).disconnectSockets(true);
|
||||
}
|
||||
}
|
||||
},
|
||||
return res.json({ success: true, count: activeIds.size });
|
||||
}
|
||||
|
||||
const rawClientId = req.body && req.body.clientId ? String(req.body.clientId) : '';
|
||||
const normalizedId = rawClientId.trim().toLowerCase();
|
||||
const clientPayload =
|
||||
req.body && req.body.client && typeof req.body.client === 'object'
|
||||
? req.body.client
|
||||
: null;
|
||||
|
||||
if (normalizedId) {
|
||||
hasSyncedInMemoryClients = true;
|
||||
clientAuthCache.delete(normalizedId);
|
||||
|
||||
if (action === 'deleted') {
|
||||
deletedClientsSet.add(normalizedId);
|
||||
inMemoryClientRegistry.delete(normalizedId);
|
||||
inMemoryClientActivity.delete(normalizedId);
|
||||
io.to(`client:${normalizedId}`).disconnectSockets(true);
|
||||
} else if (action === 'revoked') {
|
||||
const existingMem = inMemoryClientRegistry.get(normalizedId);
|
||||
if (clientPayload) {
|
||||
inMemoryClientRegistry.set(normalizedId, {
|
||||
clientId: normalizedId,
|
||||
name: String(clientPayload.name || normalizedId),
|
||||
status: 'revoked',
|
||||
token: String(clientPayload.token || ''),
|
||||
rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
|
||||
});
|
||||
} else if (existingMem) {
|
||||
existingMem.status = 'revoked';
|
||||
} else {
|
||||
inMemoryClientRegistry.set(normalizedId, {
|
||||
clientId: normalizedId,
|
||||
name: normalizedId,
|
||||
status: 'revoked',
|
||||
token: '',
|
||||
rateLimit: 100,
|
||||
});
|
||||
}
|
||||
io.to(`client:${normalizedId}`).disconnectSockets(true);
|
||||
} else if (action === 'registered') {
|
||||
deletedClientsSet.delete(normalizedId);
|
||||
if (clientPayload) {
|
||||
inMemoryClientRegistry.set(normalizedId, {
|
||||
clientId: normalizedId,
|
||||
name: String(clientPayload.name || normalizedId),
|
||||
status: clientPayload.status === 'revoked' ? 'revoked' : 'active',
|
||||
token: String(clientPayload.token || ''),
|
||||
rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
|
||||
});
|
||||
}
|
||||
} else if (action === 'updated') {
|
||||
if (clientPayload) {
|
||||
inMemoryClientRegistry.set(normalizedId, {
|
||||
clientId: normalizedId,
|
||||
name: String(clientPayload.name || normalizedId),
|
||||
status: clientPayload.status === 'revoked' ? 'revoked' : 'active',
|
||||
token: String(clientPayload.token || ''),
|
||||
rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
|
||||
});
|
||||
}
|
||||
io.to(`client:${normalizedId}`).disconnectSockets(true);
|
||||
}
|
||||
}
|
||||
|
||||
res.json({ success: true });
|
||||
});
|
||||
|
||||
let isShuttingDown = false;
|
||||
async function gracefulShutdown(signal) {
|
||||
if (isShuttingDown) return;
|
||||
isShuttingDown = true;
|
||||
console.log(`[DataStack Ingestion Engine] Received ${signal}, flushing buffer before exit...`);
|
||||
|
||||
function stopTimers() {
|
||||
if (flushTimer) {
|
||||
clearInterval(flushTimer);
|
||||
flushTimer = null;
|
||||
@@ -326,6 +911,19 @@ async function gracefulShutdown(signal) {
|
||||
clearInterval(settingsSyncTimer);
|
||||
settingsSyncTimer = null;
|
||||
}
|
||||
if (pruneTimer) {
|
||||
clearInterval(pruneTimer);
|
||||
pruneTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
let isShuttingDown = false;
|
||||
async function gracefulShutdown(signal) {
|
||||
if (isShuttingDown) return;
|
||||
isShuttingDown = true;
|
||||
console.log(`[DataStack Ingestion Engine] Received ${signal}, flushing buffer before exit...`);
|
||||
|
||||
stopTimers();
|
||||
|
||||
try {
|
||||
await flushBuffer();
|
||||
@@ -336,6 +934,9 @@ async function gracefulShutdown(signal) {
|
||||
try {
|
||||
io.close();
|
||||
server.close();
|
||||
if (redisClient && typeof redisClient.quit === 'function') {
|
||||
await redisClient.quit().catch(() => undefined);
|
||||
}
|
||||
if (dbPool) {
|
||||
await dbPool.end();
|
||||
}
|
||||
@@ -346,17 +947,32 @@ async function gracefulShutdown(signal) {
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
process.on('SIGTERM', () => {
|
||||
void gracefulShutdown('SIGTERM');
|
||||
});
|
||||
process.on('SIGINT', () => {
|
||||
void gracefulShutdown('SIGINT');
|
||||
});
|
||||
if (process.env.DATASTACK_DISABLE_LISTEN !== '1') {
|
||||
process.on('SIGTERM', () => {
|
||||
void gracefulShutdown('SIGTERM');
|
||||
});
|
||||
process.on('SIGINT', () => {
|
||||
void gracefulShutdown('SIGINT');
|
||||
});
|
||||
|
||||
server.listen(PORT, () => {
|
||||
console.log(
|
||||
`[DataStack Ingestion Engine] Express + Socket.IO listening on port ${PORT}`,
|
||||
);
|
||||
});
|
||||
server.listen(PORT, () => {
|
||||
console.log(
|
||||
`[DataStack Ingestion Engine] Express + Socket.IO listening on port ${PORT}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { app, server, io, bufferQueue, flushBuffer };
|
||||
module.exports = {
|
||||
app,
|
||||
server,
|
||||
io,
|
||||
writer,
|
||||
bufferQueue,
|
||||
flushBuffer,
|
||||
runtimeConfig,
|
||||
verifyClientInDb,
|
||||
recordClientActivity,
|
||||
applyRuntimeSettingsObject,
|
||||
setRedisClientForTests,
|
||||
stopTimers,
|
||||
};
|
||||
@@ -11,8 +11,10 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
: () => ({
|
||||
jwtSecret:
|
||||
options.jwtSecret ||
|
||||
process.env.DATASTACK_JWT_SECRET ||
|
||||
"datastack-jwt-default-secret-key-32-chars-long",
|
||||
(process.env.DATASTACK_JWT_SECRET || "").trim() ||
|
||||
(process.env.NODE_ENV === "production"
|
||||
? ""
|
||||
: "datastack-jwt-default-secret-key-32-chars-long"),
|
||||
maxEventsPerSec: 100,
|
||||
maxPayloadSizeBytes: 100 * 1024,
|
||||
batchFlushThreshold: 500,
|
||||
@@ -38,8 +40,9 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
const rateLimitMap = new Map();
|
||||
let lastCleanupAt = Date.now();
|
||||
|
||||
function cleanupMaps(now) {
|
||||
function cleanupMaps(now, force = false) {
|
||||
if (
|
||||
!force &&
|
||||
now - lastCleanupAt < CLEANUP_INTERVAL_MS &&
|
||||
idempotencyMap.size <= IDEMPOTENCY_MAX_ENTRIES
|
||||
) {
|
||||
@@ -60,6 +63,16 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
}
|
||||
}
|
||||
|
||||
function getSecurityStats() {
|
||||
cleanupMaps(Date.now(), true);
|
||||
const config = getRuntimeConfig();
|
||||
return {
|
||||
activeIdempotencyKeys: idempotencyMap.size,
|
||||
rateLimitedClientsCount: rateLimitMap.size,
|
||||
maxPayloadSizeBytes: Number(config.maxPayloadSizeBytes) || 100 * 1024,
|
||||
};
|
||||
}
|
||||
|
||||
function validateSingleEvent(clientId, eventId, rawData, clientRateLimit) {
|
||||
const config = getRuntimeConfig();
|
||||
const now = Date.now();
|
||||
@@ -136,10 +149,19 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
? String(authHeader).slice(7).trim()
|
||||
: String(authHeader).trim();
|
||||
const config = getRuntimeConfig();
|
||||
const minSecretLen = process.env.NODE_ENV === "production" ? 32 : 1;
|
||||
if (!config.jwtSecret || config.jwtSecret.length < minSecretLen) {
|
||||
return next(
|
||||
new Error("AUTH_UNAVAILABLE: DATASTACK_JWT_SECRET is not configured"),
|
||||
);
|
||||
}
|
||||
const decoded = jwt.verify(cleanToken, config.jwtSecret);
|
||||
|
||||
const normalizedClientId = String(decoded.clientId || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const dbCheck = await verifyClientInDb(
|
||||
decoded.clientId,
|
||||
normalizedClientId,
|
||||
cleanToken,
|
||||
true,
|
||||
);
|
||||
@@ -147,7 +169,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
return next(new Error(dbCheck.reason || "CLIENT_REVOKED"));
|
||||
}
|
||||
|
||||
socket.data.clientId = decoded.clientId;
|
||||
socket.data.clientId = normalizedClientId;
|
||||
socket.data.token = cleanToken;
|
||||
socket.data.rateLimit =
|
||||
dbCheck.rateLimit || decoded.rateLimit || config.maxEventsPerSec || 100;
|
||||
@@ -157,6 +179,22 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
}
|
||||
});
|
||||
|
||||
const ENVELOPE_KEYS = new Set(["eventId", "timestamp", "data", "clientId"]);
|
||||
function extractBatchItemData(safeItem) {
|
||||
if (
|
||||
safeItem &&
|
||||
typeof safeItem === "object" &&
|
||||
!Array.isArray(safeItem) &&
|
||||
safeItem.data !== undefined
|
||||
) {
|
||||
const keys = Object.keys(safeItem);
|
||||
if (keys.every((k) => ENVELOPE_KEYS.has(k))) {
|
||||
return safeItem.data;
|
||||
}
|
||||
}
|
||||
return safeItem;
|
||||
}
|
||||
|
||||
io.on("connection", (socket) => {
|
||||
const clientId = socket.data.clientId;
|
||||
socket.join(`client:${clientId}`);
|
||||
@@ -274,8 +312,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
safeItem.eventId ||
|
||||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||
const timestamp = safeItem.timestamp || new Date().toISOString();
|
||||
const rawData =
|
||||
safeItem.data !== undefined ? safeItem.data : safeItem;
|
||||
const rawData = extractBatchItemData(safeItem);
|
||||
|
||||
const check = validateSingleEvent(
|
||||
clientId,
|
||||
@@ -403,6 +440,10 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
return {
|
||||
getSecurityStats,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { setupSocketHandlers };
|
||||
+123
-24
@@ -2,6 +2,15 @@ const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { Parser } = require('json2csv');
|
||||
|
||||
const LOCK_RETRY_MIN_MS = 5;
|
||||
const LOCK_RETRY_MAX_MS = 18;
|
||||
const LOCK_TIMEOUT_MS = 10000;
|
||||
const STALE_LOCK_AGE_MS = 30000;
|
||||
|
||||
function sleepMs(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/**
|
||||
* Background CSV Writer Worker
|
||||
* Flushes batches of real-time events to partitioned CSV files.
|
||||
@@ -27,14 +36,100 @@ class CsvWriterWorker {
|
||||
}
|
||||
}
|
||||
|
||||
getTargetFilePath(clientId, dateStr) {
|
||||
ensureClientDir(clientId) {
|
||||
const safeClientId = String(clientId || 'unknown').replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
const clientDir = path.join(this.baseDir, safeClientId);
|
||||
if (!fs.existsSync(clientDir)) {
|
||||
fs.mkdirSync(clientDir, { recursive: true });
|
||||
}
|
||||
return clientDir;
|
||||
}
|
||||
|
||||
const date = dateStr || new Date().toISOString().split('T')[0];
|
||||
async withPartitionLock(clientId, dateStr, fn) {
|
||||
const clientDir = this.ensureClientDir(clientId);
|
||||
const date = this.extractSafePartitionDate(dateStr);
|
||||
const lockPath = path.join(clientDir, `.${date}.csv.lock`);
|
||||
const deadline = Date.now() + LOCK_TIMEOUT_MS;
|
||||
|
||||
let acquired = false;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const fd = fs.openSync(lockPath, 'wx');
|
||||
try {
|
||||
fs.writeFileSync(fd, `${process.pid}:${Date.now()}`, 'utf8');
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
acquired = true;
|
||||
break;
|
||||
} catch (err) {
|
||||
if (!err || err.code !== 'EEXIST') {
|
||||
throw err;
|
||||
}
|
||||
try {
|
||||
const lockStat = fs.statSync(lockPath);
|
||||
if (Date.now() - lockStat.mtimeMs > STALE_LOCK_AGE_MS) {
|
||||
fs.unlinkSync(lockPath);
|
||||
continue;
|
||||
}
|
||||
} catch {
|
||||
// Released concurrently
|
||||
}
|
||||
const jitter =
|
||||
LOCK_RETRY_MIN_MS +
|
||||
Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS));
|
||||
await sleepMs(jitter);
|
||||
}
|
||||
}
|
||||
|
||||
if (!acquired) {
|
||||
throw new Error(`Timed out waiting for CSV partition lock: ${lockPath}`);
|
||||
}
|
||||
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
try {
|
||||
if (fs.existsSync(lockPath)) {
|
||||
fs.unlinkSync(lockPath);
|
||||
}
|
||||
} catch {
|
||||
// Ignore if already removed
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
extractSafePartitionDate(rawTimestamp) {
|
||||
const fallback = new Date().toISOString().slice(0, 10);
|
||||
if (!rawTimestamp || typeof rawTimestamp !== 'string') {
|
||||
return fallback;
|
||||
}
|
||||
const trimmed = rawTimestamp.trim();
|
||||
const prefix = trimmed.slice(0, 10);
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(prefix)) {
|
||||
const parsedPrefix = new Date(`${prefix}T00:00:00.000Z`);
|
||||
if (!Number.isNaN(parsedPrefix.getTime())) {
|
||||
return prefix;
|
||||
}
|
||||
}
|
||||
if (/^\d+(\.\d+)?$/.test(trimmed)) {
|
||||
const num = Number(trimmed);
|
||||
const ms = num < 1e11 ? num * 1000 : num;
|
||||
const d = new Date(ms);
|
||||
if (!Number.isNaN(d.getTime())) {
|
||||
return d.toISOString().slice(0, 10);
|
||||
}
|
||||
}
|
||||
const parsed = new Date(trimmed);
|
||||
if (!Number.isNaN(parsed.getTime())) {
|
||||
return parsed.toISOString().slice(0, 10);
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
getTargetFilePath(clientId, dateStr) {
|
||||
const clientDir = this.ensureClientDir(clientId);
|
||||
const date = this.extractSafePartitionDate(dateStr);
|
||||
let baseFileName = `${date}.csv`;
|
||||
let targetPath = path.join(clientDir, baseFileName);
|
||||
|
||||
@@ -62,7 +157,7 @@ class CsvWriterWorker {
|
||||
|
||||
const grouped = new Map();
|
||||
for (const evt of events) {
|
||||
const date = (evt.timestamp || new Date().toISOString()).split('T')[0];
|
||||
const date = this.extractSafePartitionDate(evt.timestamp);
|
||||
const key = `${evt.clientId}:::${date}`;
|
||||
if (!grouped.has(key)) grouped.set(key, []);
|
||||
grouped.get(key).push(evt);
|
||||
@@ -73,30 +168,34 @@ class CsvWriterWorker {
|
||||
|
||||
for (const [key, batch] of grouped.entries()) {
|
||||
const [clientId, date] = key.split(':::');
|
||||
const targetPath = this.getTargetFilePath(clientId, date);
|
||||
let hasContent = false;
|
||||
try {
|
||||
hasContent =
|
||||
fs.existsSync(targetPath) && fs.statSync(targetPath).size > 0;
|
||||
} catch {
|
||||
hasContent = false;
|
||||
}
|
||||
const targetPath = await this.withPartitionLock(clientId, date, () => {
|
||||
const resolvedPath = this.getTargetFilePath(clientId, date);
|
||||
let hasContent = false;
|
||||
try {
|
||||
hasContent =
|
||||
fs.existsSync(resolvedPath) && fs.statSync(resolvedPath).size > 0;
|
||||
} catch {
|
||||
hasContent = false;
|
||||
}
|
||||
|
||||
const cleanBatch = batch.map((item) => ({
|
||||
clientId: item.clientId,
|
||||
eventId: item.eventId,
|
||||
timestamp: item.timestamp,
|
||||
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
|
||||
}));
|
||||
const cleanBatch = batch.map((item) => ({
|
||||
clientId: item.clientId,
|
||||
eventId: item.eventId,
|
||||
timestamp: item.timestamp,
|
||||
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
|
||||
}));
|
||||
|
||||
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
|
||||
const csv = parser.parse(cleanBatch);
|
||||
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
|
||||
const csv = parser.parse(cleanBatch);
|
||||
|
||||
if (hasContent) {
|
||||
fs.appendFileSync(targetPath, '\n' + csv, 'utf8');
|
||||
} else {
|
||||
fs.writeFileSync(targetPath, csv, 'utf8');
|
||||
}
|
||||
if (hasContent) {
|
||||
fs.appendFileSync(resolvedPath, '\n' + csv, 'utf8');
|
||||
} else {
|
||||
fs.writeFileSync(resolvedPath, csv, 'utf8');
|
||||
}
|
||||
|
||||
return resolvedPath;
|
||||
});
|
||||
|
||||
files.push(targetPath);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,340 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { after, afterEach, beforeEach, describe, it } from "node:test";
|
||||
import { NextRequest } from "next/server";
|
||||
import { GET as profileGet, PUT as profilePut } from "@/app/api/auth/profile/route";
|
||||
import { GET as clientsGet, POST as clientsPost } from "@/app/api/clients/route";
|
||||
import {
|
||||
DELETE as clientDelete,
|
||||
GET as clientGet,
|
||||
} from "@/app/api/clients/[id]/route";
|
||||
import { POST as eventsPost } from "@/app/api/events/route";
|
||||
import {
|
||||
createAdminSessionToken,
|
||||
getAdminSessionPayload,
|
||||
getAdminSessionSecret,
|
||||
} from "@/core/auth/adminSession";
|
||||
import {
|
||||
consumeAdminLoginAttempt,
|
||||
createInitialAdmin,
|
||||
getAdminSetupKey,
|
||||
isAdminSetupComplete,
|
||||
resetAdminStateForTests,
|
||||
verifyAdminCredentials,
|
||||
verifyAdminSetupKey,
|
||||
verifyRequestAdminSession,
|
||||
} from "@/core/auth/adminUsers";
|
||||
import { AuthService, resolveDefaultJwtSecret } from "@/core/auth/jwt";
|
||||
import { isSameOriginRequest } from "@/middleware";
|
||||
|
||||
const ORIGINAL_ENV = { ...process.env };
|
||||
|
||||
function restoreEnv() {
|
||||
for (const key of Object.keys(process.env)) {
|
||||
if (!(key in ORIGINAL_ENV)) {
|
||||
delete process.env[key];
|
||||
}
|
||||
}
|
||||
Object.assign(process.env, ORIGINAL_ENV);
|
||||
AuthService.setSecret(resolveDefaultJwtSecret() || "");
|
||||
}
|
||||
|
||||
describe("Phase-01 Auth, Secrets, Revocation & Proxy Hardening", () => {
|
||||
beforeEach(() => {
|
||||
restoreEnv();
|
||||
resetAdminStateForTests();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
restoreEnv();
|
||||
resetAdminStateForTests();
|
||||
});
|
||||
|
||||
after(() => {
|
||||
const bufState = (
|
||||
globalThis as {
|
||||
__dataStackBufferState?: { flushTimer?: ReturnType<typeof setInterval> | null };
|
||||
}
|
||||
).__dataStackBufferState;
|
||||
if (bufState?.flushTimer) {
|
||||
clearInterval(bufState.flushTimer);
|
||||
bufState.flushTimer = null;
|
||||
}
|
||||
});
|
||||
|
||||
it("BUG-01 & BUG-02: fails closed in production when secrets or DB/setup are missing", async () => {
|
||||
process.env.NODE_ENV = "production";
|
||||
delete process.env.DATABASE_URL;
|
||||
delete process.env.DATASTACK_ADMIN_SESSION_SECRET;
|
||||
delete process.env.DATASTACK_ADMIN_SETUP_KEY;
|
||||
delete process.env.DATASTACK_JWT_SECRET;
|
||||
delete process.env.DATASTACK_ADMIN_PASSWORD;
|
||||
|
||||
assert.equal(getAdminSessionSecret(), null);
|
||||
assert.equal(getAdminSetupKey(), null);
|
||||
assert.equal(resolveDefaultJwtSecret(), "");
|
||||
assert.equal(
|
||||
verifyAdminSetupKey("datastack-default-setup-key-32-chars"),
|
||||
false,
|
||||
);
|
||||
assert.equal(await isAdminSetupComplete(), false);
|
||||
|
||||
await assert.rejects(
|
||||
() => verifyAdminCredentials("admin@datastack.dev", "admin12345678"),
|
||||
/setup/i,
|
||||
);
|
||||
|
||||
AuthService.setSecret(resolveDefaultJwtSecret());
|
||||
assert.equal(AuthService.isConfigured(), false);
|
||||
assert.throws(() => AuthService.generateClientToken({ clientId: "c1" }));
|
||||
});
|
||||
|
||||
it("BUG-03: bumps tokenVersion on credential update and rejects old session cookies", async () => {
|
||||
delete process.env.DATABASE_URL;
|
||||
process.env.DATASTACK_ALLOW_DEMO_ADMIN = "false";
|
||||
const created = await createInitialAdmin("owner@datastack.dev", "InitialPass123!");
|
||||
assert.equal(created, true);
|
||||
assert.equal(await isAdminSetupComplete(), true);
|
||||
|
||||
const secret = getAdminSessionSecret()!;
|
||||
const v1Token = await createAdminSessionToken(
|
||||
"owner@datastack.dev",
|
||||
secret,
|
||||
Date.now() + 3600_000,
|
||||
1,
|
||||
);
|
||||
|
||||
const checkV1 = await verifyRequestAdminSession(v1Token);
|
||||
assert.equal(checkV1.valid, true);
|
||||
|
||||
const updateReq = new NextRequest("http://localhost:3000/api/auth/profile", {
|
||||
method: "PUT",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
cookie: `datastack_admin_session=${v1Token}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
email: "owner2@datastack.dev",
|
||||
currentPassword: "InitialPass123!",
|
||||
newPassword: "UpdatedPass456!",
|
||||
}),
|
||||
});
|
||||
|
||||
const updateRes = await profilePut(updateReq);
|
||||
assert.equal(updateRes.status, 200);
|
||||
const setCookie = updateRes.cookies.get("datastack_admin_session")?.value;
|
||||
assert.ok(setCookie);
|
||||
|
||||
const newPayload = await getAdminSessionPayload(setCookie!, secret);
|
||||
assert.ok(newPayload);
|
||||
assert.equal(newPayload!.tokenVersion, 2);
|
||||
|
||||
// Old v1 session must now return 401
|
||||
const staleProfileReq = new NextRequest("http://localhost:3000/api/auth/profile", {
|
||||
method: "GET",
|
||||
headers: { cookie: `datastack_admin_session=${v1Token}` },
|
||||
});
|
||||
const staleRes = await profileGet(staleProfileReq);
|
||||
assert.equal(staleRes.status, 401);
|
||||
|
||||
// New v2 session succeeds
|
||||
const freshProfileReq = new NextRequest("http://localhost:3000/api/auth/profile", {
|
||||
method: "GET",
|
||||
headers: { cookie: `datastack_admin_session=${setCookie}` },
|
||||
});
|
||||
const freshRes = await profileGet(freshProfileReq);
|
||||
assert.equal(freshRes.status, 200);
|
||||
});
|
||||
|
||||
it("BUG-04: throttles by composite IP+email without locking out other admin IPs", async () => {
|
||||
const secret = "test-throttle-secret-at-least-32-chars-long";
|
||||
const targetEmail = "admin@datastack.dev";
|
||||
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const dec = await consumeAdminLoginAttempt(targetEmail, secret, "198.51.100.10");
|
||||
assert.equal(dec.allowed, true);
|
||||
}
|
||||
|
||||
// 6th attempt from attacker IP is blocked
|
||||
const blocked = await consumeAdminLoginAttempt(targetEmail, secret, "198.51.100.10");
|
||||
assert.equal(blocked.allowed, false);
|
||||
assert.ok(blocked.retryAfterSeconds >= 1);
|
||||
|
||||
// Real admin from different IP is still allowed
|
||||
const otherIpAllowed = await consumeAdminLoginAttempt(
|
||||
targetEmail,
|
||||
secret,
|
||||
"203.0.113.55",
|
||||
);
|
||||
assert.equal(otherIpAllowed.allowed, true);
|
||||
|
||||
// Same IP spraying 25 distinct emails triggers IP bucket block on 26th
|
||||
for (let i = 0; i < 25; i++) {
|
||||
const spray = await consumeAdminLoginAttempt(
|
||||
`spray${i}@example.com`,
|
||||
secret,
|
||||
"198.51.100.99",
|
||||
);
|
||||
assert.equal(spray.allowed, true);
|
||||
}
|
||||
const sprayBlocked = await consumeAdminLoginAttempt(
|
||||
"spray25@example.com",
|
||||
secret,
|
||||
"198.51.100.99",
|
||||
);
|
||||
assert.equal(sprayBlocked.allowed, false);
|
||||
});
|
||||
|
||||
it("BUG-05 & BUG-15: redacts tokens in client list, returns 400 on malformed JSON, and blocks deleted/unregistered Bearer tokens", async () => {
|
||||
delete process.env.DATABASE_URL;
|
||||
await createInitialAdmin("admin@datastack.dev", "AdminPass123!");
|
||||
const secret = getAdminSessionSecret()!;
|
||||
const adminCookie = await createAdminSessionToken(
|
||||
"admin@datastack.dev",
|
||||
secret,
|
||||
Date.now() + 3600_000,
|
||||
1,
|
||||
);
|
||||
|
||||
const createReq = new NextRequest("http://localhost:3000/api/clients", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
cookie: `datastack_admin_session=${adminCookie}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
clientId: "phase01_probe_client",
|
||||
name: "Phase01 Probe",
|
||||
rateLimit: 50,
|
||||
}),
|
||||
});
|
||||
const createRes = await clientsPost(createReq);
|
||||
assert.equal(createRes.status, 201);
|
||||
const createdJson = await createRes.json();
|
||||
const clientToken = createdJson.client.token as string;
|
||||
assert.ok(clientToken.length > 20);
|
||||
|
||||
// List endpoint must redact full token & secret
|
||||
const listReq = new NextRequest("http://localhost:3000/api/clients", {
|
||||
method: "GET",
|
||||
headers: { cookie: `datastack_admin_session=${adminCookie}` },
|
||||
});
|
||||
const listRes = await clientsGet(listReq);
|
||||
assert.equal(listRes.status, 200);
|
||||
const listJson = await listRes.json();
|
||||
const listed = listJson.clients.find(
|
||||
(c: { clientId: string }) => c.clientId === "phase01_probe_client",
|
||||
);
|
||||
assert.ok(listed);
|
||||
assert.equal("token" in listed, false);
|
||||
assert.equal("secret" in listed, false);
|
||||
assert.equal(listed.tokenPreview, `${clientToken.slice(0, 12)}...`);
|
||||
|
||||
// Single client GET omits token by default, reveals on ?includeToken=true
|
||||
const getDefaultRes = await clientGet(
|
||||
new NextRequest("http://localhost:3000/api/clients/phase01_probe_client", {
|
||||
headers: { cookie: `datastack_admin_session=${adminCookie}` },
|
||||
}),
|
||||
{ params: Promise.resolve({ id: "phase01_probe_client" }) },
|
||||
);
|
||||
const defaultClient = (await getDefaultRes.json()).client;
|
||||
assert.equal("token" in defaultClient, false);
|
||||
|
||||
const getIncludeRes = await clientGet(
|
||||
new NextRequest(
|
||||
"http://localhost:3000/api/clients/phase01_probe_client?includeToken=true",
|
||||
{ headers: { cookie: `datastack_admin_session=${adminCookie}` } },
|
||||
),
|
||||
{ params: Promise.resolve({ id: "phase01_probe_client" }) },
|
||||
);
|
||||
const includeClient = (await getIncludeRes.json()).client;
|
||||
assert.equal(includeClient.token, clientToken);
|
||||
|
||||
// Malformed JSON on POST /api/events returns 400 (not 500)
|
||||
const malformedRes = await eventsPost(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
authorization: `Bearer ${clientToken}`,
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: "{bad-json",
|
||||
}),
|
||||
);
|
||||
assert.equal(malformedRes.status, 400);
|
||||
|
||||
// Delete client -> subsequent Bearer ingestion fails closed with 403 CLIENT_REVOKED
|
||||
const delRes = await clientDelete(
|
||||
new NextRequest("http://localhost:3000/api/clients/phase01_probe_client", {
|
||||
method: "DELETE",
|
||||
headers: { cookie: `datastack_admin_session=${adminCookie}` },
|
||||
}),
|
||||
{ params: Promise.resolve({ id: "phase01_probe_client" }) },
|
||||
);
|
||||
assert.equal(delRes.status, 200);
|
||||
|
||||
const rejectedRes = await eventsPost(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
authorization: `Bearer ${clientToken}`,
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ eventId: "evt_after_delete", data: { ok: 1 } }),
|
||||
}),
|
||||
);
|
||||
assert.equal(rejectedRes.status, 403);
|
||||
|
||||
// Unregistered signed Bearer token also rejected with 403 CLIENT_REVOKED
|
||||
const ghostToken = AuthService.generateClientToken({
|
||||
clientId: "unregistered_ghost_client",
|
||||
});
|
||||
const ghostRes = await eventsPost(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
authorization: `Bearer ${ghostToken}`,
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ eventId: "evt_ghost", data: { ok: 1 } }),
|
||||
}),
|
||||
);
|
||||
assert.equal(ghostRes.status, 403);
|
||||
});
|
||||
|
||||
it("BUG-26: validates Origin or Referer against Host / X-Forwarded-Host", () => {
|
||||
const reqOriginOk = new NextRequest("http://localhost:3000/api/clients", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
origin: "https://console.example.com",
|
||||
"x-forwarded-host": "console.example.com",
|
||||
},
|
||||
});
|
||||
assert.equal(isSameOriginRequest(reqOriginOk), true);
|
||||
|
||||
const reqRefererOk = new NextRequest("http://localhost:3000/api/clients", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
referer: "https://console.example.com/vcon/clients",
|
||||
host: "console.example.com",
|
||||
},
|
||||
});
|
||||
assert.equal(isSameOriginRequest(reqRefererOk), true);
|
||||
|
||||
const reqCrossOrigin = new NextRequest("http://localhost:3000/api/clients", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
origin: "https://evil.example.com",
|
||||
host: "console.example.com",
|
||||
},
|
||||
});
|
||||
assert.equal(isSameOriginRequest(reqCrossOrigin), false);
|
||||
|
||||
const reqMissingBoth = new NextRequest("http://localhost:3000/api/clients", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
host: "console.example.com",
|
||||
},
|
||||
});
|
||||
assert.equal(isSameOriginRequest(reqMissingBoth), false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,375 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createRequire } from "node:module";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import test, { after, before } from "node:test";
|
||||
import { NextRequest } from "next/server";
|
||||
import { io as ioClient, type Socket } from "socket.io-client";
|
||||
import { GET as getSettingsRoute, POST as postSettingsRoute } from "../app/api/settings/route";
|
||||
import { SettingsManager } from "../core/adminSettings/settingsManager";
|
||||
import {
|
||||
createAdminSessionToken,
|
||||
getAdminSessionSecret,
|
||||
} from "../core/auth/adminSession";
|
||||
import {
|
||||
createInitialAdmin,
|
||||
resetAdminStateForTests,
|
||||
} from "../core/auth/adminUsers";
|
||||
import { ApiEngine } from "../core/apiEngine/apiService";
|
||||
import { SDKManager } from "../core/sdk/sdkManager";
|
||||
import {
|
||||
SecurityEngine,
|
||||
getInternalIngestionHeaders,
|
||||
} from "../core/Security/security";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
interface IngestionServerModule {
|
||||
server: import("node:http").Server;
|
||||
io: import("socket.io").Server;
|
||||
bufferQueue: Array<{ clientId: string; eventId: string; timestamp: string; payload: string }>;
|
||||
runtimeConfig: {
|
||||
jwtSecret: string;
|
||||
maxEventsPerSec: number;
|
||||
maxPayloadSizeBytes: number;
|
||||
batchFlushThreshold: number;
|
||||
batchFlushIntervalMs: number;
|
||||
fileRotationMb: number;
|
||||
enableIdempotency: boolean;
|
||||
autoPruneDays: number;
|
||||
};
|
||||
stopTimers: () => void;
|
||||
}
|
||||
|
||||
const g = globalThis as typeof globalThis & {
|
||||
__dataStackBufferState?: {
|
||||
flushTimer: ReturnType<typeof setInterval> | null;
|
||||
};
|
||||
__dataStackAdminSettings?: unknown;
|
||||
__dataStackClientsMap?: Map<string, unknown>;
|
||||
__dataStackDeletedClients?: Set<string>;
|
||||
__dataStackSyncedSocketEvents?: Map<string, number>;
|
||||
};
|
||||
|
||||
let serverMod: IngestionServerModule;
|
||||
let ingestionUrl = "";
|
||||
const prevIngestionUrl = process.env.INGESTION_INTERNAL_URL;
|
||||
const prevDisableListen = process.env.DATASTACK_DISABLE_LISTEN;
|
||||
const prevDbUrl = process.env.DATABASE_URL;
|
||||
const prevInternalSecret = process.env.DATASTACK_INTERNAL_SECRET;
|
||||
const openSockets: Socket[] = [];
|
||||
|
||||
function connectSocket(token: string): Promise<Socket> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const client = ioClient(ingestionUrl, {
|
||||
auth: { token },
|
||||
transports: ["websocket"],
|
||||
reconnection: false,
|
||||
timeout: 3000,
|
||||
});
|
||||
openSockets.push(client);
|
||||
const onConnect = () => {
|
||||
cleanup();
|
||||
resolve(client);
|
||||
};
|
||||
const onError = (err: Error) => {
|
||||
cleanup();
|
||||
client.disconnect();
|
||||
reject(err);
|
||||
};
|
||||
function cleanup() {
|
||||
client.off("connect", onConnect);
|
||||
client.off("connect_error", onError);
|
||||
}
|
||||
client.on("connect", onConnect);
|
||||
client.on("connect_error", onError);
|
||||
});
|
||||
}
|
||||
|
||||
function emitWithAck<T = unknown>(
|
||||
socket: Socket,
|
||||
event: string,
|
||||
payload: unknown,
|
||||
): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
socket.timeout(3000).emit(event, payload, (err: unknown, res: T) => {
|
||||
if (err) reject(err);
|
||||
else resolve(res);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function createAdminCookieHeader(): Promise<string> {
|
||||
resetAdminStateForTests();
|
||||
await createInitialAdmin("admin@datastack.dev", "InitialAdminPass123!");
|
||||
const secret = getAdminSessionSecret();
|
||||
assert.ok(secret, "Expected admin session secret in test environment");
|
||||
const token = await createAdminSessionToken(
|
||||
"admin@datastack.dev",
|
||||
secret,
|
||||
Date.now() + 3600_000,
|
||||
1,
|
||||
);
|
||||
return `datastack_admin_session=${token}`;
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
process.env.DATASTACK_DISABLE_LISTEN = "1";
|
||||
delete process.env.DATABASE_URL;
|
||||
delete process.env.DATASTACK_INTERNAL_SECRET;
|
||||
delete g.__dataStackAdminSettings;
|
||||
g.__dataStackClientsMap?.clear();
|
||||
g.__dataStackDeletedClients?.clear();
|
||||
g.__dataStackSyncedSocketEvents?.clear();
|
||||
resetAdminStateForTests();
|
||||
|
||||
serverMod = require("../server/index.js") as IngestionServerModule;
|
||||
await new Promise<void>((resolve) => {
|
||||
serverMod.server.listen(0, "127.0.0.1", () => resolve());
|
||||
});
|
||||
const addr = serverMod.server.address() as AddressInfo;
|
||||
ingestionUrl = `http://127.0.0.1:${addr.port}`;
|
||||
process.env.INGESTION_INTERNAL_URL = ingestionUrl;
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
for (const sock of openSockets) {
|
||||
try {
|
||||
sock.disconnect();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
if (serverMod) {
|
||||
serverMod.stopTimers();
|
||||
await new Promise<void>((resolve) => {
|
||||
serverMod.io.close(() => {
|
||||
serverMod.server.close(() => resolve());
|
||||
});
|
||||
});
|
||||
}
|
||||
if (g.__dataStackBufferState?.flushTimer) {
|
||||
clearInterval(g.__dataStackBufferState.flushTimer);
|
||||
g.__dataStackBufferState.flushTimer = null;
|
||||
}
|
||||
|
||||
if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL;
|
||||
else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl;
|
||||
|
||||
if (prevDisableListen === undefined) delete process.env.DATASTACK_DISABLE_LISTEN;
|
||||
else process.env.DATASTACK_DISABLE_LISTEN = prevDisableListen;
|
||||
|
||||
if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
|
||||
else process.env.DATABASE_URL = prevDbUrl;
|
||||
|
||||
if (prevInternalSecret === undefined) delete process.env.DATASTACK_INTERNAL_SECRET;
|
||||
else process.env.DATASTACK_INTERNAL_SECRET = prevInternalSecret;
|
||||
});
|
||||
|
||||
test("BUG-13: Port 4000 /internal/* rejects unauthenticated callers and accepts valid internal secret while /health remains public", async () => {
|
||||
const healthRes = await fetch(`${ingestionUrl}/health`);
|
||||
assert.equal(healthRes.status, 200);
|
||||
|
||||
for (const [method, path, body] of [
|
||||
["GET", "/internal/metrics", undefined],
|
||||
["POST", "/internal/flush", {}],
|
||||
["GET", "/internal/recent", undefined],
|
||||
["POST", "/internal/client-sync", { clientId: "x", action: "registered" }],
|
||||
["POST", "/internal/settings-sync", { maxEventsPerSec: 120 }],
|
||||
] as const) {
|
||||
const noAuth = await fetch(`${ingestionUrl}${path}`, {
|
||||
method,
|
||||
headers: body ? { "Content-Type": "application/json" } : undefined,
|
||||
body: body ? JSON.stringify(body) : undefined,
|
||||
});
|
||||
assert.equal(noAuth.status, 401, `${method} ${path} must reject missing secret`);
|
||||
|
||||
const wrongAuth = await fetch(`${ingestionUrl}${path}`, {
|
||||
method,
|
||||
headers: {
|
||||
...(body ? { "Content-Type": "application/json" } : {}),
|
||||
"x-datastack-internal-secret": "wrong-secret-value",
|
||||
},
|
||||
body: body ? JSON.stringify(body) : undefined,
|
||||
});
|
||||
assert.equal(wrongAuth.status, 401, `${method} ${path} must reject wrong secret`);
|
||||
|
||||
const validAuth = await fetch(`${ingestionUrl}${path}`, {
|
||||
method,
|
||||
headers: {
|
||||
...(body ? { "Content-Type": "application/json" } : {}),
|
||||
...getInternalIngestionHeaders(),
|
||||
},
|
||||
body: body ? JSON.stringify(body) : undefined,
|
||||
});
|
||||
assert.equal(validAuth.status, 200, `${method} ${path} must accept valid internal secret`);
|
||||
}
|
||||
});
|
||||
|
||||
test("BUG-06: POST /api/settings validates allowlisted bounds, syncs Port 4000 immediately, preserves autoPruneDays=0, and disconnects sockets on JWT rotation", async () => {
|
||||
const unauthGet = await getSettingsRoute(new NextRequest("http://localhost:3000/api/settings"));
|
||||
assert.equal(unauthGet.status, 401);
|
||||
|
||||
const cookieHeader = await createAdminCookieHeader();
|
||||
|
||||
const badJsonReq = new NextRequest("http://localhost:3000/api/settings", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", cookie: cookieHeader },
|
||||
body: "{bad-json",
|
||||
});
|
||||
const badJsonRes = await postSettingsRoute(badJsonReq);
|
||||
assert.equal(badJsonRes.status, 400);
|
||||
|
||||
const unsupportedReq = new NextRequest("http://localhost:3000/api/settings", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", cookie: cookieHeader },
|
||||
body: JSON.stringify({ serverPort: 9999 }),
|
||||
});
|
||||
assert.equal((await postSettingsRoute(unsupportedReq)).status, 400);
|
||||
|
||||
const weakSecretReq = new NextRequest("http://localhost:3000/api/settings", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", cookie: cookieHeader },
|
||||
body: JSON.stringify({ jwtSecret: "too-short" }),
|
||||
});
|
||||
assert.equal((await postSettingsRoute(weakSecretReq)).status, 400);
|
||||
|
||||
const validUpdateReq = new NextRequest("http://localhost:3000/api/settings", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", cookie: cookieHeader },
|
||||
body: JSON.stringify({
|
||||
maxEventsPerSec: 42,
|
||||
autoPruneDays: 0,
|
||||
enableIdempotency: true,
|
||||
}),
|
||||
});
|
||||
const validUpdateRes = await postSettingsRoute(validUpdateReq);
|
||||
assert.equal(validUpdateRes.status, 200);
|
||||
const updatedJson = await validUpdateRes.json();
|
||||
assert.equal(updatedJson.settings.maxEventsPerSec, 42);
|
||||
assert.equal(updatedJson.settings.autoPruneDays, 0);
|
||||
assert.equal(serverMod.runtimeConfig.maxEventsPerSec, 42);
|
||||
assert.equal(serverMod.runtimeConfig.autoPruneDays, 0);
|
||||
|
||||
// Register client & open WebSocket before JWT rotation
|
||||
const client = await SDKManager.registerClientAsync({
|
||||
clientId: "phase02_jwt_rot",
|
||||
name: "Phase02 Rotation",
|
||||
rateLimit: 50,
|
||||
});
|
||||
const oldToken = client.token;
|
||||
const socket = await connectSocket(oldToken);
|
||||
assert.equal(socket.connected, true);
|
||||
|
||||
const disconnectedPromise = new Promise<string>((resolve) => {
|
||||
socket.once("disconnect", (reason) => resolve(reason));
|
||||
});
|
||||
|
||||
const newSecret = "phase02-rotated-production-secret-key-minimum-32-chars";
|
||||
const rotateReq = new NextRequest("http://localhost:3000/api/settings", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", cookie: cookieHeader },
|
||||
body: JSON.stringify({ jwtSecret: newSecret }),
|
||||
});
|
||||
const rotateRes = await postSettingsRoute(rotateReq);
|
||||
assert.equal(rotateRes.status, 200);
|
||||
assert.equal(serverMod.runtimeConfig.jwtSecret, newSecret);
|
||||
|
||||
const disconnectReason = await Promise.race([
|
||||
disconnectedPromise,
|
||||
new Promise<string>((_, reject) =>
|
||||
setTimeout(() => reject(new Error("Expected socket disconnect on JWT rotation")), 2000),
|
||||
),
|
||||
]);
|
||||
assert.ok(disconnectReason.length > 0);
|
||||
|
||||
// Old JWT token must fail handshake after rotation
|
||||
await assert.rejects(() => connectSocket(oldToken), /AUTH_INVALID|jwt|signature/i);
|
||||
|
||||
// New resigned client token must succeed cleanly
|
||||
const refreshedClient = SDKManager.getClient("phase02_jwt_rot");
|
||||
assert.ok(refreshedClient && refreshedClient.token !== oldToken);
|
||||
const reconnected = await connectSocket(refreshedClient.token);
|
||||
assert.equal(reconnected.connected, true);
|
||||
reconnected.disconnect();
|
||||
});
|
||||
|
||||
test("BUG-07: In-memory client registry, socket event counting, token rotation & revocation enforce cleanly on Port 4000", async () => {
|
||||
const client = await SDKManager.registerClientAsync({
|
||||
clientId: "phase02_mem_client",
|
||||
name: "Phase02 Memory Sync",
|
||||
rateLimit: 25,
|
||||
});
|
||||
|
||||
const socket = await connectSocket(client.token);
|
||||
const ack = await emitWithAck<{ success: boolean; eventId: string }>(
|
||||
socket,
|
||||
"sync_data",
|
||||
{ eventId: "evt-mem-1", data: { ok: true } },
|
||||
);
|
||||
assert.equal(ack.success, true);
|
||||
|
||||
const listFirst = await SDKManager.listClientsAsync();
|
||||
const syncedFirst = listFirst.find((c) => c.clientId === "phase02_mem_client");
|
||||
assert.ok(syncedFirst);
|
||||
assert.equal(syncedFirst.totalEvents, 1);
|
||||
assert.ok(syncedFirst.lastActive);
|
||||
|
||||
// Polling again must NOT double-count socket activity
|
||||
const listSecond = await SDKManager.listClientsAsync();
|
||||
const syncedSecond = listSecond.find((c) => c.clientId === "phase02_mem_client");
|
||||
assert.equal(syncedSecond?.totalEvents, 1);
|
||||
|
||||
// Advance across JWT iat 1-second boundary before regenerating client token
|
||||
await new Promise((r) => setTimeout(r, 1100));
|
||||
|
||||
// Regenerating client token must kick active socket and reject old token
|
||||
const rotatedClient = await SDKManager.regenerateTokenAsync("phase02_mem_client");
|
||||
assert.ok(rotatedClient && rotatedClient.token !== client.token);
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
assert.equal(socket.connected, false);
|
||||
await assert.rejects(() => connectSocket(client.token), /TOKEN_ROTATED/);
|
||||
|
||||
// New token works; revoking kicks and rejects
|
||||
const socket2 = await connectSocket(rotatedClient.token);
|
||||
assert.equal(socket2.connected, true);
|
||||
await SDKManager.updateClientAsync("phase02_mem_client", { status: "revoked" });
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
assert.equal(socket2.connected, false);
|
||||
await assert.rejects(() => connectSocket(rotatedClient.token), /CLIENT_REVOKED/);
|
||||
});
|
||||
|
||||
test("BUG-08: Rate-limit error reports effectiveRateLimit and ApiEngine aggregates Port 3000 + Port 4000 security metrics", async () => {
|
||||
await SettingsManager.updateSettingsAsync({ maxEventsPerSec: 3, enableIdempotency: true });
|
||||
|
||||
const rlClientId = "phase02_rl_msg";
|
||||
let lastError = "";
|
||||
for (let i = 0; i < 5; i += 1) {
|
||||
const res = SecurityEngine.validateEvent(rlClientId, `rl-evt-${i}`, { i }, 100);
|
||||
if (!res.allowed && res.code === "RATE_LIMIT_EXCEEDED") {
|
||||
lastError = res.reason || "";
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert.match(lastError, /Rate limit of 3 events\/sec exceeded/);
|
||||
|
||||
const client = await SDKManager.registerClientAsync({
|
||||
clientId: "phase02_stats_client",
|
||||
name: "Phase02 Stats Client",
|
||||
rateLimit: 50,
|
||||
});
|
||||
await new Promise((r) => setTimeout(r, 80));
|
||||
|
||||
const socket = await connectSocket(client.token);
|
||||
await emitWithAck(socket, "sync_data", { eventId: "evt-stats-ws-1", data: { a: 1 } });
|
||||
socket.disconnect();
|
||||
|
||||
const metrics = await ApiEngine.getDashboardMetrics();
|
||||
assert.ok(
|
||||
metrics.securityStats.activeIdempotencyKeys >= 2,
|
||||
`Expected aggregated idempotency keys >= 2 (HTTP + WS), got ${metrics.securityStats.activeIdempotencyKeys}`,
|
||||
);
|
||||
assert.ok(
|
||||
metrics.securityStats.rateLimitedClientsCount >= 2,
|
||||
`Expected aggregated rateLimitedClientsCount >= 2, got ${metrics.securityStats.rateLimitedClientsCount}`,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,511 @@
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import { createServer, type Server } from "node:http";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import { createRequire } from "node:module";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import test, { after, before, beforeEach } from "node:test";
|
||||
import { NextRequest } from "next/server";
|
||||
import { GET as getDataRoute } from "../app/api/data/route";
|
||||
import { GET as getEventsRoute } from "../app/api/events/route";
|
||||
import { GET as getExportRoute } from "../app/api/export/route";
|
||||
import { POST as postStorageRoute } from "../app/api/storage/route";
|
||||
import { ApiEngine } from "../core/apiEngine/apiService";
|
||||
import {
|
||||
DataBufferEngine,
|
||||
REDIS_BUFFER_KEY,
|
||||
REDIS_FLUSH_LOCK_KEY,
|
||||
type RedisBufferClient,
|
||||
} from "../core/data/buffer";
|
||||
import { SDKManager } from "../core/sdk/sdkManager";
|
||||
import { CSVStorageManager, type DataEvent } from "../core/Storage/csvStorage";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const { CsvWriterWorker } = require("../server/workers/csvWriter") as {
|
||||
CsvWriterWorker: new (baseDir?: string, maxFileSize?: number) => {
|
||||
baseDir: string;
|
||||
writeBatch: (events: DataEvent[]) => Promise<{ written: number; files: string[] }>;
|
||||
pruneExpiredPartitions: (days: number) => number;
|
||||
};
|
||||
};
|
||||
|
||||
interface IngestionServerModule {
|
||||
server: Server;
|
||||
io: import("socket.io").Server;
|
||||
writer: { baseDir: string };
|
||||
bufferQueue: DataEvent[];
|
||||
runtimeConfig: {
|
||||
batchFlushThreshold: number;
|
||||
autoPruneDays: number;
|
||||
};
|
||||
applyRuntimeSettingsObject: (obj: Record<string, unknown>) => { jwtRotated: boolean };
|
||||
flushBuffer: () => Promise<{ written: number; files: string[] }>;
|
||||
setRedisClientForTests: (client: RedisBufferClient | null | undefined) => void;
|
||||
stopTimers: () => void;
|
||||
}
|
||||
|
||||
class FakeRedisClient implements RedisBufferClient {
|
||||
public status = "ready";
|
||||
public lists = new Map<string, string[]>();
|
||||
public strings = new Map<string, { value: string; expiresAt: number }>();
|
||||
|
||||
public async rpush(key: string, ...values: string[]): Promise<number> {
|
||||
const list = this.lists.get(key) || [];
|
||||
list.push(...values);
|
||||
this.lists.set(key, list);
|
||||
return list.length;
|
||||
}
|
||||
|
||||
public async lpush(key: string, ...values: string[]): Promise<number> {
|
||||
const list = this.lists.get(key) || [];
|
||||
for (const v of values) {
|
||||
list.unshift(v);
|
||||
}
|
||||
this.lists.set(key, list);
|
||||
return list.length;
|
||||
}
|
||||
|
||||
public async ltrim(key: string, start: number, stop: number): Promise<"OK"> {
|
||||
const list = this.lists.get(key) || [];
|
||||
const len = list.length;
|
||||
const normStart = start < 0 ? Math.max(0, len + start) : start;
|
||||
const normStop = stop < 0 ? len + stop : stop;
|
||||
if (normStart > normStop || normStart >= len) {
|
||||
this.lists.set(key, []);
|
||||
} else {
|
||||
this.lists.set(key, list.slice(normStart, normStop + 1));
|
||||
}
|
||||
return "OK";
|
||||
}
|
||||
|
||||
public async lrange(key: string, start: number, stop: number): Promise<string[]> {
|
||||
const list = this.lists.get(key) || [];
|
||||
const len = list.length;
|
||||
if (len === 0) return [];
|
||||
const normStart = start < 0 ? Math.max(0, len + start) : start;
|
||||
const normStop = stop < 0 ? len + stop : stop;
|
||||
if (normStart > normStop || normStart >= len) return [];
|
||||
return list.slice(normStart, normStop + 1);
|
||||
}
|
||||
|
||||
public async llen(key: string): Promise<number> {
|
||||
return (this.lists.get(key) || []).length;
|
||||
}
|
||||
|
||||
public async set(
|
||||
key: string,
|
||||
value: string,
|
||||
_pxMode: "PX",
|
||||
ttlMs: number,
|
||||
_nxMode: "NX",
|
||||
): Promise<string | null> {
|
||||
const existing = this.strings.get(key);
|
||||
if (existing && existing.expiresAt > Date.now()) {
|
||||
return null;
|
||||
}
|
||||
this.strings.set(key, { value, expiresAt: Date.now() + ttlMs });
|
||||
return "OK";
|
||||
}
|
||||
|
||||
public async eval(_script: string, _numKeys: number, key: string, token: string): Promise<number> {
|
||||
const existing = this.strings.get(key);
|
||||
if (existing && existing.value === token) {
|
||||
this.strings.delete(key);
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
const g = globalThis as typeof globalThis & {
|
||||
__dataStackBufferState?: {
|
||||
buffer: DataEvent[];
|
||||
recentEvents: DataEvent[];
|
||||
flushTimer: ReturnType<typeof setInterval> | null;
|
||||
};
|
||||
__dataStackAdminSettings?: unknown;
|
||||
__dataStackClientsMap?: Map<string, unknown>;
|
||||
__dataStackDeletedClients?: Set<string>;
|
||||
};
|
||||
|
||||
let tmpStorageDir = "";
|
||||
const origBaseStorageDir = CSVStorageManager.baseStorageDir;
|
||||
const prevDisableListen = process.env.DATASTACK_DISABLE_LISTEN;
|
||||
const prevIngestionUrl = process.env.INGESTION_INTERNAL_URL;
|
||||
const prevDbUrl = process.env.DATABASE_URL;
|
||||
const prevRedisUrl = process.env.REDIS_URL;
|
||||
|
||||
before(async () => {
|
||||
process.env.DATASTACK_DISABLE_LISTEN = "1";
|
||||
delete process.env.DATABASE_URL;
|
||||
delete process.env.REDIS_URL;
|
||||
delete process.env.INGESTION_INTERNAL_URL;
|
||||
|
||||
tmpStorageDir = fs.mkdtempSync(path.join(os.tmpdir(), "datastack-phase03-"));
|
||||
CSVStorageManager.baseStorageDir = tmpStorageDir;
|
||||
CSVStorageManager.clearRowCountCache();
|
||||
});
|
||||
|
||||
beforeEach(async () => {
|
||||
DataBufferEngine.setRedisClientForTests(undefined);
|
||||
if (g.__dataStackBufferState) {
|
||||
g.__dataStackBufferState.buffer = [];
|
||||
g.__dataStackBufferState.recentEvents = [];
|
||||
}
|
||||
CSVStorageManager.clearRowCountCache();
|
||||
fs.rmSync(tmpStorageDir, { recursive: true, force: true });
|
||||
fs.mkdirSync(tmpStorageDir, { recursive: true });
|
||||
delete g.__dataStackAdminSettings;
|
||||
g.__dataStackClientsMap?.clear();
|
||||
g.__dataStackDeletedClients?.clear();
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await DataBufferEngine.shutdown();
|
||||
CSVStorageManager.baseStorageDir = origBaseStorageDir;
|
||||
CSVStorageManager.clearRowCountCache();
|
||||
if (tmpStorageDir) {
|
||||
fs.rmSync(tmpStorageDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
if (prevDisableListen === undefined) delete process.env.DATASTACK_DISABLE_LISTEN;
|
||||
else process.env.DATASTACK_DISABLE_LISTEN = prevDisableListen;
|
||||
|
||||
if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL;
|
||||
else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl;
|
||||
|
||||
if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
|
||||
else process.env.DATABASE_URL = prevDbUrl;
|
||||
|
||||
if (prevRedisUrl === undefined) delete process.env.REDIS_URL;
|
||||
else process.env.REDIS_URL = prevRedisUrl;
|
||||
});
|
||||
|
||||
test("BUG-11: Shared Redis buffer preserves events on CSV error, honors distributed flush lock, avoids double-counting, and falls back to RAM cleanly", async () => {
|
||||
const fakeRedis = new FakeRedisClient();
|
||||
DataBufferEngine.setRedisClientForTests(fakeRedis);
|
||||
|
||||
const sampleEvents: DataEvent[] = [
|
||||
{
|
||||
clientId: "phase03_redis",
|
||||
eventId: "r-evt-1",
|
||||
timestamp: "2026-11-17T10:00:00.000Z",
|
||||
payload: JSON.stringify({ seq: 1 }),
|
||||
},
|
||||
{
|
||||
clientId: "phase03_redis",
|
||||
eventId: "r-evt-2",
|
||||
timestamp: "2026-11-17T10:00:01.000Z",
|
||||
payload: JSON.stringify({ seq: 2 }),
|
||||
},
|
||||
];
|
||||
|
||||
await DataBufferEngine.pushBatchAsync(sampleEvents);
|
||||
|
||||
assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 2);
|
||||
const recentFromRedis = await DataBufferEngine.getRecentEventsAsync(10, "phase03_redis");
|
||||
assert.equal(recentFromRedis.length, 2);
|
||||
assert.equal(recentFromRedis[0].eventId, "r-evt-2");
|
||||
|
||||
// Lock contention must skip concurrent Redis flush without corrupting queue
|
||||
await fakeRedis.set(REDIS_FLUSH_LOCK_KEY, "other-worker-token", "PX", 15000, "NX");
|
||||
const skipped = await DataBufferEngine.flush();
|
||||
assert.equal(skipped.count, 0);
|
||||
assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 2);
|
||||
await fakeRedis.eval("", 1, REDIS_FLUSH_LOCK_KEY, "other-worker-token");
|
||||
|
||||
// Crash/error safety: if appendBatch throws before finishing, Redis LTRIM must NOT discard events
|
||||
const originalAppend = CSVStorageManager.appendBatch;
|
||||
CSVStorageManager.appendBatch = async () => {
|
||||
throw new Error("Simulated disk write failure");
|
||||
};
|
||||
await assert.rejects(() => DataBufferEngine.flush(), /Simulated disk write failure/);
|
||||
CSVStorageManager.appendBatch = originalAppend;
|
||||
assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 2);
|
||||
|
||||
// Subsequent healthy flush drains Redis list, releases distributed lock, and writes CSV
|
||||
const flushed = await DataBufferEngine.flush();
|
||||
assert.equal(flushed.count, 2);
|
||||
assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 0);
|
||||
assert.equal(fakeRedis.strings.has(REDIS_FLUSH_LOCK_KEY), false);
|
||||
|
||||
// Verify Port 4000 /internal/recent & /internal/metrics read shared Redis without double-counting
|
||||
const serverMod = require("../server/index.js") as IngestionServerModule;
|
||||
const origWriterDir = serverMod.writer.baseDir;
|
||||
serverMod.writer.baseDir = tmpStorageDir;
|
||||
serverMod.setRedisClientForTests(fakeRedis);
|
||||
await new Promise<void>((resolve) => serverMod.server.listen(0, "127.0.0.1", () => resolve()));
|
||||
const addr = serverMod.server.address() as AddressInfo;
|
||||
process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${addr.port}`;
|
||||
|
||||
try {
|
||||
await DataBufferEngine.pushAsync({
|
||||
clientId: "phase03_redis",
|
||||
eventId: "r-evt-3",
|
||||
timestamp: "2026-11-17T10:00:02.000Z",
|
||||
payload: JSON.stringify({ seq: 3 }),
|
||||
});
|
||||
const stats = await DataBufferEngine.getBufferStatsAsync();
|
||||
assert.equal(stats.redisEnabled, true);
|
||||
assert.equal(stats.redisBufferSize, 1);
|
||||
|
||||
const metrics = await ApiEngine.getDashboardMetrics();
|
||||
// Shared Redis count (1) must NOT be counted twice as 2 across Port 3000 + Port 4000
|
||||
assert.equal(metrics.bufferStats.currentBufferSize, 1);
|
||||
|
||||
// Clear recent list & buffer via Port 4000 flush
|
||||
const flushRes = await serverMod.flushBuffer();
|
||||
assert.equal(flushRes.written, 1);
|
||||
assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 0);
|
||||
} finally {
|
||||
serverMod.writer.baseDir = origWriterDir;
|
||||
serverMod.setRedisClientForTests(undefined);
|
||||
serverMod.stopTimers();
|
||||
delete process.env.INGESTION_INTERNAL_URL;
|
||||
await new Promise<void>((resolve) => serverMod.io.close(() => serverMod.server.close(() => resolve())));
|
||||
}
|
||||
|
||||
// Fallback to local RAM buffer when Redis is disabled (null)
|
||||
DataBufferEngine.setRedisClientForTests(null);
|
||||
await DataBufferEngine.pushAsync({
|
||||
clientId: "phase03_ram",
|
||||
eventId: "ram-evt-1",
|
||||
timestamp: "2026-11-17T10:01:00.000Z",
|
||||
payload: JSON.stringify({ mode: "ram" }),
|
||||
});
|
||||
const ramStats = await DataBufferEngine.getBufferStatsAsync();
|
||||
assert.equal(ramStats.redisEnabled, false);
|
||||
assert.equal(ramStats.localBufferSize, 1);
|
||||
const ramFlushed = await DataBufferEngine.flush();
|
||||
assert.equal(ramFlushed.count, 1);
|
||||
});
|
||||
|
||||
test("BUG-09: Cooperative CSV partition locking serializes concurrent Port 3000 + Port 4000 writers, recovers stale locks, and surfaces Port 4000 flush errors", async () => {
|
||||
const worker = new CsvWriterWorker(tmpStorageDir);
|
||||
const partitionDate = "2026-11-17";
|
||||
const clientId = "phase03_concurrent";
|
||||
|
||||
const batchA: DataEvent[] = Array.from({ length: 15 }, (_, idx) => ({
|
||||
clientId,
|
||||
eventId: `a-${idx}`,
|
||||
timestamp: `${partitionDate}T12:00:00.000Z`,
|
||||
payload: JSON.stringify({ writer: "A", idx }),
|
||||
}));
|
||||
const batchB: DataEvent[] = Array.from({ length: 15 }, (_, idx) => ({
|
||||
clientId,
|
||||
eventId: `b-${idx}`,
|
||||
timestamp: `${partitionDate}T12:00:01.000Z`,
|
||||
payload: JSON.stringify({ writer: "B", idx }),
|
||||
}));
|
||||
|
||||
await Promise.all([
|
||||
CSVStorageManager.appendBatch(batchA),
|
||||
worker.writeBatch(batchB),
|
||||
]);
|
||||
|
||||
const csvPath = path.join(tmpStorageDir, clientId, `${partitionDate}.csv`);
|
||||
const lockPath = path.join(tmpStorageDir, clientId, `.${partitionDate}.csv.lock`);
|
||||
assert.equal(fs.existsSync(lockPath), false, "Lock file must be cleaned up after writes");
|
||||
|
||||
const lines = fs.readFileSync(csvPath, "utf8").trim().split("\n");
|
||||
assert.equal(lines[0], '"clientId","eventId","timestamp","payload"');
|
||||
assert.equal(lines.length, 31, "Must contain exactly 1 header + 30 complete rows without interleaving");
|
||||
|
||||
// Stale lock recovery (>30s old mtime)
|
||||
fs.writeFileSync(lockPath, "999999:0", "utf8");
|
||||
const pastTime = new Date(Date.now() - 45_000);
|
||||
fs.utimesSync(lockPath, pastTime, pastTime);
|
||||
await CSVStorageManager.appendBatch([
|
||||
{
|
||||
clientId,
|
||||
eventId: "after-stale-lock",
|
||||
timestamp: `${partitionDate}T12:00:02.000Z`,
|
||||
payload: JSON.stringify({ staleRecovered: true }),
|
||||
},
|
||||
]);
|
||||
assert.equal(fs.existsSync(lockPath), false);
|
||||
const files = await CSVStorageManager.listAllFilesAsync();
|
||||
assert.equal(files[0].linesCount, 31);
|
||||
|
||||
// Explicit flush error & warning propagation in POST /api/storage
|
||||
const failingBridge = createServer((_req, res) => {
|
||||
res.statusCode = 500;
|
||||
res.setHeader("Content-Type", "application/json");
|
||||
res.end(JSON.stringify({ success: false, error: "Disk permission denied on Port 4000" }));
|
||||
});
|
||||
await new Promise<void>((resolve) => failingBridge.listen(0, "127.0.0.1", () => resolve()));
|
||||
const bridgeAddr = failingBridge.address() as AddressInfo;
|
||||
process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${bridgeAddr.port}`;
|
||||
|
||||
try {
|
||||
const errRes = await postStorageRoute(new NextRequest("http://localhost:3000/api/storage", { method: "POST" }));
|
||||
assert.equal(errRes.status, 502);
|
||||
const errJson = await errRes.json();
|
||||
assert.equal(errJson.success, false);
|
||||
assert.equal(errJson.code, "INGESTION_FLUSH_FAILED");
|
||||
assert.equal(errJson.ingestionFlushed, false);
|
||||
} finally {
|
||||
delete process.env.INGESTION_INTERNAL_URL;
|
||||
await new Promise<void>((resolve) => failingBridge.close(() => resolve()));
|
||||
}
|
||||
|
||||
// When Port 4000 is down and INGESTION_INTERNAL_URL is unset, standalone dev surfaces explicit warning + ingestionFlushed: false
|
||||
const standaloneRes = await postStorageRoute(new NextRequest("http://localhost:3000/api/storage", { method: "POST" }));
|
||||
assert.equal(standaloneRes.status, 200);
|
||||
const standaloneJson = await standaloneRes.json();
|
||||
assert.equal(standaloneJson.ingestionFlushed, false);
|
||||
assert.match(String(standaloneJson.warning || ""), /unreachable/i);
|
||||
});
|
||||
|
||||
test("BUG-10: Cached streaming row counter counts multiline RFC 4180 payloads accurately and avoids repeated disk reads", async () => {
|
||||
await CSVStorageManager.appendBatch([
|
||||
{
|
||||
clientId: "phase03_perf",
|
||||
eventId: "multiline-1",
|
||||
timestamp: "2026-11-17T08:00:00.000Z",
|
||||
payload: JSON.stringify({ note: "line1\nline2\n\"quoted\"\nline4" }),
|
||||
},
|
||||
{
|
||||
clientId: "phase03_perf",
|
||||
eventId: "multiline-2",
|
||||
timestamp: "2026-11-17T08:00:01.000Z",
|
||||
payload: JSON.stringify({ note: "single" }),
|
||||
},
|
||||
]);
|
||||
|
||||
const firstStats = await CSVStorageManager.getStorageStatsAsync();
|
||||
assert.equal(firstStats.totalRows, 2);
|
||||
|
||||
// Spy on fs.createReadStream to verify repeated dashboard polls hit mtime+size cache with 0 file scans
|
||||
const origCreateReadStream = fs.createReadStream;
|
||||
let streamReads = 0;
|
||||
(fs as unknown as { createReadStream: typeof fs.createReadStream }).createReadStream = ((
|
||||
...args: Parameters<typeof fs.createReadStream>
|
||||
) => {
|
||||
streamReads += 1;
|
||||
return origCreateReadStream(...args);
|
||||
}) as typeof fs.createReadStream;
|
||||
|
||||
try {
|
||||
const secondStats = await CSVStorageManager.getStorageStatsAsync();
|
||||
assert.equal(secondStats.totalRows, 2);
|
||||
assert.equal(streamReads, 0, "Unchanged CSV files must use cached row count");
|
||||
} finally {
|
||||
(fs as unknown as { createReadStream: typeof fs.createReadStream }).createReadStream =
|
||||
origCreateReadStream;
|
||||
}
|
||||
|
||||
// Server 3s settings sync must not repeatedly prune when autoPruneDays is unchanged
|
||||
const serverMod = require("../server/index.js") as IngestionServerModule;
|
||||
const oldPartitionDir = path.join(tmpStorageDir, "phase03_prune_check");
|
||||
fs.mkdirSync(oldPartitionDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(oldPartitionDir, "2020-01-01.csv"), "h\nr\n", "utf8");
|
||||
const prunedCount = CSVStorageManager.pruneExpiredPartitions(30);
|
||||
assert.equal(prunedCount, 1);
|
||||
|
||||
serverMod.applyRuntimeSettingsObject({ autoPruneDays: 30 });
|
||||
fs.writeFileSync(path.join(oldPartitionDir, "2020-01-02.csv"), "h\nr\n", "utf8");
|
||||
// Same autoPruneDays=30 applied again (simulating 3s DB poll) must NOT prune on every tick
|
||||
serverMod.applyRuntimeSettingsObject({ autoPruneDays: 30 });
|
||||
assert.equal(fs.existsSync(path.join(oldPartitionDir, "2020-01-02.csv")), true);
|
||||
serverMod.stopTimers();
|
||||
});
|
||||
|
||||
test("BUG-12: Prisma schema models and mappings align cleanly with runtime SQL DDL and Docker Compose Redis wiring", () => {
|
||||
const schemaPath = path.resolve(process.cwd(), "prisma/schema.prisma");
|
||||
const schemaText = fs.readFileSync(schemaPath, "utf8");
|
||||
|
||||
assert.match(schemaText, /model ClientSDK[\s\S]*clientId\s+String\s+@id\s+@map\("client_id"\)/);
|
||||
assert.match(schemaText, /token\s+String/);
|
||||
assert.match(schemaText, /model DeletedClient[\s\S]*@@map\("deleted_clients"\)/);
|
||||
assert.match(schemaText, /model AdminUser[\s\S]*tokenVersion\s+Int\s+@default\(1\)\s+@map\("token_version"\)/);
|
||||
assert.match(schemaText, /model AdminLoginAttempt[\s\S]*@@map\("admin_login_attempts"\)/);
|
||||
assert.match(schemaText, /model AdminSettings[\s\S]*maxEventsPerSec\s+Int\s+@default\(100\)\s+@map\("max_events_per_sec"\)/);
|
||||
assert.equal(/model ApiKey\b/.test(schemaText), false);
|
||||
assert.equal(/model AuditLog\b/.test(schemaText), false);
|
||||
|
||||
const composeText = fs.readFileSync(path.resolve(process.cwd(), "docker/docker-compose.yml"), "utf8");
|
||||
assert.equal((composeText.match(/REDIS_URL:\s*"redis:\/\/redis:6379"/g) || []).length, 2);
|
||||
assert.match(composeText, /redis:\s*\n\s*condition:\s*service_healthy/);
|
||||
});
|
||||
|
||||
test("BUG-14: GET /api/data searches eventId & clientId + UTC dates, GET /api/events caps CSV tail, and GET /api/export rejects ambiguous filename & honors ?flush=false", async () => {
|
||||
DataBufferEngine.setRedisClientForTests(null);
|
||||
|
||||
await SDKManager.registerClientAsync({ clientId: "alpha_cli", name: "Alpha", rateLimit: 100 });
|
||||
await SDKManager.registerClientAsync({ clientId: "beta_cli", name: "Beta", rateLimit: 100 });
|
||||
|
||||
const tsMillis = String(Date.UTC(2026, 10, 17, 15, 0, 0)); // 2026-11-17
|
||||
await DataBufferEngine.pushBatchAsync([
|
||||
{
|
||||
clientId: "alpha_cli",
|
||||
eventId: "unique-findme-id-99",
|
||||
timestamp: tsMillis,
|
||||
payload: JSON.stringify({ temp: 22 }),
|
||||
},
|
||||
{
|
||||
clientId: "beta_cli",
|
||||
eventId: "beta-id-1",
|
||||
timestamp: "2026-11-17T15:01:00.000Z",
|
||||
payload: JSON.stringify({ temp: 33 }),
|
||||
},
|
||||
]);
|
||||
|
||||
// 1. Search by eventId with from/to UTC normalization while still buffered
|
||||
const dataByEvent = await getDataRoute(
|
||||
new NextRequest("http://localhost:3000/api/data?search=unique-findme-id-99&from=2026-11-17&to=2026-11-17"),
|
||||
);
|
||||
assert.equal(dataByEvent.status, 200);
|
||||
const dataEventJson = await dataByEvent.json();
|
||||
assert.equal(dataEventJson.total, 1);
|
||||
assert.equal(dataEventJson.events[0].eventId, "unique-findme-id-99");
|
||||
|
||||
// 2. Search by clientId substring
|
||||
const dataByClient = await getDataRoute(
|
||||
new NextRequest("http://localhost:3000/api/data?search=beta_cli"),
|
||||
);
|
||||
const dataClientJson = await dataByClient.json();
|
||||
assert.equal(dataClientJson.total, 1);
|
||||
assert.equal(dataClientJson.events[0].clientId, "beta_cli");
|
||||
|
||||
// 3. Export with ?flush=false must NOT drain active buffer
|
||||
const noFlushRes = await getExportRoute(
|
||||
new NextRequest("http://localhost:3000/api/export?flush=false&format=json"),
|
||||
);
|
||||
assert.equal(noFlushRes.status, 200);
|
||||
assert.equal((await noFlushRes.text()).trim(), "[]");
|
||||
assert.equal((await DataBufferEngine.getBufferStatsAsync()).localBufferSize, 2);
|
||||
|
||||
// 4. Default flush writes both 2026-11-17.csv files -> clientId=all + filename must 400 AMBIGUOUS_FILENAME
|
||||
const flushExportRes = await getExportRoute(
|
||||
new NextRequest("http://localhost:3000/api/export?format=json"),
|
||||
);
|
||||
assert.equal(flushExportRes.status, 200);
|
||||
const flushedPayload = await flushExportRes.json();
|
||||
assert.equal(flushedPayload.length, 2);
|
||||
|
||||
const ambiguousRes = await getExportRoute(
|
||||
new NextRequest("http://localhost:3000/api/export?clientId=all&filename=2026-11-17.csv"),
|
||||
);
|
||||
assert.equal(ambiguousRes.status, 400);
|
||||
const ambiguousJson = await ambiguousRes.json();
|
||||
assert.equal(ambiguousJson.code, "AMBIGUOUS_FILENAME");
|
||||
|
||||
// Disambiguated download with explicit clientId succeeds cleanly
|
||||
const specificRes = await getExportRoute(
|
||||
new NextRequest("http://localhost:3000/api/export?clientId=alpha_cli&filename=2026-11-17.csv"),
|
||||
);
|
||||
assert.equal(specificRes.status, 200);
|
||||
const csvBody = await specificRes.text();
|
||||
assert.match(csvBody, /unique-findme-id-99/);
|
||||
|
||||
// 5. GET /api/events with limit=1 returns latest single stored event cleanly
|
||||
const eventsRes = await getEventsRoute(
|
||||
new NextRequest("http://localhost:3000/api/events?clientId=alpha_cli&limit=1"),
|
||||
);
|
||||
assert.equal(eventsRes.status, 200);
|
||||
const eventsJson = await eventsRes.json();
|
||||
assert.equal(eventsJson.events.length, 1);
|
||||
assert.equal(eventsJson.events[0].eventId, "unique-findme-id-99");
|
||||
});
|
||||
@@ -0,0 +1,266 @@
|
||||
"""
|
||||
Phase-05 Verification Tests (BUG-18):
|
||||
- Single Source of Truth between canonical `datastack` (`sdk/python/datastack`)
|
||||
and compatibility alias `datastack_sdk` (`sdk-python`).
|
||||
- `CLIENT_ID` config loading (`config.datastack.py`, `DATASTACK_CLIENT_ID`, kwarg),
|
||||
JWT `clientId`/`sub` fallback extraction, and mismatch validation.
|
||||
- End-to-end propagation of `clientId` & `X-DataStack-Client-Id` into HTTP
|
||||
fallback ingestion (`send`, `send_batch`, `send_file`), status snapshot, and
|
||||
offline disk spool replay against a local HTTP test server.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
CANONICAL_PKG_ROOT = REPO_ROOT / "sdk" / "python" / "datastack"
|
||||
COMPAT_PKG_ROOT = REPO_ROOT / "sdk-python"
|
||||
|
||||
for p in (str(CANONICAL_PKG_ROOT), str(COMPAT_PKG_ROOT)):
|
||||
if p not in sys.path:
|
||||
sys.path.insert(0, p)
|
||||
|
||||
import datastack # noqa: E402
|
||||
import datastack_sdk # noqa: E402
|
||||
from datastack import ( # noqa: E402
|
||||
DataStack,
|
||||
DataStackClient,
|
||||
DataStackConfig,
|
||||
DataStackConfigError,
|
||||
DataStackOfflineSpoolError,
|
||||
DiskSpooler,
|
||||
HeartbeatMonitor,
|
||||
load_config,
|
||||
)
|
||||
|
||||
|
||||
def _make_jwt(payload: Dict[str, Any]) -> str:
|
||||
header = (
|
||||
base64.urlsafe_b64encode(json.dumps({"alg": "HS256", "typ": "JWT"}).encode())
|
||||
.decode()
|
||||
.rstrip("=")
|
||||
)
|
||||
body = (
|
||||
base64.urlsafe_b64encode(json.dumps(payload).encode())
|
||||
.decode()
|
||||
.rstrip("=")
|
||||
)
|
||||
return f"{header}.{body}.testsig"
|
||||
|
||||
|
||||
class Phase05PythonSdkTests(unittest.TestCase):
|
||||
def test_single_source_of_truth_identity(self) -> None:
|
||||
self.assertIs(datastack_sdk.DataStack, datastack.DataStack)
|
||||
self.assertIs(datastack_sdk.DataStackClient, datastack.DataStackClient)
|
||||
self.assertIs(datastack_sdk.DataStackConfig, datastack.DataStackConfig)
|
||||
self.assertIs(datastack_sdk.load_config, datastack.load_config)
|
||||
self.assertIs(
|
||||
datastack_sdk.DataStackConfigError, datastack.DataStackConfigError
|
||||
)
|
||||
self.assertIs(
|
||||
datastack_sdk.DataStackOfflineSpoolError,
|
||||
datastack.DataStackOfflineSpoolError,
|
||||
)
|
||||
self.assertIs(datastack_sdk.DiskSpooler, datastack.DiskSpooler)
|
||||
self.assertIs(datastack_sdk.HeartbeatMonitor, datastack.HeartbeatMonitor)
|
||||
self.assertEqual(datastack_sdk.__version__, datastack.__version__)
|
||||
|
||||
# Submodule shims also resolve to canonical classes
|
||||
from datastack_sdk.client import DataStackClient as ShimClient
|
||||
from datastack_sdk.config import load_config as shim_load_config
|
||||
from datastack_sdk.exceptions import (
|
||||
DataStackOfflineSpoolError as ShimSpoolErr,
|
||||
)
|
||||
|
||||
self.assertIs(ShimClient, DataStackClient)
|
||||
self.assertIs(shim_load_config, load_config)
|
||||
self.assertIs(ShimSpoolErr, DataStackOfflineSpoolError)
|
||||
|
||||
def test_client_id_config_env_and_jwt_validation(self) -> None:
|
||||
tok_alpha = _make_jwt({"clientId": "alpha_svc", "sub": "alpha_svc"})
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
cfg_file = Path(tmpdir) / "config.datastack.py"
|
||||
cfg_file.write_text(
|
||||
f'DATASTACK_CONFIG = {{\n'
|
||||
f' "SERVER_URL": "http://127.0.0.1:49999",\n'
|
||||
f' "API_URL": "http://127.0.0.1:39999",\n'
|
||||
f' "CLIENT_TOKEN": "{tok_alpha}",\n'
|
||||
f' "CLIENT_ID": "alpha_svc",\n'
|
||||
f' "AUTO_CONNECT": False,\n'
|
||||
f' "OFFLINE_SPOOL_DIR": "{Path(tmpdir) / "spool"}",\n'
|
||||
f'}}\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
cfg = load_config(config_path=str(cfg_file))
|
||||
self.assertIsInstance(cfg, DataStackConfig)
|
||||
self.assertEqual(cfg.client_id, "alpha_svc")
|
||||
|
||||
# Matching explicit client_id succeeds
|
||||
client = DataStack(config_path=str(cfg_file), auto_connect=False)
|
||||
self.assertEqual(client.client_id, "alpha_svc")
|
||||
self.assertEqual(client.status()["clientId"], "alpha_svc")
|
||||
client.disconnect()
|
||||
|
||||
# Env override matching JWT succeeds; mismatching raises DataStackConfigError
|
||||
old_env = os.environ.get("DATASTACK_CLIENT_ID")
|
||||
try:
|
||||
os.environ["DATASTACK_CLIENT_ID"] = "wrong_svc"
|
||||
with self.assertRaises(DataStackConfigError):
|
||||
DataStack(config_path=str(cfg_file), auto_connect=False)
|
||||
finally:
|
||||
if old_env is None:
|
||||
os.environ.pop("DATASTACK_CLIENT_ID", None)
|
||||
else:
|
||||
os.environ["DATASTACK_CLIENT_ID"] = old_env
|
||||
|
||||
# Mismatched kwarg client_id raises DataStackConfigError
|
||||
with self.assertRaises(DataStackConfigError):
|
||||
DataStack(
|
||||
config_path=str(cfg_file),
|
||||
client_id="mismatch_svc",
|
||||
auto_connect=False,
|
||||
)
|
||||
|
||||
# Auto-derives client_id from JWT when CLIENT_ID is omitted
|
||||
tok_derived = _make_jwt({"sub": "derived_worker"})
|
||||
derived_client = DataStack(
|
||||
server_url="http://127.0.0.1:49999",
|
||||
api_url="http://127.0.0.1:39999",
|
||||
token=tok_derived,
|
||||
auto_connect=False,
|
||||
)
|
||||
self.assertEqual(derived_client.client_id, "derived_worker")
|
||||
self.assertEqual(derived_client.status()["clientId"], "derived_worker")
|
||||
derived_client.disconnect()
|
||||
|
||||
def test_http_fallback_and_offline_spool_carry_client_id(self) -> None:
|
||||
recorded_requests: List[Dict[str, Any]] = []
|
||||
tok = _make_jwt({"clientId": "sensor_01", "sub": "sensor_01"})
|
||||
|
||||
class MockHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self) -> None: # noqa: N802
|
||||
if self.path == "/api/health":
|
||||
body = json.dumps({"status": "online"}).encode("utf-8")
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
return
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
|
||||
def do_POST(self) -> None: # noqa: N802
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
raw = self.rfile.read(length).decode("utf-8")
|
||||
parsed = json.loads(raw) if raw else {}
|
||||
recorded_requests.append(
|
||||
{
|
||||
"path": self.path,
|
||||
"client_id_header": self.headers.get("X-DataStack-Client-Id"),
|
||||
"auth": self.headers.get("Authorization"),
|
||||
"body": parsed,
|
||||
}
|
||||
)
|
||||
resp_payload: Dict[str, Any] = {"success": True}
|
||||
if isinstance(parsed, dict) and "batch" in parsed:
|
||||
resp_payload["processed"] = len(parsed["batch"])
|
||||
resp_payload["rejected"] = 0
|
||||
elif isinstance(parsed, dict) and "records" in parsed:
|
||||
resp_payload["processed"] = len(parsed["records"])
|
||||
resp_payload["rejected"] = 0
|
||||
body = json.dumps(resp_payload).encode("utf-8")
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, format: str, *args: Any) -> None:
|
||||
return
|
||||
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), MockHandler)
|
||||
port = server.server_address[1]
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
spool_dir = Path(tmpdir) / "spool"
|
||||
csv_file = Path(tmpdir) / "sample.csv"
|
||||
csv_file.write_text("temp,unit\n21.5,C\n22.0,C\n", encoding="utf-8")
|
||||
|
||||
client = DataStack(
|
||||
server_url="http://127.0.0.1:1", # unreachable WS -> HTTP fallback
|
||||
api_url=f"http://127.0.0.1:{port}",
|
||||
token=tok,
|
||||
client_id="sensor_01",
|
||||
auto_connect=False,
|
||||
request_timeout_sec=1.0,
|
||||
max_retries=0,
|
||||
offline_spool_dir=str(spool_dir),
|
||||
)
|
||||
|
||||
# Single event via HTTP fallback
|
||||
res_single = client.send({"reading": 42})
|
||||
self.assertTrue(res_single.get("success"))
|
||||
self.assertEqual(res_single.get("transport"), "HTTP_FALLBACK")
|
||||
|
||||
# Batch events via HTTP fallback
|
||||
res_batch = client.send_batch([{"k": "a"}, {"k": "b"}])
|
||||
self.assertEqual(res_batch.get("processed"), 2)
|
||||
|
||||
# File ingestion via HTTP fallback
|
||||
res_file = client.send_file(csv_file)
|
||||
self.assertEqual(res_file.get("processed"), 2)
|
||||
|
||||
client.disconnect()
|
||||
|
||||
self.assertEqual(len(recorded_requests), 3)
|
||||
for req in recorded_requests:
|
||||
self.assertEqual(req["path"], "/api/events")
|
||||
self.assertEqual(req["client_id_header"], "sensor_01")
|
||||
self.assertEqual(req["auth"], f"Bearer {tok}")
|
||||
|
||||
self.assertEqual(recorded_requests[0]["body"]["clientId"], "sensor_01")
|
||||
self.assertEqual(
|
||||
[r["clientId"] for r in recorded_requests[1]["body"]["batch"]],
|
||||
["sensor_01", "sensor_01"],
|
||||
)
|
||||
self.assertEqual(
|
||||
[r["clientId"] for r in recorded_requests[2]["body"]["records"]],
|
||||
["sensor_01", "sensor_01"],
|
||||
)
|
||||
|
||||
# Offline spool check when HTTP is also unreachable
|
||||
offline_client = datastack_sdk.DataStackClient(
|
||||
server_url="http://127.0.0.1:1",
|
||||
api_url="http://127.0.0.1:1",
|
||||
token=tok,
|
||||
auto_connect=False,
|
||||
request_timeout_sec=1.0,
|
||||
max_retries=0,
|
||||
offline_spool_dir=str(spool_dir),
|
||||
)
|
||||
spool_res = offline_client.send({"offline_metric": 99})
|
||||
offline_client.disconnect()
|
||||
self.assertTrue(spool_res.get("spooled"))
|
||||
self.assertEqual(spool_res.get("transport"), "OFFLINE_SPOOL")
|
||||
self.assertGreaterEqual(spool_res.get("spooledCount", 0), 1)
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,81 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
|
||||
const ROOT = process.cwd();
|
||||
|
||||
function readPngDimensions(filePath: string): { width: number; height: number } {
|
||||
const buf = fs.readFileSync(filePath);
|
||||
const pngSig = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
||||
assert.ok(buf.subarray(0, 8).equals(pngSig), `${path.basename(filePath)} must be a valid PNG`);
|
||||
assert.equal(buf.subarray(12, 16).toString('ascii'), 'IHDR');
|
||||
return {
|
||||
width: buf.readUInt32BE(16),
|
||||
height: buf.readUInt32BE(20),
|
||||
};
|
||||
}
|
||||
|
||||
test('BUG-25: PWA manifest, icons (192/512/maskable/apple-touch), and Service Worker are complete and valid', () => {
|
||||
const manifestPath = path.join(ROOT, 'public/manifest.json');
|
||||
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
|
||||
|
||||
assert.equal(manifest.id, '/');
|
||||
assert.equal(manifest.scope, '/');
|
||||
assert.equal(manifest.display, 'standalone');
|
||||
assert.ok(manifest.short_name && manifest.short_name.length <= 12);
|
||||
assert.ok(Array.isArray(manifest.icons) && manifest.icons.length >= 3);
|
||||
|
||||
for (const icon of manifest.icons) {
|
||||
assert.ok(!String(icon.purpose || '').includes('any maskable'), 'Separate any and maskable purposes');
|
||||
const rel = String(icon.src).replace(/^\//, '');
|
||||
const full = path.join(ROOT, 'public', rel);
|
||||
assert.ok(fs.existsSync(full), `Missing manifest icon ${icon.src}`);
|
||||
const [expW, expH] = String(icon.sizes).split('x').map(Number);
|
||||
const dims = readPngDimensions(full);
|
||||
assert.equal(dims.width, expW);
|
||||
assert.equal(dims.height, expH);
|
||||
}
|
||||
|
||||
const appleDims = readPngDimensions(path.join(ROOT, 'public/apple-touch-icon.png'));
|
||||
assert.deepEqual(appleDims, { width: 180, height: 180 });
|
||||
assert.ok(fs.existsSync(path.join(ROOT, 'public/icon.svg')));
|
||||
|
||||
const swContent = fs.readFileSync(path.join(ROOT, 'public/sw.js'), 'utf8');
|
||||
assert.match(swContent, /request\.method !== ['"]GET['"]/);
|
||||
assert.match(swContent, /\/api\//);
|
||||
assert.match(swContent, /\/socket\.io\//);
|
||||
assert.match(swContent, /new Response\(/);
|
||||
|
||||
const layoutContent = fs.readFileSync(path.join(ROOT, 'app/layout.tsx'), 'utf8');
|
||||
assert.match(layoutContent, /PwaLifecycle/);
|
||||
assert.match(layoutContent, /export const viewport: Viewport/);
|
||||
assert.match(layoutContent, /apple-touch-icon\.png/);
|
||||
|
||||
const pwaClient = fs.readFileSync(path.join(ROOT, 'components/pwa/PwaLifecycle.tsx'), 'utf8');
|
||||
assert.match(pwaClient, /serviceWorker\.register\(['"]\/sw\.js['"]/);
|
||||
assert.match(pwaClient, /beforeinstallprompt/);
|
||||
});
|
||||
|
||||
test('BUG-17: Landing/Navbar/Footer/404 navigation uses root-relative anchors and valid Link semantics', () => {
|
||||
const files = [
|
||||
'components/layout/Navbar.tsx',
|
||||
'components/layout/Footer.tsx',
|
||||
'components/sections/Hero.tsx',
|
||||
'components/sections/LandingAbout.tsx',
|
||||
'app/not-found.tsx',
|
||||
];
|
||||
|
||||
for (const rel of files) {
|
||||
const content = fs.readFileSync(path.join(ROOT, rel), 'utf8');
|
||||
assert.equal(/href=["']#/.test(content), false, `${rel} should use /#... root anchors instead of bare #...`);
|
||||
const linkBlocks = content.match(/<Link\b[\s\S]*?<\/Link>/g) || [];
|
||||
for (const block of linkBlocks) {
|
||||
assert.equal(/<Button\b/.test(block), false, `${rel} should not nest <Button> inside <Link>: ${block}`);
|
||||
}
|
||||
}
|
||||
|
||||
const notFoundContent = fs.readFileSync(path.join(ROOT, 'app/not-found.tsx'), 'utf8');
|
||||
assert.match(notFoundContent, /href=["']\/["']/);
|
||||
assert.match(notFoundContent, /href=["']\/vcon\/dashboard["']/);
|
||||
});
|
||||
Reference in new issue
Block a user