refactor(auth): standardize session management and security
- Centralize secret retrieval using `getAdminSessionSecret` - Replace manual session parsing with `verifyRequestAdminSession` utility - Tighten API validation and improve security headers handling - Enforce environment variable security for production deployments
This commit is contained in:
1 parent
34519c07df
commit
177ed4a844
78 files changed
+6748
-3157
No files matched your search
+17
-10
@@ -1,16 +1,23 @@
|
|||||||
# DataStack Environment Variables Configuration
|
# DataStack Environment Variables Configuration
|
||||||
|
# NOTE: In production (NODE_ENV=production), replace all placeholder values below with strong 32+ char secrets.
|
||||||
|
|
||||||
# Administrator Account (defaults to admin@datastack.dev / admin12345678 if omitted)
|
# Security & Session Secrets (REQUIRED in production, min 32 characters)
|
||||||
DATASTACK_ADMIN_EMAIL=admin@datastack.dev
|
DATASTACK_ADMIN_SESSION_SECRET=replace-with-32-plus-char-admin-session-secret
|
||||||
DATASTACK_ADMIN_PASSWORD=admin12345678
|
DATASTACK_JWT_SECRET=replace-with-32-plus-char-client-jwt-secret
|
||||||
|
DATASTACK_ADMIN_SETUP_KEY=replace-with-32-plus-char-admin-setup-key
|
||||||
|
DATASTACK_INTERNAL_SECRET=replace-with-32-plus-char-internal-service-secret
|
||||||
|
|
||||||
# Security & Session Secrets (min 32 characters)
|
# Administrator Account (development/in-memory bootstrap; production uses /vcon/setup)
|
||||||
DATASTACK_ADMIN_SESSION_SECRET=datastack-admin-session-default-secret-32-chars
|
DATASTACK_ADMIN_EMAIL=admin@example.com
|
||||||
DATASTACK_JWT_SECRET=datastack-jwt-default-secret-key-32-chars-long
|
# DATASTACK_ADMIN_PASSWORD=replace-with-strong-password
|
||||||
DATASTACK_ADMIN_SETUP_KEY=datastack-default-setup-key-32-chars
|
# DATASTACK_ALLOW_DEMO_ADMIN=false
|
||||||
|
|
||||||
# PostgreSQL Database (optional - in-memory fallback active when not configured)
|
# PostgreSQL Database (recommended for production persistence)
|
||||||
# DATABASE_URL=postgresql://datastack:password@localhost:5432/datastack_db
|
# DATABASE_URL=postgresql://datastack:password@localhost:5432/datastack_db
|
||||||
|
|
||||||
# Ingestion Engine Port (defaults to 4000)
|
# Shared Redis Buffer (optional; enables cross-process buffering & distributed flush locks)
|
||||||
PORT_SOCKET=4000
|
# REDIS_URL=redis://127.0.0.1:6379
|
||||||
|
|
||||||
|
# Service Networking
|
||||||
|
PORT_SOCKET=4000
|
||||||
|
INGESTION_INTERNAL_URL=http://127.0.0.1:4000
|
||||||
@@ -9,10 +9,8 @@ import { redirect } from 'next/navigation';
|
|||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
export default async function LandingPage() {
|
export default async function LandingPage() {
|
||||||
if (process.env.DATABASE_URL) {
|
const isConfigured = await isAdminSetupComplete().catch(() => false);
|
||||||
const isConfigured = await isAdminSetupComplete().catch(() => false);
|
if (!isConfigured) redirect('/vcon/setup');
|
||||||
if (!isConfigured) redirect('/vcon/setup');
|
|
||||||
}
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<MainLayout>
|
<MainLayout>
|
||||||
|
|||||||
+102
-79
@@ -1,11 +1,12 @@
|
|||||||
import {
|
import {
|
||||||
createAdminSessionToken,
|
createAdminSessionToken,
|
||||||
|
getAdminSessionSecret,
|
||||||
getSessionCookieOptions,
|
getSessionCookieOptions,
|
||||||
} from "@/core/auth/adminSession";
|
} from "@/core/auth/adminSession";
|
||||||
import {
|
import {
|
||||||
|
authenticateAdminCredentials,
|
||||||
clearAdminLoginAttempts,
|
clearAdminLoginAttempts,
|
||||||
consumeAdminLoginAttempt,
|
consumeAdminLoginAttempt,
|
||||||
verifyAdminCredentials,
|
|
||||||
} from "@/core/auth/adminUsers";
|
} from "@/core/auth/adminUsers";
|
||||||
import { NextRequest, NextResponse } from "next/server";
|
import { NextRequest, NextResponse } from "next/server";
|
||||||
|
|
||||||
@@ -13,10 +14,19 @@ const SESSION_COOKIE = "datastack_admin_session";
|
|||||||
const SESSION_SECONDS = 8 * 60 * 60;
|
const SESSION_SECONDS = 8 * 60 * 60;
|
||||||
const REMEMBERED_SESSION_SECONDS = 30 * 24 * 60 * 60;
|
const REMEMBERED_SESSION_SECONDS = 30 * 24 * 60 * 60;
|
||||||
|
|
||||||
|
function extractClientIp(request: NextRequest) {
|
||||||
|
const forwardedFor = request.headers.get("x-forwarded-for");
|
||||||
|
if (forwardedFor) {
|
||||||
|
const first = forwardedFor.split(",")[0]?.trim();
|
||||||
|
if (first) return first;
|
||||||
|
}
|
||||||
|
const realIp = request.headers.get("x-real-ip")?.trim();
|
||||||
|
if (realIp) return realIp;
|
||||||
|
return "unknown";
|
||||||
|
}
|
||||||
|
|
||||||
export async function POST(request: NextRequest) {
|
export async function POST(request: NextRequest) {
|
||||||
const sessionSecret =
|
const sessionSecret = getAdminSessionSecret();
|
||||||
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
|
|
||||||
"datastack-admin-session-default-secret-32-chars";
|
|
||||||
|
|
||||||
if (!sessionSecret || sessionSecret.length < 32) {
|
if (!sessionSecret || sessionSecret.length < 32) {
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
@@ -27,86 +37,99 @@ export async function POST(request: NextRequest) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let body: Record<string, unknown>;
|
||||||
try {
|
try {
|
||||||
const body = await request.json();
|
const parsed = await request.json();
|
||||||
const email =
|
body =
|
||||||
typeof body.email === "string" ? body.email.trim().toLowerCase() : "";
|
parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
||||||
const password = typeof body.password === "string" ? body.password : "";
|
? (parsed as Record<string, unknown>)
|
||||||
if (!email || password.length === 0 || password.length > 1024) {
|
: {};
|
||||||
return NextResponse.json(
|
|
||||||
{ error: "Invalid email or password." },
|
|
||||||
{ status: 401 },
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
let throttleDecision;
|
|
||||||
try {
|
|
||||||
throttleDecision = await consumeAdminLoginAttempt(email, sessionSecret);
|
|
||||||
} catch {
|
|
||||||
return NextResponse.json(
|
|
||||||
{ error: "Admin authentication is temporarily unavailable." },
|
|
||||||
{ status: 503 },
|
|
||||||
);
|
|
||||||
}
|
|
||||||
if (!throttleDecision.allowed) {
|
|
||||||
return NextResponse.json(
|
|
||||||
{ error: "Too many sign-in attempts. Try again later." },
|
|
||||||
{
|
|
||||||
status: 429,
|
|
||||||
headers: {
|
|
||||||
"Retry-After": String(throttleDecision.retryAfterSeconds),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
let authenticated = false;
|
|
||||||
try {
|
|
||||||
authenticated = await verifyAdminCredentials(email, password);
|
|
||||||
} catch {
|
|
||||||
return NextResponse.json(
|
|
||||||
{ error: "Admin authentication is temporarily unavailable." },
|
|
||||||
{ status: 503 },
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!authenticated) {
|
|
||||||
return NextResponse.json(
|
|
||||||
{ error: "Invalid email or password." },
|
|
||||||
{ status: 401 },
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
await clearAdminLoginAttempts(email, sessionSecret);
|
|
||||||
} catch {
|
|
||||||
return NextResponse.json(
|
|
||||||
{ error: "Admin authentication is temporarily unavailable." },
|
|
||||||
{ status: 503 },
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const remember = body.remember === true;
|
|
||||||
const maxAge = remember ? REMEMBERED_SESSION_SECONDS : SESSION_SECONDS;
|
|
||||||
const expiresAt = Date.now() + maxAge * 1000;
|
|
||||||
const token = await createAdminSessionToken(
|
|
||||||
email,
|
|
||||||
sessionSecret,
|
|
||||||
expiresAt,
|
|
||||||
);
|
|
||||||
const response = NextResponse.json({ success: true });
|
|
||||||
|
|
||||||
response.cookies.set(
|
|
||||||
SESSION_COOKIE,
|
|
||||||
token,
|
|
||||||
getSessionCookieOptions(request, maxAge),
|
|
||||||
);
|
|
||||||
|
|
||||||
return response;
|
|
||||||
} catch {
|
} catch {
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{ error: "Invalid login request." },
|
{ error: "Invalid login request." },
|
||||||
{ status: 400 },
|
{ status: 400 },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const email =
|
||||||
|
typeof body.email === "string" ? body.email.trim().toLowerCase() : "";
|
||||||
|
const password = typeof body.password === "string" ? body.password : "";
|
||||||
|
if (!email || password.length === 0 || password.length > 1024) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: "Invalid email or password." },
|
||||||
|
{ status: 401 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const clientIp = extractClientIp(request);
|
||||||
|
|
||||||
|
let throttleDecision;
|
||||||
|
try {
|
||||||
|
throttleDecision = await consumeAdminLoginAttempt(
|
||||||
|
email,
|
||||||
|
sessionSecret,
|
||||||
|
clientIp,
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: "Admin authentication is temporarily unavailable." },
|
||||||
|
{ status: 503 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (!throttleDecision.allowed) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: "Too many sign-in attempts. Try again later." },
|
||||||
|
{
|
||||||
|
status: 429,
|
||||||
|
headers: {
|
||||||
|
"Retry-After": String(throttleDecision.retryAfterSeconds),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let authResult: { authenticated: boolean; tokenVersion: number };
|
||||||
|
try {
|
||||||
|
authResult = await authenticateAdminCredentials(email, password);
|
||||||
|
} catch {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: "Admin authentication is temporarily unavailable." },
|
||||||
|
{ status: 503 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!authResult.authenticated) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: "Invalid email or password." },
|
||||||
|
{ status: 401 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
await clearAdminLoginAttempts(email, sessionSecret, clientIp);
|
||||||
|
} catch {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: "Admin authentication is temporarily unavailable." },
|
||||||
|
{ status: 503 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const remember = body.remember === true;
|
||||||
|
const maxAge = remember ? REMEMBERED_SESSION_SECONDS : SESSION_SECONDS;
|
||||||
|
const expiresAt = Date.now() + maxAge * 1000;
|
||||||
|
const token = await createAdminSessionToken(
|
||||||
|
email,
|
||||||
|
sessionSecret,
|
||||||
|
expiresAt,
|
||||||
|
authResult.tokenVersion,
|
||||||
|
);
|
||||||
|
const response = NextResponse.json({ success: true });
|
||||||
|
|
||||||
|
response.cookies.set(
|
||||||
|
SESSION_COOKIE,
|
||||||
|
token,
|
||||||
|
getSessionCookieOptions(request, maxAge),
|
||||||
|
);
|
||||||
|
|
||||||
|
return response;
|
||||||
}
|
}
|
||||||
@@ -1,11 +1,11 @@
|
|||||||
import {
|
import {
|
||||||
createAdminSessionToken,
|
createAdminSessionToken,
|
||||||
getAdminSessionPayload,
|
|
||||||
getSessionCookieOptions,
|
getSessionCookieOptions,
|
||||||
} from "@/core/auth/adminSession";
|
} from "@/core/auth/adminSession";
|
||||||
import {
|
import {
|
||||||
getAdminAccountInfo,
|
getAdminAccountInfo,
|
||||||
updateAdminAccount,
|
updateAdminAccount,
|
||||||
|
verifyRequestAdminSession,
|
||||||
} from "@/core/auth/adminUsers";
|
} from "@/core/auth/adminUsers";
|
||||||
import { NextRequest, NextResponse } from "next/server";
|
import { NextRequest, NextResponse } from "next/server";
|
||||||
|
|
||||||
@@ -17,51 +17,69 @@ function validEmail(email: string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export async function GET(request: NextRequest) {
|
export async function GET(request: NextRequest) {
|
||||||
const sessionSecret =
|
const token = request.cookies.get(SESSION_COOKIE)?.value;
|
||||||
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
|
const auth = await verifyRequestAdminSession(token);
|
||||||
"datastack-admin-session-default-secret-32-chars";
|
|
||||||
const token = request.cookies.get(SESSION_COOKIE)?.value || "";
|
|
||||||
const session = await getAdminSessionPayload(token, sessionSecret);
|
|
||||||
|
|
||||||
if (!session) {
|
if (!auth.valid) {
|
||||||
return NextResponse.json(
|
const status = auth.reason === "UNAUTHORIZED" ? 401 : 503;
|
||||||
{ error: "Authentication required." },
|
const message =
|
||||||
{ status: 401 },
|
auth.reason === "UNAUTHORIZED"
|
||||||
);
|
? "Authentication required."
|
||||||
|
: "Admin authentication is temporarily unavailable.";
|
||||||
|
return NextResponse.json({ error: message }, { status });
|
||||||
}
|
}
|
||||||
|
|
||||||
const account = await getAdminAccountInfo(session.email);
|
try {
|
||||||
return NextResponse.json({
|
const account = await getAdminAccountInfo(auth.session.email);
|
||||||
profile: {
|
return NextResponse.json({
|
||||||
email: account.email,
|
profile: {
|
||||||
role: "Owner",
|
email: account.email,
|
||||||
storageMode: account.storageMode,
|
role: "Owner",
|
||||||
createdAt: account.createdAt,
|
storageMode: account.storageMode,
|
||||||
sessionExpiresAt: new Date(session.expiresAt).toISOString(),
|
createdAt: account.createdAt,
|
||||||
},
|
sessionExpiresAt: new Date(auth.session.expiresAt).toISOString(),
|
||||||
});
|
},
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: "Unable to load admin profile." },
|
||||||
|
{ status: 503 },
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function PUT(request: NextRequest) {
|
export async function PUT(request: NextRequest) {
|
||||||
const sessionSecret =
|
const token = request.cookies.get(SESSION_COOKIE)?.value;
|
||||||
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
|
const auth = await verifyRequestAdminSession(token);
|
||||||
"datastack-admin-session-default-secret-32-chars";
|
|
||||||
const token = request.cookies.get(SESSION_COOKIE)?.value || "";
|
|
||||||
const session = await getAdminSessionPayload(token, sessionSecret);
|
|
||||||
|
|
||||||
if (!session) {
|
if (!auth.valid) {
|
||||||
|
const status = auth.reason === "UNAUTHORIZED" ? 401 : 503;
|
||||||
|
const message =
|
||||||
|
auth.reason === "UNAUTHORIZED"
|
||||||
|
? "Authentication required."
|
||||||
|
: "Admin authentication is temporarily unavailable.";
|
||||||
|
return NextResponse.json({ error: message }, { status });
|
||||||
|
}
|
||||||
|
|
||||||
|
let body: Record<string, unknown>;
|
||||||
|
try {
|
||||||
|
const parsed = await request.json();
|
||||||
|
body =
|
||||||
|
parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
||||||
|
? (parsed as Record<string, unknown>)
|
||||||
|
: {};
|
||||||
|
} catch {
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{ error: "Authentication required." },
|
{ error: "Invalid profile payload." },
|
||||||
{ status: 401 },
|
{ status: 400 },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const body = await request.json();
|
|
||||||
const nextEmail =
|
const nextEmail =
|
||||||
typeof body.email === "string"
|
typeof body.email === "string"
|
||||||
? body.email.trim().toLowerCase()
|
? body.email.trim().toLowerCase()
|
||||||
: session.email;
|
: auth.session.email;
|
||||||
const currentPassword =
|
const currentPassword =
|
||||||
typeof body.currentPassword === "string" ? body.currentPassword : "";
|
typeof body.currentPassword === "string" ? body.currentPassword : "";
|
||||||
const newPassword =
|
const newPassword =
|
||||||
@@ -88,7 +106,7 @@ export async function PUT(request: NextRequest) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const account = await getAdminAccountInfo(session.email);
|
const account = await getAdminAccountInfo(auth.session.email);
|
||||||
const result = await updateAdminAccount({
|
const result = await updateAdminAccount({
|
||||||
currentEmail: account.email,
|
currentEmail: account.email,
|
||||||
currentPassword,
|
currentPassword,
|
||||||
@@ -103,13 +121,14 @@ export async function PUT(request: NextRequest) {
|
|||||||
// Preserve remaining session duration (e.g., 30-day "Remember me" session)
|
// Preserve remaining session duration (e.g., 30-day "Remember me" session)
|
||||||
const remainingSeconds = Math.max(
|
const remainingSeconds = Math.max(
|
||||||
SESSION_SECONDS,
|
SESSION_SECONDS,
|
||||||
Math.ceil((session.expiresAt - Date.now()) / 1000),
|
Math.ceil((auth.session.expiresAt - Date.now()) / 1000),
|
||||||
);
|
);
|
||||||
const expiresAt = Date.now() + remainingSeconds * 1000;
|
const expiresAt = Date.now() + remainingSeconds * 1000;
|
||||||
const nextToken = await createAdminSessionToken(
|
const nextToken = await createAdminSessionToken(
|
||||||
result.email,
|
result.email,
|
||||||
sessionSecret,
|
auth.sessionSecret,
|
||||||
expiresAt,
|
expiresAt,
|
||||||
|
result.tokenVersion,
|
||||||
);
|
);
|
||||||
|
|
||||||
const response = NextResponse.json({
|
const response = NextResponse.json({
|
||||||
@@ -133,7 +152,7 @@ export async function PUT(request: NextRequest) {
|
|||||||
} catch {
|
} catch {
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{ error: "Unable to update profile." },
|
{ error: "Unable to update profile." },
|
||||||
{ status: 400 },
|
{ status: 500 },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
import {
|
import {
|
||||||
createAdminSessionToken,
|
createAdminSessionToken,
|
||||||
|
getAdminSessionSecret,
|
||||||
getSessionCookieOptions,
|
getSessionCookieOptions,
|
||||||
} from "@/core/auth/adminSession";
|
} from "@/core/auth/adminSession";
|
||||||
import {
|
import {
|
||||||
@@ -20,9 +21,7 @@ function validEmail(email: string) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export async function POST(request: NextRequest) {
|
export async function POST(request: NextRequest) {
|
||||||
const sessionSecret =
|
const sessionSecret = getAdminSessionSecret();
|
||||||
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
|
|
||||||
"datastack-admin-session-default-secret-32-chars";
|
|
||||||
if (!sessionSecret || sessionSecret.length < 32) {
|
if (!sessionSecret || sessionSecret.length < 32) {
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{ error: "Admin session configuration is unavailable." },
|
{ error: "Admin session configuration is unavailable." },
|
||||||
@@ -30,8 +29,21 @@ export async function POST(request: NextRequest) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let body: Record<string, unknown>;
|
||||||
|
try {
|
||||||
|
const parsed = await request.json();
|
||||||
|
body =
|
||||||
|
parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
||||||
|
? (parsed as Record<string, unknown>)
|
||||||
|
: {};
|
||||||
|
} catch {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: "Invalid setup request payload." },
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const body = await request.json();
|
|
||||||
const email =
|
const email =
|
||||||
typeof body.email === "string" ? body.email.trim().toLowerCase() : "";
|
typeof body.email === "string" ? body.email.trim().toLowerCase() : "";
|
||||||
const password = typeof body.password === "string" ? body.password : "";
|
const password = typeof body.password === "string" ? body.password : "";
|
||||||
@@ -74,6 +86,7 @@ export async function POST(request: NextRequest) {
|
|||||||
email,
|
email,
|
||||||
sessionSecret,
|
sessionSecret,
|
||||||
Date.now() + SESSION_SECONDS * 1000,
|
Date.now() + SESSION_SECONDS * 1000,
|
||||||
|
1,
|
||||||
);
|
);
|
||||||
const response = NextResponse.json({ success: true });
|
const response = NextResponse.json({ success: true });
|
||||||
response.cookies.set(
|
response.cookies.set(
|
||||||
|
|||||||
@@ -1,23 +1,54 @@
|
|||||||
|
import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
|
||||||
import { SDKManager } from "@/core/sdk/sdkManager";
|
import { SDKManager } from "@/core/sdk/sdkManager";
|
||||||
import { NextRequest, NextResponse } from "next/server";
|
import { NextRequest, NextResponse } from "next/server";
|
||||||
|
|
||||||
|
const SESSION_COOKIE = "datastack_admin_session";
|
||||||
|
|
||||||
export async function GET(
|
export async function GET(
|
||||||
_req: NextRequest,
|
req: NextRequest,
|
||||||
{ params }: { params: Promise<{ id: string }> },
|
{ params }: { params: Promise<{ id: string }> },
|
||||||
) {
|
) {
|
||||||
|
const auth = await verifyRequestAdminSession(
|
||||||
|
req.cookies.get(SESSION_COOKIE)?.value,
|
||||||
|
);
|
||||||
|
if (!auth.valid) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: "Authentication required." },
|
||||||
|
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const { id } = await params;
|
const { id } = await params;
|
||||||
const client = await SDKManager.getClientAsync(id);
|
const client = await SDKManager.getClientAsync(id);
|
||||||
if (!client) {
|
if (!client) {
|
||||||
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
||||||
}
|
}
|
||||||
const { secret, token, ...clientSummary } = client;
|
const { secret, token, ...clientSummary } = client;
|
||||||
return NextResponse.json({ client: clientSummary });
|
void secret;
|
||||||
|
const includeToken = req.nextUrl.searchParams.get("includeToken") === "true";
|
||||||
|
return NextResponse.json({
|
||||||
|
client: {
|
||||||
|
...clientSummary,
|
||||||
|
tokenPreview: token ? `${token.slice(0, 12)}...` : "",
|
||||||
|
...(includeToken ? { token } : {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function PATCH(
|
export async function PATCH(
|
||||||
req: NextRequest,
|
req: NextRequest,
|
||||||
{ params }: { params: Promise<{ id: string }> },
|
{ params }: { params: Promise<{ id: string }> },
|
||||||
) {
|
) {
|
||||||
|
const auth = await verifyRequestAdminSession(
|
||||||
|
req.cookies.get(SESSION_COOKIE)?.value,
|
||||||
|
);
|
||||||
|
if (!auth.valid) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: "Authentication required." },
|
||||||
|
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const { id } = await params;
|
const { id } = await params;
|
||||||
try {
|
try {
|
||||||
const body = await req.json();
|
const body = await req.json();
|
||||||
@@ -46,11 +77,22 @@ export async function PATCH(
|
|||||||
}
|
}
|
||||||
|
|
||||||
export async function DELETE(
|
export async function DELETE(
|
||||||
_req: NextRequest,
|
req: NextRequest,
|
||||||
{ params }: { params: Promise<{ id: string }> },
|
{ params }: { params: Promise<{ id: string }> },
|
||||||
) {
|
) {
|
||||||
|
const auth = await verifyRequestAdminSession(
|
||||||
|
req.cookies.get(SESSION_COOKIE)?.value,
|
||||||
|
);
|
||||||
|
if (!auth.valid) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: "Authentication required." },
|
||||||
|
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const { id } = await params;
|
const { id } = await params;
|
||||||
const deleted = await SDKManager.deleteClientAsync(id);
|
const purgeFiles = req.nextUrl.searchParams.get("purgeFiles") === "true";
|
||||||
|
const deleted = await SDKManager.deleteClientAsync(id, { purgeFiles });
|
||||||
if (!deleted) {
|
if (!deleted) {
|
||||||
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,12 +1,28 @@
|
|||||||
|
import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
|
||||||
import { SDKManager } from "@/core/sdk/sdkManager";
|
import { SDKManager } from "@/core/sdk/sdkManager";
|
||||||
import { NextRequest, NextResponse } from "next/server";
|
import { NextRequest, NextResponse } from "next/server";
|
||||||
|
|
||||||
export async function GET() {
|
const SESSION_COOKIE = "datastack_admin_session";
|
||||||
|
|
||||||
|
export async function GET(req: NextRequest) {
|
||||||
|
const auth = await verifyRequestAdminSession(
|
||||||
|
req.cookies.get(SESSION_COOKIE)?.value,
|
||||||
|
);
|
||||||
|
if (!auth.valid) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: "Authentication required." },
|
||||||
|
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const allClients = await SDKManager.listClientsAsync();
|
const allClients = await SDKManager.listClientsAsync();
|
||||||
const clients = allClients.map(({ secret, ...client }) => {
|
const clients = allClients.map(({ secret, token, ...client }) => {
|
||||||
void secret;
|
void secret;
|
||||||
return client;
|
return {
|
||||||
|
...client,
|
||||||
|
tokenPreview: token ? `${token.slice(0, 12)}...` : "",
|
||||||
|
};
|
||||||
});
|
});
|
||||||
return NextResponse.json({ clients, count: clients.length });
|
return NextResponse.json({ clients, count: clients.length });
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
@@ -17,8 +33,31 @@ export async function GET() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export async function POST(req: NextRequest) {
|
export async function POST(req: NextRequest) {
|
||||||
|
const auth = await verifyRequestAdminSession(
|
||||||
|
req.cookies.get(SESSION_COOKIE)?.value,
|
||||||
|
);
|
||||||
|
if (!auth.valid) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: "Authentication required." },
|
||||||
|
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let body: Record<string, unknown>;
|
||||||
|
try {
|
||||||
|
const parsed = await req.json();
|
||||||
|
body =
|
||||||
|
parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
||||||
|
? (parsed as Record<string, unknown>)
|
||||||
|
: {};
|
||||||
|
} catch {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: "Invalid request payload." },
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const body = await req.json();
|
|
||||||
const { clientId, name, rateLimit } = body;
|
const { clientId, name, rateLimit } = body;
|
||||||
|
|
||||||
if (!clientId || !name) {
|
if (!clientId || !name) {
|
||||||
|
|||||||
+35
-9
@@ -1,4 +1,6 @@
|
|||||||
|
import { SettingsManager } from "@/core/adminSettings/settingsManager";
|
||||||
import { DataBufferEngine } from "@/core/data/buffer";
|
import { DataBufferEngine } from "@/core/data/buffer";
|
||||||
|
import { getInternalIngestionHeaders } from "@/core/Security/security";
|
||||||
import { CSVStorageManager } from "@/core/Storage/csvStorage";
|
import { CSVStorageManager } from "@/core/Storage/csvStorage";
|
||||||
import { parse } from "csv-parse";
|
import { parse } from "csv-parse";
|
||||||
import { NextRequest, NextResponse } from "next/server";
|
import { NextRequest, NextResponse } from "next/server";
|
||||||
@@ -31,18 +33,25 @@ export async function GET(request: NextRequest) {
|
|||||||
let total = 0;
|
let total = 0;
|
||||||
|
|
||||||
const addIfMatching = (event: DataEvent) => {
|
const addIfMatching = (event: DataEvent) => {
|
||||||
const eventDate = event.timestamp.slice(0, 10);
|
const eventDate = CSVStorageManager.extractSafePartitionDate(event.timestamp);
|
||||||
|
const dedupeKey = `${event.clientId}:${event.eventId}`;
|
||||||
|
const matchesSearch =
|
||||||
|
!search ||
|
||||||
|
event.payload.toLowerCase().includes(search) ||
|
||||||
|
event.eventId.toLowerCase().includes(search) ||
|
||||||
|
event.clientId.toLowerCase().includes(search);
|
||||||
|
|
||||||
if (
|
if (
|
||||||
(clientId !== "all" && event.clientId !== clientId) ||
|
(clientId !== "all" && event.clientId !== clientId) ||
|
||||||
(from && eventDate < from) ||
|
(from && eventDate < from) ||
|
||||||
(to && eventDate > to) ||
|
(to && eventDate > to) ||
|
||||||
(search && !event.payload.toLowerCase().includes(search)) ||
|
!matchesSearch ||
|
||||||
eventIds.has(event.eventId)
|
eventIds.has(dedupeKey)
|
||||||
) {
|
) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
eventIds.add(event.eventId);
|
eventIds.add(dedupeKey);
|
||||||
total += 1;
|
total += 1;
|
||||||
|
|
||||||
let low = 0;
|
let low = 0;
|
||||||
@@ -109,7 +118,16 @@ export async function GET(request: NextRequest) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const event of DataBufferEngine.getRecentEvents(200)) {
|
const settings = await SettingsManager.getSettingsAsync();
|
||||||
|
const recentWindowLimit = Math.min(
|
||||||
|
2000,
|
||||||
|
Math.max(200, settings.batchFlushThreshold || 500),
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const event of await DataBufferEngine.getRecentEventsAsync(
|
||||||
|
recentWindowLimit,
|
||||||
|
clientId,
|
||||||
|
)) {
|
||||||
addIfMatching({ ...event, status: "buffered" });
|
addIfMatching({ ...event, status: "buffered" });
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -117,10 +135,18 @@ export async function GET(request: NextRequest) {
|
|||||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||||
).replace(/\/$/, "");
|
).replace(/\/$/, "");
|
||||||
try {
|
try {
|
||||||
const res = await fetch(`${ingestionBaseUrl}/internal/recent?limit=200`, {
|
const qs = new URLSearchParams({ limit: String(recentWindowLimit) });
|
||||||
cache: "no-store",
|
if (clientId && clientId !== "all") {
|
||||||
signal: AbortSignal.timeout(1000),
|
qs.set("clientId", clientId);
|
||||||
});
|
}
|
||||||
|
const res = await fetch(
|
||||||
|
`${ingestionBaseUrl}/internal/recent?${qs.toString()}`,
|
||||||
|
{
|
||||||
|
headers: getInternalIngestionHeaders(),
|
||||||
|
cache: "no-store",
|
||||||
|
signal: AbortSignal.timeout(1000),
|
||||||
|
},
|
||||||
|
);
|
||||||
if (res.ok) {
|
if (res.ok) {
|
||||||
const data = await res.json();
|
const data = await res.json();
|
||||||
if (Array.isArray(data.events)) {
|
if (Array.isArray(data.events)) {
|
||||||
|
|||||||
+75
-30
@@ -1,9 +1,12 @@
|
|||||||
import { SettingsManager } from '@/core/adminSettings/settingsManager';
|
import { SettingsManager } from '@/core/adminSettings/settingsManager';
|
||||||
import { verifyAdminSessionToken } from '@/core/auth/adminSession';
|
import { verifyRequestAdminSession } from '@/core/auth/adminUsers';
|
||||||
import { AuthService } from '@/core/auth/jwt';
|
import { AuthService } from '@/core/auth/jwt';
|
||||||
import { DataBufferEngine } from '@/core/data/buffer';
|
import { DataBufferEngine } from '@/core/data/buffer';
|
||||||
import { SDKManager } from '@/core/sdk/sdkManager';
|
import { SDKManager } from '@/core/sdk/sdkManager';
|
||||||
import { SecurityEngine } from '@/core/Security/security';
|
import {
|
||||||
|
SecurityEngine,
|
||||||
|
getInternalIngestionHeaders,
|
||||||
|
} from '@/core/Security/security';
|
||||||
import { CSVStorageManager, type DataEvent } from '@/core/Storage/csvStorage';
|
import { CSVStorageManager, type DataEvent } from '@/core/Storage/csvStorage';
|
||||||
import { parse } from 'csv-parse';
|
import { parse } from 'csv-parse';
|
||||||
import { NextRequest, NextResponse } from 'next/server';
|
import { NextRequest, NextResponse } from 'next/server';
|
||||||
@@ -15,14 +18,15 @@ async function getRecentStoredEvents(
|
|||||||
limit: number,
|
limit: number,
|
||||||
clientId?: string,
|
clientId?: string,
|
||||||
): Promise<DataEvent[]> {
|
): Promise<DataEvent[]> {
|
||||||
const files = CSVStorageManager.listAllFiles()
|
const allFiles = await CSVStorageManager.listAllFilesAsync();
|
||||||
|
const files = allFiles
|
||||||
.filter((f) => !clientId || clientId === 'all' || f.clientId === clientId)
|
.filter((f) => !clientId || clientId === 'all' || f.clientId === clientId)
|
||||||
.slice(0, 5);
|
.slice(0, 5);
|
||||||
|
|
||||||
const collected: DataEvent[] = [];
|
const collected: DataEvent[] = [];
|
||||||
|
|
||||||
for (const file of files) {
|
for (const file of files) {
|
||||||
const fileEvents: DataEvent[] = [];
|
const fileTail: DataEvent[] = [];
|
||||||
const parser = createReadStream(file.filePath).pipe(
|
const parser = createReadStream(file.filePath).pipe(
|
||||||
parse({ columns: true, skip_empty_lines: true, bom: true }),
|
parse({ columns: true, skip_empty_lines: true, bom: true }),
|
||||||
);
|
);
|
||||||
@@ -32,7 +36,10 @@ async function getRecentStoredEvents(
|
|||||||
Record<string, string>
|
Record<string, string>
|
||||||
>) {
|
>) {
|
||||||
if (!record.eventId) continue;
|
if (!record.eventId) continue;
|
||||||
fileEvents.push({
|
if (fileTail.length >= limit) {
|
||||||
|
fileTail.shift();
|
||||||
|
}
|
||||||
|
fileTail.push({
|
||||||
clientId: record.clientId || file.clientId,
|
clientId: record.clientId || file.clientId,
|
||||||
eventId: record.eventId,
|
eventId: record.eventId,
|
||||||
timestamp: record.timestamp || `${file.date}T00:00:00.000Z`,
|
timestamp: record.timestamp || `${file.date}T00:00:00.000Z`,
|
||||||
@@ -43,8 +50,8 @@ async function getRecentStoredEvents(
|
|||||||
parser.destroy();
|
parser.destroy();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (fileEvents.length > 0) {
|
if (fileTail.length > 0) {
|
||||||
collected.push(...fileEvents.slice(-limit));
|
collected.push(...fileTail);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -55,9 +62,10 @@ export async function GET(req: NextRequest) {
|
|||||||
try {
|
try {
|
||||||
const { searchParams } = new URL(req.url);
|
const { searchParams } = new URL(req.url);
|
||||||
const clientId = searchParams.get('clientId') || undefined;
|
const clientId = searchParams.get('clientId') || undefined;
|
||||||
const limit = Math.min(200, Math.max(1, Number(searchParams.get('limit')) || 50));
|
const limit = Math.min(2000, Math.max(1, Number(searchParams.get('limit')) || 50));
|
||||||
|
|
||||||
const bufferedEvents = DataBufferEngine.getRecentEvents(limit, clientId);
|
await SettingsManager.getSettingsAsync();
|
||||||
|
const bufferedEvents = await DataBufferEngine.getRecentEventsAsync(limit, clientId);
|
||||||
let socketBufferedEvents: DataEvent[] = [];
|
let socketBufferedEvents: DataEvent[] = [];
|
||||||
const ingestionBaseUrl = (
|
const ingestionBaseUrl = (
|
||||||
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
|
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
|
||||||
@@ -67,6 +75,7 @@ export async function GET(req: NextRequest) {
|
|||||||
const qs = new URLSearchParams({ limit: String(limit) });
|
const qs = new URLSearchParams({ limit: String(limit) });
|
||||||
if (clientId) qs.set('clientId', clientId);
|
if (clientId) qs.set('clientId', clientId);
|
||||||
const res = await fetch(`${ingestionBaseUrl}/internal/recent?${qs.toString()}`, {
|
const res = await fetch(`${ingestionBaseUrl}/internal/recent?${qs.toString()}`, {
|
||||||
|
headers: getInternalIngestionHeaders(),
|
||||||
cache: 'no-store',
|
cache: 'no-store',
|
||||||
signal: AbortSignal.timeout(1000),
|
signal: AbortSignal.timeout(1000),
|
||||||
});
|
});
|
||||||
@@ -86,8 +95,10 @@ export async function GET(req: NextRequest) {
|
|||||||
const merged: DataEvent[] = [];
|
const merged: DataEvent[] = [];
|
||||||
|
|
||||||
for (const evt of [...bufferedEvents, ...socketBufferedEvents, ...storedEvents]) {
|
for (const evt of [...bufferedEvents, ...socketBufferedEvents, ...storedEvents]) {
|
||||||
if (!evt.eventId || seenIds.has(evt.eventId)) continue;
|
if (!evt.eventId) continue;
|
||||||
seenIds.add(evt.eventId);
|
const dedupeKey = `${evt.clientId}:${evt.eventId}`;
|
||||||
|
if (seenIds.has(dedupeKey)) continue;
|
||||||
|
seenIds.add(dedupeKey);
|
||||||
merged.push(evt);
|
merged.push(evt);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -107,9 +118,6 @@ export async function POST(req: NextRequest) {
|
|||||||
|
|
||||||
// Verify either Client Bearer JWT or Admin Session Cookie
|
// Verify either Client Bearer JWT or Admin Session Cookie
|
||||||
const authHeader = req.headers.get('authorization');
|
const authHeader = req.headers.get('authorization');
|
||||||
const sessionSecret =
|
|
||||||
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
|
|
||||||
'datastack-admin-session-default-secret-32-chars';
|
|
||||||
const sessionCookie = req.cookies.get(SESSION_COOKIE)?.value;
|
const sessionCookie = req.cookies.get(SESSION_COOKIE)?.value;
|
||||||
|
|
||||||
let tokenClientId: string | null = null;
|
let tokenClientId: string | null = null;
|
||||||
@@ -128,20 +136,30 @@ export async function POST(req: NextRequest) {
|
|||||||
tokenClientId = verified.payload.clientId;
|
tokenClientId = verified.payload.clientId;
|
||||||
tokenRateLimit = verified.payload.rateLimit;
|
tokenRateLimit = verified.payload.rateLimit;
|
||||||
} else {
|
} else {
|
||||||
const isAdmin = Boolean(
|
const auth = await verifyRequestAdminSession(sessionCookie);
|
||||||
sessionSecret &&
|
if (!auth.valid) {
|
||||||
sessionCookie &&
|
|
||||||
(await verifyAdminSessionToken(sessionCookie, sessionSecret)),
|
|
||||||
);
|
|
||||||
if (!isAdmin) {
|
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{ error: 'Authentication required. Provide a valid Client Bearer JWT or Admin Session.' },
|
{
|
||||||
{ status: 401 },
|
error:
|
||||||
|
auth.reason === 'UNAUTHORIZED'
|
||||||
|
? 'Authentication required. Provide a valid Client Bearer JWT or Admin Session.'
|
||||||
|
: 'Admin authentication is temporarily unavailable.',
|
||||||
|
},
|
||||||
|
{ status: auth.reason === 'UNAUTHORIZED' ? 401 : 503 },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const body = await req.json();
|
let body: unknown;
|
||||||
|
try {
|
||||||
|
body = await req.json();
|
||||||
|
} catch {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: 'Invalid JSON request payload.', code: 'INVALID_JSON' },
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const bodyObj =
|
const bodyObj =
|
||||||
body && typeof body === 'object' && !Array.isArray(body)
|
body && typeof body === 'object' && !Array.isArray(body)
|
||||||
? (body as Record<string, unknown>)
|
? (body as Record<string, unknown>)
|
||||||
@@ -154,7 +172,27 @@ export async function POST(req: NextRequest) {
|
|||||||
.toLowerCase()
|
.toLowerCase()
|
||||||
.replace(/[^a-z0-9_-]/g, '_');
|
.replace(/[^a-z0-9_-]/g, '_');
|
||||||
|
|
||||||
|
if (await SDKManager.isClientDeletedAsync(clientId)) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
error: `CLIENT_REVOKED: Client '${clientId}' has been deleted by administrator.`,
|
||||||
|
code: 'CLIENT_REVOKED',
|
||||||
|
},
|
||||||
|
{ status: 403 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const registeredClient = await SDKManager.getClientAsync(clientId);
|
const registeredClient = await SDKManager.getClientAsync(clientId);
|
||||||
|
if (rawBearerToken && !registeredClient) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
error: `CLIENT_REVOKED: Client '${clientId}' registration not found or has been deleted.`,
|
||||||
|
code: 'CLIENT_REVOKED',
|
||||||
|
},
|
||||||
|
{ status: 403 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
if (registeredClient) {
|
if (registeredClient) {
|
||||||
if (registeredClient.status === 'revoked') {
|
if (registeredClient.status === 'revoked') {
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
@@ -196,6 +234,7 @@ export async function POST(req: NextRequest) {
|
|||||||
: null;
|
: null;
|
||||||
|
|
||||||
if (batchItems !== null) {
|
if (batchItems !== null) {
|
||||||
|
const envelopeKeys = new Set(['eventId', 'timestamp', 'data', 'clientId']);
|
||||||
let processed = 0;
|
let processed = 0;
|
||||||
let rejected = 0;
|
let rejected = 0;
|
||||||
let duplicates = 0;
|
let duplicates = 0;
|
||||||
@@ -205,6 +244,8 @@ export async function POST(req: NextRequest) {
|
|||||||
const oversizedEventIds: string[] = [];
|
const oversizedEventIds: string[] = [];
|
||||||
const errors: string[] = [];
|
const errors: string[] = [];
|
||||||
|
|
||||||
|
const batchEvents: DataEvent[] = [];
|
||||||
|
|
||||||
for (const item of batchItems) {
|
for (const item of batchItems) {
|
||||||
const safeItem =
|
const safeItem =
|
||||||
item && typeof item === 'object' && !Array.isArray(item)
|
item && typeof item === 'object' && !Array.isArray(item)
|
||||||
@@ -221,8 +262,10 @@ export async function POST(req: NextRequest) {
|
|||||||
? safeItem.timestamp.trim()
|
? safeItem.timestamp.trim()
|
||||||
: new Date().toISOString();
|
: new Date().toISOString();
|
||||||
|
|
||||||
const itemData =
|
const isEnvelope =
|
||||||
safeItem.data !== undefined ? safeItem.data : safeItem;
|
safeItem.data !== undefined &&
|
||||||
|
Object.keys(safeItem).every((k) => envelopeKeys.has(k));
|
||||||
|
const itemData = isEnvelope ? safeItem.data : safeItem;
|
||||||
|
|
||||||
const val = SecurityEngine.validateEvent(
|
const val = SecurityEngine.validateEvent(
|
||||||
clientId,
|
clientId,
|
||||||
@@ -251,18 +294,20 @@ export async function POST(req: NextRequest) {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
const event: DataEvent = {
|
batchEvents.push({
|
||||||
clientId,
|
clientId,
|
||||||
eventId: itemEventId,
|
eventId: itemEventId,
|
||||||
timestamp: itemTimestamp,
|
timestamp: itemTimestamp,
|
||||||
payload:
|
payload:
|
||||||
typeof itemData === 'string' ? itemData : JSON.stringify(itemData),
|
typeof itemData === 'string' ? itemData : JSON.stringify(itemData),
|
||||||
};
|
});
|
||||||
|
|
||||||
DataBufferEngine.push(event);
|
|
||||||
processed += 1;
|
processed += 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (batchEvents.length > 0) {
|
||||||
|
await DataBufferEngine.pushBatchAsync(batchEvents);
|
||||||
|
}
|
||||||
|
|
||||||
if (processed > 0) {
|
if (processed > 0) {
|
||||||
SDKManager.incrementEventCount(clientId, processed);
|
SDKManager.incrementEventCount(clientId, processed);
|
||||||
}
|
}
|
||||||
@@ -347,7 +392,7 @@ export async function POST(req: NextRequest) {
|
|||||||
payload: typeof data === 'string' ? data : JSON.stringify(data),
|
payload: typeof data === 'string' ? data : JSON.stringify(data),
|
||||||
};
|
};
|
||||||
|
|
||||||
DataBufferEngine.push(event);
|
await DataBufferEngine.pushAsync(event);
|
||||||
SDKManager.incrementEventCount(clientId, 1);
|
SDKManager.incrementEventCount(clientId, 1);
|
||||||
|
|
||||||
return NextResponse.json({ success: true, eventId, event });
|
return NextResponse.json({ success: true, eventId, event });
|
||||||
|
|||||||
+40
-14
@@ -1,4 +1,6 @@
|
|||||||
|
import { SettingsManager } from "@/core/adminSettings/settingsManager";
|
||||||
import { DataBufferEngine } from "@/core/data/buffer";
|
import { DataBufferEngine } from "@/core/data/buffer";
|
||||||
|
import { getInternalIngestionHeaders } from "@/core/Security/security";
|
||||||
import {
|
import {
|
||||||
CSVStorageManager,
|
CSVStorageManager,
|
||||||
type StorageFileMeta,
|
type StorageFileMeta,
|
||||||
@@ -6,6 +8,7 @@ import {
|
|||||||
import { parse } from "csv-parse";
|
import { parse } from "csv-parse";
|
||||||
import { NextRequest, NextResponse } from "next/server";
|
import { NextRequest, NextResponse } from "next/server";
|
||||||
import { createReadStream } from "node:fs";
|
import { createReadStream } from "node:fs";
|
||||||
|
import path from "node:path";
|
||||||
|
|
||||||
type ExportFormat = "csv" | "json";
|
type ExportFormat = "csv" | "json";
|
||||||
type CsvRecord = Record<string, string>;
|
type CsvRecord = Record<string, string>;
|
||||||
@@ -108,19 +111,26 @@ function streamFile(file: StorageFileMeta) {
|
|||||||
|
|
||||||
export async function GET(request: NextRequest) {
|
export async function GET(request: NextRequest) {
|
||||||
try {
|
try {
|
||||||
await DataBufferEngine.flush();
|
const { searchParams } = new URL(request.url);
|
||||||
const ingestionBaseUrl =
|
const shouldFlush = searchParams.get("flush") !== "false";
|
||||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000";
|
|
||||||
try {
|
await SettingsManager.getSettingsAsync();
|
||||||
await fetch(`${ingestionBaseUrl}/internal/flush`, {
|
if (shouldFlush) {
|
||||||
method: "POST",
|
await DataBufferEngine.flush();
|
||||||
signal: AbortSignal.timeout(1500),
|
const ingestionBaseUrl = (
|
||||||
});
|
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||||
} catch {
|
).replace(/\/$/, "");
|
||||||
// Port 4000 may not be running in isolated single-process dev mode
|
try {
|
||||||
|
await fetch(`${ingestionBaseUrl}/internal/flush`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: getInternalIngestionHeaders(),
|
||||||
|
signal: AbortSignal.timeout(1500),
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Port 4000 may not be running in isolated single-process dev mode
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const { searchParams } = new URL(request.url);
|
|
||||||
const clientId = searchParams.get("clientId") || "all";
|
const clientId = searchParams.get("clientId") || "all";
|
||||||
const filename = searchParams.get("filename");
|
const filename = searchParams.get("filename");
|
||||||
const from = searchParams.get("from") || "";
|
const from = searchParams.get("from") || "";
|
||||||
@@ -129,17 +139,33 @@ export async function GET(request: NextRequest) {
|
|||||||
searchParams.get("format") === "json" ? "json" : "csv";
|
searchParams.get("format") === "json" ? "json" : "csv";
|
||||||
const includeHeader = searchParams.get("includeHeader") !== "false";
|
const includeHeader = searchParams.get("includeHeader") !== "false";
|
||||||
|
|
||||||
let files = CSVStorageManager.listAllFiles();
|
let files = await CSVStorageManager.listAllFilesAsync();
|
||||||
if (clientId !== "all")
|
if (clientId !== "all")
|
||||||
files = files.filter((file) => file.clientId === clientId);
|
files = files.filter((file) => file.clientId === clientId);
|
||||||
|
|
||||||
if (filename) {
|
if (filename) {
|
||||||
const file = files.find((candidate) => candidate.filename === filename);
|
const safeFilename = path.basename(filename);
|
||||||
if (!file)
|
const matches = files.filter(
|
||||||
|
(candidate) => candidate.filename === safeFilename,
|
||||||
|
);
|
||||||
|
if (matches.length === 0) {
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{ error: "CSV file not found" },
|
{ error: "CSV file not found" },
|
||||||
{ status: 404 },
|
{ status: 404 },
|
||||||
);
|
);
|
||||||
|
}
|
||||||
|
if (matches.length > 1) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
error:
|
||||||
|
"Ambiguous filename matches multiple clients. Specify clientId.",
|
||||||
|
code: "AMBIGUOUS_FILENAME",
|
||||||
|
matchingClients: matches.map((m) => m.clientId),
|
||||||
|
},
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const file = matches[0];
|
||||||
return new NextResponse(streamFile(file), {
|
return new NextResponse(streamFile(file), {
|
||||||
status: 200,
|
status: 200,
|
||||||
headers: {
|
headers: {
|
||||||
|
|||||||
+13
-1
@@ -23,7 +23,19 @@ export async function GET(req: NextRequest) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const clientId = verified.payload.clientId;
|
const clientId = String(verified.payload.clientId || '')
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
|
if (SDKManager.isClientDeleted(clientId)) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
status: 'error',
|
||||||
|
code: 'CLIENT_REVOKED',
|
||||||
|
error: `CLIENT_REVOKED: Client '${clientId}' has been deleted by administrator.`,
|
||||||
|
},
|
||||||
|
{ status: 403 },
|
||||||
|
);
|
||||||
|
}
|
||||||
const registeredClient = await SDKManager.getClientAsync(clientId);
|
const registeredClient = await SDKManager.getClientAsync(clientId);
|
||||||
if (registeredClient) {
|
if (registeredClient) {
|
||||||
if (registeredClient.status === 'revoked') {
|
if (registeredClient.status === 'revoked') {
|
||||||
|
|||||||
+176
-4
@@ -1,8 +1,55 @@
|
|||||||
import { SettingsManager } from "@/core/adminSettings/settingsManager";
|
import {
|
||||||
|
SettingsManager,
|
||||||
|
type AdminSettings,
|
||||||
|
} from "@/core/adminSettings/settingsManager";
|
||||||
|
import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
|
||||||
import { NextRequest, NextResponse } from "next/server";
|
import { NextRequest, NextResponse } from "next/server";
|
||||||
|
|
||||||
export async function GET() {
|
const SESSION_COOKIE = "datastack_admin_session";
|
||||||
|
|
||||||
|
const ALLOWED_SETTING_KEYS = new Set([
|
||||||
|
"maxEventsPerSec",
|
||||||
|
"maxPayloadSizeKb",
|
||||||
|
"batchFlushThreshold",
|
||||||
|
"batchFlushIntervalMs",
|
||||||
|
"fileRotationMb",
|
||||||
|
"enableIdempotency",
|
||||||
|
"autoPruneDays",
|
||||||
|
"jwtSecret",
|
||||||
|
]);
|
||||||
|
|
||||||
|
function parseBoundedInt(
|
||||||
|
raw: unknown,
|
||||||
|
field: string,
|
||||||
|
min: number,
|
||||||
|
max: number,
|
||||||
|
): number {
|
||||||
|
if (typeof raw !== "number" || !Number.isInteger(raw)) {
|
||||||
|
throw new Error(`${field} must be an integer between ${min} and ${max}.`);
|
||||||
|
}
|
||||||
|
if (raw < min || raw > max) {
|
||||||
|
throw new Error(`${field} must be between ${min} and ${max}.`);
|
||||||
|
}
|
||||||
|
return raw;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function GET(req: NextRequest) {
|
||||||
try {
|
try {
|
||||||
|
const auth = await verifyRequestAdminSession(
|
||||||
|
req.cookies.get(SESSION_COOKIE)?.value,
|
||||||
|
);
|
||||||
|
if (!auth.valid) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
error:
|
||||||
|
auth.reason === "UNAUTHORIZED"
|
||||||
|
? "Authentication required."
|
||||||
|
: "Admin authentication is temporarily unavailable.",
|
||||||
|
},
|
||||||
|
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const settings = await SettingsManager.getPublicSettingsAsync();
|
const settings = await SettingsManager.getPublicSettingsAsync();
|
||||||
return NextResponse.json({ settings });
|
return NextResponse.json({ settings });
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
@@ -13,8 +60,133 @@ export async function GET() {
|
|||||||
|
|
||||||
export async function POST(req: NextRequest) {
|
export async function POST(req: NextRequest) {
|
||||||
try {
|
try {
|
||||||
const body = await req.json();
|
const auth = await verifyRequestAdminSession(
|
||||||
await SettingsManager.updateSettingsAsync(body);
|
req.cookies.get(SESSION_COOKIE)?.value,
|
||||||
|
);
|
||||||
|
if (!auth.valid) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
error:
|
||||||
|
auth.reason === "UNAUTHORIZED"
|
||||||
|
? "Authentication required."
|
||||||
|
: "Admin authentication is temporarily unavailable.",
|
||||||
|
},
|
||||||
|
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let rawBody: unknown;
|
||||||
|
try {
|
||||||
|
rawBody = await req.json();
|
||||||
|
} catch {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: "Invalid JSON request payload.", code: "INVALID_JSON" },
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!rawBody || typeof rawBody !== "object" || Array.isArray(rawBody)) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
error: "Settings payload must be a JSON object.",
|
||||||
|
code: "VALIDATION_ERROR",
|
||||||
|
},
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const body = rawBody as Record<string, unknown>;
|
||||||
|
const unknownKeys = Object.keys(body).filter(
|
||||||
|
(key) => !ALLOWED_SETTING_KEYS.has(key),
|
||||||
|
);
|
||||||
|
if (unknownKeys.length > 0) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
error: `Unsupported settings field(s): ${unknownKeys.join(", ")}`,
|
||||||
|
code: "VALIDATION_ERROR",
|
||||||
|
},
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const validated: Partial<AdminSettings> = {};
|
||||||
|
try {
|
||||||
|
if (body.maxEventsPerSec !== undefined) {
|
||||||
|
validated.maxEventsPerSec = parseBoundedInt(
|
||||||
|
body.maxEventsPerSec,
|
||||||
|
"maxEventsPerSec",
|
||||||
|
1,
|
||||||
|
100000,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (body.maxPayloadSizeKb !== undefined) {
|
||||||
|
validated.maxPayloadSizeKb = parseBoundedInt(
|
||||||
|
body.maxPayloadSizeKb,
|
||||||
|
"maxPayloadSizeKb",
|
||||||
|
1,
|
||||||
|
10240,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (body.batchFlushThreshold !== undefined) {
|
||||||
|
validated.batchFlushThreshold = parseBoundedInt(
|
||||||
|
body.batchFlushThreshold,
|
||||||
|
"batchFlushThreshold",
|
||||||
|
1,
|
||||||
|
50000,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (body.batchFlushIntervalMs !== undefined) {
|
||||||
|
validated.batchFlushIntervalMs = parseBoundedInt(
|
||||||
|
body.batchFlushIntervalMs,
|
||||||
|
"batchFlushIntervalMs",
|
||||||
|
100,
|
||||||
|
60000,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (body.fileRotationMb !== undefined) {
|
||||||
|
validated.fileRotationMb = parseBoundedInt(
|
||||||
|
body.fileRotationMb,
|
||||||
|
"fileRotationMb",
|
||||||
|
1,
|
||||||
|
2048,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (body.autoPruneDays !== undefined) {
|
||||||
|
validated.autoPruneDays = parseBoundedInt(
|
||||||
|
body.autoPruneDays,
|
||||||
|
"autoPruneDays",
|
||||||
|
0,
|
||||||
|
3650,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (body.enableIdempotency !== undefined) {
|
||||||
|
if (typeof body.enableIdempotency !== "boolean") {
|
||||||
|
throw new Error("enableIdempotency must be a boolean.");
|
||||||
|
}
|
||||||
|
validated.enableIdempotency = body.enableIdempotency;
|
||||||
|
}
|
||||||
|
if (body.jwtSecret !== undefined) {
|
||||||
|
if (typeof body.jwtSecret !== "string" || body.jwtSecret.trim().length < 32) {
|
||||||
|
throw new Error(
|
||||||
|
"jwtSecret must be a string with at least 32 characters.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
validated.jwtSecret = body.jwtSecret.trim();
|
||||||
|
}
|
||||||
|
} catch (validationErr: unknown) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
error:
|
||||||
|
validationErr instanceof Error
|
||||||
|
? validationErr.message
|
||||||
|
: "Invalid settings value.",
|
||||||
|
code: "VALIDATION_ERROR",
|
||||||
|
},
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
await SettingsManager.updateSettingsAsync(validated);
|
||||||
const settings = await SettingsManager.getPublicSettingsAsync();
|
const settings = await SettingsManager.getPublicSettingsAsync();
|
||||||
return NextResponse.json({
|
return NextResponse.json({
|
||||||
settings,
|
settings,
|
||||||
|
|||||||
+59
-11
@@ -1,4 +1,6 @@
|
|||||||
import { NextRequest, NextResponse } from 'next/server';
|
import { NextRequest, NextResponse } from 'next/server';
|
||||||
|
import { SettingsManager } from '@/core/adminSettings/settingsManager';
|
||||||
|
import { getInternalIngestionHeaders } from '@/core/Security/security';
|
||||||
import { CSVStorageManager } from '@/core/Storage/csvStorage';
|
import { CSVStorageManager } from '@/core/Storage/csvStorage';
|
||||||
import { DataBufferEngine } from '@/core/data/buffer';
|
import { DataBufferEngine } from '@/core/data/buffer';
|
||||||
|
|
||||||
@@ -7,8 +9,11 @@ export async function GET(req: NextRequest) {
|
|||||||
const { searchParams } = new URL(req.url);
|
const { searchParams } = new URL(req.url);
|
||||||
const clientId = searchParams.get('clientId') || undefined;
|
const clientId = searchParams.get('clientId') || undefined;
|
||||||
|
|
||||||
const stats = CSVStorageManager.getStorageStats();
|
const [stats, allFiles] = await Promise.all([
|
||||||
let files = CSVStorageManager.listAllFiles();
|
CSVStorageManager.getStorageStatsAsync(),
|
||||||
|
CSVStorageManager.listAllFilesAsync(),
|
||||||
|
]);
|
||||||
|
let files = allFiles;
|
||||||
|
|
||||||
if (clientId && clientId !== 'all') {
|
if (clientId && clientId !== 'all') {
|
||||||
files = files.filter((f) => f.clientId === clientId);
|
files = files.filter((f) => f.clientId === clientId);
|
||||||
@@ -23,12 +28,17 @@ export async function GET(req: NextRequest) {
|
|||||||
|
|
||||||
export async function POST(_req: NextRequest) {
|
export async function POST(_req: NextRequest) {
|
||||||
try {
|
try {
|
||||||
// 1. Flush local Next.js DataBufferEngine
|
await SettingsManager.getSettingsAsync();
|
||||||
|
|
||||||
|
// 1. Flush local / shared Next.js DataBufferEngine
|
||||||
const localRes = await DataBufferEngine.flush();
|
const localRes = await DataBufferEngine.flush();
|
||||||
|
|
||||||
// 2. Also flush Port 4000 Socket.IO ingestion buffer
|
// 2. Also flush Port 4000 Socket.IO ingestion buffer
|
||||||
let remoteCount = 0;
|
let remoteCount = 0;
|
||||||
let remoteFiles: string[] = [];
|
let remoteFiles: string[] = [];
|
||||||
|
let ingestionFlushed = false;
|
||||||
|
let warning: string | undefined;
|
||||||
|
const explicitIngestionUrl = Boolean(process.env.INGESTION_INTERNAL_URL?.trim());
|
||||||
const ingestionBaseUrl = (
|
const ingestionBaseUrl = (
|
||||||
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
|
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
|
||||||
).replace(/\/$/, '');
|
).replace(/\/$/, '');
|
||||||
@@ -36,17 +46,53 @@ export async function POST(_req: NextRequest) {
|
|||||||
try {
|
try {
|
||||||
const ingestRes = await fetch(`${ingestionBaseUrl}/internal/flush`, {
|
const ingestRes = await fetch(`${ingestionBaseUrl}/internal/flush`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
|
headers: getInternalIngestionHeaders(),
|
||||||
cache: 'no-store',
|
cache: 'no-store',
|
||||||
|
signal: AbortSignal.timeout(2500),
|
||||||
});
|
});
|
||||||
if (ingestRes.ok) {
|
if (!ingestRes.ok) {
|
||||||
const ingestData = await ingestRes.json();
|
const errPayload = await ingestRes.json().catch(() => ({}));
|
||||||
remoteCount = Number(ingestData.flushedCount) || 0;
|
const remoteErr =
|
||||||
if (Array.isArray(ingestData.filesUpdated)) {
|
typeof errPayload?.error === 'string'
|
||||||
remoteFiles = ingestData.filesUpdated;
|
? errPayload.error
|
||||||
}
|
: `Ingestion flush returned HTTP ${ingestRes.status}`;
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
success: false,
|
||||||
|
error: remoteErr,
|
||||||
|
code: 'INGESTION_FLUSH_FAILED',
|
||||||
|
flushedCount: localRes.count,
|
||||||
|
filesUpdated: localRes.files,
|
||||||
|
ingestionFlushed: false,
|
||||||
|
ingestionStatus: ingestRes.status,
|
||||||
|
},
|
||||||
|
{ status: 502 },
|
||||||
|
);
|
||||||
}
|
}
|
||||||
} catch {
|
|
||||||
// Port 4000 server may be offline in standalone Next.js dev mode
|
ingestionFlushed = true;
|
||||||
|
const ingestData = await ingestRes.json();
|
||||||
|
remoteCount = Number(ingestData.flushedCount) || 0;
|
||||||
|
if (Array.isArray(ingestData.filesUpdated)) {
|
||||||
|
remoteFiles = ingestData.filesUpdated;
|
||||||
|
}
|
||||||
|
} catch (err: unknown) {
|
||||||
|
const errMsg =
|
||||||
|
err instanceof Error ? err.message : 'Ingestion server unreachable';
|
||||||
|
if (explicitIngestionUrl) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
success: false,
|
||||||
|
error: `Ingestion server flush failed: ${errMsg}`,
|
||||||
|
code: 'INGESTION_UNREACHABLE',
|
||||||
|
flushedCount: localRes.count,
|
||||||
|
filesUpdated: localRes.files,
|
||||||
|
ingestionFlushed: false,
|
||||||
|
},
|
||||||
|
{ status: 502 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
warning = `Port 4000 ingestion server is unreachable (${errMsg}); flushed Next.js buffer only.`;
|
||||||
}
|
}
|
||||||
|
|
||||||
const combinedFiles = Array.from(
|
const combinedFiles = Array.from(
|
||||||
@@ -57,6 +103,8 @@ export async function POST(_req: NextRequest) {
|
|||||||
success: true,
|
success: true,
|
||||||
flushedCount: localRes.count + remoteCount,
|
flushedCount: localRes.count + remoteCount,
|
||||||
filesUpdated: combinedFiles,
|
filesUpdated: combinedFiles,
|
||||||
|
ingestionFlushed,
|
||||||
|
...(warning ? { warning } : {}),
|
||||||
});
|
});
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
const message = err instanceof Error ? err.message : 'Manual flush failed';
|
const message = err instanceof Error ? err.message : 'Manual flush failed';
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
'use client';
|
||||||
|
|
||||||
|
import { Button } from '@/components/ui/Button';
|
||||||
|
import { AlertTriangle, RotateCcw } from 'lucide-react';
|
||||||
|
import Link from 'next/link';
|
||||||
|
import { useEffect } from 'react';
|
||||||
|
|
||||||
|
export default function RootErrorBoundary({
|
||||||
|
error,
|
||||||
|
reset,
|
||||||
|
}: {
|
||||||
|
error: Error & { digest?: string };
|
||||||
|
reset: () => void;
|
||||||
|
}) {
|
||||||
|
useEffect(() => {
|
||||||
|
console.error('Application error boundary:', error);
|
||||||
|
}, [error]);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<main className="flex min-h-screen items-center justify-center bg-[var(--bg-main)] px-4 py-8 text-[var(--text)]">
|
||||||
|
<section className="w-full max-w-[440px] border border-[var(--border-strong)] bg-[var(--surface)] p-5">
|
||||||
|
<div className="mb-3 flex items-center gap-2 border-b border-[var(--border)] pb-3">
|
||||||
|
<AlertTriangle className="h-4 w-4 shrink-0 text-[var(--danger)]" aria-hidden="true" />
|
||||||
|
<h1 className="text-[13px] font-medium uppercase tracking-wider text-[var(--text)]">
|
||||||
|
Runtime error
|
||||||
|
</h1>
|
||||||
|
</div>
|
||||||
|
<p className="mb-3 text-[12px] leading-relaxed text-[var(--text-muted)]">
|
||||||
|
An unexpected rendering error occurred. You can retry the view or return to the overview page.
|
||||||
|
</p>
|
||||||
|
{error?.message && (
|
||||||
|
<div className="mb-4 border border-[var(--border)] bg-[var(--bg-main)] px-2.5 py-2 font-mono text-[11px] text-[var(--danger)] break-words">
|
||||||
|
{error.message}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
<div className="flex flex-wrap items-center justify-end gap-2 border-t border-[var(--border)] pt-3">
|
||||||
|
<Link href="/">
|
||||||
|
<Button type="button" size="xs" variant="ghost">
|
||||||
|
Return home
|
||||||
|
</Button>
|
||||||
|
</Link>
|
||||||
|
<Button type="button" size="xs" variant="primary" onClick={() => reset()}>
|
||||||
|
<RotateCcw className="h-3 w-3" aria-hidden="true" />
|
||||||
|
Try again
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
</main>
|
||||||
|
);
|
||||||
|
}
|
||||||
+25
-6
@@ -1,11 +1,31 @@
|
|||||||
|
import { PwaLifecycle } from '@/components/pwa/PwaLifecycle';
|
||||||
import { ToastProvider } from '@/components/ui/Toast';
|
import { ToastProvider } from '@/components/ui/Toast';
|
||||||
import type { Metadata } from 'next';
|
import type { Metadata, Viewport } from 'next';
|
||||||
import './globals.css';
|
import './globals.css';
|
||||||
|
|
||||||
|
export const viewport: Viewport = {
|
||||||
|
themeColor: '#0a0a0b',
|
||||||
|
width: 'device-width',
|
||||||
|
initialScale: 1,
|
||||||
|
};
|
||||||
|
|
||||||
export const metadata: Metadata = {
|
export const metadata: Metadata = {
|
||||||
title: 'DataStack - Realtime Data Collection Platform',
|
title: 'DataStack - Realtime Data Collection Platform',
|
||||||
description: 'Self-hosted universal realtime data collection engine with Socket.IO ingestion, Redis buffer, JWT client auth, CSV batch rotation, and /vcon/ admin panel.',
|
description: 'Self-hosted universal realtime data collection engine with Socket.IO ingestion, Redis buffer, JWT client auth, CSV batch rotation, and /vcon/ admin panel.',
|
||||||
manifest: '/manifest.json',
|
manifest: '/manifest.json',
|
||||||
|
appleWebApp: {
|
||||||
|
capable: true,
|
||||||
|
statusBarStyle: 'black-translucent',
|
||||||
|
title: 'DataStack',
|
||||||
|
},
|
||||||
|
icons: {
|
||||||
|
icon: [
|
||||||
|
{ url: '/icon.svg', type: 'image/svg+xml' },
|
||||||
|
{ url: '/icon-192.png', sizes: '192x192', type: 'image/png' },
|
||||||
|
{ url: '/icon-512.png', sizes: '512x512', type: 'image/png' },
|
||||||
|
],
|
||||||
|
apple: [{ url: '/apple-touch-icon.png', sizes: '180x180', type: 'image/png' }],
|
||||||
|
},
|
||||||
openGraph: {
|
openGraph: {
|
||||||
title: 'DataStack - Realtime Data Collection Platform',
|
title: 'DataStack - Realtime Data Collection Platform',
|
||||||
description: 'Self-hosted universal realtime data collection engine with Socket.IO ingestion, Redis buffer, JWT client auth, CSV batch rotation, and /vcon/ admin panel.',
|
description: 'Self-hosted universal realtime data collection engine with Socket.IO ingestion, Redis buffer, JWT client auth, CSV batch rotation, and /vcon/ admin panel.',
|
||||||
@@ -21,12 +41,11 @@ export const metadata: Metadata = {
|
|||||||
export default function RootLayout({ children }: { children: React.ReactNode }) {
|
export default function RootLayout({ children }: { children: React.ReactNode }) {
|
||||||
return (
|
return (
|
||||||
<html lang="en" className="dark">
|
<html lang="en" className="dark">
|
||||||
<head>
|
|
||||||
<link rel="manifest" href="/manifest.json" />
|
|
||||||
<meta name="theme-color" content="#0a0a0b" />
|
|
||||||
</head>
|
|
||||||
<body suppressHydrationWarning className="min-h-screen bg-[var(--bg-main)] text-[var(--text)] antialiased">
|
<body suppressHydrationWarning className="min-h-screen bg-[var(--bg-main)] text-[var(--text)] antialiased">
|
||||||
<ToastProvider>{children}</ToastProvider>
|
<ToastProvider>
|
||||||
|
{children}
|
||||||
|
<PwaLifecycle />
|
||||||
|
</ToastProvider>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
);
|
);
|
||||||
|
|||||||
+6
-5
@@ -1,5 +1,5 @@
|
|||||||
import { MainLayout } from '@/components/layout/MainLayout';
|
import { MainLayout } from '@/components/layout/MainLayout';
|
||||||
import { Button } from '@/components/ui/Button';
|
import { buttonClassNames } from '@/components/ui/Button';
|
||||||
import Link from 'next/link';
|
import Link from 'next/link';
|
||||||
|
|
||||||
export default function NotFound() {
|
export default function NotFound() {
|
||||||
@@ -11,10 +11,11 @@ export default function NotFound() {
|
|||||||
The requested page or route does not exist on DataStack.
|
The requested page or route does not exist on DataStack.
|
||||||
</p>
|
</p>
|
||||||
<div className="flex items-center gap-2 pt-2">
|
<div className="flex items-center gap-2 pt-2">
|
||||||
<Link href="/">
|
<Link href="/" className={buttonClassNames({ variant: 'primary', size: 'sm' })}>
|
||||||
<Button variant="secondary" size="sm">
|
Return Home
|
||||||
Return Home
|
</Link>
|
||||||
</Button>
|
<Link href="/vcon/dashboard" className={buttonClassNames({ variant: 'secondary', size: 'sm' })}>
|
||||||
|
Open Console
|
||||||
</Link>
|
</Link>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -23,7 +23,11 @@ export default function ClientsPage() {
|
|||||||
const [now, setNow] = useState(() => Date.now());
|
const [now, setNow] = useState(() => Date.now());
|
||||||
|
|
||||||
const fetchClients = useCallback(async () => {
|
const fetchClients = useCallback(async () => {
|
||||||
const response = await fetch('/api/clients');
|
const response = await fetch('/api/clients', { cache: 'no-store' });
|
||||||
|
if (response.status === 401) {
|
||||||
|
window.location.assign('/vcon/login');
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (!response.ok) throw new Error('Unable to load clients');
|
if (!response.ok) throw new Error('Unable to load clients');
|
||||||
const data = await response.json();
|
const data = await response.json();
|
||||||
setClients(setCached('vcon:clients', data.clients || []));
|
setClients(setCached('vcon:clients', data.clients || []));
|
||||||
@@ -49,16 +53,19 @@ export default function ClientsPage() {
|
|||||||
!query ||
|
!query ||
|
||||||
client.clientId.toLowerCase().includes(query) ||
|
client.clientId.toLowerCase().includes(query) ||
|
||||||
client.name.toLowerCase().includes(query);
|
client.name.toLowerCase().includes(query);
|
||||||
|
const revoked = client.status === 'revoked';
|
||||||
const online =
|
const online =
|
||||||
client.status === 'active' &&
|
client.status === 'active' &&
|
||||||
Boolean(client.lastActive) &&
|
Boolean(client.lastActive) &&
|
||||||
now - new Date(client.lastActive!).getTime() < 60_000;
|
now - new Date(client.lastActive!).getTime() < 60_000;
|
||||||
const pending = client.status === 'active' && !client.lastActive;
|
const pending = client.status === 'active' && !client.lastActive;
|
||||||
|
const offline = client.status === 'active' && !online && !pending;
|
||||||
const matchesStatus =
|
const matchesStatus =
|
||||||
statusFilter === 'all' ||
|
statusFilter === 'all' ||
|
||||||
(statusFilter === 'online' && online) ||
|
(statusFilter === 'online' && online) ||
|
||||||
(statusFilter === 'offline' && !online && !pending) ||
|
(statusFilter === 'offline' && offline) ||
|
||||||
(statusFilter === 'pending' && pending);
|
(statusFilter === 'pending' && pending) ||
|
||||||
|
(statusFilter === 'revoked' && revoked);
|
||||||
return matchesSearch && matchesStatus;
|
return matchesSearch && matchesStatus;
|
||||||
});
|
});
|
||||||
}, [clients, now, search, statusFilter]);
|
}, [clients, now, search, statusFilter]);
|
||||||
@@ -72,7 +79,10 @@ export default function ClientsPage() {
|
|||||||
const pendingCount = clients.filter(
|
const pendingCount = clients.filter(
|
||||||
(client) => client.status === 'active' && !client.lastActive
|
(client) => client.status === 'active' && !client.lastActive
|
||||||
).length;
|
).length;
|
||||||
const offlineCount = clients.length - onlineCount - pendingCount;
|
const revokedCount = clients.filter(
|
||||||
|
(client) => client.status === 'revoked'
|
||||||
|
).length;
|
||||||
|
const offlineCount = clients.length - onlineCount - pendingCount - revokedCount;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<PageShell
|
<PageShell
|
||||||
@@ -90,11 +100,13 @@ export default function ClientsPage() {
|
|||||||
}
|
}
|
||||||
>
|
>
|
||||||
<StatStrip
|
<StatStrip
|
||||||
|
className="md:grid-cols-5"
|
||||||
items={[
|
items={[
|
||||||
{ label: 'Total', value: clients.length },
|
{ label: 'Total', value: clients.length },
|
||||||
{ label: 'Online', value: onlineCount, dot: 'ok' },
|
{ label: 'Online', value: onlineCount, dot: 'ok' },
|
||||||
{ label: 'Offline', value: offlineCount, dot: 'off' },
|
{ label: 'Offline', value: offlineCount, dot: 'off' },
|
||||||
{ label: 'Pending', value: pendingCount, dot: 'pending' },
|
{ label: 'Pending', value: pendingCount, dot: 'pending' },
|
||||||
|
{ label: 'Revoked', value: revokedCount, dot: 'off' },
|
||||||
]}
|
]}
|
||||||
/>
|
/>
|
||||||
|
|
||||||
@@ -109,6 +121,7 @@ export default function ClientsPage() {
|
|||||||
online: onlineCount,
|
online: onlineCount,
|
||||||
offline: offlineCount,
|
offline: offlineCount,
|
||||||
pending: pendingCount,
|
pending: pendingCount,
|
||||||
|
revoked: revokedCount,
|
||||||
}}
|
}}
|
||||||
/>
|
/>
|
||||||
<ClientsTable
|
<ClientsTable
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
'use client';
|
||||||
|
|
||||||
|
import { Button } from '@/components/ui/Button';
|
||||||
|
import { AlertTriangle, RotateCcw } from 'lucide-react';
|
||||||
|
import Link from 'next/link';
|
||||||
|
import { useEffect } from 'react';
|
||||||
|
|
||||||
|
export default function VconErrorBoundary({
|
||||||
|
error,
|
||||||
|
reset,
|
||||||
|
}: {
|
||||||
|
error: Error & { digest?: string };
|
||||||
|
reset: () => void;
|
||||||
|
}) {
|
||||||
|
useEffect(() => {
|
||||||
|
console.error('Admin console error boundary:', error);
|
||||||
|
}, [error]);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="flex min-h-[60vh] items-center justify-center p-4 text-[var(--text)]">
|
||||||
|
<section className="w-full max-w-[440px] border border-[var(--border-strong)] bg-[var(--surface)] p-5">
|
||||||
|
<div className="mb-3 flex items-center gap-2 border-b border-[var(--border)] pb-3">
|
||||||
|
<AlertTriangle className="h-4 w-4 shrink-0 text-[var(--danger)]" aria-hidden="true" />
|
||||||
|
<h1 className="text-[13px] font-medium uppercase tracking-wider text-[var(--text)]">
|
||||||
|
Console panel error
|
||||||
|
</h1>
|
||||||
|
</div>
|
||||||
|
<p className="mb-3 text-[12px] leading-relaxed text-[var(--text-muted)]">
|
||||||
|
This console view encountered an unexpected failure. Retry rendering or navigate back to the dashboard.
|
||||||
|
</p>
|
||||||
|
{error?.message && (
|
||||||
|
<div className="mb-4 border border-[var(--border)] bg-[var(--bg-main)] px-2.5 py-2 font-mono text-[11px] text-[var(--danger)] break-words">
|
||||||
|
{error.message}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
<div className="flex flex-wrap items-center justify-end gap-2 border-t border-[var(--border)] pt-3">
|
||||||
|
<Link href="/vcon/dashboard">
|
||||||
|
<Button type="button" size="xs" variant="ghost">
|
||||||
|
Dashboard
|
||||||
|
</Button>
|
||||||
|
</Link>
|
||||||
|
<Button type="button" size="xs" variant="primary" onClick={() => reset()}>
|
||||||
|
<RotateCcw className="h-3 w-3" aria-hidden="true" />
|
||||||
|
Retry panel
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -6,12 +6,13 @@ export const runtime = 'nodejs';
|
|||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
export default async function LoginPage() {
|
export default async function LoginPage() {
|
||||||
if (process.env.DATABASE_URL) {
|
const isConfigured = await isAdminSetupComplete().catch(() => false);
|
||||||
const isConfigured = await isAdminSetupComplete().catch(() => false);
|
if (!isConfigured) redirect('/vcon/setup');
|
||||||
if (!isConfigured) redirect('/vcon/setup');
|
|
||||||
}
|
|
||||||
|
|
||||||
const showDemoHint = !process.env.DATABASE_URL && process.env.NODE_ENV !== 'production';
|
const showDemoHint =
|
||||||
|
!process.env.DATABASE_URL &&
|
||||||
|
process.env.NODE_ENV !== 'production' &&
|
||||||
|
process.env.DATASTACK_ALLOW_DEMO_ADMIN !== 'false';
|
||||||
|
|
||||||
return <AdminLogin showDemoHint={showDemoHint} />;
|
return <AdminLogin showDemoHint={showDemoHint} />;
|
||||||
}
|
}
|
||||||
+96
-21
@@ -11,15 +11,25 @@ import { useToast } from '@/components/ui/Toast';
|
|||||||
import { getCached, setCached } from '@/lib/vconCache';
|
import { getCached, setCached } from '@/lib/vconCache';
|
||||||
import { useEffect, useState } from 'react';
|
import { useEffect, useState } from 'react';
|
||||||
|
|
||||||
|
interface SdkClientItem {
|
||||||
|
clientId: string;
|
||||||
|
name: string;
|
||||||
|
status?: 'active' | 'revoked';
|
||||||
|
tokenPreview?: string;
|
||||||
|
token?: string;
|
||||||
|
}
|
||||||
|
|
||||||
export default function SdkGuidePage() {
|
export default function SdkGuidePage() {
|
||||||
const { success } = useToast();
|
const { success, error } = useToast();
|
||||||
const [clients, setClients] = useState<any[]>(() =>
|
const [clients, setClients] = useState<SdkClientItem[]>(() =>
|
||||||
getCached<any[]>('vcon:clients', [])
|
getCached<SdkClientItem[]>('vcon:clients', [])
|
||||||
);
|
);
|
||||||
const [selectedClientId, setSelectedClientId] = useState<string>(() => {
|
const [selectedClientId, setSelectedClientId] = useState<string>(() => {
|
||||||
const cached = getCached<any[]>('vcon:clients', []);
|
const cached = getCached<SdkClientItem[]>('vcon:clients', []);
|
||||||
return cached[0]?.clientId ?? '';
|
return cached.find((c) => c.status !== 'revoked')?.clientId ?? cached[0]?.clientId ?? '';
|
||||||
});
|
});
|
||||||
|
const [selectedClientToken, setSelectedClientToken] = useState<string>('');
|
||||||
|
const [loadingToken, setLoadingToken] = useState<boolean>(false);
|
||||||
const [serverUrl, setServerUrl] = useState<string>('http://localhost:4000');
|
const [serverUrl, setServerUrl] = useState<string>('http://localhost:4000');
|
||||||
const [apiUrl, setApiUrl] = useState<string>('http://localhost:3000');
|
const [apiUrl, setApiUrl] = useState<string>('http://localhost:3000');
|
||||||
|
|
||||||
@@ -34,33 +44,81 @@ export default function SdkGuidePage() {
|
|||||||
|
|
||||||
async function loadClients() {
|
async function loadClients() {
|
||||||
try {
|
try {
|
||||||
const res = await fetch('/api/clients');
|
const res = await fetch('/api/clients', { cache: 'no-store' });
|
||||||
|
if (res.status === 401) {
|
||||||
|
window.location.assign('/vcon/login');
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (res.ok) {
|
if (res.ok) {
|
||||||
const data = await res.json();
|
const data = await res.json();
|
||||||
const list = setCached('vcon:clients', data.clients || []);
|
const list = setCached<SdkClientItem[]>('vcon:clients', data.clients || []);
|
||||||
setClients(list);
|
setClients(list);
|
||||||
if (list.length > 0) {
|
if (list.length > 0) {
|
||||||
setSelectedClientId((prev) =>
|
setSelectedClientId((prev) =>
|
||||||
list.some((c: any) => c.clientId === prev) ? prev : list[0].clientId
|
list.some((c) => c.clientId === prev)
|
||||||
|
? prev
|
||||||
|
: list.find((c) => c.status !== 'revoked')?.clientId ?? list[0].clientId
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
// ignore
|
// Retain cached list on transient network error
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
loadClients();
|
void loadClients();
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!selectedClientId) {
|
||||||
|
setSelectedClientToken('');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let cancelled = false;
|
||||||
|
setLoadingToken(true);
|
||||||
|
|
||||||
|
async function fetchToken() {
|
||||||
|
try {
|
||||||
|
const res = await fetch(
|
||||||
|
`/api/clients/${encodeURIComponent(selectedClientId)}?includeToken=true`,
|
||||||
|
{ cache: 'no-store' }
|
||||||
|
);
|
||||||
|
if (res.status === 401) {
|
||||||
|
window.location.assign('/vcon/login');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (res.ok) {
|
||||||
|
const data = await res.json();
|
||||||
|
if (!cancelled) {
|
||||||
|
setSelectedClientToken(String(data?.client?.token || ''));
|
||||||
|
}
|
||||||
|
} else if (!cancelled) {
|
||||||
|
setSelectedClientToken('');
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
if (!cancelled) setSelectedClientToken('');
|
||||||
|
} finally {
|
||||||
|
if (!cancelled) setLoadingToken(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void fetchToken();
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
};
|
||||||
|
}, [selectedClientId]);
|
||||||
|
|
||||||
const activeClient =
|
const activeClient =
|
||||||
clients.find((c) => c.clientId === selectedClientId) || clients[0];
|
clients.find((c) => c.clientId === selectedClientId) || clients[0];
|
||||||
const sampleToken = activeClient?.token || 'YOUR_SIGNED_CLIENT_JWT_TOKEN';
|
const resolvedToken = selectedClientToken || activeClient?.token || '';
|
||||||
|
const sampleToken = resolvedToken || 'YOUR_SIGNED_CLIENT_JWT_TOKEN';
|
||||||
|
|
||||||
const configSnippet = `# config.datastack.py — Place in your Python project root
|
const configSnippet = `# config.datastack.py — Place in your Python project root
|
||||||
DATASTACK_CONFIG = {
|
DATASTACK_CONFIG = {
|
||||||
"SERVER_URL": "${serverUrl}",
|
"SERVER_URL": "${serverUrl}",
|
||||||
"API_URL": "${apiUrl}",
|
"API_URL": "${apiUrl}",
|
||||||
"CLIENT_TOKEN": "${sampleToken}",
|
"CLIENT_TOKEN": "${sampleToken}",
|
||||||
|
"CLIENT_ID": "${activeClient?.clientId || ''}",
|
||||||
"AUTO_CONNECT": True,
|
"AUTO_CONNECT": True,
|
||||||
"ENABLE_HTTP_FALLBACK": True,
|
"ENABLE_HTTP_FALLBACK": True,
|
||||||
"HEARTBEAT_INTERVAL_SEC": 5.0,
|
"HEARTBEAT_INTERVAL_SEC": 5.0,
|
||||||
@@ -99,9 +157,21 @@ print("SDK Status:", client.status())
|
|||||||
|
|
||||||
client.disconnect()`;
|
client.disconnect()`;
|
||||||
|
|
||||||
const copySnippet = (text: string, label: string) => {
|
const copySnippet = async (text: string, label: string) => {
|
||||||
navigator.clipboard.writeText(text);
|
try {
|
||||||
success('Copied', `${label} copied`);
|
if (!navigator?.clipboard?.writeText) {
|
||||||
|
throw new Error('Clipboard API unavailable in this browser context');
|
||||||
|
}
|
||||||
|
await navigator.clipboard.writeText(text);
|
||||||
|
success('Copied', `${label} copied`);
|
||||||
|
} catch (copyErr) {
|
||||||
|
error(
|
||||||
|
'Copy failed',
|
||||||
|
copyErr instanceof Error
|
||||||
|
? copyErr.message
|
||||||
|
: 'Unable to copy to clipboard. Copy manually from the code block.'
|
||||||
|
);
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
return (
|
return (
|
||||||
@@ -109,12 +179,13 @@ client.disconnect()`;
|
|||||||
title="Python SDK"
|
title="Python SDK"
|
||||||
actions={
|
actions={
|
||||||
<>
|
<>
|
||||||
{activeClient?.token && (
|
{resolvedToken && (
|
||||||
<Button
|
<Button
|
||||||
size="xs"
|
size="xs"
|
||||||
variant="secondary"
|
variant="secondary"
|
||||||
icon="Key"
|
icon="Key"
|
||||||
onClick={() => copySnippet(activeClient.token, 'Client JWT token')}
|
disabled={loadingToken}
|
||||||
|
onClick={() => void copySnippet(resolvedToken, 'Client JWT token')}
|
||||||
>
|
>
|
||||||
Copy Token
|
Copy Token
|
||||||
</Button>
|
</Button>
|
||||||
@@ -123,7 +194,7 @@ client.disconnect()`;
|
|||||||
size="xs"
|
size="xs"
|
||||||
variant="secondary"
|
variant="secondary"
|
||||||
icon="FileCode"
|
icon="FileCode"
|
||||||
onClick={() => copySnippet(configSnippet, 'config.datastack.py')}
|
onClick={() => void copySnippet(configSnippet, 'config.datastack.py')}
|
||||||
>
|
>
|
||||||
Copy Config
|
Copy Config
|
||||||
</Button>
|
</Button>
|
||||||
@@ -131,7 +202,7 @@ client.disconnect()`;
|
|||||||
size="xs"
|
size="xs"
|
||||||
variant="secondary"
|
variant="secondary"
|
||||||
icon="Copy"
|
icon="Copy"
|
||||||
onClick={() => copySnippet(pythonSnippet, 'Python SDK snippet')}
|
onClick={() => void copySnippet(pythonSnippet, 'Python SDK snippet')}
|
||||||
>
|
>
|
||||||
Copy Code
|
Copy Code
|
||||||
</Button>
|
</Button>
|
||||||
@@ -161,8 +232,9 @@ client.disconnect()`;
|
|||||||
size="xs"
|
size="xs"
|
||||||
variant="ghost"
|
variant="ghost"
|
||||||
icon="Copy"
|
icon="Copy"
|
||||||
|
aria-label="Copy pip install command"
|
||||||
onClick={() =>
|
onClick={() =>
|
||||||
copySnippet('pip install ./sdk/python/datastack', 'Pip command')
|
void copySnippet('pip install ./sdk/python/datastack', 'Pip command')
|
||||||
}
|
}
|
||||||
/>
|
/>
|
||||||
</Card>
|
</Card>
|
||||||
@@ -176,6 +248,7 @@ client.disconnect()`;
|
|||||||
<div className="w-[190px]">
|
<div className="w-[190px]">
|
||||||
<Select
|
<Select
|
||||||
compact
|
compact
|
||||||
|
aria-label="Select SDK client"
|
||||||
value={selectedClientId}
|
value={selectedClientId}
|
||||||
onChange={(e) => setSelectedClientId(e.target.value)}
|
onChange={(e) => setSelectedClientId(e.target.value)}
|
||||||
>
|
>
|
||||||
@@ -191,7 +264,8 @@ client.disconnect()`;
|
|||||||
size="xs"
|
size="xs"
|
||||||
variant="ghost"
|
variant="ghost"
|
||||||
icon="Copy"
|
icon="Copy"
|
||||||
onClick={() => copySnippet(configSnippet, 'config.datastack.py')}
|
aria-label="Copy config.datastack.py"
|
||||||
|
onClick={() => void copySnippet(configSnippet, 'config.datastack.py')}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
}
|
}
|
||||||
@@ -207,7 +281,8 @@ client.disconnect()`;
|
|||||||
size="xs"
|
size="xs"
|
||||||
variant="ghost"
|
variant="ghost"
|
||||||
icon="Copy"
|
icon="Copy"
|
||||||
onClick={() => copySnippet(pythonSnippet, 'Python SDK snippet')}
|
aria-label="Copy Python SDK snippet"
|
||||||
|
onClick={() => void copySnippet(pythonSnippet, 'Python SDK snippet')}
|
||||||
/>
|
/>
|
||||||
}
|
}
|
||||||
>
|
>
|
||||||
|
|||||||
+310
-43
@@ -5,20 +5,40 @@ import { Button } from '@/components/ui/Button';
|
|||||||
import { CodeBlock } from '@/components/ui/CodeBlock';
|
import { CodeBlock } from '@/components/ui/CodeBlock';
|
||||||
import { Select, Textarea } from '@/components/ui/Input';
|
import { Select, Textarea } from '@/components/ui/Input';
|
||||||
import { Panel } from '@/components/ui/Panel';
|
import { Panel } from '@/components/ui/Panel';
|
||||||
|
import { Pill } from '@/components/ui/Pill';
|
||||||
import { StatStrip } from '@/components/ui/StatStrip';
|
import { StatStrip } from '@/components/ui/StatStrip';
|
||||||
import { useToast } from '@/components/ui/Toast';
|
import { useToast } from '@/components/ui/Toast';
|
||||||
import { getCached, setCached } from '@/lib/vconCache';
|
import { getCached, setCached } from '@/lib/vconCache';
|
||||||
|
import Link from 'next/link';
|
||||||
import { useEffect, useState } from 'react';
|
import { useEffect, useState } from 'react';
|
||||||
|
import { io } from 'socket.io-client';
|
||||||
|
|
||||||
|
interface SimulatorClient {
|
||||||
|
clientId: string;
|
||||||
|
name: string;
|
||||||
|
status: 'active' | 'revoked';
|
||||||
|
}
|
||||||
|
|
||||||
|
type TransportMode = 'http' | 'websocket';
|
||||||
|
type DispatchStatus = 'Ready' | 'Accepted' | 'Rejected' | 'Error';
|
||||||
|
|
||||||
|
function getSocketOrigin(): string {
|
||||||
|
const isLocalDev = window.location.port === '3000';
|
||||||
|
const protocol = window.location.protocol === 'https:' ? 'wss' : 'ws';
|
||||||
|
const host = isLocalDev ? `${window.location.hostname}:4000` : window.location.host;
|
||||||
|
return `${protocol}://${host}`;
|
||||||
|
}
|
||||||
|
|
||||||
export default function SimulatorPage() {
|
export default function SimulatorPage() {
|
||||||
const { success, error } = useToast();
|
const { success, warning, error } = useToast();
|
||||||
const [clients, setClients] = useState<any[]>(() =>
|
const [clients, setClients] = useState<SimulatorClient[]>(() =>
|
||||||
getCached<any[]>('vcon:clients', [])
|
getCached<SimulatorClient[]>('vcon:clients', [])
|
||||||
);
|
);
|
||||||
const [selectedClient, setSelectedClient] = useState<string>(() => {
|
const [selectedClient, setSelectedClient] = useState<string>(() => {
|
||||||
const cached = getCached<any[]>('vcon:clients', []);
|
const cached = getCached<SimulatorClient[]>('vcon:clients', []);
|
||||||
return cached[0]?.clientId ?? '';
|
return cached.find((c) => c.status === 'active')?.clientId ?? cached[0]?.clientId ?? '';
|
||||||
});
|
});
|
||||||
|
const [transport, setTransport] = useState<TransportMode>('http');
|
||||||
const [payloadText, setPayloadText] = useState<string>(
|
const [payloadText, setPayloadText] = useState<string>(
|
||||||
JSON.stringify(
|
JSON.stringify(
|
||||||
{
|
{
|
||||||
@@ -33,30 +53,136 @@ export default function SimulatorPage() {
|
|||||||
)
|
)
|
||||||
);
|
);
|
||||||
const [sending, setSending] = useState(false);
|
const [sending, setSending] = useState(false);
|
||||||
const [lastResponse, setLastResponse] = useState<any>(null);
|
const [dispatchStatus, setDispatchStatus] = useState<DispatchStatus>('Ready');
|
||||||
|
const [lastResponse, setLastResponse] = useState<Record<string, unknown> | null>(null);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
async function loadClients() {
|
async function loadClients() {
|
||||||
try {
|
try {
|
||||||
const res = await fetch('/api/clients');
|
const res = await fetch('/api/clients', { cache: 'no-store' });
|
||||||
|
if (res.status === 401) {
|
||||||
|
window.location.assign('/vcon/login');
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (res.ok) {
|
if (res.ok) {
|
||||||
const data = await res.json();
|
const data = await res.json();
|
||||||
const list = setCached('vcon:clients', data.clients || []);
|
const list = setCached<SimulatorClient[]>('vcon:clients', data.clients || []);
|
||||||
setClients(list);
|
setClients(list);
|
||||||
if (list.length > 0)
|
setSelectedClient((prev) => {
|
||||||
setSelectedClient((prev) => prev || list[0].clientId);
|
if (prev && list.some((c) => c.clientId === prev)) return prev;
|
||||||
|
return list.find((c) => c.status === 'active')?.clientId ?? list[0]?.clientId ?? '';
|
||||||
|
});
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
// ignore
|
// Ignore transient failure and retain cached list
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
loadClients();
|
void loadClients();
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
const fetchClientJwt = async (clientId: string): Promise<string> => {
|
||||||
|
const res = await fetch(`/api/clients/${encodeURIComponent(clientId)}?includeToken=true`, {
|
||||||
|
cache: 'no-store',
|
||||||
|
});
|
||||||
|
if (res.status === 401) {
|
||||||
|
window.location.assign('/vcon/login');
|
||||||
|
throw new Error('Authentication required');
|
||||||
|
}
|
||||||
|
const data = await res.json();
|
||||||
|
if (!res.ok || !data?.client?.token) {
|
||||||
|
throw new Error(data?.error || 'Unable to retrieve client JWT token');
|
||||||
|
}
|
||||||
|
return String(data.client.token);
|
||||||
|
};
|
||||||
|
|
||||||
|
const emitSocketEvents = async (
|
||||||
|
clientId: string,
|
||||||
|
items: Array<Record<string, unknown>>
|
||||||
|
): Promise<{ accepted: number; rejected: number; responses: Record<string, unknown>[] }> => {
|
||||||
|
const token = await fetchClientJwt(clientId);
|
||||||
|
const socketUrl = getSocketOrigin();
|
||||||
|
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const socket = io(socketUrl, {
|
||||||
|
auth: { token },
|
||||||
|
transports: ['websocket', 'polling'],
|
||||||
|
reconnection: false,
|
||||||
|
timeout: 4000,
|
||||||
|
});
|
||||||
|
|
||||||
|
const timer = window.setTimeout(() => {
|
||||||
|
socket.disconnect();
|
||||||
|
reject(new Error('WebSocket handshake timed out (is Port 4000 ingestion server running?)'));
|
||||||
|
}, 4500);
|
||||||
|
|
||||||
|
socket.on('connect_error', (connectErr) => {
|
||||||
|
window.clearTimeout(timer);
|
||||||
|
socket.disconnect();
|
||||||
|
reject(new Error(connectErr.message || 'WebSocket authentication failed'));
|
||||||
|
});
|
||||||
|
|
||||||
|
socket.on('connect', () => {
|
||||||
|
const responses: Record<string, unknown>[] = [];
|
||||||
|
let accepted = 0;
|
||||||
|
let rejected = 0;
|
||||||
|
let settled = 0;
|
||||||
|
|
||||||
|
const finishIfDone = () => {
|
||||||
|
if (settled >= items.length) {
|
||||||
|
window.clearTimeout(timer);
|
||||||
|
socket.disconnect();
|
||||||
|
resolve({ accepted, rejected, responses });
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const onAck = (ack: Record<string, unknown>) => {
|
||||||
|
responses.push(ack);
|
||||||
|
if (ack?.status === 'ok' || ack?.status === 'duplicate') {
|
||||||
|
accepted++;
|
||||||
|
} else {
|
||||||
|
rejected++;
|
||||||
|
}
|
||||||
|
settled++;
|
||||||
|
finishIfDone();
|
||||||
|
};
|
||||||
|
|
||||||
|
const onErrorEvent = (errPayload: Record<string, unknown>) => {
|
||||||
|
responses.push(errPayload);
|
||||||
|
rejected++;
|
||||||
|
settled++;
|
||||||
|
finishIfDone();
|
||||||
|
};
|
||||||
|
|
||||||
|
socket.on('ack', onAck);
|
||||||
|
socket.on('error_event', onErrorEvent);
|
||||||
|
|
||||||
|
items.forEach((data, index) => {
|
||||||
|
const eventId = `sim_ws_${Date.now()}_${index}`;
|
||||||
|
socket.emit('telemetry_event', {
|
||||||
|
clientId,
|
||||||
|
eventId,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
data,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
const handleSendSingle = async () => {
|
const handleSendSingle = async () => {
|
||||||
let parsedData: any;
|
if (!selectedClient) {
|
||||||
|
error('No target client', 'Register a client in /vcon/clients first.');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let parsedData: Record<string, unknown>;
|
||||||
try {
|
try {
|
||||||
parsedData = JSON.parse(payloadText);
|
const raw = JSON.parse(payloadText);
|
||||||
|
if (!raw || typeof raw !== 'object' || Array.isArray(raw)) {
|
||||||
|
error('Invalid payload', 'Payload must be a JSON object.');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
parsedData = raw as Record<string, unknown>;
|
||||||
} catch {
|
} catch {
|
||||||
error('Invalid JSON', 'Fix payload syntax');
|
error('Invalid JSON', 'Fix payload syntax');
|
||||||
return;
|
return;
|
||||||
@@ -64,54 +190,145 @@ export default function SimulatorPage() {
|
|||||||
|
|
||||||
setSending(true);
|
setSending(true);
|
||||||
try {
|
try {
|
||||||
|
if (transport === 'websocket') {
|
||||||
|
const result = await emitSocketEvents(selectedClient, [parsedData]);
|
||||||
|
const first = result.responses[0] || {};
|
||||||
|
setLastResponse({ transport: 'websocket', ...first });
|
||||||
|
if (result.accepted > 0) {
|
||||||
|
setDispatchStatus('Accepted');
|
||||||
|
success('WebSocket Event Accepted', `${String(first.eventId || selectedClient)} acknowledged`);
|
||||||
|
} else {
|
||||||
|
setDispatchStatus('Rejected');
|
||||||
|
error('WebSocket Event Rejected', String(first.message || first.error || 'Rejected by Port 4000'));
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const res = await fetch('/api/events', {
|
const res = await fetch('/api/events', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
clientId: selectedClient || 'py_telemetry_agent_01',
|
clientId: selectedClient,
|
||||||
data: parsedData,
|
data: parsedData,
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
|
if (res.status === 401) {
|
||||||
|
window.location.assign('/vcon/login');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const data = await res.json();
|
const data = await res.json();
|
||||||
setLastResponse(data);
|
setLastResponse({ transport: 'http', status: res.status, ...data });
|
||||||
|
|
||||||
if (res.ok) {
|
if (res.ok) {
|
||||||
success('Event Accepted', `${data.event.eventId} buffered`);
|
setDispatchStatus('Accepted');
|
||||||
|
success('Event Accepted', `${data.event?.eventId || selectedClient} buffered`);
|
||||||
} else {
|
} else {
|
||||||
|
setDispatchStatus('Rejected');
|
||||||
error('Event Rejected', data.error || 'Server rejected');
|
error('Event Rejected', data.error || 'Server rejected');
|
||||||
}
|
}
|
||||||
} catch (err: any) {
|
} catch (err: unknown) {
|
||||||
error('Error', err.message);
|
const message = err instanceof Error ? err.message : 'Dispatch error';
|
||||||
|
setDispatchStatus('Error');
|
||||||
|
setLastResponse({ transport, error: message });
|
||||||
|
error('Dispatch failed', message);
|
||||||
} finally {
|
} finally {
|
||||||
setSending(false);
|
setSending(false);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleSendBatch = async (count = 10) => {
|
const handleSendBatch = async (count = 10) => {
|
||||||
|
if (!selectedClient) {
|
||||||
|
error('No target client', 'Register a client in /vcon/clients first.');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const items = Array.from({ length: count }, (_, i) => ({
|
||||||
|
batch_index: i + 1,
|
||||||
|
sensor: `sensor_${(i % 3) + 1}`,
|
||||||
|
reading: +(Math.random() * 100).toFixed(2),
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
}));
|
||||||
|
|
||||||
setSending(true);
|
setSending(true);
|
||||||
try {
|
try {
|
||||||
|
if (transport === 'websocket') {
|
||||||
|
const result = await emitSocketEvents(selectedClient, items);
|
||||||
|
setLastResponse({
|
||||||
|
transport: 'websocket',
|
||||||
|
batchSent: count,
|
||||||
|
accepted: result.accepted,
|
||||||
|
rejected: result.rejected,
|
||||||
|
sample: result.responses.slice(0, 3),
|
||||||
|
});
|
||||||
|
if (result.accepted === 0) {
|
||||||
|
setDispatchStatus('Rejected');
|
||||||
|
error('WebSocket Batch Rejected', `0/${count} events ingested`);
|
||||||
|
} else if (result.accepted < count) {
|
||||||
|
setDispatchStatus('Rejected');
|
||||||
|
warning('WebSocket Batch Partial', `${result.accepted}/${count} ingested (${result.rejected} rejected)`);
|
||||||
|
} else {
|
||||||
|
setDispatchStatus('Accepted');
|
||||||
|
success('WebSocket Batch Streamed', `${result.accepted}/${count} ingested`);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
let accepted = 0;
|
let accepted = 0;
|
||||||
for (let i = 0; i < count; i++) {
|
let rejected = 0;
|
||||||
|
const errors: string[] = [];
|
||||||
|
|
||||||
|
for (const item of items) {
|
||||||
const res = await fetch('/api/events', {
|
const res = await fetch('/api/events', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
clientId: selectedClient || 'py_telemetry_agent_01',
|
clientId: selectedClient,
|
||||||
data: {
|
data: item,
|
||||||
batch_index: i + 1,
|
|
||||||
sensor: `sensor_${(i % 3) + 1}`,
|
|
||||||
reading: +(Math.random() * 100).toFixed(2),
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
},
|
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
if (res.ok) accepted++;
|
if (res.status === 401) {
|
||||||
|
window.location.assign('/vcon/login');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (res.ok) {
|
||||||
|
accepted++;
|
||||||
|
} else {
|
||||||
|
rejected++;
|
||||||
|
try {
|
||||||
|
const errJson = await res.json();
|
||||||
|
if (errJson?.error && errors.length < 3) {
|
||||||
|
errors.push(String(errJson.error));
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// ignore non-json error body
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
success('Batch Streamed', `${accepted}/${count} ingested`);
|
|
||||||
setLastResponse({ success: true, batchSent: count, accepted });
|
setLastResponse({
|
||||||
} catch (err: any) {
|
transport: 'http',
|
||||||
error('Batch Error', err.message);
|
batchSent: count,
|
||||||
|
accepted,
|
||||||
|
rejected,
|
||||||
|
...(errors.length > 0 ? { errors } : {}),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (accepted === 0) {
|
||||||
|
setDispatchStatus('Rejected');
|
||||||
|
error('Batch Rejected', `0/${count} ingested${errors[0] ? ` (${errors[0]})` : ''}`);
|
||||||
|
} else if (accepted < count) {
|
||||||
|
setDispatchStatus('Rejected');
|
||||||
|
warning('Batch Partial', `${accepted}/${count} ingested (${rejected} rejected)`);
|
||||||
|
} else {
|
||||||
|
setDispatchStatus('Accepted');
|
||||||
|
success('Batch Streamed', `${accepted}/${count} ingested`);
|
||||||
|
}
|
||||||
|
} catch (err: unknown) {
|
||||||
|
const message = err instanceof Error ? err.message : 'Batch dispatch error';
|
||||||
|
setDispatchStatus('Error');
|
||||||
|
setLastResponse({ transport, error: message });
|
||||||
|
error('Batch Error', message);
|
||||||
} finally {
|
} finally {
|
||||||
setSending(false);
|
setSending(false);
|
||||||
}
|
}
|
||||||
@@ -126,7 +343,8 @@ export default function SimulatorPage() {
|
|||||||
size="xs"
|
size="xs"
|
||||||
variant="secondary"
|
variant="secondary"
|
||||||
loading={sending}
|
loading={sending}
|
||||||
onClick={() => handleSendBatch(10)}
|
disabled={sending || !selectedClient}
|
||||||
|
onClick={() => void handleSendBatch(10)}
|
||||||
>
|
>
|
||||||
Burst 10
|
Burst 10
|
||||||
</Button>
|
</Button>
|
||||||
@@ -135,44 +353,93 @@ export default function SimulatorPage() {
|
|||||||
variant="primary"
|
variant="primary"
|
||||||
icon="Send"
|
icon="Send"
|
||||||
loading={sending}
|
loading={sending}
|
||||||
onClick={handleSendSingle}
|
disabled={sending || !selectedClient}
|
||||||
|
onClick={() => void handleSendSingle()}
|
||||||
>
|
>
|
||||||
Send Event
|
Send Event
|
||||||
</Button>
|
</Button>
|
||||||
</>
|
</>
|
||||||
}
|
}
|
||||||
>
|
>
|
||||||
|
{clients.length === 0 && (
|
||||||
|
<div
|
||||||
|
role="alert"
|
||||||
|
className="mb-3 flex flex-wrap items-center justify-between gap-2 border border-[var(--warning)] bg-[var(--warning-soft)] px-3 py-2 text-[11.5px] text-[var(--text)]"
|
||||||
|
>
|
||||||
|
<span>No registered SDK clients found. Register a client before dispatching events.</span>
|
||||||
|
<Link href="/vcon/clients" className="font-medium text-[var(--accent)] underline">
|
||||||
|
Go to Clients →
|
||||||
|
</Link>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
<StatStrip
|
<StatStrip
|
||||||
items={[
|
items={[
|
||||||
{
|
{
|
||||||
label: 'Target client',
|
label: 'Target client',
|
||||||
value: selectedClient || 'py_telemetry_agent_01',
|
value: selectedClient || 'None selected',
|
||||||
dot: 'ok',
|
dot: selectedClient ? 'ok' : 'off',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: 'Ingest transport',
|
||||||
|
value: transport === 'http' ? 'POST /api/events' : 'WS Port 4000',
|
||||||
},
|
},
|
||||||
{ label: 'Ingest route', value: 'POST /api/events' },
|
|
||||||
{ label: 'Payload format', value: 'JSON' },
|
{ label: 'Payload format', value: 'JSON' },
|
||||||
{
|
{
|
||||||
label: 'Last dispatch',
|
label: 'Last dispatch',
|
||||||
value: lastResponse ? 'Accepted' : 'Ready',
|
value: dispatchStatus,
|
||||||
|
dot:
|
||||||
|
dispatchStatus === 'Accepted'
|
||||||
|
? 'ok'
|
||||||
|
: dispatchStatus === 'Ready'
|
||||||
|
? undefined
|
||||||
|
: 'off',
|
||||||
},
|
},
|
||||||
]}
|
]}
|
||||||
/>
|
/>
|
||||||
|
|
||||||
<div className="grid grid-cols-1 gap-3 lg:grid-cols-2">
|
<div className="grid grid-cols-1 gap-3 lg:grid-cols-2">
|
||||||
<Panel title="Payload" bodyClassName="flex flex-col gap-2.5 p-3">
|
<Panel title="Payload" bodyClassName="flex flex-col gap-2.5 p-3">
|
||||||
|
<div className="flex flex-wrap items-center justify-between gap-2">
|
||||||
|
<span className="text-[10px] uppercase tracking-wider text-[var(--text-faint)]">
|
||||||
|
Transport
|
||||||
|
</span>
|
||||||
|
<div className="flex gap-1" role="group" aria-label="Ingest transport">
|
||||||
|
<Pill
|
||||||
|
active={transport === 'http'}
|
||||||
|
onClick={() => setTransport('http')}
|
||||||
|
>
|
||||||
|
HTTP /api/events
|
||||||
|
</Pill>
|
||||||
|
<Pill
|
||||||
|
active={transport === 'websocket'}
|
||||||
|
onClick={() => setTransport('websocket')}
|
||||||
|
>
|
||||||
|
WebSocket Port 4000
|
||||||
|
</Pill>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<Select
|
<Select
|
||||||
compact
|
compact
|
||||||
|
label="Target client"
|
||||||
value={selectedClient}
|
value={selectedClient}
|
||||||
|
disabled={clients.length === 0}
|
||||||
onChange={(e) => setSelectedClient(e.target.value)}
|
onChange={(e) => setSelectedClient(e.target.value)}
|
||||||
>
|
>
|
||||||
{clients.map((c) => (
|
{clients.length === 0 ? (
|
||||||
<option key={c.clientId} value={c.clientId}>
|
<option value="">No registered clients</option>
|
||||||
{c.clientId}
|
) : (
|
||||||
</option>
|
clients.map((c) => (
|
||||||
))}
|
<option key={c.clientId} value={c.clientId}>
|
||||||
|
{c.clientId} ({c.status})
|
||||||
|
</option>
|
||||||
|
))
|
||||||
|
)}
|
||||||
</Select>
|
</Select>
|
||||||
|
|
||||||
<Textarea
|
<Textarea
|
||||||
|
label="Event JSON"
|
||||||
rows={7}
|
rows={7}
|
||||||
value={payloadText}
|
value={payloadText}
|
||||||
onChange={(e) => setPayloadText(e.target.value)}
|
onChange={(e) => setPayloadText(e.target.value)}
|
||||||
|
|||||||
@@ -12,9 +12,9 @@ export const Footer: React.FC<FooterProps> = ({ isAdmin = false }) => {
|
|||||||
<footer className="shrink-0 min-h-9 bg-[var(--bg-activity)] border-t border-[var(--border)] px-4 sm:px-6 py-2 flex flex-wrap items-center justify-between gap-2 text-[11px] text-[var(--text-dim)] select-none">
|
<footer className="shrink-0 min-h-9 bg-[var(--bg-activity)] border-t border-[var(--border)] px-4 sm:px-6 py-2 flex flex-wrap items-center justify-between gap-2 text-[11px] text-[var(--text-dim)] select-none">
|
||||||
<span>DataStack</span>
|
<span>DataStack</span>
|
||||||
<nav aria-label="Footer navigation" className="flex items-center gap-5">
|
<nav aria-label="Footer navigation" className="flex items-center gap-5">
|
||||||
<Link href="#features" className="hover:text-[var(--text-muted)]">Features</Link>
|
<Link href="/#features" className="hover:text-[var(--text-muted)]">Features</Link>
|
||||||
<Link href="#sdk" className="hover:text-[var(--text-muted)]">SDK</Link>
|
<Link href="/#sdk" className="hover:text-[var(--text-muted)]">SDK</Link>
|
||||||
<Link href="#about" className="hover:text-[var(--text-muted)]">About</Link>
|
<Link href="/#about" className="hover:text-[var(--text-muted)]">About</Link>
|
||||||
</nav>
|
</nav>
|
||||||
</footer>
|
</footer>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
'use client';
|
'use client';
|
||||||
|
|
||||||
import { Button } from '@/components/ui/Button';
|
import { PWAInstallButton } from '@/components/pwa/PwaLifecycle';
|
||||||
|
import { Button, buttonClassNames } from '@/components/ui/Button';
|
||||||
import Link from 'next/link';
|
import Link from 'next/link';
|
||||||
import { usePathname } from 'next/navigation';
|
import { usePathname } from 'next/navigation';
|
||||||
import React, { useEffect, useState } from 'react';
|
import React, { useEffect, useState } from 'react';
|
||||||
@@ -95,12 +96,14 @@ export const Navbar: React.FC<NavbarProps> = ({
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
<nav className="hidden sm:flex items-center gap-5 text-[12px]">
|
<nav className="hidden sm:flex items-center gap-5 text-[12px]">
|
||||||
<Link href="#features" className="text-[var(--text-muted)] hover:text-[var(--text)] whitespace-nowrap shrink-0">Features</Link>
|
<Link href="/#features" className="text-[var(--text-muted)] hover:text-[var(--text)] whitespace-nowrap shrink-0">Features</Link>
|
||||||
<Link href="#sdk" className="text-[var(--text-muted)] hover:text-[var(--text)] whitespace-nowrap shrink-0">SDK</Link>
|
<Link href="/#sdk" className="text-[var(--text-muted)] hover:text-[var(--text)] whitespace-nowrap shrink-0">SDK</Link>
|
||||||
<Link href="#about" className="text-[var(--text-muted)] hover:text-[var(--text)] whitespace-nowrap shrink-0">About</Link>
|
<Link href="/#about" className="text-[var(--text-muted)] hover:text-[var(--text)] whitespace-nowrap shrink-0">About</Link>
|
||||||
</nav>
|
</nav>
|
||||||
|
|
||||||
<div className="flex items-center gap-2 shrink-0">
|
<div className="flex items-center gap-2 shrink-0">
|
||||||
|
<PWAInstallButton />
|
||||||
|
|
||||||
<Button
|
<Button
|
||||||
type="button"
|
type="button"
|
||||||
variant="icon"
|
variant="icon"
|
||||||
@@ -112,10 +115,11 @@ export const Navbar: React.FC<NavbarProps> = ({
|
|||||||
className="h-7 w-7 border border-[var(--border)] bg-[var(--bg-elevated)]"
|
className="h-7 w-7 border border-[var(--border)] bg-[var(--bg-elevated)]"
|
||||||
/>
|
/>
|
||||||
|
|
||||||
<Link href="/vcon/dashboard">
|
<Link
|
||||||
<Button size="xs" variant="primary">
|
href="/vcon/dashboard"
|
||||||
Console
|
className={buttonClassNames({ size: 'xs', variant: 'primary' })}
|
||||||
</Button>
|
>
|
||||||
|
Console
|
||||||
</Link>
|
</Link>
|
||||||
</div>
|
</div>
|
||||||
</header>
|
</header>
|
||||||
|
|||||||
@@ -0,0 +1,175 @@
|
|||||||
|
'use client';
|
||||||
|
|
||||||
|
import { Button } from '@/components/ui/Button';
|
||||||
|
import { Icon } from '@/components/ui/Icon';
|
||||||
|
import React, { useEffect, useState } from 'react';
|
||||||
|
|
||||||
|
interface BeforeInstallPromptEvent extends Event {
|
||||||
|
prompt: () => Promise<void>;
|
||||||
|
userChoice: Promise<{ outcome: 'accepted' | 'dismissed'; platform: string }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function usePWAInstall() {
|
||||||
|
const [deferredPrompt, setDeferredPrompt] = useState<BeforeInstallPromptEvent | null>(null);
|
||||||
|
const [isInstalled, setIsInstalled] = useState(false);
|
||||||
|
const [isIOS, setIsIOS] = useState(false);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const isStandalone =
|
||||||
|
window.matchMedia('(display-mode: standalone)').matches ||
|
||||||
|
(window.navigator as unknown as { standalone?: boolean }).standalone === true;
|
||||||
|
setIsInstalled(isStandalone);
|
||||||
|
|
||||||
|
const ua = window.navigator.userAgent.toLowerCase();
|
||||||
|
setIsIOS(/iphone|ipad|ipod/.test(ua));
|
||||||
|
|
||||||
|
const handleBeforeInstallPrompt = (e: Event) => {
|
||||||
|
e.preventDefault();
|
||||||
|
setDeferredPrompt(e as BeforeInstallPromptEvent);
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleAppInstalled = () => {
|
||||||
|
setIsInstalled(true);
|
||||||
|
setDeferredPrompt(null);
|
||||||
|
};
|
||||||
|
|
||||||
|
window.addEventListener('beforeinstallprompt', handleBeforeInstallPrompt);
|
||||||
|
window.addEventListener('appinstalled', handleAppInstalled);
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
window.removeEventListener('beforeinstallprompt', handleBeforeInstallPrompt);
|
||||||
|
window.removeEventListener('appinstalled', handleAppInstalled);
|
||||||
|
};
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const install = async () => {
|
||||||
|
if (!deferredPrompt) return false;
|
||||||
|
await deferredPrompt.prompt();
|
||||||
|
const { outcome } = await deferredPrompt.userChoice;
|
||||||
|
if (outcome === 'accepted') {
|
||||||
|
setIsInstalled(true);
|
||||||
|
setDeferredPrompt(null);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
|
||||||
|
return {
|
||||||
|
isInstallable: Boolean(deferredPrompt),
|
||||||
|
isInstalled,
|
||||||
|
isIOS,
|
||||||
|
install,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export const PWAInstallButton: React.FC = () => {
|
||||||
|
const { isInstallable, isInstalled, isIOS, install } = usePWAInstall();
|
||||||
|
const [showIOSGuide, setShowIOSGuide] = useState(false);
|
||||||
|
|
||||||
|
if (isInstalled) return null;
|
||||||
|
|
||||||
|
if (isInstallable) {
|
||||||
|
return (
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
size="xs"
|
||||||
|
variant="secondary"
|
||||||
|
icon="Download"
|
||||||
|
onClick={() => void install()}
|
||||||
|
>
|
||||||
|
Install
|
||||||
|
</Button>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isIOS) {
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
size="xs"
|
||||||
|
variant="secondary"
|
||||||
|
icon="Download"
|
||||||
|
onClick={() => setShowIOSGuide(true)}
|
||||||
|
>
|
||||||
|
Install
|
||||||
|
</Button>
|
||||||
|
|
||||||
|
{showIOSGuide && (
|
||||||
|
<div className="fixed inset-0 z-50 flex items-center justify-center bg-black/60 p-4">
|
||||||
|
<div className="w-full max-w-sm border border-[var(--border-strong)] bg-[var(--surface)] p-5">
|
||||||
|
<div className="mb-2 flex items-center justify-between gap-2">
|
||||||
|
<h3 className="text-[13px] font-medium text-[var(--text)]">
|
||||||
|
Install DataStack on iOS
|
||||||
|
</h3>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
aria-label="Close install instructions"
|
||||||
|
onClick={() => setShowIOSGuide(false)}
|
||||||
|
className="text-[var(--text-muted)] hover:text-[var(--text)]"
|
||||||
|
>
|
||||||
|
<Icon name="X" size={14} />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<p className="text-[12px] leading-relaxed text-[var(--text-muted)]">
|
||||||
|
1. Tap the <strong className="text-[var(--text)]">Share</strong> icon in Safari.
|
||||||
|
<br />
|
||||||
|
2. Select <strong className="text-[var(--text)]">Add to Home Screen</strong>.
|
||||||
|
</p>
|
||||||
|
<div className="mt-4 flex justify-end">
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
size="xs"
|
||||||
|
variant="primary"
|
||||||
|
onClick={() => setShowIOSGuide(false)}
|
||||||
|
>
|
||||||
|
Done
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const PwaLifecycle: React.FC = () => {
|
||||||
|
const [isOnline, setIsOnline] = useState(true);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
setIsOnline(typeof navigator !== 'undefined' ? navigator.onLine : true);
|
||||||
|
|
||||||
|
const onOnline = () => setIsOnline(true);
|
||||||
|
const onOffline = () => setIsOnline(false);
|
||||||
|
window.addEventListener('online', onOnline);
|
||||||
|
window.addEventListener('offline', onOffline);
|
||||||
|
|
||||||
|
if (typeof navigator !== 'undefined' && 'serviceWorker' in navigator) {
|
||||||
|
navigator.serviceWorker.register('/sw.js', { scope: '/' }).catch(() => {
|
||||||
|
// Non-fatal in environments without SW origin permissions
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
window.removeEventListener('online', onOnline);
|
||||||
|
window.removeEventListener('offline', onOffline);
|
||||||
|
};
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
if (isOnline) return null;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
role="status"
|
||||||
|
className="fixed bottom-4 left-4 z-50 flex items-center gap-2 border border-[var(--warning)] bg-[var(--surface)] px-3 py-1.5 text-[11px] font-mono text-[var(--text)]"
|
||||||
|
>
|
||||||
|
<Icon name="WifiOff" size={12} className="text-[var(--warning)]" />
|
||||||
|
<span>Offline mode — cached shell active</span>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
export default PwaLifecycle;
|
||||||
@@ -19,8 +19,8 @@ export function AdminLogin({
|
|||||||
}: {
|
}: {
|
||||||
showDemoHint?: boolean;
|
showDemoHint?: boolean;
|
||||||
}) {
|
}) {
|
||||||
const [email, setEmail] = useState(showDemoHint ? 'admin@datastack.dev' : '');
|
const [email, setEmail] = useState('');
|
||||||
const [password, setPassword] = useState(showDemoHint ? 'admin12345678' : '');
|
const [password, setPassword] = useState('');
|
||||||
const [remember, setRemember] = useState(true);
|
const [remember, setRemember] = useState(true);
|
||||||
const [showPassword, setShowPassword] = useState(false);
|
const [showPassword, setShowPassword] = useState(false);
|
||||||
const [isSubmitting, setIsSubmitting] = useState(false);
|
const [isSubmitting, setIsSubmitting] = useState(false);
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ const FILTERS: { key: ClientStatusFilter; label: string }[] = [
|
|||||||
{ key: 'online', label: 'Online' },
|
{ key: 'online', label: 'Online' },
|
||||||
{ key: 'offline', label: 'Offline' },
|
{ key: 'offline', label: 'Offline' },
|
||||||
{ key: 'pending', label: 'Pending' },
|
{ key: 'pending', label: 'Pending' },
|
||||||
|
{ key: 'revoked', label: 'Revoked' },
|
||||||
];
|
];
|
||||||
|
|
||||||
export function ClientsFilters({
|
export function ClientsFilters({
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
import { Button } from '@/components/ui/Button';
|
import { Button } from '@/components/ui/Button';
|
||||||
import { CopyButton } from '@/components/ui/CopyButton';
|
import { CopyButton } from '@/components/ui/CopyButton';
|
||||||
import { Textarea } from '@/components/ui/Input';
|
import { Input, Textarea } from '@/components/ui/Input';
|
||||||
import { Modal } from '@/components/ui/Modal';
|
import { Modal } from '@/components/ui/Modal';
|
||||||
import {
|
import {
|
||||||
Table,
|
Table,
|
||||||
@@ -43,8 +43,61 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
|||||||
type: 'rotate' | 'revoke' | 'delete';
|
type: 'rotate' | 'revoke' | 'delete';
|
||||||
client: ClientItem;
|
client: ClientItem;
|
||||||
} | null>(null);
|
} | null>(null);
|
||||||
|
const [editClient, setEditClient] = useState<ClientItem | null>(null);
|
||||||
|
const [editName, setEditName] = useState('');
|
||||||
|
const [editRateLimit, setEditRateLimit] = useState('100');
|
||||||
|
const [editError, setEditError] = useState('');
|
||||||
const [processing, setProcessing] = useState(false);
|
const [processing, setProcessing] = useState(false);
|
||||||
|
|
||||||
|
const openEditModal = (client: ClientItem) => {
|
||||||
|
setEditClient(client);
|
||||||
|
setEditName(client.name);
|
||||||
|
setEditRateLimit(String(client.rateLimit || 100));
|
||||||
|
setEditError('');
|
||||||
|
};
|
||||||
|
|
||||||
|
const executeEditClient = async (event: React.FormEvent<HTMLFormElement>) => {
|
||||||
|
event.preventDefault();
|
||||||
|
if (!editClient) return;
|
||||||
|
|
||||||
|
const trimmedName = editName.trim();
|
||||||
|
const numericRateLimit = Number(editRateLimit);
|
||||||
|
if (!trimmedName) {
|
||||||
|
setEditError('Display name is required.');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!Number.isFinite(numericRateLimit) || numericRateLimit < 1 || numericRateLimit > 10000) {
|
||||||
|
setEditError('Rate limit must be between 1 and 10,000 events/s.');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
setEditError('');
|
||||||
|
setProcessing(true);
|
||||||
|
try {
|
||||||
|
const response = await fetch(`/api/clients/${encodeURIComponent(editClient.clientId)}`, {
|
||||||
|
method: 'PATCH',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({
|
||||||
|
name: trimmedName,
|
||||||
|
rateLimit: Math.floor(numericRateLimit),
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
if (response.status === 401) {
|
||||||
|
window.location.assign('/vcon/login');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const data = await response.json();
|
||||||
|
if (!response.ok) throw new Error(data.error || 'Unable to update client');
|
||||||
|
success('Client updated', `${editClient.clientId} settings saved.`);
|
||||||
|
setEditClient(null);
|
||||||
|
await onStatusChange();
|
||||||
|
} catch (updateError) {
|
||||||
|
error('Client update failed', updateError instanceof Error ? updateError.message : 'Unknown error');
|
||||||
|
} finally {
|
||||||
|
setProcessing(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const executeRotateKey = async (client: ClientItem) => {
|
const executeRotateKey = async (client: ClientItem) => {
|
||||||
setProcessing(true);
|
setProcessing(true);
|
||||||
try {
|
try {
|
||||||
@@ -53,6 +106,10 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
|||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
body: JSON.stringify({ action: 'regenerate_token' }),
|
body: JSON.stringify({ action: 'regenerate_token' }),
|
||||||
});
|
});
|
||||||
|
if (response.status === 401) {
|
||||||
|
window.location.assign('/vcon/login');
|
||||||
|
return;
|
||||||
|
}
|
||||||
const data = await response.json();
|
const data = await response.json();
|
||||||
if (!response.ok) throw new Error(data.error || 'Unable to rotate credentials');
|
if (!response.ok) throw new Error(data.error || 'Unable to rotate credentials');
|
||||||
setConfirmAction(null);
|
setConfirmAction(null);
|
||||||
@@ -74,6 +131,10 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
|||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
body: JSON.stringify({ status }),
|
body: JSON.stringify({ status }),
|
||||||
});
|
});
|
||||||
|
if (response.status === 401) {
|
||||||
|
window.location.assign('/vcon/login');
|
||||||
|
return;
|
||||||
|
}
|
||||||
const data = await response.json();
|
const data = await response.json();
|
||||||
if (!response.ok) throw new Error(data.error || 'Unable to update client status');
|
if (!response.ok) throw new Error(data.error || 'Unable to update client status');
|
||||||
setConfirmAction(null);
|
setConfirmAction(null);
|
||||||
@@ -92,6 +153,10 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
|||||||
const response = await fetch(`/api/clients/${encodeURIComponent(client.clientId)}`, {
|
const response = await fetch(`/api/clients/${encodeURIComponent(client.clientId)}`, {
|
||||||
method: 'DELETE',
|
method: 'DELETE',
|
||||||
});
|
});
|
||||||
|
if (response.status === 401) {
|
||||||
|
window.location.assign('/vcon/login');
|
||||||
|
return;
|
||||||
|
}
|
||||||
const data = await response.json();
|
const data = await response.json();
|
||||||
if (!response.ok) throw new Error(data.error || 'Unable to delete client');
|
if (!response.ok) throw new Error(data.error || 'Unable to delete client');
|
||||||
setConfirmAction(null);
|
setConfirmAction(null);
|
||||||
@@ -130,7 +195,7 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
|||||||
<TableHeader>
|
<TableHeader>
|
||||||
<TableRow>
|
<TableRow>
|
||||||
<TableHead>Client</TableHead>
|
<TableHead>Client</TableHead>
|
||||||
<TableHead>API key</TableHead>
|
<TableHead>Credentials</TableHead>
|
||||||
<TableHead className="text-right">Events</TableHead>
|
<TableHead className="text-right">Events</TableHead>
|
||||||
<TableHead>Last seen</TableHead>
|
<TableHead>Last seen</TableHead>
|
||||||
<TableHead className="text-right">Actions</TableHead>
|
<TableHead className="text-right">Actions</TableHead>
|
||||||
@@ -141,15 +206,18 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
|||||||
<TableEmpty colSpan={5} message="No matching clients" />
|
<TableEmpty colSpan={5} message="No matching clients" />
|
||||||
) : (
|
) : (
|
||||||
clients.map((client) => {
|
clients.map((client) => {
|
||||||
|
const revoked = client.status === 'revoked';
|
||||||
const online = client.status === 'active' && Boolean(client.lastActive) &&
|
const online = client.status === 'active' && Boolean(client.lastActive) &&
|
||||||
now - new Date(client.lastActive!).getTime() < 60_000;
|
now - new Date(client.lastActive!).getTime() < 60_000;
|
||||||
const pending = client.status === 'active' && !client.lastActive;
|
const pending = client.status === 'active' && !client.lastActive;
|
||||||
const label = pending ? 'pending' : online ? 'online' : 'offline';
|
const label = revoked ? 'revoked' : pending ? 'pending' : online ? 'online' : 'offline';
|
||||||
const dotClass = online
|
const dotClass = revoked
|
||||||
? 'bg-[var(--accent)]'
|
? 'bg-[var(--danger)]'
|
||||||
: pending
|
: online
|
||||||
? 'bg-[var(--warning)]'
|
? 'bg-[var(--accent)]'
|
||||||
: 'bg-[var(--text-faint)]';
|
: pending
|
||||||
|
? 'bg-[var(--warning)]'
|
||||||
|
: 'bg-[var(--text-faint)]';
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<TableRow key={client.clientId}>
|
<TableRow key={client.clientId}>
|
||||||
@@ -157,13 +225,25 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
|||||||
<div className="flex min-w-0 items-center gap-2">
|
<div className="flex min-w-0 items-center gap-2">
|
||||||
<span className={`h-1.5 w-1.5 shrink-0 ${dotClass}`} title={label} />
|
<span className={`h-1.5 w-1.5 shrink-0 ${dotClass}`} title={label} />
|
||||||
<div className="min-w-0 leading-tight">
|
<div className="min-w-0 leading-tight">
|
||||||
<div className="truncate font-medium text-[var(--text)]">{client.name}</div>
|
<div className="flex items-center gap-1.5">
|
||||||
|
<span className="truncate font-medium text-[var(--text)]">{client.name}</span>
|
||||||
|
{revoked && (
|
||||||
|
<span className="border border-[var(--danger)] bg-[var(--danger-soft)] px-1 py-0 text-[9px] uppercase tracking-wider text-[var(--danger)]">
|
||||||
|
revoked
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
<div className="mt-0.5 truncate font-mono text-[10.5px] text-[var(--text-faint)]">{client.clientId}</div>
|
<div className="mt-0.5 truncate font-mono text-[10.5px] text-[var(--text-faint)]">{client.clientId}</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</TableCell>
|
</TableCell>
|
||||||
<TableCell className="font-mono">
|
<TableCell className="font-mono">
|
||||||
sec_••••••••
|
<div className="truncate text-[11px] text-[var(--text)]">
|
||||||
|
{client.tokenPreview || 'JWT Active'}
|
||||||
|
</div>
|
||||||
|
<div className="mt-0.5 text-[10px] text-[var(--text-faint)]">
|
||||||
|
{client.rateLimit}/s limit
|
||||||
|
</div>
|
||||||
</TableCell>
|
</TableCell>
|
||||||
<TableCell className="text-right font-mono tabular-nums">
|
<TableCell className="text-right font-mono tabular-nums">
|
||||||
{client.totalEvents.toLocaleString()}
|
{client.totalEvents.toLocaleString()}
|
||||||
@@ -184,6 +264,16 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
|||||||
aria-label={`View events for ${client.clientId}`}
|
aria-label={`View events for ${client.clientId}`}
|
||||||
onClick={() => router.push(`/vcon/data?clientId=${encodeURIComponent(client.clientId)}`)}
|
onClick={() => router.push(`/vcon/data?clientId=${encodeURIComponent(client.clientId)}`)}
|
||||||
/>
|
/>
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
variant="icon"
|
||||||
|
size="icon"
|
||||||
|
icon="Pencil"
|
||||||
|
className="h-6 w-6 border border-transparent hover:border-[var(--border)] hover:bg-[var(--surface-2)]"
|
||||||
|
title={`Edit ${client.clientId}`}
|
||||||
|
aria-label={`Edit ${client.clientId}`}
|
||||||
|
onClick={() => openEditModal(client)}
|
||||||
|
/>
|
||||||
<Button
|
<Button
|
||||||
type="button"
|
type="button"
|
||||||
variant="icon"
|
variant="icon"
|
||||||
@@ -224,6 +314,55 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
|||||||
</Table>
|
</Table>
|
||||||
</TableWrap>
|
</TableWrap>
|
||||||
|
|
||||||
|
<Modal
|
||||||
|
isOpen={Boolean(editClient)}
|
||||||
|
onClose={() => !processing && setEditClient(null)}
|
||||||
|
title={`Edit client: ${editClient?.clientId || ''}`}
|
||||||
|
maxWidth="xs"
|
||||||
|
footer={
|
||||||
|
<>
|
||||||
|
<Button type="button" variant="ghost" size="xs" disabled={processing} onClick={() => setEditClient(null)}>
|
||||||
|
Cancel
|
||||||
|
</Button>
|
||||||
|
<Button
|
||||||
|
type="submit"
|
||||||
|
form="edit-client-form"
|
||||||
|
variant="primary"
|
||||||
|
size="xs"
|
||||||
|
loading={processing}
|
||||||
|
>
|
||||||
|
Save changes
|
||||||
|
</Button>
|
||||||
|
</>
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<form id="edit-client-form" onSubmit={executeEditClient} className="flex flex-col gap-2.5">
|
||||||
|
<Input
|
||||||
|
label="Display name"
|
||||||
|
compact
|
||||||
|
value={editName}
|
||||||
|
onChange={(event) => setEditName(event.target.value)}
|
||||||
|
placeholder="Service display name"
|
||||||
|
required
|
||||||
|
/>
|
||||||
|
<Input
|
||||||
|
label="Rate limit (events/s)"
|
||||||
|
compact
|
||||||
|
type="number"
|
||||||
|
min="1"
|
||||||
|
max="10000"
|
||||||
|
value={editRateLimit}
|
||||||
|
onChange={(event) => setEditRateLimit(event.target.value)}
|
||||||
|
required
|
||||||
|
/>
|
||||||
|
{editError && (
|
||||||
|
<p role="alert" className="text-[11px] text-[var(--danger)]">
|
||||||
|
{editError}
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
</form>
|
||||||
|
</Modal>
|
||||||
|
|
||||||
<Modal
|
<Modal
|
||||||
isOpen={Boolean(confirmAction)}
|
isOpen={Boolean(confirmAction)}
|
||||||
onClose={() => !processing && setConfirmAction(null)}
|
onClose={() => !processing && setConfirmAction(null)}
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import {
|
|||||||
Table,
|
Table,
|
||||||
TableBody,
|
TableBody,
|
||||||
TableCell,
|
TableCell,
|
||||||
|
TableEmpty,
|
||||||
TableHead,
|
TableHead,
|
||||||
TableHeader,
|
TableHeader,
|
||||||
TableRow,
|
TableRow,
|
||||||
@@ -26,6 +27,7 @@ type EventItem = {
|
|||||||
};
|
};
|
||||||
|
|
||||||
type DashboardMetrics = {
|
type DashboardMetrics = {
|
||||||
|
activeSockets?: number;
|
||||||
activeClients: number;
|
activeClients: number;
|
||||||
totalClients: number;
|
totalClients: number;
|
||||||
bufferStats: {
|
bufferStats: {
|
||||||
@@ -62,6 +64,7 @@ export const DashboardOverview: React.FC = () => {
|
|||||||
getCached<StorageFile[]>('vcon:storage-files', [])
|
getCached<StorageFile[]>('vcon:storage-files', [])
|
||||||
);
|
);
|
||||||
const [isRefreshing, setIsRefreshing] = useState(false);
|
const [isRefreshing, setIsRefreshing] = useState(false);
|
||||||
|
const [fetchError, setFetchError] = useState('');
|
||||||
|
|
||||||
const refresh = useCallback(async () => {
|
const refresh = useCallback(async () => {
|
||||||
setIsRefreshing(true);
|
setIsRefreshing(true);
|
||||||
@@ -74,25 +77,58 @@ export const DashboardOverview: React.FC = () => {
|
|||||||
fetch('/api/clients', { cache: 'no-store' }),
|
fetch('/api/clients', { cache: 'no-store' }),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
if (
|
||||||
|
metricsResponse.status === 401 ||
|
||||||
|
eventsResponse.status === 401 ||
|
||||||
|
storageResponse.status === 401 ||
|
||||||
|
clientsResponse.status === 401
|
||||||
|
) {
|
||||||
|
window.location.assign('/vcon/login');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const errors: string[] = [];
|
||||||
|
|
||||||
if (metricsResponse.ok) {
|
if (metricsResponse.ok) {
|
||||||
const nextMetrics = await metricsResponse.json();
|
const nextMetrics = await metricsResponse.json();
|
||||||
setMetrics(setCached('vcon:metrics', nextMetrics));
|
setMetrics(setCached('vcon:metrics', nextMetrics));
|
||||||
|
} else {
|
||||||
|
errors.push('metrics');
|
||||||
}
|
}
|
||||||
|
|
||||||
if (eventsResponse.ok) {
|
if (eventsResponse.ok) {
|
||||||
const data = await eventsResponse.json();
|
const data = await eventsResponse.json();
|
||||||
setEvents(setCached('vcon:dashboard-events', data.events ?? []));
|
setEvents(setCached('vcon:dashboard-events', data.events ?? []));
|
||||||
|
} else {
|
||||||
|
errors.push('events');
|
||||||
}
|
}
|
||||||
|
|
||||||
if (storageResponse.ok) {
|
if (storageResponse.ok) {
|
||||||
const data = await storageResponse.json();
|
const data = await storageResponse.json();
|
||||||
setFiles(setCached('vcon:storage-files', data.files ?? []));
|
setFiles(setCached('vcon:storage-files', data.files ?? []));
|
||||||
if (data.stats) setCached('vcon:storage-stats', data.stats);
|
if (data.stats) setCached('vcon:storage-stats', data.stats);
|
||||||
|
} else {
|
||||||
|
errors.push('storage');
|
||||||
}
|
}
|
||||||
|
|
||||||
if (clientsResponse.ok) {
|
if (clientsResponse.ok) {
|
||||||
const data = await clientsResponse.json();
|
const data = await clientsResponse.json();
|
||||||
setClients(setCached('vcon:clients', data.clients ?? []));
|
setClients(setCached('vcon:clients', data.clients ?? []));
|
||||||
|
} else {
|
||||||
|
errors.push('clients');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (errors.length > 0) {
|
||||||
|
setFetchError(
|
||||||
|
`Partial telemetry refresh failed (${errors.join(', ')}). Showing last available snapshot.`
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
setFetchError('');
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
// Keep the last successful dashboard snapshot when one request fails.
|
setFetchError(
|
||||||
|
'Unable to reach telemetry endpoints. Showing last available snapshot.'
|
||||||
|
);
|
||||||
} finally {
|
} finally {
|
||||||
setIsRefreshing(false);
|
setIsRefreshing(false);
|
||||||
}
|
}
|
||||||
@@ -108,18 +144,21 @@ export const DashboardOverview: React.FC = () => {
|
|||||||
const storedRecordsToday = files
|
const storedRecordsToday = files
|
||||||
.filter((file) => file.date === today)
|
.filter((file) => file.date === today)
|
||||||
.reduce((total, file) => total + file.linesCount, 0);
|
.reduce((total, file) => total + file.linesCount, 0);
|
||||||
const eventsToday =
|
const bufferedCount = metrics?.bufferStats.currentBufferSize ?? 0;
|
||||||
storedRecordsToday + (metrics?.bufferStats.currentBufferSize ?? 0);
|
const eventsToday = storedRecordsToday + bufferedCount;
|
||||||
const activeClients =
|
const activeClients =
|
||||||
metrics?.activeClients ??
|
metrics?.activeClients ??
|
||||||
clients.filter((client) => client.status === 'active').length;
|
clients.filter((client) => client.status === 'active').length;
|
||||||
const totalClients = metrics?.totalClients ?? clients.length;
|
const totalClients = metrics?.totalClients ?? clients.length;
|
||||||
|
const activeSockets = metrics?.activeSockets ?? 0;
|
||||||
const totalStorage = metrics?.storageStats.totalMB ?? '0.00';
|
const totalStorage = metrics?.storageStats.totalMB ?? '0.00';
|
||||||
const totalStorageBytes = metrics?.storageStats.totalBytes ?? 0;
|
const totalStorageBytes = metrics?.storageStats.totalBytes ?? 0;
|
||||||
const storageValue =
|
const storageValue =
|
||||||
totalStorageBytes < 1024 * 1024
|
totalStorageBytes === 0
|
||||||
? String(Math.max(1, Math.round(totalStorageBytes / 1024)))
|
? '0'
|
||||||
: totalStorage;
|
: totalStorageBytes < 1024 * 1024
|
||||||
|
? String(Math.max(1, Math.round(totalStorageBytes / 1024)))
|
||||||
|
: totalStorage;
|
||||||
const storageUnit = totalStorageBytes < 1024 * 1024 ? 'KB' : 'MB';
|
const storageUnit = totalStorageBytes < 1024 * 1024 ? 'KB' : 'MB';
|
||||||
const totalFiles = metrics?.storageStats.totalFiles ?? files.length;
|
const totalFiles = metrics?.storageStats.totalFiles ?? files.length;
|
||||||
const totalRows =
|
const totalRows =
|
||||||
@@ -153,13 +192,22 @@ export const DashboardOverview: React.FC = () => {
|
|||||||
</>
|
</>
|
||||||
}
|
}
|
||||||
>
|
>
|
||||||
|
{fetchError && (
|
||||||
|
<div
|
||||||
|
role="alert"
|
||||||
|
className="mb-3 border border-[var(--danger)] bg-[var(--danger-soft)] px-3 py-2 text-[11.5px] text-[var(--text)]"
|
||||||
|
>
|
||||||
|
{fetchError}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
<StatStrip
|
<StatStrip
|
||||||
items={[
|
items={[
|
||||||
{ label: 'Events today', value: eventsToday.toLocaleString() },
|
{ label: 'Events today', value: eventsToday.toLocaleString() },
|
||||||
{
|
{
|
||||||
label: 'Active clients',
|
label: 'Active clients · Sockets',
|
||||||
value: `${activeClients} / ${totalClients}`,
|
value: `${activeClients} / ${totalClients} · ${activeSockets} ws`,
|
||||||
dot: activeClients > 0 ? 'ok' : 'off',
|
dot: activeClients > 0 || activeSockets > 0 ? 'ok' : 'off',
|
||||||
},
|
},
|
||||||
{ label: 'Storage used', value: `${storageValue} ${storageUnit}` },
|
{ label: 'Storage used', value: `${storageValue} ${storageUnit}` },
|
||||||
{ label: 'Stored rows', value: totalRows.toLocaleString() },
|
{ label: 'Stored rows', value: totalRows.toLocaleString() },
|
||||||
@@ -169,7 +217,11 @@ export const DashboardOverview: React.FC = () => {
|
|||||||
<div className="grid min-w-0 grid-cols-1 gap-3 lg:grid-cols-[minmax(0,1.85fr)_minmax(0,1fr)]">
|
<div className="grid min-w-0 grid-cols-1 gap-3 lg:grid-cols-[minmax(0,1.85fr)_minmax(0,1fr)]">
|
||||||
<Panel
|
<Panel
|
||||||
title="Recent events"
|
title="Recent events"
|
||||||
meta={`${events.length}`}
|
meta={
|
||||||
|
bufferedCount > 0
|
||||||
|
? `${events.length} · ${bufferedCount} buffered`
|
||||||
|
: `${events.length}`
|
||||||
|
}
|
||||||
action={
|
action={
|
||||||
<Link
|
<Link
|
||||||
href="/vcon/data"
|
href="/vcon/data"
|
||||||
@@ -190,24 +242,28 @@ export const DashboardOverview: React.FC = () => {
|
|||||||
</TableRow>
|
</TableRow>
|
||||||
</TableHeader>
|
</TableHeader>
|
||||||
<TableBody>
|
<TableBody>
|
||||||
{events.map((event) => (
|
{events.length === 0 ? (
|
||||||
<TableRow key={event.eventId}>
|
<TableEmpty colSpan={4} message="No recent events" />
|
||||||
<TableCell className="font-mono tabular-nums text-[var(--text-faint)]">
|
) : (
|
||||||
{event.timestamp.includes('T')
|
events.map((event) => (
|
||||||
? new Date(event.timestamp).toLocaleTimeString()
|
<TableRow key={event.eventId}>
|
||||||
: event.timestamp}
|
<TableCell className="font-mono tabular-nums text-[var(--text-faint)]">
|
||||||
</TableCell>
|
{event.timestamp.includes('T')
|
||||||
<TableCell className="font-mono text-[var(--accent)]">
|
? new Date(event.timestamp).toLocaleTimeString()
|
||||||
{event.clientId}
|
: event.timestamp}
|
||||||
</TableCell>
|
</TableCell>
|
||||||
<TableCell className="max-w-[380px] truncate font-mono text-[var(--text)]">
|
<TableCell className="font-mono text-[var(--accent)]">
|
||||||
{event.payload}
|
{event.clientId}
|
||||||
</TableCell>
|
</TableCell>
|
||||||
<TableCell className="text-right font-mono tabular-nums text-[var(--text-faint)]">
|
<TableCell className="max-w-[380px] truncate font-mono text-[var(--text)]">
|
||||||
{new TextEncoder().encode(event.payload).length} B
|
{event.payload}
|
||||||
</TableCell>
|
</TableCell>
|
||||||
</TableRow>
|
<TableCell className="text-right font-mono tabular-nums text-[var(--text-faint)]">
|
||||||
))}
|
{new TextEncoder().encode(event.payload).length} B
|
||||||
|
</TableCell>
|
||||||
|
</TableRow>
|
||||||
|
))
|
||||||
|
)}
|
||||||
</TableBody>
|
</TableBody>
|
||||||
</Table>
|
</Table>
|
||||||
</TableWrap>
|
</TableWrap>
|
||||||
|
|||||||
@@ -3,7 +3,10 @@
|
|||||||
import { PageShell } from '@/components/layout/PageShell';
|
import { PageShell } from '@/components/layout/PageShell';
|
||||||
import { Button } from '@/components/ui/Button';
|
import { Button } from '@/components/ui/Button';
|
||||||
import { Card } from '@/components/ui/Card';
|
import { Card } from '@/components/ui/Card';
|
||||||
|
import { CodeBlock } from '@/components/ui/CodeBlock';
|
||||||
|
import { CopyButton } from '@/components/ui/CopyButton';
|
||||||
import { Input, Select } from '@/components/ui/Input';
|
import { Input, Select } from '@/components/ui/Input';
|
||||||
|
import { Modal } from '@/components/ui/Modal';
|
||||||
import { Panel } from '@/components/ui/Panel';
|
import { Panel } from '@/components/ui/Panel';
|
||||||
import { Pill } from '@/components/ui/Pill';
|
import { Pill } from '@/components/ui/Pill';
|
||||||
import {
|
import {
|
||||||
@@ -34,6 +37,18 @@ interface Client {
|
|||||||
|
|
||||||
const PAGE_SIZE = 12;
|
const PAGE_SIZE = 12;
|
||||||
|
|
||||||
|
function formatEventPayload(raw: string): string {
|
||||||
|
try {
|
||||||
|
return JSON.stringify(JSON.parse(raw), null, 2);
|
||||||
|
} catch {
|
||||||
|
return raw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function getPayloadBytes(raw: string): number {
|
||||||
|
return new TextEncoder().encode(raw).length;
|
||||||
|
}
|
||||||
|
|
||||||
export function DataBrowser() {
|
export function DataBrowser() {
|
||||||
const router = useRouter();
|
const router = useRouter();
|
||||||
const [events, setEvents] = useState<StreamEvent[]>([]);
|
const [events, setEvents] = useState<StreamEvent[]>([]);
|
||||||
@@ -54,6 +69,7 @@ export function DataBrowser() {
|
|||||||
const [filtersReady, setFiltersReady] = useState(false);
|
const [filtersReady, setFiltersReady] = useState(false);
|
||||||
const [refreshing, setRefreshing] = useState(false);
|
const [refreshing, setRefreshing] = useState(false);
|
||||||
const [errorMessage, setErrorMessage] = useState('');
|
const [errorMessage, setErrorMessage] = useState('');
|
||||||
|
const [selectedEvent, setSelectedEvent] = useState<StreamEvent | null>(null);
|
||||||
|
|
||||||
const fetchEvents = useCallback(async () => {
|
const fetchEvents = useCallback(async () => {
|
||||||
try {
|
try {
|
||||||
@@ -65,7 +81,11 @@ export function DataBrowser() {
|
|||||||
page: String(page),
|
page: String(page),
|
||||||
pageSize: String(PAGE_SIZE),
|
pageSize: String(PAGE_SIZE),
|
||||||
});
|
});
|
||||||
const response = await fetch(`/api/data?${params}`);
|
const response = await fetch(`/api/data?${params}`, { cache: 'no-store' });
|
||||||
|
if (response.status === 401) {
|
||||||
|
window.location.assign('/vcon/login');
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (!response.ok) throw new Error('Unable to load events');
|
if (!response.ok) throw new Error('Unable to load events');
|
||||||
const data = await response.json();
|
const data = await response.json();
|
||||||
setEvents(data.events || []);
|
setEvents(data.events || []);
|
||||||
@@ -78,8 +98,14 @@ export function DataBrowser() {
|
|||||||
}, [appliedFilters, page]);
|
}, [appliedFilters, page]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
fetch('/api/clients')
|
fetch('/api/clients', { cache: 'no-store' })
|
||||||
.then((response) => (response.ok ? response.json() : { clients: [] }))
|
.then((response) => {
|
||||||
|
if (response.status === 401) {
|
||||||
|
window.location.assign('/vcon/login');
|
||||||
|
return { clients: [] };
|
||||||
|
}
|
||||||
|
return response.ok ? response.json() : { clients: [] };
|
||||||
|
})
|
||||||
.then((data) => setClients(data.clients || []))
|
.then((data) => setClients(data.clients || []))
|
||||||
.catch(() => setClients([]));
|
.catch(() => setClients([]));
|
||||||
}, []);
|
}, []);
|
||||||
@@ -131,6 +157,10 @@ export function DataBrowser() {
|
|||||||
setRefreshing(false);
|
setRefreshing(false);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const formattedSelectedPayload = selectedEvent
|
||||||
|
? formatEventPayload(selectedEvent.payload)
|
||||||
|
: '';
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<PageShell
|
<PageShell
|
||||||
title="Events"
|
title="Events"
|
||||||
@@ -238,7 +268,11 @@ export function DataBrowser() {
|
|||||||
<TableEmpty colSpan={6} message="No matching events" />
|
<TableEmpty colSpan={6} message="No matching events" />
|
||||||
) : (
|
) : (
|
||||||
events.map((event) => (
|
events.map((event) => (
|
||||||
<TableRow key={event.eventId}>
|
<TableRow
|
||||||
|
key={`${event.clientId}:${event.eventId}`}
|
||||||
|
className="cursor-pointer"
|
||||||
|
onClick={() => setSelectedEvent(event)}
|
||||||
|
>
|
||||||
<TableCell className="font-mono tabular-nums text-[var(--text)]">
|
<TableCell className="font-mono tabular-nums text-[var(--text)]">
|
||||||
{new Date(event.timestamp).toLocaleTimeString([], {
|
{new Date(event.timestamp).toLocaleTimeString([], {
|
||||||
hour: '2-digit',
|
hour: '2-digit',
|
||||||
@@ -257,13 +291,34 @@ export function DataBrowser() {
|
|||||||
{event.payload}
|
{event.payload}
|
||||||
</TableCell>
|
</TableCell>
|
||||||
<TableCell className="hidden font-mono tabular-nums md:table-cell">
|
<TableCell className="hidden font-mono tabular-nums md:table-cell">
|
||||||
{new TextEncoder().encode(event.payload).length} B
|
{getPayloadBytes(event.payload)} B
|
||||||
</TableCell>
|
</TableCell>
|
||||||
<TableCell className="text-right">
|
<TableCell className="text-right">
|
||||||
<span className="inline-flex items-center gap-1.5 text-[var(--text-muted)]">
|
<div className="inline-flex items-center gap-2">
|
||||||
<span className="h-1.5 w-1.5 bg-[var(--accent)]" />
|
<span className="inline-flex items-center gap-1.5 text-[var(--text-muted)]">
|
||||||
{event.status}
|
<span
|
||||||
</span>
|
className={`h-1.5 w-1.5 ${
|
||||||
|
event.status === 'buffered'
|
||||||
|
? 'bg-[var(--warning)]'
|
||||||
|
: 'bg-[var(--accent)]'
|
||||||
|
}`}
|
||||||
|
/>
|
||||||
|
{event.status}
|
||||||
|
</span>
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
variant="icon"
|
||||||
|
size="icon"
|
||||||
|
icon="Eye"
|
||||||
|
className="h-5 w-5 border border-transparent hover:border-[var(--border)] hover:bg-[var(--surface-2)]"
|
||||||
|
title={`Inspect event ${event.eventId}`}
|
||||||
|
aria-label={`Inspect event ${event.eventId}`}
|
||||||
|
onClick={(e) => {
|
||||||
|
e.stopPropagation();
|
||||||
|
setSelectedEvent(event);
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
</TableCell>
|
</TableCell>
|
||||||
</TableRow>
|
</TableRow>
|
||||||
))
|
))
|
||||||
@@ -338,6 +393,69 @@ export function DataBrowser() {
|
|||||||
</nav>
|
</nav>
|
||||||
</footer>
|
</footer>
|
||||||
</Panel>
|
</Panel>
|
||||||
|
|
||||||
|
<Modal
|
||||||
|
isOpen={Boolean(selectedEvent)}
|
||||||
|
onClose={() => setSelectedEvent(null)}
|
||||||
|
title={`Event payload: ${selectedEvent?.eventId || ''}`}
|
||||||
|
maxWidth="lg"
|
||||||
|
footer={
|
||||||
|
<Button type="button" size="xs" onClick={() => setSelectedEvent(null)}>
|
||||||
|
Close
|
||||||
|
</Button>
|
||||||
|
}
|
||||||
|
>
|
||||||
|
{selectedEvent && (
|
||||||
|
<div className="flex flex-col gap-3">
|
||||||
|
<div className="grid grid-cols-2 gap-px border border-[var(--border)] bg-[var(--border)] sm:grid-cols-4">
|
||||||
|
<div className="bg-[var(--surface)] p-2">
|
||||||
|
<div className="text-[9.5px] uppercase tracking-wider text-[var(--text-faint)]">
|
||||||
|
Client
|
||||||
|
</div>
|
||||||
|
<div className="truncate font-mono text-[11.5px] text-[var(--accent)]">
|
||||||
|
{selectedEvent.clientId}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="bg-[var(--surface)] p-2">
|
||||||
|
<div className="text-[9.5px] uppercase tracking-wider text-[var(--text-faint)]">
|
||||||
|
Status
|
||||||
|
</div>
|
||||||
|
<div className="font-mono text-[11.5px] text-[var(--text)]">
|
||||||
|
{selectedEvent.status}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="bg-[var(--surface)] p-2">
|
||||||
|
<div className="text-[9.5px] uppercase tracking-wider text-[var(--text-faint)]">
|
||||||
|
Size
|
||||||
|
</div>
|
||||||
|
<div className="font-mono text-[11.5px] tabular-nums text-[var(--text)]">
|
||||||
|
{getPayloadBytes(selectedEvent.payload)} B
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="bg-[var(--surface)] p-2">
|
||||||
|
<div className="text-[9.5px] uppercase tracking-wider text-[var(--text-faint)]">
|
||||||
|
Timestamp
|
||||||
|
</div>
|
||||||
|
<div className="truncate font-mono text-[11px] tabular-nums text-[var(--text-muted)]">
|
||||||
|
{selectedEvent.timestamp}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex flex-col gap-1">
|
||||||
|
<div className="flex items-center justify-between gap-2">
|
||||||
|
<span className="text-[10px] uppercase tracking-wider text-[var(--text-faint)]">
|
||||||
|
JSON / Raw Payload
|
||||||
|
</span>
|
||||||
|
<CopyButton value={formattedSelectedPayload} />
|
||||||
|
</div>
|
||||||
|
<CodeBlock className="max-h-[340px] border border-[var(--border)] text-[11px]">
|
||||||
|
{formattedSelectedPayload}
|
||||||
|
</CodeBlock>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</Modal>
|
||||||
</PageShell>
|
</PageShell>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -56,6 +56,7 @@ export function ExportWorkspace() {
|
|||||||
const [files, setFiles] = useState<StorageFile[]>(() =>
|
const [files, setFiles] = useState<StorageFile[]>(() =>
|
||||||
getCached<StorageFile[]>('vcon:storage-files', [])
|
getCached<StorageFile[]>('vcon:storage-files', [])
|
||||||
);
|
);
|
||||||
|
const [bufferedCount, setBufferedCount] = useState<number>(0);
|
||||||
const [selectedClient, setSelectedClient] = useState('all');
|
const [selectedClient, setSelectedClient] = useState('all');
|
||||||
const [fromDate, setFromDate] = useState('');
|
const [fromDate, setFromDate] = useState('');
|
||||||
const [toDate, setToDate] = useState('');
|
const [toDate, setToDate] = useState('');
|
||||||
@@ -69,19 +70,33 @@ export function ExportWorkspace() {
|
|||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
Promise.all([
|
Promise.all([
|
||||||
fetch('/api/clients').then(async (response) => {
|
fetch('/api/clients', { cache: 'no-store' }).then(async (response) => {
|
||||||
|
if (response.status === 401) {
|
||||||
|
window.location.assign('/vcon/login');
|
||||||
|
return { clients: [] };
|
||||||
|
}
|
||||||
if (!response.ok) throw new Error('Unable to load clients');
|
if (!response.ok) throw new Error('Unable to load clients');
|
||||||
return response.json();
|
return response.json();
|
||||||
}),
|
}),
|
||||||
fetch('/api/storage').then(async (response) => {
|
fetch('/api/storage', { cache: 'no-store' }).then(async (response) => {
|
||||||
|
if (response.status === 401) {
|
||||||
|
window.location.assign('/vcon/login');
|
||||||
|
return { files: [] };
|
||||||
|
}
|
||||||
if (!response.ok) throw new Error('Unable to load stored files');
|
if (!response.ok) throw new Error('Unable to load stored files');
|
||||||
return response.json();
|
return response.json();
|
||||||
}),
|
}),
|
||||||
|
fetch('/api/metrics', { cache: 'no-store' })
|
||||||
|
.then(async (response) => (response.ok ? response.json() : null))
|
||||||
|
.catch(() => null),
|
||||||
])
|
])
|
||||||
.then(([clientData, storageData]) => {
|
.then(([clientData, storageData, metricsData]) => {
|
||||||
setClients(setCached('vcon:clients', clientData.clients || []));
|
setClients(setCached('vcon:clients', clientData.clients || []));
|
||||||
setFiles(setCached('vcon:storage-files', storageData.files || []));
|
setFiles(setCached('vcon:storage-files', storageData.files || []));
|
||||||
if (storageData.stats) setCached('vcon:storage-stats', storageData.stats);
|
if (storageData.stats) setCached('vcon:storage-stats', storageData.stats);
|
||||||
|
if (metricsData?.bufferStats) {
|
||||||
|
setBufferedCount(Number(metricsData.bufferStats.currentBufferSize) || 0);
|
||||||
|
}
|
||||||
setLoadError('');
|
setLoadError('');
|
||||||
})
|
})
|
||||||
.catch((fetchError) => {
|
.catch((fetchError) => {
|
||||||
@@ -162,15 +177,24 @@ export function ExportWorkspace() {
|
|||||||
return `/api/export?${params.toString()}`;
|
return `/api/export?${params.toString()}`;
|
||||||
}, [format, fromDate, selectedClient, toDate]);
|
}, [format, fromDate, selectedClient, toDate]);
|
||||||
|
|
||||||
|
const invalidRange = Boolean(fromDate && toDate && fromDate > toDate);
|
||||||
|
const nothingToExport = preview.rows === 0 && bufferedCount === 0;
|
||||||
|
|
||||||
const downloadExport = () => {
|
const downloadExport = () => {
|
||||||
if (fromDate && toDate && fromDate > toDate) {
|
if (invalidRange) {
|
||||||
error('Invalid date range', 'Start date must be on or before end date.');
|
error('Invalid date range', 'Start date must be on or before end date.');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
if (nothingToExport) {
|
||||||
|
error('Nothing to export', 'No stored rows or buffered events match the selected filters.');
|
||||||
|
return;
|
||||||
|
}
|
||||||
window.location.assign(exportUrl);
|
window.location.assign(exportUrl);
|
||||||
info(
|
info(
|
||||||
'Export started',
|
'Export started',
|
||||||
`${format.toUpperCase()} download for ${preview.rows.toLocaleString()} rows.`
|
bufferedCount > 0
|
||||||
|
? `${format.toUpperCase()} download (${preview.rows.toLocaleString()} stored + ${bufferedCount} buffered auto-flushed).`
|
||||||
|
: `${format.toUpperCase()} download for ${preview.rows.toLocaleString()} rows.`
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -307,17 +331,30 @@ export function ExportWorkspace() {
|
|||||||
</strong>{' '}
|
</strong>{' '}
|
||||||
files
|
files
|
||||||
</span>
|
</span>
|
||||||
|
{bufferedCount > 0 && (
|
||||||
|
<>
|
||||||
|
<span aria-hidden="true">·</span>
|
||||||
|
<span className="text-[var(--warning)]">
|
||||||
|
+{bufferedCount} buffered (flushed on export)
|
||||||
|
</span>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
{loadError && (
|
{loadError && (
|
||||||
<p role="alert" className="text-[11px] text-[var(--text-muted)]">
|
<p role="alert" className="text-[11px] text-[var(--danger)]">
|
||||||
{loadError}
|
{loadError}
|
||||||
</p>
|
</p>
|
||||||
)}
|
)}
|
||||||
{fromDate && toDate && fromDate > toDate && (
|
{invalidRange && (
|
||||||
<p role="alert" className="text-[11px] text-[var(--text-muted)]">
|
<p role="alert" className="text-[11px] text-[var(--danger)]">
|
||||||
Start date must be on or before end date.
|
Start date must be on or before end date.
|
||||||
</p>
|
</p>
|
||||||
)}
|
)}
|
||||||
|
{!loading && !invalidRange && nothingToExport && (
|
||||||
|
<p role="status" className="text-[11px] text-[var(--text-faint)]">
|
||||||
|
No stored rows or buffered events match the selected filters.
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
<div className="flex flex-wrap items-center justify-end gap-1.5 border-t border-[var(--border)] pt-2.5">
|
<div className="flex flex-wrap items-center justify-end gap-1.5 border-t border-[var(--border)] pt-2.5">
|
||||||
<Button type="button" size="xs" onClick={resetExport}>
|
<Button type="button" size="xs" onClick={resetExport}>
|
||||||
@@ -328,7 +365,7 @@ export function ExportWorkspace() {
|
|||||||
variant="primary"
|
variant="primary"
|
||||||
size="xs"
|
size="xs"
|
||||||
icon="Download"
|
icon="Download"
|
||||||
disabled={loading || Boolean(fromDate && toDate && fromDate > toDate)}
|
disabled={loading || invalidRange || nothingToExport}
|
||||||
onClick={downloadExport}
|
onClick={downloadExport}
|
||||||
>
|
>
|
||||||
Download {format.toUpperCase()}
|
Download {format.toUpperCase()}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { ActivityPreview } from '@/components/ui/ActivityPreview';
|
import { ActivityPreview } from '@/components/ui/ActivityPreview';
|
||||||
import { Button } from '@/components/ui/Button';
|
import { buttonClassNames } from '@/components/ui/Button';
|
||||||
import Link from 'next/link';
|
import Link from 'next/link';
|
||||||
import React from 'react';
|
import React from 'react';
|
||||||
|
|
||||||
@@ -17,11 +17,11 @@ export const Hero: React.FC = () => (
|
|||||||
Keep activity organized, review the records that matter, and export them when you need them.
|
Keep activity organized, review the records that matter, and export them when you need them.
|
||||||
</p>
|
</p>
|
||||||
<div className="flex flex-wrap gap-2">
|
<div className="flex flex-wrap gap-2">
|
||||||
<Link href="#features">
|
<Link href="/#features" className={buttonClassNames({ variant: 'primary', size: 'sm' })}>
|
||||||
<Button variant="primary" size="sm">Explore features</Button>
|
Explore features
|
||||||
</Link>
|
</Link>
|
||||||
<Link href="#sdk">
|
<Link href="/#sdk" className={buttonClassNames({ variant: 'secondary', size: 'sm' })}>
|
||||||
<Button variant="secondary" size="sm">Python SDK</Button>
|
Python SDK
|
||||||
</Link>
|
</Link>
|
||||||
</div>
|
</div>
|
||||||
<ActivityPreview />
|
<ActivityPreview />
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { Button } from '@/components/ui/Button';
|
import { buttonClassNames } from '@/components/ui/Button';
|
||||||
import Link from 'next/link';
|
import Link from 'next/link';
|
||||||
import React from 'react';
|
import React from 'react';
|
||||||
|
|
||||||
@@ -8,8 +8,8 @@ export const LandingAbout: React.FC = () => (
|
|||||||
<h2 className="mb-1 text-[20px] font-medium text-[var(--text)]">Ready to simplify your data workflow?</h2>
|
<h2 className="mb-1 text-[20px] font-medium text-[var(--text)]">Ready to simplify your data workflow?</h2>
|
||||||
<p className="text-[13px] text-[var(--text-muted)]">Start with the SDK guide and make the workspace your own.</p>
|
<p className="text-[13px] text-[var(--text-muted)]">Start with the SDK guide and make the workspace your own.</p>
|
||||||
</div>
|
</div>
|
||||||
<Link href="#sdk">
|
<Link href="/#sdk" className={buttonClassNames({ variant: 'primary', size: 'sm' })}>
|
||||||
<Button variant="primary" size="sm">View Python SDK</Button>
|
View Python SDK
|
||||||
</Link>
|
</Link>
|
||||||
</section>
|
</section>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import { PageShell } from '@/components/layout/PageShell';
|
|||||||
import { Button } from '@/components/ui/Button';
|
import { Button } from '@/components/ui/Button';
|
||||||
import { FormSwitch } from '@/components/ui/Form';
|
import { FormSwitch } from '@/components/ui/Form';
|
||||||
import { Input } from '@/components/ui/Input';
|
import { Input } from '@/components/ui/Input';
|
||||||
|
import { Modal } from '@/components/ui/Modal';
|
||||||
import {
|
import {
|
||||||
SettingsField,
|
SettingsField,
|
||||||
SettingsFooter,
|
SettingsFooter,
|
||||||
@@ -14,7 +15,7 @@ import {
|
|||||||
import { StatStrip } from '@/components/ui/StatStrip';
|
import { StatStrip } from '@/components/ui/StatStrip';
|
||||||
import { useToast } from '@/components/ui/Toast';
|
import { useToast } from '@/components/ui/Toast';
|
||||||
import { getCached, setCached } from '@/lib/vconCache';
|
import { getCached, setCached } from '@/lib/vconCache';
|
||||||
import { useEffect, useMemo, useState } from 'react';
|
import { useCallback, useEffect, useMemo, useState } from 'react';
|
||||||
|
|
||||||
interface EditableSettings {
|
interface EditableSettings {
|
||||||
maxEventsPerSec: number;
|
maxEventsPerSec: number;
|
||||||
@@ -50,6 +51,7 @@ const TABS = [
|
|||||||
];
|
];
|
||||||
|
|
||||||
type TabId = (typeof TABS)[number]['id'];
|
type TabId = (typeof TABS)[number]['id'];
|
||||||
|
type SocketHealthState = 'idle' | 'checking' | 'online' | 'offline';
|
||||||
|
|
||||||
export function SettingsWorkspace() {
|
export function SettingsWorkspace() {
|
||||||
const { success, error } = useToast();
|
const { success, error } = useToast();
|
||||||
@@ -61,11 +63,15 @@ export function SettingsWorkspace() {
|
|||||||
);
|
);
|
||||||
const [activeTab, setActiveTab] = useState<TabId>('general');
|
const [activeTab, setActiveTab] = useState<TabId>('general');
|
||||||
const [jwtSecret, setJwtSecret] = useState('');
|
const [jwtSecret, setJwtSecret] = useState('');
|
||||||
|
const [jwtError, setJwtError] = useState('');
|
||||||
|
const [confirmRotateOpen, setConfirmRotateOpen] = useState(false);
|
||||||
const [storagePath, setStoragePath] = useState(() =>
|
const [storagePath, setStoragePath] = useState(() =>
|
||||||
getCached<string>('vcon:storage-path', '')
|
getCached<string>('vcon:storage-path', '')
|
||||||
);
|
);
|
||||||
const [baseUrl, setBaseUrl] = useState('');
|
const [baseUrl, setBaseUrl] = useState('');
|
||||||
const [webSocketUrl, setWebSocketUrl] = useState('');
|
const [webSocketUrl, setWebSocketUrl] = useState('');
|
||||||
|
const [socketStatus, setSocketStatus] = useState<SocketHealthState>('idle');
|
||||||
|
const [activeSockets, setActiveSockets] = useState<number>(0);
|
||||||
const [serverPort, setServerPort] = useState(() =>
|
const [serverPort, setServerPort] = useState(() =>
|
||||||
getCached<number>('vcon:server-port', 4000)
|
getCached<number>('vcon:server-port', 4000)
|
||||||
);
|
);
|
||||||
@@ -74,26 +80,60 @@ export function SettingsWorkspace() {
|
|||||||
);
|
);
|
||||||
const [saving, setSaving] = useState(false);
|
const [saving, setSaving] = useState(false);
|
||||||
|
|
||||||
|
const probeSocketServer = useCallback(async (port: number) => {
|
||||||
|
setSocketStatus('checking');
|
||||||
|
const isLocalDev = window.location.port === '3000';
|
||||||
|
const httpHost = isLocalDev
|
||||||
|
? `${window.location.hostname}:${port}`
|
||||||
|
: window.location.host;
|
||||||
|
const healthUrl = `${window.location.protocol}//${httpHost}/health`;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch(healthUrl, {
|
||||||
|
cache: 'no-store',
|
||||||
|
signal: AbortSignal.timeout(1500),
|
||||||
|
});
|
||||||
|
setSocketStatus(response.ok ? 'online' : 'offline');
|
||||||
|
} catch {
|
||||||
|
setSocketStatus('offline');
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
setBaseUrl(window.location.origin);
|
setBaseUrl(window.location.origin);
|
||||||
const isLocalDev = window.location.port === '3000';
|
const isLocalDev = window.location.port === '3000';
|
||||||
const socketProtocol = window.location.protocol === 'https:' ? 'wss' : 'ws';
|
const socketProtocol = window.location.protocol === 'https:' ? 'wss' : 'ws';
|
||||||
const socketHost = isLocalDev
|
const socketHost = isLocalDev
|
||||||
? `${window.location.hostname}:4000`
|
? `${window.location.hostname}:${serverPort}`
|
||||||
: window.location.host;
|
: window.location.host;
|
||||||
setWebSocketUrl(`${socketProtocol}://${socketHost}`);
|
setWebSocketUrl(`${socketProtocol}://${socketHost}`);
|
||||||
|
|
||||||
Promise.all([
|
Promise.all([
|
||||||
fetch('/api/settings').then(async (response) => {
|
fetch('/api/settings', { cache: 'no-store' }).then(async (response) => {
|
||||||
|
if (response.status === 401) {
|
||||||
|
window.location.assign('/vcon/login');
|
||||||
|
return null;
|
||||||
|
}
|
||||||
if (!response.ok) throw new Error('Unable to load settings');
|
if (!response.ok) throw new Error('Unable to load settings');
|
||||||
return response.json();
|
return response.json();
|
||||||
}),
|
}),
|
||||||
fetch('/api/storage').then(async (response) => {
|
fetch('/api/storage', { cache: 'no-store' }).then(async (response) => {
|
||||||
|
if (response.status === 401) {
|
||||||
|
window.location.assign('/vcon/login');
|
||||||
|
return null;
|
||||||
|
}
|
||||||
if (!response.ok) throw new Error('Unable to load storage information');
|
if (!response.ok) throw new Error('Unable to load storage information');
|
||||||
return response.json();
|
return response.json();
|
||||||
}),
|
}),
|
||||||
|
fetch('/api/metrics', { cache: 'no-store' })
|
||||||
|
.then(async (response) => (response.ok ? response.json() : null))
|
||||||
|
.catch(() => null),
|
||||||
])
|
])
|
||||||
.then(([settingsData, storageData]) => {
|
.then(([settingsData, storageData, metricsData]) => {
|
||||||
|
if (!settingsData || !storageData) return;
|
||||||
|
const loadedPort = Number(
|
||||||
|
(settingsData.settings as PublicSettings).serverPort ?? 4000
|
||||||
|
);
|
||||||
const loaded: EditableSettings = {
|
const loaded: EditableSettings = {
|
||||||
maxEventsPerSec: Number(
|
maxEventsPerSec: Number(
|
||||||
settingsData.settings.maxEventsPerSec ??
|
settingsData.settings.maxEventsPerSec ??
|
||||||
@@ -126,25 +166,26 @@ export function SettingsWorkspace() {
|
|||||||
};
|
};
|
||||||
setSettings(setCached('vcon:settings', loaded));
|
setSettings(setCached('vcon:settings', loaded));
|
||||||
setSavedSettings(loaded);
|
setSavedSettings(loaded);
|
||||||
setServerPort(
|
setServerPort(setCached('vcon:server-port', loadedPort));
|
||||||
setCached(
|
|
||||||
'vcon:server-port',
|
|
||||||
Number((settingsData.settings as PublicSettings).serverPort ?? 4000)
|
|
||||||
)
|
|
||||||
);
|
|
||||||
setStoragePath(
|
setStoragePath(
|
||||||
setCached(
|
setCached(
|
||||||
'vcon:storage-path',
|
'vcon:storage-path',
|
||||||
String(storageData.stats?.storageDirectory ?? '')
|
String(storageData.stats?.storageDirectory ?? '')
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
|
if (metricsData && typeof metricsData.activeSockets === 'number') {
|
||||||
|
setActiveSockets(metricsData.activeSockets);
|
||||||
|
if (metricsData.activeSockets > 0) {
|
||||||
|
setSocketStatus('online');
|
||||||
|
}
|
||||||
|
}
|
||||||
setCached('vcon:settings-loaded', true);
|
setCached('vcon:settings-loaded', true);
|
||||||
})
|
})
|
||||||
.catch(() => {
|
.catch(() => {
|
||||||
error('Settings load failed', 'Using default runtime settings.');
|
error('Settings load failed', 'Using default runtime settings.');
|
||||||
})
|
})
|
||||||
.finally(() => setLoading(false));
|
.finally(() => setLoading(false));
|
||||||
}, [error]);
|
}, [error, serverPort]);
|
||||||
|
|
||||||
const isDirty = useMemo(
|
const isDirty = useMemo(
|
||||||
() =>
|
() =>
|
||||||
@@ -157,9 +198,9 @@ export function SettingsWorkspace() {
|
|||||||
setSettings((current) => ({ ...current, [key]: Number(value) || 0 }));
|
setSettings((current) => ({ ...current, [key]: Number(value) || 0 }));
|
||||||
};
|
};
|
||||||
|
|
||||||
const saveSettings = async (event: React.FormEvent<HTMLFormElement>) => {
|
const buildNormalizedSettings = (): EditableSettings => {
|
||||||
event.preventDefault();
|
const parsedPrune = Number(settings.autoPruneDays);
|
||||||
const normalized: EditableSettings = {
|
return {
|
||||||
maxEventsPerSec: Math.max(
|
maxEventsPerSec: Math.max(
|
||||||
1,
|
1,
|
||||||
Math.min(
|
Math.min(
|
||||||
@@ -187,11 +228,16 @@ export function SettingsWorkspace() {
|
|||||||
settings.fileRotationMb || DEFAULT_SETTINGS.fileRotationMb
|
settings.fileRotationMb || DEFAULT_SETTINGS.fileRotationMb
|
||||||
),
|
),
|
||||||
enableIdempotency: settings.enableIdempotency,
|
enableIdempotency: settings.enableIdempotency,
|
||||||
autoPruneDays: Math.max(
|
autoPruneDays: Number.isFinite(parsedPrune)
|
||||||
1,
|
? Math.max(0, Math.min(365, Math.floor(parsedPrune)))
|
||||||
Math.min(365, settings.autoPruneDays || DEFAULT_SETTINGS.autoPruneDays)
|
: DEFAULT_SETTINGS.autoPruneDays,
|
||||||
),
|
|
||||||
};
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const executeSave = async () => {
|
||||||
|
const normalized = buildNormalizedSettings();
|
||||||
|
const trimmedSecret = jwtSecret.trim();
|
||||||
|
setConfirmRotateOpen(false);
|
||||||
setSaving(true);
|
setSaving(true);
|
||||||
try {
|
try {
|
||||||
const response = await fetch('/api/settings', {
|
const response = await fetch('/api/settings', {
|
||||||
@@ -199,15 +245,25 @@ export function SettingsWorkspace() {
|
|||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
...normalized,
|
...normalized,
|
||||||
...(jwtSecret.trim() ? { jwtSecret: jwtSecret.trim() } : {}),
|
...(trimmedSecret ? { jwtSecret: trimmedSecret } : {}),
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
|
if (response.status === 401) {
|
||||||
|
window.location.assign('/vcon/login');
|
||||||
|
return;
|
||||||
|
}
|
||||||
const data = await response.json();
|
const data = await response.json();
|
||||||
if (!response.ok) throw new Error(data.error || 'Settings update failed');
|
if (!response.ok) throw new Error(data.error || 'Settings update failed');
|
||||||
setSettings(setCached('vcon:settings', normalized));
|
setSettings(setCached('vcon:settings', normalized));
|
||||||
setSavedSettings(normalized);
|
setSavedSettings(normalized);
|
||||||
setJwtSecret('');
|
setJwtSecret('');
|
||||||
success('Settings saved', 'Runtime settings updated.');
|
setJwtError('');
|
||||||
|
success(
|
||||||
|
'Settings saved',
|
||||||
|
trimmedSecret
|
||||||
|
? 'Runtime settings saved and client JWT credentials rotated.'
|
||||||
|
: 'Runtime settings updated.'
|
||||||
|
);
|
||||||
} catch (saveError) {
|
} catch (saveError) {
|
||||||
error(
|
error(
|
||||||
'Save failed',
|
'Save failed',
|
||||||
@@ -220,9 +276,30 @@ export function SettingsWorkspace() {
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const handleFormSubmit = async (event: React.FormEvent<HTMLFormElement>) => {
|
||||||
|
event.preventDefault();
|
||||||
|
const trimmedSecret = jwtSecret.trim();
|
||||||
|
if (trimmedSecret.length > 0 && trimmedSecret.length < 32) {
|
||||||
|
const message = 'JWT secret must be at least 32 characters.';
|
||||||
|
setJwtError(message);
|
||||||
|
setActiveTab('security');
|
||||||
|
error('Invalid JWT secret', message);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
setJwtError('');
|
||||||
|
if (trimmedSecret.length >= 32) {
|
||||||
|
setConfirmRotateOpen(true);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await executeSave();
|
||||||
|
};
|
||||||
|
|
||||||
const discardChanges = () => {
|
const discardChanges = () => {
|
||||||
setSettings(savedSettings);
|
setSettings(savedSettings);
|
||||||
setJwtSecret('');
|
setJwtSecret('');
|
||||||
|
setJwtError('');
|
||||||
};
|
};
|
||||||
|
|
||||||
return (
|
return (
|
||||||
@@ -237,7 +314,9 @@ export function SettingsWorkspace() {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
label: 'CSV rotation',
|
label: 'CSV rotation',
|
||||||
value: `${settings.fileRotationMb}MB · ${settings.autoPruneDays}d`,
|
value: `${settings.fileRotationMb}MB · ${
|
||||||
|
settings.autoPruneDays === 0 ? 'Keep all' : `${settings.autoPruneDays}d`
|
||||||
|
}`,
|
||||||
},
|
},
|
||||||
]}
|
]}
|
||||||
/>
|
/>
|
||||||
@@ -249,7 +328,7 @@ export function SettingsWorkspace() {
|
|||||||
onChange={(tab) => setActiveTab(tab as TabId)}
|
onChange={(tab) => setActiveTab(tab as TabId)}
|
||||||
/>
|
/>
|
||||||
|
|
||||||
<form onSubmit={saveSettings} className="w-full">
|
<form onSubmit={handleFormSubmit} className="w-full">
|
||||||
<SettingsPanel
|
<SettingsPanel
|
||||||
id={`settings-panel-${activeTab}`}
|
id={`settings-panel-${activeTab}`}
|
||||||
role="tabpanel"
|
role="tabpanel"
|
||||||
@@ -307,6 +386,40 @@ export function SettingsWorkspace() {
|
|||||||
{activeTab === 'connections' && (
|
{activeTab === 'connections' && (
|
||||||
<>
|
<>
|
||||||
<SettingsGroup title="WebSocket server">
|
<SettingsGroup title="WebSocket server">
|
||||||
|
<SettingsField label="Live health">
|
||||||
|
<div className="flex items-center justify-between gap-2 border border-[var(--border)] bg-[var(--surface-2)] px-2.5 py-1.5">
|
||||||
|
<span className="inline-flex items-center gap-1.5 font-mono text-[11.5px] text-[var(--text)]">
|
||||||
|
<span
|
||||||
|
aria-hidden="true"
|
||||||
|
className={`h-1.5 w-1.5 ${
|
||||||
|
socketStatus === 'online'
|
||||||
|
? 'bg-[var(--accent)]'
|
||||||
|
: socketStatus === 'checking'
|
||||||
|
? 'bg-[var(--warning)]'
|
||||||
|
: socketStatus === 'offline'
|
||||||
|
? 'bg-[var(--danger)]'
|
||||||
|
: 'bg-[var(--text-faint)]'
|
||||||
|
}`}
|
||||||
|
/>
|
||||||
|
{socketStatus === 'online'
|
||||||
|
? `Online (Port ${serverPort} · ${activeSockets} active)`
|
||||||
|
: socketStatus === 'checking'
|
||||||
|
? 'Probing ingestion server...'
|
||||||
|
: socketStatus === 'offline'
|
||||||
|
? `Offline (Port ${serverPort} unreachable)`
|
||||||
|
: `Port ${serverPort} · ${activeSockets} active socket(s)`}
|
||||||
|
</span>
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
size="xs"
|
||||||
|
variant="ghost"
|
||||||
|
icon="RefreshCw"
|
||||||
|
onClick={() => void probeSocketServer(serverPort)}
|
||||||
|
>
|
||||||
|
Probe
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</SettingsField>
|
||||||
<SettingsField label="WebSocket URL">
|
<SettingsField label="WebSocket URL">
|
||||||
<Input
|
<Input
|
||||||
compact
|
compact
|
||||||
@@ -378,11 +491,11 @@ export function SettingsWorkspace() {
|
|||||||
className="font-mono"
|
className="font-mono"
|
||||||
/>
|
/>
|
||||||
</SettingsField>
|
</SettingsField>
|
||||||
<SettingsField label="Retention">
|
<SettingsField label="Retention (0 = never prune)">
|
||||||
<Input
|
<Input
|
||||||
compact
|
compact
|
||||||
type="number"
|
type="number"
|
||||||
min="1"
|
min="0"
|
||||||
max="365"
|
max="365"
|
||||||
suffix="days"
|
suffix="days"
|
||||||
value={settings.autoPruneDays}
|
value={settings.autoPruneDays}
|
||||||
@@ -408,14 +521,21 @@ export function SettingsWorkspace() {
|
|||||||
|
|
||||||
{activeTab === 'security' && (
|
{activeTab === 'security' && (
|
||||||
<SettingsGroup title="Authentication">
|
<SettingsGroup title="Authentication">
|
||||||
<SettingsField label="JWT secret">
|
<SettingsField label="JWT secret (min 32 chars)">
|
||||||
<Input
|
<Input
|
||||||
compact
|
compact
|
||||||
type="password"
|
type="password"
|
||||||
autoComplete="new-password"
|
autoComplete="new-password"
|
||||||
value={jwtSecret}
|
value={jwtSecret}
|
||||||
onChange={(event) => setJwtSecret(event.target.value)}
|
error={jwtError}
|
||||||
placeholder="New secret..."
|
onChange={(event) => {
|
||||||
|
const val = event.target.value;
|
||||||
|
setJwtSecret(val);
|
||||||
|
if (!val.trim() || val.trim().length >= 32) {
|
||||||
|
setJwtError('');
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
placeholder="Minimum 32 characters to rotate..."
|
||||||
/>
|
/>
|
||||||
</SettingsField>
|
</SettingsField>
|
||||||
<SettingsField label="Payload size limit">
|
<SettingsField label="Payload size limit">
|
||||||
@@ -471,6 +591,39 @@ export function SettingsWorkspace() {
|
|||||||
</SettingsPanel>
|
</SettingsPanel>
|
||||||
</form>
|
</form>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<Modal
|
||||||
|
isOpen={confirmRotateOpen}
|
||||||
|
onClose={() => !saving && setConfirmRotateOpen(false)}
|
||||||
|
title="Confirm JWT secret rotation"
|
||||||
|
maxWidth="xs"
|
||||||
|
footer={
|
||||||
|
<>
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
variant="ghost"
|
||||||
|
size="xs"
|
||||||
|
disabled={saving}
|
||||||
|
onClick={() => setConfirmRotateOpen(false)}
|
||||||
|
>
|
||||||
|
Cancel
|
||||||
|
</Button>
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
variant="danger"
|
||||||
|
size="xs"
|
||||||
|
loading={saving}
|
||||||
|
onClick={() => void executeSave()}
|
||||||
|
>
|
||||||
|
Rotate & Save
|
||||||
|
</Button>
|
||||||
|
</>
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<p className="text-[12px] text-[var(--text-muted)]">
|
||||||
|
Rotating the signing JWT secret will immediately re-sign registered client tokens and invalidate all older SDK JWT tokens and active WebSocket sessions. Continue?
|
||||||
|
</p>
|
||||||
|
</Modal>
|
||||||
</PageShell>
|
</PageShell>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -6,6 +6,7 @@ export interface ClientItem {
|
|||||||
totalEvents: number;
|
totalEvents: number;
|
||||||
lastActive?: string;
|
lastActive?: string;
|
||||||
createdAt: string;
|
createdAt: string;
|
||||||
|
tokenPreview?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ClientCredentials extends ClientItem {
|
export interface ClientCredentials extends ClientItem {
|
||||||
@@ -13,4 +14,9 @@ export interface ClientCredentials extends ClientItem {
|
|||||||
token: string;
|
token: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type ClientStatusFilter = "all" | "online" | "offline" | "pending";
|
export type ClientStatusFilter =
|
||||||
|
| "all"
|
||||||
|
| "online"
|
||||||
|
| "offline"
|
||||||
|
| "pending"
|
||||||
|
| "revoked";
|
||||||
+41
-29
@@ -10,6 +10,46 @@ export interface ButtonProps extends React.ButtonHTMLAttributes<HTMLButtonElemen
|
|||||||
loading?: boolean;
|
loading?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const BUTTON_BASE_STYLES =
|
||||||
|
'inline-flex items-center justify-center font-normal transition-colors select-none cursor-pointer disabled:opacity-50 disabled:cursor-not-allowed border border-transparent whitespace-nowrap outline-none';
|
||||||
|
|
||||||
|
const BUTTON_VARIANT_STYLES = {
|
||||||
|
primary:
|
||||||
|
'bg-[var(--text)] text-[var(--bg)] hover:opacity-90 border-[var(--text)]',
|
||||||
|
secondary:
|
||||||
|
'bg-[var(--bg-elevated)] text-[var(--text)] border-[var(--border-strong)] hover:bg-[var(--bg-hover)]',
|
||||||
|
ghost:
|
||||||
|
'bg-transparent text-[var(--text-muted)] hover:text-[var(--text)] hover:bg-[var(--bg-hover)] border-transparent',
|
||||||
|
danger:
|
||||||
|
'bg-transparent text-[var(--danger)] border-[var(--danger)] hover:bg-[var(--danger-soft)]',
|
||||||
|
icon:
|
||||||
|
'bg-transparent text-[var(--text-muted)] hover:text-[var(--text)] hover:bg-[var(--bg-hover)] border-transparent p-0',
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
const BUTTON_SIZE_STYLES = {
|
||||||
|
default: 'min-h-8.5 px-3 text-[12px] gap-1.5',
|
||||||
|
sm: 'h-7.5 px-2.5 text-[11.5px] gap-1.5',
|
||||||
|
xs: 'h-7 px-2 text-[11px] gap-1',
|
||||||
|
icon: 'w-7.5 h-7.5 p-0',
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
export function buttonClassNames({
|
||||||
|
variant = 'secondary',
|
||||||
|
size = 'default',
|
||||||
|
className,
|
||||||
|
}: {
|
||||||
|
variant?: keyof typeof BUTTON_VARIANT_STYLES;
|
||||||
|
size?: keyof typeof BUTTON_SIZE_STYLES;
|
||||||
|
className?: string;
|
||||||
|
} = {}): string {
|
||||||
|
return cn(
|
||||||
|
BUTTON_BASE_STYLES,
|
||||||
|
BUTTON_VARIANT_STYLES[variant],
|
||||||
|
size === 'icon' ? BUTTON_SIZE_STYLES.icon : BUTTON_SIZE_STYLES[size],
|
||||||
|
className
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export const Button = React.forwardRef<HTMLButtonElement, ButtonProps>(
|
export const Button = React.forwardRef<HTMLButtonElement, ButtonProps>(
|
||||||
(
|
(
|
||||||
{
|
{
|
||||||
@@ -25,39 +65,11 @@ export const Button = React.forwardRef<HTMLButtonElement, ButtonProps>(
|
|||||||
},
|
},
|
||||||
ref
|
ref
|
||||||
) => {
|
) => {
|
||||||
const baseStyles =
|
|
||||||
'inline-flex items-center justify-center font-normal transition-colors select-none cursor-pointer disabled:opacity-50 disabled:cursor-not-allowed border border-transparent whitespace-nowrap outline-none';
|
|
||||||
|
|
||||||
const variantStyles = {
|
|
||||||
primary:
|
|
||||||
'bg-[var(--text)] text-[var(--bg)] hover:opacity-90 border-[var(--text)]',
|
|
||||||
secondary:
|
|
||||||
'bg-[var(--bg-elevated)] text-[var(--text)] border-[var(--border-strong)] hover:bg-[var(--bg-hover)]',
|
|
||||||
ghost:
|
|
||||||
'bg-transparent text-[var(--text-muted)] hover:text-[var(--text)] hover:bg-[var(--bg-hover)] border-transparent',
|
|
||||||
danger:
|
|
||||||
'bg-transparent text-[var(--danger)] border-[var(--danger)] hover:bg-[var(--danger-soft)]',
|
|
||||||
icon:
|
|
||||||
'bg-transparent text-[var(--text-muted)] hover:text-[var(--text)] hover:bg-[var(--bg-hover)] border-transparent p-0',
|
|
||||||
};
|
|
||||||
|
|
||||||
const sizeStyles = {
|
|
||||||
default: 'min-h-8.5 px-3 text-[12px] gap-1.5',
|
|
||||||
sm: 'h-7.5 px-2.5 text-[11.5px] gap-1.5',
|
|
||||||
xs: 'h-7 px-2 text-[11px] gap-1',
|
|
||||||
icon: 'w-7.5 h-7.5 p-0',
|
|
||||||
};
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<button
|
<button
|
||||||
ref={ref}
|
ref={ref}
|
||||||
disabled={disabled || loading}
|
disabled={disabled || loading}
|
||||||
className={cn(
|
className={buttonClassNames({ variant, size, className })}
|
||||||
baseStyles,
|
|
||||||
variantStyles[variant],
|
|
||||||
size === 'icon' ? sizeStyles.icon : sizeStyles[size],
|
|
||||||
className
|
|
||||||
)}
|
|
||||||
{...props}
|
{...props}
|
||||||
>
|
>
|
||||||
{loading ? (
|
{loading ? (
|
||||||
|
|||||||
@@ -73,6 +73,31 @@ export class DatabaseEngine {
|
|||||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||||
)
|
)
|
||||||
`),
|
`),
|
||||||
|
pool.query(`
|
||||||
|
CREATE TABLE IF NOT EXISTS deleted_clients (
|
||||||
|
client_id TEXT PRIMARY KEY,
|
||||||
|
deleted_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||||
|
)
|
||||||
|
`),
|
||||||
|
pool.query(`
|
||||||
|
CREATE TABLE IF NOT EXISTS admin_users (
|
||||||
|
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
|
||||||
|
email TEXT NOT NULL UNIQUE,
|
||||||
|
password_salt BYTEA NOT NULL,
|
||||||
|
password_hash BYTEA NOT NULL,
|
||||||
|
token_version INTEGER NOT NULL DEFAULT 1,
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||||
|
)
|
||||||
|
`),
|
||||||
|
pool.query(`
|
||||||
|
CREATE TABLE IF NOT EXISTS admin_login_attempts (
|
||||||
|
key_hash TEXT PRIMARY KEY,
|
||||||
|
window_started_at TIMESTAMPTZ NOT NULL,
|
||||||
|
failed_attempts INTEGER NOT NULL,
|
||||||
|
blocked_until TIMESTAMPTZ,
|
||||||
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||||
|
)
|
||||||
|
`),
|
||||||
]).then(() => undefined);
|
]).then(() => undefined);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -1,3 +1,38 @@
|
|||||||
|
import { createHash } from "node:crypto";
|
||||||
|
import { AuthService } from "../auth/jwt";
|
||||||
|
|
||||||
|
export function resolveInternalIngestionSecret(
|
||||||
|
jwtSecretOverride?: string,
|
||||||
|
): string {
|
||||||
|
const explicit = (process.env.DATASTACK_INTERNAL_SECRET || "").trim();
|
||||||
|
const isProd = process.env.NODE_ENV === "production";
|
||||||
|
if (isProd) {
|
||||||
|
if (explicit.length >= 32) return explicit;
|
||||||
|
if (explicit.length > 0) return "";
|
||||||
|
} else if (explicit.length > 0) {
|
||||||
|
return explicit;
|
||||||
|
}
|
||||||
|
|
||||||
|
const activeJwt = String(
|
||||||
|
jwtSecretOverride ?? AuthService.getSecret() ?? "",
|
||||||
|
).trim();
|
||||||
|
const minJwtLen = isProd ? 32 : 1;
|
||||||
|
if (activeJwt.length < minJwtLen) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
return createHash("sha256")
|
||||||
|
.update(`datastack-internal:${activeJwt}`)
|
||||||
|
.digest("hex");
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getInternalIngestionHeaders(
|
||||||
|
jwtSecretOverride?: string,
|
||||||
|
): Record<string, string> {
|
||||||
|
const secret = resolveInternalIngestionSecret(jwtSecretOverride);
|
||||||
|
return secret ? { "x-datastack-internal-secret": secret } : {};
|
||||||
|
}
|
||||||
|
|
||||||
export interface ValidationResult {
|
export interface ValidationResult {
|
||||||
allowed: boolean;
|
allowed: boolean;
|
||||||
reason?: string;
|
reason?: string;
|
||||||
@@ -130,7 +165,7 @@ export class SecurityEngine {
|
|||||||
if (!withinLimit) {
|
if (!withinLimit) {
|
||||||
return {
|
return {
|
||||||
allowed: false,
|
allowed: false,
|
||||||
reason: `Rate limit of ${clientMaxRate} events/sec exceeded for client ${clientId}`,
|
reason: `Rate limit of ${effectiveRateLimit} events/sec exceeded for client ${clientId}`,
|
||||||
code: "RATE_LIMIT_EXCEEDED",
|
code: "RATE_LIMIT_EXCEEDED",
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -144,6 +179,18 @@ export class SecurityEngine {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public static getStats() {
|
public static getStats() {
|
||||||
|
const now = Date.now();
|
||||||
|
for (const [k, timestamp] of this.idempotencySet.entries()) {
|
||||||
|
if (now - timestamp > this.IDEMPOTENCY_TTL_MS) {
|
||||||
|
this.idempotencySet.delete(k);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const [clientId, entry] of this.rateLimitMap.entries()) {
|
||||||
|
if (now - entry.windowStart > 10000) {
|
||||||
|
this.rateLimitMap.delete(clientId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
activeIdempotencyKeys: this.idempotencySet.size,
|
activeIdempotencyKeys: this.idempotencySet.size,
|
||||||
rateLimitedClientsCount: this.rateLimitMap.size,
|
rateLimitedClientsCount: this.rateLimitMap.size,
|
||||||
|
|||||||
+392
-53
@@ -19,10 +19,30 @@ export interface StorageFileMeta {
|
|||||||
modifiedAt: Date;
|
modifiedAt: Date;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface RowCountCacheEntry {
|
||||||
|
sizeBytes: number;
|
||||||
|
mtimeMs: number;
|
||||||
|
linesCount: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
const LOCK_RETRY_MIN_MS = 5;
|
||||||
|
const LOCK_RETRY_MAX_MS = 18;
|
||||||
|
const LOCK_TIMEOUT_MS = 10_000;
|
||||||
|
const STALE_LOCK_AGE_MS = 30_000;
|
||||||
|
|
||||||
|
function sleepMs(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
|
||||||
export class CSVStorageManager {
|
export class CSVStorageManager {
|
||||||
// Base storage directory: /storage/data
|
// Base storage directory: /storage/data
|
||||||
public static baseStorageDir = path.resolve(process.cwd(), 'storage', 'data');
|
public static baseStorageDir = path.resolve(process.cwd(), 'storage', 'data');
|
||||||
public static maxFileSizeBytes = 100 * 1024 * 1024; // 100MB rotation limit
|
public static maxFileSizeBytes = 100 * 1024 * 1024; // 100MB rotation limit
|
||||||
|
private static rowCountCache = new Map<string, RowCountCacheEntry>();
|
||||||
|
|
||||||
|
public static clearRowCountCache(): void {
|
||||||
|
this.rowCountCache.clear();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Ensures client directory exists
|
* Ensures client directory exists
|
||||||
@@ -37,15 +57,194 @@ export class CSVStorageManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Resolves target CSV file path with daily and 100MB rotation rules
|
* Acquires an exclusive cooperative lock file for a client+date partition across Port 3000 & Port 4000
|
||||||
|
*/
|
||||||
|
public static async withPartitionLock<T>(
|
||||||
|
clientId: string,
|
||||||
|
dateStr: string,
|
||||||
|
fn: () => Promise<T> | T,
|
||||||
|
): Promise<T> {
|
||||||
|
const clientDir = this.ensureClientDir(clientId);
|
||||||
|
const date = this.extractSafePartitionDate(dateStr);
|
||||||
|
const lockPath = path.join(clientDir, `.${date}.csv.lock`);
|
||||||
|
const deadline = Date.now() + LOCK_TIMEOUT_MS;
|
||||||
|
|
||||||
|
let acquired = false;
|
||||||
|
while (Date.now() < deadline) {
|
||||||
|
try {
|
||||||
|
const fd = fs.openSync(lockPath, 'wx');
|
||||||
|
try {
|
||||||
|
fs.writeFileSync(fd, `${process.pid}:${Date.now()}`, 'utf8');
|
||||||
|
} finally {
|
||||||
|
fs.closeSync(fd);
|
||||||
|
}
|
||||||
|
acquired = true;
|
||||||
|
break;
|
||||||
|
} catch (err: unknown) {
|
||||||
|
const code = (err as NodeJS.ErrnoException)?.code;
|
||||||
|
if (code !== 'EEXIST') {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const lockStat = fs.statSync(lockPath);
|
||||||
|
if (Date.now() - lockStat.mtimeMs > STALE_LOCK_AGE_MS) {
|
||||||
|
fs.unlinkSync(lockPath);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Lock was released concurrently
|
||||||
|
}
|
||||||
|
const jitter =
|
||||||
|
LOCK_RETRY_MIN_MS +
|
||||||
|
Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS));
|
||||||
|
await sleepMs(jitter);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!acquired) {
|
||||||
|
throw new Error(`Timed out waiting for CSV partition lock: ${lockPath}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await fn();
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
if (fs.existsSync(lockPath)) {
|
||||||
|
fs.unlinkSync(lockPath);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Ignore cleanup failure if already unlinked
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Normalizes and validates any timestamp or date string into a strict YYYY-MM-DD partition key
|
||||||
|
*/
|
||||||
|
public static extractSafePartitionDate(rawTimestamp?: string): string {
|
||||||
|
const fallback = new Date().toISOString().slice(0, 10);
|
||||||
|
if (!rawTimestamp || typeof rawTimestamp !== 'string') {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
const trimmed = rawTimestamp.trim();
|
||||||
|
const prefix = trimmed.slice(0, 10);
|
||||||
|
if (/^\d{4}-\d{2}-\d{2}$/.test(prefix)) {
|
||||||
|
const parsedPrefix = new Date(`${prefix}T00:00:00.000Z`);
|
||||||
|
if (!Number.isNaN(parsedPrefix.getTime())) {
|
||||||
|
return prefix;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (/^\d+(\.\d+)?$/.test(trimmed)) {
|
||||||
|
const num = Number(trimmed);
|
||||||
|
const ms = num < 1e11 ? num * 1000 : num;
|
||||||
|
const d = new Date(ms);
|
||||||
|
if (!Number.isNaN(d.getTime())) {
|
||||||
|
return d.toISOString().slice(0, 10);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const parsed = new Date(trimmed);
|
||||||
|
if (!Number.isNaN(parsed.getTime())) {
|
||||||
|
return parsed.toISOString().slice(0, 10);
|
||||||
|
}
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Stateful RFC 4180 quote-aware chunked row counter so multi-100MB CSVs never load wholly into RAM
|
||||||
|
*/
|
||||||
|
private static createChunkRowCounter() {
|
||||||
|
let inQuotes = false;
|
||||||
|
let pendingQuote = false;
|
||||||
|
let currentLineHasChars = false;
|
||||||
|
let totalRows = 0;
|
||||||
|
|
||||||
|
return {
|
||||||
|
pushChunk(chunk: Buffer | string) {
|
||||||
|
const str = typeof chunk === 'string' ? chunk : chunk.toString('utf8');
|
||||||
|
for (let i = 0; i < str.length; i += 1) {
|
||||||
|
const ch = str[i];
|
||||||
|
if (pendingQuote) {
|
||||||
|
pendingQuote = false;
|
||||||
|
if (ch === '"') {
|
||||||
|
// Escaped quote "" across chunk boundary or inline
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
inQuotes = !inQuotes;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ch === '"') {
|
||||||
|
currentLineHasChars = true;
|
||||||
|
if (inQuotes) {
|
||||||
|
if (i + 1 < str.length) {
|
||||||
|
if (str[i + 1] === '"') {
|
||||||
|
i += 1;
|
||||||
|
} else {
|
||||||
|
inQuotes = false;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
pendingQuote = true;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
inQuotes = true;
|
||||||
|
}
|
||||||
|
} else if (ch === '\n' && !inQuotes) {
|
||||||
|
if (currentLineHasChars) {
|
||||||
|
totalRows += 1;
|
||||||
|
currentLineHasChars = false;
|
||||||
|
}
|
||||||
|
} else if (ch !== '\r' && ch !== ' ' && ch !== '\t') {
|
||||||
|
currentLineHasChars = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
finish(): number {
|
||||||
|
if (currentLineHasChars) {
|
||||||
|
totalRows += 1;
|
||||||
|
}
|
||||||
|
return Math.max(0, totalRows - 1);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static countCsvDataRowsChunkedSync(filePath: string): number {
|
||||||
|
const counter = this.createChunkRowCounter();
|
||||||
|
const buf = Buffer.allocUnsafe(64 * 1024);
|
||||||
|
const fd = fs.openSync(filePath, 'r');
|
||||||
|
try {
|
||||||
|
let bytesRead = 0;
|
||||||
|
while ((bytesRead = fs.readSync(fd, buf, 0, buf.length, null)) > 0) {
|
||||||
|
counter.pushChunk(buf.subarray(0, bytesRead));
|
||||||
|
}
|
||||||
|
return counter.finish();
|
||||||
|
} finally {
|
||||||
|
fs.closeSync(fd);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async countCsvDataRowsStreamAsync(
|
||||||
|
filePath: string,
|
||||||
|
): Promise<number> {
|
||||||
|
const counter = this.createChunkRowCounter();
|
||||||
|
return new Promise<number>((resolve, reject) => {
|
||||||
|
const stream = fs.createReadStream(filePath, {
|
||||||
|
highWaterMark: 64 * 1024,
|
||||||
|
});
|
||||||
|
stream.on('data', (chunk) => counter.pushChunk(chunk));
|
||||||
|
stream.on('end', () => resolve(counter.finish()));
|
||||||
|
stream.on('error', (err) => reject(err));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolves target CSV file path with daily and configured MB rotation rules
|
||||||
*/
|
*/
|
||||||
public static getTargetFilePath(clientId: string, dateStr?: string): string {
|
public static getTargetFilePath(clientId: string, dateStr?: string): string {
|
||||||
const clientDir = this.ensureClientDir(clientId);
|
const clientDir = this.ensureClientDir(clientId);
|
||||||
const date = dateStr || new Date().toISOString().split('T')[0];
|
const date = this.extractSafePartitionDate(dateStr);
|
||||||
let baseFileName = `${date}.csv`;
|
let baseFileName = `${date}.csv`;
|
||||||
let targetPath = path.join(clientDir, baseFileName);
|
let targetPath = path.join(clientDir, baseFileName);
|
||||||
|
|
||||||
// Check for 100MB rotation
|
// Check for file rotation
|
||||||
let partIndex = 1;
|
let partIndex = 1;
|
||||||
while (fs.existsSync(targetPath)) {
|
while (fs.existsSync(targetPath)) {
|
||||||
try {
|
try {
|
||||||
@@ -66,10 +265,12 @@ export class CSVStorageManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Batch append events into target CSV file.
|
* Batch append events into target CSV file under cooperative partition lock.
|
||||||
* If file doesn't exist, writes header: clientId,eventId,timestamp,payload
|
* If file doesn't exist, writes header: clientId,eventId,timestamp,payload
|
||||||
*/
|
*/
|
||||||
public static async appendBatch(events: DataEvent[]): Promise<{ written: number; filesUpdated: string[] }> {
|
public static async appendBatch(
|
||||||
|
events: DataEvent[],
|
||||||
|
): Promise<{ written: number; filesUpdated: string[] }> {
|
||||||
if (!events || events.length === 0) {
|
if (!events || events.length === 0) {
|
||||||
return { written: 0, filesUpdated: [] };
|
return { written: 0, filesUpdated: [] };
|
||||||
}
|
}
|
||||||
@@ -77,7 +278,7 @@ export class CSVStorageManager {
|
|||||||
// Group events by client + date
|
// Group events by client + date
|
||||||
const grouped = new Map<string, DataEvent[]>();
|
const grouped = new Map<string, DataEvent[]>();
|
||||||
for (const evt of events) {
|
for (const evt of events) {
|
||||||
const date = (evt.timestamp || new Date().toISOString()).split('T')[0];
|
const date = this.extractSafePartitionDate(evt.timestamp);
|
||||||
const key = `${evt.clientId}:::${date}`;
|
const key = `${evt.clientId}:::${date}`;
|
||||||
if (!grouped.has(key)) {
|
if (!grouped.has(key)) {
|
||||||
grouped.set(key, []);
|
grouped.set(key, []);
|
||||||
@@ -90,31 +291,39 @@ export class CSVStorageManager {
|
|||||||
|
|
||||||
for (const [key, batch] of grouped.entries()) {
|
for (const [key, batch] of grouped.entries()) {
|
||||||
const [clientId, date] = key.split(':::');
|
const [clientId, date] = key.split(':::');
|
||||||
const targetPath = this.getTargetFilePath(clientId, date);
|
const targetPath = await this.withPartitionLock(clientId, date, () => {
|
||||||
let hasContent = false;
|
const resolvedPath = this.getTargetFilePath(clientId, date);
|
||||||
try {
|
let hasContent = false;
|
||||||
hasContent =
|
try {
|
||||||
fs.existsSync(targetPath) && fs.statSync(targetPath).size > 0;
|
hasContent =
|
||||||
} catch {
|
fs.existsSync(resolvedPath) && fs.statSync(resolvedPath).size > 0;
|
||||||
hasContent = false;
|
} catch {
|
||||||
}
|
hasContent = false;
|
||||||
|
}
|
||||||
|
|
||||||
// Format payload as JSON string if not already
|
const cleanBatch = batch.map((item) => ({
|
||||||
const cleanBatch = batch.map((item) => ({
|
clientId: item.clientId,
|
||||||
clientId: item.clientId,
|
eventId: item.eventId,
|
||||||
eventId: item.eventId,
|
timestamp: item.timestamp,
|
||||||
timestamp: item.timestamp,
|
payload:
|
||||||
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
|
typeof item.payload === 'string'
|
||||||
}));
|
? item.payload
|
||||||
|
: JSON.stringify(item.payload),
|
||||||
|
}));
|
||||||
|
|
||||||
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
|
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
|
||||||
const csvData = parser.parse(cleanBatch);
|
const csvData = parser.parse(cleanBatch);
|
||||||
|
|
||||||
if (hasContent) {
|
if (hasContent) {
|
||||||
fs.appendFileSync(targetPath, '\n' + csvData, 'utf8');
|
fs.appendFileSync(resolvedPath, '\n' + csvData, 'utf8');
|
||||||
} else {
|
} else {
|
||||||
fs.writeFileSync(targetPath, csvData, 'utf8');
|
fs.writeFileSync(resolvedPath, csvData, 'utf8');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Invalidate row cache for modified file immediately
|
||||||
|
this.rowCountCache.delete(resolvedPath);
|
||||||
|
return resolvedPath;
|
||||||
|
});
|
||||||
|
|
||||||
filesUpdated.push(targetPath);
|
filesUpdated.push(targetPath);
|
||||||
}
|
}
|
||||||
@@ -123,14 +332,16 @@ export class CSVStorageManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* List all stored CSV files with sizes and dates
|
* List all stored CSV files with cached row counts (invalidated on mtime/size changes)
|
||||||
*/
|
*/
|
||||||
public static listAllFiles(): StorageFileMeta[] {
|
public static listAllFiles(): StorageFileMeta[] {
|
||||||
const results: StorageFileMeta[] = [];
|
const results: StorageFileMeta[] = [];
|
||||||
if (!fs.existsSync(this.baseStorageDir)) {
|
if (!fs.existsSync(this.baseStorageDir)) {
|
||||||
|
this.rowCountCache.clear();
|
||||||
return results;
|
return results;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const activePaths = new Set<string>();
|
||||||
try {
|
try {
|
||||||
const clientDirs = fs.readdirSync(this.baseStorageDir);
|
const clientDirs = fs.readdirSync(this.baseStorageDir);
|
||||||
for (const clientDir of clientDirs) {
|
for (const clientDir of clientDirs) {
|
||||||
@@ -142,15 +353,27 @@ export class CSVStorageManager {
|
|||||||
if (!file.endsWith('.csv')) continue;
|
if (!file.endsWith('.csv')) continue;
|
||||||
const fullFilePath = path.join(fullClientPath, file);
|
const fullFilePath = path.join(fullClientPath, file);
|
||||||
const stats = fs.statSync(fullFilePath);
|
const stats = fs.statSync(fullFilePath);
|
||||||
|
activePaths.add(fullFilePath);
|
||||||
|
|
||||||
// Count lines approximately (or sample)
|
const cached = this.rowCountCache.get(fullFilePath);
|
||||||
let linesCount = 0;
|
let linesCount = 0;
|
||||||
try {
|
if (
|
||||||
const content = fs.readFileSync(fullFilePath, 'utf8');
|
cached &&
|
||||||
linesCount = content.split('\n').filter((l) => l.trim().length > 0).length - 1; // subtract header
|
cached.sizeBytes === stats.size &&
|
||||||
if (linesCount < 0) linesCount = 0;
|
cached.mtimeMs === stats.mtimeMs
|
||||||
} catch {
|
) {
|
||||||
linesCount = 0;
|
linesCount = cached.linesCount;
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
linesCount = this.countCsvDataRowsChunkedSync(fullFilePath);
|
||||||
|
this.rowCountCache.set(fullFilePath, {
|
||||||
|
sizeBytes: stats.size,
|
||||||
|
mtimeMs: stats.mtimeMs,
|
||||||
|
linesCount,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
linesCount = 0;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const datePart = file.replace('.csv', '').split('-part')[0];
|
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||||
@@ -166,6 +389,12 @@ export class CSVStorageManager {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for (const key of this.rowCountCache.keys()) {
|
||||||
|
if (!activePaths.has(key)) {
|
||||||
|
this.rowCountCache.delete(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
} catch {
|
} catch {
|
||||||
// Return whatever collected
|
// Return whatever collected
|
||||||
}
|
}
|
||||||
@@ -174,12 +403,88 @@ export class CSVStorageManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Prune CSV partitions older than autoPruneDays
|
* Non-blocking async listing using fs.promises + streaming row counting on cache misses
|
||||||
|
*/
|
||||||
|
public static async listAllFilesAsync(): Promise<StorageFileMeta[]> {
|
||||||
|
const results: StorageFileMeta[] = [];
|
||||||
|
if (!fs.existsSync(this.baseStorageDir)) {
|
||||||
|
this.rowCountCache.clear();
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
const activePaths = new Set<string>();
|
||||||
|
try {
|
||||||
|
const clientDirs = await fs.promises.readdir(this.baseStorageDir);
|
||||||
|
for (const clientDir of clientDirs) {
|
||||||
|
const fullClientPath = path.join(this.baseStorageDir, clientDir);
|
||||||
|
const dirStat = await fs.promises.stat(fullClientPath).catch(() => null);
|
||||||
|
if (!dirStat || !dirStat.isDirectory()) continue;
|
||||||
|
|
||||||
|
const files = await fs.promises.readdir(fullClientPath);
|
||||||
|
for (const file of files) {
|
||||||
|
if (!file.endsWith('.csv')) continue;
|
||||||
|
const fullFilePath = path.join(fullClientPath, file);
|
||||||
|
const stats = await fs.promises.stat(fullFilePath).catch(() => null);
|
||||||
|
if (!stats) continue;
|
||||||
|
activePaths.add(fullFilePath);
|
||||||
|
|
||||||
|
const cached = this.rowCountCache.get(fullFilePath);
|
||||||
|
let linesCount = 0;
|
||||||
|
if (
|
||||||
|
cached &&
|
||||||
|
cached.sizeBytes === stats.size &&
|
||||||
|
cached.mtimeMs === stats.mtimeMs
|
||||||
|
) {
|
||||||
|
linesCount = cached.linesCount;
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
linesCount = await this.countCsvDataRowsStreamAsync(fullFilePath);
|
||||||
|
this.rowCountCache.set(fullFilePath, {
|
||||||
|
sizeBytes: stats.size,
|
||||||
|
mtimeMs: stats.mtimeMs,
|
||||||
|
linesCount,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
linesCount = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||||
|
|
||||||
|
results.push({
|
||||||
|
clientId: clientDir,
|
||||||
|
date: datePart,
|
||||||
|
filename: file,
|
||||||
|
filePath: fullFilePath,
|
||||||
|
sizeBytes: stats.size,
|
||||||
|
linesCount,
|
||||||
|
modifiedAt: stats.mtime,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const key of this.rowCountCache.keys()) {
|
||||||
|
if (!activePaths.has(key)) {
|
||||||
|
this.rowCountCache.delete(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Return whatever collected
|
||||||
|
}
|
||||||
|
|
||||||
|
return results.sort((a, b) => b.modifiedAt.getTime() - a.modifiedAt.getTime());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Prune CSV partitions older than autoPruneDays without counting row contents
|
||||||
*/
|
*/
|
||||||
public static pruneExpiredPartitions(autoPruneDays: number): number {
|
public static pruneExpiredPartitions(autoPruneDays: number): number {
|
||||||
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
|
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
if (!fs.existsSync(this.baseStorageDir)) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
const cutoffDate = new Date(
|
const cutoffDate = new Date(
|
||||||
Date.now() - Math.floor(autoPruneDays) * 86_400_000,
|
Date.now() - Math.floor(autoPruneDays) * 86_400_000,
|
||||||
@@ -188,28 +493,38 @@ export class CSVStorageManager {
|
|||||||
.split('T')[0];
|
.split('T')[0];
|
||||||
|
|
||||||
let prunedCount = 0;
|
let prunedCount = 0;
|
||||||
const files = this.listAllFiles();
|
try {
|
||||||
for (const file of files) {
|
const clientDirs = fs.readdirSync(this.baseStorageDir);
|
||||||
if (/^\d{4}-\d{2}-\d{2}$/.test(file.date) && file.date < cutoffDate) {
|
for (const clientDir of clientDirs) {
|
||||||
try {
|
const fullClientPath = path.join(this.baseStorageDir, clientDir);
|
||||||
if (fs.existsSync(file.filePath)) {
|
if (!fs.statSync(fullClientPath).isDirectory()) continue;
|
||||||
fs.unlinkSync(file.filePath);
|
|
||||||
prunedCount += 1;
|
const files = fs.readdirSync(fullClientPath);
|
||||||
|
for (const file of files) {
|
||||||
|
if (!file.endsWith('.csv')) continue;
|
||||||
|
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||||
|
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
|
||||||
|
const fullPath = path.join(fullClientPath, file);
|
||||||
|
try {
|
||||||
|
if (fs.existsSync(fullPath)) {
|
||||||
|
fs.unlinkSync(fullPath);
|
||||||
|
this.rowCountCache.delete(fullPath);
|
||||||
|
prunedCount += 1;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Ignore file removal error
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} catch {
|
|
||||||
// Ignore file removal error
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} catch {
|
||||||
|
// Ignore directory scan failure
|
||||||
}
|
}
|
||||||
|
|
||||||
return prunedCount;
|
return prunedCount;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
private static formatStorageStats(files: StorageFileMeta[]) {
|
||||||
* Get aggregate storage statistics
|
|
||||||
*/
|
|
||||||
public static getStorageStats() {
|
|
||||||
const files = this.listAllFiles();
|
|
||||||
const totalBytes = files.reduce((acc, f) => acc + f.sizeBytes, 0);
|
const totalBytes = files.reduce((acc, f) => acc + f.sizeBytes, 0);
|
||||||
const totalRows = files.reduce((acc, f) => acc + f.linesCount, 0);
|
const totalRows = files.reduce((acc, f) => acc + f.linesCount, 0);
|
||||||
const uniqueClients = new Set(files.map((f) => f.clientId)).size;
|
const uniqueClients = new Set(files.map((f) => f.clientId)).size;
|
||||||
@@ -231,10 +546,34 @@ export class CSVStorageManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Reads file content for export / download
|
* Get aggregate storage statistics (sync cached)
|
||||||
|
*/
|
||||||
|
public static getStorageStats() {
|
||||||
|
return this.formatStorageStats(this.listAllFiles());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get aggregate storage statistics (non-blocking async)
|
||||||
|
*/
|
||||||
|
public static async getStorageStatsAsync() {
|
||||||
|
const files = await this.listAllFilesAsync();
|
||||||
|
return this.formatStorageStats(files);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads file content for export / download with path traversal protection
|
||||||
*/
|
*/
|
||||||
public static getFileContent(clientId: string, filename: string): string | null {
|
public static getFileContent(clientId: string, filename: string): string | null {
|
||||||
const fullPath = path.join(this.baseStorageDir, clientId, filename);
|
const safeClientId = String(clientId || '').replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||||
|
const safeFilename = path.basename(String(filename || ''));
|
||||||
|
if (!safeClientId || !safeFilename || !safeFilename.endsWith('.csv')) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const fullPath = path.resolve(this.baseStorageDir, safeClientId, safeFilename);
|
||||||
|
const resolvedBase = path.resolve(this.baseStorageDir);
|
||||||
|
if (!fullPath.startsWith(resolvedBase + path.sep)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
if (fs.existsSync(fullPath)) {
|
if (fs.existsSync(fullPath)) {
|
||||||
return fs.readFileSync(fullPath, 'utf8');
|
return fs.readFileSync(fullPath, 'utf8');
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,8 +1,11 @@
|
|||||||
import { AuthService } from "../auth/jwt";
|
import { AuthService, resolveDefaultJwtSecret } from "../auth/jwt";
|
||||||
import { DataBufferEngine } from "../data/buffer";
|
import { DataBufferEngine } from "../data/buffer";
|
||||||
import { DatabaseEngine } from "../Database/database";
|
import { DatabaseEngine } from "../Database/database";
|
||||||
import { SDKManager } from "../sdk/sdkManager";
|
import { SDKManager } from "../sdk/sdkManager";
|
||||||
import { SecurityEngine } from "../Security/security";
|
import {
|
||||||
|
SecurityEngine,
|
||||||
|
getInternalIngestionHeaders,
|
||||||
|
} from "../Security/security";
|
||||||
import { CSVStorageManager } from "../Storage/csvStorage";
|
import { CSVStorageManager } from "../Storage/csvStorage";
|
||||||
|
|
||||||
export interface AdminSettings {
|
export interface AdminSettings {
|
||||||
@@ -35,9 +38,7 @@ function getDefaultSettings(): AdminSettings {
|
|||||||
batchFlushThreshold: 500,
|
batchFlushThreshold: 500,
|
||||||
batchFlushIntervalMs: 2000,
|
batchFlushIntervalMs: 2000,
|
||||||
fileRotationMb: 100,
|
fileRotationMb: 100,
|
||||||
jwtSecret:
|
jwtSecret: resolveDefaultJwtSecret(),
|
||||||
process.env.DATASTACK_JWT_SECRET ||
|
|
||||||
"datastack-jwt-default-secret-key-32-chars-long",
|
|
||||||
enableIdempotency: true,
|
enableIdempotency: true,
|
||||||
autoPruneDays: 30,
|
autoPruneDays: 30,
|
||||||
updatedAt: new Date().toISOString(),
|
updatedAt: new Date().toISOString(),
|
||||||
@@ -62,7 +63,9 @@ export class SettingsManager {
|
|||||||
AuthService.setSecret(settings.jwtSecret);
|
AuthService.setSecret(settings.jwtSecret);
|
||||||
}
|
}
|
||||||
CSVStorageManager.maxFileSizeBytes = settings.fileRotationMb * 1024 * 1024;
|
CSVStorageManager.maxFileSizeBytes = settings.fileRotationMb * 1024 * 1024;
|
||||||
CSVStorageManager.pruneExpiredPartitions(settings.autoPruneDays);
|
if (settings.autoPruneDays > 0) {
|
||||||
|
CSVStorageManager.pruneExpiredPartitions(settings.autoPruneDays);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static async syncFromDatabase(): Promise<AdminSettings | null> {
|
private static async syncFromDatabase(): Promise<AdminSettings | null> {
|
||||||
@@ -83,6 +86,10 @@ export class SettingsManager {
|
|||||||
|
|
||||||
if (result.rowCount && result.rowCount > 0) {
|
if (result.rowCount && result.rowCount > 0) {
|
||||||
const row = result.rows[0];
|
const row = result.rows[0];
|
||||||
|
const parsedPruneDays =
|
||||||
|
row.auto_prune_days !== undefined && row.auto_prune_days !== null
|
||||||
|
? Number(row.auto_prune_days)
|
||||||
|
: NaN;
|
||||||
const loaded: AdminSettings = {
|
const loaded: AdminSettings = {
|
||||||
serverPort: Number(row.server_port) || 4000,
|
serverPort: Number(row.server_port) || 4000,
|
||||||
webPort: Number(row.web_port) || 3000,
|
webPort: Number(row.web_port) || 3000,
|
||||||
@@ -91,12 +98,9 @@ export class SettingsManager {
|
|||||||
batchFlushThreshold: Number(row.batch_flush_threshold) || 500,
|
batchFlushThreshold: Number(row.batch_flush_threshold) || 500,
|
||||||
batchFlushIntervalMs: Number(row.batch_flush_interval_ms) || 2000,
|
batchFlushIntervalMs: Number(row.batch_flush_interval_ms) || 2000,
|
||||||
fileRotationMb: Number(row.file_rotation_mb) || 100,
|
fileRotationMb: Number(row.file_rotation_mb) || 100,
|
||||||
jwtSecret:
|
jwtSecret: String(row.jwt_secret || "").trim() || resolveDefaultJwtSecret(),
|
||||||
String(row.jwt_secret || "") ||
|
|
||||||
process.env.DATASTACK_JWT_SECRET ||
|
|
||||||
"datastack-jwt-default-secret-key-32-chars-long",
|
|
||||||
enableIdempotency: Boolean(row.enable_idempotency),
|
enableIdempotency: Boolean(row.enable_idempotency),
|
||||||
autoPruneDays: Number(row.auto_prune_days) || 30,
|
autoPruneDays: Number.isFinite(parsedPruneDays) ? parsedPruneDays : 30,
|
||||||
updatedAt: row.updated_at
|
updatedAt: row.updated_at
|
||||||
? new Date(row.updated_at as string | Date).toISOString()
|
? new Date(row.updated_at as string | Date).toISOString()
|
||||||
: new Date().toISOString(),
|
: new Date().toISOString(),
|
||||||
@@ -214,8 +218,46 @@ export class SettingsManager {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static async notifyIngestionSettingsSync(
|
||||||
|
settings: AdminSettings,
|
||||||
|
authJwtSecret: string,
|
||||||
|
jwtSecretChanged: boolean,
|
||||||
|
): Promise<void> {
|
||||||
|
const headers = getInternalIngestionHeaders(authJwtSecret);
|
||||||
|
if (!headers["x-datastack-internal-secret"]) return;
|
||||||
|
|
||||||
|
const ingestionBaseUrl = (
|
||||||
|
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||||
|
).replace(/\/$/, "");
|
||||||
|
|
||||||
|
try {
|
||||||
|
await fetch(`${ingestionBaseUrl}/internal/settings-sync`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
...headers,
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
maxEventsPerSec: settings.maxEventsPerSec,
|
||||||
|
maxPayloadSizeKb: settings.maxPayloadSizeKb,
|
||||||
|
batchFlushThreshold: settings.batchFlushThreshold,
|
||||||
|
batchFlushIntervalMs: settings.batchFlushIntervalMs,
|
||||||
|
fileRotationMb: settings.fileRotationMb,
|
||||||
|
enableIdempotency: settings.enableIdempotency,
|
||||||
|
autoPruneDays: settings.autoPruneDays,
|
||||||
|
...(jwtSecretChanged ? { jwtSecret: settings.jwtSecret } : {}),
|
||||||
|
}),
|
||||||
|
cache: "no-store",
|
||||||
|
signal: AbortSignal.timeout(1500),
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Port 4000 may not be running in isolated single-process dev mode
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public static updateSettings(updates: Partial<AdminSettings>): AdminSettings {
|
public static updateSettings(updates: Partial<AdminSettings>): AdminSettings {
|
||||||
const current = getSettingsState();
|
const current = getSettingsState();
|
||||||
|
const previousJwtSecret = current.jwtSecret;
|
||||||
const jwtSecretChanged =
|
const jwtSecretChanged =
|
||||||
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
|
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
|
||||||
const nextSettings: AdminSettings = {
|
const nextSettings: AdminSettings = {
|
||||||
@@ -229,6 +271,13 @@ export class SettingsManager {
|
|||||||
SDKManager.resignAllClientTokens();
|
SDKManager.resignAllClientTokens();
|
||||||
}
|
}
|
||||||
void this.persistSettingsToDb(nextSettings);
|
void this.persistSettingsToDb(nextSettings);
|
||||||
|
void this.notifyIngestionSettingsSync(
|
||||||
|
nextSettings,
|
||||||
|
previousJwtSecret,
|
||||||
|
jwtSecretChanged,
|
||||||
|
).then(() =>
|
||||||
|
jwtSecretChanged ? SDKManager.syncAllClientsToIngestionAsync() : undefined,
|
||||||
|
);
|
||||||
return { ...nextSettings };
|
return { ...nextSettings };
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -236,6 +285,7 @@ export class SettingsManager {
|
|||||||
updates: Partial<AdminSettings>,
|
updates: Partial<AdminSettings>,
|
||||||
): Promise<AdminSettings> {
|
): Promise<AdminSettings> {
|
||||||
const current = await this.getSettingsAsync();
|
const current = await this.getSettingsAsync();
|
||||||
|
const previousJwtSecret = current.jwtSecret;
|
||||||
const jwtSecretChanged =
|
const jwtSecretChanged =
|
||||||
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
|
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
|
||||||
const nextSettings: AdminSettings = {
|
const nextSettings: AdminSettings = {
|
||||||
@@ -249,6 +299,14 @@ export class SettingsManager {
|
|||||||
await SDKManager.resignAllClientTokensAsync();
|
await SDKManager.resignAllClientTokensAsync();
|
||||||
}
|
}
|
||||||
await this.persistSettingsToDb(nextSettings);
|
await this.persistSettingsToDb(nextSettings);
|
||||||
|
await this.notifyIngestionSettingsSync(
|
||||||
|
nextSettings,
|
||||||
|
previousJwtSecret,
|
||||||
|
jwtSecretChanged,
|
||||||
|
);
|
||||||
|
if (jwtSecretChanged) {
|
||||||
|
await SDKManager.syncAllClientsToIngestionAsync();
|
||||||
|
}
|
||||||
return { ...nextSettings };
|
return { ...nextSettings };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,44 +1,93 @@
|
|||||||
|
import { SettingsManager } from '../adminSettings/settingsManager';
|
||||||
import { SDKManager } from '../sdk/sdkManager';
|
import { SDKManager } from '../sdk/sdkManager';
|
||||||
import { CSVStorageManager } from '../Storage/csvStorage';
|
import { CSVStorageManager } from '../Storage/csvStorage';
|
||||||
import { DataBufferEngine } from '../data/buffer';
|
import { DataBufferEngine } from '../data/buffer';
|
||||||
import { SecurityEngine } from '../Security/security';
|
import {
|
||||||
|
SecurityEngine,
|
||||||
|
getInternalIngestionHeaders,
|
||||||
|
} from '../Security/security';
|
||||||
|
|
||||||
export class ApiEngine {
|
export class ApiEngine {
|
||||||
/**
|
/**
|
||||||
* Retrieves overall telemetry and health metrics for the dashboard
|
* Retrieves overall telemetry and health metrics for the dashboard
|
||||||
*/
|
*/
|
||||||
public static async getDashboardMetrics() {
|
public static async getDashboardMetrics() {
|
||||||
const storageStats = CSVStorageManager.getStorageStats();
|
const settings = await SettingsManager.getSettingsAsync();
|
||||||
const localBufferStats = DataBufferEngine.getBufferStats();
|
const [storageStats, localBufferStats, clients] = await Promise.all([
|
||||||
const clients = await SDKManager.listClientsAsync();
|
CSVStorageManager.getStorageStatsAsync(),
|
||||||
|
DataBufferEngine.getBufferStatsAsync(),
|
||||||
|
SDKManager.listClientsAsync(),
|
||||||
|
]);
|
||||||
const activeClients = clients.filter((c) => c.status === 'active').length;
|
const activeClients = clients.filter((c) => c.status === 'active').length;
|
||||||
let activeSockets = 0;
|
let activeSockets = 0;
|
||||||
let ingestionBufferSize = 0;
|
let ingestionLocalBufferSize = 0;
|
||||||
|
let ingestionRedisBufferSize = 0;
|
||||||
|
let ingestionRedisConnected = false;
|
||||||
|
let remoteSecurityStats: {
|
||||||
|
activeIdempotencyKeys?: number;
|
||||||
|
rateLimitedClientsCount?: number;
|
||||||
|
maxPayloadSizeBytes?: number;
|
||||||
|
} | null = null;
|
||||||
|
|
||||||
const ingestionBaseUrl = (
|
const ingestionBaseUrl = (
|
||||||
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
|
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
|
||||||
).replace(/\/$/, '');
|
).replace(/\/$/, '');
|
||||||
|
const internalHeaders = getInternalIngestionHeaders();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const res = await fetch(`${ingestionBaseUrl}/health`, {
|
const res = await fetch(`${ingestionBaseUrl}/internal/metrics`, {
|
||||||
|
headers: internalHeaders,
|
||||||
cache: 'no-store',
|
cache: 'no-store',
|
||||||
|
signal: AbortSignal.timeout(1000),
|
||||||
});
|
});
|
||||||
if (res.ok) {
|
if (res.ok) {
|
||||||
const data = await res.json();
|
const data = await res.json();
|
||||||
activeSockets = Number(data.activeSockets) || 0;
|
activeSockets = Number(data.activeSockets) || 0;
|
||||||
ingestionBufferSize = Number(data.bufferSize) || 0;
|
ingestionRedisConnected = Boolean(data.redisConnected);
|
||||||
|
if (data.localBufferSize !== undefined) {
|
||||||
|
ingestionLocalBufferSize = Number(data.localBufferSize) || 0;
|
||||||
|
ingestionRedisBufferSize = Number(data.redisBufferSize) || 0;
|
||||||
|
} else {
|
||||||
|
ingestionLocalBufferSize = Number(data.bufferSize) || 0;
|
||||||
|
}
|
||||||
|
if (data.securityStats && typeof data.securityStats === 'object') {
|
||||||
|
remoteSecurityStats = data.securityStats;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
// Ingestion server may not be running in single-process dev mode
|
// Ingestion server may not be running in single-process dev mode
|
||||||
activeSockets = 0;
|
activeSockets = 0;
|
||||||
ingestionBufferSize = 0;
|
ingestionLocalBufferSize = 0;
|
||||||
|
ingestionRedisBufferSize = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
const securityStats = SecurityEngine.getStats();
|
const localSecurityStats = SecurityEngine.getStats();
|
||||||
|
const securityStats = {
|
||||||
|
activeIdempotencyKeys:
|
||||||
|
localSecurityStats.activeIdempotencyKeys +
|
||||||
|
(Number(remoteSecurityStats?.activeIdempotencyKeys) || 0),
|
||||||
|
rateLimitedClientsCount:
|
||||||
|
localSecurityStats.rateLimitedClientsCount +
|
||||||
|
(Number(remoteSecurityStats?.rateLimitedClientsCount) || 0),
|
||||||
|
maxPayloadSizeBytes: Math.min(
|
||||||
|
localSecurityStats.maxPayloadSizeBytes,
|
||||||
|
Number(remoteSecurityStats?.maxPayloadSizeBytes) ||
|
||||||
|
localSecurityStats.maxPayloadSizeBytes,
|
||||||
|
),
|
||||||
|
};
|
||||||
|
|
||||||
|
const effectiveRedisBufferSize = Math.max(
|
||||||
|
localBufferStats.redisBufferSize || 0,
|
||||||
|
ingestionRedisBufferSize,
|
||||||
|
);
|
||||||
const bufferStats = {
|
const bufferStats = {
|
||||||
...localBufferStats,
|
...localBufferStats,
|
||||||
|
redisConnected: localBufferStats.redisConnected || ingestionRedisConnected,
|
||||||
|
redisBufferSize: effectiveRedisBufferSize,
|
||||||
currentBufferSize:
|
currentBufferSize:
|
||||||
localBufferStats.currentBufferSize + ingestionBufferSize,
|
localBufferStats.localBufferSize +
|
||||||
|
ingestionLocalBufferSize +
|
||||||
|
effectiveRedisBufferSize,
|
||||||
};
|
};
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -49,8 +98,8 @@ export class ApiEngine {
|
|||||||
storageStats,
|
storageStats,
|
||||||
securityStats,
|
securityStats,
|
||||||
runtime: {
|
runtime: {
|
||||||
serverPort: 4000,
|
serverPort: settings.serverPort || 4000,
|
||||||
webPort: 3000,
|
webPort: settings.webPort || 3000,
|
||||||
uptimeSeconds: Math.floor(process.uptime()),
|
uptimeSeconds: Math.floor(process.uptime()),
|
||||||
memoryUsageMb: (process.memoryUsage().rss / (1024 * 1024)).toFixed(1),
|
memoryUsageMb: (process.memoryUsage().rss / (1024 * 1024)).toFixed(1),
|
||||||
nodeVersion: process.version,
|
nodeVersion: process.version,
|
||||||
@@ -59,7 +108,7 @@ export class ApiEngine {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Filters and streams stored CSV data for export
|
* Filters and lists stored CSV files for export
|
||||||
*/
|
*/
|
||||||
public static getExportFiles(clientId?: string) {
|
public static getExportFiles(clientId?: string) {
|
||||||
const allFiles = CSVStorageManager.listAllFiles();
|
const allFiles = CSVStorageManager.listAllFiles();
|
||||||
@@ -68,6 +117,14 @@ export class ApiEngine {
|
|||||||
}
|
}
|
||||||
return allFiles;
|
return allFiles;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static async getExportFilesAsync(clientId?: string) {
|
||||||
|
const allFiles = await CSVStorageManager.listAllFilesAsync();
|
||||||
|
if (clientId && clientId !== 'all') {
|
||||||
|
return allFiles.filter((f) => f.clientId === clientId);
|
||||||
|
}
|
||||||
|
return allFiles;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export default ApiEngine;
|
export default ApiEngine;
|
||||||
@@ -1,8 +1,23 @@
|
|||||||
const encoder = new TextEncoder();
|
const encoder = new TextEncoder();
|
||||||
|
|
||||||
interface AdminSessionPayload {
|
const DEV_DEFAULT_SESSION_SECRET =
|
||||||
|
"datastack-admin-session-default-secret-32-chars";
|
||||||
|
|
||||||
|
export interface AdminSessionPayload {
|
||||||
email: string;
|
email: string;
|
||||||
expiresAt: number;
|
expiresAt: number;
|
||||||
|
tokenVersion: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getAdminSessionSecret(): string | null {
|
||||||
|
const configured = process.env.DATASTACK_ADMIN_SESSION_SECRET?.trim() || "";
|
||||||
|
if (process.env.NODE_ENV === "production") {
|
||||||
|
return configured.length >= 32 ? configured : null;
|
||||||
|
}
|
||||||
|
if (configured.length > 0) {
|
||||||
|
return configured.length >= 32 ? configured : null;
|
||||||
|
}
|
||||||
|
return DEV_DEFAULT_SESSION_SECRET;
|
||||||
}
|
}
|
||||||
|
|
||||||
function encodeBase64Url(bytes: Uint8Array) {
|
function encodeBase64Url(bytes: Uint8Array) {
|
||||||
@@ -34,9 +49,14 @@ export async function createAdminSessionToken(
|
|||||||
email: string,
|
email: string,
|
||||||
secret: string,
|
secret: string,
|
||||||
expiresAt: number,
|
expiresAt: number,
|
||||||
|
tokenVersion = 1,
|
||||||
) {
|
) {
|
||||||
|
const safeVersion =
|
||||||
|
Number.isInteger(tokenVersion) && tokenVersion >= 1 ? tokenVersion : 1;
|
||||||
const payload = encodeBase64Url(
|
const payload = encodeBase64Url(
|
||||||
encoder.encode(JSON.stringify({ email, expiresAt })),
|
encoder.encode(
|
||||||
|
JSON.stringify({ email, expiresAt, tokenVersion: safeVersion }),
|
||||||
|
),
|
||||||
);
|
);
|
||||||
const signature = await crypto.subtle.sign(
|
const signature = await crypto.subtle.sign(
|
||||||
"HMAC",
|
"HMAC",
|
||||||
@@ -50,6 +70,7 @@ export async function getAdminSessionPayload(
|
|||||||
token: string,
|
token: string,
|
||||||
secret: string,
|
secret: string,
|
||||||
): Promise<AdminSessionPayload | null> {
|
): Promise<AdminSessionPayload | null> {
|
||||||
|
if (!secret || secret.length < 32) return null;
|
||||||
const [payload, signature, extra] = token.split(".");
|
const [payload, signature, extra] = token.split(".");
|
||||||
if (!payload || !signature || extra) return null;
|
if (!payload || !signature || extra) return null;
|
||||||
|
|
||||||
@@ -68,7 +89,10 @@ export async function getAdminSessionPayload(
|
|||||||
if (
|
if (
|
||||||
typeof session.email === "string" &&
|
typeof session.email === "string" &&
|
||||||
typeof session.expiresAt === "number" &&
|
typeof session.expiresAt === "number" &&
|
||||||
session.expiresAt > Date.now()
|
session.expiresAt > Date.now() &&
|
||||||
|
typeof session.tokenVersion === "number" &&
|
||||||
|
Number.isInteger(session.tokenVersion) &&
|
||||||
|
session.tokenVersion >= 1
|
||||||
) {
|
) {
|
||||||
return session;
|
return session;
|
||||||
}
|
}
|
||||||
|
|||||||
+279
-163
@@ -6,15 +6,22 @@ import {
|
|||||||
timingSafeEqual,
|
timingSafeEqual,
|
||||||
} from "node:crypto";
|
} from "node:crypto";
|
||||||
import { Pool, type PoolClient } from "pg";
|
import { Pool, type PoolClient } from "pg";
|
||||||
|
import {
|
||||||
|
type AdminSessionPayload,
|
||||||
|
getAdminSessionPayload,
|
||||||
|
getAdminSessionSecret,
|
||||||
|
} from "./adminSession";
|
||||||
|
|
||||||
const PASSWORD_HASH_BYTES = 64;
|
const PASSWORD_HASH_BYTES = 64;
|
||||||
const PASSWORD_SALT_BYTES = 16;
|
const PASSWORD_SALT_BYTES = 16;
|
||||||
const DATABASE_LOCK_ID = "72025102501";
|
const DATABASE_LOCK_ID = "72025102501";
|
||||||
const LOGIN_ATTEMPT_LIMIT = 5;
|
const LOGIN_ATTEMPT_LIMIT = 5;
|
||||||
|
const IP_LOGIN_ATTEMPT_LIMIT = 25;
|
||||||
const LOGIN_WINDOW_MS = 15 * 60 * 1000;
|
const LOGIN_WINDOW_MS = 15 * 60 * 1000;
|
||||||
const LOGIN_BLOCK_MS = 15 * 60 * 1000;
|
const LOGIN_BLOCK_MS = 15 * 60 * 1000;
|
||||||
const LOCAL_THROTTLE_MAX_ENTRIES = 5000;
|
const LOCAL_THROTTLE_MAX_ENTRIES = 5000;
|
||||||
const LOCAL_THROTTLE_CLEANUP_MS = 60 * 1000;
|
const LOCAL_THROTTLE_CLEANUP_MS = 60 * 1000;
|
||||||
|
const DEV_DEFAULT_SETUP_KEY = "datastack-default-setup-key-32-chars";
|
||||||
|
|
||||||
interface LoginThrottleEntry {
|
interface LoginThrottleEntry {
|
||||||
windowStartedAt: number;
|
windowStartedAt: number;
|
||||||
@@ -28,11 +35,17 @@ interface LoginThrottleDecision {
|
|||||||
retryAfterSeconds: number;
|
retryAfterSeconds: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface LoginThrottleBucket {
|
||||||
|
keyHash: string;
|
||||||
|
limit: number;
|
||||||
|
}
|
||||||
|
|
||||||
interface InMemoryAdmin {
|
interface InMemoryAdmin {
|
||||||
id: number;
|
id: number;
|
||||||
email: string;
|
email: string;
|
||||||
passwordSalt: Buffer;
|
passwordSalt: Buffer;
|
||||||
passwordHash: Buffer;
|
passwordHash: Buffer;
|
||||||
|
tokenVersion: number;
|
||||||
createdAt: Date;
|
createdAt: Date;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -47,8 +60,12 @@ const localLoginThrottle = new Map<string, LoginThrottleEntry>();
|
|||||||
let lastLocalThrottleCleanup = 0;
|
let lastLocalThrottleCleanup = 0;
|
||||||
let lastDatabaseThrottleCleanup = 0;
|
let lastDatabaseThrottleCleanup = 0;
|
||||||
|
|
||||||
|
function isProductionEnv() {
|
||||||
|
return process.env.NODE_ENV === "production";
|
||||||
|
}
|
||||||
|
|
||||||
function getPool(): Pool | null {
|
function getPool(): Pool | null {
|
||||||
const connectionString = process.env.DATABASE_URL;
|
const connectionString = process.env.DATABASE_URL?.trim();
|
||||||
if (!connectionString) {
|
if (!connectionString) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@@ -74,17 +91,24 @@ async function ensureAdminTable() {
|
|||||||
if (!pool) return;
|
if (!pool) return;
|
||||||
|
|
||||||
globals.__dataStackAdminSchema ??= Promise.all([
|
globals.__dataStackAdminSchema ??= Promise.all([
|
||||||
pool.query(
|
pool
|
||||||
`
|
.query(
|
||||||
|
`
|
||||||
CREATE TABLE IF NOT EXISTS admin_users (
|
CREATE TABLE IF NOT EXISTS admin_users (
|
||||||
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
|
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
|
||||||
email TEXT NOT NULL UNIQUE,
|
email TEXT NOT NULL UNIQUE,
|
||||||
password_salt BYTEA NOT NULL,
|
password_salt BYTEA NOT NULL,
|
||||||
password_hash BYTEA NOT NULL,
|
password_hash BYTEA NOT NULL,
|
||||||
|
token_version INTEGER NOT NULL DEFAULT 1,
|
||||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||||
)
|
)
|
||||||
`,
|
`,
|
||||||
),
|
)
|
||||||
|
.then(() =>
|
||||||
|
pool.query(
|
||||||
|
`ALTER TABLE admin_users ADD COLUMN IF NOT EXISTS token_version INTEGER NOT NULL DEFAULT 1`,
|
||||||
|
),
|
||||||
|
),
|
||||||
pool.query(
|
pool.query(
|
||||||
`
|
`
|
||||||
CREATE TABLE IF NOT EXISTS admin_login_attempts (
|
CREATE TABLE IF NOT EXISTS admin_login_attempts (
|
||||||
@@ -121,8 +145,41 @@ function matchesSecret(candidate: string, expected: string) {
|
|||||||
return timingSafeEqual(candidateHash, expectedHash);
|
return timingSafeEqual(candidateHash, expectedHash);
|
||||||
}
|
}
|
||||||
|
|
||||||
function createLoginThrottleKeys(email: string, secret: string) {
|
export function getAdminSetupKey(): string | null {
|
||||||
return [createHmac("sha256", secret).update(`email:${email}`).digest("hex")];
|
const configured = process.env.DATASTACK_ADMIN_SETUP_KEY?.trim() || "";
|
||||||
|
if (isProductionEnv()) {
|
||||||
|
return configured.length >= 32 ? configured : null;
|
||||||
|
}
|
||||||
|
if (configured.length > 0) {
|
||||||
|
return configured.length >= 32 ? configured : null;
|
||||||
|
}
|
||||||
|
return DEV_DEFAULT_SETUP_KEY;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createLoginThrottleBuckets(
|
||||||
|
email: string,
|
||||||
|
secret: string,
|
||||||
|
ip = "unknown",
|
||||||
|
): LoginThrottleBucket[] {
|
||||||
|
const cleanEmail = email.trim().toLowerCase();
|
||||||
|
const cleanIp = ip.trim().toLowerCase() || "unknown";
|
||||||
|
const buckets: LoginThrottleBucket[] = [
|
||||||
|
{
|
||||||
|
keyHash: createHmac("sha256", secret)
|
||||||
|
.update(`ip_email:${cleanIp}:${cleanEmail}`)
|
||||||
|
.digest("hex"),
|
||||||
|
limit: LOGIN_ATTEMPT_LIMIT,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
if (cleanIp !== "unknown") {
|
||||||
|
buckets.push({
|
||||||
|
keyHash: createHmac("sha256", secret)
|
||||||
|
.update(`ip:${cleanIp}`)
|
||||||
|
.digest("hex"),
|
||||||
|
limit: IP_LOGIN_ATTEMPT_LIMIT,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return buckets;
|
||||||
}
|
}
|
||||||
|
|
||||||
function getAdvisoryLockId(keyHash: string) {
|
function getAdvisoryLockId(keyHash: string) {
|
||||||
@@ -157,14 +214,14 @@ function cleanupLocalLoginThrottle(now: number) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function consumeLocalLoginAttempt(
|
function consumeLocalLoginAttempt(
|
||||||
keyHashes: string[],
|
buckets: LoginThrottleBucket[],
|
||||||
now: number,
|
now: number,
|
||||||
): LoginThrottleDecision {
|
): LoginThrottleDecision {
|
||||||
cleanupLocalLoginThrottle(now);
|
cleanupLocalLoginThrottle(now);
|
||||||
|
|
||||||
const blockedUntil = keyHashes.reduce(
|
const blockedUntil = buckets.reduce(
|
||||||
(latest, keyHash) =>
|
(latest, bucket) =>
|
||||||
Math.max(latest, localLoginThrottle.get(keyHash)?.blockedUntil ?? 0),
|
Math.max(latest, localLoginThrottle.get(bucket.keyHash)?.blockedUntil ?? 0),
|
||||||
0,
|
0,
|
||||||
);
|
);
|
||||||
if (blockedUntil > now) {
|
if (blockedUntil > now) {
|
||||||
@@ -174,16 +231,16 @@ function consumeLocalLoginAttempt(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const keyHash of keyHashes) {
|
for (const bucket of buckets) {
|
||||||
const previous = localLoginThrottle.get(keyHash);
|
const previous = localLoginThrottle.get(bucket.keyHash);
|
||||||
const windowExpired =
|
const windowExpired =
|
||||||
!previous || now - previous.windowStartedAt >= LOGIN_WINDOW_MS;
|
!previous || now - previous.windowStartedAt >= LOGIN_WINDOW_MS;
|
||||||
const failedAttempts = windowExpired ? 1 : previous.failedAttempts + 1;
|
const failedAttempts = windowExpired ? 1 : previous.failedAttempts + 1;
|
||||||
localLoginThrottle.set(keyHash, {
|
localLoginThrottle.set(bucket.keyHash, {
|
||||||
windowStartedAt: windowExpired ? now : previous.windowStartedAt,
|
windowStartedAt: windowExpired ? now : previous.windowStartedAt,
|
||||||
failedAttempts,
|
failedAttempts,
|
||||||
blockedUntil:
|
blockedUntil:
|
||||||
failedAttempts >= LOGIN_ATTEMPT_LIMIT ? now + LOGIN_BLOCK_MS : 0,
|
failedAttempts >= bucket.limit ? now + LOGIN_BLOCK_MS : 0,
|
||||||
updatedAt: now,
|
updatedAt: now,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -192,12 +249,13 @@ function consumeLocalLoginAttempt(
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function consumeDatabaseLoginAttempt(
|
async function consumeDatabaseLoginAttempt(
|
||||||
keyHashes: string[],
|
buckets: LoginThrottleBucket[],
|
||||||
now: number,
|
now: number,
|
||||||
): Promise<LoginThrottleDecision> {
|
): Promise<LoginThrottleDecision> {
|
||||||
const pool = getPool();
|
const pool = getPool();
|
||||||
if (!pool) return consumeLocalLoginAttempt(keyHashes, now);
|
if (!pool) return consumeLocalLoginAttempt(buckets, now);
|
||||||
await ensureAdminTable();
|
await ensureAdminTable();
|
||||||
|
const keyHashes = buckets.map((b) => b.keyHash);
|
||||||
const client = await pool.connect();
|
const client = await pool.connect();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -229,8 +287,8 @@ async function consumeDatabaseLoginAttempt(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const keyHash of keyHashes) {
|
for (const bucket of buckets) {
|
||||||
const previous = entries.get(keyHash);
|
const previous = entries.get(bucket.keyHash);
|
||||||
const windowStartedAt = previous
|
const windowStartedAt = previous
|
||||||
? new Date(previous.window_started_at as string | Date).getTime()
|
? new Date(previous.window_started_at as string | Date).getTime()
|
||||||
: 0;
|
: 0;
|
||||||
@@ -243,7 +301,7 @@ async function consumeDatabaseLoginAttempt(
|
|||||||
? new Date(now)
|
? new Date(now)
|
||||||
: new Date(windowStartedAt);
|
: new Date(windowStartedAt);
|
||||||
const nextBlockedUntil =
|
const nextBlockedUntil =
|
||||||
failedAttempts >= LOGIN_ATTEMPT_LIMIT
|
failedAttempts >= bucket.limit
|
||||||
? new Date(now + LOGIN_BLOCK_MS)
|
? new Date(now + LOGIN_BLOCK_MS)
|
||||||
: null;
|
: null;
|
||||||
|
|
||||||
@@ -256,7 +314,7 @@ async function consumeDatabaseLoginAttempt(
|
|||||||
failed_attempts = EXCLUDED.failed_attempts,
|
failed_attempts = EXCLUDED.failed_attempts,
|
||||||
blocked_until = EXCLUDED.blocked_until,
|
blocked_until = EXCLUDED.blocked_until,
|
||||||
updated_at = NOW()`,
|
updated_at = NOW()`,
|
||||||
[keyHash, nextWindowStartedAt, failedAttempts, nextBlockedUntil],
|
[bucket.keyHash, nextWindowStartedAt, failedAttempts, nextBlockedUntil],
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -279,13 +337,20 @@ async function consumeDatabaseLoginAttempt(
|
|||||||
|
|
||||||
async function getOrInitInMemoryAdmin(): Promise<InMemoryAdmin> {
|
async function getOrInitInMemoryAdmin(): Promise<InMemoryAdmin> {
|
||||||
if (!globals.__dataStackInMemoryAdmin) {
|
if (!globals.__dataStackInMemoryAdmin) {
|
||||||
|
const envPassword = process.env.DATASTACK_ADMIN_PASSWORD;
|
||||||
|
const allowDemo =
|
||||||
|
!isProductionEnv() && process.env.DATASTACK_ALLOW_DEMO_ADMIN !== "false";
|
||||||
|
|
||||||
|
if (!envPassword && !allowDemo) {
|
||||||
|
throw new Error("Admin account has not completed initial setup.");
|
||||||
|
}
|
||||||
|
|
||||||
const defaultEmail = (
|
const defaultEmail = (
|
||||||
process.env.DATASTACK_ADMIN_EMAIL || "admin@datastack.dev"
|
process.env.DATASTACK_ADMIN_EMAIL || "admin@datastack.dev"
|
||||||
)
|
)
|
||||||
.trim()
|
.trim()
|
||||||
.toLowerCase();
|
.toLowerCase();
|
||||||
const defaultPassword =
|
const defaultPassword = envPassword || "admin12345678";
|
||||||
process.env.DATASTACK_ADMIN_PASSWORD || "admin12345678";
|
|
||||||
const salt = randomBytes(PASSWORD_SALT_BYTES);
|
const salt = randomBytes(PASSWORD_SALT_BYTES);
|
||||||
const hash = await derivePasswordHash(defaultPassword, salt);
|
const hash = await derivePasswordHash(defaultPassword, salt);
|
||||||
globals.__dataStackInMemoryAdmin = {
|
globals.__dataStackInMemoryAdmin = {
|
||||||
@@ -293,6 +358,7 @@ async function getOrInitInMemoryAdmin(): Promise<InMemoryAdmin> {
|
|||||||
email: defaultEmail,
|
email: defaultEmail,
|
||||||
passwordSalt: salt,
|
passwordSalt: salt,
|
||||||
passwordHash: hash,
|
passwordHash: hash,
|
||||||
|
tokenVersion: 1,
|
||||||
createdAt: new Date(),
|
createdAt: new Date(),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -302,24 +368,26 @@ async function getOrInitInMemoryAdmin(): Promise<InMemoryAdmin> {
|
|||||||
export async function consumeAdminLoginAttempt(
|
export async function consumeAdminLoginAttempt(
|
||||||
email: string,
|
email: string,
|
||||||
secret: string,
|
secret: string,
|
||||||
|
ip = "unknown",
|
||||||
): Promise<LoginThrottleDecision> {
|
): Promise<LoginThrottleDecision> {
|
||||||
const keyHashes = createLoginThrottleKeys(email, secret);
|
const buckets = createLoginThrottleBuckets(email, secret, ip);
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
|
|
||||||
const pool = getPool();
|
const pool = getPool();
|
||||||
if (!pool) {
|
if (!pool) {
|
||||||
return consumeLocalLoginAttempt(keyHashes, now);
|
return consumeLocalLoginAttempt(buckets, now);
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
return await consumeDatabaseLoginAttempt(buckets, now);
|
||||||
return await consumeDatabaseLoginAttempt(keyHashes, now);
|
|
||||||
} catch {
|
|
||||||
return consumeLocalLoginAttempt(keyHashes, now);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function clearAdminLoginAttempts(email: string, secret: string) {
|
export async function clearAdminLoginAttempts(
|
||||||
const keyHashes = createLoginThrottleKeys(email, secret);
|
email: string,
|
||||||
|
secret: string,
|
||||||
|
ip = "unknown",
|
||||||
|
) {
|
||||||
|
const buckets = createLoginThrottleBuckets(email, secret, ip);
|
||||||
|
const keyHashes = buckets.map((b) => b.keyHash);
|
||||||
|
|
||||||
const pool = getPool();
|
const pool = getPool();
|
||||||
if (!pool) {
|
if (!pool) {
|
||||||
@@ -327,50 +395,45 @@ export async function clearAdminLoginAttempts(email: string, secret: string) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
await ensureAdminTable();
|
||||||
|
const client = await pool.connect();
|
||||||
try {
|
try {
|
||||||
await ensureAdminTable();
|
await client.query("BEGIN");
|
||||||
const client = await pool.connect();
|
await lockLoginThrottleKeys(client, keyHashes);
|
||||||
try {
|
await client.query(
|
||||||
await client.query("BEGIN");
|
"DELETE FROM admin_login_attempts WHERE key_hash = ANY($1::text[])",
|
||||||
await lockLoginThrottleKeys(client, keyHashes);
|
[keyHashes],
|
||||||
await client.query(
|
);
|
||||||
"DELETE FROM admin_login_attempts WHERE key_hash = ANY($1::text[])",
|
await client.query("COMMIT");
|
||||||
[keyHashes],
|
} catch (error) {
|
||||||
);
|
await client.query("ROLLBACK").catch(() => undefined);
|
||||||
await client.query("COMMIT");
|
throw error;
|
||||||
} catch (error) {
|
} finally {
|
||||||
await client.query("ROLLBACK").catch(() => undefined);
|
client.release();
|
||||||
throw error;
|
|
||||||
} finally {
|
|
||||||
client.release();
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
keyHashes.forEach((keyHash) => localLoginThrottle.delete(keyHash));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export function verifyAdminSetupKey(candidate: string) {
|
export function verifyAdminSetupKey(candidate: string) {
|
||||||
const expected =
|
const expected = getAdminSetupKey();
|
||||||
process.env.DATASTACK_ADMIN_SETUP_KEY ||
|
if (!expected || expected.length < 32 || !candidate) return false;
|
||||||
"datastack-default-setup-key-32-chars";
|
|
||||||
if (!expected || expected.length < 32) return false;
|
|
||||||
return matchesSecret(candidate, expected);
|
return matchesSecret(candidate, expected);
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function isAdminSetupComplete() {
|
export async function isAdminSetupComplete() {
|
||||||
const pool = getPool();
|
const pool = getPool();
|
||||||
if (!pool) {
|
if (!pool) {
|
||||||
return true;
|
if (globals.__dataStackInMemoryAdmin) {
|
||||||
}
|
return true;
|
||||||
try {
|
}
|
||||||
await ensureAdminTable();
|
if (isProductionEnv()) {
|
||||||
const result = await pool.query(
|
return Boolean(process.env.DATASTACK_ADMIN_PASSWORD);
|
||||||
"SELECT 1 FROM admin_users WHERE id = 1",
|
}
|
||||||
);
|
return process.env.DATASTACK_ALLOW_DEMO_ADMIN !== "false";
|
||||||
return result.rowCount === 1;
|
|
||||||
} catch {
|
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
await ensureAdminTable();
|
||||||
|
const result = await pool.query("SELECT 1 FROM admin_users WHERE id = 1");
|
||||||
|
return result.rowCount === 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function createInitialAdmin(email: string, password: string) {
|
export async function createInitialAdmin(email: string, password: string) {
|
||||||
@@ -380,136 +443,176 @@ export async function createInitialAdmin(email: string, password: string) {
|
|||||||
const pool = getPool();
|
const pool = getPool();
|
||||||
|
|
||||||
if (!pool) {
|
if (!pool) {
|
||||||
|
if (globals.__dataStackInMemoryAdmin) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
globals.__dataStackInMemoryAdmin = {
|
globals.__dataStackInMemoryAdmin = {
|
||||||
id: 1,
|
id: 1,
|
||||||
email: cleanEmail,
|
email: cleanEmail,
|
||||||
passwordSalt: salt,
|
passwordSalt: salt,
|
||||||
passwordHash,
|
passwordHash,
|
||||||
|
tokenVersion: 1,
|
||||||
createdAt: new Date(),
|
createdAt: new Date(),
|
||||||
};
|
};
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
await ensureAdminTable();
|
||||||
|
const client = await pool.connect();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await ensureAdminTable();
|
await client.query("BEGIN");
|
||||||
const client = await pool.connect();
|
await client.query("SELECT pg_advisory_xact_lock($1::bigint)", [
|
||||||
|
DATABASE_LOCK_ID,
|
||||||
|
]);
|
||||||
|
|
||||||
try {
|
const existing = await client.query(
|
||||||
await client.query("BEGIN");
|
"SELECT 1 FROM admin_users WHERE id = 1",
|
||||||
await client.query("SELECT pg_advisory_xact_lock($1::bigint)", [
|
);
|
||||||
DATABASE_LOCK_ID,
|
if (existing.rowCount) {
|
||||||
]);
|
await client.query("ROLLBACK");
|
||||||
|
return false;
|
||||||
const existing = await client.query(
|
|
||||||
"SELECT 1 FROM admin_users WHERE id = 1",
|
|
||||||
);
|
|
||||||
if (existing.rowCount) {
|
|
||||||
await client.query("ROLLBACK");
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
await client.query(
|
|
||||||
`INSERT INTO admin_users (id, email, password_salt, password_hash)
|
|
||||||
VALUES (1, $1, $2, $3)`,
|
|
||||||
[cleanEmail, salt, passwordHash],
|
|
||||||
);
|
|
||||||
await client.query("COMMIT");
|
|
||||||
return true;
|
|
||||||
} catch (error) {
|
|
||||||
await client.query("ROLLBACK").catch(() => undefined);
|
|
||||||
throw error;
|
|
||||||
} finally {
|
|
||||||
client.release();
|
|
||||||
}
|
}
|
||||||
} catch {
|
|
||||||
globals.__dataStackInMemoryAdmin = {
|
await client.query(
|
||||||
id: 1,
|
`INSERT INTO admin_users (id, email, password_salt, password_hash, token_version)
|
||||||
email: cleanEmail,
|
VALUES (1, $1, $2, $3, 1)`,
|
||||||
passwordSalt: salt,
|
[cleanEmail, salt, passwordHash],
|
||||||
passwordHash,
|
);
|
||||||
createdAt: new Date(),
|
await client.query("COMMIT");
|
||||||
};
|
|
||||||
return true;
|
return true;
|
||||||
|
} catch (error) {
|
||||||
|
await client.query("ROLLBACK").catch(() => undefined);
|
||||||
|
throw error;
|
||||||
|
} finally {
|
||||||
|
client.release();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function verifyAdminCredentials(email: string, password: string) {
|
export async function authenticateAdminCredentials(
|
||||||
|
email: string,
|
||||||
|
password: string,
|
||||||
|
): Promise<{ authenticated: boolean; tokenVersion: number }> {
|
||||||
const cleanEmail = email.trim().toLowerCase();
|
const cleanEmail = email.trim().toLowerCase();
|
||||||
const pool = getPool();
|
const pool = getPool();
|
||||||
|
|
||||||
if (pool) {
|
if (pool) {
|
||||||
try {
|
await ensureAdminTable();
|
||||||
await ensureAdminTable();
|
const result = await pool.query(
|
||||||
const result = await pool.query(
|
`SELECT email, password_salt, password_hash, COALESCE(token_version, 1) AS token_version
|
||||||
`SELECT email, password_salt, password_hash
|
FROM admin_users
|
||||||
FROM admin_users
|
WHERE id = 1`,
|
||||||
WHERE id = 1`,
|
);
|
||||||
);
|
if (!result.rowCount || result.rowCount === 0) {
|
||||||
if (result.rowCount && result.rowCount > 0) {
|
return { authenticated: false, tokenVersion: 0 };
|
||||||
const admin = result.rows[0] as {
|
|
||||||
email: string;
|
|
||||||
password_salt: Buffer;
|
|
||||||
password_hash: Buffer;
|
|
||||||
};
|
|
||||||
const emailMatches = matchesSecret(
|
|
||||||
cleanEmail,
|
|
||||||
admin.email.trim().toLowerCase(),
|
|
||||||
);
|
|
||||||
const actualHash = await derivePasswordHash(
|
|
||||||
password,
|
|
||||||
admin.password_salt,
|
|
||||||
);
|
|
||||||
const passwordMatches = timingSafeEqual(
|
|
||||||
actualHash,
|
|
||||||
admin.password_hash,
|
|
||||||
);
|
|
||||||
return emailMatches && passwordMatches;
|
|
||||||
}
|
|
||||||
if (!process.env.DATASTACK_ADMIN_PASSWORD) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
// Database connection error, fall back to in-memory admin
|
|
||||||
}
|
}
|
||||||
|
const admin = result.rows[0] as {
|
||||||
|
email: string;
|
||||||
|
password_salt: Buffer;
|
||||||
|
password_hash: Buffer;
|
||||||
|
token_version: number;
|
||||||
|
};
|
||||||
|
const emailMatches = matchesSecret(
|
||||||
|
cleanEmail,
|
||||||
|
admin.email.trim().toLowerCase(),
|
||||||
|
);
|
||||||
|
const actualHash = await derivePasswordHash(password, admin.password_salt);
|
||||||
|
const passwordMatches = timingSafeEqual(actualHash, admin.password_hash);
|
||||||
|
return {
|
||||||
|
authenticated: emailMatches && passwordMatches,
|
||||||
|
tokenVersion: Number(admin.token_version) || 1,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const inMem = await getOrInitInMemoryAdmin();
|
const inMem = await getOrInitInMemoryAdmin();
|
||||||
if (!matchesSecret(cleanEmail, inMem.email)) {
|
if (!matchesSecret(cleanEmail, inMem.email)) {
|
||||||
return false;
|
return { authenticated: false, tokenVersion: 0 };
|
||||||
}
|
}
|
||||||
const actualHash = await derivePasswordHash(password, inMem.passwordSalt);
|
const actualHash = await derivePasswordHash(password, inMem.passwordSalt);
|
||||||
return timingSafeEqual(actualHash, inMem.passwordHash);
|
const authenticated = timingSafeEqual(actualHash, inMem.passwordHash);
|
||||||
|
return {
|
||||||
|
authenticated,
|
||||||
|
tokenVersion: authenticated ? inMem.tokenVersion : 0,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function verifyAdminCredentials(email: string, password: string) {
|
||||||
|
const result = await authenticateAdminCredentials(email, password);
|
||||||
|
return result.authenticated;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function getAdminAccountInfo(fallbackEmail?: string) {
|
export async function getAdminAccountInfo(fallbackEmail?: string) {
|
||||||
const pool = getPool();
|
const pool = getPool();
|
||||||
if (pool) {
|
if (pool) {
|
||||||
try {
|
await ensureAdminTable();
|
||||||
await ensureAdminTable();
|
const result = await pool.query(
|
||||||
const result = await pool.query(
|
`SELECT email, created_at, COALESCE(token_version, 1) AS token_version
|
||||||
`SELECT email, created_at FROM admin_users WHERE id = 1`,
|
FROM admin_users
|
||||||
);
|
WHERE id = 1`,
|
||||||
if (result.rowCount && result.rowCount > 0) {
|
);
|
||||||
const row = result.rows[0] as { email: string; created_at: Date | string };
|
if (result.rowCount && result.rowCount > 0) {
|
||||||
return {
|
const row = result.rows[0] as {
|
||||||
email: row.email,
|
email: string;
|
||||||
createdAt: new Date(row.created_at).toISOString(),
|
created_at: Date | string;
|
||||||
storageMode: "PostgreSQL" as const,
|
token_version: number;
|
||||||
};
|
};
|
||||||
}
|
return {
|
||||||
} catch {
|
email: row.email,
|
||||||
// Fall back to in-memory admin
|
createdAt: new Date(row.created_at).toISOString(),
|
||||||
|
tokenVersion: Number(row.token_version) || 1,
|
||||||
|
storageMode: "PostgreSQL" as const,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
throw new Error("Admin account is not initialized.");
|
||||||
}
|
}
|
||||||
|
|
||||||
const inMem = await getOrInitInMemoryAdmin();
|
const inMem = await getOrInitInMemoryAdmin();
|
||||||
return {
|
return {
|
||||||
email: fallbackEmail || inMem.email,
|
email: fallbackEmail || inMem.email,
|
||||||
createdAt: inMem.createdAt.toISOString(),
|
createdAt: inMem.createdAt.toISOString(),
|
||||||
|
tokenVersion: inMem.tokenVersion,
|
||||||
storageMode: "In-Memory" as const,
|
storageMode: "In-Memory" as const,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function verifyRequestAdminSession(
|
||||||
|
cookieToken: string | undefined,
|
||||||
|
): Promise<
|
||||||
|
| { valid: true; session: AdminSessionPayload; sessionSecret: string }
|
||||||
|
| { valid: false; reason: "UNCONFIGURED" | "UNAUTHORIZED" | "UNAVAILABLE" }
|
||||||
|
> {
|
||||||
|
const sessionSecret = getAdminSessionSecret();
|
||||||
|
if (!sessionSecret) {
|
||||||
|
return { valid: false, reason: "UNCONFIGURED" };
|
||||||
|
}
|
||||||
|
if (!cookieToken) {
|
||||||
|
return { valid: false, reason: "UNAUTHORIZED" };
|
||||||
|
}
|
||||||
|
|
||||||
|
const session = await getAdminSessionPayload(cookieToken, sessionSecret);
|
||||||
|
if (!session) {
|
||||||
|
return { valid: false, reason: "UNAUTHORIZED" };
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const account = await getAdminAccountInfo();
|
||||||
|
const emailMatches = matchesSecret(
|
||||||
|
session.email.trim().toLowerCase(),
|
||||||
|
account.email.trim().toLowerCase(),
|
||||||
|
);
|
||||||
|
if (!emailMatches || session.tokenVersion !== account.tokenVersion) {
|
||||||
|
return { valid: false, reason: "UNAUTHORIZED" };
|
||||||
|
}
|
||||||
|
return { valid: true, session, sessionSecret };
|
||||||
|
} catch (error) {
|
||||||
|
if (getPool()) {
|
||||||
|
return { valid: false, reason: "UNAVAILABLE" };
|
||||||
|
}
|
||||||
|
void error;
|
||||||
|
return { valid: false, reason: "UNAUTHORIZED" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export async function updateAdminAccount({
|
export async function updateAdminAccount({
|
||||||
currentEmail,
|
currentEmail,
|
||||||
currentPassword,
|
currentPassword,
|
||||||
@@ -534,29 +637,42 @@ export async function updateAdminAccount({
|
|||||||
|
|
||||||
const pool = getPool();
|
const pool = getPool();
|
||||||
if (pool) {
|
if (pool) {
|
||||||
try {
|
await ensureAdminTable();
|
||||||
await ensureAdminTable();
|
const updated = await pool.query(
|
||||||
const updated = await pool.query(
|
`UPDATE admin_users
|
||||||
`UPDATE admin_users
|
SET email = $1,
|
||||||
SET email = $1, password_salt = $2, password_hash = $3
|
password_salt = $2,
|
||||||
WHERE id = 1`,
|
password_hash = $3,
|
||||||
[cleanEmail, salt, passwordHash],
|
token_version = COALESCE(token_version, 1) + 1
|
||||||
);
|
WHERE id = 1
|
||||||
if (updated.rowCount && updated.rowCount > 0) {
|
RETURNING token_version`,
|
||||||
return { ok: true as const, email: cleanEmail };
|
[cleanEmail, salt, passwordHash],
|
||||||
}
|
);
|
||||||
} catch {
|
if (!updated.rowCount || updated.rowCount === 0) {
|
||||||
// Fall back to in-memory update
|
return { ok: false as const, error: "Admin account not found." };
|
||||||
}
|
}
|
||||||
|
return {
|
||||||
|
ok: true as const,
|
||||||
|
email: cleanEmail,
|
||||||
|
tokenVersion: Number(updated.rows[0].token_version) || 1,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const existing = await getOrInitInMemoryAdmin();
|
const existing = await getOrInitInMemoryAdmin();
|
||||||
|
const nextVersion = existing.tokenVersion + 1;
|
||||||
globals.__dataStackInMemoryAdmin = {
|
globals.__dataStackInMemoryAdmin = {
|
||||||
id: 1,
|
id: 1,
|
||||||
email: cleanEmail,
|
email: cleanEmail,
|
||||||
passwordSalt: salt,
|
passwordSalt: salt,
|
||||||
passwordHash,
|
passwordHash,
|
||||||
|
tokenVersion: nextVersion,
|
||||||
createdAt: existing.createdAt,
|
createdAt: existing.createdAt,
|
||||||
};
|
};
|
||||||
return { ok: true as const, email: cleanEmail };
|
return { ok: true as const, email: cleanEmail, tokenVersion: nextVersion };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resetAdminStateForTests() {
|
||||||
|
globals.__dataStackInMemoryAdmin = undefined;
|
||||||
|
localLoginThrottle.clear();
|
||||||
|
lastLocalThrottleCleanup = 0;
|
||||||
}
|
}
|
||||||
+33
-10
@@ -8,16 +8,31 @@ export interface ClientTokenPayload {
|
|||||||
exp?: number;
|
exp?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
const DEFAULT_SECRET =
|
const DEV_DEFAULT_JWT_SECRET =
|
||||||
process.env.DATASTACK_JWT_SECRET ||
|
|
||||||
"datastack-jwt-default-secret-key-32-chars-long";
|
"datastack-jwt-default-secret-key-32-chars-long";
|
||||||
|
|
||||||
|
export function resolveDefaultJwtSecret(): string {
|
||||||
|
const configured = process.env.DATASTACK_JWT_SECRET?.trim() || "";
|
||||||
|
if (process.env.NODE_ENV === "production") {
|
||||||
|
return configured.length >= 32 ? configured : "";
|
||||||
|
}
|
||||||
|
if (configured.length > 0) {
|
||||||
|
return configured.length >= 32 ? configured : "";
|
||||||
|
}
|
||||||
|
return DEV_DEFAULT_JWT_SECRET;
|
||||||
|
}
|
||||||
|
|
||||||
export class AuthService {
|
export class AuthService {
|
||||||
private static secret: string = DEFAULT_SECRET;
|
private static secret: string = resolveDefaultJwtSecret();
|
||||||
|
|
||||||
public static setSecret(newSecret: string) {
|
public static setSecret(newSecret: string) {
|
||||||
if (newSecret && newSecret.trim().length > 0) {
|
const trimmed = newSecret ? newSecret.trim() : "";
|
||||||
this.secret = newSecret.trim();
|
if (trimmed.length >= 32) {
|
||||||
|
this.secret = trimmed;
|
||||||
|
} else if (process.env.NODE_ENV !== "production" && trimmed.length > 0) {
|
||||||
|
this.secret = trimmed;
|
||||||
|
} else if (process.env.NODE_ENV === "production") {
|
||||||
|
this.secret = "";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -25,6 +40,11 @@ export class AuthService {
|
|||||||
return this.secret;
|
return this.secret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static isConfigured(): boolean {
|
||||||
|
const minLen = process.env.NODE_ENV === "production" ? 32 : 1;
|
||||||
|
return this.secret.length >= minLen;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Generates a signed JWT for client SDK authentication.
|
* Generates a signed JWT for client SDK authentication.
|
||||||
*/
|
*/
|
||||||
@@ -32,9 +52,9 @@ export class AuthService {
|
|||||||
payload: { clientId: string; name?: string; rateLimit?: number },
|
payload: { clientId: string; name?: string; rateLimit?: number },
|
||||||
expiresIn = "30d",
|
expiresIn = "30d",
|
||||||
): string {
|
): string {
|
||||||
if (!this.secret) {
|
if (!this.isConfigured()) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
"DATASTACK_JWT_SECRET must be configured before generating client tokens.",
|
"DATASTACK_JWT_SECRET (>= 32 characters) must be configured before generating client tokens.",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -50,15 +70,18 @@ export class AuthService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Verifies and decodes incoming token from Socket.IO handshake.
|
* Verifies and decodes incoming token from Socket.IO handshake or HTTP ingestion.
|
||||||
*/
|
*/
|
||||||
public static verifyClientToken(token: string): {
|
public static verifyClientToken(token: string): {
|
||||||
valid: boolean;
|
valid: boolean;
|
||||||
payload?: ClientTokenPayload;
|
payload?: ClientTokenPayload;
|
||||||
error?: string;
|
error?: string;
|
||||||
} {
|
} {
|
||||||
if (!this.secret) {
|
if (!this.isConfigured()) {
|
||||||
return { valid: false, error: "DATASTACK_JWT_SECRET is not configured." };
|
return {
|
||||||
|
valid: false,
|
||||||
|
error: "DATASTACK_JWT_SECRET (>= 32 characters) is not configured.",
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|||||||
+380
-28
@@ -1,8 +1,37 @@
|
|||||||
|
import crypto from 'node:crypto';
|
||||||
|
import Redis from 'ioredis';
|
||||||
import { CSVStorageManager, DataEvent } from '../Storage/csvStorage';
|
import { CSVStorageManager, DataEvent } from '../Storage/csvStorage';
|
||||||
|
|
||||||
export type EventSubscriber = (event: DataEvent) => void;
|
export type EventSubscriber = (event: DataEvent) => void;
|
||||||
export type FlushCallback = (count: number, files: string[]) => void;
|
export type FlushCallback = (count: number, files: string[]) => void;
|
||||||
|
|
||||||
|
export interface RedisBufferClient {
|
||||||
|
status?: string;
|
||||||
|
rpush(key: string, ...values: string[]): Promise<number>;
|
||||||
|
lpush(key: string, ...values: string[]): Promise<number>;
|
||||||
|
ltrim(key: string, start: number, stop: number): Promise<unknown>;
|
||||||
|
lrange(key: string, start: number, stop: number): Promise<string[]>;
|
||||||
|
llen(key: string): Promise<number>;
|
||||||
|
set(
|
||||||
|
key: string,
|
||||||
|
value: string,
|
||||||
|
pxMode: 'PX',
|
||||||
|
ttlMs: number,
|
||||||
|
nxMode: 'NX',
|
||||||
|
): Promise<string | null>;
|
||||||
|
eval(script: string, numKeys: number, ...args: string[]): Promise<unknown>;
|
||||||
|
quit?: () => Promise<unknown>;
|
||||||
|
disconnect?: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const REDIS_BUFFER_KEY = 'datastack:buffer:events';
|
||||||
|
export const REDIS_RECENT_KEY = 'datastack:recent:events';
|
||||||
|
export const REDIS_FLUSH_LOCK_KEY = 'datastack:flush:lock';
|
||||||
|
const REDIS_RECENT_LIMIT = 500;
|
||||||
|
const FLUSH_LOCK_TTL_MS = 15000;
|
||||||
|
const UNLOCK_LUA_SCRIPT =
|
||||||
|
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end';
|
||||||
|
|
||||||
interface BufferGlobalState {
|
interface BufferGlobalState {
|
||||||
buffer: DataEvent[];
|
buffer: DataEvent[];
|
||||||
recentEvents: DataEvent[];
|
recentEvents: DataEvent[];
|
||||||
@@ -14,6 +43,11 @@ interface BufferGlobalState {
|
|||||||
totalIngested: number;
|
totalIngested: number;
|
||||||
totalFlushed: number;
|
totalFlushed: number;
|
||||||
isFlushing: boolean;
|
isFlushing: boolean;
|
||||||
|
redisClient: RedisBufferClient | null | undefined;
|
||||||
|
redisUrlConfigured: string;
|
||||||
|
redisConnected: boolean;
|
||||||
|
redisTestOverride: RedisBufferClient | null | undefined;
|
||||||
|
pendingRedisWrites: Set<Promise<void>>;
|
||||||
}
|
}
|
||||||
|
|
||||||
type BufferGlobals = typeof globalThis & {
|
type BufferGlobals = typeof globalThis & {
|
||||||
@@ -36,13 +70,86 @@ function getState(): BufferGlobalState {
|
|||||||
totalIngested: 0,
|
totalIngested: 0,
|
||||||
totalFlushed: 0,
|
totalFlushed: 0,
|
||||||
isFlushing: false,
|
isFlushing: false,
|
||||||
|
redisClient: undefined,
|
||||||
|
redisUrlConfigured: '',
|
||||||
|
redisConnected: false,
|
||||||
|
redisTestOverride: undefined,
|
||||||
|
pendingRedisWrites: new Set(),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
return globals.__dataStackBufferState;
|
return globals.__dataStackBufferState;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getRedisClient(): RedisBufferClient | null {
|
||||||
|
const state = getState();
|
||||||
|
if (state.redisTestOverride !== undefined) {
|
||||||
|
return state.redisTestOverride;
|
||||||
|
}
|
||||||
|
|
||||||
|
const configuredUrl = (process.env.REDIS_URL || '').trim();
|
||||||
|
if (!configuredUrl) {
|
||||||
|
if (state.redisClient && typeof state.redisClient.disconnect === 'function') {
|
||||||
|
try {
|
||||||
|
state.redisClient.disconnect();
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
state.redisClient = null;
|
||||||
|
state.redisUrlConfigured = '';
|
||||||
|
state.redisConnected = false;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (state.redisClient && state.redisUrlConfigured === configuredUrl) {
|
||||||
|
return state.redisClient;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (state.redisClient && typeof state.redisClient.disconnect === 'function') {
|
||||||
|
try {
|
||||||
|
state.redisClient.disconnect();
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const client = new Redis(configuredUrl, {
|
||||||
|
maxRetriesPerRequest: 2,
|
||||||
|
enableReadyCheck: true,
|
||||||
|
retryStrategy: (times: number) => Math.min(times * 250, 3000),
|
||||||
|
});
|
||||||
|
|
||||||
|
client.on('ready', () => {
|
||||||
|
state.redisConnected = true;
|
||||||
|
});
|
||||||
|
client.on('connect', () => {
|
||||||
|
state.redisConnected = true;
|
||||||
|
});
|
||||||
|
client.on('close', () => {
|
||||||
|
state.redisConnected = false;
|
||||||
|
});
|
||||||
|
client.on('end', () => {
|
||||||
|
state.redisConnected = false;
|
||||||
|
});
|
||||||
|
client.on('error', () => {
|
||||||
|
state.redisConnected = false;
|
||||||
|
});
|
||||||
|
|
||||||
|
state.redisClient = client as unknown as RedisBufferClient;
|
||||||
|
state.redisUrlConfigured = configuredUrl;
|
||||||
|
state.redisConnected = client.status === 'ready' || client.status === 'connect';
|
||||||
|
return state.redisClient;
|
||||||
|
} catch {
|
||||||
|
state.redisClient = null;
|
||||||
|
state.redisUrlConfigured = configuredUrl;
|
||||||
|
state.redisConnected = false;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export class DataBufferEngine {
|
export class DataBufferEngine {
|
||||||
private static maxRecentEvents = 200;
|
private static maxRecentEvents = REDIS_RECENT_LIMIT;
|
||||||
|
|
||||||
public static get batchFlushThreshold(): number {
|
public static get batchFlushThreshold(): number {
|
||||||
return getState().batchFlushThreshold;
|
return getState().batchFlushThreshold;
|
||||||
@@ -60,11 +167,46 @@ export class DataBufferEngine {
|
|||||||
this.setFlushInterval(value);
|
this.setFlushInterval(value);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static setRedisClientForTests(client: RedisBufferClient | null | undefined): void {
|
||||||
|
const state = getState();
|
||||||
|
state.redisTestOverride = client;
|
||||||
|
state.redisConnected = Boolean(client);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async shutdown(): Promise<void> {
|
||||||
|
const state = getState();
|
||||||
|
if (state.flushTimer) {
|
||||||
|
clearInterval(state.flushTimer);
|
||||||
|
state.flushTimer = null;
|
||||||
|
}
|
||||||
|
if (state.pendingRedisWrites.size > 0) {
|
||||||
|
await Promise.allSettled([...state.pendingRedisWrites]);
|
||||||
|
}
|
||||||
|
const client = state.redisClient;
|
||||||
|
state.redisClient = undefined;
|
||||||
|
state.redisUrlConfigured = '';
|
||||||
|
state.redisConnected = false;
|
||||||
|
state.redisTestOverride = undefined;
|
||||||
|
if (client) {
|
||||||
|
try {
|
||||||
|
if (typeof client.quit === 'function') {
|
||||||
|
await client.quit();
|
||||||
|
} else if (typeof client.disconnect === 'function') {
|
||||||
|
client.disconnect();
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
if (typeof client.disconnect === 'function') {
|
||||||
|
client.disconnect();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public static init() {
|
public static init() {
|
||||||
const state = getState();
|
const state = getState();
|
||||||
if (!state.flushTimer) {
|
if (!state.flushTimer) {
|
||||||
state.flushTimer = setInterval(() => {
|
state.flushTimer = setInterval(() => {
|
||||||
void this.flush();
|
void this.flush().catch(() => undefined);
|
||||||
}, state.batchFlushIntervalMs);
|
}, state.batchFlushIntervalMs);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -96,26 +238,18 @@ export class DataBufferEngine {
|
|||||||
}
|
}
|
||||||
|
|
||||||
state.flushTimer = setInterval(() => {
|
state.flushTimer = setInterval(() => {
|
||||||
void this.flush();
|
void this.flush().catch(() => undefined);
|
||||||
}, state.batchFlushIntervalMs);
|
}, state.batchFlushIntervalMs);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
private static recordLocalMetaAndNotify(event: DataEvent): void {
|
||||||
* Push incoming event to in-memory buffer.
|
|
||||||
* If buffer hits threshold (e.g. 500 rows), triggers immediate batch flush.
|
|
||||||
*/
|
|
||||||
public static push(event: DataEvent): void {
|
|
||||||
const state = getState();
|
const state = getState();
|
||||||
state.buffer.push(event);
|
|
||||||
state.totalIngested += 1;
|
state.totalIngested += 1;
|
||||||
|
|
||||||
// Cache in recent events for live dashboard viewer
|
|
||||||
state.recentEvents.unshift(event);
|
state.recentEvents.unshift(event);
|
||||||
if (state.recentEvents.length > this.maxRecentEvents) {
|
if (state.recentEvents.length > this.maxRecentEvents) {
|
||||||
state.recentEvents.pop();
|
state.recentEvents.pop();
|
||||||
}
|
}
|
||||||
|
|
||||||
// Notify live subscribers
|
|
||||||
for (const sub of state.subscribers) {
|
for (const sub of state.subscribers) {
|
||||||
try {
|
try {
|
||||||
sub(event);
|
sub(event);
|
||||||
@@ -123,43 +257,188 @@ export class DataBufferEngine {
|
|||||||
// Ignore subscriber error
|
// Ignore subscriber error
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Check if buffer threshold reached
|
private static async writeEventsToRedisOrFallback(events: DataEvent[]): Promise<number> {
|
||||||
|
if (events.length === 0) return 0;
|
||||||
|
const state = getState();
|
||||||
|
const redis = getRedisClient();
|
||||||
|
|
||||||
|
if (redis) {
|
||||||
|
const serialized = events.map((e) => JSON.stringify(e));
|
||||||
|
try {
|
||||||
|
const bufferLen = await redis.rpush(REDIS_BUFFER_KEY, ...serialized);
|
||||||
|
await redis.lpush(REDIS_RECENT_KEY, ...serialized);
|
||||||
|
await redis.ltrim(REDIS_RECENT_KEY, 0, REDIS_RECENT_LIMIT - 1);
|
||||||
|
state.redisConnected = true;
|
||||||
|
return Number(bufferLen) + state.buffer.length;
|
||||||
|
} catch {
|
||||||
|
state.redisConnected = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
state.buffer.push(...events);
|
||||||
|
return state.buffer.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Synchronous-signature push (queues to Redis asynchronously when configured, with automatic RAM fallback).
|
||||||
|
*/
|
||||||
|
public static push(event: DataEvent): void {
|
||||||
|
const state = getState();
|
||||||
|
this.recordLocalMetaAndNotify(event);
|
||||||
|
|
||||||
|
const redis = getRedisClient();
|
||||||
|
if (redis) {
|
||||||
|
const op = this.writeEventsToRedisOrFallback([event]).then((queuedCount) => {
|
||||||
|
if (queuedCount >= state.batchFlushThreshold) {
|
||||||
|
void this.flush().catch(() => undefined);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
state.pendingRedisWrites.add(op);
|
||||||
|
void op.finally(() => {
|
||||||
|
state.pendingRedisWrites.delete(op);
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
state.buffer.push(event);
|
||||||
if (state.buffer.length >= state.batchFlushThreshold) {
|
if (state.buffer.length >= state.batchFlushThreshold) {
|
||||||
void this.flush();
|
void this.flush().catch(() => undefined);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Flushes current buffer to CSV files via CSVStorageManager
|
* Async single-event push (awaited by API routes so Redis persistence is guaranteed before HTTP response).
|
||||||
|
*/
|
||||||
|
public static async pushAsync(event: DataEvent): Promise<void> {
|
||||||
|
await this.pushBatchAsync([event]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Async batch push with shared Redis durability and RAM fallback.
|
||||||
|
*/
|
||||||
|
public static async pushBatchAsync(events: DataEvent[]): Promise<void> {
|
||||||
|
if (!events || events.length === 0) return;
|
||||||
|
const state = getState();
|
||||||
|
for (const evt of events) {
|
||||||
|
this.recordLocalMetaAndNotify(evt);
|
||||||
|
}
|
||||||
|
|
||||||
|
const queuedCount = await this.writeEventsToRedisOrFallback(events);
|
||||||
|
if (queuedCount >= state.batchFlushThreshold) {
|
||||||
|
void this.flush().catch(() => undefined);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Flushes queued events (shared Redis list under distributed lock + local RAM fallback) to partitioned CSV files.
|
||||||
|
* Redis LTRIM runs ONLY after CSVStorageManager.appendBatch succeeds so crashes never lose data.
|
||||||
*/
|
*/
|
||||||
public static async flush(): Promise<{ count: number; files: string[] }> {
|
public static async flush(): Promise<{ count: number; files: string[] }> {
|
||||||
const state = getState();
|
const state = getState();
|
||||||
if (state.buffer.length === 0 || state.isFlushing) {
|
if (state.pendingRedisWrites.size > 0) {
|
||||||
|
await Promise.allSettled([...state.pendingRedisWrites]);
|
||||||
|
}
|
||||||
|
if (state.isFlushing) {
|
||||||
return { count: 0, files: [] };
|
return { count: 0, files: [] };
|
||||||
}
|
}
|
||||||
|
|
||||||
state.isFlushing = true;
|
state.isFlushing = true;
|
||||||
const batchToFlush = [...state.buffer];
|
let totalWritten = 0;
|
||||||
state.buffer = [];
|
const filesSet = new Set<string>();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const result = await CSVStorageManager.appendBatch(batchToFlush);
|
const redis = getRedisClient();
|
||||||
state.totalFlushed += result.written;
|
if (redis) {
|
||||||
|
const lockToken = crypto.randomUUID();
|
||||||
|
let lockAcquired = false;
|
||||||
|
|
||||||
for (const listener of state.flushListeners) {
|
|
||||||
try {
|
try {
|
||||||
listener(result.written, result.filesUpdated);
|
const lockResult = await redis.set(
|
||||||
|
REDIS_FLUSH_LOCK_KEY,
|
||||||
|
lockToken,
|
||||||
|
'PX',
|
||||||
|
FLUSH_LOCK_TTL_MS,
|
||||||
|
'NX',
|
||||||
|
);
|
||||||
|
lockAcquired = lockResult === 'OK';
|
||||||
|
state.redisConnected = true;
|
||||||
} catch {
|
} catch {
|
||||||
// ignore
|
state.redisConnected = false;
|
||||||
|
lockAcquired = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (lockAcquired) {
|
||||||
|
try {
|
||||||
|
const chunkSize = Math.max(500, state.batchFlushThreshold);
|
||||||
|
while (true) {
|
||||||
|
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, chunkSize - 1);
|
||||||
|
if (!rawItems || rawItems.length === 0) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
const parsedBatch: DataEvent[] = [];
|
||||||
|
for (const raw of rawItems) {
|
||||||
|
try {
|
||||||
|
const evt = JSON.parse(raw) as DataEvent;
|
||||||
|
if (evt && evt.eventId) {
|
||||||
|
parsedBatch.push(evt);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Ignore malformed legacy entry while still trimming it
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (parsedBatch.length > 0) {
|
||||||
|
const res = await CSVStorageManager.appendBatch(parsedBatch);
|
||||||
|
totalWritten += res.written;
|
||||||
|
for (const f of res.filesUpdated) filesSet.add(f);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Trim strictly AFTER CSVStorageManager.appendBatch resolves safely
|
||||||
|
await redis.ltrim(REDIS_BUFFER_KEY, rawItems.length, -1);
|
||||||
|
|
||||||
|
if (rawItems.length < chunkSize) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
|
||||||
|
} catch {
|
||||||
|
// Lock auto-expires after FLUSH_LOCK_TTL_MS
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return { count: result.written, files: result.filesUpdated };
|
if (state.buffer.length > 0) {
|
||||||
} catch (err) {
|
const ramBatch = [...state.buffer];
|
||||||
// On failure, re-queue to buffer front for retry
|
state.buffer = [];
|
||||||
state.buffer.unshift(...batchToFlush);
|
try {
|
||||||
throw err;
|
const res = await CSVStorageManager.appendBatch(ramBatch);
|
||||||
|
totalWritten += res.written;
|
||||||
|
for (const f of res.filesUpdated) filesSet.add(f);
|
||||||
|
} catch (err) {
|
||||||
|
state.buffer.unshift(...ramBatch);
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (totalWritten > 0) {
|
||||||
|
state.totalFlushed += totalWritten;
|
||||||
|
const filesUpdated = Array.from(filesSet);
|
||||||
|
for (const listener of state.flushListeners) {
|
||||||
|
try {
|
||||||
|
listener(totalWritten, filesUpdated);
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { count: totalWritten, files: Array.from(filesSet) };
|
||||||
} finally {
|
} finally {
|
||||||
state.isFlushing = false;
|
state.isFlushing = false;
|
||||||
}
|
}
|
||||||
@@ -194,10 +473,83 @@ export class DataBufferEngine {
|
|||||||
return state.recentEvents.slice(0, limit);
|
return state.recentEvents.slice(0, limit);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static async getRecentEventsAsync(
|
||||||
|
limit = 50,
|
||||||
|
filterClientId?: string,
|
||||||
|
): Promise<DataEvent[]> {
|
||||||
|
const state = getState();
|
||||||
|
const redis = getRedisClient();
|
||||||
|
|
||||||
|
if (redis) {
|
||||||
|
try {
|
||||||
|
const readCap = Math.min(REDIS_RECENT_LIMIT, Math.max(limit * 4, limit));
|
||||||
|
const rawItems = await redis.lrange(REDIS_RECENT_KEY, 0, readCap - 1);
|
||||||
|
state.redisConnected = true;
|
||||||
|
|
||||||
|
const redisEvents: DataEvent[] = [];
|
||||||
|
for (const raw of rawItems) {
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(raw) as DataEvent;
|
||||||
|
if (!parsed || !parsed.eventId) continue;
|
||||||
|
if (filterClientId && filterClientId !== 'all' && parsed.clientId !== filterClientId) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
redisEvents.push(parsed);
|
||||||
|
if (redisEvents.length >= limit) break;
|
||||||
|
} catch {
|
||||||
|
// skip malformed JSON entry
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (redisEvents.length > 0 || state.recentEvents.length === 0) {
|
||||||
|
return redisEvents;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
state.redisConnected = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.getRecentEvents(limit, filterClientId);
|
||||||
|
}
|
||||||
|
|
||||||
public static getBufferStats() {
|
public static getBufferStats() {
|
||||||
const state = getState();
|
const state = getState();
|
||||||
return {
|
return {
|
||||||
currentBufferSize: state.buffer.length,
|
currentBufferSize: state.buffer.length,
|
||||||
|
localBufferSize: state.buffer.length,
|
||||||
|
redisBufferSize: 0,
|
||||||
|
redisEnabled: Boolean(getRedisClient()),
|
||||||
|
redisConnected: state.redisConnected,
|
||||||
|
batchFlushThreshold: state.batchFlushThreshold,
|
||||||
|
batchFlushIntervalMs: state.batchFlushIntervalMs,
|
||||||
|
totalIngested: state.totalIngested,
|
||||||
|
totalFlushed: state.totalFlushed,
|
||||||
|
recentEventsCached: state.recentEvents.length,
|
||||||
|
activeSubscribers: state.subscribers.size,
|
||||||
|
isFlushing: state.isFlushing,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async getBufferStatsAsync() {
|
||||||
|
const state = getState();
|
||||||
|
const redis = getRedisClient();
|
||||||
|
let redisBufferSize = 0;
|
||||||
|
|
||||||
|
if (redis) {
|
||||||
|
try {
|
||||||
|
redisBufferSize = Number(await redis.llen(REDIS_BUFFER_KEY)) || 0;
|
||||||
|
state.redisConnected = true;
|
||||||
|
} catch {
|
||||||
|
state.redisConnected = false;
|
||||||
|
redisBufferSize = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
currentBufferSize: state.buffer.length + redisBufferSize,
|
||||||
|
localBufferSize: state.buffer.length,
|
||||||
|
redisBufferSize,
|
||||||
|
redisEnabled: Boolean(redis),
|
||||||
|
redisConnected: state.redisConnected,
|
||||||
batchFlushThreshold: state.batchFlushThreshold,
|
batchFlushThreshold: state.batchFlushThreshold,
|
||||||
batchFlushIntervalMs: state.batchFlushIntervalMs,
|
batchFlushIntervalMs: state.batchFlushIntervalMs,
|
||||||
totalIngested: state.totalIngested,
|
totalIngested: state.totalIngested,
|
||||||
|
|||||||
+315
-47
@@ -1,5 +1,8 @@
|
|||||||
|
import { existsSync, rmSync } from "node:fs";
|
||||||
|
import { resolve, sep } from "node:path";
|
||||||
import { AuthService } from "../auth/jwt";
|
import { AuthService } from "../auth/jwt";
|
||||||
import { DatabaseEngine } from "../Database/database";
|
import { DatabaseEngine } from "../Database/database";
|
||||||
|
import { getInternalIngestionHeaders } from "../Security/security";
|
||||||
|
|
||||||
export interface ClientSDK {
|
export interface ClientSDK {
|
||||||
clientId: string;
|
clientId: string;
|
||||||
@@ -15,7 +18,9 @@ export interface ClientSDK {
|
|||||||
|
|
||||||
type SDKDatabaseGlobals = typeof globalThis & {
|
type SDKDatabaseGlobals = typeof globalThis & {
|
||||||
__dataStackClientsMap?: Map<string, ClientSDK>;
|
__dataStackClientsMap?: Map<string, ClientSDK>;
|
||||||
|
__dataStackDeletedClients?: Set<string>;
|
||||||
__dataStackClientsHydrated?: boolean;
|
__dataStackClientsHydrated?: boolean;
|
||||||
|
__dataStackSyncedSocketEvents?: Map<string, number>;
|
||||||
};
|
};
|
||||||
|
|
||||||
const globals = globalThis as SDKDatabaseGlobals;
|
const globals = globalThis as SDKDatabaseGlobals;
|
||||||
@@ -27,6 +32,67 @@ function getClientsMap(): Map<string, ClientSDK> {
|
|||||||
return globals.__dataStackClientsMap;
|
return globals.__dataStackClientsMap;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getDeletedClientsSet(): Set<string> {
|
||||||
|
if (!globals.__dataStackDeletedClients) {
|
||||||
|
globals.__dataStackDeletedClients = new Set<string>();
|
||||||
|
}
|
||||||
|
return globals.__dataStackDeletedClients;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getSyncedSocketEventsMap(): Map<string, number> {
|
||||||
|
if (!globals.__dataStackSyncedSocketEvents) {
|
||||||
|
globals.__dataStackSyncedSocketEvents = new Map<string, number>();
|
||||||
|
}
|
||||||
|
return globals.__dataStackSyncedSocketEvents;
|
||||||
|
}
|
||||||
|
|
||||||
|
function serializeSyncClient(client: ClientSDK) {
|
||||||
|
return {
|
||||||
|
clientId: client.clientId,
|
||||||
|
name: client.name,
|
||||||
|
status: client.status,
|
||||||
|
token: client.token,
|
||||||
|
rateLimit: client.rateLimit,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function notifyIngestionClientSyncAsync(
|
||||||
|
clientId: string,
|
||||||
|
action: "deleted" | "revoked" | "updated" | "registered",
|
||||||
|
client?: ClientSDK,
|
||||||
|
): Promise<void> {
|
||||||
|
const headers = getInternalIngestionHeaders();
|
||||||
|
if (!headers["x-datastack-internal-secret"]) return;
|
||||||
|
const ingestionBaseUrl = (
|
||||||
|
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||||
|
).replace(/\/$/, "");
|
||||||
|
try {
|
||||||
|
await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
...headers,
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
clientId,
|
||||||
|
action,
|
||||||
|
...(client ? { client: serializeSyncClient(client) } : {}),
|
||||||
|
}),
|
||||||
|
signal: AbortSignal.timeout(1000),
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Port 4000 may not be running in isolated single-process dev mode
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function notifyIngestionClientSync(
|
||||||
|
clientId: string,
|
||||||
|
action: "deleted" | "revoked" | "updated" | "registered",
|
||||||
|
client?: ClientSDK,
|
||||||
|
): void {
|
||||||
|
void notifyIngestionClientSyncAsync(clientId, action, client);
|
||||||
|
}
|
||||||
|
|
||||||
function mapRowToClient(row: Record<string, unknown>): ClientSDK {
|
function mapRowToClient(row: Record<string, unknown>): ClientSDK {
|
||||||
return {
|
return {
|
||||||
clientId: String(row.client_id),
|
clientId: String(row.client_id),
|
||||||
@@ -54,11 +120,20 @@ export class SDKManager {
|
|||||||
const ready = await DatabaseEngine.ensureSchema();
|
const ready = await DatabaseEngine.ensureSchema();
|
||||||
if (!ready) return null;
|
if (!ready) return null;
|
||||||
|
|
||||||
const result = await pool.query(
|
const [result, deletedResult] = await Promise.all([
|
||||||
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
|
pool.query(
|
||||||
FROM client_sdks
|
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
|
||||||
ORDER BY created_at DESC`,
|
FROM client_sdks
|
||||||
);
|
ORDER BY created_at DESC`,
|
||||||
|
),
|
||||||
|
pool.query(`SELECT client_id FROM deleted_clients`),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const deletedSet = getDeletedClientsSet();
|
||||||
|
deletedSet.clear();
|
||||||
|
for (const row of deletedResult.rows) {
|
||||||
|
deletedSet.add(String(row.client_id).trim().toLowerCase());
|
||||||
|
}
|
||||||
|
|
||||||
const map = getClientsMap();
|
const map = getClientsMap();
|
||||||
map.clear();
|
map.clear();
|
||||||
@@ -84,12 +159,91 @@ export class SDKManager {
|
|||||||
return Array.from(getClientsMap().values());
|
return Array.from(getClientsMap().values());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static async syncAllClientsToIngestionAsync(): Promise<void> {
|
||||||
|
const headers = getInternalIngestionHeaders();
|
||||||
|
if (!headers["x-datastack-internal-secret"]) return;
|
||||||
|
const ingestionBaseUrl = (
|
||||||
|
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||||
|
).replace(/\/$/, "");
|
||||||
|
const clients = Array.from(getClientsMap().values()).map(serializeSyncClient);
|
||||||
|
try {
|
||||||
|
await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
...headers,
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
action: "bulk_sync",
|
||||||
|
clients,
|
||||||
|
}),
|
||||||
|
signal: AbortSignal.timeout(1500),
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Port 4000 may not be running in isolated single-process dev mode
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public static async listClientsAsync(): Promise<ClientSDK[]> {
|
public static async listClientsAsync(): Promise<ClientSDK[]> {
|
||||||
const fromDb = await this.syncFromDatabase();
|
const fromDb = await this.syncFromDatabase();
|
||||||
if (fromDb !== null) {
|
if (fromDb !== null) {
|
||||||
return fromDb;
|
return fromDb;
|
||||||
}
|
}
|
||||||
return Array.from(getClientsMap().values());
|
|
||||||
|
const map = getClientsMap();
|
||||||
|
const headers = getInternalIngestionHeaders();
|
||||||
|
if (headers["x-datastack-internal-secret"]) {
|
||||||
|
const ingestionBaseUrl = (
|
||||||
|
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||||
|
).replace(/\/$/, "");
|
||||||
|
try {
|
||||||
|
const res = await fetch(`${ingestionBaseUrl}/internal/metrics`, {
|
||||||
|
headers,
|
||||||
|
cache: "no-store",
|
||||||
|
signal: AbortSignal.timeout(1000),
|
||||||
|
});
|
||||||
|
if (res.ok) {
|
||||||
|
const data = (await res.json()) as {
|
||||||
|
hasSyncedInMemoryClients?: boolean;
|
||||||
|
clientActivity?: Record<
|
||||||
|
string,
|
||||||
|
{ totalEvents?: number; lastActive?: string }
|
||||||
|
>;
|
||||||
|
};
|
||||||
|
if (data.hasSyncedInMemoryClients === false && map.size > 0) {
|
||||||
|
await this.syncAllClientsToIngestionAsync();
|
||||||
|
}
|
||||||
|
if (data.clientActivity && typeof data.clientActivity === "object") {
|
||||||
|
const syncedMap = getSyncedSocketEventsMap();
|
||||||
|
for (const [cid, activity] of Object.entries(data.clientActivity)) {
|
||||||
|
const client = map.get(cid);
|
||||||
|
if (!client) continue;
|
||||||
|
const remoteTotal = Math.max(0, Number(activity?.totalEvents) || 0);
|
||||||
|
const prevSynced = syncedMap.get(cid) || 0;
|
||||||
|
const delta =
|
||||||
|
remoteTotal >= prevSynced
|
||||||
|
? remoteTotal - prevSynced
|
||||||
|
: remoteTotal;
|
||||||
|
if (delta > 0) {
|
||||||
|
client.totalEvents += delta;
|
||||||
|
}
|
||||||
|
syncedMap.set(cid, remoteTotal);
|
||||||
|
if (
|
||||||
|
activity?.lastActive &&
|
||||||
|
(!client.lastActive || activity.lastActive > client.lastActive)
|
||||||
|
) {
|
||||||
|
client.lastActive = activity.lastActive;
|
||||||
|
}
|
||||||
|
map.set(cid, client);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Port 4000 may not be running in isolated single-process dev mode
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return Array.from(map.values());
|
||||||
}
|
}
|
||||||
|
|
||||||
public static getClient(clientId: string): ClientSDK | undefined {
|
public static getClient(clientId: string): ClientSDK | undefined {
|
||||||
@@ -152,8 +306,10 @@ export class SDKManager {
|
|||||||
createdAt: new Date().toISOString(),
|
createdAt: new Date().toISOString(),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
getDeletedClientsSet().delete(safeClientId);
|
||||||
getClientsMap().set(safeClientId, newClient);
|
getClientsMap().set(safeClientId, newClient);
|
||||||
void this.persistNewClient(newClient);
|
void this.persistNewClient(newClient);
|
||||||
|
notifyIngestionClientSync(safeClientId, "registered", newClient);
|
||||||
return newClient;
|
return newClient;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -163,6 +319,10 @@ export class SDKManager {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
await DatabaseEngine.ensureSchema();
|
await DatabaseEngine.ensureSchema();
|
||||||
|
await pool.query(
|
||||||
|
`DELETE FROM deleted_clients WHERE client_id = $1`,
|
||||||
|
[client.clientId],
|
||||||
|
);
|
||||||
await pool.query(
|
await pool.query(
|
||||||
`INSERT INTO client_sdks (client_id, name, secret, token, rate_limit, status, total_events, created_at, updated_at)
|
`INSERT INTO client_sdks (client_id, name, secret, token, rate_limit, status, total_events, created_at, updated_at)
|
||||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW())
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW())
|
||||||
@@ -218,8 +378,10 @@ export class SDKManager {
|
|||||||
createdAt: new Date().toISOString(),
|
createdAt: new Date().toISOString(),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
getDeletedClientsSet().delete(safeClientId);
|
||||||
getClientsMap().set(safeClientId, newClient);
|
getClientsMap().set(safeClientId, newClient);
|
||||||
await this.persistNewClient(newClient);
|
await this.persistNewClient(newClient);
|
||||||
|
await notifyIngestionClientSyncAsync(safeClientId, "registered", newClient);
|
||||||
return newClient;
|
return newClient;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -227,16 +389,20 @@ export class SDKManager {
|
|||||||
const client = getClientsMap().get(clientId);
|
const client = getClientsMap().get(clientId);
|
||||||
if (!client) return null;
|
if (!client) return null;
|
||||||
|
|
||||||
client.secret = `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`;
|
const updated: ClientSDK = {
|
||||||
client.token = AuthService.generateClientToken({
|
...client,
|
||||||
clientId: client.clientId,
|
secret: `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`,
|
||||||
name: client.name,
|
token: AuthService.generateClientToken({
|
||||||
rateLimit: client.rateLimit,
|
clientId: client.clientId,
|
||||||
});
|
name: client.name,
|
||||||
|
rateLimit: client.rateLimit,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
|
||||||
getClientsMap().set(clientId, client);
|
getClientsMap().set(clientId, updated);
|
||||||
void this.persistClientCredentials(client);
|
void this.persistClientCredentials(updated);
|
||||||
return client;
|
notifyIngestionClientSync(clientId, "updated", updated);
|
||||||
|
return updated;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static async persistClientCredentials(
|
private static async persistClientCredentials(
|
||||||
@@ -264,16 +430,20 @@ export class SDKManager {
|
|||||||
const existing = await this.getClientAsync(clientId);
|
const existing = await this.getClientAsync(clientId);
|
||||||
if (!existing) return null;
|
if (!existing) return null;
|
||||||
|
|
||||||
existing.secret = `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`;
|
const updated: ClientSDK = {
|
||||||
existing.token = AuthService.generateClientToken({
|
...existing,
|
||||||
clientId: existing.clientId,
|
secret: `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`,
|
||||||
name: existing.name,
|
token: AuthService.generateClientToken({
|
||||||
rateLimit: existing.rateLimit,
|
clientId: existing.clientId,
|
||||||
});
|
name: existing.name,
|
||||||
|
rateLimit: existing.rateLimit,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
|
||||||
getClientsMap().set(clientId, existing);
|
getClientsMap().set(clientId, updated);
|
||||||
await this.persistClientCredentials(existing);
|
await this.persistClientCredentials(updated);
|
||||||
return existing;
|
await notifyIngestionClientSyncAsync(clientId, "updated", updated);
|
||||||
|
return updated;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static updateClient(
|
public static updateClient(
|
||||||
@@ -283,14 +453,23 @@ export class SDKManager {
|
|||||||
const client = getClientsMap().get(clientId);
|
const client = getClientsMap().get(clientId);
|
||||||
if (!client) return null;
|
if (!client) return null;
|
||||||
|
|
||||||
if (updates.name !== undefined) client.name = updates.name.trim();
|
const updated: ClientSDK = {
|
||||||
if (updates.rateLimit !== undefined)
|
...client,
|
||||||
client.rateLimit = Math.max(1, Number(updates.rateLimit) || 100);
|
...(updates.name !== undefined ? { name: updates.name.trim() } : {}),
|
||||||
if (updates.status !== undefined) client.status = updates.status;
|
...(updates.rateLimit !== undefined
|
||||||
|
? { rateLimit: Math.max(1, Number(updates.rateLimit) || 100) }
|
||||||
|
: {}),
|
||||||
|
...(updates.status !== undefined ? { status: updates.status } : {}),
|
||||||
|
};
|
||||||
|
|
||||||
getClientsMap().set(clientId, client);
|
getClientsMap().set(clientId, updated);
|
||||||
void this.persistClientUpdates(client);
|
void this.persistClientUpdates(updated);
|
||||||
return client;
|
notifyIngestionClientSync(
|
||||||
|
clientId,
|
||||||
|
updated.status === "revoked" ? "revoked" : "updated",
|
||||||
|
updated,
|
||||||
|
);
|
||||||
|
return updated;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static async persistClientUpdates(client: ClientSDK): Promise<void> {
|
private static async persistClientUpdates(client: ClientSDK): Promise<void> {
|
||||||
@@ -317,19 +496,67 @@ export class SDKManager {
|
|||||||
const client = await this.getClientAsync(clientId);
|
const client = await this.getClientAsync(clientId);
|
||||||
if (!client) return null;
|
if (!client) return null;
|
||||||
|
|
||||||
if (updates.name !== undefined) client.name = updates.name.trim();
|
const updated: ClientSDK = {
|
||||||
if (updates.rateLimit !== undefined)
|
...client,
|
||||||
client.rateLimit = Math.max(1, Number(updates.rateLimit) || 100);
|
...(updates.name !== undefined ? { name: updates.name.trim() } : {}),
|
||||||
if (updates.status !== undefined) client.status = updates.status;
|
...(updates.rateLimit !== undefined
|
||||||
|
? { rateLimit: Math.max(1, Number(updates.rateLimit) || 100) }
|
||||||
|
: {}),
|
||||||
|
...(updates.status !== undefined ? { status: updates.status } : {}),
|
||||||
|
};
|
||||||
|
|
||||||
getClientsMap().set(clientId, client);
|
getClientsMap().set(clientId, updated);
|
||||||
await this.persistClientUpdates(client);
|
await this.persistClientUpdates(updated);
|
||||||
return client;
|
await notifyIngestionClientSyncAsync(
|
||||||
|
clientId,
|
||||||
|
updated.status === "revoked" ? "revoked" : "updated",
|
||||||
|
updated,
|
||||||
|
);
|
||||||
|
return updated;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static isClientDeleted(clientId: string): boolean {
|
||||||
|
const normalized = String(clientId || "")
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
|
return getDeletedClientsSet().has(normalized);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async isClientDeletedAsync(clientId: string): Promise<boolean> {
|
||||||
|
const normalized = String(clientId || "")
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
|
if (!normalized) return false;
|
||||||
|
if (getDeletedClientsSet().has(normalized)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
const pool = DatabaseEngine.getPool();
|
||||||
|
if (!pool) return false;
|
||||||
|
|
||||||
|
await DatabaseEngine.ensureSchema();
|
||||||
|
const res = await pool.query(
|
||||||
|
`SELECT 1 FROM deleted_clients WHERE client_id = $1`,
|
||||||
|
[normalized],
|
||||||
|
);
|
||||||
|
if (res.rowCount && res.rowCount > 0) {
|
||||||
|
getDeletedClientsSet().add(normalized);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static deleteClient(clientId: string): boolean {
|
public static deleteClient(clientId: string): boolean {
|
||||||
const deleted = getClientsMap().delete(clientId);
|
const normalized = String(clientId || "")
|
||||||
void this.deleteClientFromDb(clientId);
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
|
const deleted = getClientsMap().delete(normalized);
|
||||||
|
getSyncedSocketEventsMap().delete(normalized);
|
||||||
|
if (deleted) {
|
||||||
|
getDeletedClientsSet().add(normalized);
|
||||||
|
}
|
||||||
|
void this.deleteClientFromDb(normalized);
|
||||||
|
notifyIngestionClientSync(normalized, "deleted");
|
||||||
return deleted;
|
return deleted;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -337,21 +564,62 @@ export class SDKManager {
|
|||||||
const pool = DatabaseEngine.getPool();
|
const pool = DatabaseEngine.getPool();
|
||||||
if (!pool) return false;
|
if (!pool) return false;
|
||||||
|
|
||||||
|
await DatabaseEngine.ensureSchema();
|
||||||
|
const dbClient = await pool.connect();
|
||||||
try {
|
try {
|
||||||
await DatabaseEngine.ensureSchema();
|
await dbClient.query("BEGIN");
|
||||||
const res = await pool.query(
|
const res = await dbClient.query(
|
||||||
`DELETE FROM client_sdks WHERE client_id = $1`,
|
`DELETE FROM client_sdks WHERE client_id = $1`,
|
||||||
[clientId],
|
[clientId],
|
||||||
);
|
);
|
||||||
return Boolean(res.rowCount && res.rowCount > 0);
|
if (res.rowCount && res.rowCount > 0) {
|
||||||
} catch {
|
await dbClient.query(
|
||||||
|
`INSERT INTO deleted_clients (client_id, deleted_at)
|
||||||
|
VALUES ($1, NOW())
|
||||||
|
ON CONFLICT (client_id) DO UPDATE SET deleted_at = NOW()`,
|
||||||
|
[clientId],
|
||||||
|
);
|
||||||
|
await dbClient.query("COMMIT");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
await dbClient.query("ROLLBACK");
|
||||||
return false;
|
return false;
|
||||||
|
} catch {
|
||||||
|
await dbClient.query("ROLLBACK").catch(() => undefined);
|
||||||
|
return false;
|
||||||
|
} finally {
|
||||||
|
dbClient.release();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static async deleteClientAsync(clientId: string): Promise<boolean> {
|
private static purgeClientStorageDir(clientId: string): void {
|
||||||
const existedInMem = getClientsMap().delete(clientId);
|
const safeId = clientId.replace(/[^a-z0-9_-]/gi, "_");
|
||||||
const deletedFromDb = await this.deleteClientFromDb(clientId);
|
if (!safeId) return;
|
||||||
|
const storageRoot = resolve(process.cwd(), "storage", "data");
|
||||||
|
const targetDir = resolve(storageRoot, safeId);
|
||||||
|
if (!targetDir.startsWith(`${storageRoot}${sep}`)) return;
|
||||||
|
if (existsSync(targetDir)) {
|
||||||
|
rmSync(targetDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async deleteClientAsync(
|
||||||
|
clientId: string,
|
||||||
|
options?: { purgeFiles?: boolean },
|
||||||
|
): Promise<boolean> {
|
||||||
|
const normalized = String(clientId || "")
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
|
const existedInMem = getClientsMap().delete(normalized);
|
||||||
|
getSyncedSocketEventsMap().delete(normalized);
|
||||||
|
const deletedFromDb = await this.deleteClientFromDb(normalized);
|
||||||
|
if (existedInMem || deletedFromDb) {
|
||||||
|
getDeletedClientsSet().add(normalized);
|
||||||
|
if (options?.purgeFiles) {
|
||||||
|
this.purgeClientStorageDir(normalized);
|
||||||
|
}
|
||||||
|
await notifyIngestionClientSyncAsync(normalized, "deleted");
|
||||||
|
}
|
||||||
return existedInMem || deletedFromDb;
|
return existedInMem || deletedFromDb;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -17,9 +17,12 @@ services:
|
|||||||
DATASTACK_ADMIN_SESSION_SECRET: "${SERVICE_REALBASE64_64_ADMINSESSION:?Coolify must generate SERVICE_REALBASE64_64_ADMINSESSION}"
|
DATASTACK_ADMIN_SESSION_SECRET: "${SERVICE_REALBASE64_64_ADMINSESSION:?Coolify must generate SERVICE_REALBASE64_64_ADMINSESSION}"
|
||||||
DATASTACK_ADMIN_SETUP_KEY: "${SERVICE_REALBASE64_64_ADMININSTALL:?Coolify must generate SERVICE_REALBASE64_64_ADMININSTALL}"
|
DATASTACK_ADMIN_SETUP_KEY: "${SERVICE_REALBASE64_64_ADMININSTALL:?Coolify must generate SERVICE_REALBASE64_64_ADMININSTALL}"
|
||||||
DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}"
|
DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}"
|
||||||
|
REDIS_URL: "redis://redis:6379"
|
||||||
depends_on:
|
depends_on:
|
||||||
postgres:
|
postgres:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
redis:
|
||||||
|
condition: service_healthy
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test:
|
test:
|
||||||
@@ -48,9 +51,12 @@ services:
|
|||||||
PORT_SOCKET: "4000"
|
PORT_SOCKET: "4000"
|
||||||
DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}"
|
DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}"
|
||||||
DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}"
|
DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}"
|
||||||
|
REDIS_URL: "redis://redis:6379"
|
||||||
depends_on:
|
depends_on:
|
||||||
postgres:
|
postgres:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
redis:
|
||||||
|
condition: service_healthy
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test:
|
test:
|
||||||
@@ -85,7 +91,7 @@ services:
|
|||||||
retries: 3
|
retries: 3
|
||||||
start_period: 10s
|
start_period: 10s
|
||||||
|
|
||||||
# Reserved for the planned shared buffer integration; not consumed by the current runtime.
|
# Shared Redis ingestion buffer & distributed flush coordination
|
||||||
redis:
|
redis:
|
||||||
image: redis:7-alpine
|
image: redis:7-alpine
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|||||||
+33
-16
@@ -1,4 +1,7 @@
|
|||||||
import { verifyAdminSessionToken } from "@/core/auth/adminSession";
|
import {
|
||||||
|
getAdminSessionSecret,
|
||||||
|
verifyAdminSessionToken,
|
||||||
|
} from "@/core/auth/adminSession";
|
||||||
import { NextRequest, NextResponse } from "next/server";
|
import { NextRequest, NextResponse } from "next/server";
|
||||||
|
|
||||||
const SESSION_COOKIE = "datastack_admin_session";
|
const SESSION_COOKIE = "datastack_admin_session";
|
||||||
@@ -13,28 +16,37 @@ function isProtectedApi(pathname: string, method: string) {
|
|||||||
return pathname.startsWith("/api/");
|
return pathname.startsWith("/api/");
|
||||||
}
|
}
|
||||||
|
|
||||||
function isSameOriginRequest(request: NextRequest) {
|
function matchesRequestHost(candidateUrl: string, request: NextRequest) {
|
||||||
const originHeader = request.headers.get("origin");
|
|
||||||
const requestHost = request.headers.get("host");
|
const requestHost = request.headers.get("host");
|
||||||
const forwardedHost = request.headers
|
const forwardedHost = request.headers
|
||||||
.get("x-forwarded-host")
|
.get("x-forwarded-host")
|
||||||
?.split(",")[0]
|
?.split(",")[0]
|
||||||
?.trim();
|
?.trim();
|
||||||
if (!originHeader || (!requestHost && !forwardedHost)) return false;
|
if (!requestHost && !forwardedHost) return false;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const origin = new URL(originHeader);
|
const parsedHost = new URL(candidateUrl).host.toLowerCase();
|
||||||
const originHost = origin.host.toLowerCase();
|
return Boolean(
|
||||||
const matchesHost =
|
(requestHost && parsedHost === requestHost.toLowerCase()) ||
|
||||||
(requestHost && originHost === requestHost.toLowerCase()) ||
|
(forwardedHost && parsedHost === forwardedHost.toLowerCase()),
|
||||||
(forwardedHost && originHost === forwardedHost.toLowerCase());
|
);
|
||||||
|
|
||||||
return Boolean(matchesHost);
|
|
||||||
} catch {
|
} catch {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function isSameOriginRequest(request: NextRequest) {
|
||||||
|
const originHeader = request.headers.get("origin");
|
||||||
|
if (originHeader) {
|
||||||
|
return matchesRequestHost(originHeader, request);
|
||||||
|
}
|
||||||
|
const refererHeader = request.headers.get("referer");
|
||||||
|
if (refererHeader) {
|
||||||
|
return matchesRequestHost(refererHeader, request);
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
export async function middleware(request: NextRequest) {
|
export async function middleware(request: NextRequest) {
|
||||||
const { pathname } = request.nextUrl;
|
const { pathname } = request.nextUrl;
|
||||||
const isLoginPage = pathname === "/vcon/login";
|
const isLoginPage = pathname === "/vcon/login";
|
||||||
@@ -73,14 +85,19 @@ export async function middleware(request: NextRequest) {
|
|||||||
return NextResponse.next();
|
return NextResponse.next();
|
||||||
}
|
}
|
||||||
|
|
||||||
const sessionSecret =
|
const sessionSecret = getAdminSessionSecret();
|
||||||
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
|
if (!sessionSecret && isAdminApi) {
|
||||||
"datastack-admin-session-default-secret-32-chars";
|
return NextResponse.json(
|
||||||
|
{ error: "Admin authentication is not configured." },
|
||||||
|
{ status: 503 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const token = request.cookies.get(SESSION_COOKIE)?.value;
|
const token = request.cookies.get(SESSION_COOKIE)?.value;
|
||||||
const authenticated = Boolean(
|
const authenticated = Boolean(
|
||||||
sessionSecret &&
|
sessionSecret &&
|
||||||
token &&
|
token &&
|
||||||
(await verifyAdminSessionToken(token, sessionSecret)),
|
(await verifyAdminSessionToken(token, sessionSecret)),
|
||||||
);
|
);
|
||||||
|
|
||||||
if (isLoginPage) {
|
if (isLoginPage) {
|
||||||
|
|||||||
+20
-1
@@ -1,3 +1,18 @@
|
|||||||
|
map $http_upgrade $connection_upgrade {
|
||||||
|
default upgrade;
|
||||||
|
'' close;
|
||||||
|
}
|
||||||
|
|
||||||
|
map $http_x_forwarded_proto $forwarded_proto {
|
||||||
|
default $http_x_forwarded_proto;
|
||||||
|
'' $scheme;
|
||||||
|
}
|
||||||
|
|
||||||
|
map $http_x_forwarded_host $forwarded_host {
|
||||||
|
default $http_x_forwarded_host;
|
||||||
|
'' $host;
|
||||||
|
}
|
||||||
|
|
||||||
upstream nextjs_upstream {
|
upstream nextjs_upstream {
|
||||||
server 127.0.0.1:3000;
|
server 127.0.0.1:3000;
|
||||||
}
|
}
|
||||||
@@ -17,10 +32,12 @@ server {
|
|||||||
proxy_pass http://socketio_upstream;
|
proxy_pass http://socketio_upstream;
|
||||||
proxy_http_version 1.1;
|
proxy_http_version 1.1;
|
||||||
proxy_set_header Upgrade $http_upgrade;
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
proxy_set_header Connection "upgrade";
|
proxy_set_header Connection $connection_upgrade;
|
||||||
proxy_set_header Host $host;
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Forwarded-Host $forwarded_host;
|
||||||
proxy_set_header X-Real-IP $remote_addr;
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $forwarded_proto;
|
||||||
proxy_read_timeout 86400;
|
proxy_read_timeout 86400;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -29,7 +46,9 @@ server {
|
|||||||
proxy_pass http://nextjs_upstream;
|
proxy_pass http://nextjs_upstream;
|
||||||
proxy_http_version 1.1;
|
proxy_http_version 1.1;
|
||||||
proxy_set_header Host $host;
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Forwarded-Host $forwarded_host;
|
||||||
proxy_set_header X-Real-IP $remote_addr;
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $forwarded_proto;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -19,6 +19,7 @@
|
|||||||
"cors": "^2.8.6",
|
"cors": "^2.8.6",
|
||||||
"csv-parse": "^7.0.3",
|
"csv-parse": "^7.0.3",
|
||||||
"express": "^5.3.0",
|
"express": "^5.3.0",
|
||||||
|
"ioredis": "^6.0.0",
|
||||||
"json2csv": "^6.0.0-alpha.2",
|
"json2csv": "^6.0.0-alpha.2",
|
||||||
"jsonwebtoken": "^9.0.3",
|
"jsonwebtoken": "^9.0.3",
|
||||||
"lucide-react": "^0.553.0",
|
"lucide-react": "^0.553.0",
|
||||||
|
|||||||
+36
-42
@@ -14,31 +14,25 @@ datasource db {
|
|||||||
// All collected data is strictly written to CSV files in /storage/data/{clientId}/{YYYY-MM-DD}.csv
|
// All collected data is strictly written to CSV files in /storage/data/{clientId}/{YYYY-MM-DD}.csv
|
||||||
|
|
||||||
model ClientSDK {
|
model ClientSDK {
|
||||||
id String @id @default(cuid())
|
clientId String @id @map("client_id")
|
||||||
clientId String @unique
|
|
||||||
name String
|
name String
|
||||||
secret String
|
secret String
|
||||||
rateLimit Int @default(100) // Max events per second
|
token String
|
||||||
status String @default("active") // active | revoked
|
rateLimit Int @default(100) @map("rate_limit") // Max events per second
|
||||||
totalEvents BigInt @default(0)
|
status String @default("active") // active | revoked
|
||||||
lastActive DateTime?
|
totalEvents BigInt @default(0) @map("total_events")
|
||||||
createdAt DateTime @default(now())
|
lastActive DateTime? @map("last_active") @db.Timestamptz(6)
|
||||||
updatedAt DateTime @updatedAt
|
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(6)
|
||||||
|
updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(6)
|
||||||
apiKeys ApiKey[]
|
|
||||||
|
|
||||||
@@map("client_sdks")
|
@@map("client_sdks")
|
||||||
}
|
}
|
||||||
|
|
||||||
model ApiKey {
|
model DeletedClient {
|
||||||
id String @id @default(cuid())
|
clientId String @id @map("client_id")
|
||||||
clientId String
|
deletedAt DateTime @default(now()) @map("deleted_at") @db.Timestamptz(6)
|
||||||
client ClientSDK @relation(fields: [clientId], references: [clientId], onDelete: Cascade)
|
|
||||||
keyHash String
|
|
||||||
expiresAt DateTime?
|
|
||||||
createdAt DateTime @default(now())
|
|
||||||
|
|
||||||
@@map("api_keys")
|
@@map("deleted_clients")
|
||||||
}
|
}
|
||||||
|
|
||||||
model AdminUser {
|
model AdminUser {
|
||||||
@@ -46,35 +40,35 @@ model AdminUser {
|
|||||||
email String @unique
|
email String @unique
|
||||||
passwordSalt Bytes @map("password_salt")
|
passwordSalt Bytes @map("password_salt")
|
||||||
passwordHash Bytes @map("password_hash")
|
passwordHash Bytes @map("password_hash")
|
||||||
createdAt DateTime @default(now()) @map("created_at")
|
tokenVersion Int @default(1) @map("token_version")
|
||||||
|
createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(6)
|
||||||
|
|
||||||
@@map("admin_users")
|
@@map("admin_users")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
model AdminLoginAttempt {
|
||||||
|
keyHash String @id @map("key_hash")
|
||||||
|
windowStartedAt DateTime @map("window_started_at") @db.Timestamptz(6)
|
||||||
|
failedAttempts Int @map("failed_attempts")
|
||||||
|
blockedUntil DateTime? @map("blocked_until") @db.Timestamptz(6)
|
||||||
|
updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(6)
|
||||||
|
|
||||||
|
@@map("admin_login_attempts")
|
||||||
|
}
|
||||||
|
|
||||||
model AdminSettings {
|
model AdminSettings {
|
||||||
id String @id @default("default")
|
id String @id @default("default")
|
||||||
serverPort Int @default(4000)
|
serverPort Int @default(4000) @map("server_port")
|
||||||
webPort Int @default(3000)
|
webPort Int @default(3000) @map("web_port")
|
||||||
maxEventsPerSec Int @default(100)
|
maxEventsPerSec Int @default(100) @map("max_events_per_sec")
|
||||||
maxPayloadSizeKb Int @default(100)
|
maxPayloadSizeKb Int @default(100) @map("max_payload_size_kb")
|
||||||
batchFlushThreshold Int @default(500)
|
batchFlushThreshold Int @default(500) @map("batch_flush_threshold")
|
||||||
batchFlushIntervalMs Int @default(2000)
|
batchFlushIntervalMs Int @default(2000) @map("batch_flush_interval_ms")
|
||||||
fileRotationMb Int @default(100)
|
fileRotationMb Int @default(100) @map("file_rotation_mb")
|
||||||
jwtSecret String
|
jwtSecret String @map("jwt_secret")
|
||||||
enableIdempotency Boolean @default(true)
|
enableIdempotency Boolean @default(true) @map("enable_idempotency")
|
||||||
autoPruneDays Int @default(30)
|
autoPruneDays Int @default(30) @map("auto_prune_days")
|
||||||
updatedAt DateTime @updatedAt
|
updatedAt DateTime @default(now()) @updatedAt @map("updated_at") @db.Timestamptz(6)
|
||||||
|
|
||||||
@@map("admin_settings")
|
@@map("admin_settings")
|
||||||
}
|
}
|
||||||
|
|
||||||
model AuditLog {
|
|
||||||
id String @id @default(cuid())
|
|
||||||
action String
|
|
||||||
actor String
|
|
||||||
details String? // JSON string
|
|
||||||
ipAddress String?
|
|
||||||
createdAt DateTime @default(now())
|
|
||||||
|
|
||||||
@@map("audit_logs")
|
|
||||||
}
|
|
||||||
Binary file not shown.
|
After Width: | Height: | Size: 632 B |
Binary file not shown.
|
After Width: | Height: | Size: 718 B |
Binary file not shown.
|
After Width: | Height: | Size: 3.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.7 KiB |
@@ -0,0 +1,7 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512" width="512" height="512">
|
||||||
|
<rect width="512" height="512" fill="#0a0a0b" />
|
||||||
|
<rect x="24" y="24" width="464" height="464" fill="#111113" stroke="#2e2e33" stroke-width="8" />
|
||||||
|
<rect x="112" y="112" width="288" height="288" fill="#a3e635" />
|
||||||
|
<rect x="192" y="192" width="128" height="128" fill="#0a0a0b" />
|
||||||
|
<rect x="232" y="232" width="48" height="48" fill="#e8e8ea" />
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 444 B |
+16
-6
@@ -1,21 +1,31 @@
|
|||||||
{
|
{
|
||||||
"name": "DataStack - Realtime Data Platform",
|
"id": "/",
|
||||||
|
"name": "DataStack - Realtime Data Collection Platform",
|
||||||
"short_name": "DataStack",
|
"short_name": "DataStack",
|
||||||
"description": "Self-hosted universal realtime data collection engine",
|
"description": "Self-hosted universal realtime data collection engine with Socket.IO ingestion, Redis buffer, JWT client auth, CSV batch rotation, and /vcon/ admin panel.",
|
||||||
"start_url": "/vcon/dashboard",
|
"start_url": "/vcon/dashboard",
|
||||||
|
"scope": "/",
|
||||||
"display": "standalone",
|
"display": "standalone",
|
||||||
"background_color": "#0b0f17",
|
"background_color": "#0a0a0b",
|
||||||
"theme_color": "#2563eb",
|
"theme_color": "#0a0a0b",
|
||||||
"icons": [
|
"icons": [
|
||||||
{
|
{
|
||||||
"src": "/icon-192.png",
|
"src": "/icon-192.png",
|
||||||
"sizes": "192x192",
|
"sizes": "192x192",
|
||||||
"type": "image/png"
|
"type": "image/png",
|
||||||
|
"purpose": "any"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"src": "/icon-512.png",
|
"src": "/icon-512.png",
|
||||||
"sizes": "512x512",
|
"sizes": "512x512",
|
||||||
"type": "image/png"
|
"type": "image/png",
|
||||||
|
"purpose": "any"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"src": "/icon-maskable-512.png",
|
||||||
|
"sizes": "512x512",
|
||||||
|
"type": "image/png",
|
||||||
|
"purpose": "maskable"
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
+45
-7
@@ -1,5 +1,13 @@
|
|||||||
const CACHE_NAME = 'datastack-cache-v1';
|
const CACHE_NAME = 'datastack-cache-v2';
|
||||||
const STATIC_ASSETS = ['/', '/vcon/dashboard', '/manifest.json'];
|
const STATIC_ASSETS = [
|
||||||
|
'/',
|
||||||
|
'/manifest.json',
|
||||||
|
'/icon.svg',
|
||||||
|
'/icon-192.png',
|
||||||
|
'/icon-512.png',
|
||||||
|
'/icon-maskable-512.png',
|
||||||
|
'/apple-touch-icon.png',
|
||||||
|
];
|
||||||
|
|
||||||
self.addEventListener('install', (event) => {
|
self.addEventListener('install', (event) => {
|
||||||
event.waitUntil(
|
event.waitUntil(
|
||||||
@@ -22,13 +30,43 @@ self.addEventListener('activate', (event) => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
self.addEventListener('fetch', (event) => {
|
self.addEventListener('fetch', (event) => {
|
||||||
// Let WebSocket and API requests pass through
|
const { request } = event;
|
||||||
if (event.request.url.includes('/socket.io/') || event.request.url.includes('/api/')) {
|
if (request.method !== 'GET') return;
|
||||||
|
|
||||||
|
let url;
|
||||||
|
try {
|
||||||
|
url = new URL(request.url);
|
||||||
|
} catch {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Only handle same-origin GET requests; never intercept WebSocket, HMR, or API routes
|
||||||
|
if (url.origin !== self.location.origin) return;
|
||||||
|
if (
|
||||||
|
url.pathname.startsWith('/socket.io/') ||
|
||||||
|
url.pathname.startsWith('/api/') ||
|
||||||
|
url.pathname.startsWith('/_next/webpack-hmr')
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
event.respondWith(
|
event.respondWith(
|
||||||
fetch(event.request).catch(() => {
|
fetch(request)
|
||||||
return caches.match(event.request);
|
.then((response) => {
|
||||||
})
|
if (response && response.ok && response.type === 'basic') {
|
||||||
|
const clone = response.clone();
|
||||||
|
caches.open(CACHE_NAME).then((cache) => cache.put(request, clone)).catch(() => {});
|
||||||
|
}
|
||||||
|
return response;
|
||||||
|
})
|
||||||
|
.catch(async () => {
|
||||||
|
const cached = await caches.match(request);
|
||||||
|
if (cached) return cached;
|
||||||
|
return new Response('Offline — network unavailable.', {
|
||||||
|
status: 503,
|
||||||
|
statusText: 'Service Unavailable',
|
||||||
|
headers: { 'Content-Type': 'text/plain; charset=utf-8' },
|
||||||
|
});
|
||||||
|
})
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
+11
-9
@@ -6,24 +6,24 @@
|
|||||||
|
|
||||||
Official Python client library for **DataStack — Self-Hosted Real-Time Telemetry & Event Logging Platform**, developed by **[Vib Tools](https://vib.tools/)**.
|
Official Python client library for **DataStack — Self-Hosted Real-Time Telemetry & Event Logging Platform**, developed by **[Vib Tools](https://vib.tools/)**.
|
||||||
|
|
||||||
`datastack-sdk` enables Python services, IoT sensor nodes, microservice workers, and research pipelines to stream structured JSON telemetry events to a DataStack server over Socket.IO with deterministic `threading.Event` acknowledgments.
|
`datastack-sdk` is a compatibility alias for the canonical **`datastack`** package located at [`../sdk/python/datastack`](../sdk/python/datastack). Both `from datastack import DataStack` and `from datastack_sdk import DataStackClient` share a single source of truth with `config.datastack.py` auto-discovery, `CLIENT_ID` wiring, Socket.IO streaming, automatic HTTP fallback, offline disk spooling, and direct `.csv`/`.json`/`.jsonl` file streaming.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
Install directly from the `sdk-python` directory:
|
Install the canonical package directly from `sdk/python/datastack`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pip install ./sdk/python/datastack
|
||||||
|
```
|
||||||
|
|
||||||
|
Or install the compatibility alias from `sdk-python`:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pip install ./sdk-python
|
pip install ./sdk-python
|
||||||
```
|
```
|
||||||
|
|
||||||
For editable development mode:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
cd sdk-python && pip install -e .
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Quick Start
|
## Quick Start
|
||||||
@@ -34,10 +34,12 @@ from datastack_sdk import DataStackClient
|
|||||||
# 1. Initialize the client with a signed JWT token from /vcon/clients or /vcon/sdk
|
# 1. Initialize the client with a signed JWT token from /vcon/clients or /vcon/sdk
|
||||||
client = DataStackClient(
|
client = DataStackClient(
|
||||||
server_url="http://localhost:4000",
|
server_url="http://localhost:4000",
|
||||||
|
api_url="http://localhost:3000",
|
||||||
token="YOUR_CLIENT_JWT_TOKEN",
|
token="YOUR_CLIENT_JWT_TOKEN",
|
||||||
|
client_id="YOUR_CLIENT_ID", # optional (auto-derived from JWT sub/clientId)
|
||||||
)
|
)
|
||||||
|
|
||||||
# 2. Connect to the DataStack Socket.IO server
|
# 2. Connect to the DataStack Socket.IO server (or auto HTTP fallback)
|
||||||
if client.connect():
|
if client.connect():
|
||||||
# 3. Stream a single structured event and wait for server acknowledgment
|
# 3. Stream a single structured event and wait for server acknowledgment
|
||||||
res = client.sync(
|
res = client.sync(
|
||||||
|
|||||||
@@ -1,10 +1,29 @@
|
|||||||
"""
|
"""
|
||||||
DataStack Python SDK (`datastack_sdk`)
|
DataStack Python SDK Compatibility Alias (`datastack_sdk`).
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
Re-exports all canonical symbols from `sdk/python/datastack` (`datastack`)
|
||||||
Official Python client library for DataStack — Self-Hosted Real-Time Telemetry & Event Logging Platform.
|
so both `from datastack import DataStack` and legacy
|
||||||
|
`from datastack_sdk import DataStackClient` share a single source of truth.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from .client import (
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_canonical_datastack_on_path() -> None:
|
||||||
|
try:
|
||||||
|
import datastack # noqa: F401
|
||||||
|
return
|
||||||
|
except ImportError:
|
||||||
|
repo_canonical = (
|
||||||
|
Path(__file__).resolve().parents[2] / "sdk" / "python" / "datastack"
|
||||||
|
)
|
||||||
|
if repo_canonical.is_dir() and str(repo_canonical) not in sys.path:
|
||||||
|
sys.path.insert(0, str(repo_canonical))
|
||||||
|
|
||||||
|
|
||||||
|
_ensure_canonical_datastack_on_path()
|
||||||
|
|
||||||
|
from datastack import ( # noqa: E402, F401
|
||||||
ConnectionState,
|
ConnectionState,
|
||||||
DataStack,
|
DataStack,
|
||||||
DataStackAuthError,
|
DataStackAuthError,
|
||||||
@@ -19,11 +38,10 @@ from .client import (
|
|||||||
DataStackRateLimitError,
|
DataStackRateLimitError,
|
||||||
DiskSpooler,
|
DiskSpooler,
|
||||||
HeartbeatMonitor,
|
HeartbeatMonitor,
|
||||||
|
__version__,
|
||||||
load_config,
|
load_config,
|
||||||
)
|
)
|
||||||
|
|
||||||
__version__ = "2.2.0"
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"DataStack",
|
"DataStack",
|
||||||
"DataStackClient",
|
"DataStackClient",
|
||||||
|
|||||||
File diff suppressed because it is too large.
Load diff
@@ -1,321 +1,7 @@
|
|||||||
"""
|
"""Compatibility re-export of canonical `datastack.config`."""
|
||||||
Auto-Discovery Configuration Loader for `config.datastack.py`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import importlib.util
|
from . import _ensure_canonical_datastack_on_path
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any, Dict, List, Optional
|
|
||||||
from urllib.parse import urlparse
|
|
||||||
|
|
||||||
from .exceptions import DataStackConfigError
|
_ensure_canonical_datastack_on_path()
|
||||||
|
|
||||||
CONFIG_FILENAME = "config.datastack.py"
|
from datastack.config import * # noqa: E402, F401, F403
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class DataStackConfig:
|
|
||||||
server_url: str = "http://localhost:4000"
|
|
||||||
api_url: str = "http://localhost:3000"
|
|
||||||
client_token: str = ""
|
|
||||||
client_id: str = ""
|
|
||||||
auto_connect: bool = True
|
|
||||||
enable_http_fallback: bool = True
|
|
||||||
heartbeat_interval_sec: float = 5.0
|
|
||||||
request_timeout_sec: float = 8.0
|
|
||||||
max_retries: int = 3
|
|
||||||
retry_backoff_base_sec: float = 0.5
|
|
||||||
batch_chunk_size: int = 250
|
|
||||||
offline_spool_enabled: bool = True
|
|
||||||
offline_spool_dir: str = ".datastack_spool"
|
|
||||||
max_spool_size_mb: int = 50
|
|
||||||
config_source_path: Optional[str] = None
|
|
||||||
|
|
||||||
|
|
||||||
def _derive_default_api_url(server_url: str) -> str:
|
|
||||||
"""
|
|
||||||
If API_URL is not explicitly set, infer Port 3000 on the same host when SERVER_URL
|
|
||||||
uses Port 4000, or fall back to http://localhost:3000.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
parsed = urlparse(server_url)
|
|
||||||
if parsed.scheme and parsed.hostname:
|
|
||||||
if parsed.port == 4000:
|
|
||||||
return f"{parsed.scheme}://{parsed.hostname}:3000"
|
|
||||||
return f"{parsed.scheme}://{parsed.netloc}"
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
return "http://localhost:3000"
|
|
||||||
|
|
||||||
|
|
||||||
def _candidate_config_paths(explicit_path: Optional[str] = None) -> List[Path]:
|
|
||||||
candidates: List[Path] = []
|
|
||||||
|
|
||||||
if explicit_path:
|
|
||||||
p = Path(explicit_path).expanduser().resolve()
|
|
||||||
if p.is_dir():
|
|
||||||
candidates.append(p / CONFIG_FILENAME)
|
|
||||||
else:
|
|
||||||
candidates.append(p)
|
|
||||||
|
|
||||||
env_path = os.environ.get("DATASTACK_CONFIG_PATH", "").strip()
|
|
||||||
if env_path:
|
|
||||||
ep = Path(env_path).expanduser().resolve()
|
|
||||||
if ep.is_dir():
|
|
||||||
candidates.append(ep / CONFIG_FILENAME)
|
|
||||||
else:
|
|
||||||
candidates.append(ep)
|
|
||||||
|
|
||||||
# Search current working directory and up to 3 parent levels
|
|
||||||
try:
|
|
||||||
cwd = Path.cwd().resolve()
|
|
||||||
candidates.append(cwd / CONFIG_FILENAME)
|
|
||||||
for parent in list(cwd.parents)[:3]:
|
|
||||||
candidates.append(parent / CONFIG_FILENAME)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Search main entry script directory
|
|
||||||
try:
|
|
||||||
if sys.argv and sys.argv[0]:
|
|
||||||
script_dir = Path(sys.argv[0]).resolve().parent
|
|
||||||
candidates.append(script_dir / CONFIG_FILENAME)
|
|
||||||
for parent in list(script_dir.parents)[:2]:
|
|
||||||
candidates.append(parent / CONFIG_FILENAME)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Deduplicate while preserving order
|
|
||||||
seen = set()
|
|
||||||
unique: List[Path] = []
|
|
||||||
for c in candidates:
|
|
||||||
key = str(c)
|
|
||||||
if key not in seen:
|
|
||||||
seen.add(key)
|
|
||||||
unique.append(c)
|
|
||||||
return unique
|
|
||||||
|
|
||||||
|
|
||||||
def _load_python_config_file(file_path: Path) -> Dict[str, Any]:
|
|
||||||
try:
|
|
||||||
spec = importlib.util.spec_from_file_location(
|
|
||||||
"datastack_user_config", str(file_path)
|
|
||||||
)
|
|
||||||
if spec is None or spec.loader is None:
|
|
||||||
raise DataStackConfigError(
|
|
||||||
f"Unable to load configuration specification from {file_path}"
|
|
||||||
)
|
|
||||||
module = importlib.util.module_from_spec(spec)
|
|
||||||
spec.loader.exec_module(module)
|
|
||||||
|
|
||||||
raw: Dict[str, Any] = {}
|
|
||||||
if hasattr(module, "DATASTACK_CONFIG") and isinstance(
|
|
||||||
getattr(module, "DATASTACK_CONFIG"), dict
|
|
||||||
):
|
|
||||||
raw.update(getattr(module, "DATASTACK_CONFIG"))
|
|
||||||
|
|
||||||
# Also support top-level module variables (e.g., SERVER_URL = "...", CLIENT_TOKEN = "...")
|
|
||||||
for attr in dir(module):
|
|
||||||
if attr.isupper() and attr != "DATASTACK_CONFIG":
|
|
||||||
raw[attr] = getattr(module, attr)
|
|
||||||
|
|
||||||
return raw
|
|
||||||
except DataStackConfigError:
|
|
||||||
raise
|
|
||||||
except Exception as exc:
|
|
||||||
raise DataStackConfigError(
|
|
||||||
f"Failed to parse {file_path}: {exc}"
|
|
||||||
) from exc
|
|
||||||
|
|
||||||
|
|
||||||
def load_config(
|
|
||||||
config_path: Optional[str] = None,
|
|
||||||
require_token: bool = False,
|
|
||||||
**overrides: Any,
|
|
||||||
) -> DataStackConfig:
|
|
||||||
"""
|
|
||||||
Discovers and loads `config.datastack.py` from the client application's
|
|
||||||
directory, merges environment variables (`DATASTACK_*`), and applies
|
|
||||||
any explicit keyword overrides.
|
|
||||||
"""
|
|
||||||
file_values: Dict[str, Any] = {}
|
|
||||||
resolved_source: Optional[str] = None
|
|
||||||
|
|
||||||
for candidate in _candidate_config_paths(config_path):
|
|
||||||
if candidate.is_file():
|
|
||||||
file_values = _load_python_config_file(candidate)
|
|
||||||
resolved_source = str(candidate)
|
|
||||||
break
|
|
||||||
|
|
||||||
if config_path and not resolved_source:
|
|
||||||
raise DataStackConfigError(
|
|
||||||
f"Explicitly requested config file was not found: {config_path}"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Normalize keys to uppercase for lookup
|
|
||||||
normalized_file = {str(k).upper(): v for k, v in file_values.items()}
|
|
||||||
normalized_overrides = {
|
|
||||||
str(k).upper(): v for k, v in overrides.items() if v is not None
|
|
||||||
}
|
|
||||||
|
|
||||||
def pick(keys: List[str], env_keys: List[str], default: Any) -> Any:
|
|
||||||
for k in keys:
|
|
||||||
if k in normalized_overrides:
|
|
||||||
return normalized_overrides[k]
|
|
||||||
for ek in env_keys:
|
|
||||||
env_val = os.environ.get(ek)
|
|
||||||
if env_val is not None and str(env_val).strip() != "":
|
|
||||||
return env_val
|
|
||||||
for k in keys:
|
|
||||||
if k in normalized_file and normalized_file[k] is not None:
|
|
||||||
return normalized_file[k]
|
|
||||||
return default
|
|
||||||
|
|
||||||
def to_bool(val: Any, default: bool) -> bool:
|
|
||||||
if isinstance(val, bool):
|
|
||||||
return val
|
|
||||||
if isinstance(val, str):
|
|
||||||
return val.strip().lower() in ("1", "true", "yes", "on")
|
|
||||||
if val is None:
|
|
||||||
return default
|
|
||||||
return bool(val)
|
|
||||||
|
|
||||||
server_url = str(
|
|
||||||
pick(
|
|
||||||
["SERVER_URL", "SOCKET_URL", "WS_URL"],
|
|
||||||
["DATASTACK_SERVER_URL", "DATASTACK_SOCKET_URL"],
|
|
||||||
"http://localhost:4000",
|
|
||||||
)
|
|
||||||
).rstrip("/")
|
|
||||||
|
|
||||||
default_api = _derive_default_api_url(server_url)
|
|
||||||
api_url = str(
|
|
||||||
pick(
|
|
||||||
["API_URL", "HTTP_URL", "WEB_URL"],
|
|
||||||
["DATASTACK_API_URL", "DATASTACK_HTTP_URL"],
|
|
||||||
default_api,
|
|
||||||
)
|
|
||||||
).rstrip("/")
|
|
||||||
|
|
||||||
client_token = str(
|
|
||||||
pick(
|
|
||||||
["CLIENT_TOKEN", "TOKEN", "JWT_TOKEN", "API_TOKEN"],
|
|
||||||
["DATASTACK_CLIENT_TOKEN", "DATASTACK_TOKEN"],
|
|
||||||
"",
|
|
||||||
)
|
|
||||||
).strip()
|
|
||||||
|
|
||||||
client_id = str(
|
|
||||||
pick(
|
|
||||||
["CLIENT_ID"],
|
|
||||||
["DATASTACK_CLIENT_ID"],
|
|
||||||
"",
|
|
||||||
)
|
|
||||||
).strip()
|
|
||||||
|
|
||||||
cfg = DataStackConfig(
|
|
||||||
server_url=server_url,
|
|
||||||
api_url=api_url,
|
|
||||||
client_token=client_token,
|
|
||||||
client_id=client_id,
|
|
||||||
auto_connect=to_bool(
|
|
||||||
pick(["AUTO_CONNECT"], ["DATASTACK_AUTO_CONNECT"], True), True
|
|
||||||
),
|
|
||||||
enable_http_fallback=to_bool(
|
|
||||||
pick(
|
|
||||||
["ENABLE_HTTP_FALLBACK", "HTTP_FALLBACK"],
|
|
||||||
["DATASTACK_ENABLE_HTTP_FALLBACK"],
|
|
||||||
True,
|
|
||||||
),
|
|
||||||
True,
|
|
||||||
),
|
|
||||||
heartbeat_interval_sec=max(
|
|
||||||
1.0,
|
|
||||||
float(
|
|
||||||
pick(
|
|
||||||
["HEARTBEAT_INTERVAL_SEC", "HEARTBEAT_INTERVAL"],
|
|
||||||
["DATASTACK_HEARTBEAT_INTERVAL_SEC"],
|
|
||||||
5.0,
|
|
||||||
)
|
|
||||||
),
|
|
||||||
),
|
|
||||||
request_timeout_sec=max(
|
|
||||||
1.0,
|
|
||||||
float(
|
|
||||||
pick(
|
|
||||||
["REQUEST_TIMEOUT_SEC", "TIMEOUT"],
|
|
||||||
["DATASTACK_REQUEST_TIMEOUT_SEC"],
|
|
||||||
8.0,
|
|
||||||
)
|
|
||||||
),
|
|
||||||
),
|
|
||||||
max_retries=max(
|
|
||||||
0,
|
|
||||||
int(
|
|
||||||
pick(
|
|
||||||
["MAX_RETRIES", "RETRIES"],
|
|
||||||
["DATASTACK_MAX_RETRIES"],
|
|
||||||
3,
|
|
||||||
)
|
|
||||||
),
|
|
||||||
),
|
|
||||||
retry_backoff_base_sec=max(
|
|
||||||
0.1,
|
|
||||||
float(
|
|
||||||
pick(
|
|
||||||
["RETRY_BACKOFF_BASE_SEC"],
|
|
||||||
["DATASTACK_RETRY_BACKOFF_BASE_SEC"],
|
|
||||||
0.5,
|
|
||||||
)
|
|
||||||
),
|
|
||||||
),
|
|
||||||
batch_chunk_size=max(
|
|
||||||
1,
|
|
||||||
int(
|
|
||||||
pick(
|
|
||||||
["BATCH_CHUNK_SIZE", "CHUNK_SIZE"],
|
|
||||||
["DATASTACK_BATCH_CHUNK_SIZE"],
|
|
||||||
250,
|
|
||||||
)
|
|
||||||
),
|
|
||||||
),
|
|
||||||
offline_spool_enabled=to_bool(
|
|
||||||
pick(
|
|
||||||
["OFFLINE_SPOOL_ENABLED"],
|
|
||||||
["DATASTACK_OFFLINE_SPOOL_ENABLED"],
|
|
||||||
True,
|
|
||||||
),
|
|
||||||
True,
|
|
||||||
),
|
|
||||||
offline_spool_dir=str(
|
|
||||||
pick(
|
|
||||||
["OFFLINE_SPOOL_DIR", "SPOOL_DIR"],
|
|
||||||
["DATASTACK_OFFLINE_SPOOL_DIR"],
|
|
||||||
".datastack_spool",
|
|
||||||
)
|
|
||||||
),
|
|
||||||
max_spool_size_mb=max(
|
|
||||||
1,
|
|
||||||
int(
|
|
||||||
pick(
|
|
||||||
["MAX_SPOOL_SIZE_MB"],
|
|
||||||
["DATASTACK_MAX_SPOOL_SIZE_MB"],
|
|
||||||
50,
|
|
||||||
)
|
|
||||||
),
|
|
||||||
),
|
|
||||||
config_source_path=resolved_source,
|
|
||||||
)
|
|
||||||
|
|
||||||
if require_token and (
|
|
||||||
not cfg.client_token
|
|
||||||
or cfg.client_token == "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN"
|
|
||||||
):
|
|
||||||
raise DataStackConfigError(
|
|
||||||
"DataStack CLIENT_TOKEN is missing. Add your token to `config.datastack.py`, "
|
|
||||||
"set DATASTACK_TOKEN in your environment, or pass token='...' to DataStack()."
|
|
||||||
)
|
|
||||||
|
|
||||||
return cfg
|
|
||||||
@@ -1,72 +1,7 @@
|
|||||||
"""
|
"""Compatibility re-export of canonical `datastack.exceptions`."""
|
||||||
Structured Exception Hierarchy for the DataStack Python SDK.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from typing import Optional
|
from . import _ensure_canonical_datastack_on_path
|
||||||
|
|
||||||
|
_ensure_canonical_datastack_on_path()
|
||||||
|
|
||||||
class DataStackError(Exception):
|
from datastack.exceptions import * # noqa: E402, F401, F403
|
||||||
"""Base exception for all DataStack SDK errors."""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
message: str,
|
|
||||||
code: Optional[str] = None,
|
|
||||||
retryable: bool = False,
|
|
||||||
) -> None:
|
|
||||||
super().__init__(message)
|
|
||||||
self.message = message
|
|
||||||
self.code = code or "DATASTACK_ERROR"
|
|
||||||
self.retryable = retryable
|
|
||||||
|
|
||||||
|
|
||||||
class DataStackConfigError(DataStackError):
|
|
||||||
"""Raised when `config.datastack.py` or required credentials are missing or invalid."""
|
|
||||||
|
|
||||||
def __init__(self, message: str) -> None:
|
|
||||||
super().__init__(message, code="CONFIG_ERROR", retryable=False)
|
|
||||||
|
|
||||||
|
|
||||||
class DataStackAuthError(DataStackError):
|
|
||||||
"""
|
|
||||||
Raised when authentication fails (`AUTH_REQUIRED`, `AUTH_INVALID`,
|
|
||||||
`CLIENT_REVOKED`, or `TOKEN_ROTATED`). Non-retryable until credentials are updated.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, message: str, code: str = "AUTH_ERROR") -> None:
|
|
||||||
super().__init__(message, code=code, retryable=False)
|
|
||||||
|
|
||||||
|
|
||||||
class DataStackRateLimitError(DataStackError):
|
|
||||||
"""Raised when `RATE_LIMIT_EXCEEDED` persists after all backoff retries."""
|
|
||||||
|
|
||||||
def __init__(self, message: str) -> None:
|
|
||||||
super().__init__(message, code="RATE_LIMIT_EXCEEDED", retryable=True)
|
|
||||||
|
|
||||||
|
|
||||||
class DataStackPayloadTooLargeError(DataStackError):
|
|
||||||
"""Raised when a single event exceeds the server's `maxPayloadSizeKb` limit."""
|
|
||||||
|
|
||||||
def __init__(self, message: str) -> None:
|
|
||||||
super().__init__(message, code="PAYLOAD_TOO_LARGE", retryable=False)
|
|
||||||
|
|
||||||
|
|
||||||
class DataStackConnectionError(DataStackError):
|
|
||||||
"""Raised when both WebSocket and HTTP fallback are unreachable and offline spooling is disabled."""
|
|
||||||
|
|
||||||
def __init__(self, message: str) -> None:
|
|
||||||
super().__init__(message, code="CONNECTION_ERROR", retryable=True)
|
|
||||||
|
|
||||||
|
|
||||||
class DataStackFileError(DataStackError):
|
|
||||||
"""Raised when a file passed to `send_file()` is missing, unsupported, or malformed."""
|
|
||||||
|
|
||||||
def __init__(self, message: str) -> None:
|
|
||||||
super().__init__(message, code="FILE_ERROR", retryable=False)
|
|
||||||
|
|
||||||
|
|
||||||
class DataStackOfflineSpoolError(DataStackError):
|
|
||||||
"""Raised when the local offline disk spool directory is full or unwriteable."""
|
|
||||||
|
|
||||||
def __init__(self, message: str) -> None:
|
|
||||||
super().__init__(message, code="OFFLINE_SPOOL_ERROR", retryable=False)
|
|
||||||
@@ -1,186 +1,7 @@
|
|||||||
"""
|
"""Compatibility re-export of canonical `datastack.heartbeat`."""
|
||||||
Smart Heartbeat Monitor & Connection State Machine (`HeartbeatMonitor`).
|
|
||||||
|
|
||||||
Lifecycle States:
|
from . import _ensure_canonical_datastack_on_path
|
||||||
- `CONNECTED_IDLE`: WebSocket (Port 4000) is connected and verified. Heartbeat sleeps in IDLE mode.
|
|
||||||
- `FALLBACK_HTTP`: WebSocket disconnected. Client automatically sends events via HTTP (`POST /api/events`)
|
|
||||||
while Heartbeat actively probes to reconnect and verify WebSocket, then returns to `CONNECTED_IDLE`.
|
|
||||||
- `OFFLINE_SPOOL`: Device is completely offline. Events spool to disk (`.datastack_spool/spool.jsonl`).
|
|
||||||
Heartbeat monitors network restoration, reconnects, replays spooled events, and returns to `CONNECTED_IDLE`.
|
|
||||||
- `RECONNECTING_VERIFY`: Heartbeat is verifying handshake (`ping_verify`) and draining spooled events.
|
|
||||||
- `STOPPED`: Client was closed intentionally or halted due to non-retryable auth revocation.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import logging
|
_ensure_canonical_datastack_on_path()
|
||||||
import threading
|
|
||||||
import time
|
|
||||||
from typing import Any, Callable, Dict, Optional
|
|
||||||
|
|
||||||
logger = logging.getLogger("datastack.heartbeat")
|
from datastack.heartbeat import * # noqa: E402, F401, F403
|
||||||
|
|
||||||
|
|
||||||
class ConnectionState:
|
|
||||||
CONNECTED_IDLE = "CONNECTED_IDLE"
|
|
||||||
FALLBACK_HTTP = "FALLBACK_HTTP"
|
|
||||||
OFFLINE_SPOOL = "OFFLINE_SPOOL"
|
|
||||||
RECONNECTING_VERIFY = "RECONNECTING_VERIFY"
|
|
||||||
STOPPED = "STOPPED"
|
|
||||||
|
|
||||||
|
|
||||||
class HeartbeatMonitor:
|
|
||||||
"""
|
|
||||||
Background recovery thread that sleeps in zero-CPU IDLE mode while WebSocket is
|
|
||||||
healthy (`CONNECTED_IDLE`) and wakes up automatically when WebSocket disconnects
|
|
||||||
or when the client device goes offline.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
interval_sec: float,
|
|
||||||
recovery_callback: Callable[[], bool],
|
|
||||||
) -> None:
|
|
||||||
self.interval_sec = max(0.2, float(interval_sec))
|
|
||||||
self._recovery_callback = recovery_callback
|
|
||||||
self._state = ConnectionState.CONNECTED_IDLE
|
|
||||||
self._closed = False
|
|
||||||
self._last_error: Optional[str] = None
|
|
||||||
self._recovery_attempts = 0
|
|
||||||
self._last_verified_at: Optional[float] = None
|
|
||||||
|
|
||||||
self._lock = threading.RLock()
|
|
||||||
self._wake_event = threading.Event()
|
|
||||||
self._stop_event = threading.Event()
|
|
||||||
self._thread: Optional[threading.Thread] = None
|
|
||||||
|
|
||||||
@property
|
|
||||||
def state(self) -> str:
|
|
||||||
with self._lock:
|
|
||||||
return self._state
|
|
||||||
|
|
||||||
@property
|
|
||||||
def is_idle(self) -> bool:
|
|
||||||
with self._lock:
|
|
||||||
return self._state == ConnectionState.CONNECTED_IDLE
|
|
||||||
|
|
||||||
def set_idle_connected(self) -> None:
|
|
||||||
"""
|
|
||||||
Marks the connection as healthy (`CONNECTED_IDLE`) and puts the Heartbeat
|
|
||||||
Monitor into IDLE mode (`_wake_event.clear()`).
|
|
||||||
"""
|
|
||||||
with self._lock:
|
|
||||||
self._closed = False
|
|
||||||
self._state = ConnectionState.CONNECTED_IDLE
|
|
||||||
self._recovery_attempts = 0
|
|
||||||
self._last_error = None
|
|
||||||
self._last_verified_at = time.time()
|
|
||||||
self._wake_event.clear()
|
|
||||||
|
|
||||||
def trigger_fallback(self, reason: str = "WebSocket disconnected") -> None:
|
|
||||||
"""
|
|
||||||
Transitions to `FALLBACK_HTTP` and wakes up the Heartbeat Monitor thread
|
|
||||||
to repair the WebSocket connection in the background.
|
|
||||||
"""
|
|
||||||
with self._lock:
|
|
||||||
if self._closed:
|
|
||||||
return
|
|
||||||
self._state = ConnectionState.FALLBACK_HTTP
|
|
||||||
self._last_error = reason
|
|
||||||
self._ensure_thread_started()
|
|
||||||
self._wake_event.set()
|
|
||||||
|
|
||||||
def trigger_offline(self, reason: str = "Device offline") -> None:
|
|
||||||
"""
|
|
||||||
Transitions to `OFFLINE_SPOOL` when both WebSocket and HTTP fallback are
|
|
||||||
unreachable, waking the Heartbeat Monitor to watch for network return.
|
|
||||||
"""
|
|
||||||
with self._lock:
|
|
||||||
if self._closed:
|
|
||||||
return
|
|
||||||
self._state = ConnectionState.OFFLINE_SPOOL
|
|
||||||
self._last_error = reason
|
|
||||||
self._ensure_thread_started()
|
|
||||||
self._wake_event.set()
|
|
||||||
|
|
||||||
def set_reconnecting(self) -> None:
|
|
||||||
with self._lock:
|
|
||||||
if not self._closed:
|
|
||||||
self._state = ConnectionState.RECONNECTING_VERIFY
|
|
||||||
|
|
||||||
def start(self) -> None:
|
|
||||||
with self._lock:
|
|
||||||
self._closed = False
|
|
||||||
self._stop_event.clear()
|
|
||||||
self._ensure_thread_started()
|
|
||||||
|
|
||||||
def stop(self, reason: Optional[str] = None) -> None:
|
|
||||||
with self._lock:
|
|
||||||
self._closed = True
|
|
||||||
self._state = ConnectionState.STOPPED
|
|
||||||
if reason:
|
|
||||||
self._last_error = reason
|
|
||||||
self._stop_event.set()
|
|
||||||
self._wake_event.set()
|
|
||||||
|
|
||||||
def status_snapshot(self) -> Dict[str, Any]:
|
|
||||||
with self._lock:
|
|
||||||
return {
|
|
||||||
"state": self._state,
|
|
||||||
"heartbeatMode": (
|
|
||||||
"IDLE"
|
|
||||||
if self._state == ConnectionState.CONNECTED_IDLE
|
|
||||||
else "ACTIVE_RECOVERY"
|
|
||||||
if self._state
|
|
||||||
in (
|
|
||||||
ConnectionState.FALLBACK_HTTP,
|
|
||||||
ConnectionState.OFFLINE_SPOOL,
|
|
||||||
ConnectionState.RECONNECTING_VERIFY,
|
|
||||||
)
|
|
||||||
else "STOPPED"
|
|
||||||
),
|
|
||||||
"recoveryAttempts": self._recovery_attempts,
|
|
||||||
"lastError": self._last_error,
|
|
||||||
"lastVerifiedAt": self._last_verified_at,
|
|
||||||
"intervalSec": self.interval_sec,
|
|
||||||
}
|
|
||||||
|
|
||||||
def _ensure_thread_started(self) -> None:
|
|
||||||
if self._thread is not None and self._thread.is_alive():
|
|
||||||
return
|
|
||||||
self._stop_event.clear()
|
|
||||||
self._thread = threading.Thread(
|
|
||||||
target=self._run_loop,
|
|
||||||
name="DataStack-HeartbeatMonitor",
|
|
||||||
daemon=True,
|
|
||||||
)
|
|
||||||
self._thread.start()
|
|
||||||
|
|
||||||
def _run_loop(self) -> None:
|
|
||||||
while not self._stop_event.is_set():
|
|
||||||
# Wait in zero-CPU IDLE mode until woken by a disconnect or offline trigger
|
|
||||||
self._wake_event.wait()
|
|
||||||
if self._stop_event.is_set():
|
|
||||||
break
|
|
||||||
|
|
||||||
# Wait for the heartbeat interval before probing
|
|
||||||
if self._stop_event.wait(self.interval_sec):
|
|
||||||
break
|
|
||||||
|
|
||||||
if not self._wake_event.is_set():
|
|
||||||
continue
|
|
||||||
|
|
||||||
with self._lock:
|
|
||||||
if self._closed or self._state == ConnectionState.STOPPED:
|
|
||||||
break
|
|
||||||
self._recovery_attempts += 1
|
|
||||||
|
|
||||||
try:
|
|
||||||
recovered_to_idle = self._recovery_callback()
|
|
||||||
if recovered_to_idle:
|
|
||||||
self.set_idle_connected()
|
|
||||||
logger.info(
|
|
||||||
"[DataStack Heartbeat] WebSocket reconnected & verified. Returning to IDLE mode."
|
|
||||||
)
|
|
||||||
except Exception as exc:
|
|
||||||
with self._lock:
|
|
||||||
self._last_error = str(exc)
|
|
||||||
logger.debug("[DataStack Heartbeat] Recovery probe failed: %s", exc)
|
|
||||||
@@ -1,164 +1,7 @@
|
|||||||
"""
|
"""Compatibility re-export of canonical `datastack.spool`."""
|
||||||
Thread-Safe Offline Disk Spooler (`DiskSpooler`).
|
|
||||||
Persists unsent telemetry events to `.datastack_spool/spool.jsonl` when a client
|
|
||||||
device loses network connectivity completely, and drains them in deterministic
|
|
||||||
batches once the Heartbeat Monitor restores connectivity.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import json
|
from . import _ensure_canonical_datastack_on_path
|
||||||
import os
|
|
||||||
import threading
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any, Dict, List
|
|
||||||
|
|
||||||
from .exceptions import DataStackOfflineSpoolError
|
_ensure_canonical_datastack_on_path()
|
||||||
|
|
||||||
|
from datastack.spool import * # noqa: E402, F401, F403
|
||||||
class DiskSpooler:
|
|
||||||
"""
|
|
||||||
Append-only JSONL disk queue with atomic batch drainage and size cap protection.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
spool_dir: str = ".datastack_spool",
|
|
||||||
max_size_mb: int = 50,
|
|
||||||
enabled: bool = True,
|
|
||||||
) -> None:
|
|
||||||
self.enabled = enabled
|
|
||||||
self.spool_dir = Path(spool_dir)
|
|
||||||
self.spool_file = self.spool_dir / "spool.jsonl"
|
|
||||||
self.max_bytes = max(1, int(max_size_mb)) * 1024 * 1024
|
|
||||||
self._lock = threading.RLock()
|
|
||||||
|
|
||||||
def _ensure_dir(self) -> None:
|
|
||||||
try:
|
|
||||||
self.spool_dir.mkdir(parents=True, exist_ok=True)
|
|
||||||
except Exception as exc:
|
|
||||||
raise DataStackOfflineSpoolError(
|
|
||||||
f"Cannot create offline spool directory '{self.spool_dir}': {exc}"
|
|
||||||
) from exc
|
|
||||||
|
|
||||||
def enqueue(self, events: List[Dict[str, Any]]) -> int:
|
|
||||||
"""
|
|
||||||
Appends one or more formatted events (`{eventId, timestamp, data}`)
|
|
||||||
to `.datastack_spool/spool.jsonl`. Returns count of spooled events.
|
|
||||||
"""
|
|
||||||
if not self.enabled or not events:
|
|
||||||
return 0
|
|
||||||
|
|
||||||
with self._lock:
|
|
||||||
self._ensure_dir()
|
|
||||||
current_size = (
|
|
||||||
self.spool_file.stat().st_size if self.spool_file.exists() else 0
|
|
||||||
)
|
|
||||||
|
|
||||||
lines: List[str] = []
|
|
||||||
added_bytes = 0
|
|
||||||
for evt in events:
|
|
||||||
serialized = json.dumps(evt, ensure_ascii=False) + "\n"
|
|
||||||
encoded_len = len(serialized.encode("utf-8"))
|
|
||||||
if current_size + added_bytes + encoded_len > self.max_bytes:
|
|
||||||
raise DataStackOfflineSpoolError(
|
|
||||||
f"Offline spool limit ({self.max_bytes // (1024 * 1024)} MB) "
|
|
||||||
f"reached at '{self.spool_file}'."
|
|
||||||
)
|
|
||||||
lines.append(serialized)
|
|
||||||
added_bytes += encoded_len
|
|
||||||
|
|
||||||
try:
|
|
||||||
with self.spool_file.open("a", encoding="utf-8") as fh:
|
|
||||||
fh.writelines(lines)
|
|
||||||
fh.flush()
|
|
||||||
except Exception as exc:
|
|
||||||
raise DataStackOfflineSpoolError(
|
|
||||||
f"Failed writing to offline spool file '{self.spool_file}': {exc}"
|
|
||||||
) from exc
|
|
||||||
|
|
||||||
return len(lines)
|
|
||||||
|
|
||||||
def peek_batch(self, limit: int = 250) -> List[Dict[str, Any]]:
|
|
||||||
"""
|
|
||||||
Reads up to `limit` spooled events from the front of the spool file
|
|
||||||
without removing them until `ack_batch()` confirms delivery.
|
|
||||||
"""
|
|
||||||
if not self.enabled:
|
|
||||||
return []
|
|
||||||
|
|
||||||
with self._lock:
|
|
||||||
if not self.spool_file.exists():
|
|
||||||
return []
|
|
||||||
|
|
||||||
batch: List[Dict[str, Any]] = []
|
|
||||||
try:
|
|
||||||
with self.spool_file.open("r", encoding="utf-8") as fh:
|
|
||||||
for line in fh:
|
|
||||||
if len(batch) >= limit:
|
|
||||||
break
|
|
||||||
stripped = line.strip()
|
|
||||||
if not stripped:
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
parsed = json.loads(stripped)
|
|
||||||
if isinstance(parsed, dict):
|
|
||||||
batch.append(parsed)
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
continue
|
|
||||||
except Exception:
|
|
||||||
return []
|
|
||||||
|
|
||||||
return batch
|
|
||||||
|
|
||||||
def ack_batch(self, count: int) -> int:
|
|
||||||
"""
|
|
||||||
Removes the first `count` non-empty lines from the spool file atomically
|
|
||||||
after the Heartbeat Monitor confirms they have been ingested by DataStack.
|
|
||||||
"""
|
|
||||||
if not self.enabled or count <= 0:
|
|
||||||
return 0
|
|
||||||
|
|
||||||
with self._lock:
|
|
||||||
if not self.spool_file.exists():
|
|
||||||
return 0
|
|
||||||
|
|
||||||
try:
|
|
||||||
with self.spool_file.open("r", encoding="utf-8") as fh:
|
|
||||||
all_lines = [ln for ln in fh if ln.strip()]
|
|
||||||
|
|
||||||
if count >= len(all_lines):
|
|
||||||
self.spool_file.unlink(missing_ok=True)
|
|
||||||
return len(all_lines)
|
|
||||||
|
|
||||||
remaining = all_lines[count:]
|
|
||||||
tmp_file = self.spool_dir / "spool.jsonl.tmp"
|
|
||||||
with tmp_file.open("w", encoding="utf-8") as fh:
|
|
||||||
fh.writelines(remaining)
|
|
||||||
fh.flush()
|
|
||||||
os.replace(str(tmp_file), str(self.spool_file))
|
|
||||||
return count
|
|
||||||
except Exception:
|
|
||||||
return 0
|
|
||||||
|
|
||||||
def pending_count(self) -> int:
|
|
||||||
"""Returns the number of queued events waiting in the disk spool."""
|
|
||||||
if not self.enabled:
|
|
||||||
return 0
|
|
||||||
|
|
||||||
with self._lock:
|
|
||||||
if not self.spool_file.exists():
|
|
||||||
return 0
|
|
||||||
try:
|
|
||||||
with self.spool_file.open("r", encoding="utf-8") as fh:
|
|
||||||
return sum(1 for ln in fh if ln.strip())
|
|
||||||
except Exception:
|
|
||||||
return 0
|
|
||||||
|
|
||||||
def size_bytes(self) -> int:
|
|
||||||
"""Returns current disk usage of the spool file in bytes."""
|
|
||||||
with self._lock:
|
|
||||||
if not self.spool_file.exists():
|
|
||||||
return 0
|
|
||||||
try:
|
|
||||||
return self.spool_file.stat().st_size
|
|
||||||
except Exception:
|
|
||||||
return 0
|
|
||||||
@@ -33,6 +33,7 @@ DATASTACK_CONFIG = {
|
|||||||
"SERVER_URL": "http://localhost:4000", # Port 4000 Socket.IO Ingestion Service
|
"SERVER_URL": "http://localhost:4000", # Port 4000 Socket.IO Ingestion Service
|
||||||
"API_URL": "http://localhost:3000", # Port 3000 HTTP Fallback & File Ingestion API
|
"API_URL": "http://localhost:3000", # Port 3000 HTTP Fallback & File Ingestion API
|
||||||
"CLIENT_TOKEN": "YOUR_CLIENT_JWT_TOKEN", # Provisioned in /vcon/clients
|
"CLIENT_TOKEN": "YOUR_CLIENT_JWT_TOKEN", # Provisioned in /vcon/clients
|
||||||
|
"CLIENT_ID": "", # Optional override (defaults to JWT sub/clientId)
|
||||||
"AUTO_CONNECT": True,
|
"AUTO_CONNECT": True,
|
||||||
"ENABLE_HTTP_FALLBACK": True,
|
"ENABLE_HTTP_FALLBACK": True,
|
||||||
"HEARTBEAT_INTERVAL_SEC": 5.0,
|
"HEARTBEAT_INTERVAL_SEC": 5.0,
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ Socket.IO ingestion (Port 4000), automatic Heartbeat HTTP fallback (Port 3000),
|
|||||||
offline disk spooling, and direct `.csv` / `.json` / `.jsonl` file streaming.
|
offline disk spooling, and direct `.csv` / `.json` / `.jsonl` file streaming.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
from .client import DataStack, DataStackClient
|
||||||
from .config import DataStackConfig, load_config
|
from .config import DataStackConfig, load_config
|
||||||
from .exceptions import (
|
from .exceptions import (
|
||||||
DataStackAuthError,
|
DataStackAuthError,
|
||||||
@@ -16,14 +17,9 @@ from .exceptions import (
|
|||||||
DataStackPayloadTooLargeError,
|
DataStackPayloadTooLargeError,
|
||||||
DataStackRateLimitError,
|
DataStackRateLimitError,
|
||||||
)
|
)
|
||||||
|
from .heartbeat import ConnectionState, HeartbeatMonitor
|
||||||
from .spool import DiskSpooler
|
from .spool import DiskSpooler
|
||||||
|
|
||||||
try:
|
|
||||||
from .client import DataStack, DataStackClient
|
|
||||||
except ImportError:
|
|
||||||
DataStack = None # type: ignore
|
|
||||||
DataStackClient = None # type: ignore
|
|
||||||
|
|
||||||
__version__ = "2.2.0"
|
__version__ = "2.2.0"
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
@@ -31,6 +27,8 @@ __all__ = [
|
|||||||
"DataStackClient",
|
"DataStackClient",
|
||||||
"DataStackConfig",
|
"DataStackConfig",
|
||||||
"load_config",
|
"load_config",
|
||||||
|
"ConnectionState",
|
||||||
|
"HeartbeatMonitor",
|
||||||
"DiskSpooler",
|
"DiskSpooler",
|
||||||
"DataStackError",
|
"DataStackError",
|
||||||
"DataStackConfigError",
|
"DataStackConfigError",
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ Features:
|
|||||||
and lossless partial-batch rate-limit retry (`429 RATE_LIMIT_EXCEEDED`).
|
and lossless partial-batch rate-limit retry (`429 RATE_LIMIT_EXCEEDED`).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import base64
|
||||||
import csv
|
import csv
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
@@ -44,6 +45,7 @@ from .exceptions import (
|
|||||||
DataStackConnectionError,
|
DataStackConnectionError,
|
||||||
DataStackError,
|
DataStackError,
|
||||||
DataStackFileError,
|
DataStackFileError,
|
||||||
|
DataStackOfflineSpoolError,
|
||||||
DataStackPayloadTooLargeError,
|
DataStackPayloadTooLargeError,
|
||||||
DataStackRateLimitError,
|
DataStackRateLimitError,
|
||||||
)
|
)
|
||||||
@@ -72,6 +74,23 @@ def _make_deterministic_file_event_id(
|
|||||||
return f"file_{safe_stem}_{row_index}_{digest}"
|
return f"file_{safe_stem}_{row_index}_{digest}"
|
||||||
|
|
||||||
|
|
||||||
|
def _decode_jwt_client_id(token: str) -> str:
|
||||||
|
"""Safely extracts `clientId` or `sub` claim from an unverified JWT payload string."""
|
||||||
|
try:
|
||||||
|
parts = (token or "").split(".")
|
||||||
|
if len(parts) != 3:
|
||||||
|
return ""
|
||||||
|
payload_b64 = parts[1]
|
||||||
|
padding = "=" * (-len(payload_b64) % 4)
|
||||||
|
raw = base64.urlsafe_b64decode(payload_b64 + padding).decode("utf-8")
|
||||||
|
claims = json.loads(raw)
|
||||||
|
if isinstance(claims, dict):
|
||||||
|
return str(claims.get("clientId") or claims.get("sub") or "").strip()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
class DataStackClient:
|
class DataStackClient:
|
||||||
"""
|
"""
|
||||||
Production-grade DataStack Python Client with `config.datastack.py` auto-loading,
|
Production-grade DataStack Python Client with `config.datastack.py` auto-loading,
|
||||||
@@ -87,6 +106,7 @@ class DataStackClient:
|
|||||||
auto_connect: Optional[bool] = None,
|
auto_connect: Optional[bool] = None,
|
||||||
reconnection_attempts: Optional[int] = None,
|
reconnection_attempts: Optional[int] = None,
|
||||||
reconnection_delay: Optional[float] = None,
|
reconnection_delay: Optional[float] = None,
|
||||||
|
client_id: Optional[str] = None,
|
||||||
**kwargs: Any,
|
**kwargs: Any,
|
||||||
) -> None:
|
) -> None:
|
||||||
if reconnection_attempts is not None and "MAX_RETRIES" not in kwargs:
|
if reconnection_attempts is not None and "MAX_RETRIES" not in kwargs:
|
||||||
@@ -105,6 +125,7 @@ class DataStackClient:
|
|||||||
SERVER_URL=server_url,
|
SERVER_URL=server_url,
|
||||||
CLIENT_TOKEN=token,
|
CLIENT_TOKEN=token,
|
||||||
API_URL=api_url,
|
API_URL=api_url,
|
||||||
|
CLIENT_ID=client_id,
|
||||||
AUTO_CONNECT=auto_connect,
|
AUTO_CONNECT=auto_connect,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
)
|
)
|
||||||
@@ -112,6 +133,20 @@ class DataStackClient:
|
|||||||
self.server_url = self.config.server_url
|
self.server_url = self.config.server_url
|
||||||
self.api_url = self.config.api_url
|
self.api_url = self.config.api_url
|
||||||
self.token = self.config.client_token
|
self.token = self.config.client_token
|
||||||
|
|
||||||
|
configured_client_id = (self.config.client_id or "").strip()
|
||||||
|
token_client_id = _decode_jwt_client_id(self.token)
|
||||||
|
if (
|
||||||
|
configured_client_id
|
||||||
|
and token_client_id
|
||||||
|
and configured_client_id != token_client_id
|
||||||
|
):
|
||||||
|
raise DataStackConfigError(
|
||||||
|
f"Configured CLIENT_ID '{configured_client_id}' does not match "
|
||||||
|
f"CLIENT_TOKEN subject '{token_client_id}'."
|
||||||
|
)
|
||||||
|
self.client_id: str = configured_client_id or token_client_id
|
||||||
|
|
||||||
self.is_connected = False
|
self.is_connected = False
|
||||||
self._lock = threading.RLock()
|
self._lock = threading.RLock()
|
||||||
self._is_draining_spool = False
|
self._is_draining_spool = False
|
||||||
@@ -311,8 +346,11 @@ class DataStackClient:
|
|||||||
raise DataStackAuthError(msg) from exc
|
raise DataStackAuthError(msg) from exc
|
||||||
return False
|
return False
|
||||||
|
|
||||||
# Verify handshake & PostgreSQL client status via `ping_verify`
|
# Verify handshake & PostgreSQL client status via `ping_verify` while still serialized under RLock
|
||||||
return self._verify_websocket_handshake(timeout=effective_timeout)
|
verified = self._verify_websocket_handshake(timeout=effective_timeout)
|
||||||
|
if not verified:
|
||||||
|
self.is_connected = False
|
||||||
|
return verified
|
||||||
|
|
||||||
def _verify_websocket_handshake(
|
def _verify_websocket_handshake(
|
||||||
self, timeout: Optional[float] = None
|
self, timeout: Optional[float] = None
|
||||||
@@ -424,6 +462,8 @@ class DataStackClient:
|
|||||||
"Content-Type": "application/json",
|
"Content-Type": "application/json",
|
||||||
"Accept": "application/json",
|
"Accept": "application/json",
|
||||||
}
|
}
|
||||||
|
if self.client_id:
|
||||||
|
headers["X-DataStack-Client-Id"] = self.client_id
|
||||||
|
|
||||||
if requests is not None:
|
if requests is not None:
|
||||||
resp = requests.request(
|
resp = requests.request(
|
||||||
@@ -735,10 +775,14 @@ class DataStackClient:
|
|||||||
):
|
):
|
||||||
break
|
break
|
||||||
|
|
||||||
self.spooler.ack_batch(len(batch))
|
unprocessed_list = (
|
||||||
if isinstance(unprocessed, list) and len(unprocessed) > 0:
|
unprocessed if isinstance(unprocessed, list) else []
|
||||||
self.spooler.enqueue(unprocessed)
|
)
|
||||||
total_flushed += max(0, len(batch) - len(unprocessed))
|
self.spooler.ack_and_requeue_front(
|
||||||
|
len(batch), unprocessed_list
|
||||||
|
)
|
||||||
|
if unprocessed_list:
|
||||||
|
total_flushed += max(0, len(batch) - len(unprocessed_list))
|
||||||
break
|
break
|
||||||
|
|
||||||
total_flushed += len(batch)
|
total_flushed += len(batch)
|
||||||
@@ -767,11 +811,13 @@ class DataStackClient:
|
|||||||
"""
|
"""
|
||||||
resolved_id = (event_id or "").strip() or _make_event_id()
|
resolved_id = (event_id or "").strip() or _make_event_id()
|
||||||
resolved_ts = (timestamp or "").strip() or _utc_now_iso()
|
resolved_ts = (timestamp or "").strip() or _utc_now_iso()
|
||||||
envelope = {
|
envelope: Dict[str, Any] = {
|
||||||
"eventId": resolved_id,
|
"eventId": resolved_id,
|
||||||
"timestamp": resolved_ts,
|
"timestamp": resolved_ts,
|
||||||
"data": data,
|
"data": data,
|
||||||
}
|
}
|
||||||
|
if self.client_id:
|
||||||
|
envelope["clientId"] = self.client_id
|
||||||
return self._execute_with_retry_and_fallback(
|
return self._execute_with_retry_and_fallback(
|
||||||
ws_event="sync_data",
|
ws_event="sync_data",
|
||||||
ws_payload=envelope,
|
ws_payload=envelope,
|
||||||
@@ -810,6 +856,7 @@ class DataStackClient:
|
|||||||
if not isinstance(source_list, list) or len(source_list) == 0:
|
if not isinstance(source_list, list) or len(source_list) == 0:
|
||||||
return {"success": True, "processed": 0, "rejected": 0}
|
return {"success": True, "processed": 0, "rejected": 0}
|
||||||
|
|
||||||
|
envelope_keys = {"eventId", "timestamp", "data", "clientId"}
|
||||||
formatted: List[Dict[str, Any]] = []
|
formatted: List[Dict[str, Any]] = []
|
||||||
for item in source_list:
|
for item in source_list:
|
||||||
safe_item = item if isinstance(item, dict) else {"value": item}
|
safe_item = item if isinstance(item, dict) else {"value": item}
|
||||||
@@ -819,14 +866,21 @@ class DataStackClient:
|
|||||||
evt_ts = (
|
evt_ts = (
|
||||||
str(safe_item.get("timestamp") or "").strip() or _utc_now_iso()
|
str(safe_item.get("timestamp") or "").strip() or _utc_now_iso()
|
||||||
)
|
)
|
||||||
evt_data = safe_item.get("data", safe_item)
|
is_envelope = "data" in safe_item and set(safe_item.keys()).issubset(
|
||||||
formatted.append(
|
envelope_keys
|
||||||
{
|
|
||||||
"eventId": evt_id,
|
|
||||||
"timestamp": evt_ts,
|
|
||||||
"data": evt_data,
|
|
||||||
}
|
|
||||||
)
|
)
|
||||||
|
evt_data = safe_item["data"] if is_envelope else safe_item
|
||||||
|
evt_client_id = (
|
||||||
|
str(safe_item.get("clientId") or "").strip() or self.client_id
|
||||||
|
)
|
||||||
|
entry: Dict[str, Any] = {
|
||||||
|
"eventId": evt_id,
|
||||||
|
"timestamp": evt_ts,
|
||||||
|
"data": evt_data,
|
||||||
|
}
|
||||||
|
if evt_client_id:
|
||||||
|
entry["clientId"] = evt_client_id
|
||||||
|
formatted.append(entry)
|
||||||
|
|
||||||
return self._send_formatted_batch_recursive(
|
return self._send_formatted_batch_recursive(
|
||||||
formatted,
|
formatted,
|
||||||
@@ -1057,13 +1111,14 @@ class DataStackClient:
|
|||||||
)
|
)
|
||||||
evt_ts = existing_ts or _utc_now_iso()
|
evt_ts = existing_ts or _utc_now_iso()
|
||||||
|
|
||||||
buffer.append(
|
file_entry: Dict[str, Any] = {
|
||||||
{
|
"eventId": evt_id,
|
||||||
"eventId": evt_id,
|
"timestamp": evt_ts,
|
||||||
"timestamp": evt_ts,
|
"data": enriched,
|
||||||
"data": enriched,
|
}
|
||||||
}
|
if self.client_id:
|
||||||
)
|
file_entry["clientId"] = self.client_id
|
||||||
|
buffer.append(file_entry)
|
||||||
if len(buffer) >= effective_chunk:
|
if len(buffer) >= effective_chunk:
|
||||||
res = self.send_batch(buffer, file_name=path.name)
|
res = self.send_batch(buffer, file_name=path.name)
|
||||||
total_processed += int(res.get("processed", 0))
|
total_processed += int(res.get("processed", 0))
|
||||||
@@ -1162,6 +1217,7 @@ class DataStackClient:
|
|||||||
hb = self.heartbeat.status_snapshot()
|
hb = self.heartbeat.status_snapshot()
|
||||||
return {
|
return {
|
||||||
"connectedWebSocket": self.is_connected,
|
"connectedWebSocket": self.is_connected,
|
||||||
|
"clientId": self.client_id or None,
|
||||||
"serverUrl": self.server_url,
|
"serverUrl": self.server_url,
|
||||||
"apiUrl": self.api_url,
|
"apiUrl": self.api_url,
|
||||||
"configSource": self.config.config_source_path,
|
"configSource": self.config.config_source_path,
|
||||||
@@ -1176,3 +1232,21 @@ class DataStackClient:
|
|||||||
|
|
||||||
# Convenient alias so developers can write `from datastack import DataStack`
|
# Convenient alias so developers can write `from datastack import DataStack`
|
||||||
DataStack = DataStackClient
|
DataStack = DataStackClient
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"DataStack",
|
||||||
|
"DataStackClient",
|
||||||
|
"DataStackConfig",
|
||||||
|
"load_config",
|
||||||
|
"ConnectionState",
|
||||||
|
"HeartbeatMonitor",
|
||||||
|
"DiskSpooler",
|
||||||
|
"DataStackError",
|
||||||
|
"DataStackConfigError",
|
||||||
|
"DataStackAuthError",
|
||||||
|
"DataStackConnectionError",
|
||||||
|
"DataStackRateLimitError",
|
||||||
|
"DataStackPayloadTooLargeError",
|
||||||
|
"DataStackFileError",
|
||||||
|
"DataStackOfflineSpoolError",
|
||||||
|
]
|
||||||
@@ -171,6 +171,11 @@ class HeartbeatMonitor:
|
|||||||
with self._lock:
|
with self._lock:
|
||||||
if self._closed or self._state == ConnectionState.STOPPED:
|
if self._closed or self._state == ConnectionState.STOPPED:
|
||||||
break
|
break
|
||||||
|
if (
|
||||||
|
self._state == ConnectionState.CONNECTED_IDLE
|
||||||
|
or not self._wake_event.is_set()
|
||||||
|
):
|
||||||
|
continue
|
||||||
self._recovery_attempts += 1
|
self._recovery_attempts += 1
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import json
|
|||||||
import os
|
import os
|
||||||
import threading
|
import threading
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Dict, List
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
from .exceptions import DataStackOfflineSpoolError
|
from .exceptions import DataStackOfflineSpoolError
|
||||||
|
|
||||||
@@ -139,6 +139,49 @@ class DiskSpooler:
|
|||||||
except Exception:
|
except Exception:
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
def ack_and_requeue_front(
|
||||||
|
self,
|
||||||
|
acked_count: int,
|
||||||
|
unprocessed: Optional[List[Dict[str, Any]]] = None,
|
||||||
|
) -> int:
|
||||||
|
"""
|
||||||
|
Atomically removes `acked_count` head lines from `spool.jsonl` while
|
||||||
|
preserving any `unprocessed` events at the FRONT of the queue in a single
|
||||||
|
atomic `os.replace()` write (FIFO safe, zero crash-window data loss).
|
||||||
|
"""
|
||||||
|
if not self.enabled or acked_count <= 0:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
with self._lock:
|
||||||
|
if not self.spool_file.exists():
|
||||||
|
return 0
|
||||||
|
|
||||||
|
try:
|
||||||
|
with self.spool_file.open("r", encoding="utf-8") as fh:
|
||||||
|
all_lines = [ln for ln in fh if ln.strip()]
|
||||||
|
|
||||||
|
consumed = min(acked_count, len(all_lines))
|
||||||
|
tail_lines = all_lines[consumed:]
|
||||||
|
head_lines = [
|
||||||
|
json.dumps(evt, ensure_ascii=False) + "\n"
|
||||||
|
for evt in (unprocessed or [])
|
||||||
|
]
|
||||||
|
combined = head_lines + tail_lines
|
||||||
|
|
||||||
|
if not combined:
|
||||||
|
self.spool_file.unlink(missing_ok=True)
|
||||||
|
return consumed
|
||||||
|
|
||||||
|
self._ensure_dir()
|
||||||
|
tmp_file = self.spool_dir / "spool.jsonl.tmp"
|
||||||
|
with tmp_file.open("w", encoding="utf-8") as fh:
|
||||||
|
fh.writelines(combined)
|
||||||
|
fh.flush()
|
||||||
|
os.replace(str(tmp_file), str(self.spool_file))
|
||||||
|
return max(0, consumed - len(head_lines))
|
||||||
|
except Exception:
|
||||||
|
return 0
|
||||||
|
|
||||||
def pending_count(self) -> int:
|
def pending_count(self) -> int:
|
||||||
"""Returns the number of queued events waiting in the disk spool."""
|
"""Returns the number of queued events waiting in the disk spool."""
|
||||||
if not self.enabled:
|
if not self.enabled:
|
||||||
|
|||||||
+714
-98
@@ -1,8 +1,10 @@
|
|||||||
|
const crypto = require('node:crypto');
|
||||||
const express = require('express');
|
const express = require('express');
|
||||||
const http = require('http');
|
const http = require('http');
|
||||||
const cors = require('cors');
|
const cors = require('cors');
|
||||||
const { Server } = require('socket.io');
|
const { Server } = require('socket.io');
|
||||||
const { Pool } = require('pg');
|
const { Pool } = require('pg');
|
||||||
|
const Redis = require('ioredis');
|
||||||
const { setupSocketHandlers } = require('./socket/handlers');
|
const { setupSocketHandlers } = require('./socket/handlers');
|
||||||
const { CsvWriterWorker } = require('./workers/csvWriter');
|
const { CsvWriterWorker } = require('./workers/csvWriter');
|
||||||
|
|
||||||
@@ -11,21 +13,185 @@ const app = express();
|
|||||||
app.use(cors({ origin: '*' }));
|
app.use(cors({ origin: '*' }));
|
||||||
app.use(express.json({ limit: '10mb' }));
|
app.use(express.json({ limit: '10mb' }));
|
||||||
|
|
||||||
|
const REDIS_BUFFER_KEY = 'datastack:buffer:events';
|
||||||
|
const REDIS_RECENT_KEY = 'datastack:recent:events';
|
||||||
|
const REDIS_FLUSH_LOCK_KEY = 'datastack:flush:lock';
|
||||||
|
const REDIS_RECENT_LIMIT = 500;
|
||||||
|
const FLUSH_LOCK_TTL_MS = 15000;
|
||||||
|
const UNLOCK_LUA_SCRIPT =
|
||||||
|
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end';
|
||||||
|
|
||||||
const bufferQueue = [];
|
const bufferQueue = [];
|
||||||
const writer = new CsvWriterWorker();
|
const writer = new CsvWriterWorker();
|
||||||
|
let redisClient = undefined;
|
||||||
|
let redisUrlConfigured = '';
|
||||||
|
let redisConnected = false;
|
||||||
|
let redisTestOverride = undefined;
|
||||||
|
const pendingRedisWrites = new Set();
|
||||||
|
|
||||||
|
function getRedisClient() {
|
||||||
|
if (redisTestOverride !== undefined) {
|
||||||
|
return redisTestOverride;
|
||||||
|
}
|
||||||
|
|
||||||
|
const configuredUrl = (process.env.REDIS_URL || '').trim();
|
||||||
|
if (!configuredUrl) {
|
||||||
|
if (redisClient && typeof redisClient.disconnect === 'function') {
|
||||||
|
try {
|
||||||
|
redisClient.disconnect();
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
redisClient = null;
|
||||||
|
redisUrlConfigured = '';
|
||||||
|
redisConnected = false;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (redisClient && redisUrlConfigured === configuredUrl) {
|
||||||
|
return redisClient;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (redisClient && typeof redisClient.disconnect === 'function') {
|
||||||
|
try {
|
||||||
|
redisClient.disconnect();
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const client = new Redis(configuredUrl, {
|
||||||
|
maxRetriesPerRequest: 2,
|
||||||
|
enableReadyCheck: true,
|
||||||
|
retryStrategy: (times) => Math.min(times * 250, 3000),
|
||||||
|
});
|
||||||
|
client.on('ready', () => {
|
||||||
|
redisConnected = true;
|
||||||
|
});
|
||||||
|
client.on('connect', () => {
|
||||||
|
redisConnected = true;
|
||||||
|
});
|
||||||
|
client.on('close', () => {
|
||||||
|
redisConnected = false;
|
||||||
|
});
|
||||||
|
client.on('end', () => {
|
||||||
|
redisConnected = false;
|
||||||
|
});
|
||||||
|
client.on('error', () => {
|
||||||
|
redisConnected = false;
|
||||||
|
});
|
||||||
|
redisClient = client;
|
||||||
|
redisUrlConfigured = configuredUrl;
|
||||||
|
redisConnected = client.status === 'ready' || client.status === 'connect';
|
||||||
|
return redisClient;
|
||||||
|
} catch {
|
||||||
|
redisClient = null;
|
||||||
|
redisUrlConfigured = configuredUrl;
|
||||||
|
redisConnected = false;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function setRedisClientForTests(client) {
|
||||||
|
redisTestOverride = client;
|
||||||
|
redisConnected = Boolean(client);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function pushEventsToRedisOrFallback(events) {
|
||||||
|
if (!events || events.length === 0) return bufferQueue.length;
|
||||||
|
const redis = getRedisClient();
|
||||||
|
if (redis) {
|
||||||
|
const serialized = events.map((e) => JSON.stringify(e));
|
||||||
|
try {
|
||||||
|
const len = await redis.rpush(REDIS_BUFFER_KEY, ...serialized);
|
||||||
|
await redis.lpush(REDIS_RECENT_KEY, ...serialized);
|
||||||
|
await redis.ltrim(REDIS_RECENT_KEY, 0, REDIS_RECENT_LIMIT - 1);
|
||||||
|
redisConnected = true;
|
||||||
|
return Number(len) + bufferQueue.length;
|
||||||
|
} catch {
|
||||||
|
redisConnected = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bufferQueue.push(...events);
|
||||||
|
return bufferQueue.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
const IS_PRODUCTION = process.env.NODE_ENV === 'production';
|
||||||
|
const DEV_DEFAULT_JWT_SECRET = 'datastack-jwt-default-secret-key-32-chars-long';
|
||||||
|
|
||||||
|
function resolveInitialJwtSecret() {
|
||||||
|
const configured = (process.env.DATASTACK_JWT_SECRET || '').trim();
|
||||||
|
if (IS_PRODUCTION) {
|
||||||
|
return configured.length >= 32 ? configured : '';
|
||||||
|
}
|
||||||
|
if (configured.length > 0) {
|
||||||
|
return configured.length >= 32 ? configured : '';
|
||||||
|
}
|
||||||
|
return DEV_DEFAULT_JWT_SECRET;
|
||||||
|
}
|
||||||
|
|
||||||
const runtimeConfig = {
|
const runtimeConfig = {
|
||||||
jwtSecret:
|
jwtSecret: resolveInitialJwtSecret(),
|
||||||
process.env.DATASTACK_JWT_SECRET ||
|
|
||||||
'datastack-jwt-default-secret-key-32-chars-long',
|
|
||||||
maxEventsPerSec: 100,
|
maxEventsPerSec: 100,
|
||||||
maxPayloadSizeBytes: 100 * 1024,
|
maxPayloadSizeBytes: 100 * 1024,
|
||||||
batchFlushThreshold: 500,
|
batchFlushThreshold: 500,
|
||||||
batchFlushIntervalMs: 2000,
|
batchFlushIntervalMs: 2000,
|
||||||
fileRotationMb: 100,
|
fileRotationMb: 100,
|
||||||
enableIdempotency: true,
|
enableIdempotency: true,
|
||||||
|
autoPruneDays: 30,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
function resolveInternalSecret(jwtSecretOverride) {
|
||||||
|
const explicit = (process.env.DATASTACK_INTERNAL_SECRET || '').trim();
|
||||||
|
if (IS_PRODUCTION) {
|
||||||
|
if (explicit.length >= 32) return explicit;
|
||||||
|
if (explicit.length > 0) return '';
|
||||||
|
} else if (explicit.length > 0) {
|
||||||
|
return explicit;
|
||||||
|
}
|
||||||
|
const activeJwt = String(
|
||||||
|
jwtSecretOverride !== undefined ? jwtSecretOverride : runtimeConfig.jwtSecret || '',
|
||||||
|
).trim();
|
||||||
|
const minJwtLen = IS_PRODUCTION ? 32 : 1;
|
||||||
|
if (activeJwt.length < minJwtLen) {
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
return crypto
|
||||||
|
.createHash('sha256')
|
||||||
|
.update(`datastack-internal:${activeJwt}`)
|
||||||
|
.digest('hex');
|
||||||
|
}
|
||||||
|
|
||||||
|
function timingSafeSecretEqual(candidate, expected) {
|
||||||
|
if (!candidate || !expected) return false;
|
||||||
|
const a = crypto.createHash('sha256').update(String(candidate)).digest();
|
||||||
|
const b = crypto.createHash('sha256').update(String(expected)).digest();
|
||||||
|
return crypto.timingSafeEqual(a, b);
|
||||||
|
}
|
||||||
|
|
||||||
|
function requireInternalAuth(req, res, next) {
|
||||||
|
const expected = resolveInternalSecret();
|
||||||
|
if (!expected) {
|
||||||
|
return res.status(503).json({
|
||||||
|
success: false,
|
||||||
|
error: 'INTERNAL_AUTH_UNAVAILABLE: Internal bridge secret is not configured',
|
||||||
|
code: 'INTERNAL_AUTH_UNAVAILABLE',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const presented = req.get('x-datastack-internal-secret');
|
||||||
|
if (!presented || !timingSafeSecretEqual(presented, expected)) {
|
||||||
|
return res.status(401).json({
|
||||||
|
success: false,
|
||||||
|
error: 'UNAUTHORIZED: Missing or invalid internal authentication header',
|
||||||
|
code: 'UNAUTHORIZED',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return next();
|
||||||
|
}
|
||||||
|
|
||||||
let dbPool = null;
|
let dbPool = null;
|
||||||
if (process.env.DATABASE_URL) {
|
if (process.env.DATABASE_URL) {
|
||||||
try {
|
try {
|
||||||
@@ -40,43 +206,95 @@ if (process.env.DATABASE_URL) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Short-lived cache (2s TTL) for client status & token checks on high-frequency events
|
// Short-lived cache (2s TTL) for DB client status & token checks on high-frequency events
|
||||||
const clientAuthCache = new Map();
|
const clientAuthCache = new Map();
|
||||||
|
const deletedClientsSet = new Set();
|
||||||
|
const inMemoryClientRegistry = new Map();
|
||||||
|
const inMemoryClientActivity = new Map();
|
||||||
|
let hasSyncedInMemoryClients = false;
|
||||||
const CLIENT_CACHE_TTL_MS = 2000;
|
const CLIENT_CACHE_TTL_MS = 2000;
|
||||||
|
|
||||||
async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
|
async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
|
||||||
if (!dbPool || !clientId) {
|
if (!clientId) {
|
||||||
return { allowed: true };
|
return { allowed: true };
|
||||||
}
|
}
|
||||||
|
|
||||||
const now = Date.now();
|
const normalizedId = String(clientId).trim().toLowerCase();
|
||||||
if (!bypassCache) {
|
if (deletedClientsSet.has(normalizedId)) {
|
||||||
const cached = clientAuthCache.get(clientId);
|
return {
|
||||||
if (cached && now - cached.checkedAt < CLIENT_CACHE_TTL_MS) {
|
allowed: false,
|
||||||
if (cached.status === 'revoked') {
|
reason: 'CLIENT_REVOKED: Client access has been deleted or revoked by admin',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!dbPool) {
|
||||||
|
const memClient = inMemoryClientRegistry.get(normalizedId);
|
||||||
|
if (memClient) {
|
||||||
|
if (memClient.status === 'revoked') {
|
||||||
return {
|
return {
|
||||||
allowed: false,
|
allowed: false,
|
||||||
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
|
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
if (cached.token && presentedToken && cached.token !== presentedToken) {
|
if (memClient.token && presentedToken && memClient.token !== presentedToken) {
|
||||||
return {
|
return {
|
||||||
allowed: false,
|
allowed: false,
|
||||||
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
|
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
return { allowed: true, rateLimit: cached.rateLimit };
|
return { allowed: true, rateLimit: memClient.rateLimit || 100 };
|
||||||
}
|
}
|
||||||
|
if (hasSyncedInMemoryClients) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
reason: 'CLIENT_REVOKED: Client registration not found or has been deleted',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return { allowed: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
const now = Date.now();
|
||||||
|
const prevCached = clientAuthCache.get(normalizedId);
|
||||||
|
if (!bypassCache && prevCached && now - prevCached.checkedAt < CLIENT_CACHE_TTL_MS) {
|
||||||
|
if (prevCached.status === 'revoked') {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (prevCached.token && presentedToken && prevCached.token !== presentedToken) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return { allowed: true, rateLimit: prevCached.rateLimit };
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
const deletedRes = await dbPool.query(
|
||||||
|
`SELECT 1 FROM deleted_clients WHERE client_id = $1`,
|
||||||
|
[normalizedId],
|
||||||
|
);
|
||||||
|
if (deletedRes.rowCount && deletedRes.rowCount > 0) {
|
||||||
|
clientAuthCache.delete(normalizedId);
|
||||||
|
deletedClientsSet.add(normalizedId);
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
reason: 'CLIENT_REVOKED: Client access has been deleted by admin',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
const res = await dbPool.query(
|
const res = await dbPool.query(
|
||||||
`SELECT status, token, rate_limit FROM client_sdks WHERE client_id = $1`,
|
`SELECT status, token, rate_limit FROM client_sdks WHERE client_id = $1`,
|
||||||
[clientId],
|
[normalizedId],
|
||||||
);
|
);
|
||||||
if (!res.rowCount || res.rowCount === 0) {
|
if (!res.rowCount || res.rowCount === 0) {
|
||||||
// Allow token-authenticated client if not explicitly revoked in table
|
clientAuthCache.delete(normalizedId);
|
||||||
return { allowed: true };
|
return {
|
||||||
|
allowed: false,
|
||||||
|
reason: 'CLIENT_REVOKED: Client registration not found or has been deleted',
|
||||||
|
};
|
||||||
}
|
}
|
||||||
const row = res.rows[0];
|
const row = res.rows[0];
|
||||||
const entry = {
|
const entry = {
|
||||||
@@ -85,7 +303,7 @@ async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
|
|||||||
rateLimit: Number(row.rate_limit) || 100,
|
rateLimit: Number(row.rate_limit) || 100,
|
||||||
checkedAt: now,
|
checkedAt: now,
|
||||||
};
|
};
|
||||||
clientAuthCache.set(clientId, entry);
|
clientAuthCache.set(normalizedId, entry);
|
||||||
|
|
||||||
if (entry.status === 'revoked') {
|
if (entry.status === 'revoked') {
|
||||||
return {
|
return {
|
||||||
@@ -101,13 +319,30 @@ async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
|
|||||||
}
|
}
|
||||||
return { allowed: true, rateLimit: entry.rateLimit };
|
return { allowed: true, rateLimit: entry.rateLimit };
|
||||||
} catch {
|
} catch {
|
||||||
return { allowed: true };
|
return {
|
||||||
|
allowed: false,
|
||||||
|
reason: 'AUTH_UNAVAILABLE: Unable to verify client credentials against database',
|
||||||
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function recordClientActivity(clientId, count = 1) {
|
function recordClientActivity(clientId, count = 1) {
|
||||||
if (!dbPool || !clientId || count <= 0) return;
|
if (!clientId || count <= 0) return;
|
||||||
|
const normalizedId = String(clientId).trim().toLowerCase();
|
||||||
const nowIso = new Date().toISOString();
|
const nowIso = new Date().toISOString();
|
||||||
|
|
||||||
|
if (!dbPool) {
|
||||||
|
const prev = inMemoryClientActivity.get(normalizedId) || {
|
||||||
|
totalEvents: 0,
|
||||||
|
lastActive: nowIso,
|
||||||
|
};
|
||||||
|
inMemoryClientActivity.set(normalizedId, {
|
||||||
|
totalEvents: prev.totalEvents + count,
|
||||||
|
lastActive: nowIso,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
dbPool
|
dbPool
|
||||||
.query(
|
.query(
|
||||||
`UPDATE client_sdks
|
`UPDATE client_sdks
|
||||||
@@ -115,7 +350,7 @@ function recordClientActivity(clientId, count = 1) {
|
|||||||
last_active = $3,
|
last_active = $3,
|
||||||
updated_at = NOW()
|
updated_at = NOW()
|
||||||
WHERE client_id = $1`,
|
WHERE client_id = $1`,
|
||||||
[clientId, count, nowIso],
|
[normalizedId, count, nowIso],
|
||||||
)
|
)
|
||||||
.catch(() => undefined);
|
.catch(() => undefined);
|
||||||
}
|
}
|
||||||
@@ -124,23 +359,103 @@ let isFlushing = false;
|
|||||||
let flushTimer = null;
|
let flushTimer = null;
|
||||||
|
|
||||||
async function flushBuffer() {
|
async function flushBuffer() {
|
||||||
if (bufferQueue.length === 0 || isFlushing) {
|
if (pendingRedisWrites.size > 0) {
|
||||||
|
await Promise.allSettled([...pendingRedisWrites]);
|
||||||
|
}
|
||||||
|
if (isFlushing) {
|
||||||
return { written: 0, files: [] };
|
return { written: 0, files: [] };
|
||||||
}
|
}
|
||||||
isFlushing = true;
|
isFlushing = true;
|
||||||
const batch = bufferQueue.splice(0, bufferQueue.length);
|
let totalWritten = 0;
|
||||||
|
const filesSet = new Set();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const result = await writer.writeBatch(batch);
|
const redis = getRedisClient();
|
||||||
if (result.written > 0) {
|
if (redis) {
|
||||||
|
const lockToken = crypto.randomUUID();
|
||||||
|
let lockAcquired = false;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const lockResult = await redis.set(
|
||||||
|
REDIS_FLUSH_LOCK_KEY,
|
||||||
|
lockToken,
|
||||||
|
'PX',
|
||||||
|
FLUSH_LOCK_TTL_MS,
|
||||||
|
'NX',
|
||||||
|
);
|
||||||
|
lockAcquired = lockResult === 'OK';
|
||||||
|
redisConnected = true;
|
||||||
|
} catch {
|
||||||
|
redisConnected = false;
|
||||||
|
lockAcquired = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (lockAcquired) {
|
||||||
|
try {
|
||||||
|
const chunkSize = Math.max(500, runtimeConfig.batchFlushThreshold);
|
||||||
|
while (true) {
|
||||||
|
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, chunkSize - 1);
|
||||||
|
if (!rawItems || rawItems.length === 0) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
const parsedBatch = [];
|
||||||
|
for (const raw of rawItems) {
|
||||||
|
try {
|
||||||
|
const evt = JSON.parse(raw);
|
||||||
|
if (evt && evt.eventId) {
|
||||||
|
parsedBatch.push(evt);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Skip malformed entry while still trimming consumed range
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (parsedBatch.length > 0) {
|
||||||
|
const res = await writer.writeBatch(parsedBatch);
|
||||||
|
totalWritten += res.written;
|
||||||
|
for (const f of res.files) filesSet.add(f);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Trim ONLY after CSV batch write succeeds
|
||||||
|
await redis.ltrim(REDIS_BUFFER_KEY, rawItems.length, -1);
|
||||||
|
|
||||||
|
if (rawItems.length < chunkSize) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
|
||||||
|
} catch {
|
||||||
|
// Lock expires automatically via PX TTL
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (bufferQueue.length > 0) {
|
||||||
|
const batch = bufferQueue.splice(0, bufferQueue.length);
|
||||||
|
try {
|
||||||
|
const res = await writer.writeBatch(batch);
|
||||||
|
totalWritten += res.written;
|
||||||
|
for (const f of res.files) filesSet.add(f);
|
||||||
|
} catch (err) {
|
||||||
|
bufferQueue.unshift(...batch);
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const files = Array.from(filesSet);
|
||||||
|
if (totalWritten > 0) {
|
||||||
console.log(
|
console.log(
|
||||||
`[CSV Worker] Flushed ${result.written} events to ${result.files.length} file(s)`,
|
`[CSV Worker] Flushed ${totalWritten} events to ${files.length} file(s)`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
return result;
|
return { written: totalWritten, files };
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('[CSV Worker] Flush error:', err);
|
console.error('[CSV Worker] Flush error:', err);
|
||||||
bufferQueue.unshift(...batch);
|
return { written: totalWritten, files: Array.from(filesSet) };
|
||||||
return { written: 0, files: [] };
|
|
||||||
} finally {
|
} finally {
|
||||||
isFlushing = false;
|
isFlushing = false;
|
||||||
}
|
}
|
||||||
@@ -159,6 +474,92 @@ function scheduleFlushTimer(intervalMs) {
|
|||||||
|
|
||||||
scheduleFlushTimer(runtimeConfig.batchFlushIntervalMs);
|
scheduleFlushTimer(runtimeConfig.batchFlushIntervalMs);
|
||||||
|
|
||||||
|
const server = http.createServer(app);
|
||||||
|
const io = new Server(server, {
|
||||||
|
cors: { origin: '*', methods: ['GET', 'POST'] },
|
||||||
|
});
|
||||||
|
|
||||||
|
function applyRuntimeSettingsObject(settingsObj) {
|
||||||
|
if (!settingsObj || typeof settingsObj !== 'object') {
|
||||||
|
return { jwtRotated: false };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (settingsObj.maxEventsPerSec !== undefined) {
|
||||||
|
const val = Number(settingsObj.maxEventsPerSec);
|
||||||
|
if (Number.isFinite(val) && val >= 1) {
|
||||||
|
runtimeConfig.maxEventsPerSec = Math.floor(val);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (settingsObj.maxPayloadSizeKb !== undefined) {
|
||||||
|
const kb = Number(settingsObj.maxPayloadSizeKb);
|
||||||
|
if (Number.isFinite(kb) && kb >= 1) {
|
||||||
|
runtimeConfig.maxPayloadSizeBytes = Math.floor(kb) * 1024;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (settingsObj.batchFlushThreshold !== undefined) {
|
||||||
|
const threshold = Number(settingsObj.batchFlushThreshold);
|
||||||
|
if (Number.isFinite(threshold) && threshold >= 1) {
|
||||||
|
runtimeConfig.batchFlushThreshold = Math.floor(threshold);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (settingsObj.enableIdempotency !== undefined) {
|
||||||
|
runtimeConfig.enableIdempotency = Boolean(settingsObj.enableIdempotency);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (settingsObj.fileRotationMb !== undefined) {
|
||||||
|
const nextRotationMb = Number(settingsObj.fileRotationMb);
|
||||||
|
if (
|
||||||
|
Number.isFinite(nextRotationMb) &&
|
||||||
|
nextRotationMb >= 1 &&
|
||||||
|
nextRotationMb !== runtimeConfig.fileRotationMb
|
||||||
|
) {
|
||||||
|
runtimeConfig.fileRotationMb = nextRotationMb;
|
||||||
|
writer.setMaxFileSizeMb(nextRotationMb);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (settingsObj.autoPruneDays !== undefined && settingsObj.autoPruneDays !== null) {
|
||||||
|
const parsedPrune = Number(settingsObj.autoPruneDays);
|
||||||
|
if (Number.isFinite(parsedPrune) && parsedPrune >= 0) {
|
||||||
|
const nextPruneDays = Math.floor(parsedPrune);
|
||||||
|
const changed = nextPruneDays !== runtimeConfig.autoPruneDays;
|
||||||
|
runtimeConfig.autoPruneDays = nextPruneDays;
|
||||||
|
if (changed && nextPruneDays > 0) {
|
||||||
|
writer.pruneExpiredPartitions(nextPruneDays);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (settingsObj.batchFlushIntervalMs !== undefined) {
|
||||||
|
const nextIntervalMs = Math.max(
|
||||||
|
100,
|
||||||
|
Number(settingsObj.batchFlushIntervalMs) || 2000,
|
||||||
|
);
|
||||||
|
if (nextIntervalMs !== runtimeConfig.batchFlushIntervalMs) {
|
||||||
|
scheduleFlushTimer(nextIntervalMs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let jwtRotated = false;
|
||||||
|
if (settingsObj.jwtSecret !== undefined && settingsObj.jwtSecret !== null) {
|
||||||
|
const candidateSecret = String(settingsObj.jwtSecret).trim();
|
||||||
|
const validSecret =
|
||||||
|
candidateSecret.length >= 32 ||
|
||||||
|
(!IS_PRODUCTION && candidateSecret.length > 0);
|
||||||
|
if (validSecret && candidateSecret !== runtimeConfig.jwtSecret) {
|
||||||
|
runtimeConfig.jwtSecret = candidateSecret;
|
||||||
|
clientAuthCache.clear();
|
||||||
|
io.disconnectSockets(true);
|
||||||
|
jwtRotated = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { jwtRotated };
|
||||||
|
}
|
||||||
|
|
||||||
async function ensureDatabaseSchema() {
|
async function ensureDatabaseSchema() {
|
||||||
if (!dbPool) return;
|
if (!dbPool) return;
|
||||||
try {
|
try {
|
||||||
@@ -192,6 +593,12 @@ async function ensureDatabaseSchema() {
|
|||||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||||
)
|
)
|
||||||
`);
|
`);
|
||||||
|
await dbPool.query(`
|
||||||
|
CREATE TABLE IF NOT EXISTS deleted_clients (
|
||||||
|
client_id TEXT PRIMARY KEY,
|
||||||
|
deleted_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||||
|
)
|
||||||
|
`);
|
||||||
} catch {
|
} catch {
|
||||||
// Ignore transient startup DB error; retried on next sync cycle
|
// Ignore transient startup DB error; retried on next sync cycle
|
||||||
}
|
}
|
||||||
@@ -210,69 +617,132 @@ async function syncSettingsFromDatabase() {
|
|||||||
);
|
);
|
||||||
if (!res.rowCount || res.rowCount === 0) return;
|
if (!res.rowCount || res.rowCount === 0) return;
|
||||||
const row = res.rows[0];
|
const row = res.rows[0];
|
||||||
|
const parsedPrune =
|
||||||
|
row.auto_prune_days !== undefined && row.auto_prune_days !== null
|
||||||
|
? Number(row.auto_prune_days)
|
||||||
|
: NaN;
|
||||||
|
|
||||||
runtimeConfig.maxEventsPerSec = Number(row.max_events_per_sec) || 100;
|
applyRuntimeSettingsObject({
|
||||||
runtimeConfig.maxPayloadSizeBytes =
|
maxEventsPerSec: Number(row.max_events_per_sec) || 100,
|
||||||
(Number(row.max_payload_size_kb) || 100) * 1024;
|
maxPayloadSizeKb: Number(row.max_payload_size_kb) || 100,
|
||||||
runtimeConfig.batchFlushThreshold =
|
batchFlushThreshold: Math.max(1, Number(row.batch_flush_threshold) || 500),
|
||||||
Math.max(1, Number(row.batch_flush_threshold) || 500);
|
batchFlushIntervalMs: Math.max(
|
||||||
runtimeConfig.enableIdempotency = Boolean(row.enable_idempotency);
|
100,
|
||||||
|
Number(row.batch_flush_interval_ms) || 2000,
|
||||||
if (row.jwt_secret && String(row.jwt_secret).trim().length > 0) {
|
),
|
||||||
runtimeConfig.jwtSecret = String(row.jwt_secret).trim();
|
fileRotationMb: Number(row.file_rotation_mb) || 100,
|
||||||
}
|
enableIdempotency: Boolean(row.enable_idempotency),
|
||||||
|
autoPruneDays: Number.isFinite(parsedPrune) ? parsedPrune : 30,
|
||||||
const nextRotationMb = Number(row.file_rotation_mb) || 100;
|
jwtSecret: row.jwt_secret ? String(row.jwt_secret).trim() : undefined,
|
||||||
if (nextRotationMb !== runtimeConfig.fileRotationMb) {
|
});
|
||||||
runtimeConfig.fileRotationMb = nextRotationMb;
|
|
||||||
writer.setMaxFileSizeMb(nextRotationMb);
|
|
||||||
}
|
|
||||||
|
|
||||||
const nextPruneDays = Number(row.auto_prune_days) || 30;
|
|
||||||
if (nextPruneDays > 0) {
|
|
||||||
writer.pruneExpiredPartitions(nextPruneDays);
|
|
||||||
}
|
|
||||||
|
|
||||||
const nextIntervalMs = Math.max(
|
|
||||||
100,
|
|
||||||
Number(row.batch_flush_interval_ms) || 2000,
|
|
||||||
);
|
|
||||||
if (nextIntervalMs !== runtimeConfig.batchFlushIntervalMs) {
|
|
||||||
scheduleFlushTimer(nextIntervalMs);
|
|
||||||
}
|
|
||||||
} catch {
|
} catch {
|
||||||
// Keep existing runtimeConfig if table is not ready yet
|
// Keep existing runtimeConfig if table is not ready yet
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const PRUNE_INTERVAL_MS = 60 * 60 * 1000;
|
||||||
let settingsSyncTimer = null;
|
let settingsSyncTimer = null;
|
||||||
if (dbPool) {
|
let pruneTimer = null;
|
||||||
void syncSettingsFromDatabase();
|
if (process.env.DATASTACK_DISABLE_LISTEN !== '1') {
|
||||||
settingsSyncTimer = setInterval(() => {
|
if (dbPool) {
|
||||||
void syncSettingsFromDatabase();
|
void syncSettingsFromDatabase();
|
||||||
}, 3000);
|
settingsSyncTimer = setInterval(() => {
|
||||||
|
void syncSettingsFromDatabase();
|
||||||
|
}, 3000);
|
||||||
|
}
|
||||||
|
pruneTimer = setInterval(() => {
|
||||||
|
if (runtimeConfig.autoPruneDays > 0) {
|
||||||
|
writer.pruneExpiredPartitions(runtimeConfig.autoPruneDays);
|
||||||
|
}
|
||||||
|
}, PRUNE_INTERVAL_MS);
|
||||||
}
|
}
|
||||||
|
|
||||||
const server = http.createServer(app);
|
const socketHandlersApi = setupSocketHandlers(io, bufferQueue, writer, {
|
||||||
const io = new Server(server, {
|
getRuntimeConfig: () => runtimeConfig,
|
||||||
cors: { origin: '*', methods: ['GET', 'POST'] },
|
verifyClientInDb,
|
||||||
|
recordClientActivity,
|
||||||
|
onBufferPush: () => {
|
||||||
|
const redis = getRedisClient();
|
||||||
|
if (redis && bufferQueue.length > 0) {
|
||||||
|
const pushed = bufferQueue.splice(0, bufferQueue.length);
|
||||||
|
const op = pushEventsToRedisOrFallback(pushed).then((totalQueued) => {
|
||||||
|
if (totalQueued >= runtimeConfig.batchFlushThreshold) {
|
||||||
|
void flushBuffer();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
pendingRedisWrites.add(op);
|
||||||
|
void op.finally(() => {
|
||||||
|
pendingRedisWrites.delete(op);
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (bufferQueue.length >= runtimeConfig.batchFlushThreshold) {
|
||||||
|
void flushBuffer();
|
||||||
|
}
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
// Health check endpoint with activeSockets count and runtime capabilities
|
async function getRedisBufferLen() {
|
||||||
app.get('/health', (_req, res) => {
|
const redis = getRedisClient();
|
||||||
|
if (!redis) return 0;
|
||||||
|
try {
|
||||||
|
const len = Number(await redis.llen(REDIS_BUFFER_KEY)) || 0;
|
||||||
|
redisConnected = true;
|
||||||
|
return len;
|
||||||
|
} catch {
|
||||||
|
redisConnected = false;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Public health check endpoint
|
||||||
|
app.get('/health', async (_req, res) => {
|
||||||
|
const redisBufferSize = await getRedisBufferLen();
|
||||||
res.json({
|
res.json({
|
||||||
status: 'online',
|
status: 'online',
|
||||||
service: 'DataStack Express Ingestion',
|
service: 'DataStack Express Ingestion',
|
||||||
port: PORT,
|
port: PORT,
|
||||||
activeSockets: io.engine ? io.engine.clientsCount : 0,
|
activeSockets: io.engine ? io.engine.clientsCount : 0,
|
||||||
bufferSize: bufferQueue.length,
|
bufferSize: bufferQueue.length + redisBufferSize,
|
||||||
|
localBufferSize: bufferQueue.length,
|
||||||
|
redisBufferSize,
|
||||||
|
redisConnected,
|
||||||
maxEventsPerSec: runtimeConfig.maxEventsPerSec,
|
maxEventsPerSec: runtimeConfig.maxEventsPerSec,
|
||||||
maxPayloadSizeBytes: runtimeConfig.maxPayloadSizeBytes,
|
maxPayloadSizeBytes: runtimeConfig.maxPayloadSizeBytes,
|
||||||
uptime: process.uptime(),
|
uptime: process.uptime(),
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
// Internal flush endpoint triggered by Next.js POST /api/storage
|
// Require internal shared authentication on all /internal/* bridge endpoints
|
||||||
|
app.use('/internal', requireInternalAuth);
|
||||||
|
|
||||||
|
// Internal metrics endpoint for Next.js dashboard & in-memory client activity aggregation
|
||||||
|
app.get('/internal/metrics', async (_req, res) => {
|
||||||
|
const redisBufferSize = await getRedisBufferLen();
|
||||||
|
res.json({
|
||||||
|
activeSockets: io.engine ? io.engine.clientsCount : 0,
|
||||||
|
bufferSize: bufferQueue.length + redisBufferSize,
|
||||||
|
localBufferSize: bufferQueue.length,
|
||||||
|
redisBufferSize,
|
||||||
|
redisConnected,
|
||||||
|
registrySize: inMemoryClientRegistry.size,
|
||||||
|
hasSyncedInMemoryClients,
|
||||||
|
securityStats: socketHandlersApi.getSecurityStats(),
|
||||||
|
clientActivity: Object.fromEntries(inMemoryClientActivity.entries()),
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// Internal instant settings synchronization triggered by Next.js POST /api/settings
|
||||||
|
app.post('/internal/settings-sync', (req, res) => {
|
||||||
|
const result = applyRuntimeSettingsObject(req.body || {});
|
||||||
|
res.json({
|
||||||
|
success: true,
|
||||||
|
jwtRotated: result.jwtRotated,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// Internal flush endpoint triggered by Next.js POST /api/storage & GET /api/export
|
||||||
app.post('/internal/flush', async (_req, res) => {
|
app.post('/internal/flush', async (_req, res) => {
|
||||||
try {
|
try {
|
||||||
const result = await flushBuffer();
|
const result = await flushBuffer();
|
||||||
@@ -290,34 +760,149 @@ app.post('/internal/flush', async (_req, res) => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Internal endpoint to expose Port 4000 buffered events before CSV flush
|
// Internal endpoint to expose Port 4000 buffered events before CSV flush
|
||||||
app.get('/internal/recent', (req, res) => {
|
app.get('/internal/recent', async (req, res) => {
|
||||||
const clientId = req.query.clientId ? String(req.query.clientId) : '';
|
const clientId = req.query.clientId ? String(req.query.clientId) : '';
|
||||||
const limit = Math.min(200, Math.max(1, Number(req.query.limit) || 50));
|
const limit = Math.min(2000, Math.max(1, Number(req.query.limit) || 50));
|
||||||
const filtered =
|
|
||||||
|
const redis = getRedisClient();
|
||||||
|
let redisEvents = [];
|
||||||
|
if (redis) {
|
||||||
|
try {
|
||||||
|
const readCap = Math.min(REDIS_RECENT_LIMIT, Math.max(limit * 2, limit));
|
||||||
|
const rawItems = await redis.lrange(REDIS_RECENT_KEY, 0, readCap - 1);
|
||||||
|
redisConnected = true;
|
||||||
|
for (const raw of rawItems) {
|
||||||
|
try {
|
||||||
|
const evt = JSON.parse(raw);
|
||||||
|
if (!evt || !evt.eventId) continue;
|
||||||
|
if (clientId && clientId !== 'all' && evt.clientId !== clientId) continue;
|
||||||
|
redisEvents.push(evt);
|
||||||
|
if (redisEvents.length >= limit) break;
|
||||||
|
} catch {
|
||||||
|
// ignore malformed item
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
redisConnected = false;
|
||||||
|
redisEvents = [];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const localFiltered =
|
||||||
clientId && clientId !== 'all'
|
clientId && clientId !== 'all'
|
||||||
? bufferQueue.filter((evt) => evt.clientId === clientId)
|
? bufferQueue.filter((evt) => evt.clientId === clientId)
|
||||||
: bufferQueue;
|
: bufferQueue;
|
||||||
const recent = filtered.slice(-limit).reverse();
|
const localRecent = localFiltered.slice(-limit).reverse();
|
||||||
res.json({ events: recent, count: recent.length });
|
|
||||||
|
const merged = [];
|
||||||
|
const seen = new Set();
|
||||||
|
for (const evt of [...localRecent, ...redisEvents]) {
|
||||||
|
if (!evt || !evt.eventId) continue;
|
||||||
|
const key = `${evt.clientId}:${evt.eventId}`;
|
||||||
|
if (seen.has(key)) continue;
|
||||||
|
seen.add(key);
|
||||||
|
merged.push(evt);
|
||||||
|
if (merged.length >= limit) break;
|
||||||
|
}
|
||||||
|
|
||||||
|
res.json({ events: merged, count: merged.length });
|
||||||
});
|
});
|
||||||
|
|
||||||
setupSocketHandlers(io, bufferQueue, writer, {
|
// Internal endpoint triggered by Next.js SDKManager when a client is deleted, revoked, rotated, or synced
|
||||||
getRuntimeConfig: () => runtimeConfig,
|
app.post('/internal/client-sync', (req, res) => {
|
||||||
verifyClientInDb,
|
const action = req.body && req.body.action ? String(req.body.action) : '';
|
||||||
recordClientActivity,
|
|
||||||
onBufferPush: () => {
|
if (action === 'bulk_sync' && Array.isArray(req.body.clients)) {
|
||||||
if (bufferQueue.length >= runtimeConfig.batchFlushThreshold) {
|
hasSyncedInMemoryClients = true;
|
||||||
void flushBuffer();
|
inMemoryClientRegistry.clear();
|
||||||
|
clientAuthCache.clear();
|
||||||
|
const activeIds = new Set();
|
||||||
|
for (const item of req.body.clients) {
|
||||||
|
if (!item || !item.clientId) continue;
|
||||||
|
const cid = String(item.clientId).trim().toLowerCase();
|
||||||
|
activeIds.add(cid);
|
||||||
|
deletedClientsSet.delete(cid);
|
||||||
|
inMemoryClientRegistry.set(cid, {
|
||||||
|
clientId: cid,
|
||||||
|
name: String(item.name || cid),
|
||||||
|
status: item.status === 'revoked' ? 'revoked' : 'active',
|
||||||
|
token: String(item.token || ''),
|
||||||
|
rateLimit: Math.max(1, Number(item.rateLimit) || 100),
|
||||||
|
});
|
||||||
|
if (item.status === 'revoked') {
|
||||||
|
io.to(`client:${cid}`).disconnectSockets(true);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
return res.json({ success: true, count: activeIds.size });
|
||||||
|
}
|
||||||
|
|
||||||
|
const rawClientId = req.body && req.body.clientId ? String(req.body.clientId) : '';
|
||||||
|
const normalizedId = rawClientId.trim().toLowerCase();
|
||||||
|
const clientPayload =
|
||||||
|
req.body && req.body.client && typeof req.body.client === 'object'
|
||||||
|
? req.body.client
|
||||||
|
: null;
|
||||||
|
|
||||||
|
if (normalizedId) {
|
||||||
|
hasSyncedInMemoryClients = true;
|
||||||
|
clientAuthCache.delete(normalizedId);
|
||||||
|
|
||||||
|
if (action === 'deleted') {
|
||||||
|
deletedClientsSet.add(normalizedId);
|
||||||
|
inMemoryClientRegistry.delete(normalizedId);
|
||||||
|
inMemoryClientActivity.delete(normalizedId);
|
||||||
|
io.to(`client:${normalizedId}`).disconnectSockets(true);
|
||||||
|
} else if (action === 'revoked') {
|
||||||
|
const existingMem = inMemoryClientRegistry.get(normalizedId);
|
||||||
|
if (clientPayload) {
|
||||||
|
inMemoryClientRegistry.set(normalizedId, {
|
||||||
|
clientId: normalizedId,
|
||||||
|
name: String(clientPayload.name || normalizedId),
|
||||||
|
status: 'revoked',
|
||||||
|
token: String(clientPayload.token || ''),
|
||||||
|
rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
|
||||||
|
});
|
||||||
|
} else if (existingMem) {
|
||||||
|
existingMem.status = 'revoked';
|
||||||
|
} else {
|
||||||
|
inMemoryClientRegistry.set(normalizedId, {
|
||||||
|
clientId: normalizedId,
|
||||||
|
name: normalizedId,
|
||||||
|
status: 'revoked',
|
||||||
|
token: '',
|
||||||
|
rateLimit: 100,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
io.to(`client:${normalizedId}`).disconnectSockets(true);
|
||||||
|
} else if (action === 'registered') {
|
||||||
|
deletedClientsSet.delete(normalizedId);
|
||||||
|
if (clientPayload) {
|
||||||
|
inMemoryClientRegistry.set(normalizedId, {
|
||||||
|
clientId: normalizedId,
|
||||||
|
name: String(clientPayload.name || normalizedId),
|
||||||
|
status: clientPayload.status === 'revoked' ? 'revoked' : 'active',
|
||||||
|
token: String(clientPayload.token || ''),
|
||||||
|
rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} else if (action === 'updated') {
|
||||||
|
if (clientPayload) {
|
||||||
|
inMemoryClientRegistry.set(normalizedId, {
|
||||||
|
clientId: normalizedId,
|
||||||
|
name: String(clientPayload.name || normalizedId),
|
||||||
|
status: clientPayload.status === 'revoked' ? 'revoked' : 'active',
|
||||||
|
token: String(clientPayload.token || ''),
|
||||||
|
rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
io.to(`client:${normalizedId}`).disconnectSockets(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
res.json({ success: true });
|
||||||
});
|
});
|
||||||
|
|
||||||
let isShuttingDown = false;
|
function stopTimers() {
|
||||||
async function gracefulShutdown(signal) {
|
|
||||||
if (isShuttingDown) return;
|
|
||||||
isShuttingDown = true;
|
|
||||||
console.log(`[DataStack Ingestion Engine] Received ${signal}, flushing buffer before exit...`);
|
|
||||||
|
|
||||||
if (flushTimer) {
|
if (flushTimer) {
|
||||||
clearInterval(flushTimer);
|
clearInterval(flushTimer);
|
||||||
flushTimer = null;
|
flushTimer = null;
|
||||||
@@ -326,6 +911,19 @@ async function gracefulShutdown(signal) {
|
|||||||
clearInterval(settingsSyncTimer);
|
clearInterval(settingsSyncTimer);
|
||||||
settingsSyncTimer = null;
|
settingsSyncTimer = null;
|
||||||
}
|
}
|
||||||
|
if (pruneTimer) {
|
||||||
|
clearInterval(pruneTimer);
|
||||||
|
pruneTimer = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let isShuttingDown = false;
|
||||||
|
async function gracefulShutdown(signal) {
|
||||||
|
if (isShuttingDown) return;
|
||||||
|
isShuttingDown = true;
|
||||||
|
console.log(`[DataStack Ingestion Engine] Received ${signal}, flushing buffer before exit...`);
|
||||||
|
|
||||||
|
stopTimers();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await flushBuffer();
|
await flushBuffer();
|
||||||
@@ -336,6 +934,9 @@ async function gracefulShutdown(signal) {
|
|||||||
try {
|
try {
|
||||||
io.close();
|
io.close();
|
||||||
server.close();
|
server.close();
|
||||||
|
if (redisClient && typeof redisClient.quit === 'function') {
|
||||||
|
await redisClient.quit().catch(() => undefined);
|
||||||
|
}
|
||||||
if (dbPool) {
|
if (dbPool) {
|
||||||
await dbPool.end();
|
await dbPool.end();
|
||||||
}
|
}
|
||||||
@@ -346,17 +947,32 @@ async function gracefulShutdown(signal) {
|
|||||||
process.exit(0);
|
process.exit(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
process.on('SIGTERM', () => {
|
if (process.env.DATASTACK_DISABLE_LISTEN !== '1') {
|
||||||
void gracefulShutdown('SIGTERM');
|
process.on('SIGTERM', () => {
|
||||||
});
|
void gracefulShutdown('SIGTERM');
|
||||||
process.on('SIGINT', () => {
|
});
|
||||||
void gracefulShutdown('SIGINT');
|
process.on('SIGINT', () => {
|
||||||
});
|
void gracefulShutdown('SIGINT');
|
||||||
|
});
|
||||||
|
|
||||||
server.listen(PORT, () => {
|
server.listen(PORT, () => {
|
||||||
console.log(
|
console.log(
|
||||||
`[DataStack Ingestion Engine] Express + Socket.IO listening on port ${PORT}`,
|
`[DataStack Ingestion Engine] Express + Socket.IO listening on port ${PORT}`,
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
}
|
||||||
|
|
||||||
module.exports = { app, server, io, bufferQueue, flushBuffer };
|
module.exports = {
|
||||||
|
app,
|
||||||
|
server,
|
||||||
|
io,
|
||||||
|
writer,
|
||||||
|
bufferQueue,
|
||||||
|
flushBuffer,
|
||||||
|
runtimeConfig,
|
||||||
|
verifyClientInDb,
|
||||||
|
recordClientActivity,
|
||||||
|
applyRuntimeSettingsObject,
|
||||||
|
setRedisClientForTests,
|
||||||
|
stopTimers,
|
||||||
|
};
|
||||||
@@ -11,8 +11,10 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
: () => ({
|
: () => ({
|
||||||
jwtSecret:
|
jwtSecret:
|
||||||
options.jwtSecret ||
|
options.jwtSecret ||
|
||||||
process.env.DATASTACK_JWT_SECRET ||
|
(process.env.DATASTACK_JWT_SECRET || "").trim() ||
|
||||||
"datastack-jwt-default-secret-key-32-chars-long",
|
(process.env.NODE_ENV === "production"
|
||||||
|
? ""
|
||||||
|
: "datastack-jwt-default-secret-key-32-chars-long"),
|
||||||
maxEventsPerSec: 100,
|
maxEventsPerSec: 100,
|
||||||
maxPayloadSizeBytes: 100 * 1024,
|
maxPayloadSizeBytes: 100 * 1024,
|
||||||
batchFlushThreshold: 500,
|
batchFlushThreshold: 500,
|
||||||
@@ -38,8 +40,9 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
const rateLimitMap = new Map();
|
const rateLimitMap = new Map();
|
||||||
let lastCleanupAt = Date.now();
|
let lastCleanupAt = Date.now();
|
||||||
|
|
||||||
function cleanupMaps(now) {
|
function cleanupMaps(now, force = false) {
|
||||||
if (
|
if (
|
||||||
|
!force &&
|
||||||
now - lastCleanupAt < CLEANUP_INTERVAL_MS &&
|
now - lastCleanupAt < CLEANUP_INTERVAL_MS &&
|
||||||
idempotencyMap.size <= IDEMPOTENCY_MAX_ENTRIES
|
idempotencyMap.size <= IDEMPOTENCY_MAX_ENTRIES
|
||||||
) {
|
) {
|
||||||
@@ -60,6 +63,16 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getSecurityStats() {
|
||||||
|
cleanupMaps(Date.now(), true);
|
||||||
|
const config = getRuntimeConfig();
|
||||||
|
return {
|
||||||
|
activeIdempotencyKeys: idempotencyMap.size,
|
||||||
|
rateLimitedClientsCount: rateLimitMap.size,
|
||||||
|
maxPayloadSizeBytes: Number(config.maxPayloadSizeBytes) || 100 * 1024,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function validateSingleEvent(clientId, eventId, rawData, clientRateLimit) {
|
function validateSingleEvent(clientId, eventId, rawData, clientRateLimit) {
|
||||||
const config = getRuntimeConfig();
|
const config = getRuntimeConfig();
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
@@ -136,10 +149,19 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
? String(authHeader).slice(7).trim()
|
? String(authHeader).slice(7).trim()
|
||||||
: String(authHeader).trim();
|
: String(authHeader).trim();
|
||||||
const config = getRuntimeConfig();
|
const config = getRuntimeConfig();
|
||||||
|
const minSecretLen = process.env.NODE_ENV === "production" ? 32 : 1;
|
||||||
|
if (!config.jwtSecret || config.jwtSecret.length < minSecretLen) {
|
||||||
|
return next(
|
||||||
|
new Error("AUTH_UNAVAILABLE: DATASTACK_JWT_SECRET is not configured"),
|
||||||
|
);
|
||||||
|
}
|
||||||
const decoded = jwt.verify(cleanToken, config.jwtSecret);
|
const decoded = jwt.verify(cleanToken, config.jwtSecret);
|
||||||
|
|
||||||
|
const normalizedClientId = String(decoded.clientId || "")
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
const dbCheck = await verifyClientInDb(
|
const dbCheck = await verifyClientInDb(
|
||||||
decoded.clientId,
|
normalizedClientId,
|
||||||
cleanToken,
|
cleanToken,
|
||||||
true,
|
true,
|
||||||
);
|
);
|
||||||
@@ -147,7 +169,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
return next(new Error(dbCheck.reason || "CLIENT_REVOKED"));
|
return next(new Error(dbCheck.reason || "CLIENT_REVOKED"));
|
||||||
}
|
}
|
||||||
|
|
||||||
socket.data.clientId = decoded.clientId;
|
socket.data.clientId = normalizedClientId;
|
||||||
socket.data.token = cleanToken;
|
socket.data.token = cleanToken;
|
||||||
socket.data.rateLimit =
|
socket.data.rateLimit =
|
||||||
dbCheck.rateLimit || decoded.rateLimit || config.maxEventsPerSec || 100;
|
dbCheck.rateLimit || decoded.rateLimit || config.maxEventsPerSec || 100;
|
||||||
@@ -157,6 +179,22 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const ENVELOPE_KEYS = new Set(["eventId", "timestamp", "data", "clientId"]);
|
||||||
|
function extractBatchItemData(safeItem) {
|
||||||
|
if (
|
||||||
|
safeItem &&
|
||||||
|
typeof safeItem === "object" &&
|
||||||
|
!Array.isArray(safeItem) &&
|
||||||
|
safeItem.data !== undefined
|
||||||
|
) {
|
||||||
|
const keys = Object.keys(safeItem);
|
||||||
|
if (keys.every((k) => ENVELOPE_KEYS.has(k))) {
|
||||||
|
return safeItem.data;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return safeItem;
|
||||||
|
}
|
||||||
|
|
||||||
io.on("connection", (socket) => {
|
io.on("connection", (socket) => {
|
||||||
const clientId = socket.data.clientId;
|
const clientId = socket.data.clientId;
|
||||||
socket.join(`client:${clientId}`);
|
socket.join(`client:${clientId}`);
|
||||||
@@ -274,8 +312,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
safeItem.eventId ||
|
safeItem.eventId ||
|
||||||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||||
const timestamp = safeItem.timestamp || new Date().toISOString();
|
const timestamp = safeItem.timestamp || new Date().toISOString();
|
||||||
const rawData =
|
const rawData = extractBatchItemData(safeItem);
|
||||||
safeItem.data !== undefined ? safeItem.data : safeItem;
|
|
||||||
|
|
||||||
const check = validateSingleEvent(
|
const check = validateSingleEvent(
|
||||||
clientId,
|
clientId,
|
||||||
@@ -403,6 +440,10 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
getSecurityStats,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = { setupSocketHandlers };
|
module.exports = { setupSocketHandlers };
|
||||||
+123
-24
@@ -2,6 +2,15 @@ const fs = require('fs');
|
|||||||
const path = require('path');
|
const path = require('path');
|
||||||
const { Parser } = require('json2csv');
|
const { Parser } = require('json2csv');
|
||||||
|
|
||||||
|
const LOCK_RETRY_MIN_MS = 5;
|
||||||
|
const LOCK_RETRY_MAX_MS = 18;
|
||||||
|
const LOCK_TIMEOUT_MS = 10000;
|
||||||
|
const STALE_LOCK_AGE_MS = 30000;
|
||||||
|
|
||||||
|
function sleepMs(ms) {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Background CSV Writer Worker
|
* Background CSV Writer Worker
|
||||||
* Flushes batches of real-time events to partitioned CSV files.
|
* Flushes batches of real-time events to partitioned CSV files.
|
||||||
@@ -27,14 +36,100 @@ class CsvWriterWorker {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
getTargetFilePath(clientId, dateStr) {
|
ensureClientDir(clientId) {
|
||||||
const safeClientId = String(clientId || 'unknown').replace(/[^a-zA-Z0-9_-]/g, '_');
|
const safeClientId = String(clientId || 'unknown').replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||||
const clientDir = path.join(this.baseDir, safeClientId);
|
const clientDir = path.join(this.baseDir, safeClientId);
|
||||||
if (!fs.existsSync(clientDir)) {
|
if (!fs.existsSync(clientDir)) {
|
||||||
fs.mkdirSync(clientDir, { recursive: true });
|
fs.mkdirSync(clientDir, { recursive: true });
|
||||||
}
|
}
|
||||||
|
return clientDir;
|
||||||
|
}
|
||||||
|
|
||||||
const date = dateStr || new Date().toISOString().split('T')[0];
|
async withPartitionLock(clientId, dateStr, fn) {
|
||||||
|
const clientDir = this.ensureClientDir(clientId);
|
||||||
|
const date = this.extractSafePartitionDate(dateStr);
|
||||||
|
const lockPath = path.join(clientDir, `.${date}.csv.lock`);
|
||||||
|
const deadline = Date.now() + LOCK_TIMEOUT_MS;
|
||||||
|
|
||||||
|
let acquired = false;
|
||||||
|
while (Date.now() < deadline) {
|
||||||
|
try {
|
||||||
|
const fd = fs.openSync(lockPath, 'wx');
|
||||||
|
try {
|
||||||
|
fs.writeFileSync(fd, `${process.pid}:${Date.now()}`, 'utf8');
|
||||||
|
} finally {
|
||||||
|
fs.closeSync(fd);
|
||||||
|
}
|
||||||
|
acquired = true;
|
||||||
|
break;
|
||||||
|
} catch (err) {
|
||||||
|
if (!err || err.code !== 'EEXIST') {
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const lockStat = fs.statSync(lockPath);
|
||||||
|
if (Date.now() - lockStat.mtimeMs > STALE_LOCK_AGE_MS) {
|
||||||
|
fs.unlinkSync(lockPath);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Released concurrently
|
||||||
|
}
|
||||||
|
const jitter =
|
||||||
|
LOCK_RETRY_MIN_MS +
|
||||||
|
Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS));
|
||||||
|
await sleepMs(jitter);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!acquired) {
|
||||||
|
throw new Error(`Timed out waiting for CSV partition lock: ${lockPath}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await fn();
|
||||||
|
} finally {
|
||||||
|
try {
|
||||||
|
if (fs.existsSync(lockPath)) {
|
||||||
|
fs.unlinkSync(lockPath);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Ignore if already removed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
extractSafePartitionDate(rawTimestamp) {
|
||||||
|
const fallback = new Date().toISOString().slice(0, 10);
|
||||||
|
if (!rawTimestamp || typeof rawTimestamp !== 'string') {
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
const trimmed = rawTimestamp.trim();
|
||||||
|
const prefix = trimmed.slice(0, 10);
|
||||||
|
if (/^\d{4}-\d{2}-\d{2}$/.test(prefix)) {
|
||||||
|
const parsedPrefix = new Date(`${prefix}T00:00:00.000Z`);
|
||||||
|
if (!Number.isNaN(parsedPrefix.getTime())) {
|
||||||
|
return prefix;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (/^\d+(\.\d+)?$/.test(trimmed)) {
|
||||||
|
const num = Number(trimmed);
|
||||||
|
const ms = num < 1e11 ? num * 1000 : num;
|
||||||
|
const d = new Date(ms);
|
||||||
|
if (!Number.isNaN(d.getTime())) {
|
||||||
|
return d.toISOString().slice(0, 10);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const parsed = new Date(trimmed);
|
||||||
|
if (!Number.isNaN(parsed.getTime())) {
|
||||||
|
return parsed.toISOString().slice(0, 10);
|
||||||
|
}
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
getTargetFilePath(clientId, dateStr) {
|
||||||
|
const clientDir = this.ensureClientDir(clientId);
|
||||||
|
const date = this.extractSafePartitionDate(dateStr);
|
||||||
let baseFileName = `${date}.csv`;
|
let baseFileName = `${date}.csv`;
|
||||||
let targetPath = path.join(clientDir, baseFileName);
|
let targetPath = path.join(clientDir, baseFileName);
|
||||||
|
|
||||||
@@ -62,7 +157,7 @@ class CsvWriterWorker {
|
|||||||
|
|
||||||
const grouped = new Map();
|
const grouped = new Map();
|
||||||
for (const evt of events) {
|
for (const evt of events) {
|
||||||
const date = (evt.timestamp || new Date().toISOString()).split('T')[0];
|
const date = this.extractSafePartitionDate(evt.timestamp);
|
||||||
const key = `${evt.clientId}:::${date}`;
|
const key = `${evt.clientId}:::${date}`;
|
||||||
if (!grouped.has(key)) grouped.set(key, []);
|
if (!grouped.has(key)) grouped.set(key, []);
|
||||||
grouped.get(key).push(evt);
|
grouped.get(key).push(evt);
|
||||||
@@ -73,30 +168,34 @@ class CsvWriterWorker {
|
|||||||
|
|
||||||
for (const [key, batch] of grouped.entries()) {
|
for (const [key, batch] of grouped.entries()) {
|
||||||
const [clientId, date] = key.split(':::');
|
const [clientId, date] = key.split(':::');
|
||||||
const targetPath = this.getTargetFilePath(clientId, date);
|
const targetPath = await this.withPartitionLock(clientId, date, () => {
|
||||||
let hasContent = false;
|
const resolvedPath = this.getTargetFilePath(clientId, date);
|
||||||
try {
|
let hasContent = false;
|
||||||
hasContent =
|
try {
|
||||||
fs.existsSync(targetPath) && fs.statSync(targetPath).size > 0;
|
hasContent =
|
||||||
} catch {
|
fs.existsSync(resolvedPath) && fs.statSync(resolvedPath).size > 0;
|
||||||
hasContent = false;
|
} catch {
|
||||||
}
|
hasContent = false;
|
||||||
|
}
|
||||||
|
|
||||||
const cleanBatch = batch.map((item) => ({
|
const cleanBatch = batch.map((item) => ({
|
||||||
clientId: item.clientId,
|
clientId: item.clientId,
|
||||||
eventId: item.eventId,
|
eventId: item.eventId,
|
||||||
timestamp: item.timestamp,
|
timestamp: item.timestamp,
|
||||||
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
|
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
|
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
|
||||||
const csv = parser.parse(cleanBatch);
|
const csv = parser.parse(cleanBatch);
|
||||||
|
|
||||||
if (hasContent) {
|
if (hasContent) {
|
||||||
fs.appendFileSync(targetPath, '\n' + csv, 'utf8');
|
fs.appendFileSync(resolvedPath, '\n' + csv, 'utf8');
|
||||||
} else {
|
} else {
|
||||||
fs.writeFileSync(targetPath, csv, 'utf8');
|
fs.writeFileSync(resolvedPath, csv, 'utf8');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return resolvedPath;
|
||||||
|
});
|
||||||
|
|
||||||
files.push(targetPath);
|
files.push(targetPath);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,340 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { after, afterEach, beforeEach, describe, it } from "node:test";
|
||||||
|
import { NextRequest } from "next/server";
|
||||||
|
import { GET as profileGet, PUT as profilePut } from "@/app/api/auth/profile/route";
|
||||||
|
import { GET as clientsGet, POST as clientsPost } from "@/app/api/clients/route";
|
||||||
|
import {
|
||||||
|
DELETE as clientDelete,
|
||||||
|
GET as clientGet,
|
||||||
|
} from "@/app/api/clients/[id]/route";
|
||||||
|
import { POST as eventsPost } from "@/app/api/events/route";
|
||||||
|
import {
|
||||||
|
createAdminSessionToken,
|
||||||
|
getAdminSessionPayload,
|
||||||
|
getAdminSessionSecret,
|
||||||
|
} from "@/core/auth/adminSession";
|
||||||
|
import {
|
||||||
|
consumeAdminLoginAttempt,
|
||||||
|
createInitialAdmin,
|
||||||
|
getAdminSetupKey,
|
||||||
|
isAdminSetupComplete,
|
||||||
|
resetAdminStateForTests,
|
||||||
|
verifyAdminCredentials,
|
||||||
|
verifyAdminSetupKey,
|
||||||
|
verifyRequestAdminSession,
|
||||||
|
} from "@/core/auth/adminUsers";
|
||||||
|
import { AuthService, resolveDefaultJwtSecret } from "@/core/auth/jwt";
|
||||||
|
import { isSameOriginRequest } from "@/middleware";
|
||||||
|
|
||||||
|
const ORIGINAL_ENV = { ...process.env };
|
||||||
|
|
||||||
|
function restoreEnv() {
|
||||||
|
for (const key of Object.keys(process.env)) {
|
||||||
|
if (!(key in ORIGINAL_ENV)) {
|
||||||
|
delete process.env[key];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Object.assign(process.env, ORIGINAL_ENV);
|
||||||
|
AuthService.setSecret(resolveDefaultJwtSecret() || "");
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("Phase-01 Auth, Secrets, Revocation & Proxy Hardening", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
restoreEnv();
|
||||||
|
resetAdminStateForTests();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
restoreEnv();
|
||||||
|
resetAdminStateForTests();
|
||||||
|
});
|
||||||
|
|
||||||
|
after(() => {
|
||||||
|
const bufState = (
|
||||||
|
globalThis as {
|
||||||
|
__dataStackBufferState?: { flushTimer?: ReturnType<typeof setInterval> | null };
|
||||||
|
}
|
||||||
|
).__dataStackBufferState;
|
||||||
|
if (bufState?.flushTimer) {
|
||||||
|
clearInterval(bufState.flushTimer);
|
||||||
|
bufState.flushTimer = null;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("BUG-01 & BUG-02: fails closed in production when secrets or DB/setup are missing", async () => {
|
||||||
|
process.env.NODE_ENV = "production";
|
||||||
|
delete process.env.DATABASE_URL;
|
||||||
|
delete process.env.DATASTACK_ADMIN_SESSION_SECRET;
|
||||||
|
delete process.env.DATASTACK_ADMIN_SETUP_KEY;
|
||||||
|
delete process.env.DATASTACK_JWT_SECRET;
|
||||||
|
delete process.env.DATASTACK_ADMIN_PASSWORD;
|
||||||
|
|
||||||
|
assert.equal(getAdminSessionSecret(), null);
|
||||||
|
assert.equal(getAdminSetupKey(), null);
|
||||||
|
assert.equal(resolveDefaultJwtSecret(), "");
|
||||||
|
assert.equal(
|
||||||
|
verifyAdminSetupKey("datastack-default-setup-key-32-chars"),
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
assert.equal(await isAdminSetupComplete(), false);
|
||||||
|
|
||||||
|
await assert.rejects(
|
||||||
|
() => verifyAdminCredentials("admin@datastack.dev", "admin12345678"),
|
||||||
|
/setup/i,
|
||||||
|
);
|
||||||
|
|
||||||
|
AuthService.setSecret(resolveDefaultJwtSecret());
|
||||||
|
assert.equal(AuthService.isConfigured(), false);
|
||||||
|
assert.throws(() => AuthService.generateClientToken({ clientId: "c1" }));
|
||||||
|
});
|
||||||
|
|
||||||
|
it("BUG-03: bumps tokenVersion on credential update and rejects old session cookies", async () => {
|
||||||
|
delete process.env.DATABASE_URL;
|
||||||
|
process.env.DATASTACK_ALLOW_DEMO_ADMIN = "false";
|
||||||
|
const created = await createInitialAdmin("owner@datastack.dev", "InitialPass123!");
|
||||||
|
assert.equal(created, true);
|
||||||
|
assert.equal(await isAdminSetupComplete(), true);
|
||||||
|
|
||||||
|
const secret = getAdminSessionSecret()!;
|
||||||
|
const v1Token = await createAdminSessionToken(
|
||||||
|
"owner@datastack.dev",
|
||||||
|
secret,
|
||||||
|
Date.now() + 3600_000,
|
||||||
|
1,
|
||||||
|
);
|
||||||
|
|
||||||
|
const checkV1 = await verifyRequestAdminSession(v1Token);
|
||||||
|
assert.equal(checkV1.valid, true);
|
||||||
|
|
||||||
|
const updateReq = new NextRequest("http://localhost:3000/api/auth/profile", {
|
||||||
|
method: "PUT",
|
||||||
|
headers: {
|
||||||
|
"content-type": "application/json",
|
||||||
|
cookie: `datastack_admin_session=${v1Token}`,
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
email: "owner2@datastack.dev",
|
||||||
|
currentPassword: "InitialPass123!",
|
||||||
|
newPassword: "UpdatedPass456!",
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
const updateRes = await profilePut(updateReq);
|
||||||
|
assert.equal(updateRes.status, 200);
|
||||||
|
const setCookie = updateRes.cookies.get("datastack_admin_session")?.value;
|
||||||
|
assert.ok(setCookie);
|
||||||
|
|
||||||
|
const newPayload = await getAdminSessionPayload(setCookie!, secret);
|
||||||
|
assert.ok(newPayload);
|
||||||
|
assert.equal(newPayload!.tokenVersion, 2);
|
||||||
|
|
||||||
|
// Old v1 session must now return 401
|
||||||
|
const staleProfileReq = new NextRequest("http://localhost:3000/api/auth/profile", {
|
||||||
|
method: "GET",
|
||||||
|
headers: { cookie: `datastack_admin_session=${v1Token}` },
|
||||||
|
});
|
||||||
|
const staleRes = await profileGet(staleProfileReq);
|
||||||
|
assert.equal(staleRes.status, 401);
|
||||||
|
|
||||||
|
// New v2 session succeeds
|
||||||
|
const freshProfileReq = new NextRequest("http://localhost:3000/api/auth/profile", {
|
||||||
|
method: "GET",
|
||||||
|
headers: { cookie: `datastack_admin_session=${setCookie}` },
|
||||||
|
});
|
||||||
|
const freshRes = await profileGet(freshProfileReq);
|
||||||
|
assert.equal(freshRes.status, 200);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("BUG-04: throttles by composite IP+email without locking out other admin IPs", async () => {
|
||||||
|
const secret = "test-throttle-secret-at-least-32-chars-long";
|
||||||
|
const targetEmail = "admin@datastack.dev";
|
||||||
|
|
||||||
|
for (let i = 0; i < 5; i++) {
|
||||||
|
const dec = await consumeAdminLoginAttempt(targetEmail, secret, "198.51.100.10");
|
||||||
|
assert.equal(dec.allowed, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 6th attempt from attacker IP is blocked
|
||||||
|
const blocked = await consumeAdminLoginAttempt(targetEmail, secret, "198.51.100.10");
|
||||||
|
assert.equal(blocked.allowed, false);
|
||||||
|
assert.ok(blocked.retryAfterSeconds >= 1);
|
||||||
|
|
||||||
|
// Real admin from different IP is still allowed
|
||||||
|
const otherIpAllowed = await consumeAdminLoginAttempt(
|
||||||
|
targetEmail,
|
||||||
|
secret,
|
||||||
|
"203.0.113.55",
|
||||||
|
);
|
||||||
|
assert.equal(otherIpAllowed.allowed, true);
|
||||||
|
|
||||||
|
// Same IP spraying 25 distinct emails triggers IP bucket block on 26th
|
||||||
|
for (let i = 0; i < 25; i++) {
|
||||||
|
const spray = await consumeAdminLoginAttempt(
|
||||||
|
`spray${i}@example.com`,
|
||||||
|
secret,
|
||||||
|
"198.51.100.99",
|
||||||
|
);
|
||||||
|
assert.equal(spray.allowed, true);
|
||||||
|
}
|
||||||
|
const sprayBlocked = await consumeAdminLoginAttempt(
|
||||||
|
"spray25@example.com",
|
||||||
|
secret,
|
||||||
|
"198.51.100.99",
|
||||||
|
);
|
||||||
|
assert.equal(sprayBlocked.allowed, false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("BUG-05 & BUG-15: redacts tokens in client list, returns 400 on malformed JSON, and blocks deleted/unregistered Bearer tokens", async () => {
|
||||||
|
delete process.env.DATABASE_URL;
|
||||||
|
await createInitialAdmin("admin@datastack.dev", "AdminPass123!");
|
||||||
|
const secret = getAdminSessionSecret()!;
|
||||||
|
const adminCookie = await createAdminSessionToken(
|
||||||
|
"admin@datastack.dev",
|
||||||
|
secret,
|
||||||
|
Date.now() + 3600_000,
|
||||||
|
1,
|
||||||
|
);
|
||||||
|
|
||||||
|
const createReq = new NextRequest("http://localhost:3000/api/clients", {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"content-type": "application/json",
|
||||||
|
cookie: `datastack_admin_session=${adminCookie}`,
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
clientId: "phase01_probe_client",
|
||||||
|
name: "Phase01 Probe",
|
||||||
|
rateLimit: 50,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
const createRes = await clientsPost(createReq);
|
||||||
|
assert.equal(createRes.status, 201);
|
||||||
|
const createdJson = await createRes.json();
|
||||||
|
const clientToken = createdJson.client.token as string;
|
||||||
|
assert.ok(clientToken.length > 20);
|
||||||
|
|
||||||
|
// List endpoint must redact full token & secret
|
||||||
|
const listReq = new NextRequest("http://localhost:3000/api/clients", {
|
||||||
|
method: "GET",
|
||||||
|
headers: { cookie: `datastack_admin_session=${adminCookie}` },
|
||||||
|
});
|
||||||
|
const listRes = await clientsGet(listReq);
|
||||||
|
assert.equal(listRes.status, 200);
|
||||||
|
const listJson = await listRes.json();
|
||||||
|
const listed = listJson.clients.find(
|
||||||
|
(c: { clientId: string }) => c.clientId === "phase01_probe_client",
|
||||||
|
);
|
||||||
|
assert.ok(listed);
|
||||||
|
assert.equal("token" in listed, false);
|
||||||
|
assert.equal("secret" in listed, false);
|
||||||
|
assert.equal(listed.tokenPreview, `${clientToken.slice(0, 12)}...`);
|
||||||
|
|
||||||
|
// Single client GET omits token by default, reveals on ?includeToken=true
|
||||||
|
const getDefaultRes = await clientGet(
|
||||||
|
new NextRequest("http://localhost:3000/api/clients/phase01_probe_client", {
|
||||||
|
headers: { cookie: `datastack_admin_session=${adminCookie}` },
|
||||||
|
}),
|
||||||
|
{ params: Promise.resolve({ id: "phase01_probe_client" }) },
|
||||||
|
);
|
||||||
|
const defaultClient = (await getDefaultRes.json()).client;
|
||||||
|
assert.equal("token" in defaultClient, false);
|
||||||
|
|
||||||
|
const getIncludeRes = await clientGet(
|
||||||
|
new NextRequest(
|
||||||
|
"http://localhost:3000/api/clients/phase01_probe_client?includeToken=true",
|
||||||
|
{ headers: { cookie: `datastack_admin_session=${adminCookie}` } },
|
||||||
|
),
|
||||||
|
{ params: Promise.resolve({ id: "phase01_probe_client" }) },
|
||||||
|
);
|
||||||
|
const includeClient = (await getIncludeRes.json()).client;
|
||||||
|
assert.equal(includeClient.token, clientToken);
|
||||||
|
|
||||||
|
// Malformed JSON on POST /api/events returns 400 (not 500)
|
||||||
|
const malformedRes = await eventsPost(
|
||||||
|
new NextRequest("http://localhost:3000/api/events", {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
authorization: `Bearer ${clientToken}`,
|
||||||
|
"content-type": "application/json",
|
||||||
|
},
|
||||||
|
body: "{bad-json",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
assert.equal(malformedRes.status, 400);
|
||||||
|
|
||||||
|
// Delete client -> subsequent Bearer ingestion fails closed with 403 CLIENT_REVOKED
|
||||||
|
const delRes = await clientDelete(
|
||||||
|
new NextRequest("http://localhost:3000/api/clients/phase01_probe_client", {
|
||||||
|
method: "DELETE",
|
||||||
|
headers: { cookie: `datastack_admin_session=${adminCookie}` },
|
||||||
|
}),
|
||||||
|
{ params: Promise.resolve({ id: "phase01_probe_client" }) },
|
||||||
|
);
|
||||||
|
assert.equal(delRes.status, 200);
|
||||||
|
|
||||||
|
const rejectedRes = await eventsPost(
|
||||||
|
new NextRequest("http://localhost:3000/api/events", {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
authorization: `Bearer ${clientToken}`,
|
||||||
|
"content-type": "application/json",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({ eventId: "evt_after_delete", data: { ok: 1 } }),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
assert.equal(rejectedRes.status, 403);
|
||||||
|
|
||||||
|
// Unregistered signed Bearer token also rejected with 403 CLIENT_REVOKED
|
||||||
|
const ghostToken = AuthService.generateClientToken({
|
||||||
|
clientId: "unregistered_ghost_client",
|
||||||
|
});
|
||||||
|
const ghostRes = await eventsPost(
|
||||||
|
new NextRequest("http://localhost:3000/api/events", {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
authorization: `Bearer ${ghostToken}`,
|
||||||
|
"content-type": "application/json",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({ eventId: "evt_ghost", data: { ok: 1 } }),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
assert.equal(ghostRes.status, 403);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("BUG-26: validates Origin or Referer against Host / X-Forwarded-Host", () => {
|
||||||
|
const reqOriginOk = new NextRequest("http://localhost:3000/api/clients", {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
origin: "https://console.example.com",
|
||||||
|
"x-forwarded-host": "console.example.com",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
assert.equal(isSameOriginRequest(reqOriginOk), true);
|
||||||
|
|
||||||
|
const reqRefererOk = new NextRequest("http://localhost:3000/api/clients", {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
referer: "https://console.example.com/vcon/clients",
|
||||||
|
host: "console.example.com",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
assert.equal(isSameOriginRequest(reqRefererOk), true);
|
||||||
|
|
||||||
|
const reqCrossOrigin = new NextRequest("http://localhost:3000/api/clients", {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
origin: "https://evil.example.com",
|
||||||
|
host: "console.example.com",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
assert.equal(isSameOriginRequest(reqCrossOrigin), false);
|
||||||
|
|
||||||
|
const reqMissingBoth = new NextRequest("http://localhost:3000/api/clients", {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
host: "console.example.com",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
assert.equal(isSameOriginRequest(reqMissingBoth), false);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,375 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { createRequire } from "node:module";
|
||||||
|
import type { AddressInfo } from "node:net";
|
||||||
|
import test, { after, before } from "node:test";
|
||||||
|
import { NextRequest } from "next/server";
|
||||||
|
import { io as ioClient, type Socket } from "socket.io-client";
|
||||||
|
import { GET as getSettingsRoute, POST as postSettingsRoute } from "../app/api/settings/route";
|
||||||
|
import { SettingsManager } from "../core/adminSettings/settingsManager";
|
||||||
|
import {
|
||||||
|
createAdminSessionToken,
|
||||||
|
getAdminSessionSecret,
|
||||||
|
} from "../core/auth/adminSession";
|
||||||
|
import {
|
||||||
|
createInitialAdmin,
|
||||||
|
resetAdminStateForTests,
|
||||||
|
} from "../core/auth/adminUsers";
|
||||||
|
import { ApiEngine } from "../core/apiEngine/apiService";
|
||||||
|
import { SDKManager } from "../core/sdk/sdkManager";
|
||||||
|
import {
|
||||||
|
SecurityEngine,
|
||||||
|
getInternalIngestionHeaders,
|
||||||
|
} from "../core/Security/security";
|
||||||
|
|
||||||
|
const require = createRequire(import.meta.url);
|
||||||
|
|
||||||
|
interface IngestionServerModule {
|
||||||
|
server: import("node:http").Server;
|
||||||
|
io: import("socket.io").Server;
|
||||||
|
bufferQueue: Array<{ clientId: string; eventId: string; timestamp: string; payload: string }>;
|
||||||
|
runtimeConfig: {
|
||||||
|
jwtSecret: string;
|
||||||
|
maxEventsPerSec: number;
|
||||||
|
maxPayloadSizeBytes: number;
|
||||||
|
batchFlushThreshold: number;
|
||||||
|
batchFlushIntervalMs: number;
|
||||||
|
fileRotationMb: number;
|
||||||
|
enableIdempotency: boolean;
|
||||||
|
autoPruneDays: number;
|
||||||
|
};
|
||||||
|
stopTimers: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
const g = globalThis as typeof globalThis & {
|
||||||
|
__dataStackBufferState?: {
|
||||||
|
flushTimer: ReturnType<typeof setInterval> | null;
|
||||||
|
};
|
||||||
|
__dataStackAdminSettings?: unknown;
|
||||||
|
__dataStackClientsMap?: Map<string, unknown>;
|
||||||
|
__dataStackDeletedClients?: Set<string>;
|
||||||
|
__dataStackSyncedSocketEvents?: Map<string, number>;
|
||||||
|
};
|
||||||
|
|
||||||
|
let serverMod: IngestionServerModule;
|
||||||
|
let ingestionUrl = "";
|
||||||
|
const prevIngestionUrl = process.env.INGESTION_INTERNAL_URL;
|
||||||
|
const prevDisableListen = process.env.DATASTACK_DISABLE_LISTEN;
|
||||||
|
const prevDbUrl = process.env.DATABASE_URL;
|
||||||
|
const prevInternalSecret = process.env.DATASTACK_INTERNAL_SECRET;
|
||||||
|
const openSockets: Socket[] = [];
|
||||||
|
|
||||||
|
function connectSocket(token: string): Promise<Socket> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const client = ioClient(ingestionUrl, {
|
||||||
|
auth: { token },
|
||||||
|
transports: ["websocket"],
|
||||||
|
reconnection: false,
|
||||||
|
timeout: 3000,
|
||||||
|
});
|
||||||
|
openSockets.push(client);
|
||||||
|
const onConnect = () => {
|
||||||
|
cleanup();
|
||||||
|
resolve(client);
|
||||||
|
};
|
||||||
|
const onError = (err: Error) => {
|
||||||
|
cleanup();
|
||||||
|
client.disconnect();
|
||||||
|
reject(err);
|
||||||
|
};
|
||||||
|
function cleanup() {
|
||||||
|
client.off("connect", onConnect);
|
||||||
|
client.off("connect_error", onError);
|
||||||
|
}
|
||||||
|
client.on("connect", onConnect);
|
||||||
|
client.on("connect_error", onError);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function emitWithAck<T = unknown>(
|
||||||
|
socket: Socket,
|
||||||
|
event: string,
|
||||||
|
payload: unknown,
|
||||||
|
): Promise<T> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
socket.timeout(3000).emit(event, payload, (err: unknown, res: T) => {
|
||||||
|
if (err) reject(err);
|
||||||
|
else resolve(res);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function createAdminCookieHeader(): Promise<string> {
|
||||||
|
resetAdminStateForTests();
|
||||||
|
await createInitialAdmin("admin@datastack.dev", "InitialAdminPass123!");
|
||||||
|
const secret = getAdminSessionSecret();
|
||||||
|
assert.ok(secret, "Expected admin session secret in test environment");
|
||||||
|
const token = await createAdminSessionToken(
|
||||||
|
"admin@datastack.dev",
|
||||||
|
secret,
|
||||||
|
Date.now() + 3600_000,
|
||||||
|
1,
|
||||||
|
);
|
||||||
|
return `datastack_admin_session=${token}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
before(async () => {
|
||||||
|
process.env.DATASTACK_DISABLE_LISTEN = "1";
|
||||||
|
delete process.env.DATABASE_URL;
|
||||||
|
delete process.env.DATASTACK_INTERNAL_SECRET;
|
||||||
|
delete g.__dataStackAdminSettings;
|
||||||
|
g.__dataStackClientsMap?.clear();
|
||||||
|
g.__dataStackDeletedClients?.clear();
|
||||||
|
g.__dataStackSyncedSocketEvents?.clear();
|
||||||
|
resetAdminStateForTests();
|
||||||
|
|
||||||
|
serverMod = require("../server/index.js") as IngestionServerModule;
|
||||||
|
await new Promise<void>((resolve) => {
|
||||||
|
serverMod.server.listen(0, "127.0.0.1", () => resolve());
|
||||||
|
});
|
||||||
|
const addr = serverMod.server.address() as AddressInfo;
|
||||||
|
ingestionUrl = `http://127.0.0.1:${addr.port}`;
|
||||||
|
process.env.INGESTION_INTERNAL_URL = ingestionUrl;
|
||||||
|
});
|
||||||
|
|
||||||
|
after(async () => {
|
||||||
|
for (const sock of openSockets) {
|
||||||
|
try {
|
||||||
|
sock.disconnect();
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (serverMod) {
|
||||||
|
serverMod.stopTimers();
|
||||||
|
await new Promise<void>((resolve) => {
|
||||||
|
serverMod.io.close(() => {
|
||||||
|
serverMod.server.close(() => resolve());
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (g.__dataStackBufferState?.flushTimer) {
|
||||||
|
clearInterval(g.__dataStackBufferState.flushTimer);
|
||||||
|
g.__dataStackBufferState.flushTimer = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL;
|
||||||
|
else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl;
|
||||||
|
|
||||||
|
if (prevDisableListen === undefined) delete process.env.DATASTACK_DISABLE_LISTEN;
|
||||||
|
else process.env.DATASTACK_DISABLE_LISTEN = prevDisableListen;
|
||||||
|
|
||||||
|
if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
|
||||||
|
else process.env.DATABASE_URL = prevDbUrl;
|
||||||
|
|
||||||
|
if (prevInternalSecret === undefined) delete process.env.DATASTACK_INTERNAL_SECRET;
|
||||||
|
else process.env.DATASTACK_INTERNAL_SECRET = prevInternalSecret;
|
||||||
|
});
|
||||||
|
|
||||||
|
test("BUG-13: Port 4000 /internal/* rejects unauthenticated callers and accepts valid internal secret while /health remains public", async () => {
|
||||||
|
const healthRes = await fetch(`${ingestionUrl}/health`);
|
||||||
|
assert.equal(healthRes.status, 200);
|
||||||
|
|
||||||
|
for (const [method, path, body] of [
|
||||||
|
["GET", "/internal/metrics", undefined],
|
||||||
|
["POST", "/internal/flush", {}],
|
||||||
|
["GET", "/internal/recent", undefined],
|
||||||
|
["POST", "/internal/client-sync", { clientId: "x", action: "registered" }],
|
||||||
|
["POST", "/internal/settings-sync", { maxEventsPerSec: 120 }],
|
||||||
|
] as const) {
|
||||||
|
const noAuth = await fetch(`${ingestionUrl}${path}`, {
|
||||||
|
method,
|
||||||
|
headers: body ? { "Content-Type": "application/json" } : undefined,
|
||||||
|
body: body ? JSON.stringify(body) : undefined,
|
||||||
|
});
|
||||||
|
assert.equal(noAuth.status, 401, `${method} ${path} must reject missing secret`);
|
||||||
|
|
||||||
|
const wrongAuth = await fetch(`${ingestionUrl}${path}`, {
|
||||||
|
method,
|
||||||
|
headers: {
|
||||||
|
...(body ? { "Content-Type": "application/json" } : {}),
|
||||||
|
"x-datastack-internal-secret": "wrong-secret-value",
|
||||||
|
},
|
||||||
|
body: body ? JSON.stringify(body) : undefined,
|
||||||
|
});
|
||||||
|
assert.equal(wrongAuth.status, 401, `${method} ${path} must reject wrong secret`);
|
||||||
|
|
||||||
|
const validAuth = await fetch(`${ingestionUrl}${path}`, {
|
||||||
|
method,
|
||||||
|
headers: {
|
||||||
|
...(body ? { "Content-Type": "application/json" } : {}),
|
||||||
|
...getInternalIngestionHeaders(),
|
||||||
|
},
|
||||||
|
body: body ? JSON.stringify(body) : undefined,
|
||||||
|
});
|
||||||
|
assert.equal(validAuth.status, 200, `${method} ${path} must accept valid internal secret`);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("BUG-06: POST /api/settings validates allowlisted bounds, syncs Port 4000 immediately, preserves autoPruneDays=0, and disconnects sockets on JWT rotation", async () => {
|
||||||
|
const unauthGet = await getSettingsRoute(new NextRequest("http://localhost:3000/api/settings"));
|
||||||
|
assert.equal(unauthGet.status, 401);
|
||||||
|
|
||||||
|
const cookieHeader = await createAdminCookieHeader();
|
||||||
|
|
||||||
|
const badJsonReq = new NextRequest("http://localhost:3000/api/settings", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json", cookie: cookieHeader },
|
||||||
|
body: "{bad-json",
|
||||||
|
});
|
||||||
|
const badJsonRes = await postSettingsRoute(badJsonReq);
|
||||||
|
assert.equal(badJsonRes.status, 400);
|
||||||
|
|
||||||
|
const unsupportedReq = new NextRequest("http://localhost:3000/api/settings", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json", cookie: cookieHeader },
|
||||||
|
body: JSON.stringify({ serverPort: 9999 }),
|
||||||
|
});
|
||||||
|
assert.equal((await postSettingsRoute(unsupportedReq)).status, 400);
|
||||||
|
|
||||||
|
const weakSecretReq = new NextRequest("http://localhost:3000/api/settings", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json", cookie: cookieHeader },
|
||||||
|
body: JSON.stringify({ jwtSecret: "too-short" }),
|
||||||
|
});
|
||||||
|
assert.equal((await postSettingsRoute(weakSecretReq)).status, 400);
|
||||||
|
|
||||||
|
const validUpdateReq = new NextRequest("http://localhost:3000/api/settings", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json", cookie: cookieHeader },
|
||||||
|
body: JSON.stringify({
|
||||||
|
maxEventsPerSec: 42,
|
||||||
|
autoPruneDays: 0,
|
||||||
|
enableIdempotency: true,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
const validUpdateRes = await postSettingsRoute(validUpdateReq);
|
||||||
|
assert.equal(validUpdateRes.status, 200);
|
||||||
|
const updatedJson = await validUpdateRes.json();
|
||||||
|
assert.equal(updatedJson.settings.maxEventsPerSec, 42);
|
||||||
|
assert.equal(updatedJson.settings.autoPruneDays, 0);
|
||||||
|
assert.equal(serverMod.runtimeConfig.maxEventsPerSec, 42);
|
||||||
|
assert.equal(serverMod.runtimeConfig.autoPruneDays, 0);
|
||||||
|
|
||||||
|
// Register client & open WebSocket before JWT rotation
|
||||||
|
const client = await SDKManager.registerClientAsync({
|
||||||
|
clientId: "phase02_jwt_rot",
|
||||||
|
name: "Phase02 Rotation",
|
||||||
|
rateLimit: 50,
|
||||||
|
});
|
||||||
|
const oldToken = client.token;
|
||||||
|
const socket = await connectSocket(oldToken);
|
||||||
|
assert.equal(socket.connected, true);
|
||||||
|
|
||||||
|
const disconnectedPromise = new Promise<string>((resolve) => {
|
||||||
|
socket.once("disconnect", (reason) => resolve(reason));
|
||||||
|
});
|
||||||
|
|
||||||
|
const newSecret = "phase02-rotated-production-secret-key-minimum-32-chars";
|
||||||
|
const rotateReq = new NextRequest("http://localhost:3000/api/settings", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json", cookie: cookieHeader },
|
||||||
|
body: JSON.stringify({ jwtSecret: newSecret }),
|
||||||
|
});
|
||||||
|
const rotateRes = await postSettingsRoute(rotateReq);
|
||||||
|
assert.equal(rotateRes.status, 200);
|
||||||
|
assert.equal(serverMod.runtimeConfig.jwtSecret, newSecret);
|
||||||
|
|
||||||
|
const disconnectReason = await Promise.race([
|
||||||
|
disconnectedPromise,
|
||||||
|
new Promise<string>((_, reject) =>
|
||||||
|
setTimeout(() => reject(new Error("Expected socket disconnect on JWT rotation")), 2000),
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
assert.ok(disconnectReason.length > 0);
|
||||||
|
|
||||||
|
// Old JWT token must fail handshake after rotation
|
||||||
|
await assert.rejects(() => connectSocket(oldToken), /AUTH_INVALID|jwt|signature/i);
|
||||||
|
|
||||||
|
// New resigned client token must succeed cleanly
|
||||||
|
const refreshedClient = SDKManager.getClient("phase02_jwt_rot");
|
||||||
|
assert.ok(refreshedClient && refreshedClient.token !== oldToken);
|
||||||
|
const reconnected = await connectSocket(refreshedClient.token);
|
||||||
|
assert.equal(reconnected.connected, true);
|
||||||
|
reconnected.disconnect();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("BUG-07: In-memory client registry, socket event counting, token rotation & revocation enforce cleanly on Port 4000", async () => {
|
||||||
|
const client = await SDKManager.registerClientAsync({
|
||||||
|
clientId: "phase02_mem_client",
|
||||||
|
name: "Phase02 Memory Sync",
|
||||||
|
rateLimit: 25,
|
||||||
|
});
|
||||||
|
|
||||||
|
const socket = await connectSocket(client.token);
|
||||||
|
const ack = await emitWithAck<{ success: boolean; eventId: string }>(
|
||||||
|
socket,
|
||||||
|
"sync_data",
|
||||||
|
{ eventId: "evt-mem-1", data: { ok: true } },
|
||||||
|
);
|
||||||
|
assert.equal(ack.success, true);
|
||||||
|
|
||||||
|
const listFirst = await SDKManager.listClientsAsync();
|
||||||
|
const syncedFirst = listFirst.find((c) => c.clientId === "phase02_mem_client");
|
||||||
|
assert.ok(syncedFirst);
|
||||||
|
assert.equal(syncedFirst.totalEvents, 1);
|
||||||
|
assert.ok(syncedFirst.lastActive);
|
||||||
|
|
||||||
|
// Polling again must NOT double-count socket activity
|
||||||
|
const listSecond = await SDKManager.listClientsAsync();
|
||||||
|
const syncedSecond = listSecond.find((c) => c.clientId === "phase02_mem_client");
|
||||||
|
assert.equal(syncedSecond?.totalEvents, 1);
|
||||||
|
|
||||||
|
// Advance across JWT iat 1-second boundary before regenerating client token
|
||||||
|
await new Promise((r) => setTimeout(r, 1100));
|
||||||
|
|
||||||
|
// Regenerating client token must kick active socket and reject old token
|
||||||
|
const rotatedClient = await SDKManager.regenerateTokenAsync("phase02_mem_client");
|
||||||
|
assert.ok(rotatedClient && rotatedClient.token !== client.token);
|
||||||
|
await new Promise((r) => setTimeout(r, 50));
|
||||||
|
assert.equal(socket.connected, false);
|
||||||
|
await assert.rejects(() => connectSocket(client.token), /TOKEN_ROTATED/);
|
||||||
|
|
||||||
|
// New token works; revoking kicks and rejects
|
||||||
|
const socket2 = await connectSocket(rotatedClient.token);
|
||||||
|
assert.equal(socket2.connected, true);
|
||||||
|
await SDKManager.updateClientAsync("phase02_mem_client", { status: "revoked" });
|
||||||
|
await new Promise((r) => setTimeout(r, 50));
|
||||||
|
assert.equal(socket2.connected, false);
|
||||||
|
await assert.rejects(() => connectSocket(rotatedClient.token), /CLIENT_REVOKED/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("BUG-08: Rate-limit error reports effectiveRateLimit and ApiEngine aggregates Port 3000 + Port 4000 security metrics", async () => {
|
||||||
|
await SettingsManager.updateSettingsAsync({ maxEventsPerSec: 3, enableIdempotency: true });
|
||||||
|
|
||||||
|
const rlClientId = "phase02_rl_msg";
|
||||||
|
let lastError = "";
|
||||||
|
for (let i = 0; i < 5; i += 1) {
|
||||||
|
const res = SecurityEngine.validateEvent(rlClientId, `rl-evt-${i}`, { i }, 100);
|
||||||
|
if (!res.allowed && res.code === "RATE_LIMIT_EXCEEDED") {
|
||||||
|
lastError = res.reason || "";
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert.match(lastError, /Rate limit of 3 events\/sec exceeded/);
|
||||||
|
|
||||||
|
const client = await SDKManager.registerClientAsync({
|
||||||
|
clientId: "phase02_stats_client",
|
||||||
|
name: "Phase02 Stats Client",
|
||||||
|
rateLimit: 50,
|
||||||
|
});
|
||||||
|
await new Promise((r) => setTimeout(r, 80));
|
||||||
|
|
||||||
|
const socket = await connectSocket(client.token);
|
||||||
|
await emitWithAck(socket, "sync_data", { eventId: "evt-stats-ws-1", data: { a: 1 } });
|
||||||
|
socket.disconnect();
|
||||||
|
|
||||||
|
const metrics = await ApiEngine.getDashboardMetrics();
|
||||||
|
assert.ok(
|
||||||
|
metrics.securityStats.activeIdempotencyKeys >= 2,
|
||||||
|
`Expected aggregated idempotency keys >= 2 (HTTP + WS), got ${metrics.securityStats.activeIdempotencyKeys}`,
|
||||||
|
);
|
||||||
|
assert.ok(
|
||||||
|
metrics.securityStats.rateLimitedClientsCount >= 2,
|
||||||
|
`Expected aggregated rateLimitedClientsCount >= 2, got ${metrics.securityStats.rateLimitedClientsCount}`,
|
||||||
|
);
|
||||||
|
});
|
||||||
@@ -0,0 +1,511 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import fs from "node:fs";
|
||||||
|
import { createServer, type Server } from "node:http";
|
||||||
|
import type { AddressInfo } from "node:net";
|
||||||
|
import { createRequire } from "node:module";
|
||||||
|
import os from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import test, { after, before, beforeEach } from "node:test";
|
||||||
|
import { NextRequest } from "next/server";
|
||||||
|
import { GET as getDataRoute } from "../app/api/data/route";
|
||||||
|
import { GET as getEventsRoute } from "../app/api/events/route";
|
||||||
|
import { GET as getExportRoute } from "../app/api/export/route";
|
||||||
|
import { POST as postStorageRoute } from "../app/api/storage/route";
|
||||||
|
import { ApiEngine } from "../core/apiEngine/apiService";
|
||||||
|
import {
|
||||||
|
DataBufferEngine,
|
||||||
|
REDIS_BUFFER_KEY,
|
||||||
|
REDIS_FLUSH_LOCK_KEY,
|
||||||
|
type RedisBufferClient,
|
||||||
|
} from "../core/data/buffer";
|
||||||
|
import { SDKManager } from "../core/sdk/sdkManager";
|
||||||
|
import { CSVStorageManager, type DataEvent } from "../core/Storage/csvStorage";
|
||||||
|
|
||||||
|
const require = createRequire(import.meta.url);
|
||||||
|
const { CsvWriterWorker } = require("../server/workers/csvWriter") as {
|
||||||
|
CsvWriterWorker: new (baseDir?: string, maxFileSize?: number) => {
|
||||||
|
baseDir: string;
|
||||||
|
writeBatch: (events: DataEvent[]) => Promise<{ written: number; files: string[] }>;
|
||||||
|
pruneExpiredPartitions: (days: number) => number;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
interface IngestionServerModule {
|
||||||
|
server: Server;
|
||||||
|
io: import("socket.io").Server;
|
||||||
|
writer: { baseDir: string };
|
||||||
|
bufferQueue: DataEvent[];
|
||||||
|
runtimeConfig: {
|
||||||
|
batchFlushThreshold: number;
|
||||||
|
autoPruneDays: number;
|
||||||
|
};
|
||||||
|
applyRuntimeSettingsObject: (obj: Record<string, unknown>) => { jwtRotated: boolean };
|
||||||
|
flushBuffer: () => Promise<{ written: number; files: string[] }>;
|
||||||
|
setRedisClientForTests: (client: RedisBufferClient | null | undefined) => void;
|
||||||
|
stopTimers: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
class FakeRedisClient implements RedisBufferClient {
|
||||||
|
public status = "ready";
|
||||||
|
public lists = new Map<string, string[]>();
|
||||||
|
public strings = new Map<string, { value: string; expiresAt: number }>();
|
||||||
|
|
||||||
|
public async rpush(key: string, ...values: string[]): Promise<number> {
|
||||||
|
const list = this.lists.get(key) || [];
|
||||||
|
list.push(...values);
|
||||||
|
this.lists.set(key, list);
|
||||||
|
return list.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async lpush(key: string, ...values: string[]): Promise<number> {
|
||||||
|
const list = this.lists.get(key) || [];
|
||||||
|
for (const v of values) {
|
||||||
|
list.unshift(v);
|
||||||
|
}
|
||||||
|
this.lists.set(key, list);
|
||||||
|
return list.length;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async ltrim(key: string, start: number, stop: number): Promise<"OK"> {
|
||||||
|
const list = this.lists.get(key) || [];
|
||||||
|
const len = list.length;
|
||||||
|
const normStart = start < 0 ? Math.max(0, len + start) : start;
|
||||||
|
const normStop = stop < 0 ? len + stop : stop;
|
||||||
|
if (normStart > normStop || normStart >= len) {
|
||||||
|
this.lists.set(key, []);
|
||||||
|
} else {
|
||||||
|
this.lists.set(key, list.slice(normStart, normStop + 1));
|
||||||
|
}
|
||||||
|
return "OK";
|
||||||
|
}
|
||||||
|
|
||||||
|
public async lrange(key: string, start: number, stop: number): Promise<string[]> {
|
||||||
|
const list = this.lists.get(key) || [];
|
||||||
|
const len = list.length;
|
||||||
|
if (len === 0) return [];
|
||||||
|
const normStart = start < 0 ? Math.max(0, len + start) : start;
|
||||||
|
const normStop = stop < 0 ? len + stop : stop;
|
||||||
|
if (normStart > normStop || normStart >= len) return [];
|
||||||
|
return list.slice(normStart, normStop + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async llen(key: string): Promise<number> {
|
||||||
|
return (this.lists.get(key) || []).length;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async set(
|
||||||
|
key: string,
|
||||||
|
value: string,
|
||||||
|
_pxMode: "PX",
|
||||||
|
ttlMs: number,
|
||||||
|
_nxMode: "NX",
|
||||||
|
): Promise<string | null> {
|
||||||
|
const existing = this.strings.get(key);
|
||||||
|
if (existing && existing.expiresAt > Date.now()) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
this.strings.set(key, { value, expiresAt: Date.now() + ttlMs });
|
||||||
|
return "OK";
|
||||||
|
}
|
||||||
|
|
||||||
|
public async eval(_script: string, _numKeys: number, key: string, token: string): Promise<number> {
|
||||||
|
const existing = this.strings.get(key);
|
||||||
|
if (existing && existing.value === token) {
|
||||||
|
this.strings.delete(key);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const g = globalThis as typeof globalThis & {
|
||||||
|
__dataStackBufferState?: {
|
||||||
|
buffer: DataEvent[];
|
||||||
|
recentEvents: DataEvent[];
|
||||||
|
flushTimer: ReturnType<typeof setInterval> | null;
|
||||||
|
};
|
||||||
|
__dataStackAdminSettings?: unknown;
|
||||||
|
__dataStackClientsMap?: Map<string, unknown>;
|
||||||
|
__dataStackDeletedClients?: Set<string>;
|
||||||
|
};
|
||||||
|
|
||||||
|
let tmpStorageDir = "";
|
||||||
|
const origBaseStorageDir = CSVStorageManager.baseStorageDir;
|
||||||
|
const prevDisableListen = process.env.DATASTACK_DISABLE_LISTEN;
|
||||||
|
const prevIngestionUrl = process.env.INGESTION_INTERNAL_URL;
|
||||||
|
const prevDbUrl = process.env.DATABASE_URL;
|
||||||
|
const prevRedisUrl = process.env.REDIS_URL;
|
||||||
|
|
||||||
|
before(async () => {
|
||||||
|
process.env.DATASTACK_DISABLE_LISTEN = "1";
|
||||||
|
delete process.env.DATABASE_URL;
|
||||||
|
delete process.env.REDIS_URL;
|
||||||
|
delete process.env.INGESTION_INTERNAL_URL;
|
||||||
|
|
||||||
|
tmpStorageDir = fs.mkdtempSync(path.join(os.tmpdir(), "datastack-phase03-"));
|
||||||
|
CSVStorageManager.baseStorageDir = tmpStorageDir;
|
||||||
|
CSVStorageManager.clearRowCountCache();
|
||||||
|
});
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
DataBufferEngine.setRedisClientForTests(undefined);
|
||||||
|
if (g.__dataStackBufferState) {
|
||||||
|
g.__dataStackBufferState.buffer = [];
|
||||||
|
g.__dataStackBufferState.recentEvents = [];
|
||||||
|
}
|
||||||
|
CSVStorageManager.clearRowCountCache();
|
||||||
|
fs.rmSync(tmpStorageDir, { recursive: true, force: true });
|
||||||
|
fs.mkdirSync(tmpStorageDir, { recursive: true });
|
||||||
|
delete g.__dataStackAdminSettings;
|
||||||
|
g.__dataStackClientsMap?.clear();
|
||||||
|
g.__dataStackDeletedClients?.clear();
|
||||||
|
});
|
||||||
|
|
||||||
|
after(async () => {
|
||||||
|
await DataBufferEngine.shutdown();
|
||||||
|
CSVStorageManager.baseStorageDir = origBaseStorageDir;
|
||||||
|
CSVStorageManager.clearRowCountCache();
|
||||||
|
if (tmpStorageDir) {
|
||||||
|
fs.rmSync(tmpStorageDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
|
||||||
|
if (prevDisableListen === undefined) delete process.env.DATASTACK_DISABLE_LISTEN;
|
||||||
|
else process.env.DATASTACK_DISABLE_LISTEN = prevDisableListen;
|
||||||
|
|
||||||
|
if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL;
|
||||||
|
else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl;
|
||||||
|
|
||||||
|
if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
|
||||||
|
else process.env.DATABASE_URL = prevDbUrl;
|
||||||
|
|
||||||
|
if (prevRedisUrl === undefined) delete process.env.REDIS_URL;
|
||||||
|
else process.env.REDIS_URL = prevRedisUrl;
|
||||||
|
});
|
||||||
|
|
||||||
|
test("BUG-11: Shared Redis buffer preserves events on CSV error, honors distributed flush lock, avoids double-counting, and falls back to RAM cleanly", async () => {
|
||||||
|
const fakeRedis = new FakeRedisClient();
|
||||||
|
DataBufferEngine.setRedisClientForTests(fakeRedis);
|
||||||
|
|
||||||
|
const sampleEvents: DataEvent[] = [
|
||||||
|
{
|
||||||
|
clientId: "phase03_redis",
|
||||||
|
eventId: "r-evt-1",
|
||||||
|
timestamp: "2026-11-17T10:00:00.000Z",
|
||||||
|
payload: JSON.stringify({ seq: 1 }),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
clientId: "phase03_redis",
|
||||||
|
eventId: "r-evt-2",
|
||||||
|
timestamp: "2026-11-17T10:00:01.000Z",
|
||||||
|
payload: JSON.stringify({ seq: 2 }),
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
await DataBufferEngine.pushBatchAsync(sampleEvents);
|
||||||
|
|
||||||
|
assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 2);
|
||||||
|
const recentFromRedis = await DataBufferEngine.getRecentEventsAsync(10, "phase03_redis");
|
||||||
|
assert.equal(recentFromRedis.length, 2);
|
||||||
|
assert.equal(recentFromRedis[0].eventId, "r-evt-2");
|
||||||
|
|
||||||
|
// Lock contention must skip concurrent Redis flush without corrupting queue
|
||||||
|
await fakeRedis.set(REDIS_FLUSH_LOCK_KEY, "other-worker-token", "PX", 15000, "NX");
|
||||||
|
const skipped = await DataBufferEngine.flush();
|
||||||
|
assert.equal(skipped.count, 0);
|
||||||
|
assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 2);
|
||||||
|
await fakeRedis.eval("", 1, REDIS_FLUSH_LOCK_KEY, "other-worker-token");
|
||||||
|
|
||||||
|
// Crash/error safety: if appendBatch throws before finishing, Redis LTRIM must NOT discard events
|
||||||
|
const originalAppend = CSVStorageManager.appendBatch;
|
||||||
|
CSVStorageManager.appendBatch = async () => {
|
||||||
|
throw new Error("Simulated disk write failure");
|
||||||
|
};
|
||||||
|
await assert.rejects(() => DataBufferEngine.flush(), /Simulated disk write failure/);
|
||||||
|
CSVStorageManager.appendBatch = originalAppend;
|
||||||
|
assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 2);
|
||||||
|
|
||||||
|
// Subsequent healthy flush drains Redis list, releases distributed lock, and writes CSV
|
||||||
|
const flushed = await DataBufferEngine.flush();
|
||||||
|
assert.equal(flushed.count, 2);
|
||||||
|
assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 0);
|
||||||
|
assert.equal(fakeRedis.strings.has(REDIS_FLUSH_LOCK_KEY), false);
|
||||||
|
|
||||||
|
// Verify Port 4000 /internal/recent & /internal/metrics read shared Redis without double-counting
|
||||||
|
const serverMod = require("../server/index.js") as IngestionServerModule;
|
||||||
|
const origWriterDir = serverMod.writer.baseDir;
|
||||||
|
serverMod.writer.baseDir = tmpStorageDir;
|
||||||
|
serverMod.setRedisClientForTests(fakeRedis);
|
||||||
|
await new Promise<void>((resolve) => serverMod.server.listen(0, "127.0.0.1", () => resolve()));
|
||||||
|
const addr = serverMod.server.address() as AddressInfo;
|
||||||
|
process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${addr.port}`;
|
||||||
|
|
||||||
|
try {
|
||||||
|
await DataBufferEngine.pushAsync({
|
||||||
|
clientId: "phase03_redis",
|
||||||
|
eventId: "r-evt-3",
|
||||||
|
timestamp: "2026-11-17T10:00:02.000Z",
|
||||||
|
payload: JSON.stringify({ seq: 3 }),
|
||||||
|
});
|
||||||
|
const stats = await DataBufferEngine.getBufferStatsAsync();
|
||||||
|
assert.equal(stats.redisEnabled, true);
|
||||||
|
assert.equal(stats.redisBufferSize, 1);
|
||||||
|
|
||||||
|
const metrics = await ApiEngine.getDashboardMetrics();
|
||||||
|
// Shared Redis count (1) must NOT be counted twice as 2 across Port 3000 + Port 4000
|
||||||
|
assert.equal(metrics.bufferStats.currentBufferSize, 1);
|
||||||
|
|
||||||
|
// Clear recent list & buffer via Port 4000 flush
|
||||||
|
const flushRes = await serverMod.flushBuffer();
|
||||||
|
assert.equal(flushRes.written, 1);
|
||||||
|
assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 0);
|
||||||
|
} finally {
|
||||||
|
serverMod.writer.baseDir = origWriterDir;
|
||||||
|
serverMod.setRedisClientForTests(undefined);
|
||||||
|
serverMod.stopTimers();
|
||||||
|
delete process.env.INGESTION_INTERNAL_URL;
|
||||||
|
await new Promise<void>((resolve) => serverMod.io.close(() => serverMod.server.close(() => resolve())));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback to local RAM buffer when Redis is disabled (null)
|
||||||
|
DataBufferEngine.setRedisClientForTests(null);
|
||||||
|
await DataBufferEngine.pushAsync({
|
||||||
|
clientId: "phase03_ram",
|
||||||
|
eventId: "ram-evt-1",
|
||||||
|
timestamp: "2026-11-17T10:01:00.000Z",
|
||||||
|
payload: JSON.stringify({ mode: "ram" }),
|
||||||
|
});
|
||||||
|
const ramStats = await DataBufferEngine.getBufferStatsAsync();
|
||||||
|
assert.equal(ramStats.redisEnabled, false);
|
||||||
|
assert.equal(ramStats.localBufferSize, 1);
|
||||||
|
const ramFlushed = await DataBufferEngine.flush();
|
||||||
|
assert.equal(ramFlushed.count, 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("BUG-09: Cooperative CSV partition locking serializes concurrent Port 3000 + Port 4000 writers, recovers stale locks, and surfaces Port 4000 flush errors", async () => {
|
||||||
|
const worker = new CsvWriterWorker(tmpStorageDir);
|
||||||
|
const partitionDate = "2026-11-17";
|
||||||
|
const clientId = "phase03_concurrent";
|
||||||
|
|
||||||
|
const batchA: DataEvent[] = Array.from({ length: 15 }, (_, idx) => ({
|
||||||
|
clientId,
|
||||||
|
eventId: `a-${idx}`,
|
||||||
|
timestamp: `${partitionDate}T12:00:00.000Z`,
|
||||||
|
payload: JSON.stringify({ writer: "A", idx }),
|
||||||
|
}));
|
||||||
|
const batchB: DataEvent[] = Array.from({ length: 15 }, (_, idx) => ({
|
||||||
|
clientId,
|
||||||
|
eventId: `b-${idx}`,
|
||||||
|
timestamp: `${partitionDate}T12:00:01.000Z`,
|
||||||
|
payload: JSON.stringify({ writer: "B", idx }),
|
||||||
|
}));
|
||||||
|
|
||||||
|
await Promise.all([
|
||||||
|
CSVStorageManager.appendBatch(batchA),
|
||||||
|
worker.writeBatch(batchB),
|
||||||
|
]);
|
||||||
|
|
||||||
|
const csvPath = path.join(tmpStorageDir, clientId, `${partitionDate}.csv`);
|
||||||
|
const lockPath = path.join(tmpStorageDir, clientId, `.${partitionDate}.csv.lock`);
|
||||||
|
assert.equal(fs.existsSync(lockPath), false, "Lock file must be cleaned up after writes");
|
||||||
|
|
||||||
|
const lines = fs.readFileSync(csvPath, "utf8").trim().split("\n");
|
||||||
|
assert.equal(lines[0], '"clientId","eventId","timestamp","payload"');
|
||||||
|
assert.equal(lines.length, 31, "Must contain exactly 1 header + 30 complete rows without interleaving");
|
||||||
|
|
||||||
|
// Stale lock recovery (>30s old mtime)
|
||||||
|
fs.writeFileSync(lockPath, "999999:0", "utf8");
|
||||||
|
const pastTime = new Date(Date.now() - 45_000);
|
||||||
|
fs.utimesSync(lockPath, pastTime, pastTime);
|
||||||
|
await CSVStorageManager.appendBatch([
|
||||||
|
{
|
||||||
|
clientId,
|
||||||
|
eventId: "after-stale-lock",
|
||||||
|
timestamp: `${partitionDate}T12:00:02.000Z`,
|
||||||
|
payload: JSON.stringify({ staleRecovered: true }),
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
assert.equal(fs.existsSync(lockPath), false);
|
||||||
|
const files = await CSVStorageManager.listAllFilesAsync();
|
||||||
|
assert.equal(files[0].linesCount, 31);
|
||||||
|
|
||||||
|
// Explicit flush error & warning propagation in POST /api/storage
|
||||||
|
const failingBridge = createServer((_req, res) => {
|
||||||
|
res.statusCode = 500;
|
||||||
|
res.setHeader("Content-Type", "application/json");
|
||||||
|
res.end(JSON.stringify({ success: false, error: "Disk permission denied on Port 4000" }));
|
||||||
|
});
|
||||||
|
await new Promise<void>((resolve) => failingBridge.listen(0, "127.0.0.1", () => resolve()));
|
||||||
|
const bridgeAddr = failingBridge.address() as AddressInfo;
|
||||||
|
process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${bridgeAddr.port}`;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const errRes = await postStorageRoute(new NextRequest("http://localhost:3000/api/storage", { method: "POST" }));
|
||||||
|
assert.equal(errRes.status, 502);
|
||||||
|
const errJson = await errRes.json();
|
||||||
|
assert.equal(errJson.success, false);
|
||||||
|
assert.equal(errJson.code, "INGESTION_FLUSH_FAILED");
|
||||||
|
assert.equal(errJson.ingestionFlushed, false);
|
||||||
|
} finally {
|
||||||
|
delete process.env.INGESTION_INTERNAL_URL;
|
||||||
|
await new Promise<void>((resolve) => failingBridge.close(() => resolve()));
|
||||||
|
}
|
||||||
|
|
||||||
|
// When Port 4000 is down and INGESTION_INTERNAL_URL is unset, standalone dev surfaces explicit warning + ingestionFlushed: false
|
||||||
|
const standaloneRes = await postStorageRoute(new NextRequest("http://localhost:3000/api/storage", { method: "POST" }));
|
||||||
|
assert.equal(standaloneRes.status, 200);
|
||||||
|
const standaloneJson = await standaloneRes.json();
|
||||||
|
assert.equal(standaloneJson.ingestionFlushed, false);
|
||||||
|
assert.match(String(standaloneJson.warning || ""), /unreachable/i);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("BUG-10: Cached streaming row counter counts multiline RFC 4180 payloads accurately and avoids repeated disk reads", async () => {
|
||||||
|
await CSVStorageManager.appendBatch([
|
||||||
|
{
|
||||||
|
clientId: "phase03_perf",
|
||||||
|
eventId: "multiline-1",
|
||||||
|
timestamp: "2026-11-17T08:00:00.000Z",
|
||||||
|
payload: JSON.stringify({ note: "line1\nline2\n\"quoted\"\nline4" }),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
clientId: "phase03_perf",
|
||||||
|
eventId: "multiline-2",
|
||||||
|
timestamp: "2026-11-17T08:00:01.000Z",
|
||||||
|
payload: JSON.stringify({ note: "single" }),
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const firstStats = await CSVStorageManager.getStorageStatsAsync();
|
||||||
|
assert.equal(firstStats.totalRows, 2);
|
||||||
|
|
||||||
|
// Spy on fs.createReadStream to verify repeated dashboard polls hit mtime+size cache with 0 file scans
|
||||||
|
const origCreateReadStream = fs.createReadStream;
|
||||||
|
let streamReads = 0;
|
||||||
|
(fs as unknown as { createReadStream: typeof fs.createReadStream }).createReadStream = ((
|
||||||
|
...args: Parameters<typeof fs.createReadStream>
|
||||||
|
) => {
|
||||||
|
streamReads += 1;
|
||||||
|
return origCreateReadStream(...args);
|
||||||
|
}) as typeof fs.createReadStream;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const secondStats = await CSVStorageManager.getStorageStatsAsync();
|
||||||
|
assert.equal(secondStats.totalRows, 2);
|
||||||
|
assert.equal(streamReads, 0, "Unchanged CSV files must use cached row count");
|
||||||
|
} finally {
|
||||||
|
(fs as unknown as { createReadStream: typeof fs.createReadStream }).createReadStream =
|
||||||
|
origCreateReadStream;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Server 3s settings sync must not repeatedly prune when autoPruneDays is unchanged
|
||||||
|
const serverMod = require("../server/index.js") as IngestionServerModule;
|
||||||
|
const oldPartitionDir = path.join(tmpStorageDir, "phase03_prune_check");
|
||||||
|
fs.mkdirSync(oldPartitionDir, { recursive: true });
|
||||||
|
fs.writeFileSync(path.join(oldPartitionDir, "2020-01-01.csv"), "h\nr\n", "utf8");
|
||||||
|
const prunedCount = CSVStorageManager.pruneExpiredPartitions(30);
|
||||||
|
assert.equal(prunedCount, 1);
|
||||||
|
|
||||||
|
serverMod.applyRuntimeSettingsObject({ autoPruneDays: 30 });
|
||||||
|
fs.writeFileSync(path.join(oldPartitionDir, "2020-01-02.csv"), "h\nr\n", "utf8");
|
||||||
|
// Same autoPruneDays=30 applied again (simulating 3s DB poll) must NOT prune on every tick
|
||||||
|
serverMod.applyRuntimeSettingsObject({ autoPruneDays: 30 });
|
||||||
|
assert.equal(fs.existsSync(path.join(oldPartitionDir, "2020-01-02.csv")), true);
|
||||||
|
serverMod.stopTimers();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("BUG-12: Prisma schema models and mappings align cleanly with runtime SQL DDL and Docker Compose Redis wiring", () => {
|
||||||
|
const schemaPath = path.resolve(process.cwd(), "prisma/schema.prisma");
|
||||||
|
const schemaText = fs.readFileSync(schemaPath, "utf8");
|
||||||
|
|
||||||
|
assert.match(schemaText, /model ClientSDK[\s\S]*clientId\s+String\s+@id\s+@map\("client_id"\)/);
|
||||||
|
assert.match(schemaText, /token\s+String/);
|
||||||
|
assert.match(schemaText, /model DeletedClient[\s\S]*@@map\("deleted_clients"\)/);
|
||||||
|
assert.match(schemaText, /model AdminUser[\s\S]*tokenVersion\s+Int\s+@default\(1\)\s+@map\("token_version"\)/);
|
||||||
|
assert.match(schemaText, /model AdminLoginAttempt[\s\S]*@@map\("admin_login_attempts"\)/);
|
||||||
|
assert.match(schemaText, /model AdminSettings[\s\S]*maxEventsPerSec\s+Int\s+@default\(100\)\s+@map\("max_events_per_sec"\)/);
|
||||||
|
assert.equal(/model ApiKey\b/.test(schemaText), false);
|
||||||
|
assert.equal(/model AuditLog\b/.test(schemaText), false);
|
||||||
|
|
||||||
|
const composeText = fs.readFileSync(path.resolve(process.cwd(), "docker/docker-compose.yml"), "utf8");
|
||||||
|
assert.equal((composeText.match(/REDIS_URL:\s*"redis:\/\/redis:6379"/g) || []).length, 2);
|
||||||
|
assert.match(composeText, /redis:\s*\n\s*condition:\s*service_healthy/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("BUG-14: GET /api/data searches eventId & clientId + UTC dates, GET /api/events caps CSV tail, and GET /api/export rejects ambiguous filename & honors ?flush=false", async () => {
|
||||||
|
DataBufferEngine.setRedisClientForTests(null);
|
||||||
|
|
||||||
|
await SDKManager.registerClientAsync({ clientId: "alpha_cli", name: "Alpha", rateLimit: 100 });
|
||||||
|
await SDKManager.registerClientAsync({ clientId: "beta_cli", name: "Beta", rateLimit: 100 });
|
||||||
|
|
||||||
|
const tsMillis = String(Date.UTC(2026, 10, 17, 15, 0, 0)); // 2026-11-17
|
||||||
|
await DataBufferEngine.pushBatchAsync([
|
||||||
|
{
|
||||||
|
clientId: "alpha_cli",
|
||||||
|
eventId: "unique-findme-id-99",
|
||||||
|
timestamp: tsMillis,
|
||||||
|
payload: JSON.stringify({ temp: 22 }),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
clientId: "beta_cli",
|
||||||
|
eventId: "beta-id-1",
|
||||||
|
timestamp: "2026-11-17T15:01:00.000Z",
|
||||||
|
payload: JSON.stringify({ temp: 33 }),
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
// 1. Search by eventId with from/to UTC normalization while still buffered
|
||||||
|
const dataByEvent = await getDataRoute(
|
||||||
|
new NextRequest("http://localhost:3000/api/data?search=unique-findme-id-99&from=2026-11-17&to=2026-11-17"),
|
||||||
|
);
|
||||||
|
assert.equal(dataByEvent.status, 200);
|
||||||
|
const dataEventJson = await dataByEvent.json();
|
||||||
|
assert.equal(dataEventJson.total, 1);
|
||||||
|
assert.equal(dataEventJson.events[0].eventId, "unique-findme-id-99");
|
||||||
|
|
||||||
|
// 2. Search by clientId substring
|
||||||
|
const dataByClient = await getDataRoute(
|
||||||
|
new NextRequest("http://localhost:3000/api/data?search=beta_cli"),
|
||||||
|
);
|
||||||
|
const dataClientJson = await dataByClient.json();
|
||||||
|
assert.equal(dataClientJson.total, 1);
|
||||||
|
assert.equal(dataClientJson.events[0].clientId, "beta_cli");
|
||||||
|
|
||||||
|
// 3. Export with ?flush=false must NOT drain active buffer
|
||||||
|
const noFlushRes = await getExportRoute(
|
||||||
|
new NextRequest("http://localhost:3000/api/export?flush=false&format=json"),
|
||||||
|
);
|
||||||
|
assert.equal(noFlushRes.status, 200);
|
||||||
|
assert.equal((await noFlushRes.text()).trim(), "[]");
|
||||||
|
assert.equal((await DataBufferEngine.getBufferStatsAsync()).localBufferSize, 2);
|
||||||
|
|
||||||
|
// 4. Default flush writes both 2026-11-17.csv files -> clientId=all + filename must 400 AMBIGUOUS_FILENAME
|
||||||
|
const flushExportRes = await getExportRoute(
|
||||||
|
new NextRequest("http://localhost:3000/api/export?format=json"),
|
||||||
|
);
|
||||||
|
assert.equal(flushExportRes.status, 200);
|
||||||
|
const flushedPayload = await flushExportRes.json();
|
||||||
|
assert.equal(flushedPayload.length, 2);
|
||||||
|
|
||||||
|
const ambiguousRes = await getExportRoute(
|
||||||
|
new NextRequest("http://localhost:3000/api/export?clientId=all&filename=2026-11-17.csv"),
|
||||||
|
);
|
||||||
|
assert.equal(ambiguousRes.status, 400);
|
||||||
|
const ambiguousJson = await ambiguousRes.json();
|
||||||
|
assert.equal(ambiguousJson.code, "AMBIGUOUS_FILENAME");
|
||||||
|
|
||||||
|
// Disambiguated download with explicit clientId succeeds cleanly
|
||||||
|
const specificRes = await getExportRoute(
|
||||||
|
new NextRequest("http://localhost:3000/api/export?clientId=alpha_cli&filename=2026-11-17.csv"),
|
||||||
|
);
|
||||||
|
assert.equal(specificRes.status, 200);
|
||||||
|
const csvBody = await specificRes.text();
|
||||||
|
assert.match(csvBody, /unique-findme-id-99/);
|
||||||
|
|
||||||
|
// 5. GET /api/events with limit=1 returns latest single stored event cleanly
|
||||||
|
const eventsRes = await getEventsRoute(
|
||||||
|
new NextRequest("http://localhost:3000/api/events?clientId=alpha_cli&limit=1"),
|
||||||
|
);
|
||||||
|
assert.equal(eventsRes.status, 200);
|
||||||
|
const eventsJson = await eventsRes.json();
|
||||||
|
assert.equal(eventsJson.events.length, 1);
|
||||||
|
assert.equal(eventsJson.events[0].eventId, "unique-findme-id-99");
|
||||||
|
});
|
||||||
@@ -0,0 +1,266 @@
|
|||||||
|
"""
|
||||||
|
Phase-05 Verification Tests (BUG-18):
|
||||||
|
- Single Source of Truth between canonical `datastack` (`sdk/python/datastack`)
|
||||||
|
and compatibility alias `datastack_sdk` (`sdk-python`).
|
||||||
|
- `CLIENT_ID` config loading (`config.datastack.py`, `DATASTACK_CLIENT_ID`, kwarg),
|
||||||
|
JWT `clientId`/`sub` fallback extraction, and mismatch validation.
|
||||||
|
- End-to-end propagation of `clientId` & `X-DataStack-Client-Id` into HTTP
|
||||||
|
fallback ingestion (`send`, `send_batch`, `send_file`), status snapshot, and
|
||||||
|
offline disk spool replay against a local HTTP test server.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import unittest
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
CANONICAL_PKG_ROOT = REPO_ROOT / "sdk" / "python" / "datastack"
|
||||||
|
COMPAT_PKG_ROOT = REPO_ROOT / "sdk-python"
|
||||||
|
|
||||||
|
for p in (str(CANONICAL_PKG_ROOT), str(COMPAT_PKG_ROOT)):
|
||||||
|
if p not in sys.path:
|
||||||
|
sys.path.insert(0, p)
|
||||||
|
|
||||||
|
import datastack # noqa: E402
|
||||||
|
import datastack_sdk # noqa: E402
|
||||||
|
from datastack import ( # noqa: E402
|
||||||
|
DataStack,
|
||||||
|
DataStackClient,
|
||||||
|
DataStackConfig,
|
||||||
|
DataStackConfigError,
|
||||||
|
DataStackOfflineSpoolError,
|
||||||
|
DiskSpooler,
|
||||||
|
HeartbeatMonitor,
|
||||||
|
load_config,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_jwt(payload: Dict[str, Any]) -> str:
|
||||||
|
header = (
|
||||||
|
base64.urlsafe_b64encode(json.dumps({"alg": "HS256", "typ": "JWT"}).encode())
|
||||||
|
.decode()
|
||||||
|
.rstrip("=")
|
||||||
|
)
|
||||||
|
body = (
|
||||||
|
base64.urlsafe_b64encode(json.dumps(payload).encode())
|
||||||
|
.decode()
|
||||||
|
.rstrip("=")
|
||||||
|
)
|
||||||
|
return f"{header}.{body}.testsig"
|
||||||
|
|
||||||
|
|
||||||
|
class Phase05PythonSdkTests(unittest.TestCase):
|
||||||
|
def test_single_source_of_truth_identity(self) -> None:
|
||||||
|
self.assertIs(datastack_sdk.DataStack, datastack.DataStack)
|
||||||
|
self.assertIs(datastack_sdk.DataStackClient, datastack.DataStackClient)
|
||||||
|
self.assertIs(datastack_sdk.DataStackConfig, datastack.DataStackConfig)
|
||||||
|
self.assertIs(datastack_sdk.load_config, datastack.load_config)
|
||||||
|
self.assertIs(
|
||||||
|
datastack_sdk.DataStackConfigError, datastack.DataStackConfigError
|
||||||
|
)
|
||||||
|
self.assertIs(
|
||||||
|
datastack_sdk.DataStackOfflineSpoolError,
|
||||||
|
datastack.DataStackOfflineSpoolError,
|
||||||
|
)
|
||||||
|
self.assertIs(datastack_sdk.DiskSpooler, datastack.DiskSpooler)
|
||||||
|
self.assertIs(datastack_sdk.HeartbeatMonitor, datastack.HeartbeatMonitor)
|
||||||
|
self.assertEqual(datastack_sdk.__version__, datastack.__version__)
|
||||||
|
|
||||||
|
# Submodule shims also resolve to canonical classes
|
||||||
|
from datastack_sdk.client import DataStackClient as ShimClient
|
||||||
|
from datastack_sdk.config import load_config as shim_load_config
|
||||||
|
from datastack_sdk.exceptions import (
|
||||||
|
DataStackOfflineSpoolError as ShimSpoolErr,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertIs(ShimClient, DataStackClient)
|
||||||
|
self.assertIs(shim_load_config, load_config)
|
||||||
|
self.assertIs(ShimSpoolErr, DataStackOfflineSpoolError)
|
||||||
|
|
||||||
|
def test_client_id_config_env_and_jwt_validation(self) -> None:
|
||||||
|
tok_alpha = _make_jwt({"clientId": "alpha_svc", "sub": "alpha_svc"})
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
cfg_file = Path(tmpdir) / "config.datastack.py"
|
||||||
|
cfg_file.write_text(
|
||||||
|
f'DATASTACK_CONFIG = {{\n'
|
||||||
|
f' "SERVER_URL": "http://127.0.0.1:49999",\n'
|
||||||
|
f' "API_URL": "http://127.0.0.1:39999",\n'
|
||||||
|
f' "CLIENT_TOKEN": "{tok_alpha}",\n'
|
||||||
|
f' "CLIENT_ID": "alpha_svc",\n'
|
||||||
|
f' "AUTO_CONNECT": False,\n'
|
||||||
|
f' "OFFLINE_SPOOL_DIR": "{Path(tmpdir) / "spool"}",\n'
|
||||||
|
f'}}\n',
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
cfg = load_config(config_path=str(cfg_file))
|
||||||
|
self.assertIsInstance(cfg, DataStackConfig)
|
||||||
|
self.assertEqual(cfg.client_id, "alpha_svc")
|
||||||
|
|
||||||
|
# Matching explicit client_id succeeds
|
||||||
|
client = DataStack(config_path=str(cfg_file), auto_connect=False)
|
||||||
|
self.assertEqual(client.client_id, "alpha_svc")
|
||||||
|
self.assertEqual(client.status()["clientId"], "alpha_svc")
|
||||||
|
client.disconnect()
|
||||||
|
|
||||||
|
# Env override matching JWT succeeds; mismatching raises DataStackConfigError
|
||||||
|
old_env = os.environ.get("DATASTACK_CLIENT_ID")
|
||||||
|
try:
|
||||||
|
os.environ["DATASTACK_CLIENT_ID"] = "wrong_svc"
|
||||||
|
with self.assertRaises(DataStackConfigError):
|
||||||
|
DataStack(config_path=str(cfg_file), auto_connect=False)
|
||||||
|
finally:
|
||||||
|
if old_env is None:
|
||||||
|
os.environ.pop("DATASTACK_CLIENT_ID", None)
|
||||||
|
else:
|
||||||
|
os.environ["DATASTACK_CLIENT_ID"] = old_env
|
||||||
|
|
||||||
|
# Mismatched kwarg client_id raises DataStackConfigError
|
||||||
|
with self.assertRaises(DataStackConfigError):
|
||||||
|
DataStack(
|
||||||
|
config_path=str(cfg_file),
|
||||||
|
client_id="mismatch_svc",
|
||||||
|
auto_connect=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Auto-derives client_id from JWT when CLIENT_ID is omitted
|
||||||
|
tok_derived = _make_jwt({"sub": "derived_worker"})
|
||||||
|
derived_client = DataStack(
|
||||||
|
server_url="http://127.0.0.1:49999",
|
||||||
|
api_url="http://127.0.0.1:39999",
|
||||||
|
token=tok_derived,
|
||||||
|
auto_connect=False,
|
||||||
|
)
|
||||||
|
self.assertEqual(derived_client.client_id, "derived_worker")
|
||||||
|
self.assertEqual(derived_client.status()["clientId"], "derived_worker")
|
||||||
|
derived_client.disconnect()
|
||||||
|
|
||||||
|
def test_http_fallback_and_offline_spool_carry_client_id(self) -> None:
|
||||||
|
recorded_requests: List[Dict[str, Any]] = []
|
||||||
|
tok = _make_jwt({"clientId": "sensor_01", "sub": "sensor_01"})
|
||||||
|
|
||||||
|
class MockHandler(BaseHTTPRequestHandler):
|
||||||
|
def do_GET(self) -> None: # noqa: N802
|
||||||
|
if self.path == "/api/health":
|
||||||
|
body = json.dumps({"status": "online"}).encode("utf-8")
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
self.send_header("Content-Length", str(len(body)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(body)
|
||||||
|
return
|
||||||
|
self.send_response(404)
|
||||||
|
self.end_headers()
|
||||||
|
|
||||||
|
def do_POST(self) -> None: # noqa: N802
|
||||||
|
length = int(self.headers.get("Content-Length", "0"))
|
||||||
|
raw = self.rfile.read(length).decode("utf-8")
|
||||||
|
parsed = json.loads(raw) if raw else {}
|
||||||
|
recorded_requests.append(
|
||||||
|
{
|
||||||
|
"path": self.path,
|
||||||
|
"client_id_header": self.headers.get("X-DataStack-Client-Id"),
|
||||||
|
"auth": self.headers.get("Authorization"),
|
||||||
|
"body": parsed,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
resp_payload: Dict[str, Any] = {"success": True}
|
||||||
|
if isinstance(parsed, dict) and "batch" in parsed:
|
||||||
|
resp_payload["processed"] = len(parsed["batch"])
|
||||||
|
resp_payload["rejected"] = 0
|
||||||
|
elif isinstance(parsed, dict) and "records" in parsed:
|
||||||
|
resp_payload["processed"] = len(parsed["records"])
|
||||||
|
resp_payload["rejected"] = 0
|
||||||
|
body = json.dumps(resp_payload).encode("utf-8")
|
||||||
|
self.send_response(200)
|
||||||
|
self.send_header("Content-Type", "application/json")
|
||||||
|
self.send_header("Content-Length", str(len(body)))
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(body)
|
||||||
|
|
||||||
|
def log_message(self, format: str, *args: Any) -> None:
|
||||||
|
return
|
||||||
|
|
||||||
|
server = ThreadingHTTPServer(("127.0.0.1", 0), MockHandler)
|
||||||
|
port = server.server_address[1]
|
||||||
|
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||||
|
thread.start()
|
||||||
|
|
||||||
|
try:
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
spool_dir = Path(tmpdir) / "spool"
|
||||||
|
csv_file = Path(tmpdir) / "sample.csv"
|
||||||
|
csv_file.write_text("temp,unit\n21.5,C\n22.0,C\n", encoding="utf-8")
|
||||||
|
|
||||||
|
client = DataStack(
|
||||||
|
server_url="http://127.0.0.1:1", # unreachable WS -> HTTP fallback
|
||||||
|
api_url=f"http://127.0.0.1:{port}",
|
||||||
|
token=tok,
|
||||||
|
client_id="sensor_01",
|
||||||
|
auto_connect=False,
|
||||||
|
request_timeout_sec=1.0,
|
||||||
|
max_retries=0,
|
||||||
|
offline_spool_dir=str(spool_dir),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Single event via HTTP fallback
|
||||||
|
res_single = client.send({"reading": 42})
|
||||||
|
self.assertTrue(res_single.get("success"))
|
||||||
|
self.assertEqual(res_single.get("transport"), "HTTP_FALLBACK")
|
||||||
|
|
||||||
|
# Batch events via HTTP fallback
|
||||||
|
res_batch = client.send_batch([{"k": "a"}, {"k": "b"}])
|
||||||
|
self.assertEqual(res_batch.get("processed"), 2)
|
||||||
|
|
||||||
|
# File ingestion via HTTP fallback
|
||||||
|
res_file = client.send_file(csv_file)
|
||||||
|
self.assertEqual(res_file.get("processed"), 2)
|
||||||
|
|
||||||
|
client.disconnect()
|
||||||
|
|
||||||
|
self.assertEqual(len(recorded_requests), 3)
|
||||||
|
for req in recorded_requests:
|
||||||
|
self.assertEqual(req["path"], "/api/events")
|
||||||
|
self.assertEqual(req["client_id_header"], "sensor_01")
|
||||||
|
self.assertEqual(req["auth"], f"Bearer {tok}")
|
||||||
|
|
||||||
|
self.assertEqual(recorded_requests[0]["body"]["clientId"], "sensor_01")
|
||||||
|
self.assertEqual(
|
||||||
|
[r["clientId"] for r in recorded_requests[1]["body"]["batch"]],
|
||||||
|
["sensor_01", "sensor_01"],
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
[r["clientId"] for r in recorded_requests[2]["body"]["records"]],
|
||||||
|
["sensor_01", "sensor_01"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# Offline spool check when HTTP is also unreachable
|
||||||
|
offline_client = datastack_sdk.DataStackClient(
|
||||||
|
server_url="http://127.0.0.1:1",
|
||||||
|
api_url="http://127.0.0.1:1",
|
||||||
|
token=tok,
|
||||||
|
auto_connect=False,
|
||||||
|
request_timeout_sec=1.0,
|
||||||
|
max_retries=0,
|
||||||
|
offline_spool_dir=str(spool_dir),
|
||||||
|
)
|
||||||
|
spool_res = offline_client.send({"offline_metric": 99})
|
||||||
|
offline_client.disconnect()
|
||||||
|
self.assertTrue(spool_res.get("spooled"))
|
||||||
|
self.assertEqual(spool_res.get("transport"), "OFFLINE_SPOOL")
|
||||||
|
self.assertGreaterEqual(spool_res.get("spooledCount", 0), 1)
|
||||||
|
finally:
|
||||||
|
server.shutdown()
|
||||||
|
server.server_close()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main(verbosity=2)
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import path from 'node:path';
|
||||||
|
import test from 'node:test';
|
||||||
|
|
||||||
|
const ROOT = process.cwd();
|
||||||
|
|
||||||
|
function readPngDimensions(filePath: string): { width: number; height: number } {
|
||||||
|
const buf = fs.readFileSync(filePath);
|
||||||
|
const pngSig = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
||||||
|
assert.ok(buf.subarray(0, 8).equals(pngSig), `${path.basename(filePath)} must be a valid PNG`);
|
||||||
|
assert.equal(buf.subarray(12, 16).toString('ascii'), 'IHDR');
|
||||||
|
return {
|
||||||
|
width: buf.readUInt32BE(16),
|
||||||
|
height: buf.readUInt32BE(20),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
test('BUG-25: PWA manifest, icons (192/512/maskable/apple-touch), and Service Worker are complete and valid', () => {
|
||||||
|
const manifestPath = path.join(ROOT, 'public/manifest.json');
|
||||||
|
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
|
||||||
|
|
||||||
|
assert.equal(manifest.id, '/');
|
||||||
|
assert.equal(manifest.scope, '/');
|
||||||
|
assert.equal(manifest.display, 'standalone');
|
||||||
|
assert.ok(manifest.short_name && manifest.short_name.length <= 12);
|
||||||
|
assert.ok(Array.isArray(manifest.icons) && manifest.icons.length >= 3);
|
||||||
|
|
||||||
|
for (const icon of manifest.icons) {
|
||||||
|
assert.ok(!String(icon.purpose || '').includes('any maskable'), 'Separate any and maskable purposes');
|
||||||
|
const rel = String(icon.src).replace(/^\//, '');
|
||||||
|
const full = path.join(ROOT, 'public', rel);
|
||||||
|
assert.ok(fs.existsSync(full), `Missing manifest icon ${icon.src}`);
|
||||||
|
const [expW, expH] = String(icon.sizes).split('x').map(Number);
|
||||||
|
const dims = readPngDimensions(full);
|
||||||
|
assert.equal(dims.width, expW);
|
||||||
|
assert.equal(dims.height, expH);
|
||||||
|
}
|
||||||
|
|
||||||
|
const appleDims = readPngDimensions(path.join(ROOT, 'public/apple-touch-icon.png'));
|
||||||
|
assert.deepEqual(appleDims, { width: 180, height: 180 });
|
||||||
|
assert.ok(fs.existsSync(path.join(ROOT, 'public/icon.svg')));
|
||||||
|
|
||||||
|
const swContent = fs.readFileSync(path.join(ROOT, 'public/sw.js'), 'utf8');
|
||||||
|
assert.match(swContent, /request\.method !== ['"]GET['"]/);
|
||||||
|
assert.match(swContent, /\/api\//);
|
||||||
|
assert.match(swContent, /\/socket\.io\//);
|
||||||
|
assert.match(swContent, /new Response\(/);
|
||||||
|
|
||||||
|
const layoutContent = fs.readFileSync(path.join(ROOT, 'app/layout.tsx'), 'utf8');
|
||||||
|
assert.match(layoutContent, /PwaLifecycle/);
|
||||||
|
assert.match(layoutContent, /export const viewport: Viewport/);
|
||||||
|
assert.match(layoutContent, /apple-touch-icon\.png/);
|
||||||
|
|
||||||
|
const pwaClient = fs.readFileSync(path.join(ROOT, 'components/pwa/PwaLifecycle.tsx'), 'utf8');
|
||||||
|
assert.match(pwaClient, /serviceWorker\.register\(['"]\/sw\.js['"]/);
|
||||||
|
assert.match(pwaClient, /beforeinstallprompt/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('BUG-17: Landing/Navbar/Footer/404 navigation uses root-relative anchors and valid Link semantics', () => {
|
||||||
|
const files = [
|
||||||
|
'components/layout/Navbar.tsx',
|
||||||
|
'components/layout/Footer.tsx',
|
||||||
|
'components/sections/Hero.tsx',
|
||||||
|
'components/sections/LandingAbout.tsx',
|
||||||
|
'app/not-found.tsx',
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const rel of files) {
|
||||||
|
const content = fs.readFileSync(path.join(ROOT, rel), 'utf8');
|
||||||
|
assert.equal(/href=["']#/.test(content), false, `${rel} should use /#... root anchors instead of bare #...`);
|
||||||
|
const linkBlocks = content.match(/<Link\b[\s\S]*?<\/Link>/g) || [];
|
||||||
|
for (const block of linkBlocks) {
|
||||||
|
assert.equal(/<Button\b/.test(block), false, `${rel} should not nest <Button> inside <Link>: ${block}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const notFoundContent = fs.readFileSync(path.join(ROOT, 'app/not-found.tsx'), 'utf8');
|
||||||
|
assert.match(notFoundContent, /href=["']\/["']/);
|
||||||
|
assert.match(notFoundContent, /href=["']\/vcon\/dashboard["']/);
|
||||||
|
});
|
||||||
Reference in new issue
Block a user