refactor: switch SDK methods to asynchronous

Migrate SDK interactions to asynchronous calls, relax admin setup email constraints, and add event retrieval functionality.
This commit is contained in:
Team Dark Devil committed 2026-11-17 20:05:00 +06:00
1 parent 81c5e4e2fd
commit bb938c33c0
36 files changed
+1641 -716

No files matched your search

+1
View File
@@ -6,3 +6,4 @@ coverage/
.env* .env*
!.env.example !.env.example
notes/ notes/
storage/data/
+1 -1
View File
@@ -9,7 +9,7 @@ import { redirect } from 'next/navigation';
export const dynamic = 'force-dynamic'; export const dynamic = 'force-dynamic';
export default async function LandingPage() { export default async function LandingPage() {
if (process.env.NODE_ENV === 'production' && process.env.DATABASE_URL) { if (process.env.DATABASE_URL) {
const isConfigured = await isAdminSetupComplete().catch(() => false); const isConfigured = await isAdminSetupComplete().catch(() => false);
if (!isConfigured) redirect('/vcon/setup'); if (!isConfigured) redirect('/vcon/setup');
} }
+7 -2
View File
@@ -100,7 +100,12 @@ export async function PUT(request: NextRequest) {
return NextResponse.json({ error: result.error }, { status: 401 }); return NextResponse.json({ error: result.error }, { status: 401 });
} }
const expiresAt = Date.now() + SESSION_SECONDS * 1000; // Preserve remaining session duration (e.g., 30-day "Remember me" session)
const remainingSeconds = Math.max(
SESSION_SECONDS,
Math.ceil((session.expiresAt - Date.now()) / 1000),
);
const expiresAt = Date.now() + remainingSeconds * 1000;
const nextToken = await createAdminSessionToken( const nextToken = await createAdminSessionToken(
result.email, result.email,
sessionSecret, sessionSecret,
@@ -121,7 +126,7 @@ export async function PUT(request: NextRequest) {
response.cookies.set( response.cookies.set(
SESSION_COOKIE, SESSION_COOKIE,
nextToken, nextToken,
getSessionCookieOptions(request, SESSION_SECONDS), getSessionCookieOptions(request, remainingSeconds),
); );
return response; return response;
+2 -19
View File
@@ -23,21 +23,12 @@ export async function POST(request: NextRequest) {
const sessionSecret = const sessionSecret =
process.env.DATASTACK_ADMIN_SESSION_SECRET || process.env.DATASTACK_ADMIN_SESSION_SECRET ||
"datastack-admin-session-default-secret-32-chars"; "datastack-admin-session-default-secret-32-chars";
const configuredAdminEmail =
process.env.DATASTACK_ADMIN_EMAIL?.trim().toLowerCase() ||
"admin@datastack.dev";
if (!sessionSecret || sessionSecret.length < 32) { if (!sessionSecret || sessionSecret.length < 32) {
return NextResponse.json( return NextResponse.json(
{ error: "Admin session configuration is unavailable." }, { error: "Admin session configuration is unavailable." },
{ status: 503 }, { status: 503 },
); );
} }
if (!configuredAdminEmail || !validEmail(configuredAdminEmail)) {
return NextResponse.json(
{ error: "Admin setup configuration is unavailable." },
{ status: 503 },
);
}
try { try {
const body = await request.json(); const body = await request.json();
@@ -52,17 +43,9 @@ export async function POST(request: NextRequest) {
{ status: 400 }, { status: 400 },
); );
} }
if (email !== configuredAdminEmail) { if (password.length < 8 || password.length > 256) {
return NextResponse.json( return NextResponse.json(
{ { error: "Choose a password between 8 and 256 characters." },
error: "Use the administrator email configured for this deployment.",
},
{ status: 403 },
);
}
if (password.length < 12 || password.length > 256) {
return NextResponse.json(
{ error: "Choose a password between 12 and 256 characters." },
{ status: 400 }, { status: 400 },
); );
} }
+4 -4
View File
@@ -6,7 +6,7 @@ export async function GET(
{ params }: { params: Promise<{ id: string }> }, { params }: { params: Promise<{ id: string }> },
) { ) {
const { id } = await params; const { id } = await params;
const client = SDKManager.getClient(id); const client = await SDKManager.getClientAsync(id);
if (!client) { if (!client) {
return NextResponse.json({ error: "Client not found" }, { status: 404 }); return NextResponse.json({ error: "Client not found" }, { status: 404 });
} }
@@ -24,7 +24,7 @@ export async function PATCH(
// Special action: regenerate token // Special action: regenerate token
if (body.action === "regenerate_token") { if (body.action === "regenerate_token") {
const refreshed = SDKManager.regenerateToken(id); const refreshed = await SDKManager.regenerateTokenAsync(id);
if (!refreshed) { if (!refreshed) {
return NextResponse.json( return NextResponse.json(
{ error: "Client not found" }, { error: "Client not found" },
@@ -34,7 +34,7 @@ export async function PATCH(
return NextResponse.json({ client: refreshed }); return NextResponse.json({ client: refreshed });
} }
const updated = SDKManager.updateClient(id, body); const updated = await SDKManager.updateClientAsync(id, body);
if (!updated) { if (!updated) {
return NextResponse.json({ error: "Client not found" }, { status: 404 }); return NextResponse.json({ error: "Client not found" }, { status: 404 });
} }
@@ -50,7 +50,7 @@ export async function DELETE(
{ params }: { params: Promise<{ id: string }> }, { params }: { params: Promise<{ id: string }> },
) { ) {
const { id } = await params; const { id } = await params;
const deleted = SDKManager.deleteClient(id); const deleted = await SDKManager.deleteClientAsync(id);
if (!deleted) { if (!deleted) {
return NextResponse.json({ error: "Client not found" }, { status: 404 }); return NextResponse.json({ error: "Client not found" }, { status: 404 });
} }
+14 -8
View File
@@ -3,9 +3,11 @@ import { NextRequest, NextResponse } from "next/server";
export async function GET() { export async function GET() {
try { try {
const clients = SDKManager.listClients().map( const allClients = await SDKManager.listClientsAsync();
({ secret, token, ...client }) => client, const clients = allClients.map(({ secret, ...client }) => {
); void secret;
return client;
});
return NextResponse.json({ clients, count: clients.length }); return NextResponse.json({ clients, count: clients.length });
} catch (err: unknown) { } catch (err: unknown) {
const message = const message =
@@ -26,17 +28,21 @@ export async function POST(req: NextRequest) {
); );
} }
const existing = SDKManager.getClient(clientId); const safeClientId = String(clientId)
.trim()
.toLowerCase()
.replace(/[^a-z0-9_-]/g, "_");
const existing = await SDKManager.getClientAsync(safeClientId);
if (existing) { if (existing) {
return NextResponse.json( return NextResponse.json(
{ error: `Client '${clientId}' already exists` }, { error: `Client '${safeClientId}' already exists` },
{ status: 409 }, { status: 409 },
); );
} }
const newClient = SDKManager.registerClient({ const newClient = await SDKManager.registerClientAsync({
clientId, clientId: safeClientId,
name, name: String(name),
rateLimit: rateLimit ? Number(rateLimit) : 100, rateLimit: rateLimit ? Number(rateLimit) : 100,
}); });
+155 -12
View File
@@ -1,15 +1,77 @@
import { NextRequest, NextResponse } from 'next/server'; import { SettingsManager } from '@/core/adminSettings/settingsManager';
import { verifyAdminSessionToken } from '@/core/auth/adminSession';
import { AuthService } from '@/core/auth/jwt';
import { DataBufferEngine } from '@/core/data/buffer'; import { DataBufferEngine } from '@/core/data/buffer';
import { SDKManager } from '@/core/sdk/sdkManager'; import { SDKManager } from '@/core/sdk/sdkManager';
import { SecurityEngine } from '@/core/Security/security'; import { SecurityEngine } from '@/core/Security/security';
import { CSVStorageManager, type DataEvent } from '@/core/Storage/csvStorage';
import { parse } from 'csv-parse';
import { NextRequest, NextResponse } from 'next/server';
import { createReadStream } from 'node:fs';
const SESSION_COOKIE = 'datastack_admin_session';
async function getRecentStoredEvents(
limit: number,
clientId?: string,
): Promise<DataEvent[]> {
const files = CSVStorageManager.listAllFiles()
.filter((f) => !clientId || clientId === 'all' || f.clientId === clientId)
.slice(0, 5);
const collected: DataEvent[] = [];
for (const file of files) {
const fileEvents: DataEvent[] = [];
const parser = createReadStream(file.filePath).pipe(
parse({ columns: true, skip_empty_lines: true, bom: true }),
);
try {
for await (const record of parser as AsyncIterable<
Record<string, string>
>) {
if (!record.eventId) continue;
fileEvents.push({
clientId: record.clientId || file.clientId,
eventId: record.eventId,
timestamp: record.timestamp || `${file.date}T00:00:00.000Z`,
payload: record.payload || '',
});
}
} catch {
parser.destroy();
}
if (fileEvents.length > 0) {
collected.push(...fileEvents.slice(-limit));
}
}
return collected;
}
export async function GET(req: NextRequest) { export async function GET(req: NextRequest) {
try { try {
const { searchParams } = new URL(req.url); const { searchParams } = new URL(req.url);
const clientId = searchParams.get('clientId') || undefined; const clientId = searchParams.get('clientId') || undefined;
const limit = Number(searchParams.get('limit')) || 50; const limit = Math.min(200, Math.max(1, Number(searchParams.get('limit')) || 50));
const bufferedEvents = DataBufferEngine.getRecentEvents(limit, clientId);
const storedEvents = await getRecentStoredEvents(limit, clientId);
const seenIds = new Set<string>();
const merged: DataEvent[] = [];
for (const evt of [...bufferedEvents, ...storedEvents]) {
if (!evt.eventId || seenIds.has(evt.eventId)) continue;
seenIds.add(evt.eventId);
merged.push(evt);
}
merged.sort((a, b) => (b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0));
const events = merged.slice(0, limit);
const events = DataBufferEngine.getRecentEvents(limit, clientId);
return NextResponse.json({ events, count: events.length }); return NextResponse.json({ events, count: events.length });
} catch (err: unknown) { } catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Events retrieval failed'; const message = err instanceof Error ? err.message : 'Events retrieval failed';
@@ -19,16 +81,97 @@ export async function GET(req: NextRequest) {
export async function POST(req: NextRequest) { export async function POST(req: NextRequest) {
try { try {
const body = await req.json(); const settings = await SettingsManager.getSettingsAsync();
const clientId = body.clientId || 'py_telemetry_agent_01';
const eventId = body.eventId || `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
const timestamp = body.timestamp || new Date().toISOString();
const data = body.data || { sample: 'Realtime event via DataStack API', rand: Math.random() };
// Validate security // Verify either Client Bearer JWT or Admin Session Cookie
const val = SecurityEngine.validateEvent(clientId, eventId, data, 200); const authHeader = req.headers.get('authorization');
const sessionSecret =
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
'datastack-admin-session-default-secret-32-chars';
const sessionCookie = req.cookies.get(SESSION_COOKIE)?.value;
let tokenClientId: string | null = null;
let tokenRateLimit: number | undefined;
let rawBearerToken: string | null = null;
if (authHeader && authHeader.toLowerCase().startsWith('bearer ')) {
rawBearerToken = authHeader.slice(7).trim();
const verified = AuthService.verifyClientToken(rawBearerToken);
if (!verified.valid || !verified.payload) {
return NextResponse.json(
{ error: verified.error || 'Invalid client JWT token.' },
{ status: 401 },
);
}
tokenClientId = verified.payload.clientId;
tokenRateLimit = verified.payload.rateLimit;
} else {
const isAdmin = Boolean(
sessionSecret &&
sessionCookie &&
(await verifyAdminSessionToken(sessionCookie, sessionSecret)),
);
if (!isAdmin) {
return NextResponse.json(
{ error: 'Authentication required. Provide a valid Client Bearer JWT or Admin Session.' },
{ status: 401 },
);
}
}
const body = await req.json();
const clientId = String(
tokenClientId || body.clientId || 'py_telemetry_agent_01',
)
.trim()
.toLowerCase()
.replace(/[^a-z0-9_-]/g, '_');
const registeredClient = await SDKManager.getClientAsync(clientId);
if (registeredClient) {
if (registeredClient.status === 'revoked') {
return NextResponse.json(
{ error: `Client '${clientId}' access has been revoked by administrator.` },
{ status: 403 },
);
}
if (
rawBearerToken &&
registeredClient.token &&
registeredClient.token !== rawBearerToken
) {
return NextResponse.json(
{ error: 'Client credentials have been rotated. Use the latest token.' },
{ status: 401 },
);
}
}
const eventId =
body.eventId ||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
const timestamp = body.timestamp || new Date().toISOString();
const data =
body.data !== undefined
? body.data
: { sample: 'Realtime event via DataStack API', rand: Math.random() };
const effectiveClientRate =
registeredClient?.rateLimit ||
tokenRateLimit ||
settings.maxEventsPerSec ||
100;
// Validate security (idempotency, rate limit, payload size)
const val = SecurityEngine.validateEvent(
clientId,
eventId,
data,
effectiveClientRate,
);
if (!val.allowed) { if (!val.allowed) {
return NextResponse.json({ error: val.reason }, { status: 429 }); const status = val.code === 'PAYLOAD_TOO_LARGE' ? 413 : 429;
return NextResponse.json({ error: val.reason, code: val.code }, { status });
} }
const event = { const event = {
@@ -43,7 +186,7 @@ export async function POST(req: NextRequest) {
return NextResponse.json({ success: true, event }); return NextResponse.json({ success: true, event });
} catch (err: unknown) { } catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Event simulation failed'; const message = err instanceof Error ? err.message : 'Event ingestion failed';
return NextResponse.json({ error: message }, { status: 500 }); return NextResponse.json({ error: message }, { status: 500 });
} }
} }
+13
View File
@@ -1,3 +1,4 @@
import { DataBufferEngine } from "@/core/data/buffer";
import { import {
CSVStorageManager, CSVStorageManager,
type StorageFileMeta, type StorageFileMeta,
@@ -107,6 +108,18 @@ function streamFile(file: StorageFileMeta) {
export async function GET(request: NextRequest) { export async function GET(request: NextRequest) {
try { try {
await DataBufferEngine.flush();
const ingestionBaseUrl =
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000";
try {
await fetch(`${ingestionBaseUrl}/internal/flush`, {
method: "POST",
signal: AbortSignal.timeout(1500),
});
} catch {
// Port 4000 may not be running in isolated single-process dev mode
}
const { searchParams } = new URL(request.url); const { searchParams } = new URL(request.url);
const clientId = searchParams.get("clientId") || "all"; const clientId = searchParams.get("clientId") || "all";
const filename = searchParams.get("filename"); const filename = searchParams.get("filename");
+4 -3
View File
@@ -3,7 +3,7 @@ import { NextRequest, NextResponse } from "next/server";
export async function GET() { export async function GET() {
try { try {
const settings = SettingsManager.getPublicSettings(); const settings = await SettingsManager.getPublicSettingsAsync();
return NextResponse.json({ settings }); return NextResponse.json({ settings });
} catch (err: unknown) { } catch (err: unknown) {
const message = err instanceof Error ? err.message : "Settings read failed"; const message = err instanceof Error ? err.message : "Settings read failed";
@@ -14,9 +14,10 @@ export async function GET() {
export async function POST(req: NextRequest) { export async function POST(req: NextRequest) {
try { try {
const body = await req.json(); const body = await req.json();
SettingsManager.updateSettings(body); await SettingsManager.updateSettingsAsync(body);
const settings = await SettingsManager.getPublicSettingsAsync();
return NextResponse.json({ return NextResponse.json({
settings: SettingsManager.getPublicSettings(), settings,
success: true, success: true,
}); });
} catch (err: unknown) { } catch (err: unknown) {
+32 -4
View File
@@ -23,12 +23,40 @@ export async function GET(req: NextRequest) {
export async function POST(_req: NextRequest) { export async function POST(_req: NextRequest) {
try { try {
// Manual flush buffer to disk // 1. Flush local Next.js DataBufferEngine
const res = await DataBufferEngine.flush(); const localRes = await DataBufferEngine.flush();
// 2. Also flush Port 4000 Socket.IO ingestion buffer
let remoteCount = 0;
let remoteFiles: string[] = [];
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
).replace(/\/$/, '');
try {
const ingestRes = await fetch(`${ingestionBaseUrl}/internal/flush`, {
method: 'POST',
cache: 'no-store',
});
if (ingestRes.ok) {
const ingestData = await ingestRes.json();
remoteCount = Number(ingestData.flushedCount) || 0;
if (Array.isArray(ingestData.filesUpdated)) {
remoteFiles = ingestData.filesUpdated;
}
}
} catch {
// Port 4000 server may be offline in standalone Next.js dev mode
}
const combinedFiles = Array.from(
new Set([...localRes.files, ...remoteFiles]),
);
return NextResponse.json({ return NextResponse.json({
success: true, success: true,
flushedCount: res.count, flushedCount: localRes.count + remoteCount,
filesUpdated: res.files, filesUpdated: combinedFiles,
}); });
} catch (err: unknown) { } catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Manual flush failed'; const message = err instanceof Error ? err.message : 'Manual flush failed';
+4 -2
View File
@@ -6,10 +6,12 @@ export const runtime = 'nodejs';
export const dynamic = 'force-dynamic'; export const dynamic = 'force-dynamic';
export default async function LoginPage() { export default async function LoginPage() {
if (process.env.NODE_ENV === 'production' && process.env.DATABASE_URL) { if (process.env.DATABASE_URL) {
const isConfigured = await isAdminSetupComplete().catch(() => false); const isConfigured = await isAdminSetupComplete().catch(() => false);
if (!isConfigured) redirect('/vcon/setup'); if (!isConfigured) redirect('/vcon/setup');
} }
return <AdminLogin />; const showDemoHint = !process.env.DATABASE_URL && process.env.NODE_ENV !== 'production';
return <AdminLogin showDemoHint={showDemoHint} />;
} }
+15 -1
View File
@@ -38,7 +38,9 @@ export default function SdkGuidePage() {
const list = setCached('vcon:clients', data.clients || []); const list = setCached('vcon:clients', data.clients || []);
setClients(list); setClients(list);
if (list.length > 0) { if (list.length > 0) {
setSelectedClientId((prev) => prev || list[0].clientId); setSelectedClientId((prev) =>
list.some((c: any) => c.clientId === prev) ? prev : list[0].clientId
);
} }
} }
} catch { } catch {
@@ -84,6 +86,17 @@ if client.connect():
<PageShell <PageShell
title="Python SDK" title="Python SDK"
actions={ actions={
<>
{activeClient?.token && (
<Button
size="xs"
variant="secondary"
icon="Key"
onClick={() => copySnippet(activeClient.token, 'Client JWT token')}
>
Copy Token
</Button>
)}
<Button <Button
size="xs" size="xs"
variant="secondary" variant="secondary"
@@ -92,6 +105,7 @@ if client.connect():
> >
Copy Code Copy Code
</Button> </Button>
</>
} }
> >
<StatStrip <StatStrip
+4 -1
View File
@@ -95,7 +95,10 @@ export default function StoragePage() {
label: 'Persisted rows', label: 'Persisted rows',
value: stats?.totalRows ? stats.totalRows.toLocaleString() : 0, value: stats?.totalRows ? stats.totalRows.toLocaleString() : 0,
}, },
{ label: 'Rotation', value: '100MB / Daily' }, {
label: 'Rotation',
value: stats?.rotationRule || `${stats?.rotationMb ?? 100}MB / Daily`,
},
]} ]}
/> />
+9 -3
View File
@@ -14,9 +14,13 @@ function safeNextPath() {
: '/vcon/dashboard'; : '/vcon/dashboard';
} }
export function AdminLogin() { export function AdminLogin({
const [email, setEmail] = useState('admin@datastack.dev'); showDemoHint = false,
const [password, setPassword] = useState('admin12345678'); }: {
showDemoHint?: boolean;
}) {
const [email, setEmail] = useState(showDemoHint ? 'admin@datastack.dev' : '');
const [password, setPassword] = useState(showDemoHint ? 'admin12345678' : '');
const [remember, setRemember] = useState(true); const [remember, setRemember] = useState(true);
const [showPassword, setShowPassword] = useState(false); const [showPassword, setShowPassword] = useState(false);
const [isSubmitting, setIsSubmitting] = useState(false); const [isSubmitting, setIsSubmitting] = useState(false);
@@ -144,6 +148,7 @@ export function AdminLogin() {
Sign in Sign in
</Button> </Button>
{showDemoHint && (
<button <button
type="button" type="button"
onClick={() => { onClick={() => {
@@ -154,6 +159,7 @@ export function AdminLogin() {
> >
admin@datastack.dev / admin12345678 admin@datastack.dev / admin12345678
</button> </button>
)}
</form> </form>
<footer className="flex items-center justify-end border-t border-[var(--border)] px-4 py-2 text-[10.5px]"> <footer className="flex items-center justify-end border-t border-[var(--border)] px-4 py-2 text-[10.5px]">
+4 -4
View File
@@ -26,8 +26,8 @@ export function AdminSetup({
event.preventDefault(); event.preventDefault();
setErrorMessage(''); setErrorMessage('');
if (password.length < 12) { if (password.length < 8) {
setErrorMessage('Password must be at least 12 characters.'); setErrorMessage('Password must be at least 8 characters.');
return; return;
} }
if (password !== confirmPassword) { if (password !== confirmPassword) {
@@ -94,7 +94,7 @@ export function AdminSetup({
value={password} value={password}
onChange={(event) => setPassword(event.target.value)} onChange={(event) => setPassword(event.target.value)}
compact compact
minLength={12} minLength={8}
required required
/> />
<Input <Input
@@ -104,7 +104,7 @@ export function AdminSetup({
value={confirmPassword} value={confirmPassword}
onChange={(event) => setConfirmPassword(event.target.value)} onChange={(event) => setConfirmPassword(event.target.value)}
compact compact
minLength={12} minLength={8}
required required
/> />
<Input <Input
+48 -6
View File
@@ -40,7 +40,7 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
const { success, error } = useToast(); const { success, error } = useToast();
const [credentials, setCredentials] = useState<ClientCredentials | null>(null); const [credentials, setCredentials] = useState<ClientCredentials | null>(null);
const [confirmAction, setConfirmAction] = useState<{ const [confirmAction, setConfirmAction] = useState<{
type: 'rotate' | 'revoke'; type: 'rotate' | 'revoke' | 'delete';
client: ClientItem; client: ClientItem;
} | null>(null); } | null>(null);
const [processing, setProcessing] = useState(false); const [processing, setProcessing] = useState(false);
@@ -86,6 +86,24 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
} }
}; };
const executeDeleteClient = async (client: ClientItem) => {
setProcessing(true);
try {
const response = await fetch(`/api/clients/${encodeURIComponent(client.clientId)}`, {
method: 'DELETE',
});
const data = await response.json();
if (!response.ok) throw new Error(data.error || 'Unable to delete client');
setConfirmAction(null);
success('Client deleted', `${client.clientId} has been permanently removed.`);
await onStatusChange();
} catch (deleteError) {
error('Client deletion failed', deleteError instanceof Error ? deleteError.message : 'Unknown error');
} finally {
setProcessing(false);
}
};
const toggleStatus = (client: ClientItem) => { const toggleStatus = (client: ClientItem) => {
if (client.status === 'active') { if (client.status === 'active') {
setConfirmAction({ type: 'revoke', client }); setConfirmAction({ type: 'revoke', client });
@@ -98,6 +116,8 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
if (!confirmAction) return; if (!confirmAction) return;
if (confirmAction.type === 'rotate') { if (confirmAction.type === 'rotate') {
await executeRotateKey(confirmAction.client); await executeRotateKey(confirmAction.client);
} else if (confirmAction.type === 'delete') {
await executeDeleteClient(confirmAction.client);
} else { } else {
await executeStatusChange(confirmAction.client, 'revoked'); await executeStatusChange(confirmAction.client, 'revoked');
} }
@@ -178,12 +198,22 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
type="button" type="button"
variant="icon" variant="icon"
size="icon" size="icon"
icon={client.status === 'active' ? 'Trash2' : 'ShieldCheck'} icon={client.status === 'active' ? 'ShieldOff' : 'ShieldCheck'}
className={`h-6 w-6 border border-transparent ${client.status === 'active' ? 'hover:border-[var(--danger)] hover:text-[var(--danger)]' : 'hover:border-[var(--border)] hover:bg-[var(--surface-2)]'}`} className={`h-6 w-6 border border-transparent ${client.status === 'active' ? 'hover:border-[var(--warning)] hover:text-[var(--warning)]' : 'hover:border-[var(--border)] hover:bg-[var(--surface-2)]'}`}
title={`${client.status === 'active' ? 'Revoke' : 'Reactivate'} ${client.clientId}`} title={`${client.status === 'active' ? 'Revoke' : 'Reactivate'} ${client.clientId}`}
aria-label={`${client.status === 'active' ? 'Revoke' : 'Reactivate'} ${client.clientId}`} aria-label={`${client.status === 'active' ? 'Revoke' : 'Reactivate'} ${client.clientId}`}
onClick={() => toggleStatus(client)} onClick={() => toggleStatus(client)}
/> />
<Button
type="button"
variant="icon"
size="icon"
icon="Trash2"
className="h-6 w-6 border border-transparent hover:border-[var(--danger)] hover:text-[var(--danger)]"
title={`Delete ${client.clientId}`}
aria-label={`Delete ${client.clientId}`}
onClick={() => setConfirmAction({ type: 'delete', client })}
/>
</div> </div>
</TableCell> </TableCell>
</TableRow> </TableRow>
@@ -197,7 +227,13 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
<Modal <Modal
isOpen={Boolean(confirmAction)} isOpen={Boolean(confirmAction)}
onClose={() => !processing && setConfirmAction(null)} onClose={() => !processing && setConfirmAction(null)}
title={confirmAction?.type === 'rotate' ? 'Rotate credentials' : 'Revoke access'} title={
confirmAction?.type === 'rotate'
? 'Rotate credentials'
: confirmAction?.type === 'delete'
? 'Delete client'
: 'Revoke access'
}
maxWidth="xs" maxWidth="xs"
footer={ footer={
<> <>
@@ -206,12 +242,16 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
</Button> </Button>
<Button <Button
type="button" type="button"
variant={confirmAction?.type === 'revoke' ? 'danger' : 'primary'} variant={confirmAction?.type === 'rotate' ? 'primary' : 'danger'}
size="xs" size="xs"
loading={processing} loading={processing}
onClick={handleConfirm} onClick={handleConfirm}
> >
{confirmAction?.type === 'rotate' ? 'Rotate' : 'Revoke'} {confirmAction?.type === 'rotate'
? 'Rotate'
: confirmAction?.type === 'delete'
? 'Delete'
: 'Revoke'}
</Button> </Button>
</> </>
} }
@@ -219,6 +259,8 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
<p className="text-[12px] text-[var(--text-muted)]"> <p className="text-[12px] text-[var(--text-muted)]">
{confirmAction?.type === 'rotate' {confirmAction?.type === 'rotate'
? `Rotate credentials for ${confirmAction.client.clientId}? Existing credentials will stop working.` ? `Rotate credentials for ${confirmAction.client.clientId}? Existing credentials will stop working.`
: confirmAction?.type === 'delete'
? `Permanently delete client ${confirmAction.client.clientId}? This removes the client registration and credentials.`
: `Revoke access for ${confirmAction?.client.clientId || ''}?`} : `Revoke access for ${confirmAction?.client.clientId || ''}?`}
</p> </p>
</Modal> </Modal>
+3 -3
View File
@@ -57,10 +57,10 @@ export function ExportWorkspace() {
getCached<StorageFile[]>('vcon:storage-files', []) getCached<StorageFile[]>('vcon:storage-files', [])
); );
const [selectedClient, setSelectedClient] = useState('all'); const [selectedClient, setSelectedClient] = useState('all');
const [fromDate, setFromDate] = useState(() => dateString(new Date())); const [fromDate, setFromDate] = useState('');
const [toDate, setToDate] = useState(() => dateString(new Date())); const [toDate, setToDate] = useState('');
const [format, setFormat] = useState<ExportFormat>('csv'); const [format, setFormat] = useState<ExportFormat>('csv');
const [preset, setPreset] = useState<RangePreset>('today'); const [preset, setPreset] = useState<RangePreset>('all');
const [loading, setLoading] = useState( const [loading, setLoading] = useState(
() => getCached<StorageFile[]>('vcon:storage-files', []).length === 0 () => getCached<StorageFile[]>('vcon:storage-files', []).length === 0
); );
+114 -9
View File
@@ -1,28 +1,133 @@
import { Pool } from "pg";
import { SettingsManager } from "../adminSettings/settingsManager"; import { SettingsManager } from "../adminSettings/settingsManager";
import { SDKManager } from "../sdk/sdkManager"; import { SDKManager } from "../sdk/sdkManager";
type CoreDatabaseGlobals = typeof globalThis & {
__dataStackCorePool?: Pool | null;
__dataStackCoreSchema?: Promise<void>;
};
const globals = globalThis as CoreDatabaseGlobals;
/** /**
* Database Module (PostgreSQL + Prisma abstraction) * Database Module (PostgreSQL Persistence Engine)
* CRITICAL RULE: PostgreSQL stores strictly administrative metadata, * CRITICAL RULE: PostgreSQL stores strictly administrative metadata,
* client registrations, and configuration. Collected data is NEVER stored in PostgreSQL. * client registrations (client_sdks), and configuration (admin_settings).
* Collected telemetry streams are NEVER stored in PostgreSQL.
*/ */
export class DatabaseEngine { export class DatabaseEngine {
public static isConnected = true; public static getPool(): Pool | null {
const connectionString = process.env.DATABASE_URL;
if (!connectionString) {
return null;
}
if (globals.__dataStackCorePool === undefined) {
try {
globals.__dataStackCorePool = new Pool({
connectionString,
connectionTimeoutMillis: 3000,
idleTimeoutMillis: 30000,
max: 5,
});
} catch {
globals.__dataStackCorePool = null;
}
}
return globals.__dataStackCorePool ?? null;
}
public static async ensureSchema(): Promise<boolean> {
const pool = this.getPool();
if (!pool) return false;
globals.__dataStackCoreSchema ??= Promise.all([
pool.query(`
CREATE TABLE IF NOT EXISTS client_sdks (
client_id TEXT PRIMARY KEY,
name TEXT NOT NULL,
secret TEXT NOT NULL,
token TEXT NOT NULL,
rate_limit INTEGER NOT NULL DEFAULT 100,
status TEXT NOT NULL DEFAULT 'active',
total_events BIGINT NOT NULL DEFAULT 0,
last_active TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`),
pool.query(`
CREATE TABLE IF NOT EXISTS admin_settings (
id TEXT PRIMARY KEY DEFAULT 'default',
server_port INTEGER NOT NULL DEFAULT 4000,
web_port INTEGER NOT NULL DEFAULT 3000,
max_events_per_sec INTEGER NOT NULL DEFAULT 100,
max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
file_rotation_mb INTEGER NOT NULL DEFAULT 100,
jwt_secret TEXT NOT NULL,
enable_idempotency BOOLEAN NOT NULL DEFAULT true,
auto_prune_days INTEGER NOT NULL DEFAULT 30,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`),
]).then(() => undefined);
try {
await globals.__dataStackCoreSchema;
return true;
} catch {
globals.__dataStackCoreSchema = undefined;
return false;
}
}
public static async getHealth() { public static async getHealth() {
const pool = this.getPool();
if (!pool) {
return { return {
status: "healthy", status: "healthy" as const,
type: "PostgreSQL + Prisma Schema Managed", storageMode: "In-Memory" as const,
database: "datastack_db", type: "In-Memory Fallback (DATABASE_URL not set)",
database: "memory",
role: "Metadata & SDK Registrations Only", role: "Metadata & SDK Registrations Only",
collectedDataStorage: "Strictly CSV (/storage/data)", collectedDataStorage: "Strictly CSV (/storage/data)",
}; };
} }
public static async syncState() { try {
await this.ensureSchema();
await pool.query("SELECT 1");
return { return {
clientsCount: SDKManager.listClients().length, status: "healthy" as const,
settings: SettingsManager.getPublicSettings(), storageMode: "PostgreSQL" as const,
type: "PostgreSQL Managed Schema",
database: "datastack_db",
role: "Metadata & SDK Registrations Only",
collectedDataStorage: "Strictly CSV (/storage/data)",
};
} catch {
return {
status: "degraded" as const,
storageMode: "In-Memory" as const,
type: "PostgreSQL Unreachable (In-Memory Fallback Active)",
database: "datastack_db",
role: "Metadata & SDK Registrations Only",
collectedDataStorage: "Strictly CSV (/storage/data)",
};
}
}
public static async syncState() {
const [clients, settings] = await Promise.all([
SDKManager.listClientsAsync(),
SettingsManager.getPublicSettingsAsync(),
]);
return {
clientsCount: clients.length,
settings,
}; };
} }
} }
-100
View File
@@ -1,100 +0,0 @@
import express from 'express';
import http from 'http';
import cors from 'cors';
import { Server as SocketIOServer } from 'socket.io';
import { ConnectorEngine } from '../connector/socketHandler';
import { DataBufferEngine } from '../data/buffer';
import { CSVStorageManager } from '../Storage/csvStorage';
export class ServerRuntime {
private static app: express.Express | null = null;
private static httpServer: http.Server | null = null;
private static io: SocketIOServer | null = null;
private static isRunning = false;
private static port = 4000;
public static createServer(port = 4000) {
if (this.isRunning && this.httpServer) {
return { app: this.app, server: this.httpServer, io: this.io };
}
this.port = port;
const app = express();
app.use(cors({ origin: '*' }));
app.use(express.json({ limit: '10mb' }));
// Health route
app.get('/health', (_req, res) => {
res.json({
status: 'online',
service: 'DataStack Express + Socket.IO Intake',
port: this.port,
activeSockets: ConnectorEngine.getActiveConnectionsCount(),
uptime: process.uptime(),
});
});
// HTTP fallback intake endpoint for non-socket clients
app.post('/api/intake', (req, res) => {
const authHeader = req.headers.authorization;
if (!authHeader) {
res.status(401).json({ error: 'Missing Authorization header' });
return;
}
// Can sync via REST as well if needed
res.json({ success: true, message: 'REST sync supported' });
});
const httpServer = http.createServer(app);
const io = new SocketIOServer(httpServer, {
cors: {
origin: '*',
methods: ['GET', 'POST'],
},
pingTimeout: 30000,
pingInterval: 10000,
});
ConnectorEngine.initialize(io);
this.app = app;
this.httpServer = httpServer;
this.io = io;
this.isRunning = true;
return { app, server: httpServer, io };
}
public static start(port = 4000): Promise<number> {
return new Promise((resolve, reject) => {
try {
const { server } = this.createServer(port);
server.listen(port, () => {
resolve(port);
});
server.on('error', (err: NodeJS.ErrnoException) => {
if (err.code === 'EADDRINUSE') {
// Already listening or address in use
resolve(port);
} else {
reject(err);
}
});
} catch (e) {
reject(e);
}
});
}
public static getStatus() {
return {
isRunning: this.isRunning,
port: this.port,
activeSockets: ConnectorEngine.getActiveConnectionsCount(),
bufferSize: DataBufferEngine.getBufferStats().currentBufferSize,
storageStats: CSVStorageManager.getStorageStats(),
};
}
}
export default ServerRuntime;
+38 -1
View File
@@ -167,6 +167,38 @@ export class CSVStorageManager {
return results.sort((a, b) => b.modifiedAt.getTime() - a.modifiedAt.getTime()); return results.sort((a, b) => b.modifiedAt.getTime() - a.modifiedAt.getTime());
} }
/**
* Prune CSV partitions older than autoPruneDays
*/
public static pruneExpiredPartitions(autoPruneDays: number): number {
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
return 0;
}
const cutoffDate = new Date(
Date.now() - Math.floor(autoPruneDays) * 86_400_000,
)
.toISOString()
.split('T')[0];
let prunedCount = 0;
const files = this.listAllFiles();
for (const file of files) {
if (/^\d{4}-\d{2}-\d{2}$/.test(file.date) && file.date < cutoffDate) {
try {
if (fs.existsSync(file.filePath)) {
fs.unlinkSync(file.filePath);
prunedCount += 1;
}
} catch {
// Ignore file removal error
}
}
}
return prunedCount;
}
/** /**
* Get aggregate storage statistics * Get aggregate storage statistics
*/ */
@@ -175,6 +207,10 @@ export class CSVStorageManager {
const totalBytes = files.reduce((acc, f) => acc + f.sizeBytes, 0); const totalBytes = files.reduce((acc, f) => acc + f.sizeBytes, 0);
const totalRows = files.reduce((acc, f) => acc + f.linesCount, 0); const totalRows = files.reduce((acc, f) => acc + f.linesCount, 0);
const uniqueClients = new Set(files.map((f) => f.clientId)).size; const uniqueClients = new Set(files.map((f) => f.clientId)).size;
const rotationMb = Math.max(
1,
Math.round(this.maxFileSizeBytes / (1024 * 1024)),
);
return { return {
totalFiles: files.length, totalFiles: files.length,
@@ -183,7 +219,8 @@ export class CSVStorageManager {
totalRows, totalRows,
uniqueClients, uniqueClients,
storageDirectory: this.baseStorageDir, storageDirectory: this.baseStorageDir,
rotationRule: 'Per-day OR per-100MB rotation', rotationMb,
rotationRule: `${rotationMb}MB / Daily`,
}; };
} }
+191 -30
View File
@@ -1,5 +1,7 @@
import { AuthService } from "../auth/jwt"; import { AuthService } from "../auth/jwt";
import { DataBufferEngine } from "../data/buffer"; import { DataBufferEngine } from "../data/buffer";
import { DatabaseEngine } from "../Database/database";
import { SDKManager } from "../sdk/sdkManager";
import { SecurityEngine } from "../Security/security"; import { SecurityEngine } from "../Security/security";
import { CSVStorageManager } from "../Storage/csvStorage"; import { CSVStorageManager } from "../Storage/csvStorage";
@@ -17,10 +19,17 @@ export interface AdminSettings {
updatedAt: string; updatedAt: string;
} }
export class SettingsManager { type SettingsGlobals = typeof globalThis & {
private static settings: AdminSettings = { __dataStackAdminSettings?: AdminSettings;
serverPort: 4000, __dataStackSettingsHydrated?: boolean;
webPort: 3000, };
const globals = globalThis as SettingsGlobals;
function getDefaultSettings(): AdminSettings {
return {
serverPort: Number(process.env.PORT_SOCKET) || 4000,
webPort: Number(process.env.PORT) || 3000,
maxEventsPerSec: 100, maxEventsPerSec: 100,
maxPayloadSizeKb: 100, maxPayloadSizeKb: 100,
batchFlushThreshold: 500, batchFlushThreshold: 500,
@@ -33,13 +42,160 @@ export class SettingsManager {
autoPruneDays: 30, autoPruneDays: 30,
updatedAt: new Date().toISOString(), updatedAt: new Date().toISOString(),
}; };
}
function getSettingsState(): AdminSettings {
if (!globals.__dataStackAdminSettings) {
globals.__dataStackAdminSettings = getDefaultSettings();
}
return globals.__dataStackAdminSettings;
}
export class SettingsManager {
private static applyRuntimeSettings(settings: AdminSettings) {
SecurityEngine.maxEventsPerSec = settings.maxEventsPerSec;
SecurityEngine.enableIdempotency = settings.enableIdempotency;
SecurityEngine.maxPayloadSizeBytes = settings.maxPayloadSizeKb * 1024;
DataBufferEngine.batchFlushThreshold = settings.batchFlushThreshold;
DataBufferEngine.setFlushInterval(settings.batchFlushIntervalMs);
if (settings.jwtSecret) {
AuthService.setSecret(settings.jwtSecret);
}
CSVStorageManager.maxFileSizeBytes = settings.fileRotationMb * 1024 * 1024;
CSVStorageManager.pruneExpiredPartitions(settings.autoPruneDays);
}
private static async syncFromDatabase(): Promise<AdminSettings | null> {
const pool = DatabaseEngine.getPool();
if (!pool) return null;
try {
const ready = await DatabaseEngine.ensureSchema();
if (!ready) return null;
const result = await pool.query(
`SELECT server_port, web_port, max_events_per_sec, max_payload_size_kb,
batch_flush_threshold, batch_flush_interval_ms, file_rotation_mb,
jwt_secret, enable_idempotency, auto_prune_days, updated_at
FROM admin_settings
WHERE id = 'default'`,
);
if (result.rowCount && result.rowCount > 0) {
const row = result.rows[0];
const loaded: AdminSettings = {
serverPort: Number(row.server_port) || 4000,
webPort: Number(row.web_port) || 3000,
maxEventsPerSec: Number(row.max_events_per_sec) || 100,
maxPayloadSizeKb: Number(row.max_payload_size_kb) || 100,
batchFlushThreshold: Number(row.batch_flush_threshold) || 500,
batchFlushIntervalMs: Number(row.batch_flush_interval_ms) || 2000,
fileRotationMb: Number(row.file_rotation_mb) || 100,
jwtSecret:
String(row.jwt_secret || "") ||
process.env.DATASTACK_JWT_SECRET ||
"datastack-jwt-default-secret-key-32-chars-long",
enableIdempotency: Boolean(row.enable_idempotency),
autoPruneDays: Number(row.auto_prune_days) || 30,
updatedAt: row.updated_at
? new Date(row.updated_at as string | Date).toISOString()
: new Date().toISOString(),
};
globals.__dataStackAdminSettings = loaded;
globals.__dataStackSettingsHydrated = true;
this.applyRuntimeSettings(loaded);
return loaded;
}
// Initialize default row in PostgreSQL if not present yet
const defaults = getSettingsState();
await this.persistSettingsToDb(defaults);
globals.__dataStackSettingsHydrated = true;
return defaults;
} catch {
return null;
}
}
private static async persistSettingsToDb(
settings: AdminSettings,
): Promise<void> {
const pool = DatabaseEngine.getPool();
if (!pool) return;
try {
await DatabaseEngine.ensureSchema();
await pool.query(
`INSERT INTO admin_settings (
id, server_port, web_port, max_events_per_sec, max_payload_size_kb,
batch_flush_threshold, batch_flush_interval_ms, file_rotation_mb,
jwt_secret, enable_idempotency, auto_prune_days, updated_at
) VALUES ('default', $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, NOW())
ON CONFLICT (id) DO UPDATE SET
server_port = EXCLUDED.server_port,
web_port = EXCLUDED.web_port,
max_events_per_sec = EXCLUDED.max_events_per_sec,
max_payload_size_kb = EXCLUDED.max_payload_size_kb,
batch_flush_threshold = EXCLUDED.batch_flush_threshold,
batch_flush_interval_ms = EXCLUDED.batch_flush_interval_ms,
file_rotation_mb = EXCLUDED.file_rotation_mb,
jwt_secret = EXCLUDED.jwt_secret,
enable_idempotency = EXCLUDED.enable_idempotency,
auto_prune_days = EXCLUDED.auto_prune_days,
updated_at = NOW()`,
[
settings.serverPort,
settings.webPort,
settings.maxEventsPerSec,
settings.maxPayloadSizeKb,
settings.batchFlushThreshold,
settings.batchFlushIntervalMs,
settings.fileRotationMb,
settings.jwtSecret,
settings.enableIdempotency,
settings.autoPruneDays,
],
);
} catch {
// Ignore DB failure; in-memory state remains active
}
}
public static getSettings(): AdminSettings { public static getSettings(): AdminSettings {
return { ...this.settings }; if (!globals.__dataStackSettingsHydrated) {
void this.syncFromDatabase();
}
return { ...getSettingsState() };
}
public static async getSettingsAsync(): Promise<AdminSettings> {
const fromDb = await this.syncFromDatabase();
if (fromDb) {
return { ...fromDb };
}
return { ...getSettingsState() };
} }
public static getPublicSettings(): Omit<AdminSettings, "jwtSecret"> { public static getPublicSettings(): Omit<AdminSettings, "jwtSecret"> {
const settings = this.settings; const settings = this.getSettings();
return {
serverPort: settings.serverPort,
webPort: settings.webPort,
maxEventsPerSec: settings.maxEventsPerSec,
maxPayloadSizeKb: settings.maxPayloadSizeKb,
batchFlushThreshold: settings.batchFlushThreshold,
batchFlushIntervalMs: settings.batchFlushIntervalMs,
fileRotationMb: settings.fileRotationMb,
enableIdempotency: settings.enableIdempotency,
autoPruneDays: settings.autoPruneDays,
updatedAt: settings.updatedAt,
};
}
public static async getPublicSettingsAsync(): Promise<
Omit<AdminSettings, "jwtSecret">
> {
const settings = await this.getSettingsAsync();
return { return {
serverPort: settings.serverPort, serverPort: settings.serverPort,
webPort: settings.webPort, webPort: settings.webPort,
@@ -55,36 +211,41 @@ export class SettingsManager {
} }
public static updateSettings(updates: Partial<AdminSettings>): AdminSettings { public static updateSettings(updates: Partial<AdminSettings>): AdminSettings {
this.settings = { const current = getSettingsState();
...this.settings, const jwtSecretChanged =
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
const nextSettings: AdminSettings = {
...current,
...updates, ...updates,
updatedAt: new Date().toISOString(), updatedAt: new Date().toISOString(),
}; };
globals.__dataStackAdminSettings = nextSettings;
// Apply live updates to subsystems this.applyRuntimeSettings(nextSettings);
if (updates.maxEventsPerSec !== undefined) { if (jwtSecretChanged) {
SecurityEngine.maxEventsPerSec = updates.maxEventsPerSec; SDKManager.resignAllClientTokens();
} }
if (updates.enableIdempotency !== undefined) { void this.persistSettingsToDb(nextSettings);
SecurityEngine.enableIdempotency = updates.enableIdempotency; return { ...nextSettings };
}
if (updates.maxPayloadSizeKb !== undefined) {
SecurityEngine.maxPayloadSizeBytes = updates.maxPayloadSizeKb * 1024;
}
if (updates.batchFlushThreshold !== undefined) {
DataBufferEngine.batchFlushThreshold = updates.batchFlushThreshold;
}
if (updates.batchFlushIntervalMs !== undefined) {
DataBufferEngine.batchFlushIntervalMs = updates.batchFlushIntervalMs;
}
if (updates.jwtSecret !== undefined) {
AuthService.setSecret(updates.jwtSecret);
}
if (updates.fileRotationMb !== undefined) {
CSVStorageManager.maxFileSizeBytes = updates.fileRotationMb * 1024 * 1024;
} }
return this.settings; public static async updateSettingsAsync(
updates: Partial<AdminSettings>,
): Promise<AdminSettings> {
const current = await this.getSettingsAsync();
const jwtSecretChanged =
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
const nextSettings: AdminSettings = {
...current,
...updates,
updatedAt: new Date().toISOString(),
};
globals.__dataStackAdminSettings = nextSettings;
this.applyRuntimeSettings(nextSettings);
if (jwtSecretChanged) {
await SDKManager.resignAllClientTokensAsync();
}
await this.persistSettingsToDb(nextSettings);
return { ...nextSettings };
} }
} }
+19 -6
View File
@@ -1,7 +1,6 @@
import { SDKManager } from '../sdk/sdkManager'; import { SDKManager } from '../sdk/sdkManager';
import { CSVStorageManager } from '../Storage/csvStorage'; import { CSVStorageManager } from '../Storage/csvStorage';
import { DataBufferEngine } from '../data/buffer'; import { DataBufferEngine } from '../data/buffer';
import { SettingsManager } from '../adminSettings/settingsManager';
import { SecurityEngine } from '../Security/security'; import { SecurityEngine } from '../Security/security';
export class ApiEngine { export class ApiEngine {
@@ -10,23 +9,37 @@ export class ApiEngine {
*/ */
public static async getDashboardMetrics() { public static async getDashboardMetrics() {
const storageStats = CSVStorageManager.getStorageStats(); const storageStats = CSVStorageManager.getStorageStats();
const bufferStats = DataBufferEngine.getBufferStats(); const localBufferStats = DataBufferEngine.getBufferStats();
const clients = SDKManager.listClients(); const clients = await SDKManager.listClientsAsync();
const activeClients = clients.filter((c) => c.status === 'active').length; const activeClients = clients.filter((c) => c.status === 'active').length;
let activeSockets = 0; let activeSockets = 0;
let ingestionBufferSize = 0;
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
).replace(/\/$/, '');
try { try {
const res = await fetch('http://localhost:4000/health', { cache: 'no-store' }); const res = await fetch(`${ingestionBaseUrl}/health`, {
cache: 'no-store',
});
if (res.ok) { if (res.ok) {
const data = await res.json(); const data = await res.json();
activeSockets = data.activeSockets ?? 0; activeSockets = Number(data.activeSockets) || 0;
ingestionBufferSize = Number(data.bufferSize) || 0;
} }
} catch { } catch {
// Server may be starting // Ingestion server may not be running in single-process dev mode
activeSockets = 0; activeSockets = 0;
ingestionBufferSize = 0;
} }
const securityStats = SecurityEngine.getStats(); const securityStats = SecurityEngine.getStats();
const bufferStats = {
...localBufferStats,
currentBufferSize:
localBufferStats.currentBufferSize + ingestionBufferSize,
};
return { return {
activeSockets, activeSockets,
-162
View File
@@ -1,162 +0,0 @@
import { Server as SocketIOServer, Socket } from 'socket.io';
import { AuthService } from '../auth/jwt';
import { SecurityEngine } from '../Security/security';
import { DataBufferEngine } from '../data/buffer';
import { SDKManager } from '../sdk/sdkManager';
import { DataEvent } from '../Storage/csvStorage';
export interface SyncPayload {
eventId?: string;
timestamp?: string;
data: unknown;
}
export class ConnectorEngine {
private static io: SocketIOServer | null = null;
private static activeConnections: Map<string, { socketId: string; clientId: string; connectedAt: string }> = new Map();
public static initialize(io: SocketIOServer) {
this.io = io;
// Authentication middleware
io.use((socket: Socket, next) => {
const authHeader =
socket.handshake.auth?.token ||
socket.handshake.headers?.authorization ||
(socket.handshake.query?.token as string);
if (!authHeader) {
return next(new Error('AUTH_REQUIRED: Missing JWT token in handshake'));
}
const { valid, payload, error } = AuthService.verifyClientToken(authHeader);
if (!valid || !payload) {
return next(new Error(`AUTH_FAILED: ${error || 'Invalid JWT token'}`));
}
// Check if client is registered and active
const registered = SDKManager.getClient(payload.clientId);
if (registered && registered.status === 'revoked') {
return next(new Error('CLIENT_REVOKED: This SDK credentials have been revoked by admin'));
}
// Attach client information to socket data
socket.data.clientId = payload.clientId;
socket.data.rateLimit = registered?.rateLimit || payload.rateLimit || 100;
next();
});
// Connection handler
io.on('connection', (socket: Socket) => {
const clientId = socket.data.clientId as string;
const rateLimit = socket.data.rateLimit as number;
this.activeConnections.set(socket.id, {
socketId: socket.id,
clientId,
connectedAt: new Date().toISOString(),
});
// Join room for this client
socket.join(`client:${clientId}`);
// Single Event Sync: "sync_data"
socket.on('sync_data', (payload: SyncPayload, callback?: (res: { success: boolean; eventId?: string; error?: string }) => void) => {
try {
const eventId = payload.eventId || `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
const timestamp = payload.timestamp || new Date().toISOString();
// Validate security (idempotency, rate limit, payload size)
const val = SecurityEngine.validateEvent(clientId, eventId, payload.data, rateLimit);
if (!val.allowed) {
if (val.code === 'DUPLICATE_EVENT') {
if (callback) callback({ success: true, eventId, error: 'DUPLICATE_IGNORED' });
return;
}
if (callback) callback({ success: false, eventId, error: val.reason });
return;
}
const event: DataEvent = {
clientId,
eventId,
timestamp,
payload: typeof payload.data === 'string' ? payload.data : JSON.stringify(payload.data),
};
DataBufferEngine.push(event);
SDKManager.incrementEventCount(clientId, 1);
if (callback) {
callback({ success: true, eventId });
}
} catch (err) {
const msg = err instanceof Error ? err.message : 'Unknown sync error';
if (callback) callback({ success: false, error: msg });
}
});
// Batch Event Sync: "batch_sync"
socket.on('batch_sync', (batch: SyncPayload[], callback?: (res: { success: boolean; processed: number; rejected: number; errors?: string[] }) => void) => {
try {
if (!Array.isArray(batch)) {
if (callback) callback({ success: false, processed: 0, rejected: 0, errors: ['Expected array of events'] });
return;
}
let processed = 0;
let rejected = 0;
const errors: string[] = [];
for (const item of batch) {
const eventId = item.eventId || `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
const timestamp = item.timestamp || new Date().toISOString();
const val = SecurityEngine.validateEvent(clientId, eventId, item.data, rateLimit);
if (!val.allowed) {
rejected += 1;
if (val.code !== 'DUPLICATE_EVENT') {
errors.push(`${eventId}: ${val.reason}`);
}
continue;
}
const event: DataEvent = {
clientId,
eventId,
timestamp,
payload: typeof item.data === 'string' ? item.data : JSON.stringify(item.data),
};
DataBufferEngine.push(event);
processed += 1;
}
SDKManager.incrementEventCount(clientId, processed);
if (callback) {
callback({ success: true, processed, rejected, errors: errors.length > 0 ? errors.slice(0, 5) : undefined });
}
} catch (err) {
const msg = err instanceof Error ? err.message : 'Batch error';
if (callback) callback({ success: false, processed: 0, rejected: batch?.length || 0, errors: [msg] });
}
});
// Disconnect
socket.on('disconnect', () => {
this.activeConnections.delete(socket.id);
});
});
}
public static getActiveConnections() {
return Array.from(this.activeConnections.values());
}
public static getActiveConnectionsCount(): number {
return this.activeConnections.size;
}
}
export default ConnectorEngine;
+110 -80
View File
@@ -3,93 +3,116 @@ import { CSVStorageManager, DataEvent } from '../Storage/csvStorage';
export type EventSubscriber = (event: DataEvent) => void; export type EventSubscriber = (event: DataEvent) => void;
export type FlushCallback = (count: number, files: string[]) => void; export type FlushCallback = (count: number, files: string[]) => void;
interface BufferGlobalState {
buffer: DataEvent[];
recentEvents: DataEvent[];
batchFlushThreshold: number;
batchFlushIntervalMs: number;
flushTimer: NodeJS.Timeout | null;
subscribers: Set<EventSubscriber>;
flushListeners: Set<FlushCallback>;
totalIngested: number;
totalFlushed: number;
isFlushing: boolean;
}
type BufferGlobals = typeof globalThis & {
__dataStackBufferState?: BufferGlobalState;
};
const globals = globalThis as BufferGlobals;
function getState(): BufferGlobalState {
if (!globals.__dataStackBufferState) {
globals.__dataStackBufferState = {
buffer: [],
recentEvents: [],
batchFlushThreshold: 500,
batchFlushIntervalMs: 2000,
flushTimer: null,
subscribers: new Set(),
flushListeners: new Set(),
totalIngested: 0,
totalFlushed: 0,
isFlushing: false,
};
}
return globals.__dataStackBufferState;
}
export class DataBufferEngine { export class DataBufferEngine {
private static buffer: DataEvent[] = [];
private static recentEvents: DataEvent[] = [];
private static maxRecentEvents = 200; private static maxRecentEvents = 200;
// Thresholds public static get batchFlushThreshold(): number {
public static batchFlushThreshold = 500; // rows return getState().batchFlushThreshold;
public static batchFlushIntervalMs = 2000; // 2 seconds }
private static flushTimer: NodeJS.Timeout | null = null; public static set batchFlushThreshold(value: number) {
private static subscribers: Set<EventSubscriber> = new Set(); getState().batchFlushThreshold = Math.max(1, Number(value) || 500);
private static flushListeners: Set<FlushCallback> = new Set(); }
private static totalIngested = 5; public static get batchFlushIntervalMs(): number {
private static totalFlushed = 1420; return getState().batchFlushIntervalMs;
private static isFlushing = false; }
static { public static set batchFlushIntervalMs(value: number) {
const now = Date.now(); this.setFlushInterval(value);
DataBufferEngine.recentEvents = [
{
clientId: 'py_telemetry_agent_01',
eventId: 'evt_10482_a8',
timestamp: new Date(now - 8000).toISOString(),
payload: JSON.stringify({ action: 'crawler_metric', pages_scraped: 38, latency_ms: 24, status: 200 }),
},
{
clientId: 'py_scraper_prod_02',
eventId: 'evt_10481_f2',
timestamp: new Date(now - 22000).toISOString(),
payload: JSON.stringify({ spider: 'market_catalog', items: 114, parse_time_s: 1.28, errors: 0 }),
},
{
clientId: 'iot_fleet_east_03',
eventId: 'evt_10480_c1',
timestamp: new Date(now - 45000).toISOString(),
payload: JSON.stringify({ sensor_node: 'gw_alpha_09', temp_c: 24.3, humidity_pct: 58.1, battery_v: 3.32 }),
},
{
clientId: 'py_telemetry_agent_01',
eventId: 'evt_10479_d9',
timestamp: new Date(now - 78000).toISOString(),
payload: JSON.stringify({ action: 'heartbeat', memory_usage_mb: 284, cpu_load_pct: 12.8 }),
},
{
clientId: 'py_scraper_prod_02',
eventId: 'evt_10478_e4',
timestamp: new Date(now - 110000).toISOString(),
payload: JSON.stringify({ action: 'batch_sync', batch_size: 50, duration_ms: 82 }),
},
];
} }
public static init() { public static init() {
if (!this.flushTimer) { const state = getState();
this.flushTimer = setInterval(() => { if (!state.flushTimer) {
this.flush(); state.flushTimer = setInterval(() => {
}, this.batchFlushIntervalMs); void this.flush();
}, state.batchFlushIntervalMs);
} }
} }
/**
* Updates flush interval and immediately restarts the active interval timer.
*/
public static setFlushInterval(intervalMs: number) {
const state = getState();
const nextInterval = Math.max(100, Number(intervalMs) || 2000);
state.batchFlushIntervalMs = nextInterval;
if (state.flushTimer) {
clearInterval(state.flushTimer);
state.flushTimer = null;
}
state.flushTimer = setInterval(() => {
void this.flush();
}, state.batchFlushIntervalMs);
}
/** /**
* Push incoming event to in-memory buffer. * Push incoming event to in-memory buffer.
* If buffer hits threshold (e.g. 500 rows), triggers immediate batch flush. * If buffer hits threshold (e.g. 500 rows), triggers immediate batch flush.
*/ */
public static push(event: DataEvent): void { public static push(event: DataEvent): void {
this.buffer.push(event); const state = getState();
this.totalIngested += 1; state.buffer.push(event);
state.totalIngested += 1;
// Cache in recent events for live dashboard viewer // Cache in recent events for live dashboard viewer
this.recentEvents.unshift(event); state.recentEvents.unshift(event);
if (this.recentEvents.length > this.maxRecentEvents) { if (state.recentEvents.length > this.maxRecentEvents) {
this.recentEvents.pop(); state.recentEvents.pop();
} }
// Notify live subscribers // Notify live subscribers
for (const sub of this.subscribers) { for (const sub of state.subscribers) {
try { try {
sub(event); sub(event);
} catch { } catch {
// Ignore subscriber err // Ignore subscriber error
} }
} }
// Check if buffer threshold reached // Check if buffer threshold reached
if (this.buffer.length >= this.batchFlushThreshold) { if (state.buffer.length >= state.batchFlushThreshold) {
this.flush(); void this.flush();
} }
} }
@@ -97,19 +120,20 @@ export class DataBufferEngine {
* Flushes current buffer to CSV files via CSVStorageManager * Flushes current buffer to CSV files via CSVStorageManager
*/ */
public static async flush(): Promise<{ count: number; files: string[] }> { public static async flush(): Promise<{ count: number; files: string[] }> {
if (this.buffer.length === 0 || this.isFlushing) { const state = getState();
if (state.buffer.length === 0 || state.isFlushing) {
return { count: 0, files: [] }; return { count: 0, files: [] };
} }
this.isFlushing = true; state.isFlushing = true;
const batchToFlush = [...this.buffer]; const batchToFlush = [...state.buffer];
this.buffer = []; state.buffer = [];
try { try {
const result = await CSVStorageManager.appendBatch(batchToFlush); const result = await CSVStorageManager.appendBatch(batchToFlush);
this.totalFlushed += result.written; state.totalFlushed += result.written;
for (const listener of this.flushListeners) { for (const listener of state.flushListeners) {
try { try {
listener(result.written, result.filesUpdated); listener(result.written, result.filesUpdated);
} catch { } catch {
@@ -120,10 +144,10 @@ export class DataBufferEngine {
return { count: result.written, files: result.filesUpdated }; return { count: result.written, files: result.filesUpdated };
} catch (err) { } catch (err) {
// On failure, re-queue to buffer front for retry // On failure, re-queue to buffer front for retry
this.buffer.unshift(...batchToFlush); state.buffer.unshift(...batchToFlush);
throw err; throw err;
} finally { } finally {
this.isFlushing = false; state.isFlushing = false;
} }
} }
@@ -131,36 +155,42 @@ export class DataBufferEngine {
* Subscribe to real-time events (for UI monitoring) * Subscribe to real-time events (for UI monitoring)
*/ */
public static subscribe(subscriber: EventSubscriber): () => void { public static subscribe(subscriber: EventSubscriber): () => void {
this.subscribers.add(subscriber); const state = getState();
state.subscribers.add(subscriber);
return () => { return () => {
this.subscribers.delete(subscriber); state.subscribers.delete(subscriber);
}; };
} }
public static onFlush(callback: FlushCallback): () => void { public static onFlush(callback: FlushCallback): () => void {
this.flushListeners.add(callback); const state = getState();
state.flushListeners.add(callback);
return () => { return () => {
this.flushListeners.delete(callback); state.flushListeners.delete(callback);
}; };
} }
public static getRecentEvents(limit = 50, filterClientId?: string): DataEvent[] { public static getRecentEvents(limit = 50, filterClientId?: string): DataEvent[] {
const state = getState();
if (filterClientId && filterClientId !== 'all') { if (filterClientId && filterClientId !== 'all') {
return this.recentEvents.filter((e) => e.clientId === filterClientId).slice(0, limit); return state.recentEvents
.filter((e) => e.clientId === filterClientId)
.slice(0, limit);
} }
return this.recentEvents.slice(0, limit); return state.recentEvents.slice(0, limit);
} }
public static getBufferStats() { public static getBufferStats() {
const state = getState();
return { return {
currentBufferSize: this.buffer.length, currentBufferSize: state.buffer.length,
batchFlushThreshold: this.batchFlushThreshold, batchFlushThreshold: state.batchFlushThreshold,
batchFlushIntervalMs: this.batchFlushIntervalMs, batchFlushIntervalMs: state.batchFlushIntervalMs,
totalIngested: this.totalIngested, totalIngested: state.totalIngested,
totalFlushed: this.totalFlushed, totalFlushed: state.totalFlushed,
recentEventsCached: this.recentEvents.length, recentEventsCached: state.recentEvents.length,
activeSubscribers: this.subscribers.size, activeSubscribers: state.subscribers.size,
isFlushing: this.isFlushing, isFlushing: state.isFlushing,
}; };
} }
} }
-61
View File
@@ -1,61 +0,0 @@
export interface SystemNotification {
id: string;
type: 'info' | 'success' | 'warning' | 'error';
title: string;
message: string;
source: 'socket' | 'storage' | 'security' | 'worker';
timestamp: string;
read: boolean;
}
export class NotificationEngine {
private static notifications: SystemNotification[] = [
{
id: 'notif-1',
type: 'info',
title: 'DataStack Engine Booted',
message: 'Ingestion server listening on port 4000. CSV buffer worker initialized.',
source: 'worker',
timestamp: new Date().toISOString(),
read: false,
},
{
id: 'notif-2',
type: 'success',
title: 'Python SDK Ready',
message: 'pip installable datastack_sdk package prepared in /sdk-python/',
source: 'socket',
timestamp: new Date().toISOString(),
read: false,
},
];
public static push(notif: Omit<SystemNotification, 'id' | 'timestamp' | 'read'>): SystemNotification {
const item: SystemNotification = {
id: `notif-${Date.now()}-${Math.random().toString(36).substring(2, 6)}`,
timestamp: new Date().toISOString(),
read: false,
...notif,
};
this.notifications.unshift(item);
if (this.notifications.length > 100) {
this.notifications.pop();
}
return item;
}
public static list(): SystemNotification[] {
return [...this.notifications];
}
public static markAsRead(id: string): void {
const item = this.notifications.find((n) => n.id === id);
if (item) item.read = true;
}
public static clearAll(): void {
this.notifications = [];
}
}
export default NotificationEngine;
+320 -55
View File
@@ -1,4 +1,5 @@
import { AuthService } from "../auth/jwt"; import { AuthService } from "../auth/jwt";
import { DatabaseEngine } from "../Database/database";
export interface ClientSDK { export interface ClientSDK {
clientId: string; clientId: string;
@@ -12,59 +13,114 @@ export interface ClientSDK {
createdAt: string; createdAt: string;
} }
type SDKDatabaseGlobals = typeof globalThis & {
__dataStackClientsMap?: Map<string, ClientSDK>;
__dataStackClientsHydrated?: boolean;
};
const globals = globalThis as SDKDatabaseGlobals;
function getClientsMap(): Map<string, ClientSDK> {
if (!globals.__dataStackClientsMap) {
globals.__dataStackClientsMap = new Map<string, ClientSDK>();
}
return globals.__dataStackClientsMap;
}
function mapRowToClient(row: Record<string, unknown>): ClientSDK {
return {
clientId: String(row.client_id),
name: String(row.name),
secret: String(row.secret),
token: String(row.token),
rateLimit: Number(row.rate_limit) || 100,
status: row.status === "revoked" ? "revoked" : "active",
totalEvents: Number(row.total_events) || 0,
lastActive: row.last_active
? new Date(row.last_active as string | Date).toISOString()
: undefined,
createdAt: row.created_at
? new Date(row.created_at as string | Date).toISOString()
: new Date().toISOString(),
};
}
export class SDKManager { export class SDKManager {
private static clients: Map<string, ClientSDK> = new Map(); private static async syncFromDatabase(): Promise<ClientSDK[] | null> {
const pool = DatabaseEngine.getPool();
if (!pool) return null;
// Initialize with production starter clients try {
static { const ready = await DatabaseEngine.ensureSchema();
const seedClients = [ if (!ready) return null;
{
clientId: "py_telemetry_agent_01",
name: "Python Telemetry Agent (US-East)",
rateLimit: 150,
},
{
clientId: "py_scraper_prod_02",
name: "Python Web Scraper Engine",
rateLimit: 200,
},
{
clientId: "iot_fleet_east_03",
name: "IoT Sensor Gateway Node",
rateLimit: 50,
},
];
for (const c of seedClients) { const result = await pool.query(
const secret = `sec_${c.clientId}_${Math.random().toString(36).substring(2, 10)}`; `SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
const token = AuthService.getSecret() FROM client_sdks
? AuthService.generateClientToken({ ORDER BY created_at DESC`,
clientId: c.clientId, );
name: c.name,
rateLimit: c.rateLimit,
})
: "";
SDKManager.clients.set(c.clientId, { const map = getClientsMap();
clientId: c.clientId, map.clear();
name: c.name, const list: ClientSDK[] = [];
secret,
token, for (const row of result.rows) {
rateLimit: c.rateLimit, const client = mapRowToClient(row);
status: "active", map.set(client.clientId, client);
totalEvents: 4280, list.push(client);
lastActive: new Date().toISOString(), }
createdAt: "2026-10-20T10:00:00.000Z",
}); globals.__dataStackClientsHydrated = true;
return list;
} catch {
return null;
} }
} }
public static listClients(): ClientSDK[] { public static listClients(): ClientSDK[] {
return Array.from(this.clients.values()); if (!globals.__dataStackClientsHydrated) {
void this.syncFromDatabase();
}
return Array.from(getClientsMap().values());
}
public static async listClientsAsync(): Promise<ClientSDK[]> {
const fromDb = await this.syncFromDatabase();
if (fromDb !== null) {
return fromDb;
}
return Array.from(getClientsMap().values());
} }
public static getClient(clientId: string): ClientSDK | undefined { public static getClient(clientId: string): ClientSDK | undefined {
return this.clients.get(clientId); return getClientsMap().get(clientId);
}
public static async getClientAsync(
clientId: string,
): Promise<ClientSDK | undefined> {
const pool = DatabaseEngine.getPool();
if (pool) {
try {
await DatabaseEngine.ensureSchema();
const result = await pool.query(
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
FROM client_sdks
WHERE client_id = $1`,
[clientId],
);
if (result.rowCount && result.rowCount > 0) {
const client = mapRowToClient(result.rows[0]);
getClientsMap().set(client.clientId, client);
return client;
}
getClientsMap().delete(clientId);
return undefined;
} catch {
// Fall back to in-memory map
}
}
return getClientsMap().get(clientId);
} }
public static registerClient(data: { public static registerClient(data: {
@@ -77,11 +133,11 @@ export class SDKManager {
.toLowerCase() .toLowerCase()
.replace(/[^a-z0-9_-]/g, "_"); .replace(/[^a-z0-9_-]/g, "_");
const secret = `sec_${safeClientId}_${Math.random().toString(36).substring(2, 10)}`; const secret = `sec_${safeClientId}_${Math.random().toString(36).substring(2, 10)}`;
const rateLimit = data.rateLimit || 100; const rateLimit = Math.max(1, Number(data.rateLimit) || 100);
const token = AuthService.generateClientToken({ const token = AuthService.generateClientToken({
clientId: safeClientId, clientId: safeClientId,
name: data.name, name: data.name.trim(),
rateLimit, rateLimit,
}); });
@@ -96,12 +152,79 @@ export class SDKManager {
createdAt: new Date().toISOString(), createdAt: new Date().toISOString(),
}; };
this.clients.set(safeClientId, newClient); getClientsMap().set(safeClientId, newClient);
void this.persistNewClient(newClient);
return newClient;
}
private static async persistNewClient(client: ClientSDK): Promise<void> {
const pool = DatabaseEngine.getPool();
if (!pool) return;
try {
await DatabaseEngine.ensureSchema();
await pool.query(
`INSERT INTO client_sdks (client_id, name, secret, token, rate_limit, status, total_events, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW())
ON CONFLICT (client_id) DO UPDATE SET
name = EXCLUDED.name,
secret = EXCLUDED.secret,
token = EXCLUDED.token,
rate_limit = EXCLUDED.rate_limit,
status = EXCLUDED.status,
updated_at = NOW()`,
[
client.clientId,
client.name,
client.secret,
client.token,
client.rateLimit,
client.status,
client.totalEvents,
client.createdAt,
],
);
} catch {
// In-memory map already updated
}
}
public static async registerClientAsync(data: {
clientId: string;
name: string;
rateLimit?: number;
}): Promise<ClientSDK> {
const safeClientId = data.clientId
.trim()
.toLowerCase()
.replace(/[^a-z0-9_-]/g, "_");
const secret = `sec_${safeClientId}_${Math.random().toString(36).substring(2, 10)}`;
const rateLimit = Math.max(1, Number(data.rateLimit) || 100);
const token = AuthService.generateClientToken({
clientId: safeClientId,
name: data.name.trim(),
rateLimit,
});
const newClient: ClientSDK = {
clientId: safeClientId,
name: data.name.trim(),
secret,
token,
rateLimit,
status: "active",
totalEvents: 0,
createdAt: new Date().toISOString(),
};
getClientsMap().set(safeClientId, newClient);
await this.persistNewClient(newClient);
return newClient; return newClient;
} }
public static regenerateToken(clientId: string): ClientSDK | null { public static regenerateToken(clientId: string): ClientSDK | null {
const client = this.clients.get(clientId); const client = getClientsMap().get(clientId);
if (!client) return null; if (!client) return null;
client.secret = `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`; client.secret = `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`;
@@ -111,35 +234,177 @@ export class SDKManager {
rateLimit: client.rateLimit, rateLimit: client.rateLimit,
}); });
this.clients.set(clientId, client); getClientsMap().set(clientId, client);
void this.persistClientCredentials(client);
return client; return client;
} }
private static async persistClientCredentials(
client: ClientSDK,
): Promise<void> {
const pool = DatabaseEngine.getPool();
if (!pool) return;
try {
await DatabaseEngine.ensureSchema();
await pool.query(
`UPDATE client_sdks
SET secret = $2, token = $3, updated_at = NOW()
WHERE client_id = $1`,
[client.clientId, client.secret, client.token],
);
} catch {
// Ignore DB failure
}
}
public static async regenerateTokenAsync(
clientId: string,
): Promise<ClientSDK | null> {
const existing = await this.getClientAsync(clientId);
if (!existing) return null;
existing.secret = `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`;
existing.token = AuthService.generateClientToken({
clientId: existing.clientId,
name: existing.name,
rateLimit: existing.rateLimit,
});
getClientsMap().set(clientId, existing);
await this.persistClientCredentials(existing);
return existing;
}
public static updateClient( public static updateClient(
clientId: string, clientId: string,
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>, updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
): ClientSDK | null { ): ClientSDK | null {
const client = this.clients.get(clientId); const client = getClientsMap().get(clientId);
if (!client) return null; if (!client) return null;
if (updates.name !== undefined) client.name = updates.name; if (updates.name !== undefined) client.name = updates.name.trim();
if (updates.rateLimit !== undefined) client.rateLimit = updates.rateLimit; if (updates.rateLimit !== undefined)
client.rateLimit = Math.max(1, Number(updates.rateLimit) || 100);
if (updates.status !== undefined) client.status = updates.status; if (updates.status !== undefined) client.status = updates.status;
this.clients.set(clientId, client); getClientsMap().set(clientId, client);
void this.persistClientUpdates(client);
return client;
}
private static async persistClientUpdates(client: ClientSDK): Promise<void> {
const pool = DatabaseEngine.getPool();
if (!pool) return;
try {
await DatabaseEngine.ensureSchema();
await pool.query(
`UPDATE client_sdks
SET name = $2, rate_limit = $3, status = $4, updated_at = NOW()
WHERE client_id = $1`,
[client.clientId, client.name, client.rateLimit, client.status],
);
} catch {
// Ignore DB failure
}
}
public static async updateClientAsync(
clientId: string,
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
): Promise<ClientSDK | null> {
const client = await this.getClientAsync(clientId);
if (!client) return null;
if (updates.name !== undefined) client.name = updates.name.trim();
if (updates.rateLimit !== undefined)
client.rateLimit = Math.max(1, Number(updates.rateLimit) || 100);
if (updates.status !== undefined) client.status = updates.status;
getClientsMap().set(clientId, client);
await this.persistClientUpdates(client);
return client; return client;
} }
public static deleteClient(clientId: string): boolean { public static deleteClient(clientId: string): boolean {
return this.clients.delete(clientId); const deleted = getClientsMap().delete(clientId);
void this.deleteClientFromDb(clientId);
return deleted;
}
private static async deleteClientFromDb(clientId: string): Promise<boolean> {
const pool = DatabaseEngine.getPool();
if (!pool) return false;
try {
await DatabaseEngine.ensureSchema();
const res = await pool.query(
`DELETE FROM client_sdks WHERE client_id = $1`,
[clientId],
);
return Boolean(res.rowCount && res.rowCount > 0);
} catch {
return false;
}
}
public static async deleteClientAsync(clientId: string): Promise<boolean> {
const existedInMem = getClientsMap().delete(clientId);
const deletedFromDb = await this.deleteClientFromDb(clientId);
return existedInMem || deletedFromDb;
} }
public static incrementEventCount(clientId: string, count = 1): void { public static incrementEventCount(clientId: string, count = 1): void {
const client = this.clients.get(clientId); const nowIso = new Date().toISOString();
const client = getClientsMap().get(clientId);
if (client) { if (client) {
client.totalEvents += count; client.totalEvents += count;
client.lastActive = new Date().toISOString(); client.lastActive = nowIso;
this.clients.set(clientId, client); getClientsMap().set(clientId, client);
}
const pool = DatabaseEngine.getPool();
if (pool) {
void DatabaseEngine.ensureSchema()
.then(() =>
pool.query(
`UPDATE client_sdks
SET total_events = total_events + $2,
last_active = $3,
updated_at = NOW()
WHERE client_id = $1`,
[clientId, count, nowIso],
),
)
.catch(() => undefined);
}
}
public static resignAllClientTokens(): void {
const map = getClientsMap();
for (const [clientId, client] of map.entries()) {
client.token = AuthService.generateClientToken({
clientId: client.clientId,
name: client.name,
rateLimit: client.rateLimit,
});
map.set(clientId, client);
void this.persistClientCredentials(client);
}
}
public static async resignAllClientTokensAsync(): Promise<void> {
const clients = await this.listClientsAsync();
const map = getClientsMap();
for (const client of clients) {
client.token = AuthService.generateClientToken({
clientId: client.clientId,
name: client.name,
rateLimit: client.rateLimit,
});
map.set(client.clientId, client);
await this.persistClientCredentials(client);
} }
} }
} }
+5
View File
@@ -11,6 +11,7 @@ services:
environment: environment:
NODE_ENV: production NODE_ENV: production
PORT: "3000" PORT: "3000"
INGESTION_INTERNAL_URL: "http://ingestion:4000"
DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}" DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}"
DATASTACK_ADMIN_EMAIL: "${DATASTACK_ADMIN_EMAIL:-admin@datastack.dev}" DATASTACK_ADMIN_EMAIL: "${DATASTACK_ADMIN_EMAIL:-admin@datastack.dev}"
DATASTACK_ADMIN_SESSION_SECRET: "${SERVICE_REALBASE64_64_ADMINSESSION:?Coolify must generate SERVICE_REALBASE64_64_ADMINSESSION}" DATASTACK_ADMIN_SESSION_SECRET: "${SERVICE_REALBASE64_64_ADMINSESSION:?Coolify must generate SERVICE_REALBASE64_64_ADMINSESSION}"
@@ -46,6 +47,10 @@ services:
NODE_ENV: production NODE_ENV: production
PORT_SOCKET: "4000" PORT_SOCKET: "4000"
DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}" DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}"
DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}"
depends_on:
postgres:
condition: service_healthy
restart: unless-stopped restart: unless-stopped
healthcheck: healthcheck:
test: test:
+16 -2
View File
@@ -43,13 +43,23 @@ export async function middleware(request: NextRequest) {
!isLoginPage && !isLoginPage &&
!isSetupPage; !isSetupPage;
const isAdminApi = isProtectedApi(pathname, request.method); const isAdminApi = isProtectedApi(pathname, request.method);
const hasBearerAuth = Boolean(
request.headers.get("authorization")?.toLowerCase().startsWith("bearer "),
);
const isStateChangingApi = const isStateChangingApi =
pathname.startsWith("/api/") && pathname.startsWith("/api/") &&
!SAFE_METHODS.has(request.method) && !SAFE_METHODS.has(request.method) &&
pathname !== "/api/events"; !(pathname === "/api/events" && hasBearerAuth);
if (isSetupPage) return NextResponse.next(); if (isSetupPage) return NextResponse.next();
if (!isConsolePage && !isAdminApi && !isLoginPage) return NextResponse.next(); if (
!isConsolePage &&
!isAdminApi &&
!isLoginPage &&
!(pathname === "/api/events" && request.method === "POST")
) {
return NextResponse.next();
}
if (isStateChangingApi && !isSameOriginRequest(request)) { if (isStateChangingApi && !isSameOriginRequest(request)) {
return NextResponse.json( return NextResponse.json(
@@ -58,6 +68,10 @@ export async function middleware(request: NextRequest) {
); );
} }
if (pathname === "/api/events" && request.method === "POST") {
return NextResponse.next();
}
const sessionSecret = const sessionSecret =
process.env.DATASTACK_ADMIN_SESSION_SECRET || process.env.DATASTACK_ADMIN_SESSION_SECRET ||
"datastack-admin-session-default-secret-32-chars"; "datastack-admin-session-default-secret-32-chars";
+32 -8
View File
@@ -1,6 +1,7 @@
import time import time
import uuid import uuid
import logging import logging
import threading
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
import socketio import socketio
@@ -63,10 +64,15 @@ class DataStackClient:
logger.error(f"Failed to connect to DataStack: {e}") logger.error(f"Failed to connect to DataStack: {e}")
return False return False
def sync(self, data: Any, event_id: Optional[str] = None) -> Dict[str, Any]: def sync(
self,
data: Any,
event_id: Optional[str] = None,
timeout: float = 5.0,
) -> Dict[str, Any]:
""" """
Sync single event payload in realtime. Sync single event payload in realtime.
Returns server response dict. Blocks until the server ack callback fires or timeout expires.
""" """
if not self.is_connected: if not self.is_connected:
raise ConnectionError("DataStack client is not connected. Call connect() first.") raise ConnectionError("DataStack client is not connected. Call connect() first.")
@@ -80,24 +86,35 @@ class DataStackClient:
"data": data, "data": data,
} }
response = {"success": False, "eventId": evt_id} ack_event = threading.Event()
response: Dict[str, Any] = {
"success": False,
"eventId": evt_id,
"error": "Timeout waiting for server acknowledgment",
}
def on_ack(res): def on_ack(res):
nonlocal response nonlocal response
if isinstance(res, dict):
response = res response = res
ack_event.set()
try: try:
self.sio.emit("sync_data", payload, callback=on_ack) self.sio.emit("sync_data", payload, callback=on_ack)
# Short wait for ack in synchronous invocation ack_event.wait(timeout=timeout)
time.sleep(0.05)
return response return response
except Exception as e: except Exception as e:
logger.error(f"Sync error for event {evt_id}: {e}") logger.error(f"Sync error for event {evt_id}: {e}")
return {"success": False, "eventId": evt_id, "error": str(e)} return {"success": False, "eventId": evt_id, "error": str(e)}
def batch_sync(self, records: List[Dict[str, Any]]) -> Dict[str, Any]: def batch_sync(
self,
records: List[Dict[str, Any]],
timeout: float = 10.0,
) -> Dict[str, Any]:
""" """
Sync list of records in a single batch. Sync list of records in a single batch.
Blocks until the server ack callback fires or timeout expires.
""" """
if not self.is_connected: if not self.is_connected:
raise ConnectionError("DataStack client is not connected.") raise ConnectionError("DataStack client is not connected.")
@@ -110,15 +127,22 @@ class DataStackClient:
"data": r.get("data", r), "data": r.get("data", r),
}) })
response = {"success": False, "processed": 0} ack_event = threading.Event()
response: Dict[str, Any] = {
"success": False,
"processed": 0,
"error": "Timeout waiting for server batch acknowledgment",
}
def on_ack(res): def on_ack(res):
nonlocal response nonlocal response
if isinstance(res, dict):
response = res response = res
ack_event.set()
try: try:
self.sio.emit("batch_sync", batch_payload, callback=on_ack) self.sio.emit("batch_sync", batch_payload, callback=on_ack)
time.sleep(0.1) ack_event.wait(timeout=timeout)
return response return response
except Exception as e: except Exception as e:
logger.error(f"Batch sync error: {e}") logger.error(f"Batch sync error: {e}")
+227 -24
View File
@@ -2,57 +2,260 @@ const express = require('express');
const http = require('http'); const http = require('http');
const cors = require('cors'); const cors = require('cors');
const { Server } = require('socket.io'); const { Server } = require('socket.io');
const { Pool } = require('pg');
const { setupSocketHandlers } = require('./socket/handlers'); const { setupSocketHandlers } = require('./socket/handlers');
const { CsvWriterWorker } = require('./workers/csvWriter'); const { CsvWriterWorker } = require('./workers/csvWriter');
const PORT = process.env.PORT_SOCKET || 4000; const PORT = Number(process.env.PORT_SOCKET) || 4000;
const app = express(); const app = express();
app.use(cors({ origin: '*' })); app.use(cors({ origin: '*' }));
app.use(express.json()); app.use(express.json({ limit: '10mb' }));
const bufferQueue = []; const bufferQueue = [];
const writer = new CsvWriterWorker(); const writer = new CsvWriterWorker();
// Periodic Batch Flush: every 2000ms or when buffer hits 500 const runtimeConfig = {
const FLUSH_INTERVAL_MS = 2000; jwtSecret:
const FLUSH_THRESHOLD = 500; process.env.DATASTACK_JWT_SECRET ||
'datastack-jwt-default-secret-key-32-chars-long',
maxEventsPerSec: 100,
maxPayloadSizeBytes: 100 * 1024,
batchFlushThreshold: 500,
batchFlushIntervalMs: 2000,
fileRotationMb: 100,
enableIdempotency: true,
};
let dbPool = null;
if (process.env.DATABASE_URL) {
try {
dbPool = new Pool({
connectionString: process.env.DATABASE_URL,
connectionTimeoutMillis: 3000,
idleTimeoutMillis: 30000,
max: 5,
});
} catch {
dbPool = null;
}
}
// Short-lived cache (2s TTL) for client status & token checks on high-frequency events
const clientAuthCache = new Map();
const CLIENT_CACHE_TTL_MS = 2000;
async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
if (!dbPool || !clientId) {
return { allowed: true };
}
const now = Date.now();
if (!bypassCache) {
const cached = clientAuthCache.get(clientId);
if (cached && now - cached.checkedAt < CLIENT_CACHE_TTL_MS) {
if (cached.status === 'revoked') {
return {
allowed: false,
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
};
}
if (cached.token && presentedToken && cached.token !== presentedToken) {
return {
allowed: false,
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
};
}
return { allowed: true, rateLimit: cached.rateLimit };
}
}
try {
const res = await dbPool.query(
`SELECT status, token, rate_limit FROM client_sdks WHERE client_id = $1`,
[clientId],
);
if (!res.rowCount || res.rowCount === 0) {
// Allow token-authenticated client if not explicitly revoked in table
return { allowed: true };
}
const row = res.rows[0];
const entry = {
status: row.status,
token: row.token,
rateLimit: Number(row.rate_limit) || 100,
checkedAt: now,
};
clientAuthCache.set(clientId, entry);
if (entry.status === 'revoked') {
return {
allowed: false,
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
};
}
if (entry.token && presentedToken && entry.token !== presentedToken) {
return {
allowed: false,
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
};
}
return { allowed: true, rateLimit: entry.rateLimit };
} catch {
return { allowed: true };
}
}
function recordClientActivity(clientId, count = 1) {
if (!dbPool || !clientId || count <= 0) return;
const nowIso = new Date().toISOString();
dbPool
.query(
`UPDATE client_sdks
SET total_events = total_events + $2,
last_active = $3,
updated_at = NOW()
WHERE client_id = $1`,
[clientId, count, nowIso],
)
.catch(() => undefined);
}
let isFlushing = false;
let flushTimer = null;
async function flushBuffer() { async function flushBuffer() {
if (bufferQueue.length === 0) return; if (bufferQueue.length === 0 || isFlushing) {
return { written: 0, files: [] };
}
isFlushing = true;
const batch = bufferQueue.splice(0, bufferQueue.length); const batch = bufferQueue.splice(0, bufferQueue.length);
try { try {
const result = await writer.writeBatch(batch); const result = await writer.writeBatch(batch);
if (result.written > 0) { if (result.written > 0) {
console.log(`[CSV Worker] Flushed ${result.written} events to ${result.files.length} file(s)`); console.log(
`[CSV Worker] Flushed ${result.written} events to ${result.files.length} file(s)`,
);
} }
return result;
} catch (err) { } catch (err) {
console.error('[CSV Worker] Flush error:', err); console.error('[CSV Worker] Flush error:', err);
bufferQueue.unshift(...batch); // Retry next cycle bufferQueue.unshift(...batch);
return { written: 0, files: [] };
} finally {
isFlushing = false;
} }
} }
setInterval(flushBuffer, FLUSH_INTERVAL_MS); function scheduleFlushTimer(intervalMs) {
const safeInterval = Math.max(100, Number(intervalMs) || 2000);
runtimeConfig.batchFlushIntervalMs = safeInterval;
if (flushTimer) {
clearInterval(flushTimer);
}
flushTimer = setInterval(() => {
void flushBuffer();
}, safeInterval);
}
// Health check endpoint scheduleFlushTimer(runtimeConfig.batchFlushIntervalMs);
app.get('/health', (_req, res) => {
res.json({ async function syncSettingsFromDatabase() {
status: 'online', if (!dbPool) return;
service: 'DataStack Express Ingestion', try {
port: PORT, const res = await dbPool.query(
bufferSize: bufferQueue.length, `SELECT max_events_per_sec, max_payload_size_kb, batch_flush_threshold,
uptime: process.uptime(), batch_flush_interval_ms, file_rotation_mb, jwt_secret, enable_idempotency
}); FROM admin_settings
}); WHERE id = 'default'`,
);
if (!res.rowCount || res.rowCount === 0) return;
const row = res.rows[0];
runtimeConfig.maxEventsPerSec = Number(row.max_events_per_sec) || 100;
runtimeConfig.maxPayloadSizeBytes =
(Number(row.max_payload_size_kb) || 100) * 1024;
runtimeConfig.batchFlushThreshold =
Math.max(1, Number(row.batch_flush_threshold) || 500);
runtimeConfig.enableIdempotency = Boolean(row.enable_idempotency);
if (row.jwt_secret && String(row.jwt_secret).trim().length > 0) {
runtimeConfig.jwtSecret = String(row.jwt_secret).trim();
}
const nextRotationMb = Number(row.file_rotation_mb) || 100;
if (nextRotationMb !== runtimeConfig.fileRotationMb) {
runtimeConfig.fileRotationMb = nextRotationMb;
writer.setMaxFileSizeMb(nextRotationMb);
}
const nextIntervalMs = Math.max(
100,
Number(row.batch_flush_interval_ms) || 2000,
);
if (nextIntervalMs !== runtimeConfig.batchFlushIntervalMs) {
scheduleFlushTimer(nextIntervalMs);
}
} catch {
// Keep existing runtimeConfig if table is not created yet
}
}
if (dbPool) {
void syncSettingsFromDatabase();
setInterval(() => {
void syncSettingsFromDatabase();
}, 3000);
}
const server = http.createServer(app); const server = http.createServer(app);
const io = new Server(server, { const io = new Server(server, {
cors: { origin: '*', methods: ['GET', 'POST'] }, cors: { origin: '*', methods: ['GET', 'POST'] },
}); });
setupSocketHandlers(io, bufferQueue, writer); // Health check endpoint with activeSockets count
app.get('/health', (_req, res) => {
server.listen(PORT, () => { res.json({
console.log(`[DataStack Ingestion Engine] Express + Socket.IO listening on port ${PORT}`); status: 'online',
service: 'DataStack Express Ingestion',
port: PORT,
activeSockets: io.engine ? io.engine.clientsCount : 0,
bufferSize: bufferQueue.length,
uptime: process.uptime(),
});
}); });
module.exports = { app, server, io, bufferQueue }; // Internal flush endpoint triggered by Next.js POST /api/storage
app.post('/internal/flush', async (_req, res) => {
try {
const result = await flushBuffer();
res.json({
success: true,
flushedCount: result.written,
filesUpdated: result.files,
});
} catch (err) {
res.status(500).json({
success: false,
error: err instanceof Error ? err.message : 'Flush failed',
});
}
});
setupSocketHandlers(io, bufferQueue, writer, {
getRuntimeConfig: () => runtimeConfig,
verifyClientInDb,
recordClientActivity,
onBufferPush: () => {
if (bufferQueue.length >= runtimeConfig.batchFlushThreshold) {
void flushBuffer();
}
},
});
server.listen(PORT, () => {
console.log(
`[DataStack Ingestion Engine] Express + Socket.IO listening on port ${PORT}`,
);
});
module.exports = { app, server, io, bufferQueue, flushBuffer };
+241 -49
View File
@@ -1,17 +1,120 @@
const jwt = require("jsonwebtoken"); const jwt = require("jsonwebtoken");
const IDEMPOTENCY_TTL_MS = 5 * 60 * 1000; // 5 minutes
const IDEMPOTENCY_MAX_ENTRIES = 20000;
const CLEANUP_INTERVAL_MS = 60 * 1000;
function setupSocketHandlers(io, bufferQueue, worker, options = {}) { function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
const jwtSecret = options.jwtSecret || process.env.DATASTACK_JWT_SECRET; const getRuntimeConfig =
if (!jwtSecret) { typeof options.getRuntimeConfig === "function"
throw new Error( ? options.getRuntimeConfig
"DATASTACK_JWT_SECRET must be configured before starting the socket server.", : () => ({
); jwtSecret:
} options.jwtSecret ||
process.env.DATASTACK_JWT_SECRET ||
"datastack-jwt-default-secret-key-32-chars-long",
maxEventsPerSec: 100,
maxPayloadSizeBytes: 100 * 1024,
batchFlushThreshold: 500,
enableIdempotency: true,
});
const verifyClientInDb =
typeof options.verifyClientInDb === "function"
? options.verifyClientInDb
: async () => ({ allowed: true });
const recordClientActivity =
typeof options.recordClientActivity === "function"
? options.recordClientActivity
: () => undefined;
const onBufferPush =
typeof options.onBufferPush === "function"
? options.onBufferPush
: () => undefined;
const idempotencyMap = new Map(); const idempotencyMap = new Map();
const rateLimitMap = new Map(); const rateLimitMap = new Map();
let lastCleanupAt = Date.now();
function cleanupMaps(now) {
if (
now - lastCleanupAt < CLEANUP_INTERVAL_MS &&
idempotencyMap.size <= IDEMPOTENCY_MAX_ENTRIES
) {
return;
}
lastCleanupAt = now;
for (const [key, timestamp] of idempotencyMap.entries()) {
if (now - timestamp > IDEMPOTENCY_TTL_MS) {
idempotencyMap.delete(key);
}
}
for (const [clientId, rl] of rateLimitMap.entries()) {
if (now - rl.start > 10000) {
rateLimitMap.delete(clientId);
}
}
}
function validateSingleEvent(clientId, eventId, rawData, clientRateLimit) {
const config = getRuntimeConfig();
const now = Date.now();
cleanupMaps(now);
// 1. Idempotency check (with 5-minute TTL)
if (config.enableIdempotency && eventId) {
const key = `${clientId}:${eventId}`;
const existingTime = idempotencyMap.get(key);
if (existingTime && now - existingTime < IDEMPOTENCY_TTL_MS) {
return {
allowed: false,
duplicate: true,
reason: "Duplicate event dropped",
};
}
idempotencyMap.set(key, now);
}
// 2. Rate limit check
const effectiveLimit = Math.min(
Number(clientRateLimit) || 100,
Number(config.maxEventsPerSec) || 100,
);
const rl = rateLimitMap.get(clientId);
if (!rl || now - rl.start >= 1000) {
rateLimitMap.set(clientId, { count: 1, start: now });
} else if (rl.count >= effectiveLimit) {
return {
allowed: false,
duplicate: false,
reason: `RATE_LIMIT_EXCEEDED: Limit of ${effectiveLimit} events/sec exceeded`,
};
} else {
rl.count += 1;
}
// 3. Payload byte size check
const payloadStr =
typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {});
const byteLength = Buffer.byteLength(payloadStr, "utf8");
const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024;
if (byteLength > maxBytes) {
return {
allowed: false,
duplicate: false,
reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`,
};
}
return { allowed: true, payloadStr };
}
// Middleware: Handshake Auth // Middleware: Handshake Auth
io.use((socket, next) => { io.use(async (socket, next) => {
const authHeader = const authHeader =
socket.handshake.auth?.token || socket.handshake.auth?.token ||
socket.handshake.headers?.authorization || socket.handshake.headers?.authorization ||
@@ -22,12 +125,25 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
} }
try { try {
const cleanToken = authHeader.startsWith("Bearer ") const cleanToken = String(authHeader).startsWith("Bearer ")
? authHeader.slice(7) ? String(authHeader).slice(7).trim()
: authHeader; : String(authHeader).trim();
const decoded = jwt.verify(cleanToken, jwtSecret); const config = getRuntimeConfig();
const decoded = jwt.verify(cleanToken, config.jwtSecret);
const dbCheck = await verifyClientInDb(
decoded.clientId,
cleanToken,
true,
);
if (!dbCheck.allowed) {
return next(new Error(dbCheck.reason || "CLIENT_REVOKED"));
}
socket.data.clientId = decoded.clientId; socket.data.clientId = decoded.clientId;
socket.data.rateLimit = decoded.rateLimit || 100; socket.data.token = cleanToken;
socket.data.rateLimit =
dbCheck.rateLimit || decoded.rateLimit || config.maxEventsPerSec || 100;
next(); next();
} catch (err) { } catch (err) {
next(new Error(`AUTH_INVALID: ${err.message}`)); next(new Error(`AUTH_INVALID: ${err.message}`));
@@ -36,50 +152,64 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
io.on("connection", (socket) => { io.on("connection", (socket) => {
const clientId = socket.data.clientId; const clientId = socket.data.clientId;
const rateLimit = socket.data.rateLimit;
socket.join(`client:${clientId}`); socket.join(`client:${clientId}`);
// Single event // Single event
socket.on("sync_data", (payload, callback) => { socket.on("sync_data", async (payload, callback) => {
try { try {
const eventId = const dbCheck = await verifyClientInDb(
payload.eventId || clientId,
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`; socket.data.token,
const timestamp = payload.timestamp || new Date().toISOString(); false,
);
// Idempotency if (!dbCheck.allowed) {
const key = `${clientId}:${eventId}`; if (callback) {
if (idempotencyMap.has(key)) { callback({
if (callback) success: false,
callback({ success: true, eventId, note: "DUPLICATE_DROPPED" }); error: dbCheck.reason || "CLIENT_REVOKED",
});
}
socket.disconnect(true);
return; return;
} }
idempotencyMap.set(key, Date.now());
// Rate limit const effectiveClientRate =
const now = Date.now(); dbCheck.rateLimit || socket.data.rateLimit || 100;
const rl = rateLimitMap.get(clientId); const safePayload = payload && typeof payload === "object" ? payload : {};
if (!rl || now - rl.start >= 1000) { const eventId =
rateLimitMap.set(clientId, { count: 1, start: now }); safePayload.eventId ||
} else if (rl.count >= rateLimit) { `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
if (callback) const timestamp = safePayload.timestamp || new Date().toISOString();
callback({ success: false, error: "RATE_LIMIT_EXCEEDED" });
const check = validateSingleEvent(
clientId,
eventId,
safePayload.data,
effectiveClientRate,
);
if (!check.allowed) {
if (check.duplicate) {
if (callback) {
callback({ success: true, eventId, note: "DUPLICATE_DROPPED" });
}
return;
}
if (callback) {
callback({ success: false, eventId, error: check.reason });
}
return; return;
} else {
rl.count += 1;
} }
const event = { const event = {
clientId, clientId,
eventId, eventId,
timestamp, timestamp,
payload: payload: check.payloadStr,
typeof payload.data === "string"
? payload.data
: JSON.stringify(payload.data),
}; };
bufferQueue.push(event); bufferQueue.push(event);
recordClientActivity(clientId, 1);
onBufferPush();
if (callback) callback({ success: true, eventId }); if (callback) callback({ success: true, eventId });
} catch (err) { } catch (err) {
@@ -88,32 +218,94 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
}); });
// Batch event // Batch event
socket.on("batch_sync", (batch, callback) => { socket.on("batch_sync", async (batch, callback) => {
try {
if (!Array.isArray(batch)) { if (!Array.isArray(batch)) {
if (callback) callback({ success: false, error: "Expected array" }); if (callback) callback({ success: false, error: "Expected array" });
return; return;
} }
let count = 0; const dbCheck = await verifyClientInDb(
clientId,
socket.data.token,
false,
);
if (!dbCheck.allowed) {
if (callback) {
callback({
success: false,
processed: 0,
rejected: batch.length,
error: dbCheck.reason || "CLIENT_REVOKED",
});
}
socket.disconnect(true);
return;
}
const effectiveClientRate =
dbCheck.rateLimit || socket.data.rateLimit || 100;
let processed = 0;
let rejected = 0;
const errors = [];
for (const item of batch) { for (const item of batch) {
const safeItem = item && typeof item === "object" ? item : {};
const eventId = const eventId =
item.eventId || safeItem.eventId ||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`; `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
const timestamp = item.timestamp || new Date().toISOString(); const timestamp = safeItem.timestamp || new Date().toISOString();
const rawData =
safeItem.data !== undefined ? safeItem.data : safeItem;
const check = validateSingleEvent(
clientId,
eventId,
rawData,
effectiveClientRate,
);
if (!check.allowed) {
if (!check.duplicate) {
rejected += 1;
if (errors.length < 5) {
errors.push(`${eventId}: ${check.reason}`);
}
}
continue;
}
bufferQueue.push({ bufferQueue.push({
clientId, clientId,
eventId, eventId,
timestamp, timestamp,
payload: payload: check.payloadStr,
typeof item.data === "string"
? item.data
: JSON.stringify(item.data),
}); });
count++; processed += 1;
} }
if (callback) callback({ success: true, processed: count }); if (processed > 0) {
recordClientActivity(clientId, processed);
onBufferPush();
}
if (callback) {
callback({
success: true,
processed,
rejected,
...(errors.length > 0 ? { errors } : {}),
});
}
} catch (err) {
if (callback) {
callback({
success: false,
processed: 0,
rejected: Array.isArray(batch) ? batch.length : 0,
error: err.message,
});
}
}
}); });
}); });
} }
+8 -1
View File
@@ -14,6 +14,13 @@ class CsvWriterWorker {
this.ensureBaseDir(); this.ensureBaseDir();
} }
setMaxFileSizeMb(megabytes) {
const mb = Number(megabytes);
if (Number.isFinite(mb) && mb >= 1) {
this.maxFileSize = Math.floor(mb * 1024 * 1024);
}
}
ensureBaseDir() { ensureBaseDir() {
if (!fs.existsSync(this.baseDir)) { if (!fs.existsSync(this.baseDir)) {
fs.mkdirSync(this.baseDir, { recursive: true }); fs.mkdirSync(this.baseDir, { recursive: true });
@@ -21,7 +28,7 @@ class CsvWriterWorker {
} }
getTargetFilePath(clientId, dateStr) { getTargetFilePath(clientId, dateStr) {
const safeClientId = clientId.replace(/[^a-zA-Z0-9_-]/g, '_'); const safeClientId = String(clientId || 'unknown').replace(/[^a-zA-Z0-9_-]/g, '_');
const clientDir = path.join(this.baseDir, safeClientId); const clientDir = path.join(this.baseDir, safeClientId);
if (!fs.existsSync(clientDir)) { if (!fs.existsSync(clientDir)) {
fs.mkdirSync(clientDir, { recursive: true }); fs.mkdirSync(clientDir, { recursive: true });
@@ -1,3 +0,0 @@
clientId,eventId,timestamp,payload
py_scraper_prod_02,evt_scrape_101,2026-10-23T06:10:00.000Z,"{""spider"":""market_catalog"",""pages"":42,""errors"":0,""bytes"":89200}"
py_scraper_prod_02,evt_scrape_102,2026-10-23T06:25:00.000Z,"{""spider"":""price_indexer"",""pages"":18,""errors"":0,""bytes"":41200}"
1 clientId eventId timestamp payload
2 py_scraper_prod_02 evt_scrape_101 2026-10-23T06:10:00.000Z {"spider":"market_catalog","pages":42,"errors":0,"bytes":89200}
3 py_scraper_prod_02 evt_scrape_102 2026-10-23T06:25:00.000Z {"spider":"price_indexer","pages":18,"errors":0,"bytes":41200}
@@ -1,4 +0,0 @@
clientId,eventId,timestamp,payload
py_telemetry_agent_01,evt_seed_001,2026-10-23T06:00:00.000Z,"{""action"":""agent_boot"",""cpu"":18.4,""ram_mb"":512,""status"":""nominal""}"
py_telemetry_agent_01,evt_seed_002,2026-10-23T06:15:00.000Z,"{""action"":""heartbeat"",""cpu"":22.1,""ram_mb"":524,""status"":""nominal""}"
py_telemetry_agent_01,evt_seed_003,2026-10-23T06:30:00.000Z,"{""action"":""batch_processed"",""items_count"":142,""latency_ms"":18}"
1 clientId eventId timestamp payload
2 py_telemetry_agent_01 evt_seed_001 2026-10-23T06:00:00.000Z {"action":"agent_boot","cpu":18.4,"ram_mb":512,"status":"nominal"}
3 py_telemetry_agent_01 evt_seed_002 2026-10-23T06:15:00.000Z {"action":"heartbeat","cpu":22.1,"ram_mb":524,"status":"nominal"}
4 py_telemetry_agent_01 evt_seed_003 2026-10-23T06:30:00.000Z {"action":"batch_processed","items_count":142,"latency_ms":18}
-32
View File
@@ -1,32 +0,0 @@
import { useState, useEffect, useCallback } from 'react';
export function useVconMetrics(refreshIntervalMs = 3000) {
const [metrics, setMetrics] = useState<any>(null);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const fetchMetrics = useCallback(async () => {
try {
const res = await fetch('/api/metrics');
if (res.ok) {
const data = await res.json();
setMetrics(data);
setError(null);
} else {
setError('Failed to fetch metrics');
}
} catch (err: any) {
setError(err.message);
} finally {
setLoading(false);
}
}, []);
useEffect(() => {
fetchMetrics();
const timer = setInterval(fetchMetrics, refreshIntervalMs);
return () => clearInterval(timer);
}, [fetchMetrics, refreshIntervalMs]);
return { metrics, loading, error, refetch: fetchMetrics };
}
-16
View File
@@ -1,16 +0,0 @@
export async function vconApiFetch<T>(endpoint: string, options?: RequestInit): Promise<T> {
const res = await fetch(endpoint, {
...options,
headers: {
'Content-Type': 'application/json',
...options?.headers,
},
});
if (!res.ok) {
const errorBody = await res.json().catch(() => ({}));
throw new Error(errorBody.error || `API error ${res.status}: ${res.statusText}`);
}
return res.json();
}