refactor: switch SDK methods to asynchronous
Migrate SDK interactions to asynchronous calls, relax admin setup email constraints, and add event retrieval functionality.
This commit is contained in:
1 parent
81c5e4e2fd
commit
bb938c33c0
36 files changed
+1641
-716
No files matched your search
@@ -6,3 +6,4 @@ coverage/
|
|||||||
.env*
|
.env*
|
||||||
!.env.example
|
!.env.example
|
||||||
notes/
|
notes/
|
||||||
|
storage/data/
|
||||||
@@ -9,7 +9,7 @@ import { redirect } from 'next/navigation';
|
|||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
export default async function LandingPage() {
|
export default async function LandingPage() {
|
||||||
if (process.env.NODE_ENV === 'production' && process.env.DATABASE_URL) {
|
if (process.env.DATABASE_URL) {
|
||||||
const isConfigured = await isAdminSetupComplete().catch(() => false);
|
const isConfigured = await isAdminSetupComplete().catch(() => false);
|
||||||
if (!isConfigured) redirect('/vcon/setup');
|
if (!isConfigured) redirect('/vcon/setup');
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -100,7 +100,12 @@ export async function PUT(request: NextRequest) {
|
|||||||
return NextResponse.json({ error: result.error }, { status: 401 });
|
return NextResponse.json({ error: result.error }, { status: 401 });
|
||||||
}
|
}
|
||||||
|
|
||||||
const expiresAt = Date.now() + SESSION_SECONDS * 1000;
|
// Preserve remaining session duration (e.g., 30-day "Remember me" session)
|
||||||
|
const remainingSeconds = Math.max(
|
||||||
|
SESSION_SECONDS,
|
||||||
|
Math.ceil((session.expiresAt - Date.now()) / 1000),
|
||||||
|
);
|
||||||
|
const expiresAt = Date.now() + remainingSeconds * 1000;
|
||||||
const nextToken = await createAdminSessionToken(
|
const nextToken = await createAdminSessionToken(
|
||||||
result.email,
|
result.email,
|
||||||
sessionSecret,
|
sessionSecret,
|
||||||
@@ -121,7 +126,7 @@ export async function PUT(request: NextRequest) {
|
|||||||
response.cookies.set(
|
response.cookies.set(
|
||||||
SESSION_COOKIE,
|
SESSION_COOKIE,
|
||||||
nextToken,
|
nextToken,
|
||||||
getSessionCookieOptions(request, SESSION_SECONDS),
|
getSessionCookieOptions(request, remainingSeconds),
|
||||||
);
|
);
|
||||||
|
|
||||||
return response;
|
return response;
|
||||||
|
|||||||
@@ -23,21 +23,12 @@ export async function POST(request: NextRequest) {
|
|||||||
const sessionSecret =
|
const sessionSecret =
|
||||||
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
|
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
|
||||||
"datastack-admin-session-default-secret-32-chars";
|
"datastack-admin-session-default-secret-32-chars";
|
||||||
const configuredAdminEmail =
|
|
||||||
process.env.DATASTACK_ADMIN_EMAIL?.trim().toLowerCase() ||
|
|
||||||
"admin@datastack.dev";
|
|
||||||
if (!sessionSecret || sessionSecret.length < 32) {
|
if (!sessionSecret || sessionSecret.length < 32) {
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{ error: "Admin session configuration is unavailable." },
|
{ error: "Admin session configuration is unavailable." },
|
||||||
{ status: 503 },
|
{ status: 503 },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if (!configuredAdminEmail || !validEmail(configuredAdminEmail)) {
|
|
||||||
return NextResponse.json(
|
|
||||||
{ error: "Admin setup configuration is unavailable." },
|
|
||||||
{ status: 503 },
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const body = await request.json();
|
const body = await request.json();
|
||||||
@@ -52,17 +43,9 @@ export async function POST(request: NextRequest) {
|
|||||||
{ status: 400 },
|
{ status: 400 },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
if (email !== configuredAdminEmail) {
|
if (password.length < 8 || password.length > 256) {
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{
|
{ error: "Choose a password between 8 and 256 characters." },
|
||||||
error: "Use the administrator email configured for this deployment.",
|
|
||||||
},
|
|
||||||
{ status: 403 },
|
|
||||||
);
|
|
||||||
}
|
|
||||||
if (password.length < 12 || password.length > 256) {
|
|
||||||
return NextResponse.json(
|
|
||||||
{ error: "Choose a password between 12 and 256 characters." },
|
|
||||||
{ status: 400 },
|
{ status: 400 },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ export async function GET(
|
|||||||
{ params }: { params: Promise<{ id: string }> },
|
{ params }: { params: Promise<{ id: string }> },
|
||||||
) {
|
) {
|
||||||
const { id } = await params;
|
const { id } = await params;
|
||||||
const client = SDKManager.getClient(id);
|
const client = await SDKManager.getClientAsync(id);
|
||||||
if (!client) {
|
if (!client) {
|
||||||
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
||||||
}
|
}
|
||||||
@@ -24,7 +24,7 @@ export async function PATCH(
|
|||||||
|
|
||||||
// Special action: regenerate token
|
// Special action: regenerate token
|
||||||
if (body.action === "regenerate_token") {
|
if (body.action === "regenerate_token") {
|
||||||
const refreshed = SDKManager.regenerateToken(id);
|
const refreshed = await SDKManager.regenerateTokenAsync(id);
|
||||||
if (!refreshed) {
|
if (!refreshed) {
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{ error: "Client not found" },
|
{ error: "Client not found" },
|
||||||
@@ -34,7 +34,7 @@ export async function PATCH(
|
|||||||
return NextResponse.json({ client: refreshed });
|
return NextResponse.json({ client: refreshed });
|
||||||
}
|
}
|
||||||
|
|
||||||
const updated = SDKManager.updateClient(id, body);
|
const updated = await SDKManager.updateClientAsync(id, body);
|
||||||
if (!updated) {
|
if (!updated) {
|
||||||
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
||||||
}
|
}
|
||||||
@@ -50,7 +50,7 @@ export async function DELETE(
|
|||||||
{ params }: { params: Promise<{ id: string }> },
|
{ params }: { params: Promise<{ id: string }> },
|
||||||
) {
|
) {
|
||||||
const { id } = await params;
|
const { id } = await params;
|
||||||
const deleted = SDKManager.deleteClient(id);
|
const deleted = await SDKManager.deleteClientAsync(id);
|
||||||
if (!deleted) {
|
if (!deleted) {
|
||||||
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,9 +3,11 @@ import { NextRequest, NextResponse } from "next/server";
|
|||||||
|
|
||||||
export async function GET() {
|
export async function GET() {
|
||||||
try {
|
try {
|
||||||
const clients = SDKManager.listClients().map(
|
const allClients = await SDKManager.listClientsAsync();
|
||||||
({ secret, token, ...client }) => client,
|
const clients = allClients.map(({ secret, ...client }) => {
|
||||||
);
|
void secret;
|
||||||
|
return client;
|
||||||
|
});
|
||||||
return NextResponse.json({ clients, count: clients.length });
|
return NextResponse.json({ clients, count: clients.length });
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
const message =
|
const message =
|
||||||
@@ -26,17 +28,21 @@ export async function POST(req: NextRequest) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const existing = SDKManager.getClient(clientId);
|
const safeClientId = String(clientId)
|
||||||
|
.trim()
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/[^a-z0-9_-]/g, "_");
|
||||||
|
const existing = await SDKManager.getClientAsync(safeClientId);
|
||||||
if (existing) {
|
if (existing) {
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{ error: `Client '${clientId}' already exists` },
|
{ error: `Client '${safeClientId}' already exists` },
|
||||||
{ status: 409 },
|
{ status: 409 },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const newClient = SDKManager.registerClient({
|
const newClient = await SDKManager.registerClientAsync({
|
||||||
clientId,
|
clientId: safeClientId,
|
||||||
name,
|
name: String(name),
|
||||||
rateLimit: rateLimit ? Number(rateLimit) : 100,
|
rateLimit: rateLimit ? Number(rateLimit) : 100,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
+155
-12
@@ -1,15 +1,77 @@
|
|||||||
import { NextRequest, NextResponse } from 'next/server';
|
import { SettingsManager } from '@/core/adminSettings/settingsManager';
|
||||||
|
import { verifyAdminSessionToken } from '@/core/auth/adminSession';
|
||||||
|
import { AuthService } from '@/core/auth/jwt';
|
||||||
import { DataBufferEngine } from '@/core/data/buffer';
|
import { DataBufferEngine } from '@/core/data/buffer';
|
||||||
import { SDKManager } from '@/core/sdk/sdkManager';
|
import { SDKManager } from '@/core/sdk/sdkManager';
|
||||||
import { SecurityEngine } from '@/core/Security/security';
|
import { SecurityEngine } from '@/core/Security/security';
|
||||||
|
import { CSVStorageManager, type DataEvent } from '@/core/Storage/csvStorage';
|
||||||
|
import { parse } from 'csv-parse';
|
||||||
|
import { NextRequest, NextResponse } from 'next/server';
|
||||||
|
import { createReadStream } from 'node:fs';
|
||||||
|
|
||||||
|
const SESSION_COOKIE = 'datastack_admin_session';
|
||||||
|
|
||||||
|
async function getRecentStoredEvents(
|
||||||
|
limit: number,
|
||||||
|
clientId?: string,
|
||||||
|
): Promise<DataEvent[]> {
|
||||||
|
const files = CSVStorageManager.listAllFiles()
|
||||||
|
.filter((f) => !clientId || clientId === 'all' || f.clientId === clientId)
|
||||||
|
.slice(0, 5);
|
||||||
|
|
||||||
|
const collected: DataEvent[] = [];
|
||||||
|
|
||||||
|
for (const file of files) {
|
||||||
|
const fileEvents: DataEvent[] = [];
|
||||||
|
const parser = createReadStream(file.filePath).pipe(
|
||||||
|
parse({ columns: true, skip_empty_lines: true, bom: true }),
|
||||||
|
);
|
||||||
|
|
||||||
|
try {
|
||||||
|
for await (const record of parser as AsyncIterable<
|
||||||
|
Record<string, string>
|
||||||
|
>) {
|
||||||
|
if (!record.eventId) continue;
|
||||||
|
fileEvents.push({
|
||||||
|
clientId: record.clientId || file.clientId,
|
||||||
|
eventId: record.eventId,
|
||||||
|
timestamp: record.timestamp || `${file.date}T00:00:00.000Z`,
|
||||||
|
payload: record.payload || '',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
parser.destroy();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (fileEvents.length > 0) {
|
||||||
|
collected.push(...fileEvents.slice(-limit));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return collected;
|
||||||
|
}
|
||||||
|
|
||||||
export async function GET(req: NextRequest) {
|
export async function GET(req: NextRequest) {
|
||||||
try {
|
try {
|
||||||
const { searchParams } = new URL(req.url);
|
const { searchParams } = new URL(req.url);
|
||||||
const clientId = searchParams.get('clientId') || undefined;
|
const clientId = searchParams.get('clientId') || undefined;
|
||||||
const limit = Number(searchParams.get('limit')) || 50;
|
const limit = Math.min(200, Math.max(1, Number(searchParams.get('limit')) || 50));
|
||||||
|
|
||||||
|
const bufferedEvents = DataBufferEngine.getRecentEvents(limit, clientId);
|
||||||
|
const storedEvents = await getRecentStoredEvents(limit, clientId);
|
||||||
|
|
||||||
|
const seenIds = new Set<string>();
|
||||||
|
const merged: DataEvent[] = [];
|
||||||
|
|
||||||
|
for (const evt of [...bufferedEvents, ...storedEvents]) {
|
||||||
|
if (!evt.eventId || seenIds.has(evt.eventId)) continue;
|
||||||
|
seenIds.add(evt.eventId);
|
||||||
|
merged.push(evt);
|
||||||
|
}
|
||||||
|
|
||||||
|
merged.sort((a, b) => (b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0));
|
||||||
|
const events = merged.slice(0, limit);
|
||||||
|
|
||||||
const events = DataBufferEngine.getRecentEvents(limit, clientId);
|
|
||||||
return NextResponse.json({ events, count: events.length });
|
return NextResponse.json({ events, count: events.length });
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
const message = err instanceof Error ? err.message : 'Events retrieval failed';
|
const message = err instanceof Error ? err.message : 'Events retrieval failed';
|
||||||
@@ -19,16 +81,97 @@ export async function GET(req: NextRequest) {
|
|||||||
|
|
||||||
export async function POST(req: NextRequest) {
|
export async function POST(req: NextRequest) {
|
||||||
try {
|
try {
|
||||||
const body = await req.json();
|
const settings = await SettingsManager.getSettingsAsync();
|
||||||
const clientId = body.clientId || 'py_telemetry_agent_01';
|
|
||||||
const eventId = body.eventId || `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
|
||||||
const timestamp = body.timestamp || new Date().toISOString();
|
|
||||||
const data = body.data || { sample: 'Realtime event via DataStack API', rand: Math.random() };
|
|
||||||
|
|
||||||
// Validate security
|
// Verify either Client Bearer JWT or Admin Session Cookie
|
||||||
const val = SecurityEngine.validateEvent(clientId, eventId, data, 200);
|
const authHeader = req.headers.get('authorization');
|
||||||
|
const sessionSecret =
|
||||||
|
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
|
||||||
|
'datastack-admin-session-default-secret-32-chars';
|
||||||
|
const sessionCookie = req.cookies.get(SESSION_COOKIE)?.value;
|
||||||
|
|
||||||
|
let tokenClientId: string | null = null;
|
||||||
|
let tokenRateLimit: number | undefined;
|
||||||
|
let rawBearerToken: string | null = null;
|
||||||
|
|
||||||
|
if (authHeader && authHeader.toLowerCase().startsWith('bearer ')) {
|
||||||
|
rawBearerToken = authHeader.slice(7).trim();
|
||||||
|
const verified = AuthService.verifyClientToken(rawBearerToken);
|
||||||
|
if (!verified.valid || !verified.payload) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: verified.error || 'Invalid client JWT token.' },
|
||||||
|
{ status: 401 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
tokenClientId = verified.payload.clientId;
|
||||||
|
tokenRateLimit = verified.payload.rateLimit;
|
||||||
|
} else {
|
||||||
|
const isAdmin = Boolean(
|
||||||
|
sessionSecret &&
|
||||||
|
sessionCookie &&
|
||||||
|
(await verifyAdminSessionToken(sessionCookie, sessionSecret)),
|
||||||
|
);
|
||||||
|
if (!isAdmin) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: 'Authentication required. Provide a valid Client Bearer JWT or Admin Session.' },
|
||||||
|
{ status: 401 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const body = await req.json();
|
||||||
|
const clientId = String(
|
||||||
|
tokenClientId || body.clientId || 'py_telemetry_agent_01',
|
||||||
|
)
|
||||||
|
.trim()
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/[^a-z0-9_-]/g, '_');
|
||||||
|
|
||||||
|
const registeredClient = await SDKManager.getClientAsync(clientId);
|
||||||
|
if (registeredClient) {
|
||||||
|
if (registeredClient.status === 'revoked') {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: `Client '${clientId}' access has been revoked by administrator.` },
|
||||||
|
{ status: 403 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
rawBearerToken &&
|
||||||
|
registeredClient.token &&
|
||||||
|
registeredClient.token !== rawBearerToken
|
||||||
|
) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ error: 'Client credentials have been rotated. Use the latest token.' },
|
||||||
|
{ status: 401 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const eventId =
|
||||||
|
body.eventId ||
|
||||||
|
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||||
|
const timestamp = body.timestamp || new Date().toISOString();
|
||||||
|
const data =
|
||||||
|
body.data !== undefined
|
||||||
|
? body.data
|
||||||
|
: { sample: 'Realtime event via DataStack API', rand: Math.random() };
|
||||||
|
|
||||||
|
const effectiveClientRate =
|
||||||
|
registeredClient?.rateLimit ||
|
||||||
|
tokenRateLimit ||
|
||||||
|
settings.maxEventsPerSec ||
|
||||||
|
100;
|
||||||
|
|
||||||
|
// Validate security (idempotency, rate limit, payload size)
|
||||||
|
const val = SecurityEngine.validateEvent(
|
||||||
|
clientId,
|
||||||
|
eventId,
|
||||||
|
data,
|
||||||
|
effectiveClientRate,
|
||||||
|
);
|
||||||
if (!val.allowed) {
|
if (!val.allowed) {
|
||||||
return NextResponse.json({ error: val.reason }, { status: 429 });
|
const status = val.code === 'PAYLOAD_TOO_LARGE' ? 413 : 429;
|
||||||
|
return NextResponse.json({ error: val.reason, code: val.code }, { status });
|
||||||
}
|
}
|
||||||
|
|
||||||
const event = {
|
const event = {
|
||||||
@@ -43,7 +186,7 @@ export async function POST(req: NextRequest) {
|
|||||||
|
|
||||||
return NextResponse.json({ success: true, event });
|
return NextResponse.json({ success: true, event });
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
const message = err instanceof Error ? err.message : 'Event simulation failed';
|
const message = err instanceof Error ? err.message : 'Event ingestion failed';
|
||||||
return NextResponse.json({ error: message }, { status: 500 });
|
return NextResponse.json({ error: message }, { status: 500 });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import { DataBufferEngine } from "@/core/data/buffer";
|
||||||
import {
|
import {
|
||||||
CSVStorageManager,
|
CSVStorageManager,
|
||||||
type StorageFileMeta,
|
type StorageFileMeta,
|
||||||
@@ -107,6 +108,18 @@ function streamFile(file: StorageFileMeta) {
|
|||||||
|
|
||||||
export async function GET(request: NextRequest) {
|
export async function GET(request: NextRequest) {
|
||||||
try {
|
try {
|
||||||
|
await DataBufferEngine.flush();
|
||||||
|
const ingestionBaseUrl =
|
||||||
|
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000";
|
||||||
|
try {
|
||||||
|
await fetch(`${ingestionBaseUrl}/internal/flush`, {
|
||||||
|
method: "POST",
|
||||||
|
signal: AbortSignal.timeout(1500),
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Port 4000 may not be running in isolated single-process dev mode
|
||||||
|
}
|
||||||
|
|
||||||
const { searchParams } = new URL(request.url);
|
const { searchParams } = new URL(request.url);
|
||||||
const clientId = searchParams.get("clientId") || "all";
|
const clientId = searchParams.get("clientId") || "all";
|
||||||
const filename = searchParams.get("filename");
|
const filename = searchParams.get("filename");
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import { NextRequest, NextResponse } from "next/server";
|
|||||||
|
|
||||||
export async function GET() {
|
export async function GET() {
|
||||||
try {
|
try {
|
||||||
const settings = SettingsManager.getPublicSettings();
|
const settings = await SettingsManager.getPublicSettingsAsync();
|
||||||
return NextResponse.json({ settings });
|
return NextResponse.json({ settings });
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
const message = err instanceof Error ? err.message : "Settings read failed";
|
const message = err instanceof Error ? err.message : "Settings read failed";
|
||||||
@@ -14,9 +14,10 @@ export async function GET() {
|
|||||||
export async function POST(req: NextRequest) {
|
export async function POST(req: NextRequest) {
|
||||||
try {
|
try {
|
||||||
const body = await req.json();
|
const body = await req.json();
|
||||||
SettingsManager.updateSettings(body);
|
await SettingsManager.updateSettingsAsync(body);
|
||||||
|
const settings = await SettingsManager.getPublicSettingsAsync();
|
||||||
return NextResponse.json({
|
return NextResponse.json({
|
||||||
settings: SettingsManager.getPublicSettings(),
|
settings,
|
||||||
success: true,
|
success: true,
|
||||||
});
|
});
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
|
|||||||
@@ -23,12 +23,40 @@ export async function GET(req: NextRequest) {
|
|||||||
|
|
||||||
export async function POST(_req: NextRequest) {
|
export async function POST(_req: NextRequest) {
|
||||||
try {
|
try {
|
||||||
// Manual flush buffer to disk
|
// 1. Flush local Next.js DataBufferEngine
|
||||||
const res = await DataBufferEngine.flush();
|
const localRes = await DataBufferEngine.flush();
|
||||||
|
|
||||||
|
// 2. Also flush Port 4000 Socket.IO ingestion buffer
|
||||||
|
let remoteCount = 0;
|
||||||
|
let remoteFiles: string[] = [];
|
||||||
|
const ingestionBaseUrl = (
|
||||||
|
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
|
||||||
|
).replace(/\/$/, '');
|
||||||
|
|
||||||
|
try {
|
||||||
|
const ingestRes = await fetch(`${ingestionBaseUrl}/internal/flush`, {
|
||||||
|
method: 'POST',
|
||||||
|
cache: 'no-store',
|
||||||
|
});
|
||||||
|
if (ingestRes.ok) {
|
||||||
|
const ingestData = await ingestRes.json();
|
||||||
|
remoteCount = Number(ingestData.flushedCount) || 0;
|
||||||
|
if (Array.isArray(ingestData.filesUpdated)) {
|
||||||
|
remoteFiles = ingestData.filesUpdated;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Port 4000 server may be offline in standalone Next.js dev mode
|
||||||
|
}
|
||||||
|
|
||||||
|
const combinedFiles = Array.from(
|
||||||
|
new Set([...localRes.files, ...remoteFiles]),
|
||||||
|
);
|
||||||
|
|
||||||
return NextResponse.json({
|
return NextResponse.json({
|
||||||
success: true,
|
success: true,
|
||||||
flushedCount: res.count,
|
flushedCount: localRes.count + remoteCount,
|
||||||
filesUpdated: res.files,
|
filesUpdated: combinedFiles,
|
||||||
});
|
});
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
const message = err instanceof Error ? err.message : 'Manual flush failed';
|
const message = err instanceof Error ? err.message : 'Manual flush failed';
|
||||||
|
|||||||
@@ -6,10 +6,12 @@ export const runtime = 'nodejs';
|
|||||||
export const dynamic = 'force-dynamic';
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
export default async function LoginPage() {
|
export default async function LoginPage() {
|
||||||
if (process.env.NODE_ENV === 'production' && process.env.DATABASE_URL) {
|
if (process.env.DATABASE_URL) {
|
||||||
const isConfigured = await isAdminSetupComplete().catch(() => false);
|
const isConfigured = await isAdminSetupComplete().catch(() => false);
|
||||||
if (!isConfigured) redirect('/vcon/setup');
|
if (!isConfigured) redirect('/vcon/setup');
|
||||||
}
|
}
|
||||||
|
|
||||||
return <AdminLogin />;
|
const showDemoHint = !process.env.DATABASE_URL && process.env.NODE_ENV !== 'production';
|
||||||
|
|
||||||
|
return <AdminLogin showDemoHint={showDemoHint} />;
|
||||||
}
|
}
|
||||||
+15
-1
@@ -38,7 +38,9 @@ export default function SdkGuidePage() {
|
|||||||
const list = setCached('vcon:clients', data.clients || []);
|
const list = setCached('vcon:clients', data.clients || []);
|
||||||
setClients(list);
|
setClients(list);
|
||||||
if (list.length > 0) {
|
if (list.length > 0) {
|
||||||
setSelectedClientId((prev) => prev || list[0].clientId);
|
setSelectedClientId((prev) =>
|
||||||
|
list.some((c: any) => c.clientId === prev) ? prev : list[0].clientId
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
@@ -84,6 +86,17 @@ if client.connect():
|
|||||||
<PageShell
|
<PageShell
|
||||||
title="Python SDK"
|
title="Python SDK"
|
||||||
actions={
|
actions={
|
||||||
|
<>
|
||||||
|
{activeClient?.token && (
|
||||||
|
<Button
|
||||||
|
size="xs"
|
||||||
|
variant="secondary"
|
||||||
|
icon="Key"
|
||||||
|
onClick={() => copySnippet(activeClient.token, 'Client JWT token')}
|
||||||
|
>
|
||||||
|
Copy Token
|
||||||
|
</Button>
|
||||||
|
)}
|
||||||
<Button
|
<Button
|
||||||
size="xs"
|
size="xs"
|
||||||
variant="secondary"
|
variant="secondary"
|
||||||
@@ -92,6 +105,7 @@ if client.connect():
|
|||||||
>
|
>
|
||||||
Copy Code
|
Copy Code
|
||||||
</Button>
|
</Button>
|
||||||
|
</>
|
||||||
}
|
}
|
||||||
>
|
>
|
||||||
<StatStrip
|
<StatStrip
|
||||||
|
|||||||
@@ -95,7 +95,10 @@ export default function StoragePage() {
|
|||||||
label: 'Persisted rows',
|
label: 'Persisted rows',
|
||||||
value: stats?.totalRows ? stats.totalRows.toLocaleString() : 0,
|
value: stats?.totalRows ? stats.totalRows.toLocaleString() : 0,
|
||||||
},
|
},
|
||||||
{ label: 'Rotation', value: '100MB / Daily' },
|
{
|
||||||
|
label: 'Rotation',
|
||||||
|
value: stats?.rotationRule || `${stats?.rotationMb ?? 100}MB / Daily`,
|
||||||
|
},
|
||||||
]}
|
]}
|
||||||
/>
|
/>
|
||||||
|
|
||||||
|
|||||||
@@ -14,9 +14,13 @@ function safeNextPath() {
|
|||||||
: '/vcon/dashboard';
|
: '/vcon/dashboard';
|
||||||
}
|
}
|
||||||
|
|
||||||
export function AdminLogin() {
|
export function AdminLogin({
|
||||||
const [email, setEmail] = useState('admin@datastack.dev');
|
showDemoHint = false,
|
||||||
const [password, setPassword] = useState('admin12345678');
|
}: {
|
||||||
|
showDemoHint?: boolean;
|
||||||
|
}) {
|
||||||
|
const [email, setEmail] = useState(showDemoHint ? 'admin@datastack.dev' : '');
|
||||||
|
const [password, setPassword] = useState(showDemoHint ? 'admin12345678' : '');
|
||||||
const [remember, setRemember] = useState(true);
|
const [remember, setRemember] = useState(true);
|
||||||
const [showPassword, setShowPassword] = useState(false);
|
const [showPassword, setShowPassword] = useState(false);
|
||||||
const [isSubmitting, setIsSubmitting] = useState(false);
|
const [isSubmitting, setIsSubmitting] = useState(false);
|
||||||
@@ -144,6 +148,7 @@ export function AdminLogin() {
|
|||||||
Sign in
|
Sign in
|
||||||
</Button>
|
</Button>
|
||||||
|
|
||||||
|
{showDemoHint && (
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
onClick={() => {
|
onClick={() => {
|
||||||
@@ -154,6 +159,7 @@ export function AdminLogin() {
|
|||||||
>
|
>
|
||||||
admin@datastack.dev / admin12345678
|
admin@datastack.dev / admin12345678
|
||||||
</button>
|
</button>
|
||||||
|
)}
|
||||||
</form>
|
</form>
|
||||||
|
|
||||||
<footer className="flex items-center justify-end border-t border-[var(--border)] px-4 py-2 text-[10.5px]">
|
<footer className="flex items-center justify-end border-t border-[var(--border)] px-4 py-2 text-[10.5px]">
|
||||||
|
|||||||
@@ -26,8 +26,8 @@ export function AdminSetup({
|
|||||||
event.preventDefault();
|
event.preventDefault();
|
||||||
setErrorMessage('');
|
setErrorMessage('');
|
||||||
|
|
||||||
if (password.length < 12) {
|
if (password.length < 8) {
|
||||||
setErrorMessage('Password must be at least 12 characters.');
|
setErrorMessage('Password must be at least 8 characters.');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (password !== confirmPassword) {
|
if (password !== confirmPassword) {
|
||||||
@@ -94,7 +94,7 @@ export function AdminSetup({
|
|||||||
value={password}
|
value={password}
|
||||||
onChange={(event) => setPassword(event.target.value)}
|
onChange={(event) => setPassword(event.target.value)}
|
||||||
compact
|
compact
|
||||||
minLength={12}
|
minLength={8}
|
||||||
required
|
required
|
||||||
/>
|
/>
|
||||||
<Input
|
<Input
|
||||||
@@ -104,7 +104,7 @@ export function AdminSetup({
|
|||||||
value={confirmPassword}
|
value={confirmPassword}
|
||||||
onChange={(event) => setConfirmPassword(event.target.value)}
|
onChange={(event) => setConfirmPassword(event.target.value)}
|
||||||
compact
|
compact
|
||||||
minLength={12}
|
minLength={8}
|
||||||
required
|
required
|
||||||
/>
|
/>
|
||||||
<Input
|
<Input
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
|||||||
const { success, error } = useToast();
|
const { success, error } = useToast();
|
||||||
const [credentials, setCredentials] = useState<ClientCredentials | null>(null);
|
const [credentials, setCredentials] = useState<ClientCredentials | null>(null);
|
||||||
const [confirmAction, setConfirmAction] = useState<{
|
const [confirmAction, setConfirmAction] = useState<{
|
||||||
type: 'rotate' | 'revoke';
|
type: 'rotate' | 'revoke' | 'delete';
|
||||||
client: ClientItem;
|
client: ClientItem;
|
||||||
} | null>(null);
|
} | null>(null);
|
||||||
const [processing, setProcessing] = useState(false);
|
const [processing, setProcessing] = useState(false);
|
||||||
@@ -86,6 +86,24 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const executeDeleteClient = async (client: ClientItem) => {
|
||||||
|
setProcessing(true);
|
||||||
|
try {
|
||||||
|
const response = await fetch(`/api/clients/${encodeURIComponent(client.clientId)}`, {
|
||||||
|
method: 'DELETE',
|
||||||
|
});
|
||||||
|
const data = await response.json();
|
||||||
|
if (!response.ok) throw new Error(data.error || 'Unable to delete client');
|
||||||
|
setConfirmAction(null);
|
||||||
|
success('Client deleted', `${client.clientId} has been permanently removed.`);
|
||||||
|
await onStatusChange();
|
||||||
|
} catch (deleteError) {
|
||||||
|
error('Client deletion failed', deleteError instanceof Error ? deleteError.message : 'Unknown error');
|
||||||
|
} finally {
|
||||||
|
setProcessing(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const toggleStatus = (client: ClientItem) => {
|
const toggleStatus = (client: ClientItem) => {
|
||||||
if (client.status === 'active') {
|
if (client.status === 'active') {
|
||||||
setConfirmAction({ type: 'revoke', client });
|
setConfirmAction({ type: 'revoke', client });
|
||||||
@@ -98,6 +116,8 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
|||||||
if (!confirmAction) return;
|
if (!confirmAction) return;
|
||||||
if (confirmAction.type === 'rotate') {
|
if (confirmAction.type === 'rotate') {
|
||||||
await executeRotateKey(confirmAction.client);
|
await executeRotateKey(confirmAction.client);
|
||||||
|
} else if (confirmAction.type === 'delete') {
|
||||||
|
await executeDeleteClient(confirmAction.client);
|
||||||
} else {
|
} else {
|
||||||
await executeStatusChange(confirmAction.client, 'revoked');
|
await executeStatusChange(confirmAction.client, 'revoked');
|
||||||
}
|
}
|
||||||
@@ -178,12 +198,22 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
|||||||
type="button"
|
type="button"
|
||||||
variant="icon"
|
variant="icon"
|
||||||
size="icon"
|
size="icon"
|
||||||
icon={client.status === 'active' ? 'Trash2' : 'ShieldCheck'}
|
icon={client.status === 'active' ? 'ShieldOff' : 'ShieldCheck'}
|
||||||
className={`h-6 w-6 border border-transparent ${client.status === 'active' ? 'hover:border-[var(--danger)] hover:text-[var(--danger)]' : 'hover:border-[var(--border)] hover:bg-[var(--surface-2)]'}`}
|
className={`h-6 w-6 border border-transparent ${client.status === 'active' ? 'hover:border-[var(--warning)] hover:text-[var(--warning)]' : 'hover:border-[var(--border)] hover:bg-[var(--surface-2)]'}`}
|
||||||
title={`${client.status === 'active' ? 'Revoke' : 'Reactivate'} ${client.clientId}`}
|
title={`${client.status === 'active' ? 'Revoke' : 'Reactivate'} ${client.clientId}`}
|
||||||
aria-label={`${client.status === 'active' ? 'Revoke' : 'Reactivate'} ${client.clientId}`}
|
aria-label={`${client.status === 'active' ? 'Revoke' : 'Reactivate'} ${client.clientId}`}
|
||||||
onClick={() => toggleStatus(client)}
|
onClick={() => toggleStatus(client)}
|
||||||
/>
|
/>
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
variant="icon"
|
||||||
|
size="icon"
|
||||||
|
icon="Trash2"
|
||||||
|
className="h-6 w-6 border border-transparent hover:border-[var(--danger)] hover:text-[var(--danger)]"
|
||||||
|
title={`Delete ${client.clientId}`}
|
||||||
|
aria-label={`Delete ${client.clientId}`}
|
||||||
|
onClick={() => setConfirmAction({ type: 'delete', client })}
|
||||||
|
/>
|
||||||
</div>
|
</div>
|
||||||
</TableCell>
|
</TableCell>
|
||||||
</TableRow>
|
</TableRow>
|
||||||
@@ -197,7 +227,13 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
|||||||
<Modal
|
<Modal
|
||||||
isOpen={Boolean(confirmAction)}
|
isOpen={Boolean(confirmAction)}
|
||||||
onClose={() => !processing && setConfirmAction(null)}
|
onClose={() => !processing && setConfirmAction(null)}
|
||||||
title={confirmAction?.type === 'rotate' ? 'Rotate credentials' : 'Revoke access'}
|
title={
|
||||||
|
confirmAction?.type === 'rotate'
|
||||||
|
? 'Rotate credentials'
|
||||||
|
: confirmAction?.type === 'delete'
|
||||||
|
? 'Delete client'
|
||||||
|
: 'Revoke access'
|
||||||
|
}
|
||||||
maxWidth="xs"
|
maxWidth="xs"
|
||||||
footer={
|
footer={
|
||||||
<>
|
<>
|
||||||
@@ -206,12 +242,16 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
|||||||
</Button>
|
</Button>
|
||||||
<Button
|
<Button
|
||||||
type="button"
|
type="button"
|
||||||
variant={confirmAction?.type === 'revoke' ? 'danger' : 'primary'}
|
variant={confirmAction?.type === 'rotate' ? 'primary' : 'danger'}
|
||||||
size="xs"
|
size="xs"
|
||||||
loading={processing}
|
loading={processing}
|
||||||
onClick={handleConfirm}
|
onClick={handleConfirm}
|
||||||
>
|
>
|
||||||
{confirmAction?.type === 'rotate' ? 'Rotate' : 'Revoke'}
|
{confirmAction?.type === 'rotate'
|
||||||
|
? 'Rotate'
|
||||||
|
: confirmAction?.type === 'delete'
|
||||||
|
? 'Delete'
|
||||||
|
: 'Revoke'}
|
||||||
</Button>
|
</Button>
|
||||||
</>
|
</>
|
||||||
}
|
}
|
||||||
@@ -219,6 +259,8 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
|||||||
<p className="text-[12px] text-[var(--text-muted)]">
|
<p className="text-[12px] text-[var(--text-muted)]">
|
||||||
{confirmAction?.type === 'rotate'
|
{confirmAction?.type === 'rotate'
|
||||||
? `Rotate credentials for ${confirmAction.client.clientId}? Existing credentials will stop working.`
|
? `Rotate credentials for ${confirmAction.client.clientId}? Existing credentials will stop working.`
|
||||||
|
: confirmAction?.type === 'delete'
|
||||||
|
? `Permanently delete client ${confirmAction.client.clientId}? This removes the client registration and credentials.`
|
||||||
: `Revoke access for ${confirmAction?.client.clientId || ''}?`}
|
: `Revoke access for ${confirmAction?.client.clientId || ''}?`}
|
||||||
</p>
|
</p>
|
||||||
</Modal>
|
</Modal>
|
||||||
|
|||||||
@@ -57,10 +57,10 @@ export function ExportWorkspace() {
|
|||||||
getCached<StorageFile[]>('vcon:storage-files', [])
|
getCached<StorageFile[]>('vcon:storage-files', [])
|
||||||
);
|
);
|
||||||
const [selectedClient, setSelectedClient] = useState('all');
|
const [selectedClient, setSelectedClient] = useState('all');
|
||||||
const [fromDate, setFromDate] = useState(() => dateString(new Date()));
|
const [fromDate, setFromDate] = useState('');
|
||||||
const [toDate, setToDate] = useState(() => dateString(new Date()));
|
const [toDate, setToDate] = useState('');
|
||||||
const [format, setFormat] = useState<ExportFormat>('csv');
|
const [format, setFormat] = useState<ExportFormat>('csv');
|
||||||
const [preset, setPreset] = useState<RangePreset>('today');
|
const [preset, setPreset] = useState<RangePreset>('all');
|
||||||
const [loading, setLoading] = useState(
|
const [loading, setLoading] = useState(
|
||||||
() => getCached<StorageFile[]>('vcon:storage-files', []).length === 0
|
() => getCached<StorageFile[]>('vcon:storage-files', []).length === 0
|
||||||
);
|
);
|
||||||
|
|||||||
+114
-9
@@ -1,28 +1,133 @@
|
|||||||
|
import { Pool } from "pg";
|
||||||
import { SettingsManager } from "../adminSettings/settingsManager";
|
import { SettingsManager } from "../adminSettings/settingsManager";
|
||||||
import { SDKManager } from "../sdk/sdkManager";
|
import { SDKManager } from "../sdk/sdkManager";
|
||||||
|
|
||||||
|
type CoreDatabaseGlobals = typeof globalThis & {
|
||||||
|
__dataStackCorePool?: Pool | null;
|
||||||
|
__dataStackCoreSchema?: Promise<void>;
|
||||||
|
};
|
||||||
|
|
||||||
|
const globals = globalThis as CoreDatabaseGlobals;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Database Module (PostgreSQL + Prisma abstraction)
|
* Database Module (PostgreSQL Persistence Engine)
|
||||||
* CRITICAL RULE: PostgreSQL stores strictly administrative metadata,
|
* CRITICAL RULE: PostgreSQL stores strictly administrative metadata,
|
||||||
* client registrations, and configuration. Collected data is NEVER stored in PostgreSQL.
|
* client registrations (client_sdks), and configuration (admin_settings).
|
||||||
|
* Collected telemetry streams are NEVER stored in PostgreSQL.
|
||||||
*/
|
*/
|
||||||
export class DatabaseEngine {
|
export class DatabaseEngine {
|
||||||
public static isConnected = true;
|
public static getPool(): Pool | null {
|
||||||
|
const connectionString = process.env.DATABASE_URL;
|
||||||
|
if (!connectionString) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (globals.__dataStackCorePool === undefined) {
|
||||||
|
try {
|
||||||
|
globals.__dataStackCorePool = new Pool({
|
||||||
|
connectionString,
|
||||||
|
connectionTimeoutMillis: 3000,
|
||||||
|
idleTimeoutMillis: 30000,
|
||||||
|
max: 5,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
globals.__dataStackCorePool = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return globals.__dataStackCorePool ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async ensureSchema(): Promise<boolean> {
|
||||||
|
const pool = this.getPool();
|
||||||
|
if (!pool) return false;
|
||||||
|
|
||||||
|
globals.__dataStackCoreSchema ??= Promise.all([
|
||||||
|
pool.query(`
|
||||||
|
CREATE TABLE IF NOT EXISTS client_sdks (
|
||||||
|
client_id TEXT PRIMARY KEY,
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
secret TEXT NOT NULL,
|
||||||
|
token TEXT NOT NULL,
|
||||||
|
rate_limit INTEGER NOT NULL DEFAULT 100,
|
||||||
|
status TEXT NOT NULL DEFAULT 'active',
|
||||||
|
total_events BIGINT NOT NULL DEFAULT 0,
|
||||||
|
last_active TIMESTAMPTZ,
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||||
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||||
|
)
|
||||||
|
`),
|
||||||
|
pool.query(`
|
||||||
|
CREATE TABLE IF NOT EXISTS admin_settings (
|
||||||
|
id TEXT PRIMARY KEY DEFAULT 'default',
|
||||||
|
server_port INTEGER NOT NULL DEFAULT 4000,
|
||||||
|
web_port INTEGER NOT NULL DEFAULT 3000,
|
||||||
|
max_events_per_sec INTEGER NOT NULL DEFAULT 100,
|
||||||
|
max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
|
||||||
|
batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
|
||||||
|
batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
|
||||||
|
file_rotation_mb INTEGER NOT NULL DEFAULT 100,
|
||||||
|
jwt_secret TEXT NOT NULL,
|
||||||
|
enable_idempotency BOOLEAN NOT NULL DEFAULT true,
|
||||||
|
auto_prune_days INTEGER NOT NULL DEFAULT 30,
|
||||||
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||||
|
)
|
||||||
|
`),
|
||||||
|
]).then(() => undefined);
|
||||||
|
|
||||||
|
try {
|
||||||
|
await globals.__dataStackCoreSchema;
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
globals.__dataStackCoreSchema = undefined;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public static async getHealth() {
|
public static async getHealth() {
|
||||||
|
const pool = this.getPool();
|
||||||
|
if (!pool) {
|
||||||
return {
|
return {
|
||||||
status: "healthy",
|
status: "healthy" as const,
|
||||||
type: "PostgreSQL + Prisma Schema Managed",
|
storageMode: "In-Memory" as const,
|
||||||
database: "datastack_db",
|
type: "In-Memory Fallback (DATABASE_URL not set)",
|
||||||
|
database: "memory",
|
||||||
role: "Metadata & SDK Registrations Only",
|
role: "Metadata & SDK Registrations Only",
|
||||||
collectedDataStorage: "Strictly CSV (/storage/data)",
|
collectedDataStorage: "Strictly CSV (/storage/data)",
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
public static async syncState() {
|
try {
|
||||||
|
await this.ensureSchema();
|
||||||
|
await pool.query("SELECT 1");
|
||||||
return {
|
return {
|
||||||
clientsCount: SDKManager.listClients().length,
|
status: "healthy" as const,
|
||||||
settings: SettingsManager.getPublicSettings(),
|
storageMode: "PostgreSQL" as const,
|
||||||
|
type: "PostgreSQL Managed Schema",
|
||||||
|
database: "datastack_db",
|
||||||
|
role: "Metadata & SDK Registrations Only",
|
||||||
|
collectedDataStorage: "Strictly CSV (/storage/data)",
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
return {
|
||||||
|
status: "degraded" as const,
|
||||||
|
storageMode: "In-Memory" as const,
|
||||||
|
type: "PostgreSQL Unreachable (In-Memory Fallback Active)",
|
||||||
|
database: "datastack_db",
|
||||||
|
role: "Metadata & SDK Registrations Only",
|
||||||
|
collectedDataStorage: "Strictly CSV (/storage/data)",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async syncState() {
|
||||||
|
const [clients, settings] = await Promise.all([
|
||||||
|
SDKManager.listClientsAsync(),
|
||||||
|
SettingsManager.getPublicSettingsAsync(),
|
||||||
|
]);
|
||||||
|
return {
|
||||||
|
clientsCount: clients.length,
|
||||||
|
settings,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,100 +0,0 @@
|
|||||||
import express from 'express';
|
|
||||||
import http from 'http';
|
|
||||||
import cors from 'cors';
|
|
||||||
import { Server as SocketIOServer } from 'socket.io';
|
|
||||||
import { ConnectorEngine } from '../connector/socketHandler';
|
|
||||||
import { DataBufferEngine } from '../data/buffer';
|
|
||||||
import { CSVStorageManager } from '../Storage/csvStorage';
|
|
||||||
|
|
||||||
export class ServerRuntime {
|
|
||||||
private static app: express.Express | null = null;
|
|
||||||
private static httpServer: http.Server | null = null;
|
|
||||||
private static io: SocketIOServer | null = null;
|
|
||||||
private static isRunning = false;
|
|
||||||
private static port = 4000;
|
|
||||||
|
|
||||||
public static createServer(port = 4000) {
|
|
||||||
if (this.isRunning && this.httpServer) {
|
|
||||||
return { app: this.app, server: this.httpServer, io: this.io };
|
|
||||||
}
|
|
||||||
|
|
||||||
this.port = port;
|
|
||||||
const app = express();
|
|
||||||
app.use(cors({ origin: '*' }));
|
|
||||||
app.use(express.json({ limit: '10mb' }));
|
|
||||||
|
|
||||||
// Health route
|
|
||||||
app.get('/health', (_req, res) => {
|
|
||||||
res.json({
|
|
||||||
status: 'online',
|
|
||||||
service: 'DataStack Express + Socket.IO Intake',
|
|
||||||
port: this.port,
|
|
||||||
activeSockets: ConnectorEngine.getActiveConnectionsCount(),
|
|
||||||
uptime: process.uptime(),
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
// HTTP fallback intake endpoint for non-socket clients
|
|
||||||
app.post('/api/intake', (req, res) => {
|
|
||||||
const authHeader = req.headers.authorization;
|
|
||||||
if (!authHeader) {
|
|
||||||
res.status(401).json({ error: 'Missing Authorization header' });
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
// Can sync via REST as well if needed
|
|
||||||
res.json({ success: true, message: 'REST sync supported' });
|
|
||||||
});
|
|
||||||
|
|
||||||
const httpServer = http.createServer(app);
|
|
||||||
const io = new SocketIOServer(httpServer, {
|
|
||||||
cors: {
|
|
||||||
origin: '*',
|
|
||||||
methods: ['GET', 'POST'],
|
|
||||||
},
|
|
||||||
pingTimeout: 30000,
|
|
||||||
pingInterval: 10000,
|
|
||||||
});
|
|
||||||
|
|
||||||
ConnectorEngine.initialize(io);
|
|
||||||
|
|
||||||
this.app = app;
|
|
||||||
this.httpServer = httpServer;
|
|
||||||
this.io = io;
|
|
||||||
this.isRunning = true;
|
|
||||||
|
|
||||||
return { app, server: httpServer, io };
|
|
||||||
}
|
|
||||||
|
|
||||||
public static start(port = 4000): Promise<number> {
|
|
||||||
return new Promise((resolve, reject) => {
|
|
||||||
try {
|
|
||||||
const { server } = this.createServer(port);
|
|
||||||
server.listen(port, () => {
|
|
||||||
resolve(port);
|
|
||||||
});
|
|
||||||
server.on('error', (err: NodeJS.ErrnoException) => {
|
|
||||||
if (err.code === 'EADDRINUSE') {
|
|
||||||
// Already listening or address in use
|
|
||||||
resolve(port);
|
|
||||||
} else {
|
|
||||||
reject(err);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
} catch (e) {
|
|
||||||
reject(e);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
public static getStatus() {
|
|
||||||
return {
|
|
||||||
isRunning: this.isRunning,
|
|
||||||
port: this.port,
|
|
||||||
activeSockets: ConnectorEngine.getActiveConnectionsCount(),
|
|
||||||
bufferSize: DataBufferEngine.getBufferStats().currentBufferSize,
|
|
||||||
storageStats: CSVStorageManager.getStorageStats(),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export default ServerRuntime;
|
|
||||||
@@ -167,6 +167,38 @@ export class CSVStorageManager {
|
|||||||
return results.sort((a, b) => b.modifiedAt.getTime() - a.modifiedAt.getTime());
|
return results.sort((a, b) => b.modifiedAt.getTime() - a.modifiedAt.getTime());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Prune CSV partitions older than autoPruneDays
|
||||||
|
*/
|
||||||
|
public static pruneExpiredPartitions(autoPruneDays: number): number {
|
||||||
|
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
const cutoffDate = new Date(
|
||||||
|
Date.now() - Math.floor(autoPruneDays) * 86_400_000,
|
||||||
|
)
|
||||||
|
.toISOString()
|
||||||
|
.split('T')[0];
|
||||||
|
|
||||||
|
let prunedCount = 0;
|
||||||
|
const files = this.listAllFiles();
|
||||||
|
for (const file of files) {
|
||||||
|
if (/^\d{4}-\d{2}-\d{2}$/.test(file.date) && file.date < cutoffDate) {
|
||||||
|
try {
|
||||||
|
if (fs.existsSync(file.filePath)) {
|
||||||
|
fs.unlinkSync(file.filePath);
|
||||||
|
prunedCount += 1;
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Ignore file removal error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return prunedCount;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get aggregate storage statistics
|
* Get aggregate storage statistics
|
||||||
*/
|
*/
|
||||||
@@ -175,6 +207,10 @@ export class CSVStorageManager {
|
|||||||
const totalBytes = files.reduce((acc, f) => acc + f.sizeBytes, 0);
|
const totalBytes = files.reduce((acc, f) => acc + f.sizeBytes, 0);
|
||||||
const totalRows = files.reduce((acc, f) => acc + f.linesCount, 0);
|
const totalRows = files.reduce((acc, f) => acc + f.linesCount, 0);
|
||||||
const uniqueClients = new Set(files.map((f) => f.clientId)).size;
|
const uniqueClients = new Set(files.map((f) => f.clientId)).size;
|
||||||
|
const rotationMb = Math.max(
|
||||||
|
1,
|
||||||
|
Math.round(this.maxFileSizeBytes / (1024 * 1024)),
|
||||||
|
);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
totalFiles: files.length,
|
totalFiles: files.length,
|
||||||
@@ -183,7 +219,8 @@ export class CSVStorageManager {
|
|||||||
totalRows,
|
totalRows,
|
||||||
uniqueClients,
|
uniqueClients,
|
||||||
storageDirectory: this.baseStorageDir,
|
storageDirectory: this.baseStorageDir,
|
||||||
rotationRule: 'Per-day OR per-100MB rotation',
|
rotationMb,
|
||||||
|
rotationRule: `${rotationMb}MB / Daily`,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
import { AuthService } from "../auth/jwt";
|
import { AuthService } from "../auth/jwt";
|
||||||
import { DataBufferEngine } from "../data/buffer";
|
import { DataBufferEngine } from "../data/buffer";
|
||||||
|
import { DatabaseEngine } from "../Database/database";
|
||||||
|
import { SDKManager } from "../sdk/sdkManager";
|
||||||
import { SecurityEngine } from "../Security/security";
|
import { SecurityEngine } from "../Security/security";
|
||||||
import { CSVStorageManager } from "../Storage/csvStorage";
|
import { CSVStorageManager } from "../Storage/csvStorage";
|
||||||
|
|
||||||
@@ -17,10 +19,17 @@ export interface AdminSettings {
|
|||||||
updatedAt: string;
|
updatedAt: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class SettingsManager {
|
type SettingsGlobals = typeof globalThis & {
|
||||||
private static settings: AdminSettings = {
|
__dataStackAdminSettings?: AdminSettings;
|
||||||
serverPort: 4000,
|
__dataStackSettingsHydrated?: boolean;
|
||||||
webPort: 3000,
|
};
|
||||||
|
|
||||||
|
const globals = globalThis as SettingsGlobals;
|
||||||
|
|
||||||
|
function getDefaultSettings(): AdminSettings {
|
||||||
|
return {
|
||||||
|
serverPort: Number(process.env.PORT_SOCKET) || 4000,
|
||||||
|
webPort: Number(process.env.PORT) || 3000,
|
||||||
maxEventsPerSec: 100,
|
maxEventsPerSec: 100,
|
||||||
maxPayloadSizeKb: 100,
|
maxPayloadSizeKb: 100,
|
||||||
batchFlushThreshold: 500,
|
batchFlushThreshold: 500,
|
||||||
@@ -33,13 +42,160 @@ export class SettingsManager {
|
|||||||
autoPruneDays: 30,
|
autoPruneDays: 30,
|
||||||
updatedAt: new Date().toISOString(),
|
updatedAt: new Date().toISOString(),
|
||||||
};
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function getSettingsState(): AdminSettings {
|
||||||
|
if (!globals.__dataStackAdminSettings) {
|
||||||
|
globals.__dataStackAdminSettings = getDefaultSettings();
|
||||||
|
}
|
||||||
|
return globals.__dataStackAdminSettings;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class SettingsManager {
|
||||||
|
private static applyRuntimeSettings(settings: AdminSettings) {
|
||||||
|
SecurityEngine.maxEventsPerSec = settings.maxEventsPerSec;
|
||||||
|
SecurityEngine.enableIdempotency = settings.enableIdempotency;
|
||||||
|
SecurityEngine.maxPayloadSizeBytes = settings.maxPayloadSizeKb * 1024;
|
||||||
|
DataBufferEngine.batchFlushThreshold = settings.batchFlushThreshold;
|
||||||
|
DataBufferEngine.setFlushInterval(settings.batchFlushIntervalMs);
|
||||||
|
if (settings.jwtSecret) {
|
||||||
|
AuthService.setSecret(settings.jwtSecret);
|
||||||
|
}
|
||||||
|
CSVStorageManager.maxFileSizeBytes = settings.fileRotationMb * 1024 * 1024;
|
||||||
|
CSVStorageManager.pruneExpiredPartitions(settings.autoPruneDays);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async syncFromDatabase(): Promise<AdminSettings | null> {
|
||||||
|
const pool = DatabaseEngine.getPool();
|
||||||
|
if (!pool) return null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const ready = await DatabaseEngine.ensureSchema();
|
||||||
|
if (!ready) return null;
|
||||||
|
|
||||||
|
const result = await pool.query(
|
||||||
|
`SELECT server_port, web_port, max_events_per_sec, max_payload_size_kb,
|
||||||
|
batch_flush_threshold, batch_flush_interval_ms, file_rotation_mb,
|
||||||
|
jwt_secret, enable_idempotency, auto_prune_days, updated_at
|
||||||
|
FROM admin_settings
|
||||||
|
WHERE id = 'default'`,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (result.rowCount && result.rowCount > 0) {
|
||||||
|
const row = result.rows[0];
|
||||||
|
const loaded: AdminSettings = {
|
||||||
|
serverPort: Number(row.server_port) || 4000,
|
||||||
|
webPort: Number(row.web_port) || 3000,
|
||||||
|
maxEventsPerSec: Number(row.max_events_per_sec) || 100,
|
||||||
|
maxPayloadSizeKb: Number(row.max_payload_size_kb) || 100,
|
||||||
|
batchFlushThreshold: Number(row.batch_flush_threshold) || 500,
|
||||||
|
batchFlushIntervalMs: Number(row.batch_flush_interval_ms) || 2000,
|
||||||
|
fileRotationMb: Number(row.file_rotation_mb) || 100,
|
||||||
|
jwtSecret:
|
||||||
|
String(row.jwt_secret || "") ||
|
||||||
|
process.env.DATASTACK_JWT_SECRET ||
|
||||||
|
"datastack-jwt-default-secret-key-32-chars-long",
|
||||||
|
enableIdempotency: Boolean(row.enable_idempotency),
|
||||||
|
autoPruneDays: Number(row.auto_prune_days) || 30,
|
||||||
|
updatedAt: row.updated_at
|
||||||
|
? new Date(row.updated_at as string | Date).toISOString()
|
||||||
|
: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
globals.__dataStackAdminSettings = loaded;
|
||||||
|
globals.__dataStackSettingsHydrated = true;
|
||||||
|
this.applyRuntimeSettings(loaded);
|
||||||
|
return loaded;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize default row in PostgreSQL if not present yet
|
||||||
|
const defaults = getSettingsState();
|
||||||
|
await this.persistSettingsToDb(defaults);
|
||||||
|
globals.__dataStackSettingsHydrated = true;
|
||||||
|
return defaults;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async persistSettingsToDb(
|
||||||
|
settings: AdminSettings,
|
||||||
|
): Promise<void> {
|
||||||
|
const pool = DatabaseEngine.getPool();
|
||||||
|
if (!pool) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
await DatabaseEngine.ensureSchema();
|
||||||
|
await pool.query(
|
||||||
|
`INSERT INTO admin_settings (
|
||||||
|
id, server_port, web_port, max_events_per_sec, max_payload_size_kb,
|
||||||
|
batch_flush_threshold, batch_flush_interval_ms, file_rotation_mb,
|
||||||
|
jwt_secret, enable_idempotency, auto_prune_days, updated_at
|
||||||
|
) VALUES ('default', $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, NOW())
|
||||||
|
ON CONFLICT (id) DO UPDATE SET
|
||||||
|
server_port = EXCLUDED.server_port,
|
||||||
|
web_port = EXCLUDED.web_port,
|
||||||
|
max_events_per_sec = EXCLUDED.max_events_per_sec,
|
||||||
|
max_payload_size_kb = EXCLUDED.max_payload_size_kb,
|
||||||
|
batch_flush_threshold = EXCLUDED.batch_flush_threshold,
|
||||||
|
batch_flush_interval_ms = EXCLUDED.batch_flush_interval_ms,
|
||||||
|
file_rotation_mb = EXCLUDED.file_rotation_mb,
|
||||||
|
jwt_secret = EXCLUDED.jwt_secret,
|
||||||
|
enable_idempotency = EXCLUDED.enable_idempotency,
|
||||||
|
auto_prune_days = EXCLUDED.auto_prune_days,
|
||||||
|
updated_at = NOW()`,
|
||||||
|
[
|
||||||
|
settings.serverPort,
|
||||||
|
settings.webPort,
|
||||||
|
settings.maxEventsPerSec,
|
||||||
|
settings.maxPayloadSizeKb,
|
||||||
|
settings.batchFlushThreshold,
|
||||||
|
settings.batchFlushIntervalMs,
|
||||||
|
settings.fileRotationMb,
|
||||||
|
settings.jwtSecret,
|
||||||
|
settings.enableIdempotency,
|
||||||
|
settings.autoPruneDays,
|
||||||
|
],
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
// Ignore DB failure; in-memory state remains active
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public static getSettings(): AdminSettings {
|
public static getSettings(): AdminSettings {
|
||||||
return { ...this.settings };
|
if (!globals.__dataStackSettingsHydrated) {
|
||||||
|
void this.syncFromDatabase();
|
||||||
|
}
|
||||||
|
return { ...getSettingsState() };
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async getSettingsAsync(): Promise<AdminSettings> {
|
||||||
|
const fromDb = await this.syncFromDatabase();
|
||||||
|
if (fromDb) {
|
||||||
|
return { ...fromDb };
|
||||||
|
}
|
||||||
|
return { ...getSettingsState() };
|
||||||
}
|
}
|
||||||
|
|
||||||
public static getPublicSettings(): Omit<AdminSettings, "jwtSecret"> {
|
public static getPublicSettings(): Omit<AdminSettings, "jwtSecret"> {
|
||||||
const settings = this.settings;
|
const settings = this.getSettings();
|
||||||
|
return {
|
||||||
|
serverPort: settings.serverPort,
|
||||||
|
webPort: settings.webPort,
|
||||||
|
maxEventsPerSec: settings.maxEventsPerSec,
|
||||||
|
maxPayloadSizeKb: settings.maxPayloadSizeKb,
|
||||||
|
batchFlushThreshold: settings.batchFlushThreshold,
|
||||||
|
batchFlushIntervalMs: settings.batchFlushIntervalMs,
|
||||||
|
fileRotationMb: settings.fileRotationMb,
|
||||||
|
enableIdempotency: settings.enableIdempotency,
|
||||||
|
autoPruneDays: settings.autoPruneDays,
|
||||||
|
updatedAt: settings.updatedAt,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async getPublicSettingsAsync(): Promise<
|
||||||
|
Omit<AdminSettings, "jwtSecret">
|
||||||
|
> {
|
||||||
|
const settings = await this.getSettingsAsync();
|
||||||
return {
|
return {
|
||||||
serverPort: settings.serverPort,
|
serverPort: settings.serverPort,
|
||||||
webPort: settings.webPort,
|
webPort: settings.webPort,
|
||||||
@@ -55,36 +211,41 @@ export class SettingsManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public static updateSettings(updates: Partial<AdminSettings>): AdminSettings {
|
public static updateSettings(updates: Partial<AdminSettings>): AdminSettings {
|
||||||
this.settings = {
|
const current = getSettingsState();
|
||||||
...this.settings,
|
const jwtSecretChanged =
|
||||||
|
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
|
||||||
|
const nextSettings: AdminSettings = {
|
||||||
|
...current,
|
||||||
...updates,
|
...updates,
|
||||||
updatedAt: new Date().toISOString(),
|
updatedAt: new Date().toISOString(),
|
||||||
};
|
};
|
||||||
|
globals.__dataStackAdminSettings = nextSettings;
|
||||||
// Apply live updates to subsystems
|
this.applyRuntimeSettings(nextSettings);
|
||||||
if (updates.maxEventsPerSec !== undefined) {
|
if (jwtSecretChanged) {
|
||||||
SecurityEngine.maxEventsPerSec = updates.maxEventsPerSec;
|
SDKManager.resignAllClientTokens();
|
||||||
}
|
}
|
||||||
if (updates.enableIdempotency !== undefined) {
|
void this.persistSettingsToDb(nextSettings);
|
||||||
SecurityEngine.enableIdempotency = updates.enableIdempotency;
|
return { ...nextSettings };
|
||||||
}
|
|
||||||
if (updates.maxPayloadSizeKb !== undefined) {
|
|
||||||
SecurityEngine.maxPayloadSizeBytes = updates.maxPayloadSizeKb * 1024;
|
|
||||||
}
|
|
||||||
if (updates.batchFlushThreshold !== undefined) {
|
|
||||||
DataBufferEngine.batchFlushThreshold = updates.batchFlushThreshold;
|
|
||||||
}
|
|
||||||
if (updates.batchFlushIntervalMs !== undefined) {
|
|
||||||
DataBufferEngine.batchFlushIntervalMs = updates.batchFlushIntervalMs;
|
|
||||||
}
|
|
||||||
if (updates.jwtSecret !== undefined) {
|
|
||||||
AuthService.setSecret(updates.jwtSecret);
|
|
||||||
}
|
|
||||||
if (updates.fileRotationMb !== undefined) {
|
|
||||||
CSVStorageManager.maxFileSizeBytes = updates.fileRotationMb * 1024 * 1024;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return this.settings;
|
public static async updateSettingsAsync(
|
||||||
|
updates: Partial<AdminSettings>,
|
||||||
|
): Promise<AdminSettings> {
|
||||||
|
const current = await this.getSettingsAsync();
|
||||||
|
const jwtSecretChanged =
|
||||||
|
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
|
||||||
|
const nextSettings: AdminSettings = {
|
||||||
|
...current,
|
||||||
|
...updates,
|
||||||
|
updatedAt: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
globals.__dataStackAdminSettings = nextSettings;
|
||||||
|
this.applyRuntimeSettings(nextSettings);
|
||||||
|
if (jwtSecretChanged) {
|
||||||
|
await SDKManager.resignAllClientTokensAsync();
|
||||||
|
}
|
||||||
|
await this.persistSettingsToDb(nextSettings);
|
||||||
|
return { ...nextSettings };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
import { SDKManager } from '../sdk/sdkManager';
|
import { SDKManager } from '../sdk/sdkManager';
|
||||||
import { CSVStorageManager } from '../Storage/csvStorage';
|
import { CSVStorageManager } from '../Storage/csvStorage';
|
||||||
import { DataBufferEngine } from '../data/buffer';
|
import { DataBufferEngine } from '../data/buffer';
|
||||||
import { SettingsManager } from '../adminSettings/settingsManager';
|
|
||||||
import { SecurityEngine } from '../Security/security';
|
import { SecurityEngine } from '../Security/security';
|
||||||
|
|
||||||
export class ApiEngine {
|
export class ApiEngine {
|
||||||
@@ -10,23 +9,37 @@ export class ApiEngine {
|
|||||||
*/
|
*/
|
||||||
public static async getDashboardMetrics() {
|
public static async getDashboardMetrics() {
|
||||||
const storageStats = CSVStorageManager.getStorageStats();
|
const storageStats = CSVStorageManager.getStorageStats();
|
||||||
const bufferStats = DataBufferEngine.getBufferStats();
|
const localBufferStats = DataBufferEngine.getBufferStats();
|
||||||
const clients = SDKManager.listClients();
|
const clients = await SDKManager.listClientsAsync();
|
||||||
const activeClients = clients.filter((c) => c.status === 'active').length;
|
const activeClients = clients.filter((c) => c.status === 'active').length;
|
||||||
let activeSockets = 0;
|
let activeSockets = 0;
|
||||||
|
let ingestionBufferSize = 0;
|
||||||
|
|
||||||
|
const ingestionBaseUrl = (
|
||||||
|
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
|
||||||
|
).replace(/\/$/, '');
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const res = await fetch('http://localhost:4000/health', { cache: 'no-store' });
|
const res = await fetch(`${ingestionBaseUrl}/health`, {
|
||||||
|
cache: 'no-store',
|
||||||
|
});
|
||||||
if (res.ok) {
|
if (res.ok) {
|
||||||
const data = await res.json();
|
const data = await res.json();
|
||||||
activeSockets = data.activeSockets ?? 0;
|
activeSockets = Number(data.activeSockets) || 0;
|
||||||
|
ingestionBufferSize = Number(data.bufferSize) || 0;
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
// Server may be starting
|
// Ingestion server may not be running in single-process dev mode
|
||||||
activeSockets = 0;
|
activeSockets = 0;
|
||||||
|
ingestionBufferSize = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
const securityStats = SecurityEngine.getStats();
|
const securityStats = SecurityEngine.getStats();
|
||||||
|
const bufferStats = {
|
||||||
|
...localBufferStats,
|
||||||
|
currentBufferSize:
|
||||||
|
localBufferStats.currentBufferSize + ingestionBufferSize,
|
||||||
|
};
|
||||||
|
|
||||||
return {
|
return {
|
||||||
activeSockets,
|
activeSockets,
|
||||||
|
|||||||
@@ -1,162 +0,0 @@
|
|||||||
import { Server as SocketIOServer, Socket } from 'socket.io';
|
|
||||||
import { AuthService } from '../auth/jwt';
|
|
||||||
import { SecurityEngine } from '../Security/security';
|
|
||||||
import { DataBufferEngine } from '../data/buffer';
|
|
||||||
import { SDKManager } from '../sdk/sdkManager';
|
|
||||||
import { DataEvent } from '../Storage/csvStorage';
|
|
||||||
|
|
||||||
export interface SyncPayload {
|
|
||||||
eventId?: string;
|
|
||||||
timestamp?: string;
|
|
||||||
data: unknown;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class ConnectorEngine {
|
|
||||||
private static io: SocketIOServer | null = null;
|
|
||||||
private static activeConnections: Map<string, { socketId: string; clientId: string; connectedAt: string }> = new Map();
|
|
||||||
|
|
||||||
public static initialize(io: SocketIOServer) {
|
|
||||||
this.io = io;
|
|
||||||
|
|
||||||
// Authentication middleware
|
|
||||||
io.use((socket: Socket, next) => {
|
|
||||||
const authHeader =
|
|
||||||
socket.handshake.auth?.token ||
|
|
||||||
socket.handshake.headers?.authorization ||
|
|
||||||
(socket.handshake.query?.token as string);
|
|
||||||
|
|
||||||
if (!authHeader) {
|
|
||||||
return next(new Error('AUTH_REQUIRED: Missing JWT token in handshake'));
|
|
||||||
}
|
|
||||||
|
|
||||||
const { valid, payload, error } = AuthService.verifyClientToken(authHeader);
|
|
||||||
if (!valid || !payload) {
|
|
||||||
return next(new Error(`AUTH_FAILED: ${error || 'Invalid JWT token'}`));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if client is registered and active
|
|
||||||
const registered = SDKManager.getClient(payload.clientId);
|
|
||||||
if (registered && registered.status === 'revoked') {
|
|
||||||
return next(new Error('CLIENT_REVOKED: This SDK credentials have been revoked by admin'));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Attach client information to socket data
|
|
||||||
socket.data.clientId = payload.clientId;
|
|
||||||
socket.data.rateLimit = registered?.rateLimit || payload.rateLimit || 100;
|
|
||||||
next();
|
|
||||||
});
|
|
||||||
|
|
||||||
// Connection handler
|
|
||||||
io.on('connection', (socket: Socket) => {
|
|
||||||
const clientId = socket.data.clientId as string;
|
|
||||||
const rateLimit = socket.data.rateLimit as number;
|
|
||||||
|
|
||||||
this.activeConnections.set(socket.id, {
|
|
||||||
socketId: socket.id,
|
|
||||||
clientId,
|
|
||||||
connectedAt: new Date().toISOString(),
|
|
||||||
});
|
|
||||||
|
|
||||||
// Join room for this client
|
|
||||||
socket.join(`client:${clientId}`);
|
|
||||||
|
|
||||||
// Single Event Sync: "sync_data"
|
|
||||||
socket.on('sync_data', (payload: SyncPayload, callback?: (res: { success: boolean; eventId?: string; error?: string }) => void) => {
|
|
||||||
try {
|
|
||||||
const eventId = payload.eventId || `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
|
||||||
const timestamp = payload.timestamp || new Date().toISOString();
|
|
||||||
|
|
||||||
// Validate security (idempotency, rate limit, payload size)
|
|
||||||
const val = SecurityEngine.validateEvent(clientId, eventId, payload.data, rateLimit);
|
|
||||||
if (!val.allowed) {
|
|
||||||
if (val.code === 'DUPLICATE_EVENT') {
|
|
||||||
if (callback) callback({ success: true, eventId, error: 'DUPLICATE_IGNORED' });
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (callback) callback({ success: false, eventId, error: val.reason });
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const event: DataEvent = {
|
|
||||||
clientId,
|
|
||||||
eventId,
|
|
||||||
timestamp,
|
|
||||||
payload: typeof payload.data === 'string' ? payload.data : JSON.stringify(payload.data),
|
|
||||||
};
|
|
||||||
|
|
||||||
DataBufferEngine.push(event);
|
|
||||||
SDKManager.incrementEventCount(clientId, 1);
|
|
||||||
|
|
||||||
if (callback) {
|
|
||||||
callback({ success: true, eventId });
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
const msg = err instanceof Error ? err.message : 'Unknown sync error';
|
|
||||||
if (callback) callback({ success: false, error: msg });
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
// Batch Event Sync: "batch_sync"
|
|
||||||
socket.on('batch_sync', (batch: SyncPayload[], callback?: (res: { success: boolean; processed: number; rejected: number; errors?: string[] }) => void) => {
|
|
||||||
try {
|
|
||||||
if (!Array.isArray(batch)) {
|
|
||||||
if (callback) callback({ success: false, processed: 0, rejected: 0, errors: ['Expected array of events'] });
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
let processed = 0;
|
|
||||||
let rejected = 0;
|
|
||||||
const errors: string[] = [];
|
|
||||||
|
|
||||||
for (const item of batch) {
|
|
||||||
const eventId = item.eventId || `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
|
||||||
const timestamp = item.timestamp || new Date().toISOString();
|
|
||||||
|
|
||||||
const val = SecurityEngine.validateEvent(clientId, eventId, item.data, rateLimit);
|
|
||||||
if (!val.allowed) {
|
|
||||||
rejected += 1;
|
|
||||||
if (val.code !== 'DUPLICATE_EVENT') {
|
|
||||||
errors.push(`${eventId}: ${val.reason}`);
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
const event: DataEvent = {
|
|
||||||
clientId,
|
|
||||||
eventId,
|
|
||||||
timestamp,
|
|
||||||
payload: typeof item.data === 'string' ? item.data : JSON.stringify(item.data),
|
|
||||||
};
|
|
||||||
|
|
||||||
DataBufferEngine.push(event);
|
|
||||||
processed += 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
SDKManager.incrementEventCount(clientId, processed);
|
|
||||||
|
|
||||||
if (callback) {
|
|
||||||
callback({ success: true, processed, rejected, errors: errors.length > 0 ? errors.slice(0, 5) : undefined });
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
const msg = err instanceof Error ? err.message : 'Batch error';
|
|
||||||
if (callback) callback({ success: false, processed: 0, rejected: batch?.length || 0, errors: [msg] });
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
// Disconnect
|
|
||||||
socket.on('disconnect', () => {
|
|
||||||
this.activeConnections.delete(socket.id);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
public static getActiveConnections() {
|
|
||||||
return Array.from(this.activeConnections.values());
|
|
||||||
}
|
|
||||||
|
|
||||||
public static getActiveConnectionsCount(): number {
|
|
||||||
return this.activeConnections.size;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export default ConnectorEngine;
|
|
||||||
+110
-80
@@ -3,93 +3,116 @@ import { CSVStorageManager, DataEvent } from '../Storage/csvStorage';
|
|||||||
export type EventSubscriber = (event: DataEvent) => void;
|
export type EventSubscriber = (event: DataEvent) => void;
|
||||||
export type FlushCallback = (count: number, files: string[]) => void;
|
export type FlushCallback = (count: number, files: string[]) => void;
|
||||||
|
|
||||||
|
interface BufferGlobalState {
|
||||||
|
buffer: DataEvent[];
|
||||||
|
recentEvents: DataEvent[];
|
||||||
|
batchFlushThreshold: number;
|
||||||
|
batchFlushIntervalMs: number;
|
||||||
|
flushTimer: NodeJS.Timeout | null;
|
||||||
|
subscribers: Set<EventSubscriber>;
|
||||||
|
flushListeners: Set<FlushCallback>;
|
||||||
|
totalIngested: number;
|
||||||
|
totalFlushed: number;
|
||||||
|
isFlushing: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
type BufferGlobals = typeof globalThis & {
|
||||||
|
__dataStackBufferState?: BufferGlobalState;
|
||||||
|
};
|
||||||
|
|
||||||
|
const globals = globalThis as BufferGlobals;
|
||||||
|
|
||||||
|
function getState(): BufferGlobalState {
|
||||||
|
if (!globals.__dataStackBufferState) {
|
||||||
|
globals.__dataStackBufferState = {
|
||||||
|
buffer: [],
|
||||||
|
recentEvents: [],
|
||||||
|
batchFlushThreshold: 500,
|
||||||
|
batchFlushIntervalMs: 2000,
|
||||||
|
flushTimer: null,
|
||||||
|
subscribers: new Set(),
|
||||||
|
flushListeners: new Set(),
|
||||||
|
totalIngested: 0,
|
||||||
|
totalFlushed: 0,
|
||||||
|
isFlushing: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return globals.__dataStackBufferState;
|
||||||
|
}
|
||||||
|
|
||||||
export class DataBufferEngine {
|
export class DataBufferEngine {
|
||||||
private static buffer: DataEvent[] = [];
|
|
||||||
private static recentEvents: DataEvent[] = [];
|
|
||||||
private static maxRecentEvents = 200;
|
private static maxRecentEvents = 200;
|
||||||
|
|
||||||
// Thresholds
|
public static get batchFlushThreshold(): number {
|
||||||
public static batchFlushThreshold = 500; // rows
|
return getState().batchFlushThreshold;
|
||||||
public static batchFlushIntervalMs = 2000; // 2 seconds
|
}
|
||||||
|
|
||||||
private static flushTimer: NodeJS.Timeout | null = null;
|
public static set batchFlushThreshold(value: number) {
|
||||||
private static subscribers: Set<EventSubscriber> = new Set();
|
getState().batchFlushThreshold = Math.max(1, Number(value) || 500);
|
||||||
private static flushListeners: Set<FlushCallback> = new Set();
|
}
|
||||||
|
|
||||||
private static totalIngested = 5;
|
public static get batchFlushIntervalMs(): number {
|
||||||
private static totalFlushed = 1420;
|
return getState().batchFlushIntervalMs;
|
||||||
private static isFlushing = false;
|
}
|
||||||
|
|
||||||
static {
|
public static set batchFlushIntervalMs(value: number) {
|
||||||
const now = Date.now();
|
this.setFlushInterval(value);
|
||||||
DataBufferEngine.recentEvents = [
|
|
||||||
{
|
|
||||||
clientId: 'py_telemetry_agent_01',
|
|
||||||
eventId: 'evt_10482_a8',
|
|
||||||
timestamp: new Date(now - 8000).toISOString(),
|
|
||||||
payload: JSON.stringify({ action: 'crawler_metric', pages_scraped: 38, latency_ms: 24, status: 200 }),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
clientId: 'py_scraper_prod_02',
|
|
||||||
eventId: 'evt_10481_f2',
|
|
||||||
timestamp: new Date(now - 22000).toISOString(),
|
|
||||||
payload: JSON.stringify({ spider: 'market_catalog', items: 114, parse_time_s: 1.28, errors: 0 }),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
clientId: 'iot_fleet_east_03',
|
|
||||||
eventId: 'evt_10480_c1',
|
|
||||||
timestamp: new Date(now - 45000).toISOString(),
|
|
||||||
payload: JSON.stringify({ sensor_node: 'gw_alpha_09', temp_c: 24.3, humidity_pct: 58.1, battery_v: 3.32 }),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
clientId: 'py_telemetry_agent_01',
|
|
||||||
eventId: 'evt_10479_d9',
|
|
||||||
timestamp: new Date(now - 78000).toISOString(),
|
|
||||||
payload: JSON.stringify({ action: 'heartbeat', memory_usage_mb: 284, cpu_load_pct: 12.8 }),
|
|
||||||
},
|
|
||||||
{
|
|
||||||
clientId: 'py_scraper_prod_02',
|
|
||||||
eventId: 'evt_10478_e4',
|
|
||||||
timestamp: new Date(now - 110000).toISOString(),
|
|
||||||
payload: JSON.stringify({ action: 'batch_sync', batch_size: 50, duration_ms: 82 }),
|
|
||||||
},
|
|
||||||
];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static init() {
|
public static init() {
|
||||||
if (!this.flushTimer) {
|
const state = getState();
|
||||||
this.flushTimer = setInterval(() => {
|
if (!state.flushTimer) {
|
||||||
this.flush();
|
state.flushTimer = setInterval(() => {
|
||||||
}, this.batchFlushIntervalMs);
|
void this.flush();
|
||||||
|
}, state.batchFlushIntervalMs);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Updates flush interval and immediately restarts the active interval timer.
|
||||||
|
*/
|
||||||
|
public static setFlushInterval(intervalMs: number) {
|
||||||
|
const state = getState();
|
||||||
|
const nextInterval = Math.max(100, Number(intervalMs) || 2000);
|
||||||
|
state.batchFlushIntervalMs = nextInterval;
|
||||||
|
|
||||||
|
if (state.flushTimer) {
|
||||||
|
clearInterval(state.flushTimer);
|
||||||
|
state.flushTimer = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
state.flushTimer = setInterval(() => {
|
||||||
|
void this.flush();
|
||||||
|
}, state.batchFlushIntervalMs);
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Push incoming event to in-memory buffer.
|
* Push incoming event to in-memory buffer.
|
||||||
* If buffer hits threshold (e.g. 500 rows), triggers immediate batch flush.
|
* If buffer hits threshold (e.g. 500 rows), triggers immediate batch flush.
|
||||||
*/
|
*/
|
||||||
public static push(event: DataEvent): void {
|
public static push(event: DataEvent): void {
|
||||||
this.buffer.push(event);
|
const state = getState();
|
||||||
this.totalIngested += 1;
|
state.buffer.push(event);
|
||||||
|
state.totalIngested += 1;
|
||||||
|
|
||||||
// Cache in recent events for live dashboard viewer
|
// Cache in recent events for live dashboard viewer
|
||||||
this.recentEvents.unshift(event);
|
state.recentEvents.unshift(event);
|
||||||
if (this.recentEvents.length > this.maxRecentEvents) {
|
if (state.recentEvents.length > this.maxRecentEvents) {
|
||||||
this.recentEvents.pop();
|
state.recentEvents.pop();
|
||||||
}
|
}
|
||||||
|
|
||||||
// Notify live subscribers
|
// Notify live subscribers
|
||||||
for (const sub of this.subscribers) {
|
for (const sub of state.subscribers) {
|
||||||
try {
|
try {
|
||||||
sub(event);
|
sub(event);
|
||||||
} catch {
|
} catch {
|
||||||
// Ignore subscriber err
|
// Ignore subscriber error
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if buffer threshold reached
|
// Check if buffer threshold reached
|
||||||
if (this.buffer.length >= this.batchFlushThreshold) {
|
if (state.buffer.length >= state.batchFlushThreshold) {
|
||||||
this.flush();
|
void this.flush();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -97,19 +120,20 @@ export class DataBufferEngine {
|
|||||||
* Flushes current buffer to CSV files via CSVStorageManager
|
* Flushes current buffer to CSV files via CSVStorageManager
|
||||||
*/
|
*/
|
||||||
public static async flush(): Promise<{ count: number; files: string[] }> {
|
public static async flush(): Promise<{ count: number; files: string[] }> {
|
||||||
if (this.buffer.length === 0 || this.isFlushing) {
|
const state = getState();
|
||||||
|
if (state.buffer.length === 0 || state.isFlushing) {
|
||||||
return { count: 0, files: [] };
|
return { count: 0, files: [] };
|
||||||
}
|
}
|
||||||
|
|
||||||
this.isFlushing = true;
|
state.isFlushing = true;
|
||||||
const batchToFlush = [...this.buffer];
|
const batchToFlush = [...state.buffer];
|
||||||
this.buffer = [];
|
state.buffer = [];
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const result = await CSVStorageManager.appendBatch(batchToFlush);
|
const result = await CSVStorageManager.appendBatch(batchToFlush);
|
||||||
this.totalFlushed += result.written;
|
state.totalFlushed += result.written;
|
||||||
|
|
||||||
for (const listener of this.flushListeners) {
|
for (const listener of state.flushListeners) {
|
||||||
try {
|
try {
|
||||||
listener(result.written, result.filesUpdated);
|
listener(result.written, result.filesUpdated);
|
||||||
} catch {
|
} catch {
|
||||||
@@ -120,10 +144,10 @@ export class DataBufferEngine {
|
|||||||
return { count: result.written, files: result.filesUpdated };
|
return { count: result.written, files: result.filesUpdated };
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
// On failure, re-queue to buffer front for retry
|
// On failure, re-queue to buffer front for retry
|
||||||
this.buffer.unshift(...batchToFlush);
|
state.buffer.unshift(...batchToFlush);
|
||||||
throw err;
|
throw err;
|
||||||
} finally {
|
} finally {
|
||||||
this.isFlushing = false;
|
state.isFlushing = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -131,36 +155,42 @@ export class DataBufferEngine {
|
|||||||
* Subscribe to real-time events (for UI monitoring)
|
* Subscribe to real-time events (for UI monitoring)
|
||||||
*/
|
*/
|
||||||
public static subscribe(subscriber: EventSubscriber): () => void {
|
public static subscribe(subscriber: EventSubscriber): () => void {
|
||||||
this.subscribers.add(subscriber);
|
const state = getState();
|
||||||
|
state.subscribers.add(subscriber);
|
||||||
return () => {
|
return () => {
|
||||||
this.subscribers.delete(subscriber);
|
state.subscribers.delete(subscriber);
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
public static onFlush(callback: FlushCallback): () => void {
|
public static onFlush(callback: FlushCallback): () => void {
|
||||||
this.flushListeners.add(callback);
|
const state = getState();
|
||||||
|
state.flushListeners.add(callback);
|
||||||
return () => {
|
return () => {
|
||||||
this.flushListeners.delete(callback);
|
state.flushListeners.delete(callback);
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
public static getRecentEvents(limit = 50, filterClientId?: string): DataEvent[] {
|
public static getRecentEvents(limit = 50, filterClientId?: string): DataEvent[] {
|
||||||
|
const state = getState();
|
||||||
if (filterClientId && filterClientId !== 'all') {
|
if (filterClientId && filterClientId !== 'all') {
|
||||||
return this.recentEvents.filter((e) => e.clientId === filterClientId).slice(0, limit);
|
return state.recentEvents
|
||||||
|
.filter((e) => e.clientId === filterClientId)
|
||||||
|
.slice(0, limit);
|
||||||
}
|
}
|
||||||
return this.recentEvents.slice(0, limit);
|
return state.recentEvents.slice(0, limit);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static getBufferStats() {
|
public static getBufferStats() {
|
||||||
|
const state = getState();
|
||||||
return {
|
return {
|
||||||
currentBufferSize: this.buffer.length,
|
currentBufferSize: state.buffer.length,
|
||||||
batchFlushThreshold: this.batchFlushThreshold,
|
batchFlushThreshold: state.batchFlushThreshold,
|
||||||
batchFlushIntervalMs: this.batchFlushIntervalMs,
|
batchFlushIntervalMs: state.batchFlushIntervalMs,
|
||||||
totalIngested: this.totalIngested,
|
totalIngested: state.totalIngested,
|
||||||
totalFlushed: this.totalFlushed,
|
totalFlushed: state.totalFlushed,
|
||||||
recentEventsCached: this.recentEvents.length,
|
recentEventsCached: state.recentEvents.length,
|
||||||
activeSubscribers: this.subscribers.size,
|
activeSubscribers: state.subscribers.size,
|
||||||
isFlushing: this.isFlushing,
|
isFlushing: state.isFlushing,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,61 +0,0 @@
|
|||||||
export interface SystemNotification {
|
|
||||||
id: string;
|
|
||||||
type: 'info' | 'success' | 'warning' | 'error';
|
|
||||||
title: string;
|
|
||||||
message: string;
|
|
||||||
source: 'socket' | 'storage' | 'security' | 'worker';
|
|
||||||
timestamp: string;
|
|
||||||
read: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class NotificationEngine {
|
|
||||||
private static notifications: SystemNotification[] = [
|
|
||||||
{
|
|
||||||
id: 'notif-1',
|
|
||||||
type: 'info',
|
|
||||||
title: 'DataStack Engine Booted',
|
|
||||||
message: 'Ingestion server listening on port 4000. CSV buffer worker initialized.',
|
|
||||||
source: 'worker',
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
read: false,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
id: 'notif-2',
|
|
||||||
type: 'success',
|
|
||||||
title: 'Python SDK Ready',
|
|
||||||
message: 'pip installable datastack_sdk package prepared in /sdk-python/',
|
|
||||||
source: 'socket',
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
read: false,
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
public static push(notif: Omit<SystemNotification, 'id' | 'timestamp' | 'read'>): SystemNotification {
|
|
||||||
const item: SystemNotification = {
|
|
||||||
id: `notif-${Date.now()}-${Math.random().toString(36).substring(2, 6)}`,
|
|
||||||
timestamp: new Date().toISOString(),
|
|
||||||
read: false,
|
|
||||||
...notif,
|
|
||||||
};
|
|
||||||
this.notifications.unshift(item);
|
|
||||||
if (this.notifications.length > 100) {
|
|
||||||
this.notifications.pop();
|
|
||||||
}
|
|
||||||
return item;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static list(): SystemNotification[] {
|
|
||||||
return [...this.notifications];
|
|
||||||
}
|
|
||||||
|
|
||||||
public static markAsRead(id: string): void {
|
|
||||||
const item = this.notifications.find((n) => n.id === id);
|
|
||||||
if (item) item.read = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static clearAll(): void {
|
|
||||||
this.notifications = [];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export default NotificationEngine;
|
|
||||||
+320
-55
@@ -1,4 +1,5 @@
|
|||||||
import { AuthService } from "../auth/jwt";
|
import { AuthService } from "../auth/jwt";
|
||||||
|
import { DatabaseEngine } from "../Database/database";
|
||||||
|
|
||||||
export interface ClientSDK {
|
export interface ClientSDK {
|
||||||
clientId: string;
|
clientId: string;
|
||||||
@@ -12,59 +13,114 @@ export interface ClientSDK {
|
|||||||
createdAt: string;
|
createdAt: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type SDKDatabaseGlobals = typeof globalThis & {
|
||||||
|
__dataStackClientsMap?: Map<string, ClientSDK>;
|
||||||
|
__dataStackClientsHydrated?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
const globals = globalThis as SDKDatabaseGlobals;
|
||||||
|
|
||||||
|
function getClientsMap(): Map<string, ClientSDK> {
|
||||||
|
if (!globals.__dataStackClientsMap) {
|
||||||
|
globals.__dataStackClientsMap = new Map<string, ClientSDK>();
|
||||||
|
}
|
||||||
|
return globals.__dataStackClientsMap;
|
||||||
|
}
|
||||||
|
|
||||||
|
function mapRowToClient(row: Record<string, unknown>): ClientSDK {
|
||||||
|
return {
|
||||||
|
clientId: String(row.client_id),
|
||||||
|
name: String(row.name),
|
||||||
|
secret: String(row.secret),
|
||||||
|
token: String(row.token),
|
||||||
|
rateLimit: Number(row.rate_limit) || 100,
|
||||||
|
status: row.status === "revoked" ? "revoked" : "active",
|
||||||
|
totalEvents: Number(row.total_events) || 0,
|
||||||
|
lastActive: row.last_active
|
||||||
|
? new Date(row.last_active as string | Date).toISOString()
|
||||||
|
: undefined,
|
||||||
|
createdAt: row.created_at
|
||||||
|
? new Date(row.created_at as string | Date).toISOString()
|
||||||
|
: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export class SDKManager {
|
export class SDKManager {
|
||||||
private static clients: Map<string, ClientSDK> = new Map();
|
private static async syncFromDatabase(): Promise<ClientSDK[] | null> {
|
||||||
|
const pool = DatabaseEngine.getPool();
|
||||||
|
if (!pool) return null;
|
||||||
|
|
||||||
// Initialize with production starter clients
|
try {
|
||||||
static {
|
const ready = await DatabaseEngine.ensureSchema();
|
||||||
const seedClients = [
|
if (!ready) return null;
|
||||||
{
|
|
||||||
clientId: "py_telemetry_agent_01",
|
|
||||||
name: "Python Telemetry Agent (US-East)",
|
|
||||||
rateLimit: 150,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
clientId: "py_scraper_prod_02",
|
|
||||||
name: "Python Web Scraper Engine",
|
|
||||||
rateLimit: 200,
|
|
||||||
},
|
|
||||||
{
|
|
||||||
clientId: "iot_fleet_east_03",
|
|
||||||
name: "IoT Sensor Gateway Node",
|
|
||||||
rateLimit: 50,
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
for (const c of seedClients) {
|
const result = await pool.query(
|
||||||
const secret = `sec_${c.clientId}_${Math.random().toString(36).substring(2, 10)}`;
|
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
|
||||||
const token = AuthService.getSecret()
|
FROM client_sdks
|
||||||
? AuthService.generateClientToken({
|
ORDER BY created_at DESC`,
|
||||||
clientId: c.clientId,
|
);
|
||||||
name: c.name,
|
|
||||||
rateLimit: c.rateLimit,
|
|
||||||
})
|
|
||||||
: "";
|
|
||||||
|
|
||||||
SDKManager.clients.set(c.clientId, {
|
const map = getClientsMap();
|
||||||
clientId: c.clientId,
|
map.clear();
|
||||||
name: c.name,
|
const list: ClientSDK[] = [];
|
||||||
secret,
|
|
||||||
token,
|
for (const row of result.rows) {
|
||||||
rateLimit: c.rateLimit,
|
const client = mapRowToClient(row);
|
||||||
status: "active",
|
map.set(client.clientId, client);
|
||||||
totalEvents: 4280,
|
list.push(client);
|
||||||
lastActive: new Date().toISOString(),
|
}
|
||||||
createdAt: "2026-10-20T10:00:00.000Z",
|
|
||||||
});
|
globals.__dataStackClientsHydrated = true;
|
||||||
|
return list;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static listClients(): ClientSDK[] {
|
public static listClients(): ClientSDK[] {
|
||||||
return Array.from(this.clients.values());
|
if (!globals.__dataStackClientsHydrated) {
|
||||||
|
void this.syncFromDatabase();
|
||||||
|
}
|
||||||
|
return Array.from(getClientsMap().values());
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async listClientsAsync(): Promise<ClientSDK[]> {
|
||||||
|
const fromDb = await this.syncFromDatabase();
|
||||||
|
if (fromDb !== null) {
|
||||||
|
return fromDb;
|
||||||
|
}
|
||||||
|
return Array.from(getClientsMap().values());
|
||||||
}
|
}
|
||||||
|
|
||||||
public static getClient(clientId: string): ClientSDK | undefined {
|
public static getClient(clientId: string): ClientSDK | undefined {
|
||||||
return this.clients.get(clientId);
|
return getClientsMap().get(clientId);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async getClientAsync(
|
||||||
|
clientId: string,
|
||||||
|
): Promise<ClientSDK | undefined> {
|
||||||
|
const pool = DatabaseEngine.getPool();
|
||||||
|
if (pool) {
|
||||||
|
try {
|
||||||
|
await DatabaseEngine.ensureSchema();
|
||||||
|
const result = await pool.query(
|
||||||
|
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
|
||||||
|
FROM client_sdks
|
||||||
|
WHERE client_id = $1`,
|
||||||
|
[clientId],
|
||||||
|
);
|
||||||
|
if (result.rowCount && result.rowCount > 0) {
|
||||||
|
const client = mapRowToClient(result.rows[0]);
|
||||||
|
getClientsMap().set(client.clientId, client);
|
||||||
|
return client;
|
||||||
|
}
|
||||||
|
getClientsMap().delete(clientId);
|
||||||
|
return undefined;
|
||||||
|
} catch {
|
||||||
|
// Fall back to in-memory map
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return getClientsMap().get(clientId);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static registerClient(data: {
|
public static registerClient(data: {
|
||||||
@@ -77,11 +133,11 @@ export class SDKManager {
|
|||||||
.toLowerCase()
|
.toLowerCase()
|
||||||
.replace(/[^a-z0-9_-]/g, "_");
|
.replace(/[^a-z0-9_-]/g, "_");
|
||||||
const secret = `sec_${safeClientId}_${Math.random().toString(36).substring(2, 10)}`;
|
const secret = `sec_${safeClientId}_${Math.random().toString(36).substring(2, 10)}`;
|
||||||
const rateLimit = data.rateLimit || 100;
|
const rateLimit = Math.max(1, Number(data.rateLimit) || 100);
|
||||||
|
|
||||||
const token = AuthService.generateClientToken({
|
const token = AuthService.generateClientToken({
|
||||||
clientId: safeClientId,
|
clientId: safeClientId,
|
||||||
name: data.name,
|
name: data.name.trim(),
|
||||||
rateLimit,
|
rateLimit,
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -96,12 +152,79 @@ export class SDKManager {
|
|||||||
createdAt: new Date().toISOString(),
|
createdAt: new Date().toISOString(),
|
||||||
};
|
};
|
||||||
|
|
||||||
this.clients.set(safeClientId, newClient);
|
getClientsMap().set(safeClientId, newClient);
|
||||||
|
void this.persistNewClient(newClient);
|
||||||
|
return newClient;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async persistNewClient(client: ClientSDK): Promise<void> {
|
||||||
|
const pool = DatabaseEngine.getPool();
|
||||||
|
if (!pool) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
await DatabaseEngine.ensureSchema();
|
||||||
|
await pool.query(
|
||||||
|
`INSERT INTO client_sdks (client_id, name, secret, token, rate_limit, status, total_events, created_at, updated_at)
|
||||||
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW())
|
||||||
|
ON CONFLICT (client_id) DO UPDATE SET
|
||||||
|
name = EXCLUDED.name,
|
||||||
|
secret = EXCLUDED.secret,
|
||||||
|
token = EXCLUDED.token,
|
||||||
|
rate_limit = EXCLUDED.rate_limit,
|
||||||
|
status = EXCLUDED.status,
|
||||||
|
updated_at = NOW()`,
|
||||||
|
[
|
||||||
|
client.clientId,
|
||||||
|
client.name,
|
||||||
|
client.secret,
|
||||||
|
client.token,
|
||||||
|
client.rateLimit,
|
||||||
|
client.status,
|
||||||
|
client.totalEvents,
|
||||||
|
client.createdAt,
|
||||||
|
],
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
// In-memory map already updated
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async registerClientAsync(data: {
|
||||||
|
clientId: string;
|
||||||
|
name: string;
|
||||||
|
rateLimit?: number;
|
||||||
|
}): Promise<ClientSDK> {
|
||||||
|
const safeClientId = data.clientId
|
||||||
|
.trim()
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/[^a-z0-9_-]/g, "_");
|
||||||
|
const secret = `sec_${safeClientId}_${Math.random().toString(36).substring(2, 10)}`;
|
||||||
|
const rateLimit = Math.max(1, Number(data.rateLimit) || 100);
|
||||||
|
|
||||||
|
const token = AuthService.generateClientToken({
|
||||||
|
clientId: safeClientId,
|
||||||
|
name: data.name.trim(),
|
||||||
|
rateLimit,
|
||||||
|
});
|
||||||
|
|
||||||
|
const newClient: ClientSDK = {
|
||||||
|
clientId: safeClientId,
|
||||||
|
name: data.name.trim(),
|
||||||
|
secret,
|
||||||
|
token,
|
||||||
|
rateLimit,
|
||||||
|
status: "active",
|
||||||
|
totalEvents: 0,
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
};
|
||||||
|
|
||||||
|
getClientsMap().set(safeClientId, newClient);
|
||||||
|
await this.persistNewClient(newClient);
|
||||||
return newClient;
|
return newClient;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static regenerateToken(clientId: string): ClientSDK | null {
|
public static regenerateToken(clientId: string): ClientSDK | null {
|
||||||
const client = this.clients.get(clientId);
|
const client = getClientsMap().get(clientId);
|
||||||
if (!client) return null;
|
if (!client) return null;
|
||||||
|
|
||||||
client.secret = `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`;
|
client.secret = `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`;
|
||||||
@@ -111,35 +234,177 @@ export class SDKManager {
|
|||||||
rateLimit: client.rateLimit,
|
rateLimit: client.rateLimit,
|
||||||
});
|
});
|
||||||
|
|
||||||
this.clients.set(clientId, client);
|
getClientsMap().set(clientId, client);
|
||||||
|
void this.persistClientCredentials(client);
|
||||||
return client;
|
return client;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static async persistClientCredentials(
|
||||||
|
client: ClientSDK,
|
||||||
|
): Promise<void> {
|
||||||
|
const pool = DatabaseEngine.getPool();
|
||||||
|
if (!pool) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
await DatabaseEngine.ensureSchema();
|
||||||
|
await pool.query(
|
||||||
|
`UPDATE client_sdks
|
||||||
|
SET secret = $2, token = $3, updated_at = NOW()
|
||||||
|
WHERE client_id = $1`,
|
||||||
|
[client.clientId, client.secret, client.token],
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
// Ignore DB failure
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async regenerateTokenAsync(
|
||||||
|
clientId: string,
|
||||||
|
): Promise<ClientSDK | null> {
|
||||||
|
const existing = await this.getClientAsync(clientId);
|
||||||
|
if (!existing) return null;
|
||||||
|
|
||||||
|
existing.secret = `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`;
|
||||||
|
existing.token = AuthService.generateClientToken({
|
||||||
|
clientId: existing.clientId,
|
||||||
|
name: existing.name,
|
||||||
|
rateLimit: existing.rateLimit,
|
||||||
|
});
|
||||||
|
|
||||||
|
getClientsMap().set(clientId, existing);
|
||||||
|
await this.persistClientCredentials(existing);
|
||||||
|
return existing;
|
||||||
|
}
|
||||||
|
|
||||||
public static updateClient(
|
public static updateClient(
|
||||||
clientId: string,
|
clientId: string,
|
||||||
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
|
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
|
||||||
): ClientSDK | null {
|
): ClientSDK | null {
|
||||||
const client = this.clients.get(clientId);
|
const client = getClientsMap().get(clientId);
|
||||||
if (!client) return null;
|
if (!client) return null;
|
||||||
|
|
||||||
if (updates.name !== undefined) client.name = updates.name;
|
if (updates.name !== undefined) client.name = updates.name.trim();
|
||||||
if (updates.rateLimit !== undefined) client.rateLimit = updates.rateLimit;
|
if (updates.rateLimit !== undefined)
|
||||||
|
client.rateLimit = Math.max(1, Number(updates.rateLimit) || 100);
|
||||||
if (updates.status !== undefined) client.status = updates.status;
|
if (updates.status !== undefined) client.status = updates.status;
|
||||||
|
|
||||||
this.clients.set(clientId, client);
|
getClientsMap().set(clientId, client);
|
||||||
|
void this.persistClientUpdates(client);
|
||||||
|
return client;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async persistClientUpdates(client: ClientSDK): Promise<void> {
|
||||||
|
const pool = DatabaseEngine.getPool();
|
||||||
|
if (!pool) return;
|
||||||
|
|
||||||
|
try {
|
||||||
|
await DatabaseEngine.ensureSchema();
|
||||||
|
await pool.query(
|
||||||
|
`UPDATE client_sdks
|
||||||
|
SET name = $2, rate_limit = $3, status = $4, updated_at = NOW()
|
||||||
|
WHERE client_id = $1`,
|
||||||
|
[client.clientId, client.name, client.rateLimit, client.status],
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
// Ignore DB failure
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async updateClientAsync(
|
||||||
|
clientId: string,
|
||||||
|
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
|
||||||
|
): Promise<ClientSDK | null> {
|
||||||
|
const client = await this.getClientAsync(clientId);
|
||||||
|
if (!client) return null;
|
||||||
|
|
||||||
|
if (updates.name !== undefined) client.name = updates.name.trim();
|
||||||
|
if (updates.rateLimit !== undefined)
|
||||||
|
client.rateLimit = Math.max(1, Number(updates.rateLimit) || 100);
|
||||||
|
if (updates.status !== undefined) client.status = updates.status;
|
||||||
|
|
||||||
|
getClientsMap().set(clientId, client);
|
||||||
|
await this.persistClientUpdates(client);
|
||||||
return client;
|
return client;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static deleteClient(clientId: string): boolean {
|
public static deleteClient(clientId: string): boolean {
|
||||||
return this.clients.delete(clientId);
|
const deleted = getClientsMap().delete(clientId);
|
||||||
|
void this.deleteClientFromDb(clientId);
|
||||||
|
return deleted;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async deleteClientFromDb(clientId: string): Promise<boolean> {
|
||||||
|
const pool = DatabaseEngine.getPool();
|
||||||
|
if (!pool) return false;
|
||||||
|
|
||||||
|
try {
|
||||||
|
await DatabaseEngine.ensureSchema();
|
||||||
|
const res = await pool.query(
|
||||||
|
`DELETE FROM client_sdks WHERE client_id = $1`,
|
||||||
|
[clientId],
|
||||||
|
);
|
||||||
|
return Boolean(res.rowCount && res.rowCount > 0);
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async deleteClientAsync(clientId: string): Promise<boolean> {
|
||||||
|
const existedInMem = getClientsMap().delete(clientId);
|
||||||
|
const deletedFromDb = await this.deleteClientFromDb(clientId);
|
||||||
|
return existedInMem || deletedFromDb;
|
||||||
}
|
}
|
||||||
|
|
||||||
public static incrementEventCount(clientId: string, count = 1): void {
|
public static incrementEventCount(clientId: string, count = 1): void {
|
||||||
const client = this.clients.get(clientId);
|
const nowIso = new Date().toISOString();
|
||||||
|
const client = getClientsMap().get(clientId);
|
||||||
if (client) {
|
if (client) {
|
||||||
client.totalEvents += count;
|
client.totalEvents += count;
|
||||||
client.lastActive = new Date().toISOString();
|
client.lastActive = nowIso;
|
||||||
this.clients.set(clientId, client);
|
getClientsMap().set(clientId, client);
|
||||||
|
}
|
||||||
|
|
||||||
|
const pool = DatabaseEngine.getPool();
|
||||||
|
if (pool) {
|
||||||
|
void DatabaseEngine.ensureSchema()
|
||||||
|
.then(() =>
|
||||||
|
pool.query(
|
||||||
|
`UPDATE client_sdks
|
||||||
|
SET total_events = total_events + $2,
|
||||||
|
last_active = $3,
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE client_id = $1`,
|
||||||
|
[clientId, count, nowIso],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.catch(() => undefined);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static resignAllClientTokens(): void {
|
||||||
|
const map = getClientsMap();
|
||||||
|
for (const [clientId, client] of map.entries()) {
|
||||||
|
client.token = AuthService.generateClientToken({
|
||||||
|
clientId: client.clientId,
|
||||||
|
name: client.name,
|
||||||
|
rateLimit: client.rateLimit,
|
||||||
|
});
|
||||||
|
map.set(clientId, client);
|
||||||
|
void this.persistClientCredentials(client);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static async resignAllClientTokensAsync(): Promise<void> {
|
||||||
|
const clients = await this.listClientsAsync();
|
||||||
|
const map = getClientsMap();
|
||||||
|
for (const client of clients) {
|
||||||
|
client.token = AuthService.generateClientToken({
|
||||||
|
clientId: client.clientId,
|
||||||
|
name: client.name,
|
||||||
|
rateLimit: client.rateLimit,
|
||||||
|
});
|
||||||
|
map.set(client.clientId, client);
|
||||||
|
await this.persistClientCredentials(client);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ services:
|
|||||||
environment:
|
environment:
|
||||||
NODE_ENV: production
|
NODE_ENV: production
|
||||||
PORT: "3000"
|
PORT: "3000"
|
||||||
|
INGESTION_INTERNAL_URL: "http://ingestion:4000"
|
||||||
DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}"
|
DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}"
|
||||||
DATASTACK_ADMIN_EMAIL: "${DATASTACK_ADMIN_EMAIL:-admin@datastack.dev}"
|
DATASTACK_ADMIN_EMAIL: "${DATASTACK_ADMIN_EMAIL:-admin@datastack.dev}"
|
||||||
DATASTACK_ADMIN_SESSION_SECRET: "${SERVICE_REALBASE64_64_ADMINSESSION:?Coolify must generate SERVICE_REALBASE64_64_ADMINSESSION}"
|
DATASTACK_ADMIN_SESSION_SECRET: "${SERVICE_REALBASE64_64_ADMINSESSION:?Coolify must generate SERVICE_REALBASE64_64_ADMINSESSION}"
|
||||||
@@ -46,6 +47,10 @@ services:
|
|||||||
NODE_ENV: production
|
NODE_ENV: production
|
||||||
PORT_SOCKET: "4000"
|
PORT_SOCKET: "4000"
|
||||||
DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}"
|
DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}"
|
||||||
|
DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}"
|
||||||
|
depends_on:
|
||||||
|
postgres:
|
||||||
|
condition: service_healthy
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test:
|
test:
|
||||||
|
|||||||
+16
-2
@@ -43,13 +43,23 @@ export async function middleware(request: NextRequest) {
|
|||||||
!isLoginPage &&
|
!isLoginPage &&
|
||||||
!isSetupPage;
|
!isSetupPage;
|
||||||
const isAdminApi = isProtectedApi(pathname, request.method);
|
const isAdminApi = isProtectedApi(pathname, request.method);
|
||||||
|
const hasBearerAuth = Boolean(
|
||||||
|
request.headers.get("authorization")?.toLowerCase().startsWith("bearer "),
|
||||||
|
);
|
||||||
const isStateChangingApi =
|
const isStateChangingApi =
|
||||||
pathname.startsWith("/api/") &&
|
pathname.startsWith("/api/") &&
|
||||||
!SAFE_METHODS.has(request.method) &&
|
!SAFE_METHODS.has(request.method) &&
|
||||||
pathname !== "/api/events";
|
!(pathname === "/api/events" && hasBearerAuth);
|
||||||
|
|
||||||
if (isSetupPage) return NextResponse.next();
|
if (isSetupPage) return NextResponse.next();
|
||||||
if (!isConsolePage && !isAdminApi && !isLoginPage) return NextResponse.next();
|
if (
|
||||||
|
!isConsolePage &&
|
||||||
|
!isAdminApi &&
|
||||||
|
!isLoginPage &&
|
||||||
|
!(pathname === "/api/events" && request.method === "POST")
|
||||||
|
) {
|
||||||
|
return NextResponse.next();
|
||||||
|
}
|
||||||
|
|
||||||
if (isStateChangingApi && !isSameOriginRequest(request)) {
|
if (isStateChangingApi && !isSameOriginRequest(request)) {
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
@@ -58,6 +68,10 @@ export async function middleware(request: NextRequest) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (pathname === "/api/events" && request.method === "POST") {
|
||||||
|
return NextResponse.next();
|
||||||
|
}
|
||||||
|
|
||||||
const sessionSecret =
|
const sessionSecret =
|
||||||
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
|
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
|
||||||
"datastack-admin-session-default-secret-32-chars";
|
"datastack-admin-session-default-secret-32-chars";
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import time
|
import time
|
||||||
import uuid
|
import uuid
|
||||||
import logging
|
import logging
|
||||||
|
import threading
|
||||||
from typing import Any, Dict, List, Optional
|
from typing import Any, Dict, List, Optional
|
||||||
import socketio
|
import socketio
|
||||||
|
|
||||||
@@ -63,10 +64,15 @@ class DataStackClient:
|
|||||||
logger.error(f"Failed to connect to DataStack: {e}")
|
logger.error(f"Failed to connect to DataStack: {e}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def sync(self, data: Any, event_id: Optional[str] = None) -> Dict[str, Any]:
|
def sync(
|
||||||
|
self,
|
||||||
|
data: Any,
|
||||||
|
event_id: Optional[str] = None,
|
||||||
|
timeout: float = 5.0,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Sync single event payload in realtime.
|
Sync single event payload in realtime.
|
||||||
Returns server response dict.
|
Blocks until the server ack callback fires or timeout expires.
|
||||||
"""
|
"""
|
||||||
if not self.is_connected:
|
if not self.is_connected:
|
||||||
raise ConnectionError("DataStack client is not connected. Call connect() first.")
|
raise ConnectionError("DataStack client is not connected. Call connect() first.")
|
||||||
@@ -80,24 +86,35 @@ class DataStackClient:
|
|||||||
"data": data,
|
"data": data,
|
||||||
}
|
}
|
||||||
|
|
||||||
response = {"success": False, "eventId": evt_id}
|
ack_event = threading.Event()
|
||||||
|
response: Dict[str, Any] = {
|
||||||
|
"success": False,
|
||||||
|
"eventId": evt_id,
|
||||||
|
"error": "Timeout waiting for server acknowledgment",
|
||||||
|
}
|
||||||
|
|
||||||
def on_ack(res):
|
def on_ack(res):
|
||||||
nonlocal response
|
nonlocal response
|
||||||
|
if isinstance(res, dict):
|
||||||
response = res
|
response = res
|
||||||
|
ack_event.set()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
self.sio.emit("sync_data", payload, callback=on_ack)
|
self.sio.emit("sync_data", payload, callback=on_ack)
|
||||||
# Short wait for ack in synchronous invocation
|
ack_event.wait(timeout=timeout)
|
||||||
time.sleep(0.05)
|
|
||||||
return response
|
return response
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Sync error for event {evt_id}: {e}")
|
logger.error(f"Sync error for event {evt_id}: {e}")
|
||||||
return {"success": False, "eventId": evt_id, "error": str(e)}
|
return {"success": False, "eventId": evt_id, "error": str(e)}
|
||||||
|
|
||||||
def batch_sync(self, records: List[Dict[str, Any]]) -> Dict[str, Any]:
|
def batch_sync(
|
||||||
|
self,
|
||||||
|
records: List[Dict[str, Any]],
|
||||||
|
timeout: float = 10.0,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Sync list of records in a single batch.
|
Sync list of records in a single batch.
|
||||||
|
Blocks until the server ack callback fires or timeout expires.
|
||||||
"""
|
"""
|
||||||
if not self.is_connected:
|
if not self.is_connected:
|
||||||
raise ConnectionError("DataStack client is not connected.")
|
raise ConnectionError("DataStack client is not connected.")
|
||||||
@@ -110,15 +127,22 @@ class DataStackClient:
|
|||||||
"data": r.get("data", r),
|
"data": r.get("data", r),
|
||||||
})
|
})
|
||||||
|
|
||||||
response = {"success": False, "processed": 0}
|
ack_event = threading.Event()
|
||||||
|
response: Dict[str, Any] = {
|
||||||
|
"success": False,
|
||||||
|
"processed": 0,
|
||||||
|
"error": "Timeout waiting for server batch acknowledgment",
|
||||||
|
}
|
||||||
|
|
||||||
def on_ack(res):
|
def on_ack(res):
|
||||||
nonlocal response
|
nonlocal response
|
||||||
|
if isinstance(res, dict):
|
||||||
response = res
|
response = res
|
||||||
|
ack_event.set()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
self.sio.emit("batch_sync", batch_payload, callback=on_ack)
|
self.sio.emit("batch_sync", batch_payload, callback=on_ack)
|
||||||
time.sleep(0.1)
|
ack_event.wait(timeout=timeout)
|
||||||
return response
|
return response
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Batch sync error: {e}")
|
logger.error(f"Batch sync error: {e}")
|
||||||
|
|||||||
+227
-24
@@ -2,57 +2,260 @@ const express = require('express');
|
|||||||
const http = require('http');
|
const http = require('http');
|
||||||
const cors = require('cors');
|
const cors = require('cors');
|
||||||
const { Server } = require('socket.io');
|
const { Server } = require('socket.io');
|
||||||
|
const { Pool } = require('pg');
|
||||||
const { setupSocketHandlers } = require('./socket/handlers');
|
const { setupSocketHandlers } = require('./socket/handlers');
|
||||||
const { CsvWriterWorker } = require('./workers/csvWriter');
|
const { CsvWriterWorker } = require('./workers/csvWriter');
|
||||||
|
|
||||||
const PORT = process.env.PORT_SOCKET || 4000;
|
const PORT = Number(process.env.PORT_SOCKET) || 4000;
|
||||||
const app = express();
|
const app = express();
|
||||||
app.use(cors({ origin: '*' }));
|
app.use(cors({ origin: '*' }));
|
||||||
app.use(express.json());
|
app.use(express.json({ limit: '10mb' }));
|
||||||
|
|
||||||
const bufferQueue = [];
|
const bufferQueue = [];
|
||||||
const writer = new CsvWriterWorker();
|
const writer = new CsvWriterWorker();
|
||||||
|
|
||||||
// Periodic Batch Flush: every 2000ms or when buffer hits 500
|
const runtimeConfig = {
|
||||||
const FLUSH_INTERVAL_MS = 2000;
|
jwtSecret:
|
||||||
const FLUSH_THRESHOLD = 500;
|
process.env.DATASTACK_JWT_SECRET ||
|
||||||
|
'datastack-jwt-default-secret-key-32-chars-long',
|
||||||
|
maxEventsPerSec: 100,
|
||||||
|
maxPayloadSizeBytes: 100 * 1024,
|
||||||
|
batchFlushThreshold: 500,
|
||||||
|
batchFlushIntervalMs: 2000,
|
||||||
|
fileRotationMb: 100,
|
||||||
|
enableIdempotency: true,
|
||||||
|
};
|
||||||
|
|
||||||
|
let dbPool = null;
|
||||||
|
if (process.env.DATABASE_URL) {
|
||||||
|
try {
|
||||||
|
dbPool = new Pool({
|
||||||
|
connectionString: process.env.DATABASE_URL,
|
||||||
|
connectionTimeoutMillis: 3000,
|
||||||
|
idleTimeoutMillis: 30000,
|
||||||
|
max: 5,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
dbPool = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Short-lived cache (2s TTL) for client status & token checks on high-frequency events
|
||||||
|
const clientAuthCache = new Map();
|
||||||
|
const CLIENT_CACHE_TTL_MS = 2000;
|
||||||
|
|
||||||
|
async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
|
||||||
|
if (!dbPool || !clientId) {
|
||||||
|
return { allowed: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
const now = Date.now();
|
||||||
|
if (!bypassCache) {
|
||||||
|
const cached = clientAuthCache.get(clientId);
|
||||||
|
if (cached && now - cached.checkedAt < CLIENT_CACHE_TTL_MS) {
|
||||||
|
if (cached.status === 'revoked') {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (cached.token && presentedToken && cached.token !== presentedToken) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return { allowed: true, rateLimit: cached.rateLimit };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const res = await dbPool.query(
|
||||||
|
`SELECT status, token, rate_limit FROM client_sdks WHERE client_id = $1`,
|
||||||
|
[clientId],
|
||||||
|
);
|
||||||
|
if (!res.rowCount || res.rowCount === 0) {
|
||||||
|
// Allow token-authenticated client if not explicitly revoked in table
|
||||||
|
return { allowed: true };
|
||||||
|
}
|
||||||
|
const row = res.rows[0];
|
||||||
|
const entry = {
|
||||||
|
status: row.status,
|
||||||
|
token: row.token,
|
||||||
|
rateLimit: Number(row.rate_limit) || 100,
|
||||||
|
checkedAt: now,
|
||||||
|
};
|
||||||
|
clientAuthCache.set(clientId, entry);
|
||||||
|
|
||||||
|
if (entry.status === 'revoked') {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (entry.token && presentedToken && entry.token !== presentedToken) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return { allowed: true, rateLimit: entry.rateLimit };
|
||||||
|
} catch {
|
||||||
|
return { allowed: true };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function recordClientActivity(clientId, count = 1) {
|
||||||
|
if (!dbPool || !clientId || count <= 0) return;
|
||||||
|
const nowIso = new Date().toISOString();
|
||||||
|
dbPool
|
||||||
|
.query(
|
||||||
|
`UPDATE client_sdks
|
||||||
|
SET total_events = total_events + $2,
|
||||||
|
last_active = $3,
|
||||||
|
updated_at = NOW()
|
||||||
|
WHERE client_id = $1`,
|
||||||
|
[clientId, count, nowIso],
|
||||||
|
)
|
||||||
|
.catch(() => undefined);
|
||||||
|
}
|
||||||
|
|
||||||
|
let isFlushing = false;
|
||||||
|
let flushTimer = null;
|
||||||
|
|
||||||
async function flushBuffer() {
|
async function flushBuffer() {
|
||||||
if (bufferQueue.length === 0) return;
|
if (bufferQueue.length === 0 || isFlushing) {
|
||||||
|
return { written: 0, files: [] };
|
||||||
|
}
|
||||||
|
isFlushing = true;
|
||||||
const batch = bufferQueue.splice(0, bufferQueue.length);
|
const batch = bufferQueue.splice(0, bufferQueue.length);
|
||||||
try {
|
try {
|
||||||
const result = await writer.writeBatch(batch);
|
const result = await writer.writeBatch(batch);
|
||||||
if (result.written > 0) {
|
if (result.written > 0) {
|
||||||
console.log(`[CSV Worker] Flushed ${result.written} events to ${result.files.length} file(s)`);
|
console.log(
|
||||||
|
`[CSV Worker] Flushed ${result.written} events to ${result.files.length} file(s)`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
return result;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error('[CSV Worker] Flush error:', err);
|
console.error('[CSV Worker] Flush error:', err);
|
||||||
bufferQueue.unshift(...batch); // Retry next cycle
|
bufferQueue.unshift(...batch);
|
||||||
|
return { written: 0, files: [] };
|
||||||
|
} finally {
|
||||||
|
isFlushing = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
setInterval(flushBuffer, FLUSH_INTERVAL_MS);
|
function scheduleFlushTimer(intervalMs) {
|
||||||
|
const safeInterval = Math.max(100, Number(intervalMs) || 2000);
|
||||||
|
runtimeConfig.batchFlushIntervalMs = safeInterval;
|
||||||
|
if (flushTimer) {
|
||||||
|
clearInterval(flushTimer);
|
||||||
|
}
|
||||||
|
flushTimer = setInterval(() => {
|
||||||
|
void flushBuffer();
|
||||||
|
}, safeInterval);
|
||||||
|
}
|
||||||
|
|
||||||
// Health check endpoint
|
scheduleFlushTimer(runtimeConfig.batchFlushIntervalMs);
|
||||||
app.get('/health', (_req, res) => {
|
|
||||||
res.json({
|
async function syncSettingsFromDatabase() {
|
||||||
status: 'online',
|
if (!dbPool) return;
|
||||||
service: 'DataStack Express Ingestion',
|
try {
|
||||||
port: PORT,
|
const res = await dbPool.query(
|
||||||
bufferSize: bufferQueue.length,
|
`SELECT max_events_per_sec, max_payload_size_kb, batch_flush_threshold,
|
||||||
uptime: process.uptime(),
|
batch_flush_interval_ms, file_rotation_mb, jwt_secret, enable_idempotency
|
||||||
});
|
FROM admin_settings
|
||||||
});
|
WHERE id = 'default'`,
|
||||||
|
);
|
||||||
|
if (!res.rowCount || res.rowCount === 0) return;
|
||||||
|
const row = res.rows[0];
|
||||||
|
|
||||||
|
runtimeConfig.maxEventsPerSec = Number(row.max_events_per_sec) || 100;
|
||||||
|
runtimeConfig.maxPayloadSizeBytes =
|
||||||
|
(Number(row.max_payload_size_kb) || 100) * 1024;
|
||||||
|
runtimeConfig.batchFlushThreshold =
|
||||||
|
Math.max(1, Number(row.batch_flush_threshold) || 500);
|
||||||
|
runtimeConfig.enableIdempotency = Boolean(row.enable_idempotency);
|
||||||
|
|
||||||
|
if (row.jwt_secret && String(row.jwt_secret).trim().length > 0) {
|
||||||
|
runtimeConfig.jwtSecret = String(row.jwt_secret).trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
const nextRotationMb = Number(row.file_rotation_mb) || 100;
|
||||||
|
if (nextRotationMb !== runtimeConfig.fileRotationMb) {
|
||||||
|
runtimeConfig.fileRotationMb = nextRotationMb;
|
||||||
|
writer.setMaxFileSizeMb(nextRotationMb);
|
||||||
|
}
|
||||||
|
|
||||||
|
const nextIntervalMs = Math.max(
|
||||||
|
100,
|
||||||
|
Number(row.batch_flush_interval_ms) || 2000,
|
||||||
|
);
|
||||||
|
if (nextIntervalMs !== runtimeConfig.batchFlushIntervalMs) {
|
||||||
|
scheduleFlushTimer(nextIntervalMs);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Keep existing runtimeConfig if table is not created yet
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (dbPool) {
|
||||||
|
void syncSettingsFromDatabase();
|
||||||
|
setInterval(() => {
|
||||||
|
void syncSettingsFromDatabase();
|
||||||
|
}, 3000);
|
||||||
|
}
|
||||||
|
|
||||||
const server = http.createServer(app);
|
const server = http.createServer(app);
|
||||||
const io = new Server(server, {
|
const io = new Server(server, {
|
||||||
cors: { origin: '*', methods: ['GET', 'POST'] },
|
cors: { origin: '*', methods: ['GET', 'POST'] },
|
||||||
});
|
});
|
||||||
|
|
||||||
setupSocketHandlers(io, bufferQueue, writer);
|
// Health check endpoint with activeSockets count
|
||||||
|
app.get('/health', (_req, res) => {
|
||||||
server.listen(PORT, () => {
|
res.json({
|
||||||
console.log(`[DataStack Ingestion Engine] Express + Socket.IO listening on port ${PORT}`);
|
status: 'online',
|
||||||
|
service: 'DataStack Express Ingestion',
|
||||||
|
port: PORT,
|
||||||
|
activeSockets: io.engine ? io.engine.clientsCount : 0,
|
||||||
|
bufferSize: bufferQueue.length,
|
||||||
|
uptime: process.uptime(),
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
module.exports = { app, server, io, bufferQueue };
|
// Internal flush endpoint triggered by Next.js POST /api/storage
|
||||||
|
app.post('/internal/flush', async (_req, res) => {
|
||||||
|
try {
|
||||||
|
const result = await flushBuffer();
|
||||||
|
res.json({
|
||||||
|
success: true,
|
||||||
|
flushedCount: result.written,
|
||||||
|
filesUpdated: result.files,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
res.status(500).json({
|
||||||
|
success: false,
|
||||||
|
error: err instanceof Error ? err.message : 'Flush failed',
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
setupSocketHandlers(io, bufferQueue, writer, {
|
||||||
|
getRuntimeConfig: () => runtimeConfig,
|
||||||
|
verifyClientInDb,
|
||||||
|
recordClientActivity,
|
||||||
|
onBufferPush: () => {
|
||||||
|
if (bufferQueue.length >= runtimeConfig.batchFlushThreshold) {
|
||||||
|
void flushBuffer();
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
server.listen(PORT, () => {
|
||||||
|
console.log(
|
||||||
|
`[DataStack Ingestion Engine] Express + Socket.IO listening on port ${PORT}`,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
module.exports = { app, server, io, bufferQueue, flushBuffer };
|
||||||
+241
-49
@@ -1,17 +1,120 @@
|
|||||||
const jwt = require("jsonwebtoken");
|
const jwt = require("jsonwebtoken");
|
||||||
|
|
||||||
|
const IDEMPOTENCY_TTL_MS = 5 * 60 * 1000; // 5 minutes
|
||||||
|
const IDEMPOTENCY_MAX_ENTRIES = 20000;
|
||||||
|
const CLEANUP_INTERVAL_MS = 60 * 1000;
|
||||||
|
|
||||||
function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||||
const jwtSecret = options.jwtSecret || process.env.DATASTACK_JWT_SECRET;
|
const getRuntimeConfig =
|
||||||
if (!jwtSecret) {
|
typeof options.getRuntimeConfig === "function"
|
||||||
throw new Error(
|
? options.getRuntimeConfig
|
||||||
"DATASTACK_JWT_SECRET must be configured before starting the socket server.",
|
: () => ({
|
||||||
);
|
jwtSecret:
|
||||||
}
|
options.jwtSecret ||
|
||||||
|
process.env.DATASTACK_JWT_SECRET ||
|
||||||
|
"datastack-jwt-default-secret-key-32-chars-long",
|
||||||
|
maxEventsPerSec: 100,
|
||||||
|
maxPayloadSizeBytes: 100 * 1024,
|
||||||
|
batchFlushThreshold: 500,
|
||||||
|
enableIdempotency: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
const verifyClientInDb =
|
||||||
|
typeof options.verifyClientInDb === "function"
|
||||||
|
? options.verifyClientInDb
|
||||||
|
: async () => ({ allowed: true });
|
||||||
|
|
||||||
|
const recordClientActivity =
|
||||||
|
typeof options.recordClientActivity === "function"
|
||||||
|
? options.recordClientActivity
|
||||||
|
: () => undefined;
|
||||||
|
|
||||||
|
const onBufferPush =
|
||||||
|
typeof options.onBufferPush === "function"
|
||||||
|
? options.onBufferPush
|
||||||
|
: () => undefined;
|
||||||
|
|
||||||
const idempotencyMap = new Map();
|
const idempotencyMap = new Map();
|
||||||
const rateLimitMap = new Map();
|
const rateLimitMap = new Map();
|
||||||
|
let lastCleanupAt = Date.now();
|
||||||
|
|
||||||
|
function cleanupMaps(now) {
|
||||||
|
if (
|
||||||
|
now - lastCleanupAt < CLEANUP_INTERVAL_MS &&
|
||||||
|
idempotencyMap.size <= IDEMPOTENCY_MAX_ENTRIES
|
||||||
|
) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
lastCleanupAt = now;
|
||||||
|
|
||||||
|
for (const [key, timestamp] of idempotencyMap.entries()) {
|
||||||
|
if (now - timestamp > IDEMPOTENCY_TTL_MS) {
|
||||||
|
idempotencyMap.delete(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [clientId, rl] of rateLimitMap.entries()) {
|
||||||
|
if (now - rl.start > 10000) {
|
||||||
|
rateLimitMap.delete(clientId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function validateSingleEvent(clientId, eventId, rawData, clientRateLimit) {
|
||||||
|
const config = getRuntimeConfig();
|
||||||
|
const now = Date.now();
|
||||||
|
cleanupMaps(now);
|
||||||
|
|
||||||
|
// 1. Idempotency check (with 5-minute TTL)
|
||||||
|
if (config.enableIdempotency && eventId) {
|
||||||
|
const key = `${clientId}:${eventId}`;
|
||||||
|
const existingTime = idempotencyMap.get(key);
|
||||||
|
if (existingTime && now - existingTime < IDEMPOTENCY_TTL_MS) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
duplicate: true,
|
||||||
|
reason: "Duplicate event dropped",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
idempotencyMap.set(key, now);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Rate limit check
|
||||||
|
const effectiveLimit = Math.min(
|
||||||
|
Number(clientRateLimit) || 100,
|
||||||
|
Number(config.maxEventsPerSec) || 100,
|
||||||
|
);
|
||||||
|
const rl = rateLimitMap.get(clientId);
|
||||||
|
if (!rl || now - rl.start >= 1000) {
|
||||||
|
rateLimitMap.set(clientId, { count: 1, start: now });
|
||||||
|
} else if (rl.count >= effectiveLimit) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
duplicate: false,
|
||||||
|
reason: `RATE_LIMIT_EXCEEDED: Limit of ${effectiveLimit} events/sec exceeded`,
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
rl.count += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Payload byte size check
|
||||||
|
const payloadStr =
|
||||||
|
typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {});
|
||||||
|
const byteLength = Buffer.byteLength(payloadStr, "utf8");
|
||||||
|
const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024;
|
||||||
|
if (byteLength > maxBytes) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
duplicate: false,
|
||||||
|
reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return { allowed: true, payloadStr };
|
||||||
|
}
|
||||||
|
|
||||||
// Middleware: Handshake Auth
|
// Middleware: Handshake Auth
|
||||||
io.use((socket, next) => {
|
io.use(async (socket, next) => {
|
||||||
const authHeader =
|
const authHeader =
|
||||||
socket.handshake.auth?.token ||
|
socket.handshake.auth?.token ||
|
||||||
socket.handshake.headers?.authorization ||
|
socket.handshake.headers?.authorization ||
|
||||||
@@ -22,12 +125,25 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const cleanToken = authHeader.startsWith("Bearer ")
|
const cleanToken = String(authHeader).startsWith("Bearer ")
|
||||||
? authHeader.slice(7)
|
? String(authHeader).slice(7).trim()
|
||||||
: authHeader;
|
: String(authHeader).trim();
|
||||||
const decoded = jwt.verify(cleanToken, jwtSecret);
|
const config = getRuntimeConfig();
|
||||||
|
const decoded = jwt.verify(cleanToken, config.jwtSecret);
|
||||||
|
|
||||||
|
const dbCheck = await verifyClientInDb(
|
||||||
|
decoded.clientId,
|
||||||
|
cleanToken,
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
if (!dbCheck.allowed) {
|
||||||
|
return next(new Error(dbCheck.reason || "CLIENT_REVOKED"));
|
||||||
|
}
|
||||||
|
|
||||||
socket.data.clientId = decoded.clientId;
|
socket.data.clientId = decoded.clientId;
|
||||||
socket.data.rateLimit = decoded.rateLimit || 100;
|
socket.data.token = cleanToken;
|
||||||
|
socket.data.rateLimit =
|
||||||
|
dbCheck.rateLimit || decoded.rateLimit || config.maxEventsPerSec || 100;
|
||||||
next();
|
next();
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
next(new Error(`AUTH_INVALID: ${err.message}`));
|
next(new Error(`AUTH_INVALID: ${err.message}`));
|
||||||
@@ -36,50 +152,64 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
|
|
||||||
io.on("connection", (socket) => {
|
io.on("connection", (socket) => {
|
||||||
const clientId = socket.data.clientId;
|
const clientId = socket.data.clientId;
|
||||||
const rateLimit = socket.data.rateLimit;
|
|
||||||
socket.join(`client:${clientId}`);
|
socket.join(`client:${clientId}`);
|
||||||
|
|
||||||
// Single event
|
// Single event
|
||||||
socket.on("sync_data", (payload, callback) => {
|
socket.on("sync_data", async (payload, callback) => {
|
||||||
try {
|
try {
|
||||||
const eventId =
|
const dbCheck = await verifyClientInDb(
|
||||||
payload.eventId ||
|
clientId,
|
||||||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
socket.data.token,
|
||||||
const timestamp = payload.timestamp || new Date().toISOString();
|
false,
|
||||||
|
);
|
||||||
// Idempotency
|
if (!dbCheck.allowed) {
|
||||||
const key = `${clientId}:${eventId}`;
|
if (callback) {
|
||||||
if (idempotencyMap.has(key)) {
|
callback({
|
||||||
if (callback)
|
success: false,
|
||||||
callback({ success: true, eventId, note: "DUPLICATE_DROPPED" });
|
error: dbCheck.reason || "CLIENT_REVOKED",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
socket.disconnect(true);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
idempotencyMap.set(key, Date.now());
|
|
||||||
|
|
||||||
// Rate limit
|
const effectiveClientRate =
|
||||||
const now = Date.now();
|
dbCheck.rateLimit || socket.data.rateLimit || 100;
|
||||||
const rl = rateLimitMap.get(clientId);
|
const safePayload = payload && typeof payload === "object" ? payload : {};
|
||||||
if (!rl || now - rl.start >= 1000) {
|
const eventId =
|
||||||
rateLimitMap.set(clientId, { count: 1, start: now });
|
safePayload.eventId ||
|
||||||
} else if (rl.count >= rateLimit) {
|
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||||
if (callback)
|
const timestamp = safePayload.timestamp || new Date().toISOString();
|
||||||
callback({ success: false, error: "RATE_LIMIT_EXCEEDED" });
|
|
||||||
|
const check = validateSingleEvent(
|
||||||
|
clientId,
|
||||||
|
eventId,
|
||||||
|
safePayload.data,
|
||||||
|
effectiveClientRate,
|
||||||
|
);
|
||||||
|
if (!check.allowed) {
|
||||||
|
if (check.duplicate) {
|
||||||
|
if (callback) {
|
||||||
|
callback({ success: true, eventId, note: "DUPLICATE_DROPPED" });
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (callback) {
|
||||||
|
callback({ success: false, eventId, error: check.reason });
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
} else {
|
|
||||||
rl.count += 1;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const event = {
|
const event = {
|
||||||
clientId,
|
clientId,
|
||||||
eventId,
|
eventId,
|
||||||
timestamp,
|
timestamp,
|
||||||
payload:
|
payload: check.payloadStr,
|
||||||
typeof payload.data === "string"
|
|
||||||
? payload.data
|
|
||||||
: JSON.stringify(payload.data),
|
|
||||||
};
|
};
|
||||||
|
|
||||||
bufferQueue.push(event);
|
bufferQueue.push(event);
|
||||||
|
recordClientActivity(clientId, 1);
|
||||||
|
onBufferPush();
|
||||||
|
|
||||||
if (callback) callback({ success: true, eventId });
|
if (callback) callback({ success: true, eventId });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@@ -88,32 +218,94 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Batch event
|
// Batch event
|
||||||
socket.on("batch_sync", (batch, callback) => {
|
socket.on("batch_sync", async (batch, callback) => {
|
||||||
|
try {
|
||||||
if (!Array.isArray(batch)) {
|
if (!Array.isArray(batch)) {
|
||||||
if (callback) callback({ success: false, error: "Expected array" });
|
if (callback) callback({ success: false, error: "Expected array" });
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let count = 0;
|
const dbCheck = await verifyClientInDb(
|
||||||
|
clientId,
|
||||||
|
socket.data.token,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
if (!dbCheck.allowed) {
|
||||||
|
if (callback) {
|
||||||
|
callback({
|
||||||
|
success: false,
|
||||||
|
processed: 0,
|
||||||
|
rejected: batch.length,
|
||||||
|
error: dbCheck.reason || "CLIENT_REVOKED",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
socket.disconnect(true);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const effectiveClientRate =
|
||||||
|
dbCheck.rateLimit || socket.data.rateLimit || 100;
|
||||||
|
let processed = 0;
|
||||||
|
let rejected = 0;
|
||||||
|
const errors = [];
|
||||||
|
|
||||||
for (const item of batch) {
|
for (const item of batch) {
|
||||||
|
const safeItem = item && typeof item === "object" ? item : {};
|
||||||
const eventId =
|
const eventId =
|
||||||
item.eventId ||
|
safeItem.eventId ||
|
||||||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||||
const timestamp = item.timestamp || new Date().toISOString();
|
const timestamp = safeItem.timestamp || new Date().toISOString();
|
||||||
|
const rawData =
|
||||||
|
safeItem.data !== undefined ? safeItem.data : safeItem;
|
||||||
|
|
||||||
|
const check = validateSingleEvent(
|
||||||
|
clientId,
|
||||||
|
eventId,
|
||||||
|
rawData,
|
||||||
|
effectiveClientRate,
|
||||||
|
);
|
||||||
|
if (!check.allowed) {
|
||||||
|
if (!check.duplicate) {
|
||||||
|
rejected += 1;
|
||||||
|
if (errors.length < 5) {
|
||||||
|
errors.push(`${eventId}: ${check.reason}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
bufferQueue.push({
|
bufferQueue.push({
|
||||||
clientId,
|
clientId,
|
||||||
eventId,
|
eventId,
|
||||||
timestamp,
|
timestamp,
|
||||||
payload:
|
payload: check.payloadStr,
|
||||||
typeof item.data === "string"
|
|
||||||
? item.data
|
|
||||||
: JSON.stringify(item.data),
|
|
||||||
});
|
});
|
||||||
count++;
|
processed += 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (callback) callback({ success: true, processed: count });
|
if (processed > 0) {
|
||||||
|
recordClientActivity(clientId, processed);
|
||||||
|
onBufferPush();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (callback) {
|
||||||
|
callback({
|
||||||
|
success: true,
|
||||||
|
processed,
|
||||||
|
rejected,
|
||||||
|
...(errors.length > 0 ? { errors } : {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
if (callback) {
|
||||||
|
callback({
|
||||||
|
success: false,
|
||||||
|
processed: 0,
|
||||||
|
rejected: Array.isArray(batch) ? batch.length : 0,
|
||||||
|
error: err.message,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,6 +14,13 @@ class CsvWriterWorker {
|
|||||||
this.ensureBaseDir();
|
this.ensureBaseDir();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
setMaxFileSizeMb(megabytes) {
|
||||||
|
const mb = Number(megabytes);
|
||||||
|
if (Number.isFinite(mb) && mb >= 1) {
|
||||||
|
this.maxFileSize = Math.floor(mb * 1024 * 1024);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
ensureBaseDir() {
|
ensureBaseDir() {
|
||||||
if (!fs.existsSync(this.baseDir)) {
|
if (!fs.existsSync(this.baseDir)) {
|
||||||
fs.mkdirSync(this.baseDir, { recursive: true });
|
fs.mkdirSync(this.baseDir, { recursive: true });
|
||||||
@@ -21,7 +28,7 @@ class CsvWriterWorker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
getTargetFilePath(clientId, dateStr) {
|
getTargetFilePath(clientId, dateStr) {
|
||||||
const safeClientId = clientId.replace(/[^a-zA-Z0-9_-]/g, '_');
|
const safeClientId = String(clientId || 'unknown').replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||||
const clientDir = path.join(this.baseDir, safeClientId);
|
const clientDir = path.join(this.baseDir, safeClientId);
|
||||||
if (!fs.existsSync(clientDir)) {
|
if (!fs.existsSync(clientDir)) {
|
||||||
fs.mkdirSync(clientDir, { recursive: true });
|
fs.mkdirSync(clientDir, { recursive: true });
|
||||||
|
|||||||
@@ -1,3 +0,0 @@
|
|||||||
clientId,eventId,timestamp,payload
|
|
||||||
py_scraper_prod_02,evt_scrape_101,2026-10-23T06:10:00.000Z,"{""spider"":""market_catalog"",""pages"":42,""errors"":0,""bytes"":89200}"
|
|
||||||
py_scraper_prod_02,evt_scrape_102,2026-10-23T06:25:00.000Z,"{""spider"":""price_indexer"",""pages"":18,""errors"":0,""bytes"":41200}"
|
|
||||||
|
@@ -1,4 +0,0 @@
|
|||||||
clientId,eventId,timestamp,payload
|
|
||||||
py_telemetry_agent_01,evt_seed_001,2026-10-23T06:00:00.000Z,"{""action"":""agent_boot"",""cpu"":18.4,""ram_mb"":512,""status"":""nominal""}"
|
|
||||||
py_telemetry_agent_01,evt_seed_002,2026-10-23T06:15:00.000Z,"{""action"":""heartbeat"",""cpu"":22.1,""ram_mb"":524,""status"":""nominal""}"
|
|
||||||
py_telemetry_agent_01,evt_seed_003,2026-10-23T06:30:00.000Z,"{""action"":""batch_processed"",""items_count"":142,""latency_ms"":18}"
|
|
||||||
|
@@ -1,32 +0,0 @@
|
|||||||
import { useState, useEffect, useCallback } from 'react';
|
|
||||||
|
|
||||||
export function useVconMetrics(refreshIntervalMs = 3000) {
|
|
||||||
const [metrics, setMetrics] = useState<any>(null);
|
|
||||||
const [loading, setLoading] = useState(true);
|
|
||||||
const [error, setError] = useState<string | null>(null);
|
|
||||||
|
|
||||||
const fetchMetrics = useCallback(async () => {
|
|
||||||
try {
|
|
||||||
const res = await fetch('/api/metrics');
|
|
||||||
if (res.ok) {
|
|
||||||
const data = await res.json();
|
|
||||||
setMetrics(data);
|
|
||||||
setError(null);
|
|
||||||
} else {
|
|
||||||
setError('Failed to fetch metrics');
|
|
||||||
}
|
|
||||||
} catch (err: any) {
|
|
||||||
setError(err.message);
|
|
||||||
} finally {
|
|
||||||
setLoading(false);
|
|
||||||
}
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
fetchMetrics();
|
|
||||||
const timer = setInterval(fetchMetrics, refreshIntervalMs);
|
|
||||||
return () => clearInterval(timer);
|
|
||||||
}, [fetchMetrics, refreshIntervalMs]);
|
|
||||||
|
|
||||||
return { metrics, loading, error, refetch: fetchMetrics };
|
|
||||||
}
|
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
export async function vconApiFetch<T>(endpoint: string, options?: RequestInit): Promise<T> {
|
|
||||||
const res = await fetch(endpoint, {
|
|
||||||
...options,
|
|
||||||
headers: {
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
...options?.headers,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!res.ok) {
|
|
||||||
const errorBody = await res.json().catch(() => ({}));
|
|
||||||
throw new Error(errorBody.error || `API error ${res.status}: ${res.statusText}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return res.json();
|
|
||||||
}
|
|
||||||
Reference in new issue
Block a user