diff --git a/.gitignore b/.gitignore index 17cc55b..5cf545b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ coverage/ .env* !.env.example notes/ +storage/data/ diff --git a/app/(landing)/page.tsx b/app/(landing)/page.tsx index 92aabbe..5be914c 100644 --- a/app/(landing)/page.tsx +++ b/app/(landing)/page.tsx @@ -9,7 +9,7 @@ import { redirect } from 'next/navigation'; export const dynamic = 'force-dynamic'; export default async function LandingPage() { - if (process.env.NODE_ENV === 'production' && process.env.DATABASE_URL) { + if (process.env.DATABASE_URL) { const isConfigured = await isAdminSetupComplete().catch(() => false); if (!isConfigured) redirect('/vcon/setup'); } diff --git a/app/api/auth/profile/route.ts b/app/api/auth/profile/route.ts index ef36524..8baee38 100644 --- a/app/api/auth/profile/route.ts +++ b/app/api/auth/profile/route.ts @@ -100,7 +100,12 @@ export async function PUT(request: NextRequest) { return NextResponse.json({ error: result.error }, { status: 401 }); } - const expiresAt = Date.now() + SESSION_SECONDS * 1000; + // Preserve remaining session duration (e.g., 30-day "Remember me" session) + const remainingSeconds = Math.max( + SESSION_SECONDS, + Math.ceil((session.expiresAt - Date.now()) / 1000), + ); + const expiresAt = Date.now() + remainingSeconds * 1000; const nextToken = await createAdminSessionToken( result.email, sessionSecret, @@ -121,7 +126,7 @@ export async function PUT(request: NextRequest) { response.cookies.set( SESSION_COOKIE, nextToken, - getSessionCookieOptions(request, SESSION_SECONDS), + getSessionCookieOptions(request, remainingSeconds), ); return response; diff --git a/app/api/auth/setup/route.ts b/app/api/auth/setup/route.ts index d9839b0..2881166 100644 --- a/app/api/auth/setup/route.ts +++ b/app/api/auth/setup/route.ts @@ -23,21 +23,12 @@ export async function POST(request: NextRequest) { const sessionSecret = process.env.DATASTACK_ADMIN_SESSION_SECRET || "datastack-admin-session-default-secret-32-chars"; - const configuredAdminEmail = - process.env.DATASTACK_ADMIN_EMAIL?.trim().toLowerCase() || - "admin@datastack.dev"; if (!sessionSecret || sessionSecret.length < 32) { return NextResponse.json( { error: "Admin session configuration is unavailable." }, { status: 503 }, ); } - if (!configuredAdminEmail || !validEmail(configuredAdminEmail)) { - return NextResponse.json( - { error: "Admin setup configuration is unavailable." }, - { status: 503 }, - ); - } try { const body = await request.json(); @@ -52,17 +43,9 @@ export async function POST(request: NextRequest) { { status: 400 }, ); } - if (email !== configuredAdminEmail) { + if (password.length < 8 || password.length > 256) { return NextResponse.json( - { - error: "Use the administrator email configured for this deployment.", - }, - { status: 403 }, - ); - } - if (password.length < 12 || password.length > 256) { - return NextResponse.json( - { error: "Choose a password between 12 and 256 characters." }, + { error: "Choose a password between 8 and 256 characters." }, { status: 400 }, ); } diff --git a/app/api/clients/[id]/route.ts b/app/api/clients/[id]/route.ts index 94d3ece..ce37e00 100644 --- a/app/api/clients/[id]/route.ts +++ b/app/api/clients/[id]/route.ts @@ -6,7 +6,7 @@ export async function GET( { params }: { params: Promise<{ id: string }> }, ) { const { id } = await params; - const client = SDKManager.getClient(id); + const client = await SDKManager.getClientAsync(id); if (!client) { return NextResponse.json({ error: "Client not found" }, { status: 404 }); } @@ -24,7 +24,7 @@ export async function PATCH( // Special action: regenerate token if (body.action === "regenerate_token") { - const refreshed = SDKManager.regenerateToken(id); + const refreshed = await SDKManager.regenerateTokenAsync(id); if (!refreshed) { return NextResponse.json( { error: "Client not found" }, @@ -34,7 +34,7 @@ export async function PATCH( return NextResponse.json({ client: refreshed }); } - const updated = SDKManager.updateClient(id, body); + const updated = await SDKManager.updateClientAsync(id, body); if (!updated) { return NextResponse.json({ error: "Client not found" }, { status: 404 }); } @@ -50,7 +50,7 @@ export async function DELETE( { params }: { params: Promise<{ id: string }> }, ) { const { id } = await params; - const deleted = SDKManager.deleteClient(id); + const deleted = await SDKManager.deleteClientAsync(id); if (!deleted) { return NextResponse.json({ error: "Client not found" }, { status: 404 }); } diff --git a/app/api/clients/route.ts b/app/api/clients/route.ts index 6b51cbb..2b7250f 100644 --- a/app/api/clients/route.ts +++ b/app/api/clients/route.ts @@ -3,9 +3,11 @@ import { NextRequest, NextResponse } from "next/server"; export async function GET() { try { - const clients = SDKManager.listClients().map( - ({ secret, token, ...client }) => client, - ); + const allClients = await SDKManager.listClientsAsync(); + const clients = allClients.map(({ secret, ...client }) => { + void secret; + return client; + }); return NextResponse.json({ clients, count: clients.length }); } catch (err: unknown) { const message = @@ -26,17 +28,21 @@ export async function POST(req: NextRequest) { ); } - const existing = SDKManager.getClient(clientId); + const safeClientId = String(clientId) + .trim() + .toLowerCase() + .replace(/[^a-z0-9_-]/g, "_"); + const existing = await SDKManager.getClientAsync(safeClientId); if (existing) { return NextResponse.json( - { error: `Client '${clientId}' already exists` }, + { error: `Client '${safeClientId}' already exists` }, { status: 409 }, ); } - const newClient = SDKManager.registerClient({ - clientId, - name, + const newClient = await SDKManager.registerClientAsync({ + clientId: safeClientId, + name: String(name), rateLimit: rateLimit ? Number(rateLimit) : 100, }); diff --git a/app/api/events/route.ts b/app/api/events/route.ts index f6bf8e2..07aaeca 100644 --- a/app/api/events/route.ts +++ b/app/api/events/route.ts @@ -1,15 +1,77 @@ -import { NextRequest, NextResponse } from 'next/server'; +import { SettingsManager } from '@/core/adminSettings/settingsManager'; +import { verifyAdminSessionToken } from '@/core/auth/adminSession'; +import { AuthService } from '@/core/auth/jwt'; import { DataBufferEngine } from '@/core/data/buffer'; import { SDKManager } from '@/core/sdk/sdkManager'; import { SecurityEngine } from '@/core/Security/security'; +import { CSVStorageManager, type DataEvent } from '@/core/Storage/csvStorage'; +import { parse } from 'csv-parse'; +import { NextRequest, NextResponse } from 'next/server'; +import { createReadStream } from 'node:fs'; + +const SESSION_COOKIE = 'datastack_admin_session'; + +async function getRecentStoredEvents( + limit: number, + clientId?: string, +): Promise { + const files = CSVStorageManager.listAllFiles() + .filter((f) => !clientId || clientId === 'all' || f.clientId === clientId) + .slice(0, 5); + + const collected: DataEvent[] = []; + + for (const file of files) { + const fileEvents: DataEvent[] = []; + const parser = createReadStream(file.filePath).pipe( + parse({ columns: true, skip_empty_lines: true, bom: true }), + ); + + try { + for await (const record of parser as AsyncIterable< + Record + >) { + if (!record.eventId) continue; + fileEvents.push({ + clientId: record.clientId || file.clientId, + eventId: record.eventId, + timestamp: record.timestamp || `${file.date}T00:00:00.000Z`, + payload: record.payload || '', + }); + } + } catch { + parser.destroy(); + } + + if (fileEvents.length > 0) { + collected.push(...fileEvents.slice(-limit)); + } + } + + return collected; +} export async function GET(req: NextRequest) { try { const { searchParams } = new URL(req.url); const clientId = searchParams.get('clientId') || undefined; - const limit = Number(searchParams.get('limit')) || 50; + const limit = Math.min(200, Math.max(1, Number(searchParams.get('limit')) || 50)); + + const bufferedEvents = DataBufferEngine.getRecentEvents(limit, clientId); + const storedEvents = await getRecentStoredEvents(limit, clientId); + + const seenIds = new Set(); + const merged: DataEvent[] = []; + + for (const evt of [...bufferedEvents, ...storedEvents]) { + if (!evt.eventId || seenIds.has(evt.eventId)) continue; + seenIds.add(evt.eventId); + merged.push(evt); + } + + merged.sort((a, b) => (b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0)); + const events = merged.slice(0, limit); - const events = DataBufferEngine.getRecentEvents(limit, clientId); return NextResponse.json({ events, count: events.length }); } catch (err: unknown) { const message = err instanceof Error ? err.message : 'Events retrieval failed'; @@ -19,16 +81,97 @@ export async function GET(req: NextRequest) { export async function POST(req: NextRequest) { try { - const body = await req.json(); - const clientId = body.clientId || 'py_telemetry_agent_01'; - const eventId = body.eventId || `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`; - const timestamp = body.timestamp || new Date().toISOString(); - const data = body.data || { sample: 'Realtime event via DataStack API', rand: Math.random() }; + const settings = await SettingsManager.getSettingsAsync(); - // Validate security - const val = SecurityEngine.validateEvent(clientId, eventId, data, 200); + // Verify either Client Bearer JWT or Admin Session Cookie + const authHeader = req.headers.get('authorization'); + const sessionSecret = + process.env.DATASTACK_ADMIN_SESSION_SECRET || + 'datastack-admin-session-default-secret-32-chars'; + const sessionCookie = req.cookies.get(SESSION_COOKIE)?.value; + + let tokenClientId: string | null = null; + let tokenRateLimit: number | undefined; + let rawBearerToken: string | null = null; + + if (authHeader && authHeader.toLowerCase().startsWith('bearer ')) { + rawBearerToken = authHeader.slice(7).trim(); + const verified = AuthService.verifyClientToken(rawBearerToken); + if (!verified.valid || !verified.payload) { + return NextResponse.json( + { error: verified.error || 'Invalid client JWT token.' }, + { status: 401 }, + ); + } + tokenClientId = verified.payload.clientId; + tokenRateLimit = verified.payload.rateLimit; + } else { + const isAdmin = Boolean( + sessionSecret && + sessionCookie && + (await verifyAdminSessionToken(sessionCookie, sessionSecret)), + ); + if (!isAdmin) { + return NextResponse.json( + { error: 'Authentication required. Provide a valid Client Bearer JWT or Admin Session.' }, + { status: 401 }, + ); + } + } + + const body = await req.json(); + const clientId = String( + tokenClientId || body.clientId || 'py_telemetry_agent_01', + ) + .trim() + .toLowerCase() + .replace(/[^a-z0-9_-]/g, '_'); + + const registeredClient = await SDKManager.getClientAsync(clientId); + if (registeredClient) { + if (registeredClient.status === 'revoked') { + return NextResponse.json( + { error: `Client '${clientId}' access has been revoked by administrator.` }, + { status: 403 }, + ); + } + if ( + rawBearerToken && + registeredClient.token && + registeredClient.token !== rawBearerToken + ) { + return NextResponse.json( + { error: 'Client credentials have been rotated. Use the latest token.' }, + { status: 401 }, + ); + } + } + + const eventId = + body.eventId || + `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`; + const timestamp = body.timestamp || new Date().toISOString(); + const data = + body.data !== undefined + ? body.data + : { sample: 'Realtime event via DataStack API', rand: Math.random() }; + + const effectiveClientRate = + registeredClient?.rateLimit || + tokenRateLimit || + settings.maxEventsPerSec || + 100; + + // Validate security (idempotency, rate limit, payload size) + const val = SecurityEngine.validateEvent( + clientId, + eventId, + data, + effectiveClientRate, + ); if (!val.allowed) { - return NextResponse.json({ error: val.reason }, { status: 429 }); + const status = val.code === 'PAYLOAD_TOO_LARGE' ? 413 : 429; + return NextResponse.json({ error: val.reason, code: val.code }, { status }); } const event = { @@ -43,7 +186,7 @@ export async function POST(req: NextRequest) { return NextResponse.json({ success: true, event }); } catch (err: unknown) { - const message = err instanceof Error ? err.message : 'Event simulation failed'; + const message = err instanceof Error ? err.message : 'Event ingestion failed'; return NextResponse.json({ error: message }, { status: 500 }); } } diff --git a/app/api/export/route.ts b/app/api/export/route.ts index 44e9b08..7d31b8e 100644 --- a/app/api/export/route.ts +++ b/app/api/export/route.ts @@ -1,3 +1,4 @@ +import { DataBufferEngine } from "@/core/data/buffer"; import { CSVStorageManager, type StorageFileMeta, @@ -107,6 +108,18 @@ function streamFile(file: StorageFileMeta) { export async function GET(request: NextRequest) { try { + await DataBufferEngine.flush(); + const ingestionBaseUrl = + process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"; + try { + await fetch(`${ingestionBaseUrl}/internal/flush`, { + method: "POST", + signal: AbortSignal.timeout(1500), + }); + } catch { + // Port 4000 may not be running in isolated single-process dev mode + } + const { searchParams } = new URL(request.url); const clientId = searchParams.get("clientId") || "all"; const filename = searchParams.get("filename"); diff --git a/app/api/settings/route.ts b/app/api/settings/route.ts index 8c8b8ad..4cb6bbe 100644 --- a/app/api/settings/route.ts +++ b/app/api/settings/route.ts @@ -3,7 +3,7 @@ import { NextRequest, NextResponse } from "next/server"; export async function GET() { try { - const settings = SettingsManager.getPublicSettings(); + const settings = await SettingsManager.getPublicSettingsAsync(); return NextResponse.json({ settings }); } catch (err: unknown) { const message = err instanceof Error ? err.message : "Settings read failed"; @@ -14,9 +14,10 @@ export async function GET() { export async function POST(req: NextRequest) { try { const body = await req.json(); - SettingsManager.updateSettings(body); + await SettingsManager.updateSettingsAsync(body); + const settings = await SettingsManager.getPublicSettingsAsync(); return NextResponse.json({ - settings: SettingsManager.getPublicSettings(), + settings, success: true, }); } catch (err: unknown) { diff --git a/app/api/storage/route.ts b/app/api/storage/route.ts index 4a7a20e..9f098ea 100644 --- a/app/api/storage/route.ts +++ b/app/api/storage/route.ts @@ -23,12 +23,40 @@ export async function GET(req: NextRequest) { export async function POST(_req: NextRequest) { try { - // Manual flush buffer to disk - const res = await DataBufferEngine.flush(); + // 1. Flush local Next.js DataBufferEngine + const localRes = await DataBufferEngine.flush(); + + // 2. Also flush Port 4000 Socket.IO ingestion buffer + let remoteCount = 0; + let remoteFiles: string[] = []; + const ingestionBaseUrl = ( + process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000' + ).replace(/\/$/, ''); + + try { + const ingestRes = await fetch(`${ingestionBaseUrl}/internal/flush`, { + method: 'POST', + cache: 'no-store', + }); + if (ingestRes.ok) { + const ingestData = await ingestRes.json(); + remoteCount = Number(ingestData.flushedCount) || 0; + if (Array.isArray(ingestData.filesUpdated)) { + remoteFiles = ingestData.filesUpdated; + } + } + } catch { + // Port 4000 server may be offline in standalone Next.js dev mode + } + + const combinedFiles = Array.from( + new Set([...localRes.files, ...remoteFiles]), + ); + return NextResponse.json({ success: true, - flushedCount: res.count, - filesUpdated: res.files, + flushedCount: localRes.count + remoteCount, + filesUpdated: combinedFiles, }); } catch (err: unknown) { const message = err instanceof Error ? err.message : 'Manual flush failed'; diff --git a/app/vcon/login/page.tsx b/app/vcon/login/page.tsx index a69a17d..30578fd 100644 --- a/app/vcon/login/page.tsx +++ b/app/vcon/login/page.tsx @@ -6,10 +6,12 @@ export const runtime = 'nodejs'; export const dynamic = 'force-dynamic'; export default async function LoginPage() { - if (process.env.NODE_ENV === 'production' && process.env.DATABASE_URL) { + if (process.env.DATABASE_URL) { const isConfigured = await isAdminSetupComplete().catch(() => false); if (!isConfigured) redirect('/vcon/setup'); } - return ; + const showDemoHint = !process.env.DATABASE_URL && process.env.NODE_ENV !== 'production'; + + return ; } diff --git a/app/vcon/sdk/page.tsx b/app/vcon/sdk/page.tsx index 0beb7c4..1ad0422 100644 --- a/app/vcon/sdk/page.tsx +++ b/app/vcon/sdk/page.tsx @@ -38,7 +38,9 @@ export default function SdkGuidePage() { const list = setCached('vcon:clients', data.clients || []); setClients(list); if (list.length > 0) { - setSelectedClientId((prev) => prev || list[0].clientId); + setSelectedClientId((prev) => + list.some((c: any) => c.clientId === prev) ? prev : list[0].clientId + ); } } } catch { @@ -84,14 +86,26 @@ if client.connect(): copySnippet(pythonSnippet, 'Snippet')} - > - Copy Code - + <> + {activeClient?.token && ( + + )} + + } > diff --git a/components/sections/AdminLogin.tsx b/components/sections/AdminLogin.tsx index 8f4c6b0..1b614e1 100644 --- a/components/sections/AdminLogin.tsx +++ b/components/sections/AdminLogin.tsx @@ -14,9 +14,13 @@ function safeNextPath() { : '/vcon/dashboard'; } -export function AdminLogin() { - const [email, setEmail] = useState('admin@datastack.dev'); - const [password, setPassword] = useState('admin12345678'); +export function AdminLogin({ + showDemoHint = false, +}: { + showDemoHint?: boolean; +}) { + const [email, setEmail] = useState(showDemoHint ? 'admin@datastack.dev' : ''); + const [password, setPassword] = useState(showDemoHint ? 'admin12345678' : ''); const [remember, setRemember] = useState(true); const [showPassword, setShowPassword] = useState(false); const [isSubmitting, setIsSubmitting] = useState(false); @@ -144,16 +148,18 @@ export function AdminLogin() { Sign in - + {showDemoHint && ( + + )}