From fcc4f4db2d7f4be9125c834e41777fa5e8ad2e57 Mon Sep 17 00:00:00 2001 From: i vib <43686281-ivibtools@users.noreply.gitlab.com> Date: Fri, 23 Oct 2026 19:56:38 -0700 Subject: [PATCH] chore: publish current DataStack updates Publish the current application, design, and Phase 1 deployment updates. Excludes ignored personal notes, local MCP config, and generated TypeScript build info. --- .dockerignore | 13 + .env.example | 12 + .gitignore | 1 + .vscode/extensions.json | 5 + AGENTS.md | 10 + Design-Files/DataStack -Data-Pages.html | 483 + Design-Files/DataStack — Clients-Pages.html | 626 + Design-Files/DataStack — Export-page.html | 538 + Design-Files/DataStack — Settings-pages.html | 714 + Design-Files/DataStack-Dashboard-Pages.html | 546 + Design-Files/Landing-Pages.html | 290 + Design-Files/Login Page — Design.html | 293 + README.md | 34 +- app/(landing)/page.tsx | 16 + app/api/auth/login/route.ts | 76 + app/api/clients/[id]/route.ts | 28 +- app/api/clients/route.ts | 20 +- app/api/data/route.ts | 128 + app/api/export/route.ts | 206 +- app/api/settings/route.ts | 18 +- app/globals.css | 135 +- app/layout.tsx | 10 +- app/not-found.tsx | 8 +- app/page.tsx | 17 - app/vcon/clients/page.tsx | 449 +- app/vcon/dashboard/page.tsx | 520 +- app/vcon/data/page.tsx | 287 +- app/vcon/export/page.tsx | 195 +- app/vcon/layout.tsx | 6 +- app/vcon/login/page.tsx | 5 + app/vcon/sdk/page.tsx | 15 +- app/vcon/settings/page.tsx | 209 +- app/vcon/simulator/page.tsx | 17 +- app/vcon/storage/page.tsx | 53 +- components/layout/AdminPageHeader.tsx | 19 + components/layout/Footer.tsx | 58 +- components/layout/MainLayout.tsx | 18 +- components/layout/Navbar.tsx | 121 +- components/layout/Sidebar.tsx | 87 +- components/sections/AdminLogin.tsx | 180 + components/sections/ClientsFilters.tsx | 58 + components/sections/ClientsManager.tsx | 475 + components/sections/ClientsTable.tsx | 196 + components/sections/DashboardOverview.tsx | 241 + components/sections/DataBrowser.tsx | 353 + components/sections/ExportWorkspace.tsx | 316 + components/sections/Features.tsx | 68 +- components/sections/Hero.tsx | 111 +- components/sections/LandingAbout.tsx | 17 + components/sections/NewClientModal.tsx | 132 + components/sections/SdkSection.tsx | 33 + components/sections/SettingsWorkspace.tsx | 273 + components/sections/clients/types.ts | 16 + components/sections/index.ts | 8 + components/ui/ActivityPreview.tsx | 35 + components/ui/Button.tsx | 12 +- components/ui/Card.tsx | 16 +- components/ui/ClientList.tsx | 37 + components/ui/CodeBlock.tsx | 14 + components/ui/CopyButton.tsx | 49 + components/ui/FeatureItem.tsx | 20 + components/ui/Form.tsx | 16 +- components/ui/Input.tsx | 20 +- components/ui/Modal.tsx | 29 +- components/ui/Panel.tsx | 33 + components/ui/Pill.tsx | 29 + components/ui/SectionHeading.tsx | 18 + components/ui/SettingsLayout.tsx | 72 + components/ui/StatCard.tsx | 22 + components/ui/StatStrip.tsx | 28 + components/ui/Table.tsx | 12 +- components/ui/Text.tsx | 20 +- components/ui/Toast.tsx | 10 +- components/ui/index.ts | 29 +- core/Database/database.ts | 16 +- core/Security/security.ts | 42 +- core/adminSettings/settingsManager.ts | 32 +- core/auth/adminSession.ts | 73 + core/auth/jwt.ts | 33 +- core/sdk/sdkManager.ts | 50 +- docker/Dockerfile.server | 16 + docker/Dockerfile.web | 28 + docker/docker-compose.yml | 75 +- docker/manifest.md | 22 +- middleware.ts | 51 + package-lock.json | 13691 +++++++++++++++++ package.json | 1 + server/socket/handlers.js | 47 +- 88 files changed, 21248 insertions(+), 2213 deletions(-) create mode 100644 .dockerignore create mode 100644 .vscode/extensions.json create mode 100644 Design-Files/DataStack -Data-Pages.html create mode 100644 Design-Files/DataStack — Clients-Pages.html create mode 100644 Design-Files/DataStack — Export-page.html create mode 100644 Design-Files/DataStack — Settings-pages.html create mode 100644 Design-Files/DataStack-Dashboard-Pages.html create mode 100644 Design-Files/Landing-Pages.html create mode 100644 Design-Files/Login Page — Design.html create mode 100644 app/(landing)/page.tsx create mode 100644 app/api/auth/login/route.ts create mode 100644 app/api/data/route.ts delete mode 100644 app/page.tsx create mode 100644 app/vcon/login/page.tsx create mode 100644 components/layout/AdminPageHeader.tsx create mode 100644 components/sections/AdminLogin.tsx create mode 100644 components/sections/ClientsFilters.tsx create mode 100644 components/sections/ClientsManager.tsx create mode 100644 components/sections/ClientsTable.tsx create mode 100644 components/sections/DashboardOverview.tsx create mode 100644 components/sections/DataBrowser.tsx create mode 100644 components/sections/ExportWorkspace.tsx create mode 100644 components/sections/LandingAbout.tsx create mode 100644 components/sections/NewClientModal.tsx create mode 100644 components/sections/SdkSection.tsx create mode 100644 components/sections/SettingsWorkspace.tsx create mode 100644 components/sections/clients/types.ts create mode 100644 components/sections/index.ts create mode 100644 components/ui/ActivityPreview.tsx create mode 100644 components/ui/ClientList.tsx create mode 100644 components/ui/CodeBlock.tsx create mode 100644 components/ui/CopyButton.tsx create mode 100644 components/ui/FeatureItem.tsx create mode 100644 components/ui/Panel.tsx create mode 100644 components/ui/Pill.tsx create mode 100644 components/ui/SectionHeading.tsx create mode 100644 components/ui/SettingsLayout.tsx create mode 100644 components/ui/StatCard.tsx create mode 100644 components/ui/StatStrip.tsx create mode 100644 core/auth/adminSession.ts create mode 100644 docker/Dockerfile.server create mode 100644 docker/Dockerfile.web create mode 100644 middleware.ts create mode 100644 package-lock.json diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..991f4d7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +.git +.next +node_modules +coverage +notes +**/notes +storage/data +.env* +.mcp.json +.vscode +Design-Files +*.log +tsconfig.tsbuildinfo diff --git a/.env.example b/.env.example index 7a550fe..b4a55ab 100644 --- a/.env.example +++ b/.env.example @@ -7,3 +7,15 @@ GEMINI_API_KEY="MY_GEMINI_API_KEY" # AI Studio automatically injects this at runtime with the Cloud Run service URL. # Used for self-referential links, OAuth callbacks, and API endpoints. APP_URL="MY_APP_URL" + +# Production Compose requires unique values for these variables. Copy this file to `.env` +# and replace every blank value. Never commit `.env` or use generated placeholders in production. +DATASTACK_JWT_SECRET= +# Use the Compose service host `postgres`, database `datastack_db`, and user `datastack`. +DATABASE_URL= +POSTGRES_PASSWORD= + +# Required to protect the admin console. Use a long, unique session secret (32+ characters). +DATASTACK_ADMIN_EMAIL= +DATASTACK_ADMIN_PASSWORD= +DATASTACK_ADMIN_SESSION_SECRET= diff --git a/.gitignore b/.gitignore index 71b810c..17cc55b 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ coverage/ *.log .env* !.env.example +notes/ diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..301c37d --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,5 @@ +{ + "recommendations": [ + "gitlab.gitlab-workflow" + ] +} diff --git a/AGENTS.md b/AGENTS.md index 125f1a0..4e50370 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,16 +1,19 @@ # AGENTS.md — Component-Only Architecture Rules & System Directives ## 1. Single Source of Truth Components + - All UI design styles are designed once as reusable components in `components/ui/` and `components/layout/`. - Pages must never implement isolated, inline styles or custom ad-hoc styling blocks. - Spacing, colors, fonts, borders, buttons, cards, modals, and tables are defined centrally. Any design token alteration automatically cascades across all pages. ## 2. Decoupled Runtime Split + - **Port 3000**: Next.js App Router (Frontend UI, `/vcon/*` admin console, streaming CSV export `/api/export`). - **Port 4000**: Express.js + Socket.IO process (WebSocket handshake, JWT verification, real-time ingestion). - **Socket.IO Rule**: Socket.IO must never reside inside Next.js serverless route handlers. ## 3. Data Storage Rules + - **Collected Realtime Data**: Strictly stored in rotated CSV files at `/storage/data/{clientId}/{YYYY-MM-DD}.csv`. - **Zero Database Inserts**: Per-row DB inserts are prohibited for collected data streams. - **File Rotation**: Rotates daily or upon reaching 100MB (`-part2.csv`). @@ -18,6 +21,13 @@ - **PostgreSQL / Prisma**: Exclusively dedicated to metadata, SDK registrations, and admin settings. ## 4. Design & Typography Tokens + - **Font Stack**: `"Segoe UI", "Segoe UI Web", system-ui, -apple-system, "Helvetica Neue", sans-serif`. - **Border Radius**: `--radius: 0rem` strictly across all shadcn and custom UI components. - **Shadows**: None. Clean, soft, compact flat design with subtle 1px borders. + +## 5. Private Development Notes + +- The root `notes/` directory is local-only and must remain ignored by Git. +- `notes/` is exclusively for private development-time notes, plans, reports, and update/fix phase logs. Never put public-facing or publishable documentation, Markdown, or public information in this directory. +- Before starting any repository update, fix, or addition, record its report and plan in `notes/` first. Keep the relevant steps and phase status/logs updated there as work progresses. diff --git a/Design-Files/DataStack -Data-Pages.html b/Design-Files/DataStack -Data-Pages.html new file mode 100644 index 0000000..2745c91 --- /dev/null +++ b/Design-Files/DataStack -Data-Pages.html @@ -0,0 +1,483 @@ + + + + + +DataStack — Data + + + + + + + + + + + + + + + + + + + + + + + +
+ + + +
+ +
+
+
+ + +
+
+ Live · 1,284 ev/min + + +
+
+ +
+ +
+
+

Data

+
Collected events across all clients
+
+ +
+ + +
+
+
+
Client
+ +
+
+
From
+ +
+
+
To
+ +
+ +
+ + +
+
+
+ + +
+
+
Events
+
42,891 results · page 1 of 429
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
TimeClientEvent IDPayloadSizeStatus
14:28:01py-client-01evt_8f3a…c21{ sensor: "temp", value: 22.4 }42 Bok
14:28:01py-client-02evt_9b1e…44a{ user: "u_8291", action: "click" }38 Bok
14:28:02py-client-01evt_2c7f…88e{ sensor: "humidity", value: 61 }40 Bok
14:28:02py-client-03evt_5d0a…1fb{ order_id: "A-2041", status: "shipped" }44 Bok
14:28:03py-client-02evt_7e2d…09c{ user: "u_1840", action: "view" }36 Bretry
14:28:03py-client-01evt_1a9b…72d{ sensor: "temp", value: 22.6 }42 Bok
14:28:04py-client-04evt_3c4e…b81{ device: "d_01", battery: 87 }40 Bok
14:28:04py-client-02evt_6f7a…d94{ user: "u_9903", action: "purchase" }42 Bok
14:28:05py-client-03evt_0b2c…55f{ order_id: "A-2042", status: "pending" }46 Bok
14:28:05py-client-04evt_8d1c…2a3{ device: "d_02", battery: 92 }40 Bfailed
14:28:06py-client-01evt_4e9f…7d0{ sensor: "temp", value: 22.7 }42 Bok
14:28:06py-client-02evt_5c3b…11e{ user: "u_3312", action: "signup" }40 Bok
+ + +
+ +
Data is served from compact CSV files · live updates stream in over WebSocket
+ +
+
+
+ + + + \ No newline at end of file diff --git a/Design-Files/DataStack — Clients-Pages.html b/Design-Files/DataStack — Clients-Pages.html new file mode 100644 index 0000000..91847e7 --- /dev/null +++ b/Design-Files/DataStack — Clients-Pages.html @@ -0,0 +1,626 @@ + + + + + +DataStack — Clients + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + +
+ +
+
+
+ + +
+
+ Live · 8 online + + +
+
+ +
+ +
+
+

Clients

+
Registered SDK clients and their API access
+
+
+ +
+
+ + +
+
+
Total clients
+
12
+
+
+
Online now
+
8
+
+
+
Offline
+
3
+
+
+
Pending setup
+
1
+
+
+ + +
+ +
+ + + + +
+
+ + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ClientAPI keyEvents todayStorageLast seen
+
+ +
+
py-client-01
+
sensors-app · Python
+
+
+
+ sk_live_••••8f3a + + 18,4204.2 MB14:28:01 2s ago +
+ + + +
+
+
+ +
+
py-client-02
+
web-analytics · Python
+
+
+
+ sk_live_••••3b71 + + 12,1822.8 MB14:27:58 4s ago +
+ + + +
+
+
+ +
+
py-client-03
+
orders-sync · Python
+
+
+
+ sk_live_••••e0c9 + + 8,2041.9 MB14:28:00 1s ago +
+ + + +
+
+
+ +
+
py-client-04
+
devices-iote · Python
+
+
+
+ sk_live_••••7d42 + + 2,1440.5 MB14:27:52 8s ago +
+ + + +
+
+
+ +
+
py-client-06
+
mobile-backend · Python
+
+
+
+ sk_live_••••1a09 + + 1,2040.3 MB14:27:48 12s ago +
+ + + +
+
+
+ +
+
py-client-05
+
staging-test · Python
+
+
+
+ sk_test_••••f5b2 + + 3400.1 MB12:24:11 2h ago +
+ + + +
+
+
+ +
+
py-client-07
+
cron-jobs · Python
+
+
+
+ sk_test_••••9c14 + + 8212 KB08:12:04 6h ago +
+ + + +
+
+
+ +
+
py-client-08
+
new-proto · Python
+
+
+
+ sk_live_••••4e88 + + 0—— never +
+ + + +
+
+
+ +
Rotating a key invalidates the current one instantly · Revoking disconnects all active sessions
+ +
+
+
+ + + + + + + \ No newline at end of file diff --git a/Design-Files/DataStack — Export-page.html b/Design-Files/DataStack — Export-page.html new file mode 100644 index 0000000..ede14c6 --- /dev/null +++ b/Design-Files/DataStack — Export-page.html @@ -0,0 +1,538 @@ + + + + + +DataStack — Export + + + + + + + + + + + + + + + + + + + + + + +
+ + + +
+ +
+
+
+ + +
+
+ Estimate live + + +
+
+ +
+ +
+
+

Export

+
Generate CSV or JSON exports from collected data
+
+
+ + +
+
+
Exports today
+
24
+
+
+
Last export
+
2hago
+
+
+
Total exported
+
1.2GB
+
+
+
Retention
+
30days
+
+
+ +
+ + +
+
+
+ + Export builder +
+ Estimate updates on change +
+ +
+ + +
+ Quick range + + + + +
+ + +
+
+
Client
+ +
+
+
From
+ +
+
+
To
+ +
+
+ + + +
+ + +
+ + + +
+ + + +
+ + +
+ + 42,891 rows · ~18.2 MB · 3 CSV files +
+ +
+ +
+ + + +
+
+ + +
+
+
Recent exports
+ +
+
+
+ CSV +
+
oct23_sensors_01.csv
+
2h ago · 4,120 rows · 1.2 MB
+
+ +
+
+ JSON +
+
oct23_web_analytics.json
+
3h ago · 8,204 rows · 2.4 MB
+
+ +
+
+ CSV +
+
oct22_all_clients.csv
+
18h ago · 22,140 rows · 9.1 MB
+
+ +
+
+ CSV +
+
oct21_orders_sync.csv
+
1d ago · 6,022 rows · 2.1 MB
+
+ +
+
+ CSV +
+
oct20_failed_only.csv
+
2d ago · 84 rows · 12 KB
+
+ +
+
+ JSON +
+
oct19_devices_snapshot.json
+
3d ago · 1,204 rows · 340 KB
+
+ +
+
+
+ +
+ +
Exports are generated from CSV files on disk · Estimates are approximations · Files over 100 MB download via streaming
+ +
+
+
+ + + + \ No newline at end of file diff --git a/Design-Files/DataStack — Settings-pages.html b/Design-Files/DataStack — Settings-pages.html new file mode 100644 index 0000000..f3499c8 --- /dev/null +++ b/Design-Files/DataStack — Settings-pages.html @@ -0,0 +1,714 @@ + + + + + +DataStack — Settings + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + +
+ +
+
+
+ + +
+
+ + +
+
+ +
+ +
+
+

Settings

+
Configure your DataStack instance
+
+
+ + +
+ + + + + +
+ + +
+ + +
+
+
Identity
+
+
+
Instance name
+
Shown in dashboard headers and emails
+
+
+ +
+
+
+
+
Base URL
+
Used in SDK setup and export links
+
+
+ +
+
+
+
+
Instance ID
+
Immutable · used in SDK fingerprint
+
+
+
+ + +
+
+
+
+ +
+
Locale
+
+
+
Timezone
+
Applied to dashboard and exports
+
+
+ +
+
+
+
+
Date format
+
+
+ +
+
+
+
+
Language
+
+
+ +
+
+
+
+ + +
+
+
WebSocket server
+
+
+
Public WebSocket URL
+
Clients connect here (behind Nginx)
+
+
+ +
+
+
+
+
Socket port
+
Internal Express process port
+
+
+
+ + TCP +
+
+
+
+
+
Heartbeat interval
+
Disconnect clients that miss 2 beats
+
+
+
+ + sec +
+
+
+
+ +
+
Redis
+
+
+
Redis URL
+
+
+ +
+
+
+
+
Key prefix
+
Prevents collisions on shared Redis
+
+
+ +
+
+
+
+
Buffer flush
+
Trigger CSV write on rows or timer
+
+
+
+ + +
+
+
+
+
+ + +
+
+
CSV files
+
+
+
Root path
+
Files written at /{clientId}/{date}.csv
+
+
+ +
+
+
+
+
File rotation
+
+
+ +
+
+
+
+
Max file size
+
Rotate file when threshold is hit
+
+
+
+ + MB +
+
+
+
+
+
Retention
+
Older files auto-deleted by worker
+
+
+
+ + days +
+
+
+
+
+
Compress completed files
+
Gzip closed CSV files (saves ~70%)
+
+
+ +
+
+
+ +
+
Schema
+
+
+
Row format
+
Fixed for version 1.0
+
+
+ +
+
+
+
+ + +
+
+
Authentication
+
+
+
JWT secret
+
Rotating invalidates all live sessions
+
+
+
+ + +
+
+
+
+
+
JWT TTL
+
Clients re-handshake after expiry
+
+
+
+ + hours +
+
+
+
+ +
+
Rate limiting
+
+
+
Events per second
+
Per-client cap · excess dropped silently
+
+
+
+ + ev/s +
+
+
+
+
+
Max payload size
+
+
+
+ + KB +
+
+
+
+
+
Allowed origins
+
Comma-separated · * allows all
+
+
+ +
+
+
+ +
+
Danger zone
+
+
+
Delete all collected data
+
Permanently removes every CSV file and Redis buffer. Admin config is preserved.
+
+ +
+
+
+ + +
+
+
Delivery
+
+
+
Alert email
+
Where critical alerts are sent
+
+
+ +
+
+
+
+
Webhook URL
+
POST JSON on every event below
+
+
+ +
+
+
+ +
+
Events to notify
+
+
+
Client events
+
+
+ + + +
+
+
+
+
System events
+
+
+ + + + +
+
+
+
+ + +
+
+ + All changes saved +
+ + +
+ +
+ +
+ + Some settings require a service restart to take effect — restart via systemctl or Docker +
+ +
+
+
+ + + + \ No newline at end of file diff --git a/Design-Files/DataStack-Dashboard-Pages.html b/Design-Files/DataStack-Dashboard-Pages.html new file mode 100644 index 0000000..a65f03a --- /dev/null +++ b/Design-Files/DataStack-Dashboard-Pages.html @@ -0,0 +1,546 @@ + + + + + +DataStack — Console + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ + +
+ + +
+
+ + +
+
+ Live · 1,284 ev/min + + +
+
+ + +
+ +
+
+

Dashboard

+
Last 24 hours · UTC
+
+
+ + +
+
+ + +
+
+
Events today
+
42,891
+
+12.4% vs yesterday
+
+
+
Clients online
+
8/ 12
+
4 offline · 2 pending
+
+
+
Storage used
+
18.2MB
+
3 CSV files · today
+
+
+
Last export
+
2hago
+
2026-10-23 · CSV · 4,120 rows
+
+
+ + +
+ + +
+
+
Live ingestion
+
+ 1,284 ev/min + View all → +
+
+
+
14:28:01py-client-01{ sensor: "temp", value: 22.4 }42B
+
14:28:01py-client-02{ user: "u_8291", action: "click" }38B
+
14:28:02py-client-01{ sensor: "humidity", value: 61 }40B
+
14:28:02py-client-03{ order_id: "A-2041", status: "shipped" }44B
+
14:28:03py-client-02{ user: "u_1840", action: "view" }36B
+
14:28:03py-client-01{ sensor: "temp", value: 22.6 }42B
+
14:28:04py-client-04{ device: "d_01", battery: 87 }40B
+
14:28:04py-client-02{ user: "u_9903", action: "purchase" }42B
+
+
+ + +
+ + +
+
+
Active clients
+ All → +
+
+
+
py-client-01
sensors-app
+
18.4k
+
+
+
py-client-02
web-analytics
+
12.1k
+
+
+
py-client-03
orders-sync
+
8.2k
+
+
+
py-client-05
staging-test
+
offline
+
+
+
+ + +
+
+
Storage
+ 34% of 50 GB +
+
+
+
18.2 GB used31.8 GB free
+
+
+ +
+
+ +
+
+
+ + + + \ No newline at end of file diff --git a/Design-Files/Landing-Pages.html b/Design-Files/Landing-Pages.html new file mode 100644 index 0000000..d16f3b0 --- /dev/null +++ b/Design-Files/Landing-Pages.html @@ -0,0 +1,290 @@ + + + + + +DataStack — Self-hosted realtime data collection + + + + + + +
+
+ Self-hosted · v1.0 +

Realtime data collection,
without the database overhead.

+

Stream events from any client over WebSocket. Store them compactly as CSV. Query, monitor and export — all from one lightweight portal.

+ + +
+
+
Live ingestion stream
+ 1,284 events / min +
+
+
14:28:01py-client-01{ sensor: "temp", value: 22.4 }42 B
+
14:28:01py-client-02{ user: "u_8291", action: "click" }38 B
+
14:28:02py-client-01{ sensor: "humidity", value: 61 }40 B
+
14:28:02py-client-03{ order_id: "A-2041", status: "shipped" }44 B
+
+
+
+
+ +
+
+
+

Built for ingestion at scale.

+ 01 — Features +
+
+
⚡

Realtime over WebSocket

Durable Socket.IO connections. No polling, no repeated API calls.

+
▤

CSV-first storage

Compact per-day rotation. Minimal database load, easy to export.

+
⚿

JWT authentication

Every SDK client verified before a single byte is accepted.

+
◈

Redis buffer & cache

Batch writes to disk. Pub/Sub ready for horizontal scale-out.

+
◐

Live dashboard

Watch events arrive in real time. Filter by client, date or payload.

+
⇩

One-click export

Download filtered ranges as CSV or JSON. Streaming, not blocking.

+
+
+
+ +
+
+
+

Drop in the SDK. Start streaming.

+ 02 — Python SDK +
+
+
# pip install datastack-sdk +from datastack import Client + +client = Client( + url="wss://your-host:4000", + api_key="sk_live_..." +) + +client.connect() + +while running: + client.send({ + "sensor": "temp", + "value": 22.4 + })
+
+
How it works
+

A single line to send data.

+

The SDK holds a durable WebSocket connection, auto-reconnects on network drops, and batches events under backpressure. The server verifies your JWT, buffers events in Redis, and appends them to a per-client CSV file.

+
+
+
+
+ +
+
+
+

Ready to self-host?

+

Deploy with Docker in minutes. Full source, no vendor lock-in.

+
+ +
+
+ + + + + \ No newline at end of file diff --git a/Design-Files/Login Page — Design.html b/Design-Files/Login Page — Design.html new file mode 100644 index 0000000..c2e869e --- /dev/null +++ b/Design-Files/Login Page — Design.html @@ -0,0 +1,293 @@ + + + + + +DataStack — Sign in + + + + + + + + + + + + + + + + +
+
+ + +
+ + + DataStack + CONSOLE + +
+
Sign in
+
Access your admin console
+
+
+ + +
+ + +
+ + Invalid email or password. Try again or reset your credentials. +
+ + +
+ +
+ +
+
+ + +
+ +
+ + +
+
+ + +
+ + Forgot password? +
+ + + + + +
or
+ + + + +
+ + +
+ Server online + Docs → +
+ +
+
+ +
+ DataStack v1.0.0 + · + Self-hosted + · + GitHub +
+ + + + \ No newline at end of file diff --git a/README.md b/README.md index 324c0a0..91f2097 100644 --- a/README.md +++ b/README.md @@ -4,25 +4,29 @@ DataStack is a high-throughput, self-hosted data collection platform designed fo ## Architecture -| Process | Port | Role | -| :--- | :--- | :--- | -| **Next.js App Router** | 3000 | Frontend UI, `/vcon/*` admin console, streaming CSV export `/api/export` | -| **Express + Socket.IO** | 4000 | WebSocket handshake, JWT client auth, event intake | -| **CSV Writer Worker** | — | In-memory buffer flush to CSV (`500 rows / 2s`) | +| Process | Port | Role | +| :---------------------- | :--- | :----------------------------------------------------------------------- | +| **Next.js App Router** | 3000 | Frontend UI, `/vcon/*` admin console, streaming CSV export `/api/export` | +| **Express + Socket.IO** | 4000 | WebSocket handshake, JWT client auth, event intake | +| **CSV Writer Worker** | — | In-memory buffer flush to CSV (`500 rows / 2s`) | ## Data Persistence + - **CSV Storage Path**: `/storage/data/{clientId}/{YYYY-MM-DD}.csv` - **Schema**: `clientId, eventId, timestamp(ISO), payload(JSON)` - **Rotation Rule**: Per-day OR 100MB threshold (whichever triggers first). -- **PostgreSQL**: Used strictly for SDK client registration and admin settings. Zero collected data is inserted into the database. +- **PostgreSQL**: Reserved for SDK client registration and admin settings. Runtime persistence is not wired yet; collected data is never inserted into PostgreSQL. ## Python SDK + Install the client: + ```bash pip install ./sdk-python ``` Usage: + ```python from datastack_sdk import DataStackClient @@ -33,6 +37,7 @@ if client.connect(): ``` ## Running the Platform + ```bash # Terminal 1: Web Interface & Admin Console npm run dev @@ -40,3 +45,20 @@ npm run dev # Terminal 2: Express Socket.IO Ingestion Engine npm run server ``` + +## Production Containers + +Docker Compose builds the standalone Next.js web image and the separate Socket.IO ingestion image. Docker Engine with the Compose plugin is required. + +1. Copy `.env.example` to `.env` and set unique values for `DATASTACK_JWT_SECRET`, `DATASTACK_ADMIN_SESSION_SECRET`, `DATASTACK_ADMIN_EMAIL`, `DATASTACK_ADMIN_PASSWORD`, `DATABASE_URL`, and `POSTGRES_PASSWORD`. Keep `.env` private and out of source control. + `DATABASE_URL` must use the `postgres` service host and a password matching `POSTGRES_PASSWORD`. +2. From the repository root, validate and start the stack: + +```powershell +docker compose --env-file .env -f docker/docker-compose.yml config +docker compose --env-file .env -f docker/docker-compose.yml up --build -d +``` + +The web console is published on port 3000 and the ingestion service on port 4000. Redis and PostgreSQL are private to the Compose network. CSV files written by the ingestion container persist in the `datastack-data` named volume; PostgreSQL metadata persists in `pgdata`. + +The current web process still has a separate in-memory buffer and container-local storage view. Redis and Prisma/PostgreSQL are not yet used by application code, and the web process does not yet read the ingestion volume. Cross-process event visibility and shared storage are Phase P4 work; metadata persistence is Phase P5 work. Do not treat this Compose stack as production-ready until those phases and runtime checks are complete. diff --git a/app/(landing)/page.tsx b/app/(landing)/page.tsx new file mode 100644 index 0000000..842f448 --- /dev/null +++ b/app/(landing)/page.tsx @@ -0,0 +1,16 @@ +import { MainLayout } from '@/components/layout/MainLayout'; +import { Features } from '@/components/sections/Features'; +import { Hero } from '@/components/sections/Hero'; +import { LandingAbout } from '@/components/sections/LandingAbout'; +import { SdkSection } from '@/components/sections/SdkSection'; + +export default function LandingPage() { + return ( + + + + + + + ); +} \ No newline at end of file diff --git a/app/api/auth/login/route.ts b/app/api/auth/login/route.ts new file mode 100644 index 0000000..f83f2cf --- /dev/null +++ b/app/api/auth/login/route.ts @@ -0,0 +1,76 @@ +import { createAdminSessionToken } from "@/core/auth/adminSession"; +import { NextRequest, NextResponse } from "next/server"; +import { createHash, timingSafeEqual } from "node:crypto"; + +const SESSION_COOKIE = "datastack_admin_session"; +const SESSION_SECONDS = 8 * 60 * 60; +const REMEMBERED_SESSION_SECONDS = 30 * 24 * 60 * 60; + +function constantTimeEqual(left: string, right: string) { + const leftHash = createHash("sha256").update(left).digest(); + const rightHash = createHash("sha256").update(right).digest(); + return timingSafeEqual(leftHash, rightHash); +} + +export async function POST(request: NextRequest) { + const adminEmail = process.env.DATASTACK_ADMIN_EMAIL; + const adminPassword = process.env.DATASTACK_ADMIN_PASSWORD; + const sessionSecret = process.env.DATASTACK_ADMIN_SESSION_SECRET; + + if ( + !adminEmail || + !adminPassword || + !sessionSecret || + sessionSecret.length < 32 + ) { + return NextResponse.json( + { + error: + "Admin login is not configured. Set the required server environment variables.", + }, + { status: 503 }, + ); + } + + try { + const body = await request.json(); + const email = + typeof body.email === "string" ? body.email.trim().toLowerCase() : ""; + const password = typeof body.password === "string" ? body.password : ""; + + if ( + !constantTimeEqual(email, adminEmail.trim().toLowerCase()) || + !constantTimeEqual(password, adminPassword) + ) { + return NextResponse.json( + { error: "Invalid email or password." }, + { status: 401 }, + ); + } + + const remember = body.remember === true; + const maxAge = remember ? REMEMBERED_SESSION_SECONDS : SESSION_SECONDS; + const expiresAt = Date.now() + maxAge * 1000; + const token = await createAdminSessionToken( + email, + sessionSecret, + expiresAt, + ); + const response = NextResponse.json({ success: true }); + + response.cookies.set(SESSION_COOKIE, token, { + httpOnly: true, + secure: process.env.NODE_ENV === "production", + sameSite: "strict", + path: "/", + maxAge, + }); + + return response; + } catch { + return NextResponse.json( + { error: "Invalid login request." }, + { status: 400 }, + ); + } +} diff --git a/app/api/clients/[id]/route.ts b/app/api/clients/[id]/route.ts index 04363b5..94d3ece 100644 --- a/app/api/clients/[id]/route.ts +++ b/app/api/clients/[id]/route.ts @@ -1,54 +1,58 @@ -import { NextRequest, NextResponse } from 'next/server'; -import { SDKManager } from '@/core/sdk/sdkManager'; +import { SDKManager } from "@/core/sdk/sdkManager"; +import { NextRequest, NextResponse } from "next/server"; export async function GET( _req: NextRequest, - { params }: { params: Promise<{ id: string }> } + { params }: { params: Promise<{ id: string }> }, ) { const { id } = await params; const client = SDKManager.getClient(id); if (!client) { - return NextResponse.json({ error: 'Client not found' }, { status: 404 }); + return NextResponse.json({ error: "Client not found" }, { status: 404 }); } - return NextResponse.json({ client }); + const { secret, token, ...clientSummary } = client; + return NextResponse.json({ client: clientSummary }); } export async function PATCH( req: NextRequest, - { params }: { params: Promise<{ id: string }> } + { params }: { params: Promise<{ id: string }> }, ) { const { id } = await params; try { const body = await req.json(); // Special action: regenerate token - if (body.action === 'regenerate_token') { + if (body.action === "regenerate_token") { const refreshed = SDKManager.regenerateToken(id); if (!refreshed) { - return NextResponse.json({ error: 'Client not found' }, { status: 404 }); + return NextResponse.json( + { error: "Client not found" }, + { status: 404 }, + ); } return NextResponse.json({ client: refreshed }); } const updated = SDKManager.updateClient(id, body); if (!updated) { - return NextResponse.json({ error: 'Client not found' }, { status: 404 }); + return NextResponse.json({ error: "Client not found" }, { status: 404 }); } return NextResponse.json({ client: updated }); } catch (err: unknown) { - const message = err instanceof Error ? err.message : 'Update failed'; + const message = err instanceof Error ? err.message : "Update failed"; return NextResponse.json({ error: message }, { status: 500 }); } } export async function DELETE( _req: NextRequest, - { params }: { params: Promise<{ id: string }> } + { params }: { params: Promise<{ id: string }> }, ) { const { id } = await params; const deleted = SDKManager.deleteClient(id); if (!deleted) { - return NextResponse.json({ error: 'Client not found' }, { status: 404 }); + return NextResponse.json({ error: "Client not found" }, { status: 404 }); } return NextResponse.json({ success: true, message: `Client ${id} deleted` }); } diff --git a/app/api/clients/route.ts b/app/api/clients/route.ts index 1c19af8..6b51cbb 100644 --- a/app/api/clients/route.ts +++ b/app/api/clients/route.ts @@ -1,12 +1,15 @@ -import { NextRequest, NextResponse } from 'next/server'; -import { SDKManager } from '@/core/sdk/sdkManager'; +import { SDKManager } from "@/core/sdk/sdkManager"; +import { NextRequest, NextResponse } from "next/server"; export async function GET() { try { - const clients = SDKManager.listClients(); + const clients = SDKManager.listClients().map( + ({ secret, token, ...client }) => client, + ); return NextResponse.json({ clients, count: clients.length }); } catch (err: unknown) { - const message = err instanceof Error ? err.message : 'Failed to list clients'; + const message = + err instanceof Error ? err.message : "Failed to list clients"; return NextResponse.json({ error: message }, { status: 500 }); } } @@ -18,8 +21,8 @@ export async function POST(req: NextRequest) { if (!clientId || !name) { return NextResponse.json( - { error: 'clientId and name are required' }, - { status: 400 } + { error: "clientId and name are required" }, + { status: 400 }, ); } @@ -27,7 +30,7 @@ export async function POST(req: NextRequest) { if (existing) { return NextResponse.json( { error: `Client '${clientId}' already exists` }, - { status: 409 } + { status: 409 }, ); } @@ -39,7 +42,8 @@ export async function POST(req: NextRequest) { return NextResponse.json({ client: newClient }, { status: 201 }); } catch (err: unknown) { - const message = err instanceof Error ? err.message : 'Client creation failed'; + const message = + err instanceof Error ? err.message : "Client creation failed"; return NextResponse.json({ error: message }, { status: 500 }); } } diff --git a/app/api/data/route.ts b/app/api/data/route.ts new file mode 100644 index 0000000..7a59b9f --- /dev/null +++ b/app/api/data/route.ts @@ -0,0 +1,128 @@ +import { DataBufferEngine } from "@/core/data/buffer"; +import { CSVStorageManager } from "@/core/Storage/csvStorage"; +import { parse } from "csv-parse"; +import { NextRequest, NextResponse } from "next/server"; +import { createReadStream, readdirSync, statSync } from "node:fs"; +import path from "node:path"; + +interface DataEvent { + clientId: string; + eventId: string; + timestamp: string; + payload: string; + status: "stored" | "buffered"; +} + +export async function GET(request: NextRequest) { + try { + const { searchParams } = new URL(request.url); + const clientId = searchParams.get("clientId") || "all"; + const from = searchParams.get("from") || ""; + const to = searchParams.get("to") || ""; + const search = (searchParams.get("search") || "").toLowerCase(); + const page = Math.max(1, Number(searchParams.get("page")) || 1); + const pageSize = Math.min( + 100, + Math.max(1, Number(searchParams.get("pageSize")) || 12), + ); + const offset = (page - 1) * pageSize; + const selectedRows: DataEvent[] = []; + const eventIds = new Set(); + let total = 0; + + const addIfMatching = (event: DataEvent) => { + const eventDate = event.timestamp.slice(0, 10); + if ( + (clientId !== "all" && event.clientId !== clientId) || + (from && eventDate < from) || + (to && eventDate > to) || + (search && !event.payload.toLowerCase().includes(search)) || + eventIds.has(event.eventId) + ) { + return; + } + + eventIds.add(event.eventId); + total += 1; + + let low = 0; + let high = selectedRows.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (selectedRows[middle].timestamp >= event.timestamp) low = middle + 1; + else high = middle; + } + + if (low < offset + pageSize) { + selectedRows.splice(low, 0, event); + if (selectedRows.length > offset + pageSize) selectedRows.pop(); + } + }; + + const storageDir = CSVStorageManager.baseStorageDir; + let clientDirs: string[] = []; + try { + clientDirs = readdirSync(storageDir); + } catch { + clientDirs = []; + } + + for (const storedClientId of clientDirs) { + if (clientId !== "all" && storedClientId !== clientId) continue; + + const clientDir = path.join(storageDir, storedClientId); + if (!statSync(clientDir).isDirectory()) continue; + + let filenames: string[]; + try { + filenames = readdirSync(clientDir).filter((filename) => + /^\d{4}-\d{2}-\d{2}(?:-part\d+)?\.csv$/.test(filename), + ); + } catch { + continue; + } + + for (const filename of filenames) { + const fileDate = filename.slice(0, 10); + if ((from && fileDate < from) || (to && fileDate > to)) continue; + + const parser = createReadStream(path.join(clientDir, filename)).pipe( + parse({ columns: true, skip_empty_lines: true, bom: true }), + ); + + try { + for await (const record of parser as AsyncIterable< + Record + >) { + const event = { + clientId: record.clientId || storedClientId, + eventId: record.eventId || "", + timestamp: record.timestamp || `${fileDate}T00:00:00.000Z`, + payload: record.payload || "", + status: "stored" as const, + }; + if (event.eventId) addIfMatching(event); + } + } catch { + parser.destroy(); + } + } + } + + for (const event of DataBufferEngine.getRecentEvents(200)) { + addIfMatching({ ...event, status: "buffered" }); + } + + return NextResponse.json({ + events: selectedRows.slice(offset), + total, + page, + pageSize, + pageCount: Math.max(1, Math.ceil(total / pageSize)), + }); + } catch (error) { + const message = + error instanceof Error ? error.message : "Data retrieval failed"; + return NextResponse.json({ error: message }, { status: 500 }); + } +} diff --git a/app/api/export/route.ts b/app/api/export/route.ts index 63fa57f..44e9b08 100644 --- a/app/api/export/route.ts +++ b/app/api/export/route.ts @@ -1,76 +1,162 @@ -import { NextRequest, NextResponse } from 'next/server'; -import { CSVStorageManager } from '@/core/Storage/csvStorage'; -import fs from 'fs'; -import path from 'path'; +import { + CSVStorageManager, + type StorageFileMeta, +} from "@/core/Storage/csvStorage"; +import { parse } from "csv-parse"; +import { NextRequest, NextResponse } from "next/server"; +import { createReadStream } from "node:fs"; -export async function GET(req: NextRequest) { - try { - const { searchParams } = new URL(req.url); - const clientId = searchParams.get('clientId'); - const filename = searchParams.get('filename'); +type ExportFormat = "csv" | "json"; +type CsvRecord = Record; +type StreamChunk = string | Uint8Array; - // Single specific file export - if (clientId && filename) { - const content = CSVStorageManager.getFileContent(clientId, filename); - if (!content) { - return NextResponse.json({ error: 'CSV file not found' }, { status: 404 }); - } +function toByteStream(chunks: AsyncIterable) { + const iterator = chunks[Symbol.asyncIterator](); + const encoder = new TextEncoder(); - return new NextResponse(content, { - status: 200, - headers: { - 'Content-Type': 'text/csv; charset=utf-8', - 'Content-Disposition': `attachment; filename="${filename}"`, - }, - }); - } - - // Stream aggregate or list files - const allFiles = CSVStorageManager.listAllFiles(); - const filtered = clientId && clientId !== 'all' - ? allFiles.filter((f) => f.clientId === clientId) - : allFiles; - - if (filtered.length === 0) { - const emptyCsv = 'clientId,eventId,timestamp,payload\n'; - return new NextResponse(emptyCsv, { - status: 200, - headers: { - 'Content-Type': 'text/csv; charset=utf-8', - 'Content-Disposition': `attachment; filename="datastack_export_${Date.now()}.csv"`, - }, - }); - } - - // Combine matching files with a single header - let combined = 'clientId,eventId,timestamp,payload\n'; - for (const f of filtered) { + return new ReadableStream({ + async pull(controller) { try { - const raw = fs.readFileSync(f.filePath, 'utf8'); - const lines = raw.split('\n'); - // skip header on subsequent lines - const dataLines = lines.slice(1).filter((l) => l.trim().length > 0); - if (dataLines.length > 0) { - combined += dataLines.join('\n') + '\n'; + const result = await iterator.next(); + if (result.done) { + controller.close(); + } else { + controller.enqueue( + typeof result.value === "string" + ? encoder.encode(result.value) + : result.value, + ); } - } catch { - // continue + } catch (error) { + controller.error(error); + } + }, + async cancel() { + await iterator.return?.(); + }, + }); +} + +async function* readFileRecords( + file: StorageFileMeta, +): AsyncGenerator { + const parser = createReadStream(file.filePath).pipe( + parse({ columns: true, skip_empty_lines: true, bom: true }), + ); + + try { + for await (const record of parser as AsyncIterable) { + yield record; + } + } catch { + parser.destroy(); + } +} + +function makeExportStream( + files: StorageFileMeta[], + format: ExportFormat, + includeHeader: boolean, +) { + const fields = ["clientId", "eventId", "timestamp", "payload"]; + const chunks = (async function* () { + let isFirst = true; + + if (format === "json") { + yield "["; + } else if (includeHeader) { + yield `${fields.join(",")}\n`; + } + + for (const file of files) { + for await (const record of readFileRecords(file)) { + if (format === "json") { + const payload = record.payload ?? ""; + let parsedPayload: unknown = payload; + try { + parsedPayload = JSON.parse(payload); + } catch { + // Keep non-JSON payloads as strings. + } + yield `${isFirst ? "" : ","}${JSON.stringify({ ...record, payload: parsedPayload })}`; + } else { + const line = fields + .map((field) => `"${(record[field] ?? "").replaceAll('"', '""')}"`) + .join(","); + yield `${line}\n`; + } + isFirst = false; } } - const exportFileName = clientId && clientId !== 'all' - ? `${clientId}_export_${new Date().toISOString().split('T')[0]}.csv` - : `datastack_universal_export_${new Date().toISOString().split('T')[0]}.csv`; + if (format === "json") yield "]"; + })(); - return new NextResponse(combined, { + return toByteStream(chunks); +} + +function streamFile(file: StorageFileMeta) { + async function* chunks() { + for await (const chunk of createReadStream(file.filePath)) { + yield new Uint8Array(chunk); + } + } + + return toByteStream(chunks()); +} + +export async function GET(request: NextRequest) { + try { + const { searchParams } = new URL(request.url); + const clientId = searchParams.get("clientId") || "all"; + const filename = searchParams.get("filename"); + const from = searchParams.get("from") || ""; + const to = searchParams.get("to") || ""; + const format: ExportFormat = + searchParams.get("format") === "json" ? "json" : "csv"; + const includeHeader = searchParams.get("includeHeader") !== "false"; + + let files = CSVStorageManager.listAllFiles(); + if (clientId !== "all") + files = files.filter((file) => file.clientId === clientId); + + if (filename) { + const file = files.find((candidate) => candidate.filename === filename); + if (!file) + return NextResponse.json( + { error: "CSV file not found" }, + { status: 404 }, + ); + return new NextResponse(streamFile(file), { + status: 200, + headers: { + "Content-Type": "text/csv; charset=utf-8", + "Content-Disposition": `attachment; filename="${file.filename}"`, + }, + }); + } + + files = files.filter( + (file) => (!from || file.date >= from) && (!to || file.date <= to), + ); + const datePart = new Date().toISOString().slice(0, 10); + const safeClientId = clientId.replace(/[^a-zA-Z0-9_-]/g, "_"); + const baseName = + clientId === "all" ? "datastack_export" : `${safeClientId}_export`; + + return new NextResponse(makeExportStream(files, format, includeHeader), { status: 200, headers: { - 'Content-Type': 'text/csv; charset=utf-8', - 'Content-Disposition': `attachment; filename="${exportFileName}"`, + "Content-Type": + format === "json" + ? "application/json; charset=utf-8" + : "text/csv; charset=utf-8", + "Content-Disposition": `attachment; filename="${baseName}_${datePart}.${format}"`, + "Cache-Control": "no-store", }, }); - } catch (err: unknown) { - const message = err instanceof Error ? err.message : 'Export failed'; + } catch (error) { + const message = error instanceof Error ? error.message : "Export failed"; return NextResponse.json({ error: message }, { status: 500 }); } } diff --git a/app/api/settings/route.ts b/app/api/settings/route.ts index a1c2e05..8c8b8ad 100644 --- a/app/api/settings/route.ts +++ b/app/api/settings/route.ts @@ -1,12 +1,12 @@ -import { NextRequest, NextResponse } from 'next/server'; -import { SettingsManager } from '@/core/adminSettings/settingsManager'; +import { SettingsManager } from "@/core/adminSettings/settingsManager"; +import { NextRequest, NextResponse } from "next/server"; export async function GET() { try { - const settings = SettingsManager.getSettings(); + const settings = SettingsManager.getPublicSettings(); return NextResponse.json({ settings }); } catch (err: unknown) { - const message = err instanceof Error ? err.message : 'Settings read failed'; + const message = err instanceof Error ? err.message : "Settings read failed"; return NextResponse.json({ error: message }, { status: 500 }); } } @@ -14,10 +14,14 @@ export async function GET() { export async function POST(req: NextRequest) { try { const body = await req.json(); - const updated = SettingsManager.updateSettings(body); - return NextResponse.json({ settings: updated, success: true }); + SettingsManager.updateSettings(body); + return NextResponse.json({ + settings: SettingsManager.getPublicSettings(), + success: true, + }); } catch (err: unknown) { - const message = err instanceof Error ? err.message : 'Settings update failed'; + const message = + err instanceof Error ? err.message : "Settings update failed"; return NextResponse.json({ error: message }, { status: 500 }); } } diff --git a/app/globals.css b/app/globals.css index c8c0348..ee9ef7a 100644 --- a/app/globals.css +++ b/app/globals.css @@ -2,73 +2,89 @@ :root { --radius: 0rem; + --font-weight-semibold: 500; + --font-weight-bold: 500; + --font-weight-extrabold: 500; --font-sans: "Segoe UI", "Segoe UI Web", system-ui, -apple-system, BlinkMacSystemFont, "Helvetica Neue", Arial, sans-serif; - --font-mono: ui-monospace, "SF Mono", "Liberation Mono", Menlo, Consolas, monospace; + --font-mono: "Consolas", "Courier New", monospace; + --container-width: 1080px; + --console-width: 1240px; + --surface-3: #1d1d21; + --accent-text: var(--accent); - /* Theme tokens - Dark Mode (Sleek Graphite Workbench) */ - --bg-main: #0c0e12; - --bg-sidebar: #0f1117; - --bg-activity: #131720; - --bg-elevated: #161b24; - --bg-hover: #1c2230; - --bg-active: #242c3d; - --border: #212836; - --border-strong: #2e374a; - --text: #e6ebf2; - --text-muted: #8c96a8; - --text-dim: #5c6577; - --accent: #3b82f6; - --accent-soft: rgba(59, 130, 246, 0.08); - --accent-hover: #60a5fa; - --text-on-accent: #ffffff; - --success: #10b981; - --success-soft: rgba(16, 185, 129, 0.08); - --warning: #f59e0b; - --warning-soft: rgba(245, 158, 11, 0.08); - --danger: #ef4444; - --danger-soft: rgba(239, 68, 68, 0.08); + --bg: #0a0a0b; + --surface: #111113; + --surface-2: #17171a; + --border: #232326; + --border-strong: #2e2e33; + --text: #e8e8ea; + --text-muted: #85858c; + --text-faint: #5a5a62; + --accent: #a3e635; + --accent-soft: rgba(163, 230, 53, 0.08); + --font: var(--font-sans); + --radius: 0rem; + --bg-main: var(--bg); + --bg-sidebar: var(--surface); + --bg-activity: var(--surface); + --bg-elevated: var(--surface); + --bg-hover: var(--surface-2); + --bg-active: var(--surface-2); + --text-dim: var(--text-faint); + --accent-hover: var(--accent); + --text-on-accent: var(--bg); + --success: var(--accent); + --success-soft: var(--accent-soft); + --warning: var(--text-muted); + --warning-soft: transparent; + --danger: var(--text-muted); + --danger-soft: transparent; } html.light, body.light { - /* Theme tokens - Light Mode (Clean, soft contrast) */ - --bg-main: #ffffff; - --bg-sidebar: #f8fafc; - --bg-activity: #f1f5f9; - --bg-elevated: #ffffff; - --bg-hover: #f1f5f9; - --bg-active: #e2e8f0; - --border: #e2e8f0; - --border-strong: #cbd5e1; - --text: #0f172a; - --text-muted: #475569; - --text-dim: #94a3b8; - --accent: #2563eb; - --accent-soft: rgba(37, 99, 235, 0.06); - --accent-hover: #1d4ed8; - --text-on-accent: #ffffff; - --success: #059669; - --success-soft: rgba(5, 150, 105, 0.08); - --warning: #d97706; - --warning-soft: rgba(217, 119, 6, 0.08); - --danger: #dc2626; - --danger-soft: rgba(220, 38, 38, 0.08); + --bg: #fafafa; + --surface: #f4f4f5; + --surface-2: #ececed; + --surface-3: #ececed; + --border: #e2e2e5; + --border-strong: #d0d0d3; + --text: #111113; + --text-muted: #5a5a62; + --text-faint: #85858c; + --accent: #65a30d; + --accent-soft: rgba(101, 163, 13, 0.08); + --accent-text: #4d7c0f; + --font: var(--font-sans); + --bg-main: var(--bg); + --bg-sidebar: var(--surface); + --bg-activity: var(--surface); + --bg-elevated: var(--surface); + --bg-hover: var(--surface-2); + --bg-active: var(--surface-2); + --text-dim: var(--text-faint); + --accent-hover: var(--accent); + --text-on-accent: var(--bg); + --success: var(--accent); + --success-soft: var(--accent-soft); + --warning: var(--text-muted); + --warning-soft: transparent; + --danger: var(--text-muted); + --danger-soft: transparent; } * { box-sizing: border-box; - margin: 0; - padding: 0; border-radius: 0 !important; box-shadow: none !important; text-shadow: none !important; } html, body { - font-family: var(--font-sans) !important; + font-family: var(--font) !important; background-color: var(--bg-main); color: var(--text); - font-size: 12px; - line-height: 1.4; + font-size: 14px; + line-height: 1.5; -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale; } @@ -82,7 +98,11 @@ h1, h2, h3, h4, h5, h6 { font-family: var(--font-sans) !important; font-weight: 500; color: var(--text); - letter-spacing: -0.01em; + letter-spacing: 0; +} + +b, strong { + font-weight: 500; } /* Code & pre font */ @@ -92,8 +112,8 @@ pre, code, kbd, samp { /* Scrollbar clean styling */ ::-webkit-scrollbar { - width: 5px; - height: 5px; + width: 8px; + height: 8px; } ::-webkit-scrollbar-track { background: transparent; @@ -107,8 +127,13 @@ pre, code, kbd, samp { /* Interactive focus */ button:focus-visible, input:focus-visible, select:focus-visible, textarea:focus-visible { - outline: 1px solid var(--accent); - outline-offset: 1px; + outline: 2px solid var(--accent); + outline-offset: 2px; +} + +::selection { + background: var(--accent-soft); + color: var(--text); } /* Hide floating dev badges */ diff --git a/app/layout.tsx b/app/layout.tsx index a9e162c..0d565fb 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -1,20 +1,20 @@ +import { ToastProvider } from '@/components/ui/Toast'; import type { Metadata } from 'next'; import './globals.css'; -import { ToastProvider } from '@/components/ui/Toast'; export const metadata: Metadata = { title: 'DataStack - Realtime Data Collection Platform', - description: 'Self-hosted universal realtime data collection engine with Socket.IO ingestion, Redis buffer, JWT client auth, CSV batch rotation, and /vcon/ admin panel.', + description: 'A self-hosted workspace for collecting, reviewing, and exporting realtime data.', manifest: '/manifest.json', openGraph: { title: 'DataStack - Realtime Data Collection Platform', - description: 'Self-hosted universal realtime data collection engine with Socket.IO ingestion, Redis buffer, and CSV batch storage.', + description: 'Collect, review, and export realtime data from one clear workspace.', type: 'website', }, twitter: { card: 'summary_large_image', title: 'DataStack - Realtime Data Collection Platform', - description: 'Self-hosted universal realtime data collection engine with Socket.IO ingestion.', + description: 'A clear workspace for collecting, reviewing, and exporting realtime data.', }, }; @@ -23,7 +23,7 @@ export default function RootLayout({ children }: { children: React.ReactNode }) - + {children} diff --git a/app/not-found.tsx b/app/not-found.tsx index d705385..01fe544 100644 --- a/app/not-found.tsx +++ b/app/not-found.tsx @@ -1,7 +1,6 @@ -import React from 'react'; -import Link from 'next/link'; import { MainLayout } from '@/components/layout/MainLayout'; import { Button } from '@/components/ui/Button'; +import Link from 'next/link'; export default function NotFound() { return ( @@ -17,11 +16,6 @@ export default function NotFound() { Return Home - - - diff --git a/app/page.tsx b/app/page.tsx deleted file mode 100644 index d04ab0d..0000000 --- a/app/page.tsx +++ /dev/null @@ -1,17 +0,0 @@ -import React from 'react'; -import { MainLayout } from '@/components/layout/MainLayout'; -import { Hero } from '@/components/sections/Hero'; -import { Features } from '@/components/sections/Features'; -import { Architecture } from '@/components/sections/Architecture'; - -export default function RootHomePage() { - return ( - -
- - - -
-
- ); -} diff --git a/app/vcon/clients/page.tsx b/app/vcon/clients/page.tsx index 37f0c11..b241d30 100644 --- a/app/vcon/clients/page.tsx +++ b/app/vcon/clients/page.tsx @@ -1,393 +1,106 @@ 'use client'; -import React, { useState, useEffect, useCallback } from 'react'; -import { Card } from '@/components/ui/Card'; +import { AdminPageHeader } from '@/components/layout/AdminPageHeader'; +import type { ClientItem, ClientStatusFilter } from '@/components/sections/clients/types'; +import { ClientsFilters } from '@/components/sections/ClientsFilters'; +import { ClientsTable } from '@/components/sections/ClientsTable'; +import { NewClientModal } from '@/components/sections/NewClientModal'; import { Button } from '@/components/ui/Button'; -import { Input } from '@/components/ui/Input'; -import { Modal } from '@/components/ui/Modal'; -import { Icon } from '@/components/ui/Icon'; -import { - TableWrap, - Table, - TableHeader, - TableBody, - TableRow, - TableHead, - TableCell, - TableEmpty, -} from '@/components/ui/Table'; -import { useToast } from '@/components/ui/Toast'; - -interface ClientItem { - clientId: string; - name: string; - secret: string; - token: string; - rateLimit: number; - status: 'active' | 'revoked'; - totalEvents: number; - lastActive?: string; - createdAt: string; -} +import { StatStrip } from '@/components/ui/StatStrip'; +import { Plus } from 'lucide-react'; +import { useCallback, useEffect, useMemo, useState } from 'react'; export default function ClientsPage() { - const { success, error, warning } = useToast(); + const [modalOpen, setModalOpen] = useState(false); const [clients, setClients] = useState([]); - const [loading, setLoading] = useState(true); - - // Modals - const [createModalOpen, setCreateModalOpen] = useState(false); - const [tokenModalOpen, setTokenModalOpen] = useState(false); - const [selectedClient, setSelectedClient] = useState(null); - - // Form State - const [newClientId, setNewClientId] = useState(''); - const [newClientName, setNewClientName] = useState(''); - const [newRateLimit, setNewRateLimit] = useState('100'); - const [submitting, setSubmitting] = useState(false); + const [search, setSearch] = useState(''); + const [statusFilter, setStatusFilter] = useState('all'); + const [now, setNow] = useState(0); const fetchClients = useCallback(async () => { - try { - const res = await fetch('/api/clients'); - if (res.ok) { - const data = await res.json(); - setClients(data.clients || []); - } - } catch { - // ignore - } finally { - setLoading(false); - } + const response = await fetch('/api/clients'); + if (!response.ok) throw new Error('Unable to load clients'); + const data = await response.json(); + setClients(data.clients || []); }, []); useEffect(() => { - fetchClients(); + fetchClients().catch(() => setClients([])); }, [fetchClients]); - const handleCreateClient = async (e: React.FormEvent) => { - e.preventDefault(); - if (!newClientId.trim() || !newClientName.trim()) { - error('Validation Error', 'Client ID and Client Name are required'); - return; - } + useEffect(() => { + const updateClock = () => setNow(Date.now()); + updateClock(); + const interval = setInterval(updateClock, 15_000); + return () => clearInterval(interval); + }, []); - setSubmitting(true); - try { - const res = await fetch('/api/clients', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ - clientId: newClientId.trim(), - name: newClientName.trim(), - rateLimit: Number(newRateLimit) || 100, - }), - }); + const visibleClients = useMemo(() => { + const query = search.trim().toLowerCase(); + return clients.filter((client) => { + const matchesSearch = !query || + client.clientId.toLowerCase().includes(query) || + client.name.toLowerCase().includes(query); + const online = client.status === 'active' && Boolean(client.lastActive) && + now - new Date(client.lastActive!).getTime() < 60_000; + const pending = client.status === 'active' && !client.lastActive; + const matchesStatus = statusFilter === 'all' || + (statusFilter === 'online' && online) || + (statusFilter === 'offline' && !online && !pending) || + (statusFilter === 'pending' && pending); + return matchesSearch && matchesStatus; + }); + }, [clients, now, search, statusFilter]); - if (res.ok) { - const data = await res.json(); - success('Client Registered', `Client ${data.client.clientId} created successfully`); - setCreateModalOpen(false); - setNewClientId(''); - setNewClientName(''); - setNewRateLimit('100'); - fetchClients(); - // Open token modal immediately - setSelectedClient(data.client); - setTokenModalOpen(true); - } else { - const errData = await res.json(); - error('Registration Failed', errData.error || 'Unknown error'); - } - } catch (err: any) { - error('Error', err.message); - } finally { - setSubmitting(false); - } - }; - - const handleToggleStatus = async (client: ClientItem) => { - const newStatus = client.status === 'active' ? 'revoked' : 'active'; - try { - const res = await fetch(`/api/clients/${client.clientId}`, { - method: 'PATCH', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ status: newStatus }), - }); - if (res.ok) { - warning( - `Status Changed`, - `Client ${client.clientId} is now ${newStatus.toUpperCase()}` - ); - fetchClients(); - } - } catch (err: any) { - error('Update Failed', err.message); - } - }; - - const handleRegenerateToken = async (client: ClientItem) => { - try { - const res = await fetch(`/api/clients/${client.clientId}`, { - method: 'PATCH', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ action: 'regenerate_token' }), - }); - if (res.ok) { - const data = await res.json(); - success('Token Regenerated', `New JWT issued for ${client.clientId}`); - setSelectedClient(data.client); - setTokenModalOpen(true); - fetchClients(); - } - } catch (err: any) { - error('Regeneration Failed', err.message); - } - }; - - const handleDeleteClient = async (client: ClientItem) => { - if (!confirm(`Are you sure you want to delete client '${client.clientId}'?`)) return; - - try { - const res = await fetch(`/api/clients/${client.clientId}`, { - method: 'DELETE', - }); - if (res.ok) { - success('Client Removed', `Client ${client.clientId} deleted`); - fetchClients(); - } - } catch (err: any) { - error('Deletion Failed', err.message); - } - }; - - const copyToClipboard = (text: string, label: string) => { - navigator.clipboard.writeText(text); - success('Copied to Clipboard', `${label} copied`); - }; + const onlineCount = clients.filter((client) => + client.status === 'active' && client.lastActive && + now - new Date(client.lastActive).getTime() < 60_000 + ).length; + const pendingCount = clients.filter((client) => + client.status === 'active' && !client.lastActive + ).length; + const offlineCount = clients.length - onlineCount - pendingCount; return ( -
- {/* Header */} -
-

- Client SDKs -

- +
+
+ Console + + Clients
- - {/* Clients Table */} - - - - - Client ID - Client Name - Status - Rate Limit - Total Ingested - Created - Actions - - - - {clients.length === 0 ? ( - - ) : ( - clients.map((client) => ( - - - {client.clientId} - - - {client.name} - - - - {client.status.toUpperCase()} - - - - {client.rateLimit} req/s - - - {client.totalEvents.toLocaleString()} - - - {new Date(client.createdAt).toLocaleDateString()} - - -
- - -
-
-
- )) - )} -
-
-
- - {/* Register Client Modal */} - setCreateModalOpen(false)} - title="Register Client SDK" - footer={ - <> - - - - } - > -
- setNewClientId(e.target.value)} - required - /> - setNewClientName(e.target.value)} - required - /> - setNewRateLimit(e.target.value)} - /> -
-
- - {/* View Token & Credentials Modal */} - setTokenModalOpen(false)} - title={`Credentials: ${selectedClient?.clientId}`} - footer={ - } - > - {selectedClient && ( -
-
- - Client ID - -
- {selectedClient.clientId} -
-
+ /> -
- - Client Secret Key - -
- {selectedClient.secret} -
-
+ -
-
- - JWT Handshake Bearer Token - - -
-