feat(auth): harden session management and security

- Implement server-side session revocation on logout
- Add IP extraction utility and admin setup error handling
- Improve error codes and response consistency for auth endpoints
- Refine landing page redirect logic to handle transient DB outages
- Update deployment documentation and configuration defaults
This commit is contained in:
Team Dark Devil committed 2026-11-19 20:22:01 +06:00
1 parent 935f12a1e9
commit 89e4d68690
81 files changed
+11130 -6516

No files matched your search

+6
View File
@@ -18,6 +18,12 @@ DATASTACK_ADMIN_EMAIL=admin@datastack.dev
# Shared Redis Buffer (optional; enables cross-process buffering & distributed flush locks) # Shared Redis Buffer (optional; enables cross-process buffering & distributed flush locks)
# REDIS_URL=redis://127.0.0.1:6379 # REDIS_URL=redis://127.0.0.1:6379
# Shared Partitioned CSV Storage Directory (optional; defaults to ./storage/data)
# Set an absolute path so Port 3000 (Next.js) and Port 4000 (Ingestion) always share one storage root
# DATASTACK_STORAGE_DIR=/var/lib/datastack/storage/data
# Service Networking # Service Networking
PORT_SOCKET=4000 PORT_SOCKET=4000
INGESTION_INTERNAL_URL=http://127.0.0.1:4000 INGESTION_INTERNAL_URL=http://127.0.0.1:4000
# Optional comma-separated browser CORS origin allowlist for Port 4000 ingestion/Socket.IO (defaults to wildcard if unset)
# DATASTACK_CORS_ORIGINS=https://console.example.com,https://admin.example.com
+130 -10
View File
@@ -1,7 +1,7 @@
# MEMORY.md — DataStack Cross-IDE Persistent Project Memory # MEMORY.md — DataStack Cross-IDE Persistent Project Memory
> **Last Synchronized**: 2026-11-18 11:00:00 UTC > **Last Synchronized**: 2026-11-19
> **Status**: Fresh Source-Code Forensic Audit Complete (`notes/forensic-audit-report.md` — 27 verified findings across 5 categories; awaiting approval to plan incremental remediation phases) > **Status**: **Production & Deployment Hardening Complete (`BASELINE-2026-11-19-R2` + Deployment Hardening) — `4 / 4` R2 Phases (`18/18` bugs) + `5/5` Docker/Coolify Redeploy Fixes (`DEPLOY-BUG-01`–`05`) Verified (`85/85` TS tests, `10/10` Python tests, ESLint-enabled build & lint `0` errors / `0` warnings).**
> **Scope Rule**: Maintain this file accurately across VS Code, Google Antigravity, Cursor, and any other IDE after every modification, feature update, or bug fix (see `.ai-rules` and `AGENTS.md`). > **Scope Rule**: Maintain this file accurately across VS Code, Google Antigravity, Cursor, and any other IDE after every modification, feature update, or bug fix (see `.ai-rules` and `AGENTS.md`).
--- ---
@@ -69,9 +69,13 @@ datastack/
│ ├── dashboard/ clients/ data/ export/ login/ profile/ sdk/ settings/ setup/ simulator/ storage/ │ ├── dashboard/ clients/ data/ export/ login/ profile/ sdk/ settings/ setup/ simulator/ storage/
├── components/ ├── components/
│ ├── ui/ # Single-source-of-truth UI primitives (Button, Input, Table, Modal, Panel, etc.) │ ├── ui/ # Single-source-of-truth UI primitives (Button, Input, Table, Modal, Panel, etc.)
│ ├── layout/ # MainLayout, Navbar, Sidebar, PageShell, AdminPageHeader, Footer │ ├── layout/ # MainLayout, Navbar, Sidebar, VconShell, PageShell, AdminPageHeader, Footer
│ ├── sections/ # Modular feature sections (DashboardOverview, ClientsTable, DataBrowser, ExportWorkspace, SettingsWorkspace, etc.) │ ├── sections/ # Modular feature sections (DashboardOverview, ClientsTable, DataBrowser, ExportWorkspace, SettingsWorkspace, etc.)
│ └── pwa/ # PwaLifecycle.tsx service worker registration │ └── pwa/ # PwaLifecycle.tsx service worker registration
├── lib/
│ ├── adminAuth.ts # Uniform API session guard (requireAdminApiSession) + proxy-safe extractClientIp()
│ ├── vconCache.ts # In-memory SWR cache + clearAuthCaches() for logout/SW cleanup
│ └── vconUiHelpers.ts # Pure Admin UI sync helpers (settings bounds, bridge sync, UTC presets, timestamps)
├── core/ ├── core/
│ ├── adminSettings/ # settingsManager.ts (DB hydration, Port 4000 sync, unhydrated prune guard) │ ├── adminSettings/ # settingsManager.ts (DB hydration, Port 4000 sync, unhydrated prune guard)
│ ├── apiEngine/ # apiService.ts (dashboard metrics & Port 4000 internal bridge queries) │ ├── apiEngine/ # apiService.ts (dashboard metrics & Port 4000 internal bridge queries)
@@ -81,6 +85,9 @@ datastack/
│ ├── sdk/ # sdkManager.ts (DB JWT hydration, client lifecycle, deleted_clients tombstone) │ ├── sdk/ # sdkManager.ts (DB JWT hydration, client lifecycle, deleted_clients tombstone)
│ ├── Security/ # security.ts (rate limit, maxPayloadSizeKb, idempotency, internal headers) │ ├── Security/ # security.ts (rate limit, maxPayloadSizeKb, idempotency, internal headers)
│ └── Storage/ # csvStorage.ts (cooperative lock, rotation, cached streaming row count, prune) │ └── Storage/ # csvStorage.ts (cooperative lock, rotation, cached streaming row count, prune)
├── scripts/
│ ├── dev-all.js # Local concurrent Next.js (3000) + Ingestion (4000) dev runner
│ └── start-all.js # Production concurrent Next.js (`next start`) + Ingestion (`server/index.js`) runner
├── server/ ├── server/
│ ├── index.js # Express 5 + Socket.IO (Port 4000), /health, /internal/* bridge, PG sync, graceful exit │ ├── index.js # Express 5 + Socket.IO (Port 4000), /health, /internal/* bridge, PG sync, graceful exit
│ ├── socket/handlers.js # Handshake JWT auth, ping_verify, sync_data, batch_sync │ ├── socket/handlers.js # Handshake JWT auth, ping_verify, sync_data, batch_sync
@@ -139,13 +146,22 @@ datastack/
│ ├── configuration-reference.md # Complete environment variable & setting bounds │ ├── configuration-reference.md # Complete environment variable & setting bounds
│ ├── dependencies.md # Runtime & dev dependencies audit with rationales │ ├── dependencies.md # Runtime & dev dependencies audit with rationales
│ └── glossary.md # Technical domain terminology & concept glossary │ └── glossary.md # Technical domain terminology & concept glossary
├── notes/ # Fresh forensic audit output only (legacy reports/plans cleared): ├── notes/ # Production Hardening Governance & Scope-Lock Suite (`BASELINE-2026-11-19-R2`):
│ └── forensic-audit-report.md # 100% source-verified audit report (27 issues: 5C, 11H, 10M, 1L) │ ├── README.md # Master governance index & 6-step scope-locked execution protocol
│ ├── FORENSIC_AUDIT_REPORT.md # 100% raw-code verified audit (18 bugs: 0C/6H/8M/4L), Demo vs Prod & UI-vs-Reality matrices
│ ├── BASELINE_FREEZE.md # Frozen baseline snapshot (`BASELINE-2026-11-19-R2`) & 1:1 bug alignment
│ ├── PRODUCTION_READINESS_SCOPE_LOCK.md # (Doc 001) Exact bug fix contracts & per-phase Scope Locks (`Phase-01`..`Phase-04`)
│ ├── PRODUCTION_ROADMAP.md # (Doc 002) Dependency-ordered A-Z 4-phase roadmap & burn-down tracker
│ ├── PHASE_EXECUTION_LOG.md # (Doc 003) Live phase completion log (`4/4` complete) & zero-regression guardrails
│ ├── ERROR_HANDLING_MATRIX.md # (Doc 004) Active error-handling catalog & 9 planned hardening additions
│ └── ACTUAL_IMPLEMENTATION_STATUS.md # Ground-truth working vs completed-per-phase vs remaining ledger
└── tests/ # Automated verification suites: └── tests/ # Automated verification suites:
├── phase01-auth.test.ts # Auth fail-closed, tokenVersion, brute-force lockout, Bearer revocation, CSRF ├── phase01-auth.test.ts # Auth fail-closed, tokenVersion, brute-force lockout, Bearer revocation, CSRF
├── phase02-sync.test.ts # Internal bridge auth, settings sync, JWT rotation kick, rate/security metrics ├── phase02-sync.test.ts # Internal bridge auth, settings sync, JWT rotation kick, rate/security metrics
├── phase03-storage-buffer.test.ts # Redis durability, cooperative CSV locks, streaming row count, export filters ├── phase03-storage-buffer.test.ts # Redis durability, cooperative CSV locks, streaming row count, export filters
├── phase05_sdk_test.py # Python SDK WebSocket/HTTP fallback, offline spool, file streaming ├── phase04-ui-sync.test.ts # UI/API settings bounds, bridge status, UTC presets, timestamp & folder matching
├── phase05-packaging.test.ts # Python/Node packaging parity, PEP 561 py.typed, Node engines & start-all exit
├── phase05_sdk_test.py # Python SDK WebSocket/HTTP fallback, offline spool, file streaming, edge hardening
├── phase06-pwa-landing.test.ts # PWA manifest/icons/sw + navigation routes ├── phase06-pwa-landing.test.ts # PWA manifest/icons/sw + navigation routes
└── phase07-deployment.test.ts # Compose/Redis AOF, Nginx /health, lockfile, prune guard, shutdown flush └── phase07-deployment.test.ts # Compose/Redis AOF, Nginx /health, lockfile, prune guard, shutdown flush
``` ```
@@ -175,10 +191,114 @@ All planned hardening & deployment phases (**Phase-01 through Phase-07**) are **
7. **Docker / Coolify Deployment & Persistence Hardening (Completed — Phase-07)** 7. **Docker / Coolify Deployment & Persistence Hardening (Completed — Phase-07)**
- Redis AOF persistence (`redisdata:/data`), deterministic `package-lock.json` for `npm ci`, Nginx `location = /health` → `ingestion:4000`, `/api/health` container probes, non-blocking startup ownership check in Dockerfiles, unhydrated-DB `autoPruneDays` safety guard in `SettingsManager`, DB JWT hydration in `SDKManager`, graceful `SIGTERM`/`SIGINT` flush in `DataBufferEngine`, and non-destructive `ADD COLUMN IF NOT EXISTS` DDL guards. - Redis AOF persistence (`redisdata:/data`), deterministic `package-lock.json` for `npm ci`, Nginx `location = /health` → `ingestion:4000`, `/api/health` container probes, non-blocking startup ownership check in Dockerfiles, unhydrated-DB `autoPruneDays` safety guard in `SettingsManager`, DB JWT hydration in `SDKManager`, graceful `SIGTERM`/`SIGINT` flush in `DataBufferEngine`, and non-destructive `ADD COLUMN IF NOT EXISTS` DDL guards.
- **Current Forensic Audit Status (2026-11-18)**: - **Current Remediation Status (2026-11-18)**:
- All prior notes in `notes/` were cleared and replaced exclusively with `notes/forensic-audit-report.md` backed strictly by source-code citations and verbatim proofs (zero markdown extrapolation). - **Phase-01 Complete & Verified** (`BUG-A01`, `BUG-A02`, `BUG-A03`, `BUG-A04`, `BUG-A05`, `BUG-E02`):
- Baseline checks currently pass (`24/24` TS tests, `eslint .` clean, `3/3` Python unit tests), while deep source inspection uncovered **27 verified defects/gaps** (`5` Critical, `11` High, `10` Medium, `1` Low) across Auth/Middleware (`A01–A05`), Demo-vs-Prod Split-Brain (`B01–B05`), Buffer/CSV Storage (`C01–C06`), Admin UI vs Backend Reality (`D01–D07`), and SDK/Packaging/PWA (`E01–E04`). - Removed spoofable header/UA CSRF bypasses in `middleware.ts` and rejected `sec-fetch-site: cross-site`.
- Next step pending user approval: organize remediation into an incremental phased plan based strictly on `notes/forensic-audit-report.md`. - Enforced DB-backed `tokenVersion` session validation across `/vcon/*` layout (`app/vcon/layout.tsx` + `components/layout/VconShell.tsx`) and read/export APIs (`/api/data`, `GET /api/events`, `/api/export`, `/api/metrics`, `/api/storage`).
- Prevented transient DB errors from triggering false `/vcon/setup` redirects and prevented swallowed `NEXT_REDIRECT` errors on `/`, `/vcon/login`, and `/vcon/setup`.
- Standardized `SDKManager.normalizeClientId`, blocked deleted/unregistered clients on `/api/health` and Admin `POST /api/events`, hardened `POST /api/clients` & `PATCH /api/clients/[id]` input validation, and excluded `/vcon/*` from Service Worker caching with `CLEAR_AUTH_CACHE` on sign-out.
- Verified: `26/26` TS tests, `3/3` Python tests, `npm run build` clean, `eslint .` clean, Playwright browser auth/logout E2E clean.
- **Phase-02 Complete & Verified** (`BUG-B01`–`BUG-B05`):
- Enforced fail-closed client verification in `server/index.js` (`verifyClientInDb`) backed by PG, deletion tombstones, Redis `datastack:clients:*` mirrors, and normalized `clientId` keys across both runtimes.
- Added cross-runtime Redis idempotency (`datastack:idem:*`) and fixed-window rate limiting (`datastack:rl:*`) in `core/Security/security.ts` (`validateEventAsync`) and `server/socket/handlers.js` (`validateSingleEventAsync`) with automatic local RAM fallback.
- Surfaced bridge sync telemetry (`BridgeSyncResult`, `SettingsBridgeSyncStatus`, `runtime.ingestionBridgeStatus`, `POST /api/settings` `bridgeSync`) and normalized `SDKManager.incrementEventCountAsync()`.
- Fixed Socket `sync_data` flat payload preservation (`extractSingleEventData`), reconciled `PORT_SOCKET`/`PORT` precedence and live `/internal/metrics` ports, and added concurrent production `scripts/start-all.js` for `npm start`.
- Verified: `29/29` TS tests, `3/3` Python tests, `npm run build` clean, `eslint .` clean, Playwright browser check (`/vcon/dashboard`, `/vcon/settings`, `/api/metrics`) clean with `0` console errors.
- **Phase-03 Complete & Verified** (`BUG-C01`–`BUG-C06`):
- Hardened Redis flush lock TTL (`30000ms`), added in-process `activeFlushPromise` coalescing with a `flushDone` identity guard (`core/data/buffer.ts`, `server/index.js`), isolated `redis.rpush` from best-effort `REDIS_RECENT_KEY`, and logged malformed buffered JSON warnings.
- Enforced `sanitizeClientId()` path containment and `lstat` symlink exclusion, unique lock owner tokens (`${pid}:${ts}:${uuid}`), atomic `rename` stale-lock takeover, owner-verified lock unlock, and structured `JSON.stringify([clientId, date])` partition grouping in `core/Storage/csvStorage.ts` and `server/workers/csvWriter.js`.
- Fixed trailing newline blank-row prevention (`fileEndsWithNewline`), proactive size-boundary rotation (`-partN.csv`), `StringDecoder('utf8')` streaming row counting, cooperatively locked `pruneExpiredPartitions`, and 1-hour `PRUNE_COOLDOWN_MS` in `core/adminSettings/settingsManager.ts`.
- Enforced `parser`/`fileStream` destruction in `finally` blocks across `/api/data`, `GET /api/events`, and `/api/export`, error propagation inside `makeExportStream`, `502` surfacing on Port `4000` flush failure in `/api/export`, normalized `clientId` filtering, numeric timestamp sorting (`parseTimestampMs`), and `?format=json` / `?includeHeader=false` parity for `?filename=` exports.
- Verified: `32/32` TS tests, `3/3` Python tests, `npm run build` clean, `eslint .` clean, Playwright browser check (`/vcon/dashboard`, `/vcon/data`, `/vcon/storage`, `/vcon/export`, `/api/data`, `/api/storage`, `/api/export`) clean with `0` console errors.
- **Phase-04 Complete & Verified** (`BUG-D01`–`BUG-D07`):
- Extracted pure UI synchronization helpers in `lib/vconUiHelpers.ts` (`SETTINGS_LIMITS`, `normalizeEditableSettings`, `describeSettingsBridgeSync`, `mapBridgeStatusToSocketUiState`, `parseEventTimestamp`, `formatEventTimeLabel`, `getUtcDateRangeForPreset`, `matchesSelectedClientFolder`, `extractContentDispositionFilename`).
- Updated `components/sections/SettingsWorkspace.tsx` (`BUG-D01`) to probe `/api/metrics` honestly (removing false `/api/health` fallback), surface `bridgeSync` warning/error toasts on save, align integer bounds with `/api/settings` (`1..100000`, `1..10240`, `0..3650`), and eliminate the startup `serverPort` double-fetch.
- Updated `app/vcon/storage/page.tsx` (`BUG-D02`) and `components/sections/ExportWorkspace.tsx` (`BUG-D03`) with `cache: 'no-store'`, `401` redirects, UTC preset ranges, normalized client folder filters, and fetch+Blob downloads with `502`/JSON error surfacing.
- Updated `components/sections/DashboardOverview.tsx` (`BUG-D04`) and `components/sections/DataBrowser.tsx` (`BUG-D05`) with `ingestionBridgeStatus` display, `auth_error` alert banner, safe ISO/epoch-ms timestamp formatting, invalid date-range guard, and Enter-key filter submission.
- Updated `components/sections/NewClientModal.tsx`, `components/sections/ClientsTable.tsx`, `app/vcon/simulator/page.tsx`, `app/vcon/sdk/page.tsx`, `components/sections/ProfileWorkspace.tsx`, and `components/layout/Sidebar.tsx` (`BUG-D06`, `BUG-D07`) with `1..100000` rate limit parity, live `runtime.serverPort` sync, real HTTP batch ingestion in Simulator, password bounds check, and `datastack:profile-updated` live Sidebar sync.
- Verified: `37/37` TS tests (`tests/phase04-ui-sync.test.ts`), `3/3` Python tests, `npm run build` clean, `eslint .` clean, Playwright browser check across all Admin Console routes clean with `0` console errors.
- **Phase-05 Complete & Verified** (`BUG-E01`, `BUG-E03`, `BUG-E04`):
- Hardened canonical Python SDK (`sdk/python/datastack/datastack/*`): normalized `CLIENT_ID` comparison (`_normalize_client_id`), sanitized ragged CSV extra columns (`restkey="_extra"`), accurate `send_file()` `success` + `unprocessedItems`, partial-batch `rateLimitedEventIds` retry exception fallback, `DataStackConfigError` wrapping for malformed numeric env/config values (`config.py`), concurrent `stop()` vs `set_idle_connected()` race fix + clean thread join (`heartbeat.py`), `os.fsync` + orphan `.tmp` unlink + warning logs (`spool.py`), and PEP 561 `py.typed` marker.
- Aligned packaging metadata across `sdk/python/datastack/pyproject.toml`, `sdk/python/datastack/setup.py`, and `sdk-python/setup.py`, coordinated child-exit shutdown in `scripts/start-all.js`, and added `engines.node >=20.0.0` in `package.json`.
- Verified: `39/39` TS tests (`tests/phase05-packaging.test.ts`), `6/6` Python tests (`tests/phase05_sdk_test.py`), `py_compile`, `npm run build`, and `eslint .` clean.
- **Previous Remediation Cycle (2026-11-18)**: Completed Phases 01–05 (`39/39` TS tests, `6/6` Python tests).
- **New Production Hardening Cycle (`BASELINE-2026-11-19`)**:
- Re-audited entire codebase from raw source code (`notes/FORENSIC_AUDIT_REPORT.md`): **29** verified bugs/mismatches (`4` Critical, `8` High, `13` Medium, `4` Low).
- **Phase-01 Complete & Verified (`2026-11-19`)** (`BUG-01`, `BUG-04`, `BUG-05`, `BUG-06`):
- Blocked volatile RAM admin creation in production without `DATABASE_URL` (`AdminDatabaseRequiredError`), surfaced production DB warning on `/vcon/setup`, and mapped pre-setup login to `409 SETUP_REQUIRED`.
- Added brute-force attempt throttling (`429` + `Retry-After`) and completion-first ordering to `POST /api/auth/setup`.
- Added server-side `token_version` revocation (`revokeAdminSessions()`) on `POST /api/auth/logout`, removed client `Origin` trust from cookie `Secure` flag, and added `sec-fetch-site: same-origin` CSRF fallback.
- Prevented empty production `jwt_secret` persistence in PG and replaced silent DB write `catch {}` swallows in `SettingsManager` and `SDKManager` with `DATABASE_WRITE_FAILED` propagation and safe RAM rollback.
- Verified: `43/43` TS tests, `6/6` Python tests, `npm run build`, `eslint .`, and Playwright login/logout revocation check clean.
- **Phase-02 Complete & Verified (`2026-11-19`)** (`BUG-02`, `BUG-03`, `BUG-17`, `BUG-22`):
- Protected Port 4000's `inMemoryClientRegistry` from empty Port 3000 `bulk_sync` wipes (`!forceReplace`), exposed `inMemoryClients` on `/internal/metrics`, added non-destructive hydration in `SDKManager` (`listClientsAsync`, `getClientAsync`, `syncAllClientsToIngestionAsync`), and auto-reconciled Port 4000 cold starts in `ApiEngine.getDashboardMetrics()`.
- Ordered `SettingsManager.updateSettingsAsync()` JWT rotation to sync Port 4000 *before* committing Port 3000 RAM/PG/re-sign, aborting fail-closed (`JWT_ROTATION_BRIDGE_FAILED`) on bridge rejection/prod unreachable, rolling back Port 4000 on PG/re-sign error, and adding a 60s dual-secret grace window in `server/index.js` `requireInternalAuth()`.
- Added authenticated `POST /internal/security-check` on Port 4000 (backed by zero-copy `validateSingleEventLocal`) so Port 3000 HTTP `SecurityEngine.validateEventAsync()` shares rate limits & idempotency with Port 4000 WS even without Redis, and exposed explicit `securityMode` (`redis_shared` | `bridge_coordinated` | `degraded_local`) in `/api/metrics`.
- Replaced per-event PG `UPDATE client_sdks` queries on both Port 3000 (`SDKManager`) and Port 4000 (`server/index.js`) with 1s coalescing in-memory deltas flushed via batched `UNNEST` SQL updates (flushed on reads and graceful shutdown).
- Verified: `47/47` TS tests, `6/6` Python tests, `npm run build`, and `eslint .` clean.
- **Phase-03 Complete & Verified (`2026-11-19`)** (`BUG-18`, `BUG-19`, `BUG-20`, `BUG-29`):
- Added bounded Redis flush lock wait/retry, periodic Lua `PEXPIRE` TTL renewal with ownership fencing before `LTRIM`, and explicit `FLUSH_LOCK_TIMEOUT` (`503` on `/internal/flush`) across both `core/data/buffer.ts` and `server/index.js`.
- Added full deleted-client buffer purge (`DataBufferEngine.purgeClientEventsAsync`, Port 4000 `purgeClientFromLocalBuffer`) plus flush-time and recent-read tombstone filtering so deleted client directories never resurrect on subsequent flushes; ordered `SDKManager.deleteClientAsync()` to purge buffers and sync Port 4000 before deleting disk files.
- Converted `server/index.js` `onBufferPush` to `async` and awaited it before firing Socket.IO ACK callbacks in `server/socket/handlers.js` (`sync_data` and `batch_sync`), returning `BUFFER_PERSIST_FAILED` on persistence error.
- Unified canonical CSV storage root resolution via `DATASTACK_STORAGE_DIR` (`resolveDefaultStorageDir()`) across `core/Storage/csvStorage.ts`, `server/workers/csvWriter.js`, `core/sdk/sdkManager.ts`, and `.env.example`.
- Verified: `51/51` TS tests, `6/6` Python tests, `npm run build`, and `eslint .` clean.
- **Phase-04 Complete & Verified (`2026-11-19`)** (`BUG-21`, `BUG-23`, `BUG-24`):
- Aligned HTTP `POST /api/events` single-event envelope stripping 1:1 with Port 4000 WS `sync_data` and eliminated fabricated `{ sample, rand }` telemetry on empty `{}`.
- Converted `GET /api/data` directory/stat scans to non-blocking async `node:fs/promises` with newest-first date-partition pruning, and scaled `REDIS_RECENT_LIMIT` boundedly (`500..5000`) via `resolveRedisRecentLimit()` / `DATASTACK_RECENT_LIMIT` across Port 3000 & Port 4000.
- Replaced all `Math.random()` client secrets (`crypto.randomBytes(16)`) and fallback `eventId`s (`crypto.randomBytes(6)`) with CSPRNG across `core/sdk/sdkManager.ts`, `app/api/events/route.ts`, and `server/socket/handlers.js`.
- Unified canonical PostgreSQL metadata DDL (`CORE_SCHEMA_TABLE_DDLS` + `CORE_SCHEMA_ALTER_DDLS` + `ensureCoreSchemaOnPool()`) in `core/Database/database.ts`, delegated `core/auth/adminUsers.ts` to it, mirrored all 5 tables + 3 additive `ALTER TABLE` guards 1:1 in standalone `server/index.js`, and documented canonical `pg` ownership in `prisma/schema.prisma`.
- Verified: `55/55` TS tests (`tests/phase04-ingestion-hygiene.test.ts` added), `6/6` Python tests, `npm run build`, and `eslint .` clean.
- **Phase-05 Complete & Verified (`2026-11-19`)** (`BUG-07`, `BUG-08`, `BUG-09`, `BUG-14`):
- Fixed Simulator HTTP `Burst 10` to send canonical `{ clientId, batch }` (with `events` batch alias & explicit `accepted` count in `POST /api/events`) and summarize exact server-confirmed counts via `summarizeHttpBatchResult()` without `res.ok ? count : 0` falsification.
- Fixed `parseEventTimestamp()` & added `getEventUtcDate()` in `lib/vconUiHelpers.ts` for integer and fractional epoch-seconds (`< 1e11 * 1000`) 1:1 with backend CSV parsers.
- Added pending-queue-only `getPendingBufferedSummary()` (`?bufferedOnly=true` / `?includeBufferedSummary=true`) and fair multi-client partition scanning (`selectRecentPartitionFiles()`, lifting the global 5-file cap) in `GET /api/events`.
- Updated `ExportWorkspace.tsx` to match buffered events strictly against active client/date filters via `countMatchingBufferedEvents()` (preventing empty filtered header downloads) and refresh `/api/storage`, `/api/metrics`, and `vconCache` immediately post-export; updated `DashboardOverview.tsx` to attribute strictly UTC-today buffered events to `Events today` and call `await refresh()` after Quick Export.
- Verified: `58/58` TS tests, `6/6` Python tests, `npm run build`, `eslint .`, and Playwright E2E browser check (`/vcon/simulator`, `/vcon/export`, `/vcon/dashboard`, `0` console errors/warnings) clean.
- **Phase-06 Complete & Verified (`2026-11-19`)** (`BUG-10`, `BUG-11`, `BUG-12`, `BUG-13`, `BUG-15`, `BUG-16`):
- Returned explicit `SESSION_EXPIRED` / `AUTH_UNAVAILABLE` vs `INVALID_PASSWORD` codes in `/api/auth/profile`, replaced dead `401 && !currentPassword` redirect with `shouldRedirectProfileUnauthorized()`, and formatted `Session Expires` with calendar date + 24h time (`formatSessionExpiryLabel()`).
- Added opt-in `?purgeFiles=true` checkbox in `ClientsTable.tsx` client deletion modal and removed the misleading unused `Secret key` row from `CredentialsDisplay`.
- Refactored `SettingsWorkspace.tsx` numeric inputs to string drafts (`NumericSettingDrafts`) validated inline via `validateEditableSettingsDraft()` (`noValidate` submit + tab jump to first invalid field, zero snap-to-`0`, zero silent clamping) and clarified `Socket port (PORT_SOCKET)` as read-only runtime config.
- Unified public endpoint resolution (`resolvePublicEndpoints()`) across `SettingsWorkspace`, `/vcon/sdk`, and `/vcon/simulator` (consistent on port-80 `localhost` proxies) and updated `SdkSection.tsx` + `/vcon/sdk` with external path/wheel `pip install` guidance.
- Added pre-hydration `datastack-theme` bootstrap script + `suppressHydrationWarning` in `app/layout.tsx` (zero light-mode FOUC) and theme restoration/toggle in `AdminSetup.tsx`.
- Upgraded `public/sw.js` (`datastack-cache-v3`) with an uncached `503` HTML offline navigation fallback shell (`buildOfflineShellResponse()`, `Cache-Control: no-store`) for `/vcon/dashboard` and public navigations.
- Verified: `64/64` TS tests, `6/6` Python tests, `npm run build`, `eslint .`, and Playwright Chromium E2E check (`/vcon/profile`, `/vcon/clients`, `/vcon/settings`, `/vcon/sdk`, Light Mode reload, `0` console errors/warnings) clean.
- **Fresh Post-Phase-07 Forensic Audit & Baseline Freeze (`BASELINE-2026-11-19-R2`)**:
- Previous `Phase-07` hardening (`BUG-25`–`BUG-28`) landed in Python SDK (`client.py`), legacy `sdk-python`, `scripts/start-all.js` / `dev-all.js`, `nginx/default.conf`, `server/index.js`, `docker/docker-compose.yml`, and `.env.example`.
- Conducted a brand-new 100% raw-source-code audit (`notes/FORENSIC_AUDIT_REPORT.md`) identifying **18 verified remaining bugs** (`0` Critical, `6` High, `8` Medium, `4` Low).
- Frozen baseline (`BASELINE-2026-11-19-R2`) and created the complete 7-document governance & Scope-Lock suite in `notes/` (`README.md`, `BASELINE_FREEZE.md`, `PRODUCTION_READINESS_SCOPE_LOCK.md`, `PRODUCTION_ROADMAP.md`, `PHASE_EXECUTION_LOG.md`, `ERROR_HANDLING_MATRIX.md`, `ACTUAL_IMPLEMENTATION_STATUS.md`).
- **R2 Phase-01 Complete & Verified (`2026-11-19`)** (`BUG-01`, `BUG-02`):
- Moved same-origin CSRF enforcement in `middleware.ts` ahead of the public-route early return so `POST /api/auth/login` and `POST /api/auth/setup` reject cross-origin/missing-origin requests with `403` while preserving same-origin browser submissions, `sec-fetch-site: same-origin` fallback, safe `GET` routes, and Bearer `POST /api/events`.
- Extracted shared reverse-proxy-safe `extractClientIp()` in `lib/adminAuth.ts` (trusting `x-real-ip` first, then rightmost non-empty `x-forwarded-for` hop) and wired it into both `app/api/auth/login/route.ts` and `app/api/auth/setup/route.ts` to block spoofed-XFF brute-force lockout bypasses.
- Verified: `14/14` `tests/phase01-auth.test.ts` pass, `npm run build`, `npm run lint`, live HTTP `403`/`401` CSRF probes, and Chromium `/vcon/login` → `/vcon/dashboard` E2E (`0` errors).
- **R2 Phase-02 Complete & Verified (`2026-11-19`)** (`BUG-05`, `BUG-06`, `BUG-07`, `BUG-08`, `BUG-09`, `BUG-10`):
- Attached `.on('error', ...)` listeners to `DatabaseEngine.getPool()` (`max: 10`) and `server/index.js` `dbPool`, and consolidated `core/auth/adminUsers.ts` onto the shared `DatabaseEngine.getPool()`.
- Replaced non-atomic Redis `LTRIM`+`LPUSH` client deletion purging in `core/data/buffer.ts` with atomic Lua script `PURGE_CLIENT_LIST_LUA_SCRIPT` (`cjson.decode` + chunked `RPUSH`), and replaced non-atomic Redis `INCR`+`PEXPIRE` rate limiting with atomic Lua script `REDIS_RATE_LIMIT_LUA_SCRIPT` across both `core/Security/security.ts` and `server/socket/handlers.js`.
- Enforced `throwOnLockTimeout: true` locally and remotely on `POST /internal/flush` in `GET /api/export` and `POST /api/storage`, surfacing explicit `503 FLUSH_LOCK_TIMEOUT`.
- Added `DataBufferEngine.registerShutdownHook()` and registered `SDKManager.flushPendingClientActivityAsync()` so Port `3000` flushes coalesced `total_events` deltas on `SIGTERM`/`SIGINT`.
- Wrapped `GET`, `PATCH` (`regenerate_token` & update), and `DELETE` in `app/api/clients/[id]/route.ts` with structured JSON `try/catch` error responses.
- Verified: `14/14` `tests/phase02-sync.test.ts` pass, `3/3` `R2 PHASE-02` buffer/storage tests pass, `npm run build` pass, `npm run lint` pass (`0` errors).
- **R2 Phase-03 Complete & Verified (`2026-11-19`)** (`BUG-03`, `BUG-04`, `BUG-11`, `BUG-12`, `BUG-13`, `BUG-15`):
- Unified `SettingsBridgeSyncStatus` & `POST /api/settings` `bridgeSync` response (`attempted`, `ok`, `synced`, `status`, `error`, `checkedAt`) and updated `describeSettingsBridgeSync()` + `SettingsWorkspace.tsx` so Port `4000` warnings/online state work reliably.
- Counted regular non-symlink `.csv` files in `SDKManager.purgeClientStorageDir()` (`deleteClientWithStatsAsync()`) and returned exact `purgedFiles` from `DELETE /api/clients/[id]?purgeFiles=true`.
- Aligned flat batch item metadata stripping (`eventId`, `timestamp`, `clientId`) 1:1 with single-event stripping across HTTP `POST /api/events`, WS `batch_sync`, and Python SDK `send_batch()`/`send_file()`.
- Exposed `DataBufferEngine.getLocalBufferSnapshot()` for exact Port `3000` RAM pending summary in `getPendingBufferedSummary()`.
- Added non-blocking `pruneExpiredPartitionsAsync()` off live event loops in `core/Storage/csvStorage.ts` & `server/workers/csvWriter.js` (wired into `settingsManager.ts` & `server/index.js`) and deduplicated `GET /api/storage` to a single `listAllFilesAsync()` scan with accurate client-scoped `stats`.
- Cleared `selectedClientToken` on `/vcon/sdk` client switch (with loading placeholder + disabled `Copy Config`), added `400 INVALID_DATE_RANGE` for `from > to` in `GET /api/export`, and normalized `clientId` filtering in `ApiEngine.getExportFiles*` & Port `4000` `/internal/recent`.
- Verified: `21/21` `phase04-*` tests pass, `2/2` `R2 PHASE-03` storage tests pass, `9/9` Python SDK tests pass, Chromium E2E across `/vcon/settings`, `/vcon/clients`, `/vcon/sdk`, `/vcon/storage` (`0` console errors), `npm run build` & `npm run lint` (`0` errors).
- **R2 Phase-04 Complete & Verified (`2026-11-19`)** (`BUG-14`, `BUG-16`, `BUG-17`, `BUG-18`):
- Added `from __future__ import annotations`, imported `Tuple`, and replaced PEP 585 `tuple[int, Dict[str, Any]]` with Python `3.8`-safe `Tuple[int, Dict[str, Any]]` in `sdk/python/datastack/datastack/client.py`, verified by a full-package Python `3.8` AST scan test in `tests/phase05_sdk_test.py`.
- Added unbuffered streaming `location = /api/export` (`proxy_buffering off;`, `proxy_read_timeout 300s;`) targeting `http://nextjs_upstream` in both `nginx/default.conf` and `docker/coolify-nginx.conf`.
- Removed `eslint: { ignoreDuringBuilds: true }` from `next.config.ts`, set `package.json` `"name": "datastack"`, pruned unused template packages (`@google/genai`, `firebase-tools`), and synchronized `package-lock.json`.
- Isolated `DATASTACK_DISABLE_LISTEN=1` and live Port `4000` coupling across `tests/phase01-auth.test.ts`, `tests/phase02-sync.test.ts`, `tests/phase03-storage-buffer.test.ts`, `tests/phase04-ingestion-hygiene.test.ts`, and `tests/phase07-deployment.test.ts`.
- Verified: `83/83` full TS suite pass (with live Port `4000` running, without manual env flags), `10/10` Python SDK tests + `py_compile` pass, `npm run build` (with ESLint active) pass, `npm run lint` (`0` errors, `0` warnings).
- **Progress**: `4 / 4` phases complete (`18 / 18` bugs resolved — **100% Production Ready**).
- **Pre-Release Docker & Coolify Deployment Hardening Complete & Verified (`2026-11-19`)** (`DEPLOY-BUG-01`–`05`, audited in `notes/DEPLOYMENT_COOLIFY_AUDIT_REPORT.md`):
- **`DEPLOY-BUG-01`**: Replaced static `upstream` blocks in `docker/coolify-nginx.conf` with Docker embedded DNS `resolver 127.0.0.11 valid=10s ipv6=off;` and variable `proxy_pass` targets (`$nextjs_upstream` = `http://web:3000`, `$socketio_upstream` = `http://ingestion:4000`), preventing `502 Bad Gateway` stale container IP outages after container restarts/redeploys.
- **`DEPLOY-BUG-02`**: Refactored Port `4000` startup in `server/index.js` into `startIngestionServer()` awaiting bounded `syncSettingsFromDatabase()` before `server.listen(PORT)` so rotated PG `jwt_secret` and rate-limit settings hydrate prior to `/health: 200` and `/internal/*` bridge traffic.
- **`DEPLOY-BUG-03`**: Memoized `ensureDatabaseSchema()` per active `dbPool` instance in `server/index.js` (resetting only on pool swap or DDL failure) so 3-second `syncSettingsFromDatabase()` ticks no longer issue 8 `CREATE/ALTER TABLE` DDLs repeatedly.
- **`DEPLOY-BUG-04`**: Reordered `gracefulShutdown()` in `server/index.js` to close Socket.IO/HTTP ingress first (`closeServerGracefully()`), drain `pendingRedisWrites` and `flushBuffer()`/`flushPendingClientActivity()` with bounded timeouts (`2000ms`/`6000ms`), and close Redis (`quit()` with `2000ms` timeout + `.disconnect()` fallback) and PostgreSQL (`dbPool.end()` bounded `2000ms`); added `stop_grace_period: 20s` to both `web` and `ingestion` in `docker/docker-compose.yml`.
- **`DEPLOY-BUG-05`**: Clarified Coolify `Base Directory: /` and `Docker Compose Location: /docker/docker-compose.yml` in `docs/users/installation.md` and `README.md`.
- Verified: `85/85` full TS suite pass (`11/11` in `tests/phase07-deployment.test.ts`), `10/10` Python SDK tests pass, `npm run build` clean, `npm run lint` (`0` errors, `0` warnings).
--- ---
+1 -1
View File
@@ -293,7 +293,7 @@ DataStack includes a production-ready Docker Compose stack in [`docker/docker-co
### Coolify Deployment Steps ### Coolify Deployment Steps
1. Create a **Docker Compose** resource in Coolify pointing to your repository. 1. Create a **Docker Compose** resource in Coolify pointing to your repository.
2. Set **Base Directory** to `/docker` and **Docker Compose Location** to `/docker/docker-compose.yml`. 2. Set **Base Directory** to `/` and **Docker Compose Location** to `/docker/docker-compose.yml`.
3. Attach your HTTPS domain to the `proxy` service on port `80`. 3. Attach your HTTPS domain to the `proxy` service on port `80`.
4. Deploy the stack, open `https://your-domain.example/vcon/setup`, and enter your administrator email, password (8+ characters), and the generated `SERVICE_REALBASE64_64_ADMININSTALL` key from Coolify's environment variables. 4. Deploy the stack, open `https://your-domain.example/vcon/setup`, and enter your administrator email, password (8+ characters), and the generated `SERVICE_REALBASE64_64_ADMININSTALL` key from Coolify's environment variables.
+13 -1
View File
@@ -9,8 +9,20 @@ import { redirect } from 'next/navigation';
export const dynamic = 'force-dynamic'; export const dynamic = 'force-dynamic';
export default async function LandingPage() { export default async function LandingPage() {
const isConfigured = await isAdminSetupComplete().catch(() => false); try {
const isConfigured = await isAdminSetupComplete();
if (!isConfigured) redirect('/vcon/setup'); if (!isConfigured) redirect('/vcon/setup');
} catch (error) {
if (
error &&
typeof error === 'object' &&
'digest' in error &&
String((error as { digest?: unknown }).digest).startsWith('NEXT_REDIRECT')
) {
throw error;
}
// Keep public landing accessible on transient DB outage instead of redirecting to setup
}
return ( return (
<MainLayout> <MainLayout>
+12 -12
View File
@@ -4,27 +4,18 @@ import {
getSessionCookieOptions, getSessionCookieOptions,
} from "@/core/auth/adminSession"; } from "@/core/auth/adminSession";
import { import {
AdminSetupRequiredError,
authenticateAdminCredentials, authenticateAdminCredentials,
clearAdminLoginAttempts, clearAdminLoginAttempts,
consumeAdminLoginAttempt, consumeAdminLoginAttempt,
} from "@/core/auth/adminUsers"; } from "@/core/auth/adminUsers";
import { extractClientIp } from "@/lib/adminAuth";
import { NextRequest, NextResponse } from "next/server"; import { NextRequest, NextResponse } from "next/server";
const SESSION_COOKIE = "datastack_admin_session"; const SESSION_COOKIE = "datastack_admin_session";
const SESSION_SECONDS = 8 * 60 * 60; const SESSION_SECONDS = 8 * 60 * 60;
const REMEMBERED_SESSION_SECONDS = 30 * 24 * 60 * 60; const REMEMBERED_SESSION_SECONDS = 30 * 24 * 60 * 60;
function extractClientIp(request: NextRequest) {
const forwardedFor = request.headers.get("x-forwarded-for");
if (forwardedFor) {
const first = forwardedFor.split(",")[0]?.trim();
if (first) return first;
}
const realIp = request.headers.get("x-real-ip")?.trim();
if (realIp) return realIp;
return "unknown";
}
export async function POST(request: NextRequest) { export async function POST(request: NextRequest) {
const sessionSecret = getAdminSessionSecret(); const sessionSecret = getAdminSessionSecret();
@@ -91,7 +82,16 @@ export async function POST(request: NextRequest) {
let authResult: { authenticated: boolean; tokenVersion: number }; let authResult: { authenticated: boolean; tokenVersion: number };
try { try {
authResult = await authenticateAdminCredentials(email, password); authResult = await authenticateAdminCredentials(email, password);
} catch { } catch (err) {
if (err instanceof AdminSetupRequiredError) {
return NextResponse.json(
{
error: "Initial admin setup is required before signing in.",
code: err.code,
},
{ status: 409 },
);
}
return NextResponse.json( return NextResponse.json(
{ error: "Admin authentication is temporarily unavailable." }, { error: "Admin authentication is temporarily unavailable." },
{ status: 503 }, { status: 503 },
+16
View File
@@ -1,9 +1,25 @@
import { getSessionCookieOptions } from "@/core/auth/adminSession"; import { getSessionCookieOptions } from "@/core/auth/adminSession";
import {
revokeAdminSessions,
verifyRequestAdminSession,
} from "@/core/auth/adminUsers";
import { NextRequest, NextResponse } from "next/server"; import { NextRequest, NextResponse } from "next/server";
const SESSION_COOKIE = "datastack_admin_session"; const SESSION_COOKIE = "datastack_admin_session";
export async function POST(request: NextRequest) { export async function POST(request: NextRequest) {
const token = request.cookies.get(SESSION_COOKIE)?.value;
if (token) {
try {
const check = await verifyRequestAdminSession(token);
if (check.valid) {
await revokeAdminSessions();
}
} catch {
// Still clear client cookie even if server revocation encounters transient DB error
}
}
const response = NextResponse.json({ success: true }); const response = NextResponse.json({ success: true });
response.cookies.set(SESSION_COOKIE, "", { response.cookies.set(SESSION_COOKIE, "", {
...getSessionCookieOptions(request, 0), ...getSessionCookieOptions(request, 0),
+10 -3
View File
@@ -22,11 +22,13 @@ export async function GET(request: NextRequest) {
if (!auth.valid) { if (!auth.valid) {
const status = auth.reason === "UNAUTHORIZED" ? 401 : 503; const status = auth.reason === "UNAUTHORIZED" ? 401 : 503;
const code =
auth.reason === "UNAUTHORIZED" ? "SESSION_EXPIRED" : "AUTH_UNAVAILABLE";
const message = const message =
auth.reason === "UNAUTHORIZED" auth.reason === "UNAUTHORIZED"
? "Authentication required." ? "Authentication required."
: "Admin authentication is temporarily unavailable."; : "Admin authentication is temporarily unavailable.";
return NextResponse.json({ error: message }, { status }); return NextResponse.json({ error: message, code }, { status });
} }
try { try {
@@ -54,11 +56,13 @@ export async function PUT(request: NextRequest) {
if (!auth.valid) { if (!auth.valid) {
const status = auth.reason === "UNAUTHORIZED" ? 401 : 503; const status = auth.reason === "UNAUTHORIZED" ? 401 : 503;
const code =
auth.reason === "UNAUTHORIZED" ? "SESSION_EXPIRED" : "AUTH_UNAVAILABLE";
const message = const message =
auth.reason === "UNAUTHORIZED" auth.reason === "UNAUTHORIZED"
? "Authentication required." ? "Authentication required."
: "Admin authentication is temporarily unavailable."; : "Admin authentication is temporarily unavailable.";
return NextResponse.json({ error: message }, { status }); return NextResponse.json({ error: message, code }, { status });
} }
let body: Record<string, unknown>; let body: Record<string, unknown>;
@@ -115,7 +119,10 @@ export async function PUT(request: NextRequest) {
}); });
if (!result.ok) { if (!result.ok) {
return NextResponse.json({ error: result.error }, { status: 401 }); return NextResponse.json(
{ error: result.error, code: "INVALID_PASSWORD" },
{ status: 401 },
);
} }
// Preserve remaining session duration (e.g., 30-day "Remember me" session) // Preserve remaining session duration (e.g., 30-day "Remember me" session)
+59 -7
View File
@@ -4,14 +4,20 @@ import {
getSessionCookieOptions, getSessionCookieOptions,
} from "@/core/auth/adminSession"; } from "@/core/auth/adminSession";
import { import {
AdminDatabaseRequiredError,
clearAdminLoginAttempts,
consumeAdminLoginAttempt,
createInitialAdmin, createInitialAdmin,
isAdminSetupComplete, isAdminSetupComplete,
requiresPersistentAdminDatabase,
verifyAdminSetupKey, verifyAdminSetupKey,
} from "@/core/auth/adminUsers"; } from "@/core/auth/adminUsers";
import { extractClientIp } from "@/lib/adminAuth";
import { NextRequest, NextResponse } from "next/server"; import { NextRequest, NextResponse } from "next/server";
const SESSION_COOKIE = "datastack_admin_session"; const SESSION_COOKIE = "datastack_admin_session";
const SESSION_SECONDS = 8 * 60 * 60; const SESSION_SECONDS = 8 * 60 * 60;
const SETUP_THROTTLE_BUCKET = "__admin_setup__";
export const runtime = "nodejs"; export const runtime = "nodejs";
export const dynamic = "force-dynamic"; export const dynamic = "force-dynamic";
@@ -29,6 +35,17 @@ export async function POST(request: NextRequest) {
); );
} }
if (requiresPersistentAdminDatabase()) {
return NextResponse.json(
{
error:
"PostgreSQL DATABASE_URL must be configured in production before initial admin setup.",
code: "DATABASE_NOT_CONFIGURED",
},
{ status: 503 },
);
}
let body: Record<string, unknown>; let body: Record<string, unknown>;
try { try {
const parsed = await request.json(); const parsed = await request.json();
@@ -61,12 +78,7 @@ export async function POST(request: NextRequest) {
{ status: 400 }, { status: 400 },
); );
} }
if (!verifyAdminSetupKey(setupKey)) {
return NextResponse.json(
{ error: "The installation key is invalid." },
{ status: 403 },
);
}
if (await isAdminSetupComplete()) { if (await isAdminSetupComplete()) {
return NextResponse.json( return NextResponse.json(
{ error: "Initial admin setup has already been completed." }, { error: "Initial admin setup has already been completed." },
@@ -74,6 +86,34 @@ export async function POST(request: NextRequest) {
); );
} }
const clientIp = extractClientIp(request);
const throttleDecision = await consumeAdminLoginAttempt(
SETUP_THROTTLE_BUCKET,
sessionSecret,
clientIp,
);
if (!throttleDecision.allowed) {
return NextResponse.json(
{
error: "Too many setup attempts. Try again later.",
code: "TOO_MANY_ATTEMPTS",
},
{
status: 429,
headers: {
"Retry-After": String(throttleDecision.retryAfterSeconds),
},
},
);
}
if (!verifyAdminSetupKey(setupKey)) {
return NextResponse.json(
{ error: "The installation key is invalid." },
{ status: 403 },
);
}
const created = await createInitialAdmin(email, password); const created = await createInitialAdmin(email, password);
if (!created) { if (!created) {
return NextResponse.json( return NextResponse.json(
@@ -82,6 +122,12 @@ export async function POST(request: NextRequest) {
); );
} }
await clearAdminLoginAttempts(
SETUP_THROTTLE_BUCKET,
sessionSecret,
clientIp,
).catch(() => undefined);
const token = await createAdminSessionToken( const token = await createAdminSessionToken(
email, email,
sessionSecret, sessionSecret,
@@ -95,7 +141,13 @@ export async function POST(request: NextRequest) {
getSessionCookieOptions(request, SESSION_SECONDS), getSessionCookieOptions(request, SESSION_SECONDS),
); );
return response; return response;
} catch { } catch (err) {
if (err instanceof AdminDatabaseRequiredError) {
return NextResponse.json(
{ error: err.message, code: err.code },
{ status: 503 },
);
}
return NextResponse.json( return NextResponse.json(
{ error: "Admin setup is temporarily unavailable." }, { error: "Admin setup is temporarily unavailable." },
{ status: 503 }, { status: 503 },
+147 -15
View File
@@ -3,6 +3,7 @@ import { SDKManager } from "@/core/sdk/sdkManager";
import { NextRequest, NextResponse } from "next/server"; import { NextRequest, NextResponse } from "next/server";
const SESSION_COOKIE = "datastack_admin_session"; const SESSION_COOKIE = "datastack_admin_session";
const ALLOWED_PATCH_KEYS = new Set(["action", "name", "rateLimit", "status"]);
export async function GET( export async function GET(
req: NextRequest, req: NextRequest,
@@ -19,7 +20,9 @@ export async function GET(
} }
const { id } = await params; const { id } = await params;
const client = await SDKManager.getClientAsync(id); const normalizedId = SDKManager.normalizeClientId(id);
try {
const client = await SDKManager.getClientAsync(normalizedId);
if (!client) { if (!client) {
return NextResponse.json({ error: "Client not found" }, { status: 404 }); return NextResponse.json({ error: "Client not found" }, { status: 404 });
} }
@@ -33,6 +36,17 @@ export async function GET(
...(includeToken ? { token } : {}), ...(includeToken ? { token } : {}),
}, },
}); });
} catch (err: unknown) {
const message = err instanceof Error ? err.message : "Failed to load client";
const isDbError = message.includes("DATABASE_");
return NextResponse.json(
{
error: message,
code: isDbError ? "DATABASE_READ_FAILED" : "CLIENT_FETCH_FAILED",
},
{ status: isDbError ? 503 : 500 },
);
}
} }
export async function PATCH( export async function PATCH(
@@ -50,29 +64,128 @@ export async function PATCH(
} }
const { id } = await params; const { id } = await params;
try { const normalizedId = SDKManager.normalizeClientId(id);
const body = await req.json();
// Special action: regenerate token let rawBody: unknown;
if (body.action === "regenerate_token") { try {
const refreshed = await SDKManager.regenerateTokenAsync(id); rawBody = await req.json();
if (!refreshed) { } catch {
return NextResponse.json( return NextResponse.json(
{ error: "Client not found" }, { error: "Invalid JSON request payload.", code: "INVALID_JSON" },
{ status: 404 }, { status: 400 },
); );
} }
return NextResponse.json({ client: refreshed });
if (!rawBody || typeof rawBody !== "object" || Array.isArray(rawBody)) {
return NextResponse.json(
{ error: "Update payload must be a JSON object.", code: "VALIDATION_ERROR" },
{ status: 400 },
);
} }
const updated = await SDKManager.updateClientAsync(id, body); const body = rawBody as Record<string, unknown>;
const unknownKeys = Object.keys(body).filter(
(key) => !ALLOWED_PATCH_KEYS.has(key),
);
if (unknownKeys.length > 0) {
return NextResponse.json(
{
error: `Unsupported client field(s): ${unknownKeys.join(", ")}`,
code: "VALIDATION_ERROR",
},
{ status: 400 },
);
}
// Special action: regenerate token
if (body.action !== undefined) {
if (body.action !== "regenerate_token") {
return NextResponse.json(
{ error: "Unsupported client action.", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
try {
const refreshed = await SDKManager.regenerateTokenAsync(normalizedId);
if (!refreshed) {
return NextResponse.json({ error: "Client not found" }, { status: 404 });
}
return NextResponse.json({ client: refreshed });
} catch (err: unknown) {
const message =
err instanceof Error ? err.message : "Token regeneration failed";
const isDbWrite = message.includes("DATABASE_WRITE_FAILED");
return NextResponse.json(
{
error: message,
code: isDbWrite ? "DATABASE_WRITE_FAILED" : "TOKEN_REGENERATION_FAILED",
},
{ status: isDbWrite ? 503 : 500 },
);
}
}
const updates: { name?: string; rateLimit?: number; status?: "active" | "revoked" } = {};
if (body.name !== undefined) {
if (
typeof body.name !== "string" ||
body.name.trim().length < 1 ||
body.name.trim().length > 120
) {
return NextResponse.json(
{ error: "Client name must be a non-empty string (max 120 characters).", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
updates.name = body.name.trim();
}
if (body.rateLimit !== undefined) {
if (
typeof body.rateLimit !== "number" ||
!Number.isInteger(body.rateLimit) ||
body.rateLimit < 1 ||
body.rateLimit > 100000
) {
return NextResponse.json(
{ error: "rateLimit must be an integer between 1 and 100000.", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
updates.rateLimit = body.rateLimit;
}
if (body.status !== undefined) {
if (body.status !== "active" && body.status !== "revoked") {
return NextResponse.json(
{ error: "status must be 'active' or 'revoked'.", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
updates.status = body.status;
}
if (Object.keys(updates).length === 0) {
return NextResponse.json(
{ error: "No valid fields provided for update.", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
try {
const updated = await SDKManager.updateClientAsync(normalizedId, updates);
if (!updated) { if (!updated) {
return NextResponse.json({ error: "Client not found" }, { status: 404 }); return NextResponse.json({ error: "Client not found" }, { status: 404 });
} }
return NextResponse.json({ client: updated }); return NextResponse.json({ client: updated });
} catch (err: unknown) { } catch (err: unknown) {
const message = err instanceof Error ? err.message : "Update failed"; const message = err instanceof Error ? err.message : "Update failed";
return NextResponse.json({ error: message }, { status: 500 }); const isDbWrite = message.includes("DATABASE_WRITE_FAILED");
return NextResponse.json(
{
error: message,
code: isDbWrite ? "DATABASE_WRITE_FAILED" : "CLIENT_UPDATE_FAILED",
},
{ status: isDbWrite ? 503 : 500 },
);
} }
} }
@@ -91,10 +204,29 @@ export async function DELETE(
} }
const { id } = await params; const { id } = await params;
const normalizedId = SDKManager.normalizeClientId(id);
const purgeFiles = req.nextUrl.searchParams.get("purgeFiles") === "true"; const purgeFiles = req.nextUrl.searchParams.get("purgeFiles") === "true";
const deleted = await SDKManager.deleteClientAsync(id, { purgeFiles }); try {
if (!deleted) { const result = await SDKManager.deleteClientWithStatsAsync(normalizedId, {
purgeFiles,
});
if (!result.deleted) {
return NextResponse.json({ error: "Client not found" }, { status: 404 }); return NextResponse.json({ error: "Client not found" }, { status: 404 });
} }
return NextResponse.json({ success: true, message: `Client ${id} deleted` }); return NextResponse.json({
success: true,
message: `Client ${normalizedId} deleted`,
purgedFiles: result.purgedFiles,
});
} catch (err: unknown) {
const message = err instanceof Error ? err.message : "Client deletion failed";
const isDbWrite = message.includes("DATABASE_WRITE_FAILED");
return NextResponse.json(
{
error: message,
code: isDbWrite ? "DATABASE_WRITE_FAILED" : "CLIENT_DELETE_FAILED",
},
{ status: isDbWrite ? 503 : 500 },
);
}
} }
+20 -8
View File
@@ -60,17 +60,29 @@ export async function POST(req: NextRequest) {
try { try {
const { clientId, name, rateLimit } = body; const { clientId, name, rateLimit } = body;
if (!clientId || !name) { if (typeof clientId !== "string" || typeof name !== "string" || !clientId.trim() || !name.trim()) {
return NextResponse.json( return NextResponse.json(
{ error: "clientId and name are required" }, { error: "clientId and name are required strings", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
const safeClientId = SDKManager.normalizeClientId(clientId);
if (!safeClientId) {
return NextResponse.json(
{ error: "clientId is invalid", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
const parsedRateLimit = rateLimit === undefined ? 100 : Number(rateLimit);
if (!Number.isInteger(parsedRateLimit) || parsedRateLimit < 1 || parsedRateLimit > 100000) {
return NextResponse.json(
{ error: "rateLimit must be an integer between 1 and 100000.", code: "VALIDATION_ERROR" },
{ status: 400 }, { status: 400 },
); );
} }
const safeClientId = String(clientId)
.trim()
.toLowerCase()
.replace(/[^a-z0-9_-]/g, "_");
const existing = await SDKManager.getClientAsync(safeClientId); const existing = await SDKManager.getClientAsync(safeClientId);
if (existing) { if (existing) {
return NextResponse.json( return NextResponse.json(
@@ -81,8 +93,8 @@ export async function POST(req: NextRequest) {
const newClient = await SDKManager.registerClientAsync({ const newClient = await SDKManager.registerClientAsync({
clientId: safeClientId, clientId: safeClientId,
name: String(name), name: name.trim().slice(0, 120),
rateLimit: rateLimit ? Number(rateLimit) : 100, rateLimit: parsedRateLimit,
}); });
return NextResponse.json({ client: newClient }, { status: 201 }); return NextResponse.json({ client: newClient }, { status: 201 });
+88 -20
View File
@@ -1,10 +1,13 @@
import { SettingsManager } from "@/core/adminSettings/settingsManager"; import { SettingsManager } from "@/core/adminSettings/settingsManager";
import { DataBufferEngine } from "@/core/data/buffer"; import { DataBufferEngine } from "@/core/data/buffer";
import { SDKManager } from "@/core/sdk/sdkManager";
import { getInternalIngestionHeaders } from "@/core/Security/security"; import { getInternalIngestionHeaders } from "@/core/Security/security";
import { CSVStorageManager } from "@/core/Storage/csvStorage"; import { CSVStorageManager } from "@/core/Storage/csvStorage";
import { requireAdminApiSession } from "@/lib/adminAuth";
import { parse } from "csv-parse"; import { parse } from "csv-parse";
import { NextRequest, NextResponse } from "next/server"; import { NextRequest, NextResponse } from "next/server";
import { createReadStream, readdirSync, statSync } from "node:fs"; import { createReadStream } from "node:fs";
import { lstat, readdir } from "node:fs/promises";
import path from "node:path"; import path from "node:path";
interface DataEvent { interface DataEvent {
@@ -15,10 +18,40 @@ interface DataEvent {
status: "stored" | "buffered"; status: "stored" | "buffered";
} }
function parseTimestampMs(raw: string): number {
const trimmed = String(raw || "").trim();
if (/^\d+(\.\d+)?$/.test(trimmed)) {
const num = Number(trimmed);
return num < 1e11 ? num * 1000 : num;
}
const parsed = Date.parse(trimmed);
return Number.isNaN(parsed) ? Number.NEGATIVE_INFINITY : parsed;
}
function compareEventsDesc(a: DataEvent, b: DataEvent): number {
const aMs = parseTimestampMs(a.timestamp);
const bMs = parseTimestampMs(b.timestamp);
if (aMs !== bMs) {
return bMs - aMs;
}
return b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0;
}
export async function GET(request: NextRequest) { export async function GET(request: NextRequest) {
const auth = await requireAdminApiSession(request);
if (!auth.valid) {
return auth.response;
}
try { try {
const { searchParams } = new URL(request.url); const { searchParams } = new URL(request.url);
const clientId = searchParams.get("clientId") || "all"; const rawClientId = searchParams.get("clientId") || "all";
const clientId =
rawClientId === "all"
? "all"
: SDKManager.normalizeClientId(rawClientId) || "all";
const sanitizedClientId =
clientId === "all" ? "all" : CSVStorageManager.sanitizeClientId(clientId);
const from = searchParams.get("from") || ""; const from = searchParams.get("from") || "";
const to = searchParams.get("to") || ""; const to = searchParams.get("to") || "";
const search = (searchParams.get("search") || "").toLowerCase(); const search = (searchParams.get("search") || "").toLowerCase();
@@ -34,7 +67,12 @@ export async function GET(request: NextRequest) {
const addIfMatching = (event: DataEvent) => { const addIfMatching = (event: DataEvent) => {
const eventDate = CSVStorageManager.extractSafePartitionDate(event.timestamp, null); const eventDate = CSVStorageManager.extractSafePartitionDate(event.timestamp, null);
const dedupeKey = `${event.clientId}:${event.eventId}`; const normEventClient = SDKManager.normalizeClientId(event.clientId) || event.clientId;
const dedupeKey = `${normEventClient}:${event.eventId}`;
const matchesClient =
clientId === "all" ||
normEventClient === clientId ||
CSVStorageManager.sanitizeClientId(normEventClient) === sanitizedClientId;
const matchesSearch = const matchesSearch =
!search || !search ||
event.payload.toLowerCase().includes(search) || event.payload.toLowerCase().includes(search) ||
@@ -42,7 +80,7 @@ export async function GET(request: NextRequest) {
event.clientId.toLowerCase().includes(search); event.clientId.toLowerCase().includes(search);
if ( if (
(clientId !== "all" && event.clientId !== clientId) || !matchesClient ||
((from || to) && (!eventDate || (from && eventDate < from) || (to && eventDate > to))) || ((from || to) && (!eventDate || (from && eventDate < from) || (to && eventDate > to))) ||
!matchesSearch || !matchesSearch ||
eventIds.has(dedupeKey) eventIds.has(dedupeKey)
@@ -57,7 +95,7 @@ export async function GET(request: NextRequest) {
let high = selectedRows.length; let high = selectedRows.length;
while (low < high) { while (low < high) {
const middle = (low + high) >>> 1; const middle = (low + high) >>> 1;
if (selectedRows[middle].timestamp >= event.timestamp) low = middle + 1; if (compareEventsDesc(selectedRows[middle], event) <= 0) low = middle + 1;
else high = middle; else high = middle;
} }
@@ -70,31 +108,61 @@ export async function GET(request: NextRequest) {
const storageDir = CSVStorageManager.baseStorageDir; const storageDir = CSVStorageManager.baseStorageDir;
let clientDirs: string[] = []; let clientDirs: string[] = [];
try { try {
clientDirs = readdirSync(storageDir); clientDirs = await readdir(storageDir);
} catch { } catch {
clientDirs = []; clientDirs = [];
} }
for (const storedClientId of clientDirs) { for (const storedClientId of clientDirs) {
if (clientId !== "all" && storedClientId !== clientId) continue; if (
clientId !== "all" &&
storedClientId !== clientId &&
storedClientId !== sanitizedClientId
) {
continue;
}
const clientDir = path.join(storageDir, storedClientId); const clientDir = path.join(storageDir, storedClientId);
if (!statSync(clientDir).isDirectory()) continue; let dirLstat;
try {
dirLstat = await lstat(clientDir);
} catch {
continue;
}
if (dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
let filenames: string[]; let filenames: string[];
try { try {
filenames = readdirSync(clientDir).filter((filename) => filenames = (await readdir(clientDir))
/^\d{4}-\d{2}-\d{2}(?:-part\d+)?\.csv$/.test(filename), .filter((filename) => {
); if (!/^\d{4}-\d{2}-\d{2}(?:-part\d+)?\.csv$/.test(filename)) {
return false;
}
const fileDate = filename.slice(0, 10);
if ((from && fileDate < from) || (to && fileDate > to)) {
return false;
}
return true;
})
.sort((a, b) => b.localeCompare(a));
} catch { } catch {
continue; continue;
} }
for (const filename of filenames) { for (const filename of filenames) {
const fileDate = filename.slice(0, 10); const fileDate = filename.slice(0, 10);
if ((from && fileDate < from) || (to && fileDate > to)) continue;
const parser = createReadStream(path.join(clientDir, filename)).pipe( const filePath = path.join(clientDir, filename);
let fileLstat;
try {
fileLstat = await lstat(filePath);
} catch {
continue;
}
if (fileLstat.isSymbolicLink() || !fileLstat.isFile()) continue;
const fileStream = createReadStream(filePath);
const parser = fileStream.pipe(
parse({ columns: true, skip_empty_lines: true, bom: true }), parse({ columns: true, skip_empty_lines: true, bom: true }),
); );
@@ -111,17 +179,17 @@ export async function GET(request: NextRequest) {
}; };
if (event.eventId) addIfMatching(event); if (event.eventId) addIfMatching(event);
} }
} catch { } catch (err) {
parser.destroy(); console.warn(`[API /data] Failed reading CSV partition ${filePath}:`, err);
} finally {
if (!parser.destroyed) parser.destroy();
if (!fileStream.destroyed) fileStream.destroy();
} }
} }
} }
const settings = await SettingsManager.getSettingsAsync(); await SettingsManager.getSettingsAsync();
const recentWindowLimit = Math.min( const recentWindowLimit = DataBufferEngine.getRecentLimit();
2000,
Math.max(200, settings.batchFlushThreshold || 500),
);
for (const event of await DataBufferEngine.getRecentEventsAsync( for (const event of await DataBufferEngine.getRecentEventsAsync(
recentWindowLimit, recentWindowLimit,
+295 -40
View File
@@ -1,33 +1,240 @@
import { SettingsManager } from '@/core/adminSettings/settingsManager'; import { SettingsManager } from '@/core/adminSettings/settingsManager';
import { verifyRequestAdminSession } from '@/core/auth/adminUsers'; import { verifyRequestAdminSession } from '@/core/auth/adminUsers';
import { AuthService } from '@/core/auth/jwt'; import { AuthService } from '@/core/auth/jwt';
import { DataBufferEngine } from '@/core/data/buffer'; import { DataBufferEngine, REDIS_BUFFER_KEY } from '@/core/data/buffer';
import { SDKManager } from '@/core/sdk/sdkManager'; import { SDKManager } from '@/core/sdk/sdkManager';
import { import {
SecurityEngine, SecurityEngine,
getInternalIngestionHeaders, getInternalIngestionHeaders,
} from '@/core/Security/security'; } from '@/core/Security/security';
import { CSVStorageManager, type DataEvent } from '@/core/Storage/csvStorage'; import {
CSVStorageManager,
type DataEvent,
type StorageFileMeta,
} from '@/core/Storage/csvStorage';
import { parse } from 'csv-parse'; import { parse } from 'csv-parse';
import { NextRequest, NextResponse } from 'next/server'; import { NextRequest, NextResponse } from 'next/server';
import crypto from 'node:crypto';
import { createReadStream } from 'node:fs'; import { createReadStream } from 'node:fs';
import { requireAdminApiSession } from '@/lib/adminAuth';
import type { BufferedBucket, BufferedSummary } from '@/lib/vconUiHelpers';
const SESSION_COOKIE = 'datastack_admin_session'; const SESSION_COOKIE = 'datastack_admin_session';
const SINGLE_EVENT_ENVELOPE_KEYS = new Set(['eventId', 'timestamp', 'data', 'clientId']);
const MAX_SINGLE_CLIENT_SCAN_FILES = 10;
const MAX_ALL_CLIENTS_PER_CLIENT_FILES = 2;
const MAX_ALL_CLIENTS_TOTAL_SCAN_FILES = 50;
function generateFallbackEventId(): string {
return `evt_${Date.now()}_${crypto.randomBytes(6).toString('hex')}`;
}
function extractSingleHttpEventData(rawBody: Record<string, unknown>): unknown {
if (rawBody.data !== undefined) {
const keys = Object.keys(rawBody);
if (keys.every((k) => SINGLE_EVENT_ENVELOPE_KEYS.has(k))) {
return rawBody.data;
}
return rawBody;
}
const stripped: Record<string, unknown> = {};
let hasNonEnvelopeKey = false;
for (const [k, v] of Object.entries(rawBody)) {
if (k === 'eventId' || k === 'timestamp' || k === 'clientId') {
continue;
}
hasNonEnvelopeKey = true;
stripped[k] = v;
}
return hasNonEnvelopeKey ? stripped : {};
}
function parseTimestampMs(raw: string): number {
const trimmed = String(raw || '').trim();
if (/^\d+(\.\d+)?$/.test(trimmed)) {
const num = Number(trimmed);
return num < 1e11 ? num * 1000 : num;
}
const parsed = Date.parse(trimmed);
return Number.isNaN(parsed) ? Number.NEGATIVE_INFINITY : parsed;
}
function resolveEventUtcDate(rawTimestamp: string): string {
const ms = parseTimestampMs(rawTimestamp);
if (Number.isFinite(ms)) {
const parsed = new Date(ms);
if (!Number.isNaN(parsed.getTime())) {
return parsed.toISOString().slice(0, 10);
}
}
return new Date().toISOString().slice(0, 10);
}
function selectRecentPartitionFiles(
allFiles: StorageFileMeta[],
normClientId?: string,
sanitizedDir?: string,
): StorageFileMeta[] {
const matching = allFiles
.filter(
(f) =>
!normClientId ||
f.clientId === normClientId ||
f.clientId === sanitizedDir,
)
.sort((a, b) => {
if (a.date !== b.date) return b.date.localeCompare(a.date);
return new Date(b.modifiedAt).getTime() - new Date(a.modifiedAt).getTime();
});
if (normClientId) {
return matching.slice(0, MAX_SINGLE_CLIENT_SCAN_FILES);
}
// Multi-client fair scan: take up to 2 newest partitions per active client (capped at 50 files total)
// so clients beyond the 5th most-recently-modified file are never silently hidden.
const perClientCounts = new Map<string, number>();
const selected: StorageFileMeta[] = [];
for (const file of matching) {
const currentCount = perClientCounts.get(file.clientId) ?? 0;
if (currentCount >= MAX_ALL_CLIENTS_PER_CLIENT_FILES) continue;
perClientCounts.set(file.clientId, currentCount + 1);
selected.push(file);
if (selected.length >= MAX_ALL_CLIENTS_TOTAL_SCAN_FILES) break;
}
return selected;
}
async function getPendingBufferedSummary(
ingestionBaseUrl: string,
filterClientId?: string,
): Promise<{ events: DataEvent[]; summary: BufferedSummary }> {
const bufferStats = await DataBufferEngine.getBufferStatsAsync();
const recentLimit = DataBufferEngine.getRecentLimit();
const pending: DataEvent[] = [];
let totalQueued = bufferStats.localBufferSize + bufferStats.redisBufferSize;
// 1. Read exact unflushed Redis queue (when Redis is enabled & connected)
const redis = DataBufferEngine.getRedisClient();
if (redis && bufferStats.redisBufferSize > 0) {
try {
const cap = Math.min(recentLimit, bufferStats.redisBufferSize);
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, cap - 1);
for (const raw of rawItems) {
try {
const parsed = JSON.parse(raw) as DataEvent;
if (!parsed || !parsed.eventId) continue;
pending.push(parsed);
} catch {
// Skip malformed queued item
}
}
} catch {
// Fall back to local buffer telemetry
}
}
// 2. Read Port 3000 exact unflushed RAM queue snapshot
if (bufferStats.localBufferSize > 0) {
const localPending = DataBufferEngine.getLocalBufferSnapshot(recentLimit);
pending.push(...localPending);
}
// 3. When Redis is not enabled or temporarily disconnected, Port 4000 `/internal/recent` serves its unflushed RAM `bufferQueue`
if (!bufferStats.redisEnabled || !bufferStats.redisConnected) {
try {
const qs = new URLSearchParams({ limit: String(recentLimit) });
const res = await fetch(`${ingestionBaseUrl}/internal/recent?${qs.toString()}`, {
headers: getInternalIngestionHeaders(),
cache: 'no-store',
signal: AbortSignal.timeout(1000),
});
if (res.ok) {
const data = await res.json();
if (Array.isArray(data.events) && data.events.length > 0) {
pending.push(...(data.events as DataEvent[]));
totalQueued += data.events.length;
}
}
} catch {
// Port 4000 may not be running in isolated single-process dev mode
}
}
const seen = new Set<string>();
const dedupedAll: DataEvent[] = [];
const filteredEvents: DataEvent[] = [];
const bucketMap = new Map<string, BufferedBucket>();
const sanitizedFilter = filterClientId
? CSVStorageManager.sanitizeClientId(filterClientId)
: undefined;
for (const evt of pending) {
if (!evt || !evt.eventId) continue;
const normEvtClient = SDKManager.normalizeClientId(evt.clientId) || evt.clientId;
const dedupeKey = `${normEvtClient}:${evt.eventId}`;
if (seen.has(dedupeKey)) continue;
seen.add(dedupeKey);
dedupedAll.push(evt);
const matchesFilter =
!filterClientId ||
normEvtClient === filterClientId ||
evt.clientId === filterClientId ||
evt.clientId === sanitizedFilter;
if (matchesFilter) {
filteredEvents.push(evt);
const utcDate = resolveEventUtcDate(evt.timestamp);
const bucketKey = `${normEvtClient}|${utcDate}`;
const existing = bucketMap.get(bucketKey);
if (existing) {
existing.count += 1;
} else {
bucketMap.set(bucketKey, {
clientId: normEvtClient,
date: utcDate,
count: 1,
});
}
}
}
const effectiveTotalQueued = Math.max(totalQueued, dedupedAll.length);
const unattributedCount = Math.max(0, effectiveTotalQueued - dedupedAll.length);
return {
events: filteredEvents,
summary: {
total: filteredEvents.length,
totalQueued: filterClientId
? filteredEvents.length + unattributedCount
: effectiveTotalQueued,
unattributedCount,
truncated: unattributedCount > 0,
buckets: Array.from(bucketMap.values()),
},
};
}
async function getRecentStoredEvents( async function getRecentStoredEvents(
limit: number, limit: number,
clientId?: string, clientId?: string,
): Promise<DataEvent[]> { ): Promise<DataEvent[]> {
const normClientId = clientId && clientId !== 'all' ? SDKManager.normalizeClientId(clientId) : undefined;
const sanitizedDir = normClientId ? CSVStorageManager.sanitizeClientId(normClientId) : undefined;
const allFiles = await CSVStorageManager.listAllFilesAsync(); const allFiles = await CSVStorageManager.listAllFilesAsync();
const files = allFiles const files = selectRecentPartitionFiles(allFiles, normClientId, sanitizedDir);
.filter((f) => !clientId || clientId === 'all' || f.clientId === clientId)
.slice(0, 5);
const collected: DataEvent[] = []; const collected: DataEvent[] = [];
for (const file of files) { for (const file of files) {
const fileTail: DataEvent[] = []; const fileTail: DataEvent[] = [];
const parser = createReadStream(file.filePath).pipe( const fileStream = createReadStream(file.filePath);
const parser = fileStream.pipe(
parse({ columns: true, skip_empty_lines: true, bom: true }), parse({ columns: true, skip_empty_lines: true, bom: true }),
); );
@@ -46,8 +253,11 @@ async function getRecentStoredEvents(
payload: record.payload || '', payload: record.payload || '',
}); });
} }
} catch { } catch (err) {
parser.destroy(); console.warn(`[API /events] Failed reading CSV partition ${file.filePath}:`, err);
} finally {
if (!parser.destroyed) parser.destroy();
if (!fileStream.destroyed) fileStream.destroy();
} }
if (fileTail.length > 0) { if (fileTail.length > 0) {
@@ -59,18 +269,47 @@ async function getRecentStoredEvents(
} }
export async function GET(req: NextRequest) { export async function GET(req: NextRequest) {
const auth = await requireAdminApiSession(req);
if (!auth.valid) {
return auth.response;
}
try { try {
const { searchParams } = new URL(req.url); const { searchParams } = new URL(req.url);
const clientId = searchParams.get('clientId') || undefined; const rawClientId = searchParams.get('clientId');
const clientId =
rawClientId && rawClientId !== 'all'
? SDKManager.normalizeClientId(rawClientId) || undefined
: undefined;
const limit = Math.min(2000, Math.max(1, Number(searchParams.get('limit')) || 50)); const limit = Math.min(2000, Math.max(1, Number(searchParams.get('limit')) || 50));
const bufferedOnly = searchParams.get('bufferedOnly') === 'true';
const includeBufferedSummary =
bufferedOnly || searchParams.get('includeBufferedSummary') === 'true';
await SettingsManager.getSettingsAsync(); await SettingsManager.getSettingsAsync();
const bufferedEvents = await DataBufferEngine.getRecentEventsAsync(limit, clientId);
let socketBufferedEvents: DataEvent[] = [];
const ingestionBaseUrl = ( const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000' process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
).replace(/\/$/, ''); ).replace(/\/$/, '');
if (bufferedOnly) {
const pendingResult = await getPendingBufferedSummary(ingestionBaseUrl, clientId);
const sortedPending = [...pendingResult.events].sort((a, b) => {
const aMs = parseTimestampMs(a.timestamp);
const bMs = parseTimestampMs(b.timestamp);
if (aMs !== bMs) return bMs - aMs;
return b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0;
});
const events = sortedPending.slice(0, limit);
return NextResponse.json({
events,
count: events.length,
bufferedSummary: pendingResult.summary,
});
}
const bufferedEvents = await DataBufferEngine.getRecentEventsAsync(limit, clientId);
let socketBufferedEvents: DataEvent[] = [];
try { try {
const qs = new URLSearchParams({ limit: String(limit) }); const qs = new URLSearchParams({ limit: String(limit) });
if (clientId) qs.set('clientId', clientId); if (clientId) qs.set('clientId', clientId);
@@ -89,23 +328,38 @@ export async function GET(req: NextRequest) {
// Port 4000 may not be running in isolated single-process dev mode // Port 4000 may not be running in isolated single-process dev mode
} }
const storedEvents = await getRecentStoredEvents(limit, clientId); const [storedEvents, pendingResult] = await Promise.all([
getRecentStoredEvents(limit, clientId),
includeBufferedSummary
? getPendingBufferedSummary(ingestionBaseUrl, clientId)
: Promise.resolve(null),
]);
const seenIds = new Set<string>(); const seenIds = new Set<string>();
const merged: DataEvent[] = []; const merged: DataEvent[] = [];
for (const evt of [...bufferedEvents, ...socketBufferedEvents, ...storedEvents]) { for (const evt of [...bufferedEvents, ...socketBufferedEvents, ...storedEvents]) {
if (!evt.eventId) continue; if (!evt.eventId) continue;
const dedupeKey = `${evt.clientId}:${evt.eventId}`; const normEvtClient = SDKManager.normalizeClientId(evt.clientId) || evt.clientId;
const dedupeKey = `${normEvtClient}:${evt.eventId}`;
if (seenIds.has(dedupeKey)) continue; if (seenIds.has(dedupeKey)) continue;
seenIds.add(dedupeKey); seenIds.add(dedupeKey);
merged.push(evt); merged.push(evt);
} }
merged.sort((a, b) => (b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0)); merged.sort((a, b) => {
const aMs = parseTimestampMs(a.timestamp);
const bMs = parseTimestampMs(b.timestamp);
if (aMs !== bMs) return bMs - aMs;
return b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0;
});
const events = merged.slice(0, limit); const events = merged.slice(0, limit);
return NextResponse.json({ events, count: events.length }); return NextResponse.json({
events,
count: events.length,
...(pendingResult ? { bufferedSummary: pendingResult.summary } : {}),
});
} catch (err: unknown) { } catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Events retrieval failed'; const message = err instanceof Error ? err.message : 'Events retrieval failed';
return NextResponse.json({ error: message }, { status: 500 }); return NextResponse.json({ error: message }, { status: 500 });
@@ -165,12 +419,18 @@ export async function POST(req: NextRequest) {
? (body as Record<string, unknown>) ? (body as Record<string, unknown>)
: {}; : {};
const clientId = String( const clientId = SDKManager.normalizeClientId(
tokenClientId || bodyObj.clientId || 'py_telemetry_agent_01', String(tokenClientId || bodyObj.clientId || ''),
) );
.trim() if (!clientId) {
.toLowerCase() return NextResponse.json(
.replace(/[^a-z0-9_-]/g, '_'); {
error: 'clientId is required and must belong to a registered client.',
code: 'VALIDATION_ERROR',
},
{ status: 400 },
);
}
if (await SDKManager.isClientDeletedAsync(clientId)) { if (await SDKManager.isClientDeletedAsync(clientId)) {
return NextResponse.json( return NextResponse.json(
@@ -183,7 +443,7 @@ export async function POST(req: NextRequest) {
} }
const registeredClient = await SDKManager.getClientAsync(clientId); const registeredClient = await SDKManager.getClientAsync(clientId);
if (rawBearerToken && !registeredClient) { if (!registeredClient) {
return NextResponse.json( return NextResponse.json(
{ {
error: `CLIENT_REVOKED: Client '${clientId}' registration not found or has been deleted.`, error: `CLIENT_REVOKED: Client '${clientId}' registration not found or has been deleted.`,
@@ -231,10 +491,11 @@ export async function POST(req: NextRequest) {
? (bodyObj.batch as unknown[]) ? (bodyObj.batch as unknown[])
: Array.isArray(bodyObj.records) : Array.isArray(bodyObj.records)
? (bodyObj.records as unknown[]) ? (bodyObj.records as unknown[])
: bodyObj.data === undefined && Array.isArray(bodyObj.events)
? (bodyObj.events as unknown[])
: null; : null;
if (batchItems !== null) { if (batchItems !== null) {
const envelopeKeys = new Set(['eventId', 'timestamp', 'data', 'clientId']);
let processed = 0; let processed = 0;
let rejected = 0; let rejected = 0;
let duplicates = 0; let duplicates = 0;
@@ -255,19 +516,16 @@ export async function POST(req: NextRequest) {
const itemEventId = const itemEventId =
typeof safeItem.eventId === 'string' && safeItem.eventId.trim() typeof safeItem.eventId === 'string' && safeItem.eventId.trim()
? safeItem.eventId.trim() ? safeItem.eventId.trim()
: `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`; : generateFallbackEventId();
const itemTimestamp = const itemTimestamp =
typeof safeItem.timestamp === 'string' && safeItem.timestamp.trim() typeof safeItem.timestamp === 'string' && safeItem.timestamp.trim()
? safeItem.timestamp.trim() ? safeItem.timestamp.trim()
: new Date().toISOString(); : new Date().toISOString();
const isEnvelope = const itemData = extractSingleHttpEventData(safeItem);
safeItem.data !== undefined &&
Object.keys(safeItem).every((k) => envelopeKeys.has(k));
const itemData = isEnvelope ? safeItem.data : safeItem;
const val = SecurityEngine.validateEvent( const val = await SecurityEngine.validateEventAsync(
clientId, clientId,
itemEventId, itemEventId,
itemData, itemData,
@@ -309,7 +567,7 @@ export async function POST(req: NextRequest) {
} }
if (processed > 0) { if (processed > 0) {
SDKManager.incrementEventCount(clientId, processed); await SDKManager.incrementEventCountAsync(clientId, processed);
} }
// If zero items were processed or deduplicated and all failed due to payload or rate limit, return corresponding status // If zero items were processed or deduplicated and all failed due to payload or rate limit, return corresponding status
@@ -323,6 +581,7 @@ export async function POST(req: NextRequest) {
return NextResponse.json( return NextResponse.json(
{ {
success: false, success: false,
accepted: processed,
processed, processed,
rejected, rejected,
duplicates, duplicates,
@@ -338,6 +597,7 @@ export async function POST(req: NextRequest) {
return NextResponse.json({ return NextResponse.json({
success: true, success: true,
accepted: processed,
processed, processed,
rejected, rejected,
duplicates, duplicates,
@@ -350,24 +610,19 @@ export async function POST(req: NextRequest) {
}); });
} }
// Single Event Ingestion // Single Event Ingestion (exact envelope parity with Port 4000 WS sync_data)
const eventId = const eventId =
typeof bodyObj.eventId === 'string' && bodyObj.eventId.trim() typeof bodyObj.eventId === 'string' && bodyObj.eventId.trim()
? bodyObj.eventId.trim() ? bodyObj.eventId.trim()
: `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`; : generateFallbackEventId();
const timestamp = const timestamp =
typeof bodyObj.timestamp === 'string' && bodyObj.timestamp.trim() typeof bodyObj.timestamp === 'string' && bodyObj.timestamp.trim()
? bodyObj.timestamp.trim() ? bodyObj.timestamp.trim()
: new Date().toISOString(); : new Date().toISOString();
const data = const data = extractSingleHttpEventData(bodyObj);
bodyObj.data !== undefined
? bodyObj.data
: Object.keys(bodyObj).length > 0
? bodyObj
: { sample: 'Realtime event via DataStack API', rand: Math.random() };
// Validate security (idempotency, rate limit, payload size) // Validate security (idempotency, rate limit, payload size)
const val = SecurityEngine.validateEvent( const val = await SecurityEngine.validateEventAsync(
clientId, clientId,
eventId, eventId,
data, data,
@@ -393,7 +648,7 @@ export async function POST(req: NextRequest) {
}; };
await DataBufferEngine.pushAsync(event); await DataBufferEngine.pushAsync(event);
SDKManager.incrementEventCount(clientId, 1); await SDKManager.incrementEventCountAsync(clientId, 1);
return NextResponse.json({ success: true, eventId, event }); return NextResponse.json({ success: true, eventId, event });
} catch (err: unknown) { } catch (err: unknown) {
+127 -22
View File
@@ -1,10 +1,12 @@
import { SettingsManager } from "@/core/adminSettings/settingsManager"; import { SettingsManager } from "@/core/adminSettings/settingsManager";
import { DataBufferEngine } from "@/core/data/buffer"; import { DataBufferEngine } from "@/core/data/buffer";
import { SDKManager } from "@/core/sdk/sdkManager";
import { getInternalIngestionHeaders } from "@/core/Security/security"; import { getInternalIngestionHeaders } from "@/core/Security/security";
import { import {
CSVStorageManager, CSVStorageManager,
type StorageFileMeta, type StorageFileMeta,
} from "@/core/Storage/csvStorage"; } from "@/core/Storage/csvStorage";
import { requireAdminApiSession } from "@/lib/adminAuth";
import { parse } from "csv-parse"; import { parse } from "csv-parse";
import { NextRequest, NextResponse } from "next/server"; import { NextRequest, NextResponse } from "next/server";
import { createReadStream } from "node:fs"; import { createReadStream } from "node:fs";
@@ -44,7 +46,8 @@ function toByteStream(chunks: AsyncIterable<StreamChunk>) {
async function* readFileRecords( async function* readFileRecords(
file: StorageFileMeta, file: StorageFileMeta,
): AsyncGenerator<CsvRecord> { ): AsyncGenerator<CsvRecord> {
const parser = createReadStream(file.filePath).pipe( const fileStream = createReadStream(file.filePath);
const parser = fileStream.pipe(
parse({ columns: true, skip_empty_lines: true, bom: true }), parse({ columns: true, skip_empty_lines: true, bom: true }),
); );
@@ -52,8 +55,9 @@ async function* readFileRecords(
for await (const record of parser as AsyncIterable<CsvRecord>) { for await (const record of parser as AsyncIterable<CsvRecord>) {
yield record; yield record;
} }
} catch { } finally {
parser.destroy(); if (!parser.destroyed) parser.destroy();
if (!fileStream.destroyed) fileStream.destroy();
} }
} }
@@ -85,7 +89,7 @@ function makeExportStream(
yield `${isFirst ? "" : ","}${JSON.stringify({ ...record, payload: parsedPayload })}`; yield `${isFirst ? "" : ","}${JSON.stringify({ ...record, payload: parsedPayload })}`;
} else { } else {
const line = fields const line = fields
.map((field) => `"${(record[field] ?? "").replaceAll('"', '""')}"`) .map((field) => `"${String(record[field] ?? "").replaceAll('"', '""')}"`)
.join(","); .join(",");
yield `${line}\n`; yield `${line}\n`;
} }
@@ -101,8 +105,13 @@ function makeExportStream(
function streamFile(file: StorageFileMeta) { function streamFile(file: StorageFileMeta) {
async function* chunks() { async function* chunks() {
for await (const chunk of createReadStream(file.filePath)) { const fileStream = createReadStream(file.filePath);
yield new Uint8Array(chunk); try {
for await (const chunk of fileStream) {
yield new Uint8Array(chunk as Buffer);
}
} finally {
if (!fileStream.destroyed) fileStream.destroy();
} }
} }
@@ -110,38 +119,122 @@ function streamFile(file: StorageFileMeta) {
} }
export async function GET(request: NextRequest) { export async function GET(request: NextRequest) {
const auth = await requireAdminApiSession(request);
if (!auth.valid) {
return auth.response;
}
try { try {
const { searchParams } = new URL(request.url); const { searchParams } = new URL(request.url);
const from = (searchParams.get("from") || "").trim();
const to = (searchParams.get("to") || "").trim();
if (from && to && from > to) {
return NextResponse.json(
{
error: 'Start date ("from") must be on or before end date ("to").',
code: "INVALID_DATE_RANGE",
},
{ status: 400 },
);
}
const shouldFlush = searchParams.get("flush") !== "false"; const shouldFlush = searchParams.get("flush") !== "false";
await SettingsManager.getSettingsAsync(); await SettingsManager.getSettingsAsync();
if (shouldFlush) { if (shouldFlush) {
await DataBufferEngine.flush(); try {
await DataBufferEngine.flush({ waitMs: 2500, throwOnLockTimeout: true });
} catch (err: unknown) {
const message =
err instanceof Error ? err.message : "Pre-export flush failed";
if (message.includes("FLUSH_LOCK_TIMEOUT")) {
return NextResponse.json(
{
error: message,
code: "FLUSH_LOCK_TIMEOUT",
},
{ status: 503 },
);
}
throw err;
}
const explicitIngestionUrl = Boolean(process.env.INGESTION_INTERNAL_URL?.trim());
const ingestionBaseUrl = ( const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000" process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, ""); ).replace(/\/$/, "");
try { try {
await fetch(`${ingestionBaseUrl}/internal/flush`, { const flushRes = await fetch(`${ingestionBaseUrl}/internal/flush`, {
method: "POST", method: "POST",
headers: getInternalIngestionHeaders(), headers: {
signal: AbortSignal.timeout(1500), "Content-Type": "application/json",
...getInternalIngestionHeaders(),
},
body: JSON.stringify({ waitMs: 2500, throwOnLockTimeout: true }),
cache: "no-store",
signal: AbortSignal.timeout(4000),
}); });
} catch { if (!flushRes.ok) {
const errPayload = await flushRes.json().catch(() => ({}));
const remoteErr =
typeof errPayload?.error === "string"
? errPayload.error
: `Ingestion flush returned HTTP ${flushRes.status}`;
if (
flushRes.status === 503 &&
errPayload?.code === "FLUSH_LOCK_TIMEOUT"
) {
return NextResponse.json(
{
error: remoteErr,
code: "FLUSH_LOCK_TIMEOUT",
ingestionStatus: flushRes.status,
},
{ status: 503 },
);
}
return NextResponse.json(
{
error: remoteErr,
code: "INGESTION_FLUSH_FAILED",
ingestionStatus: flushRes.status,
},
{ status: 502 },
);
}
} catch (err: unknown) {
if (explicitIngestionUrl) {
const errMsg =
err instanceof Error ? err.message : "Ingestion server unreachable";
return NextResponse.json(
{
error: `Ingestion server flush failed before export: ${errMsg}`,
code: "INGESTION_UNREACHABLE",
},
{ status: 502 },
);
}
// Port 4000 may not be running in isolated single-process dev mode // Port 4000 may not be running in isolated single-process dev mode
} }
} }
const clientId = searchParams.get("clientId") || "all"; const rawClientId = searchParams.get("clientId") || "all";
const clientId =
rawClientId === "all"
? "all"
: SDKManager.normalizeClientId(rawClientId) || "all";
const sanitizedClientId =
clientId === "all" ? "all" : CSVStorageManager.sanitizeClientId(clientId);
const filename = searchParams.get("filename"); const filename = searchParams.get("filename");
const from = searchParams.get("from") || ""; const rawFormat = searchParams.get("format");
const to = searchParams.get("to") || ""; const format: ExportFormat = rawFormat === "json" ? "json" : "csv";
const format: ExportFormat =
searchParams.get("format") === "json" ? "json" : "csv";
const includeHeader = searchParams.get("includeHeader") !== "false"; const includeHeader = searchParams.get("includeHeader") !== "false";
let files = await CSVStorageManager.listAllFilesAsync(); let files = await CSVStorageManager.listAllFilesAsync();
if (clientId !== "all") if (clientId !== "all") {
files = files.filter((file) => file.clientId === clientId); files = files.filter(
(file) => file.clientId === clientId || file.clientId === sanitizedClientId,
);
}
if (filename) { if (filename) {
const safeFilename = path.basename(filename); const safeFilename = path.basename(filename);
@@ -166,11 +259,23 @@ export async function GET(request: NextRequest) {
); );
} }
const file = matches[0]; const file = matches[0];
return new NextResponse(streamFile(file), { const downloadName =
format === "json"
? `${file.filename.replace(/\.csv$/i, "")}.json`
: file.filename;
const bodyStream =
format === "json" || !includeHeader
? makeExportStream([file], format, includeHeader)
: streamFile(file);
return new NextResponse(bodyStream, {
status: 200, status: 200,
headers: { headers: {
"Content-Type": "text/csv; charset=utf-8", "Content-Type":
"Content-Disposition": `attachment; filename="${file.filename}"`, format === "json"
? "application/json; charset=utf-8"
: "text/csv; charset=utf-8",
"Content-Disposition": `attachment; filename="${downloadName.replace(/"/g, "")}"`,
"Cache-Control": "no-store",
}, },
}); });
} }
@@ -179,7 +284,7 @@ export async function GET(request: NextRequest) {
(file) => (!from || file.date >= from) && (!to || file.date <= to), (file) => (!from || file.date >= from) && (!to || file.date <= to),
); );
const datePart = new Date().toISOString().slice(0, 10); const datePart = new Date().toISOString().slice(0, 10);
const safeClientId = clientId.replace(/[^a-zA-Z0-9_-]/g, "_"); const safeClientId = sanitizedClientId === "all" ? "all" : sanitizedClientId;
const baseName = const baseName =
clientId === "all" ? "datastack_export" : `${safeClientId}_export`; clientId === "all" ? "datastack_export" : `${safeClientId}_export`;
+14 -6
View File
@@ -23,10 +23,10 @@ export async function GET(req: NextRequest) {
); );
} }
const clientId = String(verified.payload.clientId || '') const clientId = SDKManager.normalizeClientId(
.trim() String(verified.payload.clientId || ''),
.toLowerCase(); );
if (SDKManager.isClientDeleted(clientId)) { if (await SDKManager.isClientDeletedAsync(clientId)) {
return NextResponse.json( return NextResponse.json(
{ {
status: 'error', status: 'error',
@@ -37,7 +37,16 @@ export async function GET(req: NextRequest) {
); );
} }
const registeredClient = await SDKManager.getClientAsync(clientId); const registeredClient = await SDKManager.getClientAsync(clientId);
if (registeredClient) { if (!registeredClient) {
return NextResponse.json(
{
status: 'error',
code: 'CLIENT_REVOKED',
error: `CLIENT_REVOKED: Client '${clientId}' registration not found or has been deleted.`,
},
{ status: 403 },
);
}
if (registeredClient.status === 'revoked') { if (registeredClient.status === 'revoked') {
return NextResponse.json( return NextResponse.json(
{ {
@@ -59,7 +68,6 @@ export async function GET(req: NextRequest) {
); );
} }
} }
}
const bufferStats = await DataBufferEngine.getBufferStatsAsync(); const bufferStats = await DataBufferEngine.getBufferStatsAsync();
+8 -2
View File
@@ -1,7 +1,13 @@
import { NextResponse } from 'next/server'; import { NextRequest, NextResponse } from 'next/server';
import { ApiEngine } from '@/core/apiEngine/apiService'; import { ApiEngine } from '@/core/apiEngine/apiService';
import { requireAdminApiSession } from '@/lib/adminAuth';
export async function GET(req: NextRequest) {
const auth = await requireAdminApiSession(req);
if (!auth.valid) {
return auth.response;
}
export async function GET() {
try { try {
const metrics = await ApiEngine.getDashboardMetrics(); const metrics = await ApiEngine.getDashboardMetrics();
return NextResponse.json(metrics); return NextResponse.json(metrics);
+15
View File
@@ -188,9 +188,24 @@ export async function POST(req: NextRequest) {
await SettingsManager.updateSettingsAsync(validated); await SettingsManager.updateSettingsAsync(validated);
const settings = await SettingsManager.getPublicSettingsAsync(); const settings = await SettingsManager.getPublicSettingsAsync();
const bridgeSync = SettingsManager.getLastBridgeSyncStatus();
return NextResponse.json({ return NextResponse.json({
settings, settings,
success: true, success: true,
...(bridgeSync
? {
bridgeSync: {
attempted: bridgeSync.attempted ?? true,
ok: bridgeSync.ok ?? bridgeSync.synced,
synced: bridgeSync.synced,
...(bridgeSync.status !== undefined
? { status: bridgeSync.status }
: {}),
...(bridgeSync.error ? { error: bridgeSync.error } : {}),
checkedAt: bridgeSync.checkedAt,
},
}
: {}),
}); });
} catch (err: unknown) { } catch (err: unknown) {
const message = const message =
+70 -11
View File
@@ -1,24 +1,44 @@
import { NextRequest, NextResponse } from 'next/server'; import { NextRequest, NextResponse } from 'next/server';
import { SettingsManager } from '@/core/adminSettings/settingsManager'; import { SettingsManager } from '@/core/adminSettings/settingsManager';
import { SDKManager } from '@/core/sdk/sdkManager';
import { getInternalIngestionHeaders } from '@/core/Security/security'; import { getInternalIngestionHeaders } from '@/core/Security/security';
import { CSVStorageManager } from '@/core/Storage/csvStorage'; import { CSVStorageManager } from '@/core/Storage/csvStorage';
import { DataBufferEngine } from '@/core/data/buffer'; import { DataBufferEngine } from '@/core/data/buffer';
import { requireAdminApiSession } from '@/lib/adminAuth';
export async function GET(req: NextRequest) { export async function GET(req: NextRequest) {
const auth = await requireAdminApiSession(req);
if (!auth.valid) {
return auth.response;
}
try { try {
const { searchParams } = new URL(req.url); const { searchParams } = new URL(req.url);
const clientId = searchParams.get('clientId') || undefined; const rawClientId = searchParams.get('clientId');
const clientId =
rawClientId && rawClientId !== 'all'
? SDKManager.normalizeClientId(rawClientId)
: undefined;
const sanitizedClientId = clientId
? CSVStorageManager.sanitizeClientId(clientId)
: undefined;
const [stats, allFiles] = await Promise.all([ const allFiles = await CSVStorageManager.listAllFilesAsync();
CSVStorageManager.getStorageStatsAsync(),
CSVStorageManager.listAllFilesAsync(),
]);
let files = allFiles; let files = allFiles;
if (clientId && clientId !== 'all') { if (clientId) {
files = files.filter((f) => f.clientId === clientId); files = allFiles.filter((f) => {
const normFolder = SDKManager.normalizeClientId(f.clientId);
return (
normFolder === clientId ||
f.clientId === clientId ||
f.clientId === sanitizedClientId
);
});
} }
const stats = CSVStorageManager.formatStorageStats(files);
return NextResponse.json({ stats, files }); return NextResponse.json({ stats, files });
} catch (err: unknown) { } catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Storage stats failed'; const message = err instanceof Error ? err.message : 'Storage stats failed';
@@ -26,12 +46,20 @@ export async function GET(req: NextRequest) {
} }
} }
export async function POST(_req: NextRequest) { export async function POST(req: NextRequest) {
const auth = await requireAdminApiSession(req);
if (!auth.valid) {
return auth.response;
}
try { try {
await SettingsManager.getSettingsAsync(); await SettingsManager.getSettingsAsync();
// 1. Flush local / shared Next.js DataBufferEngine // 1. Flush local / shared Next.js DataBufferEngine
const localRes = await DataBufferEngine.flush(); const localRes = await DataBufferEngine.flush({
waitMs: 2500,
throwOnLockTimeout: true,
});
// 2. Also flush Port 4000 Socket.IO ingestion buffer // 2. Also flush Port 4000 Socket.IO ingestion buffer
let remoteCount = 0; let remoteCount = 0;
@@ -46,9 +74,13 @@ export async function POST(_req: NextRequest) {
try { try {
const ingestRes = await fetch(`${ingestionBaseUrl}/internal/flush`, { const ingestRes = await fetch(`${ingestionBaseUrl}/internal/flush`, {
method: 'POST', method: 'POST',
headers: getInternalIngestionHeaders(), headers: {
'Content-Type': 'application/json',
...getInternalIngestionHeaders(),
},
body: JSON.stringify({ waitMs: 2500, throwOnLockTimeout: true }),
cache: 'no-store', cache: 'no-store',
signal: AbortSignal.timeout(2500), signal: AbortSignal.timeout(4000),
}); });
if (!ingestRes.ok) { if (!ingestRes.ok) {
const errPayload = await ingestRes.json().catch(() => ({})); const errPayload = await ingestRes.json().catch(() => ({}));
@@ -56,6 +88,23 @@ export async function POST(_req: NextRequest) {
typeof errPayload?.error === 'string' typeof errPayload?.error === 'string'
? errPayload.error ? errPayload.error
: `Ingestion flush returned HTTP ${ingestRes.status}`; : `Ingestion flush returned HTTP ${ingestRes.status}`;
if (
ingestRes.status === 503 &&
errPayload?.code === 'FLUSH_LOCK_TIMEOUT'
) {
return NextResponse.json(
{
success: false,
error: remoteErr,
code: 'FLUSH_LOCK_TIMEOUT',
flushedCount: localRes.count,
filesUpdated: localRes.files,
ingestionFlushed: false,
ingestionStatus: ingestRes.status,
},
{ status: 503 },
);
}
return NextResponse.json( return NextResponse.json(
{ {
success: false, success: false,
@@ -108,6 +157,16 @@ export async function POST(_req: NextRequest) {
}); });
} catch (err: unknown) { } catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Manual flush failed'; const message = err instanceof Error ? err.message : 'Manual flush failed';
if (message.includes('FLUSH_LOCK_TIMEOUT')) {
return NextResponse.json(
{
success: false,
error: message,
code: 'FLUSH_LOCK_TIMEOUT',
},
{ status: 503 },
);
}
return NextResponse.json({ error: message }, { status: 500 }); return NextResponse.json({ error: message }, { status: 500 });
} }
} }
+6 -1
View File
@@ -38,9 +38,14 @@ export const metadata: Metadata = {
}, },
}; };
const THEME_BOOTSTRAP_SCRIPT = `(function(){try{var t=localStorage.getItem('datastack-theme');var r=document.documentElement;if(t==='light'){r.classList.remove('dark');r.classList.add('light');}else{r.classList.remove('light');r.classList.add('dark');}}catch(e){}})();`;
export default function RootLayout({ children }: { children: React.ReactNode }) { export default function RootLayout({ children }: { children: React.ReactNode }) {
return ( return (
<html lang="en" className="dark"> <html lang="en" className="dark" suppressHydrationWarning>
<head>
<script dangerouslySetInnerHTML={{ __html: THEME_BOOTSTRAP_SCRIPT }} />
</head>
<body suppressHydrationWarning className="min-h-screen bg-[var(--bg-main)] text-[var(--text)] antialiased"> <body suppressHydrationWarning className="min-h-screen bg-[var(--bg-main)] text-[var(--text)] antialiased">
<ToastProvider> <ToastProvider>
{children} {children}
+46 -8
View File
@@ -1,12 +1,50 @@
'use client'; import { VconShell } from '@/components/layout/VconShell';
import { verifyRequestAdminSession } from '@/core/auth/adminUsers';
import { MainLayout } from '@/components/layout/MainLayout'; import { cookies, headers } from 'next/headers';
import { usePathname } from 'next/navigation'; import { redirect } from 'next/navigation';
import React from 'react'; import React from 'react';
export default function VconLayout({ children }: { children: React.ReactNode }) { export const dynamic = 'force-dynamic';
const pathname = usePathname();
if (pathname === '/vcon/login' || pathname === '/vcon/setup') return <>{children}</>;
return <MainLayout showSidebar={true}>{children}</MainLayout>; export default async function VconLayout({
children,
}: {
children: React.ReactNode;
}) {
const headerList = await headers();
const pathname = headerList.get('x-datastack-pathname') || '';
if (pathname === '/vcon/login' || pathname === '/vcon/setup') {
return <>{children}</>;
}
const cookieStore = await cookies();
const token = cookieStore.get('datastack_admin_session')?.value;
const auth = await verifyRequestAdminSession(token);
if (!auth.valid) {
if (auth.reason === 'UNAUTHORIZED') {
const loginTarget = pathname
? `/vcon/login?next=${encodeURIComponent(pathname)}`
: '/vcon/login';
redirect(loginTarget);
}
return (
<main className="grid min-h-dvh place-items-center bg-[var(--bg)] p-4 text-[var(--text)]">
<section className="w-full max-w-sm border border-[var(--border)] bg-[var(--surface)] p-5">
<h1 className="text-[13px] font-medium text-[var(--text)]">
Console authentication unavailable
</h1>
<p className="mt-2 text-[12px] text-[var(--text-muted)]">
{auth.reason === 'UNCONFIGURED'
? 'Admin session secret is not configured.'
: 'Database authentication check is temporarily unavailable. Please retry shortly.'}
</p>
</section>
</main>
);
}
return <VconShell>{children}</VconShell>;
} }
+31 -3
View File
@@ -1,18 +1,46 @@
import { AdminLogin } from '@/components/sections/AdminLogin'; import { AdminLogin } from '@/components/sections/AdminLogin';
import { isAdminSetupComplete } from '@/core/auth/adminUsers'; import {
isAdminSetupComplete,
verifyRequestAdminSession,
} from '@/core/auth/adminUsers';
import { cookies } from 'next/headers';
import { redirect } from 'next/navigation'; import { redirect } from 'next/navigation';
export const runtime = 'nodejs'; export const runtime = 'nodejs';
export const dynamic = 'force-dynamic'; export const dynamic = 'force-dynamic';
export default async function LoginPage() { export default async function LoginPage() {
const isConfigured = await isAdminSetupComplete().catch(() => false); let databaseReady = true;
try {
const isConfigured = await isAdminSetupComplete();
if (!isConfigured) redirect('/vcon/setup'); if (!isConfigured) redirect('/vcon/setup');
} catch (error) {
if (
error &&
typeof error === 'object' &&
'digest' in error &&
String((error as { digest?: unknown }).digest).startsWith('NEXT_REDIRECT')
) {
throw error;
}
databaseReady = false;
}
if (databaseReady) {
const cookieStore = await cookies();
const token = cookieStore.get('datastack_admin_session')?.value;
if (token) {
const auth = await verifyRequestAdminSession(token);
if (auth.valid) {
redirect('/vcon/dashboard');
}
}
}
const showDemoHint = const showDemoHint =
!process.env.DATABASE_URL && !process.env.DATABASE_URL &&
process.env.NODE_ENV !== 'production' && process.env.NODE_ENV !== 'production' &&
process.env.DATASTACK_ALLOW_DEMO_ADMIN !== 'false'; process.env.DATASTACK_ALLOW_DEMO_ADMIN !== 'false';
return <AdminLogin showDemoHint={showDemoHint} />; return <AdminLogin showDemoHint={showDemoHint} databaseReady={databaseReady} />;
} }
+53 -20
View File
@@ -9,6 +9,7 @@ import { Panel } from '@/components/ui/Panel';
import { StatStrip } from '@/components/ui/StatStrip'; import { StatStrip } from '@/components/ui/StatStrip';
import { useToast } from '@/components/ui/Toast'; import { useToast } from '@/components/ui/Toast';
import { getCached, setCached } from '@/lib/vconCache'; import { getCached, setCached } from '@/lib/vconCache';
import { resolvePublicEndpoints } from '@/lib/vconUiHelpers';
import { useEffect, useState } from 'react'; import { useEffect, useState } from 'react';
interface SdkClientItem { interface SdkClientItem {
@@ -34,23 +35,23 @@ export default function SdkGuidePage() {
const [apiUrl, setApiUrl] = useState<string>('http://localhost:3000'); const [apiUrl, setApiUrl] = useState<string>('http://localhost:3000');
useEffect(() => { useEffect(() => {
const isLocalDev = window.location.port === '3000'; const initialPort = getCached<number>('vcon:server-port', 4000);
setServerUrl( const initialEndpoints = resolvePublicEndpoints(window.location, initialPort);
isLocalDev setServerUrl(initialEndpoints.serverUrl);
? `${window.location.protocol}//${window.location.hostname}:4000` setApiUrl(initialEndpoints.apiUrl);
: window.location.origin
);
setApiUrl(window.location.origin);
async function loadClients() { async function loadSdkContext() {
try { try {
const res = await fetch('/api/clients', { cache: 'no-store' }); const [clientsRes, metricsRes] = await Promise.all([
if (res.status === 401) { fetch('/api/clients', { cache: 'no-store' }),
fetch('/api/metrics', { cache: 'no-store' }).catch(() => null),
]);
if (clientsRes.status === 401 || metricsRes?.status === 401) {
window.location.assign('/vcon/login'); window.location.assign('/vcon/login');
return; return;
} }
if (res.ok) { if (clientsRes.ok) {
const data = await res.json(); const data = await clientsRes.json();
const list = setCached<SdkClientItem[]>('vcon:clients', data.clients || []); const list = setCached<SdkClientItem[]>('vcon:clients', data.clients || []);
setClients(list); setClients(list);
if (list.length > 0) { if (list.length > 0) {
@@ -61,11 +62,21 @@ export default function SdkGuidePage() {
); );
} }
} }
if (metricsRes && metricsRes.ok) {
const metricsData = await metricsRes.json().catch(() => null);
const runtimePort = Number(metricsData?.runtime?.serverPort);
if (Number.isFinite(runtimePort) && runtimePort > 0) {
setCached('vcon:server-port', runtimePort);
const resolved = resolvePublicEndpoints(window.location, runtimePort);
setServerUrl(resolved.serverUrl);
setApiUrl(resolved.apiUrl);
}
}
} catch { } catch {
// Retain cached list on transient network error // Retain cached list on transient network error
} }
} }
void loadClients(); void loadSdkContext();
}, []); }, []);
useEffect(() => { useEffect(() => {
@@ -75,6 +86,7 @@ export default function SdkGuidePage() {
} }
let cancelled = false; let cancelled = false;
setSelectedClientToken('');
setLoadingToken(true); setLoadingToken(true);
async function fetchToken() { async function fetchToken() {
@@ -110,8 +122,12 @@ export default function SdkGuidePage() {
const activeClient = const activeClient =
clients.find((c) => c.clientId === selectedClientId) || clients[0]; clients.find((c) => c.clientId === selectedClientId) || clients[0];
const resolvedToken = selectedClientToken || activeClient?.token || ''; const resolvedToken = loadingToken
const sampleToken = resolvedToken || 'YOUR_SIGNED_CLIENT_JWT_TOKEN'; ? ''
: selectedClientToken || activeClient?.token || '';
const sampleToken = loadingToken
? 'LOADING_CLIENT_JWT_TOKEN...'
: resolvedToken || 'YOUR_SIGNED_CLIENT_JWT_TOKEN';
const configSnippet = `# config.datastack.py — Place in your Python project root const configSnippet = `# config.datastack.py — Place in your Python project root
DATASTACK_CONFIG = { DATASTACK_CONFIG = {
@@ -194,6 +210,7 @@ client.disconnect()`;
size="xs" size="xs"
variant="secondary" variant="secondary"
icon="FileCode" icon="FileCode"
disabled={loadingToken}
onClick={() => void copySnippet(configSnippet, 'config.datastack.py')} onClick={() => void copySnippet(configSnippet, 'config.datastack.py')}
> >
Copy Config Copy Config
@@ -216,17 +233,23 @@ client.disconnect()`;
{ label: 'Server endpoint', value: serverUrl }, { label: 'Server endpoint', value: serverUrl },
{ {
label: 'Active client', label: 'Active client',
value: activeClient?.clientId || 'None selected', value: activeClient
? activeClient.status === 'revoked'
? `${activeClient.clientId} (revoked)`
: activeClient.clientId
: 'None selected',
dot: activeClient?.status === 'revoked' ? 'off' : activeClient ? 'ok' : undefined,
}, },
]} ]}
/> />
<Card <Card
compact compact
className="flex w-full items-center justify-between bg-[var(--surface)] px-3 py-2 font-mono text-[11.5px]" className="flex w-full flex-col gap-1.5 bg-[var(--surface)] px-3 py-2.5"
> >
<span className="text-[var(--accent)]"> <div className="flex items-center justify-between gap-2 font-mono text-[11.5px]">
pip install ./sdk/python/datastack <span className="truncate text-[var(--accent)]">
pip install /path/to/datastack/sdk/python/datastack
</span> </span>
<Button <Button
size="xs" size="xs"
@@ -234,9 +257,18 @@ client.disconnect()`;
icon="Copy" icon="Copy"
aria-label="Copy pip install command" aria-label="Copy pip install command"
onClick={() => onClick={() =>
void copySnippet('pip install ./sdk/python/datastack', 'Pip command') void copySnippet(
'pip install /path/to/datastack/sdk/python/datastack',
'Pip command'
)
} }
/> />
</div>
<p className="text-[11px] text-[var(--text-faint)]">
Self-hosted package bundled in <code className="font-mono text-[var(--text-muted)]">sdk/python/datastack</code>. From inside the repo run{' '}
<code className="font-mono text-[var(--text-muted)]">pip install ./sdk/python/datastack</code>, or build a portable wheel with{' '}
<code className="font-mono text-[var(--text-muted)]">pip wheel ./sdk/python/datastack -w dist/</code> for external projects.
</p>
</Card> </Card>
<Panel <Panel
@@ -265,6 +297,7 @@ client.disconnect()`;
variant="ghost" variant="ghost"
icon="Copy" icon="Copy"
aria-label="Copy config.datastack.py" aria-label="Copy config.datastack.py"
disabled={loadingToken}
onClick={() => void copySnippet(configSnippet, 'config.datastack.py')} onClick={() => void copySnippet(configSnippet, 'config.datastack.py')}
/> />
</div> </div>
+20 -2
View File
@@ -1,5 +1,8 @@
import { AdminSetup } from '@/components/sections/AdminSetup'; import { AdminSetup } from '@/components/sections/AdminSetup';
import { isAdminSetupComplete } from '@/core/auth/adminUsers'; import {
isAdminSetupComplete,
requiresPersistentAdminDatabase,
} from '@/core/auth/adminUsers';
import { redirect } from 'next/navigation'; import { redirect } from 'next/navigation';
export const runtime = 'nodejs'; export const runtime = 'nodejs';
@@ -7,15 +10,30 @@ export const dynamic = 'force-dynamic';
export default async function SetupPage() { export default async function SetupPage() {
let databaseReady = true; let databaseReady = true;
let databaseError = '';
let setupComplete = false;
if (requiresPersistentAdminDatabase()) {
databaseReady = false;
databaseError =
'PostgreSQL DATABASE_URL is required in production before creating an administrator account.';
} else {
try { try {
if (await isAdminSetupComplete()) redirect('/vcon/login'); setupComplete = await isAdminSetupComplete();
} catch { } catch {
databaseReady = false; databaseReady = false;
databaseError = 'Database connection unavailable.';
}
}
if (databaseReady && setupComplete) {
redirect('/vcon/login');
} }
return ( return (
<AdminSetup <AdminSetup
databaseReady={databaseReady} databaseReady={databaseReady}
databaseError={databaseError}
initialEmail={process.env.DATASTACK_ADMIN_EMAIL?.trim().toLowerCase() ?? ''} initialEmail={process.env.DATASTACK_ADMIN_EMAIL?.trim().toLowerCase() ?? ''}
/> />
); );
+52 -46
View File
@@ -9,6 +9,10 @@ import { Pill } from '@/components/ui/Pill';
import { StatStrip } from '@/components/ui/StatStrip'; import { StatStrip } from '@/components/ui/StatStrip';
import { useToast } from '@/components/ui/Toast'; import { useToast } from '@/components/ui/Toast';
import { getCached, setCached } from '@/lib/vconCache'; import { getCached, setCached } from '@/lib/vconCache';
import {
resolvePublicEndpoints,
summarizeHttpBatchResult,
} from '@/lib/vconUiHelpers';
import Link from 'next/link'; import Link from 'next/link';
import { useEffect, useState } from 'react'; import { useEffect, useState } from 'react';
import { io } from 'socket.io-client'; import { io } from 'socket.io-client';
@@ -22,11 +26,8 @@ interface SimulatorClient {
type TransportMode = 'http' | 'websocket'; type TransportMode = 'http' | 'websocket';
type DispatchStatus = 'Ready' | 'Accepted' | 'Rejected' | 'Error'; type DispatchStatus = 'Ready' | 'Accepted' | 'Rejected' | 'Error';
function getSocketOrigin(): string { function getSocketOrigin(serverPort: number): string {
const isLocalDev = window.location.port === '3000'; return resolvePublicEndpoints(window.location, serverPort).serverUrl;
const protocol = window.location.protocol === 'https:' ? 'https:' : 'http:';
const host = isLocalDev ? `${window.location.hostname}:4000` : window.location.host;
return `${protocol}//${host}`;
} }
export default function SimulatorPage() { export default function SimulatorPage() {
@@ -52,20 +53,26 @@ export default function SimulatorPage() {
2 2
) )
); );
const [serverPort, setServerPort] = useState<number>(() =>
getCached<number>('vcon:server-port', 4000)
);
const [sending, setSending] = useState(false); const [sending, setSending] = useState(false);
const [dispatchStatus, setDispatchStatus] = useState<DispatchStatus>('Ready'); const [dispatchStatus, setDispatchStatus] = useState<DispatchStatus>('Ready');
const [lastResponse, setLastResponse] = useState<Record<string, unknown> | null>(null); const [lastResponse, setLastResponse] = useState<Record<string, unknown> | null>(null);
useEffect(() => { useEffect(() => {
async function loadClients() { async function loadSimulatorContext() {
try { try {
const res = await fetch('/api/clients', { cache: 'no-store' }); const [clientsRes, metricsRes] = await Promise.all([
if (res.status === 401) { fetch('/api/clients', { cache: 'no-store' }),
fetch('/api/metrics', { cache: 'no-store' }).catch(() => null),
]);
if (clientsRes.status === 401 || metricsRes?.status === 401) {
window.location.assign('/vcon/login'); window.location.assign('/vcon/login');
return; return;
} }
if (res.ok) { if (clientsRes.ok) {
const data = await res.json(); const data = await clientsRes.json();
const list = setCached<SimulatorClient[]>('vcon:clients', data.clients || []); const list = setCached<SimulatorClient[]>('vcon:clients', data.clients || []);
setClients(list); setClients(list);
setSelectedClient((prev) => { setSelectedClient((prev) => {
@@ -73,11 +80,18 @@ export default function SimulatorPage() {
return list.find((c) => c.status === 'active')?.clientId ?? list[0]?.clientId ?? ''; return list.find((c) => c.status === 'active')?.clientId ?? list[0]?.clientId ?? '';
}); });
} }
if (metricsRes && metricsRes.ok) {
const metricsData = await metricsRes.json().catch(() => null);
const runtimePort = Number(metricsData?.runtime?.serverPort);
if (Number.isFinite(runtimePort) && runtimePort > 0) {
setServerPort(setCached('vcon:server-port', runtimePort));
}
}
} catch { } catch {
// Ignore transient failure and retain cached list // Ignore transient failure and retain cached list
} }
} }
void loadClients(); void loadSimulatorContext();
}, []); }, []);
const fetchClientJwt = async (clientId: string): Promise<string> => { const fetchClientJwt = async (clientId: string): Promise<string> => {
@@ -100,7 +114,7 @@ export default function SimulatorPage() {
items: Array<Record<string, unknown>> items: Array<Record<string, unknown>>
): Promise<{ accepted: number; rejected: number; responses: Record<string, unknown>[] }> => { ): Promise<{ accepted: number; rejected: number; responses: Record<string, unknown>[] }> => {
const token = await fetchClientJwt(clientId); const token = await fetchClientJwt(clientId);
const socketUrl = getSocketOrigin(); const socketUrl = getSocketOrigin(serverPort);
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const socket = io(socketUrl, { const socket = io(socketUrl, {
@@ -112,7 +126,7 @@ export default function SimulatorPage() {
const timer = window.setTimeout(() => { const timer = window.setTimeout(() => {
socket.disconnect(); socket.disconnect();
reject(new Error('WebSocket timeout (is Port 4000 ingestion server running?)')); reject(new Error(`WebSocket timeout (is Port ${serverPort} ingestion server running?)`));
}, 5000); }, 5000);
socket.on('connect_error', (connectErr) => { socket.on('connect_error', (connectErr) => {
@@ -271,55 +285,47 @@ export default function SimulatorPage() {
return; return;
} }
let accepted = 0; const batchPayload = items.map((item, idx) => ({
let rejected = 0; eventId: `sim_http_${Date.now()}_${idx}`,
const errors: string[] = []; timestamp: item.timestamp,
data: item,
for (const item of items) { }));
const res = await fetch('/api/events', { const res = await fetch('/api/events', {
method: 'POST', method: 'POST',
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ body: JSON.stringify({
clientId: selectedClient, clientId: selectedClient,
data: item, batch: batchPayload,
}), }),
}); });
if (res.status === 401) { if (res.status === 401) {
window.location.assign('/vcon/login'); window.location.assign('/vcon/login');
return; return;
} }
if (res.ok) {
accepted++;
} else {
rejected++;
try {
const errJson = await res.json();
if (errJson?.error && errors.length < 3) {
errors.push(String(errJson.error));
}
} catch {
// ignore non-json error body
}
}
}
const data = await res.json().catch(() => ({}));
const summary = summarizeHttpBatchResult(
res.ok,
data && typeof data === 'object' ? (data as Record<string, unknown>) : null,
count,
);
setLastResponse({ setLastResponse({
transport: 'http', transport: 'http',
status: res.status,
batchSent: count, batchSent: count,
accepted, accepted: summary.accepted,
rejected, rejected: summary.rejected,
...(errors.length > 0 ? { errors } : {}), duplicates: summary.duplicates,
...(data && typeof data === 'object' ? data : {}),
}); });
if (accepted === 0) { setDispatchStatus(summary.status);
setDispatchStatus('Rejected'); if (summary.level === 'error') {
error('Batch Rejected', `0/${count} ingested${errors[0] ? ` (${errors[0]})` : ''}`); error(summary.title, summary.message);
} else if (accepted < count) { } else if (summary.level === 'warning') {
setDispatchStatus('Rejected'); warning(summary.title, summary.message);
warning('Batch Partial', `${accepted}/${count} ingested (${rejected} rejected)`);
} else { } else {
setDispatchStatus('Accepted'); success(summary.title, summary.message);
success('Batch Streamed', `${accepted}/${count} ingested`);
} }
} catch (err: unknown) { } catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Batch dispatch error'; const message = err instanceof Error ? err.message : 'Batch dispatch error';
@@ -379,7 +385,7 @@ export default function SimulatorPage() {
}, },
{ {
label: 'Ingest transport', label: 'Ingest transport',
value: transport === 'http' ? 'POST /api/events' : 'WS Port 4000', value: transport === 'http' ? 'POST /api/events' : `WS Port ${serverPort}`,
}, },
{ label: 'Payload format', value: 'JSON' }, { label: 'Payload format', value: 'JSON' },
{ {
@@ -412,7 +418,7 @@ export default function SimulatorPage() {
active={transport === 'websocket'} active={transport === 'websocket'}
onClick={() => setTransport('websocket')} onClick={() => setTransport('websocket')}
> >
WebSocket Port 4000 WebSocket Port {serverPort}
</Pill> </Pill>
</div> </div>
</div> </div>
+68 -21
View File
@@ -18,52 +18,96 @@ import { useToast } from '@/components/ui/Toast';
import { getCached, setCached } from '@/lib/vconCache'; import { getCached, setCached } from '@/lib/vconCache';
import { useCallback, useEffect, useState } from 'react'; import { useCallback, useEffect, useState } from 'react';
interface StorageStatsSummary {
totalMB?: string;
totalFiles?: number;
totalRows?: number;
rotationRule?: string;
rotationMb?: number;
}
interface StoragePartitionFile {
clientId: string;
filename: string;
sizeBytes: number;
linesCount: number;
modifiedAt: string;
}
export default function StoragePage() { export default function StoragePage() {
const { success, warning, error } = useToast(); const { success, warning, error } = useToast();
const [stats, setStats] = useState<any>(() => const [stats, setStats] = useState<StorageStatsSummary | null>(() =>
getCached<any>('vcon:storage-stats', null) getCached<StorageStatsSummary | null>('vcon:storage-stats', null)
); );
const [files, setFiles] = useState<any[]>(() => const [files, setFiles] = useState<StoragePartitionFile[]>(() =>
getCached<any[]>('vcon:storage-files', []) getCached<StoragePartitionFile[]>('vcon:storage-files', [])
); );
const [refreshing, setRefreshing] = useState(false);
const [flushing, setFlushing] = useState(false); const [flushing, setFlushing] = useState(false);
const fetchStorageData = useCallback(async () => { const fetchStorageData = useCallback(async () => {
setRefreshing(true);
try { try {
const res = await fetch('/api/storage'); const res = await fetch('/api/storage', { cache: 'no-store' });
if (!res.ok) throw new Error('Unable to load storage partitions'); if (res.status === 401) {
const data = await res.json(); window.location.assign('/vcon/login');
setStats(setCached('vcon:storage-stats', data.stats)); return;
}
const data = await res.json().catch(() => ({}));
if (!res.ok) {
throw new Error(data.error || 'Unable to load storage partitions');
}
setStats(setCached('vcon:storage-stats', data.stats ?? null));
setFiles(setCached('vcon:storage-files', data.files || [])); setFiles(setCached('vcon:storage-files', data.files || []));
} catch { } catch (loadErr) {
error('Storage load failed', 'Could not load storage partitions.'); error(
'Storage load failed',
loadErr instanceof Error
? loadErr.message
: 'Could not load storage partitions.'
);
} finally {
setRefreshing(false);
} }
}, [error]); }, [error]);
useEffect(() => { useEffect(() => {
fetchStorageData(); void fetchStorageData();
}, [fetchStorageData]); }, [fetchStorageData]);
const handleManualFlush = async () => { const handleManualFlush = async () => {
setFlushing(true); setFlushing(true);
try { try {
const res = await fetch('/api/storage', { method: 'POST' }); const res = await fetch('/api/storage', {
method: 'POST',
cache: 'no-store',
});
if (res.status === 401) {
window.location.assign('/vcon/login');
return;
}
const data = await res.json().catch(() => ({})); const data = await res.json().catch(() => ({}));
if (res.ok && data.success) { if (res.ok && data.success) {
const count = Number(data.flushedCount) || 0; const count = Number(data.flushedCount) || 0;
if (data.warning) {
warning(
'Partial Flush (Web Only)',
`Flushed ${count} event${count === 1 ? '' : 's'}. ${String(data.warning)}`
);
} else {
success( success(
'Buffer Flushed', 'Buffer Flushed',
`Flushed ${count} event${count === 1 ? '' : 's'} to CSV${data.warning ? ` (${data.warning})` : ''}` `Flushed ${count} event${count === 1 ? '' : 's'} to CSV.`
); );
fetchStorageData(); }
} else if (data.flushedCount !== undefined && data.flushedCount > 0) { await fetchStorageData();
// Partial flush: local events persisted, remote Port 4000 flush returned error } else if (data.flushedCount !== undefined && Number(data.flushedCount) > 0) {
warning( warning(
'Partial Flush', 'Partial Flush',
`Flushed ${data.flushedCount} local events. Ingestion server: ${data.error || 'flush failed'}` `Flushed ${data.flushedCount} local events. Ingestion server: ${data.error || 'flush failed'}`
); );
fetchStorageData(); await fetchStorageData();
} else { } else {
error('Flush Failed', data.error || 'Could not flush buffer'); error('Flush Failed', data.error || 'Could not flush buffer');
} }
@@ -84,7 +128,9 @@ export default function StoragePage() {
size="xs" size="xs"
variant="secondary" variant="secondary"
icon="RotateCw" icon="RotateCw"
onClick={fetchStorageData} loading={refreshing}
disabled={refreshing || flushing}
onClick={() => void fetchStorageData()}
> >
Refresh Refresh
</Button> </Button>
@@ -93,7 +139,8 @@ export default function StoragePage() {
variant="primary" variant="primary"
icon="HardDrive" icon="HardDrive"
loading={flushing} loading={flushing}
onClick={handleManualFlush} disabled={flushing}
onClick={() => void handleManualFlush()}
> >
Flush Buffer Flush Buffer
</Button> </Button>
@@ -131,8 +178,8 @@ export default function StoragePage() {
{files.length === 0 ? ( {files.length === 0 ? (
<TableEmpty colSpan={5} message="No CSV files" /> <TableEmpty colSpan={5} message="No CSV files" />
) : ( ) : (
files.map((file, idx) => ( files.map((file) => (
<TableRow key={idx}> <TableRow key={`${file.clientId}/${file.filename}`}>
<TableCell className="font-mono text-[var(--accent)]"> <TableCell className="font-mono text-[var(--accent)]">
{file.clientId} {file.clientId}
</TableCell> </TableCell>
+26 -3
View File
@@ -3,7 +3,7 @@
import { Button } from '@/components/ui/Button'; import { Button } from '@/components/ui/Button';
import { Icon } from '@/components/ui/Icon'; import { Icon } from '@/components/ui/Icon';
import { cn } from '@/lib/utils'; import { cn } from '@/lib/utils';
import { getCached, setCached } from '@/lib/vconCache'; import { getCached, setCached, clearAuthCaches } from '@/lib/vconCache';
import Link from 'next/link'; import Link from 'next/link';
import { usePathname } from 'next/navigation'; import { usePathname } from 'next/navigation';
import React, { useEffect, useState } from 'react'; import React, { useEffect, useState } from 'react';
@@ -69,8 +69,26 @@ export const Sidebar: React.FC<SidebarProps> = ({
if (cached.role) setAdminRole(cached.role); if (cached.role) setAdminRole(cached.role);
} }
fetch('/api/auth/profile', { credentials: 'include' }) const handleProfileUpdated = (event: Event) => {
.then((res) => (res.ok ? res.json() : null)) const detail = (event as CustomEvent<CachedProfile>).detail;
if (detail?.email) {
setAdminEmail(detail.email);
if (detail.role) setAdminRole(detail.role);
}
};
window.addEventListener('datastack:profile-updated', handleProfileUpdated);
fetch('/api/auth/profile', {
credentials: 'include',
cache: 'no-store',
})
.then((res) => {
if (res.status === 401) {
window.location.assign('/vcon/login');
return null;
}
return res.ok ? res.json() : null;
})
.then((data) => { .then((data) => {
if (data?.profile?.email) { if (data?.profile?.email) {
setCached('vcon:profile', data.profile); setCached('vcon:profile', data.profile);
@@ -82,6 +100,10 @@ export const Sidebar: React.FC<SidebarProps> = ({
.catch(() => { .catch(() => {
// Keep cached or default identity label if offline // Keep cached or default identity label if offline
}); });
return () => {
window.removeEventListener('datastack:profile-updated', handleProfileUpdated);
};
}, [pathname]); }, [pathname]);
const toggleTheme = () => { const toggleTheme = () => {
@@ -100,6 +122,7 @@ export const Sidebar: React.FC<SidebarProps> = ({
credentials: 'include', credentials: 'include',
}); });
if (!response.ok) throw new Error('Unable to sign out'); if (!response.ok) throw new Error('Unable to sign out');
await clearAuthCaches();
window.location.assign('/vcon/login'); window.location.assign('/vcon/login');
} catch { } catch {
setIsSigningOut(false); setIsSigningOut(false);
+8
View File
@@ -0,0 +1,8 @@
'use client';
import { MainLayout } from '@/components/layout/MainLayout';
import React from 'react';
export function VconShell({ children }: { children: React.ReactNode }) {
return <MainLayout showSidebar={true}>{children}</MainLayout>;
}
+5 -1
View File
@@ -16,15 +16,19 @@ function safeNextPath() {
export function AdminLogin({ export function AdminLogin({
showDemoHint = false, showDemoHint = false,
databaseReady = true,
}: { }: {
showDemoHint?: boolean; showDemoHint?: boolean;
databaseReady?: boolean;
}) { }) {
const [email, setEmail] = useState(''); const [email, setEmail] = useState('');
const [password, setPassword] = useState(''); const [password, setPassword] = useState('');
const [remember, setRemember] = useState(true); const [remember, setRemember] = useState(true);
const [showPassword, setShowPassword] = useState(false); const [showPassword, setShowPassword] = useState(false);
const [isSubmitting, setIsSubmitting] = useState(false); const [isSubmitting, setIsSubmitting] = useState(false);
const [errorMessage, setErrorMessage] = useState(''); const [errorMessage, setErrorMessage] = useState(
databaseReady ? '' : 'Database connection unavailable. Retry once PostgreSQL recovers.',
);
const [theme, setTheme] = useState<'dark' | 'light'>('dark'); const [theme, setTheme] = useState<'dark' | 'light'>('dark');
useEffect(() => { useEffect(() => {
+32 -3
View File
@@ -4,13 +4,15 @@ import { Button } from '@/components/ui/Button';
import { Input } from '@/components/ui/Input'; import { Input } from '@/components/ui/Input';
import { ArrowRight } from 'lucide-react'; import { ArrowRight } from 'lucide-react';
import Link from 'next/link'; import Link from 'next/link';
import { useState } from 'react'; import { useEffect, useState } from 'react';
export function AdminSetup({ export function AdminSetup({
databaseReady = true, databaseReady = true,
databaseError = '',
initialEmail = '', initialEmail = '',
}: { }: {
databaseReady?: boolean; databaseReady?: boolean;
databaseError?: string;
initialEmail?: string; initialEmail?: string;
}) { }) {
const [email, setEmail] = useState(initialEmail || 'admin@datastack.dev'); const [email, setEmail] = useState(initialEmail || 'admin@datastack.dev');
@@ -19,8 +21,25 @@ export function AdminSetup({
const [setupKey, setSetupKey] = useState(''); const [setupKey, setSetupKey] = useState('');
const [isSubmitting, setIsSubmitting] = useState(false); const [isSubmitting, setIsSubmitting] = useState(false);
const [errorMessage, setErrorMessage] = useState( const [errorMessage, setErrorMessage] = useState(
databaseReady ? '' : 'Database connection unavailable.', databaseReady ? '' : databaseError || 'Database connection unavailable.',
); );
const [theme, setTheme] = useState<'dark' | 'light'>('dark');
useEffect(() => {
const savedTheme = localStorage.getItem('datastack-theme');
const initialTheme = savedTheme === 'light' ? 'light' : 'dark';
setTheme(initialTheme);
document.documentElement.classList.toggle('light', initialTheme === 'light');
document.documentElement.classList.toggle('dark', initialTheme === 'dark');
}, []);
const toggleTheme = () => {
const nextTheme = theme === 'dark' ? 'light' : 'dark';
setTheme(nextTheme);
localStorage.setItem('datastack-theme', nextTheme);
document.documentElement.classList.toggle('light', nextTheme === 'light');
document.documentElement.classList.toggle('dark', nextTheme === 'dark');
};
const submit = async (event: React.FormEvent<HTMLFormElement>) => { const submit = async (event: React.FormEvent<HTMLFormElement>) => {
event.preventDefault(); event.preventDefault();
@@ -59,7 +78,17 @@ export function AdminSetup({
return ( return (
<main className="relative grid min-h-dvh place-items-center bg-[var(--bg)] p-4 text-[var(--text)]"> <main className="relative grid min-h-dvh place-items-center bg-[var(--bg)] p-4 text-[var(--text)]">
<section className="w-full max-w-[320px] border border-[var(--border)] bg-[var(--surface)]"> <Button
type="button"
variant="icon"
size="icon"
icon={theme === 'dark' ? 'Sun' : 'Moon'}
aria-label="Toggle theme"
title="Toggle theme"
className="fixed right-3 top-3 z-10 h-7 w-7 border border-[var(--border)] text-[var(--text-muted)]"
onClick={toggleTheme}
/>
<section className="z-0 w-full max-w-[320px] border border-[var(--border)] bg-[var(--surface)]">
<header className="flex items-center justify-between border-b border-[var(--border)] px-4 py-3"> <header className="flex items-center justify-between border-b border-[var(--border)] px-4 py-3">
<Link href="/" className="flex items-center gap-2 text-[12.5px] font-medium text-[var(--text)]"> <Link href="/" className="flex items-center gap-2 text-[12.5px] font-medium text-[var(--text)]">
<span className="grid h-3 w-3 place-items-center bg-[var(--accent)]" aria-hidden="true"> <span className="grid h-3 w-3 place-items-center bg-[var(--accent)]" aria-hidden="true">
+51 -13
View File
@@ -43,12 +43,18 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
type: 'rotate' | 'revoke' | 'delete'; type: 'rotate' | 'revoke' | 'delete';
client: ClientItem; client: ClientItem;
} | null>(null); } | null>(null);
const [purgeFilesOnDelete, setPurgeFilesOnDelete] = useState(false);
const [editClient, setEditClient] = useState<ClientItem | null>(null); const [editClient, setEditClient] = useState<ClientItem | null>(null);
const [editName, setEditName] = useState(''); const [editName, setEditName] = useState('');
const [editRateLimit, setEditRateLimit] = useState('100'); const [editRateLimit, setEditRateLimit] = useState('100');
const [editError, setEditError] = useState(''); const [editError, setEditError] = useState('');
const [processing, setProcessing] = useState(false); const [processing, setProcessing] = useState(false);
const closeConfirmModal = () => {
setConfirmAction(null);
setPurgeFilesOnDelete(false);
};
const openEditModal = (client: ClientItem) => { const openEditModal = (client: ClientItem) => {
setEditClient(client); setEditClient(client);
setEditName(client.name); setEditName(client.name);
@@ -66,8 +72,8 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
setEditError('Display name is required.'); setEditError('Display name is required.');
return; return;
} }
if (!Number.isFinite(numericRateLimit) || numericRateLimit < 1 || numericRateLimit > 10000) { if (!Number.isInteger(numericRateLimit) || numericRateLimit < 1 || numericRateLimit > 100000) {
setEditError('Rate limit must be between 1 and 10,000 events/s.'); setEditError('Rate limit must be an integer between 1 and 100,000 events/s.');
return; return;
} }
@@ -147,20 +153,30 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
} }
}; };
const executeDeleteClient = async (client: ClientItem) => { const executeDeleteClient = async (client: ClientItem, purgeFiles: boolean) => {
setProcessing(true); setProcessing(true);
try { try {
const response = await fetch(`/api/clients/${encodeURIComponent(client.clientId)}`, { const query = purgeFiles ? '?purgeFiles=true' : '';
const response = await fetch(
`/api/clients/${encodeURIComponent(client.clientId)}${query}`,
{
method: 'DELETE', method: 'DELETE',
}); },
);
if (response.status === 401) { if (response.status === 401) {
window.location.assign('/vcon/login'); window.location.assign('/vcon/login');
return; return;
} }
const data = await response.json(); const data = await response.json();
if (!response.ok) throw new Error(data.error || 'Unable to delete client'); if (!response.ok) throw new Error(data.error || 'Unable to delete client');
setConfirmAction(null); closeConfirmModal();
success('Client deleted', `${client.clientId} has been permanently removed.`); const purgedCount = Number(data?.purgedFiles ?? 0);
success(
'Client deleted',
purgeFiles
? `${client.clientId} removed and ${purgedCount} stored CSV file${purgedCount === 1 ? '' : 's'} purged.`
: `${client.clientId} has been permanently removed (stored CSV files retained).`,
);
await onStatusChange(); await onStatusChange();
} catch (deleteError) { } catch (deleteError) {
error('Client deletion failed', deleteError instanceof Error ? deleteError.message : 'Unknown error'); error('Client deletion failed', deleteError instanceof Error ? deleteError.message : 'Unknown error');
@@ -171,6 +187,7 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
const toggleStatus = (client: ClientItem) => { const toggleStatus = (client: ClientItem) => {
if (client.status === 'active') { if (client.status === 'active') {
setPurgeFilesOnDelete(false);
setConfirmAction({ type: 'revoke', client }); setConfirmAction({ type: 'revoke', client });
} else { } else {
void executeStatusChange(client, 'active'); void executeStatusChange(client, 'active');
@@ -182,7 +199,7 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
if (confirmAction.type === 'rotate') { if (confirmAction.type === 'rotate') {
await executeRotateKey(confirmAction.client); await executeRotateKey(confirmAction.client);
} else if (confirmAction.type === 'delete') { } else if (confirmAction.type === 'delete') {
await executeDeleteClient(confirmAction.client); await executeDeleteClient(confirmAction.client, purgeFilesOnDelete);
} else { } else {
await executeStatusChange(confirmAction.client, 'revoked'); await executeStatusChange(confirmAction.client, 'revoked');
} }
@@ -302,7 +319,10 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
className="h-6 w-6 border border-transparent hover:border-[var(--danger)] hover:text-[var(--danger)]" className="h-6 w-6 border border-transparent hover:border-[var(--danger)] hover:text-[var(--danger)]"
title={`Delete ${client.clientId}`} title={`Delete ${client.clientId}`}
aria-label={`Delete ${client.clientId}`} aria-label={`Delete ${client.clientId}`}
onClick={() => setConfirmAction({ type: 'delete', client })} onClick={() => {
setPurgeFilesOnDelete(false);
setConfirmAction({ type: 'delete', client });
}}
/> />
</div> </div>
</TableCell> </TableCell>
@@ -350,7 +370,7 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
compact compact
type="number" type="number"
min="1" min="1"
max="10000" max="100000"
value={editRateLimit} value={editRateLimit}
onChange={(event) => setEditRateLimit(event.target.value)} onChange={(event) => setEditRateLimit(event.target.value)}
required required
@@ -365,7 +385,7 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
<Modal <Modal
isOpen={Boolean(confirmAction)} isOpen={Boolean(confirmAction)}
onClose={() => !processing && setConfirmAction(null)} onClose={() => !processing && closeConfirmModal()}
title={ title={
confirmAction?.type === 'rotate' confirmAction?.type === 'rotate'
? 'Rotate credentials' ? 'Rotate credentials'
@@ -376,7 +396,7 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
maxWidth="xs" maxWidth="xs"
footer={ footer={
<> <>
<Button type="button" variant="ghost" size="xs" disabled={processing} onClick={() => setConfirmAction(null)}> <Button type="button" variant="ghost" size="xs" disabled={processing} onClick={closeConfirmModal}>
Cancel Cancel
</Button> </Button>
<Button <Button
@@ -402,6 +422,22 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
? `Permanently delete client ${confirmAction.client.clientId}? This removes the client registration and credentials.` ? `Permanently delete client ${confirmAction.client.clientId}? This removes the client registration and credentials.`
: `Revoke access for ${confirmAction?.client.clientId || ''}?`} : `Revoke access for ${confirmAction?.client.clientId || ''}?`}
</p> </p>
{confirmAction?.type === 'delete' && (
<label className="mt-3 flex cursor-pointer items-start gap-2 border border-[var(--border)] bg-[var(--surface-2)] p-2.5 text-[11.5px] text-[var(--text)]">
<input
type="checkbox"
checked={purgeFilesOnDelete}
onChange={(event) => setPurgeFilesOnDelete(event.target.checked)}
className="mt-0.5 h-3.5 w-3.5 shrink-0 accent-[var(--danger)]"
/>
<span>
Also purge stored CSV files for this client
<span className="mt-0.5 block text-[10.5px] text-[var(--text-faint)]">
Deletes partitioned disk CSV files along with the client registration.
</span>
</span>
</label>
)}
</Modal> </Modal>
<Modal <Modal
@@ -420,7 +456,6 @@ export function CredentialsDisplay({ credentials }: { credentials: ClientCredent
return ( return (
<div className="flex flex-col gap-2.5"> <div className="flex flex-col gap-2.5">
<CredentialItem label="Client ID" value={credentials.clientId} /> <CredentialItem label="Client ID" value={credentials.clientId} />
<CredentialItem label="Secret key" value={credentials.secret} />
<div className="flex flex-col gap-1"> <div className="flex flex-col gap-1">
<div className="flex items-center justify-between gap-2"> <div className="flex items-center justify-between gap-2">
<label htmlFor={`token-${credentials.clientId}`} className="text-[10px] uppercase tracking-wider text-[var(--text-faint)]"> <label htmlFor={`token-${credentials.clientId}`} className="text-[10px] uppercase tracking-wider text-[var(--text-faint)]">
@@ -435,6 +470,9 @@ export function CredentialsDisplay({ credentials }: { credentials: ClientCredent
rows={3} rows={3}
className="font-mono text-[11px] text-[var(--accent)]" className="font-mono text-[11px] text-[var(--accent)]"
/> />
<p className="text-[10.5px] text-[var(--text-faint)]">
Use this signed JWT token for both Socket.IO handshakes and HTTP Bearer ingestion.
</p>
</div> </div>
</div> </div>
); );
+100 -17
View File
@@ -15,7 +15,14 @@ import {
TableRow, TableRow,
TableWrap, TableWrap,
} from '@/components/ui/Table'; } from '@/components/ui/Table';
import { useToast } from '@/components/ui/Toast';
import { getCached, setCached } from '@/lib/vconCache'; import { getCached, setCached } from '@/lib/vconCache';
import {
countMatchingBufferedEvents,
extractContentDispositionFilename,
formatEventTimeLabel,
type BufferedSummary,
} from '@/lib/vconUiHelpers';
import Link from 'next/link'; import Link from 'next/link';
import React, { useCallback, useEffect, useState } from 'react'; import React, { useCallback, useEffect, useState } from 'react';
@@ -40,6 +47,12 @@ type DashboardMetrics = {
totalFiles: number; totalFiles: number;
totalRows: number; totalRows: number;
}; };
runtime?: {
ingestionBridgeStatus?: 'online' | 'auth_error' | 'error' | 'offline';
ingestionBridgeError?: string;
serverPort?: number;
webPort?: number;
};
}; };
type StorageFile = { type StorageFile = {
@@ -51,6 +64,7 @@ type StorageFile = {
}; };
export const DashboardOverview: React.FC = () => { export const DashboardOverview: React.FC = () => {
const { error, info } = useToast();
const [metrics, setMetrics] = useState<DashboardMetrics | null>(() => const [metrics, setMetrics] = useState<DashboardMetrics | null>(() =>
getCached<DashboardMetrics | null>('vcon:metrics', null) getCached<DashboardMetrics | null>('vcon:metrics', null)
); );
@@ -63,7 +77,11 @@ export const DashboardOverview: React.FC = () => {
const [files, setFiles] = useState<StorageFile[]>(() => const [files, setFiles] = useState<StorageFile[]>(() =>
getCached<StorageFile[]>('vcon:storage-files', []) getCached<StorageFile[]>('vcon:storage-files', [])
); );
const [bufferedSummary, setBufferedSummary] = useState<BufferedSummary | null>(() =>
getCached<BufferedSummary | null>('vcon:buffered-summary', null)
);
const [isRefreshing, setIsRefreshing] = useState(false); const [isRefreshing, setIsRefreshing] = useState(false);
const [isExporting, setIsExporting] = useState(false);
const [fetchError, setFetchError] = useState(''); const [fetchError, setFetchError] = useState('');
const refresh = useCallback(async () => { const refresh = useCallback(async () => {
@@ -72,7 +90,7 @@ export const DashboardOverview: React.FC = () => {
const [metricsResponse, eventsResponse, storageResponse, clientsResponse] = const [metricsResponse, eventsResponse, storageResponse, clientsResponse] =
await Promise.all([ await Promise.all([
fetch('/api/metrics', { cache: 'no-store' }), fetch('/api/metrics', { cache: 'no-store' }),
fetch('/api/events?limit=8', { cache: 'no-store' }), fetch('/api/events?limit=8&includeBufferedSummary=true', { cache: 'no-store' }),
fetch('/api/storage', { cache: 'no-store' }), fetch('/api/storage', { cache: 'no-store' }),
fetch('/api/clients', { cache: 'no-store' }), fetch('/api/clients', { cache: 'no-store' }),
]); ]);
@@ -99,6 +117,9 @@ export const DashboardOverview: React.FC = () => {
if (eventsResponse.ok) { if (eventsResponse.ok) {
const data = await eventsResponse.json(); const data = await eventsResponse.json();
setEvents(setCached('vcon:dashboard-events', data.events ?? [])); setEvents(setCached('vcon:dashboard-events', data.events ?? []));
if (data.bufferedSummary) {
setBufferedSummary(setCached('vcon:buffered-summary', data.bufferedSummary));
}
} else { } else {
errors.push('events'); errors.push('events');
} }
@@ -145,12 +166,30 @@ export const DashboardOverview: React.FC = () => {
.filter((file) => file.date === today) .filter((file) => file.date === today)
.reduce((total, file) => total + file.linesCount, 0); .reduce((total, file) => total + file.linesCount, 0);
const bufferedCount = metrics?.bufferStats.currentBufferSize ?? 0; const bufferedCount = metrics?.bufferStats.currentBufferSize ?? 0;
const eventsToday = storedRecordsToday + bufferedCount; const bufferedTodayRaw = bufferedSummary
? countMatchingBufferedEvents(bufferedSummary, {
clientId: 'all',
fromDate: today,
toDate: today,
}).matchingCount
: 0;
const bufferedTodayCount = metrics
? Math.min(bufferedTodayRaw, bufferedCount)
: bufferedTodayRaw;
const eventsToday = storedRecordsToday + bufferedTodayCount;
const activeClients = const activeClients =
metrics?.activeClients ?? metrics?.activeClients ??
clients.filter((client) => client.status === 'active').length; clients.filter((client) => client.status === 'active').length;
const totalClients = metrics?.totalClients ?? clients.length; const totalClients = metrics?.totalClients ?? clients.length;
const activeSockets = metrics?.activeSockets ?? 0; const activeSockets = metrics?.activeSockets ?? 0;
const bridgeStatus = metrics?.runtime?.ingestionBridgeStatus;
const bridgeAuthError = bridgeStatus === 'auth_error';
const socketSuffix =
bridgeStatus === 'auth_error'
? `${activeSockets} ws · auth err`
: bridgeStatus === 'offline'
? `${activeSockets} ws · offline`
: `${activeSockets} ws`;
const totalStorage = metrics?.storageStats.totalMB ?? '0.00'; const totalStorage = metrics?.storageStats.totalMB ?? '0.00';
const totalStorageBytes = metrics?.storageStats.totalBytes ?? 0; const totalStorageBytes = metrics?.storageStats.totalBytes ?? 0;
const storageValue = const storageValue =
@@ -165,6 +204,41 @@ export const DashboardOverview: React.FC = () => {
metrics?.storageStats.totalRows ?? metrics?.storageStats.totalRows ??
files.reduce((sum, file) => sum + file.linesCount, 0); files.reduce((sum, file) => sum + file.linesCount, 0);
const handleQuickExport = async () => {
setIsExporting(true);
try {
const res = await fetch('/api/export?clientId=all', { cache: 'no-store' });
if (res.status === 401) {
window.location.assign('/vcon/login');
return;
}
if (!res.ok) {
const errJson = await res.json().catch(() => ({}));
error('Export failed', errJson.error || `Export failed with HTTP ${res.status}.`);
return;
}
const blob = await res.blob();
const filename = extractContentDispositionFilename(
res.headers.get('content-disposition'),
`datastack_export_all_${new Date().toISOString().slice(0, 10)}.csv`
);
const objectUrl = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = objectUrl;
a.download = filename;
document.body.appendChild(a);
a.click();
document.body.removeChild(a);
URL.revokeObjectURL(objectUrl);
info('Export downloaded', `${filename} downloaded.`);
await refresh();
} catch (err) {
error('Export failed', err instanceof Error ? err.message : 'Unable to export dataset.');
} finally {
setIsExporting(false);
}
};
return ( return (
<PageShell <PageShell
title="Dashboard" title="Dashboard"
@@ -179,11 +253,16 @@ export const DashboardOverview: React.FC = () => {
loading={isRefreshing} loading={isRefreshing}
onClick={() => void refresh()} onClick={() => void refresh()}
/> />
<a href="/api/export?clientId=all" download> <Button
<Button size="xs" variant="secondary" icon="Download"> size="xs"
variant="secondary"
icon="Download"
loading={isExporting}
disabled={isExporting}
onClick={() => void handleQuickExport()}
>
Export Export
</Button> </Button>
</a>
<Link href="/vcon/clients"> <Link href="/vcon/clients">
<Button size="xs" variant="primary" icon="Plus"> <Button size="xs" variant="primary" icon="Plus">
New client New client
@@ -192,12 +271,14 @@ export const DashboardOverview: React.FC = () => {
</> </>
} }
> >
{fetchError && ( {(fetchError || bridgeAuthError) && (
<div <div
role="alert" role="alert"
className="mb-3 border border-[var(--danger)] bg-[var(--danger-soft)] px-3 py-2 text-[11.5px] text-[var(--text)]" className="mb-3 border border-[var(--danger)] bg-[var(--danger-soft)] px-3 py-2 text-[11.5px] text-[var(--text)]"
> >
{fetchError} {bridgeAuthError
? `Ingestion bridge authentication mismatch (${metrics?.runtime?.ingestionBridgeError || '401'}). Check DATASTACK_INTERNAL_SECRET parity.`
: fetchError}
</div> </div>
)} )}
@@ -206,8 +287,13 @@ export const DashboardOverview: React.FC = () => {
{ label: 'Events today', value: eventsToday.toLocaleString() }, { label: 'Events today', value: eventsToday.toLocaleString() },
{ {
label: 'Active clients · Sockets', label: 'Active clients · Sockets',
value: `${activeClients} / ${totalClients} · ${activeSockets} ws`, value: `${activeClients} / ${totalClients} · ${socketSuffix}`,
dot: activeClients > 0 || activeSockets > 0 ? 'ok' : 'off', dot:
bridgeAuthError
? 'off'
: activeSockets > 0 || (activeClients > 0 && bridgeStatus === 'online')
? 'ok'
: 'off',
}, },
{ label: 'Storage used', value: `${storageValue} ${storageUnit}` }, { label: 'Storage used', value: `${storageValue} ${storageUnit}` },
{ label: 'Stored rows', value: totalRows.toLocaleString() }, { label: 'Stored rows', value: totalRows.toLocaleString() },
@@ -248,9 +334,7 @@ export const DashboardOverview: React.FC = () => {
events.map((event) => ( events.map((event) => (
<TableRow key={event.eventId}> <TableRow key={event.eventId}>
<TableCell className="font-mono tabular-nums text-[var(--text-faint)]"> <TableCell className="font-mono tabular-nums text-[var(--text-faint)]">
{event.timestamp.includes('T') {formatEventTimeLabel(event.timestamp)}
? new Date(event.timestamp).toLocaleTimeString()
: event.timestamp}
</TableCell> </TableCell>
<TableCell className="font-mono text-[var(--accent)]"> <TableCell className="font-mono text-[var(--accent)]">
{event.clientId} {event.clientId}
@@ -288,13 +372,12 @@ export const DashboardOverview: React.FC = () => {
title="Storage" title="Storage"
meta={`${totalFiles}`} meta={`${totalFiles}`}
action={ action={
<a <Link
href="/api/export?clientId=all" href="/vcon/storage"
download
className="text-[11px] text-[var(--text-muted)] hover:text-[var(--text)]" className="text-[11px] text-[var(--text-muted)] hover:text-[var(--text)]"
> >
Export ↓ Files →
</a> </Link>
} }
> >
<div className="divide-y divide-[var(--border)]"> <div className="divide-y divide-[var(--border)]">
+33 -11
View File
@@ -19,6 +19,7 @@ import {
TableRow, TableRow,
TableWrap, TableWrap,
} from '@/components/ui/Table'; } from '@/components/ui/Table';
import { formatEventTimeLabel } from '@/lib/vconUiHelpers';
import { useRouter } from 'next/navigation'; import { useRouter } from 'next/navigation';
import { useCallback, useEffect, useState } from 'react'; import { useCallback, useEffect, useState } from 'react';
@@ -86,8 +87,10 @@ export function DataBrowser() {
window.location.assign('/vcon/login'); window.location.assign('/vcon/login');
return; return;
} }
if (!response.ok) throw new Error('Unable to load events'); const data = await response.json().catch(() => ({}));
const data = await response.json(); if (!response.ok) {
throw new Error(data.error || 'Unable to load events');
}
setEvents(data.events || []); setEvents(data.events || []);
setTotal(data.total || 0); setTotal(data.total || 0);
setPageCount(data.pageCount || 1); setPageCount(data.pageCount || 1);
@@ -124,6 +127,8 @@ export function DataBrowser() {
fetchEvents(); fetchEvents();
}, [fetchEvents, filtersReady]); }, [fetchEvents, filtersReady]);
const invalidRange = Boolean(fromFilter && toFilter && fromFilter > toFilter);
const firstVisible = total === 0 ? 0 : (page - 1) * PAGE_SIZE + 1; const firstVisible = total === 0 ? 0 : (page - 1) * PAGE_SIZE + 1;
const lastVisible = Math.min(page * PAGE_SIZE, total); const lastVisible = Math.min(page * PAGE_SIZE, total);
const firstPage = Math.max(1, Math.min(page - 1, pageCount - 2)); const firstPage = Math.max(1, Math.min(page - 1, pageCount - 2));
@@ -133,6 +138,11 @@ export function DataBrowser() {
); );
const applyFilters = () => { const applyFilters = () => {
if (invalidRange) {
setErrorMessage('Start date ("From") must be on or before end date ("To").');
return;
}
setErrorMessage('');
setAppliedFilters({ setAppliedFilters({
clientId: clientFilter, clientId: clientFilter,
from: fromFilter, from: fromFilter,
@@ -147,6 +157,7 @@ export function DataBrowser() {
setFromFilter(''); setFromFilter('');
setToFilter(''); setToFilter('');
setSearchFilter(''); setSearchFilter('');
setErrorMessage('');
setAppliedFilters({ clientId: 'all', from: '', to: '', search: '' }); setAppliedFilters({ clientId: 'all', from: '', to: '', search: '' });
setPage(1); setPage(1);
}; };
@@ -189,7 +200,13 @@ export function DataBrowser() {
} }
> >
<Card compact className="p-0"> <Card compact className="p-0">
<div className="grid grid-cols-1 gap-px bg-[var(--border)] sm:grid-cols-2 lg:grid-cols-5"> <form
onSubmit={(e) => {
e.preventDefault();
applyFilters();
}}
className="grid grid-cols-1 gap-px bg-[var(--border)] sm:grid-cols-2 lg:grid-cols-5"
>
<div className="bg-[var(--surface)] p-2.5"> <div className="bg-[var(--surface)] p-2.5">
<Select <Select
label="Client" label="Client"
@@ -237,11 +254,21 @@ export function DataBrowser() {
<Button type="button" size="xs" icon="X" onClick={resetFilters}> <Button type="button" size="xs" icon="X" onClick={resetFilters}>
Reset Reset
</Button> </Button>
<Button type="button" size="xs" variant="primary" onClick={applyFilters}> <Button
type="submit"
size="xs"
variant="primary"
disabled={invalidRange}
>
Apply Apply
</Button> </Button>
</div> </div>
</div> </form>
{invalidRange && (
<p role="alert" className="border-t border-[var(--border)] bg-[var(--surface-2)] px-3 py-1.5 text-[11px] text-[var(--danger)]">
Start date must be on or before end date.
</p>
)}
</Card> </Card>
<Panel <Panel
@@ -274,12 +301,7 @@ export function DataBrowser() {
onClick={() => setSelectedEvent(event)} onClick={() => setSelectedEvent(event)}
> >
<TableCell className="font-mono tabular-nums text-[var(--text)]"> <TableCell className="font-mono tabular-nums text-[var(--text)]">
{new Date(event.timestamp).toLocaleTimeString([], { {formatEventTimeLabel(event.timestamp)}
hour: '2-digit',
minute: '2-digit',
second: '2-digit',
hour12: false,
})}
</TableCell> </TableCell>
<TableCell className="font-mono text-[var(--accent)]"> <TableCell className="font-mono text-[var(--accent)]">
{event.clientId} {event.clientId}
+116 -47
View File
@@ -8,9 +8,16 @@ import { Pill } from '@/components/ui/Pill';
import { StatStrip } from '@/components/ui/StatStrip'; import { StatStrip } from '@/components/ui/StatStrip';
import { useToast } from '@/components/ui/Toast'; import { useToast } from '@/components/ui/Toast';
import { getCached, setCached } from '@/lib/vconCache'; import { getCached, setCached } from '@/lib/vconCache';
import {
countMatchingBufferedEvents,
extractContentDispositionFilename,
getUtcDateRangeForPreset,
matchesSelectedClientFolder,
type BufferedSummary,
} from '@/lib/vconUiHelpers';
import { Braces, FileText } from 'lucide-react'; import { Braces, FileText } from 'lucide-react';
import Link from 'next/link'; import Link from 'next/link';
import { useEffect, useMemo, useState } from 'react'; import { useCallback, useEffect, useMemo, useState } from 'react';
interface Client { interface Client {
clientId: string; clientId: string;
@@ -29,10 +36,6 @@ interface StorageFile {
type ExportFormat = 'csv' | 'json'; type ExportFormat = 'csv' | 'json';
type RangePreset = 'today' | '7d' | '30d' | 'all' | 'custom'; type RangePreset = 'today' | '7d' | '30d' | 'all' | 'custom';
function dateString(date: Date) {
return date.toISOString().slice(0, 10);
}
function formatBytes(bytes: number) { function formatBytes(bytes: number) {
if (bytes < 1024) return `${bytes} B`; if (bytes < 1024) return `${bytes} B`;
if (bytes < 1024 ** 2) return `${(bytes / 1024).toFixed(1)} KB`; if (bytes < 1024 ** 2) return `${(bytes / 1024).toFixed(1)} KB`;
@@ -41,7 +44,9 @@ function formatBytes(bytes: number) {
} }
function relativeTime(value: string, now: number) { function relativeTime(value: string, now: number) {
const elapsedSeconds = Math.max(0, Math.floor((now - new Date(value).getTime()) / 1000)); const ts = new Date(value).getTime();
if (!Number.isFinite(ts)) return '—';
const elapsedSeconds = Math.max(0, Math.floor((now - ts) / 1000));
if (elapsedSeconds < 60) return `${elapsedSeconds}s ago`; if (elapsedSeconds < 60) return `${elapsedSeconds}s ago`;
if (elapsedSeconds < 3600) return `${Math.floor(elapsedSeconds / 60)}m ago`; if (elapsedSeconds < 3600) return `${Math.floor(elapsedSeconds / 60)}m ago`;
if (elapsedSeconds < 86_400) return `${Math.floor(elapsedSeconds / 3600)}h ago`; if (elapsedSeconds < 86_400) return `${Math.floor(elapsedSeconds / 3600)}h ago`;
@@ -57,6 +62,7 @@ export function ExportWorkspace() {
getCached<StorageFile[]>('vcon:storage-files', []) getCached<StorageFile[]>('vcon:storage-files', [])
); );
const [bufferedCount, setBufferedCount] = useState<number>(0); const [bufferedCount, setBufferedCount] = useState<number>(0);
const [bufferedSummary, setBufferedSummary] = useState<BufferedSummary | null>(null);
const [selectedClient, setSelectedClient] = useState('all'); const [selectedClient, setSelectedClient] = useState('all');
const [fromDate, setFromDate] = useState(''); const [fromDate, setFromDate] = useState('');
const [toDate, setToDate] = useState(''); const [toDate, setToDate] = useState('');
@@ -65,11 +71,13 @@ export function ExportWorkspace() {
const [loading, setLoading] = useState( const [loading, setLoading] = useState(
() => getCached<StorageFile[]>('vcon:storage-files', []).length === 0 () => getCached<StorageFile[]>('vcon:storage-files', []).length === 0
); );
const [downloading, setDownloading] = useState(false);
const [loadError, setLoadError] = useState(''); const [loadError, setLoadError] = useState('');
const [now, setNow] = useState(() => Date.now()); const [now, setNow] = useState(() => Date.now());
useEffect(() => { const refreshExportData = useCallback(async () => {
Promise.all([ try {
const [clientData, storageData, metricsData, bufferedData] = await Promise.all([
fetch('/api/clients', { cache: 'no-store' }).then(async (response) => { fetch('/api/clients', { cache: 'no-store' }).then(async (response) => {
if (response.status === 401) { if (response.status === 401) {
window.location.assign('/vcon/login'); window.location.assign('/vcon/login');
@@ -89,24 +97,37 @@ export function ExportWorkspace() {
fetch('/api/metrics', { cache: 'no-store' }) fetch('/api/metrics', { cache: 'no-store' })
.then(async (response) => (response.ok ? response.json() : null)) .then(async (response) => (response.ok ? response.json() : null))
.catch(() => null), .catch(() => null),
]) fetch('/api/events?bufferedOnly=true&limit=1', { cache: 'no-store' })
.then(([clientData, storageData, metricsData]) => { .then(async (response) => (response.ok ? response.json() : null))
.catch(() => null),
]);
setClients(setCached('vcon:clients', clientData.clients || [])); setClients(setCached('vcon:clients', clientData.clients || []));
setFiles(setCached('vcon:storage-files', storageData.files || [])); setFiles(setCached('vcon:storage-files', storageData.files || []));
if (storageData.stats) setCached('vcon:storage-stats', storageData.stats); if (storageData.stats) setCached('vcon:storage-stats', storageData.stats);
if (metricsData?.bufferStats) { if (metricsData) {
setCached('vcon:metrics', metricsData);
if (metricsData.bufferStats) {
setBufferedCount(Number(metricsData.bufferStats.currentBufferSize) || 0); setBufferedCount(Number(metricsData.bufferStats.currentBufferSize) || 0);
} }
}
if (bufferedData?.bufferedSummary) {
setBufferedSummary(setCached('vcon:buffered-summary', bufferedData.bufferedSummary));
}
setLoadError(''); setLoadError('');
}) } catch (fetchError) {
.catch((fetchError) => {
setLoadError( setLoadError(
fetchError instanceof Error ? fetchError.message : 'Unable to load export data' fetchError instanceof Error ? fetchError.message : 'Unable to load export data'
); );
}) } finally {
.finally(() => setLoading(false)); setLoading(false);
}
}, []); }, []);
useEffect(() => {
void refreshExportData();
}, [refreshExportData]);
useEffect(() => { useEffect(() => {
const updateClock = () => setNow(Date.now()); const updateClock = () => setNow(Date.now());
updateClock(); updateClock();
@@ -118,7 +139,7 @@ export function ExportWorkspace() {
() => () =>
files.filter( files.filter(
(file) => (file) =>
(selectedClient === 'all' || file.clientId === selectedClient) && matchesSelectedClientFolder(file.clientId, selectedClient) &&
(!fromDate || file.date >= fromDate) && (!fromDate || file.date >= fromDate) &&
(!toDate || file.date <= toDate) (!toDate || file.date <= toDate)
), ),
@@ -156,18 +177,9 @@ export function ExportWorkspace() {
const setQuickRange = (nextPreset: Exclude<RangePreset, 'custom'>) => { const setQuickRange = (nextPreset: Exclude<RangePreset, 'custom'>) => {
setPreset(nextPreset); setPreset(nextPreset);
if (nextPreset === 'all') { const range = getUtcDateRangeForPreset(nextPreset);
setFromDate(''); setFromDate(range.from);
setToDate(''); setToDate(range.to);
return;
}
const today = new Date();
const start = new Date(today);
if (nextPreset === '7d') start.setDate(today.getDate() - 6);
if (nextPreset === '30d') start.setDate(today.getDate() - 29);
setFromDate(dateString(start));
setToDate(dateString(today));
}; };
const exportUrl = useMemo(() => { const exportUrl = useMemo(() => {
@@ -177,10 +189,31 @@ export function ExportWorkspace() {
return `/api/export?${params.toString()}`; return `/api/export?${params.toString()}`;
}, [format, fromDate, selectedClient, toDate]); }, [format, fromDate, selectedClient, toDate]);
const invalidRange = Boolean(fromDate && toDate && fromDate > toDate); const filteredBuffered = useMemo(
const nothingToExport = preview.rows === 0 && bufferedCount === 0; () =>
countMatchingBufferedEvents(
bufferedSummary,
{ clientId: selectedClient, fromDate, toDate },
bufferedCount,
),
[bufferedCount, bufferedSummary, fromDate, selectedClient, toDate],
);
const downloadExport = () => { const invalidRange = Boolean(fromDate && toDate && fromDate > toDate);
const nothingToExport = preview.rows === 0 && !filteredBuffered.hasPotentialBuffered;
const triggerBlobDownload = (blob: Blob, filename: string) => {
const objectUrl = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = objectUrl;
a.download = filename;
document.body.appendChild(a);
a.click();
document.body.removeChild(a);
URL.revokeObjectURL(objectUrl);
};
const downloadExport = async () => {
if (invalidRange) { if (invalidRange) {
error('Invalid date range', 'Start date must be on or before end date.'); error('Invalid date range', 'Start date must be on or before end date.');
return; return;
@@ -189,33 +222,60 @@ export function ExportWorkspace() {
error('Nothing to export', 'No stored rows or buffered events match the selected filters.'); error('Nothing to export', 'No stored rows or buffered events match the selected filters.');
return; return;
} }
window.location.assign(exportUrl); setDownloading(true);
try {
const res = await fetch(exportUrl, { cache: 'no-store' });
if (res.status === 401) {
window.location.assign('/vcon/login');
return;
}
if (!res.ok) {
const errorJson = await res.json().catch(() => ({}));
error(
'Export failed',
errorJson.error || `Export endpoint returned HTTP ${res.status}.`
);
return;
}
const blob = await res.blob();
const defaultName = `datastack_export_${selectedClient || 'all'}_${new Date().toISOString().slice(0, 10)}.${format}`;
const filename = extractContentDispositionFilename(
res.headers.get('content-disposition'),
defaultName
);
triggerBlobDownload(blob, filename);
info( info(
'Export started', 'Export downloaded',
bufferedCount > 0 filteredBuffered.matchingCount > 0
? `${format.toUpperCase()} download (${preview.rows.toLocaleString()} stored + ${bufferedCount} buffered auto-flushed).` ? `${format.toUpperCase()} download (${preview.rows.toLocaleString()} stored + ${filteredBuffered.matchingCount.toLocaleString()} buffered flushed).`
: filteredBuffered.unattributedCount > 0
? `${format.toUpperCase()} download (${preview.rows.toLocaleString()} stored + pending backlog flushed).`
: `${format.toUpperCase()} download for ${preview.rows.toLocaleString()} rows.` : `${format.toUpperCase()} download for ${preview.rows.toLocaleString()} rows.`
); );
await refreshExportData();
} catch (err: unknown) {
error('Export failed', err instanceof Error ? err.message : 'Unable to export dataset.');
} finally {
setDownloading(false);
}
}; };
const downloadFile = async (file: StorageFile) => { const downloadFile = async (file: StorageFile) => {
const url = `/api/export?clientId=${encodeURIComponent(file.clientId)}&filename=${encodeURIComponent(file.filename)}`; const url = `/api/export?clientId=${encodeURIComponent(file.clientId)}&filename=${encodeURIComponent(file.filename)}`;
try { try {
const res = await fetch(url, { cache: 'no-store' }); const res = await fetch(url, { cache: 'no-store' });
if (res.status === 401) {
window.location.assign('/vcon/login');
return;
}
if (!res.ok) { if (!res.ok) {
const errorJson = await res.json().catch(() => ({})); const errorJson = await res.json().catch(() => ({}));
error('Download Failed', errorJson.error || 'Requested CSV file could not be exported.'); error('Download Failed', errorJson.error || 'Requested CSV file could not be exported.');
return; return;
} }
const blob = await res.blob(); const blob = await res.blob();
const objectUrl = URL.createObjectURL(blob); triggerBlobDownload(blob, file.filename);
const a = document.createElement('a'); await refreshExportData();
a.href = objectUrl;
a.download = file.filename;
document.body.appendChild(a);
a.click();
document.body.removeChild(a);
URL.revokeObjectURL(objectUrl);
} catch (err: unknown) { } catch (err: unknown) {
error('Download Failed', err instanceof Error ? err.message : 'Export download error'); error('Download Failed', err instanceof Error ? err.message : 'Export download error');
} }
@@ -348,11 +408,19 @@ export function ExportWorkspace() {
</strong>{' '} </strong>{' '}
files files
</span> </span>
{bufferedCount > 0 && ( {filteredBuffered.matchingCount > 0 && (
<> <>
<span aria-hidden="true">·</span> <span aria-hidden="true">·</span>
<span className="text-[var(--warning)]"> <span className="text-[var(--warning)]">
+{bufferedCount} buffered (flushed on export) +{filteredBuffered.matchingCount.toLocaleString()} buffered (flushed on export)
</span>
</>
)}
{filteredBuffered.matchingCount === 0 && filteredBuffered.unattributedCount > 0 && (
<>
<span aria-hidden="true">·</span>
<span className="text-[var(--warning)]">
+{filteredBuffered.unattributedCount.toLocaleString()} queued backlog (flushed on export)
</span> </span>
</> </>
)} )}
@@ -382,8 +450,9 @@ export function ExportWorkspace() {
variant="primary" variant="primary"
size="xs" size="xs"
icon="Download" icon="Download"
disabled={loading || invalidRange || nothingToExport} loading={downloading}
onClick={downloadExport} disabled={loading || downloading || invalidRange || nothingToExport}
onClick={() => void downloadExport()}
> >
Download {format.toUpperCase()} Download {format.toUpperCase()}
</Button> </Button>
+11 -3
View File
@@ -30,14 +30,22 @@ export function NewClientModal({ open, onClose, onCreated }: NewClientModalProps
error('Validation error', 'Client ID and Name are required.'); error('Validation error', 'Client ID and Name are required.');
return; return;
} }
const clampedRateLimit = Math.max(1, Math.min(5000, Number(rateLimit) || 100)); const numericRate = Number(rateLimit);
if (!Number.isInteger(numericRate) || numericRate < 1 || numericRate > 100000) {
error('Validation error', 'Rate limit must be an integer between 1 and 100,000 events/s.');
return;
}
setSubmitting(true); setSubmitting(true);
try { try {
const response = await fetch('/api/clients', { const response = await fetch('/api/clients', {
method: 'POST', method: 'POST',
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ clientId: trimmedId, name: trimmedName, rateLimit: clampedRateLimit }), body: JSON.stringify({ clientId: trimmedId, name: trimmedName, rateLimit: numericRate }),
}); });
if (response.status === 401) {
window.location.assign('/vcon/login');
return;
}
const data = await response.json(); const data = await response.json();
if (!response.ok) throw new Error(data.error || 'Client registration failed'); if (!response.ok) throw new Error(data.error || 'Client registration failed');
@@ -91,7 +99,7 @@ export function NewClientModal({ open, onClose, onCreated }: NewClientModalProps
label="Rate limit (req/s)" label="Rate limit (req/s)"
type="number" type="number"
min="1" min="1"
max="5000" max="100000"
value={rateLimit} value={rateLimit}
onChange={(event) => setRateLimit(event.target.value)} onChange={(event) => setRateLimit(event.target.value)}
/> />
+37 -15
View File
@@ -11,7 +11,11 @@ import {
} from '@/components/ui/SettingsLayout'; } from '@/components/ui/SettingsLayout';
import { StatStrip } from '@/components/ui/StatStrip'; import { StatStrip } from '@/components/ui/StatStrip';
import { useToast } from '@/components/ui/Toast'; import { useToast } from '@/components/ui/Toast';
import { getCached, setCached } from '@/lib/vconCache'; import { getCached, setCached, clearAuthCaches } from '@/lib/vconCache';
import {
formatSessionExpiryLabel,
shouldRedirectProfileUnauthorized,
} from '@/lib/vconUiHelpers';
import Link from 'next/link'; import Link from 'next/link';
import { useEffect, useState } from 'react'; import { useEffect, useState } from 'react';
@@ -49,20 +53,31 @@ export function ProfileWorkspace() {
const [signingOut, setSigningOut] = useState(false); const [signingOut, setSigningOut] = useState(false);
useEffect(() => { useEffect(() => {
fetch('/api/auth/profile', { credentials: 'include' }) fetch('/api/auth/profile', {
credentials: 'include',
cache: 'no-store',
})
.then(async (res) => { .then(async (res) => {
if (!res.ok) throw new Error('Unable to load profile'); if (res.status === 401) {
return res.json(); window.location.assign('/vcon/login');
return null;
}
const body = await res.json().catch(() => ({}));
if (!res.ok) throw new Error(body.error || 'Unable to load profile');
return body;
}) })
.then((data) => { .then((data) => {
if (data.profile) { if (data?.profile) {
setProfile(setCached('vcon:profile', data.profile)); setProfile(setCached('vcon:profile', data.profile));
setEmail(data.profile.email); setEmail(data.profile.email);
setCached('vcon:profile-loaded', true); setCached('vcon:profile-loaded', true);
} }
}) })
.catch(() => { .catch((loadErr) => {
error('Profile load failed', 'Could not load administrator profile.'); error(
'Profile load failed',
loadErr instanceof Error ? loadErr.message : 'Could not load administrator profile.'
);
}) })
.finally(() => setLoading(false)); .finally(() => setLoading(false));
}, [error]); }, [error]);
@@ -78,6 +93,10 @@ export function ProfileWorkspace() {
error('Validation error', 'Enter your current password to save changes.'); error('Validation error', 'Enter your current password to save changes.');
return; return;
} }
if (newPassword && (newPassword.length < 8 || newPassword.length > 256)) {
error('Validation error', 'New password must be between 8 and 256 characters.');
return;
}
if (newPassword && newPassword !== confirmPassword) { if (newPassword && newPassword !== confirmPassword) {
error('Validation error', 'New password and confirmation do not match.'); error('Validation error', 'New password and confirmation do not match.');
return; return;
@@ -89,16 +108,24 @@ export function ProfileWorkspace() {
method: 'PUT', method: 'PUT',
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
credentials: 'include', credentials: 'include',
cache: 'no-store',
body: JSON.stringify({ body: JSON.stringify({
email: trimmedEmail, email: trimmedEmail,
currentPassword, currentPassword,
newPassword: newPassword || undefined, newPassword: newPassword || undefined,
}), }),
}); });
const data = await response.json(); const data = await response.json().catch(() => ({}));
if (shouldRedirectProfileUnauthorized(response.status, data)) {
window.location.assign('/vcon/login');
return;
}
if (!response.ok) throw new Error(data.error || 'Profile update failed'); if (!response.ok) throw new Error(data.error || 'Profile update failed');
setProfile(setCached('vcon:profile', data.profile)); setProfile(setCached('vcon:profile', data.profile));
setEmail(data.profile.email); setEmail(data.profile.email);
window.dispatchEvent(
new CustomEvent('datastack:profile-updated', { detail: data.profile })
);
setCurrentPassword(''); setCurrentPassword('');
setNewPassword(''); setNewPassword('');
setConfirmPassword(''); setConfirmPassword('');
@@ -123,6 +150,7 @@ export function ProfileWorkspace() {
credentials: 'include', credentials: 'include',
}); });
if (!response.ok) throw new Error('Unable to sign out'); if (!response.ok) throw new Error('Unable to sign out');
await clearAuthCaches();
window.location.assign('/vcon/login'); window.location.assign('/vcon/login');
} catch { } catch {
error('Sign out failed', 'Please try again.'); error('Sign out failed', 'Please try again.');
@@ -160,13 +188,7 @@ export function ProfileWorkspace() {
{ label: 'Auth Store', value: profile.storageMode }, { label: 'Auth Store', value: profile.storageMode },
{ {
label: 'Session Expires', label: 'Session Expires',
value: profile.sessionExpiresAt value: formatSessionExpiryLabel(profile.sessionExpiresAt),
? new Date(profile.sessionExpiresAt).toLocaleTimeString([], {
hour: '2-digit',
minute: '2-digit',
hour12: false,
})
: 'Active',
}, },
]} ]}
/> />
+3 -3
View File
@@ -7,8 +7,8 @@ export const SdkSection: React.FC = () => (
<SectionHeading index="02" label="Python SDK" title="Drop in the SDK. Start streaming." /> <SectionHeading index="02" label="Python SDK" title="Drop in the SDK. Start streaming." />
<div className="grid grid-cols-1 gap-px border border-[var(--border)] bg-[var(--border)] md:grid-cols-2"> <div className="grid grid-cols-1 gap-px border border-[var(--border)] bg-[var(--border)] md:grid-cols-2">
<CodeBlock> <CodeBlock>
<span className="text-[var(--text-faint)]"># Install the official Python SDK</span>{'\n'} <span className="text-[var(--text-faint)]"># Install the bundled Python SDK (from repo or local wheel/path)</span>{'\n'}
<span className="text-[var(--text)]">pip install ./sdk/python/datastack</span>{'\n\n'} <span className="text-[var(--text)]">pip install /path/to/datastack/sdk/python/datastack</span>{'\n\n'}
<span className="text-[var(--text-faint)]"># Auto-loads config.datastack.py from project root</span>{'\n'} <span className="text-[var(--text-faint)]"># Auto-loads config.datastack.py from project root</span>{'\n'}
<span className="text-[var(--text)]">from datastack import DataStack</span>{'\n\n'} <span className="text-[var(--text)]">from datastack import DataStack</span>{'\n\n'}
<span className="text-[var(--text)]">client = DataStack()</span>{'\n'} <span className="text-[var(--text)]">client = DataStack()</span>{'\n'}
@@ -20,7 +20,7 @@ export const SdkSection: React.FC = () => (
<p className="mb-2 text-[11px] uppercase text-[var(--text-faint)]">How it works</p> <p className="mb-2 text-[11px] uppercase text-[var(--text-faint)]">How it works</p>
<h3 className="mb-2 text-[14px] font-medium text-[var(--text)]">Zero-boilerplate config. Auto-recovery built in.</h3> <h3 className="mb-2 text-[14px] font-medium text-[var(--text)]">Zero-boilerplate config. Auto-recovery built in.</h3>
<p className="text-[12px] leading-relaxed text-[var(--text-muted)]"> <p className="text-[12px] leading-relaxed text-[var(--text-muted)]">
Drop <code className="text-[var(--text)]">config.datastack.py</code> into your Python project to stream single events, batches, or <code className="text-[var(--text)]">.csv</code> / <code className="text-[var(--text)]">.json</code> files directly—with automatic HTTP fallback, offline disk spooling, and Heartbeat IDLE recovery. Install the self-hosted SDK from <code className="text-[var(--text)]">sdk/python/datastack</code> (or build a wheel with <code className="text-[var(--text)]">pip wheel ./sdk/python/datastack -w dist/</code>) and drop <code className="text-[var(--text)]">config.datastack.py</code> into your Python project to stream single events, batches, or <code className="text-[var(--text)]">.csv</code> / <code className="text-[var(--text)]">.json</code> files directly—with automatic HTTP fallback, offline disk spooling, and Heartbeat IDLE recovery.
</p> </p>
</div> </div>
</div> </div>
+251 -168
View File
@@ -15,34 +15,31 @@ import {
import { StatStrip } from '@/components/ui/StatStrip'; import { StatStrip } from '@/components/ui/StatStrip';
import { useToast } from '@/components/ui/Toast'; import { useToast } from '@/components/ui/Toast';
import { getCached, setCached } from '@/lib/vconCache'; import { getCached, setCached } from '@/lib/vconCache';
import {
DEFAULT_EDITABLE_SETTINGS,
describeSettingsBridgeSync,
type EditableSettings,
mapBridgeStatusToSocketUiState,
normalizeEditableSettings,
NUMERIC_SETTING_KEYS,
type NumericSettingDrafts,
type NumericSettingKey,
resolvePublicEndpoints,
SETTINGS_LIMITS,
settingsToNumericDrafts,
type SettingsValidationErrors,
type SocketHealthUiState,
validateEditableSettingsDraft,
validateSingleNumericSetting,
} from '@/lib/vconUiHelpers';
import { useCallback, useEffect, useMemo, useState } from 'react'; import { useCallback, useEffect, useMemo, useState } from 'react';
interface EditableSettings {
maxEventsPerSec: number;
maxPayloadSizeKb: number;
batchFlushThreshold: number;
batchFlushIntervalMs: number;
fileRotationMb: number;
enableIdempotency: boolean;
autoPruneDays: number;
}
interface PublicSettings extends EditableSettings { interface PublicSettings extends EditableSettings {
serverPort: number; serverPort: number;
webPort: number; webPort: number;
updatedAt: string; updatedAt: string;
} }
const DEFAULT_SETTINGS: EditableSettings = {
maxEventsPerSec: 100,
maxPayloadSizeKb: 100,
batchFlushThreshold: 500,
batchFlushIntervalMs: 2000,
fileRotationMb: 100,
enableIdempotency: true,
autoPruneDays: 30,
};
const TABS = [ const TABS = [
{ id: 'general', label: 'General' }, { id: 'general', label: 'General' },
{ id: 'connections', label: 'Connections' }, { id: 'connections', label: 'Connections' },
@@ -51,16 +48,32 @@ const TABS = [
]; ];
type TabId = (typeof TABS)[number]['id']; type TabId = (typeof TABS)[number]['id'];
type SocketHealthState = 'idle' | 'checking' | 'online' | 'offline';
const FIELD_TO_TAB: Record<NumericSettingKey, TabId> = {
maxEventsPerSec: 'general',
batchFlushThreshold: 'connections',
batchFlushIntervalMs: 'connections',
fileRotationMb: 'storage',
autoPruneDays: 'storage',
maxPayloadSizeKb: 'security',
};
export function SettingsWorkspace() { export function SettingsWorkspace() {
const { success, error } = useToast(); const { success, warning, error } = useToast();
const [settings, setSettings] = useState<EditableSettings>(() =>
getCached<EditableSettings>('vcon:settings', DEFAULT_SETTINGS)
);
const [savedSettings, setSavedSettings] = useState<EditableSettings>(() => const [savedSettings, setSavedSettings] = useState<EditableSettings>(() =>
getCached<EditableSettings>('vcon:settings', DEFAULT_SETTINGS) getCached<EditableSettings>('vcon:settings', DEFAULT_EDITABLE_SETTINGS)
); );
const [numericDrafts, setNumericDrafts] = useState<NumericSettingDrafts>(() =>
settingsToNumericDrafts(
getCached<EditableSettings>('vcon:settings', DEFAULT_EDITABLE_SETTINGS)
)
);
const [enableIdempotency, setEnableIdempotency] = useState<boolean>(
() =>
getCached<EditableSettings>('vcon:settings', DEFAULT_EDITABLE_SETTINGS)
.enableIdempotency
);
const [fieldErrors, setFieldErrors] = useState<SettingsValidationErrors>({});
const [activeTab, setActiveTab] = useState<TabId>('general'); const [activeTab, setActiveTab] = useState<TabId>('general');
const [jwtSecret, setJwtSecret] = useState(''); const [jwtSecret, setJwtSecret] = useState('');
const [jwtError, setJwtError] = useState(''); const [jwtError, setJwtError] = useState('');
@@ -69,8 +82,8 @@ export function SettingsWorkspace() {
getCached<string>('vcon:storage-path', '') getCached<string>('vcon:storage-path', '')
); );
const [baseUrl, setBaseUrl] = useState(''); const [baseUrl, setBaseUrl] = useState('');
const [webSocketUrl, setWebSocketUrl] = useState(''); const [socketStatus, setSocketStatus] = useState<SocketHealthUiState>('idle');
const [socketStatus, setSocketStatus] = useState<SocketHealthState>('idle'); const [bridgeError, setBridgeError] = useState('');
const [activeSockets, setActiveSockets] = useState<number>(0); const [activeSockets, setActiveSockets] = useState<number>(0);
const [serverPort, setServerPort] = useState(() => const [serverPort, setServerPort] = useState(() =>
getCached<number>('vcon:server-port', 4000) getCached<number>('vcon:server-port', 4000)
@@ -80,48 +93,46 @@ export function SettingsWorkspace() {
); );
const [saving, setSaving] = useState(false); const [saving, setSaving] = useState(false);
const probeSocketServer = useCallback(async (port: number) => { const webSocketUrl = useMemo(() => {
setSocketStatus('checking'); if (!baseUrl || typeof window === 'undefined') return '';
const isLocalHost = return resolvePublicEndpoints(window.location, serverPort).webSocketUrl;
window.location.hostname === 'localhost' || }, [baseUrl, serverPort]);
window.location.hostname === '127.0.0.1' ||
window.location.port === '3000';
const httpHost = isLocalHost
? `${window.location.hostname}:${port}`
: window.location.host;
const healthUrl = `${window.location.protocol}//${httpHost}/health`;
const probeSocketServer = useCallback(async () => {
setSocketStatus('checking');
try { try {
const response = await fetch(healthUrl, { const response = await fetch('/api/metrics', {
cache: 'no-store', cache: 'no-store',
signal: AbortSignal.timeout(1500), signal: AbortSignal.timeout(3000),
}); });
setSocketStatus(response.ok ? 'online' : 'offline'); if (response.status === 401) {
} catch { window.location.assign('/vcon/login');
// In remote/preview environments where Port 4000 is proxied through web API or reverse proxy, probe /api/health return;
try { }
const fallbackRes = await fetch('/api/health', { if (!response.ok) {
cache: 'no-store', setSocketStatus('offline');
signal: AbortSignal.timeout(1500), setBridgeError(`Metrics probe returned HTTP ${response.status}`);
}); return;
setSocketStatus(fallbackRes.ok ? 'online' : 'offline'); }
const metricsData = await response.json();
const runtimePort = Number(metricsData?.runtime?.serverPort);
if (Number.isFinite(runtimePort) && runtimePort > 0) {
setServerPort(setCached('vcon:server-port', runtimePort));
}
setActiveSockets(Number(metricsData?.activeSockets) || 0);
const nextState = mapBridgeStatusToSocketUiState(
metricsData?.runtime?.ingestionBridgeStatus
);
setSocketStatus(nextState);
setBridgeError(String(metricsData?.runtime?.ingestionBridgeError || ''));
} catch { } catch {
setSocketStatus('offline'); setSocketStatus('offline');
} setBridgeError('Unable to reach /api/metrics bridge probe');
} }
}, []); }, []);
useEffect(() => { useEffect(() => {
setBaseUrl(window.location.origin); setBaseUrl(window.location.origin);
const isLocalHost =
window.location.hostname === 'localhost' ||
window.location.hostname === '127.0.0.1' ||
window.location.port === '3000';
const socketProtocol = window.location.protocol === 'https:' ? 'wss' : 'ws';
const socketHost = isLocalHost
? `${window.location.hostname}:${serverPort}`
: window.location.host;
setWebSocketUrl(`${socketProtocol}://${socketHost}`);
Promise.all([ Promise.all([
fetch('/api/settings', { cache: 'no-store' }).then(async (response) => { fetch('/api/settings', { cache: 'no-store' }).then(async (response) => {
@@ -141,46 +152,58 @@ export function SettingsWorkspace() {
return response.json(); return response.json();
}), }),
fetch('/api/metrics', { cache: 'no-store' }) fetch('/api/metrics', { cache: 'no-store' })
.then(async (response) => (response.ok ? response.json() : null)) .then(async (response) => {
if (response.status === 401) {
window.location.assign('/vcon/login');
return null;
}
return response.ok ? response.json() : null;
})
.catch(() => null), .catch(() => null),
]) ])
.then(([settingsData, storageData, metricsData]) => { .then(([settingsData, storageData, metricsData]) => {
if (!settingsData || !storageData) return; if (!settingsData || !storageData) return;
const loadedPort = Number( const runtimePort = Number(metricsData?.runtime?.serverPort);
const settingsPort = Number(
(settingsData.settings as PublicSettings).serverPort ?? 4000 (settingsData.settings as PublicSettings).serverPort ?? 4000
); );
const loadedPort =
Number.isFinite(runtimePort) && runtimePort > 0 ? runtimePort : settingsPort;
const loaded: EditableSettings = { const loaded: EditableSettings = {
maxEventsPerSec: Number( maxEventsPerSec: Number(
settingsData.settings.maxEventsPerSec ?? settingsData.settings.maxEventsPerSec ??
DEFAULT_SETTINGS.maxEventsPerSec DEFAULT_EDITABLE_SETTINGS.maxEventsPerSec
), ),
maxPayloadSizeKb: Number( maxPayloadSizeKb: Number(
settingsData.settings.maxPayloadSizeKb ?? settingsData.settings.maxPayloadSizeKb ??
DEFAULT_SETTINGS.maxPayloadSizeKb DEFAULT_EDITABLE_SETTINGS.maxPayloadSizeKb
), ),
batchFlushThreshold: Number( batchFlushThreshold: Number(
settingsData.settings.batchFlushThreshold ?? settingsData.settings.batchFlushThreshold ??
DEFAULT_SETTINGS.batchFlushThreshold DEFAULT_EDITABLE_SETTINGS.batchFlushThreshold
), ),
batchFlushIntervalMs: Number( batchFlushIntervalMs: Number(
settingsData.settings.batchFlushIntervalMs ?? settingsData.settings.batchFlushIntervalMs ??
DEFAULT_SETTINGS.batchFlushIntervalMs DEFAULT_EDITABLE_SETTINGS.batchFlushIntervalMs
), ),
fileRotationMb: Number( fileRotationMb: Number(
settingsData.settings.fileRotationMb ?? settingsData.settings.fileRotationMb ??
DEFAULT_SETTINGS.fileRotationMb DEFAULT_EDITABLE_SETTINGS.fileRotationMb
), ),
enableIdempotency: Boolean( enableIdempotency: Boolean(
settingsData.settings.enableIdempotency ?? settingsData.settings.enableIdempotency ??
DEFAULT_SETTINGS.enableIdempotency DEFAULT_EDITABLE_SETTINGS.enableIdempotency
), ),
autoPruneDays: Number( autoPruneDays: Number(
settingsData.settings.autoPruneDays ?? settingsData.settings.autoPruneDays ??
DEFAULT_SETTINGS.autoPruneDays DEFAULT_EDITABLE_SETTINGS.autoPruneDays
), ),
}; };
setSettings(setCached('vcon:settings', loaded)); const cachedLoaded = setCached('vcon:settings', loaded);
setSavedSettings(loaded); setSavedSettings(cachedLoaded);
setNumericDrafts(settingsToNumericDrafts(cachedLoaded));
setEnableIdempotency(cachedLoaded.enableIdempotency);
setFieldErrors({});
setServerPort(setCached('vcon:server-port', loadedPort)); setServerPort(setCached('vcon:server-port', loadedPort));
setStoragePath( setStoragePath(
setCached( setCached(
@@ -188,69 +211,51 @@ export function SettingsWorkspace() {
String(storageData.stats?.storageDirectory ?? '') String(storageData.stats?.storageDirectory ?? '')
) )
); );
if (metricsData && typeof metricsData.activeSockets === 'number') { if (metricsData) {
setActiveSockets(metricsData.activeSockets); setActiveSockets(Number(metricsData.activeSockets) || 0);
if (metricsData.activeSockets > 0) { setSocketStatus(
setSocketStatus('online'); mapBridgeStatusToSocketUiState(
} metricsData.runtime?.ingestionBridgeStatus
)
);
setBridgeError(String(metricsData.runtime?.ingestionBridgeError || ''));
} }
setCached('vcon:settings-loaded', true); setCached('vcon:settings-loaded', true);
}) })
.catch(() => { .catch(() => {
error('Settings load failed', 'Using default runtime settings.'); error('Settings load failed', 'Using last cached runtime settings.');
}) })
.finally(() => setLoading(false)); .finally(() => setLoading(false));
}, [error, serverPort]); }, [error]);
const isDirty = useMemo( const isDirty = useMemo(
() => () =>
jwtSecret.trim().length > 0 || jwtSecret.trim().length > 0 ||
JSON.stringify(settings) !== JSON.stringify(savedSettings), enableIdempotency !== savedSettings.enableIdempotency ||
[jwtSecret, savedSettings, settings] NUMERIC_SETTING_KEYS.some(
(key) => numericDrafts[key].trim() !== String(savedSettings[key])
),
[enableIdempotency, jwtSecret, numericDrafts, savedSettings]
); );
const updateNumber = (key: keyof EditableSettings, value: string) => { const updateNumberDraft = (key: NumericSettingKey, value: string) => {
setSettings((current) => ({ ...current, [key]: Number(value) || 0 })); setNumericDrafts((current) => ({
...current,
[key]: value,
}));
const check = validateSingleNumericSetting(key, value);
setFieldErrors((current) => {
if (check.valid) {
if (!current[key]) return current;
const next = { ...current };
delete next[key];
return next;
}
return { ...current, [key]: check.error };
});
}; };
const buildNormalizedSettings = (): EditableSettings => { const executeSave = async (validated: EditableSettings) => {
const parsedPrune = Number(settings.autoPruneDays);
return {
maxEventsPerSec: Math.max(
1,
Math.min(
10000,
settings.maxEventsPerSec || DEFAULT_SETTINGS.maxEventsPerSec
)
),
maxPayloadSizeKb: Math.max(
1,
Math.min(
5120,
settings.maxPayloadSizeKb || DEFAULT_SETTINGS.maxPayloadSizeKb
)
),
batchFlushThreshold: Math.max(
1,
settings.batchFlushThreshold || DEFAULT_SETTINGS.batchFlushThreshold
),
batchFlushIntervalMs: Math.max(
100,
settings.batchFlushIntervalMs || DEFAULT_SETTINGS.batchFlushIntervalMs
),
fileRotationMb: Math.max(
1,
settings.fileRotationMb || DEFAULT_SETTINGS.fileRotationMb
),
enableIdempotency: settings.enableIdempotency,
autoPruneDays: Number.isFinite(parsedPrune)
? Math.max(0, Math.min(365, Math.floor(parsedPrune)))
: DEFAULT_SETTINGS.autoPruneDays,
};
};
const executeSave = async () => {
const normalized = buildNormalizedSettings();
const trimmedSecret = jwtSecret.trim(); const trimmedSecret = jwtSecret.trim();
setConfirmRotateOpen(false); setConfirmRotateOpen(false);
setSaving(true); setSaving(true);
@@ -259,7 +264,7 @@ export function SettingsWorkspace() {
method: 'POST', method: 'POST',
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ body: JSON.stringify({
...normalized, ...validated,
...(trimmedSecret ? { jwtSecret: trimmedSecret } : {}), ...(trimmedSecret ? { jwtSecret: trimmedSecret } : {}),
}), }),
}); });
@@ -269,16 +274,36 @@ export function SettingsWorkspace() {
} }
const data = await response.json(); const data = await response.json();
if (!response.ok) throw new Error(data.error || 'Settings update failed'); if (!response.ok) throw new Error(data.error || 'Settings update failed');
setSettings(setCached('vcon:settings', normalized)); const synced = normalizeEditableSettings(data.settings ?? validated);
setSavedSettings(normalized); setCached('vcon:settings', synced);
setSavedSettings(synced);
setNumericDrafts(settingsToNumericDrafts(synced));
setEnableIdempotency(synced.enableIdempotency);
setFieldErrors({});
setJwtSecret(''); setJwtSecret('');
setJwtError(''); setJwtError('');
const bridgeSummary = describeSettingsBridgeSync(data.bridgeSync);
if (bridgeSummary.level === 'error') {
setSocketStatus('auth_error');
setBridgeError(bridgeSummary.message || '');
error(bridgeSummary.title, bridgeSummary.message);
} else if (bridgeSummary.level === 'warning') {
setSocketStatus('offline');
setBridgeError(bridgeSummary.message || '');
warning(bridgeSummary.title, bridgeSummary.message);
} else {
if (Boolean(data.bridgeSync?.ok ?? data.bridgeSync?.synced)) {
setSocketStatus('online');
setBridgeError('');
}
success( success(
'Settings saved', 'Settings saved',
trimmedSecret trimmedSecret
? 'Runtime settings saved and client JWT credentials rotated.' ? 'Runtime settings saved and client JWT credentials rotated.'
: 'Runtime settings updated.' : 'Runtime settings updated across Web & Ingestion.'
); );
}
} catch (saveError) { } catch (saveError) {
error( error(
'Save failed', 'Save failed',
@@ -293,6 +318,24 @@ export function SettingsWorkspace() {
const handleFormSubmit = async (event: React.FormEvent<HTMLFormElement>) => { const handleFormSubmit = async (event: React.FormEvent<HTMLFormElement>) => {
event.preventDefault(); event.preventDefault();
const validation = validateEditableSettingsDraft(
numericDrafts,
enableIdempotency
);
if (!validation.valid || !validation.values) {
setFieldErrors(validation.errors);
if (validation.firstInvalidKey) {
setActiveTab(FIELD_TO_TAB[validation.firstInvalidKey]);
}
const firstMessage =
(validation.firstInvalidKey &&
validation.errors[validation.firstInvalidKey]) ||
'Review highlighted numeric settings before saving.';
error('Invalid setting value', firstMessage);
return;
}
setFieldErrors({});
const trimmedSecret = jwtSecret.trim(); const trimmedSecret = jwtSecret.trim();
if (trimmedSecret.length > 0 && trimmedSecret.length < 32) { if (trimmedSecret.length > 0 && trimmedSecret.length < 32) {
const message = 'JWT secret must be at least 32 characters.'; const message = 'JWT secret must be at least 32 characters.';
@@ -308,30 +351,57 @@ export function SettingsWorkspace() {
return; return;
} }
await executeSave(); await executeSave(validation.values);
};
const handleConfirmedRotateSave = async () => {
const validation = validateEditableSettingsDraft(
numericDrafts,
enableIdempotency
);
if (!validation.valid || !validation.values) {
setConfirmRotateOpen(false);
setFieldErrors(validation.errors);
if (validation.firstInvalidKey) {
setActiveTab(FIELD_TO_TAB[validation.firstInvalidKey]);
}
return;
}
await executeSave(validation.values);
}; };
const discardChanges = () => { const discardChanges = () => {
setSettings(savedSettings); setNumericDrafts(settingsToNumericDrafts(savedSettings));
setEnableIdempotency(savedSettings.enableIdempotency);
setFieldErrors({});
setJwtSecret(''); setJwtSecret('');
setJwtError(''); setJwtError('');
}; };
const rateLimitStat = numericDrafts.maxEventsPerSec.trim() || '—';
const flushThresholdStat = numericDrafts.batchFlushThreshold.trim() || '—';
const flushIntervalStat = numericDrafts.batchFlushIntervalMs.trim() || '—';
const rotationStat = numericDrafts.fileRotationMb.trim() || '—';
const pruneStat =
numericDrafts.autoPruneDays.trim() === '0'
? 'Keep all'
: numericDrafts.autoPruneDays.trim()
? `${numericDrafts.autoPruneDays.trim()}d`
: '—';
return ( return (
<PageShell title="Settings"> <PageShell title="Settings">
<StatStrip <StatStrip
items={[ items={[
{ label: 'Instance', value: 'DataStack', dot: 'ok' }, { label: 'Instance', value: 'DataStack', dot: 'ok' },
{ label: 'Rate limit', value: `${settings.maxEventsPerSec} / sec` }, { label: 'Rate limit', value: `${rateLimitStat} / sec` },
{ {
label: 'Batch flush', label: 'Batch flush',
value: `${settings.batchFlushThreshold} / ${settings.batchFlushIntervalMs}ms`, value: `${flushThresholdStat} / ${flushIntervalStat}ms`,
}, },
{ {
label: 'CSV rotation', label: 'CSV rotation',
value: `${settings.fileRotationMb}MB · ${ value: `${rotationStat}MB · ${pruneStat}`,
settings.autoPruneDays === 0 ? 'Keep all' : `${settings.autoPruneDays}d`
}`,
}, },
]} ]}
/> />
@@ -343,7 +413,7 @@ export function SettingsWorkspace() {
onChange={(tab) => setActiveTab(tab as TabId)} onChange={(tab) => setActiveTab(tab as TabId)}
/> />
<form onSubmit={handleFormSubmit} className="w-full"> <form onSubmit={handleFormSubmit} noValidate className="w-full">
<SettingsPanel <SettingsPanel
id={`settings-panel-${activeTab}`} id={`settings-panel-${activeTab}`}
role="tabpanel" role="tabpanel"
@@ -374,24 +444,20 @@ export function SettingsWorkspace() {
<Input <Input
compact compact
type="number" type="number"
min="1" min={SETTINGS_LIMITS.maxEventsPerSec.min}
max="10000" max={SETTINGS_LIMITS.maxEventsPerSec.max}
value={settings.maxEventsPerSec} value={numericDrafts.maxEventsPerSec}
error={fieldErrors.maxEventsPerSec}
onChange={(event) => onChange={(event) =>
updateNumber('maxEventsPerSec', event.target.value) updateNumberDraft('maxEventsPerSec', event.target.value)
} }
/> />
</SettingsField> </SettingsField>
<SettingsField label="Idempotency"> <SettingsField label="Idempotency">
<FormSwitch <FormSwitch
label="Deduplicate events" label="Deduplicate events"
checked={settings.enableIdempotency} checked={enableIdempotency}
onChange={(enableIdempotency) => onChange={setEnableIdempotency}
setSettings((current) => ({
...current,
enableIdempotency,
}))
}
/> />
</SettingsField> </SettingsField>
</SettingsGroup> </SettingsGroup>
@@ -403,7 +469,10 @@ export function SettingsWorkspace() {
<SettingsGroup title="WebSocket server"> <SettingsGroup title="WebSocket server">
<SettingsField label="Live health"> <SettingsField label="Live health">
<div className="flex items-center justify-between gap-2 border border-[var(--border)] bg-[var(--surface-2)] px-2.5 py-1.5"> <div className="flex items-center justify-between gap-2 border border-[var(--border)] bg-[var(--surface-2)] px-2.5 py-1.5">
<span className="inline-flex items-center gap-1.5 font-mono text-[11.5px] text-[var(--text)]"> <span
className="inline-flex items-center gap-1.5 font-mono text-[11.5px] text-[var(--text)]"
title={bridgeError || undefined}
>
<span <span
aria-hidden="true" aria-hidden="true"
className={`h-1.5 w-1.5 ${ className={`h-1.5 w-1.5 ${
@@ -411,7 +480,7 @@ export function SettingsWorkspace() {
? 'bg-[var(--accent)]' ? 'bg-[var(--accent)]'
: socketStatus === 'checking' : socketStatus === 'checking'
? 'bg-[var(--warning)]' ? 'bg-[var(--warning)]'
: socketStatus === 'offline' : socketStatus === 'auth_error' || socketStatus === 'offline'
? 'bg-[var(--danger)]' ? 'bg-[var(--danger)]'
: 'bg-[var(--text-faint)]' : 'bg-[var(--text-faint)]'
}`} }`}
@@ -419,7 +488,9 @@ export function SettingsWorkspace() {
{socketStatus === 'online' {socketStatus === 'online'
? `Online (Port ${serverPort} · ${activeSockets} active)` ? `Online (Port ${serverPort} · ${activeSockets} active)`
: socketStatus === 'checking' : socketStatus === 'checking'
? 'Probing ingestion server...' ? 'Probing ingestion bridge...'
: socketStatus === 'auth_error'
? `Bridge Auth Error (Port ${serverPort})`
: socketStatus === 'offline' : socketStatus === 'offline'
? `Offline (Port ${serverPort} unreachable)` ? `Offline (Port ${serverPort} unreachable)`
: `Port ${serverPort} · ${activeSockets} active socket(s)`} : `Port ${serverPort} · ${activeSockets} active socket(s)`}
@@ -429,7 +500,7 @@ export function SettingsWorkspace() {
size="xs" size="xs"
variant="ghost" variant="ghost"
icon="RefreshCw" icon="RefreshCw"
onClick={() => void probeSocketServer(serverPort)} onClick={() => void probeSocketServer()}
> >
Probe Probe
</Button> </Button>
@@ -443,14 +514,18 @@ export function SettingsWorkspace() {
className="border-[var(--border)] bg-[var(--surface-2)] font-mono text-[11px] text-[var(--text-muted)]" className="border-[var(--border)] bg-[var(--surface-2)] font-mono text-[11px] text-[var(--text-muted)]"
/> />
</SettingsField> </SettingsField>
<SettingsField label="Socket port"> <SettingsField label="Socket port (PORT_SOCKET)">
<div className="flex flex-col gap-1">
<Input <Input
compact compact
type="number" value={String(serverPort)}
value={serverPort}
readOnly readOnly
className="border-[var(--border)] bg-[var(--surface-2)] font-mono text-[var(--text-muted)]" className="border-[var(--border)] bg-[var(--surface-2)] font-mono text-[11px] text-[var(--text-muted)]"
/> />
<span className="text-[10.5px] text-[var(--text-faint)]">
Read-only runtime port managed via PORT_SOCKET environment variable.
</span>
</div>
</SettingsField> </SettingsField>
</SettingsGroup> </SettingsGroup>
<SettingsGroup title="Event buffer"> <SettingsGroup title="Event buffer">
@@ -458,10 +533,12 @@ export function SettingsWorkspace() {
<Input <Input
compact compact
type="number" type="number"
min="1" min={SETTINGS_LIMITS.batchFlushThreshold.min}
value={settings.batchFlushThreshold} max={SETTINGS_LIMITS.batchFlushThreshold.max}
value={numericDrafts.batchFlushThreshold}
error={fieldErrors.batchFlushThreshold}
onChange={(event) => onChange={(event) =>
updateNumber('batchFlushThreshold', event.target.value) updateNumberDraft('batchFlushThreshold', event.target.value)
} }
/> />
</SettingsField> </SettingsField>
@@ -469,11 +546,13 @@ export function SettingsWorkspace() {
<Input <Input
compact compact
type="number" type="number"
min="100" min={SETTINGS_LIMITS.batchFlushIntervalMs.min}
max={SETTINGS_LIMITS.batchFlushIntervalMs.max}
suffix="ms" suffix="ms"
value={settings.batchFlushIntervalMs} value={numericDrafts.batchFlushIntervalMs}
error={fieldErrors.batchFlushIntervalMs}
onChange={(event) => onChange={(event) =>
updateNumber('batchFlushIntervalMs', event.target.value) updateNumberDraft('batchFlushIntervalMs', event.target.value)
} }
className="font-mono" className="font-mono"
/> />
@@ -497,11 +576,13 @@ export function SettingsWorkspace() {
<Input <Input
compact compact
type="number" type="number"
min="1" min={SETTINGS_LIMITS.fileRotationMb.min}
max={SETTINGS_LIMITS.fileRotationMb.max}
suffix="MB" suffix="MB"
value={settings.fileRotationMb} value={numericDrafts.fileRotationMb}
error={fieldErrors.fileRotationMb}
onChange={(event) => onChange={(event) =>
updateNumber('fileRotationMb', event.target.value) updateNumberDraft('fileRotationMb', event.target.value)
} }
className="font-mono" className="font-mono"
/> />
@@ -510,12 +591,13 @@ export function SettingsWorkspace() {
<Input <Input
compact compact
type="number" type="number"
min="0" min={SETTINGS_LIMITS.autoPruneDays.min}
max="365" max={SETTINGS_LIMITS.autoPruneDays.max}
suffix="days" suffix="days"
value={settings.autoPruneDays} value={numericDrafts.autoPruneDays}
error={fieldErrors.autoPruneDays}
onChange={(event) => onChange={(event) =>
updateNumber('autoPruneDays', event.target.value) updateNumberDraft('autoPruneDays', event.target.value)
} }
className="font-mono" className="font-mono"
/> />
@@ -557,12 +639,13 @@ export function SettingsWorkspace() {
<Input <Input
compact compact
type="number" type="number"
min="1" min={SETTINGS_LIMITS.maxPayloadSizeKb.min}
max="5120" max={SETTINGS_LIMITS.maxPayloadSizeKb.max}
suffix="KB" suffix="KB"
value={settings.maxPayloadSizeKb} value={numericDrafts.maxPayloadSizeKb}
error={fieldErrors.maxPayloadSizeKb}
onChange={(event) => onChange={(event) =>
updateNumber('maxPayloadSizeKb', event.target.value) updateNumberDraft('maxPayloadSizeKb', event.target.value)
} }
className="font-mono" className="font-mono"
/> />
@@ -628,7 +711,7 @@ export function SettingsWorkspace() {
variant="danger" variant="danger"
size="xs" size="xs"
loading={saving} loading={saving}
onClick={() => void executeSave()} onClick={() => void handleConfirmedRotateSave()}
> >
Rotate & Save Rotate & Save
</Button> </Button>
+70 -52
View File
@@ -9,41 +9,8 @@ type CoreDatabaseGlobals = typeof globalThis & {
const globals = globalThis as CoreDatabaseGlobals; const globals = globalThis as CoreDatabaseGlobals;
/** export const CORE_SCHEMA_TABLE_DDLS = [
* Database Module (PostgreSQL Persistence Engine) `
* CRITICAL RULE: PostgreSQL stores strictly administrative metadata,
* client registrations (client_sdks), and configuration (admin_settings).
* Collected telemetry streams are NEVER stored in PostgreSQL.
*/
export class DatabaseEngine {
public static getPool(): Pool | null {
const connectionString = process.env.DATABASE_URL;
if (!connectionString) {
return null;
}
if (globals.__dataStackCorePool === undefined) {
try {
globals.__dataStackCorePool = new Pool({
connectionString,
connectionTimeoutMillis: 3000,
idleTimeoutMillis: 30000,
max: 5,
});
} catch {
globals.__dataStackCorePool = null;
}
}
return globals.__dataStackCorePool ?? null;
}
public static async ensureSchema(): Promise<boolean> {
const pool = this.getPool();
if (!pool) return false;
globals.__dataStackCoreSchema ??= Promise.all([
pool.query(`
CREATE TABLE IF NOT EXISTS client_sdks ( CREATE TABLE IF NOT EXISTS client_sdks (
client_id TEXT PRIMARY KEY, client_id TEXT PRIMARY KEY,
name TEXT NOT NULL, name TEXT NOT NULL,
@@ -56,8 +23,8 @@ export class DatabaseEngine {
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
) )
`), `,
pool.query(` `
CREATE TABLE IF NOT EXISTS admin_settings ( CREATE TABLE IF NOT EXISTS admin_settings (
id TEXT PRIMARY KEY DEFAULT 'default', id TEXT PRIMARY KEY DEFAULT 'default',
server_port INTEGER NOT NULL DEFAULT 4000, server_port INTEGER NOT NULL DEFAULT 4000,
@@ -72,14 +39,14 @@ export class DatabaseEngine {
auto_prune_days INTEGER NOT NULL DEFAULT 30, auto_prune_days INTEGER NOT NULL DEFAULT 30,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
) )
`), `,
pool.query(` `
CREATE TABLE IF NOT EXISTS deleted_clients ( CREATE TABLE IF NOT EXISTS deleted_clients (
client_id TEXT PRIMARY KEY, client_id TEXT PRIMARY KEY,
deleted_at TIMESTAMPTZ NOT NULL DEFAULT NOW() deleted_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
) )
`), `,
pool.query(` `
CREATE TABLE IF NOT EXISTS admin_users ( CREATE TABLE IF NOT EXISTS admin_users (
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1), id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
email TEXT NOT NULL UNIQUE, email TEXT NOT NULL UNIQUE,
@@ -88,8 +55,8 @@ export class DatabaseEngine {
token_version INTEGER NOT NULL DEFAULT 1, token_version INTEGER NOT NULL DEFAULT 1,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
) )
`), `,
pool.query(` `
CREATE TABLE IF NOT EXISTS admin_login_attempts ( CREATE TABLE IF NOT EXISTS admin_login_attempts (
key_hash TEXT PRIMARY KEY, key_hash TEXT PRIMARY KEY,
window_started_at TIMESTAMPTZ NOT NULL, window_started_at TIMESTAMPTZ NOT NULL,
@@ -97,10 +64,11 @@ export class DatabaseEngine {
blocked_until TIMESTAMPTZ, blocked_until TIMESTAMPTZ,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
) )
`), `,
]).then(() => ] as const;
Promise.all([
pool.query(` export const CORE_SCHEMA_ALTER_DDLS = [
`
ALTER TABLE client_sdks ALTER TABLE client_sdks
ADD COLUMN IF NOT EXISTS rate_limit INTEGER NOT NULL DEFAULT 100, ADD COLUMN IF NOT EXISTS rate_limit INTEGER NOT NULL DEFAULT 100,
ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'active', ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'active',
@@ -108,8 +76,8 @@ export class DatabaseEngine {
ADD COLUMN IF NOT EXISTS last_active TIMESTAMPTZ, ADD COLUMN IF NOT EXISTS last_active TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
`), `,
pool.query(` `
ALTER TABLE admin_settings ALTER TABLE admin_settings
ADD COLUMN IF NOT EXISTS server_port INTEGER NOT NULL DEFAULT 4000, ADD COLUMN IF NOT EXISTS server_port INTEGER NOT NULL DEFAULT 4000,
ADD COLUMN IF NOT EXISTS web_port INTEGER NOT NULL DEFAULT 3000, ADD COLUMN IF NOT EXISTS web_port INTEGER NOT NULL DEFAULT 3000,
@@ -121,9 +89,59 @@ export class DatabaseEngine {
ADD COLUMN IF NOT EXISTS enable_idempotency BOOLEAN NOT NULL DEFAULT true, ADD COLUMN IF NOT EXISTS enable_idempotency BOOLEAN NOT NULL DEFAULT true,
ADD COLUMN IF NOT EXISTS auto_prune_days INTEGER NOT NULL DEFAULT 30, ADD COLUMN IF NOT EXISTS auto_prune_days INTEGER NOT NULL DEFAULT 30,
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
`), `,
]), `
).then(() => undefined); ALTER TABLE admin_users
ADD COLUMN IF NOT EXISTS token_version INTEGER NOT NULL DEFAULT 1
`,
] as const;
export async function ensureCoreSchemaOnPool(pool: Pool): Promise<void> {
await Promise.all(CORE_SCHEMA_TABLE_DDLS.map((sql) => pool.query(sql)));
await Promise.all(CORE_SCHEMA_ALTER_DDLS.map((sql) => pool.query(sql)));
}
/**
* Database Module (PostgreSQL Persistence Engine)
* CRITICAL RULE: PostgreSQL stores strictly administrative metadata,
* client registrations (client_sdks), and configuration (admin_settings).
* Collected telemetry streams are NEVER stored in PostgreSQL.
*/
export class DatabaseEngine {
public static getPool(): Pool | null {
const connectionString = process.env.DATABASE_URL?.trim();
if (!connectionString) {
return null;
}
if (globals.__dataStackCorePool === undefined) {
try {
const pool = new Pool({
connectionString,
connectionTimeoutMillis: 3000,
idleTimeoutMillis: 30000,
max: 10,
});
pool.on("error", (err) => {
console.error(
"[DataStack DB] Unexpected idle PostgreSQL pool error:",
err instanceof Error ? err.message : err,
);
});
globals.__dataStackCorePool = pool;
} catch {
globals.__dataStackCorePool = null;
}
}
return globals.__dataStackCorePool ?? null;
}
public static async ensureSchema(): Promise<boolean> {
const pool = this.getPool();
if (!pool) return false;
globals.__dataStackCoreSchema ??= ensureCoreSchemaOnPool(pool);
try { try {
await globals.__dataStackCoreSchema; await globals.__dataStackCoreSchema;
+206 -1
View File
@@ -1,5 +1,17 @@
import { createHash } from "node:crypto"; import { createHash } from "node:crypto";
import { AuthService } from "../auth/jwt"; import { AuthService } from "../auth/jwt";
import { DataBufferEngine, type RedisBufferClient } from "../data/buffer";
const REDIS_IDEMPOTENCY_PREFIX = "datastack:idem:";
const REDIS_RATE_LIMIT_PREFIX = "datastack:rl:";
export const REDIS_RATE_LIMIT_LUA_SCRIPT = `
-- DATASTACK_RATE_LIMIT_INCR
local c = redis.call("INCR", KEYS[1])
if c == 1 or redis.call("PTTL", KEYS[1]) < 0 then
redis.call("PEXPIRE", KEYS[1], tonumber(ARGV[1]))
end
return c
`.trim();
export function resolveInternalIngestionSecret( export function resolveInternalIngestionSecret(
jwtSecretOverride?: string, jwtSecretOverride?: string,
@@ -43,9 +55,15 @@ export interface ValidationResult {
| "MALFORMED_PAYLOAD"; | "MALFORMED_PAYLOAD";
} }
export type SecurityMode =
| "redis_shared"
| "bridge_coordinated"
| "degraded_local";
export class SecurityEngine { export class SecurityEngine {
public static maxEventsPerSec = 100; public static maxEventsPerSec = 100;
public static enableIdempotency = true; public static enableIdempotency = true;
private static lastSecurityMode: SecurityMode = "degraded_local";
// Idempotency cache: key -> timestamp (expires after TTL) // Idempotency cache: key -> timestamp (expires after TTL)
private static idempotencySet = new Map<string, number>(); private static idempotencySet = new Map<string, number>();
@@ -126,6 +144,32 @@ export class SecurityEngine {
return true; return true;
} }
private static trackLocalRateWindow(clientId: string): void {
const now = Date.now();
const current = this.rateLimitMap.get(clientId);
if (!current || now - current.windowStart >= 1000) {
this.rateLimitMap.set(clientId, { count: 1, windowStart: now });
} else {
current.count += 1;
}
}
private static supportsSharedSecurityRedis(
redis: RedisBufferClient | null,
): redis is RedisBufferClient & {
get: (key: string) => Promise<string | null>;
incr: (key: string) => Promise<number>;
pexpire: (key: string, ttlMs: number) => Promise<number>;
} {
return Boolean(
redis &&
typeof redis.get === "function" &&
typeof redis.incr === "function" &&
typeof redis.pexpire === "function" &&
typeof redis.set === "function",
);
}
/** /**
* Validates incoming event payload: * Validates incoming event payload:
* Checks duplicate idempotency (read-only), max byte size, and rate limit, * Checks duplicate idempotency (read-only), max byte size, and rate limit,
@@ -148,7 +192,7 @@ export class SecurityEngine {
// 2. Check payload size before consuming rate-limit quota // 2. Check payload size before consuming rate-limit quota
const payloadStr = const payloadStr =
typeof payload === "string" ? payload : JSON.stringify(payload); typeof payload === "string" ? payload : JSON.stringify(payload ?? {});
const byteLength = Buffer.byteLength(payloadStr, "utf8"); const byteLength = Buffer.byteLength(payloadStr, "utf8");
if (byteLength > this.maxPayloadSizeBytes) { if (byteLength > this.maxPayloadSizeBytes) {
@@ -178,6 +222,158 @@ export class SecurityEngine {
return { allowed: true }; return { allowed: true };
} }
/**
* Shared Redis-backed async validator with Port 4000 internal-bridge fallback
* (so HTTP + WS share rate limits & idempotency even when Redis is unconfigured)
* and final local-Map fallback for isolated single-process dev.
*/
public static async validateEventAsync(
clientId: string,
eventId: string,
payload: unknown,
clientMaxRate = 100,
): Promise<ValidationResult> {
const redis = DataBufferEngine.getRedisClient();
if (!this.supportsSharedSecurityRedis(redis)) {
const payloadStr =
typeof payload === "string" ? payload : JSON.stringify(payload ?? {});
const payloadBytes = Buffer.byteLength(payloadStr, "utf8");
if (payloadBytes > this.maxPayloadSizeBytes) {
return {
allowed: false,
reason: `Payload size (${(payloadBytes / 1024).toFixed(1)} KB) exceeds limit of ${this.maxPayloadSizeBytes / 1024} KB`,
code: "PAYLOAD_TOO_LARGE",
};
}
const headers = getInternalIngestionHeaders();
if (headers["x-datastack-internal-secret"]) {
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, "");
try {
const res = await fetch(`${ingestionBaseUrl}/internal/security-check`, {
method: "POST",
headers: {
"Content-Type": "application/json",
...headers,
},
body: JSON.stringify({
clientId,
eventId,
payloadBytes,
clientMaxRate,
}),
cache: "no-store",
signal: AbortSignal.timeout(400),
});
if (res.ok) {
const data = (await res.json()) as {
allowed?: boolean;
code?: ValidationResult["code"];
reason?: string;
};
this.lastSecurityMode = "bridge_coordinated";
return {
allowed: Boolean(data.allowed),
...(data.code ? { code: data.code } : {}),
...(data.reason ? { reason: data.reason } : {}),
};
}
} catch {
// Port 4000 unreachable — fall back to local validation
}
}
this.lastSecurityMode = "degraded_local";
return this.validateEvent(clientId, eventId, payload, clientMaxRate);
}
this.lastSecurityMode = "redis_shared";
const idemKey = `${REDIS_IDEMPOTENCY_PREFIX}${clientId}:${eventId}`;
try {
// 1. Read-only idempotency check
if (this.enableIdempotency && eventId) {
if (this.hasDuplicate(clientId, eventId)) {
return {
allowed: false,
reason: "Duplicate event dropped silently",
code: "DUPLICATE_EVENT",
};
}
const existing = await redis.get(idemKey);
if (existing !== null) {
this.recordIdempotency(clientId, eventId);
return {
allowed: false,
reason: "Duplicate event dropped silently",
code: "DUPLICATE_EVENT",
};
}
}
// 2. Payload byte size check before rate-limit consumption
const payloadStr =
typeof payload === "string" ? payload : JSON.stringify(payload ?? {});
const byteLength = Buffer.byteLength(payloadStr, "utf8");
if (byteLength > this.maxPayloadSizeBytes) {
return {
allowed: false,
reason: `Payload size (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${this.maxPayloadSizeBytes / 1024} KB`,
code: "PAYLOAD_TOO_LARGE",
};
}
// 3. Shared atomic per-second rate-limit counter in Redis (single Lua round-trip)
const effectiveRateLimit = Math.min(clientMaxRate, this.maxEventsPerSec);
const windowBucket = Math.floor(Date.now() / 1000);
const rlKey = `${REDIS_RATE_LIMIT_PREFIX}${clientId}:${windowBucket}`;
let count: number;
if (typeof redis.eval === "function") {
count = Number(
await redis.eval(REDIS_RATE_LIMIT_LUA_SCRIPT, 1, rlKey, "2000"),
);
} else {
count = Number(await redis.incr(rlKey));
if (count === 1) {
await redis.pexpire(rlKey, 2000);
}
}
this.trackLocalRateWindow(clientId);
if (count > effectiveRateLimit) {
return {
allowed: false,
reason: `Rate limit of ${effectiveRateLimit} events/sec exceeded for client ${clientId}`,
code: "RATE_LIMIT_EXCEEDED",
};
}
// 4. Atomic claim of idempotency key after all checks succeed
if (this.enableIdempotency && eventId) {
const claimed = await redis.set(
idemKey,
"1",
"PX",
this.IDEMPOTENCY_TTL_MS,
"NX",
);
this.recordIdempotency(clientId, eventId);
if (claimed !== "OK") {
return {
allowed: false,
reason: "Duplicate event dropped silently",
code: "DUPLICATE_EVENT",
};
}
}
return { allowed: true };
} catch {
return this.validateEvent(clientId, eventId, payload, clientMaxRate);
}
}
public static getStats() { public static getStats() {
const now = Date.now(); const now = Date.now();
for (const [k, timestamp] of this.idempotencySet.entries()) { for (const [k, timestamp] of this.idempotencySet.entries()) {
@@ -191,10 +387,19 @@ export class SecurityEngine {
} }
} }
const mode: SecurityMode = this.supportsSharedSecurityRedis(
DataBufferEngine.getRedisClient(),
)
? "redis_shared"
: this.lastSecurityMode === "redis_shared"
? "degraded_local"
: this.lastSecurityMode;
return { return {
activeIdempotencyKeys: this.idempotencySet.size, activeIdempotencyKeys: this.idempotencySet.size,
rateLimitedClientsCount: this.rateLimitMap.size, rateLimitedClientsCount: this.rateLimitMap.size,
maxPayloadSizeBytes: this.maxPayloadSizeBytes, maxPayloadSizeBytes: this.maxPayloadSizeBytes,
mode,
}; };
} }
} }
+328 -64
View File
@@ -1,5 +1,7 @@
import crypto from 'node:crypto';
import fs from 'fs'; import fs from 'fs';
import path from 'path'; import path from 'path';
import { StringDecoder } from 'node:string_decoder';
import { Parser } from 'json2csv'; import { Parser } from 'json2csv';
export interface DataEvent { export interface DataEvent {
@@ -34,9 +36,17 @@ function sleepMs(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms)); return new Promise((resolve) => setTimeout(resolve, ms));
} }
export function resolveDefaultStorageDir(): string {
const configured = (process.env.DATASTACK_STORAGE_DIR || '').trim();
if (configured) {
return path.resolve(configured);
}
return path.resolve(process.cwd(), 'storage', 'data');
}
export class CSVStorageManager { export class CSVStorageManager {
// Base storage directory: /storage/data // Base storage directory: DATASTACK_STORAGE_DIR or /storage/data
public static baseStorageDir = path.resolve(process.cwd(), 'storage', 'data'); public static baseStorageDir = resolveDefaultStorageDir();
public static maxFileSizeBytes = 100 * 1024 * 1024; // 100MB rotation limit public static maxFileSizeBytes = 100 * 1024 * 1024; // 100MB rotation limit
private static rowCountCache = new Map<string, RowCountCacheEntry>(); private static rowCountCache = new Map<string, RowCountCacheEntry>();
@@ -44,12 +54,94 @@ export class CSVStorageManager {
this.rowCountCache.clear(); this.rowCountCache.clear();
} }
public static sanitizeClientId(clientId: string): string {
const raw = String(clientId || 'unknown').trim().replace(/[^a-zA-Z0-9_-]/g, '_');
return raw || 'unknown';
}
/** /**
* Ensures client directory exists * Checks whether the last byte of a non-empty file is LF (\n)
*/
private static fileEndsWithNewline(filePath: string, sizeBytes: number): boolean {
if (sizeBytes <= 0) return false;
const fd = fs.openSync(filePath, 'r');
try {
const lastByte = Buffer.allocUnsafe(1);
const read = fs.readSync(fd, lastByte, 0, 1, sizeBytes - 1);
return read === 1 && lastByte[0] === 0x0a;
} finally {
fs.closeSync(fd);
}
}
private static removeStaleLockAtomically(lockPath: string): void {
try {
const lockStat = fs.lstatSync(lockPath);
if (Date.now() - lockStat.mtimeMs <= STALE_LOCK_AGE_MS) {
return;
}
const staleTmp = `${lockPath}.stale.${crypto.randomUUID()}`;
fs.renameSync(lockPath, staleTmp);
try {
fs.unlinkSync(staleTmp);
} catch {
// Ignore cleanup failure of already-renamed stale token
}
} catch {
// Lock was released or claimed concurrently
}
}
private static releaseLockIfOwned(lockPath: string, lockValue: string): void {
try {
if (!fs.existsSync(lockPath)) return;
const current = fs.readFileSync(lockPath, 'utf8');
if (current === lockValue) {
fs.unlinkSync(lockPath);
}
} catch {
// Ignore cleanup failure if already unlinked
}
}
private static async removeStaleLockAtomicallyAsync(lockPath: string): Promise<void> {
try {
const lockStat = await fs.promises.lstat(lockPath);
if (Date.now() - lockStat.mtimeMs <= STALE_LOCK_AGE_MS) {
return;
}
const staleTmp = `${lockPath}.stale.${crypto.randomUUID()}`;
await fs.promises.rename(lockPath, staleTmp);
await fs.promises.unlink(staleTmp).catch(() => undefined);
} catch {
// Lock was released or claimed concurrently
}
}
private static async releaseLockIfOwnedAsync(
lockPath: string,
lockValue: string,
): Promise<void> {
try {
const current = await fs.promises.readFile(lockPath, 'utf8');
if (current === lockValue) {
await fs.promises.unlink(lockPath);
}
} catch {
// Ignore cleanup failure if already unlinked
}
}
/**
* Ensures client directory exists safely within baseStorageDir
*/ */
public static ensureClientDir(clientId: string): string { public static ensureClientDir(clientId: string): string {
const safeClientId = clientId.replace(/[^a-zA-Z0-9_-]/g, '_'); const safeClientId = this.sanitizeClientId(clientId);
const clientDir = path.join(this.baseStorageDir, safeClientId); const resolvedBase = path.resolve(this.baseStorageDir);
const clientDir = path.resolve(resolvedBase, safeClientId);
if (!clientDir.startsWith(resolvedBase + path.sep)) {
throw new Error(`Invalid storage client directory: ${clientId}`);
}
if (!fs.existsSync(clientDir)) { if (!fs.existsSync(clientDir)) {
fs.mkdirSync(clientDir, { recursive: true }); fs.mkdirSync(clientDir, { recursive: true });
} }
@@ -67,6 +159,7 @@ export class CSVStorageManager {
const clientDir = this.ensureClientDir(clientId); const clientDir = this.ensureClientDir(clientId);
const date = this.extractSafePartitionDate(dateStr); const date = this.extractSafePartitionDate(dateStr);
const lockPath = path.join(clientDir, `.${date}.csv.lock`); const lockPath = path.join(clientDir, `.${date}.csv.lock`);
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
const deadline = Date.now() + LOCK_TIMEOUT_MS; const deadline = Date.now() + LOCK_TIMEOUT_MS;
let acquired = false; let acquired = false;
@@ -74,7 +167,7 @@ export class CSVStorageManager {
try { try {
const fd = fs.openSync(lockPath, 'wx'); const fd = fs.openSync(lockPath, 'wx');
try { try {
fs.writeFileSync(fd, `${process.pid}:${Date.now()}`, 'utf8'); fs.writeFileSync(fd, lockValue, 'utf8');
} finally { } finally {
fs.closeSync(fd); fs.closeSync(fd);
} }
@@ -85,15 +178,7 @@ export class CSVStorageManager {
if (code !== 'EEXIST') { if (code !== 'EEXIST') {
throw err; throw err;
} }
try { this.removeStaleLockAtomically(lockPath);
const lockStat = fs.statSync(lockPath);
if (Date.now() - lockStat.mtimeMs > STALE_LOCK_AGE_MS) {
fs.unlinkSync(lockPath);
continue;
}
} catch {
// Lock was released concurrently
}
const jitter = const jitter =
LOCK_RETRY_MIN_MS + LOCK_RETRY_MIN_MS +
Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS)); Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS));
@@ -108,13 +193,7 @@ export class CSVStorageManager {
try { try {
return await fn(); return await fn();
} finally { } finally {
try { this.releaseLockIfOwned(lockPath, lockValue);
if (fs.existsSync(lockPath)) {
fs.unlinkSync(lockPath);
}
} catch {
// Ignore cleanup failure if already unlinked
}
} }
} }
@@ -158,14 +237,13 @@ export class CSVStorageManager {
* Stateful RFC 4180 quote-aware chunked row counter so multi-100MB CSVs never load wholly into RAM * Stateful RFC 4180 quote-aware chunked row counter so multi-100MB CSVs never load wholly into RAM
*/ */
private static createChunkRowCounter() { private static createChunkRowCounter() {
const decoder = new StringDecoder('utf8');
let inQuotes = false; let inQuotes = false;
let pendingQuote = false; let pendingQuote = false;
let currentLineHasChars = false; let currentLineHasChars = false;
let totalRows = 0; let totalRows = 0;
return { const processStr = (str: string) => {
pushChunk(chunk: Buffer | string) {
const str = typeof chunk === 'string' ? chunk : chunk.toString('utf8');
for (let i = 0; i < str.length; i += 1) { for (let i = 0; i < str.length; i += 1) {
const ch = str[i]; const ch = str[i];
if (pendingQuote) { if (pendingQuote) {
@@ -201,8 +279,16 @@ export class CSVStorageManager {
currentLineHasChars = true; currentLineHasChars = true;
} }
} }
};
return {
pushChunk(chunk: Buffer | string) {
const str = typeof chunk === 'string' ? chunk : decoder.write(chunk);
if (str) processStr(str);
}, },
finish(): number { finish(): number {
const trailing = decoder.end();
if (trailing) processStr(trailing);
if (currentLineHasChars) { if (currentLineHasChars) {
totalRows += 1; totalRows += 1;
} }
@@ -243,7 +329,11 @@ export class CSVStorageManager {
/** /**
* Resolves target CSV file path with daily and configured MB rotation rules * Resolves target CSV file path with daily and configured MB rotation rules
*/ */
public static getTargetFilePath(clientId: string, dateStr?: string): string { public static getTargetFilePath(
clientId: string,
dateStr?: string,
incomingBytes = 0,
): string {
const clientDir = this.ensureClientDir(clientId); const clientDir = this.ensureClientDir(clientId);
const date = this.extractSafePartitionDate(dateStr); const date = this.extractSafePartitionDate(dateStr);
let baseFileName = `${date}.csv`; let baseFileName = `${date}.csv`;
@@ -252,18 +342,26 @@ export class CSVStorageManager {
// Check for file rotation // Check for file rotation
let partIndex = 1; let partIndex = 1;
while (fs.existsSync(targetPath)) { while (fs.existsSync(targetPath)) {
let stats: fs.Stats;
try { try {
const stats = fs.statSync(targetPath); stats = fs.statSync(targetPath);
if (stats.size >= this.maxFileSizeBytes) { } catch (err: unknown) {
if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') {
break;
}
throw err;
}
const overflowsLimit =
stats.size > 0 &&
incomingBytes > 0 &&
stats.size + incomingBytes > this.maxFileSizeBytes;
if (stats.size >= this.maxFileSizeBytes || overflowsLimit) {
partIndex += 1; partIndex += 1;
baseFileName = `${date}-part${partIndex}.csv`; baseFileName = `${date}-part${partIndex}.csv`;
targetPath = path.join(clientDir, baseFileName); targetPath = path.join(clientDir, baseFileName);
} else { } else {
break; break;
} }
} catch {
break;
}
} }
return targetPath; return targetPath;
@@ -280,32 +378,24 @@ export class CSVStorageManager {
return { written: 0, filesUpdated: [] }; return { written: 0, filesUpdated: [] };
} }
// Group events by client + date // Group events by client + date without ambiguous delimiter splitting
const grouped = new Map<string, DataEvent[]>(); const grouped = new Map<string, { clientId: string; date: string; events: DataEvent[] }>();
for (const evt of events) { for (const evt of events) {
const date = this.extractSafePartitionDate(evt.timestamp); const date = this.extractSafePartitionDate(evt.timestamp);
const key = `${evt.clientId}:::${date}`; const clientId = String(evt.clientId || 'unknown');
if (!grouped.has(key)) { const key = JSON.stringify([clientId, date]);
grouped.set(key, []); let bucket = grouped.get(key);
if (!bucket) {
bucket = { clientId, date, events: [] };
grouped.set(key, bucket);
} }
grouped.get(key)!.push(evt); bucket.events.push(evt);
} }
const filesUpdated: string[] = []; const filesUpdated: string[] = [];
const fields = ['clientId', 'eventId', 'timestamp', 'payload']; const fields = ['clientId', 'eventId', 'timestamp', 'payload'];
for (const [key, batch] of grouped.entries()) { for (const { clientId, date, events: batch } of grouped.values()) {
const [clientId, date] = key.split(':::');
const targetPath = await this.withPartitionLock(clientId, date, () => {
const resolvedPath = this.getTargetFilePath(clientId, date);
let hasContent = false;
try {
hasContent =
fs.existsSync(resolvedPath) && fs.statSync(resolvedPath).size > 0;
} catch {
hasContent = false;
}
const cleanBatch = batch.map((item) => ({ const cleanBatch = batch.map((item) => ({
clientId: item.clientId, clientId: item.clientId,
eventId: item.eventId, eventId: item.eventId,
@@ -316,13 +406,35 @@ export class CSVStorageManager {
: JSON.stringify(item.payload), : JSON.stringify(item.payload),
})); }));
const parser = new Parser({ fields, header: !hasContent, eol: '\n' }); const bodyCsvNoHeader = new Parser({ fields, header: false, eol: '\n' }).parse(
const csvData = parser.parse(cleanBatch); cleanBatch,
);
const incomingBytes = Buffer.byteLength(bodyCsvNoHeader, 'utf8') + 1;
const targetPath = await this.withPartitionLock(clientId, date, () => {
const resolvedPath = this.getTargetFilePath(clientId, date, incomingBytes);
let sizeBytes = 0;
try {
if (fs.existsSync(resolvedPath)) {
sizeBytes = fs.statSync(resolvedPath).size;
}
} catch {
sizeBytes = 0;
}
const hasContent = sizeBytes > 0;
if (hasContent) { if (hasContent) {
fs.appendFileSync(resolvedPath, '\n' + csvData, 'utf8'); const endsWithNl = this.fileEndsWithNewline(resolvedPath, sizeBytes);
fs.appendFileSync(
resolvedPath,
(endsWithNl ? '' : '\n') + bodyCsvNoHeader,
'utf8',
);
} else { } else {
fs.writeFileSync(resolvedPath, csvData, 'utf8'); const fullCsv = new Parser({ fields, header: true, eol: '\n' }).parse(
cleanBatch,
);
fs.writeFileSync(resolvedPath, fullCsv, 'utf8');
} }
// Invalidate row cache for modified file immediately // Invalidate row cache for modified file immediately
@@ -351,13 +463,25 @@ export class CSVStorageManager {
const clientDirs = fs.readdirSync(this.baseStorageDir); const clientDirs = fs.readdirSync(this.baseStorageDir);
for (const clientDir of clientDirs) { for (const clientDir of clientDirs) {
const fullClientPath = path.join(this.baseStorageDir, clientDir); const fullClientPath = path.join(this.baseStorageDir, clientDir);
if (!fs.statSync(fullClientPath).isDirectory()) continue; let dirLstat: fs.Stats;
try {
dirLstat = fs.lstatSync(fullClientPath);
} catch {
continue;
}
if (dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
const files = fs.readdirSync(fullClientPath); const files = fs.readdirSync(fullClientPath);
for (const file of files) { for (const file of files) {
if (!file.endsWith('.csv')) continue; if (!file.endsWith('.csv')) continue;
const fullFilePath = path.join(fullClientPath, file); const fullFilePath = path.join(fullClientPath, file);
const stats = fs.statSync(fullFilePath); let stats: fs.Stats;
try {
stats = fs.lstatSync(fullFilePath);
} catch {
continue;
}
if (stats.isSymbolicLink() || !stats.isFile()) continue;
activePaths.add(fullFilePath); activePaths.add(fullFilePath);
const cached = this.rowCountCache.get(fullFilePath); const cached = this.rowCountCache.get(fullFilePath);
@@ -422,15 +546,15 @@ export class CSVStorageManager {
const clientDirs = await fs.promises.readdir(this.baseStorageDir); const clientDirs = await fs.promises.readdir(this.baseStorageDir);
for (const clientDir of clientDirs) { for (const clientDir of clientDirs) {
const fullClientPath = path.join(this.baseStorageDir, clientDir); const fullClientPath = path.join(this.baseStorageDir, clientDir);
const dirStat = await fs.promises.stat(fullClientPath).catch(() => null); const dirLstat = await fs.promises.lstat(fullClientPath).catch(() => null);
if (!dirStat || !dirStat.isDirectory()) continue; if (!dirLstat || dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
const files = await fs.promises.readdir(fullClientPath); const files = await fs.promises.readdir(fullClientPath);
for (const file of files) { for (const file of files) {
if (!file.endsWith('.csv')) continue; if (!file.endsWith('.csv')) continue;
const fullFilePath = path.join(fullClientPath, file); const fullFilePath = path.join(fullClientPath, file);
const stats = await fs.promises.stat(fullFilePath).catch(() => null); const stats = await fs.promises.lstat(fullFilePath).catch(() => null);
if (!stats) continue; if (!stats || stats.isSymbolicLink() || !stats.isFile()) continue;
activePaths.add(fullFilePath); activePaths.add(fullFilePath);
const cached = this.rowCountCache.get(fullFilePath); const cached = this.rowCountCache.get(fullFilePath);
@@ -481,7 +605,7 @@ export class CSVStorageManager {
} }
/** /**
* Prune CSV partitions older than autoPruneDays without counting row contents * Prune CSV partitions older than autoPruneDays under cooperative partition locks
*/ */
public static pruneExpiredPartitions(autoPruneDays: number): number { public static pruneExpiredPartitions(autoPruneDays: number): number {
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) { if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
@@ -502,7 +626,13 @@ export class CSVStorageManager {
const clientDirs = fs.readdirSync(this.baseStorageDir); const clientDirs = fs.readdirSync(this.baseStorageDir);
for (const clientDir of clientDirs) { for (const clientDir of clientDirs) {
const fullClientPath = path.join(this.baseStorageDir, clientDir); const fullClientPath = path.join(this.baseStorageDir, clientDir);
if (!fs.statSync(fullClientPath).isDirectory()) continue; let dirLstat: fs.Stats;
try {
dirLstat = fs.lstatSync(fullClientPath);
} catch {
continue;
}
if (dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
const files = fs.readdirSync(fullClientPath); const files = fs.readdirSync(fullClientPath);
for (const file of files) { for (const file of files) {
@@ -510,6 +640,45 @@ export class CSVStorageManager {
const datePart = file.replace('.csv', '').split('-part')[0]; const datePart = file.replace('.csv', '').split('-part')[0];
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) { if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
const fullPath = path.join(fullClientPath, file); const fullPath = path.join(fullClientPath, file);
let fileLstat: fs.Stats;
try {
fileLstat = fs.lstatSync(fullPath);
} catch {
continue;
}
if (fileLstat.isSymbolicLink() || !fileLstat.isFile()) continue;
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
let acquired = false;
try {
const fd = fs.openSync(lockPath, 'wx');
try {
fs.writeFileSync(fd, lockValue, 'utf8');
} finally {
fs.closeSync(fd);
}
acquired = true;
} catch (err: unknown) {
if ((err as NodeJS.ErrnoException)?.code === 'EEXIST') {
this.removeStaleLockAtomically(lockPath);
try {
const fd = fs.openSync(lockPath, 'wx');
try {
fs.writeFileSync(fd, lockValue, 'utf8');
} finally {
fs.closeSync(fd);
}
acquired = true;
} catch {
// Partition is currently locked by an active writer; skip safely
acquired = false;
}
}
}
if (!acquired) continue;
try { try {
if (fs.existsSync(fullPath)) { if (fs.existsSync(fullPath)) {
fs.unlinkSync(fullPath); fs.unlinkSync(fullPath);
@@ -518,6 +687,8 @@ export class CSVStorageManager {
} }
} catch { } catch {
// Ignore file removal error // Ignore file removal error
} finally {
this.releaseLockIfOwned(lockPath, lockValue);
} }
} }
} }
@@ -529,7 +700,96 @@ export class CSVStorageManager {
return prunedCount; return prunedCount;
} }
private static formatStorageStats(files: StorageFileMeta[]) { /**
* Non-blocking async partition pruner using fs.promises under cooperative partition locks
*/
public static async pruneExpiredPartitionsAsync(
autoPruneDays: number,
): Promise<number> {
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
return 0;
}
const baseStat = await fs.promises.lstat(this.baseStorageDir).catch(() => null);
if (!baseStat || !baseStat.isDirectory()) {
return 0;
}
const cutoffDate = new Date(
Date.now() - Math.floor(autoPruneDays) * 86_400_000,
)
.toISOString()
.split('T')[0];
let prunedCount = 0;
try {
const clientDirs = await fs.promises.readdir(this.baseStorageDir);
for (const clientDir of clientDirs) {
const fullClientPath = path.join(this.baseStorageDir, clientDir);
const dirLstat = await fs.promises.lstat(fullClientPath).catch(() => null);
if (!dirLstat || dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) {
continue;
}
const files = await fs.promises.readdir(fullClientPath).catch(() => [] as string[]);
for (const file of files) {
if (!file.endsWith('.csv')) continue;
const datePart = file.replace('.csv', '').split('-part')[0];
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
const fullPath = path.join(fullClientPath, file);
const fileLstat = await fs.promises.lstat(fullPath).catch(() => null);
if (!fileLstat || fileLstat.isSymbolicLink() || !fileLstat.isFile()) {
continue;
}
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
let acquired = false;
const tryAcquireLock = async (): Promise<boolean> => {
const handle = await fs.promises.open(lockPath, 'wx');
try {
await handle.writeFile(lockValue, 'utf8');
} finally {
await handle.close();
}
return true;
};
try {
acquired = await tryAcquireLock();
} catch (err: unknown) {
if ((err as NodeJS.ErrnoException)?.code === 'EEXIST') {
await this.removeStaleLockAtomicallyAsync(lockPath);
try {
acquired = await tryAcquireLock();
} catch {
acquired = false;
}
}
}
if (!acquired) continue;
try {
await fs.promises.unlink(fullPath);
this.rowCountCache.delete(fullPath);
prunedCount += 1;
} catch {
// Ignore individual unlink error or concurrent removal
} finally {
await this.releaseLockIfOwnedAsync(lockPath, lockValue);
}
}
}
}
} catch {
// Ignore directory scan failure
}
return prunedCount;
}
public static formatStorageStats(files: StorageFileMeta[]) {
const totalBytes = files.reduce((acc, f) => acc + f.sizeBytes, 0); const totalBytes = files.reduce((acc, f) => acc + f.sizeBytes, 0);
const totalRows = files.reduce((acc, f) => acc + f.linesCount, 0); const totalRows = files.reduce((acc, f) => acc + f.linesCount, 0);
const uniqueClients = new Set(files.map((f) => f.clientId)).size; const uniqueClients = new Set(files.map((f) => f.clientId)).size;
@@ -569,17 +829,21 @@ export class CSVStorageManager {
* Reads file content for export / download with path traversal protection * Reads file content for export / download with path traversal protection
*/ */
public static getFileContent(clientId: string, filename: string): string | null { public static getFileContent(clientId: string, filename: string): string | null {
const safeClientId = String(clientId || '').replace(/[^a-zA-Z0-9_-]/g, '_'); const safeClientId = this.sanitizeClientId(clientId);
const safeFilename = path.basename(String(filename || '')); const safeFilename = path.basename(String(filename || ''));
if (!safeClientId || !safeFilename || !safeFilename.endsWith('.csv')) { if (!safeClientId || !safeFilename || !safeFilename.endsWith('.csv')) {
return null; return null;
} }
const fullPath = path.resolve(this.baseStorageDir, safeClientId, safeFilename);
const resolvedBase = path.resolve(this.baseStorageDir); const resolvedBase = path.resolve(this.baseStorageDir);
const fullPath = path.resolve(resolvedBase, safeClientId, safeFilename);
if (!fullPath.startsWith(resolvedBase + path.sep)) { if (!fullPath.startsWith(resolvedBase + path.sep)) {
return null; return null;
} }
if (fs.existsSync(fullPath)) { if (fs.existsSync(fullPath)) {
const stats = fs.lstatSync(fullPath);
if (stats.isSymbolicLink() || !stats.isFile()) {
return null;
}
return fs.readFileSync(fullPath, 'utf8'); return fs.readFileSync(fullPath, 'utf8');
} }
return null; return null;
+199 -22
View File
@@ -22,11 +22,23 @@ export interface AdminSettings {
updatedAt: string; updatedAt: string;
} }
export interface SettingsBridgeSyncStatus {
attempted: boolean;
ok: boolean;
synced: boolean;
status?: number;
error?: string;
checkedAt: string;
}
type SettingsGlobals = typeof globalThis & { type SettingsGlobals = typeof globalThis & {
__dataStackAdminSettings?: AdminSettings; __dataStackAdminSettings?: AdminSettings;
__dataStackSettingsHydrated?: boolean; __dataStackSettingsHydrated?: boolean;
__dataStackLastSettingsBridgeSync?: SettingsBridgeSyncStatus;
__dataStackLastPruneMeta?: { days: number; ranAtMs: number };
}; };
const PRUNE_COOLDOWN_MS = 60 * 60 * 1000;
const globals = globalThis as SettingsGlobals; const globals = globalThis as SettingsGlobals;
function getDefaultSettings(): AdminSettings { function getDefaultSettings(): AdminSettings {
@@ -53,7 +65,7 @@ function getSettingsState(): AdminSettings {
} }
export class SettingsManager { export class SettingsManager {
private static applyRuntimeSettings(settings: AdminSettings) { private static applyRuntimeSettings(settings: AdminSettings, forcePrune = false) {
SecurityEngine.maxEventsPerSec = settings.maxEventsPerSec; SecurityEngine.maxEventsPerSec = settings.maxEventsPerSec;
SecurityEngine.enableIdempotency = settings.enableIdempotency; SecurityEngine.enableIdempotency = settings.enableIdempotency;
SecurityEngine.maxPayloadSizeBytes = settings.maxPayloadSizeKb * 1024; SecurityEngine.maxPayloadSizeBytes = settings.maxPayloadSizeKb * 1024;
@@ -68,7 +80,27 @@ export class SettingsManager {
settings.autoPruneDays > 0 && settings.autoPruneDays > 0 &&
(!hasConfiguredDb || globals.__dataStackSettingsHydrated === true) (!hasConfiguredDb || globals.__dataStackSettingsHydrated === true)
) { ) {
CSVStorageManager.pruneExpiredPartitions(settings.autoPruneDays); const prev = globals.__dataStackLastPruneMeta;
const now = Date.now();
const shouldPrune =
forcePrune ||
!prev ||
prev.days !== settings.autoPruneDays ||
now - prev.ranAtMs >= PRUNE_COOLDOWN_MS;
if (shouldPrune) {
globals.__dataStackLastPruneMeta = {
days: settings.autoPruneDays,
ranAtMs: now,
};
void CSVStorageManager.pruneExpiredPartitionsAsync(
settings.autoPruneDays,
).catch((err) => {
console.warn(
"[DataStack Storage] Background async partition prune failed:",
err,
);
});
}
} }
} }
@@ -94,9 +126,17 @@ export class SettingsManager {
row.auto_prune_days !== undefined && row.auto_prune_days !== null row.auto_prune_days !== undefined && row.auto_prune_days !== null
? Number(row.auto_prune_days) ? Number(row.auto_prune_days)
: NaN; : NaN;
const envSocketPort = Number(process.env.PORT_SOCKET);
const envWebPort = Number(process.env.PORT);
const loaded: AdminSettings = { const loaded: AdminSettings = {
serverPort: Number(row.server_port) || 4000, serverPort:
webPort: Number(row.web_port) || 3000, Number.isFinite(envSocketPort) && envSocketPort > 0
? envSocketPort
: Number(row.server_port) || 4000,
webPort:
Number.isFinite(envWebPort) && envWebPort > 0
? envWebPort
: Number(row.web_port) || 3000,
maxEventsPerSec: Number(row.max_events_per_sec) || 100, maxEventsPerSec: Number(row.max_events_per_sec) || 100,
maxPayloadSizeKb: Number(row.max_payload_size_kb) || 100, maxPayloadSizeKb: Number(row.max_payload_size_kb) || 100,
batchFlushThreshold: Number(row.batch_flush_threshold) || 500, batchFlushThreshold: Number(row.batch_flush_threshold) || 500,
@@ -117,6 +157,12 @@ export class SettingsManager {
// Initialize default row in PostgreSQL if not present yet // Initialize default row in PostgreSQL if not present yet
const defaults = getSettingsState(); const defaults = getSettingsState();
const hasValidProductionJwt =
process.env.NODE_ENV !== "production" ||
defaults.jwtSecret.trim().length >= 32;
if (!hasValidProductionJwt) {
return defaults;
}
await this.persistSettingsToDb(defaults); await this.persistSettingsToDb(defaults);
globals.__dataStackSettingsHydrated = true; globals.__dataStackSettingsHydrated = true;
return defaults; return defaults;
@@ -131,8 +177,23 @@ export class SettingsManager {
const pool = DatabaseEngine.getPool(); const pool = DatabaseEngine.getPool();
if (!pool) return; if (!pool) return;
if (
process.env.NODE_ENV === "production" &&
settings.jwtSecret.trim().length < 32
) {
throw new Error(
"DATABASE_WRITE_FAILED: Cannot persist settings without a valid DATASTACK_JWT_SECRET (>= 32 characters) in production.",
);
}
const ready = await DatabaseEngine.ensureSchema();
if (!ready) {
throw new Error(
"DATABASE_WRITE_FAILED: PostgreSQL schema is unavailable while saving settings.",
);
}
try { try {
await DatabaseEngine.ensureSchema();
await pool.query( await pool.query(
`INSERT INTO admin_settings ( `INSERT INTO admin_settings (
id, server_port, web_port, max_events_per_sec, max_payload_size_kb, id, server_port, web_port, max_events_per_sec, max_payload_size_kb,
@@ -164,8 +225,9 @@ export class SettingsManager {
settings.autoPruneDays, settings.autoPruneDays,
], ],
); );
} catch { } catch (err: unknown) {
// Ignore DB failure; in-memory state remains active const detail = err instanceof Error ? err.message : "Unknown DB error";
throw new Error(`DATABASE_WRITE_FAILED: Failed to persist settings (${detail})`);
} }
} }
@@ -222,20 +284,35 @@ export class SettingsManager {
}; };
} }
public static getLastBridgeSyncStatus(): SettingsBridgeSyncStatus | null {
return globals.__dataStackLastSettingsBridgeSync ?? null;
}
private static async notifyIngestionSettingsSync( private static async notifyIngestionSettingsSync(
settings: AdminSettings, settings: AdminSettings,
authJwtSecret: string, authJwtSecret: string,
jwtSecretChanged: boolean, jwtSecretChanged: boolean,
): Promise<void> { ): Promise<SettingsBridgeSyncStatus> {
const checkedAt = new Date().toISOString();
const headers = getInternalIngestionHeaders(authJwtSecret); const headers = getInternalIngestionHeaders(authJwtSecret);
if (!headers["x-datastack-internal-secret"]) return; if (!headers["x-datastack-internal-secret"]) {
const result: SettingsBridgeSyncStatus = {
attempted: true,
ok: false,
synced: false,
error: "Internal bridge secret is not configured",
checkedAt,
};
globals.__dataStackLastSettingsBridgeSync = result;
return result;
}
const ingestionBaseUrl = ( const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000" process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, ""); ).replace(/\/$/, "");
try { try {
await fetch(`${ingestionBaseUrl}/internal/settings-sync`, { const res = await fetch(`${ingestionBaseUrl}/internal/settings-sync`, {
method: "POST", method: "POST",
headers: { headers: {
"Content-Type": "application/json", "Content-Type": "application/json",
@@ -254,8 +331,46 @@ export class SettingsManager {
cache: "no-store", cache: "no-store",
signal: AbortSignal.timeout(1500), signal: AbortSignal.timeout(1500),
}); });
} catch { if (!res.ok) {
// Port 4000 may not be running in isolated single-process dev mode const errorMsg = `Port 4000 settings-sync failed with HTTP ${res.status}`;
console.warn(`[DataStack Bridge] ${errorMsg}`);
const result: SettingsBridgeSyncStatus = {
attempted: true,
ok: false,
synced: false,
status: res.status,
error: errorMsg,
checkedAt,
};
globals.__dataStackLastSettingsBridgeSync = result;
return result;
}
const result: SettingsBridgeSyncStatus = {
attempted: true,
ok: true,
synced: true,
status: res.status,
checkedAt,
};
globals.__dataStackLastSettingsBridgeSync = result;
return result;
} catch (err: unknown) {
const message = err instanceof Error ? err.message : "Port 4000 settings-sync unreachable";
if (
process.env.NODE_ENV === "production" ||
Boolean(process.env.INGESTION_INTERNAL_URL)
) {
console.warn(`[DataStack Bridge] ${message}`);
}
const result: SettingsBridgeSyncStatus = {
attempted: true,
ok: false,
synced: false,
error: message,
checkedAt,
};
globals.__dataStackLastSettingsBridgeSync = result;
return result;
} }
} }
@@ -270,11 +385,13 @@ export class SettingsManager {
updatedAt: new Date().toISOString(), updatedAt: new Date().toISOString(),
}; };
globals.__dataStackAdminSettings = nextSettings; globals.__dataStackAdminSettings = nextSettings;
this.applyRuntimeSettings(nextSettings); this.applyRuntimeSettings(nextSettings, updates.autoPruneDays !== undefined);
if (jwtSecretChanged) { if (jwtSecretChanged) {
SDKManager.resignAllClientTokens(); SDKManager.resignAllClientTokens();
} }
void this.persistSettingsToDb(nextSettings); void this.persistSettingsToDb(nextSettings).catch((err) => {
console.error("[DataStack Settings] Background DB persist failed:", err);
});
void this.notifyIngestionSettingsSync( void this.notifyIngestionSettingsSync(
nextSettings, nextSettings,
previousJwtSecret, previousJwtSecret,
@@ -297,20 +414,80 @@ export class SettingsManager {
...updates, ...updates,
updatedAt: new Date().toISOString(), updatedAt: new Date().toISOString(),
}; };
globals.__dataStackAdminSettings = nextSettings;
this.applyRuntimeSettings(nextSettings); if (
if (jwtSecretChanged) { jwtSecretChanged &&
await SDKManager.resignAllClientTokensAsync(); process.env.NODE_ENV === "production" &&
nextSettings.jwtSecret.trim().length < 32
) {
throw new Error(
"INVALID_JWT_SECRET: JWT secret must be at least 32 characters in production.",
);
} }
if (jwtSecretChanged) {
// BUG-17: Coordinate Port 4000 BEFORE committing JWT rotation on Port 3000
// so a bridge rejection or unreachable production ingestion server never causes secret split-brain.
const preSyncStatus = await this.notifyIngestionSettingsSync(
nextSettings,
previousJwtSecret,
true,
);
if (!preSyncStatus.synced) {
const isExplicitHttpRejection = typeof preSyncStatus.status === "number";
const isProductionBridgeRequired =
process.env.NODE_ENV === "production" &&
Boolean(process.env.INGESTION_INTERNAL_URL);
if (isExplicitHttpRejection || isProductionBridgeRequired) {
throw new Error(
`JWT_ROTATION_BRIDGE_FAILED: Aborted JWT rotation because Port 4000 could not synchronize (${preSyncStatus.error || "bridge error"}).`,
);
}
}
try {
await this.persistSettingsToDb(nextSettings); await this.persistSettingsToDb(nextSettings);
globals.__dataStackAdminSettings = nextSettings;
this.applyRuntimeSettings(nextSettings, updates.autoPruneDays !== undefined);
await SDKManager.resignAllClientTokensAsync();
} catch (err) {
globals.__dataStackAdminSettings = current;
this.applyRuntimeSettings(current, false);
await this.persistSettingsToDb(current).catch(() => undefined);
if (preSyncStatus.synced) {
await this.notifyIngestionSettingsSync(
current,
nextSettings.jwtSecret,
true,
).catch(() => undefined);
}
throw err;
}
const clientSync = await SDKManager.syncAllClientsToIngestionAsync();
if (!clientSync.synced && preSyncStatus.synced) {
globals.__dataStackLastSettingsBridgeSync = {
attempted: true,
ok: false,
synced: false,
status: clientSync.status,
error: clientSync.error,
checkedAt: new Date().toISOString(),
};
}
return { ...nextSettings };
}
await this.persistSettingsToDb(nextSettings);
globals.__dataStackAdminSettings = nextSettings;
this.applyRuntimeSettings(nextSettings, updates.autoPruneDays !== undefined);
await this.notifyIngestionSettingsSync( await this.notifyIngestionSettingsSync(
nextSettings, nextSettings,
previousJwtSecret, previousJwtSecret,
jwtSecretChanged, false,
); );
if (jwtSecretChanged) {
await SDKManager.syncAllClientsToIngestionAsync();
}
return { ...nextSettings }; return { ...nextSettings };
} }
} }
+64 -16
View File
@@ -34,6 +34,11 @@ export class ApiEngine {
).replace(/\/$/, ''); ).replace(/\/$/, '');
const internalHeaders = getInternalIngestionHeaders(); const internalHeaders = getInternalIngestionHeaders();
let ingestionBridgeStatus: 'online' | 'auth_error' | 'error' | 'offline' = 'offline';
let ingestionBridgeError: string | undefined;
let liveServerPort = settings.serverPort || 4000;
let liveWebPort = settings.webPort || 3000;
try { try {
const res = await fetch(`${ingestionBaseUrl}/internal/metrics`, { const res = await fetch(`${ingestionBaseUrl}/internal/metrics`, {
headers: internalHeaders, headers: internalHeaders,
@@ -42,8 +47,17 @@ export class ApiEngine {
}); });
if (res.ok) { if (res.ok) {
const data = await res.json(); const data = await res.json();
ingestionBridgeStatus = 'online';
activeSockets = Number(data.activeSockets) || 0; activeSockets = Number(data.activeSockets) || 0;
ingestionRedisConnected = Boolean(data.redisConnected); ingestionRedisConnected = Boolean(data.redisConnected);
if (Number.isFinite(Number(data.serverPort)) && Number(data.serverPort) > 0) {
liveServerPort = Number(data.serverPort);
} else if (Number.isFinite(Number(data.port)) && Number(data.port) > 0) {
liveServerPort = Number(data.port);
}
if (Number.isFinite(Number(data.webPort)) && Number(data.webPort) > 0) {
liveWebPort = Number(data.webPort);
}
if (data.localBufferSize !== undefined) { if (data.localBufferSize !== undefined) {
ingestionLocalBufferSize = Number(data.localBufferSize) || 0; ingestionLocalBufferSize = Number(data.localBufferSize) || 0;
ingestionRedisBufferSize = Number(data.redisBufferSize) || 0; ingestionRedisBufferSize = Number(data.redisBufferSize) || 0;
@@ -53,14 +67,36 @@ export class ApiEngine {
if (data.securityStats && typeof data.securityStats === 'object') { if (data.securityStats && typeof data.securityStats === 'object') {
remoteSecurityStats = data.securityStats; remoteSecurityStats = data.securityStats;
} }
// BUG-02: Auto-reconcile Port 4000 cold start whenever metrics observe unsynced registry
if (data.hasSyncedInMemoryClients === false && clients.length > 0) {
await SDKManager.syncAllClientsToIngestionAsync();
} }
} catch { } else {
ingestionBridgeStatus = res.status === 401 || res.status === 503 ? 'auth_error' : 'error';
ingestionBridgeError = `HTTP ${res.status}`;
console.warn(`[DataStack Bridge] /internal/metrics failed with ${ingestionBridgeError}`);
}
} catch (err: unknown) {
// Ingestion server may not be running in single-process dev mode // Ingestion server may not be running in single-process dev mode
activeSockets = 0; activeSockets = 0;
ingestionLocalBufferSize = 0; ingestionLocalBufferSize = 0;
ingestionRedisBufferSize = 0; ingestionRedisBufferSize = 0;
ingestionBridgeStatus = 'offline';
ingestionBridgeError = err instanceof Error ? err.message : 'Unreachable';
} }
const effectiveRedisBufferSize = Math.max(
localBufferStats.redisBufferSize || 0,
ingestionRedisBufferSize,
);
const redisConnected = localBufferStats.redisConnected || ingestionRedisConnected;
const securityMode: 'redis_shared' | 'bridge_coordinated' | 'degraded_local' =
redisConnected
? 'redis_shared'
: ingestionBridgeStatus === 'online'
? 'bridge_coordinated'
: 'degraded_local';
const localSecurityStats = SecurityEngine.getStats(); const localSecurityStats = SecurityEngine.getStats();
const securityStats = { const securityStats = {
activeIdempotencyKeys: activeIdempotencyKeys:
@@ -74,15 +110,12 @@ export class ApiEngine {
Number(remoteSecurityStats?.maxPayloadSizeBytes) || Number(remoteSecurityStats?.maxPayloadSizeBytes) ||
localSecurityStats.maxPayloadSizeBytes, localSecurityStats.maxPayloadSizeBytes,
), ),
mode: securityMode,
}; };
const effectiveRedisBufferSize = Math.max(
localBufferStats.redisBufferSize || 0,
ingestionRedisBufferSize,
);
const bufferStats = { const bufferStats = {
...localBufferStats, ...localBufferStats,
redisConnected: localBufferStats.redisConnected || ingestionRedisConnected, redisConnected,
redisBufferSize: effectiveRedisBufferSize, redisBufferSize: effectiveRedisBufferSize,
currentBufferSize: currentBufferSize:
localBufferStats.localBufferSize + localBufferStats.localBufferSize +
@@ -98,8 +131,11 @@ export class ApiEngine {
storageStats, storageStats,
securityStats, securityStats,
runtime: { runtime: {
serverPort: settings.serverPort || 4000, serverPort: liveServerPort,
webPort: settings.webPort || 3000, webPort: liveWebPort,
ingestionBridgeStatus,
...(ingestionBridgeError ? { ingestionBridgeError } : {}),
securityMode,
uptimeSeconds: Math.floor(process.uptime()), uptimeSeconds: Math.floor(process.uptime()),
memoryUsageMb: (process.memoryUsage().rss / (1024 * 1024)).toFixed(1), memoryUsageMb: (process.memoryUsage().rss / (1024 * 1024)).toFixed(1),
nodeVersion: process.version, nodeVersion: process.version,
@@ -107,23 +143,35 @@ export class ApiEngine {
}; };
} }
private static filterFilesByClientId<T extends { clientId: string }>(
files: T[],
clientId?: string,
): T[] {
if (!clientId || clientId === 'all') return files;
const normalized = SDKManager.normalizeClientId(clientId);
if (!normalized) return files;
const sanitized = CSVStorageManager.sanitizeClientId(normalized);
return files.filter((f) => {
const folderNorm = SDKManager.normalizeClientId(f.clientId);
return (
folderNorm === normalized ||
f.clientId === normalized ||
f.clientId === sanitized
);
});
}
/** /**
* Filters and lists stored CSV files for export * Filters and lists stored CSV files for export
*/ */
public static getExportFiles(clientId?: string) { public static getExportFiles(clientId?: string) {
const allFiles = CSVStorageManager.listAllFiles(); const allFiles = CSVStorageManager.listAllFiles();
if (clientId && clientId !== 'all') { return this.filterFilesByClientId(allFiles, clientId);
return allFiles.filter((f) => f.clientId === clientId);
}
return allFiles;
} }
public static async getExportFilesAsync(clientId?: string) { public static async getExportFilesAsync(clientId?: string) {
const allFiles = await CSVStorageManager.listAllFilesAsync(); const allFiles = await CSVStorageManager.listAllFilesAsync();
if (clientId && clientId !== 'all') { return this.filterFilesByClientId(allFiles, clientId);
return allFiles.filter((f) => f.clientId === clientId);
}
return allFiles;
} }
} }
+1 -4
View File
@@ -113,11 +113,8 @@ export function getSessionCookieOptions(request: Request, maxAge: number) {
?.split(",")[0] ?.split(",")[0]
?.trim() ?.trim()
.toLowerCase(); .toLowerCase();
const originHeader = request.headers.get("origin")?.toLowerCase() ?? "";
const isHttps = const isHttps =
forwardedProto === "https" || forwardedProto === "https" || request.url.startsWith("https://");
originHeader.startsWith("https://") ||
request.url.startsWith("https://");
if (isHttps) { if (isHttps) {
return { return {
+62 -53
View File
@@ -5,12 +5,13 @@ import {
scrypt as scryptCallback, scrypt as scryptCallback,
timingSafeEqual, timingSafeEqual,
} from "node:crypto"; } from "node:crypto";
import { Pool, type PoolClient } from "pg"; import type { Pool, PoolClient } from "pg";
import { import {
type AdminSessionPayload, type AdminSessionPayload,
getAdminSessionPayload, getAdminSessionPayload,
getAdminSessionSecret, getAdminSessionSecret,
} from "./adminSession"; } from "./adminSession";
import { DatabaseEngine, ensureCoreSchemaOnPool } from "../Database/database";
const PASSWORD_HASH_BYTES = 64; const PASSWORD_HASH_BYTES = 64;
const PASSWORD_SALT_BYTES = 16; const PASSWORD_SALT_BYTES = 16;
@@ -50,7 +51,6 @@ interface InMemoryAdmin {
} }
type AdminDatabaseGlobals = typeof globalThis & { type AdminDatabaseGlobals = typeof globalThis & {
__dataStackAdminPool?: Pool | null;
__dataStackAdminSchema?: Promise<void>; __dataStackAdminSchema?: Promise<void>;
__dataStackInMemoryAdmin?: InMemoryAdmin; __dataStackInMemoryAdmin?: InMemoryAdmin;
}; };
@@ -60,67 +60,51 @@ const localLoginThrottle = new Map<string, LoginThrottleEntry>();
let lastLocalThrottleCleanup = 0; let lastLocalThrottleCleanup = 0;
let lastDatabaseThrottleCleanup = 0; let lastDatabaseThrottleCleanup = 0;
export class AdminSetupRequiredError extends Error {
public readonly code = "SETUP_REQUIRED";
constructor(message = "Admin account has not completed initial setup.") {
super(message);
this.name = "AdminSetupRequiredError";
}
}
export class AdminDatabaseRequiredError extends Error {
public readonly code = "DATABASE_NOT_CONFIGURED";
constructor(
message = "PostgreSQL DATABASE_URL is required for initial admin setup in production.",
) {
super(message);
this.name = "AdminDatabaseRequiredError";
}
}
function isProductionEnv() { function isProductionEnv() {
return process.env.NODE_ENV === "production"; return process.env.NODE_ENV === "production";
} }
export function isAdminDatabaseConfigured(): boolean {
return Boolean(getPool());
}
export function requiresPersistentAdminDatabase(): boolean {
return (
isProductionEnv() &&
!getPool() &&
!process.env.DATASTACK_ADMIN_PASSWORD
);
}
function getPool(): Pool | null { function getPool(): Pool | null {
const connectionString = process.env.DATABASE_URL?.trim(); return DatabaseEngine.getPool();
if (!connectionString) {
return null;
}
if (globals.__dataStackAdminPool === undefined) {
try {
globals.__dataStackAdminPool = new Pool({
connectionString,
connectionTimeoutMillis: 3000,
idleTimeoutMillis: 30000,
max: 5,
});
} catch {
globals.__dataStackAdminPool = null;
}
}
return globals.__dataStackAdminPool ?? null;
} }
async function ensureAdminTable() { async function ensureAdminTable() {
const pool = getPool(); const pool = getPool();
if (!pool) return; if (!pool) return;
globals.__dataStackAdminSchema ??= Promise.all([ globals.__dataStackAdminSchema ??= ensureCoreSchemaOnPool(pool);
pool
.query(
`
CREATE TABLE IF NOT EXISTS admin_users (
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
email TEXT NOT NULL UNIQUE,
password_salt BYTEA NOT NULL,
password_hash BYTEA NOT NULL,
token_version INTEGER NOT NULL DEFAULT 1,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`,
)
.then(() =>
pool.query(
`ALTER TABLE admin_users ADD COLUMN IF NOT EXISTS token_version INTEGER NOT NULL DEFAULT 1`,
),
),
pool.query(
`
CREATE TABLE IF NOT EXISTS admin_login_attempts (
key_hash TEXT PRIMARY KEY,
window_started_at TIMESTAMPTZ NOT NULL,
failed_attempts INTEGER NOT NULL,
blocked_until TIMESTAMPTZ,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`,
),
]).then(() => undefined);
try { try {
await globals.__dataStackAdminSchema; await globals.__dataStackAdminSchema;
@@ -342,7 +326,7 @@ async function getOrInitInMemoryAdmin(): Promise<InMemoryAdmin> {
!isProductionEnv() && process.env.DATASTACK_ALLOW_DEMO_ADMIN !== "false"; !isProductionEnv() && process.env.DATASTACK_ALLOW_DEMO_ADMIN !== "false";
if (!envPassword && !allowDemo) { if (!envPassword && !allowDemo) {
throw new Error("Admin account has not completed initial setup."); throw new AdminSetupRequiredError();
} }
const defaultEmail = ( const defaultEmail = (
@@ -443,6 +427,9 @@ export async function createInitialAdmin(email: string, password: string) {
const pool = getPool(); const pool = getPool();
if (!pool) { if (!pool) {
if (isProductionEnv()) {
throw new AdminDatabaseRequiredError();
}
if (globals.__dataStackInMemoryAdmin) { if (globals.__dataStackInMemoryAdmin) {
return false; return false;
} }
@@ -671,6 +658,28 @@ export async function updateAdminAccount({
return { ok: true as const, email: cleanEmail, tokenVersion: nextVersion }; return { ok: true as const, email: cleanEmail, tokenVersion: nextVersion };
} }
export async function revokeAdminSessions(): Promise<number> {
const pool = getPool();
if (pool) {
await ensureAdminTable();
const updated = await pool.query(
`UPDATE admin_users
SET token_version = COALESCE(token_version, 1) + 1
WHERE id = 1
RETURNING token_version`,
);
if (!updated.rowCount || updated.rowCount === 0) {
return 0;
}
return Number(updated.rows[0].token_version) || 1;
}
const existing = globals.__dataStackInMemoryAdmin ?? (await getOrInitInMemoryAdmin());
existing.tokenVersion += 1;
globals.__dataStackInMemoryAdmin = existing;
return existing.tokenVersion;
}
export function resetAdminStateForTests() { export function resetAdminStateForTests() {
globals.__dataStackInMemoryAdmin = undefined; globals.__dataStackInMemoryAdmin = undefined;
localLoginThrottle.clear(); localLoginThrottle.clear();
+465 -31
View File
@@ -19,6 +19,15 @@ export interface RedisBufferClient {
ttlMs: number, ttlMs: number,
nxMode: 'NX', nxMode: 'NX',
): Promise<string | null>; ): Promise<string | null>;
get?(key: string): Promise<string | null>;
incr?(key: string): Promise<number>;
pexpire?(key: string, ttlMs: number): Promise<number>;
hget?(key: string, field: string): Promise<string | null>;
hset?(key: string, field: string, value: string): Promise<number>;
hdel?(key: string, ...fields: string[]): Promise<number>;
sismember?(key: string, member: string): Promise<number>;
sadd?(key: string, ...members: string[]): Promise<number>;
srem?(key: string, ...members: string[]): Promise<number>;
eval(script: string, numKeys: number, ...args: string[]): Promise<unknown>; eval(script: string, numKeys: number, ...args: string[]): Promise<unknown>;
quit?: () => Promise<unknown>; quit?: () => Promise<unknown>;
disconnect?: () => void; disconnect?: () => void;
@@ -27,10 +36,83 @@ export interface RedisBufferClient {
export const REDIS_BUFFER_KEY = 'datastack:buffer:events'; export const REDIS_BUFFER_KEY = 'datastack:buffer:events';
export const REDIS_RECENT_KEY = 'datastack:recent:events'; export const REDIS_RECENT_KEY = 'datastack:recent:events';
export const REDIS_FLUSH_LOCK_KEY = 'datastack:flush:lock'; export const REDIS_FLUSH_LOCK_KEY = 'datastack:flush:lock';
const REDIS_RECENT_LIMIT = 500; const REDIS_CLIENTS_DELETED_KEY = 'datastack:clients:deleted';
const FLUSH_LOCK_TTL_MS = 15000; export const DEFAULT_REDIS_RECENT_LIMIT = 500;
export const MAX_REDIS_RECENT_LIMIT = 5000;
const FLUSH_LOCK_TTL_MS = 30000;
const FLUSH_LOCK_WAIT_MS = 400;
const FLUSH_LOCK_RETRY_MS = 25;
const UNLOCK_LUA_SCRIPT = const UNLOCK_LUA_SCRIPT =
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end'; 'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end';
const RENEW_LOCK_LUA_SCRIPT =
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("pexpire", KEYS[1], ARGV[2]) else return 0 end';
export const PURGE_CLIENT_LIST_LUA_SCRIPT = `
-- DATASTACK_PURGE_CLIENT_LIST
local key = KEYS[1]
local target = ARGV[1]
local items = redis.call("LRANGE", key, 0, -1)
local removed = 0
local kept = {}
for i = 1, #items do
local raw = items[i]
local ok, obj = pcall(cjson.decode, raw)
local cid = ""
if ok and type(obj) == "table" and obj.clientId ~= nil then
cid = string.lower(tostring(obj.clientId))
cid = string.gsub(cid, "^%s*(.-)%s*$", "%1")
cid = string.gsub(cid, "[^a-z0-9_-]", "_")
end
if cid == target then
removed = removed + 1
else
kept[#kept + 1] = raw
end
end
if removed > 0 then
redis.call("DEL", key)
local chunk = 500
for i = 1, #kept, chunk do
local stop = math.min(i + chunk - 1, #kept)
redis.call("RPUSH", key, unpack(kept, i, stop))
end
end
return removed
`.trim();
export function resolveRedisRecentLimit(batchFlushThreshold?: number): number {
const envRaw = Number(process.env.DATASTACK_RECENT_LIMIT);
if (Number.isFinite(envRaw) && envRaw >= 100) {
return Math.min(MAX_REDIS_RECENT_LIMIT, Math.floor(envRaw));
}
const threshold = Number(batchFlushThreshold) || DEFAULT_REDIS_RECENT_LIMIT;
return Math.min(
MAX_REDIS_RECENT_LIMIT,
Math.max(DEFAULT_REDIS_RECENT_LIMIT, Math.floor(threshold)),
);
}
export interface FlushOptions {
waitMs?: number;
lockTtlMs?: number;
throwOnLockTimeout?: boolean;
}
export interface FlushResult {
count: number;
files: string[];
lockTimedOut?: boolean;
}
function sleepMs(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function normalizeBufferClientId(clientId: string): string {
return String(clientId || '')
.trim()
.toLowerCase()
.replace(/[^a-z0-9_-]/g, '_');
}
interface BufferGlobalState { interface BufferGlobalState {
buffer: DataEvent[]; buffer: DataEvent[];
@@ -43,11 +125,14 @@ interface BufferGlobalState {
totalIngested: number; totalIngested: number;
totalFlushed: number; totalFlushed: number;
isFlushing: boolean; isFlushing: boolean;
activeFlushPromise: Promise<FlushResult> | null;
redisClient: RedisBufferClient | null | undefined; redisClient: RedisBufferClient | null | undefined;
redisUrlConfigured: string; redisUrlConfigured: string;
redisConnected: boolean; redisConnected: boolean;
redisTestOverride: RedisBufferClient | null | undefined; redisTestOverride: RedisBufferClient | null | undefined;
pendingRedisWrites: Set<Promise<void>>; pendingRedisWrites: Set<Promise<void>>;
deletedClientIds: Set<string>;
shutdownHooks: Set<() => Promise<unknown>>;
} }
type BufferGlobals = typeof globalThis & { type BufferGlobals = typeof globalThis & {
@@ -71,13 +156,22 @@ function getState(): BufferGlobalState {
totalIngested: 0, totalIngested: 0,
totalFlushed: 0, totalFlushed: 0,
isFlushing: false, isFlushing: false,
activeFlushPromise: null,
redisClient: undefined, redisClient: undefined,
redisUrlConfigured: '', redisUrlConfigured: '',
redisConnected: false, redisConnected: false,
redisTestOverride: undefined, redisTestOverride: undefined,
pendingRedisWrites: new Set(), pendingRedisWrites: new Set(),
deletedClientIds: new Set(),
shutdownHooks: new Set(),
}; };
} }
if (!globals.__dataStackBufferState.deletedClientIds) {
globals.__dataStackBufferState.deletedClientIds = new Set();
}
if (!globals.__dataStackBufferState.shutdownHooks) {
globals.__dataStackBufferState.shutdownHooks = new Set();
}
return globals.__dataStackBufferState; return globals.__dataStackBufferState;
} }
@@ -150,7 +244,9 @@ function getRedisClient(): RedisBufferClient | null {
} }
export class DataBufferEngine { export class DataBufferEngine {
private static maxRecentEvents = REDIS_RECENT_LIMIT; public static getRecentLimit(): number {
return resolveRedisRecentLimit(getState().batchFlushThreshold);
}
public static get batchFlushThreshold(): number { public static get batchFlushThreshold(): number {
return getState().batchFlushThreshold; return getState().batchFlushThreshold;
@@ -174,6 +270,10 @@ export class DataBufferEngine {
state.redisConnected = Boolean(client); state.redisConnected = Boolean(client);
} }
public static getRedisClient(): RedisBufferClient | null {
return getRedisClient();
}
public static async shutdown(): Promise<void> { public static async shutdown(): Promise<void> {
const state = getState(); const state = getState();
if (state.flushTimer) { if (state.flushTimer) {
@@ -188,6 +288,7 @@ export class DataBufferEngine {
state.redisUrlConfigured = ''; state.redisUrlConfigured = '';
state.redisConnected = false; state.redisConnected = false;
state.redisTestOverride = undefined; state.redisTestOverride = undefined;
state.deletedClientIds.clear();
if (client) { if (client) {
try { try {
if (typeof client.quit === 'function') { if (typeof client.quit === 'function') {
@@ -203,6 +304,14 @@ export class DataBufferEngine {
} }
} }
public static registerShutdownHook(hook: () => Promise<unknown>): () => void {
const state = getState();
state.shutdownHooks.add(hook);
return () => {
state.shutdownHooks.delete(hook);
};
}
public static async flushAndShutdownOnSignal(): Promise<void> { public static async flushAndShutdownOnSignal(): Promise<void> {
if (globals.__dataStackBufferShuttingDown) { if (globals.__dataStackBufferShuttingDown) {
return globals.__dataStackBufferShuttingDown; return globals.__dataStackBufferShuttingDown;
@@ -210,8 +319,16 @@ export class DataBufferEngine {
const run = (async () => { const run = (async () => {
try { try {
const hooks = [...getState().shutdownHooks];
await Promise.race([ await Promise.race([
Promise.allSettled([
this.flush().catch(() => undefined), this.flush().catch(() => undefined),
...hooks.map((hook) =>
Promise.resolve()
.then(() => hook())
.catch(() => undefined),
),
]),
new Promise<void>((resolve) => setTimeout(resolve, 4000)), new Promise<void>((resolve) => setTimeout(resolve, 4000)),
]); ]);
} finally { } finally {
@@ -278,7 +395,8 @@ export class DataBufferEngine {
const state = getState(); const state = getState();
state.totalIngested += 1; state.totalIngested += 1;
state.recentEvents.unshift(event); state.recentEvents.unshift(event);
if (state.recentEvents.length > this.maxRecentEvents) { const recentLimit = this.getRecentLimit();
while (state.recentEvents.length > recentLimit) {
state.recentEvents.pop(); state.recentEvents.pop();
} }
@@ -298,15 +416,25 @@ export class DataBufferEngine {
if (redis) { if (redis) {
const serialized = events.map((e) => JSON.stringify(e)); const serialized = events.map((e) => JSON.stringify(e));
let bufferLen: number | null = null;
try { try {
const bufferLen = await redis.rpush(REDIS_BUFFER_KEY, ...serialized); bufferLen = await redis.rpush(REDIS_BUFFER_KEY, ...serialized);
await redis.lpush(REDIS_RECENT_KEY, ...serialized);
await redis.ltrim(REDIS_RECENT_KEY, 0, REDIS_RECENT_LIMIT - 1);
state.redisConnected = true; state.redisConnected = true;
return Number(bufferLen) + state.buffer.length;
} catch { } catch {
state.redisConnected = false; state.redisConnected = false;
} }
if (bufferLen !== null) {
// Best-effort recent list mirror: never duplicate into RAM buffer once rpush succeeded
try {
const recentLimit = this.getRecentLimit();
await redis.lpush(REDIS_RECENT_KEY, ...serialized);
await redis.ltrim(REDIS_RECENT_KEY, 0, recentLimit - 1);
} catch {
// Non-fatal: events are already committed in REDIS_BUFFER_KEY and state.recentEvents
}
return Number(bufferLen) + state.buffer.length;
}
} }
state.buffer.push(...events); state.buffer.push(...events);
@@ -363,64 +491,306 @@ export class DataBufferEngine {
} }
} }
/** private static async isClientDeletedForBuffer(
* Flushes queued events (shared Redis list under distributed lock + local RAM fallback) to partitioned CSV files. clientId: string,
* Redis LTRIM runs ONLY after CSVStorageManager.appendBatch succeeds so crashes never lose data. redis: RedisBufferClient | null,
*/ cache?: Map<string, boolean>,
public static async flush(): Promise<{ count: number; files: string[] }> { ): Promise<boolean> {
const normalized = normalizeBufferClientId(clientId);
if (!normalized) return false;
const state = getState(); const state = getState();
if (state.deletedClientIds.has(normalized)) return true;
if (cache && cache.has(normalized)) return Boolean(cache.get(normalized));
if (redis && typeof redis.sismember === 'function') {
try {
const isDeleted =
Number(await redis.sismember(REDIS_CLIENTS_DELETED_KEY, normalized)) === 1;
if (cache) cache.set(normalized, isDeleted);
if (isDeleted) state.deletedClientIds.add(normalized);
return isDeleted;
} catch {
// Ignore transient Redis tombstone lookup error
}
}
if (cache) cache.set(normalized, false);
return false;
}
/**
* Marks a client deleted in the buffer layer and immediately strips queued local events.
*/
public static markClientDeleted(clientId: string): void {
const normalized = normalizeBufferClientId(clientId);
if (!normalized) return;
const state = getState();
state.deletedClientIds.add(normalized);
state.buffer = state.buffer.filter(
(evt) => normalizeBufferClientId(evt.clientId) !== normalized,
);
state.recentEvents = state.recentEvents.filter(
(evt) => normalizeBufferClientId(evt.clientId) !== normalized,
);
}
/**
* Clears deleted tombstone when a client ID is explicitly re-registered.
*/
public static unmarkClientDeleted(clientId: string): void {
const normalized = normalizeBufferClientId(clientId);
if (!normalized) return;
getState().deletedClientIds.delete(normalized);
}
private static async filterRedisListExcludingClient(
redis: RedisBufferClient,
listKey: string,
normalizedClientId: string,
): Promise<number> {
const removed = await redis.eval(
PURGE_CLIENT_LIST_LUA_SCRIPT,
1,
listKey,
normalizedClientId,
);
return Number(removed) || 0;
}
/**
* Purges all queued events for a deleted client from both local RAM and shared Redis queues (BUG-19).
*/
public static async purgeClientEventsAsync(
clientId: string,
): Promise<{ removedLocal: number; removedRedis: number }> {
const normalized = normalizeBufferClientId(clientId);
if (!normalized) return { removedLocal: 0, removedRedis: 0 };
const state = getState();
state.deletedClientIds.add(normalized);
if (state.pendingRedisWrites.size > 0) { if (state.pendingRedisWrites.size > 0) {
await Promise.allSettled([...state.pendingRedisWrites]); await Promise.allSettled([...state.pendingRedisWrites]);
} }
if (state.isFlushing) {
return { count: 0, files: [] };
}
state.isFlushing = true; const beforeLocal = state.buffer.length;
let totalWritten = 0; state.buffer = state.buffer.filter(
const filesSet = new Set<string>(); (evt) => normalizeBufferClientId(evt.clientId) !== normalized,
);
const removedLocal = beforeLocal - state.buffer.length;
state.recentEvents = state.recentEvents.filter(
(evt) => normalizeBufferClientId(evt.clientId) !== normalized,
);
try { let removedRedis = 0;
const redis = getRedisClient(); const redis = getRedisClient();
if (redis) { if (redis) {
const lockToken = crypto.randomUUID(); const lockToken = crypto.randomUUID();
let lockAcquired = false; let lockAcquired = false;
const deadline = Date.now() + 1500;
while (true) {
try { try {
const lockResult = await redis.set( const lockRes = await redis.set(
REDIS_FLUSH_LOCK_KEY, REDIS_FLUSH_LOCK_KEY,
lockToken, lockToken,
'PX', 'PX',
FLUSH_LOCK_TTL_MS, FLUSH_LOCK_TTL_MS,
'NX', 'NX',
); );
lockAcquired = lockResult === 'OK';
state.redisConnected = true; state.redisConnected = true;
if (lockRes === 'OK') {
lockAcquired = true;
break;
}
} catch { } catch {
state.redisConnected = false; state.redisConnected = false;
lockAcquired = false; break;
}
const remaining = deadline - Date.now();
if (remaining <= 0) break;
await sleepMs(Math.min(FLUSH_LOCK_RETRY_MS, remaining));
} }
if (lockAcquired) { if (lockAcquired) {
try {
removedRedis += await this.filterRedisListExcludingClient(
redis,
REDIS_BUFFER_KEY,
normalized,
);
await this.filterRedisListExcludingClient(
redis,
REDIS_RECENT_KEY,
normalized,
).catch(() => 0);
} catch {
// Flush-time deletedClientIds/tombstone filter remains active as secondary guard
} finally {
try {
await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
} catch {
// Lock expires via PX TTL
}
}
}
}
return { removedLocal, removedRedis };
}
/**
* Flushes queued events (shared Redis list under distributed lock + local RAM fallback) to partitioned CSV files.
* Redis LTRIM runs ONLY after CSVStorageManager.appendBatch succeeds and lock ownership is verified.
*/
public static async flush(options?: FlushOptions): Promise<FlushResult> {
const state = getState();
if (state.pendingRedisWrites.size > 0) {
await Promise.allSettled([...state.pendingRedisWrites]);
}
if (state.activeFlushPromise) {
return state.activeFlushPromise;
}
const executeFlush = async (): Promise<FlushResult> => {
state.isFlushing = true;
let totalWritten = 0;
let lockTimedOut = false;
const filesSet = new Set<string>();
const deletedCheckCache = new Map<string, boolean>();
try {
if (state.pendingRedisWrites.size > 0) {
await Promise.allSettled([...state.pendingRedisWrites]);
}
const redis = getRedisClient();
if (redis) {
const lockToken = crypto.randomUUID();
const lockTtlMs = Math.max(50, Number(options?.lockTtlMs) || FLUSH_LOCK_TTL_MS);
const waitMs = Math.max(
0,
options?.waitMs !== undefined ? Number(options.waitMs) : FLUSH_LOCK_WAIT_MS,
);
const deadline = Date.now() + waitMs;
let lockAcquired = false;
let redisError = false;
while (true) {
try {
const lockResult = await redis.set(
REDIS_FLUSH_LOCK_KEY,
lockToken,
'PX',
lockTtlMs,
'NX',
);
state.redisConnected = true;
if (lockResult === 'OK') {
lockAcquired = true;
break;
}
} catch {
state.redisConnected = false;
redisError = true;
break;
}
const remaining = deadline - Date.now();
if (remaining <= 0) {
break;
}
await sleepMs(Math.min(FLUSH_LOCK_RETRY_MS, remaining));
}
if (!lockAcquired && !redisError) {
lockTimedOut = true;
if (options?.throwOnLockTimeout === true) {
throw new Error(
'FLUSH_LOCK_TIMEOUT: Timed out waiting for distributed Redis flush lock',
);
}
}
if (lockAcquired) {
const renewIntervalMs = Math.max(20, Math.floor(lockTtlMs / 3));
let lastRenewAt = Date.now();
let lockLost = false;
let renewInFlight: Promise<boolean> | null = null;
const renewLockOnce = (): Promise<boolean> => {
if (renewInFlight) return renewInFlight;
renewInFlight = (async () => {
try {
const renewed = await redis.eval(
RENEW_LOCK_LUA_SCRIPT,
1,
REDIS_FLUSH_LOCK_KEY,
lockToken,
String(lockTtlMs),
);
if (Number(renewed) === 1) {
lastRenewAt = Date.now();
return true;
}
lockLost = true;
return false;
} catch {
return false;
} finally {
renewInFlight = null;
}
})();
return renewInFlight;
};
const renewTimer = setInterval(() => {
void renewLockOnce();
}, renewIntervalMs);
if (typeof renewTimer.unref === 'function') {
renewTimer.unref();
}
try { try {
const chunkSize = Math.max(500, state.batchFlushThreshold); const chunkSize = Math.max(500, state.batchFlushThreshold);
while (true) { while (true) {
if (lockLost) break;
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, chunkSize - 1); const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, chunkSize - 1);
if (!rawItems || rawItems.length === 0) { if (!rawItems || rawItems.length === 0) {
break; break;
} }
const parsedBatch: DataEvent[] = []; const parsedBatch: DataEvent[] = [];
let malformedCount = 0;
for (const raw of rawItems) { for (const raw of rawItems) {
try { try {
const evt = JSON.parse(raw) as DataEvent; const evt = JSON.parse(raw) as DataEvent;
if (evt && evt.eventId) { if (evt && evt.eventId) {
if (
await this.isClientDeletedForBuffer(
evt.clientId,
redis,
deletedCheckCache,
)
) {
continue;
}
parsedBatch.push(evt); parsedBatch.push(evt);
} else {
malformedCount += 1;
} }
} catch { } catch {
// Ignore malformed legacy entry while still trimming it malformedCount += 1;
} }
} }
if (malformedCount > 0) {
console.warn(
`[DataBufferEngine] Skipped ${malformedCount} malformed buffered Redis event(s) during flush`,
);
}
if (Date.now() - lastRenewAt >= renewIntervalMs) {
await renewLockOnce();
}
if (lockLost) break;
if (parsedBatch.length > 0) { if (parsedBatch.length > 0) {
const res = await CSVStorageManager.appendBatch(parsedBatch); const res = await CSVStorageManager.appendBatch(parsedBatch);
@@ -428,7 +798,14 @@ export class DataBufferEngine {
for (const f of res.filesUpdated) filesSet.add(f); for (const f of res.filesUpdated) filesSet.add(f);
} }
// Trim strictly AFTER CSVStorageManager.appendBatch resolves safely if (Date.now() - lastRenewAt >= renewIntervalMs) {
await renewLockOnce();
}
if (lockLost) {
break;
}
// Trim strictly AFTER CSVStorageManager.appendBatch resolves and lock ownership holds
await redis.ltrim(REDIS_BUFFER_KEY, rawItems.length, -1); await redis.ltrim(REDIS_BUFFER_KEY, rawItems.length, -1);
if (rawItems.length < chunkSize) { if (rawItems.length < chunkSize) {
@@ -436,10 +813,14 @@ export class DataBufferEngine {
} }
} }
} finally { } finally {
clearInterval(renewTimer);
if (renewInFlight) {
await renewInFlight.catch(() => false);
}
try { try {
await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken); await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
} catch { } catch {
// Lock auto-expires after FLUSH_LOCK_TTL_MS // Lock auto-expires after lockTtlMs
} }
} }
} }
@@ -448,15 +829,31 @@ export class DataBufferEngine {
if (state.buffer.length > 0) { if (state.buffer.length > 0) {
const ramBatch = [...state.buffer]; const ramBatch = [...state.buffer];
state.buffer = []; state.buffer = [];
const validRamBatch: DataEvent[] = [];
for (const evt of ramBatch) {
if (
await this.isClientDeletedForBuffer(
evt.clientId,
redis,
deletedCheckCache,
)
) {
continue;
}
validRamBatch.push(evt);
}
if (validRamBatch.length > 0) {
try { try {
const res = await CSVStorageManager.appendBatch(ramBatch); const res = await CSVStorageManager.appendBatch(validRamBatch);
totalWritten += res.written; totalWritten += res.written;
for (const f of res.filesUpdated) filesSet.add(f); for (const f of res.filesUpdated) filesSet.add(f);
} catch (err) { } catch (err) {
state.buffer.unshift(...ramBatch); state.buffer.unshift(...validRamBatch);
throw err; throw err;
} }
} }
}
if (totalWritten > 0) { if (totalWritten > 0) {
state.totalFlushed += totalWritten; state.totalFlushed += totalWritten;
@@ -470,10 +867,34 @@ export class DataBufferEngine {
} }
} }
return { count: totalWritten, files: Array.from(filesSet) }; return {
count: totalWritten,
files: Array.from(filesSet),
...(lockTimedOut ? { lockTimedOut: true } : {}),
};
} finally { } finally {
state.isFlushing = false; state.isFlushing = false;
state.activeFlushPromise = null;
} }
};
let flushDone = false;
let promise: Promise<FlushResult> | null = null;
promise = (async () => {
try {
return await executeFlush();
} finally {
flushDone = true;
state.isFlushing = false;
if (state.activeFlushPromise === promise) {
state.activeFlushPromise = null;
}
}
})();
if (!flushDone) {
state.activeFlushPromise = promise;
}
return promise;
} }
/** /**
@@ -495,6 +916,15 @@ export class DataBufferEngine {
}; };
} }
public static getLocalBufferSnapshot(limit?: number): DataEvent[] {
const state = getState();
if (typeof limit === 'number' && Number.isFinite(limit) && limit >= 0) {
const cap = Math.floor(limit);
return cap === 0 ? [] : state.buffer.slice(-cap);
}
return [...state.buffer];
}
public static getRecentEvents(limit = 50, filterClientId?: string): DataEvent[] { public static getRecentEvents(limit = 50, filterClientId?: string): DataEvent[] {
const state = getState(); const state = getState();
if (filterClientId && filterClientId !== 'all') { if (filterClientId && filterClientId !== 'all') {
@@ -514,7 +944,8 @@ export class DataBufferEngine {
if (redis) { if (redis) {
try { try {
const readCap = Math.min(REDIS_RECENT_LIMIT, Math.max(limit * 4, limit)); const recentLimit = this.getRecentLimit();
const readCap = Math.min(recentLimit, Math.max(limit * 4, limit));
const rawItems = await redis.lrange(REDIS_RECENT_KEY, 0, readCap - 1); const rawItems = await redis.lrange(REDIS_RECENT_KEY, 0, readCap - 1);
state.redisConnected = true; state.redisConnected = true;
@@ -523,6 +954,9 @@ export class DataBufferEngine {
try { try {
const parsed = JSON.parse(raw) as DataEvent; const parsed = JSON.parse(raw) as DataEvent;
if (!parsed || !parsed.eventId) continue; if (!parsed || !parsed.eventId) continue;
if (state.deletedClientIds.has(normalizeBufferClientId(parsed.clientId))) {
continue;
}
if (filterClientId && filterClientId !== 'all' && parsed.clientId !== filterClientId) { if (filterClientId && filterClientId !== 'all' && parsed.clientId !== filterClientId) {
continue; continue;
} }
+515 -112
View File
@@ -1,9 +1,21 @@
import { existsSync, rmSync } from "node:fs"; import { randomBytes } from "node:crypto";
import { existsSync, lstatSync, readdirSync, rmSync } from "node:fs";
import { resolve, sep } from "node:path"; import { resolve, sep } from "node:path";
import { SettingsManager } from "../adminSettings/settingsManager"; import { SettingsManager } from "../adminSettings/settingsManager";
import { AuthService } from "../auth/jwt"; import { AuthService } from "../auth/jwt";
import { DataBufferEngine } from "../data/buffer";
import { DatabaseEngine } from "../Database/database"; import { DatabaseEngine } from "../Database/database";
import { getInternalIngestionHeaders } from "../Security/security"; import { getInternalIngestionHeaders } from "../Security/security";
import { CSVStorageManager } from "../Storage/csvStorage";
export interface BridgeSyncResult {
synced: boolean;
status?: number;
error?: string;
}
const REDIS_CLIENTS_REGISTRY_KEY = "datastack:clients:registry";
const REDIS_CLIENTS_DELETED_KEY = "datastack:clients:deleted";
export interface ClientSDK { export interface ClientSDK {
clientId: string; clientId: string;
@@ -17,13 +29,24 @@ export interface ClientSDK {
createdAt: string; createdAt: string;
} }
type PendingActivityEntry = {
count: number;
lastActive: string;
};
type SDKDatabaseGlobals = typeof globalThis & { type SDKDatabaseGlobals = typeof globalThis & {
__dataStackClientsMap?: Map<string, ClientSDK>; __dataStackClientsMap?: Map<string, ClientSDK>;
__dataStackDeletedClients?: Set<string>; __dataStackDeletedClients?: Set<string>;
__dataStackClientsHydrated?: boolean; __dataStackClientsHydrated?: boolean;
__dataStackSyncedSocketEvents?: Map<string, number>; __dataStackSyncedSocketEvents?: Map<string, number>;
__dataStackPendingClientActivity?: Map<string, PendingActivityEntry>;
__dataStackActivityFlushTimer?: ReturnType<typeof setInterval> | null;
__dataStackActivityFlushPromise?: Promise<number> | null;
__dataStackActivityShutdownRegistered?: boolean;
}; };
const CLIENT_ACTIVITY_FLUSH_INTERVAL_MS = 1000;
const CLIENT_ACTIVITY_IMMEDIATE_BATCH_SIZE = 500;
const globals = globalThis as SDKDatabaseGlobals; const globals = globalThis as SDKDatabaseGlobals;
async function ensureDbSettingsHydrated(): Promise<void> { async function ensureDbSettingsHydrated(): Promise<void> {
@@ -53,6 +76,58 @@ function getSyncedSocketEventsMap(): Map<string, number> {
return globals.__dataStackSyncedSocketEvents; return globals.__dataStackSyncedSocketEvents;
} }
function getPendingClientActivityMap(): Map<string, PendingActivityEntry> {
if (!globals.__dataStackPendingClientActivity) {
globals.__dataStackPendingClientActivity = new Map<
string,
PendingActivityEntry
>();
}
return globals.__dataStackPendingClientActivity;
}
function ensureActivityFlushTimer(): void {
if (globals.__dataStackActivityFlushTimer) return;
const timer = setInterval(() => {
void SDKManager.flushPendingClientActivityAsync().catch(() => undefined);
}, CLIENT_ACTIVITY_FLUSH_INTERVAL_MS);
if (typeof timer.unref === "function") {
timer.unref();
}
globals.__dataStackActivityFlushTimer = timer;
}
interface RemoteInMemoryClient {
clientId?: string;
name?: string;
status?: string;
token?: string;
rateLimit?: number;
}
function hydrateLocalMapFromRemote(remoteClients: RemoteInMemoryClient[]): number {
const map = getClientsMap();
const deletedSet = getDeletedClientsSet();
let hydrated = 0;
for (const item of remoteClients) {
if (!item || !item.clientId) continue;
const cid = SDKManager.normalizeClientId(item.clientId);
if (!cid || deletedSet.has(cid) || map.has(cid)) continue;
map.set(cid, {
clientId: cid,
name: String(item.name || cid),
secret: "",
token: String(item.token || ""),
rateLimit: Math.max(1, Number(item.rateLimit) || 100),
status: item.status === "revoked" ? "revoked" : "active",
totalEvents: 0,
createdAt: new Date().toISOString(),
});
hydrated += 1;
}
return hydrated;
}
function serializeSyncClient(client: ClientSDK) { function serializeSyncClient(client: ClientSDK) {
return { return {
clientId: client.clientId, clientId: client.clientId,
@@ -63,18 +138,53 @@ function serializeSyncClient(client: ClientSDK) {
}; };
} }
async function notifyIngestionClientSyncAsync( async function mirrorClientToRedis(
clientId: string, clientId: string,
action: "deleted" | "revoked" | "updated" | "registered", action: "deleted" | "revoked" | "updated" | "registered",
client?: ClientSDK, client?: ClientSDK,
): Promise<void> { ): Promise<void> {
const redis = DataBufferEngine.getRedisClient();
if (!redis) return;
try {
if (action === "deleted") {
if (typeof redis.sadd === "function") {
await redis.sadd(REDIS_CLIENTS_DELETED_KEY, clientId);
}
if (typeof redis.hdel === "function") {
await redis.hdel(REDIS_CLIENTS_REGISTRY_KEY, clientId);
}
return;
}
if (action === "registered" && typeof redis.srem === "function") {
await redis.srem(REDIS_CLIENTS_DELETED_KEY, clientId);
}
if (client && typeof redis.hset === "function") {
await redis.hset(
REDIS_CLIENTS_REGISTRY_KEY,
clientId,
JSON.stringify(serializeSyncClient(client)),
);
}
} catch {
// Non-fatal; DB and direct HTTP bridge remain primary sources of truth
}
}
async function notifyIngestionClientSyncAsync(
clientId: string,
action: "deleted" | "revoked" | "updated" | "registered",
client?: ClientSDK,
): Promise<BridgeSyncResult> {
await mirrorClientToRedis(clientId, action, client);
const headers = getInternalIngestionHeaders(); const headers = getInternalIngestionHeaders();
if (!headers["x-datastack-internal-secret"]) return; if (!headers["x-datastack-internal-secret"]) {
return { synced: false, error: "Internal bridge secret is not configured" };
}
const ingestionBaseUrl = ( const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000" process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, ""); ).replace(/\/$/, "");
try { try {
await fetch(`${ingestionBaseUrl}/internal/client-sync`, { const res = await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
method: "POST", method: "POST",
headers: { headers: {
"Content-Type": "application/json", "Content-Type": "application/json",
@@ -87,8 +197,21 @@ async function notifyIngestionClientSyncAsync(
}), }),
signal: AbortSignal.timeout(1000), signal: AbortSignal.timeout(1000),
}); });
} catch { if (!res.ok) {
// Port 4000 may not be running in isolated single-process dev mode const errorMsg = `Port 4000 client-sync (${action}:${clientId}) failed with HTTP ${res.status}`;
console.warn(`[DataStack Bridge] ${errorMsg}`);
return { synced: false, status: res.status, error: errorMsg };
}
return { synced: true, status: res.status };
} catch (err: unknown) {
const message = err instanceof Error ? err.message : "Bridge request failed";
if (
process.env.NODE_ENV === "production" ||
Boolean(process.env.INGESTION_INTERNAL_URL)
) {
console.warn(`[DataStack Bridge] Port 4000 client-sync unreachable: ${message}`);
}
return { synced: false, error: message };
} }
} }
@@ -127,6 +250,10 @@ export class SDKManager {
const ready = await DatabaseEngine.ensureSchema(); const ready = await DatabaseEngine.ensureSchema();
if (!ready) return null; if (!ready) return null;
if (getPendingClientActivityMap().size > 0) {
await this.flushPendingClientActivityAsync();
}
const [result, deletedResult] = await Promise.all([ const [result, deletedResult] = await Promise.all([
pool.query( pool.query(
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at `SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
@@ -166,16 +293,31 @@ export class SDKManager {
return Array.from(getClientsMap().values()); return Array.from(getClientsMap().values());
} }
public static async syncAllClientsToIngestionAsync(): Promise<void> { public static async syncAllClientsToIngestionAsync(options?: {
forceReplace?: boolean;
}): Promise<BridgeSyncResult> {
await ensureDbSettingsHydrated(); await ensureDbSettingsHydrated();
const clients = Array.from(getClientsMap().values()).map(serializeSyncClient);
const forceReplace = options?.forceReplace ?? Boolean(DatabaseEngine.getPool());
const redis = DataBufferEngine.getRedisClient();
if (redis && typeof redis.hset === "function") {
try {
for (const c of clients) {
await redis.hset(REDIS_CLIENTS_REGISTRY_KEY, c.clientId, JSON.stringify(c));
}
} catch {
// ignore Redis mirror failure
}
}
const headers = getInternalIngestionHeaders(); const headers = getInternalIngestionHeaders();
if (!headers["x-datastack-internal-secret"]) return; if (!headers["x-datastack-internal-secret"]) {
return { synced: false, error: "Internal bridge secret is not configured" };
}
const ingestionBaseUrl = ( const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000" process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, ""); ).replace(/\/$/, "");
const clients = Array.from(getClientsMap().values()).map(serializeSyncClient);
try { try {
await fetch(`${ingestionBaseUrl}/internal/client-sync`, { const res = await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
method: "POST", method: "POST",
headers: { headers: {
"Content-Type": "application/json", "Content-Type": "application/json",
@@ -184,11 +326,43 @@ export class SDKManager {
body: JSON.stringify({ body: JSON.stringify({
action: "bulk_sync", action: "bulk_sync",
clients, clients,
forceReplace,
}), }),
signal: AbortSignal.timeout(1500), signal: AbortSignal.timeout(1500),
}); });
if (!res.ok) {
const errorMsg = `Port 4000 bulk_sync failed with HTTP ${res.status}`;
console.warn(`[DataStack Bridge] ${errorMsg}`);
return { synced: false, status: res.status, error: errorMsg };
}
// BUG-02: If Port 4000 preserved an existing no-DB registry because Port 3000 restarted empty,
// hydrate Port 3000's local map non-destructively from the returned clients.
try {
const payload = (await res.json()) as {
preserved?: boolean;
clients?: RemoteInMemoryClient[];
};
if (
payload?.preserved &&
Array.isArray(payload.clients) &&
payload.clients.length > 0 &&
!DatabaseEngine.getPool()
) {
hydrateLocalMapFromRemote(payload.clients);
}
} catch { } catch {
// Port 4000 may not be running in isolated single-process dev mode // Response body optional
}
return { synced: true, status: res.status };
} catch (err: unknown) {
const message = err instanceof Error ? err.message : "Bridge bulk_sync unreachable";
if (
process.env.NODE_ENV === "production" ||
Boolean(process.env.INGESTION_INTERNAL_URL)
) {
console.warn(`[DataStack Bridge] ${message}`);
}
return { synced: false, error: message };
} }
} }
@@ -214,12 +388,20 @@ export class SDKManager {
if (res.ok) { if (res.ok) {
const data = (await res.json()) as { const data = (await res.json()) as {
hasSyncedInMemoryClients?: boolean; hasSyncedInMemoryClients?: boolean;
inMemoryClients?: RemoteInMemoryClient[];
clientActivity?: Record< clientActivity?: Record<
string, string,
{ totalEvents?: number; lastActive?: string } { totalEvents?: number; lastActive?: string }
>; >;
}; };
if (data.hasSyncedInMemoryClients === false && map.size > 0) { // BUG-02: Non-destructive hydration when Port 3000 restarted empty while Port 4000 still holds clients
if (
map.size === 0 &&
Array.isArray(data.inMemoryClients) &&
data.inMemoryClients.length > 0
) {
hydrateLocalMapFromRemote(data.inMemoryClients);
} else if (data.hasSyncedInMemoryClients === false && map.size > 0) {
await this.syncAllClientsToIngestionAsync(); await this.syncAllClientsToIngestionAsync();
} }
if (data.clientActivity && typeof data.clientActivity === "object") { if (data.clientActivity && typeof data.clientActivity === "object") {
@@ -255,36 +437,63 @@ export class SDKManager {
return Array.from(map.values()); return Array.from(map.values());
} }
public static normalizeClientId(clientId: string): string {
return String(clientId || "")
.trim()
.toLowerCase()
.replace(/[^a-z0-9_-]/g, "_");
}
public static getClient(clientId: string): ClientSDK | undefined { public static getClient(clientId: string): ClientSDK | undefined {
return getClientsMap().get(clientId); const normalizedId = this.normalizeClientId(clientId);
if (!normalizedId) return undefined;
return getClientsMap().get(normalizedId);
} }
public static async getClientAsync( public static async getClientAsync(
clientId: string, clientId: string,
): Promise<ClientSDK | undefined> { ): Promise<ClientSDK | undefined> {
await ensureDbSettingsHydrated(); await ensureDbSettingsHydrated();
const normalizedId = this.normalizeClientId(clientId);
if (!normalizedId) return undefined;
const pool = DatabaseEngine.getPool(); const pool = DatabaseEngine.getPool();
if (pool) { if (pool) {
try { try {
await DatabaseEngine.ensureSchema(); await DatabaseEngine.ensureSchema();
if (getPendingClientActivityMap().has(normalizedId)) {
await this.flushPendingClientActivityAsync();
}
const result = await pool.query( const result = await pool.query(
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at `SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
FROM client_sdks FROM client_sdks
WHERE client_id = $1`, WHERE client_id = $1`,
[clientId], [normalizedId],
); );
if (result.rowCount && result.rowCount > 0) { if (result.rowCount && result.rowCount > 0) {
const client = mapRowToClient(result.rows[0]); const client = mapRowToClient(result.rows[0]);
getClientsMap().set(client.clientId, client); getClientsMap().set(client.clientId, client);
return client; return client;
} }
getClientsMap().delete(clientId); getClientsMap().delete(normalizedId);
return undefined; return undefined;
} catch { } catch {
// Fall back to in-memory map // Fall back to in-memory map
} }
} }
return getClientsMap().get(clientId); const existing = getClientsMap().get(normalizedId);
if (existing) {
return existing;
}
// BUG-02: If Port 3000 restarted without DB and local map is empty, attempt non-destructive hydration from Port 4000
if (getClientsMap().size === 0 && !getDeletedClientsSet().has(normalizedId)) {
await this.listClientsAsync();
return getClientsMap().get(normalizedId);
}
return undefined;
}
private static generateClientSecret(safeClientId: string): string {
return `sec_${safeClientId}_${randomBytes(16).toString("hex")}`;
} }
public static registerClient(data: { public static registerClient(data: {
@@ -292,11 +501,8 @@ export class SDKManager {
name: string; name: string;
rateLimit?: number; rateLimit?: number;
}): ClientSDK { }): ClientSDK {
const safeClientId = data.clientId const safeClientId = this.normalizeClientId(data.clientId);
.trim() const secret = this.generateClientSecret(safeClientId);
.toLowerCase()
.replace(/[^a-z0-9_-]/g, "_");
const secret = `sec_${safeClientId}_${Math.random().toString(36).substring(2, 10)}`;
const rateLimit = Math.max(1, Number(data.rateLimit) || 100); const rateLimit = Math.max(1, Number(data.rateLimit) || 100);
const token = AuthService.generateClientToken({ const token = AuthService.generateClientToken({
@@ -317,8 +523,12 @@ export class SDKManager {
}; };
getDeletedClientsSet().delete(safeClientId); getDeletedClientsSet().delete(safeClientId);
DataBufferEngine.unmarkClientDeleted(safeClientId);
getSyncedSocketEventsMap().delete(safeClientId);
getClientsMap().set(safeClientId, newClient); getClientsMap().set(safeClientId, newClient);
void this.persistNewClient(newClient); void this.persistNewClient(newClient).catch((err) => {
console.error("[DataStack SDK] Background client create DB persist failed:", err);
});
notifyIngestionClientSync(safeClientId, "registered", newClient); notifyIngestionClientSync(safeClientId, "registered", newClient);
return newClient; return newClient;
} }
@@ -327,8 +537,12 @@ export class SDKManager {
const pool = DatabaseEngine.getPool(); const pool = DatabaseEngine.getPool();
if (!pool) return; if (!pool) return;
const ready = await DatabaseEngine.ensureSchema();
if (!ready) {
throw new Error("DATABASE_WRITE_FAILED: PostgreSQL schema unavailable for client registration.");
}
try { try {
await DatabaseEngine.ensureSchema();
await pool.query( await pool.query(
`DELETE FROM deleted_clients WHERE client_id = $1`, `DELETE FROM deleted_clients WHERE client_id = $1`,
[client.clientId], [client.clientId],
@@ -354,8 +568,9 @@ export class SDKManager {
client.createdAt, client.createdAt,
], ],
); );
} catch { } catch (err: unknown) {
// In-memory map already updated const detail = err instanceof Error ? err.message : "Unknown DB error";
throw new Error(`DATABASE_WRITE_FAILED: Failed to register client (${detail})`);
} }
} }
@@ -365,11 +580,8 @@ export class SDKManager {
rateLimit?: number; rateLimit?: number;
}): Promise<ClientSDK> { }): Promise<ClientSDK> {
await ensureDbSettingsHydrated(); await ensureDbSettingsHydrated();
const safeClientId = data.clientId const safeClientId = this.normalizeClientId(data.clientId);
.trim() const secret = this.generateClientSecret(safeClientId);
.toLowerCase()
.replace(/[^a-z0-9_-]/g, "_");
const secret = `sec_${safeClientId}_${Math.random().toString(36).substring(2, 10)}`;
const rateLimit = Math.max(1, Number(data.rateLimit) || 100); const rateLimit = Math.max(1, Number(data.rateLimit) || 100);
const token = AuthService.generateClientToken({ const token = AuthService.generateClientToken({
@@ -389,20 +601,23 @@ export class SDKManager {
createdAt: new Date().toISOString(), createdAt: new Date().toISOString(),
}; };
getDeletedClientsSet().delete(safeClientId);
getClientsMap().set(safeClientId, newClient);
await this.persistNewClient(newClient); await this.persistNewClient(newClient);
getDeletedClientsSet().delete(safeClientId);
DataBufferEngine.unmarkClientDeleted(safeClientId);
getSyncedSocketEventsMap().delete(safeClientId);
getClientsMap().set(safeClientId, newClient);
await notifyIngestionClientSyncAsync(safeClientId, "registered", newClient); await notifyIngestionClientSyncAsync(safeClientId, "registered", newClient);
return newClient; return newClient;
} }
public static regenerateToken(clientId: string): ClientSDK | null { public static regenerateToken(clientId: string): ClientSDK | null {
const client = getClientsMap().get(clientId); const normalizedId = this.normalizeClientId(clientId);
const client = getClientsMap().get(normalizedId);
if (!client) return null; if (!client) return null;
const updated: ClientSDK = { const updated: ClientSDK = {
...client, ...client,
secret: `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`, secret: this.generateClientSecret(normalizedId),
token: AuthService.generateClientToken({ token: AuthService.generateClientToken({
clientId: client.clientId, clientId: client.clientId,
name: client.name, name: client.name,
@@ -410,9 +625,11 @@ export class SDKManager {
}), }),
}; };
getClientsMap().set(clientId, updated); getClientsMap().set(normalizedId, updated);
void this.persistClientCredentials(updated); void this.persistClientCredentials(updated).catch((err) => {
notifyIngestionClientSync(clientId, "updated", updated); console.error("[DataStack SDK] Background token rotate DB persist failed:", err);
});
notifyIngestionClientSync(normalizedId, "updated", updated);
return updated; return updated;
} }
@@ -422,16 +639,21 @@ export class SDKManager {
const pool = DatabaseEngine.getPool(); const pool = DatabaseEngine.getPool();
if (!pool) return; if (!pool) return;
const ready = await DatabaseEngine.ensureSchema();
if (!ready) {
throw new Error("DATABASE_WRITE_FAILED: PostgreSQL schema unavailable for credential update.");
}
try { try {
await DatabaseEngine.ensureSchema();
await pool.query( await pool.query(
`UPDATE client_sdks `UPDATE client_sdks
SET secret = $2, token = $3, updated_at = NOW() SET secret = $2, token = $3, updated_at = NOW()
WHERE client_id = $1`, WHERE client_id = $1`,
[client.clientId, client.secret, client.token], [client.clientId, client.secret, client.token],
); );
} catch { } catch (err: unknown) {
// Ignore DB failure const detail = err instanceof Error ? err.message : "Unknown DB error";
throw new Error(`DATABASE_WRITE_FAILED: Failed to update client credentials (${detail})`);
} }
} }
@@ -439,12 +661,13 @@ export class SDKManager {
clientId: string, clientId: string,
): Promise<ClientSDK | null> { ): Promise<ClientSDK | null> {
await ensureDbSettingsHydrated(); await ensureDbSettingsHydrated();
const existing = await this.getClientAsync(clientId); const normalizedId = this.normalizeClientId(clientId);
const existing = await this.getClientAsync(normalizedId);
if (!existing) return null; if (!existing) return null;
const updated: ClientSDK = { const updated: ClientSDK = {
...existing, ...existing,
secret: `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`, secret: this.generateClientSecret(normalizedId),
token: AuthService.generateClientToken({ token: AuthService.generateClientToken({
clientId: existing.clientId, clientId: existing.clientId,
name: existing.name, name: existing.name,
@@ -452,9 +675,9 @@ export class SDKManager {
}), }),
}; };
getClientsMap().set(clientId, updated);
await this.persistClientCredentials(updated); await this.persistClientCredentials(updated);
await notifyIngestionClientSyncAsync(clientId, "updated", updated); getClientsMap().set(normalizedId, updated);
await notifyIngestionClientSyncAsync(normalizedId, "updated", updated);
return updated; return updated;
} }
@@ -462,7 +685,8 @@ export class SDKManager {
clientId: string, clientId: string,
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>, updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
): ClientSDK | null { ): ClientSDK | null {
const client = getClientsMap().get(clientId); const normalizedId = this.normalizeClientId(clientId);
const client = getClientsMap().get(normalizedId);
if (!client) return null; if (!client) return null;
const updated: ClientSDK = { const updated: ClientSDK = {
@@ -474,10 +698,12 @@ export class SDKManager {
...(updates.status !== undefined ? { status: updates.status } : {}), ...(updates.status !== undefined ? { status: updates.status } : {}),
}; };
getClientsMap().set(clientId, updated); getClientsMap().set(normalizedId, updated);
void this.persistClientUpdates(updated); void this.persistClientUpdates(updated).catch((err) => {
console.error("[DataStack SDK] Background client update DB persist failed:", err);
});
notifyIngestionClientSync( notifyIngestionClientSync(
clientId, normalizedId,
updated.status === "revoked" ? "revoked" : "updated", updated.status === "revoked" ? "revoked" : "updated",
updated, updated,
); );
@@ -488,16 +714,21 @@ export class SDKManager {
const pool = DatabaseEngine.getPool(); const pool = DatabaseEngine.getPool();
if (!pool) return; if (!pool) return;
const ready = await DatabaseEngine.ensureSchema();
if (!ready) {
throw new Error("DATABASE_WRITE_FAILED: PostgreSQL schema unavailable for client update.");
}
try { try {
await DatabaseEngine.ensureSchema();
await pool.query( await pool.query(
`UPDATE client_sdks `UPDATE client_sdks
SET name = $2, rate_limit = $3, status = $4, updated_at = NOW() SET name = $2, rate_limit = $3, status = $4, updated_at = NOW()
WHERE client_id = $1`, WHERE client_id = $1`,
[client.clientId, client.name, client.rateLimit, client.status], [client.clientId, client.name, client.rateLimit, client.status],
); );
} catch { } catch (err: unknown) {
// Ignore DB failure const detail = err instanceof Error ? err.message : "Unknown DB error";
throw new Error(`DATABASE_WRITE_FAILED: Failed to update client (${detail})`);
} }
} }
@@ -506,7 +737,8 @@ export class SDKManager {
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>, updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
): Promise<ClientSDK | null> { ): Promise<ClientSDK | null> {
await ensureDbSettingsHydrated(); await ensureDbSettingsHydrated();
const client = await this.getClientAsync(clientId); const normalizedId = this.normalizeClientId(clientId);
const client = await this.getClientAsync(normalizedId);
if (!client) return null; if (!client) return null;
const updated: ClientSDK = { const updated: ClientSDK = {
@@ -518,10 +750,10 @@ export class SDKManager {
...(updates.status !== undefined ? { status: updates.status } : {}), ...(updates.status !== undefined ? { status: updates.status } : {}),
}; };
getClientsMap().set(clientId, updated);
await this.persistClientUpdates(updated); await this.persistClientUpdates(updated);
getClientsMap().set(normalizedId, updated);
await notifyIngestionClientSyncAsync( await notifyIngestionClientSyncAsync(
clientId, normalizedId,
updated.status === "revoked" ? "revoked" : "updated", updated.status === "revoked" ? "revoked" : "updated",
updated, updated,
); );
@@ -529,16 +761,12 @@ export class SDKManager {
} }
public static isClientDeleted(clientId: string): boolean { public static isClientDeleted(clientId: string): boolean {
const normalized = String(clientId || "") const normalized = this.normalizeClientId(clientId);
.trim()
.toLowerCase();
return getDeletedClientsSet().has(normalized); return getDeletedClientsSet().has(normalized);
} }
public static async isClientDeletedAsync(clientId: string): Promise<boolean> { public static async isClientDeletedAsync(clientId: string): Promise<boolean> {
const normalized = String(clientId || "") const normalized = this.normalizeClientId(clientId);
.trim()
.toLowerCase();
if (!normalized) return false; if (!normalized) return false;
if (getDeletedClientsSet().has(normalized)) { if (getDeletedClientsSet().has(normalized)) {
return true; return true;
@@ -560,15 +788,18 @@ export class SDKManager {
} }
public static deleteClient(clientId: string): boolean { public static deleteClient(clientId: string): boolean {
const normalized = String(clientId || "") const normalized = this.normalizeClientId(clientId);
.trim()
.toLowerCase();
const deleted = getClientsMap().delete(normalized); const deleted = getClientsMap().delete(normalized);
getSyncedSocketEventsMap().delete(normalized); getSyncedSocketEventsMap().delete(normalized);
getPendingClientActivityMap().delete(normalized);
if (deleted) { if (deleted) {
getDeletedClientsSet().add(normalized); getDeletedClientsSet().add(normalized);
DataBufferEngine.markClientDeleted(normalized);
void DataBufferEngine.purgeClientEventsAsync(normalized).catch(() => undefined);
} }
void this.deleteClientFromDb(normalized); void this.deleteClientFromDb(normalized).catch((err) => {
console.error("[DataStack SDK] Background client delete DB persist failed:", err);
});
notifyIngestionClientSync(normalized, "deleted"); notifyIngestionClientSync(normalized, "deleted");
return deleted; return deleted;
} }
@@ -577,7 +808,10 @@ export class SDKManager {
const pool = DatabaseEngine.getPool(); const pool = DatabaseEngine.getPool();
if (!pool) return false; if (!pool) return false;
await DatabaseEngine.ensureSchema(); const ready = await DatabaseEngine.ensureSchema();
if (!ready) {
throw new Error("DATABASE_WRITE_FAILED: PostgreSQL schema unavailable for client deletion.");
}
const dbClient = await pool.connect(); const dbClient = await pool.connect();
try { try {
await dbClient.query("BEGIN"); await dbClient.query("BEGIN");
@@ -597,69 +831,226 @@ export class SDKManager {
} }
await dbClient.query("ROLLBACK"); await dbClient.query("ROLLBACK");
return false; return false;
} catch { } catch (err: unknown) {
await dbClient.query("ROLLBACK").catch(() => undefined); await dbClient.query("ROLLBACK").catch(() => undefined);
return false; const detail = err instanceof Error ? err.message : "Unknown DB error";
throw new Error(`DATABASE_WRITE_FAILED: Failed to delete client (${detail})`);
} finally { } finally {
dbClient.release(); dbClient.release();
} }
} }
private static purgeClientStorageDir(clientId: string): void { private static purgeClientStorageDir(clientId: string): number {
const safeId = clientId.replace(/[^a-z0-9_-]/gi, "_"); const safeId = clientId.replace(/[^a-z0-9_-]/gi, "_");
if (!safeId) return; if (!safeId) return 0;
const storageRoot = resolve(process.cwd(), "storage", "data"); const storageRoot = resolve(CSVStorageManager.baseStorageDir);
const targetDir = resolve(storageRoot, safeId); const targetDir = resolve(storageRoot, safeId);
if (!targetDir.startsWith(`${storageRoot}${sep}`)) return; if (!targetDir.startsWith(`${storageRoot}${sep}`)) return 0;
if (existsSync(targetDir)) { if (!existsSync(targetDir)) return 0;
rmSync(targetDir, { recursive: true, force: true });
let purgedFiles = 0;
try {
const dirStat = lstatSync(targetDir);
if (!dirStat.isSymbolicLink() && dirStat.isDirectory()) {
for (const entry of readdirSync(targetDir)) {
if (!entry.endsWith(".csv")) continue;
const entryPath = resolve(targetDir, entry);
try {
const entryStat = lstatSync(entryPath);
if (!entryStat.isSymbolicLink() && entryStat.isFile()) {
purgedFiles += 1;
} }
} catch {
// Ignore unreadable file entry during pre-purge count
}
}
}
} catch {
purgedFiles = 0;
}
rmSync(targetDir, { recursive: true, force: true });
CSVStorageManager.clearRowCountCache();
return purgedFiles;
}
public static async deleteClientWithStatsAsync(
clientId: string,
options?: { purgeFiles?: boolean },
): Promise<{ deleted: boolean; purgedFiles: number }> {
await ensureDbSettingsHydrated();
const normalized = this.normalizeClientId(clientId);
getPendingClientActivityMap().delete(normalized);
const deletedFromDb = await this.deleteClientFromDb(normalized);
const existedInMem = getClientsMap().delete(normalized);
getSyncedSocketEventsMap().delete(normalized);
let purgedFiles = 0;
if (existedInMem || deletedFromDb) {
getDeletedClientsSet().add(normalized);
DataBufferEngine.markClientDeleted(normalized);
await DataBufferEngine.purgeClientEventsAsync(normalized);
await notifyIngestionClientSyncAsync(normalized, "deleted");
if (options?.purgeFiles) {
purgedFiles = this.purgeClientStorageDir(normalized);
}
}
return { deleted: existedInMem || deletedFromDb, purgedFiles };
} }
public static async deleteClientAsync( public static async deleteClientAsync(
clientId: string, clientId: string,
options?: { purgeFiles?: boolean }, options?: { purgeFiles?: boolean },
): Promise<boolean> { ): Promise<boolean> {
await ensureDbSettingsHydrated(); const result = await this.deleteClientWithStatsAsync(clientId, options);
const normalized = String(clientId || "") return result.deleted;
.trim()
.toLowerCase();
const existedInMem = getClientsMap().delete(normalized);
getSyncedSocketEventsMap().delete(normalized);
const deletedFromDb = await this.deleteClientFromDb(normalized);
if (existedInMem || deletedFromDb) {
getDeletedClientsSet().add(normalized);
if (options?.purgeFiles) {
this.purgeClientStorageDir(normalized);
}
await notifyIngestionClientSyncAsync(normalized, "deleted");
}
return existedInMem || deletedFromDb;
} }
public static incrementEventCount(clientId: string, count = 1): void { private static queuePendingActivityDelta(
const nowIso = new Date().toISOString(); normalizedId: string,
const client = getClientsMap().get(clientId); count: number,
if (client) { nowIso: string,
client.totalEvents += count; ): number {
client.lastActive = nowIso; const pending = getPendingClientActivityMap();
getClientsMap().set(clientId, client); const existing = pending.get(normalizedId);
if (existing) {
pending.set(normalizedId, {
count: existing.count + count,
lastActive:
nowIso > existing.lastActive ? nowIso : existing.lastActive,
});
} else {
pending.set(normalizedId, {
count,
lastActive: nowIso,
});
}
ensureActivityFlushTimer();
let totalDelta = 0;
for (const entry of pending.values()) {
totalDelta += entry.count;
}
return totalDelta;
}
public static async flushPendingClientActivityAsync(): Promise<number> {
if (globals.__dataStackActivityFlushPromise) {
await globals.__dataStackActivityFlushPromise;
if (getPendingClientActivityMap().size === 0) return 0;
} }
const pool = DatabaseEngine.getPool(); const pool = DatabaseEngine.getPool();
if (pool) { const pending = getPendingClientActivityMap();
void DatabaseEngine.ensureSchema() if (!pool || pending.size === 0) return 0;
.then(() =>
pool.query( const snapshot = Array.from(pending.entries());
`UPDATE client_sdks pending.clear();
SET total_events = total_events + $2,
last_active = $3, const clientIds: string[] = [];
const deltas: number[] = [];
const lastActives: string[] = [];
for (const [cid, entry] of snapshot) {
clientIds.push(cid);
deltas.push(entry.count);
lastActives.push(entry.lastActive);
}
const run = (async () => {
try {
await DatabaseEngine.ensureSchema();
await pool.query(
`UPDATE client_sdks AS c
SET total_events = c.total_events + v.delta,
last_active = GREATEST(COALESCE(c.last_active, v.last_active), v.last_active),
updated_at = NOW() updated_at = NOW()
WHERE client_id = $1`, FROM (
[clientId, count, nowIso], SELECT UNNEST($1::text[]) AS client_id,
), UNNEST($2::bigint[]) AS delta,
) UNNEST($3::timestamptz[]) AS last_active
.catch(() => undefined); ) AS v
WHERE c.client_id = v.client_id`,
[clientIds, deltas, lastActives],
);
return clientIds.length;
} catch {
// Re-merge deltas so transient DB errors do not drop event counts
for (const [cid, entry] of snapshot) {
const current = pending.get(cid);
if (current) {
pending.set(cid, {
count: current.count + entry.count,
lastActive:
current.lastActive > entry.lastActive
? current.lastActive
: entry.lastActive,
});
} else {
pending.set(cid, { ...entry });
}
}
return 0;
} finally {
globals.__dataStackActivityFlushPromise = null;
}
})();
globals.__dataStackActivityFlushPromise = run;
return run;
}
public static stopActivityFlushTimerForTests(): void {
if (globals.__dataStackActivityFlushTimer) {
clearInterval(globals.__dataStackActivityFlushTimer);
globals.__dataStackActivityFlushTimer = null;
}
}
public static incrementEventCount(clientId: string, count = 1): void {
const normalizedId = this.normalizeClientId(clientId);
if (!normalizedId || count <= 0) return;
const nowIso = new Date().toISOString();
const client = getClientsMap().get(normalizedId);
if (client) {
client.totalEvents += count;
client.lastActive = nowIso;
getClientsMap().set(normalizedId, client);
}
if (DatabaseEngine.getPool()) {
const queuedTotal = this.queuePendingActivityDelta(
normalizedId,
count,
nowIso,
);
if (queuedTotal >= CLIENT_ACTIVITY_IMMEDIATE_BATCH_SIZE) {
void this.flushPendingClientActivityAsync().catch(() => undefined);
}
}
}
public static async incrementEventCountAsync(
clientId: string,
count = 1,
): Promise<void> {
const normalizedId = this.normalizeClientId(clientId);
if (!normalizedId || count <= 0) return;
const nowIso = new Date().toISOString();
const client = getClientsMap().get(normalizedId);
if (client) {
client.totalEvents += count;
client.lastActive = nowIso;
getClientsMap().set(normalizedId, client);
}
// BUG-22: Coalesce per-event DB updates into batched UNNEST writes instead of 1 SQL query/event
if (DatabaseEngine.getPool()) {
const queuedTotal = this.queuePendingActivityDelta(
normalizedId,
count,
nowIso,
);
if (queuedTotal >= CLIENT_ACTIVITY_IMMEDIATE_BATCH_SIZE) {
await this.flushPendingClientActivityAsync();
}
} }
} }
@@ -672,7 +1063,9 @@ export class SDKManager {
rateLimit: client.rateLimit, rateLimit: client.rateLimit,
}); });
map.set(clientId, client); map.set(clientId, client);
void this.persistClientCredentials(client); void this.persistClientCredentials(client).catch((err) => {
console.error("[DataStack SDK] Background re-sign DB persist failed:", err);
});
} }
} }
@@ -680,15 +1073,25 @@ export class SDKManager {
const clients = await this.listClientsAsync(); const clients = await this.listClientsAsync();
const map = getClientsMap(); const map = getClientsMap();
for (const client of clients) { for (const client of clients) {
client.token = AuthService.generateClientToken({ const updated: ClientSDK = {
...client,
token: AuthService.generateClientToken({
clientId: client.clientId, clientId: client.clientId,
name: client.name, name: client.name,
rateLimit: client.rateLimit, rateLimit: client.rateLimit,
}); }),
map.set(client.clientId, client); };
await this.persistClientCredentials(client); await this.persistClientCredentials(updated);
map.set(updated.clientId, updated);
} }
} }
} }
if (!globals.__dataStackActivityShutdownRegistered) {
globals.__dataStackActivityShutdownRegistered = true;
DataBufferEngine.registerShutdownHook(() =>
SDKManager.flushPendingClientActivityAsync(),
);
}
export default SDKManager; export default SDKManager;
+19 -11
View File
@@ -13,22 +13,18 @@ map $http_x_forwarded_host $forwarded_host {
'' $host; '' $host;
} }
upstream nextjs_upstream {
server web:3000;
}
upstream socketio_upstream {
server ingestion:4000;
}
server { server {
listen 80; listen 80;
server_name _; server_name _;
resolver 127.0.0.11 valid=10s ipv6=off;
set $nextjs_upstream "http://web:3000";
set $socketio_upstream "http://ingestion:4000";
client_max_body_size 50M; client_max_body_size 50M;
location = /health { location = /health {
proxy_pass http://socketio_upstream; proxy_pass $socketio_upstream;
proxy_http_version 1.1; proxy_http_version 1.1;
proxy_set_header Host $host; proxy_set_header Host $host;
proxy_set_header X-Forwarded-Host $forwarded_host; proxy_set_header X-Forwarded-Host $forwarded_host;
@@ -38,7 +34,7 @@ server {
} }
location /socket.io/ { location /socket.io/ {
proxy_pass http://socketio_upstream; proxy_pass $socketio_upstream;
proxy_http_version 1.1; proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade; proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade; proxy_set_header Connection $connection_upgrade;
@@ -52,8 +48,20 @@ server {
proxy_buffering off; proxy_buffering off;
} }
location = /api/export {
proxy_pass $nextjs_upstream;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-Host $forwarded_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $forwarded_proto;
proxy_buffering off;
proxy_read_timeout 300s;
}
location / { location / {
proxy_pass http://nextjs_upstream; proxy_pass $nextjs_upstream;
proxy_http_version 1.1; proxy_http_version 1.1;
proxy_set_header Host $host; proxy_set_header Host $host;
proxy_set_header X-Forwarded-Host $forwarded_host; proxy_set_header X-Forwarded-Host $forwarded_host;
+3
View File
@@ -25,6 +25,7 @@ services:
redis: redis:
condition: service_healthy condition: service_healthy
restart: unless-stopped restart: unless-stopped
stop_grace_period: 20s
healthcheck: healthcheck:
test: test:
[ [
@@ -54,12 +55,14 @@ services:
DATASTACK_INTERNAL_SECRET: "${DATASTACK_INTERNAL_SECRET:-${SERVICE_REALBASE64_64_INTERNAL:-}}" DATASTACK_INTERNAL_SECRET: "${DATASTACK_INTERNAL_SECRET:-${SERVICE_REALBASE64_64_INTERNAL:-}}"
DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}" DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}"
REDIS_URL: "redis://redis:6379" REDIS_URL: "redis://redis:6379"
DATASTACK_CORS_ORIGINS: "${DATASTACK_CORS_ORIGINS:-}"
depends_on: depends_on:
postgres: postgres:
condition: service_healthy condition: service_healthy
redis: redis:
condition: service_healthy condition: service_healthy
restart: unless-stopped restart: unless-stopped
stop_grace_period: 20s
healthcheck: healthcheck:
test: test:
[ [
+1 -1
View File
@@ -72,7 +72,7 @@ DataStack is optimized for deployment via [Coolify](https://coolify.io/):
1. **Create Resource**: In Coolify, click **+ Add Resource** → **Docker Compose**. 1. **Create Resource**: In Coolify, click **+ Add Resource** → **Docker Compose**.
2. **Repository Settings**: 2. **Repository Settings**:
- Point to your DataStack Git repository. - Point to your DataStack Git repository.
- Set **Base Directory** to `/docker`. - Set **Base Directory** to `/` (so Compose build context `..` from `/docker` resolves cleanly to the repository root; if an existing VPS deployment already has saved working Coolify paths, keep its current settings).
- Set **Docker Compose Location** to `/docker/docker-compose.yml`. - Set **Docker Compose Location** to `/docker/docker-compose.yml`.
3. **Domain Assignment**: 3. **Domain Assignment**:
- Attach your custom domain (e.g. `https://datastack.yourdomain.com`) to the **`proxy`** service on internal port `80`. - Attach your custom domain (e.g. `https://datastack.yourdomain.com`) to the **`proxy`** service on internal port `80`.
+53
View File
@@ -0,0 +1,53 @@
import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
import { NextRequest, NextResponse } from "next/server";
export const ADMIN_SESSION_COOKIE = "datastack_admin_session";
export async function requireAdminApiSession(req: NextRequest) {
const token = req.cookies.get(ADMIN_SESSION_COOKIE)?.value;
const auth = await verifyRequestAdminSession(token);
if (auth.valid) {
return { valid: true as const, auth };
}
return {
valid: false as const,
response: NextResponse.json(
{
error:
auth.reason === "UNAUTHORIZED"
? "Authentication required."
: "Admin authentication is temporarily unavailable.",
},
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
),
};
}
/**
* Extracts the client IP for authentication throttling in a reverse-proxy-safe order:
* 1. `x-real-ip` (overwritten directly by Nginx/Coolify with `$remote_addr`)
* 2. Rightmost non-empty hop of `x-forwarded-for` (appended by the trusted edge proxy,
* preventing attackers from spoofing arbitrary leftmost client IPs)
* 3. Fallback `"unknown"`
*/
export function extractClientIp(request: NextRequest): string {
const realIp = request.headers.get("x-real-ip")?.trim();
if (realIp) {
return realIp.slice(0, 128);
}
const forwardedFor = request.headers.get("x-forwarded-for");
if (forwardedFor) {
const hops = forwardedFor
.split(",")
.map((part) => part.trim())
.filter(Boolean);
const rightmost = hops[hops.length - 1];
if (rightmost) {
return rightmost.slice(0, 128);
}
}
return "unknown";
}
+33
View File
@@ -15,3 +15,36 @@ export function setCached<T>(key: string, value: T): T {
memoryStore.set(key, value); memoryStore.set(key, value);
return value; return value;
} }
export async function clearAuthCaches(): Promise<void> {
memoryStore.clear();
if (typeof window === 'undefined') return;
try {
if ('serviceWorker' in navigator && navigator.serviceWorker.controller) {
navigator.serviceWorker.controller.postMessage({ type: 'CLEAR_AUTH_CACHE' });
}
if ('caches' in window) {
const keys = await window.caches.keys();
await Promise.all(
keys.map(async (cacheKey) => {
const cache = await window.caches.open(cacheKey);
const requests = await cache.keys();
await Promise.all(
requests
.filter((req) => {
try {
const u = new URL(req.url);
return u.pathname === '/vcon' || u.pathname.startsWith('/vcon/');
} catch {
return false;
}
})
.map((req) => cache.delete(req))
);
})
);
}
} catch {
// Best-effort cache cleanup
}
}
+537
View File
@@ -0,0 +1,537 @@
export interface EditableSettings {
maxEventsPerSec: number;
maxPayloadSizeKb: number;
batchFlushThreshold: number;
batchFlushIntervalMs: number;
fileRotationMb: number;
enableIdempotency: boolean;
autoPruneDays: number;
}
export const SETTINGS_LIMITS = {
maxEventsPerSec: { min: 1, max: 100000, defaultValue: 100 },
maxPayloadSizeKb: { min: 1, max: 10240, defaultValue: 100 },
batchFlushThreshold: { min: 1, max: 50000, defaultValue: 500 },
batchFlushIntervalMs: { min: 100, max: 60000, defaultValue: 2000 },
fileRotationMb: { min: 1, max: 2048, defaultValue: 100 },
autoPruneDays: { min: 0, max: 3650, defaultValue: 30 },
} as const;
export const DEFAULT_EDITABLE_SETTINGS: EditableSettings = {
maxEventsPerSec: SETTINGS_LIMITS.maxEventsPerSec.defaultValue,
maxPayloadSizeKb: SETTINGS_LIMITS.maxPayloadSizeKb.defaultValue,
batchFlushThreshold: SETTINGS_LIMITS.batchFlushThreshold.defaultValue,
batchFlushIntervalMs: SETTINGS_LIMITS.batchFlushIntervalMs.defaultValue,
fileRotationMb: SETTINGS_LIMITS.fileRotationMb.defaultValue,
enableIdempotency: true,
autoPruneDays: SETTINGS_LIMITS.autoPruneDays.defaultValue,
};
function clampInt(raw: unknown, min: number, max: number, fallback: number): number {
const numeric = typeof raw === 'number' ? raw : Number(raw);
if (!Number.isFinite(numeric)) return fallback;
const rounded = Math.trunc(numeric);
return Math.max(min, Math.min(max, rounded));
}
export function normalizeEditableSettings(
input: Partial<EditableSettings>,
): EditableSettings {
return {
maxEventsPerSec: clampInt(
input.maxEventsPerSec,
SETTINGS_LIMITS.maxEventsPerSec.min,
SETTINGS_LIMITS.maxEventsPerSec.max,
SETTINGS_LIMITS.maxEventsPerSec.defaultValue,
),
maxPayloadSizeKb: clampInt(
input.maxPayloadSizeKb,
SETTINGS_LIMITS.maxPayloadSizeKb.min,
SETTINGS_LIMITS.maxPayloadSizeKb.max,
SETTINGS_LIMITS.maxPayloadSizeKb.defaultValue,
),
batchFlushThreshold: clampInt(
input.batchFlushThreshold,
SETTINGS_LIMITS.batchFlushThreshold.min,
SETTINGS_LIMITS.batchFlushThreshold.max,
SETTINGS_LIMITS.batchFlushThreshold.defaultValue,
),
batchFlushIntervalMs: clampInt(
input.batchFlushIntervalMs,
SETTINGS_LIMITS.batchFlushIntervalMs.min,
SETTINGS_LIMITS.batchFlushIntervalMs.max,
SETTINGS_LIMITS.batchFlushIntervalMs.defaultValue,
),
fileRotationMb: clampInt(
input.fileRotationMb,
SETTINGS_LIMITS.fileRotationMb.min,
SETTINGS_LIMITS.fileRotationMb.max,
SETTINGS_LIMITS.fileRotationMb.defaultValue,
),
enableIdempotency:
typeof input.enableIdempotency === 'boolean'
? input.enableIdempotency
: DEFAULT_EDITABLE_SETTINGS.enableIdempotency,
autoPruneDays: clampInt(
input.autoPruneDays,
SETTINGS_LIMITS.autoPruneDays.min,
SETTINGS_LIMITS.autoPruneDays.max,
SETTINGS_LIMITS.autoPruneDays.defaultValue,
),
};
}
export interface BridgeSyncPayload {
attempted?: boolean;
ok?: boolean;
synced?: boolean;
status?: number;
error?: string;
checkedAt?: string;
}
export function describeSettingsBridgeSync(
bridgeSync: BridgeSyncPayload | undefined,
): { level: 'success' | 'warning' | 'error'; title: string; message?: string } {
if (!bridgeSync) {
return { level: 'success', title: 'Settings saved' };
}
const attempted =
bridgeSync.attempted !== undefined
? bridgeSync.attempted
: typeof bridgeSync.synced === 'boolean';
const ok =
bridgeSync.ok !== undefined
? bridgeSync.ok
: Boolean(bridgeSync.synced);
if (!attempted || ok) {
return { level: 'success', title: 'Settings saved' };
}
if (bridgeSync.status === 401 || bridgeSync.status === 503) {
return {
level: 'error',
title: 'Saved to DB — Ingestion bridge rejected sync',
message:
bridgeSync.error ||
`Port 4000 returned HTTP ${bridgeSync.status}. Verify DATASTACK_INTERNAL_SECRET matches across runtimes.`,
};
}
return {
level: 'warning',
title: 'Saved to DB — Ingestion bridge unreachable',
message:
bridgeSync.error ||
'Port 4000 ingestion server is offline; changes will apply once it reconnects.',
};
}
export type IngestionBridgeStatus = 'online' | 'auth_error' | 'error' | 'offline';
export type SocketHealthUiState =
| 'idle'
| 'checking'
| 'online'
| 'auth_error'
| 'offline';
export function mapBridgeStatusToSocketUiState(
status?: string,
): SocketHealthUiState {
if (status === 'online') return 'online';
if (status === 'auth_error') return 'auth_error';
return 'offline';
}
export function parseEventTimestamp(value: string): Date | null {
const trimmed = String(value ?? '').trim();
if (!trimmed) return null;
if (/^\d+(\.\d+)?$/.test(trimmed)) {
const asNumber = Number(trimmed);
if (Number.isFinite(asNumber)) {
const ms = asNumber < 1e11 ? asNumber * 1000 : asNumber;
const date = new Date(ms);
return Number.isNaN(date.getTime()) ? null : date;
}
}
const date = new Date(trimmed);
return Number.isNaN(date.getTime()) ? null : date;
}
export function getEventUtcDate(value: string): string | null {
const parsed = parseEventTimestamp(value);
return parsed ? parsed.toISOString().slice(0, 10) : null;
}
export function formatEventTimeLabel(
value: string,
options: Intl.DateTimeFormatOptions = {
hour: '2-digit',
minute: '2-digit',
second: '2-digit',
hour12: false,
},
): string {
const parsed = parseEventTimestamp(value);
return parsed ? parsed.toLocaleTimeString([], options) : value || '—';
}
export function getUtcDateRangeForPreset(
preset: 'today' | '7d' | '30d' | 'all',
referenceNow: Date = new Date(),
): { from: string; to: string } {
if (preset === 'all') {
return { from: '', to: '' };
}
const toUtc = new Date(
Date.UTC(
referenceNow.getUTCFullYear(),
referenceNow.getUTCMonth(),
referenceNow.getUTCDate(),
),
);
const offsetDays = preset === '7d' ? 6 : preset === '30d' ? 29 : 0;
const fromUtc = new Date(toUtc.getTime() - offsetDays * 86_400_000);
return {
from: fromUtc.toISOString().slice(0, 10),
to: toUtc.toISOString().slice(0, 10),
};
}
export function sanitizeUiClientId(clientId: string): string {
return clientId.trim().replace(/[^a-zA-Z0-9_-]/g, '_');
}
export function matchesSelectedClientFolder(
folderClientId: string,
selectedClientId: string,
): boolean {
if (!selectedClientId || selectedClientId === 'all') return true;
const trimmed = selectedClientId.trim();
return (
folderClientId === trimmed ||
folderClientId === sanitizeUiClientId(trimmed)
);
}
export interface BufferedBucket {
clientId: string;
date: string;
count: number;
}
export interface BufferedSummary {
total: number;
totalQueued: number;
unattributedCount: number;
truncated: boolean;
buckets: BufferedBucket[];
}
export interface FilteredBufferedMatch {
matchingCount: number;
unattributedCount: number;
totalQueued: number;
hasPotentialBuffered: boolean;
}
export function countMatchingBufferedEvents(
summary: BufferedSummary | null | undefined,
filter: { clientId?: string; fromDate?: string; toDate?: string } = {},
fallbackGlobalBufferCount = 0,
): FilteredBufferedMatch {
if (!summary) {
const safeFallback = Math.max(0, Number(fallbackGlobalBufferCount) || 0);
return {
matchingCount: 0,
unattributedCount: safeFallback,
totalQueued: safeFallback,
hasPotentialBuffered: safeFallback > 0,
};
}
const selectedClient = filter.clientId || 'all';
const fromDate = filter.fromDate || '';
const toDate = filter.toDate || '';
const buckets = Array.isArray(summary.buckets) ? summary.buckets : [];
let matchingCount = 0;
for (const bucket of buckets) {
if (!bucket || typeof bucket.count !== 'number' || bucket.count <= 0) continue;
if (!matchesSelectedClientFolder(bucket.clientId, selectedClient)) continue;
if (fromDate && bucket.date < fromDate) continue;
if (toDate && bucket.date > toDate) continue;
matchingCount += bucket.count;
}
const unattributedCount = Math.max(0, Number(summary.unattributedCount) || 0);
const totalQueued = Math.max(
matchingCount + unattributedCount,
Number(summary.totalQueued) || 0,
);
return {
matchingCount,
unattributedCount,
totalQueued,
hasPotentialBuffered: matchingCount > 0 || unattributedCount > 0,
};
}
export interface HttpBatchSummary {
accepted: number;
rejected: number;
duplicates: number;
status: 'Accepted' | 'Rejected';
level: 'success' | 'warning' | 'error';
title: string;
message: string;
}
export function summarizeHttpBatchResult(
resOk: boolean,
data: Record<string, unknown> | null | undefined,
sentCount: number,
): HttpBatchSummary {
const safeSent = Math.max(0, Math.trunc(sentCount));
const rawAccepted =
typeof data?.accepted === 'number' && Number.isFinite(data.accepted)
? Math.max(0, Math.trunc(data.accepted))
: typeof data?.processed === 'number' && Number.isFinite(data.processed)
? Math.max(0, Math.trunc(data.processed))
: null;
const rawDuplicates =
typeof data?.duplicates === 'number' && Number.isFinite(data.duplicates)
? Math.max(0, Math.trunc(data.duplicates))
: 0;
const rawRejected =
typeof data?.rejected === 'number' && Number.isFinite(data.rejected)
? Math.max(0, Math.trunc(data.rejected))
: null;
const serverError =
typeof data?.error === 'string' && data.error.trim() ? data.error.trim() : '';
if (rawAccepted === null) {
return {
accepted: 0,
rejected: rawRejected ?? safeSent,
duplicates: rawDuplicates,
status: 'Rejected',
level: 'error',
title: 'Batch Rejected',
message:
serverError ||
`Server response did not confirm batch ingestion (0/${safeSent} ingested).`,
};
}
const accepted = rawAccepted;
const duplicates = rawDuplicates;
const rejected =
rawRejected !== null
? rawRejected
: Math.max(0, safeSent - accepted - duplicates);
if (!resOk || (accepted === 0 && duplicates === 0)) {
return {
accepted,
rejected,
duplicates,
status: 'Rejected',
level: 'error',
title: 'Batch Rejected',
message: serverError || `${accepted}/${safeSent} ingested`,
};
}
if (rejected > 0 || accepted + duplicates < safeSent) {
const detail =
duplicates > 0
? `${accepted}/${safeSent} ingested (${rejected} rejected, ${duplicates} duplicate)`
: `${accepted}/${safeSent} ingested (${rejected} rejected)`;
return {
accepted,
rejected,
duplicates,
status: 'Rejected',
level: 'warning',
title: 'Batch Partial',
message: detail,
};
}
const message =
duplicates > 0
? `${accepted}/${safeSent} ingested (${duplicates} duplicate skipped)`
: `${accepted}/${safeSent} ingested`;
return {
accepted,
rejected,
duplicates,
status: 'Accepted',
level: 'success',
title: 'Batch Streamed',
message,
};
}
export function extractContentDispositionFilename(
headerValue: string | null,
fallback: string,
): string {
if (!headerValue) return fallback;
const quoted = headerValue.match(/filename="([^"]+)"/i);
if (quoted && quoted[1]) return quoted[1];
const bare = headerValue.match(/filename=([^;]+)/i);
if (bare && bare[1]) return bare[1].trim();
return fallback;
}
export type NumericSettingKey = keyof typeof SETTINGS_LIMITS;
export type NumericSettingDrafts = Record<NumericSettingKey, string>;
export type SettingsValidationErrors = Partial<Record<NumericSettingKey, string>>;
export const NUMERIC_SETTING_KEYS: readonly NumericSettingKey[] = [
'maxEventsPerSec',
'maxPayloadSizeKb',
'batchFlushThreshold',
'batchFlushIntervalMs',
'fileRotationMb',
'autoPruneDays',
] as const;
export function settingsToNumericDrafts(
settings: EditableSettings,
): NumericSettingDrafts {
return {
maxEventsPerSec: String(settings.maxEventsPerSec),
maxPayloadSizeKb: String(settings.maxPayloadSizeKb),
batchFlushThreshold: String(settings.batchFlushThreshold),
batchFlushIntervalMs: String(settings.batchFlushIntervalMs),
fileRotationMb: String(settings.fileRotationMb),
autoPruneDays: String(settings.autoPruneDays),
};
}
export function validateSingleNumericSetting(
key: NumericSettingKey,
rawValue: string,
): { valid: true; value: number } | { valid: false; error: string } {
const trimmed = String(rawValue ?? '').trim();
const { min, max } = SETTINGS_LIMITS[key];
if (!trimmed) {
return { valid: false, error: 'Value is required.' };
}
if (!/^-?\d+$/.test(trimmed)) {
return { valid: false, error: 'Enter a whole number.' };
}
const parsed = Number(trimmed);
if (!Number.isSafeInteger(parsed)) {
return { valid: false, error: 'Enter a valid integer.' };
}
if (parsed < min || parsed > max) {
return {
valid: false,
error: `Must be between ${min.toLocaleString()} and ${max.toLocaleString()}.`,
};
}
return { valid: true, value: parsed };
}
export function validateEditableSettingsDraft(
drafts: NumericSettingDrafts,
enableIdempotency: boolean,
): {
valid: boolean;
values?: EditableSettings;
errors: SettingsValidationErrors;
firstInvalidKey?: NumericSettingKey;
} {
const errors: SettingsValidationErrors = {};
const parsedValues: Partial<Record<NumericSettingKey, number>> = {};
let firstInvalidKey: NumericSettingKey | undefined;
for (const key of NUMERIC_SETTING_KEYS) {
const result = validateSingleNumericSetting(key, drafts[key]);
if (!result.valid) {
errors[key] = result.error;
if (!firstInvalidKey) firstInvalidKey = key;
} else {
parsedValues[key] = result.value;
}
}
if (firstInvalidKey) {
return { valid: false, errors, firstInvalidKey };
}
return {
valid: true,
errors,
values: {
maxEventsPerSec: parsedValues.maxEventsPerSec!,
maxPayloadSizeKb: parsedValues.maxPayloadSizeKb!,
batchFlushThreshold: parsedValues.batchFlushThreshold!,
batchFlushIntervalMs: parsedValues.batchFlushIntervalMs!,
fileRotationMb: parsedValues.fileRotationMb!,
autoPruneDays: parsedValues.autoPruneDays!,
enableIdempotency: Boolean(enableIdempotency),
},
};
}
export interface BrowserLocationLike {
protocol: string;
hostname: string;
host: string;
port: string;
origin?: string;
}
export interface PublicEndpoints {
apiUrl: string;
serverUrl: string;
webSocketUrl: string;
}
export function resolvePublicEndpoints(
loc: BrowserLocationLike,
serverPort: number,
): PublicEndpoints {
const safePort =
Number.isFinite(serverPort) && serverPort > 0 ? Math.trunc(serverPort) : 4000;
const httpProtocol = loc.protocol === 'https:' ? 'https:' : 'http:';
const wsProtocol = httpProtocol === 'https:' ? 'wss:' : 'ws:';
const apiUrl = loc.origin || `${httpProtocol}//${loc.host}`;
const isDirectNextDev = loc.port === '3000';
const ingestionHost = isDirectNextDev
? `${loc.hostname}:${safePort}`
: loc.host;
return {
apiUrl,
serverUrl: `${httpProtocol}//${ingestionHost}`,
webSocketUrl: `${wsProtocol}//${ingestionHost}`,
};
}
export function shouldRedirectProfileUnauthorized(
status: number,
body: Record<string, unknown> | null | undefined,
): boolean {
if (status !== 401) return false;
return body?.code !== 'INVALID_PASSWORD';
}
export function formatSessionExpiryLabel(value: string): string {
if (!value) return 'Active';
const parsed = parseEventTimestamp(value);
if (!parsed) return 'Active';
return parsed.toLocaleString([], {
month: 'short',
day: 'numeric',
hour: '2-digit',
minute: '2-digit',
hour12: false,
});
}
+24 -24
View File
@@ -36,6 +36,10 @@ function matchesRequestHost(candidateUrl: string, request: NextRequest) {
} }
export function isSameOriginRequest(request: NextRequest) { export function isSameOriginRequest(request: NextRequest) {
const secFetchSite = request.headers.get("sec-fetch-site")?.toLowerCase();
if (secFetchSite === "cross-site") {
return false;
}
const originHeader = request.headers.get("origin"); const originHeader = request.headers.get("origin");
if (originHeader) { if (originHeader) {
return matchesRequestHost(originHeader, request); return matchesRequestHost(originHeader, request);
@@ -44,6 +48,9 @@ export function isSameOriginRequest(request: NextRequest) {
if (refererHeader) { if (refererHeader) {
return matchesRequestHost(refererHeader, request); return matchesRequestHost(refererHeader, request);
} }
if (secFetchSite === "same-origin") {
return true;
}
return false; return false;
} }
@@ -61,34 +68,29 @@ export async function middleware(request: NextRequest) {
); );
const isStateChangingApi = const isStateChangingApi =
pathname.startsWith("/api/") && pathname.startsWith("/api/") &&
!SAFE_METHODS.has(request.method) && !SAFE_METHODS.has(request.method.toUpperCase()) &&
!(pathname === "/api/events" && hasBearerAuth); !(pathname === "/api/events" && hasBearerAuth);
if (isSetupPage) return NextResponse.next(); const requestHeaders = new Headers(request.headers);
if ( requestHeaders.set("x-datastack-pathname", pathname);
!isConsolePage && const nextWithPathname = () =>
!isAdminApi && NextResponse.next({
!isLoginPage && request: {
!(pathname === "/api/events" && request.method === "POST") headers: requestHeaders,
) { },
return NextResponse.next(); });
}
const isHeadlessAdminClient = if (isSetupPage) return nextWithPathname();
request.headers.get("x-datastack-admin-client") === "true" ||
Boolean(request.headers.get("user-agent")?.startsWith("DataStack-CLI"));
if (isStateChangingApi && !isSameOriginRequest(request)) { if (isStateChangingApi && !isSameOriginRequest(request)) {
if (!isHeadlessAdminClient) {
return NextResponse.json( return NextResponse.json(
{ error: "Cross-origin request rejected." }, { error: "Cross-origin request rejected." },
{ status: 403 }, { status: 403 },
); );
} }
}
if (pathname === "/api/events" && request.method === "POST") { if (!isConsolePage && !isAdminApi && !isLoginPage) {
return NextResponse.next(); return nextWithPathname();
} }
const sessionSecret = getAdminSessionSecret(); const sessionSecret = getAdminSessionSecret();
@@ -99,6 +101,10 @@ export async function middleware(request: NextRequest) {
); );
} }
if (isLoginPage) {
return nextWithPathname();
}
const token = request.cookies.get(SESSION_COOKIE)?.value; const token = request.cookies.get(SESSION_COOKIE)?.value;
const authenticated = Boolean( const authenticated = Boolean(
sessionSecret && sessionSecret &&
@@ -106,13 +112,7 @@ export async function middleware(request: NextRequest) {
(await verifyAdminSessionToken(token, sessionSecret)), (await verifyAdminSessionToken(token, sessionSecret)),
); );
if (isLoginPage) { if (authenticated) return nextWithPathname();
if (authenticated)
return NextResponse.redirect(new URL("/vcon/dashboard", request.url));
return NextResponse.next();
}
if (authenticated) return NextResponse.next();
if (isAdminApi) { if (isAdminApi) {
return NextResponse.json( return NextResponse.json(
-3
View File
@@ -2,9 +2,6 @@ import type {NextConfig} from 'next';
const nextConfig: NextConfig = { const nextConfig: NextConfig = {
reactStrictMode: true, reactStrictMode: true,
eslint: {
ignoreDuringBuilds: true,
},
typescript: { typescript: {
ignoreBuildErrors: false, ignoreBuildErrors: false,
}, },
+16 -1
View File
@@ -49,7 +49,22 @@ server {
proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $forwarded_proto; proxy_set_header X-Forwarded-Proto $forwarded_proto;
proxy_read_timeout 86400; proxy_read_timeout 86400s;
proxy_send_timeout 86400s;
proxy_buffering off;
}
# Streaming CSV/JSON export endpoint on port 3000 (disable response buffering)
location = /api/export {
proxy_pass http://nextjs_upstream;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-Host $forwarded_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $forwarded_proto;
proxy_buffering off;
proxy_read_timeout 300s;
} }
# Next.js web application and admin panel to port 3000 # Next.js web application and admin panel to port 3000
+130 -5287
View File
File diff suppressed because it is too large. Load diff
+7 -4
View File
@@ -1,18 +1,22 @@
{ {
"name": "ai-studio-applet", "name": "datastack",
"version": "0.1.0", "version": "0.1.0",
"private": true, "private": true,
"engines": {
"node": ">=20.0.0"
},
"scripts": { "scripts": {
"dev": "next dev -p 3000 -H 0.0.0.0", "dev": "next dev -p 3000 -H 0.0.0.0",
"dev:all": "node scripts/dev-all.js", "dev:all": "node scripts/dev-all.js",
"server": "node server/index.js", "server": "node server/index.js",
"build": "NODE_ENV=production next build", "build": "NODE_ENV=production next build",
"start": "next start", "start": "node scripts/start-all.js",
"start:web": "next start",
"start:server": "node server/index.js",
"lint": "eslint .", "lint": "eslint .",
"clean": "next clean" "clean": "next clean"
}, },
"dependencies": { "dependencies": {
"@google/genai": "^2.4.0",
"@hookform/resolvers": "^5.2.1", "@hookform/resolvers": "^5.2.1",
"autoprefixer": "^10.4.21", "autoprefixer": "^10.4.21",
"class-variance-authority": "^0.7.1", "class-variance-authority": "^0.7.1",
@@ -46,7 +50,6 @@
"@types/react-dom": "^19", "@types/react-dom": "^19",
"eslint": "9.39.1", "eslint": "9.39.1",
"eslint-config-next": "16.0.8", "eslint-config-next": "16.0.8",
"firebase-tools": "^15.0.0",
"tailwindcss": "4.1.11", "tailwindcss": "4.1.11",
"tw-animate-css": "^1.4.0", "tw-animate-css": "^1.4.0",
"typescript": "5.9.3" "typescript": "5.9.3"
+4 -2
View File
@@ -1,5 +1,7 @@
// This is your Prisma schema file, // REFERENCE SCHEMA ONLY — NOT EXECUTED BY PRISMA CLI AT RUNTIME.
// learn more about it in the docs: https://pris.ly/d/prisma-schema // Canonical PostgreSQL runtime DDL is owned by `core/Database/database.ts` (Port 3000 via `pg`)
// and mirrored 1:1 in `server/index.js` (Port 4000 standalone container).
// Keep all 5 models below strictly synchronized with `core/Database/database.ts` (verified by tests).
generator client { generator client {
provider = "prisma-client-js" provider = "prisma-client-js"
+127 -2
View File
@@ -1,4 +1,4 @@
const CACHE_NAME = 'datastack-cache-v2'; const CACHE_NAME = 'datastack-cache-v3';
const STATIC_ASSETS = [ const STATIC_ASSETS = [
'/', '/',
'/manifest.json', '/manifest.json',
@@ -9,6 +9,92 @@ const STATIC_ASSETS = [
'/apple-touch-icon.png', '/apple-touch-icon.png',
]; ];
function buildOfflineShellResponse(pathname) {
const safePath = String(pathname || '/').replace(/[<>&"']/g, '');
const html = `<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>DataStack — Offline</title>
<style>
:root { color-scheme: dark light; }
body {
margin: 0;
min-height: 100vh;
display: grid;
place-items: center;
padding: 16px;
background: #0a0a0b;
color: #f4f4f5;
font-family: "Segoe UI", system-ui, -apple-system, sans-serif;
}
.panel {
width: 100%;
max-width: 360px;
border: 1px solid #27272a;
background: #111113;
padding: 20px;
}
.brand {
display: flex;
align-items: center;
gap: 8px;
font-size: 13px;
font-weight: 600;
margin-bottom: 12px;
}
.mark {
width: 10px;
height: 10px;
background: #10b981;
display: inline-block;
}
h1 { font-size: 15px; margin: 0 0 8px; font-weight: 600; }
p { font-size: 12.5px; line-height: 1.5; color: #a1a1aa; margin: 0 0 16px; }
code { color: #e4e4e7; font-family: ui-monospace, SFMono-Regular, monospace; font-size: 11.5px; }
.actions { display: flex; gap: 8px; }
button, a {
appearance: none;
border: 1px solid #27272a;
background: #18181b;
color: #f4f4f5;
padding: 7px 12px;
font-size: 12px;
text-decoration: none;
cursor: pointer;
}
button.primary {
border-color: #10b981;
background: #10b981;
color: #09090b;
font-weight: 600;
}
</style>
</head>
<body>
<main class="panel">
<div class="brand"><span class="mark"></span>DataStack</div>
<h1>Network connection unavailable</h1>
<p>Unable to reach <code>${safePath}</code> while offline. Live ingestion metrics and authenticated admin routes require a connection to the DataStack server.</p>
<div class="actions">
<button type="button" class="primary" onclick="window.location.reload()">Retry</button>
<a href="/">Overview</a>
</div>
</main>
</body>
</html>`;
return new Response(html, {
status: 503,
statusText: 'Service Unavailable',
headers: {
'Content-Type': 'text/html; charset=utf-8',
'Cache-Control': 'no-store',
},
});
}
self.addEventListener('install', (event) => { self.addEventListener('install', (event) => {
event.waitUntil( event.waitUntil(
caches.open(CACHE_NAME).then((cache) => { caches.open(CACHE_NAME).then((cache) => {
@@ -29,6 +115,28 @@ self.addEventListener('activate', (event) => {
self.clients.claim(); self.clients.claim();
}); });
self.addEventListener('message', (event) => {
if (event.data && event.data.type === 'CLEAR_AUTH_CACHE') {
event.waitUntil(
caches.open(CACHE_NAME).then(async (cache) => {
const requests = await cache.keys();
await Promise.all(
requests
.filter((req) => {
try {
const reqUrl = new URL(req.url);
return reqUrl.pathname === '/vcon' || reqUrl.pathname.startsWith('/vcon/');
} catch {
return false;
}
})
.map((req) => cache.delete(req))
);
}).catch(() => {})
);
}
});
self.addEventListener('fetch', (event) => { self.addEventListener('fetch', (event) => {
const { request } = event; const { request } = event;
if (request.method !== 'GET') return; if (request.method !== 'GET') return;
@@ -50,10 +158,24 @@ self.addEventListener('fetch', (event) => {
return; return;
} }
// Never cache authenticated /vcon/* responses, but provide an offline HTML shell on PWA launch / navigation failure
if (url.pathname === '/vcon' || url.pathname.startsWith('/vcon/')) {
if (request.mode !== 'navigate') {
return;
}
event.respondWith(
fetch(request).catch(() => buildOfflineShellResponse(url.pathname))
);
return;
}
event.respondWith( event.respondWith(
fetch(request) fetch(request)
.then((response) => { .then((response) => {
if (response && response.ok && response.type === 'basic') { const cacheControl = response.headers.get('Cache-Control') || '';
const shouldSkipCache =
cacheControl.includes('no-store') || cacheControl.includes('private');
if (response && response.ok && response.type === 'basic' && !shouldSkipCache) {
const clone = response.clone(); const clone = response.clone();
caches.open(CACHE_NAME).then((cache) => cache.put(request, clone)).catch(() => {}); caches.open(CACHE_NAME).then((cache) => cache.put(request, clone)).catch(() => {});
} }
@@ -62,6 +184,9 @@ self.addEventListener('fetch', (event) => {
.catch(async () => { .catch(async () => {
const cached = await caches.match(request); const cached = await caches.match(request);
if (cached) return cached; if (cached) return cached;
if (request.mode === 'navigate') {
return buildOfflineShellResponse(url.pathname);
}
return new Response('Offline — network unavailable.', { return new Response('Offline — network unavailable.', {
status: 503, status: 503,
statusText: 'Service Unavailable', statusText: 'Service Unavailable',
+61 -16
View File
@@ -5,44 +5,89 @@
* concurrently with unified terminal logging and clean shutdown handling. * concurrently with unified terminal logging and clean shutdown handling.
*/ */
const path = require('node:path');
const { spawn } = require('node:child_process'); const { spawn } = require('node:child_process');
const webPort = String(process.env.PORT || '3000');
const nextBin = require.resolve('next/dist/bin/next');
const serverEntry = path.resolve(__dirname, '..', 'server', 'index.js');
const GRACEFUL_SHUTDOWN_TIMEOUT_MS = 7000;
console.log('\x1b[36m%s\x1b[0m', '======================================================='); console.log('\x1b[36m%s\x1b[0m', '=======================================================');
console.log('\x1b[36m%s\x1b[0m', ' DataStack Unified Development Runner'); console.log('\x1b[36m%s\x1b[0m', ' DataStack Unified Development Runner');
console.log('\x1b[36m%s\x1b[0m', ' - Port 3000: Next.js 15 Web & Admin Console (/vcon/*)'); console.log('\x1b[36m%s\x1b[0m', ` - Port ${webPort}: Next.js 15 Web & Admin Console (/vcon/*)`);
console.log('\x1b[36m%s\x1b[0m', ' - Port 4000: Ingestion & Socket.IO Streaming Engine'); console.log('\x1b[36m%s\x1b[0m', ' - Port 4000: Ingestion & Socket.IO Streaming Engine');
console.log('\x1b[36m%s\x1b[0m', '=======================================================\n'); console.log('\x1b[36m%s\x1b[0m', '=======================================================\n');
const nextProc = spawn('npm', ['run', 'dev'], { const nextProc = spawn(process.execPath, [nextBin, 'dev', '-p', webPort], {
stdio: 'inherit', stdio: 'inherit',
shell: true, shell: false,
env: { ...process.env }, env: { ...process.env },
}); });
const serverProc = spawn('npm', ['run', 'server'], { const serverProc = spawn(process.execPath, [serverEntry], {
stdio: 'inherit', stdio: 'inherit',
shell: true, shell: false,
env: { ...process.env }, env: { ...process.env },
}); });
function shutdown(signal) { let shuttingDown = false;
console.log(`\n[DataStack] Received ${signal}. Gracefully stopping all dev services...`); let exitedChildren = 0;
if (!nextProc.killed) nextProc.kill(signal); let nextExited = false;
if (!serverProc.killed) serverProc.kill(signal); let serverExited = false;
process.exit(0); let exitCodePending = 0;
function markChildExited() {
exitedChildren += 1;
if (shuttingDown && exitedChildren >= 2) {
process.exit(exitCodePending);
}
} }
process.on('SIGINT', () => shutdown('SIGINT')); function signalChild(child, exited, signal) {
process.on('SIGTERM', () => shutdown('SIGTERM')); if (exited || child.killed) return;
try {
child.kill(signal);
} catch {
// Child already exited
}
}
function shutdown(signal, exitCode = 0) {
if (shuttingDown) return;
shuttingDown = true;
exitCodePending = exitCode;
console.log(`\n[DataStack] Received ${signal}. Gracefully stopping all dev services...`);
signalChild(nextProc, nextExited, signal);
signalChild(serverProc, serverExited, signal);
if (exitedChildren >= 2) {
process.exit(exitCodePending);
return;
}
setTimeout(() => {
signalChild(nextProc, nextExited, 'SIGKILL');
signalChild(serverProc, serverExited, 'SIGKILL');
process.exit(exitCodePending);
}, GRACEFUL_SHUTDOWN_TIMEOUT_MS).unref();
}
process.on('SIGINT', () => shutdown('SIGINT', 0));
process.on('SIGTERM', () => shutdown('SIGTERM', 0));
nextProc.on('exit', (code) => { nextProc.on('exit', (code) => {
if (code && code !== 0) { nextExited = true;
console.error(`[DataStack] Next.js development server exited with code ${code}`); markChildExited();
if (!shuttingDown) {
console.error(`[DataStack] Next.js development server exited with code ${code ?? 0}`);
shutdown('SIGTERM', code || 1);
} }
}); });
serverProc.on('exit', (code) => { serverProc.on('exit', (code) => {
if (code && code !== 0) { serverExited = true;
console.error(`[DataStack] Ingestion server exited with code ${code}`); markChildExited();
if (!shuttingDown) {
console.error(`[DataStack] Ingestion server exited with code ${code ?? 0}`);
shutdown('SIGTERM', code || 1);
} }
}); });
+90
View File
@@ -0,0 +1,90 @@
#!/usr/bin/env node
/**
* DataStack Unified Production Runner
* Starts both the Next.js Web Console (Port 3000 / $PORT) and
* the Express + Socket.IO Ingestion Server (Port 4000 / $PORT_SOCKET).
*/
const path = require('node:path');
const { spawn } = require('node:child_process');
const webPort = String(process.env.PORT || '3000');
const nextBin = require.resolve('next/dist/bin/next');
const serverEntry = path.resolve(__dirname, '..', 'server', 'index.js');
const GRACEFUL_SHUTDOWN_TIMEOUT_MS = 7000;
const webProc = spawn(
process.execPath,
[nextBin, 'start', '-p', webPort, '-H', '0.0.0.0'],
{
stdio: 'inherit',
shell: false,
env: { ...process.env },
},
);
const serverProc = spawn(process.execPath, [serverEntry], {
stdio: 'inherit',
shell: false,
env: { ...process.env },
});
let shuttingDown = false;
let exitedChildren = 0;
let webExited = false;
let serverExited = false;
let exitCodePending = 0;
function markChildExited() {
exitedChildren += 1;
if (shuttingDown && exitedChildren >= 2) {
process.exit(exitCodePending);
}
}
function signalChild(child, exited, signal) {
if (exited || child.killed) return;
try {
child.kill(signal);
} catch {
// Child already exited
}
}
function shutdown(signal, exitCode = 0) {
if (shuttingDown) return;
shuttingDown = true;
exitCodePending = exitCode;
signalChild(webProc, webExited, signal);
signalChild(serverProc, serverExited, signal);
if (exitedChildren >= 2) {
process.exit(exitCodePending);
return;
}
setTimeout(() => {
signalChild(webProc, webExited, 'SIGKILL');
signalChild(serverProc, serverExited, 'SIGKILL');
process.exit(exitCodePending);
}, GRACEFUL_SHUTDOWN_TIMEOUT_MS).unref();
}
process.on('SIGINT', () => shutdown('SIGINT', 0));
process.on('SIGTERM', () => shutdown('SIGTERM', 0));
webProc.on('exit', (code) => {
webExited = true;
markChildExited();
if (!shuttingDown) {
console.error(`[DataStack] Next.js production server exited with code ${code ?? 0}`);
shutdown('SIGTERM', code || 1);
}
});
serverProc.on('exit', (code) => {
serverExited = true;
markChildExited();
if (!shuttingDown) {
console.error(`[DataStack] Ingestion server exited with code ${code ?? 0}`);
shutdown('SIGTERM', code || 1);
}
});
+9
View File
@@ -20,6 +20,15 @@ def _ensure_canonical_datastack_on_path() -> None:
if repo_canonical.is_dir() and str(repo_canonical) not in sys.path: if repo_canonical.is_dir() and str(repo_canonical) not in sys.path:
sys.path.insert(0, str(repo_canonical)) sys.path.insert(0, str(repo_canonical))
try:
import datastack # noqa: F401
except ImportError as exc:
raise ImportError(
"datastack-sdk requires the canonical 'datastack>=2.2.0' package. "
"Install it via `pip install ./sdk/python/datastack` or install "
"`datastack>=2.2.0` from your package index."
) from exc
_ensure_canonical_datastack_on_path() _ensure_canonical_datastack_on_path()
+5 -3
View File
@@ -26,9 +26,10 @@ setup(
license="MIT", license="MIT",
packages=find_packages(), packages=find_packages(),
install_requires=[ install_requires=[
"python-socketio[client]>=5.11.0", "datastack>=2.2.0",
"requests>=2.31.0", "python-socketio[client]>=5.8.0",
"websocket-client>=1.7.0", "requests>=2.28.0",
"websocket-client>=1.5.0",
], ],
classifiers=[ classifiers=[
"Development Status :: 5 - Production/Stable", "Development Status :: 5 - Production/Stable",
@@ -44,6 +45,7 @@ setup(
"Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"License :: OSI Approved :: MIT License", "License :: OSI Approved :: MIT License",
"Operating System :: OS Independent", "Operating System :: OS Independent",
], ],
+122 -27
View File
@@ -13,12 +13,15 @@ Features:
and lossless partial-batch rate-limit retry (`429 RATE_LIMIT_EXCEEDED`). and lossless partial-batch rate-limit retry (`429 RATE_LIMIT_EXCEEDED`).
""" """
from __future__ import annotations
import base64 import base64
import csv import csv
import hashlib import hashlib
import json import json
import logging import logging
import random import random
import re
import threading import threading
import time import time
import urllib.error import urllib.error
@@ -26,7 +29,7 @@ import urllib.request
import uuid import uuid
from datetime import datetime, timezone from datetime import datetime, timezone
from pathlib import Path from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional, Union from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
try: try:
import socketio # type: ignore import socketio # type: ignore
@@ -74,6 +77,14 @@ def _make_deterministic_file_event_id(
return f"file_{safe_stem}_{row_index}_{digest}" return f"file_{safe_stem}_{row_index}_{digest}"
def _normalize_client_id(value: Optional[str]) -> str:
"""Matches server SDKManager.normalizeClientId() normalization rules."""
raw = str(value or "").strip().lower()
if not raw:
return ""
return re.sub(r"[^a-z0-9_-]", "_", raw)
def _decode_jwt_client_id(token: str) -> str: def _decode_jwt_client_id(token: str) -> str:
"""Safely extracts `clientId` or `sub` claim from an unverified JWT payload string.""" """Safely extracts `clientId` or `sub` claim from an unverified JWT payload string."""
try: try:
@@ -136,16 +147,18 @@ class DataStackClient:
configured_client_id = (self.config.client_id or "").strip() configured_client_id = (self.config.client_id or "").strip()
token_client_id = _decode_jwt_client_id(self.token) token_client_id = _decode_jwt_client_id(self.token)
norm_configured = _normalize_client_id(configured_client_id)
norm_token = _normalize_client_id(token_client_id)
if ( if (
configured_client_id norm_configured
and token_client_id and norm_token
and configured_client_id != token_client_id and norm_configured != norm_token
): ):
raise DataStackConfigError( raise DataStackConfigError(
f"Configured CLIENT_ID '{configured_client_id}' does not match " f"Configured CLIENT_ID '{configured_client_id}' does not match "
f"CLIENT_TOKEN subject '{token_client_id}'." f"CLIENT_TOKEN subject '{token_client_id}'."
) )
self.client_id: str = configured_client_id or token_client_id self.client_id: str = norm_token or norm_configured
self.is_connected = False self.is_connected = False
self._lock = threading.RLock() self._lock = threading.RLock()
@@ -186,6 +199,18 @@ class DataStackClient:
self.connect() self.connect()
except DataStackAuthError: except DataStackAuthError:
raise raise
except DataStackConnectionError:
if (
not self.config.enable_http_fallback
and not self.config.offline_spool_enabled
):
self.heartbeat.stop(
reason="Initial connect failed with fallback and spool disabled"
)
raise
logger.warning(
"[DataStack] Initial connect entered fallback/recovery mode.",
)
except Exception as exc: except Exception as exc:
logger.warning( logger.warning(
"[DataStack] Initial connect entered fallback/recovery mode: %s", "[DataStack] Initial connect entered fallback/recovery mode: %s",
@@ -202,6 +227,15 @@ class DataStackClient:
self.connect() self.connect()
except DataStackAuthError: except DataStackAuthError:
raise raise
except DataStackConnectionError:
if (
not self.config.enable_http_fallback
and not self.config.offline_spool_enabled
):
self.heartbeat.stop(
reason="Context connect failed with fallback and spool disabled"
)
raise
except Exception: except Exception:
pass pass
return self return self
@@ -295,13 +329,16 @@ class DataStackClient:
) )
return False return False
self.heartbeat.stop(
reason="WebSocket and HTTP unreachable with fallback and spool disabled"
)
raise DataStackConnectionError( raise DataStackConnectionError(
f"Unable to connect to DataStack WebSocket ({self.server_url}) or HTTP API ({self.api_url})." f"Unable to connect to DataStack WebSocket ({self.server_url}) or HTTP API ({self.api_url})."
) )
def disconnect(self) -> None: def disconnect(self) -> None:
"""Gracefully stops the Heartbeat Monitor and disconnects the WebSocket.""" """Gracefully stops the Heartbeat Monitor and disconnects the WebSocket."""
self.heartbeat.stop(reason="Client closed by application") self.heartbeat.stop(reason="Client closed by application", join_timeout=0.5)
with self._lock: with self._lock:
self.is_connected = False self.is_connected = False
if self.sio is not None and self.sio.connected: if self.sio is not None and self.sio.connected:
@@ -347,9 +384,24 @@ class DataStackClient:
return False return False
# Verify handshake & PostgreSQL client status via `ping_verify` while still serialized under RLock # Verify handshake & PostgreSQL client status via `ping_verify` while still serialized under RLock
try:
verified = self._verify_websocket_handshake(timeout=effective_timeout) verified = self._verify_websocket_handshake(timeout=effective_timeout)
except Exception:
self.is_connected = False
if self.sio.connected:
try:
self.sio.disconnect()
except Exception:
pass
raise
if not verified: if not verified:
self.is_connected = False self.is_connected = False
if self.sio.connected:
try:
self.sio.disconnect()
except Exception:
pass
return verified return verified
def _verify_websocket_handshake( def _verify_websocket_handshake(
@@ -455,7 +507,7 @@ class DataStackClient:
url: str, url: str,
payload: Optional[Dict[str, Any]] = None, payload: Optional[Dict[str, Any]] = None,
timeout: Optional[float] = None, timeout: Optional[float] = None,
) -> tuple[int, Dict[str, Any]]: ) -> Tuple[int, Dict[str, Any]]:
req_timeout = timeout or self.config.request_timeout_sec req_timeout = timeout or self.config.request_timeout_sec
headers = { headers = {
"Authorization": f"Bearer {self.token}", "Authorization": f"Bearer {self.token}",
@@ -869,7 +921,16 @@ class DataStackClient:
is_envelope = "data" in safe_item and set(safe_item.keys()).issubset( is_envelope = "data" in safe_item and set(safe_item.keys()).issubset(
envelope_keys envelope_keys
) )
evt_data = safe_item["data"] if is_envelope else safe_item if is_envelope:
evt_data = safe_item["data"]
elif "data" in safe_item:
evt_data = safe_item
else:
evt_data = {
k: v
for k, v in safe_item.items()
if k not in ("eventId", "timestamp", "clientId")
}
evt_client_id = ( evt_client_id = (
str(safe_item.get("clientId") or "").strip() or self.client_id str(safe_item.get("clientId") or "").strip() or self.client_id
) )
@@ -993,6 +1054,7 @@ class DataStackClient:
if file_name if file_name
else {"batch": remaining_items} else {"batch": remaining_items}
) )
try:
sub_res = self._execute_with_retry_and_fallback( sub_res = self._execute_with_retry_and_fallback(
ws_event="batch_sync", ws_event="batch_sync",
ws_payload=remaining_items, ws_payload=remaining_items,
@@ -1001,6 +1063,9 @@ class DataStackClient:
timeout=timeout, timeout=timeout,
allow_spool=allow_spool, allow_spool=allow_spool,
) )
except (DataStackRateLimitError, DataStackConnectionError):
break
total_processed += int(sub_res.get("processed", 0)) total_processed += int(sub_res.get("processed", 0))
total_duplicates += int(sub_res.get("duplicates", 0)) total_duplicates += int(sub_res.get("duplicates", 0))
total_spooled += int(sub_res.get("spooledCount", 0)) total_spooled += int(sub_res.get("spooledCount", 0))
@@ -1088,6 +1153,20 @@ class DataStackClient:
total_spooled = 0 total_spooled = 0
chunks_sent = 0 chunks_sent = 0
last_transport = "WEBSOCKET" last_transport = "WEBSOCKET"
unprocessed_items: List[Dict[str, Any]] = []
def _flush_chunk(chunk_items: List[Dict[str, Any]]) -> None:
nonlocal total_processed, total_rejected, total_duplicates, total_spooled, chunks_sent, last_transport
res = self.send_batch(chunk_items, file_name=path.name)
total_processed += int(res.get("processed", 0))
total_rejected += int(res.get("rejected", 0))
total_duplicates += int(res.get("duplicates", 0))
total_spooled += int(res.get("spooledCount", 0))
last_transport = str(res.get("transport", last_transport))
chunk_unprocessed = res.get("unprocessedItems")
if isinstance(chunk_unprocessed, list) and chunk_unprocessed:
unprocessed_items.extend(chunk_unprocessed)
chunks_sent += 1
buffer: List[Dict[str, Any]] = [] buffer: List[Dict[str, Any]] = []
for idx, record in enumerate(self._iter_file_records(path, ext), start=1): for idx, record in enumerate(self._iter_file_records(path, ext), start=1):
@@ -1111,35 +1190,44 @@ class DataStackClient:
) )
evt_ts = existing_ts or _utc_now_iso() evt_ts = existing_ts or _utc_now_iso()
if isinstance(enriched, dict):
is_file_envelope = "data" in enriched and set(
enriched.keys()
).issubset({"eventId", "timestamp", "data", "clientId"})
if is_file_envelope:
file_data: Any = enriched["data"]
elif "data" in enriched:
file_data = enriched
else:
file_data = {
k: v
for k, v in enriched.items()
if k not in ("eventId", "timestamp", "clientId")
}
else:
file_data = enriched
file_entry: Dict[str, Any] = { file_entry: Dict[str, Any] = {
"eventId": evt_id, "eventId": evt_id,
"timestamp": evt_ts, "timestamp": evt_ts,
"data": enriched, "data": file_data,
} }
if self.client_id: if self.client_id:
file_entry["clientId"] = self.client_id file_entry["clientId"] = self.client_id
buffer.append(file_entry) buffer.append(file_entry)
if len(buffer) >= effective_chunk: if len(buffer) >= effective_chunk:
res = self.send_batch(buffer, file_name=path.name) _flush_chunk(buffer)
total_processed += int(res.get("processed", 0))
total_rejected += int(res.get("rejected", 0))
total_duplicates += int(res.get("duplicates", 0))
total_spooled += int(res.get("spooledCount", 0))
last_transport = str(res.get("transport", last_transport))
chunks_sent += 1
buffer = [] buffer = []
if buffer: if buffer:
res = self.send_batch(buffer, file_name=path.name) _flush_chunk(buffer)
total_processed += int(res.get("processed", 0))
total_rejected += int(res.get("rejected", 0))
total_duplicates += int(res.get("duplicates", 0))
total_spooled += int(res.get("spooledCount", 0))
last_transport = str(res.get("transport", last_transport))
chunks_sent += 1
return { summary: Dict[str, Any] = {
"success": True, "success": (
total_rejected == 0
or total_processed > 0
or total_spooled > 0
),
"fileName": path.name, "fileName": path.name,
"fileType": ext.lstrip("."), "fileType": ext.lstrip("."),
"chunksSent": chunks_sent, "chunksSent": chunks_sent,
@@ -1149,15 +1237,22 @@ class DataStackClient:
"spooledCount": total_spooled, "spooledCount": total_spooled,
"transport": last_transport, "transport": last_transport,
} }
if unprocessed_items:
summary["unprocessedItems"] = unprocessed_items
return summary
@staticmethod @staticmethod
def _iter_file_records(path: Path, ext: str) -> Iterable[Dict[str, Any]]: def _iter_file_records(path: Path, ext: str) -> Iterable[Dict[str, Any]]:
try: try:
if ext == ".csv": if ext == ".csv":
with path.open("r", encoding="utf-8-sig", newline="") as fh: with path.open("r", encoding="utf-8-sig", newline="") as fh:
reader = csv.DictReader(fh) reader = csv.DictReader(fh, restkey="_extra")
for row in reader: for row in reader:
yield dict(row) clean_row = {
("_extra" if k is None else str(k)): v
for k, v in row.items()
}
yield clean_row
elif ext in (".jsonl", ".ndjson"): elif ext in (".jsonl", ".ndjson"):
with path.open("r", encoding="utf-8-sig") as fh: with path.open("r", encoding="utf-8-sig") as fh:
for line_no, line in enumerate(fh, start=1): for line_no, line in enumerate(fh, start=1):
+34 -12
View File
@@ -182,6 +182,22 @@ def load_config(
return default return default
return bool(val) return bool(val)
def to_float(val: Any, name: str) -> float:
try:
return float(val)
except (TypeError, ValueError) as exc:
raise DataStackConfigError(
f"Invalid numeric configuration for '{name}': {val!r}"
) from exc
def to_int(val: Any, name: str) -> int:
try:
return int(val)
except (TypeError, ValueError) as exc:
raise DataStackConfigError(
f"Invalid integer configuration for '{name}': {val!r}"
) from exc
server_url = str( server_url = str(
pick( pick(
["SERVER_URL", "SOCKET_URL", "WS_URL"], ["SERVER_URL", "SOCKET_URL", "WS_URL"],
@@ -233,52 +249,57 @@ def load_config(
), ),
heartbeat_interval_sec=max( heartbeat_interval_sec=max(
1.0, 1.0,
float( to_float(
pick( pick(
["HEARTBEAT_INTERVAL_SEC", "HEARTBEAT_INTERVAL"], ["HEARTBEAT_INTERVAL_SEC", "HEARTBEAT_INTERVAL"],
["DATASTACK_HEARTBEAT_INTERVAL_SEC"], ["DATASTACK_HEARTBEAT_INTERVAL_SEC"],
5.0, 5.0,
) ),
"HEARTBEAT_INTERVAL_SEC",
), ),
), ),
request_timeout_sec=max( request_timeout_sec=max(
1.0, 1.0,
float( to_float(
pick( pick(
["REQUEST_TIMEOUT_SEC", "TIMEOUT"], ["REQUEST_TIMEOUT_SEC", "TIMEOUT"],
["DATASTACK_REQUEST_TIMEOUT_SEC"], ["DATASTACK_REQUEST_TIMEOUT_SEC"],
8.0, 8.0,
) ),
"REQUEST_TIMEOUT_SEC",
), ),
), ),
max_retries=max( max_retries=max(
0, 0,
int( to_int(
pick( pick(
["MAX_RETRIES", "RETRIES"], ["MAX_RETRIES", "RETRIES"],
["DATASTACK_MAX_RETRIES"], ["DATASTACK_MAX_RETRIES"],
3, 3,
) ),
"MAX_RETRIES",
), ),
), ),
retry_backoff_base_sec=max( retry_backoff_base_sec=max(
0.1, 0.1,
float( to_float(
pick( pick(
["RETRY_BACKOFF_BASE_SEC"], ["RETRY_BACKOFF_BASE_SEC"],
["DATASTACK_RETRY_BACKOFF_BASE_SEC"], ["DATASTACK_RETRY_BACKOFF_BASE_SEC"],
0.5, 0.5,
) ),
"RETRY_BACKOFF_BASE_SEC",
), ),
), ),
batch_chunk_size=max( batch_chunk_size=max(
1, 1,
int( to_int(
pick( pick(
["BATCH_CHUNK_SIZE", "CHUNK_SIZE"], ["BATCH_CHUNK_SIZE", "CHUNK_SIZE"],
["DATASTACK_BATCH_CHUNK_SIZE"], ["DATASTACK_BATCH_CHUNK_SIZE"],
250, 250,
) ),
"BATCH_CHUNK_SIZE",
), ),
), ),
offline_spool_enabled=to_bool( offline_spool_enabled=to_bool(
@@ -298,12 +319,13 @@ def load_config(
), ),
max_spool_size_mb=max( max_spool_size_mb=max(
1, 1,
int( to_int(
pick( pick(
["MAX_SPOOL_SIZE_MB"], ["MAX_SPOOL_SIZE_MB"],
["DATASTACK_MAX_SPOOL_SIZE_MB"], ["DATASTACK_MAX_SPOOL_SIZE_MB"],
50, 50,
) ),
"MAX_SPOOL_SIZE_MB",
), ),
), ),
config_source_path=resolved_source, config_source_path=resolved_source,
+19 -2
View File
@@ -68,7 +68,8 @@ class HeartbeatMonitor:
Monitor into IDLE mode (`_wake_event.clear()`). Monitor into IDLE mode (`_wake_event.clear()`).
""" """
with self._lock: with self._lock:
self._closed = False if self._closed or self._stop_event.is_set():
return
self._state = ConnectionState.CONNECTED_IDLE self._state = ConnectionState.CONNECTED_IDLE
self._recovery_attempts = 0 self._recovery_attempts = 0
self._last_error = None self._last_error = None
@@ -112,7 +113,8 @@ class HeartbeatMonitor:
self._stop_event.clear() self._stop_event.clear()
self._ensure_thread_started() self._ensure_thread_started()
def stop(self, reason: Optional[str] = None) -> None: def stop(self, reason: Optional[str] = None, join_timeout: float = 0.0) -> None:
thread_to_join: Optional[threading.Thread] = None
with self._lock: with self._lock:
self._closed = True self._closed = True
self._state = ConnectionState.STOPPED self._state = ConnectionState.STOPPED
@@ -120,6 +122,14 @@ class HeartbeatMonitor:
self._last_error = reason self._last_error = reason
self._stop_event.set() self._stop_event.set()
self._wake_event.set() self._wake_event.set()
thread_to_join = self._thread
if (
join_timeout > 0
and thread_to_join is not None
and thread_to_join.is_alive()
and threading.current_thread() is not thread_to_join
):
thread_to_join.join(timeout=join_timeout)
def status_snapshot(self) -> Dict[str, Any]: def status_snapshot(self) -> Dict[str, Any]:
with self._lock: with self._lock:
@@ -181,6 +191,13 @@ class HeartbeatMonitor:
try: try:
recovered_to_idle = self._recovery_callback() recovered_to_idle = self._recovery_callback()
if recovered_to_idle: if recovered_to_idle:
with self._lock:
still_active = (
not self._closed
and not self._stop_event.is_set()
and self._state != ConnectionState.STOPPED
)
if still_active:
self.set_idle_connected() self.set_idle_connected()
logger.info( logger.info(
"[DataStack Heartbeat] WebSocket reconnected & verified. Returning to IDLE mode." "[DataStack Heartbeat] WebSocket reconnected & verified. Returning to IDLE mode."
Whitespace-only changes.
+15 -4
View File
@@ -6,6 +6,7 @@ batches once the Heartbeat Monitor restores connectivity.
""" """
import json import json
import logging
import os import os
import threading import threading
from pathlib import Path from pathlib import Path
@@ -13,6 +14,8 @@ from typing import Any, Dict, List, Optional
from .exceptions import DataStackOfflineSpoolError from .exceptions import DataStackOfflineSpoolError
logger = logging.getLogger("datastack.spool")
class DiskSpooler: class DiskSpooler:
""" """
@@ -121,6 +124,7 @@ class DiskSpooler:
if not self.spool_file.exists(): if not self.spool_file.exists():
return 0 return 0
tmp_file = self.spool_dir / "spool.jsonl.tmp"
try: try:
with self.spool_file.open("r", encoding="utf-8") as fh: with self.spool_file.open("r", encoding="utf-8") as fh:
all_lines = [ln for ln in fh if ln.strip()] all_lines = [ln for ln in fh if ln.strip()]
@@ -130,13 +134,15 @@ class DiskSpooler:
return len(all_lines) return len(all_lines)
remaining = all_lines[count:] remaining = all_lines[count:]
tmp_file = self.spool_dir / "spool.jsonl.tmp"
with tmp_file.open("w", encoding="utf-8") as fh: with tmp_file.open("w", encoding="utf-8") as fh:
fh.writelines(remaining) fh.writelines(remaining)
fh.flush() fh.flush()
os.fsync(fh.fileno())
os.replace(str(tmp_file), str(self.spool_file)) os.replace(str(tmp_file), str(self.spool_file))
return count return count
except Exception: except Exception as exc:
tmp_file.unlink(missing_ok=True)
logger.warning("[DataStack Spool] ack_batch failed: %s", exc)
return 0 return 0
def ack_and_requeue_front( def ack_and_requeue_front(
@@ -156,6 +162,7 @@ class DiskSpooler:
if not self.spool_file.exists(): if not self.spool_file.exists():
return 0 return 0
tmp_file = self.spool_dir / "spool.jsonl.tmp"
try: try:
with self.spool_file.open("r", encoding="utf-8") as fh: with self.spool_file.open("r", encoding="utf-8") as fh:
all_lines = [ln for ln in fh if ln.strip()] all_lines = [ln for ln in fh if ln.strip()]
@@ -173,13 +180,17 @@ class DiskSpooler:
return consumed return consumed
self._ensure_dir() self._ensure_dir()
tmp_file = self.spool_dir / "spool.jsonl.tmp"
with tmp_file.open("w", encoding="utf-8") as fh: with tmp_file.open("w", encoding="utf-8") as fh:
fh.writelines(combined) fh.writelines(combined)
fh.flush() fh.flush()
os.fsync(fh.fileno())
os.replace(str(tmp_file), str(self.spool_file)) os.replace(str(tmp_file), str(self.spool_file))
return max(0, consumed - len(head_lines)) return max(0, consumed - len(head_lines))
except Exception: except Exception as exc:
tmp_file.unlink(missing_ok=True)
logger.warning(
"[DataStack Spool] ack_and_requeue_front failed: %s", exc
)
return 0 return 0
def pending_count(self) -> int: def pending_count(self) -> int:
+4
View File
@@ -33,6 +33,7 @@ classifiers = [
"Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Software Development :: Libraries :: Python Modules",
"Topic :: System :: Logging", "Topic :: System :: Logging",
"Topic :: System :: Monitoring" "Topic :: System :: Monitoring"
@@ -52,3 +53,6 @@ Issues = "https://github.com/anik-d-art/datastack/issues"
[tool.setuptools.packages.find] [tool.setuptools.packages.find]
where = ["."] where = ["."]
include = ["datastack*"] include = ["datastack*"]
[tool.setuptools.package-data]
datastack = ["py.typed"]
+3
View File
@@ -24,6 +24,8 @@ setup(
url="https://vibtools.top/", url="https://vibtools.top/",
license="MIT", license="MIT",
packages=find_packages(include=["datastack", "datastack.*"]), packages=find_packages(include=["datastack", "datastack.*"]),
package_data={"datastack": ["py.typed"]},
include_package_data=True,
install_requires=[ install_requires=[
"python-socketio[client]>=5.8.0", "python-socketio[client]>=5.8.0",
"requests>=2.28.0", "requests>=2.28.0",
@@ -40,5 +42,6 @@ setup(
"Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
], ],
) )
+706 -109
View File
File diff suppressed because it is too large. Load diff
+271 -26
View File
@@ -1,8 +1,30 @@
const crypto = require("node:crypto");
const jwt = require("jsonwebtoken"); const jwt = require("jsonwebtoken");
const IDEMPOTENCY_TTL_MS = 5 * 60 * 1000; // 5 minutes const IDEMPOTENCY_TTL_MS = 5 * 60 * 1000; // 5 minutes
const IDEMPOTENCY_MAX_ENTRIES = 20000; const IDEMPOTENCY_MAX_ENTRIES = 20000;
const CLEANUP_INTERVAL_MS = 60 * 1000; const CLEANUP_INTERVAL_MS = 60 * 1000;
const REDIS_IDEMPOTENCY_PREFIX = "datastack:idem:";
const REDIS_RATE_LIMIT_PREFIX = "datastack:rl:";
const REDIS_RATE_LIMIT_LUA_SCRIPT = [
"-- DATASTACK_RATE_LIMIT_INCR",
'local c = redis.call("INCR", KEYS[1])',
'if c == 1 or redis.call("PTTL", KEYS[1]) < 0 then',
' redis.call("PEXPIRE", KEYS[1], tonumber(ARGV[1]))',
"end",
"return c",
].join("\n");
function generateFallbackEventId() {
return `evt_${Date.now()}_${crypto.randomBytes(6).toString("hex")}`;
}
function normalizeClientId(clientId) {
return String(clientId || "")
.trim()
.toLowerCase()
.replace(/[^a-z0-9_-]/g, "_");
}
function setupSocketHandlers(io, bufferQueue, worker, options = {}) { function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
const getRuntimeConfig = const getRuntimeConfig =
@@ -36,6 +58,11 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
? options.onBufferPush ? options.onBufferPush
: () => undefined; : () => undefined;
const getRedisClient =
typeof options.getRedisClient === "function"
? options.getRedisClient
: () => null;
const idempotencyMap = new Map(); const idempotencyMap = new Map();
const rateLimitMap = new Map(); const rateLimitMap = new Map();
let lastCleanupAt = Date.now(); let lastCleanupAt = Date.now();
@@ -63,6 +90,25 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
} }
} }
function trackLocalRateWindow(clientId, now) {
const rl = rateLimitMap.get(clientId);
if (!rl || now - rl.start >= 1000) {
rateLimitMap.set(clientId, { count: 1, start: now });
} else {
rl.count += 1;
}
}
function supportsSharedSecurityRedis(redis) {
return Boolean(
redis &&
typeof redis.get === "function" &&
typeof redis.incr === "function" &&
typeof redis.pexpire === "function" &&
typeof redis.set === "function",
);
}
function getSecurityStats() { function getSecurityStats() {
cleanupMaps(Date.now(), true); cleanupMaps(Date.now(), true);
const config = getRuntimeConfig(); const config = getRuntimeConfig();
@@ -73,7 +119,13 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
}; };
} }
function validateSingleEvent(clientId, eventId, rawData, clientRateLimit) { function validateSingleEventLocal(
clientId,
eventId,
rawData,
clientRateLimit,
explicitByteLength,
) {
const config = getRuntimeConfig(); const config = getRuntimeConfig();
const now = Date.now(); const now = Date.now();
cleanupMaps(now); cleanupMaps(now);
@@ -93,9 +145,19 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
} }
// 2. Payload byte size check (before consuming rate-limit quota) // 2. Payload byte size check (before consuming rate-limit quota)
const payloadStr = let payloadStr = "";
let byteLength = 0;
if (
typeof explicitByteLength === "number" &&
Number.isFinite(explicitByteLength) &&
explicitByteLength >= 0
) {
byteLength = explicitByteLength;
} else {
payloadStr =
typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {}); typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {});
const byteLength = Buffer.byteLength(payloadStr, "utf8"); byteLength = Buffer.byteLength(payloadStr, "utf8");
}
const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024; const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024;
if (byteLength > maxBytes) { if (byteLength > maxBytes) {
return { return {
@@ -133,6 +195,126 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
return { allowed: true, payloadStr }; return { allowed: true, payloadStr };
} }
async function validateSingleEventAsync(
clientId,
eventId,
rawData,
clientRateLimit,
) {
const redis = getRedisClient();
if (!supportsSharedSecurityRedis(redis)) {
return validateSingleEventLocal(
clientId,
eventId,
rawData,
clientRateLimit,
);
}
const config = getRuntimeConfig();
const now = Date.now();
cleanupMaps(now);
const idempotencyKey = `${clientId}:${eventId}`;
const redisIdemKey = `${REDIS_IDEMPOTENCY_PREFIX}${clientId}:${eventId}`;
try {
// 1. Read-only idempotency check
if (config.enableIdempotency && eventId) {
const localTime = idempotencyMap.get(idempotencyKey);
if (localTime && now - localTime < IDEMPOTENCY_TTL_MS) {
return {
allowed: false,
duplicate: true,
code: "DUPLICATE_EVENT",
reason: "Duplicate event dropped",
};
}
const existing = await redis.get(redisIdemKey);
if (existing !== null) {
idempotencyMap.set(idempotencyKey, now);
return {
allowed: false,
duplicate: true,
code: "DUPLICATE_EVENT",
reason: "Duplicate event dropped",
};
}
}
// 2. Payload byte size check
const payloadStr =
typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {});
const byteLength = Buffer.byteLength(payloadStr, "utf8");
const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024;
if (byteLength > maxBytes) {
return {
allowed: false,
duplicate: false,
code: "PAYLOAD_TOO_LARGE",
reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`,
};
}
// 3. Shared atomic Redis rate limit check (single Lua round-trip)
const effectiveLimit = Math.min(
Number(clientRateLimit) || 100,
Number(config.maxEventsPerSec) || 100,
);
const windowBucket = Math.floor(now / 1000);
const rlKey = `${REDIS_RATE_LIMIT_PREFIX}${clientId}:${windowBucket}`;
let count;
if (typeof redis.eval === "function") {
count = Number(
await redis.eval(REDIS_RATE_LIMIT_LUA_SCRIPT, 1, rlKey, "2000"),
);
} else {
count = Number(await redis.incr(rlKey));
if (count === 1) {
await redis.pexpire(rlKey, 2000);
}
}
trackLocalRateWindow(clientId, now);
if (count > effectiveLimit) {
return {
allowed: false,
duplicate: false,
code: "RATE_LIMIT_EXCEEDED",
reason: `RATE_LIMIT_EXCEEDED: Limit of ${effectiveLimit} events/sec exceeded`,
};
}
// 4. Atomic claim of idempotency key after all checks succeed
if (config.enableIdempotency && eventId) {
const claimed = await redis.set(
redisIdemKey,
"1",
"PX",
IDEMPOTENCY_TTL_MS,
"NX",
);
idempotencyMap.set(idempotencyKey, now);
if (claimed !== "OK") {
return {
allowed: false,
duplicate: true,
code: "DUPLICATE_EVENT",
reason: "Duplicate event dropped",
};
}
}
return { allowed: true, payloadStr };
} catch {
return validateSingleEventLocal(
clientId,
eventId,
rawData,
clientRateLimit,
);
}
}
// Middleware: Handshake Auth // Middleware: Handshake Auth
io.use(async (socket, next) => { io.use(async (socket, next) => {
const authHeader = const authHeader =
@@ -157,9 +339,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
} }
const decoded = jwt.verify(cleanToken, config.jwtSecret); const decoded = jwt.verify(cleanToken, config.jwtSecret);
const normalizedClientId = String(decoded.clientId || "") const normalizedClientId = normalizeClientId(decoded.clientId);
.trim()
.toLowerCase();
const dbCheck = await verifyClientInDb( const dbCheck = await verifyClientInDb(
normalizedClientId, normalizedClientId,
cleanToken, cleanToken,
@@ -181,18 +361,37 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
const ENVELOPE_KEYS = new Set(["eventId", "timestamp", "data", "clientId"]); const ENVELOPE_KEYS = new Set(["eventId", "timestamp", "data", "clientId"]);
function extractBatchItemData(safeItem) { function extractBatchItemData(safeItem) {
return extractSingleEventData(safeItem);
}
function extractSingleEventData(rawPayload) {
if ( if (
safeItem && rawPayload === null ||
typeof safeItem === "object" && rawPayload === undefined ||
!Array.isArray(safeItem) && typeof rawPayload !== "object" ||
safeItem.data !== undefined Array.isArray(rawPayload)
) { ) {
const keys = Object.keys(safeItem); return rawPayload === undefined ? {} : rawPayload;
}
if (rawPayload.data !== undefined) {
const keys = Object.keys(rawPayload);
if (keys.every((k) => ENVELOPE_KEYS.has(k))) { if (keys.every((k) => ENVELOPE_KEYS.has(k))) {
return safeItem.data; return rawPayload.data;
} }
return rawPayload;
} }
return safeItem;
const stripped = {};
let hasNonEnvelopeKey = false;
for (const [k, v] of Object.entries(rawPayload)) {
if (k === "eventId" || k === "timestamp" || k === "clientId") {
continue;
}
hasNonEnvelopeKey = true;
stripped[k] = v;
}
return hasNonEnvelopeKey ? stripped : {};
} }
io.on("connection", (socket) => { io.on("connection", (socket) => {
@@ -221,16 +420,25 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
const effectiveClientRate = const effectiveClientRate =
dbCheck.rateLimit || socket.data.rateLimit || 100; dbCheck.rateLimit || socket.data.rateLimit || 100;
const safePayload = payload && typeof payload === "object" ? payload : {}; const safePayload =
payload && typeof payload === "object" && !Array.isArray(payload)
? payload
: {};
const eventId = const eventId =
safePayload.eventId || typeof safePayload.eventId === "string" && safePayload.eventId.trim()
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`; ? safePayload.eventId.trim()
const timestamp = safePayload.timestamp || new Date().toISOString(); : generateFallbackEventId();
const timestamp =
typeof safePayload.timestamp === "string" &&
safePayload.timestamp.trim()
? safePayload.timestamp.trim()
: new Date().toISOString();
const eventData = extractSingleEventData(payload);
const check = validateSingleEvent( const check = await validateSingleEventAsync(
clientId, clientId,
eventId, eventId,
safePayload.data, eventData,
effectiveClientRate, effectiveClientRate,
); );
if (!check.allowed) { if (!check.allowed) {
@@ -260,7 +468,22 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
bufferQueue.push(event); bufferQueue.push(event);
recordClientActivity(clientId, 1); recordClientActivity(clientId, 1);
onBufferPush(); try {
await onBufferPush();
} catch (persistErr) {
if (callback) {
callback({
success: false,
eventId,
code: "BUFFER_PERSIST_FAILED",
error:
persistErr instanceof Error
? persistErr.message
: "Failed to persist event to buffer",
});
}
return;
}
if (callback) callback({ success: true, eventId }); if (callback) callback({ success: true, eventId });
} catch (err) { } catch (err) {
@@ -309,12 +532,16 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
for (const item of batch) { for (const item of batch) {
const safeItem = item && typeof item === "object" ? item : {}; const safeItem = item && typeof item === "object" ? item : {};
const eventId = const eventId =
safeItem.eventId || typeof safeItem.eventId === "string" && safeItem.eventId.trim()
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`; ? safeItem.eventId.trim()
const timestamp = safeItem.timestamp || new Date().toISOString(); : generateFallbackEventId();
const timestamp =
typeof safeItem.timestamp === "string" && safeItem.timestamp.trim()
? safeItem.timestamp.trim()
: new Date().toISOString();
const rawData = extractBatchItemData(safeItem); const rawData = extractBatchItemData(safeItem);
const check = validateSingleEvent( const check = await validateSingleEventAsync(
clientId, clientId,
eventId, eventId,
rawData, rawData,
@@ -351,7 +578,24 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
if (processed > 0) { if (processed > 0) {
recordClientActivity(clientId, processed); recordClientActivity(clientId, processed);
onBufferPush(); try {
await onBufferPush();
} catch (persistErr) {
if (callback) {
callback({
success: false,
processed: 0,
rejected: batch.length,
duplicates,
code: "BUFFER_PERSIST_FAILED",
error:
persistErr instanceof Error
? persistErr.message
: "Failed to persist batch to buffer",
});
}
return;
}
} }
if (processed === 0 && duplicates === 0 && rejected > 0) { if (processed === 0 && duplicates === 0 && rejected > 0) {
@@ -443,7 +687,8 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
return { return {
getSecurityStats, getSecurityStats,
validateSingleEventLocal,
}; };
} }
module.exports = { setupSocketHandlers }; module.exports = { setupSocketHandlers, normalizeClientId };
+262 -48
View File
@@ -1,3 +1,4 @@
const crypto = require('node:crypto');
const fs = require('fs'); const fs = require('fs');
const path = require('path'); const path = require('path');
const { Parser } = require('json2csv'); const { Parser } = require('json2csv');
@@ -11,13 +12,21 @@ function sleepMs(ms) {
return new Promise((resolve) => setTimeout(resolve, ms)); return new Promise((resolve) => setTimeout(resolve, ms));
} }
function resolveDefaultStorageDir() {
const configured = (process.env.DATASTACK_STORAGE_DIR || '').trim();
if (configured) {
return path.resolve(configured);
}
return path.resolve(__dirname, '../../storage/data');
}
/** /**
* Background CSV Writer Worker * Background CSV Writer Worker
* Flushes batches of real-time events to partitioned CSV files. * Flushes batches of real-time events to partitioned CSV files.
* Path convention: /storage/data/{clientId}/{YYYY-MM-DD}.csv * Path convention: /storage/data/{clientId}/{YYYY-MM-DD}.csv
*/ */
class CsvWriterWorker { class CsvWriterWorker {
constructor(baseDir = path.resolve(__dirname, '../../storage/data'), maxFileSize = 100 * 1024 * 1024) { constructor(baseDir = resolveDefaultStorageDir(), maxFileSize = 100 * 1024 * 1024) {
this.baseDir = baseDir; this.baseDir = baseDir;
this.maxFileSize = maxFileSize; this.maxFileSize = maxFileSize;
this.ensureBaseDir(); this.ensureBaseDir();
@@ -36,9 +45,85 @@ class CsvWriterWorker {
} }
} }
sanitizeClientId(clientId) {
const raw = String(clientId || 'unknown').trim().replace(/[^a-zA-Z0-9_-]/g, '_');
return raw || 'unknown';
}
fileEndsWithNewline(filePath, sizeBytes) {
if (sizeBytes <= 0) return false;
const fd = fs.openSync(filePath, 'r');
try {
const lastByte = Buffer.allocUnsafe(1);
const read = fs.readSync(fd, lastByte, 0, 1, sizeBytes - 1);
return read === 1 && lastByte[0] === 0x0a;
} finally {
fs.closeSync(fd);
}
}
removeStaleLockAtomically(lockPath) {
try {
const lockStat = fs.lstatSync(lockPath);
if (Date.now() - lockStat.mtimeMs <= STALE_LOCK_AGE_MS) {
return;
}
const staleTmp = `${lockPath}.stale.${crypto.randomUUID()}`;
fs.renameSync(lockPath, staleTmp);
try {
fs.unlinkSync(staleTmp);
} catch {
// Ignore cleanup failure
}
} catch {
// Lock was released or claimed concurrently
}
}
releaseLockIfOwned(lockPath, lockValue) {
try {
if (!fs.existsSync(lockPath)) return;
const current = fs.readFileSync(lockPath, 'utf8');
if (current === lockValue) {
fs.unlinkSync(lockPath);
}
} catch {
// Ignore if already removed
}
}
async removeStaleLockAtomicallyAsync(lockPath) {
try {
const lockStat = await fs.promises.lstat(lockPath);
if (Date.now() - lockStat.mtimeMs <= STALE_LOCK_AGE_MS) {
return;
}
const staleTmp = `${lockPath}.stale.${crypto.randomUUID()}`;
await fs.promises.rename(lockPath, staleTmp);
await fs.promises.unlink(staleTmp).catch(() => undefined);
} catch {
// Lock was released or claimed concurrently
}
}
async releaseLockIfOwnedAsync(lockPath, lockValue) {
try {
const current = await fs.promises.readFile(lockPath, 'utf8');
if (current === lockValue) {
await fs.promises.unlink(lockPath);
}
} catch {
// Ignore if already removed
}
}
ensureClientDir(clientId) { ensureClientDir(clientId) {
const safeClientId = String(clientId || 'unknown').replace(/[^a-zA-Z0-9_-]/g, '_'); const safeClientId = this.sanitizeClientId(clientId);
const clientDir = path.join(this.baseDir, safeClientId); const resolvedBase = path.resolve(this.baseDir);
const clientDir = path.resolve(resolvedBase, safeClientId);
if (!clientDir.startsWith(resolvedBase + path.sep)) {
throw new Error(`Invalid storage client directory: ${clientId}`);
}
if (!fs.existsSync(clientDir)) { if (!fs.existsSync(clientDir)) {
fs.mkdirSync(clientDir, { recursive: true }); fs.mkdirSync(clientDir, { recursive: true });
} }
@@ -49,6 +134,7 @@ class CsvWriterWorker {
const clientDir = this.ensureClientDir(clientId); const clientDir = this.ensureClientDir(clientId);
const date = this.extractSafePartitionDate(dateStr); const date = this.extractSafePartitionDate(dateStr);
const lockPath = path.join(clientDir, `.${date}.csv.lock`); const lockPath = path.join(clientDir, `.${date}.csv.lock`);
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
const deadline = Date.now() + LOCK_TIMEOUT_MS; const deadline = Date.now() + LOCK_TIMEOUT_MS;
let acquired = false; let acquired = false;
@@ -56,7 +142,7 @@ class CsvWriterWorker {
try { try {
const fd = fs.openSync(lockPath, 'wx'); const fd = fs.openSync(lockPath, 'wx');
try { try {
fs.writeFileSync(fd, `${process.pid}:${Date.now()}`, 'utf8'); fs.writeFileSync(fd, lockValue, 'utf8');
} finally { } finally {
fs.closeSync(fd); fs.closeSync(fd);
} }
@@ -66,15 +152,7 @@ class CsvWriterWorker {
if (!err || err.code !== 'EEXIST') { if (!err || err.code !== 'EEXIST') {
throw err; throw err;
} }
try { this.removeStaleLockAtomically(lockPath);
const lockStat = fs.statSync(lockPath);
if (Date.now() - lockStat.mtimeMs > STALE_LOCK_AGE_MS) {
fs.unlinkSync(lockPath);
continue;
}
} catch {
// Released concurrently
}
const jitter = const jitter =
LOCK_RETRY_MIN_MS + LOCK_RETRY_MIN_MS +
Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS)); Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS));
@@ -89,13 +167,7 @@ class CsvWriterWorker {
try { try {
return await fn(); return await fn();
} finally { } finally {
try { this.releaseLockIfOwned(lockPath, lockValue);
if (fs.existsSync(lockPath)) {
fs.unlinkSync(lockPath);
}
} catch {
// Ignore if already removed
}
} }
} }
@@ -127,7 +199,7 @@ class CsvWriterWorker {
return fallback; return fallback;
} }
getTargetFilePath(clientId, dateStr) { getTargetFilePath(clientId, dateStr, incomingBytes = 0) {
const clientDir = this.ensureClientDir(clientId); const clientDir = this.ensureClientDir(clientId);
const date = this.extractSafePartitionDate(dateStr); const date = this.extractSafePartitionDate(dateStr);
let baseFileName = `${date}.csv`; let baseFileName = `${date}.csv`;
@@ -135,18 +207,26 @@ class CsvWriterWorker {
let partIndex = 1; let partIndex = 1;
while (fs.existsSync(targetPath)) { while (fs.existsSync(targetPath)) {
let stats;
try { try {
const stats = fs.statSync(targetPath); stats = fs.statSync(targetPath);
if (stats.size >= this.maxFileSize) { } catch (err) {
if (err && err.code === 'ENOENT') {
break;
}
throw err;
}
const overflowsLimit =
stats.size > 0 &&
incomingBytes > 0 &&
stats.size + incomingBytes > this.maxFileSize;
if (stats.size >= this.maxFileSize || overflowsLimit) {
partIndex += 1; partIndex += 1;
baseFileName = `${date}-part${partIndex}.csv`; baseFileName = `${date}-part${partIndex}.csv`;
targetPath = path.join(clientDir, baseFileName); targetPath = path.join(clientDir, baseFileName);
} else { } else {
break; break;
} }
} catch {
break;
}
} }
return targetPath; return targetPath;
@@ -158,26 +238,20 @@ class CsvWriterWorker {
const grouped = new Map(); const grouped = new Map();
for (const evt of events) { for (const evt of events) {
const date = this.extractSafePartitionDate(evt.timestamp); const date = this.extractSafePartitionDate(evt.timestamp);
const key = `${evt.clientId}:::${date}`; const clientId = String(evt.clientId || 'unknown');
if (!grouped.has(key)) grouped.set(key, []); const key = JSON.stringify([clientId, date]);
grouped.get(key).push(evt); let bucket = grouped.get(key);
if (!bucket) {
bucket = { clientId, date, events: [] };
grouped.set(key, bucket);
}
bucket.events.push(evt);
} }
const files = []; const files = [];
const fields = ['clientId', 'eventId', 'timestamp', 'payload']; const fields = ['clientId', 'eventId', 'timestamp', 'payload'];
for (const [key, batch] of grouped.entries()) { for (const { clientId, date, events: batch } of grouped.values()) {
const [clientId, date] = key.split(':::');
const targetPath = await this.withPartitionLock(clientId, date, () => {
const resolvedPath = this.getTargetFilePath(clientId, date);
let hasContent = false;
try {
hasContent =
fs.existsSync(resolvedPath) && fs.statSync(resolvedPath).size > 0;
} catch {
hasContent = false;
}
const cleanBatch = batch.map((item) => ({ const cleanBatch = batch.map((item) => ({
clientId: item.clientId, clientId: item.clientId,
eventId: item.eventId, eventId: item.eventId,
@@ -185,13 +259,27 @@ class CsvWriterWorker {
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload), payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
})); }));
const parser = new Parser({ fields, header: !hasContent, eol: '\n' }); const bodyCsvNoHeader = new Parser({ fields, header: false, eol: '\n' }).parse(cleanBatch);
const csv = parser.parse(cleanBatch); const incomingBytes = Buffer.byteLength(bodyCsvNoHeader, 'utf8') + 1;
const targetPath = await this.withPartitionLock(clientId, date, () => {
const resolvedPath = this.getTargetFilePath(clientId, date, incomingBytes);
let sizeBytes = 0;
try {
if (fs.existsSync(resolvedPath)) {
sizeBytes = fs.statSync(resolvedPath).size;
}
} catch {
sizeBytes = 0;
}
const hasContent = sizeBytes > 0;
if (hasContent) { if (hasContent) {
fs.appendFileSync(resolvedPath, '\n' + csv, 'utf8'); const endsWithNl = this.fileEndsWithNewline(resolvedPath, sizeBytes);
fs.appendFileSync(resolvedPath, (endsWithNl ? '' : '\n') + bodyCsvNoHeader, 'utf8');
} else { } else {
fs.writeFileSync(resolvedPath, csv, 'utf8'); const fullCsv = new Parser({ fields, header: true, eol: '\n' }).parse(cleanBatch);
fs.writeFileSync(resolvedPath, fullCsv, 'utf8');
} }
return resolvedPath; return resolvedPath;
@@ -218,18 +306,144 @@ class CsvWriterWorker {
const clientDirs = fs.readdirSync(this.baseDir); const clientDirs = fs.readdirSync(this.baseDir);
for (const clientDir of clientDirs) { for (const clientDir of clientDirs) {
const fullClientPath = path.join(this.baseDir, clientDir); const fullClientPath = path.join(this.baseDir, clientDir);
if (!fs.statSync(fullClientPath).isDirectory()) continue; let dirLstat;
try {
dirLstat = fs.lstatSync(fullClientPath);
} catch {
continue;
}
if (dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
const files = fs.readdirSync(fullClientPath); const files = fs.readdirSync(fullClientPath);
for (const file of files) { for (const file of files) {
if (!file.endsWith('.csv')) continue; if (!file.endsWith('.csv')) continue;
const datePart = file.replace('.csv', '').split('-part')[0]; const datePart = file.replace('.csv', '').split('-part')[0];
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) { if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
const fullPath = path.join(fullClientPath, file);
let fileLstat;
try { try {
fs.unlinkSync(path.join(fullClientPath, file)); fileLstat = fs.lstatSync(fullPath);
} catch {
continue;
}
if (fileLstat.isSymbolicLink() || !fileLstat.isFile()) continue;
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
let acquired = false;
try {
const fd = fs.openSync(lockPath, 'wx');
try {
fs.writeFileSync(fd, lockValue, 'utf8');
} finally {
fs.closeSync(fd);
}
acquired = true;
} catch (err) {
if (err && err.code === 'EEXIST') {
this.removeStaleLockAtomically(lockPath);
try {
const fd = fs.openSync(lockPath, 'wx');
try {
fs.writeFileSync(fd, lockValue, 'utf8');
} finally {
fs.closeSync(fd);
}
acquired = true;
} catch {
acquired = false;
}
}
}
if (!acquired) continue;
try {
if (fs.existsSync(fullPath)) {
fs.unlinkSync(fullPath);
prunedCount += 1;
}
} catch {
// Ignore individual file unlink failure
} finally {
this.releaseLockIfOwned(lockPath, lockValue);
}
}
}
}
} catch {
// Ignore directory scan failure
}
return prunedCount;
}
async pruneExpiredPartitionsAsync(autoPruneDays) {
const days = Number(autoPruneDays);
if (!Number.isFinite(days) || days <= 0) {
return 0;
}
const baseStat = await fs.promises.lstat(this.baseDir).catch(() => null);
if (!baseStat || !baseStat.isDirectory()) {
return 0;
}
const cutoffDate = new Date(Date.now() - Math.floor(days) * 86400000)
.toISOString()
.split('T')[0];
let prunedCount = 0;
try {
const clientDirs = await fs.promises.readdir(this.baseDir);
for (const clientDir of clientDirs) {
const fullClientPath = path.join(this.baseDir, clientDir);
const dirLstat = await fs.promises.lstat(fullClientPath).catch(() => null);
if (!dirLstat || dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
const files = await fs.promises.readdir(fullClientPath).catch(() => []);
for (const file of files) {
if (!file.endsWith('.csv')) continue;
const datePart = file.replace('.csv', '').split('-part')[0];
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
const fullPath = path.join(fullClientPath, file);
const fileLstat = await fs.promises.lstat(fullPath).catch(() => null);
if (!fileLstat || fileLstat.isSymbolicLink() || !fileLstat.isFile()) continue;
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
let acquired = false;
const tryAcquireLock = async () => {
const handle = await fs.promises.open(lockPath, 'wx');
try {
await handle.writeFile(lockValue, 'utf8');
} finally {
await handle.close();
}
return true;
};
try {
acquired = await tryAcquireLock();
} catch (err) {
if (err && err.code === 'EEXIST') {
await this.removeStaleLockAtomicallyAsync(lockPath);
try {
acquired = await tryAcquireLock();
} catch {
acquired = false;
}
}
}
if (!acquired) continue;
try {
await fs.promises.unlink(fullPath);
prunedCount += 1; prunedCount += 1;
} catch { } catch {
// Ignore individual file unlink failure // Ignore individual file unlink failure
} finally {
await this.releaseLockIfOwnedAsync(lockPath, lockValue);
} }
} }
} }
@@ -241,4 +455,4 @@ class CsvWriterWorker {
} }
} }
module.exports = { CsvWriterWorker }; module.exports = { CsvWriterWorker, resolveDefaultStorageDir };
+695 -14
View File
@@ -1,29 +1,45 @@
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { after, afterEach, beforeEach, describe, it } from "node:test"; import { after, afterEach, beforeEach, describe, it } from "node:test";
import { NextRequest } from "next/server"; import { NextRequest } from "next/server";
import { POST as loginPost } from "@/app/api/auth/login/route";
import { POST as logoutPost } from "@/app/api/auth/logout/route";
import { GET as profileGet, PUT as profilePut } from "@/app/api/auth/profile/route"; import { GET as profileGet, PUT as profilePut } from "@/app/api/auth/profile/route";
import { POST as setupPost } from "@/app/api/auth/setup/route";
import { GET as clientsGet, POST as clientsPost } from "@/app/api/clients/route"; import { GET as clientsGet, POST as clientsPost } from "@/app/api/clients/route";
import { import {
DELETE as clientDelete, DELETE as clientDelete,
GET as clientGet, GET as clientGet,
PATCH as clientPatch,
} from "@/app/api/clients/[id]/route"; } from "@/app/api/clients/[id]/route";
import { POST as eventsPost } from "@/app/api/events/route"; import { GET as dataGet } from "@/app/api/data/route";
import { GET as eventsGet, POST as eventsPost } from "@/app/api/events/route";
import { GET as exportGet } from "@/app/api/export/route";
import { GET as healthGet } from "@/app/api/health/route";
import { GET as metricsGet } from "@/app/api/metrics/route";
import { GET as storageGet, POST as storagePost } from "@/app/api/storage/route";
import { SettingsManager } from "@/core/adminSettings/settingsManager";
import { import {
createAdminSessionToken, createAdminSessionToken,
getAdminSessionPayload, getAdminSessionPayload,
getAdminSessionSecret, getAdminSessionSecret,
getSessionCookieOptions,
} from "@/core/auth/adminSession"; } from "@/core/auth/adminSession";
import { import {
AdminDatabaseRequiredError,
consumeAdminLoginAttempt, consumeAdminLoginAttempt,
createInitialAdmin, createInitialAdmin,
getAdminSetupKey, getAdminSetupKey,
isAdminSetupComplete, isAdminSetupComplete,
requiresPersistentAdminDatabase,
resetAdminStateForTests, resetAdminStateForTests,
verifyAdminCredentials, verifyAdminCredentials,
verifyAdminSetupKey, verifyAdminSetupKey,
verifyRequestAdminSession, verifyRequestAdminSession,
} from "@/core/auth/adminUsers"; } from "@/core/auth/adminUsers";
import { AuthService, resolveDefaultJwtSecret } from "@/core/auth/jwt"; import { AuthService, resolveDefaultJwtSecret } from "@/core/auth/jwt";
import { DatabaseEngine } from "@/core/Database/database";
import { SDKManager } from "@/core/sdk/sdkManager";
import { extractClientIp } from "@/lib/adminAuth";
import { isSameOriginRequest, middleware } from "@/middleware"; import { isSameOriginRequest, middleware } from "@/middleware";
const ORIGINAL_ENV = { ...process.env }; const ORIGINAL_ENV = { ...process.env };
@@ -41,7 +57,14 @@ function restoreEnv() {
describe("Phase-01 Auth, Secrets, Revocation & Proxy Hardening", () => { describe("Phase-01 Auth, Secrets, Revocation & Proxy Hardening", () => {
beforeEach(() => { beforeEach(() => {
restoreEnv(); restoreEnv();
process.env.INGESTION_INTERNAL_URL = "http://127.0.0.1:1";
resetAdminStateForTests(); resetAdminStateForTests();
const g = globalThis as {
__dataStackClientsMap?: Map<string, unknown>;
__dataStackDeletedClients?: Set<string>;
};
g.__dataStackClientsMap?.clear();
g.__dataStackDeletedClients?.clear();
}); });
afterEach(() => { afterEach(() => {
@@ -338,22 +361,18 @@ describe("Phase-01 Auth, Secrets, Revocation & Proxy Hardening", () => {
assert.equal(isSameOriginRequest(reqMissingBoth), false); assert.equal(isSameOriginRequest(reqMissingBoth), false);
}); });
it("SPEC-08: allows headless scripts with x-datastack-admin-client or DataStack-CLI to bypass browser CSRF check", async () => { it("BUG-A02: rejects spoofable x-datastack-admin-client, DataStack-CLI, and cross-site Sec-Fetch-Site headers without same-origin proof", async () => {
process.env.DATASTACK_ADMIN_SESSION_SECRET = "test-secret-at-least-32-chars-long-for-session"; process.env.DATASTACK_ADMIN_SESSION_SECRET = "test-secret-at-least-32-chars-long-for-session";
// 1. Without headless header or origin -> 403 CSRF rejected // 1. Missing origin/referer -> 403
const blockedReq = new NextRequest("http://localhost:3000/api/clients", { const blockedReq = new NextRequest("http://localhost:3000/api/clients", {
method: "POST", method: "POST",
headers: { headers: { host: "localhost:3000" },
host: "localhost:3000",
},
}); });
const blockedRes = await middleware(blockedReq); const blockedRes = await middleware(blockedReq);
assert.equal(blockedRes.status, 403); assert.equal(blockedRes.status, 403);
const blockedJson = await blockedRes.json();
assert.match(blockedJson.error, /cross-origin/i);
// 2. With x-datastack-admin-client header -> bypasses CSRF (reaches auth check, returns 401 Unauthorized if no cookie) // 2. Spoofed x-datastack-admin-client header must still be rejected 403
const headlessReq = new NextRequest("http://localhost:3000/api/clients", { const headlessReq = new NextRequest("http://localhost:3000/api/clients", {
method: "POST", method: "POST",
headers: { headers: {
@@ -362,19 +381,681 @@ describe("Phase-01 Auth, Secrets, Revocation & Proxy Hardening", () => {
}, },
}); });
const headlessRes = await middleware(headlessReq); const headlessRes = await middleware(headlessReq);
assert.equal(headlessRes.status, 401); assert.equal(headlessRes.status, 403);
const headlessJson = await headlessRes.json();
assert.match(headlessJson.error, /authentication required/i);
// 3. With DataStack-CLI User-Agent -> bypasses CSRF // 3. Spoofed DataStack-CLI User-Agent + cross-origin must still be rejected 403
const cliReq = new NextRequest("http://localhost:3000/api/clients", { const cliReq = new NextRequest("http://localhost:3000/api/clients", {
method: "POST", method: "POST",
headers: { headers: {
host: "localhost:3000", host: "localhost:3000",
origin: "https://evil.example.com",
"user-agent": "DataStack-CLI/1.0", "user-agent": "DataStack-CLI/1.0",
}, },
}); });
const cliRes = await middleware(cliReq); const cliRes = await middleware(cliReq);
assert.equal(cliRes.status, 401); assert.equal(cliRes.status, 403);
// 4. Genuine same-origin passes CSRF and reaches auth check (401 without cookie)
const sameOriginReq = new NextRequest("http://localhost:3000/api/clients", {
method: "POST",
headers: {
host: "localhost:3000",
origin: "http://localhost:3000",
},
});
const sameOriginRes = await middleware(sameOriginReq);
assert.equal(sameOriginRes.status, 401);
});
it("BUG-A01: enforces tokenVersion revocation across /api/data, GET /api/events, /api/export, /api/metrics, and /api/storage", async () => {
delete process.env.DATABASE_URL;
process.env.DATASTACK_ALLOW_DEMO_ADMIN = "false";
await createInitialAdmin("owner@datastack.dev", "InitialPass123!");
const secret = getAdminSessionSecret()!;
const v1Token = await createAdminSessionToken(
"owner@datastack.dev",
secret,
Date.now() + 3600_000,
1,
);
const updateRes = await profilePut(
new NextRequest("http://localhost:3000/api/auth/profile", {
method: "PUT",
headers: {
"content-type": "application/json",
cookie: `datastack_admin_session=${v1Token}`,
},
body: JSON.stringify({
email: "owner@datastack.dev",
currentPassword: "InitialPass123!",
newPassword: "RotatedPass456!",
}),
}),
);
assert.equal(updateRes.status, 200);
const v2Cookie = updateRes.cookies.get("datastack_admin_session")?.value;
assert.ok(v2Cookie);
const endpoints: Array<() => Promise<Response>> = [
() => dataGet(new NextRequest("http://localhost:3000/api/data", { headers: { cookie: `datastack_admin_session=${v1Token}` } })),
() => eventsGet(new NextRequest("http://localhost:3000/api/events", { headers: { cookie: `datastack_admin_session=${v1Token}` } })),
() => exportGet(new NextRequest("http://localhost:3000/api/export?flush=false", { headers: { cookie: `datastack_admin_session=${v1Token}` } })),
() => metricsGet(new NextRequest("http://localhost:3000/api/metrics", { headers: { cookie: `datastack_admin_session=${v1Token}` } })),
() => storageGet(new NextRequest("http://localhost:3000/api/storage", { headers: { cookie: `datastack_admin_session=${v1Token}` } })),
() => storagePost(new NextRequest("http://localhost:3000/api/storage", { method: "POST", headers: { cookie: `datastack_admin_session=${v1Token}` } })),
];
for (const invoke of endpoints) {
const res = await invoke();
assert.equal(res.status, 401);
}
const okMetrics = await metricsGet(
new NextRequest("http://localhost:3000/api/metrics", {
headers: { cookie: `datastack_admin_session=${v2Cookie}` },
}),
);
assert.equal(okMetrics.status, 200);
});
it("BUG-A04 & BUG-A05: rejects unregistered clients on /api/health & admin POST /api/events and validates PATCH /api/clients/[id]", async () => {
delete process.env.DATABASE_URL;
await createInitialAdmin("admin@datastack.dev", "AdminPass123!");
const secret = getAdminSessionSecret()!;
const adminCookie = await createAdminSessionToken(
"admin@datastack.dev",
secret,
Date.now() + 3600_000,
1,
);
// Public unauthenticated /api/health probe stays 200
const publicHealth = await healthGet(new NextRequest("http://localhost:3000/api/health"));
assert.equal(publicHealth.status, 200);
// Unregistered signed client Bearer token on /api/health -> 403 CLIENT_REVOKED
const ghostToken = AuthService.generateClientToken({ clientId: "ghost_health_client" });
const ghostHealth = await healthGet(
new NextRequest("http://localhost:3000/api/health", {
headers: { authorization: `Bearer ${ghostToken}` },
}),
);
assert.equal(ghostHealth.status, 403);
// Admin cookie POST /api/events for unregistered clientId -> 403 CLIENT_REVOKED
const ghostAdminPost = await eventsPost(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
"content-type": "application/json",
cookie: `datastack_admin_session=${adminCookie}`,
},
body: JSON.stringify({ clientId: "unregistered_admin_target", data: { ok: 1 } }),
}),
);
assert.equal(ghostAdminPost.status, 403);
// Create normalized client and verify mixed-case/unvalidated PATCH checks
const createdRes = await clientsPost(
new NextRequest("http://localhost:3000/api/clients", {
method: "POST",
headers: {
"content-type": "application/json",
cookie: `datastack_admin_session=${adminCookie}`,
},
body: JSON.stringify({ clientId: "Norm_Client_01", name: "Normalized", rateLimit: 50 }),
}),
);
assert.equal(createdRes.status, 201);
// Invalid non-string name -> 400 (not 500)
const badNameRes = await clientPatch(
new NextRequest("http://localhost:3000/api/clients/Norm_Client_01", {
method: "PATCH",
headers: {
"content-type": "application/json",
cookie: `datastack_admin_session=${adminCookie}`,
},
body: JSON.stringify({ name: 12345 }),
}),
{ params: Promise.resolve({ id: "Norm_Client_01" }) },
);
assert.equal(badNameRes.status, 400);
// Invalid status -> 400
const badStatusRes = await clientPatch(
new NextRequest("http://localhost:3000/api/clients/Norm_Client_01", {
method: "PATCH",
headers: {
"content-type": "application/json",
cookie: `datastack_admin_session=${adminCookie}`,
},
body: JSON.stringify({ status: "suspended" }),
}),
{ params: Promise.resolve({ id: "Norm_Client_01" }) },
);
assert.equal(badStatusRes.status, 400);
// Valid mixed-case normalized PATCH -> 200
const goodPatchRes = await clientPatch(
new NextRequest("http://localhost:3000/api/clients/NORM_CLIENT_01", {
method: "PATCH",
headers: {
"content-type": "application/json",
cookie: `datastack_admin_session=${adminCookie}`,
},
body: JSON.stringify({ name: "Updated Clean Name", rateLimit: 250 }),
}),
{ params: Promise.resolve({ id: "NORM_CLIENT_01" }) },
);
assert.equal(goodPatchRes.status, 200);
const patchedJson = await goodPatchRes.json();
assert.equal(patchedJson.client.clientId, "norm_client_01");
assert.equal(patchedJson.client.rateLimit, 250);
});
it("FORENSIC BUG-01: blocks volatile RAM admin setup in production without DATABASE_URL and returns 409 SETUP_REQUIRED on pre-setup login", async () => {
process.env.NODE_ENV = "production";
delete process.env.DATABASE_URL;
delete process.env.DATASTACK_ADMIN_PASSWORD;
process.env.DATASTACK_ADMIN_SESSION_SECRET =
"prod-session-secret-at-least-32-characters-long";
process.env.DATASTACK_ADMIN_SETUP_KEY =
"prod-setup-key-at-least-32-characters-long";
assert.equal(requiresPersistentAdminDatabase(), true);
await assert.rejects(
() => createInitialAdmin("owner@datastack.dev", "StrongPass123!"),
AdminDatabaseRequiredError,
);
const setupRes = await setupPost(
new NextRequest("http://localhost:3000/api/auth/setup", {
method: "POST",
headers: {
"content-type": "application/json",
host: "localhost:3000",
origin: "http://localhost:3000",
},
body: JSON.stringify({
email: "owner@datastack.dev",
password: "StrongPass123!",
setupKey: process.env.DATASTACK_ADMIN_SETUP_KEY,
}),
}),
);
assert.equal(setupRes.status, 503);
const setupJson = await setupRes.json();
assert.equal(setupJson.code, "DATABASE_NOT_CONFIGURED");
const loginRes = await loginPost(
new NextRequest("http://localhost:3000/api/auth/login", {
method: "POST",
headers: {
"content-type": "application/json",
host: "localhost:3000",
origin: "http://localhost:3000",
},
body: JSON.stringify({
email: "owner@datastack.dev",
password: "StrongPass123!",
}),
}),
);
assert.equal(loginRes.status, 409);
const loginJson = await loginRes.json();
assert.equal(loginJson.code, "SETUP_REQUIRED");
});
it("FORENSIC BUG-04: throttles POST /api/auth/setup brute-force attempts and checks setup completion before verifying setupKey", async () => {
delete process.env.DATABASE_URL;
process.env.DATASTACK_ALLOW_DEMO_ADMIN = "false";
process.env.DATASTACK_ADMIN_SETUP_KEY =
"custom-setup-key-at-least-32-characters-long";
for (let i = 0; i < 5; i++) {
const badRes = await setupPost(
new NextRequest("http://localhost:3000/api/auth/setup", {
method: "POST",
headers: {
"content-type": "application/json",
"x-forwarded-for": "198.51.100.77",
},
body: JSON.stringify({
email: "admin@datastack.dev",
password: "StrongPass123!",
setupKey: "wrong-key-attempt",
}),
}),
);
assert.equal(badRes.status, 403);
}
const throttledRes = await setupPost(
new NextRequest("http://localhost:3000/api/auth/setup", {
method: "POST",
headers: {
"content-type": "application/json",
"x-forwarded-for": "198.51.100.77",
},
body: JSON.stringify({
email: "admin@datastack.dev",
password: "StrongPass123!",
setupKey: process.env.DATASTACK_ADMIN_SETUP_KEY,
}),
}),
);
assert.equal(throttledRes.status, 429);
assert.ok(Number(throttledRes.headers.get("retry-after")) >= 1);
// Different IP succeeds and initializes admin
const okRes = await setupPost(
new NextRequest("http://localhost:3000/api/auth/setup", {
method: "POST",
headers: {
"content-type": "application/json",
"x-forwarded-for": "203.0.113.88",
},
body: JSON.stringify({
email: "admin@datastack.dev",
password: "StrongPass123!",
setupKey: process.env.DATASTACK_ADMIN_SETUP_KEY,
}),
}),
);
assert.equal(okRes.status, 200);
// Subsequent setup attempt returns 409 even if setupKey is invalid
const postInitRes = await setupPost(
new NextRequest("http://localhost:3000/api/auth/setup", {
method: "POST",
headers: {
"content-type": "application/json",
"x-forwarded-for": "203.0.113.89",
},
body: JSON.stringify({
email: "other@datastack.dev",
password: "StrongPass123!",
setupKey: "completely-invalid-key",
}),
}),
);
assert.equal(postInitRes.status, 409);
});
it("FORENSIC BUG-05: revokes session cookie server-side on logout, ignores spoofed Origin for Secure flag, and allows privacy-stripped same-origin requests", async () => {
delete process.env.DATABASE_URL;
process.env.DATASTACK_ALLOW_DEMO_ADMIN = "false";
await createInitialAdmin("logout-test@datastack.dev", "LogoutPass123!");
const secret = getAdminSessionSecret()!;
const sessionToken = await createAdminSessionToken(
"logout-test@datastack.dev",
secret,
Date.now() + 3600_000,
1,
);
// Valid before logout
const preLogoutRes = await profileGet(
new NextRequest("http://localhost:3000/api/auth/profile", {
headers: { cookie: `datastack_admin_session=${sessionToken}` },
}),
);
assert.equal(preLogoutRes.status, 200);
// Logout bumps tokenVersion server-side
const logoutRes = await logoutPost(
new NextRequest("http://localhost:3000/api/auth/logout", {
method: "POST",
headers: { cookie: `datastack_admin_session=${sessionToken}` },
}),
);
assert.equal(logoutRes.status, 200);
// Replaying captured pre-logout cookie must return 401
const replayProfile = await profileGet(
new NextRequest("http://localhost:3000/api/auth/profile", {
headers: { cookie: `datastack_admin_session=${sessionToken}` },
}),
);
assert.equal(replayProfile.status, 401);
// Cookie Secure flag must ignore spoofed https Origin on plain http request
const spoofedOriginReq = new Request("http://localhost:3000/api/auth/login", {
method: "POST",
headers: { origin: "https://evil.example.com" },
});
const plainOpts = getSessionCookieOptions(spoofedOriginReq, 3600);
assert.equal(plainOpts.secure, false);
assert.equal(plainOpts.sameSite, "lax");
const httpsProxyReq = new Request("http://localhost:3000/api/auth/login", {
method: "POST",
headers: { "x-forwarded-proto": "https" },
});
const httpsOpts = getSessionCookieOptions(httpsProxyReq, 3600);
assert.equal(httpsOpts.secure, true);
assert.equal(httpsOpts.sameSite, "none");
// Privacy extension stripping Origin & Referer with Sec-Fetch-Site: same-origin passes CSRF
const privacySameOriginReq = new NextRequest("http://localhost:3000/api/clients", {
method: "POST",
headers: {
host: "localhost:3000",
"sec-fetch-site": "same-origin",
},
});
assert.equal(isSameOriginRequest(privacySameOriginReq), true);
const crossSiteReq = new NextRequest("http://localhost:3000/api/clients", {
method: "POST",
headers: {
host: "localhost:3000",
"sec-fetch-site": "cross-site",
},
});
assert.equal(isSameOriginRequest(crossSiteReq), false);
});
it("FORENSIC BUG-06: prevents empty production jwtSecret PG persistence and propagates DB write errors instead of silently swallowing them", async () => {
const origGetPool = DatabaseEngine.getPool;
const origEnsureSchema = DatabaseEngine.ensureSchema;
const settingsGlobals = globalThis as {
__dataStackAdminSettings?: unknown;
__dataStackSettingsHydrated?: boolean;
};
const queries: string[] = [];
try {
// 1. Production with empty jwtSecret must not persist empty jwt_secret row into PG
process.env.NODE_ENV = "production";
delete process.env.DATASTACK_JWT_SECRET;
settingsGlobals.__dataStackAdminSettings = undefined;
settingsGlobals.__dataStackSettingsHydrated = false;
DatabaseEngine.getPool = (() =>
({
query: async (sql: string) => {
queries.push(sql);
return { rowCount: 0, rows: [] };
},
}) as unknown as ReturnType<typeof DatabaseEngine.getPool>);
DatabaseEngine.ensureSchema = async () => true;
await SettingsManager.getSettingsAsync();
assert.equal(
queries.some((sql) => sql.includes("INSERT INTO admin_settings")),
false,
);
// 2. Failing PG write must reject (not silently return 200) and must not mutate RAM
process.env.NODE_ENV = "test";
process.env.DATASTACK_JWT_SECRET =
"valid-jwt-secret-at-least-32-characters-long";
AuthService.setSecret(process.env.DATASTACK_JWT_SECRET);
settingsGlobals.__dataStackAdminSettings = undefined;
settingsGlobals.__dataStackSettingsHydrated = false;
DatabaseEngine.getPool = (() =>
({
query: async (sql: string) => {
if (sql.includes("INSERT INTO") || sql.includes("UPDATE")) {
throw new Error("simulated PG outage");
}
return { rowCount: 0, rows: [] };
},
}) as unknown as ReturnType<typeof DatabaseEngine.getPool>);
await assert.rejects(
() => SettingsManager.updateSettingsAsync({ maxEventsPerSec: 777 }),
/DATABASE_WRITE_FAILED/,
);
await assert.rejects(
() =>
SDKManager.registerClientAsync({
clientId: "db_fail_client",
name: "DB Fail Client",
}),
/DATABASE_WRITE_FAILED/,
);
assert.equal(SDKManager.getClient("db_fail_client"), undefined);
} finally {
DatabaseEngine.getPool = origGetPool;
DatabaseEngine.ensureSchema = origEnsureSchema;
settingsGlobals.__dataStackAdminSettings = undefined;
settingsGlobals.__dataStackSettingsHydrated = false;
}
});
it("R2 PHASE-01 BUG-01: enforces same-origin CSRF check on POST /api/auth/login and POST /api/auth/setup while preserving same-origin and Bearer /api/events", async () => {
process.env.DATASTACK_ADMIN_SESSION_SECRET =
"test-secret-at-least-32-chars-long-for-session";
// Cross-origin POST /api/auth/login -> 403
const crossLoginRes = await middleware(
new NextRequest("http://localhost:3000/api/auth/login", {
method: "POST",
headers: {
host: "localhost:3000",
origin: "https://evil.example.com",
},
}),
);
assert.equal(crossLoginRes.status, 403);
assert.deepEqual(await crossLoginRes.json(), {
error: "Cross-origin request rejected.",
});
// Missing origin/referer/sec-fetch-site POST /api/auth/login -> 403
const bareLoginRes = await middleware(
new NextRequest("http://localhost:3000/api/auth/login", {
method: "POST",
headers: { host: "localhost:3000" },
}),
);
assert.equal(bareLoginRes.status, 403);
// Cross-origin POST /api/auth/setup -> 403
const crossSetupRes = await middleware(
new NextRequest("http://localhost:3000/api/auth/setup", {
method: "POST",
headers: {
host: "localhost:3000",
origin: "https://evil.example.com",
},
}),
);
assert.equal(crossSetupRes.status, 403);
// Missing origin POST /api/auth/setup -> 403
const bareSetupRes = await middleware(
new NextRequest("http://localhost:3000/api/auth/setup", {
method: "POST",
headers: { host: "localhost:3000" },
}),
);
assert.equal(bareSetupRes.status, 403);
// Same-origin POST /api/auth/login passes middleware (200 next)
const sameOriginLoginRes = await middleware(
new NextRequest("http://localhost:3000/api/auth/login", {
method: "POST",
headers: {
host: "localhost:3000",
origin: "http://localhost:3000",
},
}),
);
assert.equal(sameOriginLoginRes.status, 200);
// Privacy-stripped same-origin POST /api/auth/login via Sec-Fetch-Site passes middleware
const privacyLoginRes = await middleware(
new NextRequest("http://localhost:3000/api/auth/login", {
method: "POST",
headers: {
host: "localhost:3000",
"sec-fetch-site": "same-origin",
},
}),
);
assert.equal(privacyLoginRes.status, 200);
// Same-origin POST /api/auth/setup passes middleware (200 next)
const sameOriginSetupRes = await middleware(
new NextRequest("http://localhost:3000/api/auth/setup", {
method: "POST",
headers: {
host: "localhost:3000",
origin: "http://localhost:3000",
},
}),
);
assert.equal(sameOriginSetupRes.status, 200);
// Bearer-authenticated POST /api/events without Origin stays allowed (machine-to-machine SDK)
const bearerEventsRes = await middleware(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
host: "localhost:3000",
authorization: "Bearer sdk.jwt.token",
},
}),
);
assert.equal(bearerEventsRes.status, 200);
// Non-Bearer cross-origin POST /api/events is rejected 403
const crossEventsRes = await middleware(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
host: "localhost:3000",
origin: "https://evil.example.com",
},
}),
);
assert.equal(crossEventsRes.status, 403);
// Safe GET public API endpoints remain accessible without Origin
const healthRes = await middleware(
new NextRequest("http://localhost:3000/api/health", {
method: "GET",
headers: { host: "localhost:3000" },
}),
);
assert.equal(healthRes.status, 200);
});
it("R2 PHASE-01 BUG-02: extractClientIp trusts X-Real-IP and rightmost X-Forwarded-For hop, blocking spoofed-XFF lockouts on login & setup", async () => {
// Direct unit assertions on extractClientIp
assert.equal(
extractClientIp(
new NextRequest("http://localhost:3000/api/auth/login", {
headers: {
"x-forwarded-for": "1.2.3.4, 203.0.113.9",
"x-real-ip": "203.0.113.9",
},
}),
),
"203.0.113.9",
);
assert.equal(
extractClientIp(
new NextRequest("http://localhost:3000/api/auth/login", {
headers: {
"x-forwarded-for": "1.2.3.4, 198.51.100.50, ",
"x-real-ip": " ",
},
}),
),
"198.51.100.50",
);
assert.equal(
extractClientIp(new NextRequest("http://localhost:3000/api/auth/login")),
"unknown",
);
// End-to-end brute-force lockout on POST /api/auth/login despite rotating spoofed leftmost XFF
delete process.env.DATABASE_URL;
process.env.DATASTACK_ALLOW_DEMO_ADMIN = "false";
process.env.DATASTACK_ADMIN_SETUP_KEY =
"phase01-setup-key-at-least-32-characters-long";
// First verify setup route blocks rotating spoofed leftmost XFF
for (let i = 0; i < 5; i++) {
const badSetup = await setupPost(
new NextRequest("http://localhost:3000/api/auth/setup", {
method: "POST",
headers: {
"content-type": "application/json",
"x-forwarded-for": `10.1.0.${i + 1}, 198.51.100.42`,
},
body: JSON.stringify({
email: "admin@datastack.dev",
password: "StrongPass123!",
setupKey: "wrong-key",
}),
}),
);
assert.equal(badSetup.status, 403);
}
const blockedSetup = await setupPost(
new NextRequest("http://localhost:3000/api/auth/setup", {
method: "POST",
headers: {
"content-type": "application/json",
"x-forwarded-for": "10.1.0.99, 198.51.100.42",
},
body: JSON.stringify({
email: "admin@datastack.dev",
password: "StrongPass123!",
setupKey: process.env.DATASTACK_ADMIN_SETUP_KEY,
}),
}),
);
assert.equal(blockedSetup.status, 429);
// Create admin and verify login route blocks rotating spoofed leftmost XFF with X-Real-IP
await createInitialAdmin("lockout@datastack.dev", "RealPassword123!");
for (let i = 0; i < 5; i++) {
const badLogin = await loginPost(
new NextRequest("http://localhost:3000/api/auth/login", {
method: "POST",
headers: {
"content-type": "application/json",
"x-forwarded-for": `172.16.0.${i + 1}, 203.0.113.9`,
"x-real-ip": "203.0.113.9",
},
body: JSON.stringify({
email: "lockout@datastack.dev",
password: "WrongPassword!",
}),
}),
);
assert.equal(badLogin.status, 401);
}
const blockedLogin = await loginPost(
new NextRequest("http://localhost:3000/api/auth/login", {
method: "POST",
headers: {
"content-type": "application/json",
"x-forwarded-for": "172.16.0.99, 203.0.113.9",
"x-real-ip": "203.0.113.9",
},
body: JSON.stringify({
email: "lockout@datastack.dev",
password: "RealPassword123!",
}),
}),
);
assert.equal(blockedLogin.status, 429);
assert.ok(Number(blockedLogin.headers.get("retry-after")) >= 1);
}); });
}); });
+847
View File
@@ -1,20 +1,32 @@
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { createRequire } from "node:module"; import { createRequire } from "node:module";
import type { AddressInfo } from "node:net"; import type { AddressInfo } from "node:net";
import { resolve } from "node:path";
import test, { after, before } from "node:test"; import test, { after, before } from "node:test";
import { NextRequest } from "next/server"; import { NextRequest } from "next/server";
import { io as ioClient, type Socket } from "socket.io-client"; import { io as ioClient, type Socket } from "socket.io-client";
import {
DELETE as deleteClientRoute,
GET as getClientRoute,
PATCH as patchClientRoute,
} from "../app/api/clients/[id]/route";
import { POST as postEventsRoute } from "../app/api/events/route";
import { GET as getSettingsRoute, POST as postSettingsRoute } from "../app/api/settings/route"; import { GET as getSettingsRoute, POST as postSettingsRoute } from "../app/api/settings/route";
import { SettingsManager } from "../core/adminSettings/settingsManager"; import { SettingsManager } from "../core/adminSettings/settingsManager";
import { AuthService } from "../core/auth/jwt";
import { import {
createAdminSessionToken, createAdminSessionToken,
getAdminSessionSecret, getAdminSessionSecret,
} from "../core/auth/adminSession"; } from "../core/auth/adminSession";
import { import {
createInitialAdmin, createInitialAdmin,
isAdminDatabaseConfigured,
resetAdminStateForTests, resetAdminStateForTests,
} from "../core/auth/adminUsers"; } from "../core/auth/adminUsers";
import { ApiEngine } from "../core/apiEngine/apiService"; import { ApiEngine } from "../core/apiEngine/apiService";
import { DataBufferEngine, type RedisBufferClient } from "../core/data/buffer";
import { DatabaseEngine } from "../core/Database/database";
import { SDKManager } from "../core/sdk/sdkManager"; import { SDKManager } from "../core/sdk/sdkManager";
import { import {
SecurityEngine, SecurityEngine,
@@ -37,6 +49,15 @@ interface IngestionServerModule {
enableIdempotency: boolean; enableIdempotency: boolean;
autoPruneDays: number; autoPruneDays: number;
}; };
verifyClientInDb: (
clientId: string,
token: string,
bypass?: boolean,
) => Promise<{ allowed: boolean; reason?: string; rateLimit?: number }>;
recordClientActivity: (clientId: string, count?: number) => void;
flushPendingClientActivity: () => Promise<number>;
setRedisClientForTests: (client: unknown) => void;
setDbPoolForTests: (pool: unknown) => void;
stopTimers: () => void; stopTimers: () => void;
} }
@@ -48,6 +69,9 @@ const g = globalThis as typeof globalThis & {
__dataStackClientsMap?: Map<string, unknown>; __dataStackClientsMap?: Map<string, unknown>;
__dataStackDeletedClients?: Set<string>; __dataStackDeletedClients?: Set<string>;
__dataStackSyncedSocketEvents?: Map<string, number>; __dataStackSyncedSocketEvents?: Map<string, number>;
__dataStackPendingClientActivity?: Map<string, unknown>;
__dataStackCorePool?: unknown;
__dataStackCoreSchema?: Promise<void>;
}; };
let serverMod: IngestionServerModule; let serverMod: IngestionServerModule;
@@ -151,6 +175,7 @@ after(async () => {
clearInterval(g.__dataStackBufferState.flushTimer); clearInterval(g.__dataStackBufferState.flushTimer);
g.__dataStackBufferState.flushTimer = null; g.__dataStackBufferState.flushTimer = null;
} }
SDKManager.stopActivityFlushTimerForTests();
if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL; if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL;
else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl; else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl;
@@ -175,6 +200,7 @@ test("BUG-13: Port 4000 /internal/* rejects unauthenticated callers and accepts
["GET", "/internal/recent", undefined], ["GET", "/internal/recent", undefined],
["POST", "/internal/client-sync", { clientId: "x", action: "registered" }], ["POST", "/internal/client-sync", { clientId: "x", action: "registered" }],
["POST", "/internal/settings-sync", { maxEventsPerSec: 120 }], ["POST", "/internal/settings-sync", { maxEventsPerSec: 120 }],
["POST", "/internal/security-check", { clientId: "x", eventId: "auth_chk_1", payloadBytes: 12 }],
] as const) { ] as const) {
const noAuth = await fetch(`${ingestionUrl}${path}`, { const noAuth = await fetch(`${ingestionUrl}${path}`, {
method, method,
@@ -373,3 +399,824 @@ test("BUG-08: Rate-limit error reports effectiveRateLimit and ApiEngine aggregat
`Expected aggregated rateLimitedClientsCount >= 2, got ${metrics.securityStats.rateLimitedClientsCount}`, `Expected aggregated rateLimitedClientsCount >= 2, got ${metrics.securityStats.rateLimitedClientsCount}`,
); );
}); });
function createSharedMockRedis(): RedisBufferClient {
const kv = new Map<string, string>();
const lists = new Map<string, string[]>();
const hashes = new Map<string, Map<string, string>>();
const sets = new Map<string, Set<string>>();
return {
status: "ready",
async get(key: string) {
return kv.has(key) ? kv.get(key)! : null;
},
async incr(key: string) {
const next = (Number(kv.get(key)) || 0) + 1;
kv.set(key, String(next));
return next;
},
async pexpire() {
return 1;
},
async set(key: string, value: string, _px?: "PX", _ttlMs?: number, nxMode?: "NX") {
if (nxMode === "NX" && kv.has(key)) return null;
kv.set(key, value);
return "OK";
},
async hget(key: string, field: string) {
return hashes.get(key)?.get(field) ?? null;
},
async hset(key: string, field: string, value: string) {
if (!hashes.has(key)) hashes.set(key, new Map());
hashes.get(key)!.set(field, value);
return 1;
},
async hdel(key: string, ...fields: string[]) {
let deleted = 0;
const h = hashes.get(key);
if (!h) return 0;
for (const f of fields) {
if (h.delete(f)) deleted += 1;
}
return deleted;
},
async sismember(key: string, member: string) {
return sets.get(key)?.has(member) ? 1 : 0;
},
async sadd(key: string, ...members: string[]) {
if (!sets.has(key)) sets.set(key, new Set());
for (const m of members) sets.get(key)!.add(m);
return members.length;
},
async srem(key: string, ...members: string[]) {
const s = sets.get(key);
if (!s) return 0;
let removed = 0;
for (const m of members) {
if (s.delete(m)) removed += 1;
}
return removed;
},
async rpush(key: string, ...values: string[]) {
const list = lists.get(key) ?? [];
list.push(...values);
lists.set(key, list);
return list.length;
},
async lpush(key: string, ...values: string[]) {
const list = lists.get(key) ?? [];
list.unshift(...values);
lists.set(key, list);
return list.length;
},
async ltrim(key: string, start: number, stop: number) {
const list = lists.get(key) ?? [];
const len = list.length;
const end = stop < 0 ? len + stop : stop;
lists.set(key, list.slice(start, end + 1));
return "OK";
},
async lrange(key: string, start: number, stop: number) {
const list = lists.get(key) ?? [];
const len = list.length;
const end = stop < 0 ? len + stop : stop;
return list.slice(start, end + 1);
},
async llen(key: string) {
return (lists.get(key) ?? []).length;
},
async eval(script: string, _numKeys: number, ...args: string[]) {
if (script.includes("DATASTACK_RATE_LIMIT_INCR")) {
const key = args[0];
const next = (Number(kv.get(key)) || 0) + 1;
kv.set(key, String(next));
return next;
}
if (script.includes("DATASTACK_PURGE_CLIENT_LIST")) {
const [key, target] = args;
const list = lists.get(key) ?? [];
const kept: string[] = [];
let removed = 0;
for (const raw of list) {
try {
const parsed = JSON.parse(raw) as { clientId?: string };
const cid = String(parsed?.clientId || "")
.trim()
.toLowerCase()
.replace(/[^a-z0-9_-]/g, "_");
if (cid === target) {
removed += 1;
continue;
}
} catch {
// preserve unparseable entries
}
kept.push(raw);
}
if (removed > 0) lists.set(key, kept);
return removed;
}
return 1;
},
};
}
test("BUG-B01: Port 4000 verifyClientInDb fails closed for unregistered/empty clients even without DB", async () => {
const emptyCheck = await serverMod.verifyClientInDb("", "any-token", true);
assert.equal(emptyCheck.allowed, false);
assert.match(emptyCheck.reason || "", /CLIENT_REVOKED/);
const ghostCheck = await serverMod.verifyClientInDb("Unregistered Ghost", "any-token", true);
assert.equal(ghostCheck.allowed, false);
assert.match(ghostCheck.reason || "", /CLIENT_REVOKED/);
const ghostToken = AuthService.generateClientToken({ clientId: "never_registered_ws" });
await assert.rejects(() => connectSocket(ghostToken), /CLIENT_REVOKED/);
});
test("BUG-B02 & BUG-B04: Shared Redis idempotency/rate-limit across Port 3000+4000 and flat sync_data payload parity", async () => {
const mockRedis = createSharedMockRedis();
DataBufferEngine.setRedisClientForTests(mockRedis);
serverMod.setRedisClientForTests(mockRedis);
try {
await SettingsManager.updateSettingsAsync({
maxEventsPerSec: 3,
enableIdempotency: true,
});
const client = await SDKManager.registerClientAsync({
clientId: "phase02_redis_parity",
name: "Phase02 Redis Parity",
rateLimit: 3,
});
// 1. Flat sync_data payload (no .data wrapper) preserves fields cleanly (BUG-B04)
serverMod.bufferQueue.length = 0;
const socket = await connectSocket(client.token);
const flatAck = await emitWithAck<{ success: boolean; eventId: string }>(
socket,
"sync_data",
{ eventId: "evt_shared_flat_1", metric: "voltage", value: 3.3 },
);
assert.equal(flatAck.success, true);
const bufferedItems = await mockRedis.lrange("datastack:buffer:events", 0, -1);
const storedFlat = bufferedItems
.map((raw) => JSON.parse(raw) as { eventId: string; payload: string })
.find((e) => e.eventId === "evt_shared_flat_1");
assert.ok(storedFlat, "Flat sync_data event must be buffered in Redis");
const parsedFlat = JSON.parse(storedFlat.payload) as Record<string, unknown>;
assert.equal(parsedFlat.metric, "voltage");
assert.equal(parsedFlat.value, 3.3);
// 2. Replaying same eventId over HTTP POST /api/events triggers shared DUPLICATE_EVENT (BUG-B02)
const replayHttpRes = await postEventsRoute(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
"content-type": "application/json",
authorization: `Bearer ${client.token}`,
},
body: JSON.stringify({
eventId: "evt_shared_flat_1",
data: { metric: "voltage", value: 3.3 },
}),
}),
);
assert.equal(replayHttpRes.status, 200);
const replayHttpJson = await replayHttpRes.json();
assert.equal(replayHttpJson.note, "DUPLICATE_DROPPED");
// 3. Cross-runtime shared per-second rate-limit: 1 WS already consumed quota; 2 HTTP pass, 3rd hits 429 (BUG-B02)
const ok2 = await postEventsRoute(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
"content-type": "application/json",
authorization: `Bearer ${client.token}`,
},
body: JSON.stringify({ eventId: "evt_rl_shared_2", data: { n: 2 } }),
}),
);
const ok3 = await postEventsRoute(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
"content-type": "application/json",
authorization: `Bearer ${client.token}`,
},
body: JSON.stringify({ eventId: "evt_rl_shared_3", data: { n: 3 } }),
}),
);
assert.equal(ok2.status, 200);
assert.equal(ok3.status, 200);
const limited4 = await postEventsRoute(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
"content-type": "application/json",
authorization: `Bearer ${client.token}`,
},
body: JSON.stringify({ eventId: "evt_rl_shared_4", data: { n: 4 } }),
}),
);
assert.equal(limited4.status, 429);
socket.disconnect();
} finally {
DataBufferEngine.setRedisClientForTests(undefined);
serverMod.setRedisClientForTests(undefined);
}
});
test("BUG-B03 & BUG-B05: Surfacing bridge sync failures, live runtime ports, normalized counter & production start script", async () => {
const cookieHeader = await createAdminCookieHeader();
const res = await postSettingsRoute(
new NextRequest("http://localhost:3000/api/settings", {
method: "POST",
headers: { "content-type": "application/json", cookie: cookieHeader },
body: JSON.stringify({ maxEventsPerSec: 120 }),
}),
);
assert.equal(res.status, 200);
const body = await res.json();
assert.equal(body.bridgeSync?.synced, true);
// Mismatched secret on remote Port 4000 must surface auth_error on dashboard metrics
const http = await import("node:http");
const rejectServer = http.createServer((_req, res) => {
res.writeHead(401, { "Content-Type": "application/json" });
res.end(JSON.stringify({ success: false, code: "UNAUTHORIZED" }));
});
await new Promise<void>((resolveListen) => rejectServer.listen(0, "127.0.0.1", resolveListen));
const rejectAddr = rejectServer.address() as AddressInfo;
const savedIngestionUrl = process.env.INGESTION_INTERNAL_URL;
process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${rejectAddr.port}`;
try {
const metrics = await ApiEngine.getDashboardMetrics();
assert.equal(metrics.runtime.ingestionBridgeStatus, "auth_error");
assert.match(metrics.runtime.ingestionBridgeError || "", /HTTP 401/);
} finally {
process.env.INGESTION_INTERNAL_URL = savedIngestionUrl;
await new Promise<void>((resolveClose) => rejectServer.close(() => resolveClose()));
}
// Normalized totalEvents counter updates cleanly
const normClient = await SDKManager.registerClientAsync({
clientId: "Mixed.Case.Counter",
name: "Counter Check",
rateLimit: 100,
});
await SDKManager.incrementEventCountAsync("MIXED.CASE.COUNTER", 3);
const fetched = await SDKManager.getClientAsync(normClient.clientId);
assert.equal(fetched?.totalEvents, 3);
// Production start script runs dual-runtime launcher
const pkg = JSON.parse(readFileSync(resolve(process.cwd(), "package.json"), "utf8")) as {
scripts?: Record<string, string>;
};
assert.equal(pkg.scripts?.start, "node scripts/start-all.js");
assert.equal(pkg.scripts?.["start:server"], "node server/index.js");
});
test("FORENSIC BUG-02: Empty Port 3000 bulk_sync preserves Port 4000 registry and hydrates Port 3000 non-destructively", async () => {
const survivor = await SDKManager.registerClientAsync({
clientId: "phase02_survivor_client",
name: "Survivor Client",
rateLimit: 40,
});
// Verify Port 4000 knows survivor before simulated Port 3000 restart
const preCheck = await serverMod.verifyClientInDb(survivor.clientId, survivor.token, true);
assert.equal(preCheck.allowed, true);
// Simulate Port 3000 cold restart without PostgreSQL (local map cleared)
g.__dataStackClientsMap?.clear();
assert.equal(SDKManager.getClient(survivor.clientId), undefined);
// Calling syncAllClientsToIngestionAsync() with an empty Port 3000 map must NOT wipe Port 4000
const syncRes = await SDKManager.syncAllClientsToIngestionAsync();
assert.equal(syncRes.synced, true);
const postCheck = await serverMod.verifyClientInDb(survivor.clientId, survivor.token, true);
assert.equal(postCheck.allowed, true, "Port 4000 registry must NOT be wiped by empty Port 3000 bulk_sync");
// Port 3000 should also have hydrated non-destructively from Port 4000
const hydratedAfterBulk = SDKManager.getClient(survivor.clientId);
assert.ok(hydratedAfterBulk, "Port 3000 should hydrate preserved client after empty bulk_sync");
assert.equal(hydratedAfterBulk.token, survivor.token);
// Also test cold hydration via getClientAsync & listClientsAsync after clearing map again
g.__dataStackClientsMap?.clear();
const hydratedViaGet = await SDKManager.getClientAsync(survivor.clientId);
assert.ok(hydratedViaGet, "getClientAsync should non-destructively hydrate from Port 4000");
assert.equal(hydratedViaGet.clientId, survivor.clientId);
});
test("FORENSIC BUG-17: JWT rotation coordinates Port 4000 first, aborts on bridge rejection without split-brain, and supports rotation grace", async () => {
const originalSecret = AuthService.getSecret();
const http = await import("node:http");
// 1. If Port 4000 rejects /internal/settings-sync with 503, Port 3000 MUST abort rotation cleanly
const rejectServer = http.createServer((_req, res) => {
res.writeHead(503, { "Content-Type": "application/json" });
res.end(JSON.stringify({ success: false, code: "BRIDGE_REJECTED" }));
});
await new Promise<void>((r) => rejectServer.listen(0, "127.0.0.1", r));
const rejectPort = (rejectServer.address() as AddressInfo).port;
const savedUrl = process.env.INGESTION_INTERNAL_URL;
process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${rejectPort}`;
try {
await assert.rejects(
() =>
SettingsManager.updateSettingsAsync({
jwtSecret: "aborted-rotation-secret-must-never-commit-32-chars",
}),
/JWT_ROTATION_BRIDGE_FAILED/,
);
assert.equal(
AuthService.getSecret(),
originalSecret,
"Port 3000 AuthService secret must remain unchanged when Port 4000 rejects rotation",
);
assert.equal(
SettingsManager.getSettings().jwtSecret,
originalSecret,
"SettingsManager secret must remain unchanged after aborted rotation",
);
} finally {
process.env.INGESTION_INTERNAL_URL = savedUrl;
await new Promise<void>((r) => rejectServer.close(() => r()));
}
// 2. Successful rotation updates both runtimes AND keeps previous secret valid on Port 4000 bridge grace window
const rotatedSecret = "phase02-bug17-coordinated-rotation-secret-32-chars";
await SettingsManager.updateSettingsAsync({ jwtSecret: rotatedSecret });
assert.equal(AuthService.getSecret(), rotatedSecret);
assert.equal(serverMod.runtimeConfig.jwtSecret, rotatedSecret);
const graceHeaders = getInternalIngestionHeaders(originalSecret);
const graceMetricsRes = await fetch(`${ingestionUrl}/internal/metrics`, {
headers: graceHeaders,
});
assert.equal(
graceMetricsRes.status,
200,
"Port 4000 internal auth must accept previous secret during rotation grace window",
);
// 3. New secret also works on internal bridge
const currentMetricsRes = await fetch(`${ingestionUrl}/internal/metrics`, {
headers: getInternalIngestionHeaders(rotatedSecret),
});
assert.equal(currentMetricsRes.status, 200);
// 4. If PG write fails AFTER Port 4000 accepts the new secret, BOTH Port 3000 & Port 4000 roll back cleanly
const prevDbUrlLocal = process.env.DATABASE_URL;
const prevPoolLocal = g.__dataStackCorePool;
const prevSchemaLocal = g.__dataStackCoreSchema;
process.env.DATABASE_URL = "postgres://mock:mock@127.0.0.1:5432/mock";
g.__dataStackCoreSchema = Promise.resolve();
g.__dataStackCorePool = {
async query(sql: string) {
if (sql.includes("INSERT INTO admin_settings")) {
throw new Error("Simulated PG disk full during rotation");
}
return { rowCount: 0, rows: [] };
},
};
try {
await assert.rejects(
() =>
SettingsManager.updateSettingsAsync({
jwtSecret: "rolled-back-secret-after-pg-failure-32-chars-min",
}),
/DATABASE_WRITE_FAILED/,
);
assert.equal(
AuthService.getSecret(),
rotatedSecret,
"Port 3000 must roll back to previous secret after DB write failure",
);
assert.equal(
serverMod.runtimeConfig.jwtSecret,
rotatedSecret,
"Port 4000 must roll back to previous secret after Port 3000 DB write failure",
);
} finally {
if (prevDbUrlLocal === undefined) delete process.env.DATABASE_URL;
else process.env.DATABASE_URL = prevDbUrlLocal;
g.__dataStackCorePool = prevPoolLocal;
g.__dataStackCoreSchema = prevSchemaLocal;
}
});
test("FORENSIC BUG-03: Without Redis, WS and HTTP share idempotency and rate limits via Port 4000 /internal/security-check", async () => {
// Explicitly ensure Redis is disabled
DataBufferEngine.setRedisClientForTests(null);
serverMod.setRedisClientForTests(null);
try {
await SettingsManager.updateSettingsAsync({
maxEventsPerSec: 3,
enableIdempotency: true,
});
const client = await SDKManager.registerClientAsync({
clientId: "phase02_no_redis_sec",
name: "No-Redis Shared Security",
rateLimit: 3,
});
const socket = await connectSocket(client.token);
const wsAck = await emitWithAck<{ success: boolean; eventId: string }>(
socket,
"sync_data",
{ eventId: "evt_noredis_shared_1", data: { step: 1 } },
);
assert.equal(wsAck.success, true);
// Same eventId over HTTP POST /api/events MUST be deduplicated via Port 4000 bridge check
const dupHttpRes = await postEventsRoute(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
"content-type": "application/json",
authorization: `Bearer ${client.token}`,
},
body: JSON.stringify({
eventId: "evt_noredis_shared_1",
data: { step: 1 },
}),
}),
);
assert.equal(dupHttpRes.status, 200);
const dupJson = await dupHttpRes.json();
assert.equal(
dupJson.note,
"DUPLICATE_DROPPED",
"HTTP replay of WS eventId must be dropped even without Redis",
);
// Rate-limit parity without Redis: WS consumed 1 of 3; 2 HTTP pass, 3rd returns 429
const http2 = await postEventsRoute(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
"content-type": "application/json",
authorization: `Bearer ${client.token}`,
},
body: JSON.stringify({ eventId: "evt_noredis_rl_2", data: { step: 2 } }),
}),
);
const http3 = await postEventsRoute(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
"content-type": "application/json",
authorization: `Bearer ${client.token}`,
},
body: JSON.stringify({ eventId: "evt_noredis_rl_3", data: { step: 3 } }),
}),
);
assert.equal(http2.status, 200);
assert.equal(http3.status, 200);
const http4 = await postEventsRoute(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
"content-type": "application/json",
authorization: `Bearer ${client.token}`,
},
body: JSON.stringify({ eventId: "evt_noredis_rl_4", data: { step: 4 } }),
}),
);
assert.equal(http4.status, 429, "Combined WS+HTTP quota must enforce 429 without Redis");
// Metrics explicitly surface bridge_coordinated securityMode
const metrics = await ApiEngine.getDashboardMetrics();
assert.equal(metrics.securityStats.mode, "bridge_coordinated");
assert.equal(metrics.runtime.securityMode, "bridge_coordinated");
socket.disconnect();
} finally {
DataBufferEngine.setRedisClientForTests(undefined);
serverMod.setRedisClientForTests(undefined);
}
});
test("FORENSIC BUG-22: High-frequency client event increments coalesce into batched UNNEST SQL updates on both Port 3000 and Port 4000", async () => {
const sdkQueries: Array<{ sql: string; params: unknown[] }> = [];
const serverQueries: Array<{ sql: string; params: unknown[] }> = [];
const mockSdkPool = {
async query(sql: string, params: unknown[] = []) {
sdkQueries.push({ sql, params });
return { rowCount: 1, rows: [] };
},
};
const mockServerPool = {
async query(sql: string, params: unknown[] = []) {
serverQueries.push({ sql, params });
return { rowCount: 1, rows: [] };
},
};
const prevDatabaseUrl = process.env.DATABASE_URL;
const prevCorePool = g.__dataStackCorePool;
const prevCoreSchema = g.__dataStackCoreSchema;
process.env.DATABASE_URL = "postgres://mock:mock@127.0.0.1:5432/mock";
g.__dataStackCorePool = mockSdkPool;
g.__dataStackCoreSchema = Promise.resolve();
g.__dataStackPendingClientActivity?.clear();
serverMod.setDbPoolForTests(mockServerPool);
try {
// 1. Port 3000 SDKManager: 25 rapid increments across 2 clients -> 0 immediate per-event queries, 1 batched UNNEST query on flush
for (let i = 0; i < 15; i += 1) {
await SDKManager.incrementEventCountAsync("batch_client_a", 1);
}
for (let i = 0; i < 10; i += 1) {
await SDKManager.incrementEventCountAsync("batch_client_b", 2);
}
const preFlushUpdates = sdkQueries.filter((q) =>
q.sql.includes("UPDATE client_sdks"),
);
assert.equal(
preFlushUpdates.length,
0,
"Per-event increments must coalesce in memory instead of firing immediate per-row SQL",
);
const flushedSdkClients = await SDKManager.flushPendingClientActivityAsync();
assert.equal(flushedSdkClients, 2);
const postFlushSdkUpdates = sdkQueries.filter((q) =>
q.sql.includes("UPDATE client_sdks"),
);
assert.equal(
postFlushSdkUpdates.length,
1,
"25 increments across 2 clients must execute exactly ONE batched UPDATE query",
);
assert.match(postFlushSdkUpdates[0].sql, /UNNEST\(\$1::text\[\]\)/);
assert.deepEqual(postFlushSdkUpdates[0].params[0], [
"batch_client_a",
"batch_client_b",
]);
assert.deepEqual(postFlushSdkUpdates[0].params[1], [15, 20]);
// 2. Port 4000 server/index.js: 30 rapid recordClientActivity calls -> 1 batched UNNEST query on flush
for (let i = 0; i < 20; i += 1) {
serverMod.recordClientActivity("ws_batch_a", 1);
}
for (let i = 0; i < 10; i += 1) {
serverMod.recordClientActivity("ws_batch_b", 3);
}
assert.equal(
serverQueries.length,
0,
"Port 4000 recordClientActivity must coalesce instead of running per-event SQL",
);
const flushedServerClients = await serverMod.flushPendingClientActivity();
assert.equal(flushedServerClients, 2);
assert.equal(
serverQueries.length,
1,
"30 Port 4000 activity events must coalesce into ONE batched UPDATE query",
);
assert.match(serverQueries[0].sql, /UNNEST\(\$1::text\[\]\)/);
assert.deepEqual(serverQueries[0].params[0], ["ws_batch_a", "ws_batch_b"]);
assert.deepEqual(serverQueries[0].params[1], [20, 30]);
} finally {
SDKManager.stopActivityFlushTimerForTests();
g.__dataStackPendingClientActivity?.clear();
serverMod.setDbPoolForTests(null);
if (prevDatabaseUrl === undefined) delete process.env.DATABASE_URL;
else process.env.DATABASE_URL = prevDatabaseUrl;
g.__dataStackCorePool = prevCorePool;
g.__dataStackCoreSchema = prevCoreSchema;
}
});
test("R2 PHASE-02 BUG-05: PG pools attach safe error listeners and Port 3000 shares a single unified DatabaseEngine pool", async () => {
const prevDbUrl = process.env.DATABASE_URL;
const prevCorePool = g.__dataStackCorePool;
const origConsoleError = console.error;
const loggedErrors: string[] = [];
console.error = (...args: unknown[]) => {
loggedErrors.push(args.map((a) => String(a)).join(" "));
};
try {
process.env.DATABASE_URL = "postgres://unit:test@127.0.0.1:5432/datastack_unit";
g.__dataStackCorePool = undefined;
const pool = DatabaseEngine.getPool();
assert.ok(pool, "DatabaseEngine.getPool() must instantiate a pool when DATABASE_URL is set");
assert.ok(
pool.listenerCount("error") >= 1,
"DatabaseEngine PG pool must attach at least one 'error' listener",
);
assert.equal(
isAdminDatabaseConfigured(),
true,
"adminUsers must delegate to the shared DatabaseEngine.getPool()",
);
// Emitting an idle client error must log safely instead of throwing an uncaught EventEmitter crash
assert.doesNotThrow(() => {
pool.emit("error", new Error("simulated idle PG restart"));
});
assert.ok(
loggedErrors.some((msg) => msg.includes("simulated idle PG restart")),
"Pool error listener must log unexpected idle pool errors",
);
// Static verification that adminUsers.ts has zero independent `new Pool` and server/index.js attaches dbPool.on('error')
const adminUsersSrc = readFileSync(
resolve(process.cwd(), "core", "auth", "adminUsers.ts"),
"utf8",
);
assert.equal(/\bnew Pool\s*\(/.test(adminUsersSrc), false);
const serverSrc = readFileSync(
resolve(process.cwd(), "server", "index.js"),
"utf8",
);
assert.match(serverSrc, /dbPool\.on\(\s*['"]error['"]/);
await pool.end().catch(() => undefined);
} finally {
console.error = origConsoleError;
if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
else process.env.DATABASE_URL = prevDbUrl;
g.__dataStackCorePool = prevCorePool;
}
});
test("R2 PHASE-02 BUG-09: Redis rate-limiting uses atomic single-roundtrip Lua EVAL on both Port 3000 and Port 4000", async () => {
const baseRedis = createSharedMockRedis();
let evalRateLimitCalls = 0;
let separateIncrCalls = 0;
let separatePexpireCalls = 0;
const instrumentedRedis: RedisBufferClient = {
...baseRedis,
async incr(key: string) {
separateIncrCalls += 1;
return baseRedis.incr!(key);
},
async pexpire(key: string, ttlMs: number) {
separatePexpireCalls += 1;
return baseRedis.pexpire!(key, ttlMs);
},
async eval(script: string, numKeys: number, ...args: string[]) {
if (script.includes("DATASTACK_RATE_LIMIT_INCR")) {
evalRateLimitCalls += 1;
assert.equal(args[1], "2000", "Atomic Lua rate-limit script must pass 2000ms TTL");
}
return baseRedis.eval(script, numKeys, ...args);
},
};
DataBufferEngine.setRedisClientForTests(instrumentedRedis);
serverMod.setRedisClientForTests(instrumentedRedis);
try {
await SettingsManager.updateSettingsAsync({
maxEventsPerSec: 2,
enableIdempotency: true,
});
const client = await SDKManager.registerClientAsync({
clientId: "phase02_lua_rl_client",
name: "Lua RL Client",
rateLimit: 2,
});
// 1 event via Port 3000 HTTP SecurityEngine
const http1 = await SecurityEngine.validateEventAsync(
client.clientId,
"lua_rl_1",
{ v: 1 },
client.rateLimit,
);
assert.equal(http1.allowed, true);
// 2nd event via Port 4000 Socket.IO handler (allowed) + 3rd event (rate limited)
const socket = await connectSocket(client.token);
const ws2 = await emitWithAck<{ success: boolean; code?: string }>(
socket,
"sync_data",
{ eventId: "lua_rl_2", data: { v: 2 } },
);
assert.equal(ws2.success, true);
const ws3 = await emitWithAck<{ success: boolean; code?: string }>(
socket,
"sync_data",
{ eventId: "lua_rl_3", data: { v: 3 } },
);
assert.equal(ws3.success, false);
assert.equal(ws3.code, "RATE_LIMIT_EXCEEDED");
socket.disconnect();
assert.equal(
evalRateLimitCalls,
3,
"All 3 rate-limit checks across Port 3000 & Port 4000 must execute atomic Lua EVAL",
);
assert.equal(
separateIncrCalls,
0,
"Separate non-atomic redis.incr must NOT be called when redis.eval is available",
);
assert.equal(
separatePexpireCalls,
0,
"Separate non-atomic redis.pexpire must NOT be called when redis.eval is available",
);
} finally {
DataBufferEngine.setRedisClientForTests(undefined);
serverMod.setRedisClientForTests(undefined);
await SettingsManager.updateSettingsAsync({ maxEventsPerSec: 100 });
}
});
test("R2 PHASE-02 BUG-10: GET, PATCH (regenerate_token), and DELETE /api/clients/[id] catch DB errors and return structured JSON", async () => {
const secret = getAdminSessionSecret()!;
const adminToken = await createAdminSessionToken(
"admin@datastack.dev",
secret,
Date.now() + 3600_000,
1,
);
const cookieHeader = `datastack_admin_session=${adminToken}`;
const origGet = SDKManager.getClientAsync;
const origRegen = SDKManager.regenerateTokenAsync;
const origDeleteWithStats = SDKManager.deleteClientWithStatsAsync;
try {
SDKManager.getClientAsync = async () => {
throw new Error("DATABASE_READ_FAILED: simulated pg read outage");
};
const getRes = await getClientRoute(
new NextRequest("http://localhost:3000/api/clients/broken_client", {
method: "GET",
headers: { cookie: cookieHeader },
}),
{ params: Promise.resolve({ id: "broken_client" }) },
);
assert.equal(getRes.status, 503);
const getJson = await getRes.json();
assert.equal(getJson.code, "DATABASE_READ_FAILED");
assert.match(getJson.error, /simulated pg read outage/);
SDKManager.regenerateTokenAsync = async () => {
throw new Error("DATABASE_WRITE_FAILED: simulated token rotation DB failure");
};
const regenRes = await patchClientRoute(
new NextRequest("http://localhost:3000/api/clients/broken_client", {
method: "PATCH",
headers: {
"content-type": "application/json",
cookie: cookieHeader,
},
body: JSON.stringify({ action: "regenerate_token" }),
}),
{ params: Promise.resolve({ id: "broken_client" }) },
);
assert.equal(regenRes.status, 503);
const regenJson = await regenRes.json();
assert.equal(regenJson.code, "DATABASE_WRITE_FAILED");
assert.match(regenJson.error, /simulated token rotation DB failure/);
SDKManager.deleteClientWithStatsAsync = async () => {
throw new Error("DATABASE_WRITE_FAILED: simulated delete DB failure");
};
const delRes = await deleteClientRoute(
new NextRequest("http://localhost:3000/api/clients/broken_client", {
method: "DELETE",
headers: { cookie: cookieHeader },
}),
{ params: Promise.resolve({ id: "broken_client" }) },
);
assert.equal(delRes.status, 503);
const delJson = await delRes.json();
assert.equal(delJson.code, "DATABASE_WRITE_FAILED");
assert.match(delJson.error, /simulated delete DB failure/);
} finally {
SDKManager.getClientAsync = origGet;
SDKManager.regenerateTokenAsync = origRegen;
SDKManager.deleteClientWithStatsAsync = origDeleteWithStats;
}
});
File diff suppressed because it is too large. Load diff
+495
View File
@@ -0,0 +1,495 @@
import assert from "node:assert/strict";
import fs from "node:fs";
import { createRequire } from "node:module";
import os from "node:os";
import path from "node:path";
import test, { after, before, beforeEach } from "node:test";
import { NextRequest } from "next/server";
import { GET as getDataRoute } from "../app/api/data/route";
import {
GET as getEventsRoute,
POST as postEventsRoute,
} from "../app/api/events/route";
import {
createAdminSessionToken,
getAdminSessionSecret,
} from "../core/auth/adminSession";
import {
createInitialAdmin,
resetAdminStateForTests,
} from "../core/auth/adminUsers";
import {
DataBufferEngine,
MAX_REDIS_RECENT_LIMIT,
resolveRedisRecentLimit,
} from "../core/data/buffer";
import {
CORE_SCHEMA_ALTER_DDLS,
CORE_SCHEMA_TABLE_DDLS,
} from "../core/Database/database";
import { SDKManager } from "../core/sdk/sdkManager";
import { CSVStorageManager, type DataEvent } from "../core/Storage/csvStorage";
const g = globalThis as typeof globalThis & {
__dataStackBufferState?: {
buffer: DataEvent[];
recentEvents: DataEvent[];
deletedClientIds?: Set<string>;
batchFlushThreshold: number;
};
__dataStackAdminSettings?: unknown;
__dataStackClientsMap?: Map<string, unknown>;
__dataStackDeletedClients?: Set<string>;
};
let tmpStorageDir = "";
let adminCookieHeader = "";
const origBaseStorageDir = CSVStorageManager.baseStorageDir;
const prevDbUrl = process.env.DATABASE_URL;
const prevRedisUrl = process.env.REDIS_URL;
const prevRecentEnv = process.env.DATASTACK_RECENT_LIMIT;
const prevIngestionUrl = process.env.INGESTION_INTERNAL_URL;
before(async () => {
delete process.env.DATABASE_URL;
delete process.env.REDIS_URL;
delete process.env.DATASTACK_RECENT_LIMIT;
process.env.INGESTION_INTERNAL_URL = "http://127.0.0.1:1";
resetAdminStateForTests();
await createInitialAdmin("admin@datastack.dev", "Phase04Pass123!");
const secret = getAdminSessionSecret()!;
const token = await createAdminSessionToken(
"admin@datastack.dev",
secret,
Date.now() + 3600_000,
1,
);
adminCookieHeader = `datastack_admin_session=${token}`;
tmpStorageDir = fs.mkdtempSync(path.join(os.tmpdir(), "datastack-phase04-"));
CSVStorageManager.baseStorageDir = tmpStorageDir;
CSVStorageManager.clearRowCountCache();
});
beforeEach(() => {
process.env.INGESTION_INTERNAL_URL = "http://127.0.0.1:1";
DataBufferEngine.setRedisClientForTests(null);
DataBufferEngine.batchFlushThreshold = 500;
delete process.env.DATASTACK_RECENT_LIMIT;
if (g.__dataStackBufferState) {
g.__dataStackBufferState.buffer = [];
g.__dataStackBufferState.recentEvents = [];
g.__dataStackBufferState.deletedClientIds?.clear();
}
CSVStorageManager.clearRowCountCache();
fs.rmSync(tmpStorageDir, { recursive: true, force: true });
fs.mkdirSync(tmpStorageDir, { recursive: true });
delete g.__dataStackAdminSettings;
g.__dataStackClientsMap?.clear();
g.__dataStackDeletedClients?.clear();
});
after(async () => {
await DataBufferEngine.shutdown();
CSVStorageManager.baseStorageDir = origBaseStorageDir;
CSVStorageManager.clearRowCountCache();
if (tmpStorageDir) {
fs.rmSync(tmpStorageDir, { recursive: true, force: true });
}
if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
else process.env.DATABASE_URL = prevDbUrl;
if (prevRedisUrl === undefined) delete process.env.REDIS_URL;
else process.env.REDIS_URL = prevRedisUrl;
if (prevRecentEnv === undefined) delete process.env.DATASTACK_RECENT_LIMIT;
else process.env.DATASTACK_RECENT_LIMIT = prevRecentEnv;
if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL;
else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl;
});
test("FORENSIC BUG-21 (Part A): HTTP POST /api/events matches WS envelope stripping parity and never fabricates sample telemetry for {}", async () => {
const client = await SDKManager.registerClientAsync({
clientId: "phase04_parity",
name: "Parity Client",
rateLimit: 100,
});
// 1. Empty {} body must store "{}" (NO fake { sample, rand })
const emptyRes = await postEventsRoute(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
"content-type": "application/json",
authorization: `Bearer ${client.token}`,
},
body: JSON.stringify({}),
}),
);
assert.equal(emptyRes.status, 200);
const emptyJson = await emptyRes.json();
assert.equal(emptyJson.success, true);
assert.equal(emptyJson.event.payload, "{}");
assert.equal(emptyJson.event.payload.includes("sample"), false);
assert.equal(emptyJson.event.payload.includes("rand"), false);
// 2. Flat body with envelope keys must strip eventId/timestamp/clientId cleanly
const flatRes = await postEventsRoute(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
"content-type": "application/json",
authorization: `Bearer ${client.token}`,
},
body: JSON.stringify({
eventId: "flat-parity-1",
timestamp: "2026-11-19T12:00:00.000Z",
clientId: client.clientId,
temp: 24.5,
unit: "C",
}),
}),
);
assert.equal(flatRes.status, 200);
const flatJson = await flatRes.json();
assert.deepEqual(JSON.parse(flatJson.event.payload), { temp: 24.5, unit: "C" });
// 3. Pure envelope { eventId, timestamp, data } extracts data identically to WS
const wrappedRes = await postEventsRoute(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
"content-type": "application/json",
authorization: `Bearer ${client.token}`,
},
body: JSON.stringify({
eventId: "wrapped-parity-1",
timestamp: "2026-11-19T12:00:01.000Z",
data: { pressure: 101.3 },
}),
}),
);
assert.equal(wrappedRes.status, 200);
const wrappedJson = await wrappedRes.json();
assert.deepEqual(JSON.parse(wrappedJson.event.payload), { pressure: 101.3 });
// 4. Flat payload that contains both `data` and extra domain fields preserves the object identically to WS
const mixedRes = await postEventsRoute(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
"content-type": "application/json",
authorization: `Bearer ${client.token}`,
},
body: JSON.stringify({
eventId: "mixed-parity-1",
data: "sensor-raw",
extraMetric: 42,
}),
}),
);
assert.equal(mixedRes.status, 200);
const mixedJson = await mixedRes.json();
assert.deepEqual(JSON.parse(mixedJson.event.payload), {
eventId: "mixed-parity-1",
data: "sensor-raw",
extraMetric: 42,
});
});
test("FORENSIC BUG-21 (Part B): GET /api/data uses non-blocking async fs.promises with partition pruning and REDIS_RECENT_LIMIT scales boundedly", async () => {
const dataRouteSrc = fs.readFileSync(
path.resolve(process.cwd(), "app/api/data/route.ts"),
"utf8",
);
assert.match(dataRouteSrc, /from "node:fs\/promises"/);
assert.equal(/\breaddirSync\b/.test(dataRouteSrc), false);
assert.equal(/\blstatSync\b/.test(dataRouteSrc), false);
// Create an out-of-range corrupt CSV partition and an in-range healthy CSV partition
const clientDir = CSVStorageManager.ensureClientDir("phase04_async");
fs.writeFileSync(
path.join(clientDir, "2026-11-10.csv"),
'"clientId","eventId","timestamp","payload"\n"phase04_async","bad","2026-11-10T00:00:00Z","unclosed',
"utf8",
);
await CSVStorageManager.appendBatch([
{
clientId: "phase04_async",
eventId: "in-range-1",
timestamp: "2026-11-19T14:00:00.000Z",
payload: JSON.stringify({ healthy: true }),
},
]);
const res = await getDataRoute(
new NextRequest(
"http://localhost:3000/api/data?clientId=phase04_async&from=2026-11-19&to=2026-11-19",
{ headers: { cookie: adminCookieHeader } },
),
);
assert.equal(res.status, 200);
const json = await res.json();
assert.equal(json.total, 1);
assert.equal(json.events[0].eventId, "in-range-1");
// Bounded configurable recent limit scaling
assert.equal(resolveRedisRecentLimit(500), 500);
assert.equal(resolveRedisRecentLimit(1200), 1200);
assert.equal(resolveRedisRecentLimit(50000), MAX_REDIS_RECENT_LIMIT);
process.env.DATASTACK_RECENT_LIMIT = "900";
assert.equal(DataBufferEngine.getRecentLimit(), 900);
delete process.env.DATASTACK_RECENT_LIMIT;
DataBufferEngine.batchFlushThreshold = 650;
assert.equal(DataBufferEngine.getRecentLimit(), 650);
const batch600: DataEvent[] = Array.from({ length: 600 }, (_, i) => ({
clientId: "phase04_recent_scale",
eventId: `recent-scale-${i}`,
timestamp: "2026-11-19T15:00:00.000Z",
payload: JSON.stringify({ i }),
}));
await DataBufferEngine.pushBatchAsync(batch600);
assert.equal(DataBufferEngine.getRecentEvents(650, "phase04_recent_scale").length, 600);
});
test("FORENSIC BUG-23: Client secrets and fallback eventIds use CSPRNG crypto.randomBytes with zero Math.random()", async () => {
const client = await SDKManager.registerClientAsync({
clientId: "phase04_csprng",
name: "CSPRNG Client",
rateLimit: 100,
});
assert.match(client.secret, /^sec_phase04_csprng_[0-9a-f]{32}$/);
const rotated = await SDKManager.regenerateTokenAsync(client.clientId);
assert.ok(rotated);
assert.match(rotated.secret, /^sec_phase04_csprng_[0-9a-f]{32}$/);
assert.notEqual(rotated.secret, client.secret);
const res = await postEventsRoute(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
"content-type": "application/json",
authorization: `Bearer ${rotated.token}`,
},
body: JSON.stringify({ metric: "ok" }),
}),
);
assert.equal(res.status, 200);
const json = await res.json();
assert.match(json.eventId, /^evt_\d+_[0-9a-f]{12}$/);
for (const relPath of [
"core/sdk/sdkManager.ts",
"app/api/events/route.ts",
"server/socket/handlers.js",
]) {
const src = fs.readFileSync(path.resolve(process.cwd(), relPath), "utf8");
assert.equal(
/\bMath\.random\s*\(/.test(src),
false,
`Expected zero Math.random() calls in ${relPath}`,
);
}
});
test("FORENSIC BUG-24: Canonical PostgreSQL DDL is unified in database.ts, delegated by adminUsers.ts, and mirrored 1:1 in server/index.js & prisma/schema.prisma", () => {
const adminUsersSrc = fs.readFileSync(
path.resolve(process.cwd(), "core/auth/adminUsers.ts"),
"utf8",
);
const serverIndexSrc = fs.readFileSync(
path.resolve(process.cwd(), "server/index.js"),
"utf8",
);
const prismaSchemaSrc = fs.readFileSync(
path.resolve(process.cwd(), "prisma/schema.prisma"),
"utf8",
);
// adminUsers.ts must delegate to ensureCoreSchemaOnPool rather than duplicating CREATE TABLE
assert.match(adminUsersSrc, /ensureCoreSchemaOnPool/);
assert.equal(/CREATE TABLE IF NOT EXISTS/.test(adminUsersSrc), false);
// All 5 canonical tables must exist identically in database.ts, server/index.js, and prisma/schema.prisma
const expectedTables = [
"client_sdks",
"admin_settings",
"deleted_clients",
"admin_users",
"admin_login_attempts",
];
const dbTablesText = CORE_SCHEMA_TABLE_DDLS.join("\n");
for (const table of expectedTables) {
const createRe = new RegExp(`CREATE TABLE IF NOT EXISTS\\s+${table}\\b`);
assert.match(dbTablesText, createRe, `database.ts missing ${table}`);
assert.match(serverIndexSrc, createRe, `server/index.js missing ${table}`);
assert.match(
prismaSchemaSrc,
new RegExp(`@@map\\("${table}"\\)`),
`prisma/schema.prisma missing @@map("${table}")`,
);
}
// All 3 additive ALTER TABLE guards must exist in both database.ts and server/index.js
const dbAltersText = CORE_SCHEMA_ALTER_DDLS.join("\n");
for (const alterTable of ["client_sdks", "admin_settings", "admin_users"]) {
const alterRe = new RegExp(`ALTER TABLE\\s+${alterTable}\\s+ADD COLUMN IF NOT EXISTS`);
assert.match(dbAltersText, alterRe, `database.ts missing ALTER TABLE ${alterTable}`);
assert.match(serverIndexSrc, alterRe, `server/index.js missing ALTER TABLE ${alterTable}`);
}
// prisma/schema.prisma must explicitly document canonical runtime pg ownership
assert.match(prismaSchemaSrc, /REFERENCE SCHEMA ONLY/);
assert.match(prismaSchemaSrc, /core\/Database\/database\.ts/);
});
test("R2 PHASE-03 BUG-11: Flat batch items strip eventId/timestamp/clientId across both HTTP POST /api/events and Socket.IO batch_sync", async () => {
const client = await SDKManager.registerClientAsync({
clientId: "phase03_batch_strip",
name: "Batch Strip Client",
rateLimit: 200,
});
// 1. HTTP POST /api/events batch array
const httpBatchRes = await postEventsRoute(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
"content-type": "application/json",
authorization: `Bearer ${client.token}`,
},
body: JSON.stringify({
batch: [
{
eventId: "http-flat-batch-1",
timestamp: "2026-11-19T12:10:00.000Z",
clientId: client.clientId,
sensor: "temp",
value: 23.4,
},
{
eventId: "http-env-batch-2",
timestamp: "2026-11-19T12:10:01.000Z",
data: { sensor: "humidity", value: 60 },
},
],
}),
}),
);
assert.equal(httpBatchRes.status, 200);
const httpBatchJson = await httpBatchRes.json();
assert.equal(httpBatchJson.processed, 2);
const localSnap = DataBufferEngine.getLocalBufferSnapshot();
const flatEvt = localSnap.find((e) => e.eventId === "http-flat-batch-1");
const envEvt = localSnap.find((e) => e.eventId === "http-env-batch-2");
assert.ok(flatEvt);
assert.ok(envEvt);
assert.deepEqual(JSON.parse(flatEvt.payload), { sensor: "temp", value: 23.4 });
assert.deepEqual(JSON.parse(envEvt.payload), { sensor: "humidity", value: 60 });
// 2. Socket.IO batch_sync handler
const require = createRequire(import.meta.url);
const { setupSocketHandlers } = require("../server/socket/handlers") as {
setupSocketHandlers: (
io: unknown,
bufferQueue: DataEvent[],
worker: unknown,
options?: Record<string, unknown>,
) => unknown;
};
let connectionHandler: ((socket: unknown) => void) | null = null;
const fakeIo = {
use: () => undefined,
on: (evt: string, fn: (socket: unknown) => void) => {
if (evt === "connection") connectionHandler = fn;
},
};
const wsQueue: DataEvent[] = [];
setupSocketHandlers(fakeIo, wsQueue, {}, {
verifyClientInDb: async () => ({ allowed: true, rateLimit: 200 }),
});
assert.ok(connectionHandler);
const listeners = new Map<string, (...args: unknown[]) => Promise<void>>();
const fakeSocket = {
data: { clientId: "phase03_batch_strip", token: client.token, rateLimit: 200 },
join: () => undefined,
disconnect: () => undefined,
on: (name: string, fn: (...args: unknown[]) => Promise<void>) => {
listeners.set(name, fn);
},
};
(connectionHandler as (socket: unknown) => void)(fakeSocket);
const batchSyncHandler = listeners.get("batch_sync");
assert.ok(batchSyncHandler);
let wsAck: Record<string, unknown> | null = null;
await batchSyncHandler(
[
{
eventId: "ws-flat-batch-1",
timestamp: "2026-11-19T12:11:00.000Z",
clientId: "phase03_batch_strip",
sensor: "pressure",
value: 101.2,
},
],
(ack: Record<string, unknown>) => {
wsAck = ack;
},
);
assert.ok(wsAck);
assert.equal((wsAck as Record<string, unknown>).success, true);
assert.equal(wsQueue.length, 1);
assert.deepEqual(JSON.parse(wsQueue[0].payload), {
sensor: "pressure",
value: 101.2,
});
});
test("R2 PHASE-03 BUG-12: DataBufferEngine.getLocalBufferSnapshot returns exact state.buffer contents for getPendingBufferedSummary", async () => {
const prevIngestionUrl = process.env.INGESTION_INTERNAL_URL;
process.env.INGESTION_INTERNAL_URL = "http://127.0.0.1:1";
try {
// Seed 1 unflushed RAM event directly in buffer while recentEvents contains a DIFFERENT newer flushed item
await DataBufferEngine.pushAsync({
clientId: "phase03_buf_exact",
eventId: "real-unflushed-1",
timestamp: "2026-11-19T13:00:00.000Z",
payload: JSON.stringify({ state: "unflushed" }),
});
// Simulate a flushed event sitting at the front of recentEvents (which old getRecentEvents slice would wrongly return!)
g.__dataStackBufferState!.recentEvents.unshift({
clientId: "other_flushed_client",
eventId: "already-flushed-99",
timestamp: "2026-11-19T13:05:00.000Z",
payload: JSON.stringify({ state: "already-flushed" }),
});
const snap = DataBufferEngine.getLocalBufferSnapshot();
assert.equal(snap.length, 1);
assert.equal(snap[0].eventId, "real-unflushed-1");
const res = await getEventsRoute(
new NextRequest("http://localhost:3000/api/events?bufferedOnly=true&limit=10", {
headers: { cookie: adminCookieHeader },
}),
);
assert.equal(res.status, 200);
const json = await res.json();
assert.equal(json.events.length, 1);
assert.equal(json.events[0].eventId, "real-unflushed-1");
assert.equal(json.bufferedSummary.total, 1);
assert.equal(json.bufferedSummary.buckets[0].clientId, "phase03_buf_exact");
} finally {
if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL;
else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl;
}
});
+850
View File
@@ -0,0 +1,850 @@
import assert from 'node:assert/strict';
import fs from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import { NextRequest } from 'next/server';
import {
countMatchingBufferedEvents,
DEFAULT_EDITABLE_SETTINGS,
describeSettingsBridgeSync,
extractContentDispositionFilename,
formatEventTimeLabel,
formatSessionExpiryLabel,
getEventUtcDate,
getUtcDateRangeForPreset,
mapBridgeStatusToSocketUiState,
matchesSelectedClientFolder,
normalizeEditableSettings,
parseEventTimestamp,
resolvePublicEndpoints,
SETTINGS_LIMITS,
settingsToNumericDrafts,
shouldRedirectProfileUnauthorized,
summarizeHttpBatchResult,
validateEditableSettingsDraft,
validateSingleNumericSetting,
} from '../lib/vconUiHelpers';
import {
createAdminSessionToken,
getAdminSessionSecret,
} from '../core/auth/adminSession';
import {
createInitialAdmin,
resetAdminStateForTests,
} from '../core/auth/adminUsers';
import { DataBufferEngine } from '../core/data/buffer';
import { SDKManager } from '../core/sdk/sdkManager';
import { CSVStorageManager } from '../core/Storage/csvStorage';
import { GET as profileGet, PUT as profilePut } from '../app/api/auth/profile/route';
import { DELETE as deleteClientRoute } from '../app/api/clients/[id]/route';
import { GET as eventsGet, POST as eventsPost } from '../app/api/events/route';
import { POST as settingsPost } from '../app/api/settings/route';
test('BUG-D01: normalizeEditableSettings enforces exact /api/settings backend bounds', () => {
const high = normalizeEditableSettings({
maxEventsPerSec: 95000,
maxPayloadSizeKb: 10240,
batchFlushThreshold: 50000,
batchFlushIntervalMs: 60000,
fileRotationMb: 2048,
enableIdempotency: false,
autoPruneDays: 3650,
});
assert.equal(high.maxEventsPerSec, 95000);
assert.equal(high.maxPayloadSizeKb, 10240);
assert.equal(high.batchFlushThreshold, 50000);
assert.equal(high.batchFlushIntervalMs, 60000);
assert.equal(high.fileRotationMb, 2048);
assert.equal(high.enableIdempotency, false);
assert.equal(high.autoPruneDays, 3650);
// Zero on positive-min fields clamps to min (1 / 100), while autoPruneDays keeps 0 ("never prune")
const low = normalizeEditableSettings({
maxEventsPerSec: 0,
maxPayloadSizeKb: 0,
batchFlushThreshold: 0,
batchFlushIntervalMs: 10,
fileRotationMb: 0,
autoPruneDays: 0,
});
assert.equal(low.maxEventsPerSec, SETTINGS_LIMITS.maxEventsPerSec.min);
assert.equal(low.maxPayloadSizeKb, SETTINGS_LIMITS.maxPayloadSizeKb.min);
assert.equal(low.batchFlushThreshold, SETTINGS_LIMITS.batchFlushThreshold.min);
assert.equal(low.batchFlushIntervalMs, SETTINGS_LIMITS.batchFlushIntervalMs.min);
assert.equal(low.fileRotationMb, SETTINGS_LIMITS.fileRotationMb.min);
assert.equal(low.autoPruneDays, 0);
// Non-finite falls back to defaults
const fallback = normalizeEditableSettings({
maxEventsPerSec: Number.NaN,
autoPruneDays: Number.NaN,
});
assert.equal(fallback.maxEventsPerSec, DEFAULT_EDITABLE_SETTINGS.maxEventsPerSec);
assert.equal(fallback.autoPruneDays, DEFAULT_EDITABLE_SETTINGS.autoPruneDays);
});
test('BUG-D01 & BUG-D04: bridgeSync feedback and socket UI state accurately distinguish online, auth_error, and offline', () => {
assert.equal(describeSettingsBridgeSync(undefined).level, 'success');
assert.equal(describeSettingsBridgeSync({ attempted: true, ok: true }).level, 'success');
const authFailure = describeSettingsBridgeSync({
attempted: true,
ok: false,
status: 401,
error: 'Unauthorized internal bridge token',
});
assert.equal(authFailure.level, 'error');
assert.match(authFailure.title, /rejected sync/i);
const offlineFailure = describeSettingsBridgeSync({
attempted: true,
ok: false,
error: 'connect ECONNREFUSED 127.0.0.1:4000',
});
assert.equal(offlineFailure.level, 'warning');
assert.match(offlineFailure.title, /unreachable/i);
assert.equal(mapBridgeStatusToSocketUiState('online'), 'online');
assert.equal(mapBridgeStatusToSocketUiState('auth_error'), 'auth_error');
assert.equal(mapBridgeStatusToSocketUiState('offline'), 'offline');
assert.equal(mapBridgeStatusToSocketUiState('error'), 'offline');
assert.equal(mapBridgeStatusToSocketUiState(undefined), 'offline');
});
test('BUG-D03: getUtcDateRangeForPreset computes deterministic UTC ranges across day boundaries', () => {
const ref = new Date('2026-11-19T02:15:00.000Z');
assert.deepEqual(getUtcDateRangeForPreset('today', ref), {
from: '2026-11-19',
to: '2026-11-19',
});
assert.deepEqual(getUtcDateRangeForPreset('7d', ref), {
from: '2026-11-13',
to: '2026-11-19',
});
assert.deepEqual(getUtcDateRangeForPreset('30d', ref), {
from: '2026-10-21',
to: '2026-11-19',
});
assert.deepEqual(getUtcDateRangeForPreset('all', ref), {
from: '',
to: '',
});
});
test('BUG-D03 & BUG-D05: timestamp formatting handles ISO & epoch-ms strings cleanly without Invalid Date', () => {
const isoParsed = parseEventTimestamp('2026-11-18T12:34:56.000Z');
assert.ok(isoParsed instanceof Date);
assert.equal(isoParsed.toISOString(), '2026-11-18T12:34:56.000Z');
const epochParsed = parseEventTimestamp('1700000000000');
assert.ok(epochParsed instanceof Date);
assert.equal(epochParsed.getTime(), 1700000000000);
assert.notEqual(formatEventTimeLabel('1700000000000'), 'Invalid Date');
assert.notEqual(formatEventTimeLabel('1700000000000'), '1700000000000');
assert.equal(parseEventTimestamp('not-a-date'), null);
assert.equal(formatEventTimeLabel('not-a-date'), 'not-a-date');
assert.equal(formatEventTimeLabel(''), '—');
});
test('BUG-D03 & BUG-D06: matchesSelectedClientFolder and Content-Disposition parsing handle sanitized folders and quoted filenames', () => {
assert.equal(matchesSelectedClientFolder('sensor_01', 'all'), true);
assert.equal(matchesSelectedClientFolder('sensor_01', 'sensor_01'), true);
assert.equal(matchesSelectedClientFolder('client_dev', 'client@dev'), true);
assert.equal(matchesSelectedClientFolder('other_client', 'sensor_01'), false);
assert.equal(
extractContentDispositionFilename(
'attachment; filename="datastack_export_sensor_01_2026-11-18.json"',
'fallback.json'
),
'datastack_export_sensor_01_2026-11-18.json'
);
assert.equal(
extractContentDispositionFilename(null, 'fallback.csv'),
'fallback.csv'
);
});
test('FORENSIC BUG-09: parseEventTimestamp and getEventUtcDate handle integer and fractional epoch-seconds with backend parity', () => {
const intSec = parseEventTimestamp('1735689600');
assert.ok(intSec instanceof Date);
assert.equal(intSec.toISOString(), '2025-01-01T00:00:00.000Z');
assert.equal(getEventUtcDate('1735689600'), '2025-01-01');
assert.notEqual(formatEventTimeLabel('1735689600'), '1735689600');
const floatSec = parseEventTimestamp('1735689600.25');
assert.ok(floatSec instanceof Date);
assert.equal(floatSec.toISOString(), '2025-01-01T00:00:00.250Z');
assert.equal(getEventUtcDate('1735689600.25'), '2025-01-01');
assert.notEqual(formatEventTimeLabel('1735689600.25'), '1735689600.25');
const msVal = parseEventTimestamp('1735689600500');
assert.ok(msVal instanceof Date);
assert.equal(msVal.toISOString(), '2025-01-01T00:00:00.500Z');
});
test('FORENSIC BUG-07: POST /api/events ingests batch and events arrays cleanly with accepted count and Simulator never falsifies counts', async () => {
const prevStorageDir = process.env.DATASTACK_STORAGE_DIR;
const origBaseStorageDir = CSVStorageManager.baseStorageDir;
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'datastack-p5-bug07-'));
process.env.DATASTACK_STORAGE_DIR = tmpDir;
CSVStorageManager.baseStorageDir = tmpDir;
await DataBufferEngine.shutdown();
DataBufferEngine.setRedisClientForTests(null);
try {
const client = await SDKManager.registerClientAsync({
clientId: 'p5_sim_client',
name: 'Phase 05 Simulator Client',
rateLimit: 500,
});
const ts = new Date().toISOString();
// 1. Canonical { clientId, batch } payload (10 individual events)
const batchItems = Array.from({ length: 10 }, (_, idx) => ({
eventId: `p5_batch_${Date.now()}_${idx}`,
timestamp: ts,
data: { batch_index: idx + 1, sensor: 's1' },
}));
const reqBatch = new NextRequest('http://localhost:3000/api/events', {
method: 'POST',
headers: {
'content-type': 'application/json',
authorization: `Bearer ${client.token}`,
},
body: JSON.stringify({ clientId: client.clientId, batch: batchItems }),
});
const resBatch = await eventsPost(reqBatch);
assert.equal(resBatch.status, 200);
const batchJson = await resBatch.json();
assert.equal(batchJson.success, true);
assert.equal(batchJson.accepted, 10);
assert.equal(batchJson.processed, 10);
assert.equal(batchJson.rejected, 0);
// 2. Alias { clientId, events } payload also ingests individual events (never 1 wrapper event)
const aliasItems = Array.from({ length: 4 }, (_, idx) => ({
eventId: `p5_alias_${Date.now()}_${idx}`,
timestamp: ts,
data: { alias_index: idx + 1 },
}));
const reqAlias = new NextRequest('http://localhost:3000/api/events', {
method: 'POST',
headers: {
'content-type': 'application/json',
authorization: `Bearer ${client.token}`,
},
body: JSON.stringify({ clientId: client.clientId, events: aliasItems }),
});
const resAlias = await eventsPost(reqAlias);
assert.equal(resAlias.status, 200);
const aliasJson = await resAlias.json();
assert.equal(aliasJson.accepted, 4);
assert.equal(aliasJson.processed, 4);
// 3. summarizeHttpBatchResult never falsifies accepted=10 when server omits accepted/processed
const fakeSingleRes = summarizeHttpBatchResult(
true,
{ success: true, eventId: 'evt_wrapper', event: {} },
10,
);
assert.equal(fakeSingleRes.accepted, 0);
assert.equal(fakeSingleRes.rejected, 10);
assert.equal(fakeSingleRes.status, 'Rejected');
assert.equal(fakeSingleRes.level, 'error');
const partialRes = summarizeHttpBatchResult(
true,
{ success: true, accepted: 7, processed: 7, rejected: 3, duplicates: 0 },
10,
);
assert.equal(partialRes.accepted, 7);
assert.equal(partialRes.rejected, 3);
assert.equal(partialRes.status, 'Rejected');
assert.equal(partialRes.level, 'warning');
const okRes = summarizeHttpBatchResult(true, batchJson, 10);
assert.equal(okRes.accepted, 10);
assert.equal(okRes.rejected, 0);
assert.equal(okRes.status, 'Accepted');
assert.equal(okRes.level, 'success');
// Verify Simulator source sends canonical `batch:` and uses summarizeHttpBatchResult
const simSrc = await fs.readFile(
path.join(process.cwd(), 'app', 'vcon', 'simulator', 'page.tsx'),
'utf-8',
);
assert.match(simSrc, /batch:\s*batchPayload/);
assert.match(simSrc, /summarizeHttpBatchResult\(/);
assert.doesNotMatch(simSrc, /res\.ok\s*\?\s*count\s*:\s*0/);
} finally {
await DataBufferEngine.flush();
await DataBufferEngine.shutdown();
CSVStorageManager.baseStorageDir = origBaseStorageDir;
if (prevStorageDir === undefined) delete process.env.DATASTACK_STORAGE_DIR;
else process.env.DATASTACK_STORAGE_DIR = prevStorageDir;
await fs.rm(tmpDir, { recursive: true, force: true });
}
});
test('FORENSIC BUG-08 & BUG-14: filtered buffered matching, post-flush zeroing, UTC-today attribution, and multi-client GET /api/events scan beyond 5 files', async () => {
const prevStorageDir = process.env.DATASTACK_STORAGE_DIR;
const prevIngestionUrl = process.env.INGESTION_INTERNAL_URL;
const origBaseStorageDir = CSVStorageManager.baseStorageDir;
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'datastack-p5-bug08-14-'));
process.env.DATASTACK_STORAGE_DIR = tmpDir;
CSVStorageManager.baseStorageDir = tmpDir;
process.env.INGESTION_INTERNAL_URL = 'http://127.0.0.1:1';
await DataBufferEngine.shutdown();
DataBufferEngine.setRedisClientForTests(null);
try {
const today = new Date().toISOString().slice(0, 10);
resetAdminStateForTests();
await createInitialAdmin('admin@datastack.dev', 'Phase05Pass123!');
const secret = getAdminSessionSecret()!;
const adminToken = await createAdminSessionToken(
'admin@datastack.dev',
secret,
Date.now() + 3600_000,
1,
);
// 1. Pure filtered buffer matcher prevents silent empty export downloads on non-matching client/date slices
const syntheticSummary = {
total: 8,
totalQueued: 8,
unattributedCount: 0,
truncated: false,
buckets: [
{ clientId: 'alpha_client', date: today, count: 3 },
{ clientId: 'alpha_client', date: '2025-01-01', count: 5 },
],
};
const otherClientMatch = countMatchingBufferedEvents(syntheticSummary, {
clientId: 'beta_client',
});
assert.equal(otherClientMatch.matchingCount, 0);
assert.equal(otherClientMatch.hasPotentialBuffered, false);
const outOfRangeMatch = countMatchingBufferedEvents(syntheticSummary, {
clientId: 'alpha_client',
fromDate: '2024-01-01',
toDate: '2024-12-31',
});
assert.equal(outOfRangeMatch.matchingCount, 0);
assert.equal(outOfRangeMatch.hasPotentialBuffered, false);
const todayOnlyMatch = countMatchingBufferedEvents(syntheticSummary, {
clientId: 'all',
fromDate: today,
toDate: today,
});
assert.equal(todayOnlyMatch.matchingCount, 3);
assert.equal(todayOnlyMatch.hasPotentialBuffered, true);
// 2. Write CSV partitions for 8 distinct clients (plus 6 extra dated files on client_01) to prove BUG-14 scan exceeds old 5-file cap
for (let d = 1; d <= 6; d += 1) {
await CSVStorageManager.appendBatch([
{
clientId: 'p5_client_01',
eventId: `evt_c1_old_${d}`,
timestamp: `2026-10-0${d}T10:00:00.000Z`,
payload: JSON.stringify({ n: d }),
},
]);
}
for (let i = 1; i <= 8; i += 1) {
const cid = `p5_client_0${i}`;
await CSVStorageManager.appendBatch([
{
clientId: cid,
eventId: `evt_${cid}_latest`,
timestamp: `${today}T12:00:0${i}.000Z`,
payload: JSON.stringify({ client: cid }),
},
]);
}
// 3. Push 2 UTC-today buffered events + 3 backdated buffered events into RAM buffer
await DataBufferEngine.pushBatchAsync([
{
clientId: 'p5_buf_client',
eventId: 'buf_today_1',
timestamp: `${today}T13:00:01.000Z`,
payload: '{"kind":"today"}',
},
{
clientId: 'p5_buf_client',
eventId: 'buf_today_2',
timestamp: `${today}T13:00:02.000Z`,
payload: '{"kind":"today"}',
},
{
clientId: 'p5_buf_client',
eventId: 'buf_old_1',
timestamp: '2025-02-01T08:00:00.000Z',
payload: '{"kind":"backdated"}',
},
{
clientId: 'p5_buf_client',
eventId: 'buf_old_2',
timestamp: '2025-02-01T08:00:01.000Z',
payload: '{"kind":"backdated"}',
},
{
clientId: 'p5_buf_client',
eventId: 'buf_old_3',
timestamp: '2025-02-01T08:00:02.000Z',
payload: '{"kind":"backdated"}',
},
]);
const reqAll = new NextRequest(
'http://localhost:3000/api/events?limit=25&includeBufferedSummary=true',
{
headers: { cookie: `datastack_admin_session=${adminToken}` },
},
);
const resAll = await eventsGet(reqAll);
assert.equal(resAll.status, 200);
const allJson = await resAll.json();
// All 8 disk clients must be represented in GET /api/events (past the old 5-file global cap)
const seenClients = new Set<string>(
(allJson.events as Array<{ clientId: string }>).map((e) => e.clientId),
);
for (let i = 1; i <= 8; i += 1) {
assert.ok(
seenClients.has(`p5_client_0${i}`),
`Expected p5_client_0${i} to be visible in GET /api/events multi-client scan`,
);
}
// bufferedSummary accurately separates 2 today events from 3 backdated events
assert.ok(allJson.bufferedSummary);
assert.equal(allJson.bufferedSummary.total, 5);
const todayBuffered = countMatchingBufferedEvents(allJson.bufferedSummary, {
clientId: 'all',
fromDate: today,
toDate: today,
});
assert.equal(todayBuffered.matchingCount, 2);
// After flush, GET /api/events?bufferedOnly=true drops to 0 (never double-counting flushed recentEvents)
await DataBufferEngine.flush();
const reqBufAfterFlush = new NextRequest(
'http://localhost:3000/api/events?bufferedOnly=true&limit=1',
{
headers: { cookie: `datastack_admin_session=${adminToken}` },
},
);
const resBufAfterFlush = await eventsGet(reqBufAfterFlush);
assert.equal(resBufAfterFlush.status, 200);
const bufAfterJson = await resBufAfterFlush.json();
assert.equal(bufAfterJson.bufferedSummary.total, 0);
assert.equal(bufAfterJson.bufferedSummary.buckets.length, 0);
// Verify ExportWorkspace & DashboardOverview refresh caches after export
const exportSrc = await fs.readFile(
path.join(process.cwd(), 'components', 'sections', 'ExportWorkspace.tsx'),
'utf-8',
);
assert.match(exportSrc, /countMatchingBufferedEvents\(/);
assert.match(exportSrc, /await refreshExportData\(\)/);
const dashSrc = await fs.readFile(
path.join(process.cwd(), 'components', 'sections', 'DashboardOverview.tsx'),
'utf-8',
);
assert.match(dashSrc, /includeBufferedSummary=true/);
assert.match(dashSrc, /await refresh\(\)/);
} finally {
await DataBufferEngine.flush();
await DataBufferEngine.shutdown();
CSVStorageManager.baseStorageDir = origBaseStorageDir;
if (prevStorageDir === undefined) delete process.env.DATASTACK_STORAGE_DIR;
else process.env.DATASTACK_STORAGE_DIR = prevStorageDir;
if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL;
else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl;
await fs.rm(tmpDir, { recursive: true, force: true });
}
});
test('FORENSIC BUG-10: /api/auth/profile returns explicit SESSION_EXPIRED vs INVALID_PASSWORD codes and ProfileWorkspace formats full expiry date+time', async () => {
const prevDbUrl = process.env.DATABASE_URL;
const prevDemoFlag = process.env.DATASTACK_ALLOW_DEMO_ADMIN;
delete process.env.DATABASE_URL;
process.env.DATASTACK_ALLOW_DEMO_ADMIN = 'false';
try {
resetAdminStateForTests();
await createInitialAdmin('p6-admin@datastack.dev', 'Phase06Pass123!');
const secret = getAdminSessionSecret()!;
const validToken = await createAdminSessionToken(
'p6-admin@datastack.dev',
secret,
Date.now() + 30 * 86_400_000,
1,
);
// 1. Missing/expired session on GET & PUT returns 401 with code: SESSION_EXPIRED
const unauthGet = await profileGet(
new NextRequest('http://localhost:3000/api/auth/profile'),
);
assert.equal(unauthGet.status, 401);
const unauthGetJson = await unauthGet.json();
assert.equal(unauthGetJson.code, 'SESSION_EXPIRED');
const unauthPut = await profilePut(
new NextRequest('http://localhost:3000/api/auth/profile', {
method: 'PUT',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
email: 'p6-admin@datastack.dev',
currentPassword: 'Phase06Pass123!',
}),
}),
);
assert.equal(unauthPut.status, 401);
const unauthPutJson = await unauthPut.json();
assert.equal(unauthPutJson.code, 'SESSION_EXPIRED');
assert.equal(shouldRedirectProfileUnauthorized(401, unauthPutJson), true);
// 2. Wrong currentPassword on valid session returns 401 with code: INVALID_PASSWORD (no redirect)
const wrongPassRes = await profilePut(
new NextRequest('http://localhost:3000/api/auth/profile', {
method: 'PUT',
headers: {
'content-type': 'application/json',
cookie: `datastack_admin_session=${validToken}`,
},
body: JSON.stringify({
email: 'p6-admin@datastack.dev',
currentPassword: 'WrongPassword999!',
}),
}),
);
assert.equal(wrongPassRes.status, 401);
const wrongPassJson = await wrongPassRes.json();
assert.equal(wrongPassJson.code, 'INVALID_PASSWORD');
assert.equal(shouldRedirectProfileUnauthorized(401, wrongPassJson), false);
// 3. Session expiry label includes calendar date (not time-only)
const expiryLabel = formatSessionExpiryLabel('2026-12-19T14:30:00.000Z');
assert.match(expiryLabel, /\d{1,2}/);
assert.ok(
expiryLabel.length > 5 && /[A-Za-z]|\d{1,2}\/\d{1,2}/.test(expiryLabel),
`Expected date+time in session expiry label, got: ${expiryLabel}`,
);
assert.equal(formatSessionExpiryLabel(''), 'Active');
// 4. Dead code `response.status === 401 && !currentPassword` is removed from ProfileWorkspace
const profileUiSrc = await fs.readFile(
path.join(process.cwd(), 'components', 'sections', 'ProfileWorkspace.tsx'),
'utf-8',
);
assert.doesNotMatch(profileUiSrc, /response\.status === 401 && !currentPassword/);
assert.match(profileUiSrc, /shouldRedirectProfileUnauthorized\(/);
assert.match(profileUiSrc, /formatSessionExpiryLabel\(/);
} finally {
if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
else process.env.DATABASE_URL = prevDbUrl;
if (prevDemoFlag === undefined) delete process.env.DATASTACK_ALLOW_DEMO_ADMIN;
else process.env.DATASTACK_ALLOW_DEMO_ADMIN = prevDemoFlag;
resetAdminStateForTests();
}
});
test('FORENSIC BUG-11: ClientsTable exposes ?purgeFiles=true opt-in checkbox on delete and omits unused Secret key row', async () => {
const clientsTableSrc = await fs.readFile(
path.join(process.cwd(), 'components', 'sections', 'ClientsTable.tsx'),
'utf-8',
);
assert.match(clientsTableSrc, /\?purgeFiles=true/);
assert.match(clientsTableSrc, /Also purge stored CSV files for this client/);
assert.doesNotMatch(clientsTableSrc, /label="Secret key"/);
});
test('FORENSIC BUG-12: Settings numeric drafts preserve empty inputs and report explicit validation errors without silent clamping', async () => {
const validDrafts = settingsToNumericDrafts(DEFAULT_EDITABLE_SETTINGS);
const okResult = validateEditableSettingsDraft(validDrafts, true);
assert.equal(okResult.valid, true);
assert.deepEqual(okResult.values, DEFAULT_EDITABLE_SETTINGS);
// Empty string returns explicit required error (never resets to 0)
const emptyCheck = validateSingleNumericSetting('maxEventsPerSec', '');
assert.equal(emptyCheck.valid, false);
assert.match(emptyCheck.error, /required/i);
// Out-of-bounds value returns explicit range error (never silently clamped)
const invalidDrafts = {
...validDrafts,
batchFlushIntervalMs: '50',
fileRotationMb: '99999',
};
const badResult = validateEditableSettingsDraft(invalidDrafts, false);
assert.equal(badResult.valid, false);
assert.equal(badResult.firstInvalidKey, 'batchFlushIntervalMs');
assert.match(badResult.errors.batchFlushIntervalMs || '', /100/);
assert.match(badResult.errors.fileRotationMb || '', /2,048|2048/);
// autoPruneDays = "0" ("never prune") remains explicitly valid
const zeroPrune = validateSingleNumericSetting('autoPruneDays', '0');
assert.equal(zeroPrune.valid, true);
if (zeroPrune.valid) assert.equal(zeroPrune.value, 0);
const settingsUiSrc = await fs.readFile(
path.join(process.cwd(), 'components', 'sections', 'SettingsWorkspace.tsx'),
'utf-8',
);
assert.match(settingsUiSrc, /validateEditableSettingsDraft\(/);
assert.match(settingsUiSrc, /Socket port \(PORT_SOCKET\)/);
assert.match(settingsUiSrc, /noValidate/);
});
test('FORENSIC BUG-13: resolvePublicEndpoints unifies Settings, Simulator, and SDK Guide URLs across direct dev and port-80 localhost proxies', async () => {
// 1. Direct Next.js dev on :3000 maps ingestion to :serverPort
const directDev = resolvePublicEndpoints(
{
protocol: 'http:',
hostname: 'localhost',
host: 'localhost:3000',
port: '3000',
origin: 'http://localhost:3000',
},
4000,
);
assert.deepEqual(directDev, {
apiUrl: 'http://localhost:3000',
serverUrl: 'http://localhost:4000',
webSocketUrl: 'ws://localhost:4000',
});
// 2. Reverse-proxied localhost on port 80 stays same-origin (no bogus :4000)
const localProxy = resolvePublicEndpoints(
{
protocol: 'http:',
hostname: 'localhost',
host: 'localhost',
port: '',
origin: 'http://localhost',
},
4000,
);
assert.deepEqual(localProxy, {
apiUrl: 'http://localhost',
serverUrl: 'http://localhost',
webSocketUrl: 'ws://localhost',
});
// 3. HTTPS production domain uses wss:// and same-origin
const prodHttps = resolvePublicEndpoints(
{
protocol: 'https:',
hostname: 'data.example.com',
host: 'data.example.com',
port: '',
origin: 'https://data.example.com',
},
4000,
);
assert.deepEqual(prodHttps, {
apiUrl: 'https://data.example.com',
serverUrl: 'https://data.example.com',
webSocketUrl: 'wss://data.example.com',
});
// All 3 UI pages import and use resolvePublicEndpoints
for (const rel of [
path.join('components', 'sections', 'SettingsWorkspace.tsx'),
path.join('app', 'vcon', 'sdk', 'page.tsx'),
path.join('app', 'vcon', 'simulator', 'page.tsx'),
]) {
const src = await fs.readFile(path.join(process.cwd(), rel), 'utf-8');
assert.match(src, /resolvePublicEndpoints\(/, `${rel} must use resolvePublicEndpoints`);
}
});
test('R2 PHASE-03 BUG-03: POST /api/settings returns normalized bridgeSync contract and describeSettingsBridgeSync handles both synced and attempted/ok shapes', async () => {
const prevDbUrl = process.env.DATABASE_URL;
const prevIngestionUrl = process.env.INGESTION_INTERNAL_URL;
delete process.env.DATABASE_URL;
process.env.INGESTION_INTERNAL_URL = 'http://127.0.0.1:1';
try {
resetAdminStateForTests();
await createInitialAdmin('p3-settings@datastack.dev', 'Phase03Pass123!');
const secret = getAdminSessionSecret()!;
const adminToken = await createAdminSessionToken(
'p3-settings@datastack.dev',
secret,
Date.now() + 3600_000,
1,
);
const res = await settingsPost(
new NextRequest('http://localhost:3000/api/settings', {
method: 'POST',
headers: {
'content-type': 'application/json',
cookie: `datastack_admin_session=${adminToken}`,
},
body: JSON.stringify({ maxEventsPerSec: 150 }),
}),
);
assert.equal(res.status, 200);
const json = await res.json();
assert.ok(json.bridgeSync, 'POST /api/settings must return bridgeSync status');
assert.equal(json.bridgeSync.attempted, true);
assert.equal(json.bridgeSync.ok, false);
assert.equal(json.bridgeSync.synced, false);
// describeSettingsBridgeSync must produce a warning for offline bridgeSync from backend
const uiFeedback = describeSettingsBridgeSync(json.bridgeSync);
assert.equal(uiFeedback.level, 'warning');
assert.match(uiFeedback.title, /unreachable/i);
// Also accepts legacy { synced } shape directly without attempted/ok
assert.equal(describeSettingsBridgeSync({ synced: true }).level, 'success');
assert.equal(
describeSettingsBridgeSync({ synced: false, status: 401 }).level,
'error',
);
assert.equal(
describeSettingsBridgeSync({ synced: false, error: 'timeout' }).level,
'warning',
);
const settingsUiSrc = await fs.readFile(
path.join(process.cwd(), 'components', 'sections', 'SettingsWorkspace.tsx'),
'utf-8',
);
assert.match(settingsUiSrc, /data\.bridgeSync\?\.ok \?\? data\.bridgeSync\?\.synced/);
} finally {
await DataBufferEngine.shutdown();
if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
else process.env.DATABASE_URL = prevDbUrl;
if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL;
else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl;
resetAdminStateForTests();
}
});
test('R2 PHASE-03 BUG-04: DELETE /api/clients/[id]?purgeFiles=true returns exact integer purgedFiles count', async () => {
const prevDbUrl = process.env.DATABASE_URL;
const prevStorageDir = process.env.DATASTACK_STORAGE_DIR;
const origBaseStorageDir = CSVStorageManager.baseStorageDir;
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'datastack-r2p3-bug04-'));
delete process.env.DATABASE_URL;
process.env.DATASTACK_STORAGE_DIR = tmpDir;
CSVStorageManager.baseStorageDir = tmpDir;
try {
resetAdminStateForTests();
await createInitialAdmin('p3-purge@datastack.dev', 'Phase03Pass123!');
const secret = getAdminSessionSecret()!;
const adminToken = await createAdminSessionToken(
'p3-purge@datastack.dev',
secret,
Date.now() + 3600_000,
1,
);
const clientPurge = await SDKManager.registerClientAsync({
clientId: 'p3_purge_me',
name: 'Purge Me',
rateLimit: 100,
});
await CSVStorageManager.appendBatch([
{
clientId: clientPurge.clientId,
eventId: 'purge-e1',
timestamp: '2026-11-18T10:00:00.000Z',
payload: '{"a":1}',
},
{
clientId: clientPurge.clientId,
eventId: 'purge-e2',
timestamp: '2026-11-19T10:00:00.000Z',
payload: '{"a":2}',
},
]);
const delPurgeRes = await deleteClientRoute(
new NextRequest(
`http://localhost:3000/api/clients/${clientPurge.clientId}?purgeFiles=true`,
{
method: 'DELETE',
headers: { cookie: `datastack_admin_session=${adminToken}` },
},
),
{ params: Promise.resolve({ id: clientPurge.clientId }) },
);
assert.equal(delPurgeRes.status, 200);
const delPurgeJson = await delPurgeRes.json();
assert.equal(delPurgeJson.success, true);
assert.equal(delPurgeJson.purgedFiles, 2);
// Deleting without ?purgeFiles=true returns purgedFiles: 0 and retains CSV files
const clientKeep = await SDKManager.registerClientAsync({
clientId: 'p3_keep_files',
name: 'Keep Files',
rateLimit: 100,
});
await CSVStorageManager.appendBatch([
{
clientId: clientKeep.clientId,
eventId: 'keep-e1',
timestamp: '2026-11-19T10:00:00.000Z',
payload: '{"keep":true}',
},
]);
const delKeepRes = await deleteClientRoute(
new NextRequest(
`http://localhost:3000/api/clients/${clientKeep.clientId}`,
{
method: 'DELETE',
headers: { cookie: `datastack_admin_session=${adminToken}` },
},
),
{ params: Promise.resolve({ id: clientKeep.clientId }) },
);
assert.equal(delKeepRes.status, 200);
const delKeepJson = await delKeepRes.json();
assert.equal(delKeepJson.purgedFiles, 0);
const remaining = await CSVStorageManager.listAllFilesAsync();
assert.equal(
remaining.filter((f) => f.clientId === clientKeep.clientId).length,
1,
);
} finally {
await DataBufferEngine.shutdown();
CSVStorageManager.baseStorageDir = origBaseStorageDir;
if (prevStorageDir === undefined) delete process.env.DATASTACK_STORAGE_DIR;
else process.env.DATASTACK_STORAGE_DIR = prevStorageDir;
if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
else process.env.DATABASE_URL = prevDbUrl;
await fs.rm(tmpDir, { recursive: true, force: true });
resetAdminStateForTests();
}
});
test('R2 PHASE-03 BUG-15: /vcon/sdk clears previous client JWT synchronously on switch and guards config copy while loadingToken is true', async () => {
const sdkSrc = await fs.readFile(
path.join(process.cwd(), 'app', 'vcon', 'sdk', 'page.tsx'),
'utf-8',
);
assert.match(sdkSrc, /setSelectedClientToken\(''\);\s*\n\s*setLoadingToken\(true\);/);
const disabledMatches = sdkSrc.match(/disabled=\{loadingToken\}/g) || [];
assert.ok(
disabledMatches.length >= 3,
`Expected Copy Token and both Copy Config buttons to be guarded with disabled={loadingToken}, found ${disabledMatches.length}`,
);
});
+114
View File
@@ -0,0 +1,114 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import test from 'node:test';
const ROOT = process.cwd();
test('BUG-E01 & BUG-E03: Canonical datastack and compat datastack-sdk packaging metadata, versions, and py.typed align', () => {
const pyproject = fs.readFileSync(
path.join(ROOT, 'sdk/python/datastack/pyproject.toml'),
'utf8',
);
const canonicalSetup = fs.readFileSync(
path.join(ROOT, 'sdk/python/datastack/setup.py'),
'utf8',
);
const compatSetup = fs.readFileSync(
path.join(ROOT, 'sdk-python/setup.py'),
'utf8',
);
const canonicalInit = fs.readFileSync(
path.join(ROOT, 'sdk/python/datastack/datastack/__init__.py'),
'utf8',
);
assert.match(canonicalInit, /__version__\s*=\s*["']2\.2\.0["']/);
assert.match(pyproject, /version\s*=\s*"2\.2\.0"/);
assert.match(canonicalSetup, /version="2\.2\.0"/);
assert.match(compatSetup, /version="2\.2\.0"/);
assert.equal(
fs.existsSync(path.join(ROOT, 'sdk/python/datastack/datastack/py.typed')),
true,
'PEP 561 py.typed marker must exist in datastack package',
);
assert.match(pyproject, /datastack\s*=\s*\["py\.typed"\]/);
assert.match(canonicalSetup, /package_data=\{"datastack": \["py\.typed"\]\}/);
for (const dep of [
'"python-socketio[client]>=5.8.0"',
'"requests>=2.28.0"',
'"websocket-client>=1.5.0"',
]) {
assert.ok(pyproject.includes(dep), `pyproject.toml missing ${dep}`);
assert.ok(canonicalSetup.includes(dep), `setup.py missing ${dep}`);
assert.ok(compatSetup.includes(dep), `sdk-python/setup.py missing ${dep}`);
}
});
test('BUG-E03 & BUG-E04: Root package.json engines and production start-all shutdown coordination are valid', () => {
const pkg = JSON.parse(
fs.readFileSync(path.join(ROOT, 'package.json'), 'utf8'),
) as {
engines?: { node?: string };
scripts?: Record<string, string>;
};
assert.equal(pkg.engines?.node, '>=20.0.0');
assert.equal(pkg.scripts?.start, 'node scripts/start-all.js');
assert.equal(pkg.scripts?.['start:web'], 'next start');
assert.equal(pkg.scripts?.['start:server'], 'node server/index.js');
const startAll = fs.readFileSync(
path.join(ROOT, 'scripts/start-all.js'),
'utf8',
);
assert.match(startAll, /markChildExited/);
assert.match(startAll, /exitedChildren >= 2/);
});
test('BUG-26: Legacy datastack-sdk declares datastack>=2.2.0 dependency and raises actionable ImportError', () => {
const compatSetup = fs.readFileSync(
path.join(ROOT, 'sdk-python/setup.py'),
'utf8',
);
const compatInit = fs.readFileSync(
path.join(ROOT, 'sdk-python/datastack_sdk/__init__.py'),
'utf8',
);
assert.ok(
compatSetup.includes('"datastack>=2.2.0"'),
'sdk-python/setup.py install_requires must include datastack>=2.2.0',
);
assert.match(
compatInit,
/raise ImportError\([\s\S]*datastack>=2\.2\.0[\s\S]*pip install \.\/sdk\/python\/datastack/,
'sdk-python/datastack_sdk/__init__.py must provide actionable install guidance if canonical datastack is missing',
);
});
test('BUG-27: start-all.js and dev-all.js avoid shell: true orphaning and allow >= 7000ms graceful shutdown flush', () => {
for (const relPath of ['scripts/start-all.js', 'scripts/dev-all.js']) {
const src = fs.readFileSync(path.join(ROOT, relPath), 'utf8');
assert.equal(
/shell:\s*true/.test(src),
false,
`${relPath} must NOT use shell: true (prevents orphaned child processes on SIGTERM)`,
);
assert.match(src, /shell:\s*false/, `${relPath} must explicitly set shell: false`);
assert.match(
src,
/spawn\(\s*process\.execPath/,
`${relPath} must spawn child services directly via process.execPath`,
);
assert.match(
src,
/GRACEFUL_SHUTDOWN_TIMEOUT_MS\s*=\s*7000/,
`${relPath} must wait at least 7000ms for graceful CSV/Redis shutdown flush`,
);
assert.match(src, /SIGKILL/, `${relPath} must escalate to SIGKILL after grace timeout`);
assert.match(src, /exitedChildren >= 2/, `${relPath} must wait for both child processes`);
}
});
+337
View File
@@ -9,6 +9,7 @@ Phase-05 Verification Tests (BUG-18):
offline disk spool replay against a local HTTP test server. offline disk spool replay against a local HTTP test server.
""" """
import ast
import base64 import base64
import json import json
import os import os
@@ -32,9 +33,11 @@ import datastack # noqa: E402
import datastack_sdk # noqa: E402 import datastack_sdk # noqa: E402
from datastack import ( # noqa: E402 from datastack import ( # noqa: E402
DataStack, DataStack,
DataStackAuthError,
DataStackClient, DataStackClient,
DataStackConfig, DataStackConfig,
DataStackConfigError, DataStackConfigError,
DataStackConnectionError,
DataStackOfflineSpoolError, DataStackOfflineSpoolError,
DiskSpooler, DiskSpooler,
HeartbeatMonitor, HeartbeatMonitor,
@@ -265,6 +268,340 @@ class Phase05PythonSdkTests(unittest.TestCase):
server.shutdown() server.shutdown()
server.server_close() server.server_close()
def test_client_id_normalization_and_invalid_numeric_env(self) -> None:
tok_norm = _make_jwt({"clientId": "sensor_01", "sub": "sensor_01"})
# Case/punctuation variations matching server SDKManager.normalizeClientId() must succeed
client = DataStack(
server_url="http://127.0.0.1:49999",
api_url="http://127.0.0.1:39999",
token=tok_norm,
client_id="Sensor.01",
auto_connect=False,
)
self.assertEqual(client.client_id, "sensor_01")
client.disconnect()
# Invalid numeric env var must raise DataStackConfigError instead of raw ValueError
old_retries = os.environ.get("DATASTACK_MAX_RETRIES")
try:
os.environ["DATASTACK_MAX_RETRIES"] = "not-an-int"
with self.assertRaises(DataStackConfigError):
load_config()
finally:
if old_retries is None:
os.environ.pop("DATASTACK_MAX_RETRIES", None)
else:
os.environ["DATASTACK_MAX_RETRIES"] = old_retries
def test_ragged_csv_all_rejected_send_file_and_partial_rl_recovery(self) -> None:
tok = _make_jwt({"clientId": "csv_worker", "sub": "csv_worker"})
mode = {"behavior": "reject_oversized"}
seen_records: List[Dict[str, Any]] = []
class EdgeHandler(BaseHTTPRequestHandler):
def do_POST(self) -> None: # noqa: N802
length = int(self.headers.get("Content-Length", "0"))
raw = self.rfile.read(length).decode("utf-8")
parsed = json.loads(raw) if raw else {}
items = parsed.get("records") or parsed.get("batch") or []
seen_records.extend(items)
if mode["behavior"] == "reject_oversized":
resp = {
"success": True,
"processed": 0,
"rejected": len(items),
"duplicates": 0,
"oversizedEventIds": [it.get("eventId") for it in items],
}
status = 200
else:
# First round processes first item, rate-limits second; second round disconnects/429
if len(items) > 1:
resp = {
"success": True,
"processed": 1,
"rejected": 1,
"duplicates": 0,
"rateLimitedEventIds": [items[1]["eventId"]],
}
status = 200
else:
resp = {
"success": False,
"code": "RATE_LIMIT_EXCEEDED",
"error": "Still limited",
}
status = 429
body = json.dumps(resp).encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, format: str, *args: Any) -> None:
return
server = ThreadingHTTPServer(("127.0.0.1", 0), EdgeHandler)
port = server.server_address[1]
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
with tempfile.TemporaryDirectory() as tmpdir:
# Ragged CSV with extra trailing column (used to crash sort_keys=True with None key)
ragged_csv = Path(tmpdir) / "ragged.csv"
ragged_csv.write_text(
"sensor,val\ns1,10,extra_trailing_col\n",
encoding="utf-8",
)
client = DataStack(
server_url="http://127.0.0.1:1",
api_url=f"http://127.0.0.1:{port}",
token=tok,
auto_connect=False,
request_timeout_sec=1.0,
max_retries=0,
retry_backoff_base_sec=0.1,
offline_spool_enabled=False,
)
res_file = client.send_file(ragged_csv)
self.assertFalse(res_file["success"])
self.assertEqual(res_file["processed"], 0)
self.assertEqual(res_file["rejected"], 1)
self.assertEqual(seen_records[0]["data"]["_extra"], ["extra_trailing_col"])
# Partial rate-limit where follow-up retry exhausts retries must preserve round-1 processed count
mode["behavior"] = "partial_rl"
spool_dir = Path(tmpdir) / "spool"
rl_client = DataStack(
server_url="http://127.0.0.1:1",
api_url=f"http://127.0.0.1:{port}",
token=tok,
auto_connect=False,
request_timeout_sec=1.0,
max_retries=1,
retry_backoff_base_sec=0.1,
offline_spool_dir=str(spool_dir),
)
rl_res = rl_client.send_batch(
[
{"eventId": "evt_ok_1", "data": {"n": 1}},
{"eventId": "evt_rl_2", "data": {"n": 2}},
]
)
self.assertTrue(rl_res["success"])
self.assertEqual(rl_res["processed"], 1)
self.assertEqual(rl_res["spooledCount"], 1)
self.assertEqual(rl_client.spooler.pending_count(), 1)
client.disconnect()
rl_client.disconnect()
finally:
server.shutdown()
server.server_close()
def test_heartbeat_stop_race_and_spooler_clean_ops(self) -> None:
hb = HeartbeatMonitor(interval_sec=5.0, recovery_callback=lambda: True)
hb.start()
hb.trigger_fallback("disconnect")
hb.stop("shutdown", join_timeout=0.5)
# Calling set_idle_connected after stop() must NOT resurrect IDLE state
hb.set_idle_connected()
self.assertEqual(hb.state, "STOPPED")
with tempfile.TemporaryDirectory() as tmpdir:
spool = DiskSpooler(spool_dir=tmpdir, max_size_mb=5, enabled=True)
spool.enqueue([{"eventId": "a"}, {"eventId": "b"}, {"eventId": "c"}])
self.assertEqual(spool.pending_count(), 3)
spool.ack_and_requeue_front(2, [{"eventId": "b_retry"}])
batch = spool.peek_batch(limit=10)
self.assertEqual([item["eventId"] for item in batch], ["b_retry", "c"])
self.assertFalse((Path(tmpdir) / "spool.jsonl.tmp").exists())
spool.ack_batch(2)
self.assertEqual(spool.pending_count(), 0)
def test_bug25_disconnects_leaked_socket_on_failed_ping_verify(self) -> None:
tok = _make_jwt({"clientId": "cli_bug25_sock"})
client = DataStack(
server_url="http://127.0.0.1:4000",
api_url="http://127.0.0.1:3000",
token=tok,
auto_connect=False,
enable_http_fallback=False,
offline_spool_enabled=False,
request_timeout_sec=0.5,
)
class FakeSocket:
def __init__(self) -> None:
self.connected = False
self.disconnect_calls = 0
def connect(self, *args: Any, **kwargs: Any) -> None:
self.connected = True
def disconnect(self) -> None:
self.disconnect_calls += 1
self.connected = False
fake_sio = FakeSocket()
client.sio = fake_sio
# Case 1: ping_verify returns False -> socket must be disconnected
client._verify_websocket_handshake = lambda timeout=None: False # type: ignore[method-assign]
self.assertFalse(client._connect_and_verify_websocket(wait_timeout=0.5))
self.assertFalse(client.is_connected)
self.assertFalse(fake_sio.connected)
self.assertEqual(fake_sio.disconnect_calls, 1)
# Case 2: ping_verify raises DataStackAuthError -> socket must be disconnected before re-raising
def raise_auth(timeout: Any = None) -> bool:
raise DataStackAuthError("CLIENT_REVOKED")
client._verify_websocket_handshake = raise_auth # type: ignore[method-assign]
with self.assertRaises(DataStackAuthError):
client._connect_and_verify_websocket(wait_timeout=0.5)
self.assertFalse(client.is_connected)
self.assertFalse(fake_sio.connected)
self.assertEqual(fake_sio.disconnect_calls, 2)
client.disconnect()
def test_bug25_strict_auto_connect_and_context_raise_connection_error(self) -> None:
tok = _make_jwt({"clientId": "cli_bug25_strict"})
# auto_connect=True with both fallback and offline spool disabled must raise DataStackConnectionError
with self.assertRaises(DataStackConnectionError):
DataStack(
server_url="http://127.0.0.1:1",
api_url="http://127.0.0.1:1",
token=tok,
auto_connect=True,
enable_http_fallback=False,
offline_spool_enabled=False,
request_timeout_sec=0.2,
)
# Context manager __enter__ with both fallback and offline spool disabled must also raise
strict_client = DataStack(
server_url="http://127.0.0.1:1",
api_url="http://127.0.0.1:1",
token=tok,
auto_connect=False,
enable_http_fallback=False,
offline_spool_enabled=False,
request_timeout_sec=0.2,
)
with self.assertRaises(DataStackConnectionError):
with strict_client:
pass
self.assertEqual(strict_client.heartbeat.state, "STOPPED")
def test_r2_phase03_bug11_flat_batch_and_file_strip_envelope_keys(self) -> None:
tok = _make_jwt({"clientId": "cli_bug11_strip"})
captured_batches: List[List[Dict[str, Any]]] = []
with tempfile.TemporaryDirectory() as tmpdir:
client = DataStack(
server_url="http://127.0.0.1:1",
api_url="http://127.0.0.1:1",
token=tok,
auto_connect=False,
offline_spool_enabled=False,
)
def fake_execute(**kwargs: Any) -> Dict[str, Any]:
ws_payload = kwargs.get("ws_payload")
if isinstance(ws_payload, list):
captured_batches.append(ws_payload)
return {
"success": True,
"processed": len(ws_payload),
"rejected": 0,
"transport": "HTTP",
}
return {"success": True, "processed": 0, "rejected": 0}
client._execute_with_retry_and_fallback = fake_execute # type: ignore[method-assign]
res_batch = client.send_batch(
[
{
"eventId": "py-flat-1",
"timestamp": "2026-11-19T12:00:00Z",
"clientId": "cli_bug11_strip",
"sensor": "temp",
"value": 24.5,
},
{
"eventId": "py-env-2",
"timestamp": "2026-11-19T12:00:01Z",
"data": {"sensor": "humidity", "value": 55},
},
]
)
self.assertTrue(res_batch["success"])
self.assertEqual(len(captured_batches), 1)
self.assertEqual(captured_batches[0][0]["eventId"], "py-flat-1")
self.assertEqual(
captured_batches[0][0]["data"],
{"sensor": "temp", "value": 24.5},
)
self.assertEqual(
captured_batches[0][1]["data"],
{"sensor": "humidity", "value": 55},
)
csv_path = Path(tmpdir) / "flat_records.csv"
csv_path.write_text(
"eventId,timestamp,clientId,sensor,reading\n"
"file-flat-1,2026-11-19T13:00:00Z,cli_bug11_strip,pressure,101.3\n",
encoding="utf-8",
)
res_file = client.send_file(csv_path)
self.assertTrue(res_file["success"])
self.assertEqual(len(captured_batches), 2)
self.assertEqual(captured_batches[1][0]["eventId"], "file-flat-1")
self.assertEqual(
captured_batches[1][0]["data"],
{"sensor": "pressure", "reading": "101.3"},
)
def test_r2_phase04_bug14_python38_ast_typing_compatibility(self) -> None:
pkg_dir = CANONICAL_PKG_ROOT / "datastack"
py_files = sorted(pkg_dir.glob("*.py"))
self.assertGreater(len(py_files), 0)
pep585_builtins = {"tuple", "list", "dict", "set", "frozenset", "type"}
for py_file in py_files:
src = py_file.read_text(encoding="utf-8")
tree = ast.parse(src, filename=str(py_file), feature_version=(3, 8))
has_future_annotations = any(
isinstance(node, ast.ImportFrom)
and node.module == "__future__"
and any(alias.name == "annotations" for alias in node.names)
for node in tree.body
)
if py_file.name == "client.py":
self.assertTrue(
has_future_annotations,
"client.py must declare `from __future__ import annotations`",
)
for node in ast.walk(tree):
if (
isinstance(node, ast.Subscript)
and isinstance(node.value, ast.Name)
and node.value.id in pep585_builtins
):
self.fail(
f"{py_file.name}:{node.lineno} uses Python 3.9+ PEP 585 generic `{node.value.id}[...]`"
)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main(verbosity=2) unittest.main(verbosity=2)
+36
View File
@@ -45,6 +45,8 @@ test('BUG-25: PWA manifest, icons (192/512/maskable/apple-touch), and Service Wo
assert.match(swContent, /request\.method !== ['"]GET['"]/); assert.match(swContent, /request\.method !== ['"]GET['"]/);
assert.match(swContent, /\/api\//); assert.match(swContent, /\/api\//);
assert.match(swContent, /\/socket\.io\//); assert.match(swContent, /\/socket\.io\//);
assert.match(swContent, /\/vcon\//);
assert.match(swContent, /CLEAR_AUTH_CACHE/);
assert.match(swContent, /new Response\(/); assert.match(swContent, /new Response\(/);
const layoutContent = fs.readFileSync(path.join(ROOT, 'app/layout.tsx'), 'utf8'); const layoutContent = fs.readFileSync(path.join(ROOT, 'app/layout.tsx'), 'utf8');
@@ -79,3 +81,37 @@ test('BUG-17: Landing/Navbar/Footer/404 navigation uses root-relative anchors an
assert.match(notFoundContent, /href=["']\/["']/); assert.match(notFoundContent, /href=["']\/["']/);
assert.match(notFoundContent, /href=["']\/vcon\/dashboard["']/); assert.match(notFoundContent, /href=["']\/vcon\/dashboard["']/);
}); });
test('FORENSIC BUG-15: RootLayout injects pre-hydration theme bootstrap script and AdminSetup supports theme restoration & toggle', () => {
const layoutContent = fs.readFileSync(path.join(ROOT, 'app/layout.tsx'), 'utf8');
assert.match(layoutContent, /<html[^>]*suppressHydrationWarning/);
assert.match(layoutContent, /datastack-theme/);
assert.match(layoutContent, /<head>/);
const setupContent = fs.readFileSync(
path.join(ROOT, 'components/sections/AdminSetup.tsx'),
'utf8',
);
assert.match(setupContent, /datastack-theme/);
assert.match(setupContent, /Toggle theme/);
assert.match(setupContent, /document\.documentElement\.classList\.toggle\(['"]light['"]/);
});
test('FORENSIC BUG-16: Service Worker serves uncached HTML offline fallback shell for /vcon/dashboard PWA navigations', () => {
const manifest = JSON.parse(
fs.readFileSync(path.join(ROOT, 'public/manifest.json'), 'utf8'),
);
assert.equal(manifest.start_url, '/vcon/dashboard');
const swContent = fs.readFileSync(path.join(ROOT, 'public/sw.js'), 'utf8');
assert.match(swContent, /buildOfflineShellResponse/);
assert.match(swContent, /request\.mode !== ['"]navigate['"]/);
assert.match(swContent, /text\/html; charset=utf-8/);
assert.match(swContent, /Cache-Control['"]:\s*['"]no-store['"]/);
// Ensure /vcon/* branch never calls cache.put on authenticated admin HTML
const vconBranch = swContent.match(
/if \(url\.pathname === '\/vcon' \|\| url\.pathname\.startsWith\('\/vcon\/'\)\) \{[\s\S]*?return;\s*\}/,
);
assert.ok(vconBranch, 'Expected dedicated /vcon/* navigation fallback branch in public/sw.js');
assert.doesNotMatch(vconBranch[0], /cache\.put/);
});
+285 -6
View File
@@ -1,5 +1,6 @@
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { existsSync, mkdirSync, readFileSync, rmSync, utimesSync, writeFileSync } from "node:fs"; import { existsSync, mkdirSync, readFileSync, rmSync, utimesSync, writeFileSync } from "node:fs";
import { createRequire } from "node:module";
import { resolve } from "node:path"; import { resolve } from "node:path";
import test, { after } from "node:test"; import test, { after } from "node:test";
import { NextRequest } from "next/server"; import { NextRequest } from "next/server";
@@ -7,6 +8,8 @@ import { GET as getHealthRoute } from "../app/api/health/route";
import { SettingsManager } from "../core/adminSettings/settingsManager"; import { SettingsManager } from "../core/adminSettings/settingsManager";
import { DataBufferEngine, type RedisBufferClient } from "../core/data/buffer"; import { DataBufferEngine, type RedisBufferClient } from "../core/data/buffer";
const require = createRequire(import.meta.url);
const g = globalThis as typeof globalThis & { const g = globalThis as typeof globalThis & {
__dataStackAdminSettings?: unknown; __dataStackAdminSettings?: unknown;
__dataStackSettingsHydrated?: boolean; __dataStackSettingsHydrated?: boolean;
@@ -19,6 +22,8 @@ const g = globalThis as typeof globalThis & {
const prevDbUrl = process.env.DATABASE_URL; const prevDbUrl = process.env.DATABASE_URL;
const prevRedisUrl = process.env.REDIS_URL; const prevRedisUrl = process.env.REDIS_URL;
const prevDisableListen = process.env.DATASTACK_DISABLE_LISTEN;
process.env.DATASTACK_DISABLE_LISTEN = "1";
after(async () => { after(async () => {
DataBufferEngine.setRedisClientForTests(undefined); DataBufferEngine.setRedisClientForTests(undefined);
@@ -39,6 +44,9 @@ after(async () => {
if (prevRedisUrl === undefined) delete process.env.REDIS_URL; if (prevRedisUrl === undefined) delete process.env.REDIS_URL;
else process.env.REDIS_URL = prevRedisUrl; else process.env.REDIS_URL = prevRedisUrl;
if (prevDisableListen === undefined) delete process.env.DATASTACK_DISABLE_LISTEN;
else process.env.DATASTACK_DISABLE_LISTEN = prevDisableListen;
const testDirs = [ const testDirs = [
resolve(process.cwd(), "storage", "data", "phase07_prune_guard"), resolve(process.cwd(), "storage", "data", "phase07_prune_guard"),
resolve(process.cwd(), "storage", "data", "phase07_shutdown_flush"), resolve(process.cwd(), "storage", "data", "phase07_shutdown_flush"),
@@ -87,15 +95,20 @@ test("DEPLOY-BLK-03 & 05: Dockerfiles guard recursive chown on /app/storage/data
assert.ok(lockJson.packages && Object.keys(lockJson.packages).length > 10); assert.ok(lockJson.packages && Object.keys(lockJson.packages).length > 10);
}); });
test("DEPLOY-BLK-04: Both Nginx configs proxy = /health to socketio_upstream", () => { test("DEPLOY-BLK-04: Both Nginx configs proxy = /health to ingestion upstream", () => {
for (const relPath of ["docker/coolify-nginx.conf", "nginx/default.conf"]) { const defaultConf = readFileSync(resolve(process.cwd(), "nginx/default.conf"), "utf8");
const conf = readFileSync(resolve(process.cwd(), relPath), "utf8");
assert.match( assert.match(
conf, defaultConf,
/location\s*=\s*\/health\s*\{[\s\S]*?proxy_pass\s+http:\/\/socketio_upstream;/, /location\s*=\s*\/health\s*\{[\s\S]*?proxy_pass\s+http:\/\/socketio_upstream;/,
`${relPath} must route location = /health to socketio_upstream`, "nginx/default.conf must route location = /health to socketio_upstream",
);
const coolifyConf = readFileSync(resolve(process.cwd(), "docker/coolify-nginx.conf"), "utf8");
assert.match(
coolifyConf,
/location\s*=\s*\/health\s*\{[\s\S]*?proxy_pass\s+\$socketio_upstream;/,
"docker/coolify-nginx.conf must route location = /health to $socketio_upstream",
); );
}
}); });
test("DEPLOY-BLK-03 docs: manifest.md and README.md align volume mount path (/app/storage) and include sdk/python/datastack", () => { test("DEPLOY-BLK-03 docs: manifest.md and README.md align volume mount path (/app/storage) and include sdk/python/datastack", () => {
@@ -197,3 +210,269 @@ test("DEPLOY-BLK-06 shutdown & health: DataBufferEngine.flushAndShutdownOnSignal
assert.equal(json.status, "online"); assert.equal(json.status, "online");
assert.equal(json.bufferSize, 7); assert.equal(json.bufferSize, 7);
}); });
test("BUG-28: nginx/default.conf aligns Socket.IO directives and server/index.js enforces configurable DATASTACK_CORS_ORIGINS", () => {
const nginxDefault = readFileSync(resolve(process.cwd(), "nginx", "default.conf"), "utf8");
assert.match(
nginxDefault,
/location \/socket\.io\/ \{[\s\S]*proxy_read_timeout 86400s;[\s\S]*proxy_send_timeout 86400s;[\s\S]*proxy_buffering off;/,
"nginx/default.conf must disable proxy_buffering and set 86400s read/send timeouts on /socket.io/",
);
const compose = readFileSync(resolve(process.cwd(), "docker", "docker-compose.yml"), "utf8");
assert.match(compose, /DATASTACK_CORS_ORIGINS:\s*"\$\{DATASTACK_CORS_ORIGINS:-\}"/);
const envExample = readFileSync(resolve(process.cwd(), ".env.example"), "utf8");
assert.match(envExample, /DATASTACK_CORS_ORIGINS=/);
const serverSrc = readFileSync(resolve(process.cwd(), "server", "index.js"), "utf8");
assert.equal(
serverSrc.includes("origin: '*'"),
false,
"server/index.js must not hardcode wildcard origin: '*'",
);
process.env.DATASTACK_DISABLE_LISTEN = "1";
const ingestionServer = require("../server/index.js") as {
isOriginAllowedByCors: (origin: string | undefined, raw?: string) => boolean;
corsOriginDelegate: (
origin: string | undefined,
cb: (err: Error | null, allow: boolean) => void,
) => void;
stopTimers: () => void;
};
try {
// Default / empty / wildcard allows any origin
assert.equal(ingestionServer.isOriginAllowedByCors("https://any.example.com", ""), true);
assert.equal(ingestionServer.isOriginAllowedByCors("https://any.example.com", "*"), true);
// Non-browser / SDK calls (no Origin header) remain allowed even with strict allowlist
const strictList = "https://console.example.com, https://admin.example.com/";
assert.equal(ingestionServer.isOriginAllowedByCors(undefined, strictList), true);
assert.equal(ingestionServer.isOriginAllowedByCors("", strictList), true);
// Allowed origins (with trailing slash normalization) pass
assert.equal(ingestionServer.isOriginAllowedByCors("https://console.example.com", strictList), true);
assert.equal(ingestionServer.isOriginAllowedByCors("https://console.example.com/", strictList), true);
assert.equal(ingestionServer.isOriginAllowedByCors("https://admin.example.com", strictList), true);
// Disallowed browser origins are rejected cleanly
assert.equal(ingestionServer.isOriginAllowedByCors("https://evil.example.com", strictList), false);
// Delegate callback respects live process.env.DATASTACK_CORS_ORIGINS without throwing
const prevCors = process.env.DATASTACK_CORS_ORIGINS;
try {
process.env.DATASTACK_CORS_ORIGINS = "https://allowed.datastack.dev";
let delegateErr: Error | null = null;
let delegateAllowed = true;
ingestionServer.corsOriginDelegate("https://blocked.datastack.dev", (err, allow) => {
delegateErr = err;
delegateAllowed = allow;
});
assert.equal(delegateErr, null);
assert.equal(delegateAllowed, false);
ingestionServer.corsOriginDelegate("https://allowed.datastack.dev", (err, allow) => {
delegateErr = err;
delegateAllowed = allow;
});
assert.equal(delegateErr, null);
assert.equal(delegateAllowed, true);
} finally {
if (prevCors === undefined) delete process.env.DATASTACK_CORS_ORIGINS;
else process.env.DATASTACK_CORS_ORIGINS = prevCors;
}
} finally {
ingestionServer.stopTimers();
}
});
test("R2 PHASE-04 BUG-16 & BUG-17: Nginx disables buffering on /api/export, next.config enforces ESLint during builds, and package.json prunes unused template deps", () => {
const nginxDefault = readFileSync(resolve(process.cwd(), "nginx/default.conf"), "utf8");
assert.match(
nginxDefault,
/location\s*=\s*\/api\/export\s*\{[\s\S]*?proxy_pass\s+http:\/\/nextjs_upstream;[\s\S]*?proxy_buffering\s+off;[\s\S]*?proxy_read_timeout\s+300s;/,
"nginx/default.conf must disable proxy_buffering and extend proxy_read_timeout on location = /api/export",
);
const coolifyConf = readFileSync(resolve(process.cwd(), "docker/coolify-nginx.conf"), "utf8");
assert.match(
coolifyConf,
/location\s*=\s*\/api\/export\s*\{[\s\S]*?proxy_pass\s+\$nextjs_upstream;[\s\S]*?proxy_buffering\s+off;[\s\S]*?proxy_read_timeout\s+300s;/,
"docker/coolify-nginx.conf must disable proxy_buffering and extend proxy_read_timeout on location = /api/export",
);
const nextConfig = readFileSync(resolve(process.cwd(), "next.config.ts"), "utf8");
assert.equal(
/ignoreDuringBuilds\s*:\s*true/.test(nextConfig),
false,
"next.config.ts must not ignore ESLint errors during builds",
);
const pkg = JSON.parse(readFileSync(resolve(process.cwd(), "package.json"), "utf8")) as {
name?: string;
dependencies?: Record<string, string>;
devDependencies?: Record<string, string>;
};
assert.equal(pkg.name, "datastack");
assert.equal("@google/genai" in (pkg.dependencies ?? {}), false);
assert.equal("firebase-tools" in (pkg.devDependencies ?? {}), false);
});
test("DEPLOY-BUG-01 & 05: coolify-nginx.conf resolves Docker DNS dynamically and Coolify docs align Base Directory /", () => {
const coolifyConf = readFileSync(resolve(process.cwd(), "docker", "coolify-nginx.conf"), "utf8");
assert.match(coolifyConf, /resolver\s+127\.0\.0\.11\s+valid=10s\s+ipv6=off;/);
assert.match(coolifyConf, /set\s+\$nextjs_upstream\s+"http:\/\/web:3000";/);
assert.match(coolifyConf, /set\s+\$socketio_upstream\s+"http:\/\/ingestion:4000";/);
assert.equal(/\bupstream\s+\w+\s*\{/.test(coolifyConf), false, "coolify-nginx.conf must not define static upstream blocks");
const compose = readFileSync(resolve(process.cwd(), "docker", "docker-compose.yml"), "utf8");
const graceMatches = compose.match(/^\s+stop_grace_period:\s*20s/gm) ?? [];
assert.equal(graceMatches.length, 2, "web and ingestion must set stop_grace_period: 20s");
const installDoc = readFileSync(resolve(process.cwd(), "docs", "users", "installation.md"), "utf8");
const readmeDoc = readFileSync(resolve(process.cwd(), "README.md"), "utf8");
assert.match(installDoc, /\*\*Base Directory\*\*\s+to\s+`\/`/);
assert.match(readmeDoc, /\*\*Base Directory\*\*\s+to\s+`\/`\s+and\s+\*\*Docker Compose Location\*\*\s+to\s+`\/docker\/docker-compose\.yml`/);
});
test("DEPLOY-BUG-02, 03 & 04: Port 4000 memoizes schema DDLs, hydrates rotated JWT before listening, and drains buffers gracefully", async () => {
const serverSrc = readFileSync(resolve(process.cwd(), "server", "index.js"), "utf8");
assert.match(
serverSrc,
/async function startIngestionServer\(\)[\s\S]*syncSettingsFromDatabase\(\)[\s\S]*server\.listen\(PORT/,
"startIngestionServer must await bounded syncSettingsFromDatabase() before server.listen(PORT)",
);
assert.match(
serverSrc,
/async function gracefulShutdown\([\s\S]*await closeServerGracefully\(\);[\s\S]*Promise\.allSettled\(\[flushBuffer\(\),\s*flushPendingClientActivity\(\)\]\)/,
"gracefulShutdown must close HTTP/Socket.IO ingress before flushing buffers",
);
process.env.DATASTACK_DISABLE_LISTEN = "1";
const ingestionServer = require("../server/index.js") as {
runtimeConfig: { jwtSecret: string };
bufferQueue: Array<{ clientId: string; eventId: string; timestamp: string; payload: string }>;
setDbPoolForTests: (pool: unknown) => void;
setRedisClientForTests: (client: unknown) => void;
ensureDatabaseSchema: () => Promise<boolean>;
syncSettingsFromDatabase: () => Promise<boolean>;
gracefulShutdown: (signal: string, options?: { exit?: boolean }) => Promise<void>;
stopTimers: () => void;
};
const prevJwt = ingestionServer.runtimeConfig.jwtSecret;
let ddlCount = 0;
let selectCount = 0;
let failDdl = true;
let poolEnded = false;
const rotatedSecret = "rotated-production-jwt-secret-value-over-32-chars";
const mockPool = {
async query(sql: string) {
const normalized = sql.trim().toUpperCase();
if (normalized.startsWith("CREATE TABLE") || normalized.startsWith("ALTER TABLE")) {
ddlCount += 1;
if (failDdl) {
throw new Error("transient startup PG failure");
}
return { rowCount: 0, rows: [] };
}
if (normalized.includes("FROM ADMIN_SETTINGS")) {
selectCount += 1;
return {
rowCount: 1,
rows: [
{
max_events_per_sec: 250,
max_payload_size_kb: 128,
batch_flush_threshold: 100,
batch_flush_interval_ms: 2000,
file_rotation_mb: 50,
enable_idempotency: true,
auto_prune_days: 45,
jwt_secret: rotatedSecret,
},
],
};
}
return { rowCount: 0, rows: [] };
},
async end() {
poolEnded = true;
},
};
let redisQuitCalled = false;
const mockRedis = {
status: "ready",
async rpush() {
return 0;
},
async lpush() {
return 0;
},
async ltrim() {
return "OK";
},
async lrange() {
return [];
},
async llen() {
return 0;
},
async set() {
return "OK";
},
async eval() {
return 1;
},
async sismember() {
return 0;
},
async quit() {
redisQuitCalled = true;
return "OK";
},
};
try {
ingestionServer.setDbPoolForTests(mockPool);
// 1. Transient DDL error must return false and allow retry
const failedSync = await ingestionServer.syncSettingsFromDatabase();
assert.equal(failedSync, false);
assert.equal(ddlCount, 1);
// 2. Recovery runs 8 DDLs once and hydrates rotated jwt_secret; repeated calls do zero extra DDLs
failDdl = false;
ddlCount = 0;
assert.equal(await ingestionServer.syncSettingsFromDatabase(), true);
assert.equal(ddlCount, 8);
assert.equal(selectCount, 1);
assert.equal(ingestionServer.runtimeConfig.jwtSecret, rotatedSecret);
assert.equal(await ingestionServer.syncSettingsFromDatabase(), true);
assert.equal(await ingestionServer.syncSettingsFromDatabase(), true);
assert.equal(ddlCount, 8, "Repeated 3s sync ticks must NOT re-run CREATE/ALTER DDL statements");
assert.equal(selectCount, 3);
// 3. Graceful shutdown flushes buffered events, quits Redis, and closes PostgreSQL pool cleanly
ingestionServer.setRedisClientForTests(mockRedis);
ingestionServer.bufferQueue.push({
clientId: "phase07_shutdown_flush",
eventId: `evt-graceful-${Date.now()}`,
timestamp: new Date().toISOString(),
payload: JSON.stringify({ ok: true }),
});
await ingestionServer.gracefulShutdown("SIGTERM", { exit: false });
assert.equal(ingestionServer.bufferQueue.length, 0, "gracefulShutdown must drain RAM queue to CSV");
assert.equal(redisQuitCalled, true, "gracefulShutdown must invoke redis.quit()");
assert.equal(poolEnded, true, "gracefulShutdown must close PostgreSQL pool");
} finally {
ingestionServer.runtimeConfig.jwtSecret = prevJwt;
ingestionServer.setDbPoolForTests(null);
ingestionServer.setRedisClientForTests(undefined);
ingestionServer.stopTimers();
}
});