feat(auth): harden session management and security
- Implement server-side session revocation on logout - Add IP extraction utility and admin setup error handling - Improve error codes and response consistency for auth endpoints - Refine landing page redirect logic to handle transient DB outages - Update deployment documentation and configuration defaults
This commit is contained in:
1 parent
935f12a1e9
commit
89e4d68690
81 files changed
+11530
-6916
No files matched your search
+7
-1
@@ -18,6 +18,12 @@ DATASTACK_ADMIN_EMAIL=admin@datastack.dev
|
||||
# Shared Redis Buffer (optional; enables cross-process buffering & distributed flush locks)
|
||||
# REDIS_URL=redis://127.0.0.1:6379
|
||||
|
||||
# Shared Partitioned CSV Storage Directory (optional; defaults to ./storage/data)
|
||||
# Set an absolute path so Port 3000 (Next.js) and Port 4000 (Ingestion) always share one storage root
|
||||
# DATASTACK_STORAGE_DIR=/var/lib/datastack/storage/data
|
||||
|
||||
# Service Networking
|
||||
PORT_SOCKET=4000
|
||||
INGESTION_INTERNAL_URL=http://127.0.0.1:4000
|
||||
INGESTION_INTERNAL_URL=http://127.0.0.1:4000
|
||||
# Optional comma-separated browser CORS origin allowlist for Port 4000 ingestion/Socket.IO (defaults to wildcard if unset)
|
||||
# DATASTACK_CORS_ORIGINS=https://console.example.com,https://admin.example.com
|
||||
@@ -1,7 +1,7 @@
|
||||
# MEMORY.md — DataStack Cross-IDE Persistent Project Memory
|
||||
|
||||
> **Last Synchronized**: 2026-11-18 11:00:00 UTC
|
||||
> **Status**: Fresh Source-Code Forensic Audit Complete (`notes/forensic-audit-report.md` — 27 verified findings across 5 categories; awaiting approval to plan incremental remediation phases)
|
||||
> **Last Synchronized**: 2026-11-19
|
||||
> **Status**: **Production & Deployment Hardening Complete (`BASELINE-2026-11-19-R2` + Deployment Hardening) — `4 / 4` R2 Phases (`18/18` bugs) + `5/5` Docker/Coolify Redeploy Fixes (`DEPLOY-BUG-01`–`05`) Verified (`85/85` TS tests, `10/10` Python tests, ESLint-enabled build & lint `0` errors / `0` warnings).**
|
||||
> **Scope Rule**: Maintain this file accurately across VS Code, Google Antigravity, Cursor, and any other IDE after every modification, feature update, or bug fix (see `.ai-rules` and `AGENTS.md`).
|
||||
|
||||
---
|
||||
@@ -69,9 +69,13 @@ datastack/
|
||||
│ ├── dashboard/ clients/ data/ export/ login/ profile/ sdk/ settings/ setup/ simulator/ storage/
|
||||
├── components/
|
||||
│ ├── ui/ # Single-source-of-truth UI primitives (Button, Input, Table, Modal, Panel, etc.)
|
||||
│ ├── layout/ # MainLayout, Navbar, Sidebar, PageShell, AdminPageHeader, Footer
|
||||
│ ├── layout/ # MainLayout, Navbar, Sidebar, VconShell, PageShell, AdminPageHeader, Footer
|
||||
│ ├── sections/ # Modular feature sections (DashboardOverview, ClientsTable, DataBrowser, ExportWorkspace, SettingsWorkspace, etc.)
|
||||
│ └── pwa/ # PwaLifecycle.tsx service worker registration
|
||||
├── lib/
|
||||
│ ├── adminAuth.ts # Uniform API session guard (requireAdminApiSession) + proxy-safe extractClientIp()
|
||||
│ ├── vconCache.ts # In-memory SWR cache + clearAuthCaches() for logout/SW cleanup
|
||||
│ └── vconUiHelpers.ts # Pure Admin UI sync helpers (settings bounds, bridge sync, UTC presets, timestamps)
|
||||
├── core/
|
||||
│ ├── adminSettings/ # settingsManager.ts (DB hydration, Port 4000 sync, unhydrated prune guard)
|
||||
│ ├── apiEngine/ # apiService.ts (dashboard metrics & Port 4000 internal bridge queries)
|
||||
@@ -81,6 +85,9 @@ datastack/
|
||||
│ ├── sdk/ # sdkManager.ts (DB JWT hydration, client lifecycle, deleted_clients tombstone)
|
||||
│ ├── Security/ # security.ts (rate limit, maxPayloadSizeKb, idempotency, internal headers)
|
||||
│ └── Storage/ # csvStorage.ts (cooperative lock, rotation, cached streaming row count, prune)
|
||||
├── scripts/
|
||||
│ ├── dev-all.js # Local concurrent Next.js (3000) + Ingestion (4000) dev runner
|
||||
│ └── start-all.js # Production concurrent Next.js (`next start`) + Ingestion (`server/index.js`) runner
|
||||
├── server/
|
||||
│ ├── index.js # Express 5 + Socket.IO (Port 4000), /health, /internal/* bridge, PG sync, graceful exit
|
||||
│ ├── socket/handlers.js # Handshake JWT auth, ping_verify, sync_data, batch_sync
|
||||
@@ -139,15 +146,24 @@ datastack/
|
||||
│ ├── configuration-reference.md # Complete environment variable & setting bounds
|
||||
│ ├── dependencies.md # Runtime & dev dependencies audit with rationales
|
||||
│ └── glossary.md # Technical domain terminology & concept glossary
|
||||
├── notes/ # Fresh forensic audit output only (legacy reports/plans cleared):
|
||||
│ └── forensic-audit-report.md # 100% source-verified audit report (27 issues: 5C, 11H, 10M, 1L)
|
||||
├── notes/ # Production Hardening Governance & Scope-Lock Suite (`BASELINE-2026-11-19-R2`):
|
||||
│ ├── README.md # Master governance index & 6-step scope-locked execution protocol
|
||||
│ ├── FORENSIC_AUDIT_REPORT.md # 100% raw-code verified audit (18 bugs: 0C/6H/8M/4L), Demo vs Prod & UI-vs-Reality matrices
|
||||
│ ├── BASELINE_FREEZE.md # Frozen baseline snapshot (`BASELINE-2026-11-19-R2`) & 1:1 bug alignment
|
||||
│ ├── PRODUCTION_READINESS_SCOPE_LOCK.md # (Doc 001) Exact bug fix contracts & per-phase Scope Locks (`Phase-01`..`Phase-04`)
|
||||
│ ├── PRODUCTION_ROADMAP.md # (Doc 002) Dependency-ordered A-Z 4-phase roadmap & burn-down tracker
|
||||
│ ├── PHASE_EXECUTION_LOG.md # (Doc 003) Live phase completion log (`4/4` complete) & zero-regression guardrails
|
||||
│ ├── ERROR_HANDLING_MATRIX.md # (Doc 004) Active error-handling catalog & 9 planned hardening additions
|
||||
│ └── ACTUAL_IMPLEMENTATION_STATUS.md # Ground-truth working vs completed-per-phase vs remaining ledger
|
||||
└── tests/ # Automated verification suites:
|
||||
├── phase01-auth.test.ts # Auth fail-closed, tokenVersion, brute-force lockout, Bearer revocation, CSRF
|
||||
├── phase02-sync.test.ts # Internal bridge auth, settings sync, JWT rotation kick, rate/security metrics
|
||||
├── phase03-storage-buffer.test.ts # Redis durability, cooperative CSV locks, streaming row count, export filters
|
||||
├── phase05_sdk_test.py # Python SDK WebSocket/HTTP fallback, offline spool, file streaming
|
||||
├── phase06-pwa-landing.test.ts # PWA manifest/icons/sw + navigation routes
|
||||
└── phase07-deployment.test.ts # Compose/Redis AOF, Nginx /health, lockfile, prune guard, shutdown flush
|
||||
├── phase04-ui-sync.test.ts # UI/API settings bounds, bridge status, UTC presets, timestamp & folder matching
|
||||
├── phase05-packaging.test.ts # Python/Node packaging parity, PEP 561 py.typed, Node engines & start-all exit
|
||||
├── phase05_sdk_test.py # Python SDK WebSocket/HTTP fallback, offline spool, file streaming, edge hardening
|
||||
├── phase06-pwa-landing.test.ts # PWA manifest/icons/sw + navigation routes
|
||||
└── phase07-deployment.test.ts # Compose/Redis AOF, Nginx /health, lockfile, prune guard, shutdown flush
|
||||
```
|
||||
|
||||
---
|
||||
@@ -175,10 +191,114 @@ All planned hardening & deployment phases (**Phase-01 through Phase-07**) are **
|
||||
7. **Docker / Coolify Deployment & Persistence Hardening (Completed — Phase-07)**
|
||||
- Redis AOF persistence (`redisdata:/data`), deterministic `package-lock.json` for `npm ci`, Nginx `location = /health` → `ingestion:4000`, `/api/health` container probes, non-blocking startup ownership check in Dockerfiles, unhydrated-DB `autoPruneDays` safety guard in `SettingsManager`, DB JWT hydration in `SDKManager`, graceful `SIGTERM`/`SIGINT` flush in `DataBufferEngine`, and non-destructive `ADD COLUMN IF NOT EXISTS` DDL guards.
|
||||
|
||||
- **Current Forensic Audit Status (2026-11-18)**:
|
||||
- All prior notes in `notes/` were cleared and replaced exclusively with `notes/forensic-audit-report.md` backed strictly by source-code citations and verbatim proofs (zero markdown extrapolation).
|
||||
- Baseline checks currently pass (`24/24` TS tests, `eslint .` clean, `3/3` Python unit tests), while deep source inspection uncovered **27 verified defects/gaps** (`5` Critical, `11` High, `10` Medium, `1` Low) across Auth/Middleware (`A01–A05`), Demo-vs-Prod Split-Brain (`B01–B05`), Buffer/CSV Storage (`C01–C06`), Admin UI vs Backend Reality (`D01–D07`), and SDK/Packaging/PWA (`E01–E04`).
|
||||
- Next step pending user approval: organize remediation into an incremental phased plan based strictly on `notes/forensic-audit-report.md`.
|
||||
- **Current Remediation Status (2026-11-18)**:
|
||||
- **Phase-01 Complete & Verified** (`BUG-A01`, `BUG-A02`, `BUG-A03`, `BUG-A04`, `BUG-A05`, `BUG-E02`):
|
||||
- Removed spoofable header/UA CSRF bypasses in `middleware.ts` and rejected `sec-fetch-site: cross-site`.
|
||||
- Enforced DB-backed `tokenVersion` session validation across `/vcon/*` layout (`app/vcon/layout.tsx` + `components/layout/VconShell.tsx`) and read/export APIs (`/api/data`, `GET /api/events`, `/api/export`, `/api/metrics`, `/api/storage`).
|
||||
- Prevented transient DB errors from triggering false `/vcon/setup` redirects and prevented swallowed `NEXT_REDIRECT` errors on `/`, `/vcon/login`, and `/vcon/setup`.
|
||||
- Standardized `SDKManager.normalizeClientId`, blocked deleted/unregistered clients on `/api/health` and Admin `POST /api/events`, hardened `POST /api/clients` & `PATCH /api/clients/[id]` input validation, and excluded `/vcon/*` from Service Worker caching with `CLEAR_AUTH_CACHE` on sign-out.
|
||||
- Verified: `26/26` TS tests, `3/3` Python tests, `npm run build` clean, `eslint .` clean, Playwright browser auth/logout E2E clean.
|
||||
- **Phase-02 Complete & Verified** (`BUG-B01`–`BUG-B05`):
|
||||
- Enforced fail-closed client verification in `server/index.js` (`verifyClientInDb`) backed by PG, deletion tombstones, Redis `datastack:clients:*` mirrors, and normalized `clientId` keys across both runtimes.
|
||||
- Added cross-runtime Redis idempotency (`datastack:idem:*`) and fixed-window rate limiting (`datastack:rl:*`) in `core/Security/security.ts` (`validateEventAsync`) and `server/socket/handlers.js` (`validateSingleEventAsync`) with automatic local RAM fallback.
|
||||
- Surfaced bridge sync telemetry (`BridgeSyncResult`, `SettingsBridgeSyncStatus`, `runtime.ingestionBridgeStatus`, `POST /api/settings` `bridgeSync`) and normalized `SDKManager.incrementEventCountAsync()`.
|
||||
- Fixed Socket `sync_data` flat payload preservation (`extractSingleEventData`), reconciled `PORT_SOCKET`/`PORT` precedence and live `/internal/metrics` ports, and added concurrent production `scripts/start-all.js` for `npm start`.
|
||||
- Verified: `29/29` TS tests, `3/3` Python tests, `npm run build` clean, `eslint .` clean, Playwright browser check (`/vcon/dashboard`, `/vcon/settings`, `/api/metrics`) clean with `0` console errors.
|
||||
- **Phase-03 Complete & Verified** (`BUG-C01`–`BUG-C06`):
|
||||
- Hardened Redis flush lock TTL (`30000ms`), added in-process `activeFlushPromise` coalescing with a `flushDone` identity guard (`core/data/buffer.ts`, `server/index.js`), isolated `redis.rpush` from best-effort `REDIS_RECENT_KEY`, and logged malformed buffered JSON warnings.
|
||||
- Enforced `sanitizeClientId()` path containment and `lstat` symlink exclusion, unique lock owner tokens (`${pid}:${ts}:${uuid}`), atomic `rename` stale-lock takeover, owner-verified lock unlock, and structured `JSON.stringify([clientId, date])` partition grouping in `core/Storage/csvStorage.ts` and `server/workers/csvWriter.js`.
|
||||
- Fixed trailing newline blank-row prevention (`fileEndsWithNewline`), proactive size-boundary rotation (`-partN.csv`), `StringDecoder('utf8')` streaming row counting, cooperatively locked `pruneExpiredPartitions`, and 1-hour `PRUNE_COOLDOWN_MS` in `core/adminSettings/settingsManager.ts`.
|
||||
- Enforced `parser`/`fileStream` destruction in `finally` blocks across `/api/data`, `GET /api/events`, and `/api/export`, error propagation inside `makeExportStream`, `502` surfacing on Port `4000` flush failure in `/api/export`, normalized `clientId` filtering, numeric timestamp sorting (`parseTimestampMs`), and `?format=json` / `?includeHeader=false` parity for `?filename=` exports.
|
||||
- Verified: `32/32` TS tests, `3/3` Python tests, `npm run build` clean, `eslint .` clean, Playwright browser check (`/vcon/dashboard`, `/vcon/data`, `/vcon/storage`, `/vcon/export`, `/api/data`, `/api/storage`, `/api/export`) clean with `0` console errors.
|
||||
- **Phase-04 Complete & Verified** (`BUG-D01`–`BUG-D07`):
|
||||
- Extracted pure UI synchronization helpers in `lib/vconUiHelpers.ts` (`SETTINGS_LIMITS`, `normalizeEditableSettings`, `describeSettingsBridgeSync`, `mapBridgeStatusToSocketUiState`, `parseEventTimestamp`, `formatEventTimeLabel`, `getUtcDateRangeForPreset`, `matchesSelectedClientFolder`, `extractContentDispositionFilename`).
|
||||
- Updated `components/sections/SettingsWorkspace.tsx` (`BUG-D01`) to probe `/api/metrics` honestly (removing false `/api/health` fallback), surface `bridgeSync` warning/error toasts on save, align integer bounds with `/api/settings` (`1..100000`, `1..10240`, `0..3650`), and eliminate the startup `serverPort` double-fetch.
|
||||
- Updated `app/vcon/storage/page.tsx` (`BUG-D02`) and `components/sections/ExportWorkspace.tsx` (`BUG-D03`) with `cache: 'no-store'`, `401` redirects, UTC preset ranges, normalized client folder filters, and fetch+Blob downloads with `502`/JSON error surfacing.
|
||||
- Updated `components/sections/DashboardOverview.tsx` (`BUG-D04`) and `components/sections/DataBrowser.tsx` (`BUG-D05`) with `ingestionBridgeStatus` display, `auth_error` alert banner, safe ISO/epoch-ms timestamp formatting, invalid date-range guard, and Enter-key filter submission.
|
||||
- Updated `components/sections/NewClientModal.tsx`, `components/sections/ClientsTable.tsx`, `app/vcon/simulator/page.tsx`, `app/vcon/sdk/page.tsx`, `components/sections/ProfileWorkspace.tsx`, and `components/layout/Sidebar.tsx` (`BUG-D06`, `BUG-D07`) with `1..100000` rate limit parity, live `runtime.serverPort` sync, real HTTP batch ingestion in Simulator, password bounds check, and `datastack:profile-updated` live Sidebar sync.
|
||||
- Verified: `37/37` TS tests (`tests/phase04-ui-sync.test.ts`), `3/3` Python tests, `npm run build` clean, `eslint .` clean, Playwright browser check across all Admin Console routes clean with `0` console errors.
|
||||
- **Phase-05 Complete & Verified** (`BUG-E01`, `BUG-E03`, `BUG-E04`):
|
||||
- Hardened canonical Python SDK (`sdk/python/datastack/datastack/*`): normalized `CLIENT_ID` comparison (`_normalize_client_id`), sanitized ragged CSV extra columns (`restkey="_extra"`), accurate `send_file()` `success` + `unprocessedItems`, partial-batch `rateLimitedEventIds` retry exception fallback, `DataStackConfigError` wrapping for malformed numeric env/config values (`config.py`), concurrent `stop()` vs `set_idle_connected()` race fix + clean thread join (`heartbeat.py`), `os.fsync` + orphan `.tmp` unlink + warning logs (`spool.py`), and PEP 561 `py.typed` marker.
|
||||
- Aligned packaging metadata across `sdk/python/datastack/pyproject.toml`, `sdk/python/datastack/setup.py`, and `sdk-python/setup.py`, coordinated child-exit shutdown in `scripts/start-all.js`, and added `engines.node >=20.0.0` in `package.json`.
|
||||
- Verified: `39/39` TS tests (`tests/phase05-packaging.test.ts`), `6/6` Python tests (`tests/phase05_sdk_test.py`), `py_compile`, `npm run build`, and `eslint .` clean.
|
||||
- **Previous Remediation Cycle (2026-11-18)**: Completed Phases 01–05 (`39/39` TS tests, `6/6` Python tests).
|
||||
- **New Production Hardening Cycle (`BASELINE-2026-11-19`)**:
|
||||
- Re-audited entire codebase from raw source code (`notes/FORENSIC_AUDIT_REPORT.md`): **29** verified bugs/mismatches (`4` Critical, `8` High, `13` Medium, `4` Low).
|
||||
- **Phase-01 Complete & Verified (`2026-11-19`)** (`BUG-01`, `BUG-04`, `BUG-05`, `BUG-06`):
|
||||
- Blocked volatile RAM admin creation in production without `DATABASE_URL` (`AdminDatabaseRequiredError`), surfaced production DB warning on `/vcon/setup`, and mapped pre-setup login to `409 SETUP_REQUIRED`.
|
||||
- Added brute-force attempt throttling (`429` + `Retry-After`) and completion-first ordering to `POST /api/auth/setup`.
|
||||
- Added server-side `token_version` revocation (`revokeAdminSessions()`) on `POST /api/auth/logout`, removed client `Origin` trust from cookie `Secure` flag, and added `sec-fetch-site: same-origin` CSRF fallback.
|
||||
- Prevented empty production `jwt_secret` persistence in PG and replaced silent DB write `catch {}` swallows in `SettingsManager` and `SDKManager` with `DATABASE_WRITE_FAILED` propagation and safe RAM rollback.
|
||||
- Verified: `43/43` TS tests, `6/6` Python tests, `npm run build`, `eslint .`, and Playwright login/logout revocation check clean.
|
||||
- **Phase-02 Complete & Verified (`2026-11-19`)** (`BUG-02`, `BUG-03`, `BUG-17`, `BUG-22`):
|
||||
- Protected Port 4000's `inMemoryClientRegistry` from empty Port 3000 `bulk_sync` wipes (`!forceReplace`), exposed `inMemoryClients` on `/internal/metrics`, added non-destructive hydration in `SDKManager` (`listClientsAsync`, `getClientAsync`, `syncAllClientsToIngestionAsync`), and auto-reconciled Port 4000 cold starts in `ApiEngine.getDashboardMetrics()`.
|
||||
- Ordered `SettingsManager.updateSettingsAsync()` JWT rotation to sync Port 4000 *before* committing Port 3000 RAM/PG/re-sign, aborting fail-closed (`JWT_ROTATION_BRIDGE_FAILED`) on bridge rejection/prod unreachable, rolling back Port 4000 on PG/re-sign error, and adding a 60s dual-secret grace window in `server/index.js` `requireInternalAuth()`.
|
||||
- Added authenticated `POST /internal/security-check` on Port 4000 (backed by zero-copy `validateSingleEventLocal`) so Port 3000 HTTP `SecurityEngine.validateEventAsync()` shares rate limits & idempotency with Port 4000 WS even without Redis, and exposed explicit `securityMode` (`redis_shared` | `bridge_coordinated` | `degraded_local`) in `/api/metrics`.
|
||||
- Replaced per-event PG `UPDATE client_sdks` queries on both Port 3000 (`SDKManager`) and Port 4000 (`server/index.js`) with 1s coalescing in-memory deltas flushed via batched `UNNEST` SQL updates (flushed on reads and graceful shutdown).
|
||||
- Verified: `47/47` TS tests, `6/6` Python tests, `npm run build`, and `eslint .` clean.
|
||||
- **Phase-03 Complete & Verified (`2026-11-19`)** (`BUG-18`, `BUG-19`, `BUG-20`, `BUG-29`):
|
||||
- Added bounded Redis flush lock wait/retry, periodic Lua `PEXPIRE` TTL renewal with ownership fencing before `LTRIM`, and explicit `FLUSH_LOCK_TIMEOUT` (`503` on `/internal/flush`) across both `core/data/buffer.ts` and `server/index.js`.
|
||||
- Added full deleted-client buffer purge (`DataBufferEngine.purgeClientEventsAsync`, Port 4000 `purgeClientFromLocalBuffer`) plus flush-time and recent-read tombstone filtering so deleted client directories never resurrect on subsequent flushes; ordered `SDKManager.deleteClientAsync()` to purge buffers and sync Port 4000 before deleting disk files.
|
||||
- Converted `server/index.js` `onBufferPush` to `async` and awaited it before firing Socket.IO ACK callbacks in `server/socket/handlers.js` (`sync_data` and `batch_sync`), returning `BUFFER_PERSIST_FAILED` on persistence error.
|
||||
- Unified canonical CSV storage root resolution via `DATASTACK_STORAGE_DIR` (`resolveDefaultStorageDir()`) across `core/Storage/csvStorage.ts`, `server/workers/csvWriter.js`, `core/sdk/sdkManager.ts`, and `.env.example`.
|
||||
- Verified: `51/51` TS tests, `6/6` Python tests, `npm run build`, and `eslint .` clean.
|
||||
- **Phase-04 Complete & Verified (`2026-11-19`)** (`BUG-21`, `BUG-23`, `BUG-24`):
|
||||
- Aligned HTTP `POST /api/events` single-event envelope stripping 1:1 with Port 4000 WS `sync_data` and eliminated fabricated `{ sample, rand }` telemetry on empty `{}`.
|
||||
- Converted `GET /api/data` directory/stat scans to non-blocking async `node:fs/promises` with newest-first date-partition pruning, and scaled `REDIS_RECENT_LIMIT` boundedly (`500..5000`) via `resolveRedisRecentLimit()` / `DATASTACK_RECENT_LIMIT` across Port 3000 & Port 4000.
|
||||
- Replaced all `Math.random()` client secrets (`crypto.randomBytes(16)`) and fallback `eventId`s (`crypto.randomBytes(6)`) with CSPRNG across `core/sdk/sdkManager.ts`, `app/api/events/route.ts`, and `server/socket/handlers.js`.
|
||||
- Unified canonical PostgreSQL metadata DDL (`CORE_SCHEMA_TABLE_DDLS` + `CORE_SCHEMA_ALTER_DDLS` + `ensureCoreSchemaOnPool()`) in `core/Database/database.ts`, delegated `core/auth/adminUsers.ts` to it, mirrored all 5 tables + 3 additive `ALTER TABLE` guards 1:1 in standalone `server/index.js`, and documented canonical `pg` ownership in `prisma/schema.prisma`.
|
||||
- Verified: `55/55` TS tests (`tests/phase04-ingestion-hygiene.test.ts` added), `6/6` Python tests, `npm run build`, and `eslint .` clean.
|
||||
- **Phase-05 Complete & Verified (`2026-11-19`)** (`BUG-07`, `BUG-08`, `BUG-09`, `BUG-14`):
|
||||
- Fixed Simulator HTTP `Burst 10` to send canonical `{ clientId, batch }` (with `events` batch alias & explicit `accepted` count in `POST /api/events`) and summarize exact server-confirmed counts via `summarizeHttpBatchResult()` without `res.ok ? count : 0` falsification.
|
||||
- Fixed `parseEventTimestamp()` & added `getEventUtcDate()` in `lib/vconUiHelpers.ts` for integer and fractional epoch-seconds (`< 1e11 * 1000`) 1:1 with backend CSV parsers.
|
||||
- Added pending-queue-only `getPendingBufferedSummary()` (`?bufferedOnly=true` / `?includeBufferedSummary=true`) and fair multi-client partition scanning (`selectRecentPartitionFiles()`, lifting the global 5-file cap) in `GET /api/events`.
|
||||
- Updated `ExportWorkspace.tsx` to match buffered events strictly against active client/date filters via `countMatchingBufferedEvents()` (preventing empty filtered header downloads) and refresh `/api/storage`, `/api/metrics`, and `vconCache` immediately post-export; updated `DashboardOverview.tsx` to attribute strictly UTC-today buffered events to `Events today` and call `await refresh()` after Quick Export.
|
||||
- Verified: `58/58` TS tests, `6/6` Python tests, `npm run build`, `eslint .`, and Playwright E2E browser check (`/vcon/simulator`, `/vcon/export`, `/vcon/dashboard`, `0` console errors/warnings) clean.
|
||||
- **Phase-06 Complete & Verified (`2026-11-19`)** (`BUG-10`, `BUG-11`, `BUG-12`, `BUG-13`, `BUG-15`, `BUG-16`):
|
||||
- Returned explicit `SESSION_EXPIRED` / `AUTH_UNAVAILABLE` vs `INVALID_PASSWORD` codes in `/api/auth/profile`, replaced dead `401 && !currentPassword` redirect with `shouldRedirectProfileUnauthorized()`, and formatted `Session Expires` with calendar date + 24h time (`formatSessionExpiryLabel()`).
|
||||
- Added opt-in `?purgeFiles=true` checkbox in `ClientsTable.tsx` client deletion modal and removed the misleading unused `Secret key` row from `CredentialsDisplay`.
|
||||
- Refactored `SettingsWorkspace.tsx` numeric inputs to string drafts (`NumericSettingDrafts`) validated inline via `validateEditableSettingsDraft()` (`noValidate` submit + tab jump to first invalid field, zero snap-to-`0`, zero silent clamping) and clarified `Socket port (PORT_SOCKET)` as read-only runtime config.
|
||||
- Unified public endpoint resolution (`resolvePublicEndpoints()`) across `SettingsWorkspace`, `/vcon/sdk`, and `/vcon/simulator` (consistent on port-80 `localhost` proxies) and updated `SdkSection.tsx` + `/vcon/sdk` with external path/wheel `pip install` guidance.
|
||||
- Added pre-hydration `datastack-theme` bootstrap script + `suppressHydrationWarning` in `app/layout.tsx` (zero light-mode FOUC) and theme restoration/toggle in `AdminSetup.tsx`.
|
||||
- Upgraded `public/sw.js` (`datastack-cache-v3`) with an uncached `503` HTML offline navigation fallback shell (`buildOfflineShellResponse()`, `Cache-Control: no-store`) for `/vcon/dashboard` and public navigations.
|
||||
- Verified: `64/64` TS tests, `6/6` Python tests, `npm run build`, `eslint .`, and Playwright Chromium E2E check (`/vcon/profile`, `/vcon/clients`, `/vcon/settings`, `/vcon/sdk`, Light Mode reload, `0` console errors/warnings) clean.
|
||||
- **Fresh Post-Phase-07 Forensic Audit & Baseline Freeze (`BASELINE-2026-11-19-R2`)**:
|
||||
- Previous `Phase-07` hardening (`BUG-25`–`BUG-28`) landed in Python SDK (`client.py`), legacy `sdk-python`, `scripts/start-all.js` / `dev-all.js`, `nginx/default.conf`, `server/index.js`, `docker/docker-compose.yml`, and `.env.example`.
|
||||
- Conducted a brand-new 100% raw-source-code audit (`notes/FORENSIC_AUDIT_REPORT.md`) identifying **18 verified remaining bugs** (`0` Critical, `6` High, `8` Medium, `4` Low).
|
||||
- Frozen baseline (`BASELINE-2026-11-19-R2`) and created the complete 7-document governance & Scope-Lock suite in `notes/` (`README.md`, `BASELINE_FREEZE.md`, `PRODUCTION_READINESS_SCOPE_LOCK.md`, `PRODUCTION_ROADMAP.md`, `PHASE_EXECUTION_LOG.md`, `ERROR_HANDLING_MATRIX.md`, `ACTUAL_IMPLEMENTATION_STATUS.md`).
|
||||
- **R2 Phase-01 Complete & Verified (`2026-11-19`)** (`BUG-01`, `BUG-02`):
|
||||
- Moved same-origin CSRF enforcement in `middleware.ts` ahead of the public-route early return so `POST /api/auth/login` and `POST /api/auth/setup` reject cross-origin/missing-origin requests with `403` while preserving same-origin browser submissions, `sec-fetch-site: same-origin` fallback, safe `GET` routes, and Bearer `POST /api/events`.
|
||||
- Extracted shared reverse-proxy-safe `extractClientIp()` in `lib/adminAuth.ts` (trusting `x-real-ip` first, then rightmost non-empty `x-forwarded-for` hop) and wired it into both `app/api/auth/login/route.ts` and `app/api/auth/setup/route.ts` to block spoofed-XFF brute-force lockout bypasses.
|
||||
- Verified: `14/14` `tests/phase01-auth.test.ts` pass, `npm run build`, `npm run lint`, live HTTP `403`/`401` CSRF probes, and Chromium `/vcon/login` → `/vcon/dashboard` E2E (`0` errors).
|
||||
- **R2 Phase-02 Complete & Verified (`2026-11-19`)** (`BUG-05`, `BUG-06`, `BUG-07`, `BUG-08`, `BUG-09`, `BUG-10`):
|
||||
- Attached `.on('error', ...)` listeners to `DatabaseEngine.getPool()` (`max: 10`) and `server/index.js` `dbPool`, and consolidated `core/auth/adminUsers.ts` onto the shared `DatabaseEngine.getPool()`.
|
||||
- Replaced non-atomic Redis `LTRIM`+`LPUSH` client deletion purging in `core/data/buffer.ts` with atomic Lua script `PURGE_CLIENT_LIST_LUA_SCRIPT` (`cjson.decode` + chunked `RPUSH`), and replaced non-atomic Redis `INCR`+`PEXPIRE` rate limiting with atomic Lua script `REDIS_RATE_LIMIT_LUA_SCRIPT` across both `core/Security/security.ts` and `server/socket/handlers.js`.
|
||||
- Enforced `throwOnLockTimeout: true` locally and remotely on `POST /internal/flush` in `GET /api/export` and `POST /api/storage`, surfacing explicit `503 FLUSH_LOCK_TIMEOUT`.
|
||||
- Added `DataBufferEngine.registerShutdownHook()` and registered `SDKManager.flushPendingClientActivityAsync()` so Port `3000` flushes coalesced `total_events` deltas on `SIGTERM`/`SIGINT`.
|
||||
- Wrapped `GET`, `PATCH` (`regenerate_token` & update), and `DELETE` in `app/api/clients/[id]/route.ts` with structured JSON `try/catch` error responses.
|
||||
- Verified: `14/14` `tests/phase02-sync.test.ts` pass, `3/3` `R2 PHASE-02` buffer/storage tests pass, `npm run build` pass, `npm run lint` pass (`0` errors).
|
||||
- **R2 Phase-03 Complete & Verified (`2026-11-19`)** (`BUG-03`, `BUG-04`, `BUG-11`, `BUG-12`, `BUG-13`, `BUG-15`):
|
||||
- Unified `SettingsBridgeSyncStatus` & `POST /api/settings` `bridgeSync` response (`attempted`, `ok`, `synced`, `status`, `error`, `checkedAt`) and updated `describeSettingsBridgeSync()` + `SettingsWorkspace.tsx` so Port `4000` warnings/online state work reliably.
|
||||
- Counted regular non-symlink `.csv` files in `SDKManager.purgeClientStorageDir()` (`deleteClientWithStatsAsync()`) and returned exact `purgedFiles` from `DELETE /api/clients/[id]?purgeFiles=true`.
|
||||
- Aligned flat batch item metadata stripping (`eventId`, `timestamp`, `clientId`) 1:1 with single-event stripping across HTTP `POST /api/events`, WS `batch_sync`, and Python SDK `send_batch()`/`send_file()`.
|
||||
- Exposed `DataBufferEngine.getLocalBufferSnapshot()` for exact Port `3000` RAM pending summary in `getPendingBufferedSummary()`.
|
||||
- Added non-blocking `pruneExpiredPartitionsAsync()` off live event loops in `core/Storage/csvStorage.ts` & `server/workers/csvWriter.js` (wired into `settingsManager.ts` & `server/index.js`) and deduplicated `GET /api/storage` to a single `listAllFilesAsync()` scan with accurate client-scoped `stats`.
|
||||
- Cleared `selectedClientToken` on `/vcon/sdk` client switch (with loading placeholder + disabled `Copy Config`), added `400 INVALID_DATE_RANGE` for `from > to` in `GET /api/export`, and normalized `clientId` filtering in `ApiEngine.getExportFiles*` & Port `4000` `/internal/recent`.
|
||||
- Verified: `21/21` `phase04-*` tests pass, `2/2` `R2 PHASE-03` storage tests pass, `9/9` Python SDK tests pass, Chromium E2E across `/vcon/settings`, `/vcon/clients`, `/vcon/sdk`, `/vcon/storage` (`0` console errors), `npm run build` & `npm run lint` (`0` errors).
|
||||
- **R2 Phase-04 Complete & Verified (`2026-11-19`)** (`BUG-14`, `BUG-16`, `BUG-17`, `BUG-18`):
|
||||
- Added `from __future__ import annotations`, imported `Tuple`, and replaced PEP 585 `tuple[int, Dict[str, Any]]` with Python `3.8`-safe `Tuple[int, Dict[str, Any]]` in `sdk/python/datastack/datastack/client.py`, verified by a full-package Python `3.8` AST scan test in `tests/phase05_sdk_test.py`.
|
||||
- Added unbuffered streaming `location = /api/export` (`proxy_buffering off;`, `proxy_read_timeout 300s;`) targeting `http://nextjs_upstream` in both `nginx/default.conf` and `docker/coolify-nginx.conf`.
|
||||
- Removed `eslint: { ignoreDuringBuilds: true }` from `next.config.ts`, set `package.json` `"name": "datastack"`, pruned unused template packages (`@google/genai`, `firebase-tools`), and synchronized `package-lock.json`.
|
||||
- Isolated `DATASTACK_DISABLE_LISTEN=1` and live Port `4000` coupling across `tests/phase01-auth.test.ts`, `tests/phase02-sync.test.ts`, `tests/phase03-storage-buffer.test.ts`, `tests/phase04-ingestion-hygiene.test.ts`, and `tests/phase07-deployment.test.ts`.
|
||||
- Verified: `83/83` full TS suite pass (with live Port `4000` running, without manual env flags), `10/10` Python SDK tests + `py_compile` pass, `npm run build` (with ESLint active) pass, `npm run lint` (`0` errors, `0` warnings).
|
||||
- **Progress**: `4 / 4` phases complete (`18 / 18` bugs resolved — **100% Production Ready**).
|
||||
- **Pre-Release Docker & Coolify Deployment Hardening Complete & Verified (`2026-11-19`)** (`DEPLOY-BUG-01`–`05`, audited in `notes/DEPLOYMENT_COOLIFY_AUDIT_REPORT.md`):
|
||||
- **`DEPLOY-BUG-01`**: Replaced static `upstream` blocks in `docker/coolify-nginx.conf` with Docker embedded DNS `resolver 127.0.0.11 valid=10s ipv6=off;` and variable `proxy_pass` targets (`$nextjs_upstream` = `http://web:3000`, `$socketio_upstream` = `http://ingestion:4000`), preventing `502 Bad Gateway` stale container IP outages after container restarts/redeploys.
|
||||
- **`DEPLOY-BUG-02`**: Refactored Port `4000` startup in `server/index.js` into `startIngestionServer()` awaiting bounded `syncSettingsFromDatabase()` before `server.listen(PORT)` so rotated PG `jwt_secret` and rate-limit settings hydrate prior to `/health: 200` and `/internal/*` bridge traffic.
|
||||
- **`DEPLOY-BUG-03`**: Memoized `ensureDatabaseSchema()` per active `dbPool` instance in `server/index.js` (resetting only on pool swap or DDL failure) so 3-second `syncSettingsFromDatabase()` ticks no longer issue 8 `CREATE/ALTER TABLE` DDLs repeatedly.
|
||||
- **`DEPLOY-BUG-04`**: Reordered `gracefulShutdown()` in `server/index.js` to close Socket.IO/HTTP ingress first (`closeServerGracefully()`), drain `pendingRedisWrites` and `flushBuffer()`/`flushPendingClientActivity()` with bounded timeouts (`2000ms`/`6000ms`), and close Redis (`quit()` with `2000ms` timeout + `.disconnect()` fallback) and PostgreSQL (`dbPool.end()` bounded `2000ms`); added `stop_grace_period: 20s` to both `web` and `ingestion` in `docker/docker-compose.yml`.
|
||||
- **`DEPLOY-BUG-05`**: Clarified Coolify `Base Directory: /` and `Docker Compose Location: /docker/docker-compose.yml` in `docs/users/installation.md` and `README.md`.
|
||||
- Verified: `85/85` full TS suite pass (`11/11` in `tests/phase07-deployment.test.ts`), `10/10` Python SDK tests pass, `npm run build` clean, `npm run lint` (`0` errors, `0` warnings).
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -293,7 +293,7 @@ DataStack includes a production-ready Docker Compose stack in [`docker/docker-co
|
||||
|
||||
### Coolify Deployment Steps
|
||||
1. Create a **Docker Compose** resource in Coolify pointing to your repository.
|
||||
2. Set **Base Directory** to `/docker` and **Docker Compose Location** to `/docker/docker-compose.yml`.
|
||||
2. Set **Base Directory** to `/` and **Docker Compose Location** to `/docker/docker-compose.yml`.
|
||||
3. Attach your HTTPS domain to the `proxy` service on port `80`.
|
||||
4. Deploy the stack, open `https://your-domain.example/vcon/setup`, and enter your administrator email, password (8+ characters), and the generated `SERVICE_REALBASE64_64_ADMININSTALL` key from Coolify's environment variables.
|
||||
|
||||
|
||||
+14
-2
@@ -9,8 +9,20 @@ import { redirect } from 'next/navigation';
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
export default async function LandingPage() {
|
||||
const isConfigured = await isAdminSetupComplete().catch(() => false);
|
||||
if (!isConfigured) redirect('/vcon/setup');
|
||||
try {
|
||||
const isConfigured = await isAdminSetupComplete();
|
||||
if (!isConfigured) redirect('/vcon/setup');
|
||||
} catch (error) {
|
||||
if (
|
||||
error &&
|
||||
typeof error === 'object' &&
|
||||
'digest' in error &&
|
||||
String((error as { digest?: unknown }).digest).startsWith('NEXT_REDIRECT')
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
// Keep public landing accessible on transient DB outage instead of redirecting to setup
|
||||
}
|
||||
|
||||
return (
|
||||
<MainLayout>
|
||||
|
||||
+12
-12
@@ -4,27 +4,18 @@ import {
|
||||
getSessionCookieOptions,
|
||||
} from "@/core/auth/adminSession";
|
||||
import {
|
||||
AdminSetupRequiredError,
|
||||
authenticateAdminCredentials,
|
||||
clearAdminLoginAttempts,
|
||||
consumeAdminLoginAttempt,
|
||||
} from "@/core/auth/adminUsers";
|
||||
import { extractClientIp } from "@/lib/adminAuth";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
const SESSION_COOKIE = "datastack_admin_session";
|
||||
const SESSION_SECONDS = 8 * 60 * 60;
|
||||
const REMEMBERED_SESSION_SECONDS = 30 * 24 * 60 * 60;
|
||||
|
||||
function extractClientIp(request: NextRequest) {
|
||||
const forwardedFor = request.headers.get("x-forwarded-for");
|
||||
if (forwardedFor) {
|
||||
const first = forwardedFor.split(",")[0]?.trim();
|
||||
if (first) return first;
|
||||
}
|
||||
const realIp = request.headers.get("x-real-ip")?.trim();
|
||||
if (realIp) return realIp;
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
const sessionSecret = getAdminSessionSecret();
|
||||
|
||||
@@ -91,7 +82,16 @@ export async function POST(request: NextRequest) {
|
||||
let authResult: { authenticated: boolean; tokenVersion: number };
|
||||
try {
|
||||
authResult = await authenticateAdminCredentials(email, password);
|
||||
} catch {
|
||||
} catch (err) {
|
||||
if (err instanceof AdminSetupRequiredError) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: "Initial admin setup is required before signing in.",
|
||||
code: err.code,
|
||||
},
|
||||
{ status: 409 },
|
||||
);
|
||||
}
|
||||
return NextResponse.json(
|
||||
{ error: "Admin authentication is temporarily unavailable." },
|
||||
{ status: 503 },
|
||||
|
||||
@@ -1,9 +1,25 @@
|
||||
import { getSessionCookieOptions } from "@/core/auth/adminSession";
|
||||
import {
|
||||
revokeAdminSessions,
|
||||
verifyRequestAdminSession,
|
||||
} from "@/core/auth/adminUsers";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
const SESSION_COOKIE = "datastack_admin_session";
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
const token = request.cookies.get(SESSION_COOKIE)?.value;
|
||||
if (token) {
|
||||
try {
|
||||
const check = await verifyRequestAdminSession(token);
|
||||
if (check.valid) {
|
||||
await revokeAdminSessions();
|
||||
}
|
||||
} catch {
|
||||
// Still clear client cookie even if server revocation encounters transient DB error
|
||||
}
|
||||
}
|
||||
|
||||
const response = NextResponse.json({ success: true });
|
||||
response.cookies.set(SESSION_COOKIE, "", {
|
||||
...getSessionCookieOptions(request, 0),
|
||||
|
||||
@@ -22,11 +22,13 @@ export async function GET(request: NextRequest) {
|
||||
|
||||
if (!auth.valid) {
|
||||
const status = auth.reason === "UNAUTHORIZED" ? 401 : 503;
|
||||
const code =
|
||||
auth.reason === "UNAUTHORIZED" ? "SESSION_EXPIRED" : "AUTH_UNAVAILABLE";
|
||||
const message =
|
||||
auth.reason === "UNAUTHORIZED"
|
||||
? "Authentication required."
|
||||
: "Admin authentication is temporarily unavailable.";
|
||||
return NextResponse.json({ error: message }, { status });
|
||||
return NextResponse.json({ error: message, code }, { status });
|
||||
}
|
||||
|
||||
try {
|
||||
@@ -54,11 +56,13 @@ export async function PUT(request: NextRequest) {
|
||||
|
||||
if (!auth.valid) {
|
||||
const status = auth.reason === "UNAUTHORIZED" ? 401 : 503;
|
||||
const code =
|
||||
auth.reason === "UNAUTHORIZED" ? "SESSION_EXPIRED" : "AUTH_UNAVAILABLE";
|
||||
const message =
|
||||
auth.reason === "UNAUTHORIZED"
|
||||
? "Authentication required."
|
||||
: "Admin authentication is temporarily unavailable.";
|
||||
return NextResponse.json({ error: message }, { status });
|
||||
return NextResponse.json({ error: message, code }, { status });
|
||||
}
|
||||
|
||||
let body: Record<string, unknown>;
|
||||
@@ -115,7 +119,10 @@ export async function PUT(request: NextRequest) {
|
||||
});
|
||||
|
||||
if (!result.ok) {
|
||||
return NextResponse.json({ error: result.error }, { status: 401 });
|
||||
return NextResponse.json(
|
||||
{ error: result.error, code: "INVALID_PASSWORD" },
|
||||
{ status: 401 },
|
||||
);
|
||||
}
|
||||
|
||||
// Preserve remaining session duration (e.g., 30-day "Remember me" session)
|
||||
|
||||
@@ -4,14 +4,20 @@ import {
|
||||
getSessionCookieOptions,
|
||||
} from "@/core/auth/adminSession";
|
||||
import {
|
||||
AdminDatabaseRequiredError,
|
||||
clearAdminLoginAttempts,
|
||||
consumeAdminLoginAttempt,
|
||||
createInitialAdmin,
|
||||
isAdminSetupComplete,
|
||||
requiresPersistentAdminDatabase,
|
||||
verifyAdminSetupKey,
|
||||
} from "@/core/auth/adminUsers";
|
||||
import { extractClientIp } from "@/lib/adminAuth";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
const SESSION_COOKIE = "datastack_admin_session";
|
||||
const SESSION_SECONDS = 8 * 60 * 60;
|
||||
const SETUP_THROTTLE_BUCKET = "__admin_setup__";
|
||||
|
||||
export const runtime = "nodejs";
|
||||
export const dynamic = "force-dynamic";
|
||||
@@ -29,6 +35,17 @@ export async function POST(request: NextRequest) {
|
||||
);
|
||||
}
|
||||
|
||||
if (requiresPersistentAdminDatabase()) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error:
|
||||
"PostgreSQL DATABASE_URL must be configured in production before initial admin setup.",
|
||||
code: "DATABASE_NOT_CONFIGURED",
|
||||
},
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
let body: Record<string, unknown>;
|
||||
try {
|
||||
const parsed = await request.json();
|
||||
@@ -61,12 +78,7 @@ export async function POST(request: NextRequest) {
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
if (!verifyAdminSetupKey(setupKey)) {
|
||||
return NextResponse.json(
|
||||
{ error: "The installation key is invalid." },
|
||||
{ status: 403 },
|
||||
);
|
||||
}
|
||||
|
||||
if (await isAdminSetupComplete()) {
|
||||
return NextResponse.json(
|
||||
{ error: "Initial admin setup has already been completed." },
|
||||
@@ -74,6 +86,34 @@ export async function POST(request: NextRequest) {
|
||||
);
|
||||
}
|
||||
|
||||
const clientIp = extractClientIp(request);
|
||||
const throttleDecision = await consumeAdminLoginAttempt(
|
||||
SETUP_THROTTLE_BUCKET,
|
||||
sessionSecret,
|
||||
clientIp,
|
||||
);
|
||||
if (!throttleDecision.allowed) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: "Too many setup attempts. Try again later.",
|
||||
code: "TOO_MANY_ATTEMPTS",
|
||||
},
|
||||
{
|
||||
status: 429,
|
||||
headers: {
|
||||
"Retry-After": String(throttleDecision.retryAfterSeconds),
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
if (!verifyAdminSetupKey(setupKey)) {
|
||||
return NextResponse.json(
|
||||
{ error: "The installation key is invalid." },
|
||||
{ status: 403 },
|
||||
);
|
||||
}
|
||||
|
||||
const created = await createInitialAdmin(email, password);
|
||||
if (!created) {
|
||||
return NextResponse.json(
|
||||
@@ -82,6 +122,12 @@ export async function POST(request: NextRequest) {
|
||||
);
|
||||
}
|
||||
|
||||
await clearAdminLoginAttempts(
|
||||
SETUP_THROTTLE_BUCKET,
|
||||
sessionSecret,
|
||||
clientIp,
|
||||
).catch(() => undefined);
|
||||
|
||||
const token = await createAdminSessionToken(
|
||||
email,
|
||||
sessionSecret,
|
||||
@@ -95,7 +141,13 @@ export async function POST(request: NextRequest) {
|
||||
getSessionCookieOptions(request, SESSION_SECONDS),
|
||||
);
|
||||
return response;
|
||||
} catch {
|
||||
} catch (err) {
|
||||
if (err instanceof AdminDatabaseRequiredError) {
|
||||
return NextResponse.json(
|
||||
{ error: err.message, code: err.code },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
return NextResponse.json(
|
||||
{ error: "Admin setup is temporarily unavailable." },
|
||||
{ status: 503 },
|
||||
|
||||
+160
-28
@@ -3,6 +3,7 @@ import { SDKManager } from "@/core/sdk/sdkManager";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
const SESSION_COOKIE = "datastack_admin_session";
|
||||
const ALLOWED_PATCH_KEYS = new Set(["action", "name", "rateLimit", "status"]);
|
||||
|
||||
export async function GET(
|
||||
req: NextRequest,
|
||||
@@ -19,20 +20,33 @@ export async function GET(
|
||||
}
|
||||
|
||||
const { id } = await params;
|
||||
const client = await SDKManager.getClientAsync(id);
|
||||
if (!client) {
|
||||
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
||||
const normalizedId = SDKManager.normalizeClientId(id);
|
||||
try {
|
||||
const client = await SDKManager.getClientAsync(normalizedId);
|
||||
if (!client) {
|
||||
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
||||
}
|
||||
const { secret, token, ...clientSummary } = client;
|
||||
void secret;
|
||||
const includeToken = req.nextUrl.searchParams.get("includeToken") === "true";
|
||||
return NextResponse.json({
|
||||
client: {
|
||||
...clientSummary,
|
||||
tokenPreview: token ? `${token.slice(0, 12)}...` : "",
|
||||
...(includeToken ? { token } : {}),
|
||||
},
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : "Failed to load client";
|
||||
const isDbError = message.includes("DATABASE_");
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: message,
|
||||
code: isDbError ? "DATABASE_READ_FAILED" : "CLIENT_FETCH_FAILED",
|
||||
},
|
||||
{ status: isDbError ? 503 : 500 },
|
||||
);
|
||||
}
|
||||
const { secret, token, ...clientSummary } = client;
|
||||
void secret;
|
||||
const includeToken = req.nextUrl.searchParams.get("includeToken") === "true";
|
||||
return NextResponse.json({
|
||||
client: {
|
||||
...clientSummary,
|
||||
tokenPreview: token ? `${token.slice(0, 12)}...` : "",
|
||||
...(includeToken ? { token } : {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function PATCH(
|
||||
@@ -50,29 +64,128 @@ export async function PATCH(
|
||||
}
|
||||
|
||||
const { id } = await params;
|
||||
try {
|
||||
const body = await req.json();
|
||||
const normalizedId = SDKManager.normalizeClientId(id);
|
||||
|
||||
// Special action: regenerate token
|
||||
if (body.action === "regenerate_token") {
|
||||
const refreshed = await SDKManager.regenerateTokenAsync(id);
|
||||
let rawBody: unknown;
|
||||
try {
|
||||
rawBody = await req.json();
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid JSON request payload.", code: "INVALID_JSON" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
if (!rawBody || typeof rawBody !== "object" || Array.isArray(rawBody)) {
|
||||
return NextResponse.json(
|
||||
{ error: "Update payload must be a JSON object.", code: "VALIDATION_ERROR" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
const body = rawBody as Record<string, unknown>;
|
||||
const unknownKeys = Object.keys(body).filter(
|
||||
(key) => !ALLOWED_PATCH_KEYS.has(key),
|
||||
);
|
||||
if (unknownKeys.length > 0) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: `Unsupported client field(s): ${unknownKeys.join(", ")}`,
|
||||
code: "VALIDATION_ERROR",
|
||||
},
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
// Special action: regenerate token
|
||||
if (body.action !== undefined) {
|
||||
if (body.action !== "regenerate_token") {
|
||||
return NextResponse.json(
|
||||
{ error: "Unsupported client action.", code: "VALIDATION_ERROR" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
try {
|
||||
const refreshed = await SDKManager.regenerateTokenAsync(normalizedId);
|
||||
if (!refreshed) {
|
||||
return NextResponse.json(
|
||||
{ error: "Client not found" },
|
||||
{ status: 404 },
|
||||
);
|
||||
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
||||
}
|
||||
return NextResponse.json({ client: refreshed });
|
||||
} catch (err: unknown) {
|
||||
const message =
|
||||
err instanceof Error ? err.message : "Token regeneration failed";
|
||||
const isDbWrite = message.includes("DATABASE_WRITE_FAILED");
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: message,
|
||||
code: isDbWrite ? "DATABASE_WRITE_FAILED" : "TOKEN_REGENERATION_FAILED",
|
||||
},
|
||||
{ status: isDbWrite ? 503 : 500 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const updated = await SDKManager.updateClientAsync(id, body);
|
||||
const updates: { name?: string; rateLimit?: number; status?: "active" | "revoked" } = {};
|
||||
if (body.name !== undefined) {
|
||||
if (
|
||||
typeof body.name !== "string" ||
|
||||
body.name.trim().length < 1 ||
|
||||
body.name.trim().length > 120
|
||||
) {
|
||||
return NextResponse.json(
|
||||
{ error: "Client name must be a non-empty string (max 120 characters).", code: "VALIDATION_ERROR" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
updates.name = body.name.trim();
|
||||
}
|
||||
if (body.rateLimit !== undefined) {
|
||||
if (
|
||||
typeof body.rateLimit !== "number" ||
|
||||
!Number.isInteger(body.rateLimit) ||
|
||||
body.rateLimit < 1 ||
|
||||
body.rateLimit > 100000
|
||||
) {
|
||||
return NextResponse.json(
|
||||
{ error: "rateLimit must be an integer between 1 and 100000.", code: "VALIDATION_ERROR" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
updates.rateLimit = body.rateLimit;
|
||||
}
|
||||
if (body.status !== undefined) {
|
||||
if (body.status !== "active" && body.status !== "revoked") {
|
||||
return NextResponse.json(
|
||||
{ error: "status must be 'active' or 'revoked'.", code: "VALIDATION_ERROR" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
updates.status = body.status;
|
||||
}
|
||||
|
||||
if (Object.keys(updates).length === 0) {
|
||||
return NextResponse.json(
|
||||
{ error: "No valid fields provided for update.", code: "VALIDATION_ERROR" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const updated = await SDKManager.updateClientAsync(normalizedId, updates);
|
||||
if (!updated) {
|
||||
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
||||
}
|
||||
return NextResponse.json({ client: updated });
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : "Update failed";
|
||||
return NextResponse.json({ error: message }, { status: 500 });
|
||||
const isDbWrite = message.includes("DATABASE_WRITE_FAILED");
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: message,
|
||||
code: isDbWrite ? "DATABASE_WRITE_FAILED" : "CLIENT_UPDATE_FAILED",
|
||||
},
|
||||
{ status: isDbWrite ? 503 : 500 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,10 +204,29 @@ export async function DELETE(
|
||||
}
|
||||
|
||||
const { id } = await params;
|
||||
const normalizedId = SDKManager.normalizeClientId(id);
|
||||
const purgeFiles = req.nextUrl.searchParams.get("purgeFiles") === "true";
|
||||
const deleted = await SDKManager.deleteClientAsync(id, { purgeFiles });
|
||||
if (!deleted) {
|
||||
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
||||
try {
|
||||
const result = await SDKManager.deleteClientWithStatsAsync(normalizedId, {
|
||||
purgeFiles,
|
||||
});
|
||||
if (!result.deleted) {
|
||||
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
||||
}
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
message: `Client ${normalizedId} deleted`,
|
||||
purgedFiles: result.purgedFiles,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : "Client deletion failed";
|
||||
const isDbWrite = message.includes("DATABASE_WRITE_FAILED");
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: message,
|
||||
code: isDbWrite ? "DATABASE_WRITE_FAILED" : "CLIENT_DELETE_FAILED",
|
||||
},
|
||||
{ status: isDbWrite ? 503 : 500 },
|
||||
);
|
||||
}
|
||||
return NextResponse.json({ success: true, message: `Client ${id} deleted` });
|
||||
}
|
||||
@@ -60,17 +60,29 @@ export async function POST(req: NextRequest) {
|
||||
try {
|
||||
const { clientId, name, rateLimit } = body;
|
||||
|
||||
if (!clientId || !name) {
|
||||
if (typeof clientId !== "string" || typeof name !== "string" || !clientId.trim() || !name.trim()) {
|
||||
return NextResponse.json(
|
||||
{ error: "clientId and name are required" },
|
||||
{ error: "clientId and name are required strings", code: "VALIDATION_ERROR" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
const safeClientId = SDKManager.normalizeClientId(clientId);
|
||||
if (!safeClientId) {
|
||||
return NextResponse.json(
|
||||
{ error: "clientId is invalid", code: "VALIDATION_ERROR" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
const parsedRateLimit = rateLimit === undefined ? 100 : Number(rateLimit);
|
||||
if (!Number.isInteger(parsedRateLimit) || parsedRateLimit < 1 || parsedRateLimit > 100000) {
|
||||
return NextResponse.json(
|
||||
{ error: "rateLimit must be an integer between 1 and 100000.", code: "VALIDATION_ERROR" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
const safeClientId = String(clientId)
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9_-]/g, "_");
|
||||
const existing = await SDKManager.getClientAsync(safeClientId);
|
||||
if (existing) {
|
||||
return NextResponse.json(
|
||||
@@ -81,8 +93,8 @@ export async function POST(req: NextRequest) {
|
||||
|
||||
const newClient = await SDKManager.registerClientAsync({
|
||||
clientId: safeClientId,
|
||||
name: String(name),
|
||||
rateLimit: rateLimit ? Number(rateLimit) : 100,
|
||||
name: name.trim().slice(0, 120),
|
||||
rateLimit: parsedRateLimit,
|
||||
});
|
||||
|
||||
return NextResponse.json({ client: newClient }, { status: 201 });
|
||||
|
||||
+88
-20
@@ -1,10 +1,13 @@
|
||||
import { SettingsManager } from "@/core/adminSettings/settingsManager";
|
||||
import { DataBufferEngine } from "@/core/data/buffer";
|
||||
import { SDKManager } from "@/core/sdk/sdkManager";
|
||||
import { getInternalIngestionHeaders } from "@/core/Security/security";
|
||||
import { CSVStorageManager } from "@/core/Storage/csvStorage";
|
||||
import { requireAdminApiSession } from "@/lib/adminAuth";
|
||||
import { parse } from "csv-parse";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { createReadStream, readdirSync, statSync } from "node:fs";
|
||||
import { createReadStream } from "node:fs";
|
||||
import { lstat, readdir } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
interface DataEvent {
|
||||
@@ -15,10 +18,40 @@ interface DataEvent {
|
||||
status: "stored" | "buffered";
|
||||
}
|
||||
|
||||
function parseTimestampMs(raw: string): number {
|
||||
const trimmed = String(raw || "").trim();
|
||||
if (/^\d+(\.\d+)?$/.test(trimmed)) {
|
||||
const num = Number(trimmed);
|
||||
return num < 1e11 ? num * 1000 : num;
|
||||
}
|
||||
const parsed = Date.parse(trimmed);
|
||||
return Number.isNaN(parsed) ? Number.NEGATIVE_INFINITY : parsed;
|
||||
}
|
||||
|
||||
function compareEventsDesc(a: DataEvent, b: DataEvent): number {
|
||||
const aMs = parseTimestampMs(a.timestamp);
|
||||
const bMs = parseTimestampMs(b.timestamp);
|
||||
if (aMs !== bMs) {
|
||||
return bMs - aMs;
|
||||
}
|
||||
return b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0;
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const auth = await requireAdminApiSession(request);
|
||||
if (!auth.valid) {
|
||||
return auth.response;
|
||||
}
|
||||
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const clientId = searchParams.get("clientId") || "all";
|
||||
const rawClientId = searchParams.get("clientId") || "all";
|
||||
const clientId =
|
||||
rawClientId === "all"
|
||||
? "all"
|
||||
: SDKManager.normalizeClientId(rawClientId) || "all";
|
||||
const sanitizedClientId =
|
||||
clientId === "all" ? "all" : CSVStorageManager.sanitizeClientId(clientId);
|
||||
const from = searchParams.get("from") || "";
|
||||
const to = searchParams.get("to") || "";
|
||||
const search = (searchParams.get("search") || "").toLowerCase();
|
||||
@@ -34,7 +67,12 @@ export async function GET(request: NextRequest) {
|
||||
|
||||
const addIfMatching = (event: DataEvent) => {
|
||||
const eventDate = CSVStorageManager.extractSafePartitionDate(event.timestamp, null);
|
||||
const dedupeKey = `${event.clientId}:${event.eventId}`;
|
||||
const normEventClient = SDKManager.normalizeClientId(event.clientId) || event.clientId;
|
||||
const dedupeKey = `${normEventClient}:${event.eventId}`;
|
||||
const matchesClient =
|
||||
clientId === "all" ||
|
||||
normEventClient === clientId ||
|
||||
CSVStorageManager.sanitizeClientId(normEventClient) === sanitizedClientId;
|
||||
const matchesSearch =
|
||||
!search ||
|
||||
event.payload.toLowerCase().includes(search) ||
|
||||
@@ -42,7 +80,7 @@ export async function GET(request: NextRequest) {
|
||||
event.clientId.toLowerCase().includes(search);
|
||||
|
||||
if (
|
||||
(clientId !== "all" && event.clientId !== clientId) ||
|
||||
!matchesClient ||
|
||||
((from || to) && (!eventDate || (from && eventDate < from) || (to && eventDate > to))) ||
|
||||
!matchesSearch ||
|
||||
eventIds.has(dedupeKey)
|
||||
@@ -57,7 +95,7 @@ export async function GET(request: NextRequest) {
|
||||
let high = selectedRows.length;
|
||||
while (low < high) {
|
||||
const middle = (low + high) >>> 1;
|
||||
if (selectedRows[middle].timestamp >= event.timestamp) low = middle + 1;
|
||||
if (compareEventsDesc(selectedRows[middle], event) <= 0) low = middle + 1;
|
||||
else high = middle;
|
||||
}
|
||||
|
||||
@@ -70,31 +108,61 @@ export async function GET(request: NextRequest) {
|
||||
const storageDir = CSVStorageManager.baseStorageDir;
|
||||
let clientDirs: string[] = [];
|
||||
try {
|
||||
clientDirs = readdirSync(storageDir);
|
||||
clientDirs = await readdir(storageDir);
|
||||
} catch {
|
||||
clientDirs = [];
|
||||
}
|
||||
|
||||
for (const storedClientId of clientDirs) {
|
||||
if (clientId !== "all" && storedClientId !== clientId) continue;
|
||||
if (
|
||||
clientId !== "all" &&
|
||||
storedClientId !== clientId &&
|
||||
storedClientId !== sanitizedClientId
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const clientDir = path.join(storageDir, storedClientId);
|
||||
if (!statSync(clientDir).isDirectory()) continue;
|
||||
let dirLstat;
|
||||
try {
|
||||
dirLstat = await lstat(clientDir);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
|
||||
|
||||
let filenames: string[];
|
||||
try {
|
||||
filenames = readdirSync(clientDir).filter((filename) =>
|
||||
/^\d{4}-\d{2}-\d{2}(?:-part\d+)?\.csv$/.test(filename),
|
||||
);
|
||||
filenames = (await readdir(clientDir))
|
||||
.filter((filename) => {
|
||||
if (!/^\d{4}-\d{2}-\d{2}(?:-part\d+)?\.csv$/.test(filename)) {
|
||||
return false;
|
||||
}
|
||||
const fileDate = filename.slice(0, 10);
|
||||
if ((from && fileDate < from) || (to && fileDate > to)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
})
|
||||
.sort((a, b) => b.localeCompare(a));
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const filename of filenames) {
|
||||
const fileDate = filename.slice(0, 10);
|
||||
if ((from && fileDate < from) || (to && fileDate > to)) continue;
|
||||
|
||||
const parser = createReadStream(path.join(clientDir, filename)).pipe(
|
||||
const filePath = path.join(clientDir, filename);
|
||||
let fileLstat;
|
||||
try {
|
||||
fileLstat = await lstat(filePath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (fileLstat.isSymbolicLink() || !fileLstat.isFile()) continue;
|
||||
|
||||
const fileStream = createReadStream(filePath);
|
||||
const parser = fileStream.pipe(
|
||||
parse({ columns: true, skip_empty_lines: true, bom: true }),
|
||||
);
|
||||
|
||||
@@ -111,17 +179,17 @@ export async function GET(request: NextRequest) {
|
||||
};
|
||||
if (event.eventId) addIfMatching(event);
|
||||
}
|
||||
} catch {
|
||||
parser.destroy();
|
||||
} catch (err) {
|
||||
console.warn(`[API /data] Failed reading CSV partition ${filePath}:`, err);
|
||||
} finally {
|
||||
if (!parser.destroyed) parser.destroy();
|
||||
if (!fileStream.destroyed) fileStream.destroy();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const settings = await SettingsManager.getSettingsAsync();
|
||||
const recentWindowLimit = Math.min(
|
||||
2000,
|
||||
Math.max(200, settings.batchFlushThreshold || 500),
|
||||
);
|
||||
await SettingsManager.getSettingsAsync();
|
||||
const recentWindowLimit = DataBufferEngine.getRecentLimit();
|
||||
|
||||
for (const event of await DataBufferEngine.getRecentEventsAsync(
|
||||
recentWindowLimit,
|
||||
|
||||
+296
-41
@@ -1,33 +1,240 @@
|
||||
import { SettingsManager } from '@/core/adminSettings/settingsManager';
|
||||
import { verifyRequestAdminSession } from '@/core/auth/adminUsers';
|
||||
import { AuthService } from '@/core/auth/jwt';
|
||||
import { DataBufferEngine } from '@/core/data/buffer';
|
||||
import { DataBufferEngine, REDIS_BUFFER_KEY } from '@/core/data/buffer';
|
||||
import { SDKManager } from '@/core/sdk/sdkManager';
|
||||
import {
|
||||
SecurityEngine,
|
||||
getInternalIngestionHeaders,
|
||||
} from '@/core/Security/security';
|
||||
import { CSVStorageManager, type DataEvent } from '@/core/Storage/csvStorage';
|
||||
import {
|
||||
CSVStorageManager,
|
||||
type DataEvent,
|
||||
type StorageFileMeta,
|
||||
} from '@/core/Storage/csvStorage';
|
||||
import { parse } from 'csv-parse';
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import crypto from 'node:crypto';
|
||||
import { createReadStream } from 'node:fs';
|
||||
import { requireAdminApiSession } from '@/lib/adminAuth';
|
||||
import type { BufferedBucket, BufferedSummary } from '@/lib/vconUiHelpers';
|
||||
|
||||
const SESSION_COOKIE = 'datastack_admin_session';
|
||||
const SINGLE_EVENT_ENVELOPE_KEYS = new Set(['eventId', 'timestamp', 'data', 'clientId']);
|
||||
const MAX_SINGLE_CLIENT_SCAN_FILES = 10;
|
||||
const MAX_ALL_CLIENTS_PER_CLIENT_FILES = 2;
|
||||
const MAX_ALL_CLIENTS_TOTAL_SCAN_FILES = 50;
|
||||
|
||||
function generateFallbackEventId(): string {
|
||||
return `evt_${Date.now()}_${crypto.randomBytes(6).toString('hex')}`;
|
||||
}
|
||||
|
||||
function extractSingleHttpEventData(rawBody: Record<string, unknown>): unknown {
|
||||
if (rawBody.data !== undefined) {
|
||||
const keys = Object.keys(rawBody);
|
||||
if (keys.every((k) => SINGLE_EVENT_ENVELOPE_KEYS.has(k))) {
|
||||
return rawBody.data;
|
||||
}
|
||||
return rawBody;
|
||||
}
|
||||
|
||||
const stripped: Record<string, unknown> = {};
|
||||
let hasNonEnvelopeKey = false;
|
||||
for (const [k, v] of Object.entries(rawBody)) {
|
||||
if (k === 'eventId' || k === 'timestamp' || k === 'clientId') {
|
||||
continue;
|
||||
}
|
||||
hasNonEnvelopeKey = true;
|
||||
stripped[k] = v;
|
||||
}
|
||||
return hasNonEnvelopeKey ? stripped : {};
|
||||
}
|
||||
|
||||
function parseTimestampMs(raw: string): number {
|
||||
const trimmed = String(raw || '').trim();
|
||||
if (/^\d+(\.\d+)?$/.test(trimmed)) {
|
||||
const num = Number(trimmed);
|
||||
return num < 1e11 ? num * 1000 : num;
|
||||
}
|
||||
const parsed = Date.parse(trimmed);
|
||||
return Number.isNaN(parsed) ? Number.NEGATIVE_INFINITY : parsed;
|
||||
}
|
||||
|
||||
function resolveEventUtcDate(rawTimestamp: string): string {
|
||||
const ms = parseTimestampMs(rawTimestamp);
|
||||
if (Number.isFinite(ms)) {
|
||||
const parsed = new Date(ms);
|
||||
if (!Number.isNaN(parsed.getTime())) {
|
||||
return parsed.toISOString().slice(0, 10);
|
||||
}
|
||||
}
|
||||
return new Date().toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function selectRecentPartitionFiles(
|
||||
allFiles: StorageFileMeta[],
|
||||
normClientId?: string,
|
||||
sanitizedDir?: string,
|
||||
): StorageFileMeta[] {
|
||||
const matching = allFiles
|
||||
.filter(
|
||||
(f) =>
|
||||
!normClientId ||
|
||||
f.clientId === normClientId ||
|
||||
f.clientId === sanitizedDir,
|
||||
)
|
||||
.sort((a, b) => {
|
||||
if (a.date !== b.date) return b.date.localeCompare(a.date);
|
||||
return new Date(b.modifiedAt).getTime() - new Date(a.modifiedAt).getTime();
|
||||
});
|
||||
|
||||
if (normClientId) {
|
||||
return matching.slice(0, MAX_SINGLE_CLIENT_SCAN_FILES);
|
||||
}
|
||||
|
||||
// Multi-client fair scan: take up to 2 newest partitions per active client (capped at 50 files total)
|
||||
// so clients beyond the 5th most-recently-modified file are never silently hidden.
|
||||
const perClientCounts = new Map<string, number>();
|
||||
const selected: StorageFileMeta[] = [];
|
||||
|
||||
for (const file of matching) {
|
||||
const currentCount = perClientCounts.get(file.clientId) ?? 0;
|
||||
if (currentCount >= MAX_ALL_CLIENTS_PER_CLIENT_FILES) continue;
|
||||
perClientCounts.set(file.clientId, currentCount + 1);
|
||||
selected.push(file);
|
||||
if (selected.length >= MAX_ALL_CLIENTS_TOTAL_SCAN_FILES) break;
|
||||
}
|
||||
|
||||
return selected;
|
||||
}
|
||||
|
||||
async function getPendingBufferedSummary(
|
||||
ingestionBaseUrl: string,
|
||||
filterClientId?: string,
|
||||
): Promise<{ events: DataEvent[]; summary: BufferedSummary }> {
|
||||
const bufferStats = await DataBufferEngine.getBufferStatsAsync();
|
||||
const recentLimit = DataBufferEngine.getRecentLimit();
|
||||
const pending: DataEvent[] = [];
|
||||
let totalQueued = bufferStats.localBufferSize + bufferStats.redisBufferSize;
|
||||
|
||||
// 1. Read exact unflushed Redis queue (when Redis is enabled & connected)
|
||||
const redis = DataBufferEngine.getRedisClient();
|
||||
if (redis && bufferStats.redisBufferSize > 0) {
|
||||
try {
|
||||
const cap = Math.min(recentLimit, bufferStats.redisBufferSize);
|
||||
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, cap - 1);
|
||||
for (const raw of rawItems) {
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as DataEvent;
|
||||
if (!parsed || !parsed.eventId) continue;
|
||||
pending.push(parsed);
|
||||
} catch {
|
||||
// Skip malformed queued item
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Fall back to local buffer telemetry
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Read Port 3000 exact unflushed RAM queue snapshot
|
||||
if (bufferStats.localBufferSize > 0) {
|
||||
const localPending = DataBufferEngine.getLocalBufferSnapshot(recentLimit);
|
||||
pending.push(...localPending);
|
||||
}
|
||||
|
||||
// 3. When Redis is not enabled or temporarily disconnected, Port 4000 `/internal/recent` serves its unflushed RAM `bufferQueue`
|
||||
if (!bufferStats.redisEnabled || !bufferStats.redisConnected) {
|
||||
try {
|
||||
const qs = new URLSearchParams({ limit: String(recentLimit) });
|
||||
const res = await fetch(`${ingestionBaseUrl}/internal/recent?${qs.toString()}`, {
|
||||
headers: getInternalIngestionHeaders(),
|
||||
cache: 'no-store',
|
||||
signal: AbortSignal.timeout(1000),
|
||||
});
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
if (Array.isArray(data.events) && data.events.length > 0) {
|
||||
pending.push(...(data.events as DataEvent[]));
|
||||
totalQueued += data.events.length;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
}
|
||||
}
|
||||
|
||||
const seen = new Set<string>();
|
||||
const dedupedAll: DataEvent[] = [];
|
||||
const filteredEvents: DataEvent[] = [];
|
||||
const bucketMap = new Map<string, BufferedBucket>();
|
||||
const sanitizedFilter = filterClientId
|
||||
? CSVStorageManager.sanitizeClientId(filterClientId)
|
||||
: undefined;
|
||||
|
||||
for (const evt of pending) {
|
||||
if (!evt || !evt.eventId) continue;
|
||||
const normEvtClient = SDKManager.normalizeClientId(evt.clientId) || evt.clientId;
|
||||
const dedupeKey = `${normEvtClient}:${evt.eventId}`;
|
||||
if (seen.has(dedupeKey)) continue;
|
||||
seen.add(dedupeKey);
|
||||
dedupedAll.push(evt);
|
||||
|
||||
const matchesFilter =
|
||||
!filterClientId ||
|
||||
normEvtClient === filterClientId ||
|
||||
evt.clientId === filterClientId ||
|
||||
evt.clientId === sanitizedFilter;
|
||||
|
||||
if (matchesFilter) {
|
||||
filteredEvents.push(evt);
|
||||
const utcDate = resolveEventUtcDate(evt.timestamp);
|
||||
const bucketKey = `${normEvtClient}|${utcDate}`;
|
||||
const existing = bucketMap.get(bucketKey);
|
||||
if (existing) {
|
||||
existing.count += 1;
|
||||
} else {
|
||||
bucketMap.set(bucketKey, {
|
||||
clientId: normEvtClient,
|
||||
date: utcDate,
|
||||
count: 1,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const effectiveTotalQueued = Math.max(totalQueued, dedupedAll.length);
|
||||
const unattributedCount = Math.max(0, effectiveTotalQueued - dedupedAll.length);
|
||||
|
||||
return {
|
||||
events: filteredEvents,
|
||||
summary: {
|
||||
total: filteredEvents.length,
|
||||
totalQueued: filterClientId
|
||||
? filteredEvents.length + unattributedCount
|
||||
: effectiveTotalQueued,
|
||||
unattributedCount,
|
||||
truncated: unattributedCount > 0,
|
||||
buckets: Array.from(bucketMap.values()),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function getRecentStoredEvents(
|
||||
limit: number,
|
||||
clientId?: string,
|
||||
): Promise<DataEvent[]> {
|
||||
const normClientId = clientId && clientId !== 'all' ? SDKManager.normalizeClientId(clientId) : undefined;
|
||||
const sanitizedDir = normClientId ? CSVStorageManager.sanitizeClientId(normClientId) : undefined;
|
||||
const allFiles = await CSVStorageManager.listAllFilesAsync();
|
||||
const files = allFiles
|
||||
.filter((f) => !clientId || clientId === 'all' || f.clientId === clientId)
|
||||
.slice(0, 5);
|
||||
const files = selectRecentPartitionFiles(allFiles, normClientId, sanitizedDir);
|
||||
|
||||
const collected: DataEvent[] = [];
|
||||
|
||||
for (const file of files) {
|
||||
const fileTail: DataEvent[] = [];
|
||||
const parser = createReadStream(file.filePath).pipe(
|
||||
const fileStream = createReadStream(file.filePath);
|
||||
const parser = fileStream.pipe(
|
||||
parse({ columns: true, skip_empty_lines: true, bom: true }),
|
||||
);
|
||||
|
||||
@@ -46,8 +253,11 @@ async function getRecentStoredEvents(
|
||||
payload: record.payload || '',
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
parser.destroy();
|
||||
} catch (err) {
|
||||
console.warn(`[API /events] Failed reading CSV partition ${file.filePath}:`, err);
|
||||
} finally {
|
||||
if (!parser.destroyed) parser.destroy();
|
||||
if (!fileStream.destroyed) fileStream.destroy();
|
||||
}
|
||||
|
||||
if (fileTail.length > 0) {
|
||||
@@ -59,18 +269,47 @@ async function getRecentStoredEvents(
|
||||
}
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
const auth = await requireAdminApiSession(req);
|
||||
if (!auth.valid) {
|
||||
return auth.response;
|
||||
}
|
||||
|
||||
try {
|
||||
const { searchParams } = new URL(req.url);
|
||||
const clientId = searchParams.get('clientId') || undefined;
|
||||
const rawClientId = searchParams.get('clientId');
|
||||
const clientId =
|
||||
rawClientId && rawClientId !== 'all'
|
||||
? SDKManager.normalizeClientId(rawClientId) || undefined
|
||||
: undefined;
|
||||
const limit = Math.min(2000, Math.max(1, Number(searchParams.get('limit')) || 50));
|
||||
const bufferedOnly = searchParams.get('bufferedOnly') === 'true';
|
||||
const includeBufferedSummary =
|
||||
bufferedOnly || searchParams.get('includeBufferedSummary') === 'true';
|
||||
|
||||
await SettingsManager.getSettingsAsync();
|
||||
const bufferedEvents = await DataBufferEngine.getRecentEventsAsync(limit, clientId);
|
||||
let socketBufferedEvents: DataEvent[] = [];
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
|
||||
).replace(/\/$/, '');
|
||||
|
||||
if (bufferedOnly) {
|
||||
const pendingResult = await getPendingBufferedSummary(ingestionBaseUrl, clientId);
|
||||
const sortedPending = [...pendingResult.events].sort((a, b) => {
|
||||
const aMs = parseTimestampMs(a.timestamp);
|
||||
const bMs = parseTimestampMs(b.timestamp);
|
||||
if (aMs !== bMs) return bMs - aMs;
|
||||
return b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0;
|
||||
});
|
||||
const events = sortedPending.slice(0, limit);
|
||||
return NextResponse.json({
|
||||
events,
|
||||
count: events.length,
|
||||
bufferedSummary: pendingResult.summary,
|
||||
});
|
||||
}
|
||||
|
||||
const bufferedEvents = await DataBufferEngine.getRecentEventsAsync(limit, clientId);
|
||||
let socketBufferedEvents: DataEvent[] = [];
|
||||
|
||||
try {
|
||||
const qs = new URLSearchParams({ limit: String(limit) });
|
||||
if (clientId) qs.set('clientId', clientId);
|
||||
@@ -89,23 +328,38 @@ export async function GET(req: NextRequest) {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
}
|
||||
|
||||
const storedEvents = await getRecentStoredEvents(limit, clientId);
|
||||
const [storedEvents, pendingResult] = await Promise.all([
|
||||
getRecentStoredEvents(limit, clientId),
|
||||
includeBufferedSummary
|
||||
? getPendingBufferedSummary(ingestionBaseUrl, clientId)
|
||||
: Promise.resolve(null),
|
||||
]);
|
||||
|
||||
const seenIds = new Set<string>();
|
||||
const merged: DataEvent[] = [];
|
||||
|
||||
for (const evt of [...bufferedEvents, ...socketBufferedEvents, ...storedEvents]) {
|
||||
if (!evt.eventId) continue;
|
||||
const dedupeKey = `${evt.clientId}:${evt.eventId}`;
|
||||
const normEvtClient = SDKManager.normalizeClientId(evt.clientId) || evt.clientId;
|
||||
const dedupeKey = `${normEvtClient}:${evt.eventId}`;
|
||||
if (seenIds.has(dedupeKey)) continue;
|
||||
seenIds.add(dedupeKey);
|
||||
merged.push(evt);
|
||||
}
|
||||
|
||||
merged.sort((a, b) => (b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0));
|
||||
merged.sort((a, b) => {
|
||||
const aMs = parseTimestampMs(a.timestamp);
|
||||
const bMs = parseTimestampMs(b.timestamp);
|
||||
if (aMs !== bMs) return bMs - aMs;
|
||||
return b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0;
|
||||
});
|
||||
const events = merged.slice(0, limit);
|
||||
|
||||
return NextResponse.json({ events, count: events.length });
|
||||
return NextResponse.json({
|
||||
events,
|
||||
count: events.length,
|
||||
...(pendingResult ? { bufferedSummary: pendingResult.summary } : {}),
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : 'Events retrieval failed';
|
||||
return NextResponse.json({ error: message }, { status: 500 });
|
||||
@@ -165,12 +419,18 @@ export async function POST(req: NextRequest) {
|
||||
? (body as Record<string, unknown>)
|
||||
: {};
|
||||
|
||||
const clientId = String(
|
||||
tokenClientId || bodyObj.clientId || 'py_telemetry_agent_01',
|
||||
)
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9_-]/g, '_');
|
||||
const clientId = SDKManager.normalizeClientId(
|
||||
String(tokenClientId || bodyObj.clientId || ''),
|
||||
);
|
||||
if (!clientId) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: 'clientId is required and must belong to a registered client.',
|
||||
code: 'VALIDATION_ERROR',
|
||||
},
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
if (await SDKManager.isClientDeletedAsync(clientId)) {
|
||||
return NextResponse.json(
|
||||
@@ -183,7 +443,7 @@ export async function POST(req: NextRequest) {
|
||||
}
|
||||
|
||||
const registeredClient = await SDKManager.getClientAsync(clientId);
|
||||
if (rawBearerToken && !registeredClient) {
|
||||
if (!registeredClient) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: `CLIENT_REVOKED: Client '${clientId}' registration not found or has been deleted.`,
|
||||
@@ -231,10 +491,11 @@ export async function POST(req: NextRequest) {
|
||||
? (bodyObj.batch as unknown[])
|
||||
: Array.isArray(bodyObj.records)
|
||||
? (bodyObj.records as unknown[])
|
||||
: null;
|
||||
: bodyObj.data === undefined && Array.isArray(bodyObj.events)
|
||||
? (bodyObj.events as unknown[])
|
||||
: null;
|
||||
|
||||
if (batchItems !== null) {
|
||||
const envelopeKeys = new Set(['eventId', 'timestamp', 'data', 'clientId']);
|
||||
let processed = 0;
|
||||
let rejected = 0;
|
||||
let duplicates = 0;
|
||||
@@ -255,19 +516,16 @@ export async function POST(req: NextRequest) {
|
||||
const itemEventId =
|
||||
typeof safeItem.eventId === 'string' && safeItem.eventId.trim()
|
||||
? safeItem.eventId.trim()
|
||||
: `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||
: generateFallbackEventId();
|
||||
|
||||
const itemTimestamp =
|
||||
typeof safeItem.timestamp === 'string' && safeItem.timestamp.trim()
|
||||
? safeItem.timestamp.trim()
|
||||
: new Date().toISOString();
|
||||
|
||||
const isEnvelope =
|
||||
safeItem.data !== undefined &&
|
||||
Object.keys(safeItem).every((k) => envelopeKeys.has(k));
|
||||
const itemData = isEnvelope ? safeItem.data : safeItem;
|
||||
const itemData = extractSingleHttpEventData(safeItem);
|
||||
|
||||
const val = SecurityEngine.validateEvent(
|
||||
const val = await SecurityEngine.validateEventAsync(
|
||||
clientId,
|
||||
itemEventId,
|
||||
itemData,
|
||||
@@ -309,7 +567,7 @@ export async function POST(req: NextRequest) {
|
||||
}
|
||||
|
||||
if (processed > 0) {
|
||||
SDKManager.incrementEventCount(clientId, processed);
|
||||
await SDKManager.incrementEventCountAsync(clientId, processed);
|
||||
}
|
||||
|
||||
// If zero items were processed or deduplicated and all failed due to payload or rate limit, return corresponding status
|
||||
@@ -323,6 +581,7 @@ export async function POST(req: NextRequest) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
accepted: processed,
|
||||
processed,
|
||||
rejected,
|
||||
duplicates,
|
||||
@@ -338,6 +597,7 @@ export async function POST(req: NextRequest) {
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
accepted: processed,
|
||||
processed,
|
||||
rejected,
|
||||
duplicates,
|
||||
@@ -350,24 +610,19 @@ export async function POST(req: NextRequest) {
|
||||
});
|
||||
}
|
||||
|
||||
// Single Event Ingestion
|
||||
// Single Event Ingestion (exact envelope parity with Port 4000 WS sync_data)
|
||||
const eventId =
|
||||
typeof bodyObj.eventId === 'string' && bodyObj.eventId.trim()
|
||||
? bodyObj.eventId.trim()
|
||||
: `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||
: generateFallbackEventId();
|
||||
const timestamp =
|
||||
typeof bodyObj.timestamp === 'string' && bodyObj.timestamp.trim()
|
||||
? bodyObj.timestamp.trim()
|
||||
: new Date().toISOString();
|
||||
const data =
|
||||
bodyObj.data !== undefined
|
||||
? bodyObj.data
|
||||
: Object.keys(bodyObj).length > 0
|
||||
? bodyObj
|
||||
: { sample: 'Realtime event via DataStack API', rand: Math.random() };
|
||||
const data = extractSingleHttpEventData(bodyObj);
|
||||
|
||||
// Validate security (idempotency, rate limit, payload size)
|
||||
const val = SecurityEngine.validateEvent(
|
||||
const val = await SecurityEngine.validateEventAsync(
|
||||
clientId,
|
||||
eventId,
|
||||
data,
|
||||
@@ -393,7 +648,7 @@ export async function POST(req: NextRequest) {
|
||||
};
|
||||
|
||||
await DataBufferEngine.pushAsync(event);
|
||||
SDKManager.incrementEventCount(clientId, 1);
|
||||
await SDKManager.incrementEventCountAsync(clientId, 1);
|
||||
|
||||
return NextResponse.json({ success: true, eventId, event });
|
||||
} catch (err: unknown) {
|
||||
|
||||
+127
-22
@@ -1,10 +1,12 @@
|
||||
import { SettingsManager } from "@/core/adminSettings/settingsManager";
|
||||
import { DataBufferEngine } from "@/core/data/buffer";
|
||||
import { SDKManager } from "@/core/sdk/sdkManager";
|
||||
import { getInternalIngestionHeaders } from "@/core/Security/security";
|
||||
import {
|
||||
CSVStorageManager,
|
||||
type StorageFileMeta,
|
||||
} from "@/core/Storage/csvStorage";
|
||||
import { requireAdminApiSession } from "@/lib/adminAuth";
|
||||
import { parse } from "csv-parse";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { createReadStream } from "node:fs";
|
||||
@@ -44,7 +46,8 @@ function toByteStream(chunks: AsyncIterable<StreamChunk>) {
|
||||
async function* readFileRecords(
|
||||
file: StorageFileMeta,
|
||||
): AsyncGenerator<CsvRecord> {
|
||||
const parser = createReadStream(file.filePath).pipe(
|
||||
const fileStream = createReadStream(file.filePath);
|
||||
const parser = fileStream.pipe(
|
||||
parse({ columns: true, skip_empty_lines: true, bom: true }),
|
||||
);
|
||||
|
||||
@@ -52,8 +55,9 @@ async function* readFileRecords(
|
||||
for await (const record of parser as AsyncIterable<CsvRecord>) {
|
||||
yield record;
|
||||
}
|
||||
} catch {
|
||||
parser.destroy();
|
||||
} finally {
|
||||
if (!parser.destroyed) parser.destroy();
|
||||
if (!fileStream.destroyed) fileStream.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,7 +89,7 @@ function makeExportStream(
|
||||
yield `${isFirst ? "" : ","}${JSON.stringify({ ...record, payload: parsedPayload })}`;
|
||||
} else {
|
||||
const line = fields
|
||||
.map((field) => `"${(record[field] ?? "").replaceAll('"', '""')}"`)
|
||||
.map((field) => `"${String(record[field] ?? "").replaceAll('"', '""')}"`)
|
||||
.join(",");
|
||||
yield `${line}\n`;
|
||||
}
|
||||
@@ -101,8 +105,13 @@ function makeExportStream(
|
||||
|
||||
function streamFile(file: StorageFileMeta) {
|
||||
async function* chunks() {
|
||||
for await (const chunk of createReadStream(file.filePath)) {
|
||||
yield new Uint8Array(chunk);
|
||||
const fileStream = createReadStream(file.filePath);
|
||||
try {
|
||||
for await (const chunk of fileStream) {
|
||||
yield new Uint8Array(chunk as Buffer);
|
||||
}
|
||||
} finally {
|
||||
if (!fileStream.destroyed) fileStream.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -110,38 +119,122 @@ function streamFile(file: StorageFileMeta) {
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const auth = await requireAdminApiSession(request);
|
||||
if (!auth.valid) {
|
||||
return auth.response;
|
||||
}
|
||||
|
||||
try {
|
||||
const { searchParams } = new URL(request.url);
|
||||
const from = (searchParams.get("from") || "").trim();
|
||||
const to = (searchParams.get("to") || "").trim();
|
||||
if (from && to && from > to) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: 'Start date ("from") must be on or before end date ("to").',
|
||||
code: "INVALID_DATE_RANGE",
|
||||
},
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
const shouldFlush = searchParams.get("flush") !== "false";
|
||||
|
||||
await SettingsManager.getSettingsAsync();
|
||||
if (shouldFlush) {
|
||||
await DataBufferEngine.flush();
|
||||
try {
|
||||
await DataBufferEngine.flush({ waitMs: 2500, throwOnLockTimeout: true });
|
||||
} catch (err: unknown) {
|
||||
const message =
|
||||
err instanceof Error ? err.message : "Pre-export flush failed";
|
||||
if (message.includes("FLUSH_LOCK_TIMEOUT")) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: message,
|
||||
code: "FLUSH_LOCK_TIMEOUT",
|
||||
},
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
const explicitIngestionUrl = Boolean(process.env.INGESTION_INTERNAL_URL?.trim());
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||
).replace(/\/$/, "");
|
||||
try {
|
||||
await fetch(`${ingestionBaseUrl}/internal/flush`, {
|
||||
const flushRes = await fetch(`${ingestionBaseUrl}/internal/flush`, {
|
||||
method: "POST",
|
||||
headers: getInternalIngestionHeaders(),
|
||||
signal: AbortSignal.timeout(1500),
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...getInternalIngestionHeaders(),
|
||||
},
|
||||
body: JSON.stringify({ waitMs: 2500, throwOnLockTimeout: true }),
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(4000),
|
||||
});
|
||||
} catch {
|
||||
if (!flushRes.ok) {
|
||||
const errPayload = await flushRes.json().catch(() => ({}));
|
||||
const remoteErr =
|
||||
typeof errPayload?.error === "string"
|
||||
? errPayload.error
|
||||
: `Ingestion flush returned HTTP ${flushRes.status}`;
|
||||
if (
|
||||
flushRes.status === 503 &&
|
||||
errPayload?.code === "FLUSH_LOCK_TIMEOUT"
|
||||
) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: remoteErr,
|
||||
code: "FLUSH_LOCK_TIMEOUT",
|
||||
ingestionStatus: flushRes.status,
|
||||
},
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: remoteErr,
|
||||
code: "INGESTION_FLUSH_FAILED",
|
||||
ingestionStatus: flushRes.status,
|
||||
},
|
||||
{ status: 502 },
|
||||
);
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
if (explicitIngestionUrl) {
|
||||
const errMsg =
|
||||
err instanceof Error ? err.message : "Ingestion server unreachable";
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: `Ingestion server flush failed before export: ${errMsg}`,
|
||||
code: "INGESTION_UNREACHABLE",
|
||||
},
|
||||
{ status: 502 },
|
||||
);
|
||||
}
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
}
|
||||
}
|
||||
|
||||
const clientId = searchParams.get("clientId") || "all";
|
||||
const rawClientId = searchParams.get("clientId") || "all";
|
||||
const clientId =
|
||||
rawClientId === "all"
|
||||
? "all"
|
||||
: SDKManager.normalizeClientId(rawClientId) || "all";
|
||||
const sanitizedClientId =
|
||||
clientId === "all" ? "all" : CSVStorageManager.sanitizeClientId(clientId);
|
||||
const filename = searchParams.get("filename");
|
||||
const from = searchParams.get("from") || "";
|
||||
const to = searchParams.get("to") || "";
|
||||
const format: ExportFormat =
|
||||
searchParams.get("format") === "json" ? "json" : "csv";
|
||||
const rawFormat = searchParams.get("format");
|
||||
const format: ExportFormat = rawFormat === "json" ? "json" : "csv";
|
||||
const includeHeader = searchParams.get("includeHeader") !== "false";
|
||||
|
||||
let files = await CSVStorageManager.listAllFilesAsync();
|
||||
if (clientId !== "all")
|
||||
files = files.filter((file) => file.clientId === clientId);
|
||||
if (clientId !== "all") {
|
||||
files = files.filter(
|
||||
(file) => file.clientId === clientId || file.clientId === sanitizedClientId,
|
||||
);
|
||||
}
|
||||
|
||||
if (filename) {
|
||||
const safeFilename = path.basename(filename);
|
||||
@@ -166,11 +259,23 @@ export async function GET(request: NextRequest) {
|
||||
);
|
||||
}
|
||||
const file = matches[0];
|
||||
return new NextResponse(streamFile(file), {
|
||||
const downloadName =
|
||||
format === "json"
|
||||
? `${file.filename.replace(/\.csv$/i, "")}.json`
|
||||
: file.filename;
|
||||
const bodyStream =
|
||||
format === "json" || !includeHeader
|
||||
? makeExportStream([file], format, includeHeader)
|
||||
: streamFile(file);
|
||||
return new NextResponse(bodyStream, {
|
||||
status: 200,
|
||||
headers: {
|
||||
"Content-Type": "text/csv; charset=utf-8",
|
||||
"Content-Disposition": `attachment; filename="${file.filename}"`,
|
||||
"Content-Type":
|
||||
format === "json"
|
||||
? "application/json; charset=utf-8"
|
||||
: "text/csv; charset=utf-8",
|
||||
"Content-Disposition": `attachment; filename="${downloadName.replace(/"/g, "")}"`,
|
||||
"Cache-Control": "no-store",
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -179,7 +284,7 @@ export async function GET(request: NextRequest) {
|
||||
(file) => (!from || file.date >= from) && (!to || file.date <= to),
|
||||
);
|
||||
const datePart = new Date().toISOString().slice(0, 10);
|
||||
const safeClientId = clientId.replace(/[^a-zA-Z0-9_-]/g, "_");
|
||||
const safeClientId = sanitizedClientId === "all" ? "all" : sanitizedClientId;
|
||||
const baseName =
|
||||
clientId === "all" ? "datastack_export" : `${safeClientId}_export`;
|
||||
|
||||
|
||||
+33
-25
@@ -23,10 +23,10 @@ export async function GET(req: NextRequest) {
|
||||
);
|
||||
}
|
||||
|
||||
const clientId = String(verified.payload.clientId || '')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
if (SDKManager.isClientDeleted(clientId)) {
|
||||
const clientId = SDKManager.normalizeClientId(
|
||||
String(verified.payload.clientId || ''),
|
||||
);
|
||||
if (await SDKManager.isClientDeletedAsync(clientId)) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
status: 'error',
|
||||
@@ -37,27 +37,35 @@ export async function GET(req: NextRequest) {
|
||||
);
|
||||
}
|
||||
const registeredClient = await SDKManager.getClientAsync(clientId);
|
||||
if (registeredClient) {
|
||||
if (registeredClient.status === 'revoked') {
|
||||
return NextResponse.json(
|
||||
{
|
||||
status: 'error',
|
||||
code: 'CLIENT_REVOKED',
|
||||
error: `CLIENT_REVOKED: Client '${clientId}' access has been revoked by administrator.`,
|
||||
},
|
||||
{ status: 403 },
|
||||
);
|
||||
}
|
||||
if (registeredClient.token && registeredClient.token !== rawToken) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
status: 'error',
|
||||
code: 'TOKEN_ROTATED',
|
||||
error: 'TOKEN_ROTATED: Client credentials have been rotated. Use the latest token.',
|
||||
},
|
||||
{ status: 401 },
|
||||
);
|
||||
}
|
||||
if (!registeredClient) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
status: 'error',
|
||||
code: 'CLIENT_REVOKED',
|
||||
error: `CLIENT_REVOKED: Client '${clientId}' registration not found or has been deleted.`,
|
||||
},
|
||||
{ status: 403 },
|
||||
);
|
||||
}
|
||||
if (registeredClient.status === 'revoked') {
|
||||
return NextResponse.json(
|
||||
{
|
||||
status: 'error',
|
||||
code: 'CLIENT_REVOKED',
|
||||
error: `CLIENT_REVOKED: Client '${clientId}' access has been revoked by administrator.`,
|
||||
},
|
||||
{ status: 403 },
|
||||
);
|
||||
}
|
||||
if (registeredClient.token && registeredClient.token !== rawToken) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
status: 'error',
|
||||
code: 'TOKEN_ROTATED',
|
||||
error: 'TOKEN_ROTATED: Client credentials have been rotated. Use the latest token.',
|
||||
},
|
||||
{ status: 401 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,13 @@
|
||||
import { NextResponse } from 'next/server';
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { ApiEngine } from '@/core/apiEngine/apiService';
|
||||
import { requireAdminApiSession } from '@/lib/adminAuth';
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
const auth = await requireAdminApiSession(req);
|
||||
if (!auth.valid) {
|
||||
return auth.response;
|
||||
}
|
||||
|
||||
export async function GET() {
|
||||
try {
|
||||
const metrics = await ApiEngine.getDashboardMetrics();
|
||||
return NextResponse.json(metrics);
|
||||
|
||||
@@ -188,9 +188,24 @@ export async function POST(req: NextRequest) {
|
||||
|
||||
await SettingsManager.updateSettingsAsync(validated);
|
||||
const settings = await SettingsManager.getPublicSettingsAsync();
|
||||
const bridgeSync = SettingsManager.getLastBridgeSyncStatus();
|
||||
return NextResponse.json({
|
||||
settings,
|
||||
success: true,
|
||||
...(bridgeSync
|
||||
? {
|
||||
bridgeSync: {
|
||||
attempted: bridgeSync.attempted ?? true,
|
||||
ok: bridgeSync.ok ?? bridgeSync.synced,
|
||||
synced: bridgeSync.synced,
|
||||
...(bridgeSync.status !== undefined
|
||||
? { status: bridgeSync.status }
|
||||
: {}),
|
||||
...(bridgeSync.error ? { error: bridgeSync.error } : {}),
|
||||
checkedAt: bridgeSync.checkedAt,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
const message =
|
||||
|
||||
+70
-11
@@ -1,24 +1,44 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { SettingsManager } from '@/core/adminSettings/settingsManager';
|
||||
import { SDKManager } from '@/core/sdk/sdkManager';
|
||||
import { getInternalIngestionHeaders } from '@/core/Security/security';
|
||||
import { CSVStorageManager } from '@/core/Storage/csvStorage';
|
||||
import { DataBufferEngine } from '@/core/data/buffer';
|
||||
import { requireAdminApiSession } from '@/lib/adminAuth';
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
const auth = await requireAdminApiSession(req);
|
||||
if (!auth.valid) {
|
||||
return auth.response;
|
||||
}
|
||||
|
||||
try {
|
||||
const { searchParams } = new URL(req.url);
|
||||
const clientId = searchParams.get('clientId') || undefined;
|
||||
const rawClientId = searchParams.get('clientId');
|
||||
const clientId =
|
||||
rawClientId && rawClientId !== 'all'
|
||||
? SDKManager.normalizeClientId(rawClientId)
|
||||
: undefined;
|
||||
const sanitizedClientId = clientId
|
||||
? CSVStorageManager.sanitizeClientId(clientId)
|
||||
: undefined;
|
||||
|
||||
const [stats, allFiles] = await Promise.all([
|
||||
CSVStorageManager.getStorageStatsAsync(),
|
||||
CSVStorageManager.listAllFilesAsync(),
|
||||
]);
|
||||
const allFiles = await CSVStorageManager.listAllFilesAsync();
|
||||
let files = allFiles;
|
||||
|
||||
if (clientId && clientId !== 'all') {
|
||||
files = files.filter((f) => f.clientId === clientId);
|
||||
if (clientId) {
|
||||
files = allFiles.filter((f) => {
|
||||
const normFolder = SDKManager.normalizeClientId(f.clientId);
|
||||
return (
|
||||
normFolder === clientId ||
|
||||
f.clientId === clientId ||
|
||||
f.clientId === sanitizedClientId
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
const stats = CSVStorageManager.formatStorageStats(files);
|
||||
|
||||
return NextResponse.json({ stats, files });
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : 'Storage stats failed';
|
||||
@@ -26,12 +46,20 @@ export async function GET(req: NextRequest) {
|
||||
}
|
||||
}
|
||||
|
||||
export async function POST(_req: NextRequest) {
|
||||
export async function POST(req: NextRequest) {
|
||||
const auth = await requireAdminApiSession(req);
|
||||
if (!auth.valid) {
|
||||
return auth.response;
|
||||
}
|
||||
|
||||
try {
|
||||
await SettingsManager.getSettingsAsync();
|
||||
|
||||
// 1. Flush local / shared Next.js DataBufferEngine
|
||||
const localRes = await DataBufferEngine.flush();
|
||||
const localRes = await DataBufferEngine.flush({
|
||||
waitMs: 2500,
|
||||
throwOnLockTimeout: true,
|
||||
});
|
||||
|
||||
// 2. Also flush Port 4000 Socket.IO ingestion buffer
|
||||
let remoteCount = 0;
|
||||
@@ -46,9 +74,13 @@ export async function POST(_req: NextRequest) {
|
||||
try {
|
||||
const ingestRes = await fetch(`${ingestionBaseUrl}/internal/flush`, {
|
||||
method: 'POST',
|
||||
headers: getInternalIngestionHeaders(),
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
...getInternalIngestionHeaders(),
|
||||
},
|
||||
body: JSON.stringify({ waitMs: 2500, throwOnLockTimeout: true }),
|
||||
cache: 'no-store',
|
||||
signal: AbortSignal.timeout(2500),
|
||||
signal: AbortSignal.timeout(4000),
|
||||
});
|
||||
if (!ingestRes.ok) {
|
||||
const errPayload = await ingestRes.json().catch(() => ({}));
|
||||
@@ -56,6 +88,23 @@ export async function POST(_req: NextRequest) {
|
||||
typeof errPayload?.error === 'string'
|
||||
? errPayload.error
|
||||
: `Ingestion flush returned HTTP ${ingestRes.status}`;
|
||||
if (
|
||||
ingestRes.status === 503 &&
|
||||
errPayload?.code === 'FLUSH_LOCK_TIMEOUT'
|
||||
) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: remoteErr,
|
||||
code: 'FLUSH_LOCK_TIMEOUT',
|
||||
flushedCount: localRes.count,
|
||||
filesUpdated: localRes.files,
|
||||
ingestionFlushed: false,
|
||||
ingestionStatus: ingestRes.status,
|
||||
},
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
@@ -108,6 +157,16 @@ export async function POST(_req: NextRequest) {
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : 'Manual flush failed';
|
||||
if (message.includes('FLUSH_LOCK_TIMEOUT')) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: message,
|
||||
code: 'FLUSH_LOCK_TIMEOUT',
|
||||
},
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
return NextResponse.json({ error: message }, { status: 500 });
|
||||
}
|
||||
}
|
||||
+6
-1
@@ -38,9 +38,14 @@ export const metadata: Metadata = {
|
||||
},
|
||||
};
|
||||
|
||||
const THEME_BOOTSTRAP_SCRIPT = `(function(){try{var t=localStorage.getItem('datastack-theme');var r=document.documentElement;if(t==='light'){r.classList.remove('dark');r.classList.add('light');}else{r.classList.remove('light');r.classList.add('dark');}}catch(e){}})();`;
|
||||
|
||||
export default function RootLayout({ children }: { children: React.ReactNode }) {
|
||||
return (
|
||||
<html lang="en" className="dark">
|
||||
<html lang="en" className="dark" suppressHydrationWarning>
|
||||
<head>
|
||||
<script dangerouslySetInnerHTML={{ __html: THEME_BOOTSTRAP_SCRIPT }} />
|
||||
</head>
|
||||
<body suppressHydrationWarning className="min-h-screen bg-[var(--bg-main)] text-[var(--text)] antialiased">
|
||||
<ToastProvider>
|
||||
{children}
|
||||
|
||||
+46
-8
@@ -1,12 +1,50 @@
|
||||
'use client';
|
||||
|
||||
import { MainLayout } from '@/components/layout/MainLayout';
|
||||
import { usePathname } from 'next/navigation';
|
||||
import { VconShell } from '@/components/layout/VconShell';
|
||||
import { verifyRequestAdminSession } from '@/core/auth/adminUsers';
|
||||
import { cookies, headers } from 'next/headers';
|
||||
import { redirect } from 'next/navigation';
|
||||
import React from 'react';
|
||||
|
||||
export default function VconLayout({ children }: { children: React.ReactNode }) {
|
||||
const pathname = usePathname();
|
||||
if (pathname === '/vcon/login' || pathname === '/vcon/setup') return <>{children}</>;
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
return <MainLayout showSidebar={true}>{children}</MainLayout>;
|
||||
export default async function VconLayout({
|
||||
children,
|
||||
}: {
|
||||
children: React.ReactNode;
|
||||
}) {
|
||||
const headerList = await headers();
|
||||
const pathname = headerList.get('x-datastack-pathname') || '';
|
||||
|
||||
if (pathname === '/vcon/login' || pathname === '/vcon/setup') {
|
||||
return <>{children}</>;
|
||||
}
|
||||
|
||||
const cookieStore = await cookies();
|
||||
const token = cookieStore.get('datastack_admin_session')?.value;
|
||||
const auth = await verifyRequestAdminSession(token);
|
||||
|
||||
if (!auth.valid) {
|
||||
if (auth.reason === 'UNAUTHORIZED') {
|
||||
const loginTarget = pathname
|
||||
? `/vcon/login?next=${encodeURIComponent(pathname)}`
|
||||
: '/vcon/login';
|
||||
redirect(loginTarget);
|
||||
}
|
||||
|
||||
return (
|
||||
<main className="grid min-h-dvh place-items-center bg-[var(--bg)] p-4 text-[var(--text)]">
|
||||
<section className="w-full max-w-sm border border-[var(--border)] bg-[var(--surface)] p-5">
|
||||
<h1 className="text-[13px] font-medium text-[var(--text)]">
|
||||
Console authentication unavailable
|
||||
</h1>
|
||||
<p className="mt-2 text-[12px] text-[var(--text-muted)]">
|
||||
{auth.reason === 'UNCONFIGURED'
|
||||
? 'Admin session secret is not configured.'
|
||||
: 'Database authentication check is temporarily unavailable. Please retry shortly.'}
|
||||
</p>
|
||||
</section>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
|
||||
return <VconShell>{children}</VconShell>;
|
||||
}
|
||||
+32
-4
@@ -1,18 +1,46 @@
|
||||
import { AdminLogin } from '@/components/sections/AdminLogin';
|
||||
import { isAdminSetupComplete } from '@/core/auth/adminUsers';
|
||||
import {
|
||||
isAdminSetupComplete,
|
||||
verifyRequestAdminSession,
|
||||
} from '@/core/auth/adminUsers';
|
||||
import { cookies } from 'next/headers';
|
||||
import { redirect } from 'next/navigation';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
export const dynamic = 'force-dynamic';
|
||||
|
||||
export default async function LoginPage() {
|
||||
const isConfigured = await isAdminSetupComplete().catch(() => false);
|
||||
if (!isConfigured) redirect('/vcon/setup');
|
||||
let databaseReady = true;
|
||||
try {
|
||||
const isConfigured = await isAdminSetupComplete();
|
||||
if (!isConfigured) redirect('/vcon/setup');
|
||||
} catch (error) {
|
||||
if (
|
||||
error &&
|
||||
typeof error === 'object' &&
|
||||
'digest' in error &&
|
||||
String((error as { digest?: unknown }).digest).startsWith('NEXT_REDIRECT')
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
databaseReady = false;
|
||||
}
|
||||
|
||||
if (databaseReady) {
|
||||
const cookieStore = await cookies();
|
||||
const token = cookieStore.get('datastack_admin_session')?.value;
|
||||
if (token) {
|
||||
const auth = await verifyRequestAdminSession(token);
|
||||
if (auth.valid) {
|
||||
redirect('/vcon/dashboard');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const showDemoHint =
|
||||
!process.env.DATABASE_URL &&
|
||||
process.env.NODE_ENV !== 'production' &&
|
||||
process.env.DATASTACK_ALLOW_DEMO_ADMIN !== 'false';
|
||||
|
||||
return <AdminLogin showDemoHint={showDemoHint} />;
|
||||
return <AdminLogin showDemoHint={showDemoHint} databaseReady={databaseReady} />;
|
||||
}
|
||||
+62
-29
@@ -9,6 +9,7 @@ import { Panel } from '@/components/ui/Panel';
|
||||
import { StatStrip } from '@/components/ui/StatStrip';
|
||||
import { useToast } from '@/components/ui/Toast';
|
||||
import { getCached, setCached } from '@/lib/vconCache';
|
||||
import { resolvePublicEndpoints } from '@/lib/vconUiHelpers';
|
||||
import { useEffect, useState } from 'react';
|
||||
|
||||
interface SdkClientItem {
|
||||
@@ -34,23 +35,23 @@ export default function SdkGuidePage() {
|
||||
const [apiUrl, setApiUrl] = useState<string>('http://localhost:3000');
|
||||
|
||||
useEffect(() => {
|
||||
const isLocalDev = window.location.port === '3000';
|
||||
setServerUrl(
|
||||
isLocalDev
|
||||
? `${window.location.protocol}//${window.location.hostname}:4000`
|
||||
: window.location.origin
|
||||
);
|
||||
setApiUrl(window.location.origin);
|
||||
const initialPort = getCached<number>('vcon:server-port', 4000);
|
||||
const initialEndpoints = resolvePublicEndpoints(window.location, initialPort);
|
||||
setServerUrl(initialEndpoints.serverUrl);
|
||||
setApiUrl(initialEndpoints.apiUrl);
|
||||
|
||||
async function loadClients() {
|
||||
async function loadSdkContext() {
|
||||
try {
|
||||
const res = await fetch('/api/clients', { cache: 'no-store' });
|
||||
if (res.status === 401) {
|
||||
const [clientsRes, metricsRes] = await Promise.all([
|
||||
fetch('/api/clients', { cache: 'no-store' }),
|
||||
fetch('/api/metrics', { cache: 'no-store' }).catch(() => null),
|
||||
]);
|
||||
if (clientsRes.status === 401 || metricsRes?.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
if (clientsRes.ok) {
|
||||
const data = await clientsRes.json();
|
||||
const list = setCached<SdkClientItem[]>('vcon:clients', data.clients || []);
|
||||
setClients(list);
|
||||
if (list.length > 0) {
|
||||
@@ -61,11 +62,21 @@ export default function SdkGuidePage() {
|
||||
);
|
||||
}
|
||||
}
|
||||
if (metricsRes && metricsRes.ok) {
|
||||
const metricsData = await metricsRes.json().catch(() => null);
|
||||
const runtimePort = Number(metricsData?.runtime?.serverPort);
|
||||
if (Number.isFinite(runtimePort) && runtimePort > 0) {
|
||||
setCached('vcon:server-port', runtimePort);
|
||||
const resolved = resolvePublicEndpoints(window.location, runtimePort);
|
||||
setServerUrl(resolved.serverUrl);
|
||||
setApiUrl(resolved.apiUrl);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Retain cached list on transient network error
|
||||
}
|
||||
}
|
||||
void loadClients();
|
||||
void loadSdkContext();
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -75,6 +86,7 @@ export default function SdkGuidePage() {
|
||||
}
|
||||
|
||||
let cancelled = false;
|
||||
setSelectedClientToken('');
|
||||
setLoadingToken(true);
|
||||
|
||||
async function fetchToken() {
|
||||
@@ -110,8 +122,12 @@ export default function SdkGuidePage() {
|
||||
|
||||
const activeClient =
|
||||
clients.find((c) => c.clientId === selectedClientId) || clients[0];
|
||||
const resolvedToken = selectedClientToken || activeClient?.token || '';
|
||||
const sampleToken = resolvedToken || 'YOUR_SIGNED_CLIENT_JWT_TOKEN';
|
||||
const resolvedToken = loadingToken
|
||||
? ''
|
||||
: selectedClientToken || activeClient?.token || '';
|
||||
const sampleToken = loadingToken
|
||||
? 'LOADING_CLIENT_JWT_TOKEN...'
|
||||
: resolvedToken || 'YOUR_SIGNED_CLIENT_JWT_TOKEN';
|
||||
|
||||
const configSnippet = `# config.datastack.py — Place in your Python project root
|
||||
DATASTACK_CONFIG = {
|
||||
@@ -194,6 +210,7 @@ client.disconnect()`;
|
||||
size="xs"
|
||||
variant="secondary"
|
||||
icon="FileCode"
|
||||
disabled={loadingToken}
|
||||
onClick={() => void copySnippet(configSnippet, 'config.datastack.py')}
|
||||
>
|
||||
Copy Config
|
||||
@@ -216,27 +233,42 @@ client.disconnect()`;
|
||||
{ label: 'Server endpoint', value: serverUrl },
|
||||
{
|
||||
label: 'Active client',
|
||||
value: activeClient?.clientId || 'None selected',
|
||||
value: activeClient
|
||||
? activeClient.status === 'revoked'
|
||||
? `${activeClient.clientId} (revoked)`
|
||||
: activeClient.clientId
|
||||
: 'None selected',
|
||||
dot: activeClient?.status === 'revoked' ? 'off' : activeClient ? 'ok' : undefined,
|
||||
},
|
||||
]}
|
||||
/>
|
||||
|
||||
<Card
|
||||
compact
|
||||
className="flex w-full items-center justify-between bg-[var(--surface)] px-3 py-2 font-mono text-[11.5px]"
|
||||
className="flex w-full flex-col gap-1.5 bg-[var(--surface)] px-3 py-2.5"
|
||||
>
|
||||
<span className="text-[var(--accent)]">
|
||||
pip install ./sdk/python/datastack
|
||||
</span>
|
||||
<Button
|
||||
size="xs"
|
||||
variant="ghost"
|
||||
icon="Copy"
|
||||
aria-label="Copy pip install command"
|
||||
onClick={() =>
|
||||
void copySnippet('pip install ./sdk/python/datastack', 'Pip command')
|
||||
}
|
||||
/>
|
||||
<div className="flex items-center justify-between gap-2 font-mono text-[11.5px]">
|
||||
<span className="truncate text-[var(--accent)]">
|
||||
pip install /path/to/datastack/sdk/python/datastack
|
||||
</span>
|
||||
<Button
|
||||
size="xs"
|
||||
variant="ghost"
|
||||
icon="Copy"
|
||||
aria-label="Copy pip install command"
|
||||
onClick={() =>
|
||||
void copySnippet(
|
||||
'pip install /path/to/datastack/sdk/python/datastack',
|
||||
'Pip command'
|
||||
)
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
<p className="text-[11px] text-[var(--text-faint)]">
|
||||
Self-hosted package bundled in <code className="font-mono text-[var(--text-muted)]">sdk/python/datastack</code>. From inside the repo run{' '}
|
||||
<code className="font-mono text-[var(--text-muted)]">pip install ./sdk/python/datastack</code>, or build a portable wheel with{' '}
|
||||
<code className="font-mono text-[var(--text-muted)]">pip wheel ./sdk/python/datastack -w dist/</code> for external projects.
|
||||
</p>
|
||||
</Card>
|
||||
|
||||
<Panel
|
||||
@@ -265,6 +297,7 @@ client.disconnect()`;
|
||||
variant="ghost"
|
||||
icon="Copy"
|
||||
aria-label="Copy config.datastack.py"
|
||||
disabled={loadingToken}
|
||||
onClick={() => void copySnippet(configSnippet, 'config.datastack.py')}
|
||||
/>
|
||||
</div>
|
||||
|
||||
+22
-4
@@ -1,5 +1,8 @@
|
||||
import { AdminSetup } from '@/components/sections/AdminSetup';
|
||||
import { isAdminSetupComplete } from '@/core/auth/adminUsers';
|
||||
import {
|
||||
isAdminSetupComplete,
|
||||
requiresPersistentAdminDatabase,
|
||||
} from '@/core/auth/adminUsers';
|
||||
import { redirect } from 'next/navigation';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
@@ -7,15 +10,30 @@ export const dynamic = 'force-dynamic';
|
||||
|
||||
export default async function SetupPage() {
|
||||
let databaseReady = true;
|
||||
try {
|
||||
if (await isAdminSetupComplete()) redirect('/vcon/login');
|
||||
} catch {
|
||||
let databaseError = '';
|
||||
let setupComplete = false;
|
||||
|
||||
if (requiresPersistentAdminDatabase()) {
|
||||
databaseReady = false;
|
||||
databaseError =
|
||||
'PostgreSQL DATABASE_URL is required in production before creating an administrator account.';
|
||||
} else {
|
||||
try {
|
||||
setupComplete = await isAdminSetupComplete();
|
||||
} catch {
|
||||
databaseReady = false;
|
||||
databaseError = 'Database connection unavailable.';
|
||||
}
|
||||
}
|
||||
|
||||
if (databaseReady && setupComplete) {
|
||||
redirect('/vcon/login');
|
||||
}
|
||||
|
||||
return (
|
||||
<AdminSetup
|
||||
databaseReady={databaseReady}
|
||||
databaseError={databaseError}
|
||||
initialEmail={process.env.DATASTACK_ADMIN_EMAIL?.trim().toLowerCase() ?? ''}
|
||||
/>
|
||||
);
|
||||
|
||||
+62
-56
@@ -9,6 +9,10 @@ import { Pill } from '@/components/ui/Pill';
|
||||
import { StatStrip } from '@/components/ui/StatStrip';
|
||||
import { useToast } from '@/components/ui/Toast';
|
||||
import { getCached, setCached } from '@/lib/vconCache';
|
||||
import {
|
||||
resolvePublicEndpoints,
|
||||
summarizeHttpBatchResult,
|
||||
} from '@/lib/vconUiHelpers';
|
||||
import Link from 'next/link';
|
||||
import { useEffect, useState } from 'react';
|
||||
import { io } from 'socket.io-client';
|
||||
@@ -22,11 +26,8 @@ interface SimulatorClient {
|
||||
type TransportMode = 'http' | 'websocket';
|
||||
type DispatchStatus = 'Ready' | 'Accepted' | 'Rejected' | 'Error';
|
||||
|
||||
function getSocketOrigin(): string {
|
||||
const isLocalDev = window.location.port === '3000';
|
||||
const protocol = window.location.protocol === 'https:' ? 'https:' : 'http:';
|
||||
const host = isLocalDev ? `${window.location.hostname}:4000` : window.location.host;
|
||||
return `${protocol}//${host}`;
|
||||
function getSocketOrigin(serverPort: number): string {
|
||||
return resolvePublicEndpoints(window.location, serverPort).serverUrl;
|
||||
}
|
||||
|
||||
export default function SimulatorPage() {
|
||||
@@ -52,20 +53,26 @@ export default function SimulatorPage() {
|
||||
2
|
||||
)
|
||||
);
|
||||
const [serverPort, setServerPort] = useState<number>(() =>
|
||||
getCached<number>('vcon:server-port', 4000)
|
||||
);
|
||||
const [sending, setSending] = useState(false);
|
||||
const [dispatchStatus, setDispatchStatus] = useState<DispatchStatus>('Ready');
|
||||
const [lastResponse, setLastResponse] = useState<Record<string, unknown> | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
async function loadClients() {
|
||||
async function loadSimulatorContext() {
|
||||
try {
|
||||
const res = await fetch('/api/clients', { cache: 'no-store' });
|
||||
if (res.status === 401) {
|
||||
const [clientsRes, metricsRes] = await Promise.all([
|
||||
fetch('/api/clients', { cache: 'no-store' }),
|
||||
fetch('/api/metrics', { cache: 'no-store' }).catch(() => null),
|
||||
]);
|
||||
if (clientsRes.status === 401 || metricsRes?.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
if (clientsRes.ok) {
|
||||
const data = await clientsRes.json();
|
||||
const list = setCached<SimulatorClient[]>('vcon:clients', data.clients || []);
|
||||
setClients(list);
|
||||
setSelectedClient((prev) => {
|
||||
@@ -73,11 +80,18 @@ export default function SimulatorPage() {
|
||||
return list.find((c) => c.status === 'active')?.clientId ?? list[0]?.clientId ?? '';
|
||||
});
|
||||
}
|
||||
if (metricsRes && metricsRes.ok) {
|
||||
const metricsData = await metricsRes.json().catch(() => null);
|
||||
const runtimePort = Number(metricsData?.runtime?.serverPort);
|
||||
if (Number.isFinite(runtimePort) && runtimePort > 0) {
|
||||
setServerPort(setCached('vcon:server-port', runtimePort));
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore transient failure and retain cached list
|
||||
}
|
||||
}
|
||||
void loadClients();
|
||||
void loadSimulatorContext();
|
||||
}, []);
|
||||
|
||||
const fetchClientJwt = async (clientId: string): Promise<string> => {
|
||||
@@ -100,7 +114,7 @@ export default function SimulatorPage() {
|
||||
items: Array<Record<string, unknown>>
|
||||
): Promise<{ accepted: number; rejected: number; responses: Record<string, unknown>[] }> => {
|
||||
const token = await fetchClientJwt(clientId);
|
||||
const socketUrl = getSocketOrigin();
|
||||
const socketUrl = getSocketOrigin(serverPort);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const socket = io(socketUrl, {
|
||||
@@ -112,7 +126,7 @@ export default function SimulatorPage() {
|
||||
|
||||
const timer = window.setTimeout(() => {
|
||||
socket.disconnect();
|
||||
reject(new Error('WebSocket timeout (is Port 4000 ingestion server running?)'));
|
||||
reject(new Error(`WebSocket timeout (is Port ${serverPort} ingestion server running?)`));
|
||||
}, 5000);
|
||||
|
||||
socket.on('connect_error', (connectErr) => {
|
||||
@@ -271,55 +285,47 @@ export default function SimulatorPage() {
|
||||
return;
|
||||
}
|
||||
|
||||
let accepted = 0;
|
||||
let rejected = 0;
|
||||
const errors: string[] = [];
|
||||
|
||||
for (const item of items) {
|
||||
const res = await fetch('/api/events', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
clientId: selectedClient,
|
||||
data: item,
|
||||
}),
|
||||
});
|
||||
if (res.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
if (res.ok) {
|
||||
accepted++;
|
||||
} else {
|
||||
rejected++;
|
||||
try {
|
||||
const errJson = await res.json();
|
||||
if (errJson?.error && errors.length < 3) {
|
||||
errors.push(String(errJson.error));
|
||||
}
|
||||
} catch {
|
||||
// ignore non-json error body
|
||||
}
|
||||
}
|
||||
const batchPayload = items.map((item, idx) => ({
|
||||
eventId: `sim_http_${Date.now()}_${idx}`,
|
||||
timestamp: item.timestamp,
|
||||
data: item,
|
||||
}));
|
||||
const res = await fetch('/api/events', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
clientId: selectedClient,
|
||||
batch: batchPayload,
|
||||
}),
|
||||
});
|
||||
if (res.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
|
||||
const data = await res.json().catch(() => ({}));
|
||||
const summary = summarizeHttpBatchResult(
|
||||
res.ok,
|
||||
data && typeof data === 'object' ? (data as Record<string, unknown>) : null,
|
||||
count,
|
||||
);
|
||||
setLastResponse({
|
||||
transport: 'http',
|
||||
status: res.status,
|
||||
batchSent: count,
|
||||
accepted,
|
||||
rejected,
|
||||
...(errors.length > 0 ? { errors } : {}),
|
||||
accepted: summary.accepted,
|
||||
rejected: summary.rejected,
|
||||
duplicates: summary.duplicates,
|
||||
...(data && typeof data === 'object' ? data : {}),
|
||||
});
|
||||
|
||||
if (accepted === 0) {
|
||||
setDispatchStatus('Rejected');
|
||||
error('Batch Rejected', `0/${count} ingested${errors[0] ? ` (${errors[0]})` : ''}`);
|
||||
} else if (accepted < count) {
|
||||
setDispatchStatus('Rejected');
|
||||
warning('Batch Partial', `${accepted}/${count} ingested (${rejected} rejected)`);
|
||||
setDispatchStatus(summary.status);
|
||||
if (summary.level === 'error') {
|
||||
error(summary.title, summary.message);
|
||||
} else if (summary.level === 'warning') {
|
||||
warning(summary.title, summary.message);
|
||||
} else {
|
||||
setDispatchStatus('Accepted');
|
||||
success('Batch Streamed', `${accepted}/${count} ingested`);
|
||||
success(summary.title, summary.message);
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : 'Batch dispatch error';
|
||||
@@ -379,7 +385,7 @@ export default function SimulatorPage() {
|
||||
},
|
||||
{
|
||||
label: 'Ingest transport',
|
||||
value: transport === 'http' ? 'POST /api/events' : 'WS Port 4000',
|
||||
value: transport === 'http' ? 'POST /api/events' : `WS Port ${serverPort}`,
|
||||
},
|
||||
{ label: 'Payload format', value: 'JSON' },
|
||||
{
|
||||
@@ -412,7 +418,7 @@ export default function SimulatorPage() {
|
||||
active={transport === 'websocket'}
|
||||
onClick={() => setTransport('websocket')}
|
||||
>
|
||||
WebSocket Port 4000
|
||||
WebSocket Port {serverPort}
|
||||
</Pill>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
+71
-24
@@ -18,52 +18,96 @@ import { useToast } from '@/components/ui/Toast';
|
||||
import { getCached, setCached } from '@/lib/vconCache';
|
||||
import { useCallback, useEffect, useState } from 'react';
|
||||
|
||||
interface StorageStatsSummary {
|
||||
totalMB?: string;
|
||||
totalFiles?: number;
|
||||
totalRows?: number;
|
||||
rotationRule?: string;
|
||||
rotationMb?: number;
|
||||
}
|
||||
|
||||
interface StoragePartitionFile {
|
||||
clientId: string;
|
||||
filename: string;
|
||||
sizeBytes: number;
|
||||
linesCount: number;
|
||||
modifiedAt: string;
|
||||
}
|
||||
|
||||
export default function StoragePage() {
|
||||
const { success, warning, error } = useToast();
|
||||
const [stats, setStats] = useState<any>(() =>
|
||||
getCached<any>('vcon:storage-stats', null)
|
||||
const [stats, setStats] = useState<StorageStatsSummary | null>(() =>
|
||||
getCached<StorageStatsSummary | null>('vcon:storage-stats', null)
|
||||
);
|
||||
const [files, setFiles] = useState<any[]>(() =>
|
||||
getCached<any[]>('vcon:storage-files', [])
|
||||
const [files, setFiles] = useState<StoragePartitionFile[]>(() =>
|
||||
getCached<StoragePartitionFile[]>('vcon:storage-files', [])
|
||||
);
|
||||
const [refreshing, setRefreshing] = useState(false);
|
||||
const [flushing, setFlushing] = useState(false);
|
||||
|
||||
const fetchStorageData = useCallback(async () => {
|
||||
setRefreshing(true);
|
||||
try {
|
||||
const res = await fetch('/api/storage');
|
||||
if (!res.ok) throw new Error('Unable to load storage partitions');
|
||||
const data = await res.json();
|
||||
setStats(setCached('vcon:storage-stats', data.stats));
|
||||
const res = await fetch('/api/storage', { cache: 'no-store' });
|
||||
if (res.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
const data = await res.json().catch(() => ({}));
|
||||
if (!res.ok) {
|
||||
throw new Error(data.error || 'Unable to load storage partitions');
|
||||
}
|
||||
setStats(setCached('vcon:storage-stats', data.stats ?? null));
|
||||
setFiles(setCached('vcon:storage-files', data.files || []));
|
||||
} catch {
|
||||
error('Storage load failed', 'Could not load storage partitions.');
|
||||
} catch (loadErr) {
|
||||
error(
|
||||
'Storage load failed',
|
||||
loadErr instanceof Error
|
||||
? loadErr.message
|
||||
: 'Could not load storage partitions.'
|
||||
);
|
||||
} finally {
|
||||
setRefreshing(false);
|
||||
}
|
||||
}, [error]);
|
||||
|
||||
useEffect(() => {
|
||||
fetchStorageData();
|
||||
void fetchStorageData();
|
||||
}, [fetchStorageData]);
|
||||
|
||||
const handleManualFlush = async () => {
|
||||
setFlushing(true);
|
||||
try {
|
||||
const res = await fetch('/api/storage', { method: 'POST' });
|
||||
const res = await fetch('/api/storage', {
|
||||
method: 'POST',
|
||||
cache: 'no-store',
|
||||
});
|
||||
if (res.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
const data = await res.json().catch(() => ({}));
|
||||
|
||||
if (res.ok && data.success) {
|
||||
const count = Number(data.flushedCount) || 0;
|
||||
success(
|
||||
'Buffer Flushed',
|
||||
`Flushed ${count} event${count === 1 ? '' : 's'} to CSV${data.warning ? ` (${data.warning})` : ''}`
|
||||
);
|
||||
fetchStorageData();
|
||||
} else if (data.flushedCount !== undefined && data.flushedCount > 0) {
|
||||
// Partial flush: local events persisted, remote Port 4000 flush returned error
|
||||
if (data.warning) {
|
||||
warning(
|
||||
'Partial Flush (Web Only)',
|
||||
`Flushed ${count} event${count === 1 ? '' : 's'}. ${String(data.warning)}`
|
||||
);
|
||||
} else {
|
||||
success(
|
||||
'Buffer Flushed',
|
||||
`Flushed ${count} event${count === 1 ? '' : 's'} to CSV.`
|
||||
);
|
||||
}
|
||||
await fetchStorageData();
|
||||
} else if (data.flushedCount !== undefined && Number(data.flushedCount) > 0) {
|
||||
warning(
|
||||
'Partial Flush',
|
||||
`Flushed ${data.flushedCount} local events. Ingestion server: ${data.error || 'flush failed'}`
|
||||
);
|
||||
fetchStorageData();
|
||||
await fetchStorageData();
|
||||
} else {
|
||||
error('Flush Failed', data.error || 'Could not flush buffer');
|
||||
}
|
||||
@@ -84,7 +128,9 @@ export default function StoragePage() {
|
||||
size="xs"
|
||||
variant="secondary"
|
||||
icon="RotateCw"
|
||||
onClick={fetchStorageData}
|
||||
loading={refreshing}
|
||||
disabled={refreshing || flushing}
|
||||
onClick={() => void fetchStorageData()}
|
||||
>
|
||||
Refresh
|
||||
</Button>
|
||||
@@ -93,7 +139,8 @@ export default function StoragePage() {
|
||||
variant="primary"
|
||||
icon="HardDrive"
|
||||
loading={flushing}
|
||||
onClick={handleManualFlush}
|
||||
disabled={flushing}
|
||||
onClick={() => void handleManualFlush()}
|
||||
>
|
||||
Flush Buffer
|
||||
</Button>
|
||||
@@ -131,8 +178,8 @@ export default function StoragePage() {
|
||||
{files.length === 0 ? (
|
||||
<TableEmpty colSpan={5} message="No CSV files" />
|
||||
) : (
|
||||
files.map((file, idx) => (
|
||||
<TableRow key={idx}>
|
||||
files.map((file) => (
|
||||
<TableRow key={`${file.clientId}/${file.filename}`}>
|
||||
<TableCell className="font-mono text-[var(--accent)]">
|
||||
{file.clientId}
|
||||
</TableCell>
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import { Button } from '@/components/ui/Button';
|
||||
import { Icon } from '@/components/ui/Icon';
|
||||
import { cn } from '@/lib/utils';
|
||||
import { getCached, setCached } from '@/lib/vconCache';
|
||||
import { getCached, setCached, clearAuthCaches } from '@/lib/vconCache';
|
||||
import Link from 'next/link';
|
||||
import { usePathname } from 'next/navigation';
|
||||
import React, { useEffect, useState } from 'react';
|
||||
@@ -69,8 +69,26 @@ export const Sidebar: React.FC<SidebarProps> = ({
|
||||
if (cached.role) setAdminRole(cached.role);
|
||||
}
|
||||
|
||||
fetch('/api/auth/profile', { credentials: 'include' })
|
||||
.then((res) => (res.ok ? res.json() : null))
|
||||
const handleProfileUpdated = (event: Event) => {
|
||||
const detail = (event as CustomEvent<CachedProfile>).detail;
|
||||
if (detail?.email) {
|
||||
setAdminEmail(detail.email);
|
||||
if (detail.role) setAdminRole(detail.role);
|
||||
}
|
||||
};
|
||||
window.addEventListener('datastack:profile-updated', handleProfileUpdated);
|
||||
|
||||
fetch('/api/auth/profile', {
|
||||
credentials: 'include',
|
||||
cache: 'no-store',
|
||||
})
|
||||
.then((res) => {
|
||||
if (res.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return null;
|
||||
}
|
||||
return res.ok ? res.json() : null;
|
||||
})
|
||||
.then((data) => {
|
||||
if (data?.profile?.email) {
|
||||
setCached('vcon:profile', data.profile);
|
||||
@@ -82,6 +100,10 @@ export const Sidebar: React.FC<SidebarProps> = ({
|
||||
.catch(() => {
|
||||
// Keep cached or default identity label if offline
|
||||
});
|
||||
|
||||
return () => {
|
||||
window.removeEventListener('datastack:profile-updated', handleProfileUpdated);
|
||||
};
|
||||
}, [pathname]);
|
||||
|
||||
const toggleTheme = () => {
|
||||
@@ -100,6 +122,7 @@ export const Sidebar: React.FC<SidebarProps> = ({
|
||||
credentials: 'include',
|
||||
});
|
||||
if (!response.ok) throw new Error('Unable to sign out');
|
||||
await clearAuthCaches();
|
||||
window.location.assign('/vcon/login');
|
||||
} catch {
|
||||
setIsSigningOut(false);
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
'use client';
|
||||
|
||||
import { MainLayout } from '@/components/layout/MainLayout';
|
||||
import React from 'react';
|
||||
|
||||
export function VconShell({ children }: { children: React.ReactNode }) {
|
||||
return <MainLayout showSidebar={true}>{children}</MainLayout>;
|
||||
}
|
||||
@@ -16,15 +16,19 @@ function safeNextPath() {
|
||||
|
||||
export function AdminLogin({
|
||||
showDemoHint = false,
|
||||
databaseReady = true,
|
||||
}: {
|
||||
showDemoHint?: boolean;
|
||||
databaseReady?: boolean;
|
||||
}) {
|
||||
const [email, setEmail] = useState('');
|
||||
const [password, setPassword] = useState('');
|
||||
const [remember, setRemember] = useState(true);
|
||||
const [showPassword, setShowPassword] = useState(false);
|
||||
const [isSubmitting, setIsSubmitting] = useState(false);
|
||||
const [errorMessage, setErrorMessage] = useState('');
|
||||
const [errorMessage, setErrorMessage] = useState(
|
||||
databaseReady ? '' : 'Database connection unavailable. Retry once PostgreSQL recovers.',
|
||||
);
|
||||
const [theme, setTheme] = useState<'dark' | 'light'>('dark');
|
||||
|
||||
useEffect(() => {
|
||||
|
||||
@@ -4,13 +4,15 @@ import { Button } from '@/components/ui/Button';
|
||||
import { Input } from '@/components/ui/Input';
|
||||
import { ArrowRight } from 'lucide-react';
|
||||
import Link from 'next/link';
|
||||
import { useState } from 'react';
|
||||
import { useEffect, useState } from 'react';
|
||||
|
||||
export function AdminSetup({
|
||||
databaseReady = true,
|
||||
databaseError = '',
|
||||
initialEmail = '',
|
||||
}: {
|
||||
databaseReady?: boolean;
|
||||
databaseError?: string;
|
||||
initialEmail?: string;
|
||||
}) {
|
||||
const [email, setEmail] = useState(initialEmail || 'admin@datastack.dev');
|
||||
@@ -19,8 +21,25 @@ export function AdminSetup({
|
||||
const [setupKey, setSetupKey] = useState('');
|
||||
const [isSubmitting, setIsSubmitting] = useState(false);
|
||||
const [errorMessage, setErrorMessage] = useState(
|
||||
databaseReady ? '' : 'Database connection unavailable.',
|
||||
databaseReady ? '' : databaseError || 'Database connection unavailable.',
|
||||
);
|
||||
const [theme, setTheme] = useState<'dark' | 'light'>('dark');
|
||||
|
||||
useEffect(() => {
|
||||
const savedTheme = localStorage.getItem('datastack-theme');
|
||||
const initialTheme = savedTheme === 'light' ? 'light' : 'dark';
|
||||
setTheme(initialTheme);
|
||||
document.documentElement.classList.toggle('light', initialTheme === 'light');
|
||||
document.documentElement.classList.toggle('dark', initialTheme === 'dark');
|
||||
}, []);
|
||||
|
||||
const toggleTheme = () => {
|
||||
const nextTheme = theme === 'dark' ? 'light' : 'dark';
|
||||
setTheme(nextTheme);
|
||||
localStorage.setItem('datastack-theme', nextTheme);
|
||||
document.documentElement.classList.toggle('light', nextTheme === 'light');
|
||||
document.documentElement.classList.toggle('dark', nextTheme === 'dark');
|
||||
};
|
||||
|
||||
const submit = async (event: React.FormEvent<HTMLFormElement>) => {
|
||||
event.preventDefault();
|
||||
@@ -59,7 +78,17 @@ export function AdminSetup({
|
||||
|
||||
return (
|
||||
<main className="relative grid min-h-dvh place-items-center bg-[var(--bg)] p-4 text-[var(--text)]">
|
||||
<section className="w-full max-w-[320px] border border-[var(--border)] bg-[var(--surface)]">
|
||||
<Button
|
||||
type="button"
|
||||
variant="icon"
|
||||
size="icon"
|
||||
icon={theme === 'dark' ? 'Sun' : 'Moon'}
|
||||
aria-label="Toggle theme"
|
||||
title="Toggle theme"
|
||||
className="fixed right-3 top-3 z-10 h-7 w-7 border border-[var(--border)] text-[var(--text-muted)]"
|
||||
onClick={toggleTheme}
|
||||
/>
|
||||
<section className="z-0 w-full max-w-[320px] border border-[var(--border)] bg-[var(--surface)]">
|
||||
<header className="flex items-center justify-between border-b border-[var(--border)] px-4 py-3">
|
||||
<Link href="/" className="flex items-center gap-2 text-[12.5px] font-medium text-[var(--text)]">
|
||||
<span className="grid h-3 w-3 place-items-center bg-[var(--accent)]" aria-hidden="true">
|
||||
|
||||
@@ -43,12 +43,18 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
||||
type: 'rotate' | 'revoke' | 'delete';
|
||||
client: ClientItem;
|
||||
} | null>(null);
|
||||
const [purgeFilesOnDelete, setPurgeFilesOnDelete] = useState(false);
|
||||
const [editClient, setEditClient] = useState<ClientItem | null>(null);
|
||||
const [editName, setEditName] = useState('');
|
||||
const [editRateLimit, setEditRateLimit] = useState('100');
|
||||
const [editError, setEditError] = useState('');
|
||||
const [processing, setProcessing] = useState(false);
|
||||
|
||||
const closeConfirmModal = () => {
|
||||
setConfirmAction(null);
|
||||
setPurgeFilesOnDelete(false);
|
||||
};
|
||||
|
||||
const openEditModal = (client: ClientItem) => {
|
||||
setEditClient(client);
|
||||
setEditName(client.name);
|
||||
@@ -66,8 +72,8 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
||||
setEditError('Display name is required.');
|
||||
return;
|
||||
}
|
||||
if (!Number.isFinite(numericRateLimit) || numericRateLimit < 1 || numericRateLimit > 10000) {
|
||||
setEditError('Rate limit must be between 1 and 10,000 events/s.');
|
||||
if (!Number.isInteger(numericRateLimit) || numericRateLimit < 1 || numericRateLimit > 100000) {
|
||||
setEditError('Rate limit must be an integer between 1 and 100,000 events/s.');
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -147,20 +153,30 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
||||
}
|
||||
};
|
||||
|
||||
const executeDeleteClient = async (client: ClientItem) => {
|
||||
const executeDeleteClient = async (client: ClientItem, purgeFiles: boolean) => {
|
||||
setProcessing(true);
|
||||
try {
|
||||
const response = await fetch(`/api/clients/${encodeURIComponent(client.clientId)}`, {
|
||||
method: 'DELETE',
|
||||
});
|
||||
const query = purgeFiles ? '?purgeFiles=true' : '';
|
||||
const response = await fetch(
|
||||
`/api/clients/${encodeURIComponent(client.clientId)}${query}`,
|
||||
{
|
||||
method: 'DELETE',
|
||||
},
|
||||
);
|
||||
if (response.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
const data = await response.json();
|
||||
if (!response.ok) throw new Error(data.error || 'Unable to delete client');
|
||||
setConfirmAction(null);
|
||||
success('Client deleted', `${client.clientId} has been permanently removed.`);
|
||||
closeConfirmModal();
|
||||
const purgedCount = Number(data?.purgedFiles ?? 0);
|
||||
success(
|
||||
'Client deleted',
|
||||
purgeFiles
|
||||
? `${client.clientId} removed and ${purgedCount} stored CSV file${purgedCount === 1 ? '' : 's'} purged.`
|
||||
: `${client.clientId} has been permanently removed (stored CSV files retained).`,
|
||||
);
|
||||
await onStatusChange();
|
||||
} catch (deleteError) {
|
||||
error('Client deletion failed', deleteError instanceof Error ? deleteError.message : 'Unknown error');
|
||||
@@ -171,6 +187,7 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
||||
|
||||
const toggleStatus = (client: ClientItem) => {
|
||||
if (client.status === 'active') {
|
||||
setPurgeFilesOnDelete(false);
|
||||
setConfirmAction({ type: 'revoke', client });
|
||||
} else {
|
||||
void executeStatusChange(client, 'active');
|
||||
@@ -182,7 +199,7 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
||||
if (confirmAction.type === 'rotate') {
|
||||
await executeRotateKey(confirmAction.client);
|
||||
} else if (confirmAction.type === 'delete') {
|
||||
await executeDeleteClient(confirmAction.client);
|
||||
await executeDeleteClient(confirmAction.client, purgeFilesOnDelete);
|
||||
} else {
|
||||
await executeStatusChange(confirmAction.client, 'revoked');
|
||||
}
|
||||
@@ -302,7 +319,10 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
||||
className="h-6 w-6 border border-transparent hover:border-[var(--danger)] hover:text-[var(--danger)]"
|
||||
title={`Delete ${client.clientId}`}
|
||||
aria-label={`Delete ${client.clientId}`}
|
||||
onClick={() => setConfirmAction({ type: 'delete', client })}
|
||||
onClick={() => {
|
||||
setPurgeFilesOnDelete(false);
|
||||
setConfirmAction({ type: 'delete', client });
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</TableCell>
|
||||
@@ -350,7 +370,7 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
||||
compact
|
||||
type="number"
|
||||
min="1"
|
||||
max="10000"
|
||||
max="100000"
|
||||
value={editRateLimit}
|
||||
onChange={(event) => setEditRateLimit(event.target.value)}
|
||||
required
|
||||
@@ -365,7 +385,7 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
||||
|
||||
<Modal
|
||||
isOpen={Boolean(confirmAction)}
|
||||
onClose={() => !processing && setConfirmAction(null)}
|
||||
onClose={() => !processing && closeConfirmModal()}
|
||||
title={
|
||||
confirmAction?.type === 'rotate'
|
||||
? 'Rotate credentials'
|
||||
@@ -376,7 +396,7 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
||||
maxWidth="xs"
|
||||
footer={
|
||||
<>
|
||||
<Button type="button" variant="ghost" size="xs" disabled={processing} onClick={() => setConfirmAction(null)}>
|
||||
<Button type="button" variant="ghost" size="xs" disabled={processing} onClick={closeConfirmModal}>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button
|
||||
@@ -402,6 +422,22 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps
|
||||
? `Permanently delete client ${confirmAction.client.clientId}? This removes the client registration and credentials.`
|
||||
: `Revoke access for ${confirmAction?.client.clientId || ''}?`}
|
||||
</p>
|
||||
{confirmAction?.type === 'delete' && (
|
||||
<label className="mt-3 flex cursor-pointer items-start gap-2 border border-[var(--border)] bg-[var(--surface-2)] p-2.5 text-[11.5px] text-[var(--text)]">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={purgeFilesOnDelete}
|
||||
onChange={(event) => setPurgeFilesOnDelete(event.target.checked)}
|
||||
className="mt-0.5 h-3.5 w-3.5 shrink-0 accent-[var(--danger)]"
|
||||
/>
|
||||
<span>
|
||||
Also purge stored CSV files for this client
|
||||
<span className="mt-0.5 block text-[10.5px] text-[var(--text-faint)]">
|
||||
Deletes partitioned disk CSV files along with the client registration.
|
||||
</span>
|
||||
</span>
|
||||
</label>
|
||||
)}
|
||||
</Modal>
|
||||
|
||||
<Modal
|
||||
@@ -420,7 +456,6 @@ export function CredentialsDisplay({ credentials }: { credentials: ClientCredent
|
||||
return (
|
||||
<div className="flex flex-col gap-2.5">
|
||||
<CredentialItem label="Client ID" value={credentials.clientId} />
|
||||
<CredentialItem label="Secret key" value={credentials.secret} />
|
||||
<div className="flex flex-col gap-1">
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<label htmlFor={`token-${credentials.clientId}`} className="text-[10px] uppercase tracking-wider text-[var(--text-faint)]">
|
||||
@@ -435,6 +470,9 @@ export function CredentialsDisplay({ credentials }: { credentials: ClientCredent
|
||||
rows={3}
|
||||
className="font-mono text-[11px] text-[var(--accent)]"
|
||||
/>
|
||||
<p className="text-[10.5px] text-[var(--text-faint)]">
|
||||
Use this signed JWT token for both Socket.IO handshakes and HTTP Bearer ingestion.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
|
||||
@@ -15,7 +15,14 @@ import {
|
||||
TableRow,
|
||||
TableWrap,
|
||||
} from '@/components/ui/Table';
|
||||
import { useToast } from '@/components/ui/Toast';
|
||||
import { getCached, setCached } from '@/lib/vconCache';
|
||||
import {
|
||||
countMatchingBufferedEvents,
|
||||
extractContentDispositionFilename,
|
||||
formatEventTimeLabel,
|
||||
type BufferedSummary,
|
||||
} from '@/lib/vconUiHelpers';
|
||||
import Link from 'next/link';
|
||||
import React, { useCallback, useEffect, useState } from 'react';
|
||||
|
||||
@@ -40,6 +47,12 @@ type DashboardMetrics = {
|
||||
totalFiles: number;
|
||||
totalRows: number;
|
||||
};
|
||||
runtime?: {
|
||||
ingestionBridgeStatus?: 'online' | 'auth_error' | 'error' | 'offline';
|
||||
ingestionBridgeError?: string;
|
||||
serverPort?: number;
|
||||
webPort?: number;
|
||||
};
|
||||
};
|
||||
|
||||
type StorageFile = {
|
||||
@@ -51,6 +64,7 @@ type StorageFile = {
|
||||
};
|
||||
|
||||
export const DashboardOverview: React.FC = () => {
|
||||
const { error, info } = useToast();
|
||||
const [metrics, setMetrics] = useState<DashboardMetrics | null>(() =>
|
||||
getCached<DashboardMetrics | null>('vcon:metrics', null)
|
||||
);
|
||||
@@ -63,7 +77,11 @@ export const DashboardOverview: React.FC = () => {
|
||||
const [files, setFiles] = useState<StorageFile[]>(() =>
|
||||
getCached<StorageFile[]>('vcon:storage-files', [])
|
||||
);
|
||||
const [bufferedSummary, setBufferedSummary] = useState<BufferedSummary | null>(() =>
|
||||
getCached<BufferedSummary | null>('vcon:buffered-summary', null)
|
||||
);
|
||||
const [isRefreshing, setIsRefreshing] = useState(false);
|
||||
const [isExporting, setIsExporting] = useState(false);
|
||||
const [fetchError, setFetchError] = useState('');
|
||||
|
||||
const refresh = useCallback(async () => {
|
||||
@@ -72,7 +90,7 @@ export const DashboardOverview: React.FC = () => {
|
||||
const [metricsResponse, eventsResponse, storageResponse, clientsResponse] =
|
||||
await Promise.all([
|
||||
fetch('/api/metrics', { cache: 'no-store' }),
|
||||
fetch('/api/events?limit=8', { cache: 'no-store' }),
|
||||
fetch('/api/events?limit=8&includeBufferedSummary=true', { cache: 'no-store' }),
|
||||
fetch('/api/storage', { cache: 'no-store' }),
|
||||
fetch('/api/clients', { cache: 'no-store' }),
|
||||
]);
|
||||
@@ -99,6 +117,9 @@ export const DashboardOverview: React.FC = () => {
|
||||
if (eventsResponse.ok) {
|
||||
const data = await eventsResponse.json();
|
||||
setEvents(setCached('vcon:dashboard-events', data.events ?? []));
|
||||
if (data.bufferedSummary) {
|
||||
setBufferedSummary(setCached('vcon:buffered-summary', data.bufferedSummary));
|
||||
}
|
||||
} else {
|
||||
errors.push('events');
|
||||
}
|
||||
@@ -145,12 +166,30 @@ export const DashboardOverview: React.FC = () => {
|
||||
.filter((file) => file.date === today)
|
||||
.reduce((total, file) => total + file.linesCount, 0);
|
||||
const bufferedCount = metrics?.bufferStats.currentBufferSize ?? 0;
|
||||
const eventsToday = storedRecordsToday + bufferedCount;
|
||||
const bufferedTodayRaw = bufferedSummary
|
||||
? countMatchingBufferedEvents(bufferedSummary, {
|
||||
clientId: 'all',
|
||||
fromDate: today,
|
||||
toDate: today,
|
||||
}).matchingCount
|
||||
: 0;
|
||||
const bufferedTodayCount = metrics
|
||||
? Math.min(bufferedTodayRaw, bufferedCount)
|
||||
: bufferedTodayRaw;
|
||||
const eventsToday = storedRecordsToday + bufferedTodayCount;
|
||||
const activeClients =
|
||||
metrics?.activeClients ??
|
||||
clients.filter((client) => client.status === 'active').length;
|
||||
const totalClients = metrics?.totalClients ?? clients.length;
|
||||
const activeSockets = metrics?.activeSockets ?? 0;
|
||||
const bridgeStatus = metrics?.runtime?.ingestionBridgeStatus;
|
||||
const bridgeAuthError = bridgeStatus === 'auth_error';
|
||||
const socketSuffix =
|
||||
bridgeStatus === 'auth_error'
|
||||
? `${activeSockets} ws · auth err`
|
||||
: bridgeStatus === 'offline'
|
||||
? `${activeSockets} ws · offline`
|
||||
: `${activeSockets} ws`;
|
||||
const totalStorage = metrics?.storageStats.totalMB ?? '0.00';
|
||||
const totalStorageBytes = metrics?.storageStats.totalBytes ?? 0;
|
||||
const storageValue =
|
||||
@@ -165,6 +204,41 @@ export const DashboardOverview: React.FC = () => {
|
||||
metrics?.storageStats.totalRows ??
|
||||
files.reduce((sum, file) => sum + file.linesCount, 0);
|
||||
|
||||
const handleQuickExport = async () => {
|
||||
setIsExporting(true);
|
||||
try {
|
||||
const res = await fetch('/api/export?clientId=all', { cache: 'no-store' });
|
||||
if (res.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
if (!res.ok) {
|
||||
const errJson = await res.json().catch(() => ({}));
|
||||
error('Export failed', errJson.error || `Export failed with HTTP ${res.status}.`);
|
||||
return;
|
||||
}
|
||||
const blob = await res.blob();
|
||||
const filename = extractContentDispositionFilename(
|
||||
res.headers.get('content-disposition'),
|
||||
`datastack_export_all_${new Date().toISOString().slice(0, 10)}.csv`
|
||||
);
|
||||
const objectUrl = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = objectUrl;
|
||||
a.download = filename;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
info('Export downloaded', `${filename} downloaded.`);
|
||||
await refresh();
|
||||
} catch (err) {
|
||||
error('Export failed', err instanceof Error ? err.message : 'Unable to export dataset.');
|
||||
} finally {
|
||||
setIsExporting(false);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<PageShell
|
||||
title="Dashboard"
|
||||
@@ -179,11 +253,16 @@ export const DashboardOverview: React.FC = () => {
|
||||
loading={isRefreshing}
|
||||
onClick={() => void refresh()}
|
||||
/>
|
||||
<a href="/api/export?clientId=all" download>
|
||||
<Button size="xs" variant="secondary" icon="Download">
|
||||
Export
|
||||
</Button>
|
||||
</a>
|
||||
<Button
|
||||
size="xs"
|
||||
variant="secondary"
|
||||
icon="Download"
|
||||
loading={isExporting}
|
||||
disabled={isExporting}
|
||||
onClick={() => void handleQuickExport()}
|
||||
>
|
||||
Export
|
||||
</Button>
|
||||
<Link href="/vcon/clients">
|
||||
<Button size="xs" variant="primary" icon="Plus">
|
||||
New client
|
||||
@@ -192,12 +271,14 @@ export const DashboardOverview: React.FC = () => {
|
||||
</>
|
||||
}
|
||||
>
|
||||
{fetchError && (
|
||||
{(fetchError || bridgeAuthError) && (
|
||||
<div
|
||||
role="alert"
|
||||
className="mb-3 border border-[var(--danger)] bg-[var(--danger-soft)] px-3 py-2 text-[11.5px] text-[var(--text)]"
|
||||
>
|
||||
{fetchError}
|
||||
{bridgeAuthError
|
||||
? `Ingestion bridge authentication mismatch (${metrics?.runtime?.ingestionBridgeError || '401'}). Check DATASTACK_INTERNAL_SECRET parity.`
|
||||
: fetchError}
|
||||
</div>
|
||||
)}
|
||||
|
||||
@@ -206,8 +287,13 @@ export const DashboardOverview: React.FC = () => {
|
||||
{ label: 'Events today', value: eventsToday.toLocaleString() },
|
||||
{
|
||||
label: 'Active clients · Sockets',
|
||||
value: `${activeClients} / ${totalClients} · ${activeSockets} ws`,
|
||||
dot: activeClients > 0 || activeSockets > 0 ? 'ok' : 'off',
|
||||
value: `${activeClients} / ${totalClients} · ${socketSuffix}`,
|
||||
dot:
|
||||
bridgeAuthError
|
||||
? 'off'
|
||||
: activeSockets > 0 || (activeClients > 0 && bridgeStatus === 'online')
|
||||
? 'ok'
|
||||
: 'off',
|
||||
},
|
||||
{ label: 'Storage used', value: `${storageValue} ${storageUnit}` },
|
||||
{ label: 'Stored rows', value: totalRows.toLocaleString() },
|
||||
@@ -248,9 +334,7 @@ export const DashboardOverview: React.FC = () => {
|
||||
events.map((event) => (
|
||||
<TableRow key={event.eventId}>
|
||||
<TableCell className="font-mono tabular-nums text-[var(--text-faint)]">
|
||||
{event.timestamp.includes('T')
|
||||
? new Date(event.timestamp).toLocaleTimeString()
|
||||
: event.timestamp}
|
||||
{formatEventTimeLabel(event.timestamp)}
|
||||
</TableCell>
|
||||
<TableCell className="font-mono text-[var(--accent)]">
|
||||
{event.clientId}
|
||||
@@ -288,13 +372,12 @@ export const DashboardOverview: React.FC = () => {
|
||||
title="Storage"
|
||||
meta={`${totalFiles}`}
|
||||
action={
|
||||
<a
|
||||
href="/api/export?clientId=all"
|
||||
download
|
||||
<Link
|
||||
href="/vcon/storage"
|
||||
className="text-[11px] text-[var(--text-muted)] hover:text-[var(--text)]"
|
||||
>
|
||||
Export ↓
|
||||
</a>
|
||||
Files →
|
||||
</Link>
|
||||
}
|
||||
>
|
||||
<div className="divide-y divide-[var(--border)]">
|
||||
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
TableRow,
|
||||
TableWrap,
|
||||
} from '@/components/ui/Table';
|
||||
import { formatEventTimeLabel } from '@/lib/vconUiHelpers';
|
||||
import { useRouter } from 'next/navigation';
|
||||
import { useCallback, useEffect, useState } from 'react';
|
||||
|
||||
@@ -86,8 +87,10 @@ export function DataBrowser() {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
if (!response.ok) throw new Error('Unable to load events');
|
||||
const data = await response.json();
|
||||
const data = await response.json().catch(() => ({}));
|
||||
if (!response.ok) {
|
||||
throw new Error(data.error || 'Unable to load events');
|
||||
}
|
||||
setEvents(data.events || []);
|
||||
setTotal(data.total || 0);
|
||||
setPageCount(data.pageCount || 1);
|
||||
@@ -124,6 +127,8 @@ export function DataBrowser() {
|
||||
fetchEvents();
|
||||
}, [fetchEvents, filtersReady]);
|
||||
|
||||
const invalidRange = Boolean(fromFilter && toFilter && fromFilter > toFilter);
|
||||
|
||||
const firstVisible = total === 0 ? 0 : (page - 1) * PAGE_SIZE + 1;
|
||||
const lastVisible = Math.min(page * PAGE_SIZE, total);
|
||||
const firstPage = Math.max(1, Math.min(page - 1, pageCount - 2));
|
||||
@@ -133,6 +138,11 @@ export function DataBrowser() {
|
||||
);
|
||||
|
||||
const applyFilters = () => {
|
||||
if (invalidRange) {
|
||||
setErrorMessage('Start date ("From") must be on or before end date ("To").');
|
||||
return;
|
||||
}
|
||||
setErrorMessage('');
|
||||
setAppliedFilters({
|
||||
clientId: clientFilter,
|
||||
from: fromFilter,
|
||||
@@ -147,6 +157,7 @@ export function DataBrowser() {
|
||||
setFromFilter('');
|
||||
setToFilter('');
|
||||
setSearchFilter('');
|
||||
setErrorMessage('');
|
||||
setAppliedFilters({ clientId: 'all', from: '', to: '', search: '' });
|
||||
setPage(1);
|
||||
};
|
||||
@@ -189,7 +200,13 @@ export function DataBrowser() {
|
||||
}
|
||||
>
|
||||
<Card compact className="p-0">
|
||||
<div className="grid grid-cols-1 gap-px bg-[var(--border)] sm:grid-cols-2 lg:grid-cols-5">
|
||||
<form
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
applyFilters();
|
||||
}}
|
||||
className="grid grid-cols-1 gap-px bg-[var(--border)] sm:grid-cols-2 lg:grid-cols-5"
|
||||
>
|
||||
<div className="bg-[var(--surface)] p-2.5">
|
||||
<Select
|
||||
label="Client"
|
||||
@@ -237,11 +254,21 @@ export function DataBrowser() {
|
||||
<Button type="button" size="xs" icon="X" onClick={resetFilters}>
|
||||
Reset
|
||||
</Button>
|
||||
<Button type="button" size="xs" variant="primary" onClick={applyFilters}>
|
||||
<Button
|
||||
type="submit"
|
||||
size="xs"
|
||||
variant="primary"
|
||||
disabled={invalidRange}
|
||||
>
|
||||
Apply
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
{invalidRange && (
|
||||
<p role="alert" className="border-t border-[var(--border)] bg-[var(--surface-2)] px-3 py-1.5 text-[11px] text-[var(--danger)]">
|
||||
Start date must be on or before end date.
|
||||
</p>
|
||||
)}
|
||||
</Card>
|
||||
|
||||
<Panel
|
||||
@@ -274,12 +301,7 @@ export function DataBrowser() {
|
||||
onClick={() => setSelectedEvent(event)}
|
||||
>
|
||||
<TableCell className="font-mono tabular-nums text-[var(--text)]">
|
||||
{new Date(event.timestamp).toLocaleTimeString([], {
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
second: '2-digit',
|
||||
hour12: false,
|
||||
})}
|
||||
{formatEventTimeLabel(event.timestamp)}
|
||||
</TableCell>
|
||||
<TableCell className="font-mono text-[var(--accent)]">
|
||||
{event.clientId}
|
||||
|
||||
@@ -8,9 +8,16 @@ import { Pill } from '@/components/ui/Pill';
|
||||
import { StatStrip } from '@/components/ui/StatStrip';
|
||||
import { useToast } from '@/components/ui/Toast';
|
||||
import { getCached, setCached } from '@/lib/vconCache';
|
||||
import {
|
||||
countMatchingBufferedEvents,
|
||||
extractContentDispositionFilename,
|
||||
getUtcDateRangeForPreset,
|
||||
matchesSelectedClientFolder,
|
||||
type BufferedSummary,
|
||||
} from '@/lib/vconUiHelpers';
|
||||
import { Braces, FileText } from 'lucide-react';
|
||||
import Link from 'next/link';
|
||||
import { useEffect, useMemo, useState } from 'react';
|
||||
import { useCallback, useEffect, useMemo, useState } from 'react';
|
||||
|
||||
interface Client {
|
||||
clientId: string;
|
||||
@@ -29,10 +36,6 @@ interface StorageFile {
|
||||
type ExportFormat = 'csv' | 'json';
|
||||
type RangePreset = 'today' | '7d' | '30d' | 'all' | 'custom';
|
||||
|
||||
function dateString(date: Date) {
|
||||
return date.toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function formatBytes(bytes: number) {
|
||||
if (bytes < 1024) return `${bytes} B`;
|
||||
if (bytes < 1024 ** 2) return `${(bytes / 1024).toFixed(1)} KB`;
|
||||
@@ -41,7 +44,9 @@ function formatBytes(bytes: number) {
|
||||
}
|
||||
|
||||
function relativeTime(value: string, now: number) {
|
||||
const elapsedSeconds = Math.max(0, Math.floor((now - new Date(value).getTime()) / 1000));
|
||||
const ts = new Date(value).getTime();
|
||||
if (!Number.isFinite(ts)) return '—';
|
||||
const elapsedSeconds = Math.max(0, Math.floor((now - ts) / 1000));
|
||||
if (elapsedSeconds < 60) return `${elapsedSeconds}s ago`;
|
||||
if (elapsedSeconds < 3600) return `${Math.floor(elapsedSeconds / 60)}m ago`;
|
||||
if (elapsedSeconds < 86_400) return `${Math.floor(elapsedSeconds / 3600)}h ago`;
|
||||
@@ -57,6 +62,7 @@ export function ExportWorkspace() {
|
||||
getCached<StorageFile[]>('vcon:storage-files', [])
|
||||
);
|
||||
const [bufferedCount, setBufferedCount] = useState<number>(0);
|
||||
const [bufferedSummary, setBufferedSummary] = useState<BufferedSummary | null>(null);
|
||||
const [selectedClient, setSelectedClient] = useState('all');
|
||||
const [fromDate, setFromDate] = useState('');
|
||||
const [toDate, setToDate] = useState('');
|
||||
@@ -65,48 +71,63 @@ export function ExportWorkspace() {
|
||||
const [loading, setLoading] = useState(
|
||||
() => getCached<StorageFile[]>('vcon:storage-files', []).length === 0
|
||||
);
|
||||
const [downloading, setDownloading] = useState(false);
|
||||
const [loadError, setLoadError] = useState('');
|
||||
const [now, setNow] = useState(() => Date.now());
|
||||
|
||||
useEffect(() => {
|
||||
Promise.all([
|
||||
fetch('/api/clients', { cache: 'no-store' }).then(async (response) => {
|
||||
if (response.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return { clients: [] };
|
||||
}
|
||||
if (!response.ok) throw new Error('Unable to load clients');
|
||||
return response.json();
|
||||
}),
|
||||
fetch('/api/storage', { cache: 'no-store' }).then(async (response) => {
|
||||
if (response.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return { files: [] };
|
||||
}
|
||||
if (!response.ok) throw new Error('Unable to load stored files');
|
||||
return response.json();
|
||||
}),
|
||||
fetch('/api/metrics', { cache: 'no-store' })
|
||||
.then(async (response) => (response.ok ? response.json() : null))
|
||||
.catch(() => null),
|
||||
])
|
||||
.then(([clientData, storageData, metricsData]) => {
|
||||
setClients(setCached('vcon:clients', clientData.clients || []));
|
||||
setFiles(setCached('vcon:storage-files', storageData.files || []));
|
||||
if (storageData.stats) setCached('vcon:storage-stats', storageData.stats);
|
||||
if (metricsData?.bufferStats) {
|
||||
const refreshExportData = useCallback(async () => {
|
||||
try {
|
||||
const [clientData, storageData, metricsData, bufferedData] = await Promise.all([
|
||||
fetch('/api/clients', { cache: 'no-store' }).then(async (response) => {
|
||||
if (response.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return { clients: [] };
|
||||
}
|
||||
if (!response.ok) throw new Error('Unable to load clients');
|
||||
return response.json();
|
||||
}),
|
||||
fetch('/api/storage', { cache: 'no-store' }).then(async (response) => {
|
||||
if (response.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return { files: [] };
|
||||
}
|
||||
if (!response.ok) throw new Error('Unable to load stored files');
|
||||
return response.json();
|
||||
}),
|
||||
fetch('/api/metrics', { cache: 'no-store' })
|
||||
.then(async (response) => (response.ok ? response.json() : null))
|
||||
.catch(() => null),
|
||||
fetch('/api/events?bufferedOnly=true&limit=1', { cache: 'no-store' })
|
||||
.then(async (response) => (response.ok ? response.json() : null))
|
||||
.catch(() => null),
|
||||
]);
|
||||
|
||||
setClients(setCached('vcon:clients', clientData.clients || []));
|
||||
setFiles(setCached('vcon:storage-files', storageData.files || []));
|
||||
if (storageData.stats) setCached('vcon:storage-stats', storageData.stats);
|
||||
if (metricsData) {
|
||||
setCached('vcon:metrics', metricsData);
|
||||
if (metricsData.bufferStats) {
|
||||
setBufferedCount(Number(metricsData.bufferStats.currentBufferSize) || 0);
|
||||
}
|
||||
setLoadError('');
|
||||
})
|
||||
.catch((fetchError) => {
|
||||
setLoadError(
|
||||
fetchError instanceof Error ? fetchError.message : 'Unable to load export data'
|
||||
);
|
||||
})
|
||||
.finally(() => setLoading(false));
|
||||
}
|
||||
if (bufferedData?.bufferedSummary) {
|
||||
setBufferedSummary(setCached('vcon:buffered-summary', bufferedData.bufferedSummary));
|
||||
}
|
||||
setLoadError('');
|
||||
} catch (fetchError) {
|
||||
setLoadError(
|
||||
fetchError instanceof Error ? fetchError.message : 'Unable to load export data'
|
||||
);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
void refreshExportData();
|
||||
}, [refreshExportData]);
|
||||
|
||||
useEffect(() => {
|
||||
const updateClock = () => setNow(Date.now());
|
||||
updateClock();
|
||||
@@ -118,7 +139,7 @@ export function ExportWorkspace() {
|
||||
() =>
|
||||
files.filter(
|
||||
(file) =>
|
||||
(selectedClient === 'all' || file.clientId === selectedClient) &&
|
||||
matchesSelectedClientFolder(file.clientId, selectedClient) &&
|
||||
(!fromDate || file.date >= fromDate) &&
|
||||
(!toDate || file.date <= toDate)
|
||||
),
|
||||
@@ -156,18 +177,9 @@ export function ExportWorkspace() {
|
||||
|
||||
const setQuickRange = (nextPreset: Exclude<RangePreset, 'custom'>) => {
|
||||
setPreset(nextPreset);
|
||||
if (nextPreset === 'all') {
|
||||
setFromDate('');
|
||||
setToDate('');
|
||||
return;
|
||||
}
|
||||
|
||||
const today = new Date();
|
||||
const start = new Date(today);
|
||||
if (nextPreset === '7d') start.setDate(today.getDate() - 6);
|
||||
if (nextPreset === '30d') start.setDate(today.getDate() - 29);
|
||||
setFromDate(dateString(start));
|
||||
setToDate(dateString(today));
|
||||
const range = getUtcDateRangeForPreset(nextPreset);
|
||||
setFromDate(range.from);
|
||||
setToDate(range.to);
|
||||
};
|
||||
|
||||
const exportUrl = useMemo(() => {
|
||||
@@ -177,10 +189,31 @@ export function ExportWorkspace() {
|
||||
return `/api/export?${params.toString()}`;
|
||||
}, [format, fromDate, selectedClient, toDate]);
|
||||
|
||||
const invalidRange = Boolean(fromDate && toDate && fromDate > toDate);
|
||||
const nothingToExport = preview.rows === 0 && bufferedCount === 0;
|
||||
const filteredBuffered = useMemo(
|
||||
() =>
|
||||
countMatchingBufferedEvents(
|
||||
bufferedSummary,
|
||||
{ clientId: selectedClient, fromDate, toDate },
|
||||
bufferedCount,
|
||||
),
|
||||
[bufferedCount, bufferedSummary, fromDate, selectedClient, toDate],
|
||||
);
|
||||
|
||||
const downloadExport = () => {
|
||||
const invalidRange = Boolean(fromDate && toDate && fromDate > toDate);
|
||||
const nothingToExport = preview.rows === 0 && !filteredBuffered.hasPotentialBuffered;
|
||||
|
||||
const triggerBlobDownload = (blob: Blob, filename: string) => {
|
||||
const objectUrl = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = objectUrl;
|
||||
a.download = filename;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
};
|
||||
|
||||
const downloadExport = async () => {
|
||||
if (invalidRange) {
|
||||
error('Invalid date range', 'Start date must be on or before end date.');
|
||||
return;
|
||||
@@ -189,33 +222,60 @@ export function ExportWorkspace() {
|
||||
error('Nothing to export', 'No stored rows or buffered events match the selected filters.');
|
||||
return;
|
||||
}
|
||||
window.location.assign(exportUrl);
|
||||
info(
|
||||
'Export started',
|
||||
bufferedCount > 0
|
||||
? `${format.toUpperCase()} download (${preview.rows.toLocaleString()} stored + ${bufferedCount} buffered auto-flushed).`
|
||||
: `${format.toUpperCase()} download for ${preview.rows.toLocaleString()} rows.`
|
||||
);
|
||||
setDownloading(true);
|
||||
try {
|
||||
const res = await fetch(exportUrl, { cache: 'no-store' });
|
||||
if (res.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
if (!res.ok) {
|
||||
const errorJson = await res.json().catch(() => ({}));
|
||||
error(
|
||||
'Export failed',
|
||||
errorJson.error || `Export endpoint returned HTTP ${res.status}.`
|
||||
);
|
||||
return;
|
||||
}
|
||||
const blob = await res.blob();
|
||||
const defaultName = `datastack_export_${selectedClient || 'all'}_${new Date().toISOString().slice(0, 10)}.${format}`;
|
||||
const filename = extractContentDispositionFilename(
|
||||
res.headers.get('content-disposition'),
|
||||
defaultName
|
||||
);
|
||||
triggerBlobDownload(blob, filename);
|
||||
info(
|
||||
'Export downloaded',
|
||||
filteredBuffered.matchingCount > 0
|
||||
? `${format.toUpperCase()} download (${preview.rows.toLocaleString()} stored + ${filteredBuffered.matchingCount.toLocaleString()} buffered flushed).`
|
||||
: filteredBuffered.unattributedCount > 0
|
||||
? `${format.toUpperCase()} download (${preview.rows.toLocaleString()} stored + pending backlog flushed).`
|
||||
: `${format.toUpperCase()} download for ${preview.rows.toLocaleString()} rows.`
|
||||
);
|
||||
await refreshExportData();
|
||||
} catch (err: unknown) {
|
||||
error('Export failed', err instanceof Error ? err.message : 'Unable to export dataset.');
|
||||
} finally {
|
||||
setDownloading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const downloadFile = async (file: StorageFile) => {
|
||||
const url = `/api/export?clientId=${encodeURIComponent(file.clientId)}&filename=${encodeURIComponent(file.filename)}`;
|
||||
try {
|
||||
const res = await fetch(url, { cache: 'no-store' });
|
||||
if (res.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
if (!res.ok) {
|
||||
const errorJson = await res.json().catch(() => ({}));
|
||||
error('Download Failed', errorJson.error || 'Requested CSV file could not be exported.');
|
||||
return;
|
||||
}
|
||||
const blob = await res.blob();
|
||||
const objectUrl = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = objectUrl;
|
||||
a.download = file.filename;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
triggerBlobDownload(blob, file.filename);
|
||||
await refreshExportData();
|
||||
} catch (err: unknown) {
|
||||
error('Download Failed', err instanceof Error ? err.message : 'Export download error');
|
||||
}
|
||||
@@ -348,11 +408,19 @@ export function ExportWorkspace() {
|
||||
</strong>{' '}
|
||||
files
|
||||
</span>
|
||||
{bufferedCount > 0 && (
|
||||
{filteredBuffered.matchingCount > 0 && (
|
||||
<>
|
||||
<span aria-hidden="true">·</span>
|
||||
<span className="text-[var(--warning)]">
|
||||
+{bufferedCount} buffered (flushed on export)
|
||||
+{filteredBuffered.matchingCount.toLocaleString()} buffered (flushed on export)
|
||||
</span>
|
||||
</>
|
||||
)}
|
||||
{filteredBuffered.matchingCount === 0 && filteredBuffered.unattributedCount > 0 && (
|
||||
<>
|
||||
<span aria-hidden="true">·</span>
|
||||
<span className="text-[var(--warning)]">
|
||||
+{filteredBuffered.unattributedCount.toLocaleString()} queued backlog (flushed on export)
|
||||
</span>
|
||||
</>
|
||||
)}
|
||||
@@ -382,8 +450,9 @@ export function ExportWorkspace() {
|
||||
variant="primary"
|
||||
size="xs"
|
||||
icon="Download"
|
||||
disabled={loading || invalidRange || nothingToExport}
|
||||
onClick={downloadExport}
|
||||
loading={downloading}
|
||||
disabled={loading || downloading || invalidRange || nothingToExport}
|
||||
onClick={() => void downloadExport()}
|
||||
>
|
||||
Download {format.toUpperCase()}
|
||||
</Button>
|
||||
|
||||
@@ -30,14 +30,22 @@ export function NewClientModal({ open, onClose, onCreated }: NewClientModalProps
|
||||
error('Validation error', 'Client ID and Name are required.');
|
||||
return;
|
||||
}
|
||||
const clampedRateLimit = Math.max(1, Math.min(5000, Number(rateLimit) || 100));
|
||||
const numericRate = Number(rateLimit);
|
||||
if (!Number.isInteger(numericRate) || numericRate < 1 || numericRate > 100000) {
|
||||
error('Validation error', 'Rate limit must be an integer between 1 and 100,000 events/s.');
|
||||
return;
|
||||
}
|
||||
setSubmitting(true);
|
||||
try {
|
||||
const response = await fetch('/api/clients', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ clientId: trimmedId, name: trimmedName, rateLimit: clampedRateLimit }),
|
||||
body: JSON.stringify({ clientId: trimmedId, name: trimmedName, rateLimit: numericRate }),
|
||||
});
|
||||
if (response.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
const data = await response.json();
|
||||
if (!response.ok) throw new Error(data.error || 'Client registration failed');
|
||||
|
||||
@@ -91,7 +99,7 @@ export function NewClientModal({ open, onClose, onCreated }: NewClientModalProps
|
||||
label="Rate limit (req/s)"
|
||||
type="number"
|
||||
min="1"
|
||||
max="5000"
|
||||
max="100000"
|
||||
value={rateLimit}
|
||||
onChange={(event) => setRateLimit(event.target.value)}
|
||||
/>
|
||||
|
||||
@@ -11,7 +11,11 @@ import {
|
||||
} from '@/components/ui/SettingsLayout';
|
||||
import { StatStrip } from '@/components/ui/StatStrip';
|
||||
import { useToast } from '@/components/ui/Toast';
|
||||
import { getCached, setCached } from '@/lib/vconCache';
|
||||
import { getCached, setCached, clearAuthCaches } from '@/lib/vconCache';
|
||||
import {
|
||||
formatSessionExpiryLabel,
|
||||
shouldRedirectProfileUnauthorized,
|
||||
} from '@/lib/vconUiHelpers';
|
||||
import Link from 'next/link';
|
||||
import { useEffect, useState } from 'react';
|
||||
|
||||
@@ -49,20 +53,31 @@ export function ProfileWorkspace() {
|
||||
const [signingOut, setSigningOut] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
fetch('/api/auth/profile', { credentials: 'include' })
|
||||
fetch('/api/auth/profile', {
|
||||
credentials: 'include',
|
||||
cache: 'no-store',
|
||||
})
|
||||
.then(async (res) => {
|
||||
if (!res.ok) throw new Error('Unable to load profile');
|
||||
return res.json();
|
||||
if (res.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return null;
|
||||
}
|
||||
const body = await res.json().catch(() => ({}));
|
||||
if (!res.ok) throw new Error(body.error || 'Unable to load profile');
|
||||
return body;
|
||||
})
|
||||
.then((data) => {
|
||||
if (data.profile) {
|
||||
if (data?.profile) {
|
||||
setProfile(setCached('vcon:profile', data.profile));
|
||||
setEmail(data.profile.email);
|
||||
setCached('vcon:profile-loaded', true);
|
||||
}
|
||||
})
|
||||
.catch(() => {
|
||||
error('Profile load failed', 'Could not load administrator profile.');
|
||||
.catch((loadErr) => {
|
||||
error(
|
||||
'Profile load failed',
|
||||
loadErr instanceof Error ? loadErr.message : 'Could not load administrator profile.'
|
||||
);
|
||||
})
|
||||
.finally(() => setLoading(false));
|
||||
}, [error]);
|
||||
@@ -78,6 +93,10 @@ export function ProfileWorkspace() {
|
||||
error('Validation error', 'Enter your current password to save changes.');
|
||||
return;
|
||||
}
|
||||
if (newPassword && (newPassword.length < 8 || newPassword.length > 256)) {
|
||||
error('Validation error', 'New password must be between 8 and 256 characters.');
|
||||
return;
|
||||
}
|
||||
if (newPassword && newPassword !== confirmPassword) {
|
||||
error('Validation error', 'New password and confirmation do not match.');
|
||||
return;
|
||||
@@ -89,16 +108,24 @@ export function ProfileWorkspace() {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
credentials: 'include',
|
||||
cache: 'no-store',
|
||||
body: JSON.stringify({
|
||||
email: trimmedEmail,
|
||||
currentPassword,
|
||||
newPassword: newPassword || undefined,
|
||||
}),
|
||||
});
|
||||
const data = await response.json();
|
||||
const data = await response.json().catch(() => ({}));
|
||||
if (shouldRedirectProfileUnauthorized(response.status, data)) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
if (!response.ok) throw new Error(data.error || 'Profile update failed');
|
||||
setProfile(setCached('vcon:profile', data.profile));
|
||||
setEmail(data.profile.email);
|
||||
window.dispatchEvent(
|
||||
new CustomEvent('datastack:profile-updated', { detail: data.profile })
|
||||
);
|
||||
setCurrentPassword('');
|
||||
setNewPassword('');
|
||||
setConfirmPassword('');
|
||||
@@ -123,6 +150,7 @@ export function ProfileWorkspace() {
|
||||
credentials: 'include',
|
||||
});
|
||||
if (!response.ok) throw new Error('Unable to sign out');
|
||||
await clearAuthCaches();
|
||||
window.location.assign('/vcon/login');
|
||||
} catch {
|
||||
error('Sign out failed', 'Please try again.');
|
||||
@@ -160,13 +188,7 @@ export function ProfileWorkspace() {
|
||||
{ label: 'Auth Store', value: profile.storageMode },
|
||||
{
|
||||
label: 'Session Expires',
|
||||
value: profile.sessionExpiresAt
|
||||
? new Date(profile.sessionExpiresAt).toLocaleTimeString([], {
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
hour12: false,
|
||||
})
|
||||
: 'Active',
|
||||
value: formatSessionExpiryLabel(profile.sessionExpiresAt),
|
||||
},
|
||||
]}
|
||||
/>
|
||||
|
||||
@@ -7,8 +7,8 @@ export const SdkSection: React.FC = () => (
|
||||
<SectionHeading index="02" label="Python SDK" title="Drop in the SDK. Start streaming." />
|
||||
<div className="grid grid-cols-1 gap-px border border-[var(--border)] bg-[var(--border)] md:grid-cols-2">
|
||||
<CodeBlock>
|
||||
<span className="text-[var(--text-faint)]"># Install the official Python SDK</span>{'\n'}
|
||||
<span className="text-[var(--text)]">pip install ./sdk/python/datastack</span>{'\n\n'}
|
||||
<span className="text-[var(--text-faint)]"># Install the bundled Python SDK (from repo or local wheel/path)</span>{'\n'}
|
||||
<span className="text-[var(--text)]">pip install /path/to/datastack/sdk/python/datastack</span>{'\n\n'}
|
||||
<span className="text-[var(--text-faint)]"># Auto-loads config.datastack.py from project root</span>{'\n'}
|
||||
<span className="text-[var(--text)]">from datastack import DataStack</span>{'\n\n'}
|
||||
<span className="text-[var(--text)]">client = DataStack()</span>{'\n'}
|
||||
@@ -20,7 +20,7 @@ export const SdkSection: React.FC = () => (
|
||||
<p className="mb-2 text-[11px] uppercase text-[var(--text-faint)]">How it works</p>
|
||||
<h3 className="mb-2 text-[14px] font-medium text-[var(--text)]">Zero-boilerplate config. Auto-recovery built in.</h3>
|
||||
<p className="text-[12px] leading-relaxed text-[var(--text-muted)]">
|
||||
Drop <code className="text-[var(--text)]">config.datastack.py</code> into your Python project to stream single events, batches, or <code className="text-[var(--text)]">.csv</code> / <code className="text-[var(--text)]">.json</code> files directly—with automatic HTTP fallback, offline disk spooling, and Heartbeat IDLE recovery.
|
||||
Install the self-hosted SDK from <code className="text-[var(--text)]">sdk/python/datastack</code> (or build a wheel with <code className="text-[var(--text)]">pip wheel ./sdk/python/datastack -w dist/</code>) and drop <code className="text-[var(--text)]">config.datastack.py</code> into your Python project to stream single events, batches, or <code className="text-[var(--text)]">.csv</code> / <code className="text-[var(--text)]">.json</code> files directly—with automatic HTTP fallback, offline disk spooling, and Heartbeat IDLE recovery.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -15,34 +15,31 @@ import {
|
||||
import { StatStrip } from '@/components/ui/StatStrip';
|
||||
import { useToast } from '@/components/ui/Toast';
|
||||
import { getCached, setCached } from '@/lib/vconCache';
|
||||
import {
|
||||
DEFAULT_EDITABLE_SETTINGS,
|
||||
describeSettingsBridgeSync,
|
||||
type EditableSettings,
|
||||
mapBridgeStatusToSocketUiState,
|
||||
normalizeEditableSettings,
|
||||
NUMERIC_SETTING_KEYS,
|
||||
type NumericSettingDrafts,
|
||||
type NumericSettingKey,
|
||||
resolvePublicEndpoints,
|
||||
SETTINGS_LIMITS,
|
||||
settingsToNumericDrafts,
|
||||
type SettingsValidationErrors,
|
||||
type SocketHealthUiState,
|
||||
validateEditableSettingsDraft,
|
||||
validateSingleNumericSetting,
|
||||
} from '@/lib/vconUiHelpers';
|
||||
import { useCallback, useEffect, useMemo, useState } from 'react';
|
||||
|
||||
interface EditableSettings {
|
||||
maxEventsPerSec: number;
|
||||
maxPayloadSizeKb: number;
|
||||
batchFlushThreshold: number;
|
||||
batchFlushIntervalMs: number;
|
||||
fileRotationMb: number;
|
||||
enableIdempotency: boolean;
|
||||
autoPruneDays: number;
|
||||
}
|
||||
|
||||
interface PublicSettings extends EditableSettings {
|
||||
serverPort: number;
|
||||
webPort: number;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
const DEFAULT_SETTINGS: EditableSettings = {
|
||||
maxEventsPerSec: 100,
|
||||
maxPayloadSizeKb: 100,
|
||||
batchFlushThreshold: 500,
|
||||
batchFlushIntervalMs: 2000,
|
||||
fileRotationMb: 100,
|
||||
enableIdempotency: true,
|
||||
autoPruneDays: 30,
|
||||
};
|
||||
|
||||
const TABS = [
|
||||
{ id: 'general', label: 'General' },
|
||||
{ id: 'connections', label: 'Connections' },
|
||||
@@ -51,16 +48,32 @@ const TABS = [
|
||||
];
|
||||
|
||||
type TabId = (typeof TABS)[number]['id'];
|
||||
type SocketHealthState = 'idle' | 'checking' | 'online' | 'offline';
|
||||
|
||||
const FIELD_TO_TAB: Record<NumericSettingKey, TabId> = {
|
||||
maxEventsPerSec: 'general',
|
||||
batchFlushThreshold: 'connections',
|
||||
batchFlushIntervalMs: 'connections',
|
||||
fileRotationMb: 'storage',
|
||||
autoPruneDays: 'storage',
|
||||
maxPayloadSizeKb: 'security',
|
||||
};
|
||||
|
||||
export function SettingsWorkspace() {
|
||||
const { success, error } = useToast();
|
||||
const [settings, setSettings] = useState<EditableSettings>(() =>
|
||||
getCached<EditableSettings>('vcon:settings', DEFAULT_SETTINGS)
|
||||
);
|
||||
const { success, warning, error } = useToast();
|
||||
const [savedSettings, setSavedSettings] = useState<EditableSettings>(() =>
|
||||
getCached<EditableSettings>('vcon:settings', DEFAULT_SETTINGS)
|
||||
getCached<EditableSettings>('vcon:settings', DEFAULT_EDITABLE_SETTINGS)
|
||||
);
|
||||
const [numericDrafts, setNumericDrafts] = useState<NumericSettingDrafts>(() =>
|
||||
settingsToNumericDrafts(
|
||||
getCached<EditableSettings>('vcon:settings', DEFAULT_EDITABLE_SETTINGS)
|
||||
)
|
||||
);
|
||||
const [enableIdempotency, setEnableIdempotency] = useState<boolean>(
|
||||
() =>
|
||||
getCached<EditableSettings>('vcon:settings', DEFAULT_EDITABLE_SETTINGS)
|
||||
.enableIdempotency
|
||||
);
|
||||
const [fieldErrors, setFieldErrors] = useState<SettingsValidationErrors>({});
|
||||
const [activeTab, setActiveTab] = useState<TabId>('general');
|
||||
const [jwtSecret, setJwtSecret] = useState('');
|
||||
const [jwtError, setJwtError] = useState('');
|
||||
@@ -69,8 +82,8 @@ export function SettingsWorkspace() {
|
||||
getCached<string>('vcon:storage-path', '')
|
||||
);
|
||||
const [baseUrl, setBaseUrl] = useState('');
|
||||
const [webSocketUrl, setWebSocketUrl] = useState('');
|
||||
const [socketStatus, setSocketStatus] = useState<SocketHealthState>('idle');
|
||||
const [socketStatus, setSocketStatus] = useState<SocketHealthUiState>('idle');
|
||||
const [bridgeError, setBridgeError] = useState('');
|
||||
const [activeSockets, setActiveSockets] = useState<number>(0);
|
||||
const [serverPort, setServerPort] = useState(() =>
|
||||
getCached<number>('vcon:server-port', 4000)
|
||||
@@ -80,48 +93,46 @@ export function SettingsWorkspace() {
|
||||
);
|
||||
const [saving, setSaving] = useState(false);
|
||||
|
||||
const probeSocketServer = useCallback(async (port: number) => {
|
||||
setSocketStatus('checking');
|
||||
const isLocalHost =
|
||||
window.location.hostname === 'localhost' ||
|
||||
window.location.hostname === '127.0.0.1' ||
|
||||
window.location.port === '3000';
|
||||
const httpHost = isLocalHost
|
||||
? `${window.location.hostname}:${port}`
|
||||
: window.location.host;
|
||||
const healthUrl = `${window.location.protocol}//${httpHost}/health`;
|
||||
const webSocketUrl = useMemo(() => {
|
||||
if (!baseUrl || typeof window === 'undefined') return '';
|
||||
return resolvePublicEndpoints(window.location, serverPort).webSocketUrl;
|
||||
}, [baseUrl, serverPort]);
|
||||
|
||||
const probeSocketServer = useCallback(async () => {
|
||||
setSocketStatus('checking');
|
||||
try {
|
||||
const response = await fetch(healthUrl, {
|
||||
const response = await fetch('/api/metrics', {
|
||||
cache: 'no-store',
|
||||
signal: AbortSignal.timeout(1500),
|
||||
signal: AbortSignal.timeout(3000),
|
||||
});
|
||||
setSocketStatus(response.ok ? 'online' : 'offline');
|
||||
} catch {
|
||||
// In remote/preview environments where Port 4000 is proxied through web API or reverse proxy, probe /api/health
|
||||
try {
|
||||
const fallbackRes = await fetch('/api/health', {
|
||||
cache: 'no-store',
|
||||
signal: AbortSignal.timeout(1500),
|
||||
});
|
||||
setSocketStatus(fallbackRes.ok ? 'online' : 'offline');
|
||||
} catch {
|
||||
setSocketStatus('offline');
|
||||
if (response.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return;
|
||||
}
|
||||
if (!response.ok) {
|
||||
setSocketStatus('offline');
|
||||
setBridgeError(`Metrics probe returned HTTP ${response.status}`);
|
||||
return;
|
||||
}
|
||||
const metricsData = await response.json();
|
||||
const runtimePort = Number(metricsData?.runtime?.serverPort);
|
||||
if (Number.isFinite(runtimePort) && runtimePort > 0) {
|
||||
setServerPort(setCached('vcon:server-port', runtimePort));
|
||||
}
|
||||
setActiveSockets(Number(metricsData?.activeSockets) || 0);
|
||||
const nextState = mapBridgeStatusToSocketUiState(
|
||||
metricsData?.runtime?.ingestionBridgeStatus
|
||||
);
|
||||
setSocketStatus(nextState);
|
||||
setBridgeError(String(metricsData?.runtime?.ingestionBridgeError || ''));
|
||||
} catch {
|
||||
setSocketStatus('offline');
|
||||
setBridgeError('Unable to reach /api/metrics bridge probe');
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
setBaseUrl(window.location.origin);
|
||||
const isLocalHost =
|
||||
window.location.hostname === 'localhost' ||
|
||||
window.location.hostname === '127.0.0.1' ||
|
||||
window.location.port === '3000';
|
||||
const socketProtocol = window.location.protocol === 'https:' ? 'wss' : 'ws';
|
||||
const socketHost = isLocalHost
|
||||
? `${window.location.hostname}:${serverPort}`
|
||||
: window.location.host;
|
||||
setWebSocketUrl(`${socketProtocol}://${socketHost}`);
|
||||
|
||||
Promise.all([
|
||||
fetch('/api/settings', { cache: 'no-store' }).then(async (response) => {
|
||||
@@ -141,46 +152,58 @@ export function SettingsWorkspace() {
|
||||
return response.json();
|
||||
}),
|
||||
fetch('/api/metrics', { cache: 'no-store' })
|
||||
.then(async (response) => (response.ok ? response.json() : null))
|
||||
.then(async (response) => {
|
||||
if (response.status === 401) {
|
||||
window.location.assign('/vcon/login');
|
||||
return null;
|
||||
}
|
||||
return response.ok ? response.json() : null;
|
||||
})
|
||||
.catch(() => null),
|
||||
])
|
||||
.then(([settingsData, storageData, metricsData]) => {
|
||||
if (!settingsData || !storageData) return;
|
||||
const loadedPort = Number(
|
||||
const runtimePort = Number(metricsData?.runtime?.serverPort);
|
||||
const settingsPort = Number(
|
||||
(settingsData.settings as PublicSettings).serverPort ?? 4000
|
||||
);
|
||||
const loadedPort =
|
||||
Number.isFinite(runtimePort) && runtimePort > 0 ? runtimePort : settingsPort;
|
||||
const loaded: EditableSettings = {
|
||||
maxEventsPerSec: Number(
|
||||
settingsData.settings.maxEventsPerSec ??
|
||||
DEFAULT_SETTINGS.maxEventsPerSec
|
||||
DEFAULT_EDITABLE_SETTINGS.maxEventsPerSec
|
||||
),
|
||||
maxPayloadSizeKb: Number(
|
||||
settingsData.settings.maxPayloadSizeKb ??
|
||||
DEFAULT_SETTINGS.maxPayloadSizeKb
|
||||
DEFAULT_EDITABLE_SETTINGS.maxPayloadSizeKb
|
||||
),
|
||||
batchFlushThreshold: Number(
|
||||
settingsData.settings.batchFlushThreshold ??
|
||||
DEFAULT_SETTINGS.batchFlushThreshold
|
||||
DEFAULT_EDITABLE_SETTINGS.batchFlushThreshold
|
||||
),
|
||||
batchFlushIntervalMs: Number(
|
||||
settingsData.settings.batchFlushIntervalMs ??
|
||||
DEFAULT_SETTINGS.batchFlushIntervalMs
|
||||
DEFAULT_EDITABLE_SETTINGS.batchFlushIntervalMs
|
||||
),
|
||||
fileRotationMb: Number(
|
||||
settingsData.settings.fileRotationMb ??
|
||||
DEFAULT_SETTINGS.fileRotationMb
|
||||
DEFAULT_EDITABLE_SETTINGS.fileRotationMb
|
||||
),
|
||||
enableIdempotency: Boolean(
|
||||
settingsData.settings.enableIdempotency ??
|
||||
DEFAULT_SETTINGS.enableIdempotency
|
||||
DEFAULT_EDITABLE_SETTINGS.enableIdempotency
|
||||
),
|
||||
autoPruneDays: Number(
|
||||
settingsData.settings.autoPruneDays ??
|
||||
DEFAULT_SETTINGS.autoPruneDays
|
||||
DEFAULT_EDITABLE_SETTINGS.autoPruneDays
|
||||
),
|
||||
};
|
||||
setSettings(setCached('vcon:settings', loaded));
|
||||
setSavedSettings(loaded);
|
||||
const cachedLoaded = setCached('vcon:settings', loaded);
|
||||
setSavedSettings(cachedLoaded);
|
||||
setNumericDrafts(settingsToNumericDrafts(cachedLoaded));
|
||||
setEnableIdempotency(cachedLoaded.enableIdempotency);
|
||||
setFieldErrors({});
|
||||
setServerPort(setCached('vcon:server-port', loadedPort));
|
||||
setStoragePath(
|
||||
setCached(
|
||||
@@ -188,69 +211,51 @@ export function SettingsWorkspace() {
|
||||
String(storageData.stats?.storageDirectory ?? '')
|
||||
)
|
||||
);
|
||||
if (metricsData && typeof metricsData.activeSockets === 'number') {
|
||||
setActiveSockets(metricsData.activeSockets);
|
||||
if (metricsData.activeSockets > 0) {
|
||||
setSocketStatus('online');
|
||||
}
|
||||
if (metricsData) {
|
||||
setActiveSockets(Number(metricsData.activeSockets) || 0);
|
||||
setSocketStatus(
|
||||
mapBridgeStatusToSocketUiState(
|
||||
metricsData.runtime?.ingestionBridgeStatus
|
||||
)
|
||||
);
|
||||
setBridgeError(String(metricsData.runtime?.ingestionBridgeError || ''));
|
||||
}
|
||||
setCached('vcon:settings-loaded', true);
|
||||
})
|
||||
.catch(() => {
|
||||
error('Settings load failed', 'Using default runtime settings.');
|
||||
error('Settings load failed', 'Using last cached runtime settings.');
|
||||
})
|
||||
.finally(() => setLoading(false));
|
||||
}, [error, serverPort]);
|
||||
}, [error]);
|
||||
|
||||
const isDirty = useMemo(
|
||||
() =>
|
||||
jwtSecret.trim().length > 0 ||
|
||||
JSON.stringify(settings) !== JSON.stringify(savedSettings),
|
||||
[jwtSecret, savedSettings, settings]
|
||||
enableIdempotency !== savedSettings.enableIdempotency ||
|
||||
NUMERIC_SETTING_KEYS.some(
|
||||
(key) => numericDrafts[key].trim() !== String(savedSettings[key])
|
||||
),
|
||||
[enableIdempotency, jwtSecret, numericDrafts, savedSettings]
|
||||
);
|
||||
|
||||
const updateNumber = (key: keyof EditableSettings, value: string) => {
|
||||
setSettings((current) => ({ ...current, [key]: Number(value) || 0 }));
|
||||
const updateNumberDraft = (key: NumericSettingKey, value: string) => {
|
||||
setNumericDrafts((current) => ({
|
||||
...current,
|
||||
[key]: value,
|
||||
}));
|
||||
const check = validateSingleNumericSetting(key, value);
|
||||
setFieldErrors((current) => {
|
||||
if (check.valid) {
|
||||
if (!current[key]) return current;
|
||||
const next = { ...current };
|
||||
delete next[key];
|
||||
return next;
|
||||
}
|
||||
return { ...current, [key]: check.error };
|
||||
});
|
||||
};
|
||||
|
||||
const buildNormalizedSettings = (): EditableSettings => {
|
||||
const parsedPrune = Number(settings.autoPruneDays);
|
||||
return {
|
||||
maxEventsPerSec: Math.max(
|
||||
1,
|
||||
Math.min(
|
||||
10000,
|
||||
settings.maxEventsPerSec || DEFAULT_SETTINGS.maxEventsPerSec
|
||||
)
|
||||
),
|
||||
maxPayloadSizeKb: Math.max(
|
||||
1,
|
||||
Math.min(
|
||||
5120,
|
||||
settings.maxPayloadSizeKb || DEFAULT_SETTINGS.maxPayloadSizeKb
|
||||
)
|
||||
),
|
||||
batchFlushThreshold: Math.max(
|
||||
1,
|
||||
settings.batchFlushThreshold || DEFAULT_SETTINGS.batchFlushThreshold
|
||||
),
|
||||
batchFlushIntervalMs: Math.max(
|
||||
100,
|
||||
settings.batchFlushIntervalMs || DEFAULT_SETTINGS.batchFlushIntervalMs
|
||||
),
|
||||
fileRotationMb: Math.max(
|
||||
1,
|
||||
settings.fileRotationMb || DEFAULT_SETTINGS.fileRotationMb
|
||||
),
|
||||
enableIdempotency: settings.enableIdempotency,
|
||||
autoPruneDays: Number.isFinite(parsedPrune)
|
||||
? Math.max(0, Math.min(365, Math.floor(parsedPrune)))
|
||||
: DEFAULT_SETTINGS.autoPruneDays,
|
||||
};
|
||||
};
|
||||
|
||||
const executeSave = async () => {
|
||||
const normalized = buildNormalizedSettings();
|
||||
const executeSave = async (validated: EditableSettings) => {
|
||||
const trimmedSecret = jwtSecret.trim();
|
||||
setConfirmRotateOpen(false);
|
||||
setSaving(true);
|
||||
@@ -259,7 +264,7 @@ export function SettingsWorkspace() {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
...normalized,
|
||||
...validated,
|
||||
...(trimmedSecret ? { jwtSecret: trimmedSecret } : {}),
|
||||
}),
|
||||
});
|
||||
@@ -269,16 +274,36 @@ export function SettingsWorkspace() {
|
||||
}
|
||||
const data = await response.json();
|
||||
if (!response.ok) throw new Error(data.error || 'Settings update failed');
|
||||
setSettings(setCached('vcon:settings', normalized));
|
||||
setSavedSettings(normalized);
|
||||
const synced = normalizeEditableSettings(data.settings ?? validated);
|
||||
setCached('vcon:settings', synced);
|
||||
setSavedSettings(synced);
|
||||
setNumericDrafts(settingsToNumericDrafts(synced));
|
||||
setEnableIdempotency(synced.enableIdempotency);
|
||||
setFieldErrors({});
|
||||
setJwtSecret('');
|
||||
setJwtError('');
|
||||
success(
|
||||
'Settings saved',
|
||||
trimmedSecret
|
||||
? 'Runtime settings saved and client JWT credentials rotated.'
|
||||
: 'Runtime settings updated.'
|
||||
);
|
||||
|
||||
const bridgeSummary = describeSettingsBridgeSync(data.bridgeSync);
|
||||
if (bridgeSummary.level === 'error') {
|
||||
setSocketStatus('auth_error');
|
||||
setBridgeError(bridgeSummary.message || '');
|
||||
error(bridgeSummary.title, bridgeSummary.message);
|
||||
} else if (bridgeSummary.level === 'warning') {
|
||||
setSocketStatus('offline');
|
||||
setBridgeError(bridgeSummary.message || '');
|
||||
warning(bridgeSummary.title, bridgeSummary.message);
|
||||
} else {
|
||||
if (Boolean(data.bridgeSync?.ok ?? data.bridgeSync?.synced)) {
|
||||
setSocketStatus('online');
|
||||
setBridgeError('');
|
||||
}
|
||||
success(
|
||||
'Settings saved',
|
||||
trimmedSecret
|
||||
? 'Runtime settings saved and client JWT credentials rotated.'
|
||||
: 'Runtime settings updated across Web & Ingestion.'
|
||||
);
|
||||
}
|
||||
} catch (saveError) {
|
||||
error(
|
||||
'Save failed',
|
||||
@@ -293,6 +318,24 @@ export function SettingsWorkspace() {
|
||||
|
||||
const handleFormSubmit = async (event: React.FormEvent<HTMLFormElement>) => {
|
||||
event.preventDefault();
|
||||
const validation = validateEditableSettingsDraft(
|
||||
numericDrafts,
|
||||
enableIdempotency
|
||||
);
|
||||
if (!validation.valid || !validation.values) {
|
||||
setFieldErrors(validation.errors);
|
||||
if (validation.firstInvalidKey) {
|
||||
setActiveTab(FIELD_TO_TAB[validation.firstInvalidKey]);
|
||||
}
|
||||
const firstMessage =
|
||||
(validation.firstInvalidKey &&
|
||||
validation.errors[validation.firstInvalidKey]) ||
|
||||
'Review highlighted numeric settings before saving.';
|
||||
error('Invalid setting value', firstMessage);
|
||||
return;
|
||||
}
|
||||
|
||||
setFieldErrors({});
|
||||
const trimmedSecret = jwtSecret.trim();
|
||||
if (trimmedSecret.length > 0 && trimmedSecret.length < 32) {
|
||||
const message = 'JWT secret must be at least 32 characters.';
|
||||
@@ -308,30 +351,57 @@ export function SettingsWorkspace() {
|
||||
return;
|
||||
}
|
||||
|
||||
await executeSave();
|
||||
await executeSave(validation.values);
|
||||
};
|
||||
|
||||
const handleConfirmedRotateSave = async () => {
|
||||
const validation = validateEditableSettingsDraft(
|
||||
numericDrafts,
|
||||
enableIdempotency
|
||||
);
|
||||
if (!validation.valid || !validation.values) {
|
||||
setConfirmRotateOpen(false);
|
||||
setFieldErrors(validation.errors);
|
||||
if (validation.firstInvalidKey) {
|
||||
setActiveTab(FIELD_TO_TAB[validation.firstInvalidKey]);
|
||||
}
|
||||
return;
|
||||
}
|
||||
await executeSave(validation.values);
|
||||
};
|
||||
|
||||
const discardChanges = () => {
|
||||
setSettings(savedSettings);
|
||||
setNumericDrafts(settingsToNumericDrafts(savedSettings));
|
||||
setEnableIdempotency(savedSettings.enableIdempotency);
|
||||
setFieldErrors({});
|
||||
setJwtSecret('');
|
||||
setJwtError('');
|
||||
};
|
||||
|
||||
const rateLimitStat = numericDrafts.maxEventsPerSec.trim() || '—';
|
||||
const flushThresholdStat = numericDrafts.batchFlushThreshold.trim() || '—';
|
||||
const flushIntervalStat = numericDrafts.batchFlushIntervalMs.trim() || '—';
|
||||
const rotationStat = numericDrafts.fileRotationMb.trim() || '—';
|
||||
const pruneStat =
|
||||
numericDrafts.autoPruneDays.trim() === '0'
|
||||
? 'Keep all'
|
||||
: numericDrafts.autoPruneDays.trim()
|
||||
? `${numericDrafts.autoPruneDays.trim()}d`
|
||||
: '—';
|
||||
|
||||
return (
|
||||
<PageShell title="Settings">
|
||||
<StatStrip
|
||||
items={[
|
||||
{ label: 'Instance', value: 'DataStack', dot: 'ok' },
|
||||
{ label: 'Rate limit', value: `${settings.maxEventsPerSec} / sec` },
|
||||
{ label: 'Rate limit', value: `${rateLimitStat} / sec` },
|
||||
{
|
||||
label: 'Batch flush',
|
||||
value: `${settings.batchFlushThreshold} / ${settings.batchFlushIntervalMs}ms`,
|
||||
value: `${flushThresholdStat} / ${flushIntervalStat}ms`,
|
||||
},
|
||||
{
|
||||
label: 'CSV rotation',
|
||||
value: `${settings.fileRotationMb}MB · ${
|
||||
settings.autoPruneDays === 0 ? 'Keep all' : `${settings.autoPruneDays}d`
|
||||
}`,
|
||||
value: `${rotationStat}MB · ${pruneStat}`,
|
||||
},
|
||||
]}
|
||||
/>
|
||||
@@ -343,7 +413,7 @@ export function SettingsWorkspace() {
|
||||
onChange={(tab) => setActiveTab(tab as TabId)}
|
||||
/>
|
||||
|
||||
<form onSubmit={handleFormSubmit} className="w-full">
|
||||
<form onSubmit={handleFormSubmit} noValidate className="w-full">
|
||||
<SettingsPanel
|
||||
id={`settings-panel-${activeTab}`}
|
||||
role="tabpanel"
|
||||
@@ -374,24 +444,20 @@ export function SettingsWorkspace() {
|
||||
<Input
|
||||
compact
|
||||
type="number"
|
||||
min="1"
|
||||
max="10000"
|
||||
value={settings.maxEventsPerSec}
|
||||
min={SETTINGS_LIMITS.maxEventsPerSec.min}
|
||||
max={SETTINGS_LIMITS.maxEventsPerSec.max}
|
||||
value={numericDrafts.maxEventsPerSec}
|
||||
error={fieldErrors.maxEventsPerSec}
|
||||
onChange={(event) =>
|
||||
updateNumber('maxEventsPerSec', event.target.value)
|
||||
updateNumberDraft('maxEventsPerSec', event.target.value)
|
||||
}
|
||||
/>
|
||||
</SettingsField>
|
||||
<SettingsField label="Idempotency">
|
||||
<FormSwitch
|
||||
label="Deduplicate events"
|
||||
checked={settings.enableIdempotency}
|
||||
onChange={(enableIdempotency) =>
|
||||
setSettings((current) => ({
|
||||
...current,
|
||||
enableIdempotency,
|
||||
}))
|
||||
}
|
||||
checked={enableIdempotency}
|
||||
onChange={setEnableIdempotency}
|
||||
/>
|
||||
</SettingsField>
|
||||
</SettingsGroup>
|
||||
@@ -403,7 +469,10 @@ export function SettingsWorkspace() {
|
||||
<SettingsGroup title="WebSocket server">
|
||||
<SettingsField label="Live health">
|
||||
<div className="flex items-center justify-between gap-2 border border-[var(--border)] bg-[var(--surface-2)] px-2.5 py-1.5">
|
||||
<span className="inline-flex items-center gap-1.5 font-mono text-[11.5px] text-[var(--text)]">
|
||||
<span
|
||||
className="inline-flex items-center gap-1.5 font-mono text-[11.5px] text-[var(--text)]"
|
||||
title={bridgeError || undefined}
|
||||
>
|
||||
<span
|
||||
aria-hidden="true"
|
||||
className={`h-1.5 w-1.5 ${
|
||||
@@ -411,7 +480,7 @@ export function SettingsWorkspace() {
|
||||
? 'bg-[var(--accent)]'
|
||||
: socketStatus === 'checking'
|
||||
? 'bg-[var(--warning)]'
|
||||
: socketStatus === 'offline'
|
||||
: socketStatus === 'auth_error' || socketStatus === 'offline'
|
||||
? 'bg-[var(--danger)]'
|
||||
: 'bg-[var(--text-faint)]'
|
||||
}`}
|
||||
@@ -419,17 +488,19 @@ export function SettingsWorkspace() {
|
||||
{socketStatus === 'online'
|
||||
? `Online (Port ${serverPort} · ${activeSockets} active)`
|
||||
: socketStatus === 'checking'
|
||||
? 'Probing ingestion server...'
|
||||
: socketStatus === 'offline'
|
||||
? `Offline (Port ${serverPort} unreachable)`
|
||||
: `Port ${serverPort} · ${activeSockets} active socket(s)`}
|
||||
? 'Probing ingestion bridge...'
|
||||
: socketStatus === 'auth_error'
|
||||
? `Bridge Auth Error (Port ${serverPort})`
|
||||
: socketStatus === 'offline'
|
||||
? `Offline (Port ${serverPort} unreachable)`
|
||||
: `Port ${serverPort} · ${activeSockets} active socket(s)`}
|
||||
</span>
|
||||
<Button
|
||||
type="button"
|
||||
size="xs"
|
||||
variant="ghost"
|
||||
icon="RefreshCw"
|
||||
onClick={() => void probeSocketServer(serverPort)}
|
||||
onClick={() => void probeSocketServer()}
|
||||
>
|
||||
Probe
|
||||
</Button>
|
||||
@@ -443,14 +514,18 @@ export function SettingsWorkspace() {
|
||||
className="border-[var(--border)] bg-[var(--surface-2)] font-mono text-[11px] text-[var(--text-muted)]"
|
||||
/>
|
||||
</SettingsField>
|
||||
<SettingsField label="Socket port">
|
||||
<Input
|
||||
compact
|
||||
type="number"
|
||||
value={serverPort}
|
||||
readOnly
|
||||
className="border-[var(--border)] bg-[var(--surface-2)] font-mono text-[var(--text-muted)]"
|
||||
/>
|
||||
<SettingsField label="Socket port (PORT_SOCKET)">
|
||||
<div className="flex flex-col gap-1">
|
||||
<Input
|
||||
compact
|
||||
value={String(serverPort)}
|
||||
readOnly
|
||||
className="border-[var(--border)] bg-[var(--surface-2)] font-mono text-[11px] text-[var(--text-muted)]"
|
||||
/>
|
||||
<span className="text-[10.5px] text-[var(--text-faint)]">
|
||||
Read-only runtime port managed via PORT_SOCKET environment variable.
|
||||
</span>
|
||||
</div>
|
||||
</SettingsField>
|
||||
</SettingsGroup>
|
||||
<SettingsGroup title="Event buffer">
|
||||
@@ -458,10 +533,12 @@ export function SettingsWorkspace() {
|
||||
<Input
|
||||
compact
|
||||
type="number"
|
||||
min="1"
|
||||
value={settings.batchFlushThreshold}
|
||||
min={SETTINGS_LIMITS.batchFlushThreshold.min}
|
||||
max={SETTINGS_LIMITS.batchFlushThreshold.max}
|
||||
value={numericDrafts.batchFlushThreshold}
|
||||
error={fieldErrors.batchFlushThreshold}
|
||||
onChange={(event) =>
|
||||
updateNumber('batchFlushThreshold', event.target.value)
|
||||
updateNumberDraft('batchFlushThreshold', event.target.value)
|
||||
}
|
||||
/>
|
||||
</SettingsField>
|
||||
@@ -469,11 +546,13 @@ export function SettingsWorkspace() {
|
||||
<Input
|
||||
compact
|
||||
type="number"
|
||||
min="100"
|
||||
min={SETTINGS_LIMITS.batchFlushIntervalMs.min}
|
||||
max={SETTINGS_LIMITS.batchFlushIntervalMs.max}
|
||||
suffix="ms"
|
||||
value={settings.batchFlushIntervalMs}
|
||||
value={numericDrafts.batchFlushIntervalMs}
|
||||
error={fieldErrors.batchFlushIntervalMs}
|
||||
onChange={(event) =>
|
||||
updateNumber('batchFlushIntervalMs', event.target.value)
|
||||
updateNumberDraft('batchFlushIntervalMs', event.target.value)
|
||||
}
|
||||
className="font-mono"
|
||||
/>
|
||||
@@ -497,11 +576,13 @@ export function SettingsWorkspace() {
|
||||
<Input
|
||||
compact
|
||||
type="number"
|
||||
min="1"
|
||||
min={SETTINGS_LIMITS.fileRotationMb.min}
|
||||
max={SETTINGS_LIMITS.fileRotationMb.max}
|
||||
suffix="MB"
|
||||
value={settings.fileRotationMb}
|
||||
value={numericDrafts.fileRotationMb}
|
||||
error={fieldErrors.fileRotationMb}
|
||||
onChange={(event) =>
|
||||
updateNumber('fileRotationMb', event.target.value)
|
||||
updateNumberDraft('fileRotationMb', event.target.value)
|
||||
}
|
||||
className="font-mono"
|
||||
/>
|
||||
@@ -510,12 +591,13 @@ export function SettingsWorkspace() {
|
||||
<Input
|
||||
compact
|
||||
type="number"
|
||||
min="0"
|
||||
max="365"
|
||||
min={SETTINGS_LIMITS.autoPruneDays.min}
|
||||
max={SETTINGS_LIMITS.autoPruneDays.max}
|
||||
suffix="days"
|
||||
value={settings.autoPruneDays}
|
||||
value={numericDrafts.autoPruneDays}
|
||||
error={fieldErrors.autoPruneDays}
|
||||
onChange={(event) =>
|
||||
updateNumber('autoPruneDays', event.target.value)
|
||||
updateNumberDraft('autoPruneDays', event.target.value)
|
||||
}
|
||||
className="font-mono"
|
||||
/>
|
||||
@@ -557,12 +639,13 @@ export function SettingsWorkspace() {
|
||||
<Input
|
||||
compact
|
||||
type="number"
|
||||
min="1"
|
||||
max="5120"
|
||||
min={SETTINGS_LIMITS.maxPayloadSizeKb.min}
|
||||
max={SETTINGS_LIMITS.maxPayloadSizeKb.max}
|
||||
suffix="KB"
|
||||
value={settings.maxPayloadSizeKb}
|
||||
value={numericDrafts.maxPayloadSizeKb}
|
||||
error={fieldErrors.maxPayloadSizeKb}
|
||||
onChange={(event) =>
|
||||
updateNumber('maxPayloadSizeKb', event.target.value)
|
||||
updateNumberDraft('maxPayloadSizeKb', event.target.value)
|
||||
}
|
||||
className="font-mono"
|
||||
/>
|
||||
@@ -628,7 +711,7 @@ export function SettingsWorkspace() {
|
||||
variant="danger"
|
||||
size="xs"
|
||||
loading={saving}
|
||||
onClick={() => void executeSave()}
|
||||
onClick={() => void handleConfirmedRotateSave()}
|
||||
>
|
||||
Rotate & Save
|
||||
</Button>
|
||||
|
||||
+103
-85
@@ -9,6 +9,98 @@ type CoreDatabaseGlobals = typeof globalThis & {
|
||||
|
||||
const globals = globalThis as CoreDatabaseGlobals;
|
||||
|
||||
export const CORE_SCHEMA_TABLE_DDLS = [
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS client_sdks (
|
||||
client_id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
secret TEXT NOT NULL,
|
||||
token TEXT NOT NULL,
|
||||
rate_limit INTEGER NOT NULL DEFAULT 100,
|
||||
status TEXT NOT NULL DEFAULT 'active',
|
||||
total_events BIGINT NOT NULL DEFAULT 0,
|
||||
last_active TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`,
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS admin_settings (
|
||||
id TEXT PRIMARY KEY DEFAULT 'default',
|
||||
server_port INTEGER NOT NULL DEFAULT 4000,
|
||||
web_port INTEGER NOT NULL DEFAULT 3000,
|
||||
max_events_per_sec INTEGER NOT NULL DEFAULT 100,
|
||||
max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
|
||||
batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
|
||||
batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
|
||||
file_rotation_mb INTEGER NOT NULL DEFAULT 100,
|
||||
jwt_secret TEXT NOT NULL,
|
||||
enable_idempotency BOOLEAN NOT NULL DEFAULT true,
|
||||
auto_prune_days INTEGER NOT NULL DEFAULT 30,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`,
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS deleted_clients (
|
||||
client_id TEXT PRIMARY KEY,
|
||||
deleted_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`,
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS admin_users (
|
||||
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
|
||||
email TEXT NOT NULL UNIQUE,
|
||||
password_salt BYTEA NOT NULL,
|
||||
password_hash BYTEA NOT NULL,
|
||||
token_version INTEGER NOT NULL DEFAULT 1,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`,
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS admin_login_attempts (
|
||||
key_hash TEXT PRIMARY KEY,
|
||||
window_started_at TIMESTAMPTZ NOT NULL,
|
||||
failed_attempts INTEGER NOT NULL,
|
||||
blocked_until TIMESTAMPTZ,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`,
|
||||
] as const;
|
||||
|
||||
export const CORE_SCHEMA_ALTER_DDLS = [
|
||||
`
|
||||
ALTER TABLE client_sdks
|
||||
ADD COLUMN IF NOT EXISTS rate_limit INTEGER NOT NULL DEFAULT 100,
|
||||
ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'active',
|
||||
ADD COLUMN IF NOT EXISTS total_events BIGINT NOT NULL DEFAULT 0,
|
||||
ADD COLUMN IF NOT EXISTS last_active TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
`,
|
||||
`
|
||||
ALTER TABLE admin_settings
|
||||
ADD COLUMN IF NOT EXISTS server_port INTEGER NOT NULL DEFAULT 4000,
|
||||
ADD COLUMN IF NOT EXISTS web_port INTEGER NOT NULL DEFAULT 3000,
|
||||
ADD COLUMN IF NOT EXISTS max_events_per_sec INTEGER NOT NULL DEFAULT 100,
|
||||
ADD COLUMN IF NOT EXISTS max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
|
||||
ADD COLUMN IF NOT EXISTS batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
|
||||
ADD COLUMN IF NOT EXISTS batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
|
||||
ADD COLUMN IF NOT EXISTS file_rotation_mb INTEGER NOT NULL DEFAULT 100,
|
||||
ADD COLUMN IF NOT EXISTS enable_idempotency BOOLEAN NOT NULL DEFAULT true,
|
||||
ADD COLUMN IF NOT EXISTS auto_prune_days INTEGER NOT NULL DEFAULT 30,
|
||||
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
`,
|
||||
`
|
||||
ALTER TABLE admin_users
|
||||
ADD COLUMN IF NOT EXISTS token_version INTEGER NOT NULL DEFAULT 1
|
||||
`,
|
||||
] as const;
|
||||
|
||||
export async function ensureCoreSchemaOnPool(pool: Pool): Promise<void> {
|
||||
await Promise.all(CORE_SCHEMA_TABLE_DDLS.map((sql) => pool.query(sql)));
|
||||
await Promise.all(CORE_SCHEMA_ALTER_DDLS.map((sql) => pool.query(sql)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Database Module (PostgreSQL Persistence Engine)
|
||||
* CRITICAL RULE: PostgreSQL stores strictly administrative metadata,
|
||||
@@ -17,19 +109,26 @@ const globals = globalThis as CoreDatabaseGlobals;
|
||||
*/
|
||||
export class DatabaseEngine {
|
||||
public static getPool(): Pool | null {
|
||||
const connectionString = process.env.DATABASE_URL;
|
||||
const connectionString = process.env.DATABASE_URL?.trim();
|
||||
if (!connectionString) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (globals.__dataStackCorePool === undefined) {
|
||||
try {
|
||||
globals.__dataStackCorePool = new Pool({
|
||||
const pool = new Pool({
|
||||
connectionString,
|
||||
connectionTimeoutMillis: 3000,
|
||||
idleTimeoutMillis: 30000,
|
||||
max: 5,
|
||||
max: 10,
|
||||
});
|
||||
pool.on("error", (err) => {
|
||||
console.error(
|
||||
"[DataStack DB] Unexpected idle PostgreSQL pool error:",
|
||||
err instanceof Error ? err.message : err,
|
||||
);
|
||||
});
|
||||
globals.__dataStackCorePool = pool;
|
||||
} catch {
|
||||
globals.__dataStackCorePool = null;
|
||||
}
|
||||
@@ -42,88 +141,7 @@ export class DatabaseEngine {
|
||||
const pool = this.getPool();
|
||||
if (!pool) return false;
|
||||
|
||||
globals.__dataStackCoreSchema ??= Promise.all([
|
||||
pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS client_sdks (
|
||||
client_id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
secret TEXT NOT NULL,
|
||||
token TEXT NOT NULL,
|
||||
rate_limit INTEGER NOT NULL DEFAULT 100,
|
||||
status TEXT NOT NULL DEFAULT 'active',
|
||||
total_events BIGINT NOT NULL DEFAULT 0,
|
||||
last_active TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`),
|
||||
pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS admin_settings (
|
||||
id TEXT PRIMARY KEY DEFAULT 'default',
|
||||
server_port INTEGER NOT NULL DEFAULT 4000,
|
||||
web_port INTEGER NOT NULL DEFAULT 3000,
|
||||
max_events_per_sec INTEGER NOT NULL DEFAULT 100,
|
||||
max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
|
||||
batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
|
||||
batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
|
||||
file_rotation_mb INTEGER NOT NULL DEFAULT 100,
|
||||
jwt_secret TEXT NOT NULL,
|
||||
enable_idempotency BOOLEAN NOT NULL DEFAULT true,
|
||||
auto_prune_days INTEGER NOT NULL DEFAULT 30,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`),
|
||||
pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS deleted_clients (
|
||||
client_id TEXT PRIMARY KEY,
|
||||
deleted_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`),
|
||||
pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS admin_users (
|
||||
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
|
||||
email TEXT NOT NULL UNIQUE,
|
||||
password_salt BYTEA NOT NULL,
|
||||
password_hash BYTEA NOT NULL,
|
||||
token_version INTEGER NOT NULL DEFAULT 1,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`),
|
||||
pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS admin_login_attempts (
|
||||
key_hash TEXT PRIMARY KEY,
|
||||
window_started_at TIMESTAMPTZ NOT NULL,
|
||||
failed_attempts INTEGER NOT NULL,
|
||||
blocked_until TIMESTAMPTZ,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`),
|
||||
]).then(() =>
|
||||
Promise.all([
|
||||
pool.query(`
|
||||
ALTER TABLE client_sdks
|
||||
ADD COLUMN IF NOT EXISTS rate_limit INTEGER NOT NULL DEFAULT 100,
|
||||
ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'active',
|
||||
ADD COLUMN IF NOT EXISTS total_events BIGINT NOT NULL DEFAULT 0,
|
||||
ADD COLUMN IF NOT EXISTS last_active TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
`),
|
||||
pool.query(`
|
||||
ALTER TABLE admin_settings
|
||||
ADD COLUMN IF NOT EXISTS server_port INTEGER NOT NULL DEFAULT 4000,
|
||||
ADD COLUMN IF NOT EXISTS web_port INTEGER NOT NULL DEFAULT 3000,
|
||||
ADD COLUMN IF NOT EXISTS max_events_per_sec INTEGER NOT NULL DEFAULT 100,
|
||||
ADD COLUMN IF NOT EXISTS max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
|
||||
ADD COLUMN IF NOT EXISTS batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
|
||||
ADD COLUMN IF NOT EXISTS batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
|
||||
ADD COLUMN IF NOT EXISTS file_rotation_mb INTEGER NOT NULL DEFAULT 100,
|
||||
ADD COLUMN IF NOT EXISTS enable_idempotency BOOLEAN NOT NULL DEFAULT true,
|
||||
ADD COLUMN IF NOT EXISTS auto_prune_days INTEGER NOT NULL DEFAULT 30,
|
||||
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
`),
|
||||
]),
|
||||
).then(() => undefined);
|
||||
globals.__dataStackCoreSchema ??= ensureCoreSchemaOnPool(pool);
|
||||
|
||||
try {
|
||||
await globals.__dataStackCoreSchema;
|
||||
|
||||
+206
-1
@@ -1,5 +1,17 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { AuthService } from "../auth/jwt";
|
||||
import { DataBufferEngine, type RedisBufferClient } from "../data/buffer";
|
||||
|
||||
const REDIS_IDEMPOTENCY_PREFIX = "datastack:idem:";
|
||||
const REDIS_RATE_LIMIT_PREFIX = "datastack:rl:";
|
||||
export const REDIS_RATE_LIMIT_LUA_SCRIPT = `
|
||||
-- DATASTACK_RATE_LIMIT_INCR
|
||||
local c = redis.call("INCR", KEYS[1])
|
||||
if c == 1 or redis.call("PTTL", KEYS[1]) < 0 then
|
||||
redis.call("PEXPIRE", KEYS[1], tonumber(ARGV[1]))
|
||||
end
|
||||
return c
|
||||
`.trim();
|
||||
|
||||
export function resolveInternalIngestionSecret(
|
||||
jwtSecretOverride?: string,
|
||||
@@ -43,9 +55,15 @@ export interface ValidationResult {
|
||||
| "MALFORMED_PAYLOAD";
|
||||
}
|
||||
|
||||
export type SecurityMode =
|
||||
| "redis_shared"
|
||||
| "bridge_coordinated"
|
||||
| "degraded_local";
|
||||
|
||||
export class SecurityEngine {
|
||||
public static maxEventsPerSec = 100;
|
||||
public static enableIdempotency = true;
|
||||
private static lastSecurityMode: SecurityMode = "degraded_local";
|
||||
|
||||
// Idempotency cache: key -> timestamp (expires after TTL)
|
||||
private static idempotencySet = new Map<string, number>();
|
||||
@@ -126,6 +144,32 @@ export class SecurityEngine {
|
||||
return true;
|
||||
}
|
||||
|
||||
private static trackLocalRateWindow(clientId: string): void {
|
||||
const now = Date.now();
|
||||
const current = this.rateLimitMap.get(clientId);
|
||||
if (!current || now - current.windowStart >= 1000) {
|
||||
this.rateLimitMap.set(clientId, { count: 1, windowStart: now });
|
||||
} else {
|
||||
current.count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
private static supportsSharedSecurityRedis(
|
||||
redis: RedisBufferClient | null,
|
||||
): redis is RedisBufferClient & {
|
||||
get: (key: string) => Promise<string | null>;
|
||||
incr: (key: string) => Promise<number>;
|
||||
pexpire: (key: string, ttlMs: number) => Promise<number>;
|
||||
} {
|
||||
return Boolean(
|
||||
redis &&
|
||||
typeof redis.get === "function" &&
|
||||
typeof redis.incr === "function" &&
|
||||
typeof redis.pexpire === "function" &&
|
||||
typeof redis.set === "function",
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates incoming event payload:
|
||||
* Checks duplicate idempotency (read-only), max byte size, and rate limit,
|
||||
@@ -148,7 +192,7 @@ export class SecurityEngine {
|
||||
|
||||
// 2. Check payload size before consuming rate-limit quota
|
||||
const payloadStr =
|
||||
typeof payload === "string" ? payload : JSON.stringify(payload);
|
||||
typeof payload === "string" ? payload : JSON.stringify(payload ?? {});
|
||||
const byteLength = Buffer.byteLength(payloadStr, "utf8");
|
||||
|
||||
if (byteLength > this.maxPayloadSizeBytes) {
|
||||
@@ -178,6 +222,158 @@ export class SecurityEngine {
|
||||
return { allowed: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared Redis-backed async validator with Port 4000 internal-bridge fallback
|
||||
* (so HTTP + WS share rate limits & idempotency even when Redis is unconfigured)
|
||||
* and final local-Map fallback for isolated single-process dev.
|
||||
*/
|
||||
public static async validateEventAsync(
|
||||
clientId: string,
|
||||
eventId: string,
|
||||
payload: unknown,
|
||||
clientMaxRate = 100,
|
||||
): Promise<ValidationResult> {
|
||||
const redis = DataBufferEngine.getRedisClient();
|
||||
if (!this.supportsSharedSecurityRedis(redis)) {
|
||||
const payloadStr =
|
||||
typeof payload === "string" ? payload : JSON.stringify(payload ?? {});
|
||||
const payloadBytes = Buffer.byteLength(payloadStr, "utf8");
|
||||
if (payloadBytes > this.maxPayloadSizeBytes) {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: `Payload size (${(payloadBytes / 1024).toFixed(1)} KB) exceeds limit of ${this.maxPayloadSizeBytes / 1024} KB`,
|
||||
code: "PAYLOAD_TOO_LARGE",
|
||||
};
|
||||
}
|
||||
|
||||
const headers = getInternalIngestionHeaders();
|
||||
if (headers["x-datastack-internal-secret"]) {
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||
).replace(/\/$/, "");
|
||||
try {
|
||||
const res = await fetch(`${ingestionBaseUrl}/internal/security-check`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...headers,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
clientId,
|
||||
eventId,
|
||||
payloadBytes,
|
||||
clientMaxRate,
|
||||
}),
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(400),
|
||||
});
|
||||
if (res.ok) {
|
||||
const data = (await res.json()) as {
|
||||
allowed?: boolean;
|
||||
code?: ValidationResult["code"];
|
||||
reason?: string;
|
||||
};
|
||||
this.lastSecurityMode = "bridge_coordinated";
|
||||
return {
|
||||
allowed: Boolean(data.allowed),
|
||||
...(data.code ? { code: data.code } : {}),
|
||||
...(data.reason ? { reason: data.reason } : {}),
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
// Port 4000 unreachable — fall back to local validation
|
||||
}
|
||||
}
|
||||
|
||||
this.lastSecurityMode = "degraded_local";
|
||||
return this.validateEvent(clientId, eventId, payload, clientMaxRate);
|
||||
}
|
||||
|
||||
this.lastSecurityMode = "redis_shared";
|
||||
const idemKey = `${REDIS_IDEMPOTENCY_PREFIX}${clientId}:${eventId}`;
|
||||
try {
|
||||
// 1. Read-only idempotency check
|
||||
if (this.enableIdempotency && eventId) {
|
||||
if (this.hasDuplicate(clientId, eventId)) {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: "Duplicate event dropped silently",
|
||||
code: "DUPLICATE_EVENT",
|
||||
};
|
||||
}
|
||||
const existing = await redis.get(idemKey);
|
||||
if (existing !== null) {
|
||||
this.recordIdempotency(clientId, eventId);
|
||||
return {
|
||||
allowed: false,
|
||||
reason: "Duplicate event dropped silently",
|
||||
code: "DUPLICATE_EVENT",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Payload byte size check before rate-limit consumption
|
||||
const payloadStr =
|
||||
typeof payload === "string" ? payload : JSON.stringify(payload ?? {});
|
||||
const byteLength = Buffer.byteLength(payloadStr, "utf8");
|
||||
if (byteLength > this.maxPayloadSizeBytes) {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: `Payload size (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${this.maxPayloadSizeBytes / 1024} KB`,
|
||||
code: "PAYLOAD_TOO_LARGE",
|
||||
};
|
||||
}
|
||||
|
||||
// 3. Shared atomic per-second rate-limit counter in Redis (single Lua round-trip)
|
||||
const effectiveRateLimit = Math.min(clientMaxRate, this.maxEventsPerSec);
|
||||
const windowBucket = Math.floor(Date.now() / 1000);
|
||||
const rlKey = `${REDIS_RATE_LIMIT_PREFIX}${clientId}:${windowBucket}`;
|
||||
let count: number;
|
||||
if (typeof redis.eval === "function") {
|
||||
count = Number(
|
||||
await redis.eval(REDIS_RATE_LIMIT_LUA_SCRIPT, 1, rlKey, "2000"),
|
||||
);
|
||||
} else {
|
||||
count = Number(await redis.incr(rlKey));
|
||||
if (count === 1) {
|
||||
await redis.pexpire(rlKey, 2000);
|
||||
}
|
||||
}
|
||||
this.trackLocalRateWindow(clientId);
|
||||
|
||||
if (count > effectiveRateLimit) {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: `Rate limit of ${effectiveRateLimit} events/sec exceeded for client ${clientId}`,
|
||||
code: "RATE_LIMIT_EXCEEDED",
|
||||
};
|
||||
}
|
||||
|
||||
// 4. Atomic claim of idempotency key after all checks succeed
|
||||
if (this.enableIdempotency && eventId) {
|
||||
const claimed = await redis.set(
|
||||
idemKey,
|
||||
"1",
|
||||
"PX",
|
||||
this.IDEMPOTENCY_TTL_MS,
|
||||
"NX",
|
||||
);
|
||||
this.recordIdempotency(clientId, eventId);
|
||||
if (claimed !== "OK") {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: "Duplicate event dropped silently",
|
||||
code: "DUPLICATE_EVENT",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return { allowed: true };
|
||||
} catch {
|
||||
return this.validateEvent(clientId, eventId, payload, clientMaxRate);
|
||||
}
|
||||
}
|
||||
|
||||
public static getStats() {
|
||||
const now = Date.now();
|
||||
for (const [k, timestamp] of this.idempotencySet.entries()) {
|
||||
@@ -191,10 +387,19 @@ export class SecurityEngine {
|
||||
}
|
||||
}
|
||||
|
||||
const mode: SecurityMode = this.supportsSharedSecurityRedis(
|
||||
DataBufferEngine.getRedisClient(),
|
||||
)
|
||||
? "redis_shared"
|
||||
: this.lastSecurityMode === "redis_shared"
|
||||
? "degraded_local"
|
||||
: this.lastSecurityMode;
|
||||
|
||||
return {
|
||||
activeIdempotencyKeys: this.idempotencySet.size,
|
||||
rateLimitedClientsCount: this.rateLimitMap.size,
|
||||
maxPayloadSizeBytes: this.maxPayloadSizeBytes,
|
||||
mode,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
+364
-100
@@ -1,5 +1,7 @@
|
||||
import crypto from 'node:crypto';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { StringDecoder } from 'node:string_decoder';
|
||||
import { Parser } from 'json2csv';
|
||||
|
||||
export interface DataEvent {
|
||||
@@ -34,9 +36,17 @@ function sleepMs(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
export function resolveDefaultStorageDir(): string {
|
||||
const configured = (process.env.DATASTACK_STORAGE_DIR || '').trim();
|
||||
if (configured) {
|
||||
return path.resolve(configured);
|
||||
}
|
||||
return path.resolve(process.cwd(), 'storage', 'data');
|
||||
}
|
||||
|
||||
export class CSVStorageManager {
|
||||
// Base storage directory: /storage/data
|
||||
public static baseStorageDir = path.resolve(process.cwd(), 'storage', 'data');
|
||||
// Base storage directory: DATASTACK_STORAGE_DIR or /storage/data
|
||||
public static baseStorageDir = resolveDefaultStorageDir();
|
||||
public static maxFileSizeBytes = 100 * 1024 * 1024; // 100MB rotation limit
|
||||
private static rowCountCache = new Map<string, RowCountCacheEntry>();
|
||||
|
||||
@@ -44,12 +54,94 @@ export class CSVStorageManager {
|
||||
this.rowCountCache.clear();
|
||||
}
|
||||
|
||||
public static sanitizeClientId(clientId: string): string {
|
||||
const raw = String(clientId || 'unknown').trim().replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
return raw || 'unknown';
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures client directory exists
|
||||
* Checks whether the last byte of a non-empty file is LF (\n)
|
||||
*/
|
||||
private static fileEndsWithNewline(filePath: string, sizeBytes: number): boolean {
|
||||
if (sizeBytes <= 0) return false;
|
||||
const fd = fs.openSync(filePath, 'r');
|
||||
try {
|
||||
const lastByte = Buffer.allocUnsafe(1);
|
||||
const read = fs.readSync(fd, lastByte, 0, 1, sizeBytes - 1);
|
||||
return read === 1 && lastByte[0] === 0x0a;
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
}
|
||||
|
||||
private static removeStaleLockAtomically(lockPath: string): void {
|
||||
try {
|
||||
const lockStat = fs.lstatSync(lockPath);
|
||||
if (Date.now() - lockStat.mtimeMs <= STALE_LOCK_AGE_MS) {
|
||||
return;
|
||||
}
|
||||
const staleTmp = `${lockPath}.stale.${crypto.randomUUID()}`;
|
||||
fs.renameSync(lockPath, staleTmp);
|
||||
try {
|
||||
fs.unlinkSync(staleTmp);
|
||||
} catch {
|
||||
// Ignore cleanup failure of already-renamed stale token
|
||||
}
|
||||
} catch {
|
||||
// Lock was released or claimed concurrently
|
||||
}
|
||||
}
|
||||
|
||||
private static releaseLockIfOwned(lockPath: string, lockValue: string): void {
|
||||
try {
|
||||
if (!fs.existsSync(lockPath)) return;
|
||||
const current = fs.readFileSync(lockPath, 'utf8');
|
||||
if (current === lockValue) {
|
||||
fs.unlinkSync(lockPath);
|
||||
}
|
||||
} catch {
|
||||
// Ignore cleanup failure if already unlinked
|
||||
}
|
||||
}
|
||||
|
||||
private static async removeStaleLockAtomicallyAsync(lockPath: string): Promise<void> {
|
||||
try {
|
||||
const lockStat = await fs.promises.lstat(lockPath);
|
||||
if (Date.now() - lockStat.mtimeMs <= STALE_LOCK_AGE_MS) {
|
||||
return;
|
||||
}
|
||||
const staleTmp = `${lockPath}.stale.${crypto.randomUUID()}`;
|
||||
await fs.promises.rename(lockPath, staleTmp);
|
||||
await fs.promises.unlink(staleTmp).catch(() => undefined);
|
||||
} catch {
|
||||
// Lock was released or claimed concurrently
|
||||
}
|
||||
}
|
||||
|
||||
private static async releaseLockIfOwnedAsync(
|
||||
lockPath: string,
|
||||
lockValue: string,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const current = await fs.promises.readFile(lockPath, 'utf8');
|
||||
if (current === lockValue) {
|
||||
await fs.promises.unlink(lockPath);
|
||||
}
|
||||
} catch {
|
||||
// Ignore cleanup failure if already unlinked
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures client directory exists safely within baseStorageDir
|
||||
*/
|
||||
public static ensureClientDir(clientId: string): string {
|
||||
const safeClientId = clientId.replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
const clientDir = path.join(this.baseStorageDir, safeClientId);
|
||||
const safeClientId = this.sanitizeClientId(clientId);
|
||||
const resolvedBase = path.resolve(this.baseStorageDir);
|
||||
const clientDir = path.resolve(resolvedBase, safeClientId);
|
||||
if (!clientDir.startsWith(resolvedBase + path.sep)) {
|
||||
throw new Error(`Invalid storage client directory: ${clientId}`);
|
||||
}
|
||||
if (!fs.existsSync(clientDir)) {
|
||||
fs.mkdirSync(clientDir, { recursive: true });
|
||||
}
|
||||
@@ -67,6 +159,7 @@ export class CSVStorageManager {
|
||||
const clientDir = this.ensureClientDir(clientId);
|
||||
const date = this.extractSafePartitionDate(dateStr);
|
||||
const lockPath = path.join(clientDir, `.${date}.csv.lock`);
|
||||
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
|
||||
const deadline = Date.now() + LOCK_TIMEOUT_MS;
|
||||
|
||||
let acquired = false;
|
||||
@@ -74,7 +167,7 @@ export class CSVStorageManager {
|
||||
try {
|
||||
const fd = fs.openSync(lockPath, 'wx');
|
||||
try {
|
||||
fs.writeFileSync(fd, `${process.pid}:${Date.now()}`, 'utf8');
|
||||
fs.writeFileSync(fd, lockValue, 'utf8');
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
@@ -85,15 +178,7 @@ export class CSVStorageManager {
|
||||
if (code !== 'EEXIST') {
|
||||
throw err;
|
||||
}
|
||||
try {
|
||||
const lockStat = fs.statSync(lockPath);
|
||||
if (Date.now() - lockStat.mtimeMs > STALE_LOCK_AGE_MS) {
|
||||
fs.unlinkSync(lockPath);
|
||||
continue;
|
||||
}
|
||||
} catch {
|
||||
// Lock was released concurrently
|
||||
}
|
||||
this.removeStaleLockAtomically(lockPath);
|
||||
const jitter =
|
||||
LOCK_RETRY_MIN_MS +
|
||||
Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS));
|
||||
@@ -108,13 +193,7 @@ export class CSVStorageManager {
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
try {
|
||||
if (fs.existsSync(lockPath)) {
|
||||
fs.unlinkSync(lockPath);
|
||||
}
|
||||
} catch {
|
||||
// Ignore cleanup failure if already unlinked
|
||||
}
|
||||
this.releaseLockIfOwned(lockPath, lockValue);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -158,51 +237,58 @@ export class CSVStorageManager {
|
||||
* Stateful RFC 4180 quote-aware chunked row counter so multi-100MB CSVs never load wholly into RAM
|
||||
*/
|
||||
private static createChunkRowCounter() {
|
||||
const decoder = new StringDecoder('utf8');
|
||||
let inQuotes = false;
|
||||
let pendingQuote = false;
|
||||
let currentLineHasChars = false;
|
||||
let totalRows = 0;
|
||||
|
||||
return {
|
||||
pushChunk(chunk: Buffer | string) {
|
||||
const str = typeof chunk === 'string' ? chunk : chunk.toString('utf8');
|
||||
for (let i = 0; i < str.length; i += 1) {
|
||||
const ch = str[i];
|
||||
if (pendingQuote) {
|
||||
pendingQuote = false;
|
||||
if (ch === '"') {
|
||||
// Escaped quote "" across chunk boundary or inline
|
||||
continue;
|
||||
}
|
||||
inQuotes = !inQuotes;
|
||||
}
|
||||
|
||||
const processStr = (str: string) => {
|
||||
for (let i = 0; i < str.length; i += 1) {
|
||||
const ch = str[i];
|
||||
if (pendingQuote) {
|
||||
pendingQuote = false;
|
||||
if (ch === '"') {
|
||||
currentLineHasChars = true;
|
||||
if (inQuotes) {
|
||||
if (i + 1 < str.length) {
|
||||
if (str[i + 1] === '"') {
|
||||
i += 1;
|
||||
} else {
|
||||
inQuotes = false;
|
||||
}
|
||||
// Escaped quote "" across chunk boundary or inline
|
||||
continue;
|
||||
}
|
||||
inQuotes = !inQuotes;
|
||||
}
|
||||
|
||||
if (ch === '"') {
|
||||
currentLineHasChars = true;
|
||||
if (inQuotes) {
|
||||
if (i + 1 < str.length) {
|
||||
if (str[i + 1] === '"') {
|
||||
i += 1;
|
||||
} else {
|
||||
pendingQuote = true;
|
||||
inQuotes = false;
|
||||
}
|
||||
} else {
|
||||
inQuotes = true;
|
||||
pendingQuote = true;
|
||||
}
|
||||
} else if (ch === '\n' && !inQuotes) {
|
||||
if (currentLineHasChars) {
|
||||
totalRows += 1;
|
||||
currentLineHasChars = false;
|
||||
}
|
||||
} else if (ch !== '\r' && ch !== ' ' && ch !== '\t') {
|
||||
currentLineHasChars = true;
|
||||
} else {
|
||||
inQuotes = true;
|
||||
}
|
||||
} else if (ch === '\n' && !inQuotes) {
|
||||
if (currentLineHasChars) {
|
||||
totalRows += 1;
|
||||
currentLineHasChars = false;
|
||||
}
|
||||
} else if (ch !== '\r' && ch !== ' ' && ch !== '\t') {
|
||||
currentLineHasChars = true;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
pushChunk(chunk: Buffer | string) {
|
||||
const str = typeof chunk === 'string' ? chunk : decoder.write(chunk);
|
||||
if (str) processStr(str);
|
||||
},
|
||||
finish(): number {
|
||||
const trailing = decoder.end();
|
||||
if (trailing) processStr(trailing);
|
||||
if (currentLineHasChars) {
|
||||
totalRows += 1;
|
||||
}
|
||||
@@ -243,7 +329,11 @@ export class CSVStorageManager {
|
||||
/**
|
||||
* Resolves target CSV file path with daily and configured MB rotation rules
|
||||
*/
|
||||
public static getTargetFilePath(clientId: string, dateStr?: string): string {
|
||||
public static getTargetFilePath(
|
||||
clientId: string,
|
||||
dateStr?: string,
|
||||
incomingBytes = 0,
|
||||
): string {
|
||||
const clientDir = this.ensureClientDir(clientId);
|
||||
const date = this.extractSafePartitionDate(dateStr);
|
||||
let baseFileName = `${date}.csv`;
|
||||
@@ -252,16 +342,24 @@ export class CSVStorageManager {
|
||||
// Check for file rotation
|
||||
let partIndex = 1;
|
||||
while (fs.existsSync(targetPath)) {
|
||||
let stats: fs.Stats;
|
||||
try {
|
||||
const stats = fs.statSync(targetPath);
|
||||
if (stats.size >= this.maxFileSizeBytes) {
|
||||
partIndex += 1;
|
||||
baseFileName = `${date}-part${partIndex}.csv`;
|
||||
targetPath = path.join(clientDir, baseFileName);
|
||||
} else {
|
||||
stats = fs.statSync(targetPath);
|
||||
} catch (err: unknown) {
|
||||
if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') {
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
throw err;
|
||||
}
|
||||
const overflowsLimit =
|
||||
stats.size > 0 &&
|
||||
incomingBytes > 0 &&
|
||||
stats.size + incomingBytes > this.maxFileSizeBytes;
|
||||
if (stats.size >= this.maxFileSizeBytes || overflowsLimit) {
|
||||
partIndex += 1;
|
||||
baseFileName = `${date}-part${partIndex}.csv`;
|
||||
targetPath = path.join(clientDir, baseFileName);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -280,49 +378,63 @@ export class CSVStorageManager {
|
||||
return { written: 0, filesUpdated: [] };
|
||||
}
|
||||
|
||||
// Group events by client + date
|
||||
const grouped = new Map<string, DataEvent[]>();
|
||||
// Group events by client + date without ambiguous delimiter splitting
|
||||
const grouped = new Map<string, { clientId: string; date: string; events: DataEvent[] }>();
|
||||
for (const evt of events) {
|
||||
const date = this.extractSafePartitionDate(evt.timestamp);
|
||||
const key = `${evt.clientId}:::${date}`;
|
||||
if (!grouped.has(key)) {
|
||||
grouped.set(key, []);
|
||||
const clientId = String(evt.clientId || 'unknown');
|
||||
const key = JSON.stringify([clientId, date]);
|
||||
let bucket = grouped.get(key);
|
||||
if (!bucket) {
|
||||
bucket = { clientId, date, events: [] };
|
||||
grouped.set(key, bucket);
|
||||
}
|
||||
grouped.get(key)!.push(evt);
|
||||
bucket.events.push(evt);
|
||||
}
|
||||
|
||||
const filesUpdated: string[] = [];
|
||||
const fields = ['clientId', 'eventId', 'timestamp', 'payload'];
|
||||
|
||||
for (const [key, batch] of grouped.entries()) {
|
||||
const [clientId, date] = key.split(':::');
|
||||
for (const { clientId, date, events: batch } of grouped.values()) {
|
||||
const cleanBatch = batch.map((item) => ({
|
||||
clientId: item.clientId,
|
||||
eventId: item.eventId,
|
||||
timestamp: item.timestamp,
|
||||
payload:
|
||||
typeof item.payload === 'string'
|
||||
? item.payload
|
||||
: JSON.stringify(item.payload),
|
||||
}));
|
||||
|
||||
const bodyCsvNoHeader = new Parser({ fields, header: false, eol: '\n' }).parse(
|
||||
cleanBatch,
|
||||
);
|
||||
const incomingBytes = Buffer.byteLength(bodyCsvNoHeader, 'utf8') + 1;
|
||||
|
||||
const targetPath = await this.withPartitionLock(clientId, date, () => {
|
||||
const resolvedPath = this.getTargetFilePath(clientId, date);
|
||||
let hasContent = false;
|
||||
const resolvedPath = this.getTargetFilePath(clientId, date, incomingBytes);
|
||||
let sizeBytes = 0;
|
||||
try {
|
||||
hasContent =
|
||||
fs.existsSync(resolvedPath) && fs.statSync(resolvedPath).size > 0;
|
||||
if (fs.existsSync(resolvedPath)) {
|
||||
sizeBytes = fs.statSync(resolvedPath).size;
|
||||
}
|
||||
} catch {
|
||||
hasContent = false;
|
||||
sizeBytes = 0;
|
||||
}
|
||||
|
||||
const cleanBatch = batch.map((item) => ({
|
||||
clientId: item.clientId,
|
||||
eventId: item.eventId,
|
||||
timestamp: item.timestamp,
|
||||
payload:
|
||||
typeof item.payload === 'string'
|
||||
? item.payload
|
||||
: JSON.stringify(item.payload),
|
||||
}));
|
||||
|
||||
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
|
||||
const csvData = parser.parse(cleanBatch);
|
||||
const hasContent = sizeBytes > 0;
|
||||
|
||||
if (hasContent) {
|
||||
fs.appendFileSync(resolvedPath, '\n' + csvData, 'utf8');
|
||||
const endsWithNl = this.fileEndsWithNewline(resolvedPath, sizeBytes);
|
||||
fs.appendFileSync(
|
||||
resolvedPath,
|
||||
(endsWithNl ? '' : '\n') + bodyCsvNoHeader,
|
||||
'utf8',
|
||||
);
|
||||
} else {
|
||||
fs.writeFileSync(resolvedPath, csvData, 'utf8');
|
||||
const fullCsv = new Parser({ fields, header: true, eol: '\n' }).parse(
|
||||
cleanBatch,
|
||||
);
|
||||
fs.writeFileSync(resolvedPath, fullCsv, 'utf8');
|
||||
}
|
||||
|
||||
// Invalidate row cache for modified file immediately
|
||||
@@ -351,13 +463,25 @@ export class CSVStorageManager {
|
||||
const clientDirs = fs.readdirSync(this.baseStorageDir);
|
||||
for (const clientDir of clientDirs) {
|
||||
const fullClientPath = path.join(this.baseStorageDir, clientDir);
|
||||
if (!fs.statSync(fullClientPath).isDirectory()) continue;
|
||||
let dirLstat: fs.Stats;
|
||||
try {
|
||||
dirLstat = fs.lstatSync(fullClientPath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
|
||||
|
||||
const files = fs.readdirSync(fullClientPath);
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
const fullFilePath = path.join(fullClientPath, file);
|
||||
const stats = fs.statSync(fullFilePath);
|
||||
let stats: fs.Stats;
|
||||
try {
|
||||
stats = fs.lstatSync(fullFilePath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (stats.isSymbolicLink() || !stats.isFile()) continue;
|
||||
activePaths.add(fullFilePath);
|
||||
|
||||
const cached = this.rowCountCache.get(fullFilePath);
|
||||
@@ -422,15 +546,15 @@ export class CSVStorageManager {
|
||||
const clientDirs = await fs.promises.readdir(this.baseStorageDir);
|
||||
for (const clientDir of clientDirs) {
|
||||
const fullClientPath = path.join(this.baseStorageDir, clientDir);
|
||||
const dirStat = await fs.promises.stat(fullClientPath).catch(() => null);
|
||||
if (!dirStat || !dirStat.isDirectory()) continue;
|
||||
const dirLstat = await fs.promises.lstat(fullClientPath).catch(() => null);
|
||||
if (!dirLstat || dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
|
||||
|
||||
const files = await fs.promises.readdir(fullClientPath);
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
const fullFilePath = path.join(fullClientPath, file);
|
||||
const stats = await fs.promises.stat(fullFilePath).catch(() => null);
|
||||
if (!stats) continue;
|
||||
const stats = await fs.promises.lstat(fullFilePath).catch(() => null);
|
||||
if (!stats || stats.isSymbolicLink() || !stats.isFile()) continue;
|
||||
activePaths.add(fullFilePath);
|
||||
|
||||
const cached = this.rowCountCache.get(fullFilePath);
|
||||
@@ -481,7 +605,7 @@ export class CSVStorageManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Prune CSV partitions older than autoPruneDays without counting row contents
|
||||
* Prune CSV partitions older than autoPruneDays under cooperative partition locks
|
||||
*/
|
||||
public static pruneExpiredPartitions(autoPruneDays: number): number {
|
||||
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
|
||||
@@ -502,7 +626,13 @@ export class CSVStorageManager {
|
||||
const clientDirs = fs.readdirSync(this.baseStorageDir);
|
||||
for (const clientDir of clientDirs) {
|
||||
const fullClientPath = path.join(this.baseStorageDir, clientDir);
|
||||
if (!fs.statSync(fullClientPath).isDirectory()) continue;
|
||||
let dirLstat: fs.Stats;
|
||||
try {
|
||||
dirLstat = fs.lstatSync(fullClientPath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
|
||||
|
||||
const files = fs.readdirSync(fullClientPath);
|
||||
for (const file of files) {
|
||||
@@ -510,6 +640,45 @@ export class CSVStorageManager {
|
||||
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
|
||||
const fullPath = path.join(fullClientPath, file);
|
||||
let fileLstat: fs.Stats;
|
||||
try {
|
||||
fileLstat = fs.lstatSync(fullPath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (fileLstat.isSymbolicLink() || !fileLstat.isFile()) continue;
|
||||
|
||||
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
|
||||
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
|
||||
let acquired = false;
|
||||
try {
|
||||
const fd = fs.openSync(lockPath, 'wx');
|
||||
try {
|
||||
fs.writeFileSync(fd, lockValue, 'utf8');
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
acquired = true;
|
||||
} catch (err: unknown) {
|
||||
if ((err as NodeJS.ErrnoException)?.code === 'EEXIST') {
|
||||
this.removeStaleLockAtomically(lockPath);
|
||||
try {
|
||||
const fd = fs.openSync(lockPath, 'wx');
|
||||
try {
|
||||
fs.writeFileSync(fd, lockValue, 'utf8');
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
acquired = true;
|
||||
} catch {
|
||||
// Partition is currently locked by an active writer; skip safely
|
||||
acquired = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!acquired) continue;
|
||||
|
||||
try {
|
||||
if (fs.existsSync(fullPath)) {
|
||||
fs.unlinkSync(fullPath);
|
||||
@@ -518,6 +687,8 @@ export class CSVStorageManager {
|
||||
}
|
||||
} catch {
|
||||
// Ignore file removal error
|
||||
} finally {
|
||||
this.releaseLockIfOwned(lockPath, lockValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -529,7 +700,96 @@ export class CSVStorageManager {
|
||||
return prunedCount;
|
||||
}
|
||||
|
||||
private static formatStorageStats(files: StorageFileMeta[]) {
|
||||
/**
|
||||
* Non-blocking async partition pruner using fs.promises under cooperative partition locks
|
||||
*/
|
||||
public static async pruneExpiredPartitionsAsync(
|
||||
autoPruneDays: number,
|
||||
): Promise<number> {
|
||||
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const baseStat = await fs.promises.lstat(this.baseStorageDir).catch(() => null);
|
||||
if (!baseStat || !baseStat.isDirectory()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const cutoffDate = new Date(
|
||||
Date.now() - Math.floor(autoPruneDays) * 86_400_000,
|
||||
)
|
||||
.toISOString()
|
||||
.split('T')[0];
|
||||
|
||||
let prunedCount = 0;
|
||||
try {
|
||||
const clientDirs = await fs.promises.readdir(this.baseStorageDir);
|
||||
for (const clientDir of clientDirs) {
|
||||
const fullClientPath = path.join(this.baseStorageDir, clientDir);
|
||||
const dirLstat = await fs.promises.lstat(fullClientPath).catch(() => null);
|
||||
if (!dirLstat || dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const files = await fs.promises.readdir(fullClientPath).catch(() => [] as string[]);
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
|
||||
const fullPath = path.join(fullClientPath, file);
|
||||
const fileLstat = await fs.promises.lstat(fullPath).catch(() => null);
|
||||
if (!fileLstat || fileLstat.isSymbolicLink() || !fileLstat.isFile()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
|
||||
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
|
||||
let acquired = false;
|
||||
const tryAcquireLock = async (): Promise<boolean> => {
|
||||
const handle = await fs.promises.open(lockPath, 'wx');
|
||||
try {
|
||||
await handle.writeFile(lockValue, 'utf8');
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
try {
|
||||
acquired = await tryAcquireLock();
|
||||
} catch (err: unknown) {
|
||||
if ((err as NodeJS.ErrnoException)?.code === 'EEXIST') {
|
||||
await this.removeStaleLockAtomicallyAsync(lockPath);
|
||||
try {
|
||||
acquired = await tryAcquireLock();
|
||||
} catch {
|
||||
acquired = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!acquired) continue;
|
||||
|
||||
try {
|
||||
await fs.promises.unlink(fullPath);
|
||||
this.rowCountCache.delete(fullPath);
|
||||
prunedCount += 1;
|
||||
} catch {
|
||||
// Ignore individual unlink error or concurrent removal
|
||||
} finally {
|
||||
await this.releaseLockIfOwnedAsync(lockPath, lockValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore directory scan failure
|
||||
}
|
||||
|
||||
return prunedCount;
|
||||
}
|
||||
|
||||
public static formatStorageStats(files: StorageFileMeta[]) {
|
||||
const totalBytes = files.reduce((acc, f) => acc + f.sizeBytes, 0);
|
||||
const totalRows = files.reduce((acc, f) => acc + f.linesCount, 0);
|
||||
const uniqueClients = new Set(files.map((f) => f.clientId)).size;
|
||||
@@ -569,17 +829,21 @@ export class CSVStorageManager {
|
||||
* Reads file content for export / download with path traversal protection
|
||||
*/
|
||||
public static getFileContent(clientId: string, filename: string): string | null {
|
||||
const safeClientId = String(clientId || '').replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
const safeClientId = this.sanitizeClientId(clientId);
|
||||
const safeFilename = path.basename(String(filename || ''));
|
||||
if (!safeClientId || !safeFilename || !safeFilename.endsWith('.csv')) {
|
||||
return null;
|
||||
}
|
||||
const fullPath = path.resolve(this.baseStorageDir, safeClientId, safeFilename);
|
||||
const resolvedBase = path.resolve(this.baseStorageDir);
|
||||
const fullPath = path.resolve(resolvedBase, safeClientId, safeFilename);
|
||||
if (!fullPath.startsWith(resolvedBase + path.sep)) {
|
||||
return null;
|
||||
}
|
||||
if (fs.existsSync(fullPath)) {
|
||||
const stats = fs.lstatSync(fullPath);
|
||||
if (stats.isSymbolicLink() || !stats.isFile()) {
|
||||
return null;
|
||||
}
|
||||
return fs.readFileSync(fullPath, 'utf8');
|
||||
}
|
||||
return null;
|
||||
|
||||
@@ -22,11 +22,23 @@ export interface AdminSettings {
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface SettingsBridgeSyncStatus {
|
||||
attempted: boolean;
|
||||
ok: boolean;
|
||||
synced: boolean;
|
||||
status?: number;
|
||||
error?: string;
|
||||
checkedAt: string;
|
||||
}
|
||||
|
||||
type SettingsGlobals = typeof globalThis & {
|
||||
__dataStackAdminSettings?: AdminSettings;
|
||||
__dataStackSettingsHydrated?: boolean;
|
||||
__dataStackLastSettingsBridgeSync?: SettingsBridgeSyncStatus;
|
||||
__dataStackLastPruneMeta?: { days: number; ranAtMs: number };
|
||||
};
|
||||
|
||||
const PRUNE_COOLDOWN_MS = 60 * 60 * 1000;
|
||||
const globals = globalThis as SettingsGlobals;
|
||||
|
||||
function getDefaultSettings(): AdminSettings {
|
||||
@@ -53,7 +65,7 @@ function getSettingsState(): AdminSettings {
|
||||
}
|
||||
|
||||
export class SettingsManager {
|
||||
private static applyRuntimeSettings(settings: AdminSettings) {
|
||||
private static applyRuntimeSettings(settings: AdminSettings, forcePrune = false) {
|
||||
SecurityEngine.maxEventsPerSec = settings.maxEventsPerSec;
|
||||
SecurityEngine.enableIdempotency = settings.enableIdempotency;
|
||||
SecurityEngine.maxPayloadSizeBytes = settings.maxPayloadSizeKb * 1024;
|
||||
@@ -68,7 +80,27 @@ export class SettingsManager {
|
||||
settings.autoPruneDays > 0 &&
|
||||
(!hasConfiguredDb || globals.__dataStackSettingsHydrated === true)
|
||||
) {
|
||||
CSVStorageManager.pruneExpiredPartitions(settings.autoPruneDays);
|
||||
const prev = globals.__dataStackLastPruneMeta;
|
||||
const now = Date.now();
|
||||
const shouldPrune =
|
||||
forcePrune ||
|
||||
!prev ||
|
||||
prev.days !== settings.autoPruneDays ||
|
||||
now - prev.ranAtMs >= PRUNE_COOLDOWN_MS;
|
||||
if (shouldPrune) {
|
||||
globals.__dataStackLastPruneMeta = {
|
||||
days: settings.autoPruneDays,
|
||||
ranAtMs: now,
|
||||
};
|
||||
void CSVStorageManager.pruneExpiredPartitionsAsync(
|
||||
settings.autoPruneDays,
|
||||
).catch((err) => {
|
||||
console.warn(
|
||||
"[DataStack Storage] Background async partition prune failed:",
|
||||
err,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -94,9 +126,17 @@ export class SettingsManager {
|
||||
row.auto_prune_days !== undefined && row.auto_prune_days !== null
|
||||
? Number(row.auto_prune_days)
|
||||
: NaN;
|
||||
const envSocketPort = Number(process.env.PORT_SOCKET);
|
||||
const envWebPort = Number(process.env.PORT);
|
||||
const loaded: AdminSettings = {
|
||||
serverPort: Number(row.server_port) || 4000,
|
||||
webPort: Number(row.web_port) || 3000,
|
||||
serverPort:
|
||||
Number.isFinite(envSocketPort) && envSocketPort > 0
|
||||
? envSocketPort
|
||||
: Number(row.server_port) || 4000,
|
||||
webPort:
|
||||
Number.isFinite(envWebPort) && envWebPort > 0
|
||||
? envWebPort
|
||||
: Number(row.web_port) || 3000,
|
||||
maxEventsPerSec: Number(row.max_events_per_sec) || 100,
|
||||
maxPayloadSizeKb: Number(row.max_payload_size_kb) || 100,
|
||||
batchFlushThreshold: Number(row.batch_flush_threshold) || 500,
|
||||
@@ -117,6 +157,12 @@ export class SettingsManager {
|
||||
|
||||
// Initialize default row in PostgreSQL if not present yet
|
||||
const defaults = getSettingsState();
|
||||
const hasValidProductionJwt =
|
||||
process.env.NODE_ENV !== "production" ||
|
||||
defaults.jwtSecret.trim().length >= 32;
|
||||
if (!hasValidProductionJwt) {
|
||||
return defaults;
|
||||
}
|
||||
await this.persistSettingsToDb(defaults);
|
||||
globals.__dataStackSettingsHydrated = true;
|
||||
return defaults;
|
||||
@@ -131,8 +177,23 @@ export class SettingsManager {
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (!pool) return;
|
||||
|
||||
if (
|
||||
process.env.NODE_ENV === "production" &&
|
||||
settings.jwtSecret.trim().length < 32
|
||||
) {
|
||||
throw new Error(
|
||||
"DATABASE_WRITE_FAILED: Cannot persist settings without a valid DATASTACK_JWT_SECRET (>= 32 characters) in production.",
|
||||
);
|
||||
}
|
||||
|
||||
const ready = await DatabaseEngine.ensureSchema();
|
||||
if (!ready) {
|
||||
throw new Error(
|
||||
"DATABASE_WRITE_FAILED: PostgreSQL schema is unavailable while saving settings.",
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
await pool.query(
|
||||
`INSERT INTO admin_settings (
|
||||
id, server_port, web_port, max_events_per_sec, max_payload_size_kb,
|
||||
@@ -164,8 +225,9 @@ export class SettingsManager {
|
||||
settings.autoPruneDays,
|
||||
],
|
||||
);
|
||||
} catch {
|
||||
// Ignore DB failure; in-memory state remains active
|
||||
} catch (err: unknown) {
|
||||
const detail = err instanceof Error ? err.message : "Unknown DB error";
|
||||
throw new Error(`DATABASE_WRITE_FAILED: Failed to persist settings (${detail})`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -222,20 +284,35 @@ export class SettingsManager {
|
||||
};
|
||||
}
|
||||
|
||||
public static getLastBridgeSyncStatus(): SettingsBridgeSyncStatus | null {
|
||||
return globals.__dataStackLastSettingsBridgeSync ?? null;
|
||||
}
|
||||
|
||||
private static async notifyIngestionSettingsSync(
|
||||
settings: AdminSettings,
|
||||
authJwtSecret: string,
|
||||
jwtSecretChanged: boolean,
|
||||
): Promise<void> {
|
||||
): Promise<SettingsBridgeSyncStatus> {
|
||||
const checkedAt = new Date().toISOString();
|
||||
const headers = getInternalIngestionHeaders(authJwtSecret);
|
||||
if (!headers["x-datastack-internal-secret"]) return;
|
||||
if (!headers["x-datastack-internal-secret"]) {
|
||||
const result: SettingsBridgeSyncStatus = {
|
||||
attempted: true,
|
||||
ok: false,
|
||||
synced: false,
|
||||
error: "Internal bridge secret is not configured",
|
||||
checkedAt,
|
||||
};
|
||||
globals.__dataStackLastSettingsBridgeSync = result;
|
||||
return result;
|
||||
}
|
||||
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||
).replace(/\/$/, "");
|
||||
|
||||
try {
|
||||
await fetch(`${ingestionBaseUrl}/internal/settings-sync`, {
|
||||
const res = await fetch(`${ingestionBaseUrl}/internal/settings-sync`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
@@ -254,8 +331,46 @@ export class SettingsManager {
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(1500),
|
||||
});
|
||||
} catch {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
if (!res.ok) {
|
||||
const errorMsg = `Port 4000 settings-sync failed with HTTP ${res.status}`;
|
||||
console.warn(`[DataStack Bridge] ${errorMsg}`);
|
||||
const result: SettingsBridgeSyncStatus = {
|
||||
attempted: true,
|
||||
ok: false,
|
||||
synced: false,
|
||||
status: res.status,
|
||||
error: errorMsg,
|
||||
checkedAt,
|
||||
};
|
||||
globals.__dataStackLastSettingsBridgeSync = result;
|
||||
return result;
|
||||
}
|
||||
const result: SettingsBridgeSyncStatus = {
|
||||
attempted: true,
|
||||
ok: true,
|
||||
synced: true,
|
||||
status: res.status,
|
||||
checkedAt,
|
||||
};
|
||||
globals.__dataStackLastSettingsBridgeSync = result;
|
||||
return result;
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : "Port 4000 settings-sync unreachable";
|
||||
if (
|
||||
process.env.NODE_ENV === "production" ||
|
||||
Boolean(process.env.INGESTION_INTERNAL_URL)
|
||||
) {
|
||||
console.warn(`[DataStack Bridge] ${message}`);
|
||||
}
|
||||
const result: SettingsBridgeSyncStatus = {
|
||||
attempted: true,
|
||||
ok: false,
|
||||
synced: false,
|
||||
error: message,
|
||||
checkedAt,
|
||||
};
|
||||
globals.__dataStackLastSettingsBridgeSync = result;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -270,11 +385,13 @@ export class SettingsManager {
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
globals.__dataStackAdminSettings = nextSettings;
|
||||
this.applyRuntimeSettings(nextSettings);
|
||||
this.applyRuntimeSettings(nextSettings, updates.autoPruneDays !== undefined);
|
||||
if (jwtSecretChanged) {
|
||||
SDKManager.resignAllClientTokens();
|
||||
}
|
||||
void this.persistSettingsToDb(nextSettings);
|
||||
void this.persistSettingsToDb(nextSettings).catch((err) => {
|
||||
console.error("[DataStack Settings] Background DB persist failed:", err);
|
||||
});
|
||||
void this.notifyIngestionSettingsSync(
|
||||
nextSettings,
|
||||
previousJwtSecret,
|
||||
@@ -297,20 +414,80 @@ export class SettingsManager {
|
||||
...updates,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
globals.__dataStackAdminSettings = nextSettings;
|
||||
this.applyRuntimeSettings(nextSettings);
|
||||
if (jwtSecretChanged) {
|
||||
await SDKManager.resignAllClientTokensAsync();
|
||||
|
||||
if (
|
||||
jwtSecretChanged &&
|
||||
process.env.NODE_ENV === "production" &&
|
||||
nextSettings.jwtSecret.trim().length < 32
|
||||
) {
|
||||
throw new Error(
|
||||
"INVALID_JWT_SECRET: JWT secret must be at least 32 characters in production.",
|
||||
);
|
||||
}
|
||||
|
||||
if (jwtSecretChanged) {
|
||||
// BUG-17: Coordinate Port 4000 BEFORE committing JWT rotation on Port 3000
|
||||
// so a bridge rejection or unreachable production ingestion server never causes secret split-brain.
|
||||
const preSyncStatus = await this.notifyIngestionSettingsSync(
|
||||
nextSettings,
|
||||
previousJwtSecret,
|
||||
true,
|
||||
);
|
||||
|
||||
if (!preSyncStatus.synced) {
|
||||
const isExplicitHttpRejection = typeof preSyncStatus.status === "number";
|
||||
const isProductionBridgeRequired =
|
||||
process.env.NODE_ENV === "production" &&
|
||||
Boolean(process.env.INGESTION_INTERNAL_URL);
|
||||
|
||||
if (isExplicitHttpRejection || isProductionBridgeRequired) {
|
||||
throw new Error(
|
||||
`JWT_ROTATION_BRIDGE_FAILED: Aborted JWT rotation because Port 4000 could not synchronize (${preSyncStatus.error || "bridge error"}).`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await this.persistSettingsToDb(nextSettings);
|
||||
globals.__dataStackAdminSettings = nextSettings;
|
||||
this.applyRuntimeSettings(nextSettings, updates.autoPruneDays !== undefined);
|
||||
await SDKManager.resignAllClientTokensAsync();
|
||||
} catch (err) {
|
||||
globals.__dataStackAdminSettings = current;
|
||||
this.applyRuntimeSettings(current, false);
|
||||
await this.persistSettingsToDb(current).catch(() => undefined);
|
||||
if (preSyncStatus.synced) {
|
||||
await this.notifyIngestionSettingsSync(
|
||||
current,
|
||||
nextSettings.jwtSecret,
|
||||
true,
|
||||
).catch(() => undefined);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
const clientSync = await SDKManager.syncAllClientsToIngestionAsync();
|
||||
if (!clientSync.synced && preSyncStatus.synced) {
|
||||
globals.__dataStackLastSettingsBridgeSync = {
|
||||
attempted: true,
|
||||
ok: false,
|
||||
synced: false,
|
||||
status: clientSync.status,
|
||||
error: clientSync.error,
|
||||
checkedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
return { ...nextSettings };
|
||||
}
|
||||
|
||||
await this.persistSettingsToDb(nextSettings);
|
||||
globals.__dataStackAdminSettings = nextSettings;
|
||||
this.applyRuntimeSettings(nextSettings, updates.autoPruneDays !== undefined);
|
||||
await this.notifyIngestionSettingsSync(
|
||||
nextSettings,
|
||||
previousJwtSecret,
|
||||
jwtSecretChanged,
|
||||
false,
|
||||
);
|
||||
if (jwtSecretChanged) {
|
||||
await SDKManager.syncAllClientsToIngestionAsync();
|
||||
}
|
||||
return { ...nextSettings };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,6 +34,11 @@ export class ApiEngine {
|
||||
).replace(/\/$/, '');
|
||||
const internalHeaders = getInternalIngestionHeaders();
|
||||
|
||||
let ingestionBridgeStatus: 'online' | 'auth_error' | 'error' | 'offline' = 'offline';
|
||||
let ingestionBridgeError: string | undefined;
|
||||
let liveServerPort = settings.serverPort || 4000;
|
||||
let liveWebPort = settings.webPort || 3000;
|
||||
|
||||
try {
|
||||
const res = await fetch(`${ingestionBaseUrl}/internal/metrics`, {
|
||||
headers: internalHeaders,
|
||||
@@ -42,8 +47,17 @@ export class ApiEngine {
|
||||
});
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
ingestionBridgeStatus = 'online';
|
||||
activeSockets = Number(data.activeSockets) || 0;
|
||||
ingestionRedisConnected = Boolean(data.redisConnected);
|
||||
if (Number.isFinite(Number(data.serverPort)) && Number(data.serverPort) > 0) {
|
||||
liveServerPort = Number(data.serverPort);
|
||||
} else if (Number.isFinite(Number(data.port)) && Number(data.port) > 0) {
|
||||
liveServerPort = Number(data.port);
|
||||
}
|
||||
if (Number.isFinite(Number(data.webPort)) && Number(data.webPort) > 0) {
|
||||
liveWebPort = Number(data.webPort);
|
||||
}
|
||||
if (data.localBufferSize !== undefined) {
|
||||
ingestionLocalBufferSize = Number(data.localBufferSize) || 0;
|
||||
ingestionRedisBufferSize = Number(data.redisBufferSize) || 0;
|
||||
@@ -53,14 +67,36 @@ export class ApiEngine {
|
||||
if (data.securityStats && typeof data.securityStats === 'object') {
|
||||
remoteSecurityStats = data.securityStats;
|
||||
}
|
||||
// BUG-02: Auto-reconcile Port 4000 cold start whenever metrics observe unsynced registry
|
||||
if (data.hasSyncedInMemoryClients === false && clients.length > 0) {
|
||||
await SDKManager.syncAllClientsToIngestionAsync();
|
||||
}
|
||||
} else {
|
||||
ingestionBridgeStatus = res.status === 401 || res.status === 503 ? 'auth_error' : 'error';
|
||||
ingestionBridgeError = `HTTP ${res.status}`;
|
||||
console.warn(`[DataStack Bridge] /internal/metrics failed with ${ingestionBridgeError}`);
|
||||
}
|
||||
} catch {
|
||||
} catch (err: unknown) {
|
||||
// Ingestion server may not be running in single-process dev mode
|
||||
activeSockets = 0;
|
||||
ingestionLocalBufferSize = 0;
|
||||
ingestionRedisBufferSize = 0;
|
||||
ingestionBridgeStatus = 'offline';
|
||||
ingestionBridgeError = err instanceof Error ? err.message : 'Unreachable';
|
||||
}
|
||||
|
||||
const effectiveRedisBufferSize = Math.max(
|
||||
localBufferStats.redisBufferSize || 0,
|
||||
ingestionRedisBufferSize,
|
||||
);
|
||||
const redisConnected = localBufferStats.redisConnected || ingestionRedisConnected;
|
||||
const securityMode: 'redis_shared' | 'bridge_coordinated' | 'degraded_local' =
|
||||
redisConnected
|
||||
? 'redis_shared'
|
||||
: ingestionBridgeStatus === 'online'
|
||||
? 'bridge_coordinated'
|
||||
: 'degraded_local';
|
||||
|
||||
const localSecurityStats = SecurityEngine.getStats();
|
||||
const securityStats = {
|
||||
activeIdempotencyKeys:
|
||||
@@ -74,15 +110,12 @@ export class ApiEngine {
|
||||
Number(remoteSecurityStats?.maxPayloadSizeBytes) ||
|
||||
localSecurityStats.maxPayloadSizeBytes,
|
||||
),
|
||||
mode: securityMode,
|
||||
};
|
||||
|
||||
const effectiveRedisBufferSize = Math.max(
|
||||
localBufferStats.redisBufferSize || 0,
|
||||
ingestionRedisBufferSize,
|
||||
);
|
||||
const bufferStats = {
|
||||
...localBufferStats,
|
||||
redisConnected: localBufferStats.redisConnected || ingestionRedisConnected,
|
||||
redisConnected,
|
||||
redisBufferSize: effectiveRedisBufferSize,
|
||||
currentBufferSize:
|
||||
localBufferStats.localBufferSize +
|
||||
@@ -98,8 +131,11 @@ export class ApiEngine {
|
||||
storageStats,
|
||||
securityStats,
|
||||
runtime: {
|
||||
serverPort: settings.serverPort || 4000,
|
||||
webPort: settings.webPort || 3000,
|
||||
serverPort: liveServerPort,
|
||||
webPort: liveWebPort,
|
||||
ingestionBridgeStatus,
|
||||
...(ingestionBridgeError ? { ingestionBridgeError } : {}),
|
||||
securityMode,
|
||||
uptimeSeconds: Math.floor(process.uptime()),
|
||||
memoryUsageMb: (process.memoryUsage().rss / (1024 * 1024)).toFixed(1),
|
||||
nodeVersion: process.version,
|
||||
@@ -107,23 +143,35 @@ export class ApiEngine {
|
||||
};
|
||||
}
|
||||
|
||||
private static filterFilesByClientId<T extends { clientId: string }>(
|
||||
files: T[],
|
||||
clientId?: string,
|
||||
): T[] {
|
||||
if (!clientId || clientId === 'all') return files;
|
||||
const normalized = SDKManager.normalizeClientId(clientId);
|
||||
if (!normalized) return files;
|
||||
const sanitized = CSVStorageManager.sanitizeClientId(normalized);
|
||||
return files.filter((f) => {
|
||||
const folderNorm = SDKManager.normalizeClientId(f.clientId);
|
||||
return (
|
||||
folderNorm === normalized ||
|
||||
f.clientId === normalized ||
|
||||
f.clientId === sanitized
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Filters and lists stored CSV files for export
|
||||
*/
|
||||
public static getExportFiles(clientId?: string) {
|
||||
const allFiles = CSVStorageManager.listAllFiles();
|
||||
if (clientId && clientId !== 'all') {
|
||||
return allFiles.filter((f) => f.clientId === clientId);
|
||||
}
|
||||
return allFiles;
|
||||
return this.filterFilesByClientId(allFiles, clientId);
|
||||
}
|
||||
|
||||
public static async getExportFilesAsync(clientId?: string) {
|
||||
const allFiles = await CSVStorageManager.listAllFilesAsync();
|
||||
if (clientId && clientId !== 'all') {
|
||||
return allFiles.filter((f) => f.clientId === clientId);
|
||||
}
|
||||
return allFiles;
|
||||
return this.filterFilesByClientId(allFiles, clientId);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -113,11 +113,8 @@ export function getSessionCookieOptions(request: Request, maxAge: number) {
|
||||
?.split(",")[0]
|
||||
?.trim()
|
||||
.toLowerCase();
|
||||
const originHeader = request.headers.get("origin")?.toLowerCase() ?? "";
|
||||
const isHttps =
|
||||
forwardedProto === "https" ||
|
||||
originHeader.startsWith("https://") ||
|
||||
request.url.startsWith("https://");
|
||||
forwardedProto === "https" || request.url.startsWith("https://");
|
||||
|
||||
if (isHttps) {
|
||||
return {
|
||||
|
||||
+62
-53
@@ -5,12 +5,13 @@ import {
|
||||
scrypt as scryptCallback,
|
||||
timingSafeEqual,
|
||||
} from "node:crypto";
|
||||
import { Pool, type PoolClient } from "pg";
|
||||
import type { Pool, PoolClient } from "pg";
|
||||
import {
|
||||
type AdminSessionPayload,
|
||||
getAdminSessionPayload,
|
||||
getAdminSessionSecret,
|
||||
} from "./adminSession";
|
||||
import { DatabaseEngine, ensureCoreSchemaOnPool } from "../Database/database";
|
||||
|
||||
const PASSWORD_HASH_BYTES = 64;
|
||||
const PASSWORD_SALT_BYTES = 16;
|
||||
@@ -50,7 +51,6 @@ interface InMemoryAdmin {
|
||||
}
|
||||
|
||||
type AdminDatabaseGlobals = typeof globalThis & {
|
||||
__dataStackAdminPool?: Pool | null;
|
||||
__dataStackAdminSchema?: Promise<void>;
|
||||
__dataStackInMemoryAdmin?: InMemoryAdmin;
|
||||
};
|
||||
@@ -60,67 +60,51 @@ const localLoginThrottle = new Map<string, LoginThrottleEntry>();
|
||||
let lastLocalThrottleCleanup = 0;
|
||||
let lastDatabaseThrottleCleanup = 0;
|
||||
|
||||
export class AdminSetupRequiredError extends Error {
|
||||
public readonly code = "SETUP_REQUIRED";
|
||||
|
||||
constructor(message = "Admin account has not completed initial setup.") {
|
||||
super(message);
|
||||
this.name = "AdminSetupRequiredError";
|
||||
}
|
||||
}
|
||||
|
||||
export class AdminDatabaseRequiredError extends Error {
|
||||
public readonly code = "DATABASE_NOT_CONFIGURED";
|
||||
|
||||
constructor(
|
||||
message = "PostgreSQL DATABASE_URL is required for initial admin setup in production.",
|
||||
) {
|
||||
super(message);
|
||||
this.name = "AdminDatabaseRequiredError";
|
||||
}
|
||||
}
|
||||
|
||||
function isProductionEnv() {
|
||||
return process.env.NODE_ENV === "production";
|
||||
}
|
||||
|
||||
export function isAdminDatabaseConfigured(): boolean {
|
||||
return Boolean(getPool());
|
||||
}
|
||||
|
||||
export function requiresPersistentAdminDatabase(): boolean {
|
||||
return (
|
||||
isProductionEnv() &&
|
||||
!getPool() &&
|
||||
!process.env.DATASTACK_ADMIN_PASSWORD
|
||||
);
|
||||
}
|
||||
|
||||
function getPool(): Pool | null {
|
||||
const connectionString = process.env.DATABASE_URL?.trim();
|
||||
if (!connectionString) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (globals.__dataStackAdminPool === undefined) {
|
||||
try {
|
||||
globals.__dataStackAdminPool = new Pool({
|
||||
connectionString,
|
||||
connectionTimeoutMillis: 3000,
|
||||
idleTimeoutMillis: 30000,
|
||||
max: 5,
|
||||
});
|
||||
} catch {
|
||||
globals.__dataStackAdminPool = null;
|
||||
}
|
||||
}
|
||||
|
||||
return globals.__dataStackAdminPool ?? null;
|
||||
return DatabaseEngine.getPool();
|
||||
}
|
||||
|
||||
async function ensureAdminTable() {
|
||||
const pool = getPool();
|
||||
if (!pool) return;
|
||||
|
||||
globals.__dataStackAdminSchema ??= Promise.all([
|
||||
pool
|
||||
.query(
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS admin_users (
|
||||
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
|
||||
email TEXT NOT NULL UNIQUE,
|
||||
password_salt BYTEA NOT NULL,
|
||||
password_hash BYTEA NOT NULL,
|
||||
token_version INTEGER NOT NULL DEFAULT 1,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`,
|
||||
)
|
||||
.then(() =>
|
||||
pool.query(
|
||||
`ALTER TABLE admin_users ADD COLUMN IF NOT EXISTS token_version INTEGER NOT NULL DEFAULT 1`,
|
||||
),
|
||||
),
|
||||
pool.query(
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS admin_login_attempts (
|
||||
key_hash TEXT PRIMARY KEY,
|
||||
window_started_at TIMESTAMPTZ NOT NULL,
|
||||
failed_attempts INTEGER NOT NULL,
|
||||
blocked_until TIMESTAMPTZ,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`,
|
||||
),
|
||||
]).then(() => undefined);
|
||||
globals.__dataStackAdminSchema ??= ensureCoreSchemaOnPool(pool);
|
||||
|
||||
try {
|
||||
await globals.__dataStackAdminSchema;
|
||||
@@ -342,7 +326,7 @@ async function getOrInitInMemoryAdmin(): Promise<InMemoryAdmin> {
|
||||
!isProductionEnv() && process.env.DATASTACK_ALLOW_DEMO_ADMIN !== "false";
|
||||
|
||||
if (!envPassword && !allowDemo) {
|
||||
throw new Error("Admin account has not completed initial setup.");
|
||||
throw new AdminSetupRequiredError();
|
||||
}
|
||||
|
||||
const defaultEmail = (
|
||||
@@ -443,6 +427,9 @@ export async function createInitialAdmin(email: string, password: string) {
|
||||
const pool = getPool();
|
||||
|
||||
if (!pool) {
|
||||
if (isProductionEnv()) {
|
||||
throw new AdminDatabaseRequiredError();
|
||||
}
|
||||
if (globals.__dataStackInMemoryAdmin) {
|
||||
return false;
|
||||
}
|
||||
@@ -671,6 +658,28 @@ export async function updateAdminAccount({
|
||||
return { ok: true as const, email: cleanEmail, tokenVersion: nextVersion };
|
||||
}
|
||||
|
||||
export async function revokeAdminSessions(): Promise<number> {
|
||||
const pool = getPool();
|
||||
if (pool) {
|
||||
await ensureAdminTable();
|
||||
const updated = await pool.query(
|
||||
`UPDATE admin_users
|
||||
SET token_version = COALESCE(token_version, 1) + 1
|
||||
WHERE id = 1
|
||||
RETURNING token_version`,
|
||||
);
|
||||
if (!updated.rowCount || updated.rowCount === 0) {
|
||||
return 0;
|
||||
}
|
||||
return Number(updated.rows[0].token_version) || 1;
|
||||
}
|
||||
|
||||
const existing = globals.__dataStackInMemoryAdmin ?? (await getOrInitInMemoryAdmin());
|
||||
existing.tokenVersion += 1;
|
||||
globals.__dataStackInMemoryAdmin = existing;
|
||||
return existing.tokenVersion;
|
||||
}
|
||||
|
||||
export function resetAdminStateForTests() {
|
||||
globals.__dataStackInMemoryAdmin = undefined;
|
||||
localLoginThrottle.clear();
|
||||
|
||||
+517
-83
@@ -19,6 +19,15 @@ export interface RedisBufferClient {
|
||||
ttlMs: number,
|
||||
nxMode: 'NX',
|
||||
): Promise<string | null>;
|
||||
get?(key: string): Promise<string | null>;
|
||||
incr?(key: string): Promise<number>;
|
||||
pexpire?(key: string, ttlMs: number): Promise<number>;
|
||||
hget?(key: string, field: string): Promise<string | null>;
|
||||
hset?(key: string, field: string, value: string): Promise<number>;
|
||||
hdel?(key: string, ...fields: string[]): Promise<number>;
|
||||
sismember?(key: string, member: string): Promise<number>;
|
||||
sadd?(key: string, ...members: string[]): Promise<number>;
|
||||
srem?(key: string, ...members: string[]): Promise<number>;
|
||||
eval(script: string, numKeys: number, ...args: string[]): Promise<unknown>;
|
||||
quit?: () => Promise<unknown>;
|
||||
disconnect?: () => void;
|
||||
@@ -27,10 +36,83 @@ export interface RedisBufferClient {
|
||||
export const REDIS_BUFFER_KEY = 'datastack:buffer:events';
|
||||
export const REDIS_RECENT_KEY = 'datastack:recent:events';
|
||||
export const REDIS_FLUSH_LOCK_KEY = 'datastack:flush:lock';
|
||||
const REDIS_RECENT_LIMIT = 500;
|
||||
const FLUSH_LOCK_TTL_MS = 15000;
|
||||
const REDIS_CLIENTS_DELETED_KEY = 'datastack:clients:deleted';
|
||||
export const DEFAULT_REDIS_RECENT_LIMIT = 500;
|
||||
export const MAX_REDIS_RECENT_LIMIT = 5000;
|
||||
const FLUSH_LOCK_TTL_MS = 30000;
|
||||
const FLUSH_LOCK_WAIT_MS = 400;
|
||||
const FLUSH_LOCK_RETRY_MS = 25;
|
||||
const UNLOCK_LUA_SCRIPT =
|
||||
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end';
|
||||
const RENEW_LOCK_LUA_SCRIPT =
|
||||
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("pexpire", KEYS[1], ARGV[2]) else return 0 end';
|
||||
export const PURGE_CLIENT_LIST_LUA_SCRIPT = `
|
||||
-- DATASTACK_PURGE_CLIENT_LIST
|
||||
local key = KEYS[1]
|
||||
local target = ARGV[1]
|
||||
local items = redis.call("LRANGE", key, 0, -1)
|
||||
local removed = 0
|
||||
local kept = {}
|
||||
for i = 1, #items do
|
||||
local raw = items[i]
|
||||
local ok, obj = pcall(cjson.decode, raw)
|
||||
local cid = ""
|
||||
if ok and type(obj) == "table" and obj.clientId ~= nil then
|
||||
cid = string.lower(tostring(obj.clientId))
|
||||
cid = string.gsub(cid, "^%s*(.-)%s*$", "%1")
|
||||
cid = string.gsub(cid, "[^a-z0-9_-]", "_")
|
||||
end
|
||||
if cid == target then
|
||||
removed = removed + 1
|
||||
else
|
||||
kept[#kept + 1] = raw
|
||||
end
|
||||
end
|
||||
if removed > 0 then
|
||||
redis.call("DEL", key)
|
||||
local chunk = 500
|
||||
for i = 1, #kept, chunk do
|
||||
local stop = math.min(i + chunk - 1, #kept)
|
||||
redis.call("RPUSH", key, unpack(kept, i, stop))
|
||||
end
|
||||
end
|
||||
return removed
|
||||
`.trim();
|
||||
|
||||
export function resolveRedisRecentLimit(batchFlushThreshold?: number): number {
|
||||
const envRaw = Number(process.env.DATASTACK_RECENT_LIMIT);
|
||||
if (Number.isFinite(envRaw) && envRaw >= 100) {
|
||||
return Math.min(MAX_REDIS_RECENT_LIMIT, Math.floor(envRaw));
|
||||
}
|
||||
const threshold = Number(batchFlushThreshold) || DEFAULT_REDIS_RECENT_LIMIT;
|
||||
return Math.min(
|
||||
MAX_REDIS_RECENT_LIMIT,
|
||||
Math.max(DEFAULT_REDIS_RECENT_LIMIT, Math.floor(threshold)),
|
||||
);
|
||||
}
|
||||
|
||||
export interface FlushOptions {
|
||||
waitMs?: number;
|
||||
lockTtlMs?: number;
|
||||
throwOnLockTimeout?: boolean;
|
||||
}
|
||||
|
||||
export interface FlushResult {
|
||||
count: number;
|
||||
files: string[];
|
||||
lockTimedOut?: boolean;
|
||||
}
|
||||
|
||||
function sleepMs(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function normalizeBufferClientId(clientId: string): string {
|
||||
return String(clientId || '')
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9_-]/g, '_');
|
||||
}
|
||||
|
||||
interface BufferGlobalState {
|
||||
buffer: DataEvent[];
|
||||
@@ -43,11 +125,14 @@ interface BufferGlobalState {
|
||||
totalIngested: number;
|
||||
totalFlushed: number;
|
||||
isFlushing: boolean;
|
||||
activeFlushPromise: Promise<FlushResult> | null;
|
||||
redisClient: RedisBufferClient | null | undefined;
|
||||
redisUrlConfigured: string;
|
||||
redisConnected: boolean;
|
||||
redisTestOverride: RedisBufferClient | null | undefined;
|
||||
pendingRedisWrites: Set<Promise<void>>;
|
||||
deletedClientIds: Set<string>;
|
||||
shutdownHooks: Set<() => Promise<unknown>>;
|
||||
}
|
||||
|
||||
type BufferGlobals = typeof globalThis & {
|
||||
@@ -71,13 +156,22 @@ function getState(): BufferGlobalState {
|
||||
totalIngested: 0,
|
||||
totalFlushed: 0,
|
||||
isFlushing: false,
|
||||
activeFlushPromise: null,
|
||||
redisClient: undefined,
|
||||
redisUrlConfigured: '',
|
||||
redisConnected: false,
|
||||
redisTestOverride: undefined,
|
||||
pendingRedisWrites: new Set(),
|
||||
deletedClientIds: new Set(),
|
||||
shutdownHooks: new Set(),
|
||||
};
|
||||
}
|
||||
if (!globals.__dataStackBufferState.deletedClientIds) {
|
||||
globals.__dataStackBufferState.deletedClientIds = new Set();
|
||||
}
|
||||
if (!globals.__dataStackBufferState.shutdownHooks) {
|
||||
globals.__dataStackBufferState.shutdownHooks = new Set();
|
||||
}
|
||||
return globals.__dataStackBufferState;
|
||||
}
|
||||
|
||||
@@ -150,7 +244,9 @@ function getRedisClient(): RedisBufferClient | null {
|
||||
}
|
||||
|
||||
export class DataBufferEngine {
|
||||
private static maxRecentEvents = REDIS_RECENT_LIMIT;
|
||||
public static getRecentLimit(): number {
|
||||
return resolveRedisRecentLimit(getState().batchFlushThreshold);
|
||||
}
|
||||
|
||||
public static get batchFlushThreshold(): number {
|
||||
return getState().batchFlushThreshold;
|
||||
@@ -174,6 +270,10 @@ export class DataBufferEngine {
|
||||
state.redisConnected = Boolean(client);
|
||||
}
|
||||
|
||||
public static getRedisClient(): RedisBufferClient | null {
|
||||
return getRedisClient();
|
||||
}
|
||||
|
||||
public static async shutdown(): Promise<void> {
|
||||
const state = getState();
|
||||
if (state.flushTimer) {
|
||||
@@ -188,6 +288,7 @@ export class DataBufferEngine {
|
||||
state.redisUrlConfigured = '';
|
||||
state.redisConnected = false;
|
||||
state.redisTestOverride = undefined;
|
||||
state.deletedClientIds.clear();
|
||||
if (client) {
|
||||
try {
|
||||
if (typeof client.quit === 'function') {
|
||||
@@ -203,6 +304,14 @@ export class DataBufferEngine {
|
||||
}
|
||||
}
|
||||
|
||||
public static registerShutdownHook(hook: () => Promise<unknown>): () => void {
|
||||
const state = getState();
|
||||
state.shutdownHooks.add(hook);
|
||||
return () => {
|
||||
state.shutdownHooks.delete(hook);
|
||||
};
|
||||
}
|
||||
|
||||
public static async flushAndShutdownOnSignal(): Promise<void> {
|
||||
if (globals.__dataStackBufferShuttingDown) {
|
||||
return globals.__dataStackBufferShuttingDown;
|
||||
@@ -210,8 +319,16 @@ export class DataBufferEngine {
|
||||
|
||||
const run = (async () => {
|
||||
try {
|
||||
const hooks = [...getState().shutdownHooks];
|
||||
await Promise.race([
|
||||
this.flush().catch(() => undefined),
|
||||
Promise.allSettled([
|
||||
this.flush().catch(() => undefined),
|
||||
...hooks.map((hook) =>
|
||||
Promise.resolve()
|
||||
.then(() => hook())
|
||||
.catch(() => undefined),
|
||||
),
|
||||
]),
|
||||
new Promise<void>((resolve) => setTimeout(resolve, 4000)),
|
||||
]);
|
||||
} finally {
|
||||
@@ -278,7 +395,8 @@ export class DataBufferEngine {
|
||||
const state = getState();
|
||||
state.totalIngested += 1;
|
||||
state.recentEvents.unshift(event);
|
||||
if (state.recentEvents.length > this.maxRecentEvents) {
|
||||
const recentLimit = this.getRecentLimit();
|
||||
while (state.recentEvents.length > recentLimit) {
|
||||
state.recentEvents.pop();
|
||||
}
|
||||
|
||||
@@ -298,15 +416,25 @@ export class DataBufferEngine {
|
||||
|
||||
if (redis) {
|
||||
const serialized = events.map((e) => JSON.stringify(e));
|
||||
let bufferLen: number | null = null;
|
||||
try {
|
||||
const bufferLen = await redis.rpush(REDIS_BUFFER_KEY, ...serialized);
|
||||
await redis.lpush(REDIS_RECENT_KEY, ...serialized);
|
||||
await redis.ltrim(REDIS_RECENT_KEY, 0, REDIS_RECENT_LIMIT - 1);
|
||||
bufferLen = await redis.rpush(REDIS_BUFFER_KEY, ...serialized);
|
||||
state.redisConnected = true;
|
||||
return Number(bufferLen) + state.buffer.length;
|
||||
} catch {
|
||||
state.redisConnected = false;
|
||||
}
|
||||
|
||||
if (bufferLen !== null) {
|
||||
// Best-effort recent list mirror: never duplicate into RAM buffer once rpush succeeded
|
||||
try {
|
||||
const recentLimit = this.getRecentLimit();
|
||||
await redis.lpush(REDIS_RECENT_KEY, ...serialized);
|
||||
await redis.ltrim(REDIS_RECENT_KEY, 0, recentLimit - 1);
|
||||
} catch {
|
||||
// Non-fatal: events are already committed in REDIS_BUFFER_KEY and state.recentEvents
|
||||
}
|
||||
return Number(bufferLen) + state.buffer.length;
|
||||
}
|
||||
}
|
||||
|
||||
state.buffer.push(...events);
|
||||
@@ -363,117 +491,410 @@ export class DataBufferEngine {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Flushes queued events (shared Redis list under distributed lock + local RAM fallback) to partitioned CSV files.
|
||||
* Redis LTRIM runs ONLY after CSVStorageManager.appendBatch succeeds so crashes never lose data.
|
||||
*/
|
||||
public static async flush(): Promise<{ count: number; files: string[] }> {
|
||||
private static async isClientDeletedForBuffer(
|
||||
clientId: string,
|
||||
redis: RedisBufferClient | null,
|
||||
cache?: Map<string, boolean>,
|
||||
): Promise<boolean> {
|
||||
const normalized = normalizeBufferClientId(clientId);
|
||||
if (!normalized) return false;
|
||||
const state = getState();
|
||||
if (state.deletedClientIds.has(normalized)) return true;
|
||||
if (cache && cache.has(normalized)) return Boolean(cache.get(normalized));
|
||||
if (redis && typeof redis.sismember === 'function') {
|
||||
try {
|
||||
const isDeleted =
|
||||
Number(await redis.sismember(REDIS_CLIENTS_DELETED_KEY, normalized)) === 1;
|
||||
if (cache) cache.set(normalized, isDeleted);
|
||||
if (isDeleted) state.deletedClientIds.add(normalized);
|
||||
return isDeleted;
|
||||
} catch {
|
||||
// Ignore transient Redis tombstone lookup error
|
||||
}
|
||||
}
|
||||
if (cache) cache.set(normalized, false);
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Marks a client deleted in the buffer layer and immediately strips queued local events.
|
||||
*/
|
||||
public static markClientDeleted(clientId: string): void {
|
||||
const normalized = normalizeBufferClientId(clientId);
|
||||
if (!normalized) return;
|
||||
const state = getState();
|
||||
state.deletedClientIds.add(normalized);
|
||||
state.buffer = state.buffer.filter(
|
||||
(evt) => normalizeBufferClientId(evt.clientId) !== normalized,
|
||||
);
|
||||
state.recentEvents = state.recentEvents.filter(
|
||||
(evt) => normalizeBufferClientId(evt.clientId) !== normalized,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears deleted tombstone when a client ID is explicitly re-registered.
|
||||
*/
|
||||
public static unmarkClientDeleted(clientId: string): void {
|
||||
const normalized = normalizeBufferClientId(clientId);
|
||||
if (!normalized) return;
|
||||
getState().deletedClientIds.delete(normalized);
|
||||
}
|
||||
|
||||
private static async filterRedisListExcludingClient(
|
||||
redis: RedisBufferClient,
|
||||
listKey: string,
|
||||
normalizedClientId: string,
|
||||
): Promise<number> {
|
||||
const removed = await redis.eval(
|
||||
PURGE_CLIENT_LIST_LUA_SCRIPT,
|
||||
1,
|
||||
listKey,
|
||||
normalizedClientId,
|
||||
);
|
||||
return Number(removed) || 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Purges all queued events for a deleted client from both local RAM and shared Redis queues (BUG-19).
|
||||
*/
|
||||
public static async purgeClientEventsAsync(
|
||||
clientId: string,
|
||||
): Promise<{ removedLocal: number; removedRedis: number }> {
|
||||
const normalized = normalizeBufferClientId(clientId);
|
||||
if (!normalized) return { removedLocal: 0, removedRedis: 0 };
|
||||
|
||||
const state = getState();
|
||||
state.deletedClientIds.add(normalized);
|
||||
|
||||
if (state.pendingRedisWrites.size > 0) {
|
||||
await Promise.allSettled([...state.pendingRedisWrites]);
|
||||
}
|
||||
if (state.isFlushing) {
|
||||
return { count: 0, files: [] };
|
||||
}
|
||||
|
||||
state.isFlushing = true;
|
||||
let totalWritten = 0;
|
||||
const filesSet = new Set<string>();
|
||||
|
||||
try {
|
||||
const redis = getRedisClient();
|
||||
if (redis) {
|
||||
const lockToken = crypto.randomUUID();
|
||||
let lockAcquired = false;
|
||||
const beforeLocal = state.buffer.length;
|
||||
state.buffer = state.buffer.filter(
|
||||
(evt) => normalizeBufferClientId(evt.clientId) !== normalized,
|
||||
);
|
||||
const removedLocal = beforeLocal - state.buffer.length;
|
||||
state.recentEvents = state.recentEvents.filter(
|
||||
(evt) => normalizeBufferClientId(evt.clientId) !== normalized,
|
||||
);
|
||||
|
||||
let removedRedis = 0;
|
||||
const redis = getRedisClient();
|
||||
if (redis) {
|
||||
const lockToken = crypto.randomUUID();
|
||||
let lockAcquired = false;
|
||||
const deadline = Date.now() + 1500;
|
||||
while (true) {
|
||||
try {
|
||||
const lockResult = await redis.set(
|
||||
const lockRes = await redis.set(
|
||||
REDIS_FLUSH_LOCK_KEY,
|
||||
lockToken,
|
||||
'PX',
|
||||
FLUSH_LOCK_TTL_MS,
|
||||
'NX',
|
||||
);
|
||||
lockAcquired = lockResult === 'OK';
|
||||
state.redisConnected = true;
|
||||
if (lockRes === 'OK') {
|
||||
lockAcquired = true;
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
state.redisConnected = false;
|
||||
lockAcquired = false;
|
||||
break;
|
||||
}
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining <= 0) break;
|
||||
await sleepMs(Math.min(FLUSH_LOCK_RETRY_MS, remaining));
|
||||
}
|
||||
|
||||
if (lockAcquired) {
|
||||
try {
|
||||
removedRedis += await this.filterRedisListExcludingClient(
|
||||
redis,
|
||||
REDIS_BUFFER_KEY,
|
||||
normalized,
|
||||
);
|
||||
await this.filterRedisListExcludingClient(
|
||||
redis,
|
||||
REDIS_RECENT_KEY,
|
||||
normalized,
|
||||
).catch(() => 0);
|
||||
} catch {
|
||||
// Flush-time deletedClientIds/tombstone filter remains active as secondary guard
|
||||
} finally {
|
||||
try {
|
||||
await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
|
||||
} catch {
|
||||
// Lock expires via PX TTL
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { removedLocal, removedRedis };
|
||||
}
|
||||
|
||||
/**
|
||||
* Flushes queued events (shared Redis list under distributed lock + local RAM fallback) to partitioned CSV files.
|
||||
* Redis LTRIM runs ONLY after CSVStorageManager.appendBatch succeeds and lock ownership is verified.
|
||||
*/
|
||||
public static async flush(options?: FlushOptions): Promise<FlushResult> {
|
||||
const state = getState();
|
||||
if (state.pendingRedisWrites.size > 0) {
|
||||
await Promise.allSettled([...state.pendingRedisWrites]);
|
||||
}
|
||||
if (state.activeFlushPromise) {
|
||||
return state.activeFlushPromise;
|
||||
}
|
||||
|
||||
const executeFlush = async (): Promise<FlushResult> => {
|
||||
state.isFlushing = true;
|
||||
let totalWritten = 0;
|
||||
let lockTimedOut = false;
|
||||
const filesSet = new Set<string>();
|
||||
const deletedCheckCache = new Map<string, boolean>();
|
||||
|
||||
try {
|
||||
if (state.pendingRedisWrites.size > 0) {
|
||||
await Promise.allSettled([...state.pendingRedisWrites]);
|
||||
}
|
||||
|
||||
if (lockAcquired) {
|
||||
try {
|
||||
const chunkSize = Math.max(500, state.batchFlushThreshold);
|
||||
while (true) {
|
||||
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, chunkSize - 1);
|
||||
if (!rawItems || rawItems.length === 0) {
|
||||
const redis = getRedisClient();
|
||||
if (redis) {
|
||||
const lockToken = crypto.randomUUID();
|
||||
const lockTtlMs = Math.max(50, Number(options?.lockTtlMs) || FLUSH_LOCK_TTL_MS);
|
||||
const waitMs = Math.max(
|
||||
0,
|
||||
options?.waitMs !== undefined ? Number(options.waitMs) : FLUSH_LOCK_WAIT_MS,
|
||||
);
|
||||
const deadline = Date.now() + waitMs;
|
||||
let lockAcquired = false;
|
||||
let redisError = false;
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
const lockResult = await redis.set(
|
||||
REDIS_FLUSH_LOCK_KEY,
|
||||
lockToken,
|
||||
'PX',
|
||||
lockTtlMs,
|
||||
'NX',
|
||||
);
|
||||
state.redisConnected = true;
|
||||
if (lockResult === 'OK') {
|
||||
lockAcquired = true;
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
state.redisConnected = false;
|
||||
redisError = true;
|
||||
break;
|
||||
}
|
||||
|
||||
const parsedBatch: DataEvent[] = [];
|
||||
for (const raw of rawItems) {
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining <= 0) {
|
||||
break;
|
||||
}
|
||||
await sleepMs(Math.min(FLUSH_LOCK_RETRY_MS, remaining));
|
||||
}
|
||||
|
||||
if (!lockAcquired && !redisError) {
|
||||
lockTimedOut = true;
|
||||
if (options?.throwOnLockTimeout === true) {
|
||||
throw new Error(
|
||||
'FLUSH_LOCK_TIMEOUT: Timed out waiting for distributed Redis flush lock',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (lockAcquired) {
|
||||
const renewIntervalMs = Math.max(20, Math.floor(lockTtlMs / 3));
|
||||
let lastRenewAt = Date.now();
|
||||
let lockLost = false;
|
||||
let renewInFlight: Promise<boolean> | null = null;
|
||||
|
||||
const renewLockOnce = (): Promise<boolean> => {
|
||||
if (renewInFlight) return renewInFlight;
|
||||
renewInFlight = (async () => {
|
||||
try {
|
||||
const evt = JSON.parse(raw) as DataEvent;
|
||||
if (evt && evt.eventId) {
|
||||
parsedBatch.push(evt);
|
||||
const renewed = await redis.eval(
|
||||
RENEW_LOCK_LUA_SCRIPT,
|
||||
1,
|
||||
REDIS_FLUSH_LOCK_KEY,
|
||||
lockToken,
|
||||
String(lockTtlMs),
|
||||
);
|
||||
if (Number(renewed) === 1) {
|
||||
lastRenewAt = Date.now();
|
||||
return true;
|
||||
}
|
||||
lockLost = true;
|
||||
return false;
|
||||
} catch {
|
||||
// Ignore malformed legacy entry while still trimming it
|
||||
return false;
|
||||
} finally {
|
||||
renewInFlight = null;
|
||||
}
|
||||
})();
|
||||
return renewInFlight;
|
||||
};
|
||||
|
||||
const renewTimer = setInterval(() => {
|
||||
void renewLockOnce();
|
||||
}, renewIntervalMs);
|
||||
if (typeof renewTimer.unref === 'function') {
|
||||
renewTimer.unref();
|
||||
}
|
||||
|
||||
try {
|
||||
const chunkSize = Math.max(500, state.batchFlushThreshold);
|
||||
while (true) {
|
||||
if (lockLost) break;
|
||||
|
||||
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, chunkSize - 1);
|
||||
if (!rawItems || rawItems.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
const parsedBatch: DataEvent[] = [];
|
||||
let malformedCount = 0;
|
||||
for (const raw of rawItems) {
|
||||
try {
|
||||
const evt = JSON.parse(raw) as DataEvent;
|
||||
if (evt && evt.eventId) {
|
||||
if (
|
||||
await this.isClientDeletedForBuffer(
|
||||
evt.clientId,
|
||||
redis,
|
||||
deletedCheckCache,
|
||||
)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
parsedBatch.push(evt);
|
||||
} else {
|
||||
malformedCount += 1;
|
||||
}
|
||||
} catch {
|
||||
malformedCount += 1;
|
||||
}
|
||||
}
|
||||
if (malformedCount > 0) {
|
||||
console.warn(
|
||||
`[DataBufferEngine] Skipped ${malformedCount} malformed buffered Redis event(s) during flush`,
|
||||
);
|
||||
}
|
||||
|
||||
if (Date.now() - lastRenewAt >= renewIntervalMs) {
|
||||
await renewLockOnce();
|
||||
}
|
||||
if (lockLost) break;
|
||||
|
||||
if (parsedBatch.length > 0) {
|
||||
const res = await CSVStorageManager.appendBatch(parsedBatch);
|
||||
totalWritten += res.written;
|
||||
for (const f of res.filesUpdated) filesSet.add(f);
|
||||
}
|
||||
|
||||
if (Date.now() - lastRenewAt >= renewIntervalMs) {
|
||||
await renewLockOnce();
|
||||
}
|
||||
if (lockLost) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Trim strictly AFTER CSVStorageManager.appendBatch resolves and lock ownership holds
|
||||
await redis.ltrim(REDIS_BUFFER_KEY, rawItems.length, -1);
|
||||
|
||||
if (rawItems.length < chunkSize) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (parsedBatch.length > 0) {
|
||||
const res = await CSVStorageManager.appendBatch(parsedBatch);
|
||||
totalWritten += res.written;
|
||||
for (const f of res.filesUpdated) filesSet.add(f);
|
||||
} finally {
|
||||
clearInterval(renewTimer);
|
||||
if (renewInFlight) {
|
||||
await renewInFlight.catch(() => false);
|
||||
}
|
||||
|
||||
// Trim strictly AFTER CSVStorageManager.appendBatch resolves safely
|
||||
await redis.ltrim(REDIS_BUFFER_KEY, rawItems.length, -1);
|
||||
|
||||
if (rawItems.length < chunkSize) {
|
||||
break;
|
||||
try {
|
||||
await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
|
||||
} catch {
|
||||
// Lock auto-expires after lockTtlMs
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
}
|
||||
}
|
||||
|
||||
if (state.buffer.length > 0) {
|
||||
const ramBatch = [...state.buffer];
|
||||
state.buffer = [];
|
||||
const validRamBatch: DataEvent[] = [];
|
||||
for (const evt of ramBatch) {
|
||||
if (
|
||||
await this.isClientDeletedForBuffer(
|
||||
evt.clientId,
|
||||
redis,
|
||||
deletedCheckCache,
|
||||
)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
validRamBatch.push(evt);
|
||||
}
|
||||
|
||||
if (validRamBatch.length > 0) {
|
||||
try {
|
||||
await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
|
||||
} catch {
|
||||
// Lock auto-expires after FLUSH_LOCK_TTL_MS
|
||||
const res = await CSVStorageManager.appendBatch(validRamBatch);
|
||||
totalWritten += res.written;
|
||||
for (const f of res.filesUpdated) filesSet.add(f);
|
||||
} catch (err) {
|
||||
state.buffer.unshift(...validRamBatch);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (state.buffer.length > 0) {
|
||||
const ramBatch = [...state.buffer];
|
||||
state.buffer = [];
|
||||
try {
|
||||
const res = await CSVStorageManager.appendBatch(ramBatch);
|
||||
totalWritten += res.written;
|
||||
for (const f of res.filesUpdated) filesSet.add(f);
|
||||
} catch (err) {
|
||||
state.buffer.unshift(...ramBatch);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
if (totalWritten > 0) {
|
||||
state.totalFlushed += totalWritten;
|
||||
const filesUpdated = Array.from(filesSet);
|
||||
for (const listener of state.flushListeners) {
|
||||
try {
|
||||
listener(totalWritten, filesUpdated);
|
||||
} catch {
|
||||
// ignore
|
||||
if (totalWritten > 0) {
|
||||
state.totalFlushed += totalWritten;
|
||||
const filesUpdated = Array.from(filesSet);
|
||||
for (const listener of state.flushListeners) {
|
||||
try {
|
||||
listener(totalWritten, filesUpdated);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { count: totalWritten, files: Array.from(filesSet) };
|
||||
} finally {
|
||||
state.isFlushing = false;
|
||||
return {
|
||||
count: totalWritten,
|
||||
files: Array.from(filesSet),
|
||||
...(lockTimedOut ? { lockTimedOut: true } : {}),
|
||||
};
|
||||
} finally {
|
||||
state.isFlushing = false;
|
||||
state.activeFlushPromise = null;
|
||||
}
|
||||
};
|
||||
|
||||
let flushDone = false;
|
||||
let promise: Promise<FlushResult> | null = null;
|
||||
promise = (async () => {
|
||||
try {
|
||||
return await executeFlush();
|
||||
} finally {
|
||||
flushDone = true;
|
||||
state.isFlushing = false;
|
||||
if (state.activeFlushPromise === promise) {
|
||||
state.activeFlushPromise = null;
|
||||
}
|
||||
}
|
||||
})();
|
||||
if (!flushDone) {
|
||||
state.activeFlushPromise = promise;
|
||||
}
|
||||
return promise;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -495,6 +916,15 @@ export class DataBufferEngine {
|
||||
};
|
||||
}
|
||||
|
||||
public static getLocalBufferSnapshot(limit?: number): DataEvent[] {
|
||||
const state = getState();
|
||||
if (typeof limit === 'number' && Number.isFinite(limit) && limit >= 0) {
|
||||
const cap = Math.floor(limit);
|
||||
return cap === 0 ? [] : state.buffer.slice(-cap);
|
||||
}
|
||||
return [...state.buffer];
|
||||
}
|
||||
|
||||
public static getRecentEvents(limit = 50, filterClientId?: string): DataEvent[] {
|
||||
const state = getState();
|
||||
if (filterClientId && filterClientId !== 'all') {
|
||||
@@ -514,7 +944,8 @@ export class DataBufferEngine {
|
||||
|
||||
if (redis) {
|
||||
try {
|
||||
const readCap = Math.min(REDIS_RECENT_LIMIT, Math.max(limit * 4, limit));
|
||||
const recentLimit = this.getRecentLimit();
|
||||
const readCap = Math.min(recentLimit, Math.max(limit * 4, limit));
|
||||
const rawItems = await redis.lrange(REDIS_RECENT_KEY, 0, readCap - 1);
|
||||
state.redisConnected = true;
|
||||
|
||||
@@ -523,6 +954,9 @@ export class DataBufferEngine {
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as DataEvent;
|
||||
if (!parsed || !parsed.eventId) continue;
|
||||
if (state.deletedClientIds.has(normalizeBufferClientId(parsed.clientId))) {
|
||||
continue;
|
||||
}
|
||||
if (filterClientId && filterClientId !== 'all' && parsed.clientId !== filterClientId) {
|
||||
continue;
|
||||
}
|
||||
|
||||
+520
-117
@@ -1,9 +1,21 @@
|
||||
import { existsSync, rmSync } from "node:fs";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { existsSync, lstatSync, readdirSync, rmSync } from "node:fs";
|
||||
import { resolve, sep } from "node:path";
|
||||
import { SettingsManager } from "../adminSettings/settingsManager";
|
||||
import { AuthService } from "../auth/jwt";
|
||||
import { DataBufferEngine } from "../data/buffer";
|
||||
import { DatabaseEngine } from "../Database/database";
|
||||
import { getInternalIngestionHeaders } from "../Security/security";
|
||||
import { CSVStorageManager } from "../Storage/csvStorage";
|
||||
|
||||
export interface BridgeSyncResult {
|
||||
synced: boolean;
|
||||
status?: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
const REDIS_CLIENTS_REGISTRY_KEY = "datastack:clients:registry";
|
||||
const REDIS_CLIENTS_DELETED_KEY = "datastack:clients:deleted";
|
||||
|
||||
export interface ClientSDK {
|
||||
clientId: string;
|
||||
@@ -17,13 +29,24 @@ export interface ClientSDK {
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
type PendingActivityEntry = {
|
||||
count: number;
|
||||
lastActive: string;
|
||||
};
|
||||
|
||||
type SDKDatabaseGlobals = typeof globalThis & {
|
||||
__dataStackClientsMap?: Map<string, ClientSDK>;
|
||||
__dataStackDeletedClients?: Set<string>;
|
||||
__dataStackClientsHydrated?: boolean;
|
||||
__dataStackSyncedSocketEvents?: Map<string, number>;
|
||||
__dataStackPendingClientActivity?: Map<string, PendingActivityEntry>;
|
||||
__dataStackActivityFlushTimer?: ReturnType<typeof setInterval> | null;
|
||||
__dataStackActivityFlushPromise?: Promise<number> | null;
|
||||
__dataStackActivityShutdownRegistered?: boolean;
|
||||
};
|
||||
|
||||
const CLIENT_ACTIVITY_FLUSH_INTERVAL_MS = 1000;
|
||||
const CLIENT_ACTIVITY_IMMEDIATE_BATCH_SIZE = 500;
|
||||
const globals = globalThis as SDKDatabaseGlobals;
|
||||
|
||||
async function ensureDbSettingsHydrated(): Promise<void> {
|
||||
@@ -53,6 +76,58 @@ function getSyncedSocketEventsMap(): Map<string, number> {
|
||||
return globals.__dataStackSyncedSocketEvents;
|
||||
}
|
||||
|
||||
function getPendingClientActivityMap(): Map<string, PendingActivityEntry> {
|
||||
if (!globals.__dataStackPendingClientActivity) {
|
||||
globals.__dataStackPendingClientActivity = new Map<
|
||||
string,
|
||||
PendingActivityEntry
|
||||
>();
|
||||
}
|
||||
return globals.__dataStackPendingClientActivity;
|
||||
}
|
||||
|
||||
function ensureActivityFlushTimer(): void {
|
||||
if (globals.__dataStackActivityFlushTimer) return;
|
||||
const timer = setInterval(() => {
|
||||
void SDKManager.flushPendingClientActivityAsync().catch(() => undefined);
|
||||
}, CLIENT_ACTIVITY_FLUSH_INTERVAL_MS);
|
||||
if (typeof timer.unref === "function") {
|
||||
timer.unref();
|
||||
}
|
||||
globals.__dataStackActivityFlushTimer = timer;
|
||||
}
|
||||
|
||||
interface RemoteInMemoryClient {
|
||||
clientId?: string;
|
||||
name?: string;
|
||||
status?: string;
|
||||
token?: string;
|
||||
rateLimit?: number;
|
||||
}
|
||||
|
||||
function hydrateLocalMapFromRemote(remoteClients: RemoteInMemoryClient[]): number {
|
||||
const map = getClientsMap();
|
||||
const deletedSet = getDeletedClientsSet();
|
||||
let hydrated = 0;
|
||||
for (const item of remoteClients) {
|
||||
if (!item || !item.clientId) continue;
|
||||
const cid = SDKManager.normalizeClientId(item.clientId);
|
||||
if (!cid || deletedSet.has(cid) || map.has(cid)) continue;
|
||||
map.set(cid, {
|
||||
clientId: cid,
|
||||
name: String(item.name || cid),
|
||||
secret: "",
|
||||
token: String(item.token || ""),
|
||||
rateLimit: Math.max(1, Number(item.rateLimit) || 100),
|
||||
status: item.status === "revoked" ? "revoked" : "active",
|
||||
totalEvents: 0,
|
||||
createdAt: new Date().toISOString(),
|
||||
});
|
||||
hydrated += 1;
|
||||
}
|
||||
return hydrated;
|
||||
}
|
||||
|
||||
function serializeSyncClient(client: ClientSDK) {
|
||||
return {
|
||||
clientId: client.clientId,
|
||||
@@ -63,18 +138,53 @@ function serializeSyncClient(client: ClientSDK) {
|
||||
};
|
||||
}
|
||||
|
||||
async function notifyIngestionClientSyncAsync(
|
||||
async function mirrorClientToRedis(
|
||||
clientId: string,
|
||||
action: "deleted" | "revoked" | "updated" | "registered",
|
||||
client?: ClientSDK,
|
||||
): Promise<void> {
|
||||
const redis = DataBufferEngine.getRedisClient();
|
||||
if (!redis) return;
|
||||
try {
|
||||
if (action === "deleted") {
|
||||
if (typeof redis.sadd === "function") {
|
||||
await redis.sadd(REDIS_CLIENTS_DELETED_KEY, clientId);
|
||||
}
|
||||
if (typeof redis.hdel === "function") {
|
||||
await redis.hdel(REDIS_CLIENTS_REGISTRY_KEY, clientId);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (action === "registered" && typeof redis.srem === "function") {
|
||||
await redis.srem(REDIS_CLIENTS_DELETED_KEY, clientId);
|
||||
}
|
||||
if (client && typeof redis.hset === "function") {
|
||||
await redis.hset(
|
||||
REDIS_CLIENTS_REGISTRY_KEY,
|
||||
clientId,
|
||||
JSON.stringify(serializeSyncClient(client)),
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
// Non-fatal; DB and direct HTTP bridge remain primary sources of truth
|
||||
}
|
||||
}
|
||||
|
||||
async function notifyIngestionClientSyncAsync(
|
||||
clientId: string,
|
||||
action: "deleted" | "revoked" | "updated" | "registered",
|
||||
client?: ClientSDK,
|
||||
): Promise<BridgeSyncResult> {
|
||||
await mirrorClientToRedis(clientId, action, client);
|
||||
const headers = getInternalIngestionHeaders();
|
||||
if (!headers["x-datastack-internal-secret"]) return;
|
||||
if (!headers["x-datastack-internal-secret"]) {
|
||||
return { synced: false, error: "Internal bridge secret is not configured" };
|
||||
}
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||
).replace(/\/$/, "");
|
||||
try {
|
||||
await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
|
||||
const res = await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
@@ -87,8 +197,21 @@ async function notifyIngestionClientSyncAsync(
|
||||
}),
|
||||
signal: AbortSignal.timeout(1000),
|
||||
});
|
||||
} catch {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
if (!res.ok) {
|
||||
const errorMsg = `Port 4000 client-sync (${action}:${clientId}) failed with HTTP ${res.status}`;
|
||||
console.warn(`[DataStack Bridge] ${errorMsg}`);
|
||||
return { synced: false, status: res.status, error: errorMsg };
|
||||
}
|
||||
return { synced: true, status: res.status };
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : "Bridge request failed";
|
||||
if (
|
||||
process.env.NODE_ENV === "production" ||
|
||||
Boolean(process.env.INGESTION_INTERNAL_URL)
|
||||
) {
|
||||
console.warn(`[DataStack Bridge] Port 4000 client-sync unreachable: ${message}`);
|
||||
}
|
||||
return { synced: false, error: message };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -127,6 +250,10 @@ export class SDKManager {
|
||||
const ready = await DatabaseEngine.ensureSchema();
|
||||
if (!ready) return null;
|
||||
|
||||
if (getPendingClientActivityMap().size > 0) {
|
||||
await this.flushPendingClientActivityAsync();
|
||||
}
|
||||
|
||||
const [result, deletedResult] = await Promise.all([
|
||||
pool.query(
|
||||
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
|
||||
@@ -166,16 +293,31 @@ export class SDKManager {
|
||||
return Array.from(getClientsMap().values());
|
||||
}
|
||||
|
||||
public static async syncAllClientsToIngestionAsync(): Promise<void> {
|
||||
public static async syncAllClientsToIngestionAsync(options?: {
|
||||
forceReplace?: boolean;
|
||||
}): Promise<BridgeSyncResult> {
|
||||
await ensureDbSettingsHydrated();
|
||||
const clients = Array.from(getClientsMap().values()).map(serializeSyncClient);
|
||||
const forceReplace = options?.forceReplace ?? Boolean(DatabaseEngine.getPool());
|
||||
const redis = DataBufferEngine.getRedisClient();
|
||||
if (redis && typeof redis.hset === "function") {
|
||||
try {
|
||||
for (const c of clients) {
|
||||
await redis.hset(REDIS_CLIENTS_REGISTRY_KEY, c.clientId, JSON.stringify(c));
|
||||
}
|
||||
} catch {
|
||||
// ignore Redis mirror failure
|
||||
}
|
||||
}
|
||||
const headers = getInternalIngestionHeaders();
|
||||
if (!headers["x-datastack-internal-secret"]) return;
|
||||
if (!headers["x-datastack-internal-secret"]) {
|
||||
return { synced: false, error: "Internal bridge secret is not configured" };
|
||||
}
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||
).replace(/\/$/, "");
|
||||
const clients = Array.from(getClientsMap().values()).map(serializeSyncClient);
|
||||
try {
|
||||
await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
|
||||
const res = await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
@@ -184,11 +326,43 @@ export class SDKManager {
|
||||
body: JSON.stringify({
|
||||
action: "bulk_sync",
|
||||
clients,
|
||||
forceReplace,
|
||||
}),
|
||||
signal: AbortSignal.timeout(1500),
|
||||
});
|
||||
} catch {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
if (!res.ok) {
|
||||
const errorMsg = `Port 4000 bulk_sync failed with HTTP ${res.status}`;
|
||||
console.warn(`[DataStack Bridge] ${errorMsg}`);
|
||||
return { synced: false, status: res.status, error: errorMsg };
|
||||
}
|
||||
// BUG-02: If Port 4000 preserved an existing no-DB registry because Port 3000 restarted empty,
|
||||
// hydrate Port 3000's local map non-destructively from the returned clients.
|
||||
try {
|
||||
const payload = (await res.json()) as {
|
||||
preserved?: boolean;
|
||||
clients?: RemoteInMemoryClient[];
|
||||
};
|
||||
if (
|
||||
payload?.preserved &&
|
||||
Array.isArray(payload.clients) &&
|
||||
payload.clients.length > 0 &&
|
||||
!DatabaseEngine.getPool()
|
||||
) {
|
||||
hydrateLocalMapFromRemote(payload.clients);
|
||||
}
|
||||
} catch {
|
||||
// Response body optional
|
||||
}
|
||||
return { synced: true, status: res.status };
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : "Bridge bulk_sync unreachable";
|
||||
if (
|
||||
process.env.NODE_ENV === "production" ||
|
||||
Boolean(process.env.INGESTION_INTERNAL_URL)
|
||||
) {
|
||||
console.warn(`[DataStack Bridge] ${message}`);
|
||||
}
|
||||
return { synced: false, error: message };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -214,12 +388,20 @@ export class SDKManager {
|
||||
if (res.ok) {
|
||||
const data = (await res.json()) as {
|
||||
hasSyncedInMemoryClients?: boolean;
|
||||
inMemoryClients?: RemoteInMemoryClient[];
|
||||
clientActivity?: Record<
|
||||
string,
|
||||
{ totalEvents?: number; lastActive?: string }
|
||||
>;
|
||||
};
|
||||
if (data.hasSyncedInMemoryClients === false && map.size > 0) {
|
||||
// BUG-02: Non-destructive hydration when Port 3000 restarted empty while Port 4000 still holds clients
|
||||
if (
|
||||
map.size === 0 &&
|
||||
Array.isArray(data.inMemoryClients) &&
|
||||
data.inMemoryClients.length > 0
|
||||
) {
|
||||
hydrateLocalMapFromRemote(data.inMemoryClients);
|
||||
} else if (data.hasSyncedInMemoryClients === false && map.size > 0) {
|
||||
await this.syncAllClientsToIngestionAsync();
|
||||
}
|
||||
if (data.clientActivity && typeof data.clientActivity === "object") {
|
||||
@@ -255,36 +437,63 @@ export class SDKManager {
|
||||
return Array.from(map.values());
|
||||
}
|
||||
|
||||
public static normalizeClientId(clientId: string): string {
|
||||
return String(clientId || "")
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9_-]/g, "_");
|
||||
}
|
||||
|
||||
public static getClient(clientId: string): ClientSDK | undefined {
|
||||
return getClientsMap().get(clientId);
|
||||
const normalizedId = this.normalizeClientId(clientId);
|
||||
if (!normalizedId) return undefined;
|
||||
return getClientsMap().get(normalizedId);
|
||||
}
|
||||
|
||||
public static async getClientAsync(
|
||||
clientId: string,
|
||||
): Promise<ClientSDK | undefined> {
|
||||
await ensureDbSettingsHydrated();
|
||||
const normalizedId = this.normalizeClientId(clientId);
|
||||
if (!normalizedId) return undefined;
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (pool) {
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
if (getPendingClientActivityMap().has(normalizedId)) {
|
||||
await this.flushPendingClientActivityAsync();
|
||||
}
|
||||
const result = await pool.query(
|
||||
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
|
||||
FROM client_sdks
|
||||
WHERE client_id = $1`,
|
||||
[clientId],
|
||||
[normalizedId],
|
||||
);
|
||||
if (result.rowCount && result.rowCount > 0) {
|
||||
const client = mapRowToClient(result.rows[0]);
|
||||
getClientsMap().set(client.clientId, client);
|
||||
return client;
|
||||
}
|
||||
getClientsMap().delete(clientId);
|
||||
getClientsMap().delete(normalizedId);
|
||||
return undefined;
|
||||
} catch {
|
||||
// Fall back to in-memory map
|
||||
}
|
||||
}
|
||||
return getClientsMap().get(clientId);
|
||||
const existing = getClientsMap().get(normalizedId);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
// BUG-02: If Port 3000 restarted without DB and local map is empty, attempt non-destructive hydration from Port 4000
|
||||
if (getClientsMap().size === 0 && !getDeletedClientsSet().has(normalizedId)) {
|
||||
await this.listClientsAsync();
|
||||
return getClientsMap().get(normalizedId);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private static generateClientSecret(safeClientId: string): string {
|
||||
return `sec_${safeClientId}_${randomBytes(16).toString("hex")}`;
|
||||
}
|
||||
|
||||
public static registerClient(data: {
|
||||
@@ -292,11 +501,8 @@ export class SDKManager {
|
||||
name: string;
|
||||
rateLimit?: number;
|
||||
}): ClientSDK {
|
||||
const safeClientId = data.clientId
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9_-]/g, "_");
|
||||
const secret = `sec_${safeClientId}_${Math.random().toString(36).substring(2, 10)}`;
|
||||
const safeClientId = this.normalizeClientId(data.clientId);
|
||||
const secret = this.generateClientSecret(safeClientId);
|
||||
const rateLimit = Math.max(1, Number(data.rateLimit) || 100);
|
||||
|
||||
const token = AuthService.generateClientToken({
|
||||
@@ -317,8 +523,12 @@ export class SDKManager {
|
||||
};
|
||||
|
||||
getDeletedClientsSet().delete(safeClientId);
|
||||
DataBufferEngine.unmarkClientDeleted(safeClientId);
|
||||
getSyncedSocketEventsMap().delete(safeClientId);
|
||||
getClientsMap().set(safeClientId, newClient);
|
||||
void this.persistNewClient(newClient);
|
||||
void this.persistNewClient(newClient).catch((err) => {
|
||||
console.error("[DataStack SDK] Background client create DB persist failed:", err);
|
||||
});
|
||||
notifyIngestionClientSync(safeClientId, "registered", newClient);
|
||||
return newClient;
|
||||
}
|
||||
@@ -327,8 +537,12 @@ export class SDKManager {
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (!pool) return;
|
||||
|
||||
const ready = await DatabaseEngine.ensureSchema();
|
||||
if (!ready) {
|
||||
throw new Error("DATABASE_WRITE_FAILED: PostgreSQL schema unavailable for client registration.");
|
||||
}
|
||||
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
await pool.query(
|
||||
`DELETE FROM deleted_clients WHERE client_id = $1`,
|
||||
[client.clientId],
|
||||
@@ -354,8 +568,9 @@ export class SDKManager {
|
||||
client.createdAt,
|
||||
],
|
||||
);
|
||||
} catch {
|
||||
// In-memory map already updated
|
||||
} catch (err: unknown) {
|
||||
const detail = err instanceof Error ? err.message : "Unknown DB error";
|
||||
throw new Error(`DATABASE_WRITE_FAILED: Failed to register client (${detail})`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -365,11 +580,8 @@ export class SDKManager {
|
||||
rateLimit?: number;
|
||||
}): Promise<ClientSDK> {
|
||||
await ensureDbSettingsHydrated();
|
||||
const safeClientId = data.clientId
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9_-]/g, "_");
|
||||
const secret = `sec_${safeClientId}_${Math.random().toString(36).substring(2, 10)}`;
|
||||
const safeClientId = this.normalizeClientId(data.clientId);
|
||||
const secret = this.generateClientSecret(safeClientId);
|
||||
const rateLimit = Math.max(1, Number(data.rateLimit) || 100);
|
||||
|
||||
const token = AuthService.generateClientToken({
|
||||
@@ -389,20 +601,23 @@ export class SDKManager {
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
getDeletedClientsSet().delete(safeClientId);
|
||||
getClientsMap().set(safeClientId, newClient);
|
||||
await this.persistNewClient(newClient);
|
||||
getDeletedClientsSet().delete(safeClientId);
|
||||
DataBufferEngine.unmarkClientDeleted(safeClientId);
|
||||
getSyncedSocketEventsMap().delete(safeClientId);
|
||||
getClientsMap().set(safeClientId, newClient);
|
||||
await notifyIngestionClientSyncAsync(safeClientId, "registered", newClient);
|
||||
return newClient;
|
||||
}
|
||||
|
||||
public static regenerateToken(clientId: string): ClientSDK | null {
|
||||
const client = getClientsMap().get(clientId);
|
||||
const normalizedId = this.normalizeClientId(clientId);
|
||||
const client = getClientsMap().get(normalizedId);
|
||||
if (!client) return null;
|
||||
|
||||
const updated: ClientSDK = {
|
||||
...client,
|
||||
secret: `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`,
|
||||
secret: this.generateClientSecret(normalizedId),
|
||||
token: AuthService.generateClientToken({
|
||||
clientId: client.clientId,
|
||||
name: client.name,
|
||||
@@ -410,9 +625,11 @@ export class SDKManager {
|
||||
}),
|
||||
};
|
||||
|
||||
getClientsMap().set(clientId, updated);
|
||||
void this.persistClientCredentials(updated);
|
||||
notifyIngestionClientSync(clientId, "updated", updated);
|
||||
getClientsMap().set(normalizedId, updated);
|
||||
void this.persistClientCredentials(updated).catch((err) => {
|
||||
console.error("[DataStack SDK] Background token rotate DB persist failed:", err);
|
||||
});
|
||||
notifyIngestionClientSync(normalizedId, "updated", updated);
|
||||
return updated;
|
||||
}
|
||||
|
||||
@@ -422,16 +639,21 @@ export class SDKManager {
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (!pool) return;
|
||||
|
||||
const ready = await DatabaseEngine.ensureSchema();
|
||||
if (!ready) {
|
||||
throw new Error("DATABASE_WRITE_FAILED: PostgreSQL schema unavailable for credential update.");
|
||||
}
|
||||
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
await pool.query(
|
||||
`UPDATE client_sdks
|
||||
SET secret = $2, token = $3, updated_at = NOW()
|
||||
WHERE client_id = $1`,
|
||||
[client.clientId, client.secret, client.token],
|
||||
);
|
||||
} catch {
|
||||
// Ignore DB failure
|
||||
} catch (err: unknown) {
|
||||
const detail = err instanceof Error ? err.message : "Unknown DB error";
|
||||
throw new Error(`DATABASE_WRITE_FAILED: Failed to update client credentials (${detail})`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -439,12 +661,13 @@ export class SDKManager {
|
||||
clientId: string,
|
||||
): Promise<ClientSDK | null> {
|
||||
await ensureDbSettingsHydrated();
|
||||
const existing = await this.getClientAsync(clientId);
|
||||
const normalizedId = this.normalizeClientId(clientId);
|
||||
const existing = await this.getClientAsync(normalizedId);
|
||||
if (!existing) return null;
|
||||
|
||||
const updated: ClientSDK = {
|
||||
...existing,
|
||||
secret: `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`,
|
||||
secret: this.generateClientSecret(normalizedId),
|
||||
token: AuthService.generateClientToken({
|
||||
clientId: existing.clientId,
|
||||
name: existing.name,
|
||||
@@ -452,9 +675,9 @@ export class SDKManager {
|
||||
}),
|
||||
};
|
||||
|
||||
getClientsMap().set(clientId, updated);
|
||||
await this.persistClientCredentials(updated);
|
||||
await notifyIngestionClientSyncAsync(clientId, "updated", updated);
|
||||
getClientsMap().set(normalizedId, updated);
|
||||
await notifyIngestionClientSyncAsync(normalizedId, "updated", updated);
|
||||
return updated;
|
||||
}
|
||||
|
||||
@@ -462,7 +685,8 @@ export class SDKManager {
|
||||
clientId: string,
|
||||
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
|
||||
): ClientSDK | null {
|
||||
const client = getClientsMap().get(clientId);
|
||||
const normalizedId = this.normalizeClientId(clientId);
|
||||
const client = getClientsMap().get(normalizedId);
|
||||
if (!client) return null;
|
||||
|
||||
const updated: ClientSDK = {
|
||||
@@ -474,10 +698,12 @@ export class SDKManager {
|
||||
...(updates.status !== undefined ? { status: updates.status } : {}),
|
||||
};
|
||||
|
||||
getClientsMap().set(clientId, updated);
|
||||
void this.persistClientUpdates(updated);
|
||||
getClientsMap().set(normalizedId, updated);
|
||||
void this.persistClientUpdates(updated).catch((err) => {
|
||||
console.error("[DataStack SDK] Background client update DB persist failed:", err);
|
||||
});
|
||||
notifyIngestionClientSync(
|
||||
clientId,
|
||||
normalizedId,
|
||||
updated.status === "revoked" ? "revoked" : "updated",
|
||||
updated,
|
||||
);
|
||||
@@ -488,16 +714,21 @@ export class SDKManager {
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (!pool) return;
|
||||
|
||||
const ready = await DatabaseEngine.ensureSchema();
|
||||
if (!ready) {
|
||||
throw new Error("DATABASE_WRITE_FAILED: PostgreSQL schema unavailable for client update.");
|
||||
}
|
||||
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
await pool.query(
|
||||
`UPDATE client_sdks
|
||||
SET name = $2, rate_limit = $3, status = $4, updated_at = NOW()
|
||||
WHERE client_id = $1`,
|
||||
[client.clientId, client.name, client.rateLimit, client.status],
|
||||
);
|
||||
} catch {
|
||||
// Ignore DB failure
|
||||
} catch (err: unknown) {
|
||||
const detail = err instanceof Error ? err.message : "Unknown DB error";
|
||||
throw new Error(`DATABASE_WRITE_FAILED: Failed to update client (${detail})`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -506,7 +737,8 @@ export class SDKManager {
|
||||
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
|
||||
): Promise<ClientSDK | null> {
|
||||
await ensureDbSettingsHydrated();
|
||||
const client = await this.getClientAsync(clientId);
|
||||
const normalizedId = this.normalizeClientId(clientId);
|
||||
const client = await this.getClientAsync(normalizedId);
|
||||
if (!client) return null;
|
||||
|
||||
const updated: ClientSDK = {
|
||||
@@ -518,10 +750,10 @@ export class SDKManager {
|
||||
...(updates.status !== undefined ? { status: updates.status } : {}),
|
||||
};
|
||||
|
||||
getClientsMap().set(clientId, updated);
|
||||
await this.persistClientUpdates(updated);
|
||||
getClientsMap().set(normalizedId, updated);
|
||||
await notifyIngestionClientSyncAsync(
|
||||
clientId,
|
||||
normalizedId,
|
||||
updated.status === "revoked" ? "revoked" : "updated",
|
||||
updated,
|
||||
);
|
||||
@@ -529,16 +761,12 @@ export class SDKManager {
|
||||
}
|
||||
|
||||
public static isClientDeleted(clientId: string): boolean {
|
||||
const normalized = String(clientId || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const normalized = this.normalizeClientId(clientId);
|
||||
return getDeletedClientsSet().has(normalized);
|
||||
}
|
||||
|
||||
public static async isClientDeletedAsync(clientId: string): Promise<boolean> {
|
||||
const normalized = String(clientId || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const normalized = this.normalizeClientId(clientId);
|
||||
if (!normalized) return false;
|
||||
if (getDeletedClientsSet().has(normalized)) {
|
||||
return true;
|
||||
@@ -560,15 +788,18 @@ export class SDKManager {
|
||||
}
|
||||
|
||||
public static deleteClient(clientId: string): boolean {
|
||||
const normalized = String(clientId || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const normalized = this.normalizeClientId(clientId);
|
||||
const deleted = getClientsMap().delete(normalized);
|
||||
getSyncedSocketEventsMap().delete(normalized);
|
||||
getPendingClientActivityMap().delete(normalized);
|
||||
if (deleted) {
|
||||
getDeletedClientsSet().add(normalized);
|
||||
DataBufferEngine.markClientDeleted(normalized);
|
||||
void DataBufferEngine.purgeClientEventsAsync(normalized).catch(() => undefined);
|
||||
}
|
||||
void this.deleteClientFromDb(normalized);
|
||||
void this.deleteClientFromDb(normalized).catch((err) => {
|
||||
console.error("[DataStack SDK] Background client delete DB persist failed:", err);
|
||||
});
|
||||
notifyIngestionClientSync(normalized, "deleted");
|
||||
return deleted;
|
||||
}
|
||||
@@ -577,7 +808,10 @@ export class SDKManager {
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (!pool) return false;
|
||||
|
||||
await DatabaseEngine.ensureSchema();
|
||||
const ready = await DatabaseEngine.ensureSchema();
|
||||
if (!ready) {
|
||||
throw new Error("DATABASE_WRITE_FAILED: PostgreSQL schema unavailable for client deletion.");
|
||||
}
|
||||
const dbClient = await pool.connect();
|
||||
try {
|
||||
await dbClient.query("BEGIN");
|
||||
@@ -597,69 +831,226 @@ export class SDKManager {
|
||||
}
|
||||
await dbClient.query("ROLLBACK");
|
||||
return false;
|
||||
} catch {
|
||||
} catch (err: unknown) {
|
||||
await dbClient.query("ROLLBACK").catch(() => undefined);
|
||||
return false;
|
||||
const detail = err instanceof Error ? err.message : "Unknown DB error";
|
||||
throw new Error(`DATABASE_WRITE_FAILED: Failed to delete client (${detail})`);
|
||||
} finally {
|
||||
dbClient.release();
|
||||
}
|
||||
}
|
||||
|
||||
private static purgeClientStorageDir(clientId: string): void {
|
||||
private static purgeClientStorageDir(clientId: string): number {
|
||||
const safeId = clientId.replace(/[^a-z0-9_-]/gi, "_");
|
||||
if (!safeId) return;
|
||||
const storageRoot = resolve(process.cwd(), "storage", "data");
|
||||
if (!safeId) return 0;
|
||||
const storageRoot = resolve(CSVStorageManager.baseStorageDir);
|
||||
const targetDir = resolve(storageRoot, safeId);
|
||||
if (!targetDir.startsWith(`${storageRoot}${sep}`)) return;
|
||||
if (existsSync(targetDir)) {
|
||||
rmSync(targetDir, { recursive: true, force: true });
|
||||
if (!targetDir.startsWith(`${storageRoot}${sep}`)) return 0;
|
||||
if (!existsSync(targetDir)) return 0;
|
||||
|
||||
let purgedFiles = 0;
|
||||
try {
|
||||
const dirStat = lstatSync(targetDir);
|
||||
if (!dirStat.isSymbolicLink() && dirStat.isDirectory()) {
|
||||
for (const entry of readdirSync(targetDir)) {
|
||||
if (!entry.endsWith(".csv")) continue;
|
||||
const entryPath = resolve(targetDir, entry);
|
||||
try {
|
||||
const entryStat = lstatSync(entryPath);
|
||||
if (!entryStat.isSymbolicLink() && entryStat.isFile()) {
|
||||
purgedFiles += 1;
|
||||
}
|
||||
} catch {
|
||||
// Ignore unreadable file entry during pre-purge count
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
purgedFiles = 0;
|
||||
}
|
||||
|
||||
rmSync(targetDir, { recursive: true, force: true });
|
||||
CSVStorageManager.clearRowCountCache();
|
||||
return purgedFiles;
|
||||
}
|
||||
|
||||
public static async deleteClientWithStatsAsync(
|
||||
clientId: string,
|
||||
options?: { purgeFiles?: boolean },
|
||||
): Promise<{ deleted: boolean; purgedFiles: number }> {
|
||||
await ensureDbSettingsHydrated();
|
||||
const normalized = this.normalizeClientId(clientId);
|
||||
getPendingClientActivityMap().delete(normalized);
|
||||
const deletedFromDb = await this.deleteClientFromDb(normalized);
|
||||
const existedInMem = getClientsMap().delete(normalized);
|
||||
getSyncedSocketEventsMap().delete(normalized);
|
||||
let purgedFiles = 0;
|
||||
if (existedInMem || deletedFromDb) {
|
||||
getDeletedClientsSet().add(normalized);
|
||||
DataBufferEngine.markClientDeleted(normalized);
|
||||
await DataBufferEngine.purgeClientEventsAsync(normalized);
|
||||
await notifyIngestionClientSyncAsync(normalized, "deleted");
|
||||
if (options?.purgeFiles) {
|
||||
purgedFiles = this.purgeClientStorageDir(normalized);
|
||||
}
|
||||
}
|
||||
return { deleted: existedInMem || deletedFromDb, purgedFiles };
|
||||
}
|
||||
|
||||
public static async deleteClientAsync(
|
||||
clientId: string,
|
||||
options?: { purgeFiles?: boolean },
|
||||
): Promise<boolean> {
|
||||
await ensureDbSettingsHydrated();
|
||||
const normalized = String(clientId || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const existedInMem = getClientsMap().delete(normalized);
|
||||
getSyncedSocketEventsMap().delete(normalized);
|
||||
const deletedFromDb = await this.deleteClientFromDb(normalized);
|
||||
if (existedInMem || deletedFromDb) {
|
||||
getDeletedClientsSet().add(normalized);
|
||||
if (options?.purgeFiles) {
|
||||
this.purgeClientStorageDir(normalized);
|
||||
}
|
||||
await notifyIngestionClientSyncAsync(normalized, "deleted");
|
||||
}
|
||||
return existedInMem || deletedFromDb;
|
||||
const result = await this.deleteClientWithStatsAsync(clientId, options);
|
||||
return result.deleted;
|
||||
}
|
||||
|
||||
public static incrementEventCount(clientId: string, count = 1): void {
|
||||
const nowIso = new Date().toISOString();
|
||||
const client = getClientsMap().get(clientId);
|
||||
if (client) {
|
||||
client.totalEvents += count;
|
||||
client.lastActive = nowIso;
|
||||
getClientsMap().set(clientId, client);
|
||||
private static queuePendingActivityDelta(
|
||||
normalizedId: string,
|
||||
count: number,
|
||||
nowIso: string,
|
||||
): number {
|
||||
const pending = getPendingClientActivityMap();
|
||||
const existing = pending.get(normalizedId);
|
||||
if (existing) {
|
||||
pending.set(normalizedId, {
|
||||
count: existing.count + count,
|
||||
lastActive:
|
||||
nowIso > existing.lastActive ? nowIso : existing.lastActive,
|
||||
});
|
||||
} else {
|
||||
pending.set(normalizedId, {
|
||||
count,
|
||||
lastActive: nowIso,
|
||||
});
|
||||
}
|
||||
ensureActivityFlushTimer();
|
||||
let totalDelta = 0;
|
||||
for (const entry of pending.values()) {
|
||||
totalDelta += entry.count;
|
||||
}
|
||||
return totalDelta;
|
||||
}
|
||||
|
||||
public static async flushPendingClientActivityAsync(): Promise<number> {
|
||||
if (globals.__dataStackActivityFlushPromise) {
|
||||
await globals.__dataStackActivityFlushPromise;
|
||||
if (getPendingClientActivityMap().size === 0) return 0;
|
||||
}
|
||||
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (pool) {
|
||||
void DatabaseEngine.ensureSchema()
|
||||
.then(() =>
|
||||
pool.query(
|
||||
`UPDATE client_sdks
|
||||
SET total_events = total_events + $2,
|
||||
last_active = $3,
|
||||
updated_at = NOW()
|
||||
WHERE client_id = $1`,
|
||||
[clientId, count, nowIso],
|
||||
),
|
||||
)
|
||||
.catch(() => undefined);
|
||||
const pending = getPendingClientActivityMap();
|
||||
if (!pool || pending.size === 0) return 0;
|
||||
|
||||
const snapshot = Array.from(pending.entries());
|
||||
pending.clear();
|
||||
|
||||
const clientIds: string[] = [];
|
||||
const deltas: number[] = [];
|
||||
const lastActives: string[] = [];
|
||||
for (const [cid, entry] of snapshot) {
|
||||
clientIds.push(cid);
|
||||
deltas.push(entry.count);
|
||||
lastActives.push(entry.lastActive);
|
||||
}
|
||||
|
||||
const run = (async () => {
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
await pool.query(
|
||||
`UPDATE client_sdks AS c
|
||||
SET total_events = c.total_events + v.delta,
|
||||
last_active = GREATEST(COALESCE(c.last_active, v.last_active), v.last_active),
|
||||
updated_at = NOW()
|
||||
FROM (
|
||||
SELECT UNNEST($1::text[]) AS client_id,
|
||||
UNNEST($2::bigint[]) AS delta,
|
||||
UNNEST($3::timestamptz[]) AS last_active
|
||||
) AS v
|
||||
WHERE c.client_id = v.client_id`,
|
||||
[clientIds, deltas, lastActives],
|
||||
);
|
||||
return clientIds.length;
|
||||
} catch {
|
||||
// Re-merge deltas so transient DB errors do not drop event counts
|
||||
for (const [cid, entry] of snapshot) {
|
||||
const current = pending.get(cid);
|
||||
if (current) {
|
||||
pending.set(cid, {
|
||||
count: current.count + entry.count,
|
||||
lastActive:
|
||||
current.lastActive > entry.lastActive
|
||||
? current.lastActive
|
||||
: entry.lastActive,
|
||||
});
|
||||
} else {
|
||||
pending.set(cid, { ...entry });
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
} finally {
|
||||
globals.__dataStackActivityFlushPromise = null;
|
||||
}
|
||||
})();
|
||||
|
||||
globals.__dataStackActivityFlushPromise = run;
|
||||
return run;
|
||||
}
|
||||
|
||||
public static stopActivityFlushTimerForTests(): void {
|
||||
if (globals.__dataStackActivityFlushTimer) {
|
||||
clearInterval(globals.__dataStackActivityFlushTimer);
|
||||
globals.__dataStackActivityFlushTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
public static incrementEventCount(clientId: string, count = 1): void {
|
||||
const normalizedId = this.normalizeClientId(clientId);
|
||||
if (!normalizedId || count <= 0) return;
|
||||
const nowIso = new Date().toISOString();
|
||||
const client = getClientsMap().get(normalizedId);
|
||||
if (client) {
|
||||
client.totalEvents += count;
|
||||
client.lastActive = nowIso;
|
||||
getClientsMap().set(normalizedId, client);
|
||||
}
|
||||
|
||||
if (DatabaseEngine.getPool()) {
|
||||
const queuedTotal = this.queuePendingActivityDelta(
|
||||
normalizedId,
|
||||
count,
|
||||
nowIso,
|
||||
);
|
||||
if (queuedTotal >= CLIENT_ACTIVITY_IMMEDIATE_BATCH_SIZE) {
|
||||
void this.flushPendingClientActivityAsync().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static async incrementEventCountAsync(
|
||||
clientId: string,
|
||||
count = 1,
|
||||
): Promise<void> {
|
||||
const normalizedId = this.normalizeClientId(clientId);
|
||||
if (!normalizedId || count <= 0) return;
|
||||
const nowIso = new Date().toISOString();
|
||||
const client = getClientsMap().get(normalizedId);
|
||||
if (client) {
|
||||
client.totalEvents += count;
|
||||
client.lastActive = nowIso;
|
||||
getClientsMap().set(normalizedId, client);
|
||||
}
|
||||
|
||||
// BUG-22: Coalesce per-event DB updates into batched UNNEST writes instead of 1 SQL query/event
|
||||
if (DatabaseEngine.getPool()) {
|
||||
const queuedTotal = this.queuePendingActivityDelta(
|
||||
normalizedId,
|
||||
count,
|
||||
nowIso,
|
||||
);
|
||||
if (queuedTotal >= CLIENT_ACTIVITY_IMMEDIATE_BATCH_SIZE) {
|
||||
await this.flushPendingClientActivityAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -672,7 +1063,9 @@ export class SDKManager {
|
||||
rateLimit: client.rateLimit,
|
||||
});
|
||||
map.set(clientId, client);
|
||||
void this.persistClientCredentials(client);
|
||||
void this.persistClientCredentials(client).catch((err) => {
|
||||
console.error("[DataStack SDK] Background re-sign DB persist failed:", err);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -680,15 +1073,25 @@ export class SDKManager {
|
||||
const clients = await this.listClientsAsync();
|
||||
const map = getClientsMap();
|
||||
for (const client of clients) {
|
||||
client.token = AuthService.generateClientToken({
|
||||
clientId: client.clientId,
|
||||
name: client.name,
|
||||
rateLimit: client.rateLimit,
|
||||
});
|
||||
map.set(client.clientId, client);
|
||||
await this.persistClientCredentials(client);
|
||||
const updated: ClientSDK = {
|
||||
...client,
|
||||
token: AuthService.generateClientToken({
|
||||
clientId: client.clientId,
|
||||
name: client.name,
|
||||
rateLimit: client.rateLimit,
|
||||
}),
|
||||
};
|
||||
await this.persistClientCredentials(updated);
|
||||
map.set(updated.clientId, updated);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!globals.__dataStackActivityShutdownRegistered) {
|
||||
globals.__dataStackActivityShutdownRegistered = true;
|
||||
DataBufferEngine.registerShutdownHook(() =>
|
||||
SDKManager.flushPendingClientActivityAsync(),
|
||||
);
|
||||
}
|
||||
|
||||
export default SDKManager;
|
||||
+19
-11
@@ -13,22 +13,18 @@ map $http_x_forwarded_host $forwarded_host {
|
||||
'' $host;
|
||||
}
|
||||
|
||||
upstream nextjs_upstream {
|
||||
server web:3000;
|
||||
}
|
||||
|
||||
upstream socketio_upstream {
|
||||
server ingestion:4000;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name _;
|
||||
|
||||
resolver 127.0.0.11 valid=10s ipv6=off;
|
||||
set $nextjs_upstream "http://web:3000";
|
||||
set $socketio_upstream "http://ingestion:4000";
|
||||
|
||||
client_max_body_size 50M;
|
||||
|
||||
location = /health {
|
||||
proxy_pass http://socketio_upstream;
|
||||
proxy_pass $socketio_upstream;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-Host $forwarded_host;
|
||||
@@ -38,7 +34,7 @@ server {
|
||||
}
|
||||
|
||||
location /socket.io/ {
|
||||
proxy_pass http://socketio_upstream;
|
||||
proxy_pass $socketio_upstream;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection $connection_upgrade;
|
||||
@@ -52,8 +48,20 @@ server {
|
||||
proxy_buffering off;
|
||||
}
|
||||
|
||||
location = /api/export {
|
||||
proxy_pass $nextjs_upstream;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-Host $forwarded_host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $forwarded_proto;
|
||||
proxy_buffering off;
|
||||
proxy_read_timeout 300s;
|
||||
}
|
||||
|
||||
location / {
|
||||
proxy_pass http://nextjs_upstream;
|
||||
proxy_pass $nextjs_upstream;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-Host $forwarded_host;
|
||||
|
||||
@@ -25,6 +25,7 @@ services:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 20s
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
@@ -54,12 +55,14 @@ services:
|
||||
DATASTACK_INTERNAL_SECRET: "${DATASTACK_INTERNAL_SECRET:-${SERVICE_REALBASE64_64_INTERNAL:-}}"
|
||||
DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}"
|
||||
REDIS_URL: "redis://redis:6379"
|
||||
DATASTACK_CORS_ORIGINS: "${DATASTACK_CORS_ORIGINS:-}"
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 20s
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
|
||||
@@ -72,7 +72,7 @@ DataStack is optimized for deployment via [Coolify](https://coolify.io/):
|
||||
1. **Create Resource**: In Coolify, click **+ Add Resource** → **Docker Compose**.
|
||||
2. **Repository Settings**:
|
||||
- Point to your DataStack Git repository.
|
||||
- Set **Base Directory** to `/docker`.
|
||||
- Set **Base Directory** to `/` (so Compose build context `..` from `/docker` resolves cleanly to the repository root; if an existing VPS deployment already has saved working Coolify paths, keep its current settings).
|
||||
- Set **Docker Compose Location** to `/docker/docker-compose.yml`.
|
||||
3. **Domain Assignment**:
|
||||
- Attach your custom domain (e.g. `https://datastack.yourdomain.com`) to the **`proxy`** service on internal port `80`.
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
export const ADMIN_SESSION_COOKIE = "datastack_admin_session";
|
||||
|
||||
export async function requireAdminApiSession(req: NextRequest) {
|
||||
const token = req.cookies.get(ADMIN_SESSION_COOKIE)?.value;
|
||||
const auth = await verifyRequestAdminSession(token);
|
||||
if (auth.valid) {
|
||||
return { valid: true as const, auth };
|
||||
}
|
||||
|
||||
return {
|
||||
valid: false as const,
|
||||
response: NextResponse.json(
|
||||
{
|
||||
error:
|
||||
auth.reason === "UNAUTHORIZED"
|
||||
? "Authentication required."
|
||||
: "Admin authentication is temporarily unavailable.",
|
||||
},
|
||||
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the client IP for authentication throttling in a reverse-proxy-safe order:
|
||||
* 1. `x-real-ip` (overwritten directly by Nginx/Coolify with `$remote_addr`)
|
||||
* 2. Rightmost non-empty hop of `x-forwarded-for` (appended by the trusted edge proxy,
|
||||
* preventing attackers from spoofing arbitrary leftmost client IPs)
|
||||
* 3. Fallback `"unknown"`
|
||||
*/
|
||||
export function extractClientIp(request: NextRequest): string {
|
||||
const realIp = request.headers.get("x-real-ip")?.trim();
|
||||
if (realIp) {
|
||||
return realIp.slice(0, 128);
|
||||
}
|
||||
|
||||
const forwardedFor = request.headers.get("x-forwarded-for");
|
||||
if (forwardedFor) {
|
||||
const hops = forwardedFor
|
||||
.split(",")
|
||||
.map((part) => part.trim())
|
||||
.filter(Boolean);
|
||||
const rightmost = hops[hops.length - 1];
|
||||
if (rightmost) {
|
||||
return rightmost.slice(0, 128);
|
||||
}
|
||||
}
|
||||
|
||||
return "unknown";
|
||||
}
|
||||
@@ -15,3 +15,36 @@ export function setCached<T>(key: string, value: T): T {
|
||||
memoryStore.set(key, value);
|
||||
return value;
|
||||
}
|
||||
|
||||
export async function clearAuthCaches(): Promise<void> {
|
||||
memoryStore.clear();
|
||||
if (typeof window === 'undefined') return;
|
||||
try {
|
||||
if ('serviceWorker' in navigator && navigator.serviceWorker.controller) {
|
||||
navigator.serviceWorker.controller.postMessage({ type: 'CLEAR_AUTH_CACHE' });
|
||||
}
|
||||
if ('caches' in window) {
|
||||
const keys = await window.caches.keys();
|
||||
await Promise.all(
|
||||
keys.map(async (cacheKey) => {
|
||||
const cache = await window.caches.open(cacheKey);
|
||||
const requests = await cache.keys();
|
||||
await Promise.all(
|
||||
requests
|
||||
.filter((req) => {
|
||||
try {
|
||||
const u = new URL(req.url);
|
||||
return u.pathname === '/vcon' || u.pathname.startsWith('/vcon/');
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
})
|
||||
.map((req) => cache.delete(req))
|
||||
);
|
||||
})
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
// Best-effort cache cleanup
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,537 @@
|
||||
export interface EditableSettings {
|
||||
maxEventsPerSec: number;
|
||||
maxPayloadSizeKb: number;
|
||||
batchFlushThreshold: number;
|
||||
batchFlushIntervalMs: number;
|
||||
fileRotationMb: number;
|
||||
enableIdempotency: boolean;
|
||||
autoPruneDays: number;
|
||||
}
|
||||
|
||||
export const SETTINGS_LIMITS = {
|
||||
maxEventsPerSec: { min: 1, max: 100000, defaultValue: 100 },
|
||||
maxPayloadSizeKb: { min: 1, max: 10240, defaultValue: 100 },
|
||||
batchFlushThreshold: { min: 1, max: 50000, defaultValue: 500 },
|
||||
batchFlushIntervalMs: { min: 100, max: 60000, defaultValue: 2000 },
|
||||
fileRotationMb: { min: 1, max: 2048, defaultValue: 100 },
|
||||
autoPruneDays: { min: 0, max: 3650, defaultValue: 30 },
|
||||
} as const;
|
||||
|
||||
export const DEFAULT_EDITABLE_SETTINGS: EditableSettings = {
|
||||
maxEventsPerSec: SETTINGS_LIMITS.maxEventsPerSec.defaultValue,
|
||||
maxPayloadSizeKb: SETTINGS_LIMITS.maxPayloadSizeKb.defaultValue,
|
||||
batchFlushThreshold: SETTINGS_LIMITS.batchFlushThreshold.defaultValue,
|
||||
batchFlushIntervalMs: SETTINGS_LIMITS.batchFlushIntervalMs.defaultValue,
|
||||
fileRotationMb: SETTINGS_LIMITS.fileRotationMb.defaultValue,
|
||||
enableIdempotency: true,
|
||||
autoPruneDays: SETTINGS_LIMITS.autoPruneDays.defaultValue,
|
||||
};
|
||||
|
||||
function clampInt(raw: unknown, min: number, max: number, fallback: number): number {
|
||||
const numeric = typeof raw === 'number' ? raw : Number(raw);
|
||||
if (!Number.isFinite(numeric)) return fallback;
|
||||
const rounded = Math.trunc(numeric);
|
||||
return Math.max(min, Math.min(max, rounded));
|
||||
}
|
||||
|
||||
export function normalizeEditableSettings(
|
||||
input: Partial<EditableSettings>,
|
||||
): EditableSettings {
|
||||
return {
|
||||
maxEventsPerSec: clampInt(
|
||||
input.maxEventsPerSec,
|
||||
SETTINGS_LIMITS.maxEventsPerSec.min,
|
||||
SETTINGS_LIMITS.maxEventsPerSec.max,
|
||||
SETTINGS_LIMITS.maxEventsPerSec.defaultValue,
|
||||
),
|
||||
maxPayloadSizeKb: clampInt(
|
||||
input.maxPayloadSizeKb,
|
||||
SETTINGS_LIMITS.maxPayloadSizeKb.min,
|
||||
SETTINGS_LIMITS.maxPayloadSizeKb.max,
|
||||
SETTINGS_LIMITS.maxPayloadSizeKb.defaultValue,
|
||||
),
|
||||
batchFlushThreshold: clampInt(
|
||||
input.batchFlushThreshold,
|
||||
SETTINGS_LIMITS.batchFlushThreshold.min,
|
||||
SETTINGS_LIMITS.batchFlushThreshold.max,
|
||||
SETTINGS_LIMITS.batchFlushThreshold.defaultValue,
|
||||
),
|
||||
batchFlushIntervalMs: clampInt(
|
||||
input.batchFlushIntervalMs,
|
||||
SETTINGS_LIMITS.batchFlushIntervalMs.min,
|
||||
SETTINGS_LIMITS.batchFlushIntervalMs.max,
|
||||
SETTINGS_LIMITS.batchFlushIntervalMs.defaultValue,
|
||||
),
|
||||
fileRotationMb: clampInt(
|
||||
input.fileRotationMb,
|
||||
SETTINGS_LIMITS.fileRotationMb.min,
|
||||
SETTINGS_LIMITS.fileRotationMb.max,
|
||||
SETTINGS_LIMITS.fileRotationMb.defaultValue,
|
||||
),
|
||||
enableIdempotency:
|
||||
typeof input.enableIdempotency === 'boolean'
|
||||
? input.enableIdempotency
|
||||
: DEFAULT_EDITABLE_SETTINGS.enableIdempotency,
|
||||
autoPruneDays: clampInt(
|
||||
input.autoPruneDays,
|
||||
SETTINGS_LIMITS.autoPruneDays.min,
|
||||
SETTINGS_LIMITS.autoPruneDays.max,
|
||||
SETTINGS_LIMITS.autoPruneDays.defaultValue,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
export interface BridgeSyncPayload {
|
||||
attempted?: boolean;
|
||||
ok?: boolean;
|
||||
synced?: boolean;
|
||||
status?: number;
|
||||
error?: string;
|
||||
checkedAt?: string;
|
||||
}
|
||||
|
||||
export function describeSettingsBridgeSync(
|
||||
bridgeSync: BridgeSyncPayload | undefined,
|
||||
): { level: 'success' | 'warning' | 'error'; title: string; message?: string } {
|
||||
if (!bridgeSync) {
|
||||
return { level: 'success', title: 'Settings saved' };
|
||||
}
|
||||
const attempted =
|
||||
bridgeSync.attempted !== undefined
|
||||
? bridgeSync.attempted
|
||||
: typeof bridgeSync.synced === 'boolean';
|
||||
const ok =
|
||||
bridgeSync.ok !== undefined
|
||||
? bridgeSync.ok
|
||||
: Boolean(bridgeSync.synced);
|
||||
if (!attempted || ok) {
|
||||
return { level: 'success', title: 'Settings saved' };
|
||||
}
|
||||
if (bridgeSync.status === 401 || bridgeSync.status === 503) {
|
||||
return {
|
||||
level: 'error',
|
||||
title: 'Saved to DB — Ingestion bridge rejected sync',
|
||||
message:
|
||||
bridgeSync.error ||
|
||||
`Port 4000 returned HTTP ${bridgeSync.status}. Verify DATASTACK_INTERNAL_SECRET matches across runtimes.`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
level: 'warning',
|
||||
title: 'Saved to DB — Ingestion bridge unreachable',
|
||||
message:
|
||||
bridgeSync.error ||
|
||||
'Port 4000 ingestion server is offline; changes will apply once it reconnects.',
|
||||
};
|
||||
}
|
||||
|
||||
export type IngestionBridgeStatus = 'online' | 'auth_error' | 'error' | 'offline';
|
||||
export type SocketHealthUiState =
|
||||
| 'idle'
|
||||
| 'checking'
|
||||
| 'online'
|
||||
| 'auth_error'
|
||||
| 'offline';
|
||||
|
||||
export function mapBridgeStatusToSocketUiState(
|
||||
status?: string,
|
||||
): SocketHealthUiState {
|
||||
if (status === 'online') return 'online';
|
||||
if (status === 'auth_error') return 'auth_error';
|
||||
return 'offline';
|
||||
}
|
||||
|
||||
export function parseEventTimestamp(value: string): Date | null {
|
||||
const trimmed = String(value ?? '').trim();
|
||||
if (!trimmed) return null;
|
||||
if (/^\d+(\.\d+)?$/.test(trimmed)) {
|
||||
const asNumber = Number(trimmed);
|
||||
if (Number.isFinite(asNumber)) {
|
||||
const ms = asNumber < 1e11 ? asNumber * 1000 : asNumber;
|
||||
const date = new Date(ms);
|
||||
return Number.isNaN(date.getTime()) ? null : date;
|
||||
}
|
||||
}
|
||||
const date = new Date(trimmed);
|
||||
return Number.isNaN(date.getTime()) ? null : date;
|
||||
}
|
||||
|
||||
export function getEventUtcDate(value: string): string | null {
|
||||
const parsed = parseEventTimestamp(value);
|
||||
return parsed ? parsed.toISOString().slice(0, 10) : null;
|
||||
}
|
||||
|
||||
export function formatEventTimeLabel(
|
||||
value: string,
|
||||
options: Intl.DateTimeFormatOptions = {
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
second: '2-digit',
|
||||
hour12: false,
|
||||
},
|
||||
): string {
|
||||
const parsed = parseEventTimestamp(value);
|
||||
return parsed ? parsed.toLocaleTimeString([], options) : value || '—';
|
||||
}
|
||||
|
||||
export function getUtcDateRangeForPreset(
|
||||
preset: 'today' | '7d' | '30d' | 'all',
|
||||
referenceNow: Date = new Date(),
|
||||
): { from: string; to: string } {
|
||||
if (preset === 'all') {
|
||||
return { from: '', to: '' };
|
||||
}
|
||||
const toUtc = new Date(
|
||||
Date.UTC(
|
||||
referenceNow.getUTCFullYear(),
|
||||
referenceNow.getUTCMonth(),
|
||||
referenceNow.getUTCDate(),
|
||||
),
|
||||
);
|
||||
const offsetDays = preset === '7d' ? 6 : preset === '30d' ? 29 : 0;
|
||||
const fromUtc = new Date(toUtc.getTime() - offsetDays * 86_400_000);
|
||||
return {
|
||||
from: fromUtc.toISOString().slice(0, 10),
|
||||
to: toUtc.toISOString().slice(0, 10),
|
||||
};
|
||||
}
|
||||
|
||||
export function sanitizeUiClientId(clientId: string): string {
|
||||
return clientId.trim().replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
}
|
||||
|
||||
export function matchesSelectedClientFolder(
|
||||
folderClientId: string,
|
||||
selectedClientId: string,
|
||||
): boolean {
|
||||
if (!selectedClientId || selectedClientId === 'all') return true;
|
||||
const trimmed = selectedClientId.trim();
|
||||
return (
|
||||
folderClientId === trimmed ||
|
||||
folderClientId === sanitizeUiClientId(trimmed)
|
||||
);
|
||||
}
|
||||
|
||||
export interface BufferedBucket {
|
||||
clientId: string;
|
||||
date: string;
|
||||
count: number;
|
||||
}
|
||||
|
||||
export interface BufferedSummary {
|
||||
total: number;
|
||||
totalQueued: number;
|
||||
unattributedCount: number;
|
||||
truncated: boolean;
|
||||
buckets: BufferedBucket[];
|
||||
}
|
||||
|
||||
export interface FilteredBufferedMatch {
|
||||
matchingCount: number;
|
||||
unattributedCount: number;
|
||||
totalQueued: number;
|
||||
hasPotentialBuffered: boolean;
|
||||
}
|
||||
|
||||
export function countMatchingBufferedEvents(
|
||||
summary: BufferedSummary | null | undefined,
|
||||
filter: { clientId?: string; fromDate?: string; toDate?: string } = {},
|
||||
fallbackGlobalBufferCount = 0,
|
||||
): FilteredBufferedMatch {
|
||||
if (!summary) {
|
||||
const safeFallback = Math.max(0, Number(fallbackGlobalBufferCount) || 0);
|
||||
return {
|
||||
matchingCount: 0,
|
||||
unattributedCount: safeFallback,
|
||||
totalQueued: safeFallback,
|
||||
hasPotentialBuffered: safeFallback > 0,
|
||||
};
|
||||
}
|
||||
|
||||
const selectedClient = filter.clientId || 'all';
|
||||
const fromDate = filter.fromDate || '';
|
||||
const toDate = filter.toDate || '';
|
||||
const buckets = Array.isArray(summary.buckets) ? summary.buckets : [];
|
||||
|
||||
let matchingCount = 0;
|
||||
for (const bucket of buckets) {
|
||||
if (!bucket || typeof bucket.count !== 'number' || bucket.count <= 0) continue;
|
||||
if (!matchesSelectedClientFolder(bucket.clientId, selectedClient)) continue;
|
||||
if (fromDate && bucket.date < fromDate) continue;
|
||||
if (toDate && bucket.date > toDate) continue;
|
||||
matchingCount += bucket.count;
|
||||
}
|
||||
|
||||
const unattributedCount = Math.max(0, Number(summary.unattributedCount) || 0);
|
||||
const totalQueued = Math.max(
|
||||
matchingCount + unattributedCount,
|
||||
Number(summary.totalQueued) || 0,
|
||||
);
|
||||
|
||||
return {
|
||||
matchingCount,
|
||||
unattributedCount,
|
||||
totalQueued,
|
||||
hasPotentialBuffered: matchingCount > 0 || unattributedCount > 0,
|
||||
};
|
||||
}
|
||||
|
||||
export interface HttpBatchSummary {
|
||||
accepted: number;
|
||||
rejected: number;
|
||||
duplicates: number;
|
||||
status: 'Accepted' | 'Rejected';
|
||||
level: 'success' | 'warning' | 'error';
|
||||
title: string;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export function summarizeHttpBatchResult(
|
||||
resOk: boolean,
|
||||
data: Record<string, unknown> | null | undefined,
|
||||
sentCount: number,
|
||||
): HttpBatchSummary {
|
||||
const safeSent = Math.max(0, Math.trunc(sentCount));
|
||||
const rawAccepted =
|
||||
typeof data?.accepted === 'number' && Number.isFinite(data.accepted)
|
||||
? Math.max(0, Math.trunc(data.accepted))
|
||||
: typeof data?.processed === 'number' && Number.isFinite(data.processed)
|
||||
? Math.max(0, Math.trunc(data.processed))
|
||||
: null;
|
||||
const rawDuplicates =
|
||||
typeof data?.duplicates === 'number' && Number.isFinite(data.duplicates)
|
||||
? Math.max(0, Math.trunc(data.duplicates))
|
||||
: 0;
|
||||
const rawRejected =
|
||||
typeof data?.rejected === 'number' && Number.isFinite(data.rejected)
|
||||
? Math.max(0, Math.trunc(data.rejected))
|
||||
: null;
|
||||
|
||||
const serverError =
|
||||
typeof data?.error === 'string' && data.error.trim() ? data.error.trim() : '';
|
||||
|
||||
if (rawAccepted === null) {
|
||||
return {
|
||||
accepted: 0,
|
||||
rejected: rawRejected ?? safeSent,
|
||||
duplicates: rawDuplicates,
|
||||
status: 'Rejected',
|
||||
level: 'error',
|
||||
title: 'Batch Rejected',
|
||||
message:
|
||||
serverError ||
|
||||
`Server response did not confirm batch ingestion (0/${safeSent} ingested).`,
|
||||
};
|
||||
}
|
||||
|
||||
const accepted = rawAccepted;
|
||||
const duplicates = rawDuplicates;
|
||||
const rejected =
|
||||
rawRejected !== null
|
||||
? rawRejected
|
||||
: Math.max(0, safeSent - accepted - duplicates);
|
||||
|
||||
if (!resOk || (accepted === 0 && duplicates === 0)) {
|
||||
return {
|
||||
accepted,
|
||||
rejected,
|
||||
duplicates,
|
||||
status: 'Rejected',
|
||||
level: 'error',
|
||||
title: 'Batch Rejected',
|
||||
message: serverError || `${accepted}/${safeSent} ingested`,
|
||||
};
|
||||
}
|
||||
|
||||
if (rejected > 0 || accepted + duplicates < safeSent) {
|
||||
const detail =
|
||||
duplicates > 0
|
||||
? `${accepted}/${safeSent} ingested (${rejected} rejected, ${duplicates} duplicate)`
|
||||
: `${accepted}/${safeSent} ingested (${rejected} rejected)`;
|
||||
return {
|
||||
accepted,
|
||||
rejected,
|
||||
duplicates,
|
||||
status: 'Rejected',
|
||||
level: 'warning',
|
||||
title: 'Batch Partial',
|
||||
message: detail,
|
||||
};
|
||||
}
|
||||
|
||||
const message =
|
||||
duplicates > 0
|
||||
? `${accepted}/${safeSent} ingested (${duplicates} duplicate skipped)`
|
||||
: `${accepted}/${safeSent} ingested`;
|
||||
|
||||
return {
|
||||
accepted,
|
||||
rejected,
|
||||
duplicates,
|
||||
status: 'Accepted',
|
||||
level: 'success',
|
||||
title: 'Batch Streamed',
|
||||
message,
|
||||
};
|
||||
}
|
||||
|
||||
export function extractContentDispositionFilename(
|
||||
headerValue: string | null,
|
||||
fallback: string,
|
||||
): string {
|
||||
if (!headerValue) return fallback;
|
||||
const quoted = headerValue.match(/filename="([^"]+)"/i);
|
||||
if (quoted && quoted[1]) return quoted[1];
|
||||
const bare = headerValue.match(/filename=([^;]+)/i);
|
||||
if (bare && bare[1]) return bare[1].trim();
|
||||
return fallback;
|
||||
}
|
||||
|
||||
export type NumericSettingKey = keyof typeof SETTINGS_LIMITS;
|
||||
export type NumericSettingDrafts = Record<NumericSettingKey, string>;
|
||||
export type SettingsValidationErrors = Partial<Record<NumericSettingKey, string>>;
|
||||
|
||||
export const NUMERIC_SETTING_KEYS: readonly NumericSettingKey[] = [
|
||||
'maxEventsPerSec',
|
||||
'maxPayloadSizeKb',
|
||||
'batchFlushThreshold',
|
||||
'batchFlushIntervalMs',
|
||||
'fileRotationMb',
|
||||
'autoPruneDays',
|
||||
] as const;
|
||||
|
||||
export function settingsToNumericDrafts(
|
||||
settings: EditableSettings,
|
||||
): NumericSettingDrafts {
|
||||
return {
|
||||
maxEventsPerSec: String(settings.maxEventsPerSec),
|
||||
maxPayloadSizeKb: String(settings.maxPayloadSizeKb),
|
||||
batchFlushThreshold: String(settings.batchFlushThreshold),
|
||||
batchFlushIntervalMs: String(settings.batchFlushIntervalMs),
|
||||
fileRotationMb: String(settings.fileRotationMb),
|
||||
autoPruneDays: String(settings.autoPruneDays),
|
||||
};
|
||||
}
|
||||
|
||||
export function validateSingleNumericSetting(
|
||||
key: NumericSettingKey,
|
||||
rawValue: string,
|
||||
): { valid: true; value: number } | { valid: false; error: string } {
|
||||
const trimmed = String(rawValue ?? '').trim();
|
||||
const { min, max } = SETTINGS_LIMITS[key];
|
||||
if (!trimmed) {
|
||||
return { valid: false, error: 'Value is required.' };
|
||||
}
|
||||
if (!/^-?\d+$/.test(trimmed)) {
|
||||
return { valid: false, error: 'Enter a whole number.' };
|
||||
}
|
||||
const parsed = Number(trimmed);
|
||||
if (!Number.isSafeInteger(parsed)) {
|
||||
return { valid: false, error: 'Enter a valid integer.' };
|
||||
}
|
||||
if (parsed < min || parsed > max) {
|
||||
return {
|
||||
valid: false,
|
||||
error: `Must be between ${min.toLocaleString()} and ${max.toLocaleString()}.`,
|
||||
};
|
||||
}
|
||||
return { valid: true, value: parsed };
|
||||
}
|
||||
|
||||
export function validateEditableSettingsDraft(
|
||||
drafts: NumericSettingDrafts,
|
||||
enableIdempotency: boolean,
|
||||
): {
|
||||
valid: boolean;
|
||||
values?: EditableSettings;
|
||||
errors: SettingsValidationErrors;
|
||||
firstInvalidKey?: NumericSettingKey;
|
||||
} {
|
||||
const errors: SettingsValidationErrors = {};
|
||||
const parsedValues: Partial<Record<NumericSettingKey, number>> = {};
|
||||
let firstInvalidKey: NumericSettingKey | undefined;
|
||||
|
||||
for (const key of NUMERIC_SETTING_KEYS) {
|
||||
const result = validateSingleNumericSetting(key, drafts[key]);
|
||||
if (!result.valid) {
|
||||
errors[key] = result.error;
|
||||
if (!firstInvalidKey) firstInvalidKey = key;
|
||||
} else {
|
||||
parsedValues[key] = result.value;
|
||||
}
|
||||
}
|
||||
|
||||
if (firstInvalidKey) {
|
||||
return { valid: false, errors, firstInvalidKey };
|
||||
}
|
||||
|
||||
return {
|
||||
valid: true,
|
||||
errors,
|
||||
values: {
|
||||
maxEventsPerSec: parsedValues.maxEventsPerSec!,
|
||||
maxPayloadSizeKb: parsedValues.maxPayloadSizeKb!,
|
||||
batchFlushThreshold: parsedValues.batchFlushThreshold!,
|
||||
batchFlushIntervalMs: parsedValues.batchFlushIntervalMs!,
|
||||
fileRotationMb: parsedValues.fileRotationMb!,
|
||||
autoPruneDays: parsedValues.autoPruneDays!,
|
||||
enableIdempotency: Boolean(enableIdempotency),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export interface BrowserLocationLike {
|
||||
protocol: string;
|
||||
hostname: string;
|
||||
host: string;
|
||||
port: string;
|
||||
origin?: string;
|
||||
}
|
||||
|
||||
export interface PublicEndpoints {
|
||||
apiUrl: string;
|
||||
serverUrl: string;
|
||||
webSocketUrl: string;
|
||||
}
|
||||
|
||||
export function resolvePublicEndpoints(
|
||||
loc: BrowserLocationLike,
|
||||
serverPort: number,
|
||||
): PublicEndpoints {
|
||||
const safePort =
|
||||
Number.isFinite(serverPort) && serverPort > 0 ? Math.trunc(serverPort) : 4000;
|
||||
const httpProtocol = loc.protocol === 'https:' ? 'https:' : 'http:';
|
||||
const wsProtocol = httpProtocol === 'https:' ? 'wss:' : 'ws:';
|
||||
const apiUrl = loc.origin || `${httpProtocol}//${loc.host}`;
|
||||
const isDirectNextDev = loc.port === '3000';
|
||||
const ingestionHost = isDirectNextDev
|
||||
? `${loc.hostname}:${safePort}`
|
||||
: loc.host;
|
||||
|
||||
return {
|
||||
apiUrl,
|
||||
serverUrl: `${httpProtocol}//${ingestionHost}`,
|
||||
webSocketUrl: `${wsProtocol}//${ingestionHost}`,
|
||||
};
|
||||
}
|
||||
|
||||
export function shouldRedirectProfileUnauthorized(
|
||||
status: number,
|
||||
body: Record<string, unknown> | null | undefined,
|
||||
): boolean {
|
||||
if (status !== 401) return false;
|
||||
return body?.code !== 'INVALID_PASSWORD';
|
||||
}
|
||||
|
||||
export function formatSessionExpiryLabel(value: string): string {
|
||||
if (!value) return 'Active';
|
||||
const parsed = parseEventTimestamp(value);
|
||||
if (!parsed) return 'Active';
|
||||
return parsed.toLocaleString([], {
|
||||
month: 'short',
|
||||
day: 'numeric',
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
hour12: false,
|
||||
});
|
||||
}
|
||||
+28
-28
@@ -36,6 +36,10 @@ function matchesRequestHost(candidateUrl: string, request: NextRequest) {
|
||||
}
|
||||
|
||||
export function isSameOriginRequest(request: NextRequest) {
|
||||
const secFetchSite = request.headers.get("sec-fetch-site")?.toLowerCase();
|
||||
if (secFetchSite === "cross-site") {
|
||||
return false;
|
||||
}
|
||||
const originHeader = request.headers.get("origin");
|
||||
if (originHeader) {
|
||||
return matchesRequestHost(originHeader, request);
|
||||
@@ -44,6 +48,9 @@ export function isSameOriginRequest(request: NextRequest) {
|
||||
if (refererHeader) {
|
||||
return matchesRequestHost(refererHeader, request);
|
||||
}
|
||||
if (secFetchSite === "same-origin") {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -61,34 +68,29 @@ export async function middleware(request: NextRequest) {
|
||||
);
|
||||
const isStateChangingApi =
|
||||
pathname.startsWith("/api/") &&
|
||||
!SAFE_METHODS.has(request.method) &&
|
||||
!SAFE_METHODS.has(request.method.toUpperCase()) &&
|
||||
!(pathname === "/api/events" && hasBearerAuth);
|
||||
|
||||
if (isSetupPage) return NextResponse.next();
|
||||
if (
|
||||
!isConsolePage &&
|
||||
!isAdminApi &&
|
||||
!isLoginPage &&
|
||||
!(pathname === "/api/events" && request.method === "POST")
|
||||
) {
|
||||
return NextResponse.next();
|
||||
}
|
||||
const requestHeaders = new Headers(request.headers);
|
||||
requestHeaders.set("x-datastack-pathname", pathname);
|
||||
const nextWithPathname = () =>
|
||||
NextResponse.next({
|
||||
request: {
|
||||
headers: requestHeaders,
|
||||
},
|
||||
});
|
||||
|
||||
const isHeadlessAdminClient =
|
||||
request.headers.get("x-datastack-admin-client") === "true" ||
|
||||
Boolean(request.headers.get("user-agent")?.startsWith("DataStack-CLI"));
|
||||
if (isSetupPage) return nextWithPathname();
|
||||
|
||||
if (isStateChangingApi && !isSameOriginRequest(request)) {
|
||||
if (!isHeadlessAdminClient) {
|
||||
return NextResponse.json(
|
||||
{ error: "Cross-origin request rejected." },
|
||||
{ status: 403 },
|
||||
);
|
||||
}
|
||||
return NextResponse.json(
|
||||
{ error: "Cross-origin request rejected." },
|
||||
{ status: 403 },
|
||||
);
|
||||
}
|
||||
|
||||
if (pathname === "/api/events" && request.method === "POST") {
|
||||
return NextResponse.next();
|
||||
if (!isConsolePage && !isAdminApi && !isLoginPage) {
|
||||
return nextWithPathname();
|
||||
}
|
||||
|
||||
const sessionSecret = getAdminSessionSecret();
|
||||
@@ -99,6 +101,10 @@ export async function middleware(request: NextRequest) {
|
||||
);
|
||||
}
|
||||
|
||||
if (isLoginPage) {
|
||||
return nextWithPathname();
|
||||
}
|
||||
|
||||
const token = request.cookies.get(SESSION_COOKIE)?.value;
|
||||
const authenticated = Boolean(
|
||||
sessionSecret &&
|
||||
@@ -106,13 +112,7 @@ export async function middleware(request: NextRequest) {
|
||||
(await verifyAdminSessionToken(token, sessionSecret)),
|
||||
);
|
||||
|
||||
if (isLoginPage) {
|
||||
if (authenticated)
|
||||
return NextResponse.redirect(new URL("/vcon/dashboard", request.url));
|
||||
return NextResponse.next();
|
||||
}
|
||||
|
||||
if (authenticated) return NextResponse.next();
|
||||
if (authenticated) return nextWithPathname();
|
||||
|
||||
if (isAdminApi) {
|
||||
return NextResponse.json(
|
||||
|
||||
@@ -2,9 +2,6 @@ import type {NextConfig} from 'next';
|
||||
|
||||
const nextConfig: NextConfig = {
|
||||
reactStrictMode: true,
|
||||
eslint: {
|
||||
ignoreDuringBuilds: true,
|
||||
},
|
||||
typescript: {
|
||||
ignoreBuildErrors: false,
|
||||
},
|
||||
|
||||
+16
-1
@@ -49,7 +49,22 @@ server {
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $forwarded_proto;
|
||||
proxy_read_timeout 86400;
|
||||
proxy_read_timeout 86400s;
|
||||
proxy_send_timeout 86400s;
|
||||
proxy_buffering off;
|
||||
}
|
||||
|
||||
# Streaming CSV/JSON export endpoint on port 3000 (disable response buffering)
|
||||
location = /api/export {
|
||||
proxy_pass http://nextjs_upstream;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-Host $forwarded_host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $forwarded_proto;
|
||||
proxy_buffering off;
|
||||
proxy_read_timeout 300s;
|
||||
}
|
||||
|
||||
# Next.js web application and admin panel to port 3000
|
||||
|
||||
Generated
+130
-5287
File diff suppressed because it is too large.
Load diff
+7
-4
@@ -1,18 +1,22 @@
|
||||
{
|
||||
"name": "ai-studio-applet",
|
||||
"name": "datastack",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
},
|
||||
"scripts": {
|
||||
"dev": "next dev -p 3000 -H 0.0.0.0",
|
||||
"dev:all": "node scripts/dev-all.js",
|
||||
"server": "node server/index.js",
|
||||
"build": "NODE_ENV=production next build",
|
||||
"start": "next start",
|
||||
"start": "node scripts/start-all.js",
|
||||
"start:web": "next start",
|
||||
"start:server": "node server/index.js",
|
||||
"lint": "eslint .",
|
||||
"clean": "next clean"
|
||||
},
|
||||
"dependencies": {
|
||||
"@google/genai": "^2.4.0",
|
||||
"@hookform/resolvers": "^5.2.1",
|
||||
"autoprefixer": "^10.4.21",
|
||||
"class-variance-authority": "^0.7.1",
|
||||
@@ -46,7 +50,6 @@
|
||||
"@types/react-dom": "^19",
|
||||
"eslint": "9.39.1",
|
||||
"eslint-config-next": "16.0.8",
|
||||
"firebase-tools": "^15.0.0",
|
||||
"tailwindcss": "4.1.11",
|
||||
"tw-animate-css": "^1.4.0",
|
||||
"typescript": "5.9.3"
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
// This is your Prisma schema file,
|
||||
// learn more about it in the docs: https://pris.ly/d/prisma-schema
|
||||
// REFERENCE SCHEMA ONLY — NOT EXECUTED BY PRISMA CLI AT RUNTIME.
|
||||
// Canonical PostgreSQL runtime DDL is owned by `core/Database/database.ts` (Port 3000 via `pg`)
|
||||
// and mirrored 1:1 in `server/index.js` (Port 4000 standalone container).
|
||||
// Keep all 5 models below strictly synchronized with `core/Database/database.ts` (verified by tests).
|
||||
|
||||
generator client {
|
||||
provider = "prisma-client-js"
|
||||
|
||||
+127
-2
@@ -1,4 +1,4 @@
|
||||
const CACHE_NAME = 'datastack-cache-v2';
|
||||
const CACHE_NAME = 'datastack-cache-v3';
|
||||
const STATIC_ASSETS = [
|
||||
'/',
|
||||
'/manifest.json',
|
||||
@@ -9,6 +9,92 @@ const STATIC_ASSETS = [
|
||||
'/apple-touch-icon.png',
|
||||
];
|
||||
|
||||
function buildOfflineShellResponse(pathname) {
|
||||
const safePath = String(pathname || '/').replace(/[<>&"']/g, '');
|
||||
const html = `<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>DataStack — Offline</title>
|
||||
<style>
|
||||
:root { color-scheme: dark light; }
|
||||
body {
|
||||
margin: 0;
|
||||
min-height: 100vh;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
padding: 16px;
|
||||
background: #0a0a0b;
|
||||
color: #f4f4f5;
|
||||
font-family: "Segoe UI", system-ui, -apple-system, sans-serif;
|
||||
}
|
||||
.panel {
|
||||
width: 100%;
|
||||
max-width: 360px;
|
||||
border: 1px solid #27272a;
|
||||
background: #111113;
|
||||
padding: 20px;
|
||||
}
|
||||
.brand {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
.mark {
|
||||
width: 10px;
|
||||
height: 10px;
|
||||
background: #10b981;
|
||||
display: inline-block;
|
||||
}
|
||||
h1 { font-size: 15px; margin: 0 0 8px; font-weight: 600; }
|
||||
p { font-size: 12.5px; line-height: 1.5; color: #a1a1aa; margin: 0 0 16px; }
|
||||
code { color: #e4e4e7; font-family: ui-monospace, SFMono-Regular, monospace; font-size: 11.5px; }
|
||||
.actions { display: flex; gap: 8px; }
|
||||
button, a {
|
||||
appearance: none;
|
||||
border: 1px solid #27272a;
|
||||
background: #18181b;
|
||||
color: #f4f4f5;
|
||||
padding: 7px 12px;
|
||||
font-size: 12px;
|
||||
text-decoration: none;
|
||||
cursor: pointer;
|
||||
}
|
||||
button.primary {
|
||||
border-color: #10b981;
|
||||
background: #10b981;
|
||||
color: #09090b;
|
||||
font-weight: 600;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main class="panel">
|
||||
<div class="brand"><span class="mark"></span>DataStack</div>
|
||||
<h1>Network connection unavailable</h1>
|
||||
<p>Unable to reach <code>${safePath}</code> while offline. Live ingestion metrics and authenticated admin routes require a connection to the DataStack server.</p>
|
||||
<div class="actions">
|
||||
<button type="button" class="primary" onclick="window.location.reload()">Retry</button>
|
||||
<a href="/">Overview</a>
|
||||
</div>
|
||||
</main>
|
||||
</body>
|
||||
</html>`;
|
||||
|
||||
return new Response(html, {
|
||||
status: 503,
|
||||
statusText: 'Service Unavailable',
|
||||
headers: {
|
||||
'Content-Type': 'text/html; charset=utf-8',
|
||||
'Cache-Control': 'no-store',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
self.addEventListener('install', (event) => {
|
||||
event.waitUntil(
|
||||
caches.open(CACHE_NAME).then((cache) => {
|
||||
@@ -29,6 +115,28 @@ self.addEventListener('activate', (event) => {
|
||||
self.clients.claim();
|
||||
});
|
||||
|
||||
self.addEventListener('message', (event) => {
|
||||
if (event.data && event.data.type === 'CLEAR_AUTH_CACHE') {
|
||||
event.waitUntil(
|
||||
caches.open(CACHE_NAME).then(async (cache) => {
|
||||
const requests = await cache.keys();
|
||||
await Promise.all(
|
||||
requests
|
||||
.filter((req) => {
|
||||
try {
|
||||
const reqUrl = new URL(req.url);
|
||||
return reqUrl.pathname === '/vcon' || reqUrl.pathname.startsWith('/vcon/');
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
})
|
||||
.map((req) => cache.delete(req))
|
||||
);
|
||||
}).catch(() => {})
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
self.addEventListener('fetch', (event) => {
|
||||
const { request } = event;
|
||||
if (request.method !== 'GET') return;
|
||||
@@ -50,10 +158,24 @@ self.addEventListener('fetch', (event) => {
|
||||
return;
|
||||
}
|
||||
|
||||
// Never cache authenticated /vcon/* responses, but provide an offline HTML shell on PWA launch / navigation failure
|
||||
if (url.pathname === '/vcon' || url.pathname.startsWith('/vcon/')) {
|
||||
if (request.mode !== 'navigate') {
|
||||
return;
|
||||
}
|
||||
event.respondWith(
|
||||
fetch(request).catch(() => buildOfflineShellResponse(url.pathname))
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
event.respondWith(
|
||||
fetch(request)
|
||||
.then((response) => {
|
||||
if (response && response.ok && response.type === 'basic') {
|
||||
const cacheControl = response.headers.get('Cache-Control') || '';
|
||||
const shouldSkipCache =
|
||||
cacheControl.includes('no-store') || cacheControl.includes('private');
|
||||
if (response && response.ok && response.type === 'basic' && !shouldSkipCache) {
|
||||
const clone = response.clone();
|
||||
caches.open(CACHE_NAME).then((cache) => cache.put(request, clone)).catch(() => {});
|
||||
}
|
||||
@@ -62,6 +184,9 @@ self.addEventListener('fetch', (event) => {
|
||||
.catch(async () => {
|
||||
const cached = await caches.match(request);
|
||||
if (cached) return cached;
|
||||
if (request.mode === 'navigate') {
|
||||
return buildOfflineShellResponse(url.pathname);
|
||||
}
|
||||
return new Response('Offline — network unavailable.', {
|
||||
status: 503,
|
||||
statusText: 'Service Unavailable',
|
||||
|
||||
+61
-16
@@ -5,44 +5,89 @@
|
||||
* concurrently with unified terminal logging and clean shutdown handling.
|
||||
*/
|
||||
|
||||
const path = require('node:path');
|
||||
const { spawn } = require('node:child_process');
|
||||
|
||||
const webPort = String(process.env.PORT || '3000');
|
||||
const nextBin = require.resolve('next/dist/bin/next');
|
||||
const serverEntry = path.resolve(__dirname, '..', 'server', 'index.js');
|
||||
const GRACEFUL_SHUTDOWN_TIMEOUT_MS = 7000;
|
||||
|
||||
console.log('\x1b[36m%s\x1b[0m', '=======================================================');
|
||||
console.log('\x1b[36m%s\x1b[0m', ' DataStack Unified Development Runner');
|
||||
console.log('\x1b[36m%s\x1b[0m', ' - Port 3000: Next.js 15 Web & Admin Console (/vcon/*)');
|
||||
console.log('\x1b[36m%s\x1b[0m', ` - Port ${webPort}: Next.js 15 Web & Admin Console (/vcon/*)`);
|
||||
console.log('\x1b[36m%s\x1b[0m', ' - Port 4000: Ingestion & Socket.IO Streaming Engine');
|
||||
console.log('\x1b[36m%s\x1b[0m', '=======================================================\n');
|
||||
|
||||
const nextProc = spawn('npm', ['run', 'dev'], {
|
||||
const nextProc = spawn(process.execPath, [nextBin, 'dev', '-p', webPort], {
|
||||
stdio: 'inherit',
|
||||
shell: true,
|
||||
shell: false,
|
||||
env: { ...process.env },
|
||||
});
|
||||
|
||||
const serverProc = spawn('npm', ['run', 'server'], {
|
||||
const serverProc = spawn(process.execPath, [serverEntry], {
|
||||
stdio: 'inherit',
|
||||
shell: true,
|
||||
shell: false,
|
||||
env: { ...process.env },
|
||||
});
|
||||
|
||||
function shutdown(signal) {
|
||||
console.log(`\n[DataStack] Received ${signal}. Gracefully stopping all dev services...`);
|
||||
if (!nextProc.killed) nextProc.kill(signal);
|
||||
if (!serverProc.killed) serverProc.kill(signal);
|
||||
process.exit(0);
|
||||
let shuttingDown = false;
|
||||
let exitedChildren = 0;
|
||||
let nextExited = false;
|
||||
let serverExited = false;
|
||||
let exitCodePending = 0;
|
||||
|
||||
function markChildExited() {
|
||||
exitedChildren += 1;
|
||||
if (shuttingDown && exitedChildren >= 2) {
|
||||
process.exit(exitCodePending);
|
||||
}
|
||||
}
|
||||
|
||||
process.on('SIGINT', () => shutdown('SIGINT'));
|
||||
process.on('SIGTERM', () => shutdown('SIGTERM'));
|
||||
function signalChild(child, exited, signal) {
|
||||
if (exited || child.killed) return;
|
||||
try {
|
||||
child.kill(signal);
|
||||
} catch {
|
||||
// Child already exited
|
||||
}
|
||||
}
|
||||
|
||||
function shutdown(signal, exitCode = 0) {
|
||||
if (shuttingDown) return;
|
||||
shuttingDown = true;
|
||||
exitCodePending = exitCode;
|
||||
console.log(`\n[DataStack] Received ${signal}. Gracefully stopping all dev services...`);
|
||||
signalChild(nextProc, nextExited, signal);
|
||||
signalChild(serverProc, serverExited, signal);
|
||||
if (exitedChildren >= 2) {
|
||||
process.exit(exitCodePending);
|
||||
return;
|
||||
}
|
||||
setTimeout(() => {
|
||||
signalChild(nextProc, nextExited, 'SIGKILL');
|
||||
signalChild(serverProc, serverExited, 'SIGKILL');
|
||||
process.exit(exitCodePending);
|
||||
}, GRACEFUL_SHUTDOWN_TIMEOUT_MS).unref();
|
||||
}
|
||||
|
||||
process.on('SIGINT', () => shutdown('SIGINT', 0));
|
||||
process.on('SIGTERM', () => shutdown('SIGTERM', 0));
|
||||
|
||||
nextProc.on('exit', (code) => {
|
||||
if (code && code !== 0) {
|
||||
console.error(`[DataStack] Next.js development server exited with code ${code}`);
|
||||
nextExited = true;
|
||||
markChildExited();
|
||||
if (!shuttingDown) {
|
||||
console.error(`[DataStack] Next.js development server exited with code ${code ?? 0}`);
|
||||
shutdown('SIGTERM', code || 1);
|
||||
}
|
||||
});
|
||||
|
||||
serverProc.on('exit', (code) => {
|
||||
if (code && code !== 0) {
|
||||
console.error(`[DataStack] Ingestion server exited with code ${code}`);
|
||||
serverExited = true;
|
||||
markChildExited();
|
||||
if (!shuttingDown) {
|
||||
console.error(`[DataStack] Ingestion server exited with code ${code ?? 0}`);
|
||||
shutdown('SIGTERM', code || 1);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,90 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* DataStack Unified Production Runner
|
||||
* Starts both the Next.js Web Console (Port 3000 / $PORT) and
|
||||
* the Express + Socket.IO Ingestion Server (Port 4000 / $PORT_SOCKET).
|
||||
*/
|
||||
|
||||
const path = require('node:path');
|
||||
const { spawn } = require('node:child_process');
|
||||
|
||||
const webPort = String(process.env.PORT || '3000');
|
||||
const nextBin = require.resolve('next/dist/bin/next');
|
||||
const serverEntry = path.resolve(__dirname, '..', 'server', 'index.js');
|
||||
const GRACEFUL_SHUTDOWN_TIMEOUT_MS = 7000;
|
||||
|
||||
const webProc = spawn(
|
||||
process.execPath,
|
||||
[nextBin, 'start', '-p', webPort, '-H', '0.0.0.0'],
|
||||
{
|
||||
stdio: 'inherit',
|
||||
shell: false,
|
||||
env: { ...process.env },
|
||||
},
|
||||
);
|
||||
|
||||
const serverProc = spawn(process.execPath, [serverEntry], {
|
||||
stdio: 'inherit',
|
||||
shell: false,
|
||||
env: { ...process.env },
|
||||
});
|
||||
|
||||
let shuttingDown = false;
|
||||
let exitedChildren = 0;
|
||||
let webExited = false;
|
||||
let serverExited = false;
|
||||
let exitCodePending = 0;
|
||||
|
||||
function markChildExited() {
|
||||
exitedChildren += 1;
|
||||
if (shuttingDown && exitedChildren >= 2) {
|
||||
process.exit(exitCodePending);
|
||||
}
|
||||
}
|
||||
|
||||
function signalChild(child, exited, signal) {
|
||||
if (exited || child.killed) return;
|
||||
try {
|
||||
child.kill(signal);
|
||||
} catch {
|
||||
// Child already exited
|
||||
}
|
||||
}
|
||||
|
||||
function shutdown(signal, exitCode = 0) {
|
||||
if (shuttingDown) return;
|
||||
shuttingDown = true;
|
||||
exitCodePending = exitCode;
|
||||
signalChild(webProc, webExited, signal);
|
||||
signalChild(serverProc, serverExited, signal);
|
||||
if (exitedChildren >= 2) {
|
||||
process.exit(exitCodePending);
|
||||
return;
|
||||
}
|
||||
setTimeout(() => {
|
||||
signalChild(webProc, webExited, 'SIGKILL');
|
||||
signalChild(serverProc, serverExited, 'SIGKILL');
|
||||
process.exit(exitCodePending);
|
||||
}, GRACEFUL_SHUTDOWN_TIMEOUT_MS).unref();
|
||||
}
|
||||
|
||||
process.on('SIGINT', () => shutdown('SIGINT', 0));
|
||||
process.on('SIGTERM', () => shutdown('SIGTERM', 0));
|
||||
|
||||
webProc.on('exit', (code) => {
|
||||
webExited = true;
|
||||
markChildExited();
|
||||
if (!shuttingDown) {
|
||||
console.error(`[DataStack] Next.js production server exited with code ${code ?? 0}`);
|
||||
shutdown('SIGTERM', code || 1);
|
||||
}
|
||||
});
|
||||
|
||||
serverProc.on('exit', (code) => {
|
||||
serverExited = true;
|
||||
markChildExited();
|
||||
if (!shuttingDown) {
|
||||
console.error(`[DataStack] Ingestion server exited with code ${code ?? 0}`);
|
||||
shutdown('SIGTERM', code || 1);
|
||||
}
|
||||
});
|
||||
@@ -20,6 +20,15 @@ def _ensure_canonical_datastack_on_path() -> None:
|
||||
if repo_canonical.is_dir() and str(repo_canonical) not in sys.path:
|
||||
sys.path.insert(0, str(repo_canonical))
|
||||
|
||||
try:
|
||||
import datastack # noqa: F401
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"datastack-sdk requires the canonical 'datastack>=2.2.0' package. "
|
||||
"Install it via `pip install ./sdk/python/datastack` or install "
|
||||
"`datastack>=2.2.0` from your package index."
|
||||
) from exc
|
||||
|
||||
|
||||
_ensure_canonical_datastack_on_path()
|
||||
|
||||
|
||||
+5
-3
@@ -26,9 +26,10 @@ setup(
|
||||
license="MIT",
|
||||
packages=find_packages(),
|
||||
install_requires=[
|
||||
"python-socketio[client]>=5.11.0",
|
||||
"requests>=2.31.0",
|
||||
"websocket-client>=1.7.0",
|
||||
"datastack>=2.2.0",
|
||||
"python-socketio[client]>=5.8.0",
|
||||
"requests>=2.28.0",
|
||||
"websocket-client>=1.5.0",
|
||||
],
|
||||
classifiers=[
|
||||
"Development Status :: 5 - Production/Stable",
|
||||
@@ -44,6 +45,7 @@ setup(
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"License :: OSI Approved :: MIT License",
|
||||
"Operating System :: OS Independent",
|
||||
],
|
||||
|
||||
@@ -13,12 +13,15 @@ Features:
|
||||
and lossless partial-batch rate-limit retry (`429 RATE_LIMIT_EXCEEDED`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
@@ -26,7 +29,7 @@ import urllib.request
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Optional, Union
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
|
||||
|
||||
try:
|
||||
import socketio # type: ignore
|
||||
@@ -74,6 +77,14 @@ def _make_deterministic_file_event_id(
|
||||
return f"file_{safe_stem}_{row_index}_{digest}"
|
||||
|
||||
|
||||
def _normalize_client_id(value: Optional[str]) -> str:
|
||||
"""Matches server SDKManager.normalizeClientId() normalization rules."""
|
||||
raw = str(value or "").strip().lower()
|
||||
if not raw:
|
||||
return ""
|
||||
return re.sub(r"[^a-z0-9_-]", "_", raw)
|
||||
|
||||
|
||||
def _decode_jwt_client_id(token: str) -> str:
|
||||
"""Safely extracts `clientId` or `sub` claim from an unverified JWT payload string."""
|
||||
try:
|
||||
@@ -136,16 +147,18 @@ class DataStackClient:
|
||||
|
||||
configured_client_id = (self.config.client_id or "").strip()
|
||||
token_client_id = _decode_jwt_client_id(self.token)
|
||||
norm_configured = _normalize_client_id(configured_client_id)
|
||||
norm_token = _normalize_client_id(token_client_id)
|
||||
if (
|
||||
configured_client_id
|
||||
and token_client_id
|
||||
and configured_client_id != token_client_id
|
||||
norm_configured
|
||||
and norm_token
|
||||
and norm_configured != norm_token
|
||||
):
|
||||
raise DataStackConfigError(
|
||||
f"Configured CLIENT_ID '{configured_client_id}' does not match "
|
||||
f"CLIENT_TOKEN subject '{token_client_id}'."
|
||||
)
|
||||
self.client_id: str = configured_client_id or token_client_id
|
||||
self.client_id: str = norm_token or norm_configured
|
||||
|
||||
self.is_connected = False
|
||||
self._lock = threading.RLock()
|
||||
@@ -186,6 +199,18 @@ class DataStackClient:
|
||||
self.connect()
|
||||
except DataStackAuthError:
|
||||
raise
|
||||
except DataStackConnectionError:
|
||||
if (
|
||||
not self.config.enable_http_fallback
|
||||
and not self.config.offline_spool_enabled
|
||||
):
|
||||
self.heartbeat.stop(
|
||||
reason="Initial connect failed with fallback and spool disabled"
|
||||
)
|
||||
raise
|
||||
logger.warning(
|
||||
"[DataStack] Initial connect entered fallback/recovery mode.",
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"[DataStack] Initial connect entered fallback/recovery mode: %s",
|
||||
@@ -202,6 +227,15 @@ class DataStackClient:
|
||||
self.connect()
|
||||
except DataStackAuthError:
|
||||
raise
|
||||
except DataStackConnectionError:
|
||||
if (
|
||||
not self.config.enable_http_fallback
|
||||
and not self.config.offline_spool_enabled
|
||||
):
|
||||
self.heartbeat.stop(
|
||||
reason="Context connect failed with fallback and spool disabled"
|
||||
)
|
||||
raise
|
||||
except Exception:
|
||||
pass
|
||||
return self
|
||||
@@ -295,13 +329,16 @@ class DataStackClient:
|
||||
)
|
||||
return False
|
||||
|
||||
self.heartbeat.stop(
|
||||
reason="WebSocket and HTTP unreachable with fallback and spool disabled"
|
||||
)
|
||||
raise DataStackConnectionError(
|
||||
f"Unable to connect to DataStack WebSocket ({self.server_url}) or HTTP API ({self.api_url})."
|
||||
)
|
||||
|
||||
def disconnect(self) -> None:
|
||||
"""Gracefully stops the Heartbeat Monitor and disconnects the WebSocket."""
|
||||
self.heartbeat.stop(reason="Client closed by application")
|
||||
self.heartbeat.stop(reason="Client closed by application", join_timeout=0.5)
|
||||
with self._lock:
|
||||
self.is_connected = False
|
||||
if self.sio is not None and self.sio.connected:
|
||||
@@ -347,9 +384,24 @@ class DataStackClient:
|
||||
return False
|
||||
|
||||
# Verify handshake & PostgreSQL client status via `ping_verify` while still serialized under RLock
|
||||
verified = self._verify_websocket_handshake(timeout=effective_timeout)
|
||||
try:
|
||||
verified = self._verify_websocket_handshake(timeout=effective_timeout)
|
||||
except Exception:
|
||||
self.is_connected = False
|
||||
if self.sio.connected:
|
||||
try:
|
||||
self.sio.disconnect()
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
|
||||
if not verified:
|
||||
self.is_connected = False
|
||||
if self.sio.connected:
|
||||
try:
|
||||
self.sio.disconnect()
|
||||
except Exception:
|
||||
pass
|
||||
return verified
|
||||
|
||||
def _verify_websocket_handshake(
|
||||
@@ -455,7 +507,7 @@ class DataStackClient:
|
||||
url: str,
|
||||
payload: Optional[Dict[str, Any]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
) -> tuple[int, Dict[str, Any]]:
|
||||
) -> Tuple[int, Dict[str, Any]]:
|
||||
req_timeout = timeout or self.config.request_timeout_sec
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.token}",
|
||||
@@ -869,7 +921,16 @@ class DataStackClient:
|
||||
is_envelope = "data" in safe_item and set(safe_item.keys()).issubset(
|
||||
envelope_keys
|
||||
)
|
||||
evt_data = safe_item["data"] if is_envelope else safe_item
|
||||
if is_envelope:
|
||||
evt_data = safe_item["data"]
|
||||
elif "data" in safe_item:
|
||||
evt_data = safe_item
|
||||
else:
|
||||
evt_data = {
|
||||
k: v
|
||||
for k, v in safe_item.items()
|
||||
if k not in ("eventId", "timestamp", "clientId")
|
||||
}
|
||||
evt_client_id = (
|
||||
str(safe_item.get("clientId") or "").strip() or self.client_id
|
||||
)
|
||||
@@ -993,14 +1054,18 @@ class DataStackClient:
|
||||
if file_name
|
||||
else {"batch": remaining_items}
|
||||
)
|
||||
sub_res = self._execute_with_retry_and_fallback(
|
||||
ws_event="batch_sync",
|
||||
ws_payload=remaining_items,
|
||||
http_payload=sub_http_payload,
|
||||
spool_items=remaining_items,
|
||||
timeout=timeout,
|
||||
allow_spool=allow_spool,
|
||||
)
|
||||
try:
|
||||
sub_res = self._execute_with_retry_and_fallback(
|
||||
ws_event="batch_sync",
|
||||
ws_payload=remaining_items,
|
||||
http_payload=sub_http_payload,
|
||||
spool_items=remaining_items,
|
||||
timeout=timeout,
|
||||
allow_spool=allow_spool,
|
||||
)
|
||||
except (DataStackRateLimitError, DataStackConnectionError):
|
||||
break
|
||||
|
||||
total_processed += int(sub_res.get("processed", 0))
|
||||
total_duplicates += int(sub_res.get("duplicates", 0))
|
||||
total_spooled += int(sub_res.get("spooledCount", 0))
|
||||
@@ -1088,6 +1153,20 @@ class DataStackClient:
|
||||
total_spooled = 0
|
||||
chunks_sent = 0
|
||||
last_transport = "WEBSOCKET"
|
||||
unprocessed_items: List[Dict[str, Any]] = []
|
||||
|
||||
def _flush_chunk(chunk_items: List[Dict[str, Any]]) -> None:
|
||||
nonlocal total_processed, total_rejected, total_duplicates, total_spooled, chunks_sent, last_transport
|
||||
res = self.send_batch(chunk_items, file_name=path.name)
|
||||
total_processed += int(res.get("processed", 0))
|
||||
total_rejected += int(res.get("rejected", 0))
|
||||
total_duplicates += int(res.get("duplicates", 0))
|
||||
total_spooled += int(res.get("spooledCount", 0))
|
||||
last_transport = str(res.get("transport", last_transport))
|
||||
chunk_unprocessed = res.get("unprocessedItems")
|
||||
if isinstance(chunk_unprocessed, list) and chunk_unprocessed:
|
||||
unprocessed_items.extend(chunk_unprocessed)
|
||||
chunks_sent += 1
|
||||
|
||||
buffer: List[Dict[str, Any]] = []
|
||||
for idx, record in enumerate(self._iter_file_records(path, ext), start=1):
|
||||
@@ -1111,35 +1190,44 @@ class DataStackClient:
|
||||
)
|
||||
evt_ts = existing_ts or _utc_now_iso()
|
||||
|
||||
if isinstance(enriched, dict):
|
||||
is_file_envelope = "data" in enriched and set(
|
||||
enriched.keys()
|
||||
).issubset({"eventId", "timestamp", "data", "clientId"})
|
||||
if is_file_envelope:
|
||||
file_data: Any = enriched["data"]
|
||||
elif "data" in enriched:
|
||||
file_data = enriched
|
||||
else:
|
||||
file_data = {
|
||||
k: v
|
||||
for k, v in enriched.items()
|
||||
if k not in ("eventId", "timestamp", "clientId")
|
||||
}
|
||||
else:
|
||||
file_data = enriched
|
||||
|
||||
file_entry: Dict[str, Any] = {
|
||||
"eventId": evt_id,
|
||||
"timestamp": evt_ts,
|
||||
"data": enriched,
|
||||
"data": file_data,
|
||||
}
|
||||
if self.client_id:
|
||||
file_entry["clientId"] = self.client_id
|
||||
buffer.append(file_entry)
|
||||
if len(buffer) >= effective_chunk:
|
||||
res = self.send_batch(buffer, file_name=path.name)
|
||||
total_processed += int(res.get("processed", 0))
|
||||
total_rejected += int(res.get("rejected", 0))
|
||||
total_duplicates += int(res.get("duplicates", 0))
|
||||
total_spooled += int(res.get("spooledCount", 0))
|
||||
last_transport = str(res.get("transport", last_transport))
|
||||
chunks_sent += 1
|
||||
_flush_chunk(buffer)
|
||||
buffer = []
|
||||
|
||||
if buffer:
|
||||
res = self.send_batch(buffer, file_name=path.name)
|
||||
total_processed += int(res.get("processed", 0))
|
||||
total_rejected += int(res.get("rejected", 0))
|
||||
total_duplicates += int(res.get("duplicates", 0))
|
||||
total_spooled += int(res.get("spooledCount", 0))
|
||||
last_transport = str(res.get("transport", last_transport))
|
||||
chunks_sent += 1
|
||||
_flush_chunk(buffer)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
summary: Dict[str, Any] = {
|
||||
"success": (
|
||||
total_rejected == 0
|
||||
or total_processed > 0
|
||||
or total_spooled > 0
|
||||
),
|
||||
"fileName": path.name,
|
||||
"fileType": ext.lstrip("."),
|
||||
"chunksSent": chunks_sent,
|
||||
@@ -1149,15 +1237,22 @@ class DataStackClient:
|
||||
"spooledCount": total_spooled,
|
||||
"transport": last_transport,
|
||||
}
|
||||
if unprocessed_items:
|
||||
summary["unprocessedItems"] = unprocessed_items
|
||||
return summary
|
||||
|
||||
@staticmethod
|
||||
def _iter_file_records(path: Path, ext: str) -> Iterable[Dict[str, Any]]:
|
||||
try:
|
||||
if ext == ".csv":
|
||||
with path.open("r", encoding="utf-8-sig", newline="") as fh:
|
||||
reader = csv.DictReader(fh)
|
||||
reader = csv.DictReader(fh, restkey="_extra")
|
||||
for row in reader:
|
||||
yield dict(row)
|
||||
clean_row = {
|
||||
("_extra" if k is None else str(k)): v
|
||||
for k, v in row.items()
|
||||
}
|
||||
yield clean_row
|
||||
elif ext in (".jsonl", ".ndjson"):
|
||||
with path.open("r", encoding="utf-8-sig") as fh:
|
||||
for line_no, line in enumerate(fh, start=1):
|
||||
|
||||
@@ -182,6 +182,22 @@ def load_config(
|
||||
return default
|
||||
return bool(val)
|
||||
|
||||
def to_float(val: Any, name: str) -> float:
|
||||
try:
|
||||
return float(val)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise DataStackConfigError(
|
||||
f"Invalid numeric configuration for '{name}': {val!r}"
|
||||
) from exc
|
||||
|
||||
def to_int(val: Any, name: str) -> int:
|
||||
try:
|
||||
return int(val)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise DataStackConfigError(
|
||||
f"Invalid integer configuration for '{name}': {val!r}"
|
||||
) from exc
|
||||
|
||||
server_url = str(
|
||||
pick(
|
||||
["SERVER_URL", "SOCKET_URL", "WS_URL"],
|
||||
@@ -233,52 +249,57 @@ def load_config(
|
||||
),
|
||||
heartbeat_interval_sec=max(
|
||||
1.0,
|
||||
float(
|
||||
to_float(
|
||||
pick(
|
||||
["HEARTBEAT_INTERVAL_SEC", "HEARTBEAT_INTERVAL"],
|
||||
["DATASTACK_HEARTBEAT_INTERVAL_SEC"],
|
||||
5.0,
|
||||
)
|
||||
),
|
||||
"HEARTBEAT_INTERVAL_SEC",
|
||||
),
|
||||
),
|
||||
request_timeout_sec=max(
|
||||
1.0,
|
||||
float(
|
||||
to_float(
|
||||
pick(
|
||||
["REQUEST_TIMEOUT_SEC", "TIMEOUT"],
|
||||
["DATASTACK_REQUEST_TIMEOUT_SEC"],
|
||||
8.0,
|
||||
)
|
||||
),
|
||||
"REQUEST_TIMEOUT_SEC",
|
||||
),
|
||||
),
|
||||
max_retries=max(
|
||||
0,
|
||||
int(
|
||||
to_int(
|
||||
pick(
|
||||
["MAX_RETRIES", "RETRIES"],
|
||||
["DATASTACK_MAX_RETRIES"],
|
||||
3,
|
||||
)
|
||||
),
|
||||
"MAX_RETRIES",
|
||||
),
|
||||
),
|
||||
retry_backoff_base_sec=max(
|
||||
0.1,
|
||||
float(
|
||||
to_float(
|
||||
pick(
|
||||
["RETRY_BACKOFF_BASE_SEC"],
|
||||
["DATASTACK_RETRY_BACKOFF_BASE_SEC"],
|
||||
0.5,
|
||||
)
|
||||
),
|
||||
"RETRY_BACKOFF_BASE_SEC",
|
||||
),
|
||||
),
|
||||
batch_chunk_size=max(
|
||||
1,
|
||||
int(
|
||||
to_int(
|
||||
pick(
|
||||
["BATCH_CHUNK_SIZE", "CHUNK_SIZE"],
|
||||
["DATASTACK_BATCH_CHUNK_SIZE"],
|
||||
250,
|
||||
)
|
||||
),
|
||||
"BATCH_CHUNK_SIZE",
|
||||
),
|
||||
),
|
||||
offline_spool_enabled=to_bool(
|
||||
@@ -298,12 +319,13 @@ def load_config(
|
||||
),
|
||||
max_spool_size_mb=max(
|
||||
1,
|
||||
int(
|
||||
to_int(
|
||||
pick(
|
||||
["MAX_SPOOL_SIZE_MB"],
|
||||
["DATASTACK_MAX_SPOOL_SIZE_MB"],
|
||||
50,
|
||||
)
|
||||
),
|
||||
"MAX_SPOOL_SIZE_MB",
|
||||
),
|
||||
),
|
||||
config_source_path=resolved_source,
|
||||
|
||||
@@ -68,7 +68,8 @@ class HeartbeatMonitor:
|
||||
Monitor into IDLE mode (`_wake_event.clear()`).
|
||||
"""
|
||||
with self._lock:
|
||||
self._closed = False
|
||||
if self._closed or self._stop_event.is_set():
|
||||
return
|
||||
self._state = ConnectionState.CONNECTED_IDLE
|
||||
self._recovery_attempts = 0
|
||||
self._last_error = None
|
||||
@@ -112,7 +113,8 @@ class HeartbeatMonitor:
|
||||
self._stop_event.clear()
|
||||
self._ensure_thread_started()
|
||||
|
||||
def stop(self, reason: Optional[str] = None) -> None:
|
||||
def stop(self, reason: Optional[str] = None, join_timeout: float = 0.0) -> None:
|
||||
thread_to_join: Optional[threading.Thread] = None
|
||||
with self._lock:
|
||||
self._closed = True
|
||||
self._state = ConnectionState.STOPPED
|
||||
@@ -120,6 +122,14 @@ class HeartbeatMonitor:
|
||||
self._last_error = reason
|
||||
self._stop_event.set()
|
||||
self._wake_event.set()
|
||||
thread_to_join = self._thread
|
||||
if (
|
||||
join_timeout > 0
|
||||
and thread_to_join is not None
|
||||
and thread_to_join.is_alive()
|
||||
and threading.current_thread() is not thread_to_join
|
||||
):
|
||||
thread_to_join.join(timeout=join_timeout)
|
||||
|
||||
def status_snapshot(self) -> Dict[str, Any]:
|
||||
with self._lock:
|
||||
@@ -181,10 +191,17 @@ class HeartbeatMonitor:
|
||||
try:
|
||||
recovered_to_idle = self._recovery_callback()
|
||||
if recovered_to_idle:
|
||||
self.set_idle_connected()
|
||||
logger.info(
|
||||
"[DataStack Heartbeat] WebSocket reconnected & verified. Returning to IDLE mode."
|
||||
)
|
||||
with self._lock:
|
||||
still_active = (
|
||||
not self._closed
|
||||
and not self._stop_event.is_set()
|
||||
and self._state != ConnectionState.STOPPED
|
||||
)
|
||||
if still_active:
|
||||
self.set_idle_connected()
|
||||
logger.info(
|
||||
"[DataStack Heartbeat] WebSocket reconnected & verified. Returning to IDLE mode."
|
||||
)
|
||||
except Exception as exc:
|
||||
with self._lock:
|
||||
self._last_error = str(exc)
|
||||
|
||||
Whitespace-only changes.
@@ -6,6 +6,7 @@ batches once the Heartbeat Monitor restores connectivity.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
@@ -13,6 +14,8 @@ from typing import Any, Dict, List, Optional
|
||||
|
||||
from .exceptions import DataStackOfflineSpoolError
|
||||
|
||||
logger = logging.getLogger("datastack.spool")
|
||||
|
||||
|
||||
class DiskSpooler:
|
||||
"""
|
||||
@@ -121,6 +124,7 @@ class DiskSpooler:
|
||||
if not self.spool_file.exists():
|
||||
return 0
|
||||
|
||||
tmp_file = self.spool_dir / "spool.jsonl.tmp"
|
||||
try:
|
||||
with self.spool_file.open("r", encoding="utf-8") as fh:
|
||||
all_lines = [ln for ln in fh if ln.strip()]
|
||||
@@ -130,13 +134,15 @@ class DiskSpooler:
|
||||
return len(all_lines)
|
||||
|
||||
remaining = all_lines[count:]
|
||||
tmp_file = self.spool_dir / "spool.jsonl.tmp"
|
||||
with tmp_file.open("w", encoding="utf-8") as fh:
|
||||
fh.writelines(remaining)
|
||||
fh.flush()
|
||||
os.fsync(fh.fileno())
|
||||
os.replace(str(tmp_file), str(self.spool_file))
|
||||
return count
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
tmp_file.unlink(missing_ok=True)
|
||||
logger.warning("[DataStack Spool] ack_batch failed: %s", exc)
|
||||
return 0
|
||||
|
||||
def ack_and_requeue_front(
|
||||
@@ -156,6 +162,7 @@ class DiskSpooler:
|
||||
if not self.spool_file.exists():
|
||||
return 0
|
||||
|
||||
tmp_file = self.spool_dir / "spool.jsonl.tmp"
|
||||
try:
|
||||
with self.spool_file.open("r", encoding="utf-8") as fh:
|
||||
all_lines = [ln for ln in fh if ln.strip()]
|
||||
@@ -173,13 +180,17 @@ class DiskSpooler:
|
||||
return consumed
|
||||
|
||||
self._ensure_dir()
|
||||
tmp_file = self.spool_dir / "spool.jsonl.tmp"
|
||||
with tmp_file.open("w", encoding="utf-8") as fh:
|
||||
fh.writelines(combined)
|
||||
fh.flush()
|
||||
os.fsync(fh.fileno())
|
||||
os.replace(str(tmp_file), str(self.spool_file))
|
||||
return max(0, consumed - len(head_lines))
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
tmp_file.unlink(missing_ok=True)
|
||||
logger.warning(
|
||||
"[DataStack Spool] ack_and_requeue_front failed: %s", exc
|
||||
)
|
||||
return 0
|
||||
|
||||
def pending_count(self) -> int:
|
||||
|
||||
@@ -33,6 +33,7 @@ classifiers = [
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Topic :: Software Development :: Libraries :: Python Modules",
|
||||
"Topic :: System :: Logging",
|
||||
"Topic :: System :: Monitoring"
|
||||
@@ -52,3 +53,6 @@ Issues = "https://github.com/anik-d-art/datastack/issues"
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["."]
|
||||
include = ["datastack*"]
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
datastack = ["py.typed"]
|
||||
@@ -24,6 +24,8 @@ setup(
|
||||
url="https://vibtools.top/",
|
||||
license="MIT",
|
||||
packages=find_packages(include=["datastack", "datastack.*"]),
|
||||
package_data={"datastack": ["py.typed"]},
|
||||
include_package_data=True,
|
||||
install_requires=[
|
||||
"python-socketio[client]>=5.8.0",
|
||||
"requests>=2.28.0",
|
||||
@@ -40,5 +42,6 @@ setup(
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
],
|
||||
)
|
||||
+841
-244
File diff suppressed because it is too large.
Load diff
+274
-29
@@ -1,8 +1,30 @@
|
||||
const crypto = require("node:crypto");
|
||||
const jwt = require("jsonwebtoken");
|
||||
|
||||
const IDEMPOTENCY_TTL_MS = 5 * 60 * 1000; // 5 minutes
|
||||
const IDEMPOTENCY_MAX_ENTRIES = 20000;
|
||||
const CLEANUP_INTERVAL_MS = 60 * 1000;
|
||||
const REDIS_IDEMPOTENCY_PREFIX = "datastack:idem:";
|
||||
const REDIS_RATE_LIMIT_PREFIX = "datastack:rl:";
|
||||
const REDIS_RATE_LIMIT_LUA_SCRIPT = [
|
||||
"-- DATASTACK_RATE_LIMIT_INCR",
|
||||
'local c = redis.call("INCR", KEYS[1])',
|
||||
'if c == 1 or redis.call("PTTL", KEYS[1]) < 0 then',
|
||||
' redis.call("PEXPIRE", KEYS[1], tonumber(ARGV[1]))',
|
||||
"end",
|
||||
"return c",
|
||||
].join("\n");
|
||||
|
||||
function generateFallbackEventId() {
|
||||
return `evt_${Date.now()}_${crypto.randomBytes(6).toString("hex")}`;
|
||||
}
|
||||
|
||||
function normalizeClientId(clientId) {
|
||||
return String(clientId || "")
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9_-]/g, "_");
|
||||
}
|
||||
|
||||
function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
const getRuntimeConfig =
|
||||
@@ -36,6 +58,11 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
? options.onBufferPush
|
||||
: () => undefined;
|
||||
|
||||
const getRedisClient =
|
||||
typeof options.getRedisClient === "function"
|
||||
? options.getRedisClient
|
||||
: () => null;
|
||||
|
||||
const idempotencyMap = new Map();
|
||||
const rateLimitMap = new Map();
|
||||
let lastCleanupAt = Date.now();
|
||||
@@ -63,6 +90,25 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
}
|
||||
}
|
||||
|
||||
function trackLocalRateWindow(clientId, now) {
|
||||
const rl = rateLimitMap.get(clientId);
|
||||
if (!rl || now - rl.start >= 1000) {
|
||||
rateLimitMap.set(clientId, { count: 1, start: now });
|
||||
} else {
|
||||
rl.count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
function supportsSharedSecurityRedis(redis) {
|
||||
return Boolean(
|
||||
redis &&
|
||||
typeof redis.get === "function" &&
|
||||
typeof redis.incr === "function" &&
|
||||
typeof redis.pexpire === "function" &&
|
||||
typeof redis.set === "function",
|
||||
);
|
||||
}
|
||||
|
||||
function getSecurityStats() {
|
||||
cleanupMaps(Date.now(), true);
|
||||
const config = getRuntimeConfig();
|
||||
@@ -73,7 +119,13 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
function validateSingleEvent(clientId, eventId, rawData, clientRateLimit) {
|
||||
function validateSingleEventLocal(
|
||||
clientId,
|
||||
eventId,
|
||||
rawData,
|
||||
clientRateLimit,
|
||||
explicitByteLength,
|
||||
) {
|
||||
const config = getRuntimeConfig();
|
||||
const now = Date.now();
|
||||
cleanupMaps(now);
|
||||
@@ -93,9 +145,19 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
}
|
||||
|
||||
// 2. Payload byte size check (before consuming rate-limit quota)
|
||||
const payloadStr =
|
||||
typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {});
|
||||
const byteLength = Buffer.byteLength(payloadStr, "utf8");
|
||||
let payloadStr = "";
|
||||
let byteLength = 0;
|
||||
if (
|
||||
typeof explicitByteLength === "number" &&
|
||||
Number.isFinite(explicitByteLength) &&
|
||||
explicitByteLength >= 0
|
||||
) {
|
||||
byteLength = explicitByteLength;
|
||||
} else {
|
||||
payloadStr =
|
||||
typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {});
|
||||
byteLength = Buffer.byteLength(payloadStr, "utf8");
|
||||
}
|
||||
const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024;
|
||||
if (byteLength > maxBytes) {
|
||||
return {
|
||||
@@ -133,6 +195,126 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
return { allowed: true, payloadStr };
|
||||
}
|
||||
|
||||
async function validateSingleEventAsync(
|
||||
clientId,
|
||||
eventId,
|
||||
rawData,
|
||||
clientRateLimit,
|
||||
) {
|
||||
const redis = getRedisClient();
|
||||
if (!supportsSharedSecurityRedis(redis)) {
|
||||
return validateSingleEventLocal(
|
||||
clientId,
|
||||
eventId,
|
||||
rawData,
|
||||
clientRateLimit,
|
||||
);
|
||||
}
|
||||
|
||||
const config = getRuntimeConfig();
|
||||
const now = Date.now();
|
||||
cleanupMaps(now);
|
||||
const idempotencyKey = `${clientId}:${eventId}`;
|
||||
const redisIdemKey = `${REDIS_IDEMPOTENCY_PREFIX}${clientId}:${eventId}`;
|
||||
|
||||
try {
|
||||
// 1. Read-only idempotency check
|
||||
if (config.enableIdempotency && eventId) {
|
||||
const localTime = idempotencyMap.get(idempotencyKey);
|
||||
if (localTime && now - localTime < IDEMPOTENCY_TTL_MS) {
|
||||
return {
|
||||
allowed: false,
|
||||
duplicate: true,
|
||||
code: "DUPLICATE_EVENT",
|
||||
reason: "Duplicate event dropped",
|
||||
};
|
||||
}
|
||||
const existing = await redis.get(redisIdemKey);
|
||||
if (existing !== null) {
|
||||
idempotencyMap.set(idempotencyKey, now);
|
||||
return {
|
||||
allowed: false,
|
||||
duplicate: true,
|
||||
code: "DUPLICATE_EVENT",
|
||||
reason: "Duplicate event dropped",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Payload byte size check
|
||||
const payloadStr =
|
||||
typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {});
|
||||
const byteLength = Buffer.byteLength(payloadStr, "utf8");
|
||||
const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024;
|
||||
if (byteLength > maxBytes) {
|
||||
return {
|
||||
allowed: false,
|
||||
duplicate: false,
|
||||
code: "PAYLOAD_TOO_LARGE",
|
||||
reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`,
|
||||
};
|
||||
}
|
||||
|
||||
// 3. Shared atomic Redis rate limit check (single Lua round-trip)
|
||||
const effectiveLimit = Math.min(
|
||||
Number(clientRateLimit) || 100,
|
||||
Number(config.maxEventsPerSec) || 100,
|
||||
);
|
||||
const windowBucket = Math.floor(now / 1000);
|
||||
const rlKey = `${REDIS_RATE_LIMIT_PREFIX}${clientId}:${windowBucket}`;
|
||||
let count;
|
||||
if (typeof redis.eval === "function") {
|
||||
count = Number(
|
||||
await redis.eval(REDIS_RATE_LIMIT_LUA_SCRIPT, 1, rlKey, "2000"),
|
||||
);
|
||||
} else {
|
||||
count = Number(await redis.incr(rlKey));
|
||||
if (count === 1) {
|
||||
await redis.pexpire(rlKey, 2000);
|
||||
}
|
||||
}
|
||||
trackLocalRateWindow(clientId, now);
|
||||
|
||||
if (count > effectiveLimit) {
|
||||
return {
|
||||
allowed: false,
|
||||
duplicate: false,
|
||||
code: "RATE_LIMIT_EXCEEDED",
|
||||
reason: `RATE_LIMIT_EXCEEDED: Limit of ${effectiveLimit} events/sec exceeded`,
|
||||
};
|
||||
}
|
||||
|
||||
// 4. Atomic claim of idempotency key after all checks succeed
|
||||
if (config.enableIdempotency && eventId) {
|
||||
const claimed = await redis.set(
|
||||
redisIdemKey,
|
||||
"1",
|
||||
"PX",
|
||||
IDEMPOTENCY_TTL_MS,
|
||||
"NX",
|
||||
);
|
||||
idempotencyMap.set(idempotencyKey, now);
|
||||
if (claimed !== "OK") {
|
||||
return {
|
||||
allowed: false,
|
||||
duplicate: true,
|
||||
code: "DUPLICATE_EVENT",
|
||||
reason: "Duplicate event dropped",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return { allowed: true, payloadStr };
|
||||
} catch {
|
||||
return validateSingleEventLocal(
|
||||
clientId,
|
||||
eventId,
|
||||
rawData,
|
||||
clientRateLimit,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Middleware: Handshake Auth
|
||||
io.use(async (socket, next) => {
|
||||
const authHeader =
|
||||
@@ -157,9 +339,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
}
|
||||
const decoded = jwt.verify(cleanToken, config.jwtSecret);
|
||||
|
||||
const normalizedClientId = String(decoded.clientId || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const normalizedClientId = normalizeClientId(decoded.clientId);
|
||||
const dbCheck = await verifyClientInDb(
|
||||
normalizedClientId,
|
||||
cleanToken,
|
||||
@@ -181,18 +361,37 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
|
||||
const ENVELOPE_KEYS = new Set(["eventId", "timestamp", "data", "clientId"]);
|
||||
function extractBatchItemData(safeItem) {
|
||||
return extractSingleEventData(safeItem);
|
||||
}
|
||||
|
||||
function extractSingleEventData(rawPayload) {
|
||||
if (
|
||||
safeItem &&
|
||||
typeof safeItem === "object" &&
|
||||
!Array.isArray(safeItem) &&
|
||||
safeItem.data !== undefined
|
||||
rawPayload === null ||
|
||||
rawPayload === undefined ||
|
||||
typeof rawPayload !== "object" ||
|
||||
Array.isArray(rawPayload)
|
||||
) {
|
||||
const keys = Object.keys(safeItem);
|
||||
if (keys.every((k) => ENVELOPE_KEYS.has(k))) {
|
||||
return safeItem.data;
|
||||
}
|
||||
return rawPayload === undefined ? {} : rawPayload;
|
||||
}
|
||||
return safeItem;
|
||||
|
||||
if (rawPayload.data !== undefined) {
|
||||
const keys = Object.keys(rawPayload);
|
||||
if (keys.every((k) => ENVELOPE_KEYS.has(k))) {
|
||||
return rawPayload.data;
|
||||
}
|
||||
return rawPayload;
|
||||
}
|
||||
|
||||
const stripped = {};
|
||||
let hasNonEnvelopeKey = false;
|
||||
for (const [k, v] of Object.entries(rawPayload)) {
|
||||
if (k === "eventId" || k === "timestamp" || k === "clientId") {
|
||||
continue;
|
||||
}
|
||||
hasNonEnvelopeKey = true;
|
||||
stripped[k] = v;
|
||||
}
|
||||
return hasNonEnvelopeKey ? stripped : {};
|
||||
}
|
||||
|
||||
io.on("connection", (socket) => {
|
||||
@@ -221,16 +420,25 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
|
||||
const effectiveClientRate =
|
||||
dbCheck.rateLimit || socket.data.rateLimit || 100;
|
||||
const safePayload = payload && typeof payload === "object" ? payload : {};
|
||||
const safePayload =
|
||||
payload && typeof payload === "object" && !Array.isArray(payload)
|
||||
? payload
|
||||
: {};
|
||||
const eventId =
|
||||
safePayload.eventId ||
|
||||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||
const timestamp = safePayload.timestamp || new Date().toISOString();
|
||||
typeof safePayload.eventId === "string" && safePayload.eventId.trim()
|
||||
? safePayload.eventId.trim()
|
||||
: generateFallbackEventId();
|
||||
const timestamp =
|
||||
typeof safePayload.timestamp === "string" &&
|
||||
safePayload.timestamp.trim()
|
||||
? safePayload.timestamp.trim()
|
||||
: new Date().toISOString();
|
||||
const eventData = extractSingleEventData(payload);
|
||||
|
||||
const check = validateSingleEvent(
|
||||
const check = await validateSingleEventAsync(
|
||||
clientId,
|
||||
eventId,
|
||||
safePayload.data,
|
||||
eventData,
|
||||
effectiveClientRate,
|
||||
);
|
||||
if (!check.allowed) {
|
||||
@@ -260,7 +468,22 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
|
||||
bufferQueue.push(event);
|
||||
recordClientActivity(clientId, 1);
|
||||
onBufferPush();
|
||||
try {
|
||||
await onBufferPush();
|
||||
} catch (persistErr) {
|
||||
if (callback) {
|
||||
callback({
|
||||
success: false,
|
||||
eventId,
|
||||
code: "BUFFER_PERSIST_FAILED",
|
||||
error:
|
||||
persistErr instanceof Error
|
||||
? persistErr.message
|
||||
: "Failed to persist event to buffer",
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (callback) callback({ success: true, eventId });
|
||||
} catch (err) {
|
||||
@@ -309,12 +532,16 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
for (const item of batch) {
|
||||
const safeItem = item && typeof item === "object" ? item : {};
|
||||
const eventId =
|
||||
safeItem.eventId ||
|
||||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||
const timestamp = safeItem.timestamp || new Date().toISOString();
|
||||
typeof safeItem.eventId === "string" && safeItem.eventId.trim()
|
||||
? safeItem.eventId.trim()
|
||||
: generateFallbackEventId();
|
||||
const timestamp =
|
||||
typeof safeItem.timestamp === "string" && safeItem.timestamp.trim()
|
||||
? safeItem.timestamp.trim()
|
||||
: new Date().toISOString();
|
||||
const rawData = extractBatchItemData(safeItem);
|
||||
|
||||
const check = validateSingleEvent(
|
||||
const check = await validateSingleEventAsync(
|
||||
clientId,
|
||||
eventId,
|
||||
rawData,
|
||||
@@ -351,7 +578,24 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
|
||||
if (processed > 0) {
|
||||
recordClientActivity(clientId, processed);
|
||||
onBufferPush();
|
||||
try {
|
||||
await onBufferPush();
|
||||
} catch (persistErr) {
|
||||
if (callback) {
|
||||
callback({
|
||||
success: false,
|
||||
processed: 0,
|
||||
rejected: batch.length,
|
||||
duplicates,
|
||||
code: "BUFFER_PERSIST_FAILED",
|
||||
error:
|
||||
persistErr instanceof Error
|
||||
? persistErr.message
|
||||
: "Failed to persist batch to buffer",
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (processed === 0 && duplicates === 0 && rejected > 0) {
|
||||
@@ -443,7 +687,8 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
|
||||
return {
|
||||
getSecurityStats,
|
||||
validateSingleEventLocal,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { setupSocketHandlers };
|
||||
module.exports = { setupSocketHandlers, normalizeClientId };
|
||||
+267
-53
@@ -1,3 +1,4 @@
|
||||
const crypto = require('node:crypto');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { Parser } = require('json2csv');
|
||||
@@ -11,13 +12,21 @@ function sleepMs(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function resolveDefaultStorageDir() {
|
||||
const configured = (process.env.DATASTACK_STORAGE_DIR || '').trim();
|
||||
if (configured) {
|
||||
return path.resolve(configured);
|
||||
}
|
||||
return path.resolve(__dirname, '../../storage/data');
|
||||
}
|
||||
|
||||
/**
|
||||
* Background CSV Writer Worker
|
||||
* Flushes batches of real-time events to partitioned CSV files.
|
||||
* Path convention: /storage/data/{clientId}/{YYYY-MM-DD}.csv
|
||||
*/
|
||||
class CsvWriterWorker {
|
||||
constructor(baseDir = path.resolve(__dirname, '../../storage/data'), maxFileSize = 100 * 1024 * 1024) {
|
||||
constructor(baseDir = resolveDefaultStorageDir(), maxFileSize = 100 * 1024 * 1024) {
|
||||
this.baseDir = baseDir;
|
||||
this.maxFileSize = maxFileSize;
|
||||
this.ensureBaseDir();
|
||||
@@ -36,9 +45,85 @@ class CsvWriterWorker {
|
||||
}
|
||||
}
|
||||
|
||||
sanitizeClientId(clientId) {
|
||||
const raw = String(clientId || 'unknown').trim().replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
return raw || 'unknown';
|
||||
}
|
||||
|
||||
fileEndsWithNewline(filePath, sizeBytes) {
|
||||
if (sizeBytes <= 0) return false;
|
||||
const fd = fs.openSync(filePath, 'r');
|
||||
try {
|
||||
const lastByte = Buffer.allocUnsafe(1);
|
||||
const read = fs.readSync(fd, lastByte, 0, 1, sizeBytes - 1);
|
||||
return read === 1 && lastByte[0] === 0x0a;
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
}
|
||||
|
||||
removeStaleLockAtomically(lockPath) {
|
||||
try {
|
||||
const lockStat = fs.lstatSync(lockPath);
|
||||
if (Date.now() - lockStat.mtimeMs <= STALE_LOCK_AGE_MS) {
|
||||
return;
|
||||
}
|
||||
const staleTmp = `${lockPath}.stale.${crypto.randomUUID()}`;
|
||||
fs.renameSync(lockPath, staleTmp);
|
||||
try {
|
||||
fs.unlinkSync(staleTmp);
|
||||
} catch {
|
||||
// Ignore cleanup failure
|
||||
}
|
||||
} catch {
|
||||
// Lock was released or claimed concurrently
|
||||
}
|
||||
}
|
||||
|
||||
releaseLockIfOwned(lockPath, lockValue) {
|
||||
try {
|
||||
if (!fs.existsSync(lockPath)) return;
|
||||
const current = fs.readFileSync(lockPath, 'utf8');
|
||||
if (current === lockValue) {
|
||||
fs.unlinkSync(lockPath);
|
||||
}
|
||||
} catch {
|
||||
// Ignore if already removed
|
||||
}
|
||||
}
|
||||
|
||||
async removeStaleLockAtomicallyAsync(lockPath) {
|
||||
try {
|
||||
const lockStat = await fs.promises.lstat(lockPath);
|
||||
if (Date.now() - lockStat.mtimeMs <= STALE_LOCK_AGE_MS) {
|
||||
return;
|
||||
}
|
||||
const staleTmp = `${lockPath}.stale.${crypto.randomUUID()}`;
|
||||
await fs.promises.rename(lockPath, staleTmp);
|
||||
await fs.promises.unlink(staleTmp).catch(() => undefined);
|
||||
} catch {
|
||||
// Lock was released or claimed concurrently
|
||||
}
|
||||
}
|
||||
|
||||
async releaseLockIfOwnedAsync(lockPath, lockValue) {
|
||||
try {
|
||||
const current = await fs.promises.readFile(lockPath, 'utf8');
|
||||
if (current === lockValue) {
|
||||
await fs.promises.unlink(lockPath);
|
||||
}
|
||||
} catch {
|
||||
// Ignore if already removed
|
||||
}
|
||||
}
|
||||
|
||||
ensureClientDir(clientId) {
|
||||
const safeClientId = String(clientId || 'unknown').replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
const clientDir = path.join(this.baseDir, safeClientId);
|
||||
const safeClientId = this.sanitizeClientId(clientId);
|
||||
const resolvedBase = path.resolve(this.baseDir);
|
||||
const clientDir = path.resolve(resolvedBase, safeClientId);
|
||||
if (!clientDir.startsWith(resolvedBase + path.sep)) {
|
||||
throw new Error(`Invalid storage client directory: ${clientId}`);
|
||||
}
|
||||
if (!fs.existsSync(clientDir)) {
|
||||
fs.mkdirSync(clientDir, { recursive: true });
|
||||
}
|
||||
@@ -49,6 +134,7 @@ class CsvWriterWorker {
|
||||
const clientDir = this.ensureClientDir(clientId);
|
||||
const date = this.extractSafePartitionDate(dateStr);
|
||||
const lockPath = path.join(clientDir, `.${date}.csv.lock`);
|
||||
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
|
||||
const deadline = Date.now() + LOCK_TIMEOUT_MS;
|
||||
|
||||
let acquired = false;
|
||||
@@ -56,7 +142,7 @@ class CsvWriterWorker {
|
||||
try {
|
||||
const fd = fs.openSync(lockPath, 'wx');
|
||||
try {
|
||||
fs.writeFileSync(fd, `${process.pid}:${Date.now()}`, 'utf8');
|
||||
fs.writeFileSync(fd, lockValue, 'utf8');
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
@@ -66,15 +152,7 @@ class CsvWriterWorker {
|
||||
if (!err || err.code !== 'EEXIST') {
|
||||
throw err;
|
||||
}
|
||||
try {
|
||||
const lockStat = fs.statSync(lockPath);
|
||||
if (Date.now() - lockStat.mtimeMs > STALE_LOCK_AGE_MS) {
|
||||
fs.unlinkSync(lockPath);
|
||||
continue;
|
||||
}
|
||||
} catch {
|
||||
// Released concurrently
|
||||
}
|
||||
this.removeStaleLockAtomically(lockPath);
|
||||
const jitter =
|
||||
LOCK_RETRY_MIN_MS +
|
||||
Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS));
|
||||
@@ -89,13 +167,7 @@ class CsvWriterWorker {
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
try {
|
||||
if (fs.existsSync(lockPath)) {
|
||||
fs.unlinkSync(lockPath);
|
||||
}
|
||||
} catch {
|
||||
// Ignore if already removed
|
||||
}
|
||||
this.releaseLockIfOwned(lockPath, lockValue);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -127,7 +199,7 @@ class CsvWriterWorker {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
getTargetFilePath(clientId, dateStr) {
|
||||
getTargetFilePath(clientId, dateStr, incomingBytes = 0) {
|
||||
const clientDir = this.ensureClientDir(clientId);
|
||||
const date = this.extractSafePartitionDate(dateStr);
|
||||
let baseFileName = `${date}.csv`;
|
||||
@@ -135,16 +207,24 @@ class CsvWriterWorker {
|
||||
|
||||
let partIndex = 1;
|
||||
while (fs.existsSync(targetPath)) {
|
||||
let stats;
|
||||
try {
|
||||
const stats = fs.statSync(targetPath);
|
||||
if (stats.size >= this.maxFileSize) {
|
||||
partIndex += 1;
|
||||
baseFileName = `${date}-part${partIndex}.csv`;
|
||||
targetPath = path.join(clientDir, baseFileName);
|
||||
} else {
|
||||
stats = fs.statSync(targetPath);
|
||||
} catch (err) {
|
||||
if (err && err.code === 'ENOENT') {
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
throw err;
|
||||
}
|
||||
const overflowsLimit =
|
||||
stats.size > 0 &&
|
||||
incomingBytes > 0 &&
|
||||
stats.size + incomingBytes > this.maxFileSize;
|
||||
if (stats.size >= this.maxFileSize || overflowsLimit) {
|
||||
partIndex += 1;
|
||||
baseFileName = `${date}-part${partIndex}.csv`;
|
||||
targetPath = path.join(clientDir, baseFileName);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -158,40 +238,48 @@ class CsvWriterWorker {
|
||||
const grouped = new Map();
|
||||
for (const evt of events) {
|
||||
const date = this.extractSafePartitionDate(evt.timestamp);
|
||||
const key = `${evt.clientId}:::${date}`;
|
||||
if (!grouped.has(key)) grouped.set(key, []);
|
||||
grouped.get(key).push(evt);
|
||||
const clientId = String(evt.clientId || 'unknown');
|
||||
const key = JSON.stringify([clientId, date]);
|
||||
let bucket = grouped.get(key);
|
||||
if (!bucket) {
|
||||
bucket = { clientId, date, events: [] };
|
||||
grouped.set(key, bucket);
|
||||
}
|
||||
bucket.events.push(evt);
|
||||
}
|
||||
|
||||
const files = [];
|
||||
const fields = ['clientId', 'eventId', 'timestamp', 'payload'];
|
||||
|
||||
for (const [key, batch] of grouped.entries()) {
|
||||
const [clientId, date] = key.split(':::');
|
||||
for (const { clientId, date, events: batch } of grouped.values()) {
|
||||
const cleanBatch = batch.map((item) => ({
|
||||
clientId: item.clientId,
|
||||
eventId: item.eventId,
|
||||
timestamp: item.timestamp,
|
||||
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
|
||||
}));
|
||||
|
||||
const bodyCsvNoHeader = new Parser({ fields, header: false, eol: '\n' }).parse(cleanBatch);
|
||||
const incomingBytes = Buffer.byteLength(bodyCsvNoHeader, 'utf8') + 1;
|
||||
|
||||
const targetPath = await this.withPartitionLock(clientId, date, () => {
|
||||
const resolvedPath = this.getTargetFilePath(clientId, date);
|
||||
let hasContent = false;
|
||||
const resolvedPath = this.getTargetFilePath(clientId, date, incomingBytes);
|
||||
let sizeBytes = 0;
|
||||
try {
|
||||
hasContent =
|
||||
fs.existsSync(resolvedPath) && fs.statSync(resolvedPath).size > 0;
|
||||
if (fs.existsSync(resolvedPath)) {
|
||||
sizeBytes = fs.statSync(resolvedPath).size;
|
||||
}
|
||||
} catch {
|
||||
hasContent = false;
|
||||
sizeBytes = 0;
|
||||
}
|
||||
|
||||
const cleanBatch = batch.map((item) => ({
|
||||
clientId: item.clientId,
|
||||
eventId: item.eventId,
|
||||
timestamp: item.timestamp,
|
||||
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
|
||||
}));
|
||||
|
||||
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
|
||||
const csv = parser.parse(cleanBatch);
|
||||
const hasContent = sizeBytes > 0;
|
||||
|
||||
if (hasContent) {
|
||||
fs.appendFileSync(resolvedPath, '\n' + csv, 'utf8');
|
||||
const endsWithNl = this.fileEndsWithNewline(resolvedPath, sizeBytes);
|
||||
fs.appendFileSync(resolvedPath, (endsWithNl ? '' : '\n') + bodyCsvNoHeader, 'utf8');
|
||||
} else {
|
||||
fs.writeFileSync(resolvedPath, csv, 'utf8');
|
||||
const fullCsv = new Parser({ fields, header: true, eol: '\n' }).parse(cleanBatch);
|
||||
fs.writeFileSync(resolvedPath, fullCsv, 'utf8');
|
||||
}
|
||||
|
||||
return resolvedPath;
|
||||
@@ -218,18 +306,144 @@ class CsvWriterWorker {
|
||||
const clientDirs = fs.readdirSync(this.baseDir);
|
||||
for (const clientDir of clientDirs) {
|
||||
const fullClientPath = path.join(this.baseDir, clientDir);
|
||||
if (!fs.statSync(fullClientPath).isDirectory()) continue;
|
||||
let dirLstat;
|
||||
try {
|
||||
dirLstat = fs.lstatSync(fullClientPath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
|
||||
|
||||
const files = fs.readdirSync(fullClientPath);
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
|
||||
const fullPath = path.join(fullClientPath, file);
|
||||
let fileLstat;
|
||||
try {
|
||||
fs.unlinkSync(path.join(fullClientPath, file));
|
||||
fileLstat = fs.lstatSync(fullPath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (fileLstat.isSymbolicLink() || !fileLstat.isFile()) continue;
|
||||
|
||||
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
|
||||
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
|
||||
let acquired = false;
|
||||
try {
|
||||
const fd = fs.openSync(lockPath, 'wx');
|
||||
try {
|
||||
fs.writeFileSync(fd, lockValue, 'utf8');
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
acquired = true;
|
||||
} catch (err) {
|
||||
if (err && err.code === 'EEXIST') {
|
||||
this.removeStaleLockAtomically(lockPath);
|
||||
try {
|
||||
const fd = fs.openSync(lockPath, 'wx');
|
||||
try {
|
||||
fs.writeFileSync(fd, lockValue, 'utf8');
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
acquired = true;
|
||||
} catch {
|
||||
acquired = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!acquired) continue;
|
||||
|
||||
try {
|
||||
if (fs.existsSync(fullPath)) {
|
||||
fs.unlinkSync(fullPath);
|
||||
prunedCount += 1;
|
||||
}
|
||||
} catch {
|
||||
// Ignore individual file unlink failure
|
||||
} finally {
|
||||
this.releaseLockIfOwned(lockPath, lockValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore directory scan failure
|
||||
}
|
||||
return prunedCount;
|
||||
}
|
||||
|
||||
async pruneExpiredPartitionsAsync(autoPruneDays) {
|
||||
const days = Number(autoPruneDays);
|
||||
if (!Number.isFinite(days) || days <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const baseStat = await fs.promises.lstat(this.baseDir).catch(() => null);
|
||||
if (!baseStat || !baseStat.isDirectory()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const cutoffDate = new Date(Date.now() - Math.floor(days) * 86400000)
|
||||
.toISOString()
|
||||
.split('T')[0];
|
||||
|
||||
let prunedCount = 0;
|
||||
try {
|
||||
const clientDirs = await fs.promises.readdir(this.baseDir);
|
||||
for (const clientDir of clientDirs) {
|
||||
const fullClientPath = path.join(this.baseDir, clientDir);
|
||||
const dirLstat = await fs.promises.lstat(fullClientPath).catch(() => null);
|
||||
if (!dirLstat || dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
|
||||
|
||||
const files = await fs.promises.readdir(fullClientPath).catch(() => []);
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
|
||||
const fullPath = path.join(fullClientPath, file);
|
||||
const fileLstat = await fs.promises.lstat(fullPath).catch(() => null);
|
||||
if (!fileLstat || fileLstat.isSymbolicLink() || !fileLstat.isFile()) continue;
|
||||
|
||||
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
|
||||
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
|
||||
let acquired = false;
|
||||
const tryAcquireLock = async () => {
|
||||
const handle = await fs.promises.open(lockPath, 'wx');
|
||||
try {
|
||||
await handle.writeFile(lockValue, 'utf8');
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
try {
|
||||
acquired = await tryAcquireLock();
|
||||
} catch (err) {
|
||||
if (err && err.code === 'EEXIST') {
|
||||
await this.removeStaleLockAtomicallyAsync(lockPath);
|
||||
try {
|
||||
acquired = await tryAcquireLock();
|
||||
} catch {
|
||||
acquired = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!acquired) continue;
|
||||
|
||||
try {
|
||||
await fs.promises.unlink(fullPath);
|
||||
prunedCount += 1;
|
||||
} catch {
|
||||
// Ignore individual file unlink failure
|
||||
} finally {
|
||||
await this.releaseLockIfOwnedAsync(lockPath, lockValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -241,4 +455,4 @@ class CsvWriterWorker {
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { CsvWriterWorker };
|
||||
module.exports = { CsvWriterWorker, resolveDefaultStorageDir };
|
||||
+695
-14
@@ -1,29 +1,45 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { after, afterEach, beforeEach, describe, it } from "node:test";
|
||||
import { NextRequest } from "next/server";
|
||||
import { POST as loginPost } from "@/app/api/auth/login/route";
|
||||
import { POST as logoutPost } from "@/app/api/auth/logout/route";
|
||||
import { GET as profileGet, PUT as profilePut } from "@/app/api/auth/profile/route";
|
||||
import { POST as setupPost } from "@/app/api/auth/setup/route";
|
||||
import { GET as clientsGet, POST as clientsPost } from "@/app/api/clients/route";
|
||||
import {
|
||||
DELETE as clientDelete,
|
||||
GET as clientGet,
|
||||
PATCH as clientPatch,
|
||||
} from "@/app/api/clients/[id]/route";
|
||||
import { POST as eventsPost } from "@/app/api/events/route";
|
||||
import { GET as dataGet } from "@/app/api/data/route";
|
||||
import { GET as eventsGet, POST as eventsPost } from "@/app/api/events/route";
|
||||
import { GET as exportGet } from "@/app/api/export/route";
|
||||
import { GET as healthGet } from "@/app/api/health/route";
|
||||
import { GET as metricsGet } from "@/app/api/metrics/route";
|
||||
import { GET as storageGet, POST as storagePost } from "@/app/api/storage/route";
|
||||
import { SettingsManager } from "@/core/adminSettings/settingsManager";
|
||||
import {
|
||||
createAdminSessionToken,
|
||||
getAdminSessionPayload,
|
||||
getAdminSessionSecret,
|
||||
getSessionCookieOptions,
|
||||
} from "@/core/auth/adminSession";
|
||||
import {
|
||||
AdminDatabaseRequiredError,
|
||||
consumeAdminLoginAttempt,
|
||||
createInitialAdmin,
|
||||
getAdminSetupKey,
|
||||
isAdminSetupComplete,
|
||||
requiresPersistentAdminDatabase,
|
||||
resetAdminStateForTests,
|
||||
verifyAdminCredentials,
|
||||
verifyAdminSetupKey,
|
||||
verifyRequestAdminSession,
|
||||
} from "@/core/auth/adminUsers";
|
||||
import { AuthService, resolveDefaultJwtSecret } from "@/core/auth/jwt";
|
||||
import { DatabaseEngine } from "@/core/Database/database";
|
||||
import { SDKManager } from "@/core/sdk/sdkManager";
|
||||
import { extractClientIp } from "@/lib/adminAuth";
|
||||
import { isSameOriginRequest, middleware } from "@/middleware";
|
||||
|
||||
const ORIGINAL_ENV = { ...process.env };
|
||||
@@ -41,7 +57,14 @@ function restoreEnv() {
|
||||
describe("Phase-01 Auth, Secrets, Revocation & Proxy Hardening", () => {
|
||||
beforeEach(() => {
|
||||
restoreEnv();
|
||||
process.env.INGESTION_INTERNAL_URL = "http://127.0.0.1:1";
|
||||
resetAdminStateForTests();
|
||||
const g = globalThis as {
|
||||
__dataStackClientsMap?: Map<string, unknown>;
|
||||
__dataStackDeletedClients?: Set<string>;
|
||||
};
|
||||
g.__dataStackClientsMap?.clear();
|
||||
g.__dataStackDeletedClients?.clear();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
@@ -338,22 +361,18 @@ describe("Phase-01 Auth, Secrets, Revocation & Proxy Hardening", () => {
|
||||
assert.equal(isSameOriginRequest(reqMissingBoth), false);
|
||||
});
|
||||
|
||||
it("SPEC-08: allows headless scripts with x-datastack-admin-client or DataStack-CLI to bypass browser CSRF check", async () => {
|
||||
it("BUG-A02: rejects spoofable x-datastack-admin-client, DataStack-CLI, and cross-site Sec-Fetch-Site headers without same-origin proof", async () => {
|
||||
process.env.DATASTACK_ADMIN_SESSION_SECRET = "test-secret-at-least-32-chars-long-for-session";
|
||||
|
||||
// 1. Without headless header or origin -> 403 CSRF rejected
|
||||
// 1. Missing origin/referer -> 403
|
||||
const blockedReq = new NextRequest("http://localhost:3000/api/clients", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
host: "localhost:3000",
|
||||
},
|
||||
headers: { host: "localhost:3000" },
|
||||
});
|
||||
const blockedRes = await middleware(blockedReq);
|
||||
assert.equal(blockedRes.status, 403);
|
||||
const blockedJson = await blockedRes.json();
|
||||
assert.match(blockedJson.error, /cross-origin/i);
|
||||
|
||||
// 2. With x-datastack-admin-client header -> bypasses CSRF (reaches auth check, returns 401 Unauthorized if no cookie)
|
||||
// 2. Spoofed x-datastack-admin-client header must still be rejected 403
|
||||
const headlessReq = new NextRequest("http://localhost:3000/api/clients", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
@@ -362,19 +381,681 @@ describe("Phase-01 Auth, Secrets, Revocation & Proxy Hardening", () => {
|
||||
},
|
||||
});
|
||||
const headlessRes = await middleware(headlessReq);
|
||||
assert.equal(headlessRes.status, 401);
|
||||
const headlessJson = await headlessRes.json();
|
||||
assert.match(headlessJson.error, /authentication required/i);
|
||||
assert.equal(headlessRes.status, 403);
|
||||
|
||||
// 3. With DataStack-CLI User-Agent -> bypasses CSRF
|
||||
// 3. Spoofed DataStack-CLI User-Agent + cross-origin must still be rejected 403
|
||||
const cliReq = new NextRequest("http://localhost:3000/api/clients", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
host: "localhost:3000",
|
||||
origin: "https://evil.example.com",
|
||||
"user-agent": "DataStack-CLI/1.0",
|
||||
},
|
||||
});
|
||||
const cliRes = await middleware(cliReq);
|
||||
assert.equal(cliRes.status, 401);
|
||||
assert.equal(cliRes.status, 403);
|
||||
|
||||
// 4. Genuine same-origin passes CSRF and reaches auth check (401 without cookie)
|
||||
const sameOriginReq = new NextRequest("http://localhost:3000/api/clients", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
host: "localhost:3000",
|
||||
origin: "http://localhost:3000",
|
||||
},
|
||||
});
|
||||
const sameOriginRes = await middleware(sameOriginReq);
|
||||
assert.equal(sameOriginRes.status, 401);
|
||||
});
|
||||
|
||||
it("BUG-A01: enforces tokenVersion revocation across /api/data, GET /api/events, /api/export, /api/metrics, and /api/storage", async () => {
|
||||
delete process.env.DATABASE_URL;
|
||||
process.env.DATASTACK_ALLOW_DEMO_ADMIN = "false";
|
||||
await createInitialAdmin("owner@datastack.dev", "InitialPass123!");
|
||||
const secret = getAdminSessionSecret()!;
|
||||
|
||||
const v1Token = await createAdminSessionToken(
|
||||
"owner@datastack.dev",
|
||||
secret,
|
||||
Date.now() + 3600_000,
|
||||
1,
|
||||
);
|
||||
|
||||
const updateRes = await profilePut(
|
||||
new NextRequest("http://localhost:3000/api/auth/profile", {
|
||||
method: "PUT",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
cookie: `datastack_admin_session=${v1Token}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
email: "owner@datastack.dev",
|
||||
currentPassword: "InitialPass123!",
|
||||
newPassword: "RotatedPass456!",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(updateRes.status, 200);
|
||||
const v2Cookie = updateRes.cookies.get("datastack_admin_session")?.value;
|
||||
assert.ok(v2Cookie);
|
||||
|
||||
const endpoints: Array<() => Promise<Response>> = [
|
||||
() => dataGet(new NextRequest("http://localhost:3000/api/data", { headers: { cookie: `datastack_admin_session=${v1Token}` } })),
|
||||
() => eventsGet(new NextRequest("http://localhost:3000/api/events", { headers: { cookie: `datastack_admin_session=${v1Token}` } })),
|
||||
() => exportGet(new NextRequest("http://localhost:3000/api/export?flush=false", { headers: { cookie: `datastack_admin_session=${v1Token}` } })),
|
||||
() => metricsGet(new NextRequest("http://localhost:3000/api/metrics", { headers: { cookie: `datastack_admin_session=${v1Token}` } })),
|
||||
() => storageGet(new NextRequest("http://localhost:3000/api/storage", { headers: { cookie: `datastack_admin_session=${v1Token}` } })),
|
||||
() => storagePost(new NextRequest("http://localhost:3000/api/storage", { method: "POST", headers: { cookie: `datastack_admin_session=${v1Token}` } })),
|
||||
];
|
||||
|
||||
for (const invoke of endpoints) {
|
||||
const res = await invoke();
|
||||
assert.equal(res.status, 401);
|
||||
}
|
||||
|
||||
const okMetrics = await metricsGet(
|
||||
new NextRequest("http://localhost:3000/api/metrics", {
|
||||
headers: { cookie: `datastack_admin_session=${v2Cookie}` },
|
||||
}),
|
||||
);
|
||||
assert.equal(okMetrics.status, 200);
|
||||
});
|
||||
|
||||
it("BUG-A04 & BUG-A05: rejects unregistered clients on /api/health & admin POST /api/events and validates PATCH /api/clients/[id]", async () => {
|
||||
delete process.env.DATABASE_URL;
|
||||
await createInitialAdmin("admin@datastack.dev", "AdminPass123!");
|
||||
const secret = getAdminSessionSecret()!;
|
||||
const adminCookie = await createAdminSessionToken(
|
||||
"admin@datastack.dev",
|
||||
secret,
|
||||
Date.now() + 3600_000,
|
||||
1,
|
||||
);
|
||||
|
||||
// Public unauthenticated /api/health probe stays 200
|
||||
const publicHealth = await healthGet(new NextRequest("http://localhost:3000/api/health"));
|
||||
assert.equal(publicHealth.status, 200);
|
||||
|
||||
// Unregistered signed client Bearer token on /api/health -> 403 CLIENT_REVOKED
|
||||
const ghostToken = AuthService.generateClientToken({ clientId: "ghost_health_client" });
|
||||
const ghostHealth = await healthGet(
|
||||
new NextRequest("http://localhost:3000/api/health", {
|
||||
headers: { authorization: `Bearer ${ghostToken}` },
|
||||
}),
|
||||
);
|
||||
assert.equal(ghostHealth.status, 403);
|
||||
|
||||
// Admin cookie POST /api/events for unregistered clientId -> 403 CLIENT_REVOKED
|
||||
const ghostAdminPost = await eventsPost(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
cookie: `datastack_admin_session=${adminCookie}`,
|
||||
},
|
||||
body: JSON.stringify({ clientId: "unregistered_admin_target", data: { ok: 1 } }),
|
||||
}),
|
||||
);
|
||||
assert.equal(ghostAdminPost.status, 403);
|
||||
|
||||
// Create normalized client and verify mixed-case/unvalidated PATCH checks
|
||||
const createdRes = await clientsPost(
|
||||
new NextRequest("http://localhost:3000/api/clients", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
cookie: `datastack_admin_session=${adminCookie}`,
|
||||
},
|
||||
body: JSON.stringify({ clientId: "Norm_Client_01", name: "Normalized", rateLimit: 50 }),
|
||||
}),
|
||||
);
|
||||
assert.equal(createdRes.status, 201);
|
||||
|
||||
// Invalid non-string name -> 400 (not 500)
|
||||
const badNameRes = await clientPatch(
|
||||
new NextRequest("http://localhost:3000/api/clients/Norm_Client_01", {
|
||||
method: "PATCH",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
cookie: `datastack_admin_session=${adminCookie}`,
|
||||
},
|
||||
body: JSON.stringify({ name: 12345 }),
|
||||
}),
|
||||
{ params: Promise.resolve({ id: "Norm_Client_01" }) },
|
||||
);
|
||||
assert.equal(badNameRes.status, 400);
|
||||
|
||||
// Invalid status -> 400
|
||||
const badStatusRes = await clientPatch(
|
||||
new NextRequest("http://localhost:3000/api/clients/Norm_Client_01", {
|
||||
method: "PATCH",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
cookie: `datastack_admin_session=${adminCookie}`,
|
||||
},
|
||||
body: JSON.stringify({ status: "suspended" }),
|
||||
}),
|
||||
{ params: Promise.resolve({ id: "Norm_Client_01" }) },
|
||||
);
|
||||
assert.equal(badStatusRes.status, 400);
|
||||
|
||||
// Valid mixed-case normalized PATCH -> 200
|
||||
const goodPatchRes = await clientPatch(
|
||||
new NextRequest("http://localhost:3000/api/clients/NORM_CLIENT_01", {
|
||||
method: "PATCH",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
cookie: `datastack_admin_session=${adminCookie}`,
|
||||
},
|
||||
body: JSON.stringify({ name: "Updated Clean Name", rateLimit: 250 }),
|
||||
}),
|
||||
{ params: Promise.resolve({ id: "NORM_CLIENT_01" }) },
|
||||
);
|
||||
assert.equal(goodPatchRes.status, 200);
|
||||
const patchedJson = await goodPatchRes.json();
|
||||
assert.equal(patchedJson.client.clientId, "norm_client_01");
|
||||
assert.equal(patchedJson.client.rateLimit, 250);
|
||||
});
|
||||
|
||||
it("FORENSIC BUG-01: blocks volatile RAM admin setup in production without DATABASE_URL and returns 409 SETUP_REQUIRED on pre-setup login", async () => {
|
||||
process.env.NODE_ENV = "production";
|
||||
delete process.env.DATABASE_URL;
|
||||
delete process.env.DATASTACK_ADMIN_PASSWORD;
|
||||
process.env.DATASTACK_ADMIN_SESSION_SECRET =
|
||||
"prod-session-secret-at-least-32-characters-long";
|
||||
process.env.DATASTACK_ADMIN_SETUP_KEY =
|
||||
"prod-setup-key-at-least-32-characters-long";
|
||||
|
||||
assert.equal(requiresPersistentAdminDatabase(), true);
|
||||
await assert.rejects(
|
||||
() => createInitialAdmin("owner@datastack.dev", "StrongPass123!"),
|
||||
AdminDatabaseRequiredError,
|
||||
);
|
||||
|
||||
const setupRes = await setupPost(
|
||||
new NextRequest("http://localhost:3000/api/auth/setup", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
host: "localhost:3000",
|
||||
origin: "http://localhost:3000",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
email: "owner@datastack.dev",
|
||||
password: "StrongPass123!",
|
||||
setupKey: process.env.DATASTACK_ADMIN_SETUP_KEY,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(setupRes.status, 503);
|
||||
const setupJson = await setupRes.json();
|
||||
assert.equal(setupJson.code, "DATABASE_NOT_CONFIGURED");
|
||||
|
||||
const loginRes = await loginPost(
|
||||
new NextRequest("http://localhost:3000/api/auth/login", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
host: "localhost:3000",
|
||||
origin: "http://localhost:3000",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
email: "owner@datastack.dev",
|
||||
password: "StrongPass123!",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(loginRes.status, 409);
|
||||
const loginJson = await loginRes.json();
|
||||
assert.equal(loginJson.code, "SETUP_REQUIRED");
|
||||
});
|
||||
|
||||
it("FORENSIC BUG-04: throttles POST /api/auth/setup brute-force attempts and checks setup completion before verifying setupKey", async () => {
|
||||
delete process.env.DATABASE_URL;
|
||||
process.env.DATASTACK_ALLOW_DEMO_ADMIN = "false";
|
||||
process.env.DATASTACK_ADMIN_SETUP_KEY =
|
||||
"custom-setup-key-at-least-32-characters-long";
|
||||
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const badRes = await setupPost(
|
||||
new NextRequest("http://localhost:3000/api/auth/setup", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"x-forwarded-for": "198.51.100.77",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
email: "admin@datastack.dev",
|
||||
password: "StrongPass123!",
|
||||
setupKey: "wrong-key-attempt",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(badRes.status, 403);
|
||||
}
|
||||
|
||||
const throttledRes = await setupPost(
|
||||
new NextRequest("http://localhost:3000/api/auth/setup", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"x-forwarded-for": "198.51.100.77",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
email: "admin@datastack.dev",
|
||||
password: "StrongPass123!",
|
||||
setupKey: process.env.DATASTACK_ADMIN_SETUP_KEY,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(throttledRes.status, 429);
|
||||
assert.ok(Number(throttledRes.headers.get("retry-after")) >= 1);
|
||||
|
||||
// Different IP succeeds and initializes admin
|
||||
const okRes = await setupPost(
|
||||
new NextRequest("http://localhost:3000/api/auth/setup", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"x-forwarded-for": "203.0.113.88",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
email: "admin@datastack.dev",
|
||||
password: "StrongPass123!",
|
||||
setupKey: process.env.DATASTACK_ADMIN_SETUP_KEY,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(okRes.status, 200);
|
||||
|
||||
// Subsequent setup attempt returns 409 even if setupKey is invalid
|
||||
const postInitRes = await setupPost(
|
||||
new NextRequest("http://localhost:3000/api/auth/setup", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"x-forwarded-for": "203.0.113.89",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
email: "other@datastack.dev",
|
||||
password: "StrongPass123!",
|
||||
setupKey: "completely-invalid-key",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(postInitRes.status, 409);
|
||||
});
|
||||
|
||||
it("FORENSIC BUG-05: revokes session cookie server-side on logout, ignores spoofed Origin for Secure flag, and allows privacy-stripped same-origin requests", async () => {
|
||||
delete process.env.DATABASE_URL;
|
||||
process.env.DATASTACK_ALLOW_DEMO_ADMIN = "false";
|
||||
await createInitialAdmin("logout-test@datastack.dev", "LogoutPass123!");
|
||||
const secret = getAdminSessionSecret()!;
|
||||
const sessionToken = await createAdminSessionToken(
|
||||
"logout-test@datastack.dev",
|
||||
secret,
|
||||
Date.now() + 3600_000,
|
||||
1,
|
||||
);
|
||||
|
||||
// Valid before logout
|
||||
const preLogoutRes = await profileGet(
|
||||
new NextRequest("http://localhost:3000/api/auth/profile", {
|
||||
headers: { cookie: `datastack_admin_session=${sessionToken}` },
|
||||
}),
|
||||
);
|
||||
assert.equal(preLogoutRes.status, 200);
|
||||
|
||||
// Logout bumps tokenVersion server-side
|
||||
const logoutRes = await logoutPost(
|
||||
new NextRequest("http://localhost:3000/api/auth/logout", {
|
||||
method: "POST",
|
||||
headers: { cookie: `datastack_admin_session=${sessionToken}` },
|
||||
}),
|
||||
);
|
||||
assert.equal(logoutRes.status, 200);
|
||||
|
||||
// Replaying captured pre-logout cookie must return 401
|
||||
const replayProfile = await profileGet(
|
||||
new NextRequest("http://localhost:3000/api/auth/profile", {
|
||||
headers: { cookie: `datastack_admin_session=${sessionToken}` },
|
||||
}),
|
||||
);
|
||||
assert.equal(replayProfile.status, 401);
|
||||
|
||||
// Cookie Secure flag must ignore spoofed https Origin on plain http request
|
||||
const spoofedOriginReq = new Request("http://localhost:3000/api/auth/login", {
|
||||
method: "POST",
|
||||
headers: { origin: "https://evil.example.com" },
|
||||
});
|
||||
const plainOpts = getSessionCookieOptions(spoofedOriginReq, 3600);
|
||||
assert.equal(plainOpts.secure, false);
|
||||
assert.equal(plainOpts.sameSite, "lax");
|
||||
|
||||
const httpsProxyReq = new Request("http://localhost:3000/api/auth/login", {
|
||||
method: "POST",
|
||||
headers: { "x-forwarded-proto": "https" },
|
||||
});
|
||||
const httpsOpts = getSessionCookieOptions(httpsProxyReq, 3600);
|
||||
assert.equal(httpsOpts.secure, true);
|
||||
assert.equal(httpsOpts.sameSite, "none");
|
||||
|
||||
// Privacy extension stripping Origin & Referer with Sec-Fetch-Site: same-origin passes CSRF
|
||||
const privacySameOriginReq = new NextRequest("http://localhost:3000/api/clients", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
host: "localhost:3000",
|
||||
"sec-fetch-site": "same-origin",
|
||||
},
|
||||
});
|
||||
assert.equal(isSameOriginRequest(privacySameOriginReq), true);
|
||||
|
||||
const crossSiteReq = new NextRequest("http://localhost:3000/api/clients", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
host: "localhost:3000",
|
||||
"sec-fetch-site": "cross-site",
|
||||
},
|
||||
});
|
||||
assert.equal(isSameOriginRequest(crossSiteReq), false);
|
||||
});
|
||||
|
||||
it("FORENSIC BUG-06: prevents empty production jwtSecret PG persistence and propagates DB write errors instead of silently swallowing them", async () => {
|
||||
const origGetPool = DatabaseEngine.getPool;
|
||||
const origEnsureSchema = DatabaseEngine.ensureSchema;
|
||||
const settingsGlobals = globalThis as {
|
||||
__dataStackAdminSettings?: unknown;
|
||||
__dataStackSettingsHydrated?: boolean;
|
||||
};
|
||||
const queries: string[] = [];
|
||||
|
||||
try {
|
||||
// 1. Production with empty jwtSecret must not persist empty jwt_secret row into PG
|
||||
process.env.NODE_ENV = "production";
|
||||
delete process.env.DATASTACK_JWT_SECRET;
|
||||
settingsGlobals.__dataStackAdminSettings = undefined;
|
||||
settingsGlobals.__dataStackSettingsHydrated = false;
|
||||
|
||||
DatabaseEngine.getPool = (() =>
|
||||
({
|
||||
query: async (sql: string) => {
|
||||
queries.push(sql);
|
||||
return { rowCount: 0, rows: [] };
|
||||
},
|
||||
}) as unknown as ReturnType<typeof DatabaseEngine.getPool>);
|
||||
DatabaseEngine.ensureSchema = async () => true;
|
||||
|
||||
await SettingsManager.getSettingsAsync();
|
||||
assert.equal(
|
||||
queries.some((sql) => sql.includes("INSERT INTO admin_settings")),
|
||||
false,
|
||||
);
|
||||
|
||||
// 2. Failing PG write must reject (not silently return 200) and must not mutate RAM
|
||||
process.env.NODE_ENV = "test";
|
||||
process.env.DATASTACK_JWT_SECRET =
|
||||
"valid-jwt-secret-at-least-32-characters-long";
|
||||
AuthService.setSecret(process.env.DATASTACK_JWT_SECRET);
|
||||
settingsGlobals.__dataStackAdminSettings = undefined;
|
||||
settingsGlobals.__dataStackSettingsHydrated = false;
|
||||
|
||||
DatabaseEngine.getPool = (() =>
|
||||
({
|
||||
query: async (sql: string) => {
|
||||
if (sql.includes("INSERT INTO") || sql.includes("UPDATE")) {
|
||||
throw new Error("simulated PG outage");
|
||||
}
|
||||
return { rowCount: 0, rows: [] };
|
||||
},
|
||||
}) as unknown as ReturnType<typeof DatabaseEngine.getPool>);
|
||||
|
||||
await assert.rejects(
|
||||
() => SettingsManager.updateSettingsAsync({ maxEventsPerSec: 777 }),
|
||||
/DATABASE_WRITE_FAILED/,
|
||||
);
|
||||
|
||||
await assert.rejects(
|
||||
() =>
|
||||
SDKManager.registerClientAsync({
|
||||
clientId: "db_fail_client",
|
||||
name: "DB Fail Client",
|
||||
}),
|
||||
/DATABASE_WRITE_FAILED/,
|
||||
);
|
||||
assert.equal(SDKManager.getClient("db_fail_client"), undefined);
|
||||
} finally {
|
||||
DatabaseEngine.getPool = origGetPool;
|
||||
DatabaseEngine.ensureSchema = origEnsureSchema;
|
||||
settingsGlobals.__dataStackAdminSettings = undefined;
|
||||
settingsGlobals.__dataStackSettingsHydrated = false;
|
||||
}
|
||||
});
|
||||
|
||||
it("R2 PHASE-01 BUG-01: enforces same-origin CSRF check on POST /api/auth/login and POST /api/auth/setup while preserving same-origin and Bearer /api/events", async () => {
|
||||
process.env.DATASTACK_ADMIN_SESSION_SECRET =
|
||||
"test-secret-at-least-32-chars-long-for-session";
|
||||
|
||||
// Cross-origin POST /api/auth/login -> 403
|
||||
const crossLoginRes = await middleware(
|
||||
new NextRequest("http://localhost:3000/api/auth/login", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
host: "localhost:3000",
|
||||
origin: "https://evil.example.com",
|
||||
},
|
||||
}),
|
||||
);
|
||||
assert.equal(crossLoginRes.status, 403);
|
||||
assert.deepEqual(await crossLoginRes.json(), {
|
||||
error: "Cross-origin request rejected.",
|
||||
});
|
||||
|
||||
// Missing origin/referer/sec-fetch-site POST /api/auth/login -> 403
|
||||
const bareLoginRes = await middleware(
|
||||
new NextRequest("http://localhost:3000/api/auth/login", {
|
||||
method: "POST",
|
||||
headers: { host: "localhost:3000" },
|
||||
}),
|
||||
);
|
||||
assert.equal(bareLoginRes.status, 403);
|
||||
|
||||
// Cross-origin POST /api/auth/setup -> 403
|
||||
const crossSetupRes = await middleware(
|
||||
new NextRequest("http://localhost:3000/api/auth/setup", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
host: "localhost:3000",
|
||||
origin: "https://evil.example.com",
|
||||
},
|
||||
}),
|
||||
);
|
||||
assert.equal(crossSetupRes.status, 403);
|
||||
|
||||
// Missing origin POST /api/auth/setup -> 403
|
||||
const bareSetupRes = await middleware(
|
||||
new NextRequest("http://localhost:3000/api/auth/setup", {
|
||||
method: "POST",
|
||||
headers: { host: "localhost:3000" },
|
||||
}),
|
||||
);
|
||||
assert.equal(bareSetupRes.status, 403);
|
||||
|
||||
// Same-origin POST /api/auth/login passes middleware (200 next)
|
||||
const sameOriginLoginRes = await middleware(
|
||||
new NextRequest("http://localhost:3000/api/auth/login", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
host: "localhost:3000",
|
||||
origin: "http://localhost:3000",
|
||||
},
|
||||
}),
|
||||
);
|
||||
assert.equal(sameOriginLoginRes.status, 200);
|
||||
|
||||
// Privacy-stripped same-origin POST /api/auth/login via Sec-Fetch-Site passes middleware
|
||||
const privacyLoginRes = await middleware(
|
||||
new NextRequest("http://localhost:3000/api/auth/login", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
host: "localhost:3000",
|
||||
"sec-fetch-site": "same-origin",
|
||||
},
|
||||
}),
|
||||
);
|
||||
assert.equal(privacyLoginRes.status, 200);
|
||||
|
||||
// Same-origin POST /api/auth/setup passes middleware (200 next)
|
||||
const sameOriginSetupRes = await middleware(
|
||||
new NextRequest("http://localhost:3000/api/auth/setup", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
host: "localhost:3000",
|
||||
origin: "http://localhost:3000",
|
||||
},
|
||||
}),
|
||||
);
|
||||
assert.equal(sameOriginSetupRes.status, 200);
|
||||
|
||||
// Bearer-authenticated POST /api/events without Origin stays allowed (machine-to-machine SDK)
|
||||
const bearerEventsRes = await middleware(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
host: "localhost:3000",
|
||||
authorization: "Bearer sdk.jwt.token",
|
||||
},
|
||||
}),
|
||||
);
|
||||
assert.equal(bearerEventsRes.status, 200);
|
||||
|
||||
// Non-Bearer cross-origin POST /api/events is rejected 403
|
||||
const crossEventsRes = await middleware(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
host: "localhost:3000",
|
||||
origin: "https://evil.example.com",
|
||||
},
|
||||
}),
|
||||
);
|
||||
assert.equal(crossEventsRes.status, 403);
|
||||
|
||||
// Safe GET public API endpoints remain accessible without Origin
|
||||
const healthRes = await middleware(
|
||||
new NextRequest("http://localhost:3000/api/health", {
|
||||
method: "GET",
|
||||
headers: { host: "localhost:3000" },
|
||||
}),
|
||||
);
|
||||
assert.equal(healthRes.status, 200);
|
||||
});
|
||||
|
||||
it("R2 PHASE-01 BUG-02: extractClientIp trusts X-Real-IP and rightmost X-Forwarded-For hop, blocking spoofed-XFF lockouts on login & setup", async () => {
|
||||
// Direct unit assertions on extractClientIp
|
||||
assert.equal(
|
||||
extractClientIp(
|
||||
new NextRequest("http://localhost:3000/api/auth/login", {
|
||||
headers: {
|
||||
"x-forwarded-for": "1.2.3.4, 203.0.113.9",
|
||||
"x-real-ip": "203.0.113.9",
|
||||
},
|
||||
}),
|
||||
),
|
||||
"203.0.113.9",
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
extractClientIp(
|
||||
new NextRequest("http://localhost:3000/api/auth/login", {
|
||||
headers: {
|
||||
"x-forwarded-for": "1.2.3.4, 198.51.100.50, ",
|
||||
"x-real-ip": " ",
|
||||
},
|
||||
}),
|
||||
),
|
||||
"198.51.100.50",
|
||||
);
|
||||
|
||||
assert.equal(
|
||||
extractClientIp(new NextRequest("http://localhost:3000/api/auth/login")),
|
||||
"unknown",
|
||||
);
|
||||
|
||||
// End-to-end brute-force lockout on POST /api/auth/login despite rotating spoofed leftmost XFF
|
||||
delete process.env.DATABASE_URL;
|
||||
process.env.DATASTACK_ALLOW_DEMO_ADMIN = "false";
|
||||
process.env.DATASTACK_ADMIN_SETUP_KEY =
|
||||
"phase01-setup-key-at-least-32-characters-long";
|
||||
|
||||
// First verify setup route blocks rotating spoofed leftmost XFF
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const badSetup = await setupPost(
|
||||
new NextRequest("http://localhost:3000/api/auth/setup", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"x-forwarded-for": `10.1.0.${i + 1}, 198.51.100.42`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
email: "admin@datastack.dev",
|
||||
password: "StrongPass123!",
|
||||
setupKey: "wrong-key",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(badSetup.status, 403);
|
||||
}
|
||||
|
||||
const blockedSetup = await setupPost(
|
||||
new NextRequest("http://localhost:3000/api/auth/setup", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"x-forwarded-for": "10.1.0.99, 198.51.100.42",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
email: "admin@datastack.dev",
|
||||
password: "StrongPass123!",
|
||||
setupKey: process.env.DATASTACK_ADMIN_SETUP_KEY,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(blockedSetup.status, 429);
|
||||
|
||||
// Create admin and verify login route blocks rotating spoofed leftmost XFF with X-Real-IP
|
||||
await createInitialAdmin("lockout@datastack.dev", "RealPassword123!");
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const badLogin = await loginPost(
|
||||
new NextRequest("http://localhost:3000/api/auth/login", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"x-forwarded-for": `172.16.0.${i + 1}, 203.0.113.9`,
|
||||
"x-real-ip": "203.0.113.9",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
email: "lockout@datastack.dev",
|
||||
password: "WrongPassword!",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(badLogin.status, 401);
|
||||
}
|
||||
|
||||
const blockedLogin = await loginPost(
|
||||
new NextRequest("http://localhost:3000/api/auth/login", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
"x-forwarded-for": "172.16.0.99, 203.0.113.9",
|
||||
"x-real-ip": "203.0.113.9",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
email: "lockout@datastack.dev",
|
||||
password: "RealPassword123!",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(blockedLogin.status, 429);
|
||||
assert.ok(Number(blockedLogin.headers.get("retry-after")) >= 1);
|
||||
});
|
||||
});
|
||||
@@ -1,20 +1,32 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { createRequire } from "node:module";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import { resolve } from "node:path";
|
||||
import test, { after, before } from "node:test";
|
||||
import { NextRequest } from "next/server";
|
||||
import { io as ioClient, type Socket } from "socket.io-client";
|
||||
import {
|
||||
DELETE as deleteClientRoute,
|
||||
GET as getClientRoute,
|
||||
PATCH as patchClientRoute,
|
||||
} from "../app/api/clients/[id]/route";
|
||||
import { POST as postEventsRoute } from "../app/api/events/route";
|
||||
import { GET as getSettingsRoute, POST as postSettingsRoute } from "../app/api/settings/route";
|
||||
import { SettingsManager } from "../core/adminSettings/settingsManager";
|
||||
import { AuthService } from "../core/auth/jwt";
|
||||
import {
|
||||
createAdminSessionToken,
|
||||
getAdminSessionSecret,
|
||||
} from "../core/auth/adminSession";
|
||||
import {
|
||||
createInitialAdmin,
|
||||
isAdminDatabaseConfigured,
|
||||
resetAdminStateForTests,
|
||||
} from "../core/auth/adminUsers";
|
||||
import { ApiEngine } from "../core/apiEngine/apiService";
|
||||
import { DataBufferEngine, type RedisBufferClient } from "../core/data/buffer";
|
||||
import { DatabaseEngine } from "../core/Database/database";
|
||||
import { SDKManager } from "../core/sdk/sdkManager";
|
||||
import {
|
||||
SecurityEngine,
|
||||
@@ -37,6 +49,15 @@ interface IngestionServerModule {
|
||||
enableIdempotency: boolean;
|
||||
autoPruneDays: number;
|
||||
};
|
||||
verifyClientInDb: (
|
||||
clientId: string,
|
||||
token: string,
|
||||
bypass?: boolean,
|
||||
) => Promise<{ allowed: boolean; reason?: string; rateLimit?: number }>;
|
||||
recordClientActivity: (clientId: string, count?: number) => void;
|
||||
flushPendingClientActivity: () => Promise<number>;
|
||||
setRedisClientForTests: (client: unknown) => void;
|
||||
setDbPoolForTests: (pool: unknown) => void;
|
||||
stopTimers: () => void;
|
||||
}
|
||||
|
||||
@@ -48,6 +69,9 @@ const g = globalThis as typeof globalThis & {
|
||||
__dataStackClientsMap?: Map<string, unknown>;
|
||||
__dataStackDeletedClients?: Set<string>;
|
||||
__dataStackSyncedSocketEvents?: Map<string, number>;
|
||||
__dataStackPendingClientActivity?: Map<string, unknown>;
|
||||
__dataStackCorePool?: unknown;
|
||||
__dataStackCoreSchema?: Promise<void>;
|
||||
};
|
||||
|
||||
let serverMod: IngestionServerModule;
|
||||
@@ -151,6 +175,7 @@ after(async () => {
|
||||
clearInterval(g.__dataStackBufferState.flushTimer);
|
||||
g.__dataStackBufferState.flushTimer = null;
|
||||
}
|
||||
SDKManager.stopActivityFlushTimerForTests();
|
||||
|
||||
if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL;
|
||||
else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl;
|
||||
@@ -175,6 +200,7 @@ test("BUG-13: Port 4000 /internal/* rejects unauthenticated callers and accepts
|
||||
["GET", "/internal/recent", undefined],
|
||||
["POST", "/internal/client-sync", { clientId: "x", action: "registered" }],
|
||||
["POST", "/internal/settings-sync", { maxEventsPerSec: 120 }],
|
||||
["POST", "/internal/security-check", { clientId: "x", eventId: "auth_chk_1", payloadBytes: 12 }],
|
||||
] as const) {
|
||||
const noAuth = await fetch(`${ingestionUrl}${path}`, {
|
||||
method,
|
||||
@@ -373,3 +399,824 @@ test("BUG-08: Rate-limit error reports effectiveRateLimit and ApiEngine aggregat
|
||||
`Expected aggregated rateLimitedClientsCount >= 2, got ${metrics.securityStats.rateLimitedClientsCount}`,
|
||||
);
|
||||
});
|
||||
|
||||
function createSharedMockRedis(): RedisBufferClient {
|
||||
const kv = new Map<string, string>();
|
||||
const lists = new Map<string, string[]>();
|
||||
const hashes = new Map<string, Map<string, string>>();
|
||||
const sets = new Map<string, Set<string>>();
|
||||
|
||||
return {
|
||||
status: "ready",
|
||||
async get(key: string) {
|
||||
return kv.has(key) ? kv.get(key)! : null;
|
||||
},
|
||||
async incr(key: string) {
|
||||
const next = (Number(kv.get(key)) || 0) + 1;
|
||||
kv.set(key, String(next));
|
||||
return next;
|
||||
},
|
||||
async pexpire() {
|
||||
return 1;
|
||||
},
|
||||
async set(key: string, value: string, _px?: "PX", _ttlMs?: number, nxMode?: "NX") {
|
||||
if (nxMode === "NX" && kv.has(key)) return null;
|
||||
kv.set(key, value);
|
||||
return "OK";
|
||||
},
|
||||
async hget(key: string, field: string) {
|
||||
return hashes.get(key)?.get(field) ?? null;
|
||||
},
|
||||
async hset(key: string, field: string, value: string) {
|
||||
if (!hashes.has(key)) hashes.set(key, new Map());
|
||||
hashes.get(key)!.set(field, value);
|
||||
return 1;
|
||||
},
|
||||
async hdel(key: string, ...fields: string[]) {
|
||||
let deleted = 0;
|
||||
const h = hashes.get(key);
|
||||
if (!h) return 0;
|
||||
for (const f of fields) {
|
||||
if (h.delete(f)) deleted += 1;
|
||||
}
|
||||
return deleted;
|
||||
},
|
||||
async sismember(key: string, member: string) {
|
||||
return sets.get(key)?.has(member) ? 1 : 0;
|
||||
},
|
||||
async sadd(key: string, ...members: string[]) {
|
||||
if (!sets.has(key)) sets.set(key, new Set());
|
||||
for (const m of members) sets.get(key)!.add(m);
|
||||
return members.length;
|
||||
},
|
||||
async srem(key: string, ...members: string[]) {
|
||||
const s = sets.get(key);
|
||||
if (!s) return 0;
|
||||
let removed = 0;
|
||||
for (const m of members) {
|
||||
if (s.delete(m)) removed += 1;
|
||||
}
|
||||
return removed;
|
||||
},
|
||||
async rpush(key: string, ...values: string[]) {
|
||||
const list = lists.get(key) ?? [];
|
||||
list.push(...values);
|
||||
lists.set(key, list);
|
||||
return list.length;
|
||||
},
|
||||
async lpush(key: string, ...values: string[]) {
|
||||
const list = lists.get(key) ?? [];
|
||||
list.unshift(...values);
|
||||
lists.set(key, list);
|
||||
return list.length;
|
||||
},
|
||||
async ltrim(key: string, start: number, stop: number) {
|
||||
const list = lists.get(key) ?? [];
|
||||
const len = list.length;
|
||||
const end = stop < 0 ? len + stop : stop;
|
||||
lists.set(key, list.slice(start, end + 1));
|
||||
return "OK";
|
||||
},
|
||||
async lrange(key: string, start: number, stop: number) {
|
||||
const list = lists.get(key) ?? [];
|
||||
const len = list.length;
|
||||
const end = stop < 0 ? len + stop : stop;
|
||||
return list.slice(start, end + 1);
|
||||
},
|
||||
async llen(key: string) {
|
||||
return (lists.get(key) ?? []).length;
|
||||
},
|
||||
async eval(script: string, _numKeys: number, ...args: string[]) {
|
||||
if (script.includes("DATASTACK_RATE_LIMIT_INCR")) {
|
||||
const key = args[0];
|
||||
const next = (Number(kv.get(key)) || 0) + 1;
|
||||
kv.set(key, String(next));
|
||||
return next;
|
||||
}
|
||||
if (script.includes("DATASTACK_PURGE_CLIENT_LIST")) {
|
||||
const [key, target] = args;
|
||||
const list = lists.get(key) ?? [];
|
||||
const kept: string[] = [];
|
||||
let removed = 0;
|
||||
for (const raw of list) {
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as { clientId?: string };
|
||||
const cid = String(parsed?.clientId || "")
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9_-]/g, "_");
|
||||
if (cid === target) {
|
||||
removed += 1;
|
||||
continue;
|
||||
}
|
||||
} catch {
|
||||
// preserve unparseable entries
|
||||
}
|
||||
kept.push(raw);
|
||||
}
|
||||
if (removed > 0) lists.set(key, kept);
|
||||
return removed;
|
||||
}
|
||||
return 1;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test("BUG-B01: Port 4000 verifyClientInDb fails closed for unregistered/empty clients even without DB", async () => {
|
||||
const emptyCheck = await serverMod.verifyClientInDb("", "any-token", true);
|
||||
assert.equal(emptyCheck.allowed, false);
|
||||
assert.match(emptyCheck.reason || "", /CLIENT_REVOKED/);
|
||||
|
||||
const ghostCheck = await serverMod.verifyClientInDb("Unregistered Ghost", "any-token", true);
|
||||
assert.equal(ghostCheck.allowed, false);
|
||||
assert.match(ghostCheck.reason || "", /CLIENT_REVOKED/);
|
||||
|
||||
const ghostToken = AuthService.generateClientToken({ clientId: "never_registered_ws" });
|
||||
await assert.rejects(() => connectSocket(ghostToken), /CLIENT_REVOKED/);
|
||||
});
|
||||
|
||||
test("BUG-B02 & BUG-B04: Shared Redis idempotency/rate-limit across Port 3000+4000 and flat sync_data payload parity", async () => {
|
||||
const mockRedis = createSharedMockRedis();
|
||||
DataBufferEngine.setRedisClientForTests(mockRedis);
|
||||
serverMod.setRedisClientForTests(mockRedis);
|
||||
|
||||
try {
|
||||
await SettingsManager.updateSettingsAsync({
|
||||
maxEventsPerSec: 3,
|
||||
enableIdempotency: true,
|
||||
});
|
||||
|
||||
const client = await SDKManager.registerClientAsync({
|
||||
clientId: "phase02_redis_parity",
|
||||
name: "Phase02 Redis Parity",
|
||||
rateLimit: 3,
|
||||
});
|
||||
|
||||
// 1. Flat sync_data payload (no .data wrapper) preserves fields cleanly (BUG-B04)
|
||||
serverMod.bufferQueue.length = 0;
|
||||
const socket = await connectSocket(client.token);
|
||||
const flatAck = await emitWithAck<{ success: boolean; eventId: string }>(
|
||||
socket,
|
||||
"sync_data",
|
||||
{ eventId: "evt_shared_flat_1", metric: "voltage", value: 3.3 },
|
||||
);
|
||||
assert.equal(flatAck.success, true);
|
||||
|
||||
const bufferedItems = await mockRedis.lrange("datastack:buffer:events", 0, -1);
|
||||
const storedFlat = bufferedItems
|
||||
.map((raw) => JSON.parse(raw) as { eventId: string; payload: string })
|
||||
.find((e) => e.eventId === "evt_shared_flat_1");
|
||||
assert.ok(storedFlat, "Flat sync_data event must be buffered in Redis");
|
||||
const parsedFlat = JSON.parse(storedFlat.payload) as Record<string, unknown>;
|
||||
assert.equal(parsedFlat.metric, "voltage");
|
||||
assert.equal(parsedFlat.value, 3.3);
|
||||
|
||||
// 2. Replaying same eventId over HTTP POST /api/events triggers shared DUPLICATE_EVENT (BUG-B02)
|
||||
const replayHttpRes = await postEventsRoute(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
authorization: `Bearer ${client.token}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
eventId: "evt_shared_flat_1",
|
||||
data: { metric: "voltage", value: 3.3 },
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(replayHttpRes.status, 200);
|
||||
const replayHttpJson = await replayHttpRes.json();
|
||||
assert.equal(replayHttpJson.note, "DUPLICATE_DROPPED");
|
||||
|
||||
// 3. Cross-runtime shared per-second rate-limit: 1 WS already consumed quota; 2 HTTP pass, 3rd hits 429 (BUG-B02)
|
||||
const ok2 = await postEventsRoute(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
authorization: `Bearer ${client.token}`,
|
||||
},
|
||||
body: JSON.stringify({ eventId: "evt_rl_shared_2", data: { n: 2 } }),
|
||||
}),
|
||||
);
|
||||
const ok3 = await postEventsRoute(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
authorization: `Bearer ${client.token}`,
|
||||
},
|
||||
body: JSON.stringify({ eventId: "evt_rl_shared_3", data: { n: 3 } }),
|
||||
}),
|
||||
);
|
||||
assert.equal(ok2.status, 200);
|
||||
assert.equal(ok3.status, 200);
|
||||
|
||||
const limited4 = await postEventsRoute(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
authorization: `Bearer ${client.token}`,
|
||||
},
|
||||
body: JSON.stringify({ eventId: "evt_rl_shared_4", data: { n: 4 } }),
|
||||
}),
|
||||
);
|
||||
assert.equal(limited4.status, 429);
|
||||
socket.disconnect();
|
||||
} finally {
|
||||
DataBufferEngine.setRedisClientForTests(undefined);
|
||||
serverMod.setRedisClientForTests(undefined);
|
||||
}
|
||||
});
|
||||
|
||||
test("BUG-B03 & BUG-B05: Surfacing bridge sync failures, live runtime ports, normalized counter & production start script", async () => {
|
||||
const cookieHeader = await createAdminCookieHeader();
|
||||
const res = await postSettingsRoute(
|
||||
new NextRequest("http://localhost:3000/api/settings", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", cookie: cookieHeader },
|
||||
body: JSON.stringify({ maxEventsPerSec: 120 }),
|
||||
}),
|
||||
);
|
||||
assert.equal(res.status, 200);
|
||||
const body = await res.json();
|
||||
assert.equal(body.bridgeSync?.synced, true);
|
||||
|
||||
// Mismatched secret on remote Port 4000 must surface auth_error on dashboard metrics
|
||||
const http = await import("node:http");
|
||||
const rejectServer = http.createServer((_req, res) => {
|
||||
res.writeHead(401, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify({ success: false, code: "UNAUTHORIZED" }));
|
||||
});
|
||||
await new Promise<void>((resolveListen) => rejectServer.listen(0, "127.0.0.1", resolveListen));
|
||||
const rejectAddr = rejectServer.address() as AddressInfo;
|
||||
const savedIngestionUrl = process.env.INGESTION_INTERNAL_URL;
|
||||
process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${rejectAddr.port}`;
|
||||
try {
|
||||
const metrics = await ApiEngine.getDashboardMetrics();
|
||||
assert.equal(metrics.runtime.ingestionBridgeStatus, "auth_error");
|
||||
assert.match(metrics.runtime.ingestionBridgeError || "", /HTTP 401/);
|
||||
} finally {
|
||||
process.env.INGESTION_INTERNAL_URL = savedIngestionUrl;
|
||||
await new Promise<void>((resolveClose) => rejectServer.close(() => resolveClose()));
|
||||
}
|
||||
|
||||
// Normalized totalEvents counter updates cleanly
|
||||
const normClient = await SDKManager.registerClientAsync({
|
||||
clientId: "Mixed.Case.Counter",
|
||||
name: "Counter Check",
|
||||
rateLimit: 100,
|
||||
});
|
||||
await SDKManager.incrementEventCountAsync("MIXED.CASE.COUNTER", 3);
|
||||
const fetched = await SDKManager.getClientAsync(normClient.clientId);
|
||||
assert.equal(fetched?.totalEvents, 3);
|
||||
|
||||
// Production start script runs dual-runtime launcher
|
||||
const pkg = JSON.parse(readFileSync(resolve(process.cwd(), "package.json"), "utf8")) as {
|
||||
scripts?: Record<string, string>;
|
||||
};
|
||||
assert.equal(pkg.scripts?.start, "node scripts/start-all.js");
|
||||
assert.equal(pkg.scripts?.["start:server"], "node server/index.js");
|
||||
});
|
||||
|
||||
test("FORENSIC BUG-02: Empty Port 3000 bulk_sync preserves Port 4000 registry and hydrates Port 3000 non-destructively", async () => {
|
||||
const survivor = await SDKManager.registerClientAsync({
|
||||
clientId: "phase02_survivor_client",
|
||||
name: "Survivor Client",
|
||||
rateLimit: 40,
|
||||
});
|
||||
|
||||
// Verify Port 4000 knows survivor before simulated Port 3000 restart
|
||||
const preCheck = await serverMod.verifyClientInDb(survivor.clientId, survivor.token, true);
|
||||
assert.equal(preCheck.allowed, true);
|
||||
|
||||
// Simulate Port 3000 cold restart without PostgreSQL (local map cleared)
|
||||
g.__dataStackClientsMap?.clear();
|
||||
assert.equal(SDKManager.getClient(survivor.clientId), undefined);
|
||||
|
||||
// Calling syncAllClientsToIngestionAsync() with an empty Port 3000 map must NOT wipe Port 4000
|
||||
const syncRes = await SDKManager.syncAllClientsToIngestionAsync();
|
||||
assert.equal(syncRes.synced, true);
|
||||
|
||||
const postCheck = await serverMod.verifyClientInDb(survivor.clientId, survivor.token, true);
|
||||
assert.equal(postCheck.allowed, true, "Port 4000 registry must NOT be wiped by empty Port 3000 bulk_sync");
|
||||
|
||||
// Port 3000 should also have hydrated non-destructively from Port 4000
|
||||
const hydratedAfterBulk = SDKManager.getClient(survivor.clientId);
|
||||
assert.ok(hydratedAfterBulk, "Port 3000 should hydrate preserved client after empty bulk_sync");
|
||||
assert.equal(hydratedAfterBulk.token, survivor.token);
|
||||
|
||||
// Also test cold hydration via getClientAsync & listClientsAsync after clearing map again
|
||||
g.__dataStackClientsMap?.clear();
|
||||
const hydratedViaGet = await SDKManager.getClientAsync(survivor.clientId);
|
||||
assert.ok(hydratedViaGet, "getClientAsync should non-destructively hydrate from Port 4000");
|
||||
assert.equal(hydratedViaGet.clientId, survivor.clientId);
|
||||
});
|
||||
|
||||
test("FORENSIC BUG-17: JWT rotation coordinates Port 4000 first, aborts on bridge rejection without split-brain, and supports rotation grace", async () => {
|
||||
const originalSecret = AuthService.getSecret();
|
||||
const http = await import("node:http");
|
||||
|
||||
// 1. If Port 4000 rejects /internal/settings-sync with 503, Port 3000 MUST abort rotation cleanly
|
||||
const rejectServer = http.createServer((_req, res) => {
|
||||
res.writeHead(503, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify({ success: false, code: "BRIDGE_REJECTED" }));
|
||||
});
|
||||
await new Promise<void>((r) => rejectServer.listen(0, "127.0.0.1", r));
|
||||
const rejectPort = (rejectServer.address() as AddressInfo).port;
|
||||
const savedUrl = process.env.INGESTION_INTERNAL_URL;
|
||||
process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${rejectPort}`;
|
||||
|
||||
try {
|
||||
await assert.rejects(
|
||||
() =>
|
||||
SettingsManager.updateSettingsAsync({
|
||||
jwtSecret: "aborted-rotation-secret-must-never-commit-32-chars",
|
||||
}),
|
||||
/JWT_ROTATION_BRIDGE_FAILED/,
|
||||
);
|
||||
assert.equal(
|
||||
AuthService.getSecret(),
|
||||
originalSecret,
|
||||
"Port 3000 AuthService secret must remain unchanged when Port 4000 rejects rotation",
|
||||
);
|
||||
assert.equal(
|
||||
SettingsManager.getSettings().jwtSecret,
|
||||
originalSecret,
|
||||
"SettingsManager secret must remain unchanged after aborted rotation",
|
||||
);
|
||||
} finally {
|
||||
process.env.INGESTION_INTERNAL_URL = savedUrl;
|
||||
await new Promise<void>((r) => rejectServer.close(() => r()));
|
||||
}
|
||||
|
||||
// 2. Successful rotation updates both runtimes AND keeps previous secret valid on Port 4000 bridge grace window
|
||||
const rotatedSecret = "phase02-bug17-coordinated-rotation-secret-32-chars";
|
||||
await SettingsManager.updateSettingsAsync({ jwtSecret: rotatedSecret });
|
||||
assert.equal(AuthService.getSecret(), rotatedSecret);
|
||||
assert.equal(serverMod.runtimeConfig.jwtSecret, rotatedSecret);
|
||||
|
||||
const graceHeaders = getInternalIngestionHeaders(originalSecret);
|
||||
const graceMetricsRes = await fetch(`${ingestionUrl}/internal/metrics`, {
|
||||
headers: graceHeaders,
|
||||
});
|
||||
assert.equal(
|
||||
graceMetricsRes.status,
|
||||
200,
|
||||
"Port 4000 internal auth must accept previous secret during rotation grace window",
|
||||
);
|
||||
|
||||
// 3. New secret also works on internal bridge
|
||||
const currentMetricsRes = await fetch(`${ingestionUrl}/internal/metrics`, {
|
||||
headers: getInternalIngestionHeaders(rotatedSecret),
|
||||
});
|
||||
assert.equal(currentMetricsRes.status, 200);
|
||||
|
||||
// 4. If PG write fails AFTER Port 4000 accepts the new secret, BOTH Port 3000 & Port 4000 roll back cleanly
|
||||
const prevDbUrlLocal = process.env.DATABASE_URL;
|
||||
const prevPoolLocal = g.__dataStackCorePool;
|
||||
const prevSchemaLocal = g.__dataStackCoreSchema;
|
||||
process.env.DATABASE_URL = "postgres://mock:mock@127.0.0.1:5432/mock";
|
||||
g.__dataStackCoreSchema = Promise.resolve();
|
||||
g.__dataStackCorePool = {
|
||||
async query(sql: string) {
|
||||
if (sql.includes("INSERT INTO admin_settings")) {
|
||||
throw new Error("Simulated PG disk full during rotation");
|
||||
}
|
||||
return { rowCount: 0, rows: [] };
|
||||
},
|
||||
};
|
||||
try {
|
||||
await assert.rejects(
|
||||
() =>
|
||||
SettingsManager.updateSettingsAsync({
|
||||
jwtSecret: "rolled-back-secret-after-pg-failure-32-chars-min",
|
||||
}),
|
||||
/DATABASE_WRITE_FAILED/,
|
||||
);
|
||||
assert.equal(
|
||||
AuthService.getSecret(),
|
||||
rotatedSecret,
|
||||
"Port 3000 must roll back to previous secret after DB write failure",
|
||||
);
|
||||
assert.equal(
|
||||
serverMod.runtimeConfig.jwtSecret,
|
||||
rotatedSecret,
|
||||
"Port 4000 must roll back to previous secret after Port 3000 DB write failure",
|
||||
);
|
||||
} finally {
|
||||
if (prevDbUrlLocal === undefined) delete process.env.DATABASE_URL;
|
||||
else process.env.DATABASE_URL = prevDbUrlLocal;
|
||||
g.__dataStackCorePool = prevPoolLocal;
|
||||
g.__dataStackCoreSchema = prevSchemaLocal;
|
||||
}
|
||||
});
|
||||
|
||||
test("FORENSIC BUG-03: Without Redis, WS and HTTP share idempotency and rate limits via Port 4000 /internal/security-check", async () => {
|
||||
// Explicitly ensure Redis is disabled
|
||||
DataBufferEngine.setRedisClientForTests(null);
|
||||
serverMod.setRedisClientForTests(null);
|
||||
|
||||
try {
|
||||
await SettingsManager.updateSettingsAsync({
|
||||
maxEventsPerSec: 3,
|
||||
enableIdempotency: true,
|
||||
});
|
||||
|
||||
const client = await SDKManager.registerClientAsync({
|
||||
clientId: "phase02_no_redis_sec",
|
||||
name: "No-Redis Shared Security",
|
||||
rateLimit: 3,
|
||||
});
|
||||
|
||||
const socket = await connectSocket(client.token);
|
||||
const wsAck = await emitWithAck<{ success: boolean; eventId: string }>(
|
||||
socket,
|
||||
"sync_data",
|
||||
{ eventId: "evt_noredis_shared_1", data: { step: 1 } },
|
||||
);
|
||||
assert.equal(wsAck.success, true);
|
||||
|
||||
// Same eventId over HTTP POST /api/events MUST be deduplicated via Port 4000 bridge check
|
||||
const dupHttpRes = await postEventsRoute(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
authorization: `Bearer ${client.token}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
eventId: "evt_noredis_shared_1",
|
||||
data: { step: 1 },
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(dupHttpRes.status, 200);
|
||||
const dupJson = await dupHttpRes.json();
|
||||
assert.equal(
|
||||
dupJson.note,
|
||||
"DUPLICATE_DROPPED",
|
||||
"HTTP replay of WS eventId must be dropped even without Redis",
|
||||
);
|
||||
|
||||
// Rate-limit parity without Redis: WS consumed 1 of 3; 2 HTTP pass, 3rd returns 429
|
||||
const http2 = await postEventsRoute(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
authorization: `Bearer ${client.token}`,
|
||||
},
|
||||
body: JSON.stringify({ eventId: "evt_noredis_rl_2", data: { step: 2 } }),
|
||||
}),
|
||||
);
|
||||
const http3 = await postEventsRoute(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
authorization: `Bearer ${client.token}`,
|
||||
},
|
||||
body: JSON.stringify({ eventId: "evt_noredis_rl_3", data: { step: 3 } }),
|
||||
}),
|
||||
);
|
||||
assert.equal(http2.status, 200);
|
||||
assert.equal(http3.status, 200);
|
||||
|
||||
const http4 = await postEventsRoute(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
authorization: `Bearer ${client.token}`,
|
||||
},
|
||||
body: JSON.stringify({ eventId: "evt_noredis_rl_4", data: { step: 4 } }),
|
||||
}),
|
||||
);
|
||||
assert.equal(http4.status, 429, "Combined WS+HTTP quota must enforce 429 without Redis");
|
||||
|
||||
// Metrics explicitly surface bridge_coordinated securityMode
|
||||
const metrics = await ApiEngine.getDashboardMetrics();
|
||||
assert.equal(metrics.securityStats.mode, "bridge_coordinated");
|
||||
assert.equal(metrics.runtime.securityMode, "bridge_coordinated");
|
||||
|
||||
socket.disconnect();
|
||||
} finally {
|
||||
DataBufferEngine.setRedisClientForTests(undefined);
|
||||
serverMod.setRedisClientForTests(undefined);
|
||||
}
|
||||
});
|
||||
|
||||
test("FORENSIC BUG-22: High-frequency client event increments coalesce into batched UNNEST SQL updates on both Port 3000 and Port 4000", async () => {
|
||||
const sdkQueries: Array<{ sql: string; params: unknown[] }> = [];
|
||||
const serverQueries: Array<{ sql: string; params: unknown[] }> = [];
|
||||
|
||||
const mockSdkPool = {
|
||||
async query(sql: string, params: unknown[] = []) {
|
||||
sdkQueries.push({ sql, params });
|
||||
return { rowCount: 1, rows: [] };
|
||||
},
|
||||
};
|
||||
const mockServerPool = {
|
||||
async query(sql: string, params: unknown[] = []) {
|
||||
serverQueries.push({ sql, params });
|
||||
return { rowCount: 1, rows: [] };
|
||||
},
|
||||
};
|
||||
|
||||
const prevDatabaseUrl = process.env.DATABASE_URL;
|
||||
const prevCorePool = g.__dataStackCorePool;
|
||||
const prevCoreSchema = g.__dataStackCoreSchema;
|
||||
|
||||
process.env.DATABASE_URL = "postgres://mock:mock@127.0.0.1:5432/mock";
|
||||
g.__dataStackCorePool = mockSdkPool;
|
||||
g.__dataStackCoreSchema = Promise.resolve();
|
||||
g.__dataStackPendingClientActivity?.clear();
|
||||
serverMod.setDbPoolForTests(mockServerPool);
|
||||
|
||||
try {
|
||||
// 1. Port 3000 SDKManager: 25 rapid increments across 2 clients -> 0 immediate per-event queries, 1 batched UNNEST query on flush
|
||||
for (let i = 0; i < 15; i += 1) {
|
||||
await SDKManager.incrementEventCountAsync("batch_client_a", 1);
|
||||
}
|
||||
for (let i = 0; i < 10; i += 1) {
|
||||
await SDKManager.incrementEventCountAsync("batch_client_b", 2);
|
||||
}
|
||||
|
||||
const preFlushUpdates = sdkQueries.filter((q) =>
|
||||
q.sql.includes("UPDATE client_sdks"),
|
||||
);
|
||||
assert.equal(
|
||||
preFlushUpdates.length,
|
||||
0,
|
||||
"Per-event increments must coalesce in memory instead of firing immediate per-row SQL",
|
||||
);
|
||||
|
||||
const flushedSdkClients = await SDKManager.flushPendingClientActivityAsync();
|
||||
assert.equal(flushedSdkClients, 2);
|
||||
|
||||
const postFlushSdkUpdates = sdkQueries.filter((q) =>
|
||||
q.sql.includes("UPDATE client_sdks"),
|
||||
);
|
||||
assert.equal(
|
||||
postFlushSdkUpdates.length,
|
||||
1,
|
||||
"25 increments across 2 clients must execute exactly ONE batched UPDATE query",
|
||||
);
|
||||
assert.match(postFlushSdkUpdates[0].sql, /UNNEST\(\$1::text\[\]\)/);
|
||||
assert.deepEqual(postFlushSdkUpdates[0].params[0], [
|
||||
"batch_client_a",
|
||||
"batch_client_b",
|
||||
]);
|
||||
assert.deepEqual(postFlushSdkUpdates[0].params[1], [15, 20]);
|
||||
|
||||
// 2. Port 4000 server/index.js: 30 rapid recordClientActivity calls -> 1 batched UNNEST query on flush
|
||||
for (let i = 0; i < 20; i += 1) {
|
||||
serverMod.recordClientActivity("ws_batch_a", 1);
|
||||
}
|
||||
for (let i = 0; i < 10; i += 1) {
|
||||
serverMod.recordClientActivity("ws_batch_b", 3);
|
||||
}
|
||||
assert.equal(
|
||||
serverQueries.length,
|
||||
0,
|
||||
"Port 4000 recordClientActivity must coalesce instead of running per-event SQL",
|
||||
);
|
||||
|
||||
const flushedServerClients = await serverMod.flushPendingClientActivity();
|
||||
assert.equal(flushedServerClients, 2);
|
||||
assert.equal(
|
||||
serverQueries.length,
|
||||
1,
|
||||
"30 Port 4000 activity events must coalesce into ONE batched UPDATE query",
|
||||
);
|
||||
assert.match(serverQueries[0].sql, /UNNEST\(\$1::text\[\]\)/);
|
||||
assert.deepEqual(serverQueries[0].params[0], ["ws_batch_a", "ws_batch_b"]);
|
||||
assert.deepEqual(serverQueries[0].params[1], [20, 30]);
|
||||
} finally {
|
||||
SDKManager.stopActivityFlushTimerForTests();
|
||||
g.__dataStackPendingClientActivity?.clear();
|
||||
serverMod.setDbPoolForTests(null);
|
||||
if (prevDatabaseUrl === undefined) delete process.env.DATABASE_URL;
|
||||
else process.env.DATABASE_URL = prevDatabaseUrl;
|
||||
g.__dataStackCorePool = prevCorePool;
|
||||
g.__dataStackCoreSchema = prevCoreSchema;
|
||||
}
|
||||
});
|
||||
|
||||
test("R2 PHASE-02 BUG-05: PG pools attach safe error listeners and Port 3000 shares a single unified DatabaseEngine pool", async () => {
|
||||
const prevDbUrl = process.env.DATABASE_URL;
|
||||
const prevCorePool = g.__dataStackCorePool;
|
||||
const origConsoleError = console.error;
|
||||
const loggedErrors: string[] = [];
|
||||
console.error = (...args: unknown[]) => {
|
||||
loggedErrors.push(args.map((a) => String(a)).join(" "));
|
||||
};
|
||||
|
||||
try {
|
||||
process.env.DATABASE_URL = "postgres://unit:test@127.0.0.1:5432/datastack_unit";
|
||||
g.__dataStackCorePool = undefined;
|
||||
|
||||
const pool = DatabaseEngine.getPool();
|
||||
assert.ok(pool, "DatabaseEngine.getPool() must instantiate a pool when DATABASE_URL is set");
|
||||
assert.ok(
|
||||
pool.listenerCount("error") >= 1,
|
||||
"DatabaseEngine PG pool must attach at least one 'error' listener",
|
||||
);
|
||||
assert.equal(
|
||||
isAdminDatabaseConfigured(),
|
||||
true,
|
||||
"adminUsers must delegate to the shared DatabaseEngine.getPool()",
|
||||
);
|
||||
|
||||
// Emitting an idle client error must log safely instead of throwing an uncaught EventEmitter crash
|
||||
assert.doesNotThrow(() => {
|
||||
pool.emit("error", new Error("simulated idle PG restart"));
|
||||
});
|
||||
assert.ok(
|
||||
loggedErrors.some((msg) => msg.includes("simulated idle PG restart")),
|
||||
"Pool error listener must log unexpected idle pool errors",
|
||||
);
|
||||
|
||||
// Static verification that adminUsers.ts has zero independent `new Pool` and server/index.js attaches dbPool.on('error')
|
||||
const adminUsersSrc = readFileSync(
|
||||
resolve(process.cwd(), "core", "auth", "adminUsers.ts"),
|
||||
"utf8",
|
||||
);
|
||||
assert.equal(/\bnew Pool\s*\(/.test(adminUsersSrc), false);
|
||||
|
||||
const serverSrc = readFileSync(
|
||||
resolve(process.cwd(), "server", "index.js"),
|
||||
"utf8",
|
||||
);
|
||||
assert.match(serverSrc, /dbPool\.on\(\s*['"]error['"]/);
|
||||
|
||||
await pool.end().catch(() => undefined);
|
||||
} finally {
|
||||
console.error = origConsoleError;
|
||||
if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
|
||||
else process.env.DATABASE_URL = prevDbUrl;
|
||||
g.__dataStackCorePool = prevCorePool;
|
||||
}
|
||||
});
|
||||
|
||||
test("R2 PHASE-02 BUG-09: Redis rate-limiting uses atomic single-roundtrip Lua EVAL on both Port 3000 and Port 4000", async () => {
|
||||
const baseRedis = createSharedMockRedis();
|
||||
let evalRateLimitCalls = 0;
|
||||
let separateIncrCalls = 0;
|
||||
let separatePexpireCalls = 0;
|
||||
|
||||
const instrumentedRedis: RedisBufferClient = {
|
||||
...baseRedis,
|
||||
async incr(key: string) {
|
||||
separateIncrCalls += 1;
|
||||
return baseRedis.incr!(key);
|
||||
},
|
||||
async pexpire(key: string, ttlMs: number) {
|
||||
separatePexpireCalls += 1;
|
||||
return baseRedis.pexpire!(key, ttlMs);
|
||||
},
|
||||
async eval(script: string, numKeys: number, ...args: string[]) {
|
||||
if (script.includes("DATASTACK_RATE_LIMIT_INCR")) {
|
||||
evalRateLimitCalls += 1;
|
||||
assert.equal(args[1], "2000", "Atomic Lua rate-limit script must pass 2000ms TTL");
|
||||
}
|
||||
return baseRedis.eval(script, numKeys, ...args);
|
||||
},
|
||||
};
|
||||
|
||||
DataBufferEngine.setRedisClientForTests(instrumentedRedis);
|
||||
serverMod.setRedisClientForTests(instrumentedRedis);
|
||||
|
||||
try {
|
||||
await SettingsManager.updateSettingsAsync({
|
||||
maxEventsPerSec: 2,
|
||||
enableIdempotency: true,
|
||||
});
|
||||
const client = await SDKManager.registerClientAsync({
|
||||
clientId: "phase02_lua_rl_client",
|
||||
name: "Lua RL Client",
|
||||
rateLimit: 2,
|
||||
});
|
||||
|
||||
// 1 event via Port 3000 HTTP SecurityEngine
|
||||
const http1 = await SecurityEngine.validateEventAsync(
|
||||
client.clientId,
|
||||
"lua_rl_1",
|
||||
{ v: 1 },
|
||||
client.rateLimit,
|
||||
);
|
||||
assert.equal(http1.allowed, true);
|
||||
|
||||
// 2nd event via Port 4000 Socket.IO handler (allowed) + 3rd event (rate limited)
|
||||
const socket = await connectSocket(client.token);
|
||||
const ws2 = await emitWithAck<{ success: boolean; code?: string }>(
|
||||
socket,
|
||||
"sync_data",
|
||||
{ eventId: "lua_rl_2", data: { v: 2 } },
|
||||
);
|
||||
assert.equal(ws2.success, true);
|
||||
|
||||
const ws3 = await emitWithAck<{ success: boolean; code?: string }>(
|
||||
socket,
|
||||
"sync_data",
|
||||
{ eventId: "lua_rl_3", data: { v: 3 } },
|
||||
);
|
||||
assert.equal(ws3.success, false);
|
||||
assert.equal(ws3.code, "RATE_LIMIT_EXCEEDED");
|
||||
|
||||
socket.disconnect();
|
||||
|
||||
assert.equal(
|
||||
evalRateLimitCalls,
|
||||
3,
|
||||
"All 3 rate-limit checks across Port 3000 & Port 4000 must execute atomic Lua EVAL",
|
||||
);
|
||||
assert.equal(
|
||||
separateIncrCalls,
|
||||
0,
|
||||
"Separate non-atomic redis.incr must NOT be called when redis.eval is available",
|
||||
);
|
||||
assert.equal(
|
||||
separatePexpireCalls,
|
||||
0,
|
||||
"Separate non-atomic redis.pexpire must NOT be called when redis.eval is available",
|
||||
);
|
||||
} finally {
|
||||
DataBufferEngine.setRedisClientForTests(undefined);
|
||||
serverMod.setRedisClientForTests(undefined);
|
||||
await SettingsManager.updateSettingsAsync({ maxEventsPerSec: 100 });
|
||||
}
|
||||
});
|
||||
|
||||
test("R2 PHASE-02 BUG-10: GET, PATCH (regenerate_token), and DELETE /api/clients/[id] catch DB errors and return structured JSON", async () => {
|
||||
const secret = getAdminSessionSecret()!;
|
||||
const adminToken = await createAdminSessionToken(
|
||||
"admin@datastack.dev",
|
||||
secret,
|
||||
Date.now() + 3600_000,
|
||||
1,
|
||||
);
|
||||
const cookieHeader = `datastack_admin_session=${adminToken}`;
|
||||
|
||||
const origGet = SDKManager.getClientAsync;
|
||||
const origRegen = SDKManager.regenerateTokenAsync;
|
||||
const origDeleteWithStats = SDKManager.deleteClientWithStatsAsync;
|
||||
|
||||
try {
|
||||
SDKManager.getClientAsync = async () => {
|
||||
throw new Error("DATABASE_READ_FAILED: simulated pg read outage");
|
||||
};
|
||||
const getRes = await getClientRoute(
|
||||
new NextRequest("http://localhost:3000/api/clients/broken_client", {
|
||||
method: "GET",
|
||||
headers: { cookie: cookieHeader },
|
||||
}),
|
||||
{ params: Promise.resolve({ id: "broken_client" }) },
|
||||
);
|
||||
assert.equal(getRes.status, 503);
|
||||
const getJson = await getRes.json();
|
||||
assert.equal(getJson.code, "DATABASE_READ_FAILED");
|
||||
assert.match(getJson.error, /simulated pg read outage/);
|
||||
|
||||
SDKManager.regenerateTokenAsync = async () => {
|
||||
throw new Error("DATABASE_WRITE_FAILED: simulated token rotation DB failure");
|
||||
};
|
||||
const regenRes = await patchClientRoute(
|
||||
new NextRequest("http://localhost:3000/api/clients/broken_client", {
|
||||
method: "PATCH",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
cookie: cookieHeader,
|
||||
},
|
||||
body: JSON.stringify({ action: "regenerate_token" }),
|
||||
}),
|
||||
{ params: Promise.resolve({ id: "broken_client" }) },
|
||||
);
|
||||
assert.equal(regenRes.status, 503);
|
||||
const regenJson = await regenRes.json();
|
||||
assert.equal(regenJson.code, "DATABASE_WRITE_FAILED");
|
||||
assert.match(regenJson.error, /simulated token rotation DB failure/);
|
||||
|
||||
SDKManager.deleteClientWithStatsAsync = async () => {
|
||||
throw new Error("DATABASE_WRITE_FAILED: simulated delete DB failure");
|
||||
};
|
||||
const delRes = await deleteClientRoute(
|
||||
new NextRequest("http://localhost:3000/api/clients/broken_client", {
|
||||
method: "DELETE",
|
||||
headers: { cookie: cookieHeader },
|
||||
}),
|
||||
{ params: Promise.resolve({ id: "broken_client" }) },
|
||||
);
|
||||
assert.equal(delRes.status, 503);
|
||||
const delJson = await delRes.json();
|
||||
assert.equal(delJson.code, "DATABASE_WRITE_FAILED");
|
||||
assert.match(delJson.error, /simulated delete DB failure/);
|
||||
} finally {
|
||||
SDKManager.getClientAsync = origGet;
|
||||
SDKManager.regenerateTokenAsync = origRegen;
|
||||
SDKManager.deleteClientWithStatsAsync = origDeleteWithStats;
|
||||
}
|
||||
});
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,495 @@
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import { createRequire } from "node:module";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import test, { after, before, beforeEach } from "node:test";
|
||||
import { NextRequest } from "next/server";
|
||||
import { GET as getDataRoute } from "../app/api/data/route";
|
||||
import {
|
||||
GET as getEventsRoute,
|
||||
POST as postEventsRoute,
|
||||
} from "../app/api/events/route";
|
||||
import {
|
||||
createAdminSessionToken,
|
||||
getAdminSessionSecret,
|
||||
} from "../core/auth/adminSession";
|
||||
import {
|
||||
createInitialAdmin,
|
||||
resetAdminStateForTests,
|
||||
} from "../core/auth/adminUsers";
|
||||
import {
|
||||
DataBufferEngine,
|
||||
MAX_REDIS_RECENT_LIMIT,
|
||||
resolveRedisRecentLimit,
|
||||
} from "../core/data/buffer";
|
||||
import {
|
||||
CORE_SCHEMA_ALTER_DDLS,
|
||||
CORE_SCHEMA_TABLE_DDLS,
|
||||
} from "../core/Database/database";
|
||||
import { SDKManager } from "../core/sdk/sdkManager";
|
||||
import { CSVStorageManager, type DataEvent } from "../core/Storage/csvStorage";
|
||||
|
||||
const g = globalThis as typeof globalThis & {
|
||||
__dataStackBufferState?: {
|
||||
buffer: DataEvent[];
|
||||
recentEvents: DataEvent[];
|
||||
deletedClientIds?: Set<string>;
|
||||
batchFlushThreshold: number;
|
||||
};
|
||||
__dataStackAdminSettings?: unknown;
|
||||
__dataStackClientsMap?: Map<string, unknown>;
|
||||
__dataStackDeletedClients?: Set<string>;
|
||||
};
|
||||
|
||||
let tmpStorageDir = "";
|
||||
let adminCookieHeader = "";
|
||||
const origBaseStorageDir = CSVStorageManager.baseStorageDir;
|
||||
const prevDbUrl = process.env.DATABASE_URL;
|
||||
const prevRedisUrl = process.env.REDIS_URL;
|
||||
const prevRecentEnv = process.env.DATASTACK_RECENT_LIMIT;
|
||||
const prevIngestionUrl = process.env.INGESTION_INTERNAL_URL;
|
||||
|
||||
before(async () => {
|
||||
delete process.env.DATABASE_URL;
|
||||
delete process.env.REDIS_URL;
|
||||
delete process.env.DATASTACK_RECENT_LIMIT;
|
||||
process.env.INGESTION_INTERNAL_URL = "http://127.0.0.1:1";
|
||||
|
||||
resetAdminStateForTests();
|
||||
await createInitialAdmin("admin@datastack.dev", "Phase04Pass123!");
|
||||
const secret = getAdminSessionSecret()!;
|
||||
const token = await createAdminSessionToken(
|
||||
"admin@datastack.dev",
|
||||
secret,
|
||||
Date.now() + 3600_000,
|
||||
1,
|
||||
);
|
||||
adminCookieHeader = `datastack_admin_session=${token}`;
|
||||
|
||||
tmpStorageDir = fs.mkdtempSync(path.join(os.tmpdir(), "datastack-phase04-"));
|
||||
CSVStorageManager.baseStorageDir = tmpStorageDir;
|
||||
CSVStorageManager.clearRowCountCache();
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
process.env.INGESTION_INTERNAL_URL = "http://127.0.0.1:1";
|
||||
DataBufferEngine.setRedisClientForTests(null);
|
||||
DataBufferEngine.batchFlushThreshold = 500;
|
||||
delete process.env.DATASTACK_RECENT_LIMIT;
|
||||
if (g.__dataStackBufferState) {
|
||||
g.__dataStackBufferState.buffer = [];
|
||||
g.__dataStackBufferState.recentEvents = [];
|
||||
g.__dataStackBufferState.deletedClientIds?.clear();
|
||||
}
|
||||
CSVStorageManager.clearRowCountCache();
|
||||
fs.rmSync(tmpStorageDir, { recursive: true, force: true });
|
||||
fs.mkdirSync(tmpStorageDir, { recursive: true });
|
||||
delete g.__dataStackAdminSettings;
|
||||
g.__dataStackClientsMap?.clear();
|
||||
g.__dataStackDeletedClients?.clear();
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await DataBufferEngine.shutdown();
|
||||
CSVStorageManager.baseStorageDir = origBaseStorageDir;
|
||||
CSVStorageManager.clearRowCountCache();
|
||||
if (tmpStorageDir) {
|
||||
fs.rmSync(tmpStorageDir, { recursive: true, force: true });
|
||||
}
|
||||
if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
|
||||
else process.env.DATABASE_URL = prevDbUrl;
|
||||
|
||||
if (prevRedisUrl === undefined) delete process.env.REDIS_URL;
|
||||
else process.env.REDIS_URL = prevRedisUrl;
|
||||
|
||||
if (prevRecentEnv === undefined) delete process.env.DATASTACK_RECENT_LIMIT;
|
||||
else process.env.DATASTACK_RECENT_LIMIT = prevRecentEnv;
|
||||
|
||||
if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL;
|
||||
else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl;
|
||||
});
|
||||
|
||||
test("FORENSIC BUG-21 (Part A): HTTP POST /api/events matches WS envelope stripping parity and never fabricates sample telemetry for {}", async () => {
|
||||
const client = await SDKManager.registerClientAsync({
|
||||
clientId: "phase04_parity",
|
||||
name: "Parity Client",
|
||||
rateLimit: 100,
|
||||
});
|
||||
|
||||
// 1. Empty {} body must store "{}" (NO fake { sample, rand })
|
||||
const emptyRes = await postEventsRoute(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
authorization: `Bearer ${client.token}`,
|
||||
},
|
||||
body: JSON.stringify({}),
|
||||
}),
|
||||
);
|
||||
assert.equal(emptyRes.status, 200);
|
||||
const emptyJson = await emptyRes.json();
|
||||
assert.equal(emptyJson.success, true);
|
||||
assert.equal(emptyJson.event.payload, "{}");
|
||||
assert.equal(emptyJson.event.payload.includes("sample"), false);
|
||||
assert.equal(emptyJson.event.payload.includes("rand"), false);
|
||||
|
||||
// 2. Flat body with envelope keys must strip eventId/timestamp/clientId cleanly
|
||||
const flatRes = await postEventsRoute(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
authorization: `Bearer ${client.token}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
eventId: "flat-parity-1",
|
||||
timestamp: "2026-11-19T12:00:00.000Z",
|
||||
clientId: client.clientId,
|
||||
temp: 24.5,
|
||||
unit: "C",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(flatRes.status, 200);
|
||||
const flatJson = await flatRes.json();
|
||||
assert.deepEqual(JSON.parse(flatJson.event.payload), { temp: 24.5, unit: "C" });
|
||||
|
||||
// 3. Pure envelope { eventId, timestamp, data } extracts data identically to WS
|
||||
const wrappedRes = await postEventsRoute(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
authorization: `Bearer ${client.token}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
eventId: "wrapped-parity-1",
|
||||
timestamp: "2026-11-19T12:00:01.000Z",
|
||||
data: { pressure: 101.3 },
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(wrappedRes.status, 200);
|
||||
const wrappedJson = await wrappedRes.json();
|
||||
assert.deepEqual(JSON.parse(wrappedJson.event.payload), { pressure: 101.3 });
|
||||
|
||||
// 4. Flat payload that contains both `data` and extra domain fields preserves the object identically to WS
|
||||
const mixedRes = await postEventsRoute(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
authorization: `Bearer ${client.token}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
eventId: "mixed-parity-1",
|
||||
data: "sensor-raw",
|
||||
extraMetric: 42,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(mixedRes.status, 200);
|
||||
const mixedJson = await mixedRes.json();
|
||||
assert.deepEqual(JSON.parse(mixedJson.event.payload), {
|
||||
eventId: "mixed-parity-1",
|
||||
data: "sensor-raw",
|
||||
extraMetric: 42,
|
||||
});
|
||||
});
|
||||
|
||||
test("FORENSIC BUG-21 (Part B): GET /api/data uses non-blocking async fs.promises with partition pruning and REDIS_RECENT_LIMIT scales boundedly", async () => {
|
||||
const dataRouteSrc = fs.readFileSync(
|
||||
path.resolve(process.cwd(), "app/api/data/route.ts"),
|
||||
"utf8",
|
||||
);
|
||||
assert.match(dataRouteSrc, /from "node:fs\/promises"/);
|
||||
assert.equal(/\breaddirSync\b/.test(dataRouteSrc), false);
|
||||
assert.equal(/\blstatSync\b/.test(dataRouteSrc), false);
|
||||
|
||||
// Create an out-of-range corrupt CSV partition and an in-range healthy CSV partition
|
||||
const clientDir = CSVStorageManager.ensureClientDir("phase04_async");
|
||||
fs.writeFileSync(
|
||||
path.join(clientDir, "2026-11-10.csv"),
|
||||
'"clientId","eventId","timestamp","payload"\n"phase04_async","bad","2026-11-10T00:00:00Z","unclosed',
|
||||
"utf8",
|
||||
);
|
||||
await CSVStorageManager.appendBatch([
|
||||
{
|
||||
clientId: "phase04_async",
|
||||
eventId: "in-range-1",
|
||||
timestamp: "2026-11-19T14:00:00.000Z",
|
||||
payload: JSON.stringify({ healthy: true }),
|
||||
},
|
||||
]);
|
||||
|
||||
const res = await getDataRoute(
|
||||
new NextRequest(
|
||||
"http://localhost:3000/api/data?clientId=phase04_async&from=2026-11-19&to=2026-11-19",
|
||||
{ headers: { cookie: adminCookieHeader } },
|
||||
),
|
||||
);
|
||||
assert.equal(res.status, 200);
|
||||
const json = await res.json();
|
||||
assert.equal(json.total, 1);
|
||||
assert.equal(json.events[0].eventId, "in-range-1");
|
||||
|
||||
// Bounded configurable recent limit scaling
|
||||
assert.equal(resolveRedisRecentLimit(500), 500);
|
||||
assert.equal(resolveRedisRecentLimit(1200), 1200);
|
||||
assert.equal(resolveRedisRecentLimit(50000), MAX_REDIS_RECENT_LIMIT);
|
||||
|
||||
process.env.DATASTACK_RECENT_LIMIT = "900";
|
||||
assert.equal(DataBufferEngine.getRecentLimit(), 900);
|
||||
delete process.env.DATASTACK_RECENT_LIMIT;
|
||||
|
||||
DataBufferEngine.batchFlushThreshold = 650;
|
||||
assert.equal(DataBufferEngine.getRecentLimit(), 650);
|
||||
const batch600: DataEvent[] = Array.from({ length: 600 }, (_, i) => ({
|
||||
clientId: "phase04_recent_scale",
|
||||
eventId: `recent-scale-${i}`,
|
||||
timestamp: "2026-11-19T15:00:00.000Z",
|
||||
payload: JSON.stringify({ i }),
|
||||
}));
|
||||
await DataBufferEngine.pushBatchAsync(batch600);
|
||||
assert.equal(DataBufferEngine.getRecentEvents(650, "phase04_recent_scale").length, 600);
|
||||
});
|
||||
|
||||
test("FORENSIC BUG-23: Client secrets and fallback eventIds use CSPRNG crypto.randomBytes with zero Math.random()", async () => {
|
||||
const client = await SDKManager.registerClientAsync({
|
||||
clientId: "phase04_csprng",
|
||||
name: "CSPRNG Client",
|
||||
rateLimit: 100,
|
||||
});
|
||||
assert.match(client.secret, /^sec_phase04_csprng_[0-9a-f]{32}$/);
|
||||
|
||||
const rotated = await SDKManager.regenerateTokenAsync(client.clientId);
|
||||
assert.ok(rotated);
|
||||
assert.match(rotated.secret, /^sec_phase04_csprng_[0-9a-f]{32}$/);
|
||||
assert.notEqual(rotated.secret, client.secret);
|
||||
|
||||
const res = await postEventsRoute(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
authorization: `Bearer ${rotated.token}`,
|
||||
},
|
||||
body: JSON.stringify({ metric: "ok" }),
|
||||
}),
|
||||
);
|
||||
assert.equal(res.status, 200);
|
||||
const json = await res.json();
|
||||
assert.match(json.eventId, /^evt_\d+_[0-9a-f]{12}$/);
|
||||
|
||||
for (const relPath of [
|
||||
"core/sdk/sdkManager.ts",
|
||||
"app/api/events/route.ts",
|
||||
"server/socket/handlers.js",
|
||||
]) {
|
||||
const src = fs.readFileSync(path.resolve(process.cwd(), relPath), "utf8");
|
||||
assert.equal(
|
||||
/\bMath\.random\s*\(/.test(src),
|
||||
false,
|
||||
`Expected zero Math.random() calls in ${relPath}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("FORENSIC BUG-24: Canonical PostgreSQL DDL is unified in database.ts, delegated by adminUsers.ts, and mirrored 1:1 in server/index.js & prisma/schema.prisma", () => {
|
||||
const adminUsersSrc = fs.readFileSync(
|
||||
path.resolve(process.cwd(), "core/auth/adminUsers.ts"),
|
||||
"utf8",
|
||||
);
|
||||
const serverIndexSrc = fs.readFileSync(
|
||||
path.resolve(process.cwd(), "server/index.js"),
|
||||
"utf8",
|
||||
);
|
||||
const prismaSchemaSrc = fs.readFileSync(
|
||||
path.resolve(process.cwd(), "prisma/schema.prisma"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
// adminUsers.ts must delegate to ensureCoreSchemaOnPool rather than duplicating CREATE TABLE
|
||||
assert.match(adminUsersSrc, /ensureCoreSchemaOnPool/);
|
||||
assert.equal(/CREATE TABLE IF NOT EXISTS/.test(adminUsersSrc), false);
|
||||
|
||||
// All 5 canonical tables must exist identically in database.ts, server/index.js, and prisma/schema.prisma
|
||||
const expectedTables = [
|
||||
"client_sdks",
|
||||
"admin_settings",
|
||||
"deleted_clients",
|
||||
"admin_users",
|
||||
"admin_login_attempts",
|
||||
];
|
||||
const dbTablesText = CORE_SCHEMA_TABLE_DDLS.join("\n");
|
||||
for (const table of expectedTables) {
|
||||
const createRe = new RegExp(`CREATE TABLE IF NOT EXISTS\\s+${table}\\b`);
|
||||
assert.match(dbTablesText, createRe, `database.ts missing ${table}`);
|
||||
assert.match(serverIndexSrc, createRe, `server/index.js missing ${table}`);
|
||||
assert.match(
|
||||
prismaSchemaSrc,
|
||||
new RegExp(`@@map\\("${table}"\\)`),
|
||||
`prisma/schema.prisma missing @@map("${table}")`,
|
||||
);
|
||||
}
|
||||
|
||||
// All 3 additive ALTER TABLE guards must exist in both database.ts and server/index.js
|
||||
const dbAltersText = CORE_SCHEMA_ALTER_DDLS.join("\n");
|
||||
for (const alterTable of ["client_sdks", "admin_settings", "admin_users"]) {
|
||||
const alterRe = new RegExp(`ALTER TABLE\\s+${alterTable}\\s+ADD COLUMN IF NOT EXISTS`);
|
||||
assert.match(dbAltersText, alterRe, `database.ts missing ALTER TABLE ${alterTable}`);
|
||||
assert.match(serverIndexSrc, alterRe, `server/index.js missing ALTER TABLE ${alterTable}`);
|
||||
}
|
||||
|
||||
// prisma/schema.prisma must explicitly document canonical runtime pg ownership
|
||||
assert.match(prismaSchemaSrc, /REFERENCE SCHEMA ONLY/);
|
||||
assert.match(prismaSchemaSrc, /core\/Database\/database\.ts/);
|
||||
});
|
||||
|
||||
test("R2 PHASE-03 BUG-11: Flat batch items strip eventId/timestamp/clientId across both HTTP POST /api/events and Socket.IO batch_sync", async () => {
|
||||
const client = await SDKManager.registerClientAsync({
|
||||
clientId: "phase03_batch_strip",
|
||||
name: "Batch Strip Client",
|
||||
rateLimit: 200,
|
||||
});
|
||||
|
||||
// 1. HTTP POST /api/events batch array
|
||||
const httpBatchRes = await postEventsRoute(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
authorization: `Bearer ${client.token}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
batch: [
|
||||
{
|
||||
eventId: "http-flat-batch-1",
|
||||
timestamp: "2026-11-19T12:10:00.000Z",
|
||||
clientId: client.clientId,
|
||||
sensor: "temp",
|
||||
value: 23.4,
|
||||
},
|
||||
{
|
||||
eventId: "http-env-batch-2",
|
||||
timestamp: "2026-11-19T12:10:01.000Z",
|
||||
data: { sensor: "humidity", value: 60 },
|
||||
},
|
||||
],
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(httpBatchRes.status, 200);
|
||||
const httpBatchJson = await httpBatchRes.json();
|
||||
assert.equal(httpBatchJson.processed, 2);
|
||||
|
||||
const localSnap = DataBufferEngine.getLocalBufferSnapshot();
|
||||
const flatEvt = localSnap.find((e) => e.eventId === "http-flat-batch-1");
|
||||
const envEvt = localSnap.find((e) => e.eventId === "http-env-batch-2");
|
||||
assert.ok(flatEvt);
|
||||
assert.ok(envEvt);
|
||||
assert.deepEqual(JSON.parse(flatEvt.payload), { sensor: "temp", value: 23.4 });
|
||||
assert.deepEqual(JSON.parse(envEvt.payload), { sensor: "humidity", value: 60 });
|
||||
|
||||
// 2. Socket.IO batch_sync handler
|
||||
const require = createRequire(import.meta.url);
|
||||
const { setupSocketHandlers } = require("../server/socket/handlers") as {
|
||||
setupSocketHandlers: (
|
||||
io: unknown,
|
||||
bufferQueue: DataEvent[],
|
||||
worker: unknown,
|
||||
options?: Record<string, unknown>,
|
||||
) => unknown;
|
||||
};
|
||||
let connectionHandler: ((socket: unknown) => void) | null = null;
|
||||
const fakeIo = {
|
||||
use: () => undefined,
|
||||
on: (evt: string, fn: (socket: unknown) => void) => {
|
||||
if (evt === "connection") connectionHandler = fn;
|
||||
},
|
||||
};
|
||||
const wsQueue: DataEvent[] = [];
|
||||
setupSocketHandlers(fakeIo, wsQueue, {}, {
|
||||
verifyClientInDb: async () => ({ allowed: true, rateLimit: 200 }),
|
||||
});
|
||||
assert.ok(connectionHandler);
|
||||
|
||||
const listeners = new Map<string, (...args: unknown[]) => Promise<void>>();
|
||||
const fakeSocket = {
|
||||
data: { clientId: "phase03_batch_strip", token: client.token, rateLimit: 200 },
|
||||
join: () => undefined,
|
||||
disconnect: () => undefined,
|
||||
on: (name: string, fn: (...args: unknown[]) => Promise<void>) => {
|
||||
listeners.set(name, fn);
|
||||
},
|
||||
};
|
||||
(connectionHandler as (socket: unknown) => void)(fakeSocket);
|
||||
|
||||
const batchSyncHandler = listeners.get("batch_sync");
|
||||
assert.ok(batchSyncHandler);
|
||||
let wsAck: Record<string, unknown> | null = null;
|
||||
await batchSyncHandler(
|
||||
[
|
||||
{
|
||||
eventId: "ws-flat-batch-1",
|
||||
timestamp: "2026-11-19T12:11:00.000Z",
|
||||
clientId: "phase03_batch_strip",
|
||||
sensor: "pressure",
|
||||
value: 101.2,
|
||||
},
|
||||
],
|
||||
(ack: Record<string, unknown>) => {
|
||||
wsAck = ack;
|
||||
},
|
||||
);
|
||||
assert.ok(wsAck);
|
||||
assert.equal((wsAck as Record<string, unknown>).success, true);
|
||||
assert.equal(wsQueue.length, 1);
|
||||
assert.deepEqual(JSON.parse(wsQueue[0].payload), {
|
||||
sensor: "pressure",
|
||||
value: 101.2,
|
||||
});
|
||||
});
|
||||
|
||||
test("R2 PHASE-03 BUG-12: DataBufferEngine.getLocalBufferSnapshot returns exact state.buffer contents for getPendingBufferedSummary", async () => {
|
||||
const prevIngestionUrl = process.env.INGESTION_INTERNAL_URL;
|
||||
process.env.INGESTION_INTERNAL_URL = "http://127.0.0.1:1";
|
||||
try {
|
||||
// Seed 1 unflushed RAM event directly in buffer while recentEvents contains a DIFFERENT newer flushed item
|
||||
await DataBufferEngine.pushAsync({
|
||||
clientId: "phase03_buf_exact",
|
||||
eventId: "real-unflushed-1",
|
||||
timestamp: "2026-11-19T13:00:00.000Z",
|
||||
payload: JSON.stringify({ state: "unflushed" }),
|
||||
});
|
||||
// Simulate a flushed event sitting at the front of recentEvents (which old getRecentEvents slice would wrongly return!)
|
||||
g.__dataStackBufferState!.recentEvents.unshift({
|
||||
clientId: "other_flushed_client",
|
||||
eventId: "already-flushed-99",
|
||||
timestamp: "2026-11-19T13:05:00.000Z",
|
||||
payload: JSON.stringify({ state: "already-flushed" }),
|
||||
});
|
||||
|
||||
const snap = DataBufferEngine.getLocalBufferSnapshot();
|
||||
assert.equal(snap.length, 1);
|
||||
assert.equal(snap[0].eventId, "real-unflushed-1");
|
||||
|
||||
const res = await getEventsRoute(
|
||||
new NextRequest("http://localhost:3000/api/events?bufferedOnly=true&limit=10", {
|
||||
headers: { cookie: adminCookieHeader },
|
||||
}),
|
||||
);
|
||||
assert.equal(res.status, 200);
|
||||
const json = await res.json();
|
||||
assert.equal(json.events.length, 1);
|
||||
assert.equal(json.events[0].eventId, "real-unflushed-1");
|
||||
assert.equal(json.bufferedSummary.total, 1);
|
||||
assert.equal(json.bufferedSummary.buckets[0].clientId, "phase03_buf_exact");
|
||||
} finally {
|
||||
if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL;
|
||||
else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl;
|
||||
}
|
||||
});
|
||||
|
||||
@@ -0,0 +1,850 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { NextRequest } from 'next/server';
|
||||
import {
|
||||
countMatchingBufferedEvents,
|
||||
DEFAULT_EDITABLE_SETTINGS,
|
||||
describeSettingsBridgeSync,
|
||||
extractContentDispositionFilename,
|
||||
formatEventTimeLabel,
|
||||
formatSessionExpiryLabel,
|
||||
getEventUtcDate,
|
||||
getUtcDateRangeForPreset,
|
||||
mapBridgeStatusToSocketUiState,
|
||||
matchesSelectedClientFolder,
|
||||
normalizeEditableSettings,
|
||||
parseEventTimestamp,
|
||||
resolvePublicEndpoints,
|
||||
SETTINGS_LIMITS,
|
||||
settingsToNumericDrafts,
|
||||
shouldRedirectProfileUnauthorized,
|
||||
summarizeHttpBatchResult,
|
||||
validateEditableSettingsDraft,
|
||||
validateSingleNumericSetting,
|
||||
} from '../lib/vconUiHelpers';
|
||||
import {
|
||||
createAdminSessionToken,
|
||||
getAdminSessionSecret,
|
||||
} from '../core/auth/adminSession';
|
||||
import {
|
||||
createInitialAdmin,
|
||||
resetAdminStateForTests,
|
||||
} from '../core/auth/adminUsers';
|
||||
import { DataBufferEngine } from '../core/data/buffer';
|
||||
import { SDKManager } from '../core/sdk/sdkManager';
|
||||
import { CSVStorageManager } from '../core/Storage/csvStorage';
|
||||
import { GET as profileGet, PUT as profilePut } from '../app/api/auth/profile/route';
|
||||
import { DELETE as deleteClientRoute } from '../app/api/clients/[id]/route';
|
||||
import { GET as eventsGet, POST as eventsPost } from '../app/api/events/route';
|
||||
import { POST as settingsPost } from '../app/api/settings/route';
|
||||
|
||||
test('BUG-D01: normalizeEditableSettings enforces exact /api/settings backend bounds', () => {
|
||||
const high = normalizeEditableSettings({
|
||||
maxEventsPerSec: 95000,
|
||||
maxPayloadSizeKb: 10240,
|
||||
batchFlushThreshold: 50000,
|
||||
batchFlushIntervalMs: 60000,
|
||||
fileRotationMb: 2048,
|
||||
enableIdempotency: false,
|
||||
autoPruneDays: 3650,
|
||||
});
|
||||
|
||||
assert.equal(high.maxEventsPerSec, 95000);
|
||||
assert.equal(high.maxPayloadSizeKb, 10240);
|
||||
assert.equal(high.batchFlushThreshold, 50000);
|
||||
assert.equal(high.batchFlushIntervalMs, 60000);
|
||||
assert.equal(high.fileRotationMb, 2048);
|
||||
assert.equal(high.enableIdempotency, false);
|
||||
assert.equal(high.autoPruneDays, 3650);
|
||||
|
||||
// Zero on positive-min fields clamps to min (1 / 100), while autoPruneDays keeps 0 ("never prune")
|
||||
const low = normalizeEditableSettings({
|
||||
maxEventsPerSec: 0,
|
||||
maxPayloadSizeKb: 0,
|
||||
batchFlushThreshold: 0,
|
||||
batchFlushIntervalMs: 10,
|
||||
fileRotationMb: 0,
|
||||
autoPruneDays: 0,
|
||||
});
|
||||
|
||||
assert.equal(low.maxEventsPerSec, SETTINGS_LIMITS.maxEventsPerSec.min);
|
||||
assert.equal(low.maxPayloadSizeKb, SETTINGS_LIMITS.maxPayloadSizeKb.min);
|
||||
assert.equal(low.batchFlushThreshold, SETTINGS_LIMITS.batchFlushThreshold.min);
|
||||
assert.equal(low.batchFlushIntervalMs, SETTINGS_LIMITS.batchFlushIntervalMs.min);
|
||||
assert.equal(low.fileRotationMb, SETTINGS_LIMITS.fileRotationMb.min);
|
||||
assert.equal(low.autoPruneDays, 0);
|
||||
|
||||
// Non-finite falls back to defaults
|
||||
const fallback = normalizeEditableSettings({
|
||||
maxEventsPerSec: Number.NaN,
|
||||
autoPruneDays: Number.NaN,
|
||||
});
|
||||
assert.equal(fallback.maxEventsPerSec, DEFAULT_EDITABLE_SETTINGS.maxEventsPerSec);
|
||||
assert.equal(fallback.autoPruneDays, DEFAULT_EDITABLE_SETTINGS.autoPruneDays);
|
||||
});
|
||||
|
||||
test('BUG-D01 & BUG-D04: bridgeSync feedback and socket UI state accurately distinguish online, auth_error, and offline', () => {
|
||||
assert.equal(describeSettingsBridgeSync(undefined).level, 'success');
|
||||
assert.equal(describeSettingsBridgeSync({ attempted: true, ok: true }).level, 'success');
|
||||
|
||||
const authFailure = describeSettingsBridgeSync({
|
||||
attempted: true,
|
||||
ok: false,
|
||||
status: 401,
|
||||
error: 'Unauthorized internal bridge token',
|
||||
});
|
||||
assert.equal(authFailure.level, 'error');
|
||||
assert.match(authFailure.title, /rejected sync/i);
|
||||
|
||||
const offlineFailure = describeSettingsBridgeSync({
|
||||
attempted: true,
|
||||
ok: false,
|
||||
error: 'connect ECONNREFUSED 127.0.0.1:4000',
|
||||
});
|
||||
assert.equal(offlineFailure.level, 'warning');
|
||||
assert.match(offlineFailure.title, /unreachable/i);
|
||||
|
||||
assert.equal(mapBridgeStatusToSocketUiState('online'), 'online');
|
||||
assert.equal(mapBridgeStatusToSocketUiState('auth_error'), 'auth_error');
|
||||
assert.equal(mapBridgeStatusToSocketUiState('offline'), 'offline');
|
||||
assert.equal(mapBridgeStatusToSocketUiState('error'), 'offline');
|
||||
assert.equal(mapBridgeStatusToSocketUiState(undefined), 'offline');
|
||||
});
|
||||
|
||||
test('BUG-D03: getUtcDateRangeForPreset computes deterministic UTC ranges across day boundaries', () => {
|
||||
const ref = new Date('2026-11-19T02:15:00.000Z');
|
||||
assert.deepEqual(getUtcDateRangeForPreset('today', ref), {
|
||||
from: '2026-11-19',
|
||||
to: '2026-11-19',
|
||||
});
|
||||
assert.deepEqual(getUtcDateRangeForPreset('7d', ref), {
|
||||
from: '2026-11-13',
|
||||
to: '2026-11-19',
|
||||
});
|
||||
assert.deepEqual(getUtcDateRangeForPreset('30d', ref), {
|
||||
from: '2026-10-21',
|
||||
to: '2026-11-19',
|
||||
});
|
||||
assert.deepEqual(getUtcDateRangeForPreset('all', ref), {
|
||||
from: '',
|
||||
to: '',
|
||||
});
|
||||
});
|
||||
|
||||
test('BUG-D03 & BUG-D05: timestamp formatting handles ISO & epoch-ms strings cleanly without Invalid Date', () => {
|
||||
const isoParsed = parseEventTimestamp('2026-11-18T12:34:56.000Z');
|
||||
assert.ok(isoParsed instanceof Date);
|
||||
assert.equal(isoParsed.toISOString(), '2026-11-18T12:34:56.000Z');
|
||||
|
||||
const epochParsed = parseEventTimestamp('1700000000000');
|
||||
assert.ok(epochParsed instanceof Date);
|
||||
assert.equal(epochParsed.getTime(), 1700000000000);
|
||||
assert.notEqual(formatEventTimeLabel('1700000000000'), 'Invalid Date');
|
||||
assert.notEqual(formatEventTimeLabel('1700000000000'), '1700000000000');
|
||||
|
||||
assert.equal(parseEventTimestamp('not-a-date'), null);
|
||||
assert.equal(formatEventTimeLabel('not-a-date'), 'not-a-date');
|
||||
assert.equal(formatEventTimeLabel(''), '—');
|
||||
});
|
||||
|
||||
test('BUG-D03 & BUG-D06: matchesSelectedClientFolder and Content-Disposition parsing handle sanitized folders and quoted filenames', () => {
|
||||
assert.equal(matchesSelectedClientFolder('sensor_01', 'all'), true);
|
||||
assert.equal(matchesSelectedClientFolder('sensor_01', 'sensor_01'), true);
|
||||
assert.equal(matchesSelectedClientFolder('client_dev', 'client@dev'), true);
|
||||
assert.equal(matchesSelectedClientFolder('other_client', 'sensor_01'), false);
|
||||
|
||||
assert.equal(
|
||||
extractContentDispositionFilename(
|
||||
'attachment; filename="datastack_export_sensor_01_2026-11-18.json"',
|
||||
'fallback.json'
|
||||
),
|
||||
'datastack_export_sensor_01_2026-11-18.json'
|
||||
);
|
||||
assert.equal(
|
||||
extractContentDispositionFilename(null, 'fallback.csv'),
|
||||
'fallback.csv'
|
||||
);
|
||||
});
|
||||
|
||||
test('FORENSIC BUG-09: parseEventTimestamp and getEventUtcDate handle integer and fractional epoch-seconds with backend parity', () => {
|
||||
const intSec = parseEventTimestamp('1735689600');
|
||||
assert.ok(intSec instanceof Date);
|
||||
assert.equal(intSec.toISOString(), '2025-01-01T00:00:00.000Z');
|
||||
assert.equal(getEventUtcDate('1735689600'), '2025-01-01');
|
||||
assert.notEqual(formatEventTimeLabel('1735689600'), '1735689600');
|
||||
|
||||
const floatSec = parseEventTimestamp('1735689600.25');
|
||||
assert.ok(floatSec instanceof Date);
|
||||
assert.equal(floatSec.toISOString(), '2025-01-01T00:00:00.250Z');
|
||||
assert.equal(getEventUtcDate('1735689600.25'), '2025-01-01');
|
||||
assert.notEqual(formatEventTimeLabel('1735689600.25'), '1735689600.25');
|
||||
|
||||
const msVal = parseEventTimestamp('1735689600500');
|
||||
assert.ok(msVal instanceof Date);
|
||||
assert.equal(msVal.toISOString(), '2025-01-01T00:00:00.500Z');
|
||||
});
|
||||
|
||||
test('FORENSIC BUG-07: POST /api/events ingests batch and events arrays cleanly with accepted count and Simulator never falsifies counts', async () => {
|
||||
const prevStorageDir = process.env.DATASTACK_STORAGE_DIR;
|
||||
const origBaseStorageDir = CSVStorageManager.baseStorageDir;
|
||||
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'datastack-p5-bug07-'));
|
||||
process.env.DATASTACK_STORAGE_DIR = tmpDir;
|
||||
CSVStorageManager.baseStorageDir = tmpDir;
|
||||
await DataBufferEngine.shutdown();
|
||||
DataBufferEngine.setRedisClientForTests(null);
|
||||
|
||||
try {
|
||||
const client = await SDKManager.registerClientAsync({
|
||||
clientId: 'p5_sim_client',
|
||||
name: 'Phase 05 Simulator Client',
|
||||
rateLimit: 500,
|
||||
});
|
||||
const ts = new Date().toISOString();
|
||||
|
||||
// 1. Canonical { clientId, batch } payload (10 individual events)
|
||||
const batchItems = Array.from({ length: 10 }, (_, idx) => ({
|
||||
eventId: `p5_batch_${Date.now()}_${idx}`,
|
||||
timestamp: ts,
|
||||
data: { batch_index: idx + 1, sensor: 's1' },
|
||||
}));
|
||||
const reqBatch = new NextRequest('http://localhost:3000/api/events', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
authorization: `Bearer ${client.token}`,
|
||||
},
|
||||
body: JSON.stringify({ clientId: client.clientId, batch: batchItems }),
|
||||
});
|
||||
const resBatch = await eventsPost(reqBatch);
|
||||
assert.equal(resBatch.status, 200);
|
||||
const batchJson = await resBatch.json();
|
||||
assert.equal(batchJson.success, true);
|
||||
assert.equal(batchJson.accepted, 10);
|
||||
assert.equal(batchJson.processed, 10);
|
||||
assert.equal(batchJson.rejected, 0);
|
||||
|
||||
// 2. Alias { clientId, events } payload also ingests individual events (never 1 wrapper event)
|
||||
const aliasItems = Array.from({ length: 4 }, (_, idx) => ({
|
||||
eventId: `p5_alias_${Date.now()}_${idx}`,
|
||||
timestamp: ts,
|
||||
data: { alias_index: idx + 1 },
|
||||
}));
|
||||
const reqAlias = new NextRequest('http://localhost:3000/api/events', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
authorization: `Bearer ${client.token}`,
|
||||
},
|
||||
body: JSON.stringify({ clientId: client.clientId, events: aliasItems }),
|
||||
});
|
||||
const resAlias = await eventsPost(reqAlias);
|
||||
assert.equal(resAlias.status, 200);
|
||||
const aliasJson = await resAlias.json();
|
||||
assert.equal(aliasJson.accepted, 4);
|
||||
assert.equal(aliasJson.processed, 4);
|
||||
|
||||
// 3. summarizeHttpBatchResult never falsifies accepted=10 when server omits accepted/processed
|
||||
const fakeSingleRes = summarizeHttpBatchResult(
|
||||
true,
|
||||
{ success: true, eventId: 'evt_wrapper', event: {} },
|
||||
10,
|
||||
);
|
||||
assert.equal(fakeSingleRes.accepted, 0);
|
||||
assert.equal(fakeSingleRes.rejected, 10);
|
||||
assert.equal(fakeSingleRes.status, 'Rejected');
|
||||
assert.equal(fakeSingleRes.level, 'error');
|
||||
|
||||
const partialRes = summarizeHttpBatchResult(
|
||||
true,
|
||||
{ success: true, accepted: 7, processed: 7, rejected: 3, duplicates: 0 },
|
||||
10,
|
||||
);
|
||||
assert.equal(partialRes.accepted, 7);
|
||||
assert.equal(partialRes.rejected, 3);
|
||||
assert.equal(partialRes.status, 'Rejected');
|
||||
assert.equal(partialRes.level, 'warning');
|
||||
|
||||
const okRes = summarizeHttpBatchResult(true, batchJson, 10);
|
||||
assert.equal(okRes.accepted, 10);
|
||||
assert.equal(okRes.rejected, 0);
|
||||
assert.equal(okRes.status, 'Accepted');
|
||||
assert.equal(okRes.level, 'success');
|
||||
|
||||
// Verify Simulator source sends canonical `batch:` and uses summarizeHttpBatchResult
|
||||
const simSrc = await fs.readFile(
|
||||
path.join(process.cwd(), 'app', 'vcon', 'simulator', 'page.tsx'),
|
||||
'utf-8',
|
||||
);
|
||||
assert.match(simSrc, /batch:\s*batchPayload/);
|
||||
assert.match(simSrc, /summarizeHttpBatchResult\(/);
|
||||
assert.doesNotMatch(simSrc, /res\.ok\s*\?\s*count\s*:\s*0/);
|
||||
} finally {
|
||||
await DataBufferEngine.flush();
|
||||
await DataBufferEngine.shutdown();
|
||||
CSVStorageManager.baseStorageDir = origBaseStorageDir;
|
||||
if (prevStorageDir === undefined) delete process.env.DATASTACK_STORAGE_DIR;
|
||||
else process.env.DATASTACK_STORAGE_DIR = prevStorageDir;
|
||||
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('FORENSIC BUG-08 & BUG-14: filtered buffered matching, post-flush zeroing, UTC-today attribution, and multi-client GET /api/events scan beyond 5 files', async () => {
|
||||
const prevStorageDir = process.env.DATASTACK_STORAGE_DIR;
|
||||
const prevIngestionUrl = process.env.INGESTION_INTERNAL_URL;
|
||||
const origBaseStorageDir = CSVStorageManager.baseStorageDir;
|
||||
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'datastack-p5-bug08-14-'));
|
||||
process.env.DATASTACK_STORAGE_DIR = tmpDir;
|
||||
CSVStorageManager.baseStorageDir = tmpDir;
|
||||
process.env.INGESTION_INTERNAL_URL = 'http://127.0.0.1:1';
|
||||
await DataBufferEngine.shutdown();
|
||||
DataBufferEngine.setRedisClientForTests(null);
|
||||
|
||||
try {
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
resetAdminStateForTests();
|
||||
await createInitialAdmin('admin@datastack.dev', 'Phase05Pass123!');
|
||||
const secret = getAdminSessionSecret()!;
|
||||
const adminToken = await createAdminSessionToken(
|
||||
'admin@datastack.dev',
|
||||
secret,
|
||||
Date.now() + 3600_000,
|
||||
1,
|
||||
);
|
||||
|
||||
// 1. Pure filtered buffer matcher prevents silent empty export downloads on non-matching client/date slices
|
||||
const syntheticSummary = {
|
||||
total: 8,
|
||||
totalQueued: 8,
|
||||
unattributedCount: 0,
|
||||
truncated: false,
|
||||
buckets: [
|
||||
{ clientId: 'alpha_client', date: today, count: 3 },
|
||||
{ clientId: 'alpha_client', date: '2025-01-01', count: 5 },
|
||||
],
|
||||
};
|
||||
|
||||
const otherClientMatch = countMatchingBufferedEvents(syntheticSummary, {
|
||||
clientId: 'beta_client',
|
||||
});
|
||||
assert.equal(otherClientMatch.matchingCount, 0);
|
||||
assert.equal(otherClientMatch.hasPotentialBuffered, false);
|
||||
|
||||
const outOfRangeMatch = countMatchingBufferedEvents(syntheticSummary, {
|
||||
clientId: 'alpha_client',
|
||||
fromDate: '2024-01-01',
|
||||
toDate: '2024-12-31',
|
||||
});
|
||||
assert.equal(outOfRangeMatch.matchingCount, 0);
|
||||
assert.equal(outOfRangeMatch.hasPotentialBuffered, false);
|
||||
|
||||
const todayOnlyMatch = countMatchingBufferedEvents(syntheticSummary, {
|
||||
clientId: 'all',
|
||||
fromDate: today,
|
||||
toDate: today,
|
||||
});
|
||||
assert.equal(todayOnlyMatch.matchingCount, 3);
|
||||
assert.equal(todayOnlyMatch.hasPotentialBuffered, true);
|
||||
|
||||
// 2. Write CSV partitions for 8 distinct clients (plus 6 extra dated files on client_01) to prove BUG-14 scan exceeds old 5-file cap
|
||||
for (let d = 1; d <= 6; d += 1) {
|
||||
await CSVStorageManager.appendBatch([
|
||||
{
|
||||
clientId: 'p5_client_01',
|
||||
eventId: `evt_c1_old_${d}`,
|
||||
timestamp: `2026-10-0${d}T10:00:00.000Z`,
|
||||
payload: JSON.stringify({ n: d }),
|
||||
},
|
||||
]);
|
||||
}
|
||||
for (let i = 1; i <= 8; i += 1) {
|
||||
const cid = `p5_client_0${i}`;
|
||||
await CSVStorageManager.appendBatch([
|
||||
{
|
||||
clientId: cid,
|
||||
eventId: `evt_${cid}_latest`,
|
||||
timestamp: `${today}T12:00:0${i}.000Z`,
|
||||
payload: JSON.stringify({ client: cid }),
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
// 3. Push 2 UTC-today buffered events + 3 backdated buffered events into RAM buffer
|
||||
await DataBufferEngine.pushBatchAsync([
|
||||
{
|
||||
clientId: 'p5_buf_client',
|
||||
eventId: 'buf_today_1',
|
||||
timestamp: `${today}T13:00:01.000Z`,
|
||||
payload: '{"kind":"today"}',
|
||||
},
|
||||
{
|
||||
clientId: 'p5_buf_client',
|
||||
eventId: 'buf_today_2',
|
||||
timestamp: `${today}T13:00:02.000Z`,
|
||||
payload: '{"kind":"today"}',
|
||||
},
|
||||
{
|
||||
clientId: 'p5_buf_client',
|
||||
eventId: 'buf_old_1',
|
||||
timestamp: '2025-02-01T08:00:00.000Z',
|
||||
payload: '{"kind":"backdated"}',
|
||||
},
|
||||
{
|
||||
clientId: 'p5_buf_client',
|
||||
eventId: 'buf_old_2',
|
||||
timestamp: '2025-02-01T08:00:01.000Z',
|
||||
payload: '{"kind":"backdated"}',
|
||||
},
|
||||
{
|
||||
clientId: 'p5_buf_client',
|
||||
eventId: 'buf_old_3',
|
||||
timestamp: '2025-02-01T08:00:02.000Z',
|
||||
payload: '{"kind":"backdated"}',
|
||||
},
|
||||
]);
|
||||
|
||||
const reqAll = new NextRequest(
|
||||
'http://localhost:3000/api/events?limit=25&includeBufferedSummary=true',
|
||||
{
|
||||
headers: { cookie: `datastack_admin_session=${adminToken}` },
|
||||
},
|
||||
);
|
||||
const resAll = await eventsGet(reqAll);
|
||||
assert.equal(resAll.status, 200);
|
||||
const allJson = await resAll.json();
|
||||
|
||||
// All 8 disk clients must be represented in GET /api/events (past the old 5-file global cap)
|
||||
const seenClients = new Set<string>(
|
||||
(allJson.events as Array<{ clientId: string }>).map((e) => e.clientId),
|
||||
);
|
||||
for (let i = 1; i <= 8; i += 1) {
|
||||
assert.ok(
|
||||
seenClients.has(`p5_client_0${i}`),
|
||||
`Expected p5_client_0${i} to be visible in GET /api/events multi-client scan`,
|
||||
);
|
||||
}
|
||||
|
||||
// bufferedSummary accurately separates 2 today events from 3 backdated events
|
||||
assert.ok(allJson.bufferedSummary);
|
||||
assert.equal(allJson.bufferedSummary.total, 5);
|
||||
const todayBuffered = countMatchingBufferedEvents(allJson.bufferedSummary, {
|
||||
clientId: 'all',
|
||||
fromDate: today,
|
||||
toDate: today,
|
||||
});
|
||||
assert.equal(todayBuffered.matchingCount, 2);
|
||||
|
||||
// After flush, GET /api/events?bufferedOnly=true drops to 0 (never double-counting flushed recentEvents)
|
||||
await DataBufferEngine.flush();
|
||||
const reqBufAfterFlush = new NextRequest(
|
||||
'http://localhost:3000/api/events?bufferedOnly=true&limit=1',
|
||||
{
|
||||
headers: { cookie: `datastack_admin_session=${adminToken}` },
|
||||
},
|
||||
);
|
||||
const resBufAfterFlush = await eventsGet(reqBufAfterFlush);
|
||||
assert.equal(resBufAfterFlush.status, 200);
|
||||
const bufAfterJson = await resBufAfterFlush.json();
|
||||
assert.equal(bufAfterJson.bufferedSummary.total, 0);
|
||||
assert.equal(bufAfterJson.bufferedSummary.buckets.length, 0);
|
||||
|
||||
// Verify ExportWorkspace & DashboardOverview refresh caches after export
|
||||
const exportSrc = await fs.readFile(
|
||||
path.join(process.cwd(), 'components', 'sections', 'ExportWorkspace.tsx'),
|
||||
'utf-8',
|
||||
);
|
||||
assert.match(exportSrc, /countMatchingBufferedEvents\(/);
|
||||
assert.match(exportSrc, /await refreshExportData\(\)/);
|
||||
|
||||
const dashSrc = await fs.readFile(
|
||||
path.join(process.cwd(), 'components', 'sections', 'DashboardOverview.tsx'),
|
||||
'utf-8',
|
||||
);
|
||||
assert.match(dashSrc, /includeBufferedSummary=true/);
|
||||
assert.match(dashSrc, /await refresh\(\)/);
|
||||
} finally {
|
||||
await DataBufferEngine.flush();
|
||||
await DataBufferEngine.shutdown();
|
||||
CSVStorageManager.baseStorageDir = origBaseStorageDir;
|
||||
if (prevStorageDir === undefined) delete process.env.DATASTACK_STORAGE_DIR;
|
||||
else process.env.DATASTACK_STORAGE_DIR = prevStorageDir;
|
||||
if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL;
|
||||
else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl;
|
||||
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('FORENSIC BUG-10: /api/auth/profile returns explicit SESSION_EXPIRED vs INVALID_PASSWORD codes and ProfileWorkspace formats full expiry date+time', async () => {
|
||||
const prevDbUrl = process.env.DATABASE_URL;
|
||||
const prevDemoFlag = process.env.DATASTACK_ALLOW_DEMO_ADMIN;
|
||||
delete process.env.DATABASE_URL;
|
||||
process.env.DATASTACK_ALLOW_DEMO_ADMIN = 'false';
|
||||
|
||||
try {
|
||||
resetAdminStateForTests();
|
||||
await createInitialAdmin('p6-admin@datastack.dev', 'Phase06Pass123!');
|
||||
const secret = getAdminSessionSecret()!;
|
||||
const validToken = await createAdminSessionToken(
|
||||
'p6-admin@datastack.dev',
|
||||
secret,
|
||||
Date.now() + 30 * 86_400_000,
|
||||
1,
|
||||
);
|
||||
|
||||
// 1. Missing/expired session on GET & PUT returns 401 with code: SESSION_EXPIRED
|
||||
const unauthGet = await profileGet(
|
||||
new NextRequest('http://localhost:3000/api/auth/profile'),
|
||||
);
|
||||
assert.equal(unauthGet.status, 401);
|
||||
const unauthGetJson = await unauthGet.json();
|
||||
assert.equal(unauthGetJson.code, 'SESSION_EXPIRED');
|
||||
|
||||
const unauthPut = await profilePut(
|
||||
new NextRequest('http://localhost:3000/api/auth/profile', {
|
||||
method: 'PUT',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
email: 'p6-admin@datastack.dev',
|
||||
currentPassword: 'Phase06Pass123!',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(unauthPut.status, 401);
|
||||
const unauthPutJson = await unauthPut.json();
|
||||
assert.equal(unauthPutJson.code, 'SESSION_EXPIRED');
|
||||
assert.equal(shouldRedirectProfileUnauthorized(401, unauthPutJson), true);
|
||||
|
||||
// 2. Wrong currentPassword on valid session returns 401 with code: INVALID_PASSWORD (no redirect)
|
||||
const wrongPassRes = await profilePut(
|
||||
new NextRequest('http://localhost:3000/api/auth/profile', {
|
||||
method: 'PUT',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
cookie: `datastack_admin_session=${validToken}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
email: 'p6-admin@datastack.dev',
|
||||
currentPassword: 'WrongPassword999!',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(wrongPassRes.status, 401);
|
||||
const wrongPassJson = await wrongPassRes.json();
|
||||
assert.equal(wrongPassJson.code, 'INVALID_PASSWORD');
|
||||
assert.equal(shouldRedirectProfileUnauthorized(401, wrongPassJson), false);
|
||||
|
||||
// 3. Session expiry label includes calendar date (not time-only)
|
||||
const expiryLabel = formatSessionExpiryLabel('2026-12-19T14:30:00.000Z');
|
||||
assert.match(expiryLabel, /\d{1,2}/);
|
||||
assert.ok(
|
||||
expiryLabel.length > 5 && /[A-Za-z]|\d{1,2}\/\d{1,2}/.test(expiryLabel),
|
||||
`Expected date+time in session expiry label, got: ${expiryLabel}`,
|
||||
);
|
||||
assert.equal(formatSessionExpiryLabel(''), 'Active');
|
||||
|
||||
// 4. Dead code `response.status === 401 && !currentPassword` is removed from ProfileWorkspace
|
||||
const profileUiSrc = await fs.readFile(
|
||||
path.join(process.cwd(), 'components', 'sections', 'ProfileWorkspace.tsx'),
|
||||
'utf-8',
|
||||
);
|
||||
assert.doesNotMatch(profileUiSrc, /response\.status === 401 && !currentPassword/);
|
||||
assert.match(profileUiSrc, /shouldRedirectProfileUnauthorized\(/);
|
||||
assert.match(profileUiSrc, /formatSessionExpiryLabel\(/);
|
||||
} finally {
|
||||
if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
|
||||
else process.env.DATABASE_URL = prevDbUrl;
|
||||
if (prevDemoFlag === undefined) delete process.env.DATASTACK_ALLOW_DEMO_ADMIN;
|
||||
else process.env.DATASTACK_ALLOW_DEMO_ADMIN = prevDemoFlag;
|
||||
resetAdminStateForTests();
|
||||
}
|
||||
});
|
||||
|
||||
test('FORENSIC BUG-11: ClientsTable exposes ?purgeFiles=true opt-in checkbox on delete and omits unused Secret key row', async () => {
|
||||
const clientsTableSrc = await fs.readFile(
|
||||
path.join(process.cwd(), 'components', 'sections', 'ClientsTable.tsx'),
|
||||
'utf-8',
|
||||
);
|
||||
assert.match(clientsTableSrc, /\?purgeFiles=true/);
|
||||
assert.match(clientsTableSrc, /Also purge stored CSV files for this client/);
|
||||
assert.doesNotMatch(clientsTableSrc, /label="Secret key"/);
|
||||
});
|
||||
|
||||
test('FORENSIC BUG-12: Settings numeric drafts preserve empty inputs and report explicit validation errors without silent clamping', async () => {
|
||||
const validDrafts = settingsToNumericDrafts(DEFAULT_EDITABLE_SETTINGS);
|
||||
const okResult = validateEditableSettingsDraft(validDrafts, true);
|
||||
assert.equal(okResult.valid, true);
|
||||
assert.deepEqual(okResult.values, DEFAULT_EDITABLE_SETTINGS);
|
||||
|
||||
// Empty string returns explicit required error (never resets to 0)
|
||||
const emptyCheck = validateSingleNumericSetting('maxEventsPerSec', '');
|
||||
assert.equal(emptyCheck.valid, false);
|
||||
assert.match(emptyCheck.error, /required/i);
|
||||
|
||||
// Out-of-bounds value returns explicit range error (never silently clamped)
|
||||
const invalidDrafts = {
|
||||
...validDrafts,
|
||||
batchFlushIntervalMs: '50',
|
||||
fileRotationMb: '99999',
|
||||
};
|
||||
const badResult = validateEditableSettingsDraft(invalidDrafts, false);
|
||||
assert.equal(badResult.valid, false);
|
||||
assert.equal(badResult.firstInvalidKey, 'batchFlushIntervalMs');
|
||||
assert.match(badResult.errors.batchFlushIntervalMs || '', /100/);
|
||||
assert.match(badResult.errors.fileRotationMb || '', /2,048|2048/);
|
||||
|
||||
// autoPruneDays = "0" ("never prune") remains explicitly valid
|
||||
const zeroPrune = validateSingleNumericSetting('autoPruneDays', '0');
|
||||
assert.equal(zeroPrune.valid, true);
|
||||
if (zeroPrune.valid) assert.equal(zeroPrune.value, 0);
|
||||
|
||||
const settingsUiSrc = await fs.readFile(
|
||||
path.join(process.cwd(), 'components', 'sections', 'SettingsWorkspace.tsx'),
|
||||
'utf-8',
|
||||
);
|
||||
assert.match(settingsUiSrc, /validateEditableSettingsDraft\(/);
|
||||
assert.match(settingsUiSrc, /Socket port \(PORT_SOCKET\)/);
|
||||
assert.match(settingsUiSrc, /noValidate/);
|
||||
});
|
||||
|
||||
test('FORENSIC BUG-13: resolvePublicEndpoints unifies Settings, Simulator, and SDK Guide URLs across direct dev and port-80 localhost proxies', async () => {
|
||||
// 1. Direct Next.js dev on :3000 maps ingestion to :serverPort
|
||||
const directDev = resolvePublicEndpoints(
|
||||
{
|
||||
protocol: 'http:',
|
||||
hostname: 'localhost',
|
||||
host: 'localhost:3000',
|
||||
port: '3000',
|
||||
origin: 'http://localhost:3000',
|
||||
},
|
||||
4000,
|
||||
);
|
||||
assert.deepEqual(directDev, {
|
||||
apiUrl: 'http://localhost:3000',
|
||||
serverUrl: 'http://localhost:4000',
|
||||
webSocketUrl: 'ws://localhost:4000',
|
||||
});
|
||||
|
||||
// 2. Reverse-proxied localhost on port 80 stays same-origin (no bogus :4000)
|
||||
const localProxy = resolvePublicEndpoints(
|
||||
{
|
||||
protocol: 'http:',
|
||||
hostname: 'localhost',
|
||||
host: 'localhost',
|
||||
port: '',
|
||||
origin: 'http://localhost',
|
||||
},
|
||||
4000,
|
||||
);
|
||||
assert.deepEqual(localProxy, {
|
||||
apiUrl: 'http://localhost',
|
||||
serverUrl: 'http://localhost',
|
||||
webSocketUrl: 'ws://localhost',
|
||||
});
|
||||
|
||||
// 3. HTTPS production domain uses wss:// and same-origin
|
||||
const prodHttps = resolvePublicEndpoints(
|
||||
{
|
||||
protocol: 'https:',
|
||||
hostname: 'data.example.com',
|
||||
host: 'data.example.com',
|
||||
port: '',
|
||||
origin: 'https://data.example.com',
|
||||
},
|
||||
4000,
|
||||
);
|
||||
assert.deepEqual(prodHttps, {
|
||||
apiUrl: 'https://data.example.com',
|
||||
serverUrl: 'https://data.example.com',
|
||||
webSocketUrl: 'wss://data.example.com',
|
||||
});
|
||||
|
||||
// All 3 UI pages import and use resolvePublicEndpoints
|
||||
for (const rel of [
|
||||
path.join('components', 'sections', 'SettingsWorkspace.tsx'),
|
||||
path.join('app', 'vcon', 'sdk', 'page.tsx'),
|
||||
path.join('app', 'vcon', 'simulator', 'page.tsx'),
|
||||
]) {
|
||||
const src = await fs.readFile(path.join(process.cwd(), rel), 'utf-8');
|
||||
assert.match(src, /resolvePublicEndpoints\(/, `${rel} must use resolvePublicEndpoints`);
|
||||
}
|
||||
});
|
||||
|
||||
test('R2 PHASE-03 BUG-03: POST /api/settings returns normalized bridgeSync contract and describeSettingsBridgeSync handles both synced and attempted/ok shapes', async () => {
|
||||
const prevDbUrl = process.env.DATABASE_URL;
|
||||
const prevIngestionUrl = process.env.INGESTION_INTERNAL_URL;
|
||||
delete process.env.DATABASE_URL;
|
||||
process.env.INGESTION_INTERNAL_URL = 'http://127.0.0.1:1';
|
||||
|
||||
try {
|
||||
resetAdminStateForTests();
|
||||
await createInitialAdmin('p3-settings@datastack.dev', 'Phase03Pass123!');
|
||||
const secret = getAdminSessionSecret()!;
|
||||
const adminToken = await createAdminSessionToken(
|
||||
'p3-settings@datastack.dev',
|
||||
secret,
|
||||
Date.now() + 3600_000,
|
||||
1,
|
||||
);
|
||||
|
||||
const res = await settingsPost(
|
||||
new NextRequest('http://localhost:3000/api/settings', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
cookie: `datastack_admin_session=${adminToken}`,
|
||||
},
|
||||
body: JSON.stringify({ maxEventsPerSec: 150 }),
|
||||
}),
|
||||
);
|
||||
assert.equal(res.status, 200);
|
||||
const json = await res.json();
|
||||
assert.ok(json.bridgeSync, 'POST /api/settings must return bridgeSync status');
|
||||
assert.equal(json.bridgeSync.attempted, true);
|
||||
assert.equal(json.bridgeSync.ok, false);
|
||||
assert.equal(json.bridgeSync.synced, false);
|
||||
|
||||
// describeSettingsBridgeSync must produce a warning for offline bridgeSync from backend
|
||||
const uiFeedback = describeSettingsBridgeSync(json.bridgeSync);
|
||||
assert.equal(uiFeedback.level, 'warning');
|
||||
assert.match(uiFeedback.title, /unreachable/i);
|
||||
|
||||
// Also accepts legacy { synced } shape directly without attempted/ok
|
||||
assert.equal(describeSettingsBridgeSync({ synced: true }).level, 'success');
|
||||
assert.equal(
|
||||
describeSettingsBridgeSync({ synced: false, status: 401 }).level,
|
||||
'error',
|
||||
);
|
||||
assert.equal(
|
||||
describeSettingsBridgeSync({ synced: false, error: 'timeout' }).level,
|
||||
'warning',
|
||||
);
|
||||
|
||||
const settingsUiSrc = await fs.readFile(
|
||||
path.join(process.cwd(), 'components', 'sections', 'SettingsWorkspace.tsx'),
|
||||
'utf-8',
|
||||
);
|
||||
assert.match(settingsUiSrc, /data\.bridgeSync\?\.ok \?\? data\.bridgeSync\?\.synced/);
|
||||
} finally {
|
||||
await DataBufferEngine.shutdown();
|
||||
if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
|
||||
else process.env.DATABASE_URL = prevDbUrl;
|
||||
if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL;
|
||||
else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl;
|
||||
resetAdminStateForTests();
|
||||
}
|
||||
});
|
||||
|
||||
test('R2 PHASE-03 BUG-04: DELETE /api/clients/[id]?purgeFiles=true returns exact integer purgedFiles count', async () => {
|
||||
const prevDbUrl = process.env.DATABASE_URL;
|
||||
const prevStorageDir = process.env.DATASTACK_STORAGE_DIR;
|
||||
const origBaseStorageDir = CSVStorageManager.baseStorageDir;
|
||||
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'datastack-r2p3-bug04-'));
|
||||
delete process.env.DATABASE_URL;
|
||||
process.env.DATASTACK_STORAGE_DIR = tmpDir;
|
||||
CSVStorageManager.baseStorageDir = tmpDir;
|
||||
|
||||
try {
|
||||
resetAdminStateForTests();
|
||||
await createInitialAdmin('p3-purge@datastack.dev', 'Phase03Pass123!');
|
||||
const secret = getAdminSessionSecret()!;
|
||||
const adminToken = await createAdminSessionToken(
|
||||
'p3-purge@datastack.dev',
|
||||
secret,
|
||||
Date.now() + 3600_000,
|
||||
1,
|
||||
);
|
||||
|
||||
const clientPurge = await SDKManager.registerClientAsync({
|
||||
clientId: 'p3_purge_me',
|
||||
name: 'Purge Me',
|
||||
rateLimit: 100,
|
||||
});
|
||||
await CSVStorageManager.appendBatch([
|
||||
{
|
||||
clientId: clientPurge.clientId,
|
||||
eventId: 'purge-e1',
|
||||
timestamp: '2026-11-18T10:00:00.000Z',
|
||||
payload: '{"a":1}',
|
||||
},
|
||||
{
|
||||
clientId: clientPurge.clientId,
|
||||
eventId: 'purge-e2',
|
||||
timestamp: '2026-11-19T10:00:00.000Z',
|
||||
payload: '{"a":2}',
|
||||
},
|
||||
]);
|
||||
|
||||
const delPurgeRes = await deleteClientRoute(
|
||||
new NextRequest(
|
||||
`http://localhost:3000/api/clients/${clientPurge.clientId}?purgeFiles=true`,
|
||||
{
|
||||
method: 'DELETE',
|
||||
headers: { cookie: `datastack_admin_session=${adminToken}` },
|
||||
},
|
||||
),
|
||||
{ params: Promise.resolve({ id: clientPurge.clientId }) },
|
||||
);
|
||||
assert.equal(delPurgeRes.status, 200);
|
||||
const delPurgeJson = await delPurgeRes.json();
|
||||
assert.equal(delPurgeJson.success, true);
|
||||
assert.equal(delPurgeJson.purgedFiles, 2);
|
||||
|
||||
// Deleting without ?purgeFiles=true returns purgedFiles: 0 and retains CSV files
|
||||
const clientKeep = await SDKManager.registerClientAsync({
|
||||
clientId: 'p3_keep_files',
|
||||
name: 'Keep Files',
|
||||
rateLimit: 100,
|
||||
});
|
||||
await CSVStorageManager.appendBatch([
|
||||
{
|
||||
clientId: clientKeep.clientId,
|
||||
eventId: 'keep-e1',
|
||||
timestamp: '2026-11-19T10:00:00.000Z',
|
||||
payload: '{"keep":true}',
|
||||
},
|
||||
]);
|
||||
const delKeepRes = await deleteClientRoute(
|
||||
new NextRequest(
|
||||
`http://localhost:3000/api/clients/${clientKeep.clientId}`,
|
||||
{
|
||||
method: 'DELETE',
|
||||
headers: { cookie: `datastack_admin_session=${adminToken}` },
|
||||
},
|
||||
),
|
||||
{ params: Promise.resolve({ id: clientKeep.clientId }) },
|
||||
);
|
||||
assert.equal(delKeepRes.status, 200);
|
||||
const delKeepJson = await delKeepRes.json();
|
||||
assert.equal(delKeepJson.purgedFiles, 0);
|
||||
const remaining = await CSVStorageManager.listAllFilesAsync();
|
||||
assert.equal(
|
||||
remaining.filter((f) => f.clientId === clientKeep.clientId).length,
|
||||
1,
|
||||
);
|
||||
} finally {
|
||||
await DataBufferEngine.shutdown();
|
||||
CSVStorageManager.baseStorageDir = origBaseStorageDir;
|
||||
if (prevStorageDir === undefined) delete process.env.DATASTACK_STORAGE_DIR;
|
||||
else process.env.DATASTACK_STORAGE_DIR = prevStorageDir;
|
||||
if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
|
||||
else process.env.DATABASE_URL = prevDbUrl;
|
||||
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||
resetAdminStateForTests();
|
||||
}
|
||||
});
|
||||
|
||||
test('R2 PHASE-03 BUG-15: /vcon/sdk clears previous client JWT synchronously on switch and guards config copy while loadingToken is true', async () => {
|
||||
const sdkSrc = await fs.readFile(
|
||||
path.join(process.cwd(), 'app', 'vcon', 'sdk', 'page.tsx'),
|
||||
'utf-8',
|
||||
);
|
||||
assert.match(sdkSrc, /setSelectedClientToken\(''\);\s*\n\s*setLoadingToken\(true\);/);
|
||||
const disabledMatches = sdkSrc.match(/disabled=\{loadingToken\}/g) || [];
|
||||
assert.ok(
|
||||
disabledMatches.length >= 3,
|
||||
`Expected Copy Token and both Copy Config buttons to be guarded with disabled={loadingToken}, found ${disabledMatches.length}`,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
|
||||
const ROOT = process.cwd();
|
||||
|
||||
test('BUG-E01 & BUG-E03: Canonical datastack and compat datastack-sdk packaging metadata, versions, and py.typed align', () => {
|
||||
const pyproject = fs.readFileSync(
|
||||
path.join(ROOT, 'sdk/python/datastack/pyproject.toml'),
|
||||
'utf8',
|
||||
);
|
||||
const canonicalSetup = fs.readFileSync(
|
||||
path.join(ROOT, 'sdk/python/datastack/setup.py'),
|
||||
'utf8',
|
||||
);
|
||||
const compatSetup = fs.readFileSync(
|
||||
path.join(ROOT, 'sdk-python/setup.py'),
|
||||
'utf8',
|
||||
);
|
||||
const canonicalInit = fs.readFileSync(
|
||||
path.join(ROOT, 'sdk/python/datastack/datastack/__init__.py'),
|
||||
'utf8',
|
||||
);
|
||||
|
||||
assert.match(canonicalInit, /__version__\s*=\s*["']2\.2\.0["']/);
|
||||
assert.match(pyproject, /version\s*=\s*"2\.2\.0"/);
|
||||
assert.match(canonicalSetup, /version="2\.2\.0"/);
|
||||
assert.match(compatSetup, /version="2\.2\.0"/);
|
||||
|
||||
assert.equal(
|
||||
fs.existsSync(path.join(ROOT, 'sdk/python/datastack/datastack/py.typed')),
|
||||
true,
|
||||
'PEP 561 py.typed marker must exist in datastack package',
|
||||
);
|
||||
assert.match(pyproject, /datastack\s*=\s*\["py\.typed"\]/);
|
||||
assert.match(canonicalSetup, /package_data=\{"datastack": \["py\.typed"\]\}/);
|
||||
|
||||
for (const dep of [
|
||||
'"python-socketio[client]>=5.8.0"',
|
||||
'"requests>=2.28.0"',
|
||||
'"websocket-client>=1.5.0"',
|
||||
]) {
|
||||
assert.ok(pyproject.includes(dep), `pyproject.toml missing ${dep}`);
|
||||
assert.ok(canonicalSetup.includes(dep), `setup.py missing ${dep}`);
|
||||
assert.ok(compatSetup.includes(dep), `sdk-python/setup.py missing ${dep}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('BUG-E03 & BUG-E04: Root package.json engines and production start-all shutdown coordination are valid', () => {
|
||||
const pkg = JSON.parse(
|
||||
fs.readFileSync(path.join(ROOT, 'package.json'), 'utf8'),
|
||||
) as {
|
||||
engines?: { node?: string };
|
||||
scripts?: Record<string, string>;
|
||||
};
|
||||
|
||||
assert.equal(pkg.engines?.node, '>=20.0.0');
|
||||
assert.equal(pkg.scripts?.start, 'node scripts/start-all.js');
|
||||
assert.equal(pkg.scripts?.['start:web'], 'next start');
|
||||
assert.equal(pkg.scripts?.['start:server'], 'node server/index.js');
|
||||
|
||||
const startAll = fs.readFileSync(
|
||||
path.join(ROOT, 'scripts/start-all.js'),
|
||||
'utf8',
|
||||
);
|
||||
assert.match(startAll, /markChildExited/);
|
||||
assert.match(startAll, /exitedChildren >= 2/);
|
||||
});
|
||||
|
||||
test('BUG-26: Legacy datastack-sdk declares datastack>=2.2.0 dependency and raises actionable ImportError', () => {
|
||||
const compatSetup = fs.readFileSync(
|
||||
path.join(ROOT, 'sdk-python/setup.py'),
|
||||
'utf8',
|
||||
);
|
||||
const compatInit = fs.readFileSync(
|
||||
path.join(ROOT, 'sdk-python/datastack_sdk/__init__.py'),
|
||||
'utf8',
|
||||
);
|
||||
|
||||
assert.ok(
|
||||
compatSetup.includes('"datastack>=2.2.0"'),
|
||||
'sdk-python/setup.py install_requires must include datastack>=2.2.0',
|
||||
);
|
||||
assert.match(
|
||||
compatInit,
|
||||
/raise ImportError\([\s\S]*datastack>=2\.2\.0[\s\S]*pip install \.\/sdk\/python\/datastack/,
|
||||
'sdk-python/datastack_sdk/__init__.py must provide actionable install guidance if canonical datastack is missing',
|
||||
);
|
||||
});
|
||||
|
||||
test('BUG-27: start-all.js and dev-all.js avoid shell: true orphaning and allow >= 7000ms graceful shutdown flush', () => {
|
||||
for (const relPath of ['scripts/start-all.js', 'scripts/dev-all.js']) {
|
||||
const src = fs.readFileSync(path.join(ROOT, relPath), 'utf8');
|
||||
assert.equal(
|
||||
/shell:\s*true/.test(src),
|
||||
false,
|
||||
`${relPath} must NOT use shell: true (prevents orphaned child processes on SIGTERM)`,
|
||||
);
|
||||
assert.match(src, /shell:\s*false/, `${relPath} must explicitly set shell: false`);
|
||||
assert.match(
|
||||
src,
|
||||
/spawn\(\s*process\.execPath/,
|
||||
`${relPath} must spawn child services directly via process.execPath`,
|
||||
);
|
||||
assert.match(
|
||||
src,
|
||||
/GRACEFUL_SHUTDOWN_TIMEOUT_MS\s*=\s*7000/,
|
||||
`${relPath} must wait at least 7000ms for graceful CSV/Redis shutdown flush`,
|
||||
);
|
||||
assert.match(src, /SIGKILL/, `${relPath} must escalate to SIGKILL after grace timeout`);
|
||||
assert.match(src, /exitedChildren >= 2/, `${relPath} must wait for both child processes`);
|
||||
}
|
||||
});
|
||||
@@ -9,6 +9,7 @@ Phase-05 Verification Tests (BUG-18):
|
||||
offline disk spool replay against a local HTTP test server.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
@@ -32,9 +33,11 @@ import datastack # noqa: E402
|
||||
import datastack_sdk # noqa: E402
|
||||
from datastack import ( # noqa: E402
|
||||
DataStack,
|
||||
DataStackAuthError,
|
||||
DataStackClient,
|
||||
DataStackConfig,
|
||||
DataStackConfigError,
|
||||
DataStackConnectionError,
|
||||
DataStackOfflineSpoolError,
|
||||
DiskSpooler,
|
||||
HeartbeatMonitor,
|
||||
@@ -265,6 +268,340 @@ class Phase05PythonSdkTests(unittest.TestCase):
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
def test_client_id_normalization_and_invalid_numeric_env(self) -> None:
|
||||
tok_norm = _make_jwt({"clientId": "sensor_01", "sub": "sensor_01"})
|
||||
# Case/punctuation variations matching server SDKManager.normalizeClientId() must succeed
|
||||
client = DataStack(
|
||||
server_url="http://127.0.0.1:49999",
|
||||
api_url="http://127.0.0.1:39999",
|
||||
token=tok_norm,
|
||||
client_id="Sensor.01",
|
||||
auto_connect=False,
|
||||
)
|
||||
self.assertEqual(client.client_id, "sensor_01")
|
||||
client.disconnect()
|
||||
|
||||
# Invalid numeric env var must raise DataStackConfigError instead of raw ValueError
|
||||
old_retries = os.environ.get("DATASTACK_MAX_RETRIES")
|
||||
try:
|
||||
os.environ["DATASTACK_MAX_RETRIES"] = "not-an-int"
|
||||
with self.assertRaises(DataStackConfigError):
|
||||
load_config()
|
||||
finally:
|
||||
if old_retries is None:
|
||||
os.environ.pop("DATASTACK_MAX_RETRIES", None)
|
||||
else:
|
||||
os.environ["DATASTACK_MAX_RETRIES"] = old_retries
|
||||
|
||||
def test_ragged_csv_all_rejected_send_file_and_partial_rl_recovery(self) -> None:
|
||||
tok = _make_jwt({"clientId": "csv_worker", "sub": "csv_worker"})
|
||||
mode = {"behavior": "reject_oversized"}
|
||||
seen_records: List[Dict[str, Any]] = []
|
||||
|
||||
class EdgeHandler(BaseHTTPRequestHandler):
|
||||
def do_POST(self) -> None: # noqa: N802
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
raw = self.rfile.read(length).decode("utf-8")
|
||||
parsed = json.loads(raw) if raw else {}
|
||||
items = parsed.get("records") or parsed.get("batch") or []
|
||||
seen_records.extend(items)
|
||||
|
||||
if mode["behavior"] == "reject_oversized":
|
||||
resp = {
|
||||
"success": True,
|
||||
"processed": 0,
|
||||
"rejected": len(items),
|
||||
"duplicates": 0,
|
||||
"oversizedEventIds": [it.get("eventId") for it in items],
|
||||
}
|
||||
status = 200
|
||||
else:
|
||||
# First round processes first item, rate-limits second; second round disconnects/429
|
||||
if len(items) > 1:
|
||||
resp = {
|
||||
"success": True,
|
||||
"processed": 1,
|
||||
"rejected": 1,
|
||||
"duplicates": 0,
|
||||
"rateLimitedEventIds": [items[1]["eventId"]],
|
||||
}
|
||||
status = 200
|
||||
else:
|
||||
resp = {
|
||||
"success": False,
|
||||
"code": "RATE_LIMIT_EXCEEDED",
|
||||
"error": "Still limited",
|
||||
}
|
||||
status = 429
|
||||
|
||||
body = json.dumps(resp).encode("utf-8")
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, format: str, *args: Any) -> None:
|
||||
return
|
||||
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), EdgeHandler)
|
||||
port = server.server_address[1]
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Ragged CSV with extra trailing column (used to crash sort_keys=True with None key)
|
||||
ragged_csv = Path(tmpdir) / "ragged.csv"
|
||||
ragged_csv.write_text(
|
||||
"sensor,val\ns1,10,extra_trailing_col\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
client = DataStack(
|
||||
server_url="http://127.0.0.1:1",
|
||||
api_url=f"http://127.0.0.1:{port}",
|
||||
token=tok,
|
||||
auto_connect=False,
|
||||
request_timeout_sec=1.0,
|
||||
max_retries=0,
|
||||
retry_backoff_base_sec=0.1,
|
||||
offline_spool_enabled=False,
|
||||
)
|
||||
|
||||
res_file = client.send_file(ragged_csv)
|
||||
self.assertFalse(res_file["success"])
|
||||
self.assertEqual(res_file["processed"], 0)
|
||||
self.assertEqual(res_file["rejected"], 1)
|
||||
self.assertEqual(seen_records[0]["data"]["_extra"], ["extra_trailing_col"])
|
||||
|
||||
# Partial rate-limit where follow-up retry exhausts retries must preserve round-1 processed count
|
||||
mode["behavior"] = "partial_rl"
|
||||
spool_dir = Path(tmpdir) / "spool"
|
||||
rl_client = DataStack(
|
||||
server_url="http://127.0.0.1:1",
|
||||
api_url=f"http://127.0.0.1:{port}",
|
||||
token=tok,
|
||||
auto_connect=False,
|
||||
request_timeout_sec=1.0,
|
||||
max_retries=1,
|
||||
retry_backoff_base_sec=0.1,
|
||||
offline_spool_dir=str(spool_dir),
|
||||
)
|
||||
rl_res = rl_client.send_batch(
|
||||
[
|
||||
{"eventId": "evt_ok_1", "data": {"n": 1}},
|
||||
{"eventId": "evt_rl_2", "data": {"n": 2}},
|
||||
]
|
||||
)
|
||||
self.assertTrue(rl_res["success"])
|
||||
self.assertEqual(rl_res["processed"], 1)
|
||||
self.assertEqual(rl_res["spooledCount"], 1)
|
||||
self.assertEqual(rl_client.spooler.pending_count(), 1)
|
||||
|
||||
client.disconnect()
|
||||
rl_client.disconnect()
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
def test_heartbeat_stop_race_and_spooler_clean_ops(self) -> None:
|
||||
hb = HeartbeatMonitor(interval_sec=5.0, recovery_callback=lambda: True)
|
||||
hb.start()
|
||||
hb.trigger_fallback("disconnect")
|
||||
hb.stop("shutdown", join_timeout=0.5)
|
||||
# Calling set_idle_connected after stop() must NOT resurrect IDLE state
|
||||
hb.set_idle_connected()
|
||||
self.assertEqual(hb.state, "STOPPED")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
spool = DiskSpooler(spool_dir=tmpdir, max_size_mb=5, enabled=True)
|
||||
spool.enqueue([{"eventId": "a"}, {"eventId": "b"}, {"eventId": "c"}])
|
||||
self.assertEqual(spool.pending_count(), 3)
|
||||
spool.ack_and_requeue_front(2, [{"eventId": "b_retry"}])
|
||||
batch = spool.peek_batch(limit=10)
|
||||
self.assertEqual([item["eventId"] for item in batch], ["b_retry", "c"])
|
||||
self.assertFalse((Path(tmpdir) / "spool.jsonl.tmp").exists())
|
||||
spool.ack_batch(2)
|
||||
self.assertEqual(spool.pending_count(), 0)
|
||||
|
||||
def test_bug25_disconnects_leaked_socket_on_failed_ping_verify(self) -> None:
|
||||
tok = _make_jwt({"clientId": "cli_bug25_sock"})
|
||||
client = DataStack(
|
||||
server_url="http://127.0.0.1:4000",
|
||||
api_url="http://127.0.0.1:3000",
|
||||
token=tok,
|
||||
auto_connect=False,
|
||||
enable_http_fallback=False,
|
||||
offline_spool_enabled=False,
|
||||
request_timeout_sec=0.5,
|
||||
)
|
||||
|
||||
class FakeSocket:
|
||||
def __init__(self) -> None:
|
||||
self.connected = False
|
||||
self.disconnect_calls = 0
|
||||
|
||||
def connect(self, *args: Any, **kwargs: Any) -> None:
|
||||
self.connected = True
|
||||
|
||||
def disconnect(self) -> None:
|
||||
self.disconnect_calls += 1
|
||||
self.connected = False
|
||||
|
||||
fake_sio = FakeSocket()
|
||||
client.sio = fake_sio
|
||||
|
||||
# Case 1: ping_verify returns False -> socket must be disconnected
|
||||
client._verify_websocket_handshake = lambda timeout=None: False # type: ignore[method-assign]
|
||||
self.assertFalse(client._connect_and_verify_websocket(wait_timeout=0.5))
|
||||
self.assertFalse(client.is_connected)
|
||||
self.assertFalse(fake_sio.connected)
|
||||
self.assertEqual(fake_sio.disconnect_calls, 1)
|
||||
|
||||
# Case 2: ping_verify raises DataStackAuthError -> socket must be disconnected before re-raising
|
||||
def raise_auth(timeout: Any = None) -> bool:
|
||||
raise DataStackAuthError("CLIENT_REVOKED")
|
||||
|
||||
client._verify_websocket_handshake = raise_auth # type: ignore[method-assign]
|
||||
with self.assertRaises(DataStackAuthError):
|
||||
client._connect_and_verify_websocket(wait_timeout=0.5)
|
||||
self.assertFalse(client.is_connected)
|
||||
self.assertFalse(fake_sio.connected)
|
||||
self.assertEqual(fake_sio.disconnect_calls, 2)
|
||||
client.disconnect()
|
||||
|
||||
def test_bug25_strict_auto_connect_and_context_raise_connection_error(self) -> None:
|
||||
tok = _make_jwt({"clientId": "cli_bug25_strict"})
|
||||
|
||||
# auto_connect=True with both fallback and offline spool disabled must raise DataStackConnectionError
|
||||
with self.assertRaises(DataStackConnectionError):
|
||||
DataStack(
|
||||
server_url="http://127.0.0.1:1",
|
||||
api_url="http://127.0.0.1:1",
|
||||
token=tok,
|
||||
auto_connect=True,
|
||||
enable_http_fallback=False,
|
||||
offline_spool_enabled=False,
|
||||
request_timeout_sec=0.2,
|
||||
)
|
||||
|
||||
# Context manager __enter__ with both fallback and offline spool disabled must also raise
|
||||
strict_client = DataStack(
|
||||
server_url="http://127.0.0.1:1",
|
||||
api_url="http://127.0.0.1:1",
|
||||
token=tok,
|
||||
auto_connect=False,
|
||||
enable_http_fallback=False,
|
||||
offline_spool_enabled=False,
|
||||
request_timeout_sec=0.2,
|
||||
)
|
||||
with self.assertRaises(DataStackConnectionError):
|
||||
with strict_client:
|
||||
pass
|
||||
self.assertEqual(strict_client.heartbeat.state, "STOPPED")
|
||||
|
||||
def test_r2_phase03_bug11_flat_batch_and_file_strip_envelope_keys(self) -> None:
|
||||
tok = _make_jwt({"clientId": "cli_bug11_strip"})
|
||||
captured_batches: List[List[Dict[str, Any]]] = []
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
client = DataStack(
|
||||
server_url="http://127.0.0.1:1",
|
||||
api_url="http://127.0.0.1:1",
|
||||
token=tok,
|
||||
auto_connect=False,
|
||||
offline_spool_enabled=False,
|
||||
)
|
||||
|
||||
def fake_execute(**kwargs: Any) -> Dict[str, Any]:
|
||||
ws_payload = kwargs.get("ws_payload")
|
||||
if isinstance(ws_payload, list):
|
||||
captured_batches.append(ws_payload)
|
||||
return {
|
||||
"success": True,
|
||||
"processed": len(ws_payload),
|
||||
"rejected": 0,
|
||||
"transport": "HTTP",
|
||||
}
|
||||
return {"success": True, "processed": 0, "rejected": 0}
|
||||
|
||||
client._execute_with_retry_and_fallback = fake_execute # type: ignore[method-assign]
|
||||
|
||||
res_batch = client.send_batch(
|
||||
[
|
||||
{
|
||||
"eventId": "py-flat-1",
|
||||
"timestamp": "2026-11-19T12:00:00Z",
|
||||
"clientId": "cli_bug11_strip",
|
||||
"sensor": "temp",
|
||||
"value": 24.5,
|
||||
},
|
||||
{
|
||||
"eventId": "py-env-2",
|
||||
"timestamp": "2026-11-19T12:00:01Z",
|
||||
"data": {"sensor": "humidity", "value": 55},
|
||||
},
|
||||
]
|
||||
)
|
||||
self.assertTrue(res_batch["success"])
|
||||
self.assertEqual(len(captured_batches), 1)
|
||||
self.assertEqual(captured_batches[0][0]["eventId"], "py-flat-1")
|
||||
self.assertEqual(
|
||||
captured_batches[0][0]["data"],
|
||||
{"sensor": "temp", "value": 24.5},
|
||||
)
|
||||
self.assertEqual(
|
||||
captured_batches[0][1]["data"],
|
||||
{"sensor": "humidity", "value": 55},
|
||||
)
|
||||
|
||||
csv_path = Path(tmpdir) / "flat_records.csv"
|
||||
csv_path.write_text(
|
||||
"eventId,timestamp,clientId,sensor,reading\n"
|
||||
"file-flat-1,2026-11-19T13:00:00Z,cli_bug11_strip,pressure,101.3\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
res_file = client.send_file(csv_path)
|
||||
self.assertTrue(res_file["success"])
|
||||
self.assertEqual(len(captured_batches), 2)
|
||||
self.assertEqual(captured_batches[1][0]["eventId"], "file-flat-1")
|
||||
self.assertEqual(
|
||||
captured_batches[1][0]["data"],
|
||||
{"sensor": "pressure", "reading": "101.3"},
|
||||
)
|
||||
|
||||
def test_r2_phase04_bug14_python38_ast_typing_compatibility(self) -> None:
|
||||
pkg_dir = CANONICAL_PKG_ROOT / "datastack"
|
||||
py_files = sorted(pkg_dir.glob("*.py"))
|
||||
self.assertGreater(len(py_files), 0)
|
||||
|
||||
pep585_builtins = {"tuple", "list", "dict", "set", "frozenset", "type"}
|
||||
for py_file in py_files:
|
||||
src = py_file.read_text(encoding="utf-8")
|
||||
tree = ast.parse(src, filename=str(py_file), feature_version=(3, 8))
|
||||
has_future_annotations = any(
|
||||
isinstance(node, ast.ImportFrom)
|
||||
and node.module == "__future__"
|
||||
and any(alias.name == "annotations" for alias in node.names)
|
||||
for node in tree.body
|
||||
)
|
||||
if py_file.name == "client.py":
|
||||
self.assertTrue(
|
||||
has_future_annotations,
|
||||
"client.py must declare `from __future__ import annotations`",
|
||||
)
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if (
|
||||
isinstance(node, ast.Subscript)
|
||||
and isinstance(node.value, ast.Name)
|
||||
and node.value.id in pep585_builtins
|
||||
):
|
||||
self.fail(
|
||||
f"{py_file.name}:{node.lineno} uses Python 3.9+ PEP 585 generic `{node.value.id}[...]`"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
@@ -45,6 +45,8 @@ test('BUG-25: PWA manifest, icons (192/512/maskable/apple-touch), and Service Wo
|
||||
assert.match(swContent, /request\.method !== ['"]GET['"]/);
|
||||
assert.match(swContent, /\/api\//);
|
||||
assert.match(swContent, /\/socket\.io\//);
|
||||
assert.match(swContent, /\/vcon\//);
|
||||
assert.match(swContent, /CLEAR_AUTH_CACHE/);
|
||||
assert.match(swContent, /new Response\(/);
|
||||
|
||||
const layoutContent = fs.readFileSync(path.join(ROOT, 'app/layout.tsx'), 'utf8');
|
||||
@@ -79,3 +81,37 @@ test('BUG-17: Landing/Navbar/Footer/404 navigation uses root-relative anchors an
|
||||
assert.match(notFoundContent, /href=["']\/["']/);
|
||||
assert.match(notFoundContent, /href=["']\/vcon\/dashboard["']/);
|
||||
});
|
||||
|
||||
test('FORENSIC BUG-15: RootLayout injects pre-hydration theme bootstrap script and AdminSetup supports theme restoration & toggle', () => {
|
||||
const layoutContent = fs.readFileSync(path.join(ROOT, 'app/layout.tsx'), 'utf8');
|
||||
assert.match(layoutContent, /<html[^>]*suppressHydrationWarning/);
|
||||
assert.match(layoutContent, /datastack-theme/);
|
||||
assert.match(layoutContent, /<head>/);
|
||||
|
||||
const setupContent = fs.readFileSync(
|
||||
path.join(ROOT, 'components/sections/AdminSetup.tsx'),
|
||||
'utf8',
|
||||
);
|
||||
assert.match(setupContent, /datastack-theme/);
|
||||
assert.match(setupContent, /Toggle theme/);
|
||||
assert.match(setupContent, /document\.documentElement\.classList\.toggle\(['"]light['"]/);
|
||||
});
|
||||
|
||||
test('FORENSIC BUG-16: Service Worker serves uncached HTML offline fallback shell for /vcon/dashboard PWA navigations', () => {
|
||||
const manifest = JSON.parse(
|
||||
fs.readFileSync(path.join(ROOT, 'public/manifest.json'), 'utf8'),
|
||||
);
|
||||
assert.equal(manifest.start_url, '/vcon/dashboard');
|
||||
|
||||
const swContent = fs.readFileSync(path.join(ROOT, 'public/sw.js'), 'utf8');
|
||||
assert.match(swContent, /buildOfflineShellResponse/);
|
||||
assert.match(swContent, /request\.mode !== ['"]navigate['"]/);
|
||||
assert.match(swContent, /text\/html; charset=utf-8/);
|
||||
assert.match(swContent, /Cache-Control['"]:\s*['"]no-store['"]/);
|
||||
// Ensure /vcon/* branch never calls cache.put on authenticated admin HTML
|
||||
const vconBranch = swContent.match(
|
||||
/if \(url\.pathname === '\/vcon' \|\| url\.pathname\.startsWith\('\/vcon\/'\)\) \{[\s\S]*?return;\s*\}/,
|
||||
);
|
||||
assert.ok(vconBranch, 'Expected dedicated /vcon/* navigation fallback branch in public/sw.js');
|
||||
assert.doesNotMatch(vconBranch[0], /cache\.put/);
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { existsSync, mkdirSync, readFileSync, rmSync, utimesSync, writeFileSync } from "node:fs";
|
||||
import { createRequire } from "node:module";
|
||||
import { resolve } from "node:path";
|
||||
import test, { after } from "node:test";
|
||||
import { NextRequest } from "next/server";
|
||||
@@ -7,6 +8,8 @@ import { GET as getHealthRoute } from "../app/api/health/route";
|
||||
import { SettingsManager } from "../core/adminSettings/settingsManager";
|
||||
import { DataBufferEngine, type RedisBufferClient } from "../core/data/buffer";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
const g = globalThis as typeof globalThis & {
|
||||
__dataStackAdminSettings?: unknown;
|
||||
__dataStackSettingsHydrated?: boolean;
|
||||
@@ -19,6 +22,8 @@ const g = globalThis as typeof globalThis & {
|
||||
|
||||
const prevDbUrl = process.env.DATABASE_URL;
|
||||
const prevRedisUrl = process.env.REDIS_URL;
|
||||
const prevDisableListen = process.env.DATASTACK_DISABLE_LISTEN;
|
||||
process.env.DATASTACK_DISABLE_LISTEN = "1";
|
||||
|
||||
after(async () => {
|
||||
DataBufferEngine.setRedisClientForTests(undefined);
|
||||
@@ -39,6 +44,9 @@ after(async () => {
|
||||
if (prevRedisUrl === undefined) delete process.env.REDIS_URL;
|
||||
else process.env.REDIS_URL = prevRedisUrl;
|
||||
|
||||
if (prevDisableListen === undefined) delete process.env.DATASTACK_DISABLE_LISTEN;
|
||||
else process.env.DATASTACK_DISABLE_LISTEN = prevDisableListen;
|
||||
|
||||
const testDirs = [
|
||||
resolve(process.cwd(), "storage", "data", "phase07_prune_guard"),
|
||||
resolve(process.cwd(), "storage", "data", "phase07_shutdown_flush"),
|
||||
@@ -87,15 +95,20 @@ test("DEPLOY-BLK-03 & 05: Dockerfiles guard recursive chown on /app/storage/data
|
||||
assert.ok(lockJson.packages && Object.keys(lockJson.packages).length > 10);
|
||||
});
|
||||
|
||||
test("DEPLOY-BLK-04: Both Nginx configs proxy = /health to socketio_upstream", () => {
|
||||
for (const relPath of ["docker/coolify-nginx.conf", "nginx/default.conf"]) {
|
||||
const conf = readFileSync(resolve(process.cwd(), relPath), "utf8");
|
||||
assert.match(
|
||||
conf,
|
||||
/location\s*=\s*\/health\s*\{[\s\S]*?proxy_pass\s+http:\/\/socketio_upstream;/,
|
||||
`${relPath} must route location = /health to socketio_upstream`,
|
||||
);
|
||||
}
|
||||
test("DEPLOY-BLK-04: Both Nginx configs proxy = /health to ingestion upstream", () => {
|
||||
const defaultConf = readFileSync(resolve(process.cwd(), "nginx/default.conf"), "utf8");
|
||||
assert.match(
|
||||
defaultConf,
|
||||
/location\s*=\s*\/health\s*\{[\s\S]*?proxy_pass\s+http:\/\/socketio_upstream;/,
|
||||
"nginx/default.conf must route location = /health to socketio_upstream",
|
||||
);
|
||||
|
||||
const coolifyConf = readFileSync(resolve(process.cwd(), "docker/coolify-nginx.conf"), "utf8");
|
||||
assert.match(
|
||||
coolifyConf,
|
||||
/location\s*=\s*\/health\s*\{[\s\S]*?proxy_pass\s+\$socketio_upstream;/,
|
||||
"docker/coolify-nginx.conf must route location = /health to $socketio_upstream",
|
||||
);
|
||||
});
|
||||
|
||||
test("DEPLOY-BLK-03 docs: manifest.md and README.md align volume mount path (/app/storage) and include sdk/python/datastack", () => {
|
||||
@@ -197,3 +210,269 @@ test("DEPLOY-BLK-06 shutdown & health: DataBufferEngine.flushAndShutdownOnSignal
|
||||
assert.equal(json.status, "online");
|
||||
assert.equal(json.bufferSize, 7);
|
||||
});
|
||||
|
||||
test("BUG-28: nginx/default.conf aligns Socket.IO directives and server/index.js enforces configurable DATASTACK_CORS_ORIGINS", () => {
|
||||
const nginxDefault = readFileSync(resolve(process.cwd(), "nginx", "default.conf"), "utf8");
|
||||
assert.match(
|
||||
nginxDefault,
|
||||
/location \/socket\.io\/ \{[\s\S]*proxy_read_timeout 86400s;[\s\S]*proxy_send_timeout 86400s;[\s\S]*proxy_buffering off;/,
|
||||
"nginx/default.conf must disable proxy_buffering and set 86400s read/send timeouts on /socket.io/",
|
||||
);
|
||||
|
||||
const compose = readFileSync(resolve(process.cwd(), "docker", "docker-compose.yml"), "utf8");
|
||||
assert.match(compose, /DATASTACK_CORS_ORIGINS:\s*"\$\{DATASTACK_CORS_ORIGINS:-\}"/);
|
||||
|
||||
const envExample = readFileSync(resolve(process.cwd(), ".env.example"), "utf8");
|
||||
assert.match(envExample, /DATASTACK_CORS_ORIGINS=/);
|
||||
|
||||
const serverSrc = readFileSync(resolve(process.cwd(), "server", "index.js"), "utf8");
|
||||
assert.equal(
|
||||
serverSrc.includes("origin: '*'"),
|
||||
false,
|
||||
"server/index.js must not hardcode wildcard origin: '*'",
|
||||
);
|
||||
|
||||
process.env.DATASTACK_DISABLE_LISTEN = "1";
|
||||
const ingestionServer = require("../server/index.js") as {
|
||||
isOriginAllowedByCors: (origin: string | undefined, raw?: string) => boolean;
|
||||
corsOriginDelegate: (
|
||||
origin: string | undefined,
|
||||
cb: (err: Error | null, allow: boolean) => void,
|
||||
) => void;
|
||||
stopTimers: () => void;
|
||||
};
|
||||
|
||||
try {
|
||||
// Default / empty / wildcard allows any origin
|
||||
assert.equal(ingestionServer.isOriginAllowedByCors("https://any.example.com", ""), true);
|
||||
assert.equal(ingestionServer.isOriginAllowedByCors("https://any.example.com", "*"), true);
|
||||
|
||||
// Non-browser / SDK calls (no Origin header) remain allowed even with strict allowlist
|
||||
const strictList = "https://console.example.com, https://admin.example.com/";
|
||||
assert.equal(ingestionServer.isOriginAllowedByCors(undefined, strictList), true);
|
||||
assert.equal(ingestionServer.isOriginAllowedByCors("", strictList), true);
|
||||
|
||||
// Allowed origins (with trailing slash normalization) pass
|
||||
assert.equal(ingestionServer.isOriginAllowedByCors("https://console.example.com", strictList), true);
|
||||
assert.equal(ingestionServer.isOriginAllowedByCors("https://console.example.com/", strictList), true);
|
||||
assert.equal(ingestionServer.isOriginAllowedByCors("https://admin.example.com", strictList), true);
|
||||
|
||||
// Disallowed browser origins are rejected cleanly
|
||||
assert.equal(ingestionServer.isOriginAllowedByCors("https://evil.example.com", strictList), false);
|
||||
|
||||
// Delegate callback respects live process.env.DATASTACK_CORS_ORIGINS without throwing
|
||||
const prevCors = process.env.DATASTACK_CORS_ORIGINS;
|
||||
try {
|
||||
process.env.DATASTACK_CORS_ORIGINS = "https://allowed.datastack.dev";
|
||||
let delegateErr: Error | null = null;
|
||||
let delegateAllowed = true;
|
||||
ingestionServer.corsOriginDelegate("https://blocked.datastack.dev", (err, allow) => {
|
||||
delegateErr = err;
|
||||
delegateAllowed = allow;
|
||||
});
|
||||
assert.equal(delegateErr, null);
|
||||
assert.equal(delegateAllowed, false);
|
||||
|
||||
ingestionServer.corsOriginDelegate("https://allowed.datastack.dev", (err, allow) => {
|
||||
delegateErr = err;
|
||||
delegateAllowed = allow;
|
||||
});
|
||||
assert.equal(delegateErr, null);
|
||||
assert.equal(delegateAllowed, true);
|
||||
} finally {
|
||||
if (prevCors === undefined) delete process.env.DATASTACK_CORS_ORIGINS;
|
||||
else process.env.DATASTACK_CORS_ORIGINS = prevCors;
|
||||
}
|
||||
} finally {
|
||||
ingestionServer.stopTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test("R2 PHASE-04 BUG-16 & BUG-17: Nginx disables buffering on /api/export, next.config enforces ESLint during builds, and package.json prunes unused template deps", () => {
|
||||
const nginxDefault = readFileSync(resolve(process.cwd(), "nginx/default.conf"), "utf8");
|
||||
assert.match(
|
||||
nginxDefault,
|
||||
/location\s*=\s*\/api\/export\s*\{[\s\S]*?proxy_pass\s+http:\/\/nextjs_upstream;[\s\S]*?proxy_buffering\s+off;[\s\S]*?proxy_read_timeout\s+300s;/,
|
||||
"nginx/default.conf must disable proxy_buffering and extend proxy_read_timeout on location = /api/export",
|
||||
);
|
||||
|
||||
const coolifyConf = readFileSync(resolve(process.cwd(), "docker/coolify-nginx.conf"), "utf8");
|
||||
assert.match(
|
||||
coolifyConf,
|
||||
/location\s*=\s*\/api\/export\s*\{[\s\S]*?proxy_pass\s+\$nextjs_upstream;[\s\S]*?proxy_buffering\s+off;[\s\S]*?proxy_read_timeout\s+300s;/,
|
||||
"docker/coolify-nginx.conf must disable proxy_buffering and extend proxy_read_timeout on location = /api/export",
|
||||
);
|
||||
|
||||
const nextConfig = readFileSync(resolve(process.cwd(), "next.config.ts"), "utf8");
|
||||
assert.equal(
|
||||
/ignoreDuringBuilds\s*:\s*true/.test(nextConfig),
|
||||
false,
|
||||
"next.config.ts must not ignore ESLint errors during builds",
|
||||
);
|
||||
|
||||
const pkg = JSON.parse(readFileSync(resolve(process.cwd(), "package.json"), "utf8")) as {
|
||||
name?: string;
|
||||
dependencies?: Record<string, string>;
|
||||
devDependencies?: Record<string, string>;
|
||||
};
|
||||
assert.equal(pkg.name, "datastack");
|
||||
assert.equal("@google/genai" in (pkg.dependencies ?? {}), false);
|
||||
assert.equal("firebase-tools" in (pkg.devDependencies ?? {}), false);
|
||||
});
|
||||
|
||||
test("DEPLOY-BUG-01 & 05: coolify-nginx.conf resolves Docker DNS dynamically and Coolify docs align Base Directory /", () => {
|
||||
const coolifyConf = readFileSync(resolve(process.cwd(), "docker", "coolify-nginx.conf"), "utf8");
|
||||
assert.match(coolifyConf, /resolver\s+127\.0\.0\.11\s+valid=10s\s+ipv6=off;/);
|
||||
assert.match(coolifyConf, /set\s+\$nextjs_upstream\s+"http:\/\/web:3000";/);
|
||||
assert.match(coolifyConf, /set\s+\$socketio_upstream\s+"http:\/\/ingestion:4000";/);
|
||||
assert.equal(/\bupstream\s+\w+\s*\{/.test(coolifyConf), false, "coolify-nginx.conf must not define static upstream blocks");
|
||||
|
||||
const compose = readFileSync(resolve(process.cwd(), "docker", "docker-compose.yml"), "utf8");
|
||||
const graceMatches = compose.match(/^\s+stop_grace_period:\s*20s/gm) ?? [];
|
||||
assert.equal(graceMatches.length, 2, "web and ingestion must set stop_grace_period: 20s");
|
||||
|
||||
const installDoc = readFileSync(resolve(process.cwd(), "docs", "users", "installation.md"), "utf8");
|
||||
const readmeDoc = readFileSync(resolve(process.cwd(), "README.md"), "utf8");
|
||||
assert.match(installDoc, /\*\*Base Directory\*\*\s+to\s+`\/`/);
|
||||
assert.match(readmeDoc, /\*\*Base Directory\*\*\s+to\s+`\/`\s+and\s+\*\*Docker Compose Location\*\*\s+to\s+`\/docker\/docker-compose\.yml`/);
|
||||
});
|
||||
|
||||
test("DEPLOY-BUG-02, 03 & 04: Port 4000 memoizes schema DDLs, hydrates rotated JWT before listening, and drains buffers gracefully", async () => {
|
||||
const serverSrc = readFileSync(resolve(process.cwd(), "server", "index.js"), "utf8");
|
||||
assert.match(
|
||||
serverSrc,
|
||||
/async function startIngestionServer\(\)[\s\S]*syncSettingsFromDatabase\(\)[\s\S]*server\.listen\(PORT/,
|
||||
"startIngestionServer must await bounded syncSettingsFromDatabase() before server.listen(PORT)",
|
||||
);
|
||||
assert.match(
|
||||
serverSrc,
|
||||
/async function gracefulShutdown\([\s\S]*await closeServerGracefully\(\);[\s\S]*Promise\.allSettled\(\[flushBuffer\(\),\s*flushPendingClientActivity\(\)\]\)/,
|
||||
"gracefulShutdown must close HTTP/Socket.IO ingress before flushing buffers",
|
||||
);
|
||||
|
||||
process.env.DATASTACK_DISABLE_LISTEN = "1";
|
||||
const ingestionServer = require("../server/index.js") as {
|
||||
runtimeConfig: { jwtSecret: string };
|
||||
bufferQueue: Array<{ clientId: string; eventId: string; timestamp: string; payload: string }>;
|
||||
setDbPoolForTests: (pool: unknown) => void;
|
||||
setRedisClientForTests: (client: unknown) => void;
|
||||
ensureDatabaseSchema: () => Promise<boolean>;
|
||||
syncSettingsFromDatabase: () => Promise<boolean>;
|
||||
gracefulShutdown: (signal: string, options?: { exit?: boolean }) => Promise<void>;
|
||||
stopTimers: () => void;
|
||||
};
|
||||
|
||||
const prevJwt = ingestionServer.runtimeConfig.jwtSecret;
|
||||
let ddlCount = 0;
|
||||
let selectCount = 0;
|
||||
let failDdl = true;
|
||||
let poolEnded = false;
|
||||
const rotatedSecret = "rotated-production-jwt-secret-value-over-32-chars";
|
||||
|
||||
const mockPool = {
|
||||
async query(sql: string) {
|
||||
const normalized = sql.trim().toUpperCase();
|
||||
if (normalized.startsWith("CREATE TABLE") || normalized.startsWith("ALTER TABLE")) {
|
||||
ddlCount += 1;
|
||||
if (failDdl) {
|
||||
throw new Error("transient startup PG failure");
|
||||
}
|
||||
return { rowCount: 0, rows: [] };
|
||||
}
|
||||
if (normalized.includes("FROM ADMIN_SETTINGS")) {
|
||||
selectCount += 1;
|
||||
return {
|
||||
rowCount: 1,
|
||||
rows: [
|
||||
{
|
||||
max_events_per_sec: 250,
|
||||
max_payload_size_kb: 128,
|
||||
batch_flush_threshold: 100,
|
||||
batch_flush_interval_ms: 2000,
|
||||
file_rotation_mb: 50,
|
||||
enable_idempotency: true,
|
||||
auto_prune_days: 45,
|
||||
jwt_secret: rotatedSecret,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
return { rowCount: 0, rows: [] };
|
||||
},
|
||||
async end() {
|
||||
poolEnded = true;
|
||||
},
|
||||
};
|
||||
|
||||
let redisQuitCalled = false;
|
||||
const mockRedis = {
|
||||
status: "ready",
|
||||
async rpush() {
|
||||
return 0;
|
||||
},
|
||||
async lpush() {
|
||||
return 0;
|
||||
},
|
||||
async ltrim() {
|
||||
return "OK";
|
||||
},
|
||||
async lrange() {
|
||||
return [];
|
||||
},
|
||||
async llen() {
|
||||
return 0;
|
||||
},
|
||||
async set() {
|
||||
return "OK";
|
||||
},
|
||||
async eval() {
|
||||
return 1;
|
||||
},
|
||||
async sismember() {
|
||||
return 0;
|
||||
},
|
||||
async quit() {
|
||||
redisQuitCalled = true;
|
||||
return "OK";
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
ingestionServer.setDbPoolForTests(mockPool);
|
||||
// 1. Transient DDL error must return false and allow retry
|
||||
const failedSync = await ingestionServer.syncSettingsFromDatabase();
|
||||
assert.equal(failedSync, false);
|
||||
assert.equal(ddlCount, 1);
|
||||
|
||||
// 2. Recovery runs 8 DDLs once and hydrates rotated jwt_secret; repeated calls do zero extra DDLs
|
||||
failDdl = false;
|
||||
ddlCount = 0;
|
||||
assert.equal(await ingestionServer.syncSettingsFromDatabase(), true);
|
||||
assert.equal(ddlCount, 8);
|
||||
assert.equal(selectCount, 1);
|
||||
assert.equal(ingestionServer.runtimeConfig.jwtSecret, rotatedSecret);
|
||||
|
||||
assert.equal(await ingestionServer.syncSettingsFromDatabase(), true);
|
||||
assert.equal(await ingestionServer.syncSettingsFromDatabase(), true);
|
||||
assert.equal(ddlCount, 8, "Repeated 3s sync ticks must NOT re-run CREATE/ALTER DDL statements");
|
||||
assert.equal(selectCount, 3);
|
||||
|
||||
// 3. Graceful shutdown flushes buffered events, quits Redis, and closes PostgreSQL pool cleanly
|
||||
ingestionServer.setRedisClientForTests(mockRedis);
|
||||
ingestionServer.bufferQueue.push({
|
||||
clientId: "phase07_shutdown_flush",
|
||||
eventId: `evt-graceful-${Date.now()}`,
|
||||
timestamp: new Date().toISOString(),
|
||||
payload: JSON.stringify({ ok: true }),
|
||||
});
|
||||
await ingestionServer.gracefulShutdown("SIGTERM", { exit: false });
|
||||
assert.equal(ingestionServer.bufferQueue.length, 0, "gracefulShutdown must drain RAM queue to CSV");
|
||||
assert.equal(redisQuitCalled, true, "gracefulShutdown must invoke redis.quit()");
|
||||
assert.equal(poolEnded, true, "gracefulShutdown must close PostgreSQL pool");
|
||||
} finally {
|
||||
ingestionServer.runtimeConfig.jwtSecret = prevJwt;
|
||||
ingestionServer.setDbPoolForTests(null);
|
||||
ingestionServer.setRedisClientForTests(undefined);
|
||||
ingestionServer.stopTimers();
|
||||
}
|
||||
});
|
||||
Reference in new issue
Block a user