From 89e4d686909ec7940b147ac69964c1f14e7afe49 Mon Sep 17 00:00:00 2001 From: Team Dark Devil Date: Thu, 19 Nov 2026 20:22:01 +0600 Subject: [PATCH] feat(auth): harden session management and security - Implement server-side session revocation on logout - Add IP extraction utility and admin setup error handling - Improve error codes and response consistency for auth endpoints - Refine landing page redirect logic to handle transient DB outages - Update deployment documentation and configuration defaults --- .env.example | 8 +- MEMORY.md | 144 +- README.md | 2 +- app/(landing)/page.tsx | 16 +- app/api/auth/login/route.ts | 24 +- app/api/auth/logout/route.ts | 16 + app/api/auth/profile/route.ts | 13 +- app/api/auth/setup/route.ts | 66 +- app/api/clients/[id]/route.ts | 188 +- app/api/clients/route.ts | 28 +- app/api/data/route.ts | 108 +- app/api/events/route.ts | 337 +- app/api/export/route.ts | 149 +- app/api/health/route.ts | 58 +- app/api/metrics/route.ts | 10 +- app/api/settings/route.ts | 15 + app/api/storage/route.ts | 81 +- app/layout.tsx | 7 +- app/vcon/layout.tsx | 54 +- app/vcon/login/page.tsx | 36 +- app/vcon/sdk/page.tsx | 91 +- app/vcon/setup/page.tsx | 26 +- app/vcon/simulator/page.tsx | 118 +- app/vcon/storage/page.tsx | 95 +- components/layout/Sidebar.tsx | 29 +- components/layout/VconShell.tsx | 8 + components/sections/AdminLogin.tsx | 6 +- components/sections/AdminSetup.tsx | 35 +- components/sections/ClientsTable.tsx | 66 +- components/sections/DashboardOverview.tsx | 121 +- components/sections/DataBrowser.tsx | 44 +- components/sections/ExportWorkspace.tsx | 221 +- components/sections/NewClientModal.tsx | 14 +- components/sections/ProfileWorkspace.tsx | 52 +- components/sections/SdkSection.tsx | 6 +- components/sections/SettingsWorkspace.tsx | 445 +- core/Database/database.ts | 188 +- core/Security/security.ts | 207 +- core/Storage/csvStorage.ts | 464 +- core/adminSettings/settingsManager.ts | 221 +- core/apiEngine/apiService.ts | 80 +- core/auth/adminSession.ts | 5 +- core/auth/adminUsers.ts | 115 +- core/data/buffer.ts | 600 +- core/sdk/sdkManager.ts | 637 ++- docker/coolify-nginx.conf | 30 +- docker/docker-compose.yml | 3 + docs/users/installation.md | 2 +- lib/adminAuth.ts | 53 + lib/vconCache.ts | 33 + lib/vconUiHelpers.ts | 537 ++ middleware.ts | 56 +- next.config.ts | 3 - nginx/default.conf | 17 +- package-lock.json | 5417 +------------------ package.json | 11 +- prisma/schema.prisma | 6 +- public/sw.js | 129 +- scripts/dev-all.js | 77 +- scripts/start-all.js | 90 + sdk-python/datastack_sdk/__init__.py | 9 + sdk-python/setup.py | 8 +- sdk/python/datastack/datastack/client.py | 167 +- sdk/python/datastack/datastack/config.py | 46 +- sdk/python/datastack/datastack/heartbeat.py | 29 +- sdk/python/datastack/datastack/py.typed | 0 sdk/python/datastack/datastack/spool.py | 19 +- sdk/python/datastack/pyproject.toml | 4 + sdk/python/datastack/setup.py | 3 + server/index.js | 1085 +++- server/socket/handlers.js | 303 +- server/workers/csvWriter.js | 320 +- tests/phase01-auth.test.ts | 709 ++- tests/phase02-sync.test.ts | 847 +++ tests/phase03-storage-buffer.test.ts | 1050 +++- tests/phase04-ingestion-hygiene.test.ts | 495 ++ tests/phase04-ui-sync.test.ts | 850 +++ tests/phase05-packaging.test.ts | 114 + tests/phase05_sdk_test.py | 337 ++ tests/phase06-pwa-landing.test.ts | 36 + tests/phase07-deployment.test.ts | 297 +- 81 files changed, 11530 insertions(+), 6916 deletions(-) create mode 100644 components/layout/VconShell.tsx create mode 100644 lib/adminAuth.ts create mode 100644 lib/vconUiHelpers.ts create mode 100644 scripts/start-all.js create mode 100644 sdk/python/datastack/datastack/py.typed create mode 100644 tests/phase04-ingestion-hygiene.test.ts create mode 100644 tests/phase04-ui-sync.test.ts create mode 100644 tests/phase05-packaging.test.ts diff --git a/.env.example b/.env.example index 453f3e9..accf636 100644 --- a/.env.example +++ b/.env.example @@ -18,6 +18,12 @@ DATASTACK_ADMIN_EMAIL=admin@datastack.dev # Shared Redis Buffer (optional; enables cross-process buffering & distributed flush locks) # REDIS_URL=redis://127.0.0.1:6379 +# Shared Partitioned CSV Storage Directory (optional; defaults to ./storage/data) +# Set an absolute path so Port 3000 (Next.js) and Port 4000 (Ingestion) always share one storage root +# DATASTACK_STORAGE_DIR=/var/lib/datastack/storage/data + # Service Networking PORT_SOCKET=4000 -INGESTION_INTERNAL_URL=http://127.0.0.1:4000 \ No newline at end of file +INGESTION_INTERNAL_URL=http://127.0.0.1:4000 +# Optional comma-separated browser CORS origin allowlist for Port 4000 ingestion/Socket.IO (defaults to wildcard if unset) +# DATASTACK_CORS_ORIGINS=https://console.example.com,https://admin.example.com \ No newline at end of file diff --git a/MEMORY.md b/MEMORY.md index 0d91603..897b9b0 100644 --- a/MEMORY.md +++ b/MEMORY.md @@ -1,7 +1,7 @@ # MEMORY.md — DataStack Cross-IDE Persistent Project Memory -> **Last Synchronized**: 2026-11-18 11:00:00 UTC -> **Status**: Fresh Source-Code Forensic Audit Complete (`notes/forensic-audit-report.md` — 27 verified findings across 5 categories; awaiting approval to plan incremental remediation phases) +> **Last Synchronized**: 2026-11-19 +> **Status**: **Production & Deployment Hardening Complete (`BASELINE-2026-11-19-R2` + Deployment Hardening) — `4 / 4` R2 Phases (`18/18` bugs) + `5/5` Docker/Coolify Redeploy Fixes (`DEPLOY-BUG-01`–`05`) Verified (`85/85` TS tests, `10/10` Python tests, ESLint-enabled build & lint `0` errors / `0` warnings).** > **Scope Rule**: Maintain this file accurately across VS Code, Google Antigravity, Cursor, and any other IDE after every modification, feature update, or bug fix (see `.ai-rules` and `AGENTS.md`). --- @@ -69,9 +69,13 @@ datastack/ │ ├── dashboard/ clients/ data/ export/ login/ profile/ sdk/ settings/ setup/ simulator/ storage/ ├── components/ │ ├── ui/ # Single-source-of-truth UI primitives (Button, Input, Table, Modal, Panel, etc.) -│ ├── layout/ # MainLayout, Navbar, Sidebar, PageShell, AdminPageHeader, Footer +│ ├── layout/ # MainLayout, Navbar, Sidebar, VconShell, PageShell, AdminPageHeader, Footer │ ├── sections/ # Modular feature sections (DashboardOverview, ClientsTable, DataBrowser, ExportWorkspace, SettingsWorkspace, etc.) │ └── pwa/ # PwaLifecycle.tsx service worker registration +├── lib/ +│ ├── adminAuth.ts # Uniform API session guard (requireAdminApiSession) + proxy-safe extractClientIp() +│ ├── vconCache.ts # In-memory SWR cache + clearAuthCaches() for logout/SW cleanup +│ └── vconUiHelpers.ts # Pure Admin UI sync helpers (settings bounds, bridge sync, UTC presets, timestamps) ├── core/ │ ├── adminSettings/ # settingsManager.ts (DB hydration, Port 4000 sync, unhydrated prune guard) │ ├── apiEngine/ # apiService.ts (dashboard metrics & Port 4000 internal bridge queries) @@ -81,6 +85,9 @@ datastack/ │ ├── sdk/ # sdkManager.ts (DB JWT hydration, client lifecycle, deleted_clients tombstone) │ ├── Security/ # security.ts (rate limit, maxPayloadSizeKb, idempotency, internal headers) │ └── Storage/ # csvStorage.ts (cooperative lock, rotation, cached streaming row count, prune) +├── scripts/ +│ ├── dev-all.js # Local concurrent Next.js (3000) + Ingestion (4000) dev runner +│ └── start-all.js # Production concurrent Next.js (`next start`) + Ingestion (`server/index.js`) runner ├── server/ │ ├── index.js # Express 5 + Socket.IO (Port 4000), /health, /internal/* bridge, PG sync, graceful exit │ ├── socket/handlers.js # Handshake JWT auth, ping_verify, sync_data, batch_sync @@ -139,15 +146,24 @@ datastack/ │ ├── configuration-reference.md # Complete environment variable & setting bounds │ ├── dependencies.md # Runtime & dev dependencies audit with rationales │ └── glossary.md # Technical domain terminology & concept glossary -├── notes/ # Fresh forensic audit output only (legacy reports/plans cleared): -│ └── forensic-audit-report.md # 100% source-verified audit report (27 issues: 5C, 11H, 10M, 1L) +├── notes/ # Production Hardening Governance & Scope-Lock Suite (`BASELINE-2026-11-19-R2`): +│ ├── README.md # Master governance index & 6-step scope-locked execution protocol +│ ├── FORENSIC_AUDIT_REPORT.md # 100% raw-code verified audit (18 bugs: 0C/6H/8M/4L), Demo vs Prod & UI-vs-Reality matrices +│ ├── BASELINE_FREEZE.md # Frozen baseline snapshot (`BASELINE-2026-11-19-R2`) & 1:1 bug alignment +│ ├── PRODUCTION_READINESS_SCOPE_LOCK.md # (Doc 001) Exact bug fix contracts & per-phase Scope Locks (`Phase-01`..`Phase-04`) +│ ├── PRODUCTION_ROADMAP.md # (Doc 002) Dependency-ordered A-Z 4-phase roadmap & burn-down tracker +│ ├── PHASE_EXECUTION_LOG.md # (Doc 003) Live phase completion log (`4/4` complete) & zero-regression guardrails +│ ├── ERROR_HANDLING_MATRIX.md # (Doc 004) Active error-handling catalog & 9 planned hardening additions +│ └── ACTUAL_IMPLEMENTATION_STATUS.md # Ground-truth working vs completed-per-phase vs remaining ledger └── tests/ # Automated verification suites: ├── phase01-auth.test.ts # Auth fail-closed, tokenVersion, brute-force lockout, Bearer revocation, CSRF ├── phase02-sync.test.ts # Internal bridge auth, settings sync, JWT rotation kick, rate/security metrics ├── phase03-storage-buffer.test.ts # Redis durability, cooperative CSV locks, streaming row count, export filters - ├── phase05_sdk_test.py # Python SDK WebSocket/HTTP fallback, offline spool, file streaming - ├── phase06-pwa-landing.test.ts # PWA manifest/icons/sw + navigation routes - └── phase07-deployment.test.ts # Compose/Redis AOF, Nginx /health, lockfile, prune guard, shutdown flush + ├── phase04-ui-sync.test.ts # UI/API settings bounds, bridge status, UTC presets, timestamp & folder matching + ├── phase05-packaging.test.ts # Python/Node packaging parity, PEP 561 py.typed, Node engines & start-all exit + ├── phase05_sdk_test.py # Python SDK WebSocket/HTTP fallback, offline spool, file streaming, edge hardening + ├── phase06-pwa-landing.test.ts # PWA manifest/icons/sw + navigation routes + └── phase07-deployment.test.ts # Compose/Redis AOF, Nginx /health, lockfile, prune guard, shutdown flush ``` --- @@ -175,10 +191,114 @@ All planned hardening & deployment phases (**Phase-01 through Phase-07**) are ** 7. **Docker / Coolify Deployment & Persistence Hardening (Completed — Phase-07)** - Redis AOF persistence (`redisdata:/data`), deterministic `package-lock.json` for `npm ci`, Nginx `location = /health` → `ingestion:4000`, `/api/health` container probes, non-blocking startup ownership check in Dockerfiles, unhydrated-DB `autoPruneDays` safety guard in `SettingsManager`, DB JWT hydration in `SDKManager`, graceful `SIGTERM`/`SIGINT` flush in `DataBufferEngine`, and non-destructive `ADD COLUMN IF NOT EXISTS` DDL guards. -- **Current Forensic Audit Status (2026-11-18)**: - - All prior notes in `notes/` were cleared and replaced exclusively with `notes/forensic-audit-report.md` backed strictly by source-code citations and verbatim proofs (zero markdown extrapolation). - - Baseline checks currently pass (`24/24` TS tests, `eslint .` clean, `3/3` Python unit tests), while deep source inspection uncovered **27 verified defects/gaps** (`5` Critical, `11` High, `10` Medium, `1` Low) across Auth/Middleware (`A01–A05`), Demo-vs-Prod Split-Brain (`B01–B05`), Buffer/CSV Storage (`C01–C06`), Admin UI vs Backend Reality (`D01–D07`), and SDK/Packaging/PWA (`E01–E04`). - - Next step pending user approval: organize remediation into an incremental phased plan based strictly on `notes/forensic-audit-report.md`. +- **Current Remediation Status (2026-11-18)**: + - **Phase-01 Complete & Verified** (`BUG-A01`, `BUG-A02`, `BUG-A03`, `BUG-A04`, `BUG-A05`, `BUG-E02`): + - Removed spoofable header/UA CSRF bypasses in `middleware.ts` and rejected `sec-fetch-site: cross-site`. + - Enforced DB-backed `tokenVersion` session validation across `/vcon/*` layout (`app/vcon/layout.tsx` + `components/layout/VconShell.tsx`) and read/export APIs (`/api/data`, `GET /api/events`, `/api/export`, `/api/metrics`, `/api/storage`). + - Prevented transient DB errors from triggering false `/vcon/setup` redirects and prevented swallowed `NEXT_REDIRECT` errors on `/`, `/vcon/login`, and `/vcon/setup`. + - Standardized `SDKManager.normalizeClientId`, blocked deleted/unregistered clients on `/api/health` and Admin `POST /api/events`, hardened `POST /api/clients` & `PATCH /api/clients/[id]` input validation, and excluded `/vcon/*` from Service Worker caching with `CLEAR_AUTH_CACHE` on sign-out. + - Verified: `26/26` TS tests, `3/3` Python tests, `npm run build` clean, `eslint .` clean, Playwright browser auth/logout E2E clean. + - **Phase-02 Complete & Verified** (`BUG-B01`–`BUG-B05`): + - Enforced fail-closed client verification in `server/index.js` (`verifyClientInDb`) backed by PG, deletion tombstones, Redis `datastack:clients:*` mirrors, and normalized `clientId` keys across both runtimes. + - Added cross-runtime Redis idempotency (`datastack:idem:*`) and fixed-window rate limiting (`datastack:rl:*`) in `core/Security/security.ts` (`validateEventAsync`) and `server/socket/handlers.js` (`validateSingleEventAsync`) with automatic local RAM fallback. + - Surfaced bridge sync telemetry (`BridgeSyncResult`, `SettingsBridgeSyncStatus`, `runtime.ingestionBridgeStatus`, `POST /api/settings` `bridgeSync`) and normalized `SDKManager.incrementEventCountAsync()`. + - Fixed Socket `sync_data` flat payload preservation (`extractSingleEventData`), reconciled `PORT_SOCKET`/`PORT` precedence and live `/internal/metrics` ports, and added concurrent production `scripts/start-all.js` for `npm start`. + - Verified: `29/29` TS tests, `3/3` Python tests, `npm run build` clean, `eslint .` clean, Playwright browser check (`/vcon/dashboard`, `/vcon/settings`, `/api/metrics`) clean with `0` console errors. + - **Phase-03 Complete & Verified** (`BUG-C01`–`BUG-C06`): + - Hardened Redis flush lock TTL (`30000ms`), added in-process `activeFlushPromise` coalescing with a `flushDone` identity guard (`core/data/buffer.ts`, `server/index.js`), isolated `redis.rpush` from best-effort `REDIS_RECENT_KEY`, and logged malformed buffered JSON warnings. + - Enforced `sanitizeClientId()` path containment and `lstat` symlink exclusion, unique lock owner tokens (`${pid}:${ts}:${uuid}`), atomic `rename` stale-lock takeover, owner-verified lock unlock, and structured `JSON.stringify([clientId, date])` partition grouping in `core/Storage/csvStorage.ts` and `server/workers/csvWriter.js`. + - Fixed trailing newline blank-row prevention (`fileEndsWithNewline`), proactive size-boundary rotation (`-partN.csv`), `StringDecoder('utf8')` streaming row counting, cooperatively locked `pruneExpiredPartitions`, and 1-hour `PRUNE_COOLDOWN_MS` in `core/adminSettings/settingsManager.ts`. + - Enforced `parser`/`fileStream` destruction in `finally` blocks across `/api/data`, `GET /api/events`, and `/api/export`, error propagation inside `makeExportStream`, `502` surfacing on Port `4000` flush failure in `/api/export`, normalized `clientId` filtering, numeric timestamp sorting (`parseTimestampMs`), and `?format=json` / `?includeHeader=false` parity for `?filename=` exports. + - Verified: `32/32` TS tests, `3/3` Python tests, `npm run build` clean, `eslint .` clean, Playwright browser check (`/vcon/dashboard`, `/vcon/data`, `/vcon/storage`, `/vcon/export`, `/api/data`, `/api/storage`, `/api/export`) clean with `0` console errors. + - **Phase-04 Complete & Verified** (`BUG-D01`–`BUG-D07`): + - Extracted pure UI synchronization helpers in `lib/vconUiHelpers.ts` (`SETTINGS_LIMITS`, `normalizeEditableSettings`, `describeSettingsBridgeSync`, `mapBridgeStatusToSocketUiState`, `parseEventTimestamp`, `formatEventTimeLabel`, `getUtcDateRangeForPreset`, `matchesSelectedClientFolder`, `extractContentDispositionFilename`). + - Updated `components/sections/SettingsWorkspace.tsx` (`BUG-D01`) to probe `/api/metrics` honestly (removing false `/api/health` fallback), surface `bridgeSync` warning/error toasts on save, align integer bounds with `/api/settings` (`1..100000`, `1..10240`, `0..3650`), and eliminate the startup `serverPort` double-fetch. + - Updated `app/vcon/storage/page.tsx` (`BUG-D02`) and `components/sections/ExportWorkspace.tsx` (`BUG-D03`) with `cache: 'no-store'`, `401` redirects, UTC preset ranges, normalized client folder filters, and fetch+Blob downloads with `502`/JSON error surfacing. + - Updated `components/sections/DashboardOverview.tsx` (`BUG-D04`) and `components/sections/DataBrowser.tsx` (`BUG-D05`) with `ingestionBridgeStatus` display, `auth_error` alert banner, safe ISO/epoch-ms timestamp formatting, invalid date-range guard, and Enter-key filter submission. + - Updated `components/sections/NewClientModal.tsx`, `components/sections/ClientsTable.tsx`, `app/vcon/simulator/page.tsx`, `app/vcon/sdk/page.tsx`, `components/sections/ProfileWorkspace.tsx`, and `components/layout/Sidebar.tsx` (`BUG-D06`, `BUG-D07`) with `1..100000` rate limit parity, live `runtime.serverPort` sync, real HTTP batch ingestion in Simulator, password bounds check, and `datastack:profile-updated` live Sidebar sync. + - Verified: `37/37` TS tests (`tests/phase04-ui-sync.test.ts`), `3/3` Python tests, `npm run build` clean, `eslint .` clean, Playwright browser check across all Admin Console routes clean with `0` console errors. + - **Phase-05 Complete & Verified** (`BUG-E01`, `BUG-E03`, `BUG-E04`): + - Hardened canonical Python SDK (`sdk/python/datastack/datastack/*`): normalized `CLIENT_ID` comparison (`_normalize_client_id`), sanitized ragged CSV extra columns (`restkey="_extra"`), accurate `send_file()` `success` + `unprocessedItems`, partial-batch `rateLimitedEventIds` retry exception fallback, `DataStackConfigError` wrapping for malformed numeric env/config values (`config.py`), concurrent `stop()` vs `set_idle_connected()` race fix + clean thread join (`heartbeat.py`), `os.fsync` + orphan `.tmp` unlink + warning logs (`spool.py`), and PEP 561 `py.typed` marker. + - Aligned packaging metadata across `sdk/python/datastack/pyproject.toml`, `sdk/python/datastack/setup.py`, and `sdk-python/setup.py`, coordinated child-exit shutdown in `scripts/start-all.js`, and added `engines.node >=20.0.0` in `package.json`. + - Verified: `39/39` TS tests (`tests/phase05-packaging.test.ts`), `6/6` Python tests (`tests/phase05_sdk_test.py`), `py_compile`, `npm run build`, and `eslint .` clean. + - **Previous Remediation Cycle (2026-11-18)**: Completed Phases 01–05 (`39/39` TS tests, `6/6` Python tests). + - **New Production Hardening Cycle (`BASELINE-2026-11-19`)**: + - Re-audited entire codebase from raw source code (`notes/FORENSIC_AUDIT_REPORT.md`): **29** verified bugs/mismatches (`4` Critical, `8` High, `13` Medium, `4` Low). + - **Phase-01 Complete & Verified (`2026-11-19`)** (`BUG-01`, `BUG-04`, `BUG-05`, `BUG-06`): + - Blocked volatile RAM admin creation in production without `DATABASE_URL` (`AdminDatabaseRequiredError`), surfaced production DB warning on `/vcon/setup`, and mapped pre-setup login to `409 SETUP_REQUIRED`. + - Added brute-force attempt throttling (`429` + `Retry-After`) and completion-first ordering to `POST /api/auth/setup`. + - Added server-side `token_version` revocation (`revokeAdminSessions()`) on `POST /api/auth/logout`, removed client `Origin` trust from cookie `Secure` flag, and added `sec-fetch-site: same-origin` CSRF fallback. + - Prevented empty production `jwt_secret` persistence in PG and replaced silent DB write `catch {}` swallows in `SettingsManager` and `SDKManager` with `DATABASE_WRITE_FAILED` propagation and safe RAM rollback. + - Verified: `43/43` TS tests, `6/6` Python tests, `npm run build`, `eslint .`, and Playwright login/logout revocation check clean. + - **Phase-02 Complete & Verified (`2026-11-19`)** (`BUG-02`, `BUG-03`, `BUG-17`, `BUG-22`): + - Protected Port 4000's `inMemoryClientRegistry` from empty Port 3000 `bulk_sync` wipes (`!forceReplace`), exposed `inMemoryClients` on `/internal/metrics`, added non-destructive hydration in `SDKManager` (`listClientsAsync`, `getClientAsync`, `syncAllClientsToIngestionAsync`), and auto-reconciled Port 4000 cold starts in `ApiEngine.getDashboardMetrics()`. + - Ordered `SettingsManager.updateSettingsAsync()` JWT rotation to sync Port 4000 *before* committing Port 3000 RAM/PG/re-sign, aborting fail-closed (`JWT_ROTATION_BRIDGE_FAILED`) on bridge rejection/prod unreachable, rolling back Port 4000 on PG/re-sign error, and adding a 60s dual-secret grace window in `server/index.js` `requireInternalAuth()`. + - Added authenticated `POST /internal/security-check` on Port 4000 (backed by zero-copy `validateSingleEventLocal`) so Port 3000 HTTP `SecurityEngine.validateEventAsync()` shares rate limits & idempotency with Port 4000 WS even without Redis, and exposed explicit `securityMode` (`redis_shared` | `bridge_coordinated` | `degraded_local`) in `/api/metrics`. + - Replaced per-event PG `UPDATE client_sdks` queries on both Port 3000 (`SDKManager`) and Port 4000 (`server/index.js`) with 1s coalescing in-memory deltas flushed via batched `UNNEST` SQL updates (flushed on reads and graceful shutdown). + - Verified: `47/47` TS tests, `6/6` Python tests, `npm run build`, and `eslint .` clean. + - **Phase-03 Complete & Verified (`2026-11-19`)** (`BUG-18`, `BUG-19`, `BUG-20`, `BUG-29`): + - Added bounded Redis flush lock wait/retry, periodic Lua `PEXPIRE` TTL renewal with ownership fencing before `LTRIM`, and explicit `FLUSH_LOCK_TIMEOUT` (`503` on `/internal/flush`) across both `core/data/buffer.ts` and `server/index.js`. + - Added full deleted-client buffer purge (`DataBufferEngine.purgeClientEventsAsync`, Port 4000 `purgeClientFromLocalBuffer`) plus flush-time and recent-read tombstone filtering so deleted client directories never resurrect on subsequent flushes; ordered `SDKManager.deleteClientAsync()` to purge buffers and sync Port 4000 before deleting disk files. + - Converted `server/index.js` `onBufferPush` to `async` and awaited it before firing Socket.IO ACK callbacks in `server/socket/handlers.js` (`sync_data` and `batch_sync`), returning `BUFFER_PERSIST_FAILED` on persistence error. + - Unified canonical CSV storage root resolution via `DATASTACK_STORAGE_DIR` (`resolveDefaultStorageDir()`) across `core/Storage/csvStorage.ts`, `server/workers/csvWriter.js`, `core/sdk/sdkManager.ts`, and `.env.example`. + - Verified: `51/51` TS tests, `6/6` Python tests, `npm run build`, and `eslint .` clean. + - **Phase-04 Complete & Verified (`2026-11-19`)** (`BUG-21`, `BUG-23`, `BUG-24`): + - Aligned HTTP `POST /api/events` single-event envelope stripping 1:1 with Port 4000 WS `sync_data` and eliminated fabricated `{ sample, rand }` telemetry on empty `{}`. + - Converted `GET /api/data` directory/stat scans to non-blocking async `node:fs/promises` with newest-first date-partition pruning, and scaled `REDIS_RECENT_LIMIT` boundedly (`500..5000`) via `resolveRedisRecentLimit()` / `DATASTACK_RECENT_LIMIT` across Port 3000 & Port 4000. + - Replaced all `Math.random()` client secrets (`crypto.randomBytes(16)`) and fallback `eventId`s (`crypto.randomBytes(6)`) with CSPRNG across `core/sdk/sdkManager.ts`, `app/api/events/route.ts`, and `server/socket/handlers.js`. + - Unified canonical PostgreSQL metadata DDL (`CORE_SCHEMA_TABLE_DDLS` + `CORE_SCHEMA_ALTER_DDLS` + `ensureCoreSchemaOnPool()`) in `core/Database/database.ts`, delegated `core/auth/adminUsers.ts` to it, mirrored all 5 tables + 3 additive `ALTER TABLE` guards 1:1 in standalone `server/index.js`, and documented canonical `pg` ownership in `prisma/schema.prisma`. + - Verified: `55/55` TS tests (`tests/phase04-ingestion-hygiene.test.ts` added), `6/6` Python tests, `npm run build`, and `eslint .` clean. + - **Phase-05 Complete & Verified (`2026-11-19`)** (`BUG-07`, `BUG-08`, `BUG-09`, `BUG-14`): + - Fixed Simulator HTTP `Burst 10` to send canonical `{ clientId, batch }` (with `events` batch alias & explicit `accepted` count in `POST /api/events`) and summarize exact server-confirmed counts via `summarizeHttpBatchResult()` without `res.ok ? count : 0` falsification. + - Fixed `parseEventTimestamp()` & added `getEventUtcDate()` in `lib/vconUiHelpers.ts` for integer and fractional epoch-seconds (`< 1e11 * 1000`) 1:1 with backend CSV parsers. + - Added pending-queue-only `getPendingBufferedSummary()` (`?bufferedOnly=true` / `?includeBufferedSummary=true`) and fair multi-client partition scanning (`selectRecentPartitionFiles()`, lifting the global 5-file cap) in `GET /api/events`. + - Updated `ExportWorkspace.tsx` to match buffered events strictly against active client/date filters via `countMatchingBufferedEvents()` (preventing empty filtered header downloads) and refresh `/api/storage`, `/api/metrics`, and `vconCache` immediately post-export; updated `DashboardOverview.tsx` to attribute strictly UTC-today buffered events to `Events today` and call `await refresh()` after Quick Export. + - Verified: `58/58` TS tests, `6/6` Python tests, `npm run build`, `eslint .`, and Playwright E2E browser check (`/vcon/simulator`, `/vcon/export`, `/vcon/dashboard`, `0` console errors/warnings) clean. + - **Phase-06 Complete & Verified (`2026-11-19`)** (`BUG-10`, `BUG-11`, `BUG-12`, `BUG-13`, `BUG-15`, `BUG-16`): + - Returned explicit `SESSION_EXPIRED` / `AUTH_UNAVAILABLE` vs `INVALID_PASSWORD` codes in `/api/auth/profile`, replaced dead `401 && !currentPassword` redirect with `shouldRedirectProfileUnauthorized()`, and formatted `Session Expires` with calendar date + 24h time (`formatSessionExpiryLabel()`). + - Added opt-in `?purgeFiles=true` checkbox in `ClientsTable.tsx` client deletion modal and removed the misleading unused `Secret key` row from `CredentialsDisplay`. + - Refactored `SettingsWorkspace.tsx` numeric inputs to string drafts (`NumericSettingDrafts`) validated inline via `validateEditableSettingsDraft()` (`noValidate` submit + tab jump to first invalid field, zero snap-to-`0`, zero silent clamping) and clarified `Socket port (PORT_SOCKET)` as read-only runtime config. + - Unified public endpoint resolution (`resolvePublicEndpoints()`) across `SettingsWorkspace`, `/vcon/sdk`, and `/vcon/simulator` (consistent on port-80 `localhost` proxies) and updated `SdkSection.tsx` + `/vcon/sdk` with external path/wheel `pip install` guidance. + - Added pre-hydration `datastack-theme` bootstrap script + `suppressHydrationWarning` in `app/layout.tsx` (zero light-mode FOUC) and theme restoration/toggle in `AdminSetup.tsx`. + - Upgraded `public/sw.js` (`datastack-cache-v3`) with an uncached `503` HTML offline navigation fallback shell (`buildOfflineShellResponse()`, `Cache-Control: no-store`) for `/vcon/dashboard` and public navigations. + - Verified: `64/64` TS tests, `6/6` Python tests, `npm run build`, `eslint .`, and Playwright Chromium E2E check (`/vcon/profile`, `/vcon/clients`, `/vcon/settings`, `/vcon/sdk`, Light Mode reload, `0` console errors/warnings) clean. + - **Fresh Post-Phase-07 Forensic Audit & Baseline Freeze (`BASELINE-2026-11-19-R2`)**: + - Previous `Phase-07` hardening (`BUG-25`–`BUG-28`) landed in Python SDK (`client.py`), legacy `sdk-python`, `scripts/start-all.js` / `dev-all.js`, `nginx/default.conf`, `server/index.js`, `docker/docker-compose.yml`, and `.env.example`. + - Conducted a brand-new 100% raw-source-code audit (`notes/FORENSIC_AUDIT_REPORT.md`) identifying **18 verified remaining bugs** (`0` Critical, `6` High, `8` Medium, `4` Low). + - Frozen baseline (`BASELINE-2026-11-19-R2`) and created the complete 7-document governance & Scope-Lock suite in `notes/` (`README.md`, `BASELINE_FREEZE.md`, `PRODUCTION_READINESS_SCOPE_LOCK.md`, `PRODUCTION_ROADMAP.md`, `PHASE_EXECUTION_LOG.md`, `ERROR_HANDLING_MATRIX.md`, `ACTUAL_IMPLEMENTATION_STATUS.md`). + - **R2 Phase-01 Complete & Verified (`2026-11-19`)** (`BUG-01`, `BUG-02`): + - Moved same-origin CSRF enforcement in `middleware.ts` ahead of the public-route early return so `POST /api/auth/login` and `POST /api/auth/setup` reject cross-origin/missing-origin requests with `403` while preserving same-origin browser submissions, `sec-fetch-site: same-origin` fallback, safe `GET` routes, and Bearer `POST /api/events`. + - Extracted shared reverse-proxy-safe `extractClientIp()` in `lib/adminAuth.ts` (trusting `x-real-ip` first, then rightmost non-empty `x-forwarded-for` hop) and wired it into both `app/api/auth/login/route.ts` and `app/api/auth/setup/route.ts` to block spoofed-XFF brute-force lockout bypasses. + - Verified: `14/14` `tests/phase01-auth.test.ts` pass, `npm run build`, `npm run lint`, live HTTP `403`/`401` CSRF probes, and Chromium `/vcon/login` → `/vcon/dashboard` E2E (`0` errors). + - **R2 Phase-02 Complete & Verified (`2026-11-19`)** (`BUG-05`, `BUG-06`, `BUG-07`, `BUG-08`, `BUG-09`, `BUG-10`): + - Attached `.on('error', ...)` listeners to `DatabaseEngine.getPool()` (`max: 10`) and `server/index.js` `dbPool`, and consolidated `core/auth/adminUsers.ts` onto the shared `DatabaseEngine.getPool()`. + - Replaced non-atomic Redis `LTRIM`+`LPUSH` client deletion purging in `core/data/buffer.ts` with atomic Lua script `PURGE_CLIENT_LIST_LUA_SCRIPT` (`cjson.decode` + chunked `RPUSH`), and replaced non-atomic Redis `INCR`+`PEXPIRE` rate limiting with atomic Lua script `REDIS_RATE_LIMIT_LUA_SCRIPT` across both `core/Security/security.ts` and `server/socket/handlers.js`. + - Enforced `throwOnLockTimeout: true` locally and remotely on `POST /internal/flush` in `GET /api/export` and `POST /api/storage`, surfacing explicit `503 FLUSH_LOCK_TIMEOUT`. + - Added `DataBufferEngine.registerShutdownHook()` and registered `SDKManager.flushPendingClientActivityAsync()` so Port `3000` flushes coalesced `total_events` deltas on `SIGTERM`/`SIGINT`. + - Wrapped `GET`, `PATCH` (`regenerate_token` & update), and `DELETE` in `app/api/clients/[id]/route.ts` with structured JSON `try/catch` error responses. + - Verified: `14/14` `tests/phase02-sync.test.ts` pass, `3/3` `R2 PHASE-02` buffer/storage tests pass, `npm run build` pass, `npm run lint` pass (`0` errors). + - **R2 Phase-03 Complete & Verified (`2026-11-19`)** (`BUG-03`, `BUG-04`, `BUG-11`, `BUG-12`, `BUG-13`, `BUG-15`): + - Unified `SettingsBridgeSyncStatus` & `POST /api/settings` `bridgeSync` response (`attempted`, `ok`, `synced`, `status`, `error`, `checkedAt`) and updated `describeSettingsBridgeSync()` + `SettingsWorkspace.tsx` so Port `4000` warnings/online state work reliably. + - Counted regular non-symlink `.csv` files in `SDKManager.purgeClientStorageDir()` (`deleteClientWithStatsAsync()`) and returned exact `purgedFiles` from `DELETE /api/clients/[id]?purgeFiles=true`. + - Aligned flat batch item metadata stripping (`eventId`, `timestamp`, `clientId`) 1:1 with single-event stripping across HTTP `POST /api/events`, WS `batch_sync`, and Python SDK `send_batch()`/`send_file()`. + - Exposed `DataBufferEngine.getLocalBufferSnapshot()` for exact Port `3000` RAM pending summary in `getPendingBufferedSummary()`. + - Added non-blocking `pruneExpiredPartitionsAsync()` off live event loops in `core/Storage/csvStorage.ts` & `server/workers/csvWriter.js` (wired into `settingsManager.ts` & `server/index.js`) and deduplicated `GET /api/storage` to a single `listAllFilesAsync()` scan with accurate client-scoped `stats`. + - Cleared `selectedClientToken` on `/vcon/sdk` client switch (with loading placeholder + disabled `Copy Config`), added `400 INVALID_DATE_RANGE` for `from > to` in `GET /api/export`, and normalized `clientId` filtering in `ApiEngine.getExportFiles*` & Port `4000` `/internal/recent`. + - Verified: `21/21` `phase04-*` tests pass, `2/2` `R2 PHASE-03` storage tests pass, `9/9` Python SDK tests pass, Chromium E2E across `/vcon/settings`, `/vcon/clients`, `/vcon/sdk`, `/vcon/storage` (`0` console errors), `npm run build` & `npm run lint` (`0` errors). + - **R2 Phase-04 Complete & Verified (`2026-11-19`)** (`BUG-14`, `BUG-16`, `BUG-17`, `BUG-18`): + - Added `from __future__ import annotations`, imported `Tuple`, and replaced PEP 585 `tuple[int, Dict[str, Any]]` with Python `3.8`-safe `Tuple[int, Dict[str, Any]]` in `sdk/python/datastack/datastack/client.py`, verified by a full-package Python `3.8` AST scan test in `tests/phase05_sdk_test.py`. + - Added unbuffered streaming `location = /api/export` (`proxy_buffering off;`, `proxy_read_timeout 300s;`) targeting `http://nextjs_upstream` in both `nginx/default.conf` and `docker/coolify-nginx.conf`. + - Removed `eslint: { ignoreDuringBuilds: true }` from `next.config.ts`, set `package.json` `"name": "datastack"`, pruned unused template packages (`@google/genai`, `firebase-tools`), and synchronized `package-lock.json`. + - Isolated `DATASTACK_DISABLE_LISTEN=1` and live Port `4000` coupling across `tests/phase01-auth.test.ts`, `tests/phase02-sync.test.ts`, `tests/phase03-storage-buffer.test.ts`, `tests/phase04-ingestion-hygiene.test.ts`, and `tests/phase07-deployment.test.ts`. + - Verified: `83/83` full TS suite pass (with live Port `4000` running, without manual env flags), `10/10` Python SDK tests + `py_compile` pass, `npm run build` (with ESLint active) pass, `npm run lint` (`0` errors, `0` warnings). + - **Progress**: `4 / 4` phases complete (`18 / 18` bugs resolved — **100% Production Ready**). + - **Pre-Release Docker & Coolify Deployment Hardening Complete & Verified (`2026-11-19`)** (`DEPLOY-BUG-01`–`05`, audited in `notes/DEPLOYMENT_COOLIFY_AUDIT_REPORT.md`): + - **`DEPLOY-BUG-01`**: Replaced static `upstream` blocks in `docker/coolify-nginx.conf` with Docker embedded DNS `resolver 127.0.0.11 valid=10s ipv6=off;` and variable `proxy_pass` targets (`$nextjs_upstream` = `http://web:3000`, `$socketio_upstream` = `http://ingestion:4000`), preventing `502 Bad Gateway` stale container IP outages after container restarts/redeploys. + - **`DEPLOY-BUG-02`**: Refactored Port `4000` startup in `server/index.js` into `startIngestionServer()` awaiting bounded `syncSettingsFromDatabase()` before `server.listen(PORT)` so rotated PG `jwt_secret` and rate-limit settings hydrate prior to `/health: 200` and `/internal/*` bridge traffic. + - **`DEPLOY-BUG-03`**: Memoized `ensureDatabaseSchema()` per active `dbPool` instance in `server/index.js` (resetting only on pool swap or DDL failure) so 3-second `syncSettingsFromDatabase()` ticks no longer issue 8 `CREATE/ALTER TABLE` DDLs repeatedly. + - **`DEPLOY-BUG-04`**: Reordered `gracefulShutdown()` in `server/index.js` to close Socket.IO/HTTP ingress first (`closeServerGracefully()`), drain `pendingRedisWrites` and `flushBuffer()`/`flushPendingClientActivity()` with bounded timeouts (`2000ms`/`6000ms`), and close Redis (`quit()` with `2000ms` timeout + `.disconnect()` fallback) and PostgreSQL (`dbPool.end()` bounded `2000ms`); added `stop_grace_period: 20s` to both `web` and `ingestion` in `docker/docker-compose.yml`. + - **`DEPLOY-BUG-05`**: Clarified Coolify `Base Directory: /` and `Docker Compose Location: /docker/docker-compose.yml` in `docs/users/installation.md` and `README.md`. + - Verified: `85/85` full TS suite pass (`11/11` in `tests/phase07-deployment.test.ts`), `10/10` Python SDK tests pass, `npm run build` clean, `npm run lint` (`0` errors, `0` warnings). --- diff --git a/README.md b/README.md index 49d7b94..157e73c 100644 --- a/README.md +++ b/README.md @@ -293,7 +293,7 @@ DataStack includes a production-ready Docker Compose stack in [`docker/docker-co ### Coolify Deployment Steps 1. Create a **Docker Compose** resource in Coolify pointing to your repository. -2. Set **Base Directory** to `/docker` and **Docker Compose Location** to `/docker/docker-compose.yml`. +2. Set **Base Directory** to `/` and **Docker Compose Location** to `/docker/docker-compose.yml`. 3. Attach your HTTPS domain to the `proxy` service on port `80`. 4. Deploy the stack, open `https://your-domain.example/vcon/setup`, and enter your administrator email, password (8+ characters), and the generated `SERVICE_REALBASE64_64_ADMININSTALL` key from Coolify's environment variables. diff --git a/app/(landing)/page.tsx b/app/(landing)/page.tsx index 7545dd7..e92e705 100644 --- a/app/(landing)/page.tsx +++ b/app/(landing)/page.tsx @@ -9,8 +9,20 @@ import { redirect } from 'next/navigation'; export const dynamic = 'force-dynamic'; export default async function LandingPage() { - const isConfigured = await isAdminSetupComplete().catch(() => false); - if (!isConfigured) redirect('/vcon/setup'); + try { + const isConfigured = await isAdminSetupComplete(); + if (!isConfigured) redirect('/vcon/setup'); + } catch (error) { + if ( + error && + typeof error === 'object' && + 'digest' in error && + String((error as { digest?: unknown }).digest).startsWith('NEXT_REDIRECT') + ) { + throw error; + } + // Keep public landing accessible on transient DB outage instead of redirecting to setup + } return ( diff --git a/app/api/auth/login/route.ts b/app/api/auth/login/route.ts index afd12a6..9c72d08 100644 --- a/app/api/auth/login/route.ts +++ b/app/api/auth/login/route.ts @@ -4,27 +4,18 @@ import { getSessionCookieOptions, } from "@/core/auth/adminSession"; import { + AdminSetupRequiredError, authenticateAdminCredentials, clearAdminLoginAttempts, consumeAdminLoginAttempt, } from "@/core/auth/adminUsers"; +import { extractClientIp } from "@/lib/adminAuth"; import { NextRequest, NextResponse } from "next/server"; const SESSION_COOKIE = "datastack_admin_session"; const SESSION_SECONDS = 8 * 60 * 60; const REMEMBERED_SESSION_SECONDS = 30 * 24 * 60 * 60; -function extractClientIp(request: NextRequest) { - const forwardedFor = request.headers.get("x-forwarded-for"); - if (forwardedFor) { - const first = forwardedFor.split(",")[0]?.trim(); - if (first) return first; - } - const realIp = request.headers.get("x-real-ip")?.trim(); - if (realIp) return realIp; - return "unknown"; -} - export async function POST(request: NextRequest) { const sessionSecret = getAdminSessionSecret(); @@ -91,7 +82,16 @@ export async function POST(request: NextRequest) { let authResult: { authenticated: boolean; tokenVersion: number }; try { authResult = await authenticateAdminCredentials(email, password); - } catch { + } catch (err) { + if (err instanceof AdminSetupRequiredError) { + return NextResponse.json( + { + error: "Initial admin setup is required before signing in.", + code: err.code, + }, + { status: 409 }, + ); + } return NextResponse.json( { error: "Admin authentication is temporarily unavailable." }, { status: 503 }, diff --git a/app/api/auth/logout/route.ts b/app/api/auth/logout/route.ts index 9ee5ed8..627a604 100644 --- a/app/api/auth/logout/route.ts +++ b/app/api/auth/logout/route.ts @@ -1,9 +1,25 @@ import { getSessionCookieOptions } from "@/core/auth/adminSession"; +import { + revokeAdminSessions, + verifyRequestAdminSession, +} from "@/core/auth/adminUsers"; import { NextRequest, NextResponse } from "next/server"; const SESSION_COOKIE = "datastack_admin_session"; export async function POST(request: NextRequest) { + const token = request.cookies.get(SESSION_COOKIE)?.value; + if (token) { + try { + const check = await verifyRequestAdminSession(token); + if (check.valid) { + await revokeAdminSessions(); + } + } catch { + // Still clear client cookie even if server revocation encounters transient DB error + } + } + const response = NextResponse.json({ success: true }); response.cookies.set(SESSION_COOKIE, "", { ...getSessionCookieOptions(request, 0), diff --git a/app/api/auth/profile/route.ts b/app/api/auth/profile/route.ts index 9cdee9c..13f5f66 100644 --- a/app/api/auth/profile/route.ts +++ b/app/api/auth/profile/route.ts @@ -22,11 +22,13 @@ export async function GET(request: NextRequest) { if (!auth.valid) { const status = auth.reason === "UNAUTHORIZED" ? 401 : 503; + const code = + auth.reason === "UNAUTHORIZED" ? "SESSION_EXPIRED" : "AUTH_UNAVAILABLE"; const message = auth.reason === "UNAUTHORIZED" ? "Authentication required." : "Admin authentication is temporarily unavailable."; - return NextResponse.json({ error: message }, { status }); + return NextResponse.json({ error: message, code }, { status }); } try { @@ -54,11 +56,13 @@ export async function PUT(request: NextRequest) { if (!auth.valid) { const status = auth.reason === "UNAUTHORIZED" ? 401 : 503; + const code = + auth.reason === "UNAUTHORIZED" ? "SESSION_EXPIRED" : "AUTH_UNAVAILABLE"; const message = auth.reason === "UNAUTHORIZED" ? "Authentication required." : "Admin authentication is temporarily unavailable."; - return NextResponse.json({ error: message }, { status }); + return NextResponse.json({ error: message, code }, { status }); } let body: Record; @@ -115,7 +119,10 @@ export async function PUT(request: NextRequest) { }); if (!result.ok) { - return NextResponse.json({ error: result.error }, { status: 401 }); + return NextResponse.json( + { error: result.error, code: "INVALID_PASSWORD" }, + { status: 401 }, + ); } // Preserve remaining session duration (e.g., 30-day "Remember me" session) diff --git a/app/api/auth/setup/route.ts b/app/api/auth/setup/route.ts index 331a20b..3a7152c 100644 --- a/app/api/auth/setup/route.ts +++ b/app/api/auth/setup/route.ts @@ -4,14 +4,20 @@ import { getSessionCookieOptions, } from "@/core/auth/adminSession"; import { + AdminDatabaseRequiredError, + clearAdminLoginAttempts, + consumeAdminLoginAttempt, createInitialAdmin, isAdminSetupComplete, + requiresPersistentAdminDatabase, verifyAdminSetupKey, } from "@/core/auth/adminUsers"; +import { extractClientIp } from "@/lib/adminAuth"; import { NextRequest, NextResponse } from "next/server"; const SESSION_COOKIE = "datastack_admin_session"; const SESSION_SECONDS = 8 * 60 * 60; +const SETUP_THROTTLE_BUCKET = "__admin_setup__"; export const runtime = "nodejs"; export const dynamic = "force-dynamic"; @@ -29,6 +35,17 @@ export async function POST(request: NextRequest) { ); } + if (requiresPersistentAdminDatabase()) { + return NextResponse.json( + { + error: + "PostgreSQL DATABASE_URL must be configured in production before initial admin setup.", + code: "DATABASE_NOT_CONFIGURED", + }, + { status: 503 }, + ); + } + let body: Record; try { const parsed = await request.json(); @@ -61,12 +78,7 @@ export async function POST(request: NextRequest) { { status: 400 }, ); } - if (!verifyAdminSetupKey(setupKey)) { - return NextResponse.json( - { error: "The installation key is invalid." }, - { status: 403 }, - ); - } + if (await isAdminSetupComplete()) { return NextResponse.json( { error: "Initial admin setup has already been completed." }, @@ -74,6 +86,34 @@ export async function POST(request: NextRequest) { ); } + const clientIp = extractClientIp(request); + const throttleDecision = await consumeAdminLoginAttempt( + SETUP_THROTTLE_BUCKET, + sessionSecret, + clientIp, + ); + if (!throttleDecision.allowed) { + return NextResponse.json( + { + error: "Too many setup attempts. Try again later.", + code: "TOO_MANY_ATTEMPTS", + }, + { + status: 429, + headers: { + "Retry-After": String(throttleDecision.retryAfterSeconds), + }, + }, + ); + } + + if (!verifyAdminSetupKey(setupKey)) { + return NextResponse.json( + { error: "The installation key is invalid." }, + { status: 403 }, + ); + } + const created = await createInitialAdmin(email, password); if (!created) { return NextResponse.json( @@ -82,6 +122,12 @@ export async function POST(request: NextRequest) { ); } + await clearAdminLoginAttempts( + SETUP_THROTTLE_BUCKET, + sessionSecret, + clientIp, + ).catch(() => undefined); + const token = await createAdminSessionToken( email, sessionSecret, @@ -95,7 +141,13 @@ export async function POST(request: NextRequest) { getSessionCookieOptions(request, SESSION_SECONDS), ); return response; - } catch { + } catch (err) { + if (err instanceof AdminDatabaseRequiredError) { + return NextResponse.json( + { error: err.message, code: err.code }, + { status: 503 }, + ); + } return NextResponse.json( { error: "Admin setup is temporarily unavailable." }, { status: 503 }, diff --git a/app/api/clients/[id]/route.ts b/app/api/clients/[id]/route.ts index 8523234..2ce6929 100644 --- a/app/api/clients/[id]/route.ts +++ b/app/api/clients/[id]/route.ts @@ -3,6 +3,7 @@ import { SDKManager } from "@/core/sdk/sdkManager"; import { NextRequest, NextResponse } from "next/server"; const SESSION_COOKIE = "datastack_admin_session"; +const ALLOWED_PATCH_KEYS = new Set(["action", "name", "rateLimit", "status"]); export async function GET( req: NextRequest, @@ -19,20 +20,33 @@ export async function GET( } const { id } = await params; - const client = await SDKManager.getClientAsync(id); - if (!client) { - return NextResponse.json({ error: "Client not found" }, { status: 404 }); + const normalizedId = SDKManager.normalizeClientId(id); + try { + const client = await SDKManager.getClientAsync(normalizedId); + if (!client) { + return NextResponse.json({ error: "Client not found" }, { status: 404 }); + } + const { secret, token, ...clientSummary } = client; + void secret; + const includeToken = req.nextUrl.searchParams.get("includeToken") === "true"; + return NextResponse.json({ + client: { + ...clientSummary, + tokenPreview: token ? `${token.slice(0, 12)}...` : "", + ...(includeToken ? { token } : {}), + }, + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Failed to load client"; + const isDbError = message.includes("DATABASE_"); + return NextResponse.json( + { + error: message, + code: isDbError ? "DATABASE_READ_FAILED" : "CLIENT_FETCH_FAILED", + }, + { status: isDbError ? 503 : 500 }, + ); } - const { secret, token, ...clientSummary } = client; - void secret; - const includeToken = req.nextUrl.searchParams.get("includeToken") === "true"; - return NextResponse.json({ - client: { - ...clientSummary, - tokenPreview: token ? `${token.slice(0, 12)}...` : "", - ...(includeToken ? { token } : {}), - }, - }); } export async function PATCH( @@ -50,29 +64,128 @@ export async function PATCH( } const { id } = await params; - try { - const body = await req.json(); + const normalizedId = SDKManager.normalizeClientId(id); - // Special action: regenerate token - if (body.action === "regenerate_token") { - const refreshed = await SDKManager.regenerateTokenAsync(id); + let rawBody: unknown; + try { + rawBody = await req.json(); + } catch { + return NextResponse.json( + { error: "Invalid JSON request payload.", code: "INVALID_JSON" }, + { status: 400 }, + ); + } + + if (!rawBody || typeof rawBody !== "object" || Array.isArray(rawBody)) { + return NextResponse.json( + { error: "Update payload must be a JSON object.", code: "VALIDATION_ERROR" }, + { status: 400 }, + ); + } + + const body = rawBody as Record; + const unknownKeys = Object.keys(body).filter( + (key) => !ALLOWED_PATCH_KEYS.has(key), + ); + if (unknownKeys.length > 0) { + return NextResponse.json( + { + error: `Unsupported client field(s): ${unknownKeys.join(", ")}`, + code: "VALIDATION_ERROR", + }, + { status: 400 }, + ); + } + + // Special action: regenerate token + if (body.action !== undefined) { + if (body.action !== "regenerate_token") { + return NextResponse.json( + { error: "Unsupported client action.", code: "VALIDATION_ERROR" }, + { status: 400 }, + ); + } + try { + const refreshed = await SDKManager.regenerateTokenAsync(normalizedId); if (!refreshed) { - return NextResponse.json( - { error: "Client not found" }, - { status: 404 }, - ); + return NextResponse.json({ error: "Client not found" }, { status: 404 }); } return NextResponse.json({ client: refreshed }); + } catch (err: unknown) { + const message = + err instanceof Error ? err.message : "Token regeneration failed"; + const isDbWrite = message.includes("DATABASE_WRITE_FAILED"); + return NextResponse.json( + { + error: message, + code: isDbWrite ? "DATABASE_WRITE_FAILED" : "TOKEN_REGENERATION_FAILED", + }, + { status: isDbWrite ? 503 : 500 }, + ); } + } - const updated = await SDKManager.updateClientAsync(id, body); + const updates: { name?: string; rateLimit?: number; status?: "active" | "revoked" } = {}; + if (body.name !== undefined) { + if ( + typeof body.name !== "string" || + body.name.trim().length < 1 || + body.name.trim().length > 120 + ) { + return NextResponse.json( + { error: "Client name must be a non-empty string (max 120 characters).", code: "VALIDATION_ERROR" }, + { status: 400 }, + ); + } + updates.name = body.name.trim(); + } + if (body.rateLimit !== undefined) { + if ( + typeof body.rateLimit !== "number" || + !Number.isInteger(body.rateLimit) || + body.rateLimit < 1 || + body.rateLimit > 100000 + ) { + return NextResponse.json( + { error: "rateLimit must be an integer between 1 and 100000.", code: "VALIDATION_ERROR" }, + { status: 400 }, + ); + } + updates.rateLimit = body.rateLimit; + } + if (body.status !== undefined) { + if (body.status !== "active" && body.status !== "revoked") { + return NextResponse.json( + { error: "status must be 'active' or 'revoked'.", code: "VALIDATION_ERROR" }, + { status: 400 }, + ); + } + updates.status = body.status; + } + + if (Object.keys(updates).length === 0) { + return NextResponse.json( + { error: "No valid fields provided for update.", code: "VALIDATION_ERROR" }, + { status: 400 }, + ); + } + + try { + const updated = await SDKManager.updateClientAsync(normalizedId, updates); if (!updated) { return NextResponse.json({ error: "Client not found" }, { status: 404 }); } return NextResponse.json({ client: updated }); } catch (err: unknown) { const message = err instanceof Error ? err.message : "Update failed"; - return NextResponse.json({ error: message }, { status: 500 }); + const isDbWrite = message.includes("DATABASE_WRITE_FAILED"); + return NextResponse.json( + { + error: message, + code: isDbWrite ? "DATABASE_WRITE_FAILED" : "CLIENT_UPDATE_FAILED", + }, + { status: isDbWrite ? 503 : 500 }, + ); } } @@ -91,10 +204,29 @@ export async function DELETE( } const { id } = await params; + const normalizedId = SDKManager.normalizeClientId(id); const purgeFiles = req.nextUrl.searchParams.get("purgeFiles") === "true"; - const deleted = await SDKManager.deleteClientAsync(id, { purgeFiles }); - if (!deleted) { - return NextResponse.json({ error: "Client not found" }, { status: 404 }); + try { + const result = await SDKManager.deleteClientWithStatsAsync(normalizedId, { + purgeFiles, + }); + if (!result.deleted) { + return NextResponse.json({ error: "Client not found" }, { status: 404 }); + } + return NextResponse.json({ + success: true, + message: `Client ${normalizedId} deleted`, + purgedFiles: result.purgedFiles, + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : "Client deletion failed"; + const isDbWrite = message.includes("DATABASE_WRITE_FAILED"); + return NextResponse.json( + { + error: message, + code: isDbWrite ? "DATABASE_WRITE_FAILED" : "CLIENT_DELETE_FAILED", + }, + { status: isDbWrite ? 503 : 500 }, + ); } - return NextResponse.json({ success: true, message: `Client ${id} deleted` }); } diff --git a/app/api/clients/route.ts b/app/api/clients/route.ts index 000932f..d3ee91a 100644 --- a/app/api/clients/route.ts +++ b/app/api/clients/route.ts @@ -60,17 +60,29 @@ export async function POST(req: NextRequest) { try { const { clientId, name, rateLimit } = body; - if (!clientId || !name) { + if (typeof clientId !== "string" || typeof name !== "string" || !clientId.trim() || !name.trim()) { return NextResponse.json( - { error: "clientId and name are required" }, + { error: "clientId and name are required strings", code: "VALIDATION_ERROR" }, + { status: 400 }, + ); + } + + const safeClientId = SDKManager.normalizeClientId(clientId); + if (!safeClientId) { + return NextResponse.json( + { error: "clientId is invalid", code: "VALIDATION_ERROR" }, + { status: 400 }, + ); + } + + const parsedRateLimit = rateLimit === undefined ? 100 : Number(rateLimit); + if (!Number.isInteger(parsedRateLimit) || parsedRateLimit < 1 || parsedRateLimit > 100000) { + return NextResponse.json( + { error: "rateLimit must be an integer between 1 and 100000.", code: "VALIDATION_ERROR" }, { status: 400 }, ); } - const safeClientId = String(clientId) - .trim() - .toLowerCase() - .replace(/[^a-z0-9_-]/g, "_"); const existing = await SDKManager.getClientAsync(safeClientId); if (existing) { return NextResponse.json( @@ -81,8 +93,8 @@ export async function POST(req: NextRequest) { const newClient = await SDKManager.registerClientAsync({ clientId: safeClientId, - name: String(name), - rateLimit: rateLimit ? Number(rateLimit) : 100, + name: name.trim().slice(0, 120), + rateLimit: parsedRateLimit, }); return NextResponse.json({ client: newClient }, { status: 201 }); diff --git a/app/api/data/route.ts b/app/api/data/route.ts index 2afeca6..aba149d 100644 --- a/app/api/data/route.ts +++ b/app/api/data/route.ts @@ -1,10 +1,13 @@ import { SettingsManager } from "@/core/adminSettings/settingsManager"; import { DataBufferEngine } from "@/core/data/buffer"; +import { SDKManager } from "@/core/sdk/sdkManager"; import { getInternalIngestionHeaders } from "@/core/Security/security"; import { CSVStorageManager } from "@/core/Storage/csvStorage"; +import { requireAdminApiSession } from "@/lib/adminAuth"; import { parse } from "csv-parse"; import { NextRequest, NextResponse } from "next/server"; -import { createReadStream, readdirSync, statSync } from "node:fs"; +import { createReadStream } from "node:fs"; +import { lstat, readdir } from "node:fs/promises"; import path from "node:path"; interface DataEvent { @@ -15,10 +18,40 @@ interface DataEvent { status: "stored" | "buffered"; } +function parseTimestampMs(raw: string): number { + const trimmed = String(raw || "").trim(); + if (/^\d+(\.\d+)?$/.test(trimmed)) { + const num = Number(trimmed); + return num < 1e11 ? num * 1000 : num; + } + const parsed = Date.parse(trimmed); + return Number.isNaN(parsed) ? Number.NEGATIVE_INFINITY : parsed; +} + +function compareEventsDesc(a: DataEvent, b: DataEvent): number { + const aMs = parseTimestampMs(a.timestamp); + const bMs = parseTimestampMs(b.timestamp); + if (aMs !== bMs) { + return bMs - aMs; + } + return b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0; +} + export async function GET(request: NextRequest) { + const auth = await requireAdminApiSession(request); + if (!auth.valid) { + return auth.response; + } + try { const { searchParams } = new URL(request.url); - const clientId = searchParams.get("clientId") || "all"; + const rawClientId = searchParams.get("clientId") || "all"; + const clientId = + rawClientId === "all" + ? "all" + : SDKManager.normalizeClientId(rawClientId) || "all"; + const sanitizedClientId = + clientId === "all" ? "all" : CSVStorageManager.sanitizeClientId(clientId); const from = searchParams.get("from") || ""; const to = searchParams.get("to") || ""; const search = (searchParams.get("search") || "").toLowerCase(); @@ -34,7 +67,12 @@ export async function GET(request: NextRequest) { const addIfMatching = (event: DataEvent) => { const eventDate = CSVStorageManager.extractSafePartitionDate(event.timestamp, null); - const dedupeKey = `${event.clientId}:${event.eventId}`; + const normEventClient = SDKManager.normalizeClientId(event.clientId) || event.clientId; + const dedupeKey = `${normEventClient}:${event.eventId}`; + const matchesClient = + clientId === "all" || + normEventClient === clientId || + CSVStorageManager.sanitizeClientId(normEventClient) === sanitizedClientId; const matchesSearch = !search || event.payload.toLowerCase().includes(search) || @@ -42,7 +80,7 @@ export async function GET(request: NextRequest) { event.clientId.toLowerCase().includes(search); if ( - (clientId !== "all" && event.clientId !== clientId) || + !matchesClient || ((from || to) && (!eventDate || (from && eventDate < from) || (to && eventDate > to))) || !matchesSearch || eventIds.has(dedupeKey) @@ -57,7 +95,7 @@ export async function GET(request: NextRequest) { let high = selectedRows.length; while (low < high) { const middle = (low + high) >>> 1; - if (selectedRows[middle].timestamp >= event.timestamp) low = middle + 1; + if (compareEventsDesc(selectedRows[middle], event) <= 0) low = middle + 1; else high = middle; } @@ -70,31 +108,61 @@ export async function GET(request: NextRequest) { const storageDir = CSVStorageManager.baseStorageDir; let clientDirs: string[] = []; try { - clientDirs = readdirSync(storageDir); + clientDirs = await readdir(storageDir); } catch { clientDirs = []; } for (const storedClientId of clientDirs) { - if (clientId !== "all" && storedClientId !== clientId) continue; + if ( + clientId !== "all" && + storedClientId !== clientId && + storedClientId !== sanitizedClientId + ) { + continue; + } const clientDir = path.join(storageDir, storedClientId); - if (!statSync(clientDir).isDirectory()) continue; + let dirLstat; + try { + dirLstat = await lstat(clientDir); + } catch { + continue; + } + if (dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue; let filenames: string[]; try { - filenames = readdirSync(clientDir).filter((filename) => - /^\d{4}-\d{2}-\d{2}(?:-part\d+)?\.csv$/.test(filename), - ); + filenames = (await readdir(clientDir)) + .filter((filename) => { + if (!/^\d{4}-\d{2}-\d{2}(?:-part\d+)?\.csv$/.test(filename)) { + return false; + } + const fileDate = filename.slice(0, 10); + if ((from && fileDate < from) || (to && fileDate > to)) { + return false; + } + return true; + }) + .sort((a, b) => b.localeCompare(a)); } catch { continue; } for (const filename of filenames) { const fileDate = filename.slice(0, 10); - if ((from && fileDate < from) || (to && fileDate > to)) continue; - const parser = createReadStream(path.join(clientDir, filename)).pipe( + const filePath = path.join(clientDir, filename); + let fileLstat; + try { + fileLstat = await lstat(filePath); + } catch { + continue; + } + if (fileLstat.isSymbolicLink() || !fileLstat.isFile()) continue; + + const fileStream = createReadStream(filePath); + const parser = fileStream.pipe( parse({ columns: true, skip_empty_lines: true, bom: true }), ); @@ -111,17 +179,17 @@ export async function GET(request: NextRequest) { }; if (event.eventId) addIfMatching(event); } - } catch { - parser.destroy(); + } catch (err) { + console.warn(`[API /data] Failed reading CSV partition ${filePath}:`, err); + } finally { + if (!parser.destroyed) parser.destroy(); + if (!fileStream.destroyed) fileStream.destroy(); } } } - const settings = await SettingsManager.getSettingsAsync(); - const recentWindowLimit = Math.min( - 2000, - Math.max(200, settings.batchFlushThreshold || 500), - ); + await SettingsManager.getSettingsAsync(); + const recentWindowLimit = DataBufferEngine.getRecentLimit(); for (const event of await DataBufferEngine.getRecentEventsAsync( recentWindowLimit, diff --git a/app/api/events/route.ts b/app/api/events/route.ts index bf607f1..4944db9 100644 --- a/app/api/events/route.ts +++ b/app/api/events/route.ts @@ -1,33 +1,240 @@ import { SettingsManager } from '@/core/adminSettings/settingsManager'; import { verifyRequestAdminSession } from '@/core/auth/adminUsers'; import { AuthService } from '@/core/auth/jwt'; -import { DataBufferEngine } from '@/core/data/buffer'; +import { DataBufferEngine, REDIS_BUFFER_KEY } from '@/core/data/buffer'; import { SDKManager } from '@/core/sdk/sdkManager'; import { SecurityEngine, getInternalIngestionHeaders, } from '@/core/Security/security'; -import { CSVStorageManager, type DataEvent } from '@/core/Storage/csvStorage'; +import { + CSVStorageManager, + type DataEvent, + type StorageFileMeta, +} from '@/core/Storage/csvStorage'; import { parse } from 'csv-parse'; import { NextRequest, NextResponse } from 'next/server'; +import crypto from 'node:crypto'; import { createReadStream } from 'node:fs'; +import { requireAdminApiSession } from '@/lib/adminAuth'; +import type { BufferedBucket, BufferedSummary } from '@/lib/vconUiHelpers'; const SESSION_COOKIE = 'datastack_admin_session'; +const SINGLE_EVENT_ENVELOPE_KEYS = new Set(['eventId', 'timestamp', 'data', 'clientId']); +const MAX_SINGLE_CLIENT_SCAN_FILES = 10; +const MAX_ALL_CLIENTS_PER_CLIENT_FILES = 2; +const MAX_ALL_CLIENTS_TOTAL_SCAN_FILES = 50; + +function generateFallbackEventId(): string { + return `evt_${Date.now()}_${crypto.randomBytes(6).toString('hex')}`; +} + +function extractSingleHttpEventData(rawBody: Record): unknown { + if (rawBody.data !== undefined) { + const keys = Object.keys(rawBody); + if (keys.every((k) => SINGLE_EVENT_ENVELOPE_KEYS.has(k))) { + return rawBody.data; + } + return rawBody; + } + + const stripped: Record = {}; + let hasNonEnvelopeKey = false; + for (const [k, v] of Object.entries(rawBody)) { + if (k === 'eventId' || k === 'timestamp' || k === 'clientId') { + continue; + } + hasNonEnvelopeKey = true; + stripped[k] = v; + } + return hasNonEnvelopeKey ? stripped : {}; +} + +function parseTimestampMs(raw: string): number { + const trimmed = String(raw || '').trim(); + if (/^\d+(\.\d+)?$/.test(trimmed)) { + const num = Number(trimmed); + return num < 1e11 ? num * 1000 : num; + } + const parsed = Date.parse(trimmed); + return Number.isNaN(parsed) ? Number.NEGATIVE_INFINITY : parsed; +} + +function resolveEventUtcDate(rawTimestamp: string): string { + const ms = parseTimestampMs(rawTimestamp); + if (Number.isFinite(ms)) { + const parsed = new Date(ms); + if (!Number.isNaN(parsed.getTime())) { + return parsed.toISOString().slice(0, 10); + } + } + return new Date().toISOString().slice(0, 10); +} + +function selectRecentPartitionFiles( + allFiles: StorageFileMeta[], + normClientId?: string, + sanitizedDir?: string, +): StorageFileMeta[] { + const matching = allFiles + .filter( + (f) => + !normClientId || + f.clientId === normClientId || + f.clientId === sanitizedDir, + ) + .sort((a, b) => { + if (a.date !== b.date) return b.date.localeCompare(a.date); + return new Date(b.modifiedAt).getTime() - new Date(a.modifiedAt).getTime(); + }); + + if (normClientId) { + return matching.slice(0, MAX_SINGLE_CLIENT_SCAN_FILES); + } + + // Multi-client fair scan: take up to 2 newest partitions per active client (capped at 50 files total) + // so clients beyond the 5th most-recently-modified file are never silently hidden. + const perClientCounts = new Map(); + const selected: StorageFileMeta[] = []; + + for (const file of matching) { + const currentCount = perClientCounts.get(file.clientId) ?? 0; + if (currentCount >= MAX_ALL_CLIENTS_PER_CLIENT_FILES) continue; + perClientCounts.set(file.clientId, currentCount + 1); + selected.push(file); + if (selected.length >= MAX_ALL_CLIENTS_TOTAL_SCAN_FILES) break; + } + + return selected; +} + +async function getPendingBufferedSummary( + ingestionBaseUrl: string, + filterClientId?: string, +): Promise<{ events: DataEvent[]; summary: BufferedSummary }> { + const bufferStats = await DataBufferEngine.getBufferStatsAsync(); + const recentLimit = DataBufferEngine.getRecentLimit(); + const pending: DataEvent[] = []; + let totalQueued = bufferStats.localBufferSize + bufferStats.redisBufferSize; + + // 1. Read exact unflushed Redis queue (when Redis is enabled & connected) + const redis = DataBufferEngine.getRedisClient(); + if (redis && bufferStats.redisBufferSize > 0) { + try { + const cap = Math.min(recentLimit, bufferStats.redisBufferSize); + const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, cap - 1); + for (const raw of rawItems) { + try { + const parsed = JSON.parse(raw) as DataEvent; + if (!parsed || !parsed.eventId) continue; + pending.push(parsed); + } catch { + // Skip malformed queued item + } + } + } catch { + // Fall back to local buffer telemetry + } + } + + // 2. Read Port 3000 exact unflushed RAM queue snapshot + if (bufferStats.localBufferSize > 0) { + const localPending = DataBufferEngine.getLocalBufferSnapshot(recentLimit); + pending.push(...localPending); + } + + // 3. When Redis is not enabled or temporarily disconnected, Port 4000 `/internal/recent` serves its unflushed RAM `bufferQueue` + if (!bufferStats.redisEnabled || !bufferStats.redisConnected) { + try { + const qs = new URLSearchParams({ limit: String(recentLimit) }); + const res = await fetch(`${ingestionBaseUrl}/internal/recent?${qs.toString()}`, { + headers: getInternalIngestionHeaders(), + cache: 'no-store', + signal: AbortSignal.timeout(1000), + }); + if (res.ok) { + const data = await res.json(); + if (Array.isArray(data.events) && data.events.length > 0) { + pending.push(...(data.events as DataEvent[])); + totalQueued += data.events.length; + } + } + } catch { + // Port 4000 may not be running in isolated single-process dev mode + } + } + + const seen = new Set(); + const dedupedAll: DataEvent[] = []; + const filteredEvents: DataEvent[] = []; + const bucketMap = new Map(); + const sanitizedFilter = filterClientId + ? CSVStorageManager.sanitizeClientId(filterClientId) + : undefined; + + for (const evt of pending) { + if (!evt || !evt.eventId) continue; + const normEvtClient = SDKManager.normalizeClientId(evt.clientId) || evt.clientId; + const dedupeKey = `${normEvtClient}:${evt.eventId}`; + if (seen.has(dedupeKey)) continue; + seen.add(dedupeKey); + dedupedAll.push(evt); + + const matchesFilter = + !filterClientId || + normEvtClient === filterClientId || + evt.clientId === filterClientId || + evt.clientId === sanitizedFilter; + + if (matchesFilter) { + filteredEvents.push(evt); + const utcDate = resolveEventUtcDate(evt.timestamp); + const bucketKey = `${normEvtClient}|${utcDate}`; + const existing = bucketMap.get(bucketKey); + if (existing) { + existing.count += 1; + } else { + bucketMap.set(bucketKey, { + clientId: normEvtClient, + date: utcDate, + count: 1, + }); + } + } + } + + const effectiveTotalQueued = Math.max(totalQueued, dedupedAll.length); + const unattributedCount = Math.max(0, effectiveTotalQueued - dedupedAll.length); + + return { + events: filteredEvents, + summary: { + total: filteredEvents.length, + totalQueued: filterClientId + ? filteredEvents.length + unattributedCount + : effectiveTotalQueued, + unattributedCount, + truncated: unattributedCount > 0, + buckets: Array.from(bucketMap.values()), + }, + }; +} async function getRecentStoredEvents( limit: number, clientId?: string, ): Promise { + const normClientId = clientId && clientId !== 'all' ? SDKManager.normalizeClientId(clientId) : undefined; + const sanitizedDir = normClientId ? CSVStorageManager.sanitizeClientId(normClientId) : undefined; const allFiles = await CSVStorageManager.listAllFilesAsync(); - const files = allFiles - .filter((f) => !clientId || clientId === 'all' || f.clientId === clientId) - .slice(0, 5); + const files = selectRecentPartitionFiles(allFiles, normClientId, sanitizedDir); const collected: DataEvent[] = []; for (const file of files) { const fileTail: DataEvent[] = []; - const parser = createReadStream(file.filePath).pipe( + const fileStream = createReadStream(file.filePath); + const parser = fileStream.pipe( parse({ columns: true, skip_empty_lines: true, bom: true }), ); @@ -46,8 +253,11 @@ async function getRecentStoredEvents( payload: record.payload || '', }); } - } catch { - parser.destroy(); + } catch (err) { + console.warn(`[API /events] Failed reading CSV partition ${file.filePath}:`, err); + } finally { + if (!parser.destroyed) parser.destroy(); + if (!fileStream.destroyed) fileStream.destroy(); } if (fileTail.length > 0) { @@ -59,18 +269,47 @@ async function getRecentStoredEvents( } export async function GET(req: NextRequest) { + const auth = await requireAdminApiSession(req); + if (!auth.valid) { + return auth.response; + } + try { const { searchParams } = new URL(req.url); - const clientId = searchParams.get('clientId') || undefined; + const rawClientId = searchParams.get('clientId'); + const clientId = + rawClientId && rawClientId !== 'all' + ? SDKManager.normalizeClientId(rawClientId) || undefined + : undefined; const limit = Math.min(2000, Math.max(1, Number(searchParams.get('limit')) || 50)); + const bufferedOnly = searchParams.get('bufferedOnly') === 'true'; + const includeBufferedSummary = + bufferedOnly || searchParams.get('includeBufferedSummary') === 'true'; await SettingsManager.getSettingsAsync(); - const bufferedEvents = await DataBufferEngine.getRecentEventsAsync(limit, clientId); - let socketBufferedEvents: DataEvent[] = []; const ingestionBaseUrl = ( process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000' ).replace(/\/$/, ''); + if (bufferedOnly) { + const pendingResult = await getPendingBufferedSummary(ingestionBaseUrl, clientId); + const sortedPending = [...pendingResult.events].sort((a, b) => { + const aMs = parseTimestampMs(a.timestamp); + const bMs = parseTimestampMs(b.timestamp); + if (aMs !== bMs) return bMs - aMs; + return b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0; + }); + const events = sortedPending.slice(0, limit); + return NextResponse.json({ + events, + count: events.length, + bufferedSummary: pendingResult.summary, + }); + } + + const bufferedEvents = await DataBufferEngine.getRecentEventsAsync(limit, clientId); + let socketBufferedEvents: DataEvent[] = []; + try { const qs = new URLSearchParams({ limit: String(limit) }); if (clientId) qs.set('clientId', clientId); @@ -89,23 +328,38 @@ export async function GET(req: NextRequest) { // Port 4000 may not be running in isolated single-process dev mode } - const storedEvents = await getRecentStoredEvents(limit, clientId); + const [storedEvents, pendingResult] = await Promise.all([ + getRecentStoredEvents(limit, clientId), + includeBufferedSummary + ? getPendingBufferedSummary(ingestionBaseUrl, clientId) + : Promise.resolve(null), + ]); const seenIds = new Set(); const merged: DataEvent[] = []; for (const evt of [...bufferedEvents, ...socketBufferedEvents, ...storedEvents]) { if (!evt.eventId) continue; - const dedupeKey = `${evt.clientId}:${evt.eventId}`; + const normEvtClient = SDKManager.normalizeClientId(evt.clientId) || evt.clientId; + const dedupeKey = `${normEvtClient}:${evt.eventId}`; if (seenIds.has(dedupeKey)) continue; seenIds.add(dedupeKey); merged.push(evt); } - merged.sort((a, b) => (b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0)); + merged.sort((a, b) => { + const aMs = parseTimestampMs(a.timestamp); + const bMs = parseTimestampMs(b.timestamp); + if (aMs !== bMs) return bMs - aMs; + return b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0; + }); const events = merged.slice(0, limit); - return NextResponse.json({ events, count: events.length }); + return NextResponse.json({ + events, + count: events.length, + ...(pendingResult ? { bufferedSummary: pendingResult.summary } : {}), + }); } catch (err: unknown) { const message = err instanceof Error ? err.message : 'Events retrieval failed'; return NextResponse.json({ error: message }, { status: 500 }); @@ -165,12 +419,18 @@ export async function POST(req: NextRequest) { ? (body as Record) : {}; - const clientId = String( - tokenClientId || bodyObj.clientId || 'py_telemetry_agent_01', - ) - .trim() - .toLowerCase() - .replace(/[^a-z0-9_-]/g, '_'); + const clientId = SDKManager.normalizeClientId( + String(tokenClientId || bodyObj.clientId || ''), + ); + if (!clientId) { + return NextResponse.json( + { + error: 'clientId is required and must belong to a registered client.', + code: 'VALIDATION_ERROR', + }, + { status: 400 }, + ); + } if (await SDKManager.isClientDeletedAsync(clientId)) { return NextResponse.json( @@ -183,7 +443,7 @@ export async function POST(req: NextRequest) { } const registeredClient = await SDKManager.getClientAsync(clientId); - if (rawBearerToken && !registeredClient) { + if (!registeredClient) { return NextResponse.json( { error: `CLIENT_REVOKED: Client '${clientId}' registration not found or has been deleted.`, @@ -231,10 +491,11 @@ export async function POST(req: NextRequest) { ? (bodyObj.batch as unknown[]) : Array.isArray(bodyObj.records) ? (bodyObj.records as unknown[]) - : null; + : bodyObj.data === undefined && Array.isArray(bodyObj.events) + ? (bodyObj.events as unknown[]) + : null; if (batchItems !== null) { - const envelopeKeys = new Set(['eventId', 'timestamp', 'data', 'clientId']); let processed = 0; let rejected = 0; let duplicates = 0; @@ -255,19 +516,16 @@ export async function POST(req: NextRequest) { const itemEventId = typeof safeItem.eventId === 'string' && safeItem.eventId.trim() ? safeItem.eventId.trim() - : `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`; + : generateFallbackEventId(); const itemTimestamp = typeof safeItem.timestamp === 'string' && safeItem.timestamp.trim() ? safeItem.timestamp.trim() : new Date().toISOString(); - const isEnvelope = - safeItem.data !== undefined && - Object.keys(safeItem).every((k) => envelopeKeys.has(k)); - const itemData = isEnvelope ? safeItem.data : safeItem; + const itemData = extractSingleHttpEventData(safeItem); - const val = SecurityEngine.validateEvent( + const val = await SecurityEngine.validateEventAsync( clientId, itemEventId, itemData, @@ -309,7 +567,7 @@ export async function POST(req: NextRequest) { } if (processed > 0) { - SDKManager.incrementEventCount(clientId, processed); + await SDKManager.incrementEventCountAsync(clientId, processed); } // If zero items were processed or deduplicated and all failed due to payload or rate limit, return corresponding status @@ -323,6 +581,7 @@ export async function POST(req: NextRequest) { return NextResponse.json( { success: false, + accepted: processed, processed, rejected, duplicates, @@ -338,6 +597,7 @@ export async function POST(req: NextRequest) { return NextResponse.json({ success: true, + accepted: processed, processed, rejected, duplicates, @@ -350,24 +610,19 @@ export async function POST(req: NextRequest) { }); } - // Single Event Ingestion + // Single Event Ingestion (exact envelope parity with Port 4000 WS sync_data) const eventId = typeof bodyObj.eventId === 'string' && bodyObj.eventId.trim() ? bodyObj.eventId.trim() - : `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`; + : generateFallbackEventId(); const timestamp = typeof bodyObj.timestamp === 'string' && bodyObj.timestamp.trim() ? bodyObj.timestamp.trim() : new Date().toISOString(); - const data = - bodyObj.data !== undefined - ? bodyObj.data - : Object.keys(bodyObj).length > 0 - ? bodyObj - : { sample: 'Realtime event via DataStack API', rand: Math.random() }; + const data = extractSingleHttpEventData(bodyObj); // Validate security (idempotency, rate limit, payload size) - const val = SecurityEngine.validateEvent( + const val = await SecurityEngine.validateEventAsync( clientId, eventId, data, @@ -393,7 +648,7 @@ export async function POST(req: NextRequest) { }; await DataBufferEngine.pushAsync(event); - SDKManager.incrementEventCount(clientId, 1); + await SDKManager.incrementEventCountAsync(clientId, 1); return NextResponse.json({ success: true, eventId, event }); } catch (err: unknown) { diff --git a/app/api/export/route.ts b/app/api/export/route.ts index 27eff34..9710b40 100644 --- a/app/api/export/route.ts +++ b/app/api/export/route.ts @@ -1,10 +1,12 @@ import { SettingsManager } from "@/core/adminSettings/settingsManager"; import { DataBufferEngine } from "@/core/data/buffer"; +import { SDKManager } from "@/core/sdk/sdkManager"; import { getInternalIngestionHeaders } from "@/core/Security/security"; import { CSVStorageManager, type StorageFileMeta, } from "@/core/Storage/csvStorage"; +import { requireAdminApiSession } from "@/lib/adminAuth"; import { parse } from "csv-parse"; import { NextRequest, NextResponse } from "next/server"; import { createReadStream } from "node:fs"; @@ -44,7 +46,8 @@ function toByteStream(chunks: AsyncIterable) { async function* readFileRecords( file: StorageFileMeta, ): AsyncGenerator { - const parser = createReadStream(file.filePath).pipe( + const fileStream = createReadStream(file.filePath); + const parser = fileStream.pipe( parse({ columns: true, skip_empty_lines: true, bom: true }), ); @@ -52,8 +55,9 @@ async function* readFileRecords( for await (const record of parser as AsyncIterable) { yield record; } - } catch { - parser.destroy(); + } finally { + if (!parser.destroyed) parser.destroy(); + if (!fileStream.destroyed) fileStream.destroy(); } } @@ -85,7 +89,7 @@ function makeExportStream( yield `${isFirst ? "" : ","}${JSON.stringify({ ...record, payload: parsedPayload })}`; } else { const line = fields - .map((field) => `"${(record[field] ?? "").replaceAll('"', '""')}"`) + .map((field) => `"${String(record[field] ?? "").replaceAll('"', '""')}"`) .join(","); yield `${line}\n`; } @@ -101,8 +105,13 @@ function makeExportStream( function streamFile(file: StorageFileMeta) { async function* chunks() { - for await (const chunk of createReadStream(file.filePath)) { - yield new Uint8Array(chunk); + const fileStream = createReadStream(file.filePath); + try { + for await (const chunk of fileStream) { + yield new Uint8Array(chunk as Buffer); + } + } finally { + if (!fileStream.destroyed) fileStream.destroy(); } } @@ -110,38 +119,122 @@ function streamFile(file: StorageFileMeta) { } export async function GET(request: NextRequest) { + const auth = await requireAdminApiSession(request); + if (!auth.valid) { + return auth.response; + } + try { const { searchParams } = new URL(request.url); + const from = (searchParams.get("from") || "").trim(); + const to = (searchParams.get("to") || "").trim(); + if (from && to && from > to) { + return NextResponse.json( + { + error: 'Start date ("from") must be on or before end date ("to").', + code: "INVALID_DATE_RANGE", + }, + { status: 400 }, + ); + } const shouldFlush = searchParams.get("flush") !== "false"; await SettingsManager.getSettingsAsync(); if (shouldFlush) { - await DataBufferEngine.flush(); + try { + await DataBufferEngine.flush({ waitMs: 2500, throwOnLockTimeout: true }); + } catch (err: unknown) { + const message = + err instanceof Error ? err.message : "Pre-export flush failed"; + if (message.includes("FLUSH_LOCK_TIMEOUT")) { + return NextResponse.json( + { + error: message, + code: "FLUSH_LOCK_TIMEOUT", + }, + { status: 503 }, + ); + } + throw err; + } + + const explicitIngestionUrl = Boolean(process.env.INGESTION_INTERNAL_URL?.trim()); const ingestionBaseUrl = ( process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000" ).replace(/\/$/, ""); try { - await fetch(`${ingestionBaseUrl}/internal/flush`, { + const flushRes = await fetch(`${ingestionBaseUrl}/internal/flush`, { method: "POST", - headers: getInternalIngestionHeaders(), - signal: AbortSignal.timeout(1500), + headers: { + "Content-Type": "application/json", + ...getInternalIngestionHeaders(), + }, + body: JSON.stringify({ waitMs: 2500, throwOnLockTimeout: true }), + cache: "no-store", + signal: AbortSignal.timeout(4000), }); - } catch { + if (!flushRes.ok) { + const errPayload = await flushRes.json().catch(() => ({})); + const remoteErr = + typeof errPayload?.error === "string" + ? errPayload.error + : `Ingestion flush returned HTTP ${flushRes.status}`; + if ( + flushRes.status === 503 && + errPayload?.code === "FLUSH_LOCK_TIMEOUT" + ) { + return NextResponse.json( + { + error: remoteErr, + code: "FLUSH_LOCK_TIMEOUT", + ingestionStatus: flushRes.status, + }, + { status: 503 }, + ); + } + return NextResponse.json( + { + error: remoteErr, + code: "INGESTION_FLUSH_FAILED", + ingestionStatus: flushRes.status, + }, + { status: 502 }, + ); + } + } catch (err: unknown) { + if (explicitIngestionUrl) { + const errMsg = + err instanceof Error ? err.message : "Ingestion server unreachable"; + return NextResponse.json( + { + error: `Ingestion server flush failed before export: ${errMsg}`, + code: "INGESTION_UNREACHABLE", + }, + { status: 502 }, + ); + } // Port 4000 may not be running in isolated single-process dev mode } } - const clientId = searchParams.get("clientId") || "all"; + const rawClientId = searchParams.get("clientId") || "all"; + const clientId = + rawClientId === "all" + ? "all" + : SDKManager.normalizeClientId(rawClientId) || "all"; + const sanitizedClientId = + clientId === "all" ? "all" : CSVStorageManager.sanitizeClientId(clientId); const filename = searchParams.get("filename"); - const from = searchParams.get("from") || ""; - const to = searchParams.get("to") || ""; - const format: ExportFormat = - searchParams.get("format") === "json" ? "json" : "csv"; + const rawFormat = searchParams.get("format"); + const format: ExportFormat = rawFormat === "json" ? "json" : "csv"; const includeHeader = searchParams.get("includeHeader") !== "false"; let files = await CSVStorageManager.listAllFilesAsync(); - if (clientId !== "all") - files = files.filter((file) => file.clientId === clientId); + if (clientId !== "all") { + files = files.filter( + (file) => file.clientId === clientId || file.clientId === sanitizedClientId, + ); + } if (filename) { const safeFilename = path.basename(filename); @@ -166,11 +259,23 @@ export async function GET(request: NextRequest) { ); } const file = matches[0]; - return new NextResponse(streamFile(file), { + const downloadName = + format === "json" + ? `${file.filename.replace(/\.csv$/i, "")}.json` + : file.filename; + const bodyStream = + format === "json" || !includeHeader + ? makeExportStream([file], format, includeHeader) + : streamFile(file); + return new NextResponse(bodyStream, { status: 200, headers: { - "Content-Type": "text/csv; charset=utf-8", - "Content-Disposition": `attachment; filename="${file.filename}"`, + "Content-Type": + format === "json" + ? "application/json; charset=utf-8" + : "text/csv; charset=utf-8", + "Content-Disposition": `attachment; filename="${downloadName.replace(/"/g, "")}"`, + "Cache-Control": "no-store", }, }); } @@ -179,7 +284,7 @@ export async function GET(request: NextRequest) { (file) => (!from || file.date >= from) && (!to || file.date <= to), ); const datePart = new Date().toISOString().slice(0, 10); - const safeClientId = clientId.replace(/[^a-zA-Z0-9_-]/g, "_"); + const safeClientId = sanitizedClientId === "all" ? "all" : sanitizedClientId; const baseName = clientId === "all" ? "datastack_export" : `${safeClientId}_export`; diff --git a/app/api/health/route.ts b/app/api/health/route.ts index f022a0b..a41a7bc 100644 --- a/app/api/health/route.ts +++ b/app/api/health/route.ts @@ -23,10 +23,10 @@ export async function GET(req: NextRequest) { ); } - const clientId = String(verified.payload.clientId || '') - .trim() - .toLowerCase(); - if (SDKManager.isClientDeleted(clientId)) { + const clientId = SDKManager.normalizeClientId( + String(verified.payload.clientId || ''), + ); + if (await SDKManager.isClientDeletedAsync(clientId)) { return NextResponse.json( { status: 'error', @@ -37,27 +37,35 @@ export async function GET(req: NextRequest) { ); } const registeredClient = await SDKManager.getClientAsync(clientId); - if (registeredClient) { - if (registeredClient.status === 'revoked') { - return NextResponse.json( - { - status: 'error', - code: 'CLIENT_REVOKED', - error: `CLIENT_REVOKED: Client '${clientId}' access has been revoked by administrator.`, - }, - { status: 403 }, - ); - } - if (registeredClient.token && registeredClient.token !== rawToken) { - return NextResponse.json( - { - status: 'error', - code: 'TOKEN_ROTATED', - error: 'TOKEN_ROTATED: Client credentials have been rotated. Use the latest token.', - }, - { status: 401 }, - ); - } + if (!registeredClient) { + return NextResponse.json( + { + status: 'error', + code: 'CLIENT_REVOKED', + error: `CLIENT_REVOKED: Client '${clientId}' registration not found or has been deleted.`, + }, + { status: 403 }, + ); + } + if (registeredClient.status === 'revoked') { + return NextResponse.json( + { + status: 'error', + code: 'CLIENT_REVOKED', + error: `CLIENT_REVOKED: Client '${clientId}' access has been revoked by administrator.`, + }, + { status: 403 }, + ); + } + if (registeredClient.token && registeredClient.token !== rawToken) { + return NextResponse.json( + { + status: 'error', + code: 'TOKEN_ROTATED', + error: 'TOKEN_ROTATED: Client credentials have been rotated. Use the latest token.', + }, + { status: 401 }, + ); } } diff --git a/app/api/metrics/route.ts b/app/api/metrics/route.ts index b6d4fd8..6d8d048 100644 --- a/app/api/metrics/route.ts +++ b/app/api/metrics/route.ts @@ -1,7 +1,13 @@ -import { NextResponse } from 'next/server'; +import { NextRequest, NextResponse } from 'next/server'; import { ApiEngine } from '@/core/apiEngine/apiService'; +import { requireAdminApiSession } from '@/lib/adminAuth'; + +export async function GET(req: NextRequest) { + const auth = await requireAdminApiSession(req); + if (!auth.valid) { + return auth.response; + } -export async function GET() { try { const metrics = await ApiEngine.getDashboardMetrics(); return NextResponse.json(metrics); diff --git a/app/api/settings/route.ts b/app/api/settings/route.ts index 5d8c3b3..27a2c32 100644 --- a/app/api/settings/route.ts +++ b/app/api/settings/route.ts @@ -188,9 +188,24 @@ export async function POST(req: NextRequest) { await SettingsManager.updateSettingsAsync(validated); const settings = await SettingsManager.getPublicSettingsAsync(); + const bridgeSync = SettingsManager.getLastBridgeSyncStatus(); return NextResponse.json({ settings, success: true, + ...(bridgeSync + ? { + bridgeSync: { + attempted: bridgeSync.attempted ?? true, + ok: bridgeSync.ok ?? bridgeSync.synced, + synced: bridgeSync.synced, + ...(bridgeSync.status !== undefined + ? { status: bridgeSync.status } + : {}), + ...(bridgeSync.error ? { error: bridgeSync.error } : {}), + checkedAt: bridgeSync.checkedAt, + }, + } + : {}), }); } catch (err: unknown) { const message = diff --git a/app/api/storage/route.ts b/app/api/storage/route.ts index 9739161..b058f88 100644 --- a/app/api/storage/route.ts +++ b/app/api/storage/route.ts @@ -1,24 +1,44 @@ import { NextRequest, NextResponse } from 'next/server'; import { SettingsManager } from '@/core/adminSettings/settingsManager'; +import { SDKManager } from '@/core/sdk/sdkManager'; import { getInternalIngestionHeaders } from '@/core/Security/security'; import { CSVStorageManager } from '@/core/Storage/csvStorage'; import { DataBufferEngine } from '@/core/data/buffer'; +import { requireAdminApiSession } from '@/lib/adminAuth'; export async function GET(req: NextRequest) { + const auth = await requireAdminApiSession(req); + if (!auth.valid) { + return auth.response; + } + try { const { searchParams } = new URL(req.url); - const clientId = searchParams.get('clientId') || undefined; + const rawClientId = searchParams.get('clientId'); + const clientId = + rawClientId && rawClientId !== 'all' + ? SDKManager.normalizeClientId(rawClientId) + : undefined; + const sanitizedClientId = clientId + ? CSVStorageManager.sanitizeClientId(clientId) + : undefined; - const [stats, allFiles] = await Promise.all([ - CSVStorageManager.getStorageStatsAsync(), - CSVStorageManager.listAllFilesAsync(), - ]); + const allFiles = await CSVStorageManager.listAllFilesAsync(); let files = allFiles; - if (clientId && clientId !== 'all') { - files = files.filter((f) => f.clientId === clientId); + if (clientId) { + files = allFiles.filter((f) => { + const normFolder = SDKManager.normalizeClientId(f.clientId); + return ( + normFolder === clientId || + f.clientId === clientId || + f.clientId === sanitizedClientId + ); + }); } + const stats = CSVStorageManager.formatStorageStats(files); + return NextResponse.json({ stats, files }); } catch (err: unknown) { const message = err instanceof Error ? err.message : 'Storage stats failed'; @@ -26,12 +46,20 @@ export async function GET(req: NextRequest) { } } -export async function POST(_req: NextRequest) { +export async function POST(req: NextRequest) { + const auth = await requireAdminApiSession(req); + if (!auth.valid) { + return auth.response; + } + try { await SettingsManager.getSettingsAsync(); // 1. Flush local / shared Next.js DataBufferEngine - const localRes = await DataBufferEngine.flush(); + const localRes = await DataBufferEngine.flush({ + waitMs: 2500, + throwOnLockTimeout: true, + }); // 2. Also flush Port 4000 Socket.IO ingestion buffer let remoteCount = 0; @@ -46,9 +74,13 @@ export async function POST(_req: NextRequest) { try { const ingestRes = await fetch(`${ingestionBaseUrl}/internal/flush`, { method: 'POST', - headers: getInternalIngestionHeaders(), + headers: { + 'Content-Type': 'application/json', + ...getInternalIngestionHeaders(), + }, + body: JSON.stringify({ waitMs: 2500, throwOnLockTimeout: true }), cache: 'no-store', - signal: AbortSignal.timeout(2500), + signal: AbortSignal.timeout(4000), }); if (!ingestRes.ok) { const errPayload = await ingestRes.json().catch(() => ({})); @@ -56,6 +88,23 @@ export async function POST(_req: NextRequest) { typeof errPayload?.error === 'string' ? errPayload.error : `Ingestion flush returned HTTP ${ingestRes.status}`; + if ( + ingestRes.status === 503 && + errPayload?.code === 'FLUSH_LOCK_TIMEOUT' + ) { + return NextResponse.json( + { + success: false, + error: remoteErr, + code: 'FLUSH_LOCK_TIMEOUT', + flushedCount: localRes.count, + filesUpdated: localRes.files, + ingestionFlushed: false, + ingestionStatus: ingestRes.status, + }, + { status: 503 }, + ); + } return NextResponse.json( { success: false, @@ -108,6 +157,16 @@ export async function POST(_req: NextRequest) { }); } catch (err: unknown) { const message = err instanceof Error ? err.message : 'Manual flush failed'; + if (message.includes('FLUSH_LOCK_TIMEOUT')) { + return NextResponse.json( + { + success: false, + error: message, + code: 'FLUSH_LOCK_TIMEOUT', + }, + { status: 503 }, + ); + } return NextResponse.json({ error: message }, { status: 500 }); } } diff --git a/app/layout.tsx b/app/layout.tsx index b0c4e73..141f070 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -38,9 +38,14 @@ export const metadata: Metadata = { }, }; +const THEME_BOOTSTRAP_SCRIPT = `(function(){try{var t=localStorage.getItem('datastack-theme');var r=document.documentElement;if(t==='light'){r.classList.remove('dark');r.classList.add('light');}else{r.classList.remove('light');r.classList.add('dark');}}catch(e){}})();`; + export default function RootLayout({ children }: { children: React.ReactNode }) { return ( - + + +