From 8132087085b8ed85e90ebc5ac7c355a1000484b5 Mon Sep 17 00:00:00 2001 From: Team Dark Devil Date: Sat, 21 Nov 2026 00:06:55 +0600 Subject: [PATCH] feat(auth): enhance security and audit logging - Integrate webhook alerts for admin login lockouts. - Implement detailed audit logging for login, logout, and setup actions. - Improve client revocation event tracking. - Standardize error handling in profile updates. - Enforce 'operator' role for event API access. --- MEMORY.md | 67 ++- app/api/auth/login/route.ts | 12 + app/api/auth/logout/route.ts | 17 +- app/api/auth/profile/route.ts | 8 +- app/api/auth/setup/route.ts | 12 + app/api/clients/[id]/route.ts | 2 +- app/api/events/route.ts | 17 +- app/api/health/route.ts | 23 + app/api/prometheus/route.ts | 6 + app/api/settings/webhook-test/route.ts | 22 +- app/api/storage/route.ts | 32 +- app/api/storage/s3-test/route.ts | 2 +- app/vcon/clients/page.tsx | 161 +++--- app/vcon/layout.tsx | 2 +- app/vcon/simulator/page.tsx | 54 +- app/vcon/storage/page.tsx | 84 ++- components/layout/VconShell.tsx | 23 +- components/sections/AuditLogViewer.tsx | 9 +- components/sections/ClientsTable.tsx | 120 +++-- components/sections/DashboardOverview.tsx | 26 +- components/sections/ExportWorkspace.tsx | 12 +- components/sections/SettingsWorkspace.tsx | 91 +++- core/Security/ipAllowlist.ts | 57 +- core/Storage/csvStorage.ts | 10 +- core/adminSettings/settingsManager.ts | 253 ++++++++- core/apiEngine/apiService.ts | 10 +- core/auth/adminUsers.ts | 132 ++++- core/auth/auditLog.ts | 4 + core/auth/jwt.ts | 27 +- core/sdk/sdkManager.ts | 78 +-- docker/docker-compose.yml | 1 + lib/vconUiHelpers.ts | 161 ++++++ sdk-python/datastack_sdk/__init__.py | 2 + sdk/node/datastack/index.ts | 385 ++++++++++---- sdk/node/datastack/package.json | 13 +- sdk/node/datastack/tsconfig.json | 17 + sdk/node/datastack/types.ts | 14 +- sdk/python/datastack/datastack/__init__.py | 2 + sdk/python/datastack/datastack/client.py | 79 ++- sdk/python/datastack/datastack/exceptions.py | 13 + server/index.js | 275 +++++++++- server/socket/handlers.js | 68 ++- server/workers/csvWriter.js | 4 +- tests/phase05_sdk_test.py | 58 ++ ...r3-phase01-security-vault-webhooks.test.ts | 491 +++++++++++++++++ .../r3-phase02-storage-scheduler-gzn.test.ts | 347 ++++++++++++ tests/r3-phase03-sdk-parity.test.ts | 497 ++++++++++++++++++ tests/r3-phase04-ui-metrics-rbac.test.ts | 271 ++++++++++ tests/r3_phase03_python_sdk_test.py | 218 ++++++++ tests/v2-phase01-storage-quotas.test.ts | 90 ++++ tests/v2-phase02-alerts-ip-prometheus.test.ts | 223 ++++++++ tests/v2-phase03-rbac-audit.test.ts | 239 +++++++++ 52 files changed, 4434 insertions(+), 407 deletions(-) create mode 100644 sdk/node/datastack/tsconfig.json create mode 100644 tests/r3-phase01-security-vault-webhooks.test.ts create mode 100644 tests/r3-phase02-storage-scheduler-gzn.test.ts create mode 100644 tests/r3-phase03-sdk-parity.test.ts create mode 100644 tests/r3-phase04-ui-metrics-rbac.test.ts create mode 100644 tests/r3_phase03_python_sdk_test.py diff --git a/MEMORY.md b/MEMORY.md index 25c11de..031aff3 100644 --- a/MEMORY.md +++ b/MEMORY.md @@ -1,7 +1,7 @@ # MEMORY.md — DataStack Cross-IDE Persistent Project Memory > **Last Synchronized**: 2026-11-20 -> **Status**: **V2-Phase-04 & `V2-Deploy-Fix-01` Complete & Verified (`BASELINE-2026-11-19-V2`) — `4 / 4` V2 Phases + Final Coolify Redeploy Hardening Done (`V2-Phase-01` Storage `.csv.gz` Compression, S3/R2 Off-Site Archive & Per-Client/Disk Quotas; `V2-Phase-02` Outbound Webhook Alerting, Per-Client IP/CIDR Allowlist & Token Expiry, Prometheus `/api/prometheus` Metrics Endpoint & Streaming `.ndjson` Export; `V2-Phase-03` Multi-Admin RBAC `owner`/`admin`/`operator`/`viewer` via `admin_team_members` & Immutable Admin Audit Log `admin_audit_logs`; `V2-Phase-04` Official Node.js / TypeScript Client SDK `sdk/node/datastack` & `/vcon/sdk` Multi-Language Guide; `V2-Deploy-Fix-01` `DEPLOY-V2-BUG-01..03`) — Verified (`102/102` TS tests, `10/10` Python tests, ESLint-enabled build & lint `0` errors / `0` warnings).** +> **Status**: **R3 Production Remediation (`BASELINE-2026-11-20-R3`): `3 / 4` Phases Complete (`12 / 16` Bugs Resolved — `R3-Phase-01`, `R3-Phase-02`, & `R3-Phase-03` Verified Complete; `R3-Phase-04` ready for `Proceed to R3-Phase-04`).** > **Scope Rule**: Maintain this file accurately across VS Code, Google Antigravity, Cursor, and any other IDE after every modification, feature update, or bug fix (see `.ai-rules` and `AGENTS.md`). --- @@ -146,15 +146,15 @@ datastack/ │ ├── configuration-reference.md # Complete environment variable & setting bounds │ ├── dependencies.md # Runtime & dev dependencies audit with rationales │ └── glossary.md # Technical domain terminology & concept glossary -├── notes/ # Production Hardening Governance & Scope-Lock Suite (`BASELINE-2026-11-19-R2`): -│ ├── README.md # Master governance index & 6-step scope-locked execution protocol -│ ├── FORENSIC_AUDIT_REPORT.md # 100% raw-code verified audit (18 bugs: 0C/6H/8M/4L), Demo vs Prod & UI-vs-Reality matrices -│ ├── BASELINE_FREEZE.md # Frozen baseline snapshot (`BASELINE-2026-11-19-R2`) & 1:1 bug alignment -│ ├── PRODUCTION_READINESS_SCOPE_LOCK.md # (Doc 001) Exact bug fix contracts & per-phase Scope Locks (`Phase-01`..`Phase-04`) -│ ├── PRODUCTION_ROADMAP.md # (Doc 002) Dependency-ordered A-Z 4-phase roadmap & burn-down tracker -│ ├── PHASE_EXECUTION_LOG.md # (Doc 003) Live phase completion log (`4/4` complete) & zero-regression guardrails -│ ├── ERROR_HANDLING_MATRIX.md # (Doc 004) Active error-handling catalog & 9 planned hardening additions -│ └── ACTUAL_IMPLEMENTATION_STATUS.md # Ground-truth working vs completed-per-phase vs remaining ledger +├── notes/ # Production Remediation Governance & Scope-Lock Suite (`BASELINE-2026-11-20-R3`): +│ ├── README.md # Master governance index, 6-step execution workflow & phase approval commands +│ ├── FORENSIC_AUDIT_REPORT.md # 100% raw-code verified audit (16 bugs: BUG-01..BUG-16) & Demo vs Prod matrix +│ ├── BASELINE_FREEZE.md # Frozen baseline snapshot (`BASELINE-2026-11-20-R3`) & 1:1 bug alignment +│ ├── PRODUCTION_READINESS_SCOPE_LOCK.md # (Doc 001) Exact bug fix contracts & per-phase Scope Locks (`R3-Phase-01`..`04`) +│ ├── PRODUCTION_ROADMAP.md # (Doc 002) Dependency-ordered A–Z 4-phase roadmap & burn-down tracker +│ ├── PHASE_EXECUTION_LOG.md # (Doc 003) Live phase completion log (`0/4` complete in R3 cycle) & zero-drift guardrails +│ ├── ERROR_HANDLING_MATRIX.md # (Doc 004) Active error-handling catalog & 7 planned fixes (`ERR-R3-01`..`07`) +│ └── ACTUAL_IMPLEMENTATION_STATUS.md # Ground-truth working vs completed-per-phase vs remaining ledger (16 items remaining) └── tests/ # Automated verification suites: ├── phase01-auth.test.ts # Auth fail-closed, tokenVersion, brute-force lockout, Bearer revocation, CSRF ├── phase02-sync.test.ts # Internal bridge auth, settings sync, JWT rotation kick, rate/security metrics @@ -330,6 +330,53 @@ All planned hardening & deployment phases (**Phase-01 through Phase-07**) are ** - **Official Node.js / TypeScript Client SDK (`sdk/node/datastack/` — `@datastack/sdk` v2.2.0)**: Created `sdk/node/datastack/package.json`, `types.ts`, `spool.ts`, `index.ts`, and `README.md`. Supports `datastack.config.json` / `DATASTACK_*` env / constructor options with strict `normalizeClientId` vs JWT `clientId`/`sub` verification (`DataStackConfigError` — `V2-E12`), Socket.IO primary streaming (`connect()`, `send()`/`sync()`, `sendBatch()`/`batchSync()`), automatic HTTP `POST /api/events` fallback, recursive `413 PAYLOAD_TOO_LARGE` batch splitting, partial-batch `429 RATE_LIMIT_EXCEEDED` retry, typed terminal error mapping (`401 CLIENT_TOKEN_EXPIRED` & `403 CLIENT_IP_FORBIDDEN` -> `DataStackAuthError`, `507 STORAGE_QUOTA_EXCEEDED` & `DISK_SPACE_LOW` -> `DataStackQuotaExceededError`), durable `.datastack_spool/spool.jsonl` local disk spooling (`DiskSpooler`) & `flushSpool()`, direct `.csv`/`.json`/`.jsonl`/`.ndjson` file ingestion (`sendFile()` with deterministic SHA-256 `eventId` via `makeDeterministicFileEventId`), and graceful `await client.shutdown(timeoutMs)` drain. - **Multi-Language SDK Guide (`app/vcon/sdk/page.tsx`)**: Added interactive language tabs (`Python SDK (v2.2.0)`, `Node.js / TypeScript SDK (v2.2.0)`, `HTTP / cURL`) with live client JWT token population and copy-ready `config.datastack.py` / `datastack.config.json` snippets. - **Verified**: `101/101` TS tests (`4/4` in `tests/v2-phase04-node-sdk.test.ts`), `10/10` Python SDK tests, `npm run build` clean, and `npm run lint` (`0` errors / `0` warnings). + - **Post-V2 Full-Codebase Forensic Audit & Baseline Freeze (`BASELINE-2026-11-20-V2-R2`)**: + - Conducted a 100% raw-source-code forensic audit across all Port 3000 APIs/engines, Port 4000 Socket.IO workers, `/vcon/*` UI pages, and Python/Node SDKs (`notes/FORENSIC_AUDIT_REPORT.md`), identifying **10 verified bugs** (`BUG-V2-AUDIT-01` .. `BUG-V2-AUDIT-10`: `5` High, `4` Medium, `1` Low). + - Froze the current verified codebase (`102/102` TS tests, `10/10` Python tests, `0` build/lint errors, `0` source code changes) as **`BASELINE-2026-11-20-V2-R2`** in `notes/BASELINE_FREEZE.md`. + - Prepared the complete 7-document governance & Scope-Lock suite in `notes/`: + - `notes/FORENSIC_AUDIT_REPORT.md` (`10` verified bugs with verbatim code blocks & line numbers) + - `notes/BASELINE_FREEZE.md` (1:1 baseline-to-audit mapping) + - `notes/PRODUCTION_READINESS_SCOPE_LOCK.md` (Doc `001` — exact fix contracts & strict per-phase Scope Locks) + - `notes/PRODUCTION_ROADMAP.md` (Doc `002` — A–Z 3-phase execution roadmap: `V2-Audit-Phase-01` Auth/Email/Audit, `V2-Audit-Phase-02` Port 4000 Webhooks/IPv6 CIDR/Health/Python SDK, `V2-Audit-Phase-03` UI Disk Usage/Client Expiry/CSV.GZ Badges) + - `notes/PHASE_EXECUTION_LOG.md` (Doc `003` — `3 / 3` phases completed, `0 / 3` remaining, `10 / 10` bugs resolved) + - `notes/ERROR_HANDLING_MATRIX.md` (Doc `004` — active error catalog + `ERR-V2-AUDIT-01`..`05` all `5 / 5` completed) + - `notes/ACTUAL_IMPLEMENTATION_STATUS.md` (ground-truth working vs per-phase delta — `0` remaining items) + - `notes/README.md` (governance index & 6-step execution protocol) + - **`V2-Audit-Phase-01` Complete & Verified (`2026-11-20`)** (`BUG-V2-AUDIT-02`, `BUG-V2-AUDIT-03`, `BUG-V2-AUDIT-04`, `ERR-V2-AUDIT-01`): + - **`BUG-V2-AUDIT-04`**: Updated `revokeAdminSessions(target)` in `core/auth/adminUsers.ts` to accept explicit `{ memberId, email }` and passed `{ memberId: check.session.memberId, email: check.session.email }` from `POST /api/auth/logout` (`app/api/auth/logout/route.ts`), eliminating the multi-admin logout global race condition (`globals.__dataStackLastVerifiedMemberId`). + - **`BUG-V2-AUDIT-03` (`ERR-V2-AUDIT-01`)**: Enforced cross-table (`admin_users` ↔ `admin_team_members`) and in-memory peer duplicate email checks in `updateAdminAccount()` (`core/auth/adminUsers.ts`) returning `code: 'ADMIN_EMAIL_CONFLICT'`, mapped to HTTP `409 Conflict` in `PUT /api/auth/profile` (`app/api/auth/profile/route.ts`). + - **`BUG-V2-AUDIT-02`**: Wired all 4 missing audit actions (`ADMIN_LOGOUT` in `POST /api/auth/logout`, `ADMIN_SETUP` in `POST /api/auth/setup`, `STORAGE_FLUSH_TRIGGERED` in `POST /api/storage`, and `CLIENT_REVOKED` when `updates.status === 'revoked'` in `PATCH /api/clients/[id]`) and synchronized `ACTION_FILTER_OPTIONS` in `components/sections/AuditLogViewer.tsx` 1:1 with `AuditActionType` in `core/auth/auditLog.ts` (all 18/18 governance actions active). + - **Verified**: `103/103` TS tests (`5/5` in `tests/v2-phase03-rbac-audit.test.ts`), `10/10` Python SDK tests, `npm run build` clean, and `npm run lint` (`0` errors / `0` warnings). + - **`V2-Audit-Phase-02` Complete & Verified (`2026-11-20`)** (`BUG-V2-AUDIT-05`, `BUG-V2-AUDIT-06`, `BUG-V2-AUDIT-08`, `BUG-V2-AUDIT-09`, `ERR-V2-AUDIT-02`..`05`): + - **`BUG-V2-AUDIT-05` (`ERR-V2-AUDIT-02`)**: Wired outbound HMAC-SHA256 webhook alert dispatching (`triggerWebhookAlert` + `dispatchRejectionWebhook`) on Port 4000 Socket.IO (`server/index.js`, `server/socket/handlers.js`) for `CLIENT_TOKEN_EXPIRED`, `CLIENT_IP_FORBIDDEN`, `DISK_SPACE_LOW`, and `STORAGE_QUOTA_EXCEEDED` across handshake, `sync_data`, `batch_sync`, and `ping_verify`, with AES-256-GCM `decryptVaultSecret` in `syncSettingsFromDatabase()` and live `webhookSecret` propagation in `core/adminSettings/settingsManager.ts`. + - **`BUG-V2-AUDIT-06` (`ERR-V2-AUDIT-03`)**: Upgraded `isClientIpAllowed()` and `stripIpv6MappedPrefix()` in both `core/Security/ipAllowlist.ts` and `server/index.js` to use Node's built-in `net.BlockList` (`addSubnet`) for all IPv4 (`/0..32`) and IPv6 (`/0..128`) CIDR subnets and normalize `[ipv6]:port`, `ipv4:port`, `%zone`, and `::ffff:`. + - **`BUG-V2-AUDIT-08` (`ERR-V2-AUDIT-04`)**: Enforced `isClientTokenExpired(registeredClient.expiresAt)` (`401 CLIENT_TOKEN_EXPIRED`) and `isClientIpAllowed(extractClientIp(req), registeredClient.allowedIps)` (`403 CLIENT_IP_FORBIDDEN`) in `GET /api/health` (`app/api/health/route.ts`). + - **`BUG-V2-AUDIT-09` (`ERR-V2-AUDIT-05`)**: Added `"CLIENT_TOKEN_EXPIRED"` and `"CLIENT_IP_FORBIDDEN"` to `_is_non_retryable_auth_error()` and `_extract_auth_error_code()` in `sdk/python/datastack/datastack/client.py`, immediately raising `DataStackAuthError(msg, code=...)` matching the Node.js SDK 1:1. + - **Verified**: `104/104` TS tests (`5/5` in `tests/v2-phase02-alerts-ip-prometheus.test.ts`), `11/11` Python SDK tests (`tests/phase05_sdk_test.py`), `npm run build` clean, and `npm run lint` (`0` errors / `0` warnings). + - **`V2-Audit-Phase-03` Complete & Verified (`2026-11-20`)** (`BUG-V2-AUDIT-01`, `BUG-V2-AUDIT-07`, `BUG-V2-AUDIT-10`): + - **`BUG-V2-AUDIT-01`**: Aligned `DiskUsageSnapshot` (`core/Storage/csvStorage.ts`), `GET /api/storage` (`app/api/storage/route.ts`), and `SettingsWorkspace.tsx` (`components/sections/SettingsWorkspace.tsx`) with `available: boolean` (`totalBytes > 0`) and `freeBytes: number` (`availableBytes`) so `/vcon/settings` -> Storage -> "VPS disk usage" renders real disk utilization percentage, progress bar, and free/total GiB instead of `"Statfs unavailable"`. + - **`BUG-V2-AUDIT-07`**: Added `isClientExpired(expiresAt, nowMs)` in `app/vcon/clients/page.tsx` and excluded expired tokens from `onlineCount`, `pendingCount`, and `"online"`/`"pending"` filter tabs (classifying non-revoked expired clients under `"offline"`), matching `ClientsTable.tsx` `"expired"` badges 1:1. + - **`BUG-V2-AUDIT-10`**: Updated `components/sections/ExportWorkspace.tsx` and `app/vcon/storage/page.tsx` to render a `CSV.GZ` format badge whenever `file.compressed || file.filename.toLowerCase().endsWith('.csv.gz')` is true (`CSV` otherwise). + - **Verified**: `105/105` TS tests (`4/4` in `tests/v2-phase01-storage-quotas.test.ts`), `11/11` Python SDK tests (`tests/phase05_sdk_test.py`), `npm run build` clean, and `npm run lint` (`0` errors / `0` warnings). All `3 / 3` phases (`10 / 10` bugs) resolved. + - **R3 Full-Stack Production Remediation (`BASELINE-2026-11-20-R3` — `16` Verified Bugs across `4` Scope-Locked Phases)**: + - **`R3-Phase-01` Complete & Verified (`2026-11-20`)** (`BUG-01`, `BUG-03`, `BUG-04`, `BUG-06`, `BUG-10`, `BUG-13`, `BUG-15`; `ERR-R3-01`..`ERR-R3-04`): + - **`BUG-01`**: Aligned Port 4000 `decryptVaultSecret()` (`server/index.js`) 1:1 with `core/Security/secretVault.ts` (`v1:` prefix, `DATASTACK_ADMIN_SESSION_SECRET`, `base64url` decoding, fail-closed `""` + warning) and injected `DATASTACK_ADMIN_SESSION_SECRET` (`SERVICE_REALBASE64_64_ADMINSESSION`) into `docker/docker-compose.yml` `services.ingestion.environment`. + - **`BUG-03`**: Updated `AuthService.generateClientToken(payload, expiresIn?)` (`core/auth/jwt.ts`) to omit JWT `exp` when `expiresIn` is `null`/`undefined`/`""`, and passed `null` across all 6 minting sites in `core/sdk/sdkManager.ts` so DB `client_sdks.expires_at` (`Never expires`, `>30d`, live extension) is authoritative. + - **`BUG-04`**: Enforced `requireAdminApiSession(req, { minRole: 'operator' })` on admin-session calls to `POST /api/events` (`app/api/events/route.ts`), returning `403 INSUFFICIENT_ROLE_PERMISSION` for read-only `viewer` accounts. + - **`BUG-06`**: Wrapped `decryptSecret(settings.webhookSecretEnc)` in `SettingsManager.notifyIngestionSettingsSync()` (`core/adminSettings/settingsManager.ts`) in `try/catch` (`""` fallback + warning) and deferred `decryptSecret` in `POST /api/settings/webhook-test` (`app/api/settings/webhook-test/route.ts`) until neither draft `webhookSecret` nor `clearWebhookSecret` is provided. + - **`BUG-10`**: Wired non-blocking `ADMIN_LOGIN_LOCKOUT` webhook alert dispatch inside the `429 TOO_MANY_ATTEMPTS` lockout branch of `POST /api/auth/login` (`app/api/auth/login/route.ts`). + - **`BUG-13`**: Unified Port 4000 `formatOutboundWebhookBody()` (`server/index.js`) with Port 3000 `core/alerts/webhookNotifier.ts` (`instance`, `event`, `severity`, `title`, `message`, `clientId`, `details`, `timestamp`, `text`, `content`). + - **`BUG-15`**: Tightened `POST /api/storage/s3-test` (`app/api/storage/s3-test/route.ts`) from `operator` to `owner`-only RBAC. + - **Verified**: `6/6` in `tests/r3-phase01-security-vault-webhooks.test.ts`, `15/15` in `tests/v2-phase01..03`, `40/40` in `tests/phase01/02/07`, `npm run build` clean, `npm run lint` (`0` errors). `0` UI/SDK/CSV files modified. + - **`R3-Phase-02` Complete & Verified (`2026-11-20`)** (`BUG-05`, `BUG-09`): + - **`BUG-05`**: Added singleton unref'd periodic maintenance scheduler (`ensureMaintenanceScheduler`, `runScheduledMaintenanceTick`, `stopMaintenanceScheduler`, `stopMaintenanceSchedulerForTests`, `isMaintenanceSchedulerActive`) in `core/adminSettings/settingsManager.ts` with overlap locks (`__dataStackMaintenanceRunning`, `__dataStackArchivePruneInFlight`, `__dataStackCompressInFlight`), cooldown guards, and unhydrated-DB protection so S3 off-site archival + retention pruning and partition compression run unattended on schedule when `enableS3Backup === true`. + - **`BUG-09`**: Updated `extractDateFromFilename()` in both `core/Storage/csvStorage.ts` and `server/workers/csvWriter.js` to strip `.csv(.gz)` and trailing `-(?:part|gz)\d+` suffixes so `-gzN.csv.gz` and `-partN-gzM.csv.gz` collision files work identically across Port 3000/4000 pruning, S3 archiving, `/api/data`, and `/api/export`. + - **Verified**: `3/3` in `tests/r3-phase02-storage-scheduler-gzn.test.ts`, `40/40` in `tests/phase03/v2-phase01/phase07/r3-phase01`, `npm run build` clean, `npm run lint` (`0` errors). `0` UI/SDK/Auth/Docker files modified. + - **`R3-Phase-03` Complete & Verified (`2026-11-20`)** (`BUG-02`, `BUG-07`, `BUG-08`; `ERR-R3-05`, `ERR-R3-06`): + - **`BUG-02`**: Fixed Node.js SDK (`sdk/node/datastack/index.ts`) WS `batch_sync` to emit a raw `Array` directly (1:1 with Port 4000 `server/socket/handlers.js:595`), parse `ack.processed ?? ack.processedCount ?? ack.accepted` and `ack.duplicates ?? 0` across WS & HTTP batch responses, and propagate `duplicates` through `sendBatch()`, `batchSync()`, and `sendFile()`. + - **`BUG-07` (`ERR-R3-05`)**: Implemented real `heartbeatIntervalMs` monitor in Node.js SDK (`startHeartbeatMonitor` with `.unref?.()`, zero-CPU `IDLE_WS` pause, automatic `ping_verify` recovery & spool drain, clean timer teardown, `status().heartbeatActive`); fixed single HTTP/WS `DUPLICATE_DROPPED` reporting (`accepted: 0, duplicates: 1`); wired `TOKEN_ROTATED` (`401`) and `CLIENT_REVOKED` (`403`) fast-fail `DataStackAuthError` across WS `connect_error`, `ping_verify`, `sync_data`, `batch_sync`, and HTTP fallback; compiled CommonJS `sdk/node/datastack/dist/{index,types,spool}.{js,d.ts}` via `tsconfig.json` and pointed `package.json` `main`/`types`/`exports` to `./dist/index.js` & `./dist/index.d.ts`. + - **`BUG-08` (`ERR-R3-06`)**: Added non-retryable `DataStackQuotaExceededError(DataStackError, retryable=False)` in `sdk/python/datastack/datastack/exceptions.py`, re-exported it in both `datastack` and `datastack_sdk`, and wired immediate fail-fast handling for HTTP `507` and WS `STORAGE_QUOTA_EXCEEDED` / `DISK_SPACE_LOW` in `sdk/python/datastack/datastack/client.py` with `0` retries, `0` HTTP fallback after WS quota rejection, and `0` offline spooling. + - **Verified**: `13/13` Node SDK/packaging tests (`5/5` `tests/r3-phase03-sdk-parity.test.ts` against live Socket.IO handlers + `4/4` `v2-phase04-node-sdk.test.ts` + `4/4` `phase05-packaging.test.ts`), `14/14` Python SDK tests (`3/3` `tests/r3_phase03_python_sdk_test.py` + `11/11` `phase05_sdk_test.py`), `py_compile`, `tsc`, `npm run build` clean, `npm run lint` (`0` errors, `0` warnings). `0` Port 3000/4000/UI/Docker files modified. --- diff --git a/app/api/auth/login/route.ts b/app/api/auth/login/route.ts index 7037c40..420cfa0 100644 --- a/app/api/auth/login/route.ts +++ b/app/api/auth/login/route.ts @@ -10,6 +10,7 @@ import { clearAdminLoginAttempts, consumeAdminLoginAttempt, } from "@/core/auth/adminUsers"; +import { SettingsManager } from "@/core/adminSettings/settingsManager"; import { recordAdminAuditLogAsync } from "@/core/auth/auditLog"; import { extractClientIp } from "@/lib/adminAuth"; import { NextRequest, NextResponse } from "next/server"; @@ -70,6 +71,17 @@ export async function POST(request: NextRequest) { ); } if (!throttleDecision.allowed) { + SettingsManager.triggerWebhookAlert({ + event: "ADMIN_LOGIN_LOCKOUT", + severity: "warning", + title: "Admin Login Lockout Triggered", + message: `Admin sign-in throttled for ${email} from IP ${clientIp} (${throttleDecision.retryAfterSeconds}s lockout).`, + details: { + email, + clientIp, + retryAfterSeconds: throttleDecision.retryAfterSeconds, + }, + }); return NextResponse.json( { error: "Too many sign-in attempts. Try again later." }, { diff --git a/app/api/auth/logout/route.ts b/app/api/auth/logout/route.ts index 627a604..64447b7 100644 --- a/app/api/auth/logout/route.ts +++ b/app/api/auth/logout/route.ts @@ -1,8 +1,10 @@ import { getSessionCookieOptions } from "@/core/auth/adminSession"; +import { recordAdminAuditLogAsync } from "@/core/auth/auditLog"; import { revokeAdminSessions, verifyRequestAdminSession, } from "@/core/auth/adminUsers"; +import { extractClientIp } from "@/lib/adminAuth"; import { NextRequest, NextResponse } from "next/server"; const SESSION_COOKIE = "datastack_admin_session"; @@ -13,7 +15,20 @@ export async function POST(request: NextRequest) { try { const check = await verifyRequestAdminSession(token); if (check.valid) { - await revokeAdminSessions(); + await revokeAdminSessions({ + memberId: check.session.memberId, + email: check.session.email, + }); + await recordAdminAuditLogAsync({ + actorEmail: check.session.email, + actorRole: check.session.role || "owner", + action: "ADMIN_LOGOUT", + target: check.session.email, + ipAddress: extractClientIp(request), + metadata: { + memberId: check.session.memberId || "owner-1", + }, + }); } } catch { // Still clear client cookie even if server revocation encounters transient DB error diff --git a/app/api/auth/profile/route.ts b/app/api/auth/profile/route.ts index 682bd58..f4624d9 100644 --- a/app/api/auth/profile/route.ts +++ b/app/api/auth/profile/route.ts @@ -125,9 +125,13 @@ export async function PUT(request: NextRequest) { }); if (!result.ok) { + const isConflict = result.code === "ADMIN_EMAIL_CONFLICT"; return NextResponse.json( - { error: result.error, code: "INVALID_PASSWORD" }, - { status: 401 }, + { + error: result.error, + code: result.code || "INVALID_PASSWORD", + }, + { status: isConflict ? 409 : 401 }, ); } diff --git a/app/api/auth/setup/route.ts b/app/api/auth/setup/route.ts index 3a7152c..d7d9781 100644 --- a/app/api/auth/setup/route.ts +++ b/app/api/auth/setup/route.ts @@ -3,6 +3,7 @@ import { getAdminSessionSecret, getSessionCookieOptions, } from "@/core/auth/adminSession"; +import { recordAdminAuditLogAsync } from "@/core/auth/auditLog"; import { AdminDatabaseRequiredError, clearAdminLoginAttempts, @@ -128,6 +129,17 @@ export async function POST(request: NextRequest) { clientIp, ).catch(() => undefined); + await recordAdminAuditLogAsync({ + actorEmail: email, + actorRole: "owner", + action: "ADMIN_SETUP", + target: email, + ipAddress: clientIp, + metadata: { + memberId: "owner-1", + }, + }); + const token = await createAdminSessionToken( email, sessionSecret, diff --git a/app/api/clients/[id]/route.ts b/app/api/clients/[id]/route.ts index cffc1ff..c667043 100644 --- a/app/api/clients/[id]/route.ts +++ b/app/api/clients/[id]/route.ts @@ -271,7 +271,7 @@ export async function PATCH( await recordAdminAuditLogAsync({ actorEmail: auth.auth.session.email, actorRole: auth.auth.session.role || "owner", - action: "CLIENT_UPDATED", + action: updates.status === "revoked" ? "CLIENT_REVOKED" : "CLIENT_UPDATED", target: updated.clientId, ipAddress: extractClientIp(req), metadata: updates, diff --git a/app/api/events/route.ts b/app/api/events/route.ts index bc15d2c..33e3d8f 100644 --- a/app/api/events/route.ts +++ b/app/api/events/route.ts @@ -1,5 +1,4 @@ import { SettingsManager } from '@/core/adminSettings/settingsManager'; -import { verifyRequestAdminSession } from '@/core/auth/adminUsers'; import { AuthService } from '@/core/auth/jwt'; import { DataBufferEngine, REDIS_BUFFER_KEY } from '@/core/data/buffer'; import { SDKManager } from '@/core/sdk/sdkManager'; @@ -22,7 +21,6 @@ import crypto from 'node:crypto'; import { extractClientIp, requireAdminApiSession } from '@/lib/adminAuth'; import type { BufferedBucket, BufferedSummary } from '@/lib/vconUiHelpers'; -const SESSION_COOKIE = 'datastack_admin_session'; const SINGLE_EVENT_ENVELOPE_KEYS = new Set(['eventId', 'timestamp', 'data', 'clientId']); const MAX_SINGLE_CLIENT_SCAN_FILES = 10; const MAX_ALL_CLIENTS_PER_CLIENT_FILES = 2; @@ -376,9 +374,8 @@ export async function POST(req: NextRequest) { try { const settings = await SettingsManager.getSettingsAsync(); - // Verify either Client Bearer JWT or Admin Session Cookie + // Verify either Client Bearer JWT or Admin Session Cookie (minRole: operator) const authHeader = req.headers.get('authorization'); - const sessionCookie = req.cookies.get(SESSION_COOKIE)?.value; let tokenClientId: string | null = null; let tokenRateLimit: number | undefined; @@ -396,17 +393,9 @@ export async function POST(req: NextRequest) { tokenClientId = verified.payload.clientId; tokenRateLimit = verified.payload.rateLimit; } else { - const auth = await verifyRequestAdminSession(sessionCookie); + const auth = await requireAdminApiSession(req, { minRole: 'operator' }); if (!auth.valid) { - return NextResponse.json( - { - error: - auth.reason === 'UNAUTHORIZED' - ? 'Authentication required. Provide a valid Client Bearer JWT or Admin Session.' - : 'Admin authentication is temporarily unavailable.', - }, - { status: auth.reason === 'UNAUTHORIZED' ? 401 : 503 }, - ); + return auth.response; } } diff --git a/app/api/health/route.ts b/app/api/health/route.ts index a41a7bc..59aa429 100644 --- a/app/api/health/route.ts +++ b/app/api/health/route.ts @@ -2,6 +2,8 @@ import { SettingsManager } from '@/core/adminSettings/settingsManager'; import { AuthService } from '@/core/auth/jwt'; import { DataBufferEngine } from '@/core/data/buffer'; import { SDKManager } from '@/core/sdk/sdkManager'; +import { isClientIpAllowed, isClientTokenExpired } from '@/core/Security/ipAllowlist'; +import { extractClientIp } from '@/lib/adminAuth'; import { NextRequest, NextResponse } from 'next/server'; export async function GET(req: NextRequest) { @@ -67,6 +69,27 @@ export async function GET(req: NextRequest) { { status: 401 }, ); } + if (isClientTokenExpired(registeredClient.expiresAt)) { + return NextResponse.json( + { + status: 'error', + code: 'CLIENT_TOKEN_EXPIRED', + error: `CLIENT_TOKEN_EXPIRED: Client '${clientId}' token has expired (${registeredClient.expiresAt}).`, + }, + { status: 401 }, + ); + } + const clientIp = extractClientIp(req); + if (!isClientIpAllowed(clientIp, registeredClient.allowedIps)) { + return NextResponse.json( + { + status: 'error', + code: 'CLIENT_IP_FORBIDDEN', + error: `CLIENT_IP_FORBIDDEN: IP '${clientIp}' is not in the allowlist for client '${clientId}'.`, + }, + { status: 403 }, + ); + } } const bufferStats = await DataBufferEngine.getBufferStatsAsync(); diff --git a/app/api/prometheus/route.ts b/app/api/prometheus/route.ts index c6b7bf7..66be697 100644 --- a/app/api/prometheus/route.ts +++ b/app/api/prometheus/route.ts @@ -2,6 +2,7 @@ import { SettingsManager } from "@/core/adminSettings/settingsManager"; import { verifyRequestAdminSession } from "@/core/auth/adminUsers"; import { DataBufferEngine } from "@/core/data/buffer"; import { SDKManager } from "@/core/sdk/sdkManager"; +import { isClientTokenExpired } from "@/core/Security/ipAllowlist"; import { getInternalIngestionHeaders } from "@/core/Security/security"; import { CSVStorageManager } from "@/core/Storage/csvStorage"; import { ADMIN_SESSION_COOKIE } from "@/lib/adminAuth"; @@ -116,13 +117,17 @@ export async function GET(req: NextRequest) { ? bufferStats.redisBufferSize : remoteBufferSize); + const nowMs = Date.now(); let activeClientsCount = 0; + let expiredClientsCount = 0; let revokedClientsCount = 0; let totalEventsCounter = 0; for (const c of clients) { totalEventsCounter += Math.max(0, Number(c.totalEvents) || 0); if (c.status === "revoked") { revokedClientsCount += 1; + } else if (isClientTokenExpired(c.expiresAt, nowMs)) { + expiredClientsCount += 1; } else { activeClientsCount += 1; } @@ -165,6 +170,7 @@ export async function GET(req: NextRequest) { "# HELP datastack_clients_total Number of registered SDK clients partitioned by status.", "# TYPE datastack_clients_total gauge", `datastack_clients_total{status="active"} ${activeClientsCount}`, + `datastack_clients_total{status="expired"} ${expiredClientsCount}`, `datastack_clients_total{status="revoked"} ${revokedClientsCount}`, "# HELP datastack_client_events_total Cumulative ingested events per registered client.", "# TYPE datastack_client_events_total counter", diff --git a/app/api/settings/webhook-test/route.ts b/app/api/settings/webhook-test/route.ts index 3e56bb9..d2eb245 100644 --- a/app/api/settings/webhook-test/route.ts +++ b/app/api/settings/webhook-test/route.ts @@ -1,7 +1,10 @@ import { SettingsManager } from "@/core/adminSettings/settingsManager"; import { dispatchWebhookAlertAsync } from "@/core/alerts/webhookNotifier"; import { recordAdminAuditLogAsync } from "@/core/auth/auditLog"; -import { SecretVaultError } from "@/core/Security/secretVault"; +import { + decryptSecret, + SecretVaultError, +} from "@/core/Security/secretVault"; import { extractClientIp, requireAdminApiSession } from "@/lib/adminAuth"; import { NextRequest, NextResponse } from "next/server"; @@ -28,17 +31,20 @@ export async function POST(req: NextRequest) { } try { - const savedConfig = await SettingsManager.getDecryptedWebhookConfigAsync(); + const saved = await SettingsManager.getSettingsAsync(); const webhookUrl = typeof body.webhookUrl === "string" ? body.webhookUrl.trim() - : savedConfig.webhookUrl.trim(); + : saved.webhookUrl.trim(); const clearWebhookSecret = body.clearWebhookSecret === true; - const webhookSecret = clearWebhookSecret - ? "" - : typeof body.webhookSecret === "string" && body.webhookSecret.trim() + const draftSecret = + typeof body.webhookSecret === "string" && body.webhookSecret.trim() ? body.webhookSecret.trim() - : savedConfig.webhookSecret; + : ""; + let webhookSecret = draftSecret; + if (!webhookSecret && !clearWebhookSecret && saved.webhookSecretEnc) { + webhookSecret = decryptSecret(saved.webhookSecretEnc); + } if (!webhookUrl) { return NextResponse.json( @@ -81,7 +87,7 @@ export async function POST(req: NextRequest) { enabled: true, webhookUrl, webhookSecret, - cooldownSec: savedConfig.cooldownSec, + cooldownSec: saved.webhookCooldownSec, }, { bypassCooldown: true }, ); diff --git a/app/api/storage/route.ts b/app/api/storage/route.ts index 2c974e1..89a6b38 100644 --- a/app/api/storage/route.ts +++ b/app/api/storage/route.ts @@ -1,10 +1,11 @@ import { NextRequest, NextResponse } from 'next/server'; import { SettingsManager } from '@/core/adminSettings/settingsManager'; +import { recordAdminAuditLogAsync } from '@/core/auth/auditLog'; import { SDKManager } from '@/core/sdk/sdkManager'; import { getInternalIngestionHeaders } from '@/core/Security/security'; import { CSVStorageManager } from '@/core/Storage/csvStorage'; import { DataBufferEngine } from '@/core/data/buffer'; -import { requireAdminApiSession } from '@/lib/adminAuth'; +import { extractClientIp, requireAdminApiSession } from '@/lib/adminAuth'; export async function GET(req: NextRequest) { const auth = await requireAdminApiSession(req); @@ -42,7 +43,18 @@ export async function GET(req: NextRequest) { const stats = CSVStorageManager.formatStorageStats(files); - return NextResponse.json({ stats, files, diskUsage }); + return NextResponse.json({ + stats, + files, + diskUsage: diskUsage ?? { + available: false, + usedPct: 0, + availableBytes: 0, + freeBytes: 0, + totalBytes: 0, + checkedAt: Date.now(), + }, + }); } catch (err: unknown) { const message = err instanceof Error ? err.message : 'Storage stats failed'; return NextResponse.json({ error: message }, { status: 500 }); @@ -150,10 +162,24 @@ export async function POST(req: NextRequest) { const combinedFiles = Array.from( new Set([...localRes.files, ...remoteFiles]), ); + const totalFlushed = localRes.count + remoteCount; + + await recordAdminAuditLogAsync({ + actorEmail: auth.auth.session.email, + actorRole: auth.auth.session.role || 'owner', + action: 'STORAGE_FLUSH_TRIGGERED', + target: 'storage_buffer', + ipAddress: extractClientIp(req), + metadata: { + flushedCount: totalFlushed, + filesUpdated: combinedFiles.length, + ingestionFlushed, + }, + }); return NextResponse.json({ success: true, - flushedCount: localRes.count + remoteCount, + flushedCount: totalFlushed, filesUpdated: combinedFiles, ingestionFlushed, ...(warning ? { warning } : {}), diff --git a/app/api/storage/s3-test/route.ts b/app/api/storage/s3-test/route.ts index a60f980..454670a 100644 --- a/app/api/storage/s3-test/route.ts +++ b/app/api/storage/s3-test/route.ts @@ -11,7 +11,7 @@ import { requireAdminApiSession } from "@/lib/adminAuth"; import { NextRequest, NextResponse } from "next/server"; export async function POST(req: NextRequest) { - const auth = await requireAdminApiSession(req, { minRole: "operator" }); + const auth = await requireAdminApiSession(req, { minRole: "owner" }); if (!auth.valid) { return auth.response; } diff --git a/app/vcon/clients/page.tsx b/app/vcon/clients/page.tsx index da7b501..6f6d40b 100644 --- a/app/vcon/clients/page.tsx +++ b/app/vcon/clients/page.tsx @@ -1,25 +1,46 @@ 'use client'; import { PageShell } from '@/components/layout/PageShell'; -import type { ClientItem, ClientStatusFilter } from '@/components/sections/clients/types'; -import { ClientsFilters } from '@/components/sections/ClientsFilters'; +import { useVconRole } from '@/components/layout/VconShell'; +import type { ClientItem } from '@/components/sections/clients/types'; import { ClientsTable } from '@/components/sections/ClientsTable'; import { NewClientModal } from '@/components/sections/NewClientModal'; import { Button } from '@/components/ui/Button'; +import { Input } from '@/components/ui/Input'; import { Panel } from '@/components/ui/Panel'; +import { Pill } from '@/components/ui/Pill'; import { StatStrip } from '@/components/ui/StatStrip'; import { useToast } from '@/components/ui/Toast'; import { getCached, setCached } from '@/lib/vconCache'; +import { + classifyUiClientStatus, + getVconRoleCapabilities, + summarizeUiClientCounts, + type UiClientComputedStatus, +} from '@/lib/vconUiHelpers'; import { useCallback, useEffect, useMemo, useState } from 'react'; +type ExtendedClientFilter = 'all' | UiClientComputedStatus; + +const CLIENT_FILTER_TABS: { key: ExtendedClientFilter; label: string }[] = [ + { key: 'all', label: 'All' }, + { key: 'online', label: 'Online' }, + { key: 'offline', label: 'Offline' }, + { key: 'pending', label: 'Pending' }, + { key: 'expired', label: 'Expired' }, + { key: 'revoked', label: 'Revoked' }, +]; + export default function ClientsPage() { const { error } = useToast(); + const currentRole = useVconRole(); + const { canOperate } = getVconRoleCapabilities(currentRole); const [modalOpen, setModalOpen] = useState(false); const [clients, setClients] = useState(() => getCached('vcon:clients', []) ); const [search, setSearch] = useState(''); - const [statusFilter, setStatusFilter] = useState('all'); + const [statusFilter, setStatusFilter] = useState('all'); const [now, setNow] = useState(() => Date.now()); const fetchClients = useCallback(async () => { @@ -46,6 +67,11 @@ export default function ClientsPage() { return () => clearInterval(interval); }, []); + const counts = useMemo( + () => summarizeUiClientCounts(clients, now), + [clients, now], + ); + const visibleClients = useMemo(() => { const query = search.trim().toLowerCase(); return clients.filter((client) => { @@ -53,89 +79,102 @@ export default function ClientsPage() { !query || client.clientId.toLowerCase().includes(query) || client.name.toLowerCase().includes(query); - const revoked = client.status === 'revoked'; - const online = - client.status === 'active' && - Boolean(client.lastActive) && - now - new Date(client.lastActive!).getTime() < 60_000; - const pending = client.status === 'active' && !client.lastActive; - const offline = client.status === 'active' && !online && !pending; + const computedStatus = classifyUiClientStatus(client, now); const matchesStatus = - statusFilter === 'all' || - (statusFilter === 'online' && online) || - (statusFilter === 'offline' && offline) || - (statusFilter === 'pending' && pending) || - (statusFilter === 'revoked' && revoked); + statusFilter === 'all' || computedStatus === statusFilter; return matchesSearch && matchesStatus; }); }, [clients, now, search, statusFilter]); - const onlineCount = clients.filter( - (client) => - client.status === 'active' && - client.lastActive && - now - new Date(client.lastActive).getTime() < 60_000 - ).length; - const pendingCount = clients.filter( - (client) => client.status === 'active' && !client.lastActive - ).length; - const revokedCount = clients.filter( - (client) => client.status === 'revoked' - ).length; - const offlineCount = clients.length - onlineCount - pendingCount - revokedCount; + const filterCounts: Record = { + all: counts.total, + online: counts.online, + offline: counts.offline, + pending: counts.pending, + expired: counts.expired, + revoked: counts.revoked, + }; return ( setModalOpen(true)} - > - New client - + canOperate ? ( + + ) : ( + + Read-only ({currentRole}) + + ) } > - +
+
+ setSearch(event.target.value)} + placeholder="Search client..." + /> +
+
+ {CLIENT_FILTER_TABS.map((filter) => ( + setStatusFilter(filter.key)} + > + {filter.label} + + {filterCounts[filter.key].toLocaleString()} + + + ))} +
+
- setModalOpen(false)} - onCreated={fetchClients} - /> + {canOperate && ( + setModalOpen(false)} + onCreated={fetchClients} + /> + )}
); } diff --git a/app/vcon/layout.tsx b/app/vcon/layout.tsx index cafe967..081f1d2 100644 --- a/app/vcon/layout.tsx +++ b/app/vcon/layout.tsx @@ -46,5 +46,5 @@ export default async function VconLayout({ ); } - return {children}; + return {children}; } diff --git a/app/vcon/simulator/page.tsx b/app/vcon/simulator/page.tsx index 215163a..354cddc 100644 --- a/app/vcon/simulator/page.tsx +++ b/app/vcon/simulator/page.tsx @@ -1,6 +1,7 @@ 'use client'; import { PageShell } from '@/components/layout/PageShell'; +import { useVconRole } from '@/components/layout/VconShell'; import { Button } from '@/components/ui/Button'; import { CodeBlock } from '@/components/ui/CodeBlock'; import { Select, Textarea } from '@/components/ui/Input'; @@ -10,7 +11,9 @@ import { StatStrip } from '@/components/ui/StatStrip'; import { useToast } from '@/components/ui/Toast'; import { getCached, setCached } from '@/lib/vconCache'; import { + getVconRoleCapabilities, resolvePublicEndpoints, + shouldRedirectSimulatorUnauthorized, summarizeHttpBatchResult, } from '@/lib/vconUiHelpers'; import Link from 'next/link'; @@ -32,6 +35,8 @@ function getSocketOrigin(serverPort: number): string { export default function SimulatorPage() { const { success, warning, error } = useToast(); + const currentRole = useVconRole(); + const { canOperate } = getVconRoleCapabilities(currentRole); const [clients, setClients] = useState(() => getCached('vcon:clients', []) ); @@ -181,6 +186,10 @@ export default function SimulatorPage() { }; const handleSendSingle = async () => { + if (!canOperate) { + error('Read-only role', 'Event simulation requires operator or owner access.'); + return; + } if (!selectedClient) { error('No target client', 'Register a client in /vcon/clients first.'); return; @@ -223,20 +232,29 @@ export default function SimulatorPage() { data: parsedData, }), }); - if (res.status === 401) { + const data = await res.json().catch(() => ({})); + if ( + shouldRedirectSimulatorUnauthorized( + res.status, + data && typeof data === 'object' ? (data as Record) : null, + ) + ) { window.location.assign('/vcon/login'); return; } - const data = await res.json(); - setLastResponse({ transport: 'http', status: res.status, ...data }); + setLastResponse({ + transport: 'http', + status: res.status, + ...(data && typeof data === 'object' ? data : {}), + }); if (res.ok) { setDispatchStatus('Accepted'); success('Event Accepted', `${data.event?.eventId || selectedClient} buffered`); } else { setDispatchStatus('Rejected'); - error('Event Rejected', data.error || 'Server rejected'); + error('Event Rejected', data.error || `Server rejected (HTTP ${res.status})`); } } catch (err: unknown) { const message = err instanceof Error ? err.message : 'Dispatch error'; @@ -249,6 +267,10 @@ export default function SimulatorPage() { }; const handleSendBatch = async (count = 10) => { + if (!canOperate) { + error('Read-only role', 'Event simulation requires operator or owner access.'); + return; + } if (!selectedClient) { error('No target client', 'Register a client in /vcon/clients first.'); return; @@ -298,12 +320,17 @@ export default function SimulatorPage() { batch: batchPayload, }), }); - if (res.status === 401) { + const data = await res.json().catch(() => ({})); + if ( + shouldRedirectSimulatorUnauthorized( + res.status, + data && typeof data === 'object' ? (data as Record) : null, + ) + ) { window.location.assign('/vcon/login'); return; } - const data = await res.json().catch(() => ({})); const summary = summarizeHttpBatchResult( res.ok, data && typeof data === 'object' ? (data as Record) : null, @@ -346,7 +373,8 @@ export default function SimulatorPage() { size="xs" variant="secondary" loading={sending} - disabled={sending || !selectedClient} + disabled={!canOperate || sending || !selectedClient} + title={!canOperate ? 'Requires operator or owner role' : undefined} onClick={() => void handleSendBatch(10)} > Burst 10 @@ -356,7 +384,8 @@ export default function SimulatorPage() { variant="primary" icon="Send" loading={sending} - disabled={sending || !selectedClient} + disabled={!canOperate || sending || !selectedClient} + title={!canOperate ? 'Requires operator or owner role' : undefined} onClick={() => void handleSendSingle()} > Send Event @@ -364,6 +393,15 @@ export default function SimulatorPage() { } > + {!canOperate && ( +
+ Read-only access ({currentRole}) — dispatching simulated events requires operator or owner permissions. +
+ )} + {clients.length === 0 && (
(() => getCached('vcon:storage-stats', null) ); @@ -76,6 +81,10 @@ export default function StoragePage() { }, [fetchStorageData]); const handleManualFlush = async () => { + if (!canOperate) { + error('Read-only role', 'Buffer flush requires operator or owner permissions.'); + return; + } setFlushing(true); try { const res = await fetch('/api/storage', { @@ -134,16 +143,22 @@ export default function StoragePage() { > Refresh - + {canOperate ? ( + + ) : ( + + Read-only ({currentRole}) + + )} } > @@ -178,25 +193,36 @@ export default function StoragePage() { {files.length === 0 ? ( ) : ( - files.map((file) => ( - - - {file.clientId} - - - {file.filename} - - - {(file.sizeBytes / 1024).toFixed(1)} KB - - - {file.linesCount.toLocaleString()} - - - {new Date(file.modifiedAt).toLocaleString()} - - - )) + files.map((file) => { + const isCompressed = Boolean( + file.compressed || + file.filename.toLowerCase().endsWith('.csv.gz') + ); + return ( + + + {file.clientId} + + +
+ + {isCompressed ? 'CSV.GZ' : 'CSV'} + + {file.filename} +
+
+ + {(file.sizeBytes / 1024).toFixed(1)} KB + + + {file.linesCount.toLocaleString()} + + + {new Date(file.modifiedAt).toLocaleString()} + +
+ ); + }) )} diff --git a/components/layout/VconShell.tsx b/components/layout/VconShell.tsx index 2e0ef7f..929ccfc 100644 --- a/components/layout/VconShell.tsx +++ b/components/layout/VconShell.tsx @@ -1,8 +1,25 @@ 'use client'; import { MainLayout } from '@/components/layout/MainLayout'; -import React from 'react'; +import type { VconAdminRole } from '@/lib/vconUiHelpers'; +import React, { createContext, useContext } from 'react'; -export function VconShell({ children }: { children: React.ReactNode }) { - return {children}; +const VconRoleContext = createContext('owner'); + +export function useVconRole(): VconAdminRole { + return useContext(VconRoleContext); +} + +export function VconShell({ + role = 'owner', + children, +}: { + role?: VconAdminRole; + children: React.ReactNode; +}) { + return ( + + {children} + + ); } diff --git a/components/sections/AuditLogViewer.tsx b/components/sections/AuditLogViewer.tsx index 73368c4..e573d3a 100644 --- a/components/sections/AuditLogViewer.tsx +++ b/components/sections/AuditLogViewer.tsx @@ -32,7 +32,10 @@ interface AuditLogItem { const ACTION_FILTER_OPTIONS = [ { value: 'ALL', label: 'All Governance Actions' }, + { value: 'ADMIN_SETUP', label: 'ADMIN_SETUP' }, { value: 'ADMIN_LOGIN', label: 'ADMIN_LOGIN' }, + { value: 'ADMIN_LOGOUT', label: 'ADMIN_LOGOUT' }, + { value: 'ADMIN_PROFILE_UPDATED', label: 'ADMIN_PROFILE_UPDATED' }, { value: 'ADMIN_PASSWORD_UPDATED', label: 'ADMIN_PASSWORD_UPDATED' }, { value: 'TEAM_MEMBER_CREATED', label: 'TEAM_MEMBER_CREATED' }, { value: 'TEAM_MEMBER_UPDATED', label: 'TEAM_MEMBER_UPDATED' }, @@ -40,9 +43,11 @@ const ACTION_FILTER_OPTIONS = [ { value: 'TEAM_MEMBER_DELETED', label: 'TEAM_MEMBER_DELETED' }, { value: 'CLIENT_CREATED', label: 'CLIENT_CREATED' }, { value: 'CLIENT_UPDATED', label: 'CLIENT_UPDATED' }, + { value: 'CLIENT_REVOKED', label: 'CLIENT_REVOKED' }, { value: 'CLIENT_TOKEN_ROTATED', label: 'CLIENT_TOKEN_ROTATED' }, { value: 'CLIENT_DELETED', label: 'CLIENT_DELETED' }, { value: 'SETTINGS_UPDATED', label: 'SETTINGS_UPDATED' }, + { value: 'STORAGE_FLUSH_TRIGGERED', label: 'STORAGE_FLUSH_TRIGGERED' }, { value: 'S3_ARCHIVE_TRIGGERED', label: 'S3_ARCHIVE_TRIGGERED' }, { value: 'WEBHOOK_TEST_TRIGGERED', label: 'WEBHOOK_TEST_TRIGGERED' }, ]; @@ -90,7 +95,9 @@ export function AuditLogViewer() { }, [fetchAuditLogs]); const uniqueActors = new Set(logs.map((l) => l.actorEmail)).size; - const mutationCount = logs.filter((l) => l.action !== 'ADMIN_LOGIN').length; + const mutationCount = logs.filter( + (l) => l.action !== 'ADMIN_LOGIN' && l.action !== 'ADMIN_LOGOUT', + ).length; return ( Promise; } @@ -35,9 +41,19 @@ function relativeTime(timestamp: string | undefined, now: number) { return `${Math.floor(seconds / 86_400)}d ago`; } -export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps) { +export function ClientsTable({ + clients, + now, + canManage, + onStatusChange, +}: ClientsTableProps) { const router = useRouter(); const { success, error } = useToast(); + const currentRole = useVconRole(); + const allowMutations = + canManage !== undefined + ? canManage + : getVconRoleCapabilities(currentRole).canOperate; const [credentials, setCredentials] = useState(null); const [confirmAction, setConfirmAction] = useState<{ type: 'rotate' | 'revoke' | 'delete'; @@ -59,21 +75,13 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps }; const openEditModal = (client: ClientItem) => { + if (!allowMutations) return; setEditClient(client); setEditName(client.name); setEditRateLimit(String(client.rateLimit || 100)); setEditStorageQuotaMb(String(client.storageQuotaMb ?? 0)); setEditAllowedIps(client.allowedIps || ''); - if (client.expiresAt) { - const parsedDate = new Date(client.expiresAt); - setEditExpiresAt( - !Number.isNaN(parsedDate.getTime()) - ? parsedDate.toISOString().slice(0, 16) - : '', - ); - } else { - setEditExpiresAt(''); - } + setEditExpiresAt(formatDateForLocalInput(client.expiresAt)); setEditError(''); }; @@ -339,49 +347,53 @@ export function ClientsTable({ clients, now, onStatusChange }: ClientsTableProps aria-label={`View events for ${client.clientId}`} onClick={() => router.push(`/vcon/data?clientId=${encodeURIComponent(client.clientId)}`)} /> -
diff --git a/components/sections/DashboardOverview.tsx b/components/sections/DashboardOverview.tsx index 695e569..893f22c 100644 --- a/components/sections/DashboardOverview.tsx +++ b/components/sections/DashboardOverview.tsx @@ -1,6 +1,7 @@ 'use client'; import { PageShell } from '@/components/layout/PageShell'; +import { useVconRole } from '@/components/layout/VconShell'; import { Button } from '@/components/ui/Button'; import { ClientList, type ClientListItem } from '@/components/ui/ClientList'; import { Panel } from '@/components/ui/Panel'; @@ -21,6 +22,8 @@ import { countMatchingBufferedEvents, extractContentDispositionFilename, formatEventTimeLabel, + getVconRoleCapabilities, + isUiClientExpired, type BufferedSummary, } from '@/lib/vconUiHelpers'; import Link from 'next/link'; @@ -36,6 +39,7 @@ type EventItem = { type DashboardMetrics = { activeSockets?: number; activeClients: number; + expiredClients?: number; totalClients: number; bufferStats: { currentBufferSize: number; @@ -65,6 +69,8 @@ type StorageFile = { export const DashboardOverview: React.FC = () => { const { error, info } = useToast(); + const currentRole = useVconRole(); + const { canOperate } = getVconRoleCapabilities(currentRole); const [metrics, setMetrics] = useState(() => getCached('vcon:metrics', null) ); @@ -179,7 +185,13 @@ export const DashboardOverview: React.FC = () => { const eventsToday = storedRecordsToday + bufferedTodayCount; const activeClients = metrics?.activeClients ?? - clients.filter((client) => client.status === 'active').length; + clients.filter( + (client) => + client.status === 'active' && + !isUiClientExpired( + (client as ClientListItem & { expiresAt?: string | null }).expiresAt, + ), + ).length; const totalClients = metrics?.totalClients ?? clients.length; const activeSockets = metrics?.activeSockets ?? 0; const bridgeStatus = metrics?.runtime?.ingestionBridgeStatus; @@ -263,11 +275,13 @@ export const DashboardOverview: React.FC = () => { > Export - - - + {canOperate && ( + + + + )} } > diff --git a/components/sections/ExportWorkspace.tsx b/components/sections/ExportWorkspace.tsx index 257b832..3f18864 100644 --- a/components/sections/ExportWorkspace.tsx +++ b/components/sections/ExportWorkspace.tsx @@ -31,6 +31,7 @@ interface StorageFile { sizeBytes: number; linesCount: number; modifiedAt: string; + compressed?: boolean; } type ExportFormat = 'csv' | 'json' | 'ndjson'; @@ -487,13 +488,17 @@ export function ExportWorkspace() { No files

) : ( - sortedRecentFiles.map((file) => ( + sortedRecentFiles.map((file) => { + const isCompressed = Boolean( + file.compressed || file.filename.toLowerCase().endsWith('.csv.gz') + ); + return (
- CSV + {isCompressed ? 'CSV.GZ' : 'CSV'}
@@ -516,7 +521,8 @@ export function ExportWorkspace() { onClick={() => downloadFile(file)} />
- )) + ); + }) )}
diff --git a/components/sections/SettingsWorkspace.tsx b/components/sections/SettingsWorkspace.tsx index 27aa55f..d814f98 100644 --- a/components/sections/SettingsWorkspace.tsx +++ b/components/sections/SettingsWorkspace.tsx @@ -1,6 +1,7 @@ 'use client'; import { PageShell } from '@/components/layout/PageShell'; +import { useVconRole } from '@/components/layout/VconShell'; import { Button } from '@/components/ui/Button'; import { FormSwitch } from '@/components/ui/Form'; import { Input } from '@/components/ui/Input'; @@ -22,6 +23,7 @@ import { DEFAULT_S3_SETTINGS, describeSettingsBridgeSync, type EditableSettings, + getVconRoleCapabilities, mapBridgeStatusToSocketUiState, normalizeAlertingSettings, normalizeEditableSettings, @@ -56,11 +58,12 @@ interface PublicSettings } interface DiskUsageSummary { - available: boolean; + available?: boolean; usedPct: number; - freeBytes: number; + freeBytes?: number; + availableBytes?: number; totalBytes: number; - checkedAt?: string; + checkedAt?: string | number; } const TABS = [ @@ -90,6 +93,8 @@ function formatGiB(bytes: number): string { } export function SettingsWorkspace() { + const currentRole = useVconRole(); + const { canAdminSettings, canOperate } = getVconRoleCapabilities(currentRole); const { success, warning, error } = useToast(); const [savedSettings, setSavedSettings] = useState(() => normalizeEditableSettings( @@ -333,7 +338,20 @@ export function SettingsWorkspace() { ) ); if (storageData.diskUsage && typeof storageData.diskUsage === 'object') { - setDiskUsage(storageData.diskUsage as DiskUsageSummary); + const rawDisk = storageData.diskUsage as DiskUsageSummary; + const totalBytes = Number(rawDisk.totalBytes) || 0; + const freeBytes = Number(rawDisk.freeBytes ?? rawDisk.availableBytes) || 0; + setDiskUsage({ + ...rawDisk, + available: + rawDisk.available !== undefined + ? Boolean(rawDisk.available) + : totalBytes > 0, + usedPct: Number(rawDisk.usedPct) || 0, + freeBytes, + availableBytes: freeBytes, + totalBytes, + }); } if (metricsData) { setActiveSockets(Number(metricsData.activeSockets) || 0); @@ -535,6 +553,7 @@ export function SettingsWorkspace() { }; const handleTestWebhookAlert = async () => { + if (!canAdminSettings) return; setTestingWebhook(true); setWebhookTestStatus(null); try { @@ -574,6 +593,7 @@ export function SettingsWorkspace() { }; const handleTestS3Connection = async () => { + if (!canAdminSettings) return; setTestingS3(true); setS3ActionStatus(null); try { @@ -617,6 +637,7 @@ export function SettingsWorkspace() { }; const handleArchiveNow = async () => { + if (!canOperate) return; if (s3ConfigDirty) { warning('Unsaved S3 settings', 'Save your S3 credentials before archiving.'); return; @@ -657,6 +678,7 @@ export function SettingsWorkspace() { const handleFormSubmit = async (event: React.FormEvent) => { event.preventDefault(); + if (!canAdminSettings) return; const validation = validateEditableSettingsDraft( numericDrafts, enableIdempotency @@ -787,6 +809,15 @@ export function SettingsWorkspace() { ]} /> + {!canAdminSettings && ( +
+ Read-only configuration ({currentRole}) — only owners can modify platform settings, credentials, or run credential diagnostics. +
+ )} +
@@ -838,6 +870,7 @@ export function SettingsWorkspace() { @@ -916,6 +949,7 @@ export function SettingsWorkspace() { type="number" min={SETTINGS_LIMITS.batchFlushThreshold.min} max={SETTINGS_LIMITS.batchFlushThreshold.max} + disabled={!canAdminSettings} value={numericDrafts.batchFlushThreshold} error={fieldErrors.batchFlushThreshold} onChange={(event) => @@ -930,6 +964,7 @@ export function SettingsWorkspace() { min={SETTINGS_LIMITS.batchFlushIntervalMs.min} max={SETTINGS_LIMITS.batchFlushIntervalMs.max} suffix="ms" + disabled={!canAdminSettings} value={numericDrafts.batchFlushIntervalMs} error={fieldErrors.batchFlushIntervalMs} onChange={(event) => @@ -963,7 +998,7 @@ export function SettingsWorkspace() { {diskUsage?.available && ( - {formatGiB(diskUsage.freeBytes)} free /{' '} + {formatGiB(diskUsage.freeBytes ?? diskUsage.availableBytes ?? 0)} free /{' '} {formatGiB(diskUsage.totalBytes)} )} @@ -995,6 +1030,7 @@ export function SettingsWorkspace() { min={SETTINGS_LIMITS.fileRotationMb.min} max={SETTINGS_LIMITS.fileRotationMb.max} suffix="MB" + disabled={!canAdminSettings} value={numericDrafts.fileRotationMb} error={fieldErrors.fileRotationMb} onChange={(event) => @@ -1010,6 +1046,7 @@ export function SettingsWorkspace() { min={SETTINGS_LIMITS.autoPruneDays.min} max={SETTINGS_LIMITS.autoPruneDays.max} suffix="days" + disabled={!canAdminSettings} value={numericDrafts.autoPruneDays} error={fieldErrors.autoPruneDays} onChange={(event) => @@ -1025,6 +1062,7 @@ export function SettingsWorkspace() { min={SETTINGS_LIMITS.compressAfterDays.min} max={SETTINGS_LIMITS.compressAfterDays.max} suffix="days" + disabled={!canAdminSettings} value={numericDrafts.compressAfterDays} error={fieldErrors.compressAfterDays} onChange={(event) => @@ -1040,6 +1078,7 @@ export function SettingsWorkspace() { min={SETTINGS_LIMITS.diskPauseThresholdPct.min} max={SETTINGS_LIMITS.diskPauseThresholdPct.max} suffix="%" + disabled={!canAdminSettings} value={numericDrafts.diskPauseThresholdPct} error={fieldErrors.diskPauseThresholdPct} onChange={(event) => @@ -1055,6 +1094,7 @@ export function SettingsWorkspace() { setS3Draft((prev) => ({ ...prev, enableS3Backup: checked })) } @@ -1064,6 +1104,7 @@ export function SettingsWorkspace() { setS3Draft((prev) => ({ @@ -1078,6 +1119,7 @@ export function SettingsWorkspace() { setS3Draft((prev) => ({ @@ -1092,6 +1134,7 @@ export function SettingsWorkspace() { setS3Draft((prev) => ({ @@ -1106,6 +1149,7 @@ export function SettingsWorkspace() { setS3Draft((prev) => ({ @@ -1120,6 +1164,7 @@ export function SettingsWorkspace() { setS3Draft((prev) => ({ @@ -1136,6 +1181,7 @@ export function SettingsWorkspace() { compact type="password" autoComplete="new-password" + disabled={!canAdminSettings} value={s3SecretAccessKey} onChange={(event) => { setS3SecretAccessKey(event.target.value); @@ -1155,6 +1201,7 @@ export function SettingsWorkspace() { type="button" size="xs" variant={clearS3SecretKey ? 'danger' : 'ghost'} + disabled={!canAdminSettings} onClick={() => { setClearS3SecretKey((prev) => !prev); setS3SecretAccessKey(''); @@ -1174,7 +1221,7 @@ export function SettingsWorkspace() { variant="secondary" icon="Plug" loading={testingS3} - disabled={testingS3 || archivingNow} + disabled={!canAdminSettings || testingS3 || archivingNow} onClick={() => void handleTestS3Connection()} > Test S3 Connection @@ -1185,7 +1232,7 @@ export function SettingsWorkspace() { variant="secondary" icon="UploadCloud" loading={archivingNow} - disabled={testingS3 || archivingNow || s3ConfigDirty} + disabled={!canOperate || testingS3 || archivingNow || s3ConfigDirty} onClick={() => void handleArchiveNow()} > Archive Now @@ -1233,6 +1280,7 @@ export function SettingsWorkspace() { compact type="password" autoComplete="new-password" + disabled={!canAdminSettings} value={jwtSecret} error={jwtError} onChange={(event) => { @@ -1252,6 +1300,7 @@ export function SettingsWorkspace() { min={SETTINGS_LIMITS.maxPayloadSizeKb.min} max={SETTINGS_LIMITS.maxPayloadSizeKb.max} suffix="KB" + disabled={!canAdminSettings} value={numericDrafts.maxPayloadSizeKb} error={fieldErrors.maxPayloadSizeKb} onChange={(event) => @@ -1267,6 +1316,7 @@ export function SettingsWorkspace() { setAlertingDraft((prev) => ({ ...prev, @@ -1279,6 +1329,7 @@ export function SettingsWorkspace() { setAlertingDraft((prev) => ({ @@ -1295,6 +1346,7 @@ export function SettingsWorkspace() { compact type="password" autoComplete="new-password" + disabled={!canAdminSettings} value={webhookSecret} onChange={(event) => { setWebhookSecret(event.target.value); @@ -1314,6 +1366,7 @@ export function SettingsWorkspace() { type="button" size="xs" variant={clearWebhookSecret ? 'danger' : 'ghost'} + disabled={!canAdminSettings} onClick={() => { setClearWebhookSecret((prev) => !prev); setWebhookSecret(''); @@ -1331,6 +1384,7 @@ export function SettingsWorkspace() { min={SETTINGS_LIMITS.webhookCooldownSec.min} max={SETTINGS_LIMITS.webhookCooldownSec.max} suffix="sec" + disabled={!canAdminSettings} value={numericDrafts.webhookCooldownSec} error={fieldErrors.webhookCooldownSec} onChange={(event) => @@ -1348,7 +1402,7 @@ export function SettingsWorkspace() { variant="secondary" icon="Bell" loading={testingWebhook} - disabled={testingWebhook} + disabled={!canAdminSettings || testingWebhook} onClick={() => void handleTestWebhookAlert()} > Send Test Alert @@ -1375,6 +1429,7 @@ export function SettingsWorkspace() { setAlertingDraft((prev) => ({ ...prev, @@ -1397,6 +1452,7 @@ export function SettingsWorkspace() { compact type="password" autoComplete="new-password" + disabled={!canAdminSettings} value={prometheusMetricsToken} onChange={(event) => { setPrometheusMetricsToken(event.target.value); @@ -1416,6 +1472,7 @@ export function SettingsWorkspace() { type="button" size="xs" variant={clearPrometheusToken ? 'danger' : 'ghost'} + disabled={!canAdminSettings} onClick={() => { setClearPrometheusToken((prev) => !prev); setPrometheusMetricsToken(''); @@ -1435,11 +1492,21 @@ export function SettingsWorkspace() { <> - {loading ? 'Loading' : isDirty ? 'Unsaved changes' : 'Saved'} + {loading + ? 'Loading' + : !canAdminSettings + ? `Read-only (${currentRole})` + : isDirty + ? 'Unsaved changes' + : 'Saved'} } @@ -1447,7 +1514,7 @@ export function SettingsWorkspace() { diff --git a/core/Security/ipAllowlist.ts b/core/Security/ipAllowlist.ts index 0c2bb8e..1e98263 100644 --- a/core/Security/ipAllowlist.ts +++ b/core/Security/ipAllowlist.ts @@ -5,7 +5,17 @@ import net from "node:net"; */ export function stripIpv6MappedPrefix(ip: string): string { - const trimmed = String(ip || "").trim(); + let trimmed = String(ip || "").trim(); + if (!trimmed) return ""; + if (trimmed.startsWith("[") && trimmed.includes("]")) { + trimmed = trimmed.slice(1, trimmed.indexOf("]")); + } else if (/^\d{1,3}(?:\.\d{1,3}){3}:\d+$/.test(trimmed)) { + trimmed = trimmed.slice(0, trimmed.lastIndexOf(":")); + } + const pctIndex = trimmed.indexOf("%"); + if (pctIndex !== -1) { + trimmed = trimmed.slice(0, pctIndex); + } if (trimmed.toLowerCase().startsWith("::ffff:")) { const candidate = trimmed.slice(7); if (net.isIPv4(candidate)) { @@ -142,6 +152,7 @@ export function isClientIpAllowed( return entries.includes("*"); } + const clientFamily = net.isIP(cleanClientIp); const clientIpv4Int = ipv4ToUint32(cleanClientIp); const isLoopbackClient = cleanClientIp === "127.0.0.1" || cleanClientIp.toLowerCase() === "::1"; @@ -153,13 +164,47 @@ export function isClientIpAllowed( const [ruleIpRaw, prefixRaw] = rule.split("/"); const ruleIp = stripIpv6MappedPrefix(ruleIpRaw); const prefix = Number(prefixRaw); - const ruleIpv4Int = ipv4ToUint32(ruleIp); + const ruleFamily = net.isIP(ruleIp); - if (clientIpv4Int !== null && ruleIpv4Int !== null && prefix >= 0 && prefix <= 32) { + if ( + clientFamily === 4 && + ruleFamily === 4 && + Number.isInteger(prefix) && + prefix >= 0 && + prefix <= 32 + ) { if (prefix === 0) return true; - const mask = prefix === 32 ? 0xffffffff : (~((1 << (32 - prefix)) - 1)) >>> 0; - if ((clientIpv4Int & mask) >>> 0 === (ruleIpv4Int & mask) >>> 0) { - return true; + try { + const bl = new net.BlockList(); + bl.addSubnet(ruleIp, prefix, "ipv4"); + if (bl.check(cleanClientIp, "ipv4")) { + return true; + } + } catch { + const ruleIpv4Int = ipv4ToUint32(ruleIp); + if (clientIpv4Int !== null && ruleIpv4Int !== null) { + const mask = prefix === 32 ? 0xffffffff : (~((1 << (32 - prefix)) - 1)) >>> 0; + if ((clientIpv4Int & mask) >>> 0 === (ruleIpv4Int & mask) >>> 0) { + return true; + } + } + } + } else if ( + clientFamily === 6 && + ruleFamily === 6 && + Number.isInteger(prefix) && + prefix >= 0 && + prefix <= 128 + ) { + if (prefix === 0) return true; + try { + const bl = new net.BlockList(); + bl.addSubnet(ruleIp, prefix, "ipv6"); + if (bl.check(cleanClientIp, "ipv6")) { + return true; + } + } catch { + // ignore malformed IPv6 subnet } } continue; diff --git a/core/Storage/csvStorage.ts b/core/Storage/csvStorage.ts index 0243534..4dbb2cd 100644 --- a/core/Storage/csvStorage.ts +++ b/core/Storage/csvStorage.ts @@ -26,8 +26,10 @@ export interface StorageFileMeta { } export interface DiskUsageSnapshot { + available: boolean; usedPct: number; availableBytes: number; + freeBytes: number; totalBytes: number; checkedAt: number; } @@ -85,7 +87,9 @@ export class CSVStorageManager { } public static extractDateFromFilename(filename: string): string { - return filename.replace(/\.csv(?:\.gz)?$/, '').split('-part')[0]; + return String(filename || '') + .replace(/\.csv(?:\.gz)?$/i, '') + .replace(/(?:-(?:part|gz)\d+)+$/i, ''); } public static clearRowCountCache(): void { @@ -1073,8 +1077,10 @@ export class CSVStorageManager { ) : 0; const snapshot: DiskUsageSnapshot = { + available: totalBytes > 0, usedPct, availableBytes, + freeBytes: availableBytes, totalBytes, checkedAt: now, }; @@ -1082,8 +1088,10 @@ export class CSVStorageManager { return snapshot; } catch { const fallback: DiskUsageSnapshot = { + available: false, usedPct: 0, availableBytes: 0, + freeBytes: 0, totalBytes: 0, checkedAt: now, }; diff --git a/core/adminSettings/settingsManager.ts b/core/adminSettings/settingsManager.ts index 9af6800..ed510f7 100644 --- a/core/adminSettings/settingsManager.ts +++ b/core/adminSettings/settingsManager.ts @@ -92,12 +92,33 @@ type SettingsGlobals = typeof globalThis & { __dataStackLastSettingsBridgeSync?: SettingsBridgeSyncStatus; __dataStackLastPruneMeta?: { days: number; ranAtMs: number }; __dataStackLastCompressMeta?: { days: number; ranAtMs: number }; + __dataStackMaintenanceTimer?: ReturnType; + __dataStackMaintenanceIntervalMs?: number; + __dataStackMaintenanceRunning?: boolean; + __dataStackArchivePruneInFlight?: boolean; + __dataStackCompressInFlight?: boolean; }; const PRUNE_COOLDOWN_MS = 60 * 60 * 1000; const COMPRESS_COOLDOWN_MS = 60 * 60 * 1000; +const DEFAULT_MAINTENANCE_INTERVAL_MS = 15 * 60 * 1000; const globals = globalThis as SettingsGlobals; +function resolveMaintenanceIntervalMs(overrideMs?: number): number { + if ( + typeof overrideMs === "number" && + Number.isFinite(overrideMs) && + overrideMs >= 20 + ) { + return Math.floor(overrideMs); + } + const envVal = Number(process.env.DATASTACK_MAINTENANCE_INTERVAL_MS); + if (Number.isFinite(envVal) && envVal >= 1000) { + return Math.floor(envVal); + } + return DEFAULT_MAINTENANCE_INTERVAL_MS; +} + function getDefaultSettings(): AdminSettings { return { serverPort: Number(process.env.PORT_SOCKET) || 4000, @@ -257,10 +278,167 @@ export class SettingsManager { ); } + public static isMaintenanceSchedulerActive(): boolean { + return Boolean(globals.__dataStackMaintenanceTimer); + } + + public static ensureMaintenanceScheduler(intervalMs?: number): boolean { + const effectiveIntervalMs = resolveMaintenanceIntervalMs(intervalMs); + if (globals.__dataStackMaintenanceTimer) { + if ( + intervalMs === undefined || + globals.__dataStackMaintenanceIntervalMs === effectiveIntervalMs + ) { + return false; + } + clearInterval(globals.__dataStackMaintenanceTimer); + delete globals.__dataStackMaintenanceTimer; + } + + const timer = setInterval(() => { + void this.runScheduledMaintenanceTick().catch((err) => { + console.warn( + "[DataStack Storage] Periodic maintenance tick failed:", + err, + ); + }); + }, effectiveIntervalMs); + timer.unref?.(); + globals.__dataStackMaintenanceTimer = timer; + globals.__dataStackMaintenanceIntervalMs = effectiveIntervalMs; + return true; + } + + public static stopMaintenanceScheduler(): void { + if (globals.__dataStackMaintenanceTimer) { + clearInterval(globals.__dataStackMaintenanceTimer); + delete globals.__dataStackMaintenanceTimer; + } + delete globals.__dataStackMaintenanceIntervalMs; + globals.__dataStackMaintenanceRunning = false; + } + + public static stopMaintenanceSchedulerForTests(): void { + this.stopMaintenanceScheduler(); + globals.__dataStackArchivePruneInFlight = false; + globals.__dataStackCompressInFlight = false; + } + + public static async runScheduledMaintenanceTick(options?: { + forceCompress?: boolean; + forcePrune?: boolean; + skipDbSync?: boolean; + }): Promise<{ + ran: boolean; + skippedReason?: string; + compressed: boolean; + archivedOrPruned: boolean; + }> { + if (globals.__dataStackMaintenanceRunning) { + return { + ran: false, + skippedReason: "ALREADY_RUNNING", + compressed: false, + archivedOrPruned: false, + }; + } + + globals.__dataStackMaintenanceRunning = true; + try { + const hasConfiguredDb = Boolean(process.env.DATABASE_URL?.trim()); + if (hasConfiguredDb && !options?.skipDbSync) { + await this.syncFromDatabase(true); + } + + const dbReadyForMaintenance = + !hasConfiguredDb || globals.__dataStackSettingsHydrated === true; + if (!dbReadyForMaintenance) { + return { + ran: false, + skippedReason: "DB_UNHYDRATED", + compressed: false, + archivedOrPruned: false, + }; + } + + const settings = getSettingsState(); + const now = Date.now(); + let compressed = false; + let archivedOrPruned = false; + + if ( + settings.compressAfterDays > 0 && + !globals.__dataStackCompressInFlight + ) { + const prevComp = globals.__dataStackLastCompressMeta; + const shouldCompress = + Boolean(options?.forceCompress) || + !prevComp || + prevComp.days !== settings.compressAfterDays || + now - prevComp.ranAtMs >= COMPRESS_COOLDOWN_MS; + if (shouldCompress) { + globals.__dataStackLastCompressMeta = { + days: settings.compressAfterDays, + ranAtMs: now, + }; + globals.__dataStackCompressInFlight = true; + try { + await CSVStorageManager.compressOldPartitionsAsync( + settings.compressAfterDays, + ); + compressed = true; + } catch (err) { + console.warn( + "[DataStack Storage] Scheduled partition compression failed:", + err, + ); + } finally { + globals.__dataStackCompressInFlight = false; + } + } + } + + if ( + settings.autoPruneDays > 0 && + !globals.__dataStackArchivePruneInFlight + ) { + const prevPrune = globals.__dataStackLastPruneMeta; + const shouldPrune = + Boolean(options?.forcePrune) || + !prevPrune || + prevPrune.days !== settings.autoPruneDays || + now - prevPrune.ranAtMs >= PRUNE_COOLDOWN_MS; + if (shouldPrune) { + globals.__dataStackLastPruneMeta = { + days: settings.autoPruneDays, + ranAtMs: now, + }; + globals.__dataStackArchivePruneInFlight = true; + try { + await this.executeArchiveAndPrune(settings); + archivedOrPruned = true; + } catch (err) { + console.warn( + "[DataStack Storage] Scheduled partition prune/archive failed:", + err, + ); + } finally { + globals.__dataStackArchivePruneInFlight = false; + } + } + } + + return { ran: true, compressed, archivedOrPruned }; + } finally { + globals.__dataStackMaintenanceRunning = false; + } + } + private static applyRuntimeSettings( settings: AdminSettings, forcePrune = false, forceCompress = false, + skipBackgroundMaintenance = false, ) { SecurityEngine.maxEventsPerSec = settings.maxEventsPerSec; SecurityEngine.enableIdempotency = settings.enableIdempotency; @@ -271,11 +449,19 @@ export class SettingsManager { AuthService.setSecret(settings.jwtSecret); } CSVStorageManager.maxFileSizeBytes = settings.fileRotationMb * 1024 * 1024; + this.ensureMaintenanceScheduler(); + if (skipBackgroundMaintenance) { + return; + } const hasConfiguredDb = Boolean(process.env.DATABASE_URL?.trim()); const dbReadyForMaintenance = !hasConfiguredDb || globals.__dataStackSettingsHydrated === true; - if (settings.compressAfterDays > 0 && dbReadyForMaintenance) { + if ( + settings.compressAfterDays > 0 && + dbReadyForMaintenance && + !globals.__dataStackCompressInFlight + ) { const prevComp = globals.__dataStackLastCompressMeta; const now = Date.now(); const shouldCompress = @@ -288,18 +474,27 @@ export class SettingsManager { days: settings.compressAfterDays, ranAtMs: now, }; + globals.__dataStackCompressInFlight = true; void CSVStorageManager.compressOldPartitionsAsync( settings.compressAfterDays, - ).catch((err) => { - console.warn( - "[DataStack Storage] Background partition compression failed:", - err, - ); - }); + ) + .catch((err) => { + console.warn( + "[DataStack Storage] Background partition compression failed:", + err, + ); + }) + .finally(() => { + globals.__dataStackCompressInFlight = false; + }); } } - if (settings.autoPruneDays > 0 && dbReadyForMaintenance) { + if ( + settings.autoPruneDays > 0 && + dbReadyForMaintenance && + !globals.__dataStackArchivePruneInFlight + ) { const prev = globals.__dataStackLastPruneMeta; const now = Date.now(); const shouldPrune = @@ -312,17 +507,24 @@ export class SettingsManager { days: settings.autoPruneDays, ranAtMs: now, }; - void this.executeArchiveAndPrune(settings).catch((err) => { - console.warn( - "[DataStack Storage] Background async partition prune/archive failed:", - err, - ); - }); + globals.__dataStackArchivePruneInFlight = true; + void this.executeArchiveAndPrune(settings) + .catch((err) => { + console.warn( + "[DataStack Storage] Background async partition prune/archive failed:", + err, + ); + }) + .finally(() => { + globals.__dataStackArchivePruneInFlight = false; + }); } } } - private static async syncFromDatabase(): Promise { + private static async syncFromDatabase( + skipBackgroundMaintenance = false, + ): Promise { const pool = DatabaseEngine.getPool(); if (!pool) return null; @@ -414,7 +616,12 @@ export class SettingsManager { }; globals.__dataStackAdminSettings = loaded; globals.__dataStackSettingsHydrated = true; - this.applyRuntimeSettings(loaded); + this.applyRuntimeSettings( + loaded, + false, + false, + skipBackgroundMaintenance, + ); return loaded; } @@ -625,6 +832,19 @@ export class SettingsManager { process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000" ).replace(/\/$/, ""); + let decryptedWebhookSecret = ""; + if (settings.webhookSecretEnc) { + try { + decryptedWebhookSecret = decryptSecret(settings.webhookSecretEnc); + } catch (vaultErr: unknown) { + console.warn( + "[DataStack Bridge] Stored webhook secret could not be decrypted during Port 4000 settings sync; continuing with empty webhook secret:", + vaultErr instanceof Error ? vaultErr.message : vaultErr, + ); + decryptedWebhookSecret = ""; + } + } + try { const res = await fetch(`${ingestionBaseUrl}/internal/settings-sync`, { method: "POST", @@ -645,6 +865,7 @@ export class SettingsManager { diskPauseThresholdPct: settings.diskPauseThresholdPct, enableWebhookAlerts: settings.enableWebhookAlerts, webhookUrl: settings.webhookUrl, + webhookSecret: decryptedWebhookSecret, webhookCooldownSec: settings.webhookCooldownSec, ...(jwtSecretChanged ? { jwtSecret: settings.jwtSecret } : {}), }), diff --git a/core/apiEngine/apiService.ts b/core/apiEngine/apiService.ts index 8c93aaf..0abe55b 100644 --- a/core/apiEngine/apiService.ts +++ b/core/apiEngine/apiService.ts @@ -2,6 +2,7 @@ import { SettingsManager } from '../adminSettings/settingsManager'; import { SDKManager } from '../sdk/sdkManager'; import { CSVStorageManager } from '../Storage/csvStorage'; import { DataBufferEngine } from '../data/buffer'; +import { isClientTokenExpired } from '../Security/ipAllowlist'; import { SecurityEngine, getInternalIngestionHeaders, @@ -20,7 +21,13 @@ export class ApiEngine { DataBufferEngine.getBufferStatsAsync(), SDKManager.listClientsAsync(), ]); - const activeClients = clients.filter((c) => c.status === 'active').length; + const nowMs = Date.now(); + const expiredClients = clients.filter( + (c) => c.status === 'active' && isClientTokenExpired(c.expiresAt, nowMs), + ).length; + const activeClients = clients.filter( + (c) => c.status === 'active' && !isClientTokenExpired(c.expiresAt, nowMs), + ).length; let activeSockets = 0; let ingestionLocalBufferSize = 0; let ingestionRedisBufferSize = 0; @@ -129,6 +136,7 @@ export class ApiEngine { activeSockets, totalClients: clients.length, activeClients, + expiredClients, bufferStats, storageStats, diskUsage, diff --git a/core/auth/adminUsers.ts b/core/auth/adminUsers.ts index 0e1546a..a190cd4 100644 --- a/core/auth/adminUsers.ts +++ b/core/auth/adminUsers.ts @@ -913,7 +913,11 @@ export async function updateAdminAccount({ currentPassword, ); if (!authCheck.authenticated) { - return { ok: false as const, error: "Current password is incorrect." }; + return { + ok: false as const, + error: "Current password is incorrect.", + code: "INVALID_PASSWORD" as const, + }; } const cleanEmail = nextEmail.trim().toLowerCase(); @@ -926,6 +930,18 @@ export async function updateAdminAccount({ if (pool) { await ensureAdminTable(); if (authCheck.memberId === "owner-1") { + const teamCollision = await pool.query( + `SELECT 1 FROM admin_team_members WHERE LOWER(email) = $1 LIMIT 1`, + [cleanEmail], + ); + if (teamCollision.rowCount && teamCollision.rowCount > 0) { + return { + ok: false as const, + error: `An administrator with email '${cleanEmail}' already exists.`, + code: "ADMIN_EMAIL_CONFLICT" as const, + }; + } + const updated = await pool.query( `UPDATE admin_users SET email = $1, @@ -937,7 +953,11 @@ export async function updateAdminAccount({ [cleanEmail, salt, passwordHash], ); if (!updated.rowCount || updated.rowCount === 0) { - return { ok: false as const, error: "Admin account not found." }; + return { + ok: false as const, + error: "Admin account not found.", + code: "ACCOUNT_NOT_FOUND" as const, + }; } return { ok: true as const, @@ -948,6 +968,30 @@ export async function updateAdminAccount({ }; } + const ownerCollision = await pool.query( + `SELECT 1 FROM admin_users WHERE id = 1 AND LOWER(email) = $1 LIMIT 1`, + [cleanEmail], + ); + if (ownerCollision.rowCount && ownerCollision.rowCount > 0) { + return { + ok: false as const, + error: `An administrator with email '${cleanEmail}' already exists.`, + code: "ADMIN_EMAIL_CONFLICT" as const, + }; + } + + const peerCollision = await pool.query( + `SELECT 1 FROM admin_team_members WHERE LOWER(email) = $1 AND id <> $2 LIMIT 1`, + [cleanEmail, authCheck.memberId], + ); + if (peerCollision.rowCount && peerCollision.rowCount > 0) { + return { + ok: false as const, + error: `An administrator with email '${cleanEmail}' already exists.`, + code: "ADMIN_EMAIL_CONFLICT" as const, + }; + } + const updatedMember = await pool.query( `UPDATE admin_team_members SET email = $1, @@ -960,7 +1004,11 @@ export async function updateAdminAccount({ [cleanEmail, salt, passwordHash, authCheck.memberId], ); if (!updatedMember.rowCount || updatedMember.rowCount === 0) { - return { ok: false as const, error: "Team account not found." }; + return { + ok: false as const, + error: "Team account not found.", + code: "ACCOUNT_NOT_FOUND" as const, + }; } return { ok: true as const, @@ -972,6 +1020,16 @@ export async function updateAdminAccount({ } if (authCheck.memberId === "owner-1") { + for (const peer of getInMemoryTeamMap().values()) { + if (peer.email.toLowerCase() === cleanEmail) { + return { + ok: false as const, + error: `An administrator with email '${cleanEmail}' already exists.`, + code: "ADMIN_EMAIL_CONFLICT" as const, + }; + } + } + const existing = await getOrInitInMemoryAdmin(); const nextVersion = existing.tokenVersion + 1; globals.__dataStackInMemoryAdmin = { @@ -991,10 +1049,32 @@ export async function updateAdminAccount({ }; } + const inMemOwner = await getOrInitInMemoryAdmin(); + if (inMemOwner.email.toLowerCase() === cleanEmail) { + return { + ok: false as const, + error: `An administrator with email '${cleanEmail}' already exists.`, + code: "ADMIN_EMAIL_CONFLICT" as const, + }; + } + const teamMap = getInMemoryTeamMap(); const member = teamMap.get(authCheck.memberId); if (!member) { - return { ok: false as const, error: "Team account not found." }; + return { + ok: false as const, + error: "Team account not found.", + code: "ACCOUNT_NOT_FOUND" as const, + }; + } + for (const peer of teamMap.values()) { + if (peer.id !== member.id && peer.email.toLowerCase() === cleanEmail) { + return { + ok: false as const, + error: `An administrator with email '${cleanEmail}' already exists.`, + code: "ADMIN_EMAIL_CONFLICT" as const, + }; + } } const nextVersion = member.tokenVersion + 1; teamMap.set(member.id, { @@ -1014,12 +1094,23 @@ export async function updateAdminAccount({ }; } -export async function revokeAdminSessions(memberId?: string): Promise { +export async function revokeAdminSessions( + target?: string | { memberId?: string; email?: string }, +): Promise { + const explicitMemberId = + typeof target === "string" + ? target.trim() + : target?.memberId?.trim() || ""; + const explicitEmail = + typeof target === "object" && target !== null + ? target.email?.trim().toLowerCase() || "" + : ""; const effectiveMemberId = - memberId || globals.__dataStackLastVerifiedMemberId || "owner-1"; + explicitMemberId || + (!explicitEmail ? globals.__dataStackLastVerifiedMemberId || "owner-1" : ""); const pool = getPool(); - if (effectiveMemberId !== "owner-1" && effectiveMemberId !== "1") { + if (effectiveMemberId && effectiveMemberId !== "owner-1" && effectiveMemberId !== "1") { if (pool) { await ensureAdminTable(); const updated = await pool.query( @@ -1045,6 +1136,33 @@ export async function revokeAdminSessions(memberId?: string): Promise { } } + if (explicitEmail) { + if (pool) { + await ensureAdminTable(); + const updatedByEmail = await pool.query( + `UPDATE admin_team_members + SET token_version = COALESCE(token_version, 1) + 1, + updated_at = NOW() + WHERE LOWER(email) = $1 + RETURNING token_version`, + [explicitEmail], + ); + if (updatedByEmail.rowCount && updatedByEmail.rowCount > 0) { + return Number(updatedByEmail.rows[0].token_version) || 1; + } + } else { + const teamMap = getInMemoryTeamMap(); + for (const member of teamMap.values()) { + if (member.email.toLowerCase() === explicitEmail) { + member.tokenVersion += 1; + member.updatedAt = new Date(); + teamMap.set(member.id, member); + return member.tokenVersion; + } + } + } + } + if (pool) { await ensureAdminTable(); const updated = await pool.query( diff --git a/core/auth/auditLog.ts b/core/auth/auditLog.ts index 660fdcc..452dab0 100644 --- a/core/auth/auditLog.ts +++ b/core/auth/auditLog.ts @@ -2,7 +2,9 @@ import { randomBytes } from "node:crypto"; import { DatabaseEngine } from "@/core/Database/database"; export type AuditActionType = + | "ADMIN_SETUP" | "ADMIN_LOGIN" + | "ADMIN_LOGOUT" | "ADMIN_PROFILE_UPDATED" | "ADMIN_PASSWORD_UPDATED" | "TEAM_MEMBER_CREATED" @@ -11,9 +13,11 @@ export type AuditActionType = | "TEAM_MEMBER_DELETED" | "CLIENT_CREATED" | "CLIENT_UPDATED" + | "CLIENT_REVOKED" | "CLIENT_TOKEN_ROTATED" | "CLIENT_DELETED" | "SETTINGS_UPDATED" + | "STORAGE_FLUSH_TRIGGERED" | "S3_ARCHIVE_TRIGGERED" | "WEBHOOK_TEST_TRIGGERED" | string; diff --git a/core/auth/jwt.ts b/core/auth/jwt.ts index 2b1131a..49c7d79 100644 --- a/core/auth/jwt.ts +++ b/core/auth/jwt.ts @@ -47,10 +47,13 @@ export class AuthService { /** * Generates a signed JWT for client SDK authentication. + * When `expiresIn` is null/omitted, no hardcoded JWT `exp` claim is embedded so + * database-backed `client_sdks.expires_at` (including "Never expires" or >30d + * policies and live expiry extensions) remains the authoritative expiry check. */ public static generateClientToken( payload: { clientId: string; name?: string; rateLimit?: number }, - expiresIn = "30d", + expiresIn?: string | number | null, ): string { if (!this.isConfigured()) { throw new Error( @@ -58,15 +61,19 @@ export class AuthService { ); } - return jwt.sign( - { - clientId: payload.clientId, - name: payload.name || payload.clientId, - rateLimit: payload.rateLimit || 100, - }, - this.secret, - { expiresIn: expiresIn as jwt.SignOptions["expiresIn"] }, - ); + const tokenBody = { + clientId: payload.clientId, + name: payload.name || payload.clientId, + rateLimit: payload.rateLimit || 100, + }; + + if (expiresIn !== undefined && expiresIn !== null && expiresIn !== "") { + return jwt.sign(tokenBody, this.secret, { + expiresIn: expiresIn as jwt.SignOptions["expiresIn"], + }); + } + + return jwt.sign(tokenBody, this.secret); } /** diff --git a/core/sdk/sdkManager.ts b/core/sdk/sdkManager.ts index 7d4f9f1..cbe75be 100644 --- a/core/sdk/sdkManager.ts +++ b/core/sdk/sdkManager.ts @@ -540,11 +540,14 @@ export class SDKManager { const allowedIps = typeof data.allowedIps === "string" ? data.allowedIps.trim() : ""; const expiresAt = data.expiresAt ? String(data.expiresAt) : null; - const token = AuthService.generateClientToken({ - clientId: safeClientId, - name: data.name.trim(), - rateLimit, - }); + const token = AuthService.generateClientToken( + { + clientId: safeClientId, + name: data.name.trim(), + rateLimit, + }, + null, + ); const newClient: ClientSDK = { clientId: safeClientId, @@ -637,11 +640,14 @@ export class SDKManager { const allowedIps = typeof data.allowedIps === "string" ? data.allowedIps.trim() : ""; const expiresAt = data.expiresAt ? String(data.expiresAt) : null; - const token = AuthService.generateClientToken({ - clientId: safeClientId, - name: data.name.trim(), - rateLimit, - }); + const token = AuthService.generateClientToken( + { + clientId: safeClientId, + name: data.name.trim(), + rateLimit, + }, + null, + ); const newClient: ClientSDK = { clientId: safeClientId, @@ -674,11 +680,14 @@ export class SDKManager { const updated: ClientSDK = { ...client, secret: this.generateClientSecret(normalizedId), - token: AuthService.generateClientToken({ - clientId: client.clientId, - name: client.name, - rateLimit: client.rateLimit, - }), + token: AuthService.generateClientToken( + { + clientId: client.clientId, + name: client.name, + rateLimit: client.rateLimit, + }, + null, + ), }; getClientsMap().set(normalizedId, updated); @@ -724,11 +733,14 @@ export class SDKManager { const updated: ClientSDK = { ...existing, secret: this.generateClientSecret(normalizedId), - token: AuthService.generateClientToken({ - clientId: existing.clientId, - name: existing.name, - rateLimit: existing.rateLimit, - }), + token: AuthService.generateClientToken( + { + clientId: existing.clientId, + name: existing.name, + rateLimit: existing.rateLimit, + }, + null, + ), }; await this.persistClientCredentials(updated); @@ -1170,11 +1182,14 @@ export class SDKManager { public static resignAllClientTokens(): void { const map = getClientsMap(); for (const [clientId, client] of map.entries()) { - client.token = AuthService.generateClientToken({ - clientId: client.clientId, - name: client.name, - rateLimit: client.rateLimit, - }); + client.token = AuthService.generateClientToken( + { + clientId: client.clientId, + name: client.name, + rateLimit: client.rateLimit, + }, + null, + ); map.set(clientId, client); void this.persistClientCredentials(client).catch((err) => { console.error("[DataStack SDK] Background re-sign DB persist failed:", err); @@ -1188,11 +1203,14 @@ export class SDKManager { for (const client of clients) { const updated: ClientSDK = { ...client, - token: AuthService.generateClientToken({ - clientId: client.clientId, - name: client.name, - rateLimit: client.rateLimit, - }), + token: AuthService.generateClientToken( + { + clientId: client.clientId, + name: client.name, + rateLimit: client.rateLimit, + }, + null, + ), }; await this.persistClientCredentials(updated); map.set(updated.clientId, updated); diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index b0c67b1..38a1da4 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -53,6 +53,7 @@ services: PORT_SOCKET: "4000" DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}" DATASTACK_INTERNAL_SECRET: "${DATASTACK_INTERNAL_SECRET:-${SERVICE_REALBASE64_64_INTERNAL:-}}" + DATASTACK_ADMIN_SESSION_SECRET: "${SERVICE_REALBASE64_64_ADMINSESSION:?Coolify must generate SERVICE_REALBASE64_64_ADMINSESSION}" DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}" REDIS_URL: "redis://redis:6379" DATASTACK_CORS_ORIGINS: "${DATASTACK_CORS_ORIGINS:-}" diff --git a/lib/vconUiHelpers.ts b/lib/vconUiHelpers.ts index d642cb7..eb1471e 100644 --- a/lib/vconUiHelpers.ts +++ b/lib/vconUiHelpers.ts @@ -658,3 +658,164 @@ export function formatSessionExpiryLabel(value: string): string { hour12: false, }); } + +const SIMULATOR_CLIENT_AUTH_ERROR_CODES = new Set([ + 'CLIENT_TOKEN_EXPIRED', + 'TOKEN_ROTATED', + 'AUTH_INVALID', + 'CLIENT_REVOKED', + 'CLIENT_IP_FORBIDDEN', +]); + +/** + * BUG-11 (ERR-R3-07): Only redirect /vcon/simulator to /vcon/login when HTTP 401 + * indicates an expired/invalid Admin Console session, NOT a target SDK client rejection + * (such as CLIENT_TOKEN_EXPIRED, TOKEN_ROTATED, or AUTH_INVALID). + */ +export function shouldRedirectSimulatorUnauthorized( + status: number, + body: Record | null | undefined, +): boolean { + if (status !== 401) return false; + const code = typeof body?.code === 'string' ? body.code.trim().toUpperCase() : ''; + if (code && SIMULATOR_CLIENT_AUTH_ERROR_CODES.has(code)) { + return false; + } + return true; +} + +/** + * BUG-12: Formats an ISO timestamp string into local browser `YYYY-MM-DDTHH:mm` + * suitable for `` without UTC offset drift on save. + */ +export function formatDateForLocalInput(iso: string | null | undefined): string { + if (!iso || typeof iso !== 'string') return ''; + const parsed = new Date(iso); + if (Number.isNaN(parsed.getTime())) return ''; + const pad2 = (n: number) => String(n).padStart(2, '0'); + const year = parsed.getFullYear(); + const month = pad2(parsed.getMonth() + 1); + const day = pad2(parsed.getDate()); + const hours = pad2(parsed.getHours()); + const minutes = pad2(parsed.getMinutes()); + return `${year}-${month}-${day}T${hours}:${minutes}`; +} + +export type UiClientComputedStatus = + | 'revoked' + | 'expired' + | 'online' + | 'pending' + | 'offline'; + +export interface UiClientStatusInput { + status: 'active' | 'revoked'; + expiresAt?: string | null; + lastActive?: string; +} + +export function isUiClientExpired( + expiresAt: string | null | undefined, + nowMs: number = Date.now(), +): boolean { + if (!expiresAt) return false; + const parsed = Date.parse(expiresAt); + return Number.isFinite(parsed) && nowMs >= parsed; +} + +/** + * BUG-14: Classifies a client into one mutually exclusive UI status bucket so + * expired credentials are tracked separately from active offline clients. + */ +export function classifyUiClientStatus( + client: UiClientStatusInput, + nowMs: number = Date.now(), +): UiClientComputedStatus { + if (client.status === 'revoked') return 'revoked'; + if (isUiClientExpired(client.expiresAt, nowMs)) return 'expired'; + if (!client.lastActive) return 'pending'; + const lastActiveMs = Date.parse(client.lastActive); + if (Number.isFinite(lastActiveMs) && nowMs - lastActiveMs < 60_000) { + return 'online'; + } + return 'offline'; +} + +export interface UiClientStatusCounts { + total: number; + active: number; + online: number; + offline: number; + pending: number; + expired: number; + revoked: number; +} + +export function summarizeUiClientCounts( + clients: readonly UiClientStatusInput[], + nowMs: number = Date.now(), +): UiClientStatusCounts { + let online = 0; + let offline = 0; + let pending = 0; + let expired = 0; + let revoked = 0; + + for (const client of clients) { + const bucket = classifyUiClientStatus(client, nowMs); + if (bucket === 'online') online += 1; + else if (bucket === 'offline') offline += 1; + else if (bucket === 'pending') pending += 1; + else if (bucket === 'expired') expired += 1; + else revoked += 1; + } + + return { + total: clients.length, + active: online + offline + pending, + online, + offline, + pending, + expired, + revoked, + }; +} + +export type VconAdminRole = 'owner' | 'admin' | 'operator' | 'viewer'; + +export interface VconRoleCapabilities { + role: VconAdminRole; + isOwner: boolean; + canAdminSettings: boolean; + canOperate: boolean; + isReadOnly: boolean; +} + +/** + * BUG-16: Central RBAC UI capability mapper aligned 1:1 with server-side + * `requireAdminApiSession(req, { minRole })` rules. + */ +export function getVconRoleCapabilities( + rawRole: string | null | undefined, +): VconRoleCapabilities { + const normalized = String(rawRole || 'viewer').trim().toLowerCase(); + const role: VconAdminRole = + normalized === 'owner' + ? 'owner' + : normalized === 'admin' + ? 'admin' + : normalized === 'operator' + ? 'operator' + : 'viewer'; + + const isOwner = role === 'owner'; + const canOperate = role === 'owner' || role === 'admin' || role === 'operator'; + return { + role, + isOwner, + canAdminSettings: isOwner, + canOperate, + isReadOnly: !canOperate, + }; +} + diff --git a/sdk-python/datastack_sdk/__init__.py b/sdk-python/datastack_sdk/__init__.py index 5fa570e..e03017b 100644 --- a/sdk-python/datastack_sdk/__init__.py +++ b/sdk-python/datastack_sdk/__init__.py @@ -44,6 +44,7 @@ from datastack import ( # noqa: E402, F401 DataStackFileError, DataStackOfflineSpoolError, DataStackPayloadTooLargeError, + DataStackQuotaExceededError, DataStackRateLimitError, DiskSpooler, HeartbeatMonitor, @@ -65,6 +66,7 @@ __all__ = [ "DataStackConnectionError", "DataStackRateLimitError", "DataStackPayloadTooLargeError", + "DataStackQuotaExceededError", "DataStackFileError", "DataStackOfflineSpoolError", ] diff --git a/sdk/node/datastack/index.ts b/sdk/node/datastack/index.ts index ccb9573..f2587c2 100644 --- a/sdk/node/datastack/index.ts +++ b/sdk/node/datastack/index.ts @@ -3,7 +3,7 @@ import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; import { io, type Socket } from 'socket.io-client'; -import { DiskSpooler } from './spool'; +import { DiskSpooler, type DiskSpoolerOptions } from './spool'; import { DataStackAuthError, DataStackConfigError, @@ -13,6 +13,7 @@ import { DataStackQuotaExceededError, DataStackRateLimitError, DataStackShutdownTimeoutError, + type DataStackAuthErrorCode, type DataStackConfigOptions, type DataStackConnectionState, type DataStackEnvelope, @@ -20,14 +21,67 @@ import { type DataStackFileSendResult, type DataStackSendResult, type DataStackStatusSnapshot, + type DataStackTransportMode, type ResolvedDataStackConfig, } from './types'; -export * from './spool'; -export * from './types'; +export { + DiskSpooler, + DataStackAuthError, + DataStackConfigError, + DataStackError, + DataStackFileError, + DataStackPayloadTooLargeError, + DataStackQuotaExceededError, + DataStackRateLimitError, + DataStackShutdownTimeoutError, + type DiskSpoolerOptions, + type DataStackAuthErrorCode, + type DataStackConfigOptions, + type DataStackConnectionState, + type DataStackEnvelope, + type DataStackEventPayload, + type DataStackFileSendResult, + type DataStackSendResult, + type DataStackStatusSnapshot, + type DataStackTransportMode, + type ResolvedDataStackConfig, +}; export const SDK_VERSION = '2.2.0'; +function classifyAuthRejection( + rawCode: string, + rawMsg: string +): { code: DataStackAuthErrorCode; statusCode: number } | null { + const code = String(rawCode || '').toUpperCase(); + const upperMsg = String(rawMsg || '').toUpperCase(); + const combined = `${code} ${upperMsg}`; + + if (combined.includes('CLIENT_TOKEN_EXPIRED')) { + return { code: 'CLIENT_TOKEN_EXPIRED', statusCode: 401 }; + } + if (combined.includes('TOKEN_ROTATED')) { + return { code: 'TOKEN_ROTATED', statusCode: 401 }; + } + if (combined.includes('CLIENT_IP_FORBIDDEN')) { + return { code: 'CLIENT_IP_FORBIDDEN', statusCode: 403 }; + } + if (combined.includes('CLIENT_REVOKED') || upperMsg.includes('REVOKED')) { + return { code: 'CLIENT_REVOKED', statusCode: 403 }; + } + if ( + code === 'AUTH_FAILED' || + code === 'AUTH_INVALID' || + code === 'AUTH_REQUIRED' || + upperMsg.includes('AUTHENTICATION') || + upperMsg.includes('INVALID TOKEN') + ) { + return { code: 'AUTH_FAILED', statusCode: 401 }; + } + return null; +} + /** * Matches server `SDKManager.normalizeClientId()` and Python SDK `_normalize_client_id()`. */ @@ -124,6 +178,9 @@ export class DataStackClient { private transportState: DataStackConnectionState = 'DISCONNECTED'; private inFlightSet = new Set>(); private isDrainingSpool = false; + private heartbeatTimer: ReturnType | null = null; + private heartbeatInFlight = false; + private heartbeatStopped = false; private readonly fetchFn: typeof fetch; constructor(options: DataStackConfigOptions = {}) { @@ -204,6 +261,14 @@ export class DataStackClient { ); } + const rawHeartbeat = Number( + options.heartbeatIntervalMs ?? fileCfg.heartbeatIntervalMs ?? 5000 + ); + const heartbeatIntervalMs = + Number.isFinite(rawHeartbeat) && rawHeartbeat > 0 + ? Math.max(50, Math.floor(rawHeartbeat)) + : 0; + this.clientId = normToken || normConfigured; this.config = { serverUrl, @@ -213,9 +278,7 @@ export class DataStackClient { autoConnect: Boolean(options.autoConnect ?? fileCfg.autoConnect ?? false), enableHttpFallback: options.enableHttpFallback ?? fileCfg.enableHttpFallback ?? true, - heartbeatIntervalMs: Number( - options.heartbeatIntervalMs ?? fileCfg.heartbeatIntervalMs ?? 5000 - ), + heartbeatIntervalMs, maxRetries, batchChunkSize: Math.floor(batchChunkSize), offlineSpoolEnabled: @@ -250,6 +313,125 @@ export class DataStackClient { }); } + private startHeartbeatMonitor(): void { + if ( + this.heartbeatStopped || + this.config.heartbeatIntervalMs <= 0 || + this.heartbeatTimer !== null + ) { + return; + } + const timer = setInterval(() => { + void this.runHeartbeatStep(); + }, this.config.heartbeatIntervalMs); + timer.unref?.(); + this.heartbeatTimer = timer; + } + + private stopHeartbeatMonitor(): void { + if (this.heartbeatTimer !== null) { + clearInterval(this.heartbeatTimer); + this.heartbeatTimer = null; + } + } + + /** + * Verifies the active WebSocket handshake with Port 4000 via `ping_verify`. + * Throws `DataStackAuthError` immediately on terminal auth rejections. + */ + public async pingVerify(timeoutMs?: number): Promise { + if (!this.isConnected || !this.socket || !this.socket.connected) { + return false; + } + const waitMs = Math.max( + 100, + Math.min(timeoutMs ?? this.config.requestTimeoutMs, 5000) + ); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => resolve(false), waitMs); + this.socket!.emit( + 'ping_verify', + {}, + (ack: Record | undefined) => { + clearTimeout(timer); + if (!ack || typeof ack !== 'object') { + resolve(false); + return; + } + if (ack.success === true) { + resolve(ack.status === 'VERIFIED' || ack.status === undefined); + return; + } + const code = String(ack.code || ''); + const msg = String(ack.error || 'Handshake verification failed'); + const authErr = classifyAuthRejection(code, msg); + if (authErr) { + this.heartbeatStopped = true; + this.stopHeartbeatMonitor(); + reject( + new DataStackAuthError(msg, authErr.code, authErr.statusCode, ack) + ); + return; + } + resolve(false); + } + ); + }); + } + + /** + * Executes one zero-CPU IDLE / active recovery heartbeat step: + * - If WebSocket is already connected (`IDLE_WS`), stops the active recovery timer and returns `true`. + * - If disconnected/fallback, attempts WS reconnection + `ping_verify` verification, drains spool, + * and returns to zero-CPU `IDLE_WS` upon recovery. + */ + public async runHeartbeatStep(): Promise { + if (this.heartbeatStopped || this.heartbeatInFlight) { + return false; + } + if (this.isConnected && this.socket?.connected) { + this.transportState = 'IDLE_WS'; + this.stopHeartbeatMonitor(); + return true; + } + + this.heartbeatInFlight = true; + try { + const reconnected = await this.connect(); + if (!reconnected) { + return false; + } + const verified = await this.pingVerify(); + if (!verified) { + if (this.socket) { + this.socket.removeAllListeners(); + this.socket.disconnect(); + this.socket = null; + } + this.isConnected = false; + this.transportState = this.config.enableHttpFallback + ? 'HTTP_FALLBACK' + : 'DISCONNECTED'; + this.startHeartbeatMonitor(); + return false; + } + this.transportState = 'IDLE_WS'; + this.stopHeartbeatMonitor(); + if (this.spooler.count() > 0) { + void this.flushSpool().catch(() => {}); + } + return true; + } catch (err) { + if (err instanceof DataStackAuthError) { + this.heartbeatStopped = true; + this.stopHeartbeatMonitor(); + } + return false; + } finally { + this.heartbeatInFlight = false; + } + } + /** * Establishes a real-time Socket.IO connection to Port 4000 (`serverUrl`). */ @@ -259,7 +441,9 @@ export class DataStackClient { 'Cannot connect: clientToken is required.' ); } + this.heartbeatStopped = false; if (this.isConnected && this.socket?.connected) { + this.stopHeartbeatMonitor(); return true; } @@ -291,6 +475,7 @@ export class DataStackClient { this.transportState = this.config.enableHttpFallback ? 'HTTP_FALLBACK' : 'DISCONNECTED'; + this.startHeartbeatMonitor(); resolve(false); }, this.config.requestTimeoutMs); @@ -300,6 +485,7 @@ export class DataStackClient { clearTimeout(timer); this.isConnected = true; this.transportState = 'IDLE_WS'; + this.stopHeartbeatMonitor(); void this.flushSpool().catch(() => {}); resolve(true); }); @@ -313,43 +499,16 @@ export class DataStackClient { const msg = String(err?.message || ''); const code = String(err?.data?.code || ''); - if ( - code === 'CLIENT_TOKEN_EXPIRED' || - msg.includes('CLIENT_TOKEN_EXPIRED') - ) { - reject( - new DataStackAuthError( - msg || 'Client JWT token has expired.', - 'CLIENT_TOKEN_EXPIRED', - 401 - ) - ); - return; - } - if ( - code === 'CLIENT_IP_FORBIDDEN' || - msg.includes('CLIENT_IP_FORBIDDEN') - ) { - reject( - new DataStackAuthError( - msg || 'Client IP is not permitted by allowlist.', - 'CLIENT_IP_FORBIDDEN', - 403 - ) - ); - return; - } - if ( - code === 'AUTH_FAILED' || - msg.toLowerCase().includes('authentication') || - msg.toLowerCase().includes('invalid token') || - msg.toLowerCase().includes('revoked') - ) { + const authErr = classifyAuthRejection(code, msg); + if (authErr) { + this.heartbeatStopped = true; + this.stopHeartbeatMonitor(); reject( new DataStackAuthError( msg || 'Socket authentication failed.', - 'AUTH_FAILED', - 401 + authErr.code, + authErr.statusCode, + err?.data ) ); return; @@ -358,6 +517,7 @@ export class DataStackClient { this.transportState = this.config.enableHttpFallback ? 'HTTP_FALLBACK' : 'DISCONNECTED'; + this.startHeartbeatMonitor(); resolve(false); }); @@ -368,11 +528,16 @@ export class DataStackClient { ? 'HTTP_FALLBACK' : 'DISCONNECTED'; } + if (!this.heartbeatStopped) { + this.startHeartbeatMonitor(); + } }); }); } public disconnect(): void { + this.heartbeatStopped = true; + this.stopHeartbeatMonitor(); if (this.socket) { this.socket.removeAllListeners(); this.socket.disconnect(); @@ -463,22 +628,28 @@ export class DataStackClient { } if (ack.success === true) { this.transportState = 'IDLE_WS'; + const isDuplicate = + ack.note === 'DUPLICATE_DROPPED' || + ack.note === 'DUPLICATE_EVENT' || + ack.duplicate === true; resolve({ success: true, transport: 'websocket', eventId: String(ack.eventId || envelope.eventId), - accepted: 1, + accepted: isDuplicate ? 0 : 1, + duplicates: isDuplicate ? 1 : 0, }); return; } const code = String(ack.code || 'SOCKET_REJECTED'); const msg = String(ack.error || 'Socket event rejected'); - if (code === 'CLIENT_TOKEN_EXPIRED' || code === 'AUTH_FAILED' || code === 'CLIENT_REVOKED') { - reject(new DataStackAuthError(msg, code as 'CLIENT_TOKEN_EXPIRED' | 'AUTH_FAILED', 401)); - return; - } - if (code === 'CLIENT_IP_FORBIDDEN') { - reject(new DataStackAuthError(msg, 'CLIENT_IP_FORBIDDEN', 403)); + const authErr = classifyAuthRejection(code, msg); + if (authErr) { + this.heartbeatStopped = true; + this.stopHeartbeatMonitor(); + reject( + new DataStackAuthError(msg, authErr.code, authErr.statusCode, ack) + ); return; } if (code === 'STORAGE_QUOTA_EXCEEDED' || code === 'DISK_SPACE_LOW') { @@ -537,12 +708,16 @@ export class DataStackClient { const body = (await res.json().catch(() => ({}))) as Record; if (res.ok && body.success !== false) { this.transportState = 'HTTP_FALLBACK'; + const isDuplicate = + body.note === 'DUPLICATE_DROPPED' || + body.note === 'DUPLICATE_EVENT' || + body.duplicate === true; return { success: true, transport: 'http', eventId: String(body.eventId || envelope.eventId), - accepted: 1, - duplicates: body.duplicate ? 1 : 0, + accepted: isDuplicate ? 0 : 1, + duplicates: isDuplicate ? 1 : 0, }; } @@ -550,14 +725,19 @@ export class DataStackClient { const errMsg = String(body.error || `HTTP ${res.status} error`); if (res.status === 401) { - throw new DataStackAuthError( - errMsg, - code === 'CLIENT_TOKEN_EXPIRED' ? 'CLIENT_TOKEN_EXPIRED' : 'AUTH_FAILED', - 401, - body - ); + this.heartbeatStopped = true; + this.stopHeartbeatMonitor(); + const authCode: DataStackAuthErrorCode = + code === 'CLIENT_TOKEN_EXPIRED' + ? 'CLIENT_TOKEN_EXPIRED' + : code === 'TOKEN_ROTATED' + ? 'TOKEN_ROTATED' + : 'AUTH_FAILED'; + throw new DataStackAuthError(errMsg, authCode, 401, body); } if (res.status === 403) { + this.heartbeatStopped = true; + this.stopHeartbeatMonitor(); throw new DataStackAuthError( errMsg, code === 'CLIENT_IP_FORBIDDEN' ? 'CLIENT_IP_FORBIDDEN' : 'CLIENT_REVOKED', @@ -687,14 +867,12 @@ export class DataStackClient { ); this.socket!.emit( 'batch_sync', - { - clientId: this.clientId, - batch: chunk.map((item) => ({ - eventId: item.eventId, - timestamp: item.timestamp, - data: item.data, - })), - }, + chunk.map((item) => ({ + clientId: item.clientId || this.clientId || undefined, + eventId: item.eventId, + timestamp: item.timestamp, + data: item.data, + })), (ack: Record | undefined) => { clearTimeout(timer); if (!ack || typeof ack !== 'object') { @@ -705,26 +883,35 @@ export class DataStackClient { const rateLimited = Array.isArray(ack.rateLimitedEventIds) ? (ack.rateLimitedEventIds as string[]) : []; + const duplicates = Number(ack.duplicates ?? 0); + const acceptedVal = + ack.processed ?? + ack.processedCount ?? + ack.accepted ?? + Math.max(0, chunk.length - rateLimited.length - duplicates); resolve({ success: true, transport: 'websocket', - accepted: Number( - ack.processedCount ?? - ack.accepted ?? - chunk.length - rateLimited.length - ), + accepted: Number(acceptedVal), + duplicates, rateLimitedEventIds: rateLimited, }); return; } const code = String(ack.code || 'BATCH_REJECTED'); const msg = String(ack.error || 'Batch rejected'); - if (code === 'CLIENT_TOKEN_EXPIRED' || code === 'AUTH_FAILED') { - reject(new DataStackAuthError(msg, code as 'CLIENT_TOKEN_EXPIRED' | 'AUTH_FAILED', 401)); - return; - } - if (code === 'CLIENT_IP_FORBIDDEN') { - reject(new DataStackAuthError(msg, 'CLIENT_IP_FORBIDDEN', 403)); + const authErr = classifyAuthRejection(code, msg); + if (authErr) { + this.heartbeatStopped = true; + this.stopHeartbeatMonitor(); + reject( + new DataStackAuthError( + msg, + authErr.code, + authErr.statusCode, + ack + ) + ); return; } if (code === 'STORAGE_QUOTA_EXCEEDED' || code === 'DISK_SPACE_LOW') { @@ -801,15 +988,17 @@ export class DataStackClient { if (res.ok && body.success !== false) { this.transportState = 'HTTP_FALLBACK'; + const duplicates = Number(body.duplicates ?? 0); + const acceptedVal = + body.processed ?? + body.processedCount ?? + body.accepted ?? + Math.max(0, chunk.length - rateLimited.length - duplicates); return { success: true, transport: 'http', - accepted: Number( - body.accepted ?? - body.processedCount ?? - chunk.length - rateLimited.length - ), - duplicates: Number(body.duplicates ?? 0), + accepted: Number(acceptedVal), + duplicates, rateLimitedEventIds: rateLimited, }; } @@ -818,14 +1007,19 @@ export class DataStackClient { const errMsg = String(body.error || `HTTP ${res.status} batch error`); if (res.status === 401) { - throw new DataStackAuthError( - errMsg, - code === 'CLIENT_TOKEN_EXPIRED' ? 'CLIENT_TOKEN_EXPIRED' : 'AUTH_FAILED', - 401, - body - ); + this.heartbeatStopped = true; + this.stopHeartbeatMonitor(); + const authCode: DataStackAuthErrorCode = + code === 'CLIENT_TOKEN_EXPIRED' + ? 'CLIENT_TOKEN_EXPIRED' + : code === 'TOKEN_ROTATED' + ? 'TOKEN_ROTATED' + : 'AUTH_FAILED'; + throw new DataStackAuthError(errMsg, authCode, 401, body); } if (res.status === 403) { + this.heartbeatStopped = true; + this.stopHeartbeatMonitor(); throw new DataStackAuthError( errMsg, code === 'CLIENT_IP_FORBIDDEN' ? 'CLIENT_IP_FORBIDDEN' : 'CLIENT_REVOKED', @@ -864,6 +1058,7 @@ export class DataStackClient { ): Promise { let pending = [...chunk]; let totalAccepted = 0; + let totalDuplicates = 0; let totalSpooled = 0; let lastTransport: 'websocket' | 'http' | 'spool' = 'http'; let attempt = 0; @@ -873,15 +1068,18 @@ export class DataStackClient { const res = await this.sendBatchChunkInternal(pending); lastTransport = res.transport; const rateLimitedSet = new Set(res.rateLimitedEventIds || []); + const duplicatesNow = res.duplicates || 0; + totalDuplicates += duplicatesNow; if (rateLimitedSet.size === 0) { - totalAccepted += res.accepted ?? pending.length; + totalAccepted += res.accepted ?? Math.max(0, pending.length - duplicatesNow); pending = []; break; } const acceptedNow = - res.accepted ?? Math.max(0, pending.length - rateLimitedSet.size); + res.accepted ?? + Math.max(0, pending.length - rateLimitedSet.size - duplicatesNow); totalAccepted += acceptedNow; pending = pending.filter((item) => rateLimitedSet.has(item.eventId)); @@ -911,6 +1109,7 @@ export class DataStackClient { success: left.success && right.success, transport: right.transport, accepted: (left.accepted || 0) + (right.accepted || 0), + duplicates: (left.duplicates || 0) + (right.duplicates || 0), spooled: (left.spooled || 0) + (right.spooled || 0), }; } @@ -958,6 +1157,7 @@ export class DataStackClient { success: pending.length === 0, transport: lastTransport, accepted: totalAccepted, + duplicates: totalDuplicates, spooled: totalSpooled, unprocessedItems: pending.map((p) => p.data), }; @@ -986,6 +1186,7 @@ export class DataStackClient { ); let totalAccepted = 0; + let totalDuplicates = 0; let totalSpooled = 0; let lastTransport: 'websocket' | 'http' | 'spool' = 'http'; @@ -993,6 +1194,7 @@ export class DataStackClient { const slice = envelopes.slice(i, i + chunkSize); const res = await this.deliverChunkWithSplitAndRetry(slice, true); totalAccepted += res.accepted || 0; + totalDuplicates += res.duplicates || 0; totalSpooled += res.spooled || 0; lastTransport = res.transport; } @@ -1001,6 +1203,7 @@ export class DataStackClient { success: true, transport: lastTransport, accepted: totalAccepted, + duplicates: totalDuplicates, spooled: totalSpooled, }; })(); @@ -1116,6 +1319,7 @@ export class DataStackClient { ); let acceptedRows = 0; + let duplicateRows = 0; let spooledRows = 0; let chunksSent = 0; let lastTransport: 'websocket' | 'http' | 'spool' = 'http'; @@ -1124,18 +1328,20 @@ export class DataStackClient { const slice = envelopes.slice(i, i + chunkSize); const res = await this.deliverChunkWithSplitAndRetry(slice, true); acceptedRows += res.accepted || 0; + duplicateRows += res.duplicates || 0; spooledRows += res.spooled || 0; chunksSent += 1; lastTransport = res.transport; } return { - success: acceptedRows + spooledRows === envelopes.length, + success: acceptedRows + duplicateRows + spooledRows === envelopes.length, transport: lastTransport, filePath: resolvedPath, totalRows: envelopes.length, accepted: acceptedRows, acceptedRows, + duplicates: duplicateRows, spooled: spooledRows, spooledRows, chunksSent, @@ -1237,6 +1443,7 @@ export class DataStackClient { spooledCount: this.spooler.count(), spooledBytes: this.spooler.sizeBytes(), inFlightRequests: this.inFlightSet.size, + heartbeatActive: this.heartbeatTimer !== null, }; } } diff --git a/sdk/node/datastack/package.json b/sdk/node/datastack/package.json index 0a90dc5..c0532f0 100644 --- a/sdk/node/datastack/package.json +++ b/sdk/node/datastack/package.json @@ -2,18 +2,23 @@ "name": "@datastack/sdk", "version": "2.2.0", "description": "Official DataStack Node.js & TypeScript Client SDK — Real-time Socket.IO telemetry streaming with automatic HTTP fallback, deterministic file ingestion, and offline disk spooling.", - "main": "./index.ts", - "types": "./index.ts", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", "exports": { ".": { - "types": "./index.ts", - "default": "./index.ts" + "types": "./dist/index.d.ts", + "default": "./dist/index.js" } }, + "scripts": { + "build": "tsc -p tsconfig.json" + }, "files": [ + "dist", "index.ts", "types.ts", "spool.ts", + "tsconfig.json", "README.md" ], "engines": { diff --git a/sdk/node/datastack/tsconfig.json b/sdk/node/datastack/tsconfig.json new file mode 100644 index 0000000..ef751c6 --- /dev/null +++ b/sdk/node/datastack/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "commonjs", + "moduleResolution": "node", + "lib": ["ES2022", "DOM"], + "outDir": "./dist", + "rootDir": ".", + "declaration": true, + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["index.ts", "types.ts", "spool.ts"], + "exclude": ["dist", "node_modules"] +} diff --git a/sdk/node/datastack/types.ts b/sdk/node/datastack/types.ts index e12da34..c3228e5 100644 --- a/sdk/node/datastack/types.ts +++ b/sdk/node/datastack/types.ts @@ -96,6 +96,7 @@ export interface DataStackStatusSnapshot { spooledCount: number; spooledBytes: number; inFlightRequests: number; + heartbeatActive?: boolean; } /** @@ -127,14 +128,17 @@ export class DataStackConfigError extends DataStackError { } } +export type DataStackAuthErrorCode = + | 'AUTH_FAILED' + | 'CLIENT_TOKEN_EXPIRED' + | 'CLIENT_IP_FORBIDDEN' + | 'CLIENT_REVOKED' + | 'TOKEN_ROTATED'; + export class DataStackAuthError extends DataStackError { constructor( message: string, - code: - | 'AUTH_FAILED' - | 'CLIENT_TOKEN_EXPIRED' - | 'CLIENT_IP_FORBIDDEN' - | 'CLIENT_REVOKED' = 'AUTH_FAILED', + code: DataStackAuthErrorCode = 'AUTH_FAILED', statusCode = 401, details?: Record ) { diff --git a/sdk/python/datastack/datastack/__init__.py b/sdk/python/datastack/datastack/__init__.py index 913d13c..1808f3e 100644 --- a/sdk/python/datastack/datastack/__init__.py +++ b/sdk/python/datastack/datastack/__init__.py @@ -15,6 +15,7 @@ from .exceptions import ( DataStackFileError, DataStackOfflineSpoolError, DataStackPayloadTooLargeError, + DataStackQuotaExceededError, DataStackRateLimitError, ) from .heartbeat import ConnectionState, HeartbeatMonitor @@ -36,6 +37,7 @@ __all__ = [ "DataStackConnectionError", "DataStackRateLimitError", "DataStackPayloadTooLargeError", + "DataStackQuotaExceededError", "DataStackFileError", "DataStackOfflineSpoolError", ] diff --git a/sdk/python/datastack/datastack/client.py b/sdk/python/datastack/datastack/client.py index 71023f1..49bcbdf 100644 --- a/sdk/python/datastack/datastack/client.py +++ b/sdk/python/datastack/datastack/client.py @@ -50,6 +50,7 @@ from .exceptions import ( DataStackFileError, DataStackOfflineSpoolError, DataStackPayloadTooLargeError, + DataStackQuotaExceededError, DataStackRateLimitError, ) from .heartbeat import ConnectionState, HeartbeatMonitor @@ -284,9 +285,37 @@ class DataStackClient: "AUTH_INVALID", "CLIENT_REVOKED", "TOKEN_ROTATED", + "CLIENT_TOKEN_EXPIRED", + "CLIENT_IP_FORBIDDEN", ) ) + @staticmethod + def _extract_auth_error_code(message: str, default_code: str = "AUTH_ERROR") -> str: + upper = (message or "").upper() + for code in ( + "CLIENT_TOKEN_EXPIRED", + "CLIENT_IP_FORBIDDEN", + "CLIENT_REVOKED", + "TOKEN_ROTATED", + "AUTH_INVALID", + "AUTH_REQUIRED", + ): + if code in upper: + return code + return default_code + + @staticmethod + def _extract_quota_error_code( + message: str, default_code: str = "STORAGE_QUOTA_EXCEEDED" + ) -> str: + upper = (message or "").upper() + if "DISK_SPACE_LOW" in upper: + return "DISK_SPACE_LOW" + if "STORAGE_QUOTA_EXCEEDED" in upper: + return "STORAGE_QUOTA_EXCEEDED" + return default_code + def _ensure_token_present(self) -> None: if not self.token or self.token == "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN": raise DataStackConfigError( @@ -380,7 +409,9 @@ class DataStackClient: self.is_connected = False msg = str(exc) if self._is_non_retryable_auth_error(msg): - raise DataStackAuthError(msg) from exc + raise DataStackAuthError( + msg, code=self._extract_auth_error_code(msg) + ) from exc return False # Verify handshake & PostgreSQL client status via `ping_verify` while still serialized under RLock @@ -431,7 +462,13 @@ class DataStackClient: result_holder.get("error") or "Handshake verification failed" ) if self._is_non_retryable_auth_error(err_msg): - raise DataStackAuthError(err_msg) + raise DataStackAuthError( + err_msg, + code=str( + result_holder.get("code") + or self._extract_auth_error_code(err_msg) + ), + ) return False return result_holder.get("status") == "VERIFIED" @@ -599,6 +636,16 @@ class DataStackClient: if status_code == 413 or "PAYLOAD_TOO_LARGE" in combined: raise DataStackPayloadTooLargeError(err_msg) + if ( + status_code == 507 + or "STORAGE_QUOTA_EXCEEDED" in combined + or "DISK_SPACE_LOW" in combined + ): + raise DataStackQuotaExceededError( + err_msg, + code=self._extract_quota_error_code(combined), + ) + if status_code == 429 or "RATE_LIMIT_EXCEEDED" in combined: raise DataStackRateLimitError(err_msg) @@ -666,6 +713,14 @@ class DataStackClient: raise DataStackAuthError(err_msg, code=err_code or "AUTH_ERROR") if "PAYLOAD_TOO_LARGE" in combined: raise DataStackPayloadTooLargeError(err_msg) + if ( + "STORAGE_QUOTA_EXCEEDED" in combined + or "DISK_SPACE_LOW" in combined + ): + raise DataStackQuotaExceededError( + err_msg, + code=self._extract_quota_error_code(combined), + ) if "RATE_LIMIT_EXCEEDED" in combined: raise DataStackRateLimitError(err_msg) raise DataStackError(err_msg, code=err_code or "WS_REJECTED") @@ -722,7 +777,11 @@ class DataStackClient: ): self._drain_offline_spool() return res - except (DataStackAuthError, DataStackPayloadTooLargeError): + except ( + DataStackAuthError, + DataStackPayloadTooLargeError, + DataStackQuotaExceededError, + ): raise except DataStackRateLimitError as rl_err: last_exception = rl_err @@ -749,7 +808,11 @@ class DataStackClient: ): self._drain_offline_spool() return res - except (DataStackAuthError, DataStackPayloadTooLargeError): + except ( + DataStackAuthError, + DataStackPayloadTooLargeError, + DataStackQuotaExceededError, + ): raise except DataStackRateLimitError as rl_err: last_exception = rl_err @@ -757,10 +820,14 @@ class DataStackClient: except Exception as http_err: last_exception = http_err - # 3. If rate-limited or payload-rejected after all retries, raise directly + # 3. If rate-limited, payload-rejected, or quota-exceeded after retries, raise directly if isinstance( last_exception, - (DataStackRateLimitError, DataStackPayloadTooLargeError), + ( + DataStackRateLimitError, + DataStackPayloadTooLargeError, + DataStackQuotaExceededError, + ), ): raise last_exception diff --git a/sdk/python/datastack/datastack/exceptions.py b/sdk/python/datastack/datastack/exceptions.py index 1aa255d..611e4c4 100644 --- a/sdk/python/datastack/datastack/exceptions.py +++ b/sdk/python/datastack/datastack/exceptions.py @@ -51,6 +51,19 @@ class DataStackPayloadTooLargeError(DataStackError): super().__init__(message, code="PAYLOAD_TOO_LARGE", retryable=False) +class DataStackQuotaExceededError(DataStackError): + """ + Raised when the server rejects ingestion with HTTP `507` or WebSocket + `STORAGE_QUOTA_EXCEEDED` / `DISK_SPACE_LOW`. Non-retryable until quota + or disk space is freed. + """ + + def __init__( + self, message: str, code: str = "STORAGE_QUOTA_EXCEEDED" + ) -> None: + super().__init__(message, code=code, retryable=False) + + class DataStackConnectionError(DataStackError): """Raised when both WebSocket and HTTP fallback are unreachable and offline spooling is disabled.""" diff --git a/server/index.js b/server/index.js index bf6f878..352c275 100644 --- a/server/index.js +++ b/server/index.js @@ -1,4 +1,5 @@ const crypto = require('node:crypto'); +const net = require('node:net'); const express = require('express'); const http = require('http'); const cors = require('cors'); @@ -202,6 +203,10 @@ const runtimeConfig = { compressAfterDays: 0, enableS3Backup: false, diskPauseThresholdPct: 95, + enableWebhookAlerts: false, + webhookUrl: '', + webhookSecret: '', + webhookCooldownSec: 300, }; let previousJwtSecret = ''; @@ -338,7 +343,7 @@ function matchesIpv4Cidr(clientIp, cidrPattern) { function isClientIpAllowed(clientIp, rawAllowedIps) { if (!rawAllowedIps || !String(rawAllowedIps).trim()) return true; const entries = String(rawAllowedIps) - .split(/[\s,]+/) + .split(/[\s,;]+/) .map((entry) => entry.trim()) .filter(Boolean); if (entries.length === 0) return true; @@ -349,15 +354,53 @@ function isClientIpAllowed(clientIp, rawAllowedIps) { if (!normalizedClientIp || normalizedClientIp === 'unknown') { return false; } + const clientFamily = net.isIP(normalizedClientIp); for (const entry of entries) { const normalizedEntry = stripIpv6MappedPrefix(entry); if (normalizedEntry.includes('/')) { - if (matchesIpv4Cidr(normalizedClientIp, normalizedEntry)) { - return true; - } - const [netAddr, prefixStr] = normalizedEntry.split('/'); - if (Number(prefixStr) === 128 && normalizedClientIp === stripIpv6MappedPrefix(netAddr)) { - return true; + const [netAddrRaw, prefixStr] = normalizedEntry.split('/'); + const netAddr = stripIpv6MappedPrefix(netAddrRaw); + const prefix = Number(prefixStr); + const ruleFamily = net.isIP(netAddr); + + if ( + clientFamily === 4 && + ruleFamily === 4 && + Number.isInteger(prefix) && + prefix >= 0 && + prefix <= 32 + ) { + if (prefix === 0) return true; + try { + const bl = new net.BlockList(); + bl.addSubnet(netAddr, prefix, 'ipv4'); + if (bl.check(normalizedClientIp, 'ipv4')) { + return true; + } + } catch { + if (matchesIpv4Cidr(normalizedClientIp, `${netAddr}/${prefix}`)) { + return true; + } + } + } else if ( + clientFamily === 6 && + ruleFamily === 6 && + Number.isInteger(prefix) && + prefix >= 0 && + prefix <= 128 + ) { + if (prefix === 0) return true; + try { + const bl = new net.BlockList(); + bl.addSubnet(netAddr, prefix, 'ipv6'); + if (bl.check(normalizedClientIp, 'ipv6')) { + return true; + } + } catch { + if (prefix === 128 && normalizedClientIp === netAddr) { + return true; + } + } } } else { if (normalizedClientIp === normalizedEntry) { @@ -1128,6 +1171,33 @@ function applyRuntimeSettingsObject(settingsObj) { } } + if (settingsObj.enableWebhookAlerts !== undefined) { + runtimeConfig.enableWebhookAlerts = Boolean(settingsObj.enableWebhookAlerts); + } + + if (settingsObj.webhookUrl !== undefined && settingsObj.webhookUrl !== null) { + runtimeConfig.webhookUrl = String(settingsObj.webhookUrl).trim(); + } + + if (settingsObj.webhookSecret !== undefined && settingsObj.webhookSecret !== null) { + runtimeConfig.webhookSecret = String(settingsObj.webhookSecret); + } else if ( + settingsObj.webhookSecretEnc !== undefined && + settingsObj.webhookSecretEnc !== null + ) { + runtimeConfig.webhookSecret = decryptVaultSecret(String(settingsObj.webhookSecretEnc)); + } + + if ( + settingsObj.webhookCooldownSec !== undefined && + settingsObj.webhookCooldownSec !== null + ) { + const cooldown = Number(settingsObj.webhookCooldownSec); + if (Number.isFinite(cooldown) && cooldown >= 0) { + runtimeConfig.webhookCooldownSec = Math.floor(cooldown); + } + } + let jwtRotated = false; if (settingsObj.jwtSecret !== undefined && settingsObj.jwtSecret !== null) { const candidateSecret = String(settingsObj.jwtSecret).trim(); @@ -1149,6 +1219,179 @@ function applyRuntimeSettingsObject(settingsObj) { return { jwtRotated }; } +const VAULT_PREFIX = 'v1'; +const DEV_DEFAULT_SESSION_SECRET = 'datastack-admin-session-default-secret-32-chars'; +const webhookCooldownMap = new Map(); + +function clearWebhookCooldownCacheForTests() { + webhookCooldownMap.clear(); +} + +function resolveVaultMasterSecret() { + const candidate = (process.env.DATASTACK_ADMIN_SESSION_SECRET || '').trim(); + if (process.env.NODE_ENV === 'production') { + return candidate.length >= 32 ? candidate : null; + } + if (candidate.length > 0) { + return candidate.length >= 32 ? candidate : null; + } + return DEV_DEFAULT_SESSION_SECRET; +} + +function decryptVaultSecret(ciphertext) { + if (!ciphertext || typeof ciphertext !== 'string') return ''; + const trimmed = ciphertext.trim(); + if (!trimmed) return ''; + const parts = trimmed.split(':'); + if (parts.length !== 4 || parts[0] !== VAULT_PREFIX) { + console.warn( + '[DataStack Vault] Ignoring unrecognized stored webhook secret format on Port 4000.', + ); + return ''; + } + const master = resolveVaultMasterSecret(); + if (!master) { + console.warn( + '[DataStack Vault] DATASTACK_ADMIN_SESSION_SECRET (>= 32 chars) unavailable on Port 4000; skipping webhook secret decryption.', + ); + return ''; + } + try { + const derived = crypto.hkdfSync( + 'sha256', + Buffer.from(master, 'utf8'), + Buffer.from('datastack-vault-salt-v1', 'utf8'), + Buffer.from('datastack-aes-256-gcm-v1', 'utf8'), + 32, + ); + const key = Buffer.from(derived); + const iv = Buffer.from(parts[1], 'base64url'); + const authTag = Buffer.from(parts[2], 'base64url'); + const encrypted = Buffer.from(parts[3], 'base64url'); + if (iv.length !== 12 || authTag.length !== 16) { + return ''; + } + const decipher = crypto.createDecipheriv('aes-256-gcm', key, iv, { + authTagLength: 16, + }); + decipher.setAuthTag(authTag); + const decrypted = Buffer.concat([decipher.update(encrypted), decipher.final()]); + return decrypted.toString('utf8'); + } catch { + console.warn( + '[DataStack Vault] Failed to decrypt stored webhook secret on Port 4000. Verify DATASTACK_ADMIN_SESSION_SECRET matches Port 3000.', + ); + return ''; + } +} + +function formatOutboundWebhookBody(_url, payload, timestamp) { + const safeTimestamp = timestamp || payload.timestamp || new Date().toISOString(); + const severity = payload.severity || 'warning'; + const prefix = + severity === 'critical' + ? '[CRITICAL]' + : severity === 'warning' + ? '[WARNING]' + : '[INFO]'; + const title = payload.title || String(payload.event || 'Alert'); + const clientTag = payload.clientId ? ` (client: ${payload.clientId})` : ''; + const summaryLine = `${prefix} DataStack ${payload.event}${clientTag} — ${title}: ${payload.message}`; + + return JSON.stringify({ + instance: 'DataStack', + event: payload.event, + severity, + title, + message: payload.message, + ...(payload.clientId ? { clientId: payload.clientId } : {}), + ...(payload.details ? { details: payload.details } : {}), + timestamp: safeTimestamp, + // Slack & Discord compatible top-level fields (1:1 parity with Port 3000) + text: summaryLine, + content: summaryLine, + }); +} + +async function triggerWebhookAlert(payload, options = {}) { + try { + const enabled = + options.enableWebhookAlerts !== undefined + ? Boolean(options.enableWebhookAlerts) + : Boolean(runtimeConfig.enableWebhookAlerts); + const url = ( + options.webhookUrl !== undefined + ? String(options.webhookUrl) + : String(runtimeConfig.webhookUrl || '') + ).trim(); + + if (!enabled || !url) { + return { dispatched: false, reason: 'disabled_or_empty_url' }; + } + + const cooldownSec = Math.max( + 0, + Number( + options.webhookCooldownSec !== undefined + ? options.webhookCooldownSec + : runtimeConfig.webhookCooldownSec, + ) || 0, + ); + const dedupKey = `${payload.event}:${payload.clientId || 'global'}`; + const now = Date.now(); + const lastFired = webhookCooldownMap.get(dedupKey) || 0; + + if (!options.bypassCooldown && cooldownSec > 0 && now - lastFired < cooldownSec * 1000) { + return { dispatched: false, reason: 'cooldown_active' }; + } + + webhookCooldownMap.set(dedupKey, now); + + const timestamp = payload.timestamp || new Date(now).toISOString(); + const rawBody = formatOutboundWebhookBody(url, payload, timestamp); + const secret = + options.webhookSecret !== undefined + ? String(options.webhookSecret || '') + : String(runtimeConfig.webhookSecret || ''); + + const headers = { + 'Content-Type': 'application/json', + 'User-Agent': 'DataStack-Webhook-Notifier/2.0', + 'X-DataStack-Event': String(payload.event || ''), + 'X-DataStack-Timestamp': timestamp, + }; + + if (secret) { + const signature = crypto + .createHmac('sha256', secret) + .update(rawBody) + .digest('hex'); + headers['X-DataStack-Signature'] = `sha256=${signature}`; + } + + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 3500); + const fetchImpl = options.fetchImpl || global.fetch; + + try { + const res = await fetchImpl(url, { + method: 'POST', + headers, + body: rawBody, + signal: controller.signal, + }); + return { dispatched: true, status: res.status }; + } finally { + clearTimeout(timeout); + } + } catch (err) { + return { + dispatched: false, + reason: err && err.message ? err.message : 'webhook_dispatch_failed', + }; + } +} + async function ensureDatabaseSchema() { const pool = dbPool; if (!pool) return false; @@ -1297,7 +1540,8 @@ async function syncSettingsFromDatabase() { const res = await dbPool.query( `SELECT max_events_per_sec, max_payload_size_kb, batch_flush_threshold, batch_flush_interval_ms, file_rotation_mb, jwt_secret, enable_idempotency, - auto_prune_days, compress_after_days, enable_s3_backup, disk_pause_threshold_pct + auto_prune_days, compress_after_days, enable_s3_backup, disk_pause_threshold_pct, + enable_webhook_alerts, webhook_url, webhook_secret_enc, webhook_cooldown_sec FROM admin_settings WHERE id = 'default'`, ); @@ -1319,6 +1563,10 @@ async function syncSettingsFromDatabase() { row.disk_pause_threshold_pct !== undefined && row.disk_pause_threshold_pct !== null ? Number(row.disk_pause_threshold_pct) : NaN; + const parsedWebhookCooldown = + row.webhook_cooldown_sec !== undefined && row.webhook_cooldown_sec !== null + ? Number(row.webhook_cooldown_sec) + : NaN; applyRuntimeSettingsObject({ maxEventsPerSec: Number(row.max_events_per_sec) || 100, @@ -1334,6 +1582,12 @@ async function syncSettingsFromDatabase() { compressAfterDays: Number.isFinite(parsedCompress) ? Math.max(0, parsedCompress) : 0, enableS3Backup: Boolean(row.enable_s3_backup), diskPauseThresholdPct: Number.isFinite(parsedDiskPct) ? parsedDiskPct : 95, + enableWebhookAlerts: Boolean(row.enable_webhook_alerts), + webhookUrl: row.webhook_url ? String(row.webhook_url).trim() : '', + webhookSecretEnc: row.webhook_secret_enc ? String(row.webhook_secret_enc) : '', + webhookCooldownSec: Number.isFinite(parsedWebhookCooldown) + ? Math.max(0, parsedWebhookCooldown) + : 300, jwtSecret: row.jwt_secret ? String(row.jwt_secret).trim() : undefined, }); return true; @@ -1351,6 +1605,7 @@ const socketHandlersApi = setupSocketHandlers(io, bufferQueue, writer, { getRedisClient, verifyClientInDb, recordClientActivity, + triggerWebhookAlert, onBufferPush: async () => { const redis = getRedisClient(); if (redis) { @@ -1888,4 +2143,8 @@ module.exports = { corsOriginDelegate, isClientIpAllowed, isClientTokenExpired, + triggerWebhookAlert, + clearWebhookCooldownCacheForTests, + decryptVaultSecret, + formatOutboundWebhookBody, }; diff --git a/server/socket/handlers.js b/server/socket/handlers.js index b7779c9..fdf5840 100644 --- a/server/socket/handlers.js +++ b/server/socket/handlers.js @@ -91,6 +91,38 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { ? options.getRedisClient : () => null; + const triggerWebhookAlert = + typeof options.triggerWebhookAlert === "function" + ? options.triggerWebhookAlert + : null; + + const WEBHOOK_REJECTION_CODES = new Set([ + "CLIENT_TOKEN_EXPIRED", + "CLIENT_IP_FORBIDDEN", + "DISK_SPACE_LOW", + "STORAGE_QUOTA_EXCEEDED", + ]); + + function dispatchRejectionWebhook(code, reason, clientId, details) { + if (!triggerWebhookAlert || !WEBHOOK_REJECTION_CODES.has(code)) return; + const severity = code === "DISK_SPACE_LOW" ? "critical" : "warning"; + try { + Promise.resolve( + triggerWebhookAlert({ + event: code, + severity, + clientId, + message: + reason || + `${code}: Ingestion request rejected on Port 4000 Socket.IO.`, + details: details || { transport: "socketio" }, + }), + ).catch(() => undefined); + } catch { + // Never block socket handlers on webhook alert dispatch + } + } + const idempotencyMap = new Map(); const rateLimitMap = new Map(); let lastCleanupAt = Date.now(); @@ -376,7 +408,12 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { clientIp, ); if (!dbCheck.allowed) { - return next(new Error(dbCheck.reason || dbCheck.code || "CLIENT_REVOKED")); + const errCode = dbCheck.code || "CLIENT_REVOKED"; + dispatchRejectionWebhook(errCode, dbCheck.reason, normalizedClientId, { + transport: "socketio_handshake", + clientIp, + }); + return next(new Error(dbCheck.reason || errCode)); } socket.data.clientId = normalizedClientId; @@ -440,10 +477,15 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { clientIp, ); if (!dbCheck.allowed) { + const errCode = dbCheck.code || "CLIENT_REVOKED"; + dispatchRejectionWebhook(errCode, dbCheck.reason, clientId, { + transport: "socketio_sync_data", + clientIp, + }); if (callback) { callback({ success: false, - code: dbCheck.code || "CLIENT_REVOKED", + code: errCode, error: dbCheck.reason || "CLIENT_REVOKED", }); } @@ -459,6 +501,10 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { config.diskPauseThresholdPct ?? 95, ); if (!admission.allowed) { + dispatchRejectionWebhook(admission.code, admission.reason, clientId, { + transport: "socketio_sync_data", + clientIp, + }); if (callback) { callback({ success: false, @@ -559,12 +605,17 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { clientIp, ); if (!dbCheck.allowed) { + const errCode = dbCheck.code || "CLIENT_REVOKED"; + dispatchRejectionWebhook(errCode, dbCheck.reason, clientId, { + transport: "socketio_batch_sync", + clientIp, + }); if (callback) { callback({ success: false, processed: 0, rejected: batch.length, - code: dbCheck.code || "CLIENT_REVOKED", + code: errCode, error: dbCheck.reason || "CLIENT_REVOKED", }); } @@ -580,6 +631,10 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { config.diskPauseThresholdPct ?? 95, ); if (!admission.allowed) { + dispatchRejectionWebhook(admission.code, admission.reason, clientId, { + transport: "socketio_batch_sync", + clientIp, + }); if (callback) { callback({ success: false, @@ -729,11 +784,16 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { clientIp, ); if (!dbCheck.allowed) { + const errCode = dbCheck.code || "CLIENT_REVOKED"; + dispatchRejectionWebhook(errCode, dbCheck.reason, clientId, { + transport: "socketio_ping_verify", + clientIp, + }); if (callback) { callback({ success: false, status: "REVOKED", - code: dbCheck.code || "CLIENT_REVOKED", + code: errCode, error: dbCheck.reason || "CLIENT_REVOKED", }); } diff --git a/server/workers/csvWriter.js b/server/workers/csvWriter.js index 958277f..97600fd 100644 --- a/server/workers/csvWriter.js +++ b/server/workers/csvWriter.js @@ -44,8 +44,8 @@ class CsvWriterWorker { extractDateFromFilename(filename) { return String(filename || '') - .replace(/\.csv(?:\.gz)?$/, '') - .split('-part')[0]; + .replace(/\.csv(?:\.gz)?$/i, '') + .replace(/(?:-(?:part|gz)\d+)+$/i, ''); } clearQuotaCaches(clientId) { diff --git a/tests/phase05_sdk_test.py b/tests/phase05_sdk_test.py index 6c5aa47..0c02e27 100644 --- a/tests/phase05_sdk_test.py +++ b/tests/phase05_sdk_test.py @@ -602,6 +602,64 @@ class Phase05PythonSdkTests(unittest.TestCase): f"{py_file.name}:{node.lineno} uses Python 3.9+ PEP 585 generic `{node.value.id}[...]`" ) + def test_v2_audit_phase02_bug09_fatal_auth_codes_expired_and_ip_forbidden( + self, + ) -> None: + self.assertTrue( + DataStackClient._is_non_retryable_auth_error( + "CLIENT_TOKEN_EXPIRED: Client token has expired" + ) + ) + self.assertTrue( + DataStackClient._is_non_retryable_auth_error( + "CLIENT_IP_FORBIDDEN: IP 198.51.100.25 is not permitted" + ) + ) + self.assertFalse( + DataStackClient._is_non_retryable_auth_error("Connection reset by peer") + ) + + tok = _make_jwt({"clientId": "cli_audit09"}) + client = DataStack( + server_url="http://127.0.0.1:4000", + api_url="http://127.0.0.1:3000", + token=tok, + auto_connect=False, + enable_http_fallback=True, + offline_spool_enabled=True, + request_timeout_sec=0.5, + ) + + class FakeFailingAuthSocket: + def __init__(self, error_msg: str) -> None: + self.connected = False + self.error_msg = error_msg + + def connect(self, *args: Any, **kwargs: Any) -> None: + raise RuntimeError(self.error_msg) + + def disconnect(self) -> None: + self.connected = False + + # 1. CLIENT_TOKEN_EXPIRED on WebSocket handshake must immediately raise DataStackAuthError with code="CLIENT_TOKEN_EXPIRED" + client.sio = FakeFailingAuthSocket( + "CLIENT_TOKEN_EXPIRED: Client token has expired" + ) + with self.assertRaises(DataStackAuthError) as ctx_exp: + client._connect_and_verify_websocket(wait_timeout=0.2) + self.assertEqual(ctx_exp.exception.code, "CLIENT_TOKEN_EXPIRED") + + # 2. CLIENT_IP_FORBIDDEN on WebSocket handshake must immediately raise DataStackAuthError with code="CLIENT_IP_FORBIDDEN" + client.sio = FakeFailingAuthSocket( + "CLIENT_IP_FORBIDDEN: IP is not in allowlist" + ) + with self.assertRaises(DataStackAuthError) as ctx_ip: + client._connect_and_verify_websocket(wait_timeout=0.2) + self.assertEqual(ctx_ip.exception.code, "CLIENT_IP_FORBIDDEN") + + client.disconnect() + if __name__ == "__main__": unittest.main(verbosity=2) + diff --git a/tests/r3-phase01-security-vault-webhooks.test.ts b/tests/r3-phase01-security-vault-webhooks.test.ts new file mode 100644 index 0000000..f514d52 --- /dev/null +++ b/tests/r3-phase01-security-vault-webhooks.test.ts @@ -0,0 +1,491 @@ +import assert from "node:assert/strict"; +import crypto from "node:crypto"; +import fs from "node:fs"; +import { createRequire } from "node:module"; +import path from "node:path"; +import test from "node:test"; +import jwt from "jsonwebtoken"; +import { NextRequest } from "next/server"; + +process.env.DATASTACK_DISABLE_LISTEN = "1"; + +import { POST as loginPost } from "../app/api/auth/login/route"; +import { POST as eventsPost } from "../app/api/events/route"; +import { POST as webhookTestPost } from "../app/api/settings/webhook-test/route"; +import { POST as s3TestPost } from "../app/api/storage/s3-test/route"; +import { SettingsManager } from "../core/adminSettings/settingsManager"; +import { + clearWebhookCooldownCacheForTests, + formatOutboundWebhookBody as formatPort3000WebhookBody, +} from "../core/alerts/webhookNotifier"; +import { + createAdminSessionToken, + getAdminSessionSecret, +} from "../core/auth/adminSession"; +import { + clearAdminLoginAttempts, + createAdminTeamMemberAsync, + createInitialAdmin, + deleteAdminTeamMemberAsync, + listAdminTeamMembersAsync, + resetAdminStateForTests, +} from "../core/auth/adminUsers"; +import { AuthService } from "../core/auth/jwt"; +import { DataBufferEngine } from "../core/data/buffer"; +import { SDKManager } from "../core/sdk/sdkManager"; +import { encryptSecret } from "../core/Security/secretVault"; + +const require = createRequire(import.meta.url); +const ingestionServer = require("../server/index.js") as { + decryptVaultSecret: (ciphertext: string) => string; + formatOutboundWebhookBody: ( + url: string, + payload: Record, + timestamp?: string, + ) => string; + triggerWebhookAlert: ( + payload: Record, + options?: Record, + ) => Promise<{ dispatched: boolean; status?: number; reason?: string }>; + applyRuntimeSettingsObject: (settingsObj: Record) => { + jwtRotated: boolean; + }; + runtimeConfig: Record; + clearWebhookCooldownCacheForTests: () => void; + stopTimers: () => void; +}; + +test.after(async () => { + ingestionServer.stopTimers(); + SDKManager.stopActivityFlushTimerForTests(); + await DataBufferEngine.shutdown(); + resetAdminStateForTests(); +}); + +test("R3-Phase-01 [BUG-01]: Port 3000 encryptSecret() decrypts 1:1 on Port 4000 & Docker Compose passes DATASTACK_ADMIN_SESSION_SECRET", async () => { + const prevSecret = process.env.DATASTACK_ADMIN_SESSION_SECRET; + const prevNodeEnv = process.env.NODE_ENV; + const testMaster = "r3-phase01-vault-master-secret-at-least-32-chars!!"; + try { + process.env.DATASTACK_ADMIN_SESSION_SECRET = testMaster; + const plaintext = "whsec_live_prod_hmac_99887766"; + const encrypted = encryptSecret(plaintext); + assert.ok(encrypted.startsWith("v1:"), "Port 3000 vault must emit v1: prefix"); + + // 1. Cross-runtime decryption matches 1:1 + assert.equal(ingestionServer.decryptVaultSecret(encrypted), plaintext); + + // 2. applyRuntimeSettingsObject hydrates plaintext webhookSecret from DB webhookSecretEnc + ingestionServer.applyRuntimeSettingsObject({ webhookSecretEnc: encrypted }); + assert.equal(ingestionServer.runtimeConfig.webhookSecret, plaintext); + + // 3. Wrong master secret fails closed to "" (never leaks raw ciphertext as HMAC key) + process.env.DATASTACK_ADMIN_SESSION_SECRET = + "wrong-master-secret-that-is-32-chars-long-000"; + assert.equal(ingestionServer.decryptVaultSecret(encrypted), ""); + + // 4. Legacy / unrecognized ciphertext format fails closed to "" (never returned verbatim) + assert.equal( + ingestionServer.decryptVaultSecret("enc:v1:invalid:payload"), + "", + ); + assert.equal( + ingestionServer.decryptVaultSecret("plaintext-not-vault-format"), + "", + ); + + // 5. Production mode without DATASTACK_ADMIN_SESSION_SECRET fails closed to "" + (process.env as Record).NODE_ENV = "production"; + delete process.env.DATASTACK_ADMIN_SESSION_SECRET; + assert.equal(ingestionServer.decryptVaultSecret(encrypted), ""); + + // 6. Docker Compose passes DATASTACK_ADMIN_SESSION_SECRET to ingestion container + const composeContent = fs.readFileSync( + path.resolve(process.cwd(), "docker/docker-compose.yml"), + "utf8", + ); + const ingestionSection = composeContent.slice( + composeContent.indexOf(" ingestion:"), + composeContent.indexOf(" proxy:"), + ); + assert.match( + ingestionSection, + /DATASTACK_ADMIN_SESSION_SECRET:\s*"\$\{SERVICE_REALBASE64_64_ADMINSESSION:/, + "docker/docker-compose.yml ingestion service must receive DATASTACK_ADMIN_SESSION_SECRET", + ); + } finally { + if (prevSecret === undefined) { + delete process.env.DATASTACK_ADMIN_SESSION_SECRET; + } else { + process.env.DATASTACK_ADMIN_SESSION_SECRET = prevSecret; + } + (process.env as Record).NODE_ENV = prevNodeEnv; + } +}); + +test("R3-Phase-01 [BUG-03]: Client SDK JWTs omit hardcoded 30d exp so Never-Expires and >30d clients stay valid", async () => { + const clientId = `r3_jwt_longlived_${Date.now()}`; + const farFutureIso = new Date(Date.now() + 180 * 86_400_000).toISOString(); + + // 1. Explicit expiresIn still works when caller requests bounded JWT + const boundedToken = AuthService.generateClientToken( + { clientId: "bounded_test" }, + "1h", + ); + const boundedDecoded = jwt.decode(boundedToken) as jwt.JwtPayload; + assert.equal(typeof boundedDecoded.exp, "number"); + + // 2. Registered client with expiresAt = null ("Never expires") has NO hardcoded 30d exp + const neverClient = await SDKManager.registerClientAsync({ + clientId, + name: "Long-Lived Telemetry Client", + rateLimit: 100, + expiresAt: null, + }); + try { + const decodedNever = jwt.decode(neverClient.token) as jwt.JwtPayload; + assert.equal( + decodedNever.exp, + undefined, + "Never-expires client token must not embed a hardcoded 30d JWT exp claim", + ); + assert.equal(AuthService.verifyClientToken(neverClient.token).valid, true); + + // 3. Updating expiresAt to >30d future keeps the existing token valid without rotation + const updated = await SDKManager.updateClientAsync(clientId, { + expiresAt: farFutureIso, + }); + assert.ok(updated); + assert.equal(updated.expiresAt, farFutureIso); + assert.equal(updated.token, neverClient.token); + + // 4. Regenerated token also omits hardcoded 30d exp + const rotated = await SDKManager.regenerateTokenAsync(clientId); + assert.ok(rotated); + const decodedRotated = jwt.decode(rotated.token) as jwt.JwtPayload; + assert.equal(decodedRotated.exp, undefined); + } finally { + await SDKManager.deleteClientAsync(clientId); + } +}); + +test("R3-Phase-01 [BUG-04 & BUG-15]: Viewer blocked (403) on POST /api/events and Operator blocked (403) on POST /api/storage/s3-test", async () => { + resetAdminStateForTests(); + await createInitialAdmin("r3-owner@datastack.dev", "OwnerPassword123!"); + const sessionSecret = getAdminSessionSecret(); + const suffix = Date.now(); + const viewerEmail = `r3viewer_${suffix}@datastack.dev`; + const operatorEmail = `r3operator_${suffix}@datastack.dev`; + const clientId = `r3_rbac_client_${suffix}`; + + const viewer = await createAdminTeamMemberAsync({ + email: viewerEmail, + password: "ViewerPassword123!", + role: "viewer", + displayName: "R3 Viewer", + }); + const operator = await createAdminTeamMemberAsync({ + email: operatorEmail, + password: "OperatorPassword123!", + role: "operator", + displayName: "R3 Operator", + }); + await SDKManager.registerClientAsync({ + clientId, + name: "RBAC Test Client", + rateLimit: 50, + }); + + try { + const viewerToken = await createAdminSessionToken( + viewer.email, + sessionSecret, + Date.now() + 3600_000, + viewer.tokenVersion, + viewer.role, + viewer.id, + ); + const operatorToken = await createAdminSessionToken( + operator.email, + sessionSecret, + Date.now() + 3600_000, + operator.tokenVersion, + operator.role, + operator.id, + ); + + // BUG-04: Viewer CANNOT ingest events via POST /api/events + const viewerIngestRes = await eventsPost( + new NextRequest("http://localhost:3000/api/events", { + method: "POST", + headers: { + "content-type": "application/json", + cookie: `datastack_admin_session=${viewerToken}`, + }, + body: JSON.stringify({ clientId, data: { metric: 1 } }), + }), + ); + assert.equal(viewerIngestRes.status, 403); + assert.equal( + (await viewerIngestRes.json()).code, + "INSUFFICIENT_ROLE_PERMISSION", + ); + + // Operator CAN ingest events via POST /api/events + const operatorIngestRes = await eventsPost( + new NextRequest("http://localhost:3000/api/events", { + method: "POST", + headers: { + "content-type": "application/json", + cookie: `datastack_admin_session=${operatorToken}`, + }, + body: JSON.stringify({ clientId, data: { metric: 2 } }), + }), + ); + assert.equal(operatorIngestRes.status, 200); + + // BUG-15: Operator CANNOT call POST /api/storage/s3-test (Owner-only) + const operatorS3TestRes = await s3TestPost( + new NextRequest("http://localhost:3000/api/storage/s3-test", { + method: "POST", + headers: { + "content-type": "application/json", + cookie: `datastack_admin_session=${operatorToken}`, + }, + body: JSON.stringify({ s3Endpoint: "http://127.0.0.1:9999" }), + }), + ); + assert.equal(operatorS3TestRes.status, 403); + assert.equal( + (await operatorS3TestRes.json()).code, + "INSUFFICIENT_ROLE_PERMISSION", + ); + } finally { + await SDKManager.deleteClientAsync(clientId); + await deleteAdminTeamMemberAsync(viewer.id); + await deleteAdminTeamMemberAsync(operator.id); + } +}); + +test("R3-Phase-01 [BUG-06]: Locked webhook secret does not break unrelated settings saves or draft/clear webhook tests", async () => { + resetAdminStateForTests(); + await createInitialAdmin("r3-owner2@datastack.dev", "OwnerPassword123!"); + const prevSecret = process.env.DATASTACK_ADMIN_SESSION_SECRET; + const origFetch = global.fetch; + const members = await listAdminTeamMembersAsync(); + const owner = members.find((m) => m.role === "owner") || members[0]; + assert.ok(owner, "Bootstrap owner member must exist for test"); + + try { + // 1. Encrypt webhook secret under key A + process.env.DATASTACK_ADMIN_SESSION_SECRET = + "key-alpha-32-characters-long-for-rotation-test"; + await SettingsManager.updateSettingsAsync({ + enableWebhookAlerts: true, + webhookUrl: "https://hooks.example.com/r3-test", + webhookSecret: "old_secret_under_key_alpha", + }); + + // 2. Rotate DATASTACK_ADMIN_SESSION_SECRET to key B (making stored webhookSecretEnc locked) + process.env.DATASTACK_ADMIN_SESSION_SECRET = + "key-beta-32-characters-long-for-rotation-test!"; + + const ownerToken = await createAdminSessionToken( + owner.email, + getAdminSessionSecret(), + Date.now() + 3600_000, + owner.tokenVersion, + owner.role, + owner.id, + ); + + let capturedTestHeaders: Record = {}; + global.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.includes("/internal/settings-sync")) { + return new Response(JSON.stringify({ ok: true }), { status: 200 }); + } + if (url.includes("hooks.example.com")) { + capturedTestHeaders = (init?.headers as Record) || {}; + return new Response("ok", { status: 200 }); + } + return new Response("not found", { status: 404 }); + }) as typeof fetch; + + // Saving an unrelated setting MUST NOT throw SecretVaultError + const updated = await SettingsManager.updateSettingsAsync({ + maxEventsPerSec: 150, + }); + assert.equal(updated.maxEventsPerSec, 150); + + // Testing a brand-new draft webhookSecret MUST succeed without decrypting the old locked secret + const draftRes = await webhookTestPost( + new NextRequest("http://localhost:3000/api/settings/webhook-test", { + method: "POST", + headers: { + "content-type": "application/json", + cookie: `datastack_admin_session=${ownerToken}`, + }, + body: JSON.stringify({ + webhookUrl: "https://hooks.example.com/r3-test", + webhookSecret: "brand_new_draft_secret_1234", + }), + }), + ); + assert.equal(draftRes.status, 200); + assert.ok( + capturedTestHeaders["X-DataStack-Signature"]?.startsWith("sha256="), + "Draft secret must sign outbound test webhook", + ); + + // Testing with clearWebhookSecret: true MUST also succeed without touching locked secret + capturedTestHeaders = {}; + const clearRes = await webhookTestPost( + new NextRequest("http://localhost:3000/api/settings/webhook-test", { + method: "POST", + headers: { + "content-type": "application/json", + cookie: `datastack_admin_session=${ownerToken}`, + }, + body: JSON.stringify({ + webhookUrl: "https://hooks.example.com/r3-test", + clearWebhookSecret: true, + }), + }), + ); + assert.equal(clearRes.status, 200); + assert.equal(capturedTestHeaders["X-DataStack-Signature"], undefined); + } finally { + global.fetch = origFetch; + if (prevSecret === undefined) { + delete process.env.DATASTACK_ADMIN_SESSION_SECRET; + } else { + process.env.DATASTACK_ADMIN_SESSION_SECRET = prevSecret; + } + await SettingsManager.updateSettingsAsync({ + enableWebhookAlerts: false, + webhookUrl: "", + clearWebhookSecret: true, + }); + } +}); + +test("R3-Phase-01 [BUG-10]: Brute-force 429 admin login lockout dispatches ADMIN_LOGIN_LOCKOUT webhook alert", async () => { + resetAdminStateForTests(); + await createInitialAdmin("r3-owner3@datastack.dev", "OwnerPassword123!"); + const origFetch = global.fetch; + const email = `lockout_probe_${Date.now()}@datastack.dev`; + const testIp = "198.51.100.42"; + const dispatchedBodies: Array> = []; + + try { + clearWebhookCooldownCacheForTests(); + await SettingsManager.updateSettingsAsync({ + enableWebhookAlerts: true, + webhookUrl: "https://hooks.example.com/lockout-alert", + webhookSecret: "lockout_hmac_secret_123456", + webhookCooldownSec: 0, + }); + + global.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.includes("hooks.example.com/lockout-alert")) { + const parsed = JSON.parse(String(init?.body || "{}")) as Record< + string, + unknown + >; + dispatchedBodies.push(parsed); + return new Response("ok", { status: 200 }); + } + return new Response("ok", { status: 200 }); + }) as typeof fetch; + + let lastStatus = 0; + for (let i = 0; i < 6; i += 1) { + const res = await loginPost( + new NextRequest("http://localhost:3000/api/auth/login", { + method: "POST", + headers: { + "content-type": "application/json", + "x-real-ip": testIp, + }, + body: JSON.stringify({ email, password: "definitely-wrong-password" }), + }), + ); + lastStatus = res.status; + } + + assert.equal(lastStatus, 429); + await new Promise((resolve) => setTimeout(resolve, 60)); + + assert.ok( + dispatchedBodies.length >= 1, + "Expected ADMIN_LOGIN_LOCKOUT webhook alert to be dispatched on 429", + ); + assert.equal(dispatchedBodies[0].event, "ADMIN_LOGIN_LOCKOUT"); + assert.equal(dispatchedBodies[0].severity, "warning"); + assert.equal(dispatchedBodies[0].instance, "DataStack"); + } finally { + global.fetch = origFetch; + await clearAdminLoginAttempts(email, getAdminSessionSecret(), testIp); + await SettingsManager.updateSettingsAsync({ + enableWebhookAlerts: false, + webhookUrl: "", + clearWebhookSecret: true, + }); + } +}); + +test("R3-Phase-01 [BUG-13]: Port 4000 outbound webhook payload schema matches Port 3000 1:1 (including custom Discord proxies)", async () => { + const fixedTs = "2026-11-20T12:00:00.000Z"; + const payload = { + event: "DISK_SPACE_LOW" as const, + severity: "critical" as const, + title: "VPS Disk Threshold Reached", + message: "Disk usage reached 96%.", + clientId: "iot_edge_01", + details: { usedPct: 96 }, + timestamp: fixedTs, + }; + + const port3000Json = JSON.parse(formatPort3000WebhookBody(payload)); + const port4000Json = JSON.parse( + ingestionServer.formatOutboundWebhookBody( + "https://discord-proxy.internal.company/webhooks/123", + payload, + fixedTs, + ), + ); + + assert.deepEqual(port4000Json, port3000Json); + assert.equal(port4000Json.instance, "DataStack"); + assert.equal(port4000Json.title, "VPS Disk Threshold Reached"); + assert.equal(typeof port4000Json.text, "string"); + assert.equal(typeof port4000Json.content, "string"); + + // Also verify end-to-end Port 4000 HMAC signature uses decrypted secret cleanly + ingestionServer.clearWebhookCooldownCacheForTests(); + let capturedBody = ""; + let capturedSig = ""; + const secret = "hmac_parity_secret_2026"; + await ingestionServer.triggerWebhookAlert(payload, { + enableWebhookAlerts: true, + webhookUrl: "https://custom-webhook.example.com/ingest", + webhookSecret: secret, + bypassCooldown: true, + fetchImpl: async ( + _url: string, + init: { body: string; headers: Record }, + ) => { + capturedBody = init.body; + capturedSig = init.headers["X-DataStack-Signature"]; + return { ok: true, status: 200 }; + }, + }); + + const expectedSig = `sha256=${crypto + .createHmac("sha256", secret) + .update(capturedBody, "utf8") + .digest("hex")}`; + assert.equal(capturedSig, expectedSig); +}); diff --git a/tests/r3-phase02-storage-scheduler-gzn.test.ts b/tests/r3-phase02-storage-scheduler-gzn.test.ts new file mode 100644 index 0000000..60b908c --- /dev/null +++ b/tests/r3-phase02-storage-scheduler-gzn.test.ts @@ -0,0 +1,347 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import http from "node:http"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import zlib from "node:zlib"; +import { NextRequest } from "next/server"; + +import { GET as dataGet } from "../app/api/data/route"; +import { GET as exportGet } from "../app/api/export/route"; +import { SettingsManager } from "../core/adminSettings/settingsManager"; +import { + createAdminSessionToken, + getAdminSessionSecret, +} from "../core/auth/adminSession"; +import { + createInitialAdmin, + resetAdminStateForTests, +} from "../core/auth/adminUsers"; +import { DataBufferEngine } from "../core/data/buffer"; +import { SDKManager } from "../core/sdk/sdkManager"; +import { CSVStorageManager } from "../core/Storage/csvStorage"; + +const { CsvWriterWorker } = require("../server/workers/csvWriter.js"); + +const ORIGINAL_STORAGE_DIR = CSVStorageManager.baseStorageDir; +const ORIGINAL_DB_URL = process.env.DATABASE_URL; +const ORIGINAL_SESSION_SECRET = process.env.DATASTACK_ADMIN_SESSION_SECRET; +const ORIGINAL_JWT_SECRET = process.env.DATASTACK_JWT_SECRET; + +function makeTempDir(prefix: string): string { + return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); +} + +function writeGzipCsv(filePath: string, content: string): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, zlib.gzipSync(Buffer.from(content, "utf8"))); +} + +async function createOwnerCookie(): Promise { + delete process.env.DATABASE_URL; + const sessionSecret = getAdminSessionSecret(); + resetAdminStateForTests(); + const ownerEmail = "owner-r3p2@datastack.local"; + await createInitialAdmin(ownerEmail, "StrongPassword123!"); + const token = await createAdminSessionToken( + ownerEmail, + sessionSecret, + Date.now() + 3600_000, + 1, + "owner", + "owner-1", + ); + return `datastack_admin_session=${token}`; +} + +test.after(async () => { + SettingsManager.stopMaintenanceSchedulerForTests(); + SDKManager.stopActivityFlushTimerForTests(); + await DataBufferEngine.shutdown(); + CSVStorageManager.baseStorageDir = ORIGINAL_STORAGE_DIR; + CSVStorageManager.clearRowCountCache(); + CSVStorageManager.clearQuotaCaches(); + if (ORIGINAL_DB_URL === undefined) delete process.env.DATABASE_URL; + else process.env.DATABASE_URL = ORIGINAL_DB_URL; + if (ORIGINAL_SESSION_SECRET === undefined) { + delete process.env.DATASTACK_ADMIN_SESSION_SECRET; + } else { + process.env.DATASTACK_ADMIN_SESSION_SECRET = ORIGINAL_SESSION_SECRET; + } + if (ORIGINAL_JWT_SECRET === undefined) delete process.env.DATASTACK_JWT_SECRET; + else process.env.DATASTACK_JWT_SECRET = ORIGINAL_JWT_SECRET; + resetAdminStateForTests(); +}); + +test("R3-Phase-02 BUG-09: Port 3000 and Port 4000 extractDateFromFilename parse -gzN and -partN-gzN identically", () => { + const tmpDir = makeTempDir("datastack-r3p2-date-"); + const worker = new CsvWriterWorker(tmpDir); + try { + const cases: Array<[string, string]> = [ + ["2026-10-01.csv", "2026-10-01"], + ["2026-10-01.csv.gz", "2026-10-01"], + ["2026-10-01-part2.csv", "2026-10-01"], + ["2026-10-01-part12.csv.gz", "2026-10-01"], + ["2026-10-01-gz2.csv.gz", "2026-10-01"], + ["2026-10-01-part2-gz3.csv.gz", "2026-10-01"], + ["2026-10-01-gz2-part3.csv.gz", "2026-10-01"], + ["not-a-date.csv.gz", "not-a-date"], + ]; + + for (const [filename, expected] of cases) { + assert.equal( + CSVStorageManager.extractDateFromFilename(filename), + expected, + `Port 3000 mismatch for ${filename}`, + ); + assert.equal( + worker.extractDateFromFilename(filename), + expected, + `Port 4000 mismatch for ${filename}`, + ); + } + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } +}); + +test("R3-Phase-02 BUG-09: -gzN collision files work in /api/data (to filter + fallback timestamp), /api/export (to filter), and Port 3000/4000 pruning", async () => { + const tmpDir = makeTempDir("datastack-r3p2-gzn-"); + const cookie = await createOwnerCookie(); + SettingsManager.updateSettings({ + autoPruneDays: 0, + compressAfterDays: 0, + enableS3Backup: false, + }); + CSVStorageManager.baseStorageDir = tmpDir; + CSVStorageManager.clearRowCountCache(); + CSVStorageManager.clearQuotaCaches(); + + try { + const clientDir = path.join(tmpDir, "gzn_client"); + const targetDate = "2025-02-15"; + const gz2Path = path.join(clientDir, `${targetDate}-gz2.csv.gz`); + const partGzPath = path.join(clientDir, `${targetDate}-part2-gz3.csv.gz`); + + // Row in -gz2 intentionally omits timestamp to verify valid ISO fallback (`2025-02-15T00:00:00.000Z`) + writeGzipCsv( + gz2Path, + `clientId,eventId,timestamp,payload\ngzn_client,evt_gz2_1,,"{""src"":""gz2""}"\n`, + ); + writeGzipCsv( + partGzPath, + `clientId,eventId,timestamp,payload\ngzn_client,evt_part_gz3_1,${targetDate}T12:30:00.000Z,"{""src"":""part-gz3""}"\n`, + ); + + // 1. /api/data with exact to=2025-02-15 MUST return BOTH events with valid ISO timestamps + const dataRes = await dataGet( + new NextRequest( + `http://localhost:3000/api/data?clientId=gzn_client&from=${targetDate}&to=${targetDate}`, + { headers: { cookie } }, + ), + ); + assert.equal(dataRes.status, 200); + const dataJson = (await dataRes.json()) as { + total: number; + events: Array<{ eventId: string; timestamp: string }>; + }; + assert.equal(dataJson.total, 2); + const gz2Evt = dataJson.events.find((e) => e.eventId === "evt_gz2_1"); + assert.ok(gz2Evt, "evt_gz2_1 must not be excluded by to= filter"); + assert.equal(gz2Evt.timestamp, `${targetDate}T00:00:00.000Z`); + assert.ok( + dataJson.events.some((e) => e.eventId === "evt_part_gz3_1"), + "evt_part_gz3_1 must not be excluded by to= filter", + ); + + // 2. /api/export with exact to=2025-02-15 MUST include BOTH -gzN files + const exportRes = await exportGet( + new NextRequest( + `http://localhost:3000/api/export?clientId=gzn_client&from=${targetDate}&to=${targetDate}&format=json&flush=false`, + { headers: { cookie } }, + ), + ); + assert.equal(exportRes.status, 200); + const exported = JSON.parse(await exportRes.text()) as Array<{ + eventId: string; + }>; + assert.equal(exported.length, 2); + assert.ok(exported.some((e) => e.eventId === "evt_gz2_1")); + assert.ok(exported.some((e) => e.eventId === "evt_part_gz3_1")); + + // 3. Port 3000 pruneExpiredPartitionsAsync deletes -gzN collision files older than cutoff + const pruned3000 = await CSVStorageManager.pruneExpiredPartitionsAsync(7); + assert.equal(pruned3000, 2); + assert.equal(fs.existsSync(gz2Path), false); + assert.equal(fs.existsSync(partGzPath), false); + + // 4. Port 4000 CsvWriterWorker.pruneExpiredPartitionsAsync also deletes -gzN collision files + writeGzipCsv( + gz2Path, + `clientId,eventId,timestamp,payload\ngzn_client,evt_gz2_2,${targetDate}T00:00:00.000Z,"{}"\n`, + ); + const worker = new CsvWriterWorker(tmpDir); + const pruned4000 = await worker.pruneExpiredPartitionsAsync(7); + assert.equal(pruned4000, 1); + assert.equal(fs.existsSync(gz2Path), false); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } +}); + +test("R3-Phase-02 BUG-05: Singleton periodic scheduler automatically archives expired files (including -gzN) to S3 and prunes locally, with overlap and unhydrated-DB guards", async () => { + const tmpDir = makeTempDir("datastack-r3p2-s3sched-"); + delete process.env.DATABASE_URL; + process.env.DATASTACK_ADMIN_SESSION_SECRET = + "r3-phase02-test-session-secret-0123456789abcdef"; + CSVStorageManager.baseStorageDir = tmpDir; + CSVStorageManager.clearRowCountCache(); + CSVStorageManager.clearQuotaCaches(); + SettingsManager.stopMaintenanceSchedulerForTests(); + + const uploadedPaths: string[] = []; + let failS3 = false; + + const mockS3 = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (chunk: Buffer) => chunks.push(chunk)); + req.on("end", () => { + if (failS3) { + res.statusCode = 500; + res.end("S3 temporary error"); + return; + } + if (req.method === "PUT") { + uploadedPaths.push(req.url || ""); + res.statusCode = 200; + res.end(); + return; + } + res.statusCode = 200; + res.end(); + }); + }); + + await new Promise((resolve) => mockS3.listen(0, "127.0.0.1", resolve)); + const addr = mockS3.address(); + assert.ok(addr && typeof addr === "object"); + + try { + // Configure S3 backup + autoPruneDays (cooldown will be reset so scheduled timer runs) + await SettingsManager.updateSettingsAsync({ + enableS3Backup: true, + autoPruneDays: 7, + compressAfterDays: 0, + s3Endpoint: `http://127.0.0.1:${addr.port}`, + s3Region: "auto", + s3Bucket: "sched-bucket", + s3Prefix: "auto-archive/", + s3AccessKeyId: "AKIA_SCHED_TEST", + s3SecretAccessKey: "super_secret_sched_key_9999", + }); + + assert.equal( + SettingsManager.isMaintenanceSchedulerActive(), + true, + "Scheduler must auto-bootstrap on settings access/update", + ); + + // Confirm singleton guard returns false when called again with no interval change + assert.equal(SettingsManager.ensureMaintenanceScheduler(), false); + + // Plant expired files AFTER initial save so ONLY the scheduled timer can archive+prune them + const clientDir = path.join(tmpDir, "sched_client"); + fs.mkdirSync(clientDir, { recursive: true }); + const oldCsv = path.join(clientDir, "2024-11-01.csv"); + const oldGz2 = path.join(clientDir, "2024-11-01-gz2.csv.gz"); + const todayDate = new Date().toISOString().slice(0, 10); + const todayCsv = path.join(clientDir, `${todayDate}.csv`); + + fs.writeFileSync( + oldCsv, + `clientId,eventId,timestamp,payload\nsched_client,evt_old_csv,2024-11-01T00:00:00.000Z,"{}"\n`, + ); + writeGzipCsv( + oldGz2, + `clientId,eventId,timestamp,payload\nsched_client,evt_old_gz2,2024-11-01T01:00:00.000Z,"{}"\n`, + ); + fs.writeFileSync( + todayCsv, + `clientId,eventId,timestamp,payload\nsched_client,evt_today,${todayDate}T00:00:00.000Z,"{}"\n`, + ); + + // Expire prune cooldown and start a fast 30ms test interval to verify real timer firing + const globals = globalThis as typeof globalThis & { + __dataStackLastPruneMeta?: { days: number; ranAtMs: number }; + __dataStackMaintenanceRunning?: boolean; + __dataStackSettingsHydrated?: boolean; + }; + globals.__dataStackLastPruneMeta = { + days: 7, + ranAtMs: Date.now() - 2 * 60 * 60 * 1000, + }; + SettingsManager.ensureMaintenanceScheduler(30); + + // Wait up to 1.5s for scheduled timer tick to upload to S3 and prune both expired files + const deadline = Date.now() + 1500; + while ( + Date.now() < deadline && + (fs.existsSync(oldCsv) || fs.existsSync(oldGz2)) + ) { + await new Promise((r) => setTimeout(r, 25)); + } + + SettingsManager.stopMaintenanceSchedulerForTests(); + + assert.equal(fs.existsSync(oldCsv), false, "Expired .csv must be pruned after S3 upload"); + assert.equal(fs.existsSync(oldGz2), false, "Expired -gz2.csv.gz must be pruned after S3 upload"); + assert.equal(fs.existsSync(todayCsv), true, "Today's partition must remain intact"); + assert.ok( + uploadedPaths.some((u) => u.includes("/sched-bucket/auto-archive/sched_client/2024-11-01.csv")), + `Expected 2024-11-01.csv S3 upload, got: ${uploadedPaths.join(", ")}`, + ); + assert.ok( + uploadedPaths.some((u) => u.includes("/sched-bucket/auto-archive/sched_client/2024-11-01-gz2.csv.gz")), + `Expected 2024-11-01-gz2.csv.gz S3 upload, got: ${uploadedPaths.join(", ")}`, + ); + + // Verify fail-safe when S3 returns 500: local expired file MUST NOT be deleted + failS3 = true; + const failCsv = path.join(clientDir, "2024-10-15.csv"); + fs.writeFileSync( + failCsv, + `clientId,eventId,timestamp,payload\nsched_client,evt_fail_s3,2024-10-15T00:00:00.000Z,"{}"\n`, + ); + await SettingsManager.runScheduledMaintenanceTick({ forcePrune: true }); + assert.equal( + fs.existsSync(failCsv), + true, + "Fail-safe: local file must NEVER be deleted when S3 upload fails", + ); + + // Verify overlap guard blocks concurrent execution + globals.__dataStackMaintenanceRunning = true; + const overlapResult = await SettingsManager.runScheduledMaintenanceTick({ + forcePrune: true, + }); + globals.__dataStackMaintenanceRunning = false; + assert.equal(overlapResult.ran, false); + assert.equal(overlapResult.skippedReason, "ALREADY_RUNNING"); + + // Verify unhydrated-DB guard skips maintenance when DATABASE_URL is configured but unhydrated + process.env.DATABASE_URL = "postgresql://user:pass@127.0.0.1:1/unreachable"; + globals.__dataStackSettingsHydrated = false; + const unhydratedResult = await SettingsManager.runScheduledMaintenanceTick({ + forcePrune: true, + skipDbSync: true, + }); + delete process.env.DATABASE_URL; + assert.equal(unhydratedResult.ran, false); + assert.equal(unhydratedResult.skippedReason, "DB_UNHYDRATED"); + } finally { + SettingsManager.stopMaintenanceSchedulerForTests(); + await new Promise((resolve) => mockS3.close(() => resolve())); + fs.rmSync(tmpDir, { recursive: true, force: true }); + } +}); diff --git a/tests/r3-phase03-sdk-parity.test.ts b/tests/r3-phase03-sdk-parity.test.ts new file mode 100644 index 0000000..92c8c6d --- /dev/null +++ b/tests/r3-phase03-sdk-parity.test.ts @@ -0,0 +1,497 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import http from 'node:http'; +import { createRequire } from 'node:module'; +import os from 'node:os'; +import path from 'node:path'; +import test from 'node:test'; +import jwt from 'jsonwebtoken'; +import { Server as SocketIOServer } from 'socket.io'; +import { + DataStack, + DataStackAuthError, + DataStackQuotaExceededError, +} from '../sdk/node/datastack'; + +const require = createRequire(import.meta.url); +const { setupSocketHandlers } = require('../server/socket/handlers.js') as { + setupSocketHandlers: ( + io: SocketIOServer, + bufferQueue: Array>, + worker: { + flush?: (rows: Array>) => Promise; + checkStorageAdmission?: ( + clientId: string, + quotaMb: number, + diskPct: number + ) => Promise<{ allowed: boolean; code?: string; reason?: string }>; + }, + options?: Record + ) => unknown; +}; + +const TEST_JWT_SECRET = 'r3-phase03-sdk-parity-jwt-secret-key-32-chars'; + +function signClientJwt(clientId: string): string { + return jwt.sign({ clientId, sub: clientId }, TEST_JWT_SECRET, { + expiresIn: '1h', + }); +} + +async function startLiveIngestionSocketServer(options?: { + verifyClientInDb?: ( + clientId: string, + token: string, + forceRefresh: boolean, + clientIp: string + ) => Promise<{ + allowed: boolean; + code?: string; + reason?: string; + rateLimit?: number; + storageQuotaMb?: number; + }>; + checkStorageAdmission?: ( + clientId: string, + quotaMb: number, + diskPct: number + ) => Promise<{ allowed: boolean; code?: string; reason?: string }>; +}) { + const httpServer = http.createServer((_req, res) => { + res.writeHead(404); + res.end(); + }); + const io = new SocketIOServer(httpServer, { + cors: { origin: '*' }, + transports: ['websocket', 'polling'], + }); + const bufferQueue: Array> = []; + const flushed: Array> = []; + + setupSocketHandlers( + io, + bufferQueue, + { + flush: async (rows: Array>) => { + flushed.push(...rows); + }, + checkStorageAdmission: + options?.checkStorageAdmission ?? + (async () => ({ allowed: true })), + }, + { + getRuntimeConfig: () => ({ + jwtSecret: TEST_JWT_SECRET, + maxEventsPerSec: 200, + maxPayloadSizeBytes: 100 * 1024, + batchFlushThreshold: 500, + enableIdempotency: true, + diskPauseThresholdPct: 95, + }), + verifyClientInDb: + options?.verifyClientInDb ?? + (async () => ({ + allowed: true, + rateLimit: 200, + storageQuotaMb: 0, + })), + } + ); + + await new Promise((resolve) => { + httpServer.listen(0, '127.0.0.1', () => resolve()); + }); + const addr = httpServer.address(); + const port = typeof addr === 'object' && addr ? addr.port : 0; + + return { + port, + url: `http://127.0.0.1:${port}`, + io, + bufferQueue, + flushed, + close: async () => { + await new Promise((resolve) => { + io.close(() => { + httpServer.close(() => resolve()); + }); + }); + }, + }; +} + +test('R3-Phase-03 (BUG-07): Compiled Node.js SDK dist/index.js loads cleanly via plain CommonJS require() and exports typed package entrypoints', async () => { + const pkgPath = path.join( + process.cwd(), + 'sdk', + 'node', + 'datastack', + 'package.json' + ); + const pkg = JSON.parse(await fs.readFile(pkgPath, 'utf8')) as { + main: string; + types: string; + exports?: { '.': { default: string; types: string } }; + scripts?: { build?: string }; + }; + + assert.equal(pkg.main, './dist/index.js'); + assert.equal(pkg.types, './dist/index.d.ts'); + assert.equal(pkg.exports?.['.']?.default, './dist/index.js'); + assert.equal(pkg.exports?.['.']?.types, './dist/index.d.ts'); + assert.ok(pkg.scripts?.build?.includes('tsc')); + + // Verify compiled CommonJS output loads directly via Node require() with zero TypeScript transpilation + const distPath = path.join( + process.cwd(), + 'sdk', + 'node', + 'datastack', + 'dist', + 'index.js' + ); + const loaded = require(distPath) as Record; + assert.equal(typeof loaded.DataStack, 'function'); + assert.equal(typeof loaded.DataStackClient, 'function'); + assert.equal(typeof loaded.DataStackAuthError, 'function'); + assert.equal(typeof loaded.DataStackQuotaExceededError, 'function'); + assert.equal(typeof loaded.SDK_VERSION, 'string'); + + const dtsPath = path.join( + process.cwd(), + 'sdk', + 'node', + 'datastack', + 'dist', + 'index.d.ts' + ); + const dtsContent = await fs.readFile(dtsPath, 'utf8'); + assert.match(dtsContent, /export declare class DataStack/); +}); + +test('R3-Phase-03 (BUG-02): Live Port 4000 Socket.IO batch_sync accepts Node.js SDK raw Array payload and reports processed + duplicates accurately', async () => { + const server = await startLiveIngestionSocketServer(); + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'ds-r3-p03-ws-')); + let client: DataStack | null = null; + + try { + const token = signClientJwt('node_ws_batch_client'); + client = new DataStack({ + serverUrl: server.url, + apiUrl: 'http://127.0.0.1:1', // Unreachable HTTP to prove WS-only delivery + token, + autoConnect: false, + enableHttpFallback: false, + offlineSpoolEnabled: false, + heartbeatIntervalMs: 5000, + }); + + await client.connect(); + assert.equal(client.status().connected, true); + + // 1. Live WebSocket sendBatch with explicit eventIds (must send raw Array, NOT { clientId, batch }) + const batchResult = await client.sendBatch([ + { eventId: 'r3_ws_evt_1', metric: 'cpu', value: 41.2 }, + { eventId: 'r3_ws_evt_2', metric: 'mem', value: 68.5 }, + { eventId: 'r3_ws_evt_3', metric: 'disk', value: 52.0 }, + ]); + + assert.equal(batchResult.success, true); + assert.equal(batchResult.transport, 'websocket'); + assert.equal(batchResult.accepted, 3); + assert.equal(batchResult.duplicates, 0); + assert.equal(batchResult.spooled, 0); + assert.equal(server.bufferQueue.length, 3); + + // 2. Idempotent re-send of 2 duplicates + 1 new event over live WebSocket batch_sync + const dedupResult = await client.batchSync([ + { eventId: 'r3_ws_evt_1', metric: 'cpu', value: 41.2 }, + { eventId: 'r3_ws_evt_2', metric: 'mem', value: 68.5 }, + { eventId: 'r3_ws_evt_4', metric: 'net', value: 12.8 }, + ]); + + assert.equal(dedupResult.success, true); + assert.equal(dedupResult.transport, 'websocket'); + assert.equal(dedupResult.accepted, 1); + assert.equal(dedupResult.duplicates, 2); + assert.equal(dedupResult.spooled, 0); + assert.equal(server.bufferQueue.length, 4); + + // 3. Live WebSocket sendFile() + idempotent file re-send (deterministic eventIds) + const csvFile = path.join(tmpDir, 'telemetry.csv'); + await fs.writeFile( + csvFile, + 'node,temp\na1,23.5\na2,24.1\n', + 'utf8' + ); + + const fileRes1 = await client.sendFile(csvFile, { chunkSize: 10 }); + assert.equal(fileRes1.success, true); + assert.equal(fileRes1.totalRows, 2); + assert.equal(fileRes1.acceptedRows, 2); + assert.equal(fileRes1.duplicates, 0); + assert.equal(server.bufferQueue.length, 6); + + const fileRes2 = await client.sendFile(csvFile, { chunkSize: 10 }); + assert.equal(fileRes2.success, true); + assert.equal(fileRes2.totalRows, 2); + assert.equal(fileRes2.acceptedRows, 0); + assert.equal(fileRes2.duplicates, 2); + assert.equal(server.bufferQueue.length, 6); + } finally { + if (client) { + await client.disconnect(); + } + await server.close(); + await fs.rm(tmpDir, { recursive: true, force: true }); + } +}); + +test('R3-Phase-03 (BUG-07): Node.js SDK heartbeatIntervalMs monitor stays zero-CPU in IDLE_WS, activates on drop, recovers via ping_verify, and cleans up on disconnect', async () => { + const server = await startLiveIngestionSocketServer(); + let client: DataStack | null = null; + + try { + const token = signClientJwt('node_hb_client'); + client = new DataStack({ + serverUrl: server.url, + apiUrl: 'http://127.0.0.1:1', + token, + autoConnect: false, + enableHttpFallback: false, + offlineSpoolEnabled: false, + heartbeatIntervalMs: 120, + requestTimeoutMs: 1500, + }); + + assert.equal(client.status().heartbeatActive, false); + + await client.connect(); + // Zero-CPU IDLE_WS parity: heartbeat timer is paused while socket is healthy + assert.equal(client.status().connected, true); + assert.equal(client.status().transportState, 'IDLE_WS'); + assert.equal(client.status().heartbeatActive, false); + + // Force server-side disconnect of all sockets -> heartbeat monitor activates immediately + for (const sock of server.io.sockets.sockets.values()) { + sock.disconnect(true); + } + + // Wait briefly to observe disconnect + automatic heartbeat ping_verify recovery + let sawHeartbeatActive = false; + const deadline = Date.now() + 3000; + while (Date.now() < deadline) { + const snap = client.status(); + if (snap.heartbeatActive) { + sawHeartbeatActive = true; + } + if (sawHeartbeatActive && snap.connected && !snap.heartbeatActive) { + break; + } + await new Promise((r) => setTimeout(r, 20)); + } + + assert.equal(sawHeartbeatActive, true); + assert.equal(client.status().connected, true); + assert.equal(client.status().transportState, 'IDLE_WS'); + assert.equal(client.status().heartbeatActive, false); + + // Verify recovered WebSocket can immediately send live telemetry + const afterRecover = await client.send({ + eventId: 'hb_recovered_1', + status: 'alive', + }); + assert.equal(afterRecover.success, true); + assert.equal(afterRecover.transport, 'websocket'); + assert.equal(afterRecover.accepted, 1); + + // Disconnect must stop heartbeat timer cleanly + await client.disconnect(); + assert.equal(client.status().connected, false); + assert.equal(client.status().heartbeatActive, false); + } finally { + if (client) { + await client.disconnect(); + } + await server.close(); + } +}); + +test('R3-Phase-03 (BUG-07): Single HTTP DUPLICATE_DROPPED reports accepted=0, duplicates=1 and TOKEN_ROTATED / CLIENT_REVOKED fail fast with DataStackAuthError', async () => { + const token = signClientJwt('node_http_dedup_auth'); + let mode: 'dup' | 'rotated' | 'revoked' = 'dup'; + let fetchCalls = 0; + + const mockFetch: typeof fetch = async () => { + fetchCalls += 1; + if (mode === 'dup') { + return new Response( + JSON.stringify({ + success: true, + eventId: 'dup_single_1', + note: 'DUPLICATE_DROPPED', + }), + { status: 200, headers: { 'Content-Type': 'application/json' } } + ); + } + if (mode === 'rotated') { + return new Response( + JSON.stringify({ + success: false, + code: 'TOKEN_ROTATED', + error: 'Client token was rotated; issue new credentials', + }), + { status: 401, headers: { 'Content-Type': 'application/json' } } + ); + } + return new Response( + JSON.stringify({ + success: false, + code: 'CLIENT_REVOKED', + error: 'Client credential has been revoked', + }), + { status: 403, headers: { 'Content-Type': 'application/json' } } + ); + }; + + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'ds-r3-p03-auth-')); + const client = new DataStack({ + serverUrl: 'http://127.0.0.1:1', + apiUrl: 'http://127.0.0.1:3000', + token, + autoConnect: false, + maxRetries: 3, + offlineSpoolEnabled: true, + offlineSpoolDir: tmpDir, + fetchImpl: mockFetch, + }); + + try { + // 1. Single HTTP DUPLICATE_DROPPED -> accepted: 0, duplicates: 1 + mode = 'dup'; + const dupRes = await client.send({ + eventId: 'dup_single_1', + metric: 'temp', + val: 20, + }); + assert.equal(dupRes.success, true); + assert.equal(dupRes.accepted, 0); + assert.equal(dupRes.duplicates, 1); + assert.equal(dupRes.spooled ?? 0, 0); + + // 2. TOKEN_ROTATED -> immediate DataStackAuthError (401), 0 retries, 0 spooling + fetchCalls = 0; + mode = 'rotated'; + await assert.rejects( + () => client.send({ eventId: 'rot_1', k: 1 }), + (err: unknown) => + err instanceof DataStackAuthError && + err.code === 'TOKEN_ROTATED' && + err.statusCode === 401 + ); + assert.equal(fetchCalls, 1); + assert.equal(client.status().spooledCount, 0); + + // 3. CLIENT_REVOKED -> immediate DataStackAuthError (403), 0 retries, 0 spooling + fetchCalls = 0; + mode = 'revoked'; + await assert.rejects( + () => client.sendBatch([{ eventId: 'rev_1', k: 2 }]), + (err: unknown) => + err instanceof DataStackAuthError && + err.code === 'CLIENT_REVOKED' && + err.statusCode === 403 + ); + assert.equal(fetchCalls, 1); + assert.equal(client.status().spooledCount, 0); + } finally { + await client.disconnect(); + await fs.rm(tmpDir, { recursive: true, force: true }); + } +}); + +test('R3-Phase-03 (BUG-07): Live WebSocket TOKEN_ROTATED, CLIENT_REVOKED, and STORAGE_QUOTA_EXCEEDED fail fast without spooling', async () => { + let dbState: 'ok' | 'rotated' | 'revoked' | 'quota' = 'ok'; + const server = await startLiveIngestionSocketServer({ + verifyClientInDb: async () => { + if (dbState === 'rotated') { + return { + allowed: false, + code: 'TOKEN_ROTATED', + reason: 'Token rotated by administrator', + }; + } + if (dbState === 'revoked') { + return { + allowed: false, + code: 'CLIENT_REVOKED', + reason: 'Client revoked by administrator', + }; + } + return { allowed: true, rateLimit: 200, storageQuotaMb: 10 }; + }, + checkStorageAdmission: async () => { + if (dbState === 'quota') { + return { + allowed: false, + code: 'STORAGE_QUOTA_EXCEEDED', + reason: 'Client quota reached 10MB', + }; + } + return { allowed: true }; + }, + }); + + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'ds-r3-p03-ws-err-')); + const token = signClientJwt('node_ws_errors'); + const client = new DataStack({ + serverUrl: server.url, + apiUrl: 'http://127.0.0.1:1', + token, + autoConnect: false, + enableHttpFallback: true, + offlineSpoolEnabled: true, + offlineSpoolDir: tmpDir, + heartbeatIntervalMs: 5000, + }); + + try { + await client.connect(); + + // 1. Live WS batch_sync quota exceeded -> DataStackQuotaExceededError (507), 0 spooling + dbState = 'quota'; + await assert.rejects( + () => client.sendBatch([{ eventId: 'ws_q_1', v: 1 }]), + (err: unknown) => + err instanceof DataStackQuotaExceededError && + err.code === 'STORAGE_QUOTA_EXCEEDED' && + err.statusCode === 507 + ); + assert.equal(client.status().spooledCount, 0); + + // 2. Live WS batch_sync TOKEN_ROTATED -> DataStackAuthError (401), 0 spooling + dbState = 'rotated'; + await assert.rejects( + () => client.sendBatch([{ eventId: 'ws_rot_1', v: 2 }]), + (err: unknown) => + err instanceof DataStackAuthError && + err.code === 'TOKEN_ROTATED' && + err.statusCode === 401 + ); + assert.equal(client.status().spooledCount, 0); + + // 3. Reconnect while CLIENT_REVOKED -> connect_error maps to DataStackAuthError (403) + dbState = 'revoked'; + await assert.rejects( + () => client.connect(), + (err: unknown) => + err instanceof DataStackAuthError && + err.code === 'CLIENT_REVOKED' && + err.statusCode === 403 + ); + } finally { + await client.disconnect(); + await server.close(); + await fs.rm(tmpDir, { recursive: true, force: true }); + } +}); diff --git a/tests/r3-phase04-ui-metrics-rbac.test.ts b/tests/r3-phase04-ui-metrics-rbac.test.ts new file mode 100644 index 0000000..06bce2c --- /dev/null +++ b/tests/r3-phase04-ui-metrics-rbac.test.ts @@ -0,0 +1,271 @@ +import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import path from "node:path"; +import test from "node:test"; +import { NextRequest } from "next/server"; +import { GET as prometheusGet } from "../app/api/prometheus/route"; +import { SettingsManager } from "../core/adminSettings/settingsManager"; +import { ApiEngine } from "../core/apiEngine/apiService"; +import { + createAdminSessionToken, + getAdminSessionSecret, +} from "../core/auth/adminSession"; +import { + createInitialAdmin, + resetAdminStateForTests, +} from "../core/auth/adminUsers"; +import { SDKManager } from "../core/sdk/sdkManager"; +import { + classifyUiClientStatus, + formatDateForLocalInput, + getVconRoleCapabilities, + isUiClientExpired, + shouldRedirectSimulatorUnauthorized, + summarizeUiClientCounts, +} from "../lib/vconUiHelpers"; + +async function makeOwnerCookie(): Promise { + resetAdminStateForTests(); + await createInitialAdmin("r3-p4-owner@datastack.dev", "R3Phase04Pass!"); + const secret = getAdminSessionSecret()!; + const token = await createAdminSessionToken( + "r3-p4-owner@datastack.dev", + secret, + Date.now() + 3600_000, + 1, + "owner", + ); + return `datastack_admin_session=${token}`; +} + +test("R3-Phase-04 [BUG-11 / ERR-R3-07]: shouldRedirectSimulatorUnauthorized separates client 401s from admin session 401s", async () => { + // Non-401 statuses never redirect + assert.equal(shouldRedirectSimulatorUnauthorized(200, {}), false); + assert.equal(shouldRedirectSimulatorUnauthorized(400, { code: "UNAUTHORIZED" }), false); + assert.equal(shouldRedirectSimulatorUnauthorized(403, { code: "FORBIDDEN" }), false); + assert.equal(shouldRedirectSimulatorUnauthorized(500, null), false); + + // Client token 401 codes stay on /vcon/simulator and show inline response + for (const clientCode of [ + "CLIENT_TOKEN_EXPIRED", + "client_token_expired", + "TOKEN_ROTATED", + "AUTH_INVALID", + "CLIENT_REVOKED", + "CLIENT_IP_FORBIDDEN", + "CLIENT_UNAUTHORIZED", + ]) { + assert.equal( + shouldRedirectSimulatorUnauthorized(401, { + code: clientCode, + error: "Client rejected", + }), + false, + `Expected no redirect for client code ${clientCode}`, + ); + } + + // Admin session 401 codes or uncoded 401 responses redirect to /vcon/login + assert.equal( + shouldRedirectSimulatorUnauthorized(401, { code: "UNAUTHORIZED" }), + true, + ); + assert.equal( + shouldRedirectSimulatorUnauthorized(401, { code: "SESSION_EXPIRED" }), + true, + ); + assert.equal(shouldRedirectSimulatorUnauthorized(401, {}), true); + assert.equal(shouldRedirectSimulatorUnauthorized(401, null), true); + + // Verify simulator page wires shouldRedirectSimulatorUnauthorized in both send handlers + const simSource = await fs.readFile( + path.join(process.cwd(), "app/vcon/simulator/page.tsx"), + "utf8", + ); + assert.ok(simSource.includes("shouldRedirectSimulatorUnauthorized")); + assert.equal( + simSource.includes("if (res.status === 401) {\n window.location.assign"), + false, + "Simulator must parse JSON and evaluate shouldRedirectSimulatorUnauthorized before redirecting", + ); +}); + +test("R3-Phase-04 [BUG-12]: formatDateForLocalInput preserves local timezone round-trip without UTC offset drift", async () => { + assert.equal(formatDateForLocalInput(null), ""); + assert.equal(formatDateForLocalInput(undefined), ""); + assert.equal(formatDateForLocalInput(""), ""); + assert.equal(formatDateForLocalInput("not-a-valid-date"), ""); + + // Round-trip multiple UTC instants across winter/summer DST boundaries + for (const iso of [ + "2026-01-15T08:30:00.000Z", + "2026-07-20T23:45:00.000Z", + "2026-11-20T14:05:00.000Z", + ]) { + const formatted = formatDateForLocalInput(iso); + assert.match(formatted, /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}$/); + const roundTripped = new Date(formatted).toISOString(); + assert.equal( + roundTripped, + iso, + `Round-trip local datetime-local value for ${iso} drifted to ${roundTripped}`, + ); + } + + // Verify ClientsTable uses formatDateForLocalInput instead of toISOString().slice(0, 16) + const clientsTableSource = await fs.readFile( + path.join(process.cwd(), "components/sections/ClientsTable.tsx"), + "utf8", + ); + assert.ok(clientsTableSource.includes("formatDateForLocalInput(client.expiresAt)")); + assert.equal( + clientsTableSource.includes("toISOString().slice(0, 16)"), + false, + "ClientsTable must not format datetime-local using UTC toISOString().slice(0, 16)", + ); +}); + +test("R3-Phase-04 [BUG-14]: expired clients are separated in UI helpers, ApiEngine.getDashboardMetrics, and /api/prometheus", async () => { + const nowMs = Date.parse("2026-11-20T12:00:00.000Z"); + + assert.equal(isUiClientExpired(null, nowMs), false); + assert.equal(isUiClientExpired("2026-11-20T12:05:00.000Z", nowMs), false); + assert.equal(isUiClientExpired("2026-11-20T12:00:00.000Z", nowMs), true); + assert.equal(isUiClientExpired("2026-11-19T00:00:00.000Z", nowMs), true); + + const sampleClients = [ + { status: "active", isOnline: true, lastConnectedAt: "2026-11-20T11:59:00.000Z", expiresAt: null }, + { status: "active", isOnline: false, lastConnectedAt: "2026-11-20T10:00:00.000Z", expiresAt: "2026-12-01T00:00:00.000Z" }, + { status: "active", isOnline: false, lastConnectedAt: null, expiresAt: null }, + { status: "active", isOnline: false, lastConnectedAt: "2026-11-19T10:00:00.000Z", expiresAt: "2026-11-19T12:00:00.000Z" }, + { status: "active", isOnline: true, lastConnectedAt: "2026-11-20T11:58:00.000Z", expiresAt: "2026-11-20T11:00:00.000Z" }, + { status: "revoked", isOnline: false, lastConnectedAt: null, expiresAt: "2026-11-10T00:00:00.000Z" }, + ]; + + assert.equal(classifyUiClientStatus(sampleClients[0], nowMs), "online"); + assert.equal(classifyUiClientStatus(sampleClients[1], nowMs), "offline"); + assert.equal(classifyUiClientStatus(sampleClients[2], nowMs), "pending"); + assert.equal(classifyUiClientStatus(sampleClients[3], nowMs), "expired"); + assert.equal(classifyUiClientStatus(sampleClients[4], nowMs), "expired"); + assert.equal(classifyUiClientStatus(sampleClients[5], nowMs), "revoked"); + + const summary = summarizeUiClientCounts(sampleClients, nowMs); + assert.deepEqual(summary, { + total: 6, + online: 1, + offline: 1, + pending: 1, + expired: 2, + revoked: 1, + }); + + // Live integration: ApiEngine.getDashboardMetrics() & GET /api/prometheus + const activeClient = SDKManager.registerClient("r3p4_active_client"); + const expiredClient = SDKManager.registerClient("r3p4_expired_client", { + expiresAt: "2020-01-01T00:00:00.000Z", + }); + const revokedClient = SDKManager.registerClient("r3p4_revoked_client"); + SDKManager.revokeClient(revokedClient.id); + + const prevSettings = SettingsManager.getSettings(); + try { + SettingsManager.updateSettings({ enablePrometheusMetrics: true }); + const metrics = await ApiEngine.getDashboardMetrics(); + assert.ok( + metrics.expiredClients >= 1, + `Expected expiredClients >= 1, got ${metrics.expiredClients}`, + ); + assert.equal( + metrics.activeClients + metrics.expiredClients <= metrics.totalClients, + true, + "activeClients + expiredClients must not exceed totalClients", + ); + + // Check Prometheus output counts active vs expired vs revoked + const cookie = await makeOwnerCookie(); + const req = new NextRequest("http://localhost:3000/api/prometheus", { + headers: { cookie }, + }); + const res = await prometheusGet(req); + assert.equal(res.status, 200); + const body = await res.text(); + const activeMatch = body.match(/datastack_clients_total\{status="active"\}\s+(\d+)/); + const expiredMatch = body.match(/datastack_clients_total\{status="expired"\}\s+(\d+)/); + const revokedMatch = body.match(/datastack_clients_total\{status="revoked"\}\s+(\d+)/); + assert.ok(activeMatch, "Prometheus must expose status=\"active\" gauge"); + assert.ok(expiredMatch, "Prometheus must expose status=\"expired\" gauge"); + assert.ok(revokedMatch, "Prometheus must expose status=\"revoked\" gauge"); + assert.equal(Number(activeMatch[1]), metrics.activeClients); + assert.equal(Number(expiredMatch[1]), metrics.expiredClients); + assert.ok(Number(expiredMatch[1]) >= 1); + assert.ok(Number(revokedMatch[1]) >= 1); + } finally { + SDKManager.deleteClient(activeClient.id); + SDKManager.deleteClient(expiredClient.id); + SDKManager.deleteClient(revokedClient.id); + SettingsManager.updateSettings({ + enablePrometheusMetrics: prevSettings.enablePrometheusMetrics, + }); + } +}); + +test("R3-Phase-04 [BUG-16]: RBAC role capabilities and UI surface wiring across VconShell and Admin pages", async () => { + assert.deepEqual(getVconRoleCapabilities("owner"), { + role: "owner", + canAdminSettings: true, + canManageTeam: true, + canOperate: true, + isReadOnly: false, + }); + assert.deepEqual(getVconRoleCapabilities("admin"), { + role: "admin", + canAdminSettings: false, + canManageTeam: false, + canOperate: true, + isReadOnly: false, + }); + assert.deepEqual(getVconRoleCapabilities("operator"), { + role: "operator", + canAdminSettings: false, + canManageTeam: false, + canOperate: true, + isReadOnly: false, + }); + assert.deepEqual(getVconRoleCapabilities("viewer"), { + role: "viewer", + canAdminSettings: false, + canManageTeam: false, + canOperate: false, + isReadOnly: true, + }); + // Fail-closed default for unknown/missing roles + assert.deepEqual(getVconRoleCapabilities("unknown_role"), { + role: "viewer", + canAdminSettings: false, + canManageTeam: false, + canOperate: false, + isReadOnly: true, + }); + + // Static audit confirming all 6 UI surfaces + layout/shell wire the verified role context + const filesToCheck: Array<{ rel: string; patterns: string[] }> = [ + { rel: "app/vcon/layout.tsx", patterns: ["role={auth.session.role}"] }, + { rel: "components/layout/VconShell.tsx", patterns: ["export function useVconRole()", "VconRoleContext.Provider"] }, + { rel: "components/sections/SettingsWorkspace.tsx", patterns: ["useVconRole()", "canAdminSettings", "canOperate", "Read-only configuration"] }, + { rel: "components/sections/ClientsTable.tsx", patterns: ["useVconRole()", "canManage"] }, + { rel: "app/vcon/clients/page.tsx", patterns: ["useVconRole()", "canOperate", "summarizeUiClientCounts", "Expired"] }, + { rel: "components/sections/DashboardOverview.tsx", patterns: ["useVconRole()", "canOperate", "isUiClientExpired"] }, + { rel: "app/vcon/storage/page.tsx", patterns: ["useVconRole()", "canOperate", "Read-only"] }, + { rel: "app/vcon/simulator/page.tsx", patterns: ["useVconRole()", "canOperate", "Read-only"] }, + ]; + + for (const check of filesToCheck) { + const content = await fs.readFile(path.join(process.cwd(), check.rel), "utf8"); + for (const pattern of check.patterns) { + assert.ok( + content.includes(pattern), + `Expected ${check.rel} to contain "${pattern}"`, + ); + } + } +}); diff --git a/tests/r3_phase03_python_sdk_test.py b/tests/r3_phase03_python_sdk_test.py new file mode 100644 index 0000000..f559709 --- /dev/null +++ b/tests/r3_phase03_python_sdk_test.py @@ -0,0 +1,218 @@ +""" +R3-Phase-03 Verification Tests (BUG-08 / ERR-R3-06): +- `DataStackQuotaExceededError` exported from both canonical `datastack` + and compatibility alias `datastack_sdk` with `retryable=False`. +- HTTP 507 (`STORAGE_QUOTA_EXCEEDED` and `DISK_SPACE_LOW`) immediately raises + `DataStackQuotaExceededError` with 0 retries and 0 offline disk spool entries. +- WebSocket ACK (`STORAGE_QUOTA_EXCEEDED` and `DISK_SPACE_LOW`) immediately + raises `DataStackQuotaExceededError` with 0 retries, 0 HTTP fallback calls, + and 0 offline disk spool entries for both single `send()` and `send_batch()`. +""" + +import base64 +import json +import sys +import tempfile +import threading +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Dict + +REPO_ROOT = Path(__file__).resolve().parents[1] +CANONICAL_PKG_ROOT = REPO_ROOT / "sdk" / "python" / "datastack" +COMPAT_PKG_ROOT = REPO_ROOT / "sdk-python" + +for p in (str(CANONICAL_PKG_ROOT), str(COMPAT_PKG_ROOT)): + if p not in sys.path: + sys.path.insert(0, p) + +import datastack # noqa: E402 +import datastack_sdk # noqa: E402 +from datastack import ( # noqa: E402 + DataStack, + DataStackError, + DataStackQuotaExceededError, +) + + +def _make_jwt(client_id: str) -> str: + header = ( + base64.urlsafe_b64encode(json.dumps({"alg": "HS256", "typ": "JWT"}).encode()) + .decode() + .rstrip("=") + ) + body = ( + base64.urlsafe_b64encode( + json.dumps({"clientId": client_id, "sub": client_id}).encode() + ) + .decode() + .rstrip("=") + ) + return f"{header}.{body}.testsig" + + +class R3Phase03PythonQuotaTests(unittest.TestCase): + def test_quota_exception_exports_and_non_retryable_flag(self) -> None: + self.assertIs( + datastack_sdk.DataStackQuotaExceededError, + datastack.DataStackQuotaExceededError, + ) + self.assertIn("DataStackQuotaExceededError", datastack.__all__) + self.assertIn("DataStackQuotaExceededError", datastack_sdk.__all__) + + err = DataStackQuotaExceededError("Quota full") + self.assertIsInstance(err, DataStackError) + self.assertFalse(err.retryable) + self.assertEqual(err.code, "STORAGE_QUOTA_EXCEEDED") + + def test_http_507_quota_and_disk_low_fail_fast_without_retry_or_spool(self) -> None: + state = {"code": "STORAGE_QUOTA_EXCEEDED", "calls": 0} + tok = _make_jwt("py_quota_http") + + class QuotaHttpHandler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 + length = int(self.headers.get("Content-Length", "0")) + _ = self.rfile.read(length) + state["calls"] += 1 + body = json.dumps( + { + "success": False, + "code": state["code"], + "error": f"Storage blocked ({state['code']})", + } + ).encode("utf-8") + self.send_response(507) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: Any) -> None: + return + + server = ThreadingHTTPServer(("127.0.0.1", 0), QuotaHttpHandler) + port = server.server_address[1] + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + + try: + with tempfile.TemporaryDirectory() as tmpdir: + spool_dir = Path(tmpdir) / "spool" + client = DataStack( + server_url="http://127.0.0.1:1", + api_url=f"http://127.0.0.1:{port}", + token=tok, + auto_connect=False, + max_retries=3, + retry_backoff_base_sec=0.05, + request_timeout_sec=1.0, + offline_spool_enabled=True, + offline_spool_dir=str(spool_dir), + ) + + # 1. HTTP 507 STORAGE_QUOTA_EXCEEDED -> 1 call (0 retries), 0 spooled + state["code"] = "STORAGE_QUOTA_EXCEEDED" + state["calls"] = 0 + with self.assertRaises(DataStackQuotaExceededError) as ctx1: + client.send({"sensor": "temp", "val": 100}) + self.assertEqual(ctx1.exception.code, "STORAGE_QUOTA_EXCEEDED") + self.assertFalse(ctx1.exception.retryable) + self.assertEqual(state["calls"], 1) + self.assertEqual(client.spooler.pending_count(), 0) + + # 2. HTTP 507 DISK_SPACE_LOW on send_batch -> 1 call (0 retries), 0 spooled + state["code"] = "DISK_SPACE_LOW" + state["calls"] = 0 + with self.assertRaises(DataStackQuotaExceededError) as ctx2: + client.send_batch([{"sensor": "a", "v": 1}, {"sensor": "b", "v": 2}]) + self.assertEqual(ctx2.exception.code, "DISK_SPACE_LOW") + self.assertFalse(ctx2.exception.retryable) + self.assertEqual(state["calls"], 1) + self.assertEqual(client.spooler.pending_count(), 0) + + client.disconnect() + finally: + server.shutdown() + server.server_close() + + def test_ws_ack_quota_and_disk_low_fail_fast_without_http_fallback_or_spool( + self, + ) -> None: + tok = _make_jwt("py_quota_ws") + + with tempfile.TemporaryDirectory() as tmpdir: + spool_dir = Path(tmpdir) / "spool" + client = DataStack( + server_url="http://127.0.0.1:4000", + api_url="http://127.0.0.1:3000", + token=tok, + auto_connect=False, + max_retries=3, + retry_backoff_base_sec=0.05, + enable_http_fallback=True, + offline_spool_enabled=True, + offline_spool_dir=str(spool_dir), + ) + + http_fallback_calls = {"count": 0} + + def fail_if_http_called(*args: Any, **kwargs: Any) -> Dict[str, Any]: + http_fallback_calls["count"] += 1 + raise AssertionError( + "HTTP fallback must NOT be called after WS DataStackQuotaExceededError" + ) + + client._send_via_http_fallback = fail_if_http_called # type: ignore[method-assign] + + ws_mode = {"code": "STORAGE_QUOTA_EXCEEDED", "calls": 0} + + class FakeQuotaSocket: + connected = True + + def emit( + self, event: str, payload: Any, callback: Any = None + ) -> None: + ws_mode["calls"] += 1 + if callback is not None: + callback( + { + "success": False, + "code": ws_mode["code"], + "error": f"WS rejected due to {ws_mode['code']}", + } + ) + + def disconnect(self) -> None: + self.connected = False + + client.sio = FakeQuotaSocket() + client.is_connected = True + + # 1. Single send WS STORAGE_QUOTA_EXCEEDED + ws_mode["code"] = "STORAGE_QUOTA_EXCEEDED" + ws_mode["calls"] = 0 + with self.assertRaises(DataStackQuotaExceededError) as ctx_ws1: + client.send({"k": "v1"}) + self.assertEqual(ctx_ws1.exception.code, "STORAGE_QUOTA_EXCEEDED") + self.assertFalse(ctx_ws1.exception.retryable) + self.assertEqual(ws_mode["calls"], 1) + self.assertEqual(http_fallback_calls["count"], 0) + self.assertEqual(client.spooler.pending_count(), 0) + + # 2. Batch send WS DISK_SPACE_LOW + ws_mode["code"] = "DISK_SPACE_LOW" + ws_mode["calls"] = 0 + with self.assertRaises(DataStackQuotaExceededError) as ctx_ws2: + client.send_batch([{"k": "v2"}, {"k": "v3"}]) + self.assertEqual(ctx_ws2.exception.code, "DISK_SPACE_LOW") + self.assertFalse(ctx_ws2.exception.retryable) + self.assertEqual(ws_mode["calls"], 1) + self.assertEqual(http_fallback_calls["count"], 0) + self.assertEqual(client.spooler.pending_count(), 0) + + client.disconnect() + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/v2-phase01-storage-quotas.test.ts b/tests/v2-phase01-storage-quotas.test.ts index 6b98d79..9d11ffc 100644 --- a/tests/v2-phase01-storage-quotas.test.ts +++ b/tests/v2-phase01-storage-quotas.test.ts @@ -725,3 +725,93 @@ test("V2-Phase-01 [Quotas V2-E04 & V2-E05]: HTTP POST /api/events + Port 4000 So resetAdminStateForTests(); } }); + +test("V2-Audit-Phase-03 [BUG-V2-AUDIT-01, BUG-V2-AUDIT-07, BUG-V2-AUDIT-10]: DiskUsageSnapshot available/freeBytes schema parity, /vcon/clients expiration counter alignment & CSV.GZ partition badges", async () => { + const prevStorageDir = process.env.DATASTACK_STORAGE_DIR; + const origBaseStorageDir = CSVStorageManager.baseStorageDir; + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "datastack-v2-p3-")); + + process.env.DATASTACK_STORAGE_DIR = tmpDir; + CSVStorageManager.baseStorageDir = tmpDir; + CSVStorageManager.clearQuotaCaches(); + + try { + const cookie = await makeAdminCookie(); + + // 1. BUG-V2-AUDIT-01: CSVStorageManager.getDiskUsageAsync() & GET /api/storage return available: true and freeBytes === availableBytes + CSVStorageManager.setStatfsOverrideForTests(async () => ({ + blocks: 2000, + bavail: 1500, // 25% used, 75% free + bsize: 4096, + })); + + const snapshot = await CSVStorageManager.getDiskUsageAsync(true); + assert.equal(snapshot.available, true); + assert.equal(snapshot.usedPct, 25); + assert.equal(snapshot.totalBytes, 2000 * 4096); + assert.equal(snapshot.availableBytes, 1500 * 4096); + assert.equal(snapshot.freeBytes, snapshot.availableBytes); + + const storageRes = await storageGet( + new NextRequest("http://localhost:3000/api/storage", { headers: { cookie } }), + ); + assert.equal(storageRes.status, 200); + const storageBody = await storageRes.json(); + assert.equal(storageBody.diskUsage.available, true); + assert.equal(storageBody.diskUsage.freeBytes, 1500 * 4096); + assert.equal(storageBody.diskUsage.availableBytes, 1500 * 4096); + assert.equal(storageBody.diskUsage.usedPct, 25); + + // Also verify statfs failure fallback sets available: false and freeBytes: 0 + CSVStorageManager.setStatfsOverrideForTests(async () => { + throw new Error("Simulated statfs failure"); + }); + const fallbackSnapshot = await CSVStorageManager.getDiskUsageAsync(true); + assert.equal(fallbackSnapshot.available, false); + assert.equal(fallbackSnapshot.freeBytes, 0); + assert.equal(fallbackSnapshot.availableBytes, 0); + + // 2. BUG-V2-AUDIT-07: app/vcon/clients/page.tsx defines isClientExpired and excludes expired tokens from onlineCount, pendingCount, and online/pending filter tabs + const clientsPageSrc = await fs.readFile( + path.join(process.cwd(), "app/vcon/clients/page.tsx"), + "utf8", + ); + assert.match(clientsPageSrc, /function isClientExpired\(/); + assert.match( + clientsPageSrc, + /const expired = !revoked && isClientExpired\(client\.expiresAt,\s*now\)/, + ); + assert.match( + clientsPageSrc, + /!isClientExpired\(client\.expiresAt,\s*now\)/, + ); + assert.match( + clientsPageSrc, + /const pending =\s*client\.status === 'active' && !expired && !client\.lastActive/, + ); + + // 3. BUG-V2-AUDIT-10: ExportWorkspace.tsx and app/vcon/storage/page.tsx render CSV.GZ badge for compressed .csv.gz partitions + const exportWorkspaceSrc = await fs.readFile( + path.join(process.cwd(), "components/sections/ExportWorkspace.tsx"), + "utf8", + ); + assert.match(exportWorkspaceSrc, /CSV\.GZ/); + assert.match(exportWorkspaceSrc, /endsWith\(['"]\.csv\.gz['"]\)/); + + const storagePageSrc = await fs.readFile( + path.join(process.cwd(), "app/vcon/storage/page.tsx"), + "utf8", + ); + assert.match(storagePageSrc, /CSV\.GZ/); + assert.match(storagePageSrc, /endsWith\(['"]\.csv\.gz['"]\)/); + } finally { + CSVStorageManager.setStatfsOverrideForTests(null); + CSVStorageManager.clearQuotaCaches(); + CSVStorageManager.baseStorageDir = origBaseStorageDir; + if (prevStorageDir === undefined) delete process.env.DATASTACK_STORAGE_DIR; + else process.env.DATASTACK_STORAGE_DIR = prevStorageDir; + await fs.rm(tmpDir, { recursive: true, force: true }); + resetAdminStateForTests(); + } +}); + diff --git a/tests/v2-phase02-alerts-ip-prometheus.test.ts b/tests/v2-phase02-alerts-ip-prometheus.test.ts index 6f436a1..8e9d998 100644 --- a/tests/v2-phase02-alerts-ip-prometheus.test.ts +++ b/tests/v2-phase02-alerts-ip-prometheus.test.ts @@ -12,6 +12,7 @@ import { PATCH as patchClientRoute } from "../app/api/clients/[id]/route"; import { POST as clientsPost } from "../app/api/clients/route"; import { POST as eventsPost } from "../app/api/events/route"; import { GET as exportGet } from "../app/api/export/route"; +import { GET as healthGet } from "../app/api/health/route"; import { GET as prometheusGet } from "../app/api/prometheus/route"; import { POST as settingsPost } from "../app/api/settings/route"; import { POST as webhookTestPost } from "../app/api/settings/webhook-test/route"; @@ -110,6 +111,28 @@ test("V2-Phase-02 [IP Allowlist & Token Expiry]: validateAllowedIpsInput, isClie assert.equal(isClientIpAllowed("::1", "127.0.0.1"), true); assert.equal(isClientIpAllowed("127.0.0.1", "::1"), true); + // BUG-V2-AUDIT-06: IPv6 CIDR (/0..128) BlockList matching & bracketed/port/zone normalization + assert.equal( + isClientIpAllowed("2001:db8:abcd:0012::1", "2001:db8::/32"), + true, + ); + assert.equal( + isClientIpAllowed("[2001:db8:abcd:0012::99]:8443", "2001:db8::/32"), + true, + ); + assert.equal( + isClientIpAllowed("2001:db9::1", "2001:db8::/32"), + false, + ); + assert.equal( + isClientIpAllowed("fe80::1ff:fe23:4567:890a%eth0", "fe80::/64"), + true, + ); + assert.equal( + isClientIpAllowed("203.0.113.10:54321", "203.0.113.0/24"), + true, + ); + // Expiration validation & evaluation assert.deepEqual(validateClientExpiresAtInput(null), { valid: true, @@ -689,3 +712,203 @@ test("V2-Phase-02 [Prometheus Metrics & NDJSON Streaming Export]: GET /api/prome resetAdminStateForTests(); } }); + +test("V2-Audit-Phase-02 [BUG-V2-AUDIT-05, BUG-V2-AUDIT-06, BUG-V2-AUDIT-08]: Port 4000 outbound webhook alerts, IPv6 CIDR parity & GET /api/health expiry/IP checks", async () => { + const prevDisableListen = process.env.DATASTACK_DISABLE_LISTEN; + const prevDbUrl = process.env.DATABASE_URL; + process.env.DATASTACK_DISABLE_LISTEN = "1"; + delete process.env.DATABASE_URL; + + const serverIndex = require("../server/index.js"); + const { Server: SocketIOServer } = require("socket.io"); + const { setupSocketHandlers } = require("../server/socket/handlers.js"); + + // 1. BUG-V2-AUDIT-06: Port 4000 isClientIpAllowed supports IPv6 CIDR (/0..128) and semicolon delimiters + assert.equal( + serverIndex.isClientIpAllowed("2001:db8:1234::5", "10.0.0.0/8; 2001:db8::/32"), + true, + ); + assert.equal( + serverIndex.isClientIpAllowed("[2001:db8:9999::1]:9443", "2001:db8::/32"), + true, + ); + assert.equal( + serverIndex.isClientIpAllowed("2001:db9::1", "2001:db8::/32"), + false, + ); + assert.equal( + serverIndex.isClientIpAllowed("fe80::1234%eth0", "fe80::/64"), + true, + ); + + // 2. BUG-V2-AUDIT-05: Port 4000 Socket.IO outbound webhook alerts with HMAC-SHA256 signature + const receivedPort4000Webhooks: Array<{ + headers: http.IncomingHttpHeaders; + body: string; + json: Record; + }> = []; + + const mockWebhookServer = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (chunk) => chunks.push(Buffer.from(chunk))); + req.on("end", () => { + const bodyStr = Buffer.concat(chunks).toString("utf8"); + let parsed: Record = {}; + try { + parsed = JSON.parse(bodyStr); + } catch { + parsed = {}; + } + receivedPort4000Webhooks.push({ + headers: req.headers, + body: bodyStr, + json: parsed, + }); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ ok: true })); + }); + }); + + await new Promise((resolve) => + mockWebhookServer.listen(0, "127.0.0.1", () => resolve()), + ); + const webhookPort = (mockWebhookServer.address() as AddressInfo).port; + const webhookUrl = `http://127.0.0.1:${webhookPort}/socket-hook`; + + const testHttpServer = http.createServer(); + const testIo = new SocketIOServer(testHttpServer); + + const forbiddenClient = await SDKManager.registerClientAsync({ + clientId: "audit05_forbidden_sock", + name: "Audit05ForbiddenSocketClient", + rateLimit: 100, + storageQuotaMb: 0, + allowedIps: "203.0.113.99", + expiresAt: null, + }); + + setupSocketHandlers(testIo, [], serverIndex.writer, { + getRuntimeConfig: () => serverIndex.runtimeConfig, + verifyClientInDb: async (cid: string, _tok: string, _bypass: boolean, ip: string) => { + if (cid === forbiddenClient.clientId && !serverIndex.isClientIpAllowed(ip, "203.0.113.99")) { + return { + allowed: false, + code: "CLIENT_IP_FORBIDDEN", + reason: `CLIENT_IP_FORBIDDEN: IP ${ip} is not allowed for ${cid}`, + }; + } + return { allowed: true, rateLimit: 100 }; + }, + triggerWebhookAlert: serverIndex.triggerWebhookAlert, + }); + + await new Promise((resolve) => + testHttpServer.listen(0, "127.0.0.1", () => resolve()), + ); + const socketPort = (testHttpServer.address() as AddressInfo).port; + + try { + serverIndex.clearWebhookCooldownCacheForTests(); + serverIndex.applyRuntimeSettingsObject({ + enableWebhookAlerts: true, + webhookUrl, + webhookSecret: "port4000_shared_hmac_secret", + webhookCooldownSec: 60, + }); + + // Attempt Socket.IO connection from 127.0.0.1 (not in 203.0.113.99 allowlist) + const sock: Socket = ioClient(`http://127.0.0.1:${socketPort}`, { + auth: { token: forbiddenClient.token }, + transports: ["websocket"], + reconnection: false, + timeout: 2000, + }); + + const connectErr = await new Promise((resolve, reject) => { + sock.on("connect", () => { + sock.disconnect(); + reject(new Error("Expected Socket.IO handshake rejection for forbidden IP")); + }); + sock.on("connect_error", (err) => { + sock.disconnect(); + resolve(err); + }); + }); + + assert.ok(connectErr.message.includes("CLIENT_IP_FORBIDDEN")); + + // Wait briefly for asynchronous webhook alert dispatch + await new Promise((r) => setTimeout(r, 80)); + assert.equal(receivedPort4000Webhooks.length, 1); + assert.equal( + receivedPort4000Webhooks[0].headers["x-datastack-event"], + "CLIENT_IP_FORBIDDEN", + ); + const expectedSig = buildWebhookSignature( + receivedPort4000Webhooks[0].body, + "port4000_shared_hmac_secret", + ); + assert.equal( + receivedPort4000Webhooks[0].headers["x-datastack-signature"], + expectedSig, + ); + assert.equal( + receivedPort4000Webhooks[0].json.clientId, + forbiddenClient.clientId, + ); + + // 3. BUG-V2-AUDIT-08: GET /api/health enforces CLIENT_TOKEN_EXPIRED (401) and CLIENT_IP_FORBIDDEN (403) + const expiredHealthClient = await SDKManager.registerClientAsync({ + clientId: "audit08_expired_health", + name: "Audit08ExpiredHealthClient", + rateLimit: 100, + storageQuotaMb: 0, + allowedIps: "", + expiresAt: "2020-01-01T00:00:00.000Z", + }); + + const expiredHealthRes = await healthGet( + new NextRequest("http://localhost:3000/api/health", { + headers: { + authorization: `Bearer ${expiredHealthClient.token}`, + }, + }), + ); + assert.equal(expiredHealthRes.status, 401); + const expiredHealthBody = await expiredHealthRes.json(); + assert.equal(expiredHealthBody.code, "CLIENT_TOKEN_EXPIRED"); + + const forbiddenHealthRes = await healthGet( + new NextRequest("http://localhost:3000/api/health", { + headers: { + authorization: `Bearer ${forbiddenClient.token}`, + "x-real-ip": "198.51.100.44", + }, + }), + ); + assert.equal(forbiddenHealthRes.status, 403); + const forbiddenHealthBody = await forbiddenHealthRes.json(); + assert.equal(forbiddenHealthBody.code, "CLIENT_IP_FORBIDDEN"); + } finally { + serverIndex.applyRuntimeSettingsObject({ + enableWebhookAlerts: false, + webhookUrl: "", + webhookSecret: "", + webhookCooldownSec: 300, + }); + serverIndex.stopTimers(); + SDKManager.stopActivityFlushTimerForTests(); + await DataBufferEngine.shutdown(); + await new Promise((resolve) => { + testIo.close(() => resolve()); + }); + await new Promise((resolve) => + mockWebhookServer.close(() => resolve()), + ); + resetAdminStateForTests(); + if (prevDisableListen === undefined) delete process.env.DATASTACK_DISABLE_LISTEN; + else process.env.DATASTACK_DISABLE_LISTEN = prevDisableListen; + if (prevDbUrl === undefined) delete process.env.DATABASE_URL; + else process.env.DATABASE_URL = prevDbUrl; + } +}); diff --git a/tests/v2-phase03-rbac-audit.test.ts b/tests/v2-phase03-rbac-audit.test.ts index 46dc9ac..5f9d05e 100644 --- a/tests/v2-phase03-rbac-audit.test.ts +++ b/tests/v2-phase03-rbac-audit.test.ts @@ -3,7 +3,9 @@ import { after, beforeEach, describe, it } from "node:test"; import { NextRequest } from "next/server"; import { GET as auditGet } from "@/app/api/audit/route"; import { POST as loginPost } from "@/app/api/auth/login/route"; +import { POST as logoutPost } from "@/app/api/auth/logout/route"; import { GET as profileGet, PUT as profilePut } from "@/app/api/auth/profile/route"; +import { POST as setupPost } from "@/app/api/auth/setup/route"; import { DELETE as clientDelete, GET as clientGet, @@ -24,7 +26,10 @@ import { } from "@/core/auth/adminSession"; import { createInitialAdmin, + getAdminSetupKey, resetAdminStateForTests, + revokeAdminSessions, + verifyRequestAdminSession, } from "@/core/auth/adminUsers"; import { DataBufferEngine } from "@/core/data/buffer"; @@ -618,4 +623,238 @@ describe("V2-Phase-03: Multi-Admin RBAC & Immutable Admin Audit Log", () => { assert.equal(freshJson.profile.role, "Operator"); assert.equal(freshJson.profile.roleKey, "operator"); }); + + it("5. V2-Audit-Phase-01: Prevents cross-table/in-memory duplicate admin emails (BUG-V2-AUDIT-03), isolates multi-admin session logout from global race (BUG-V2-AUDIT-04), and records ADMIN_SETUP, ADMIN_LOGOUT, CLIENT_REVOKED, and STORAGE_FLUSH_TRIGGERED audit actions (BUG-V2-AUDIT-02)", async () => { + const setupKey = getAdminSetupKey()!; + const setupRes = await setupPost( + new NextRequest("http://localhost:3000/api/auth/setup", { + method: "POST", + headers: { + "content-type": "application/json", + "x-real-ip": "198.51.100.10", + }, + body: JSON.stringify({ + email: "founder@datastack.dev", + password: "FounderPassword123!", + setupKey, + }), + }), + ); + assert.equal(setupRes.status, 200); + const ownerCookie = setupRes.cookies.get("datastack_admin_session")?.value!; + assert.ok(ownerCookie); + + // Owner creates two team members: Alice (admin) and Bob (operator) + const aliceRes = await teamPost( + new NextRequest("http://localhost:3000/api/team", { + method: "POST", + headers: { + "content-type": "application/json", + cookie: `datastack_admin_session=${ownerCookie}`, + }, + body: JSON.stringify({ + email: "alice@datastack.dev", + password: "AlicePassword123!", + role: "admin", + }), + }), + ); + assert.equal(aliceRes.status, 201); + const aliceMember = (await aliceRes.json()).member; + + const bobRes = await teamPost( + new NextRequest("http://localhost:3000/api/team", { + method: "POST", + headers: { + "content-type": "application/json", + cookie: `datastack_admin_session=${ownerCookie}`, + }, + body: JSON.stringify({ + email: "bob@datastack.dev", + password: "BobPassword123!", + role: "operator", + }), + }), + ); + assert.equal(bobRes.status, 201); + + // Login as Alice and Bob + const aliceLogin = await loginPost( + new NextRequest("http://localhost:3000/api/auth/login", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + email: "alice@datastack.dev", + password: "AlicePassword123!", + }), + }), + ); + assert.equal(aliceLogin.status, 200); + const aliceCookie = aliceLogin.cookies.get("datastack_admin_session")?.value!; + + const bobLogin = await loginPost( + new NextRequest("http://localhost:3000/api/auth/login", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + email: "bob@datastack.dev", + password: "BobPassword123!", + }), + }), + ); + assert.equal(bobLogin.status, 200); + const bobCookie = bobLogin.cookies.get("datastack_admin_session")?.value!; + + // BUG-V2-AUDIT-03: Alice cannot change her email to Primary Owner's email OR Bob's email -> 409 ADMIN_EMAIL_CONFLICT + const aliceCollideOwner = await profilePut( + new NextRequest("http://localhost:3000/api/auth/profile", { + method: "PUT", + headers: { + "content-type": "application/json", + cookie: `datastack_admin_session=${aliceCookie}`, + }, + body: JSON.stringify({ + email: "FOUNDER@datastack.dev", + currentPassword: "AlicePassword123!", + }), + }), + ); + assert.equal(aliceCollideOwner.status, 409); + assert.equal((await aliceCollideOwner.json()).code, "ADMIN_EMAIL_CONFLICT"); + + const aliceCollideBob = await profilePut( + new NextRequest("http://localhost:3000/api/auth/profile", { + method: "PUT", + headers: { + "content-type": "application/json", + cookie: `datastack_admin_session=${aliceCookie}`, + }, + body: JSON.stringify({ + email: "bob@datastack.dev", + currentPassword: "AlicePassword123!", + }), + }), + ); + assert.equal(aliceCollideBob.status, 409); + assert.equal((await aliceCollideBob.json()).code, "ADMIN_EMAIL_CONFLICT"); + + // Primary Owner cannot change email to Alice's email -> 409 ADMIN_EMAIL_CONFLICT + const ownerCollideAlice = await profilePut( + new NextRequest("http://localhost:3000/api/auth/profile", { + method: "PUT", + headers: { + "content-type": "application/json", + cookie: `datastack_admin_session=${ownerCookie}`, + }, + body: JSON.stringify({ + email: "alice@datastack.dev", + currentPassword: "FounderPassword123!", + }), + }), + ); + assert.equal(ownerCollideAlice.status, 409); + assert.equal((await ownerCollideAlice.json()).code, "ADMIN_EMAIL_CONFLICT"); + + // BUG-V2-AUDIT-04: Simulate concurrent verification where Bob's session is verified right before Alice's session revocation + const aliceVerified = await verifyRequestAdminSession(aliceCookie); + assert.equal(aliceVerified.valid, true); + const bobVerified = await verifyRequestAdminSession(bobCookie); + assert.equal(bobVerified.valid, true); + + // Explicitly revoke Alice's session even though Bob was the last verified globally + await revokeAdminSessions({ + memberId: aliceMember.id, + email: "alice@datastack.dev", + }); + + // Alice's cookie must now be revoked, while Bob's and Owner's cookies remain valid! + const aliceAfterRevoke = await verifyRequestAdminSession(aliceCookie); + assert.equal(aliceAfterRevoke.valid, false); + const bobStillValid = await verifyRequestAdminSession(bobCookie); + assert.equal(bobStillValid.valid, true); + const ownerStillValid = await verifyRequestAdminSession(ownerCookie); + assert.equal(ownerStillValid.valid, true); + + // Create a client, revoke it (CLIENT_REVOKED), trigger manual storage flush (STORAGE_FLUSH_TRIGGERED), and log out Bob (ADMIN_LOGOUT) + const createClientRes = await clientsPost( + new NextRequest("http://localhost:3000/api/clients", { + method: "POST", + headers: { + "content-type": "application/json", + cookie: `datastack_admin_session=${ownerCookie}`, + }, + body: JSON.stringify({ + clientId: "phase01_audit_client", + name: "Phase 01 Audit Client", + }), + }), + ); + assert.equal(createClientRes.status, 201); + + const revokeClientRes = await clientPatch( + new NextRequest("http://localhost:3000/api/clients/phase01_audit_client", { + method: "PATCH", + headers: { + "content-type": "application/json", + cookie: `datastack_admin_session=${ownerCookie}`, + }, + body: JSON.stringify({ status: "revoked" }), + }), + { params: Promise.resolve({ id: "phase01_audit_client" }) }, + ); + assert.equal(revokeClientRes.status, 200); + + const flushRes = await storagePost( + new NextRequest("http://localhost:3000/api/storage", { + method: "POST", + headers: { cookie: `datastack_admin_session=${ownerCookie}` }, + }), + ); + // Even if Port 4000 is mocked to 127.0.0.1:1, test without explicit INGESTION_INTERNAL_URL so local flush succeeds + void flushRes; + delete process.env.INGESTION_INTERNAL_URL; + const flushLocalRes = await storagePost( + new NextRequest("http://localhost:3000/api/storage", { + method: "POST", + headers: { cookie: `datastack_admin_session=${ownerCookie}` }, + }), + ); + assert.equal(flushLocalRes.status, 200); + + const bobLogoutRes = await logoutPost( + new NextRequest("http://localhost:3000/api/auth/logout", { + method: "POST", + headers: { + cookie: `datastack_admin_session=${bobCookie}`, + "x-real-ip": "198.51.100.25", + }, + }), + ); + assert.equal(bobLogoutRes.status, 200); + + // Bob's session is now invalidated on server, while Owner's session is still valid + assert.equal((await verifyRequestAdminSession(bobCookie)).valid, false); + assert.equal((await verifyRequestAdminSession(ownerCookie)).valid, true); + + // BUG-V2-AUDIT-02: Verify all 4 newly wired audit actions appear in GET /api/audit + const auditRes = await auditGet( + new NextRequest("http://localhost:3000/api/audit?limit=100", { + headers: { cookie: `datastack_admin_session=${ownerCookie}` }, + }), + ); + assert.equal(auditRes.status, 200); + const logs = (await auditRes.json()).logs as Array<{ + action: string; + actorEmail: string; + target: string; + }>; + const recordedActions = new Set(logs.map((l) => l.action)); + assert.ok(recordedActions.has("ADMIN_SETUP"), "Expected ADMIN_SETUP audit log"); + assert.ok(recordedActions.has("ADMIN_LOGOUT"), "Expected ADMIN_LOGOUT audit log"); + assert.ok(recordedActions.has("CLIENT_REVOKED"), "Expected CLIENT_REVOKED audit log"); + assert.ok( + recordedActions.has("STORAGE_FLUSH_TRIGGERED"), + "Expected STORAGE_FLUSH_TRIGGERED audit log", + ); + }); });