feat: implement R4 production hardening
- Add configurable ingestion batch item limits - Improve client serialization and sanitization - Add graceful handling for corrupt GZIP partitions in data API - Expand admin session token scope - Update project documentation for R4 status
This commit is contained in:
1 parent
8132087085
commit
ae9b554189
59 files changed
+16878
-1073
No files matched your search
@@ -27,3 +27,9 @@ PORT_SOCKET=4000
|
|||||||
INGESTION_INTERNAL_URL=http://127.0.0.1:4000
|
INGESTION_INTERNAL_URL=http://127.0.0.1:4000
|
||||||
# Optional comma-separated browser CORS origin allowlist for Port 4000 ingestion/Socket.IO (defaults to wildcard if unset)
|
# Optional comma-separated browser CORS origin allowlist for Port 4000 ingestion/Socket.IO (defaults to wildcard if unset)
|
||||||
# DATASTACK_CORS_ORIGINS=https://console.example.com,https://admin.example.com
|
# DATASTACK_CORS_ORIGINS=https://console.example.com,https://admin.example.com
|
||||||
|
# Outbound SSRF protection blocks loopback/private/internal URLs by default for webhooks & S3 endpoints.
|
||||||
|
# Set to true ONLY if intentionally targeting a self-hosted LAN MinIO or private webhook endpoint (cloud metadata IPs remain always blocked).
|
||||||
|
# DATASTACK_ALLOW_PRIVATE_URLS=false
|
||||||
|
|
||||||
|
# Ingestion Batch Array Guard (default: 1000 items per HTTP POST /api/events batch or Socket.IO batch_sync frame; bounded 1..10000)
|
||||||
|
# DATASTACK_MAX_BATCH_ITEMS=1000
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
# MEMORY.md — DataStack Cross-IDE Persistent Project Memory
|
# MEMORY.md — DataStack Cross-IDE Persistent Project Memory
|
||||||
|
|
||||||
> **Last Synchronized**: 2026-11-20
|
> **Last Synchronized**: 2026-11-21
|
||||||
> **Status**: **R3 Production Remediation (`BASELINE-2026-11-20-R3`): `3 / 4` Phases Complete (`12 / 16` Bugs Resolved — `R3-Phase-01`, `R3-Phase-02`, & `R3-Phase-03` Verified Complete; `R3-Phase-04` ready for `Proceed to R3-Phase-04`).**
|
> **Status**: **`R4` Production Hardening (`BASELINE-2026-11-20-R4`: `4 / 4` Phases Complete — All `R4-Phase-01`..`04` Complete & Verified (`AUDIT-01`–`20`, `19 / 19` actionable findings resolved)).**
|
||||||
> **Scope Rule**: Maintain this file accurately across VS Code, Google Antigravity, Cursor, and any other IDE after every modification, feature update, or bug fix (see `.ai-rules` and `AGENTS.md`).
|
> **Scope Rule**: Maintain this file accurately across VS Code, Google Antigravity, Cursor, and any other IDE after every modification, feature update, or bug fix (see `.ai-rules` and `AGENTS.md`).
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -61,12 +61,13 @@ datastack/
|
|||||||
│ │ ├── data/ # Paginated CSV search/query across partitions
|
│ │ ├── data/ # Paginated CSV search/query across partitions
|
||||||
│ │ ├── events/ # Authenticated HTTP ingestion (single/batch/file records) & recent list
|
│ │ ├── events/ # Authenticated HTTP ingestion (single/batch/file records) & recent list
|
||||||
│ │ ├── export/ # Streaming CSV / JSON filtered export (UTC dates, optional flush=false)
|
│ │ ├── export/ # Streaming CSV / JSON filtered export (UTC dates, optional flush=false)
|
||||||
|
│ │ ├── files/status/ # Safe server-side Port 4000 Files Manager probe (catches ECONNREFUSED cleanly)
|
||||||
│ │ ├── health/ # Web health + client Bearer token probe + async buffer depth
|
│ │ ├── health/ # Web health + client Bearer token probe + async buffer depth
|
||||||
│ │ ├── metrics/ # Aggregated Port 3000 + Port 4000 + DB + CSV metrics
|
│ │ ├── metrics/ # Aggregated Port 3000 + Port 4000 + DB + CSV metrics
|
||||||
│ │ ├── settings/ # Allowlisted runtime config get/post + live Port 4000 sync
|
│ │ ├── settings/ # Allowlisted runtime config get/post + live Port 4000 sync
|
||||||
│ │ └── storage/ # CSV partition listing & manual cross-process buffer flush
|
│ │ └── storage/ # CSV partition listing & manual cross-process buffer flush
|
||||||
│ └── vcon/ # Protected Admin Console pages:
|
│ └── vcon/ # Protected Admin Console pages:
|
||||||
│ ├── dashboard/ clients/ data/ export/ login/ profile/ sdk/ settings/ setup/ simulator/ storage/
|
│ ├── dashboard/ clients/ data/ export/ files/ login/ profile/ sdk/ settings/ setup/ simulator/ storage/ team/ audit/
|
||||||
├── components/
|
├── components/
|
||||||
│ ├── ui/ # Single-source-of-truth UI primitives (Button, Input, Table, Modal, Panel, etc.)
|
│ ├── ui/ # Single-source-of-truth UI primitives (Button, Input, Table, Modal, Panel, etc.)
|
||||||
│ ├── layout/ # MainLayout, Navbar, Sidebar, VconShell, PageShell, AdminPageHeader, Footer
|
│ ├── layout/ # MainLayout, Navbar, Sidebar, VconShell, PageShell, AdminPageHeader, Footer
|
||||||
@@ -89,7 +90,8 @@ datastack/
|
|||||||
│ ├── dev-all.js # Local concurrent Next.js (3000) + Ingestion (4000) dev runner
|
│ ├── dev-all.js # Local concurrent Next.js (3000) + Ingestion (4000) dev runner
|
||||||
│ └── start-all.js # Production concurrent Next.js (`next start`) + Ingestion (`server/index.js`) runner
|
│ └── start-all.js # Production concurrent Next.js (`next start`) + Ingestion (`server/index.js`) runner
|
||||||
├── server/
|
├── server/
|
||||||
│ ├── index.js # Express 5 + Socket.IO (Port 4000), /health, /internal/* bridge, PG sync, graceful exit
|
│ ├── index.js # Express 5 + Socket.IO (Port 4000), /health, /internal/* bridge, PG sync, filesManager mount
|
||||||
|
│ ├── filesManager.js # Hard-locked cloudcmd@19.23.2 (/vcon/files-engine), isolated Socket.IO, Admin Cookie/DB/RBAC/CSRF/*.lock guard
|
||||||
│ ├── socket/handlers.js # Handshake JWT auth, ping_verify, sync_data, batch_sync
|
│ ├── socket/handlers.js # Handshake JWT auth, ping_verify, sync_data, batch_sync
|
||||||
│ └── workers/csvWriter.js # Partitioned CSV batch writer & rotation/pruning worker
|
│ └── workers/csvWriter.js # Partitioned CSV batch writer & rotation/pruning worker
|
||||||
├── sdk/python/datastack/ # Official pip-installable Python SDK (`datastack` v2.2.0)
|
├── sdk/python/datastack/ # Official pip-installable Python SDK (`datastack` v2.2.0)
|
||||||
@@ -146,16 +148,17 @@ datastack/
|
|||||||
│ ├── configuration-reference.md # Complete environment variable & setting bounds
|
│ ├── configuration-reference.md # Complete environment variable & setting bounds
|
||||||
│ ├── dependencies.md # Runtime & dev dependencies audit with rationales
|
│ ├── dependencies.md # Runtime & dev dependencies audit with rationales
|
||||||
│ └── glossary.md # Technical domain terminology & concept glossary
|
│ └── glossary.md # Technical domain terminology & concept glossary
|
||||||
├── notes/ # Production Remediation Governance & Scope-Lock Suite (`BASELINE-2026-11-20-R3`):
|
├── notes/ # `R4` Production Hardening Governance Suite (`BASELINE-2026-11-20-R4`):
|
||||||
│ ├── README.md # Master governance index, 6-step execution workflow & phase approval commands
|
│ ├── README.md # Master governance index, 6-step zero-drift protocol & approval triggers (`R4-Phase-01`..`04`)
|
||||||
│ ├── FORENSIC_AUDIT_REPORT.md # 100% raw-code verified audit (16 bugs: BUG-01..BUG-16) & Demo vs Prod matrix
|
│ ├── FULL_PROJECT_FORENSIC_AUDIT.md # 100% raw-source-verified deep scan (20 findings: 3 Critical, 6 High, 7 Medium, 4 Low)
|
||||||
│ ├── BASELINE_FREEZE.md # Frozen baseline snapshot (`BASELINE-2026-11-20-R3`) & 1:1 bug alignment
|
│ ├── BASELINE_FREEZE.md # Frozen baseline snapshot (`BASELINE-2026-11-20-R4`) & 1:1 audit alignment matrix
|
||||||
│ ├── PRODUCTION_READINESS_SCOPE_LOCK.md # (Doc 001) Exact bug fix contracts & per-phase Scope Locks (`R3-Phase-01`..`04`)
|
│ ├── PRODUCTION_READINESS_SCOPE_LOCK.md # (Doc 001) All 20 defect fix contracts & exact per-phase Scope Locks
|
||||||
│ ├── PRODUCTION_ROADMAP.md # (Doc 002) Dependency-ordered A–Z 4-phase roadmap & burn-down tracker
|
│ ├── PRODUCTION_ROADMAP.md # (Doc 002) Dependency-ordered A–Z 4-phase roadmap & verification gates
|
||||||
│ ├── PHASE_EXECUTION_LOG.md # (Doc 003) Live phase completion log (`0/4` complete in R3 cycle) & zero-drift guardrails
|
│ ├── PHASE_EXECUTION_LOG.md # (Doc 003) Live phase completion tracker (`4 / 4` complete) & changelog template
|
||||||
│ ├── ERROR_HANDLING_MATRIX.md # (Doc 004) Active error-handling catalog & 7 planned fixes (`ERR-R3-01`..`07`)
|
│ ├── ERROR_HANDLING_MATRIX.md # (Doc 004) Verified existing error contracts + 10 planned `R4` contracts (`ERR-R4-01`..`10`)
|
||||||
│ └── ACTUAL_IMPLEMENTATION_STATUS.md # Ground-truth working vs completed-per-phase vs remaining ledger (16 items remaining)
|
│ └── ACTUAL_IMPLEMENTATION_STATUS.md # Ground-truth working vs phase-delivered vs pending ledger (Demo & Production)
|
||||||
└── tests/ # Automated verification suites:
|
└── tests/ # Automated verification suites:
|
||||||
|
├── files-manager.test.ts # CloudCmd RCE lockdown, CRUD/ZIP/UNZIP, RBAC, CSRF, *.lock, DB revocation, proxy & UI wiring
|
||||||
├── phase01-auth.test.ts # Auth fail-closed, tokenVersion, brute-force lockout, Bearer revocation, CSRF
|
├── phase01-auth.test.ts # Auth fail-closed, tokenVersion, brute-force lockout, Bearer revocation, CSRF
|
||||||
├── phase02-sync.test.ts # Internal bridge auth, settings sync, JWT rotation kick, rate/security metrics
|
├── phase02-sync.test.ts # Internal bridge auth, settings sync, JWT rotation kick, rate/security metrics
|
||||||
├── phase03-storage-buffer.test.ts # Redis durability, cooperative CSV locks, streaming row count, export filters
|
├── phase03-storage-buffer.test.ts # Redis durability, cooperative CSV locks, streaming row count, export filters
|
||||||
@@ -377,6 +380,49 @@ All planned hardening & deployment phases (**Phase-01 through Phase-07**) are **
|
|||||||
- **`BUG-07` (`ERR-R3-05`)**: Implemented real `heartbeatIntervalMs` monitor in Node.js SDK (`startHeartbeatMonitor` with `.unref?.()`, zero-CPU `IDLE_WS` pause, automatic `ping_verify` recovery & spool drain, clean timer teardown, `status().heartbeatActive`); fixed single HTTP/WS `DUPLICATE_DROPPED` reporting (`accepted: 0, duplicates: 1`); wired `TOKEN_ROTATED` (`401`) and `CLIENT_REVOKED` (`403`) fast-fail `DataStackAuthError` across WS `connect_error`, `ping_verify`, `sync_data`, `batch_sync`, and HTTP fallback; compiled CommonJS `sdk/node/datastack/dist/{index,types,spool}.{js,d.ts}` via `tsconfig.json` and pointed `package.json` `main`/`types`/`exports` to `./dist/index.js` & `./dist/index.d.ts`.
|
- **`BUG-07` (`ERR-R3-05`)**: Implemented real `heartbeatIntervalMs` monitor in Node.js SDK (`startHeartbeatMonitor` with `.unref?.()`, zero-CPU `IDLE_WS` pause, automatic `ping_verify` recovery & spool drain, clean timer teardown, `status().heartbeatActive`); fixed single HTTP/WS `DUPLICATE_DROPPED` reporting (`accepted: 0, duplicates: 1`); wired `TOKEN_ROTATED` (`401`) and `CLIENT_REVOKED` (`403`) fast-fail `DataStackAuthError` across WS `connect_error`, `ping_verify`, `sync_data`, `batch_sync`, and HTTP fallback; compiled CommonJS `sdk/node/datastack/dist/{index,types,spool}.{js,d.ts}` via `tsconfig.json` and pointed `package.json` `main`/`types`/`exports` to `./dist/index.js` & `./dist/index.d.ts`.
|
||||||
- **`BUG-08` (`ERR-R3-06`)**: Added non-retryable `DataStackQuotaExceededError(DataStackError, retryable=False)` in `sdk/python/datastack/datastack/exceptions.py`, re-exported it in both `datastack` and `datastack_sdk`, and wired immediate fail-fast handling for HTTP `507` and WS `STORAGE_QUOTA_EXCEEDED` / `DISK_SPACE_LOW` in `sdk/python/datastack/datastack/client.py` with `0` retries, `0` HTTP fallback after WS quota rejection, and `0` offline spooling.
|
- **`BUG-08` (`ERR-R3-06`)**: Added non-retryable `DataStackQuotaExceededError(DataStackError, retryable=False)` in `sdk/python/datastack/datastack/exceptions.py`, re-exported it in both `datastack` and `datastack_sdk`, and wired immediate fail-fast handling for HTTP `507` and WS `STORAGE_QUOTA_EXCEEDED` / `DISK_SPACE_LOW` in `sdk/python/datastack/datastack/client.py` with `0` retries, `0` HTTP fallback after WS quota rejection, and `0` offline spooling.
|
||||||
- **Verified**: `13/13` Node SDK/packaging tests (`5/5` `tests/r3-phase03-sdk-parity.test.ts` against live Socket.IO handlers + `4/4` `v2-phase04-node-sdk.test.ts` + `4/4` `phase05-packaging.test.ts`), `14/14` Python SDK tests (`3/3` `tests/r3_phase03_python_sdk_test.py` + `11/11` `phase05_sdk_test.py`), `py_compile`, `tsc`, `npm run build` clean, `npm run lint` (`0` errors, `0` warnings). `0` Port 3000/4000/UI/Docker files modified.
|
- **Verified**: `13/13` Node SDK/packaging tests (`5/5` `tests/r3-phase03-sdk-parity.test.ts` against live Socket.IO handlers + `4/4` `v2-phase04-node-sdk.test.ts` + `4/4` `phase05-packaging.test.ts`), `14/14` Python SDK tests (`3/3` `tests/r3_phase03_python_sdk_test.py` + `11/11` `phase05_sdk_test.py`), `py_compile`, `tsc`, `npm run build` clean, `npm run lint` (`0` errors, `0` warnings). `0` Port 3000/4000/UI/Docker files modified.
|
||||||
|
- **`R3-Phase-04` Complete & Verified (`2026-11-20`)** (`BUG-11`, `BUG-12`, `BUG-14`, `BUG-16`; `ERR-R3-07`):
|
||||||
|
- **`BUG-11` (`ERR-R3-07`)**: Added `shouldRedirectSimulatorUnauthorized(status, body)` in `lib/vconUiHelpers.ts` and wired it post-JSON-parse across both single & batch HTTP sends in `app/vcon/simulator/page.tsx` so expired/rotated/revoked client `401` responses display inline without logging out the authenticated admin.
|
||||||
|
- **`BUG-12`**: Added browser-local `formatDateForLocalInput(iso)` in `lib/vconUiHelpers.ts` and replaced UTC `.toISOString().slice(0, 16)` inside `ClientsTable.tsx` `openEditModal()`, eliminating local timezone drift on client edits.
|
||||||
|
- **`BUG-14`**: Added `isUiClientExpired()`, `classifyUiClientStatus()`, and `summarizeUiClientCounts()` in `lib/vconUiHelpers.ts`; excluded expired clients from `activeClients` and exposed `expiredClients` in `core/apiEngine/apiService.ts` and `DashboardOverview.tsx`; emitted `datastack_clients_total{status="expired"}` and excluded expired clients from `status="active"` in `app/api/prometheus/route.ts`; expanded `/vcon/clients` with a 6-column `StatStrip` + 6-pill status filter (`Total`, `Online`, `Offline`, `Pending`, `Expired`, `Revoked`).
|
||||||
|
- **`BUG-16`**: Propagated verified session role from `app/vcon/layout.tsx` through `components/layout/VconShell.tsx` (`VconRoleContext` / `useVconRole()`) + `getVconRoleCapabilities()` (`lib/vconUiHelpers.ts`) to gate all role-restricted UI controls across `SettingsWorkspace`, `ClientsTable`, `/vcon/clients`, `DashboardOverview`, `/vcon/storage`, and `/vcon/simulator`.
|
||||||
|
- **Verified**: `29/29` TS unit/integration tests (`4/4` `r3-phase04-ui-metrics-rbac.test.ts` + `15/15` `phase04-ui-sync.test.ts` + `5/5` `v2-phase02-alerts-ip-prometheus.test.ts` + `5/5` `v2-phase03-rbac-audit.test.ts`), live headless Chromium (`playwright-cli`, `1440x900`) verification across `/vcon/dashboard`, `/vcon/clients`, `/vcon/simulator`, `/vcon/storage`, and `/vcon/settings` (`0` JS/React pageerrors), `npm run build` clean, `npm run lint` (`0` errors, `0` warnings). `0` Port 4000/SDK/Auth/Storage/Docker files modified. **All `4 / 4` R3 phases (`16 / 16` bugs) complete.**
|
||||||
|
- **Full-Project Production Forensic Audit & `R4` Baseline Freeze (`BASELINE-2026-11-20-R4`)**:
|
||||||
|
- Cleaned all 8 prior Files Manager files from `notes/` and created the 100% raw-source-verified `notes/FULL_PROJECT_FORENSIC_AUDIT.md` (**20** verified findings: `3` Critical, `6` High, `7` Medium, `4` Low — `19` actionable + `1` verified intentional preview `AUDIT-17`).
|
||||||
|
- Froze the untouched codebase as `BASELINE-2026-11-20-R4` and created the complete 8-file `R4` governance & Scope-Lock suite in `notes/` (`README.md`, `FULL_PROJECT_FORENSIC_AUDIT.md`, `BASELINE_FREEZE.md`, `PRODUCTION_READINESS_SCOPE_LOCK.md`, `PRODUCTION_ROADMAP.md`, `PHASE_EXECUTION_LOG.md`, `ERROR_HANDLING_MATRIX.md`, `ACTUAL_IMPLEMENTATION_STATUS.md`) across `4` Scope-Locked phases (`R4-Phase-01`..`R4-Phase-04`).
|
||||||
|
- **`R4-Phase-01` Complete & Verified (`2026-11-21`)** (`AUDIT-01`, `AUDIT-03`, `AUDIT-04`, `AUDIT-09`; `ERR-R4-01`–`ERR-R4-05`):
|
||||||
|
- **`AUDIT-01` (`ERR-R4-01`)**: Created shared SSRF validator `core/Security/ssrfGuard.ts` blocking non-HTTP(S) schemes, embedded credentials, loopback, RFC1918/CGNAT/ULA, `0.0.0.0/8`, cloud metadata (`169.254.0.0/16`, `fe80::/10`, `metadata.google.internal`), bare internal hostnames, and DNS-rebinding targets, plus `redirect: "manual"` 3xx rejection; enforced uniformly across `core/alerts/webhookNotifier.ts`, `core/Storage/s3Archive.ts`, `app/api/settings/route.ts`, `app/api/settings/webhook-test/route.ts`, `app/api/storage/s3-test/route.ts`, and `server/index.js` `triggerWebhookAlert`, with `DATASTACK_ALLOW_PRIVATE_URLS=true` documented in `.env.example` (metadata still unconditionally blocked).
|
||||||
|
- **`AUDIT-03` (`ERR-R4-02`)**: Mirrored `admin_team_members` and `admin_audit_logs` DDL 1:1 into `server/index.js` `ensureDatabaseSchema()` for full 7-table parity across Port `3000` and Port `4000` on fresh PostgreSQL boots.
|
||||||
|
- **`AUDIT-04` (`ERR-R4-03`)**: Added Next.js `instrumentation.ts` (`register()` on `NEXT_RUNTIME === "nodejs"`) invoking `SettingsManager.bootstrapMaintenanceSchedulerOnBoot()` to hydrate settings and start the periodic storage maintenance scheduler automatically on Port `3000` startup.
|
||||||
|
- **`AUDIT-09` (`ERR-R4-04`, `ERR-R4-05`)**: Added `console.warn` + `storageMode`, `degraded: true`, `code: "AUDIT_STORAGE_DEGRADED"`, `warning`, and `X-DataStack-Audit-Degraded: 1` header in `core/auth/auditLog.ts` & `GET /api/audit`; wired `POST /internal/admin-session-sync` in `server/index.js`, `syncInMemoryAdminSessionState` / `checkInMemorySessionRevocation` in `server/filesManager.js`, and `notifyIngestionAdminSessionSync` in `core/auth/adminUsers.ts` so `!dbPool` Files Manager sessions revoke immediately on logout/disable/delete/credential rotation.
|
||||||
|
- **Verified**: `5/5` `tests/r4-phase01-security-boot.test.ts` + `85/85` regression tests (`phase01..04`, `phase07`, `files-manager`, `r3-phase01`, `r3-phase02`, `v2-phase03`, `v2-phase04` = `90/90` PASS), `npm run build`, `npm run lint` (`0` errors). `0` forbidden UI/SDK/Docker/ingestion-batch files modified.
|
||||||
|
- **`R4-Phase-02` Complete & Verified (`2026-11-21`)** (`AUDIT-05`, `AUDIT-06`, `AUDIT-07`, `AUDIT-10`, `AUDIT-11`, `AUDIT-12`; `ERR-R4-06`–`ERR-R4-08`):
|
||||||
|
- **`AUDIT-05` (`ERR-R4-06`)**: Enforced `DATASTACK_MAX_BATCH_ITEMS` (default `1000`, clamped `1..10000`, documented in `.env.example`) across `POST /api/events` (`400 BATCH_LIMIT_EXCEEDED`, evaluated before storage admission scan) and Socket.IO `batch_sync` (`server/socket/handlers.js`).
|
||||||
|
- **`AUDIT-06`**: Added `FILES_MANAGER_MUTATION` and `DATA_EXPORT_DOWNLOADED` to `core/auth/auditLog.ts` and `ACTION_FILTER_OPTIONS` in `components/sections/AuditLogViewer.tsx`; wired export download audit logs in `app/api/export/route.ts` and CloudCmd HTTP/Socket mutation audit logs (`admin_audit_logs` PG + RAM fallback) in `server/filesManager.js`.
|
||||||
|
- **`AUDIT-07` (`ERR-R4-07`)**: Isolated corrupt `.csv.gz` decompression failures inside `GET /api/data` (`app/api/data/route.ts`) so healthy client partitions return HTTP `200` alongside `corruptPartitions` (`CORRUPT_GZIP_PARTITION`) and `warnings`.
|
||||||
|
- **`AUDIT-10`**: Added `serializeClientResponse()` in `app/api/clients/[id]/route.ts` stripping `secret` and non-rotation `token` from `PATCH /api/clients/[id]` responses while exposing `tokenPreview` (and full `token` only on `action === "regenerate_token"`).
|
||||||
|
- **`AUDIT-11`**: Made `POST /internal/client-sync` (`action === "updated"`) in `server/index.js` disconnect live sockets only on security-relevant updates (`status === "revoked"`, expired `expiresAt`, token rotation, `allowedIps` change, or live socket credential mismatch), keeping metadata-only updates (`name`, `rateLimit`, `storageQuotaMb`) connected without disruption.
|
||||||
|
- **`AUDIT-12` (`ERR-R4-08`)**: Converted `uploadFileToS3` (`core/Storage/s3Archive.ts`) to streaming SHA-256 pre-pass + `Readable.toWeb(fs.createReadStream(...))` (`duplex: "half"`) with `.destroy()` teardown, and hardened `CSVStorageManager.openPartitionTextStream` + `countCsvDataRowsStreamAsync` (`public static`) with `for await` + `.destroy()` cleanup.
|
||||||
|
- **Verified**: `5/5` `tests/r4-phase02-ingestion-audit-streaming.test.ts` PASS, regression suites (`r4-phase01`, `files-manager`, `phase02-04`, `v2-phase02-03`, `r3-phase02`, `r3-phase04`) PASS, Playwright `/vcon/audit` (`0` console errors, new action filter options visible), `npm run build`, and `npm run lint` (`0` errors). `0` forbidden SDK/Docker/Nginx/middleware/auth/unrelated UI files touched.
|
||||||
|
- **`R4-Phase-03` Complete & Verified (`2026-11-21`)** (`AUDIT-02`, `AUDIT-08`, `AUDIT-13`, `AUDIT-20`; `ERR-R4-09`):
|
||||||
|
- **`AUDIT-02`**: Added `"prepare": "npm run build"` and `"devDependencies": { "typescript": "^5.9.3" }` to `sdk/node/datastack/package.json` and compiled `sdk/node/datastack/dist/{index,spool,types}.{js,d.ts}` so CommonJS `require('./sdk/node/datastack')` works cleanly out of the box.
|
||||||
|
- **`AUDIT-20`**: Added atomic `DiskSpooler.ackAndRequeueFront(ackedCount, unprocessed)` in `sdk/node/datastack/spool.ts` and wired `deliverChunkWithSplitAndRetry()` + `flushSpool()` in `sdk/node/datastack/index.ts` (with `dist/` rebuilt) so partial-batch rate-limit recoveries commit acknowledged events without dropping or reordering unprocessed envelopes ahead of tail items.
|
||||||
|
- **`AUDIT-08` (`ERR-R4-09`)**: Added `DATASTACK_ADMIN_EMAIL: "${DATASTACK_ADMIN_EMAIL:-admin@datastack.dev}"` to `services.ingestion.environment` in `docker/docker-compose.yml`; added `proxyMode`/`wsSupported`/`wsHint` detection in `GET /api/files/status` (`app/api/files/status/route.ts`) and rendered a non-blocking `[data-testid="files-manager-proxy-hint"]` strip in `components/sections/FilesManagerWorkspace.tsx` when running standalone without Nginx WebSocket proxying.
|
||||||
|
- **`AUDIT-13`**: Removed dead `/api/auth/setup/status` whitelist line from `middleware.ts` and passed explicit `role: "owner"`, `memberId: "owner-1"` arguments to `createAdminSessionToken()` in `app/api/auth/setup/route.ts`.
|
||||||
|
- **Verified**: `4/4` `tests/r4-phase03-sdk-deploy-auth.test.ts` + `43/43` regression checks (`47/47` total PASS across `v2-phase04-node-sdk`, `r3-phase03-sdk-parity`, `phase05-packaging`, `phase07-deployment`, `phase01-auth`, `files-manager`), `node -e "require('./sdk/node/datastack')"` PASS (`2.2.0`), Playwright `/vcon/files` (`0` console errors), `npm run build`, and `npm run lint` (`0` errors). `0` forbidden Storage/Webhook/Client/Team/Profile/Landing files modified.
|
||||||
|
- **`R4-Phase-04` Complete & Verified (`2026-11-21`)** (`AUDIT-14`, `AUDIT-15`, `AUDIT-16`, `AUDIT-18`, `AUDIT-19`; `ERR-R4-10`, plus `AUDIT-07` non-blocking `corruptPartitions` UI notice):
|
||||||
|
- **`AUDIT-14`**: Updated `components/sections/TeamWorkspace.tsx` to initialize role from `useVconRole()` (`shellRole || 'viewer'`), preserve distinct `operator` and `admin` `<Select>` options and row values without coercing `operator` to `admin`, label StatStrip `'Operators / Admins'`, and require an explicit `<Modal>` (`"Confirm Team Member Removal"`, `[data-testid="confirm-remove-member"]`) before `DELETE /api/team/${id}`.
|
||||||
|
- **`AUDIT-15` (`ERR-R4-10`)**: Wired `useVconRole()` + `getVconRoleCapabilities(shellRole)` (`canOperate`) in `app/vcon/sdk/page.tsx` so `viewer` sessions skip `?includeToken=true` requests cleanly (`0` HTTP `403`s) and display inline `[data-testid="sdk-viewer-rbac-banner"]` RBAC guidance while preserving existing `loadingToken` disabled guards.
|
||||||
|
- **`AUDIT-16`**: Seeded `components/layout/Sidebar.tsx` role badge from `useVconRole()` (`formatShellRoleLabel()`), renamed `'Python SDK'` to `'Client SDKs'`, and added `components/layout/Navbar.tsx` mobile `VCON_LABELS` for `'/vcon/files'` (`'Files Manager'`), `'/vcon/sdk'` (`'Client SDKs'`), `'/vcon/team'` (`'Team & RBAC'`), and `'/vcon/audit'` (`'Audit Log'`).
|
||||||
|
- **`AUDIT-19`**: Replaced fake Demo fallback constants in `components/sections/ProfileWorkspace.tsx` with neutral `EMPTY_PROFILE` placeholders (`'Loading...'` / `'—'`) until `/api/auth/profile` resolves.
|
||||||
|
- **`AUDIT-07` UI Hook**: Surfaced non-blocking `[data-testid="data-corrupt-partitions-warning"]` banners inside `components/sections/DataBrowser.tsx` when `GET /api/data` reports `corruptPartitions`.
|
||||||
|
- **`AUDIT-18`**: Aligned `components/sections/Hero.tsx`, `components/sections/SdkSection.tsx`, and `components/sections/LandingAbout.tsx` around `'Client SDKs'` with side-by-side Python (`datastack v2.2.0`) and Node.js / TypeScript (`@datastack/sdk v2.2.0`) quick-start cards.
|
||||||
|
- **Verified**: `5/5` `tests/r4-phase04-ui-rbac-copy.test.ts` + `46/46` regression checks (`51/51` total PASS), live headless Chromium desktop (`1440x900`) & mobile (`390x844`) verification across `/`, `/vcon/team`, `/vcon/profile`, `/vcon/data`, `/vcon/files`, `/vcon/audit`, `/vcon/sdk` (`0` console/page errors, `0` viewer `403`s), `npm run build`, and `npm run lint` (`0` errors). `0` forbidden backend/SDK/Docker/Nginx files modified. **All `4 / 4` R4 phases (`19 / 19` actionable findings) complete.**
|
||||||
|
|
||||||
|
- **`PREPUSH-FIX-01` Complete & Verified (`2026-11-21`)** (`DEPLOY-BLK-01`, `DEPLOY-BUG-02`–`05`):
|
||||||
|
- **`DEPLOY-BLK-01`**: Resynchronized `package-lock.json` (`lockfileVersion: 3`) with `package.json` (`cloudcmd ^19.23.2` + full dependency tree) and verified `npm ci --include=dev --dry-run` (`Dockerfile.web`) and `npm ci --omit=dev --dry-run` (`Dockerfile.server`) both exit `0`.
|
||||||
|
- **`DEPLOY-BUG-02`**: Hardened `docker/coolify-nginx.conf` and `nginx/default.conf` with `ngx_http_realip_module` (`set_real_ip_from` loopback/RFC1918/Docker bridges + IPv6 ULA, `real_ip_header X-Forwarded-For`, `real_ip_recursive on`) so `proxy_set_header X-Real-IP $remote_addr` preserves real external client IPs behind Coolify Traefik without permitting untrusted public IP spoofing.
|
||||||
|
- **`DEPLOY-BUG-03` & `DEPLOY-BUG-04`**: Added `ARG INGESTION_INTERNAL_URL=http://ingestion:4000` + `ENV INGESTION_INTERNAL_URL` in `docker/Dockerfile.web` `builder` stage for standalone Next.js rewrites, and forwarded optional `DATASTACK_ALLOW_PRIVATE_URLS`, `DATASTACK_MAX_BATCH_ITEMS`, `DATASTACK_RECENT_LIMIT`, `DATASTACK_MAINTENANCE_INTERVAL_MS`, and `DATASTACK_PROXY_MODE` env vars in `docker/docker-compose.yml`.
|
||||||
|
- **`DEPLOY-BUG-05`**: Updated `sdk/node/datastack/index.ts` (`emitSocketSingle` and `sendBatchChunkInternal`) to call `this.disconnect()` on terminal `DataStackAuthError` rejections (`TOKEN_ROTATED`, `CLIENT_REVOKED`, etc.) and recompiled `sdk/node/datastack/dist/*` so subsequent `client.connect()` performs a fresh handshake (`tests/r3-phase03-sdk-parity.test.ts` `5/5` PASS).
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -384,4 +430,5 @@ All planned hardening & deployment phases (**Phase-01 through Phase-07**) are **
|
|||||||
- Never add per-row telemetry `INSERT` queries to PostgreSQL or add `DROP`/`TRUNCATE` auto-migrations.
|
- Never add per-row telemetry `INSERT` queries to PostgreSQL or add `DROP`/`TRUNCATE` auto-migrations.
|
||||||
- Never move Socket.IO handlers into Next.js `app/api/*` routes.
|
- Never move Socket.IO handlers into Next.js `app/api/*` routes.
|
||||||
- When deploying updates to an existing Coolify VPS, always preserve existing `SERVICE_*` environment variables and avoid `docker compose down -v` so `pgdata`, `datastack-data`, and `redisdata` volumes remain intact.
|
- When deploying updates to an existing Coolify VPS, always preserve existing `SERVICE_*` environment variables and avoid `docker compose down -v` so `pgdata`, `datastack-data`, and `redisdata` volumes remain intact.
|
||||||
- After any code change, verify with `npm run build`, `npm run lint`, the TS `tests/phase*.test.ts` suite (`npx --yes tsx --test`), and `python3 -m unittest tests/phase05_sdk_test.py`, then synchronize this `MEMORY.md`.
|
- After any code change, verify with `npm run build`, `npm run lint`, `npm ci --dry-run`, and the test suites, then synchronize this `MEMORY.md`.
|
||||||
|
|
||||||
+13
-5
@@ -1,4 +1,4 @@
|
|||||||
import { listAdminAuditLogsAsync } from "@/core/auth/auditLog";
|
import { listAdminAuditLogsWithStatusAsync } from "@/core/auth/auditLog";
|
||||||
import { requireAdminApiSession } from "@/lib/adminAuth";
|
import { requireAdminApiSession } from "@/lib/adminAuth";
|
||||||
import { NextRequest, NextResponse } from "next/server";
|
import { NextRequest, NextResponse } from "next/server";
|
||||||
|
|
||||||
@@ -22,17 +22,25 @@ export async function GET(req: NextRequest) {
|
|||||||
const search =
|
const search =
|
||||||
searchParams.get("q") || searchParams.get("search") || undefined;
|
searchParams.get("q") || searchParams.get("search") || undefined;
|
||||||
|
|
||||||
const logs = await listAdminAuditLogsAsync({
|
const result = await listAdminAuditLogsWithStatusAsync({
|
||||||
limit,
|
limit,
|
||||||
action,
|
action,
|
||||||
actorEmail,
|
actorEmail,
|
||||||
search,
|
search,
|
||||||
});
|
});
|
||||||
|
|
||||||
return NextResponse.json({
|
const response = NextResponse.json({
|
||||||
logs,
|
logs: result.logs,
|
||||||
total: logs.length,
|
total: result.logs.length,
|
||||||
|
storageMode: result.storageMode,
|
||||||
|
degraded: result.degraded,
|
||||||
|
...(result.code ? { code: result.code } : {}),
|
||||||
|
...(result.warning ? { warning: result.warning } : {}),
|
||||||
});
|
});
|
||||||
|
if (result.degraded) {
|
||||||
|
response.headers.set("X-DataStack-Audit-Degraded", "1");
|
||||||
|
}
|
||||||
|
return response;
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
const message =
|
const message =
|
||||||
err instanceof Error ? err.message : "Unable to fetch audit logs.";
|
err instanceof Error ? err.message : "Unable to fetch audit logs.";
|
||||||
|
|||||||
@@ -145,6 +145,8 @@ export async function POST(request: NextRequest) {
|
|||||||
sessionSecret,
|
sessionSecret,
|
||||||
Date.now() + SESSION_SECONDS * 1000,
|
Date.now() + SESSION_SECONDS * 1000,
|
||||||
1,
|
1,
|
||||||
|
"owner",
|
||||||
|
"owner-1",
|
||||||
);
|
);
|
||||||
const response = NextResponse.json({ success: true });
|
const response = NextResponse.json({ success: true });
|
||||||
response.cookies.set(
|
response.cookies.set(
|
||||||
|
|||||||
@@ -17,6 +17,18 @@ const ALLOWED_PATCH_KEYS = new Set([
|
|||||||
"status",
|
"status",
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
function serializeClientResponse<
|
||||||
|
T extends { secret?: string; token?: string },
|
||||||
|
>(client: T, options: { includeToken: boolean }) {
|
||||||
|
const { secret, token, ...clientSummary } = client;
|
||||||
|
void secret;
|
||||||
|
return {
|
||||||
|
...clientSummary,
|
||||||
|
tokenPreview: token ? `${token.slice(0, 12)}...` : "",
|
||||||
|
...(options.includeToken && token ? { token } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export async function GET(
|
export async function GET(
|
||||||
req: NextRequest,
|
req: NextRequest,
|
||||||
{ params }: { params: Promise<{ id: string }> },
|
{ params }: { params: Promise<{ id: string }> },
|
||||||
@@ -36,14 +48,8 @@ export async function GET(
|
|||||||
if (!client) {
|
if (!client) {
|
||||||
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
||||||
}
|
}
|
||||||
const { secret, token, ...clientSummary } = client;
|
|
||||||
void secret;
|
|
||||||
return NextResponse.json({
|
return NextResponse.json({
|
||||||
client: {
|
client: serializeClientResponse(client, { includeToken }),
|
||||||
...clientSummary,
|
|
||||||
tokenPreview: token ? `${token.slice(0, 12)}...` : "",
|
|
||||||
...(includeToken ? { token } : {}),
|
|
||||||
},
|
|
||||||
});
|
});
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
const message = err instanceof Error ? err.message : "Failed to load client";
|
const message = err instanceof Error ? err.message : "Failed to load client";
|
||||||
@@ -132,7 +138,9 @@ export async function PATCH(
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
return NextResponse.json({ client: refreshed });
|
return NextResponse.json({
|
||||||
|
client: serializeClientResponse(refreshed, { includeToken: true }),
|
||||||
|
});
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
const message =
|
const message =
|
||||||
err instanceof Error ? err.message : "Token regeneration failed";
|
err instanceof Error ? err.message : "Token regeneration failed";
|
||||||
@@ -277,7 +285,9 @@ export async function PATCH(
|
|||||||
metadata: updates,
|
metadata: updates,
|
||||||
});
|
});
|
||||||
|
|
||||||
return NextResponse.json({ client: updated });
|
return NextResponse.json({
|
||||||
|
client: serializeClientResponse(updated, { includeToken: false }),
|
||||||
|
});
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
const message = err instanceof Error ? err.message : "Update failed";
|
const message = err instanceof Error ? err.message : "Update failed";
|
||||||
const isDbWrite = message.includes("DATABASE_WRITE_FAILED");
|
const isDbWrite = message.includes("DATABASE_WRITE_FAILED");
|
||||||
|
|||||||
+16
-1
@@ -17,6 +17,13 @@ interface DataEvent {
|
|||||||
status: "stored" | "buffered";
|
status: "stored" | "buffered";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface CorruptPartitionWarning {
|
||||||
|
clientId: string;
|
||||||
|
filename: string;
|
||||||
|
code: "CORRUPT_GZIP_PARTITION";
|
||||||
|
error: string;
|
||||||
|
}
|
||||||
|
|
||||||
function parseTimestampMs(raw: string): number {
|
function parseTimestampMs(raw: string): number {
|
||||||
const trimmed = String(raw || "").trim();
|
const trimmed = String(raw || "").trim();
|
||||||
if (/^\d+(\.\d+)?$/.test(trimmed)) {
|
if (/^\d+(\.\d+)?$/.test(trimmed)) {
|
||||||
@@ -62,6 +69,7 @@ export async function GET(request: NextRequest) {
|
|||||||
const offset = (page - 1) * pageSize;
|
const offset = (page - 1) * pageSize;
|
||||||
const selectedRows: DataEvent[] = [];
|
const selectedRows: DataEvent[] = [];
|
||||||
const eventIds = new Set<string>();
|
const eventIds = new Set<string>();
|
||||||
|
const corruptPartitions: CorruptPartitionWarning[] = [];
|
||||||
let total = 0;
|
let total = 0;
|
||||||
|
|
||||||
const addIfMatching = (event: DataEvent) => {
|
const addIfMatching = (event: DataEvent) => {
|
||||||
@@ -185,7 +193,12 @@ export async function GET(request: NextRequest) {
|
|||||||
console.warn(`[API /data] Failed reading partition ${filePath}:`, err);
|
console.warn(`[API /data] Failed reading partition ${filePath}:`, err);
|
||||||
if (filename.endsWith(".csv.gz")) {
|
if (filename.endsWith(".csv.gz")) {
|
||||||
const detail = err instanceof Error ? err.message : String(err);
|
const detail = err instanceof Error ? err.message : String(err);
|
||||||
throw new Error(`Corrupt .csv.gz partition (${filename}): ${detail}`);
|
corruptPartitions.push({
|
||||||
|
clientId: storedClientId,
|
||||||
|
filename,
|
||||||
|
code: "CORRUPT_GZIP_PARTITION",
|
||||||
|
error: `Corrupt .csv.gz partition (${filename}): ${detail}`,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
if (!parser.destroyed) parser.destroy();
|
if (!parser.destroyed) parser.destroy();
|
||||||
@@ -249,6 +262,8 @@ export async function GET(request: NextRequest) {
|
|||||||
page,
|
page,
|
||||||
pageSize,
|
pageSize,
|
||||||
pageCount: Math.max(1, Math.ceil(total / pageSize)),
|
pageCount: Math.max(1, Math.ceil(total / pageSize)),
|
||||||
|
corruptPartitions,
|
||||||
|
warnings: corruptPartitions.length,
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
const message =
|
const message =
|
||||||
|
|||||||
+46
-11
@@ -25,6 +25,21 @@ const SINGLE_EVENT_ENVELOPE_KEYS = new Set(['eventId', 'timestamp', 'data', 'cli
|
|||||||
const MAX_SINGLE_CLIENT_SCAN_FILES = 10;
|
const MAX_SINGLE_CLIENT_SCAN_FILES = 10;
|
||||||
const MAX_ALL_CLIENTS_PER_CLIENT_FILES = 2;
|
const MAX_ALL_CLIENTS_PER_CLIENT_FILES = 2;
|
||||||
const MAX_ALL_CLIENTS_TOTAL_SCAN_FILES = 50;
|
const MAX_ALL_CLIENTS_TOTAL_SCAN_FILES = 50;
|
||||||
|
const DEFAULT_MAX_BATCH_ITEMS = 1000;
|
||||||
|
const MIN_BATCH_ITEMS = 1;
|
||||||
|
const MAX_BATCH_ITEMS_CEILING = 10000;
|
||||||
|
|
||||||
|
function resolveMaxBatchItems(): number {
|
||||||
|
const raw = process.env.DATASTACK_MAX_BATCH_ITEMS;
|
||||||
|
if (raw === undefined || raw.trim() === '') {
|
||||||
|
return DEFAULT_MAX_BATCH_ITEMS;
|
||||||
|
}
|
||||||
|
const parsed = Math.floor(Number(raw));
|
||||||
|
if (!Number.isFinite(parsed)) {
|
||||||
|
return DEFAULT_MAX_BATCH_ITEMS;
|
||||||
|
}
|
||||||
|
return Math.min(MAX_BATCH_ITEMS_CEILING, Math.max(MIN_BATCH_ITEMS, parsed));
|
||||||
|
}
|
||||||
|
|
||||||
function generateFallbackEventId(): string {
|
function generateFallbackEventId(): string {
|
||||||
return `evt_${Date.now()}_${crypto.randomBytes(6).toString('hex')}`;
|
return `evt_${Date.now()}_${crypto.randomBytes(6).toString('hex')}`;
|
||||||
@@ -513,6 +528,36 @@ export async function POST(req: NextRequest) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Detect if this is a batch array or direct file-records ingestion request
|
||||||
|
const batchItems: unknown[] | null = Array.isArray(body)
|
||||||
|
? body
|
||||||
|
: Array.isArray(bodyObj.batch)
|
||||||
|
? (bodyObj.batch as unknown[])
|
||||||
|
: Array.isArray(bodyObj.records)
|
||||||
|
? (bodyObj.records as unknown[])
|
||||||
|
: bodyObj.data === undefined && Array.isArray(bodyObj.events)
|
||||||
|
? (bodyObj.events as unknown[])
|
||||||
|
: null;
|
||||||
|
|
||||||
|
if (batchItems !== null) {
|
||||||
|
const maxBatchItems = resolveMaxBatchItems();
|
||||||
|
if (batchItems.length > maxBatchItems) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
success: false,
|
||||||
|
accepted: 0,
|
||||||
|
processed: 0,
|
||||||
|
rejected: batchItems.length,
|
||||||
|
duplicates: 0,
|
||||||
|
code: 'BATCH_LIMIT_EXCEEDED',
|
||||||
|
error: `BATCH_LIMIT_EXCEEDED: Batch size (${batchItems.length}) exceeds maximum allowed ${maxBatchItems} items per request.`,
|
||||||
|
maxBatchItems,
|
||||||
|
},
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const storageAdmission = await CSVStorageManager.checkStorageAdmissionAsync(
|
const storageAdmission = await CSVStorageManager.checkStorageAdmissionAsync(
|
||||||
clientId,
|
clientId,
|
||||||
registeredClient.storageQuotaMb,
|
registeredClient.storageQuotaMb,
|
||||||
@@ -550,18 +595,8 @@ export async function POST(req: NextRequest) {
|
|||||||
settings.maxEventsPerSec ||
|
settings.maxEventsPerSec ||
|
||||||
100;
|
100;
|
||||||
|
|
||||||
// Detect if this is a batch array or direct file-records ingestion request
|
|
||||||
const batchItems: unknown[] | null = Array.isArray(body)
|
|
||||||
? body
|
|
||||||
: Array.isArray(bodyObj.batch)
|
|
||||||
? (bodyObj.batch as unknown[])
|
|
||||||
: Array.isArray(bodyObj.records)
|
|
||||||
? (bodyObj.records as unknown[])
|
|
||||||
: bodyObj.data === undefined && Array.isArray(bodyObj.events)
|
|
||||||
? (bodyObj.events as unknown[])
|
|
||||||
: null;
|
|
||||||
|
|
||||||
if (batchItems !== null) {
|
if (batchItems !== null) {
|
||||||
|
|
||||||
let processed = 0;
|
let processed = 0;
|
||||||
let rejected = 0;
|
let rejected = 0;
|
||||||
let duplicates = 0;
|
let duplicates = 0;
|
||||||
|
|||||||
+40
-2
@@ -1,4 +1,5 @@
|
|||||||
import { SettingsManager } from "@/core/adminSettings/settingsManager";
|
import { SettingsManager } from "@/core/adminSettings/settingsManager";
|
||||||
|
import { recordAdminAuditLogAsync } from "@/core/auth/auditLog";
|
||||||
import { DataBufferEngine } from "@/core/data/buffer";
|
import { DataBufferEngine } from "@/core/data/buffer";
|
||||||
import { SDKManager } from "@/core/sdk/sdkManager";
|
import { SDKManager } from "@/core/sdk/sdkManager";
|
||||||
import { getInternalIngestionHeaders } from "@/core/Security/security";
|
import { getInternalIngestionHeaders } from "@/core/Security/security";
|
||||||
@@ -6,7 +7,7 @@ import {
|
|||||||
CSVStorageManager,
|
CSVStorageManager,
|
||||||
type StorageFileMeta,
|
type StorageFileMeta,
|
||||||
} from "@/core/Storage/csvStorage";
|
} from "@/core/Storage/csvStorage";
|
||||||
import { requireAdminApiSession } from "@/lib/adminAuth";
|
import { extractClientIp, requireAdminApiSession } from "@/lib/adminAuth";
|
||||||
import { parse } from "csv-parse";
|
import { parse } from "csv-parse";
|
||||||
import { NextRequest, NextResponse } from "next/server";
|
import { NextRequest, NextResponse } from "next/server";
|
||||||
import { createReadStream } from "node:fs";
|
import { createReadStream } from "node:fs";
|
||||||
@@ -296,6 +297,24 @@ export async function GET(request: NextRequest) {
|
|||||||
: file.compressed && includeHeader
|
: file.compressed && includeHeader
|
||||||
? "application/gzip"
|
? "application/gzip"
|
||||||
: "text/csv; charset=utf-8";
|
: "text/csv; charset=utf-8";
|
||||||
|
|
||||||
|
await recordAdminAuditLogAsync({
|
||||||
|
actorEmail: auth.auth.session.email,
|
||||||
|
actorRole: auth.auth.session.role || "owner",
|
||||||
|
action: "DATA_EXPORT_DOWNLOADED",
|
||||||
|
target: file.clientId,
|
||||||
|
ipAddress: extractClientIp(request),
|
||||||
|
metadata: {
|
||||||
|
mode: "single_file",
|
||||||
|
clientId: file.clientId,
|
||||||
|
filename: file.filename,
|
||||||
|
downloadName,
|
||||||
|
format,
|
||||||
|
includeHeader,
|
||||||
|
fileCount: 1,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
return new NextResponse(bodyStream, {
|
return new NextResponse(bodyStream, {
|
||||||
status: 200,
|
status: 200,
|
||||||
headers: {
|
headers: {
|
||||||
@@ -313,6 +332,25 @@ export async function GET(request: NextRequest) {
|
|||||||
const safeClientId = sanitizedClientId === "all" ? "all" : sanitizedClientId;
|
const safeClientId = sanitizedClientId === "all" ? "all" : sanitizedClientId;
|
||||||
const baseName =
|
const baseName =
|
||||||
clientId === "all" ? "datastack_export" : `${safeClientId}_export`;
|
clientId === "all" ? "datastack_export" : `${safeClientId}_export`;
|
||||||
|
const downloadName = `${baseName}_${datePart}.${format}`;
|
||||||
|
|
||||||
|
await recordAdminAuditLogAsync({
|
||||||
|
actorEmail: auth.auth.session.email,
|
||||||
|
actorRole: auth.auth.session.role || "owner",
|
||||||
|
action: "DATA_EXPORT_DOWNLOADED",
|
||||||
|
target: clientId,
|
||||||
|
ipAddress: extractClientIp(request),
|
||||||
|
metadata: {
|
||||||
|
mode: "filtered_range",
|
||||||
|
clientId,
|
||||||
|
from: from || null,
|
||||||
|
to: to || null,
|
||||||
|
format,
|
||||||
|
includeHeader,
|
||||||
|
fileCount: files.length,
|
||||||
|
filename: downloadName,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
return new NextResponse(makeExportStream(files, format, includeHeader), {
|
return new NextResponse(makeExportStream(files, format, includeHeader), {
|
||||||
status: 200,
|
status: 200,
|
||||||
@@ -323,7 +361,7 @@ export async function GET(request: NextRequest) {
|
|||||||
: format === "ndjson"
|
: format === "ndjson"
|
||||||
? "application/x-ndjson; charset=utf-8"
|
? "application/x-ndjson; charset=utf-8"
|
||||||
: "text/csv; charset=utf-8",
|
: "text/csv; charset=utf-8",
|
||||||
"Content-Disposition": `attachment; filename="${baseName}_${datePart}.${format}"`,
|
"Content-Disposition": `attachment; filename="${downloadName}"`,
|
||||||
"Cache-Control": "no-store",
|
"Cache-Control": "no-store",
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,115 @@
|
|||||||
|
import { requireAdminApiSession } from '@/lib/adminAuth';
|
||||||
|
import { NextRequest, NextResponse } from 'next/server';
|
||||||
|
|
||||||
|
export const dynamic = 'force-dynamic';
|
||||||
|
|
||||||
|
function resolveIngestionBaseUrl(): string {
|
||||||
|
return (process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000').replace(
|
||||||
|
/\/+$/,
|
||||||
|
'',
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveFilesProxyStatus(req: NextRequest): {
|
||||||
|
proxyMode: 'nginx' | 'standalone';
|
||||||
|
wsSupported: boolean;
|
||||||
|
wsHint: string | null;
|
||||||
|
} {
|
||||||
|
const configuredMode = process.env.DATASTACK_PROXY_MODE?.trim().toLowerCase();
|
||||||
|
const hasReverseProxyHeaders = Boolean(
|
||||||
|
req.headers.get('x-forwarded-host') ||
|
||||||
|
req.headers.get('x-real-ip') ||
|
||||||
|
req.headers.get('x-forwarded-proto'),
|
||||||
|
);
|
||||||
|
if (configuredMode === 'nginx' || hasReverseProxyHeaders) {
|
||||||
|
return {
|
||||||
|
proxyMode: 'nginx',
|
||||||
|
wsSupported: true,
|
||||||
|
wsHint: null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
proxyMode: 'standalone',
|
||||||
|
wsSupported: false,
|
||||||
|
wsHint:
|
||||||
|
'Direct Next.js HTTP rewrite active without Nginx WebSocket proxying. Files Manager operates over HTTP polling.',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function GET(req: NextRequest) {
|
||||||
|
const auth = await requireAdminApiSession(req);
|
||||||
|
if (!auth.valid) {
|
||||||
|
return auth.response;
|
||||||
|
}
|
||||||
|
|
||||||
|
const role = auth.auth.session.role || 'owner';
|
||||||
|
const readOnly = role === 'viewer';
|
||||||
|
const proxyInfo = resolveFilesProxyStatus(req);
|
||||||
|
const targetUrl = `${resolveIngestionBaseUrl()}/vcon/files-engine/_status`;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const cookieHeader = req.headers.get('cookie') || '';
|
||||||
|
const upstream = await fetch(targetUrl, {
|
||||||
|
method: 'GET',
|
||||||
|
headers: cookieHeader ? { cookie: cookieHeader } : undefined,
|
||||||
|
cache: 'no-store',
|
||||||
|
signal: AbortSignal.timeout(2500),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (upstream.status === 401) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{ ok: false, offline: false, error: 'Authentication required.' },
|
||||||
|
{ status: 401, headers: { 'Cache-Control': 'no-store, private' } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const payload = (await upstream.json().catch(() => ({}))) as {
|
||||||
|
ok?: boolean;
|
||||||
|
engine?: string;
|
||||||
|
role?: string;
|
||||||
|
readOnly?: boolean;
|
||||||
|
error?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
if (upstream.ok && payload.ok) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
ok: true,
|
||||||
|
offline: false,
|
||||||
|
engine: payload.engine || 'cloudcmd',
|
||||||
|
role: payload.role || role,
|
||||||
|
readOnly: Boolean(payload.readOnly ?? readOnly),
|
||||||
|
...proxyInfo,
|
||||||
|
},
|
||||||
|
{ status: 200, headers: { 'Cache-Control': 'no-store, private' } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
ok: false,
|
||||||
|
offline: true,
|
||||||
|
role,
|
||||||
|
readOnly,
|
||||||
|
...proxyInfo,
|
||||||
|
error:
|
||||||
|
payload.error ||
|
||||||
|
`Files Manager engine returned HTTP ${upstream.status}. Ensure the Port 4000 ingestion server is running.`,
|
||||||
|
},
|
||||||
|
{ status: 200, headers: { 'Cache-Control': 'no-store, private' } },
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
ok: false,
|
||||||
|
offline: true,
|
||||||
|
role,
|
||||||
|
readOnly,
|
||||||
|
...proxyInfo,
|
||||||
|
error:
|
||||||
|
'Files Manager engine on Port 4000 is not reachable. Start both services with "npm run dev" / "npm run dev:all" or Docker Compose.',
|
||||||
|
},
|
||||||
|
{ status: 200, headers: { 'Cache-Control': 'no-store, private' } },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
+39
-17
@@ -4,6 +4,10 @@ import {
|
|||||||
} from "@/core/adminSettings/settingsManager";
|
} from "@/core/adminSettings/settingsManager";
|
||||||
import { recordAdminAuditLogAsync } from "@/core/auth/auditLog";
|
import { recordAdminAuditLogAsync } from "@/core/auth/auditLog";
|
||||||
import { SecretVaultError } from "@/core/Security/secretVault";
|
import { SecretVaultError } from "@/core/Security/secretVault";
|
||||||
|
import {
|
||||||
|
SSRFBlockedError,
|
||||||
|
validateOutboundUrlSync,
|
||||||
|
} from "@/core/Security/ssrfGuard";
|
||||||
import { extractClientIp, requireAdminApiSession } from "@/lib/adminAuth";
|
import { extractClientIp, requireAdminApiSession } from "@/lib/adminAuth";
|
||||||
import { NextRequest, NextResponse } from "next/server";
|
import { NextRequest, NextResponse } from "next/server";
|
||||||
|
|
||||||
@@ -209,16 +213,20 @@ export async function POST(req: NextRequest) {
|
|||||||
if (body.s3Endpoint !== undefined) {
|
if (body.s3Endpoint !== undefined) {
|
||||||
const ep = parseOptionalString(body.s3Endpoint, "s3Endpoint", 512);
|
const ep = parseOptionalString(body.s3Endpoint, "s3Endpoint", 512);
|
||||||
if (ep) {
|
if (ep) {
|
||||||
let parsedEp: URL;
|
const normalizedEp = /^https?:\/\//i.test(ep) ? ep : `https://${ep}`;
|
||||||
try {
|
const check = validateOutboundUrlSync(normalizedEp, {
|
||||||
parsedEp = new URL(
|
fieldName: "s3Endpoint",
|
||||||
/^https?:\/\//i.test(ep) ? ep : `https://${ep}`,
|
});
|
||||||
|
if (!check.allowed) {
|
||||||
|
if (check.code === "SSRF_BLOCKED_HOST") {
|
||||||
|
throw new SSRFBlockedError(
|
||||||
|
check.error || "s3Endpoint target host is blocked.",
|
||||||
|
check.hostname || "",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
throw new Error(
|
||||||
|
check.error || "s3Endpoint must be a valid http(s) URL.",
|
||||||
);
|
);
|
||||||
} catch {
|
|
||||||
throw new Error("s3Endpoint must be a valid http(s) URL.");
|
|
||||||
}
|
|
||||||
if (parsedEp.protocol !== "http:" && parsedEp.protocol !== "https:") {
|
|
||||||
throw new Error("s3Endpoint must use http:// or https://.");
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
validated.s3Endpoint = ep;
|
validated.s3Endpoint = ep;
|
||||||
@@ -270,14 +278,19 @@ export async function POST(req: NextRequest) {
|
|||||||
if (body.webhookUrl !== undefined) {
|
if (body.webhookUrl !== undefined) {
|
||||||
const wh = parseOptionalString(body.webhookUrl, "webhookUrl", 1024);
|
const wh = parseOptionalString(body.webhookUrl, "webhookUrl", 1024);
|
||||||
if (wh) {
|
if (wh) {
|
||||||
let parsedWh: URL;
|
const check = validateOutboundUrlSync(wh, {
|
||||||
try {
|
fieldName: "webhookUrl",
|
||||||
parsedWh = new URL(wh);
|
});
|
||||||
} catch {
|
if (!check.allowed) {
|
||||||
throw new Error("webhookUrl must be a valid http(s) URL.");
|
if (check.code === "SSRF_BLOCKED_HOST") {
|
||||||
}
|
throw new SSRFBlockedError(
|
||||||
if (parsedWh.protocol !== "http:" && parsedWh.protocol !== "https:") {
|
check.error || "webhookUrl target host is blocked.",
|
||||||
throw new Error("webhookUrl must use http:// or https://.");
|
check.hostname || "",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
throw new Error(
|
||||||
|
check.error || "webhookUrl must be a valid http(s) URL.",
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
validated.webhookUrl = wh;
|
validated.webhookUrl = wh;
|
||||||
@@ -334,6 +347,15 @@ export async function POST(req: NextRequest) {
|
|||||||
validated.jwtSecret = body.jwtSecret.trim();
|
validated.jwtSecret = body.jwtSecret.trim();
|
||||||
}
|
}
|
||||||
} catch (validationErr: unknown) {
|
} catch (validationErr: unknown) {
|
||||||
|
if (validationErr instanceof SSRFBlockedError) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
error: validationErr.message,
|
||||||
|
code: validationErr.code,
|
||||||
|
},
|
||||||
|
{ status: 400 },
|
||||||
|
);
|
||||||
|
}
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{
|
{
|
||||||
error:
|
error:
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import {
|
|||||||
decryptSecret,
|
decryptSecret,
|
||||||
SecretVaultError,
|
SecretVaultError,
|
||||||
} from "@/core/Security/secretVault";
|
} from "@/core/Security/secretVault";
|
||||||
|
import { validateOutboundUrlSync } from "@/core/Security/ssrfGuard";
|
||||||
import { extractClientIp, requireAdminApiSession } from "@/lib/adminAuth";
|
import { extractClientIp, requireAdminApiSession } from "@/lib/adminAuth";
|
||||||
import { NextRequest, NextResponse } from "next/server";
|
import { NextRequest, NextResponse } from "next/server";
|
||||||
|
|
||||||
@@ -57,17 +58,20 @@ export async function POST(req: NextRequest) {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
const urlCheck = validateOutboundUrlSync(webhookUrl, {
|
||||||
const parsedUrl = new URL(webhookUrl);
|
fieldName: "Webhook URL",
|
||||||
if (parsedUrl.protocol !== "http:" && parsedUrl.protocol !== "https:") {
|
});
|
||||||
throw new Error("Webhook URL must use http:// or https://.");
|
if (!urlCheck.allowed) {
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{
|
{
|
||||||
success: false,
|
success: false,
|
||||||
code: "VALIDATION_ERROR",
|
code:
|
||||||
error: "Webhook URL must be a valid http:// or https:// endpoint.",
|
urlCheck.code === "SSRF_BLOCKED_HOST"
|
||||||
|
? "SSRF_BLOCKED_HOST"
|
||||||
|
: "VALIDATION_ERROR",
|
||||||
|
error:
|
||||||
|
urlCheck.error ||
|
||||||
|
"Webhook URL must be a valid http:// or https:// endpoint.",
|
||||||
},
|
},
|
||||||
{ status: 400 },
|
{ status: 400 },
|
||||||
);
|
);
|
||||||
@@ -105,14 +109,15 @@ export async function POST(req: NextRequest) {
|
|||||||
});
|
});
|
||||||
|
|
||||||
if (!result.ok) {
|
if (!result.ok) {
|
||||||
|
const isSsrfBlocked = result.code === "SSRF_BLOCKED_HOST";
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{
|
{
|
||||||
success: false,
|
success: false,
|
||||||
code: "WEBHOOK_DELIVERY_FAILED",
|
code: isSsrfBlocked ? "SSRF_BLOCKED_HOST" : "WEBHOOK_DELIVERY_FAILED",
|
||||||
error: result.error || "Webhook target failed to accept test alert.",
|
error: result.error || "Webhook target failed to accept test alert.",
|
||||||
status: result.status,
|
status: result.status,
|
||||||
},
|
},
|
||||||
{ status: 502 },
|
{ status: isSsrfBlocked ? 400 : 502 },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -71,6 +71,10 @@ export async function POST(req: NextRequest) {
|
|||||||
|
|
||||||
const result = await testS3Connection(config);
|
const result = await testS3Connection(config);
|
||||||
if (!result.ok) {
|
if (!result.ok) {
|
||||||
|
const isClientError =
|
||||||
|
result.code === "S3_INVALID_CONFIG" ||
|
||||||
|
result.code === "SSRF_BLOCKED_HOST" ||
|
||||||
|
result.code === "INVALID_URL";
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{
|
{
|
||||||
success: false,
|
success: false,
|
||||||
@@ -78,7 +82,7 @@ export async function POST(req: NextRequest) {
|
|||||||
error: result.error || "S3 connectivity check failed.",
|
error: result.error || "S3 connectivity check failed.",
|
||||||
...(result.status !== undefined ? { s3Status: result.status } : {}),
|
...(result.status !== undefined ? { s3Status: result.status } : {}),
|
||||||
},
|
},
|
||||||
{ status: result.code === "S3_INVALID_CONFIG" ? 400 : 502 },
|
{ status: isClientError ? 400 : 502 },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,5 @@
|
|||||||
|
import { FilesManagerWorkspace } from '@/components/sections/FilesManagerWorkspace';
|
||||||
|
|
||||||
|
export default function FilesManagerPage() {
|
||||||
|
return <FilesManagerWorkspace />;
|
||||||
|
}
|
||||||
+26
-3
@@ -1,6 +1,7 @@
|
|||||||
'use client';
|
'use client';
|
||||||
|
|
||||||
import { PageShell } from '@/components/layout/PageShell';
|
import { PageShell } from '@/components/layout/PageShell';
|
||||||
|
import { useVconRole } from '@/components/layout/VconShell';
|
||||||
import { Button } from '@/components/ui/Button';
|
import { Button } from '@/components/ui/Button';
|
||||||
import { Card } from '@/components/ui/Card';
|
import { Card } from '@/components/ui/Card';
|
||||||
import { CodeBlock } from '@/components/ui/CodeBlock';
|
import { CodeBlock } from '@/components/ui/CodeBlock';
|
||||||
@@ -9,7 +10,10 @@ import { Panel } from '@/components/ui/Panel';
|
|||||||
import { StatStrip } from '@/components/ui/StatStrip';
|
import { StatStrip } from '@/components/ui/StatStrip';
|
||||||
import { useToast } from '@/components/ui/Toast';
|
import { useToast } from '@/components/ui/Toast';
|
||||||
import { getCached, setCached } from '@/lib/vconCache';
|
import { getCached, setCached } from '@/lib/vconCache';
|
||||||
import { resolvePublicEndpoints } from '@/lib/vconUiHelpers';
|
import {
|
||||||
|
getVconRoleCapabilities,
|
||||||
|
resolvePublicEndpoints,
|
||||||
|
} from '@/lib/vconUiHelpers';
|
||||||
import { useEffect, useState } from 'react';
|
import { useEffect, useState } from 'react';
|
||||||
|
|
||||||
interface SdkClientItem {
|
interface SdkClientItem {
|
||||||
@@ -24,6 +28,8 @@ type SdkLanguageTab = 'python' | 'node' | 'curl';
|
|||||||
|
|
||||||
export default function SdkGuidePage() {
|
export default function SdkGuidePage() {
|
||||||
const { success, error } = useToast();
|
const { success, error } = useToast();
|
||||||
|
const shellRole = useVconRole();
|
||||||
|
const { canOperate } = getVconRoleCapabilities(shellRole);
|
||||||
const [activeTab, setActiveTab] = useState<SdkLanguageTab>('python');
|
const [activeTab, setActiveTab] = useState<SdkLanguageTab>('python');
|
||||||
const [clients, setClients] = useState<SdkClientItem[]>(() =>
|
const [clients, setClients] = useState<SdkClientItem[]>(() =>
|
||||||
getCached<SdkClientItem[]>('vcon:clients', [])
|
getCached<SdkClientItem[]>('vcon:clients', [])
|
||||||
@@ -83,8 +89,9 @@ export default function SdkGuidePage() {
|
|||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!selectedClientId) {
|
if (!selectedClientId || !canOperate) {
|
||||||
setSelectedClientToken('');
|
setSelectedClientToken('');
|
||||||
|
setLoadingToken(false);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -121,7 +128,7 @@ export default function SdkGuidePage() {
|
|||||||
return () => {
|
return () => {
|
||||||
cancelled = true;
|
cancelled = true;
|
||||||
};
|
};
|
||||||
}, [selectedClientId]);
|
}, [selectedClientId, canOperate]);
|
||||||
|
|
||||||
const activeClient =
|
const activeClient =
|
||||||
clients.find((c) => c.clientId === selectedClientId) || clients[0];
|
clients.find((c) => c.clientId === selectedClientId) || clients[0];
|
||||||
@@ -454,6 +461,22 @@ curl -X POST "${apiUrl}/api/events" \\
|
|||||||
</Card>
|
</Card>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
{!canOperate && (
|
||||||
|
<div
|
||||||
|
role="status"
|
||||||
|
data-testid="sdk-viewer-rbac-banner"
|
||||||
|
className="mb-3 border border-[var(--warning)]/40 bg-[var(--warning)]/10 px-3 py-2 text-[11.5px] text-[var(--text)]"
|
||||||
|
>
|
||||||
|
Read-only <span className="font-mono uppercase">{shellRole}</span> session:
|
||||||
|
Operator or Owner role required to reveal live client JWT tokens. Snippets below
|
||||||
|
safely use placeholder{' '}
|
||||||
|
<code className="font-mono text-[var(--accent)]">
|
||||||
|
YOUR_SIGNED_CLIENT_JWT_TOKEN
|
||||||
|
</code>
|
||||||
|
.
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
<Panel
|
<Panel
|
||||||
title={`Step 1 — ${activeConfigLabel}`}
|
title={`Step 1 — ${activeConfigLabel}`}
|
||||||
className="w-full"
|
className="w-full"
|
||||||
|
|||||||
@@ -17,8 +17,11 @@ const VCON_LABELS: Record<string, string> = {
|
|||||||
'/vcon/clients': 'Clients',
|
'/vcon/clients': 'Clients',
|
||||||
'/vcon/export': 'Export',
|
'/vcon/export': 'Export',
|
||||||
'/vcon/storage': 'Storage',
|
'/vcon/storage': 'Storage',
|
||||||
'/vcon/sdk': 'Python SDK',
|
'/vcon/files': 'Files Manager',
|
||||||
|
'/vcon/sdk': 'Client SDKs',
|
||||||
'/vcon/simulator': 'Simulator',
|
'/vcon/simulator': 'Simulator',
|
||||||
|
'/vcon/team': 'Team & RBAC',
|
||||||
|
'/vcon/audit': 'Audit Log',
|
||||||
'/vcon/profile': 'Profile',
|
'/vcon/profile': 'Profile',
|
||||||
'/vcon/settings': 'Settings',
|
'/vcon/settings': 'Settings',
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
'use client';
|
'use client';
|
||||||
|
|
||||||
|
import { useVconRole } from '@/components/layout/VconShell';
|
||||||
import { Button } from '@/components/ui/Button';
|
import { Button } from '@/components/ui/Button';
|
||||||
import { Icon } from '@/components/ui/Icon';
|
import { Icon } from '@/components/ui/Icon';
|
||||||
import { cn } from '@/lib/utils';
|
import { cn } from '@/lib/utils';
|
||||||
@@ -18,6 +19,14 @@ interface CachedProfile {
|
|||||||
role?: string;
|
role?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function formatShellRoleLabel(role?: string | null): string {
|
||||||
|
const clean = String(role || 'viewer').trim().toLowerCase();
|
||||||
|
if (clean === 'owner') return 'Owner';
|
||||||
|
if (clean === 'admin') return 'Admin';
|
||||||
|
if (clean === 'operator') return 'Operator';
|
||||||
|
return 'Viewer';
|
||||||
|
}
|
||||||
|
|
||||||
const NAVIGATION_SECTIONS = [
|
const NAVIGATION_SECTIONS = [
|
||||||
{
|
{
|
||||||
group: 'Main',
|
group: 'Main',
|
||||||
@@ -27,12 +36,13 @@ const NAVIGATION_SECTIONS = [
|
|||||||
{ name: 'Clients', href: '/vcon/clients', icon: 'Key' },
|
{ name: 'Clients', href: '/vcon/clients', icon: 'Key' },
|
||||||
{ name: 'Export', href: '/vcon/export', icon: 'Download' },
|
{ name: 'Export', href: '/vcon/export', icon: 'Download' },
|
||||||
{ name: 'Storage', href: '/vcon/storage', icon: 'HardDrive' },
|
{ name: 'Storage', href: '/vcon/storage', icon: 'HardDrive' },
|
||||||
|
{ name: 'Files Manager', href: '/vcon/files', icon: 'FolderOpen' },
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
group: 'System',
|
group: 'System',
|
||||||
items: [
|
items: [
|
||||||
{ name: 'Python SDK', href: '/vcon/sdk', icon: 'FileCode2' },
|
{ name: 'Client SDKs', href: '/vcon/sdk', icon: 'FileCode2' },
|
||||||
{ name: 'Simulator', href: '/vcon/simulator', icon: 'Play' },
|
{ name: 'Simulator', href: '/vcon/simulator', icon: 'Play' },
|
||||||
{ name: 'Team & RBAC', href: '/vcon/team', icon: 'Users' },
|
{ name: 'Team & RBAC', href: '/vcon/team', icon: 'Users' },
|
||||||
{ name: 'Audit Log', href: '/vcon/audit', icon: 'FileText' },
|
{ name: 'Audit Log', href: '/vcon/audit', icon: 'FileText' },
|
||||||
@@ -47,13 +57,15 @@ export const Sidebar: React.FC<SidebarProps> = ({
|
|||||||
onClose,
|
onClose,
|
||||||
}) => {
|
}) => {
|
||||||
const pathname = usePathname();
|
const pathname = usePathname();
|
||||||
|
const shellRole = useVconRole();
|
||||||
|
const initialRoleLabel = formatShellRoleLabel(shellRole);
|
||||||
const [isSigningOut, setIsSigningOut] = useState(false);
|
const [isSigningOut, setIsSigningOut] = useState(false);
|
||||||
const [theme, setTheme] = useState<'dark' | 'light'>('dark');
|
const [theme, setTheme] = useState<'dark' | 'light'>('dark');
|
||||||
const [adminEmail, setAdminEmail] = useState<string>(
|
const [adminEmail, setAdminEmail] = useState<string>(
|
||||||
() => getCached<CachedProfile>('vcon:profile', {}).email || 'Admin'
|
() => getCached<CachedProfile>('vcon:profile', {}).email || 'Admin'
|
||||||
);
|
);
|
||||||
const [adminRole, setAdminRole] = useState<string>(
|
const [adminRole, setAdminRole] = useState<string>(
|
||||||
() => getCached<CachedProfile>('vcon:profile', {}).role || 'Owner'
|
() => getCached<CachedProfile>('vcon:profile', {}).role || initialRoleLabel
|
||||||
);
|
);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
@@ -96,17 +108,17 @@ export const Sidebar: React.FC<SidebarProps> = ({
|
|||||||
setCached('vcon:profile', data.profile);
|
setCached('vcon:profile', data.profile);
|
||||||
setCached('vcon:profile-loaded', true);
|
setCached('vcon:profile-loaded', true);
|
||||||
setAdminEmail(data.profile.email);
|
setAdminEmail(data.profile.email);
|
||||||
setAdminRole(data.profile.role || 'Owner');
|
setAdminRole(data.profile.role || initialRoleLabel);
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.catch(() => {
|
.catch(() => {
|
||||||
// Keep cached or default identity label if offline
|
// Keep cached or shell role identity label if offline
|
||||||
});
|
});
|
||||||
|
|
||||||
return () => {
|
return () => {
|
||||||
window.removeEventListener('datastack:profile-updated', handleProfileUpdated);
|
window.removeEventListener('datastack:profile-updated', handleProfileUpdated);
|
||||||
};
|
};
|
||||||
}, [pathname]);
|
}, [pathname, initialRoleLabel]);
|
||||||
|
|
||||||
const toggleTheme = () => {
|
const toggleTheme = () => {
|
||||||
const nextTheme = theme === 'dark' ? 'light' : 'dark';
|
const nextTheme = theme === 'dark' ? 'light' : 'dark';
|
||||||
|
|||||||
@@ -50,6 +50,8 @@ const ACTION_FILTER_OPTIONS = [
|
|||||||
{ value: 'STORAGE_FLUSH_TRIGGERED', label: 'STORAGE_FLUSH_TRIGGERED' },
|
{ value: 'STORAGE_FLUSH_TRIGGERED', label: 'STORAGE_FLUSH_TRIGGERED' },
|
||||||
{ value: 'S3_ARCHIVE_TRIGGERED', label: 'S3_ARCHIVE_TRIGGERED' },
|
{ value: 'S3_ARCHIVE_TRIGGERED', label: 'S3_ARCHIVE_TRIGGERED' },
|
||||||
{ value: 'WEBHOOK_TEST_TRIGGERED', label: 'WEBHOOK_TEST_TRIGGERED' },
|
{ value: 'WEBHOOK_TEST_TRIGGERED', label: 'WEBHOOK_TEST_TRIGGERED' },
|
||||||
|
{ value: 'FILES_MANAGER_MUTATION', label: 'FILES_MANAGER_MUTATION' },
|
||||||
|
{ value: 'DATA_EXPORT_DOWNLOADED', label: 'DATA_EXPORT_DOWNLOADED' },
|
||||||
];
|
];
|
||||||
|
|
||||||
export function AuditLogViewer() {
|
export function AuditLogViewer() {
|
||||||
|
|||||||
@@ -36,6 +36,13 @@ interface Client {
|
|||||||
name: string;
|
name: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface CorruptPartitionWarning {
|
||||||
|
clientId: string;
|
||||||
|
filename: string;
|
||||||
|
code?: string;
|
||||||
|
error: string;
|
||||||
|
}
|
||||||
|
|
||||||
const PAGE_SIZE = 12;
|
const PAGE_SIZE = 12;
|
||||||
|
|
||||||
function formatEventPayload(raw: string): string {
|
function formatEventPayload(raw: string): string {
|
||||||
@@ -70,6 +77,7 @@ export function DataBrowser() {
|
|||||||
const [filtersReady, setFiltersReady] = useState(false);
|
const [filtersReady, setFiltersReady] = useState(false);
|
||||||
const [refreshing, setRefreshing] = useState(false);
|
const [refreshing, setRefreshing] = useState(false);
|
||||||
const [errorMessage, setErrorMessage] = useState('');
|
const [errorMessage, setErrorMessage] = useState('');
|
||||||
|
const [corruptPartitions, setCorruptPartitions] = useState<CorruptPartitionWarning[]>([]);
|
||||||
const [selectedEvent, setSelectedEvent] = useState<StreamEvent | null>(null);
|
const [selectedEvent, setSelectedEvent] = useState<StreamEvent | null>(null);
|
||||||
|
|
||||||
const fetchEvents = useCallback(async () => {
|
const fetchEvents = useCallback(async () => {
|
||||||
@@ -94,6 +102,9 @@ export function DataBrowser() {
|
|||||||
setEvents(data.events || []);
|
setEvents(data.events || []);
|
||||||
setTotal(data.total || 0);
|
setTotal(data.total || 0);
|
||||||
setPageCount(data.pageCount || 1);
|
setPageCount(data.pageCount || 1);
|
||||||
|
setCorruptPartitions(
|
||||||
|
Array.isArray(data.corruptPartitions) ? data.corruptPartitions : []
|
||||||
|
);
|
||||||
setErrorMessage('');
|
setErrorMessage('');
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
setErrorMessage(error instanceof Error ? error.message : 'Unable to load events');
|
setErrorMessage(error instanceof Error ? error.message : 'Unable to load events');
|
||||||
@@ -271,6 +282,29 @@ export function DataBrowser() {
|
|||||||
)}
|
)}
|
||||||
</Card>
|
</Card>
|
||||||
|
|
||||||
|
{corruptPartitions.length > 0 && (
|
||||||
|
<div
|
||||||
|
role="status"
|
||||||
|
data-testid="data-corrupt-partitions-warning"
|
||||||
|
className="border border-[var(--warning)]/40 bg-[var(--warning)]/10 px-3 py-2 text-[11.5px] text-[var(--text)]"
|
||||||
|
>
|
||||||
|
<div className="font-medium">
|
||||||
|
{corruptPartitions.length} corrupt .csv.gz partition
|
||||||
|
{corruptPartitions.length === 1 ? '' : 's'} skipped while loading healthy events.
|
||||||
|
</div>
|
||||||
|
<ul className="mt-1 flex flex-col gap-0.5 font-mono text-[10.5px] text-[var(--text-muted)]">
|
||||||
|
{corruptPartitions.slice(0, 5).map((item) => (
|
||||||
|
<li key={`${item.clientId}:${item.filename}`}>
|
||||||
|
{item.clientId}/{item.filename} — {item.error}
|
||||||
|
</li>
|
||||||
|
))}
|
||||||
|
{corruptPartitions.length > 5 && (
|
||||||
|
<li>+{corruptPartitions.length - 5} more unreadable archive(s)</li>
|
||||||
|
)}
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
<Panel
|
<Panel
|
||||||
title="Events"
|
title="Events"
|
||||||
meta={`${total.toLocaleString()} · ${page}/${pageCount}`}
|
meta={`${total.toLocaleString()} · ${page}/${pageCount}`}
|
||||||
|
|||||||
@@ -0,0 +1,210 @@
|
|||||||
|
'use client';
|
||||||
|
|
||||||
|
import { PageShell } from '@/components/layout/PageShell';
|
||||||
|
import { useVconRole } from '@/components/layout/VconShell';
|
||||||
|
import { Button } from '@/components/ui/Button';
|
||||||
|
import { Icon } from '@/components/ui/Icon';
|
||||||
|
import { getVconRoleCapabilities } from '@/lib/vconUiHelpers';
|
||||||
|
import React, { useCallback, useEffect, useState } from 'react';
|
||||||
|
|
||||||
|
type EngineStatus = 'checking' | 'online' | 'offline';
|
||||||
|
|
||||||
|
interface FilesEngineStatusResponse {
|
||||||
|
ok?: boolean;
|
||||||
|
engine?: string;
|
||||||
|
role?: string;
|
||||||
|
readOnly?: boolean;
|
||||||
|
proxyMode?: 'nginx' | 'standalone';
|
||||||
|
wsSupported?: boolean;
|
||||||
|
wsHint?: string | null;
|
||||||
|
error?: string;
|
||||||
|
code?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const FILES_ENGINE_STATUS_URL = '/api/files/status';
|
||||||
|
const FILES_ENGINE_FRAME_URL = '/vcon/files-engine';
|
||||||
|
|
||||||
|
export function FilesManagerWorkspace() {
|
||||||
|
const currentRole = useVconRole();
|
||||||
|
const { canOperate } = getVconRoleCapabilities(currentRole);
|
||||||
|
|
||||||
|
const [status, setStatus] = useState<EngineStatus>('checking');
|
||||||
|
const [statusError, setStatusError] = useState<string>('');
|
||||||
|
const [wsHint, setWsHint] = useState<string | null>(null);
|
||||||
|
const [engineReadOnly, setEngineReadOnly] = useState<boolean>(!canOperate);
|
||||||
|
const [frameKey, setFrameKey] = useState<number>(1);
|
||||||
|
const [probing, setProbing] = useState<boolean>(false);
|
||||||
|
|
||||||
|
const checkEngineStatus = useCallback(async (reloadFrame = false) => {
|
||||||
|
setProbing(true);
|
||||||
|
setStatusError('');
|
||||||
|
try {
|
||||||
|
const response = await fetch(FILES_ENGINE_STATUS_URL, {
|
||||||
|
method: 'GET',
|
||||||
|
cache: 'no-store',
|
||||||
|
credentials: 'same-origin',
|
||||||
|
});
|
||||||
|
|
||||||
|
if (response.status === 401) {
|
||||||
|
window.location.assign('/vcon/login?next=%2Fvcon%2Ffiles');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = (await response.json().catch(() => ({}))) as FilesEngineStatusResponse;
|
||||||
|
setWsHint(typeof data.wsHint === 'string' && data.wsHint ? data.wsHint : null);
|
||||||
|
|
||||||
|
if (response.ok && data.ok) {
|
||||||
|
setEngineReadOnly(Boolean(data.readOnly));
|
||||||
|
setStatus('online');
|
||||||
|
if (reloadFrame) {
|
||||||
|
setFrameKey((prev) => prev + 1);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
setStatus('offline');
|
||||||
|
setStatusError(
|
||||||
|
data.error ||
|
||||||
|
`Files Manager engine returned HTTP ${response.status}. Ensure the Port 4000 ingestion server is running.`
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
setStatus('offline');
|
||||||
|
setStatusError(
|
||||||
|
'Cannot reach the Port 4000 Files Manager engine. Start both services with "npm run dev:all" or Docker Compose.'
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
setProbing(false);
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
void checkEngineStatus(false);
|
||||||
|
}, [checkEngineStatus]);
|
||||||
|
|
||||||
|
const effectiveReadOnly = !canOperate || engineReadOnly;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<PageShell
|
||||||
|
title="Files Manager"
|
||||||
|
actions={
|
||||||
|
<>
|
||||||
|
<span
|
||||||
|
className="border border-[var(--border)] bg-[var(--surface-2)] px-2 py-1 font-mono text-[10.5px] uppercase tracking-wider text-[var(--text-muted)]"
|
||||||
|
data-testid="files-manager-access-badge"
|
||||||
|
>
|
||||||
|
{effectiveReadOnly
|
||||||
|
? `Read-only (${currentRole})`
|
||||||
|
: `Full Access (${currentRole})`}
|
||||||
|
</span>
|
||||||
|
<Button
|
||||||
|
size="xs"
|
||||||
|
variant="secondary"
|
||||||
|
icon="RotateCw"
|
||||||
|
loading={probing}
|
||||||
|
disabled={probing}
|
||||||
|
onClick={() => void checkEngineStatus(true)}
|
||||||
|
>
|
||||||
|
Reload
|
||||||
|
</Button>
|
||||||
|
{status === 'online' && (
|
||||||
|
<a
|
||||||
|
href={FILES_ENGINE_FRAME_URL}
|
||||||
|
target="_blank"
|
||||||
|
rel="noopener noreferrer"
|
||||||
|
className="inline-flex h-6 items-center gap-1.5 border border-[var(--border)] bg-[var(--surface)] px-2 text-[11px] font-medium text-[var(--text)] transition-colors hover:border-[var(--border-strong)] hover:bg-[var(--surface-2)]"
|
||||||
|
>
|
||||||
|
<Icon name="ExternalLink" size={12} />
|
||||||
|
<span>Full View</span>
|
||||||
|
</a>
|
||||||
|
)}
|
||||||
|
</>
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<div className="flex flex-wrap items-center justify-between gap-2 border border-[var(--border)] bg-[var(--surface)] px-3 py-1.5 text-[11.5px] text-[var(--text-muted)]">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<Icon name="FolderOpen" size={13} className="shrink-0 text-[var(--accent)]" />
|
||||||
|
<span>
|
||||||
|
Root: <code className="font-mono text-[var(--text)]">/storage/data</code> — browse, edit, rename, upload, download, and ZIP/UNZIP client partitions.
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<span className="font-mono text-[10.5px] text-[var(--text-dim)]">
|
||||||
|
{effectiveReadOnly
|
||||||
|
? 'Viewer mode: browse, view & download only'
|
||||||
|
: 'Active *.lock files are protected; avoid editing live UTC-today CSVs during peak flush'}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{status === 'online' && wsHint && (
|
||||||
|
<div
|
||||||
|
className="flex items-center gap-2 border border-[var(--border)] bg-[var(--surface-2)] px-3 py-1.5 text-[11px] text-[var(--text-muted)]"
|
||||||
|
data-testid="files-manager-proxy-hint"
|
||||||
|
>
|
||||||
|
<Icon name="Info" size={12} className="shrink-0 text-[var(--accent)]" />
|
||||||
|
<span>{wsHint}</span>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<div
|
||||||
|
className="w-full overflow-hidden border border-[var(--border)] bg-[var(--surface)]"
|
||||||
|
data-testid="files-manager-container"
|
||||||
|
>
|
||||||
|
{status === 'checking' && (
|
||||||
|
<div className="flex h-[calc(100dvh-180px)] min-h-[480px] flex-col items-center justify-center gap-2.5 p-6 text-center">
|
||||||
|
<Icon name="RotateCw" size={18} className="animate-spin text-[var(--accent)]" />
|
||||||
|
<p className="text-[12.5px] font-medium text-[var(--text)]">
|
||||||
|
Connecting to Files Manager engine...
|
||||||
|
</p>
|
||||||
|
<p className="text-[11.5px] text-[var(--text-dim)]">
|
||||||
|
Verifying admin session and storage root on Port 4000
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{status === 'offline' && (
|
||||||
|
<div
|
||||||
|
className="flex h-[calc(100dvh-180px)] min-h-[480px] flex-col items-center justify-center gap-3 p-6 text-center"
|
||||||
|
data-testid="files-manager-offline"
|
||||||
|
>
|
||||||
|
<div className="flex h-9 w-9 items-center justify-center border border-[var(--border-strong)] bg-[var(--surface-2)] text-[var(--text-muted)]">
|
||||||
|
<Icon name="HardDrive" size={18} />
|
||||||
|
</div>
|
||||||
|
<div className="max-w-md space-y-1">
|
||||||
|
<h2 className="text-[13px] font-medium text-[var(--text)]">
|
||||||
|
Files Manager Engine Unavailable
|
||||||
|
</h2>
|
||||||
|
<p className="text-[12px] text-[var(--text-muted)]">
|
||||||
|
{statusError ||
|
||||||
|
'The Port 4000 ingestion and files engine is currently offline.'}
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
<div className="border border-[var(--border)] bg-[var(--surface-2)] px-3 py-2 font-mono text-[11px] text-[var(--text-dim)]">
|
||||||
|
Local: npm run dev:all | Production: docker compose up
|
||||||
|
</div>
|
||||||
|
<Button
|
||||||
|
size="sm"
|
||||||
|
variant="primary"
|
||||||
|
icon="RotateCw"
|
||||||
|
loading={probing}
|
||||||
|
disabled={probing}
|
||||||
|
onClick={() => void checkEngineStatus(true)}
|
||||||
|
>
|
||||||
|
Retry Connection
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{status === 'online' && (
|
||||||
|
<iframe
|
||||||
|
key={frameKey}
|
||||||
|
src={FILES_ENGINE_FRAME_URL}
|
||||||
|
title="DataStack Files Manager"
|
||||||
|
data-testid="files-manager-iframe"
|
||||||
|
className="block h-[calc(100dvh-180px)] min-h-[540px] w-full border-0 bg-[var(--surface)]"
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</PageShell>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export default FilesManagerWorkspace;
|
||||||
@@ -21,7 +21,7 @@ export const Hero: React.FC = () => (
|
|||||||
Explore features
|
Explore features
|
||||||
</Link>
|
</Link>
|
||||||
<Link href="/#sdk" className={buttonClassNames({ variant: 'secondary', size: 'sm' })}>
|
<Link href="/#sdk" className={buttonClassNames({ variant: 'secondary', size: 'sm' })}>
|
||||||
Python SDK
|
Client SDKs
|
||||||
</Link>
|
</Link>
|
||||||
</div>
|
</div>
|
||||||
<ActivityPreview />
|
<ActivityPreview />
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ export const LandingAbout: React.FC = () => (
|
|||||||
<p className="text-[13px] text-[var(--text-muted)]">Start with the SDK guide and make the workspace your own.</p>
|
<p className="text-[13px] text-[var(--text-muted)]">Start with the SDK guide and make the workspace your own.</p>
|
||||||
</div>
|
</div>
|
||||||
<Link href="/#sdk" className={buttonClassNames({ variant: 'primary', size: 'sm' })}>
|
<Link href="/#sdk" className={buttonClassNames({ variant: 'primary', size: 'sm' })}>
|
||||||
View Python SDK
|
View Client SDKs
|
||||||
</Link>
|
</Link>
|
||||||
</section>
|
</section>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -27,10 +27,10 @@ interface AdminProfileData {
|
|||||||
sessionExpiresAt: string;
|
sessionExpiresAt: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
const DEFAULT_PROFILE: AdminProfileData = {
|
const EMPTY_PROFILE: AdminProfileData = {
|
||||||
email: 'admin@datastack.dev',
|
email: '',
|
||||||
role: 'Owner',
|
role: '',
|
||||||
storageMode: 'In-Memory',
|
storageMode: '',
|
||||||
createdAt: '',
|
createdAt: '',
|
||||||
sessionExpiresAt: '',
|
sessionExpiresAt: '',
|
||||||
};
|
};
|
||||||
@@ -38,10 +38,10 @@ const DEFAULT_PROFILE: AdminProfileData = {
|
|||||||
export function ProfileWorkspace() {
|
export function ProfileWorkspace() {
|
||||||
const { success, error } = useToast();
|
const { success, error } = useToast();
|
||||||
const [profile, setProfile] = useState<AdminProfileData>(() =>
|
const [profile, setProfile] = useState<AdminProfileData>(() =>
|
||||||
getCached<AdminProfileData>('vcon:profile', DEFAULT_PROFILE)
|
getCached<AdminProfileData>('vcon:profile', EMPTY_PROFILE)
|
||||||
);
|
);
|
||||||
const [email, setEmail] = useState(
|
const [email, setEmail] = useState(
|
||||||
() => getCached<AdminProfileData>('vcon:profile', DEFAULT_PROFILE).email
|
() => getCached<AdminProfileData>('vcon:profile', EMPTY_PROFILE).email
|
||||||
);
|
);
|
||||||
const [currentPassword, setCurrentPassword] = useState('');
|
const [currentPassword, setCurrentPassword] = useState('');
|
||||||
const [newPassword, setNewPassword] = useState('');
|
const [newPassword, setNewPassword] = useState('');
|
||||||
@@ -183,12 +183,26 @@ export function ProfileWorkspace() {
|
|||||||
>
|
>
|
||||||
<StatStrip
|
<StatStrip
|
||||||
items={[
|
items={[
|
||||||
{ label: 'Administrator', value: profile.email },
|
{
|
||||||
{ label: 'Role', value: profile.role, dot: 'ok' },
|
label: 'Administrator',
|
||||||
{ label: 'Auth Store', value: profile.storageMode },
|
value: profile.email || (loading ? 'Loading...' : '—'),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: 'Role',
|
||||||
|
value: profile.role || (loading ? 'Loading...' : '—'),
|
||||||
|
dot: profile.role ? 'ok' : undefined,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: 'Auth Store',
|
||||||
|
value: profile.storageMode || (loading ? 'Loading...' : '—'),
|
||||||
|
},
|
||||||
{
|
{
|
||||||
label: 'Session Expires',
|
label: 'Session Expires',
|
||||||
value: formatSessionExpiryLabel(profile.sessionExpiresAt),
|
value: profile.sessionExpiresAt
|
||||||
|
? formatSessionExpiryLabel(profile.sessionExpiresAt)
|
||||||
|
: loading
|
||||||
|
? 'Loading...'
|
||||||
|
: 'Active',
|
||||||
},
|
},
|
||||||
]}
|
]}
|
||||||
/>
|
/>
|
||||||
@@ -200,6 +214,7 @@ export function ProfileWorkspace() {
|
|||||||
<Input
|
<Input
|
||||||
compact
|
compact
|
||||||
type="email"
|
type="email"
|
||||||
|
placeholder={loading ? 'Loading account email...' : 'admin@company.com'}
|
||||||
value={email}
|
value={email}
|
||||||
onChange={(event) => setEmail(event.target.value)}
|
onChange={(event) => setEmail(event.target.value)}
|
||||||
required
|
required
|
||||||
@@ -208,7 +223,7 @@ export function ProfileWorkspace() {
|
|||||||
<SettingsField label="Access role">
|
<SettingsField label="Access role">
|
||||||
<Input
|
<Input
|
||||||
compact
|
compact
|
||||||
value={profile.role}
|
value={profile.role || (loading ? 'Loading...' : '—')}
|
||||||
readOnly
|
readOnly
|
||||||
className="border-[var(--border)] bg-[var(--surface-2)] text-[var(--text-muted)]"
|
className="border-[var(--border)] bg-[var(--surface-2)] text-[var(--text-muted)]"
|
||||||
/>
|
/>
|
||||||
|
|||||||
@@ -4,25 +4,32 @@ import React from 'react';
|
|||||||
|
|
||||||
export const SdkSection: React.FC = () => (
|
export const SdkSection: React.FC = () => (
|
||||||
<section id="sdk" className="border-b border-[var(--border)] py-10">
|
<section id="sdk" className="border-b border-[var(--border)] py-10">
|
||||||
<SectionHeading index="02" label="Python SDK" title="Drop in the SDK. Start streaming." />
|
<SectionHeading index="02" label="Client SDKs" title="Drop in the SDK. Start streaming." />
|
||||||
<div className="grid grid-cols-1 gap-px border border-[var(--border)] bg-[var(--border)] md:grid-cols-2">
|
<div className="grid grid-cols-1 gap-px border border-[var(--border)] bg-[var(--border)] md:grid-cols-2">
|
||||||
<CodeBlock>
|
<CodeBlock>
|
||||||
<span className="text-[var(--text-faint)]"># Install the bundled Python SDK (from repo or local wheel/path)</span>{'\n'}
|
<span className="text-[var(--text-faint)]"># Python SDK (datastack v2.2.0) — auto-loads config.datastack.py</span>{'\n'}
|
||||||
<span className="text-[var(--text)]">pip install /path/to/datastack/sdk/python/datastack</span>{'\n\n'}
|
<span className="text-[var(--text)]">pip install ./sdk/python/datastack</span>{'\n\n'}
|
||||||
<span className="text-[var(--text-faint)]"># Auto-loads config.datastack.py from project root</span>{'\n'}
|
|
||||||
<span className="text-[var(--text)]">from datastack import DataStack</span>{'\n\n'}
|
<span className="text-[var(--text)]">from datastack import DataStack</span>{'\n\n'}
|
||||||
<span className="text-[var(--text)]">client = DataStack()</span>{'\n'}
|
<span className="text-[var(--text)]">client = DataStack()</span>{'\n'}
|
||||||
<span className="text-[var(--text)]">client.send({'{'}<span className="text-[var(--accent)]">"event_type"</span>: <span className="text-[var(--accent)]">"sensor_telemetry"</span>, <span className="text-[var(--accent)]">"temperature"</span>: 24.8, <span className="text-[var(--accent)]">"status"</span>: <span className="text-[var(--accent)]">"NOMINAL"</span>{'}'})</span>{'\n'}
|
<span className="text-[var(--text)]">client.send({'{'}<span className="text-[var(--accent)]">"event_type"</span>: <span className="text-[var(--accent)]">"sensor_telemetry"</span>, <span className="text-[var(--accent)]">"temperature"</span>: 24.8, <span className="text-[var(--accent)]">"status"</span>: <span className="text-[var(--accent)]">"NOMINAL"</span>{'}'})</span>{'\n'}
|
||||||
<span className="text-[var(--text)]">client.send_file(<span className="text-[var(--accent)]">"telemetry_batch.csv"</span>)</span>{'\n'}
|
<span className="text-[var(--text)]">client.send_file(<span className="text-[var(--accent)]">"telemetry_batch.csv"</span>)</span>{'\n'}
|
||||||
<span className="text-[var(--text)]">client.disconnect()</span>
|
<span className="text-[var(--text)]">client.disconnect()</span>
|
||||||
</CodeBlock>
|
</CodeBlock>
|
||||||
<div className="bg-[var(--surface)] p-5 sm:p-6">
|
<CodeBlock>
|
||||||
<p className="mb-2 text-[11px] uppercase text-[var(--text-faint)]">How it works</p>
|
<span className="text-[var(--text-faint)]">{'// Node.js / TypeScript SDK (@datastack/sdk v2.2.0)'}</span>{'\n'}
|
||||||
<h3 className="mb-2 text-[14px] font-medium text-[var(--text)]">Zero-boilerplate config. Auto-recovery built in.</h3>
|
<span className="text-[var(--text)]">npm install ./sdk/node/datastack</span>{'\n\n'}
|
||||||
<p className="text-[12px] leading-relaxed text-[var(--text-muted)]">
|
<span className="text-[var(--text)]">import {'{'} DataStack {'}'} from <span className="text-[var(--accent)]">'@datastack/sdk'</span>;</span>{'\n\n'}
|
||||||
Install the self-hosted SDK from <code className="text-[var(--text)]">sdk/python/datastack</code> (or build a wheel with <code className="text-[var(--text)]">pip wheel ./sdk/python/datastack -w dist/</code>) and drop <code className="text-[var(--text)]">config.datastack.py</code> into your Python project to stream single events, batches, or <code className="text-[var(--text)]">.csv</code> / <code className="text-[var(--text)]">.json</code> files directly—with automatic HTTP fallback, offline disk spooling, and Heartbeat IDLE recovery.
|
<span className="text-[var(--text)]">const client = new DataStack();</span>{'\n'}
|
||||||
</p>
|
<span className="text-[var(--text)]">await client.send({'{'} <span className="text-[var(--accent)]">event_type</span>: <span className="text-[var(--accent)]">'sensor_telemetry'</span>, <span className="text-[var(--accent)]">temperature</span>: 24.8 {'}'});</span>{'\n'}
|
||||||
</div>
|
<span className="text-[var(--text)]">await client.sendFile(<span className="text-[var(--accent)]">'telemetry_batch.csv'</span>);</span>{'\n'}
|
||||||
|
<span className="text-[var(--text)]">await client.shutdown(5000);</span>
|
||||||
|
</CodeBlock>
|
||||||
|
</div>
|
||||||
|
<div className="border-x border-b border-[var(--border)] bg-[var(--surface)] px-5 py-4 sm:px-6">
|
||||||
|
<p className="mb-1 text-[11px] uppercase text-[var(--text-faint)]">Zero-boilerplate config · Auto-recovery built in</p>
|
||||||
|
<p className="text-[12px] leading-relaxed text-[var(--text-muted)]">
|
||||||
|
Install either <code className="text-[var(--text)]">sdk/python/datastack</code> (<code className="text-[var(--text)]">config.datastack.py</code>) or <code className="text-[var(--text)]">@datastack/sdk</code> in <code className="text-[var(--text)]">sdk/node/datastack</code> (<code className="text-[var(--text)]">datastack.config.json</code>) to stream single events, batches, or <code className="text-[var(--text)]">.csv</code> / <code className="text-[var(--text)]">.json</code> / <code className="text-[var(--text)]">.jsonl</code> files—with automatic HTTP fallback, offline disk spooling, and graceful shutdown drain.
|
||||||
|
</p>
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
'use client';
|
'use client';
|
||||||
|
|
||||||
import { PageShell } from '@/components/layout/PageShell';
|
import { PageShell } from '@/components/layout/PageShell';
|
||||||
|
import { useVconRole } from '@/components/layout/VconShell';
|
||||||
import { Button } from '@/components/ui/Button';
|
import { Button } from '@/components/ui/Button';
|
||||||
import { Input, Select } from '@/components/ui/Input';
|
import { Input, Select } from '@/components/ui/Input';
|
||||||
import { Modal } from '@/components/ui/Modal';
|
import { Modal } from '@/components/ui/Modal';
|
||||||
@@ -34,8 +35,9 @@ interface TeamMemberItem {
|
|||||||
|
|
||||||
export function TeamWorkspace() {
|
export function TeamWorkspace() {
|
||||||
const { success, error } = useToast();
|
const { success, error } = useToast();
|
||||||
|
const shellRole = useVconRole();
|
||||||
const [members, setMembers] = useState<TeamMemberItem[]>([]);
|
const [members, setMembers] = useState<TeamMemberItem[]>([]);
|
||||||
const [currentRole, setCurrentRole] = useState<string>('owner');
|
const [currentRole, setCurrentRole] = useState<string>(() => shellRole || 'viewer');
|
||||||
const [loading, setLoading] = useState<boolean>(true);
|
const [loading, setLoading] = useState<boolean>(true);
|
||||||
|
|
||||||
const [createOpen, setCreateOpen] = useState<boolean>(false);
|
const [createOpen, setCreateOpen] = useState<boolean>(false);
|
||||||
@@ -43,6 +45,7 @@ export function TeamWorkspace() {
|
|||||||
const [newPassword, setNewPassword] = useState<string>('');
|
const [newPassword, setNewPassword] = useState<string>('');
|
||||||
const [newRole, setNewRole] = useState<string>('viewer');
|
const [newRole, setNewRole] = useState<string>('viewer');
|
||||||
const [creating, setCreating] = useState<boolean>(false);
|
const [creating, setCreating] = useState<boolean>(false);
|
||||||
|
const [deleteTarget, setDeleteTarget] = useState<TeamMemberItem | null>(null);
|
||||||
|
|
||||||
const [busyId, setBusyId] = useState<string | null>(null);
|
const [busyId, setBusyId] = useState<string | null>(null);
|
||||||
|
|
||||||
@@ -186,6 +189,7 @@ export function TeamWorkspace() {
|
|||||||
throw new Error(data?.error || 'Delete failed.');
|
throw new Error(data?.error || 'Delete failed.');
|
||||||
}
|
}
|
||||||
success(`Removed ${member.email} from team.`);
|
success(`Removed ${member.email} from team.`);
|
||||||
|
setDeleteTarget(null);
|
||||||
await fetchMembers();
|
await fetchMembers();
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
error(err instanceof Error ? err.message : 'Unable to delete team member.');
|
error(err instanceof Error ? err.message : 'Unable to delete team member.');
|
||||||
@@ -246,7 +250,7 @@ export function TeamWorkspace() {
|
|||||||
value: String(ownerCount),
|
value: String(ownerCount),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
label: 'Admins / Operators',
|
label: 'Operators / Admins',
|
||||||
value: String(operatorCount),
|
value: String(operatorCount),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -306,14 +310,15 @@ export function TeamWorkspace() {
|
|||||||
<Select
|
<Select
|
||||||
compact
|
compact
|
||||||
aria-label={`Role for ${row.email}`}
|
aria-label={`Role for ${row.email}`}
|
||||||
value={row.role === 'operator' ? 'admin' : row.role}
|
value={row.role}
|
||||||
disabled={busyId === row.id}
|
disabled={busyId === row.id}
|
||||||
onChange={(e) =>
|
onChange={(e) =>
|
||||||
void handleRoleChange(row, e.target.value)
|
void handleRoleChange(row, e.target.value)
|
||||||
}
|
}
|
||||||
options={[
|
options={[
|
||||||
{ value: 'viewer', label: 'Viewer (Read-Only)' },
|
{ value: 'viewer', label: 'Viewer (Read-Only)' },
|
||||||
{ value: 'admin', label: 'Admin / Operator' },
|
{ value: 'operator', label: 'Operator (Manage Clients)' },
|
||||||
|
{ value: 'admin', label: 'Admin (Manage Clients)' },
|
||||||
{ value: 'owner', label: 'Owner (Full Control)' },
|
{ value: 'owner', label: 'Owner (Full Control)' },
|
||||||
]}
|
]}
|
||||||
/>
|
/>
|
||||||
@@ -359,7 +364,7 @@ export function TeamWorkspace() {
|
|||||||
variant="danger"
|
variant="danger"
|
||||||
size="sm"
|
size="sm"
|
||||||
disabled={busyId === row.id}
|
disabled={busyId === row.id}
|
||||||
onClick={() => void handleDeleteMember(row)}
|
onClick={() => setDeleteTarget(row)}
|
||||||
>
|
>
|
||||||
Remove
|
Remove
|
||||||
</Button>
|
</Button>
|
||||||
@@ -410,9 +415,13 @@ export function TeamWorkspace() {
|
|||||||
value: 'viewer',
|
value: 'viewer',
|
||||||
label: 'Viewer — Read-only dashboards, events, clients & exports',
|
label: 'Viewer — Read-only dashboards, events, clients & exports',
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
value: 'operator',
|
||||||
|
label: 'Operator — Manage SDK clients, tokens & manual flushes',
|
||||||
|
},
|
||||||
{
|
{
|
||||||
value: 'admin',
|
value: 'admin',
|
||||||
label: 'Admin / Operator — Manage SDK clients, tokens & flushes',
|
label: 'Admin — Operational role (manage SDK clients & flushes)',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
value: 'owner',
|
value: 'owner',
|
||||||
@@ -440,6 +449,50 @@ export function TeamWorkspace() {
|
|||||||
</div>
|
</div>
|
||||||
</form>
|
</form>
|
||||||
</Modal>
|
</Modal>
|
||||||
|
|
||||||
|
<Modal
|
||||||
|
isOpen={Boolean(deleteTarget)}
|
||||||
|
onClose={() => {
|
||||||
|
if (!busyId) setDeleteTarget(null);
|
||||||
|
}}
|
||||||
|
title="Confirm Team Member Removal"
|
||||||
|
maxWidth="sm"
|
||||||
|
>
|
||||||
|
{deleteTarget && (
|
||||||
|
<div className="flex flex-col gap-3">
|
||||||
|
<p className="text-[12px] text-[var(--text-muted)] leading-relaxed">
|
||||||
|
Remove{' '}
|
||||||
|
<span className="font-mono font-medium text-[var(--text)]">
|
||||||
|
{deleteTarget.email}
|
||||||
|
</span>{' '}
|
||||||
|
(<span className="font-mono uppercase">{deleteTarget.role}</span>) from
|
||||||
|
administrator access? Active sessions will stop working immediately.
|
||||||
|
</p>
|
||||||
|
<div className="flex justify-end gap-2 pt-1">
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
variant="secondary"
|
||||||
|
size="sm"
|
||||||
|
disabled={busyId === deleteTarget.id}
|
||||||
|
onClick={() => setDeleteTarget(null)}
|
||||||
|
>
|
||||||
|
Cancel
|
||||||
|
</Button>
|
||||||
|
<Button
|
||||||
|
type="button"
|
||||||
|
variant="danger"
|
||||||
|
size="sm"
|
||||||
|
data-testid="confirm-remove-member"
|
||||||
|
loading={busyId === deleteTarget.id}
|
||||||
|
disabled={busyId === deleteTarget.id}
|
||||||
|
onClick={() => void handleDeleteMember(deleteTarget)}
|
||||||
|
>
|
||||||
|
Confirm Remove
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</Modal>
|
||||||
</PageShell>
|
</PageShell>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,311 @@
|
|||||||
|
import dns from "node:dns/promises";
|
||||||
|
import net from "node:net";
|
||||||
|
|
||||||
|
export class SSRFBlockedError extends Error {
|
||||||
|
public readonly code = "SSRF_BLOCKED_HOST" as const;
|
||||||
|
public readonly hostname: string;
|
||||||
|
|
||||||
|
constructor(message: string, hostname: string) {
|
||||||
|
super(message);
|
||||||
|
this.name = "SSRFBlockedError";
|
||||||
|
this.hostname = hostname;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface OutboundUrlValidationOptions {
|
||||||
|
allowPrivate?: boolean;
|
||||||
|
env?: NodeJS.ProcessEnv;
|
||||||
|
dnsLookup?: (
|
||||||
|
hostname: string,
|
||||||
|
) => Promise<Array<{ address: string; family: number }>>;
|
||||||
|
fieldName?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface OutboundUrlValidationResult {
|
||||||
|
allowed: boolean;
|
||||||
|
url?: URL;
|
||||||
|
code?: "SSRF_BLOCKED_HOST" | "INVALID_URL";
|
||||||
|
error?: string;
|
||||||
|
hostname?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ALWAYS_BLOCKED_HOSTNAMES = new Set([
|
||||||
|
"metadata.google.internal",
|
||||||
|
"metadata.goog",
|
||||||
|
"metadata",
|
||||||
|
"instance-data",
|
||||||
|
"instance-data.ec2.internal",
|
||||||
|
]);
|
||||||
|
|
||||||
|
const PRIVATE_HOSTNAMES = new Set([
|
||||||
|
"localhost",
|
||||||
|
"localhost.localdomain",
|
||||||
|
"ip6-localhost",
|
||||||
|
"ip6-loopback",
|
||||||
|
]);
|
||||||
|
|
||||||
|
function buildAlwaysBlockedBlockList(): net.BlockList {
|
||||||
|
const list = new net.BlockList();
|
||||||
|
// Unspecified / zero network
|
||||||
|
list.addSubnet("0.0.0.0", 8, "ipv4");
|
||||||
|
// Link-local & cloud metadata (AWS/GCP/Azure/Oracle 169.254.169.254)
|
||||||
|
list.addSubnet("169.254.0.0", 16, "ipv4");
|
||||||
|
// Multicast & reserved
|
||||||
|
list.addSubnet("224.0.0.0", 4, "ipv4");
|
||||||
|
list.addSubnet("240.0.0.0", 4, "ipv4");
|
||||||
|
// IPv6 unspecified, link-local, multicast
|
||||||
|
list.addAddress("::", "ipv6");
|
||||||
|
list.addSubnet("fe80::", 10, "ipv6");
|
||||||
|
list.addSubnet("ff00::", 8, "ipv6");
|
||||||
|
return list;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildPrivateBlockList(): net.BlockList {
|
||||||
|
const list = new net.BlockList();
|
||||||
|
// Loopback
|
||||||
|
list.addSubnet("127.0.0.0", 8, "ipv4");
|
||||||
|
// RFC 1918 private networks
|
||||||
|
list.addSubnet("10.0.0.0", 8, "ipv4");
|
||||||
|
list.addSubnet("172.16.0.0", 12, "ipv4");
|
||||||
|
list.addSubnet("192.168.0.0", 16, "ipv4");
|
||||||
|
// Carrier-grade NAT (RFC 6598)
|
||||||
|
list.addSubnet("100.64.0.0", 10, "ipv4");
|
||||||
|
// Benchmarking / documentation
|
||||||
|
list.addSubnet("198.18.0.0", 15, "ipv4");
|
||||||
|
// IPv6 loopback & Unique Local Addresses (ULA)
|
||||||
|
list.addAddress("::1", "ipv6");
|
||||||
|
list.addSubnet("fc00::", 7, "ipv6");
|
||||||
|
return list;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ALWAYS_BLOCKED_IPS = buildAlwaysBlockedBlockList();
|
||||||
|
const PRIVATE_IPS = buildPrivateBlockList();
|
||||||
|
|
||||||
|
export function isPrivateOutboundUrlAllowed(
|
||||||
|
env: NodeJS.ProcessEnv = process.env,
|
||||||
|
): boolean {
|
||||||
|
const raw = String(env.DATASTACK_ALLOW_PRIVATE_URLS || "")
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
|
return raw === "true" || raw === "1" || raw === "yes";
|
||||||
|
}
|
||||||
|
|
||||||
|
function stripBracketsAndZone(rawHost: string): string {
|
||||||
|
let host = String(rawHost || "").trim().toLowerCase();
|
||||||
|
if (host.startsWith("[") && host.endsWith("]")) {
|
||||||
|
host = host.slice(1, -1);
|
||||||
|
}
|
||||||
|
const zoneIdx = host.indexOf("%");
|
||||||
|
if (zoneIdx >= 0) {
|
||||||
|
host = host.slice(0, zoneIdx);
|
||||||
|
}
|
||||||
|
// Remove trailing FQDN dot (e.g. "localhost.")
|
||||||
|
return host.replace(/\.+$/, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeIpForCheck(rawIp: string): { ip: string; family: 4 | 6 } | null {
|
||||||
|
const clean = stripBracketsAndZone(rawIp);
|
||||||
|
if (!clean) return null;
|
||||||
|
|
||||||
|
// Handle IPv4-mapped IPv6 addresses (e.g. ::ffff:127.0.0.1 or ::ffff:7f00:1)
|
||||||
|
if (clean.startsWith("::ffff:")) {
|
||||||
|
const mapped = clean.slice("::ffff:".length);
|
||||||
|
if (net.isIP(mapped) === 4) {
|
||||||
|
return { ip: mapped, family: 4 };
|
||||||
|
}
|
||||||
|
const hexParts = mapped.split(":");
|
||||||
|
if (hexParts.length === 2) {
|
||||||
|
const hi = Number.parseInt(hexParts[0], 16);
|
||||||
|
const lo = Number.parseInt(hexParts[1], 16);
|
||||||
|
if (
|
||||||
|
Number.isInteger(hi) &&
|
||||||
|
Number.isInteger(lo) &&
|
||||||
|
hi >= 0 &&
|
||||||
|
hi <= 0xffff &&
|
||||||
|
lo >= 0 &&
|
||||||
|
lo <= 0xffff
|
||||||
|
) {
|
||||||
|
const ipv4 = `${(hi >> 8) & 0xff}.${hi & 0xff}.${(lo >> 8) & 0xff}.${lo & 0xff}`;
|
||||||
|
if (net.isIP(ipv4) === 4) {
|
||||||
|
return { ip: ipv4, family: 4 };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const family = net.isIP(clean);
|
||||||
|
if (family === 4 || family === 6) {
|
||||||
|
return { ip: clean, family };
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function checkIpAddressAllowed(
|
||||||
|
rawIp: string,
|
||||||
|
allowPrivate: boolean,
|
||||||
|
): { allowed: boolean; reason?: string } {
|
||||||
|
const parsed = normalizeIpForCheck(rawIp);
|
||||||
|
if (!parsed) {
|
||||||
|
return { allowed: false, reason: `Invalid IP address '${rawIp}'.` };
|
||||||
|
}
|
||||||
|
const ipType = parsed.family === 4 ? "ipv4" : "ipv6";
|
||||||
|
if (ALWAYS_BLOCKED_IPS.check(parsed.ip, ipType)) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
reason: `Address '${parsed.ip}' is in a restricted link-local/metadata/reserved network range.`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (!allowPrivate && PRIVATE_IPS.check(parsed.ip, ipType)) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
reason: `Address '${parsed.ip}' is a private or loopback network address (set DATASTACK_ALLOW_PRIVATE_URLS=true only if intentionally targeting a private host).`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return { allowed: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function validateOutboundUrlSync(
|
||||||
|
rawUrl: string,
|
||||||
|
options: OutboundUrlValidationOptions = {},
|
||||||
|
): OutboundUrlValidationResult {
|
||||||
|
const fieldName = options.fieldName || "URL";
|
||||||
|
const allowPrivate =
|
||||||
|
options.allowPrivate !== undefined
|
||||||
|
? options.allowPrivate
|
||||||
|
: isPrivateOutboundUrlAllowed(options.env);
|
||||||
|
|
||||||
|
const trimmed = String(rawUrl || "").trim();
|
||||||
|
if (!trimmed) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
code: "INVALID_URL",
|
||||||
|
error: `${fieldName} is required.`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
let parsed: URL;
|
||||||
|
try {
|
||||||
|
parsed = new URL(trimmed);
|
||||||
|
} catch {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
code: "INVALID_URL",
|
||||||
|
error: `${fieldName} must be a valid http:// or https:// URL.`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
code: "SSRF_BLOCKED_HOST",
|
||||||
|
hostname: parsed.hostname,
|
||||||
|
error: `${fieldName} must use http:// or https://.`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (parsed.username || parsed.password) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
code: "SSRF_BLOCKED_HOST",
|
||||||
|
hostname: parsed.hostname,
|
||||||
|
error: `${fieldName} must not embed URL credentials.`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const host = stripBracketsAndZone(parsed.hostname);
|
||||||
|
if (!host) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
code: "INVALID_URL",
|
||||||
|
error: `${fieldName} must include a valid hostname.`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ALWAYS_BLOCKED_HOSTNAMES.has(host)) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
code: "SSRF_BLOCKED_HOST",
|
||||||
|
hostname: host,
|
||||||
|
error: `${fieldName} target '${host}' is blocked (cloud metadata/internal endpoint).`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Literal IP check (covers IPv4, IPv6, and IPv4-mapped IPv6)
|
||||||
|
const ipParsed = normalizeIpForCheck(host);
|
||||||
|
if (ipParsed) {
|
||||||
|
const ipCheck = checkIpAddressAllowed(host, allowPrivate);
|
||||||
|
if (!ipCheck.allowed) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
code: "SSRF_BLOCKED_HOST",
|
||||||
|
hostname: host,
|
||||||
|
error: `${fieldName} blocked: ${ipCheck.reason}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return { allowed: true, url: parsed, hostname: host };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Non-IP hostname checks
|
||||||
|
if (!allowPrivate) {
|
||||||
|
if (
|
||||||
|
PRIVATE_HOSTNAMES.has(host) ||
|
||||||
|
host.endsWith(".localhost") ||
|
||||||
|
host.endsWith(".local") ||
|
||||||
|
host.endsWith(".internal") ||
|
||||||
|
!host.includes(".")
|
||||||
|
) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
code: "SSRF_BLOCKED_HOST",
|
||||||
|
hostname: host,
|
||||||
|
error: `${fieldName} target '${host}' is a private/internal hostname (set DATASTACK_ALLOW_PRIVATE_URLS=true only for trusted private hosts).`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { allowed: true, url: parsed, hostname: host };
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function validateOutboundUrlAsync(
|
||||||
|
rawUrl: string,
|
||||||
|
options: OutboundUrlValidationOptions = {},
|
||||||
|
): Promise<OutboundUrlValidationResult> {
|
||||||
|
const syncResult = validateOutboundUrlSync(rawUrl, options);
|
||||||
|
if (!syncResult.allowed || !syncResult.url || !syncResult.hostname) {
|
||||||
|
return syncResult;
|
||||||
|
}
|
||||||
|
|
||||||
|
const host = syncResult.hostname;
|
||||||
|
const fieldName = options.fieldName || "URL";
|
||||||
|
const allowPrivate =
|
||||||
|
options.allowPrivate !== undefined
|
||||||
|
? options.allowPrivate
|
||||||
|
: isPrivateOutboundUrlAllowed(options.env);
|
||||||
|
|
||||||
|
// Skip DNS resolution if already validated as a literal IP
|
||||||
|
if (normalizeIpForCheck(host)) {
|
||||||
|
return syncResult;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const lookupFn =
|
||||||
|
options.dnsLookup ||
|
||||||
|
((h: string) => dns.lookup(h, { all: true, verbatim: true }));
|
||||||
|
const records = await lookupFn(host);
|
||||||
|
for (const rec of records) {
|
||||||
|
const check = checkIpAddressAllowed(rec.address, allowPrivate);
|
||||||
|
if (!check.allowed) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
code: "SSRF_BLOCKED_HOST",
|
||||||
|
hostname: host,
|
||||||
|
error: `${fieldName} hostname '${host}' resolved to a blocked address: ${check.reason}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// DNS resolution failure (e.g. ENOTFOUND/EAI_AGAIN) will be handled cleanly by the caller or fetch()
|
||||||
|
}
|
||||||
|
|
||||||
|
return syncResult;
|
||||||
|
}
|
||||||
@@ -386,6 +386,14 @@ export class CSVStorageManager {
|
|||||||
if (!gunzip.destroyed) {
|
if (!gunzip.destroyed) {
|
||||||
gunzip.destroy(err);
|
gunzip.destroy(err);
|
||||||
}
|
}
|
||||||
|
if (!fileStream.destroyed) {
|
||||||
|
fileStream.destroy();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
gunzip.on('error', () => {
|
||||||
|
if (!fileStream.destroyed) {
|
||||||
|
fileStream.destroy();
|
||||||
|
}
|
||||||
});
|
});
|
||||||
gunzip.on('close', () => {
|
gunzip.on('close', () => {
|
||||||
if (!fileStream.destroyed) {
|
if (!fileStream.destroyed) {
|
||||||
@@ -396,16 +404,21 @@ export class CSVStorageManager {
|
|||||||
return gunzip;
|
return gunzip;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static async countCsvDataRowsStreamAsync(
|
public static async countCsvDataRowsStreamAsync(
|
||||||
filePath: string,
|
filePath: string,
|
||||||
): Promise<number> {
|
): Promise<number> {
|
||||||
const counter = this.createChunkRowCounter();
|
const counter = this.createChunkRowCounter();
|
||||||
return new Promise<number>((resolve, reject) => {
|
const stream = this.openPartitionTextStream(filePath);
|
||||||
const stream = this.openPartitionTextStream(filePath);
|
try {
|
||||||
stream.on('data', (chunk) => counter.pushChunk(chunk));
|
for await (const chunk of stream) {
|
||||||
stream.on('end', () => resolve(counter.finish()));
|
counter.pushChunk(chunk as Buffer | string);
|
||||||
stream.on('error', (err) => reject(err));
|
}
|
||||||
});
|
return counter.finish();
|
||||||
|
} finally {
|
||||||
|
if (!stream.destroyed) {
|
||||||
|
stream.destroy();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
+98
-12
@@ -1,5 +1,7 @@
|
|||||||
import crypto from "node:crypto";
|
import crypto from "node:crypto";
|
||||||
import fs from "node:fs";
|
import fs from "node:fs";
|
||||||
|
import { Readable } from "node:stream";
|
||||||
|
import { validateOutboundUrlAsync } from "@/core/Security/ssrfGuard";
|
||||||
|
|
||||||
export interface S3ArchiveConfig {
|
export interface S3ArchiveConfig {
|
||||||
endpoint: string;
|
endpoint: string;
|
||||||
@@ -17,7 +19,12 @@ export interface S3OperationResult {
|
|||||||
bucket?: string;
|
bucket?: string;
|
||||||
key?: string;
|
key?: string;
|
||||||
etag?: string;
|
etag?: string;
|
||||||
code?: "S3_INVALID_CONFIG" | "S3_CONNECTION_FAILED" | "S3_ARCHIVE_FAILED";
|
code?:
|
||||||
|
| "S3_INVALID_CONFIG"
|
||||||
|
| "S3_CONNECTION_FAILED"
|
||||||
|
| "S3_ARCHIVE_FAILED"
|
||||||
|
| "SSRF_BLOCKED_HOST"
|
||||||
|
| "INVALID_URL";
|
||||||
error?: string;
|
error?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -25,6 +32,27 @@ function sha256Hex(data: Buffer | string): string {
|
|||||||
return crypto.createHash("sha256").update(data).digest("hex");
|
return crypto.createHash("sha256").update(data).digest("hex");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function computeFileSha256AndSize(
|
||||||
|
filePath: string,
|
||||||
|
): Promise<{ sha256Hex: string; sizeBytes: number }> {
|
||||||
|
const stat = await fs.promises.stat(filePath);
|
||||||
|
const hash = crypto.createHash("sha256");
|
||||||
|
const readStream = fs.createReadStream(filePath, { highWaterMark: 64 * 1024 });
|
||||||
|
try {
|
||||||
|
for await (const chunk of readStream) {
|
||||||
|
hash.update(chunk as Buffer | string);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
sha256Hex: hash.digest("hex"),
|
||||||
|
sizeBytes: stat.size,
|
||||||
|
};
|
||||||
|
} finally {
|
||||||
|
if (!readStream.destroyed) {
|
||||||
|
readStream.destroy();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function hmacSha256(key: Buffer | string, data: string): Buffer {
|
function hmacSha256(key: Buffer | string, data: string): Buffer {
|
||||||
return crypto.createHmac("sha256", key).update(data, "utf8").digest();
|
return crypto.createHmac("sha256", key).update(data, "utf8").digest();
|
||||||
}
|
}
|
||||||
@@ -204,6 +232,16 @@ export async function testS3Connection(
|
|||||||
config.endpoint,
|
config.endpoint,
|
||||||
config.region,
|
config.region,
|
||||||
);
|
);
|
||||||
|
const ssrfCheck = await validateOutboundUrlAsync(url.toString(), {
|
||||||
|
fieldName: "S3 Endpoint URL",
|
||||||
|
});
|
||||||
|
if (!ssrfCheck.allowed) {
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
code: ssrfCheck.code || "SSRF_BLOCKED_HOST",
|
||||||
|
error: ssrfCheck.error || "S3 Endpoint URL is blocked.",
|
||||||
|
};
|
||||||
|
}
|
||||||
const signed = signS3Request({
|
const signed = signS3Request({
|
||||||
method: "GET",
|
method: "GET",
|
||||||
endpointUrl: url,
|
endpointUrl: url,
|
||||||
@@ -219,9 +257,21 @@ export async function testS3Connection(
|
|||||||
method: "GET",
|
method: "GET",
|
||||||
headers: signed.headers,
|
headers: signed.headers,
|
||||||
cache: "no-store",
|
cache: "no-store",
|
||||||
|
redirect: "manual",
|
||||||
signal: AbortSignal.timeout(8000),
|
signal: AbortSignal.timeout(8000),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
if (response.status >= 300 && response.status < 400) {
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
status: response.status,
|
||||||
|
endpoint: url.origin,
|
||||||
|
bucket: config.bucket.trim(),
|
||||||
|
code: "SSRF_BLOCKED_HOST",
|
||||||
|
error: `S3 endpoint returned an unsupported HTTP ${response.status} redirect.`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
const bodyPreview = await response.text().catch(() => "");
|
const bodyPreview = await response.text().catch(() => "");
|
||||||
return {
|
return {
|
||||||
@@ -272,8 +322,19 @@ export async function uploadFileToS3(
|
|||||||
config.endpoint,
|
config.endpoint,
|
||||||
config.region,
|
config.region,
|
||||||
);
|
);
|
||||||
const bodyBuffer = await fs.promises.readFile(localFilePath);
|
const ssrfCheck = await validateOutboundUrlAsync(url.toString(), {
|
||||||
const payloadHashHex = sha256Hex(bodyBuffer);
|
fieldName: "S3 Endpoint URL",
|
||||||
|
});
|
||||||
|
if (!ssrfCheck.allowed) {
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
code: ssrfCheck.code || "SSRF_BLOCKED_HOST",
|
||||||
|
error: ssrfCheck.error || "S3 Endpoint URL is blocked.",
|
||||||
|
key: objectKey,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const { sha256Hex: payloadHashHex, sizeBytes } =
|
||||||
|
await computeFileSha256AndSize(localFilePath);
|
||||||
const contentType = localFilePath.endsWith(".gz")
|
const contentType = localFilePath.endsWith(".gz")
|
||||||
? "application/gzip"
|
? "application/gzip"
|
||||||
: "text/csv; charset=utf-8";
|
: "text/csv; charset=utf-8";
|
||||||
@@ -290,16 +351,41 @@ export async function uploadFileToS3(
|
|||||||
contentType,
|
contentType,
|
||||||
});
|
});
|
||||||
|
|
||||||
const response = await fetch(signed.targetUrl, {
|
const uploadStream = fs.createReadStream(localFilePath, {
|
||||||
method: "PUT",
|
highWaterMark: 64 * 1024,
|
||||||
headers: {
|
|
||||||
...signed.headers,
|
|
||||||
"content-length": String(bodyBuffer.byteLength),
|
|
||||||
},
|
|
||||||
body: new Uint8Array(bodyBuffer),
|
|
||||||
cache: "no-store",
|
|
||||||
signal: AbortSignal.timeout(30000),
|
|
||||||
});
|
});
|
||||||
|
let response: Response;
|
||||||
|
try {
|
||||||
|
const requestInit: RequestInit & { duplex: "half" } = {
|
||||||
|
method: "PUT",
|
||||||
|
headers: {
|
||||||
|
...signed.headers,
|
||||||
|
"content-length": String(sizeBytes),
|
||||||
|
},
|
||||||
|
body: Readable.toWeb(uploadStream) as unknown as BodyInit,
|
||||||
|
duplex: "half",
|
||||||
|
cache: "no-store",
|
||||||
|
redirect: "manual",
|
||||||
|
signal: AbortSignal.timeout(30000),
|
||||||
|
};
|
||||||
|
response = await fetch(signed.targetUrl, requestInit);
|
||||||
|
} finally {
|
||||||
|
if (!uploadStream.destroyed) {
|
||||||
|
uploadStream.destroy();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (response.status >= 300 && response.status < 400) {
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
status: response.status,
|
||||||
|
endpoint: url.origin,
|
||||||
|
bucket: config.bucket.trim(),
|
||||||
|
key: objectKey,
|
||||||
|
code: "SSRF_BLOCKED_HOST",
|
||||||
|
error: `S3 upload returned an unsupported HTTP ${response.status} redirect.`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
const bodyPreview = await response.text().catch(() => "");
|
const bodyPreview = await response.text().catch(() => "");
|
||||||
|
|||||||
@@ -309,6 +309,34 @@ export class SettingsManager {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static async bootstrapMaintenanceSchedulerOnBoot(): Promise<{
|
||||||
|
schedulerActive: boolean;
|
||||||
|
hydrated: boolean;
|
||||||
|
error?: string;
|
||||||
|
}> {
|
||||||
|
this.ensureMaintenanceScheduler();
|
||||||
|
try {
|
||||||
|
await this.getSettingsAsync();
|
||||||
|
return {
|
||||||
|
schedulerActive: this.isMaintenanceSchedulerActive(),
|
||||||
|
hydrated:
|
||||||
|
!process.env.DATABASE_URL?.trim() ||
|
||||||
|
globals.__dataStackSettingsHydrated === true,
|
||||||
|
};
|
||||||
|
} catch (err: unknown) {
|
||||||
|
const message = err instanceof Error ? err.message : String(err);
|
||||||
|
console.warn(
|
||||||
|
"[DataStack Boot] Maintenance scheduler init deferred/failed:",
|
||||||
|
message,
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
schedulerActive: this.isMaintenanceSchedulerActive(),
|
||||||
|
hydrated: false,
|
||||||
|
error: message,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public static stopMaintenanceScheduler(): void {
|
public static stopMaintenanceScheduler(): void {
|
||||||
if (globals.__dataStackMaintenanceTimer) {
|
if (globals.__dataStackMaintenanceTimer) {
|
||||||
clearInterval(globals.__dataStackMaintenanceTimer);
|
clearInterval(globals.__dataStackMaintenanceTimer);
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import crypto from "node:crypto";
|
import crypto from "node:crypto";
|
||||||
|
import { validateOutboundUrlAsync } from "@/core/Security/ssrfGuard";
|
||||||
|
|
||||||
export type WebhookAlertEventType =
|
export type WebhookAlertEventType =
|
||||||
| "DISK_SPACE_LOW"
|
| "DISK_SPACE_LOW"
|
||||||
@@ -30,6 +31,7 @@ export interface WebhookDispatchResult {
|
|||||||
ok: boolean;
|
ok: boolean;
|
||||||
skipped?: boolean;
|
skipped?: boolean;
|
||||||
reason?: string;
|
reason?: string;
|
||||||
|
code?: string;
|
||||||
status?: number;
|
status?: number;
|
||||||
error?: string;
|
error?: string;
|
||||||
}
|
}
|
||||||
@@ -106,21 +108,17 @@ export async function dispatchWebhookAlertAsync(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
let parsedUrl: URL;
|
const ssrfCheck = await validateOutboundUrlAsync(webhookUrl, {
|
||||||
try {
|
fieldName: "Webhook URL",
|
||||||
parsedUrl = new URL(webhookUrl);
|
});
|
||||||
if (parsedUrl.protocol !== "http:" && parsedUrl.protocol !== "https:") {
|
if (!ssrfCheck.allowed || !ssrfCheck.url) {
|
||||||
return {
|
|
||||||
ok: false,
|
|
||||||
error: "Webhook URL must use http:// or https://.",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
return {
|
return {
|
||||||
ok: false,
|
ok: false,
|
||||||
error: "Webhook URL is not a valid URL.",
|
code: ssrfCheck.code || "SSRF_BLOCKED_HOST",
|
||||||
|
error: ssrfCheck.error || "Webhook URL is not allowed.",
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
const parsedUrl = ssrfCheck.url;
|
||||||
|
|
||||||
const dedupeKey = `${payload.event}:${payload.clientId || "global"}`;
|
const dedupeKey = `${payload.event}:${payload.clientId || "global"}`;
|
||||||
const cooldownMs = Math.max(0, (Number(config.cooldownSec) || 300) * 1000);
|
const cooldownMs = Math.max(0, (Number(config.cooldownSec) || 300) * 1000);
|
||||||
@@ -156,9 +154,19 @@ export async function dispatchWebhookAlertAsync(
|
|||||||
headers,
|
headers,
|
||||||
body: bodyStr,
|
body: bodyStr,
|
||||||
cache: "no-store",
|
cache: "no-store",
|
||||||
|
redirect: "manual",
|
||||||
signal: AbortSignal.timeout(timeoutMs),
|
signal: AbortSignal.timeout(timeoutMs),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
if (res.status >= 300 && res.status < 400) {
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
code: "SSRF_BLOCKED_HOST",
|
||||||
|
status: res.status,
|
||||||
|
error: `Webhook endpoint attempted an unsupported HTTP ${res.status} redirect.`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
if (!options?.bypassCooldown) {
|
if (!options?.bypassCooldown) {
|
||||||
// Keep cooldown entry so failing webhooks don't DDOS a broken upstream endpoint
|
// Keep cooldown entry so failing webhooks don't DDOS a broken upstream endpoint
|
||||||
|
|||||||
@@ -15,6 +15,40 @@ import {
|
|||||||
} from "./adminSession";
|
} from "./adminSession";
|
||||||
import { clearAuditLogsForTests } from "./auditLog";
|
import { clearAuditLogsForTests } from "./auditLog";
|
||||||
import { DatabaseEngine, ensureCoreSchemaOnPool } from "../Database/database";
|
import { DatabaseEngine, ensureCoreSchemaOnPool } from "../Database/database";
|
||||||
|
import { getInternalIngestionHeaders } from "../Security/security";
|
||||||
|
|
||||||
|
async function notifyIngestionAdminSessionSync(payload: {
|
||||||
|
memberId?: string;
|
||||||
|
email?: string;
|
||||||
|
previousEmail?: string;
|
||||||
|
minTokenVersion?: number;
|
||||||
|
disabled?: boolean;
|
||||||
|
deleted?: boolean;
|
||||||
|
}): Promise<void> {
|
||||||
|
const configuredUrl = process.env.INGESTION_INTERNAL_URL?.trim();
|
||||||
|
if (process.env.DATASTACK_DISABLE_LISTEN === "1" && !configuredUrl) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const baseUrl = (configuredUrl || "http://127.0.0.1:4000").replace(
|
||||||
|
/\/+$/,
|
||||||
|
"",
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
await fetch(`${baseUrl}/internal/admin-session-sync`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
...getInternalIngestionHeaders(),
|
||||||
|
},
|
||||||
|
body: JSON.stringify(payload),
|
||||||
|
cache: "no-store",
|
||||||
|
signal: AbortSignal.timeout(1200),
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
// Best-effort bridge sync when Port 4000 is offline or not running
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
const PASSWORD_HASH_BYTES = 64;
|
const PASSWORD_HASH_BYTES = 64;
|
||||||
const PASSWORD_SALT_BYTES = 16;
|
const PASSWORD_SALT_BYTES = 16;
|
||||||
@@ -1040,6 +1074,13 @@ export async function updateAdminAccount({
|
|||||||
tokenVersion: nextVersion,
|
tokenVersion: nextVersion,
|
||||||
createdAt: existing.createdAt,
|
createdAt: existing.createdAt,
|
||||||
};
|
};
|
||||||
|
await notifyIngestionAdminSessionSync({
|
||||||
|
memberId: "owner-1",
|
||||||
|
email: cleanEmail,
|
||||||
|
previousEmail: existing.email,
|
||||||
|
minTokenVersion: nextVersion,
|
||||||
|
disabled: false,
|
||||||
|
});
|
||||||
return {
|
return {
|
||||||
ok: true as const,
|
ok: true as const,
|
||||||
email: cleanEmail,
|
email: cleanEmail,
|
||||||
@@ -1085,6 +1126,13 @@ export async function updateAdminAccount({
|
|||||||
tokenVersion: nextVersion,
|
tokenVersion: nextVersion,
|
||||||
updatedAt: new Date(),
|
updatedAt: new Date(),
|
||||||
});
|
});
|
||||||
|
await notifyIngestionAdminSessionSync({
|
||||||
|
memberId: member.id,
|
||||||
|
email: cleanEmail,
|
||||||
|
previousEmail: member.email,
|
||||||
|
minTokenVersion: nextVersion,
|
||||||
|
disabled: member.status === "disabled",
|
||||||
|
});
|
||||||
return {
|
return {
|
||||||
ok: true as const,
|
ok: true as const,
|
||||||
email: cleanEmail,
|
email: cleanEmail,
|
||||||
@@ -1131,6 +1179,12 @@ export async function revokeAdminSessions(
|
|||||||
member.tokenVersion += 1;
|
member.tokenVersion += 1;
|
||||||
member.updatedAt = new Date();
|
member.updatedAt = new Date();
|
||||||
teamMap.set(member.id, member);
|
teamMap.set(member.id, member);
|
||||||
|
await notifyIngestionAdminSessionSync({
|
||||||
|
memberId: member.id,
|
||||||
|
email: member.email,
|
||||||
|
minTokenVersion: member.tokenVersion,
|
||||||
|
disabled: member.status === "disabled",
|
||||||
|
});
|
||||||
return member.tokenVersion;
|
return member.tokenVersion;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1157,6 +1211,12 @@ export async function revokeAdminSessions(
|
|||||||
member.tokenVersion += 1;
|
member.tokenVersion += 1;
|
||||||
member.updatedAt = new Date();
|
member.updatedAt = new Date();
|
||||||
teamMap.set(member.id, member);
|
teamMap.set(member.id, member);
|
||||||
|
await notifyIngestionAdminSessionSync({
|
||||||
|
memberId: member.id,
|
||||||
|
email: member.email,
|
||||||
|
minTokenVersion: member.tokenVersion,
|
||||||
|
disabled: member.status === "disabled",
|
||||||
|
});
|
||||||
return member.tokenVersion;
|
return member.tokenVersion;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1181,6 +1241,12 @@ export async function revokeAdminSessions(
|
|||||||
globals.__dataStackInMemoryAdmin ?? (await getOrInitInMemoryAdmin());
|
globals.__dataStackInMemoryAdmin ?? (await getOrInitInMemoryAdmin());
|
||||||
existing.tokenVersion += 1;
|
existing.tokenVersion += 1;
|
||||||
globals.__dataStackInMemoryAdmin = existing;
|
globals.__dataStackInMemoryAdmin = existing;
|
||||||
|
await notifyIngestionAdminSessionSync({
|
||||||
|
memberId: "owner-1",
|
||||||
|
email: existing.email,
|
||||||
|
minTokenVersion: existing.tokenVersion,
|
||||||
|
disabled: false,
|
||||||
|
});
|
||||||
return existing.tokenVersion;
|
return existing.tokenVersion;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1588,6 +1654,12 @@ export async function updateAdminTeamMemberAsync(
|
|||||||
updatedAt: new Date(),
|
updatedAt: new Date(),
|
||||||
};
|
};
|
||||||
teamMap.set(id, updated);
|
teamMap.set(id, updated);
|
||||||
|
await notifyIngestionAdminSessionSync({
|
||||||
|
memberId: updated.id,
|
||||||
|
email: updated.email,
|
||||||
|
minTokenVersion: updated.tokenVersion,
|
||||||
|
disabled: updated.status === "disabled",
|
||||||
|
});
|
||||||
|
|
||||||
return {
|
return {
|
||||||
id: updated.id,
|
id: updated.id,
|
||||||
@@ -1654,6 +1726,13 @@ export async function deleteAdminTeamMemberAsync(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
teamMap.delete(id);
|
teamMap.delete(id);
|
||||||
|
await notifyIngestionAdminSessionSync({
|
||||||
|
memberId: existing.id,
|
||||||
|
email: existing.email,
|
||||||
|
minTokenVersion: existing.tokenVersion + 1,
|
||||||
|
disabled: true,
|
||||||
|
deleted: true,
|
||||||
|
});
|
||||||
return { id: existing.id, email: existing.email };
|
return { id: existing.id, email: existing.email };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+65
-8
@@ -20,6 +20,8 @@ export type AuditActionType =
|
|||||||
| "STORAGE_FLUSH_TRIGGERED"
|
| "STORAGE_FLUSH_TRIGGERED"
|
||||||
| "S3_ARCHIVE_TRIGGERED"
|
| "S3_ARCHIVE_TRIGGERED"
|
||||||
| "WEBHOOK_TEST_TRIGGERED"
|
| "WEBHOOK_TEST_TRIGGERED"
|
||||||
|
| "FILES_MANAGER_MUTATION"
|
||||||
|
| "DATA_EXPORT_DOWNLOADED"
|
||||||
| string;
|
| string;
|
||||||
|
|
||||||
export interface AuditLogEntry {
|
export interface AuditLogEntry {
|
||||||
@@ -49,10 +51,19 @@ export interface ListAuditLogsQuery {
|
|||||||
search?: string;
|
search?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface AuditLogListResult {
|
||||||
|
logs: AuditLogEntry[];
|
||||||
|
storageMode: "postgresql" | "memory";
|
||||||
|
degraded: boolean;
|
||||||
|
code?: "AUDIT_STORAGE_DEGRADED";
|
||||||
|
warning?: string;
|
||||||
|
}
|
||||||
|
|
||||||
const MAX_IN_MEMORY_AUDIT_LOGS = 1000;
|
const MAX_IN_MEMORY_AUDIT_LOGS = 1000;
|
||||||
|
|
||||||
type AuditLogGlobals = typeof globalThis & {
|
type AuditLogGlobals = typeof globalThis & {
|
||||||
__dataStackAuditLogs?: AuditLogEntry[];
|
__dataStackAuditLogs?: AuditLogEntry[];
|
||||||
|
__dataStackAuditLastWriteError?: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
const globals = globalThis as AuditLogGlobals;
|
const globals = globalThis as AuditLogGlobals;
|
||||||
@@ -149,8 +160,13 @@ export async function recordAdminAuditLogAsync(
|
|||||||
entry.createdAt,
|
entry.createdAt,
|
||||||
],
|
],
|
||||||
);
|
);
|
||||||
} catch {
|
globals.__dataStackAuditLastWriteError = undefined;
|
||||||
// Never block or crash administrative actions if audit table write experiences transient DB failure
|
} catch (err: unknown) {
|
||||||
|
const msg = err instanceof Error ? err.message : String(err);
|
||||||
|
globals.__dataStackAuditLastWriteError = msg;
|
||||||
|
console.warn(
|
||||||
|
`[DataStack Audit] PostgreSQL audit insert failed for action=${entry.action} (buffered in memory only): ${msg}`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -161,9 +177,9 @@ export function recordAdminAuditLog(input: RecordAuditLogInput): void {
|
|||||||
void recordAdminAuditLogAsync(input);
|
void recordAdminAuditLogAsync(input);
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function listAdminAuditLogsAsync(
|
export async function listAdminAuditLogsWithStatusAsync(
|
||||||
query: ListAuditLogsQuery = {},
|
query: ListAuditLogsQuery = {},
|
||||||
): Promise<AuditLogEntry[]> {
|
): Promise<AuditLogListResult> {
|
||||||
const rawLimit =
|
const rawLimit =
|
||||||
typeof query.limit === "number" && Number.isFinite(query.limit)
|
typeof query.limit === "number" && Number.isFinite(query.limit)
|
||||||
? Math.floor(query.limit)
|
? Math.floor(query.limit)
|
||||||
@@ -174,6 +190,7 @@ export async function listAdminAuditLogsAsync(
|
|||||||
const searchFilter = query.search?.trim().toLowerCase() || "";
|
const searchFilter = query.search?.trim().toLowerCase() || "";
|
||||||
|
|
||||||
const pool = DatabaseEngine.getPool();
|
const pool = DatabaseEngine.getPool();
|
||||||
|
let dbReadError: string | null = null;
|
||||||
if (pool) {
|
if (pool) {
|
||||||
try {
|
try {
|
||||||
await DatabaseEngine.ensureSchema();
|
await DatabaseEngine.ensureSchema();
|
||||||
@@ -227,7 +244,7 @@ export async function listAdminAuditLogsAsync(
|
|||||||
params,
|
params,
|
||||||
);
|
);
|
||||||
|
|
||||||
return result.rows.map((row) => {
|
const logs = result.rows.map((row) => {
|
||||||
let parsedMeta: Record<string, unknown> = {};
|
let parsedMeta: Record<string, unknown> = {};
|
||||||
try {
|
try {
|
||||||
const parsed = JSON.parse(row.metadata_json || "{}");
|
const parsed = JSON.parse(row.metadata_json || "{}");
|
||||||
@@ -251,12 +268,28 @@ export async function listAdminAuditLogsAsync(
|
|||||||
: new Date(row.created_at).toISOString(),
|
: new Date(row.created_at).toISOString(),
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
} catch {
|
|
||||||
// Fallback to in-memory log buffer if PostgreSQL query fails
|
const writeErr = globals.__dataStackAuditLastWriteError;
|
||||||
|
return {
|
||||||
|
logs,
|
||||||
|
storageMode: "postgresql",
|
||||||
|
degraded: Boolean(writeErr),
|
||||||
|
...(writeErr
|
||||||
|
? {
|
||||||
|
code: "AUDIT_STORAGE_DEGRADED" as const,
|
||||||
|
warning: `Recent audit log insert failed in PostgreSQL: ${writeErr}`,
|
||||||
|
}
|
||||||
|
: {}),
|
||||||
|
};
|
||||||
|
} catch (err: unknown) {
|
||||||
|
dbReadError = err instanceof Error ? err.message : String(err);
|
||||||
|
console.warn(
|
||||||
|
`[DataStack Audit] PostgreSQL audit query failed (serving ephemeral in-memory fallback): ${dbReadError}`,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return getInMemoryAuditLogs()
|
const fallbackLogs = getInMemoryAuditLogs()
|
||||||
.filter((entry) => {
|
.filter((entry) => {
|
||||||
if (
|
if (
|
||||||
actionFilter &&
|
actionFilter &&
|
||||||
@@ -278,8 +311,32 @@ export async function listAdminAuditLogsAsync(
|
|||||||
return true;
|
return true;
|
||||||
})
|
})
|
||||||
.slice(0, limit);
|
.slice(0, limit);
|
||||||
|
|
||||||
|
if (dbReadError) {
|
||||||
|
return {
|
||||||
|
logs: fallbackLogs,
|
||||||
|
storageMode: "memory",
|
||||||
|
degraded: true,
|
||||||
|
code: "AUDIT_STORAGE_DEGRADED",
|
||||||
|
warning: `PostgreSQL audit query failed (${dbReadError}); serving ephemeral in-memory logs.`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
logs: fallbackLogs,
|
||||||
|
storageMode: "memory",
|
||||||
|
degraded: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function listAdminAuditLogsAsync(
|
||||||
|
query: ListAuditLogsQuery = {},
|
||||||
|
): Promise<AuditLogEntry[]> {
|
||||||
|
const result = await listAdminAuditLogsWithStatusAsync(query);
|
||||||
|
return result.logs;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function clearAuditLogsForTests(): void {
|
export function clearAuditLogsForTests(): void {
|
||||||
globals.__dataStackAuditLogs = [];
|
globals.__dataStackAuditLogs = [];
|
||||||
|
globals.__dataStackAuditLastWriteError = undefined;
|
||||||
}
|
}
|
||||||
@@ -8,6 +8,8 @@ FROM node:22-alpine AS builder
|
|||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
ENV NODE_ENV=production
|
ENV NODE_ENV=production
|
||||||
ENV NEXT_TELEMETRY_DISABLED=1
|
ENV NEXT_TELEMETRY_DISABLED=1
|
||||||
|
ARG INGESTION_INTERNAL_URL=http://ingestion:4000
|
||||||
|
ENV INGESTION_INTERNAL_URL=${INGESTION_INTERNAL_URL}
|
||||||
COPY --from=dependencies /app/node_modules ./node_modules
|
COPY --from=dependencies /app/node_modules ./node_modules
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN npm run build
|
RUN npm run build
|
||||||
|
|||||||
@@ -17,6 +17,15 @@ server {
|
|||||||
listen 80;
|
listen 80;
|
||||||
server_name _;
|
server_name _;
|
||||||
|
|
||||||
|
# Preserve real external visitor IP when routed behind Coolify Traefik / trusted LAN proxies
|
||||||
|
set_real_ip_from 127.0.0.1;
|
||||||
|
set_real_ip_from 10.0.0.0/8;
|
||||||
|
set_real_ip_from 172.16.0.0/12;
|
||||||
|
set_real_ip_from 192.168.0.0/16;
|
||||||
|
set_real_ip_from fc00::/7;
|
||||||
|
real_ip_header X-Forwarded-For;
|
||||||
|
real_ip_recursive on;
|
||||||
|
|
||||||
resolver 127.0.0.11 valid=10s ipv6=off;
|
resolver 127.0.0.11 valid=10s ipv6=off;
|
||||||
set $nextjs_upstream "http://web:3000";
|
set $nextjs_upstream "http://web:3000";
|
||||||
set $socketio_upstream "http://ingestion:4000";
|
set $socketio_upstream "http://ingestion:4000";
|
||||||
@@ -48,6 +57,21 @@ server {
|
|||||||
proxy_buffering off;
|
proxy_buffering off;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
location /vcon/files-engine {
|
||||||
|
proxy_pass $socketio_upstream;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
|
proxy_set_header Connection $connection_upgrade;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Forwarded-Host $forwarded_host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $forwarded_proto;
|
||||||
|
proxy_read_timeout 86400s;
|
||||||
|
proxy_send_timeout 86400s;
|
||||||
|
proxy_buffering off;
|
||||||
|
}
|
||||||
|
|
||||||
location = /api/export {
|
location = /api/export {
|
||||||
proxy_pass $nextjs_upstream;
|
proxy_pass $nextjs_upstream;
|
||||||
proxy_http_version 1.1;
|
proxy_http_version 1.1;
|
||||||
|
|||||||
@@ -19,6 +19,11 @@ services:
|
|||||||
DATASTACK_ADMIN_SETUP_KEY: "${SERVICE_REALBASE64_64_ADMININSTALL:?Coolify must generate SERVICE_REALBASE64_64_ADMININSTALL}"
|
DATASTACK_ADMIN_SETUP_KEY: "${SERVICE_REALBASE64_64_ADMININSTALL:?Coolify must generate SERVICE_REALBASE64_64_ADMININSTALL}"
|
||||||
DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}"
|
DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}"
|
||||||
REDIS_URL: "redis://redis:6379"
|
REDIS_URL: "redis://redis:6379"
|
||||||
|
DATASTACK_ALLOW_PRIVATE_URLS: "${DATASTACK_ALLOW_PRIVATE_URLS:-false}"
|
||||||
|
DATASTACK_MAX_BATCH_ITEMS: "${DATASTACK_MAX_BATCH_ITEMS:-1000}"
|
||||||
|
DATASTACK_RECENT_LIMIT: "${DATASTACK_RECENT_LIMIT:-500}"
|
||||||
|
DATASTACK_MAINTENANCE_INTERVAL_MS: "${DATASTACK_MAINTENANCE_INTERVAL_MS:-}"
|
||||||
|
DATASTACK_PROXY_MODE: "${DATASTACK_PROXY_MODE:-}"
|
||||||
depends_on:
|
depends_on:
|
||||||
postgres:
|
postgres:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
@@ -53,10 +58,14 @@ services:
|
|||||||
PORT_SOCKET: "4000"
|
PORT_SOCKET: "4000"
|
||||||
DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}"
|
DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}"
|
||||||
DATASTACK_INTERNAL_SECRET: "${DATASTACK_INTERNAL_SECRET:-${SERVICE_REALBASE64_64_INTERNAL:-}}"
|
DATASTACK_INTERNAL_SECRET: "${DATASTACK_INTERNAL_SECRET:-${SERVICE_REALBASE64_64_INTERNAL:-}}"
|
||||||
|
DATASTACK_ADMIN_EMAIL: "${DATASTACK_ADMIN_EMAIL:-admin@datastack.dev}"
|
||||||
DATASTACK_ADMIN_SESSION_SECRET: "${SERVICE_REALBASE64_64_ADMINSESSION:?Coolify must generate SERVICE_REALBASE64_64_ADMINSESSION}"
|
DATASTACK_ADMIN_SESSION_SECRET: "${SERVICE_REALBASE64_64_ADMINSESSION:?Coolify must generate SERVICE_REALBASE64_64_ADMINSESSION}"
|
||||||
DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}"
|
DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}"
|
||||||
REDIS_URL: "redis://redis:6379"
|
REDIS_URL: "redis://redis:6379"
|
||||||
DATASTACK_CORS_ORIGINS: "${DATASTACK_CORS_ORIGINS:-}"
|
DATASTACK_CORS_ORIGINS: "${DATASTACK_CORS_ORIGINS:-}"
|
||||||
|
DATASTACK_ALLOW_PRIVATE_URLS: "${DATASTACK_ALLOW_PRIVATE_URLS:-false}"
|
||||||
|
DATASTACK_MAX_BATCH_ITEMS: "${DATASTACK_MAX_BATCH_ITEMS:-1000}"
|
||||||
|
DATASTACK_RECENT_LIMIT: "${DATASTACK_RECENT_LIMIT:-500}"
|
||||||
depends_on:
|
depends_on:
|
||||||
postgres:
|
postgres:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
|||||||
@@ -0,0 +1,19 @@
|
|||||||
|
export async function register(): Promise<void> {
|
||||||
|
if (process.env.NEXT_RUNTIME === "nodejs") {
|
||||||
|
if (process.env.DATASTACK_DISABLE_LISTEN === "1") {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const { SettingsManager } = await import(
|
||||||
|
"@/core/adminSettings/settingsManager"
|
||||||
|
);
|
||||||
|
await SettingsManager.bootstrapMaintenanceSchedulerOnBoot();
|
||||||
|
} catch (err: unknown) {
|
||||||
|
console.warn(
|
||||||
|
"[DataStack Boot] Maintenance scheduler init deferred/failed:",
|
||||||
|
err instanceof Error ? err.message : err,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -9,7 +9,6 @@ const SAFE_METHODS = new Set(["GET", "HEAD", "OPTIONS"]);
|
|||||||
|
|
||||||
function isProtectedApi(pathname: string, method: string) {
|
function isProtectedApi(pathname: string, method: string) {
|
||||||
if (pathname === "/api/auth/login") return false;
|
if (pathname === "/api/auth/login") return false;
|
||||||
if (pathname === "/api/auth/setup/status" && method === "GET") return false;
|
|
||||||
if (pathname === "/api/auth/setup" && method === "POST") return false;
|
if (pathname === "/api/auth/setup" && method === "POST") return false;
|
||||||
if (pathname === "/api/events" && method === "POST") return false;
|
if (pathname === "/api/events" && method === "POST") return false;
|
||||||
if (pathname === "/api/health" && method === "GET") return false;
|
if (pathname === "/api/health" && method === "GET") return false;
|
||||||
|
|||||||
@@ -18,6 +18,21 @@ const nextConfig: NextConfig = {
|
|||||||
},
|
},
|
||||||
output: 'standalone',
|
output: 'standalone',
|
||||||
transpilePackages: ['motion'],
|
transpilePackages: ['motion'],
|
||||||
|
async rewrites() {
|
||||||
|
const ingestionBase = (
|
||||||
|
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
|
||||||
|
).replace(/\/+$/, '');
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
source: '/vcon/files-engine',
|
||||||
|
destination: `${ingestionBase}/vcon/files-engine`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
source: '/vcon/files-engine/:path*',
|
||||||
|
destination: `${ingestionBase}/vcon/files-engine/:path*`,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
},
|
||||||
webpack: (config, {dev}) => {
|
webpack: (config, {dev}) => {
|
||||||
// HMR is disabled in AI Studio via DISABLE_HMR env var.
|
// HMR is disabled in AI Studio via DISABLE_HMR env var.
|
||||||
// Do not modify—file watching is disabled to prevent flickering during agent edits.
|
// Do not modify—file watching is disabled to prevent flickering during agent edits.
|
||||||
|
|||||||
@@ -25,6 +25,15 @@ server {
|
|||||||
listen 80;
|
listen 80;
|
||||||
server_name _;
|
server_name _;
|
||||||
|
|
||||||
|
# Preserve real external visitor IP when routed behind trusted reverse proxies / Docker bridges
|
||||||
|
set_real_ip_from 127.0.0.1;
|
||||||
|
set_real_ip_from 10.0.0.0/8;
|
||||||
|
set_real_ip_from 172.16.0.0/12;
|
||||||
|
set_real_ip_from 192.168.0.0/16;
|
||||||
|
set_real_ip_from fc00::/7;
|
||||||
|
real_ip_header X-Forwarded-For;
|
||||||
|
real_ip_recursive on;
|
||||||
|
|
||||||
client_max_body_size 50M;
|
client_max_body_size 50M;
|
||||||
|
|
||||||
# Express + Socket.IO health endpoint on port 4000
|
# Express + Socket.IO health endpoint on port 4000
|
||||||
@@ -54,6 +63,22 @@ server {
|
|||||||
proxy_buffering off;
|
proxy_buffering off;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Admin Files Manager (CloudCmd + isolated Socket.IO) reverse proxy to port 4000
|
||||||
|
location /vcon/files-engine {
|
||||||
|
proxy_pass http://socketio_upstream;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
|
proxy_set_header Connection $connection_upgrade;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Forwarded-Host $forwarded_host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $forwarded_proto;
|
||||||
|
proxy_read_timeout 86400s;
|
||||||
|
proxy_send_timeout 86400s;
|
||||||
|
proxy_buffering off;
|
||||||
|
}
|
||||||
|
|
||||||
# Streaming CSV/JSON export endpoint on port 3000 (disable response buffering)
|
# Streaming CSV/JSON export endpoint on port 3000 (disable response buffering)
|
||||||
location = /api/export {
|
location = /api/export {
|
||||||
proxy_pass http://nextjs_upstream;
|
proxy_pass http://nextjs_upstream;
|
||||||
|
|||||||
Generated
+10356
-223
File diff suppressed because it is too large.
Load diff
+3
-1
@@ -6,7 +6,8 @@
|
|||||||
"node": ">=20.0.0"
|
"node": ">=20.0.0"
|
||||||
},
|
},
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"dev": "next dev -p 3000 -H 0.0.0.0",
|
"dev": "node scripts/dev-all.js",
|
||||||
|
"dev:web": "next dev -p 3000 -H 0.0.0.0",
|
||||||
"dev:all": "node scripts/dev-all.js",
|
"dev:all": "node scripts/dev-all.js",
|
||||||
"server": "node server/index.js",
|
"server": "node server/index.js",
|
||||||
"build": "NODE_ENV=production next build",
|
"build": "NODE_ENV=production next build",
|
||||||
@@ -20,6 +21,7 @@
|
|||||||
"@hookform/resolvers": "^5.2.1",
|
"@hookform/resolvers": "^5.2.1",
|
||||||
"autoprefixer": "^10.4.21",
|
"autoprefixer": "^10.4.21",
|
||||||
"class-variance-authority": "^0.7.1",
|
"class-variance-authority": "^0.7.1",
|
||||||
|
"cloudcmd": "^19.23.2",
|
||||||
"clsx": "^2.1.1",
|
"clsx": "^2.1.1",
|
||||||
"cors": "^2.8.6",
|
"cors": "^2.8.6",
|
||||||
"csv-parse": "^7.0.3",
|
"csv-parse": "^7.0.3",
|
||||||
|
|||||||
@@ -153,6 +153,8 @@ self.addEventListener('fetch', (event) => {
|
|||||||
if (
|
if (
|
||||||
url.pathname.startsWith('/socket.io/') ||
|
url.pathname.startsWith('/socket.io/') ||
|
||||||
url.pathname.startsWith('/api/') ||
|
url.pathname.startsWith('/api/') ||
|
||||||
|
url.pathname === '/vcon/files-engine' ||
|
||||||
|
url.pathname.startsWith('/vcon/files-engine/') ||
|
||||||
url.pathname.startsWith('/_next/webpack-hmr')
|
url.pathname.startsWith('/_next/webpack-hmr')
|
||||||
) {
|
) {
|
||||||
return;
|
return;
|
||||||
|
|||||||
+1
-1
@@ -19,7 +19,7 @@ console.log('\x1b[36m%s\x1b[0m', ` - Port ${webPort}: Next.js 15 Web & Admin Co
|
|||||||
console.log('\x1b[36m%s\x1b[0m', ' - Port 4000: Ingestion & Socket.IO Streaming Engine');
|
console.log('\x1b[36m%s\x1b[0m', ' - Port 4000: Ingestion & Socket.IO Streaming Engine');
|
||||||
console.log('\x1b[36m%s\x1b[0m', '=======================================================\n');
|
console.log('\x1b[36m%s\x1b[0m', '=======================================================\n');
|
||||||
|
|
||||||
const nextProc = spawn(process.execPath, [nextBin, 'dev', '-p', webPort], {
|
const nextProc = spawn(process.execPath, [nextBin, 'dev', '-p', webPort, '-H', '0.0.0.0'], {
|
||||||
stdio: 'inherit',
|
stdio: 'inherit',
|
||||||
shell: false,
|
shell: false,
|
||||||
env: { ...process.env },
|
env: { ...process.env },
|
||||||
|
|||||||
+61
-17
@@ -645,8 +645,7 @@ export class DataStackClient {
|
|||||||
const msg = String(ack.error || 'Socket event rejected');
|
const msg = String(ack.error || 'Socket event rejected');
|
||||||
const authErr = classifyAuthRejection(code, msg);
|
const authErr = classifyAuthRejection(code, msg);
|
||||||
if (authErr) {
|
if (authErr) {
|
||||||
this.heartbeatStopped = true;
|
this.disconnect();
|
||||||
this.stopHeartbeatMonitor();
|
|
||||||
reject(
|
reject(
|
||||||
new DataStackAuthError(msg, authErr.code, authErr.statusCode, ack)
|
new DataStackAuthError(msg, authErr.code, authErr.statusCode, ack)
|
||||||
);
|
);
|
||||||
@@ -902,8 +901,7 @@ export class DataStackClient {
|
|||||||
const msg = String(ack.error || 'Batch rejected');
|
const msg = String(ack.error || 'Batch rejected');
|
||||||
const authErr = classifyAuthRejection(code, msg);
|
const authErr = classifyAuthRejection(code, msg);
|
||||||
if (authErr) {
|
if (authErr) {
|
||||||
this.heartbeatStopped = true;
|
this.disconnect();
|
||||||
this.stopHeartbeatMonitor();
|
|
||||||
reject(
|
reject(
|
||||||
new DataStackAuthError(
|
new DataStackAuthError(
|
||||||
msg,
|
msg,
|
||||||
@@ -1055,7 +1053,7 @@ export class DataStackClient {
|
|||||||
private async deliverChunkWithSplitAndRetry(
|
private async deliverChunkWithSplitAndRetry(
|
||||||
chunk: DataStackEnvelope[],
|
chunk: DataStackEnvelope[],
|
||||||
allowSpoolFallback = true
|
allowSpoolFallback = true
|
||||||
): Promise<DataStackSendResult> {
|
): Promise<DataStackSendResult & { unprocessedEnvelopes?: DataStackEnvelope[] }> {
|
||||||
let pending = [...chunk];
|
let pending = [...chunk];
|
||||||
let totalAccepted = 0;
|
let totalAccepted = 0;
|
||||||
let totalDuplicates = 0;
|
let totalDuplicates = 0;
|
||||||
@@ -1084,6 +1082,18 @@ export class DataStackClient {
|
|||||||
pending = pending.filter((item) => rateLimitedSet.has(item.eventId));
|
pending = pending.filter((item) => rateLimitedSet.has(item.eventId));
|
||||||
|
|
||||||
if (attempt >= this.config.maxRetries) {
|
if (attempt >= this.config.maxRetries) {
|
||||||
|
if (!allowSpoolFallback) {
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
transport: lastTransport,
|
||||||
|
accepted: totalAccepted,
|
||||||
|
duplicates: totalDuplicates,
|
||||||
|
spooled: totalSpooled,
|
||||||
|
rateLimitedEventIds: pending.map((p) => p.eventId),
|
||||||
|
unprocessedItems: pending.map((p) => p.data),
|
||||||
|
unprocessedEnvelopes: pending,
|
||||||
|
};
|
||||||
|
}
|
||||||
throw new DataStackRateLimitError(
|
throw new DataStackRateLimitError(
|
||||||
`Rate limit exceeded for ${pending.length} event(s) after ${this.config.maxRetries} retries.`,
|
`Rate limit exceeded for ${pending.length} event(s) after ${this.config.maxRetries} retries.`,
|
||||||
pending.map((p) => p.eventId)
|
pending.map((p) => p.eventId)
|
||||||
@@ -1105,34 +1115,57 @@ export class DataStackClient {
|
|||||||
pending.slice(mid),
|
pending.slice(mid),
|
||||||
allowSpoolFallback
|
allowSpoolFallback
|
||||||
);
|
);
|
||||||
|
const combinedEnvelopes = [
|
||||||
|
...(left.unprocessedEnvelopes || []),
|
||||||
|
...(right.unprocessedEnvelopes || []),
|
||||||
|
];
|
||||||
|
const combinedItems = [
|
||||||
|
...(left.unprocessedItems || []),
|
||||||
|
...(right.unprocessedItems || []),
|
||||||
|
];
|
||||||
return {
|
return {
|
||||||
success: left.success && right.success,
|
success: left.success && right.success,
|
||||||
transport: right.transport,
|
transport: right.transport,
|
||||||
accepted: (left.accepted || 0) + (right.accepted || 0),
|
accepted: (left.accepted || 0) + (right.accepted || 0),
|
||||||
duplicates: (left.duplicates || 0) + (right.duplicates || 0),
|
duplicates: (left.duplicates || 0) + (right.duplicates || 0),
|
||||||
spooled: (left.spooled || 0) + (right.spooled || 0),
|
spooled: (left.spooled || 0) + (right.spooled || 0),
|
||||||
|
...(combinedItems.length > 0 ? { unprocessedItems: combinedItems } : {}),
|
||||||
|
...(combinedEnvelopes.length > 0
|
||||||
|
? { unprocessedEnvelopes: combinedEnvelopes }
|
||||||
|
: {}),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (
|
if (err instanceof DataStackRateLimitError) {
|
||||||
err instanceof DataStackRateLimitError &&
|
|
||||||
attempt < this.config.maxRetries
|
|
||||||
) {
|
|
||||||
const rlSet = new Set(err.rateLimitedEventIds);
|
const rlSet = new Set(err.rateLimitedEventIds);
|
||||||
if (rlSet.size > 0 && rlSet.size < pending.length) {
|
if (rlSet.size > 0 && rlSet.size < pending.length) {
|
||||||
totalAccepted += pending.length - rlSet.size;
|
totalAccepted += pending.length - rlSet.size;
|
||||||
pending = pending.filter((p) => rlSet.has(p.eventId));
|
pending = pending.filter((p) => rlSet.has(p.eventId));
|
||||||
}
|
}
|
||||||
attempt += 1;
|
if (attempt < this.config.maxRetries) {
|
||||||
await new Promise((r) => setTimeout(r, 50 * attempt));
|
attempt += 1;
|
||||||
continue;
|
await new Promise((r) => setTimeout(r, 50 * attempt));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!allowSpoolFallback) {
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
transport: lastTransport,
|
||||||
|
accepted: totalAccepted,
|
||||||
|
duplicates: totalDuplicates,
|
||||||
|
spooled: totalSpooled,
|
||||||
|
rateLimitedEventIds: pending.map((p) => p.eventId),
|
||||||
|
unprocessedItems: pending.map((p) => p.data),
|
||||||
|
unprocessedEnvelopes: pending,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
err instanceof DataStackAuthError ||
|
err instanceof DataStackAuthError ||
|
||||||
err instanceof DataStackQuotaExceededError ||
|
err instanceof DataStackQuotaExceededError ||
|
||||||
err instanceof DataStackPayloadTooLargeError ||
|
err instanceof DataStackPayloadTooLargeError ||
|
||||||
err instanceof DataStackRateLimitError ||
|
|
||||||
err instanceof DataStackConfigError
|
err instanceof DataStackConfigError
|
||||||
) {
|
) {
|
||||||
throw err;
|
throw err;
|
||||||
@@ -1160,6 +1193,7 @@ export class DataStackClient {
|
|||||||
duplicates: totalDuplicates,
|
duplicates: totalDuplicates,
|
||||||
spooled: totalSpooled,
|
spooled: totalSpooled,
|
||||||
unprocessedItems: pending.map((p) => p.data),
|
unprocessedItems: pending.map((p) => p.data),
|
||||||
|
...(pending.length > 0 ? { unprocessedEnvelopes: pending } : {}),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1365,12 +1399,22 @@ export class DataStackClient {
|
|||||||
const batch = this.spooler.peekBatch(this.config.batchChunkSize);
|
const batch = this.spooler.peekBatch(this.config.batchChunkSize);
|
||||||
if (batch.length === 0) break;
|
if (batch.length === 0) break;
|
||||||
const res = await this.deliverChunkWithSplitAndRetry(batch, false);
|
const res = await this.deliverChunkWithSplitAndRetry(batch, false);
|
||||||
if (res.success) {
|
const processedNow = res.accepted || 0;
|
||||||
this.spooler.removeFirst(batch.length);
|
const duplicatesNow = res.duplicates || 0;
|
||||||
drained += batch.length;
|
const remaining = res.unprocessedEnvelopes ?? [];
|
||||||
} else {
|
|
||||||
|
if (!res.success && processedNow === 0 && duplicatesNow === 0) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (remaining.length > 0) {
|
||||||
|
const acked = this.spooler.ackAndRequeueFront(batch.length, remaining);
|
||||||
|
drained += acked;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.spooler.ackAndRequeueFront(batch.length, []);
|
||||||
|
drained += batch.length;
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
this.isDrainingSpool = false;
|
this.isDrainingSpool = false;
|
||||||
|
|||||||
@@ -11,7 +11,8 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"build": "tsc -p tsconfig.json"
|
"build": "tsc -p tsconfig.json",
|
||||||
|
"prepare": "npm run build"
|
||||||
},
|
},
|
||||||
"files": [
|
"files": [
|
||||||
"dist",
|
"dist",
|
||||||
@@ -28,6 +29,9 @@
|
|||||||
"csv-parse": "^7.0.3",
|
"csv-parse": "^7.0.3",
|
||||||
"socket.io-client": "^4.8.4"
|
"socket.io-client": "^4.8.4"
|
||||||
},
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"typescript": "^5.9.3"
|
||||||
|
},
|
||||||
"keywords": [
|
"keywords": [
|
||||||
"datastack",
|
"datastack",
|
||||||
"telemetry",
|
"telemetry",
|
||||||
|
|||||||
@@ -138,6 +138,64 @@ export class DiskSpooler {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Atomically removes `ackedCount` head lines from `spool.jsonl` while
|
||||||
|
* preserving any `unprocessed` envelopes at the FRONT of the queue in a
|
||||||
|
* single atomic temp-file + `fsyncSync` + `renameSync` operation (FIFO safe).
|
||||||
|
*/
|
||||||
|
public ackAndRequeueFront(
|
||||||
|
ackedCount: number,
|
||||||
|
unprocessed: DataStackEnvelope[] = []
|
||||||
|
): number {
|
||||||
|
if (!this.enabled || ackedCount <= 0 || !fs.existsSync(this.spoolFile)) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
const raw = fs.readFileSync(this.spoolFile, 'utf8');
|
||||||
|
const allLines = raw
|
||||||
|
.split('\n')
|
||||||
|
.map((l) => l.trim())
|
||||||
|
.filter(Boolean);
|
||||||
|
|
||||||
|
const consumed = Math.min(Math.floor(ackedCount), allLines.length);
|
||||||
|
const tailLines = allLines.slice(consumed);
|
||||||
|
const headLines = unprocessed.map((evt) => JSON.stringify(evt));
|
||||||
|
const combined = [...headLines, ...tailLines];
|
||||||
|
|
||||||
|
if (combined.length === 0) {
|
||||||
|
try {
|
||||||
|
fs.unlinkSync(this.spoolFile);
|
||||||
|
} catch {
|
||||||
|
// Ignore if already removed
|
||||||
|
}
|
||||||
|
return consumed;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.ensureDir();
|
||||||
|
const tmpFile = `${this.spoolFile}.${process.pid}.${Date.now()}.tmp`;
|
||||||
|
const payload = combined.join('\n') + '\n';
|
||||||
|
try {
|
||||||
|
const fd = fs.openSync(tmpFile, 'w');
|
||||||
|
try {
|
||||||
|
fs.writeFileSync(fd, payload, 'utf8');
|
||||||
|
fs.fsyncSync(fd);
|
||||||
|
} finally {
|
||||||
|
fs.closeSync(fd);
|
||||||
|
}
|
||||||
|
fs.renameSync(tmpFile, this.spoolFile);
|
||||||
|
} catch (err) {
|
||||||
|
try {
|
||||||
|
if (fs.existsSync(tmpFile)) {
|
||||||
|
fs.unlinkSync(tmpFile);
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Ignore cleanup failure
|
||||||
|
}
|
||||||
|
throw err;
|
||||||
|
}
|
||||||
|
|
||||||
|
return Math.max(0, consumed - headLines.length);
|
||||||
|
}
|
||||||
|
|
||||||
public count(): number {
|
public count(): number {
|
||||||
if (!this.enabled || !fs.existsSync(this.spoolFile)) {
|
if (!this.enabled || !fs.existsSync(this.spoolFile)) {
|
||||||
return 0;
|
return 0;
|
||||||
|
|||||||
File diff suppressed because it is too large.
Load diff
+345
-8
@@ -1,4 +1,5 @@
|
|||||||
const crypto = require('node:crypto');
|
const crypto = require('node:crypto');
|
||||||
|
const dnsPromises = require('node:dns/promises');
|
||||||
const net = require('node:net');
|
const net = require('node:net');
|
||||||
const express = require('express');
|
const express = require('express');
|
||||||
const http = require('http');
|
const http = require('http');
|
||||||
@@ -8,6 +9,11 @@ const { Pool } = require('pg');
|
|||||||
const Redis = require('ioredis');
|
const Redis = require('ioredis');
|
||||||
const { setupSocketHandlers, normalizeClientId } = require('./socket/handlers');
|
const { setupSocketHandlers, normalizeClientId } = require('./socket/handlers');
|
||||||
const { CsvWriterWorker } = require('./workers/csvWriter');
|
const { CsvWriterWorker } = require('./workers/csvWriter');
|
||||||
|
const {
|
||||||
|
mountFilesManager,
|
||||||
|
syncInMemoryAdminSessionState,
|
||||||
|
clearInMemoryAdminSessionStateForTests,
|
||||||
|
} = require('./filesManager');
|
||||||
|
|
||||||
function parseAllowedCorsOrigins(raw = process.env.DATASTACK_CORS_ORIGINS) {
|
function parseAllowedCorsOrigins(raw = process.env.DATASTACK_CORS_ORIGINS) {
|
||||||
if (typeof raw !== 'string') return null;
|
if (typeof raw !== 'string') return null;
|
||||||
@@ -36,7 +42,14 @@ function corsOriginDelegate(origin, callback) {
|
|||||||
const PORT = Number(process.env.PORT_SOCKET) || 4000;
|
const PORT = Number(process.env.PORT_SOCKET) || 4000;
|
||||||
const app = express();
|
const app = express();
|
||||||
app.use(cors({ origin: corsOriginDelegate }));
|
app.use(cors({ origin: corsOriginDelegate }));
|
||||||
app.use(express.json({ limit: '10mb' }));
|
const jsonBodyParser = express.json({ limit: '10mb' });
|
||||||
|
app.use((req, res, next) => {
|
||||||
|
if (req.path === '/vcon/files-engine' || req.path.startsWith('/vcon/files-engine/')) {
|
||||||
|
return next();
|
||||||
|
}
|
||||||
|
return jsonBodyParser(req, res, next);
|
||||||
|
});
|
||||||
|
let filesManagerHandle = null;
|
||||||
|
|
||||||
const REDIS_BUFFER_KEY = 'datastack:buffer:events';
|
const REDIS_BUFFER_KEY = 'datastack:buffer:events';
|
||||||
const REDIS_RECENT_KEY = 'datastack:recent:events';
|
const REDIS_RECENT_KEY = 'datastack:recent:events';
|
||||||
@@ -1313,6 +1326,204 @@ function formatOutboundWebhookBody(_url, payload, timestamp) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const WEBHOOK_ALWAYS_BLOCKED_HOSTNAMES = new Set([
|
||||||
|
'metadata.google.internal',
|
||||||
|
'metadata.goog',
|
||||||
|
'metadata',
|
||||||
|
'instance-data',
|
||||||
|
'instance-data.ec2.internal',
|
||||||
|
]);
|
||||||
|
|
||||||
|
const WEBHOOK_PRIVATE_HOSTNAMES = new Set([
|
||||||
|
'localhost',
|
||||||
|
'localhost.localdomain',
|
||||||
|
'ip6-localhost',
|
||||||
|
'ip6-loopback',
|
||||||
|
]);
|
||||||
|
|
||||||
|
const WEBHOOK_ALWAYS_BLOCKED_IPS = (() => {
|
||||||
|
const list = new net.BlockList();
|
||||||
|
list.addSubnet('0.0.0.0', 8, 'ipv4');
|
||||||
|
list.addSubnet('169.254.0.0', 16, 'ipv4');
|
||||||
|
list.addSubnet('224.0.0.0', 4, 'ipv4');
|
||||||
|
list.addSubnet('240.0.0.0', 4, 'ipv4');
|
||||||
|
list.addAddress('::', 'ipv6');
|
||||||
|
list.addSubnet('fe80::', 10, 'ipv6');
|
||||||
|
list.addSubnet('ff00::', 8, 'ipv6');
|
||||||
|
return list;
|
||||||
|
})();
|
||||||
|
|
||||||
|
const WEBHOOK_PRIVATE_IPS = (() => {
|
||||||
|
const list = new net.BlockList();
|
||||||
|
list.addSubnet('127.0.0.0', 8, 'ipv4');
|
||||||
|
list.addSubnet('10.0.0.0', 8, 'ipv4');
|
||||||
|
list.addSubnet('172.16.0.0', 12, 'ipv4');
|
||||||
|
list.addSubnet('192.168.0.0', 16, 'ipv4');
|
||||||
|
list.addSubnet('100.64.0.0', 10, 'ipv4');
|
||||||
|
list.addSubnet('198.18.0.0', 15, 'ipv4');
|
||||||
|
list.addAddress('::1', 'ipv6');
|
||||||
|
list.addSubnet('fc00::', 7, 'ipv6');
|
||||||
|
return list;
|
||||||
|
})();
|
||||||
|
|
||||||
|
function isPrivateWebhookUrlAllowed(env = process.env) {
|
||||||
|
const raw = String(env.DATASTACK_ALLOW_PRIVATE_URLS || '')
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
|
return raw === 'true' || raw === '1' || raw === 'yes';
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeWebhookIp(rawHost) {
|
||||||
|
let clean = String(rawHost || '').trim().toLowerCase();
|
||||||
|
if (clean.startsWith('[') && clean.endsWith(']')) {
|
||||||
|
clean = clean.slice(1, -1);
|
||||||
|
}
|
||||||
|
const zoneIdx = clean.indexOf('%');
|
||||||
|
if (zoneIdx >= 0) clean = clean.slice(0, zoneIdx);
|
||||||
|
clean = clean.replace(/\.+$/, '');
|
||||||
|
if (!clean) return null;
|
||||||
|
|
||||||
|
if (clean.startsWith('::ffff:')) {
|
||||||
|
const mapped = clean.slice('::ffff:'.length);
|
||||||
|
if (net.isIP(mapped) === 4) {
|
||||||
|
return { ip: mapped, family: 4 };
|
||||||
|
}
|
||||||
|
const hexParts = mapped.split(':');
|
||||||
|
if (hexParts.length === 2) {
|
||||||
|
const hi = Number.parseInt(hexParts[0], 16);
|
||||||
|
const lo = Number.parseInt(hexParts[1], 16);
|
||||||
|
if (
|
||||||
|
Number.isInteger(hi) &&
|
||||||
|
Number.isInteger(lo) &&
|
||||||
|
hi >= 0 &&
|
||||||
|
hi <= 0xffff &&
|
||||||
|
lo >= 0 &&
|
||||||
|
lo <= 0xffff
|
||||||
|
) {
|
||||||
|
const ipv4 = `${(hi >> 8) & 0xff}.${hi & 0xff}.${(lo >> 8) & 0xff}.${lo & 0xff}`;
|
||||||
|
if (net.isIP(ipv4) === 4) {
|
||||||
|
return { ip: ipv4, family: 4 };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const family = net.isIP(clean);
|
||||||
|
if (family === 4 || family === 6) {
|
||||||
|
return { ip: clean, family };
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function checkWebhookIpAllowed(rawIp, allowPrivate) {
|
||||||
|
const parsed = normalizeWebhookIp(rawIp);
|
||||||
|
if (!parsed) return { allowed: false, reason: 'Invalid IP address' };
|
||||||
|
const ipType = parsed.family === 4 ? 'ipv4' : 'ipv6';
|
||||||
|
if (WEBHOOK_ALWAYS_BLOCKED_IPS.check(parsed.ip, ipType)) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
reason: `Address '${parsed.ip}' is in a restricted link-local/metadata/reserved range`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (!allowPrivate && WEBHOOK_PRIVATE_IPS.check(parsed.ip, ipType)) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
reason: `Address '${parsed.ip}' is a private or loopback network address`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return { allowed: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function validateOutboundWebhookUrlAsync(rawUrl, options = {}) {
|
||||||
|
const allowPrivate =
|
||||||
|
options.allowPrivate !== undefined
|
||||||
|
? Boolean(options.allowPrivate)
|
||||||
|
: isPrivateWebhookUrlAllowed(options.env || process.env);
|
||||||
|
|
||||||
|
let parsed;
|
||||||
|
try {
|
||||||
|
parsed = new URL(String(rawUrl || '').trim());
|
||||||
|
} catch {
|
||||||
|
return { allowed: false, code: 'INVALID_URL', error: 'Webhook URL is not a valid URL.' };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
code: 'SSRF_BLOCKED_HOST',
|
||||||
|
error: 'Webhook URL must use http:// or https://.',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (parsed.username || parsed.password) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
code: 'SSRF_BLOCKED_HOST',
|
||||||
|
error: 'Webhook URL must not embed credentials.',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
let host = String(parsed.hostname || '').trim().toLowerCase();
|
||||||
|
if (host.startsWith('[') && host.endsWith(']')) host = host.slice(1, -1);
|
||||||
|
host = host.replace(/\.+$/, '');
|
||||||
|
if (!host) {
|
||||||
|
return { allowed: false, code: 'INVALID_URL', error: 'Webhook URL hostname is empty.' };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (WEBHOOK_ALWAYS_BLOCKED_HOSTNAMES.has(host)) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
code: 'SSRF_BLOCKED_HOST',
|
||||||
|
error: `Webhook target '${host}' is blocked (cloud metadata/internal endpoint).`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const literalIp = normalizeWebhookIp(host);
|
||||||
|
if (literalIp) {
|
||||||
|
const ipCheck = checkWebhookIpAllowed(host, allowPrivate);
|
||||||
|
if (!ipCheck.allowed) {
|
||||||
|
return { allowed: false, code: 'SSRF_BLOCKED_HOST', error: ipCheck.reason };
|
||||||
|
}
|
||||||
|
return { allowed: true, url: parsed.toString() };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!allowPrivate) {
|
||||||
|
if (
|
||||||
|
WEBHOOK_PRIVATE_HOSTNAMES.has(host) ||
|
||||||
|
host.endsWith('.localhost') ||
|
||||||
|
host.endsWith('.local') ||
|
||||||
|
host.endsWith('.internal') ||
|
||||||
|
!host.includes('.')
|
||||||
|
) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
code: 'SSRF_BLOCKED_HOST',
|
||||||
|
error: `Webhook target '${host}' is a private/internal hostname.`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const lookupFn =
|
||||||
|
options.dnsLookup ||
|
||||||
|
((h) => dnsPromises.lookup(h, { all: true, verbatim: true }));
|
||||||
|
const records = await lookupFn(host);
|
||||||
|
for (const rec of records) {
|
||||||
|
const check = checkWebhookIpAllowed(rec.address, allowPrivate);
|
||||||
|
if (!check.allowed) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
code: 'SSRF_BLOCKED_HOST',
|
||||||
|
error: `Webhook hostname '${host}' resolved to blocked address: ${check.reason}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Let fetch handle unresolved DNS names cleanly
|
||||||
|
}
|
||||||
|
|
||||||
|
return { allowed: true, url: parsed.toString() };
|
||||||
|
}
|
||||||
|
|
||||||
async function triggerWebhookAlert(payload, options = {}) {
|
async function triggerWebhookAlert(payload, options = {}) {
|
||||||
try {
|
try {
|
||||||
const enabled =
|
const enabled =
|
||||||
@@ -1329,6 +1540,19 @@ async function triggerWebhookAlert(payload, options = {}) {
|
|||||||
return { dispatched: false, reason: 'disabled_or_empty_url' };
|
return { dispatched: false, reason: 'disabled_or_empty_url' };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const ssrfCheck = await validateOutboundWebhookUrlAsync(url, {
|
||||||
|
allowPrivate: options.allowPrivate,
|
||||||
|
dnsLookup: options.dnsLookup,
|
||||||
|
});
|
||||||
|
if (!ssrfCheck.allowed) {
|
||||||
|
return {
|
||||||
|
dispatched: false,
|
||||||
|
code: ssrfCheck.code || 'SSRF_BLOCKED_HOST',
|
||||||
|
reason: ssrfCheck.code || 'SSRF_BLOCKED_HOST',
|
||||||
|
error: ssrfCheck.error,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
const cooldownSec = Math.max(
|
const cooldownSec = Math.max(
|
||||||
0,
|
0,
|
||||||
Number(
|
Number(
|
||||||
@@ -1374,12 +1598,22 @@ async function triggerWebhookAlert(payload, options = {}) {
|
|||||||
const fetchImpl = options.fetchImpl || global.fetch;
|
const fetchImpl = options.fetchImpl || global.fetch;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const res = await fetchImpl(url, {
|
const res = await fetchImpl(ssrfCheck.url, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers,
|
headers,
|
||||||
body: rawBody,
|
body: rawBody,
|
||||||
|
redirect: 'manual',
|
||||||
signal: controller.signal,
|
signal: controller.signal,
|
||||||
});
|
});
|
||||||
|
if (res.status >= 300 && res.status < 400) {
|
||||||
|
return {
|
||||||
|
dispatched: false,
|
||||||
|
code: 'SSRF_BLOCKED_HOST',
|
||||||
|
reason: 'SSRF_BLOCKED_HOST',
|
||||||
|
status: res.status,
|
||||||
|
error: `Webhook endpoint attempted an unsupported HTTP ${res.status} redirect.`,
|
||||||
|
};
|
||||||
|
}
|
||||||
return { dispatched: true, status: res.status };
|
return { dispatched: true, status: res.status };
|
||||||
} finally {
|
} finally {
|
||||||
clearTimeout(timeout);
|
clearTimeout(timeout);
|
||||||
@@ -1475,6 +1709,32 @@ async function ensureDatabaseSchema() {
|
|||||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||||
)
|
)
|
||||||
`);
|
`);
|
||||||
|
await pool.query(`
|
||||||
|
CREATE TABLE IF NOT EXISTS admin_team_members (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
email TEXT NOT NULL UNIQUE,
|
||||||
|
role TEXT NOT NULL DEFAULT 'viewer',
|
||||||
|
status TEXT NOT NULL DEFAULT 'active',
|
||||||
|
password_salt BYTEA NOT NULL,
|
||||||
|
password_hash BYTEA NOT NULL,
|
||||||
|
token_version INTEGER NOT NULL DEFAULT 1,
|
||||||
|
invited_by TEXT NOT NULL DEFAULT 'owner',
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||||
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||||
|
)
|
||||||
|
`);
|
||||||
|
await pool.query(`
|
||||||
|
CREATE TABLE IF NOT EXISTS admin_audit_logs (
|
||||||
|
id TEXT PRIMARY KEY,
|
||||||
|
actor_email TEXT NOT NULL,
|
||||||
|
actor_role TEXT NOT NULL,
|
||||||
|
action TEXT NOT NULL,
|
||||||
|
target TEXT NOT NULL,
|
||||||
|
ip_address TEXT NOT NULL DEFAULT '',
|
||||||
|
metadata_json TEXT NOT NULL DEFAULT '{}',
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||||
|
)
|
||||||
|
`);
|
||||||
await pool.query(`
|
await pool.query(`
|
||||||
ALTER TABLE client_sdks
|
ALTER TABLE client_sdks
|
||||||
ADD COLUMN IF NOT EXISTS rate_limit INTEGER NOT NULL DEFAULT 100,
|
ADD COLUMN IF NOT EXISTS rate_limit INTEGER NOT NULL DEFAULT 100,
|
||||||
@@ -1822,7 +2082,7 @@ app.get('/internal/recent', async (req, res) => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Internal endpoint triggered by Next.js SDKManager when a client is deleted, revoked, rotated, or synced
|
// Internal endpoint triggered by Next.js SDKManager when a client is deleted, revoked, rotated, or synced
|
||||||
app.post('/internal/client-sync', (req, res) => {
|
app.post('/internal/client-sync', async (req, res) => {
|
||||||
const action = req.body && req.body.action ? String(req.body.action) : '';
|
const action = req.body && req.body.action ? String(req.body.action) : '';
|
||||||
|
|
||||||
if (action === 'bulk_sync' && Array.isArray(req.body.clients)) {
|
if (action === 'bulk_sync' && Array.isArray(req.body.clients)) {
|
||||||
@@ -1942,25 +2202,83 @@ app.post('/internal/client-sync', (req, res) => {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
} else if (action === 'updated') {
|
} else if (action === 'updated') {
|
||||||
|
const prevClient = inMemoryClientRegistry.get(normalizedId);
|
||||||
|
const nextStatus =
|
||||||
|
clientPayload && clientPayload.status === 'revoked' ? 'revoked' : 'active';
|
||||||
|
const nextToken = clientPayload ? String(clientPayload.token || '') : '';
|
||||||
|
const nextAllowedIps =
|
||||||
|
clientPayload && typeof clientPayload.allowedIps === 'string'
|
||||||
|
? clientPayload.allowedIps.trim()
|
||||||
|
: '';
|
||||||
|
const nextExpiresAt =
|
||||||
|
clientPayload && clientPayload.expiresAt
|
||||||
|
? String(clientPayload.expiresAt)
|
||||||
|
: null;
|
||||||
|
|
||||||
if (clientPayload) {
|
if (clientPayload) {
|
||||||
inMemoryClientRegistry.set(normalizedId, {
|
inMemoryClientRegistry.set(normalizedId, {
|
||||||
clientId: normalizedId,
|
clientId: normalizedId,
|
||||||
name: String(clientPayload.name || normalizedId),
|
name: String(clientPayload.name || normalizedId),
|
||||||
status: clientPayload.status === 'revoked' ? 'revoked' : 'active',
|
status: nextStatus,
|
||||||
token: String(clientPayload.token || ''),
|
token: nextToken,
|
||||||
rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
|
rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
|
||||||
storageQuotaMb: Math.max(0, Math.floor(Number(clientPayload.storageQuotaMb) || 0)),
|
storageQuotaMb: Math.max(0, Math.floor(Number(clientPayload.storageQuotaMb) || 0)),
|
||||||
allowedIps: typeof clientPayload.allowedIps === 'string' ? clientPayload.allowedIps : '',
|
allowedIps: nextAllowedIps,
|
||||||
expiresAt: clientPayload.expiresAt ? String(clientPayload.expiresAt) : null,
|
expiresAt: nextExpiresAt,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
io.to(`client:${normalizedId}`).disconnectSockets(true);
|
|
||||||
|
const tokenExpired = Boolean(
|
||||||
|
nextExpiresAt && isClientTokenExpired(nextExpiresAt),
|
||||||
|
);
|
||||||
|
const shouldDisconnectRoom =
|
||||||
|
!clientPayload ||
|
||||||
|
nextStatus === 'revoked' ||
|
||||||
|
tokenExpired ||
|
||||||
|
Boolean(
|
||||||
|
prevClient &&
|
||||||
|
(prevClient.status !== nextStatus ||
|
||||||
|
(nextToken && prevClient.token !== nextToken) ||
|
||||||
|
(prevClient.allowedIps || '').trim() !== nextAllowedIps ||
|
||||||
|
(prevClient.expiresAt || null) !== nextExpiresAt),
|
||||||
|
);
|
||||||
|
|
||||||
|
let disconnected = false;
|
||||||
|
if (shouldDisconnectRoom) {
|
||||||
|
io.to(`client:${normalizedId}`).disconnectSockets(true);
|
||||||
|
disconnected = true;
|
||||||
|
} else if (!prevClient) {
|
||||||
|
const liveSockets = await io.in(`client:${normalizedId}`).fetchSockets();
|
||||||
|
for (const sock of liveSockets) {
|
||||||
|
const sockToken =
|
||||||
|
sock.data && sock.data.token ? String(sock.data.token) : '';
|
||||||
|
const sockIp =
|
||||||
|
sock.data && sock.data.clientIp ? String(sock.data.clientIp) : '';
|
||||||
|
const tokenMismatch = Boolean(
|
||||||
|
nextToken && sockToken && sockToken !== nextToken,
|
||||||
|
);
|
||||||
|
const ipForbidden = Boolean(
|
||||||
|
nextAllowedIps && (!sockIp || !isClientIpAllowed(sockIp, nextAllowedIps)),
|
||||||
|
);
|
||||||
|
if (tokenMismatch || ipForbidden) {
|
||||||
|
sock.disconnect(true);
|
||||||
|
disconnected = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return res.json({ success: true, disconnected });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
res.json({ success: true });
|
res.json({ success: true });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
app.post('/internal/admin-session-sync', requireInternalAuth, (req, res) => {
|
||||||
|
const synced = syncInMemoryAdminSessionState(req.body || {});
|
||||||
|
res.json({ success: true, synced });
|
||||||
|
});
|
||||||
|
|
||||||
function stopTimers() {
|
function stopTimers() {
|
||||||
if (flushTimer) {
|
if (flushTimer) {
|
||||||
clearInterval(flushTimer);
|
clearInterval(flushTimer);
|
||||||
@@ -2016,6 +2334,10 @@ async function gracefulShutdown(signal, options = {}) {
|
|||||||
stopTimers();
|
stopTimers();
|
||||||
|
|
||||||
// Close Socket.IO & HTTP listeners before final flush so late traffic cannot enter after drain starts
|
// Close Socket.IO & HTTP listeners before final flush so late traffic cannot enter after drain starts
|
||||||
|
if (filesManagerHandle && typeof filesManagerHandle.close === 'function') {
|
||||||
|
await Promise.race([filesManagerHandle.close().catch(() => undefined), sleepMs(2000)]);
|
||||||
|
filesManagerHandle = null;
|
||||||
|
}
|
||||||
await closeServerGracefully();
|
await closeServerGracefully();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -2097,6 +2419,16 @@ async function startIngestionServer() {
|
|||||||
|
|
||||||
if (isShuttingDown) return;
|
if (isShuttingDown) return;
|
||||||
|
|
||||||
|
if (!filesManagerHandle) {
|
||||||
|
filesManagerHandle = await mountFilesManager({
|
||||||
|
app,
|
||||||
|
server,
|
||||||
|
rootDir: writer.baseDir,
|
||||||
|
getDbPool: () => dbPool,
|
||||||
|
corsOriginDelegate,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
await new Promise((resolve) => {
|
await new Promise((resolve) => {
|
||||||
server.listen(PORT, () => {
|
server.listen(PORT, () => {
|
||||||
console.log(
|
console.log(
|
||||||
@@ -2144,7 +2476,12 @@ module.exports = {
|
|||||||
isClientIpAllowed,
|
isClientIpAllowed,
|
||||||
isClientTokenExpired,
|
isClientTokenExpired,
|
||||||
triggerWebhookAlert,
|
triggerWebhookAlert,
|
||||||
|
validateOutboundWebhookUrlAsync,
|
||||||
clearWebhookCooldownCacheForTests,
|
clearWebhookCooldownCacheForTests,
|
||||||
decryptVaultSecret,
|
decryptVaultSecret,
|
||||||
formatOutboundWebhookBody,
|
formatOutboundWebhookBody,
|
||||||
|
mountFilesManager,
|
||||||
|
syncInMemoryAdminSessionState,
|
||||||
|
clearInMemoryAdminSessionStateForTests,
|
||||||
|
getFilesManagerHandle: () => filesManagerHandle,
|
||||||
};
|
};
|
||||||
@@ -15,6 +15,22 @@ const REDIS_RATE_LIMIT_LUA_SCRIPT = [
|
|||||||
"return c",
|
"return c",
|
||||||
].join("\n");
|
].join("\n");
|
||||||
|
|
||||||
|
const DEFAULT_MAX_BATCH_ITEMS = 1000;
|
||||||
|
const MIN_BATCH_ITEMS = 1;
|
||||||
|
const MAX_BATCH_ITEMS_CEILING = 10000;
|
||||||
|
|
||||||
|
function resolveMaxBatchItems() {
|
||||||
|
const raw = process.env.DATASTACK_MAX_BATCH_ITEMS;
|
||||||
|
if (raw === undefined || String(raw).trim() === "") {
|
||||||
|
return DEFAULT_MAX_BATCH_ITEMS;
|
||||||
|
}
|
||||||
|
const parsed = Math.floor(Number(raw));
|
||||||
|
if (!Number.isFinite(parsed)) {
|
||||||
|
return DEFAULT_MAX_BATCH_ITEMS;
|
||||||
|
}
|
||||||
|
return Math.min(MAX_BATCH_ITEMS_CEILING, Math.max(MIN_BATCH_ITEMS, parsed));
|
||||||
|
}
|
||||||
|
|
||||||
function generateFallbackEventId() {
|
function generateFallbackEventId() {
|
||||||
return `evt_${Date.now()}_${crypto.randomBytes(6).toString("hex")}`;
|
return `evt_${Date.now()}_${crypto.randomBytes(6).toString("hex")}`;
|
||||||
}
|
}
|
||||||
@@ -597,6 +613,21 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const maxBatchItems = resolveMaxBatchItems();
|
||||||
|
if (batch.length > maxBatchItems) {
|
||||||
|
if (callback) {
|
||||||
|
callback({
|
||||||
|
success: false,
|
||||||
|
processed: 0,
|
||||||
|
rejected: batch.length,
|
||||||
|
code: "BATCH_LIMIT_EXCEEDED",
|
||||||
|
error: `BATCH_LIMIT_EXCEEDED: Batch size (${batch.length}) exceeds maximum allowed ${maxBatchItems} items per request.`,
|
||||||
|
maxBatchItems,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const clientIp = socket.data.clientIp || extractSocketClientIp(socket);
|
const clientIp = socket.data.clientIp || extractSocketClientIp(socket);
|
||||||
const dbCheck = await verifyClientInDb(
|
const dbCheck = await verifyClientInDb(
|
||||||
clientId,
|
clientId,
|
||||||
|
|||||||
@@ -0,0 +1,308 @@
|
|||||||
|
import test from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import http from 'node:http';
|
||||||
|
import os from 'node:os';
|
||||||
|
import path from 'node:path';
|
||||||
|
import express from 'express';
|
||||||
|
import { createAdminSessionToken } from '../core/auth/adminSession';
|
||||||
|
|
||||||
|
const fm = require('../server/filesManager');
|
||||||
|
|
||||||
|
test('FM-REQ-02: buildCloudCmdConfig enforces RCE lockdown, root jail, zip packer, and scrubs default tokens', () => {
|
||||||
|
const tmpRoot = path.resolve(os.tmpdir(), 'datastack-fm-cfg-test');
|
||||||
|
const cfg = fm.buildCloudCmdConfig(tmpRoot);
|
||||||
|
|
||||||
|
assert.equal(cfg.root, tmpRoot);
|
||||||
|
assert.equal(cfg.prefix, '/vcon/files-engine');
|
||||||
|
assert.equal(cfg.console, false);
|
||||||
|
assert.equal(cfg.terminal, false);
|
||||||
|
assert.equal(cfg.configDialog, false);
|
||||||
|
assert.equal(cfg.auth, false);
|
||||||
|
assert.equal(cfg.open, false);
|
||||||
|
assert.equal(cfg.online, false);
|
||||||
|
assert.equal(cfg.contact, false);
|
||||||
|
assert.equal(cfg.oneFilePanel, true);
|
||||||
|
assert.equal(cfg.packer, 'zip');
|
||||||
|
assert.equal(cfg.editor, 'edward');
|
||||||
|
assert.equal(cfg.password, '');
|
||||||
|
assert.equal(cfg.exportToken, '');
|
||||||
|
assert.equal(cfg.importToken, '');
|
||||||
|
assert.equal(cfg.dropboxToken, '');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('FM-REQ-04: authenticateFilesRequest fails closed on missing prod secret and DB outages', async () => {
|
||||||
|
const token = await createAdminSessionToken(
|
||||||
|
'owner@datastack.dev',
|
||||||
|
fm.DEV_DEFAULT_SESSION_SECRET,
|
||||||
|
Date.now() + 60_000,
|
||||||
|
1,
|
||||||
|
'owner',
|
||||||
|
'owner-1',
|
||||||
|
);
|
||||||
|
|
||||||
|
const prodMissing = await fm.authenticateFilesRequest({
|
||||||
|
cookieHeader: `datastack_admin_session=${token}`,
|
||||||
|
dbPool: null,
|
||||||
|
env: { NODE_ENV: 'production', DATASTACK_ADMIN_SESSION_SECRET: '' },
|
||||||
|
});
|
||||||
|
assert.equal(prodMissing.valid, false);
|
||||||
|
assert.equal(prodMissing.status, 503);
|
||||||
|
assert.equal(prodMissing.code, 'FILES_AUTH_UNCONFIGURED');
|
||||||
|
|
||||||
|
const dbUnavailable = await fm.authenticateFilesRequest({
|
||||||
|
cookieHeader: `datastack_admin_session=${token}`,
|
||||||
|
dbPool: {
|
||||||
|
query: async () => {
|
||||||
|
throw new Error('ECONNREFUSED');
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
assert.equal(dbUnavailable.valid, false);
|
||||||
|
assert.equal(dbUnavailable.status, 503);
|
||||||
|
assert.equal(dbUnavailable.code, 'AUTH_DB_UNAVAILABLE');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('FM-REQ-01..06: live CloudCmd mount enforces auth, DB revocation, RBAC, CSRF, *.lock & traversal guards, and full file CRUD + ZIP/UNZIP', async () => {
|
||||||
|
const tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'datastack-fm-e2e-'));
|
||||||
|
fs.mkdirSync(path.join(tmpRoot, 'client-alpha'), { recursive: true });
|
||||||
|
fs.writeFileSync(
|
||||||
|
path.join(tmpRoot, 'client-alpha', '2026-11-20.csv'),
|
||||||
|
'eventId,clientId\nev-1,client-alpha\n',
|
||||||
|
);
|
||||||
|
fs.writeFileSync(path.join(tmpRoot, 'client-alpha', '2026-11-20.csv.lock'), 'pid=99');
|
||||||
|
|
||||||
|
const app = express();
|
||||||
|
const server = http.createServer(app);
|
||||||
|
let activeDbPool: { query: (sql: string, params?: unknown[]) => Promise<{ rows: unknown[] }> } | null = null;
|
||||||
|
|
||||||
|
const handle = await fm.mountFilesManager({
|
||||||
|
app,
|
||||||
|
server,
|
||||||
|
rootDir: tmpRoot,
|
||||||
|
getDbPool: () => activeDbPool,
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
assert.equal(handle.engineReady, true, `CloudCmd failed to init: ${handle.initErrorMessage}`);
|
||||||
|
assert.equal(handle.socketPath, '/vcon/files-engine/socket.io');
|
||||||
|
|
||||||
|
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', () => resolve()));
|
||||||
|
const address = server.address() as { port: number };
|
||||||
|
const baseUrl = `http://127.0.0.1:${address.port}`;
|
||||||
|
const hostHeader = `127.0.0.1:${address.port}`;
|
||||||
|
|
||||||
|
const ownerToken = await createAdminSessionToken(
|
||||||
|
'owner@datastack.dev',
|
||||||
|
fm.DEV_DEFAULT_SESSION_SECRET,
|
||||||
|
Date.now() + 60_000,
|
||||||
|
1,
|
||||||
|
'owner',
|
||||||
|
'owner-1',
|
||||||
|
);
|
||||||
|
const viewerToken = await createAdminSessionToken(
|
||||||
|
'viewer@datastack.dev',
|
||||||
|
fm.DEV_DEFAULT_SESSION_SECRET,
|
||||||
|
Date.now() + 60_000,
|
||||||
|
1,
|
||||||
|
'viewer',
|
||||||
|
'member-viewer-1',
|
||||||
|
);
|
||||||
|
|
||||||
|
const ownerHeaders = {
|
||||||
|
Cookie: `datastack_admin_session=${ownerToken}`,
|
||||||
|
Origin: baseUrl,
|
||||||
|
Host: hostHeader,
|
||||||
|
};
|
||||||
|
const viewerHeaders = {
|
||||||
|
Cookie: `datastack_admin_session=${viewerToken}`,
|
||||||
|
Origin: baseUrl,
|
||||||
|
Host: hostHeader,
|
||||||
|
};
|
||||||
|
|
||||||
|
// 1. Unauthenticated access blocked
|
||||||
|
assert.equal((await fetch(`${baseUrl}/vcon/files-engine/_status`)).status, 401);
|
||||||
|
assert.equal((await fetch(`${baseUrl}/vcon/files-engine/`)).status, 401);
|
||||||
|
|
||||||
|
// 2. Authenticated status probe & locked HTML output
|
||||||
|
const statusRes = await fetch(`${baseUrl}/vcon/files-engine/_status`, {
|
||||||
|
headers: { Cookie: ownerHeaders.Cookie },
|
||||||
|
});
|
||||||
|
assert.equal(statusRes.status, 200);
|
||||||
|
const statusJson = (await statusRes.json()) as { ok: boolean; role: string; readOnly: boolean };
|
||||||
|
assert.equal(statusJson.ok, true);
|
||||||
|
assert.equal(statusJson.role, 'owner');
|
||||||
|
assert.equal(statusJson.readOnly, false);
|
||||||
|
|
||||||
|
const htmlRes = await fetch(`${baseUrl}/vcon/files-engine/`, {
|
||||||
|
headers: { Cookie: ownerHeaders.Cookie },
|
||||||
|
});
|
||||||
|
assert.equal(htmlRes.status, 200);
|
||||||
|
const html = await htmlRes.text();
|
||||||
|
assert.ok(html.includes('/vcon/files-engine'));
|
||||||
|
assert.ok(html.includes('"console":false'));
|
||||||
|
assert.ok(html.includes('"terminal":false'));
|
||||||
|
assert.ok(html.includes('"configDialog":false'));
|
||||||
|
assert.ok(html.includes('"packer":"zip"'));
|
||||||
|
|
||||||
|
// 3. Create, rename, ZIP, UNZIP, download, and delete
|
||||||
|
const createRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/fs/client-alpha/note.csv`, {
|
||||||
|
method: 'PUT',
|
||||||
|
headers: ownerHeaders,
|
||||||
|
body: 'k,v\n1,ok\n',
|
||||||
|
});
|
||||||
|
assert.equal(createRes.status, 200);
|
||||||
|
assert.equal(fs.readFileSync(path.join(tmpRoot, 'client-alpha', 'note.csv'), 'utf8'), 'k,v\n1,ok\n');
|
||||||
|
|
||||||
|
const renameRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/rename`, {
|
||||||
|
method: 'PUT',
|
||||||
|
headers: { ...ownerHeaders, 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ from: '/client-alpha/note.csv', to: '/client-alpha/note-renamed.csv' }),
|
||||||
|
});
|
||||||
|
assert.equal(renameRes.status, 200);
|
||||||
|
assert.equal(fs.existsSync(path.join(tmpRoot, 'client-alpha', 'note-renamed.csv')), true);
|
||||||
|
|
||||||
|
const packRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/pack`, {
|
||||||
|
method: 'PUT',
|
||||||
|
headers: { ...ownerHeaders, 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({
|
||||||
|
from: '/client-alpha',
|
||||||
|
to: '/client-alpha/bundle.zip',
|
||||||
|
names: ['note-renamed.csv'],
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
assert.equal(packRes.status, 200);
|
||||||
|
assert.equal(fs.existsSync(path.join(tmpRoot, 'client-alpha', 'bundle.zip')), true);
|
||||||
|
|
||||||
|
fs.mkdirSync(path.join(tmpRoot, 'client-alpha', 'extracted'), { recursive: true });
|
||||||
|
const extractRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/extract`, {
|
||||||
|
method: 'PUT',
|
||||||
|
headers: { ...ownerHeaders, 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({
|
||||||
|
from: '/client-alpha/bundle.zip',
|
||||||
|
to: '/client-alpha/extracted',
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
assert.equal(extractRes.status, 200);
|
||||||
|
assert.equal(fs.existsSync(path.join(tmpRoot, 'client-alpha', 'extracted', 'note-renamed.csv')), true);
|
||||||
|
|
||||||
|
const downloadRes = await fetch(
|
||||||
|
`${baseUrl}/vcon/files-engine/fs/client-alpha/note-renamed.csv?download`,
|
||||||
|
{ headers: { Cookie: ownerHeaders.Cookie } },
|
||||||
|
);
|
||||||
|
assert.equal(downloadRes.status, 200);
|
||||||
|
assert.equal(await downloadRes.text(), 'k,v\n1,ok\n');
|
||||||
|
|
||||||
|
const deleteRes = await fetch(
|
||||||
|
`${baseUrl}/vcon/files-engine/api/v1/fs/client-alpha/note-renamed.csv`,
|
||||||
|
{ method: 'DELETE', headers: ownerHeaders },
|
||||||
|
);
|
||||||
|
assert.equal(deleteRes.status, 200);
|
||||||
|
assert.equal(fs.existsSync(path.join(tmpRoot, 'client-alpha', 'note-renamed.csv')), false);
|
||||||
|
|
||||||
|
// 4. Viewer role: GET allowed, mutations blocked (403)
|
||||||
|
const viewerList = await fetch(`${baseUrl}/vcon/files-engine/api/v1/fs/`, {
|
||||||
|
headers: { Cookie: viewerHeaders.Cookie },
|
||||||
|
});
|
||||||
|
assert.equal(viewerList.status, 200);
|
||||||
|
|
||||||
|
const viewerWrite = await fetch(`${baseUrl}/vcon/files-engine/api/v1/fs/client-alpha/nope.csv`, {
|
||||||
|
method: 'PUT',
|
||||||
|
headers: viewerHeaders,
|
||||||
|
body: 'blocked',
|
||||||
|
});
|
||||||
|
assert.equal(viewerWrite.status, 403);
|
||||||
|
assert.equal(((await viewerWrite.json()) as { code: string }).code, 'INSUFFICIENT_ROLE_PERMISSION');
|
||||||
|
|
||||||
|
// 5. Cross-origin CSRF rejected (403)
|
||||||
|
const csrfRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/fs/client-alpha/csrf.csv`, {
|
||||||
|
method: 'PUT',
|
||||||
|
headers: { Cookie: ownerHeaders.Cookie, Origin: 'https://evil.example', Host: hostHeader },
|
||||||
|
body: 'blocked',
|
||||||
|
});
|
||||||
|
assert.equal(csrfRes.status, 403);
|
||||||
|
assert.equal(((await csrfRes.json()) as { code: string }).code, 'CSRF_REJECTED');
|
||||||
|
|
||||||
|
// 6. Cooperative *.lock protection in URL and JSON body
|
||||||
|
const lockUrlRes = await fetch(
|
||||||
|
`${baseUrl}/vcon/files-engine/api/v1/fs/client-alpha/2026-11-20.csv.lock`,
|
||||||
|
{ method: 'DELETE', headers: ownerHeaders },
|
||||||
|
);
|
||||||
|
assert.equal(lockUrlRes.status, 403);
|
||||||
|
assert.equal(((await lockUrlRes.json()) as { code: string }).code, 'LOCKFILE_PROTECTED');
|
||||||
|
|
||||||
|
const lockBodyRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/rename`, {
|
||||||
|
method: 'PUT',
|
||||||
|
headers: { ...ownerHeaders, 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({
|
||||||
|
from: '/client-alpha/2026-11-20.csv.lock',
|
||||||
|
to: '/client-alpha/hijack.csv',
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
assert.equal(lockBodyRes.status, 403);
|
||||||
|
assert.equal(((await lockBodyRes.json()) as { code: string }).code, 'LOCKFILE_PROTECTED');
|
||||||
|
|
||||||
|
// 7. Path traversal & runtime config PATCH blocked
|
||||||
|
const travRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/fs/..%2F..%2Fpackage.json`, {
|
||||||
|
headers: { Cookie: ownerHeaders.Cookie },
|
||||||
|
});
|
||||||
|
assert.equal(travRes.status, 403);
|
||||||
|
assert.equal(((await travRes.json()) as { code: string }).code, 'PATH_OUTSIDE_ROOT');
|
||||||
|
|
||||||
|
const cfgPatchRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/config`, {
|
||||||
|
method: 'PATCH',
|
||||||
|
headers: { ...ownerHeaders, 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ console: true }),
|
||||||
|
});
|
||||||
|
assert.equal(cfgPatchRes.status, 403);
|
||||||
|
assert.equal(((await cfgPatchRes.json()) as { code: string }).code, 'CONFIG_LOCKED');
|
||||||
|
|
||||||
|
// 8. Revoked DB token_version immediately rejects session (401 SESSION_REVOKED)
|
||||||
|
activeDbPool = {
|
||||||
|
query: async (sql: string) => {
|
||||||
|
if (sql.includes('admin_users')) {
|
||||||
|
return { rows: [{ email: 'owner@datastack.dev', token_version: 2 }] };
|
||||||
|
}
|
||||||
|
return { rows: [] };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const revokedRes = await fetch(`${baseUrl}/vcon/files-engine/_status`, {
|
||||||
|
headers: { Cookie: ownerHeaders.Cookie },
|
||||||
|
});
|
||||||
|
assert.equal(revokedRes.status, 401);
|
||||||
|
assert.equal(((await revokedRes.json()) as { code: string }).code, 'SESSION_REVOKED');
|
||||||
|
} finally {
|
||||||
|
await handle.close();
|
||||||
|
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||||
|
fs.rmSync(tmpRoot, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test('FM-REQ-07..09: proxy routing, SW bypass, Sidebar nav, and /vcon/files workspace are cleanly wired', () => {
|
||||||
|
const root = process.cwd();
|
||||||
|
const nextConfig = fs.readFileSync(path.join(root, 'next.config.ts'), 'utf8');
|
||||||
|
const coolifyNginx = fs.readFileSync(path.join(root, 'docker', 'coolify-nginx.conf'), 'utf8');
|
||||||
|
const localNginx = fs.readFileSync(path.join(root, 'nginx', 'default.conf'), 'utf8');
|
||||||
|
const swJs = fs.readFileSync(path.join(root, 'public', 'sw.js'), 'utf8');
|
||||||
|
const sidebar = fs.readFileSync(path.join(root, 'components', 'layout', 'Sidebar.tsx'), 'utf8');
|
||||||
|
const page = fs.readFileSync(path.join(root, 'app', 'vcon', 'files', 'page.tsx'), 'utf8');
|
||||||
|
const workspace = fs.readFileSync(
|
||||||
|
path.join(root, 'components', 'sections', 'FilesManagerWorkspace.tsx'),
|
||||||
|
'utf8',
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.ok(nextConfig.includes("source: '/vcon/files-engine/:path*'"));
|
||||||
|
assert.ok(coolifyNginx.includes('location /vcon/files-engine'));
|
||||||
|
assert.ok(localNginx.includes('location /vcon/files-engine'));
|
||||||
|
assert.ok(swJs.includes("url.pathname.startsWith('/vcon/files-engine/')"));
|
||||||
|
assert.ok(sidebar.includes("{ name: 'Files Manager', href: '/vcon/files', icon: 'FolderOpen' }"));
|
||||||
|
assert.ok(page.includes('FilesManagerWorkspace'));
|
||||||
|
const statusRoute = fs.readFileSync(
|
||||||
|
path.join(root, 'app', 'api', 'files', 'status', 'route.ts'),
|
||||||
|
'utf8',
|
||||||
|
);
|
||||||
|
assert.ok(statusRoute.includes('/vcon/files-engine/_status'));
|
||||||
|
assert.ok(workspace.includes('/api/files/status'));
|
||||||
|
assert.ok(workspace.includes('files-manager-offline'));
|
||||||
|
assert.ok(workspace.includes('files-manager-iframe'));
|
||||||
|
});
|
||||||
@@ -444,17 +444,17 @@ test("DEPLOY-BUG-02, 03 & 04: Port 4000 memoizes schema DDLs, hydrates rotated J
|
|||||||
assert.equal(failedSync, false);
|
assert.equal(failedSync, false);
|
||||||
assert.equal(ddlCount, 1);
|
assert.equal(ddlCount, 1);
|
||||||
|
|
||||||
// 2. Recovery runs 8 DDLs once and hydrates rotated jwt_secret; repeated calls do zero extra DDLs
|
// 2. Recovery runs 10 DDLs once (7 CREATE TABLE + 3 ALTER TABLE) and hydrates rotated jwt_secret; repeated calls do zero extra DDLs
|
||||||
failDdl = false;
|
failDdl = false;
|
||||||
ddlCount = 0;
|
ddlCount = 0;
|
||||||
assert.equal(await ingestionServer.syncSettingsFromDatabase(), true);
|
assert.equal(await ingestionServer.syncSettingsFromDatabase(), true);
|
||||||
assert.equal(ddlCount, 8);
|
assert.equal(ddlCount, 10);
|
||||||
assert.equal(selectCount, 1);
|
assert.equal(selectCount, 1);
|
||||||
assert.equal(ingestionServer.runtimeConfig.jwtSecret, rotatedSecret);
|
assert.equal(ingestionServer.runtimeConfig.jwtSecret, rotatedSecret);
|
||||||
|
|
||||||
assert.equal(await ingestionServer.syncSettingsFromDatabase(), true);
|
assert.equal(await ingestionServer.syncSettingsFromDatabase(), true);
|
||||||
assert.equal(await ingestionServer.syncSettingsFromDatabase(), true);
|
assert.equal(await ingestionServer.syncSettingsFromDatabase(), true);
|
||||||
assert.equal(ddlCount, 8, "Repeated 3s sync ticks must NOT re-run CREATE/ALTER DDL statements");
|
assert.equal(ddlCount, 10, "Repeated 3s sync ticks must NOT re-run CREATE/ALTER DDL statements");
|
||||||
assert.equal(selectCount, 3);
|
assert.equal(selectCount, 3);
|
||||||
|
|
||||||
// 3. Graceful shutdown flushes buffered events, quits Redis, and closes PostgreSQL pool cleanly
|
// 3. Graceful shutdown flushes buffered events, quits Redis, and closes PostgreSQL pool cleanly
|
||||||
@@ -606,3 +606,41 @@ test("V2-Deploy-Fix-01 (DEPLOY-V2-BUG-01, 02 & 03): middleware permits GET /api/
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("PREPUSH-FIX-01: Nginx preserves upstream real client IP behind Traefik, Dockerfile.web & Compose align runtime/build env vars, and package-lock includes cloudcmd", () => {
|
||||||
|
for (const confRel of ["docker/coolify-nginx.conf", "nginx/default.conf"]) {
|
||||||
|
const conf = readFileSync(resolve(process.cwd(), confRel), "utf8");
|
||||||
|
assert.match(conf, /set_real_ip_from\s+127\.0\.0\.1;/, `${confRel} must trust loopback proxy`);
|
||||||
|
assert.match(conf, /set_real_ip_from\s+172\.16\.0\.0\/12;/, `${confRel} must trust Docker bridge proxy subnet`);
|
||||||
|
assert.match(conf, /real_ip_header\s+X-Forwarded-For;/, `${confRel} must extract real IP from X-Forwarded-For`);
|
||||||
|
assert.match(conf, /real_ip_recursive\s+on;/, `${confRel} must enable real_ip_recursive`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const webDockerfile = readFileSync(resolve(process.cwd(), "docker/Dockerfile.web"), "utf8");
|
||||||
|
assert.match(
|
||||||
|
webDockerfile,
|
||||||
|
/ARG INGESTION_INTERNAL_URL=http:\/\/ingestion:4000[\s\S]*ENV INGESTION_INTERNAL_URL=\$\{INGESTION_INTERNAL_URL\}[\s\S]*RUN npm run build/,
|
||||||
|
"Dockerfile.web builder must set INGESTION_INTERNAL_URL=http://ingestion:4000 before npm run build",
|
||||||
|
);
|
||||||
|
|
||||||
|
const compose = readFileSync(resolve(process.cwd(), "docker/docker-compose.yml"), "utf8");
|
||||||
|
for (const envKey of [
|
||||||
|
"DATASTACK_ALLOW_PRIVATE_URLS",
|
||||||
|
"DATASTACK_MAX_BATCH_ITEMS",
|
||||||
|
"DATASTACK_RECENT_LIMIT",
|
||||||
|
]) {
|
||||||
|
const matches = compose.match(new RegExp(`^\\s+${envKey}:`, "gm")) ?? [];
|
||||||
|
assert.equal(matches.length, 2, `Both web and ingestion must forward ${envKey}`);
|
||||||
|
}
|
||||||
|
assert.match(compose, /^\s+DATASTACK_MAINTENANCE_INTERVAL_MS:/m);
|
||||||
|
assert.match(compose, /^\s+DATASTACK_PROXY_MODE:/m);
|
||||||
|
|
||||||
|
const lockJson = JSON.parse(
|
||||||
|
readFileSync(resolve(process.cwd(), "package-lock.json"), "utf8"),
|
||||||
|
) as {
|
||||||
|
packages?: Record<string, { dependencies?: Record<string, string> }>;
|
||||||
|
};
|
||||||
|
assert.ok(lockJson.packages?.[""]?.dependencies?.cloudcmd, "Root lockfile must declare cloudcmd");
|
||||||
|
assert.ok(lockJson.packages?.["node_modules/cloudcmd"], "package-lock.json must resolve node_modules/cloudcmd");
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
@@ -227,6 +227,9 @@ test("R3-Phase-02 BUG-05: Singleton periodic scheduler automatically archives ex
|
|||||||
const addr = mockS3.address();
|
const addr = mockS3.address();
|
||||||
assert.ok(addr && typeof addr === "object");
|
assert.ok(addr && typeof addr === "object");
|
||||||
|
|
||||||
|
const prevAllowPrivate = process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||||
|
process.env.DATASTACK_ALLOW_PRIVATE_URLS = "true";
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Configure S3 backup + autoPruneDays (cooldown will be reset so scheduled timer runs)
|
// Configure S3 backup + autoPruneDays (cooldown will be reset so scheduled timer runs)
|
||||||
await SettingsManager.updateSettingsAsync({
|
await SettingsManager.updateSettingsAsync({
|
||||||
@@ -340,6 +343,8 @@ test("R3-Phase-02 BUG-05: Singleton periodic scheduler automatically archives ex
|
|||||||
assert.equal(unhydratedResult.ran, false);
|
assert.equal(unhydratedResult.ran, false);
|
||||||
assert.equal(unhydratedResult.skippedReason, "DB_UNHYDRATED");
|
assert.equal(unhydratedResult.skippedReason, "DB_UNHYDRATED");
|
||||||
} finally {
|
} finally {
|
||||||
|
if (prevAllowPrivate === undefined) delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||||
|
else process.env.DATASTACK_ALLOW_PRIVATE_URLS = prevAllowPrivate;
|
||||||
SettingsManager.stopMaintenanceSchedulerForTests();
|
SettingsManager.stopMaintenanceSchedulerForTests();
|
||||||
await new Promise<void>((resolve) => mockS3.close(() => resolve()));
|
await new Promise<void>((resolve) => mockS3.close(() => resolve()));
|
||||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||||
|
|||||||
@@ -53,7 +53,6 @@ test("R3-Phase-04 [BUG-11 / ERR-R3-07]: shouldRedirectSimulatorUnauthorized sepa
|
|||||||
"AUTH_INVALID",
|
"AUTH_INVALID",
|
||||||
"CLIENT_REVOKED",
|
"CLIENT_REVOKED",
|
||||||
"CLIENT_IP_FORBIDDEN",
|
"CLIENT_IP_FORBIDDEN",
|
||||||
"CLIENT_UNAUTHORIZED",
|
|
||||||
]) {
|
]) {
|
||||||
assert.equal(
|
assert.equal(
|
||||||
shouldRedirectSimulatorUnauthorized(401, {
|
shouldRedirectSimulatorUnauthorized(401, {
|
||||||
@@ -134,12 +133,12 @@ test("R3-Phase-04 [BUG-14]: expired clients are separated in UI helpers, ApiEngi
|
|||||||
assert.equal(isUiClientExpired("2026-11-19T00:00:00.000Z", nowMs), true);
|
assert.equal(isUiClientExpired("2026-11-19T00:00:00.000Z", nowMs), true);
|
||||||
|
|
||||||
const sampleClients = [
|
const sampleClients = [
|
||||||
{ status: "active", isOnline: true, lastConnectedAt: "2026-11-20T11:59:00.000Z", expiresAt: null },
|
{ status: "active" as const, lastActive: "2026-11-20T11:59:30.000Z", expiresAt: null },
|
||||||
{ status: "active", isOnline: false, lastConnectedAt: "2026-11-20T10:00:00.000Z", expiresAt: "2026-12-01T00:00:00.000Z" },
|
{ status: "active" as const, lastActive: "2026-11-20T10:00:00.000Z", expiresAt: "2026-12-01T00:00:00.000Z" },
|
||||||
{ status: "active", isOnline: false, lastConnectedAt: null, expiresAt: null },
|
{ status: "active" as const, lastActive: undefined, expiresAt: null },
|
||||||
{ status: "active", isOnline: false, lastConnectedAt: "2026-11-19T10:00:00.000Z", expiresAt: "2026-11-19T12:00:00.000Z" },
|
{ status: "active" as const, lastActive: "2026-11-19T10:00:00.000Z", expiresAt: "2026-11-19T12:00:00.000Z" },
|
||||||
{ status: "active", isOnline: true, lastConnectedAt: "2026-11-20T11:58:00.000Z", expiresAt: "2026-11-20T11:00:00.000Z" },
|
{ status: "active" as const, lastActive: "2026-11-20T11:59:45.000Z", expiresAt: "2026-11-20T11:00:00.000Z" },
|
||||||
{ status: "revoked", isOnline: false, lastConnectedAt: null, expiresAt: "2026-11-10T00:00:00.000Z" },
|
{ status: "revoked" as const, lastActive: undefined, expiresAt: "2026-11-10T00:00:00.000Z" },
|
||||||
];
|
];
|
||||||
|
|
||||||
assert.equal(classifyUiClientStatus(sampleClients[0], nowMs), "online");
|
assert.equal(classifyUiClientStatus(sampleClients[0], nowMs), "online");
|
||||||
@@ -152,6 +151,7 @@ test("R3-Phase-04 [BUG-14]: expired clients are separated in UI helpers, ApiEngi
|
|||||||
const summary = summarizeUiClientCounts(sampleClients, nowMs);
|
const summary = summarizeUiClientCounts(sampleClients, nowMs);
|
||||||
assert.deepEqual(summary, {
|
assert.deepEqual(summary, {
|
||||||
total: 6,
|
total: 6,
|
||||||
|
active: 3,
|
||||||
online: 1,
|
online: 1,
|
||||||
offline: 1,
|
offline: 1,
|
||||||
pending: 1,
|
pending: 1,
|
||||||
@@ -160,12 +160,20 @@ test("R3-Phase-04 [BUG-14]: expired clients are separated in UI helpers, ApiEngi
|
|||||||
});
|
});
|
||||||
|
|
||||||
// Live integration: ApiEngine.getDashboardMetrics() & GET /api/prometheus
|
// Live integration: ApiEngine.getDashboardMetrics() & GET /api/prometheus
|
||||||
const activeClient = SDKManager.registerClient("r3p4_active_client");
|
const activeClient = SDKManager.registerClient({
|
||||||
const expiredClient = SDKManager.registerClient("r3p4_expired_client", {
|
clientId: "r3p4_active_client",
|
||||||
|
name: "R3P4 Active Client",
|
||||||
|
});
|
||||||
|
const expiredClient = SDKManager.registerClient({
|
||||||
|
clientId: "r3p4_expired_client",
|
||||||
|
name: "R3P4 Expired Client",
|
||||||
expiresAt: "2020-01-01T00:00:00.000Z",
|
expiresAt: "2020-01-01T00:00:00.000Z",
|
||||||
});
|
});
|
||||||
const revokedClient = SDKManager.registerClient("r3p4_revoked_client");
|
const revokedClient = SDKManager.registerClient({
|
||||||
SDKManager.revokeClient(revokedClient.id);
|
clientId: "r3p4_revoked_client",
|
||||||
|
name: "R3P4 Revoked Client",
|
||||||
|
});
|
||||||
|
SDKManager.updateClient(revokedClient.clientId, { status: "revoked" });
|
||||||
|
|
||||||
const prevSettings = SettingsManager.getSettings();
|
const prevSettings = SettingsManager.getSettings();
|
||||||
try {
|
try {
|
||||||
@@ -200,9 +208,9 @@ test("R3-Phase-04 [BUG-14]: expired clients are separated in UI helpers, ApiEngi
|
|||||||
assert.ok(Number(expiredMatch[1]) >= 1);
|
assert.ok(Number(expiredMatch[1]) >= 1);
|
||||||
assert.ok(Number(revokedMatch[1]) >= 1);
|
assert.ok(Number(revokedMatch[1]) >= 1);
|
||||||
} finally {
|
} finally {
|
||||||
SDKManager.deleteClient(activeClient.id);
|
SDKManager.deleteClient(activeClient.clientId);
|
||||||
SDKManager.deleteClient(expiredClient.id);
|
SDKManager.deleteClient(expiredClient.clientId);
|
||||||
SDKManager.deleteClient(revokedClient.id);
|
SDKManager.deleteClient(revokedClient.clientId);
|
||||||
SettingsManager.updateSettings({
|
SettingsManager.updateSettings({
|
||||||
enablePrometheusMetrics: prevSettings.enablePrometheusMetrics,
|
enablePrometheusMetrics: prevSettings.enablePrometheusMetrics,
|
||||||
});
|
});
|
||||||
@@ -212,37 +220,37 @@ test("R3-Phase-04 [BUG-14]: expired clients are separated in UI helpers, ApiEngi
|
|||||||
test("R3-Phase-04 [BUG-16]: RBAC role capabilities and UI surface wiring across VconShell and Admin pages", async () => {
|
test("R3-Phase-04 [BUG-16]: RBAC role capabilities and UI surface wiring across VconShell and Admin pages", async () => {
|
||||||
assert.deepEqual(getVconRoleCapabilities("owner"), {
|
assert.deepEqual(getVconRoleCapabilities("owner"), {
|
||||||
role: "owner",
|
role: "owner",
|
||||||
|
isOwner: true,
|
||||||
canAdminSettings: true,
|
canAdminSettings: true,
|
||||||
canManageTeam: true,
|
|
||||||
canOperate: true,
|
canOperate: true,
|
||||||
isReadOnly: false,
|
isReadOnly: false,
|
||||||
});
|
});
|
||||||
assert.deepEqual(getVconRoleCapabilities("admin"), {
|
assert.deepEqual(getVconRoleCapabilities("admin"), {
|
||||||
role: "admin",
|
role: "admin",
|
||||||
|
isOwner: false,
|
||||||
canAdminSettings: false,
|
canAdminSettings: false,
|
||||||
canManageTeam: false,
|
|
||||||
canOperate: true,
|
canOperate: true,
|
||||||
isReadOnly: false,
|
isReadOnly: false,
|
||||||
});
|
});
|
||||||
assert.deepEqual(getVconRoleCapabilities("operator"), {
|
assert.deepEqual(getVconRoleCapabilities("operator"), {
|
||||||
role: "operator",
|
role: "operator",
|
||||||
|
isOwner: false,
|
||||||
canAdminSettings: false,
|
canAdminSettings: false,
|
||||||
canManageTeam: false,
|
|
||||||
canOperate: true,
|
canOperate: true,
|
||||||
isReadOnly: false,
|
isReadOnly: false,
|
||||||
});
|
});
|
||||||
assert.deepEqual(getVconRoleCapabilities("viewer"), {
|
assert.deepEqual(getVconRoleCapabilities("viewer"), {
|
||||||
role: "viewer",
|
role: "viewer",
|
||||||
|
isOwner: false,
|
||||||
canAdminSettings: false,
|
canAdminSettings: false,
|
||||||
canManageTeam: false,
|
|
||||||
canOperate: false,
|
canOperate: false,
|
||||||
isReadOnly: true,
|
isReadOnly: true,
|
||||||
});
|
});
|
||||||
// Fail-closed default for unknown/missing roles
|
// Fail-closed default for unknown/missing roles
|
||||||
assert.deepEqual(getVconRoleCapabilities("unknown_role"), {
|
assert.deepEqual(getVconRoleCapabilities("unknown_role"), {
|
||||||
role: "viewer",
|
role: "viewer",
|
||||||
|
isOwner: false,
|
||||||
canAdminSettings: false,
|
canAdminSettings: false,
|
||||||
canManageTeam: false,
|
|
||||||
canOperate: false,
|
canOperate: false,
|
||||||
isReadOnly: true,
|
isReadOnly: true,
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,618 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import fs from "node:fs/promises";
|
||||||
|
import http from "node:http";
|
||||||
|
import { createRequire } from "node:module";
|
||||||
|
import type { AddressInfo } from "node:net";
|
||||||
|
import os from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import test from "node:test";
|
||||||
|
import zlib from "node:zlib";
|
||||||
|
import { NextRequest } from "next/server";
|
||||||
|
|
||||||
|
process.env.DATASTACK_DISABLE_LISTEN = "1";
|
||||||
|
|
||||||
|
import { GET as auditGet } from "../app/api/audit/route";
|
||||||
|
import { POST as settingsPost } from "../app/api/settings/route";
|
||||||
|
import { POST as webhookTestPost } from "../app/api/settings/webhook-test/route";
|
||||||
|
import { POST as s3TestPost } from "../app/api/storage/s3-test/route";
|
||||||
|
import { SettingsManager } from "../core/adminSettings/settingsManager";
|
||||||
|
import {
|
||||||
|
clearWebhookCooldownCacheForTests,
|
||||||
|
dispatchWebhookAlertAsync,
|
||||||
|
} from "../core/alerts/webhookNotifier";
|
||||||
|
import {
|
||||||
|
createAdminSessionToken,
|
||||||
|
getAdminSessionSecret,
|
||||||
|
} from "../core/auth/adminSession";
|
||||||
|
import {
|
||||||
|
createInitialAdmin,
|
||||||
|
createAdminTeamMemberAsync,
|
||||||
|
resetAdminStateForTests,
|
||||||
|
revokeAdminSessions,
|
||||||
|
updateAdminTeamMemberAsync,
|
||||||
|
deleteAdminTeamMemberAsync,
|
||||||
|
} from "../core/auth/adminUsers";
|
||||||
|
import {
|
||||||
|
clearAuditLogsForTests,
|
||||||
|
listAdminAuditLogsWithStatusAsync,
|
||||||
|
recordAdminAuditLogAsync,
|
||||||
|
} from "../core/auth/auditLog";
|
||||||
|
import { DataBufferEngine } from "../core/data/buffer";
|
||||||
|
import { SDKManager } from "../core/sdk/sdkManager";
|
||||||
|
import { getInternalIngestionHeaders } from "../core/Security/security";
|
||||||
|
import {
|
||||||
|
checkIpAddressAllowed,
|
||||||
|
validateOutboundUrlAsync,
|
||||||
|
validateOutboundUrlSync,
|
||||||
|
} from "../core/Security/ssrfGuard";
|
||||||
|
import { testS3Connection, uploadFileToS3 } from "../core/Storage/s3Archive";
|
||||||
|
import * as instrumentationHook from "../instrumentation";
|
||||||
|
|
||||||
|
const require = createRequire(import.meta.url);
|
||||||
|
const filesManager = require("../server/filesManager.js") as {
|
||||||
|
authenticateFilesRequest: (params: {
|
||||||
|
cookieHeader?: string;
|
||||||
|
dbPool?: unknown;
|
||||||
|
env?: NodeJS.ProcessEnv;
|
||||||
|
nowMs?: number;
|
||||||
|
}) => Promise<{
|
||||||
|
valid: boolean;
|
||||||
|
status?: number;
|
||||||
|
code?: string;
|
||||||
|
error?: string;
|
||||||
|
session?: { email: string; role: string; memberId: string; tokenVersion: number };
|
||||||
|
}>;
|
||||||
|
syncInMemoryAdminSessionState: (payload: Record<string, unknown>) => boolean;
|
||||||
|
clearInMemoryAdminSessionStateForTests: () => void;
|
||||||
|
};
|
||||||
|
|
||||||
|
const ingestionServer = require("../server/index.js") as {
|
||||||
|
app: (req: http.IncomingMessage, res: http.ServerResponse) => void;
|
||||||
|
setDbPoolForTests: (pool: unknown) => void;
|
||||||
|
syncSettingsFromDatabase: () => Promise<boolean>;
|
||||||
|
validateOutboundWebhookUrlAsync: (
|
||||||
|
rawUrl: string,
|
||||||
|
options?: Record<string, unknown>,
|
||||||
|
) => Promise<{ allowed: boolean; code?: string; error?: string; url?: string }>;
|
||||||
|
triggerWebhookAlert: (
|
||||||
|
payload: Record<string, unknown>,
|
||||||
|
options?: Record<string, unknown>,
|
||||||
|
) => Promise<{ dispatched: boolean; status?: number; code?: string; reason?: string; error?: string }>;
|
||||||
|
clearWebhookCooldownCacheForTests: () => void;
|
||||||
|
stopTimers: () => void;
|
||||||
|
};
|
||||||
|
|
||||||
|
const ORIGINAL_ENV = { ...process.env };
|
||||||
|
|
||||||
|
function restoreEnv(): void {
|
||||||
|
for (const key of Object.keys(process.env)) {
|
||||||
|
if (!(key in ORIGINAL_ENV)) {
|
||||||
|
delete process.env[key];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Object.assign(process.env, ORIGINAL_ENV);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function createOwnerCookie(email = "r4p1-owner@datastack.dev"): Promise<string> {
|
||||||
|
resetAdminStateForTests();
|
||||||
|
await createInitialAdmin(email, "OwnerPassword123!");
|
||||||
|
const secret = getAdminSessionSecret();
|
||||||
|
const token = await createAdminSessionToken(
|
||||||
|
email,
|
||||||
|
secret,
|
||||||
|
Date.now() + 3600_000,
|
||||||
|
1,
|
||||||
|
"owner",
|
||||||
|
"owner-1",
|
||||||
|
);
|
||||||
|
return `datastack_admin_session=${token}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
test.after(async () => {
|
||||||
|
SettingsManager.stopMaintenanceSchedulerForTests();
|
||||||
|
SDKManager.stopActivityFlushTimerForTests();
|
||||||
|
ingestionServer.stopTimers();
|
||||||
|
ingestionServer.setDbPoolForTests(null);
|
||||||
|
filesManager.clearInMemoryAdminSessionStateForTests();
|
||||||
|
await DataBufferEngine.shutdown();
|
||||||
|
clearAuditLogsForTests();
|
||||||
|
resetAdminStateForTests();
|
||||||
|
restoreEnv();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("R4-Phase-01 [AUDIT-01 / ERR-R4-01]: ssrfGuard blocks loopback, RFC1918, metadata, internal hostnames, non-HTTP(S), DNS rebinding & redirects while respecting explicit opt-in", async () => {
|
||||||
|
delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||||
|
|
||||||
|
const blockedTargets = [
|
||||||
|
"http://127.0.0.1:4000/internal/settings-sync",
|
||||||
|
"http://localhost:3000/api/health",
|
||||||
|
"http://sub.localhost/hook",
|
||||||
|
"http://[::1]:4000/",
|
||||||
|
"http://[::ffff:127.0.0.1]/secret",
|
||||||
|
"http://10.0.0.5:9000/bucket",
|
||||||
|
"http://172.16.4.2/hook",
|
||||||
|
"http://192.168.1.50/hook",
|
||||||
|
"http://0.0.0.0/",
|
||||||
|
"http://ingestion:4000/internal",
|
||||||
|
"http://postgres.internal:5432/",
|
||||||
|
"http://169.254.169.254/latest/meta-data/iam/security-credentials/",
|
||||||
|
"http://metadata.google.internal/computeMetadata/v1/",
|
||||||
|
"file:///etc/passwd",
|
||||||
|
"ftp://files.example.com/archive",
|
||||||
|
"https://user:pass@hooks.example.com/hook",
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const target of blockedTargets) {
|
||||||
|
const res = validateOutboundUrlSync(target, { fieldName: "Outbound URL" });
|
||||||
|
assert.equal(res.allowed, false, `Expected ${target} to be blocked`);
|
||||||
|
assert.equal(res.code, "SSRF_BLOCKED_HOST", `Expected SSRF_BLOCKED_HOST for ${target}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Public HTTPS host passes sync + safe DNS lookup
|
||||||
|
const okPublic = await validateOutboundUrlAsync("https://hooks.example.com/v1/alert", {
|
||||||
|
fieldName: "Webhook URL",
|
||||||
|
dnsLookup: async () => [{ address: "93.184.216.34", family: 4 }],
|
||||||
|
});
|
||||||
|
assert.equal(okPublic.allowed, true);
|
||||||
|
|
||||||
|
// DNS rebinding hostname resolving to 127.0.0.1 or 169.254.169.254 is blocked
|
||||||
|
const rebindLoopback = await validateOutboundUrlAsync("https://rebind.example.com/hook", {
|
||||||
|
fieldName: "Webhook URL",
|
||||||
|
dnsLookup: async () => [{ address: "127.0.0.1", family: 4 }],
|
||||||
|
});
|
||||||
|
assert.equal(rebindLoopback.allowed, false);
|
||||||
|
assert.equal(rebindLoopback.code, "SSRF_BLOCKED_HOST");
|
||||||
|
|
||||||
|
const rebindMeta = await validateOutboundUrlAsync("https://rebind-meta.example.com/hook", {
|
||||||
|
fieldName: "Webhook URL",
|
||||||
|
dnsLookup: async () => [{ address: "169.254.169.254", family: 4 }],
|
||||||
|
});
|
||||||
|
assert.equal(rebindMeta.allowed, false);
|
||||||
|
assert.equal(rebindMeta.code, "SSRF_BLOCKED_HOST");
|
||||||
|
|
||||||
|
// Explicit opt-in allows RFC1918/loopback/single-label internal LAN hosts, YET ALWAYS blocks cloud metadata 169.254.169.254 & metadata.google.internal
|
||||||
|
assert.equal(checkIpAddressAllowed("127.0.0.1", true).allowed, true);
|
||||||
|
assert.equal(checkIpAddressAllowed("192.168.1.25", true).allowed, true);
|
||||||
|
assert.equal(checkIpAddressAllowed("169.254.169.254", true).allowed, false);
|
||||||
|
assert.equal(checkIpAddressAllowed("fe80::1", true).allowed, false);
|
||||||
|
assert.equal(
|
||||||
|
validateOutboundUrlSync("http://minio:9000", {
|
||||||
|
allowPrivate: true,
|
||||||
|
}).allowed,
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
validateOutboundUrlSync("http://169.254.169.254/latest/meta-data/", {
|
||||||
|
allowPrivate: true,
|
||||||
|
}).allowed,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
validateOutboundUrlSync("http://metadata.google.internal/", {
|
||||||
|
allowPrivate: true,
|
||||||
|
}).allowed,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("R4-Phase-01 [AUDIT-01 / ERR-R4-01]: API routes (/api/settings, /api/settings/webhook-test, /api/storage/s3-test), Port 4000 triggerWebhookAlert & redirect:manual block SSRF", async () => {
|
||||||
|
delete process.env.DATABASE_URL;
|
||||||
|
delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||||
|
clearWebhookCooldownCacheForTests();
|
||||||
|
ingestionServer.clearWebhookCooldownCacheForTests();
|
||||||
|
|
||||||
|
const cookie = await createOwnerCookie("r4-ssrf-owner@datastack.dev");
|
||||||
|
|
||||||
|
// 1. POST /api/settings rejects private webhookUrl & private s3Endpoint with 400 SSRF_BLOCKED_HOST
|
||||||
|
const badWebhookSave = await settingsPost(
|
||||||
|
new NextRequest("http://localhost:3000/api/settings", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "content-type": "application/json", cookie },
|
||||||
|
body: JSON.stringify({ webhookUrl: "http://127.0.0.1:4000/internal/client-sync" }),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
assert.equal(badWebhookSave.status, 400);
|
||||||
|
assert.equal((await badWebhookSave.json()).code, "SSRF_BLOCKED_HOST");
|
||||||
|
|
||||||
|
const badS3Save = await settingsPost(
|
||||||
|
new NextRequest("http://localhost:3000/api/settings", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "content-type": "application/json", cookie },
|
||||||
|
body: JSON.stringify({ s3Endpoint: "http://169.254.169.254/latest/meta-data" }),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
assert.equal(badS3Save.status, 400);
|
||||||
|
assert.equal((await badS3Save.json()).code, "SSRF_BLOCKED_HOST");
|
||||||
|
|
||||||
|
// 2. POST /api/settings/webhook-test rejects loopback/metadata with 400 SSRF_BLOCKED_HOST
|
||||||
|
const badWebhookTest = await webhookTestPost(
|
||||||
|
new NextRequest("http://localhost:3000/api/settings/webhook-test", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "content-type": "application/json", cookie },
|
||||||
|
body: JSON.stringify({ webhookUrl: "http://localhost:4000/health" }),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
assert.equal(badWebhookTest.status, 400);
|
||||||
|
assert.equal((await badWebhookTest.json()).code, "SSRF_BLOCKED_HOST");
|
||||||
|
|
||||||
|
// 3. POST /api/storage/s3-test rejects internal/RFC1918 host with 400 SSRF_BLOCKED_HOST
|
||||||
|
const badS3Test = await s3TestPost(
|
||||||
|
new NextRequest("http://localhost:3000/api/storage/s3-test", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "content-type": "application/json", cookie },
|
||||||
|
body: JSON.stringify({
|
||||||
|
s3Endpoint: "http://10.0.0.12:9000",
|
||||||
|
s3Bucket: "test-bucket",
|
||||||
|
s3AccessKeyId: "AKIA_TEST",
|
||||||
|
s3SecretAccessKey: "secret_key_test",
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
assert.equal(badS3Test.status, 400);
|
||||||
|
assert.equal((await badS3Test.json()).code, "SSRF_BLOCKED_HOST");
|
||||||
|
|
||||||
|
// 4. Port 4000 triggerWebhookAlert & validateOutboundWebhookUrlAsync block loopback & metadata
|
||||||
|
const p4Blocked = await ingestionServer.triggerWebhookAlert(
|
||||||
|
{
|
||||||
|
event: "CLIENT_IP_FORBIDDEN",
|
||||||
|
severity: "warning",
|
||||||
|
title: "Probe",
|
||||||
|
message: "Test",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
enableWebhookAlerts: true,
|
||||||
|
webhookUrl: "http://127.0.0.1:4000/internal/client-sync",
|
||||||
|
bypassCooldown: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert.equal(p4Blocked.dispatched, false);
|
||||||
|
assert.equal(p4Blocked.code, "SSRF_BLOCKED_HOST");
|
||||||
|
|
||||||
|
// 5. redirect: 'manual' blocks HTTP 302 redirects across webhookNotifier, S3, and Port 4000
|
||||||
|
let targetHitAfterRedirect = false;
|
||||||
|
const redirectServer = http.createServer((req, res) => {
|
||||||
|
if (req.url && req.url.startsWith("/bounce")) {
|
||||||
|
res.writeHead(302, { Location: "http://169.254.169.254/latest/meta-data/" });
|
||||||
|
res.end();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
targetHitAfterRedirect = true;
|
||||||
|
res.writeHead(200);
|
||||||
|
res.end("ok");
|
||||||
|
});
|
||||||
|
await new Promise<void>((resolve) => redirectServer.listen(0, "127.0.0.1", resolve));
|
||||||
|
const addr = redirectServer.address() as AddressInfo;
|
||||||
|
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "datastack-r4-ssrf-redir-"));
|
||||||
|
const gzPath = path.join(tmpDir, "2026-11-20.csv.gz");
|
||||||
|
await fs.writeFile(gzPath, zlib.gzipSync(Buffer.from("clientId,eventId,timestamp,payload\n")));
|
||||||
|
|
||||||
|
try {
|
||||||
|
process.env.DATASTACK_ALLOW_PRIVATE_URLS = "true";
|
||||||
|
|
||||||
|
const whRedirect = await dispatchWebhookAlertAsync(
|
||||||
|
{
|
||||||
|
event: "TEST_ALERT",
|
||||||
|
severity: "info",
|
||||||
|
title: "Redirect check",
|
||||||
|
message: "Must not follow 302",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
enabled: true,
|
||||||
|
webhookUrl: `http://127.0.0.1:${addr.port}/bounce`,
|
||||||
|
webhookSecret: "",
|
||||||
|
cooldownSec: 0,
|
||||||
|
},
|
||||||
|
{ bypassCooldown: true },
|
||||||
|
);
|
||||||
|
assert.equal(whRedirect.ok, false);
|
||||||
|
assert.equal(whRedirect.code, "SSRF_BLOCKED_HOST");
|
||||||
|
|
||||||
|
const s3ProbeRedir = await testS3Connection({
|
||||||
|
endpoint: `http://127.0.0.1:${addr.port}/bounce`,
|
||||||
|
region: "auto",
|
||||||
|
bucket: "bkt",
|
||||||
|
prefix: "",
|
||||||
|
accessKeyId: "AKIA",
|
||||||
|
secretAccessKey: "SEC",
|
||||||
|
});
|
||||||
|
assert.equal(s3ProbeRedir.ok, false);
|
||||||
|
assert.equal(s3ProbeRedir.code, "SSRF_BLOCKED_HOST");
|
||||||
|
|
||||||
|
const s3PutRedir = await uploadFileToS3(
|
||||||
|
{
|
||||||
|
endpoint: `http://127.0.0.1:${addr.port}/bounce`,
|
||||||
|
region: "auto",
|
||||||
|
bucket: "bkt",
|
||||||
|
prefix: "",
|
||||||
|
accessKeyId: "AKIA",
|
||||||
|
secretAccessKey: "SEC",
|
||||||
|
},
|
||||||
|
gzPath,
|
||||||
|
"c/2026-11-20.csv.gz",
|
||||||
|
);
|
||||||
|
assert.equal(s3PutRedir.ok, false);
|
||||||
|
assert.equal(s3PutRedir.code, "SSRF_BLOCKED_HOST");
|
||||||
|
|
||||||
|
const p4Redirect = await ingestionServer.triggerWebhookAlert(
|
||||||
|
{
|
||||||
|
event: "DISK_SPACE_LOW",
|
||||||
|
severity: "warning",
|
||||||
|
title: "Redirect check",
|
||||||
|
message: "Must not follow 302",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
enableWebhookAlerts: true,
|
||||||
|
webhookUrl: `http://127.0.0.1:${addr.port}/bounce`,
|
||||||
|
allowPrivate: true,
|
||||||
|
bypassCooldown: true,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert.equal(p4Redirect.dispatched, false);
|
||||||
|
assert.equal(p4Redirect.code, "SSRF_BLOCKED_HOST");
|
||||||
|
assert.equal(targetHitAfterRedirect, false);
|
||||||
|
} finally {
|
||||||
|
delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||||
|
await new Promise<void>((resolve) => redirectServer.close(() => resolve()));
|
||||||
|
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("R4-Phase-01 [AUDIT-03 / ERR-R4-02]: Port 4000 ensureDatabaseSchema creates all 7 tables including admin_team_members and admin_audit_logs", async () => {
|
||||||
|
const executedSql: string[] = [];
|
||||||
|
const mockPool = {
|
||||||
|
async query(sql: string) {
|
||||||
|
executedSql.push(sql);
|
||||||
|
if (sql.includes("FROM admin_settings")) {
|
||||||
|
return { rowCount: 0, rows: [] };
|
||||||
|
}
|
||||||
|
return { rowCount: 0, rows: [] };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
ingestionServer.setDbPoolForTests(mockPool);
|
||||||
|
const synced = await ingestionServer.syncSettingsFromDatabase();
|
||||||
|
assert.equal(synced, true);
|
||||||
|
|
||||||
|
const createStatements = executedSql.filter((s) => /CREATE TABLE IF NOT EXISTS/i.test(s));
|
||||||
|
assert.equal(createStatements.length, 7, "Port 4000 must create all 7 tables");
|
||||||
|
const joined = createStatements.join("\n");
|
||||||
|
for (const tableName of [
|
||||||
|
"client_sdks",
|
||||||
|
"admin_settings",
|
||||||
|
"deleted_clients",
|
||||||
|
"admin_users",
|
||||||
|
"admin_login_attempts",
|
||||||
|
"admin_team_members",
|
||||||
|
"admin_audit_logs",
|
||||||
|
]) {
|
||||||
|
assert.ok(
|
||||||
|
joined.includes(tableName),
|
||||||
|
`Port 4000 schema missing table ${tableName}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
ingestionServer.setDbPoolForTests(null);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("R4-Phase-01 [AUDIT-04 / ERR-R4-03]: Next.js instrumentation.register() boots maintenance scheduler unattended on nodejs runtime and survives unreachable DB", async () => {
|
||||||
|
delete process.env.DATABASE_URL;
|
||||||
|
const prevDisable = process.env.DATASTACK_DISABLE_LISTEN;
|
||||||
|
const prevRuntime = process.env.NEXT_RUNTIME;
|
||||||
|
SettingsManager.stopMaintenanceSchedulerForTests();
|
||||||
|
|
||||||
|
try {
|
||||||
|
// 1. Edge runtime does NOT start Node maintenance scheduler
|
||||||
|
process.env.NEXT_RUNTIME = "edge";
|
||||||
|
delete process.env.DATASTACK_DISABLE_LISTEN;
|
||||||
|
await instrumentationHook.register();
|
||||||
|
assert.equal(SettingsManager.isMaintenanceSchedulerActive(), false);
|
||||||
|
|
||||||
|
// 2. Node.js runtime starts maintenance scheduler automatically on boot
|
||||||
|
process.env.NEXT_RUNTIME = "nodejs";
|
||||||
|
await instrumentationHook.register();
|
||||||
|
assert.equal(SettingsManager.isMaintenanceSchedulerActive(), true);
|
||||||
|
|
||||||
|
// 3. Even if DB fails during boot hydration, bootstrapMaintenanceSchedulerOnBoot catches error cleanly without crashing
|
||||||
|
SettingsManager.stopMaintenanceSchedulerForTests();
|
||||||
|
process.env.DATABASE_URL = "postgresql://user:pass@127.0.0.1:1/unreachable";
|
||||||
|
const bootRes = await SettingsManager.bootstrapMaintenanceSchedulerOnBoot();
|
||||||
|
assert.equal(bootRes.schedulerActive, true);
|
||||||
|
assert.equal(bootRes.hydrated, false);
|
||||||
|
} finally {
|
||||||
|
SettingsManager.stopMaintenanceSchedulerForTests();
|
||||||
|
delete process.env.DATABASE_URL;
|
||||||
|
if (prevDisable === undefined) delete process.env.DATASTACK_DISABLE_LISTEN;
|
||||||
|
else process.env.DATASTACK_DISABLE_LISTEN = prevDisable;
|
||||||
|
if (prevRuntime === undefined) delete process.env.NEXT_RUNTIME;
|
||||||
|
else process.env.NEXT_RUNTIME = prevRuntime;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("R4-Phase-01 [AUDIT-09 / ERR-R4-04 & ERR-R4-05]: Fail-loud audit DB degraded warnings + no-DB Files Manager session revocation sync", async () => {
|
||||||
|
delete process.env.DATABASE_URL;
|
||||||
|
clearAuditLogsForTests();
|
||||||
|
filesManager.clearInMemoryAdminSessionStateForTests();
|
||||||
|
|
||||||
|
const cookie = await createOwnerCookie("r4-audit-owner@datastack.dev");
|
||||||
|
const globals = globalThis as typeof globalThis & {
|
||||||
|
__dataStackCorePool?: unknown;
|
||||||
|
};
|
||||||
|
const prevPool = globals.__dataStackCorePool;
|
||||||
|
|
||||||
|
// 1. Simulate PostgreSQL failure on audit write + read -> must log console.warn and expose degraded metadata & header
|
||||||
|
const failingPool = {
|
||||||
|
async query(sql: string) {
|
||||||
|
if (/CREATE TABLE|ALTER TABLE/i.test(sql)) {
|
||||||
|
return { rowCount: 0, rows: [] };
|
||||||
|
}
|
||||||
|
if (sql.includes("FROM admin_users")) {
|
||||||
|
return {
|
||||||
|
rowCount: 1,
|
||||||
|
rows: [
|
||||||
|
{
|
||||||
|
email: "r4-audit-owner@datastack.dev",
|
||||||
|
created_at: new Date(),
|
||||||
|
token_version: 1,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (sql.includes("admin_audit_logs")) {
|
||||||
|
throw new Error("simulated PG outage");
|
||||||
|
}
|
||||||
|
return { rowCount: 0, rows: [] };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
process.env.DATABASE_URL = "postgresql://mock:mock@127.0.0.1:5432/mock";
|
||||||
|
globals.__dataStackCorePool = failingPool;
|
||||||
|
const warnings: string[] = [];
|
||||||
|
const origWarn = console.warn;
|
||||||
|
console.warn = (...args: unknown[]) => {
|
||||||
|
warnings.push(args.map((a) => String(a)).join(" "));
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
await recordAdminAuditLogAsync({
|
||||||
|
actorEmail: "r4-audit-owner@datastack.dev",
|
||||||
|
actorRole: "owner",
|
||||||
|
action: "SETTINGS_UPDATED",
|
||||||
|
target: "admin_settings",
|
||||||
|
});
|
||||||
|
const listResult = await listAdminAuditLogsWithStatusAsync({ limit: 10 });
|
||||||
|
assert.equal(listResult.storageMode, "memory");
|
||||||
|
assert.equal(listResult.degraded, true);
|
||||||
|
assert.equal(listResult.code, "AUDIT_STORAGE_DEGRADED");
|
||||||
|
assert.match(listResult.warning || "", /PostgreSQL/i);
|
||||||
|
assert.ok(
|
||||||
|
warnings.some((w) => w.includes("[DataStack Audit]")),
|
||||||
|
"Expected console.warn fail-loud audit log",
|
||||||
|
);
|
||||||
|
|
||||||
|
// Verify GET /api/audit surfaces degraded status & header
|
||||||
|
const apiRes = await auditGet(
|
||||||
|
new NextRequest("http://localhost:3000/api/audit?limit=10", {
|
||||||
|
headers: { cookie },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
assert.equal(apiRes.status, 200);
|
||||||
|
assert.equal(apiRes.headers.get("X-DataStack-Audit-Degraded"), "1");
|
||||||
|
const apiJson = await apiRes.json();
|
||||||
|
assert.equal(apiJson.degraded, true);
|
||||||
|
assert.equal(apiJson.code, "AUDIT_STORAGE_DEGRADED");
|
||||||
|
assert.equal(apiJson.storageMode, "memory");
|
||||||
|
} finally {
|
||||||
|
console.warn = origWarn;
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
delete process.env.DATABASE_URL;
|
||||||
|
globals.__dataStackCorePool = prevPool;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Verify no-DB (!dbPool) Files Manager session revocation via Port 4000 /internal/admin-session-sync bridge
|
||||||
|
const testBridgeServer = http.createServer(ingestionServer.app);
|
||||||
|
await new Promise<void>((resolve) => testBridgeServer.listen(0, "127.0.0.1", resolve));
|
||||||
|
const bridgePort = (testBridgeServer.address() as AddressInfo).port;
|
||||||
|
process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${bridgePort}`;
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Reject unauthenticated POST /internal/admin-session-sync
|
||||||
|
const unauthSync = await fetch(
|
||||||
|
`http://127.0.0.1:${bridgePort}/internal/admin-session-sync`,
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({ memberId: "owner-1", minTokenVersion: 99 }),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert.equal(unauthSync.status, 401);
|
||||||
|
|
||||||
|
// Owner session v1 works before logout/revocation
|
||||||
|
const secret = getAdminSessionSecret();
|
||||||
|
const ownerTokenV1 = await createAdminSessionToken(
|
||||||
|
"r4-audit-owner@datastack.dev",
|
||||||
|
secret,
|
||||||
|
Date.now() + 3600_000,
|
||||||
|
1,
|
||||||
|
"owner",
|
||||||
|
"owner-1",
|
||||||
|
);
|
||||||
|
const beforeLogout = await filesManager.authenticateFilesRequest({
|
||||||
|
cookieHeader: `datastack_admin_session=${ownerTokenV1}`,
|
||||||
|
dbPool: null,
|
||||||
|
});
|
||||||
|
assert.equal(beforeLogout.valid, true);
|
||||||
|
|
||||||
|
// Calling revokeAdminSessions syncs minTokenVersion=2 to Port 4000 bridge -> v1 Files Manager cookie is rejected
|
||||||
|
const nextVer = await revokeAdminSessions({
|
||||||
|
memberId: "owner-1",
|
||||||
|
email: "r4-audit-owner@datastack.dev",
|
||||||
|
});
|
||||||
|
assert.ok(nextVer >= 2);
|
||||||
|
const afterLogout = await filesManager.authenticateFilesRequest({
|
||||||
|
cookieHeader: `datastack_admin_session=${ownerTokenV1}`,
|
||||||
|
dbPool: null,
|
||||||
|
});
|
||||||
|
assert.equal(afterLogout.valid, false);
|
||||||
|
assert.equal(afterLogout.status, 401);
|
||||||
|
assert.equal(afterLogout.code, "SESSION_REVOKED");
|
||||||
|
|
||||||
|
// Team member disable/delete also revokes no-DB Files Manager session immediately
|
||||||
|
const opMember = await createAdminTeamMemberAsync({
|
||||||
|
email: "r4-op@datastack.dev",
|
||||||
|
password: "OperatorPassword123!",
|
||||||
|
role: "operator",
|
||||||
|
});
|
||||||
|
const opTokenV1 = await createAdminSessionToken(
|
||||||
|
opMember.email,
|
||||||
|
secret,
|
||||||
|
Date.now() + 3600_000,
|
||||||
|
opMember.tokenVersion,
|
||||||
|
opMember.role,
|
||||||
|
opMember.id,
|
||||||
|
);
|
||||||
|
const opBefore = await filesManager.authenticateFilesRequest({
|
||||||
|
cookieHeader: `datastack_admin_session=${opTokenV1}`,
|
||||||
|
dbPool: null,
|
||||||
|
});
|
||||||
|
assert.equal(opBefore.valid, true);
|
||||||
|
|
||||||
|
await updateAdminTeamMemberAsync(opMember.id, { status: "disabled" });
|
||||||
|
const opAfterDisable = await filesManager.authenticateFilesRequest({
|
||||||
|
cookieHeader: `datastack_admin_session=${opTokenV1}`,
|
||||||
|
dbPool: null,
|
||||||
|
});
|
||||||
|
assert.equal(opAfterDisable.valid, false);
|
||||||
|
assert.equal(opAfterDisable.code, "SESSION_REVOKED");
|
||||||
|
|
||||||
|
await deleteAdminTeamMemberAsync(opMember.id);
|
||||||
|
|
||||||
|
// Also verify authenticated /internal/admin-session-sync endpoint returns 200 & synced=true
|
||||||
|
const authSyncRes = await fetch(
|
||||||
|
`http://127.0.0.1:${bridgePort}/internal/admin-session-sync`,
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
...getInternalIngestionHeaders(),
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
memberId: "adm_manual_test",
|
||||||
|
email: "manual@datastack.dev",
|
||||||
|
minTokenVersion: 5,
|
||||||
|
disabled: true,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert.equal(authSyncRes.status, 200);
|
||||||
|
const authSyncJson = (await authSyncRes.json()) as { success: boolean; synced: boolean };
|
||||||
|
assert.equal(authSyncJson.success, true);
|
||||||
|
assert.equal(authSyncJson.synced, true);
|
||||||
|
} finally {
|
||||||
|
delete process.env.INGESTION_INTERNAL_URL;
|
||||||
|
filesManager.clearInMemoryAdminSessionStateForTests();
|
||||||
|
await new Promise<void>((resolve) => testBridgeServer.close(() => resolve()));
|
||||||
|
}
|
||||||
|
});
|
||||||
@@ -0,0 +1,777 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import crypto from "node:crypto";
|
||||||
|
import fs from "node:fs";
|
||||||
|
import fsp from "node:fs/promises";
|
||||||
|
import http from "node:http";
|
||||||
|
import { createRequire } from "node:module";
|
||||||
|
import type { AddressInfo } from "node:net";
|
||||||
|
import os from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import test from "node:test";
|
||||||
|
import zlib from "node:zlib";
|
||||||
|
import { NextRequest } from "next/server";
|
||||||
|
|
||||||
|
process.env.DATASTACK_DISABLE_LISTEN = "1";
|
||||||
|
|
||||||
|
import { GET as auditGet } from "../app/api/audit/route";
|
||||||
|
import { PATCH as clientPatch } from "../app/api/clients/[id]/route";
|
||||||
|
import { GET as dataGet } from "../app/api/data/route";
|
||||||
|
import { POST as eventsPost } from "../app/api/events/route";
|
||||||
|
import { GET as exportGet } from "../app/api/export/route";
|
||||||
|
import { SettingsManager } from "../core/adminSettings/settingsManager";
|
||||||
|
import {
|
||||||
|
createAdminSessionToken,
|
||||||
|
getAdminSessionSecret,
|
||||||
|
} from "../core/auth/adminSession";
|
||||||
|
import {
|
||||||
|
createInitialAdmin,
|
||||||
|
resetAdminStateForTests,
|
||||||
|
} from "../core/auth/adminUsers";
|
||||||
|
import {
|
||||||
|
clearAuditLogsForTests,
|
||||||
|
listAdminAuditLogsWithStatusAsync,
|
||||||
|
} from "../core/auth/auditLog";
|
||||||
|
import { DataBufferEngine } from "../core/data/buffer";
|
||||||
|
import { SDKManager } from "../core/sdk/sdkManager";
|
||||||
|
import { getInternalIngestionHeaders } from "../core/Security/security";
|
||||||
|
import { CSVStorageManager } from "../core/Storage/csvStorage";
|
||||||
|
import { uploadFileToS3 } from "../core/Storage/s3Archive";
|
||||||
|
|
||||||
|
const require = createRequire(import.meta.url);
|
||||||
|
const express = require("express");
|
||||||
|
const { Server } = require("socket.io");
|
||||||
|
const { io: ioClient } = require("socket.io-client");
|
||||||
|
const { setupSocketHandlers } = require("../server/socket/handlers");
|
||||||
|
const filesManager = require("../server/filesManager.js") as {
|
||||||
|
createFilesSecurityMiddleware: (opts: {
|
||||||
|
rootDir: string;
|
||||||
|
getDbPool?: () => unknown;
|
||||||
|
env?: NodeJS.ProcessEnv;
|
||||||
|
}) => (req: http.IncomingMessage, res: http.ServerResponse, next: () => void) => void;
|
||||||
|
flushPendingFilesAuditLogsForTests: () => Promise<void>;
|
||||||
|
clearInMemoryAdminSessionStateForTests: () => void;
|
||||||
|
};
|
||||||
|
|
||||||
|
const ingestionServer = require("../server/index.js") as {
|
||||||
|
app: (req: http.IncomingMessage, res: http.ServerResponse) => void;
|
||||||
|
io: {
|
||||||
|
on: (ev: string, cb: (socket: unknown) => void) => void;
|
||||||
|
close: (cb?: () => void) => void;
|
||||||
|
};
|
||||||
|
server: http.Server;
|
||||||
|
stopTimers: () => void;
|
||||||
|
setDbPoolForTests: (pool: unknown) => void;
|
||||||
|
writer: { baseDir: string };
|
||||||
|
};
|
||||||
|
|
||||||
|
const ORIGINAL_ENV = { ...process.env };
|
||||||
|
const ORIGINAL_STORAGE_DIR = CSVStorageManager.baseStorageDir;
|
||||||
|
|
||||||
|
function restoreEnv(): void {
|
||||||
|
for (const key of Object.keys(process.env)) {
|
||||||
|
if (!(key in ORIGINAL_ENV)) {
|
||||||
|
delete process.env[key];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Object.assign(process.env, ORIGINAL_ENV);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function createOwnerCookie(
|
||||||
|
email = "r4p2-owner@datastack.dev",
|
||||||
|
): Promise<string> {
|
||||||
|
resetAdminStateForTests();
|
||||||
|
await createInitialAdmin(email, "OwnerPassword123!");
|
||||||
|
const secret = getAdminSessionSecret();
|
||||||
|
const token = await createAdminSessionToken(
|
||||||
|
email,
|
||||||
|
secret,
|
||||||
|
Date.now() + 3600_000,
|
||||||
|
1,
|
||||||
|
"owner",
|
||||||
|
"owner-1",
|
||||||
|
);
|
||||||
|
return `datastack_admin_session=${token}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
test.after(async () => {
|
||||||
|
CSVStorageManager.baseStorageDir = ORIGINAL_STORAGE_DIR;
|
||||||
|
CSVStorageManager.clearRowCountCache();
|
||||||
|
CSVStorageManager.clearQuotaCaches();
|
||||||
|
SettingsManager.stopMaintenanceSchedulerForTests();
|
||||||
|
SDKManager.stopActivityFlushTimerForTests();
|
||||||
|
ingestionServer.stopTimers();
|
||||||
|
ingestionServer.setDbPoolForTests(null);
|
||||||
|
filesManager.clearInMemoryAdminSessionStateForTests();
|
||||||
|
await DataBufferEngine.shutdown();
|
||||||
|
clearAuditLogsForTests();
|
||||||
|
resetAdminStateForTests();
|
||||||
|
restoreEnv();
|
||||||
|
});
|
||||||
|
|
||||||
|
test("R4-Phase-02 AUDIT-05 / ERR-R4-06: HTTP & Socket.IO enforce DATASTACK_MAX_BATCH_ITEMS cap", async () => {
|
||||||
|
const prevStorage = CSVStorageManager.baseStorageDir;
|
||||||
|
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "ds-r4p2-batch-"));
|
||||||
|
CSVStorageManager.baseStorageDir = tmpDir;
|
||||||
|
CSVStorageManager.clearRowCountCache();
|
||||||
|
CSVStorageManager.clearQuotaCaches();
|
||||||
|
process.env.DATASTACK_MAX_BATCH_ITEMS = "3";
|
||||||
|
|
||||||
|
let httpServer: http.Server | null = null;
|
||||||
|
let io: InstanceType<typeof Server> | null = null;
|
||||||
|
let socket: ReturnType<typeof ioClient> | null = null;
|
||||||
|
|
||||||
|
const runId = `${Date.now()}_${crypto.randomBytes(3).toString("hex")}`;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const client = await SDKManager.registerClientAsync({
|
||||||
|
clientId: "r4p2-batch-client",
|
||||||
|
name: "Batch Limit Client",
|
||||||
|
rateLimit: 500,
|
||||||
|
});
|
||||||
|
|
||||||
|
// 1. HTTP POST /api/events rejects 4-item array before per-item checks
|
||||||
|
const overBatchReq = new NextRequest("http://127.0.0.1:3000/api/events", {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
Authorization: `Bearer ${client.token}`,
|
||||||
|
},
|
||||||
|
body: JSON.stringify([
|
||||||
|
{ eventId: `${runId}-b1`, data: { step: 1 } },
|
||||||
|
{ eventId: `${runId}-b2`, data: { step: 2 } },
|
||||||
|
{ eventId: `${runId}-b3`, data: { step: 3 } },
|
||||||
|
{ eventId: `${runId}-b4`, data: { step: 4 } },
|
||||||
|
]),
|
||||||
|
});
|
||||||
|
const overRes = await eventsPost(overBatchReq);
|
||||||
|
assert.equal(overRes.status, 400);
|
||||||
|
const overJson = await overRes.json();
|
||||||
|
assert.equal(overJson.success, false);
|
||||||
|
assert.equal(overJson.code, "BATCH_LIMIT_EXCEEDED");
|
||||||
|
assert.equal(overJson.rejected, 4);
|
||||||
|
assert.equal(overJson.maxBatchItems, 3);
|
||||||
|
|
||||||
|
// 2. HTTP POST /api/events accepts 3-item records array
|
||||||
|
const okBatchReq = new NextRequest("http://127.0.0.1:3000/api/events", {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
Authorization: `Bearer ${client.token}`,
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
records: [
|
||||||
|
{ eventId: `${runId}-b1`, data: { step: 1 } },
|
||||||
|
{ eventId: `${runId}-b2`, data: { step: 2 } },
|
||||||
|
{ eventId: `${runId}-b3`, data: { step: 3 } },
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
const okRes = await eventsPost(okBatchReq);
|
||||||
|
assert.equal(okRes.status, 200);
|
||||||
|
const okJson = await okRes.json();
|
||||||
|
assert.equal(okJson.success, true);
|
||||||
|
assert.equal(okJson.accepted, 3);
|
||||||
|
assert.equal(okJson.rejected, 0);
|
||||||
|
|
||||||
|
// 3. Socket.IO batch_sync rejects 4-item array and accepts 3-item array
|
||||||
|
httpServer = http.createServer();
|
||||||
|
io = new Server(httpServer);
|
||||||
|
const bufferQueue: unknown[] = [];
|
||||||
|
setupSocketHandlers(io, bufferQueue, null, {
|
||||||
|
getRuntimeConfig: () => ({
|
||||||
|
jwtSecret: "datastack-jwt-default-secret-key-32-chars-long",
|
||||||
|
maxEventsPerSec: 500,
|
||||||
|
maxPayloadSizeBytes: 100 * 1024,
|
||||||
|
enableIdempotency: true,
|
||||||
|
}),
|
||||||
|
verifyClientInDb: async () => ({ allowed: true, rateLimit: 500 }),
|
||||||
|
});
|
||||||
|
|
||||||
|
await new Promise<void>((resolve) => httpServer!.listen(0, "127.0.0.1", resolve));
|
||||||
|
const port = (httpServer.address() as AddressInfo).port;
|
||||||
|
|
||||||
|
socket = ioClient(`http://127.0.0.1:${port}`, {
|
||||||
|
auth: { token: client.token },
|
||||||
|
transports: ["websocket"],
|
||||||
|
});
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
socket!.on("connect", () => resolve());
|
||||||
|
socket!.on("connect_error", reject);
|
||||||
|
});
|
||||||
|
|
||||||
|
const overAck = await new Promise<Record<string, unknown>>((resolve) => {
|
||||||
|
socket!.emit(
|
||||||
|
"batch_sync",
|
||||||
|
[
|
||||||
|
{ eventId: "s1", data: { i: 1 } },
|
||||||
|
{ eventId: "s2", data: { i: 2 } },
|
||||||
|
{ eventId: "s3", data: { i: 3 } },
|
||||||
|
{ eventId: "s4", data: { i: 4 } },
|
||||||
|
],
|
||||||
|
resolve,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
assert.equal(overAck.success, false);
|
||||||
|
assert.equal(overAck.code, "BATCH_LIMIT_EXCEEDED");
|
||||||
|
assert.equal(overAck.rejected, 4);
|
||||||
|
assert.equal(overAck.maxBatchItems, 3);
|
||||||
|
assert.equal(bufferQueue.length, 0);
|
||||||
|
|
||||||
|
const okAck = await new Promise<Record<string, unknown>>((resolve) => {
|
||||||
|
socket!.emit(
|
||||||
|
"batch_sync",
|
||||||
|
[
|
||||||
|
{ eventId: `${runId}-s1`, data: { i: 1 } },
|
||||||
|
{ eventId: `${runId}-s2`, data: { i: 2 } },
|
||||||
|
{ eventId: `${runId}-s3`, data: { i: 3 } },
|
||||||
|
],
|
||||||
|
resolve,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
assert.equal(okAck.success, true);
|
||||||
|
assert.equal(okAck.processed, 3);
|
||||||
|
assert.equal(okAck.rejected, 0);
|
||||||
|
assert.equal(bufferQueue.length, 3);
|
||||||
|
|
||||||
|
// 4. Lower bound clamping to 1 when DATASTACK_MAX_BATCH_ITEMS=0
|
||||||
|
process.env.DATASTACK_MAX_BATCH_ITEMS = "0";
|
||||||
|
const clampAck = await new Promise<Record<string, unknown>>((resolve) => {
|
||||||
|
socket!.emit(
|
||||||
|
"batch_sync",
|
||||||
|
[
|
||||||
|
{ eventId: `${runId}-c1`, data: { i: 1 } },
|
||||||
|
{ eventId: `${runId}-c2`, data: { i: 2 } },
|
||||||
|
],
|
||||||
|
resolve,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
assert.equal(clampAck.success, false);
|
||||||
|
assert.equal(clampAck.code, "BATCH_LIMIT_EXCEEDED");
|
||||||
|
assert.equal(clampAck.maxBatchItems, 1);
|
||||||
|
} finally {
|
||||||
|
delete process.env.DATASTACK_MAX_BATCH_ITEMS;
|
||||||
|
if (socket) socket.disconnect();
|
||||||
|
if (io) await new Promise<void>((resolve) => io!.close(() => resolve()));
|
||||||
|
if (httpServer) httpServer.close();
|
||||||
|
await DataBufferEngine.shutdown();
|
||||||
|
await SDKManager.deleteClientWithStatsAsync("r4p2-batch-client", {
|
||||||
|
purgeFiles: false,
|
||||||
|
}).catch(() => undefined);
|
||||||
|
CSVStorageManager.baseStorageDir = prevStorage;
|
||||||
|
CSVStorageManager.clearRowCountCache();
|
||||||
|
CSVStorageManager.clearQuotaCaches();
|
||||||
|
await fsp.rm(tmpDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("R4-Phase-02 AUDIT-06: Governance audit logging for /api/export, Files Manager HTTP mutations (RAM + PG), and UI filters", async () => {
|
||||||
|
const prevStorage = CSVStorageManager.baseStorageDir;
|
||||||
|
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "ds-r4p2-audit-"));
|
||||||
|
CSVStorageManager.baseStorageDir = tmpDir;
|
||||||
|
CSVStorageManager.clearRowCountCache();
|
||||||
|
CSVStorageManager.clearQuotaCaches();
|
||||||
|
clearAuditLogsForTests();
|
||||||
|
|
||||||
|
let mockServer: http.Server | null = null;
|
||||||
|
let pgServer: http.Server | null = null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const cookie = await createOwnerCookie("audit-owner@datastack.dev");
|
||||||
|
const partitionDir = path.join(tmpDir, "export-client");
|
||||||
|
await fsp.mkdir(partitionDir, { recursive: true });
|
||||||
|
await fsp.writeFile(
|
||||||
|
path.join(partitionDir, "2026-11-20.csv"),
|
||||||
|
'clientId,eventId,timestamp,payload\n"export-client","evt-1","2026-11-20T10:00:00.000Z","{""ok"":true}"\n',
|
||||||
|
"utf8",
|
||||||
|
);
|
||||||
|
|
||||||
|
// 1. Filtered-range export audit log
|
||||||
|
const rangeRes = await exportGet(
|
||||||
|
new NextRequest(
|
||||||
|
"http://127.0.0.1:3000/api/export?clientId=export-client&format=csv&flush=false",
|
||||||
|
{ headers: { Cookie: cookie } },
|
||||||
|
),
|
||||||
|
);
|
||||||
|
assert.equal(rangeRes.status, 200);
|
||||||
|
await rangeRes.text();
|
||||||
|
|
||||||
|
// 2. Single-file export audit log
|
||||||
|
const singleRes = await exportGet(
|
||||||
|
new NextRequest(
|
||||||
|
"http://127.0.0.1:3000/api/export?clientId=export-client&filename=2026-11-20.csv&format=json&flush=false",
|
||||||
|
{ headers: { Cookie: cookie } },
|
||||||
|
),
|
||||||
|
);
|
||||||
|
assert.equal(singleRes.status, 200);
|
||||||
|
await singleRes.text();
|
||||||
|
|
||||||
|
// Verify /api/audit filters DATA_EXPORT_DOWNLOADED
|
||||||
|
const auditRes = await auditGet(
|
||||||
|
new NextRequest(
|
||||||
|
"http://127.0.0.1:3000/api/audit?action=DATA_EXPORT_DOWNLOADED&limit=10",
|
||||||
|
{ headers: { Cookie: cookie } },
|
||||||
|
),
|
||||||
|
);
|
||||||
|
assert.equal(auditRes.status, 200);
|
||||||
|
const auditJson = await auditRes.json();
|
||||||
|
assert.equal(auditJson.logs.length, 2);
|
||||||
|
assert.equal(auditJson.logs[0].action, "DATA_EXPORT_DOWNLOADED");
|
||||||
|
assert.equal(auditJson.logs[0].metadata.mode, "single_file");
|
||||||
|
assert.equal(auditJson.logs[1].metadata.mode, "filtered_range");
|
||||||
|
|
||||||
|
// 3. Files Manager HTTP mutations -> RAM fallback
|
||||||
|
clearAuditLogsForTests();
|
||||||
|
const ramApp = express();
|
||||||
|
ramApp.use(
|
||||||
|
"/vcon/files-engine",
|
||||||
|
filesManager.createFilesSecurityMiddleware({
|
||||||
|
rootDir: tmpDir,
|
||||||
|
getDbPool: () => null,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
ramApp.put("/vcon/files-engine/api/v1/fs/sample.csv", (_req: express.Request, res: express.Response) => {
|
||||||
|
res.status(200).json({ ok: true });
|
||||||
|
});
|
||||||
|
ramApp.put("/vcon/files-engine/api/v1/rename", (_req: express.Request, res: express.Response) => {
|
||||||
|
res.status(200).json({ ok: true });
|
||||||
|
});
|
||||||
|
ramApp.delete("/vcon/files-engine/api/v1/fs/sample.csv", (_req: express.Request, res: express.Response) => {
|
||||||
|
res.status(403).json({ ok: false }); // non-2xx should NOT emit mutation audit
|
||||||
|
});
|
||||||
|
|
||||||
|
mockServer = http.createServer(ramApp);
|
||||||
|
await new Promise<void>((resolve) => mockServer!.listen(0, "127.0.0.1", resolve));
|
||||||
|
const ramPort = (mockServer.address() as AddressInfo).port;
|
||||||
|
const originHeaders = {
|
||||||
|
Cookie: cookie,
|
||||||
|
Origin: `http://127.0.0.1:${ramPort}`,
|
||||||
|
Host: `127.0.0.1:${ramPort}`,
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
};
|
||||||
|
|
||||||
|
const putRes = await fetch(
|
||||||
|
`http://127.0.0.1:${ramPort}/vcon/files-engine/api/v1/fs/sample.csv`,
|
||||||
|
{ method: "PUT", headers: originHeaders, body: "hello" },
|
||||||
|
);
|
||||||
|
assert.equal(putRes.status, 200);
|
||||||
|
|
||||||
|
const renameRes = await fetch(
|
||||||
|
`http://127.0.0.1:${ramPort}/vcon/files-engine/api/v1/rename`,
|
||||||
|
{
|
||||||
|
method: "PUT",
|
||||||
|
headers: originHeaders,
|
||||||
|
body: JSON.stringify({ from: "/sample.csv", to: "/renamed.csv" }),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert.equal(renameRes.status, 200);
|
||||||
|
|
||||||
|
const failDelRes = await fetch(
|
||||||
|
`http://127.0.0.1:${ramPort}/vcon/files-engine/api/v1/fs/sample.csv`,
|
||||||
|
{ method: "DELETE", headers: originHeaders },
|
||||||
|
);
|
||||||
|
assert.equal(failDelRes.status, 403);
|
||||||
|
|
||||||
|
await filesManager.flushPendingFilesAuditLogsForTests();
|
||||||
|
const { logs: fmRamLogs } = await listAdminAuditLogsWithStatusAsync({
|
||||||
|
action: "FILES_MANAGER_MUTATION",
|
||||||
|
});
|
||||||
|
assert.equal(fmRamLogs.length, 2);
|
||||||
|
assert.equal(fmRamLogs[0].target, "/renamed.csv");
|
||||||
|
assert.equal(fmRamLogs[0].metadata.from, "/sample.csv");
|
||||||
|
assert.equal(fmRamLogs[1].target, "/sample.csv");
|
||||||
|
|
||||||
|
// 4. Files Manager HTTP mutation -> mock PostgreSQL pool
|
||||||
|
const pgInserts: Array<{ sql: string; params: unknown[] }> = [];
|
||||||
|
const mockDbPool = {
|
||||||
|
query: async (sql: string, params: unknown[] = []) => {
|
||||||
|
if (/FROM\s+admin_users\b/i.test(sql)) {
|
||||||
|
return {
|
||||||
|
rows: [{ email: "audit-owner@datastack.dev", token_version: 1 }],
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (/FROM\s+admin_team_members\b/i.test(sql)) {
|
||||||
|
return { rows: [] };
|
||||||
|
}
|
||||||
|
pgInserts.push({ sql, params });
|
||||||
|
return { rows: [] };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const pgApp = express();
|
||||||
|
pgApp.use(
|
||||||
|
"/vcon/files-engine",
|
||||||
|
filesManager.createFilesSecurityMiddleware({
|
||||||
|
rootDir: tmpDir,
|
||||||
|
getDbPool: () => mockDbPool,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
pgApp.delete("/vcon/files-engine/api/v1/fs/obsolete.csv", (_req: express.Request, res: express.Response) => {
|
||||||
|
res.status(200).json({ ok: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
pgServer = http.createServer(pgApp);
|
||||||
|
await new Promise<void>((resolve) => pgServer!.listen(0, "127.0.0.1", resolve));
|
||||||
|
const pgPort = (pgServer.address() as AddressInfo).port;
|
||||||
|
const pgDel = await fetch(
|
||||||
|
`http://127.0.0.1:${pgPort}/vcon/files-engine/api/v1/fs/obsolete.csv`,
|
||||||
|
{
|
||||||
|
method: "DELETE",
|
||||||
|
headers: {
|
||||||
|
Cookie: cookie,
|
||||||
|
Origin: `http://127.0.0.1:${pgPort}`,
|
||||||
|
Host: `127.0.0.1:${pgPort}`,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert.equal(pgDel.status, 200);
|
||||||
|
await filesManager.flushPendingFilesAuditLogsForTests();
|
||||||
|
|
||||||
|
assert.equal(pgInserts.length, 1);
|
||||||
|
assert.match(pgInserts[0].sql, /INSERT INTO admin_audit_logs/i);
|
||||||
|
assert.equal(pgInserts[0].params[3], "FILES_MANAGER_MUTATION");
|
||||||
|
assert.equal(pgInserts[0].params[4], "/obsolete.csv");
|
||||||
|
|
||||||
|
// 5. AuditLogViewer filter options expose both new action types
|
||||||
|
const viewerSrc = await fsp.readFile(
|
||||||
|
path.join(process.cwd(), "components/sections/AuditLogViewer.tsx"),
|
||||||
|
"utf8",
|
||||||
|
);
|
||||||
|
assert.ok(viewerSrc.includes("FILES_MANAGER_MUTATION"));
|
||||||
|
assert.ok(viewerSrc.includes("DATA_EXPORT_DOWNLOADED"));
|
||||||
|
} finally {
|
||||||
|
if (mockServer) mockServer.close();
|
||||||
|
if (pgServer) pgServer.close();
|
||||||
|
clearAuditLogsForTests();
|
||||||
|
CSVStorageManager.baseStorageDir = prevStorage;
|
||||||
|
CSVStorageManager.clearRowCountCache();
|
||||||
|
CSVStorageManager.clearQuotaCaches();
|
||||||
|
await fsp.rm(tmpDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("R4-Phase-02 AUDIT-07 / ERR-R4-07: GET /api/data isolates corrupt .csv.gz partitions and serves healthy data", async () => {
|
||||||
|
const prevStorage = CSVStorageManager.baseStorageDir;
|
||||||
|
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "ds-r4p2-corrupt-"));
|
||||||
|
CSVStorageManager.baseStorageDir = tmpDir;
|
||||||
|
CSVStorageManager.clearRowCountCache();
|
||||||
|
CSVStorageManager.clearQuotaCaches();
|
||||||
|
|
||||||
|
try {
|
||||||
|
const cookie = await createOwnerCookie("data-owner@datastack.dev");
|
||||||
|
const healthyDir = path.join(tmpDir, "healthy-app");
|
||||||
|
const corruptDir = path.join(tmpDir, "corrupt-app");
|
||||||
|
await fsp.mkdir(healthyDir, { recursive: true });
|
||||||
|
await fsp.mkdir(corruptDir, { recursive: true });
|
||||||
|
|
||||||
|
await fsp.writeFile(
|
||||||
|
path.join(healthyDir, "2026-11-20.csv"),
|
||||||
|
'clientId,eventId,timestamp,payload\n"healthy-app","ok-1","2026-11-20T12:00:00.000Z","{""status"":""alive""}"\n',
|
||||||
|
"utf8",
|
||||||
|
);
|
||||||
|
await fsp.writeFile(
|
||||||
|
path.join(corruptDir, "2026-11-20.csv.gz"),
|
||||||
|
Buffer.from("corrupted-not-gzip-bytes-123456"),
|
||||||
|
);
|
||||||
|
|
||||||
|
const res = await dataGet(
|
||||||
|
new NextRequest(
|
||||||
|
"http://127.0.0.1:3000/api/data?from=2026-11-20&to=2026-11-20&page=1&pageSize=25",
|
||||||
|
{
|
||||||
|
headers: { Cookie: cookie },
|
||||||
|
},
|
||||||
|
),
|
||||||
|
);
|
||||||
|
assert.equal(res.status, 200);
|
||||||
|
const json = await res.json();
|
||||||
|
assert.equal(json.total, 1);
|
||||||
|
assert.equal(json.events.length, 1);
|
||||||
|
assert.equal(json.events[0].eventId, "ok-1");
|
||||||
|
assert.equal(json.warnings, 1);
|
||||||
|
assert.ok(Array.isArray(json.corruptPartitions));
|
||||||
|
assert.equal(json.corruptPartitions.length, 1);
|
||||||
|
assert.equal(json.corruptPartitions[0].clientId, "corrupt-app");
|
||||||
|
assert.equal(json.corruptPartitions[0].filename, "2026-11-20.csv.gz");
|
||||||
|
assert.equal(json.corruptPartitions[0].code, "CORRUPT_GZIP_PARTITION");
|
||||||
|
} finally {
|
||||||
|
CSVStorageManager.baseStorageDir = prevStorage;
|
||||||
|
CSVStorageManager.clearRowCountCache();
|
||||||
|
CSVStorageManager.clearQuotaCaches();
|
||||||
|
await fsp.rm(tmpDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("R4-Phase-02 AUDIT-10 & AUDIT-11: PATCH redacts secret/token and /internal/client-sync disconnects selectively", async () => {
|
||||||
|
const cookie = await createOwnerCookie("clients-owner@datastack.dev");
|
||||||
|
const client = await SDKManager.registerClientAsync({
|
||||||
|
clientId: "r4p2-selective-client",
|
||||||
|
name: "Initial Client Name",
|
||||||
|
rateLimit: 100,
|
||||||
|
storageQuotaMb: 50,
|
||||||
|
allowedIps: "127.0.0.1",
|
||||||
|
});
|
||||||
|
|
||||||
|
// 1. AUDIT-10: PATCH metadata update must omit secret & full token
|
||||||
|
const metaPatchRes = await clientPatch(
|
||||||
|
new NextRequest(`http://127.0.0.1:3000/api/clients/${client.clientId}`, {
|
||||||
|
method: "PATCH",
|
||||||
|
headers: { Cookie: cookie, "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({ name: "Renamed Client", rateLimit: 250 }),
|
||||||
|
}),
|
||||||
|
{ params: Promise.resolve({ id: client.clientId }) },
|
||||||
|
);
|
||||||
|
assert.equal(metaPatchRes.status, 200);
|
||||||
|
const metaPatchJson = await metaPatchRes.json();
|
||||||
|
assert.equal(metaPatchJson.client.name, "Renamed Client");
|
||||||
|
assert.equal(metaPatchJson.client.rateLimit, 250);
|
||||||
|
assert.equal("secret" in metaPatchJson.client, false);
|
||||||
|
assert.equal("token" in metaPatchJson.client, false);
|
||||||
|
assert.equal(typeof metaPatchJson.client.tokenPreview, "string");
|
||||||
|
assert.ok(metaPatchJson.client.tokenPreview.endsWith("..."));
|
||||||
|
|
||||||
|
// 2. AUDIT-10: regenerate_token includes rotated token while still redacting secret
|
||||||
|
const regenRes = await clientPatch(
|
||||||
|
new NextRequest(`http://127.0.0.1:3000/api/clients/${client.clientId}`, {
|
||||||
|
method: "PATCH",
|
||||||
|
headers: { Cookie: cookie, "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({ action: "regenerate_token" }),
|
||||||
|
}),
|
||||||
|
{ params: Promise.resolve({ id: client.clientId }) },
|
||||||
|
);
|
||||||
|
assert.equal(regenRes.status, 200);
|
||||||
|
const regenJson = await regenRes.json();
|
||||||
|
assert.equal("secret" in regenJson.client, false);
|
||||||
|
assert.equal(typeof regenJson.client.token, "string");
|
||||||
|
assert.notEqual(regenJson.client.token, client.token);
|
||||||
|
|
||||||
|
// 3. AUDIT-11: Live Socket.IO client stays connected on metadata update, disconnects on security update
|
||||||
|
await new Promise<void>((resolve) =>
|
||||||
|
ingestionServer.server.listen(0, "127.0.0.1", resolve),
|
||||||
|
);
|
||||||
|
const port = (ingestionServer.server.address() as AddressInfo).port;
|
||||||
|
const internalHeaders = {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
...getInternalIngestionHeaders(),
|
||||||
|
};
|
||||||
|
|
||||||
|
let sock: ReturnType<typeof ioClient> | null = null;
|
||||||
|
try {
|
||||||
|
// Seed ingestion server registry via /internal/client-sync registered
|
||||||
|
const regSyncRes = await fetch(
|
||||||
|
`http://127.0.0.1:${port}/internal/client-sync`,
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
headers: internalHeaders,
|
||||||
|
body: JSON.stringify({
|
||||||
|
action: "registered",
|
||||||
|
clientId: client.clientId,
|
||||||
|
client: {
|
||||||
|
...regenJson.client,
|
||||||
|
token: regenJson.client.token,
|
||||||
|
status: "active",
|
||||||
|
allowedIps: "127.0.0.1",
|
||||||
|
expiresAt: null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert.equal(regSyncRes.status, 200);
|
||||||
|
|
||||||
|
sock = ioClient(`http://127.0.0.1:${port}`, {
|
||||||
|
auth: { token: regenJson.client.token },
|
||||||
|
transports: ["websocket"],
|
||||||
|
});
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
sock!.on("connect", () => resolve());
|
||||||
|
sock!.on("connect_error", reject);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Metadata-only update: socket MUST remain connected
|
||||||
|
const metadataSyncRes = await fetch(
|
||||||
|
`http://127.0.0.1:${port}/internal/client-sync`,
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
headers: internalHeaders,
|
||||||
|
body: JSON.stringify({
|
||||||
|
action: "updated",
|
||||||
|
clientId: client.clientId,
|
||||||
|
client: {
|
||||||
|
...regenJson.client,
|
||||||
|
name: "Hot Metadata Update",
|
||||||
|
rateLimit: 400,
|
||||||
|
storageQuotaMb: 200,
|
||||||
|
token: regenJson.client.token,
|
||||||
|
status: "active",
|
||||||
|
allowedIps: "127.0.0.1",
|
||||||
|
expiresAt: null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert.equal(metadataSyncRes.status, 200);
|
||||||
|
const metadataSyncJson = await metadataSyncRes.json();
|
||||||
|
assert.equal(metadataSyncJson.disconnected, false);
|
||||||
|
assert.equal(sock.connected, true);
|
||||||
|
|
||||||
|
// AllowedIps narrowing away from 127.0.0.1 MUST disconnect live socket
|
||||||
|
const disconnectPromise = new Promise<string>((resolve) => {
|
||||||
|
sock!.once("disconnect", resolve);
|
||||||
|
});
|
||||||
|
const ipSyncRes = await fetch(
|
||||||
|
`http://127.0.0.1:${port}/internal/client-sync`,
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
headers: internalHeaders,
|
||||||
|
body: JSON.stringify({
|
||||||
|
action: "updated",
|
||||||
|
clientId: client.clientId,
|
||||||
|
client: {
|
||||||
|
...regenJson.client,
|
||||||
|
token: regenJson.client.token,
|
||||||
|
status: "active",
|
||||||
|
allowedIps: "203.0.113.50",
|
||||||
|
expiresAt: null,
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert.equal(ipSyncRes.status, 200);
|
||||||
|
const ipSyncJson = await ipSyncRes.json();
|
||||||
|
assert.equal(ipSyncJson.disconnected, true);
|
||||||
|
await disconnectPromise;
|
||||||
|
assert.equal(sock.connected, false);
|
||||||
|
} finally {
|
||||||
|
if (sock) sock.disconnect();
|
||||||
|
await new Promise<void>((resolve) => ingestionServer.io.close(() => resolve()));
|
||||||
|
await SDKManager.deleteClientWithStatsAsync("r4p2-selective-client", {
|
||||||
|
purgeFiles: false,
|
||||||
|
}).catch(() => undefined);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("R4-Phase-02 AUDIT-12 / ERR-R4-08: Streaming S3 upload & streaming .csv.gz row counter", async () => {
|
||||||
|
process.env.DATASTACK_ALLOW_PRIVATE_URLS = "true";
|
||||||
|
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "ds-r4p2-stream-"));
|
||||||
|
|
||||||
|
let s3Server: http.Server | null = null;
|
||||||
|
try {
|
||||||
|
// 1. Multi-chunk (>256 KB) .csv and .csv.gz partitions for row counting & S3 upload
|
||||||
|
const totalRows = 3200;
|
||||||
|
const lines = ["clientId,eventId,timestamp,payload"];
|
||||||
|
for (let i = 0; i < totalRows; i += 1) {
|
||||||
|
lines.push(
|
||||||
|
`"stream-client","evt-${i}","2026-11-20T00:00:00.000Z","{""seq"":${i},""padding"":""0123456789abcdef0123456789abcdef0123456789abcdef""}"`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const rawCsv = `${lines.join("\n")}\n`;
|
||||||
|
assert.ok(Buffer.byteLength(rawCsv, "utf8") > 256 * 1024);
|
||||||
|
|
||||||
|
const csvPath = path.join(tmpDir, "2026-11-20.csv");
|
||||||
|
const gzPath = path.join(tmpDir, "2026-11-20.csv.gz");
|
||||||
|
await fsp.writeFile(csvPath, rawCsv, "utf8");
|
||||||
|
await fsp.writeFile(gzPath, zlib.gzipSync(Buffer.from(rawCsv, "utf8")));
|
||||||
|
|
||||||
|
assert.equal(
|
||||||
|
await CSVStorageManager.countCsvDataRowsStreamAsync(csvPath),
|
||||||
|
totalRows,
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
await CSVStorageManager.countCsvDataRowsStreamAsync(gzPath),
|
||||||
|
totalRows,
|
||||||
|
);
|
||||||
|
|
||||||
|
const corruptGz = path.join(tmpDir, "corrupt.csv.gz");
|
||||||
|
await fsp.writeFile(corruptGz, Buffer.from("invalid-gzip-data"));
|
||||||
|
await assert.rejects(() =>
|
||||||
|
CSVStorageManager.countCsvDataRowsStreamAsync(corruptGz),
|
||||||
|
);
|
||||||
|
|
||||||
|
// 2. Streaming PUT via uploadFileToS3 with exact content-length & SHA-256 verification
|
||||||
|
let failNextUpload = false;
|
||||||
|
const receivedUploads: Array<{
|
||||||
|
url: string;
|
||||||
|
contentLength: string | undefined;
|
||||||
|
sha256: string | undefined;
|
||||||
|
body: Buffer;
|
||||||
|
}> = [];
|
||||||
|
|
||||||
|
s3Server = http.createServer((req, res) => {
|
||||||
|
if (failNextUpload) {
|
||||||
|
req.resume();
|
||||||
|
res.destroy(new Error("Simulated mid-upload S3 reset"));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const chunks: Buffer[] = [];
|
||||||
|
req.on("data", (chunk: Buffer) => chunks.push(chunk));
|
||||||
|
req.on("end", () => {
|
||||||
|
receivedUploads.push({
|
||||||
|
url: req.url || "",
|
||||||
|
contentLength: req.headers["content-length"],
|
||||||
|
sha256: req.headers["x-amz-content-sha256"] as string | undefined,
|
||||||
|
body: Buffer.concat(chunks),
|
||||||
|
});
|
||||||
|
res.writeHead(200, { ETag: '"stream-etag-ok"' });
|
||||||
|
res.end();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
await new Promise<void>((resolve) => s3Server!.listen(0, "127.0.0.1", resolve));
|
||||||
|
const s3Port = (s3Server.address() as AddressInfo).port;
|
||||||
|
const s3Config = {
|
||||||
|
endpoint: `http://127.0.0.1:${s3Port}`,
|
||||||
|
region: "auto",
|
||||||
|
bucket: "datastack-test-bucket",
|
||||||
|
prefix: "archive",
|
||||||
|
accessKeyId: "test-ak",
|
||||||
|
secretAccessKey: "test-sk",
|
||||||
|
};
|
||||||
|
|
||||||
|
const csvRes = await uploadFileToS3(
|
||||||
|
s3Config,
|
||||||
|
csvPath,
|
||||||
|
"archive/stream-client/2026-11-20.csv",
|
||||||
|
);
|
||||||
|
assert.equal(csvRes.ok, true);
|
||||||
|
assert.equal(csvRes.etag, '"stream-etag-ok"');
|
||||||
|
|
||||||
|
const gzRes = await uploadFileToS3(
|
||||||
|
s3Config,
|
||||||
|
gzPath,
|
||||||
|
"archive/stream-client/2026-11-20.csv.gz",
|
||||||
|
);
|
||||||
|
assert.equal(gzRes.ok, true);
|
||||||
|
|
||||||
|
assert.equal(receivedUploads.length, 2);
|
||||||
|
const expectedCsvBuf = await fsp.readFile(csvPath);
|
||||||
|
const expectedCsvHash = crypto
|
||||||
|
.createHash("sha256")
|
||||||
|
.update(expectedCsvBuf)
|
||||||
|
.digest("hex");
|
||||||
|
assert.equal(
|
||||||
|
receivedUploads[0].contentLength,
|
||||||
|
String(expectedCsvBuf.length),
|
||||||
|
);
|
||||||
|
assert.equal(receivedUploads[0].sha256, expectedCsvHash);
|
||||||
|
assert.ok(receivedUploads[0].body.equals(expectedCsvBuf));
|
||||||
|
|
||||||
|
const expectedGzBuf = await fsp.readFile(gzPath);
|
||||||
|
assert.equal(
|
||||||
|
receivedUploads[1].contentLength,
|
||||||
|
String(expectedGzBuf.length),
|
||||||
|
);
|
||||||
|
assert.ok(receivedUploads[1].body.equals(expectedGzBuf));
|
||||||
|
|
||||||
|
// 3. Failure mid-upload returns S3_ARCHIVE_FAILED cleanly
|
||||||
|
failNextUpload = true;
|
||||||
|
const failRes = await uploadFileToS3(
|
||||||
|
s3Config,
|
||||||
|
csvPath,
|
||||||
|
"archive/stream-client/fail.csv",
|
||||||
|
);
|
||||||
|
assert.equal(failRes.ok, false);
|
||||||
|
assert.equal(failRes.code, "S3_ARCHIVE_FAILED");
|
||||||
|
} finally {
|
||||||
|
delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||||
|
if (s3Server) s3Server.close();
|
||||||
|
await fsp.rm(tmpDir, { recursive: true, force: true });
|
||||||
|
assert.equal(fs.existsSync(tmpDir), false);
|
||||||
|
}
|
||||||
|
});
|
||||||
@@ -0,0 +1,313 @@
|
|||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import { createRequire } from 'node:module';
|
||||||
|
import os from 'node:os';
|
||||||
|
import path from 'node:path';
|
||||||
|
import { afterEach, beforeEach, describe, it } from 'node:test';
|
||||||
|
import { NextRequest } from 'next/server';
|
||||||
|
import { POST as setupPost } from '../app/api/auth/setup/route';
|
||||||
|
import { GET as filesStatusGet } from '../app/api/files/status/route';
|
||||||
|
import {
|
||||||
|
createAdminSessionToken,
|
||||||
|
getAdminSessionPayload,
|
||||||
|
getAdminSessionSecret,
|
||||||
|
} from '../core/auth/adminSession';
|
||||||
|
import { resetAdminStateForTests } from '../core/auth/adminUsers';
|
||||||
|
import { middleware } from '../middleware';
|
||||||
|
import { DataStack, DiskSpooler } from '../sdk/node/datastack';
|
||||||
|
|
||||||
|
const require = createRequire(import.meta.url);
|
||||||
|
const ORIGINAL_ENV = { ...process.env };
|
||||||
|
const ORIGINAL_FETCH = globalThis.fetch;
|
||||||
|
|
||||||
|
function restoreEnv() {
|
||||||
|
for (const key of Object.keys(process.env)) {
|
||||||
|
if (!(key in ORIGINAL_ENV)) {
|
||||||
|
delete process.env[key];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Object.assign(process.env, ORIGINAL_ENV);
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeFakeJwt(clientId: string): string {
|
||||||
|
const header = Buffer.from(JSON.stringify({ alg: 'HS256', typ: 'JWT' })).toString('base64url');
|
||||||
|
const payload = Buffer.from(JSON.stringify({ clientId, iat: 1700000000 })).toString('base64url');
|
||||||
|
return `${header}.${payload}.sig`;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('R4-Phase-03: Node SDK Packaging & Spool Parity, Docker & Proxy Hint, Auth Hygiene', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
restoreEnv();
|
||||||
|
resetAdminStateForTests();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
globalThis.fetch = ORIGINAL_FETCH;
|
||||||
|
restoreEnv();
|
||||||
|
resetAdminStateForTests();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('AUDIT-02: prebuilds @datastack/sdk dist artifacts, sets prepare/typescript metadata, and loads directly via CJS require', () => {
|
||||||
|
const sdkDir = path.join(process.cwd(), 'sdk', 'node', 'datastack');
|
||||||
|
const pkg = JSON.parse(fs.readFileSync(path.join(sdkDir, 'package.json'), 'utf8')) as {
|
||||||
|
scripts?: Record<string, string>;
|
||||||
|
devDependencies?: Record<string, string>;
|
||||||
|
main?: string;
|
||||||
|
types?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
assert.equal(pkg.main, './dist/index.js');
|
||||||
|
assert.equal(pkg.types, './dist/index.d.ts');
|
||||||
|
assert.equal(pkg.scripts?.build, 'tsc -p tsconfig.json');
|
||||||
|
assert.equal(pkg.scripts?.prepare, 'npm run build');
|
||||||
|
assert.ok(pkg.devDependencies?.typescript, 'typescript must be listed in devDependencies');
|
||||||
|
|
||||||
|
for (const file of [
|
||||||
|
'index.js',
|
||||||
|
'index.d.ts',
|
||||||
|
'spool.js',
|
||||||
|
'spool.d.ts',
|
||||||
|
'types.js',
|
||||||
|
'types.d.ts',
|
||||||
|
]) {
|
||||||
|
assert.equal(
|
||||||
|
fs.existsSync(path.join(sdkDir, 'dist', file)),
|
||||||
|
true,
|
||||||
|
`Missing prebuilt dist artifact: ${file}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const cjsLoaded = require(path.join(sdkDir, 'dist', 'index.js')) as {
|
||||||
|
DataStack?: typeof DataStack;
|
||||||
|
DataStackClient?: typeof DataStack;
|
||||||
|
DiskSpooler?: typeof DiskSpooler;
|
||||||
|
SDK_VERSION?: string;
|
||||||
|
};
|
||||||
|
assert.equal(typeof cjsLoaded.DataStack, 'function');
|
||||||
|
assert.equal(typeof cjsLoaded.DataStackClient, 'function');
|
||||||
|
assert.equal(typeof cjsLoaded.DiskSpooler, 'function');
|
||||||
|
assert.equal(cjsLoaded.SDK_VERSION, '2.2.0');
|
||||||
|
|
||||||
|
const token = makeFakeJwt('cjs_loader_probe');
|
||||||
|
const client = new cjsLoaded.DataStack!({
|
||||||
|
token,
|
||||||
|
autoConnect: false,
|
||||||
|
offlineSpoolEnabled: false,
|
||||||
|
});
|
||||||
|
assert.equal(client.clientId, 'cjs_loader_probe');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('AUDIT-20: DiskSpooler.ackAndRequeueFront preserves FIFO tail and client.flushSpool recovers partial 429 progress atomically', async () => {
|
||||||
|
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ds-r4-p03-spool-'));
|
||||||
|
try {
|
||||||
|
// 1. Direct DiskSpooler unit check
|
||||||
|
const spooler = new DiskSpooler({ spoolDir: tmpDir, enabled: true });
|
||||||
|
const mkEnv = (id: string) => ({
|
||||||
|
clientId: 'sensor_node',
|
||||||
|
eventId: id,
|
||||||
|
timestamp: '2026-11-21T00:00:00.000Z',
|
||||||
|
data: { id },
|
||||||
|
});
|
||||||
|
spooler.enqueue([mkEnv('e1'), mkEnv('e2'), mkEnv('e3'), mkEnv('e4'), mkEnv('e5')]);
|
||||||
|
const removedCount = spooler.ackAndRequeueFront(3, [mkEnv('e2')]);
|
||||||
|
assert.equal(removedCount, 2);
|
||||||
|
|
||||||
|
const afterPartial = spooler.peekBatch(10).map((e) => e.eventId);
|
||||||
|
assert.deepEqual(afterPartial, ['e2', 'e4', 'e5']);
|
||||||
|
|
||||||
|
// Ensure no leftover .tmp files exist in spoolDir
|
||||||
|
const entries = fs.readdirSync(tmpDir);
|
||||||
|
assert.equal(entries.some((f) => f.endsWith('.tmp')), false);
|
||||||
|
|
||||||
|
// Clean and test partial recovery through DataStack.flushSpool()
|
||||||
|
spooler.clear();
|
||||||
|
const deliveredIds: string[] = [];
|
||||||
|
let simulatePartial = true;
|
||||||
|
|
||||||
|
const token = makeFakeJwt('sensor_node');
|
||||||
|
const client = new DataStack({
|
||||||
|
token,
|
||||||
|
apiUrl: 'http://localhost:3000',
|
||||||
|
serverUrl: 'http://localhost:4000',
|
||||||
|
autoConnect: false,
|
||||||
|
offlineSpoolEnabled: true,
|
||||||
|
offlineSpoolDir: tmpDir,
|
||||||
|
batchChunkSize: 3,
|
||||||
|
maxRetries: 0,
|
||||||
|
fetchImpl: async (_input, init) => {
|
||||||
|
const body = JSON.parse(String(init?.body || '{}')) as {
|
||||||
|
batch?: Array<{ eventId: string }>;
|
||||||
|
};
|
||||||
|
const batch = body.batch || [];
|
||||||
|
if (simulatePartial && batch.length === 3) {
|
||||||
|
// Simulate partial 429 HTTP rejection where batch[1] is rate-limited
|
||||||
|
const limitedId = batch[1].eventId;
|
||||||
|
deliveredIds.push(batch[0].eventId, batch[2].eventId);
|
||||||
|
return new Response(
|
||||||
|
JSON.stringify({
|
||||||
|
success: false,
|
||||||
|
code: 'RATE_LIMIT_EXCEEDED',
|
||||||
|
error: 'Rate limit reached for 1 event',
|
||||||
|
rateLimitedEventIds: [limitedId],
|
||||||
|
}),
|
||||||
|
{ status: 429, headers: { 'Content-Type': 'application/json' } }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for (const item of batch) {
|
||||||
|
deliveredIds.push(item.eventId);
|
||||||
|
}
|
||||||
|
return new Response(
|
||||||
|
JSON.stringify({ success: true, accepted: batch.length }),
|
||||||
|
{ status: 200, headers: { 'Content-Type': 'application/json' } }
|
||||||
|
);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
client.spooler.enqueue([mkEnv('s1'), mkEnv('s2'), mkEnv('s3'), mkEnv('s4')]);
|
||||||
|
const firstFlush = await client.flushSpool();
|
||||||
|
assert.equal(firstFlush.drained, 2);
|
||||||
|
assert.equal(firstFlush.remaining, 2);
|
||||||
|
assert.deepEqual(
|
||||||
|
client.spooler.peekBatch(10).map((e) => e.eventId),
|
||||||
|
['s2', 's4']
|
||||||
|
);
|
||||||
|
|
||||||
|
// Next flush delivers remaining [s2, s4] cleanly with zero duplicates or data loss
|
||||||
|
simulatePartial = false;
|
||||||
|
const secondFlush = await client.flushSpool();
|
||||||
|
assert.equal(secondFlush.drained, 2);
|
||||||
|
assert.equal(secondFlush.remaining, 0);
|
||||||
|
assert.deepEqual(deliveredIds, ['s1', 's3', 's2', 's4']);
|
||||||
|
} finally {
|
||||||
|
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('AUDIT-08 & ERR-R4-09: docker-compose ingestion configures DATASTACK_ADMIN_EMAIL and /api/files/status + UI expose non-Nginx WS hint', async () => {
|
||||||
|
const compose = fs.readFileSync(path.join(process.cwd(), 'docker', 'docker-compose.yml'), 'utf8');
|
||||||
|
const ingestionBlockMatch = compose.match(
|
||||||
|
/ingestion:\s*\n[\s\S]*?\n\s+proxy:/
|
||||||
|
);
|
||||||
|
assert.ok(ingestionBlockMatch, 'ingestion service block must exist in docker-compose.yml');
|
||||||
|
assert.match(
|
||||||
|
ingestionBlockMatch[0],
|
||||||
|
/DATASTACK_ADMIN_EMAIL:\s*"\$\{DATASTACK_ADMIN_EMAIL:-admin@datastack\.dev\}"/
|
||||||
|
);
|
||||||
|
|
||||||
|
delete process.env.DATABASE_URL;
|
||||||
|
delete process.env.DATASTACK_PROXY_MODE;
|
||||||
|
process.env.DATASTACK_ADMIN_SESSION_SECRET =
|
||||||
|
'phase03-files-status-session-secret-min-32-chars';
|
||||||
|
|
||||||
|
const secret = getAdminSessionSecret()!;
|
||||||
|
const ownerCookie = await createAdminSessionToken(
|
||||||
|
'admin@datastack.dev',
|
||||||
|
secret,
|
||||||
|
Date.now() + 60_000,
|
||||||
|
1,
|
||||||
|
'owner',
|
||||||
|
'owner-1'
|
||||||
|
);
|
||||||
|
|
||||||
|
globalThis.fetch = (async () =>
|
||||||
|
new Response(
|
||||||
|
JSON.stringify({
|
||||||
|
ok: true,
|
||||||
|
engine: 'cloudcmd',
|
||||||
|
role: 'owner',
|
||||||
|
readOnly: false,
|
||||||
|
}),
|
||||||
|
{ status: 200, headers: { 'Content-Type': 'application/json' } }
|
||||||
|
)) as typeof fetch;
|
||||||
|
|
||||||
|
// 1. Standalone Next.js mode (no reverse-proxy headers)
|
||||||
|
const standaloneRes = await filesStatusGet(
|
||||||
|
new NextRequest('http://localhost:3000/api/files/status', {
|
||||||
|
headers: { cookie: `datastack_admin_session=${ownerCookie}` },
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert.equal(standaloneRes.status, 200);
|
||||||
|
const standaloneJson = (await standaloneRes.json()) as {
|
||||||
|
ok: boolean;
|
||||||
|
proxyMode: string;
|
||||||
|
wsSupported: boolean;
|
||||||
|
wsHint: string | null;
|
||||||
|
};
|
||||||
|
assert.equal(standaloneJson.ok, true);
|
||||||
|
assert.equal(standaloneJson.proxyMode, 'standalone');
|
||||||
|
assert.equal(standaloneJson.wsSupported, false);
|
||||||
|
assert.ok(standaloneJson.wsHint && /HTTP polling/i.test(standaloneJson.wsHint));
|
||||||
|
|
||||||
|
// 2. Nginx reverse-proxy headers present
|
||||||
|
const nginxRes = await filesStatusGet(
|
||||||
|
new NextRequest('http://localhost:3000/api/files/status', {
|
||||||
|
headers: {
|
||||||
|
cookie: `datastack_admin_session=${ownerCookie}`,
|
||||||
|
'x-forwarded-host': 'console.example.com',
|
||||||
|
'x-forwarded-proto': 'https',
|
||||||
|
},
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert.equal(nginxRes.status, 200);
|
||||||
|
const nginxJson = (await nginxRes.json()) as {
|
||||||
|
proxyMode: string;
|
||||||
|
wsSupported: boolean;
|
||||||
|
wsHint: string | null;
|
||||||
|
};
|
||||||
|
assert.equal(nginxJson.proxyMode, 'nginx');
|
||||||
|
assert.equal(nginxJson.wsSupported, true);
|
||||||
|
assert.equal(nginxJson.wsHint, null);
|
||||||
|
|
||||||
|
// 3. UI component renders proxy hint element safely when present
|
||||||
|
const workspaceSrc = fs.readFileSync(
|
||||||
|
path.join(process.cwd(), 'components', 'sections', 'FilesManagerWorkspace.tsx'),
|
||||||
|
'utf8'
|
||||||
|
);
|
||||||
|
assert.ok(workspaceSrc.includes('files-manager-proxy-hint'));
|
||||||
|
assert.ok(workspaceSrc.includes('data.wsHint'));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('AUDIT-13: middleware removes dead /api/auth/setup/status exemption and POST /api/auth/setup embeds explicit owner role/memberId', async () => {
|
||||||
|
const mwSrc = fs.readFileSync(path.join(process.cwd(), 'middleware.ts'), 'utf8');
|
||||||
|
assert.equal(mwSrc.includes('/api/auth/setup/status'), false);
|
||||||
|
|
||||||
|
process.env.DATASTACK_ADMIN_SESSION_SECRET =
|
||||||
|
'phase03-setup-session-secret-at-least-32-chars';
|
||||||
|
const mwRes = await middleware(
|
||||||
|
new NextRequest('http://localhost:3000/api/auth/setup/status', { method: 'GET' })
|
||||||
|
);
|
||||||
|
assert.equal(mwRes.status, 401);
|
||||||
|
|
||||||
|
delete process.env.DATABASE_URL;
|
||||||
|
process.env.DATASTACK_ALLOW_DEMO_ADMIN = 'false';
|
||||||
|
process.env.DATASTACK_ADMIN_SETUP_KEY =
|
||||||
|
'phase03-custom-setup-key-over-32-characters-long';
|
||||||
|
|
||||||
|
const setupRes = await setupPost(
|
||||||
|
new NextRequest('http://localhost:3000/api/auth/setup', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'content-type': 'application/json',
|
||||||
|
'x-forwarded-for': '203.0.113.15',
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
email: 'founder@datastack.dev',
|
||||||
|
password: 'FounderPassword123!',
|
||||||
|
setupKey: process.env.DATASTACK_ADMIN_SETUP_KEY,
|
||||||
|
}),
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert.equal(setupRes.status, 200);
|
||||||
|
const cookieVal = setupRes.cookies.get('datastack_admin_session')?.value;
|
||||||
|
assert.ok(cookieVal);
|
||||||
|
|
||||||
|
const payload = await getAdminSessionPayload(
|
||||||
|
cookieVal!,
|
||||||
|
process.env.DATASTACK_ADMIN_SESSION_SECRET
|
||||||
|
);
|
||||||
|
assert.ok(payload);
|
||||||
|
assert.equal(payload!.email, 'founder@datastack.dev');
|
||||||
|
assert.equal(payload!.role, 'owner');
|
||||||
|
assert.equal(payload!.memberId, 'owner-1');
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import fs from 'node:fs/promises';
|
||||||
|
import path from 'node:path';
|
||||||
|
import test from 'node:test';
|
||||||
|
import { getVconRoleCapabilities } from '../lib/vconUiHelpers';
|
||||||
|
|
||||||
|
const ROOT = process.cwd();
|
||||||
|
|
||||||
|
async function readSource(relPath: string): Promise<string> {
|
||||||
|
return fs.readFile(path.join(ROOT, relPath), 'utf8');
|
||||||
|
}
|
||||||
|
|
||||||
|
test('R4-Phase-04 [AUDIT-14]: TeamWorkspace seeds role from useVconRole, confirms deletes via Modal, and distinguishes operator vs admin', async () => {
|
||||||
|
const src = await readSource('components/sections/TeamWorkspace.tsx');
|
||||||
|
|
||||||
|
// Eliminates hardcoded 'owner' initial flash
|
||||||
|
assert.match(src, /useVconRole\(\)/);
|
||||||
|
assert.doesNotMatch(src, /useState<string>\(\s*['"]owner['"]\s*\)/);
|
||||||
|
assert.match(src, /useState<string>\(\(\)\s*=>\s*shellRole\s*\|\|\s*['"]viewer['"]\)/);
|
||||||
|
|
||||||
|
// Explicit delete confirmation modal before DELETE /api/team/[id]
|
||||||
|
assert.match(src, /setDeleteTarget\(row\)/);
|
||||||
|
assert.match(src, /title="Confirm Team Member Removal"/);
|
||||||
|
assert.match(src, /data-testid="confirm-remove-member"/);
|
||||||
|
assert.doesNotMatch(src, /onClick=\{\(\)\s*=>\s*void handleDeleteMember\(row\)\}/);
|
||||||
|
|
||||||
|
// Preserves distinct operator vs admin options instead of silently coercing operator -> admin
|
||||||
|
assert.doesNotMatch(src, /row\.role === ['"]operator['"] \? ['"]admin['"]/);
|
||||||
|
assert.match(src, /value:\s*['"]operator['"]/);
|
||||||
|
assert.match(src, /value:\s*['"]admin['"]/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('R4-Phase-04 [AUDIT-15 / ERR-R4-10]: /vcon/sdk skips ?includeToken=true fetch for viewer and renders explicit RBAC notice', async () => {
|
||||||
|
const src = await readSource('app/vcon/sdk/page.tsx');
|
||||||
|
|
||||||
|
assert.match(src, /useVconRole\(\)/);
|
||||||
|
assert.match(src, /getVconRoleCapabilities\(shellRole\)/);
|
||||||
|
assert.match(src, /if \(!selectedClientId \|\| !canOperate\)/);
|
||||||
|
assert.match(src, /data-testid="sdk-viewer-rbac-banner"/);
|
||||||
|
assert.match(src, /Operator or Owner role required/i);
|
||||||
|
|
||||||
|
// Confirm capability helper blocks viewer token retrieval while allowing operator/admin/owner
|
||||||
|
assert.equal(getVconRoleCapabilities('viewer').canOperate, false);
|
||||||
|
assert.equal(getVconRoleCapabilities('operator').canOperate, true);
|
||||||
|
assert.equal(getVconRoleCapabilities('admin').canOperate, true);
|
||||||
|
assert.equal(getVconRoleCapabilities('owner').canOperate, true);
|
||||||
|
|
||||||
|
// Preserve V2-Phase-04 and R2-Phase-03 compatibility checks
|
||||||
|
assert.match(src, /Python SDK \(v2\.2\.0\)/);
|
||||||
|
assert.match(src, /Node\.js \/ TypeScript SDK \(v2\.2\.0\)/);
|
||||||
|
assert.match(src, /setSelectedClientToken\(''\);\s*\n\s*setLoadingToken\(true\);/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('R4-Phase-04 [AUDIT-16]: Sidebar uses useVconRole and Client SDKs label; Navbar covers all admin routes', async () => {
|
||||||
|
const sidebarSrc = await readSource('components/layout/Sidebar.tsx');
|
||||||
|
assert.match(sidebarSrc, /useVconRole\(\)/);
|
||||||
|
assert.match(sidebarSrc, /\{\s*name:\s*['"]Client SDKs['"],\s*href:\s*['"]\/vcon\/sdk['"]/);
|
||||||
|
assert.doesNotMatch(sidebarSrc, /\{\s*name:\s*['"]Python SDK['"]/);
|
||||||
|
assert.doesNotMatch(sidebarSrc, /getCached<CachedProfile>\('vcon:profile',\s*\{\}\)\.role\s*\|\|\s*['"]Owner['"]/);
|
||||||
|
assert.match(sidebarSrc, /getCached<CachedProfile>\('vcon:profile',\s*\{\}\)\.role\s*\|\|\s*initialRoleLabel/);
|
||||||
|
|
||||||
|
const navbarSrc = await readSource('components/layout/Navbar.tsx');
|
||||||
|
assert.match(navbarSrc, /'\/vcon\/files':\s*'Files Manager'/);
|
||||||
|
assert.match(navbarSrc, /'\/vcon\/sdk':\s*'Client SDKs'/);
|
||||||
|
assert.match(navbarSrc, /'\/vcon\/team':\s*'Team & RBAC'/);
|
||||||
|
assert.match(navbarSrc, /'\/vcon\/audit':\s*'Audit Log'/);
|
||||||
|
assert.doesNotMatch(navbarSrc, /'\/vcon\/sdk':\s*'Python SDK'/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('R4-Phase-04 [AUDIT-19 & AUDIT-07]: ProfileWorkspace drops hardcoded demo defaults and DataBrowser surfaces corruptPartitions warnings', async () => {
|
||||||
|
const profileSrc = await readSource('components/sections/ProfileWorkspace.tsx');
|
||||||
|
assert.doesNotMatch(profileSrc, /DEFAULT_PROFILE/);
|
||||||
|
assert.doesNotMatch(profileSrc, /email:\s*'admin@datastack\.dev'/);
|
||||||
|
assert.doesNotMatch(profileSrc, /storageMode:\s*'In-Memory'/);
|
||||||
|
assert.match(profileSrc, /EMPTY_PROFILE/);
|
||||||
|
assert.match(profileSrc, /Loading\.\.\./);
|
||||||
|
|
||||||
|
const dataBrowserSrc = await readSource('components/sections/DataBrowser.tsx');
|
||||||
|
assert.match(dataBrowserSrc, /corruptPartitions/);
|
||||||
|
assert.match(dataBrowserSrc, /data-testid="data-corrupt-partitions-warning"/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('R4-Phase-04 [AUDIT-18]: Public landing Hero, SdkSection, and LandingAbout reflect both Python and Node.js / TypeScript SDKs', async () => {
|
||||||
|
const heroSrc = await readSource('components/sections/Hero.tsx');
|
||||||
|
assert.match(heroSrc, /href="\/#sdk"[\s\S]*?>\s*Client SDKs\s*<\/Link>/);
|
||||||
|
assert.doesNotMatch(heroSrc, />\s*Python SDK\s*<\/Link>/);
|
||||||
|
|
||||||
|
const sdkSectionSrc = await readSource('components/sections/SdkSection.tsx');
|
||||||
|
assert.match(sdkSectionSrc, /label="Client SDKs"/);
|
||||||
|
assert.match(sdkSectionSrc, /datastack/);
|
||||||
|
assert.match(sdkSectionSrc, /@datastack\/sdk/);
|
||||||
|
assert.match(sdkSectionSrc, /npm install/);
|
||||||
|
assert.match(sdkSectionSrc, /pip install/);
|
||||||
|
|
||||||
|
const aboutSrc = await readSource('components/sections/LandingAbout.tsx');
|
||||||
|
assert.match(aboutSrc, /href="\/#sdk"[\s\S]*?>\s*View Client SDKs\s*<\/Link>/);
|
||||||
|
assert.doesNotMatch(aboutSrc, /View Python SDK/);
|
||||||
|
});
|
||||||
@@ -146,6 +146,9 @@ test("V2-Phase-01 [S3 SigV4]: buildS3ObjectKey, normalizeS3EndpointUrl, testS3Co
|
|||||||
const addr = mockS3.address() as AddressInfo;
|
const addr = mockS3.address() as AddressInfo;
|
||||||
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "datastack-v2-s3-"));
|
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "datastack-v2-s3-"));
|
||||||
|
|
||||||
|
const prevAllowPrivate = process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||||
|
process.env.DATASTACK_ALLOW_PRIVATE_URLS = "true";
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const samplePath = path.join(tmpDir, "2026-10-01.csv.gz");
|
const samplePath = path.join(tmpDir, "2026-10-01.csv.gz");
|
||||||
const gzData = zlib.gzipSync(
|
const gzData = zlib.gzipSync(
|
||||||
@@ -189,6 +192,8 @@ test("V2-Phase-01 [S3 SigV4]: buildS3ObjectKey, normalizeS3EndpointUrl, testS3Co
|
|||||||
assert.equal(uploadErr.code, "S3_ARCHIVE_FAILED");
|
assert.equal(uploadErr.code, "S3_ARCHIVE_FAILED");
|
||||||
assert.match(uploadErr.error || "", /HTTP 500/);
|
assert.match(uploadErr.error || "", /HTTP 500/);
|
||||||
} finally {
|
} finally {
|
||||||
|
if (prevAllowPrivate === undefined) delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||||
|
else process.env.DATASTACK_ALLOW_PRIVATE_URLS = prevAllowPrivate;
|
||||||
await new Promise<void>((resolve) => mockS3.close(() => resolve()));
|
await new Promise<void>((resolve) => mockS3.close(() => resolve()));
|
||||||
await fs.rm(tmpDir, { recursive: true, force: true });
|
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||||
}
|
}
|
||||||
@@ -229,6 +234,9 @@ test("V2-Phase-01 [Compression, Export, S3 Safe Prune V2-E02, Corrupt Gzip V2-E0
|
|||||||
await new Promise<void>((resolve) => mockS3.listen(0, "127.0.0.1", resolve));
|
await new Promise<void>((resolve) => mockS3.listen(0, "127.0.0.1", resolve));
|
||||||
const s3Addr = mockS3.address() as AddressInfo;
|
const s3Addr = mockS3.address() as AddressInfo;
|
||||||
|
|
||||||
|
const prevAllowPrivate = process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||||
|
process.env.DATASTACK_ALLOW_PRIVATE_URLS = "true";
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const cookie = await makeAdminCookie();
|
const cookie = await makeAdminCookie();
|
||||||
const today = new Date().toISOString().slice(0, 10);
|
const today = new Date().toISOString().slice(0, 10);
|
||||||
@@ -412,7 +420,7 @@ test("V2-Phase-01 [Compression, Export, S3 Safe Prune V2-E02, Corrupt Gzip V2-E0
|
|||||||
assert.deepEqual(uploadedKeys, [`/prod-vault/archive-test/v2_alpha/${oldDate}.csv.gz`]);
|
assert.deepEqual(uploadedKeys, [`/prod-vault/archive-test/v2_alpha/${oldDate}.csv.gz`]);
|
||||||
await fs.stat(gzMeta.filePath);
|
await fs.stat(gzMeta.filePath);
|
||||||
|
|
||||||
// 10. V2-E03: Corrupt .csv.gz cleanly surfaces error in /api/data instead of crashing
|
// 10. V2-E03 upgraded to ERR-R4-07: Corrupt .csv.gz is isolated per-file (returns 200 with corruptPartitions warning instead of crashing all clients)
|
||||||
await fs.writeFile(
|
await fs.writeFile(
|
||||||
path.join(tmpDir, "v2_alpha", "2024-01-01.csv.gz"),
|
path.join(tmpDir, "v2_alpha", "2024-01-01.csv.gz"),
|
||||||
Buffer.from("not-valid-gzip-bytes"),
|
Buffer.from("not-valid-gzip-bytes"),
|
||||||
@@ -423,10 +431,16 @@ test("V2-Phase-01 [Compression, Export, S3 Safe Prune V2-E02, Corrupt Gzip V2-E0
|
|||||||
{ headers: { cookie } },
|
{ headers: { cookie } },
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
assert.equal(corruptDataRes.status, 500);
|
assert.equal(corruptDataRes.status, 200);
|
||||||
const corruptJson = await corruptDataRes.json();
|
const corruptJson = await corruptDataRes.json();
|
||||||
assert.match(corruptJson.error || "", /Corrupt|\.gz/i);
|
assert.equal(corruptJson.warnings, 1);
|
||||||
|
assert.ok(Array.isArray(corruptJson.corruptPartitions));
|
||||||
|
assert.equal(corruptJson.corruptPartitions.length, 1);
|
||||||
|
assert.equal(corruptJson.corruptPartitions[0].code, "CORRUPT_GZIP_PARTITION");
|
||||||
|
assert.match(corruptJson.corruptPartitions[0].error || "", /Corrupt|\.gz/i);
|
||||||
} finally {
|
} finally {
|
||||||
|
if (prevAllowPrivate === undefined) delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||||
|
else process.env.DATASTACK_ALLOW_PRIVATE_URLS = prevAllowPrivate;
|
||||||
await new Promise<void>((resolve) => mockS3.close(() => resolve()));
|
await new Promise<void>((resolve) => mockS3.close(() => resolve()));
|
||||||
await DataBufferEngine.shutdown();
|
await DataBufferEngine.shutdown();
|
||||||
CSVStorageManager.baseStorageDir = origBaseStorageDir;
|
CSVStorageManager.baseStorageDir = origBaseStorageDir;
|
||||||
|
|||||||
@@ -191,6 +191,9 @@ test("V2-Phase-02 [Webhook Notifier & /api/settings/webhook-test]: HMAC-SHA256 s
|
|||||||
const port = (mockWebhookServer.address() as AddressInfo).port;
|
const port = (mockWebhookServer.address() as AddressInfo).port;
|
||||||
const webhookUrl = `http://127.0.0.1:${port}/hooks/datastack`;
|
const webhookUrl = `http://127.0.0.1:${port}/hooks/datastack`;
|
||||||
|
|
||||||
|
const prevAllowPrivate = process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||||
|
process.env.DATASTACK_ALLOW_PRIVATE_URLS = "true";
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// 1. Format & signature verification
|
// 1. Format & signature verification
|
||||||
const formattedJsonStr = formatOutboundWebhookBody({
|
const formattedJsonStr = formatOutboundWebhookBody({
|
||||||
@@ -297,6 +300,8 @@ test("V2-Phase-02 [Webhook Notifier & /api/settings/webhook-test]: HMAC-SHA256 s
|
|||||||
const failTestData = await failTestRes.json();
|
const failTestData = await failTestRes.json();
|
||||||
assert.equal(failTestData.code, "WEBHOOK_DELIVERY_FAILED");
|
assert.equal(failTestData.code, "WEBHOOK_DELIVERY_FAILED");
|
||||||
} finally {
|
} finally {
|
||||||
|
if (prevAllowPrivate === undefined) delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||||
|
else process.env.DATASTACK_ALLOW_PRIVATE_URLS = prevAllowPrivate;
|
||||||
await new Promise<void>((resolve) => mockWebhookServer.close(() => resolve()));
|
await new Promise<void>((resolve) => mockWebhookServer.close(() => resolve()));
|
||||||
await DataBufferEngine.shutdown();
|
await DataBufferEngine.shutdown();
|
||||||
resetAdminStateForTests();
|
resetAdminStateForTests();
|
||||||
@@ -807,6 +812,9 @@ test("V2-Audit-Phase-02 [BUG-V2-AUDIT-05, BUG-V2-AUDIT-06, BUG-V2-AUDIT-08]: Por
|
|||||||
);
|
);
|
||||||
const socketPort = (testHttpServer.address() as AddressInfo).port;
|
const socketPort = (testHttpServer.address() as AddressInfo).port;
|
||||||
|
|
||||||
|
const prevAllowPrivate = process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||||
|
process.env.DATASTACK_ALLOW_PRIVATE_URLS = "true";
|
||||||
|
|
||||||
try {
|
try {
|
||||||
serverIndex.clearWebhookCooldownCacheForTests();
|
serverIndex.clearWebhookCooldownCacheForTests();
|
||||||
serverIndex.applyRuntimeSettingsObject({
|
serverIndex.applyRuntimeSettingsObject({
|
||||||
@@ -890,6 +898,8 @@ test("V2-Audit-Phase-02 [BUG-V2-AUDIT-05, BUG-V2-AUDIT-06, BUG-V2-AUDIT-08]: Por
|
|||||||
const forbiddenHealthBody = await forbiddenHealthRes.json();
|
const forbiddenHealthBody = await forbiddenHealthRes.json();
|
||||||
assert.equal(forbiddenHealthBody.code, "CLIENT_IP_FORBIDDEN");
|
assert.equal(forbiddenHealthBody.code, "CLIENT_IP_FORBIDDEN");
|
||||||
} finally {
|
} finally {
|
||||||
|
if (prevAllowPrivate === undefined) delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||||
|
else process.env.DATASTACK_ALLOW_PRIVATE_URLS = prevAllowPrivate;
|
||||||
serverIndex.applyRuntimeSettingsObject({
|
serverIndex.applyRuntimeSettingsObject({
|
||||||
enableWebhookAlerts: false,
|
enableWebhookAlerts: false,
|
||||||
webhookUrl: "",
|
webhookUrl: "",
|
||||||
|
|||||||
Reference in new issue
Block a user