feat: implement R4 production hardening

- Add configurable ingestion batch item limits
- Improve client serialization and sanitization
- Add graceful handling for corrupt GZIP partitions in data API
- Expand admin session token scope
- Update project documentation for R4 status
This commit is contained in:
Team Dark Devil committed 2026-11-21 22:15:32 +06:00
1 parent 8132087085
commit ae9b554189
59 files changed
+16878 -1073

No files matched your search

+6
View File
@@ -27,3 +27,9 @@ PORT_SOCKET=4000
INGESTION_INTERNAL_URL=http://127.0.0.1:4000
# Optional comma-separated browser CORS origin allowlist for Port 4000 ingestion/Socket.IO (defaults to wildcard if unset)
# DATASTACK_CORS_ORIGINS=https://console.example.com,https://admin.example.com
# Outbound SSRF protection blocks loopback/private/internal URLs by default for webhooks & S3 endpoints.
# Set to true ONLY if intentionally targeting a self-hosted LAN MinIO or private webhook endpoint (cloud metadata IPs remain always blocked).
# DATASTACK_ALLOW_PRIVATE_URLS=false
# Ingestion Batch Array Guard (default: 1000 items per HTTP POST /api/events batch or Socket.IO batch_sync frame; bounded 1..10000)
# DATASTACK_MAX_BATCH_ITEMS=1000
+61 -14
View File
@@ -1,7 +1,7 @@
# MEMORY.md — DataStack Cross-IDE Persistent Project Memory
> **Last Synchronized**: 2026-11-20
> **Status**: **R3 Production Remediation (`BASELINE-2026-11-20-R3`): `3 / 4` Phases Complete (`12 / 16` Bugs Resolved — `R3-Phase-01`, `R3-Phase-02`, & `R3-Phase-03` Verified Complete; `R3-Phase-04` ready for `Proceed to R3-Phase-04`).**
> **Last Synchronized**: 2026-11-21
> **Status**: **`R4` Production Hardening (`BASELINE-2026-11-20-R4`: `4 / 4` Phases Complete — All `R4-Phase-01`..`04` Complete & Verified (`AUDIT-01`–`20`, `19 / 19` actionable findings resolved)).**
> **Scope Rule**: Maintain this file accurately across VS Code, Google Antigravity, Cursor, and any other IDE after every modification, feature update, or bug fix (see `.ai-rules` and `AGENTS.md`).
---
@@ -61,12 +61,13 @@ datastack/
│ │ ├── data/ # Paginated CSV search/query across partitions
│ │ ├── events/ # Authenticated HTTP ingestion (single/batch/file records) & recent list
│ │ ├── export/ # Streaming CSV / JSON filtered export (UTC dates, optional flush=false)
│ │ ├── files/status/ # Safe server-side Port 4000 Files Manager probe (catches ECONNREFUSED cleanly)
│ │ ├── health/ # Web health + client Bearer token probe + async buffer depth
│ │ ├── metrics/ # Aggregated Port 3000 + Port 4000 + DB + CSV metrics
│ │ ├── settings/ # Allowlisted runtime config get/post + live Port 4000 sync
│ │ └── storage/ # CSV partition listing & manual cross-process buffer flush
│ └── vcon/ # Protected Admin Console pages:
│ ├── dashboard/ clients/ data/ export/ login/ profile/ sdk/ settings/ setup/ simulator/ storage/
│ ├── dashboard/ clients/ data/ export/ files/ login/ profile/ sdk/ settings/ setup/ simulator/ storage/ team/ audit/
├── components/
│ ├── ui/ # Single-source-of-truth UI primitives (Button, Input, Table, Modal, Panel, etc.)
│ ├── layout/ # MainLayout, Navbar, Sidebar, VconShell, PageShell, AdminPageHeader, Footer
@@ -89,7 +90,8 @@ datastack/
│ ├── dev-all.js # Local concurrent Next.js (3000) + Ingestion (4000) dev runner
│ └── start-all.js # Production concurrent Next.js (`next start`) + Ingestion (`server/index.js`) runner
├── server/
│ ├── index.js # Express 5 + Socket.IO (Port 4000), /health, /internal/* bridge, PG sync, graceful exit
│ ├── index.js # Express 5 + Socket.IO (Port 4000), /health, /internal/* bridge, PG sync, filesManager mount
│ ├── filesManager.js # Hard-locked cloudcmd@19.23.2 (/vcon/files-engine), isolated Socket.IO, Admin Cookie/DB/RBAC/CSRF/*.lock guard
│ ├── socket/handlers.js # Handshake JWT auth, ping_verify, sync_data, batch_sync
│ └── workers/csvWriter.js # Partitioned CSV batch writer & rotation/pruning worker
├── sdk/python/datastack/ # Official pip-installable Python SDK (`datastack` v2.2.0)
@@ -146,16 +148,17 @@ datastack/
│ ├── configuration-reference.md # Complete environment variable & setting bounds
│ ├── dependencies.md # Runtime & dev dependencies audit with rationales
│ └── glossary.md # Technical domain terminology & concept glossary
├── notes/ # Production Remediation Governance & Scope-Lock Suite (`BASELINE-2026-11-20-R3`):
│ ├── README.md # Master governance index, 6-step execution workflow & phase approval commands
│ ├── FORENSIC_AUDIT_REPORT.md # 100% raw-code verified audit (16 bugs: BUG-01..BUG-16) & Demo vs Prod matrix
│ ├── BASELINE_FREEZE.md # Frozen baseline snapshot (`BASELINE-2026-11-20-R3`) & 1:1 bug alignment
│ ├── PRODUCTION_READINESS_SCOPE_LOCK.md # (Doc 001) Exact bug fix contracts & per-phase Scope Locks (`R3-Phase-01`..`04`)
│ ├── PRODUCTION_ROADMAP.md # (Doc 002) Dependency-ordered A–Z 4-phase roadmap & burn-down tracker
│ ├── PHASE_EXECUTION_LOG.md # (Doc 003) Live phase completion log (`0/4` complete in R3 cycle) & zero-drift guardrails
│ ├── ERROR_HANDLING_MATRIX.md # (Doc 004) Active error-handling catalog & 7 planned fixes (`ERR-R3-01`..`07`)
│ └── ACTUAL_IMPLEMENTATION_STATUS.md # Ground-truth working vs completed-per-phase vs remaining ledger (16 items remaining)
├── notes/ # `R4` Production Hardening Governance Suite (`BASELINE-2026-11-20-R4`):
│ ├── README.md # Master governance index, 6-step zero-drift protocol & approval triggers (`R4-Phase-01`..`04`)
│ ├── FULL_PROJECT_FORENSIC_AUDIT.md # 100% raw-source-verified deep scan (20 findings: 3 Critical, 6 High, 7 Medium, 4 Low)
│ ├── BASELINE_FREEZE.md # Frozen baseline snapshot (`BASELINE-2026-11-20-R4`) & 1:1 audit alignment matrix
│ ├── PRODUCTION_READINESS_SCOPE_LOCK.md # (Doc 001) All 20 defect fix contracts & exact per-phase Scope Locks
│ ├── PRODUCTION_ROADMAP.md # (Doc 002) Dependency-ordered A–Z 4-phase roadmap & verification gates
│ ├── PHASE_EXECUTION_LOG.md # (Doc 003) Live phase completion tracker (`4 / 4` complete) & changelog template
│ ├── ERROR_HANDLING_MATRIX.md # (Doc 004) Verified existing error contracts + 10 planned `R4` contracts (`ERR-R4-01`..`10`)
│ └── ACTUAL_IMPLEMENTATION_STATUS.md # Ground-truth working vs phase-delivered vs pending ledger (Demo & Production)
└── tests/ # Automated verification suites:
├── files-manager.test.ts # CloudCmd RCE lockdown, CRUD/ZIP/UNZIP, RBAC, CSRF, *.lock, DB revocation, proxy & UI wiring
├── phase01-auth.test.ts # Auth fail-closed, tokenVersion, brute-force lockout, Bearer revocation, CSRF
├── phase02-sync.test.ts # Internal bridge auth, settings sync, JWT rotation kick, rate/security metrics
├── phase03-storage-buffer.test.ts # Redis durability, cooperative CSV locks, streaming row count, export filters
@@ -377,6 +380,49 @@ All planned hardening & deployment phases (**Phase-01 through Phase-07**) are **
- **`BUG-07` (`ERR-R3-05`)**: Implemented real `heartbeatIntervalMs` monitor in Node.js SDK (`startHeartbeatMonitor` with `.unref?.()`, zero-CPU `IDLE_WS` pause, automatic `ping_verify` recovery & spool drain, clean timer teardown, `status().heartbeatActive`); fixed single HTTP/WS `DUPLICATE_DROPPED` reporting (`accepted: 0, duplicates: 1`); wired `TOKEN_ROTATED` (`401`) and `CLIENT_REVOKED` (`403`) fast-fail `DataStackAuthError` across WS `connect_error`, `ping_verify`, `sync_data`, `batch_sync`, and HTTP fallback; compiled CommonJS `sdk/node/datastack/dist/{index,types,spool}.{js,d.ts}` via `tsconfig.json` and pointed `package.json` `main`/`types`/`exports` to `./dist/index.js` & `./dist/index.d.ts`.
- **`BUG-08` (`ERR-R3-06`)**: Added non-retryable `DataStackQuotaExceededError(DataStackError, retryable=False)` in `sdk/python/datastack/datastack/exceptions.py`, re-exported it in both `datastack` and `datastack_sdk`, and wired immediate fail-fast handling for HTTP `507` and WS `STORAGE_QUOTA_EXCEEDED` / `DISK_SPACE_LOW` in `sdk/python/datastack/datastack/client.py` with `0` retries, `0` HTTP fallback after WS quota rejection, and `0` offline spooling.
- **Verified**: `13/13` Node SDK/packaging tests (`5/5` `tests/r3-phase03-sdk-parity.test.ts` against live Socket.IO handlers + `4/4` `v2-phase04-node-sdk.test.ts` + `4/4` `phase05-packaging.test.ts`), `14/14` Python SDK tests (`3/3` `tests/r3_phase03_python_sdk_test.py` + `11/11` `phase05_sdk_test.py`), `py_compile`, `tsc`, `npm run build` clean, `npm run lint` (`0` errors, `0` warnings). `0` Port 3000/4000/UI/Docker files modified.
- **`R3-Phase-04` Complete & Verified (`2026-11-20`)** (`BUG-11`, `BUG-12`, `BUG-14`, `BUG-16`; `ERR-R3-07`):
- **`BUG-11` (`ERR-R3-07`)**: Added `shouldRedirectSimulatorUnauthorized(status, body)` in `lib/vconUiHelpers.ts` and wired it post-JSON-parse across both single & batch HTTP sends in `app/vcon/simulator/page.tsx` so expired/rotated/revoked client `401` responses display inline without logging out the authenticated admin.
- **`BUG-12`**: Added browser-local `formatDateForLocalInput(iso)` in `lib/vconUiHelpers.ts` and replaced UTC `.toISOString().slice(0, 16)` inside `ClientsTable.tsx` `openEditModal()`, eliminating local timezone drift on client edits.
- **`BUG-14`**: Added `isUiClientExpired()`, `classifyUiClientStatus()`, and `summarizeUiClientCounts()` in `lib/vconUiHelpers.ts`; excluded expired clients from `activeClients` and exposed `expiredClients` in `core/apiEngine/apiService.ts` and `DashboardOverview.tsx`; emitted `datastack_clients_total{status="expired"}` and excluded expired clients from `status="active"` in `app/api/prometheus/route.ts`; expanded `/vcon/clients` with a 6-column `StatStrip` + 6-pill status filter (`Total`, `Online`, `Offline`, `Pending`, `Expired`, `Revoked`).
- **`BUG-16`**: Propagated verified session role from `app/vcon/layout.tsx` through `components/layout/VconShell.tsx` (`VconRoleContext` / `useVconRole()`) + `getVconRoleCapabilities()` (`lib/vconUiHelpers.ts`) to gate all role-restricted UI controls across `SettingsWorkspace`, `ClientsTable`, `/vcon/clients`, `DashboardOverview`, `/vcon/storage`, and `/vcon/simulator`.
- **Verified**: `29/29` TS unit/integration tests (`4/4` `r3-phase04-ui-metrics-rbac.test.ts` + `15/15` `phase04-ui-sync.test.ts` + `5/5` `v2-phase02-alerts-ip-prometheus.test.ts` + `5/5` `v2-phase03-rbac-audit.test.ts`), live headless Chromium (`playwright-cli`, `1440x900`) verification across `/vcon/dashboard`, `/vcon/clients`, `/vcon/simulator`, `/vcon/storage`, and `/vcon/settings` (`0` JS/React pageerrors), `npm run build` clean, `npm run lint` (`0` errors, `0` warnings). `0` Port 4000/SDK/Auth/Storage/Docker files modified. **All `4 / 4` R3 phases (`16 / 16` bugs) complete.**
- **Full-Project Production Forensic Audit & `R4` Baseline Freeze (`BASELINE-2026-11-20-R4`)**:
- Cleaned all 8 prior Files Manager files from `notes/` and created the 100% raw-source-verified `notes/FULL_PROJECT_FORENSIC_AUDIT.md` (**20** verified findings: `3` Critical, `6` High, `7` Medium, `4` Low — `19` actionable + `1` verified intentional preview `AUDIT-17`).
- Froze the untouched codebase as `BASELINE-2026-11-20-R4` and created the complete 8-file `R4` governance & Scope-Lock suite in `notes/` (`README.md`, `FULL_PROJECT_FORENSIC_AUDIT.md`, `BASELINE_FREEZE.md`, `PRODUCTION_READINESS_SCOPE_LOCK.md`, `PRODUCTION_ROADMAP.md`, `PHASE_EXECUTION_LOG.md`, `ERROR_HANDLING_MATRIX.md`, `ACTUAL_IMPLEMENTATION_STATUS.md`) across `4` Scope-Locked phases (`R4-Phase-01`..`R4-Phase-04`).
- **`R4-Phase-01` Complete & Verified (`2026-11-21`)** (`AUDIT-01`, `AUDIT-03`, `AUDIT-04`, `AUDIT-09`; `ERR-R4-01`–`ERR-R4-05`):
- **`AUDIT-01` (`ERR-R4-01`)**: Created shared SSRF validator `core/Security/ssrfGuard.ts` blocking non-HTTP(S) schemes, embedded credentials, loopback, RFC1918/CGNAT/ULA, `0.0.0.0/8`, cloud metadata (`169.254.0.0/16`, `fe80::/10`, `metadata.google.internal`), bare internal hostnames, and DNS-rebinding targets, plus `redirect: "manual"` 3xx rejection; enforced uniformly across `core/alerts/webhookNotifier.ts`, `core/Storage/s3Archive.ts`, `app/api/settings/route.ts`, `app/api/settings/webhook-test/route.ts`, `app/api/storage/s3-test/route.ts`, and `server/index.js` `triggerWebhookAlert`, with `DATASTACK_ALLOW_PRIVATE_URLS=true` documented in `.env.example` (metadata still unconditionally blocked).
- **`AUDIT-03` (`ERR-R4-02`)**: Mirrored `admin_team_members` and `admin_audit_logs` DDL 1:1 into `server/index.js` `ensureDatabaseSchema()` for full 7-table parity across Port `3000` and Port `4000` on fresh PostgreSQL boots.
- **`AUDIT-04` (`ERR-R4-03`)**: Added Next.js `instrumentation.ts` (`register()` on `NEXT_RUNTIME === "nodejs"`) invoking `SettingsManager.bootstrapMaintenanceSchedulerOnBoot()` to hydrate settings and start the periodic storage maintenance scheduler automatically on Port `3000` startup.
- **`AUDIT-09` (`ERR-R4-04`, `ERR-R4-05`)**: Added `console.warn` + `storageMode`, `degraded: true`, `code: "AUDIT_STORAGE_DEGRADED"`, `warning`, and `X-DataStack-Audit-Degraded: 1` header in `core/auth/auditLog.ts` & `GET /api/audit`; wired `POST /internal/admin-session-sync` in `server/index.js`, `syncInMemoryAdminSessionState` / `checkInMemorySessionRevocation` in `server/filesManager.js`, and `notifyIngestionAdminSessionSync` in `core/auth/adminUsers.ts` so `!dbPool` Files Manager sessions revoke immediately on logout/disable/delete/credential rotation.
- **Verified**: `5/5` `tests/r4-phase01-security-boot.test.ts` + `85/85` regression tests (`phase01..04`, `phase07`, `files-manager`, `r3-phase01`, `r3-phase02`, `v2-phase03`, `v2-phase04` = `90/90` PASS), `npm run build`, `npm run lint` (`0` errors). `0` forbidden UI/SDK/Docker/ingestion-batch files modified.
- **`R4-Phase-02` Complete & Verified (`2026-11-21`)** (`AUDIT-05`, `AUDIT-06`, `AUDIT-07`, `AUDIT-10`, `AUDIT-11`, `AUDIT-12`; `ERR-R4-06`–`ERR-R4-08`):
- **`AUDIT-05` (`ERR-R4-06`)**: Enforced `DATASTACK_MAX_BATCH_ITEMS` (default `1000`, clamped `1..10000`, documented in `.env.example`) across `POST /api/events` (`400 BATCH_LIMIT_EXCEEDED`, evaluated before storage admission scan) and Socket.IO `batch_sync` (`server/socket/handlers.js`).
- **`AUDIT-06`**: Added `FILES_MANAGER_MUTATION` and `DATA_EXPORT_DOWNLOADED` to `core/auth/auditLog.ts` and `ACTION_FILTER_OPTIONS` in `components/sections/AuditLogViewer.tsx`; wired export download audit logs in `app/api/export/route.ts` and CloudCmd HTTP/Socket mutation audit logs (`admin_audit_logs` PG + RAM fallback) in `server/filesManager.js`.
- **`AUDIT-07` (`ERR-R4-07`)**: Isolated corrupt `.csv.gz` decompression failures inside `GET /api/data` (`app/api/data/route.ts`) so healthy client partitions return HTTP `200` alongside `corruptPartitions` (`CORRUPT_GZIP_PARTITION`) and `warnings`.
- **`AUDIT-10`**: Added `serializeClientResponse()` in `app/api/clients/[id]/route.ts` stripping `secret` and non-rotation `token` from `PATCH /api/clients/[id]` responses while exposing `tokenPreview` (and full `token` only on `action === "regenerate_token"`).
- **`AUDIT-11`**: Made `POST /internal/client-sync` (`action === "updated"`) in `server/index.js` disconnect live sockets only on security-relevant updates (`status === "revoked"`, expired `expiresAt`, token rotation, `allowedIps` change, or live socket credential mismatch), keeping metadata-only updates (`name`, `rateLimit`, `storageQuotaMb`) connected without disruption.
- **`AUDIT-12` (`ERR-R4-08`)**: Converted `uploadFileToS3` (`core/Storage/s3Archive.ts`) to streaming SHA-256 pre-pass + `Readable.toWeb(fs.createReadStream(...))` (`duplex: "half"`) with `.destroy()` teardown, and hardened `CSVStorageManager.openPartitionTextStream` + `countCsvDataRowsStreamAsync` (`public static`) with `for await` + `.destroy()` cleanup.
- **Verified**: `5/5` `tests/r4-phase02-ingestion-audit-streaming.test.ts` PASS, regression suites (`r4-phase01`, `files-manager`, `phase02-04`, `v2-phase02-03`, `r3-phase02`, `r3-phase04`) PASS, Playwright `/vcon/audit` (`0` console errors, new action filter options visible), `npm run build`, and `npm run lint` (`0` errors). `0` forbidden SDK/Docker/Nginx/middleware/auth/unrelated UI files touched.
- **`R4-Phase-03` Complete & Verified (`2026-11-21`)** (`AUDIT-02`, `AUDIT-08`, `AUDIT-13`, `AUDIT-20`; `ERR-R4-09`):
- **`AUDIT-02`**: Added `"prepare": "npm run build"` and `"devDependencies": { "typescript": "^5.9.3" }` to `sdk/node/datastack/package.json` and compiled `sdk/node/datastack/dist/{index,spool,types}.{js,d.ts}` so CommonJS `require('./sdk/node/datastack')` works cleanly out of the box.
- **`AUDIT-20`**: Added atomic `DiskSpooler.ackAndRequeueFront(ackedCount, unprocessed)` in `sdk/node/datastack/spool.ts` and wired `deliverChunkWithSplitAndRetry()` + `flushSpool()` in `sdk/node/datastack/index.ts` (with `dist/` rebuilt) so partial-batch rate-limit recoveries commit acknowledged events without dropping or reordering unprocessed envelopes ahead of tail items.
- **`AUDIT-08` (`ERR-R4-09`)**: Added `DATASTACK_ADMIN_EMAIL: "${DATASTACK_ADMIN_EMAIL:-admin@datastack.dev}"` to `services.ingestion.environment` in `docker/docker-compose.yml`; added `proxyMode`/`wsSupported`/`wsHint` detection in `GET /api/files/status` (`app/api/files/status/route.ts`) and rendered a non-blocking `[data-testid="files-manager-proxy-hint"]` strip in `components/sections/FilesManagerWorkspace.tsx` when running standalone without Nginx WebSocket proxying.
- **`AUDIT-13`**: Removed dead `/api/auth/setup/status` whitelist line from `middleware.ts` and passed explicit `role: "owner"`, `memberId: "owner-1"` arguments to `createAdminSessionToken()` in `app/api/auth/setup/route.ts`.
- **Verified**: `4/4` `tests/r4-phase03-sdk-deploy-auth.test.ts` + `43/43` regression checks (`47/47` total PASS across `v2-phase04-node-sdk`, `r3-phase03-sdk-parity`, `phase05-packaging`, `phase07-deployment`, `phase01-auth`, `files-manager`), `node -e "require('./sdk/node/datastack')"` PASS (`2.2.0`), Playwright `/vcon/files` (`0` console errors), `npm run build`, and `npm run lint` (`0` errors). `0` forbidden Storage/Webhook/Client/Team/Profile/Landing files modified.
- **`R4-Phase-04` Complete & Verified (`2026-11-21`)** (`AUDIT-14`, `AUDIT-15`, `AUDIT-16`, `AUDIT-18`, `AUDIT-19`; `ERR-R4-10`, plus `AUDIT-07` non-blocking `corruptPartitions` UI notice):
- **`AUDIT-14`**: Updated `components/sections/TeamWorkspace.tsx` to initialize role from `useVconRole()` (`shellRole || 'viewer'`), preserve distinct `operator` and `admin` `<Select>` options and row values without coercing `operator` to `admin`, label StatStrip `'Operators / Admins'`, and require an explicit `<Modal>` (`"Confirm Team Member Removal"`, `[data-testid="confirm-remove-member"]`) before `DELETE /api/team/${id}`.
- **`AUDIT-15` (`ERR-R4-10`)**: Wired `useVconRole()` + `getVconRoleCapabilities(shellRole)` (`canOperate`) in `app/vcon/sdk/page.tsx` so `viewer` sessions skip `?includeToken=true` requests cleanly (`0` HTTP `403`s) and display inline `[data-testid="sdk-viewer-rbac-banner"]` RBAC guidance while preserving existing `loadingToken` disabled guards.
- **`AUDIT-16`**: Seeded `components/layout/Sidebar.tsx` role badge from `useVconRole()` (`formatShellRoleLabel()`), renamed `'Python SDK'` to `'Client SDKs'`, and added `components/layout/Navbar.tsx` mobile `VCON_LABELS` for `'/vcon/files'` (`'Files Manager'`), `'/vcon/sdk'` (`'Client SDKs'`), `'/vcon/team'` (`'Team & RBAC'`), and `'/vcon/audit'` (`'Audit Log'`).
- **`AUDIT-19`**: Replaced fake Demo fallback constants in `components/sections/ProfileWorkspace.tsx` with neutral `EMPTY_PROFILE` placeholders (`'Loading...'` / `'—'`) until `/api/auth/profile` resolves.
- **`AUDIT-07` UI Hook**: Surfaced non-blocking `[data-testid="data-corrupt-partitions-warning"]` banners inside `components/sections/DataBrowser.tsx` when `GET /api/data` reports `corruptPartitions`.
- **`AUDIT-18`**: Aligned `components/sections/Hero.tsx`, `components/sections/SdkSection.tsx`, and `components/sections/LandingAbout.tsx` around `'Client SDKs'` with side-by-side Python (`datastack v2.2.0`) and Node.js / TypeScript (`@datastack/sdk v2.2.0`) quick-start cards.
- **Verified**: `5/5` `tests/r4-phase04-ui-rbac-copy.test.ts` + `46/46` regression checks (`51/51` total PASS), live headless Chromium desktop (`1440x900`) & mobile (`390x844`) verification across `/`, `/vcon/team`, `/vcon/profile`, `/vcon/data`, `/vcon/files`, `/vcon/audit`, `/vcon/sdk` (`0` console/page errors, `0` viewer `403`s), `npm run build`, and `npm run lint` (`0` errors). `0` forbidden backend/SDK/Docker/Nginx files modified. **All `4 / 4` R4 phases (`19 / 19` actionable findings) complete.**
- **`PREPUSH-FIX-01` Complete & Verified (`2026-11-21`)** (`DEPLOY-BLK-01`, `DEPLOY-BUG-02`–`05`):
- **`DEPLOY-BLK-01`**: Resynchronized `package-lock.json` (`lockfileVersion: 3`) with `package.json` (`cloudcmd ^19.23.2` + full dependency tree) and verified `npm ci --include=dev --dry-run` (`Dockerfile.web`) and `npm ci --omit=dev --dry-run` (`Dockerfile.server`) both exit `0`.
- **`DEPLOY-BUG-02`**: Hardened `docker/coolify-nginx.conf` and `nginx/default.conf` with `ngx_http_realip_module` (`set_real_ip_from` loopback/RFC1918/Docker bridges + IPv6 ULA, `real_ip_header X-Forwarded-For`, `real_ip_recursive on`) so `proxy_set_header X-Real-IP $remote_addr` preserves real external client IPs behind Coolify Traefik without permitting untrusted public IP spoofing.
- **`DEPLOY-BUG-03` & `DEPLOY-BUG-04`**: Added `ARG INGESTION_INTERNAL_URL=http://ingestion:4000` + `ENV INGESTION_INTERNAL_URL` in `docker/Dockerfile.web` `builder` stage for standalone Next.js rewrites, and forwarded optional `DATASTACK_ALLOW_PRIVATE_URLS`, `DATASTACK_MAX_BATCH_ITEMS`, `DATASTACK_RECENT_LIMIT`, `DATASTACK_MAINTENANCE_INTERVAL_MS`, and `DATASTACK_PROXY_MODE` env vars in `docker/docker-compose.yml`.
- **`DEPLOY-BUG-05`**: Updated `sdk/node/datastack/index.ts` (`emitSocketSingle` and `sendBatchChunkInternal`) to call `this.disconnect()` on terminal `DataStackAuthError` rejections (`TOKEN_ROTATED`, `CLIENT_REVOKED`, etc.) and recompiled `sdk/node/datastack/dist/*` so subsequent `client.connect()` performs a fresh handshake (`tests/r3-phase03-sdk-parity.test.ts` `5/5` PASS).
---
@@ -384,4 +430,5 @@ All planned hardening & deployment phases (**Phase-01 through Phase-07**) are **
- Never add per-row telemetry `INSERT` queries to PostgreSQL or add `DROP`/`TRUNCATE` auto-migrations.
- Never move Socket.IO handlers into Next.js `app/api/*` routes.
- When deploying updates to an existing Coolify VPS, always preserve existing `SERVICE_*` environment variables and avoid `docker compose down -v` so `pgdata`, `datastack-data`, and `redisdata` volumes remain intact.
- After any code change, verify with `npm run build`, `npm run lint`, the TS `tests/phase*.test.ts` suite (`npx --yes tsx --test`), and `python3 -m unittest tests/phase05_sdk_test.py`, then synchronize this `MEMORY.md`.
- After any code change, verify with `npm run build`, `npm run lint`, `npm ci --dry-run`, and the test suites, then synchronize this `MEMORY.md`.
+13 -5
View File
@@ -1,4 +1,4 @@
import { listAdminAuditLogsAsync } from "@/core/auth/auditLog";
import { listAdminAuditLogsWithStatusAsync } from "@/core/auth/auditLog";
import { requireAdminApiSession } from "@/lib/adminAuth";
import { NextRequest, NextResponse } from "next/server";
@@ -22,17 +22,25 @@ export async function GET(req: NextRequest) {
const search =
searchParams.get("q") || searchParams.get("search") || undefined;
const logs = await listAdminAuditLogsAsync({
const result = await listAdminAuditLogsWithStatusAsync({
limit,
action,
actorEmail,
search,
});
return NextResponse.json({
logs,
total: logs.length,
const response = NextResponse.json({
logs: result.logs,
total: result.logs.length,
storageMode: result.storageMode,
degraded: result.degraded,
...(result.code ? { code: result.code } : {}),
...(result.warning ? { warning: result.warning } : {}),
});
if (result.degraded) {
response.headers.set("X-DataStack-Audit-Degraded", "1");
}
return response;
} catch (err: unknown) {
const message =
err instanceof Error ? err.message : "Unable to fetch audit logs.";
+2
View File
@@ -145,6 +145,8 @@ export async function POST(request: NextRequest) {
sessionSecret,
Date.now() + SESSION_SECONDS * 1000,
1,
"owner",
"owner-1",
);
const response = NextResponse.json({ success: true });
response.cookies.set(
+19 -9
View File
@@ -17,6 +17,18 @@ const ALLOWED_PATCH_KEYS = new Set([
"status",
]);
function serializeClientResponse<
T extends { secret?: string; token?: string },
>(client: T, options: { includeToken: boolean }) {
const { secret, token, ...clientSummary } = client;
void secret;
return {
...clientSummary,
tokenPreview: token ? `${token.slice(0, 12)}...` : "",
...(options.includeToken && token ? { token } : {}),
};
}
export async function GET(
req: NextRequest,
{ params }: { params: Promise<{ id: string }> },
@@ -36,14 +48,8 @@ export async function GET(
if (!client) {
return NextResponse.json({ error: "Client not found" }, { status: 404 });
}
const { secret, token, ...clientSummary } = client;
void secret;
return NextResponse.json({
client: {
...clientSummary,
tokenPreview: token ? `${token.slice(0, 12)}...` : "",
...(includeToken ? { token } : {}),
},
client: serializeClientResponse(client, { includeToken }),
});
} catch (err: unknown) {
const message = err instanceof Error ? err.message : "Failed to load client";
@@ -132,7 +138,9 @@ export async function PATCH(
},
});
return NextResponse.json({ client: refreshed });
return NextResponse.json({
client: serializeClientResponse(refreshed, { includeToken: true }),
});
} catch (err: unknown) {
const message =
err instanceof Error ? err.message : "Token regeneration failed";
@@ -277,7 +285,9 @@ export async function PATCH(
metadata: updates,
});
return NextResponse.json({ client: updated });
return NextResponse.json({
client: serializeClientResponse(updated, { includeToken: false }),
});
} catch (err: unknown) {
const message = err instanceof Error ? err.message : "Update failed";
const isDbWrite = message.includes("DATABASE_WRITE_FAILED");
+16 -1
View File
@@ -17,6 +17,13 @@ interface DataEvent {
status: "stored" | "buffered";
}
interface CorruptPartitionWarning {
clientId: string;
filename: string;
code: "CORRUPT_GZIP_PARTITION";
error: string;
}
function parseTimestampMs(raw: string): number {
const trimmed = String(raw || "").trim();
if (/^\d+(\.\d+)?$/.test(trimmed)) {
@@ -62,6 +69,7 @@ export async function GET(request: NextRequest) {
const offset = (page - 1) * pageSize;
const selectedRows: DataEvent[] = [];
const eventIds = new Set<string>();
const corruptPartitions: CorruptPartitionWarning[] = [];
let total = 0;
const addIfMatching = (event: DataEvent) => {
@@ -185,7 +193,12 @@ export async function GET(request: NextRequest) {
console.warn(`[API /data] Failed reading partition ${filePath}:`, err);
if (filename.endsWith(".csv.gz")) {
const detail = err instanceof Error ? err.message : String(err);
throw new Error(`Corrupt .csv.gz partition (${filename}): ${detail}`);
corruptPartitions.push({
clientId: storedClientId,
filename,
code: "CORRUPT_GZIP_PARTITION",
error: `Corrupt .csv.gz partition (${filename}): ${detail}`,
});
}
} finally {
if (!parser.destroyed) parser.destroy();
@@ -249,6 +262,8 @@ export async function GET(request: NextRequest) {
page,
pageSize,
pageCount: Math.max(1, Math.ceil(total / pageSize)),
corruptPartitions,
warnings: corruptPartitions.length,
});
} catch (error) {
const message =
+46 -11
View File
@@ -25,6 +25,21 @@ const SINGLE_EVENT_ENVELOPE_KEYS = new Set(['eventId', 'timestamp', 'data', 'cli
const MAX_SINGLE_CLIENT_SCAN_FILES = 10;
const MAX_ALL_CLIENTS_PER_CLIENT_FILES = 2;
const MAX_ALL_CLIENTS_TOTAL_SCAN_FILES = 50;
const DEFAULT_MAX_BATCH_ITEMS = 1000;
const MIN_BATCH_ITEMS = 1;
const MAX_BATCH_ITEMS_CEILING = 10000;
function resolveMaxBatchItems(): number {
const raw = process.env.DATASTACK_MAX_BATCH_ITEMS;
if (raw === undefined || raw.trim() === '') {
return DEFAULT_MAX_BATCH_ITEMS;
}
const parsed = Math.floor(Number(raw));
if (!Number.isFinite(parsed)) {
return DEFAULT_MAX_BATCH_ITEMS;
}
return Math.min(MAX_BATCH_ITEMS_CEILING, Math.max(MIN_BATCH_ITEMS, parsed));
}
function generateFallbackEventId(): string {
return `evt_${Date.now()}_${crypto.randomBytes(6).toString('hex')}`;
@@ -513,6 +528,36 @@ export async function POST(req: NextRequest) {
}
}
// Detect if this is a batch array or direct file-records ingestion request
const batchItems: unknown[] | null = Array.isArray(body)
? body
: Array.isArray(bodyObj.batch)
? (bodyObj.batch as unknown[])
: Array.isArray(bodyObj.records)
? (bodyObj.records as unknown[])
: bodyObj.data === undefined && Array.isArray(bodyObj.events)
? (bodyObj.events as unknown[])
: null;
if (batchItems !== null) {
const maxBatchItems = resolveMaxBatchItems();
if (batchItems.length > maxBatchItems) {
return NextResponse.json(
{
success: false,
accepted: 0,
processed: 0,
rejected: batchItems.length,
duplicates: 0,
code: 'BATCH_LIMIT_EXCEEDED',
error: `BATCH_LIMIT_EXCEEDED: Batch size (${batchItems.length}) exceeds maximum allowed ${maxBatchItems} items per request.`,
maxBatchItems,
},
{ status: 400 },
);
}
}
const storageAdmission = await CSVStorageManager.checkStorageAdmissionAsync(
clientId,
registeredClient.storageQuotaMb,
@@ -550,18 +595,8 @@ export async function POST(req: NextRequest) {
settings.maxEventsPerSec ||
100;
// Detect if this is a batch array or direct file-records ingestion request
const batchItems: unknown[] | null = Array.isArray(body)
? body
: Array.isArray(bodyObj.batch)
? (bodyObj.batch as unknown[])
: Array.isArray(bodyObj.records)
? (bodyObj.records as unknown[])
: bodyObj.data === undefined && Array.isArray(bodyObj.events)
? (bodyObj.events as unknown[])
: null;
if (batchItems !== null) {
let processed = 0;
let rejected = 0;
let duplicates = 0;
+40 -2
View File
@@ -1,4 +1,5 @@
import { SettingsManager } from "@/core/adminSettings/settingsManager";
import { recordAdminAuditLogAsync } from "@/core/auth/auditLog";
import { DataBufferEngine } from "@/core/data/buffer";
import { SDKManager } from "@/core/sdk/sdkManager";
import { getInternalIngestionHeaders } from "@/core/Security/security";
@@ -6,7 +7,7 @@ import {
CSVStorageManager,
type StorageFileMeta,
} from "@/core/Storage/csvStorage";
import { requireAdminApiSession } from "@/lib/adminAuth";
import { extractClientIp, requireAdminApiSession } from "@/lib/adminAuth";
import { parse } from "csv-parse";
import { NextRequest, NextResponse } from "next/server";
import { createReadStream } from "node:fs";
@@ -296,6 +297,24 @@ export async function GET(request: NextRequest) {
: file.compressed && includeHeader
? "application/gzip"
: "text/csv; charset=utf-8";
await recordAdminAuditLogAsync({
actorEmail: auth.auth.session.email,
actorRole: auth.auth.session.role || "owner",
action: "DATA_EXPORT_DOWNLOADED",
target: file.clientId,
ipAddress: extractClientIp(request),
metadata: {
mode: "single_file",
clientId: file.clientId,
filename: file.filename,
downloadName,
format,
includeHeader,
fileCount: 1,
},
});
return new NextResponse(bodyStream, {
status: 200,
headers: {
@@ -313,6 +332,25 @@ export async function GET(request: NextRequest) {
const safeClientId = sanitizedClientId === "all" ? "all" : sanitizedClientId;
const baseName =
clientId === "all" ? "datastack_export" : `${safeClientId}_export`;
const downloadName = `${baseName}_${datePart}.${format}`;
await recordAdminAuditLogAsync({
actorEmail: auth.auth.session.email,
actorRole: auth.auth.session.role || "owner",
action: "DATA_EXPORT_DOWNLOADED",
target: clientId,
ipAddress: extractClientIp(request),
metadata: {
mode: "filtered_range",
clientId,
from: from || null,
to: to || null,
format,
includeHeader,
fileCount: files.length,
filename: downloadName,
},
});
return new NextResponse(makeExportStream(files, format, includeHeader), {
status: 200,
@@ -323,7 +361,7 @@ export async function GET(request: NextRequest) {
: format === "ndjson"
? "application/x-ndjson; charset=utf-8"
: "text/csv; charset=utf-8",
"Content-Disposition": `attachment; filename="${baseName}_${datePart}.${format}"`,
"Content-Disposition": `attachment; filename="${downloadName}"`,
"Cache-Control": "no-store",
},
});
+115
View File
@@ -0,0 +1,115 @@
import { requireAdminApiSession } from '@/lib/adminAuth';
import { NextRequest, NextResponse } from 'next/server';
export const dynamic = 'force-dynamic';
function resolveIngestionBaseUrl(): string {
return (process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000').replace(
/\/+$/,
'',
);
}
function resolveFilesProxyStatus(req: NextRequest): {
proxyMode: 'nginx' | 'standalone';
wsSupported: boolean;
wsHint: string | null;
} {
const configuredMode = process.env.DATASTACK_PROXY_MODE?.trim().toLowerCase();
const hasReverseProxyHeaders = Boolean(
req.headers.get('x-forwarded-host') ||
req.headers.get('x-real-ip') ||
req.headers.get('x-forwarded-proto'),
);
if (configuredMode === 'nginx' || hasReverseProxyHeaders) {
return {
proxyMode: 'nginx',
wsSupported: true,
wsHint: null,
};
}
return {
proxyMode: 'standalone',
wsSupported: false,
wsHint:
'Direct Next.js HTTP rewrite active without Nginx WebSocket proxying. Files Manager operates over HTTP polling.',
};
}
export async function GET(req: NextRequest) {
const auth = await requireAdminApiSession(req);
if (!auth.valid) {
return auth.response;
}
const role = auth.auth.session.role || 'owner';
const readOnly = role === 'viewer';
const proxyInfo = resolveFilesProxyStatus(req);
const targetUrl = `${resolveIngestionBaseUrl()}/vcon/files-engine/_status`;
try {
const cookieHeader = req.headers.get('cookie') || '';
const upstream = await fetch(targetUrl, {
method: 'GET',
headers: cookieHeader ? { cookie: cookieHeader } : undefined,
cache: 'no-store',
signal: AbortSignal.timeout(2500),
});
if (upstream.status === 401) {
return NextResponse.json(
{ ok: false, offline: false, error: 'Authentication required.' },
{ status: 401, headers: { 'Cache-Control': 'no-store, private' } },
);
}
const payload = (await upstream.json().catch(() => ({}))) as {
ok?: boolean;
engine?: string;
role?: string;
readOnly?: boolean;
error?: string;
};
if (upstream.ok && payload.ok) {
return NextResponse.json(
{
ok: true,
offline: false,
engine: payload.engine || 'cloudcmd',
role: payload.role || role,
readOnly: Boolean(payload.readOnly ?? readOnly),
...proxyInfo,
},
{ status: 200, headers: { 'Cache-Control': 'no-store, private' } },
);
}
return NextResponse.json(
{
ok: false,
offline: true,
role,
readOnly,
...proxyInfo,
error:
payload.error ||
`Files Manager engine returned HTTP ${upstream.status}. Ensure the Port 4000 ingestion server is running.`,
},
{ status: 200, headers: { 'Cache-Control': 'no-store, private' } },
);
} catch {
return NextResponse.json(
{
ok: false,
offline: true,
role,
readOnly,
...proxyInfo,
error:
'Files Manager engine on Port 4000 is not reachable. Start both services with "npm run dev" / "npm run dev:all" or Docker Compose.',
},
{ status: 200, headers: { 'Cache-Control': 'no-store, private' } },
);
}
}
+39 -17
View File
@@ -4,6 +4,10 @@ import {
} from "@/core/adminSettings/settingsManager";
import { recordAdminAuditLogAsync } from "@/core/auth/auditLog";
import { SecretVaultError } from "@/core/Security/secretVault";
import {
SSRFBlockedError,
validateOutboundUrlSync,
} from "@/core/Security/ssrfGuard";
import { extractClientIp, requireAdminApiSession } from "@/lib/adminAuth";
import { NextRequest, NextResponse } from "next/server";
@@ -209,16 +213,20 @@ export async function POST(req: NextRequest) {
if (body.s3Endpoint !== undefined) {
const ep = parseOptionalString(body.s3Endpoint, "s3Endpoint", 512);
if (ep) {
let parsedEp: URL;
try {
parsedEp = new URL(
/^https?:\/\//i.test(ep) ? ep : `https://${ep}`,
const normalizedEp = /^https?:\/\//i.test(ep) ? ep : `https://${ep}`;
const check = validateOutboundUrlSync(normalizedEp, {
fieldName: "s3Endpoint",
});
if (!check.allowed) {
if (check.code === "SSRF_BLOCKED_HOST") {
throw new SSRFBlockedError(
check.error || "s3Endpoint target host is blocked.",
check.hostname || "",
);
}
throw new Error(
check.error || "s3Endpoint must be a valid http(s) URL.",
);
} catch {
throw new Error("s3Endpoint must be a valid http(s) URL.");
}
if (parsedEp.protocol !== "http:" && parsedEp.protocol !== "https:") {
throw new Error("s3Endpoint must use http:// or https://.");
}
}
validated.s3Endpoint = ep;
@@ -270,14 +278,19 @@ export async function POST(req: NextRequest) {
if (body.webhookUrl !== undefined) {
const wh = parseOptionalString(body.webhookUrl, "webhookUrl", 1024);
if (wh) {
let parsedWh: URL;
try {
parsedWh = new URL(wh);
} catch {
throw new Error("webhookUrl must be a valid http(s) URL.");
}
if (parsedWh.protocol !== "http:" && parsedWh.protocol !== "https:") {
throw new Error("webhookUrl must use http:// or https://.");
const check = validateOutboundUrlSync(wh, {
fieldName: "webhookUrl",
});
if (!check.allowed) {
if (check.code === "SSRF_BLOCKED_HOST") {
throw new SSRFBlockedError(
check.error || "webhookUrl target host is blocked.",
check.hostname || "",
);
}
throw new Error(
check.error || "webhookUrl must be a valid http(s) URL.",
);
}
}
validated.webhookUrl = wh;
@@ -334,6 +347,15 @@ export async function POST(req: NextRequest) {
validated.jwtSecret = body.jwtSecret.trim();
}
} catch (validationErr: unknown) {
if (validationErr instanceof SSRFBlockedError) {
return NextResponse.json(
{
error: validationErr.message,
code: validationErr.code,
},
{ status: 400 },
);
}
return NextResponse.json(
{
error:
+15 -10
View File
@@ -5,6 +5,7 @@ import {
decryptSecret,
SecretVaultError,
} from "@/core/Security/secretVault";
import { validateOutboundUrlSync } from "@/core/Security/ssrfGuard";
import { extractClientIp, requireAdminApiSession } from "@/lib/adminAuth";
import { NextRequest, NextResponse } from "next/server";
@@ -57,17 +58,20 @@ export async function POST(req: NextRequest) {
);
}
try {
const parsedUrl = new URL(webhookUrl);
if (parsedUrl.protocol !== "http:" && parsedUrl.protocol !== "https:") {
throw new Error("Webhook URL must use http:// or https://.");
}
} catch {
const urlCheck = validateOutboundUrlSync(webhookUrl, {
fieldName: "Webhook URL",
});
if (!urlCheck.allowed) {
return NextResponse.json(
{
success: false,
code: "VALIDATION_ERROR",
error: "Webhook URL must be a valid http:// or https:// endpoint.",
code:
urlCheck.code === "SSRF_BLOCKED_HOST"
? "SSRF_BLOCKED_HOST"
: "VALIDATION_ERROR",
error:
urlCheck.error ||
"Webhook URL must be a valid http:// or https:// endpoint.",
},
{ status: 400 },
);
@@ -105,14 +109,15 @@ export async function POST(req: NextRequest) {
});
if (!result.ok) {
const isSsrfBlocked = result.code === "SSRF_BLOCKED_HOST";
return NextResponse.json(
{
success: false,
code: "WEBHOOK_DELIVERY_FAILED",
code: isSsrfBlocked ? "SSRF_BLOCKED_HOST" : "WEBHOOK_DELIVERY_FAILED",
error: result.error || "Webhook target failed to accept test alert.",
status: result.status,
},
{ status: 502 },
{ status: isSsrfBlocked ? 400 : 502 },
);
}
+5 -1
View File
@@ -71,6 +71,10 @@ export async function POST(req: NextRequest) {
const result = await testS3Connection(config);
if (!result.ok) {
const isClientError =
result.code === "S3_INVALID_CONFIG" ||
result.code === "SSRF_BLOCKED_HOST" ||
result.code === "INVALID_URL";
return NextResponse.json(
{
success: false,
@@ -78,7 +82,7 @@ export async function POST(req: NextRequest) {
error: result.error || "S3 connectivity check failed.",
...(result.status !== undefined ? { s3Status: result.status } : {}),
},
{ status: result.code === "S3_INVALID_CONFIG" ? 400 : 502 },
{ status: isClientError ? 400 : 502 },
);
}
+5
View File
@@ -0,0 +1,5 @@
import { FilesManagerWorkspace } from '@/components/sections/FilesManagerWorkspace';
export default function FilesManagerPage() {
return <FilesManagerWorkspace />;
}
+26 -3
View File
@@ -1,6 +1,7 @@
'use client';
import { PageShell } from '@/components/layout/PageShell';
import { useVconRole } from '@/components/layout/VconShell';
import { Button } from '@/components/ui/Button';
import { Card } from '@/components/ui/Card';
import { CodeBlock } from '@/components/ui/CodeBlock';
@@ -9,7 +10,10 @@ import { Panel } from '@/components/ui/Panel';
import { StatStrip } from '@/components/ui/StatStrip';
import { useToast } from '@/components/ui/Toast';
import { getCached, setCached } from '@/lib/vconCache';
import { resolvePublicEndpoints } from '@/lib/vconUiHelpers';
import {
getVconRoleCapabilities,
resolvePublicEndpoints,
} from '@/lib/vconUiHelpers';
import { useEffect, useState } from 'react';
interface SdkClientItem {
@@ -24,6 +28,8 @@ type SdkLanguageTab = 'python' | 'node' | 'curl';
export default function SdkGuidePage() {
const { success, error } = useToast();
const shellRole = useVconRole();
const { canOperate } = getVconRoleCapabilities(shellRole);
const [activeTab, setActiveTab] = useState<SdkLanguageTab>('python');
const [clients, setClients] = useState<SdkClientItem[]>(() =>
getCached<SdkClientItem[]>('vcon:clients', [])
@@ -83,8 +89,9 @@ export default function SdkGuidePage() {
}, []);
useEffect(() => {
if (!selectedClientId) {
if (!selectedClientId || !canOperate) {
setSelectedClientToken('');
setLoadingToken(false);
return;
}
@@ -121,7 +128,7 @@ export default function SdkGuidePage() {
return () => {
cancelled = true;
};
}, [selectedClientId]);
}, [selectedClientId, canOperate]);
const activeClient =
clients.find((c) => c.clientId === selectedClientId) || clients[0];
@@ -454,6 +461,22 @@ curl -X POST "${apiUrl}/api/events" \\
</Card>
)}
{!canOperate && (
<div
role="status"
data-testid="sdk-viewer-rbac-banner"
className="mb-3 border border-[var(--warning)]/40 bg-[var(--warning)]/10 px-3 py-2 text-[11.5px] text-[var(--text)]"
>
Read-only <span className="font-mono uppercase">{shellRole}</span> session:
Operator or Owner role required to reveal live client JWT tokens. Snippets below
safely use placeholder{' '}
<code className="font-mono text-[var(--accent)]">
YOUR_SIGNED_CLIENT_JWT_TOKEN
</code>
.
</div>
)}
<Panel
title={`Step 1 — ${activeConfigLabel}`}
className="w-full"
+1190 -649
View File
File diff suppressed because it is too large. Load diff
+4 -1
View File
@@ -17,8 +17,11 @@ const VCON_LABELS: Record<string, string> = {
'/vcon/clients': 'Clients',
'/vcon/export': 'Export',
'/vcon/storage': 'Storage',
'/vcon/sdk': 'Python SDK',
'/vcon/files': 'Files Manager',
'/vcon/sdk': 'Client SDKs',
'/vcon/simulator': 'Simulator',
'/vcon/team': 'Team & RBAC',
'/vcon/audit': 'Audit Log',
'/vcon/profile': 'Profile',
'/vcon/settings': 'Settings',
};
+17 -5
View File
@@ -1,5 +1,6 @@
'use client';
import { useVconRole } from '@/components/layout/VconShell';
import { Button } from '@/components/ui/Button';
import { Icon } from '@/components/ui/Icon';
import { cn } from '@/lib/utils';
@@ -18,6 +19,14 @@ interface CachedProfile {
role?: string;
}
function formatShellRoleLabel(role?: string | null): string {
const clean = String(role || 'viewer').trim().toLowerCase();
if (clean === 'owner') return 'Owner';
if (clean === 'admin') return 'Admin';
if (clean === 'operator') return 'Operator';
return 'Viewer';
}
const NAVIGATION_SECTIONS = [
{
group: 'Main',
@@ -27,12 +36,13 @@ const NAVIGATION_SECTIONS = [
{ name: 'Clients', href: '/vcon/clients', icon: 'Key' },
{ name: 'Export', href: '/vcon/export', icon: 'Download' },
{ name: 'Storage', href: '/vcon/storage', icon: 'HardDrive' },
{ name: 'Files Manager', href: '/vcon/files', icon: 'FolderOpen' },
],
},
{
group: 'System',
items: [
{ name: 'Python SDK', href: '/vcon/sdk', icon: 'FileCode2' },
{ name: 'Client SDKs', href: '/vcon/sdk', icon: 'FileCode2' },
{ name: 'Simulator', href: '/vcon/simulator', icon: 'Play' },
{ name: 'Team & RBAC', href: '/vcon/team', icon: 'Users' },
{ name: 'Audit Log', href: '/vcon/audit', icon: 'FileText' },
@@ -47,13 +57,15 @@ export const Sidebar: React.FC<SidebarProps> = ({
onClose,
}) => {
const pathname = usePathname();
const shellRole = useVconRole();
const initialRoleLabel = formatShellRoleLabel(shellRole);
const [isSigningOut, setIsSigningOut] = useState(false);
const [theme, setTheme] = useState<'dark' | 'light'>('dark');
const [adminEmail, setAdminEmail] = useState<string>(
() => getCached<CachedProfile>('vcon:profile', {}).email || 'Admin'
);
const [adminRole, setAdminRole] = useState<string>(
() => getCached<CachedProfile>('vcon:profile', {}).role || 'Owner'
() => getCached<CachedProfile>('vcon:profile', {}).role || initialRoleLabel
);
useEffect(() => {
@@ -96,17 +108,17 @@ export const Sidebar: React.FC<SidebarProps> = ({
setCached('vcon:profile', data.profile);
setCached('vcon:profile-loaded', true);
setAdminEmail(data.profile.email);
setAdminRole(data.profile.role || 'Owner');
setAdminRole(data.profile.role || initialRoleLabel);
}
})
.catch(() => {
// Keep cached or default identity label if offline
// Keep cached or shell role identity label if offline
});
return () => {
window.removeEventListener('datastack:profile-updated', handleProfileUpdated);
};
}, [pathname]);
}, [pathname, initialRoleLabel]);
const toggleTheme = () => {
const nextTheme = theme === 'dark' ? 'light' : 'dark';
+2
View File
@@ -50,6 +50,8 @@ const ACTION_FILTER_OPTIONS = [
{ value: 'STORAGE_FLUSH_TRIGGERED', label: 'STORAGE_FLUSH_TRIGGERED' },
{ value: 'S3_ARCHIVE_TRIGGERED', label: 'S3_ARCHIVE_TRIGGERED' },
{ value: 'WEBHOOK_TEST_TRIGGERED', label: 'WEBHOOK_TEST_TRIGGERED' },
{ value: 'FILES_MANAGER_MUTATION', label: 'FILES_MANAGER_MUTATION' },
{ value: 'DATA_EXPORT_DOWNLOADED', label: 'DATA_EXPORT_DOWNLOADED' },
];
export function AuditLogViewer() {
+34
View File
@@ -36,6 +36,13 @@ interface Client {
name: string;
}
interface CorruptPartitionWarning {
clientId: string;
filename: string;
code?: string;
error: string;
}
const PAGE_SIZE = 12;
function formatEventPayload(raw: string): string {
@@ -70,6 +77,7 @@ export function DataBrowser() {
const [filtersReady, setFiltersReady] = useState(false);
const [refreshing, setRefreshing] = useState(false);
const [errorMessage, setErrorMessage] = useState('');
const [corruptPartitions, setCorruptPartitions] = useState<CorruptPartitionWarning[]>([]);
const [selectedEvent, setSelectedEvent] = useState<StreamEvent | null>(null);
const fetchEvents = useCallback(async () => {
@@ -94,6 +102,9 @@ export function DataBrowser() {
setEvents(data.events || []);
setTotal(data.total || 0);
setPageCount(data.pageCount || 1);
setCorruptPartitions(
Array.isArray(data.corruptPartitions) ? data.corruptPartitions : []
);
setErrorMessage('');
} catch (error) {
setErrorMessage(error instanceof Error ? error.message : 'Unable to load events');
@@ -271,6 +282,29 @@ export function DataBrowser() {
)}
</Card>
{corruptPartitions.length > 0 && (
<div
role="status"
data-testid="data-corrupt-partitions-warning"
className="border border-[var(--warning)]/40 bg-[var(--warning)]/10 px-3 py-2 text-[11.5px] text-[var(--text)]"
>
<div className="font-medium">
{corruptPartitions.length} corrupt .csv.gz partition
{corruptPartitions.length === 1 ? '' : 's'} skipped while loading healthy events.
</div>
<ul className="mt-1 flex flex-col gap-0.5 font-mono text-[10.5px] text-[var(--text-muted)]">
{corruptPartitions.slice(0, 5).map((item) => (
<li key={`${item.clientId}:${item.filename}`}>
{item.clientId}/{item.filename} — {item.error}
</li>
))}
{corruptPartitions.length > 5 && (
<li>+{corruptPartitions.length - 5} more unreadable archive(s)</li>
)}
</ul>
</div>
)}
<Panel
title="Events"
meta={`${total.toLocaleString()} · ${page}/${pageCount}`}
@@ -0,0 +1,210 @@
'use client';
import { PageShell } from '@/components/layout/PageShell';
import { useVconRole } from '@/components/layout/VconShell';
import { Button } from '@/components/ui/Button';
import { Icon } from '@/components/ui/Icon';
import { getVconRoleCapabilities } from '@/lib/vconUiHelpers';
import React, { useCallback, useEffect, useState } from 'react';
type EngineStatus = 'checking' | 'online' | 'offline';
interface FilesEngineStatusResponse {
ok?: boolean;
engine?: string;
role?: string;
readOnly?: boolean;
proxyMode?: 'nginx' | 'standalone';
wsSupported?: boolean;
wsHint?: string | null;
error?: string;
code?: string;
}
const FILES_ENGINE_STATUS_URL = '/api/files/status';
const FILES_ENGINE_FRAME_URL = '/vcon/files-engine';
export function FilesManagerWorkspace() {
const currentRole = useVconRole();
const { canOperate } = getVconRoleCapabilities(currentRole);
const [status, setStatus] = useState<EngineStatus>('checking');
const [statusError, setStatusError] = useState<string>('');
const [wsHint, setWsHint] = useState<string | null>(null);
const [engineReadOnly, setEngineReadOnly] = useState<boolean>(!canOperate);
const [frameKey, setFrameKey] = useState<number>(1);
const [probing, setProbing] = useState<boolean>(false);
const checkEngineStatus = useCallback(async (reloadFrame = false) => {
setProbing(true);
setStatusError('');
try {
const response = await fetch(FILES_ENGINE_STATUS_URL, {
method: 'GET',
cache: 'no-store',
credentials: 'same-origin',
});
if (response.status === 401) {
window.location.assign('/vcon/login?next=%2Fvcon%2Ffiles');
return;
}
const data = (await response.json().catch(() => ({}))) as FilesEngineStatusResponse;
setWsHint(typeof data.wsHint === 'string' && data.wsHint ? data.wsHint : null);
if (response.ok && data.ok) {
setEngineReadOnly(Boolean(data.readOnly));
setStatus('online');
if (reloadFrame) {
setFrameKey((prev) => prev + 1);
}
return;
}
setStatus('offline');
setStatusError(
data.error ||
`Files Manager engine returned HTTP ${response.status}. Ensure the Port 4000 ingestion server is running.`
);
} catch {
setStatus('offline');
setStatusError(
'Cannot reach the Port 4000 Files Manager engine. Start both services with "npm run dev:all" or Docker Compose.'
);
} finally {
setProbing(false);
}
}, []);
useEffect(() => {
void checkEngineStatus(false);
}, [checkEngineStatus]);
const effectiveReadOnly = !canOperate || engineReadOnly;
return (
<PageShell
title="Files Manager"
actions={
<>
<span
className="border border-[var(--border)] bg-[var(--surface-2)] px-2 py-1 font-mono text-[10.5px] uppercase tracking-wider text-[var(--text-muted)]"
data-testid="files-manager-access-badge"
>
{effectiveReadOnly
? `Read-only (${currentRole})`
: `Full Access (${currentRole})`}
</span>
<Button
size="xs"
variant="secondary"
icon="RotateCw"
loading={probing}
disabled={probing}
onClick={() => void checkEngineStatus(true)}
>
Reload
</Button>
{status === 'online' && (
<a
href={FILES_ENGINE_FRAME_URL}
target="_blank"
rel="noopener noreferrer"
className="inline-flex h-6 items-center gap-1.5 border border-[var(--border)] bg-[var(--surface)] px-2 text-[11px] font-medium text-[var(--text)] transition-colors hover:border-[var(--border-strong)] hover:bg-[var(--surface-2)]"
>
<Icon name="ExternalLink" size={12} />
<span>Full View</span>
</a>
)}
</>
}
>
<div className="flex flex-wrap items-center justify-between gap-2 border border-[var(--border)] bg-[var(--surface)] px-3 py-1.5 text-[11.5px] text-[var(--text-muted)]">
<div className="flex items-center gap-2">
<Icon name="FolderOpen" size={13} className="shrink-0 text-[var(--accent)]" />
<span>
Root: <code className="font-mono text-[var(--text)]">/storage/data</code> — browse, edit, rename, upload, download, and ZIP/UNZIP client partitions.
</span>
</div>
<span className="font-mono text-[10.5px] text-[var(--text-dim)]">
{effectiveReadOnly
? 'Viewer mode: browse, view & download only'
: 'Active *.lock files are protected; avoid editing live UTC-today CSVs during peak flush'}
</span>
</div>
{status === 'online' && wsHint && (
<div
className="flex items-center gap-2 border border-[var(--border)] bg-[var(--surface-2)] px-3 py-1.5 text-[11px] text-[var(--text-muted)]"
data-testid="files-manager-proxy-hint"
>
<Icon name="Info" size={12} className="shrink-0 text-[var(--accent)]" />
<span>{wsHint}</span>
</div>
)}
<div
className="w-full overflow-hidden border border-[var(--border)] bg-[var(--surface)]"
data-testid="files-manager-container"
>
{status === 'checking' && (
<div className="flex h-[calc(100dvh-180px)] min-h-[480px] flex-col items-center justify-center gap-2.5 p-6 text-center">
<Icon name="RotateCw" size={18} className="animate-spin text-[var(--accent)]" />
<p className="text-[12.5px] font-medium text-[var(--text)]">
Connecting to Files Manager engine...
</p>
<p className="text-[11.5px] text-[var(--text-dim)]">
Verifying admin session and storage root on Port 4000
</p>
</div>
)}
{status === 'offline' && (
<div
className="flex h-[calc(100dvh-180px)] min-h-[480px] flex-col items-center justify-center gap-3 p-6 text-center"
data-testid="files-manager-offline"
>
<div className="flex h-9 w-9 items-center justify-center border border-[var(--border-strong)] bg-[var(--surface-2)] text-[var(--text-muted)]">
<Icon name="HardDrive" size={18} />
</div>
<div className="max-w-md space-y-1">
<h2 className="text-[13px] font-medium text-[var(--text)]">
Files Manager Engine Unavailable
</h2>
<p className="text-[12px] text-[var(--text-muted)]">
{statusError ||
'The Port 4000 ingestion and files engine is currently offline.'}
</p>
</div>
<div className="border border-[var(--border)] bg-[var(--surface-2)] px-3 py-2 font-mono text-[11px] text-[var(--text-dim)]">
Local: npm run dev:all &nbsp;|&nbsp; Production: docker compose up
</div>
<Button
size="sm"
variant="primary"
icon="RotateCw"
loading={probing}
disabled={probing}
onClick={() => void checkEngineStatus(true)}
>
Retry Connection
</Button>
</div>
)}
{status === 'online' && (
<iframe
key={frameKey}
src={FILES_ENGINE_FRAME_URL}
title="DataStack Files Manager"
data-testid="files-manager-iframe"
className="block h-[calc(100dvh-180px)] min-h-[540px] w-full border-0 bg-[var(--surface)]"
/>
)}
</div>
</PageShell>
);
}
export default FilesManagerWorkspace;
+1 -1
View File
@@ -21,7 +21,7 @@ export const Hero: React.FC = () => (
Explore features
</Link>
<Link href="/#sdk" className={buttonClassNames({ variant: 'secondary', size: 'sm' })}>
Python SDK
Client SDKs
</Link>
</div>
<ActivityPreview />
+1 -1
View File
@@ -9,7 +9,7 @@ export const LandingAbout: React.FC = () => (
<p className="text-[13px] text-[var(--text-muted)]">Start with the SDK guide and make the workspace your own.</p>
</div>
<Link href="/#sdk" className={buttonClassNames({ variant: 'primary', size: 'sm' })}>
View Python SDK
View Client SDKs
</Link>
</section>
);
+26 -11
View File
@@ -27,10 +27,10 @@ interface AdminProfileData {
sessionExpiresAt: string;
}
const DEFAULT_PROFILE: AdminProfileData = {
email: 'admin@datastack.dev',
role: 'Owner',
storageMode: 'In-Memory',
const EMPTY_PROFILE: AdminProfileData = {
email: '',
role: '',
storageMode: '',
createdAt: '',
sessionExpiresAt: '',
};
@@ -38,10 +38,10 @@ const DEFAULT_PROFILE: AdminProfileData = {
export function ProfileWorkspace() {
const { success, error } = useToast();
const [profile, setProfile] = useState<AdminProfileData>(() =>
getCached<AdminProfileData>('vcon:profile', DEFAULT_PROFILE)
getCached<AdminProfileData>('vcon:profile', EMPTY_PROFILE)
);
const [email, setEmail] = useState(
() => getCached<AdminProfileData>('vcon:profile', DEFAULT_PROFILE).email
() => getCached<AdminProfileData>('vcon:profile', EMPTY_PROFILE).email
);
const [currentPassword, setCurrentPassword] = useState('');
const [newPassword, setNewPassword] = useState('');
@@ -183,12 +183,26 @@ export function ProfileWorkspace() {
>
<StatStrip
items={[
{ label: 'Administrator', value: profile.email },
{ label: 'Role', value: profile.role, dot: 'ok' },
{ label: 'Auth Store', value: profile.storageMode },
{
label: 'Administrator',
value: profile.email || (loading ? 'Loading...' : '—'),
},
{
label: 'Role',
value: profile.role || (loading ? 'Loading...' : '—'),
dot: profile.role ? 'ok' : undefined,
},
{
label: 'Auth Store',
value: profile.storageMode || (loading ? 'Loading...' : '—'),
},
{
label: 'Session Expires',
value: formatSessionExpiryLabel(profile.sessionExpiresAt),
value: profile.sessionExpiresAt
? formatSessionExpiryLabel(profile.sessionExpiresAt)
: loading
? 'Loading...'
: 'Active',
},
]}
/>
@@ -200,6 +214,7 @@ export function ProfileWorkspace() {
<Input
compact
type="email"
placeholder={loading ? 'Loading account email...' : 'admin@company.com'}
value={email}
onChange={(event) => setEmail(event.target.value)}
required
@@ -208,7 +223,7 @@ export function ProfileWorkspace() {
<SettingsField label="Access role">
<Input
compact
value={profile.role}
value={profile.role || (loading ? 'Loading...' : '—')}
readOnly
className="border-[var(--border)] bg-[var(--surface-2)] text-[var(--text-muted)]"
/>
+18 -11
View File
@@ -4,25 +4,32 @@ import React from 'react';
export const SdkSection: React.FC = () => (
<section id="sdk" className="border-b border-[var(--border)] py-10">
<SectionHeading index="02" label="Python SDK" title="Drop in the SDK. Start streaming." />
<SectionHeading index="02" label="Client SDKs" title="Drop in the SDK. Start streaming." />
<div className="grid grid-cols-1 gap-px border border-[var(--border)] bg-[var(--border)] md:grid-cols-2">
<CodeBlock>
<span className="text-[var(--text-faint)]"># Install the bundled Python SDK (from repo or local wheel/path)</span>{'\n'}
<span className="text-[var(--text)]">pip install /path/to/datastack/sdk/python/datastack</span>{'\n\n'}
<span className="text-[var(--text-faint)]"># Auto-loads config.datastack.py from project root</span>{'\n'}
<span className="text-[var(--text-faint)]"># Python SDK (datastack v2.2.0) — auto-loads config.datastack.py</span>{'\n'}
<span className="text-[var(--text)]">pip install ./sdk/python/datastack</span>{'\n\n'}
<span className="text-[var(--text)]">from datastack import DataStack</span>{'\n\n'}
<span className="text-[var(--text)]">client = DataStack()</span>{'\n'}
<span className="text-[var(--text)]">client.send({'{'}<span className="text-[var(--accent)]">&quot;event_type&quot;</span>: <span className="text-[var(--accent)]">&quot;sensor_telemetry&quot;</span>, <span className="text-[var(--accent)]">&quot;temperature&quot;</span>: 24.8, <span className="text-[var(--accent)]">&quot;status&quot;</span>: <span className="text-[var(--accent)]">&quot;NOMINAL&quot;</span>{'}'})</span>{'\n'}
<span className="text-[var(--text)]">client.send_file(<span className="text-[var(--accent)]">&quot;telemetry_batch.csv&quot;</span>)</span>{'\n'}
<span className="text-[var(--text)]">client.disconnect()</span>
</CodeBlock>
<div className="bg-[var(--surface)] p-5 sm:p-6">
<p className="mb-2 text-[11px] uppercase text-[var(--text-faint)]">How it works</p>
<h3 className="mb-2 text-[14px] font-medium text-[var(--text)]">Zero-boilerplate config. Auto-recovery built in.</h3>
<p className="text-[12px] leading-relaxed text-[var(--text-muted)]">
Install the self-hosted SDK from <code className="text-[var(--text)]">sdk/python/datastack</code> (or build a wheel with <code className="text-[var(--text)]">pip wheel ./sdk/python/datastack -w dist/</code>) and drop <code className="text-[var(--text)]">config.datastack.py</code> into your Python project to stream single events, batches, or <code className="text-[var(--text)]">.csv</code> / <code className="text-[var(--text)]">.json</code> files directly—with automatic HTTP fallback, offline disk spooling, and Heartbeat IDLE recovery.
</p>
</div>
<CodeBlock>
<span className="text-[var(--text-faint)]">{'// Node.js / TypeScript SDK (@datastack/sdk v2.2.0)'}</span>{'\n'}
<span className="text-[var(--text)]">npm install ./sdk/node/datastack</span>{'\n\n'}
<span className="text-[var(--text)]">import {'{'} DataStack {'}'} from <span className="text-[var(--accent)]">&apos;@datastack/sdk&apos;</span>;</span>{'\n\n'}
<span className="text-[var(--text)]">const client = new DataStack();</span>{'\n'}
<span className="text-[var(--text)]">await client.send({'{'} <span className="text-[var(--accent)]">event_type</span>: <span className="text-[var(--accent)]">&apos;sensor_telemetry&apos;</span>, <span className="text-[var(--accent)]">temperature</span>: 24.8 {'}'});</span>{'\n'}
<span className="text-[var(--text)]">await client.sendFile(<span className="text-[var(--accent)]">&apos;telemetry_batch.csv&apos;</span>);</span>{'\n'}
<span className="text-[var(--text)]">await client.shutdown(5000);</span>
</CodeBlock>
</div>
<div className="border-x border-b border-[var(--border)] bg-[var(--surface)] px-5 py-4 sm:px-6">
<p className="mb-1 text-[11px] uppercase text-[var(--text-faint)]">Zero-boilerplate config · Auto-recovery built in</p>
<p className="text-[12px] leading-relaxed text-[var(--text-muted)]">
Install either <code className="text-[var(--text)]">sdk/python/datastack</code> (<code className="text-[var(--text)]">config.datastack.py</code>) or <code className="text-[var(--text)]">@datastack/sdk</code> in <code className="text-[var(--text)]">sdk/node/datastack</code> (<code className="text-[var(--text)]">datastack.config.json</code>) to stream single events, batches, or <code className="text-[var(--text)]">.csv</code> / <code className="text-[var(--text)]">.json</code> / <code className="text-[var(--text)]">.jsonl</code> files—with automatic HTTP fallback, offline disk spooling, and graceful shutdown drain.
</p>
</div>
</section>
);
+59 -6
View File
@@ -1,6 +1,7 @@
'use client';
import { PageShell } from '@/components/layout/PageShell';
import { useVconRole } from '@/components/layout/VconShell';
import { Button } from '@/components/ui/Button';
import { Input, Select } from '@/components/ui/Input';
import { Modal } from '@/components/ui/Modal';
@@ -34,8 +35,9 @@ interface TeamMemberItem {
export function TeamWorkspace() {
const { success, error } = useToast();
const shellRole = useVconRole();
const [members, setMembers] = useState<TeamMemberItem[]>([]);
const [currentRole, setCurrentRole] = useState<string>('owner');
const [currentRole, setCurrentRole] = useState<string>(() => shellRole || 'viewer');
const [loading, setLoading] = useState<boolean>(true);
const [createOpen, setCreateOpen] = useState<boolean>(false);
@@ -43,6 +45,7 @@ export function TeamWorkspace() {
const [newPassword, setNewPassword] = useState<string>('');
const [newRole, setNewRole] = useState<string>('viewer');
const [creating, setCreating] = useState<boolean>(false);
const [deleteTarget, setDeleteTarget] = useState<TeamMemberItem | null>(null);
const [busyId, setBusyId] = useState<string | null>(null);
@@ -186,6 +189,7 @@ export function TeamWorkspace() {
throw new Error(data?.error || 'Delete failed.');
}
success(`Removed ${member.email} from team.`);
setDeleteTarget(null);
await fetchMembers();
} catch (err: unknown) {
error(err instanceof Error ? err.message : 'Unable to delete team member.');
@@ -246,7 +250,7 @@ export function TeamWorkspace() {
value: String(ownerCount),
},
{
label: 'Admins / Operators',
label: 'Operators / Admins',
value: String(operatorCount),
},
{
@@ -306,14 +310,15 @@ export function TeamWorkspace() {
<Select
compact
aria-label={`Role for ${row.email}`}
value={row.role === 'operator' ? 'admin' : row.role}
value={row.role}
disabled={busyId === row.id}
onChange={(e) =>
void handleRoleChange(row, e.target.value)
}
options={[
{ value: 'viewer', label: 'Viewer (Read-Only)' },
{ value: 'admin', label: 'Admin / Operator' },
{ value: 'operator', label: 'Operator (Manage Clients)' },
{ value: 'admin', label: 'Admin (Manage Clients)' },
{ value: 'owner', label: 'Owner (Full Control)' },
]}
/>
@@ -359,7 +364,7 @@ export function TeamWorkspace() {
variant="danger"
size="sm"
disabled={busyId === row.id}
onClick={() => void handleDeleteMember(row)}
onClick={() => setDeleteTarget(row)}
>
Remove
</Button>
@@ -410,9 +415,13 @@ export function TeamWorkspace() {
value: 'viewer',
label: 'Viewer — Read-only dashboards, events, clients & exports',
},
{
value: 'operator',
label: 'Operator — Manage SDK clients, tokens & manual flushes',
},
{
value: 'admin',
label: 'Admin / Operator — Manage SDK clients, tokens & flushes',
label: 'Admin — Operational role (manage SDK clients & flushes)',
},
{
value: 'owner',
@@ -440,6 +449,50 @@ export function TeamWorkspace() {
</div>
</form>
</Modal>
<Modal
isOpen={Boolean(deleteTarget)}
onClose={() => {
if (!busyId) setDeleteTarget(null);
}}
title="Confirm Team Member Removal"
maxWidth="sm"
>
{deleteTarget && (
<div className="flex flex-col gap-3">
<p className="text-[12px] text-[var(--text-muted)] leading-relaxed">
Remove{' '}
<span className="font-mono font-medium text-[var(--text)]">
{deleteTarget.email}
</span>{' '}
(<span className="font-mono uppercase">{deleteTarget.role}</span>) from
administrator access? Active sessions will stop working immediately.
</p>
<div className="flex justify-end gap-2 pt-1">
<Button
type="button"
variant="secondary"
size="sm"
disabled={busyId === deleteTarget.id}
onClick={() => setDeleteTarget(null)}
>
Cancel
</Button>
<Button
type="button"
variant="danger"
size="sm"
data-testid="confirm-remove-member"
loading={busyId === deleteTarget.id}
disabled={busyId === deleteTarget.id}
onClick={() => void handleDeleteMember(deleteTarget)}
>
Confirm Remove
</Button>
</div>
</div>
)}
</Modal>
</PageShell>
);
}
+311
View File
@@ -0,0 +1,311 @@
import dns from "node:dns/promises";
import net from "node:net";
export class SSRFBlockedError extends Error {
public readonly code = "SSRF_BLOCKED_HOST" as const;
public readonly hostname: string;
constructor(message: string, hostname: string) {
super(message);
this.name = "SSRFBlockedError";
this.hostname = hostname;
}
}
export interface OutboundUrlValidationOptions {
allowPrivate?: boolean;
env?: NodeJS.ProcessEnv;
dnsLookup?: (
hostname: string,
) => Promise<Array<{ address: string; family: number }>>;
fieldName?: string;
}
export interface OutboundUrlValidationResult {
allowed: boolean;
url?: URL;
code?: "SSRF_BLOCKED_HOST" | "INVALID_URL";
error?: string;
hostname?: string;
}
const ALWAYS_BLOCKED_HOSTNAMES = new Set([
"metadata.google.internal",
"metadata.goog",
"metadata",
"instance-data",
"instance-data.ec2.internal",
]);
const PRIVATE_HOSTNAMES = new Set([
"localhost",
"localhost.localdomain",
"ip6-localhost",
"ip6-loopback",
]);
function buildAlwaysBlockedBlockList(): net.BlockList {
const list = new net.BlockList();
// Unspecified / zero network
list.addSubnet("0.0.0.0", 8, "ipv4");
// Link-local & cloud metadata (AWS/GCP/Azure/Oracle 169.254.169.254)
list.addSubnet("169.254.0.0", 16, "ipv4");
// Multicast & reserved
list.addSubnet("224.0.0.0", 4, "ipv4");
list.addSubnet("240.0.0.0", 4, "ipv4");
// IPv6 unspecified, link-local, multicast
list.addAddress("::", "ipv6");
list.addSubnet("fe80::", 10, "ipv6");
list.addSubnet("ff00::", 8, "ipv6");
return list;
}
function buildPrivateBlockList(): net.BlockList {
const list = new net.BlockList();
// Loopback
list.addSubnet("127.0.0.0", 8, "ipv4");
// RFC 1918 private networks
list.addSubnet("10.0.0.0", 8, "ipv4");
list.addSubnet("172.16.0.0", 12, "ipv4");
list.addSubnet("192.168.0.0", 16, "ipv4");
// Carrier-grade NAT (RFC 6598)
list.addSubnet("100.64.0.0", 10, "ipv4");
// Benchmarking / documentation
list.addSubnet("198.18.0.0", 15, "ipv4");
// IPv6 loopback & Unique Local Addresses (ULA)
list.addAddress("::1", "ipv6");
list.addSubnet("fc00::", 7, "ipv6");
return list;
}
const ALWAYS_BLOCKED_IPS = buildAlwaysBlockedBlockList();
const PRIVATE_IPS = buildPrivateBlockList();
export function isPrivateOutboundUrlAllowed(
env: NodeJS.ProcessEnv = process.env,
): boolean {
const raw = String(env.DATASTACK_ALLOW_PRIVATE_URLS || "")
.trim()
.toLowerCase();
return raw === "true" || raw === "1" || raw === "yes";
}
function stripBracketsAndZone(rawHost: string): string {
let host = String(rawHost || "").trim().toLowerCase();
if (host.startsWith("[") && host.endsWith("]")) {
host = host.slice(1, -1);
}
const zoneIdx = host.indexOf("%");
if (zoneIdx >= 0) {
host = host.slice(0, zoneIdx);
}
// Remove trailing FQDN dot (e.g. "localhost.")
return host.replace(/\.+$/, "");
}
function normalizeIpForCheck(rawIp: string): { ip: string; family: 4 | 6 } | null {
const clean = stripBracketsAndZone(rawIp);
if (!clean) return null;
// Handle IPv4-mapped IPv6 addresses (e.g. ::ffff:127.0.0.1 or ::ffff:7f00:1)
if (clean.startsWith("::ffff:")) {
const mapped = clean.slice("::ffff:".length);
if (net.isIP(mapped) === 4) {
return { ip: mapped, family: 4 };
}
const hexParts = mapped.split(":");
if (hexParts.length === 2) {
const hi = Number.parseInt(hexParts[0], 16);
const lo = Number.parseInt(hexParts[1], 16);
if (
Number.isInteger(hi) &&
Number.isInteger(lo) &&
hi >= 0 &&
hi <= 0xffff &&
lo >= 0 &&
lo <= 0xffff
) {
const ipv4 = `${(hi >> 8) & 0xff}.${hi & 0xff}.${(lo >> 8) & 0xff}.${lo & 0xff}`;
if (net.isIP(ipv4) === 4) {
return { ip: ipv4, family: 4 };
}
}
}
}
const family = net.isIP(clean);
if (family === 4 || family === 6) {
return { ip: clean, family };
}
return null;
}
export function checkIpAddressAllowed(
rawIp: string,
allowPrivate: boolean,
): { allowed: boolean; reason?: string } {
const parsed = normalizeIpForCheck(rawIp);
if (!parsed) {
return { allowed: false, reason: `Invalid IP address '${rawIp}'.` };
}
const ipType = parsed.family === 4 ? "ipv4" : "ipv6";
if (ALWAYS_BLOCKED_IPS.check(parsed.ip, ipType)) {
return {
allowed: false,
reason: `Address '${parsed.ip}' is in a restricted link-local/metadata/reserved network range.`,
};
}
if (!allowPrivate && PRIVATE_IPS.check(parsed.ip, ipType)) {
return {
allowed: false,
reason: `Address '${parsed.ip}' is a private or loopback network address (set DATASTACK_ALLOW_PRIVATE_URLS=true only if intentionally targeting a private host).`,
};
}
return { allowed: true };
}
export function validateOutboundUrlSync(
rawUrl: string,
options: OutboundUrlValidationOptions = {},
): OutboundUrlValidationResult {
const fieldName = options.fieldName || "URL";
const allowPrivate =
options.allowPrivate !== undefined
? options.allowPrivate
: isPrivateOutboundUrlAllowed(options.env);
const trimmed = String(rawUrl || "").trim();
if (!trimmed) {
return {
allowed: false,
code: "INVALID_URL",
error: `${fieldName} is required.`,
};
}
let parsed: URL;
try {
parsed = new URL(trimmed);
} catch {
return {
allowed: false,
code: "INVALID_URL",
error: `${fieldName} must be a valid http:// or https:// URL.`,
};
}
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
return {
allowed: false,
code: "SSRF_BLOCKED_HOST",
hostname: parsed.hostname,
error: `${fieldName} must use http:// or https://.`,
};
}
if (parsed.username || parsed.password) {
return {
allowed: false,
code: "SSRF_BLOCKED_HOST",
hostname: parsed.hostname,
error: `${fieldName} must not embed URL credentials.`,
};
}
const host = stripBracketsAndZone(parsed.hostname);
if (!host) {
return {
allowed: false,
code: "INVALID_URL",
error: `${fieldName} must include a valid hostname.`,
};
}
if (ALWAYS_BLOCKED_HOSTNAMES.has(host)) {
return {
allowed: false,
code: "SSRF_BLOCKED_HOST",
hostname: host,
error: `${fieldName} target '${host}' is blocked (cloud metadata/internal endpoint).`,
};
}
// Literal IP check (covers IPv4, IPv6, and IPv4-mapped IPv6)
const ipParsed = normalizeIpForCheck(host);
if (ipParsed) {
const ipCheck = checkIpAddressAllowed(host, allowPrivate);
if (!ipCheck.allowed) {
return {
allowed: false,
code: "SSRF_BLOCKED_HOST",
hostname: host,
error: `${fieldName} blocked: ${ipCheck.reason}`,
};
}
return { allowed: true, url: parsed, hostname: host };
}
// Non-IP hostname checks
if (!allowPrivate) {
if (
PRIVATE_HOSTNAMES.has(host) ||
host.endsWith(".localhost") ||
host.endsWith(".local") ||
host.endsWith(".internal") ||
!host.includes(".")
) {
return {
allowed: false,
code: "SSRF_BLOCKED_HOST",
hostname: host,
error: `${fieldName} target '${host}' is a private/internal hostname (set DATASTACK_ALLOW_PRIVATE_URLS=true only for trusted private hosts).`,
};
}
}
return { allowed: true, url: parsed, hostname: host };
}
export async function validateOutboundUrlAsync(
rawUrl: string,
options: OutboundUrlValidationOptions = {},
): Promise<OutboundUrlValidationResult> {
const syncResult = validateOutboundUrlSync(rawUrl, options);
if (!syncResult.allowed || !syncResult.url || !syncResult.hostname) {
return syncResult;
}
const host = syncResult.hostname;
const fieldName = options.fieldName || "URL";
const allowPrivate =
options.allowPrivate !== undefined
? options.allowPrivate
: isPrivateOutboundUrlAllowed(options.env);
// Skip DNS resolution if already validated as a literal IP
if (normalizeIpForCheck(host)) {
return syncResult;
}
try {
const lookupFn =
options.dnsLookup ||
((h: string) => dns.lookup(h, { all: true, verbatim: true }));
const records = await lookupFn(host);
for (const rec of records) {
const check = checkIpAddressAllowed(rec.address, allowPrivate);
if (!check.allowed) {
return {
allowed: false,
code: "SSRF_BLOCKED_HOST",
hostname: host,
error: `${fieldName} hostname '${host}' resolved to a blocked address: ${check.reason}`,
};
}
}
} catch {
// DNS resolution failure (e.g. ENOTFOUND/EAI_AGAIN) will be handled cleanly by the caller or fetch()
}
return syncResult;
}
+20 -7
View File
@@ -386,6 +386,14 @@ export class CSVStorageManager {
if (!gunzip.destroyed) {
gunzip.destroy(err);
}
if (!fileStream.destroyed) {
fileStream.destroy();
}
});
gunzip.on('error', () => {
if (!fileStream.destroyed) {
fileStream.destroy();
}
});
gunzip.on('close', () => {
if (!fileStream.destroyed) {
@@ -396,16 +404,21 @@ export class CSVStorageManager {
return gunzip;
}
private static async countCsvDataRowsStreamAsync(
public static async countCsvDataRowsStreamAsync(
filePath: string,
): Promise<number> {
const counter = this.createChunkRowCounter();
return new Promise<number>((resolve, reject) => {
const stream = this.openPartitionTextStream(filePath);
stream.on('data', (chunk) => counter.pushChunk(chunk));
stream.on('end', () => resolve(counter.finish()));
stream.on('error', (err) => reject(err));
});
const stream = this.openPartitionTextStream(filePath);
try {
for await (const chunk of stream) {
counter.pushChunk(chunk as Buffer | string);
}
return counter.finish();
} finally {
if (!stream.destroyed) {
stream.destroy();
}
}
}
/**
+98 -12
View File
@@ -1,5 +1,7 @@
import crypto from "node:crypto";
import fs from "node:fs";
import { Readable } from "node:stream";
import { validateOutboundUrlAsync } from "@/core/Security/ssrfGuard";
export interface S3ArchiveConfig {
endpoint: string;
@@ -17,7 +19,12 @@ export interface S3OperationResult {
bucket?: string;
key?: string;
etag?: string;
code?: "S3_INVALID_CONFIG" | "S3_CONNECTION_FAILED" | "S3_ARCHIVE_FAILED";
code?:
| "S3_INVALID_CONFIG"
| "S3_CONNECTION_FAILED"
| "S3_ARCHIVE_FAILED"
| "SSRF_BLOCKED_HOST"
| "INVALID_URL";
error?: string;
}
@@ -25,6 +32,27 @@ function sha256Hex(data: Buffer | string): string {
return crypto.createHash("sha256").update(data).digest("hex");
}
async function computeFileSha256AndSize(
filePath: string,
): Promise<{ sha256Hex: string; sizeBytes: number }> {
const stat = await fs.promises.stat(filePath);
const hash = crypto.createHash("sha256");
const readStream = fs.createReadStream(filePath, { highWaterMark: 64 * 1024 });
try {
for await (const chunk of readStream) {
hash.update(chunk as Buffer | string);
}
return {
sha256Hex: hash.digest("hex"),
sizeBytes: stat.size,
};
} finally {
if (!readStream.destroyed) {
readStream.destroy();
}
}
}
function hmacSha256(key: Buffer | string, data: string): Buffer {
return crypto.createHmac("sha256", key).update(data, "utf8").digest();
}
@@ -204,6 +232,16 @@ export async function testS3Connection(
config.endpoint,
config.region,
);
const ssrfCheck = await validateOutboundUrlAsync(url.toString(), {
fieldName: "S3 Endpoint URL",
});
if (!ssrfCheck.allowed) {
return {
ok: false,
code: ssrfCheck.code || "SSRF_BLOCKED_HOST",
error: ssrfCheck.error || "S3 Endpoint URL is blocked.",
};
}
const signed = signS3Request({
method: "GET",
endpointUrl: url,
@@ -219,9 +257,21 @@ export async function testS3Connection(
method: "GET",
headers: signed.headers,
cache: "no-store",
redirect: "manual",
signal: AbortSignal.timeout(8000),
});
if (response.status >= 300 && response.status < 400) {
return {
ok: false,
status: response.status,
endpoint: url.origin,
bucket: config.bucket.trim(),
code: "SSRF_BLOCKED_HOST",
error: `S3 endpoint returned an unsupported HTTP ${response.status} redirect.`,
};
}
if (!response.ok) {
const bodyPreview = await response.text().catch(() => "");
return {
@@ -272,8 +322,19 @@ export async function uploadFileToS3(
config.endpoint,
config.region,
);
const bodyBuffer = await fs.promises.readFile(localFilePath);
const payloadHashHex = sha256Hex(bodyBuffer);
const ssrfCheck = await validateOutboundUrlAsync(url.toString(), {
fieldName: "S3 Endpoint URL",
});
if (!ssrfCheck.allowed) {
return {
ok: false,
code: ssrfCheck.code || "SSRF_BLOCKED_HOST",
error: ssrfCheck.error || "S3 Endpoint URL is blocked.",
key: objectKey,
};
}
const { sha256Hex: payloadHashHex, sizeBytes } =
await computeFileSha256AndSize(localFilePath);
const contentType = localFilePath.endsWith(".gz")
? "application/gzip"
: "text/csv; charset=utf-8";
@@ -290,16 +351,41 @@ export async function uploadFileToS3(
contentType,
});
const response = await fetch(signed.targetUrl, {
method: "PUT",
headers: {
...signed.headers,
"content-length": String(bodyBuffer.byteLength),
},
body: new Uint8Array(bodyBuffer),
cache: "no-store",
signal: AbortSignal.timeout(30000),
const uploadStream = fs.createReadStream(localFilePath, {
highWaterMark: 64 * 1024,
});
let response: Response;
try {
const requestInit: RequestInit & { duplex: "half" } = {
method: "PUT",
headers: {
...signed.headers,
"content-length": String(sizeBytes),
},
body: Readable.toWeb(uploadStream) as unknown as BodyInit,
duplex: "half",
cache: "no-store",
redirect: "manual",
signal: AbortSignal.timeout(30000),
};
response = await fetch(signed.targetUrl, requestInit);
} finally {
if (!uploadStream.destroyed) {
uploadStream.destroy();
}
}
if (response.status >= 300 && response.status < 400) {
return {
ok: false,
status: response.status,
endpoint: url.origin,
bucket: config.bucket.trim(),
key: objectKey,
code: "SSRF_BLOCKED_HOST",
error: `S3 upload returned an unsupported HTTP ${response.status} redirect.`,
};
}
if (!response.ok) {
const bodyPreview = await response.text().catch(() => "");
+28
View File
@@ -309,6 +309,34 @@ export class SettingsManager {
return true;
}
public static async bootstrapMaintenanceSchedulerOnBoot(): Promise<{
schedulerActive: boolean;
hydrated: boolean;
error?: string;
}> {
this.ensureMaintenanceScheduler();
try {
await this.getSettingsAsync();
return {
schedulerActive: this.isMaintenanceSchedulerActive(),
hydrated:
!process.env.DATABASE_URL?.trim() ||
globals.__dataStackSettingsHydrated === true,
};
} catch (err: unknown) {
const message = err instanceof Error ? err.message : String(err);
console.warn(
"[DataStack Boot] Maintenance scheduler init deferred/failed:",
message,
);
return {
schedulerActive: this.isMaintenanceSchedulerActive(),
hydrated: false,
error: message,
};
}
}
public static stopMaintenanceScheduler(): void {
if (globals.__dataStackMaintenanceTimer) {
clearInterval(globals.__dataStackMaintenanceTimer);
+19 -11
View File
@@ -1,4 +1,5 @@
import crypto from "node:crypto";
import { validateOutboundUrlAsync } from "@/core/Security/ssrfGuard";
export type WebhookAlertEventType =
| "DISK_SPACE_LOW"
@@ -30,6 +31,7 @@ export interface WebhookDispatchResult {
ok: boolean;
skipped?: boolean;
reason?: string;
code?: string;
status?: number;
error?: string;
}
@@ -106,21 +108,17 @@ export async function dispatchWebhookAlertAsync(
};
}
let parsedUrl: URL;
try {
parsedUrl = new URL(webhookUrl);
if (parsedUrl.protocol !== "http:" && parsedUrl.protocol !== "https:") {
return {
ok: false,
error: "Webhook URL must use http:// or https://.",
};
}
} catch {
const ssrfCheck = await validateOutboundUrlAsync(webhookUrl, {
fieldName: "Webhook URL",
});
if (!ssrfCheck.allowed || !ssrfCheck.url) {
return {
ok: false,
error: "Webhook URL is not a valid URL.",
code: ssrfCheck.code || "SSRF_BLOCKED_HOST",
error: ssrfCheck.error || "Webhook URL is not allowed.",
};
}
const parsedUrl = ssrfCheck.url;
const dedupeKey = `${payload.event}:${payload.clientId || "global"}`;
const cooldownMs = Math.max(0, (Number(config.cooldownSec) || 300) * 1000);
@@ -156,9 +154,19 @@ export async function dispatchWebhookAlertAsync(
headers,
body: bodyStr,
cache: "no-store",
redirect: "manual",
signal: AbortSignal.timeout(timeoutMs),
});
if (res.status >= 300 && res.status < 400) {
return {
ok: false,
code: "SSRF_BLOCKED_HOST",
status: res.status,
error: `Webhook endpoint attempted an unsupported HTTP ${res.status} redirect.`,
};
}
if (!res.ok) {
if (!options?.bypassCooldown) {
// Keep cooldown entry so failing webhooks don't DDOS a broken upstream endpoint
+79
View File
@@ -15,6 +15,40 @@ import {
} from "./adminSession";
import { clearAuditLogsForTests } from "./auditLog";
import { DatabaseEngine, ensureCoreSchemaOnPool } from "../Database/database";
import { getInternalIngestionHeaders } from "../Security/security";
async function notifyIngestionAdminSessionSync(payload: {
memberId?: string;
email?: string;
previousEmail?: string;
minTokenVersion?: number;
disabled?: boolean;
deleted?: boolean;
}): Promise<void> {
const configuredUrl = process.env.INGESTION_INTERNAL_URL?.trim();
if (process.env.DATASTACK_DISABLE_LISTEN === "1" && !configuredUrl) {
return;
}
const baseUrl = (configuredUrl || "http://127.0.0.1:4000").replace(
/\/+$/,
"",
);
try {
await fetch(`${baseUrl}/internal/admin-session-sync`, {
method: "POST",
headers: {
"Content-Type": "application/json",
...getInternalIngestionHeaders(),
},
body: JSON.stringify(payload),
cache: "no-store",
signal: AbortSignal.timeout(1200),
});
} catch {
// Best-effort bridge sync when Port 4000 is offline or not running
}
}
const PASSWORD_HASH_BYTES = 64;
const PASSWORD_SALT_BYTES = 16;
@@ -1040,6 +1074,13 @@ export async function updateAdminAccount({
tokenVersion: nextVersion,
createdAt: existing.createdAt,
};
await notifyIngestionAdminSessionSync({
memberId: "owner-1",
email: cleanEmail,
previousEmail: existing.email,
minTokenVersion: nextVersion,
disabled: false,
});
return {
ok: true as const,
email: cleanEmail,
@@ -1085,6 +1126,13 @@ export async function updateAdminAccount({
tokenVersion: nextVersion,
updatedAt: new Date(),
});
await notifyIngestionAdminSessionSync({
memberId: member.id,
email: cleanEmail,
previousEmail: member.email,
minTokenVersion: nextVersion,
disabled: member.status === "disabled",
});
return {
ok: true as const,
email: cleanEmail,
@@ -1131,6 +1179,12 @@ export async function revokeAdminSessions(
member.tokenVersion += 1;
member.updatedAt = new Date();
teamMap.set(member.id, member);
await notifyIngestionAdminSessionSync({
memberId: member.id,
email: member.email,
minTokenVersion: member.tokenVersion,
disabled: member.status === "disabled",
});
return member.tokenVersion;
}
}
@@ -1157,6 +1211,12 @@ export async function revokeAdminSessions(
member.tokenVersion += 1;
member.updatedAt = new Date();
teamMap.set(member.id, member);
await notifyIngestionAdminSessionSync({
memberId: member.id,
email: member.email,
minTokenVersion: member.tokenVersion,
disabled: member.status === "disabled",
});
return member.tokenVersion;
}
}
@@ -1181,6 +1241,12 @@ export async function revokeAdminSessions(
globals.__dataStackInMemoryAdmin ?? (await getOrInitInMemoryAdmin());
existing.tokenVersion += 1;
globals.__dataStackInMemoryAdmin = existing;
await notifyIngestionAdminSessionSync({
memberId: "owner-1",
email: existing.email,
minTokenVersion: existing.tokenVersion,
disabled: false,
});
return existing.tokenVersion;
}
@@ -1588,6 +1654,12 @@ export async function updateAdminTeamMemberAsync(
updatedAt: new Date(),
};
teamMap.set(id, updated);
await notifyIngestionAdminSessionSync({
memberId: updated.id,
email: updated.email,
minTokenVersion: updated.tokenVersion,
disabled: updated.status === "disabled",
});
return {
id: updated.id,
@@ -1654,6 +1726,13 @@ export async function deleteAdminTeamMemberAsync(
);
}
teamMap.delete(id);
await notifyIngestionAdminSessionSync({
memberId: existing.id,
email: existing.email,
minTokenVersion: existing.tokenVersion + 1,
disabled: true,
deleted: true,
});
return { id: existing.id, email: existing.email };
}
+65 -8
View File
@@ -20,6 +20,8 @@ export type AuditActionType =
| "STORAGE_FLUSH_TRIGGERED"
| "S3_ARCHIVE_TRIGGERED"
| "WEBHOOK_TEST_TRIGGERED"
| "FILES_MANAGER_MUTATION"
| "DATA_EXPORT_DOWNLOADED"
| string;
export interface AuditLogEntry {
@@ -49,10 +51,19 @@ export interface ListAuditLogsQuery {
search?: string;
}
export interface AuditLogListResult {
logs: AuditLogEntry[];
storageMode: "postgresql" | "memory";
degraded: boolean;
code?: "AUDIT_STORAGE_DEGRADED";
warning?: string;
}
const MAX_IN_MEMORY_AUDIT_LOGS = 1000;
type AuditLogGlobals = typeof globalThis & {
__dataStackAuditLogs?: AuditLogEntry[];
__dataStackAuditLastWriteError?: string;
};
const globals = globalThis as AuditLogGlobals;
@@ -149,8 +160,13 @@ export async function recordAdminAuditLogAsync(
entry.createdAt,
],
);
} catch {
// Never block or crash administrative actions if audit table write experiences transient DB failure
globals.__dataStackAuditLastWriteError = undefined;
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
globals.__dataStackAuditLastWriteError = msg;
console.warn(
`[DataStack Audit] PostgreSQL audit insert failed for action=${entry.action} (buffered in memory only): ${msg}`,
);
}
}
@@ -161,9 +177,9 @@ export function recordAdminAuditLog(input: RecordAuditLogInput): void {
void recordAdminAuditLogAsync(input);
}
export async function listAdminAuditLogsAsync(
export async function listAdminAuditLogsWithStatusAsync(
query: ListAuditLogsQuery = {},
): Promise<AuditLogEntry[]> {
): Promise<AuditLogListResult> {
const rawLimit =
typeof query.limit === "number" && Number.isFinite(query.limit)
? Math.floor(query.limit)
@@ -174,6 +190,7 @@ export async function listAdminAuditLogsAsync(
const searchFilter = query.search?.trim().toLowerCase() || "";
const pool = DatabaseEngine.getPool();
let dbReadError: string | null = null;
if (pool) {
try {
await DatabaseEngine.ensureSchema();
@@ -227,7 +244,7 @@ export async function listAdminAuditLogsAsync(
params,
);
return result.rows.map((row) => {
const logs = result.rows.map((row) => {
let parsedMeta: Record<string, unknown> = {};
try {
const parsed = JSON.parse(row.metadata_json || "{}");
@@ -251,12 +268,28 @@ export async function listAdminAuditLogsAsync(
: new Date(row.created_at).toISOString(),
};
});
} catch {
// Fallback to in-memory log buffer if PostgreSQL query fails
const writeErr = globals.__dataStackAuditLastWriteError;
return {
logs,
storageMode: "postgresql",
degraded: Boolean(writeErr),
...(writeErr
? {
code: "AUDIT_STORAGE_DEGRADED" as const,
warning: `Recent audit log insert failed in PostgreSQL: ${writeErr}`,
}
: {}),
};
} catch (err: unknown) {
dbReadError = err instanceof Error ? err.message : String(err);
console.warn(
`[DataStack Audit] PostgreSQL audit query failed (serving ephemeral in-memory fallback): ${dbReadError}`,
);
}
}
return getInMemoryAuditLogs()
const fallbackLogs = getInMemoryAuditLogs()
.filter((entry) => {
if (
actionFilter &&
@@ -278,8 +311,32 @@ export async function listAdminAuditLogsAsync(
return true;
})
.slice(0, limit);
if (dbReadError) {
return {
logs: fallbackLogs,
storageMode: "memory",
degraded: true,
code: "AUDIT_STORAGE_DEGRADED",
warning: `PostgreSQL audit query failed (${dbReadError}); serving ephemeral in-memory logs.`,
};
}
return {
logs: fallbackLogs,
storageMode: "memory",
degraded: false,
};
}
export async function listAdminAuditLogsAsync(
query: ListAuditLogsQuery = {},
): Promise<AuditLogEntry[]> {
const result = await listAdminAuditLogsWithStatusAsync(query);
return result.logs;
}
export function clearAuditLogsForTests(): void {
globals.__dataStackAuditLogs = [];
globals.__dataStackAuditLastWriteError = undefined;
}
+2
View File
@@ -8,6 +8,8 @@ FROM node:22-alpine AS builder
WORKDIR /app
ENV NODE_ENV=production
ENV NEXT_TELEMETRY_DISABLED=1
ARG INGESTION_INTERNAL_URL=http://ingestion:4000
ENV INGESTION_INTERNAL_URL=${INGESTION_INTERNAL_URL}
COPY --from=dependencies /app/node_modules ./node_modules
COPY . .
RUN npm run build
+24
View File
@@ -17,6 +17,15 @@ server {
listen 80;
server_name _;
# Preserve real external visitor IP when routed behind Coolify Traefik / trusted LAN proxies
set_real_ip_from 127.0.0.1;
set_real_ip_from 10.0.0.0/8;
set_real_ip_from 172.16.0.0/12;
set_real_ip_from 192.168.0.0/16;
set_real_ip_from fc00::/7;
real_ip_header X-Forwarded-For;
real_ip_recursive on;
resolver 127.0.0.11 valid=10s ipv6=off;
set $nextjs_upstream "http://web:3000";
set $socketio_upstream "http://ingestion:4000";
@@ -48,6 +57,21 @@ server {
proxy_buffering off;
}
location /vcon/files-engine {
proxy_pass $socketio_upstream;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-Host $forwarded_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $forwarded_proto;
proxy_read_timeout 86400s;
proxy_send_timeout 86400s;
proxy_buffering off;
}
location = /api/export {
proxy_pass $nextjs_upstream;
proxy_http_version 1.1;
+9
View File
@@ -19,6 +19,11 @@ services:
DATASTACK_ADMIN_SETUP_KEY: "${SERVICE_REALBASE64_64_ADMININSTALL:?Coolify must generate SERVICE_REALBASE64_64_ADMININSTALL}"
DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}"
REDIS_URL: "redis://redis:6379"
DATASTACK_ALLOW_PRIVATE_URLS: "${DATASTACK_ALLOW_PRIVATE_URLS:-false}"
DATASTACK_MAX_BATCH_ITEMS: "${DATASTACK_MAX_BATCH_ITEMS:-1000}"
DATASTACK_RECENT_LIMIT: "${DATASTACK_RECENT_LIMIT:-500}"
DATASTACK_MAINTENANCE_INTERVAL_MS: "${DATASTACK_MAINTENANCE_INTERVAL_MS:-}"
DATASTACK_PROXY_MODE: "${DATASTACK_PROXY_MODE:-}"
depends_on:
postgres:
condition: service_healthy
@@ -53,10 +58,14 @@ services:
PORT_SOCKET: "4000"
DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}"
DATASTACK_INTERNAL_SECRET: "${DATASTACK_INTERNAL_SECRET:-${SERVICE_REALBASE64_64_INTERNAL:-}}"
DATASTACK_ADMIN_EMAIL: "${DATASTACK_ADMIN_EMAIL:-admin@datastack.dev}"
DATASTACK_ADMIN_SESSION_SECRET: "${SERVICE_REALBASE64_64_ADMINSESSION:?Coolify must generate SERVICE_REALBASE64_64_ADMINSESSION}"
DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}"
REDIS_URL: "redis://redis:6379"
DATASTACK_CORS_ORIGINS: "${DATASTACK_CORS_ORIGINS:-}"
DATASTACK_ALLOW_PRIVATE_URLS: "${DATASTACK_ALLOW_PRIVATE_URLS:-false}"
DATASTACK_MAX_BATCH_ITEMS: "${DATASTACK_MAX_BATCH_ITEMS:-1000}"
DATASTACK_RECENT_LIMIT: "${DATASTACK_RECENT_LIMIT:-500}"
depends_on:
postgres:
condition: service_healthy
+19
View File
@@ -0,0 +1,19 @@
export async function register(): Promise<void> {
if (process.env.NEXT_RUNTIME === "nodejs") {
if (process.env.DATASTACK_DISABLE_LISTEN === "1") {
return;
}
try {
const { SettingsManager } = await import(
"@/core/adminSettings/settingsManager"
);
await SettingsManager.bootstrapMaintenanceSchedulerOnBoot();
} catch (err: unknown) {
console.warn(
"[DataStack Boot] Maintenance scheduler init deferred/failed:",
err instanceof Error ? err.message : err,
);
}
}
}
-1
View File
@@ -9,7 +9,6 @@ const SAFE_METHODS = new Set(["GET", "HEAD", "OPTIONS"]);
function isProtectedApi(pathname: string, method: string) {
if (pathname === "/api/auth/login") return false;
if (pathname === "/api/auth/setup/status" && method === "GET") return false;
if (pathname === "/api/auth/setup" && method === "POST") return false;
if (pathname === "/api/events" && method === "POST") return false;
if (pathname === "/api/health" && method === "GET") return false;
+15
View File
@@ -18,6 +18,21 @@ const nextConfig: NextConfig = {
},
output: 'standalone',
transpilePackages: ['motion'],
async rewrites() {
const ingestionBase = (
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
).replace(/\/+$/, '');
return [
{
source: '/vcon/files-engine',
destination: `${ingestionBase}/vcon/files-engine`,
},
{
source: '/vcon/files-engine/:path*',
destination: `${ingestionBase}/vcon/files-engine/:path*`,
},
];
},
webpack: (config, {dev}) => {
// HMR is disabled in AI Studio via DISABLE_HMR env var.
// Do not modify—file watching is disabled to prevent flickering during agent edits.
+25
View File
@@ -25,6 +25,15 @@ server {
listen 80;
server_name _;
# Preserve real external visitor IP when routed behind trusted reverse proxies / Docker bridges
set_real_ip_from 127.0.0.1;
set_real_ip_from 10.0.0.0/8;
set_real_ip_from 172.16.0.0/12;
set_real_ip_from 192.168.0.0/16;
set_real_ip_from fc00::/7;
real_ip_header X-Forwarded-For;
real_ip_recursive on;
client_max_body_size 50M;
# Express + Socket.IO health endpoint on port 4000
@@ -54,6 +63,22 @@ server {
proxy_buffering off;
}
# Admin Files Manager (CloudCmd + isolated Socket.IO) reverse proxy to port 4000
location /vcon/files-engine {
proxy_pass http://socketio_upstream;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-Host $forwarded_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $forwarded_proto;
proxy_read_timeout 86400s;
proxy_send_timeout 86400s;
proxy_buffering off;
}
# Streaming CSV/JSON export endpoint on port 3000 (disable response buffering)
location = /api/export {
proxy_pass http://nextjs_upstream;
+10356 -223
View File
File diff suppressed because it is too large. Load diff
+3 -1
View File
@@ -6,7 +6,8 @@
"node": ">=20.0.0"
},
"scripts": {
"dev": "next dev -p 3000 -H 0.0.0.0",
"dev": "node scripts/dev-all.js",
"dev:web": "next dev -p 3000 -H 0.0.0.0",
"dev:all": "node scripts/dev-all.js",
"server": "node server/index.js",
"build": "NODE_ENV=production next build",
@@ -20,6 +21,7 @@
"@hookform/resolvers": "^5.2.1",
"autoprefixer": "^10.4.21",
"class-variance-authority": "^0.7.1",
"cloudcmd": "^19.23.2",
"clsx": "^2.1.1",
"cors": "^2.8.6",
"csv-parse": "^7.0.3",
+2
View File
@@ -153,6 +153,8 @@ self.addEventListener('fetch', (event) => {
if (
url.pathname.startsWith('/socket.io/') ||
url.pathname.startsWith('/api/') ||
url.pathname === '/vcon/files-engine' ||
url.pathname.startsWith('/vcon/files-engine/') ||
url.pathname.startsWith('/_next/webpack-hmr')
) {
return;
+1 -1
View File
@@ -19,7 +19,7 @@ console.log('\x1b[36m%s\x1b[0m', ` - Port ${webPort}: Next.js 15 Web & Admin Co
console.log('\x1b[36m%s\x1b[0m', ' - Port 4000: Ingestion & Socket.IO Streaming Engine');
console.log('\x1b[36m%s\x1b[0m', '=======================================================\n');
const nextProc = spawn(process.execPath, [nextBin, 'dev', '-p', webPort], {
const nextProc = spawn(process.execPath, [nextBin, 'dev', '-p', webPort, '-H', '0.0.0.0'], {
stdio: 'inherit',
shell: false,
env: { ...process.env },
+61 -17
View File
@@ -645,8 +645,7 @@ export class DataStackClient {
const msg = String(ack.error || 'Socket event rejected');
const authErr = classifyAuthRejection(code, msg);
if (authErr) {
this.heartbeatStopped = true;
this.stopHeartbeatMonitor();
this.disconnect();
reject(
new DataStackAuthError(msg, authErr.code, authErr.statusCode, ack)
);
@@ -902,8 +901,7 @@ export class DataStackClient {
const msg = String(ack.error || 'Batch rejected');
const authErr = classifyAuthRejection(code, msg);
if (authErr) {
this.heartbeatStopped = true;
this.stopHeartbeatMonitor();
this.disconnect();
reject(
new DataStackAuthError(
msg,
@@ -1055,7 +1053,7 @@ export class DataStackClient {
private async deliverChunkWithSplitAndRetry(
chunk: DataStackEnvelope[],
allowSpoolFallback = true
): Promise<DataStackSendResult> {
): Promise<DataStackSendResult & { unprocessedEnvelopes?: DataStackEnvelope[] }> {
let pending = [...chunk];
let totalAccepted = 0;
let totalDuplicates = 0;
@@ -1084,6 +1082,18 @@ export class DataStackClient {
pending = pending.filter((item) => rateLimitedSet.has(item.eventId));
if (attempt >= this.config.maxRetries) {
if (!allowSpoolFallback) {
return {
success: false,
transport: lastTransport,
accepted: totalAccepted,
duplicates: totalDuplicates,
spooled: totalSpooled,
rateLimitedEventIds: pending.map((p) => p.eventId),
unprocessedItems: pending.map((p) => p.data),
unprocessedEnvelopes: pending,
};
}
throw new DataStackRateLimitError(
`Rate limit exceeded for ${pending.length} event(s) after ${this.config.maxRetries} retries.`,
pending.map((p) => p.eventId)
@@ -1105,34 +1115,57 @@ export class DataStackClient {
pending.slice(mid),
allowSpoolFallback
);
const combinedEnvelopes = [
...(left.unprocessedEnvelopes || []),
...(right.unprocessedEnvelopes || []),
];
const combinedItems = [
...(left.unprocessedItems || []),
...(right.unprocessedItems || []),
];
return {
success: left.success && right.success,
transport: right.transport,
accepted: (left.accepted || 0) + (right.accepted || 0),
duplicates: (left.duplicates || 0) + (right.duplicates || 0),
spooled: (left.spooled || 0) + (right.spooled || 0),
...(combinedItems.length > 0 ? { unprocessedItems: combinedItems } : {}),
...(combinedEnvelopes.length > 0
? { unprocessedEnvelopes: combinedEnvelopes }
: {}),
};
}
if (
err instanceof DataStackRateLimitError &&
attempt < this.config.maxRetries
) {
if (err instanceof DataStackRateLimitError) {
const rlSet = new Set(err.rateLimitedEventIds);
if (rlSet.size > 0 && rlSet.size < pending.length) {
totalAccepted += pending.length - rlSet.size;
pending = pending.filter((p) => rlSet.has(p.eventId));
}
attempt += 1;
await new Promise((r) => setTimeout(r, 50 * attempt));
continue;
if (attempt < this.config.maxRetries) {
attempt += 1;
await new Promise((r) => setTimeout(r, 50 * attempt));
continue;
}
if (!allowSpoolFallback) {
return {
success: false,
transport: lastTransport,
accepted: totalAccepted,
duplicates: totalDuplicates,
spooled: totalSpooled,
rateLimitedEventIds: pending.map((p) => p.eventId),
unprocessedItems: pending.map((p) => p.data),
unprocessedEnvelopes: pending,
};
}
throw err;
}
if (
err instanceof DataStackAuthError ||
err instanceof DataStackQuotaExceededError ||
err instanceof DataStackPayloadTooLargeError ||
err instanceof DataStackRateLimitError ||
err instanceof DataStackConfigError
) {
throw err;
@@ -1160,6 +1193,7 @@ export class DataStackClient {
duplicates: totalDuplicates,
spooled: totalSpooled,
unprocessedItems: pending.map((p) => p.data),
...(pending.length > 0 ? { unprocessedEnvelopes: pending } : {}),
};
}
@@ -1365,12 +1399,22 @@ export class DataStackClient {
const batch = this.spooler.peekBatch(this.config.batchChunkSize);
if (batch.length === 0) break;
const res = await this.deliverChunkWithSplitAndRetry(batch, false);
if (res.success) {
this.spooler.removeFirst(batch.length);
drained += batch.length;
} else {
const processedNow = res.accepted || 0;
const duplicatesNow = res.duplicates || 0;
const remaining = res.unprocessedEnvelopes ?? [];
if (!res.success && processedNow === 0 && duplicatesNow === 0) {
break;
}
if (remaining.length > 0) {
const acked = this.spooler.ackAndRequeueFront(batch.length, remaining);
drained += acked;
break;
}
this.spooler.ackAndRequeueFront(batch.length, []);
drained += batch.length;
}
} finally {
this.isDrainingSpool = false;
+5 -1
View File
@@ -11,7 +11,8 @@
}
},
"scripts": {
"build": "tsc -p tsconfig.json"
"build": "tsc -p tsconfig.json",
"prepare": "npm run build"
},
"files": [
"dist",
@@ -28,6 +29,9 @@
"csv-parse": "^7.0.3",
"socket.io-client": "^4.8.4"
},
"devDependencies": {
"typescript": "^5.9.3"
},
"keywords": [
"datastack",
"telemetry",
+58
View File
@@ -138,6 +138,64 @@ export class DiskSpooler {
}
}
/**
* Atomically removes `ackedCount` head lines from `spool.jsonl` while
* preserving any `unprocessed` envelopes at the FRONT of the queue in a
* single atomic temp-file + `fsyncSync` + `renameSync` operation (FIFO safe).
*/
public ackAndRequeueFront(
ackedCount: number,
unprocessed: DataStackEnvelope[] = []
): number {
if (!this.enabled || ackedCount <= 0 || !fs.existsSync(this.spoolFile)) {
return 0;
}
const raw = fs.readFileSync(this.spoolFile, 'utf8');
const allLines = raw
.split('\n')
.map((l) => l.trim())
.filter(Boolean);
const consumed = Math.min(Math.floor(ackedCount), allLines.length);
const tailLines = allLines.slice(consumed);
const headLines = unprocessed.map((evt) => JSON.stringify(evt));
const combined = [...headLines, ...tailLines];
if (combined.length === 0) {
try {
fs.unlinkSync(this.spoolFile);
} catch {
// Ignore if already removed
}
return consumed;
}
this.ensureDir();
const tmpFile = `${this.spoolFile}.${process.pid}.${Date.now()}.tmp`;
const payload = combined.join('\n') + '\n';
try {
const fd = fs.openSync(tmpFile, 'w');
try {
fs.writeFileSync(fd, payload, 'utf8');
fs.fsyncSync(fd);
} finally {
fs.closeSync(fd);
}
fs.renameSync(tmpFile, this.spoolFile);
} catch (err) {
try {
if (fs.existsSync(tmpFile)) {
fs.unlinkSync(tmpFile);
}
} catch {
// Ignore cleanup failure
}
throw err;
}
return Math.max(0, consumed - headLines.length);
}
public count(): number {
if (!this.enabled || !fs.existsSync(this.spoolFile)) {
return 0;
File diff suppressed because it is too large. Load diff
+345 -8
View File
@@ -1,4 +1,5 @@
const crypto = require('node:crypto');
const dnsPromises = require('node:dns/promises');
const net = require('node:net');
const express = require('express');
const http = require('http');
@@ -8,6 +9,11 @@ const { Pool } = require('pg');
const Redis = require('ioredis');
const { setupSocketHandlers, normalizeClientId } = require('./socket/handlers');
const { CsvWriterWorker } = require('./workers/csvWriter');
const {
mountFilesManager,
syncInMemoryAdminSessionState,
clearInMemoryAdminSessionStateForTests,
} = require('./filesManager');
function parseAllowedCorsOrigins(raw = process.env.DATASTACK_CORS_ORIGINS) {
if (typeof raw !== 'string') return null;
@@ -36,7 +42,14 @@ function corsOriginDelegate(origin, callback) {
const PORT = Number(process.env.PORT_SOCKET) || 4000;
const app = express();
app.use(cors({ origin: corsOriginDelegate }));
app.use(express.json({ limit: '10mb' }));
const jsonBodyParser = express.json({ limit: '10mb' });
app.use((req, res, next) => {
if (req.path === '/vcon/files-engine' || req.path.startsWith('/vcon/files-engine/')) {
return next();
}
return jsonBodyParser(req, res, next);
});
let filesManagerHandle = null;
const REDIS_BUFFER_KEY = 'datastack:buffer:events';
const REDIS_RECENT_KEY = 'datastack:recent:events';
@@ -1313,6 +1326,204 @@ function formatOutboundWebhookBody(_url, payload, timestamp) {
});
}
const WEBHOOK_ALWAYS_BLOCKED_HOSTNAMES = new Set([
'metadata.google.internal',
'metadata.goog',
'metadata',
'instance-data',
'instance-data.ec2.internal',
]);
const WEBHOOK_PRIVATE_HOSTNAMES = new Set([
'localhost',
'localhost.localdomain',
'ip6-localhost',
'ip6-loopback',
]);
const WEBHOOK_ALWAYS_BLOCKED_IPS = (() => {
const list = new net.BlockList();
list.addSubnet('0.0.0.0', 8, 'ipv4');
list.addSubnet('169.254.0.0', 16, 'ipv4');
list.addSubnet('224.0.0.0', 4, 'ipv4');
list.addSubnet('240.0.0.0', 4, 'ipv4');
list.addAddress('::', 'ipv6');
list.addSubnet('fe80::', 10, 'ipv6');
list.addSubnet('ff00::', 8, 'ipv6');
return list;
})();
const WEBHOOK_PRIVATE_IPS = (() => {
const list = new net.BlockList();
list.addSubnet('127.0.0.0', 8, 'ipv4');
list.addSubnet('10.0.0.0', 8, 'ipv4');
list.addSubnet('172.16.0.0', 12, 'ipv4');
list.addSubnet('192.168.0.0', 16, 'ipv4');
list.addSubnet('100.64.0.0', 10, 'ipv4');
list.addSubnet('198.18.0.0', 15, 'ipv4');
list.addAddress('::1', 'ipv6');
list.addSubnet('fc00::', 7, 'ipv6');
return list;
})();
function isPrivateWebhookUrlAllowed(env = process.env) {
const raw = String(env.DATASTACK_ALLOW_PRIVATE_URLS || '')
.trim()
.toLowerCase();
return raw === 'true' || raw === '1' || raw === 'yes';
}
function normalizeWebhookIp(rawHost) {
let clean = String(rawHost || '').trim().toLowerCase();
if (clean.startsWith('[') && clean.endsWith(']')) {
clean = clean.slice(1, -1);
}
const zoneIdx = clean.indexOf('%');
if (zoneIdx >= 0) clean = clean.slice(0, zoneIdx);
clean = clean.replace(/\.+$/, '');
if (!clean) return null;
if (clean.startsWith('::ffff:')) {
const mapped = clean.slice('::ffff:'.length);
if (net.isIP(mapped) === 4) {
return { ip: mapped, family: 4 };
}
const hexParts = mapped.split(':');
if (hexParts.length === 2) {
const hi = Number.parseInt(hexParts[0], 16);
const lo = Number.parseInt(hexParts[1], 16);
if (
Number.isInteger(hi) &&
Number.isInteger(lo) &&
hi >= 0 &&
hi <= 0xffff &&
lo >= 0 &&
lo <= 0xffff
) {
const ipv4 = `${(hi >> 8) & 0xff}.${hi & 0xff}.${(lo >> 8) & 0xff}.${lo & 0xff}`;
if (net.isIP(ipv4) === 4) {
return { ip: ipv4, family: 4 };
}
}
}
}
const family = net.isIP(clean);
if (family === 4 || family === 6) {
return { ip: clean, family };
}
return null;
}
function checkWebhookIpAllowed(rawIp, allowPrivate) {
const parsed = normalizeWebhookIp(rawIp);
if (!parsed) return { allowed: false, reason: 'Invalid IP address' };
const ipType = parsed.family === 4 ? 'ipv4' : 'ipv6';
if (WEBHOOK_ALWAYS_BLOCKED_IPS.check(parsed.ip, ipType)) {
return {
allowed: false,
reason: `Address '${parsed.ip}' is in a restricted link-local/metadata/reserved range`,
};
}
if (!allowPrivate && WEBHOOK_PRIVATE_IPS.check(parsed.ip, ipType)) {
return {
allowed: false,
reason: `Address '${parsed.ip}' is a private or loopback network address`,
};
}
return { allowed: true };
}
async function validateOutboundWebhookUrlAsync(rawUrl, options = {}) {
const allowPrivate =
options.allowPrivate !== undefined
? Boolean(options.allowPrivate)
: isPrivateWebhookUrlAllowed(options.env || process.env);
let parsed;
try {
parsed = new URL(String(rawUrl || '').trim());
} catch {
return { allowed: false, code: 'INVALID_URL', error: 'Webhook URL is not a valid URL.' };
}
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
return {
allowed: false,
code: 'SSRF_BLOCKED_HOST',
error: 'Webhook URL must use http:// or https://.',
};
}
if (parsed.username || parsed.password) {
return {
allowed: false,
code: 'SSRF_BLOCKED_HOST',
error: 'Webhook URL must not embed credentials.',
};
}
let host = String(parsed.hostname || '').trim().toLowerCase();
if (host.startsWith('[') && host.endsWith(']')) host = host.slice(1, -1);
host = host.replace(/\.+$/, '');
if (!host) {
return { allowed: false, code: 'INVALID_URL', error: 'Webhook URL hostname is empty.' };
}
if (WEBHOOK_ALWAYS_BLOCKED_HOSTNAMES.has(host)) {
return {
allowed: false,
code: 'SSRF_BLOCKED_HOST',
error: `Webhook target '${host}' is blocked (cloud metadata/internal endpoint).`,
};
}
const literalIp = normalizeWebhookIp(host);
if (literalIp) {
const ipCheck = checkWebhookIpAllowed(host, allowPrivate);
if (!ipCheck.allowed) {
return { allowed: false, code: 'SSRF_BLOCKED_HOST', error: ipCheck.reason };
}
return { allowed: true, url: parsed.toString() };
}
if (!allowPrivate) {
if (
WEBHOOK_PRIVATE_HOSTNAMES.has(host) ||
host.endsWith('.localhost') ||
host.endsWith('.local') ||
host.endsWith('.internal') ||
!host.includes('.')
) {
return {
allowed: false,
code: 'SSRF_BLOCKED_HOST',
error: `Webhook target '${host}' is a private/internal hostname.`,
};
}
}
try {
const lookupFn =
options.dnsLookup ||
((h) => dnsPromises.lookup(h, { all: true, verbatim: true }));
const records = await lookupFn(host);
for (const rec of records) {
const check = checkWebhookIpAllowed(rec.address, allowPrivate);
if (!check.allowed) {
return {
allowed: false,
code: 'SSRF_BLOCKED_HOST',
error: `Webhook hostname '${host}' resolved to blocked address: ${check.reason}`,
};
}
}
} catch {
// Let fetch handle unresolved DNS names cleanly
}
return { allowed: true, url: parsed.toString() };
}
async function triggerWebhookAlert(payload, options = {}) {
try {
const enabled =
@@ -1329,6 +1540,19 @@ async function triggerWebhookAlert(payload, options = {}) {
return { dispatched: false, reason: 'disabled_or_empty_url' };
}
const ssrfCheck = await validateOutboundWebhookUrlAsync(url, {
allowPrivate: options.allowPrivate,
dnsLookup: options.dnsLookup,
});
if (!ssrfCheck.allowed) {
return {
dispatched: false,
code: ssrfCheck.code || 'SSRF_BLOCKED_HOST',
reason: ssrfCheck.code || 'SSRF_BLOCKED_HOST',
error: ssrfCheck.error,
};
}
const cooldownSec = Math.max(
0,
Number(
@@ -1374,12 +1598,22 @@ async function triggerWebhookAlert(payload, options = {}) {
const fetchImpl = options.fetchImpl || global.fetch;
try {
const res = await fetchImpl(url, {
const res = await fetchImpl(ssrfCheck.url, {
method: 'POST',
headers,
body: rawBody,
redirect: 'manual',
signal: controller.signal,
});
if (res.status >= 300 && res.status < 400) {
return {
dispatched: false,
code: 'SSRF_BLOCKED_HOST',
reason: 'SSRF_BLOCKED_HOST',
status: res.status,
error: `Webhook endpoint attempted an unsupported HTTP ${res.status} redirect.`,
};
}
return { dispatched: true, status: res.status };
} finally {
clearTimeout(timeout);
@@ -1475,6 +1709,32 @@ async function ensureDatabaseSchema() {
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`);
await pool.query(`
CREATE TABLE IF NOT EXISTS admin_team_members (
id TEXT PRIMARY KEY,
email TEXT NOT NULL UNIQUE,
role TEXT NOT NULL DEFAULT 'viewer',
status TEXT NOT NULL DEFAULT 'active',
password_salt BYTEA NOT NULL,
password_hash BYTEA NOT NULL,
token_version INTEGER NOT NULL DEFAULT 1,
invited_by TEXT NOT NULL DEFAULT 'owner',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`);
await pool.query(`
CREATE TABLE IF NOT EXISTS admin_audit_logs (
id TEXT PRIMARY KEY,
actor_email TEXT NOT NULL,
actor_role TEXT NOT NULL,
action TEXT NOT NULL,
target TEXT NOT NULL,
ip_address TEXT NOT NULL DEFAULT '',
metadata_json TEXT NOT NULL DEFAULT '{}',
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`);
await pool.query(`
ALTER TABLE client_sdks
ADD COLUMN IF NOT EXISTS rate_limit INTEGER NOT NULL DEFAULT 100,
@@ -1822,7 +2082,7 @@ app.get('/internal/recent', async (req, res) => {
});
// Internal endpoint triggered by Next.js SDKManager when a client is deleted, revoked, rotated, or synced
app.post('/internal/client-sync', (req, res) => {
app.post('/internal/client-sync', async (req, res) => {
const action = req.body && req.body.action ? String(req.body.action) : '';
if (action === 'bulk_sync' && Array.isArray(req.body.clients)) {
@@ -1942,25 +2202,83 @@ app.post('/internal/client-sync', (req, res) => {
});
}
} else if (action === 'updated') {
const prevClient = inMemoryClientRegistry.get(normalizedId);
const nextStatus =
clientPayload && clientPayload.status === 'revoked' ? 'revoked' : 'active';
const nextToken = clientPayload ? String(clientPayload.token || '') : '';
const nextAllowedIps =
clientPayload && typeof clientPayload.allowedIps === 'string'
? clientPayload.allowedIps.trim()
: '';
const nextExpiresAt =
clientPayload && clientPayload.expiresAt
? String(clientPayload.expiresAt)
: null;
if (clientPayload) {
inMemoryClientRegistry.set(normalizedId, {
clientId: normalizedId,
name: String(clientPayload.name || normalizedId),
status: clientPayload.status === 'revoked' ? 'revoked' : 'active',
token: String(clientPayload.token || ''),
status: nextStatus,
token: nextToken,
rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
storageQuotaMb: Math.max(0, Math.floor(Number(clientPayload.storageQuotaMb) || 0)),
allowedIps: typeof clientPayload.allowedIps === 'string' ? clientPayload.allowedIps : '',
expiresAt: clientPayload.expiresAt ? String(clientPayload.expiresAt) : null,
allowedIps: nextAllowedIps,
expiresAt: nextExpiresAt,
});
}
io.to(`client:${normalizedId}`).disconnectSockets(true);
const tokenExpired = Boolean(
nextExpiresAt && isClientTokenExpired(nextExpiresAt),
);
const shouldDisconnectRoom =
!clientPayload ||
nextStatus === 'revoked' ||
tokenExpired ||
Boolean(
prevClient &&
(prevClient.status !== nextStatus ||
(nextToken && prevClient.token !== nextToken) ||
(prevClient.allowedIps || '').trim() !== nextAllowedIps ||
(prevClient.expiresAt || null) !== nextExpiresAt),
);
let disconnected = false;
if (shouldDisconnectRoom) {
io.to(`client:${normalizedId}`).disconnectSockets(true);
disconnected = true;
} else if (!prevClient) {
const liveSockets = await io.in(`client:${normalizedId}`).fetchSockets();
for (const sock of liveSockets) {
const sockToken =
sock.data && sock.data.token ? String(sock.data.token) : '';
const sockIp =
sock.data && sock.data.clientIp ? String(sock.data.clientIp) : '';
const tokenMismatch = Boolean(
nextToken && sockToken && sockToken !== nextToken,
);
const ipForbidden = Boolean(
nextAllowedIps && (!sockIp || !isClientIpAllowed(sockIp, nextAllowedIps)),
);
if (tokenMismatch || ipForbidden) {
sock.disconnect(true);
disconnected = true;
}
}
}
return res.json({ success: true, disconnected });
}
}
res.json({ success: true });
});
app.post('/internal/admin-session-sync', requireInternalAuth, (req, res) => {
const synced = syncInMemoryAdminSessionState(req.body || {});
res.json({ success: true, synced });
});
function stopTimers() {
if (flushTimer) {
clearInterval(flushTimer);
@@ -2016,6 +2334,10 @@ async function gracefulShutdown(signal, options = {}) {
stopTimers();
// Close Socket.IO & HTTP listeners before final flush so late traffic cannot enter after drain starts
if (filesManagerHandle && typeof filesManagerHandle.close === 'function') {
await Promise.race([filesManagerHandle.close().catch(() => undefined), sleepMs(2000)]);
filesManagerHandle = null;
}
await closeServerGracefully();
try {
@@ -2097,6 +2419,16 @@ async function startIngestionServer() {
if (isShuttingDown) return;
if (!filesManagerHandle) {
filesManagerHandle = await mountFilesManager({
app,
server,
rootDir: writer.baseDir,
getDbPool: () => dbPool,
corsOriginDelegate,
});
}
await new Promise((resolve) => {
server.listen(PORT, () => {
console.log(
@@ -2144,7 +2476,12 @@ module.exports = {
isClientIpAllowed,
isClientTokenExpired,
triggerWebhookAlert,
validateOutboundWebhookUrlAsync,
clearWebhookCooldownCacheForTests,
decryptVaultSecret,
formatOutboundWebhookBody,
mountFilesManager,
syncInMemoryAdminSessionState,
clearInMemoryAdminSessionStateForTests,
getFilesManagerHandle: () => filesManagerHandle,
};
+31
View File
@@ -15,6 +15,22 @@ const REDIS_RATE_LIMIT_LUA_SCRIPT = [
"return c",
].join("\n");
const DEFAULT_MAX_BATCH_ITEMS = 1000;
const MIN_BATCH_ITEMS = 1;
const MAX_BATCH_ITEMS_CEILING = 10000;
function resolveMaxBatchItems() {
const raw = process.env.DATASTACK_MAX_BATCH_ITEMS;
if (raw === undefined || String(raw).trim() === "") {
return DEFAULT_MAX_BATCH_ITEMS;
}
const parsed = Math.floor(Number(raw));
if (!Number.isFinite(parsed)) {
return DEFAULT_MAX_BATCH_ITEMS;
}
return Math.min(MAX_BATCH_ITEMS_CEILING, Math.max(MIN_BATCH_ITEMS, parsed));
}
function generateFallbackEventId() {
return `evt_${Date.now()}_${crypto.randomBytes(6).toString("hex")}`;
}
@@ -597,6 +613,21 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
return;
}
const maxBatchItems = resolveMaxBatchItems();
if (batch.length > maxBatchItems) {
if (callback) {
callback({
success: false,
processed: 0,
rejected: batch.length,
code: "BATCH_LIMIT_EXCEEDED",
error: `BATCH_LIMIT_EXCEEDED: Batch size (${batch.length}) exceeds maximum allowed ${maxBatchItems} items per request.`,
maxBatchItems,
});
}
return;
}
const clientIp = socket.data.clientIp || extractSocketClientIp(socket);
const dbCheck = await verifyClientInDb(
clientId,
+308
View File
@@ -0,0 +1,308 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import http from 'node:http';
import os from 'node:os';
import path from 'node:path';
import express from 'express';
import { createAdminSessionToken } from '../core/auth/adminSession';
const fm = require('../server/filesManager');
test('FM-REQ-02: buildCloudCmdConfig enforces RCE lockdown, root jail, zip packer, and scrubs default tokens', () => {
const tmpRoot = path.resolve(os.tmpdir(), 'datastack-fm-cfg-test');
const cfg = fm.buildCloudCmdConfig(tmpRoot);
assert.equal(cfg.root, tmpRoot);
assert.equal(cfg.prefix, '/vcon/files-engine');
assert.equal(cfg.console, false);
assert.equal(cfg.terminal, false);
assert.equal(cfg.configDialog, false);
assert.equal(cfg.auth, false);
assert.equal(cfg.open, false);
assert.equal(cfg.online, false);
assert.equal(cfg.contact, false);
assert.equal(cfg.oneFilePanel, true);
assert.equal(cfg.packer, 'zip');
assert.equal(cfg.editor, 'edward');
assert.equal(cfg.password, '');
assert.equal(cfg.exportToken, '');
assert.equal(cfg.importToken, '');
assert.equal(cfg.dropboxToken, '');
});
test('FM-REQ-04: authenticateFilesRequest fails closed on missing prod secret and DB outages', async () => {
const token = await createAdminSessionToken(
'owner@datastack.dev',
fm.DEV_DEFAULT_SESSION_SECRET,
Date.now() + 60_000,
1,
'owner',
'owner-1',
);
const prodMissing = await fm.authenticateFilesRequest({
cookieHeader: `datastack_admin_session=${token}`,
dbPool: null,
env: { NODE_ENV: 'production', DATASTACK_ADMIN_SESSION_SECRET: '' },
});
assert.equal(prodMissing.valid, false);
assert.equal(prodMissing.status, 503);
assert.equal(prodMissing.code, 'FILES_AUTH_UNCONFIGURED');
const dbUnavailable = await fm.authenticateFilesRequest({
cookieHeader: `datastack_admin_session=${token}`,
dbPool: {
query: async () => {
throw new Error('ECONNREFUSED');
},
},
});
assert.equal(dbUnavailable.valid, false);
assert.equal(dbUnavailable.status, 503);
assert.equal(dbUnavailable.code, 'AUTH_DB_UNAVAILABLE');
});
test('FM-REQ-01..06: live CloudCmd mount enforces auth, DB revocation, RBAC, CSRF, *.lock & traversal guards, and full file CRUD + ZIP/UNZIP', async () => {
const tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'datastack-fm-e2e-'));
fs.mkdirSync(path.join(tmpRoot, 'client-alpha'), { recursive: true });
fs.writeFileSync(
path.join(tmpRoot, 'client-alpha', '2026-11-20.csv'),
'eventId,clientId\nev-1,client-alpha\n',
);
fs.writeFileSync(path.join(tmpRoot, 'client-alpha', '2026-11-20.csv.lock'), 'pid=99');
const app = express();
const server = http.createServer(app);
let activeDbPool: { query: (sql: string, params?: unknown[]) => Promise<{ rows: unknown[] }> } | null = null;
const handle = await fm.mountFilesManager({
app,
server,
rootDir: tmpRoot,
getDbPool: () => activeDbPool,
});
try {
assert.equal(handle.engineReady, true, `CloudCmd failed to init: ${handle.initErrorMessage}`);
assert.equal(handle.socketPath, '/vcon/files-engine/socket.io');
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', () => resolve()));
const address = server.address() as { port: number };
const baseUrl = `http://127.0.0.1:${address.port}`;
const hostHeader = `127.0.0.1:${address.port}`;
const ownerToken = await createAdminSessionToken(
'owner@datastack.dev',
fm.DEV_DEFAULT_SESSION_SECRET,
Date.now() + 60_000,
1,
'owner',
'owner-1',
);
const viewerToken = await createAdminSessionToken(
'viewer@datastack.dev',
fm.DEV_DEFAULT_SESSION_SECRET,
Date.now() + 60_000,
1,
'viewer',
'member-viewer-1',
);
const ownerHeaders = {
Cookie: `datastack_admin_session=${ownerToken}`,
Origin: baseUrl,
Host: hostHeader,
};
const viewerHeaders = {
Cookie: `datastack_admin_session=${viewerToken}`,
Origin: baseUrl,
Host: hostHeader,
};
// 1. Unauthenticated access blocked
assert.equal((await fetch(`${baseUrl}/vcon/files-engine/_status`)).status, 401);
assert.equal((await fetch(`${baseUrl}/vcon/files-engine/`)).status, 401);
// 2. Authenticated status probe & locked HTML output
const statusRes = await fetch(`${baseUrl}/vcon/files-engine/_status`, {
headers: { Cookie: ownerHeaders.Cookie },
});
assert.equal(statusRes.status, 200);
const statusJson = (await statusRes.json()) as { ok: boolean; role: string; readOnly: boolean };
assert.equal(statusJson.ok, true);
assert.equal(statusJson.role, 'owner');
assert.equal(statusJson.readOnly, false);
const htmlRes = await fetch(`${baseUrl}/vcon/files-engine/`, {
headers: { Cookie: ownerHeaders.Cookie },
});
assert.equal(htmlRes.status, 200);
const html = await htmlRes.text();
assert.ok(html.includes('/vcon/files-engine'));
assert.ok(html.includes('"console":false'));
assert.ok(html.includes('"terminal":false'));
assert.ok(html.includes('"configDialog":false'));
assert.ok(html.includes('"packer":"zip"'));
// 3. Create, rename, ZIP, UNZIP, download, and delete
const createRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/fs/client-alpha/note.csv`, {
method: 'PUT',
headers: ownerHeaders,
body: 'k,v\n1,ok\n',
});
assert.equal(createRes.status, 200);
assert.equal(fs.readFileSync(path.join(tmpRoot, 'client-alpha', 'note.csv'), 'utf8'), 'k,v\n1,ok\n');
const renameRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/rename`, {
method: 'PUT',
headers: { ...ownerHeaders, 'Content-Type': 'application/json' },
body: JSON.stringify({ from: '/client-alpha/note.csv', to: '/client-alpha/note-renamed.csv' }),
});
assert.equal(renameRes.status, 200);
assert.equal(fs.existsSync(path.join(tmpRoot, 'client-alpha', 'note-renamed.csv')), true);
const packRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/pack`, {
method: 'PUT',
headers: { ...ownerHeaders, 'Content-Type': 'application/json' },
body: JSON.stringify({
from: '/client-alpha',
to: '/client-alpha/bundle.zip',
names: ['note-renamed.csv'],
}),
});
assert.equal(packRes.status, 200);
assert.equal(fs.existsSync(path.join(tmpRoot, 'client-alpha', 'bundle.zip')), true);
fs.mkdirSync(path.join(tmpRoot, 'client-alpha', 'extracted'), { recursive: true });
const extractRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/extract`, {
method: 'PUT',
headers: { ...ownerHeaders, 'Content-Type': 'application/json' },
body: JSON.stringify({
from: '/client-alpha/bundle.zip',
to: '/client-alpha/extracted',
}),
});
assert.equal(extractRes.status, 200);
assert.equal(fs.existsSync(path.join(tmpRoot, 'client-alpha', 'extracted', 'note-renamed.csv')), true);
const downloadRes = await fetch(
`${baseUrl}/vcon/files-engine/fs/client-alpha/note-renamed.csv?download`,
{ headers: { Cookie: ownerHeaders.Cookie } },
);
assert.equal(downloadRes.status, 200);
assert.equal(await downloadRes.text(), 'k,v\n1,ok\n');
const deleteRes = await fetch(
`${baseUrl}/vcon/files-engine/api/v1/fs/client-alpha/note-renamed.csv`,
{ method: 'DELETE', headers: ownerHeaders },
);
assert.equal(deleteRes.status, 200);
assert.equal(fs.existsSync(path.join(tmpRoot, 'client-alpha', 'note-renamed.csv')), false);
// 4. Viewer role: GET allowed, mutations blocked (403)
const viewerList = await fetch(`${baseUrl}/vcon/files-engine/api/v1/fs/`, {
headers: { Cookie: viewerHeaders.Cookie },
});
assert.equal(viewerList.status, 200);
const viewerWrite = await fetch(`${baseUrl}/vcon/files-engine/api/v1/fs/client-alpha/nope.csv`, {
method: 'PUT',
headers: viewerHeaders,
body: 'blocked',
});
assert.equal(viewerWrite.status, 403);
assert.equal(((await viewerWrite.json()) as { code: string }).code, 'INSUFFICIENT_ROLE_PERMISSION');
// 5. Cross-origin CSRF rejected (403)
const csrfRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/fs/client-alpha/csrf.csv`, {
method: 'PUT',
headers: { Cookie: ownerHeaders.Cookie, Origin: 'https://evil.example', Host: hostHeader },
body: 'blocked',
});
assert.equal(csrfRes.status, 403);
assert.equal(((await csrfRes.json()) as { code: string }).code, 'CSRF_REJECTED');
// 6. Cooperative *.lock protection in URL and JSON body
const lockUrlRes = await fetch(
`${baseUrl}/vcon/files-engine/api/v1/fs/client-alpha/2026-11-20.csv.lock`,
{ method: 'DELETE', headers: ownerHeaders },
);
assert.equal(lockUrlRes.status, 403);
assert.equal(((await lockUrlRes.json()) as { code: string }).code, 'LOCKFILE_PROTECTED');
const lockBodyRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/rename`, {
method: 'PUT',
headers: { ...ownerHeaders, 'Content-Type': 'application/json' },
body: JSON.stringify({
from: '/client-alpha/2026-11-20.csv.lock',
to: '/client-alpha/hijack.csv',
}),
});
assert.equal(lockBodyRes.status, 403);
assert.equal(((await lockBodyRes.json()) as { code: string }).code, 'LOCKFILE_PROTECTED');
// 7. Path traversal & runtime config PATCH blocked
const travRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/fs/..%2F..%2Fpackage.json`, {
headers: { Cookie: ownerHeaders.Cookie },
});
assert.equal(travRes.status, 403);
assert.equal(((await travRes.json()) as { code: string }).code, 'PATH_OUTSIDE_ROOT');
const cfgPatchRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/config`, {
method: 'PATCH',
headers: { ...ownerHeaders, 'Content-Type': 'application/json' },
body: JSON.stringify({ console: true }),
});
assert.equal(cfgPatchRes.status, 403);
assert.equal(((await cfgPatchRes.json()) as { code: string }).code, 'CONFIG_LOCKED');
// 8. Revoked DB token_version immediately rejects session (401 SESSION_REVOKED)
activeDbPool = {
query: async (sql: string) => {
if (sql.includes('admin_users')) {
return { rows: [{ email: 'owner@datastack.dev', token_version: 2 }] };
}
return { rows: [] };
},
};
const revokedRes = await fetch(`${baseUrl}/vcon/files-engine/_status`, {
headers: { Cookie: ownerHeaders.Cookie },
});
assert.equal(revokedRes.status, 401);
assert.equal(((await revokedRes.json()) as { code: string }).code, 'SESSION_REVOKED');
} finally {
await handle.close();
await new Promise<void>((resolve) => server.close(() => resolve()));
fs.rmSync(tmpRoot, { recursive: true, force: true });
}
});
test('FM-REQ-07..09: proxy routing, SW bypass, Sidebar nav, and /vcon/files workspace are cleanly wired', () => {
const root = process.cwd();
const nextConfig = fs.readFileSync(path.join(root, 'next.config.ts'), 'utf8');
const coolifyNginx = fs.readFileSync(path.join(root, 'docker', 'coolify-nginx.conf'), 'utf8');
const localNginx = fs.readFileSync(path.join(root, 'nginx', 'default.conf'), 'utf8');
const swJs = fs.readFileSync(path.join(root, 'public', 'sw.js'), 'utf8');
const sidebar = fs.readFileSync(path.join(root, 'components', 'layout', 'Sidebar.tsx'), 'utf8');
const page = fs.readFileSync(path.join(root, 'app', 'vcon', 'files', 'page.tsx'), 'utf8');
const workspace = fs.readFileSync(
path.join(root, 'components', 'sections', 'FilesManagerWorkspace.tsx'),
'utf8',
);
assert.ok(nextConfig.includes("source: '/vcon/files-engine/:path*'"));
assert.ok(coolifyNginx.includes('location /vcon/files-engine'));
assert.ok(localNginx.includes('location /vcon/files-engine'));
assert.ok(swJs.includes("url.pathname.startsWith('/vcon/files-engine/')"));
assert.ok(sidebar.includes("{ name: 'Files Manager', href: '/vcon/files', icon: 'FolderOpen' }"));
assert.ok(page.includes('FilesManagerWorkspace'));
const statusRoute = fs.readFileSync(
path.join(root, 'app', 'api', 'files', 'status', 'route.ts'),
'utf8',
);
assert.ok(statusRoute.includes('/vcon/files-engine/_status'));
assert.ok(workspace.includes('/api/files/status'));
assert.ok(workspace.includes('files-manager-offline'));
assert.ok(workspace.includes('files-manager-iframe'));
});
+41 -3
View File
@@ -444,17 +444,17 @@ test("DEPLOY-BUG-02, 03 & 04: Port 4000 memoizes schema DDLs, hydrates rotated J
assert.equal(failedSync, false);
assert.equal(ddlCount, 1);
// 2. Recovery runs 8 DDLs once and hydrates rotated jwt_secret; repeated calls do zero extra DDLs
// 2. Recovery runs 10 DDLs once (7 CREATE TABLE + 3 ALTER TABLE) and hydrates rotated jwt_secret; repeated calls do zero extra DDLs
failDdl = false;
ddlCount = 0;
assert.equal(await ingestionServer.syncSettingsFromDatabase(), true);
assert.equal(ddlCount, 8);
assert.equal(ddlCount, 10);
assert.equal(selectCount, 1);
assert.equal(ingestionServer.runtimeConfig.jwtSecret, rotatedSecret);
assert.equal(await ingestionServer.syncSettingsFromDatabase(), true);
assert.equal(await ingestionServer.syncSettingsFromDatabase(), true);
assert.equal(ddlCount, 8, "Repeated 3s sync ticks must NOT re-run CREATE/ALTER DDL statements");
assert.equal(ddlCount, 10, "Repeated 3s sync ticks must NOT re-run CREATE/ALTER DDL statements");
assert.equal(selectCount, 3);
// 3. Graceful shutdown flushes buffered events, quits Redis, and closes PostgreSQL pool cleanly
@@ -606,3 +606,41 @@ test("V2-Deploy-Fix-01 (DEPLOY-V2-BUG-01, 02 & 03): middleware permits GET /api/
}
});
test("PREPUSH-FIX-01: Nginx preserves upstream real client IP behind Traefik, Dockerfile.web & Compose align runtime/build env vars, and package-lock includes cloudcmd", () => {
for (const confRel of ["docker/coolify-nginx.conf", "nginx/default.conf"]) {
const conf = readFileSync(resolve(process.cwd(), confRel), "utf8");
assert.match(conf, /set_real_ip_from\s+127\.0\.0\.1;/, `${confRel} must trust loopback proxy`);
assert.match(conf, /set_real_ip_from\s+172\.16\.0\.0\/12;/, `${confRel} must trust Docker bridge proxy subnet`);
assert.match(conf, /real_ip_header\s+X-Forwarded-For;/, `${confRel} must extract real IP from X-Forwarded-For`);
assert.match(conf, /real_ip_recursive\s+on;/, `${confRel} must enable real_ip_recursive`);
}
const webDockerfile = readFileSync(resolve(process.cwd(), "docker/Dockerfile.web"), "utf8");
assert.match(
webDockerfile,
/ARG INGESTION_INTERNAL_URL=http:\/\/ingestion:4000[\s\S]*ENV INGESTION_INTERNAL_URL=\$\{INGESTION_INTERNAL_URL\}[\s\S]*RUN npm run build/,
"Dockerfile.web builder must set INGESTION_INTERNAL_URL=http://ingestion:4000 before npm run build",
);
const compose = readFileSync(resolve(process.cwd(), "docker/docker-compose.yml"), "utf8");
for (const envKey of [
"DATASTACK_ALLOW_PRIVATE_URLS",
"DATASTACK_MAX_BATCH_ITEMS",
"DATASTACK_RECENT_LIMIT",
]) {
const matches = compose.match(new RegExp(`^\\s+${envKey}:`, "gm")) ?? [];
assert.equal(matches.length, 2, `Both web and ingestion must forward ${envKey}`);
}
assert.match(compose, /^\s+DATASTACK_MAINTENANCE_INTERVAL_MS:/m);
assert.match(compose, /^\s+DATASTACK_PROXY_MODE:/m);
const lockJson = JSON.parse(
readFileSync(resolve(process.cwd(), "package-lock.json"), "utf8"),
) as {
packages?: Record<string, { dependencies?: Record<string, string> }>;
};
assert.ok(lockJson.packages?.[""]?.dependencies?.cloudcmd, "Root lockfile must declare cloudcmd");
assert.ok(lockJson.packages?.["node_modules/cloudcmd"], "package-lock.json must resolve node_modules/cloudcmd");
});
@@ -227,6 +227,9 @@ test("R3-Phase-02 BUG-05: Singleton periodic scheduler automatically archives ex
const addr = mockS3.address();
assert.ok(addr && typeof addr === "object");
const prevAllowPrivate = process.env.DATASTACK_ALLOW_PRIVATE_URLS;
process.env.DATASTACK_ALLOW_PRIVATE_URLS = "true";
try {
// Configure S3 backup + autoPruneDays (cooldown will be reset so scheduled timer runs)
await SettingsManager.updateSettingsAsync({
@@ -340,6 +343,8 @@ test("R3-Phase-02 BUG-05: Singleton periodic scheduler automatically archives ex
assert.equal(unhydratedResult.ran, false);
assert.equal(unhydratedResult.skippedReason, "DB_UNHYDRATED");
} finally {
if (prevAllowPrivate === undefined) delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
else process.env.DATASTACK_ALLOW_PRIVATE_URLS = prevAllowPrivate;
SettingsManager.stopMaintenanceSchedulerForTests();
await new Promise<void>((resolve) => mockS3.close(() => resolve()));
fs.rmSync(tmpDir, { recursive: true, force: true });
+27 -19
View File
@@ -53,7 +53,6 @@ test("R3-Phase-04 [BUG-11 / ERR-R3-07]: shouldRedirectSimulatorUnauthorized sepa
"AUTH_INVALID",
"CLIENT_REVOKED",
"CLIENT_IP_FORBIDDEN",
"CLIENT_UNAUTHORIZED",
]) {
assert.equal(
shouldRedirectSimulatorUnauthorized(401, {
@@ -134,12 +133,12 @@ test("R3-Phase-04 [BUG-14]: expired clients are separated in UI helpers, ApiEngi
assert.equal(isUiClientExpired("2026-11-19T00:00:00.000Z", nowMs), true);
const sampleClients = [
{ status: "active", isOnline: true, lastConnectedAt: "2026-11-20T11:59:00.000Z", expiresAt: null },
{ status: "active", isOnline: false, lastConnectedAt: "2026-11-20T10:00:00.000Z", expiresAt: "2026-12-01T00:00:00.000Z" },
{ status: "active", isOnline: false, lastConnectedAt: null, expiresAt: null },
{ status: "active", isOnline: false, lastConnectedAt: "2026-11-19T10:00:00.000Z", expiresAt: "2026-11-19T12:00:00.000Z" },
{ status: "active", isOnline: true, lastConnectedAt: "2026-11-20T11:58:00.000Z", expiresAt: "2026-11-20T11:00:00.000Z" },
{ status: "revoked", isOnline: false, lastConnectedAt: null, expiresAt: "2026-11-10T00:00:00.000Z" },
{ status: "active" as const, lastActive: "2026-11-20T11:59:30.000Z", expiresAt: null },
{ status: "active" as const, lastActive: "2026-11-20T10:00:00.000Z", expiresAt: "2026-12-01T00:00:00.000Z" },
{ status: "active" as const, lastActive: undefined, expiresAt: null },
{ status: "active" as const, lastActive: "2026-11-19T10:00:00.000Z", expiresAt: "2026-11-19T12:00:00.000Z" },
{ status: "active" as const, lastActive: "2026-11-20T11:59:45.000Z", expiresAt: "2026-11-20T11:00:00.000Z" },
{ status: "revoked" as const, lastActive: undefined, expiresAt: "2026-11-10T00:00:00.000Z" },
];
assert.equal(classifyUiClientStatus(sampleClients[0], nowMs), "online");
@@ -152,6 +151,7 @@ test("R3-Phase-04 [BUG-14]: expired clients are separated in UI helpers, ApiEngi
const summary = summarizeUiClientCounts(sampleClients, nowMs);
assert.deepEqual(summary, {
total: 6,
active: 3,
online: 1,
offline: 1,
pending: 1,
@@ -160,12 +160,20 @@ test("R3-Phase-04 [BUG-14]: expired clients are separated in UI helpers, ApiEngi
});
// Live integration: ApiEngine.getDashboardMetrics() & GET /api/prometheus
const activeClient = SDKManager.registerClient("r3p4_active_client");
const expiredClient = SDKManager.registerClient("r3p4_expired_client", {
const activeClient = SDKManager.registerClient({
clientId: "r3p4_active_client",
name: "R3P4 Active Client",
});
const expiredClient = SDKManager.registerClient({
clientId: "r3p4_expired_client",
name: "R3P4 Expired Client",
expiresAt: "2020-01-01T00:00:00.000Z",
});
const revokedClient = SDKManager.registerClient("r3p4_revoked_client");
SDKManager.revokeClient(revokedClient.id);
const revokedClient = SDKManager.registerClient({
clientId: "r3p4_revoked_client",
name: "R3P4 Revoked Client",
});
SDKManager.updateClient(revokedClient.clientId, { status: "revoked" });
const prevSettings = SettingsManager.getSettings();
try {
@@ -200,9 +208,9 @@ test("R3-Phase-04 [BUG-14]: expired clients are separated in UI helpers, ApiEngi
assert.ok(Number(expiredMatch[1]) >= 1);
assert.ok(Number(revokedMatch[1]) >= 1);
} finally {
SDKManager.deleteClient(activeClient.id);
SDKManager.deleteClient(expiredClient.id);
SDKManager.deleteClient(revokedClient.id);
SDKManager.deleteClient(activeClient.clientId);
SDKManager.deleteClient(expiredClient.clientId);
SDKManager.deleteClient(revokedClient.clientId);
SettingsManager.updateSettings({
enablePrometheusMetrics: prevSettings.enablePrometheusMetrics,
});
@@ -212,37 +220,37 @@ test("R3-Phase-04 [BUG-14]: expired clients are separated in UI helpers, ApiEngi
test("R3-Phase-04 [BUG-16]: RBAC role capabilities and UI surface wiring across VconShell and Admin pages", async () => {
assert.deepEqual(getVconRoleCapabilities("owner"), {
role: "owner",
isOwner: true,
canAdminSettings: true,
canManageTeam: true,
canOperate: true,
isReadOnly: false,
});
assert.deepEqual(getVconRoleCapabilities("admin"), {
role: "admin",
isOwner: false,
canAdminSettings: false,
canManageTeam: false,
canOperate: true,
isReadOnly: false,
});
assert.deepEqual(getVconRoleCapabilities("operator"), {
role: "operator",
isOwner: false,
canAdminSettings: false,
canManageTeam: false,
canOperate: true,
isReadOnly: false,
});
assert.deepEqual(getVconRoleCapabilities("viewer"), {
role: "viewer",
isOwner: false,
canAdminSettings: false,
canManageTeam: false,
canOperate: false,
isReadOnly: true,
});
// Fail-closed default for unknown/missing roles
assert.deepEqual(getVconRoleCapabilities("unknown_role"), {
role: "viewer",
isOwner: false,
canAdminSettings: false,
canManageTeam: false,
canOperate: false,
isReadOnly: true,
});
+618
View File
@@ -0,0 +1,618 @@
import assert from "node:assert/strict";
import fs from "node:fs/promises";
import http from "node:http";
import { createRequire } from "node:module";
import type { AddressInfo } from "node:net";
import os from "node:os";
import path from "node:path";
import test from "node:test";
import zlib from "node:zlib";
import { NextRequest } from "next/server";
process.env.DATASTACK_DISABLE_LISTEN = "1";
import { GET as auditGet } from "../app/api/audit/route";
import { POST as settingsPost } from "../app/api/settings/route";
import { POST as webhookTestPost } from "../app/api/settings/webhook-test/route";
import { POST as s3TestPost } from "../app/api/storage/s3-test/route";
import { SettingsManager } from "../core/adminSettings/settingsManager";
import {
clearWebhookCooldownCacheForTests,
dispatchWebhookAlertAsync,
} from "../core/alerts/webhookNotifier";
import {
createAdminSessionToken,
getAdminSessionSecret,
} from "../core/auth/adminSession";
import {
createInitialAdmin,
createAdminTeamMemberAsync,
resetAdminStateForTests,
revokeAdminSessions,
updateAdminTeamMemberAsync,
deleteAdminTeamMemberAsync,
} from "../core/auth/adminUsers";
import {
clearAuditLogsForTests,
listAdminAuditLogsWithStatusAsync,
recordAdminAuditLogAsync,
} from "../core/auth/auditLog";
import { DataBufferEngine } from "../core/data/buffer";
import { SDKManager } from "../core/sdk/sdkManager";
import { getInternalIngestionHeaders } from "../core/Security/security";
import {
checkIpAddressAllowed,
validateOutboundUrlAsync,
validateOutboundUrlSync,
} from "../core/Security/ssrfGuard";
import { testS3Connection, uploadFileToS3 } from "../core/Storage/s3Archive";
import * as instrumentationHook from "../instrumentation";
const require = createRequire(import.meta.url);
const filesManager = require("../server/filesManager.js") as {
authenticateFilesRequest: (params: {
cookieHeader?: string;
dbPool?: unknown;
env?: NodeJS.ProcessEnv;
nowMs?: number;
}) => Promise<{
valid: boolean;
status?: number;
code?: string;
error?: string;
session?: { email: string; role: string; memberId: string; tokenVersion: number };
}>;
syncInMemoryAdminSessionState: (payload: Record<string, unknown>) => boolean;
clearInMemoryAdminSessionStateForTests: () => void;
};
const ingestionServer = require("../server/index.js") as {
app: (req: http.IncomingMessage, res: http.ServerResponse) => void;
setDbPoolForTests: (pool: unknown) => void;
syncSettingsFromDatabase: () => Promise<boolean>;
validateOutboundWebhookUrlAsync: (
rawUrl: string,
options?: Record<string, unknown>,
) => Promise<{ allowed: boolean; code?: string; error?: string; url?: string }>;
triggerWebhookAlert: (
payload: Record<string, unknown>,
options?: Record<string, unknown>,
) => Promise<{ dispatched: boolean; status?: number; code?: string; reason?: string; error?: string }>;
clearWebhookCooldownCacheForTests: () => void;
stopTimers: () => void;
};
const ORIGINAL_ENV = { ...process.env };
function restoreEnv(): void {
for (const key of Object.keys(process.env)) {
if (!(key in ORIGINAL_ENV)) {
delete process.env[key];
}
}
Object.assign(process.env, ORIGINAL_ENV);
}
async function createOwnerCookie(email = "r4p1-owner@datastack.dev"): Promise<string> {
resetAdminStateForTests();
await createInitialAdmin(email, "OwnerPassword123!");
const secret = getAdminSessionSecret();
const token = await createAdminSessionToken(
email,
secret,
Date.now() + 3600_000,
1,
"owner",
"owner-1",
);
return `datastack_admin_session=${token}`;
}
test.after(async () => {
SettingsManager.stopMaintenanceSchedulerForTests();
SDKManager.stopActivityFlushTimerForTests();
ingestionServer.stopTimers();
ingestionServer.setDbPoolForTests(null);
filesManager.clearInMemoryAdminSessionStateForTests();
await DataBufferEngine.shutdown();
clearAuditLogsForTests();
resetAdminStateForTests();
restoreEnv();
});
test("R4-Phase-01 [AUDIT-01 / ERR-R4-01]: ssrfGuard blocks loopback, RFC1918, metadata, internal hostnames, non-HTTP(S), DNS rebinding & redirects while respecting explicit opt-in", async () => {
delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
const blockedTargets = [
"http://127.0.0.1:4000/internal/settings-sync",
"http://localhost:3000/api/health",
"http://sub.localhost/hook",
"http://[::1]:4000/",
"http://[::ffff:127.0.0.1]/secret",
"http://10.0.0.5:9000/bucket",
"http://172.16.4.2/hook",
"http://192.168.1.50/hook",
"http://0.0.0.0/",
"http://ingestion:4000/internal",
"http://postgres.internal:5432/",
"http://169.254.169.254/latest/meta-data/iam/security-credentials/",
"http://metadata.google.internal/computeMetadata/v1/",
"file:///etc/passwd",
"ftp://files.example.com/archive",
"https://user:pass@hooks.example.com/hook",
];
for (const target of blockedTargets) {
const res = validateOutboundUrlSync(target, { fieldName: "Outbound URL" });
assert.equal(res.allowed, false, `Expected ${target} to be blocked`);
assert.equal(res.code, "SSRF_BLOCKED_HOST", `Expected SSRF_BLOCKED_HOST for ${target}`);
}
// Public HTTPS host passes sync + safe DNS lookup
const okPublic = await validateOutboundUrlAsync("https://hooks.example.com/v1/alert", {
fieldName: "Webhook URL",
dnsLookup: async () => [{ address: "93.184.216.34", family: 4 }],
});
assert.equal(okPublic.allowed, true);
// DNS rebinding hostname resolving to 127.0.0.1 or 169.254.169.254 is blocked
const rebindLoopback = await validateOutboundUrlAsync("https://rebind.example.com/hook", {
fieldName: "Webhook URL",
dnsLookup: async () => [{ address: "127.0.0.1", family: 4 }],
});
assert.equal(rebindLoopback.allowed, false);
assert.equal(rebindLoopback.code, "SSRF_BLOCKED_HOST");
const rebindMeta = await validateOutboundUrlAsync("https://rebind-meta.example.com/hook", {
fieldName: "Webhook URL",
dnsLookup: async () => [{ address: "169.254.169.254", family: 4 }],
});
assert.equal(rebindMeta.allowed, false);
assert.equal(rebindMeta.code, "SSRF_BLOCKED_HOST");
// Explicit opt-in allows RFC1918/loopback/single-label internal LAN hosts, YET ALWAYS blocks cloud metadata 169.254.169.254 & metadata.google.internal
assert.equal(checkIpAddressAllowed("127.0.0.1", true).allowed, true);
assert.equal(checkIpAddressAllowed("192.168.1.25", true).allowed, true);
assert.equal(checkIpAddressAllowed("169.254.169.254", true).allowed, false);
assert.equal(checkIpAddressAllowed("fe80::1", true).allowed, false);
assert.equal(
validateOutboundUrlSync("http://minio:9000", {
allowPrivate: true,
}).allowed,
true,
);
assert.equal(
validateOutboundUrlSync("http://169.254.169.254/latest/meta-data/", {
allowPrivate: true,
}).allowed,
false,
);
assert.equal(
validateOutboundUrlSync("http://metadata.google.internal/", {
allowPrivate: true,
}).allowed,
false,
);
});
test("R4-Phase-01 [AUDIT-01 / ERR-R4-01]: API routes (/api/settings, /api/settings/webhook-test, /api/storage/s3-test), Port 4000 triggerWebhookAlert & redirect:manual block SSRF", async () => {
delete process.env.DATABASE_URL;
delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
clearWebhookCooldownCacheForTests();
ingestionServer.clearWebhookCooldownCacheForTests();
const cookie = await createOwnerCookie("r4-ssrf-owner@datastack.dev");
// 1. POST /api/settings rejects private webhookUrl & private s3Endpoint with 400 SSRF_BLOCKED_HOST
const badWebhookSave = await settingsPost(
new NextRequest("http://localhost:3000/api/settings", {
method: "POST",
headers: { "content-type": "application/json", cookie },
body: JSON.stringify({ webhookUrl: "http://127.0.0.1:4000/internal/client-sync" }),
}),
);
assert.equal(badWebhookSave.status, 400);
assert.equal((await badWebhookSave.json()).code, "SSRF_BLOCKED_HOST");
const badS3Save = await settingsPost(
new NextRequest("http://localhost:3000/api/settings", {
method: "POST",
headers: { "content-type": "application/json", cookie },
body: JSON.stringify({ s3Endpoint: "http://169.254.169.254/latest/meta-data" }),
}),
);
assert.equal(badS3Save.status, 400);
assert.equal((await badS3Save.json()).code, "SSRF_BLOCKED_HOST");
// 2. POST /api/settings/webhook-test rejects loopback/metadata with 400 SSRF_BLOCKED_HOST
const badWebhookTest = await webhookTestPost(
new NextRequest("http://localhost:3000/api/settings/webhook-test", {
method: "POST",
headers: { "content-type": "application/json", cookie },
body: JSON.stringify({ webhookUrl: "http://localhost:4000/health" }),
}),
);
assert.equal(badWebhookTest.status, 400);
assert.equal((await badWebhookTest.json()).code, "SSRF_BLOCKED_HOST");
// 3. POST /api/storage/s3-test rejects internal/RFC1918 host with 400 SSRF_BLOCKED_HOST
const badS3Test = await s3TestPost(
new NextRequest("http://localhost:3000/api/storage/s3-test", {
method: "POST",
headers: { "content-type": "application/json", cookie },
body: JSON.stringify({
s3Endpoint: "http://10.0.0.12:9000",
s3Bucket: "test-bucket",
s3AccessKeyId: "AKIA_TEST",
s3SecretAccessKey: "secret_key_test",
}),
}),
);
assert.equal(badS3Test.status, 400);
assert.equal((await badS3Test.json()).code, "SSRF_BLOCKED_HOST");
// 4. Port 4000 triggerWebhookAlert & validateOutboundWebhookUrlAsync block loopback & metadata
const p4Blocked = await ingestionServer.triggerWebhookAlert(
{
event: "CLIENT_IP_FORBIDDEN",
severity: "warning",
title: "Probe",
message: "Test",
},
{
enableWebhookAlerts: true,
webhookUrl: "http://127.0.0.1:4000/internal/client-sync",
bypassCooldown: true,
},
);
assert.equal(p4Blocked.dispatched, false);
assert.equal(p4Blocked.code, "SSRF_BLOCKED_HOST");
// 5. redirect: 'manual' blocks HTTP 302 redirects across webhookNotifier, S3, and Port 4000
let targetHitAfterRedirect = false;
const redirectServer = http.createServer((req, res) => {
if (req.url && req.url.startsWith("/bounce")) {
res.writeHead(302, { Location: "http://169.254.169.254/latest/meta-data/" });
res.end();
return;
}
targetHitAfterRedirect = true;
res.writeHead(200);
res.end("ok");
});
await new Promise<void>((resolve) => redirectServer.listen(0, "127.0.0.1", resolve));
const addr = redirectServer.address() as AddressInfo;
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "datastack-r4-ssrf-redir-"));
const gzPath = path.join(tmpDir, "2026-11-20.csv.gz");
await fs.writeFile(gzPath, zlib.gzipSync(Buffer.from("clientId,eventId,timestamp,payload\n")));
try {
process.env.DATASTACK_ALLOW_PRIVATE_URLS = "true";
const whRedirect = await dispatchWebhookAlertAsync(
{
event: "TEST_ALERT",
severity: "info",
title: "Redirect check",
message: "Must not follow 302",
},
{
enabled: true,
webhookUrl: `http://127.0.0.1:${addr.port}/bounce`,
webhookSecret: "",
cooldownSec: 0,
},
{ bypassCooldown: true },
);
assert.equal(whRedirect.ok, false);
assert.equal(whRedirect.code, "SSRF_BLOCKED_HOST");
const s3ProbeRedir = await testS3Connection({
endpoint: `http://127.0.0.1:${addr.port}/bounce`,
region: "auto",
bucket: "bkt",
prefix: "",
accessKeyId: "AKIA",
secretAccessKey: "SEC",
});
assert.equal(s3ProbeRedir.ok, false);
assert.equal(s3ProbeRedir.code, "SSRF_BLOCKED_HOST");
const s3PutRedir = await uploadFileToS3(
{
endpoint: `http://127.0.0.1:${addr.port}/bounce`,
region: "auto",
bucket: "bkt",
prefix: "",
accessKeyId: "AKIA",
secretAccessKey: "SEC",
},
gzPath,
"c/2026-11-20.csv.gz",
);
assert.equal(s3PutRedir.ok, false);
assert.equal(s3PutRedir.code, "SSRF_BLOCKED_HOST");
const p4Redirect = await ingestionServer.triggerWebhookAlert(
{
event: "DISK_SPACE_LOW",
severity: "warning",
title: "Redirect check",
message: "Must not follow 302",
},
{
enableWebhookAlerts: true,
webhookUrl: `http://127.0.0.1:${addr.port}/bounce`,
allowPrivate: true,
bypassCooldown: true,
},
);
assert.equal(p4Redirect.dispatched, false);
assert.equal(p4Redirect.code, "SSRF_BLOCKED_HOST");
assert.equal(targetHitAfterRedirect, false);
} finally {
delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
await new Promise<void>((resolve) => redirectServer.close(() => resolve()));
await fs.rm(tmpDir, { recursive: true, force: true });
}
});
test("R4-Phase-01 [AUDIT-03 / ERR-R4-02]: Port 4000 ensureDatabaseSchema creates all 7 tables including admin_team_members and admin_audit_logs", async () => {
const executedSql: string[] = [];
const mockPool = {
async query(sql: string) {
executedSql.push(sql);
if (sql.includes("FROM admin_settings")) {
return { rowCount: 0, rows: [] };
}
return { rowCount: 0, rows: [] };
},
};
try {
ingestionServer.setDbPoolForTests(mockPool);
const synced = await ingestionServer.syncSettingsFromDatabase();
assert.equal(synced, true);
const createStatements = executedSql.filter((s) => /CREATE TABLE IF NOT EXISTS/i.test(s));
assert.equal(createStatements.length, 7, "Port 4000 must create all 7 tables");
const joined = createStatements.join("\n");
for (const tableName of [
"client_sdks",
"admin_settings",
"deleted_clients",
"admin_users",
"admin_login_attempts",
"admin_team_members",
"admin_audit_logs",
]) {
assert.ok(
joined.includes(tableName),
`Port 4000 schema missing table ${tableName}`,
);
}
} finally {
ingestionServer.setDbPoolForTests(null);
}
});
test("R4-Phase-01 [AUDIT-04 / ERR-R4-03]: Next.js instrumentation.register() boots maintenance scheduler unattended on nodejs runtime and survives unreachable DB", async () => {
delete process.env.DATABASE_URL;
const prevDisable = process.env.DATASTACK_DISABLE_LISTEN;
const prevRuntime = process.env.NEXT_RUNTIME;
SettingsManager.stopMaintenanceSchedulerForTests();
try {
// 1. Edge runtime does NOT start Node maintenance scheduler
process.env.NEXT_RUNTIME = "edge";
delete process.env.DATASTACK_DISABLE_LISTEN;
await instrumentationHook.register();
assert.equal(SettingsManager.isMaintenanceSchedulerActive(), false);
// 2. Node.js runtime starts maintenance scheduler automatically on boot
process.env.NEXT_RUNTIME = "nodejs";
await instrumentationHook.register();
assert.equal(SettingsManager.isMaintenanceSchedulerActive(), true);
// 3. Even if DB fails during boot hydration, bootstrapMaintenanceSchedulerOnBoot catches error cleanly without crashing
SettingsManager.stopMaintenanceSchedulerForTests();
process.env.DATABASE_URL = "postgresql://user:pass@127.0.0.1:1/unreachable";
const bootRes = await SettingsManager.bootstrapMaintenanceSchedulerOnBoot();
assert.equal(bootRes.schedulerActive, true);
assert.equal(bootRes.hydrated, false);
} finally {
SettingsManager.stopMaintenanceSchedulerForTests();
delete process.env.DATABASE_URL;
if (prevDisable === undefined) delete process.env.DATASTACK_DISABLE_LISTEN;
else process.env.DATASTACK_DISABLE_LISTEN = prevDisable;
if (prevRuntime === undefined) delete process.env.NEXT_RUNTIME;
else process.env.NEXT_RUNTIME = prevRuntime;
}
});
test("R4-Phase-01 [AUDIT-09 / ERR-R4-04 & ERR-R4-05]: Fail-loud audit DB degraded warnings + no-DB Files Manager session revocation sync", async () => {
delete process.env.DATABASE_URL;
clearAuditLogsForTests();
filesManager.clearInMemoryAdminSessionStateForTests();
const cookie = await createOwnerCookie("r4-audit-owner@datastack.dev");
const globals = globalThis as typeof globalThis & {
__dataStackCorePool?: unknown;
};
const prevPool = globals.__dataStackCorePool;
// 1. Simulate PostgreSQL failure on audit write + read -> must log console.warn and expose degraded metadata & header
const failingPool = {
async query(sql: string) {
if (/CREATE TABLE|ALTER TABLE/i.test(sql)) {
return { rowCount: 0, rows: [] };
}
if (sql.includes("FROM admin_users")) {
return {
rowCount: 1,
rows: [
{
email: "r4-audit-owner@datastack.dev",
created_at: new Date(),
token_version: 1,
},
],
};
}
if (sql.includes("admin_audit_logs")) {
throw new Error("simulated PG outage");
}
return { rowCount: 0, rows: [] };
},
};
try {
process.env.DATABASE_URL = "postgresql://mock:mock@127.0.0.1:5432/mock";
globals.__dataStackCorePool = failingPool;
const warnings: string[] = [];
const origWarn = console.warn;
console.warn = (...args: unknown[]) => {
warnings.push(args.map((a) => String(a)).join(" "));
};
try {
await recordAdminAuditLogAsync({
actorEmail: "r4-audit-owner@datastack.dev",
actorRole: "owner",
action: "SETTINGS_UPDATED",
target: "admin_settings",
});
const listResult = await listAdminAuditLogsWithStatusAsync({ limit: 10 });
assert.equal(listResult.storageMode, "memory");
assert.equal(listResult.degraded, true);
assert.equal(listResult.code, "AUDIT_STORAGE_DEGRADED");
assert.match(listResult.warning || "", /PostgreSQL/i);
assert.ok(
warnings.some((w) => w.includes("[DataStack Audit]")),
"Expected console.warn fail-loud audit log",
);
// Verify GET /api/audit surfaces degraded status & header
const apiRes = await auditGet(
new NextRequest("http://localhost:3000/api/audit?limit=10", {
headers: { cookie },
}),
);
assert.equal(apiRes.status, 200);
assert.equal(apiRes.headers.get("X-DataStack-Audit-Degraded"), "1");
const apiJson = await apiRes.json();
assert.equal(apiJson.degraded, true);
assert.equal(apiJson.code, "AUDIT_STORAGE_DEGRADED");
assert.equal(apiJson.storageMode, "memory");
} finally {
console.warn = origWarn;
}
} finally {
delete process.env.DATABASE_URL;
globals.__dataStackCorePool = prevPool;
}
// 2. Verify no-DB (!dbPool) Files Manager session revocation via Port 4000 /internal/admin-session-sync bridge
const testBridgeServer = http.createServer(ingestionServer.app);
await new Promise<void>((resolve) => testBridgeServer.listen(0, "127.0.0.1", resolve));
const bridgePort = (testBridgeServer.address() as AddressInfo).port;
process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${bridgePort}`;
try {
// Reject unauthenticated POST /internal/admin-session-sync
const unauthSync = await fetch(
`http://127.0.0.1:${bridgePort}/internal/admin-session-sync`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ memberId: "owner-1", minTokenVersion: 99 }),
},
);
assert.equal(unauthSync.status, 401);
// Owner session v1 works before logout/revocation
const secret = getAdminSessionSecret();
const ownerTokenV1 = await createAdminSessionToken(
"r4-audit-owner@datastack.dev",
secret,
Date.now() + 3600_000,
1,
"owner",
"owner-1",
);
const beforeLogout = await filesManager.authenticateFilesRequest({
cookieHeader: `datastack_admin_session=${ownerTokenV1}`,
dbPool: null,
});
assert.equal(beforeLogout.valid, true);
// Calling revokeAdminSessions syncs minTokenVersion=2 to Port 4000 bridge -> v1 Files Manager cookie is rejected
const nextVer = await revokeAdminSessions({
memberId: "owner-1",
email: "r4-audit-owner@datastack.dev",
});
assert.ok(nextVer >= 2);
const afterLogout = await filesManager.authenticateFilesRequest({
cookieHeader: `datastack_admin_session=${ownerTokenV1}`,
dbPool: null,
});
assert.equal(afterLogout.valid, false);
assert.equal(afterLogout.status, 401);
assert.equal(afterLogout.code, "SESSION_REVOKED");
// Team member disable/delete also revokes no-DB Files Manager session immediately
const opMember = await createAdminTeamMemberAsync({
email: "r4-op@datastack.dev",
password: "OperatorPassword123!",
role: "operator",
});
const opTokenV1 = await createAdminSessionToken(
opMember.email,
secret,
Date.now() + 3600_000,
opMember.tokenVersion,
opMember.role,
opMember.id,
);
const opBefore = await filesManager.authenticateFilesRequest({
cookieHeader: `datastack_admin_session=${opTokenV1}`,
dbPool: null,
});
assert.equal(opBefore.valid, true);
await updateAdminTeamMemberAsync(opMember.id, { status: "disabled" });
const opAfterDisable = await filesManager.authenticateFilesRequest({
cookieHeader: `datastack_admin_session=${opTokenV1}`,
dbPool: null,
});
assert.equal(opAfterDisable.valid, false);
assert.equal(opAfterDisable.code, "SESSION_REVOKED");
await deleteAdminTeamMemberAsync(opMember.id);
// Also verify authenticated /internal/admin-session-sync endpoint returns 200 & synced=true
const authSyncRes = await fetch(
`http://127.0.0.1:${bridgePort}/internal/admin-session-sync`,
{
method: "POST",
headers: {
"Content-Type": "application/json",
...getInternalIngestionHeaders(),
},
body: JSON.stringify({
memberId: "adm_manual_test",
email: "manual@datastack.dev",
minTokenVersion: 5,
disabled: true,
}),
},
);
assert.equal(authSyncRes.status, 200);
const authSyncJson = (await authSyncRes.json()) as { success: boolean; synced: boolean };
assert.equal(authSyncJson.success, true);
assert.equal(authSyncJson.synced, true);
} finally {
delete process.env.INGESTION_INTERNAL_URL;
filesManager.clearInMemoryAdminSessionStateForTests();
await new Promise<void>((resolve) => testBridgeServer.close(() => resolve()));
}
});
@@ -0,0 +1,777 @@
import assert from "node:assert/strict";
import crypto from "node:crypto";
import fs from "node:fs";
import fsp from "node:fs/promises";
import http from "node:http";
import { createRequire } from "node:module";
import type { AddressInfo } from "node:net";
import os from "node:os";
import path from "node:path";
import test from "node:test";
import zlib from "node:zlib";
import { NextRequest } from "next/server";
process.env.DATASTACK_DISABLE_LISTEN = "1";
import { GET as auditGet } from "../app/api/audit/route";
import { PATCH as clientPatch } from "../app/api/clients/[id]/route";
import { GET as dataGet } from "../app/api/data/route";
import { POST as eventsPost } from "../app/api/events/route";
import { GET as exportGet } from "../app/api/export/route";
import { SettingsManager } from "../core/adminSettings/settingsManager";
import {
createAdminSessionToken,
getAdminSessionSecret,
} from "../core/auth/adminSession";
import {
createInitialAdmin,
resetAdminStateForTests,
} from "../core/auth/adminUsers";
import {
clearAuditLogsForTests,
listAdminAuditLogsWithStatusAsync,
} from "../core/auth/auditLog";
import { DataBufferEngine } from "../core/data/buffer";
import { SDKManager } from "../core/sdk/sdkManager";
import { getInternalIngestionHeaders } from "../core/Security/security";
import { CSVStorageManager } from "../core/Storage/csvStorage";
import { uploadFileToS3 } from "../core/Storage/s3Archive";
const require = createRequire(import.meta.url);
const express = require("express");
const { Server } = require("socket.io");
const { io: ioClient } = require("socket.io-client");
const { setupSocketHandlers } = require("../server/socket/handlers");
const filesManager = require("../server/filesManager.js") as {
createFilesSecurityMiddleware: (opts: {
rootDir: string;
getDbPool?: () => unknown;
env?: NodeJS.ProcessEnv;
}) => (req: http.IncomingMessage, res: http.ServerResponse, next: () => void) => void;
flushPendingFilesAuditLogsForTests: () => Promise<void>;
clearInMemoryAdminSessionStateForTests: () => void;
};
const ingestionServer = require("../server/index.js") as {
app: (req: http.IncomingMessage, res: http.ServerResponse) => void;
io: {
on: (ev: string, cb: (socket: unknown) => void) => void;
close: (cb?: () => void) => void;
};
server: http.Server;
stopTimers: () => void;
setDbPoolForTests: (pool: unknown) => void;
writer: { baseDir: string };
};
const ORIGINAL_ENV = { ...process.env };
const ORIGINAL_STORAGE_DIR = CSVStorageManager.baseStorageDir;
function restoreEnv(): void {
for (const key of Object.keys(process.env)) {
if (!(key in ORIGINAL_ENV)) {
delete process.env[key];
}
}
Object.assign(process.env, ORIGINAL_ENV);
}
async function createOwnerCookie(
email = "r4p2-owner@datastack.dev",
): Promise<string> {
resetAdminStateForTests();
await createInitialAdmin(email, "OwnerPassword123!");
const secret = getAdminSessionSecret();
const token = await createAdminSessionToken(
email,
secret,
Date.now() + 3600_000,
1,
"owner",
"owner-1",
);
return `datastack_admin_session=${token}`;
}
test.after(async () => {
CSVStorageManager.baseStorageDir = ORIGINAL_STORAGE_DIR;
CSVStorageManager.clearRowCountCache();
CSVStorageManager.clearQuotaCaches();
SettingsManager.stopMaintenanceSchedulerForTests();
SDKManager.stopActivityFlushTimerForTests();
ingestionServer.stopTimers();
ingestionServer.setDbPoolForTests(null);
filesManager.clearInMemoryAdminSessionStateForTests();
await DataBufferEngine.shutdown();
clearAuditLogsForTests();
resetAdminStateForTests();
restoreEnv();
});
test("R4-Phase-02 AUDIT-05 / ERR-R4-06: HTTP & Socket.IO enforce DATASTACK_MAX_BATCH_ITEMS cap", async () => {
const prevStorage = CSVStorageManager.baseStorageDir;
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "ds-r4p2-batch-"));
CSVStorageManager.baseStorageDir = tmpDir;
CSVStorageManager.clearRowCountCache();
CSVStorageManager.clearQuotaCaches();
process.env.DATASTACK_MAX_BATCH_ITEMS = "3";
let httpServer: http.Server | null = null;
let io: InstanceType<typeof Server> | null = null;
let socket: ReturnType<typeof ioClient> | null = null;
const runId = `${Date.now()}_${crypto.randomBytes(3).toString("hex")}`;
try {
const client = await SDKManager.registerClientAsync({
clientId: "r4p2-batch-client",
name: "Batch Limit Client",
rateLimit: 500,
});
// 1. HTTP POST /api/events rejects 4-item array before per-item checks
const overBatchReq = new NextRequest("http://127.0.0.1:3000/api/events", {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${client.token}`,
},
body: JSON.stringify([
{ eventId: `${runId}-b1`, data: { step: 1 } },
{ eventId: `${runId}-b2`, data: { step: 2 } },
{ eventId: `${runId}-b3`, data: { step: 3 } },
{ eventId: `${runId}-b4`, data: { step: 4 } },
]),
});
const overRes = await eventsPost(overBatchReq);
assert.equal(overRes.status, 400);
const overJson = await overRes.json();
assert.equal(overJson.success, false);
assert.equal(overJson.code, "BATCH_LIMIT_EXCEEDED");
assert.equal(overJson.rejected, 4);
assert.equal(overJson.maxBatchItems, 3);
// 2. HTTP POST /api/events accepts 3-item records array
const okBatchReq = new NextRequest("http://127.0.0.1:3000/api/events", {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${client.token}`,
},
body: JSON.stringify({
records: [
{ eventId: `${runId}-b1`, data: { step: 1 } },
{ eventId: `${runId}-b2`, data: { step: 2 } },
{ eventId: `${runId}-b3`, data: { step: 3 } },
],
}),
});
const okRes = await eventsPost(okBatchReq);
assert.equal(okRes.status, 200);
const okJson = await okRes.json();
assert.equal(okJson.success, true);
assert.equal(okJson.accepted, 3);
assert.equal(okJson.rejected, 0);
// 3. Socket.IO batch_sync rejects 4-item array and accepts 3-item array
httpServer = http.createServer();
io = new Server(httpServer);
const bufferQueue: unknown[] = [];
setupSocketHandlers(io, bufferQueue, null, {
getRuntimeConfig: () => ({
jwtSecret: "datastack-jwt-default-secret-key-32-chars-long",
maxEventsPerSec: 500,
maxPayloadSizeBytes: 100 * 1024,
enableIdempotency: true,
}),
verifyClientInDb: async () => ({ allowed: true, rateLimit: 500 }),
});
await new Promise<void>((resolve) => httpServer!.listen(0, "127.0.0.1", resolve));
const port = (httpServer.address() as AddressInfo).port;
socket = ioClient(`http://127.0.0.1:${port}`, {
auth: { token: client.token },
transports: ["websocket"],
});
await new Promise<void>((resolve, reject) => {
socket!.on("connect", () => resolve());
socket!.on("connect_error", reject);
});
const overAck = await new Promise<Record<string, unknown>>((resolve) => {
socket!.emit(
"batch_sync",
[
{ eventId: "s1", data: { i: 1 } },
{ eventId: "s2", data: { i: 2 } },
{ eventId: "s3", data: { i: 3 } },
{ eventId: "s4", data: { i: 4 } },
],
resolve,
);
});
assert.equal(overAck.success, false);
assert.equal(overAck.code, "BATCH_LIMIT_EXCEEDED");
assert.equal(overAck.rejected, 4);
assert.equal(overAck.maxBatchItems, 3);
assert.equal(bufferQueue.length, 0);
const okAck = await new Promise<Record<string, unknown>>((resolve) => {
socket!.emit(
"batch_sync",
[
{ eventId: `${runId}-s1`, data: { i: 1 } },
{ eventId: `${runId}-s2`, data: { i: 2 } },
{ eventId: `${runId}-s3`, data: { i: 3 } },
],
resolve,
);
});
assert.equal(okAck.success, true);
assert.equal(okAck.processed, 3);
assert.equal(okAck.rejected, 0);
assert.equal(bufferQueue.length, 3);
// 4. Lower bound clamping to 1 when DATASTACK_MAX_BATCH_ITEMS=0
process.env.DATASTACK_MAX_BATCH_ITEMS = "0";
const clampAck = await new Promise<Record<string, unknown>>((resolve) => {
socket!.emit(
"batch_sync",
[
{ eventId: `${runId}-c1`, data: { i: 1 } },
{ eventId: `${runId}-c2`, data: { i: 2 } },
],
resolve,
);
});
assert.equal(clampAck.success, false);
assert.equal(clampAck.code, "BATCH_LIMIT_EXCEEDED");
assert.equal(clampAck.maxBatchItems, 1);
} finally {
delete process.env.DATASTACK_MAX_BATCH_ITEMS;
if (socket) socket.disconnect();
if (io) await new Promise<void>((resolve) => io!.close(() => resolve()));
if (httpServer) httpServer.close();
await DataBufferEngine.shutdown();
await SDKManager.deleteClientWithStatsAsync("r4p2-batch-client", {
purgeFiles: false,
}).catch(() => undefined);
CSVStorageManager.baseStorageDir = prevStorage;
CSVStorageManager.clearRowCountCache();
CSVStorageManager.clearQuotaCaches();
await fsp.rm(tmpDir, { recursive: true, force: true });
}
});
test("R4-Phase-02 AUDIT-06: Governance audit logging for /api/export, Files Manager HTTP mutations (RAM + PG), and UI filters", async () => {
const prevStorage = CSVStorageManager.baseStorageDir;
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "ds-r4p2-audit-"));
CSVStorageManager.baseStorageDir = tmpDir;
CSVStorageManager.clearRowCountCache();
CSVStorageManager.clearQuotaCaches();
clearAuditLogsForTests();
let mockServer: http.Server | null = null;
let pgServer: http.Server | null = null;
try {
const cookie = await createOwnerCookie("audit-owner@datastack.dev");
const partitionDir = path.join(tmpDir, "export-client");
await fsp.mkdir(partitionDir, { recursive: true });
await fsp.writeFile(
path.join(partitionDir, "2026-11-20.csv"),
'clientId,eventId,timestamp,payload\n"export-client","evt-1","2026-11-20T10:00:00.000Z","{""ok"":true}"\n',
"utf8",
);
// 1. Filtered-range export audit log
const rangeRes = await exportGet(
new NextRequest(
"http://127.0.0.1:3000/api/export?clientId=export-client&format=csv&flush=false",
{ headers: { Cookie: cookie } },
),
);
assert.equal(rangeRes.status, 200);
await rangeRes.text();
// 2. Single-file export audit log
const singleRes = await exportGet(
new NextRequest(
"http://127.0.0.1:3000/api/export?clientId=export-client&filename=2026-11-20.csv&format=json&flush=false",
{ headers: { Cookie: cookie } },
),
);
assert.equal(singleRes.status, 200);
await singleRes.text();
// Verify /api/audit filters DATA_EXPORT_DOWNLOADED
const auditRes = await auditGet(
new NextRequest(
"http://127.0.0.1:3000/api/audit?action=DATA_EXPORT_DOWNLOADED&limit=10",
{ headers: { Cookie: cookie } },
),
);
assert.equal(auditRes.status, 200);
const auditJson = await auditRes.json();
assert.equal(auditJson.logs.length, 2);
assert.equal(auditJson.logs[0].action, "DATA_EXPORT_DOWNLOADED");
assert.equal(auditJson.logs[0].metadata.mode, "single_file");
assert.equal(auditJson.logs[1].metadata.mode, "filtered_range");
// 3. Files Manager HTTP mutations -> RAM fallback
clearAuditLogsForTests();
const ramApp = express();
ramApp.use(
"/vcon/files-engine",
filesManager.createFilesSecurityMiddleware({
rootDir: tmpDir,
getDbPool: () => null,
}),
);
ramApp.put("/vcon/files-engine/api/v1/fs/sample.csv", (_req: express.Request, res: express.Response) => {
res.status(200).json({ ok: true });
});
ramApp.put("/vcon/files-engine/api/v1/rename", (_req: express.Request, res: express.Response) => {
res.status(200).json({ ok: true });
});
ramApp.delete("/vcon/files-engine/api/v1/fs/sample.csv", (_req: express.Request, res: express.Response) => {
res.status(403).json({ ok: false }); // non-2xx should NOT emit mutation audit
});
mockServer = http.createServer(ramApp);
await new Promise<void>((resolve) => mockServer!.listen(0, "127.0.0.1", resolve));
const ramPort = (mockServer.address() as AddressInfo).port;
const originHeaders = {
Cookie: cookie,
Origin: `http://127.0.0.1:${ramPort}`,
Host: `127.0.0.1:${ramPort}`,
"Content-Type": "application/json",
};
const putRes = await fetch(
`http://127.0.0.1:${ramPort}/vcon/files-engine/api/v1/fs/sample.csv`,
{ method: "PUT", headers: originHeaders, body: "hello" },
);
assert.equal(putRes.status, 200);
const renameRes = await fetch(
`http://127.0.0.1:${ramPort}/vcon/files-engine/api/v1/rename`,
{
method: "PUT",
headers: originHeaders,
body: JSON.stringify({ from: "/sample.csv", to: "/renamed.csv" }),
},
);
assert.equal(renameRes.status, 200);
const failDelRes = await fetch(
`http://127.0.0.1:${ramPort}/vcon/files-engine/api/v1/fs/sample.csv`,
{ method: "DELETE", headers: originHeaders },
);
assert.equal(failDelRes.status, 403);
await filesManager.flushPendingFilesAuditLogsForTests();
const { logs: fmRamLogs } = await listAdminAuditLogsWithStatusAsync({
action: "FILES_MANAGER_MUTATION",
});
assert.equal(fmRamLogs.length, 2);
assert.equal(fmRamLogs[0].target, "/renamed.csv");
assert.equal(fmRamLogs[0].metadata.from, "/sample.csv");
assert.equal(fmRamLogs[1].target, "/sample.csv");
// 4. Files Manager HTTP mutation -> mock PostgreSQL pool
const pgInserts: Array<{ sql: string; params: unknown[] }> = [];
const mockDbPool = {
query: async (sql: string, params: unknown[] = []) => {
if (/FROM\s+admin_users\b/i.test(sql)) {
return {
rows: [{ email: "audit-owner@datastack.dev", token_version: 1 }],
};
}
if (/FROM\s+admin_team_members\b/i.test(sql)) {
return { rows: [] };
}
pgInserts.push({ sql, params });
return { rows: [] };
},
};
const pgApp = express();
pgApp.use(
"/vcon/files-engine",
filesManager.createFilesSecurityMiddleware({
rootDir: tmpDir,
getDbPool: () => mockDbPool,
}),
);
pgApp.delete("/vcon/files-engine/api/v1/fs/obsolete.csv", (_req: express.Request, res: express.Response) => {
res.status(200).json({ ok: true });
});
pgServer = http.createServer(pgApp);
await new Promise<void>((resolve) => pgServer!.listen(0, "127.0.0.1", resolve));
const pgPort = (pgServer.address() as AddressInfo).port;
const pgDel = await fetch(
`http://127.0.0.1:${pgPort}/vcon/files-engine/api/v1/fs/obsolete.csv`,
{
method: "DELETE",
headers: {
Cookie: cookie,
Origin: `http://127.0.0.1:${pgPort}`,
Host: `127.0.0.1:${pgPort}`,
},
},
);
assert.equal(pgDel.status, 200);
await filesManager.flushPendingFilesAuditLogsForTests();
assert.equal(pgInserts.length, 1);
assert.match(pgInserts[0].sql, /INSERT INTO admin_audit_logs/i);
assert.equal(pgInserts[0].params[3], "FILES_MANAGER_MUTATION");
assert.equal(pgInserts[0].params[4], "/obsolete.csv");
// 5. AuditLogViewer filter options expose both new action types
const viewerSrc = await fsp.readFile(
path.join(process.cwd(), "components/sections/AuditLogViewer.tsx"),
"utf8",
);
assert.ok(viewerSrc.includes("FILES_MANAGER_MUTATION"));
assert.ok(viewerSrc.includes("DATA_EXPORT_DOWNLOADED"));
} finally {
if (mockServer) mockServer.close();
if (pgServer) pgServer.close();
clearAuditLogsForTests();
CSVStorageManager.baseStorageDir = prevStorage;
CSVStorageManager.clearRowCountCache();
CSVStorageManager.clearQuotaCaches();
await fsp.rm(tmpDir, { recursive: true, force: true });
}
});
test("R4-Phase-02 AUDIT-07 / ERR-R4-07: GET /api/data isolates corrupt .csv.gz partitions and serves healthy data", async () => {
const prevStorage = CSVStorageManager.baseStorageDir;
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "ds-r4p2-corrupt-"));
CSVStorageManager.baseStorageDir = tmpDir;
CSVStorageManager.clearRowCountCache();
CSVStorageManager.clearQuotaCaches();
try {
const cookie = await createOwnerCookie("data-owner@datastack.dev");
const healthyDir = path.join(tmpDir, "healthy-app");
const corruptDir = path.join(tmpDir, "corrupt-app");
await fsp.mkdir(healthyDir, { recursive: true });
await fsp.mkdir(corruptDir, { recursive: true });
await fsp.writeFile(
path.join(healthyDir, "2026-11-20.csv"),
'clientId,eventId,timestamp,payload\n"healthy-app","ok-1","2026-11-20T12:00:00.000Z","{""status"":""alive""}"\n',
"utf8",
);
await fsp.writeFile(
path.join(corruptDir, "2026-11-20.csv.gz"),
Buffer.from("corrupted-not-gzip-bytes-123456"),
);
const res = await dataGet(
new NextRequest(
"http://127.0.0.1:3000/api/data?from=2026-11-20&to=2026-11-20&page=1&pageSize=25",
{
headers: { Cookie: cookie },
},
),
);
assert.equal(res.status, 200);
const json = await res.json();
assert.equal(json.total, 1);
assert.equal(json.events.length, 1);
assert.equal(json.events[0].eventId, "ok-1");
assert.equal(json.warnings, 1);
assert.ok(Array.isArray(json.corruptPartitions));
assert.equal(json.corruptPartitions.length, 1);
assert.equal(json.corruptPartitions[0].clientId, "corrupt-app");
assert.equal(json.corruptPartitions[0].filename, "2026-11-20.csv.gz");
assert.equal(json.corruptPartitions[0].code, "CORRUPT_GZIP_PARTITION");
} finally {
CSVStorageManager.baseStorageDir = prevStorage;
CSVStorageManager.clearRowCountCache();
CSVStorageManager.clearQuotaCaches();
await fsp.rm(tmpDir, { recursive: true, force: true });
}
});
test("R4-Phase-02 AUDIT-10 & AUDIT-11: PATCH redacts secret/token and /internal/client-sync disconnects selectively", async () => {
const cookie = await createOwnerCookie("clients-owner@datastack.dev");
const client = await SDKManager.registerClientAsync({
clientId: "r4p2-selective-client",
name: "Initial Client Name",
rateLimit: 100,
storageQuotaMb: 50,
allowedIps: "127.0.0.1",
});
// 1. AUDIT-10: PATCH metadata update must omit secret & full token
const metaPatchRes = await clientPatch(
new NextRequest(`http://127.0.0.1:3000/api/clients/${client.clientId}`, {
method: "PATCH",
headers: { Cookie: cookie, "Content-Type": "application/json" },
body: JSON.stringify({ name: "Renamed Client", rateLimit: 250 }),
}),
{ params: Promise.resolve({ id: client.clientId }) },
);
assert.equal(metaPatchRes.status, 200);
const metaPatchJson = await metaPatchRes.json();
assert.equal(metaPatchJson.client.name, "Renamed Client");
assert.equal(metaPatchJson.client.rateLimit, 250);
assert.equal("secret" in metaPatchJson.client, false);
assert.equal("token" in metaPatchJson.client, false);
assert.equal(typeof metaPatchJson.client.tokenPreview, "string");
assert.ok(metaPatchJson.client.tokenPreview.endsWith("..."));
// 2. AUDIT-10: regenerate_token includes rotated token while still redacting secret
const regenRes = await clientPatch(
new NextRequest(`http://127.0.0.1:3000/api/clients/${client.clientId}`, {
method: "PATCH",
headers: { Cookie: cookie, "Content-Type": "application/json" },
body: JSON.stringify({ action: "regenerate_token" }),
}),
{ params: Promise.resolve({ id: client.clientId }) },
);
assert.equal(regenRes.status, 200);
const regenJson = await regenRes.json();
assert.equal("secret" in regenJson.client, false);
assert.equal(typeof regenJson.client.token, "string");
assert.notEqual(regenJson.client.token, client.token);
// 3. AUDIT-11: Live Socket.IO client stays connected on metadata update, disconnects on security update
await new Promise<void>((resolve) =>
ingestionServer.server.listen(0, "127.0.0.1", resolve),
);
const port = (ingestionServer.server.address() as AddressInfo).port;
const internalHeaders = {
"Content-Type": "application/json",
...getInternalIngestionHeaders(),
};
let sock: ReturnType<typeof ioClient> | null = null;
try {
// Seed ingestion server registry via /internal/client-sync registered
const regSyncRes = await fetch(
`http://127.0.0.1:${port}/internal/client-sync`,
{
method: "POST",
headers: internalHeaders,
body: JSON.stringify({
action: "registered",
clientId: client.clientId,
client: {
...regenJson.client,
token: regenJson.client.token,
status: "active",
allowedIps: "127.0.0.1",
expiresAt: null,
},
}),
},
);
assert.equal(regSyncRes.status, 200);
sock = ioClient(`http://127.0.0.1:${port}`, {
auth: { token: regenJson.client.token },
transports: ["websocket"],
});
await new Promise<void>((resolve, reject) => {
sock!.on("connect", () => resolve());
sock!.on("connect_error", reject);
});
// Metadata-only update: socket MUST remain connected
const metadataSyncRes = await fetch(
`http://127.0.0.1:${port}/internal/client-sync`,
{
method: "POST",
headers: internalHeaders,
body: JSON.stringify({
action: "updated",
clientId: client.clientId,
client: {
...regenJson.client,
name: "Hot Metadata Update",
rateLimit: 400,
storageQuotaMb: 200,
token: regenJson.client.token,
status: "active",
allowedIps: "127.0.0.1",
expiresAt: null,
},
}),
},
);
assert.equal(metadataSyncRes.status, 200);
const metadataSyncJson = await metadataSyncRes.json();
assert.equal(metadataSyncJson.disconnected, false);
assert.equal(sock.connected, true);
// AllowedIps narrowing away from 127.0.0.1 MUST disconnect live socket
const disconnectPromise = new Promise<string>((resolve) => {
sock!.once("disconnect", resolve);
});
const ipSyncRes = await fetch(
`http://127.0.0.1:${port}/internal/client-sync`,
{
method: "POST",
headers: internalHeaders,
body: JSON.stringify({
action: "updated",
clientId: client.clientId,
client: {
...regenJson.client,
token: regenJson.client.token,
status: "active",
allowedIps: "203.0.113.50",
expiresAt: null,
},
}),
},
);
assert.equal(ipSyncRes.status, 200);
const ipSyncJson = await ipSyncRes.json();
assert.equal(ipSyncJson.disconnected, true);
await disconnectPromise;
assert.equal(sock.connected, false);
} finally {
if (sock) sock.disconnect();
await new Promise<void>((resolve) => ingestionServer.io.close(() => resolve()));
await SDKManager.deleteClientWithStatsAsync("r4p2-selective-client", {
purgeFiles: false,
}).catch(() => undefined);
}
});
test("R4-Phase-02 AUDIT-12 / ERR-R4-08: Streaming S3 upload & streaming .csv.gz row counter", async () => {
process.env.DATASTACK_ALLOW_PRIVATE_URLS = "true";
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "ds-r4p2-stream-"));
let s3Server: http.Server | null = null;
try {
// 1. Multi-chunk (>256 KB) .csv and .csv.gz partitions for row counting & S3 upload
const totalRows = 3200;
const lines = ["clientId,eventId,timestamp,payload"];
for (let i = 0; i < totalRows; i += 1) {
lines.push(
`"stream-client","evt-${i}","2026-11-20T00:00:00.000Z","{""seq"":${i},""padding"":""0123456789abcdef0123456789abcdef0123456789abcdef""}"`,
);
}
const rawCsv = `${lines.join("\n")}\n`;
assert.ok(Buffer.byteLength(rawCsv, "utf8") > 256 * 1024);
const csvPath = path.join(tmpDir, "2026-11-20.csv");
const gzPath = path.join(tmpDir, "2026-11-20.csv.gz");
await fsp.writeFile(csvPath, rawCsv, "utf8");
await fsp.writeFile(gzPath, zlib.gzipSync(Buffer.from(rawCsv, "utf8")));
assert.equal(
await CSVStorageManager.countCsvDataRowsStreamAsync(csvPath),
totalRows,
);
assert.equal(
await CSVStorageManager.countCsvDataRowsStreamAsync(gzPath),
totalRows,
);
const corruptGz = path.join(tmpDir, "corrupt.csv.gz");
await fsp.writeFile(corruptGz, Buffer.from("invalid-gzip-data"));
await assert.rejects(() =>
CSVStorageManager.countCsvDataRowsStreamAsync(corruptGz),
);
// 2. Streaming PUT via uploadFileToS3 with exact content-length & SHA-256 verification
let failNextUpload = false;
const receivedUploads: Array<{
url: string;
contentLength: string | undefined;
sha256: string | undefined;
body: Buffer;
}> = [];
s3Server = http.createServer((req, res) => {
if (failNextUpload) {
req.resume();
res.destroy(new Error("Simulated mid-upload S3 reset"));
return;
}
const chunks: Buffer[] = [];
req.on("data", (chunk: Buffer) => chunks.push(chunk));
req.on("end", () => {
receivedUploads.push({
url: req.url || "",
contentLength: req.headers["content-length"],
sha256: req.headers["x-amz-content-sha256"] as string | undefined,
body: Buffer.concat(chunks),
});
res.writeHead(200, { ETag: '"stream-etag-ok"' });
res.end();
});
});
await new Promise<void>((resolve) => s3Server!.listen(0, "127.0.0.1", resolve));
const s3Port = (s3Server.address() as AddressInfo).port;
const s3Config = {
endpoint: `http://127.0.0.1:${s3Port}`,
region: "auto",
bucket: "datastack-test-bucket",
prefix: "archive",
accessKeyId: "test-ak",
secretAccessKey: "test-sk",
};
const csvRes = await uploadFileToS3(
s3Config,
csvPath,
"archive/stream-client/2026-11-20.csv",
);
assert.equal(csvRes.ok, true);
assert.equal(csvRes.etag, '"stream-etag-ok"');
const gzRes = await uploadFileToS3(
s3Config,
gzPath,
"archive/stream-client/2026-11-20.csv.gz",
);
assert.equal(gzRes.ok, true);
assert.equal(receivedUploads.length, 2);
const expectedCsvBuf = await fsp.readFile(csvPath);
const expectedCsvHash = crypto
.createHash("sha256")
.update(expectedCsvBuf)
.digest("hex");
assert.equal(
receivedUploads[0].contentLength,
String(expectedCsvBuf.length),
);
assert.equal(receivedUploads[0].sha256, expectedCsvHash);
assert.ok(receivedUploads[0].body.equals(expectedCsvBuf));
const expectedGzBuf = await fsp.readFile(gzPath);
assert.equal(
receivedUploads[1].contentLength,
String(expectedGzBuf.length),
);
assert.ok(receivedUploads[1].body.equals(expectedGzBuf));
// 3. Failure mid-upload returns S3_ARCHIVE_FAILED cleanly
failNextUpload = true;
const failRes = await uploadFileToS3(
s3Config,
csvPath,
"archive/stream-client/fail.csv",
);
assert.equal(failRes.ok, false);
assert.equal(failRes.code, "S3_ARCHIVE_FAILED");
} finally {
delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
if (s3Server) s3Server.close();
await fsp.rm(tmpDir, { recursive: true, force: true });
assert.equal(fs.existsSync(tmpDir), false);
}
});
+313
View File
@@ -0,0 +1,313 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import { createRequire } from 'node:module';
import os from 'node:os';
import path from 'node:path';
import { afterEach, beforeEach, describe, it } from 'node:test';
import { NextRequest } from 'next/server';
import { POST as setupPost } from '../app/api/auth/setup/route';
import { GET as filesStatusGet } from '../app/api/files/status/route';
import {
createAdminSessionToken,
getAdminSessionPayload,
getAdminSessionSecret,
} from '../core/auth/adminSession';
import { resetAdminStateForTests } from '../core/auth/adminUsers';
import { middleware } from '../middleware';
import { DataStack, DiskSpooler } from '../sdk/node/datastack';
const require = createRequire(import.meta.url);
const ORIGINAL_ENV = { ...process.env };
const ORIGINAL_FETCH = globalThis.fetch;
function restoreEnv() {
for (const key of Object.keys(process.env)) {
if (!(key in ORIGINAL_ENV)) {
delete process.env[key];
}
}
Object.assign(process.env, ORIGINAL_ENV);
}
function makeFakeJwt(clientId: string): string {
const header = Buffer.from(JSON.stringify({ alg: 'HS256', typ: 'JWT' })).toString('base64url');
const payload = Buffer.from(JSON.stringify({ clientId, iat: 1700000000 })).toString('base64url');
return `${header}.${payload}.sig`;
}
describe('R4-Phase-03: Node SDK Packaging & Spool Parity, Docker & Proxy Hint, Auth Hygiene', () => {
beforeEach(() => {
restoreEnv();
resetAdminStateForTests();
});
afterEach(() => {
globalThis.fetch = ORIGINAL_FETCH;
restoreEnv();
resetAdminStateForTests();
});
it('AUDIT-02: prebuilds @datastack/sdk dist artifacts, sets prepare/typescript metadata, and loads directly via CJS require', () => {
const sdkDir = path.join(process.cwd(), 'sdk', 'node', 'datastack');
const pkg = JSON.parse(fs.readFileSync(path.join(sdkDir, 'package.json'), 'utf8')) as {
scripts?: Record<string, string>;
devDependencies?: Record<string, string>;
main?: string;
types?: string;
};
assert.equal(pkg.main, './dist/index.js');
assert.equal(pkg.types, './dist/index.d.ts');
assert.equal(pkg.scripts?.build, 'tsc -p tsconfig.json');
assert.equal(pkg.scripts?.prepare, 'npm run build');
assert.ok(pkg.devDependencies?.typescript, 'typescript must be listed in devDependencies');
for (const file of [
'index.js',
'index.d.ts',
'spool.js',
'spool.d.ts',
'types.js',
'types.d.ts',
]) {
assert.equal(
fs.existsSync(path.join(sdkDir, 'dist', file)),
true,
`Missing prebuilt dist artifact: ${file}`
);
}
const cjsLoaded = require(path.join(sdkDir, 'dist', 'index.js')) as {
DataStack?: typeof DataStack;
DataStackClient?: typeof DataStack;
DiskSpooler?: typeof DiskSpooler;
SDK_VERSION?: string;
};
assert.equal(typeof cjsLoaded.DataStack, 'function');
assert.equal(typeof cjsLoaded.DataStackClient, 'function');
assert.equal(typeof cjsLoaded.DiskSpooler, 'function');
assert.equal(cjsLoaded.SDK_VERSION, '2.2.0');
const token = makeFakeJwt('cjs_loader_probe');
const client = new cjsLoaded.DataStack!({
token,
autoConnect: false,
offlineSpoolEnabled: false,
});
assert.equal(client.clientId, 'cjs_loader_probe');
});
it('AUDIT-20: DiskSpooler.ackAndRequeueFront preserves FIFO tail and client.flushSpool recovers partial 429 progress atomically', async () => {
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ds-r4-p03-spool-'));
try {
// 1. Direct DiskSpooler unit check
const spooler = new DiskSpooler({ spoolDir: tmpDir, enabled: true });
const mkEnv = (id: string) => ({
clientId: 'sensor_node',
eventId: id,
timestamp: '2026-11-21T00:00:00.000Z',
data: { id },
});
spooler.enqueue([mkEnv('e1'), mkEnv('e2'), mkEnv('e3'), mkEnv('e4'), mkEnv('e5')]);
const removedCount = spooler.ackAndRequeueFront(3, [mkEnv('e2')]);
assert.equal(removedCount, 2);
const afterPartial = spooler.peekBatch(10).map((e) => e.eventId);
assert.deepEqual(afterPartial, ['e2', 'e4', 'e5']);
// Ensure no leftover .tmp files exist in spoolDir
const entries = fs.readdirSync(tmpDir);
assert.equal(entries.some((f) => f.endsWith('.tmp')), false);
// Clean and test partial recovery through DataStack.flushSpool()
spooler.clear();
const deliveredIds: string[] = [];
let simulatePartial = true;
const token = makeFakeJwt('sensor_node');
const client = new DataStack({
token,
apiUrl: 'http://localhost:3000',
serverUrl: 'http://localhost:4000',
autoConnect: false,
offlineSpoolEnabled: true,
offlineSpoolDir: tmpDir,
batchChunkSize: 3,
maxRetries: 0,
fetchImpl: async (_input, init) => {
const body = JSON.parse(String(init?.body || '{}')) as {
batch?: Array<{ eventId: string }>;
};
const batch = body.batch || [];
if (simulatePartial && batch.length === 3) {
// Simulate partial 429 HTTP rejection where batch[1] is rate-limited
const limitedId = batch[1].eventId;
deliveredIds.push(batch[0].eventId, batch[2].eventId);
return new Response(
JSON.stringify({
success: false,
code: 'RATE_LIMIT_EXCEEDED',
error: 'Rate limit reached for 1 event',
rateLimitedEventIds: [limitedId],
}),
{ status: 429, headers: { 'Content-Type': 'application/json' } }
);
}
for (const item of batch) {
deliveredIds.push(item.eventId);
}
return new Response(
JSON.stringify({ success: true, accepted: batch.length }),
{ status: 200, headers: { 'Content-Type': 'application/json' } }
);
},
});
client.spooler.enqueue([mkEnv('s1'), mkEnv('s2'), mkEnv('s3'), mkEnv('s4')]);
const firstFlush = await client.flushSpool();
assert.equal(firstFlush.drained, 2);
assert.equal(firstFlush.remaining, 2);
assert.deepEqual(
client.spooler.peekBatch(10).map((e) => e.eventId),
['s2', 's4']
);
// Next flush delivers remaining [s2, s4] cleanly with zero duplicates or data loss
simulatePartial = false;
const secondFlush = await client.flushSpool();
assert.equal(secondFlush.drained, 2);
assert.equal(secondFlush.remaining, 0);
assert.deepEqual(deliveredIds, ['s1', 's3', 's2', 's4']);
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});
it('AUDIT-08 & ERR-R4-09: docker-compose ingestion configures DATASTACK_ADMIN_EMAIL and /api/files/status + UI expose non-Nginx WS hint', async () => {
const compose = fs.readFileSync(path.join(process.cwd(), 'docker', 'docker-compose.yml'), 'utf8');
const ingestionBlockMatch = compose.match(
/ingestion:\s*\n[\s\S]*?\n\s+proxy:/
);
assert.ok(ingestionBlockMatch, 'ingestion service block must exist in docker-compose.yml');
assert.match(
ingestionBlockMatch[0],
/DATASTACK_ADMIN_EMAIL:\s*"\$\{DATASTACK_ADMIN_EMAIL:-admin@datastack\.dev\}"/
);
delete process.env.DATABASE_URL;
delete process.env.DATASTACK_PROXY_MODE;
process.env.DATASTACK_ADMIN_SESSION_SECRET =
'phase03-files-status-session-secret-min-32-chars';
const secret = getAdminSessionSecret()!;
const ownerCookie = await createAdminSessionToken(
'admin@datastack.dev',
secret,
Date.now() + 60_000,
1,
'owner',
'owner-1'
);
globalThis.fetch = (async () =>
new Response(
JSON.stringify({
ok: true,
engine: 'cloudcmd',
role: 'owner',
readOnly: false,
}),
{ status: 200, headers: { 'Content-Type': 'application/json' } }
)) as typeof fetch;
// 1. Standalone Next.js mode (no reverse-proxy headers)
const standaloneRes = await filesStatusGet(
new NextRequest('http://localhost:3000/api/files/status', {
headers: { cookie: `datastack_admin_session=${ownerCookie}` },
})
);
assert.equal(standaloneRes.status, 200);
const standaloneJson = (await standaloneRes.json()) as {
ok: boolean;
proxyMode: string;
wsSupported: boolean;
wsHint: string | null;
};
assert.equal(standaloneJson.ok, true);
assert.equal(standaloneJson.proxyMode, 'standalone');
assert.equal(standaloneJson.wsSupported, false);
assert.ok(standaloneJson.wsHint && /HTTP polling/i.test(standaloneJson.wsHint));
// 2. Nginx reverse-proxy headers present
const nginxRes = await filesStatusGet(
new NextRequest('http://localhost:3000/api/files/status', {
headers: {
cookie: `datastack_admin_session=${ownerCookie}`,
'x-forwarded-host': 'console.example.com',
'x-forwarded-proto': 'https',
},
})
);
assert.equal(nginxRes.status, 200);
const nginxJson = (await nginxRes.json()) as {
proxyMode: string;
wsSupported: boolean;
wsHint: string | null;
};
assert.equal(nginxJson.proxyMode, 'nginx');
assert.equal(nginxJson.wsSupported, true);
assert.equal(nginxJson.wsHint, null);
// 3. UI component renders proxy hint element safely when present
const workspaceSrc = fs.readFileSync(
path.join(process.cwd(), 'components', 'sections', 'FilesManagerWorkspace.tsx'),
'utf8'
);
assert.ok(workspaceSrc.includes('files-manager-proxy-hint'));
assert.ok(workspaceSrc.includes('data.wsHint'));
});
it('AUDIT-13: middleware removes dead /api/auth/setup/status exemption and POST /api/auth/setup embeds explicit owner role/memberId', async () => {
const mwSrc = fs.readFileSync(path.join(process.cwd(), 'middleware.ts'), 'utf8');
assert.equal(mwSrc.includes('/api/auth/setup/status'), false);
process.env.DATASTACK_ADMIN_SESSION_SECRET =
'phase03-setup-session-secret-at-least-32-chars';
const mwRes = await middleware(
new NextRequest('http://localhost:3000/api/auth/setup/status', { method: 'GET' })
);
assert.equal(mwRes.status, 401);
delete process.env.DATABASE_URL;
process.env.DATASTACK_ALLOW_DEMO_ADMIN = 'false';
process.env.DATASTACK_ADMIN_SETUP_KEY =
'phase03-custom-setup-key-over-32-characters-long';
const setupRes = await setupPost(
new NextRequest('http://localhost:3000/api/auth/setup', {
method: 'POST',
headers: {
'content-type': 'application/json',
'x-forwarded-for': '203.0.113.15',
},
body: JSON.stringify({
email: 'founder@datastack.dev',
password: 'FounderPassword123!',
setupKey: process.env.DATASTACK_ADMIN_SETUP_KEY,
}),
})
);
assert.equal(setupRes.status, 200);
const cookieVal = setupRes.cookies.get('datastack_admin_session')?.value;
assert.ok(cookieVal);
const payload = await getAdminSessionPayload(
cookieVal!,
process.env.DATASTACK_ADMIN_SESSION_SECRET
);
assert.ok(payload);
assert.equal(payload!.email, 'founder@datastack.dev');
assert.equal(payload!.role, 'owner');
assert.equal(payload!.memberId, 'owner-1');
});
});
+98
View File
@@ -0,0 +1,98 @@
import assert from 'node:assert/strict';
import fs from 'node:fs/promises';
import path from 'node:path';
import test from 'node:test';
import { getVconRoleCapabilities } from '../lib/vconUiHelpers';
const ROOT = process.cwd();
async function readSource(relPath: string): Promise<string> {
return fs.readFile(path.join(ROOT, relPath), 'utf8');
}
test('R4-Phase-04 [AUDIT-14]: TeamWorkspace seeds role from useVconRole, confirms deletes via Modal, and distinguishes operator vs admin', async () => {
const src = await readSource('components/sections/TeamWorkspace.tsx');
// Eliminates hardcoded 'owner' initial flash
assert.match(src, /useVconRole\(\)/);
assert.doesNotMatch(src, /useState<string>\(\s*['"]owner['"]\s*\)/);
assert.match(src, /useState<string>\(\(\)\s*=>\s*shellRole\s*\|\|\s*['"]viewer['"]\)/);
// Explicit delete confirmation modal before DELETE /api/team/[id]
assert.match(src, /setDeleteTarget\(row\)/);
assert.match(src, /title="Confirm Team Member Removal"/);
assert.match(src, /data-testid="confirm-remove-member"/);
assert.doesNotMatch(src, /onClick=\{\(\)\s*=>\s*void handleDeleteMember\(row\)\}/);
// Preserves distinct operator vs admin options instead of silently coercing operator -> admin
assert.doesNotMatch(src, /row\.role === ['"]operator['"] \? ['"]admin['"]/);
assert.match(src, /value:\s*['"]operator['"]/);
assert.match(src, /value:\s*['"]admin['"]/);
});
test('R4-Phase-04 [AUDIT-15 / ERR-R4-10]: /vcon/sdk skips ?includeToken=true fetch for viewer and renders explicit RBAC notice', async () => {
const src = await readSource('app/vcon/sdk/page.tsx');
assert.match(src, /useVconRole\(\)/);
assert.match(src, /getVconRoleCapabilities\(shellRole\)/);
assert.match(src, /if \(!selectedClientId \|\| !canOperate\)/);
assert.match(src, /data-testid="sdk-viewer-rbac-banner"/);
assert.match(src, /Operator or Owner role required/i);
// Confirm capability helper blocks viewer token retrieval while allowing operator/admin/owner
assert.equal(getVconRoleCapabilities('viewer').canOperate, false);
assert.equal(getVconRoleCapabilities('operator').canOperate, true);
assert.equal(getVconRoleCapabilities('admin').canOperate, true);
assert.equal(getVconRoleCapabilities('owner').canOperate, true);
// Preserve V2-Phase-04 and R2-Phase-03 compatibility checks
assert.match(src, /Python SDK \(v2\.2\.0\)/);
assert.match(src, /Node\.js \/ TypeScript SDK \(v2\.2\.0\)/);
assert.match(src, /setSelectedClientToken\(''\);\s*\n\s*setLoadingToken\(true\);/);
});
test('R4-Phase-04 [AUDIT-16]: Sidebar uses useVconRole and Client SDKs label; Navbar covers all admin routes', async () => {
const sidebarSrc = await readSource('components/layout/Sidebar.tsx');
assert.match(sidebarSrc, /useVconRole\(\)/);
assert.match(sidebarSrc, /\{\s*name:\s*['"]Client SDKs['"],\s*href:\s*['"]\/vcon\/sdk['"]/);
assert.doesNotMatch(sidebarSrc, /\{\s*name:\s*['"]Python SDK['"]/);
assert.doesNotMatch(sidebarSrc, /getCached<CachedProfile>\('vcon:profile',\s*\{\}\)\.role\s*\|\|\s*['"]Owner['"]/);
assert.match(sidebarSrc, /getCached<CachedProfile>\('vcon:profile',\s*\{\}\)\.role\s*\|\|\s*initialRoleLabel/);
const navbarSrc = await readSource('components/layout/Navbar.tsx');
assert.match(navbarSrc, /'\/vcon\/files':\s*'Files Manager'/);
assert.match(navbarSrc, /'\/vcon\/sdk':\s*'Client SDKs'/);
assert.match(navbarSrc, /'\/vcon\/team':\s*'Team & RBAC'/);
assert.match(navbarSrc, /'\/vcon\/audit':\s*'Audit Log'/);
assert.doesNotMatch(navbarSrc, /'\/vcon\/sdk':\s*'Python SDK'/);
});
test('R4-Phase-04 [AUDIT-19 & AUDIT-07]: ProfileWorkspace drops hardcoded demo defaults and DataBrowser surfaces corruptPartitions warnings', async () => {
const profileSrc = await readSource('components/sections/ProfileWorkspace.tsx');
assert.doesNotMatch(profileSrc, /DEFAULT_PROFILE/);
assert.doesNotMatch(profileSrc, /email:\s*'admin@datastack\.dev'/);
assert.doesNotMatch(profileSrc, /storageMode:\s*'In-Memory'/);
assert.match(profileSrc, /EMPTY_PROFILE/);
assert.match(profileSrc, /Loading\.\.\./);
const dataBrowserSrc = await readSource('components/sections/DataBrowser.tsx');
assert.match(dataBrowserSrc, /corruptPartitions/);
assert.match(dataBrowserSrc, /data-testid="data-corrupt-partitions-warning"/);
});
test('R4-Phase-04 [AUDIT-18]: Public landing Hero, SdkSection, and LandingAbout reflect both Python and Node.js / TypeScript SDKs', async () => {
const heroSrc = await readSource('components/sections/Hero.tsx');
assert.match(heroSrc, /href="\/#sdk"[\s\S]*?>\s*Client SDKs\s*<\/Link>/);
assert.doesNotMatch(heroSrc, />\s*Python SDK\s*<\/Link>/);
const sdkSectionSrc = await readSource('components/sections/SdkSection.tsx');
assert.match(sdkSectionSrc, /label="Client SDKs"/);
assert.match(sdkSectionSrc, /datastack/);
assert.match(sdkSectionSrc, /@datastack\/sdk/);
assert.match(sdkSectionSrc, /npm install/);
assert.match(sdkSectionSrc, /pip install/);
const aboutSrc = await readSource('components/sections/LandingAbout.tsx');
assert.match(aboutSrc, /href="\/#sdk"[\s\S]*?>\s*View Client SDKs\s*<\/Link>/);
assert.doesNotMatch(aboutSrc, /View Python SDK/);
});
+17 -3
View File
@@ -146,6 +146,9 @@ test("V2-Phase-01 [S3 SigV4]: buildS3ObjectKey, normalizeS3EndpointUrl, testS3Co
const addr = mockS3.address() as AddressInfo;
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "datastack-v2-s3-"));
const prevAllowPrivate = process.env.DATASTACK_ALLOW_PRIVATE_URLS;
process.env.DATASTACK_ALLOW_PRIVATE_URLS = "true";
try {
const samplePath = path.join(tmpDir, "2026-10-01.csv.gz");
const gzData = zlib.gzipSync(
@@ -189,6 +192,8 @@ test("V2-Phase-01 [S3 SigV4]: buildS3ObjectKey, normalizeS3EndpointUrl, testS3Co
assert.equal(uploadErr.code, "S3_ARCHIVE_FAILED");
assert.match(uploadErr.error || "", /HTTP 500/);
} finally {
if (prevAllowPrivate === undefined) delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
else process.env.DATASTACK_ALLOW_PRIVATE_URLS = prevAllowPrivate;
await new Promise<void>((resolve) => mockS3.close(() => resolve()));
await fs.rm(tmpDir, { recursive: true, force: true });
}
@@ -229,6 +234,9 @@ test("V2-Phase-01 [Compression, Export, S3 Safe Prune V2-E02, Corrupt Gzip V2-E0
await new Promise<void>((resolve) => mockS3.listen(0, "127.0.0.1", resolve));
const s3Addr = mockS3.address() as AddressInfo;
const prevAllowPrivate = process.env.DATASTACK_ALLOW_PRIVATE_URLS;
process.env.DATASTACK_ALLOW_PRIVATE_URLS = "true";
try {
const cookie = await makeAdminCookie();
const today = new Date().toISOString().slice(0, 10);
@@ -412,7 +420,7 @@ test("V2-Phase-01 [Compression, Export, S3 Safe Prune V2-E02, Corrupt Gzip V2-E0
assert.deepEqual(uploadedKeys, [`/prod-vault/archive-test/v2_alpha/${oldDate}.csv.gz`]);
await fs.stat(gzMeta.filePath);
// 10. V2-E03: Corrupt .csv.gz cleanly surfaces error in /api/data instead of crashing
// 10. V2-E03 upgraded to ERR-R4-07: Corrupt .csv.gz is isolated per-file (returns 200 with corruptPartitions warning instead of crashing all clients)
await fs.writeFile(
path.join(tmpDir, "v2_alpha", "2024-01-01.csv.gz"),
Buffer.from("not-valid-gzip-bytes"),
@@ -423,10 +431,16 @@ test("V2-Phase-01 [Compression, Export, S3 Safe Prune V2-E02, Corrupt Gzip V2-E0
{ headers: { cookie } },
),
);
assert.equal(corruptDataRes.status, 500);
assert.equal(corruptDataRes.status, 200);
const corruptJson = await corruptDataRes.json();
assert.match(corruptJson.error || "", /Corrupt|\.gz/i);
assert.equal(corruptJson.warnings, 1);
assert.ok(Array.isArray(corruptJson.corruptPartitions));
assert.equal(corruptJson.corruptPartitions.length, 1);
assert.equal(corruptJson.corruptPartitions[0].code, "CORRUPT_GZIP_PARTITION");
assert.match(corruptJson.corruptPartitions[0].error || "", /Corrupt|\.gz/i);
} finally {
if (prevAllowPrivate === undefined) delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
else process.env.DATASTACK_ALLOW_PRIVATE_URLS = prevAllowPrivate;
await new Promise<void>((resolve) => mockS3.close(() => resolve()));
await DataBufferEngine.shutdown();
CSVStorageManager.baseStorageDir = origBaseStorageDir;
@@ -191,6 +191,9 @@ test("V2-Phase-02 [Webhook Notifier & /api/settings/webhook-test]: HMAC-SHA256 s
const port = (mockWebhookServer.address() as AddressInfo).port;
const webhookUrl = `http://127.0.0.1:${port}/hooks/datastack`;
const prevAllowPrivate = process.env.DATASTACK_ALLOW_PRIVATE_URLS;
process.env.DATASTACK_ALLOW_PRIVATE_URLS = "true";
try {
// 1. Format & signature verification
const formattedJsonStr = formatOutboundWebhookBody({
@@ -297,6 +300,8 @@ test("V2-Phase-02 [Webhook Notifier & /api/settings/webhook-test]: HMAC-SHA256 s
const failTestData = await failTestRes.json();
assert.equal(failTestData.code, "WEBHOOK_DELIVERY_FAILED");
} finally {
if (prevAllowPrivate === undefined) delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
else process.env.DATASTACK_ALLOW_PRIVATE_URLS = prevAllowPrivate;
await new Promise<void>((resolve) => mockWebhookServer.close(() => resolve()));
await DataBufferEngine.shutdown();
resetAdminStateForTests();
@@ -807,6 +812,9 @@ test("V2-Audit-Phase-02 [BUG-V2-AUDIT-05, BUG-V2-AUDIT-06, BUG-V2-AUDIT-08]: Por
);
const socketPort = (testHttpServer.address() as AddressInfo).port;
const prevAllowPrivate = process.env.DATASTACK_ALLOW_PRIVATE_URLS;
process.env.DATASTACK_ALLOW_PRIVATE_URLS = "true";
try {
serverIndex.clearWebhookCooldownCacheForTests();
serverIndex.applyRuntimeSettingsObject({
@@ -890,6 +898,8 @@ test("V2-Audit-Phase-02 [BUG-V2-AUDIT-05, BUG-V2-AUDIT-06, BUG-V2-AUDIT-08]: Por
const forbiddenHealthBody = await forbiddenHealthRes.json();
assert.equal(forbiddenHealthBody.code, "CLIENT_IP_FORBIDDEN");
} finally {
if (prevAllowPrivate === undefined) delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
else process.env.DATASTACK_ALLOW_PRIVATE_URLS = prevAllowPrivate;
serverIndex.applyRuntimeSettingsObject({
enableWebhookAlerts: false,
webhookUrl: "",