From b2669e2ad5e501a86317c5f9b99b18981fd4ef5f Mon Sep 17 00:00:00 2001 From: Team Dark Devil Date: Sat, 21 Nov 2026 23:00:29 +0600 Subject: [PATCH] feat(mcp): implement MCP infrastructure - Add `mcp_settings` table to database schema - Enable internal MCP API endpoint via middleware header authentication - Introduce runtime log directory and capture utilities for diagnostics - Update build pipeline to capture build logs --- MEMORY.md | 11 +- app/api/internal/mcp/route.ts | 1018 ++++++++++++++++++ core/Database/database.ts | 15 + core/auth/auditLog.ts | 11 + core/logs/runtimeLogCapture.ts | 448 ++++++++ core/mcp/mcpSettings.ts | 672 ++++++++++++ docker/Dockerfile.web | 4 +- instrumentation.ts | 11 + middleware.ts | 21 +- package.json | 2 +- scripts/capture-build-log.js | 95 ++ scripts/dev-all.js | 32 +- scripts/start-all.js | 32 +- server/index.js | 19 + server/runtimeLogCapture.js | 371 +++++++ storage/.runtime-logs/last-build-meta.json | 11 + tests/phase07-deployment.test.ts | 6 +- tests/r4-phase01-security-boot.test.ts | 5 +- tests/r5-phase01-mcp-foundation-logs.test.ts | 516 +++++++++ 19 files changed, 3280 insertions(+), 20 deletions(-) create mode 100644 app/api/internal/mcp/route.ts create mode 100644 core/logs/runtimeLogCapture.ts create mode 100644 core/mcp/mcpSettings.ts create mode 100644 scripts/capture-build-log.js create mode 100644 server/runtimeLogCapture.js create mode 100644 storage/.runtime-logs/last-build-meta.json create mode 100644 tests/r5-phase01-mcp-foundation-logs.test.ts diff --git a/MEMORY.md b/MEMORY.md index 3aa6ede..dfa205e 100644 --- a/MEMORY.md +++ b/MEMORY.md @@ -1,7 +1,7 @@ # MEMORY.md — DataStack Cross-IDE Persistent Project Memory > **Last Synchronized**: 2026-11-21 -> **Status**: **`R4` Production Hardening (`BASELINE-2026-11-20-R4`: `4 / 4` Phases Complete — All `R4-Phase-01`..`04` Complete & Verified (`AUDIT-01`–`20`, `19 / 19` actionable findings resolved)).** +> **Status**: **`R5-MCP` (`BASELINE-2026-11-21-R5-MCP`: `1 / 3` Phases Complete — `R5-Phase-01` Complete & Verified; `R4` `4/4` & `PREPUSH-FIX-01` `5/5` Preserved).** > **Scope Rule**: Maintain this file accurately across VS Code, Google Antigravity, Cursor, and any other IDE after every modification, feature update, or bug fix (see `.ai-rules` and `AGENTS.md`). --- @@ -423,6 +423,15 @@ All planned hardening & deployment phases (**Phase-01 through Phase-07**) are ** - **`DEPLOY-BUG-02`**: Hardened `docker/coolify-nginx.conf` and `nginx/default.conf` with `ngx_http_realip_module` (`set_real_ip_from` loopback/RFC1918/Docker bridges + IPv6 ULA, `real_ip_header X-Forwarded-For`, `real_ip_recursive on`) so `proxy_set_header X-Real-IP $remote_addr` preserves real external client IPs behind Coolify Traefik without permitting untrusted public IP spoofing. - **`DEPLOY-BUG-03` & `DEPLOY-BUG-04`**: Added `ARG INGESTION_INTERNAL_URL=http://ingestion:4000` + `ENV INGESTION_INTERNAL_URL` in `docker/Dockerfile.web` `builder` stage for standalone Next.js rewrites, and forwarded optional `DATASTACK_ALLOW_PRIVATE_URLS`, `DATASTACK_MAX_BATCH_ITEMS`, `DATASTACK_RECENT_LIMIT`, `DATASTACK_MAINTENANCE_INTERVAL_MS`, and `DATASTACK_PROXY_MODE` env vars in `docker/docker-compose.yml`. - **`DEPLOY-BUG-05`**: Updated `sdk/node/datastack/index.ts` (`emitSocketSingle` and `sendBatchChunkInternal`) to call `this.disconnect()` on terminal `DataStackAuthError` rejections (`TOKEN_ROTATED`, `CLIENT_REVOKED`, etc.) and recompiled `sdk/node/datastack/dist/*` so subsequent `client.connect()` performs a fresh handshake (`tests/r3-phase03-sdk-parity.test.ts` `5/5` PASS). + - **`R5-MCP` Planning, Foundation & Server Track (`BASELINE-2026-11-21-R5-MCP`)**: + - Froze post-`R4` / post-`PREPUSH-FIX-01` codebase and documented the full 8-item raw-source-verified MCP forensic audit (`MCP-AUDIT-01`–`MCP-AUDIT-08`), Scope Locks, A–Z 3-phase roadmap (`R5-Phase-01`–`03`), Phase Execution Log, `ERR-R5-01`–`09` error contracts, and Actual Status Ledger across `/notes/`. + - **`R5-Phase-01` Complete & Verified (`2026-11-21`)** (`MCP-AUDIT-02` Complete; `MCP-AUDIT-03/04/05/07` Foundation; `ERR-R5-01` & `ERR-R5-02`): + - Added 8th PostgreSQL table `mcp_settings` identically in `core/Database/database.ts` and `server/index.js` (`ensureDatabaseSchema()`). + - Created TS (`core/logs/runtimeLogCapture.ts`) and CJS (`server/runtimeLogCapture.js`) bounded rotating runtime/build log writers & readers (`.runtime-logs/{web,ingestion}.log`, max 2MB + `.1` backup, secret redaction for Bearer/`dsmcp_`/JWT/JSON keys, graceful `LOG_CAPTURE_UNAVAILABLE` fallback), wired into `instrumentation.ts`, `server/index.js`, `scripts/start-all.js`, and `scripts/dev-all.js` (`DATASTACK_LOG_CAPTURED_BY_RUNNER=1` prevents duplicate tee writes). + - Created `scripts/capture-build-log.js` (`package.json` `"build"`) and updated `docker/Dockerfile.web` so standalone Next.js builds emit `.next/standalone/build.log`, `.next/standalone/build-meta.json`, and `.runtime-logs/last-build*`. + - Created `core/mcp/mcpSettings.ts` (`McpSettingsManager` with default `enabled: false`, `17` permission definitions across 4 categories, `dsmcp_` token generation + SHA-256 digest + AES-256-GCM vault encryption, IP CIDR validation, SSRF-guarded `coolifyUrl`, PG/RAM roundtrip, and `POST /internal/mcp-sync` notifier) and added `MCP_*` action types + secret redactions in `core/auth/auditLog.ts`. + - Added `middleware.ts` internal secret pass + timing-safe `/api/internal/mcp` bridge (`app/api/internal/mcp/route.ts`) supporting metrics, runtime/build logs, audit queries, client CRUD, safe settings updates, CSV+buffer data queries, storage stats/flush, and S3 archive. + - **Verified**: `5/5` `tests/r5-phase01-mcp-foundation-logs.test.ts` + `78/78` regression checks across 8 suites (`83/83` PASS), `compile_applet` (`npm run build` -> `.next/standalone/build.log` + `build-meta.json` generated), `lint_applet` (`eslint .` -> `0` errors), and `npm ci --include=dev --dry-run` / `--omit=dev --dry-run` clean. Ready for `Proceed to R5-Phase-02`. --- diff --git a/app/api/internal/mcp/route.ts b/app/api/internal/mcp/route.ts new file mode 100644 index 0000000..9b6f4b2 --- /dev/null +++ b/app/api/internal/mcp/route.ts @@ -0,0 +1,1018 @@ +import { timingSafeEqual } from "node:crypto"; +import { NextRequest, NextResponse } from "next/server"; +import { + SettingsManager, + type AdminSettingsUpdateInput, +} from "@/core/adminSettings/settingsManager"; +import { ApiEngine } from "@/core/apiEngine/apiService"; +import { + listAdminAuditLogsWithStatusAsync, + recordAdminAuditLogAsync, +} from "@/core/auth/auditLog"; +import { DataBufferEngine, REDIS_BUFFER_KEY } from "@/core/data/buffer"; +import { + readBuildLogs, + readRuntimeLogs, +} from "@/core/logs/runtimeLogCapture"; +import { SDKManager } from "@/core/sdk/sdkManager"; +import { + validateAllowedIpsInput, + validateClientExpiresAtInput, +} from "@/core/Security/ipAllowlist"; +import { resolveInternalIngestionSecret } from "@/core/Security/security"; +import { SSRFBlockedError, validateOutboundUrlSync } from "@/core/Security/ssrfGuard"; +import { CSVStorageManager } from "@/core/Storage/csvStorage"; +import { + buildS3ObjectKey, + normalizeS3EndpointUrl, + uploadFileToS3, +} from "@/core/Storage/s3Archive"; +import { parse } from "csv-parse"; +import { lstat, readdir } from "node:fs/promises"; +import path from "node:path"; + +const MCP_ALLOWED_SETTINGS_FIELDS = new Set([ + "maxEventsPerSec", + "maxPayloadSizeKb", + "batchFlushThreshold", + "batchFlushIntervalMs", + "fileRotationMb", + "enableIdempotency", + "autoPruneDays", + "compressAfterDays", + "diskPauseThresholdPct", + "enableWebhookAlerts", + "webhookUrl", + "webhookCooldownSec", + "enableS3Backup", + "s3Endpoint", + "s3Region", + "s3Bucket", + "s3Prefix", + "s3AccessKeyId", + "enablePrometheusMetrics", +]); + +function verifyInternalBridgeSecret(request: NextRequest): boolean { + const presented = + request.headers.get("x-datastack-internal-secret")?.trim() || ""; + const expected = resolveInternalIngestionSecret(); + if (!presented || !expected) return false; + const presentedBuf = Buffer.from(presented, "utf8"); + const expectedBuf = Buffer.from(expected, "utf8"); + if (presentedBuf.length !== expectedBuf.length) return false; + return timingSafeEqual(presentedBuf, expectedBuf); +} + +function serializeRedactedClient( + client: { + clientId: string; + name: string; + secret?: string; + token?: string; + rateLimit: number; + storageQuotaMb: number; + allowedIps: string; + expiresAt?: string | null; + status: "active" | "revoked"; + totalEvents: number; + lastActive?: string; + createdAt: string; + }, + includeToken = false, +) { + const { secret, token, ...rest } = client; + void secret; + return { + ...rest, + tokenPreview: token ? `${token.slice(0, 12)}...` : "", + ...(includeToken && token ? { token } : {}), + }; +} + +function parseBoundedInt( + raw: unknown, + field: string, + min: number, + max: number, +): number { + if (typeof raw !== "number" || !Number.isInteger(raw)) { + throw new Error(`${field} must be an integer between ${min} and ${max}.`); + } + if (raw < min || raw > max) { + throw new Error(`${field} must be between ${min} and ${max}.`); + } + return raw; +} + +export async function POST(req: NextRequest) { + if (!verifyInternalBridgeSecret(req)) { + return NextResponse.json( + { + error: "Internal MCP bridge authentication failed.", + code: "MCP_BRIDGE_UNAUTHORIZED", + }, + { status: 401 }, + ); + } + + let body: { action?: unknown; params?: unknown; actorIp?: unknown }; + try { + const parsed = await req.json(); + body = + parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as { action?: unknown; params?: unknown; actorIp?: unknown }) + : {}; + } catch { + return NextResponse.json( + { error: "Invalid JSON request payload.", code: "INVALID_JSON" }, + { status: 400 }, + ); + } + + const action = typeof body.action === "string" ? body.action.trim() : ""; + const params = + body.params && typeof body.params === "object" && !Array.isArray(body.params) + ? (body.params as Record) + : {}; + const actorIp = + typeof body.actorIp === "string" ? body.actorIp.trim().slice(0, 128) : ""; + + try { + switch (action) { + case "metrics_get": { + const [metrics, diskUsage] = await Promise.all([ + ApiEngine.getDashboardMetrics(), + CSVStorageManager.getDiskUsageAsync(), + ]); + return NextResponse.json({ success: true, metrics, diskUsage }); + } + + case "runtime_logs_read": { + const result = readRuntimeLogs({ + service: + params.service === "web" || params.service === "ingestion" + ? params.service + : "all", + lines: typeof params.lines === "number" ? params.lines : 120, + search: typeof params.search === "string" ? params.search : undefined, + level: typeof params.level === "string" ? params.level : undefined, + }); + return NextResponse.json({ success: true, ...result }); + } + + case "build_logs_read": { + const result = readBuildLogs({ + lines: typeof params.lines === "number" ? params.lines : 200, + search: typeof params.search === "string" ? params.search : undefined, + }); + return NextResponse.json({ success: true, ...result }); + } + + case "audit_query": { + const result = await listAdminAuditLogsWithStatusAsync({ + limit: typeof params.limit === "number" ? params.limit : 100, + action: typeof params.action === "string" ? params.action : undefined, + actorEmail: + typeof params.actorEmail === "string" + ? params.actorEmail + : undefined, + search: typeof params.search === "string" ? params.search : undefined, + }); + return NextResponse.json({ success: true, ...result }); + } + + case "clients_list": { + const clients = (await SDKManager.listClientsAsync()).map((c) => + serializeRedactedClient(c, false), + ); + return NextResponse.json({ success: true, clients, count: clients.length }); + } + + case "clients_get": { + const rawId = typeof params.clientId === "string" ? params.clientId : ""; + const normalizedId = SDKManager.normalizeClientId(rawId); + if (!normalizedId) { + return NextResponse.json( + { error: "clientId is required.", code: "VALIDATION_ERROR" }, + { status: 400 }, + ); + } + const client = await SDKManager.getClientAsync(normalizedId); + if (!client) { + return NextResponse.json( + { error: `Client '${normalizedId}' not found.`, code: "NOT_FOUND" }, + { status: 404 }, + ); + } + return NextResponse.json({ + success: true, + client: serializeRedactedClient(client, false), + }); + } + + case "clients_create": { + const rawClientId = + typeof params.clientId === "string" ? params.clientId : ""; + const rawName = typeof params.name === "string" ? params.name : ""; + if (!rawClientId.trim() || !rawName.trim()) { + return NextResponse.json( + { + error: "clientId and name are required non-empty strings.", + code: "VALIDATION_ERROR", + }, + { status: 400 }, + ); + } + const safeClientId = SDKManager.normalizeClientId(rawClientId); + if (!safeClientId) { + return NextResponse.json( + { error: "clientId is invalid.", code: "VALIDATION_ERROR" }, + { status: 400 }, + ); + } + const parsedRateLimit = + params.rateLimit === undefined ? 100 : Number(params.rateLimit); + if ( + !Number.isInteger(parsedRateLimit) || + parsedRateLimit < 1 || + parsedRateLimit > 100000 + ) { + return NextResponse.json( + { + error: "rateLimit must be an integer between 1 and 100000.", + code: "VALIDATION_ERROR", + }, + { status: 400 }, + ); + } + const parsedQuotaMb = + params.storageQuotaMb === undefined + ? 0 + : Number(params.storageQuotaMb); + if ( + !Number.isInteger(parsedQuotaMb) || + parsedQuotaMb < 0 || + parsedQuotaMb > 1048576 + ) { + return NextResponse.json( + { + error: "storageQuotaMb must be an integer between 0 and 1048576.", + code: "VALIDATION_ERROR", + }, + { status: 400 }, + ); + } + let normalizedAllowedIps = ""; + if (params.allowedIps !== undefined && params.allowedIps !== null) { + if (typeof params.allowedIps !== "string") { + return NextResponse.json( + { error: "allowedIps must be a string.", code: "VALIDATION_ERROR" }, + { status: 400 }, + ); + } + const ipRes = validateAllowedIpsInput(params.allowedIps); + if (!ipRes.valid) { + return NextResponse.json( + { + error: ipRes.error || "Invalid allowedIps.", + code: "VALIDATION_ERROR", + }, + { status: 400 }, + ); + } + normalizedAllowedIps = ipRes.normalized; + } + let normalizedExpiresAt: string | null = null; + if (params.expiresAt !== undefined) { + const expRes = validateClientExpiresAtInput(params.expiresAt); + if (!expRes.valid) { + return NextResponse.json( + { + error: expRes.error || "Invalid expiresAt.", + code: "VALIDATION_ERROR", + }, + { status: 400 }, + ); + } + normalizedExpiresAt = expRes.iso ?? null; + } + const existing = await SDKManager.getClientAsync(safeClientId); + if (existing) { + return NextResponse.json( + { error: `Client '${safeClientId}' already exists.`, code: "CONFLICT" }, + { status: 409 }, + ); + } + const created = await SDKManager.registerClientAsync({ + clientId: safeClientId, + name: rawName.trim().slice(0, 120), + rateLimit: parsedRateLimit, + storageQuotaMb: parsedQuotaMb, + allowedIps: normalizedAllowedIps, + expiresAt: normalizedExpiresAt, + }); + await recordAdminAuditLogAsync({ + actorEmail: "mcp-agent", + actorRole: "mcp", + action: "CLIENT_CREATED", + target: created.clientId, + ipAddress: actorIp, + metadata: { + via: "mcp", + name: created.name, + rateLimit: created.rateLimit, + storageQuotaMb: created.storageQuotaMb, + }, + }); + return NextResponse.json( + { success: true, client: serializeRedactedClient(created, true) }, + { status: 201 }, + ); + } + + case "clients_update": { + const rawId = typeof params.clientId === "string" ? params.clientId : ""; + const normalizedId = SDKManager.normalizeClientId(rawId); + if (!normalizedId) { + return NextResponse.json( + { error: "clientId is required.", code: "VALIDATION_ERROR" }, + { status: 400 }, + ); + } + const updates: { + name?: string; + rateLimit?: number; + storageQuotaMb?: number; + allowedIps?: string; + expiresAt?: string | null; + status?: "active" | "revoked"; + } = {}; + if (params.name !== undefined) { + if ( + typeof params.name !== "string" || + params.name.trim().length < 1 || + params.name.trim().length > 120 + ) { + return NextResponse.json( + { error: "name must be 1..120 characters.", code: "VALIDATION_ERROR" }, + { status: 400 }, + ); + } + updates.name = params.name.trim(); + } + if (params.rateLimit !== undefined) { + updates.rateLimit = parseBoundedInt( + params.rateLimit, + "rateLimit", + 1, + 100000, + ); + } + if (params.storageQuotaMb !== undefined) { + updates.storageQuotaMb = parseBoundedInt( + params.storageQuotaMb, + "storageQuotaMb", + 0, + 1048576, + ); + } + if (params.allowedIps !== undefined) { + if (typeof params.allowedIps !== "string") { + return NextResponse.json( + { error: "allowedIps must be a string.", code: "VALIDATION_ERROR" }, + { status: 400 }, + ); + } + const ipRes = validateAllowedIpsInput(params.allowedIps); + if (!ipRes.valid) { + return NextResponse.json( + { + error: ipRes.error || "Invalid allowedIps.", + code: "VALIDATION_ERROR", + }, + { status: 400 }, + ); + } + updates.allowedIps = ipRes.normalized; + } + if (params.expiresAt !== undefined) { + const expRes = validateClientExpiresAtInput(params.expiresAt); + if (!expRes.valid) { + return NextResponse.json( + { + error: expRes.error || "Invalid expiresAt.", + code: "VALIDATION_ERROR", + }, + { status: 400 }, + ); + } + updates.expiresAt = expRes.iso ?? null; + } + if (params.status !== undefined) { + if (params.status !== "active" && params.status !== "revoked") { + return NextResponse.json( + { + error: "status must be 'active' or 'revoked'.", + code: "VALIDATION_ERROR", + }, + { status: 400 }, + ); + } + updates.status = params.status; + } + if (Object.keys(updates).length === 0) { + return NextResponse.json( + { error: "No update fields supplied.", code: "VALIDATION_ERROR" }, + { status: 400 }, + ); + } + const updated = await SDKManager.updateClientAsync(normalizedId, updates); + if (!updated) { + return NextResponse.json( + { error: `Client '${normalizedId}' not found.`, code: "NOT_FOUND" }, + { status: 404 }, + ); + } + await recordAdminAuditLogAsync({ + actorEmail: "mcp-agent", + actorRole: "mcp", + action: updates.status === "revoked" ? "CLIENT_REVOKED" : "CLIENT_UPDATED", + target: updated.clientId, + ipAddress: actorIp, + metadata: { via: "mcp", ...updates }, + }); + return NextResponse.json({ + success: true, + client: serializeRedactedClient(updated, false), + }); + } + + case "clients_rotate_token": { + const rawId = typeof params.clientId === "string" ? params.clientId : ""; + const normalizedId = SDKManager.normalizeClientId(rawId); + if (!normalizedId) { + return NextResponse.json( + { error: "clientId is required.", code: "VALIDATION_ERROR" }, + { status: 400 }, + ); + } + const refreshed = await SDKManager.regenerateTokenAsync(normalizedId); + if (!refreshed) { + return NextResponse.json( + { error: `Client '${normalizedId}' not found.`, code: "NOT_FOUND" }, + { status: 404 }, + ); + } + await recordAdminAuditLogAsync({ + actorEmail: "mcp-agent", + actorRole: "mcp", + action: "CLIENT_TOKEN_ROTATED", + target: refreshed.clientId, + ipAddress: actorIp, + metadata: { via: "mcp" }, + }); + return NextResponse.json({ + success: true, + client: serializeRedactedClient(refreshed, true), + }); + } + + case "clients_revoke": { + const rawId = typeof params.clientId === "string" ? params.clientId : ""; + const normalizedId = SDKManager.normalizeClientId(rawId); + if (!normalizedId) { + return NextResponse.json( + { error: "clientId is required.", code: "VALIDATION_ERROR" }, + { status: 400 }, + ); + } + const updated = await SDKManager.updateClientAsync(normalizedId, { + status: "revoked", + }); + if (!updated) { + return NextResponse.json( + { error: `Client '${normalizedId}' not found.`, code: "NOT_FOUND" }, + { status: 404 }, + ); + } + await recordAdminAuditLogAsync({ + actorEmail: "mcp-agent", + actorRole: "mcp", + action: "CLIENT_REVOKED", + target: updated.clientId, + ipAddress: actorIp, + metadata: { via: "mcp" }, + }); + return NextResponse.json({ + success: true, + client: serializeRedactedClient(updated, false), + }); + } + + case "clients_delete": { + const rawId = typeof params.clientId === "string" ? params.clientId : ""; + const normalizedId = SDKManager.normalizeClientId(rawId); + if (!normalizedId) { + return NextResponse.json( + { error: "clientId is required.", code: "VALIDATION_ERROR" }, + { status: 400 }, + ); + } + const purgeFiles = params.purgeFiles === true; + const result = await SDKManager.deleteClientWithStatsAsync(normalizedId, { + purgeFiles, + }); + if (!result.deleted) { + return NextResponse.json( + { error: `Client '${normalizedId}' not found.`, code: "NOT_FOUND" }, + { status: 404 }, + ); + } + await recordAdminAuditLogAsync({ + actorEmail: "mcp-agent", + actorRole: "mcp", + action: "CLIENT_DELETED", + target: normalizedId, + ipAddress: actorIp, + metadata: { via: "mcp", purgeFiles, purgedFiles: result.purgedFiles }, + }); + return NextResponse.json({ + success: true, + clientId: normalizedId, + purgedFiles: result.purgedFiles, + }); + } + + case "settings_get": { + const settings = await SettingsManager.getPublicSettingsAsync(); + return NextResponse.json({ success: true, settings }); + } + + case "settings_update": { + const unknown = Object.keys(params).filter( + (k) => !MCP_ALLOWED_SETTINGS_FIELDS.has(k), + ); + if (unknown.length > 0) { + return NextResponse.json( + { + error: `Disallowed or unknown settings field(s) for MCP: ${unknown.join(", ")}`, + code: "VALIDATION_ERROR", + }, + { status: 400 }, + ); + } + const validated: AdminSettingsUpdateInput = {}; + if (params.maxEventsPerSec !== undefined) { + validated.maxEventsPerSec = parseBoundedInt( + params.maxEventsPerSec, + "maxEventsPerSec", + 1, + 100000, + ); + } + if (params.maxPayloadSizeKb !== undefined) { + validated.maxPayloadSizeKb = parseBoundedInt( + params.maxPayloadSizeKb, + "maxPayloadSizeKb", + 1, + 10240, + ); + } + if (params.batchFlushThreshold !== undefined) { + validated.batchFlushThreshold = parseBoundedInt( + params.batchFlushThreshold, + "batchFlushThreshold", + 1, + 50000, + ); + } + if (params.batchFlushIntervalMs !== undefined) { + validated.batchFlushIntervalMs = parseBoundedInt( + params.batchFlushIntervalMs, + "batchFlushIntervalMs", + 100, + 60000, + ); + } + if (params.fileRotationMb !== undefined) { + validated.fileRotationMb = parseBoundedInt( + params.fileRotationMb, + "fileRotationMb", + 1, + 2048, + ); + } + if (params.autoPruneDays !== undefined) { + validated.autoPruneDays = parseBoundedInt( + params.autoPruneDays, + "autoPruneDays", + 0, + 3650, + ); + } + if (params.compressAfterDays !== undefined) { + validated.compressAfterDays = parseBoundedInt( + params.compressAfterDays, + "compressAfterDays", + 0, + 3650, + ); + } + if (params.diskPauseThresholdPct !== undefined) { + const pct = parseBoundedInt( + params.diskPauseThresholdPct, + "diskPauseThresholdPct", + 0, + 99, + ); + if (pct !== 0 && pct < 50) { + throw new Error( + "diskPauseThresholdPct must be 0 or between 50 and 99.", + ); + } + validated.diskPauseThresholdPct = pct; + } + if (params.enableIdempotency !== undefined) { + if (typeof params.enableIdempotency !== "boolean") { + throw new Error("enableIdempotency must be a boolean."); + } + validated.enableIdempotency = params.enableIdempotency; + } + if (params.enableWebhookAlerts !== undefined) { + if (typeof params.enableWebhookAlerts !== "boolean") { + throw new Error("enableWebhookAlerts must be a boolean."); + } + validated.enableWebhookAlerts = params.enableWebhookAlerts; + } + if (params.webhookUrl !== undefined) { + const wh = String(params.webhookUrl || "").trim(); + if (wh) { + const check = validateOutboundUrlSync(wh, { + fieldName: "webhookUrl", + }); + if (!check.allowed) { + if (check.code === "SSRF_BLOCKED_HOST") { + throw new SSRFBlockedError( + check.error || "webhookUrl blocked.", + check.hostname || "", + ); + } + throw new Error(check.error || "Invalid webhookUrl."); + } + } + validated.webhookUrl = wh; + } + if (params.webhookCooldownSec !== undefined) { + validated.webhookCooldownSec = parseBoundedInt( + params.webhookCooldownSec, + "webhookCooldownSec", + 10, + 86400, + ); + } + if (params.enableS3Backup !== undefined) { + if (typeof params.enableS3Backup !== "boolean") { + throw new Error("enableS3Backup must be a boolean."); + } + validated.enableS3Backup = params.enableS3Backup; + } + if (params.s3Endpoint !== undefined) { + const ep = String(params.s3Endpoint || "").trim(); + if (ep) { + const norm = /^https?:\/\//i.test(ep) ? ep : `https://${ep}`; + const check = validateOutboundUrlSync(norm, { + fieldName: "s3Endpoint", + }); + if (!check.allowed) { + if (check.code === "SSRF_BLOCKED_HOST") { + throw new SSRFBlockedError( + check.error || "s3Endpoint blocked.", + check.hostname || "", + ); + } + throw new Error(check.error || "Invalid s3Endpoint."); + } + } + validated.s3Endpoint = ep; + } + if (params.s3Region !== undefined) { + validated.s3Region = String(params.s3Region || "").trim().slice(0, 64) || "auto"; + } + if (params.s3Bucket !== undefined) { + validated.s3Bucket = String(params.s3Bucket || "").trim().slice(0, 128); + } + if (params.s3Prefix !== undefined) { + validated.s3Prefix = String(params.s3Prefix || "").trim().slice(0, 256); + } + if (params.s3AccessKeyId !== undefined) { + validated.s3AccessKeyId = String(params.s3AccessKeyId || "").trim().slice(0, 256); + } + if (params.enablePrometheusMetrics !== undefined) { + if (typeof params.enablePrometheusMetrics !== "boolean") { + throw new Error("enablePrometheusMetrics must be a boolean."); + } + validated.enablePrometheusMetrics = params.enablePrometheusMetrics; + } + + if (Object.keys(validated).length === 0) { + return NextResponse.json( + { error: "No valid settings supplied.", code: "VALIDATION_ERROR" }, + { status: 400 }, + ); + } + + await SettingsManager.updateSettingsAsync(validated); + await recordAdminAuditLogAsync({ + actorEmail: "mcp-agent", + actorRole: "mcp", + action: "SETTINGS_UPDATED", + target: "admin_settings", + ipAddress: actorIp, + metadata: { via: "mcp", updatedFields: Object.keys(validated) }, + }); + const settings = await SettingsManager.getPublicSettingsAsync(); + return NextResponse.json({ success: true, settings }); + } + + case "data_query": { + const rawClientId = + typeof params.clientId === "string" && params.clientId.trim() + ? params.clientId.trim() + : "all"; + const clientId = + rawClientId === "all" + ? "all" + : SDKManager.normalizeClientId(rawClientId) || "all"; + const sanitizedClientId = + clientId === "all" + ? "all" + : CSVStorageManager.sanitizeClientId(clientId); + const search = + typeof params.search === "string" + ? params.search.trim().toLowerCase() + : ""; + const limit = Math.min( + 100, + Math.max(1, Number(params.limit) || 25), + ); + const from = typeof params.from === "string" ? params.from.trim() : ""; + const to = typeof params.to === "string" ? params.to.trim() : ""; + + const results: Array<{ + clientId: string; + eventId: string; + timestamp: string; + payload: string; + status: "stored" | "buffered"; + }> = []; + const seen = new Set(); + const corruptPartitions: Array<{ + clientId: string; + filename: string; + error: string; + }> = []; + + const buffered = await DataBufferEngine.getRecentEventsAsync( + limit, + clientId === "all" ? undefined : clientId, + ); + for (const evt of buffered) { + const dedupe = `${evt.clientId}:${evt.eventId}`; + if (seen.has(dedupe)) continue; + if ( + search && + !evt.payload.toLowerCase().includes(search) && + !evt.eventId.toLowerCase().includes(search) && + !evt.clientId.toLowerCase().includes(search) + ) { + continue; + } + seen.add(dedupe); + results.push({ ...evt, status: "buffered" }); + if (results.length >= limit) break; + } + + const storageDir = CSVStorageManager.baseStorageDir; + try { + const clientFolders = await readdir(storageDir); + for (const folder of clientFolders) { + if (results.length >= limit) break; + if ( + clientId !== "all" && + folder !== sanitizedClientId && + folder !== clientId + ) { + continue; + } + const clientDir = path.join(storageDir, folder); + const stat = await lstat(clientDir).catch(() => null); + if (!stat || stat.isSymbolicLink() || !stat.isDirectory()) { + continue; + } + const entries = await readdir(clientDir).catch(() => [] as string[]); + const partitionFiles = entries + .filter((f) => CSVStorageManager.isPartitionFilename(f)) + .sort() + .reverse(); + for (const filename of partitionFiles) { + if (results.length >= limit) break; + const fileDate = CSVStorageManager.extractDateFromFilename(filename); + if (from && fileDate < from) continue; + if (to && fileDate > to) continue; + const fullPath = path.join(clientDir, filename); + const fileLstat = await lstat(fullPath).catch(() => null); + if (!fileLstat || fileLstat.isSymbolicLink() || !fileLstat.isFile()) { + continue; + } + const fileStream = CSVStorageManager.openPartitionTextStream(fullPath); + const parser = fileStream.pipe( + parse({ columns: true, skip_empty_lines: true, bom: true, trim: true }), + ); + fileStream.on("error", (streamErr) => { + if (!parser.destroyed) parser.destroy(streamErr); + }); + try { + for await (const row of parser as AsyncIterable<{ + clientId?: string; + eventId?: string; + timestamp?: string; + payload?: string; + }>) { + if (!row.eventId) continue; + const cid = row.clientId || folder; + const dedupe = `${cid}:${row.eventId}`; + if (seen.has(dedupe)) continue; + const payload = row.payload || "{}"; + if ( + search && + !payload.toLowerCase().includes(search) && + !row.eventId.toLowerCase().includes(search) && + !cid.toLowerCase().includes(search) + ) { + continue; + } + seen.add(dedupe); + results.push({ + clientId: cid, + eventId: row.eventId, + timestamp: row.timestamp || `${fileDate}T00:00:00.000Z`, + payload, + status: "stored", + }); + if (results.length >= limit) break; + } + } catch (fileErr) { + corruptPartitions.push({ + clientId: folder, + filename, + error: + fileErr instanceof Error ? fileErr.message : String(fileErr), + }); + } finally { + if (!parser.destroyed) parser.destroy(); + if (!fileStream.destroyed) fileStream.destroy(); + } + } + } + } catch { + // storage dir may not exist yet + } + + const redisQueueLen = await (async () => { + try { + const redis = DataBufferEngine.getRedisClient(); + return redis ? Number(await redis.llen(REDIS_BUFFER_KEY)) || 0 : 0; + } catch { + return 0; + } + })(); + + return NextResponse.json({ + success: true, + events: results, + count: results.length, + redisQueueLen, + corruptPartitions, + }); + } + + case "storage_stats": { + const [files, diskUsage] = await Promise.all([ + CSVStorageManager.listAllFilesAsync(), + CSVStorageManager.getDiskUsageAsync(), + ]); + const stats = CSVStorageManager.formatStorageStats(files); + return NextResponse.json({ + success: true, + stats, + fileCount: files.length, + diskUsage, + }); + } + + case "storage_flush": { + await SettingsManager.getSettingsAsync(); + const localRes = await DataBufferEngine.flush({ + waitMs: 2500, + throwOnLockTimeout: true, + }); + await recordAdminAuditLogAsync({ + actorEmail: "mcp-agent", + actorRole: "mcp", + action: "STORAGE_FLUSH_TRIGGERED", + target: "storage_buffer", + ipAddress: actorIp, + metadata: { + via: "mcp", + flushedCount: localRes.count, + filesUpdated: localRes.files.length, + }, + }); + return NextResponse.json({ + success: true, + flushedCount: localRes.count, + filesUpdated: localRes.files, + }); + } + + case "storage_s3_archive": { + const settings = await SettingsManager.getSettingsAsync(); + const s3Config = await SettingsManager.getDecryptedS3ConfigAsync(); + if ( + !s3Config.bucket.trim() || + !s3Config.accessKeyId.trim() || + !s3Config.secretAccessKey.trim() + ) { + return NextResponse.json( + { + success: false, + code: "S3_INVALID_CONFIG", + error: + "Configure S3 Bucket, Access Key ID, and Secret Access Key before archiving.", + }, + { status: 400 }, + ); + } + normalizeS3EndpointUrl(s3Config.endpoint, s3Config.region); + const pruneAfterUpload = params.pruneAfterUpload === true; + const archiveRes = + await CSVStorageManager.pruneOrArchiveExpiredPartitionsAsync( + settings.autoPruneDays, + { + archiveOnlyWithoutDelete: !pruneAfterUpload, + beforeDeleteFile: async (file) => { + const key = buildS3ObjectKey( + s3Config.prefix, + file.clientId, + file.filename, + ); + const uploadRes = await uploadFileToS3( + s3Config, + file.filePath, + key, + ); + if (!uploadRes.ok) { + throw new Error( + uploadRes.error || `Failed to upload ${file.filename}`, + ); + } + }, + }, + ); + await recordAdminAuditLogAsync({ + actorEmail: "mcp-agent", + actorRole: "mcp", + action: "S3_ARCHIVE_TRIGGERED", + target: s3Config.bucket, + ipAddress: actorIp, + metadata: { + via: "mcp", + archivedCount: archiveRes.archivedCount, + prunedCount: archiveRes.prunedCount, + pruneAfterUpload, + }, + }); + return NextResponse.json({ success: true, ...archiveRes }); + } + + default: + return NextResponse.json( + { error: `Unknown MCP bridge action '${action}'.`, code: "VALIDATION_ERROR" }, + { status: 400 }, + ); + } + } catch (err: unknown) { + if (err instanceof SSRFBlockedError) { + return NextResponse.json( + { error: err.message, code: err.code }, + { status: 400 }, + ); + } + const message = err instanceof Error ? err.message : "MCP bridge error"; + return NextResponse.json( + { error: message, code: "MCP_BRIDGE_ERROR" }, + { status: 400 }, + ); + } +} diff --git a/core/Database/database.ts b/core/Database/database.ts index 0764371..468a8ed 100644 --- a/core/Database/database.ts +++ b/core/Database/database.ts @@ -109,6 +109,21 @@ export const CORE_SCHEMA_TABLE_DDLS = [ created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ) `, + ` + CREATE TABLE IF NOT EXISTS mcp_settings ( + id TEXT PRIMARY KEY DEFAULT 'default', + enabled BOOLEAN NOT NULL DEFAULT false, + token_hash TEXT NOT NULL DEFAULT '', + token_enc TEXT NOT NULL DEFAULT '', + token_preview TEXT NOT NULL DEFAULT '', + allowed_ips TEXT NOT NULL DEFAULT '', + permissions_json TEXT NOT NULL DEFAULT '{}', + coolify_url TEXT NOT NULL DEFAULT '', + coolify_token_enc TEXT NOT NULL DEFAULT '', + coolify_app_uuid TEXT NOT NULL DEFAULT '', + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ) + `, ] as const; export const CORE_SCHEMA_ALTER_DDLS = [ diff --git a/core/auth/auditLog.ts b/core/auth/auditLog.ts index c374310..e23df8c 100644 --- a/core/auth/auditLog.ts +++ b/core/auth/auditLog.ts @@ -22,6 +22,10 @@ export type AuditActionType = | "WEBHOOK_TEST_TRIGGERED" | "FILES_MANAGER_MUTATION" | "DATA_EXPORT_DOWNLOADED" + | "MCP_SETTINGS_UPDATED" + | "MCP_TOKEN_ROTATED" + | "MCP_TOOL_CALLED" + | "MCP_TOOL_DENIED" | string; export interface AuditLogEntry { @@ -98,6 +102,13 @@ function sanitizeMetadata( "webhookSecretEnc", "prometheusMetricsToken", "prometheusMetricsTokenEnc", + "mcpToken", + "mcpBearerToken", + "issuedToken", + "tokenHash", + "tokenEnc", + "coolifyToken", + "coolifyTokenEnc", ]); const clean: Record = {}; for (const [key, value] of Object.entries(metadata)) { diff --git a/core/logs/runtimeLogCapture.ts b/core/logs/runtimeLogCapture.ts new file mode 100644 index 0000000..d2bc231 --- /dev/null +++ b/core/logs/runtimeLogCapture.ts @@ -0,0 +1,448 @@ +import fs from "node:fs"; +import path from "node:path"; + +const MAX_LOG_FILE_BYTES = 2 * 1024 * 1024; +const DEFAULT_TAIL_LINES = 120; +const MAX_TAIL_LINES = 500; +const MAX_SCAN_TAIL_BYTES = 512 * 1024; + +export interface RuntimeLogEntry { + ts: string; + service: string; + level: string; + message: string; +} + +export interface RuntimeLogReadResult { + entries: RuntimeLogEntry[]; + totalMatched: number; + filesChecked: string[]; + logsDir: string; + exists: boolean; + code?: "LOG_CAPTURE_UNAVAILABLE"; + warning?: string; +} + +export interface BuildLogReadResult { + meta: Record | null; + logFile?: string | null; + lines: string[]; + totalLines: number; + exists: boolean; + code?: "LOG_CAPTURE_UNAVAILABLE"; + warning?: string; +} + +const SECRET_SCRUB_PATTERNS: Array<[RegExp, string]> = [ + [/\b(Bearer\s+)[A-Za-z0-9\-._~+/]+=*/gi, "$1[REDACTED]"], + [/\b(dsmcp_)[A-Za-z0-9_-]{8,}\b/gi, "$1[REDACTED]"], + [ + /\b(eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.)[A-Za-z0-9_-]+\b/g, + "$1[REDACTED]", + ], + [ + /("(?:token|secret|password|jwtSecret|s3SecretAccessKey|webhookSecret|prometheusMetricsToken|coolifyToken|mcpToken|x-datastack-internal-secret)"\s*:\s*")([^"]+)(")/gi, + "$1[REDACTED]$3", + ], +]; + +export function scrubSensitiveLogText(rawText: string): string { + let clean = String(rawText ?? ""); + for (const [pattern, replacement] of SECRET_SCRUB_PATTERNS) { + clean = clean.replace(pattern, replacement); + } + return clean; +} + +export function resolveRuntimeLogsDir( + env: NodeJS.ProcessEnv = process.env, +): string { + const explicitDir = env.DATASTACK_RUNTIME_LOGS_DIR?.trim() || ""; + if (explicitDir) { + return path.resolve(explicitDir); + } + + const storageDir = env.DATASTACK_STORAGE_DIR?.trim() || ""; + if (storageDir) { + const resolved = path.resolve(storageDir); + if (path.basename(resolved).toLowerCase() === "data") { + return path.join(path.dirname(resolved), ".runtime-logs"); + } + return path.join(resolved, ".runtime-logs"); + } + + return path.resolve(process.cwd(), "storage", ".runtime-logs"); +} + +function normalizeServiceName(service: string): "web" | "ingestion" { + const s = String(service || "").trim().toLowerCase(); + if (s === "ingestion" || s === "server" || s === "socket") { + return "ingestion"; + } + return "web"; +} + +function getServiceLogFilePath(service: "web" | "ingestion", logsDir: string): string { + return path.join(logsDir, `${service}.log`); +} + +function rotateLogFileIfNeeded(filePath: string): void { + try { + const stat = fs.statSync(filePath); + if (stat.size >= MAX_LOG_FILE_BYTES) { + const backupPath = `${filePath}.1`; + try { + if (fs.existsSync(backupPath)) { + fs.unlinkSync(backupPath); + } + } catch { + // ignore backup cleanup error + } + fs.renameSync(filePath, backupPath); + } + } catch { + // file does not exist yet + } +} + +export function appendRuntimeLogLine( + service: "web" | "ingestion" | string, + streamName: "stdout" | "stderr", + rawChunk: string, + options?: { logsDir?: string; env?: NodeJS.ProcessEnv }, +): number { + const logsDir = options?.logsDir || resolveRuntimeLogsDir(options?.env); + const safeService = normalizeServiceName(service); + const text = scrubSensitiveLogText(rawChunk); + if (!text) return 0; + + const lines = text.split(/\r?\n/).filter((line) => line.trim().length > 0); + if (lines.length === 0) return 0; + + try { + fs.mkdirSync(logsDir, { recursive: true }); + const filePath = getServiceLogFilePath(safeService, logsDir); + rotateLogFileIfNeeded(filePath); + const level = streamName === "stderr" ? "error" : "info"; + const ts = new Date().toISOString(); + const formatted = + lines + .map((line) => + JSON.stringify({ ts, service: safeService, level, message: line }), + ) + .join("\n") + "\n"; + fs.appendFileSync(filePath, formatted, "utf8"); + return lines.length; + } catch { + return 0; + } +} + +export function installRuntimeLogHook( + service: "web" | "ingestion", + options?: { logsDir?: string; env?: NodeJS.ProcessEnv; force?: boolean }, +): { installed: boolean; reason?: string; service?: string } { + const safeService = normalizeServiceName(service); + if (process.env.DATASTACK_LOG_CAPTURED_BY_RUNNER === "1" && !options?.force) { + return { installed: false, reason: "captured_by_runner" }; + } + + const globals = globalThis as Record; + const guardKey = `__dataStackRuntimeLogHook_${safeService}`; + if (globals[guardKey]) { + return { installed: false, reason: "already_installed" }; + } + globals[guardKey] = true; + + let isWriting = false; + const wrapStream = ( + stream: NodeJS.WriteStream, + streamName: "stdout" | "stderr", + ) => { + const originalWrite = stream.write.bind(stream) as ( + chunk: Uint8Array | string, + encoding?: BufferEncoding, + cb?: (err?: Error | null) => void, + ) => boolean; + + stream.write = (( + chunk: Uint8Array | string, + encodingOrCb?: BufferEncoding | ((err?: Error | null) => void), + cb?: (err?: Error | null) => void, + ): boolean => { + const encoding = + typeof encodingOrCb === "string" ? encodingOrCb : undefined; + const callback = typeof encodingOrCb === "function" ? encodingOrCb : cb; + const result = originalWrite(chunk, encoding, callback); + if (!isWriting) { + isWriting = true; + try { + const raw = + typeof chunk === "string" + ? chunk + : Buffer.isBuffer(chunk) + ? chunk.toString(encoding || "utf8") + : Buffer.from(chunk).toString("utf8"); + appendRuntimeLogLine(safeService, streamName, raw, options); + } catch { + // Never disrupt terminal output + } finally { + isWriting = false; + } + } + return result; + }) as typeof stream.write; + }; + + wrapStream(process.stdout, "stdout"); + wrapStream(process.stderr, "stderr"); + return { installed: true, service: safeService }; +} + +function readTailLinesFromFile( + filePath: string, + maxBytes = MAX_SCAN_TAIL_BYTES, +): string[] { + if (!fs.existsSync(filePath)) return []; + const stat = fs.statSync(filePath); + if (!stat.isFile() || stat.size === 0) return []; + + const readSize = Math.min(stat.size, maxBytes); + const startPos = Math.max(0, stat.size - readSize); + const fd = fs.openSync(filePath, "r"); + try { + const buf = Buffer.allocUnsafe(readSize); + const bytesRead = fs.readSync(fd, buf, 0, readSize, startPos); + let content = buf.subarray(0, bytesRead).toString("utf8"); + if (startPos > 0) { + const firstNewline = content.indexOf("\n"); + content = firstNewline >= 0 ? content.slice(firstNewline + 1) : ""; + } + return content.split(/\r?\n/).filter(Boolean); + } finally { + fs.closeSync(fd); + } +} + +function parseStoredLogLine( + rawLine: string, + fallbackService: string, +): RuntimeLogEntry { + try { + const parsed = JSON.parse(rawLine) as Record; + if (parsed && typeof parsed === "object" && typeof parsed.message === "string") { + return { + ts: typeof parsed.ts === "string" ? parsed.ts : "", + service: + typeof parsed.service === "string" ? parsed.service : fallbackService, + level: typeof parsed.level === "string" ? parsed.level : "info", + message: parsed.message, + }; + } + } catch { + // Plain-text fallback + } + return { + ts: "", + service: fallbackService, + level: "info", + message: rawLine, + }; +} + +export function readRuntimeLogs(options?: { + logsDir?: string; + env?: NodeJS.ProcessEnv; + service?: "web" | "ingestion" | "all"; + lines?: number; + search?: string; + level?: string; +}): RuntimeLogReadResult { + const logsDir = options?.logsDir || resolveRuntimeLogsDir(options?.env); + const serviceFilter = String(options?.service || "all") + .trim() + .toLowerCase(); + const requestedLines = Number(options?.lines); + const limit = Number.isFinite(requestedLines) + ? Math.min(MAX_TAIL_LINES, Math.max(1, Math.floor(requestedLines))) + : DEFAULT_TAIL_LINES; + const search = options?.search?.trim().toLowerCase() || ""; + const levelFilter = options?.level?.trim().toLowerCase() || ""; + + const services: Array<"web" | "ingestion"> = + serviceFilter === "web" + ? ["web"] + : serviceFilter === "ingestion" + ? ["ingestion"] + : ["web", "ingestion"]; + + const collected: RuntimeLogEntry[] = []; + const filesChecked: string[] = []; + + for (const svc of services) { + const mainFile = getServiceLogFilePath(svc, logsDir); + const backupFile = `${mainFile}.1`; + for (const candidate of [backupFile, mainFile]) { + if (fs.existsSync(candidate)) { + filesChecked.push(path.basename(candidate)); + const rawLines = readTailLinesFromFile(candidate); + for (const rawLine of rawLines) { + const entry = parseStoredLogLine(rawLine, svc); + if (levelFilter && levelFilter !== "all" && entry.level !== levelFilter) { + continue; + } + if ( + search && + !entry.message.toLowerCase().includes(search) && + !entry.service.toLowerCase().includes(search) + ) { + continue; + } + collected.push(entry); + } + } + } + } + + if (filesChecked.length === 0) { + return { + entries: [], + totalMatched: 0, + filesChecked: [], + logsDir, + exists: false, + code: "LOG_CAPTURE_UNAVAILABLE", + warning: + "No runtime logs captured yet in .runtime-logs/. Start services or emit stdout logs first.", + }; + } + + collected.sort((a, b) => { + if (a.ts && b.ts && a.ts !== b.ts) { + return a.ts.localeCompare(b.ts); + } + return 0; + }); + + return { + entries: collected.slice(-limit), + totalMatched: collected.length, + filesChecked, + logsDir, + exists: true, + }; +} + +export function readBuildLogs(options?: { + logsDir?: string; + env?: NodeJS.ProcessEnv; + cwd?: string; + lines?: number; + search?: string; +}): BuildLogReadResult { + const logsDir = options?.logsDir || resolveRuntimeLogsDir(options?.env); + const cwd = options?.cwd || process.cwd(); + const requestedLines = Number(options?.lines); + const limit = Number.isFinite(requestedLines) + ? Math.min(MAX_TAIL_LINES, Math.max(1, Math.floor(requestedLines))) + : 200; + const search = options?.search?.trim().toLowerCase() || ""; + + const metaCandidates = [ + path.join(logsDir, "last-build-meta.json"), + path.join(cwd, "build-meta.json"), + path.join(cwd, ".next", "standalone", "build-meta.json"), + ]; + const logCandidates = [ + path.join(logsDir, "last-build.log"), + path.join(cwd, "build.log"), + path.join(cwd, ".next", "standalone", "build.log"), + ]; + + let meta: Record | null = null; + for (const metaPath of metaCandidates) { + try { + if (fs.existsSync(metaPath)) { + meta = JSON.parse(fs.readFileSync(metaPath, "utf8")) as Record< + string, + unknown + >; + break; + } + } catch { + // try next candidate + } + } + + let logPathUsed: string | null = null; + let allLines: string[] = []; + for (const logPath of logCandidates) { + try { + if (fs.existsSync(logPath)) { + logPathUsed = path.basename(logPath); + allLines = readTailLinesFromFile(logPath); + break; + } + } catch { + // try next candidate + } + } + + if (!meta && !logPathUsed) { + return { + meta: null, + lines: [], + totalLines: 0, + exists: false, + code: "LOG_CAPTURE_UNAVAILABLE", + warning: + "No captured build log or build-meta.json found yet. Run npm run build first.", + }; + } + + const filtered = search + ? allLines.filter((line) => line.toLowerCase().includes(search)) + : allLines; + + return { + meta, + logFile: logPathUsed, + lines: filtered.slice(-limit), + totalLines: filtered.length, + exists: true, + }; +} + +export function syncStandaloneBuildArtifactsToLogsDir(options?: { + logsDir?: string; + env?: NodeJS.ProcessEnv; + cwd?: string; +}): { synced: boolean; logsDir?: string } { + const logsDir = options?.logsDir || resolveRuntimeLogsDir(options?.env); + const cwd = options?.cwd || process.cwd(); + const buildLogSrc = [ + path.join(cwd, "build.log"), + path.join(cwd, ".next", "standalone", "build.log"), + ].find((candidate) => fs.existsSync(candidate)); + const buildMetaSrc = [ + path.join(cwd, "build-meta.json"), + path.join(cwd, ".next", "standalone", "build-meta.json"), + ].find((candidate) => fs.existsSync(candidate)); + + if (!buildLogSrc && !buildMetaSrc) { + return { synced: false }; + } + + try { + fs.mkdirSync(logsDir, { recursive: true }); + if (buildLogSrc) { + fs.copyFileSync(buildLogSrc, path.join(logsDir, "last-build.log")); + } + if (buildMetaSrc) { + fs.copyFileSync(buildMetaSrc, path.join(logsDir, "last-build-meta.json")); + } + return { synced: true, logsDir }; + } catch { + return { synced: false }; + } +} diff --git a/core/mcp/mcpSettings.ts b/core/mcp/mcpSettings.ts new file mode 100644 index 0000000..9a8805f --- /dev/null +++ b/core/mcp/mcpSettings.ts @@ -0,0 +1,672 @@ +import { createHash, randomBytes, timingSafeEqual } from "node:crypto"; +import { DatabaseEngine } from "../Database/database"; +import { getInternalIngestionHeaders } from "../Security/security"; +import { + decryptSecret, + encryptSecret, + getMaskedHintFromEncrypted, +} from "../Security/secretVault"; +import { SSRFBlockedError, validateOutboundUrlSync } from "../Security/ssrfGuard"; +import { validateAllowedIpsInput } from "../Security/ipAllowlist"; + +export type McpPermissionKey = + | "metrics_health_read" + | "runtime_logs_read" + | "build_logs_read" + | "audit_logs_read" + | "ingestion_events_read" + | "clients_read" + | "clients_create" + | "clients_update" + | "clients_revoke_delete" + | "files_list_read" + | "files_download" + | "files_upload_create" + | "files_edit" + | "files_delete" + | "settings_read" + | "settings_write" + | "storage_maintenance"; + +export type McpPermissions = Record; + +export interface McpPermissionDefinition { + key: McpPermissionKey; + label: string; + description: string; + category: + | "Observability & Logs" + | "Clients & SDKs" + | "Files & Storage" + | "Admin & Maintenance"; + defaultEnabled: boolean; + risk: "read" | "write" | "destructive"; +} + +export const MCP_PERMISSION_DEFINITIONS: readonly McpPermissionDefinition[] = [ + { + key: "metrics_health_read", + label: "Dashboard Metrics & Health", + description: "Inspect dashboard metrics, Redis/PG health, buffer depth, and storage usage.", + category: "Observability & Logs", + defaultEnabled: true, + risk: "read", + }, + { + key: "runtime_logs_read", + label: "Runtime Logs (Web & Ingestion)", + description: "Read rotating Next.js (Port 3000) and Ingestion (Port 4000) stdout/stderr logs.", + category: "Observability & Logs", + defaultEnabled: true, + risk: "read", + }, + { + key: "build_logs_read", + label: "Build & Coolify Logs", + description: "Read captured Next.js build manifest/logs and optional Coolify deploy logs.", + category: "Observability & Logs", + defaultEnabled: true, + risk: "read", + }, + { + key: "audit_logs_read", + label: "Admin Audit Logs", + description: "Search administrative, RBAC, client, and MCP action history.", + category: "Observability & Logs", + defaultEnabled: true, + risk: "read", + }, + { + key: "ingestion_events_read", + label: "Telemetry Events Read", + description: "Query buffered recent events and stored CSV/GZ telemetry partitions.", + category: "Observability & Logs", + defaultEnabled: true, + risk: "read", + }, + { + key: "clients_read", + label: "List & Inspect Clients", + description: "View registered SDK client IDs, quotas, allowlists, and token previews.", + category: "Clients & SDKs", + defaultEnabled: true, + risk: "read", + }, + { + key: "clients_create", + label: "Create SDK Clients", + description: "Register new SDK client credentials from an AI agent session.", + category: "Clients & SDKs", + defaultEnabled: false, + risk: "write", + }, + { + key: "clients_update", + label: "Update SDK Clients", + description: "Modify client names, rate limits, storage quotas, allowed IPs, or expiry.", + category: "Clients & SDKs", + defaultEnabled: false, + risk: "write", + }, + { + key: "clients_revoke_delete", + label: "Rotate, Revoke & Delete Clients", + description: "Regenerate client JWTs, revoke active client tokens, or permanently delete clients.", + category: "Clients & SDKs", + defaultEnabled: false, + risk: "destructive", + }, + { + key: "files_list_read", + label: "Browse & Read Storage Files", + description: "List directories and inspect bounded lines/bytes strictly inside /app/storage.", + category: "Files & Storage", + defaultEnabled: true, + risk: "read", + }, + { + key: "files_download", + label: "Download Storage File Chunks", + description: "Fetch bounded text or base64 file chunks inside /app/storage.", + category: "Files & Storage", + defaultEnabled: true, + risk: "read", + }, + { + key: "files_upload_create", + label: "Upload & Create Files", + description: "Upload or create new files inside /app/storage (respecting .lock protection).", + category: "Files & Storage", + defaultEnabled: false, + risk: "write", + }, + { + key: "files_edit", + label: "Edit Storage Files", + description: "Perform targeted search/replace edits on non-locked files inside /app/storage.", + category: "Files & Storage", + defaultEnabled: false, + risk: "write", + }, + { + key: "files_delete", + label: "Delete Storage Files", + description: "Delete non-locked files inside /app/storage.", + category: "Files & Storage", + defaultEnabled: false, + risk: "destructive", + }, + { + key: "settings_read", + label: "Read System Settings", + description: "Inspect masked public runtime settings (raw secrets are always hidden).", + category: "Admin & Maintenance", + defaultEnabled: true, + risk: "read", + }, + { + key: "settings_write", + label: "Update System Settings", + description: "Modify ingestion/storage thresholds (SSRF-guarded; never touches admin users or JWT keys).", + category: "Admin & Maintenance", + defaultEnabled: false, + risk: "destructive", + }, + { + key: "storage_maintenance", + label: "Buffer Flush & S3 Archive", + description: "Trigger manual buffer flushes to CSV or on-demand S3 cold backups.", + category: "Admin & Maintenance", + defaultEnabled: false, + risk: "write", + }, +] as const; + +export interface McpSettingsRecord { + enabled: boolean; + tokenHash: string; + tokenEnc: string; + tokenPreview: string; + allowedIps: string; + permissions: McpPermissions; + coolifyUrl: string; + coolifyTokenEnc: string; + coolifyAppUuid: string; + updatedAt: string; +} + +export interface PublicMcpSettings { + enabled: boolean; + hasToken: boolean; + tokenPreview: string; + allowedIps: string; + permissions: McpPermissions; + coolifyUrl: string; + hasCoolifyToken: boolean; + coolifyTokenMask: string; + coolifyAppUuid: string; + endpointPath: string; + updatedAt: string; +} + +export interface McpBridgeSyncResult { + attempted: boolean; + ok: boolean; + synced: boolean; + status?: number; + error?: string; + checkedAt: string; +} + +export type McpSettingsUpdateInput = { + enabled?: boolean; + allowedIps?: string; + permissions?: Partial>; + rotateToken?: boolean; + revokeToken?: boolean; + coolifyUrl?: string; + coolifyToken?: string; + clearCoolifyToken?: boolean; + coolifyAppUuid?: string; +}; + +type McpGlobals = typeof globalThis & { + __dataStackMcpSettings?: McpSettingsRecord; + __dataStackLastMcpBridgeSync?: McpBridgeSyncResult; +}; + +const globals = globalThis as McpGlobals; + +export function getDefaultMcpPermissions(): McpPermissions { + const out = {} as McpPermissions; + for (const def of MCP_PERMISSION_DEFINITIONS) { + out[def.key] = def.defaultEnabled; + } + return out; +} + +export function normalizeMcpPermissions( + raw: unknown, + base: McpPermissions = getDefaultMcpPermissions(), +): McpPermissions { + const merged: McpPermissions = { ...base }; + if (!raw || typeof raw !== "object" || Array.isArray(raw)) { + return merged; + } + const source = raw as Record; + for (const def of MCP_PERMISSION_DEFINITIONS) { + if (typeof source[def.key] === "boolean") { + merged[def.key] = source[def.key] as boolean; + } + } + return merged; +} + +export function getDefaultMcpSettings(): McpSettingsRecord { + return { + enabled: false, + tokenHash: "", + tokenEnc: "", + tokenPreview: "", + allowedIps: "", + permissions: getDefaultMcpPermissions(), + coolifyUrl: "", + coolifyTokenEnc: "", + coolifyAppUuid: "", + updatedAt: new Date().toISOString(), + }; +} + +export function generateMcpBearerToken(): string { + return `dsmcp_${randomBytes(32).toString("hex")}`; +} + +export function hashMcpToken(rawToken: string): string { + return createHash("sha256").update(String(rawToken || ""), "utf8").digest("hex"); +} + +export function formatMcpTokenPreview(rawToken: string): string { + const clean = String(rawToken || "").trim(); + if (!clean) return ""; + if (clean.length <= 14) return `${clean.slice(0, 4)}...`; + return `${clean.slice(0, 10)}...${clean.slice(-4)}`; +} + +export function verifyMcpBearerTokenHash( + presentedToken: string, + expectedHashHex: string, +): boolean { + const cleanPresented = String(presentedToken || "").trim(); + const cleanExpected = String(expectedHashHex || "").trim().toLowerCase(); + if (!cleanPresented || cleanExpected.length !== 64) { + return false; + } + try { + const presentedBuf = Buffer.from(hashMcpToken(cleanPresented), "hex"); + const expectedBuf = Buffer.from(cleanExpected, "hex"); + if (presentedBuf.length !== expectedBuf.length) { + return false; + } + return timingSafeEqual(presentedBuf, expectedBuf); + } catch { + return false; + } +} + +function getMcpState(): McpSettingsRecord { + if (!globals.__dataStackMcpSettings) { + globals.__dataStackMcpSettings = getDefaultMcpSettings(); + } + return globals.__dataStackMcpSettings; +} + +export function toPublicMcpSettings(record: McpSettingsRecord): PublicMcpSettings { + return { + enabled: Boolean(record.enabled), + hasToken: Boolean(record.tokenHash), + tokenPreview: record.tokenPreview || "", + allowedIps: record.allowedIps || "", + permissions: normalizeMcpPermissions(record.permissions), + coolifyUrl: record.coolifyUrl || "", + hasCoolifyToken: Boolean(record.coolifyTokenEnc), + coolifyTokenMask: getMaskedHintFromEncrypted(record.coolifyTokenEnc), + coolifyAppUuid: record.coolifyAppUuid || "", + endpointPath: "/mcp", + updatedAt: record.updatedAt || new Date().toISOString(), + }; +} + +export class McpSettingsManager { + public static getSettings(): McpSettingsRecord { + const current = getMcpState(); + return { + ...current, + permissions: { ...current.permissions }, + }; + } + + public static async getSettingsAsync(): Promise { + const pool = DatabaseEngine.getPool(); + if (!pool) { + return this.getSettings(); + } + + try { + await DatabaseEngine.ensureSchema(); + const res = await pool.query<{ + enabled: boolean; + token_hash: string; + token_enc: string; + token_preview: string; + allowed_ips: string; + permissions_json: string; + coolify_url: string; + coolify_token_enc: string; + coolify_app_uuid: string; + updated_at: Date | string; + }>( + `SELECT enabled, token_hash, token_enc, token_preview, allowed_ips, + permissions_json, coolify_url, coolify_token_enc, coolify_app_uuid, updated_at + FROM mcp_settings + WHERE id = 'default'`, + ); + + if (!res.rowCount || res.rowCount === 0) { + return this.getSettings(); + } + + const row = res.rows[0]; + let parsedPermissions: unknown = {}; + try { + parsedPermissions = JSON.parse(row.permissions_json || "{}"); + } catch { + parsedPermissions = {}; + } + + const hydrated: McpSettingsRecord = { + enabled: Boolean(row.enabled), + tokenHash: String(row.token_hash || ""), + tokenEnc: String(row.token_enc || ""), + tokenPreview: String(row.token_preview || ""), + allowedIps: String(row.allowed_ips || ""), + permissions: normalizeMcpPermissions(parsedPermissions), + coolifyUrl: String(row.coolify_url || ""), + coolifyTokenEnc: String(row.coolify_token_enc || ""), + coolifyAppUuid: String(row.coolify_app_uuid || ""), + updatedAt: + row.updated_at instanceof Date + ? row.updated_at.toISOString() + : String(row.updated_at || new Date().toISOString()), + }; + globals.__dataStackMcpSettings = hydrated; + return { ...hydrated, permissions: { ...hydrated.permissions } }; + } catch (err) { + console.warn( + "[DataStack MCP] Failed to read mcp_settings from PostgreSQL (using cached/default state):", + err instanceof Error ? err.message : err, + ); + return this.getSettings(); + } + } + + public static async getPublicSettingsAsync(): Promise { + const settings = await this.getSettingsAsync(); + return toPublicMcpSettings(settings); + } + + public static getLastBridgeSyncStatus(): McpBridgeSyncResult | null { + return globals.__dataStackLastMcpBridgeSync ?? null; + } + + private static async persistToDb(record: McpSettingsRecord): Promise { + const pool = DatabaseEngine.getPool(); + if (!pool) return; + + const schemaOk = await DatabaseEngine.ensureSchema(); + if (!schemaOk) { + throw new Error( + "DATABASE_WRITE_FAILED: Unable to verify PostgreSQL schema for mcp_settings.", + ); + } + + await pool.query( + ` + INSERT INTO mcp_settings ( + id, enabled, token_hash, token_enc, token_preview, + allowed_ips, permissions_json, coolify_url, coolify_token_enc, + coolify_app_uuid, updated_at + ) + VALUES ('default', $1, $2, $3, $4, $5, $6, $7, $8, $9, $10::timestamptz) + ON CONFLICT (id) DO UPDATE SET + enabled = EXCLUDED.enabled, + token_hash = EXCLUDED.token_hash, + token_enc = EXCLUDED.token_enc, + token_preview = EXCLUDED.token_preview, + allowed_ips = EXCLUDED.allowed_ips, + permissions_json = EXCLUDED.permissions_json, + coolify_url = EXCLUDED.coolify_url, + coolify_token_enc = EXCLUDED.coolify_token_enc, + coolify_app_uuid = EXCLUDED.coolify_app_uuid, + updated_at = EXCLUDED.updated_at + `, + [ + record.enabled, + record.tokenHash, + record.tokenEnc, + record.tokenPreview, + record.allowedIps, + JSON.stringify(record.permissions), + record.coolifyUrl, + record.coolifyTokenEnc, + record.coolifyAppUuid, + record.updatedAt, + ], + ); + } + + private static async notifyIngestionMcpSync( + record: McpSettingsRecord, + tokenRotated: boolean, + ): Promise { + const checkedAt = new Date().toISOString(); + const configuredUrl = process.env.INGESTION_INTERNAL_URL?.trim(); + if (process.env.DATASTACK_DISABLE_LISTEN === "1" && !configuredUrl) { + const skipped: McpBridgeSyncResult = { + attempted: false, + ok: true, + synced: false, + checkedAt, + }; + globals.__dataStackLastMcpBridgeSync = skipped; + return skipped; + } + + const headers = getInternalIngestionHeaders(); + if (!headers["x-datastack-internal-secret"]) { + const result: McpBridgeSyncResult = { + attempted: true, + ok: false, + synced: false, + error: "Internal bridge secret is not configured", + checkedAt, + }; + globals.__dataStackLastMcpBridgeSync = result; + return result; + } + + const ingestionBaseUrl = ( + configuredUrl || "http://127.0.0.1:4000" + ).replace(/\/$/, ""); + + let coolifyTokenPlain = ""; + if (record.coolifyTokenEnc) { + try { + coolifyTokenPlain = decryptSecret(record.coolifyTokenEnc); + } catch { + coolifyTokenPlain = ""; + } + } + + try { + const res = await fetch(`${ingestionBaseUrl}/internal/mcp-sync`, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...headers, + }, + body: JSON.stringify({ + enabled: record.enabled, + tokenHash: record.tokenHash, + tokenPreview: record.tokenPreview, + allowedIps: record.allowedIps, + permissions: record.permissions, + coolifyUrl: record.coolifyUrl, + coolifyToken: coolifyTokenPlain, + coolifyAppUuid: record.coolifyAppUuid, + updatedAt: record.updatedAt, + tokenRotated, + }), + cache: "no-store", + signal: AbortSignal.timeout(1500), + }); + const result: McpBridgeSyncResult = { + attempted: true, + ok: res.ok, + synced: res.ok, + status: res.status, + ...(res.ok ? {} : { error: `Port 4000 mcp-sync returned HTTP ${res.status}` }), + checkedAt, + }; + globals.__dataStackLastMcpBridgeSync = result; + return result; + } catch (err: unknown) { + const result: McpBridgeSyncResult = { + attempted: true, + ok: false, + synced: false, + error: err instanceof Error ? err.message : "Port 4000 unreachable", + checkedAt, + }; + globals.__dataStackLastMcpBridgeSync = result; + return result; + } + } + + public static async updateSettingsAsync( + updates: McpSettingsUpdateInput, + ): Promise<{ + settings: PublicMcpSettings; + issuedToken?: string; + tokenRotated: boolean; + bridgeSync: McpBridgeSyncResult; + }> { + const current = await this.getSettingsAsync(); + let nextEnabled = + typeof updates.enabled === "boolean" ? updates.enabled : current.enabled; + let nextAllowedIps = current.allowedIps; + + if (updates.allowedIps !== undefined) { + const ipCheck = validateAllowedIpsInput(updates.allowedIps); + if (!ipCheck.valid) { + throw new Error(ipCheck.error || "Invalid allowedIps list."); + } + nextAllowedIps = ipCheck.normalized; + } + + let nextCoolifyUrl = current.coolifyUrl; + if (updates.coolifyUrl !== undefined) { + const trimmed = String(updates.coolifyUrl || "").trim(); + if (trimmed) { + const ssrf = validateOutboundUrlSync(trimmed, { + fieldName: "coolifyUrl", + }); + if (!ssrf.allowed) { + if (ssrf.code === "SSRF_BLOCKED_HOST") { + throw new SSRFBlockedError( + ssrf.error || "coolifyUrl target host is blocked.", + ssrf.hostname || "", + ); + } + throw new Error(ssrf.error || "Invalid coolifyUrl."); + } + } + nextCoolifyUrl = trimmed; + } + + const nextCoolifyAppUuid = + updates.coolifyAppUuid !== undefined + ? String(updates.coolifyAppUuid || "").trim().slice(0, 128) + : current.coolifyAppUuid; + + let nextCoolifyTokenEnc = current.coolifyTokenEnc; + if (updates.clearCoolifyToken) { + nextCoolifyTokenEnc = ""; + } else if ( + typeof updates.coolifyToken === "string" && + updates.coolifyToken.trim().length > 0 + ) { + nextCoolifyTokenEnc = encryptSecret(updates.coolifyToken.trim()); + } + + const nextPermissions = normalizeMcpPermissions( + updates.permissions, + current.permissions, + ); + + let nextTokenHash = current.tokenHash; + let nextTokenEnc = current.tokenEnc; + let nextTokenPreview = current.tokenPreview; + let issuedToken: string | undefined; + let tokenRotated = false; + + if (updates.revokeToken) { + nextTokenHash = ""; + nextTokenEnc = ""; + nextTokenPreview = ""; + nextEnabled = false; + tokenRotated = true; + } else if ( + updates.rotateToken || + (nextEnabled && !current.tokenHash) + ) { + const freshToken = generateMcpBearerToken(); + issuedToken = freshToken; + nextTokenHash = hashMcpToken(freshToken); + nextTokenEnc = encryptSecret(freshToken); + nextTokenPreview = formatMcpTokenPreview(freshToken); + tokenRotated = true; + } + + const nextRecord: McpSettingsRecord = { + enabled: nextEnabled, + tokenHash: nextTokenHash, + tokenEnc: nextTokenEnc, + tokenPreview: nextTokenPreview, + allowedIps: nextAllowedIps, + permissions: nextPermissions, + coolifyUrl: nextCoolifyUrl, + coolifyTokenEnc: nextCoolifyTokenEnc, + coolifyAppUuid: nextCoolifyAppUuid, + updatedAt: new Date().toISOString(), + }; + + await this.persistToDb(nextRecord); + globals.__dataStackMcpSettings = nextRecord; + const bridgeSync = await this.notifyIngestionMcpSync( + nextRecord, + tokenRotated || updates.enabled === false, + ); + + return { + settings: toPublicMcpSettings(nextRecord), + ...(issuedToken ? { issuedToken } : {}), + tokenRotated, + bridgeSync, + }; + } + + public static resetForTests(): void { + delete globals.__dataStackMcpSettings; + delete globals.__dataStackLastMcpBridgeSync; + } +} diff --git a/docker/Dockerfile.web b/docker/Dockerfile.web index f19a099..7a3609a 100644 --- a/docker/Dockerfile.web +++ b/docker/Dockerfile.web @@ -21,10 +21,10 @@ ENV NEXT_TELEMETRY_DISABLED=1 ENV PORT=3000 ENV HOSTNAME=0.0.0.0 RUN apk add --no-cache su-exec \ - && mkdir -p /app/storage/data \ + && mkdir -p /app/storage/data /app/storage/.runtime-logs \ && chown -R node:node /app/storage COPY --from=builder --chown=node:node /app/public ./public COPY --from=builder --chown=node:node /app/.next/standalone ./ COPY --from=builder --chown=node:node /app/.next/static ./.next/static EXPOSE 3000 -CMD ["sh", "-c", "mkdir -p /app/storage/data && NODE_UG=\"$(id -u node):$(id -g node)\" && if [ \"$(stat -c '%u:%g' /app/storage/data)\" != \"$NODE_UG\" ]; then chown -R node:node /app/storage; else chown node:node /app/storage /app/storage/data; fi && exec su-exec node:node node server.js"] +CMD ["sh", "-c", "mkdir -p /app/storage/data /app/storage/.runtime-logs && NODE_UG=\"$(id -u node):$(id -g node)\" && if [ \"$(stat -c '%u:%g' /app/storage/data)\" != \"$NODE_UG\" ]; then chown -R node:node /app/storage; else chown node:node /app/storage /app/storage/data /app/storage/.runtime-logs; fi && exec su-exec node:node node server.js"] diff --git a/instrumentation.ts b/instrumentation.ts index 75adca6..426c292 100644 --- a/instrumentation.ts +++ b/instrumentation.ts @@ -4,6 +4,17 @@ export async function register(): Promise { return; } + try { + const { + installRuntimeLogHook, + syncStandaloneBuildArtifactsToLogsDir, + } = await import("@/core/logs/runtimeLogCapture"); + installRuntimeLogHook("web"); + syncStandaloneBuildArtifactsToLogsDir(); + } catch { + // Non-fatal if runtime log hook cannot initialize + } + try { const { SettingsManager } = await import( "@/core/adminSettings/settingsManager" diff --git a/middleware.ts b/middleware.ts index 8a7618b..51b0e99 100644 --- a/middleware.ts +++ b/middleware.ts @@ -7,12 +7,15 @@ import { NextRequest, NextResponse } from "next/server"; const SESSION_COOKIE = "datastack_admin_session"; const SAFE_METHODS = new Set(["GET", "HEAD", "OPTIONS"]); -function isProtectedApi(pathname: string, method: string) { +function isProtectedApi(pathname: string, method: string, hasInternalSecretHeader: boolean) { if (pathname === "/api/auth/login") return false; if (pathname === "/api/auth/setup" && method === "POST") return false; if (pathname === "/api/events" && method === "POST") return false; if (pathname === "/api/health" && method === "GET") return false; if (pathname === "/api/prometheus" && method === "GET") return false; + if (pathname === "/api/internal/mcp" && method === "POST" && hasInternalSecretHeader) { + return false; + } return pathname.startsWith("/api/"); } @@ -62,14 +65,26 @@ export async function middleware(request: NextRequest) { (pathname === "/vcon" || pathname.startsWith("/vcon/")) && !isLoginPage && !isSetupPage; - const isAdminApi = isProtectedApi(pathname, request.method); + const hasInternalSecretHeader = Boolean( + request.headers.get("x-datastack-internal-secret")?.trim(), + ); + const isAdminApi = isProtectedApi( + pathname, + request.method, + hasInternalSecretHeader, + ); const hasBearerAuth = Boolean( request.headers.get("authorization")?.toLowerCase().startsWith("bearer "), ); const isStateChangingApi = pathname.startsWith("/api/") && !SAFE_METHODS.has(request.method.toUpperCase()) && - !(pathname === "/api/events" && hasBearerAuth); + !(pathname === "/api/events" && hasBearerAuth) && + !( + pathname === "/api/internal/mcp" && + request.method.toUpperCase() === "POST" && + hasInternalSecretHeader + ); const requestHeaders = new Headers(request.headers); requestHeaders.set("x-datastack-pathname", pathname); diff --git a/package.json b/package.json index 407e388..7563423 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "dev:web": "next dev -p 3000 -H 0.0.0.0", "dev:all": "node scripts/dev-all.js", "server": "node server/index.js", - "build": "NODE_ENV=production next build", + "build": "node scripts/capture-build-log.js", "start": "node scripts/start-all.js", "start:web": "next start", "start:server": "node server/index.js", diff --git a/scripts/capture-build-log.js b/scripts/capture-build-log.js new file mode 100644 index 0000000..488d440 --- /dev/null +++ b/scripts/capture-build-log.js @@ -0,0 +1,95 @@ +#!/usr/bin/env node +/** + * Captures Next.js production build output & metadata into `.next/standalone/build.log` + * and `.next/standalone/build-meta.json` (plus runtime logs dir when writable) so + * build diagnostics are inspectable via MCP `read_build_logs` inside running containers. + */ + +const fs = require('node:fs'); +const path = require('node:path'); +const { spawn } = require('node:child_process'); +const { resolveRuntimeLogsDir, scrubSensitiveLogText } = require('../server/runtimeLogCapture'); + +const MAX_BUILD_LOG_BYTES = 512 * 1024; +const nextBin = require.resolve('next/dist/bin/next'); +const pkg = require(path.resolve(__dirname, '..', 'package.json')); + +const startedAt = new Date().toISOString(); +const startMs = Date.now(); +const capturedChunks = []; +let capturedBytes = 0; + +function recordChunk(text) { + const clean = scrubSensitiveLogText(text); + capturedChunks.push(clean); + capturedBytes += Buffer.byteLength(clean, 'utf8'); + while (capturedBytes > MAX_BUILD_LOG_BYTES && capturedChunks.length > 1) { + const dropped = capturedChunks.shift(); + capturedBytes -= Buffer.byteLength(dropped, 'utf8'); + } +} + +const childEnv = { + ...process.env, + NODE_ENV: 'production', +}; + +const child = spawn(process.execPath, [nextBin, 'build'], { + stdio: ['inherit', 'pipe', 'pipe'], + shell: false, + env: childEnv, +}); + +child.stdout.on('data', (buf) => { + process.stdout.write(buf); + recordChunk(buf.toString('utf8')); +}); + +child.stderr.on('data', (buf) => { + process.stderr.write(buf); + recordChunk(buf.toString('utf8')); +}); + +child.on('error', (err) => { + console.error('[DataStack Build] Failed to spawn Next.js build:', err.message); + process.exit(1); +}); + +child.on('close', (code) => { + const exitCode = typeof code === 'number' ? code : 1; + const finishedAt = new Date().toISOString(); + const durationMs = Date.now() - startMs; + + const meta = { + appName: pkg.name || 'datastack', + appVersion: pkg.version || '0.1.0', + nodeVersion: process.version, + nextVersion: (pkg.dependencies && pkg.dependencies.next) || 'unknown', + startedAt, + finishedAt, + durationMs, + exitCode, + status: exitCode === 0 ? 'succeeded' : 'failed', + }; + + const fullLog = capturedChunks.join(''); + const writeArtifactDir = (dir, logName, metaName) => { + try { + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, logName), fullLog, 'utf8'); + fs.writeFileSync(path.join(dir, metaName), JSON.stringify(meta, null, 2) + '\n', 'utf8'); + } catch { + // Non-fatal if target dir is read-only or unavailable + } + }; + + const standaloneDir = path.resolve(process.cwd(), '.next', 'standalone'); + if (exitCode === 0 && fs.existsSync(standaloneDir)) { + writeArtifactDir(standaloneDir, 'build.log', 'build-meta.json'); + } + + const runtimeLogsDir = resolveRuntimeLogsDir(process.env); + writeArtifactDir(runtimeLogsDir, 'last-build.log', 'last-build-meta.json'); + + process.exit(exitCode); +}); diff --git a/scripts/dev-all.js b/scripts/dev-all.js index e8f9ed9..6be14aa 100644 --- a/scripts/dev-all.js +++ b/scripts/dev-all.js @@ -7,6 +7,7 @@ const path = require('node:path'); const { spawn } = require('node:child_process'); +const { appendRuntimeLogLine } = require('../server/runtimeLogCapture'); const webPort = String(process.env.PORT || '3000'); const nextBin = require.resolve('next/dist/bin/next'); @@ -19,18 +20,41 @@ console.log('\x1b[36m%s\x1b[0m', ` - Port ${webPort}: Next.js 15 Web & Admin Co console.log('\x1b[36m%s\x1b[0m', ' - Port 4000: Ingestion & Socket.IO Streaming Engine'); console.log('\x1b[36m%s\x1b[0m', '=======================================================\n'); +const childEnv = { + ...process.env, + DATASTACK_LOG_CAPTURED_BY_RUNNER: '1', +}; + const nextProc = spawn(process.execPath, [nextBin, 'dev', '-p', webPort, '-H', '0.0.0.0'], { - stdio: 'inherit', + stdio: ['inherit', 'pipe', 'pipe'], shell: false, - env: { ...process.env }, + env: childEnv, }); const serverProc = spawn(process.execPath, [serverEntry], { - stdio: 'inherit', + stdio: ['inherit', 'pipe', 'pipe'], shell: false, - env: { ...process.env }, + env: childEnv, }); +function wireStreamCapture(proc, serviceName) { + if (proc.stdout) { + proc.stdout.on('data', (chunk) => { + process.stdout.write(chunk); + appendRuntimeLogLine(serviceName, 'stdout', chunk.toString('utf8')); + }); + } + if (proc.stderr) { + proc.stderr.on('data', (chunk) => { + process.stderr.write(chunk); + appendRuntimeLogLine(serviceName, 'stderr', chunk.toString('utf8')); + }); + } +} + +wireStreamCapture(nextProc, 'web'); +wireStreamCapture(serverProc, 'ingestion'); + let shuttingDown = false; let exitedChildren = 0; let nextExited = false; diff --git a/scripts/start-all.js b/scripts/start-all.js index 4da653b..393f986 100644 --- a/scripts/start-all.js +++ b/scripts/start-all.js @@ -7,28 +7,52 @@ const path = require('node:path'); const { spawn } = require('node:child_process'); +const { appendRuntimeLogLine } = require('../server/runtimeLogCapture'); const webPort = String(process.env.PORT || '3000'); const nextBin = require.resolve('next/dist/bin/next'); const serverEntry = path.resolve(__dirname, '..', 'server', 'index.js'); const GRACEFUL_SHUTDOWN_TIMEOUT_MS = 7000; +const childEnv = { + ...process.env, + DATASTACK_LOG_CAPTURED_BY_RUNNER: '1', +}; + const webProc = spawn( process.execPath, [nextBin, 'start', '-p', webPort, '-H', '0.0.0.0'], { - stdio: 'inherit', + stdio: ['inherit', 'pipe', 'pipe'], shell: false, - env: { ...process.env }, + env: childEnv, }, ); const serverProc = spawn(process.execPath, [serverEntry], { - stdio: 'inherit', + stdio: ['inherit', 'pipe', 'pipe'], shell: false, - env: { ...process.env }, + env: childEnv, }); +function wireStreamCapture(proc, serviceName) { + if (proc.stdout) { + proc.stdout.on('data', (chunk) => { + process.stdout.write(chunk); + appendRuntimeLogLine(serviceName, 'stdout', chunk.toString('utf8')); + }); + } + if (proc.stderr) { + proc.stderr.on('data', (chunk) => { + process.stderr.write(chunk); + appendRuntimeLogLine(serviceName, 'stderr', chunk.toString('utf8')); + }); + } +} + +wireStreamCapture(webProc, 'web'); +wireStreamCapture(serverProc, 'ingestion'); + let shuttingDown = false; let exitedChildren = 0; let webExited = false; diff --git a/server/index.js b/server/index.js index 149bc30..88d00a6 100644 --- a/server/index.js +++ b/server/index.js @@ -14,6 +14,7 @@ const { syncInMemoryAdminSessionState, clearInMemoryAdminSessionStateForTests, } = require('./filesManager'); +const { installRuntimeLogHook } = require('./runtimeLogCapture'); function parseAllowedCorsOrigins(raw = process.env.DATASTACK_CORS_ORIGINS) { if (typeof raw !== 'string') return null; @@ -1735,6 +1736,21 @@ async function ensureDatabaseSchema() { created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ) `); + await pool.query(` + CREATE TABLE IF NOT EXISTS mcp_settings ( + id TEXT PRIMARY KEY DEFAULT 'default', + enabled BOOLEAN NOT NULL DEFAULT false, + token_hash TEXT NOT NULL DEFAULT '', + token_enc TEXT NOT NULL DEFAULT '', + token_preview TEXT NOT NULL DEFAULT '', + allowed_ips TEXT NOT NULL DEFAULT '', + permissions_json TEXT NOT NULL DEFAULT '{}', + coolify_url TEXT NOT NULL DEFAULT '', + coolify_token_enc TEXT NOT NULL DEFAULT '', + coolify_app_uuid TEXT NOT NULL DEFAULT '', + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ) + `); await pool.query(` ALTER TABLE client_sdks ADD COLUMN IF NOT EXISTS rate_limit INTEGER NOT NULL DEFAULT 100, @@ -2393,6 +2409,9 @@ async function gracefulShutdown(signal, options = {}) { } async function startIngestionServer() { + if (process.env.DATASTACK_DISABLE_LISTEN !== '1') { + installRuntimeLogHook('ingestion'); + } if (dbPool) { await Promise.race([ syncSettingsFromDatabase().catch(() => false), diff --git a/server/runtimeLogCapture.js b/server/runtimeLogCapture.js new file mode 100644 index 0000000..e3ffc67 --- /dev/null +++ b/server/runtimeLogCapture.js @@ -0,0 +1,371 @@ +const fs = require('node:fs'); +const path = require('node:path'); + +const MAX_LOG_FILE_BYTES = 2 * 1024 * 1024; // 2 MB per service file (+ .1 backup) +const DEFAULT_TAIL_LINES = 120; +const MAX_TAIL_LINES = 500; +const MAX_SCAN_TAIL_BYTES = 512 * 1024; // Bound in-memory file slice reads + +const SECRET_SCRUB_PATTERNS = [ + [/\b(Bearer\s+)[A-Za-z0-9\-._~+/]+=*/gi, '$1[REDACTED]'], + [/\b(dsmcp_)[A-Za-z0-9_-]{8,}\b/gi, '$1[REDACTED]'], + [/\b(eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.)[A-Za-z0-9_-]+\b/g, '$1[REDACTED]'], + [/("(?:token|secret|password|jwtSecret|s3SecretAccessKey|webhookSecret|prometheusMetricsToken|coolifyToken|mcpToken|x-datastack-internal-secret)"\s*:\s*")([^"]+)(")/gi, '$1[REDACTED]$3'], +]; + +function scrubSensitiveLogText(rawText) { + let clean = String(rawText ?? ''); + for (const [pattern, replacement] of SECRET_SCRUB_PATTERNS) { + clean = clean.replace(pattern, replacement); + } + return clean; +} + +function resolveRuntimeLogsDir(env = process.env) { + const explicitDir = typeof env.DATASTACK_RUNTIME_LOGS_DIR === 'string' + ? env.DATASTACK_RUNTIME_LOGS_DIR.trim() + : ''; + if (explicitDir) { + return path.resolve(explicitDir); + } + + const storageDir = typeof env.DATASTACK_STORAGE_DIR === 'string' + ? env.DATASTACK_STORAGE_DIR.trim() + : ''; + if (storageDir) { + const resolved = path.resolve(storageDir); + if (path.basename(resolved).toLowerCase() === 'data') { + return path.join(path.dirname(resolved), '.runtime-logs'); + } + return path.join(resolved, '.runtime-logs'); + } + + return path.resolve(process.cwd(), 'storage', '.runtime-logs'); +} + +function normalizeServiceName(service) { + const s = String(service || '').trim().toLowerCase(); + if (s === 'web' || s === 'nextjs') return 'web'; + if (s === 'ingestion' || s === 'server' || s === 'socket') return 'ingestion'; + return 'web'; +} + +function getServiceLogFilePath(service, logsDir = resolveRuntimeLogsDir()) { + const safeService = normalizeServiceName(service); + return path.join(logsDir, `${safeService}.log`); +} + +function rotateLogFileIfNeeded(filePath) { + try { + const stat = fs.statSync(filePath); + if (stat.size >= MAX_LOG_FILE_BYTES) { + const backupPath = `${filePath}.1`; + try { + if (fs.existsSync(backupPath)) { + fs.unlinkSync(backupPath); + } + } catch { + // ignore backup cleanup error + } + fs.renameSync(filePath, backupPath); + } + } catch { + // file does not exist yet + } +} + +function appendRuntimeLogLine(service, streamName, rawChunk, options = {}) { + const logsDir = options.logsDir || resolveRuntimeLogsDir(options.env); + const safeService = normalizeServiceName(service); + const text = scrubSensitiveLogText(rawChunk); + if (!text) return 0; + + const lines = text.split(/\r?\n/).filter((line) => line.trim().length > 0); + if (lines.length === 0) return 0; + + try { + fs.mkdirSync(logsDir, { recursive: true }); + const filePath = getServiceLogFilePath(safeService, logsDir); + rotateLogFileIfNeeded(filePath); + const level = streamName === 'stderr' ? 'error' : 'info'; + const ts = new Date().toISOString(); + const formatted = + lines + .map((line) => JSON.stringify({ ts, service: safeService, level, message: line })) + .join('\n') + '\n'; + fs.appendFileSync(filePath, formatted, 'utf8'); + return lines.length; + } catch { + return 0; + } +} + +function installRuntimeLogHook(service, options = {}) { + const safeService = normalizeServiceName(service); + if (process.env.DATASTACK_LOG_CAPTURED_BY_RUNNER === '1' && !options.force) { + return { installed: false, reason: 'captured_by_runner' }; + } + const guardKey = `__dataStackRuntimeLogHook_${safeService}`; + if (globalThis[guardKey]) { + return { installed: false, reason: 'already_installed' }; + } + globalThis[guardKey] = true; + + let isWriting = false; + const wrapStream = (stream, streamName) => { + const originalWrite = stream.write.bind(stream); + stream.write = function patchedWrite(chunk, encoding, callback) { + const result = originalWrite(chunk, encoding, callback); + if (!isWriting) { + isWriting = true; + try { + const raw = + typeof chunk === 'string' + ? chunk + : Buffer.isBuffer(chunk) + ? chunk.toString(typeof encoding === 'string' ? encoding : 'utf8') + : String(chunk ?? ''); + appendRuntimeLogLine(safeService, streamName, raw, options); + } catch { + // Never interrupt stdout/stderr + } finally { + isWriting = false; + } + } + return result; + }; + }; + + wrapStream(process.stdout, 'stdout'); + wrapStream(process.stderr, 'stderr'); + return { installed: true, service: safeService }; +} + +function readTailLinesFromFile(filePath, maxBytes = MAX_SCAN_TAIL_BYTES) { + if (!fs.existsSync(filePath)) return []; + const stat = fs.statSync(filePath); + if (!stat.isFile() || stat.size === 0) return []; + + const readSize = Math.min(stat.size, maxBytes); + const startPos = Math.max(0, stat.size - readSize); + const fd = fs.openSync(filePath, 'r'); + try { + const buf = Buffer.allocUnsafe(readSize); + const bytesRead = fs.readSync(fd, buf, 0, readSize, startPos); + let content = buf.subarray(0, bytesRead).toString('utf8'); + if (startPos > 0) { + const firstNewline = content.indexOf('\n'); + content = firstNewline >= 0 ? content.slice(firstNewline + 1) : ''; + } + return content.split(/\r?\n/).filter(Boolean); + } finally { + fs.closeSync(fd); + } +} + +function parseStoredLogLine(rawLine, fallbackService) { + try { + const parsed = JSON.parse(rawLine); + if (parsed && typeof parsed === 'object' && typeof parsed.message === 'string') { + return { + ts: typeof parsed.ts === 'string' ? parsed.ts : '', + service: typeof parsed.service === 'string' ? parsed.service : fallbackService, + level: typeof parsed.level === 'string' ? parsed.level : 'info', + message: parsed.message, + }; + } + } catch { + // Plain-text fallback + } + return { + ts: '', + service: fallbackService, + level: 'info', + message: rawLine, + }; +} + +function readRuntimeLogs(options = {}) { + const logsDir = options.logsDir || resolveRuntimeLogsDir(options.env); + const serviceFilter = String(options.service || 'all').trim().toLowerCase(); + const requestedLines = Number(options.lines); + const limit = Number.isFinite(requestedLines) + ? Math.min(MAX_TAIL_LINES, Math.max(1, Math.floor(requestedLines))) + : DEFAULT_TAIL_LINES; + const search = typeof options.search === 'string' ? options.search.trim().toLowerCase() : ''; + const levelFilter = typeof options.level === 'string' ? options.level.trim().toLowerCase() : ''; + + const services = + serviceFilter === 'web' + ? ['web'] + : serviceFilter === 'ingestion' + ? ['ingestion'] + : ['web', 'ingestion']; + + const collected = []; + const filesChecked = []; + for (const svc of services) { + const mainFile = getServiceLogFilePath(svc, logsDir); + const backupFile = `${mainFile}.1`; + for (const candidate of [backupFile, mainFile]) { + if (fs.existsSync(candidate)) { + filesChecked.push(path.basename(candidate)); + const rawLines = readTailLinesFromFile(candidate); + for (const rawLine of rawLines) { + const entry = parseStoredLogLine(rawLine, svc); + if (levelFilter && levelFilter !== 'all' && entry.level !== levelFilter) { + continue; + } + if ( + search && + !entry.message.toLowerCase().includes(search) && + !entry.service.toLowerCase().includes(search) + ) { + continue; + } + collected.push(entry); + } + } + } + } + + if (filesChecked.length === 0) { + return { + entries: [], + totalMatched: 0, + filesChecked: [], + logsDir, + exists: false, + code: 'LOG_CAPTURE_UNAVAILABLE', + warning: 'No runtime logs captured yet in .runtime-logs/. Start services or emit stdout logs first.', + }; + } + + collected.sort((a, b) => { + if (a.ts && b.ts && a.ts !== b.ts) { + return a.ts.localeCompare(b.ts); + } + return 0; + }); + + const entries = collected.slice(-limit); + return { + entries, + totalMatched: collected.length, + filesChecked, + logsDir, + exists: true, + }; +} + +function readBuildLogs(options = {}) { + const logsDir = options.logsDir || resolveRuntimeLogsDir(options.env); + const cwd = options.cwd || process.cwd(); + const requestedLines = Number(options.lines); + const limit = Number.isFinite(requestedLines) + ? Math.min(MAX_TAIL_LINES, Math.max(1, Math.floor(requestedLines))) + : 200; + const search = typeof options.search === 'string' ? options.search.trim().toLowerCase() : ''; + + const metaCandidates = [ + path.join(logsDir, 'last-build-meta.json'), + path.join(cwd, 'build-meta.json'), + path.join(cwd, '.next', 'standalone', 'build-meta.json'), + ]; + const logCandidates = [ + path.join(logsDir, 'last-build.log'), + path.join(cwd, 'build.log'), + path.join(cwd, '.next', 'standalone', 'build.log'), + ]; + + let meta = null; + for (const metaPath of metaCandidates) { + try { + if (fs.existsSync(metaPath)) { + meta = JSON.parse(fs.readFileSync(metaPath, 'utf8')); + break; + } + } catch { + // try next candidate + } + } + + let logPathUsed = null; + let allLines = []; + for (const logPath of logCandidates) { + try { + if (fs.existsSync(logPath)) { + logPathUsed = path.basename(logPath); + allLines = readTailLinesFromFile(logPath); + break; + } + } catch { + // try next candidate + } + } + + if (!meta && !logPathUsed) { + return { + meta: null, + lines: [], + totalLines: 0, + exists: false, + code: 'LOG_CAPTURE_UNAVAILABLE', + warning: 'No captured build log or build-meta.json found yet. Run npm run build first.', + }; + } + + const filtered = search + ? allLines.filter((line) => line.toLowerCase().includes(search)) + : allLines; + + return { + meta, + logFile: logPathUsed, + lines: filtered.slice(-limit), + totalLines: filtered.length, + exists: true, + }; +} + +function syncStandaloneBuildArtifactsToLogsDir(options = {}) { + const logsDir = options.logsDir || resolveRuntimeLogsDir(options.env); + const cwd = options.cwd || process.cwd(); + const buildLogSrc = [ + path.join(cwd, 'build.log'), + path.join(cwd, '.next', 'standalone', 'build.log'), + ].find((candidate) => fs.existsSync(candidate)); + const buildMetaSrc = [ + path.join(cwd, 'build-meta.json'), + path.join(cwd, '.next', 'standalone', 'build-meta.json'), + ].find((candidate) => fs.existsSync(candidate)); + + if (!buildLogSrc && !buildMetaSrc) { + return { synced: false }; + } + + try { + fs.mkdirSync(logsDir, { recursive: true }); + if (buildLogSrc) { + fs.copyFileSync(buildLogSrc, path.join(logsDir, 'last-build.log')); + } + if (buildMetaSrc) { + fs.copyFileSync(buildMetaSrc, path.join(logsDir, 'last-build-meta.json')); + } + return { synced: true, logsDir }; + } catch { + return { synced: false }; + } +} + +module.exports = { + MAX_LOG_FILE_BYTES, + scrubSensitiveLogText, + resolveRuntimeLogsDir, + getServiceLogFilePath, + appendRuntimeLogLine, + installRuntimeLogHook, + readRuntimeLogs, + readBuildLogs, + syncStandaloneBuildArtifactsToLogsDir, +}; diff --git a/storage/.runtime-logs/last-build-meta.json b/storage/.runtime-logs/last-build-meta.json new file mode 100644 index 0000000..bc6f06d --- /dev/null +++ b/storage/.runtime-logs/last-build-meta.json @@ -0,0 +1,11 @@ +{ + "appName": "datastack", + "appVersion": "0.1.0", + "nodeVersion": "v22.23.2", + "nextVersion": "^15.4.9", + "startedAt": "2026-11-21T16:53:22.930Z", + "finishedAt": "2026-11-21T16:54:54.446Z", + "durationMs": 91516, + "exitCode": 0, + "status": "succeeded" +} diff --git a/tests/phase07-deployment.test.ts b/tests/phase07-deployment.test.ts index b5b48db..0723f3b 100644 --- a/tests/phase07-deployment.test.ts +++ b/tests/phase07-deployment.test.ts @@ -444,17 +444,17 @@ test("DEPLOY-BUG-02, 03 & 04: Port 4000 memoizes schema DDLs, hydrates rotated J assert.equal(failedSync, false); assert.equal(ddlCount, 1); - // 2. Recovery runs 10 DDLs once (7 CREATE TABLE + 3 ALTER TABLE) and hydrates rotated jwt_secret; repeated calls do zero extra DDLs + // 2. Recovery runs 11 DDLs once (8 CREATE TABLE + 3 ALTER TABLE) and hydrates rotated jwt_secret; repeated calls do zero extra DDLs failDdl = false; ddlCount = 0; assert.equal(await ingestionServer.syncSettingsFromDatabase(), true); - assert.equal(ddlCount, 10); + assert.equal(ddlCount, 11); assert.equal(selectCount, 1); assert.equal(ingestionServer.runtimeConfig.jwtSecret, rotatedSecret); assert.equal(await ingestionServer.syncSettingsFromDatabase(), true); assert.equal(await ingestionServer.syncSettingsFromDatabase(), true); - assert.equal(ddlCount, 10, "Repeated 3s sync ticks must NOT re-run CREATE/ALTER DDL statements"); + assert.equal(ddlCount, 11, "Repeated 3s sync ticks must NOT re-run CREATE/ALTER DDL statements"); assert.equal(selectCount, 3); // 3. Graceful shutdown flushes buffered events, quits Redis, and closes PostgreSQL pool cleanly diff --git a/tests/r4-phase01-security-boot.test.ts b/tests/r4-phase01-security-boot.test.ts index 9c78e55..ea9b463 100644 --- a/tests/r4-phase01-security-boot.test.ts +++ b/tests/r4-phase01-security-boot.test.ts @@ -357,7 +357,7 @@ test("R4-Phase-01 [AUDIT-01 / ERR-R4-01]: API routes (/api/settings, /api/settin } }); -test("R4-Phase-01 [AUDIT-03 / ERR-R4-02]: Port 4000 ensureDatabaseSchema creates all 7 tables including admin_team_members and admin_audit_logs", async () => { +test("R4-Phase-01 [AUDIT-03 / ERR-R4-02]: Port 4000 ensureDatabaseSchema creates all 8 tables including admin_team_members, admin_audit_logs, and mcp_settings", async () => { const executedSql: string[] = []; const mockPool = { async query(sql: string) { @@ -375,7 +375,7 @@ test("R4-Phase-01 [AUDIT-03 / ERR-R4-02]: Port 4000 ensureDatabaseSchema creates assert.equal(synced, true); const createStatements = executedSql.filter((s) => /CREATE TABLE IF NOT EXISTS/i.test(s)); - assert.equal(createStatements.length, 7, "Port 4000 must create all 7 tables"); + assert.equal(createStatements.length, 8, "Port 4000 must create all 8 tables"); const joined = createStatements.join("\n"); for (const tableName of [ "client_sdks", @@ -385,6 +385,7 @@ test("R4-Phase-01 [AUDIT-03 / ERR-R4-02]: Port 4000 ensureDatabaseSchema creates "admin_login_attempts", "admin_team_members", "admin_audit_logs", + "mcp_settings", ]) { assert.ok( joined.includes(tableName), diff --git a/tests/r5-phase01-mcp-foundation-logs.test.ts b/tests/r5-phase01-mcp-foundation-logs.test.ts new file mode 100644 index 0000000..dfc3c19 --- /dev/null +++ b/tests/r5-phase01-mcp-foundation-logs.test.ts @@ -0,0 +1,516 @@ +import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import http from "node:http"; +import { createRequire } from "node:module"; +import type { AddressInfo } from "node:net"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { NextRequest } from "next/server"; + +process.env.DATASTACK_DISABLE_LISTEN = "1"; + +import { POST as internalMcpPost } from "../app/api/internal/mcp/route"; +import { + clearAuditLogsForTests, + listAdminAuditLogsWithStatusAsync, + recordAdminAuditLogAsync, +} from "../core/auth/auditLog"; +import { CORE_SCHEMA_TABLE_DDLS } from "../core/Database/database"; +import { + appendRuntimeLogLine, + readBuildLogs, + readRuntimeLogs, + scrubSensitiveLogText, + syncStandaloneBuildArtifactsToLogsDir, +} from "../core/logs/runtimeLogCapture"; +import { + MCP_PERMISSION_DEFINITIONS, + McpSettingsManager, + generateMcpBearerToken, + hashMcpToken, + verifyMcpBearerTokenHash, +} from "../core/mcp/mcpSettings"; +import { SDKManager } from "../core/sdk/sdkManager"; +import { getInternalIngestionHeaders } from "../core/Security/security"; +import { SSRFBlockedError } from "../core/Security/ssrfGuard"; +import { CSVStorageManager } from "../core/Storage/csvStorage"; +import { middleware } from "../middleware"; + +const require = createRequire(import.meta.url); +const cjsRuntimeLogCapture = require("../server/runtimeLogCapture.js") as { + MAX_LOG_FILE_BYTES: number; + scrubSensitiveLogText: (raw: string) => string; + appendRuntimeLogLine: ( + service: string, + streamName: "stdout" | "stderr", + rawChunk: string, + options?: { logsDir?: string; env?: NodeJS.ProcessEnv }, + ) => number; + readRuntimeLogs: (options?: { + logsDir?: string; + service?: string; + lines?: number; + search?: string; + level?: string; + }) => { + entries: Array<{ ts: string; service: string; level: string; message: string }>; + totalMatched: number; + exists: boolean; + code?: string; + warning?: string; + }; + readBuildLogs: (options?: { + logsDir?: string; + cwd?: string; + lines?: number; + search?: string; + }) => { + meta: Record | null; + lines: string[]; + totalLines: number; + exists: boolean; + code?: string; + warning?: string; + }; +}; + +const ingestionServer = require("../server/index.js") as { + setDbPoolForTests: (pool: unknown) => void; + syncSettingsFromDatabase: () => Promise; +}; + +test("R5-Phase-01 [MCP-AUDIT-02 / ERR-R5-02]: Runtime log capture scrubs secrets, rotates at 2MB, filters/tails safely, and handles missing dirs gracefully", async () => { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "datastack-r5-logs-")); + const logsDir = path.join(tmpDir, ".runtime-logs"); + try { + // 1. Missing directory returns LOG_CAPTURE_UNAVAILABLE cleanly without throwing + const missingRead = readRuntimeLogs({ logsDir }); + assert.equal(missingRead.exists, false); + assert.equal(missingRead.code, "LOG_CAPTURE_UNAVAILABLE"); + assert.equal(missingRead.entries.length, 0); + + const missingBuild = readBuildLogs({ logsDir, cwd: tmpDir }); + assert.equal(missingBuild.exists, false); + assert.equal(missingBuild.code, "LOG_CAPTURE_UNAVAILABLE"); + + // 2. Secret scrubbing across TS + CJS implementations + const rawLine = + 'Auth Bearer secret_jwt_token_123 dsmcp_abcdef1234567890 eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.SuperSecretSignature {"coolifyToken":"cf_live_999","mcpToken":"dsmcp_xyz987654321"}'; + const scrubbedTs = scrubSensitiveLogText(rawLine); + const scrubbedCjs = cjsRuntimeLogCapture.scrubSensitiveLogText(rawLine); + assert.equal(scrubbedTs, scrubbedCjs); + assert.ok(!scrubbedTs.includes("secret_jwt_token_123")); + assert.ok(!scrubbedTs.includes("abcdef1234567890")); + assert.ok(!scrubbedTs.includes("SuperSecretSignature")); + assert.ok(!scrubbedTs.includes("cf_live_999")); + assert.ok(scrubbedTs.includes("[REDACTED]")); + + // 3. Multi-service append + read filtering + const writtenWeb = appendRuntimeLogLine("web", "stdout", `${rawLine}\n[web] ready on port 3000\n`, { + logsDir, + }); + const writtenIng = cjsRuntimeLogCapture.appendRuntimeLogLine( + "ingestion", + "stderr", + "[ingestion] redis reconnect warning\n", + { logsDir }, + ); + assert.equal(writtenWeb, 2); + assert.equal(writtenIng, 1); + + const allLogs = readRuntimeLogs({ logsDir, service: "all", lines: 50 }); + assert.equal(allLogs.exists, true); + assert.equal(allLogs.totalMatched, 3); + assert.equal(allLogs.entries.length, 3); + assert.ok(allLogs.entries.every((e) => !e.message.includes("secret_jwt_token_123"))); + + const errOnly = cjsRuntimeLogCapture.readRuntimeLogs({ + logsDir, + service: "ingestion", + level: "error", + }); + assert.equal(errOnly.exists, true); + assert.equal(errOnly.entries.length, 1); + assert.equal(errOnly.entries[0]?.service, "ingestion"); + assert.equal(errOnly.entries[0]?.level, "error"); + + // 4. Rotation to .1 backup when file size >= 2MB + const webLogPath = path.join(logsDir, "web.log"); + await fs.writeFile(webLogPath, "x".repeat(cjsRuntimeLogCapture.MAX_LOG_FILE_BYTES + 128), "utf8"); + appendRuntimeLogLine("web", "stdout", "[web] post-rotation line\n", { logsDir }); + const backupStat = await fs.stat(`${webLogPath}.1`); + assert.ok(backupStat.size >= cjsRuntimeLogCapture.MAX_LOG_FILE_BYTES); + const afterRotate = readRuntimeLogs({ logsDir, service: "web", search: "post-rotation" }); + assert.equal(afterRotate.entries.length, 1); + assert.match(afterRotate.entries[0]?.message || "", /post-rotation/); + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }); + } +}); + +test("R5-Phase-01 [MCP-AUDIT-03]: Standalone build.log & build-meta.json sync to .runtime-logs and Dockerfile.web/capture-build-log stay intact", async () => { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "datastack-r5-build-")); + const standaloneDir = path.join(tmpDir, ".next", "standalone"); + const logsDir = path.join(tmpDir, "storage", ".runtime-logs"); + try { + await fs.mkdir(standaloneDir, { recursive: true }); + await fs.writeFile( + path.join(standaloneDir, "build.log"), + "Creating an optimized production build ...\nCompiled successfully in 12.4s\n", + "utf8", + ); + await fs.writeFile( + path.join(standaloneDir, "build-meta.json"), + JSON.stringify({ appName: "datastack", status: "succeeded", exitCode: 0 }), + "utf8", + ); + + const synced = syncStandaloneBuildArtifactsToLogsDir({ cwd: tmpDir, logsDir }); + assert.equal(synced.synced, true); + + const buildRead = readBuildLogs({ logsDir, cwd: tmpDir, search: "compiled" }); + assert.equal(buildRead.exists, true); + assert.equal(buildRead.meta?.status, "succeeded"); + assert.equal(buildRead.lines.length, 1); + assert.match(buildRead.lines[0] || "", /Compiled successfully/); + + const pkgJson = JSON.parse(await fs.readFile(path.resolve(process.cwd(), "package.json"), "utf8")) as { + scripts?: Record; + }; + assert.equal(pkgJson.scripts?.build, "node scripts/capture-build-log.js"); + + const dockerfileWeb = await fs.readFile(path.resolve(process.cwd(), "docker/Dockerfile.web"), "utf8"); + assert.match(dockerfileWeb, /\.runtime-logs/); + } finally { + await fs.rm(tmpDir, { recursive: true, force: true }); + } +}); + +test("R5-Phase-01 [MCP-AUDIT-05]: McpSettingsManager enforces OFF default, 17 granular toggles, token hash/vault, CIDR/SSRF validation, DB roundtrip & Port 4000 sync", async () => { + McpSettingsManager.resetForTests(); + const prevDbUrl = process.env.DATABASE_URL; + const prevCorePool = (globalThis as Record).__dataStackCorePool; + const prevCoreSchema = (globalThis as Record).__dataStackCoreSchema; + const prevIngestionUrl = process.env.INGESTION_INTERNAL_URL; + + let persistedRow: Record | null = null; + const syncPayloads: Array> = []; + + const syncServer = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c) => chunks.push(Buffer.from(c))); + req.on("end", () => { + if (req.url === "/internal/mcp-sync" && req.method === "POST") { + syncPayloads.push(JSON.parse(Buffer.concat(chunks).toString("utf8")) as Record); + res.statusCode = 200; + res.setHeader("Content-Type", "application/json"); + res.end(JSON.stringify({ success: true })); + return; + } + res.statusCode = 404; + res.end(); + }); + }); + + await new Promise((resolve) => syncServer.listen(0, "127.0.0.1", () => resolve())); + const syncPort = (syncServer.address() as AddressInfo).port; + + try { + process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${syncPort}`; + assert.equal(MCP_PERMISSION_DEFINITIONS.length, 17); + + const mockPool = { + async query(sql: string, params?: unknown[]) { + if (/INSERT INTO mcp_settings/i.test(sql) && params) { + persistedRow = { + enabled: params[0], + token_hash: params[1], + token_enc: params[2], + token_preview: params[3], + allowed_ips: params[4], + permissions_json: params[5], + coolify_url: params[6], + coolify_token_enc: params[7], + coolify_app_uuid: params[8], + updated_at: params[9], + }; + return { rowCount: 1, rows: [] }; + } + if (/FROM mcp_settings/i.test(sql)) { + return persistedRow + ? { rowCount: 1, rows: [persistedRow] } + : { rowCount: 0, rows: [] }; + } + return { rowCount: 0, rows: [] }; + }, + }; + + process.env.DATABASE_URL = "postgres://datastack:test@127.0.0.1:5432/datastack"; + (globalThis as Record).__dataStackCorePool = mockPool; + (globalThis as Record).__dataStackCoreSchema = Promise.resolve(); + + // 1. Default safe-by-default OFF state + const initial = await McpSettingsManager.getPublicSettingsAsync(); + assert.equal(initial.enabled, false); + assert.equal(initial.hasToken, false); + assert.equal(initial.permissions.clients_create, false); + assert.equal(initial.permissions.metrics_health_read, true); + + // 2. Enabling auto-mints Bearer token + encrypts Coolify token + syncs Port 4000 + const enabledRes = await McpSettingsManager.updateSettingsAsync({ + enabled: true, + allowedIps: "203.0.113.0/24", + permissions: { clients_create: true, files_edit: true }, + coolifyUrl: "https://coolify.example.com", + coolifyToken: "coolify_pat_secret_123", + coolifyAppUuid: "app-uuid-01", + }); + + assert.equal(enabledRes.settings.enabled, true); + assert.equal(enabledRes.settings.hasToken, true); + assert.ok(enabledRes.issuedToken?.startsWith("dsmcp_")); + assert.equal(enabledRes.issuedToken?.length, 70); + assert.equal(enabledRes.settings.permissions.clients_create, true); + assert.equal(enabledRes.settings.permissions.files_edit, true); + assert.equal(enabledRes.settings.hasCoolifyToken, true); + assert.equal(enabledRes.bridgeSync.ok, true); + assert.equal(syncPayloads.length, 1); + assert.equal(syncPayloads[0]?.coolifyToken, "coolify_pat_secret_123"); + + // 3. SHA-256 digest & timing-safe verification + const rawToken = generateMcpBearerToken(); + const digest = hashMcpToken(rawToken); + assert.equal(verifyMcpBearerTokenHash(rawToken, digest), true); + assert.equal(verifyMcpBearerTokenHash(`${rawToken}x`, digest), false); + + // 4. Re-hydrate from mock PG row cleanly + McpSettingsManager.resetForTests(); + const rehydrated = await McpSettingsManager.getPublicSettingsAsync(); + assert.equal(rehydrated.enabled, true); + assert.equal(rehydrated.allowedIps, "203.0.113.0/24"); + assert.equal(rehydrated.permissions.clients_create, true); + assert.equal(rehydrated.hasCoolifyToken, true); + + // 5. Reject invalid CIDR & SSRF target for coolifyUrl + await assert.rejects( + () => McpSettingsManager.updateSettingsAsync({ allowedIps: "not-a-cidr" }), + /Invalid/i, + ); + await assert.rejects( + () => + McpSettingsManager.updateSettingsAsync({ + coolifyUrl: "http://169.254.169.254/latest/meta-data", + }), + (err: unknown) => err instanceof SSRFBlockedError, + ); + + // 6. Revoke forces enabled: false and clears token + const revoked = await McpSettingsManager.updateSettingsAsync({ revokeToken: true }); + assert.equal(revoked.settings.enabled, false); + assert.equal(revoked.settings.hasToken, false); + } finally { + McpSettingsManager.resetForTests(); + if (prevDbUrl === undefined) delete process.env.DATABASE_URL; + else process.env.DATABASE_URL = prevDbUrl; + (globalThis as Record).__dataStackCorePool = prevCorePool; + (globalThis as Record).__dataStackCoreSchema = prevCoreSchema; + if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL; + else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl; + await new Promise((resolve) => syncServer.close(() => resolve())); + } +}); + +test("R5-Phase-01 [MCP-AUDIT-05 / MCP-AUDIT-07]: Port 3000 & Port 4000 share 8-table mcp_settings DDL parity and auditLog records/redacts MCP actions", async () => { + // 1. Port 3000 DDL contains mcp_settings as 8th table + assert.equal(CORE_SCHEMA_TABLE_DDLS.length, 8); + assert.ok(CORE_SCHEMA_TABLE_DDLS.some((ddl) => /CREATE TABLE IF NOT EXISTS mcp_settings/i.test(ddl))); + + // 2. Port 4000 ensureDatabaseSchema creates mcp_settings as well + const executedSql: string[] = []; + try { + ingestionServer.setDbPoolForTests({ + async query(sql: string) { + executedSql.push(sql); + return { rowCount: 0, rows: [] }; + }, + }); + const synced = await ingestionServer.syncSettingsFromDatabase(); + assert.equal(synced, true); + const createStatements = executedSql.filter((s) => /CREATE TABLE IF NOT EXISTS/i.test(s)); + assert.equal(createStatements.length, 8); + assert.ok(createStatements.some((s) => /\bmcp_settings\b/.test(s))); + } finally { + ingestionServer.setDbPoolForTests(null); + } + + // 3. MCP_* audit actions & secret redaction + clearAuditLogsForTests(); + try { + await recordAdminAuditLogAsync({ + actorEmail: "owner@datastack.dev", + actorRole: "owner", + action: "MCP_SETTINGS_UPDATED", + target: "mcp_settings", + metadata: { + mcpToken: "dsmcp_plain_should_redact", + mcpBearerToken: "dsmcp_bearer_should_redact", + issuedToken: "dsmcp_issued_should_redact", + tokenHash: "abc123", + tokenEnc: "v1:enc", + coolifyToken: "cf_secret", + coolifyTokenEnc: "v1:cf_enc", + safeFlag: true, + }, + }); + await recordAdminAuditLogAsync({ + actorEmail: "mcp-agent", + actorRole: "mcp", + action: "MCP_TOOL_DENIED", + target: "clients_create", + metadata: { permission: "clients_create" }, + }); + + const list = await listAdminAuditLogsWithStatusAsync({ limit: 10 }); + const settingsEvt = list.logs.find((l) => l.action === "MCP_SETTINGS_UPDATED"); + const deniedEvt = list.logs.find((l) => l.action === "MCP_TOOL_DENIED"); + assert.ok(settingsEvt); + assert.ok(deniedEvt); + for (const redactedKey of [ + "mcpToken", + "mcpBearerToken", + "issuedToken", + "tokenHash", + "tokenEnc", + "coolifyToken", + "coolifyTokenEnc", + ]) { + assert.equal(settingsEvt.metadata[redactedKey], "[REDACTED]"); + } + assert.equal(settingsEvt.metadata.safeFlag, true); + } finally { + clearAuditLogsForTests(); + } +}); + +test("R5-Phase-01 [MCP-AUDIT-04 / ERR-R5-01]: middleware permits POST /api/internal/mcp only with x-datastack-internal-secret and route enforces timing-safe auth & actions", async () => { + const prevStorageDir = process.env.DATASTACK_STORAGE_DIR; + const tmpStorage = await fs.mkdtemp(path.join(os.tmpdir(), "datastack-r5-bridge-")); + const originalBaseDir = CSVStorageManager.baseStorageDir; + + try { + CSVStorageManager.baseStorageDir = path.join(tmpStorage, "data"); + const internalHeaders = getInternalIngestionHeaders(); + assert.ok(internalHeaders["x-datastack-internal-secret"]); + + // 1. Middleware rejects POST /api/internal/mcp without internal secret (cross-origin check -> 403) + const noSecretMwReq = new NextRequest("http://localhost:3000/api/internal/mcp", { + method: "POST", + headers: { host: "localhost:3000" }, + }); + const noSecretMwRes = await middleware(noSecretMwReq); + assert.equal(noSecretMwRes.status, 403); + + // 2. Middleware allows POST /api/internal/mcp to reach route when header is present + const withSecretMwReq = new NextRequest("http://localhost:3000/api/internal/mcp", { + method: "POST", + headers: { + host: "localhost:3000", + "x-datastack-internal-secret": internalHeaders["x-datastack-internal-secret"], + }, + }); + const withSecretMwRes = await middleware(withSecretMwReq); + assert.equal(withSecretMwRes.status, 200); + + // 3. Route rejects wrong/missing internal secret with 401 MCP_BRIDGE_UNAUTHORIZED (ERR-R5-01) + const badSecretReq = new NextRequest("http://localhost:3000/api/internal/mcp", { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-datastack-internal-secret": "wrong-secret", + }, + body: JSON.stringify({ action: "metrics_get" }), + }); + const badSecretRes = await internalMcpPost(badSecretReq); + assert.equal(badSecretRes.status, 401); + const badJson = (await badSecretRes.json()) as { code: string }; + assert.equal(badJson.code, "MCP_BRIDGE_UNAUTHORIZED"); + + const callBridge = async (action: string, params: Record = {}) => { + const req = new NextRequest("http://localhost:3000/api/internal/mcp", { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-datastack-internal-secret": internalHeaders["x-datastack-internal-secret"], + }, + body: JSON.stringify({ action, params }), + }); + const res = await internalMcpPost(req); + return { status: res.status, body: (await res.json()) as Record }; + }; + + // 4. Client create / list / update / rotate / revoke / delete + const uniqueId = `mcp-test-client-${Date.now()}`; + const created = await callBridge("clients_create", { + clientId: uniqueId, + name: "MCP Test Client", + rateLimit: 250, + }); + assert.equal(created.status, 201); + const createdClient = created.body.client as { clientId: string; token?: string; secret?: string }; + assert.equal(createdClient.clientId, uniqueId); + assert.ok(createdClient.token); + assert.equal(createdClient.secret, undefined); + + const listed = await callBridge("clients_list"); + assert.equal(listed.status, 200); + const listArr = listed.body.clients as Array<{ clientId: string; token?: string }>; + const found = listArr.find((c) => c.clientId === uniqueId); + assert.ok(found); + assert.equal(found?.token, undefined); + + const updated = await callBridge("clients_update", { + clientId: uniqueId, + rateLimit: 500, + }); + assert.equal(updated.status, 200); + assert.equal((updated.body.client as { rateLimit: number }).rateLimit, 500); + + // 5. settings_update blocks forbidden sensitive field jwtSecret + const badSettings = await callBridge("settings_update", { + jwtSecret: "attempted-overwrite-secret", + }); + assert.equal(badSettings.status, 400); + assert.equal(badSettings.body.code, "VALIDATION_ERROR"); + + // 6. Metrics, settings_get, data_query (with stored CSV row), storage_stats + const metricsRes = await callBridge("metrics_get"); + assert.equal(metricsRes.status, 200); + const settingsRes = await callBridge("settings_get"); + assert.equal(settingsRes.status, 200); + + const clientDataDir = path.join(CSVStorageManager.baseStorageDir, CSVStorageManager.sanitizeClientId(uniqueId)); + await fs.mkdir(clientDataDir, { recursive: true }); + await fs.writeFile( + path.join(clientDataDir, "2026-11-21.csv"), + `"clientId","eventId","timestamp","payload"\n"${uniqueId}","evt-mcp-csv-1","2026-11-21T10:00:00.000Z","{""ok"":true}"\n`, + "utf8", + ); + const dataRes = await callBridge("data_query", { clientId: uniqueId, limit: 10 }); + assert.equal(dataRes.status, 200); + assert.equal(dataRes.body.count, 1); + const statsRes = await callBridge("storage_stats"); + assert.equal(statsRes.status, 200); + + // Cleanup client + const delRes = await callBridge("clients_delete", { clientId: uniqueId, purgeFiles: true }); + assert.equal(delRes.status, 200); + await SDKManager.deleteClientWithStatsAsync(uniqueId, { purgeFiles: true }); + } finally { + CSVStorageManager.baseStorageDir = originalBaseDir; + if (prevStorageDir === undefined) delete process.env.DATASTACK_STORAGE_DIR; + else process.env.DATASTACK_STORAGE_DIR = prevStorageDir; + clearAuditLogsForTests(); + await fs.rm(tmpStorage, { recursive: true, force: true }); + } +});