feat: implement R4 production hardening
- Add configurable ingestion batch item limits - Improve client serialization and sanitization - Add graceful handling for corrupt GZIP partitions in data API - Expand admin session token scope - Update project documentation for R4 status
This commit is contained in:
1 parent
8132087085
commit
ae9b554189
59 files changed
+16879
-1074
No files matched your search
@@ -0,0 +1,308 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import http from 'node:http';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import express from 'express';
|
||||
import { createAdminSessionToken } from '../core/auth/adminSession';
|
||||
|
||||
const fm = require('../server/filesManager');
|
||||
|
||||
test('FM-REQ-02: buildCloudCmdConfig enforces RCE lockdown, root jail, zip packer, and scrubs default tokens', () => {
|
||||
const tmpRoot = path.resolve(os.tmpdir(), 'datastack-fm-cfg-test');
|
||||
const cfg = fm.buildCloudCmdConfig(tmpRoot);
|
||||
|
||||
assert.equal(cfg.root, tmpRoot);
|
||||
assert.equal(cfg.prefix, '/vcon/files-engine');
|
||||
assert.equal(cfg.console, false);
|
||||
assert.equal(cfg.terminal, false);
|
||||
assert.equal(cfg.configDialog, false);
|
||||
assert.equal(cfg.auth, false);
|
||||
assert.equal(cfg.open, false);
|
||||
assert.equal(cfg.online, false);
|
||||
assert.equal(cfg.contact, false);
|
||||
assert.equal(cfg.oneFilePanel, true);
|
||||
assert.equal(cfg.packer, 'zip');
|
||||
assert.equal(cfg.editor, 'edward');
|
||||
assert.equal(cfg.password, '');
|
||||
assert.equal(cfg.exportToken, '');
|
||||
assert.equal(cfg.importToken, '');
|
||||
assert.equal(cfg.dropboxToken, '');
|
||||
});
|
||||
|
||||
test('FM-REQ-04: authenticateFilesRequest fails closed on missing prod secret and DB outages', async () => {
|
||||
const token = await createAdminSessionToken(
|
||||
'owner@datastack.dev',
|
||||
fm.DEV_DEFAULT_SESSION_SECRET,
|
||||
Date.now() + 60_000,
|
||||
1,
|
||||
'owner',
|
||||
'owner-1',
|
||||
);
|
||||
|
||||
const prodMissing = await fm.authenticateFilesRequest({
|
||||
cookieHeader: `datastack_admin_session=${token}`,
|
||||
dbPool: null,
|
||||
env: { NODE_ENV: 'production', DATASTACK_ADMIN_SESSION_SECRET: '' },
|
||||
});
|
||||
assert.equal(prodMissing.valid, false);
|
||||
assert.equal(prodMissing.status, 503);
|
||||
assert.equal(prodMissing.code, 'FILES_AUTH_UNCONFIGURED');
|
||||
|
||||
const dbUnavailable = await fm.authenticateFilesRequest({
|
||||
cookieHeader: `datastack_admin_session=${token}`,
|
||||
dbPool: {
|
||||
query: async () => {
|
||||
throw new Error('ECONNREFUSED');
|
||||
},
|
||||
},
|
||||
});
|
||||
assert.equal(dbUnavailable.valid, false);
|
||||
assert.equal(dbUnavailable.status, 503);
|
||||
assert.equal(dbUnavailable.code, 'AUTH_DB_UNAVAILABLE');
|
||||
});
|
||||
|
||||
test('FM-REQ-01..06: live CloudCmd mount enforces auth, DB revocation, RBAC, CSRF, *.lock & traversal guards, and full file CRUD + ZIP/UNZIP', async () => {
|
||||
const tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'datastack-fm-e2e-'));
|
||||
fs.mkdirSync(path.join(tmpRoot, 'client-alpha'), { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(tmpRoot, 'client-alpha', '2026-11-20.csv'),
|
||||
'eventId,clientId\nev-1,client-alpha\n',
|
||||
);
|
||||
fs.writeFileSync(path.join(tmpRoot, 'client-alpha', '2026-11-20.csv.lock'), 'pid=99');
|
||||
|
||||
const app = express();
|
||||
const server = http.createServer(app);
|
||||
let activeDbPool: { query: (sql: string, params?: unknown[]) => Promise<{ rows: unknown[] }> } | null = null;
|
||||
|
||||
const handle = await fm.mountFilesManager({
|
||||
app,
|
||||
server,
|
||||
rootDir: tmpRoot,
|
||||
getDbPool: () => activeDbPool,
|
||||
});
|
||||
|
||||
try {
|
||||
assert.equal(handle.engineReady, true, `CloudCmd failed to init: ${handle.initErrorMessage}`);
|
||||
assert.equal(handle.socketPath, '/vcon/files-engine/socket.io');
|
||||
|
||||
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', () => resolve()));
|
||||
const address = server.address() as { port: number };
|
||||
const baseUrl = `http://127.0.0.1:${address.port}`;
|
||||
const hostHeader = `127.0.0.1:${address.port}`;
|
||||
|
||||
const ownerToken = await createAdminSessionToken(
|
||||
'owner@datastack.dev',
|
||||
fm.DEV_DEFAULT_SESSION_SECRET,
|
||||
Date.now() + 60_000,
|
||||
1,
|
||||
'owner',
|
||||
'owner-1',
|
||||
);
|
||||
const viewerToken = await createAdminSessionToken(
|
||||
'viewer@datastack.dev',
|
||||
fm.DEV_DEFAULT_SESSION_SECRET,
|
||||
Date.now() + 60_000,
|
||||
1,
|
||||
'viewer',
|
||||
'member-viewer-1',
|
||||
);
|
||||
|
||||
const ownerHeaders = {
|
||||
Cookie: `datastack_admin_session=${ownerToken}`,
|
||||
Origin: baseUrl,
|
||||
Host: hostHeader,
|
||||
};
|
||||
const viewerHeaders = {
|
||||
Cookie: `datastack_admin_session=${viewerToken}`,
|
||||
Origin: baseUrl,
|
||||
Host: hostHeader,
|
||||
};
|
||||
|
||||
// 1. Unauthenticated access blocked
|
||||
assert.equal((await fetch(`${baseUrl}/vcon/files-engine/_status`)).status, 401);
|
||||
assert.equal((await fetch(`${baseUrl}/vcon/files-engine/`)).status, 401);
|
||||
|
||||
// 2. Authenticated status probe & locked HTML output
|
||||
const statusRes = await fetch(`${baseUrl}/vcon/files-engine/_status`, {
|
||||
headers: { Cookie: ownerHeaders.Cookie },
|
||||
});
|
||||
assert.equal(statusRes.status, 200);
|
||||
const statusJson = (await statusRes.json()) as { ok: boolean; role: string; readOnly: boolean };
|
||||
assert.equal(statusJson.ok, true);
|
||||
assert.equal(statusJson.role, 'owner');
|
||||
assert.equal(statusJson.readOnly, false);
|
||||
|
||||
const htmlRes = await fetch(`${baseUrl}/vcon/files-engine/`, {
|
||||
headers: { Cookie: ownerHeaders.Cookie },
|
||||
});
|
||||
assert.equal(htmlRes.status, 200);
|
||||
const html = await htmlRes.text();
|
||||
assert.ok(html.includes('/vcon/files-engine'));
|
||||
assert.ok(html.includes('"console":false'));
|
||||
assert.ok(html.includes('"terminal":false'));
|
||||
assert.ok(html.includes('"configDialog":false'));
|
||||
assert.ok(html.includes('"packer":"zip"'));
|
||||
|
||||
// 3. Create, rename, ZIP, UNZIP, download, and delete
|
||||
const createRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/fs/client-alpha/note.csv`, {
|
||||
method: 'PUT',
|
||||
headers: ownerHeaders,
|
||||
body: 'k,v\n1,ok\n',
|
||||
});
|
||||
assert.equal(createRes.status, 200);
|
||||
assert.equal(fs.readFileSync(path.join(tmpRoot, 'client-alpha', 'note.csv'), 'utf8'), 'k,v\n1,ok\n');
|
||||
|
||||
const renameRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/rename`, {
|
||||
method: 'PUT',
|
||||
headers: { ...ownerHeaders, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ from: '/client-alpha/note.csv', to: '/client-alpha/note-renamed.csv' }),
|
||||
});
|
||||
assert.equal(renameRes.status, 200);
|
||||
assert.equal(fs.existsSync(path.join(tmpRoot, 'client-alpha', 'note-renamed.csv')), true);
|
||||
|
||||
const packRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/pack`, {
|
||||
method: 'PUT',
|
||||
headers: { ...ownerHeaders, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
from: '/client-alpha',
|
||||
to: '/client-alpha/bundle.zip',
|
||||
names: ['note-renamed.csv'],
|
||||
}),
|
||||
});
|
||||
assert.equal(packRes.status, 200);
|
||||
assert.equal(fs.existsSync(path.join(tmpRoot, 'client-alpha', 'bundle.zip')), true);
|
||||
|
||||
fs.mkdirSync(path.join(tmpRoot, 'client-alpha', 'extracted'), { recursive: true });
|
||||
const extractRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/extract`, {
|
||||
method: 'PUT',
|
||||
headers: { ...ownerHeaders, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
from: '/client-alpha/bundle.zip',
|
||||
to: '/client-alpha/extracted',
|
||||
}),
|
||||
});
|
||||
assert.equal(extractRes.status, 200);
|
||||
assert.equal(fs.existsSync(path.join(tmpRoot, 'client-alpha', 'extracted', 'note-renamed.csv')), true);
|
||||
|
||||
const downloadRes = await fetch(
|
||||
`${baseUrl}/vcon/files-engine/fs/client-alpha/note-renamed.csv?download`,
|
||||
{ headers: { Cookie: ownerHeaders.Cookie } },
|
||||
);
|
||||
assert.equal(downloadRes.status, 200);
|
||||
assert.equal(await downloadRes.text(), 'k,v\n1,ok\n');
|
||||
|
||||
const deleteRes = await fetch(
|
||||
`${baseUrl}/vcon/files-engine/api/v1/fs/client-alpha/note-renamed.csv`,
|
||||
{ method: 'DELETE', headers: ownerHeaders },
|
||||
);
|
||||
assert.equal(deleteRes.status, 200);
|
||||
assert.equal(fs.existsSync(path.join(tmpRoot, 'client-alpha', 'note-renamed.csv')), false);
|
||||
|
||||
// 4. Viewer role: GET allowed, mutations blocked (403)
|
||||
const viewerList = await fetch(`${baseUrl}/vcon/files-engine/api/v1/fs/`, {
|
||||
headers: { Cookie: viewerHeaders.Cookie },
|
||||
});
|
||||
assert.equal(viewerList.status, 200);
|
||||
|
||||
const viewerWrite = await fetch(`${baseUrl}/vcon/files-engine/api/v1/fs/client-alpha/nope.csv`, {
|
||||
method: 'PUT',
|
||||
headers: viewerHeaders,
|
||||
body: 'blocked',
|
||||
});
|
||||
assert.equal(viewerWrite.status, 403);
|
||||
assert.equal(((await viewerWrite.json()) as { code: string }).code, 'INSUFFICIENT_ROLE_PERMISSION');
|
||||
|
||||
// 5. Cross-origin CSRF rejected (403)
|
||||
const csrfRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/fs/client-alpha/csrf.csv`, {
|
||||
method: 'PUT',
|
||||
headers: { Cookie: ownerHeaders.Cookie, Origin: 'https://evil.example', Host: hostHeader },
|
||||
body: 'blocked',
|
||||
});
|
||||
assert.equal(csrfRes.status, 403);
|
||||
assert.equal(((await csrfRes.json()) as { code: string }).code, 'CSRF_REJECTED');
|
||||
|
||||
// 6. Cooperative *.lock protection in URL and JSON body
|
||||
const lockUrlRes = await fetch(
|
||||
`${baseUrl}/vcon/files-engine/api/v1/fs/client-alpha/2026-11-20.csv.lock`,
|
||||
{ method: 'DELETE', headers: ownerHeaders },
|
||||
);
|
||||
assert.equal(lockUrlRes.status, 403);
|
||||
assert.equal(((await lockUrlRes.json()) as { code: string }).code, 'LOCKFILE_PROTECTED');
|
||||
|
||||
const lockBodyRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/rename`, {
|
||||
method: 'PUT',
|
||||
headers: { ...ownerHeaders, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
from: '/client-alpha/2026-11-20.csv.lock',
|
||||
to: '/client-alpha/hijack.csv',
|
||||
}),
|
||||
});
|
||||
assert.equal(lockBodyRes.status, 403);
|
||||
assert.equal(((await lockBodyRes.json()) as { code: string }).code, 'LOCKFILE_PROTECTED');
|
||||
|
||||
// 7. Path traversal & runtime config PATCH blocked
|
||||
const travRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/fs/..%2F..%2Fpackage.json`, {
|
||||
headers: { Cookie: ownerHeaders.Cookie },
|
||||
});
|
||||
assert.equal(travRes.status, 403);
|
||||
assert.equal(((await travRes.json()) as { code: string }).code, 'PATH_OUTSIDE_ROOT');
|
||||
|
||||
const cfgPatchRes = await fetch(`${baseUrl}/vcon/files-engine/api/v1/config`, {
|
||||
method: 'PATCH',
|
||||
headers: { ...ownerHeaders, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ console: true }),
|
||||
});
|
||||
assert.equal(cfgPatchRes.status, 403);
|
||||
assert.equal(((await cfgPatchRes.json()) as { code: string }).code, 'CONFIG_LOCKED');
|
||||
|
||||
// 8. Revoked DB token_version immediately rejects session (401 SESSION_REVOKED)
|
||||
activeDbPool = {
|
||||
query: async (sql: string) => {
|
||||
if (sql.includes('admin_users')) {
|
||||
return { rows: [{ email: 'owner@datastack.dev', token_version: 2 }] };
|
||||
}
|
||||
return { rows: [] };
|
||||
},
|
||||
};
|
||||
const revokedRes = await fetch(`${baseUrl}/vcon/files-engine/_status`, {
|
||||
headers: { Cookie: ownerHeaders.Cookie },
|
||||
});
|
||||
assert.equal(revokedRes.status, 401);
|
||||
assert.equal(((await revokedRes.json()) as { code: string }).code, 'SESSION_REVOKED');
|
||||
} finally {
|
||||
await handle.close();
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
fs.rmSync(tmpRoot, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('FM-REQ-07..09: proxy routing, SW bypass, Sidebar nav, and /vcon/files workspace are cleanly wired', () => {
|
||||
const root = process.cwd();
|
||||
const nextConfig = fs.readFileSync(path.join(root, 'next.config.ts'), 'utf8');
|
||||
const coolifyNginx = fs.readFileSync(path.join(root, 'docker', 'coolify-nginx.conf'), 'utf8');
|
||||
const localNginx = fs.readFileSync(path.join(root, 'nginx', 'default.conf'), 'utf8');
|
||||
const swJs = fs.readFileSync(path.join(root, 'public', 'sw.js'), 'utf8');
|
||||
const sidebar = fs.readFileSync(path.join(root, 'components', 'layout', 'Sidebar.tsx'), 'utf8');
|
||||
const page = fs.readFileSync(path.join(root, 'app', 'vcon', 'files', 'page.tsx'), 'utf8');
|
||||
const workspace = fs.readFileSync(
|
||||
path.join(root, 'components', 'sections', 'FilesManagerWorkspace.tsx'),
|
||||
'utf8',
|
||||
);
|
||||
|
||||
assert.ok(nextConfig.includes("source: '/vcon/files-engine/:path*'"));
|
||||
assert.ok(coolifyNginx.includes('location /vcon/files-engine'));
|
||||
assert.ok(localNginx.includes('location /vcon/files-engine'));
|
||||
assert.ok(swJs.includes("url.pathname.startsWith('/vcon/files-engine/')"));
|
||||
assert.ok(sidebar.includes("{ name: 'Files Manager', href: '/vcon/files', icon: 'FolderOpen' }"));
|
||||
assert.ok(page.includes('FilesManagerWorkspace'));
|
||||
const statusRoute = fs.readFileSync(
|
||||
path.join(root, 'app', 'api', 'files', 'status', 'route.ts'),
|
||||
'utf8',
|
||||
);
|
||||
assert.ok(statusRoute.includes('/vcon/files-engine/_status'));
|
||||
assert.ok(workspace.includes('/api/files/status'));
|
||||
assert.ok(workspace.includes('files-manager-offline'));
|
||||
assert.ok(workspace.includes('files-manager-iframe'));
|
||||
});
|
||||
@@ -444,17 +444,17 @@ test("DEPLOY-BUG-02, 03 & 04: Port 4000 memoizes schema DDLs, hydrates rotated J
|
||||
assert.equal(failedSync, false);
|
||||
assert.equal(ddlCount, 1);
|
||||
|
||||
// 2. Recovery runs 8 DDLs once and hydrates rotated jwt_secret; repeated calls do zero extra DDLs
|
||||
// 2. Recovery runs 10 DDLs once (7 CREATE TABLE + 3 ALTER TABLE) and hydrates rotated jwt_secret; repeated calls do zero extra DDLs
|
||||
failDdl = false;
|
||||
ddlCount = 0;
|
||||
assert.equal(await ingestionServer.syncSettingsFromDatabase(), true);
|
||||
assert.equal(ddlCount, 8);
|
||||
assert.equal(ddlCount, 10);
|
||||
assert.equal(selectCount, 1);
|
||||
assert.equal(ingestionServer.runtimeConfig.jwtSecret, rotatedSecret);
|
||||
|
||||
assert.equal(await ingestionServer.syncSettingsFromDatabase(), true);
|
||||
assert.equal(await ingestionServer.syncSettingsFromDatabase(), true);
|
||||
assert.equal(ddlCount, 8, "Repeated 3s sync ticks must NOT re-run CREATE/ALTER DDL statements");
|
||||
assert.equal(ddlCount, 10, "Repeated 3s sync ticks must NOT re-run CREATE/ALTER DDL statements");
|
||||
assert.equal(selectCount, 3);
|
||||
|
||||
// 3. Graceful shutdown flushes buffered events, quits Redis, and closes PostgreSQL pool cleanly
|
||||
@@ -606,3 +606,41 @@ test("V2-Deploy-Fix-01 (DEPLOY-V2-BUG-01, 02 & 03): middleware permits GET /api/
|
||||
}
|
||||
});
|
||||
|
||||
test("PREPUSH-FIX-01: Nginx preserves upstream real client IP behind Traefik, Dockerfile.web & Compose align runtime/build env vars, and package-lock includes cloudcmd", () => {
|
||||
for (const confRel of ["docker/coolify-nginx.conf", "nginx/default.conf"]) {
|
||||
const conf = readFileSync(resolve(process.cwd(), confRel), "utf8");
|
||||
assert.match(conf, /set_real_ip_from\s+127\.0\.0\.1;/, `${confRel} must trust loopback proxy`);
|
||||
assert.match(conf, /set_real_ip_from\s+172\.16\.0\.0\/12;/, `${confRel} must trust Docker bridge proxy subnet`);
|
||||
assert.match(conf, /real_ip_header\s+X-Forwarded-For;/, `${confRel} must extract real IP from X-Forwarded-For`);
|
||||
assert.match(conf, /real_ip_recursive\s+on;/, `${confRel} must enable real_ip_recursive`);
|
||||
}
|
||||
|
||||
const webDockerfile = readFileSync(resolve(process.cwd(), "docker/Dockerfile.web"), "utf8");
|
||||
assert.match(
|
||||
webDockerfile,
|
||||
/ARG INGESTION_INTERNAL_URL=http:\/\/ingestion:4000[\s\S]*ENV INGESTION_INTERNAL_URL=\$\{INGESTION_INTERNAL_URL\}[\s\S]*RUN npm run build/,
|
||||
"Dockerfile.web builder must set INGESTION_INTERNAL_URL=http://ingestion:4000 before npm run build",
|
||||
);
|
||||
|
||||
const compose = readFileSync(resolve(process.cwd(), "docker/docker-compose.yml"), "utf8");
|
||||
for (const envKey of [
|
||||
"DATASTACK_ALLOW_PRIVATE_URLS",
|
||||
"DATASTACK_MAX_BATCH_ITEMS",
|
||||
"DATASTACK_RECENT_LIMIT",
|
||||
]) {
|
||||
const matches = compose.match(new RegExp(`^\\s+${envKey}:`, "gm")) ?? [];
|
||||
assert.equal(matches.length, 2, `Both web and ingestion must forward ${envKey}`);
|
||||
}
|
||||
assert.match(compose, /^\s+DATASTACK_MAINTENANCE_INTERVAL_MS:/m);
|
||||
assert.match(compose, /^\s+DATASTACK_PROXY_MODE:/m);
|
||||
|
||||
const lockJson = JSON.parse(
|
||||
readFileSync(resolve(process.cwd(), "package-lock.json"), "utf8"),
|
||||
) as {
|
||||
packages?: Record<string, { dependencies?: Record<string, string> }>;
|
||||
};
|
||||
assert.ok(lockJson.packages?.[""]?.dependencies?.cloudcmd, "Root lockfile must declare cloudcmd");
|
||||
assert.ok(lockJson.packages?.["node_modules/cloudcmd"], "package-lock.json must resolve node_modules/cloudcmd");
|
||||
});
|
||||
|
||||
|
||||
@@ -227,6 +227,9 @@ test("R3-Phase-02 BUG-05: Singleton periodic scheduler automatically archives ex
|
||||
const addr = mockS3.address();
|
||||
assert.ok(addr && typeof addr === "object");
|
||||
|
||||
const prevAllowPrivate = process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||
process.env.DATASTACK_ALLOW_PRIVATE_URLS = "true";
|
||||
|
||||
try {
|
||||
// Configure S3 backup + autoPruneDays (cooldown will be reset so scheduled timer runs)
|
||||
await SettingsManager.updateSettingsAsync({
|
||||
@@ -340,6 +343,8 @@ test("R3-Phase-02 BUG-05: Singleton periodic scheduler automatically archives ex
|
||||
assert.equal(unhydratedResult.ran, false);
|
||||
assert.equal(unhydratedResult.skippedReason, "DB_UNHYDRATED");
|
||||
} finally {
|
||||
if (prevAllowPrivate === undefined) delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||
else process.env.DATASTACK_ALLOW_PRIVATE_URLS = prevAllowPrivate;
|
||||
SettingsManager.stopMaintenanceSchedulerForTests();
|
||||
await new Promise<void>((resolve) => mockS3.close(() => resolve()));
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
|
||||
@@ -53,7 +53,6 @@ test("R3-Phase-04 [BUG-11 / ERR-R3-07]: shouldRedirectSimulatorUnauthorized sepa
|
||||
"AUTH_INVALID",
|
||||
"CLIENT_REVOKED",
|
||||
"CLIENT_IP_FORBIDDEN",
|
||||
"CLIENT_UNAUTHORIZED",
|
||||
]) {
|
||||
assert.equal(
|
||||
shouldRedirectSimulatorUnauthorized(401, {
|
||||
@@ -134,12 +133,12 @@ test("R3-Phase-04 [BUG-14]: expired clients are separated in UI helpers, ApiEngi
|
||||
assert.equal(isUiClientExpired("2026-11-19T00:00:00.000Z", nowMs), true);
|
||||
|
||||
const sampleClients = [
|
||||
{ status: "active", isOnline: true, lastConnectedAt: "2026-11-20T11:59:00.000Z", expiresAt: null },
|
||||
{ status: "active", isOnline: false, lastConnectedAt: "2026-11-20T10:00:00.000Z", expiresAt: "2026-12-01T00:00:00.000Z" },
|
||||
{ status: "active", isOnline: false, lastConnectedAt: null, expiresAt: null },
|
||||
{ status: "active", isOnline: false, lastConnectedAt: "2026-11-19T10:00:00.000Z", expiresAt: "2026-11-19T12:00:00.000Z" },
|
||||
{ status: "active", isOnline: true, lastConnectedAt: "2026-11-20T11:58:00.000Z", expiresAt: "2026-11-20T11:00:00.000Z" },
|
||||
{ status: "revoked", isOnline: false, lastConnectedAt: null, expiresAt: "2026-11-10T00:00:00.000Z" },
|
||||
{ status: "active" as const, lastActive: "2026-11-20T11:59:30.000Z", expiresAt: null },
|
||||
{ status: "active" as const, lastActive: "2026-11-20T10:00:00.000Z", expiresAt: "2026-12-01T00:00:00.000Z" },
|
||||
{ status: "active" as const, lastActive: undefined, expiresAt: null },
|
||||
{ status: "active" as const, lastActive: "2026-11-19T10:00:00.000Z", expiresAt: "2026-11-19T12:00:00.000Z" },
|
||||
{ status: "active" as const, lastActive: "2026-11-20T11:59:45.000Z", expiresAt: "2026-11-20T11:00:00.000Z" },
|
||||
{ status: "revoked" as const, lastActive: undefined, expiresAt: "2026-11-10T00:00:00.000Z" },
|
||||
];
|
||||
|
||||
assert.equal(classifyUiClientStatus(sampleClients[0], nowMs), "online");
|
||||
@@ -152,6 +151,7 @@ test("R3-Phase-04 [BUG-14]: expired clients are separated in UI helpers, ApiEngi
|
||||
const summary = summarizeUiClientCounts(sampleClients, nowMs);
|
||||
assert.deepEqual(summary, {
|
||||
total: 6,
|
||||
active: 3,
|
||||
online: 1,
|
||||
offline: 1,
|
||||
pending: 1,
|
||||
@@ -160,12 +160,20 @@ test("R3-Phase-04 [BUG-14]: expired clients are separated in UI helpers, ApiEngi
|
||||
});
|
||||
|
||||
// Live integration: ApiEngine.getDashboardMetrics() & GET /api/prometheus
|
||||
const activeClient = SDKManager.registerClient("r3p4_active_client");
|
||||
const expiredClient = SDKManager.registerClient("r3p4_expired_client", {
|
||||
const activeClient = SDKManager.registerClient({
|
||||
clientId: "r3p4_active_client",
|
||||
name: "R3P4 Active Client",
|
||||
});
|
||||
const expiredClient = SDKManager.registerClient({
|
||||
clientId: "r3p4_expired_client",
|
||||
name: "R3P4 Expired Client",
|
||||
expiresAt: "2020-01-01T00:00:00.000Z",
|
||||
});
|
||||
const revokedClient = SDKManager.registerClient("r3p4_revoked_client");
|
||||
SDKManager.revokeClient(revokedClient.id);
|
||||
const revokedClient = SDKManager.registerClient({
|
||||
clientId: "r3p4_revoked_client",
|
||||
name: "R3P4 Revoked Client",
|
||||
});
|
||||
SDKManager.updateClient(revokedClient.clientId, { status: "revoked" });
|
||||
|
||||
const prevSettings = SettingsManager.getSettings();
|
||||
try {
|
||||
@@ -200,9 +208,9 @@ test("R3-Phase-04 [BUG-14]: expired clients are separated in UI helpers, ApiEngi
|
||||
assert.ok(Number(expiredMatch[1]) >= 1);
|
||||
assert.ok(Number(revokedMatch[1]) >= 1);
|
||||
} finally {
|
||||
SDKManager.deleteClient(activeClient.id);
|
||||
SDKManager.deleteClient(expiredClient.id);
|
||||
SDKManager.deleteClient(revokedClient.id);
|
||||
SDKManager.deleteClient(activeClient.clientId);
|
||||
SDKManager.deleteClient(expiredClient.clientId);
|
||||
SDKManager.deleteClient(revokedClient.clientId);
|
||||
SettingsManager.updateSettings({
|
||||
enablePrometheusMetrics: prevSettings.enablePrometheusMetrics,
|
||||
});
|
||||
@@ -212,37 +220,37 @@ test("R3-Phase-04 [BUG-14]: expired clients are separated in UI helpers, ApiEngi
|
||||
test("R3-Phase-04 [BUG-16]: RBAC role capabilities and UI surface wiring across VconShell and Admin pages", async () => {
|
||||
assert.deepEqual(getVconRoleCapabilities("owner"), {
|
||||
role: "owner",
|
||||
isOwner: true,
|
||||
canAdminSettings: true,
|
||||
canManageTeam: true,
|
||||
canOperate: true,
|
||||
isReadOnly: false,
|
||||
});
|
||||
assert.deepEqual(getVconRoleCapabilities("admin"), {
|
||||
role: "admin",
|
||||
isOwner: false,
|
||||
canAdminSettings: false,
|
||||
canManageTeam: false,
|
||||
canOperate: true,
|
||||
isReadOnly: false,
|
||||
});
|
||||
assert.deepEqual(getVconRoleCapabilities("operator"), {
|
||||
role: "operator",
|
||||
isOwner: false,
|
||||
canAdminSettings: false,
|
||||
canManageTeam: false,
|
||||
canOperate: true,
|
||||
isReadOnly: false,
|
||||
});
|
||||
assert.deepEqual(getVconRoleCapabilities("viewer"), {
|
||||
role: "viewer",
|
||||
isOwner: false,
|
||||
canAdminSettings: false,
|
||||
canManageTeam: false,
|
||||
canOperate: false,
|
||||
isReadOnly: true,
|
||||
});
|
||||
// Fail-closed default for unknown/missing roles
|
||||
assert.deepEqual(getVconRoleCapabilities("unknown_role"), {
|
||||
role: "viewer",
|
||||
isOwner: false,
|
||||
canAdminSettings: false,
|
||||
canManageTeam: false,
|
||||
canOperate: false,
|
||||
isReadOnly: true,
|
||||
});
|
||||
|
||||
@@ -0,0 +1,618 @@
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs/promises";
|
||||
import http from "node:http";
|
||||
import { createRequire } from "node:module";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import zlib from "node:zlib";
|
||||
import { NextRequest } from "next/server";
|
||||
|
||||
process.env.DATASTACK_DISABLE_LISTEN = "1";
|
||||
|
||||
import { GET as auditGet } from "../app/api/audit/route";
|
||||
import { POST as settingsPost } from "../app/api/settings/route";
|
||||
import { POST as webhookTestPost } from "../app/api/settings/webhook-test/route";
|
||||
import { POST as s3TestPost } from "../app/api/storage/s3-test/route";
|
||||
import { SettingsManager } from "../core/adminSettings/settingsManager";
|
||||
import {
|
||||
clearWebhookCooldownCacheForTests,
|
||||
dispatchWebhookAlertAsync,
|
||||
} from "../core/alerts/webhookNotifier";
|
||||
import {
|
||||
createAdminSessionToken,
|
||||
getAdminSessionSecret,
|
||||
} from "../core/auth/adminSession";
|
||||
import {
|
||||
createInitialAdmin,
|
||||
createAdminTeamMemberAsync,
|
||||
resetAdminStateForTests,
|
||||
revokeAdminSessions,
|
||||
updateAdminTeamMemberAsync,
|
||||
deleteAdminTeamMemberAsync,
|
||||
} from "../core/auth/adminUsers";
|
||||
import {
|
||||
clearAuditLogsForTests,
|
||||
listAdminAuditLogsWithStatusAsync,
|
||||
recordAdminAuditLogAsync,
|
||||
} from "../core/auth/auditLog";
|
||||
import { DataBufferEngine } from "../core/data/buffer";
|
||||
import { SDKManager } from "../core/sdk/sdkManager";
|
||||
import { getInternalIngestionHeaders } from "../core/Security/security";
|
||||
import {
|
||||
checkIpAddressAllowed,
|
||||
validateOutboundUrlAsync,
|
||||
validateOutboundUrlSync,
|
||||
} from "../core/Security/ssrfGuard";
|
||||
import { testS3Connection, uploadFileToS3 } from "../core/Storage/s3Archive";
|
||||
import * as instrumentationHook from "../instrumentation";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const filesManager = require("../server/filesManager.js") as {
|
||||
authenticateFilesRequest: (params: {
|
||||
cookieHeader?: string;
|
||||
dbPool?: unknown;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
nowMs?: number;
|
||||
}) => Promise<{
|
||||
valid: boolean;
|
||||
status?: number;
|
||||
code?: string;
|
||||
error?: string;
|
||||
session?: { email: string; role: string; memberId: string; tokenVersion: number };
|
||||
}>;
|
||||
syncInMemoryAdminSessionState: (payload: Record<string, unknown>) => boolean;
|
||||
clearInMemoryAdminSessionStateForTests: () => void;
|
||||
};
|
||||
|
||||
const ingestionServer = require("../server/index.js") as {
|
||||
app: (req: http.IncomingMessage, res: http.ServerResponse) => void;
|
||||
setDbPoolForTests: (pool: unknown) => void;
|
||||
syncSettingsFromDatabase: () => Promise<boolean>;
|
||||
validateOutboundWebhookUrlAsync: (
|
||||
rawUrl: string,
|
||||
options?: Record<string, unknown>,
|
||||
) => Promise<{ allowed: boolean; code?: string; error?: string; url?: string }>;
|
||||
triggerWebhookAlert: (
|
||||
payload: Record<string, unknown>,
|
||||
options?: Record<string, unknown>,
|
||||
) => Promise<{ dispatched: boolean; status?: number; code?: string; reason?: string; error?: string }>;
|
||||
clearWebhookCooldownCacheForTests: () => void;
|
||||
stopTimers: () => void;
|
||||
};
|
||||
|
||||
const ORIGINAL_ENV = { ...process.env };
|
||||
|
||||
function restoreEnv(): void {
|
||||
for (const key of Object.keys(process.env)) {
|
||||
if (!(key in ORIGINAL_ENV)) {
|
||||
delete process.env[key];
|
||||
}
|
||||
}
|
||||
Object.assign(process.env, ORIGINAL_ENV);
|
||||
}
|
||||
|
||||
async function createOwnerCookie(email = "r4p1-owner@datastack.dev"): Promise<string> {
|
||||
resetAdminStateForTests();
|
||||
await createInitialAdmin(email, "OwnerPassword123!");
|
||||
const secret = getAdminSessionSecret();
|
||||
const token = await createAdminSessionToken(
|
||||
email,
|
||||
secret,
|
||||
Date.now() + 3600_000,
|
||||
1,
|
||||
"owner",
|
||||
"owner-1",
|
||||
);
|
||||
return `datastack_admin_session=${token}`;
|
||||
}
|
||||
|
||||
test.after(async () => {
|
||||
SettingsManager.stopMaintenanceSchedulerForTests();
|
||||
SDKManager.stopActivityFlushTimerForTests();
|
||||
ingestionServer.stopTimers();
|
||||
ingestionServer.setDbPoolForTests(null);
|
||||
filesManager.clearInMemoryAdminSessionStateForTests();
|
||||
await DataBufferEngine.shutdown();
|
||||
clearAuditLogsForTests();
|
||||
resetAdminStateForTests();
|
||||
restoreEnv();
|
||||
});
|
||||
|
||||
test("R4-Phase-01 [AUDIT-01 / ERR-R4-01]: ssrfGuard blocks loopback, RFC1918, metadata, internal hostnames, non-HTTP(S), DNS rebinding & redirects while respecting explicit opt-in", async () => {
|
||||
delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||
|
||||
const blockedTargets = [
|
||||
"http://127.0.0.1:4000/internal/settings-sync",
|
||||
"http://localhost:3000/api/health",
|
||||
"http://sub.localhost/hook",
|
||||
"http://[::1]:4000/",
|
||||
"http://[::ffff:127.0.0.1]/secret",
|
||||
"http://10.0.0.5:9000/bucket",
|
||||
"http://172.16.4.2/hook",
|
||||
"http://192.168.1.50/hook",
|
||||
"http://0.0.0.0/",
|
||||
"http://ingestion:4000/internal",
|
||||
"http://postgres.internal:5432/",
|
||||
"http://169.254.169.254/latest/meta-data/iam/security-credentials/",
|
||||
"http://metadata.google.internal/computeMetadata/v1/",
|
||||
"file:///etc/passwd",
|
||||
"ftp://files.example.com/archive",
|
||||
"https://user:pass@hooks.example.com/hook",
|
||||
];
|
||||
|
||||
for (const target of blockedTargets) {
|
||||
const res = validateOutboundUrlSync(target, { fieldName: "Outbound URL" });
|
||||
assert.equal(res.allowed, false, `Expected ${target} to be blocked`);
|
||||
assert.equal(res.code, "SSRF_BLOCKED_HOST", `Expected SSRF_BLOCKED_HOST for ${target}`);
|
||||
}
|
||||
|
||||
// Public HTTPS host passes sync + safe DNS lookup
|
||||
const okPublic = await validateOutboundUrlAsync("https://hooks.example.com/v1/alert", {
|
||||
fieldName: "Webhook URL",
|
||||
dnsLookup: async () => [{ address: "93.184.216.34", family: 4 }],
|
||||
});
|
||||
assert.equal(okPublic.allowed, true);
|
||||
|
||||
// DNS rebinding hostname resolving to 127.0.0.1 or 169.254.169.254 is blocked
|
||||
const rebindLoopback = await validateOutboundUrlAsync("https://rebind.example.com/hook", {
|
||||
fieldName: "Webhook URL",
|
||||
dnsLookup: async () => [{ address: "127.0.0.1", family: 4 }],
|
||||
});
|
||||
assert.equal(rebindLoopback.allowed, false);
|
||||
assert.equal(rebindLoopback.code, "SSRF_BLOCKED_HOST");
|
||||
|
||||
const rebindMeta = await validateOutboundUrlAsync("https://rebind-meta.example.com/hook", {
|
||||
fieldName: "Webhook URL",
|
||||
dnsLookup: async () => [{ address: "169.254.169.254", family: 4 }],
|
||||
});
|
||||
assert.equal(rebindMeta.allowed, false);
|
||||
assert.equal(rebindMeta.code, "SSRF_BLOCKED_HOST");
|
||||
|
||||
// Explicit opt-in allows RFC1918/loopback/single-label internal LAN hosts, YET ALWAYS blocks cloud metadata 169.254.169.254 & metadata.google.internal
|
||||
assert.equal(checkIpAddressAllowed("127.0.0.1", true).allowed, true);
|
||||
assert.equal(checkIpAddressAllowed("192.168.1.25", true).allowed, true);
|
||||
assert.equal(checkIpAddressAllowed("169.254.169.254", true).allowed, false);
|
||||
assert.equal(checkIpAddressAllowed("fe80::1", true).allowed, false);
|
||||
assert.equal(
|
||||
validateOutboundUrlSync("http://minio:9000", {
|
||||
allowPrivate: true,
|
||||
}).allowed,
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
validateOutboundUrlSync("http://169.254.169.254/latest/meta-data/", {
|
||||
allowPrivate: true,
|
||||
}).allowed,
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
validateOutboundUrlSync("http://metadata.google.internal/", {
|
||||
allowPrivate: true,
|
||||
}).allowed,
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test("R4-Phase-01 [AUDIT-01 / ERR-R4-01]: API routes (/api/settings, /api/settings/webhook-test, /api/storage/s3-test), Port 4000 triggerWebhookAlert & redirect:manual block SSRF", async () => {
|
||||
delete process.env.DATABASE_URL;
|
||||
delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||
clearWebhookCooldownCacheForTests();
|
||||
ingestionServer.clearWebhookCooldownCacheForTests();
|
||||
|
||||
const cookie = await createOwnerCookie("r4-ssrf-owner@datastack.dev");
|
||||
|
||||
// 1. POST /api/settings rejects private webhookUrl & private s3Endpoint with 400 SSRF_BLOCKED_HOST
|
||||
const badWebhookSave = await settingsPost(
|
||||
new NextRequest("http://localhost:3000/api/settings", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", cookie },
|
||||
body: JSON.stringify({ webhookUrl: "http://127.0.0.1:4000/internal/client-sync" }),
|
||||
}),
|
||||
);
|
||||
assert.equal(badWebhookSave.status, 400);
|
||||
assert.equal((await badWebhookSave.json()).code, "SSRF_BLOCKED_HOST");
|
||||
|
||||
const badS3Save = await settingsPost(
|
||||
new NextRequest("http://localhost:3000/api/settings", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", cookie },
|
||||
body: JSON.stringify({ s3Endpoint: "http://169.254.169.254/latest/meta-data" }),
|
||||
}),
|
||||
);
|
||||
assert.equal(badS3Save.status, 400);
|
||||
assert.equal((await badS3Save.json()).code, "SSRF_BLOCKED_HOST");
|
||||
|
||||
// 2. POST /api/settings/webhook-test rejects loopback/metadata with 400 SSRF_BLOCKED_HOST
|
||||
const badWebhookTest = await webhookTestPost(
|
||||
new NextRequest("http://localhost:3000/api/settings/webhook-test", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", cookie },
|
||||
body: JSON.stringify({ webhookUrl: "http://localhost:4000/health" }),
|
||||
}),
|
||||
);
|
||||
assert.equal(badWebhookTest.status, 400);
|
||||
assert.equal((await badWebhookTest.json()).code, "SSRF_BLOCKED_HOST");
|
||||
|
||||
// 3. POST /api/storage/s3-test rejects internal/RFC1918 host with 400 SSRF_BLOCKED_HOST
|
||||
const badS3Test = await s3TestPost(
|
||||
new NextRequest("http://localhost:3000/api/storage/s3-test", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", cookie },
|
||||
body: JSON.stringify({
|
||||
s3Endpoint: "http://10.0.0.12:9000",
|
||||
s3Bucket: "test-bucket",
|
||||
s3AccessKeyId: "AKIA_TEST",
|
||||
s3SecretAccessKey: "secret_key_test",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(badS3Test.status, 400);
|
||||
assert.equal((await badS3Test.json()).code, "SSRF_BLOCKED_HOST");
|
||||
|
||||
// 4. Port 4000 triggerWebhookAlert & validateOutboundWebhookUrlAsync block loopback & metadata
|
||||
const p4Blocked = await ingestionServer.triggerWebhookAlert(
|
||||
{
|
||||
event: "CLIENT_IP_FORBIDDEN",
|
||||
severity: "warning",
|
||||
title: "Probe",
|
||||
message: "Test",
|
||||
},
|
||||
{
|
||||
enableWebhookAlerts: true,
|
||||
webhookUrl: "http://127.0.0.1:4000/internal/client-sync",
|
||||
bypassCooldown: true,
|
||||
},
|
||||
);
|
||||
assert.equal(p4Blocked.dispatched, false);
|
||||
assert.equal(p4Blocked.code, "SSRF_BLOCKED_HOST");
|
||||
|
||||
// 5. redirect: 'manual' blocks HTTP 302 redirects across webhookNotifier, S3, and Port 4000
|
||||
let targetHitAfterRedirect = false;
|
||||
const redirectServer = http.createServer((req, res) => {
|
||||
if (req.url && req.url.startsWith("/bounce")) {
|
||||
res.writeHead(302, { Location: "http://169.254.169.254/latest/meta-data/" });
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
targetHitAfterRedirect = true;
|
||||
res.writeHead(200);
|
||||
res.end("ok");
|
||||
});
|
||||
await new Promise<void>((resolve) => redirectServer.listen(0, "127.0.0.1", resolve));
|
||||
const addr = redirectServer.address() as AddressInfo;
|
||||
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "datastack-r4-ssrf-redir-"));
|
||||
const gzPath = path.join(tmpDir, "2026-11-20.csv.gz");
|
||||
await fs.writeFile(gzPath, zlib.gzipSync(Buffer.from("clientId,eventId,timestamp,payload\n")));
|
||||
|
||||
try {
|
||||
process.env.DATASTACK_ALLOW_PRIVATE_URLS = "true";
|
||||
|
||||
const whRedirect = await dispatchWebhookAlertAsync(
|
||||
{
|
||||
event: "TEST_ALERT",
|
||||
severity: "info",
|
||||
title: "Redirect check",
|
||||
message: "Must not follow 302",
|
||||
},
|
||||
{
|
||||
enabled: true,
|
||||
webhookUrl: `http://127.0.0.1:${addr.port}/bounce`,
|
||||
webhookSecret: "",
|
||||
cooldownSec: 0,
|
||||
},
|
||||
{ bypassCooldown: true },
|
||||
);
|
||||
assert.equal(whRedirect.ok, false);
|
||||
assert.equal(whRedirect.code, "SSRF_BLOCKED_HOST");
|
||||
|
||||
const s3ProbeRedir = await testS3Connection({
|
||||
endpoint: `http://127.0.0.1:${addr.port}/bounce`,
|
||||
region: "auto",
|
||||
bucket: "bkt",
|
||||
prefix: "",
|
||||
accessKeyId: "AKIA",
|
||||
secretAccessKey: "SEC",
|
||||
});
|
||||
assert.equal(s3ProbeRedir.ok, false);
|
||||
assert.equal(s3ProbeRedir.code, "SSRF_BLOCKED_HOST");
|
||||
|
||||
const s3PutRedir = await uploadFileToS3(
|
||||
{
|
||||
endpoint: `http://127.0.0.1:${addr.port}/bounce`,
|
||||
region: "auto",
|
||||
bucket: "bkt",
|
||||
prefix: "",
|
||||
accessKeyId: "AKIA",
|
||||
secretAccessKey: "SEC",
|
||||
},
|
||||
gzPath,
|
||||
"c/2026-11-20.csv.gz",
|
||||
);
|
||||
assert.equal(s3PutRedir.ok, false);
|
||||
assert.equal(s3PutRedir.code, "SSRF_BLOCKED_HOST");
|
||||
|
||||
const p4Redirect = await ingestionServer.triggerWebhookAlert(
|
||||
{
|
||||
event: "DISK_SPACE_LOW",
|
||||
severity: "warning",
|
||||
title: "Redirect check",
|
||||
message: "Must not follow 302",
|
||||
},
|
||||
{
|
||||
enableWebhookAlerts: true,
|
||||
webhookUrl: `http://127.0.0.1:${addr.port}/bounce`,
|
||||
allowPrivate: true,
|
||||
bypassCooldown: true,
|
||||
},
|
||||
);
|
||||
assert.equal(p4Redirect.dispatched, false);
|
||||
assert.equal(p4Redirect.code, "SSRF_BLOCKED_HOST");
|
||||
assert.equal(targetHitAfterRedirect, false);
|
||||
} finally {
|
||||
delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||
await new Promise<void>((resolve) => redirectServer.close(() => resolve()));
|
||||
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("R4-Phase-01 [AUDIT-03 / ERR-R4-02]: Port 4000 ensureDatabaseSchema creates all 7 tables including admin_team_members and admin_audit_logs", async () => {
|
||||
const executedSql: string[] = [];
|
||||
const mockPool = {
|
||||
async query(sql: string) {
|
||||
executedSql.push(sql);
|
||||
if (sql.includes("FROM admin_settings")) {
|
||||
return { rowCount: 0, rows: [] };
|
||||
}
|
||||
return { rowCount: 0, rows: [] };
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
ingestionServer.setDbPoolForTests(mockPool);
|
||||
const synced = await ingestionServer.syncSettingsFromDatabase();
|
||||
assert.equal(synced, true);
|
||||
|
||||
const createStatements = executedSql.filter((s) => /CREATE TABLE IF NOT EXISTS/i.test(s));
|
||||
assert.equal(createStatements.length, 7, "Port 4000 must create all 7 tables");
|
||||
const joined = createStatements.join("\n");
|
||||
for (const tableName of [
|
||||
"client_sdks",
|
||||
"admin_settings",
|
||||
"deleted_clients",
|
||||
"admin_users",
|
||||
"admin_login_attempts",
|
||||
"admin_team_members",
|
||||
"admin_audit_logs",
|
||||
]) {
|
||||
assert.ok(
|
||||
joined.includes(tableName),
|
||||
`Port 4000 schema missing table ${tableName}`,
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
ingestionServer.setDbPoolForTests(null);
|
||||
}
|
||||
});
|
||||
|
||||
test("R4-Phase-01 [AUDIT-04 / ERR-R4-03]: Next.js instrumentation.register() boots maintenance scheduler unattended on nodejs runtime and survives unreachable DB", async () => {
|
||||
delete process.env.DATABASE_URL;
|
||||
const prevDisable = process.env.DATASTACK_DISABLE_LISTEN;
|
||||
const prevRuntime = process.env.NEXT_RUNTIME;
|
||||
SettingsManager.stopMaintenanceSchedulerForTests();
|
||||
|
||||
try {
|
||||
// 1. Edge runtime does NOT start Node maintenance scheduler
|
||||
process.env.NEXT_RUNTIME = "edge";
|
||||
delete process.env.DATASTACK_DISABLE_LISTEN;
|
||||
await instrumentationHook.register();
|
||||
assert.equal(SettingsManager.isMaintenanceSchedulerActive(), false);
|
||||
|
||||
// 2. Node.js runtime starts maintenance scheduler automatically on boot
|
||||
process.env.NEXT_RUNTIME = "nodejs";
|
||||
await instrumentationHook.register();
|
||||
assert.equal(SettingsManager.isMaintenanceSchedulerActive(), true);
|
||||
|
||||
// 3. Even if DB fails during boot hydration, bootstrapMaintenanceSchedulerOnBoot catches error cleanly without crashing
|
||||
SettingsManager.stopMaintenanceSchedulerForTests();
|
||||
process.env.DATABASE_URL = "postgresql://user:pass@127.0.0.1:1/unreachable";
|
||||
const bootRes = await SettingsManager.bootstrapMaintenanceSchedulerOnBoot();
|
||||
assert.equal(bootRes.schedulerActive, true);
|
||||
assert.equal(bootRes.hydrated, false);
|
||||
} finally {
|
||||
SettingsManager.stopMaintenanceSchedulerForTests();
|
||||
delete process.env.DATABASE_URL;
|
||||
if (prevDisable === undefined) delete process.env.DATASTACK_DISABLE_LISTEN;
|
||||
else process.env.DATASTACK_DISABLE_LISTEN = prevDisable;
|
||||
if (prevRuntime === undefined) delete process.env.NEXT_RUNTIME;
|
||||
else process.env.NEXT_RUNTIME = prevRuntime;
|
||||
}
|
||||
});
|
||||
|
||||
test("R4-Phase-01 [AUDIT-09 / ERR-R4-04 & ERR-R4-05]: Fail-loud audit DB degraded warnings + no-DB Files Manager session revocation sync", async () => {
|
||||
delete process.env.DATABASE_URL;
|
||||
clearAuditLogsForTests();
|
||||
filesManager.clearInMemoryAdminSessionStateForTests();
|
||||
|
||||
const cookie = await createOwnerCookie("r4-audit-owner@datastack.dev");
|
||||
const globals = globalThis as typeof globalThis & {
|
||||
__dataStackCorePool?: unknown;
|
||||
};
|
||||
const prevPool = globals.__dataStackCorePool;
|
||||
|
||||
// 1. Simulate PostgreSQL failure on audit write + read -> must log console.warn and expose degraded metadata & header
|
||||
const failingPool = {
|
||||
async query(sql: string) {
|
||||
if (/CREATE TABLE|ALTER TABLE/i.test(sql)) {
|
||||
return { rowCount: 0, rows: [] };
|
||||
}
|
||||
if (sql.includes("FROM admin_users")) {
|
||||
return {
|
||||
rowCount: 1,
|
||||
rows: [
|
||||
{
|
||||
email: "r4-audit-owner@datastack.dev",
|
||||
created_at: new Date(),
|
||||
token_version: 1,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (sql.includes("admin_audit_logs")) {
|
||||
throw new Error("simulated PG outage");
|
||||
}
|
||||
return { rowCount: 0, rows: [] };
|
||||
},
|
||||
};
|
||||
|
||||
try {
|
||||
process.env.DATABASE_URL = "postgresql://mock:mock@127.0.0.1:5432/mock";
|
||||
globals.__dataStackCorePool = failingPool;
|
||||
const warnings: string[] = [];
|
||||
const origWarn = console.warn;
|
||||
console.warn = (...args: unknown[]) => {
|
||||
warnings.push(args.map((a) => String(a)).join(" "));
|
||||
};
|
||||
try {
|
||||
await recordAdminAuditLogAsync({
|
||||
actorEmail: "r4-audit-owner@datastack.dev",
|
||||
actorRole: "owner",
|
||||
action: "SETTINGS_UPDATED",
|
||||
target: "admin_settings",
|
||||
});
|
||||
const listResult = await listAdminAuditLogsWithStatusAsync({ limit: 10 });
|
||||
assert.equal(listResult.storageMode, "memory");
|
||||
assert.equal(listResult.degraded, true);
|
||||
assert.equal(listResult.code, "AUDIT_STORAGE_DEGRADED");
|
||||
assert.match(listResult.warning || "", /PostgreSQL/i);
|
||||
assert.ok(
|
||||
warnings.some((w) => w.includes("[DataStack Audit]")),
|
||||
"Expected console.warn fail-loud audit log",
|
||||
);
|
||||
|
||||
// Verify GET /api/audit surfaces degraded status & header
|
||||
const apiRes = await auditGet(
|
||||
new NextRequest("http://localhost:3000/api/audit?limit=10", {
|
||||
headers: { cookie },
|
||||
}),
|
||||
);
|
||||
assert.equal(apiRes.status, 200);
|
||||
assert.equal(apiRes.headers.get("X-DataStack-Audit-Degraded"), "1");
|
||||
const apiJson = await apiRes.json();
|
||||
assert.equal(apiJson.degraded, true);
|
||||
assert.equal(apiJson.code, "AUDIT_STORAGE_DEGRADED");
|
||||
assert.equal(apiJson.storageMode, "memory");
|
||||
} finally {
|
||||
console.warn = origWarn;
|
||||
}
|
||||
} finally {
|
||||
delete process.env.DATABASE_URL;
|
||||
globals.__dataStackCorePool = prevPool;
|
||||
}
|
||||
|
||||
// 2. Verify no-DB (!dbPool) Files Manager session revocation via Port 4000 /internal/admin-session-sync bridge
|
||||
const testBridgeServer = http.createServer(ingestionServer.app);
|
||||
await new Promise<void>((resolve) => testBridgeServer.listen(0, "127.0.0.1", resolve));
|
||||
const bridgePort = (testBridgeServer.address() as AddressInfo).port;
|
||||
process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${bridgePort}`;
|
||||
|
||||
try {
|
||||
// Reject unauthenticated POST /internal/admin-session-sync
|
||||
const unauthSync = await fetch(
|
||||
`http://127.0.0.1:${bridgePort}/internal/admin-session-sync`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ memberId: "owner-1", minTokenVersion: 99 }),
|
||||
},
|
||||
);
|
||||
assert.equal(unauthSync.status, 401);
|
||||
|
||||
// Owner session v1 works before logout/revocation
|
||||
const secret = getAdminSessionSecret();
|
||||
const ownerTokenV1 = await createAdminSessionToken(
|
||||
"r4-audit-owner@datastack.dev",
|
||||
secret,
|
||||
Date.now() + 3600_000,
|
||||
1,
|
||||
"owner",
|
||||
"owner-1",
|
||||
);
|
||||
const beforeLogout = await filesManager.authenticateFilesRequest({
|
||||
cookieHeader: `datastack_admin_session=${ownerTokenV1}`,
|
||||
dbPool: null,
|
||||
});
|
||||
assert.equal(beforeLogout.valid, true);
|
||||
|
||||
// Calling revokeAdminSessions syncs minTokenVersion=2 to Port 4000 bridge -> v1 Files Manager cookie is rejected
|
||||
const nextVer = await revokeAdminSessions({
|
||||
memberId: "owner-1",
|
||||
email: "r4-audit-owner@datastack.dev",
|
||||
});
|
||||
assert.ok(nextVer >= 2);
|
||||
const afterLogout = await filesManager.authenticateFilesRequest({
|
||||
cookieHeader: `datastack_admin_session=${ownerTokenV1}`,
|
||||
dbPool: null,
|
||||
});
|
||||
assert.equal(afterLogout.valid, false);
|
||||
assert.equal(afterLogout.status, 401);
|
||||
assert.equal(afterLogout.code, "SESSION_REVOKED");
|
||||
|
||||
// Team member disable/delete also revokes no-DB Files Manager session immediately
|
||||
const opMember = await createAdminTeamMemberAsync({
|
||||
email: "r4-op@datastack.dev",
|
||||
password: "OperatorPassword123!",
|
||||
role: "operator",
|
||||
});
|
||||
const opTokenV1 = await createAdminSessionToken(
|
||||
opMember.email,
|
||||
secret,
|
||||
Date.now() + 3600_000,
|
||||
opMember.tokenVersion,
|
||||
opMember.role,
|
||||
opMember.id,
|
||||
);
|
||||
const opBefore = await filesManager.authenticateFilesRequest({
|
||||
cookieHeader: `datastack_admin_session=${opTokenV1}`,
|
||||
dbPool: null,
|
||||
});
|
||||
assert.equal(opBefore.valid, true);
|
||||
|
||||
await updateAdminTeamMemberAsync(opMember.id, { status: "disabled" });
|
||||
const opAfterDisable = await filesManager.authenticateFilesRequest({
|
||||
cookieHeader: `datastack_admin_session=${opTokenV1}`,
|
||||
dbPool: null,
|
||||
});
|
||||
assert.equal(opAfterDisable.valid, false);
|
||||
assert.equal(opAfterDisable.code, "SESSION_REVOKED");
|
||||
|
||||
await deleteAdminTeamMemberAsync(opMember.id);
|
||||
|
||||
// Also verify authenticated /internal/admin-session-sync endpoint returns 200 & synced=true
|
||||
const authSyncRes = await fetch(
|
||||
`http://127.0.0.1:${bridgePort}/internal/admin-session-sync`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...getInternalIngestionHeaders(),
|
||||
},
|
||||
body: JSON.stringify({
|
||||
memberId: "adm_manual_test",
|
||||
email: "manual@datastack.dev",
|
||||
minTokenVersion: 5,
|
||||
disabled: true,
|
||||
}),
|
||||
},
|
||||
);
|
||||
assert.equal(authSyncRes.status, 200);
|
||||
const authSyncJson = (await authSyncRes.json()) as { success: boolean; synced: boolean };
|
||||
assert.equal(authSyncJson.success, true);
|
||||
assert.equal(authSyncJson.synced, true);
|
||||
} finally {
|
||||
delete process.env.INGESTION_INTERNAL_URL;
|
||||
filesManager.clearInMemoryAdminSessionStateForTests();
|
||||
await new Promise<void>((resolve) => testBridgeServer.close(() => resolve()));
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,777 @@
|
||||
import assert from "node:assert/strict";
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import fsp from "node:fs/promises";
|
||||
import http from "node:http";
|
||||
import { createRequire } from "node:module";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import zlib from "node:zlib";
|
||||
import { NextRequest } from "next/server";
|
||||
|
||||
process.env.DATASTACK_DISABLE_LISTEN = "1";
|
||||
|
||||
import { GET as auditGet } from "../app/api/audit/route";
|
||||
import { PATCH as clientPatch } from "../app/api/clients/[id]/route";
|
||||
import { GET as dataGet } from "../app/api/data/route";
|
||||
import { POST as eventsPost } from "../app/api/events/route";
|
||||
import { GET as exportGet } from "../app/api/export/route";
|
||||
import { SettingsManager } from "../core/adminSettings/settingsManager";
|
||||
import {
|
||||
createAdminSessionToken,
|
||||
getAdminSessionSecret,
|
||||
} from "../core/auth/adminSession";
|
||||
import {
|
||||
createInitialAdmin,
|
||||
resetAdminStateForTests,
|
||||
} from "../core/auth/adminUsers";
|
||||
import {
|
||||
clearAuditLogsForTests,
|
||||
listAdminAuditLogsWithStatusAsync,
|
||||
} from "../core/auth/auditLog";
|
||||
import { DataBufferEngine } from "../core/data/buffer";
|
||||
import { SDKManager } from "../core/sdk/sdkManager";
|
||||
import { getInternalIngestionHeaders } from "../core/Security/security";
|
||||
import { CSVStorageManager } from "../core/Storage/csvStorage";
|
||||
import { uploadFileToS3 } from "../core/Storage/s3Archive";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const express = require("express");
|
||||
const { Server } = require("socket.io");
|
||||
const { io: ioClient } = require("socket.io-client");
|
||||
const { setupSocketHandlers } = require("../server/socket/handlers");
|
||||
const filesManager = require("../server/filesManager.js") as {
|
||||
createFilesSecurityMiddleware: (opts: {
|
||||
rootDir: string;
|
||||
getDbPool?: () => unknown;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}) => (req: http.IncomingMessage, res: http.ServerResponse, next: () => void) => void;
|
||||
flushPendingFilesAuditLogsForTests: () => Promise<void>;
|
||||
clearInMemoryAdminSessionStateForTests: () => void;
|
||||
};
|
||||
|
||||
const ingestionServer = require("../server/index.js") as {
|
||||
app: (req: http.IncomingMessage, res: http.ServerResponse) => void;
|
||||
io: {
|
||||
on: (ev: string, cb: (socket: unknown) => void) => void;
|
||||
close: (cb?: () => void) => void;
|
||||
};
|
||||
server: http.Server;
|
||||
stopTimers: () => void;
|
||||
setDbPoolForTests: (pool: unknown) => void;
|
||||
writer: { baseDir: string };
|
||||
};
|
||||
|
||||
const ORIGINAL_ENV = { ...process.env };
|
||||
const ORIGINAL_STORAGE_DIR = CSVStorageManager.baseStorageDir;
|
||||
|
||||
function restoreEnv(): void {
|
||||
for (const key of Object.keys(process.env)) {
|
||||
if (!(key in ORIGINAL_ENV)) {
|
||||
delete process.env[key];
|
||||
}
|
||||
}
|
||||
Object.assign(process.env, ORIGINAL_ENV);
|
||||
}
|
||||
|
||||
async function createOwnerCookie(
|
||||
email = "r4p2-owner@datastack.dev",
|
||||
): Promise<string> {
|
||||
resetAdminStateForTests();
|
||||
await createInitialAdmin(email, "OwnerPassword123!");
|
||||
const secret = getAdminSessionSecret();
|
||||
const token = await createAdminSessionToken(
|
||||
email,
|
||||
secret,
|
||||
Date.now() + 3600_000,
|
||||
1,
|
||||
"owner",
|
||||
"owner-1",
|
||||
);
|
||||
return `datastack_admin_session=${token}`;
|
||||
}
|
||||
|
||||
test.after(async () => {
|
||||
CSVStorageManager.baseStorageDir = ORIGINAL_STORAGE_DIR;
|
||||
CSVStorageManager.clearRowCountCache();
|
||||
CSVStorageManager.clearQuotaCaches();
|
||||
SettingsManager.stopMaintenanceSchedulerForTests();
|
||||
SDKManager.stopActivityFlushTimerForTests();
|
||||
ingestionServer.stopTimers();
|
||||
ingestionServer.setDbPoolForTests(null);
|
||||
filesManager.clearInMemoryAdminSessionStateForTests();
|
||||
await DataBufferEngine.shutdown();
|
||||
clearAuditLogsForTests();
|
||||
resetAdminStateForTests();
|
||||
restoreEnv();
|
||||
});
|
||||
|
||||
test("R4-Phase-02 AUDIT-05 / ERR-R4-06: HTTP & Socket.IO enforce DATASTACK_MAX_BATCH_ITEMS cap", async () => {
|
||||
const prevStorage = CSVStorageManager.baseStorageDir;
|
||||
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "ds-r4p2-batch-"));
|
||||
CSVStorageManager.baseStorageDir = tmpDir;
|
||||
CSVStorageManager.clearRowCountCache();
|
||||
CSVStorageManager.clearQuotaCaches();
|
||||
process.env.DATASTACK_MAX_BATCH_ITEMS = "3";
|
||||
|
||||
let httpServer: http.Server | null = null;
|
||||
let io: InstanceType<typeof Server> | null = null;
|
||||
let socket: ReturnType<typeof ioClient> | null = null;
|
||||
|
||||
const runId = `${Date.now()}_${crypto.randomBytes(3).toString("hex")}`;
|
||||
|
||||
try {
|
||||
const client = await SDKManager.registerClientAsync({
|
||||
clientId: "r4p2-batch-client",
|
||||
name: "Batch Limit Client",
|
||||
rateLimit: 500,
|
||||
});
|
||||
|
||||
// 1. HTTP POST /api/events rejects 4-item array before per-item checks
|
||||
const overBatchReq = new NextRequest("http://127.0.0.1:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${client.token}`,
|
||||
},
|
||||
body: JSON.stringify([
|
||||
{ eventId: `${runId}-b1`, data: { step: 1 } },
|
||||
{ eventId: `${runId}-b2`, data: { step: 2 } },
|
||||
{ eventId: `${runId}-b3`, data: { step: 3 } },
|
||||
{ eventId: `${runId}-b4`, data: { step: 4 } },
|
||||
]),
|
||||
});
|
||||
const overRes = await eventsPost(overBatchReq);
|
||||
assert.equal(overRes.status, 400);
|
||||
const overJson = await overRes.json();
|
||||
assert.equal(overJson.success, false);
|
||||
assert.equal(overJson.code, "BATCH_LIMIT_EXCEEDED");
|
||||
assert.equal(overJson.rejected, 4);
|
||||
assert.equal(overJson.maxBatchItems, 3);
|
||||
|
||||
// 2. HTTP POST /api/events accepts 3-item records array
|
||||
const okBatchReq = new NextRequest("http://127.0.0.1:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${client.token}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
records: [
|
||||
{ eventId: `${runId}-b1`, data: { step: 1 } },
|
||||
{ eventId: `${runId}-b2`, data: { step: 2 } },
|
||||
{ eventId: `${runId}-b3`, data: { step: 3 } },
|
||||
],
|
||||
}),
|
||||
});
|
||||
const okRes = await eventsPost(okBatchReq);
|
||||
assert.equal(okRes.status, 200);
|
||||
const okJson = await okRes.json();
|
||||
assert.equal(okJson.success, true);
|
||||
assert.equal(okJson.accepted, 3);
|
||||
assert.equal(okJson.rejected, 0);
|
||||
|
||||
// 3. Socket.IO batch_sync rejects 4-item array and accepts 3-item array
|
||||
httpServer = http.createServer();
|
||||
io = new Server(httpServer);
|
||||
const bufferQueue: unknown[] = [];
|
||||
setupSocketHandlers(io, bufferQueue, null, {
|
||||
getRuntimeConfig: () => ({
|
||||
jwtSecret: "datastack-jwt-default-secret-key-32-chars-long",
|
||||
maxEventsPerSec: 500,
|
||||
maxPayloadSizeBytes: 100 * 1024,
|
||||
enableIdempotency: true,
|
||||
}),
|
||||
verifyClientInDb: async () => ({ allowed: true, rateLimit: 500 }),
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve) => httpServer!.listen(0, "127.0.0.1", resolve));
|
||||
const port = (httpServer.address() as AddressInfo).port;
|
||||
|
||||
socket = ioClient(`http://127.0.0.1:${port}`, {
|
||||
auth: { token: client.token },
|
||||
transports: ["websocket"],
|
||||
});
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
socket!.on("connect", () => resolve());
|
||||
socket!.on("connect_error", reject);
|
||||
});
|
||||
|
||||
const overAck = await new Promise<Record<string, unknown>>((resolve) => {
|
||||
socket!.emit(
|
||||
"batch_sync",
|
||||
[
|
||||
{ eventId: "s1", data: { i: 1 } },
|
||||
{ eventId: "s2", data: { i: 2 } },
|
||||
{ eventId: "s3", data: { i: 3 } },
|
||||
{ eventId: "s4", data: { i: 4 } },
|
||||
],
|
||||
resolve,
|
||||
);
|
||||
});
|
||||
assert.equal(overAck.success, false);
|
||||
assert.equal(overAck.code, "BATCH_LIMIT_EXCEEDED");
|
||||
assert.equal(overAck.rejected, 4);
|
||||
assert.equal(overAck.maxBatchItems, 3);
|
||||
assert.equal(bufferQueue.length, 0);
|
||||
|
||||
const okAck = await new Promise<Record<string, unknown>>((resolve) => {
|
||||
socket!.emit(
|
||||
"batch_sync",
|
||||
[
|
||||
{ eventId: `${runId}-s1`, data: { i: 1 } },
|
||||
{ eventId: `${runId}-s2`, data: { i: 2 } },
|
||||
{ eventId: `${runId}-s3`, data: { i: 3 } },
|
||||
],
|
||||
resolve,
|
||||
);
|
||||
});
|
||||
assert.equal(okAck.success, true);
|
||||
assert.equal(okAck.processed, 3);
|
||||
assert.equal(okAck.rejected, 0);
|
||||
assert.equal(bufferQueue.length, 3);
|
||||
|
||||
// 4. Lower bound clamping to 1 when DATASTACK_MAX_BATCH_ITEMS=0
|
||||
process.env.DATASTACK_MAX_BATCH_ITEMS = "0";
|
||||
const clampAck = await new Promise<Record<string, unknown>>((resolve) => {
|
||||
socket!.emit(
|
||||
"batch_sync",
|
||||
[
|
||||
{ eventId: `${runId}-c1`, data: { i: 1 } },
|
||||
{ eventId: `${runId}-c2`, data: { i: 2 } },
|
||||
],
|
||||
resolve,
|
||||
);
|
||||
});
|
||||
assert.equal(clampAck.success, false);
|
||||
assert.equal(clampAck.code, "BATCH_LIMIT_EXCEEDED");
|
||||
assert.equal(clampAck.maxBatchItems, 1);
|
||||
} finally {
|
||||
delete process.env.DATASTACK_MAX_BATCH_ITEMS;
|
||||
if (socket) socket.disconnect();
|
||||
if (io) await new Promise<void>((resolve) => io!.close(() => resolve()));
|
||||
if (httpServer) httpServer.close();
|
||||
await DataBufferEngine.shutdown();
|
||||
await SDKManager.deleteClientWithStatsAsync("r4p2-batch-client", {
|
||||
purgeFiles: false,
|
||||
}).catch(() => undefined);
|
||||
CSVStorageManager.baseStorageDir = prevStorage;
|
||||
CSVStorageManager.clearRowCountCache();
|
||||
CSVStorageManager.clearQuotaCaches();
|
||||
await fsp.rm(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("R4-Phase-02 AUDIT-06: Governance audit logging for /api/export, Files Manager HTTP mutations (RAM + PG), and UI filters", async () => {
|
||||
const prevStorage = CSVStorageManager.baseStorageDir;
|
||||
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "ds-r4p2-audit-"));
|
||||
CSVStorageManager.baseStorageDir = tmpDir;
|
||||
CSVStorageManager.clearRowCountCache();
|
||||
CSVStorageManager.clearQuotaCaches();
|
||||
clearAuditLogsForTests();
|
||||
|
||||
let mockServer: http.Server | null = null;
|
||||
let pgServer: http.Server | null = null;
|
||||
|
||||
try {
|
||||
const cookie = await createOwnerCookie("audit-owner@datastack.dev");
|
||||
const partitionDir = path.join(tmpDir, "export-client");
|
||||
await fsp.mkdir(partitionDir, { recursive: true });
|
||||
await fsp.writeFile(
|
||||
path.join(partitionDir, "2026-11-20.csv"),
|
||||
'clientId,eventId,timestamp,payload\n"export-client","evt-1","2026-11-20T10:00:00.000Z","{""ok"":true}"\n',
|
||||
"utf8",
|
||||
);
|
||||
|
||||
// 1. Filtered-range export audit log
|
||||
const rangeRes = await exportGet(
|
||||
new NextRequest(
|
||||
"http://127.0.0.1:3000/api/export?clientId=export-client&format=csv&flush=false",
|
||||
{ headers: { Cookie: cookie } },
|
||||
),
|
||||
);
|
||||
assert.equal(rangeRes.status, 200);
|
||||
await rangeRes.text();
|
||||
|
||||
// 2. Single-file export audit log
|
||||
const singleRes = await exportGet(
|
||||
new NextRequest(
|
||||
"http://127.0.0.1:3000/api/export?clientId=export-client&filename=2026-11-20.csv&format=json&flush=false",
|
||||
{ headers: { Cookie: cookie } },
|
||||
),
|
||||
);
|
||||
assert.equal(singleRes.status, 200);
|
||||
await singleRes.text();
|
||||
|
||||
// Verify /api/audit filters DATA_EXPORT_DOWNLOADED
|
||||
const auditRes = await auditGet(
|
||||
new NextRequest(
|
||||
"http://127.0.0.1:3000/api/audit?action=DATA_EXPORT_DOWNLOADED&limit=10",
|
||||
{ headers: { Cookie: cookie } },
|
||||
),
|
||||
);
|
||||
assert.equal(auditRes.status, 200);
|
||||
const auditJson = await auditRes.json();
|
||||
assert.equal(auditJson.logs.length, 2);
|
||||
assert.equal(auditJson.logs[0].action, "DATA_EXPORT_DOWNLOADED");
|
||||
assert.equal(auditJson.logs[0].metadata.mode, "single_file");
|
||||
assert.equal(auditJson.logs[1].metadata.mode, "filtered_range");
|
||||
|
||||
// 3. Files Manager HTTP mutations -> RAM fallback
|
||||
clearAuditLogsForTests();
|
||||
const ramApp = express();
|
||||
ramApp.use(
|
||||
"/vcon/files-engine",
|
||||
filesManager.createFilesSecurityMiddleware({
|
||||
rootDir: tmpDir,
|
||||
getDbPool: () => null,
|
||||
}),
|
||||
);
|
||||
ramApp.put("/vcon/files-engine/api/v1/fs/sample.csv", (_req: express.Request, res: express.Response) => {
|
||||
res.status(200).json({ ok: true });
|
||||
});
|
||||
ramApp.put("/vcon/files-engine/api/v1/rename", (_req: express.Request, res: express.Response) => {
|
||||
res.status(200).json({ ok: true });
|
||||
});
|
||||
ramApp.delete("/vcon/files-engine/api/v1/fs/sample.csv", (_req: express.Request, res: express.Response) => {
|
||||
res.status(403).json({ ok: false }); // non-2xx should NOT emit mutation audit
|
||||
});
|
||||
|
||||
mockServer = http.createServer(ramApp);
|
||||
await new Promise<void>((resolve) => mockServer!.listen(0, "127.0.0.1", resolve));
|
||||
const ramPort = (mockServer.address() as AddressInfo).port;
|
||||
const originHeaders = {
|
||||
Cookie: cookie,
|
||||
Origin: `http://127.0.0.1:${ramPort}`,
|
||||
Host: `127.0.0.1:${ramPort}`,
|
||||
"Content-Type": "application/json",
|
||||
};
|
||||
|
||||
const putRes = await fetch(
|
||||
`http://127.0.0.1:${ramPort}/vcon/files-engine/api/v1/fs/sample.csv`,
|
||||
{ method: "PUT", headers: originHeaders, body: "hello" },
|
||||
);
|
||||
assert.equal(putRes.status, 200);
|
||||
|
||||
const renameRes = await fetch(
|
||||
`http://127.0.0.1:${ramPort}/vcon/files-engine/api/v1/rename`,
|
||||
{
|
||||
method: "PUT",
|
||||
headers: originHeaders,
|
||||
body: JSON.stringify({ from: "/sample.csv", to: "/renamed.csv" }),
|
||||
},
|
||||
);
|
||||
assert.equal(renameRes.status, 200);
|
||||
|
||||
const failDelRes = await fetch(
|
||||
`http://127.0.0.1:${ramPort}/vcon/files-engine/api/v1/fs/sample.csv`,
|
||||
{ method: "DELETE", headers: originHeaders },
|
||||
);
|
||||
assert.equal(failDelRes.status, 403);
|
||||
|
||||
await filesManager.flushPendingFilesAuditLogsForTests();
|
||||
const { logs: fmRamLogs } = await listAdminAuditLogsWithStatusAsync({
|
||||
action: "FILES_MANAGER_MUTATION",
|
||||
});
|
||||
assert.equal(fmRamLogs.length, 2);
|
||||
assert.equal(fmRamLogs[0].target, "/renamed.csv");
|
||||
assert.equal(fmRamLogs[0].metadata.from, "/sample.csv");
|
||||
assert.equal(fmRamLogs[1].target, "/sample.csv");
|
||||
|
||||
// 4. Files Manager HTTP mutation -> mock PostgreSQL pool
|
||||
const pgInserts: Array<{ sql: string; params: unknown[] }> = [];
|
||||
const mockDbPool = {
|
||||
query: async (sql: string, params: unknown[] = []) => {
|
||||
if (/FROM\s+admin_users\b/i.test(sql)) {
|
||||
return {
|
||||
rows: [{ email: "audit-owner@datastack.dev", token_version: 1 }],
|
||||
};
|
||||
}
|
||||
if (/FROM\s+admin_team_members\b/i.test(sql)) {
|
||||
return { rows: [] };
|
||||
}
|
||||
pgInserts.push({ sql, params });
|
||||
return { rows: [] };
|
||||
},
|
||||
};
|
||||
const pgApp = express();
|
||||
pgApp.use(
|
||||
"/vcon/files-engine",
|
||||
filesManager.createFilesSecurityMiddleware({
|
||||
rootDir: tmpDir,
|
||||
getDbPool: () => mockDbPool,
|
||||
}),
|
||||
);
|
||||
pgApp.delete("/vcon/files-engine/api/v1/fs/obsolete.csv", (_req: express.Request, res: express.Response) => {
|
||||
res.status(200).json({ ok: true });
|
||||
});
|
||||
|
||||
pgServer = http.createServer(pgApp);
|
||||
await new Promise<void>((resolve) => pgServer!.listen(0, "127.0.0.1", resolve));
|
||||
const pgPort = (pgServer.address() as AddressInfo).port;
|
||||
const pgDel = await fetch(
|
||||
`http://127.0.0.1:${pgPort}/vcon/files-engine/api/v1/fs/obsolete.csv`,
|
||||
{
|
||||
method: "DELETE",
|
||||
headers: {
|
||||
Cookie: cookie,
|
||||
Origin: `http://127.0.0.1:${pgPort}`,
|
||||
Host: `127.0.0.1:${pgPort}`,
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.equal(pgDel.status, 200);
|
||||
await filesManager.flushPendingFilesAuditLogsForTests();
|
||||
|
||||
assert.equal(pgInserts.length, 1);
|
||||
assert.match(pgInserts[0].sql, /INSERT INTO admin_audit_logs/i);
|
||||
assert.equal(pgInserts[0].params[3], "FILES_MANAGER_MUTATION");
|
||||
assert.equal(pgInserts[0].params[4], "/obsolete.csv");
|
||||
|
||||
// 5. AuditLogViewer filter options expose both new action types
|
||||
const viewerSrc = await fsp.readFile(
|
||||
path.join(process.cwd(), "components/sections/AuditLogViewer.tsx"),
|
||||
"utf8",
|
||||
);
|
||||
assert.ok(viewerSrc.includes("FILES_MANAGER_MUTATION"));
|
||||
assert.ok(viewerSrc.includes("DATA_EXPORT_DOWNLOADED"));
|
||||
} finally {
|
||||
if (mockServer) mockServer.close();
|
||||
if (pgServer) pgServer.close();
|
||||
clearAuditLogsForTests();
|
||||
CSVStorageManager.baseStorageDir = prevStorage;
|
||||
CSVStorageManager.clearRowCountCache();
|
||||
CSVStorageManager.clearQuotaCaches();
|
||||
await fsp.rm(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("R4-Phase-02 AUDIT-07 / ERR-R4-07: GET /api/data isolates corrupt .csv.gz partitions and serves healthy data", async () => {
|
||||
const prevStorage = CSVStorageManager.baseStorageDir;
|
||||
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "ds-r4p2-corrupt-"));
|
||||
CSVStorageManager.baseStorageDir = tmpDir;
|
||||
CSVStorageManager.clearRowCountCache();
|
||||
CSVStorageManager.clearQuotaCaches();
|
||||
|
||||
try {
|
||||
const cookie = await createOwnerCookie("data-owner@datastack.dev");
|
||||
const healthyDir = path.join(tmpDir, "healthy-app");
|
||||
const corruptDir = path.join(tmpDir, "corrupt-app");
|
||||
await fsp.mkdir(healthyDir, { recursive: true });
|
||||
await fsp.mkdir(corruptDir, { recursive: true });
|
||||
|
||||
await fsp.writeFile(
|
||||
path.join(healthyDir, "2026-11-20.csv"),
|
||||
'clientId,eventId,timestamp,payload\n"healthy-app","ok-1","2026-11-20T12:00:00.000Z","{""status"":""alive""}"\n',
|
||||
"utf8",
|
||||
);
|
||||
await fsp.writeFile(
|
||||
path.join(corruptDir, "2026-11-20.csv.gz"),
|
||||
Buffer.from("corrupted-not-gzip-bytes-123456"),
|
||||
);
|
||||
|
||||
const res = await dataGet(
|
||||
new NextRequest(
|
||||
"http://127.0.0.1:3000/api/data?from=2026-11-20&to=2026-11-20&page=1&pageSize=25",
|
||||
{
|
||||
headers: { Cookie: cookie },
|
||||
},
|
||||
),
|
||||
);
|
||||
assert.equal(res.status, 200);
|
||||
const json = await res.json();
|
||||
assert.equal(json.total, 1);
|
||||
assert.equal(json.events.length, 1);
|
||||
assert.equal(json.events[0].eventId, "ok-1");
|
||||
assert.equal(json.warnings, 1);
|
||||
assert.ok(Array.isArray(json.corruptPartitions));
|
||||
assert.equal(json.corruptPartitions.length, 1);
|
||||
assert.equal(json.corruptPartitions[0].clientId, "corrupt-app");
|
||||
assert.equal(json.corruptPartitions[0].filename, "2026-11-20.csv.gz");
|
||||
assert.equal(json.corruptPartitions[0].code, "CORRUPT_GZIP_PARTITION");
|
||||
} finally {
|
||||
CSVStorageManager.baseStorageDir = prevStorage;
|
||||
CSVStorageManager.clearRowCountCache();
|
||||
CSVStorageManager.clearQuotaCaches();
|
||||
await fsp.rm(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("R4-Phase-02 AUDIT-10 & AUDIT-11: PATCH redacts secret/token and /internal/client-sync disconnects selectively", async () => {
|
||||
const cookie = await createOwnerCookie("clients-owner@datastack.dev");
|
||||
const client = await SDKManager.registerClientAsync({
|
||||
clientId: "r4p2-selective-client",
|
||||
name: "Initial Client Name",
|
||||
rateLimit: 100,
|
||||
storageQuotaMb: 50,
|
||||
allowedIps: "127.0.0.1",
|
||||
});
|
||||
|
||||
// 1. AUDIT-10: PATCH metadata update must omit secret & full token
|
||||
const metaPatchRes = await clientPatch(
|
||||
new NextRequest(`http://127.0.0.1:3000/api/clients/${client.clientId}`, {
|
||||
method: "PATCH",
|
||||
headers: { Cookie: cookie, "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ name: "Renamed Client", rateLimit: 250 }),
|
||||
}),
|
||||
{ params: Promise.resolve({ id: client.clientId }) },
|
||||
);
|
||||
assert.equal(metaPatchRes.status, 200);
|
||||
const metaPatchJson = await metaPatchRes.json();
|
||||
assert.equal(metaPatchJson.client.name, "Renamed Client");
|
||||
assert.equal(metaPatchJson.client.rateLimit, 250);
|
||||
assert.equal("secret" in metaPatchJson.client, false);
|
||||
assert.equal("token" in metaPatchJson.client, false);
|
||||
assert.equal(typeof metaPatchJson.client.tokenPreview, "string");
|
||||
assert.ok(metaPatchJson.client.tokenPreview.endsWith("..."));
|
||||
|
||||
// 2. AUDIT-10: regenerate_token includes rotated token while still redacting secret
|
||||
const regenRes = await clientPatch(
|
||||
new NextRequest(`http://127.0.0.1:3000/api/clients/${client.clientId}`, {
|
||||
method: "PATCH",
|
||||
headers: { Cookie: cookie, "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ action: "regenerate_token" }),
|
||||
}),
|
||||
{ params: Promise.resolve({ id: client.clientId }) },
|
||||
);
|
||||
assert.equal(regenRes.status, 200);
|
||||
const regenJson = await regenRes.json();
|
||||
assert.equal("secret" in regenJson.client, false);
|
||||
assert.equal(typeof regenJson.client.token, "string");
|
||||
assert.notEqual(regenJson.client.token, client.token);
|
||||
|
||||
// 3. AUDIT-11: Live Socket.IO client stays connected on metadata update, disconnects on security update
|
||||
await new Promise<void>((resolve) =>
|
||||
ingestionServer.server.listen(0, "127.0.0.1", resolve),
|
||||
);
|
||||
const port = (ingestionServer.server.address() as AddressInfo).port;
|
||||
const internalHeaders = {
|
||||
"Content-Type": "application/json",
|
||||
...getInternalIngestionHeaders(),
|
||||
};
|
||||
|
||||
let sock: ReturnType<typeof ioClient> | null = null;
|
||||
try {
|
||||
// Seed ingestion server registry via /internal/client-sync registered
|
||||
const regSyncRes = await fetch(
|
||||
`http://127.0.0.1:${port}/internal/client-sync`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: internalHeaders,
|
||||
body: JSON.stringify({
|
||||
action: "registered",
|
||||
clientId: client.clientId,
|
||||
client: {
|
||||
...regenJson.client,
|
||||
token: regenJson.client.token,
|
||||
status: "active",
|
||||
allowedIps: "127.0.0.1",
|
||||
expiresAt: null,
|
||||
},
|
||||
}),
|
||||
},
|
||||
);
|
||||
assert.equal(regSyncRes.status, 200);
|
||||
|
||||
sock = ioClient(`http://127.0.0.1:${port}`, {
|
||||
auth: { token: regenJson.client.token },
|
||||
transports: ["websocket"],
|
||||
});
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
sock!.on("connect", () => resolve());
|
||||
sock!.on("connect_error", reject);
|
||||
});
|
||||
|
||||
// Metadata-only update: socket MUST remain connected
|
||||
const metadataSyncRes = await fetch(
|
||||
`http://127.0.0.1:${port}/internal/client-sync`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: internalHeaders,
|
||||
body: JSON.stringify({
|
||||
action: "updated",
|
||||
clientId: client.clientId,
|
||||
client: {
|
||||
...regenJson.client,
|
||||
name: "Hot Metadata Update",
|
||||
rateLimit: 400,
|
||||
storageQuotaMb: 200,
|
||||
token: regenJson.client.token,
|
||||
status: "active",
|
||||
allowedIps: "127.0.0.1",
|
||||
expiresAt: null,
|
||||
},
|
||||
}),
|
||||
},
|
||||
);
|
||||
assert.equal(metadataSyncRes.status, 200);
|
||||
const metadataSyncJson = await metadataSyncRes.json();
|
||||
assert.equal(metadataSyncJson.disconnected, false);
|
||||
assert.equal(sock.connected, true);
|
||||
|
||||
// AllowedIps narrowing away from 127.0.0.1 MUST disconnect live socket
|
||||
const disconnectPromise = new Promise<string>((resolve) => {
|
||||
sock!.once("disconnect", resolve);
|
||||
});
|
||||
const ipSyncRes = await fetch(
|
||||
`http://127.0.0.1:${port}/internal/client-sync`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: internalHeaders,
|
||||
body: JSON.stringify({
|
||||
action: "updated",
|
||||
clientId: client.clientId,
|
||||
client: {
|
||||
...regenJson.client,
|
||||
token: regenJson.client.token,
|
||||
status: "active",
|
||||
allowedIps: "203.0.113.50",
|
||||
expiresAt: null,
|
||||
},
|
||||
}),
|
||||
},
|
||||
);
|
||||
assert.equal(ipSyncRes.status, 200);
|
||||
const ipSyncJson = await ipSyncRes.json();
|
||||
assert.equal(ipSyncJson.disconnected, true);
|
||||
await disconnectPromise;
|
||||
assert.equal(sock.connected, false);
|
||||
} finally {
|
||||
if (sock) sock.disconnect();
|
||||
await new Promise<void>((resolve) => ingestionServer.io.close(() => resolve()));
|
||||
await SDKManager.deleteClientWithStatsAsync("r4p2-selective-client", {
|
||||
purgeFiles: false,
|
||||
}).catch(() => undefined);
|
||||
}
|
||||
});
|
||||
|
||||
test("R4-Phase-02 AUDIT-12 / ERR-R4-08: Streaming S3 upload & streaming .csv.gz row counter", async () => {
|
||||
process.env.DATASTACK_ALLOW_PRIVATE_URLS = "true";
|
||||
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "ds-r4p2-stream-"));
|
||||
|
||||
let s3Server: http.Server | null = null;
|
||||
try {
|
||||
// 1. Multi-chunk (>256 KB) .csv and .csv.gz partitions for row counting & S3 upload
|
||||
const totalRows = 3200;
|
||||
const lines = ["clientId,eventId,timestamp,payload"];
|
||||
for (let i = 0; i < totalRows; i += 1) {
|
||||
lines.push(
|
||||
`"stream-client","evt-${i}","2026-11-20T00:00:00.000Z","{""seq"":${i},""padding"":""0123456789abcdef0123456789abcdef0123456789abcdef""}"`,
|
||||
);
|
||||
}
|
||||
const rawCsv = `${lines.join("\n")}\n`;
|
||||
assert.ok(Buffer.byteLength(rawCsv, "utf8") > 256 * 1024);
|
||||
|
||||
const csvPath = path.join(tmpDir, "2026-11-20.csv");
|
||||
const gzPath = path.join(tmpDir, "2026-11-20.csv.gz");
|
||||
await fsp.writeFile(csvPath, rawCsv, "utf8");
|
||||
await fsp.writeFile(gzPath, zlib.gzipSync(Buffer.from(rawCsv, "utf8")));
|
||||
|
||||
assert.equal(
|
||||
await CSVStorageManager.countCsvDataRowsStreamAsync(csvPath),
|
||||
totalRows,
|
||||
);
|
||||
assert.equal(
|
||||
await CSVStorageManager.countCsvDataRowsStreamAsync(gzPath),
|
||||
totalRows,
|
||||
);
|
||||
|
||||
const corruptGz = path.join(tmpDir, "corrupt.csv.gz");
|
||||
await fsp.writeFile(corruptGz, Buffer.from("invalid-gzip-data"));
|
||||
await assert.rejects(() =>
|
||||
CSVStorageManager.countCsvDataRowsStreamAsync(corruptGz),
|
||||
);
|
||||
|
||||
// 2. Streaming PUT via uploadFileToS3 with exact content-length & SHA-256 verification
|
||||
let failNextUpload = false;
|
||||
const receivedUploads: Array<{
|
||||
url: string;
|
||||
contentLength: string | undefined;
|
||||
sha256: string | undefined;
|
||||
body: Buffer;
|
||||
}> = [];
|
||||
|
||||
s3Server = http.createServer((req, res) => {
|
||||
if (failNextUpload) {
|
||||
req.resume();
|
||||
res.destroy(new Error("Simulated mid-upload S3 reset"));
|
||||
return;
|
||||
}
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (chunk: Buffer) => chunks.push(chunk));
|
||||
req.on("end", () => {
|
||||
receivedUploads.push({
|
||||
url: req.url || "",
|
||||
contentLength: req.headers["content-length"],
|
||||
sha256: req.headers["x-amz-content-sha256"] as string | undefined,
|
||||
body: Buffer.concat(chunks),
|
||||
});
|
||||
res.writeHead(200, { ETag: '"stream-etag-ok"' });
|
||||
res.end();
|
||||
});
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve) => s3Server!.listen(0, "127.0.0.1", resolve));
|
||||
const s3Port = (s3Server.address() as AddressInfo).port;
|
||||
const s3Config = {
|
||||
endpoint: `http://127.0.0.1:${s3Port}`,
|
||||
region: "auto",
|
||||
bucket: "datastack-test-bucket",
|
||||
prefix: "archive",
|
||||
accessKeyId: "test-ak",
|
||||
secretAccessKey: "test-sk",
|
||||
};
|
||||
|
||||
const csvRes = await uploadFileToS3(
|
||||
s3Config,
|
||||
csvPath,
|
||||
"archive/stream-client/2026-11-20.csv",
|
||||
);
|
||||
assert.equal(csvRes.ok, true);
|
||||
assert.equal(csvRes.etag, '"stream-etag-ok"');
|
||||
|
||||
const gzRes = await uploadFileToS3(
|
||||
s3Config,
|
||||
gzPath,
|
||||
"archive/stream-client/2026-11-20.csv.gz",
|
||||
);
|
||||
assert.equal(gzRes.ok, true);
|
||||
|
||||
assert.equal(receivedUploads.length, 2);
|
||||
const expectedCsvBuf = await fsp.readFile(csvPath);
|
||||
const expectedCsvHash = crypto
|
||||
.createHash("sha256")
|
||||
.update(expectedCsvBuf)
|
||||
.digest("hex");
|
||||
assert.equal(
|
||||
receivedUploads[0].contentLength,
|
||||
String(expectedCsvBuf.length),
|
||||
);
|
||||
assert.equal(receivedUploads[0].sha256, expectedCsvHash);
|
||||
assert.ok(receivedUploads[0].body.equals(expectedCsvBuf));
|
||||
|
||||
const expectedGzBuf = await fsp.readFile(gzPath);
|
||||
assert.equal(
|
||||
receivedUploads[1].contentLength,
|
||||
String(expectedGzBuf.length),
|
||||
);
|
||||
assert.ok(receivedUploads[1].body.equals(expectedGzBuf));
|
||||
|
||||
// 3. Failure mid-upload returns S3_ARCHIVE_FAILED cleanly
|
||||
failNextUpload = true;
|
||||
const failRes = await uploadFileToS3(
|
||||
s3Config,
|
||||
csvPath,
|
||||
"archive/stream-client/fail.csv",
|
||||
);
|
||||
assert.equal(failRes.ok, false);
|
||||
assert.equal(failRes.code, "S3_ARCHIVE_FAILED");
|
||||
} finally {
|
||||
delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||
if (s3Server) s3Server.close();
|
||||
await fsp.rm(tmpDir, { recursive: true, force: true });
|
||||
assert.equal(fs.existsSync(tmpDir), false);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,313 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import { createRequire } from 'node:module';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { afterEach, beforeEach, describe, it } from 'node:test';
|
||||
import { NextRequest } from 'next/server';
|
||||
import { POST as setupPost } from '../app/api/auth/setup/route';
|
||||
import { GET as filesStatusGet } from '../app/api/files/status/route';
|
||||
import {
|
||||
createAdminSessionToken,
|
||||
getAdminSessionPayload,
|
||||
getAdminSessionSecret,
|
||||
} from '../core/auth/adminSession';
|
||||
import { resetAdminStateForTests } from '../core/auth/adminUsers';
|
||||
import { middleware } from '../middleware';
|
||||
import { DataStack, DiskSpooler } from '../sdk/node/datastack';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const ORIGINAL_ENV = { ...process.env };
|
||||
const ORIGINAL_FETCH = globalThis.fetch;
|
||||
|
||||
function restoreEnv() {
|
||||
for (const key of Object.keys(process.env)) {
|
||||
if (!(key in ORIGINAL_ENV)) {
|
||||
delete process.env[key];
|
||||
}
|
||||
}
|
||||
Object.assign(process.env, ORIGINAL_ENV);
|
||||
}
|
||||
|
||||
function makeFakeJwt(clientId: string): string {
|
||||
const header = Buffer.from(JSON.stringify({ alg: 'HS256', typ: 'JWT' })).toString('base64url');
|
||||
const payload = Buffer.from(JSON.stringify({ clientId, iat: 1700000000 })).toString('base64url');
|
||||
return `${header}.${payload}.sig`;
|
||||
}
|
||||
|
||||
describe('R4-Phase-03: Node SDK Packaging & Spool Parity, Docker & Proxy Hint, Auth Hygiene', () => {
|
||||
beforeEach(() => {
|
||||
restoreEnv();
|
||||
resetAdminStateForTests();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = ORIGINAL_FETCH;
|
||||
restoreEnv();
|
||||
resetAdminStateForTests();
|
||||
});
|
||||
|
||||
it('AUDIT-02: prebuilds @datastack/sdk dist artifacts, sets prepare/typescript metadata, and loads directly via CJS require', () => {
|
||||
const sdkDir = path.join(process.cwd(), 'sdk', 'node', 'datastack');
|
||||
const pkg = JSON.parse(fs.readFileSync(path.join(sdkDir, 'package.json'), 'utf8')) as {
|
||||
scripts?: Record<string, string>;
|
||||
devDependencies?: Record<string, string>;
|
||||
main?: string;
|
||||
types?: string;
|
||||
};
|
||||
|
||||
assert.equal(pkg.main, './dist/index.js');
|
||||
assert.equal(pkg.types, './dist/index.d.ts');
|
||||
assert.equal(pkg.scripts?.build, 'tsc -p tsconfig.json');
|
||||
assert.equal(pkg.scripts?.prepare, 'npm run build');
|
||||
assert.ok(pkg.devDependencies?.typescript, 'typescript must be listed in devDependencies');
|
||||
|
||||
for (const file of [
|
||||
'index.js',
|
||||
'index.d.ts',
|
||||
'spool.js',
|
||||
'spool.d.ts',
|
||||
'types.js',
|
||||
'types.d.ts',
|
||||
]) {
|
||||
assert.equal(
|
||||
fs.existsSync(path.join(sdkDir, 'dist', file)),
|
||||
true,
|
||||
`Missing prebuilt dist artifact: ${file}`
|
||||
);
|
||||
}
|
||||
|
||||
const cjsLoaded = require(path.join(sdkDir, 'dist', 'index.js')) as {
|
||||
DataStack?: typeof DataStack;
|
||||
DataStackClient?: typeof DataStack;
|
||||
DiskSpooler?: typeof DiskSpooler;
|
||||
SDK_VERSION?: string;
|
||||
};
|
||||
assert.equal(typeof cjsLoaded.DataStack, 'function');
|
||||
assert.equal(typeof cjsLoaded.DataStackClient, 'function');
|
||||
assert.equal(typeof cjsLoaded.DiskSpooler, 'function');
|
||||
assert.equal(cjsLoaded.SDK_VERSION, '2.2.0');
|
||||
|
||||
const token = makeFakeJwt('cjs_loader_probe');
|
||||
const client = new cjsLoaded.DataStack!({
|
||||
token,
|
||||
autoConnect: false,
|
||||
offlineSpoolEnabled: false,
|
||||
});
|
||||
assert.equal(client.clientId, 'cjs_loader_probe');
|
||||
});
|
||||
|
||||
it('AUDIT-20: DiskSpooler.ackAndRequeueFront preserves FIFO tail and client.flushSpool recovers partial 429 progress atomically', async () => {
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ds-r4-p03-spool-'));
|
||||
try {
|
||||
// 1. Direct DiskSpooler unit check
|
||||
const spooler = new DiskSpooler({ spoolDir: tmpDir, enabled: true });
|
||||
const mkEnv = (id: string) => ({
|
||||
clientId: 'sensor_node',
|
||||
eventId: id,
|
||||
timestamp: '2026-11-21T00:00:00.000Z',
|
||||
data: { id },
|
||||
});
|
||||
spooler.enqueue([mkEnv('e1'), mkEnv('e2'), mkEnv('e3'), mkEnv('e4'), mkEnv('e5')]);
|
||||
const removedCount = spooler.ackAndRequeueFront(3, [mkEnv('e2')]);
|
||||
assert.equal(removedCount, 2);
|
||||
|
||||
const afterPartial = spooler.peekBatch(10).map((e) => e.eventId);
|
||||
assert.deepEqual(afterPartial, ['e2', 'e4', 'e5']);
|
||||
|
||||
// Ensure no leftover .tmp files exist in spoolDir
|
||||
const entries = fs.readdirSync(tmpDir);
|
||||
assert.equal(entries.some((f) => f.endsWith('.tmp')), false);
|
||||
|
||||
// Clean and test partial recovery through DataStack.flushSpool()
|
||||
spooler.clear();
|
||||
const deliveredIds: string[] = [];
|
||||
let simulatePartial = true;
|
||||
|
||||
const token = makeFakeJwt('sensor_node');
|
||||
const client = new DataStack({
|
||||
token,
|
||||
apiUrl: 'http://localhost:3000',
|
||||
serverUrl: 'http://localhost:4000',
|
||||
autoConnect: false,
|
||||
offlineSpoolEnabled: true,
|
||||
offlineSpoolDir: tmpDir,
|
||||
batchChunkSize: 3,
|
||||
maxRetries: 0,
|
||||
fetchImpl: async (_input, init) => {
|
||||
const body = JSON.parse(String(init?.body || '{}')) as {
|
||||
batch?: Array<{ eventId: string }>;
|
||||
};
|
||||
const batch = body.batch || [];
|
||||
if (simulatePartial && batch.length === 3) {
|
||||
// Simulate partial 429 HTTP rejection where batch[1] is rate-limited
|
||||
const limitedId = batch[1].eventId;
|
||||
deliveredIds.push(batch[0].eventId, batch[2].eventId);
|
||||
return new Response(
|
||||
JSON.stringify({
|
||||
success: false,
|
||||
code: 'RATE_LIMIT_EXCEEDED',
|
||||
error: 'Rate limit reached for 1 event',
|
||||
rateLimitedEventIds: [limitedId],
|
||||
}),
|
||||
{ status: 429, headers: { 'Content-Type': 'application/json' } }
|
||||
);
|
||||
}
|
||||
for (const item of batch) {
|
||||
deliveredIds.push(item.eventId);
|
||||
}
|
||||
return new Response(
|
||||
JSON.stringify({ success: true, accepted: batch.length }),
|
||||
{ status: 200, headers: { 'Content-Type': 'application/json' } }
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
client.spooler.enqueue([mkEnv('s1'), mkEnv('s2'), mkEnv('s3'), mkEnv('s4')]);
|
||||
const firstFlush = await client.flushSpool();
|
||||
assert.equal(firstFlush.drained, 2);
|
||||
assert.equal(firstFlush.remaining, 2);
|
||||
assert.deepEqual(
|
||||
client.spooler.peekBatch(10).map((e) => e.eventId),
|
||||
['s2', 's4']
|
||||
);
|
||||
|
||||
// Next flush delivers remaining [s2, s4] cleanly with zero duplicates or data loss
|
||||
simulatePartial = false;
|
||||
const secondFlush = await client.flushSpool();
|
||||
assert.equal(secondFlush.drained, 2);
|
||||
assert.equal(secondFlush.remaining, 0);
|
||||
assert.deepEqual(deliveredIds, ['s1', 's3', 's2', 's4']);
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('AUDIT-08 & ERR-R4-09: docker-compose ingestion configures DATASTACK_ADMIN_EMAIL and /api/files/status + UI expose non-Nginx WS hint', async () => {
|
||||
const compose = fs.readFileSync(path.join(process.cwd(), 'docker', 'docker-compose.yml'), 'utf8');
|
||||
const ingestionBlockMatch = compose.match(
|
||||
/ingestion:\s*\n[\s\S]*?\n\s+proxy:/
|
||||
);
|
||||
assert.ok(ingestionBlockMatch, 'ingestion service block must exist in docker-compose.yml');
|
||||
assert.match(
|
||||
ingestionBlockMatch[0],
|
||||
/DATASTACK_ADMIN_EMAIL:\s*"\$\{DATASTACK_ADMIN_EMAIL:-admin@datastack\.dev\}"/
|
||||
);
|
||||
|
||||
delete process.env.DATABASE_URL;
|
||||
delete process.env.DATASTACK_PROXY_MODE;
|
||||
process.env.DATASTACK_ADMIN_SESSION_SECRET =
|
||||
'phase03-files-status-session-secret-min-32-chars';
|
||||
|
||||
const secret = getAdminSessionSecret()!;
|
||||
const ownerCookie = await createAdminSessionToken(
|
||||
'admin@datastack.dev',
|
||||
secret,
|
||||
Date.now() + 60_000,
|
||||
1,
|
||||
'owner',
|
||||
'owner-1'
|
||||
);
|
||||
|
||||
globalThis.fetch = (async () =>
|
||||
new Response(
|
||||
JSON.stringify({
|
||||
ok: true,
|
||||
engine: 'cloudcmd',
|
||||
role: 'owner',
|
||||
readOnly: false,
|
||||
}),
|
||||
{ status: 200, headers: { 'Content-Type': 'application/json' } }
|
||||
)) as typeof fetch;
|
||||
|
||||
// 1. Standalone Next.js mode (no reverse-proxy headers)
|
||||
const standaloneRes = await filesStatusGet(
|
||||
new NextRequest('http://localhost:3000/api/files/status', {
|
||||
headers: { cookie: `datastack_admin_session=${ownerCookie}` },
|
||||
})
|
||||
);
|
||||
assert.equal(standaloneRes.status, 200);
|
||||
const standaloneJson = (await standaloneRes.json()) as {
|
||||
ok: boolean;
|
||||
proxyMode: string;
|
||||
wsSupported: boolean;
|
||||
wsHint: string | null;
|
||||
};
|
||||
assert.equal(standaloneJson.ok, true);
|
||||
assert.equal(standaloneJson.proxyMode, 'standalone');
|
||||
assert.equal(standaloneJson.wsSupported, false);
|
||||
assert.ok(standaloneJson.wsHint && /HTTP polling/i.test(standaloneJson.wsHint));
|
||||
|
||||
// 2. Nginx reverse-proxy headers present
|
||||
const nginxRes = await filesStatusGet(
|
||||
new NextRequest('http://localhost:3000/api/files/status', {
|
||||
headers: {
|
||||
cookie: `datastack_admin_session=${ownerCookie}`,
|
||||
'x-forwarded-host': 'console.example.com',
|
||||
'x-forwarded-proto': 'https',
|
||||
},
|
||||
})
|
||||
);
|
||||
assert.equal(nginxRes.status, 200);
|
||||
const nginxJson = (await nginxRes.json()) as {
|
||||
proxyMode: string;
|
||||
wsSupported: boolean;
|
||||
wsHint: string | null;
|
||||
};
|
||||
assert.equal(nginxJson.proxyMode, 'nginx');
|
||||
assert.equal(nginxJson.wsSupported, true);
|
||||
assert.equal(nginxJson.wsHint, null);
|
||||
|
||||
// 3. UI component renders proxy hint element safely when present
|
||||
const workspaceSrc = fs.readFileSync(
|
||||
path.join(process.cwd(), 'components', 'sections', 'FilesManagerWorkspace.tsx'),
|
||||
'utf8'
|
||||
);
|
||||
assert.ok(workspaceSrc.includes('files-manager-proxy-hint'));
|
||||
assert.ok(workspaceSrc.includes('data.wsHint'));
|
||||
});
|
||||
|
||||
it('AUDIT-13: middleware removes dead /api/auth/setup/status exemption and POST /api/auth/setup embeds explicit owner role/memberId', async () => {
|
||||
const mwSrc = fs.readFileSync(path.join(process.cwd(), 'middleware.ts'), 'utf8');
|
||||
assert.equal(mwSrc.includes('/api/auth/setup/status'), false);
|
||||
|
||||
process.env.DATASTACK_ADMIN_SESSION_SECRET =
|
||||
'phase03-setup-session-secret-at-least-32-chars';
|
||||
const mwRes = await middleware(
|
||||
new NextRequest('http://localhost:3000/api/auth/setup/status', { method: 'GET' })
|
||||
);
|
||||
assert.equal(mwRes.status, 401);
|
||||
|
||||
delete process.env.DATABASE_URL;
|
||||
process.env.DATASTACK_ALLOW_DEMO_ADMIN = 'false';
|
||||
process.env.DATASTACK_ADMIN_SETUP_KEY =
|
||||
'phase03-custom-setup-key-over-32-characters-long';
|
||||
|
||||
const setupRes = await setupPost(
|
||||
new NextRequest('http://localhost:3000/api/auth/setup', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
'x-forwarded-for': '203.0.113.15',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
email: 'founder@datastack.dev',
|
||||
password: 'FounderPassword123!',
|
||||
setupKey: process.env.DATASTACK_ADMIN_SETUP_KEY,
|
||||
}),
|
||||
})
|
||||
);
|
||||
assert.equal(setupRes.status, 200);
|
||||
const cookieVal = setupRes.cookies.get('datastack_admin_session')?.value;
|
||||
assert.ok(cookieVal);
|
||||
|
||||
const payload = await getAdminSessionPayload(
|
||||
cookieVal!,
|
||||
process.env.DATASTACK_ADMIN_SESSION_SECRET
|
||||
);
|
||||
assert.ok(payload);
|
||||
assert.equal(payload!.email, 'founder@datastack.dev');
|
||||
assert.equal(payload!.role, 'owner');
|
||||
assert.equal(payload!.memberId, 'owner-1');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,98 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { getVconRoleCapabilities } from '../lib/vconUiHelpers';
|
||||
|
||||
const ROOT = process.cwd();
|
||||
|
||||
async function readSource(relPath: string): Promise<string> {
|
||||
return fs.readFile(path.join(ROOT, relPath), 'utf8');
|
||||
}
|
||||
|
||||
test('R4-Phase-04 [AUDIT-14]: TeamWorkspace seeds role from useVconRole, confirms deletes via Modal, and distinguishes operator vs admin', async () => {
|
||||
const src = await readSource('components/sections/TeamWorkspace.tsx');
|
||||
|
||||
// Eliminates hardcoded 'owner' initial flash
|
||||
assert.match(src, /useVconRole\(\)/);
|
||||
assert.doesNotMatch(src, /useState<string>\(\s*['"]owner['"]\s*\)/);
|
||||
assert.match(src, /useState<string>\(\(\)\s*=>\s*shellRole\s*\|\|\s*['"]viewer['"]\)/);
|
||||
|
||||
// Explicit delete confirmation modal before DELETE /api/team/[id]
|
||||
assert.match(src, /setDeleteTarget\(row\)/);
|
||||
assert.match(src, /title="Confirm Team Member Removal"/);
|
||||
assert.match(src, /data-testid="confirm-remove-member"/);
|
||||
assert.doesNotMatch(src, /onClick=\{\(\)\s*=>\s*void handleDeleteMember\(row\)\}/);
|
||||
|
||||
// Preserves distinct operator vs admin options instead of silently coercing operator -> admin
|
||||
assert.doesNotMatch(src, /row\.role === ['"]operator['"] \? ['"]admin['"]/);
|
||||
assert.match(src, /value:\s*['"]operator['"]/);
|
||||
assert.match(src, /value:\s*['"]admin['"]/);
|
||||
});
|
||||
|
||||
test('R4-Phase-04 [AUDIT-15 / ERR-R4-10]: /vcon/sdk skips ?includeToken=true fetch for viewer and renders explicit RBAC notice', async () => {
|
||||
const src = await readSource('app/vcon/sdk/page.tsx');
|
||||
|
||||
assert.match(src, /useVconRole\(\)/);
|
||||
assert.match(src, /getVconRoleCapabilities\(shellRole\)/);
|
||||
assert.match(src, /if \(!selectedClientId \|\| !canOperate\)/);
|
||||
assert.match(src, /data-testid="sdk-viewer-rbac-banner"/);
|
||||
assert.match(src, /Operator or Owner role required/i);
|
||||
|
||||
// Confirm capability helper blocks viewer token retrieval while allowing operator/admin/owner
|
||||
assert.equal(getVconRoleCapabilities('viewer').canOperate, false);
|
||||
assert.equal(getVconRoleCapabilities('operator').canOperate, true);
|
||||
assert.equal(getVconRoleCapabilities('admin').canOperate, true);
|
||||
assert.equal(getVconRoleCapabilities('owner').canOperate, true);
|
||||
|
||||
// Preserve V2-Phase-04 and R2-Phase-03 compatibility checks
|
||||
assert.match(src, /Python SDK \(v2\.2\.0\)/);
|
||||
assert.match(src, /Node\.js \/ TypeScript SDK \(v2\.2\.0\)/);
|
||||
assert.match(src, /setSelectedClientToken\(''\);\s*\n\s*setLoadingToken\(true\);/);
|
||||
});
|
||||
|
||||
test('R4-Phase-04 [AUDIT-16]: Sidebar uses useVconRole and Client SDKs label; Navbar covers all admin routes', async () => {
|
||||
const sidebarSrc = await readSource('components/layout/Sidebar.tsx');
|
||||
assert.match(sidebarSrc, /useVconRole\(\)/);
|
||||
assert.match(sidebarSrc, /\{\s*name:\s*['"]Client SDKs['"],\s*href:\s*['"]\/vcon\/sdk['"]/);
|
||||
assert.doesNotMatch(sidebarSrc, /\{\s*name:\s*['"]Python SDK['"]/);
|
||||
assert.doesNotMatch(sidebarSrc, /getCached<CachedProfile>\('vcon:profile',\s*\{\}\)\.role\s*\|\|\s*['"]Owner['"]/);
|
||||
assert.match(sidebarSrc, /getCached<CachedProfile>\('vcon:profile',\s*\{\}\)\.role\s*\|\|\s*initialRoleLabel/);
|
||||
|
||||
const navbarSrc = await readSource('components/layout/Navbar.tsx');
|
||||
assert.match(navbarSrc, /'\/vcon\/files':\s*'Files Manager'/);
|
||||
assert.match(navbarSrc, /'\/vcon\/sdk':\s*'Client SDKs'/);
|
||||
assert.match(navbarSrc, /'\/vcon\/team':\s*'Team & RBAC'/);
|
||||
assert.match(navbarSrc, /'\/vcon\/audit':\s*'Audit Log'/);
|
||||
assert.doesNotMatch(navbarSrc, /'\/vcon\/sdk':\s*'Python SDK'/);
|
||||
});
|
||||
|
||||
test('R4-Phase-04 [AUDIT-19 & AUDIT-07]: ProfileWorkspace drops hardcoded demo defaults and DataBrowser surfaces corruptPartitions warnings', async () => {
|
||||
const profileSrc = await readSource('components/sections/ProfileWorkspace.tsx');
|
||||
assert.doesNotMatch(profileSrc, /DEFAULT_PROFILE/);
|
||||
assert.doesNotMatch(profileSrc, /email:\s*'admin@datastack\.dev'/);
|
||||
assert.doesNotMatch(profileSrc, /storageMode:\s*'In-Memory'/);
|
||||
assert.match(profileSrc, /EMPTY_PROFILE/);
|
||||
assert.match(profileSrc, /Loading\.\.\./);
|
||||
|
||||
const dataBrowserSrc = await readSource('components/sections/DataBrowser.tsx');
|
||||
assert.match(dataBrowserSrc, /corruptPartitions/);
|
||||
assert.match(dataBrowserSrc, /data-testid="data-corrupt-partitions-warning"/);
|
||||
});
|
||||
|
||||
test('R4-Phase-04 [AUDIT-18]: Public landing Hero, SdkSection, and LandingAbout reflect both Python and Node.js / TypeScript SDKs', async () => {
|
||||
const heroSrc = await readSource('components/sections/Hero.tsx');
|
||||
assert.match(heroSrc, /href="\/#sdk"[\s\S]*?>\s*Client SDKs\s*<\/Link>/);
|
||||
assert.doesNotMatch(heroSrc, />\s*Python SDK\s*<\/Link>/);
|
||||
|
||||
const sdkSectionSrc = await readSource('components/sections/SdkSection.tsx');
|
||||
assert.match(sdkSectionSrc, /label="Client SDKs"/);
|
||||
assert.match(sdkSectionSrc, /datastack/);
|
||||
assert.match(sdkSectionSrc, /@datastack\/sdk/);
|
||||
assert.match(sdkSectionSrc, /npm install/);
|
||||
assert.match(sdkSectionSrc, /pip install/);
|
||||
|
||||
const aboutSrc = await readSource('components/sections/LandingAbout.tsx');
|
||||
assert.match(aboutSrc, /href="\/#sdk"[\s\S]*?>\s*View Client SDKs\s*<\/Link>/);
|
||||
assert.doesNotMatch(aboutSrc, /View Python SDK/);
|
||||
});
|
||||
@@ -146,6 +146,9 @@ test("V2-Phase-01 [S3 SigV4]: buildS3ObjectKey, normalizeS3EndpointUrl, testS3Co
|
||||
const addr = mockS3.address() as AddressInfo;
|
||||
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "datastack-v2-s3-"));
|
||||
|
||||
const prevAllowPrivate = process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||
process.env.DATASTACK_ALLOW_PRIVATE_URLS = "true";
|
||||
|
||||
try {
|
||||
const samplePath = path.join(tmpDir, "2026-10-01.csv.gz");
|
||||
const gzData = zlib.gzipSync(
|
||||
@@ -189,6 +192,8 @@ test("V2-Phase-01 [S3 SigV4]: buildS3ObjectKey, normalizeS3EndpointUrl, testS3Co
|
||||
assert.equal(uploadErr.code, "S3_ARCHIVE_FAILED");
|
||||
assert.match(uploadErr.error || "", /HTTP 500/);
|
||||
} finally {
|
||||
if (prevAllowPrivate === undefined) delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||
else process.env.DATASTACK_ALLOW_PRIVATE_URLS = prevAllowPrivate;
|
||||
await new Promise<void>((resolve) => mockS3.close(() => resolve()));
|
||||
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
@@ -229,6 +234,9 @@ test("V2-Phase-01 [Compression, Export, S3 Safe Prune V2-E02, Corrupt Gzip V2-E0
|
||||
await new Promise<void>((resolve) => mockS3.listen(0, "127.0.0.1", resolve));
|
||||
const s3Addr = mockS3.address() as AddressInfo;
|
||||
|
||||
const prevAllowPrivate = process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||
process.env.DATASTACK_ALLOW_PRIVATE_URLS = "true";
|
||||
|
||||
try {
|
||||
const cookie = await makeAdminCookie();
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
@@ -412,7 +420,7 @@ test("V2-Phase-01 [Compression, Export, S3 Safe Prune V2-E02, Corrupt Gzip V2-E0
|
||||
assert.deepEqual(uploadedKeys, [`/prod-vault/archive-test/v2_alpha/${oldDate}.csv.gz`]);
|
||||
await fs.stat(gzMeta.filePath);
|
||||
|
||||
// 10. V2-E03: Corrupt .csv.gz cleanly surfaces error in /api/data instead of crashing
|
||||
// 10. V2-E03 upgraded to ERR-R4-07: Corrupt .csv.gz is isolated per-file (returns 200 with corruptPartitions warning instead of crashing all clients)
|
||||
await fs.writeFile(
|
||||
path.join(tmpDir, "v2_alpha", "2024-01-01.csv.gz"),
|
||||
Buffer.from("not-valid-gzip-bytes"),
|
||||
@@ -423,10 +431,16 @@ test("V2-Phase-01 [Compression, Export, S3 Safe Prune V2-E02, Corrupt Gzip V2-E0
|
||||
{ headers: { cookie } },
|
||||
),
|
||||
);
|
||||
assert.equal(corruptDataRes.status, 500);
|
||||
assert.equal(corruptDataRes.status, 200);
|
||||
const corruptJson = await corruptDataRes.json();
|
||||
assert.match(corruptJson.error || "", /Corrupt|\.gz/i);
|
||||
assert.equal(corruptJson.warnings, 1);
|
||||
assert.ok(Array.isArray(corruptJson.corruptPartitions));
|
||||
assert.equal(corruptJson.corruptPartitions.length, 1);
|
||||
assert.equal(corruptJson.corruptPartitions[0].code, "CORRUPT_GZIP_PARTITION");
|
||||
assert.match(corruptJson.corruptPartitions[0].error || "", /Corrupt|\.gz/i);
|
||||
} finally {
|
||||
if (prevAllowPrivate === undefined) delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||
else process.env.DATASTACK_ALLOW_PRIVATE_URLS = prevAllowPrivate;
|
||||
await new Promise<void>((resolve) => mockS3.close(() => resolve()));
|
||||
await DataBufferEngine.shutdown();
|
||||
CSVStorageManager.baseStorageDir = origBaseStorageDir;
|
||||
|
||||
@@ -191,6 +191,9 @@ test("V2-Phase-02 [Webhook Notifier & /api/settings/webhook-test]: HMAC-SHA256 s
|
||||
const port = (mockWebhookServer.address() as AddressInfo).port;
|
||||
const webhookUrl = `http://127.0.0.1:${port}/hooks/datastack`;
|
||||
|
||||
const prevAllowPrivate = process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||
process.env.DATASTACK_ALLOW_PRIVATE_URLS = "true";
|
||||
|
||||
try {
|
||||
// 1. Format & signature verification
|
||||
const formattedJsonStr = formatOutboundWebhookBody({
|
||||
@@ -297,6 +300,8 @@ test("V2-Phase-02 [Webhook Notifier & /api/settings/webhook-test]: HMAC-SHA256 s
|
||||
const failTestData = await failTestRes.json();
|
||||
assert.equal(failTestData.code, "WEBHOOK_DELIVERY_FAILED");
|
||||
} finally {
|
||||
if (prevAllowPrivate === undefined) delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||
else process.env.DATASTACK_ALLOW_PRIVATE_URLS = prevAllowPrivate;
|
||||
await new Promise<void>((resolve) => mockWebhookServer.close(() => resolve()));
|
||||
await DataBufferEngine.shutdown();
|
||||
resetAdminStateForTests();
|
||||
@@ -807,6 +812,9 @@ test("V2-Audit-Phase-02 [BUG-V2-AUDIT-05, BUG-V2-AUDIT-06, BUG-V2-AUDIT-08]: Por
|
||||
);
|
||||
const socketPort = (testHttpServer.address() as AddressInfo).port;
|
||||
|
||||
const prevAllowPrivate = process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||
process.env.DATASTACK_ALLOW_PRIVATE_URLS = "true";
|
||||
|
||||
try {
|
||||
serverIndex.clearWebhookCooldownCacheForTests();
|
||||
serverIndex.applyRuntimeSettingsObject({
|
||||
@@ -890,6 +898,8 @@ test("V2-Audit-Phase-02 [BUG-V2-AUDIT-05, BUG-V2-AUDIT-06, BUG-V2-AUDIT-08]: Por
|
||||
const forbiddenHealthBody = await forbiddenHealthRes.json();
|
||||
assert.equal(forbiddenHealthBody.code, "CLIENT_IP_FORBIDDEN");
|
||||
} finally {
|
||||
if (prevAllowPrivate === undefined) delete process.env.DATASTACK_ALLOW_PRIVATE_URLS;
|
||||
else process.env.DATASTACK_ALLOW_PRIVATE_URLS = prevAllowPrivate;
|
||||
serverIndex.applyRuntimeSettingsObject({
|
||||
enableWebhookAlerts: false,
|
||||
webhookUrl: "",
|
||||
|
||||
Reference in new issue
Block a user