feat(auth): enhance security and audit logging

- Integrate webhook alerts for admin login lockouts.
- Implement detailed audit logging for login, logout, and setup actions.
- Improve client revocation event tracking.
- Standardize error handling in profile updates.
- Enforce 'operator' role for event API access.
This commit is contained in:
Team Dark Devil committed 2026-11-21 00:06:55 +06:00
1 parent 5c977d6977
commit 8132087085
52 files changed
+4434 -407

No files matched your search

+58
View File
@@ -602,6 +602,64 @@ class Phase05PythonSdkTests(unittest.TestCase):
f"{py_file.name}:{node.lineno} uses Python 3.9+ PEP 585 generic `{node.value.id}[...]`"
)
def test_v2_audit_phase02_bug09_fatal_auth_codes_expired_and_ip_forbidden(
self,
) -> None:
self.assertTrue(
DataStackClient._is_non_retryable_auth_error(
"CLIENT_TOKEN_EXPIRED: Client token has expired"
)
)
self.assertTrue(
DataStackClient._is_non_retryable_auth_error(
"CLIENT_IP_FORBIDDEN: IP 198.51.100.25 is not permitted"
)
)
self.assertFalse(
DataStackClient._is_non_retryable_auth_error("Connection reset by peer")
)
tok = _make_jwt({"clientId": "cli_audit09"})
client = DataStack(
server_url="http://127.0.0.1:4000",
api_url="http://127.0.0.1:3000",
token=tok,
auto_connect=False,
enable_http_fallback=True,
offline_spool_enabled=True,
request_timeout_sec=0.5,
)
class FakeFailingAuthSocket:
def __init__(self, error_msg: str) -> None:
self.connected = False
self.error_msg = error_msg
def connect(self, *args: Any, **kwargs: Any) -> None:
raise RuntimeError(self.error_msg)
def disconnect(self) -> None:
self.connected = False
# 1. CLIENT_TOKEN_EXPIRED on WebSocket handshake must immediately raise DataStackAuthError with code="CLIENT_TOKEN_EXPIRED"
client.sio = FakeFailingAuthSocket(
"CLIENT_TOKEN_EXPIRED: Client token has expired"
)
with self.assertRaises(DataStackAuthError) as ctx_exp:
client._connect_and_verify_websocket(wait_timeout=0.2)
self.assertEqual(ctx_exp.exception.code, "CLIENT_TOKEN_EXPIRED")
# 2. CLIENT_IP_FORBIDDEN on WebSocket handshake must immediately raise DataStackAuthError with code="CLIENT_IP_FORBIDDEN"
client.sio = FakeFailingAuthSocket(
"CLIENT_IP_FORBIDDEN: IP is not in allowlist"
)
with self.assertRaises(DataStackAuthError) as ctx_ip:
client._connect_and_verify_websocket(wait_timeout=0.2)
self.assertEqual(ctx_ip.exception.code, "CLIENT_IP_FORBIDDEN")
client.disconnect()
if __name__ == "__main__":
unittest.main(verbosity=2)
@@ -0,0 +1,491 @@
import assert from "node:assert/strict";
import crypto from "node:crypto";
import fs from "node:fs";
import { createRequire } from "node:module";
import path from "node:path";
import test from "node:test";
import jwt from "jsonwebtoken";
import { NextRequest } from "next/server";
process.env.DATASTACK_DISABLE_LISTEN = "1";
import { POST as loginPost } from "../app/api/auth/login/route";
import { POST as eventsPost } from "../app/api/events/route";
import { POST as webhookTestPost } from "../app/api/settings/webhook-test/route";
import { POST as s3TestPost } from "../app/api/storage/s3-test/route";
import { SettingsManager } from "../core/adminSettings/settingsManager";
import {
clearWebhookCooldownCacheForTests,
formatOutboundWebhookBody as formatPort3000WebhookBody,
} from "../core/alerts/webhookNotifier";
import {
createAdminSessionToken,
getAdminSessionSecret,
} from "../core/auth/adminSession";
import {
clearAdminLoginAttempts,
createAdminTeamMemberAsync,
createInitialAdmin,
deleteAdminTeamMemberAsync,
listAdminTeamMembersAsync,
resetAdminStateForTests,
} from "../core/auth/adminUsers";
import { AuthService } from "../core/auth/jwt";
import { DataBufferEngine } from "../core/data/buffer";
import { SDKManager } from "../core/sdk/sdkManager";
import { encryptSecret } from "../core/Security/secretVault";
const require = createRequire(import.meta.url);
const ingestionServer = require("../server/index.js") as {
decryptVaultSecret: (ciphertext: string) => string;
formatOutboundWebhookBody: (
url: string,
payload: Record<string, unknown>,
timestamp?: string,
) => string;
triggerWebhookAlert: (
payload: Record<string, unknown>,
options?: Record<string, unknown>,
) => Promise<{ dispatched: boolean; status?: number; reason?: string }>;
applyRuntimeSettingsObject: (settingsObj: Record<string, unknown>) => {
jwtRotated: boolean;
};
runtimeConfig: Record<string, unknown>;
clearWebhookCooldownCacheForTests: () => void;
stopTimers: () => void;
};
test.after(async () => {
ingestionServer.stopTimers();
SDKManager.stopActivityFlushTimerForTests();
await DataBufferEngine.shutdown();
resetAdminStateForTests();
});
test("R3-Phase-01 [BUG-01]: Port 3000 encryptSecret() decrypts 1:1 on Port 4000 & Docker Compose passes DATASTACK_ADMIN_SESSION_SECRET", async () => {
const prevSecret = process.env.DATASTACK_ADMIN_SESSION_SECRET;
const prevNodeEnv = process.env.NODE_ENV;
const testMaster = "r3-phase01-vault-master-secret-at-least-32-chars!!";
try {
process.env.DATASTACK_ADMIN_SESSION_SECRET = testMaster;
const plaintext = "whsec_live_prod_hmac_99887766";
const encrypted = encryptSecret(plaintext);
assert.ok(encrypted.startsWith("v1:"), "Port 3000 vault must emit v1: prefix");
// 1. Cross-runtime decryption matches 1:1
assert.equal(ingestionServer.decryptVaultSecret(encrypted), plaintext);
// 2. applyRuntimeSettingsObject hydrates plaintext webhookSecret from DB webhookSecretEnc
ingestionServer.applyRuntimeSettingsObject({ webhookSecretEnc: encrypted });
assert.equal(ingestionServer.runtimeConfig.webhookSecret, plaintext);
// 3. Wrong master secret fails closed to "" (never leaks raw ciphertext as HMAC key)
process.env.DATASTACK_ADMIN_SESSION_SECRET =
"wrong-master-secret-that-is-32-chars-long-000";
assert.equal(ingestionServer.decryptVaultSecret(encrypted), "");
// 4. Legacy / unrecognized ciphertext format fails closed to "" (never returned verbatim)
assert.equal(
ingestionServer.decryptVaultSecret("enc:v1:invalid:payload"),
"",
);
assert.equal(
ingestionServer.decryptVaultSecret("plaintext-not-vault-format"),
"",
);
// 5. Production mode without DATASTACK_ADMIN_SESSION_SECRET fails closed to ""
(process.env as Record<string, string | undefined>).NODE_ENV = "production";
delete process.env.DATASTACK_ADMIN_SESSION_SECRET;
assert.equal(ingestionServer.decryptVaultSecret(encrypted), "");
// 6. Docker Compose passes DATASTACK_ADMIN_SESSION_SECRET to ingestion container
const composeContent = fs.readFileSync(
path.resolve(process.cwd(), "docker/docker-compose.yml"),
"utf8",
);
const ingestionSection = composeContent.slice(
composeContent.indexOf(" ingestion:"),
composeContent.indexOf(" proxy:"),
);
assert.match(
ingestionSection,
/DATASTACK_ADMIN_SESSION_SECRET:\s*"\$\{SERVICE_REALBASE64_64_ADMINSESSION:/,
"docker/docker-compose.yml ingestion service must receive DATASTACK_ADMIN_SESSION_SECRET",
);
} finally {
if (prevSecret === undefined) {
delete process.env.DATASTACK_ADMIN_SESSION_SECRET;
} else {
process.env.DATASTACK_ADMIN_SESSION_SECRET = prevSecret;
}
(process.env as Record<string, string | undefined>).NODE_ENV = prevNodeEnv;
}
});
test("R3-Phase-01 [BUG-03]: Client SDK JWTs omit hardcoded 30d exp so Never-Expires and >30d clients stay valid", async () => {
const clientId = `r3_jwt_longlived_${Date.now()}`;
const farFutureIso = new Date(Date.now() + 180 * 86_400_000).toISOString();
// 1. Explicit expiresIn still works when caller requests bounded JWT
const boundedToken = AuthService.generateClientToken(
{ clientId: "bounded_test" },
"1h",
);
const boundedDecoded = jwt.decode(boundedToken) as jwt.JwtPayload;
assert.equal(typeof boundedDecoded.exp, "number");
// 2. Registered client with expiresAt = null ("Never expires") has NO hardcoded 30d exp
const neverClient = await SDKManager.registerClientAsync({
clientId,
name: "Long-Lived Telemetry Client",
rateLimit: 100,
expiresAt: null,
});
try {
const decodedNever = jwt.decode(neverClient.token) as jwt.JwtPayload;
assert.equal(
decodedNever.exp,
undefined,
"Never-expires client token must not embed a hardcoded 30d JWT exp claim",
);
assert.equal(AuthService.verifyClientToken(neverClient.token).valid, true);
// 3. Updating expiresAt to >30d future keeps the existing token valid without rotation
const updated = await SDKManager.updateClientAsync(clientId, {
expiresAt: farFutureIso,
});
assert.ok(updated);
assert.equal(updated.expiresAt, farFutureIso);
assert.equal(updated.token, neverClient.token);
// 4. Regenerated token also omits hardcoded 30d exp
const rotated = await SDKManager.regenerateTokenAsync(clientId);
assert.ok(rotated);
const decodedRotated = jwt.decode(rotated.token) as jwt.JwtPayload;
assert.equal(decodedRotated.exp, undefined);
} finally {
await SDKManager.deleteClientAsync(clientId);
}
});
test("R3-Phase-01 [BUG-04 & BUG-15]: Viewer blocked (403) on POST /api/events and Operator blocked (403) on POST /api/storage/s3-test", async () => {
resetAdminStateForTests();
await createInitialAdmin("r3-owner@datastack.dev", "OwnerPassword123!");
const sessionSecret = getAdminSessionSecret();
const suffix = Date.now();
const viewerEmail = `r3viewer_${suffix}@datastack.dev`;
const operatorEmail = `r3operator_${suffix}@datastack.dev`;
const clientId = `r3_rbac_client_${suffix}`;
const viewer = await createAdminTeamMemberAsync({
email: viewerEmail,
password: "ViewerPassword123!",
role: "viewer",
displayName: "R3 Viewer",
});
const operator = await createAdminTeamMemberAsync({
email: operatorEmail,
password: "OperatorPassword123!",
role: "operator",
displayName: "R3 Operator",
});
await SDKManager.registerClientAsync({
clientId,
name: "RBAC Test Client",
rateLimit: 50,
});
try {
const viewerToken = await createAdminSessionToken(
viewer.email,
sessionSecret,
Date.now() + 3600_000,
viewer.tokenVersion,
viewer.role,
viewer.id,
);
const operatorToken = await createAdminSessionToken(
operator.email,
sessionSecret,
Date.now() + 3600_000,
operator.tokenVersion,
operator.role,
operator.id,
);
// BUG-04: Viewer CANNOT ingest events via POST /api/events
const viewerIngestRes = await eventsPost(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
"content-type": "application/json",
cookie: `datastack_admin_session=${viewerToken}`,
},
body: JSON.stringify({ clientId, data: { metric: 1 } }),
}),
);
assert.equal(viewerIngestRes.status, 403);
assert.equal(
(await viewerIngestRes.json()).code,
"INSUFFICIENT_ROLE_PERMISSION",
);
// Operator CAN ingest events via POST /api/events
const operatorIngestRes = await eventsPost(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
"content-type": "application/json",
cookie: `datastack_admin_session=${operatorToken}`,
},
body: JSON.stringify({ clientId, data: { metric: 2 } }),
}),
);
assert.equal(operatorIngestRes.status, 200);
// BUG-15: Operator CANNOT call POST /api/storage/s3-test (Owner-only)
const operatorS3TestRes = await s3TestPost(
new NextRequest("http://localhost:3000/api/storage/s3-test", {
method: "POST",
headers: {
"content-type": "application/json",
cookie: `datastack_admin_session=${operatorToken}`,
},
body: JSON.stringify({ s3Endpoint: "http://127.0.0.1:9999" }),
}),
);
assert.equal(operatorS3TestRes.status, 403);
assert.equal(
(await operatorS3TestRes.json()).code,
"INSUFFICIENT_ROLE_PERMISSION",
);
} finally {
await SDKManager.deleteClientAsync(clientId);
await deleteAdminTeamMemberAsync(viewer.id);
await deleteAdminTeamMemberAsync(operator.id);
}
});
test("R3-Phase-01 [BUG-06]: Locked webhook secret does not break unrelated settings saves or draft/clear webhook tests", async () => {
resetAdminStateForTests();
await createInitialAdmin("r3-owner2@datastack.dev", "OwnerPassword123!");
const prevSecret = process.env.DATASTACK_ADMIN_SESSION_SECRET;
const origFetch = global.fetch;
const members = await listAdminTeamMembersAsync();
const owner = members.find((m) => m.role === "owner") || members[0];
assert.ok(owner, "Bootstrap owner member must exist for test");
try {
// 1. Encrypt webhook secret under key A
process.env.DATASTACK_ADMIN_SESSION_SECRET =
"key-alpha-32-characters-long-for-rotation-test";
await SettingsManager.updateSettingsAsync({
enableWebhookAlerts: true,
webhookUrl: "https://hooks.example.com/r3-test",
webhookSecret: "old_secret_under_key_alpha",
});
// 2. Rotate DATASTACK_ADMIN_SESSION_SECRET to key B (making stored webhookSecretEnc locked)
process.env.DATASTACK_ADMIN_SESSION_SECRET =
"key-beta-32-characters-long-for-rotation-test!";
const ownerToken = await createAdminSessionToken(
owner.email,
getAdminSessionSecret(),
Date.now() + 3600_000,
owner.tokenVersion,
owner.role,
owner.id,
);
let capturedTestHeaders: Record<string, string> = {};
global.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
const url = String(input);
if (url.includes("/internal/settings-sync")) {
return new Response(JSON.stringify({ ok: true }), { status: 200 });
}
if (url.includes("hooks.example.com")) {
capturedTestHeaders = (init?.headers as Record<string, string>) || {};
return new Response("ok", { status: 200 });
}
return new Response("not found", { status: 404 });
}) as typeof fetch;
// Saving an unrelated setting MUST NOT throw SecretVaultError
const updated = await SettingsManager.updateSettingsAsync({
maxEventsPerSec: 150,
});
assert.equal(updated.maxEventsPerSec, 150);
// Testing a brand-new draft webhookSecret MUST succeed without decrypting the old locked secret
const draftRes = await webhookTestPost(
new NextRequest("http://localhost:3000/api/settings/webhook-test", {
method: "POST",
headers: {
"content-type": "application/json",
cookie: `datastack_admin_session=${ownerToken}`,
},
body: JSON.stringify({
webhookUrl: "https://hooks.example.com/r3-test",
webhookSecret: "brand_new_draft_secret_1234",
}),
}),
);
assert.equal(draftRes.status, 200);
assert.ok(
capturedTestHeaders["X-DataStack-Signature"]?.startsWith("sha256="),
"Draft secret must sign outbound test webhook",
);
// Testing with clearWebhookSecret: true MUST also succeed without touching locked secret
capturedTestHeaders = {};
const clearRes = await webhookTestPost(
new NextRequest("http://localhost:3000/api/settings/webhook-test", {
method: "POST",
headers: {
"content-type": "application/json",
cookie: `datastack_admin_session=${ownerToken}`,
},
body: JSON.stringify({
webhookUrl: "https://hooks.example.com/r3-test",
clearWebhookSecret: true,
}),
}),
);
assert.equal(clearRes.status, 200);
assert.equal(capturedTestHeaders["X-DataStack-Signature"], undefined);
} finally {
global.fetch = origFetch;
if (prevSecret === undefined) {
delete process.env.DATASTACK_ADMIN_SESSION_SECRET;
} else {
process.env.DATASTACK_ADMIN_SESSION_SECRET = prevSecret;
}
await SettingsManager.updateSettingsAsync({
enableWebhookAlerts: false,
webhookUrl: "",
clearWebhookSecret: true,
});
}
});
test("R3-Phase-01 [BUG-10]: Brute-force 429 admin login lockout dispatches ADMIN_LOGIN_LOCKOUT webhook alert", async () => {
resetAdminStateForTests();
await createInitialAdmin("r3-owner3@datastack.dev", "OwnerPassword123!");
const origFetch = global.fetch;
const email = `lockout_probe_${Date.now()}@datastack.dev`;
const testIp = "198.51.100.42";
const dispatchedBodies: Array<Record<string, unknown>> = [];
try {
clearWebhookCooldownCacheForTests();
await SettingsManager.updateSettingsAsync({
enableWebhookAlerts: true,
webhookUrl: "https://hooks.example.com/lockout-alert",
webhookSecret: "lockout_hmac_secret_123456",
webhookCooldownSec: 0,
});
global.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
const url = String(input);
if (url.includes("hooks.example.com/lockout-alert")) {
const parsed = JSON.parse(String(init?.body || "{}")) as Record<
string,
unknown
>;
dispatchedBodies.push(parsed);
return new Response("ok", { status: 200 });
}
return new Response("ok", { status: 200 });
}) as typeof fetch;
let lastStatus = 0;
for (let i = 0; i < 6; i += 1) {
const res = await loginPost(
new NextRequest("http://localhost:3000/api/auth/login", {
method: "POST",
headers: {
"content-type": "application/json",
"x-real-ip": testIp,
},
body: JSON.stringify({ email, password: "definitely-wrong-password" }),
}),
);
lastStatus = res.status;
}
assert.equal(lastStatus, 429);
await new Promise((resolve) => setTimeout(resolve, 60));
assert.ok(
dispatchedBodies.length >= 1,
"Expected ADMIN_LOGIN_LOCKOUT webhook alert to be dispatched on 429",
);
assert.equal(dispatchedBodies[0].event, "ADMIN_LOGIN_LOCKOUT");
assert.equal(dispatchedBodies[0].severity, "warning");
assert.equal(dispatchedBodies[0].instance, "DataStack");
} finally {
global.fetch = origFetch;
await clearAdminLoginAttempts(email, getAdminSessionSecret(), testIp);
await SettingsManager.updateSettingsAsync({
enableWebhookAlerts: false,
webhookUrl: "",
clearWebhookSecret: true,
});
}
});
test("R3-Phase-01 [BUG-13]: Port 4000 outbound webhook payload schema matches Port 3000 1:1 (including custom Discord proxies)", async () => {
const fixedTs = "2026-11-20T12:00:00.000Z";
const payload = {
event: "DISK_SPACE_LOW" as const,
severity: "critical" as const,
title: "VPS Disk Threshold Reached",
message: "Disk usage reached 96%.",
clientId: "iot_edge_01",
details: { usedPct: 96 },
timestamp: fixedTs,
};
const port3000Json = JSON.parse(formatPort3000WebhookBody(payload));
const port4000Json = JSON.parse(
ingestionServer.formatOutboundWebhookBody(
"https://discord-proxy.internal.company/webhooks/123",
payload,
fixedTs,
),
);
assert.deepEqual(port4000Json, port3000Json);
assert.equal(port4000Json.instance, "DataStack");
assert.equal(port4000Json.title, "VPS Disk Threshold Reached");
assert.equal(typeof port4000Json.text, "string");
assert.equal(typeof port4000Json.content, "string");
// Also verify end-to-end Port 4000 HMAC signature uses decrypted secret cleanly
ingestionServer.clearWebhookCooldownCacheForTests();
let capturedBody = "";
let capturedSig = "";
const secret = "hmac_parity_secret_2026";
await ingestionServer.triggerWebhookAlert(payload, {
enableWebhookAlerts: true,
webhookUrl: "https://custom-webhook.example.com/ingest",
webhookSecret: secret,
bypassCooldown: true,
fetchImpl: async (
_url: string,
init: { body: string; headers: Record<string, string> },
) => {
capturedBody = init.body;
capturedSig = init.headers["X-DataStack-Signature"];
return { ok: true, status: 200 };
},
});
const expectedSig = `sha256=${crypto
.createHmac("sha256", secret)
.update(capturedBody, "utf8")
.digest("hex")}`;
assert.equal(capturedSig, expectedSig);
});
@@ -0,0 +1,347 @@
import assert from "node:assert/strict";
import fs from "node:fs";
import http from "node:http";
import os from "node:os";
import path from "node:path";
import test from "node:test";
import zlib from "node:zlib";
import { NextRequest } from "next/server";
import { GET as dataGet } from "../app/api/data/route";
import { GET as exportGet } from "../app/api/export/route";
import { SettingsManager } from "../core/adminSettings/settingsManager";
import {
createAdminSessionToken,
getAdminSessionSecret,
} from "../core/auth/adminSession";
import {
createInitialAdmin,
resetAdminStateForTests,
} from "../core/auth/adminUsers";
import { DataBufferEngine } from "../core/data/buffer";
import { SDKManager } from "../core/sdk/sdkManager";
import { CSVStorageManager } from "../core/Storage/csvStorage";
const { CsvWriterWorker } = require("../server/workers/csvWriter.js");
const ORIGINAL_STORAGE_DIR = CSVStorageManager.baseStorageDir;
const ORIGINAL_DB_URL = process.env.DATABASE_URL;
const ORIGINAL_SESSION_SECRET = process.env.DATASTACK_ADMIN_SESSION_SECRET;
const ORIGINAL_JWT_SECRET = process.env.DATASTACK_JWT_SECRET;
function makeTempDir(prefix: string): string {
return fs.mkdtempSync(path.join(os.tmpdir(), prefix));
}
function writeGzipCsv(filePath: string, content: string): void {
fs.mkdirSync(path.dirname(filePath), { recursive: true });
fs.writeFileSync(filePath, zlib.gzipSync(Buffer.from(content, "utf8")));
}
async function createOwnerCookie(): Promise<string> {
delete process.env.DATABASE_URL;
const sessionSecret = getAdminSessionSecret();
resetAdminStateForTests();
const ownerEmail = "owner-r3p2@datastack.local";
await createInitialAdmin(ownerEmail, "StrongPassword123!");
const token = await createAdminSessionToken(
ownerEmail,
sessionSecret,
Date.now() + 3600_000,
1,
"owner",
"owner-1",
);
return `datastack_admin_session=${token}`;
}
test.after(async () => {
SettingsManager.stopMaintenanceSchedulerForTests();
SDKManager.stopActivityFlushTimerForTests();
await DataBufferEngine.shutdown();
CSVStorageManager.baseStorageDir = ORIGINAL_STORAGE_DIR;
CSVStorageManager.clearRowCountCache();
CSVStorageManager.clearQuotaCaches();
if (ORIGINAL_DB_URL === undefined) delete process.env.DATABASE_URL;
else process.env.DATABASE_URL = ORIGINAL_DB_URL;
if (ORIGINAL_SESSION_SECRET === undefined) {
delete process.env.DATASTACK_ADMIN_SESSION_SECRET;
} else {
process.env.DATASTACK_ADMIN_SESSION_SECRET = ORIGINAL_SESSION_SECRET;
}
if (ORIGINAL_JWT_SECRET === undefined) delete process.env.DATASTACK_JWT_SECRET;
else process.env.DATASTACK_JWT_SECRET = ORIGINAL_JWT_SECRET;
resetAdminStateForTests();
});
test("R3-Phase-02 BUG-09: Port 3000 and Port 4000 extractDateFromFilename parse -gzN and -partN-gzN identically", () => {
const tmpDir = makeTempDir("datastack-r3p2-date-");
const worker = new CsvWriterWorker(tmpDir);
try {
const cases: Array<[string, string]> = [
["2026-10-01.csv", "2026-10-01"],
["2026-10-01.csv.gz", "2026-10-01"],
["2026-10-01-part2.csv", "2026-10-01"],
["2026-10-01-part12.csv.gz", "2026-10-01"],
["2026-10-01-gz2.csv.gz", "2026-10-01"],
["2026-10-01-part2-gz3.csv.gz", "2026-10-01"],
["2026-10-01-gz2-part3.csv.gz", "2026-10-01"],
["not-a-date.csv.gz", "not-a-date"],
];
for (const [filename, expected] of cases) {
assert.equal(
CSVStorageManager.extractDateFromFilename(filename),
expected,
`Port 3000 mismatch for ${filename}`,
);
assert.equal(
worker.extractDateFromFilename(filename),
expected,
`Port 4000 mismatch for ${filename}`,
);
}
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});
test("R3-Phase-02 BUG-09: -gzN collision files work in /api/data (to filter + fallback timestamp), /api/export (to filter), and Port 3000/4000 pruning", async () => {
const tmpDir = makeTempDir("datastack-r3p2-gzn-");
const cookie = await createOwnerCookie();
SettingsManager.updateSettings({
autoPruneDays: 0,
compressAfterDays: 0,
enableS3Backup: false,
});
CSVStorageManager.baseStorageDir = tmpDir;
CSVStorageManager.clearRowCountCache();
CSVStorageManager.clearQuotaCaches();
try {
const clientDir = path.join(tmpDir, "gzn_client");
const targetDate = "2025-02-15";
const gz2Path = path.join(clientDir, `${targetDate}-gz2.csv.gz`);
const partGzPath = path.join(clientDir, `${targetDate}-part2-gz3.csv.gz`);
// Row in -gz2 intentionally omits timestamp to verify valid ISO fallback (`2025-02-15T00:00:00.000Z`)
writeGzipCsv(
gz2Path,
`clientId,eventId,timestamp,payload\ngzn_client,evt_gz2_1,,"{""src"":""gz2""}"\n`,
);
writeGzipCsv(
partGzPath,
`clientId,eventId,timestamp,payload\ngzn_client,evt_part_gz3_1,${targetDate}T12:30:00.000Z,"{""src"":""part-gz3""}"\n`,
);
// 1. /api/data with exact to=2025-02-15 MUST return BOTH events with valid ISO timestamps
const dataRes = await dataGet(
new NextRequest(
`http://localhost:3000/api/data?clientId=gzn_client&from=${targetDate}&to=${targetDate}`,
{ headers: { cookie } },
),
);
assert.equal(dataRes.status, 200);
const dataJson = (await dataRes.json()) as {
total: number;
events: Array<{ eventId: string; timestamp: string }>;
};
assert.equal(dataJson.total, 2);
const gz2Evt = dataJson.events.find((e) => e.eventId === "evt_gz2_1");
assert.ok(gz2Evt, "evt_gz2_1 must not be excluded by to= filter");
assert.equal(gz2Evt.timestamp, `${targetDate}T00:00:00.000Z`);
assert.ok(
dataJson.events.some((e) => e.eventId === "evt_part_gz3_1"),
"evt_part_gz3_1 must not be excluded by to= filter",
);
// 2. /api/export with exact to=2025-02-15 MUST include BOTH -gzN files
const exportRes = await exportGet(
new NextRequest(
`http://localhost:3000/api/export?clientId=gzn_client&from=${targetDate}&to=${targetDate}&format=json&flush=false`,
{ headers: { cookie } },
),
);
assert.equal(exportRes.status, 200);
const exported = JSON.parse(await exportRes.text()) as Array<{
eventId: string;
}>;
assert.equal(exported.length, 2);
assert.ok(exported.some((e) => e.eventId === "evt_gz2_1"));
assert.ok(exported.some((e) => e.eventId === "evt_part_gz3_1"));
// 3. Port 3000 pruneExpiredPartitionsAsync deletes -gzN collision files older than cutoff
const pruned3000 = await CSVStorageManager.pruneExpiredPartitionsAsync(7);
assert.equal(pruned3000, 2);
assert.equal(fs.existsSync(gz2Path), false);
assert.equal(fs.existsSync(partGzPath), false);
// 4. Port 4000 CsvWriterWorker.pruneExpiredPartitionsAsync also deletes -gzN collision files
writeGzipCsv(
gz2Path,
`clientId,eventId,timestamp,payload\ngzn_client,evt_gz2_2,${targetDate}T00:00:00.000Z,"{}"\n`,
);
const worker = new CsvWriterWorker(tmpDir);
const pruned4000 = await worker.pruneExpiredPartitionsAsync(7);
assert.equal(pruned4000, 1);
assert.equal(fs.existsSync(gz2Path), false);
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});
test("R3-Phase-02 BUG-05: Singleton periodic scheduler automatically archives expired files (including -gzN) to S3 and prunes locally, with overlap and unhydrated-DB guards", async () => {
const tmpDir = makeTempDir("datastack-r3p2-s3sched-");
delete process.env.DATABASE_URL;
process.env.DATASTACK_ADMIN_SESSION_SECRET =
"r3-phase02-test-session-secret-0123456789abcdef";
CSVStorageManager.baseStorageDir = tmpDir;
CSVStorageManager.clearRowCountCache();
CSVStorageManager.clearQuotaCaches();
SettingsManager.stopMaintenanceSchedulerForTests();
const uploadedPaths: string[] = [];
let failS3 = false;
const mockS3 = http.createServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (chunk: Buffer) => chunks.push(chunk));
req.on("end", () => {
if (failS3) {
res.statusCode = 500;
res.end("S3 temporary error");
return;
}
if (req.method === "PUT") {
uploadedPaths.push(req.url || "");
res.statusCode = 200;
res.end();
return;
}
res.statusCode = 200;
res.end();
});
});
await new Promise<void>((resolve) => mockS3.listen(0, "127.0.0.1", resolve));
const addr = mockS3.address();
assert.ok(addr && typeof addr === "object");
try {
// Configure S3 backup + autoPruneDays (cooldown will be reset so scheduled timer runs)
await SettingsManager.updateSettingsAsync({
enableS3Backup: true,
autoPruneDays: 7,
compressAfterDays: 0,
s3Endpoint: `http://127.0.0.1:${addr.port}`,
s3Region: "auto",
s3Bucket: "sched-bucket",
s3Prefix: "auto-archive/",
s3AccessKeyId: "AKIA_SCHED_TEST",
s3SecretAccessKey: "super_secret_sched_key_9999",
});
assert.equal(
SettingsManager.isMaintenanceSchedulerActive(),
true,
"Scheduler must auto-bootstrap on settings access/update",
);
// Confirm singleton guard returns false when called again with no interval change
assert.equal(SettingsManager.ensureMaintenanceScheduler(), false);
// Plant expired files AFTER initial save so ONLY the scheduled timer can archive+prune them
const clientDir = path.join(tmpDir, "sched_client");
fs.mkdirSync(clientDir, { recursive: true });
const oldCsv = path.join(clientDir, "2024-11-01.csv");
const oldGz2 = path.join(clientDir, "2024-11-01-gz2.csv.gz");
const todayDate = new Date().toISOString().slice(0, 10);
const todayCsv = path.join(clientDir, `${todayDate}.csv`);
fs.writeFileSync(
oldCsv,
`clientId,eventId,timestamp,payload\nsched_client,evt_old_csv,2024-11-01T00:00:00.000Z,"{}"\n`,
);
writeGzipCsv(
oldGz2,
`clientId,eventId,timestamp,payload\nsched_client,evt_old_gz2,2024-11-01T01:00:00.000Z,"{}"\n`,
);
fs.writeFileSync(
todayCsv,
`clientId,eventId,timestamp,payload\nsched_client,evt_today,${todayDate}T00:00:00.000Z,"{}"\n`,
);
// Expire prune cooldown and start a fast 30ms test interval to verify real timer firing
const globals = globalThis as typeof globalThis & {
__dataStackLastPruneMeta?: { days: number; ranAtMs: number };
__dataStackMaintenanceRunning?: boolean;
__dataStackSettingsHydrated?: boolean;
};
globals.__dataStackLastPruneMeta = {
days: 7,
ranAtMs: Date.now() - 2 * 60 * 60 * 1000,
};
SettingsManager.ensureMaintenanceScheduler(30);
// Wait up to 1.5s for scheduled timer tick to upload to S3 and prune both expired files
const deadline = Date.now() + 1500;
while (
Date.now() < deadline &&
(fs.existsSync(oldCsv) || fs.existsSync(oldGz2))
) {
await new Promise((r) => setTimeout(r, 25));
}
SettingsManager.stopMaintenanceSchedulerForTests();
assert.equal(fs.existsSync(oldCsv), false, "Expired .csv must be pruned after S3 upload");
assert.equal(fs.existsSync(oldGz2), false, "Expired -gz2.csv.gz must be pruned after S3 upload");
assert.equal(fs.existsSync(todayCsv), true, "Today's partition must remain intact");
assert.ok(
uploadedPaths.some((u) => u.includes("/sched-bucket/auto-archive/sched_client/2024-11-01.csv")),
`Expected 2024-11-01.csv S3 upload, got: ${uploadedPaths.join(", ")}`,
);
assert.ok(
uploadedPaths.some((u) => u.includes("/sched-bucket/auto-archive/sched_client/2024-11-01-gz2.csv.gz")),
`Expected 2024-11-01-gz2.csv.gz S3 upload, got: ${uploadedPaths.join(", ")}`,
);
// Verify fail-safe when S3 returns 500: local expired file MUST NOT be deleted
failS3 = true;
const failCsv = path.join(clientDir, "2024-10-15.csv");
fs.writeFileSync(
failCsv,
`clientId,eventId,timestamp,payload\nsched_client,evt_fail_s3,2024-10-15T00:00:00.000Z,"{}"\n`,
);
await SettingsManager.runScheduledMaintenanceTick({ forcePrune: true });
assert.equal(
fs.existsSync(failCsv),
true,
"Fail-safe: local file must NEVER be deleted when S3 upload fails",
);
// Verify overlap guard blocks concurrent execution
globals.__dataStackMaintenanceRunning = true;
const overlapResult = await SettingsManager.runScheduledMaintenanceTick({
forcePrune: true,
});
globals.__dataStackMaintenanceRunning = false;
assert.equal(overlapResult.ran, false);
assert.equal(overlapResult.skippedReason, "ALREADY_RUNNING");
// Verify unhydrated-DB guard skips maintenance when DATABASE_URL is configured but unhydrated
process.env.DATABASE_URL = "postgresql://user:pass@127.0.0.1:1/unreachable";
globals.__dataStackSettingsHydrated = false;
const unhydratedResult = await SettingsManager.runScheduledMaintenanceTick({
forcePrune: true,
skipDbSync: true,
});
delete process.env.DATABASE_URL;
assert.equal(unhydratedResult.ran, false);
assert.equal(unhydratedResult.skippedReason, "DB_UNHYDRATED");
} finally {
SettingsManager.stopMaintenanceSchedulerForTests();
await new Promise<void>((resolve) => mockS3.close(() => resolve()));
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});
+497
View File
@@ -0,0 +1,497 @@
import assert from 'node:assert/strict';
import fs from 'node:fs/promises';
import http from 'node:http';
import { createRequire } from 'node:module';
import os from 'node:os';
import path from 'node:path';
import test from 'node:test';
import jwt from 'jsonwebtoken';
import { Server as SocketIOServer } from 'socket.io';
import {
DataStack,
DataStackAuthError,
DataStackQuotaExceededError,
} from '../sdk/node/datastack';
const require = createRequire(import.meta.url);
const { setupSocketHandlers } = require('../server/socket/handlers.js') as {
setupSocketHandlers: (
io: SocketIOServer,
bufferQueue: Array<Record<string, unknown>>,
worker: {
flush?: (rows: Array<Record<string, unknown>>) => Promise<void>;
checkStorageAdmission?: (
clientId: string,
quotaMb: number,
diskPct: number
) => Promise<{ allowed: boolean; code?: string; reason?: string }>;
},
options?: Record<string, unknown>
) => unknown;
};
const TEST_JWT_SECRET = 'r3-phase03-sdk-parity-jwt-secret-key-32-chars';
function signClientJwt(clientId: string): string {
return jwt.sign({ clientId, sub: clientId }, TEST_JWT_SECRET, {
expiresIn: '1h',
});
}
async function startLiveIngestionSocketServer(options?: {
verifyClientInDb?: (
clientId: string,
token: string,
forceRefresh: boolean,
clientIp: string
) => Promise<{
allowed: boolean;
code?: string;
reason?: string;
rateLimit?: number;
storageQuotaMb?: number;
}>;
checkStorageAdmission?: (
clientId: string,
quotaMb: number,
diskPct: number
) => Promise<{ allowed: boolean; code?: string; reason?: string }>;
}) {
const httpServer = http.createServer((_req, res) => {
res.writeHead(404);
res.end();
});
const io = new SocketIOServer(httpServer, {
cors: { origin: '*' },
transports: ['websocket', 'polling'],
});
const bufferQueue: Array<Record<string, unknown>> = [];
const flushed: Array<Record<string, unknown>> = [];
setupSocketHandlers(
io,
bufferQueue,
{
flush: async (rows: Array<Record<string, unknown>>) => {
flushed.push(...rows);
},
checkStorageAdmission:
options?.checkStorageAdmission ??
(async () => ({ allowed: true })),
},
{
getRuntimeConfig: () => ({
jwtSecret: TEST_JWT_SECRET,
maxEventsPerSec: 200,
maxPayloadSizeBytes: 100 * 1024,
batchFlushThreshold: 500,
enableIdempotency: true,
diskPauseThresholdPct: 95,
}),
verifyClientInDb:
options?.verifyClientInDb ??
(async () => ({
allowed: true,
rateLimit: 200,
storageQuotaMb: 0,
})),
}
);
await new Promise<void>((resolve) => {
httpServer.listen(0, '127.0.0.1', () => resolve());
});
const addr = httpServer.address();
const port = typeof addr === 'object' && addr ? addr.port : 0;
return {
port,
url: `http://127.0.0.1:${port}`,
io,
bufferQueue,
flushed,
close: async () => {
await new Promise<void>((resolve) => {
io.close(() => {
httpServer.close(() => resolve());
});
});
},
};
}
test('R3-Phase-03 (BUG-07): Compiled Node.js SDK dist/index.js loads cleanly via plain CommonJS require() and exports typed package entrypoints', async () => {
const pkgPath = path.join(
process.cwd(),
'sdk',
'node',
'datastack',
'package.json'
);
const pkg = JSON.parse(await fs.readFile(pkgPath, 'utf8')) as {
main: string;
types: string;
exports?: { '.': { default: string; types: string } };
scripts?: { build?: string };
};
assert.equal(pkg.main, './dist/index.js');
assert.equal(pkg.types, './dist/index.d.ts');
assert.equal(pkg.exports?.['.']?.default, './dist/index.js');
assert.equal(pkg.exports?.['.']?.types, './dist/index.d.ts');
assert.ok(pkg.scripts?.build?.includes('tsc'));
// Verify compiled CommonJS output loads directly via Node require() with zero TypeScript transpilation
const distPath = path.join(
process.cwd(),
'sdk',
'node',
'datastack',
'dist',
'index.js'
);
const loaded = require(distPath) as Record<string, unknown>;
assert.equal(typeof loaded.DataStack, 'function');
assert.equal(typeof loaded.DataStackClient, 'function');
assert.equal(typeof loaded.DataStackAuthError, 'function');
assert.equal(typeof loaded.DataStackQuotaExceededError, 'function');
assert.equal(typeof loaded.SDK_VERSION, 'string');
const dtsPath = path.join(
process.cwd(),
'sdk',
'node',
'datastack',
'dist',
'index.d.ts'
);
const dtsContent = await fs.readFile(dtsPath, 'utf8');
assert.match(dtsContent, /export declare class DataStack/);
});
test('R3-Phase-03 (BUG-02): Live Port 4000 Socket.IO batch_sync accepts Node.js SDK raw Array payload and reports processed + duplicates accurately', async () => {
const server = await startLiveIngestionSocketServer();
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'ds-r3-p03-ws-'));
let client: DataStack | null = null;
try {
const token = signClientJwt('node_ws_batch_client');
client = new DataStack({
serverUrl: server.url,
apiUrl: 'http://127.0.0.1:1', // Unreachable HTTP to prove WS-only delivery
token,
autoConnect: false,
enableHttpFallback: false,
offlineSpoolEnabled: false,
heartbeatIntervalMs: 5000,
});
await client.connect();
assert.equal(client.status().connected, true);
// 1. Live WebSocket sendBatch with explicit eventIds (must send raw Array, NOT { clientId, batch })
const batchResult = await client.sendBatch([
{ eventId: 'r3_ws_evt_1', metric: 'cpu', value: 41.2 },
{ eventId: 'r3_ws_evt_2', metric: 'mem', value: 68.5 },
{ eventId: 'r3_ws_evt_3', metric: 'disk', value: 52.0 },
]);
assert.equal(batchResult.success, true);
assert.equal(batchResult.transport, 'websocket');
assert.equal(batchResult.accepted, 3);
assert.equal(batchResult.duplicates, 0);
assert.equal(batchResult.spooled, 0);
assert.equal(server.bufferQueue.length, 3);
// 2. Idempotent re-send of 2 duplicates + 1 new event over live WebSocket batch_sync
const dedupResult = await client.batchSync([
{ eventId: 'r3_ws_evt_1', metric: 'cpu', value: 41.2 },
{ eventId: 'r3_ws_evt_2', metric: 'mem', value: 68.5 },
{ eventId: 'r3_ws_evt_4', metric: 'net', value: 12.8 },
]);
assert.equal(dedupResult.success, true);
assert.equal(dedupResult.transport, 'websocket');
assert.equal(dedupResult.accepted, 1);
assert.equal(dedupResult.duplicates, 2);
assert.equal(dedupResult.spooled, 0);
assert.equal(server.bufferQueue.length, 4);
// 3. Live WebSocket sendFile() + idempotent file re-send (deterministic eventIds)
const csvFile = path.join(tmpDir, 'telemetry.csv');
await fs.writeFile(
csvFile,
'node,temp\na1,23.5\na2,24.1\n',
'utf8'
);
const fileRes1 = await client.sendFile(csvFile, { chunkSize: 10 });
assert.equal(fileRes1.success, true);
assert.equal(fileRes1.totalRows, 2);
assert.equal(fileRes1.acceptedRows, 2);
assert.equal(fileRes1.duplicates, 0);
assert.equal(server.bufferQueue.length, 6);
const fileRes2 = await client.sendFile(csvFile, { chunkSize: 10 });
assert.equal(fileRes2.success, true);
assert.equal(fileRes2.totalRows, 2);
assert.equal(fileRes2.acceptedRows, 0);
assert.equal(fileRes2.duplicates, 2);
assert.equal(server.bufferQueue.length, 6);
} finally {
if (client) {
await client.disconnect();
}
await server.close();
await fs.rm(tmpDir, { recursive: true, force: true });
}
});
test('R3-Phase-03 (BUG-07): Node.js SDK heartbeatIntervalMs monitor stays zero-CPU in IDLE_WS, activates on drop, recovers via ping_verify, and cleans up on disconnect', async () => {
const server = await startLiveIngestionSocketServer();
let client: DataStack | null = null;
try {
const token = signClientJwt('node_hb_client');
client = new DataStack({
serverUrl: server.url,
apiUrl: 'http://127.0.0.1:1',
token,
autoConnect: false,
enableHttpFallback: false,
offlineSpoolEnabled: false,
heartbeatIntervalMs: 120,
requestTimeoutMs: 1500,
});
assert.equal(client.status().heartbeatActive, false);
await client.connect();
// Zero-CPU IDLE_WS parity: heartbeat timer is paused while socket is healthy
assert.equal(client.status().connected, true);
assert.equal(client.status().transportState, 'IDLE_WS');
assert.equal(client.status().heartbeatActive, false);
// Force server-side disconnect of all sockets -> heartbeat monitor activates immediately
for (const sock of server.io.sockets.sockets.values()) {
sock.disconnect(true);
}
// Wait briefly to observe disconnect + automatic heartbeat ping_verify recovery
let sawHeartbeatActive = false;
const deadline = Date.now() + 3000;
while (Date.now() < deadline) {
const snap = client.status();
if (snap.heartbeatActive) {
sawHeartbeatActive = true;
}
if (sawHeartbeatActive && snap.connected && !snap.heartbeatActive) {
break;
}
await new Promise((r) => setTimeout(r, 20));
}
assert.equal(sawHeartbeatActive, true);
assert.equal(client.status().connected, true);
assert.equal(client.status().transportState, 'IDLE_WS');
assert.equal(client.status().heartbeatActive, false);
// Verify recovered WebSocket can immediately send live telemetry
const afterRecover = await client.send({
eventId: 'hb_recovered_1',
status: 'alive',
});
assert.equal(afterRecover.success, true);
assert.equal(afterRecover.transport, 'websocket');
assert.equal(afterRecover.accepted, 1);
// Disconnect must stop heartbeat timer cleanly
await client.disconnect();
assert.equal(client.status().connected, false);
assert.equal(client.status().heartbeatActive, false);
} finally {
if (client) {
await client.disconnect();
}
await server.close();
}
});
test('R3-Phase-03 (BUG-07): Single HTTP DUPLICATE_DROPPED reports accepted=0, duplicates=1 and TOKEN_ROTATED / CLIENT_REVOKED fail fast with DataStackAuthError', async () => {
const token = signClientJwt('node_http_dedup_auth');
let mode: 'dup' | 'rotated' | 'revoked' = 'dup';
let fetchCalls = 0;
const mockFetch: typeof fetch = async () => {
fetchCalls += 1;
if (mode === 'dup') {
return new Response(
JSON.stringify({
success: true,
eventId: 'dup_single_1',
note: 'DUPLICATE_DROPPED',
}),
{ status: 200, headers: { 'Content-Type': 'application/json' } }
);
}
if (mode === 'rotated') {
return new Response(
JSON.stringify({
success: false,
code: 'TOKEN_ROTATED',
error: 'Client token was rotated; issue new credentials',
}),
{ status: 401, headers: { 'Content-Type': 'application/json' } }
);
}
return new Response(
JSON.stringify({
success: false,
code: 'CLIENT_REVOKED',
error: 'Client credential has been revoked',
}),
{ status: 403, headers: { 'Content-Type': 'application/json' } }
);
};
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'ds-r3-p03-auth-'));
const client = new DataStack({
serverUrl: 'http://127.0.0.1:1',
apiUrl: 'http://127.0.0.1:3000',
token,
autoConnect: false,
maxRetries: 3,
offlineSpoolEnabled: true,
offlineSpoolDir: tmpDir,
fetchImpl: mockFetch,
});
try {
// 1. Single HTTP DUPLICATE_DROPPED -> accepted: 0, duplicates: 1
mode = 'dup';
const dupRes = await client.send({
eventId: 'dup_single_1',
metric: 'temp',
val: 20,
});
assert.equal(dupRes.success, true);
assert.equal(dupRes.accepted, 0);
assert.equal(dupRes.duplicates, 1);
assert.equal(dupRes.spooled ?? 0, 0);
// 2. TOKEN_ROTATED -> immediate DataStackAuthError (401), 0 retries, 0 spooling
fetchCalls = 0;
mode = 'rotated';
await assert.rejects(
() => client.send({ eventId: 'rot_1', k: 1 }),
(err: unknown) =>
err instanceof DataStackAuthError &&
err.code === 'TOKEN_ROTATED' &&
err.statusCode === 401
);
assert.equal(fetchCalls, 1);
assert.equal(client.status().spooledCount, 0);
// 3. CLIENT_REVOKED -> immediate DataStackAuthError (403), 0 retries, 0 spooling
fetchCalls = 0;
mode = 'revoked';
await assert.rejects(
() => client.sendBatch([{ eventId: 'rev_1', k: 2 }]),
(err: unknown) =>
err instanceof DataStackAuthError &&
err.code === 'CLIENT_REVOKED' &&
err.statusCode === 403
);
assert.equal(fetchCalls, 1);
assert.equal(client.status().spooledCount, 0);
} finally {
await client.disconnect();
await fs.rm(tmpDir, { recursive: true, force: true });
}
});
test('R3-Phase-03 (BUG-07): Live WebSocket TOKEN_ROTATED, CLIENT_REVOKED, and STORAGE_QUOTA_EXCEEDED fail fast without spooling', async () => {
let dbState: 'ok' | 'rotated' | 'revoked' | 'quota' = 'ok';
const server = await startLiveIngestionSocketServer({
verifyClientInDb: async () => {
if (dbState === 'rotated') {
return {
allowed: false,
code: 'TOKEN_ROTATED',
reason: 'Token rotated by administrator',
};
}
if (dbState === 'revoked') {
return {
allowed: false,
code: 'CLIENT_REVOKED',
reason: 'Client revoked by administrator',
};
}
return { allowed: true, rateLimit: 200, storageQuotaMb: 10 };
},
checkStorageAdmission: async () => {
if (dbState === 'quota') {
return {
allowed: false,
code: 'STORAGE_QUOTA_EXCEEDED',
reason: 'Client quota reached 10MB',
};
}
return { allowed: true };
},
});
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'ds-r3-p03-ws-err-'));
const token = signClientJwt('node_ws_errors');
const client = new DataStack({
serverUrl: server.url,
apiUrl: 'http://127.0.0.1:1',
token,
autoConnect: false,
enableHttpFallback: true,
offlineSpoolEnabled: true,
offlineSpoolDir: tmpDir,
heartbeatIntervalMs: 5000,
});
try {
await client.connect();
// 1. Live WS batch_sync quota exceeded -> DataStackQuotaExceededError (507), 0 spooling
dbState = 'quota';
await assert.rejects(
() => client.sendBatch([{ eventId: 'ws_q_1', v: 1 }]),
(err: unknown) =>
err instanceof DataStackQuotaExceededError &&
err.code === 'STORAGE_QUOTA_EXCEEDED' &&
err.statusCode === 507
);
assert.equal(client.status().spooledCount, 0);
// 2. Live WS batch_sync TOKEN_ROTATED -> DataStackAuthError (401), 0 spooling
dbState = 'rotated';
await assert.rejects(
() => client.sendBatch([{ eventId: 'ws_rot_1', v: 2 }]),
(err: unknown) =>
err instanceof DataStackAuthError &&
err.code === 'TOKEN_ROTATED' &&
err.statusCode === 401
);
assert.equal(client.status().spooledCount, 0);
// 3. Reconnect while CLIENT_REVOKED -> connect_error maps to DataStackAuthError (403)
dbState = 'revoked';
await assert.rejects(
() => client.connect(),
(err: unknown) =>
err instanceof DataStackAuthError &&
err.code === 'CLIENT_REVOKED' &&
err.statusCode === 403
);
} finally {
await client.disconnect();
await server.close();
await fs.rm(tmpDir, { recursive: true, force: true });
}
});
+271
View File
@@ -0,0 +1,271 @@
import assert from "node:assert/strict";
import fs from "node:fs/promises";
import path from "node:path";
import test from "node:test";
import { NextRequest } from "next/server";
import { GET as prometheusGet } from "../app/api/prometheus/route";
import { SettingsManager } from "../core/adminSettings/settingsManager";
import { ApiEngine } from "../core/apiEngine/apiService";
import {
createAdminSessionToken,
getAdminSessionSecret,
} from "../core/auth/adminSession";
import {
createInitialAdmin,
resetAdminStateForTests,
} from "../core/auth/adminUsers";
import { SDKManager } from "../core/sdk/sdkManager";
import {
classifyUiClientStatus,
formatDateForLocalInput,
getVconRoleCapabilities,
isUiClientExpired,
shouldRedirectSimulatorUnauthorized,
summarizeUiClientCounts,
} from "../lib/vconUiHelpers";
async function makeOwnerCookie(): Promise<string> {
resetAdminStateForTests();
await createInitialAdmin("r3-p4-owner@datastack.dev", "R3Phase04Pass!");
const secret = getAdminSessionSecret()!;
const token = await createAdminSessionToken(
"r3-p4-owner@datastack.dev",
secret,
Date.now() + 3600_000,
1,
"owner",
);
return `datastack_admin_session=${token}`;
}
test("R3-Phase-04 [BUG-11 / ERR-R3-07]: shouldRedirectSimulatorUnauthorized separates client 401s from admin session 401s", async () => {
// Non-401 statuses never redirect
assert.equal(shouldRedirectSimulatorUnauthorized(200, {}), false);
assert.equal(shouldRedirectSimulatorUnauthorized(400, { code: "UNAUTHORIZED" }), false);
assert.equal(shouldRedirectSimulatorUnauthorized(403, { code: "FORBIDDEN" }), false);
assert.equal(shouldRedirectSimulatorUnauthorized(500, null), false);
// Client token 401 codes stay on /vcon/simulator and show inline response
for (const clientCode of [
"CLIENT_TOKEN_EXPIRED",
"client_token_expired",
"TOKEN_ROTATED",
"AUTH_INVALID",
"CLIENT_REVOKED",
"CLIENT_IP_FORBIDDEN",
"CLIENT_UNAUTHORIZED",
]) {
assert.equal(
shouldRedirectSimulatorUnauthorized(401, {
code: clientCode,
error: "Client rejected",
}),
false,
`Expected no redirect for client code ${clientCode}`,
);
}
// Admin session 401 codes or uncoded 401 responses redirect to /vcon/login
assert.equal(
shouldRedirectSimulatorUnauthorized(401, { code: "UNAUTHORIZED" }),
true,
);
assert.equal(
shouldRedirectSimulatorUnauthorized(401, { code: "SESSION_EXPIRED" }),
true,
);
assert.equal(shouldRedirectSimulatorUnauthorized(401, {}), true);
assert.equal(shouldRedirectSimulatorUnauthorized(401, null), true);
// Verify simulator page wires shouldRedirectSimulatorUnauthorized in both send handlers
const simSource = await fs.readFile(
path.join(process.cwd(), "app/vcon/simulator/page.tsx"),
"utf8",
);
assert.ok(simSource.includes("shouldRedirectSimulatorUnauthorized"));
assert.equal(
simSource.includes("if (res.status === 401) {\n window.location.assign"),
false,
"Simulator must parse JSON and evaluate shouldRedirectSimulatorUnauthorized before redirecting",
);
});
test("R3-Phase-04 [BUG-12]: formatDateForLocalInput preserves local timezone round-trip without UTC offset drift", async () => {
assert.equal(formatDateForLocalInput(null), "");
assert.equal(formatDateForLocalInput(undefined), "");
assert.equal(formatDateForLocalInput(""), "");
assert.equal(formatDateForLocalInput("not-a-valid-date"), "");
// Round-trip multiple UTC instants across winter/summer DST boundaries
for (const iso of [
"2026-01-15T08:30:00.000Z",
"2026-07-20T23:45:00.000Z",
"2026-11-20T14:05:00.000Z",
]) {
const formatted = formatDateForLocalInput(iso);
assert.match(formatted, /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}$/);
const roundTripped = new Date(formatted).toISOString();
assert.equal(
roundTripped,
iso,
`Round-trip local datetime-local value for ${iso} drifted to ${roundTripped}`,
);
}
// Verify ClientsTable uses formatDateForLocalInput instead of toISOString().slice(0, 16)
const clientsTableSource = await fs.readFile(
path.join(process.cwd(), "components/sections/ClientsTable.tsx"),
"utf8",
);
assert.ok(clientsTableSource.includes("formatDateForLocalInput(client.expiresAt)"));
assert.equal(
clientsTableSource.includes("toISOString().slice(0, 16)"),
false,
"ClientsTable must not format datetime-local using UTC toISOString().slice(0, 16)",
);
});
test("R3-Phase-04 [BUG-14]: expired clients are separated in UI helpers, ApiEngine.getDashboardMetrics, and /api/prometheus", async () => {
const nowMs = Date.parse("2026-11-20T12:00:00.000Z");
assert.equal(isUiClientExpired(null, nowMs), false);
assert.equal(isUiClientExpired("2026-11-20T12:05:00.000Z", nowMs), false);
assert.equal(isUiClientExpired("2026-11-20T12:00:00.000Z", nowMs), true);
assert.equal(isUiClientExpired("2026-11-19T00:00:00.000Z", nowMs), true);
const sampleClients = [
{ status: "active", isOnline: true, lastConnectedAt: "2026-11-20T11:59:00.000Z", expiresAt: null },
{ status: "active", isOnline: false, lastConnectedAt: "2026-11-20T10:00:00.000Z", expiresAt: "2026-12-01T00:00:00.000Z" },
{ status: "active", isOnline: false, lastConnectedAt: null, expiresAt: null },
{ status: "active", isOnline: false, lastConnectedAt: "2026-11-19T10:00:00.000Z", expiresAt: "2026-11-19T12:00:00.000Z" },
{ status: "active", isOnline: true, lastConnectedAt: "2026-11-20T11:58:00.000Z", expiresAt: "2026-11-20T11:00:00.000Z" },
{ status: "revoked", isOnline: false, lastConnectedAt: null, expiresAt: "2026-11-10T00:00:00.000Z" },
];
assert.equal(classifyUiClientStatus(sampleClients[0], nowMs), "online");
assert.equal(classifyUiClientStatus(sampleClients[1], nowMs), "offline");
assert.equal(classifyUiClientStatus(sampleClients[2], nowMs), "pending");
assert.equal(classifyUiClientStatus(sampleClients[3], nowMs), "expired");
assert.equal(classifyUiClientStatus(sampleClients[4], nowMs), "expired");
assert.equal(classifyUiClientStatus(sampleClients[5], nowMs), "revoked");
const summary = summarizeUiClientCounts(sampleClients, nowMs);
assert.deepEqual(summary, {
total: 6,
online: 1,
offline: 1,
pending: 1,
expired: 2,
revoked: 1,
});
// Live integration: ApiEngine.getDashboardMetrics() & GET /api/prometheus
const activeClient = SDKManager.registerClient("r3p4_active_client");
const expiredClient = SDKManager.registerClient("r3p4_expired_client", {
expiresAt: "2020-01-01T00:00:00.000Z",
});
const revokedClient = SDKManager.registerClient("r3p4_revoked_client");
SDKManager.revokeClient(revokedClient.id);
const prevSettings = SettingsManager.getSettings();
try {
SettingsManager.updateSettings({ enablePrometheusMetrics: true });
const metrics = await ApiEngine.getDashboardMetrics();
assert.ok(
metrics.expiredClients >= 1,
`Expected expiredClients >= 1, got ${metrics.expiredClients}`,
);
assert.equal(
metrics.activeClients + metrics.expiredClients <= metrics.totalClients,
true,
"activeClients + expiredClients must not exceed totalClients",
);
// Check Prometheus output counts active vs expired vs revoked
const cookie = await makeOwnerCookie();
const req = new NextRequest("http://localhost:3000/api/prometheus", {
headers: { cookie },
});
const res = await prometheusGet(req);
assert.equal(res.status, 200);
const body = await res.text();
const activeMatch = body.match(/datastack_clients_total\{status="active"\}\s+(\d+)/);
const expiredMatch = body.match(/datastack_clients_total\{status="expired"\}\s+(\d+)/);
const revokedMatch = body.match(/datastack_clients_total\{status="revoked"\}\s+(\d+)/);
assert.ok(activeMatch, "Prometheus must expose status=\"active\" gauge");
assert.ok(expiredMatch, "Prometheus must expose status=\"expired\" gauge");
assert.ok(revokedMatch, "Prometheus must expose status=\"revoked\" gauge");
assert.equal(Number(activeMatch[1]), metrics.activeClients);
assert.equal(Number(expiredMatch[1]), metrics.expiredClients);
assert.ok(Number(expiredMatch[1]) >= 1);
assert.ok(Number(revokedMatch[1]) >= 1);
} finally {
SDKManager.deleteClient(activeClient.id);
SDKManager.deleteClient(expiredClient.id);
SDKManager.deleteClient(revokedClient.id);
SettingsManager.updateSettings({
enablePrometheusMetrics: prevSettings.enablePrometheusMetrics,
});
}
});
test("R3-Phase-04 [BUG-16]: RBAC role capabilities and UI surface wiring across VconShell and Admin pages", async () => {
assert.deepEqual(getVconRoleCapabilities("owner"), {
role: "owner",
canAdminSettings: true,
canManageTeam: true,
canOperate: true,
isReadOnly: false,
});
assert.deepEqual(getVconRoleCapabilities("admin"), {
role: "admin",
canAdminSettings: false,
canManageTeam: false,
canOperate: true,
isReadOnly: false,
});
assert.deepEqual(getVconRoleCapabilities("operator"), {
role: "operator",
canAdminSettings: false,
canManageTeam: false,
canOperate: true,
isReadOnly: false,
});
assert.deepEqual(getVconRoleCapabilities("viewer"), {
role: "viewer",
canAdminSettings: false,
canManageTeam: false,
canOperate: false,
isReadOnly: true,
});
// Fail-closed default for unknown/missing roles
assert.deepEqual(getVconRoleCapabilities("unknown_role"), {
role: "viewer",
canAdminSettings: false,
canManageTeam: false,
canOperate: false,
isReadOnly: true,
});
// Static audit confirming all 6 UI surfaces + layout/shell wire the verified role context
const filesToCheck: Array<{ rel: string; patterns: string[] }> = [
{ rel: "app/vcon/layout.tsx", patterns: ["role={auth.session.role}"] },
{ rel: "components/layout/VconShell.tsx", patterns: ["export function useVconRole()", "VconRoleContext.Provider"] },
{ rel: "components/sections/SettingsWorkspace.tsx", patterns: ["useVconRole()", "canAdminSettings", "canOperate", "Read-only configuration"] },
{ rel: "components/sections/ClientsTable.tsx", patterns: ["useVconRole()", "canManage"] },
{ rel: "app/vcon/clients/page.tsx", patterns: ["useVconRole()", "canOperate", "summarizeUiClientCounts", "Expired"] },
{ rel: "components/sections/DashboardOverview.tsx", patterns: ["useVconRole()", "canOperate", "isUiClientExpired"] },
{ rel: "app/vcon/storage/page.tsx", patterns: ["useVconRole()", "canOperate", "Read-only"] },
{ rel: "app/vcon/simulator/page.tsx", patterns: ["useVconRole()", "canOperate", "Read-only"] },
];
for (const check of filesToCheck) {
const content = await fs.readFile(path.join(process.cwd(), check.rel), "utf8");
for (const pattern of check.patterns) {
assert.ok(
content.includes(pattern),
`Expected ${check.rel} to contain "${pattern}"`,
);
}
}
});
+218
View File
@@ -0,0 +1,218 @@
"""
R3-Phase-03 Verification Tests (BUG-08 / ERR-R3-06):
- `DataStackQuotaExceededError` exported from both canonical `datastack`
and compatibility alias `datastack_sdk` with `retryable=False`.
- HTTP 507 (`STORAGE_QUOTA_EXCEEDED` and `DISK_SPACE_LOW`) immediately raises
`DataStackQuotaExceededError` with 0 retries and 0 offline disk spool entries.
- WebSocket ACK (`STORAGE_QUOTA_EXCEEDED` and `DISK_SPACE_LOW`) immediately
raises `DataStackQuotaExceededError` with 0 retries, 0 HTTP fallback calls,
and 0 offline disk spool entries for both single `send()` and `send_batch()`.
"""
import base64
import json
import sys
import tempfile
import threading
import unittest
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import Any, Dict
REPO_ROOT = Path(__file__).resolve().parents[1]
CANONICAL_PKG_ROOT = REPO_ROOT / "sdk" / "python" / "datastack"
COMPAT_PKG_ROOT = REPO_ROOT / "sdk-python"
for p in (str(CANONICAL_PKG_ROOT), str(COMPAT_PKG_ROOT)):
if p not in sys.path:
sys.path.insert(0, p)
import datastack # noqa: E402
import datastack_sdk # noqa: E402
from datastack import ( # noqa: E402
DataStack,
DataStackError,
DataStackQuotaExceededError,
)
def _make_jwt(client_id: str) -> str:
header = (
base64.urlsafe_b64encode(json.dumps({"alg": "HS256", "typ": "JWT"}).encode())
.decode()
.rstrip("=")
)
body = (
base64.urlsafe_b64encode(
json.dumps({"clientId": client_id, "sub": client_id}).encode()
)
.decode()
.rstrip("=")
)
return f"{header}.{body}.testsig"
class R3Phase03PythonQuotaTests(unittest.TestCase):
def test_quota_exception_exports_and_non_retryable_flag(self) -> None:
self.assertIs(
datastack_sdk.DataStackQuotaExceededError,
datastack.DataStackQuotaExceededError,
)
self.assertIn("DataStackQuotaExceededError", datastack.__all__)
self.assertIn("DataStackQuotaExceededError", datastack_sdk.__all__)
err = DataStackQuotaExceededError("Quota full")
self.assertIsInstance(err, DataStackError)
self.assertFalse(err.retryable)
self.assertEqual(err.code, "STORAGE_QUOTA_EXCEEDED")
def test_http_507_quota_and_disk_low_fail_fast_without_retry_or_spool(self) -> None:
state = {"code": "STORAGE_QUOTA_EXCEEDED", "calls": 0}
tok = _make_jwt("py_quota_http")
class QuotaHttpHandler(BaseHTTPRequestHandler):
def do_POST(self) -> None: # noqa: N802
length = int(self.headers.get("Content-Length", "0"))
_ = self.rfile.read(length)
state["calls"] += 1
body = json.dumps(
{
"success": False,
"code": state["code"],
"error": f"Storage blocked ({state['code']})",
}
).encode("utf-8")
self.send_response(507)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, format: str, *args: Any) -> None:
return
server = ThreadingHTTPServer(("127.0.0.1", 0), QuotaHttpHandler)
port = server.server_address[1]
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
with tempfile.TemporaryDirectory() as tmpdir:
spool_dir = Path(tmpdir) / "spool"
client = DataStack(
server_url="http://127.0.0.1:1",
api_url=f"http://127.0.0.1:{port}",
token=tok,
auto_connect=False,
max_retries=3,
retry_backoff_base_sec=0.05,
request_timeout_sec=1.0,
offline_spool_enabled=True,
offline_spool_dir=str(spool_dir),
)
# 1. HTTP 507 STORAGE_QUOTA_EXCEEDED -> 1 call (0 retries), 0 spooled
state["code"] = "STORAGE_QUOTA_EXCEEDED"
state["calls"] = 0
with self.assertRaises(DataStackQuotaExceededError) as ctx1:
client.send({"sensor": "temp", "val": 100})
self.assertEqual(ctx1.exception.code, "STORAGE_QUOTA_EXCEEDED")
self.assertFalse(ctx1.exception.retryable)
self.assertEqual(state["calls"], 1)
self.assertEqual(client.spooler.pending_count(), 0)
# 2. HTTP 507 DISK_SPACE_LOW on send_batch -> 1 call (0 retries), 0 spooled
state["code"] = "DISK_SPACE_LOW"
state["calls"] = 0
with self.assertRaises(DataStackQuotaExceededError) as ctx2:
client.send_batch([{"sensor": "a", "v": 1}, {"sensor": "b", "v": 2}])
self.assertEqual(ctx2.exception.code, "DISK_SPACE_LOW")
self.assertFalse(ctx2.exception.retryable)
self.assertEqual(state["calls"], 1)
self.assertEqual(client.spooler.pending_count(), 0)
client.disconnect()
finally:
server.shutdown()
server.server_close()
def test_ws_ack_quota_and_disk_low_fail_fast_without_http_fallback_or_spool(
self,
) -> None:
tok = _make_jwt("py_quota_ws")
with tempfile.TemporaryDirectory() as tmpdir:
spool_dir = Path(tmpdir) / "spool"
client = DataStack(
server_url="http://127.0.0.1:4000",
api_url="http://127.0.0.1:3000",
token=tok,
auto_connect=False,
max_retries=3,
retry_backoff_base_sec=0.05,
enable_http_fallback=True,
offline_spool_enabled=True,
offline_spool_dir=str(spool_dir),
)
http_fallback_calls = {"count": 0}
def fail_if_http_called(*args: Any, **kwargs: Any) -> Dict[str, Any]:
http_fallback_calls["count"] += 1
raise AssertionError(
"HTTP fallback must NOT be called after WS DataStackQuotaExceededError"
)
client._send_via_http_fallback = fail_if_http_called # type: ignore[method-assign]
ws_mode = {"code": "STORAGE_QUOTA_EXCEEDED", "calls": 0}
class FakeQuotaSocket:
connected = True
def emit(
self, event: str, payload: Any, callback: Any = None
) -> None:
ws_mode["calls"] += 1
if callback is not None:
callback(
{
"success": False,
"code": ws_mode["code"],
"error": f"WS rejected due to {ws_mode['code']}",
}
)
def disconnect(self) -> None:
self.connected = False
client.sio = FakeQuotaSocket()
client.is_connected = True
# 1. Single send WS STORAGE_QUOTA_EXCEEDED
ws_mode["code"] = "STORAGE_QUOTA_EXCEEDED"
ws_mode["calls"] = 0
with self.assertRaises(DataStackQuotaExceededError) as ctx_ws1:
client.send({"k": "v1"})
self.assertEqual(ctx_ws1.exception.code, "STORAGE_QUOTA_EXCEEDED")
self.assertFalse(ctx_ws1.exception.retryable)
self.assertEqual(ws_mode["calls"], 1)
self.assertEqual(http_fallback_calls["count"], 0)
self.assertEqual(client.spooler.pending_count(), 0)
# 2. Batch send WS DISK_SPACE_LOW
ws_mode["code"] = "DISK_SPACE_LOW"
ws_mode["calls"] = 0
with self.assertRaises(DataStackQuotaExceededError) as ctx_ws2:
client.send_batch([{"k": "v2"}, {"k": "v3"}])
self.assertEqual(ctx_ws2.exception.code, "DISK_SPACE_LOW")
self.assertFalse(ctx_ws2.exception.retryable)
self.assertEqual(ws_mode["calls"], 1)
self.assertEqual(http_fallback_calls["count"], 0)
self.assertEqual(client.spooler.pending_count(), 0)
client.disconnect()
if __name__ == "__main__":
unittest.main(verbosity=2)
+90
View File
@@ -725,3 +725,93 @@ test("V2-Phase-01 [Quotas V2-E04 & V2-E05]: HTTP POST /api/events + Port 4000 So
resetAdminStateForTests();
}
});
test("V2-Audit-Phase-03 [BUG-V2-AUDIT-01, BUG-V2-AUDIT-07, BUG-V2-AUDIT-10]: DiskUsageSnapshot available/freeBytes schema parity, /vcon/clients expiration counter alignment & CSV.GZ partition badges", async () => {
const prevStorageDir = process.env.DATASTACK_STORAGE_DIR;
const origBaseStorageDir = CSVStorageManager.baseStorageDir;
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "datastack-v2-p3-"));
process.env.DATASTACK_STORAGE_DIR = tmpDir;
CSVStorageManager.baseStorageDir = tmpDir;
CSVStorageManager.clearQuotaCaches();
try {
const cookie = await makeAdminCookie();
// 1. BUG-V2-AUDIT-01: CSVStorageManager.getDiskUsageAsync() & GET /api/storage return available: true and freeBytes === availableBytes
CSVStorageManager.setStatfsOverrideForTests(async () => ({
blocks: 2000,
bavail: 1500, // 25% used, 75% free
bsize: 4096,
}));
const snapshot = await CSVStorageManager.getDiskUsageAsync(true);
assert.equal(snapshot.available, true);
assert.equal(snapshot.usedPct, 25);
assert.equal(snapshot.totalBytes, 2000 * 4096);
assert.equal(snapshot.availableBytes, 1500 * 4096);
assert.equal(snapshot.freeBytes, snapshot.availableBytes);
const storageRes = await storageGet(
new NextRequest("http://localhost:3000/api/storage", { headers: { cookie } }),
);
assert.equal(storageRes.status, 200);
const storageBody = await storageRes.json();
assert.equal(storageBody.diskUsage.available, true);
assert.equal(storageBody.diskUsage.freeBytes, 1500 * 4096);
assert.equal(storageBody.diskUsage.availableBytes, 1500 * 4096);
assert.equal(storageBody.diskUsage.usedPct, 25);
// Also verify statfs failure fallback sets available: false and freeBytes: 0
CSVStorageManager.setStatfsOverrideForTests(async () => {
throw new Error("Simulated statfs failure");
});
const fallbackSnapshot = await CSVStorageManager.getDiskUsageAsync(true);
assert.equal(fallbackSnapshot.available, false);
assert.equal(fallbackSnapshot.freeBytes, 0);
assert.equal(fallbackSnapshot.availableBytes, 0);
// 2. BUG-V2-AUDIT-07: app/vcon/clients/page.tsx defines isClientExpired and excludes expired tokens from onlineCount, pendingCount, and online/pending filter tabs
const clientsPageSrc = await fs.readFile(
path.join(process.cwd(), "app/vcon/clients/page.tsx"),
"utf8",
);
assert.match(clientsPageSrc, /function isClientExpired\(/);
assert.match(
clientsPageSrc,
/const expired = !revoked && isClientExpired\(client\.expiresAt,\s*now\)/,
);
assert.match(
clientsPageSrc,
/!isClientExpired\(client\.expiresAt,\s*now\)/,
);
assert.match(
clientsPageSrc,
/const pending =\s*client\.status === 'active' && !expired && !client\.lastActive/,
);
// 3. BUG-V2-AUDIT-10: ExportWorkspace.tsx and app/vcon/storage/page.tsx render CSV.GZ badge for compressed .csv.gz partitions
const exportWorkspaceSrc = await fs.readFile(
path.join(process.cwd(), "components/sections/ExportWorkspace.tsx"),
"utf8",
);
assert.match(exportWorkspaceSrc, /CSV\.GZ/);
assert.match(exportWorkspaceSrc, /endsWith\(['"]\.csv\.gz['"]\)/);
const storagePageSrc = await fs.readFile(
path.join(process.cwd(), "app/vcon/storage/page.tsx"),
"utf8",
);
assert.match(storagePageSrc, /CSV\.GZ/);
assert.match(storagePageSrc, /endsWith\(['"]\.csv\.gz['"]\)/);
} finally {
CSVStorageManager.setStatfsOverrideForTests(null);
CSVStorageManager.clearQuotaCaches();
CSVStorageManager.baseStorageDir = origBaseStorageDir;
if (prevStorageDir === undefined) delete process.env.DATASTACK_STORAGE_DIR;
else process.env.DATASTACK_STORAGE_DIR = prevStorageDir;
await fs.rm(tmpDir, { recursive: true, force: true });
resetAdminStateForTests();
}
});
@@ -12,6 +12,7 @@ import { PATCH as patchClientRoute } from "../app/api/clients/[id]/route";
import { POST as clientsPost } from "../app/api/clients/route";
import { POST as eventsPost } from "../app/api/events/route";
import { GET as exportGet } from "../app/api/export/route";
import { GET as healthGet } from "../app/api/health/route";
import { GET as prometheusGet } from "../app/api/prometheus/route";
import { POST as settingsPost } from "../app/api/settings/route";
import { POST as webhookTestPost } from "../app/api/settings/webhook-test/route";
@@ -110,6 +111,28 @@ test("V2-Phase-02 [IP Allowlist & Token Expiry]: validateAllowedIpsInput, isClie
assert.equal(isClientIpAllowed("::1", "127.0.0.1"), true);
assert.equal(isClientIpAllowed("127.0.0.1", "::1"), true);
// BUG-V2-AUDIT-06: IPv6 CIDR (/0..128) BlockList matching & bracketed/port/zone normalization
assert.equal(
isClientIpAllowed("2001:db8:abcd:0012::1", "2001:db8::/32"),
true,
);
assert.equal(
isClientIpAllowed("[2001:db8:abcd:0012::99]:8443", "2001:db8::/32"),
true,
);
assert.equal(
isClientIpAllowed("2001:db9::1", "2001:db8::/32"),
false,
);
assert.equal(
isClientIpAllowed("fe80::1ff:fe23:4567:890a%eth0", "fe80::/64"),
true,
);
assert.equal(
isClientIpAllowed("203.0.113.10:54321", "203.0.113.0/24"),
true,
);
// Expiration validation & evaluation
assert.deepEqual(validateClientExpiresAtInput(null), {
valid: true,
@@ -689,3 +712,203 @@ test("V2-Phase-02 [Prometheus Metrics & NDJSON Streaming Export]: GET /api/prome
resetAdminStateForTests();
}
});
test("V2-Audit-Phase-02 [BUG-V2-AUDIT-05, BUG-V2-AUDIT-06, BUG-V2-AUDIT-08]: Port 4000 outbound webhook alerts, IPv6 CIDR parity & GET /api/health expiry/IP checks", async () => {
const prevDisableListen = process.env.DATASTACK_DISABLE_LISTEN;
const prevDbUrl = process.env.DATABASE_URL;
process.env.DATASTACK_DISABLE_LISTEN = "1";
delete process.env.DATABASE_URL;
const serverIndex = require("../server/index.js");
const { Server: SocketIOServer } = require("socket.io");
const { setupSocketHandlers } = require("../server/socket/handlers.js");
// 1. BUG-V2-AUDIT-06: Port 4000 isClientIpAllowed supports IPv6 CIDR (/0..128) and semicolon delimiters
assert.equal(
serverIndex.isClientIpAllowed("2001:db8:1234::5", "10.0.0.0/8; 2001:db8::/32"),
true,
);
assert.equal(
serverIndex.isClientIpAllowed("[2001:db8:9999::1]:9443", "2001:db8::/32"),
true,
);
assert.equal(
serverIndex.isClientIpAllowed("2001:db9::1", "2001:db8::/32"),
false,
);
assert.equal(
serverIndex.isClientIpAllowed("fe80::1234%eth0", "fe80::/64"),
true,
);
// 2. BUG-V2-AUDIT-05: Port 4000 Socket.IO outbound webhook alerts with HMAC-SHA256 signature
const receivedPort4000Webhooks: Array<{
headers: http.IncomingHttpHeaders;
body: string;
json: Record<string, unknown>;
}> = [];
const mockWebhookServer = http.createServer((req, res) => {
const chunks: Buffer[] = [];
req.on("data", (chunk) => chunks.push(Buffer.from(chunk)));
req.on("end", () => {
const bodyStr = Buffer.concat(chunks).toString("utf8");
let parsed: Record<string, unknown> = {};
try {
parsed = JSON.parse(bodyStr);
} catch {
parsed = {};
}
receivedPort4000Webhooks.push({
headers: req.headers,
body: bodyStr,
json: parsed,
});
res.writeHead(200, { "Content-Type": "application/json" });
res.end(JSON.stringify({ ok: true }));
});
});
await new Promise<void>((resolve) =>
mockWebhookServer.listen(0, "127.0.0.1", () => resolve()),
);
const webhookPort = (mockWebhookServer.address() as AddressInfo).port;
const webhookUrl = `http://127.0.0.1:${webhookPort}/socket-hook`;
const testHttpServer = http.createServer();
const testIo = new SocketIOServer(testHttpServer);
const forbiddenClient = await SDKManager.registerClientAsync({
clientId: "audit05_forbidden_sock",
name: "Audit05ForbiddenSocketClient",
rateLimit: 100,
storageQuotaMb: 0,
allowedIps: "203.0.113.99",
expiresAt: null,
});
setupSocketHandlers(testIo, [], serverIndex.writer, {
getRuntimeConfig: () => serverIndex.runtimeConfig,
verifyClientInDb: async (cid: string, _tok: string, _bypass: boolean, ip: string) => {
if (cid === forbiddenClient.clientId && !serverIndex.isClientIpAllowed(ip, "203.0.113.99")) {
return {
allowed: false,
code: "CLIENT_IP_FORBIDDEN",
reason: `CLIENT_IP_FORBIDDEN: IP ${ip} is not allowed for ${cid}`,
};
}
return { allowed: true, rateLimit: 100 };
},
triggerWebhookAlert: serverIndex.triggerWebhookAlert,
});
await new Promise<void>((resolve) =>
testHttpServer.listen(0, "127.0.0.1", () => resolve()),
);
const socketPort = (testHttpServer.address() as AddressInfo).port;
try {
serverIndex.clearWebhookCooldownCacheForTests();
serverIndex.applyRuntimeSettingsObject({
enableWebhookAlerts: true,
webhookUrl,
webhookSecret: "port4000_shared_hmac_secret",
webhookCooldownSec: 60,
});
// Attempt Socket.IO connection from 127.0.0.1 (not in 203.0.113.99 allowlist)
const sock: Socket = ioClient(`http://127.0.0.1:${socketPort}`, {
auth: { token: forbiddenClient.token },
transports: ["websocket"],
reconnection: false,
timeout: 2000,
});
const connectErr = await new Promise<Error>((resolve, reject) => {
sock.on("connect", () => {
sock.disconnect();
reject(new Error("Expected Socket.IO handshake rejection for forbidden IP"));
});
sock.on("connect_error", (err) => {
sock.disconnect();
resolve(err);
});
});
assert.ok(connectErr.message.includes("CLIENT_IP_FORBIDDEN"));
// Wait briefly for asynchronous webhook alert dispatch
await new Promise((r) => setTimeout(r, 80));
assert.equal(receivedPort4000Webhooks.length, 1);
assert.equal(
receivedPort4000Webhooks[0].headers["x-datastack-event"],
"CLIENT_IP_FORBIDDEN",
);
const expectedSig = buildWebhookSignature(
receivedPort4000Webhooks[0].body,
"port4000_shared_hmac_secret",
);
assert.equal(
receivedPort4000Webhooks[0].headers["x-datastack-signature"],
expectedSig,
);
assert.equal(
receivedPort4000Webhooks[0].json.clientId,
forbiddenClient.clientId,
);
// 3. BUG-V2-AUDIT-08: GET /api/health enforces CLIENT_TOKEN_EXPIRED (401) and CLIENT_IP_FORBIDDEN (403)
const expiredHealthClient = await SDKManager.registerClientAsync({
clientId: "audit08_expired_health",
name: "Audit08ExpiredHealthClient",
rateLimit: 100,
storageQuotaMb: 0,
allowedIps: "",
expiresAt: "2020-01-01T00:00:00.000Z",
});
const expiredHealthRes = await healthGet(
new NextRequest("http://localhost:3000/api/health", {
headers: {
authorization: `Bearer ${expiredHealthClient.token}`,
},
}),
);
assert.equal(expiredHealthRes.status, 401);
const expiredHealthBody = await expiredHealthRes.json();
assert.equal(expiredHealthBody.code, "CLIENT_TOKEN_EXPIRED");
const forbiddenHealthRes = await healthGet(
new NextRequest("http://localhost:3000/api/health", {
headers: {
authorization: `Bearer ${forbiddenClient.token}`,
"x-real-ip": "198.51.100.44",
},
}),
);
assert.equal(forbiddenHealthRes.status, 403);
const forbiddenHealthBody = await forbiddenHealthRes.json();
assert.equal(forbiddenHealthBody.code, "CLIENT_IP_FORBIDDEN");
} finally {
serverIndex.applyRuntimeSettingsObject({
enableWebhookAlerts: false,
webhookUrl: "",
webhookSecret: "",
webhookCooldownSec: 300,
});
serverIndex.stopTimers();
SDKManager.stopActivityFlushTimerForTests();
await DataBufferEngine.shutdown();
await new Promise<void>((resolve) => {
testIo.close(() => resolve());
});
await new Promise<void>((resolve) =>
mockWebhookServer.close(() => resolve()),
);
resetAdminStateForTests();
if (prevDisableListen === undefined) delete process.env.DATASTACK_DISABLE_LISTEN;
else process.env.DATASTACK_DISABLE_LISTEN = prevDisableListen;
if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
else process.env.DATABASE_URL = prevDbUrl;
}
});
+239
View File
@@ -3,7 +3,9 @@ import { after, beforeEach, describe, it } from "node:test";
import { NextRequest } from "next/server";
import { GET as auditGet } from "@/app/api/audit/route";
import { POST as loginPost } from "@/app/api/auth/login/route";
import { POST as logoutPost } from "@/app/api/auth/logout/route";
import { GET as profileGet, PUT as profilePut } from "@/app/api/auth/profile/route";
import { POST as setupPost } from "@/app/api/auth/setup/route";
import {
DELETE as clientDelete,
GET as clientGet,
@@ -24,7 +26,10 @@ import {
} from "@/core/auth/adminSession";
import {
createInitialAdmin,
getAdminSetupKey,
resetAdminStateForTests,
revokeAdminSessions,
verifyRequestAdminSession,
} from "@/core/auth/adminUsers";
import { DataBufferEngine } from "@/core/data/buffer";
@@ -618,4 +623,238 @@ describe("V2-Phase-03: Multi-Admin RBAC & Immutable Admin Audit Log", () => {
assert.equal(freshJson.profile.role, "Operator");
assert.equal(freshJson.profile.roleKey, "operator");
});
it("5. V2-Audit-Phase-01: Prevents cross-table/in-memory duplicate admin emails (BUG-V2-AUDIT-03), isolates multi-admin session logout from global race (BUG-V2-AUDIT-04), and records ADMIN_SETUP, ADMIN_LOGOUT, CLIENT_REVOKED, and STORAGE_FLUSH_TRIGGERED audit actions (BUG-V2-AUDIT-02)", async () => {
const setupKey = getAdminSetupKey()!;
const setupRes = await setupPost(
new NextRequest("http://localhost:3000/api/auth/setup", {
method: "POST",
headers: {
"content-type": "application/json",
"x-real-ip": "198.51.100.10",
},
body: JSON.stringify({
email: "founder@datastack.dev",
password: "FounderPassword123!",
setupKey,
}),
}),
);
assert.equal(setupRes.status, 200);
const ownerCookie = setupRes.cookies.get("datastack_admin_session")?.value!;
assert.ok(ownerCookie);
// Owner creates two team members: Alice (admin) and Bob (operator)
const aliceRes = await teamPost(
new NextRequest("http://localhost:3000/api/team", {
method: "POST",
headers: {
"content-type": "application/json",
cookie: `datastack_admin_session=${ownerCookie}`,
},
body: JSON.stringify({
email: "alice@datastack.dev",
password: "AlicePassword123!",
role: "admin",
}),
}),
);
assert.equal(aliceRes.status, 201);
const aliceMember = (await aliceRes.json()).member;
const bobRes = await teamPost(
new NextRequest("http://localhost:3000/api/team", {
method: "POST",
headers: {
"content-type": "application/json",
cookie: `datastack_admin_session=${ownerCookie}`,
},
body: JSON.stringify({
email: "bob@datastack.dev",
password: "BobPassword123!",
role: "operator",
}),
}),
);
assert.equal(bobRes.status, 201);
// Login as Alice and Bob
const aliceLogin = await loginPost(
new NextRequest("http://localhost:3000/api/auth/login", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
email: "alice@datastack.dev",
password: "AlicePassword123!",
}),
}),
);
assert.equal(aliceLogin.status, 200);
const aliceCookie = aliceLogin.cookies.get("datastack_admin_session")?.value!;
const bobLogin = await loginPost(
new NextRequest("http://localhost:3000/api/auth/login", {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
email: "bob@datastack.dev",
password: "BobPassword123!",
}),
}),
);
assert.equal(bobLogin.status, 200);
const bobCookie = bobLogin.cookies.get("datastack_admin_session")?.value!;
// BUG-V2-AUDIT-03: Alice cannot change her email to Primary Owner's email OR Bob's email -> 409 ADMIN_EMAIL_CONFLICT
const aliceCollideOwner = await profilePut(
new NextRequest("http://localhost:3000/api/auth/profile", {
method: "PUT",
headers: {
"content-type": "application/json",
cookie: `datastack_admin_session=${aliceCookie}`,
},
body: JSON.stringify({
email: "FOUNDER@datastack.dev",
currentPassword: "AlicePassword123!",
}),
}),
);
assert.equal(aliceCollideOwner.status, 409);
assert.equal((await aliceCollideOwner.json()).code, "ADMIN_EMAIL_CONFLICT");
const aliceCollideBob = await profilePut(
new NextRequest("http://localhost:3000/api/auth/profile", {
method: "PUT",
headers: {
"content-type": "application/json",
cookie: `datastack_admin_session=${aliceCookie}`,
},
body: JSON.stringify({
email: "bob@datastack.dev",
currentPassword: "AlicePassword123!",
}),
}),
);
assert.equal(aliceCollideBob.status, 409);
assert.equal((await aliceCollideBob.json()).code, "ADMIN_EMAIL_CONFLICT");
// Primary Owner cannot change email to Alice's email -> 409 ADMIN_EMAIL_CONFLICT
const ownerCollideAlice = await profilePut(
new NextRequest("http://localhost:3000/api/auth/profile", {
method: "PUT",
headers: {
"content-type": "application/json",
cookie: `datastack_admin_session=${ownerCookie}`,
},
body: JSON.stringify({
email: "alice@datastack.dev",
currentPassword: "FounderPassword123!",
}),
}),
);
assert.equal(ownerCollideAlice.status, 409);
assert.equal((await ownerCollideAlice.json()).code, "ADMIN_EMAIL_CONFLICT");
// BUG-V2-AUDIT-04: Simulate concurrent verification where Bob's session is verified right before Alice's session revocation
const aliceVerified = await verifyRequestAdminSession(aliceCookie);
assert.equal(aliceVerified.valid, true);
const bobVerified = await verifyRequestAdminSession(bobCookie);
assert.equal(bobVerified.valid, true);
// Explicitly revoke Alice's session even though Bob was the last verified globally
await revokeAdminSessions({
memberId: aliceMember.id,
email: "alice@datastack.dev",
});
// Alice's cookie must now be revoked, while Bob's and Owner's cookies remain valid!
const aliceAfterRevoke = await verifyRequestAdminSession(aliceCookie);
assert.equal(aliceAfterRevoke.valid, false);
const bobStillValid = await verifyRequestAdminSession(bobCookie);
assert.equal(bobStillValid.valid, true);
const ownerStillValid = await verifyRequestAdminSession(ownerCookie);
assert.equal(ownerStillValid.valid, true);
// Create a client, revoke it (CLIENT_REVOKED), trigger manual storage flush (STORAGE_FLUSH_TRIGGERED), and log out Bob (ADMIN_LOGOUT)
const createClientRes = await clientsPost(
new NextRequest("http://localhost:3000/api/clients", {
method: "POST",
headers: {
"content-type": "application/json",
cookie: `datastack_admin_session=${ownerCookie}`,
},
body: JSON.stringify({
clientId: "phase01_audit_client",
name: "Phase 01 Audit Client",
}),
}),
);
assert.equal(createClientRes.status, 201);
const revokeClientRes = await clientPatch(
new NextRequest("http://localhost:3000/api/clients/phase01_audit_client", {
method: "PATCH",
headers: {
"content-type": "application/json",
cookie: `datastack_admin_session=${ownerCookie}`,
},
body: JSON.stringify({ status: "revoked" }),
}),
{ params: Promise.resolve({ id: "phase01_audit_client" }) },
);
assert.equal(revokeClientRes.status, 200);
const flushRes = await storagePost(
new NextRequest("http://localhost:3000/api/storage", {
method: "POST",
headers: { cookie: `datastack_admin_session=${ownerCookie}` },
}),
);
// Even if Port 4000 is mocked to 127.0.0.1:1, test without explicit INGESTION_INTERNAL_URL so local flush succeeds
void flushRes;
delete process.env.INGESTION_INTERNAL_URL;
const flushLocalRes = await storagePost(
new NextRequest("http://localhost:3000/api/storage", {
method: "POST",
headers: { cookie: `datastack_admin_session=${ownerCookie}` },
}),
);
assert.equal(flushLocalRes.status, 200);
const bobLogoutRes = await logoutPost(
new NextRequest("http://localhost:3000/api/auth/logout", {
method: "POST",
headers: {
cookie: `datastack_admin_session=${bobCookie}`,
"x-real-ip": "198.51.100.25",
},
}),
);
assert.equal(bobLogoutRes.status, 200);
// Bob's session is now invalidated on server, while Owner's session is still valid
assert.equal((await verifyRequestAdminSession(bobCookie)).valid, false);
assert.equal((await verifyRequestAdminSession(ownerCookie)).valid, true);
// BUG-V2-AUDIT-02: Verify all 4 newly wired audit actions appear in GET /api/audit
const auditRes = await auditGet(
new NextRequest("http://localhost:3000/api/audit?limit=100", {
headers: { cookie: `datastack_admin_session=${ownerCookie}` },
}),
);
assert.equal(auditRes.status, 200);
const logs = (await auditRes.json()).logs as Array<{
action: string;
actorEmail: string;
target: string;
}>;
const recordedActions = new Set(logs.map((l) => l.action));
assert.ok(recordedActions.has("ADMIN_SETUP"), "Expected ADMIN_SETUP audit log");
assert.ok(recordedActions.has("ADMIN_LOGOUT"), "Expected ADMIN_LOGOUT audit log");
assert.ok(recordedActions.has("CLIENT_REVOKED"), "Expected CLIENT_REVOKED audit log");
assert.ok(
recordedActions.has("STORAGE_FLUSH_TRIGGERED"),
"Expected STORAGE_FLUSH_TRIGGERED audit log",
);
});
});