refactor(auth): standardize session management and security

- Centralize secret retrieval using `getAdminSessionSecret`
- Replace manual session parsing with `verifyRequestAdminSession` utility
- Tighten API validation and improve security headers handling
- Enforce environment variable security for production deployments
This commit is contained in:
Team Dark Devil committed 2026-11-18 05:18:55 +06:00
1 parent 34519c07df
commit 177ed4a844
78 files changed
+6748 -3157

No files matched your search

+340
View File
@@ -0,0 +1,340 @@
import assert from "node:assert/strict";
import { after, afterEach, beforeEach, describe, it } from "node:test";
import { NextRequest } from "next/server";
import { GET as profileGet, PUT as profilePut } from "@/app/api/auth/profile/route";
import { GET as clientsGet, POST as clientsPost } from "@/app/api/clients/route";
import {
DELETE as clientDelete,
GET as clientGet,
} from "@/app/api/clients/[id]/route";
import { POST as eventsPost } from "@/app/api/events/route";
import {
createAdminSessionToken,
getAdminSessionPayload,
getAdminSessionSecret,
} from "@/core/auth/adminSession";
import {
consumeAdminLoginAttempt,
createInitialAdmin,
getAdminSetupKey,
isAdminSetupComplete,
resetAdminStateForTests,
verifyAdminCredentials,
verifyAdminSetupKey,
verifyRequestAdminSession,
} from "@/core/auth/adminUsers";
import { AuthService, resolveDefaultJwtSecret } from "@/core/auth/jwt";
import { isSameOriginRequest } from "@/middleware";
const ORIGINAL_ENV = { ...process.env };
function restoreEnv() {
for (const key of Object.keys(process.env)) {
if (!(key in ORIGINAL_ENV)) {
delete process.env[key];
}
}
Object.assign(process.env, ORIGINAL_ENV);
AuthService.setSecret(resolveDefaultJwtSecret() || "");
}
describe("Phase-01 Auth, Secrets, Revocation & Proxy Hardening", () => {
beforeEach(() => {
restoreEnv();
resetAdminStateForTests();
});
afterEach(() => {
restoreEnv();
resetAdminStateForTests();
});
after(() => {
const bufState = (
globalThis as {
__dataStackBufferState?: { flushTimer?: ReturnType<typeof setInterval> | null };
}
).__dataStackBufferState;
if (bufState?.flushTimer) {
clearInterval(bufState.flushTimer);
bufState.flushTimer = null;
}
});
it("BUG-01 & BUG-02: fails closed in production when secrets or DB/setup are missing", async () => {
process.env.NODE_ENV = "production";
delete process.env.DATABASE_URL;
delete process.env.DATASTACK_ADMIN_SESSION_SECRET;
delete process.env.DATASTACK_ADMIN_SETUP_KEY;
delete process.env.DATASTACK_JWT_SECRET;
delete process.env.DATASTACK_ADMIN_PASSWORD;
assert.equal(getAdminSessionSecret(), null);
assert.equal(getAdminSetupKey(), null);
assert.equal(resolveDefaultJwtSecret(), "");
assert.equal(
verifyAdminSetupKey("datastack-default-setup-key-32-chars"),
false,
);
assert.equal(await isAdminSetupComplete(), false);
await assert.rejects(
() => verifyAdminCredentials("admin@datastack.dev", "admin12345678"),
/setup/i,
);
AuthService.setSecret(resolveDefaultJwtSecret());
assert.equal(AuthService.isConfigured(), false);
assert.throws(() => AuthService.generateClientToken({ clientId: "c1" }));
});
it("BUG-03: bumps tokenVersion on credential update and rejects old session cookies", async () => {
delete process.env.DATABASE_URL;
process.env.DATASTACK_ALLOW_DEMO_ADMIN = "false";
const created = await createInitialAdmin("owner@datastack.dev", "InitialPass123!");
assert.equal(created, true);
assert.equal(await isAdminSetupComplete(), true);
const secret = getAdminSessionSecret()!;
const v1Token = await createAdminSessionToken(
"owner@datastack.dev",
secret,
Date.now() + 3600_000,
1,
);
const checkV1 = await verifyRequestAdminSession(v1Token);
assert.equal(checkV1.valid, true);
const updateReq = new NextRequest("http://localhost:3000/api/auth/profile", {
method: "PUT",
headers: {
"content-type": "application/json",
cookie: `datastack_admin_session=${v1Token}`,
},
body: JSON.stringify({
email: "owner2@datastack.dev",
currentPassword: "InitialPass123!",
newPassword: "UpdatedPass456!",
}),
});
const updateRes = await profilePut(updateReq);
assert.equal(updateRes.status, 200);
const setCookie = updateRes.cookies.get("datastack_admin_session")?.value;
assert.ok(setCookie);
const newPayload = await getAdminSessionPayload(setCookie!, secret);
assert.ok(newPayload);
assert.equal(newPayload!.tokenVersion, 2);
// Old v1 session must now return 401
const staleProfileReq = new NextRequest("http://localhost:3000/api/auth/profile", {
method: "GET",
headers: { cookie: `datastack_admin_session=${v1Token}` },
});
const staleRes = await profileGet(staleProfileReq);
assert.equal(staleRes.status, 401);
// New v2 session succeeds
const freshProfileReq = new NextRequest("http://localhost:3000/api/auth/profile", {
method: "GET",
headers: { cookie: `datastack_admin_session=${setCookie}` },
});
const freshRes = await profileGet(freshProfileReq);
assert.equal(freshRes.status, 200);
});
it("BUG-04: throttles by composite IP+email without locking out other admin IPs", async () => {
const secret = "test-throttle-secret-at-least-32-chars-long";
const targetEmail = "admin@datastack.dev";
for (let i = 0; i < 5; i++) {
const dec = await consumeAdminLoginAttempt(targetEmail, secret, "198.51.100.10");
assert.equal(dec.allowed, true);
}
// 6th attempt from attacker IP is blocked
const blocked = await consumeAdminLoginAttempt(targetEmail, secret, "198.51.100.10");
assert.equal(blocked.allowed, false);
assert.ok(blocked.retryAfterSeconds >= 1);
// Real admin from different IP is still allowed
const otherIpAllowed = await consumeAdminLoginAttempt(
targetEmail,
secret,
"203.0.113.55",
);
assert.equal(otherIpAllowed.allowed, true);
// Same IP spraying 25 distinct emails triggers IP bucket block on 26th
for (let i = 0; i < 25; i++) {
const spray = await consumeAdminLoginAttempt(
`spray${i}@example.com`,
secret,
"198.51.100.99",
);
assert.equal(spray.allowed, true);
}
const sprayBlocked = await consumeAdminLoginAttempt(
"spray25@example.com",
secret,
"198.51.100.99",
);
assert.equal(sprayBlocked.allowed, false);
});
it("BUG-05 & BUG-15: redacts tokens in client list, returns 400 on malformed JSON, and blocks deleted/unregistered Bearer tokens", async () => {
delete process.env.DATABASE_URL;
await createInitialAdmin("admin@datastack.dev", "AdminPass123!");
const secret = getAdminSessionSecret()!;
const adminCookie = await createAdminSessionToken(
"admin@datastack.dev",
secret,
Date.now() + 3600_000,
1,
);
const createReq = new NextRequest("http://localhost:3000/api/clients", {
method: "POST",
headers: {
"content-type": "application/json",
cookie: `datastack_admin_session=${adminCookie}`,
},
body: JSON.stringify({
clientId: "phase01_probe_client",
name: "Phase01 Probe",
rateLimit: 50,
}),
});
const createRes = await clientsPost(createReq);
assert.equal(createRes.status, 201);
const createdJson = await createRes.json();
const clientToken = createdJson.client.token as string;
assert.ok(clientToken.length > 20);
// List endpoint must redact full token & secret
const listReq = new NextRequest("http://localhost:3000/api/clients", {
method: "GET",
headers: { cookie: `datastack_admin_session=${adminCookie}` },
});
const listRes = await clientsGet(listReq);
assert.equal(listRes.status, 200);
const listJson = await listRes.json();
const listed = listJson.clients.find(
(c: { clientId: string }) => c.clientId === "phase01_probe_client",
);
assert.ok(listed);
assert.equal("token" in listed, false);
assert.equal("secret" in listed, false);
assert.equal(listed.tokenPreview, `${clientToken.slice(0, 12)}...`);
// Single client GET omits token by default, reveals on ?includeToken=true
const getDefaultRes = await clientGet(
new NextRequest("http://localhost:3000/api/clients/phase01_probe_client", {
headers: { cookie: `datastack_admin_session=${adminCookie}` },
}),
{ params: Promise.resolve({ id: "phase01_probe_client" }) },
);
const defaultClient = (await getDefaultRes.json()).client;
assert.equal("token" in defaultClient, false);
const getIncludeRes = await clientGet(
new NextRequest(
"http://localhost:3000/api/clients/phase01_probe_client?includeToken=true",
{ headers: { cookie: `datastack_admin_session=${adminCookie}` } },
),
{ params: Promise.resolve({ id: "phase01_probe_client" }) },
);
const includeClient = (await getIncludeRes.json()).client;
assert.equal(includeClient.token, clientToken);
// Malformed JSON on POST /api/events returns 400 (not 500)
const malformedRes = await eventsPost(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
authorization: `Bearer ${clientToken}`,
"content-type": "application/json",
},
body: "{bad-json",
}),
);
assert.equal(malformedRes.status, 400);
// Delete client -> subsequent Bearer ingestion fails closed with 403 CLIENT_REVOKED
const delRes = await clientDelete(
new NextRequest("http://localhost:3000/api/clients/phase01_probe_client", {
method: "DELETE",
headers: { cookie: `datastack_admin_session=${adminCookie}` },
}),
{ params: Promise.resolve({ id: "phase01_probe_client" }) },
);
assert.equal(delRes.status, 200);
const rejectedRes = await eventsPost(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
authorization: `Bearer ${clientToken}`,
"content-type": "application/json",
},
body: JSON.stringify({ eventId: "evt_after_delete", data: { ok: 1 } }),
}),
);
assert.equal(rejectedRes.status, 403);
// Unregistered signed Bearer token also rejected with 403 CLIENT_REVOKED
const ghostToken = AuthService.generateClientToken({
clientId: "unregistered_ghost_client",
});
const ghostRes = await eventsPost(
new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: {
authorization: `Bearer ${ghostToken}`,
"content-type": "application/json",
},
body: JSON.stringify({ eventId: "evt_ghost", data: { ok: 1 } }),
}),
);
assert.equal(ghostRes.status, 403);
});
it("BUG-26: validates Origin or Referer against Host / X-Forwarded-Host", () => {
const reqOriginOk = new NextRequest("http://localhost:3000/api/clients", {
method: "POST",
headers: {
origin: "https://console.example.com",
"x-forwarded-host": "console.example.com",
},
});
assert.equal(isSameOriginRequest(reqOriginOk), true);
const reqRefererOk = new NextRequest("http://localhost:3000/api/clients", {
method: "POST",
headers: {
referer: "https://console.example.com/vcon/clients",
host: "console.example.com",
},
});
assert.equal(isSameOriginRequest(reqRefererOk), true);
const reqCrossOrigin = new NextRequest("http://localhost:3000/api/clients", {
method: "POST",
headers: {
origin: "https://evil.example.com",
host: "console.example.com",
},
});
assert.equal(isSameOriginRequest(reqCrossOrigin), false);
const reqMissingBoth = new NextRequest("http://localhost:3000/api/clients", {
method: "POST",
headers: {
host: "console.example.com",
},
});
assert.equal(isSameOriginRequest(reqMissingBoth), false);
});
});
+375
View File
@@ -0,0 +1,375 @@
import assert from "node:assert/strict";
import { createRequire } from "node:module";
import type { AddressInfo } from "node:net";
import test, { after, before } from "node:test";
import { NextRequest } from "next/server";
import { io as ioClient, type Socket } from "socket.io-client";
import { GET as getSettingsRoute, POST as postSettingsRoute } from "../app/api/settings/route";
import { SettingsManager } from "../core/adminSettings/settingsManager";
import {
createAdminSessionToken,
getAdminSessionSecret,
} from "../core/auth/adminSession";
import {
createInitialAdmin,
resetAdminStateForTests,
} from "../core/auth/adminUsers";
import { ApiEngine } from "../core/apiEngine/apiService";
import { SDKManager } from "../core/sdk/sdkManager";
import {
SecurityEngine,
getInternalIngestionHeaders,
} from "../core/Security/security";
const require = createRequire(import.meta.url);
interface IngestionServerModule {
server: import("node:http").Server;
io: import("socket.io").Server;
bufferQueue: Array<{ clientId: string; eventId: string; timestamp: string; payload: string }>;
runtimeConfig: {
jwtSecret: string;
maxEventsPerSec: number;
maxPayloadSizeBytes: number;
batchFlushThreshold: number;
batchFlushIntervalMs: number;
fileRotationMb: number;
enableIdempotency: boolean;
autoPruneDays: number;
};
stopTimers: () => void;
}
const g = globalThis as typeof globalThis & {
__dataStackBufferState?: {
flushTimer: ReturnType<typeof setInterval> | null;
};
__dataStackAdminSettings?: unknown;
__dataStackClientsMap?: Map<string, unknown>;
__dataStackDeletedClients?: Set<string>;
__dataStackSyncedSocketEvents?: Map<string, number>;
};
let serverMod: IngestionServerModule;
let ingestionUrl = "";
const prevIngestionUrl = process.env.INGESTION_INTERNAL_URL;
const prevDisableListen = process.env.DATASTACK_DISABLE_LISTEN;
const prevDbUrl = process.env.DATABASE_URL;
const prevInternalSecret = process.env.DATASTACK_INTERNAL_SECRET;
const openSockets: Socket[] = [];
function connectSocket(token: string): Promise<Socket> {
return new Promise((resolve, reject) => {
const client = ioClient(ingestionUrl, {
auth: { token },
transports: ["websocket"],
reconnection: false,
timeout: 3000,
});
openSockets.push(client);
const onConnect = () => {
cleanup();
resolve(client);
};
const onError = (err: Error) => {
cleanup();
client.disconnect();
reject(err);
};
function cleanup() {
client.off("connect", onConnect);
client.off("connect_error", onError);
}
client.on("connect", onConnect);
client.on("connect_error", onError);
});
}
function emitWithAck<T = unknown>(
socket: Socket,
event: string,
payload: unknown,
): Promise<T> {
return new Promise((resolve, reject) => {
socket.timeout(3000).emit(event, payload, (err: unknown, res: T) => {
if (err) reject(err);
else resolve(res);
});
});
}
async function createAdminCookieHeader(): Promise<string> {
resetAdminStateForTests();
await createInitialAdmin("admin@datastack.dev", "InitialAdminPass123!");
const secret = getAdminSessionSecret();
assert.ok(secret, "Expected admin session secret in test environment");
const token = await createAdminSessionToken(
"admin@datastack.dev",
secret,
Date.now() + 3600_000,
1,
);
return `datastack_admin_session=${token}`;
}
before(async () => {
process.env.DATASTACK_DISABLE_LISTEN = "1";
delete process.env.DATABASE_URL;
delete process.env.DATASTACK_INTERNAL_SECRET;
delete g.__dataStackAdminSettings;
g.__dataStackClientsMap?.clear();
g.__dataStackDeletedClients?.clear();
g.__dataStackSyncedSocketEvents?.clear();
resetAdminStateForTests();
serverMod = require("../server/index.js") as IngestionServerModule;
await new Promise<void>((resolve) => {
serverMod.server.listen(0, "127.0.0.1", () => resolve());
});
const addr = serverMod.server.address() as AddressInfo;
ingestionUrl = `http://127.0.0.1:${addr.port}`;
process.env.INGESTION_INTERNAL_URL = ingestionUrl;
});
after(async () => {
for (const sock of openSockets) {
try {
sock.disconnect();
} catch {
// ignore
}
}
if (serverMod) {
serverMod.stopTimers();
await new Promise<void>((resolve) => {
serverMod.io.close(() => {
serverMod.server.close(() => resolve());
});
});
}
if (g.__dataStackBufferState?.flushTimer) {
clearInterval(g.__dataStackBufferState.flushTimer);
g.__dataStackBufferState.flushTimer = null;
}
if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL;
else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl;
if (prevDisableListen === undefined) delete process.env.DATASTACK_DISABLE_LISTEN;
else process.env.DATASTACK_DISABLE_LISTEN = prevDisableListen;
if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
else process.env.DATABASE_URL = prevDbUrl;
if (prevInternalSecret === undefined) delete process.env.DATASTACK_INTERNAL_SECRET;
else process.env.DATASTACK_INTERNAL_SECRET = prevInternalSecret;
});
test("BUG-13: Port 4000 /internal/* rejects unauthenticated callers and accepts valid internal secret while /health remains public", async () => {
const healthRes = await fetch(`${ingestionUrl}/health`);
assert.equal(healthRes.status, 200);
for (const [method, path, body] of [
["GET", "/internal/metrics", undefined],
["POST", "/internal/flush", {}],
["GET", "/internal/recent", undefined],
["POST", "/internal/client-sync", { clientId: "x", action: "registered" }],
["POST", "/internal/settings-sync", { maxEventsPerSec: 120 }],
] as const) {
const noAuth = await fetch(`${ingestionUrl}${path}`, {
method,
headers: body ? { "Content-Type": "application/json" } : undefined,
body: body ? JSON.stringify(body) : undefined,
});
assert.equal(noAuth.status, 401, `${method} ${path} must reject missing secret`);
const wrongAuth = await fetch(`${ingestionUrl}${path}`, {
method,
headers: {
...(body ? { "Content-Type": "application/json" } : {}),
"x-datastack-internal-secret": "wrong-secret-value",
},
body: body ? JSON.stringify(body) : undefined,
});
assert.equal(wrongAuth.status, 401, `${method} ${path} must reject wrong secret`);
const validAuth = await fetch(`${ingestionUrl}${path}`, {
method,
headers: {
...(body ? { "Content-Type": "application/json" } : {}),
...getInternalIngestionHeaders(),
},
body: body ? JSON.stringify(body) : undefined,
});
assert.equal(validAuth.status, 200, `${method} ${path} must accept valid internal secret`);
}
});
test("BUG-06: POST /api/settings validates allowlisted bounds, syncs Port 4000 immediately, preserves autoPruneDays=0, and disconnects sockets on JWT rotation", async () => {
const unauthGet = await getSettingsRoute(new NextRequest("http://localhost:3000/api/settings"));
assert.equal(unauthGet.status, 401);
const cookieHeader = await createAdminCookieHeader();
const badJsonReq = new NextRequest("http://localhost:3000/api/settings", {
method: "POST",
headers: { "Content-Type": "application/json", cookie: cookieHeader },
body: "{bad-json",
});
const badJsonRes = await postSettingsRoute(badJsonReq);
assert.equal(badJsonRes.status, 400);
const unsupportedReq = new NextRequest("http://localhost:3000/api/settings", {
method: "POST",
headers: { "Content-Type": "application/json", cookie: cookieHeader },
body: JSON.stringify({ serverPort: 9999 }),
});
assert.equal((await postSettingsRoute(unsupportedReq)).status, 400);
const weakSecretReq = new NextRequest("http://localhost:3000/api/settings", {
method: "POST",
headers: { "Content-Type": "application/json", cookie: cookieHeader },
body: JSON.stringify({ jwtSecret: "too-short" }),
});
assert.equal((await postSettingsRoute(weakSecretReq)).status, 400);
const validUpdateReq = new NextRequest("http://localhost:3000/api/settings", {
method: "POST",
headers: { "Content-Type": "application/json", cookie: cookieHeader },
body: JSON.stringify({
maxEventsPerSec: 42,
autoPruneDays: 0,
enableIdempotency: true,
}),
});
const validUpdateRes = await postSettingsRoute(validUpdateReq);
assert.equal(validUpdateRes.status, 200);
const updatedJson = await validUpdateRes.json();
assert.equal(updatedJson.settings.maxEventsPerSec, 42);
assert.equal(updatedJson.settings.autoPruneDays, 0);
assert.equal(serverMod.runtimeConfig.maxEventsPerSec, 42);
assert.equal(serverMod.runtimeConfig.autoPruneDays, 0);
// Register client & open WebSocket before JWT rotation
const client = await SDKManager.registerClientAsync({
clientId: "phase02_jwt_rot",
name: "Phase02 Rotation",
rateLimit: 50,
});
const oldToken = client.token;
const socket = await connectSocket(oldToken);
assert.equal(socket.connected, true);
const disconnectedPromise = new Promise<string>((resolve) => {
socket.once("disconnect", (reason) => resolve(reason));
});
const newSecret = "phase02-rotated-production-secret-key-minimum-32-chars";
const rotateReq = new NextRequest("http://localhost:3000/api/settings", {
method: "POST",
headers: { "Content-Type": "application/json", cookie: cookieHeader },
body: JSON.stringify({ jwtSecret: newSecret }),
});
const rotateRes = await postSettingsRoute(rotateReq);
assert.equal(rotateRes.status, 200);
assert.equal(serverMod.runtimeConfig.jwtSecret, newSecret);
const disconnectReason = await Promise.race([
disconnectedPromise,
new Promise<string>((_, reject) =>
setTimeout(() => reject(new Error("Expected socket disconnect on JWT rotation")), 2000),
),
]);
assert.ok(disconnectReason.length > 0);
// Old JWT token must fail handshake after rotation
await assert.rejects(() => connectSocket(oldToken), /AUTH_INVALID|jwt|signature/i);
// New resigned client token must succeed cleanly
const refreshedClient = SDKManager.getClient("phase02_jwt_rot");
assert.ok(refreshedClient && refreshedClient.token !== oldToken);
const reconnected = await connectSocket(refreshedClient.token);
assert.equal(reconnected.connected, true);
reconnected.disconnect();
});
test("BUG-07: In-memory client registry, socket event counting, token rotation & revocation enforce cleanly on Port 4000", async () => {
const client = await SDKManager.registerClientAsync({
clientId: "phase02_mem_client",
name: "Phase02 Memory Sync",
rateLimit: 25,
});
const socket = await connectSocket(client.token);
const ack = await emitWithAck<{ success: boolean; eventId: string }>(
socket,
"sync_data",
{ eventId: "evt-mem-1", data: { ok: true } },
);
assert.equal(ack.success, true);
const listFirst = await SDKManager.listClientsAsync();
const syncedFirst = listFirst.find((c) => c.clientId === "phase02_mem_client");
assert.ok(syncedFirst);
assert.equal(syncedFirst.totalEvents, 1);
assert.ok(syncedFirst.lastActive);
// Polling again must NOT double-count socket activity
const listSecond = await SDKManager.listClientsAsync();
const syncedSecond = listSecond.find((c) => c.clientId === "phase02_mem_client");
assert.equal(syncedSecond?.totalEvents, 1);
// Advance across JWT iat 1-second boundary before regenerating client token
await new Promise((r) => setTimeout(r, 1100));
// Regenerating client token must kick active socket and reject old token
const rotatedClient = await SDKManager.regenerateTokenAsync("phase02_mem_client");
assert.ok(rotatedClient && rotatedClient.token !== client.token);
await new Promise((r) => setTimeout(r, 50));
assert.equal(socket.connected, false);
await assert.rejects(() => connectSocket(client.token), /TOKEN_ROTATED/);
// New token works; revoking kicks and rejects
const socket2 = await connectSocket(rotatedClient.token);
assert.equal(socket2.connected, true);
await SDKManager.updateClientAsync("phase02_mem_client", { status: "revoked" });
await new Promise((r) => setTimeout(r, 50));
assert.equal(socket2.connected, false);
await assert.rejects(() => connectSocket(rotatedClient.token), /CLIENT_REVOKED/);
});
test("BUG-08: Rate-limit error reports effectiveRateLimit and ApiEngine aggregates Port 3000 + Port 4000 security metrics", async () => {
await SettingsManager.updateSettingsAsync({ maxEventsPerSec: 3, enableIdempotency: true });
const rlClientId = "phase02_rl_msg";
let lastError = "";
for (let i = 0; i < 5; i += 1) {
const res = SecurityEngine.validateEvent(rlClientId, `rl-evt-${i}`, { i }, 100);
if (!res.allowed && res.code === "RATE_LIMIT_EXCEEDED") {
lastError = res.reason || "";
break;
}
}
assert.match(lastError, /Rate limit of 3 events\/sec exceeded/);
const client = await SDKManager.registerClientAsync({
clientId: "phase02_stats_client",
name: "Phase02 Stats Client",
rateLimit: 50,
});
await new Promise((r) => setTimeout(r, 80));
const socket = await connectSocket(client.token);
await emitWithAck(socket, "sync_data", { eventId: "evt-stats-ws-1", data: { a: 1 } });
socket.disconnect();
const metrics = await ApiEngine.getDashboardMetrics();
assert.ok(
metrics.securityStats.activeIdempotencyKeys >= 2,
`Expected aggregated idempotency keys >= 2 (HTTP + WS), got ${metrics.securityStats.activeIdempotencyKeys}`,
);
assert.ok(
metrics.securityStats.rateLimitedClientsCount >= 2,
`Expected aggregated rateLimitedClientsCount >= 2, got ${metrics.securityStats.rateLimitedClientsCount}`,
);
});
+511
View File
@@ -0,0 +1,511 @@
import assert from "node:assert/strict";
import fs from "node:fs";
import { createServer, type Server } from "node:http";
import type { AddressInfo } from "node:net";
import { createRequire } from "node:module";
import os from "node:os";
import path from "node:path";
import test, { after, before, beforeEach } from "node:test";
import { NextRequest } from "next/server";
import { GET as getDataRoute } from "../app/api/data/route";
import { GET as getEventsRoute } from "../app/api/events/route";
import { GET as getExportRoute } from "../app/api/export/route";
import { POST as postStorageRoute } from "../app/api/storage/route";
import { ApiEngine } from "../core/apiEngine/apiService";
import {
DataBufferEngine,
REDIS_BUFFER_KEY,
REDIS_FLUSH_LOCK_KEY,
type RedisBufferClient,
} from "../core/data/buffer";
import { SDKManager } from "../core/sdk/sdkManager";
import { CSVStorageManager, type DataEvent } from "../core/Storage/csvStorage";
const require = createRequire(import.meta.url);
const { CsvWriterWorker } = require("../server/workers/csvWriter") as {
CsvWriterWorker: new (baseDir?: string, maxFileSize?: number) => {
baseDir: string;
writeBatch: (events: DataEvent[]) => Promise<{ written: number; files: string[] }>;
pruneExpiredPartitions: (days: number) => number;
};
};
interface IngestionServerModule {
server: Server;
io: import("socket.io").Server;
writer: { baseDir: string };
bufferQueue: DataEvent[];
runtimeConfig: {
batchFlushThreshold: number;
autoPruneDays: number;
};
applyRuntimeSettingsObject: (obj: Record<string, unknown>) => { jwtRotated: boolean };
flushBuffer: () => Promise<{ written: number; files: string[] }>;
setRedisClientForTests: (client: RedisBufferClient | null | undefined) => void;
stopTimers: () => void;
}
class FakeRedisClient implements RedisBufferClient {
public status = "ready";
public lists = new Map<string, string[]>();
public strings = new Map<string, { value: string; expiresAt: number }>();
public async rpush(key: string, ...values: string[]): Promise<number> {
const list = this.lists.get(key) || [];
list.push(...values);
this.lists.set(key, list);
return list.length;
}
public async lpush(key: string, ...values: string[]): Promise<number> {
const list = this.lists.get(key) || [];
for (const v of values) {
list.unshift(v);
}
this.lists.set(key, list);
return list.length;
}
public async ltrim(key: string, start: number, stop: number): Promise<"OK"> {
const list = this.lists.get(key) || [];
const len = list.length;
const normStart = start < 0 ? Math.max(0, len + start) : start;
const normStop = stop < 0 ? len + stop : stop;
if (normStart > normStop || normStart >= len) {
this.lists.set(key, []);
} else {
this.lists.set(key, list.slice(normStart, normStop + 1));
}
return "OK";
}
public async lrange(key: string, start: number, stop: number): Promise<string[]> {
const list = this.lists.get(key) || [];
const len = list.length;
if (len === 0) return [];
const normStart = start < 0 ? Math.max(0, len + start) : start;
const normStop = stop < 0 ? len + stop : stop;
if (normStart > normStop || normStart >= len) return [];
return list.slice(normStart, normStop + 1);
}
public async llen(key: string): Promise<number> {
return (this.lists.get(key) || []).length;
}
public async set(
key: string,
value: string,
_pxMode: "PX",
ttlMs: number,
_nxMode: "NX",
): Promise<string | null> {
const existing = this.strings.get(key);
if (existing && existing.expiresAt > Date.now()) {
return null;
}
this.strings.set(key, { value, expiresAt: Date.now() + ttlMs });
return "OK";
}
public async eval(_script: string, _numKeys: number, key: string, token: string): Promise<number> {
const existing = this.strings.get(key);
if (existing && existing.value === token) {
this.strings.delete(key);
return 1;
}
return 0;
}
}
const g = globalThis as typeof globalThis & {
__dataStackBufferState?: {
buffer: DataEvent[];
recentEvents: DataEvent[];
flushTimer: ReturnType<typeof setInterval> | null;
};
__dataStackAdminSettings?: unknown;
__dataStackClientsMap?: Map<string, unknown>;
__dataStackDeletedClients?: Set<string>;
};
let tmpStorageDir = "";
const origBaseStorageDir = CSVStorageManager.baseStorageDir;
const prevDisableListen = process.env.DATASTACK_DISABLE_LISTEN;
const prevIngestionUrl = process.env.INGESTION_INTERNAL_URL;
const prevDbUrl = process.env.DATABASE_URL;
const prevRedisUrl = process.env.REDIS_URL;
before(async () => {
process.env.DATASTACK_DISABLE_LISTEN = "1";
delete process.env.DATABASE_URL;
delete process.env.REDIS_URL;
delete process.env.INGESTION_INTERNAL_URL;
tmpStorageDir = fs.mkdtempSync(path.join(os.tmpdir(), "datastack-phase03-"));
CSVStorageManager.baseStorageDir = tmpStorageDir;
CSVStorageManager.clearRowCountCache();
});
beforeEach(async () => {
DataBufferEngine.setRedisClientForTests(undefined);
if (g.__dataStackBufferState) {
g.__dataStackBufferState.buffer = [];
g.__dataStackBufferState.recentEvents = [];
}
CSVStorageManager.clearRowCountCache();
fs.rmSync(tmpStorageDir, { recursive: true, force: true });
fs.mkdirSync(tmpStorageDir, { recursive: true });
delete g.__dataStackAdminSettings;
g.__dataStackClientsMap?.clear();
g.__dataStackDeletedClients?.clear();
});
after(async () => {
await DataBufferEngine.shutdown();
CSVStorageManager.baseStorageDir = origBaseStorageDir;
CSVStorageManager.clearRowCountCache();
if (tmpStorageDir) {
fs.rmSync(tmpStorageDir, { recursive: true, force: true });
}
if (prevDisableListen === undefined) delete process.env.DATASTACK_DISABLE_LISTEN;
else process.env.DATASTACK_DISABLE_LISTEN = prevDisableListen;
if (prevIngestionUrl === undefined) delete process.env.INGESTION_INTERNAL_URL;
else process.env.INGESTION_INTERNAL_URL = prevIngestionUrl;
if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
else process.env.DATABASE_URL = prevDbUrl;
if (prevRedisUrl === undefined) delete process.env.REDIS_URL;
else process.env.REDIS_URL = prevRedisUrl;
});
test("BUG-11: Shared Redis buffer preserves events on CSV error, honors distributed flush lock, avoids double-counting, and falls back to RAM cleanly", async () => {
const fakeRedis = new FakeRedisClient();
DataBufferEngine.setRedisClientForTests(fakeRedis);
const sampleEvents: DataEvent[] = [
{
clientId: "phase03_redis",
eventId: "r-evt-1",
timestamp: "2026-11-17T10:00:00.000Z",
payload: JSON.stringify({ seq: 1 }),
},
{
clientId: "phase03_redis",
eventId: "r-evt-2",
timestamp: "2026-11-17T10:00:01.000Z",
payload: JSON.stringify({ seq: 2 }),
},
];
await DataBufferEngine.pushBatchAsync(sampleEvents);
assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 2);
const recentFromRedis = await DataBufferEngine.getRecentEventsAsync(10, "phase03_redis");
assert.equal(recentFromRedis.length, 2);
assert.equal(recentFromRedis[0].eventId, "r-evt-2");
// Lock contention must skip concurrent Redis flush without corrupting queue
await fakeRedis.set(REDIS_FLUSH_LOCK_KEY, "other-worker-token", "PX", 15000, "NX");
const skipped = await DataBufferEngine.flush();
assert.equal(skipped.count, 0);
assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 2);
await fakeRedis.eval("", 1, REDIS_FLUSH_LOCK_KEY, "other-worker-token");
// Crash/error safety: if appendBatch throws before finishing, Redis LTRIM must NOT discard events
const originalAppend = CSVStorageManager.appendBatch;
CSVStorageManager.appendBatch = async () => {
throw new Error("Simulated disk write failure");
};
await assert.rejects(() => DataBufferEngine.flush(), /Simulated disk write failure/);
CSVStorageManager.appendBatch = originalAppend;
assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 2);
// Subsequent healthy flush drains Redis list, releases distributed lock, and writes CSV
const flushed = await DataBufferEngine.flush();
assert.equal(flushed.count, 2);
assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 0);
assert.equal(fakeRedis.strings.has(REDIS_FLUSH_LOCK_KEY), false);
// Verify Port 4000 /internal/recent & /internal/metrics read shared Redis without double-counting
const serverMod = require("../server/index.js") as IngestionServerModule;
const origWriterDir = serverMod.writer.baseDir;
serverMod.writer.baseDir = tmpStorageDir;
serverMod.setRedisClientForTests(fakeRedis);
await new Promise<void>((resolve) => serverMod.server.listen(0, "127.0.0.1", () => resolve()));
const addr = serverMod.server.address() as AddressInfo;
process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${addr.port}`;
try {
await DataBufferEngine.pushAsync({
clientId: "phase03_redis",
eventId: "r-evt-3",
timestamp: "2026-11-17T10:00:02.000Z",
payload: JSON.stringify({ seq: 3 }),
});
const stats = await DataBufferEngine.getBufferStatsAsync();
assert.equal(stats.redisEnabled, true);
assert.equal(stats.redisBufferSize, 1);
const metrics = await ApiEngine.getDashboardMetrics();
// Shared Redis count (1) must NOT be counted twice as 2 across Port 3000 + Port 4000
assert.equal(metrics.bufferStats.currentBufferSize, 1);
// Clear recent list & buffer via Port 4000 flush
const flushRes = await serverMod.flushBuffer();
assert.equal(flushRes.written, 1);
assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 0);
} finally {
serverMod.writer.baseDir = origWriterDir;
serverMod.setRedisClientForTests(undefined);
serverMod.stopTimers();
delete process.env.INGESTION_INTERNAL_URL;
await new Promise<void>((resolve) => serverMod.io.close(() => serverMod.server.close(() => resolve())));
}
// Fallback to local RAM buffer when Redis is disabled (null)
DataBufferEngine.setRedisClientForTests(null);
await DataBufferEngine.pushAsync({
clientId: "phase03_ram",
eventId: "ram-evt-1",
timestamp: "2026-11-17T10:01:00.000Z",
payload: JSON.stringify({ mode: "ram" }),
});
const ramStats = await DataBufferEngine.getBufferStatsAsync();
assert.equal(ramStats.redisEnabled, false);
assert.equal(ramStats.localBufferSize, 1);
const ramFlushed = await DataBufferEngine.flush();
assert.equal(ramFlushed.count, 1);
});
test("BUG-09: Cooperative CSV partition locking serializes concurrent Port 3000 + Port 4000 writers, recovers stale locks, and surfaces Port 4000 flush errors", async () => {
const worker = new CsvWriterWorker(tmpStorageDir);
const partitionDate = "2026-11-17";
const clientId = "phase03_concurrent";
const batchA: DataEvent[] = Array.from({ length: 15 }, (_, idx) => ({
clientId,
eventId: `a-${idx}`,
timestamp: `${partitionDate}T12:00:00.000Z`,
payload: JSON.stringify({ writer: "A", idx }),
}));
const batchB: DataEvent[] = Array.from({ length: 15 }, (_, idx) => ({
clientId,
eventId: `b-${idx}`,
timestamp: `${partitionDate}T12:00:01.000Z`,
payload: JSON.stringify({ writer: "B", idx }),
}));
await Promise.all([
CSVStorageManager.appendBatch(batchA),
worker.writeBatch(batchB),
]);
const csvPath = path.join(tmpStorageDir, clientId, `${partitionDate}.csv`);
const lockPath = path.join(tmpStorageDir, clientId, `.${partitionDate}.csv.lock`);
assert.equal(fs.existsSync(lockPath), false, "Lock file must be cleaned up after writes");
const lines = fs.readFileSync(csvPath, "utf8").trim().split("\n");
assert.equal(lines[0], '"clientId","eventId","timestamp","payload"');
assert.equal(lines.length, 31, "Must contain exactly 1 header + 30 complete rows without interleaving");
// Stale lock recovery (>30s old mtime)
fs.writeFileSync(lockPath, "999999:0", "utf8");
const pastTime = new Date(Date.now() - 45_000);
fs.utimesSync(lockPath, pastTime, pastTime);
await CSVStorageManager.appendBatch([
{
clientId,
eventId: "after-stale-lock",
timestamp: `${partitionDate}T12:00:02.000Z`,
payload: JSON.stringify({ staleRecovered: true }),
},
]);
assert.equal(fs.existsSync(lockPath), false);
const files = await CSVStorageManager.listAllFilesAsync();
assert.equal(files[0].linesCount, 31);
// Explicit flush error & warning propagation in POST /api/storage
const failingBridge = createServer((_req, res) => {
res.statusCode = 500;
res.setHeader("Content-Type", "application/json");
res.end(JSON.stringify({ success: false, error: "Disk permission denied on Port 4000" }));
});
await new Promise<void>((resolve) => failingBridge.listen(0, "127.0.0.1", () => resolve()));
const bridgeAddr = failingBridge.address() as AddressInfo;
process.env.INGESTION_INTERNAL_URL = `http://127.0.0.1:${bridgeAddr.port}`;
try {
const errRes = await postStorageRoute(new NextRequest("http://localhost:3000/api/storage", { method: "POST" }));
assert.equal(errRes.status, 502);
const errJson = await errRes.json();
assert.equal(errJson.success, false);
assert.equal(errJson.code, "INGESTION_FLUSH_FAILED");
assert.equal(errJson.ingestionFlushed, false);
} finally {
delete process.env.INGESTION_INTERNAL_URL;
await new Promise<void>((resolve) => failingBridge.close(() => resolve()));
}
// When Port 4000 is down and INGESTION_INTERNAL_URL is unset, standalone dev surfaces explicit warning + ingestionFlushed: false
const standaloneRes = await postStorageRoute(new NextRequest("http://localhost:3000/api/storage", { method: "POST" }));
assert.equal(standaloneRes.status, 200);
const standaloneJson = await standaloneRes.json();
assert.equal(standaloneJson.ingestionFlushed, false);
assert.match(String(standaloneJson.warning || ""), /unreachable/i);
});
test("BUG-10: Cached streaming row counter counts multiline RFC 4180 payloads accurately and avoids repeated disk reads", async () => {
await CSVStorageManager.appendBatch([
{
clientId: "phase03_perf",
eventId: "multiline-1",
timestamp: "2026-11-17T08:00:00.000Z",
payload: JSON.stringify({ note: "line1\nline2\n\"quoted\"\nline4" }),
},
{
clientId: "phase03_perf",
eventId: "multiline-2",
timestamp: "2026-11-17T08:00:01.000Z",
payload: JSON.stringify({ note: "single" }),
},
]);
const firstStats = await CSVStorageManager.getStorageStatsAsync();
assert.equal(firstStats.totalRows, 2);
// Spy on fs.createReadStream to verify repeated dashboard polls hit mtime+size cache with 0 file scans
const origCreateReadStream = fs.createReadStream;
let streamReads = 0;
(fs as unknown as { createReadStream: typeof fs.createReadStream }).createReadStream = ((
...args: Parameters<typeof fs.createReadStream>
) => {
streamReads += 1;
return origCreateReadStream(...args);
}) as typeof fs.createReadStream;
try {
const secondStats = await CSVStorageManager.getStorageStatsAsync();
assert.equal(secondStats.totalRows, 2);
assert.equal(streamReads, 0, "Unchanged CSV files must use cached row count");
} finally {
(fs as unknown as { createReadStream: typeof fs.createReadStream }).createReadStream =
origCreateReadStream;
}
// Server 3s settings sync must not repeatedly prune when autoPruneDays is unchanged
const serverMod = require("../server/index.js") as IngestionServerModule;
const oldPartitionDir = path.join(tmpStorageDir, "phase03_prune_check");
fs.mkdirSync(oldPartitionDir, { recursive: true });
fs.writeFileSync(path.join(oldPartitionDir, "2020-01-01.csv"), "h\nr\n", "utf8");
const prunedCount = CSVStorageManager.pruneExpiredPartitions(30);
assert.equal(prunedCount, 1);
serverMod.applyRuntimeSettingsObject({ autoPruneDays: 30 });
fs.writeFileSync(path.join(oldPartitionDir, "2020-01-02.csv"), "h\nr\n", "utf8");
// Same autoPruneDays=30 applied again (simulating 3s DB poll) must NOT prune on every tick
serverMod.applyRuntimeSettingsObject({ autoPruneDays: 30 });
assert.equal(fs.existsSync(path.join(oldPartitionDir, "2020-01-02.csv")), true);
serverMod.stopTimers();
});
test("BUG-12: Prisma schema models and mappings align cleanly with runtime SQL DDL and Docker Compose Redis wiring", () => {
const schemaPath = path.resolve(process.cwd(), "prisma/schema.prisma");
const schemaText = fs.readFileSync(schemaPath, "utf8");
assert.match(schemaText, /model ClientSDK[\s\S]*clientId\s+String\s+@id\s+@map\("client_id"\)/);
assert.match(schemaText, /token\s+String/);
assert.match(schemaText, /model DeletedClient[\s\S]*@@map\("deleted_clients"\)/);
assert.match(schemaText, /model AdminUser[\s\S]*tokenVersion\s+Int\s+@default\(1\)\s+@map\("token_version"\)/);
assert.match(schemaText, /model AdminLoginAttempt[\s\S]*@@map\("admin_login_attempts"\)/);
assert.match(schemaText, /model AdminSettings[\s\S]*maxEventsPerSec\s+Int\s+@default\(100\)\s+@map\("max_events_per_sec"\)/);
assert.equal(/model ApiKey\b/.test(schemaText), false);
assert.equal(/model AuditLog\b/.test(schemaText), false);
const composeText = fs.readFileSync(path.resolve(process.cwd(), "docker/docker-compose.yml"), "utf8");
assert.equal((composeText.match(/REDIS_URL:\s*"redis:\/\/redis:6379"/g) || []).length, 2);
assert.match(composeText, /redis:\s*\n\s*condition:\s*service_healthy/);
});
test("BUG-14: GET /api/data searches eventId & clientId + UTC dates, GET /api/events caps CSV tail, and GET /api/export rejects ambiguous filename & honors ?flush=false", async () => {
DataBufferEngine.setRedisClientForTests(null);
await SDKManager.registerClientAsync({ clientId: "alpha_cli", name: "Alpha", rateLimit: 100 });
await SDKManager.registerClientAsync({ clientId: "beta_cli", name: "Beta", rateLimit: 100 });
const tsMillis = String(Date.UTC(2026, 10, 17, 15, 0, 0)); // 2026-11-17
await DataBufferEngine.pushBatchAsync([
{
clientId: "alpha_cli",
eventId: "unique-findme-id-99",
timestamp: tsMillis,
payload: JSON.stringify({ temp: 22 }),
},
{
clientId: "beta_cli",
eventId: "beta-id-1",
timestamp: "2026-11-17T15:01:00.000Z",
payload: JSON.stringify({ temp: 33 }),
},
]);
// 1. Search by eventId with from/to UTC normalization while still buffered
const dataByEvent = await getDataRoute(
new NextRequest("http://localhost:3000/api/data?search=unique-findme-id-99&from=2026-11-17&to=2026-11-17"),
);
assert.equal(dataByEvent.status, 200);
const dataEventJson = await dataByEvent.json();
assert.equal(dataEventJson.total, 1);
assert.equal(dataEventJson.events[0].eventId, "unique-findme-id-99");
// 2. Search by clientId substring
const dataByClient = await getDataRoute(
new NextRequest("http://localhost:3000/api/data?search=beta_cli"),
);
const dataClientJson = await dataByClient.json();
assert.equal(dataClientJson.total, 1);
assert.equal(dataClientJson.events[0].clientId, "beta_cli");
// 3. Export with ?flush=false must NOT drain active buffer
const noFlushRes = await getExportRoute(
new NextRequest("http://localhost:3000/api/export?flush=false&format=json"),
);
assert.equal(noFlushRes.status, 200);
assert.equal((await noFlushRes.text()).trim(), "[]");
assert.equal((await DataBufferEngine.getBufferStatsAsync()).localBufferSize, 2);
// 4. Default flush writes both 2026-11-17.csv files -> clientId=all + filename must 400 AMBIGUOUS_FILENAME
const flushExportRes = await getExportRoute(
new NextRequest("http://localhost:3000/api/export?format=json"),
);
assert.equal(flushExportRes.status, 200);
const flushedPayload = await flushExportRes.json();
assert.equal(flushedPayload.length, 2);
const ambiguousRes = await getExportRoute(
new NextRequest("http://localhost:3000/api/export?clientId=all&filename=2026-11-17.csv"),
);
assert.equal(ambiguousRes.status, 400);
const ambiguousJson = await ambiguousRes.json();
assert.equal(ambiguousJson.code, "AMBIGUOUS_FILENAME");
// Disambiguated download with explicit clientId succeeds cleanly
const specificRes = await getExportRoute(
new NextRequest("http://localhost:3000/api/export?clientId=alpha_cli&filename=2026-11-17.csv"),
);
assert.equal(specificRes.status, 200);
const csvBody = await specificRes.text();
assert.match(csvBody, /unique-findme-id-99/);
// 5. GET /api/events with limit=1 returns latest single stored event cleanly
const eventsRes = await getEventsRoute(
new NextRequest("http://localhost:3000/api/events?clientId=alpha_cli&limit=1"),
);
assert.equal(eventsRes.status, 200);
const eventsJson = await eventsRes.json();
assert.equal(eventsJson.events.length, 1);
assert.equal(eventsJson.events[0].eventId, "unique-findme-id-99");
});
+266
View File
@@ -0,0 +1,266 @@
"""
Phase-05 Verification Tests (BUG-18):
- Single Source of Truth between canonical `datastack` (`sdk/python/datastack`)
and compatibility alias `datastack_sdk` (`sdk-python`).
- `CLIENT_ID` config loading (`config.datastack.py`, `DATASTACK_CLIENT_ID`, kwarg),
JWT `clientId`/`sub` fallback extraction, and mismatch validation.
- End-to-end propagation of `clientId` & `X-DataStack-Client-Id` into HTTP
fallback ingestion (`send`, `send_batch`, `send_file`), status snapshot, and
offline disk spool replay against a local HTTP test server.
"""
import base64
import json
import os
import sys
import tempfile
import threading
import unittest
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import Any, Dict, List
REPO_ROOT = Path(__file__).resolve().parents[1]
CANONICAL_PKG_ROOT = REPO_ROOT / "sdk" / "python" / "datastack"
COMPAT_PKG_ROOT = REPO_ROOT / "sdk-python"
for p in (str(CANONICAL_PKG_ROOT), str(COMPAT_PKG_ROOT)):
if p not in sys.path:
sys.path.insert(0, p)
import datastack # noqa: E402
import datastack_sdk # noqa: E402
from datastack import ( # noqa: E402
DataStack,
DataStackClient,
DataStackConfig,
DataStackConfigError,
DataStackOfflineSpoolError,
DiskSpooler,
HeartbeatMonitor,
load_config,
)
def _make_jwt(payload: Dict[str, Any]) -> str:
header = (
base64.urlsafe_b64encode(json.dumps({"alg": "HS256", "typ": "JWT"}).encode())
.decode()
.rstrip("=")
)
body = (
base64.urlsafe_b64encode(json.dumps(payload).encode())
.decode()
.rstrip("=")
)
return f"{header}.{body}.testsig"
class Phase05PythonSdkTests(unittest.TestCase):
def test_single_source_of_truth_identity(self) -> None:
self.assertIs(datastack_sdk.DataStack, datastack.DataStack)
self.assertIs(datastack_sdk.DataStackClient, datastack.DataStackClient)
self.assertIs(datastack_sdk.DataStackConfig, datastack.DataStackConfig)
self.assertIs(datastack_sdk.load_config, datastack.load_config)
self.assertIs(
datastack_sdk.DataStackConfigError, datastack.DataStackConfigError
)
self.assertIs(
datastack_sdk.DataStackOfflineSpoolError,
datastack.DataStackOfflineSpoolError,
)
self.assertIs(datastack_sdk.DiskSpooler, datastack.DiskSpooler)
self.assertIs(datastack_sdk.HeartbeatMonitor, datastack.HeartbeatMonitor)
self.assertEqual(datastack_sdk.__version__, datastack.__version__)
# Submodule shims also resolve to canonical classes
from datastack_sdk.client import DataStackClient as ShimClient
from datastack_sdk.config import load_config as shim_load_config
from datastack_sdk.exceptions import (
DataStackOfflineSpoolError as ShimSpoolErr,
)
self.assertIs(ShimClient, DataStackClient)
self.assertIs(shim_load_config, load_config)
self.assertIs(ShimSpoolErr, DataStackOfflineSpoolError)
def test_client_id_config_env_and_jwt_validation(self) -> None:
tok_alpha = _make_jwt({"clientId": "alpha_svc", "sub": "alpha_svc"})
with tempfile.TemporaryDirectory() as tmpdir:
cfg_file = Path(tmpdir) / "config.datastack.py"
cfg_file.write_text(
f'DATASTACK_CONFIG = {{\n'
f' "SERVER_URL": "http://127.0.0.1:49999",\n'
f' "API_URL": "http://127.0.0.1:39999",\n'
f' "CLIENT_TOKEN": "{tok_alpha}",\n'
f' "CLIENT_ID": "alpha_svc",\n'
f' "AUTO_CONNECT": False,\n'
f' "OFFLINE_SPOOL_DIR": "{Path(tmpdir) / "spool"}",\n'
f'}}\n',
encoding="utf-8",
)
cfg = load_config(config_path=str(cfg_file))
self.assertIsInstance(cfg, DataStackConfig)
self.assertEqual(cfg.client_id, "alpha_svc")
# Matching explicit client_id succeeds
client = DataStack(config_path=str(cfg_file), auto_connect=False)
self.assertEqual(client.client_id, "alpha_svc")
self.assertEqual(client.status()["clientId"], "alpha_svc")
client.disconnect()
# Env override matching JWT succeeds; mismatching raises DataStackConfigError
old_env = os.environ.get("DATASTACK_CLIENT_ID")
try:
os.environ["DATASTACK_CLIENT_ID"] = "wrong_svc"
with self.assertRaises(DataStackConfigError):
DataStack(config_path=str(cfg_file), auto_connect=False)
finally:
if old_env is None:
os.environ.pop("DATASTACK_CLIENT_ID", None)
else:
os.environ["DATASTACK_CLIENT_ID"] = old_env
# Mismatched kwarg client_id raises DataStackConfigError
with self.assertRaises(DataStackConfigError):
DataStack(
config_path=str(cfg_file),
client_id="mismatch_svc",
auto_connect=False,
)
# Auto-derives client_id from JWT when CLIENT_ID is omitted
tok_derived = _make_jwt({"sub": "derived_worker"})
derived_client = DataStack(
server_url="http://127.0.0.1:49999",
api_url="http://127.0.0.1:39999",
token=tok_derived,
auto_connect=False,
)
self.assertEqual(derived_client.client_id, "derived_worker")
self.assertEqual(derived_client.status()["clientId"], "derived_worker")
derived_client.disconnect()
def test_http_fallback_and_offline_spool_carry_client_id(self) -> None:
recorded_requests: List[Dict[str, Any]] = []
tok = _make_jwt({"clientId": "sensor_01", "sub": "sensor_01"})
class MockHandler(BaseHTTPRequestHandler):
def do_GET(self) -> None: # noqa: N802
if self.path == "/api/health":
body = json.dumps({"status": "online"}).encode("utf-8")
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
return
self.send_response(404)
self.end_headers()
def do_POST(self) -> None: # noqa: N802
length = int(self.headers.get("Content-Length", "0"))
raw = self.rfile.read(length).decode("utf-8")
parsed = json.loads(raw) if raw else {}
recorded_requests.append(
{
"path": self.path,
"client_id_header": self.headers.get("X-DataStack-Client-Id"),
"auth": self.headers.get("Authorization"),
"body": parsed,
}
)
resp_payload: Dict[str, Any] = {"success": True}
if isinstance(parsed, dict) and "batch" in parsed:
resp_payload["processed"] = len(parsed["batch"])
resp_payload["rejected"] = 0
elif isinstance(parsed, dict) and "records" in parsed:
resp_payload["processed"] = len(parsed["records"])
resp_payload["rejected"] = 0
body = json.dumps(resp_payload).encode("utf-8")
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, format: str, *args: Any) -> None:
return
server = ThreadingHTTPServer(("127.0.0.1", 0), MockHandler)
port = server.server_address[1]
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
with tempfile.TemporaryDirectory() as tmpdir:
spool_dir = Path(tmpdir) / "spool"
csv_file = Path(tmpdir) / "sample.csv"
csv_file.write_text("temp,unit\n21.5,C\n22.0,C\n", encoding="utf-8")
client = DataStack(
server_url="http://127.0.0.1:1", # unreachable WS -> HTTP fallback
api_url=f"http://127.0.0.1:{port}",
token=tok,
client_id="sensor_01",
auto_connect=False,
request_timeout_sec=1.0,
max_retries=0,
offline_spool_dir=str(spool_dir),
)
# Single event via HTTP fallback
res_single = client.send({"reading": 42})
self.assertTrue(res_single.get("success"))
self.assertEqual(res_single.get("transport"), "HTTP_FALLBACK")
# Batch events via HTTP fallback
res_batch = client.send_batch([{"k": "a"}, {"k": "b"}])
self.assertEqual(res_batch.get("processed"), 2)
# File ingestion via HTTP fallback
res_file = client.send_file(csv_file)
self.assertEqual(res_file.get("processed"), 2)
client.disconnect()
self.assertEqual(len(recorded_requests), 3)
for req in recorded_requests:
self.assertEqual(req["path"], "/api/events")
self.assertEqual(req["client_id_header"], "sensor_01")
self.assertEqual(req["auth"], f"Bearer {tok}")
self.assertEqual(recorded_requests[0]["body"]["clientId"], "sensor_01")
self.assertEqual(
[r["clientId"] for r in recorded_requests[1]["body"]["batch"]],
["sensor_01", "sensor_01"],
)
self.assertEqual(
[r["clientId"] for r in recorded_requests[2]["body"]["records"]],
["sensor_01", "sensor_01"],
)
# Offline spool check when HTTP is also unreachable
offline_client = datastack_sdk.DataStackClient(
server_url="http://127.0.0.1:1",
api_url="http://127.0.0.1:1",
token=tok,
auto_connect=False,
request_timeout_sec=1.0,
max_retries=0,
offline_spool_dir=str(spool_dir),
)
spool_res = offline_client.send({"offline_metric": 99})
offline_client.disconnect()
self.assertTrue(spool_res.get("spooled"))
self.assertEqual(spool_res.get("transport"), "OFFLINE_SPOOL")
self.assertGreaterEqual(spool_res.get("spooledCount", 0), 1)
finally:
server.shutdown()
server.server_close()
if __name__ == "__main__":
unittest.main(verbosity=2)
+81
View File
@@ -0,0 +1,81 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import test from 'node:test';
const ROOT = process.cwd();
function readPngDimensions(filePath: string): { width: number; height: number } {
const buf = fs.readFileSync(filePath);
const pngSig = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
assert.ok(buf.subarray(0, 8).equals(pngSig), `${path.basename(filePath)} must be a valid PNG`);
assert.equal(buf.subarray(12, 16).toString('ascii'), 'IHDR');
return {
width: buf.readUInt32BE(16),
height: buf.readUInt32BE(20),
};
}
test('BUG-25: PWA manifest, icons (192/512/maskable/apple-touch), and Service Worker are complete and valid', () => {
const manifestPath = path.join(ROOT, 'public/manifest.json');
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
assert.equal(manifest.id, '/');
assert.equal(manifest.scope, '/');
assert.equal(manifest.display, 'standalone');
assert.ok(manifest.short_name && manifest.short_name.length <= 12);
assert.ok(Array.isArray(manifest.icons) && manifest.icons.length >= 3);
for (const icon of manifest.icons) {
assert.ok(!String(icon.purpose || '').includes('any maskable'), 'Separate any and maskable purposes');
const rel = String(icon.src).replace(/^\//, '');
const full = path.join(ROOT, 'public', rel);
assert.ok(fs.existsSync(full), `Missing manifest icon ${icon.src}`);
const [expW, expH] = String(icon.sizes).split('x').map(Number);
const dims = readPngDimensions(full);
assert.equal(dims.width, expW);
assert.equal(dims.height, expH);
}
const appleDims = readPngDimensions(path.join(ROOT, 'public/apple-touch-icon.png'));
assert.deepEqual(appleDims, { width: 180, height: 180 });
assert.ok(fs.existsSync(path.join(ROOT, 'public/icon.svg')));
const swContent = fs.readFileSync(path.join(ROOT, 'public/sw.js'), 'utf8');
assert.match(swContent, /request\.method !== ['"]GET['"]/);
assert.match(swContent, /\/api\//);
assert.match(swContent, /\/socket\.io\//);
assert.match(swContent, /new Response\(/);
const layoutContent = fs.readFileSync(path.join(ROOT, 'app/layout.tsx'), 'utf8');
assert.match(layoutContent, /PwaLifecycle/);
assert.match(layoutContent, /export const viewport: Viewport/);
assert.match(layoutContent, /apple-touch-icon\.png/);
const pwaClient = fs.readFileSync(path.join(ROOT, 'components/pwa/PwaLifecycle.tsx'), 'utf8');
assert.match(pwaClient, /serviceWorker\.register\(['"]\/sw\.js['"]/);
assert.match(pwaClient, /beforeinstallprompt/);
});
test('BUG-17: Landing/Navbar/Footer/404 navigation uses root-relative anchors and valid Link semantics', () => {
const files = [
'components/layout/Navbar.tsx',
'components/layout/Footer.tsx',
'components/sections/Hero.tsx',
'components/sections/LandingAbout.tsx',
'app/not-found.tsx',
];
for (const rel of files) {
const content = fs.readFileSync(path.join(ROOT, rel), 'utf8');
assert.equal(/href=["']#/.test(content), false, `${rel} should use /#... root anchors instead of bare #...`);
const linkBlocks = content.match(/<Link\b[\s\S]*?<\/Link>/g) || [];
for (const block of linkBlocks) {
assert.equal(/<Button\b/.test(block), false, `${rel} should not nest <Button> inside <Link>: ${block}`);
}
}
const notFoundContent = fs.readFileSync(path.join(ROOT, 'app/not-found.tsx'), 'utf8');
assert.match(notFoundContent, /href=["']\/["']/);
assert.match(notFoundContent, /href=["']\/vcon\/dashboard["']/);
});