feat(client): implement per-client storage and security controls
- Added storage quota, IP allowlisting, and expiration logic to client management APIs. - Updated CSV storage manager to support auto-compression and improved partition file parsing. - Integrated security checks into event ingestion and data retrieval routes.
This commit is contained in:
1 parent
89e4d68690
commit
2873b9904a
32 files changed
+6075
-241
No files matched your search
@@ -0,0 +1,727 @@
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs/promises";
|
||||
import http from "node:http";
|
||||
import { createRequire } from "node:module";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import zlib from "node:zlib";
|
||||
import { NextRequest } from "next/server";
|
||||
import { io as ioClient, type Socket } from "socket.io-client";
|
||||
import { DELETE as deleteClientRoute, PATCH as patchClientRoute } from "../app/api/clients/[id]/route";
|
||||
import { GET as clientsGet, POST as clientsPost } from "../app/api/clients/route";
|
||||
import { GET as dataGet } from "../app/api/data/route";
|
||||
import { GET as eventsGet, POST as eventsPost } from "../app/api/events/route";
|
||||
import { GET as exportGet } from "../app/api/export/route";
|
||||
import { GET as settingsGet, POST as settingsPost } from "../app/api/settings/route";
|
||||
import { POST as archivePost } from "../app/api/storage/archive/route";
|
||||
import { GET as storageGet } from "../app/api/storage/route";
|
||||
import { POST as s3TestPost } from "../app/api/storage/s3-test/route";
|
||||
import { SettingsManager } from "../core/adminSettings/settingsManager";
|
||||
import {
|
||||
createAdminSessionToken,
|
||||
getAdminSessionSecret,
|
||||
} from "../core/auth/adminSession";
|
||||
import {
|
||||
createInitialAdmin,
|
||||
resetAdminStateForTests,
|
||||
} from "../core/auth/adminUsers";
|
||||
import { DataBufferEngine } from "../core/data/buffer";
|
||||
import { SDKManager } from "../core/sdk/sdkManager";
|
||||
import {
|
||||
decryptSecret,
|
||||
encryptSecret,
|
||||
getMaskedHintFromEncrypted,
|
||||
SecretVaultError,
|
||||
} from "../core/Security/secretVault";
|
||||
import { CSVStorageManager } from "../core/Storage/csvStorage";
|
||||
import {
|
||||
buildS3ObjectKey,
|
||||
normalizeS3EndpointUrl,
|
||||
testS3Connection,
|
||||
uploadFileToS3,
|
||||
} from "../core/Storage/s3Archive";
|
||||
import {
|
||||
DEFAULT_EDITABLE_SETTINGS,
|
||||
settingsToNumericDrafts,
|
||||
validateEditableSettingsDraft,
|
||||
validateSingleNumericSetting,
|
||||
} from "../lib/vconUiHelpers";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
async function makeAdminCookie(): Promise<string> {
|
||||
resetAdminStateForTests();
|
||||
await createInitialAdmin("v2-p1@datastack.dev", "V2Phase01Pass!");
|
||||
const secret = getAdminSessionSecret()!;
|
||||
const token = await createAdminSessionToken(
|
||||
"v2-p1@datastack.dev",
|
||||
secret,
|
||||
Date.now() + 3600_000,
|
||||
1,
|
||||
);
|
||||
return `datastack_admin_session=${token}`;
|
||||
}
|
||||
|
||||
test("V2-Phase-01 [Vault]: encryptSecret/decryptSecret AES-256-GCM roundtrip, tamper detection & masked hints", () => {
|
||||
const secretA = "test-master-secret-key-at-least-32-characters";
|
||||
const secretB = "different-master-secret-key-at-least-32-chars";
|
||||
|
||||
const encrypted = encryptSecret("r2_secret_value_987654321", secretA);
|
||||
assert.ok(encrypted.startsWith("v1:"));
|
||||
assert.equal(decryptSecret(encrypted, secretA), "r2_secret_value_987654321");
|
||||
assert.equal(getMaskedHintFromEncrypted(encrypted, secretA), "••••4321");
|
||||
|
||||
// Tampered ciphertext throws SecretVaultError (V2-E01)
|
||||
const parts = encrypted.split(":");
|
||||
const flippedChar = parts[3].startsWith("A") ? "B" : "A";
|
||||
const tampered = `${parts[0]}:${parts[1]}:${parts[2]}:${flippedChar}${parts[3].slice(1)}`;
|
||||
assert.throws(
|
||||
() => decryptSecret(tampered, secretA),
|
||||
(err: unknown) =>
|
||||
err instanceof SecretVaultError && err.code === "SECRET_VAULT_DECRYPT_FAILED",
|
||||
);
|
||||
|
||||
// Wrong master key throws SecretVaultError
|
||||
assert.throws(
|
||||
() => decryptSecret(encrypted, secretB),
|
||||
(err: unknown) =>
|
||||
err instanceof SecretVaultError && err.code === "SECRET_VAULT_DECRYPT_FAILED",
|
||||
);
|
||||
|
||||
// Too-short master secret rejects key derivation
|
||||
assert.throws(
|
||||
() => encryptSecret("hello", "short"),
|
||||
(err: unknown) =>
|
||||
err instanceof SecretVaultError && err.code === "SECRET_VAULT_KEY_UNAVAILABLE",
|
||||
);
|
||||
});
|
||||
|
||||
test("V2-Phase-01 [S3 SigV4]: buildS3ObjectKey, normalizeS3EndpointUrl, testS3Connection & uploadFileToS3 against mock S3 server", async () => {
|
||||
assert.equal(
|
||||
buildS3ObjectKey("/datastack/", "client_01", "2026-10-01.csv.gz"),
|
||||
"datastack/client_01/2026-10-01.csv.gz",
|
||||
);
|
||||
assert.equal(
|
||||
normalizeS3EndpointUrl("", "us-east-1").url.origin,
|
||||
"https://s3.us-east-1.amazonaws.com",
|
||||
);
|
||||
|
||||
const requests: Array<{
|
||||
method: string;
|
||||
url: string;
|
||||
auth: string;
|
||||
bodyLen: number;
|
||||
}> = [];
|
||||
let failUploads = false;
|
||||
|
||||
const mockS3 = http.createServer((req, res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (chunk: Buffer) => chunks.push(chunk));
|
||||
req.on("end", () => {
|
||||
const body = Buffer.concat(chunks);
|
||||
requests.push({
|
||||
method: req.method || "",
|
||||
url: req.url || "",
|
||||
auth: String(req.headers.authorization || ""),
|
||||
bodyLen: body.byteLength,
|
||||
});
|
||||
if (req.method === "PUT" && failUploads) {
|
||||
res.writeHead(500, { "content-type": "text/plain" });
|
||||
res.end("Simulated S3 outage");
|
||||
return;
|
||||
}
|
||||
if (req.method === "PUT") {
|
||||
res.writeHead(200, { etag: '"mock-etag-1"' });
|
||||
res.end("");
|
||||
return;
|
||||
}
|
||||
res.writeHead(200, { "content-type": "application/xml" });
|
||||
res.end("<ListBucketResult></ListBucketResult>");
|
||||
});
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve) => mockS3.listen(0, "127.0.0.1", resolve));
|
||||
const addr = mockS3.address() as AddressInfo;
|
||||
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "datastack-v2-s3-"));
|
||||
|
||||
try {
|
||||
const samplePath = path.join(tmpDir, "2026-10-01.csv.gz");
|
||||
const gzData = zlib.gzipSync(
|
||||
Buffer.from('clientId,eventId,timestamp,payload\n"c1","e1","2026-10-01T00:00:00Z","{}"\n'),
|
||||
);
|
||||
await fs.writeFile(samplePath, gzData);
|
||||
|
||||
const cfg = {
|
||||
endpoint: `http://127.0.0.1:${addr.port}`,
|
||||
region: "auto",
|
||||
bucket: "ds-archive",
|
||||
prefix: "datastack/",
|
||||
accessKeyId: "AKIA_TEST_123",
|
||||
secretAccessKey: "secret_access_key_xyz_999",
|
||||
};
|
||||
|
||||
const probe = await testS3Connection(cfg);
|
||||
assert.equal(probe.ok, true);
|
||||
assert.equal(probe.status, 200);
|
||||
assert.match(requests[0].auth, /^AWS4-HMAC-SHA256 Credential=AKIA_TEST_123\//);
|
||||
assert.match(requests[0].url, /^\/ds-archive\?list-type=2&max-keys=1$/);
|
||||
|
||||
const uploadOk = await uploadFileToS3(
|
||||
cfg,
|
||||
samplePath,
|
||||
"datastack/c1/2026-10-01.csv.gz",
|
||||
);
|
||||
assert.equal(uploadOk.ok, true);
|
||||
assert.equal(uploadOk.etag, '"mock-etag-1"');
|
||||
assert.equal(requests[1].method, "PUT");
|
||||
assert.equal(requests[1].url, "/ds-archive/datastack/c1/2026-10-01.csv.gz");
|
||||
assert.equal(requests[1].bodyLen, gzData.byteLength);
|
||||
|
||||
failUploads = true;
|
||||
const uploadErr = await uploadFileToS3(
|
||||
cfg,
|
||||
samplePath,
|
||||
"datastack/c1/2026-10-01.csv.gz",
|
||||
);
|
||||
assert.equal(uploadErr.ok, false);
|
||||
assert.equal(uploadErr.code, "S3_ARCHIVE_FAILED");
|
||||
assert.match(uploadErr.error || "", /HTTP 500/);
|
||||
} finally {
|
||||
await new Promise<void>((resolve) => mockS3.close(() => resolve()));
|
||||
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("V2-Phase-01 [Compression, Export, S3 Safe Prune V2-E02, Corrupt Gzip V2-E03]", async () => {
|
||||
const prevStorageDir = process.env.DATASTACK_STORAGE_DIR;
|
||||
const prevDbUrl = process.env.DATABASE_URL;
|
||||
const origBaseStorageDir = CSVStorageManager.baseStorageDir;
|
||||
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "datastack-v2-gz-"));
|
||||
delete process.env.DATABASE_URL;
|
||||
process.env.DATASTACK_STORAGE_DIR = tmpDir;
|
||||
CSVStorageManager.baseStorageDir = tmpDir;
|
||||
CSVStorageManager.clearRowCountCache();
|
||||
CSVStorageManager.clearQuotaCaches();
|
||||
|
||||
const uploadedKeys: string[] = [];
|
||||
let failNextUpload = false;
|
||||
const mockS3 = http.createServer((req, res) => {
|
||||
req.on("data", () => undefined);
|
||||
req.on("end", () => {
|
||||
if (req.method === "PUT") {
|
||||
if (failNextUpload) {
|
||||
res.writeHead(502);
|
||||
res.end("Upstream error");
|
||||
return;
|
||||
}
|
||||
uploadedKeys.push(req.url || "");
|
||||
res.writeHead(200, { etag: '"ok"' });
|
||||
res.end("");
|
||||
return;
|
||||
}
|
||||
res.writeHead(200);
|
||||
res.end("<OK/>");
|
||||
});
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve) => mockS3.listen(0, "127.0.0.1", resolve));
|
||||
const s3Addr = mockS3.address() as AddressInfo;
|
||||
|
||||
try {
|
||||
const cookie = await makeAdminCookie();
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
const oldDate = new Date(Date.now() - 45 * 86_400_000).toISOString().slice(0, 10);
|
||||
await SettingsManager.updateSettings({
|
||||
autoPruneDays: 90,
|
||||
compressAfterDays: 0,
|
||||
enableS3Backup: false,
|
||||
diskPauseThresholdPct: 0,
|
||||
});
|
||||
|
||||
// 1. Write 2 old rows (45 days ago) and 1 today row
|
||||
await CSVStorageManager.appendBatch([
|
||||
{
|
||||
clientId: "v2_alpha",
|
||||
eventId: "evt_old_1",
|
||||
timestamp: `${oldDate}T10:00:00.000Z`,
|
||||
payload: '{"stage":"old","seq":1}',
|
||||
},
|
||||
{
|
||||
clientId: "v2_alpha",
|
||||
eventId: "evt_old_2",
|
||||
timestamp: `${oldDate}T11:00:00.000Z`,
|
||||
payload: '{"stage":"old","seq":2}',
|
||||
},
|
||||
{
|
||||
clientId: "v2_alpha",
|
||||
eventId: "evt_today_1",
|
||||
timestamp: `${today}T09:00:00.000Z`,
|
||||
payload: '{"stage":"live","seq":3}',
|
||||
},
|
||||
]);
|
||||
|
||||
// 2. Compress partitions older than 30 days -> ${oldDate}.csv becomes .csv.gz; today stays .csv
|
||||
const compressed = await CSVStorageManager.compressOldPartitionsAsync(30);
|
||||
assert.equal(compressed, 1);
|
||||
|
||||
const files = await CSVStorageManager.listAllFilesAsync();
|
||||
assert.equal(files.length, 2);
|
||||
const gzMeta = files.find((f) => f.filename === `${oldDate}.csv.gz`);
|
||||
assert.ok(gzMeta, `Expected compressed ${oldDate}.csv.gz partition`);
|
||||
assert.equal(gzMeta.linesCount, 2);
|
||||
assert.equal(gzMeta.compressed, true);
|
||||
|
||||
// Original uncompressed ${oldDate}.csv was removed after atomic rename
|
||||
const legacyCsv = path.join(tmpDir, "v2_alpha", `${oldDate}.csv`);
|
||||
await assert.rejects(() => fs.stat(legacyCsv));
|
||||
|
||||
// 3. Verify GET /api/data reads .csv.gz transparently
|
||||
const dataRes = await dataGet(
|
||||
new NextRequest(
|
||||
`http://localhost:3000/api/data?clientId=v2_alpha&from=${oldDate}&to=${oldDate}`,
|
||||
{ headers: { cookie } },
|
||||
),
|
||||
);
|
||||
assert.equal(dataRes.status, 200);
|
||||
const dataJson = await dataRes.json();
|
||||
assert.equal(dataJson.total, 2);
|
||||
assert.equal(dataJson.events.length, 2);
|
||||
assert.equal(dataJson.events[0].eventId, "evt_old_2");
|
||||
assert.equal(dataJson.events[1].eventId, "evt_old_1");
|
||||
|
||||
// 4. Verify GET /api/events reads compressed partition seamlessly
|
||||
const evRes = await eventsGet(
|
||||
new NextRequest("http://localhost:3000/api/events?clientId=v2_alpha&limit=10", {
|
||||
headers: { cookie },
|
||||
}),
|
||||
);
|
||||
assert.equal(evRes.status, 200);
|
||||
const evJson = await evRes.json();
|
||||
const ids = (evJson.events as Array<{ eventId: string }>).map((e) => e.eventId);
|
||||
assert.ok(ids.includes("evt_old_1"));
|
||||
assert.ok(ids.includes("evt_today_1"));
|
||||
|
||||
// 5. Verify GET /api/export CSV, JSON, and raw single .csv.gz download
|
||||
const exportCsvRes = await exportGet(
|
||||
new NextRequest(
|
||||
`http://localhost:3000/api/export?clientId=v2_alpha&from=${oldDate}&to=${oldDate}&format=csv`,
|
||||
{ headers: { cookie } },
|
||||
),
|
||||
);
|
||||
assert.equal(exportCsvRes.status, 200);
|
||||
const exportedCsvText = await exportCsvRes.text();
|
||||
assert.match(exportedCsvText, /evt_old_1/);
|
||||
assert.match(exportedCsvText, /evt_old_2/);
|
||||
|
||||
const exportJsonRes = await exportGet(
|
||||
new NextRequest(
|
||||
`http://localhost:3000/api/export?clientId=v2_alpha&from=${oldDate}&to=${oldDate}&format=json`,
|
||||
{ headers: { cookie } },
|
||||
),
|
||||
);
|
||||
assert.equal(exportJsonRes.status, 200);
|
||||
const exportedJson = await exportJsonRes.json();
|
||||
assert.equal(exportedJson.length, 2);
|
||||
|
||||
const exportSingleGzRes = await exportGet(
|
||||
new NextRequest(
|
||||
`http://localhost:3000/api/export?clientId=v2_alpha&file=${oldDate}.csv.gz`,
|
||||
{ headers: { cookie } },
|
||||
),
|
||||
);
|
||||
assert.equal(exportSingleGzRes.status, 200);
|
||||
assert.equal(exportSingleGzRes.headers.get("content-type"), "application/gzip");
|
||||
const downloadedBuf = Buffer.from(await exportSingleGzRes.arrayBuffer());
|
||||
assert.match(zlib.gunzipSync(downloadedBuf).toString("utf8"), /evt_old_1/);
|
||||
|
||||
// 6. Save S3 settings via POST /api/settings and verify GET /api/settings masks secret (zero plaintext leak)
|
||||
const saveRes = await settingsPost(
|
||||
new NextRequest("http://localhost:3000/api/settings", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", cookie },
|
||||
body: JSON.stringify({
|
||||
compressAfterDays: 30,
|
||||
autoPruneDays: 60,
|
||||
diskPauseThresholdPct: 92,
|
||||
enableS3Backup: true,
|
||||
s3Endpoint: `http://127.0.0.1:${s3Addr.port}`,
|
||||
s3Region: "auto",
|
||||
s3Bucket: "prod-vault",
|
||||
s3Prefix: "archive-test/",
|
||||
s3AccessKeyId: "AKIA_V2_KEY",
|
||||
s3SecretAccessKey: "super_secret_v2_token_7788",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(saveRes.status, 200);
|
||||
|
||||
const getRes = await settingsGet(
|
||||
new NextRequest("http://localhost:3000/api/settings", { headers: { cookie } }),
|
||||
);
|
||||
assert.equal(getRes.status, 200);
|
||||
const getText = await getRes.text();
|
||||
assert.doesNotMatch(getText, /super_secret_v2_token_7788/);
|
||||
const parsedSettings = JSON.parse(getText).settings;
|
||||
assert.equal(parsedSettings.hasS3SecretKey, true);
|
||||
assert.equal(parsedSettings.s3SecretKeyMask, "••••7788");
|
||||
assert.equal(parsedSettings.compressAfterDays, 30);
|
||||
assert.equal(parsedSettings.diskPauseThresholdPct, 92);
|
||||
|
||||
// 7. POST /api/storage/s3-test succeeds with saved credentials
|
||||
const s3TestRes = await s3TestPost(
|
||||
new NextRequest("http://localhost:3000/api/storage/s3-test", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", cookie },
|
||||
body: JSON.stringify({}),
|
||||
}),
|
||||
);
|
||||
assert.equal(s3TestRes.status, 200);
|
||||
assert.equal((await s3TestRes.json()).success, true);
|
||||
|
||||
// 8. V2-E02: When S3 upload fails, POST /api/storage/archive returns 502 and NEVER deletes local partition
|
||||
failNextUpload = true;
|
||||
const failArchiveRes = await archivePost(
|
||||
new NextRequest("http://localhost:3000/api/storage/archive", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", cookie },
|
||||
body: JSON.stringify({ pruneAfterUpload: true, olderThanDays: 30 }),
|
||||
}),
|
||||
);
|
||||
assert.equal(failArchiveRes.status, 502);
|
||||
const failArchiveJson = await failArchiveRes.json();
|
||||
assert.equal(failArchiveJson.code, "S3_ARCHIVE_FAILED");
|
||||
assert.equal(failArchiveJson.prunedCount, 0);
|
||||
// File MUST still exist on disk!
|
||||
await fs.stat(gzMeta.filePath);
|
||||
|
||||
// 9. When S3 upload succeeds with pruneAfterUpload: false, local file stays + uploadedKeys receives .csv.gz
|
||||
failNextUpload = false;
|
||||
const okArchiveRes = await archivePost(
|
||||
new NextRequest("http://localhost:3000/api/storage/archive", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", cookie },
|
||||
body: JSON.stringify({ pruneAfterUpload: false, olderThanDays: 30 }),
|
||||
}),
|
||||
);
|
||||
assert.equal(okArchiveRes.status, 200);
|
||||
const okArchiveJson = await okArchiveRes.json();
|
||||
assert.equal(okArchiveJson.archivedCount, 1);
|
||||
assert.equal(okArchiveJson.prunedCount, 0);
|
||||
assert.deepEqual(uploadedKeys, [`/prod-vault/archive-test/v2_alpha/${oldDate}.csv.gz`]);
|
||||
await fs.stat(gzMeta.filePath);
|
||||
|
||||
// 10. V2-E03: Corrupt .csv.gz cleanly surfaces error in /api/data instead of crashing
|
||||
await fs.writeFile(
|
||||
path.join(tmpDir, "v2_alpha", "2024-01-01.csv.gz"),
|
||||
Buffer.from("not-valid-gzip-bytes"),
|
||||
);
|
||||
const corruptDataRes = await dataGet(
|
||||
new NextRequest(
|
||||
"http://localhost:3000/api/data?clientId=v2_alpha&from=2024-01-01&to=2024-01-01",
|
||||
{ headers: { cookie } },
|
||||
),
|
||||
);
|
||||
assert.equal(corruptDataRes.status, 500);
|
||||
const corruptJson = await corruptDataRes.json();
|
||||
assert.match(corruptJson.error || "", /Corrupt|\.gz/i);
|
||||
} finally {
|
||||
await new Promise<void>((resolve) => mockS3.close(() => resolve()));
|
||||
await DataBufferEngine.shutdown();
|
||||
CSVStorageManager.baseStorageDir = origBaseStorageDir;
|
||||
CSVStorageManager.clearRowCountCache();
|
||||
CSVStorageManager.clearQuotaCaches();
|
||||
if (prevStorageDir === undefined) delete process.env.DATASTACK_STORAGE_DIR;
|
||||
else process.env.DATASTACK_STORAGE_DIR = prevStorageDir;
|
||||
if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
|
||||
else process.env.DATABASE_URL = prevDbUrl;
|
||||
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||
resetAdminStateForTests();
|
||||
}
|
||||
});
|
||||
|
||||
test("V2-Phase-01 [Quotas V2-E04 & V2-E05]: HTTP POST /api/events + Port 4000 Socket.IO enforce disk threshold & per-client storageQuotaMb while keeping Admin reads online", async () => {
|
||||
const prevStorageDir = process.env.DATASTACK_STORAGE_DIR;
|
||||
const prevDbUrl = process.env.DATABASE_URL;
|
||||
const prevDisableListen = process.env.DATASTACK_DISABLE_LISTEN;
|
||||
const origBaseStorageDir = CSVStorageManager.baseStorageDir;
|
||||
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "datastack-v2-quota-"));
|
||||
|
||||
delete process.env.DATABASE_URL;
|
||||
process.env.DATASTACK_STORAGE_DIR = tmpDir;
|
||||
CSVStorageManager.baseStorageDir = tmpDir;
|
||||
process.env.DATASTACK_DISABLE_LISTEN = "1";
|
||||
CSVStorageManager.clearQuotaCaches();
|
||||
await DataBufferEngine.shutdown();
|
||||
DataBufferEngine.setRedisClientForTests(null);
|
||||
|
||||
const serverMod = require("../server/index.js") as {
|
||||
server: http.Server;
|
||||
io: { close: (cb: () => void) => void };
|
||||
writer: {
|
||||
baseDir: string;
|
||||
setStatfsOverrideForTests: (
|
||||
fn: ((dir: string) => Promise<{ blocks: number; bavail: number; bsize: number }>) | null,
|
||||
) => void;
|
||||
clearQuotaCaches: () => void;
|
||||
};
|
||||
runtimeConfig: { diskPauseThresholdPct: number };
|
||||
setDbPoolForTests: (pool: unknown) => void;
|
||||
stopTimers: () => void;
|
||||
};
|
||||
const origWriterBaseDir = serverMod.writer.baseDir;
|
||||
serverMod.writer.baseDir = tmpDir;
|
||||
serverMod.writer.clearQuotaCaches();
|
||||
|
||||
const openSockets: Socket[] = [];
|
||||
let listeningUrl = "";
|
||||
|
||||
try {
|
||||
const cookie = await makeAdminCookie();
|
||||
|
||||
// 1. Validate UI helper & API validation for diskPauseThresholdPct (0 or 50..99)
|
||||
assert.equal(validateSingleNumericSetting("diskPauseThresholdPct", "0").valid, true);
|
||||
assert.equal(validateSingleNumericSetting("diskPauseThresholdPct", "95").valid, true);
|
||||
assert.equal(validateSingleNumericSetting("diskPauseThresholdPct", "25").valid, false);
|
||||
const fullDraft = settingsToNumericDrafts(DEFAULT_EDITABLE_SETTINGS);
|
||||
assert.equal(validateEditableSettingsDraft(fullDraft, true).valid, true);
|
||||
|
||||
const badPctRes = await settingsPost(
|
||||
new NextRequest("http://localhost:3000/api/settings", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", cookie },
|
||||
body: JSON.stringify({ diskPauseThresholdPct: 25 }),
|
||||
}),
|
||||
);
|
||||
assert.equal(badPctRes.status, 400);
|
||||
|
||||
await SettingsManager.updateSettingsAsync({ diskPauseThresholdPct: 90 });
|
||||
|
||||
// 2. Register client via /api/clients with 1 MB quota & edit quota via PATCH /api/clients/[id]
|
||||
const createRes = await clientsPost(
|
||||
new NextRequest("http://localhost:3000/api/clients", {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json", cookie },
|
||||
body: JSON.stringify({
|
||||
clientId: "quota_client",
|
||||
name: "Quota Client",
|
||||
rateLimit: 200,
|
||||
storageQuotaMb: 2,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(createRes.status, 201);
|
||||
const createdClient = (await createRes.json()).client;
|
||||
assert.equal(createdClient.storageQuotaMb, 2);
|
||||
|
||||
const patchRes = await patchClientRoute(
|
||||
new NextRequest("http://localhost:3000/api/clients/quota_client", {
|
||||
method: "PATCH",
|
||||
headers: { "content-type": "application/json", cookie },
|
||||
body: JSON.stringify({ storageQuotaMb: 1 }),
|
||||
}),
|
||||
{ params: Promise.resolve({ id: "quota_client" }) },
|
||||
);
|
||||
assert.equal(patchRes.status, 200);
|
||||
assert.equal((await patchRes.json()).client.storageQuotaMb, 1);
|
||||
|
||||
const listRes = await clientsGet(
|
||||
new NextRequest("http://localhost:3000/api/clients", { headers: { cookie } }),
|
||||
);
|
||||
const listed = (await listRes.json()).clients.find(
|
||||
(c: { clientId: string }) => c.clientId === "quota_client",
|
||||
);
|
||||
assert.equal(listed.storageQuotaMb, 1);
|
||||
|
||||
// Seed 1.1 MB into client dir to breach 1 MB quota (V2-E05)
|
||||
const clientDir = path.join(tmpDir, "quota_client");
|
||||
await fs.mkdir(clientDir, { recursive: true });
|
||||
await fs.writeFile(
|
||||
path.join(clientDir, "2026-10-10.csv"),
|
||||
Buffer.alloc(Math.ceil(1.1 * 1024 * 1024), "x"),
|
||||
);
|
||||
CSVStorageManager.clearQuotaCaches("quota_client");
|
||||
|
||||
const httpQuotaRes = await eventsPost(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
authorization: `Bearer ${createdClient.token}`,
|
||||
},
|
||||
body: JSON.stringify({ eventId: "q_evt_1", data: { ok: true } }),
|
||||
}),
|
||||
);
|
||||
assert.equal(httpQuotaRes.status, 507);
|
||||
assert.equal((await httpQuotaRes.json()).code, "STORAGE_QUOTA_EXCEEDED");
|
||||
|
||||
// Now test global VPS disk threshold >= 90% (V2-E04) across HTTP + Socket.IO
|
||||
await patchClientRoute(
|
||||
new NextRequest("http://localhost:3000/api/clients/quota_client", {
|
||||
method: "PATCH",
|
||||
headers: { "content-type": "application/json", cookie },
|
||||
body: JSON.stringify({ storageQuotaMb: 0 }),
|
||||
}),
|
||||
{ params: Promise.resolve({ id: "quota_client" }) },
|
||||
);
|
||||
CSVStorageManager.setStatfsOverrideForTests(async () => ({
|
||||
blocks: 1000,
|
||||
bavail: 50, // 95% used >= 90% threshold
|
||||
bsize: 4096,
|
||||
}));
|
||||
|
||||
const httpDiskRes = await eventsPost(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
authorization: `Bearer ${createdClient.token}`,
|
||||
},
|
||||
body: JSON.stringify({ eventId: "q_evt_2", data: { ok: true } }),
|
||||
}),
|
||||
);
|
||||
assert.equal(httpDiskRes.status, 507);
|
||||
assert.equal((await httpDiskRes.json()).code, "DISK_SPACE_LOW");
|
||||
|
||||
// Admin read/storage/export MUST remain 100% operational while ingestion is paused!
|
||||
const storageRes = await storageGet(
|
||||
new NextRequest("http://localhost:3000/api/storage", { headers: { cookie } }),
|
||||
);
|
||||
assert.equal(storageRes.status, 200);
|
||||
const storageJson = await storageRes.json();
|
||||
assert.equal(storageJson.diskUsage.usedPct, 95);
|
||||
|
||||
// Port 4000 Socket.IO enforcement for DISK_SPACE_LOW & STORAGE_QUOTA_EXCEEDED
|
||||
let mockQuotaMb = 0;
|
||||
serverMod.runtimeConfig.diskPauseThresholdPct = 90;
|
||||
serverMod.setDbPoolForTests({
|
||||
query: async (sql: string) => {
|
||||
if (sql.includes("FROM client_sdks")) {
|
||||
return {
|
||||
rowCount: 1,
|
||||
rows: [
|
||||
{
|
||||
client_id: "quota_client",
|
||||
token: createdClient.token,
|
||||
status: "active",
|
||||
rate_limit: 200,
|
||||
storage_quota_mb: mockQuotaMb,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
return { rowCount: 0, rows: [] };
|
||||
},
|
||||
});
|
||||
serverMod.writer.setStatfsOverrideForTests(async () => ({
|
||||
blocks: 1000,
|
||||
bavail: 50, // 95% used
|
||||
bsize: 4096,
|
||||
}));
|
||||
|
||||
await new Promise<void>((resolve) =>
|
||||
serverMod.server.listen(0, "127.0.0.1", resolve),
|
||||
);
|
||||
const addr = serverMod.server.address() as AddressInfo;
|
||||
listeningUrl = `http://127.0.0.1:${addr.port}`;
|
||||
|
||||
const socket = ioClient(listeningUrl, {
|
||||
auth: { token: createdClient.token },
|
||||
transports: ["websocket"],
|
||||
reconnection: false,
|
||||
timeout: 3000,
|
||||
});
|
||||
openSockets.push(socket);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
socket.on("connect", () => resolve());
|
||||
socket.on("connect_error", reject);
|
||||
});
|
||||
|
||||
const singleDiskAck = await new Promise<{
|
||||
success: boolean;
|
||||
code?: string;
|
||||
}>((resolve, reject) => {
|
||||
socket.timeout(3000).emit("sync_data", { data: { k: 1 } }, (err: unknown, ack: { success: boolean; code?: string }) => {
|
||||
if (err) reject(err);
|
||||
else resolve(ack);
|
||||
});
|
||||
});
|
||||
assert.equal(singleDiskAck.success, false);
|
||||
assert.equal(singleDiskAck.code, "DISK_SPACE_LOW");
|
||||
|
||||
// Relax disk threshold & activate per-client 1 MB quota in Socket.IO worker
|
||||
serverMod.writer.setStatfsOverrideForTests(async () => ({
|
||||
blocks: 1000,
|
||||
bavail: 800, // 20% used
|
||||
bsize: 4096,
|
||||
}));
|
||||
serverMod.writer.clearQuotaCaches();
|
||||
mockQuotaMb = 1;
|
||||
// Wait out 2s verifyClientInDb cache or update query response cleanly
|
||||
await new Promise((r) => setTimeout(r, 2100));
|
||||
|
||||
const batchQuotaAck = await new Promise<{
|
||||
success: boolean;
|
||||
code?: string;
|
||||
rejected?: number;
|
||||
}>((resolve, reject) => {
|
||||
socket.timeout(3000).emit(
|
||||
"batch_sync",
|
||||
[{ eventId: "b1", data: { k: 1 } }],
|
||||
(err: unknown, ack: { success: boolean; code?: string; rejected?: number }) => {
|
||||
if (err) reject(err);
|
||||
else resolve(ack);
|
||||
},
|
||||
);
|
||||
});
|
||||
assert.equal(batchQuotaAck.success, false);
|
||||
assert.equal(batchQuotaAck.code, "STORAGE_QUOTA_EXCEEDED");
|
||||
assert.equal(batchQuotaAck.rejected, 1);
|
||||
|
||||
// Clean delete route check
|
||||
const delRes = await deleteClientRoute(
|
||||
new NextRequest("http://localhost:3000/api/clients/quota_client?purgeFiles=true", {
|
||||
method: "DELETE",
|
||||
headers: { cookie },
|
||||
}),
|
||||
{ params: Promise.resolve({ id: "quota_client" }) },
|
||||
);
|
||||
assert.equal(delRes.status, 200);
|
||||
} finally {
|
||||
for (const s of openSockets) {
|
||||
try {
|
||||
s.disconnect();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
serverMod.writer.setStatfsOverrideForTests(null);
|
||||
serverMod.writer.baseDir = origWriterBaseDir;
|
||||
serverMod.writer.clearQuotaCaches();
|
||||
serverMod.setDbPoolForTests(null);
|
||||
serverMod.stopTimers();
|
||||
await new Promise<void>((resolve) => {
|
||||
serverMod.io.close(() => {
|
||||
if (serverMod.server.listening) {
|
||||
serverMod.server.close(() => resolve());
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
});
|
||||
SDKManager.stopActivityFlushTimerForTests();
|
||||
CSVStorageManager.setStatfsOverrideForTests(null);
|
||||
CSVStorageManager.clearQuotaCaches();
|
||||
await DataBufferEngine.shutdown();
|
||||
CSVStorageManager.baseStorageDir = origBaseStorageDir;
|
||||
if (prevStorageDir === undefined) delete process.env.DATASTACK_STORAGE_DIR;
|
||||
else process.env.DATASTACK_STORAGE_DIR = prevStorageDir;
|
||||
if (prevDisableListen === undefined) delete process.env.DATASTACK_DISABLE_LISTEN;
|
||||
else process.env.DATASTACK_DISABLE_LISTEN = prevDisableListen;
|
||||
if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
|
||||
else process.env.DATABASE_URL = prevDbUrl;
|
||||
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||
resetAdminStateForTests();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,631 @@
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs/promises";
|
||||
import http from "node:http";
|
||||
import { createRequire } from "node:module";
|
||||
import type { AddressInfo } from "node:net";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import { NextRequest } from "next/server";
|
||||
import { io as ioClient, type Socket } from "socket.io-client";
|
||||
import { PATCH as patchClientRoute } from "../app/api/clients/[id]/route";
|
||||
import { GET as clientsGet, POST as clientsPost } from "../app/api/clients/route";
|
||||
import { POST as eventsPost } from "../app/api/events/route";
|
||||
import { GET as exportGet } from "../app/api/export/route";
|
||||
import { GET as prometheusGet } from "../app/api/prometheus/route";
|
||||
import { GET as settingsGet, POST as settingsPost } from "../app/api/settings/route";
|
||||
import { POST as webhookTestPost } from "../app/api/settings/webhook-test/route";
|
||||
import { SettingsManager } from "../core/adminSettings/settingsManager";
|
||||
import {
|
||||
buildWebhookSignature,
|
||||
clearWebhookCooldownCacheForTests,
|
||||
dispatchWebhookAlertAsync,
|
||||
formatOutboundWebhookBody,
|
||||
} from "../core/alerts/webhookNotifier";
|
||||
import {
|
||||
createAdminSessionToken,
|
||||
getAdminSessionSecret,
|
||||
} from "../core/auth/adminSession";
|
||||
import {
|
||||
createInitialAdmin,
|
||||
resetAdminStateForTests,
|
||||
} from "../core/auth/adminUsers";
|
||||
import { DataBufferEngine } from "../core/data/buffer";
|
||||
import { SDKManager } from "../core/sdk/sdkManager";
|
||||
import {
|
||||
isClientIpAllowed,
|
||||
isClientTokenExpired,
|
||||
validateAllowedIpsInput,
|
||||
validateClientExpiresAtInput,
|
||||
} from "../core/Security/ipAllowlist";
|
||||
import { CSVStorageManager } from "../core/Storage/csvStorage";
|
||||
import {
|
||||
DEFAULT_ALERTING_SETTINGS,
|
||||
DEFAULT_EDITABLE_SETTINGS,
|
||||
normalizeAlertingSettings,
|
||||
settingsToNumericDrafts,
|
||||
validateEditableSettingsDraft,
|
||||
validateSingleNumericSetting,
|
||||
} from "../lib/vconUiHelpers";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
async function makeAdminCookie(): Promise<string> {
|
||||
resetAdminStateForTests();
|
||||
await createInitialAdmin("v2-p2@datastack.dev", "V2Phase02Pass!");
|
||||
const secret = getAdminSessionSecret()!;
|
||||
const token = await createAdminSessionToken(
|
||||
"v2-p2@datastack.dev",
|
||||
secret,
|
||||
Date.now() + 3600_000,
|
||||
1,
|
||||
);
|
||||
return `datastack_admin_session=${token}`;
|
||||
}
|
||||
|
||||
test("V2-Phase-02 [IP Allowlist & Token Expiry]: validateAllowedIpsInput, isClientIpAllowed (IPv4/IPv6/CIDR) & isClientTokenExpired", () => {
|
||||
// Valid empty or wildcard
|
||||
assert.deepEqual(validateAllowedIpsInput(""), { valid: true, normalized: "" });
|
||||
assert.deepEqual(validateAllowedIpsInput(" "), { valid: true, normalized: "" });
|
||||
assert.deepEqual(validateAllowedIpsInput("*"), { valid: true, normalized: "*" });
|
||||
|
||||
// Valid comma-separated IPv4, IPv4 CIDR, IPv6, IPv6 CIDR
|
||||
const validList = validateAllowedIpsInput(
|
||||
"203.0.113.10, 10.0.0.0/8, ::ffff:192.0.2.5, 2001:db8::/32, 203.0.113.10",
|
||||
);
|
||||
assert.equal(validList.valid, true);
|
||||
if (validList.valid) {
|
||||
assert.equal(
|
||||
validList.normalized,
|
||||
"203.0.113.10, 10.0.0.0/8, 192.0.2.5, 2001:db8::/32",
|
||||
);
|
||||
}
|
||||
|
||||
// Invalid IP or CIDR prefix rejected
|
||||
const badCidr = validateAllowedIpsInput("10.0.0.0/33");
|
||||
assert.equal(badCidr.valid, false);
|
||||
const badIp = validateAllowedIpsInput("999.888.777.666");
|
||||
assert.equal(badIp.valid, false);
|
||||
|
||||
// Matching behavior
|
||||
assert.equal(isClientIpAllowed("198.51.100.77", ""), true);
|
||||
assert.equal(isClientIpAllowed("198.51.100.77", "*"), true);
|
||||
assert.equal(
|
||||
isClientIpAllowed("203.0.113.10", "203.0.113.10, 10.0.0.0/8"),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
isClientIpAllowed("::ffff:203.0.113.10", "203.0.113.10, 10.0.0.0/8"),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
isClientIpAllowed("10.24.99.1", "203.0.113.10, 10.0.0.0/8"),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
isClientIpAllowed("192.168.1.50", "203.0.113.10, 10.0.0.0/8"),
|
||||
false,
|
||||
);
|
||||
// Loopback equivalence between 127.0.0.1 and ::1
|
||||
assert.equal(isClientIpAllowed("::1", "127.0.0.1"), true);
|
||||
assert.equal(isClientIpAllowed("127.0.0.1", "::1"), true);
|
||||
|
||||
// Expiration validation & evaluation
|
||||
assert.deepEqual(validateClientExpiresAtInput(null), {
|
||||
valid: true,
|
||||
normalized: null,
|
||||
});
|
||||
assert.deepEqual(validateClientExpiresAtInput(""), {
|
||||
valid: true,
|
||||
normalized: null,
|
||||
});
|
||||
const validExp = validateClientExpiresAtInput("2030-06-15T12:00:00.000Z");
|
||||
assert.equal(validExp.valid, true);
|
||||
if (validExp.valid) {
|
||||
assert.equal(validExp.normalized, "2030-06-15T12:00:00.000Z");
|
||||
}
|
||||
const badExp = validateClientExpiresAtInput("not-a-timestamp");
|
||||
assert.equal(badExp.valid, false);
|
||||
|
||||
const nowMs = Date.parse("2026-11-19T12:00:00.000Z");
|
||||
assert.equal(isClientTokenExpired(null, nowMs), false);
|
||||
assert.equal(isClientTokenExpired("2026-11-19T12:00:01.000Z", nowMs), false);
|
||||
assert.equal(isClientTokenExpired("2026-11-19T12:00:00.000Z", nowMs), true);
|
||||
assert.equal(isClientTokenExpired("2026-11-19T11:59:59.000Z", nowMs), true);
|
||||
});
|
||||
|
||||
test("V2-Phase-02 [Webhook Notifier & /api/settings/webhook-test]: HMAC-SHA256 signing, cooldown suppression & V2-E06 error handling", async () => {
|
||||
clearWebhookCooldownCacheForTests();
|
||||
const receivedRequests: Array<{
|
||||
headers: http.IncomingHttpHeaders;
|
||||
body: string;
|
||||
json: Record<string, unknown>;
|
||||
}> = [];
|
||||
let responseStatus = 200;
|
||||
|
||||
const mockWebhookServer = http.createServer((req, res) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on("data", (chunk) => chunks.push(Buffer.from(chunk)));
|
||||
req.on("end", () => {
|
||||
const bodyStr = Buffer.concat(chunks).toString("utf8");
|
||||
let parsed: Record<string, unknown> = {};
|
||||
try {
|
||||
parsed = JSON.parse(bodyStr);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
receivedRequests.push({
|
||||
headers: req.headers,
|
||||
body: bodyStr,
|
||||
json: parsed,
|
||||
});
|
||||
res.writeHead(responseStatus, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify({ ok: responseStatus < 400 }));
|
||||
});
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve) => mockWebhookServer.listen(0, "127.0.0.1", resolve));
|
||||
const port = (mockWebhookServer.address() as AddressInfo).port;
|
||||
const webhookUrl = `http://127.0.0.1:${port}/hooks/datastack`;
|
||||
|
||||
try {
|
||||
// 1. Format & signature verification
|
||||
const formatted = formatOutboundWebhookBody({
|
||||
event: "DISK_SPACE_LOW",
|
||||
severity: "critical",
|
||||
title: "Disk Threshold Reached",
|
||||
message: "VPS disk is at 95.4%.",
|
||||
clientId: "svc_alpha",
|
||||
timestamp: "2026-11-19T12:00:00.000Z",
|
||||
details: { usedPct: 95.4 },
|
||||
});
|
||||
assert.ok(formatted.text.includes("DISK_SPACE_LOW"));
|
||||
assert.equal(formatted.content, formatted.text);
|
||||
|
||||
const bodyJson = JSON.stringify(formatted);
|
||||
const sig = buildWebhookSignature(bodyJson, "webhook_secret_xyz");
|
||||
assert.ok(sig.startsWith("sha256="));
|
||||
|
||||
// 2. Cooldown deduplication test
|
||||
const first = await dispatchWebhookAlertAsync(
|
||||
{
|
||||
enabled: true,
|
||||
url: webhookUrl,
|
||||
secret: "webhook_secret_xyz",
|
||||
cooldownSec: 120,
|
||||
},
|
||||
{
|
||||
event: "CLIENT_QUOTA_EXCEEDED",
|
||||
severity: "warning",
|
||||
title: "Client Quota Exceeded",
|
||||
message: "Quota reached for svc_alpha",
|
||||
clientId: "svc_alpha",
|
||||
},
|
||||
);
|
||||
assert.equal(first.ok, true);
|
||||
assert.equal(receivedRequests.length, 1);
|
||||
assert.equal(
|
||||
receivedRequests[0].headers["x-datastack-event"],
|
||||
"CLIENT_QUOTA_EXCEEDED",
|
||||
);
|
||||
assert.equal(
|
||||
receivedRequests[0].headers["x-datastack-signature"],
|
||||
buildWebhookSignature(receivedRequests[0].body, "webhook_secret_xyz"),
|
||||
);
|
||||
|
||||
// Duplicate event for same client within cooldown window is suppressed
|
||||
const second = await dispatchWebhookAlertAsync(
|
||||
{
|
||||
enabled: true,
|
||||
url: webhookUrl,
|
||||
secret: "webhook_secret_xyz",
|
||||
cooldownSec: 120,
|
||||
},
|
||||
{
|
||||
event: "CLIENT_QUOTA_EXCEEDED",
|
||||
severity: "warning",
|
||||
title: "Client Quota Exceeded",
|
||||
message: "Quota reached again for svc_alpha",
|
||||
clientId: "svc_alpha",
|
||||
},
|
||||
);
|
||||
assert.equal(second.ok, true);
|
||||
assert.equal(second.suppressedByCooldown, true);
|
||||
assert.equal(receivedRequests.length, 1);
|
||||
|
||||
// 3. Test POST /api/settings/webhook-test route (200 & 502 WEBHOOK_DELIVERY_FAILED V2-E06)
|
||||
const adminCookie = await makeAdminCookie();
|
||||
const okTestReq = new NextRequest("http://localhost:3000/api/settings/webhook-test", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
cookie: adminCookie,
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
webhookUrl,
|
||||
webhookSecret: "test_route_hmac_secret",
|
||||
}),
|
||||
});
|
||||
const okTestRes = await webhookTestPost(okTestReq);
|
||||
assert.equal(okTestRes.status, 200);
|
||||
const okTestData = await okTestRes.json();
|
||||
assert.equal(okTestData.success, true);
|
||||
assert.equal(receivedRequests.length, 2);
|
||||
assert.equal(receivedRequests[1].headers["x-datastack-event"], "TEST_ALERT");
|
||||
|
||||
// Simulate upstream 500 error -> 502 WEBHOOK_DELIVERY_FAILED (V2-E06)
|
||||
responseStatus = 500;
|
||||
const failTestReq = new NextRequest("http://localhost:3000/api/settings/webhook-test", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
cookie: adminCookie,
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
webhookUrl,
|
||||
}),
|
||||
});
|
||||
const failTestRes = await webhookTestPost(failTestReq);
|
||||
assert.equal(failTestRes.status, 502);
|
||||
const failTestData = await failTestRes.json();
|
||||
assert.equal(failTestData.code, "WEBHOOK_DELIVERY_FAILED");
|
||||
} finally {
|
||||
await new Promise<void>((resolve) => mockWebhookServer.close(() => resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("V2-Phase-02 [HTTP & Socket Ingestion Security]: Per-client IP allowlist (403 V2-E07) & Token expiration (401 V2-E08)", async () => {
|
||||
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "datastack-v2-p2-sec-"));
|
||||
const prevStorageDir = process.env.DATASTACK_STORAGE_DIR;
|
||||
process.env.DATASTACK_STORAGE_DIR = tmpDir;
|
||||
CSVStorageManager.resetForTests(tmpDir);
|
||||
await CSVStorageManager.initializeAsync();
|
||||
clearWebhookCooldownCacheForTests();
|
||||
|
||||
try {
|
||||
const adminCookie = await makeAdminCookie();
|
||||
|
||||
// 1. Create client with strict IP allowlist (only 10.10.0.0/16) and future expiration
|
||||
const futureIso = new Date(Date.now() + 3600_000).toISOString();
|
||||
const createRes = await clientsPost(
|
||||
new NextRequest("http://localhost:3000/api/clients", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
cookie: adminCookie,
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
name: "IP Restricted Client",
|
||||
rateLimit: 100,
|
||||
storageQuotaMb: 0,
|
||||
allowedIps: "10.10.0.0/16",
|
||||
expiresAt: futureIso,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(createRes.status, 200);
|
||||
const createData = await createRes.json();
|
||||
const client = createData.client;
|
||||
assert.equal(client.allowedIps, "10.10.0.0/16");
|
||||
assert.equal(client.expiresAt, futureIso);
|
||||
|
||||
// 2. Request from forbidden IP 192.168.1.25 -> 403 CLIENT_IP_FORBIDDEN (V2-E07)
|
||||
const forbiddenIpRes = await eventsPost(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
authorization: `Bearer ${client.token}`,
|
||||
"content-type": "application/json",
|
||||
"x-forwarded-for": "192.168.1.25",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
eventId: "evt_ip_denied",
|
||||
payload: { step: 1 },
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(forbiddenIpRes.status, 403);
|
||||
const forbiddenIpJson = await forbiddenIpRes.json();
|
||||
assert.equal(forbiddenIpJson.code, "CLIENT_IP_FORBIDDEN");
|
||||
|
||||
// 3. Request from allowed CIDR IP 10.10.42.9 -> 200 OK
|
||||
const allowedIpRes = await eventsPost(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
authorization: `Bearer ${client.token}`,
|
||||
"content-type": "application/json",
|
||||
"x-forwarded-for": "10.10.42.9",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
eventId: "evt_ip_allowed",
|
||||
payload: { step: 2 },
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(allowedIpRes.status, 200);
|
||||
|
||||
// 4. Patch client with past expiresAt -> 401 CLIENT_TOKEN_EXPIRED (V2-E08)
|
||||
const pastIso = new Date(Date.now() - 60_000).toISOString();
|
||||
const patchRes = await patchClientRoute(
|
||||
new NextRequest(
|
||||
`http://localhost:3000/api/clients/${encodeURIComponent(client.clientId)}`,
|
||||
{
|
||||
method: "PATCH",
|
||||
headers: {
|
||||
cookie: adminCookie,
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
expiresAt: pastIso,
|
||||
}),
|
||||
},
|
||||
),
|
||||
{ params: Promise.resolve({ id: client.clientId }) },
|
||||
);
|
||||
assert.equal(patchRes.status, 200);
|
||||
const patchData = await patchRes.json();
|
||||
assert.equal(patchData.client.expiresAt, pastIso);
|
||||
|
||||
const expiredRes = await eventsPost(
|
||||
new NextRequest("http://localhost:3000/api/events", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
authorization: `Bearer ${client.token}`,
|
||||
"content-type": "application/json",
|
||||
"x-forwarded-for": "10.10.42.9",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
eventId: "evt_expired",
|
||||
payload: { step: 3 },
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(expiredRes.status, 401);
|
||||
const expiredJson = await expiredRes.json();
|
||||
assert.equal(expiredJson.code, "CLIENT_TOKEN_EXPIRED");
|
||||
|
||||
// 5. Verify Port 4000 Socket.IO server enforces IP allowlist & token expiry
|
||||
process.env.DATASTACK_DISABLE_LISTEN = "1";
|
||||
const ingestionApp = require("../server/index.js");
|
||||
|
||||
// Sync expired client to Port 4000 in-memory cache and verify verifyClientInDb returns CLIENT_TOKEN_EXPIRED
|
||||
await new Promise<void>((resolve) =>
|
||||
ingestionApp.server.listen(0, "127.0.0.1", () => resolve()),
|
||||
);
|
||||
const port4000 = (ingestionApp.server.address() as AddressInfo).port;
|
||||
|
||||
try {
|
||||
const expiredCheck = await ingestionApp.verifyClientInDb(
|
||||
client.clientId,
|
||||
client.token,
|
||||
true,
|
||||
"10.10.42.9",
|
||||
);
|
||||
assert.equal(expiredCheck.ok, false);
|
||||
assert.equal(expiredCheck.reason, "CLIENT_TOKEN_EXPIRED");
|
||||
|
||||
// Clear expiration via PATCH, keep IP allowlist 10.10.0.0/16, and check Socket.IO handshake from 127.0.0.1
|
||||
await patchClientRoute(
|
||||
new NextRequest(
|
||||
`http://localhost:3000/api/clients/${encodeURIComponent(client.clientId)}`,
|
||||
{
|
||||
method: "PATCH",
|
||||
headers: {
|
||||
cookie: adminCookie,
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
expiresAt: null,
|
||||
}),
|
||||
},
|
||||
),
|
||||
{ params: Promise.resolve({ id: client.clientId }) },
|
||||
);
|
||||
|
||||
const ipForbiddenCheck = await ingestionApp.verifyClientInDb(
|
||||
client.clientId,
|
||||
client.token,
|
||||
true,
|
||||
"127.0.0.1",
|
||||
);
|
||||
assert.equal(ipForbiddenCheck.ok, false);
|
||||
assert.equal(ipForbiddenCheck.reason, "CLIENT_IP_FORBIDDEN");
|
||||
|
||||
// Socket.IO connection from 127.0.0.1 should fail handshake with CLIENT_IP_FORBIDDEN
|
||||
const socketError = await new Promise<string>((resolve) => {
|
||||
const sock: Socket = ioClient(`http://127.0.0.1:${port4000}`, {
|
||||
auth: { clientId: client.clientId, token: client.token },
|
||||
transports: ["websocket"],
|
||||
reconnection: false,
|
||||
});
|
||||
sock.on("connect_error", (err) => {
|
||||
sock.close();
|
||||
resolve(err.message);
|
||||
});
|
||||
sock.on("connect", () => {
|
||||
sock.close();
|
||||
resolve("CONNECTED_UNEXPECTEDLY");
|
||||
});
|
||||
});
|
||||
assert.ok(socketError.includes("CLIENT_IP_FORBIDDEN"));
|
||||
} finally {
|
||||
await new Promise<void>((resolve) =>
|
||||
ingestionApp.server.close(() => resolve()),
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
await DataBufferEngine.flushAllAsync();
|
||||
SDKManager.stopActivityFlushTimerForTests();
|
||||
if (prevStorageDir === undefined) {
|
||||
delete process.env.DATASTACK_STORAGE_DIR;
|
||||
} else {
|
||||
process.env.DATASTACK_STORAGE_DIR = prevStorageDir;
|
||||
}
|
||||
CSVStorageManager.resetForTests();
|
||||
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("V2-Phase-02 [Prometheus Metrics & NDJSON Streaming Export]: GET /api/prometheus (V2-FEAT-04 / V2-E09) & GET /api/export?format=ndjson (V2-FEAT-06)", async () => {
|
||||
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "datastack-v2-p2-prom-"));
|
||||
const prevStorageDir = process.env.DATASTACK_STORAGE_DIR;
|
||||
process.env.DATASTACK_STORAGE_DIR = tmpDir;
|
||||
CSVStorageManager.resetForTests(tmpDir);
|
||||
await CSVStorageManager.initializeAsync();
|
||||
|
||||
try {
|
||||
const adminCookie = await makeAdminCookie();
|
||||
|
||||
// 1. Unauthenticated scrape when Prometheus is disabled -> 401 PROMETHEUS_UNAUTHORIZED (V2-E09)
|
||||
const unauthPromRes = await prometheusGet(
|
||||
new NextRequest("http://localhost:3000/api/prometheus"),
|
||||
);
|
||||
assert.equal(unauthPromRes.status, 401);
|
||||
const unauthPromJson = await unauthPromRes.json();
|
||||
assert.equal(unauthPromJson.code, "PROMETHEUS_UNAUTHORIZED");
|
||||
|
||||
// 2. Configure Webhook & Prometheus settings via POST /api/settings
|
||||
const saveSettingsRes = await settingsPost(
|
||||
new NextRequest("http://localhost:3000/api/settings", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
cookie: adminCookie,
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
enableWebhookAlerts: true,
|
||||
webhookUrl: "https://hooks.example.com/datastack",
|
||||
webhookSecret: "super_secret_webhook_hmac_9988",
|
||||
webhookCooldownSec: 180,
|
||||
enablePrometheusMetrics: true,
|
||||
prometheusMetricsToken: "prom_scrape_bearer_token_7766",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
assert.equal(saveSettingsRes.status, 200);
|
||||
const savedData = await saveSettingsRes.json();
|
||||
assert.equal(savedData.settings.enableWebhookAlerts, true);
|
||||
assert.equal(savedData.settings.webhookCooldownSec, 180);
|
||||
assert.equal(savedData.settings.hasWebhookSecret, true);
|
||||
assert.equal(savedData.settings.webhookSecretMask, "••••9988");
|
||||
assert.equal("webhookSecretEnc" in savedData.settings, false);
|
||||
assert.equal(savedData.settings.enablePrometheusMetrics, true);
|
||||
assert.equal(savedData.settings.hasPrometheusToken, true);
|
||||
assert.equal(savedData.settings.prometheusTokenMask, "••••7766");
|
||||
assert.equal("prometheusMetricsTokenEnc" in savedData.settings, false);
|
||||
|
||||
// 3. Wrong Bearer token -> 401 PROMETHEUS_UNAUTHORIZED (V2-E09)
|
||||
const wrongTokenRes = await prometheusGet(
|
||||
new NextRequest("http://localhost:3000/api/prometheus", {
|
||||
headers: {
|
||||
authorization: "Bearer wrong_token_value",
|
||||
},
|
||||
}),
|
||||
);
|
||||
assert.equal(wrongTokenRes.status, 401);
|
||||
|
||||
// 4. Valid Bearer token -> 200 Prometheus text exposition format
|
||||
const validTokenRes = await prometheusGet(
|
||||
new NextRequest("http://localhost:3000/api/prometheus", {
|
||||
headers: {
|
||||
authorization: "Bearer prom_scrape_bearer_token_7766",
|
||||
},
|
||||
}),
|
||||
);
|
||||
assert.equal(validTokenRes.status, 200);
|
||||
assert.ok(
|
||||
validTokenRes.headers
|
||||
.get("content-type")
|
||||
?.includes("text/plain; version=0.0.4"),
|
||||
);
|
||||
const metricsBody = await validTokenRes.text();
|
||||
assert.ok(metricsBody.includes("datastack_up 1"));
|
||||
assert.ok(metricsBody.includes("datastack_events_total"));
|
||||
assert.ok(metricsBody.includes("datastack_storage_bytes_total"));
|
||||
assert.ok(metricsBody.includes("datastack_disk_used_percent"));
|
||||
|
||||
// 5. Seed events and export as streaming NDJSON (GET /api/export?format=ndjson) (V2-FEAT-06)
|
||||
await CSVStorageManager.appendBatchAsync("client_ndjson", [
|
||||
{
|
||||
clientId: "client_ndjson",
|
||||
eventId: "evt_nd_1",
|
||||
timestamp: "2026-11-19T10:00:00.000Z",
|
||||
payload: JSON.stringify({ action: "purchase", amount: 49 }),
|
||||
},
|
||||
{
|
||||
clientId: "client_ndjson",
|
||||
eventId: "evt_nd_2",
|
||||
timestamp: "2026-11-19T10:01:00.000Z",
|
||||
payload: JSON.stringify({ action: "refund", amount: 12 }),
|
||||
},
|
||||
]);
|
||||
|
||||
const ndjsonRes = await exportGet(
|
||||
new NextRequest(
|
||||
"http://localhost:3000/api/export?format=ndjson&clientId=client_ndjson",
|
||||
{
|
||||
headers: { cookie: adminCookie },
|
||||
},
|
||||
),
|
||||
);
|
||||
assert.equal(ndjsonRes.status, 200);
|
||||
assert.ok(
|
||||
ndjsonRes.headers
|
||||
.get("content-type")
|
||||
?.includes("application/x-ndjson"),
|
||||
);
|
||||
assert.ok(
|
||||
ndjsonRes.headers
|
||||
.get("content-disposition")
|
||||
?.includes("datastack-client_ndjson.ndjson"),
|
||||
);
|
||||
const ndjsonText = await ndjsonRes.text();
|
||||
const lines = ndjsonText
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line));
|
||||
assert.equal(lines.length, 2);
|
||||
assert.equal(lines[0].clientId, "client_ndjson");
|
||||
assert.equal(lines[0].eventId, "evt_nd_1");
|
||||
assert.deepEqual(lines[0].payload, { action: "purchase", amount: 49 });
|
||||
assert.equal(lines[1].eventId, "evt_nd_2");
|
||||
assert.deepEqual(lines[1].payload, { action: "refund", amount: 12 });
|
||||
|
||||
// 6. Verify UI helper validation for webhookCooldownSec & normalizeAlertingSettings
|
||||
assert.equal(
|
||||
validateSingleNumericSetting("webhookCooldownSec", "5").valid,
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
validateSingleNumericSetting("webhookCooldownSec", "300").valid,
|
||||
true,
|
||||
);
|
||||
const drafts = settingsToNumericDrafts(DEFAULT_EDITABLE_SETTINGS);
|
||||
const validatedDraft = validateEditableSettingsDraft(drafts, true);
|
||||
assert.equal(validatedDraft.valid, true);
|
||||
assert.deepEqual(validatedDraft.values, DEFAULT_EDITABLE_SETTINGS);
|
||||
assert.deepEqual(
|
||||
normalizeAlertingSettings(undefined),
|
||||
DEFAULT_ALERTING_SETTINGS,
|
||||
);
|
||||
} finally {
|
||||
await DataBufferEngine.flushAllAsync();
|
||||
SDKManager.stopActivityFlushTimerForTests();
|
||||
await SettingsManager.updateSettingsAsync({
|
||||
enableWebhookAlerts: false,
|
||||
webhookUrl: "",
|
||||
clearWebhookSecret: true,
|
||||
webhookCooldownSec: 300,
|
||||
enablePrometheusMetrics: false,
|
||||
clearPrometheusToken: true,
|
||||
});
|
||||
if (prevStorageDir === undefined) {
|
||||
delete process.env.DATASTACK_STORAGE_DIR;
|
||||
} else {
|
||||
process.env.DATASTACK_STORAGE_DIR = prevStorageDir;
|
||||
}
|
||||
CSVStorageManager.resetForTests();
|
||||
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
Reference in new issue
Block a user