feat(client): implement per-client storage and security controls

- Added storage quota, IP allowlisting, and expiration logic to client management APIs.
- Updated CSV storage manager to support auto-compression and improved partition file parsing.
- Integrated security checks into event ingestion and data retrieval routes.
This commit is contained in:
Team Dark Devil committed 2026-11-19 23:02:27 +06:00
1 parent 89e4d68690
commit 2873b9904a
32 files changed
+6075 -241

No files matched your search

+36
View File
@@ -17,6 +17,9 @@ export const CORE_SCHEMA_TABLE_DDLS = [
secret TEXT NOT NULL,
token TEXT NOT NULL,
rate_limit INTEGER NOT NULL DEFAULT 100,
storage_quota_mb INTEGER NOT NULL DEFAULT 0,
allowed_ips TEXT NOT NULL DEFAULT '',
expires_at TIMESTAMPTZ,
status TEXT NOT NULL DEFAULT 'active',
total_events BIGINT NOT NULL DEFAULT 0,
last_active TIMESTAMPTZ,
@@ -37,6 +40,21 @@ export const CORE_SCHEMA_TABLE_DDLS = [
jwt_secret TEXT NOT NULL,
enable_idempotency BOOLEAN NOT NULL DEFAULT true,
auto_prune_days INTEGER NOT NULL DEFAULT 30,
compress_after_days INTEGER NOT NULL DEFAULT 0,
enable_s3_backup BOOLEAN NOT NULL DEFAULT false,
s3_endpoint TEXT NOT NULL DEFAULT '',
s3_region TEXT NOT NULL DEFAULT 'auto',
s3_bucket TEXT NOT NULL DEFAULT '',
s3_prefix TEXT NOT NULL DEFAULT 'datastack/',
s3_access_key_id TEXT NOT NULL DEFAULT '',
s3_secret_access_key_enc TEXT NOT NULL DEFAULT '',
disk_pause_threshold_pct INTEGER NOT NULL DEFAULT 95,
enable_webhook_alerts BOOLEAN NOT NULL DEFAULT false,
webhook_url TEXT NOT NULL DEFAULT '',
webhook_secret_enc TEXT NOT NULL DEFAULT '',
webhook_cooldown_sec INTEGER NOT NULL DEFAULT 300,
enable_prometheus_metrics BOOLEAN NOT NULL DEFAULT false,
prometheus_metrics_token_enc TEXT NOT NULL DEFAULT '',
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`,
@@ -71,6 +89,9 @@ export const CORE_SCHEMA_ALTER_DDLS = [
`
ALTER TABLE client_sdks
ADD COLUMN IF NOT EXISTS rate_limit INTEGER NOT NULL DEFAULT 100,
ADD COLUMN IF NOT EXISTS storage_quota_mb INTEGER NOT NULL DEFAULT 0,
ADD COLUMN IF NOT EXISTS allowed_ips TEXT NOT NULL DEFAULT '',
ADD COLUMN IF NOT EXISTS expires_at TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'active',
ADD COLUMN IF NOT EXISTS total_events BIGINT NOT NULL DEFAULT 0,
ADD COLUMN IF NOT EXISTS last_active TIMESTAMPTZ,
@@ -88,6 +109,21 @@ export const CORE_SCHEMA_ALTER_DDLS = [
ADD COLUMN IF NOT EXISTS file_rotation_mb INTEGER NOT NULL DEFAULT 100,
ADD COLUMN IF NOT EXISTS enable_idempotency BOOLEAN NOT NULL DEFAULT true,
ADD COLUMN IF NOT EXISTS auto_prune_days INTEGER NOT NULL DEFAULT 30,
ADD COLUMN IF NOT EXISTS compress_after_days INTEGER NOT NULL DEFAULT 0,
ADD COLUMN IF NOT EXISTS enable_s3_backup BOOLEAN NOT NULL DEFAULT false,
ADD COLUMN IF NOT EXISTS s3_endpoint TEXT NOT NULL DEFAULT '',
ADD COLUMN IF NOT EXISTS s3_region TEXT NOT NULL DEFAULT 'auto',
ADD COLUMN IF NOT EXISTS s3_bucket TEXT NOT NULL DEFAULT '',
ADD COLUMN IF NOT EXISTS s3_prefix TEXT NOT NULL DEFAULT 'datastack/',
ADD COLUMN IF NOT EXISTS s3_access_key_id TEXT NOT NULL DEFAULT '',
ADD COLUMN IF NOT EXISTS s3_secret_access_key_enc TEXT NOT NULL DEFAULT '',
ADD COLUMN IF NOT EXISTS disk_pause_threshold_pct INTEGER NOT NULL DEFAULT 95,
ADD COLUMN IF NOT EXISTS enable_webhook_alerts BOOLEAN NOT NULL DEFAULT false,
ADD COLUMN IF NOT EXISTS webhook_url TEXT NOT NULL DEFAULT '',
ADD COLUMN IF NOT EXISTS webhook_secret_enc TEXT NOT NULL DEFAULT '',
ADD COLUMN IF NOT EXISTS webhook_cooldown_sec INTEGER NOT NULL DEFAULT 300,
ADD COLUMN IF NOT EXISTS enable_prometheus_metrics BOOLEAN NOT NULL DEFAULT false,
ADD COLUMN IF NOT EXISTS prometheus_metrics_token_enc TEXT NOT NULL DEFAULT '',
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
`,
`
+219
View File
@@ -0,0 +1,219 @@
import net from "node:net";
/**
* Per-Client IP / CIDR Allowlist & Token Expiration Validator (V2-Phase-02)
*/
export function stripIpv6MappedPrefix(ip: string): string {
const trimmed = String(ip || "").trim();
if (trimmed.toLowerCase().startsWith("::ffff:")) {
const candidate = trimmed.slice(7);
if (net.isIPv4(candidate)) {
return candidate;
}
}
return trimmed;
}
export function ipv4ToUint32(ip: string): number | null {
const clean = stripIpv6MappedPrefix(ip);
if (!net.isIPv4(clean)) return null;
const parts = clean.split(".").map((octet) => Number(octet));
if (
parts.length !== 4 ||
parts.some((n) => !Number.isInteger(n) || n < 0 || n > 255)
) {
return null;
}
return (
(((parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3]) >>> 0)
);
}
export function parseAllowedIpsList(raw?: string | null): string[] {
if (!raw || typeof raw !== "string") return [];
return raw
.split(/[\s,;]+/)
.map((entry) => entry.trim())
.filter(Boolean);
}
export function validateAllowedIpsInput(
raw: unknown,
): { valid: true; normalized: string } | { valid: false; error: string } {
if (raw === undefined || raw === null || raw === "") {
return { valid: true, normalized: "" };
}
if (typeof raw !== "string") {
return { valid: false, error: "allowedIps must be a string." };
}
if (raw.length > 2048) {
return {
valid: false,
error: "allowedIps cannot exceed 2048 characters.",
};
}
const entries = parseAllowedIpsList(raw);
if (entries.length === 0) {
return { valid: true, normalized: "" };
}
if (entries.length > 64) {
return {
valid: false,
error: "allowedIps cannot contain more than 64 IP/CIDR entries.",
};
}
const normalizedEntries: string[] = [];
for (const entry of entries) {
if (entry === "*") {
normalizedEntries.push("*");
continue;
}
if (entry.includes("/")) {
const [ipPart, prefixPart, ...extra] = entry.split("/");
if (extra.length > 0 || !ipPart || !prefixPart || !/^\d+$/.test(prefixPart)) {
return {
valid: false,
error: `Invalid CIDR entry '${entry}'. Expected format like 203.0.113.0/24.`,
};
}
const prefix = Number(prefixPart);
const cleanIp = stripIpv6MappedPrefix(ipPart);
if (net.isIPv4(cleanIp)) {
if (prefix < 0 || prefix > 32) {
return {
valid: false,
error: `Invalid IPv4 CIDR prefix in '${entry}' (must be 0..32).`,
};
}
normalizedEntries.push(`${cleanIp}/${prefix}`);
continue;
}
if (net.isIPv6(cleanIp)) {
if (prefix < 0 || prefix > 128) {
return {
valid: false,
error: `Invalid IPv6 CIDR prefix in '${entry}' (must be 0..128).`,
};
}
normalizedEntries.push(`${cleanIp.toLowerCase()}/${prefix}`);
continue;
}
return {
valid: false,
error: `Invalid IP address in CIDR entry '${entry}'.`,
};
}
const cleanIp = stripIpv6MappedPrefix(entry);
const ipVersion = net.isIP(cleanIp);
if (ipVersion === 4) {
normalizedEntries.push(cleanIp);
} else if (ipVersion === 6) {
normalizedEntries.push(cleanIp.toLowerCase());
} else {
return {
valid: false,
error: `Invalid IP or CIDR entry '${entry}'.`,
};
}
}
return {
valid: true,
normalized: Array.from(new Set(normalizedEntries)).join(", "),
};
}
export function isClientIpAllowed(
clientIp: string,
allowedIpsRaw?: string | null,
): boolean {
const entries = parseAllowedIpsList(allowedIpsRaw);
if (entries.length === 0) {
return true;
}
const cleanClientIp = stripIpv6MappedPrefix(clientIp);
if (!cleanClientIp || cleanClientIp === "unknown") {
return entries.includes("*");
}
const clientIpv4Int = ipv4ToUint32(cleanClientIp);
const isLoopbackClient =
cleanClientIp === "127.0.0.1" || cleanClientIp.toLowerCase() === "::1";
for (const rule of entries) {
if (rule === "*") return true;
if (rule.includes("/")) {
const [ruleIpRaw, prefixRaw] = rule.split("/");
const ruleIp = stripIpv6MappedPrefix(ruleIpRaw);
const prefix = Number(prefixRaw);
const ruleIpv4Int = ipv4ToUint32(ruleIp);
if (clientIpv4Int !== null && ruleIpv4Int !== null && prefix >= 0 && prefix <= 32) {
if (prefix === 0) return true;
const mask = prefix === 32 ? 0xffffffff : (~((1 << (32 - prefix)) - 1)) >>> 0;
if ((clientIpv4Int & mask) >>> 0 === (ruleIpv4Int & mask) >>> 0) {
return true;
}
}
continue;
}
const cleanRule = stripIpv6MappedPrefix(rule);
if (cleanRule.toLowerCase() === cleanClientIp.toLowerCase()) {
return true;
}
if (
isLoopbackClient &&
(cleanRule === "127.0.0.1" || cleanRule.toLowerCase() === "::1")
) {
return true;
}
}
return false;
}
export function validateClientExpiresAtInput(
raw: unknown,
): { valid: true; iso: string | null } | { valid: false; error: string } {
if (raw === undefined || raw === null || raw === "") {
return { valid: true, iso: null };
}
if (typeof raw !== "string") {
return {
valid: false,
error: "expiresAt must be an ISO timestamp string or null.",
};
}
const trimmed = raw.trim();
if (!trimmed) {
return { valid: true, iso: null };
}
const parsedMs = Date.parse(trimmed);
if (!Number.isFinite(parsedMs)) {
return {
valid: false,
error: "expiresAt must be a valid ISO-8601 date or timestamp.",
};
}
return { valid: true, iso: new Date(parsedMs).toISOString() };
}
export function isClientTokenExpired(
expiresAt?: string | null,
nowMs = Date.now(),
): boolean {
if (!expiresAt || typeof expiresAt !== "string") return false;
const trimmed = expiresAt.trim();
if (!trimmed) return false;
const parsedMs = Date.parse(trimmed);
if (!Number.isFinite(parsedMs)) return false;
return nowMs >= parsedMs;
}
+152
View File
@@ -0,0 +1,152 @@
import {
createCipheriv,
createDecipheriv,
hkdfSync,
randomBytes,
} from "node:crypto";
const DEV_DEFAULT_SESSION_SECRET =
"datastack-admin-session-default-secret-32-chars";
const VAULT_PREFIX = "v1";
const HKDF_SALT = Buffer.from("datastack-vault-salt-v1", "utf8");
const HKDF_INFO = Buffer.from("datastack-aes-256-gcm-v1", "utf8");
export type SecretVaultErrorCode =
| "SECRET_VAULT_KEY_UNAVAILABLE"
| "SECRET_VAULT_DECRYPT_FAILED"
| "SECRET_VAULT_INVALID_INPUT";
export class SecretVaultError extends Error {
public readonly code: SecretVaultErrorCode;
constructor(code: SecretVaultErrorCode, message: string) {
super(`${code}: ${message}`);
this.name = "SecretVaultError";
this.code = code;
}
}
export function resolveVaultMasterSecret(secretOverride?: string): string | null {
const candidate = (
secretOverride ??
process.env.DATASTACK_ADMIN_SESSION_SECRET ??
""
).trim();
if (process.env.NODE_ENV === "production") {
return candidate.length >= 32 ? candidate : null;
}
if (candidate.length > 0) {
return candidate.length >= 32 ? candidate : null;
}
return DEV_DEFAULT_SESSION_SECRET;
}
function deriveAesKey(secretOverride?: string): Buffer {
const master = resolveVaultMasterSecret(secretOverride);
if (!master) {
throw new SecretVaultError(
"SECRET_VAULT_KEY_UNAVAILABLE",
"DATASTACK_ADMIN_SESSION_SECRET (>= 32 characters) is required to encrypt or decrypt stored secrets.",
);
}
const derived = hkdfSync(
"sha256",
Buffer.from(master, "utf8"),
HKDF_SALT,
HKDF_INFO,
32,
);
return Buffer.from(derived);
}
export function encryptSecret(
plaintext: string,
secretOverride?: string,
): string {
if (typeof plaintext !== "string") {
throw new SecretVaultError(
"SECRET_VAULT_INVALID_INPUT",
"Secret value must be a string.",
);
}
if (!plaintext) {
return "";
}
const key = deriveAesKey(secretOverride);
const iv = randomBytes(12);
const cipher = createCipheriv("aes-256-gcm", key, iv);
const encrypted = Buffer.concat([
cipher.update(plaintext, "utf8"),
cipher.final(),
]);
const authTag = cipher.getAuthTag();
return [
VAULT_PREFIX,
iv.toString("base64url"),
authTag.toString("base64url"),
encrypted.toString("base64url"),
].join(":");
}
export function decryptSecret(
ciphertext: string,
secretOverride?: string,
): string {
if (!ciphertext || typeof ciphertext !== "string") {
return "";
}
const parts = ciphertext.trim().split(":");
if (parts.length !== 4 || parts[0] !== VAULT_PREFIX) {
throw new SecretVaultError(
"SECRET_VAULT_DECRYPT_FAILED",
"Stored secret ciphertext has an unrecognized format.",
);
}
const key = deriveAesKey(secretOverride);
try {
const iv = Buffer.from(parts[1], "base64url");
const authTag = Buffer.from(parts[2], "base64url");
const payload = Buffer.from(parts[3], "base64url");
if (iv.length !== 12 || authTag.length !== 16) {
throw new Error("Invalid IV or auth tag length");
}
const decipher = createDecipheriv("aes-256-gcm", key, iv);
decipher.setAuthTag(authTag);
const decrypted = Buffer.concat([
decipher.update(payload),
decipher.final(),
]);
return decrypted.toString("utf8");
} catch (err: unknown) {
if (err instanceof SecretVaultError) {
throw err;
}
throw new SecretVaultError(
"SECRET_VAULT_DECRYPT_FAILED",
"Unable to decrypt stored secret. Verify DATASTACK_ADMIN_SESSION_SECRET matches the key used to encrypt it.",
);
}
}
export function maskSecretHint(plaintext: string): string {
const trimmed = String(plaintext || "").trim();
if (!trimmed) return "";
if (trimmed.length >= 8) {
return `••••${trimmed.slice(-4)}`;
}
return "••••••••";
}
export function getMaskedHintFromEncrypted(
ciphertext: string,
secretOverride?: string,
): string {
if (!ciphertext || !ciphertext.trim()) return "";
try {
const plain = decryptSecret(ciphertext, secretOverride);
return maskSecretHint(plain);
} catch {
return "••••(locked)";
}
}
+444 -59
View File
@@ -1,7 +1,10 @@
import crypto from 'node:crypto';
import fs from 'fs';
import path from 'path';
import { Readable } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import { StringDecoder } from 'node:string_decoder';
import zlib from 'node:zlib';
import { Parser } from 'json2csv';
export interface DataEvent {
@@ -19,8 +22,28 @@ export interface StorageFileMeta {
sizeBytes: number;
linesCount: number;
modifiedAt: Date;
compressed?: boolean;
}
export interface DiskUsageSnapshot {
usedPct: number;
availableBytes: number;
totalBytes: number;
checkedAt: number;
}
export interface StorageAdmissionResult {
allowed: boolean;
code?: 'DISK_SPACE_LOW' | 'STORAGE_QUOTA_EXCEEDED';
reason?: string;
}
export type StatfsProvider = (dir: string) => Promise<{
blocks: number;
bavail: number;
bsize: number;
}>;
interface RowCountCacheEntry {
sizeBytes: number;
mtimeMs: number;
@@ -31,6 +54,7 @@ const LOCK_RETRY_MIN_MS = 5;
const LOCK_RETRY_MAX_MS = 18;
const LOCK_TIMEOUT_MS = 10_000;
const STALE_LOCK_AGE_MS = 30_000;
const QUOTA_CACHE_TTL_MS = 30_000;
function sleepMs(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
@@ -49,11 +73,39 @@ export class CSVStorageManager {
public static baseStorageDir = resolveDefaultStorageDir();
public static maxFileSizeBytes = 100 * 1024 * 1024; // 100MB rotation limit
private static rowCountCache = new Map<string, RowCountCacheEntry>();
private static diskUsageCache: DiskUsageSnapshot | null = null;
private static clientBytesCache = new Map<
string,
{ checkedAt: number; bytes: number }
>();
private static statfsOverrideForTests: StatfsProvider | null = null;
public static isPartitionFilename(filename: string): boolean {
return filename.endsWith('.csv') || filename.endsWith('.csv.gz');
}
public static extractDateFromFilename(filename: string): string {
return filename.replace(/\.csv(?:\.gz)?$/, '').split('-part')[0];
}
public static clearRowCountCache(): void {
this.rowCountCache.clear();
}
public static clearQuotaCaches(clientId?: string): void {
this.diskUsageCache = null;
if (clientId) {
this.clientBytesCache.delete(this.sanitizeClientId(clientId));
} else {
this.clientBytesCache.clear();
}
}
public static setStatfsOverrideForTests(provider: StatfsProvider | null): void {
this.statfsOverrideForTests = provider;
this.diskUsageCache = null;
}
public static sanitizeClientId(clientId: string): string {
const raw = String(clientId || 'unknown').trim().replace(/[^a-zA-Z0-9_-]/g, '_');
return raw || 'unknown';
@@ -299,6 +351,12 @@ export class CSVStorageManager {
private static countCsvDataRowsChunkedSync(filePath: string): number {
const counter = this.createChunkRowCounter();
if (filePath.endsWith('.gz')) {
const compressed = fs.readFileSync(filePath);
const decompressed = zlib.gunzipSync(compressed);
counter.pushChunk(decompressed);
return counter.finish();
}
const buf = Buffer.allocUnsafe(64 * 1024);
const fd = fs.openSync(filePath, 'r');
try {
@@ -312,14 +370,34 @@ export class CSVStorageManager {
}
}
public static openPartitionTextStream(filePath: string): Readable {
const fileStream = fs.createReadStream(filePath, {
highWaterMark: 64 * 1024,
});
if (!filePath.endsWith('.gz')) {
return fileStream;
}
const gunzip = zlib.createGunzip();
fileStream.on('error', (err) => {
if (!gunzip.destroyed) {
gunzip.destroy(err);
}
});
gunzip.on('close', () => {
if (!fileStream.destroyed) {
fileStream.destroy();
}
});
fileStream.pipe(gunzip);
return gunzip;
}
private static async countCsvDataRowsStreamAsync(
filePath: string,
): Promise<number> {
const counter = this.createChunkRowCounter();
return new Promise<number>((resolve, reject) => {
const stream = fs.createReadStream(filePath, {
highWaterMark: 64 * 1024,
});
const stream = this.openPartitionTextStream(filePath);
stream.on('data', (chunk) => counter.pushChunk(chunk));
stream.on('end', () => resolve(counter.finish()));
stream.on('error', (err) => reject(err));
@@ -437,8 +515,9 @@ export class CSVStorageManager {
fs.writeFileSync(resolvedPath, fullCsv, 'utf8');
}
// Invalidate row cache for modified file immediately
// Invalidate row & quota caches for modified file immediately
this.rowCountCache.delete(resolvedPath);
this.clearQuotaCaches(clientId);
return resolvedPath;
});
@@ -449,7 +528,7 @@ export class CSVStorageManager {
}
/**
* List all stored CSV files with cached row counts (invalidated on mtime/size changes)
* List all stored CSV and gzipped CSV (.csv / .csv.gz) files with cached row counts
*/
public static listAllFiles(): StorageFileMeta[] {
const results: StorageFileMeta[] = [];
@@ -473,7 +552,7 @@ export class CSVStorageManager {
const files = fs.readdirSync(fullClientPath);
for (const file of files) {
if (!file.endsWith('.csv')) continue;
if (!this.isPartitionFilename(file)) continue;
const fullFilePath = path.join(fullClientPath, file);
let stats: fs.Stats;
try {
@@ -505,7 +584,7 @@ export class CSVStorageManager {
}
}
const datePart = file.replace('.csv', '').split('-part')[0];
const datePart = this.extractDateFromFilename(file);
results.push({
clientId: clientDir,
@@ -515,6 +594,7 @@ export class CSVStorageManager {
sizeBytes: stats.size,
linesCount,
modifiedAt: stats.mtime,
compressed: file.endsWith('.csv.gz'),
});
}
}
@@ -551,7 +631,7 @@ export class CSVStorageManager {
const files = await fs.promises.readdir(fullClientPath);
for (const file of files) {
if (!file.endsWith('.csv')) continue;
if (!this.isPartitionFilename(file)) continue;
const fullFilePath = path.join(fullClientPath, file);
const stats = await fs.promises.lstat(fullFilePath).catch(() => null);
if (!stats || stats.isSymbolicLink() || !stats.isFile()) continue;
@@ -578,7 +658,7 @@ export class CSVStorageManager {
}
}
const datePart = file.replace('.csv', '').split('-part')[0];
const datePart = this.extractDateFromFilename(file);
results.push({
clientId: clientDir,
@@ -588,6 +668,7 @@ export class CSVStorageManager {
sizeBytes: stats.size,
linesCount,
modifiedAt: stats.mtime,
compressed: file.endsWith('.csv.gz'),
});
}
}
@@ -605,7 +686,121 @@ export class CSVStorageManager {
}
/**
* Prune CSV partitions older than autoPruneDays under cooperative partition locks
* Compress uncompressed .csv partitions older than compressAfterDays into .csv.gz under cooperative partition locks
*/
public static async compressOldPartitionsAsync(
compressAfterDays: number,
): Promise<number> {
if (!Number.isFinite(compressAfterDays) || compressAfterDays <= 0) {
return 0;
}
const baseStat = await fs.promises.lstat(this.baseStorageDir).catch(() => null);
if (!baseStat || !baseStat.isDirectory()) {
return 0;
}
const cutoffDate = new Date(
Date.now() - Math.floor(compressAfterDays) * 86_400_000,
)
.toISOString()
.split('T')[0];
let compressedCount = 0;
try {
const clientDirs = await fs.promises.readdir(this.baseStorageDir);
for (const clientDir of clientDirs) {
const fullClientPath = path.join(this.baseStorageDir, clientDir);
const dirLstat = await fs.promises.lstat(fullClientPath).catch(() => null);
if (!dirLstat || dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) {
continue;
}
const files = await fs.promises
.readdir(fullClientPath)
.catch(() => [] as string[]);
for (const file of files) {
if (!file.endsWith('.csv')) continue;
const datePart = this.extractDateFromFilename(file);
if (!(/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate)) {
continue;
}
const fullPath = path.join(fullClientPath, file);
const fileLstat = await fs.promises.lstat(fullPath).catch(() => null);
if (!fileLstat || fileLstat.isSymbolicLink() || !fileLstat.isFile()) {
continue;
}
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
let acquired = false;
const tryAcquireLock = async (): Promise<boolean> => {
const handle = await fs.promises.open(lockPath, 'wx');
try {
await handle.writeFile(lockValue, 'utf8');
} finally {
await handle.close();
}
return true;
};
try {
acquired = await tryAcquireLock();
} catch (err: unknown) {
if ((err as NodeJS.ErrnoException)?.code === 'EEXIST') {
await this.removeStaleLockAtomicallyAsync(lockPath);
try {
acquired = await tryAcquireLock();
} catch {
acquired = false;
}
}
}
if (!acquired) continue;
let targetGzPath = `${fullPath}.gz`;
if (fs.existsSync(targetGzPath)) {
const stem = fullPath.replace(/\.csv$/, '');
let suffix = 2;
while (fs.existsSync(`${stem}-gz${suffix}.csv.gz`)) {
suffix += 1;
}
targetGzPath = `${stem}-gz${suffix}.csv.gz`;
}
const tmpGzPath = `${targetGzPath}.tmp.${crypto.randomUUID()}`;
try {
await pipeline(
fs.createReadStream(fullPath, { highWaterMark: 64 * 1024 }),
zlib.createGzip({ level: 6 }),
fs.createWriteStream(tmpGzPath),
);
await fs.promises
.utimes(tmpGzPath, fileLstat.atime, fileLstat.mtime)
.catch(() => undefined);
await fs.promises.rename(tmpGzPath, targetGzPath);
await fs.promises.unlink(fullPath);
this.rowCountCache.delete(fullPath);
this.rowCountCache.delete(targetGzPath);
this.clearQuotaCaches(clientDir);
compressedCount += 1;
} catch {
await fs.promises.unlink(tmpGzPath).catch(() => undefined);
} finally {
await this.releaseLockIfOwnedAsync(lockPath, lockValue);
}
}
}
} catch {
// Ignore directory scan failure
}
return compressedCount;
}
/**
* Prune CSV / CSV.GZ partitions older than autoPruneDays under cooperative partition locks
*/
public static pruneExpiredPartitions(autoPruneDays: number): number {
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
@@ -636,8 +831,8 @@ export class CSVStorageManager {
const files = fs.readdirSync(fullClientPath);
for (const file of files) {
if (!file.endsWith('.csv')) continue;
const datePart = file.replace('.csv', '').split('-part')[0];
if (!this.isPartitionFilename(file)) continue;
const datePart = this.extractDateFromFilename(file);
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
const fullPath = path.join(fullClientPath, file);
let fileLstat: fs.Stats;
@@ -683,6 +878,7 @@ export class CSVStorageManager {
if (fs.existsSync(fullPath)) {
fs.unlinkSync(fullPath);
this.rowCountCache.delete(fullPath);
this.clearQuotaCaches(clientDir);
prunedCount += 1;
}
} catch {
@@ -701,27 +897,54 @@ export class CSVStorageManager {
}
/**
* Non-blocking async partition pruner using fs.promises under cooperative partition locks
* Archive and/or prune expired partitions under cooperative partition locks.
* Never deletes a local partition if options.beforeDeleteFile throws or rejects (V2-E02).
*/
public static async pruneExpiredPartitionsAsync(
public static async pruneOrArchiveExpiredPartitionsAsync(
autoPruneDays: number,
): Promise<number> {
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
return 0;
options?: {
beforeDeleteFile?: (file: {
clientId: string;
filename: string;
filePath: string;
date: string;
}) => Promise<void>;
archiveOnlyWithoutDelete?: boolean;
maxCutoffDays?: number;
},
): Promise<{
scannedCount: number;
archivedCount: number;
prunedCount: number;
errors: string[];
}> {
const targetDays =
options?.maxCutoffDays !== undefined
? options.maxCutoffDays
: autoPruneDays;
if (!Number.isFinite(targetDays) || targetDays < 0) {
return { scannedCount: 0, archivedCount: 0, prunedCount: 0, errors: [] };
}
if (options?.maxCutoffDays === undefined && targetDays <= 0) {
return { scannedCount: 0, archivedCount: 0, prunedCount: 0, errors: [] };
}
const baseStat = await fs.promises.lstat(this.baseStorageDir).catch(() => null);
if (!baseStat || !baseStat.isDirectory()) {
return 0;
return { scannedCount: 0, archivedCount: 0, prunedCount: 0, errors: [] };
}
const cutoffDate = new Date(
Date.now() - Math.floor(autoPruneDays) * 86_400_000,
Date.now() - Math.floor(targetDays) * 86_400_000,
)
.toISOString()
.split('T')[0];
let scannedCount = 0;
let archivedCount = 0;
let prunedCount = 0;
const errors: string[] = [];
try {
const clientDirs = await fs.promises.readdir(this.baseStorageDir);
for (const clientDir of clientDirs) {
@@ -731,54 +954,71 @@ export class CSVStorageManager {
continue;
}
const files = await fs.promises.readdir(fullClientPath).catch(() => [] as string[]);
const files = await fs.promises
.readdir(fullClientPath)
.catch(() => [] as string[]);
for (const file of files) {
if (!file.endsWith('.csv')) continue;
const datePart = file.replace('.csv', '').split('-part')[0];
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
const fullPath = path.join(fullClientPath, file);
const fileLstat = await fs.promises.lstat(fullPath).catch(() => null);
if (!fileLstat || fileLstat.isSymbolicLink() || !fileLstat.isFile()) {
continue;
}
if (!this.isPartitionFilename(file)) continue;
const datePart = this.extractDateFromFilename(file);
if (!(/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate)) {
continue;
}
const fullPath = path.join(fullClientPath, file);
const fileLstat = await fs.promises.lstat(fullPath).catch(() => null);
if (!fileLstat || fileLstat.isSymbolicLink() || !fileLstat.isFile()) {
continue;
}
scannedCount += 1;
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
let acquired = false;
const tryAcquireLock = async (): Promise<boolean> => {
const handle = await fs.promises.open(lockPath, 'wx');
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
let acquired = false;
const tryAcquireLock = async (): Promise<boolean> => {
const handle = await fs.promises.open(lockPath, 'wx');
try {
await handle.writeFile(lockValue, 'utf8');
} finally {
await handle.close();
}
return true;
};
try {
acquired = await tryAcquireLock();
} catch (err: unknown) {
if ((err as NodeJS.ErrnoException)?.code === 'EEXIST') {
await this.removeStaleLockAtomicallyAsync(lockPath);
try {
await handle.writeFile(lockValue, 'utf8');
} finally {
await handle.close();
}
return true;
};
try {
acquired = await tryAcquireLock();
} catch (err: unknown) {
if ((err as NodeJS.ErrnoException)?.code === 'EEXIST') {
await this.removeStaleLockAtomicallyAsync(lockPath);
try {
acquired = await tryAcquireLock();
} catch {
acquired = false;
}
acquired = await tryAcquireLock();
} catch {
acquired = false;
}
}
}
if (!acquired) continue;
if (!acquired) continue;
try {
try {
if (options?.beforeDeleteFile) {
await options.beforeDeleteFile({
clientId: clientDir,
filename: file,
filePath: fullPath,
date: datePart,
});
archivedCount += 1;
}
if (!options?.archiveOnlyWithoutDelete) {
await fs.promises.unlink(fullPath);
this.rowCountCache.delete(fullPath);
this.clearQuotaCaches(clientDir);
prunedCount += 1;
} catch {
// Ignore individual unlink error or concurrent removal
} finally {
await this.releaseLockIfOwnedAsync(lockPath, lockValue);
}
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : 'Archive/prune error';
errors.push(`${clientDir}/${file}: ${msg}`);
} finally {
await this.releaseLockIfOwnedAsync(lockPath, lockValue);
}
}
}
@@ -786,7 +1026,149 @@ export class CSVStorageManager {
// Ignore directory scan failure
}
return prunedCount;
return { scannedCount, archivedCount, prunedCount, errors };
}
/**
* Non-blocking async partition pruner using fs.promises under cooperative partition locks
*/
public static async pruneExpiredPartitionsAsync(
autoPruneDays: number,
): Promise<number> {
const res = await this.pruneOrArchiveExpiredPartitionsAsync(autoPruneDays);
return res.prunedCount;
}
/**
* Cached VPS filesystem disk usage snapshot (30s TTL) for disk-full protection
*/
public static async getDiskUsageAsync(): Promise<DiskUsageSnapshot> {
const now = Date.now();
if (
this.diskUsageCache &&
now - this.diskUsageCache.checkedAt < QUOTA_CACHE_TTL_MS
) {
return this.diskUsageCache;
}
let probeDir = path.resolve(this.baseStorageDir);
while (!fs.existsSync(probeDir)) {
const parent = path.dirname(probeDir);
if (parent === probeDir) break;
probeDir = parent;
}
try {
const statfsRes = this.statfsOverrideForTests
? await this.statfsOverrideForTests(probeDir)
: await fs.promises.statfs(probeDir);
const bsize = Math.max(1, Number(statfsRes.bsize) || 4096);
const totalBytes = Math.max(0, Number(statfsRes.blocks) * bsize);
const availableBytes = Math.max(0, Number(statfsRes.bavail) * bsize);
const usedBytes = Math.max(0, totalBytes - availableBytes);
const usedPct =
totalBytes > 0
? Number(
Math.min(100, Math.max(0, (usedBytes / totalBytes) * 100)).toFixed(2),
)
: 0;
const snapshot: DiskUsageSnapshot = {
usedPct,
availableBytes,
totalBytes,
checkedAt: now,
};
this.diskUsageCache = snapshot;
return snapshot;
} catch {
const fallback: DiskUsageSnapshot = {
usedPct: 0,
availableBytes: 0,
totalBytes: 0,
checkedAt: now,
};
this.diskUsageCache = fallback;
return fallback;
}
}
/**
* Cached per-client directory byte count (30s TTL)
*/
public static async getClientStorageBytesAsync(
clientId: string,
): Promise<number> {
const safeClientId = this.sanitizeClientId(clientId);
const now = Date.now();
const cached = this.clientBytesCache.get(safeClientId);
if (cached && now - cached.checkedAt < QUOTA_CACHE_TTL_MS) {
return cached.bytes;
}
const resolvedBase = path.resolve(this.baseStorageDir);
const clientDir = path.resolve(resolvedBase, safeClientId);
if (!clientDir.startsWith(resolvedBase + path.sep)) {
return 0;
}
let totalBytes = 0;
try {
const dirStat = await fs.promises.lstat(clientDir).catch(() => null);
if (dirStat && !dirStat.isSymbolicLink() && dirStat.isDirectory()) {
const entries = await fs.promises
.readdir(clientDir)
.catch(() => [] as string[]);
for (const entry of entries) {
if (!this.isPartitionFilename(entry)) continue;
const filePath = path.join(clientDir, entry);
const stats = await fs.promises.lstat(filePath).catch(() => null);
if (stats && !stats.isSymbolicLink() && stats.isFile()) {
totalBytes += stats.size;
}
}
}
} catch {
// Fall back to 0 on unreadable directory
}
this.clientBytesCache.set(safeClientId, { checkedAt: now, bytes: totalBytes });
return totalBytes;
}
/**
* Checks global VPS disk threshold and per-client MB quota before admitting new events
*/
public static async checkStorageAdmissionAsync(
clientId: string,
clientQuotaMb: number,
diskPauseThresholdPct: number,
): Promise<StorageAdmissionResult> {
const threshold = Number(diskPauseThresholdPct);
if (Number.isFinite(threshold) && threshold >= 50 && threshold <= 99) {
const disk = await this.getDiskUsageAsync();
if (disk.totalBytes > 0 && disk.usedPct >= threshold) {
return {
allowed: false,
code: 'DISK_SPACE_LOW',
reason: `DISK_SPACE_LOW: Disk utilization (${disk.usedPct.toFixed(1)}%) reached safety pause threshold (${threshold}%).`,
};
}
}
const quotaMb = Number(clientQuotaMb);
if (Number.isFinite(quotaMb) && quotaMb > 0) {
const currentBytes = await this.getClientStorageBytesAsync(clientId);
const quotaBytes = Math.floor(quotaMb) * 1024 * 1024;
if (currentBytes >= quotaBytes) {
return {
allowed: false,
code: 'STORAGE_QUOTA_EXCEEDED',
reason: `STORAGE_QUOTA_EXCEEDED: Client '${clientId}' used ${(currentBytes / (1024 * 1024)).toFixed(2)} MB of ${quotaMb} MB quota.`,
};
}
}
return { allowed: true };
}
public static formatStorageStats(files: StorageFileMeta[]) {
@@ -826,12 +1208,12 @@ export class CSVStorageManager {
}
/**
* Reads file content for export / download with path traversal protection
* Reads file content for export / download with path traversal protection (supports .csv & .csv.gz)
*/
public static getFileContent(clientId: string, filename: string): string | null {
const safeClientId = this.sanitizeClientId(clientId);
const safeFilename = path.basename(String(filename || ''));
if (!safeClientId || !safeFilename || !safeFilename.endsWith('.csv')) {
if (!safeClientId || !safeFilename || !this.isPartitionFilename(safeFilename)) {
return null;
}
const resolvedBase = path.resolve(this.baseStorageDir);
@@ -844,6 +1226,9 @@ export class CSVStorageManager {
if (stats.isSymbolicLink() || !stats.isFile()) {
return null;
}
if (safeFilename.endsWith('.gz')) {
return zlib.gunzipSync(fs.readFileSync(fullPath)).toString('utf8');
}
return fs.readFileSync(fullPath, 'utf8');
}
return null;
+335
View File
@@ -0,0 +1,335 @@
import crypto from "node:crypto";
import fs from "node:fs";
export interface S3ArchiveConfig {
endpoint: string;
region: string;
bucket: string;
prefix: string;
accessKeyId: string;
secretAccessKey: string;
}
export interface S3OperationResult {
ok: boolean;
status?: number;
endpoint?: string;
bucket?: string;
key?: string;
etag?: string;
code?: "S3_INVALID_CONFIG" | "S3_CONNECTION_FAILED" | "S3_ARCHIVE_FAILED";
error?: string;
}
function sha256Hex(data: Buffer | string): string {
return crypto.createHash("sha256").update(data).digest("hex");
}
function hmacSha256(key: Buffer | string, data: string): Buffer {
return crypto.createHmac("sha256", key).update(data, "utf8").digest();
}
function encodeRfc3986(value: string): string {
return encodeURIComponent(value).replace(
/[!'()*]/g,
(ch) => `%${ch.charCodeAt(0).toString(16).toUpperCase()}`,
);
}
function encodeS3Path(rawPath: string): string {
return rawPath
.split("/")
.map((segment) => encodeRfc3986(segment))
.join("/");
}
export function buildS3ObjectKey(
prefix: string,
clientId: string,
filename: string,
): string {
const cleanPrefix = String(prefix || "")
.trim()
.replace(/^\/+|\/+$/g, "");
const cleanClient = String(clientId || "unknown")
.trim()
.replace(/^\/+|\/+$/g, "");
const cleanFile = String(filename || "")
.trim()
.replace(/^\/+/, "");
return [cleanPrefix, cleanClient, cleanFile].filter(Boolean).join("/");
}
export function normalizeS3EndpointUrl(
endpoint: string,
region: string,
): { url: URL; region: string } {
const trimmedRegion = String(region || "").trim() || "auto";
const trimmedEndpoint = String(endpoint || "").trim().replace(/\/+$/, "");
if (trimmedEndpoint) {
const withProtocol = /^https?:\/\//i.test(trimmedEndpoint)
? trimmedEndpoint
: `https://${trimmedEndpoint}`;
const parsed = new URL(withProtocol);
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
throw new Error("S3 endpoint must use http:// or https://");
}
return { url: parsed, region: trimmedRegion };
}
if (trimmedRegion === "auto") {
throw new Error(
"S3 Endpoint URL is required when Region is set to 'auto' (e.g. Cloudflare R2 or MinIO).",
);
}
return {
url: new URL(`https://s3.${trimmedRegion}.amazonaws.com`),
region: trimmedRegion,
};
}
export function signS3Request(params: {
method: "GET" | "HEAD" | "PUT";
endpointUrl: URL;
bucket: string;
objectKey?: string;
queryParams?: Record<string, string>;
region: string;
accessKeyId: string;
secretAccessKey: string;
payloadHashHex: string;
contentType?: string;
now?: Date;
}): { targetUrl: string; headers: Record<string, string> } {
const now = params.now ?? new Date();
const amzDate = now.toISOString().replace(/[:-]|\.\d{3}/g, "");
const dateStamp = amzDate.slice(0, 8);
const basePath = params.endpointUrl.pathname.replace(/\/+$/, "");
const bucketPart = encodeRfc3986(params.bucket.trim());
const keyPart = params.objectKey ? `/${encodeS3Path(params.objectKey)}` : "";
const canonicalUri = `${basePath}/${bucketPart}${keyPart}`.replace(
/\/{2,}/g,
"/",
);
const queryEntries = Object.entries(params.queryParams ?? {}).sort(([a], [b]) =>
a < b ? -1 : a > b ? 1 : 0,
);
const canonicalQueryString = queryEntries
.map(([k, v]) => `${encodeRfc3986(k)}=${encodeRfc3986(v)}`)
.join("&");
const hostHeader = params.endpointUrl.host;
const headerMap: Record<string, string> = {
host: hostHeader,
"x-amz-content-sha256": params.payloadHashHex,
"x-amz-date": amzDate,
};
if (params.contentType) {
headerMap["content-type"] = params.contentType;
}
const sortedHeaderKeys = Object.keys(headerMap).sort();
const canonicalHeaders = sortedHeaderKeys
.map((key) => `${key}:${headerMap[key].trim()}\n`)
.join("");
const signedHeaders = sortedHeaderKeys.join(";");
const canonicalRequest = [
params.method,
canonicalUri,
canonicalQueryString,
canonicalHeaders,
signedHeaders,
params.payloadHashHex,
].join("\n");
const credentialScope = `${dateStamp}/${params.region}/s3/aws4_request`;
const stringToSign = [
"AWS4-HMAC-SHA256",
amzDate,
credentialScope,
sha256Hex(canonicalRequest),
].join("\n");
const kDate = hmacSha256(`AWS4${params.secretAccessKey}`, dateStamp);
const kRegion = hmacSha256(kDate, params.region);
const kService = hmacSha256(kRegion, "s3");
const kSigning = hmacSha256(kService, "aws4_request");
const signature = crypto
.createHmac("sha256", kSigning)
.update(stringToSign, "utf8")
.digest("hex");
const authorization = `AWS4-HMAC-SHA256 Credential=${params.accessKeyId}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`;
const targetUrl = `${params.endpointUrl.origin}${canonicalUri}${
canonicalQueryString ? `?${canonicalQueryString}` : ""
}`;
return {
targetUrl,
headers: {
"x-amz-date": amzDate,
"x-amz-content-sha256": params.payloadHashHex,
...(params.contentType ? { "content-type": params.contentType } : {}),
Authorization: authorization,
},
};
}
function validateS3Config(config: S3ArchiveConfig): string | null {
if (!config.bucket?.trim()) return "S3 Bucket name is required.";
if (!config.accessKeyId?.trim()) return "S3 Access Key ID is required.";
if (!config.secretAccessKey?.trim()) return "S3 Secret Access Key is required.";
return null;
}
export async function testS3Connection(
config: S3ArchiveConfig,
): Promise<S3OperationResult> {
const validationError = validateS3Config(config);
if (validationError) {
return {
ok: false,
code: "S3_INVALID_CONFIG",
error: validationError,
};
}
try {
const { url, region } = normalizeS3EndpointUrl(
config.endpoint,
config.region,
);
const signed = signS3Request({
method: "GET",
endpointUrl: url,
bucket: config.bucket.trim(),
queryParams: { "list-type": "2", "max-keys": "1" },
region,
accessKeyId: config.accessKeyId.trim(),
secretAccessKey: config.secretAccessKey,
payloadHashHex: sha256Hex(""),
});
const response = await fetch(signed.targetUrl, {
method: "GET",
headers: signed.headers,
cache: "no-store",
signal: AbortSignal.timeout(8000),
});
if (!response.ok) {
const bodyPreview = await response.text().catch(() => "");
return {
ok: false,
status: response.status,
endpoint: url.origin,
bucket: config.bucket.trim(),
code: "S3_CONNECTION_FAILED",
error: `S3 endpoint returned HTTP ${response.status}${
bodyPreview ? `: ${bodyPreview.slice(0, 180)}` : ""
}`,
};
}
return {
ok: true,
status: response.status,
endpoint: url.origin,
bucket: config.bucket.trim(),
};
} catch (err: unknown) {
return {
ok: false,
code: "S3_CONNECTION_FAILED",
error:
err instanceof Error ? err.message : "Unable to reach S3 endpoint",
};
}
}
export async function uploadFileToS3(
config: S3ArchiveConfig,
localFilePath: string,
objectKey: string,
): Promise<S3OperationResult> {
const validationError = validateS3Config(config);
if (validationError) {
return {
ok: false,
code: "S3_INVALID_CONFIG",
error: validationError,
key: objectKey,
};
}
try {
const { url, region } = normalizeS3EndpointUrl(
config.endpoint,
config.region,
);
const bodyBuffer = await fs.promises.readFile(localFilePath);
const payloadHashHex = sha256Hex(bodyBuffer);
const contentType = localFilePath.endsWith(".gz")
? "application/gzip"
: "text/csv; charset=utf-8";
const signed = signS3Request({
method: "PUT",
endpointUrl: url,
bucket: config.bucket.trim(),
objectKey,
region,
accessKeyId: config.accessKeyId.trim(),
secretAccessKey: config.secretAccessKey,
payloadHashHex,
contentType,
});
const response = await fetch(signed.targetUrl, {
method: "PUT",
headers: {
...signed.headers,
"content-length": String(bodyBuffer.byteLength),
},
body: new Uint8Array(bodyBuffer),
cache: "no-store",
signal: AbortSignal.timeout(30000),
});
if (!response.ok) {
const bodyPreview = await response.text().catch(() => "");
return {
ok: false,
status: response.status,
endpoint: url.origin,
bucket: config.bucket.trim(),
key: objectKey,
code: "S3_ARCHIVE_FAILED",
error: `S3 upload failed with HTTP ${response.status}${
bodyPreview ? `: ${bodyPreview.slice(0, 180)}` : ""
}`,
};
}
return {
ok: true,
status: response.status,
endpoint: url.origin,
bucket: config.bucket.trim(),
key: objectKey,
etag: response.headers.get("etag") ?? undefined,
};
} catch (err: unknown) {
return {
ok: false,
key: objectKey,
code: "S3_ARCHIVE_FAILED",
error: err instanceof Error ? err.message : "S3 upload request failed",
};
}
}
+443 -55
View File
@@ -1,12 +1,27 @@
import {
dispatchWebhookAlertAsync,
type WebhookAlertPayload,
type WebhookRuntimeConfig,
} from "../alerts/webhookNotifier";
import { AuthService, resolveDefaultJwtSecret } from "../auth/jwt";
import { DataBufferEngine } from "../data/buffer";
import { DatabaseEngine } from "../Database/database";
import { SDKManager } from "../sdk/sdkManager";
import {
decryptSecret,
encryptSecret,
getMaskedHintFromEncrypted,
} from "../Security/secretVault";
import {
SecurityEngine,
getInternalIngestionHeaders,
} from "../Security/security";
import { CSVStorageManager } from "../Storage/csvStorage";
import {
buildS3ObjectKey,
type S3ArchiveConfig,
uploadFileToS3,
} from "../Storage/s3Archive";
export interface AdminSettings {
serverPort: number;
@@ -19,9 +34,49 @@ export interface AdminSettings {
jwtSecret: string;
enableIdempotency: boolean;
autoPruneDays: number;
compressAfterDays: number;
enableS3Backup: boolean;
s3Endpoint: string;
s3Region: string;
s3Bucket: string;
s3Prefix: string;
s3AccessKeyId: string;
s3SecretAccessKeyEnc: string;
diskPauseThresholdPct: number;
enableWebhookAlerts: boolean;
webhookUrl: string;
webhookSecretEnc: string;
webhookCooldownSec: number;
enablePrometheusMetrics: boolean;
prometheusMetricsTokenEnc: string;
updatedAt: string;
}
export interface PublicAdminSettings
extends Omit<
AdminSettings,
| "jwtSecret"
| "s3SecretAccessKeyEnc"
| "webhookSecretEnc"
| "prometheusMetricsTokenEnc"
> {
hasS3SecretKey: boolean;
s3SecretKeyMask: string;
hasWebhookSecret: boolean;
webhookSecretMask: string;
hasPrometheusToken: boolean;
prometheusTokenMask: string;
}
export type AdminSettingsUpdateInput = Partial<AdminSettings> & {
s3SecretAccessKey?: string;
clearS3SecretKey?: boolean;
webhookSecret?: string;
clearWebhookSecret?: boolean;
prometheusMetricsToken?: string;
clearPrometheusToken?: boolean;
};
export interface SettingsBridgeSyncStatus {
attempted: boolean;
ok: boolean;
@@ -36,9 +91,11 @@ type SettingsGlobals = typeof globalThis & {
__dataStackSettingsHydrated?: boolean;
__dataStackLastSettingsBridgeSync?: SettingsBridgeSyncStatus;
__dataStackLastPruneMeta?: { days: number; ranAtMs: number };
__dataStackLastCompressMeta?: { days: number; ranAtMs: number };
};
const PRUNE_COOLDOWN_MS = 60 * 60 * 1000;
const COMPRESS_COOLDOWN_MS = 60 * 60 * 1000;
const globals = globalThis as SettingsGlobals;
function getDefaultSettings(): AdminSettings {
@@ -53,6 +110,21 @@ function getDefaultSettings(): AdminSettings {
jwtSecret: resolveDefaultJwtSecret(),
enableIdempotency: true,
autoPruneDays: 30,
compressAfterDays: 0,
enableS3Backup: false,
s3Endpoint: "",
s3Region: "auto",
s3Bucket: "",
s3Prefix: "datastack/",
s3AccessKeyId: "",
s3SecretAccessKeyEnc: "",
diskPauseThresholdPct: 95,
enableWebhookAlerts: false,
webhookUrl: "",
webhookSecretEnc: "",
webhookCooldownSec: 300,
enablePrometheusMetrics: false,
prometheusMetricsTokenEnc: "",
updatedAt: new Date().toISOString(),
};
}
@@ -64,8 +136,132 @@ function getSettingsState(): AdminSettings {
return globals.__dataStackAdminSettings;
}
function toPublicSettings(settings: AdminSettings): PublicAdminSettings {
const s3SecretKeyMask = getMaskedHintFromEncrypted(
settings.s3SecretAccessKeyEnc,
);
const webhookSecretMask = getMaskedHintFromEncrypted(
settings.webhookSecretEnc,
);
const prometheusTokenMask = getMaskedHintFromEncrypted(
settings.prometheusMetricsTokenEnc,
);
return {
serverPort: settings.serverPort,
webPort: settings.webPort,
maxEventsPerSec: settings.maxEventsPerSec,
maxPayloadSizeKb: settings.maxPayloadSizeKb,
batchFlushThreshold: settings.batchFlushThreshold,
batchFlushIntervalMs: settings.batchFlushIntervalMs,
fileRotationMb: settings.fileRotationMb,
enableIdempotency: settings.enableIdempotency,
autoPruneDays: settings.autoPruneDays,
compressAfterDays: settings.compressAfterDays,
enableS3Backup: settings.enableS3Backup,
s3Endpoint: settings.s3Endpoint,
s3Region: settings.s3Region,
s3Bucket: settings.s3Bucket,
s3Prefix: settings.s3Prefix,
s3AccessKeyId: settings.s3AccessKeyId,
hasS3SecretKey: Boolean(settings.s3SecretAccessKeyEnc),
s3SecretKeyMask,
diskPauseThresholdPct: settings.diskPauseThresholdPct,
enableWebhookAlerts: settings.enableWebhookAlerts,
webhookUrl: settings.webhookUrl,
hasWebhookSecret: Boolean(settings.webhookSecretEnc),
webhookSecretMask,
webhookCooldownSec: settings.webhookCooldownSec,
enablePrometheusMetrics: settings.enablePrometheusMetrics,
hasPrometheusToken: Boolean(settings.prometheusMetricsTokenEnc),
prometheusTokenMask,
updatedAt: settings.updatedAt,
};
}
export class SettingsManager {
private static applyRuntimeSettings(settings: AdminSettings, forcePrune = false) {
public static triggerWebhookAlert(payload: WebhookAlertPayload): void {
const current = getSettingsState();
if (!current.enableWebhookAlerts || !current.webhookUrl.trim()) {
return;
}
void (async () => {
try {
let secret = "";
if (current.webhookSecretEnc) {
try {
secret = decryptSecret(current.webhookSecretEnc);
} catch {
secret = "";
}
}
await dispatchWebhookAlertAsync(payload, {
enabled: current.enableWebhookAlerts,
webhookUrl: current.webhookUrl,
webhookSecret: secret,
cooldownSec: current.webhookCooldownSec,
});
} catch {
// Never allow background webhook dispatch to throw into callers
}
})();
}
private static async executeArchiveAndPrune(
settings: AdminSettings,
): Promise<void> {
if (!settings.enableS3Backup) {
await CSVStorageManager.pruneExpiredPartitionsAsync(
settings.autoPruneDays,
);
return;
}
const s3Config: S3ArchiveConfig = {
endpoint: settings.s3Endpoint,
region: settings.s3Region,
bucket: settings.s3Bucket,
prefix: settings.s3Prefix,
accessKeyId: settings.s3AccessKeyId,
secretAccessKey: decryptSecret(settings.s3SecretAccessKeyEnc),
};
await CSVStorageManager.pruneOrArchiveExpiredPartitionsAsync(
settings.autoPruneDays,
{
beforeDeleteFile: async (file) => {
const objectKey = buildS3ObjectKey(
s3Config.prefix,
file.clientId,
file.filename,
);
const result = await uploadFileToS3(
s3Config,
file.filePath,
objectKey,
);
if (!result.ok) {
const errMsg =
result.error || `Failed to upload ${file.filename} to S3`;
this.triggerWebhookAlert({
event: "S3_ARCHIVE_FAILED",
severity: "critical",
title: "Off-Site S3 Archive Upload Failed",
message: errMsg,
clientId: file.clientId,
details: { filename: file.filename, bucket: s3Config.bucket },
});
throw new Error(errMsg);
}
},
},
);
}
private static applyRuntimeSettings(
settings: AdminSettings,
forcePrune = false,
forceCompress = false,
) {
SecurityEngine.maxEventsPerSec = settings.maxEventsPerSec;
SecurityEngine.enableIdempotency = settings.enableIdempotency;
SecurityEngine.maxPayloadSizeBytes = settings.maxPayloadSizeKb * 1024;
@@ -76,10 +272,34 @@ export class SettingsManager {
}
CSVStorageManager.maxFileSizeBytes = settings.fileRotationMb * 1024 * 1024;
const hasConfiguredDb = Boolean(process.env.DATABASE_URL?.trim());
if (
settings.autoPruneDays > 0 &&
(!hasConfiguredDb || globals.__dataStackSettingsHydrated === true)
) {
const dbReadyForMaintenance =
!hasConfiguredDb || globals.__dataStackSettingsHydrated === true;
if (settings.compressAfterDays > 0 && dbReadyForMaintenance) {
const prevComp = globals.__dataStackLastCompressMeta;
const now = Date.now();
const shouldCompress =
forceCompress ||
!prevComp ||
prevComp.days !== settings.compressAfterDays ||
now - prevComp.ranAtMs >= COMPRESS_COOLDOWN_MS;
if (shouldCompress) {
globals.__dataStackLastCompressMeta = {
days: settings.compressAfterDays,
ranAtMs: now,
};
void CSVStorageManager.compressOldPartitionsAsync(
settings.compressAfterDays,
).catch((err) => {
console.warn(
"[DataStack Storage] Background partition compression failed:",
err,
);
});
}
}
if (settings.autoPruneDays > 0 && dbReadyForMaintenance) {
const prev = globals.__dataStackLastPruneMeta;
const now = Date.now();
const shouldPrune =
@@ -92,11 +312,9 @@ export class SettingsManager {
days: settings.autoPruneDays,
ranAtMs: now,
};
void CSVStorageManager.pruneExpiredPartitionsAsync(
settings.autoPruneDays,
).catch((err) => {
void this.executeArchiveAndPrune(settings).catch((err) => {
console.warn(
"[DataStack Storage] Background async partition prune failed:",
"[DataStack Storage] Background async partition prune/archive failed:",
err,
);
});
@@ -115,7 +333,12 @@ export class SettingsManager {
const result = await pool.query(
`SELECT server_port, web_port, max_events_per_sec, max_payload_size_kb,
batch_flush_threshold, batch_flush_interval_ms, file_rotation_mb,
jwt_secret, enable_idempotency, auto_prune_days, updated_at
jwt_secret, enable_idempotency, auto_prune_days,
compress_after_days, enable_s3_backup, s3_endpoint, s3_region,
s3_bucket, s3_prefix, s3_access_key_id, s3_secret_access_key_enc,
disk_pause_threshold_pct, enable_webhook_alerts, webhook_url,
webhook_secret_enc, webhook_cooldown_sec, enable_prometheus_metrics,
prometheus_metrics_token_enc, updated_at
FROM admin_settings
WHERE id = 'default'`,
);
@@ -126,6 +349,21 @@ export class SettingsManager {
row.auto_prune_days !== undefined && row.auto_prune_days !== null
? Number(row.auto_prune_days)
: NaN;
const parsedCompressDays =
row.compress_after_days !== undefined &&
row.compress_after_days !== null
? Number(row.compress_after_days)
: NaN;
const parsedDiskThreshold =
row.disk_pause_threshold_pct !== undefined &&
row.disk_pause_threshold_pct !== null
? Number(row.disk_pause_threshold_pct)
: NaN;
const parsedWebhookCooldown =
row.webhook_cooldown_sec !== undefined &&
row.webhook_cooldown_sec !== null
? Number(row.webhook_cooldown_sec)
: NaN;
const envSocketPort = Number(process.env.PORT_SOCKET);
const envWebPort = Number(process.env.PORT);
const loaded: AdminSettings = {
@@ -142,9 +380,34 @@ export class SettingsManager {
batchFlushThreshold: Number(row.batch_flush_threshold) || 500,
batchFlushIntervalMs: Number(row.batch_flush_interval_ms) || 2000,
fileRotationMb: Number(row.file_rotation_mb) || 100,
jwtSecret: String(row.jwt_secret || "").trim() || resolveDefaultJwtSecret(),
jwtSecret:
String(row.jwt_secret || "").trim() || resolveDefaultJwtSecret(),
enableIdempotency: Boolean(row.enable_idempotency),
autoPruneDays: Number.isFinite(parsedPruneDays) ? parsedPruneDays : 30,
compressAfterDays: Number.isFinite(parsedCompressDays)
? Math.max(0, parsedCompressDays)
: 0,
enableS3Backup: Boolean(row.enable_s3_backup),
s3Endpoint: String(row.s3_endpoint ?? ""),
s3Region: String(row.s3_region || "auto"),
s3Bucket: String(row.s3_bucket ?? ""),
s3Prefix: String(row.s3_prefix ?? "datastack/"),
s3AccessKeyId: String(row.s3_access_key_id ?? ""),
s3SecretAccessKeyEnc: String(row.s3_secret_access_key_enc ?? ""),
diskPauseThresholdPct: Number.isFinite(parsedDiskThreshold)
? parsedDiskThreshold
: 95,
enableWebhookAlerts: Boolean(row.enable_webhook_alerts),
webhookUrl: String(row.webhook_url ?? ""),
webhookSecretEnc: String(row.webhook_secret_enc ?? ""),
webhookCooldownSec:
Number.isFinite(parsedWebhookCooldown) && parsedWebhookCooldown >= 10
? Math.floor(parsedWebhookCooldown)
: 300,
enablePrometheusMetrics: Boolean(row.enable_prometheus_metrics),
prometheusMetricsTokenEnc: String(
row.prometheus_metrics_token_enc ?? "",
),
updatedAt: row.updated_at
? new Date(row.updated_at as string | Date).toISOString()
: new Date().toISOString(),
@@ -198,8 +461,17 @@ export class SettingsManager {
`INSERT INTO admin_settings (
id, server_port, web_port, max_events_per_sec, max_payload_size_kb,
batch_flush_threshold, batch_flush_interval_ms, file_rotation_mb,
jwt_secret, enable_idempotency, auto_prune_days, updated_at
) VALUES ('default', $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, NOW())
jwt_secret, enable_idempotency, auto_prune_days,
compress_after_days, enable_s3_backup, s3_endpoint, s3_region,
s3_bucket, s3_prefix, s3_access_key_id, s3_secret_access_key_enc,
disk_pause_threshold_pct, enable_webhook_alerts, webhook_url,
webhook_secret_enc, webhook_cooldown_sec, enable_prometheus_metrics,
prometheus_metrics_token_enc, updated_at
) VALUES (
'default', $1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
$11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21,
$22, $23, $24, $25, NOW()
)
ON CONFLICT (id) DO UPDATE SET
server_port = EXCLUDED.server_port,
web_port = EXCLUDED.web_port,
@@ -211,6 +483,21 @@ export class SettingsManager {
jwt_secret = EXCLUDED.jwt_secret,
enable_idempotency = EXCLUDED.enable_idempotency,
auto_prune_days = EXCLUDED.auto_prune_days,
compress_after_days = EXCLUDED.compress_after_days,
enable_s3_backup = EXCLUDED.enable_s3_backup,
s3_endpoint = EXCLUDED.s3_endpoint,
s3_region = EXCLUDED.s3_region,
s3_bucket = EXCLUDED.s3_bucket,
s3_prefix = EXCLUDED.s3_prefix,
s3_access_key_id = EXCLUDED.s3_access_key_id,
s3_secret_access_key_enc = EXCLUDED.s3_secret_access_key_enc,
disk_pause_threshold_pct = EXCLUDED.disk_pause_threshold_pct,
enable_webhook_alerts = EXCLUDED.enable_webhook_alerts,
webhook_url = EXCLUDED.webhook_url,
webhook_secret_enc = EXCLUDED.webhook_secret_enc,
webhook_cooldown_sec = EXCLUDED.webhook_cooldown_sec,
enable_prometheus_metrics = EXCLUDED.enable_prometheus_metrics,
prometheus_metrics_token_enc = EXCLUDED.prometheus_metrics_token_enc,
updated_at = NOW()`,
[
settings.serverPort,
@@ -223,6 +510,21 @@ export class SettingsManager {
settings.jwtSecret,
settings.enableIdempotency,
settings.autoPruneDays,
settings.compressAfterDays,
settings.enableS3Backup,
settings.s3Endpoint,
settings.s3Region,
settings.s3Bucket,
settings.s3Prefix,
settings.s3AccessKeyId,
settings.s3SecretAccessKeyEnc,
settings.diskPauseThresholdPct,
settings.enableWebhookAlerts,
settings.webhookUrl,
settings.webhookSecretEnc,
settings.webhookCooldownSec,
settings.enablePrometheusMetrics,
settings.prometheusMetricsTokenEnc,
],
);
} catch (err: unknown) {
@@ -250,37 +552,49 @@ export class SettingsManager {
return { ...current };
}
public static getPublicSettings(): Omit<AdminSettings, "jwtSecret"> {
const settings = this.getSettings();
public static getPublicSettings(): PublicAdminSettings {
return toPublicSettings(this.getSettings());
}
public static async getPublicSettingsAsync(): Promise<PublicAdminSettings> {
const settings = await this.getSettingsAsync();
return toPublicSettings(settings);
}
public static async getDecryptedS3ConfigAsync(): Promise<S3ArchiveConfig> {
const settings = await this.getSettingsAsync();
return {
serverPort: settings.serverPort,
webPort: settings.webPort,
maxEventsPerSec: settings.maxEventsPerSec,
maxPayloadSizeKb: settings.maxPayloadSizeKb,
batchFlushThreshold: settings.batchFlushThreshold,
batchFlushIntervalMs: settings.batchFlushIntervalMs,
fileRotationMb: settings.fileRotationMb,
enableIdempotency: settings.enableIdempotency,
autoPruneDays: settings.autoPruneDays,
updatedAt: settings.updatedAt,
endpoint: settings.s3Endpoint,
region: settings.s3Region,
bucket: settings.s3Bucket,
prefix: settings.s3Prefix,
accessKeyId: settings.s3AccessKeyId,
secretAccessKey: decryptSecret(settings.s3SecretAccessKeyEnc),
};
}
public static async getPublicSettingsAsync(): Promise<
Omit<AdminSettings, "jwtSecret">
> {
public static async getDecryptedWebhookConfigAsync(): Promise<WebhookRuntimeConfig> {
const settings = await this.getSettingsAsync();
return {
serverPort: settings.serverPort,
webPort: settings.webPort,
maxEventsPerSec: settings.maxEventsPerSec,
maxPayloadSizeKb: settings.maxPayloadSizeKb,
batchFlushThreshold: settings.batchFlushThreshold,
batchFlushIntervalMs: settings.batchFlushIntervalMs,
fileRotationMb: settings.fileRotationMb,
enableIdempotency: settings.enableIdempotency,
autoPruneDays: settings.autoPruneDays,
updatedAt: settings.updatedAt,
enabled: settings.enableWebhookAlerts,
webhookUrl: settings.webhookUrl,
webhookSecret: settings.webhookSecretEnc
? decryptSecret(settings.webhookSecretEnc)
: "",
cooldownSec: settings.webhookCooldownSec,
};
}
public static async getDecryptedPrometheusConfigAsync(): Promise<{
enabled: boolean;
token: string;
}> {
const settings = await this.getSettingsAsync();
return {
enabled: settings.enablePrometheusMetrics,
token: settings.prometheusMetricsTokenEnc
? decryptSecret(settings.prometheusMetricsTokenEnc)
: "",
};
}
@@ -326,6 +640,12 @@ export class SettingsManager {
fileRotationMb: settings.fileRotationMb,
enableIdempotency: settings.enableIdempotency,
autoPruneDays: settings.autoPruneDays,
compressAfterDays: settings.compressAfterDays,
enableS3Backup: settings.enableS3Backup,
diskPauseThresholdPct: settings.diskPauseThresholdPct,
enableWebhookAlerts: settings.enableWebhookAlerts,
webhookUrl: settings.webhookUrl,
webhookCooldownSec: settings.webhookCooldownSec,
...(jwtSecretChanged ? { jwtSecret: settings.jwtSecret } : {}),
}),
cache: "no-store",
@@ -355,7 +675,10 @@ export class SettingsManager {
globals.__dataStackLastSettingsBridgeSync = result;
return result;
} catch (err: unknown) {
const message = err instanceof Error ? err.message : "Port 4000 settings-sync unreachable";
const message =
err instanceof Error
? err.message
: "Port 4000 settings-sync unreachable";
if (
process.env.NODE_ENV === "production" ||
Boolean(process.env.INGESTION_INTERNAL_URL)
@@ -374,18 +697,79 @@ export class SettingsManager {
}
}
public static updateSettings(updates: Partial<AdminSettings>): AdminSettings {
private static mergeSettingsUpdate(
current: AdminSettings,
updates: AdminSettingsUpdateInput,
): AdminSettings {
const {
s3SecretAccessKey,
clearS3SecretKey,
webhookSecret,
clearWebhookSecret,
prometheusMetricsToken,
clearPrometheusToken,
...rest
} = updates;
let nextEncryptedSecret = current.s3SecretAccessKeyEnc;
if (clearS3SecretKey) {
nextEncryptedSecret = "";
} else if (
typeof s3SecretAccessKey === "string" &&
s3SecretAccessKey.trim().length > 0
) {
nextEncryptedSecret = encryptSecret(s3SecretAccessKey.trim());
} else if (rest.s3SecretAccessKeyEnc !== undefined) {
nextEncryptedSecret = rest.s3SecretAccessKeyEnc;
}
let nextWebhookSecretEnc = current.webhookSecretEnc;
if (clearWebhookSecret) {
nextWebhookSecretEnc = "";
} else if (
typeof webhookSecret === "string" &&
webhookSecret.trim().length > 0
) {
nextWebhookSecretEnc = encryptSecret(webhookSecret.trim());
} else if (rest.webhookSecretEnc !== undefined) {
nextWebhookSecretEnc = rest.webhookSecretEnc;
}
let nextPrometheusTokenEnc = current.prometheusMetricsTokenEnc;
if (clearPrometheusToken) {
nextPrometheusTokenEnc = "";
} else if (
typeof prometheusMetricsToken === "string" &&
prometheusMetricsToken.trim().length > 0
) {
nextPrometheusTokenEnc = encryptSecret(prometheusMetricsToken.trim());
} else if (rest.prometheusMetricsTokenEnc !== undefined) {
nextPrometheusTokenEnc = rest.prometheusMetricsTokenEnc;
}
return {
...current,
...rest,
s3SecretAccessKeyEnc: nextEncryptedSecret,
webhookSecretEnc: nextWebhookSecretEnc,
prometheusMetricsTokenEnc: nextPrometheusTokenEnc,
updatedAt: new Date().toISOString(),
};
}
public static updateSettings(
updates: AdminSettingsUpdateInput,
): AdminSettings {
const current = getSettingsState();
const previousJwtSecret = current.jwtSecret;
const jwtSecretChanged =
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
const nextSettings: AdminSettings = {
...current,
...updates,
updatedAt: new Date().toISOString(),
};
const nextSettings = this.mergeSettingsUpdate(current, updates);
globals.__dataStackAdminSettings = nextSettings;
this.applyRuntimeSettings(nextSettings, updates.autoPruneDays !== undefined);
this.applyRuntimeSettings(
nextSettings,
updates.autoPruneDays !== undefined,
updates.compressAfterDays !== undefined,
);
if (jwtSecretChanged) {
SDKManager.resignAllClientTokens();
}
@@ -403,17 +787,13 @@ export class SettingsManager {
}
public static async updateSettingsAsync(
updates: Partial<AdminSettings>,
updates: AdminSettingsUpdateInput,
): Promise<AdminSettings> {
const current = await this.getSettingsAsync();
const previousJwtSecret = current.jwtSecret;
const jwtSecretChanged =
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
const nextSettings: AdminSettings = {
...current,
...updates,
updatedAt: new Date().toISOString(),
};
const nextSettings = this.mergeSettingsUpdate(current, updates);
if (
jwtSecretChanged &&
@@ -450,11 +830,15 @@ export class SettingsManager {
try {
await this.persistSettingsToDb(nextSettings);
globals.__dataStackAdminSettings = nextSettings;
this.applyRuntimeSettings(nextSettings, updates.autoPruneDays !== undefined);
this.applyRuntimeSettings(
nextSettings,
updates.autoPruneDays !== undefined,
updates.compressAfterDays !== undefined,
);
await SDKManager.resignAllClientTokensAsync();
} catch (err) {
globals.__dataStackAdminSettings = current;
this.applyRuntimeSettings(current, false);
this.applyRuntimeSettings(current, false, false);
await this.persistSettingsToDb(current).catch(() => undefined);
if (preSyncStatus.synced) {
await this.notifyIngestionSettingsSync(
@@ -482,7 +866,11 @@ export class SettingsManager {
await this.persistSettingsToDb(nextSettings);
globals.__dataStackAdminSettings = nextSettings;
this.applyRuntimeSettings(nextSettings, updates.autoPruneDays !== undefined);
this.applyRuntimeSettings(
nextSettings,
updates.autoPruneDays !== undefined,
updates.compressAfterDays !== undefined,
);
await this.notifyIngestionSettingsSync(
nextSettings,
previousJwtSecret,
+183
View File
@@ -0,0 +1,183 @@
import crypto from "node:crypto";
export type WebhookAlertEventType =
| "DISK_SPACE_LOW"
| "STORAGE_QUOTA_EXCEEDED"
| "S3_ARCHIVE_FAILED"
| "CLIENT_IP_FORBIDDEN"
| "CLIENT_TOKEN_EXPIRED"
| "ADMIN_LOGIN_LOCKOUT"
| "TEST_ALERT";
export interface WebhookAlertPayload {
event: WebhookAlertEventType;
severity: "info" | "warning" | "critical";
title: string;
message: string;
clientId?: string;
details?: Record<string, unknown>;
timestamp?: string;
}
export interface WebhookRuntimeConfig {
enabled: boolean;
webhookUrl: string;
webhookSecret: string;
cooldownSec: number;
}
export interface WebhookDispatchResult {
ok: boolean;
skipped?: boolean;
reason?: string;
status?: number;
error?: string;
}
type WebhookGlobals = typeof globalThis & {
__dataStackWebhookCooldownMap?: Map<string, number>;
};
const globals = globalThis as WebhookGlobals;
const DEFAULT_WEBHOOK_TIMEOUT_MS = 5000;
function getCooldownMap(): Map<string, number> {
if (!globals.__dataStackWebhookCooldownMap) {
globals.__dataStackWebhookCooldownMap = new Map<string, number>();
}
return globals.__dataStackWebhookCooldownMap;
}
export function clearWebhookCooldownCacheForTests(): void {
getCooldownMap().clear();
}
export function buildWebhookSignature(
rawBody: string,
secret: string,
): string {
const hmac = crypto
.createHmac("sha256", secret)
.update(rawBody, "utf8")
.digest("hex");
return `sha256=${hmac}`;
}
export function formatOutboundWebhookBody(payload: WebhookAlertPayload): string {
const timestamp = payload.timestamp || new Date().toISOString();
const prefix =
payload.severity === "critical"
? "[CRITICAL]"
: payload.severity === "warning"
? "[WARNING]"
: "[INFO]";
const clientTag = payload.clientId ? ` (client: ${payload.clientId})` : "";
const summaryLine = `${prefix} DataStack ${payload.event}${clientTag} — ${payload.title}: ${payload.message}`;
return JSON.stringify({
instance: "DataStack",
event: payload.event,
severity: payload.severity,
title: payload.title,
message: payload.message,
...(payload.clientId ? { clientId: payload.clientId } : {}),
...(payload.details ? { details: payload.details } : {}),
timestamp,
// Slack & Discord compatible top-level fields
text: summaryLine,
content: summaryLine,
});
}
export async function dispatchWebhookAlertAsync(
payload: WebhookAlertPayload,
config: WebhookRuntimeConfig,
options?: { bypassCooldown?: boolean; timeoutMs?: number },
): Promise<WebhookDispatchResult> {
const webhookUrl = String(config.webhookUrl || "").trim();
if (!options?.bypassCooldown && !config.enabled) {
return { ok: false, skipped: true, reason: "WEBHOOK_DISABLED" };
}
if (!webhookUrl) {
return {
ok: false,
skipped: !options?.bypassCooldown,
error: "Webhook URL is not configured.",
};
}
let parsedUrl: URL;
try {
parsedUrl = new URL(webhookUrl);
if (parsedUrl.protocol !== "http:" && parsedUrl.protocol !== "https:") {
return {
ok: false,
error: "Webhook URL must use http:// or https://.",
};
}
} catch {
return {
ok: false,
error: "Webhook URL is not a valid URL.",
};
}
const dedupeKey = `${payload.event}:${payload.clientId || "global"}`;
const cooldownMs = Math.max(0, (Number(config.cooldownSec) || 300) * 1000);
const now = Date.now();
const cooldownMap = getCooldownMap();
if (!options?.bypassCooldown && cooldownMs > 0) {
const lastSentAt = cooldownMap.get(dedupeKey);
if (lastSentAt !== undefined && now - lastSentAt < cooldownMs) {
return { ok: true, skipped: true, reason: "COOLDOWN_ACTIVE" };
}
cooldownMap.set(dedupeKey, now);
}
const timestamp = payload.timestamp || new Date(now).toISOString();
const bodyStr = formatOutboundWebhookBody({ ...payload, timestamp });
const headers: Record<string, string> = {
"Content-Type": "application/json",
"User-Agent": "DataStack-Webhook-Notifier/2.0",
"X-DataStack-Event": payload.event,
"X-DataStack-Timestamp": timestamp,
};
const secret = String(config.webhookSecret || "").trim();
if (secret) {
headers["X-DataStack-Signature"] = buildWebhookSignature(bodyStr, secret);
}
const timeoutMs = Math.max(500, options?.timeoutMs || DEFAULT_WEBHOOK_TIMEOUT_MS);
try {
const res = await fetch(parsedUrl.toString(), {
method: "POST",
headers,
body: bodyStr,
cache: "no-store",
signal: AbortSignal.timeout(timeoutMs),
});
if (!res.ok) {
if (!options?.bypassCooldown) {
// Keep cooldown entry so failing webhooks don't DDOS a broken upstream endpoint
}
const errText = await res.text().catch(() => "");
return {
ok: false,
status: res.status,
error: `Webhook endpoint returned HTTP ${res.status}${errText ? `: ${errText.slice(0, 160)}` : ""}`,
};
}
return { ok: true, status: res.status };
} catch (err: unknown) {
const msg =
err instanceof Error ? err.message : "Webhook delivery failed";
return {
ok: false,
error: `Webhook delivery failed: ${msg}`,
};
}
}
+8 -5
View File
@@ -13,11 +13,13 @@ export class ApiEngine {
*/
public static async getDashboardMetrics() {
const settings = await SettingsManager.getSettingsAsync();
const [storageStats, localBufferStats, clients] = await Promise.all([
CSVStorageManager.getStorageStatsAsync(),
DataBufferEngine.getBufferStatsAsync(),
SDKManager.listClientsAsync(),
]);
const [storageStats, diskUsage, localBufferStats, clients] =
await Promise.all([
CSVStorageManager.getStorageStatsAsync(),
CSVStorageManager.getDiskUsageAsync(),
DataBufferEngine.getBufferStatsAsync(),
SDKManager.listClientsAsync(),
]);
const activeClients = clients.filter((c) => c.status === 'active').length;
let activeSockets = 0;
let ingestionLocalBufferSize = 0;
@@ -129,6 +131,7 @@ export class ApiEngine {
activeClients,
bufferStats,
storageStats,
diskUsage,
securityStats,
runtime: {
serverPort: liveServerPort,
+122 -9
View File
@@ -23,6 +23,9 @@ export interface ClientSDK {
secret: string;
token: string;
rateLimit: number; // max events per sec
storageQuotaMb: number; // max storage in MB per client (0 = unlimited)
allowedIps: string; // comma-separated IPv4/IPv6/CIDR list ("" = allow all)
expiresAt?: string | null; // ISO timestamp or null/undefined if never expires
status: "active" | "revoked";
totalEvents: number;
lastActive?: string;
@@ -103,6 +106,9 @@ interface RemoteInMemoryClient {
status?: string;
token?: string;
rateLimit?: number;
storageQuotaMb?: number;
allowedIps?: string;
expiresAt?: string | null;
}
function hydrateLocalMapFromRemote(remoteClients: RemoteInMemoryClient[]): number {
@@ -119,6 +125,9 @@ function hydrateLocalMapFromRemote(remoteClients: RemoteInMemoryClient[]): numbe
secret: "",
token: String(item.token || ""),
rateLimit: Math.max(1, Number(item.rateLimit) || 100),
storageQuotaMb: Math.max(0, Math.floor(Number(item.storageQuotaMb) || 0)),
allowedIps: typeof item.allowedIps === "string" ? item.allowedIps : "",
expiresAt: item.expiresAt ? String(item.expiresAt) : null,
status: item.status === "revoked" ? "revoked" : "active",
totalEvents: 0,
createdAt: new Date().toISOString(),
@@ -135,6 +144,9 @@ function serializeSyncClient(client: ClientSDK) {
status: client.status,
token: client.token,
rateLimit: client.rateLimit,
storageQuotaMb: client.storageQuotaMb,
allowedIps: client.allowedIps || "",
expiresAt: client.expiresAt ?? null,
};
}
@@ -224,12 +236,26 @@ function notifyIngestionClientSync(
}
function mapRowToClient(row: Record<string, unknown>): ClientSDK {
const parsedQuota = Number(row.storage_quota_mb);
let expiresAt: string | null = null;
if (row.expires_at) {
const d = new Date(row.expires_at as string | Date);
if (!Number.isNaN(d.getTime())) {
expiresAt = d.toISOString();
}
}
return {
clientId: String(row.client_id),
name: String(row.name),
secret: String(row.secret),
token: String(row.token),
rateLimit: Number(row.rate_limit) || 100,
storageQuotaMb:
Number.isFinite(parsedQuota) && parsedQuota >= 0
? Math.floor(parsedQuota)
: 0,
allowedIps: typeof row.allowed_ips === "string" ? row.allowed_ips : "",
expiresAt,
status: row.status === "revoked" ? "revoked" : "active",
totalEvents: Number(row.total_events) || 0,
lastActive: row.last_active
@@ -256,7 +282,7 @@ export class SDKManager {
const [result, deletedResult] = await Promise.all([
pool.query(
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
`SELECT client_id, name, secret, token, rate_limit, storage_quota_mb, allowed_ips, expires_at, status, total_events, last_active, created_at
FROM client_sdks
ORDER BY created_at DESC`,
),
@@ -464,7 +490,7 @@ export class SDKManager {
await this.flushPendingClientActivityAsync();
}
const result = await pool.query(
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
`SELECT client_id, name, secret, token, rate_limit, storage_quota_mb, allowed_ips, expires_at, status, total_events, last_active, created_at
FROM client_sdks
WHERE client_id = $1`,
[normalizedId],
@@ -500,10 +526,19 @@ export class SDKManager {
clientId: string;
name: string;
rateLimit?: number;
storageQuotaMb?: number;
allowedIps?: string;
expiresAt?: string | null;
}): ClientSDK {
const safeClientId = this.normalizeClientId(data.clientId);
const secret = this.generateClientSecret(safeClientId);
const rateLimit = Math.max(1, Number(data.rateLimit) || 100);
const storageQuotaMb = Math.max(
0,
Math.floor(Number(data.storageQuotaMb) || 0),
);
const allowedIps = typeof data.allowedIps === "string" ? data.allowedIps.trim() : "";
const expiresAt = data.expiresAt ? String(data.expiresAt) : null;
const token = AuthService.generateClientToken({
clientId: safeClientId,
@@ -517,6 +552,9 @@ export class SDKManager {
secret,
token,
rateLimit,
storageQuotaMb,
allowedIps,
expiresAt,
status: "active",
totalEvents: 0,
createdAt: new Date().toISOString(),
@@ -548,13 +586,16 @@ export class SDKManager {
[client.clientId],
);
await pool.query(
`INSERT INTO client_sdks (client_id, name, secret, token, rate_limit, status, total_events, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW())
`INSERT INTO client_sdks (client_id, name, secret, token, rate_limit, storage_quota_mb, allowed_ips, expires_at, status, total_events, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, NOW())
ON CONFLICT (client_id) DO UPDATE SET
name = EXCLUDED.name,
secret = EXCLUDED.secret,
token = EXCLUDED.token,
rate_limit = EXCLUDED.rate_limit,
storage_quota_mb = EXCLUDED.storage_quota_mb,
allowed_ips = EXCLUDED.allowed_ips,
expires_at = EXCLUDED.expires_at,
status = EXCLUDED.status,
updated_at = NOW()`,
[
@@ -563,6 +604,9 @@ export class SDKManager {
client.secret,
client.token,
client.rateLimit,
client.storageQuotaMb,
client.allowedIps || "",
client.expiresAt ?? null,
client.status,
client.totalEvents,
client.createdAt,
@@ -578,11 +622,20 @@ export class SDKManager {
clientId: string;
name: string;
rateLimit?: number;
storageQuotaMb?: number;
allowedIps?: string;
expiresAt?: string | null;
}): Promise<ClientSDK> {
await ensureDbSettingsHydrated();
const safeClientId = this.normalizeClientId(data.clientId);
const secret = this.generateClientSecret(safeClientId);
const rateLimit = Math.max(1, Number(data.rateLimit) || 100);
const storageQuotaMb = Math.max(
0,
Math.floor(Number(data.storageQuotaMb) || 0),
);
const allowedIps = typeof data.allowedIps === "string" ? data.allowedIps.trim() : "";
const expiresAt = data.expiresAt ? String(data.expiresAt) : null;
const token = AuthService.generateClientToken({
clientId: safeClientId,
@@ -596,6 +649,9 @@ export class SDKManager {
secret,
token,
rateLimit,
storageQuotaMb,
allowedIps,
expiresAt,
status: "active",
totalEvents: 0,
createdAt: new Date().toISOString(),
@@ -683,7 +739,17 @@ export class SDKManager {
public static updateClient(
clientId: string,
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
updates: Partial<
Pick<
ClientSDK,
| "name"
| "rateLimit"
| "storageQuotaMb"
| "allowedIps"
| "expiresAt"
| "status"
>
>,
): ClientSDK | null {
const normalizedId = this.normalizeClientId(clientId);
const client = getClientsMap().get(normalizedId);
@@ -695,6 +761,20 @@ export class SDKManager {
...(updates.rateLimit !== undefined
? { rateLimit: Math.max(1, Number(updates.rateLimit) || 100) }
: {}),
...(updates.storageQuotaMb !== undefined
? {
storageQuotaMb: Math.max(
0,
Math.floor(Number(updates.storageQuotaMb) || 0),
),
}
: {}),
...(updates.allowedIps !== undefined
? { allowedIps: String(updates.allowedIps).trim() }
: {}),
...(updates.expiresAt !== undefined
? { expiresAt: updates.expiresAt ? String(updates.expiresAt) : null }
: {}),
...(updates.status !== undefined ? { status: updates.status } : {}),
};
@@ -722,9 +802,17 @@ export class SDKManager {
try {
await pool.query(
`UPDATE client_sdks
SET name = $2, rate_limit = $3, status = $4, updated_at = NOW()
SET name = $2, rate_limit = $3, storage_quota_mb = $4, allowed_ips = $5, expires_at = $6, status = $7, updated_at = NOW()
WHERE client_id = $1`,
[client.clientId, client.name, client.rateLimit, client.status],
[
client.clientId,
client.name,
client.rateLimit,
client.storageQuotaMb,
client.allowedIps || "",
client.expiresAt ?? null,
client.status,
],
);
} catch (err: unknown) {
const detail = err instanceof Error ? err.message : "Unknown DB error";
@@ -734,7 +822,17 @@ export class SDKManager {
public static async updateClientAsync(
clientId: string,
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
updates: Partial<
Pick<
ClientSDK,
| "name"
| "rateLimit"
| "storageQuotaMb"
| "allowedIps"
| "expiresAt"
| "status"
>
>,
): Promise<ClientSDK | null> {
await ensureDbSettingsHydrated();
const normalizedId = this.normalizeClientId(clientId);
@@ -747,6 +845,20 @@ export class SDKManager {
...(updates.rateLimit !== undefined
? { rateLimit: Math.max(1, Number(updates.rateLimit) || 100) }
: {}),
...(updates.storageQuotaMb !== undefined
? {
storageQuotaMb: Math.max(
0,
Math.floor(Number(updates.storageQuotaMb) || 0),
),
}
: {}),
...(updates.allowedIps !== undefined
? { allowedIps: String(updates.allowedIps).trim() }
: {}),
...(updates.expiresAt !== undefined
? { expiresAt: updates.expiresAt ? String(updates.expiresAt) : null }
: {}),
...(updates.status !== undefined ? { status: updates.status } : {}),
};
@@ -853,7 +965,7 @@ export class SDKManager {
const dirStat = lstatSync(targetDir);
if (!dirStat.isSymbolicLink() && dirStat.isDirectory()) {
for (const entry of readdirSync(targetDir)) {
if (!entry.endsWith(".csv")) continue;
if (!CSVStorageManager.isPartitionFilename(entry)) continue;
const entryPath = resolve(targetDir, entry);
try {
const entryStat = lstatSync(entryPath);
@@ -871,6 +983,7 @@ export class SDKManager {
rmSync(targetDir, { recursive: true, force: true });
CSVStorageManager.clearRowCountCache();
CSVStorageManager.clearQuotaCaches(safeId);
return purgedFiles;
}