feat(auth): harden session management and security

- Implement server-side session revocation on logout
- Add IP extraction utility and admin setup error handling
- Improve error codes and response consistency for auth endpoints
- Refine landing page redirect logic to handle transient DB outages
- Update deployment documentation and configuration defaults
This commit is contained in:
Team Dark Devil committed 2026-11-19 20:22:01 +06:00
1 parent 935f12a1e9
commit 89e4d68690
81 files changed
+11530 -6916

No files matched your search

+103 -85
View File
@@ -9,6 +9,98 @@ type CoreDatabaseGlobals = typeof globalThis & {
const globals = globalThis as CoreDatabaseGlobals;
export const CORE_SCHEMA_TABLE_DDLS = [
`
CREATE TABLE IF NOT EXISTS client_sdks (
client_id TEXT PRIMARY KEY,
name TEXT NOT NULL,
secret TEXT NOT NULL,
token TEXT NOT NULL,
rate_limit INTEGER NOT NULL DEFAULT 100,
status TEXT NOT NULL DEFAULT 'active',
total_events BIGINT NOT NULL DEFAULT 0,
last_active TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`,
`
CREATE TABLE IF NOT EXISTS admin_settings (
id TEXT PRIMARY KEY DEFAULT 'default',
server_port INTEGER NOT NULL DEFAULT 4000,
web_port INTEGER NOT NULL DEFAULT 3000,
max_events_per_sec INTEGER NOT NULL DEFAULT 100,
max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
file_rotation_mb INTEGER NOT NULL DEFAULT 100,
jwt_secret TEXT NOT NULL,
enable_idempotency BOOLEAN NOT NULL DEFAULT true,
auto_prune_days INTEGER NOT NULL DEFAULT 30,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`,
`
CREATE TABLE IF NOT EXISTS deleted_clients (
client_id TEXT PRIMARY KEY,
deleted_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`,
`
CREATE TABLE IF NOT EXISTS admin_users (
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
email TEXT NOT NULL UNIQUE,
password_salt BYTEA NOT NULL,
password_hash BYTEA NOT NULL,
token_version INTEGER NOT NULL DEFAULT 1,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`,
`
CREATE TABLE IF NOT EXISTS admin_login_attempts (
key_hash TEXT PRIMARY KEY,
window_started_at TIMESTAMPTZ NOT NULL,
failed_attempts INTEGER NOT NULL,
blocked_until TIMESTAMPTZ,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`,
] as const;
export const CORE_SCHEMA_ALTER_DDLS = [
`
ALTER TABLE client_sdks
ADD COLUMN IF NOT EXISTS rate_limit INTEGER NOT NULL DEFAULT 100,
ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'active',
ADD COLUMN IF NOT EXISTS total_events BIGINT NOT NULL DEFAULT 0,
ADD COLUMN IF NOT EXISTS last_active TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
`,
`
ALTER TABLE admin_settings
ADD COLUMN IF NOT EXISTS server_port INTEGER NOT NULL DEFAULT 4000,
ADD COLUMN IF NOT EXISTS web_port INTEGER NOT NULL DEFAULT 3000,
ADD COLUMN IF NOT EXISTS max_events_per_sec INTEGER NOT NULL DEFAULT 100,
ADD COLUMN IF NOT EXISTS max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
ADD COLUMN IF NOT EXISTS batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
ADD COLUMN IF NOT EXISTS batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
ADD COLUMN IF NOT EXISTS file_rotation_mb INTEGER NOT NULL DEFAULT 100,
ADD COLUMN IF NOT EXISTS enable_idempotency BOOLEAN NOT NULL DEFAULT true,
ADD COLUMN IF NOT EXISTS auto_prune_days INTEGER NOT NULL DEFAULT 30,
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
`,
`
ALTER TABLE admin_users
ADD COLUMN IF NOT EXISTS token_version INTEGER NOT NULL DEFAULT 1
`,
] as const;
export async function ensureCoreSchemaOnPool(pool: Pool): Promise<void> {
await Promise.all(CORE_SCHEMA_TABLE_DDLS.map((sql) => pool.query(sql)));
await Promise.all(CORE_SCHEMA_ALTER_DDLS.map((sql) => pool.query(sql)));
}
/**
* Database Module (PostgreSQL Persistence Engine)
* CRITICAL RULE: PostgreSQL stores strictly administrative metadata,
@@ -17,19 +109,26 @@ const globals = globalThis as CoreDatabaseGlobals;
*/
export class DatabaseEngine {
public static getPool(): Pool | null {
const connectionString = process.env.DATABASE_URL;
const connectionString = process.env.DATABASE_URL?.trim();
if (!connectionString) {
return null;
}
if (globals.__dataStackCorePool === undefined) {
try {
globals.__dataStackCorePool = new Pool({
const pool = new Pool({
connectionString,
connectionTimeoutMillis: 3000,
idleTimeoutMillis: 30000,
max: 5,
max: 10,
});
pool.on("error", (err) => {
console.error(
"[DataStack DB] Unexpected idle PostgreSQL pool error:",
err instanceof Error ? err.message : err,
);
});
globals.__dataStackCorePool = pool;
} catch {
globals.__dataStackCorePool = null;
}
@@ -42,88 +141,7 @@ export class DatabaseEngine {
const pool = this.getPool();
if (!pool) return false;
globals.__dataStackCoreSchema ??= Promise.all([
pool.query(`
CREATE TABLE IF NOT EXISTS client_sdks (
client_id TEXT PRIMARY KEY,
name TEXT NOT NULL,
secret TEXT NOT NULL,
token TEXT NOT NULL,
rate_limit INTEGER NOT NULL DEFAULT 100,
status TEXT NOT NULL DEFAULT 'active',
total_events BIGINT NOT NULL DEFAULT 0,
last_active TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`),
pool.query(`
CREATE TABLE IF NOT EXISTS admin_settings (
id TEXT PRIMARY KEY DEFAULT 'default',
server_port INTEGER NOT NULL DEFAULT 4000,
web_port INTEGER NOT NULL DEFAULT 3000,
max_events_per_sec INTEGER NOT NULL DEFAULT 100,
max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
file_rotation_mb INTEGER NOT NULL DEFAULT 100,
jwt_secret TEXT NOT NULL,
enable_idempotency BOOLEAN NOT NULL DEFAULT true,
auto_prune_days INTEGER NOT NULL DEFAULT 30,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`),
pool.query(`
CREATE TABLE IF NOT EXISTS deleted_clients (
client_id TEXT PRIMARY KEY,
deleted_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`),
pool.query(`
CREATE TABLE IF NOT EXISTS admin_users (
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
email TEXT NOT NULL UNIQUE,
password_salt BYTEA NOT NULL,
password_hash BYTEA NOT NULL,
token_version INTEGER NOT NULL DEFAULT 1,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`),
pool.query(`
CREATE TABLE IF NOT EXISTS admin_login_attempts (
key_hash TEXT PRIMARY KEY,
window_started_at TIMESTAMPTZ NOT NULL,
failed_attempts INTEGER NOT NULL,
blocked_until TIMESTAMPTZ,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`),
]).then(() =>
Promise.all([
pool.query(`
ALTER TABLE client_sdks
ADD COLUMN IF NOT EXISTS rate_limit INTEGER NOT NULL DEFAULT 100,
ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'active',
ADD COLUMN IF NOT EXISTS total_events BIGINT NOT NULL DEFAULT 0,
ADD COLUMN IF NOT EXISTS last_active TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
`),
pool.query(`
ALTER TABLE admin_settings
ADD COLUMN IF NOT EXISTS server_port INTEGER NOT NULL DEFAULT 4000,
ADD COLUMN IF NOT EXISTS web_port INTEGER NOT NULL DEFAULT 3000,
ADD COLUMN IF NOT EXISTS max_events_per_sec INTEGER NOT NULL DEFAULT 100,
ADD COLUMN IF NOT EXISTS max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
ADD COLUMN IF NOT EXISTS batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
ADD COLUMN IF NOT EXISTS batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
ADD COLUMN IF NOT EXISTS file_rotation_mb INTEGER NOT NULL DEFAULT 100,
ADD COLUMN IF NOT EXISTS enable_idempotency BOOLEAN NOT NULL DEFAULT true,
ADD COLUMN IF NOT EXISTS auto_prune_days INTEGER NOT NULL DEFAULT 30,
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
`),
]),
).then(() => undefined);
globals.__dataStackCoreSchema ??= ensureCoreSchemaOnPool(pool);
try {
await globals.__dataStackCoreSchema;
+206 -1
View File
@@ -1,5 +1,17 @@
import { createHash } from "node:crypto";
import { AuthService } from "../auth/jwt";
import { DataBufferEngine, type RedisBufferClient } from "../data/buffer";
const REDIS_IDEMPOTENCY_PREFIX = "datastack:idem:";
const REDIS_RATE_LIMIT_PREFIX = "datastack:rl:";
export const REDIS_RATE_LIMIT_LUA_SCRIPT = `
-- DATASTACK_RATE_LIMIT_INCR
local c = redis.call("INCR", KEYS[1])
if c == 1 or redis.call("PTTL", KEYS[1]) < 0 then
redis.call("PEXPIRE", KEYS[1], tonumber(ARGV[1]))
end
return c
`.trim();
export function resolveInternalIngestionSecret(
jwtSecretOverride?: string,
@@ -43,9 +55,15 @@ export interface ValidationResult {
| "MALFORMED_PAYLOAD";
}
export type SecurityMode =
| "redis_shared"
| "bridge_coordinated"
| "degraded_local";
export class SecurityEngine {
public static maxEventsPerSec = 100;
public static enableIdempotency = true;
private static lastSecurityMode: SecurityMode = "degraded_local";
// Idempotency cache: key -> timestamp (expires after TTL)
private static idempotencySet = new Map<string, number>();
@@ -126,6 +144,32 @@ export class SecurityEngine {
return true;
}
private static trackLocalRateWindow(clientId: string): void {
const now = Date.now();
const current = this.rateLimitMap.get(clientId);
if (!current || now - current.windowStart >= 1000) {
this.rateLimitMap.set(clientId, { count: 1, windowStart: now });
} else {
current.count += 1;
}
}
private static supportsSharedSecurityRedis(
redis: RedisBufferClient | null,
): redis is RedisBufferClient & {
get: (key: string) => Promise<string | null>;
incr: (key: string) => Promise<number>;
pexpire: (key: string, ttlMs: number) => Promise<number>;
} {
return Boolean(
redis &&
typeof redis.get === "function" &&
typeof redis.incr === "function" &&
typeof redis.pexpire === "function" &&
typeof redis.set === "function",
);
}
/**
* Validates incoming event payload:
* Checks duplicate idempotency (read-only), max byte size, and rate limit,
@@ -148,7 +192,7 @@ export class SecurityEngine {
// 2. Check payload size before consuming rate-limit quota
const payloadStr =
typeof payload === "string" ? payload : JSON.stringify(payload);
typeof payload === "string" ? payload : JSON.stringify(payload ?? {});
const byteLength = Buffer.byteLength(payloadStr, "utf8");
if (byteLength > this.maxPayloadSizeBytes) {
@@ -178,6 +222,158 @@ export class SecurityEngine {
return { allowed: true };
}
/**
* Shared Redis-backed async validator with Port 4000 internal-bridge fallback
* (so HTTP + WS share rate limits & idempotency even when Redis is unconfigured)
* and final local-Map fallback for isolated single-process dev.
*/
public static async validateEventAsync(
clientId: string,
eventId: string,
payload: unknown,
clientMaxRate = 100,
): Promise<ValidationResult> {
const redis = DataBufferEngine.getRedisClient();
if (!this.supportsSharedSecurityRedis(redis)) {
const payloadStr =
typeof payload === "string" ? payload : JSON.stringify(payload ?? {});
const payloadBytes = Buffer.byteLength(payloadStr, "utf8");
if (payloadBytes > this.maxPayloadSizeBytes) {
return {
allowed: false,
reason: `Payload size (${(payloadBytes / 1024).toFixed(1)} KB) exceeds limit of ${this.maxPayloadSizeBytes / 1024} KB`,
code: "PAYLOAD_TOO_LARGE",
};
}
const headers = getInternalIngestionHeaders();
if (headers["x-datastack-internal-secret"]) {
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, "");
try {
const res = await fetch(`${ingestionBaseUrl}/internal/security-check`, {
method: "POST",
headers: {
"Content-Type": "application/json",
...headers,
},
body: JSON.stringify({
clientId,
eventId,
payloadBytes,
clientMaxRate,
}),
cache: "no-store",
signal: AbortSignal.timeout(400),
});
if (res.ok) {
const data = (await res.json()) as {
allowed?: boolean;
code?: ValidationResult["code"];
reason?: string;
};
this.lastSecurityMode = "bridge_coordinated";
return {
allowed: Boolean(data.allowed),
...(data.code ? { code: data.code } : {}),
...(data.reason ? { reason: data.reason } : {}),
};
}
} catch {
// Port 4000 unreachable — fall back to local validation
}
}
this.lastSecurityMode = "degraded_local";
return this.validateEvent(clientId, eventId, payload, clientMaxRate);
}
this.lastSecurityMode = "redis_shared";
const idemKey = `${REDIS_IDEMPOTENCY_PREFIX}${clientId}:${eventId}`;
try {
// 1. Read-only idempotency check
if (this.enableIdempotency && eventId) {
if (this.hasDuplicate(clientId, eventId)) {
return {
allowed: false,
reason: "Duplicate event dropped silently",
code: "DUPLICATE_EVENT",
};
}
const existing = await redis.get(idemKey);
if (existing !== null) {
this.recordIdempotency(clientId, eventId);
return {
allowed: false,
reason: "Duplicate event dropped silently",
code: "DUPLICATE_EVENT",
};
}
}
// 2. Payload byte size check before rate-limit consumption
const payloadStr =
typeof payload === "string" ? payload : JSON.stringify(payload ?? {});
const byteLength = Buffer.byteLength(payloadStr, "utf8");
if (byteLength > this.maxPayloadSizeBytes) {
return {
allowed: false,
reason: `Payload size (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${this.maxPayloadSizeBytes / 1024} KB`,
code: "PAYLOAD_TOO_LARGE",
};
}
// 3. Shared atomic per-second rate-limit counter in Redis (single Lua round-trip)
const effectiveRateLimit = Math.min(clientMaxRate, this.maxEventsPerSec);
const windowBucket = Math.floor(Date.now() / 1000);
const rlKey = `${REDIS_RATE_LIMIT_PREFIX}${clientId}:${windowBucket}`;
let count: number;
if (typeof redis.eval === "function") {
count = Number(
await redis.eval(REDIS_RATE_LIMIT_LUA_SCRIPT, 1, rlKey, "2000"),
);
} else {
count = Number(await redis.incr(rlKey));
if (count === 1) {
await redis.pexpire(rlKey, 2000);
}
}
this.trackLocalRateWindow(clientId);
if (count > effectiveRateLimit) {
return {
allowed: false,
reason: `Rate limit of ${effectiveRateLimit} events/sec exceeded for client ${clientId}`,
code: "RATE_LIMIT_EXCEEDED",
};
}
// 4. Atomic claim of idempotency key after all checks succeed
if (this.enableIdempotency && eventId) {
const claimed = await redis.set(
idemKey,
"1",
"PX",
this.IDEMPOTENCY_TTL_MS,
"NX",
);
this.recordIdempotency(clientId, eventId);
if (claimed !== "OK") {
return {
allowed: false,
reason: "Duplicate event dropped silently",
code: "DUPLICATE_EVENT",
};
}
}
return { allowed: true };
} catch {
return this.validateEvent(clientId, eventId, payload, clientMaxRate);
}
}
public static getStats() {
const now = Date.now();
for (const [k, timestamp] of this.idempotencySet.entries()) {
@@ -191,10 +387,19 @@ export class SecurityEngine {
}
}
const mode: SecurityMode = this.supportsSharedSecurityRedis(
DataBufferEngine.getRedisClient(),
)
? "redis_shared"
: this.lastSecurityMode === "redis_shared"
? "degraded_local"
: this.lastSecurityMode;
return {
activeIdempotencyKeys: this.idempotencySet.size,
rateLimitedClientsCount: this.rateLimitMap.size,
maxPayloadSizeBytes: this.maxPayloadSizeBytes,
mode,
};
}
}
+364 -100
View File
@@ -1,5 +1,7 @@
import crypto from 'node:crypto';
import fs from 'fs';
import path from 'path';
import { StringDecoder } from 'node:string_decoder';
import { Parser } from 'json2csv';
export interface DataEvent {
@@ -34,9 +36,17 @@ function sleepMs(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
export function resolveDefaultStorageDir(): string {
const configured = (process.env.DATASTACK_STORAGE_DIR || '').trim();
if (configured) {
return path.resolve(configured);
}
return path.resolve(process.cwd(), 'storage', 'data');
}
export class CSVStorageManager {
// Base storage directory: /storage/data
public static baseStorageDir = path.resolve(process.cwd(), 'storage', 'data');
// Base storage directory: DATASTACK_STORAGE_DIR or /storage/data
public static baseStorageDir = resolveDefaultStorageDir();
public static maxFileSizeBytes = 100 * 1024 * 1024; // 100MB rotation limit
private static rowCountCache = new Map<string, RowCountCacheEntry>();
@@ -44,12 +54,94 @@ export class CSVStorageManager {
this.rowCountCache.clear();
}
public static sanitizeClientId(clientId: string): string {
const raw = String(clientId || 'unknown').trim().replace(/[^a-zA-Z0-9_-]/g, '_');
return raw || 'unknown';
}
/**
* Ensures client directory exists
* Checks whether the last byte of a non-empty file is LF (\n)
*/
private static fileEndsWithNewline(filePath: string, sizeBytes: number): boolean {
if (sizeBytes <= 0) return false;
const fd = fs.openSync(filePath, 'r');
try {
const lastByte = Buffer.allocUnsafe(1);
const read = fs.readSync(fd, lastByte, 0, 1, sizeBytes - 1);
return read === 1 && lastByte[0] === 0x0a;
} finally {
fs.closeSync(fd);
}
}
private static removeStaleLockAtomically(lockPath: string): void {
try {
const lockStat = fs.lstatSync(lockPath);
if (Date.now() - lockStat.mtimeMs <= STALE_LOCK_AGE_MS) {
return;
}
const staleTmp = `${lockPath}.stale.${crypto.randomUUID()}`;
fs.renameSync(lockPath, staleTmp);
try {
fs.unlinkSync(staleTmp);
} catch {
// Ignore cleanup failure of already-renamed stale token
}
} catch {
// Lock was released or claimed concurrently
}
}
private static releaseLockIfOwned(lockPath: string, lockValue: string): void {
try {
if (!fs.existsSync(lockPath)) return;
const current = fs.readFileSync(lockPath, 'utf8');
if (current === lockValue) {
fs.unlinkSync(lockPath);
}
} catch {
// Ignore cleanup failure if already unlinked
}
}
private static async removeStaleLockAtomicallyAsync(lockPath: string): Promise<void> {
try {
const lockStat = await fs.promises.lstat(lockPath);
if (Date.now() - lockStat.mtimeMs <= STALE_LOCK_AGE_MS) {
return;
}
const staleTmp = `${lockPath}.stale.${crypto.randomUUID()}`;
await fs.promises.rename(lockPath, staleTmp);
await fs.promises.unlink(staleTmp).catch(() => undefined);
} catch {
// Lock was released or claimed concurrently
}
}
private static async releaseLockIfOwnedAsync(
lockPath: string,
lockValue: string,
): Promise<void> {
try {
const current = await fs.promises.readFile(lockPath, 'utf8');
if (current === lockValue) {
await fs.promises.unlink(lockPath);
}
} catch {
// Ignore cleanup failure if already unlinked
}
}
/**
* Ensures client directory exists safely within baseStorageDir
*/
public static ensureClientDir(clientId: string): string {
const safeClientId = clientId.replace(/[^a-zA-Z0-9_-]/g, '_');
const clientDir = path.join(this.baseStorageDir, safeClientId);
const safeClientId = this.sanitizeClientId(clientId);
const resolvedBase = path.resolve(this.baseStorageDir);
const clientDir = path.resolve(resolvedBase, safeClientId);
if (!clientDir.startsWith(resolvedBase + path.sep)) {
throw new Error(`Invalid storage client directory: ${clientId}`);
}
if (!fs.existsSync(clientDir)) {
fs.mkdirSync(clientDir, { recursive: true });
}
@@ -67,6 +159,7 @@ export class CSVStorageManager {
const clientDir = this.ensureClientDir(clientId);
const date = this.extractSafePartitionDate(dateStr);
const lockPath = path.join(clientDir, `.${date}.csv.lock`);
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
const deadline = Date.now() + LOCK_TIMEOUT_MS;
let acquired = false;
@@ -74,7 +167,7 @@ export class CSVStorageManager {
try {
const fd = fs.openSync(lockPath, 'wx');
try {
fs.writeFileSync(fd, `${process.pid}:${Date.now()}`, 'utf8');
fs.writeFileSync(fd, lockValue, 'utf8');
} finally {
fs.closeSync(fd);
}
@@ -85,15 +178,7 @@ export class CSVStorageManager {
if (code !== 'EEXIST') {
throw err;
}
try {
const lockStat = fs.statSync(lockPath);
if (Date.now() - lockStat.mtimeMs > STALE_LOCK_AGE_MS) {
fs.unlinkSync(lockPath);
continue;
}
} catch {
// Lock was released concurrently
}
this.removeStaleLockAtomically(lockPath);
const jitter =
LOCK_RETRY_MIN_MS +
Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS));
@@ -108,13 +193,7 @@ export class CSVStorageManager {
try {
return await fn();
} finally {
try {
if (fs.existsSync(lockPath)) {
fs.unlinkSync(lockPath);
}
} catch {
// Ignore cleanup failure if already unlinked
}
this.releaseLockIfOwned(lockPath, lockValue);
}
}
@@ -158,51 +237,58 @@ export class CSVStorageManager {
* Stateful RFC 4180 quote-aware chunked row counter so multi-100MB CSVs never load wholly into RAM
*/
private static createChunkRowCounter() {
const decoder = new StringDecoder('utf8');
let inQuotes = false;
let pendingQuote = false;
let currentLineHasChars = false;
let totalRows = 0;
return {
pushChunk(chunk: Buffer | string) {
const str = typeof chunk === 'string' ? chunk : chunk.toString('utf8');
for (let i = 0; i < str.length; i += 1) {
const ch = str[i];
if (pendingQuote) {
pendingQuote = false;
if (ch === '"') {
// Escaped quote "" across chunk boundary or inline
continue;
}
inQuotes = !inQuotes;
}
const processStr = (str: string) => {
for (let i = 0; i < str.length; i += 1) {
const ch = str[i];
if (pendingQuote) {
pendingQuote = false;
if (ch === '"') {
currentLineHasChars = true;
if (inQuotes) {
if (i + 1 < str.length) {
if (str[i + 1] === '"') {
i += 1;
} else {
inQuotes = false;
}
// Escaped quote "" across chunk boundary or inline
continue;
}
inQuotes = !inQuotes;
}
if (ch === '"') {
currentLineHasChars = true;
if (inQuotes) {
if (i + 1 < str.length) {
if (str[i + 1] === '"') {
i += 1;
} else {
pendingQuote = true;
inQuotes = false;
}
} else {
inQuotes = true;
pendingQuote = true;
}
} else if (ch === '\n' && !inQuotes) {
if (currentLineHasChars) {
totalRows += 1;
currentLineHasChars = false;
}
} else if (ch !== '\r' && ch !== ' ' && ch !== '\t') {
currentLineHasChars = true;
} else {
inQuotes = true;
}
} else if (ch === '\n' && !inQuotes) {
if (currentLineHasChars) {
totalRows += 1;
currentLineHasChars = false;
}
} else if (ch !== '\r' && ch !== ' ' && ch !== '\t') {
currentLineHasChars = true;
}
}
};
return {
pushChunk(chunk: Buffer | string) {
const str = typeof chunk === 'string' ? chunk : decoder.write(chunk);
if (str) processStr(str);
},
finish(): number {
const trailing = decoder.end();
if (trailing) processStr(trailing);
if (currentLineHasChars) {
totalRows += 1;
}
@@ -243,7 +329,11 @@ export class CSVStorageManager {
/**
* Resolves target CSV file path with daily and configured MB rotation rules
*/
public static getTargetFilePath(clientId: string, dateStr?: string): string {
public static getTargetFilePath(
clientId: string,
dateStr?: string,
incomingBytes = 0,
): string {
const clientDir = this.ensureClientDir(clientId);
const date = this.extractSafePartitionDate(dateStr);
let baseFileName = `${date}.csv`;
@@ -252,16 +342,24 @@ export class CSVStorageManager {
// Check for file rotation
let partIndex = 1;
while (fs.existsSync(targetPath)) {
let stats: fs.Stats;
try {
const stats = fs.statSync(targetPath);
if (stats.size >= this.maxFileSizeBytes) {
partIndex += 1;
baseFileName = `${date}-part${partIndex}.csv`;
targetPath = path.join(clientDir, baseFileName);
} else {
stats = fs.statSync(targetPath);
} catch (err: unknown) {
if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') {
break;
}
} catch {
throw err;
}
const overflowsLimit =
stats.size > 0 &&
incomingBytes > 0 &&
stats.size + incomingBytes > this.maxFileSizeBytes;
if (stats.size >= this.maxFileSizeBytes || overflowsLimit) {
partIndex += 1;
baseFileName = `${date}-part${partIndex}.csv`;
targetPath = path.join(clientDir, baseFileName);
} else {
break;
}
}
@@ -280,49 +378,63 @@ export class CSVStorageManager {
return { written: 0, filesUpdated: [] };
}
// Group events by client + date
const grouped = new Map<string, DataEvent[]>();
// Group events by client + date without ambiguous delimiter splitting
const grouped = new Map<string, { clientId: string; date: string; events: DataEvent[] }>();
for (const evt of events) {
const date = this.extractSafePartitionDate(evt.timestamp);
const key = `${evt.clientId}:::${date}`;
if (!grouped.has(key)) {
grouped.set(key, []);
const clientId = String(evt.clientId || 'unknown');
const key = JSON.stringify([clientId, date]);
let bucket = grouped.get(key);
if (!bucket) {
bucket = { clientId, date, events: [] };
grouped.set(key, bucket);
}
grouped.get(key)!.push(evt);
bucket.events.push(evt);
}
const filesUpdated: string[] = [];
const fields = ['clientId', 'eventId', 'timestamp', 'payload'];
for (const [key, batch] of grouped.entries()) {
const [clientId, date] = key.split(':::');
for (const { clientId, date, events: batch } of grouped.values()) {
const cleanBatch = batch.map((item) => ({
clientId: item.clientId,
eventId: item.eventId,
timestamp: item.timestamp,
payload:
typeof item.payload === 'string'
? item.payload
: JSON.stringify(item.payload),
}));
const bodyCsvNoHeader = new Parser({ fields, header: false, eol: '\n' }).parse(
cleanBatch,
);
const incomingBytes = Buffer.byteLength(bodyCsvNoHeader, 'utf8') + 1;
const targetPath = await this.withPartitionLock(clientId, date, () => {
const resolvedPath = this.getTargetFilePath(clientId, date);
let hasContent = false;
const resolvedPath = this.getTargetFilePath(clientId, date, incomingBytes);
let sizeBytes = 0;
try {
hasContent =
fs.existsSync(resolvedPath) && fs.statSync(resolvedPath).size > 0;
if (fs.existsSync(resolvedPath)) {
sizeBytes = fs.statSync(resolvedPath).size;
}
} catch {
hasContent = false;
sizeBytes = 0;
}
const cleanBatch = batch.map((item) => ({
clientId: item.clientId,
eventId: item.eventId,
timestamp: item.timestamp,
payload:
typeof item.payload === 'string'
? item.payload
: JSON.stringify(item.payload),
}));
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
const csvData = parser.parse(cleanBatch);
const hasContent = sizeBytes > 0;
if (hasContent) {
fs.appendFileSync(resolvedPath, '\n' + csvData, 'utf8');
const endsWithNl = this.fileEndsWithNewline(resolvedPath, sizeBytes);
fs.appendFileSync(
resolvedPath,
(endsWithNl ? '' : '\n') + bodyCsvNoHeader,
'utf8',
);
} else {
fs.writeFileSync(resolvedPath, csvData, 'utf8');
const fullCsv = new Parser({ fields, header: true, eol: '\n' }).parse(
cleanBatch,
);
fs.writeFileSync(resolvedPath, fullCsv, 'utf8');
}
// Invalidate row cache for modified file immediately
@@ -351,13 +463,25 @@ export class CSVStorageManager {
const clientDirs = fs.readdirSync(this.baseStorageDir);
for (const clientDir of clientDirs) {
const fullClientPath = path.join(this.baseStorageDir, clientDir);
if (!fs.statSync(fullClientPath).isDirectory()) continue;
let dirLstat: fs.Stats;
try {
dirLstat = fs.lstatSync(fullClientPath);
} catch {
continue;
}
if (dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
const files = fs.readdirSync(fullClientPath);
for (const file of files) {
if (!file.endsWith('.csv')) continue;
const fullFilePath = path.join(fullClientPath, file);
const stats = fs.statSync(fullFilePath);
let stats: fs.Stats;
try {
stats = fs.lstatSync(fullFilePath);
} catch {
continue;
}
if (stats.isSymbolicLink() || !stats.isFile()) continue;
activePaths.add(fullFilePath);
const cached = this.rowCountCache.get(fullFilePath);
@@ -422,15 +546,15 @@ export class CSVStorageManager {
const clientDirs = await fs.promises.readdir(this.baseStorageDir);
for (const clientDir of clientDirs) {
const fullClientPath = path.join(this.baseStorageDir, clientDir);
const dirStat = await fs.promises.stat(fullClientPath).catch(() => null);
if (!dirStat || !dirStat.isDirectory()) continue;
const dirLstat = await fs.promises.lstat(fullClientPath).catch(() => null);
if (!dirLstat || dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
const files = await fs.promises.readdir(fullClientPath);
for (const file of files) {
if (!file.endsWith('.csv')) continue;
const fullFilePath = path.join(fullClientPath, file);
const stats = await fs.promises.stat(fullFilePath).catch(() => null);
if (!stats) continue;
const stats = await fs.promises.lstat(fullFilePath).catch(() => null);
if (!stats || stats.isSymbolicLink() || !stats.isFile()) continue;
activePaths.add(fullFilePath);
const cached = this.rowCountCache.get(fullFilePath);
@@ -481,7 +605,7 @@ export class CSVStorageManager {
}
/**
* Prune CSV partitions older than autoPruneDays without counting row contents
* Prune CSV partitions older than autoPruneDays under cooperative partition locks
*/
public static pruneExpiredPartitions(autoPruneDays: number): number {
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
@@ -502,7 +626,13 @@ export class CSVStorageManager {
const clientDirs = fs.readdirSync(this.baseStorageDir);
for (const clientDir of clientDirs) {
const fullClientPath = path.join(this.baseStorageDir, clientDir);
if (!fs.statSync(fullClientPath).isDirectory()) continue;
let dirLstat: fs.Stats;
try {
dirLstat = fs.lstatSync(fullClientPath);
} catch {
continue;
}
if (dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
const files = fs.readdirSync(fullClientPath);
for (const file of files) {
@@ -510,6 +640,45 @@ export class CSVStorageManager {
const datePart = file.replace('.csv', '').split('-part')[0];
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
const fullPath = path.join(fullClientPath, file);
let fileLstat: fs.Stats;
try {
fileLstat = fs.lstatSync(fullPath);
} catch {
continue;
}
if (fileLstat.isSymbolicLink() || !fileLstat.isFile()) continue;
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
let acquired = false;
try {
const fd = fs.openSync(lockPath, 'wx');
try {
fs.writeFileSync(fd, lockValue, 'utf8');
} finally {
fs.closeSync(fd);
}
acquired = true;
} catch (err: unknown) {
if ((err as NodeJS.ErrnoException)?.code === 'EEXIST') {
this.removeStaleLockAtomically(lockPath);
try {
const fd = fs.openSync(lockPath, 'wx');
try {
fs.writeFileSync(fd, lockValue, 'utf8');
} finally {
fs.closeSync(fd);
}
acquired = true;
} catch {
// Partition is currently locked by an active writer; skip safely
acquired = false;
}
}
}
if (!acquired) continue;
try {
if (fs.existsSync(fullPath)) {
fs.unlinkSync(fullPath);
@@ -518,6 +687,8 @@ export class CSVStorageManager {
}
} catch {
// Ignore file removal error
} finally {
this.releaseLockIfOwned(lockPath, lockValue);
}
}
}
@@ -529,7 +700,96 @@ export class CSVStorageManager {
return prunedCount;
}
private static formatStorageStats(files: StorageFileMeta[]) {
/**
* Non-blocking async partition pruner using fs.promises under cooperative partition locks
*/
public static async pruneExpiredPartitionsAsync(
autoPruneDays: number,
): Promise<number> {
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
return 0;
}
const baseStat = await fs.promises.lstat(this.baseStorageDir).catch(() => null);
if (!baseStat || !baseStat.isDirectory()) {
return 0;
}
const cutoffDate = new Date(
Date.now() - Math.floor(autoPruneDays) * 86_400_000,
)
.toISOString()
.split('T')[0];
let prunedCount = 0;
try {
const clientDirs = await fs.promises.readdir(this.baseStorageDir);
for (const clientDir of clientDirs) {
const fullClientPath = path.join(this.baseStorageDir, clientDir);
const dirLstat = await fs.promises.lstat(fullClientPath).catch(() => null);
if (!dirLstat || dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) {
continue;
}
const files = await fs.promises.readdir(fullClientPath).catch(() => [] as string[]);
for (const file of files) {
if (!file.endsWith('.csv')) continue;
const datePart = file.replace('.csv', '').split('-part')[0];
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
const fullPath = path.join(fullClientPath, file);
const fileLstat = await fs.promises.lstat(fullPath).catch(() => null);
if (!fileLstat || fileLstat.isSymbolicLink() || !fileLstat.isFile()) {
continue;
}
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
let acquired = false;
const tryAcquireLock = async (): Promise<boolean> => {
const handle = await fs.promises.open(lockPath, 'wx');
try {
await handle.writeFile(lockValue, 'utf8');
} finally {
await handle.close();
}
return true;
};
try {
acquired = await tryAcquireLock();
} catch (err: unknown) {
if ((err as NodeJS.ErrnoException)?.code === 'EEXIST') {
await this.removeStaleLockAtomicallyAsync(lockPath);
try {
acquired = await tryAcquireLock();
} catch {
acquired = false;
}
}
}
if (!acquired) continue;
try {
await fs.promises.unlink(fullPath);
this.rowCountCache.delete(fullPath);
prunedCount += 1;
} catch {
// Ignore individual unlink error or concurrent removal
} finally {
await this.releaseLockIfOwnedAsync(lockPath, lockValue);
}
}
}
}
} catch {
// Ignore directory scan failure
}
return prunedCount;
}
public static formatStorageStats(files: StorageFileMeta[]) {
const totalBytes = files.reduce((acc, f) => acc + f.sizeBytes, 0);
const totalRows = files.reduce((acc, f) => acc + f.linesCount, 0);
const uniqueClients = new Set(files.map((f) => f.clientId)).size;
@@ -569,17 +829,21 @@ export class CSVStorageManager {
* Reads file content for export / download with path traversal protection
*/
public static getFileContent(clientId: string, filename: string): string | null {
const safeClientId = String(clientId || '').replace(/[^a-zA-Z0-9_-]/g, '_');
const safeClientId = this.sanitizeClientId(clientId);
const safeFilename = path.basename(String(filename || ''));
if (!safeClientId || !safeFilename || !safeFilename.endsWith('.csv')) {
return null;
}
const fullPath = path.resolve(this.baseStorageDir, safeClientId, safeFilename);
const resolvedBase = path.resolve(this.baseStorageDir);
const fullPath = path.resolve(resolvedBase, safeClientId, safeFilename);
if (!fullPath.startsWith(resolvedBase + path.sep)) {
return null;
}
if (fs.existsSync(fullPath)) {
const stats = fs.lstatSync(fullPath);
if (stats.isSymbolicLink() || !stats.isFile()) {
return null;
}
return fs.readFileSync(fullPath, 'utf8');
}
return null;
+199 -22
View File
@@ -22,11 +22,23 @@ export interface AdminSettings {
updatedAt: string;
}
export interface SettingsBridgeSyncStatus {
attempted: boolean;
ok: boolean;
synced: boolean;
status?: number;
error?: string;
checkedAt: string;
}
type SettingsGlobals = typeof globalThis & {
__dataStackAdminSettings?: AdminSettings;
__dataStackSettingsHydrated?: boolean;
__dataStackLastSettingsBridgeSync?: SettingsBridgeSyncStatus;
__dataStackLastPruneMeta?: { days: number; ranAtMs: number };
};
const PRUNE_COOLDOWN_MS = 60 * 60 * 1000;
const globals = globalThis as SettingsGlobals;
function getDefaultSettings(): AdminSettings {
@@ -53,7 +65,7 @@ function getSettingsState(): AdminSettings {
}
export class SettingsManager {
private static applyRuntimeSettings(settings: AdminSettings) {
private static applyRuntimeSettings(settings: AdminSettings, forcePrune = false) {
SecurityEngine.maxEventsPerSec = settings.maxEventsPerSec;
SecurityEngine.enableIdempotency = settings.enableIdempotency;
SecurityEngine.maxPayloadSizeBytes = settings.maxPayloadSizeKb * 1024;
@@ -68,7 +80,27 @@ export class SettingsManager {
settings.autoPruneDays > 0 &&
(!hasConfiguredDb || globals.__dataStackSettingsHydrated === true)
) {
CSVStorageManager.pruneExpiredPartitions(settings.autoPruneDays);
const prev = globals.__dataStackLastPruneMeta;
const now = Date.now();
const shouldPrune =
forcePrune ||
!prev ||
prev.days !== settings.autoPruneDays ||
now - prev.ranAtMs >= PRUNE_COOLDOWN_MS;
if (shouldPrune) {
globals.__dataStackLastPruneMeta = {
days: settings.autoPruneDays,
ranAtMs: now,
};
void CSVStorageManager.pruneExpiredPartitionsAsync(
settings.autoPruneDays,
).catch((err) => {
console.warn(
"[DataStack Storage] Background async partition prune failed:",
err,
);
});
}
}
}
@@ -94,9 +126,17 @@ export class SettingsManager {
row.auto_prune_days !== undefined && row.auto_prune_days !== null
? Number(row.auto_prune_days)
: NaN;
const envSocketPort = Number(process.env.PORT_SOCKET);
const envWebPort = Number(process.env.PORT);
const loaded: AdminSettings = {
serverPort: Number(row.server_port) || 4000,
webPort: Number(row.web_port) || 3000,
serverPort:
Number.isFinite(envSocketPort) && envSocketPort > 0
? envSocketPort
: Number(row.server_port) || 4000,
webPort:
Number.isFinite(envWebPort) && envWebPort > 0
? envWebPort
: Number(row.web_port) || 3000,
maxEventsPerSec: Number(row.max_events_per_sec) || 100,
maxPayloadSizeKb: Number(row.max_payload_size_kb) || 100,
batchFlushThreshold: Number(row.batch_flush_threshold) || 500,
@@ -117,6 +157,12 @@ export class SettingsManager {
// Initialize default row in PostgreSQL if not present yet
const defaults = getSettingsState();
const hasValidProductionJwt =
process.env.NODE_ENV !== "production" ||
defaults.jwtSecret.trim().length >= 32;
if (!hasValidProductionJwt) {
return defaults;
}
await this.persistSettingsToDb(defaults);
globals.__dataStackSettingsHydrated = true;
return defaults;
@@ -131,8 +177,23 @@ export class SettingsManager {
const pool = DatabaseEngine.getPool();
if (!pool) return;
if (
process.env.NODE_ENV === "production" &&
settings.jwtSecret.trim().length < 32
) {
throw new Error(
"DATABASE_WRITE_FAILED: Cannot persist settings without a valid DATASTACK_JWT_SECRET (>= 32 characters) in production.",
);
}
const ready = await DatabaseEngine.ensureSchema();
if (!ready) {
throw new Error(
"DATABASE_WRITE_FAILED: PostgreSQL schema is unavailable while saving settings.",
);
}
try {
await DatabaseEngine.ensureSchema();
await pool.query(
`INSERT INTO admin_settings (
id, server_port, web_port, max_events_per_sec, max_payload_size_kb,
@@ -164,8 +225,9 @@ export class SettingsManager {
settings.autoPruneDays,
],
);
} catch {
// Ignore DB failure; in-memory state remains active
} catch (err: unknown) {
const detail = err instanceof Error ? err.message : "Unknown DB error";
throw new Error(`DATABASE_WRITE_FAILED: Failed to persist settings (${detail})`);
}
}
@@ -222,20 +284,35 @@ export class SettingsManager {
};
}
public static getLastBridgeSyncStatus(): SettingsBridgeSyncStatus | null {
return globals.__dataStackLastSettingsBridgeSync ?? null;
}
private static async notifyIngestionSettingsSync(
settings: AdminSettings,
authJwtSecret: string,
jwtSecretChanged: boolean,
): Promise<void> {
): Promise<SettingsBridgeSyncStatus> {
const checkedAt = new Date().toISOString();
const headers = getInternalIngestionHeaders(authJwtSecret);
if (!headers["x-datastack-internal-secret"]) return;
if (!headers["x-datastack-internal-secret"]) {
const result: SettingsBridgeSyncStatus = {
attempted: true,
ok: false,
synced: false,
error: "Internal bridge secret is not configured",
checkedAt,
};
globals.__dataStackLastSettingsBridgeSync = result;
return result;
}
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, "");
try {
await fetch(`${ingestionBaseUrl}/internal/settings-sync`, {
const res = await fetch(`${ingestionBaseUrl}/internal/settings-sync`, {
method: "POST",
headers: {
"Content-Type": "application/json",
@@ -254,8 +331,46 @@ export class SettingsManager {
cache: "no-store",
signal: AbortSignal.timeout(1500),
});
} catch {
// Port 4000 may not be running in isolated single-process dev mode
if (!res.ok) {
const errorMsg = `Port 4000 settings-sync failed with HTTP ${res.status}`;
console.warn(`[DataStack Bridge] ${errorMsg}`);
const result: SettingsBridgeSyncStatus = {
attempted: true,
ok: false,
synced: false,
status: res.status,
error: errorMsg,
checkedAt,
};
globals.__dataStackLastSettingsBridgeSync = result;
return result;
}
const result: SettingsBridgeSyncStatus = {
attempted: true,
ok: true,
synced: true,
status: res.status,
checkedAt,
};
globals.__dataStackLastSettingsBridgeSync = result;
return result;
} catch (err: unknown) {
const message = err instanceof Error ? err.message : "Port 4000 settings-sync unreachable";
if (
process.env.NODE_ENV === "production" ||
Boolean(process.env.INGESTION_INTERNAL_URL)
) {
console.warn(`[DataStack Bridge] ${message}`);
}
const result: SettingsBridgeSyncStatus = {
attempted: true,
ok: false,
synced: false,
error: message,
checkedAt,
};
globals.__dataStackLastSettingsBridgeSync = result;
return result;
}
}
@@ -270,11 +385,13 @@ export class SettingsManager {
updatedAt: new Date().toISOString(),
};
globals.__dataStackAdminSettings = nextSettings;
this.applyRuntimeSettings(nextSettings);
this.applyRuntimeSettings(nextSettings, updates.autoPruneDays !== undefined);
if (jwtSecretChanged) {
SDKManager.resignAllClientTokens();
}
void this.persistSettingsToDb(nextSettings);
void this.persistSettingsToDb(nextSettings).catch((err) => {
console.error("[DataStack Settings] Background DB persist failed:", err);
});
void this.notifyIngestionSettingsSync(
nextSettings,
previousJwtSecret,
@@ -297,20 +414,80 @@ export class SettingsManager {
...updates,
updatedAt: new Date().toISOString(),
};
globals.__dataStackAdminSettings = nextSettings;
this.applyRuntimeSettings(nextSettings);
if (jwtSecretChanged) {
await SDKManager.resignAllClientTokensAsync();
if (
jwtSecretChanged &&
process.env.NODE_ENV === "production" &&
nextSettings.jwtSecret.trim().length < 32
) {
throw new Error(
"INVALID_JWT_SECRET: JWT secret must be at least 32 characters in production.",
);
}
if (jwtSecretChanged) {
// BUG-17: Coordinate Port 4000 BEFORE committing JWT rotation on Port 3000
// so a bridge rejection or unreachable production ingestion server never causes secret split-brain.
const preSyncStatus = await this.notifyIngestionSettingsSync(
nextSettings,
previousJwtSecret,
true,
);
if (!preSyncStatus.synced) {
const isExplicitHttpRejection = typeof preSyncStatus.status === "number";
const isProductionBridgeRequired =
process.env.NODE_ENV === "production" &&
Boolean(process.env.INGESTION_INTERNAL_URL);
if (isExplicitHttpRejection || isProductionBridgeRequired) {
throw new Error(
`JWT_ROTATION_BRIDGE_FAILED: Aborted JWT rotation because Port 4000 could not synchronize (${preSyncStatus.error || "bridge error"}).`,
);
}
}
try {
await this.persistSettingsToDb(nextSettings);
globals.__dataStackAdminSettings = nextSettings;
this.applyRuntimeSettings(nextSettings, updates.autoPruneDays !== undefined);
await SDKManager.resignAllClientTokensAsync();
} catch (err) {
globals.__dataStackAdminSettings = current;
this.applyRuntimeSettings(current, false);
await this.persistSettingsToDb(current).catch(() => undefined);
if (preSyncStatus.synced) {
await this.notifyIngestionSettingsSync(
current,
nextSettings.jwtSecret,
true,
).catch(() => undefined);
}
throw err;
}
const clientSync = await SDKManager.syncAllClientsToIngestionAsync();
if (!clientSync.synced && preSyncStatus.synced) {
globals.__dataStackLastSettingsBridgeSync = {
attempted: true,
ok: false,
synced: false,
status: clientSync.status,
error: clientSync.error,
checkedAt: new Date().toISOString(),
};
}
return { ...nextSettings };
}
await this.persistSettingsToDb(nextSettings);
globals.__dataStackAdminSettings = nextSettings;
this.applyRuntimeSettings(nextSettings, updates.autoPruneDays !== undefined);
await this.notifyIngestionSettingsSync(
nextSettings,
previousJwtSecret,
jwtSecretChanged,
false,
);
if (jwtSecretChanged) {
await SDKManager.syncAllClientsToIngestionAsync();
}
return { ...nextSettings };
}
}
+64 -16
View File
@@ -34,6 +34,11 @@ export class ApiEngine {
).replace(/\/$/, '');
const internalHeaders = getInternalIngestionHeaders();
let ingestionBridgeStatus: 'online' | 'auth_error' | 'error' | 'offline' = 'offline';
let ingestionBridgeError: string | undefined;
let liveServerPort = settings.serverPort || 4000;
let liveWebPort = settings.webPort || 3000;
try {
const res = await fetch(`${ingestionBaseUrl}/internal/metrics`, {
headers: internalHeaders,
@@ -42,8 +47,17 @@ export class ApiEngine {
});
if (res.ok) {
const data = await res.json();
ingestionBridgeStatus = 'online';
activeSockets = Number(data.activeSockets) || 0;
ingestionRedisConnected = Boolean(data.redisConnected);
if (Number.isFinite(Number(data.serverPort)) && Number(data.serverPort) > 0) {
liveServerPort = Number(data.serverPort);
} else if (Number.isFinite(Number(data.port)) && Number(data.port) > 0) {
liveServerPort = Number(data.port);
}
if (Number.isFinite(Number(data.webPort)) && Number(data.webPort) > 0) {
liveWebPort = Number(data.webPort);
}
if (data.localBufferSize !== undefined) {
ingestionLocalBufferSize = Number(data.localBufferSize) || 0;
ingestionRedisBufferSize = Number(data.redisBufferSize) || 0;
@@ -53,14 +67,36 @@ export class ApiEngine {
if (data.securityStats && typeof data.securityStats === 'object') {
remoteSecurityStats = data.securityStats;
}
// BUG-02: Auto-reconcile Port 4000 cold start whenever metrics observe unsynced registry
if (data.hasSyncedInMemoryClients === false && clients.length > 0) {
await SDKManager.syncAllClientsToIngestionAsync();
}
} else {
ingestionBridgeStatus = res.status === 401 || res.status === 503 ? 'auth_error' : 'error';
ingestionBridgeError = `HTTP ${res.status}`;
console.warn(`[DataStack Bridge] /internal/metrics failed with ${ingestionBridgeError}`);
}
} catch {
} catch (err: unknown) {
// Ingestion server may not be running in single-process dev mode
activeSockets = 0;
ingestionLocalBufferSize = 0;
ingestionRedisBufferSize = 0;
ingestionBridgeStatus = 'offline';
ingestionBridgeError = err instanceof Error ? err.message : 'Unreachable';
}
const effectiveRedisBufferSize = Math.max(
localBufferStats.redisBufferSize || 0,
ingestionRedisBufferSize,
);
const redisConnected = localBufferStats.redisConnected || ingestionRedisConnected;
const securityMode: 'redis_shared' | 'bridge_coordinated' | 'degraded_local' =
redisConnected
? 'redis_shared'
: ingestionBridgeStatus === 'online'
? 'bridge_coordinated'
: 'degraded_local';
const localSecurityStats = SecurityEngine.getStats();
const securityStats = {
activeIdempotencyKeys:
@@ -74,15 +110,12 @@ export class ApiEngine {
Number(remoteSecurityStats?.maxPayloadSizeBytes) ||
localSecurityStats.maxPayloadSizeBytes,
),
mode: securityMode,
};
const effectiveRedisBufferSize = Math.max(
localBufferStats.redisBufferSize || 0,
ingestionRedisBufferSize,
);
const bufferStats = {
...localBufferStats,
redisConnected: localBufferStats.redisConnected || ingestionRedisConnected,
redisConnected,
redisBufferSize: effectiveRedisBufferSize,
currentBufferSize:
localBufferStats.localBufferSize +
@@ -98,8 +131,11 @@ export class ApiEngine {
storageStats,
securityStats,
runtime: {
serverPort: settings.serverPort || 4000,
webPort: settings.webPort || 3000,
serverPort: liveServerPort,
webPort: liveWebPort,
ingestionBridgeStatus,
...(ingestionBridgeError ? { ingestionBridgeError } : {}),
securityMode,
uptimeSeconds: Math.floor(process.uptime()),
memoryUsageMb: (process.memoryUsage().rss / (1024 * 1024)).toFixed(1),
nodeVersion: process.version,
@@ -107,23 +143,35 @@ export class ApiEngine {
};
}
private static filterFilesByClientId<T extends { clientId: string }>(
files: T[],
clientId?: string,
): T[] {
if (!clientId || clientId === 'all') return files;
const normalized = SDKManager.normalizeClientId(clientId);
if (!normalized) return files;
const sanitized = CSVStorageManager.sanitizeClientId(normalized);
return files.filter((f) => {
const folderNorm = SDKManager.normalizeClientId(f.clientId);
return (
folderNorm === normalized ||
f.clientId === normalized ||
f.clientId === sanitized
);
});
}
/**
* Filters and lists stored CSV files for export
*/
public static getExportFiles(clientId?: string) {
const allFiles = CSVStorageManager.listAllFiles();
if (clientId && clientId !== 'all') {
return allFiles.filter((f) => f.clientId === clientId);
}
return allFiles;
return this.filterFilesByClientId(allFiles, clientId);
}
public static async getExportFilesAsync(clientId?: string) {
const allFiles = await CSVStorageManager.listAllFilesAsync();
if (clientId && clientId !== 'all') {
return allFiles.filter((f) => f.clientId === clientId);
}
return allFiles;
return this.filterFilesByClientId(allFiles, clientId);
}
}
+1 -4
View File
@@ -113,11 +113,8 @@ export function getSessionCookieOptions(request: Request, maxAge: number) {
?.split(",")[0]
?.trim()
.toLowerCase();
const originHeader = request.headers.get("origin")?.toLowerCase() ?? "";
const isHttps =
forwardedProto === "https" ||
originHeader.startsWith("https://") ||
request.url.startsWith("https://");
forwardedProto === "https" || request.url.startsWith("https://");
if (isHttps) {
return {
+62 -53
View File
@@ -5,12 +5,13 @@ import {
scrypt as scryptCallback,
timingSafeEqual,
} from "node:crypto";
import { Pool, type PoolClient } from "pg";
import type { Pool, PoolClient } from "pg";
import {
type AdminSessionPayload,
getAdminSessionPayload,
getAdminSessionSecret,
} from "./adminSession";
import { DatabaseEngine, ensureCoreSchemaOnPool } from "../Database/database";
const PASSWORD_HASH_BYTES = 64;
const PASSWORD_SALT_BYTES = 16;
@@ -50,7 +51,6 @@ interface InMemoryAdmin {
}
type AdminDatabaseGlobals = typeof globalThis & {
__dataStackAdminPool?: Pool | null;
__dataStackAdminSchema?: Promise<void>;
__dataStackInMemoryAdmin?: InMemoryAdmin;
};
@@ -60,67 +60,51 @@ const localLoginThrottle = new Map<string, LoginThrottleEntry>();
let lastLocalThrottleCleanup = 0;
let lastDatabaseThrottleCleanup = 0;
export class AdminSetupRequiredError extends Error {
public readonly code = "SETUP_REQUIRED";
constructor(message = "Admin account has not completed initial setup.") {
super(message);
this.name = "AdminSetupRequiredError";
}
}
export class AdminDatabaseRequiredError extends Error {
public readonly code = "DATABASE_NOT_CONFIGURED";
constructor(
message = "PostgreSQL DATABASE_URL is required for initial admin setup in production.",
) {
super(message);
this.name = "AdminDatabaseRequiredError";
}
}
function isProductionEnv() {
return process.env.NODE_ENV === "production";
}
export function isAdminDatabaseConfigured(): boolean {
return Boolean(getPool());
}
export function requiresPersistentAdminDatabase(): boolean {
return (
isProductionEnv() &&
!getPool() &&
!process.env.DATASTACK_ADMIN_PASSWORD
);
}
function getPool(): Pool | null {
const connectionString = process.env.DATABASE_URL?.trim();
if (!connectionString) {
return null;
}
if (globals.__dataStackAdminPool === undefined) {
try {
globals.__dataStackAdminPool = new Pool({
connectionString,
connectionTimeoutMillis: 3000,
idleTimeoutMillis: 30000,
max: 5,
});
} catch {
globals.__dataStackAdminPool = null;
}
}
return globals.__dataStackAdminPool ?? null;
return DatabaseEngine.getPool();
}
async function ensureAdminTable() {
const pool = getPool();
if (!pool) return;
globals.__dataStackAdminSchema ??= Promise.all([
pool
.query(
`
CREATE TABLE IF NOT EXISTS admin_users (
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
email TEXT NOT NULL UNIQUE,
password_salt BYTEA NOT NULL,
password_hash BYTEA NOT NULL,
token_version INTEGER NOT NULL DEFAULT 1,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`,
)
.then(() =>
pool.query(
`ALTER TABLE admin_users ADD COLUMN IF NOT EXISTS token_version INTEGER NOT NULL DEFAULT 1`,
),
),
pool.query(
`
CREATE TABLE IF NOT EXISTS admin_login_attempts (
key_hash TEXT PRIMARY KEY,
window_started_at TIMESTAMPTZ NOT NULL,
failed_attempts INTEGER NOT NULL,
blocked_until TIMESTAMPTZ,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`,
),
]).then(() => undefined);
globals.__dataStackAdminSchema ??= ensureCoreSchemaOnPool(pool);
try {
await globals.__dataStackAdminSchema;
@@ -342,7 +326,7 @@ async function getOrInitInMemoryAdmin(): Promise<InMemoryAdmin> {
!isProductionEnv() && process.env.DATASTACK_ALLOW_DEMO_ADMIN !== "false";
if (!envPassword && !allowDemo) {
throw new Error("Admin account has not completed initial setup.");
throw new AdminSetupRequiredError();
}
const defaultEmail = (
@@ -443,6 +427,9 @@ export async function createInitialAdmin(email: string, password: string) {
const pool = getPool();
if (!pool) {
if (isProductionEnv()) {
throw new AdminDatabaseRequiredError();
}
if (globals.__dataStackInMemoryAdmin) {
return false;
}
@@ -671,6 +658,28 @@ export async function updateAdminAccount({
return { ok: true as const, email: cleanEmail, tokenVersion: nextVersion };
}
export async function revokeAdminSessions(): Promise<number> {
const pool = getPool();
if (pool) {
await ensureAdminTable();
const updated = await pool.query(
`UPDATE admin_users
SET token_version = COALESCE(token_version, 1) + 1
WHERE id = 1
RETURNING token_version`,
);
if (!updated.rowCount || updated.rowCount === 0) {
return 0;
}
return Number(updated.rows[0].token_version) || 1;
}
const existing = globals.__dataStackInMemoryAdmin ?? (await getOrInitInMemoryAdmin());
existing.tokenVersion += 1;
globals.__dataStackInMemoryAdmin = existing;
return existing.tokenVersion;
}
export function resetAdminStateForTests() {
globals.__dataStackInMemoryAdmin = undefined;
localLoginThrottle.clear();
+517 -83
View File
@@ -19,6 +19,15 @@ export interface RedisBufferClient {
ttlMs: number,
nxMode: 'NX',
): Promise<string | null>;
get?(key: string): Promise<string | null>;
incr?(key: string): Promise<number>;
pexpire?(key: string, ttlMs: number): Promise<number>;
hget?(key: string, field: string): Promise<string | null>;
hset?(key: string, field: string, value: string): Promise<number>;
hdel?(key: string, ...fields: string[]): Promise<number>;
sismember?(key: string, member: string): Promise<number>;
sadd?(key: string, ...members: string[]): Promise<number>;
srem?(key: string, ...members: string[]): Promise<number>;
eval(script: string, numKeys: number, ...args: string[]): Promise<unknown>;
quit?: () => Promise<unknown>;
disconnect?: () => void;
@@ -27,10 +36,83 @@ export interface RedisBufferClient {
export const REDIS_BUFFER_KEY = 'datastack:buffer:events';
export const REDIS_RECENT_KEY = 'datastack:recent:events';
export const REDIS_FLUSH_LOCK_KEY = 'datastack:flush:lock';
const REDIS_RECENT_LIMIT = 500;
const FLUSH_LOCK_TTL_MS = 15000;
const REDIS_CLIENTS_DELETED_KEY = 'datastack:clients:deleted';
export const DEFAULT_REDIS_RECENT_LIMIT = 500;
export const MAX_REDIS_RECENT_LIMIT = 5000;
const FLUSH_LOCK_TTL_MS = 30000;
const FLUSH_LOCK_WAIT_MS = 400;
const FLUSH_LOCK_RETRY_MS = 25;
const UNLOCK_LUA_SCRIPT =
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end';
const RENEW_LOCK_LUA_SCRIPT =
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("pexpire", KEYS[1], ARGV[2]) else return 0 end';
export const PURGE_CLIENT_LIST_LUA_SCRIPT = `
-- DATASTACK_PURGE_CLIENT_LIST
local key = KEYS[1]
local target = ARGV[1]
local items = redis.call("LRANGE", key, 0, -1)
local removed = 0
local kept = {}
for i = 1, #items do
local raw = items[i]
local ok, obj = pcall(cjson.decode, raw)
local cid = ""
if ok and type(obj) == "table" and obj.clientId ~= nil then
cid = string.lower(tostring(obj.clientId))
cid = string.gsub(cid, "^%s*(.-)%s*$", "%1")
cid = string.gsub(cid, "[^a-z0-9_-]", "_")
end
if cid == target then
removed = removed + 1
else
kept[#kept + 1] = raw
end
end
if removed > 0 then
redis.call("DEL", key)
local chunk = 500
for i = 1, #kept, chunk do
local stop = math.min(i + chunk - 1, #kept)
redis.call("RPUSH", key, unpack(kept, i, stop))
end
end
return removed
`.trim();
export function resolveRedisRecentLimit(batchFlushThreshold?: number): number {
const envRaw = Number(process.env.DATASTACK_RECENT_LIMIT);
if (Number.isFinite(envRaw) && envRaw >= 100) {
return Math.min(MAX_REDIS_RECENT_LIMIT, Math.floor(envRaw));
}
const threshold = Number(batchFlushThreshold) || DEFAULT_REDIS_RECENT_LIMIT;
return Math.min(
MAX_REDIS_RECENT_LIMIT,
Math.max(DEFAULT_REDIS_RECENT_LIMIT, Math.floor(threshold)),
);
}
export interface FlushOptions {
waitMs?: number;
lockTtlMs?: number;
throwOnLockTimeout?: boolean;
}
export interface FlushResult {
count: number;
files: string[];
lockTimedOut?: boolean;
}
function sleepMs(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function normalizeBufferClientId(clientId: string): string {
return String(clientId || '')
.trim()
.toLowerCase()
.replace(/[^a-z0-9_-]/g, '_');
}
interface BufferGlobalState {
buffer: DataEvent[];
@@ -43,11 +125,14 @@ interface BufferGlobalState {
totalIngested: number;
totalFlushed: number;
isFlushing: boolean;
activeFlushPromise: Promise<FlushResult> | null;
redisClient: RedisBufferClient | null | undefined;
redisUrlConfigured: string;
redisConnected: boolean;
redisTestOverride: RedisBufferClient | null | undefined;
pendingRedisWrites: Set<Promise<void>>;
deletedClientIds: Set<string>;
shutdownHooks: Set<() => Promise<unknown>>;
}
type BufferGlobals = typeof globalThis & {
@@ -71,13 +156,22 @@ function getState(): BufferGlobalState {
totalIngested: 0,
totalFlushed: 0,
isFlushing: false,
activeFlushPromise: null,
redisClient: undefined,
redisUrlConfigured: '',
redisConnected: false,
redisTestOverride: undefined,
pendingRedisWrites: new Set(),
deletedClientIds: new Set(),
shutdownHooks: new Set(),
};
}
if (!globals.__dataStackBufferState.deletedClientIds) {
globals.__dataStackBufferState.deletedClientIds = new Set();
}
if (!globals.__dataStackBufferState.shutdownHooks) {
globals.__dataStackBufferState.shutdownHooks = new Set();
}
return globals.__dataStackBufferState;
}
@@ -150,7 +244,9 @@ function getRedisClient(): RedisBufferClient | null {
}
export class DataBufferEngine {
private static maxRecentEvents = REDIS_RECENT_LIMIT;
public static getRecentLimit(): number {
return resolveRedisRecentLimit(getState().batchFlushThreshold);
}
public static get batchFlushThreshold(): number {
return getState().batchFlushThreshold;
@@ -174,6 +270,10 @@ export class DataBufferEngine {
state.redisConnected = Boolean(client);
}
public static getRedisClient(): RedisBufferClient | null {
return getRedisClient();
}
public static async shutdown(): Promise<void> {
const state = getState();
if (state.flushTimer) {
@@ -188,6 +288,7 @@ export class DataBufferEngine {
state.redisUrlConfigured = '';
state.redisConnected = false;
state.redisTestOverride = undefined;
state.deletedClientIds.clear();
if (client) {
try {
if (typeof client.quit === 'function') {
@@ -203,6 +304,14 @@ export class DataBufferEngine {
}
}
public static registerShutdownHook(hook: () => Promise<unknown>): () => void {
const state = getState();
state.shutdownHooks.add(hook);
return () => {
state.shutdownHooks.delete(hook);
};
}
public static async flushAndShutdownOnSignal(): Promise<void> {
if (globals.__dataStackBufferShuttingDown) {
return globals.__dataStackBufferShuttingDown;
@@ -210,8 +319,16 @@ export class DataBufferEngine {
const run = (async () => {
try {
const hooks = [...getState().shutdownHooks];
await Promise.race([
this.flush().catch(() => undefined),
Promise.allSettled([
this.flush().catch(() => undefined),
...hooks.map((hook) =>
Promise.resolve()
.then(() => hook())
.catch(() => undefined),
),
]),
new Promise<void>((resolve) => setTimeout(resolve, 4000)),
]);
} finally {
@@ -278,7 +395,8 @@ export class DataBufferEngine {
const state = getState();
state.totalIngested += 1;
state.recentEvents.unshift(event);
if (state.recentEvents.length > this.maxRecentEvents) {
const recentLimit = this.getRecentLimit();
while (state.recentEvents.length > recentLimit) {
state.recentEvents.pop();
}
@@ -298,15 +416,25 @@ export class DataBufferEngine {
if (redis) {
const serialized = events.map((e) => JSON.stringify(e));
let bufferLen: number | null = null;
try {
const bufferLen = await redis.rpush(REDIS_BUFFER_KEY, ...serialized);
await redis.lpush(REDIS_RECENT_KEY, ...serialized);
await redis.ltrim(REDIS_RECENT_KEY, 0, REDIS_RECENT_LIMIT - 1);
bufferLen = await redis.rpush(REDIS_BUFFER_KEY, ...serialized);
state.redisConnected = true;
return Number(bufferLen) + state.buffer.length;
} catch {
state.redisConnected = false;
}
if (bufferLen !== null) {
// Best-effort recent list mirror: never duplicate into RAM buffer once rpush succeeded
try {
const recentLimit = this.getRecentLimit();
await redis.lpush(REDIS_RECENT_KEY, ...serialized);
await redis.ltrim(REDIS_RECENT_KEY, 0, recentLimit - 1);
} catch {
// Non-fatal: events are already committed in REDIS_BUFFER_KEY and state.recentEvents
}
return Number(bufferLen) + state.buffer.length;
}
}
state.buffer.push(...events);
@@ -363,117 +491,410 @@ export class DataBufferEngine {
}
}
/**
* Flushes queued events (shared Redis list under distributed lock + local RAM fallback) to partitioned CSV files.
* Redis LTRIM runs ONLY after CSVStorageManager.appendBatch succeeds so crashes never lose data.
*/
public static async flush(): Promise<{ count: number; files: string[] }> {
private static async isClientDeletedForBuffer(
clientId: string,
redis: RedisBufferClient | null,
cache?: Map<string, boolean>,
): Promise<boolean> {
const normalized = normalizeBufferClientId(clientId);
if (!normalized) return false;
const state = getState();
if (state.deletedClientIds.has(normalized)) return true;
if (cache && cache.has(normalized)) return Boolean(cache.get(normalized));
if (redis && typeof redis.sismember === 'function') {
try {
const isDeleted =
Number(await redis.sismember(REDIS_CLIENTS_DELETED_KEY, normalized)) === 1;
if (cache) cache.set(normalized, isDeleted);
if (isDeleted) state.deletedClientIds.add(normalized);
return isDeleted;
} catch {
// Ignore transient Redis tombstone lookup error
}
}
if (cache) cache.set(normalized, false);
return false;
}
/**
* Marks a client deleted in the buffer layer and immediately strips queued local events.
*/
public static markClientDeleted(clientId: string): void {
const normalized = normalizeBufferClientId(clientId);
if (!normalized) return;
const state = getState();
state.deletedClientIds.add(normalized);
state.buffer = state.buffer.filter(
(evt) => normalizeBufferClientId(evt.clientId) !== normalized,
);
state.recentEvents = state.recentEvents.filter(
(evt) => normalizeBufferClientId(evt.clientId) !== normalized,
);
}
/**
* Clears deleted tombstone when a client ID is explicitly re-registered.
*/
public static unmarkClientDeleted(clientId: string): void {
const normalized = normalizeBufferClientId(clientId);
if (!normalized) return;
getState().deletedClientIds.delete(normalized);
}
private static async filterRedisListExcludingClient(
redis: RedisBufferClient,
listKey: string,
normalizedClientId: string,
): Promise<number> {
const removed = await redis.eval(
PURGE_CLIENT_LIST_LUA_SCRIPT,
1,
listKey,
normalizedClientId,
);
return Number(removed) || 0;
}
/**
* Purges all queued events for a deleted client from both local RAM and shared Redis queues (BUG-19).
*/
public static async purgeClientEventsAsync(
clientId: string,
): Promise<{ removedLocal: number; removedRedis: number }> {
const normalized = normalizeBufferClientId(clientId);
if (!normalized) return { removedLocal: 0, removedRedis: 0 };
const state = getState();
state.deletedClientIds.add(normalized);
if (state.pendingRedisWrites.size > 0) {
await Promise.allSettled([...state.pendingRedisWrites]);
}
if (state.isFlushing) {
return { count: 0, files: [] };
}
state.isFlushing = true;
let totalWritten = 0;
const filesSet = new Set<string>();
try {
const redis = getRedisClient();
if (redis) {
const lockToken = crypto.randomUUID();
let lockAcquired = false;
const beforeLocal = state.buffer.length;
state.buffer = state.buffer.filter(
(evt) => normalizeBufferClientId(evt.clientId) !== normalized,
);
const removedLocal = beforeLocal - state.buffer.length;
state.recentEvents = state.recentEvents.filter(
(evt) => normalizeBufferClientId(evt.clientId) !== normalized,
);
let removedRedis = 0;
const redis = getRedisClient();
if (redis) {
const lockToken = crypto.randomUUID();
let lockAcquired = false;
const deadline = Date.now() + 1500;
while (true) {
try {
const lockResult = await redis.set(
const lockRes = await redis.set(
REDIS_FLUSH_LOCK_KEY,
lockToken,
'PX',
FLUSH_LOCK_TTL_MS,
'NX',
);
lockAcquired = lockResult === 'OK';
state.redisConnected = true;
if (lockRes === 'OK') {
lockAcquired = true;
break;
}
} catch {
state.redisConnected = false;
lockAcquired = false;
break;
}
const remaining = deadline - Date.now();
if (remaining <= 0) break;
await sleepMs(Math.min(FLUSH_LOCK_RETRY_MS, remaining));
}
if (lockAcquired) {
try {
removedRedis += await this.filterRedisListExcludingClient(
redis,
REDIS_BUFFER_KEY,
normalized,
);
await this.filterRedisListExcludingClient(
redis,
REDIS_RECENT_KEY,
normalized,
).catch(() => 0);
} catch {
// Flush-time deletedClientIds/tombstone filter remains active as secondary guard
} finally {
try {
await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
} catch {
// Lock expires via PX TTL
}
}
}
}
return { removedLocal, removedRedis };
}
/**
* Flushes queued events (shared Redis list under distributed lock + local RAM fallback) to partitioned CSV files.
* Redis LTRIM runs ONLY after CSVStorageManager.appendBatch succeeds and lock ownership is verified.
*/
public static async flush(options?: FlushOptions): Promise<FlushResult> {
const state = getState();
if (state.pendingRedisWrites.size > 0) {
await Promise.allSettled([...state.pendingRedisWrites]);
}
if (state.activeFlushPromise) {
return state.activeFlushPromise;
}
const executeFlush = async (): Promise<FlushResult> => {
state.isFlushing = true;
let totalWritten = 0;
let lockTimedOut = false;
const filesSet = new Set<string>();
const deletedCheckCache = new Map<string, boolean>();
try {
if (state.pendingRedisWrites.size > 0) {
await Promise.allSettled([...state.pendingRedisWrites]);
}
if (lockAcquired) {
try {
const chunkSize = Math.max(500, state.batchFlushThreshold);
while (true) {
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, chunkSize - 1);
if (!rawItems || rawItems.length === 0) {
const redis = getRedisClient();
if (redis) {
const lockToken = crypto.randomUUID();
const lockTtlMs = Math.max(50, Number(options?.lockTtlMs) || FLUSH_LOCK_TTL_MS);
const waitMs = Math.max(
0,
options?.waitMs !== undefined ? Number(options.waitMs) : FLUSH_LOCK_WAIT_MS,
);
const deadline = Date.now() + waitMs;
let lockAcquired = false;
let redisError = false;
while (true) {
try {
const lockResult = await redis.set(
REDIS_FLUSH_LOCK_KEY,
lockToken,
'PX',
lockTtlMs,
'NX',
);
state.redisConnected = true;
if (lockResult === 'OK') {
lockAcquired = true;
break;
}
} catch {
state.redisConnected = false;
redisError = true;
break;
}
const parsedBatch: DataEvent[] = [];
for (const raw of rawItems) {
const remaining = deadline - Date.now();
if (remaining <= 0) {
break;
}
await sleepMs(Math.min(FLUSH_LOCK_RETRY_MS, remaining));
}
if (!lockAcquired && !redisError) {
lockTimedOut = true;
if (options?.throwOnLockTimeout === true) {
throw new Error(
'FLUSH_LOCK_TIMEOUT: Timed out waiting for distributed Redis flush lock',
);
}
}
if (lockAcquired) {
const renewIntervalMs = Math.max(20, Math.floor(lockTtlMs / 3));
let lastRenewAt = Date.now();
let lockLost = false;
let renewInFlight: Promise<boolean> | null = null;
const renewLockOnce = (): Promise<boolean> => {
if (renewInFlight) return renewInFlight;
renewInFlight = (async () => {
try {
const evt = JSON.parse(raw) as DataEvent;
if (evt && evt.eventId) {
parsedBatch.push(evt);
const renewed = await redis.eval(
RENEW_LOCK_LUA_SCRIPT,
1,
REDIS_FLUSH_LOCK_KEY,
lockToken,
String(lockTtlMs),
);
if (Number(renewed) === 1) {
lastRenewAt = Date.now();
return true;
}
lockLost = true;
return false;
} catch {
// Ignore malformed legacy entry while still trimming it
return false;
} finally {
renewInFlight = null;
}
})();
return renewInFlight;
};
const renewTimer = setInterval(() => {
void renewLockOnce();
}, renewIntervalMs);
if (typeof renewTimer.unref === 'function') {
renewTimer.unref();
}
try {
const chunkSize = Math.max(500, state.batchFlushThreshold);
while (true) {
if (lockLost) break;
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, chunkSize - 1);
if (!rawItems || rawItems.length === 0) {
break;
}
const parsedBatch: DataEvent[] = [];
let malformedCount = 0;
for (const raw of rawItems) {
try {
const evt = JSON.parse(raw) as DataEvent;
if (evt && evt.eventId) {
if (
await this.isClientDeletedForBuffer(
evt.clientId,
redis,
deletedCheckCache,
)
) {
continue;
}
parsedBatch.push(evt);
} else {
malformedCount += 1;
}
} catch {
malformedCount += 1;
}
}
if (malformedCount > 0) {
console.warn(
`[DataBufferEngine] Skipped ${malformedCount} malformed buffered Redis event(s) during flush`,
);
}
if (Date.now() - lastRenewAt >= renewIntervalMs) {
await renewLockOnce();
}
if (lockLost) break;
if (parsedBatch.length > 0) {
const res = await CSVStorageManager.appendBatch(parsedBatch);
totalWritten += res.written;
for (const f of res.filesUpdated) filesSet.add(f);
}
if (Date.now() - lastRenewAt >= renewIntervalMs) {
await renewLockOnce();
}
if (lockLost) {
break;
}
// Trim strictly AFTER CSVStorageManager.appendBatch resolves and lock ownership holds
await redis.ltrim(REDIS_BUFFER_KEY, rawItems.length, -1);
if (rawItems.length < chunkSize) {
break;
}
}
if (parsedBatch.length > 0) {
const res = await CSVStorageManager.appendBatch(parsedBatch);
totalWritten += res.written;
for (const f of res.filesUpdated) filesSet.add(f);
} finally {
clearInterval(renewTimer);
if (renewInFlight) {
await renewInFlight.catch(() => false);
}
// Trim strictly AFTER CSVStorageManager.appendBatch resolves safely
await redis.ltrim(REDIS_BUFFER_KEY, rawItems.length, -1);
if (rawItems.length < chunkSize) {
break;
try {
await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
} catch {
// Lock auto-expires after lockTtlMs
}
}
} finally {
}
}
if (state.buffer.length > 0) {
const ramBatch = [...state.buffer];
state.buffer = [];
const validRamBatch: DataEvent[] = [];
for (const evt of ramBatch) {
if (
await this.isClientDeletedForBuffer(
evt.clientId,
redis,
deletedCheckCache,
)
) {
continue;
}
validRamBatch.push(evt);
}
if (validRamBatch.length > 0) {
try {
await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
} catch {
// Lock auto-expires after FLUSH_LOCK_TTL_MS
const res = await CSVStorageManager.appendBatch(validRamBatch);
totalWritten += res.written;
for (const f of res.filesUpdated) filesSet.add(f);
} catch (err) {
state.buffer.unshift(...validRamBatch);
throw err;
}
}
}
}
if (state.buffer.length > 0) {
const ramBatch = [...state.buffer];
state.buffer = [];
try {
const res = await CSVStorageManager.appendBatch(ramBatch);
totalWritten += res.written;
for (const f of res.filesUpdated) filesSet.add(f);
} catch (err) {
state.buffer.unshift(...ramBatch);
throw err;
}
}
if (totalWritten > 0) {
state.totalFlushed += totalWritten;
const filesUpdated = Array.from(filesSet);
for (const listener of state.flushListeners) {
try {
listener(totalWritten, filesUpdated);
} catch {
// ignore
if (totalWritten > 0) {
state.totalFlushed += totalWritten;
const filesUpdated = Array.from(filesSet);
for (const listener of state.flushListeners) {
try {
listener(totalWritten, filesUpdated);
} catch {
// ignore
}
}
}
}
return { count: totalWritten, files: Array.from(filesSet) };
} finally {
state.isFlushing = false;
return {
count: totalWritten,
files: Array.from(filesSet),
...(lockTimedOut ? { lockTimedOut: true } : {}),
};
} finally {
state.isFlushing = false;
state.activeFlushPromise = null;
}
};
let flushDone = false;
let promise: Promise<FlushResult> | null = null;
promise = (async () => {
try {
return await executeFlush();
} finally {
flushDone = true;
state.isFlushing = false;
if (state.activeFlushPromise === promise) {
state.activeFlushPromise = null;
}
}
})();
if (!flushDone) {
state.activeFlushPromise = promise;
}
return promise;
}
/**
@@ -495,6 +916,15 @@ export class DataBufferEngine {
};
}
public static getLocalBufferSnapshot(limit?: number): DataEvent[] {
const state = getState();
if (typeof limit === 'number' && Number.isFinite(limit) && limit >= 0) {
const cap = Math.floor(limit);
return cap === 0 ? [] : state.buffer.slice(-cap);
}
return [...state.buffer];
}
public static getRecentEvents(limit = 50, filterClientId?: string): DataEvent[] {
const state = getState();
if (filterClientId && filterClientId !== 'all') {
@@ -514,7 +944,8 @@ export class DataBufferEngine {
if (redis) {
try {
const readCap = Math.min(REDIS_RECENT_LIMIT, Math.max(limit * 4, limit));
const recentLimit = this.getRecentLimit();
const readCap = Math.min(recentLimit, Math.max(limit * 4, limit));
const rawItems = await redis.lrange(REDIS_RECENT_KEY, 0, readCap - 1);
state.redisConnected = true;
@@ -523,6 +954,9 @@ export class DataBufferEngine {
try {
const parsed = JSON.parse(raw) as DataEvent;
if (!parsed || !parsed.eventId) continue;
if (state.deletedClientIds.has(normalizeBufferClientId(parsed.clientId))) {
continue;
}
if (filterClientId && filterClientId !== 'all' && parsed.clientId !== filterClientId) {
continue;
}
+520 -117
View File
@@ -1,9 +1,21 @@
import { existsSync, rmSync } from "node:fs";
import { randomBytes } from "node:crypto";
import { existsSync, lstatSync, readdirSync, rmSync } from "node:fs";
import { resolve, sep } from "node:path";
import { SettingsManager } from "../adminSettings/settingsManager";
import { AuthService } from "../auth/jwt";
import { DataBufferEngine } from "../data/buffer";
import { DatabaseEngine } from "../Database/database";
import { getInternalIngestionHeaders } from "../Security/security";
import { CSVStorageManager } from "../Storage/csvStorage";
export interface BridgeSyncResult {
synced: boolean;
status?: number;
error?: string;
}
const REDIS_CLIENTS_REGISTRY_KEY = "datastack:clients:registry";
const REDIS_CLIENTS_DELETED_KEY = "datastack:clients:deleted";
export interface ClientSDK {
clientId: string;
@@ -17,13 +29,24 @@ export interface ClientSDK {
createdAt: string;
}
type PendingActivityEntry = {
count: number;
lastActive: string;
};
type SDKDatabaseGlobals = typeof globalThis & {
__dataStackClientsMap?: Map<string, ClientSDK>;
__dataStackDeletedClients?: Set<string>;
__dataStackClientsHydrated?: boolean;
__dataStackSyncedSocketEvents?: Map<string, number>;
__dataStackPendingClientActivity?: Map<string, PendingActivityEntry>;
__dataStackActivityFlushTimer?: ReturnType<typeof setInterval> | null;
__dataStackActivityFlushPromise?: Promise<number> | null;
__dataStackActivityShutdownRegistered?: boolean;
};
const CLIENT_ACTIVITY_FLUSH_INTERVAL_MS = 1000;
const CLIENT_ACTIVITY_IMMEDIATE_BATCH_SIZE = 500;
const globals = globalThis as SDKDatabaseGlobals;
async function ensureDbSettingsHydrated(): Promise<void> {
@@ -53,6 +76,58 @@ function getSyncedSocketEventsMap(): Map<string, number> {
return globals.__dataStackSyncedSocketEvents;
}
function getPendingClientActivityMap(): Map<string, PendingActivityEntry> {
if (!globals.__dataStackPendingClientActivity) {
globals.__dataStackPendingClientActivity = new Map<
string,
PendingActivityEntry
>();
}
return globals.__dataStackPendingClientActivity;
}
function ensureActivityFlushTimer(): void {
if (globals.__dataStackActivityFlushTimer) return;
const timer = setInterval(() => {
void SDKManager.flushPendingClientActivityAsync().catch(() => undefined);
}, CLIENT_ACTIVITY_FLUSH_INTERVAL_MS);
if (typeof timer.unref === "function") {
timer.unref();
}
globals.__dataStackActivityFlushTimer = timer;
}
interface RemoteInMemoryClient {
clientId?: string;
name?: string;
status?: string;
token?: string;
rateLimit?: number;
}
function hydrateLocalMapFromRemote(remoteClients: RemoteInMemoryClient[]): number {
const map = getClientsMap();
const deletedSet = getDeletedClientsSet();
let hydrated = 0;
for (const item of remoteClients) {
if (!item || !item.clientId) continue;
const cid = SDKManager.normalizeClientId(item.clientId);
if (!cid || deletedSet.has(cid) || map.has(cid)) continue;
map.set(cid, {
clientId: cid,
name: String(item.name || cid),
secret: "",
token: String(item.token || ""),
rateLimit: Math.max(1, Number(item.rateLimit) || 100),
status: item.status === "revoked" ? "revoked" : "active",
totalEvents: 0,
createdAt: new Date().toISOString(),
});
hydrated += 1;
}
return hydrated;
}
function serializeSyncClient(client: ClientSDK) {
return {
clientId: client.clientId,
@@ -63,18 +138,53 @@ function serializeSyncClient(client: ClientSDK) {
};
}
async function notifyIngestionClientSyncAsync(
async function mirrorClientToRedis(
clientId: string,
action: "deleted" | "revoked" | "updated" | "registered",
client?: ClientSDK,
): Promise<void> {
const redis = DataBufferEngine.getRedisClient();
if (!redis) return;
try {
if (action === "deleted") {
if (typeof redis.sadd === "function") {
await redis.sadd(REDIS_CLIENTS_DELETED_KEY, clientId);
}
if (typeof redis.hdel === "function") {
await redis.hdel(REDIS_CLIENTS_REGISTRY_KEY, clientId);
}
return;
}
if (action === "registered" && typeof redis.srem === "function") {
await redis.srem(REDIS_CLIENTS_DELETED_KEY, clientId);
}
if (client && typeof redis.hset === "function") {
await redis.hset(
REDIS_CLIENTS_REGISTRY_KEY,
clientId,
JSON.stringify(serializeSyncClient(client)),
);
}
} catch {
// Non-fatal; DB and direct HTTP bridge remain primary sources of truth
}
}
async function notifyIngestionClientSyncAsync(
clientId: string,
action: "deleted" | "revoked" | "updated" | "registered",
client?: ClientSDK,
): Promise<BridgeSyncResult> {
await mirrorClientToRedis(clientId, action, client);
const headers = getInternalIngestionHeaders();
if (!headers["x-datastack-internal-secret"]) return;
if (!headers["x-datastack-internal-secret"]) {
return { synced: false, error: "Internal bridge secret is not configured" };
}
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, "");
try {
await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
const res = await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
method: "POST",
headers: {
"Content-Type": "application/json",
@@ -87,8 +197,21 @@ async function notifyIngestionClientSyncAsync(
}),
signal: AbortSignal.timeout(1000),
});
} catch {
// Port 4000 may not be running in isolated single-process dev mode
if (!res.ok) {
const errorMsg = `Port 4000 client-sync (${action}:${clientId}) failed with HTTP ${res.status}`;
console.warn(`[DataStack Bridge] ${errorMsg}`);
return { synced: false, status: res.status, error: errorMsg };
}
return { synced: true, status: res.status };
} catch (err: unknown) {
const message = err instanceof Error ? err.message : "Bridge request failed";
if (
process.env.NODE_ENV === "production" ||
Boolean(process.env.INGESTION_INTERNAL_URL)
) {
console.warn(`[DataStack Bridge] Port 4000 client-sync unreachable: ${message}`);
}
return { synced: false, error: message };
}
}
@@ -127,6 +250,10 @@ export class SDKManager {
const ready = await DatabaseEngine.ensureSchema();
if (!ready) return null;
if (getPendingClientActivityMap().size > 0) {
await this.flushPendingClientActivityAsync();
}
const [result, deletedResult] = await Promise.all([
pool.query(
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
@@ -166,16 +293,31 @@ export class SDKManager {
return Array.from(getClientsMap().values());
}
public static async syncAllClientsToIngestionAsync(): Promise<void> {
public static async syncAllClientsToIngestionAsync(options?: {
forceReplace?: boolean;
}): Promise<BridgeSyncResult> {
await ensureDbSettingsHydrated();
const clients = Array.from(getClientsMap().values()).map(serializeSyncClient);
const forceReplace = options?.forceReplace ?? Boolean(DatabaseEngine.getPool());
const redis = DataBufferEngine.getRedisClient();
if (redis && typeof redis.hset === "function") {
try {
for (const c of clients) {
await redis.hset(REDIS_CLIENTS_REGISTRY_KEY, c.clientId, JSON.stringify(c));
}
} catch {
// ignore Redis mirror failure
}
}
const headers = getInternalIngestionHeaders();
if (!headers["x-datastack-internal-secret"]) return;
if (!headers["x-datastack-internal-secret"]) {
return { synced: false, error: "Internal bridge secret is not configured" };
}
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, "");
const clients = Array.from(getClientsMap().values()).map(serializeSyncClient);
try {
await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
const res = await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
method: "POST",
headers: {
"Content-Type": "application/json",
@@ -184,11 +326,43 @@ export class SDKManager {
body: JSON.stringify({
action: "bulk_sync",
clients,
forceReplace,
}),
signal: AbortSignal.timeout(1500),
});
} catch {
// Port 4000 may not be running in isolated single-process dev mode
if (!res.ok) {
const errorMsg = `Port 4000 bulk_sync failed with HTTP ${res.status}`;
console.warn(`[DataStack Bridge] ${errorMsg}`);
return { synced: false, status: res.status, error: errorMsg };
}
// BUG-02: If Port 4000 preserved an existing no-DB registry because Port 3000 restarted empty,
// hydrate Port 3000's local map non-destructively from the returned clients.
try {
const payload = (await res.json()) as {
preserved?: boolean;
clients?: RemoteInMemoryClient[];
};
if (
payload?.preserved &&
Array.isArray(payload.clients) &&
payload.clients.length > 0 &&
!DatabaseEngine.getPool()
) {
hydrateLocalMapFromRemote(payload.clients);
}
} catch {
// Response body optional
}
return { synced: true, status: res.status };
} catch (err: unknown) {
const message = err instanceof Error ? err.message : "Bridge bulk_sync unreachable";
if (
process.env.NODE_ENV === "production" ||
Boolean(process.env.INGESTION_INTERNAL_URL)
) {
console.warn(`[DataStack Bridge] ${message}`);
}
return { synced: false, error: message };
}
}
@@ -214,12 +388,20 @@ export class SDKManager {
if (res.ok) {
const data = (await res.json()) as {
hasSyncedInMemoryClients?: boolean;
inMemoryClients?: RemoteInMemoryClient[];
clientActivity?: Record<
string,
{ totalEvents?: number; lastActive?: string }
>;
};
if (data.hasSyncedInMemoryClients === false && map.size > 0) {
// BUG-02: Non-destructive hydration when Port 3000 restarted empty while Port 4000 still holds clients
if (
map.size === 0 &&
Array.isArray(data.inMemoryClients) &&
data.inMemoryClients.length > 0
) {
hydrateLocalMapFromRemote(data.inMemoryClients);
} else if (data.hasSyncedInMemoryClients === false && map.size > 0) {
await this.syncAllClientsToIngestionAsync();
}
if (data.clientActivity && typeof data.clientActivity === "object") {
@@ -255,36 +437,63 @@ export class SDKManager {
return Array.from(map.values());
}
public static normalizeClientId(clientId: string): string {
return String(clientId || "")
.trim()
.toLowerCase()
.replace(/[^a-z0-9_-]/g, "_");
}
public static getClient(clientId: string): ClientSDK | undefined {
return getClientsMap().get(clientId);
const normalizedId = this.normalizeClientId(clientId);
if (!normalizedId) return undefined;
return getClientsMap().get(normalizedId);
}
public static async getClientAsync(
clientId: string,
): Promise<ClientSDK | undefined> {
await ensureDbSettingsHydrated();
const normalizedId = this.normalizeClientId(clientId);
if (!normalizedId) return undefined;
const pool = DatabaseEngine.getPool();
if (pool) {
try {
await DatabaseEngine.ensureSchema();
if (getPendingClientActivityMap().has(normalizedId)) {
await this.flushPendingClientActivityAsync();
}
const result = await pool.query(
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
FROM client_sdks
WHERE client_id = $1`,
[clientId],
[normalizedId],
);
if (result.rowCount && result.rowCount > 0) {
const client = mapRowToClient(result.rows[0]);
getClientsMap().set(client.clientId, client);
return client;
}
getClientsMap().delete(clientId);
getClientsMap().delete(normalizedId);
return undefined;
} catch {
// Fall back to in-memory map
}
}
return getClientsMap().get(clientId);
const existing = getClientsMap().get(normalizedId);
if (existing) {
return existing;
}
// BUG-02: If Port 3000 restarted without DB and local map is empty, attempt non-destructive hydration from Port 4000
if (getClientsMap().size === 0 && !getDeletedClientsSet().has(normalizedId)) {
await this.listClientsAsync();
return getClientsMap().get(normalizedId);
}
return undefined;
}
private static generateClientSecret(safeClientId: string): string {
return `sec_${safeClientId}_${randomBytes(16).toString("hex")}`;
}
public static registerClient(data: {
@@ -292,11 +501,8 @@ export class SDKManager {
name: string;
rateLimit?: number;
}): ClientSDK {
const safeClientId = data.clientId
.trim()
.toLowerCase()
.replace(/[^a-z0-9_-]/g, "_");
const secret = `sec_${safeClientId}_${Math.random().toString(36).substring(2, 10)}`;
const safeClientId = this.normalizeClientId(data.clientId);
const secret = this.generateClientSecret(safeClientId);
const rateLimit = Math.max(1, Number(data.rateLimit) || 100);
const token = AuthService.generateClientToken({
@@ -317,8 +523,12 @@ export class SDKManager {
};
getDeletedClientsSet().delete(safeClientId);
DataBufferEngine.unmarkClientDeleted(safeClientId);
getSyncedSocketEventsMap().delete(safeClientId);
getClientsMap().set(safeClientId, newClient);
void this.persistNewClient(newClient);
void this.persistNewClient(newClient).catch((err) => {
console.error("[DataStack SDK] Background client create DB persist failed:", err);
});
notifyIngestionClientSync(safeClientId, "registered", newClient);
return newClient;
}
@@ -327,8 +537,12 @@ export class SDKManager {
const pool = DatabaseEngine.getPool();
if (!pool) return;
const ready = await DatabaseEngine.ensureSchema();
if (!ready) {
throw new Error("DATABASE_WRITE_FAILED: PostgreSQL schema unavailable for client registration.");
}
try {
await DatabaseEngine.ensureSchema();
await pool.query(
`DELETE FROM deleted_clients WHERE client_id = $1`,
[client.clientId],
@@ -354,8 +568,9 @@ export class SDKManager {
client.createdAt,
],
);
} catch {
// In-memory map already updated
} catch (err: unknown) {
const detail = err instanceof Error ? err.message : "Unknown DB error";
throw new Error(`DATABASE_WRITE_FAILED: Failed to register client (${detail})`);
}
}
@@ -365,11 +580,8 @@ export class SDKManager {
rateLimit?: number;
}): Promise<ClientSDK> {
await ensureDbSettingsHydrated();
const safeClientId = data.clientId
.trim()
.toLowerCase()
.replace(/[^a-z0-9_-]/g, "_");
const secret = `sec_${safeClientId}_${Math.random().toString(36).substring(2, 10)}`;
const safeClientId = this.normalizeClientId(data.clientId);
const secret = this.generateClientSecret(safeClientId);
const rateLimit = Math.max(1, Number(data.rateLimit) || 100);
const token = AuthService.generateClientToken({
@@ -389,20 +601,23 @@ export class SDKManager {
createdAt: new Date().toISOString(),
};
getDeletedClientsSet().delete(safeClientId);
getClientsMap().set(safeClientId, newClient);
await this.persistNewClient(newClient);
getDeletedClientsSet().delete(safeClientId);
DataBufferEngine.unmarkClientDeleted(safeClientId);
getSyncedSocketEventsMap().delete(safeClientId);
getClientsMap().set(safeClientId, newClient);
await notifyIngestionClientSyncAsync(safeClientId, "registered", newClient);
return newClient;
}
public static regenerateToken(clientId: string): ClientSDK | null {
const client = getClientsMap().get(clientId);
const normalizedId = this.normalizeClientId(clientId);
const client = getClientsMap().get(normalizedId);
if (!client) return null;
const updated: ClientSDK = {
...client,
secret: `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`,
secret: this.generateClientSecret(normalizedId),
token: AuthService.generateClientToken({
clientId: client.clientId,
name: client.name,
@@ -410,9 +625,11 @@ export class SDKManager {
}),
};
getClientsMap().set(clientId, updated);
void this.persistClientCredentials(updated);
notifyIngestionClientSync(clientId, "updated", updated);
getClientsMap().set(normalizedId, updated);
void this.persistClientCredentials(updated).catch((err) => {
console.error("[DataStack SDK] Background token rotate DB persist failed:", err);
});
notifyIngestionClientSync(normalizedId, "updated", updated);
return updated;
}
@@ -422,16 +639,21 @@ export class SDKManager {
const pool = DatabaseEngine.getPool();
if (!pool) return;
const ready = await DatabaseEngine.ensureSchema();
if (!ready) {
throw new Error("DATABASE_WRITE_FAILED: PostgreSQL schema unavailable for credential update.");
}
try {
await DatabaseEngine.ensureSchema();
await pool.query(
`UPDATE client_sdks
SET secret = $2, token = $3, updated_at = NOW()
WHERE client_id = $1`,
[client.clientId, client.secret, client.token],
);
} catch {
// Ignore DB failure
} catch (err: unknown) {
const detail = err instanceof Error ? err.message : "Unknown DB error";
throw new Error(`DATABASE_WRITE_FAILED: Failed to update client credentials (${detail})`);
}
}
@@ -439,12 +661,13 @@ export class SDKManager {
clientId: string,
): Promise<ClientSDK | null> {
await ensureDbSettingsHydrated();
const existing = await this.getClientAsync(clientId);
const normalizedId = this.normalizeClientId(clientId);
const existing = await this.getClientAsync(normalizedId);
if (!existing) return null;
const updated: ClientSDK = {
...existing,
secret: `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`,
secret: this.generateClientSecret(normalizedId),
token: AuthService.generateClientToken({
clientId: existing.clientId,
name: existing.name,
@@ -452,9 +675,9 @@ export class SDKManager {
}),
};
getClientsMap().set(clientId, updated);
await this.persistClientCredentials(updated);
await notifyIngestionClientSyncAsync(clientId, "updated", updated);
getClientsMap().set(normalizedId, updated);
await notifyIngestionClientSyncAsync(normalizedId, "updated", updated);
return updated;
}
@@ -462,7 +685,8 @@ export class SDKManager {
clientId: string,
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
): ClientSDK | null {
const client = getClientsMap().get(clientId);
const normalizedId = this.normalizeClientId(clientId);
const client = getClientsMap().get(normalizedId);
if (!client) return null;
const updated: ClientSDK = {
@@ -474,10 +698,12 @@ export class SDKManager {
...(updates.status !== undefined ? { status: updates.status } : {}),
};
getClientsMap().set(clientId, updated);
void this.persistClientUpdates(updated);
getClientsMap().set(normalizedId, updated);
void this.persistClientUpdates(updated).catch((err) => {
console.error("[DataStack SDK] Background client update DB persist failed:", err);
});
notifyIngestionClientSync(
clientId,
normalizedId,
updated.status === "revoked" ? "revoked" : "updated",
updated,
);
@@ -488,16 +714,21 @@ export class SDKManager {
const pool = DatabaseEngine.getPool();
if (!pool) return;
const ready = await DatabaseEngine.ensureSchema();
if (!ready) {
throw new Error("DATABASE_WRITE_FAILED: PostgreSQL schema unavailable for client update.");
}
try {
await DatabaseEngine.ensureSchema();
await pool.query(
`UPDATE client_sdks
SET name = $2, rate_limit = $3, status = $4, updated_at = NOW()
WHERE client_id = $1`,
[client.clientId, client.name, client.rateLimit, client.status],
);
} catch {
// Ignore DB failure
} catch (err: unknown) {
const detail = err instanceof Error ? err.message : "Unknown DB error";
throw new Error(`DATABASE_WRITE_FAILED: Failed to update client (${detail})`);
}
}
@@ -506,7 +737,8 @@ export class SDKManager {
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
): Promise<ClientSDK | null> {
await ensureDbSettingsHydrated();
const client = await this.getClientAsync(clientId);
const normalizedId = this.normalizeClientId(clientId);
const client = await this.getClientAsync(normalizedId);
if (!client) return null;
const updated: ClientSDK = {
@@ -518,10 +750,10 @@ export class SDKManager {
...(updates.status !== undefined ? { status: updates.status } : {}),
};
getClientsMap().set(clientId, updated);
await this.persistClientUpdates(updated);
getClientsMap().set(normalizedId, updated);
await notifyIngestionClientSyncAsync(
clientId,
normalizedId,
updated.status === "revoked" ? "revoked" : "updated",
updated,
);
@@ -529,16 +761,12 @@ export class SDKManager {
}
public static isClientDeleted(clientId: string): boolean {
const normalized = String(clientId || "")
.trim()
.toLowerCase();
const normalized = this.normalizeClientId(clientId);
return getDeletedClientsSet().has(normalized);
}
public static async isClientDeletedAsync(clientId: string): Promise<boolean> {
const normalized = String(clientId || "")
.trim()
.toLowerCase();
const normalized = this.normalizeClientId(clientId);
if (!normalized) return false;
if (getDeletedClientsSet().has(normalized)) {
return true;
@@ -560,15 +788,18 @@ export class SDKManager {
}
public static deleteClient(clientId: string): boolean {
const normalized = String(clientId || "")
.trim()
.toLowerCase();
const normalized = this.normalizeClientId(clientId);
const deleted = getClientsMap().delete(normalized);
getSyncedSocketEventsMap().delete(normalized);
getPendingClientActivityMap().delete(normalized);
if (deleted) {
getDeletedClientsSet().add(normalized);
DataBufferEngine.markClientDeleted(normalized);
void DataBufferEngine.purgeClientEventsAsync(normalized).catch(() => undefined);
}
void this.deleteClientFromDb(normalized);
void this.deleteClientFromDb(normalized).catch((err) => {
console.error("[DataStack SDK] Background client delete DB persist failed:", err);
});
notifyIngestionClientSync(normalized, "deleted");
return deleted;
}
@@ -577,7 +808,10 @@ export class SDKManager {
const pool = DatabaseEngine.getPool();
if (!pool) return false;
await DatabaseEngine.ensureSchema();
const ready = await DatabaseEngine.ensureSchema();
if (!ready) {
throw new Error("DATABASE_WRITE_FAILED: PostgreSQL schema unavailable for client deletion.");
}
const dbClient = await pool.connect();
try {
await dbClient.query("BEGIN");
@@ -597,69 +831,226 @@ export class SDKManager {
}
await dbClient.query("ROLLBACK");
return false;
} catch {
} catch (err: unknown) {
await dbClient.query("ROLLBACK").catch(() => undefined);
return false;
const detail = err instanceof Error ? err.message : "Unknown DB error";
throw new Error(`DATABASE_WRITE_FAILED: Failed to delete client (${detail})`);
} finally {
dbClient.release();
}
}
private static purgeClientStorageDir(clientId: string): void {
private static purgeClientStorageDir(clientId: string): number {
const safeId = clientId.replace(/[^a-z0-9_-]/gi, "_");
if (!safeId) return;
const storageRoot = resolve(process.cwd(), "storage", "data");
if (!safeId) return 0;
const storageRoot = resolve(CSVStorageManager.baseStorageDir);
const targetDir = resolve(storageRoot, safeId);
if (!targetDir.startsWith(`${storageRoot}${sep}`)) return;
if (existsSync(targetDir)) {
rmSync(targetDir, { recursive: true, force: true });
if (!targetDir.startsWith(`${storageRoot}${sep}`)) return 0;
if (!existsSync(targetDir)) return 0;
let purgedFiles = 0;
try {
const dirStat = lstatSync(targetDir);
if (!dirStat.isSymbolicLink() && dirStat.isDirectory()) {
for (const entry of readdirSync(targetDir)) {
if (!entry.endsWith(".csv")) continue;
const entryPath = resolve(targetDir, entry);
try {
const entryStat = lstatSync(entryPath);
if (!entryStat.isSymbolicLink() && entryStat.isFile()) {
purgedFiles += 1;
}
} catch {
// Ignore unreadable file entry during pre-purge count
}
}
}
} catch {
purgedFiles = 0;
}
rmSync(targetDir, { recursive: true, force: true });
CSVStorageManager.clearRowCountCache();
return purgedFiles;
}
public static async deleteClientWithStatsAsync(
clientId: string,
options?: { purgeFiles?: boolean },
): Promise<{ deleted: boolean; purgedFiles: number }> {
await ensureDbSettingsHydrated();
const normalized = this.normalizeClientId(clientId);
getPendingClientActivityMap().delete(normalized);
const deletedFromDb = await this.deleteClientFromDb(normalized);
const existedInMem = getClientsMap().delete(normalized);
getSyncedSocketEventsMap().delete(normalized);
let purgedFiles = 0;
if (existedInMem || deletedFromDb) {
getDeletedClientsSet().add(normalized);
DataBufferEngine.markClientDeleted(normalized);
await DataBufferEngine.purgeClientEventsAsync(normalized);
await notifyIngestionClientSyncAsync(normalized, "deleted");
if (options?.purgeFiles) {
purgedFiles = this.purgeClientStorageDir(normalized);
}
}
return { deleted: existedInMem || deletedFromDb, purgedFiles };
}
public static async deleteClientAsync(
clientId: string,
options?: { purgeFiles?: boolean },
): Promise<boolean> {
await ensureDbSettingsHydrated();
const normalized = String(clientId || "")
.trim()
.toLowerCase();
const existedInMem = getClientsMap().delete(normalized);
getSyncedSocketEventsMap().delete(normalized);
const deletedFromDb = await this.deleteClientFromDb(normalized);
if (existedInMem || deletedFromDb) {
getDeletedClientsSet().add(normalized);
if (options?.purgeFiles) {
this.purgeClientStorageDir(normalized);
}
await notifyIngestionClientSyncAsync(normalized, "deleted");
}
return existedInMem || deletedFromDb;
const result = await this.deleteClientWithStatsAsync(clientId, options);
return result.deleted;
}
public static incrementEventCount(clientId: string, count = 1): void {
const nowIso = new Date().toISOString();
const client = getClientsMap().get(clientId);
if (client) {
client.totalEvents += count;
client.lastActive = nowIso;
getClientsMap().set(clientId, client);
private static queuePendingActivityDelta(
normalizedId: string,
count: number,
nowIso: string,
): number {
const pending = getPendingClientActivityMap();
const existing = pending.get(normalizedId);
if (existing) {
pending.set(normalizedId, {
count: existing.count + count,
lastActive:
nowIso > existing.lastActive ? nowIso : existing.lastActive,
});
} else {
pending.set(normalizedId, {
count,
lastActive: nowIso,
});
}
ensureActivityFlushTimer();
let totalDelta = 0;
for (const entry of pending.values()) {
totalDelta += entry.count;
}
return totalDelta;
}
public static async flushPendingClientActivityAsync(): Promise<number> {
if (globals.__dataStackActivityFlushPromise) {
await globals.__dataStackActivityFlushPromise;
if (getPendingClientActivityMap().size === 0) return 0;
}
const pool = DatabaseEngine.getPool();
if (pool) {
void DatabaseEngine.ensureSchema()
.then(() =>
pool.query(
`UPDATE client_sdks
SET total_events = total_events + $2,
last_active = $3,
updated_at = NOW()
WHERE client_id = $1`,
[clientId, count, nowIso],
),
)
.catch(() => undefined);
const pending = getPendingClientActivityMap();
if (!pool || pending.size === 0) return 0;
const snapshot = Array.from(pending.entries());
pending.clear();
const clientIds: string[] = [];
const deltas: number[] = [];
const lastActives: string[] = [];
for (const [cid, entry] of snapshot) {
clientIds.push(cid);
deltas.push(entry.count);
lastActives.push(entry.lastActive);
}
const run = (async () => {
try {
await DatabaseEngine.ensureSchema();
await pool.query(
`UPDATE client_sdks AS c
SET total_events = c.total_events + v.delta,
last_active = GREATEST(COALESCE(c.last_active, v.last_active), v.last_active),
updated_at = NOW()
FROM (
SELECT UNNEST($1::text[]) AS client_id,
UNNEST($2::bigint[]) AS delta,
UNNEST($3::timestamptz[]) AS last_active
) AS v
WHERE c.client_id = v.client_id`,
[clientIds, deltas, lastActives],
);
return clientIds.length;
} catch {
// Re-merge deltas so transient DB errors do not drop event counts
for (const [cid, entry] of snapshot) {
const current = pending.get(cid);
if (current) {
pending.set(cid, {
count: current.count + entry.count,
lastActive:
current.lastActive > entry.lastActive
? current.lastActive
: entry.lastActive,
});
} else {
pending.set(cid, { ...entry });
}
}
return 0;
} finally {
globals.__dataStackActivityFlushPromise = null;
}
})();
globals.__dataStackActivityFlushPromise = run;
return run;
}
public static stopActivityFlushTimerForTests(): void {
if (globals.__dataStackActivityFlushTimer) {
clearInterval(globals.__dataStackActivityFlushTimer);
globals.__dataStackActivityFlushTimer = null;
}
}
public static incrementEventCount(clientId: string, count = 1): void {
const normalizedId = this.normalizeClientId(clientId);
if (!normalizedId || count <= 0) return;
const nowIso = new Date().toISOString();
const client = getClientsMap().get(normalizedId);
if (client) {
client.totalEvents += count;
client.lastActive = nowIso;
getClientsMap().set(normalizedId, client);
}
if (DatabaseEngine.getPool()) {
const queuedTotal = this.queuePendingActivityDelta(
normalizedId,
count,
nowIso,
);
if (queuedTotal >= CLIENT_ACTIVITY_IMMEDIATE_BATCH_SIZE) {
void this.flushPendingClientActivityAsync().catch(() => undefined);
}
}
}
public static async incrementEventCountAsync(
clientId: string,
count = 1,
): Promise<void> {
const normalizedId = this.normalizeClientId(clientId);
if (!normalizedId || count <= 0) return;
const nowIso = new Date().toISOString();
const client = getClientsMap().get(normalizedId);
if (client) {
client.totalEvents += count;
client.lastActive = nowIso;
getClientsMap().set(normalizedId, client);
}
// BUG-22: Coalesce per-event DB updates into batched UNNEST writes instead of 1 SQL query/event
if (DatabaseEngine.getPool()) {
const queuedTotal = this.queuePendingActivityDelta(
normalizedId,
count,
nowIso,
);
if (queuedTotal >= CLIENT_ACTIVITY_IMMEDIATE_BATCH_SIZE) {
await this.flushPendingClientActivityAsync();
}
}
}
@@ -672,7 +1063,9 @@ export class SDKManager {
rateLimit: client.rateLimit,
});
map.set(clientId, client);
void this.persistClientCredentials(client);
void this.persistClientCredentials(client).catch((err) => {
console.error("[DataStack SDK] Background re-sign DB persist failed:", err);
});
}
}
@@ -680,15 +1073,25 @@ export class SDKManager {
const clients = await this.listClientsAsync();
const map = getClientsMap();
for (const client of clients) {
client.token = AuthService.generateClientToken({
clientId: client.clientId,
name: client.name,
rateLimit: client.rateLimit,
});
map.set(client.clientId, client);
await this.persistClientCredentials(client);
const updated: ClientSDK = {
...client,
token: AuthService.generateClientToken({
clientId: client.clientId,
name: client.name,
rateLimit: client.rateLimit,
}),
};
await this.persistClientCredentials(updated);
map.set(updated.clientId, updated);
}
}
}
if (!globals.__dataStackActivityShutdownRegistered) {
globals.__dataStackActivityShutdownRegistered = true;
DataBufferEngine.registerShutdownHook(() =>
SDKManager.flushPendingClientActivityAsync(),
);
}
export default SDKManager;