feat(auth): harden session management and security
- Implement server-side session revocation on logout - Add IP extraction utility and admin setup error handling - Improve error codes and response consistency for auth endpoints - Refine landing page redirect logic to handle transient DB outages - Update deployment documentation and configuration defaults
This commit is contained in:
1 parent
935f12a1e9
commit
89e4d68690
81 files changed
+11530
-6916
No files matched your search
+103
-85
@@ -9,6 +9,98 @@ type CoreDatabaseGlobals = typeof globalThis & {
|
||||
|
||||
const globals = globalThis as CoreDatabaseGlobals;
|
||||
|
||||
export const CORE_SCHEMA_TABLE_DDLS = [
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS client_sdks (
|
||||
client_id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
secret TEXT NOT NULL,
|
||||
token TEXT NOT NULL,
|
||||
rate_limit INTEGER NOT NULL DEFAULT 100,
|
||||
status TEXT NOT NULL DEFAULT 'active',
|
||||
total_events BIGINT NOT NULL DEFAULT 0,
|
||||
last_active TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`,
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS admin_settings (
|
||||
id TEXT PRIMARY KEY DEFAULT 'default',
|
||||
server_port INTEGER NOT NULL DEFAULT 4000,
|
||||
web_port INTEGER NOT NULL DEFAULT 3000,
|
||||
max_events_per_sec INTEGER NOT NULL DEFAULT 100,
|
||||
max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
|
||||
batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
|
||||
batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
|
||||
file_rotation_mb INTEGER NOT NULL DEFAULT 100,
|
||||
jwt_secret TEXT NOT NULL,
|
||||
enable_idempotency BOOLEAN NOT NULL DEFAULT true,
|
||||
auto_prune_days INTEGER NOT NULL DEFAULT 30,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`,
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS deleted_clients (
|
||||
client_id TEXT PRIMARY KEY,
|
||||
deleted_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`,
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS admin_users (
|
||||
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
|
||||
email TEXT NOT NULL UNIQUE,
|
||||
password_salt BYTEA NOT NULL,
|
||||
password_hash BYTEA NOT NULL,
|
||||
token_version INTEGER NOT NULL DEFAULT 1,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`,
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS admin_login_attempts (
|
||||
key_hash TEXT PRIMARY KEY,
|
||||
window_started_at TIMESTAMPTZ NOT NULL,
|
||||
failed_attempts INTEGER NOT NULL,
|
||||
blocked_until TIMESTAMPTZ,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`,
|
||||
] as const;
|
||||
|
||||
export const CORE_SCHEMA_ALTER_DDLS = [
|
||||
`
|
||||
ALTER TABLE client_sdks
|
||||
ADD COLUMN IF NOT EXISTS rate_limit INTEGER NOT NULL DEFAULT 100,
|
||||
ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'active',
|
||||
ADD COLUMN IF NOT EXISTS total_events BIGINT NOT NULL DEFAULT 0,
|
||||
ADD COLUMN IF NOT EXISTS last_active TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
`,
|
||||
`
|
||||
ALTER TABLE admin_settings
|
||||
ADD COLUMN IF NOT EXISTS server_port INTEGER NOT NULL DEFAULT 4000,
|
||||
ADD COLUMN IF NOT EXISTS web_port INTEGER NOT NULL DEFAULT 3000,
|
||||
ADD COLUMN IF NOT EXISTS max_events_per_sec INTEGER NOT NULL DEFAULT 100,
|
||||
ADD COLUMN IF NOT EXISTS max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
|
||||
ADD COLUMN IF NOT EXISTS batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
|
||||
ADD COLUMN IF NOT EXISTS batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
|
||||
ADD COLUMN IF NOT EXISTS file_rotation_mb INTEGER NOT NULL DEFAULT 100,
|
||||
ADD COLUMN IF NOT EXISTS enable_idempotency BOOLEAN NOT NULL DEFAULT true,
|
||||
ADD COLUMN IF NOT EXISTS auto_prune_days INTEGER NOT NULL DEFAULT 30,
|
||||
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
`,
|
||||
`
|
||||
ALTER TABLE admin_users
|
||||
ADD COLUMN IF NOT EXISTS token_version INTEGER NOT NULL DEFAULT 1
|
||||
`,
|
||||
] as const;
|
||||
|
||||
export async function ensureCoreSchemaOnPool(pool: Pool): Promise<void> {
|
||||
await Promise.all(CORE_SCHEMA_TABLE_DDLS.map((sql) => pool.query(sql)));
|
||||
await Promise.all(CORE_SCHEMA_ALTER_DDLS.map((sql) => pool.query(sql)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Database Module (PostgreSQL Persistence Engine)
|
||||
* CRITICAL RULE: PostgreSQL stores strictly administrative metadata,
|
||||
@@ -17,19 +109,26 @@ const globals = globalThis as CoreDatabaseGlobals;
|
||||
*/
|
||||
export class DatabaseEngine {
|
||||
public static getPool(): Pool | null {
|
||||
const connectionString = process.env.DATABASE_URL;
|
||||
const connectionString = process.env.DATABASE_URL?.trim();
|
||||
if (!connectionString) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (globals.__dataStackCorePool === undefined) {
|
||||
try {
|
||||
globals.__dataStackCorePool = new Pool({
|
||||
const pool = new Pool({
|
||||
connectionString,
|
||||
connectionTimeoutMillis: 3000,
|
||||
idleTimeoutMillis: 30000,
|
||||
max: 5,
|
||||
max: 10,
|
||||
});
|
||||
pool.on("error", (err) => {
|
||||
console.error(
|
||||
"[DataStack DB] Unexpected idle PostgreSQL pool error:",
|
||||
err instanceof Error ? err.message : err,
|
||||
);
|
||||
});
|
||||
globals.__dataStackCorePool = pool;
|
||||
} catch {
|
||||
globals.__dataStackCorePool = null;
|
||||
}
|
||||
@@ -42,88 +141,7 @@ export class DatabaseEngine {
|
||||
const pool = this.getPool();
|
||||
if (!pool) return false;
|
||||
|
||||
globals.__dataStackCoreSchema ??= Promise.all([
|
||||
pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS client_sdks (
|
||||
client_id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
secret TEXT NOT NULL,
|
||||
token TEXT NOT NULL,
|
||||
rate_limit INTEGER NOT NULL DEFAULT 100,
|
||||
status TEXT NOT NULL DEFAULT 'active',
|
||||
total_events BIGINT NOT NULL DEFAULT 0,
|
||||
last_active TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`),
|
||||
pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS admin_settings (
|
||||
id TEXT PRIMARY KEY DEFAULT 'default',
|
||||
server_port INTEGER NOT NULL DEFAULT 4000,
|
||||
web_port INTEGER NOT NULL DEFAULT 3000,
|
||||
max_events_per_sec INTEGER NOT NULL DEFAULT 100,
|
||||
max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
|
||||
batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
|
||||
batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
|
||||
file_rotation_mb INTEGER NOT NULL DEFAULT 100,
|
||||
jwt_secret TEXT NOT NULL,
|
||||
enable_idempotency BOOLEAN NOT NULL DEFAULT true,
|
||||
auto_prune_days INTEGER NOT NULL DEFAULT 30,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`),
|
||||
pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS deleted_clients (
|
||||
client_id TEXT PRIMARY KEY,
|
||||
deleted_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`),
|
||||
pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS admin_users (
|
||||
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
|
||||
email TEXT NOT NULL UNIQUE,
|
||||
password_salt BYTEA NOT NULL,
|
||||
password_hash BYTEA NOT NULL,
|
||||
token_version INTEGER NOT NULL DEFAULT 1,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`),
|
||||
pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS admin_login_attempts (
|
||||
key_hash TEXT PRIMARY KEY,
|
||||
window_started_at TIMESTAMPTZ NOT NULL,
|
||||
failed_attempts INTEGER NOT NULL,
|
||||
blocked_until TIMESTAMPTZ,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`),
|
||||
]).then(() =>
|
||||
Promise.all([
|
||||
pool.query(`
|
||||
ALTER TABLE client_sdks
|
||||
ADD COLUMN IF NOT EXISTS rate_limit INTEGER NOT NULL DEFAULT 100,
|
||||
ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'active',
|
||||
ADD COLUMN IF NOT EXISTS total_events BIGINT NOT NULL DEFAULT 0,
|
||||
ADD COLUMN IF NOT EXISTS last_active TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
`),
|
||||
pool.query(`
|
||||
ALTER TABLE admin_settings
|
||||
ADD COLUMN IF NOT EXISTS server_port INTEGER NOT NULL DEFAULT 4000,
|
||||
ADD COLUMN IF NOT EXISTS web_port INTEGER NOT NULL DEFAULT 3000,
|
||||
ADD COLUMN IF NOT EXISTS max_events_per_sec INTEGER NOT NULL DEFAULT 100,
|
||||
ADD COLUMN IF NOT EXISTS max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
|
||||
ADD COLUMN IF NOT EXISTS batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
|
||||
ADD COLUMN IF NOT EXISTS batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
|
||||
ADD COLUMN IF NOT EXISTS file_rotation_mb INTEGER NOT NULL DEFAULT 100,
|
||||
ADD COLUMN IF NOT EXISTS enable_idempotency BOOLEAN NOT NULL DEFAULT true,
|
||||
ADD COLUMN IF NOT EXISTS auto_prune_days INTEGER NOT NULL DEFAULT 30,
|
||||
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
`),
|
||||
]),
|
||||
).then(() => undefined);
|
||||
globals.__dataStackCoreSchema ??= ensureCoreSchemaOnPool(pool);
|
||||
|
||||
try {
|
||||
await globals.__dataStackCoreSchema;
|
||||
|
||||
+206
-1
@@ -1,5 +1,17 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { AuthService } from "../auth/jwt";
|
||||
import { DataBufferEngine, type RedisBufferClient } from "../data/buffer";
|
||||
|
||||
const REDIS_IDEMPOTENCY_PREFIX = "datastack:idem:";
|
||||
const REDIS_RATE_LIMIT_PREFIX = "datastack:rl:";
|
||||
export const REDIS_RATE_LIMIT_LUA_SCRIPT = `
|
||||
-- DATASTACK_RATE_LIMIT_INCR
|
||||
local c = redis.call("INCR", KEYS[1])
|
||||
if c == 1 or redis.call("PTTL", KEYS[1]) < 0 then
|
||||
redis.call("PEXPIRE", KEYS[1], tonumber(ARGV[1]))
|
||||
end
|
||||
return c
|
||||
`.trim();
|
||||
|
||||
export function resolveInternalIngestionSecret(
|
||||
jwtSecretOverride?: string,
|
||||
@@ -43,9 +55,15 @@ export interface ValidationResult {
|
||||
| "MALFORMED_PAYLOAD";
|
||||
}
|
||||
|
||||
export type SecurityMode =
|
||||
| "redis_shared"
|
||||
| "bridge_coordinated"
|
||||
| "degraded_local";
|
||||
|
||||
export class SecurityEngine {
|
||||
public static maxEventsPerSec = 100;
|
||||
public static enableIdempotency = true;
|
||||
private static lastSecurityMode: SecurityMode = "degraded_local";
|
||||
|
||||
// Idempotency cache: key -> timestamp (expires after TTL)
|
||||
private static idempotencySet = new Map<string, number>();
|
||||
@@ -126,6 +144,32 @@ export class SecurityEngine {
|
||||
return true;
|
||||
}
|
||||
|
||||
private static trackLocalRateWindow(clientId: string): void {
|
||||
const now = Date.now();
|
||||
const current = this.rateLimitMap.get(clientId);
|
||||
if (!current || now - current.windowStart >= 1000) {
|
||||
this.rateLimitMap.set(clientId, { count: 1, windowStart: now });
|
||||
} else {
|
||||
current.count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
private static supportsSharedSecurityRedis(
|
||||
redis: RedisBufferClient | null,
|
||||
): redis is RedisBufferClient & {
|
||||
get: (key: string) => Promise<string | null>;
|
||||
incr: (key: string) => Promise<number>;
|
||||
pexpire: (key: string, ttlMs: number) => Promise<number>;
|
||||
} {
|
||||
return Boolean(
|
||||
redis &&
|
||||
typeof redis.get === "function" &&
|
||||
typeof redis.incr === "function" &&
|
||||
typeof redis.pexpire === "function" &&
|
||||
typeof redis.set === "function",
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates incoming event payload:
|
||||
* Checks duplicate idempotency (read-only), max byte size, and rate limit,
|
||||
@@ -148,7 +192,7 @@ export class SecurityEngine {
|
||||
|
||||
// 2. Check payload size before consuming rate-limit quota
|
||||
const payloadStr =
|
||||
typeof payload === "string" ? payload : JSON.stringify(payload);
|
||||
typeof payload === "string" ? payload : JSON.stringify(payload ?? {});
|
||||
const byteLength = Buffer.byteLength(payloadStr, "utf8");
|
||||
|
||||
if (byteLength > this.maxPayloadSizeBytes) {
|
||||
@@ -178,6 +222,158 @@ export class SecurityEngine {
|
||||
return { allowed: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared Redis-backed async validator with Port 4000 internal-bridge fallback
|
||||
* (so HTTP + WS share rate limits & idempotency even when Redis is unconfigured)
|
||||
* and final local-Map fallback for isolated single-process dev.
|
||||
*/
|
||||
public static async validateEventAsync(
|
||||
clientId: string,
|
||||
eventId: string,
|
||||
payload: unknown,
|
||||
clientMaxRate = 100,
|
||||
): Promise<ValidationResult> {
|
||||
const redis = DataBufferEngine.getRedisClient();
|
||||
if (!this.supportsSharedSecurityRedis(redis)) {
|
||||
const payloadStr =
|
||||
typeof payload === "string" ? payload : JSON.stringify(payload ?? {});
|
||||
const payloadBytes = Buffer.byteLength(payloadStr, "utf8");
|
||||
if (payloadBytes > this.maxPayloadSizeBytes) {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: `Payload size (${(payloadBytes / 1024).toFixed(1)} KB) exceeds limit of ${this.maxPayloadSizeBytes / 1024} KB`,
|
||||
code: "PAYLOAD_TOO_LARGE",
|
||||
};
|
||||
}
|
||||
|
||||
const headers = getInternalIngestionHeaders();
|
||||
if (headers["x-datastack-internal-secret"]) {
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||
).replace(/\/$/, "");
|
||||
try {
|
||||
const res = await fetch(`${ingestionBaseUrl}/internal/security-check`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...headers,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
clientId,
|
||||
eventId,
|
||||
payloadBytes,
|
||||
clientMaxRate,
|
||||
}),
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(400),
|
||||
});
|
||||
if (res.ok) {
|
||||
const data = (await res.json()) as {
|
||||
allowed?: boolean;
|
||||
code?: ValidationResult["code"];
|
||||
reason?: string;
|
||||
};
|
||||
this.lastSecurityMode = "bridge_coordinated";
|
||||
return {
|
||||
allowed: Boolean(data.allowed),
|
||||
...(data.code ? { code: data.code } : {}),
|
||||
...(data.reason ? { reason: data.reason } : {}),
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
// Port 4000 unreachable — fall back to local validation
|
||||
}
|
||||
}
|
||||
|
||||
this.lastSecurityMode = "degraded_local";
|
||||
return this.validateEvent(clientId, eventId, payload, clientMaxRate);
|
||||
}
|
||||
|
||||
this.lastSecurityMode = "redis_shared";
|
||||
const idemKey = `${REDIS_IDEMPOTENCY_PREFIX}${clientId}:${eventId}`;
|
||||
try {
|
||||
// 1. Read-only idempotency check
|
||||
if (this.enableIdempotency && eventId) {
|
||||
if (this.hasDuplicate(clientId, eventId)) {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: "Duplicate event dropped silently",
|
||||
code: "DUPLICATE_EVENT",
|
||||
};
|
||||
}
|
||||
const existing = await redis.get(idemKey);
|
||||
if (existing !== null) {
|
||||
this.recordIdempotency(clientId, eventId);
|
||||
return {
|
||||
allowed: false,
|
||||
reason: "Duplicate event dropped silently",
|
||||
code: "DUPLICATE_EVENT",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Payload byte size check before rate-limit consumption
|
||||
const payloadStr =
|
||||
typeof payload === "string" ? payload : JSON.stringify(payload ?? {});
|
||||
const byteLength = Buffer.byteLength(payloadStr, "utf8");
|
||||
if (byteLength > this.maxPayloadSizeBytes) {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: `Payload size (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${this.maxPayloadSizeBytes / 1024} KB`,
|
||||
code: "PAYLOAD_TOO_LARGE",
|
||||
};
|
||||
}
|
||||
|
||||
// 3. Shared atomic per-second rate-limit counter in Redis (single Lua round-trip)
|
||||
const effectiveRateLimit = Math.min(clientMaxRate, this.maxEventsPerSec);
|
||||
const windowBucket = Math.floor(Date.now() / 1000);
|
||||
const rlKey = `${REDIS_RATE_LIMIT_PREFIX}${clientId}:${windowBucket}`;
|
||||
let count: number;
|
||||
if (typeof redis.eval === "function") {
|
||||
count = Number(
|
||||
await redis.eval(REDIS_RATE_LIMIT_LUA_SCRIPT, 1, rlKey, "2000"),
|
||||
);
|
||||
} else {
|
||||
count = Number(await redis.incr(rlKey));
|
||||
if (count === 1) {
|
||||
await redis.pexpire(rlKey, 2000);
|
||||
}
|
||||
}
|
||||
this.trackLocalRateWindow(clientId);
|
||||
|
||||
if (count > effectiveRateLimit) {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: `Rate limit of ${effectiveRateLimit} events/sec exceeded for client ${clientId}`,
|
||||
code: "RATE_LIMIT_EXCEEDED",
|
||||
};
|
||||
}
|
||||
|
||||
// 4. Atomic claim of idempotency key after all checks succeed
|
||||
if (this.enableIdempotency && eventId) {
|
||||
const claimed = await redis.set(
|
||||
idemKey,
|
||||
"1",
|
||||
"PX",
|
||||
this.IDEMPOTENCY_TTL_MS,
|
||||
"NX",
|
||||
);
|
||||
this.recordIdempotency(clientId, eventId);
|
||||
if (claimed !== "OK") {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: "Duplicate event dropped silently",
|
||||
code: "DUPLICATE_EVENT",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return { allowed: true };
|
||||
} catch {
|
||||
return this.validateEvent(clientId, eventId, payload, clientMaxRate);
|
||||
}
|
||||
}
|
||||
|
||||
public static getStats() {
|
||||
const now = Date.now();
|
||||
for (const [k, timestamp] of this.idempotencySet.entries()) {
|
||||
@@ -191,10 +387,19 @@ export class SecurityEngine {
|
||||
}
|
||||
}
|
||||
|
||||
const mode: SecurityMode = this.supportsSharedSecurityRedis(
|
||||
DataBufferEngine.getRedisClient(),
|
||||
)
|
||||
? "redis_shared"
|
||||
: this.lastSecurityMode === "redis_shared"
|
||||
? "degraded_local"
|
||||
: this.lastSecurityMode;
|
||||
|
||||
return {
|
||||
activeIdempotencyKeys: this.idempotencySet.size,
|
||||
rateLimitedClientsCount: this.rateLimitMap.size,
|
||||
maxPayloadSizeBytes: this.maxPayloadSizeBytes,
|
||||
mode,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
+364
-100
@@ -1,5 +1,7 @@
|
||||
import crypto from 'node:crypto';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { StringDecoder } from 'node:string_decoder';
|
||||
import { Parser } from 'json2csv';
|
||||
|
||||
export interface DataEvent {
|
||||
@@ -34,9 +36,17 @@ function sleepMs(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
export function resolveDefaultStorageDir(): string {
|
||||
const configured = (process.env.DATASTACK_STORAGE_DIR || '').trim();
|
||||
if (configured) {
|
||||
return path.resolve(configured);
|
||||
}
|
||||
return path.resolve(process.cwd(), 'storage', 'data');
|
||||
}
|
||||
|
||||
export class CSVStorageManager {
|
||||
// Base storage directory: /storage/data
|
||||
public static baseStorageDir = path.resolve(process.cwd(), 'storage', 'data');
|
||||
// Base storage directory: DATASTACK_STORAGE_DIR or /storage/data
|
||||
public static baseStorageDir = resolveDefaultStorageDir();
|
||||
public static maxFileSizeBytes = 100 * 1024 * 1024; // 100MB rotation limit
|
||||
private static rowCountCache = new Map<string, RowCountCacheEntry>();
|
||||
|
||||
@@ -44,12 +54,94 @@ export class CSVStorageManager {
|
||||
this.rowCountCache.clear();
|
||||
}
|
||||
|
||||
public static sanitizeClientId(clientId: string): string {
|
||||
const raw = String(clientId || 'unknown').trim().replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
return raw || 'unknown';
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures client directory exists
|
||||
* Checks whether the last byte of a non-empty file is LF (\n)
|
||||
*/
|
||||
private static fileEndsWithNewline(filePath: string, sizeBytes: number): boolean {
|
||||
if (sizeBytes <= 0) return false;
|
||||
const fd = fs.openSync(filePath, 'r');
|
||||
try {
|
||||
const lastByte = Buffer.allocUnsafe(1);
|
||||
const read = fs.readSync(fd, lastByte, 0, 1, sizeBytes - 1);
|
||||
return read === 1 && lastByte[0] === 0x0a;
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
}
|
||||
|
||||
private static removeStaleLockAtomically(lockPath: string): void {
|
||||
try {
|
||||
const lockStat = fs.lstatSync(lockPath);
|
||||
if (Date.now() - lockStat.mtimeMs <= STALE_LOCK_AGE_MS) {
|
||||
return;
|
||||
}
|
||||
const staleTmp = `${lockPath}.stale.${crypto.randomUUID()}`;
|
||||
fs.renameSync(lockPath, staleTmp);
|
||||
try {
|
||||
fs.unlinkSync(staleTmp);
|
||||
} catch {
|
||||
// Ignore cleanup failure of already-renamed stale token
|
||||
}
|
||||
} catch {
|
||||
// Lock was released or claimed concurrently
|
||||
}
|
||||
}
|
||||
|
||||
private static releaseLockIfOwned(lockPath: string, lockValue: string): void {
|
||||
try {
|
||||
if (!fs.existsSync(lockPath)) return;
|
||||
const current = fs.readFileSync(lockPath, 'utf8');
|
||||
if (current === lockValue) {
|
||||
fs.unlinkSync(lockPath);
|
||||
}
|
||||
} catch {
|
||||
// Ignore cleanup failure if already unlinked
|
||||
}
|
||||
}
|
||||
|
||||
private static async removeStaleLockAtomicallyAsync(lockPath: string): Promise<void> {
|
||||
try {
|
||||
const lockStat = await fs.promises.lstat(lockPath);
|
||||
if (Date.now() - lockStat.mtimeMs <= STALE_LOCK_AGE_MS) {
|
||||
return;
|
||||
}
|
||||
const staleTmp = `${lockPath}.stale.${crypto.randomUUID()}`;
|
||||
await fs.promises.rename(lockPath, staleTmp);
|
||||
await fs.promises.unlink(staleTmp).catch(() => undefined);
|
||||
} catch {
|
||||
// Lock was released or claimed concurrently
|
||||
}
|
||||
}
|
||||
|
||||
private static async releaseLockIfOwnedAsync(
|
||||
lockPath: string,
|
||||
lockValue: string,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const current = await fs.promises.readFile(lockPath, 'utf8');
|
||||
if (current === lockValue) {
|
||||
await fs.promises.unlink(lockPath);
|
||||
}
|
||||
} catch {
|
||||
// Ignore cleanup failure if already unlinked
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures client directory exists safely within baseStorageDir
|
||||
*/
|
||||
public static ensureClientDir(clientId: string): string {
|
||||
const safeClientId = clientId.replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
const clientDir = path.join(this.baseStorageDir, safeClientId);
|
||||
const safeClientId = this.sanitizeClientId(clientId);
|
||||
const resolvedBase = path.resolve(this.baseStorageDir);
|
||||
const clientDir = path.resolve(resolvedBase, safeClientId);
|
||||
if (!clientDir.startsWith(resolvedBase + path.sep)) {
|
||||
throw new Error(`Invalid storage client directory: ${clientId}`);
|
||||
}
|
||||
if (!fs.existsSync(clientDir)) {
|
||||
fs.mkdirSync(clientDir, { recursive: true });
|
||||
}
|
||||
@@ -67,6 +159,7 @@ export class CSVStorageManager {
|
||||
const clientDir = this.ensureClientDir(clientId);
|
||||
const date = this.extractSafePartitionDate(dateStr);
|
||||
const lockPath = path.join(clientDir, `.${date}.csv.lock`);
|
||||
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
|
||||
const deadline = Date.now() + LOCK_TIMEOUT_MS;
|
||||
|
||||
let acquired = false;
|
||||
@@ -74,7 +167,7 @@ export class CSVStorageManager {
|
||||
try {
|
||||
const fd = fs.openSync(lockPath, 'wx');
|
||||
try {
|
||||
fs.writeFileSync(fd, `${process.pid}:${Date.now()}`, 'utf8');
|
||||
fs.writeFileSync(fd, lockValue, 'utf8');
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
@@ -85,15 +178,7 @@ export class CSVStorageManager {
|
||||
if (code !== 'EEXIST') {
|
||||
throw err;
|
||||
}
|
||||
try {
|
||||
const lockStat = fs.statSync(lockPath);
|
||||
if (Date.now() - lockStat.mtimeMs > STALE_LOCK_AGE_MS) {
|
||||
fs.unlinkSync(lockPath);
|
||||
continue;
|
||||
}
|
||||
} catch {
|
||||
// Lock was released concurrently
|
||||
}
|
||||
this.removeStaleLockAtomically(lockPath);
|
||||
const jitter =
|
||||
LOCK_RETRY_MIN_MS +
|
||||
Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS));
|
||||
@@ -108,13 +193,7 @@ export class CSVStorageManager {
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
try {
|
||||
if (fs.existsSync(lockPath)) {
|
||||
fs.unlinkSync(lockPath);
|
||||
}
|
||||
} catch {
|
||||
// Ignore cleanup failure if already unlinked
|
||||
}
|
||||
this.releaseLockIfOwned(lockPath, lockValue);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -158,51 +237,58 @@ export class CSVStorageManager {
|
||||
* Stateful RFC 4180 quote-aware chunked row counter so multi-100MB CSVs never load wholly into RAM
|
||||
*/
|
||||
private static createChunkRowCounter() {
|
||||
const decoder = new StringDecoder('utf8');
|
||||
let inQuotes = false;
|
||||
let pendingQuote = false;
|
||||
let currentLineHasChars = false;
|
||||
let totalRows = 0;
|
||||
|
||||
return {
|
||||
pushChunk(chunk: Buffer | string) {
|
||||
const str = typeof chunk === 'string' ? chunk : chunk.toString('utf8');
|
||||
for (let i = 0; i < str.length; i += 1) {
|
||||
const ch = str[i];
|
||||
if (pendingQuote) {
|
||||
pendingQuote = false;
|
||||
if (ch === '"') {
|
||||
// Escaped quote "" across chunk boundary or inline
|
||||
continue;
|
||||
}
|
||||
inQuotes = !inQuotes;
|
||||
}
|
||||
|
||||
const processStr = (str: string) => {
|
||||
for (let i = 0; i < str.length; i += 1) {
|
||||
const ch = str[i];
|
||||
if (pendingQuote) {
|
||||
pendingQuote = false;
|
||||
if (ch === '"') {
|
||||
currentLineHasChars = true;
|
||||
if (inQuotes) {
|
||||
if (i + 1 < str.length) {
|
||||
if (str[i + 1] === '"') {
|
||||
i += 1;
|
||||
} else {
|
||||
inQuotes = false;
|
||||
}
|
||||
// Escaped quote "" across chunk boundary or inline
|
||||
continue;
|
||||
}
|
||||
inQuotes = !inQuotes;
|
||||
}
|
||||
|
||||
if (ch === '"') {
|
||||
currentLineHasChars = true;
|
||||
if (inQuotes) {
|
||||
if (i + 1 < str.length) {
|
||||
if (str[i + 1] === '"') {
|
||||
i += 1;
|
||||
} else {
|
||||
pendingQuote = true;
|
||||
inQuotes = false;
|
||||
}
|
||||
} else {
|
||||
inQuotes = true;
|
||||
pendingQuote = true;
|
||||
}
|
||||
} else if (ch === '\n' && !inQuotes) {
|
||||
if (currentLineHasChars) {
|
||||
totalRows += 1;
|
||||
currentLineHasChars = false;
|
||||
}
|
||||
} else if (ch !== '\r' && ch !== ' ' && ch !== '\t') {
|
||||
currentLineHasChars = true;
|
||||
} else {
|
||||
inQuotes = true;
|
||||
}
|
||||
} else if (ch === '\n' && !inQuotes) {
|
||||
if (currentLineHasChars) {
|
||||
totalRows += 1;
|
||||
currentLineHasChars = false;
|
||||
}
|
||||
} else if (ch !== '\r' && ch !== ' ' && ch !== '\t') {
|
||||
currentLineHasChars = true;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
pushChunk(chunk: Buffer | string) {
|
||||
const str = typeof chunk === 'string' ? chunk : decoder.write(chunk);
|
||||
if (str) processStr(str);
|
||||
},
|
||||
finish(): number {
|
||||
const trailing = decoder.end();
|
||||
if (trailing) processStr(trailing);
|
||||
if (currentLineHasChars) {
|
||||
totalRows += 1;
|
||||
}
|
||||
@@ -243,7 +329,11 @@ export class CSVStorageManager {
|
||||
/**
|
||||
* Resolves target CSV file path with daily and configured MB rotation rules
|
||||
*/
|
||||
public static getTargetFilePath(clientId: string, dateStr?: string): string {
|
||||
public static getTargetFilePath(
|
||||
clientId: string,
|
||||
dateStr?: string,
|
||||
incomingBytes = 0,
|
||||
): string {
|
||||
const clientDir = this.ensureClientDir(clientId);
|
||||
const date = this.extractSafePartitionDate(dateStr);
|
||||
let baseFileName = `${date}.csv`;
|
||||
@@ -252,16 +342,24 @@ export class CSVStorageManager {
|
||||
// Check for file rotation
|
||||
let partIndex = 1;
|
||||
while (fs.existsSync(targetPath)) {
|
||||
let stats: fs.Stats;
|
||||
try {
|
||||
const stats = fs.statSync(targetPath);
|
||||
if (stats.size >= this.maxFileSizeBytes) {
|
||||
partIndex += 1;
|
||||
baseFileName = `${date}-part${partIndex}.csv`;
|
||||
targetPath = path.join(clientDir, baseFileName);
|
||||
} else {
|
||||
stats = fs.statSync(targetPath);
|
||||
} catch (err: unknown) {
|
||||
if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') {
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
throw err;
|
||||
}
|
||||
const overflowsLimit =
|
||||
stats.size > 0 &&
|
||||
incomingBytes > 0 &&
|
||||
stats.size + incomingBytes > this.maxFileSizeBytes;
|
||||
if (stats.size >= this.maxFileSizeBytes || overflowsLimit) {
|
||||
partIndex += 1;
|
||||
baseFileName = `${date}-part${partIndex}.csv`;
|
||||
targetPath = path.join(clientDir, baseFileName);
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -280,49 +378,63 @@ export class CSVStorageManager {
|
||||
return { written: 0, filesUpdated: [] };
|
||||
}
|
||||
|
||||
// Group events by client + date
|
||||
const grouped = new Map<string, DataEvent[]>();
|
||||
// Group events by client + date without ambiguous delimiter splitting
|
||||
const grouped = new Map<string, { clientId: string; date: string; events: DataEvent[] }>();
|
||||
for (const evt of events) {
|
||||
const date = this.extractSafePartitionDate(evt.timestamp);
|
||||
const key = `${evt.clientId}:::${date}`;
|
||||
if (!grouped.has(key)) {
|
||||
grouped.set(key, []);
|
||||
const clientId = String(evt.clientId || 'unknown');
|
||||
const key = JSON.stringify([clientId, date]);
|
||||
let bucket = grouped.get(key);
|
||||
if (!bucket) {
|
||||
bucket = { clientId, date, events: [] };
|
||||
grouped.set(key, bucket);
|
||||
}
|
||||
grouped.get(key)!.push(evt);
|
||||
bucket.events.push(evt);
|
||||
}
|
||||
|
||||
const filesUpdated: string[] = [];
|
||||
const fields = ['clientId', 'eventId', 'timestamp', 'payload'];
|
||||
|
||||
for (const [key, batch] of grouped.entries()) {
|
||||
const [clientId, date] = key.split(':::');
|
||||
for (const { clientId, date, events: batch } of grouped.values()) {
|
||||
const cleanBatch = batch.map((item) => ({
|
||||
clientId: item.clientId,
|
||||
eventId: item.eventId,
|
||||
timestamp: item.timestamp,
|
||||
payload:
|
||||
typeof item.payload === 'string'
|
||||
? item.payload
|
||||
: JSON.stringify(item.payload),
|
||||
}));
|
||||
|
||||
const bodyCsvNoHeader = new Parser({ fields, header: false, eol: '\n' }).parse(
|
||||
cleanBatch,
|
||||
);
|
||||
const incomingBytes = Buffer.byteLength(bodyCsvNoHeader, 'utf8') + 1;
|
||||
|
||||
const targetPath = await this.withPartitionLock(clientId, date, () => {
|
||||
const resolvedPath = this.getTargetFilePath(clientId, date);
|
||||
let hasContent = false;
|
||||
const resolvedPath = this.getTargetFilePath(clientId, date, incomingBytes);
|
||||
let sizeBytes = 0;
|
||||
try {
|
||||
hasContent =
|
||||
fs.existsSync(resolvedPath) && fs.statSync(resolvedPath).size > 0;
|
||||
if (fs.existsSync(resolvedPath)) {
|
||||
sizeBytes = fs.statSync(resolvedPath).size;
|
||||
}
|
||||
} catch {
|
||||
hasContent = false;
|
||||
sizeBytes = 0;
|
||||
}
|
||||
|
||||
const cleanBatch = batch.map((item) => ({
|
||||
clientId: item.clientId,
|
||||
eventId: item.eventId,
|
||||
timestamp: item.timestamp,
|
||||
payload:
|
||||
typeof item.payload === 'string'
|
||||
? item.payload
|
||||
: JSON.stringify(item.payload),
|
||||
}));
|
||||
|
||||
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
|
||||
const csvData = parser.parse(cleanBatch);
|
||||
const hasContent = sizeBytes > 0;
|
||||
|
||||
if (hasContent) {
|
||||
fs.appendFileSync(resolvedPath, '\n' + csvData, 'utf8');
|
||||
const endsWithNl = this.fileEndsWithNewline(resolvedPath, sizeBytes);
|
||||
fs.appendFileSync(
|
||||
resolvedPath,
|
||||
(endsWithNl ? '' : '\n') + bodyCsvNoHeader,
|
||||
'utf8',
|
||||
);
|
||||
} else {
|
||||
fs.writeFileSync(resolvedPath, csvData, 'utf8');
|
||||
const fullCsv = new Parser({ fields, header: true, eol: '\n' }).parse(
|
||||
cleanBatch,
|
||||
);
|
||||
fs.writeFileSync(resolvedPath, fullCsv, 'utf8');
|
||||
}
|
||||
|
||||
// Invalidate row cache for modified file immediately
|
||||
@@ -351,13 +463,25 @@ export class CSVStorageManager {
|
||||
const clientDirs = fs.readdirSync(this.baseStorageDir);
|
||||
for (const clientDir of clientDirs) {
|
||||
const fullClientPath = path.join(this.baseStorageDir, clientDir);
|
||||
if (!fs.statSync(fullClientPath).isDirectory()) continue;
|
||||
let dirLstat: fs.Stats;
|
||||
try {
|
||||
dirLstat = fs.lstatSync(fullClientPath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
|
||||
|
||||
const files = fs.readdirSync(fullClientPath);
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
const fullFilePath = path.join(fullClientPath, file);
|
||||
const stats = fs.statSync(fullFilePath);
|
||||
let stats: fs.Stats;
|
||||
try {
|
||||
stats = fs.lstatSync(fullFilePath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (stats.isSymbolicLink() || !stats.isFile()) continue;
|
||||
activePaths.add(fullFilePath);
|
||||
|
||||
const cached = this.rowCountCache.get(fullFilePath);
|
||||
@@ -422,15 +546,15 @@ export class CSVStorageManager {
|
||||
const clientDirs = await fs.promises.readdir(this.baseStorageDir);
|
||||
for (const clientDir of clientDirs) {
|
||||
const fullClientPath = path.join(this.baseStorageDir, clientDir);
|
||||
const dirStat = await fs.promises.stat(fullClientPath).catch(() => null);
|
||||
if (!dirStat || !dirStat.isDirectory()) continue;
|
||||
const dirLstat = await fs.promises.lstat(fullClientPath).catch(() => null);
|
||||
if (!dirLstat || dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
|
||||
|
||||
const files = await fs.promises.readdir(fullClientPath);
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
const fullFilePath = path.join(fullClientPath, file);
|
||||
const stats = await fs.promises.stat(fullFilePath).catch(() => null);
|
||||
if (!stats) continue;
|
||||
const stats = await fs.promises.lstat(fullFilePath).catch(() => null);
|
||||
if (!stats || stats.isSymbolicLink() || !stats.isFile()) continue;
|
||||
activePaths.add(fullFilePath);
|
||||
|
||||
const cached = this.rowCountCache.get(fullFilePath);
|
||||
@@ -481,7 +605,7 @@ export class CSVStorageManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Prune CSV partitions older than autoPruneDays without counting row contents
|
||||
* Prune CSV partitions older than autoPruneDays under cooperative partition locks
|
||||
*/
|
||||
public static pruneExpiredPartitions(autoPruneDays: number): number {
|
||||
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
|
||||
@@ -502,7 +626,13 @@ export class CSVStorageManager {
|
||||
const clientDirs = fs.readdirSync(this.baseStorageDir);
|
||||
for (const clientDir of clientDirs) {
|
||||
const fullClientPath = path.join(this.baseStorageDir, clientDir);
|
||||
if (!fs.statSync(fullClientPath).isDirectory()) continue;
|
||||
let dirLstat: fs.Stats;
|
||||
try {
|
||||
dirLstat = fs.lstatSync(fullClientPath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
|
||||
|
||||
const files = fs.readdirSync(fullClientPath);
|
||||
for (const file of files) {
|
||||
@@ -510,6 +640,45 @@ export class CSVStorageManager {
|
||||
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
|
||||
const fullPath = path.join(fullClientPath, file);
|
||||
let fileLstat: fs.Stats;
|
||||
try {
|
||||
fileLstat = fs.lstatSync(fullPath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (fileLstat.isSymbolicLink() || !fileLstat.isFile()) continue;
|
||||
|
||||
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
|
||||
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
|
||||
let acquired = false;
|
||||
try {
|
||||
const fd = fs.openSync(lockPath, 'wx');
|
||||
try {
|
||||
fs.writeFileSync(fd, lockValue, 'utf8');
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
acquired = true;
|
||||
} catch (err: unknown) {
|
||||
if ((err as NodeJS.ErrnoException)?.code === 'EEXIST') {
|
||||
this.removeStaleLockAtomically(lockPath);
|
||||
try {
|
||||
const fd = fs.openSync(lockPath, 'wx');
|
||||
try {
|
||||
fs.writeFileSync(fd, lockValue, 'utf8');
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
acquired = true;
|
||||
} catch {
|
||||
// Partition is currently locked by an active writer; skip safely
|
||||
acquired = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!acquired) continue;
|
||||
|
||||
try {
|
||||
if (fs.existsSync(fullPath)) {
|
||||
fs.unlinkSync(fullPath);
|
||||
@@ -518,6 +687,8 @@ export class CSVStorageManager {
|
||||
}
|
||||
} catch {
|
||||
// Ignore file removal error
|
||||
} finally {
|
||||
this.releaseLockIfOwned(lockPath, lockValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -529,7 +700,96 @@ export class CSVStorageManager {
|
||||
return prunedCount;
|
||||
}
|
||||
|
||||
private static formatStorageStats(files: StorageFileMeta[]) {
|
||||
/**
|
||||
* Non-blocking async partition pruner using fs.promises under cooperative partition locks
|
||||
*/
|
||||
public static async pruneExpiredPartitionsAsync(
|
||||
autoPruneDays: number,
|
||||
): Promise<number> {
|
||||
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const baseStat = await fs.promises.lstat(this.baseStorageDir).catch(() => null);
|
||||
if (!baseStat || !baseStat.isDirectory()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const cutoffDate = new Date(
|
||||
Date.now() - Math.floor(autoPruneDays) * 86_400_000,
|
||||
)
|
||||
.toISOString()
|
||||
.split('T')[0];
|
||||
|
||||
let prunedCount = 0;
|
||||
try {
|
||||
const clientDirs = await fs.promises.readdir(this.baseStorageDir);
|
||||
for (const clientDir of clientDirs) {
|
||||
const fullClientPath = path.join(this.baseStorageDir, clientDir);
|
||||
const dirLstat = await fs.promises.lstat(fullClientPath).catch(() => null);
|
||||
if (!dirLstat || dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const files = await fs.promises.readdir(fullClientPath).catch(() => [] as string[]);
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
|
||||
const fullPath = path.join(fullClientPath, file);
|
||||
const fileLstat = await fs.promises.lstat(fullPath).catch(() => null);
|
||||
if (!fileLstat || fileLstat.isSymbolicLink() || !fileLstat.isFile()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
|
||||
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
|
||||
let acquired = false;
|
||||
const tryAcquireLock = async (): Promise<boolean> => {
|
||||
const handle = await fs.promises.open(lockPath, 'wx');
|
||||
try {
|
||||
await handle.writeFile(lockValue, 'utf8');
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
try {
|
||||
acquired = await tryAcquireLock();
|
||||
} catch (err: unknown) {
|
||||
if ((err as NodeJS.ErrnoException)?.code === 'EEXIST') {
|
||||
await this.removeStaleLockAtomicallyAsync(lockPath);
|
||||
try {
|
||||
acquired = await tryAcquireLock();
|
||||
} catch {
|
||||
acquired = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!acquired) continue;
|
||||
|
||||
try {
|
||||
await fs.promises.unlink(fullPath);
|
||||
this.rowCountCache.delete(fullPath);
|
||||
prunedCount += 1;
|
||||
} catch {
|
||||
// Ignore individual unlink error or concurrent removal
|
||||
} finally {
|
||||
await this.releaseLockIfOwnedAsync(lockPath, lockValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore directory scan failure
|
||||
}
|
||||
|
||||
return prunedCount;
|
||||
}
|
||||
|
||||
public static formatStorageStats(files: StorageFileMeta[]) {
|
||||
const totalBytes = files.reduce((acc, f) => acc + f.sizeBytes, 0);
|
||||
const totalRows = files.reduce((acc, f) => acc + f.linesCount, 0);
|
||||
const uniqueClients = new Set(files.map((f) => f.clientId)).size;
|
||||
@@ -569,17 +829,21 @@ export class CSVStorageManager {
|
||||
* Reads file content for export / download with path traversal protection
|
||||
*/
|
||||
public static getFileContent(clientId: string, filename: string): string | null {
|
||||
const safeClientId = String(clientId || '').replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
const safeClientId = this.sanitizeClientId(clientId);
|
||||
const safeFilename = path.basename(String(filename || ''));
|
||||
if (!safeClientId || !safeFilename || !safeFilename.endsWith('.csv')) {
|
||||
return null;
|
||||
}
|
||||
const fullPath = path.resolve(this.baseStorageDir, safeClientId, safeFilename);
|
||||
const resolvedBase = path.resolve(this.baseStorageDir);
|
||||
const fullPath = path.resolve(resolvedBase, safeClientId, safeFilename);
|
||||
if (!fullPath.startsWith(resolvedBase + path.sep)) {
|
||||
return null;
|
||||
}
|
||||
if (fs.existsSync(fullPath)) {
|
||||
const stats = fs.lstatSync(fullPath);
|
||||
if (stats.isSymbolicLink() || !stats.isFile()) {
|
||||
return null;
|
||||
}
|
||||
return fs.readFileSync(fullPath, 'utf8');
|
||||
}
|
||||
return null;
|
||||
|
||||
@@ -22,11 +22,23 @@ export interface AdminSettings {
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface SettingsBridgeSyncStatus {
|
||||
attempted: boolean;
|
||||
ok: boolean;
|
||||
synced: boolean;
|
||||
status?: number;
|
||||
error?: string;
|
||||
checkedAt: string;
|
||||
}
|
||||
|
||||
type SettingsGlobals = typeof globalThis & {
|
||||
__dataStackAdminSettings?: AdminSettings;
|
||||
__dataStackSettingsHydrated?: boolean;
|
||||
__dataStackLastSettingsBridgeSync?: SettingsBridgeSyncStatus;
|
||||
__dataStackLastPruneMeta?: { days: number; ranAtMs: number };
|
||||
};
|
||||
|
||||
const PRUNE_COOLDOWN_MS = 60 * 60 * 1000;
|
||||
const globals = globalThis as SettingsGlobals;
|
||||
|
||||
function getDefaultSettings(): AdminSettings {
|
||||
@@ -53,7 +65,7 @@ function getSettingsState(): AdminSettings {
|
||||
}
|
||||
|
||||
export class SettingsManager {
|
||||
private static applyRuntimeSettings(settings: AdminSettings) {
|
||||
private static applyRuntimeSettings(settings: AdminSettings, forcePrune = false) {
|
||||
SecurityEngine.maxEventsPerSec = settings.maxEventsPerSec;
|
||||
SecurityEngine.enableIdempotency = settings.enableIdempotency;
|
||||
SecurityEngine.maxPayloadSizeBytes = settings.maxPayloadSizeKb * 1024;
|
||||
@@ -68,7 +80,27 @@ export class SettingsManager {
|
||||
settings.autoPruneDays > 0 &&
|
||||
(!hasConfiguredDb || globals.__dataStackSettingsHydrated === true)
|
||||
) {
|
||||
CSVStorageManager.pruneExpiredPartitions(settings.autoPruneDays);
|
||||
const prev = globals.__dataStackLastPruneMeta;
|
||||
const now = Date.now();
|
||||
const shouldPrune =
|
||||
forcePrune ||
|
||||
!prev ||
|
||||
prev.days !== settings.autoPruneDays ||
|
||||
now - prev.ranAtMs >= PRUNE_COOLDOWN_MS;
|
||||
if (shouldPrune) {
|
||||
globals.__dataStackLastPruneMeta = {
|
||||
days: settings.autoPruneDays,
|
||||
ranAtMs: now,
|
||||
};
|
||||
void CSVStorageManager.pruneExpiredPartitionsAsync(
|
||||
settings.autoPruneDays,
|
||||
).catch((err) => {
|
||||
console.warn(
|
||||
"[DataStack Storage] Background async partition prune failed:",
|
||||
err,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -94,9 +126,17 @@ export class SettingsManager {
|
||||
row.auto_prune_days !== undefined && row.auto_prune_days !== null
|
||||
? Number(row.auto_prune_days)
|
||||
: NaN;
|
||||
const envSocketPort = Number(process.env.PORT_SOCKET);
|
||||
const envWebPort = Number(process.env.PORT);
|
||||
const loaded: AdminSettings = {
|
||||
serverPort: Number(row.server_port) || 4000,
|
||||
webPort: Number(row.web_port) || 3000,
|
||||
serverPort:
|
||||
Number.isFinite(envSocketPort) && envSocketPort > 0
|
||||
? envSocketPort
|
||||
: Number(row.server_port) || 4000,
|
||||
webPort:
|
||||
Number.isFinite(envWebPort) && envWebPort > 0
|
||||
? envWebPort
|
||||
: Number(row.web_port) || 3000,
|
||||
maxEventsPerSec: Number(row.max_events_per_sec) || 100,
|
||||
maxPayloadSizeKb: Number(row.max_payload_size_kb) || 100,
|
||||
batchFlushThreshold: Number(row.batch_flush_threshold) || 500,
|
||||
@@ -117,6 +157,12 @@ export class SettingsManager {
|
||||
|
||||
// Initialize default row in PostgreSQL if not present yet
|
||||
const defaults = getSettingsState();
|
||||
const hasValidProductionJwt =
|
||||
process.env.NODE_ENV !== "production" ||
|
||||
defaults.jwtSecret.trim().length >= 32;
|
||||
if (!hasValidProductionJwt) {
|
||||
return defaults;
|
||||
}
|
||||
await this.persistSettingsToDb(defaults);
|
||||
globals.__dataStackSettingsHydrated = true;
|
||||
return defaults;
|
||||
@@ -131,8 +177,23 @@ export class SettingsManager {
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (!pool) return;
|
||||
|
||||
if (
|
||||
process.env.NODE_ENV === "production" &&
|
||||
settings.jwtSecret.trim().length < 32
|
||||
) {
|
||||
throw new Error(
|
||||
"DATABASE_WRITE_FAILED: Cannot persist settings without a valid DATASTACK_JWT_SECRET (>= 32 characters) in production.",
|
||||
);
|
||||
}
|
||||
|
||||
const ready = await DatabaseEngine.ensureSchema();
|
||||
if (!ready) {
|
||||
throw new Error(
|
||||
"DATABASE_WRITE_FAILED: PostgreSQL schema is unavailable while saving settings.",
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
await pool.query(
|
||||
`INSERT INTO admin_settings (
|
||||
id, server_port, web_port, max_events_per_sec, max_payload_size_kb,
|
||||
@@ -164,8 +225,9 @@ export class SettingsManager {
|
||||
settings.autoPruneDays,
|
||||
],
|
||||
);
|
||||
} catch {
|
||||
// Ignore DB failure; in-memory state remains active
|
||||
} catch (err: unknown) {
|
||||
const detail = err instanceof Error ? err.message : "Unknown DB error";
|
||||
throw new Error(`DATABASE_WRITE_FAILED: Failed to persist settings (${detail})`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -222,20 +284,35 @@ export class SettingsManager {
|
||||
};
|
||||
}
|
||||
|
||||
public static getLastBridgeSyncStatus(): SettingsBridgeSyncStatus | null {
|
||||
return globals.__dataStackLastSettingsBridgeSync ?? null;
|
||||
}
|
||||
|
||||
private static async notifyIngestionSettingsSync(
|
||||
settings: AdminSettings,
|
||||
authJwtSecret: string,
|
||||
jwtSecretChanged: boolean,
|
||||
): Promise<void> {
|
||||
): Promise<SettingsBridgeSyncStatus> {
|
||||
const checkedAt = new Date().toISOString();
|
||||
const headers = getInternalIngestionHeaders(authJwtSecret);
|
||||
if (!headers["x-datastack-internal-secret"]) return;
|
||||
if (!headers["x-datastack-internal-secret"]) {
|
||||
const result: SettingsBridgeSyncStatus = {
|
||||
attempted: true,
|
||||
ok: false,
|
||||
synced: false,
|
||||
error: "Internal bridge secret is not configured",
|
||||
checkedAt,
|
||||
};
|
||||
globals.__dataStackLastSettingsBridgeSync = result;
|
||||
return result;
|
||||
}
|
||||
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||
).replace(/\/$/, "");
|
||||
|
||||
try {
|
||||
await fetch(`${ingestionBaseUrl}/internal/settings-sync`, {
|
||||
const res = await fetch(`${ingestionBaseUrl}/internal/settings-sync`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
@@ -254,8 +331,46 @@ export class SettingsManager {
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(1500),
|
||||
});
|
||||
} catch {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
if (!res.ok) {
|
||||
const errorMsg = `Port 4000 settings-sync failed with HTTP ${res.status}`;
|
||||
console.warn(`[DataStack Bridge] ${errorMsg}`);
|
||||
const result: SettingsBridgeSyncStatus = {
|
||||
attempted: true,
|
||||
ok: false,
|
||||
synced: false,
|
||||
status: res.status,
|
||||
error: errorMsg,
|
||||
checkedAt,
|
||||
};
|
||||
globals.__dataStackLastSettingsBridgeSync = result;
|
||||
return result;
|
||||
}
|
||||
const result: SettingsBridgeSyncStatus = {
|
||||
attempted: true,
|
||||
ok: true,
|
||||
synced: true,
|
||||
status: res.status,
|
||||
checkedAt,
|
||||
};
|
||||
globals.__dataStackLastSettingsBridgeSync = result;
|
||||
return result;
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : "Port 4000 settings-sync unreachable";
|
||||
if (
|
||||
process.env.NODE_ENV === "production" ||
|
||||
Boolean(process.env.INGESTION_INTERNAL_URL)
|
||||
) {
|
||||
console.warn(`[DataStack Bridge] ${message}`);
|
||||
}
|
||||
const result: SettingsBridgeSyncStatus = {
|
||||
attempted: true,
|
||||
ok: false,
|
||||
synced: false,
|
||||
error: message,
|
||||
checkedAt,
|
||||
};
|
||||
globals.__dataStackLastSettingsBridgeSync = result;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -270,11 +385,13 @@ export class SettingsManager {
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
globals.__dataStackAdminSettings = nextSettings;
|
||||
this.applyRuntimeSettings(nextSettings);
|
||||
this.applyRuntimeSettings(nextSettings, updates.autoPruneDays !== undefined);
|
||||
if (jwtSecretChanged) {
|
||||
SDKManager.resignAllClientTokens();
|
||||
}
|
||||
void this.persistSettingsToDb(nextSettings);
|
||||
void this.persistSettingsToDb(nextSettings).catch((err) => {
|
||||
console.error("[DataStack Settings] Background DB persist failed:", err);
|
||||
});
|
||||
void this.notifyIngestionSettingsSync(
|
||||
nextSettings,
|
||||
previousJwtSecret,
|
||||
@@ -297,20 +414,80 @@ export class SettingsManager {
|
||||
...updates,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
globals.__dataStackAdminSettings = nextSettings;
|
||||
this.applyRuntimeSettings(nextSettings);
|
||||
if (jwtSecretChanged) {
|
||||
await SDKManager.resignAllClientTokensAsync();
|
||||
|
||||
if (
|
||||
jwtSecretChanged &&
|
||||
process.env.NODE_ENV === "production" &&
|
||||
nextSettings.jwtSecret.trim().length < 32
|
||||
) {
|
||||
throw new Error(
|
||||
"INVALID_JWT_SECRET: JWT secret must be at least 32 characters in production.",
|
||||
);
|
||||
}
|
||||
|
||||
if (jwtSecretChanged) {
|
||||
// BUG-17: Coordinate Port 4000 BEFORE committing JWT rotation on Port 3000
|
||||
// so a bridge rejection or unreachable production ingestion server never causes secret split-brain.
|
||||
const preSyncStatus = await this.notifyIngestionSettingsSync(
|
||||
nextSettings,
|
||||
previousJwtSecret,
|
||||
true,
|
||||
);
|
||||
|
||||
if (!preSyncStatus.synced) {
|
||||
const isExplicitHttpRejection = typeof preSyncStatus.status === "number";
|
||||
const isProductionBridgeRequired =
|
||||
process.env.NODE_ENV === "production" &&
|
||||
Boolean(process.env.INGESTION_INTERNAL_URL);
|
||||
|
||||
if (isExplicitHttpRejection || isProductionBridgeRequired) {
|
||||
throw new Error(
|
||||
`JWT_ROTATION_BRIDGE_FAILED: Aborted JWT rotation because Port 4000 could not synchronize (${preSyncStatus.error || "bridge error"}).`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await this.persistSettingsToDb(nextSettings);
|
||||
globals.__dataStackAdminSettings = nextSettings;
|
||||
this.applyRuntimeSettings(nextSettings, updates.autoPruneDays !== undefined);
|
||||
await SDKManager.resignAllClientTokensAsync();
|
||||
} catch (err) {
|
||||
globals.__dataStackAdminSettings = current;
|
||||
this.applyRuntimeSettings(current, false);
|
||||
await this.persistSettingsToDb(current).catch(() => undefined);
|
||||
if (preSyncStatus.synced) {
|
||||
await this.notifyIngestionSettingsSync(
|
||||
current,
|
||||
nextSettings.jwtSecret,
|
||||
true,
|
||||
).catch(() => undefined);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
const clientSync = await SDKManager.syncAllClientsToIngestionAsync();
|
||||
if (!clientSync.synced && preSyncStatus.synced) {
|
||||
globals.__dataStackLastSettingsBridgeSync = {
|
||||
attempted: true,
|
||||
ok: false,
|
||||
synced: false,
|
||||
status: clientSync.status,
|
||||
error: clientSync.error,
|
||||
checkedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
return { ...nextSettings };
|
||||
}
|
||||
|
||||
await this.persistSettingsToDb(nextSettings);
|
||||
globals.__dataStackAdminSettings = nextSettings;
|
||||
this.applyRuntimeSettings(nextSettings, updates.autoPruneDays !== undefined);
|
||||
await this.notifyIngestionSettingsSync(
|
||||
nextSettings,
|
||||
previousJwtSecret,
|
||||
jwtSecretChanged,
|
||||
false,
|
||||
);
|
||||
if (jwtSecretChanged) {
|
||||
await SDKManager.syncAllClientsToIngestionAsync();
|
||||
}
|
||||
return { ...nextSettings };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,6 +34,11 @@ export class ApiEngine {
|
||||
).replace(/\/$/, '');
|
||||
const internalHeaders = getInternalIngestionHeaders();
|
||||
|
||||
let ingestionBridgeStatus: 'online' | 'auth_error' | 'error' | 'offline' = 'offline';
|
||||
let ingestionBridgeError: string | undefined;
|
||||
let liveServerPort = settings.serverPort || 4000;
|
||||
let liveWebPort = settings.webPort || 3000;
|
||||
|
||||
try {
|
||||
const res = await fetch(`${ingestionBaseUrl}/internal/metrics`, {
|
||||
headers: internalHeaders,
|
||||
@@ -42,8 +47,17 @@ export class ApiEngine {
|
||||
});
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
ingestionBridgeStatus = 'online';
|
||||
activeSockets = Number(data.activeSockets) || 0;
|
||||
ingestionRedisConnected = Boolean(data.redisConnected);
|
||||
if (Number.isFinite(Number(data.serverPort)) && Number(data.serverPort) > 0) {
|
||||
liveServerPort = Number(data.serverPort);
|
||||
} else if (Number.isFinite(Number(data.port)) && Number(data.port) > 0) {
|
||||
liveServerPort = Number(data.port);
|
||||
}
|
||||
if (Number.isFinite(Number(data.webPort)) && Number(data.webPort) > 0) {
|
||||
liveWebPort = Number(data.webPort);
|
||||
}
|
||||
if (data.localBufferSize !== undefined) {
|
||||
ingestionLocalBufferSize = Number(data.localBufferSize) || 0;
|
||||
ingestionRedisBufferSize = Number(data.redisBufferSize) || 0;
|
||||
@@ -53,14 +67,36 @@ export class ApiEngine {
|
||||
if (data.securityStats && typeof data.securityStats === 'object') {
|
||||
remoteSecurityStats = data.securityStats;
|
||||
}
|
||||
// BUG-02: Auto-reconcile Port 4000 cold start whenever metrics observe unsynced registry
|
||||
if (data.hasSyncedInMemoryClients === false && clients.length > 0) {
|
||||
await SDKManager.syncAllClientsToIngestionAsync();
|
||||
}
|
||||
} else {
|
||||
ingestionBridgeStatus = res.status === 401 || res.status === 503 ? 'auth_error' : 'error';
|
||||
ingestionBridgeError = `HTTP ${res.status}`;
|
||||
console.warn(`[DataStack Bridge] /internal/metrics failed with ${ingestionBridgeError}`);
|
||||
}
|
||||
} catch {
|
||||
} catch (err: unknown) {
|
||||
// Ingestion server may not be running in single-process dev mode
|
||||
activeSockets = 0;
|
||||
ingestionLocalBufferSize = 0;
|
||||
ingestionRedisBufferSize = 0;
|
||||
ingestionBridgeStatus = 'offline';
|
||||
ingestionBridgeError = err instanceof Error ? err.message : 'Unreachable';
|
||||
}
|
||||
|
||||
const effectiveRedisBufferSize = Math.max(
|
||||
localBufferStats.redisBufferSize || 0,
|
||||
ingestionRedisBufferSize,
|
||||
);
|
||||
const redisConnected = localBufferStats.redisConnected || ingestionRedisConnected;
|
||||
const securityMode: 'redis_shared' | 'bridge_coordinated' | 'degraded_local' =
|
||||
redisConnected
|
||||
? 'redis_shared'
|
||||
: ingestionBridgeStatus === 'online'
|
||||
? 'bridge_coordinated'
|
||||
: 'degraded_local';
|
||||
|
||||
const localSecurityStats = SecurityEngine.getStats();
|
||||
const securityStats = {
|
||||
activeIdempotencyKeys:
|
||||
@@ -74,15 +110,12 @@ export class ApiEngine {
|
||||
Number(remoteSecurityStats?.maxPayloadSizeBytes) ||
|
||||
localSecurityStats.maxPayloadSizeBytes,
|
||||
),
|
||||
mode: securityMode,
|
||||
};
|
||||
|
||||
const effectiveRedisBufferSize = Math.max(
|
||||
localBufferStats.redisBufferSize || 0,
|
||||
ingestionRedisBufferSize,
|
||||
);
|
||||
const bufferStats = {
|
||||
...localBufferStats,
|
||||
redisConnected: localBufferStats.redisConnected || ingestionRedisConnected,
|
||||
redisConnected,
|
||||
redisBufferSize: effectiveRedisBufferSize,
|
||||
currentBufferSize:
|
||||
localBufferStats.localBufferSize +
|
||||
@@ -98,8 +131,11 @@ export class ApiEngine {
|
||||
storageStats,
|
||||
securityStats,
|
||||
runtime: {
|
||||
serverPort: settings.serverPort || 4000,
|
||||
webPort: settings.webPort || 3000,
|
||||
serverPort: liveServerPort,
|
||||
webPort: liveWebPort,
|
||||
ingestionBridgeStatus,
|
||||
...(ingestionBridgeError ? { ingestionBridgeError } : {}),
|
||||
securityMode,
|
||||
uptimeSeconds: Math.floor(process.uptime()),
|
||||
memoryUsageMb: (process.memoryUsage().rss / (1024 * 1024)).toFixed(1),
|
||||
nodeVersion: process.version,
|
||||
@@ -107,23 +143,35 @@ export class ApiEngine {
|
||||
};
|
||||
}
|
||||
|
||||
private static filterFilesByClientId<T extends { clientId: string }>(
|
||||
files: T[],
|
||||
clientId?: string,
|
||||
): T[] {
|
||||
if (!clientId || clientId === 'all') return files;
|
||||
const normalized = SDKManager.normalizeClientId(clientId);
|
||||
if (!normalized) return files;
|
||||
const sanitized = CSVStorageManager.sanitizeClientId(normalized);
|
||||
return files.filter((f) => {
|
||||
const folderNorm = SDKManager.normalizeClientId(f.clientId);
|
||||
return (
|
||||
folderNorm === normalized ||
|
||||
f.clientId === normalized ||
|
||||
f.clientId === sanitized
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Filters and lists stored CSV files for export
|
||||
*/
|
||||
public static getExportFiles(clientId?: string) {
|
||||
const allFiles = CSVStorageManager.listAllFiles();
|
||||
if (clientId && clientId !== 'all') {
|
||||
return allFiles.filter((f) => f.clientId === clientId);
|
||||
}
|
||||
return allFiles;
|
||||
return this.filterFilesByClientId(allFiles, clientId);
|
||||
}
|
||||
|
||||
public static async getExportFilesAsync(clientId?: string) {
|
||||
const allFiles = await CSVStorageManager.listAllFilesAsync();
|
||||
if (clientId && clientId !== 'all') {
|
||||
return allFiles.filter((f) => f.clientId === clientId);
|
||||
}
|
||||
return allFiles;
|
||||
return this.filterFilesByClientId(allFiles, clientId);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -113,11 +113,8 @@ export function getSessionCookieOptions(request: Request, maxAge: number) {
|
||||
?.split(",")[0]
|
||||
?.trim()
|
||||
.toLowerCase();
|
||||
const originHeader = request.headers.get("origin")?.toLowerCase() ?? "";
|
||||
const isHttps =
|
||||
forwardedProto === "https" ||
|
||||
originHeader.startsWith("https://") ||
|
||||
request.url.startsWith("https://");
|
||||
forwardedProto === "https" || request.url.startsWith("https://");
|
||||
|
||||
if (isHttps) {
|
||||
return {
|
||||
|
||||
+62
-53
@@ -5,12 +5,13 @@ import {
|
||||
scrypt as scryptCallback,
|
||||
timingSafeEqual,
|
||||
} from "node:crypto";
|
||||
import { Pool, type PoolClient } from "pg";
|
||||
import type { Pool, PoolClient } from "pg";
|
||||
import {
|
||||
type AdminSessionPayload,
|
||||
getAdminSessionPayload,
|
||||
getAdminSessionSecret,
|
||||
} from "./adminSession";
|
||||
import { DatabaseEngine, ensureCoreSchemaOnPool } from "../Database/database";
|
||||
|
||||
const PASSWORD_HASH_BYTES = 64;
|
||||
const PASSWORD_SALT_BYTES = 16;
|
||||
@@ -50,7 +51,6 @@ interface InMemoryAdmin {
|
||||
}
|
||||
|
||||
type AdminDatabaseGlobals = typeof globalThis & {
|
||||
__dataStackAdminPool?: Pool | null;
|
||||
__dataStackAdminSchema?: Promise<void>;
|
||||
__dataStackInMemoryAdmin?: InMemoryAdmin;
|
||||
};
|
||||
@@ -60,67 +60,51 @@ const localLoginThrottle = new Map<string, LoginThrottleEntry>();
|
||||
let lastLocalThrottleCleanup = 0;
|
||||
let lastDatabaseThrottleCleanup = 0;
|
||||
|
||||
export class AdminSetupRequiredError extends Error {
|
||||
public readonly code = "SETUP_REQUIRED";
|
||||
|
||||
constructor(message = "Admin account has not completed initial setup.") {
|
||||
super(message);
|
||||
this.name = "AdminSetupRequiredError";
|
||||
}
|
||||
}
|
||||
|
||||
export class AdminDatabaseRequiredError extends Error {
|
||||
public readonly code = "DATABASE_NOT_CONFIGURED";
|
||||
|
||||
constructor(
|
||||
message = "PostgreSQL DATABASE_URL is required for initial admin setup in production.",
|
||||
) {
|
||||
super(message);
|
||||
this.name = "AdminDatabaseRequiredError";
|
||||
}
|
||||
}
|
||||
|
||||
function isProductionEnv() {
|
||||
return process.env.NODE_ENV === "production";
|
||||
}
|
||||
|
||||
export function isAdminDatabaseConfigured(): boolean {
|
||||
return Boolean(getPool());
|
||||
}
|
||||
|
||||
export function requiresPersistentAdminDatabase(): boolean {
|
||||
return (
|
||||
isProductionEnv() &&
|
||||
!getPool() &&
|
||||
!process.env.DATASTACK_ADMIN_PASSWORD
|
||||
);
|
||||
}
|
||||
|
||||
function getPool(): Pool | null {
|
||||
const connectionString = process.env.DATABASE_URL?.trim();
|
||||
if (!connectionString) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (globals.__dataStackAdminPool === undefined) {
|
||||
try {
|
||||
globals.__dataStackAdminPool = new Pool({
|
||||
connectionString,
|
||||
connectionTimeoutMillis: 3000,
|
||||
idleTimeoutMillis: 30000,
|
||||
max: 5,
|
||||
});
|
||||
} catch {
|
||||
globals.__dataStackAdminPool = null;
|
||||
}
|
||||
}
|
||||
|
||||
return globals.__dataStackAdminPool ?? null;
|
||||
return DatabaseEngine.getPool();
|
||||
}
|
||||
|
||||
async function ensureAdminTable() {
|
||||
const pool = getPool();
|
||||
if (!pool) return;
|
||||
|
||||
globals.__dataStackAdminSchema ??= Promise.all([
|
||||
pool
|
||||
.query(
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS admin_users (
|
||||
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
|
||||
email TEXT NOT NULL UNIQUE,
|
||||
password_salt BYTEA NOT NULL,
|
||||
password_hash BYTEA NOT NULL,
|
||||
token_version INTEGER NOT NULL DEFAULT 1,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`,
|
||||
)
|
||||
.then(() =>
|
||||
pool.query(
|
||||
`ALTER TABLE admin_users ADD COLUMN IF NOT EXISTS token_version INTEGER NOT NULL DEFAULT 1`,
|
||||
),
|
||||
),
|
||||
pool.query(
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS admin_login_attempts (
|
||||
key_hash TEXT PRIMARY KEY,
|
||||
window_started_at TIMESTAMPTZ NOT NULL,
|
||||
failed_attempts INTEGER NOT NULL,
|
||||
blocked_until TIMESTAMPTZ,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`,
|
||||
),
|
||||
]).then(() => undefined);
|
||||
globals.__dataStackAdminSchema ??= ensureCoreSchemaOnPool(pool);
|
||||
|
||||
try {
|
||||
await globals.__dataStackAdminSchema;
|
||||
@@ -342,7 +326,7 @@ async function getOrInitInMemoryAdmin(): Promise<InMemoryAdmin> {
|
||||
!isProductionEnv() && process.env.DATASTACK_ALLOW_DEMO_ADMIN !== "false";
|
||||
|
||||
if (!envPassword && !allowDemo) {
|
||||
throw new Error("Admin account has not completed initial setup.");
|
||||
throw new AdminSetupRequiredError();
|
||||
}
|
||||
|
||||
const defaultEmail = (
|
||||
@@ -443,6 +427,9 @@ export async function createInitialAdmin(email: string, password: string) {
|
||||
const pool = getPool();
|
||||
|
||||
if (!pool) {
|
||||
if (isProductionEnv()) {
|
||||
throw new AdminDatabaseRequiredError();
|
||||
}
|
||||
if (globals.__dataStackInMemoryAdmin) {
|
||||
return false;
|
||||
}
|
||||
@@ -671,6 +658,28 @@ export async function updateAdminAccount({
|
||||
return { ok: true as const, email: cleanEmail, tokenVersion: nextVersion };
|
||||
}
|
||||
|
||||
export async function revokeAdminSessions(): Promise<number> {
|
||||
const pool = getPool();
|
||||
if (pool) {
|
||||
await ensureAdminTable();
|
||||
const updated = await pool.query(
|
||||
`UPDATE admin_users
|
||||
SET token_version = COALESCE(token_version, 1) + 1
|
||||
WHERE id = 1
|
||||
RETURNING token_version`,
|
||||
);
|
||||
if (!updated.rowCount || updated.rowCount === 0) {
|
||||
return 0;
|
||||
}
|
||||
return Number(updated.rows[0].token_version) || 1;
|
||||
}
|
||||
|
||||
const existing = globals.__dataStackInMemoryAdmin ?? (await getOrInitInMemoryAdmin());
|
||||
existing.tokenVersion += 1;
|
||||
globals.__dataStackInMemoryAdmin = existing;
|
||||
return existing.tokenVersion;
|
||||
}
|
||||
|
||||
export function resetAdminStateForTests() {
|
||||
globals.__dataStackInMemoryAdmin = undefined;
|
||||
localLoginThrottle.clear();
|
||||
|
||||
+517
-83
@@ -19,6 +19,15 @@ export interface RedisBufferClient {
|
||||
ttlMs: number,
|
||||
nxMode: 'NX',
|
||||
): Promise<string | null>;
|
||||
get?(key: string): Promise<string | null>;
|
||||
incr?(key: string): Promise<number>;
|
||||
pexpire?(key: string, ttlMs: number): Promise<number>;
|
||||
hget?(key: string, field: string): Promise<string | null>;
|
||||
hset?(key: string, field: string, value: string): Promise<number>;
|
||||
hdel?(key: string, ...fields: string[]): Promise<number>;
|
||||
sismember?(key: string, member: string): Promise<number>;
|
||||
sadd?(key: string, ...members: string[]): Promise<number>;
|
||||
srem?(key: string, ...members: string[]): Promise<number>;
|
||||
eval(script: string, numKeys: number, ...args: string[]): Promise<unknown>;
|
||||
quit?: () => Promise<unknown>;
|
||||
disconnect?: () => void;
|
||||
@@ -27,10 +36,83 @@ export interface RedisBufferClient {
|
||||
export const REDIS_BUFFER_KEY = 'datastack:buffer:events';
|
||||
export const REDIS_RECENT_KEY = 'datastack:recent:events';
|
||||
export const REDIS_FLUSH_LOCK_KEY = 'datastack:flush:lock';
|
||||
const REDIS_RECENT_LIMIT = 500;
|
||||
const FLUSH_LOCK_TTL_MS = 15000;
|
||||
const REDIS_CLIENTS_DELETED_KEY = 'datastack:clients:deleted';
|
||||
export const DEFAULT_REDIS_RECENT_LIMIT = 500;
|
||||
export const MAX_REDIS_RECENT_LIMIT = 5000;
|
||||
const FLUSH_LOCK_TTL_MS = 30000;
|
||||
const FLUSH_LOCK_WAIT_MS = 400;
|
||||
const FLUSH_LOCK_RETRY_MS = 25;
|
||||
const UNLOCK_LUA_SCRIPT =
|
||||
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end';
|
||||
const RENEW_LOCK_LUA_SCRIPT =
|
||||
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("pexpire", KEYS[1], ARGV[2]) else return 0 end';
|
||||
export const PURGE_CLIENT_LIST_LUA_SCRIPT = `
|
||||
-- DATASTACK_PURGE_CLIENT_LIST
|
||||
local key = KEYS[1]
|
||||
local target = ARGV[1]
|
||||
local items = redis.call("LRANGE", key, 0, -1)
|
||||
local removed = 0
|
||||
local kept = {}
|
||||
for i = 1, #items do
|
||||
local raw = items[i]
|
||||
local ok, obj = pcall(cjson.decode, raw)
|
||||
local cid = ""
|
||||
if ok and type(obj) == "table" and obj.clientId ~= nil then
|
||||
cid = string.lower(tostring(obj.clientId))
|
||||
cid = string.gsub(cid, "^%s*(.-)%s*$", "%1")
|
||||
cid = string.gsub(cid, "[^a-z0-9_-]", "_")
|
||||
end
|
||||
if cid == target then
|
||||
removed = removed + 1
|
||||
else
|
||||
kept[#kept + 1] = raw
|
||||
end
|
||||
end
|
||||
if removed > 0 then
|
||||
redis.call("DEL", key)
|
||||
local chunk = 500
|
||||
for i = 1, #kept, chunk do
|
||||
local stop = math.min(i + chunk - 1, #kept)
|
||||
redis.call("RPUSH", key, unpack(kept, i, stop))
|
||||
end
|
||||
end
|
||||
return removed
|
||||
`.trim();
|
||||
|
||||
export function resolveRedisRecentLimit(batchFlushThreshold?: number): number {
|
||||
const envRaw = Number(process.env.DATASTACK_RECENT_LIMIT);
|
||||
if (Number.isFinite(envRaw) && envRaw >= 100) {
|
||||
return Math.min(MAX_REDIS_RECENT_LIMIT, Math.floor(envRaw));
|
||||
}
|
||||
const threshold = Number(batchFlushThreshold) || DEFAULT_REDIS_RECENT_LIMIT;
|
||||
return Math.min(
|
||||
MAX_REDIS_RECENT_LIMIT,
|
||||
Math.max(DEFAULT_REDIS_RECENT_LIMIT, Math.floor(threshold)),
|
||||
);
|
||||
}
|
||||
|
||||
export interface FlushOptions {
|
||||
waitMs?: number;
|
||||
lockTtlMs?: number;
|
||||
throwOnLockTimeout?: boolean;
|
||||
}
|
||||
|
||||
export interface FlushResult {
|
||||
count: number;
|
||||
files: string[];
|
||||
lockTimedOut?: boolean;
|
||||
}
|
||||
|
||||
function sleepMs(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function normalizeBufferClientId(clientId: string): string {
|
||||
return String(clientId || '')
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9_-]/g, '_');
|
||||
}
|
||||
|
||||
interface BufferGlobalState {
|
||||
buffer: DataEvent[];
|
||||
@@ -43,11 +125,14 @@ interface BufferGlobalState {
|
||||
totalIngested: number;
|
||||
totalFlushed: number;
|
||||
isFlushing: boolean;
|
||||
activeFlushPromise: Promise<FlushResult> | null;
|
||||
redisClient: RedisBufferClient | null | undefined;
|
||||
redisUrlConfigured: string;
|
||||
redisConnected: boolean;
|
||||
redisTestOverride: RedisBufferClient | null | undefined;
|
||||
pendingRedisWrites: Set<Promise<void>>;
|
||||
deletedClientIds: Set<string>;
|
||||
shutdownHooks: Set<() => Promise<unknown>>;
|
||||
}
|
||||
|
||||
type BufferGlobals = typeof globalThis & {
|
||||
@@ -71,13 +156,22 @@ function getState(): BufferGlobalState {
|
||||
totalIngested: 0,
|
||||
totalFlushed: 0,
|
||||
isFlushing: false,
|
||||
activeFlushPromise: null,
|
||||
redisClient: undefined,
|
||||
redisUrlConfigured: '',
|
||||
redisConnected: false,
|
||||
redisTestOverride: undefined,
|
||||
pendingRedisWrites: new Set(),
|
||||
deletedClientIds: new Set(),
|
||||
shutdownHooks: new Set(),
|
||||
};
|
||||
}
|
||||
if (!globals.__dataStackBufferState.deletedClientIds) {
|
||||
globals.__dataStackBufferState.deletedClientIds = new Set();
|
||||
}
|
||||
if (!globals.__dataStackBufferState.shutdownHooks) {
|
||||
globals.__dataStackBufferState.shutdownHooks = new Set();
|
||||
}
|
||||
return globals.__dataStackBufferState;
|
||||
}
|
||||
|
||||
@@ -150,7 +244,9 @@ function getRedisClient(): RedisBufferClient | null {
|
||||
}
|
||||
|
||||
export class DataBufferEngine {
|
||||
private static maxRecentEvents = REDIS_RECENT_LIMIT;
|
||||
public static getRecentLimit(): number {
|
||||
return resolveRedisRecentLimit(getState().batchFlushThreshold);
|
||||
}
|
||||
|
||||
public static get batchFlushThreshold(): number {
|
||||
return getState().batchFlushThreshold;
|
||||
@@ -174,6 +270,10 @@ export class DataBufferEngine {
|
||||
state.redisConnected = Boolean(client);
|
||||
}
|
||||
|
||||
public static getRedisClient(): RedisBufferClient | null {
|
||||
return getRedisClient();
|
||||
}
|
||||
|
||||
public static async shutdown(): Promise<void> {
|
||||
const state = getState();
|
||||
if (state.flushTimer) {
|
||||
@@ -188,6 +288,7 @@ export class DataBufferEngine {
|
||||
state.redisUrlConfigured = '';
|
||||
state.redisConnected = false;
|
||||
state.redisTestOverride = undefined;
|
||||
state.deletedClientIds.clear();
|
||||
if (client) {
|
||||
try {
|
||||
if (typeof client.quit === 'function') {
|
||||
@@ -203,6 +304,14 @@ export class DataBufferEngine {
|
||||
}
|
||||
}
|
||||
|
||||
public static registerShutdownHook(hook: () => Promise<unknown>): () => void {
|
||||
const state = getState();
|
||||
state.shutdownHooks.add(hook);
|
||||
return () => {
|
||||
state.shutdownHooks.delete(hook);
|
||||
};
|
||||
}
|
||||
|
||||
public static async flushAndShutdownOnSignal(): Promise<void> {
|
||||
if (globals.__dataStackBufferShuttingDown) {
|
||||
return globals.__dataStackBufferShuttingDown;
|
||||
@@ -210,8 +319,16 @@ export class DataBufferEngine {
|
||||
|
||||
const run = (async () => {
|
||||
try {
|
||||
const hooks = [...getState().shutdownHooks];
|
||||
await Promise.race([
|
||||
this.flush().catch(() => undefined),
|
||||
Promise.allSettled([
|
||||
this.flush().catch(() => undefined),
|
||||
...hooks.map((hook) =>
|
||||
Promise.resolve()
|
||||
.then(() => hook())
|
||||
.catch(() => undefined),
|
||||
),
|
||||
]),
|
||||
new Promise<void>((resolve) => setTimeout(resolve, 4000)),
|
||||
]);
|
||||
} finally {
|
||||
@@ -278,7 +395,8 @@ export class DataBufferEngine {
|
||||
const state = getState();
|
||||
state.totalIngested += 1;
|
||||
state.recentEvents.unshift(event);
|
||||
if (state.recentEvents.length > this.maxRecentEvents) {
|
||||
const recentLimit = this.getRecentLimit();
|
||||
while (state.recentEvents.length > recentLimit) {
|
||||
state.recentEvents.pop();
|
||||
}
|
||||
|
||||
@@ -298,15 +416,25 @@ export class DataBufferEngine {
|
||||
|
||||
if (redis) {
|
||||
const serialized = events.map((e) => JSON.stringify(e));
|
||||
let bufferLen: number | null = null;
|
||||
try {
|
||||
const bufferLen = await redis.rpush(REDIS_BUFFER_KEY, ...serialized);
|
||||
await redis.lpush(REDIS_RECENT_KEY, ...serialized);
|
||||
await redis.ltrim(REDIS_RECENT_KEY, 0, REDIS_RECENT_LIMIT - 1);
|
||||
bufferLen = await redis.rpush(REDIS_BUFFER_KEY, ...serialized);
|
||||
state.redisConnected = true;
|
||||
return Number(bufferLen) + state.buffer.length;
|
||||
} catch {
|
||||
state.redisConnected = false;
|
||||
}
|
||||
|
||||
if (bufferLen !== null) {
|
||||
// Best-effort recent list mirror: never duplicate into RAM buffer once rpush succeeded
|
||||
try {
|
||||
const recentLimit = this.getRecentLimit();
|
||||
await redis.lpush(REDIS_RECENT_KEY, ...serialized);
|
||||
await redis.ltrim(REDIS_RECENT_KEY, 0, recentLimit - 1);
|
||||
} catch {
|
||||
// Non-fatal: events are already committed in REDIS_BUFFER_KEY and state.recentEvents
|
||||
}
|
||||
return Number(bufferLen) + state.buffer.length;
|
||||
}
|
||||
}
|
||||
|
||||
state.buffer.push(...events);
|
||||
@@ -363,117 +491,410 @@ export class DataBufferEngine {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Flushes queued events (shared Redis list under distributed lock + local RAM fallback) to partitioned CSV files.
|
||||
* Redis LTRIM runs ONLY after CSVStorageManager.appendBatch succeeds so crashes never lose data.
|
||||
*/
|
||||
public static async flush(): Promise<{ count: number; files: string[] }> {
|
||||
private static async isClientDeletedForBuffer(
|
||||
clientId: string,
|
||||
redis: RedisBufferClient | null,
|
||||
cache?: Map<string, boolean>,
|
||||
): Promise<boolean> {
|
||||
const normalized = normalizeBufferClientId(clientId);
|
||||
if (!normalized) return false;
|
||||
const state = getState();
|
||||
if (state.deletedClientIds.has(normalized)) return true;
|
||||
if (cache && cache.has(normalized)) return Boolean(cache.get(normalized));
|
||||
if (redis && typeof redis.sismember === 'function') {
|
||||
try {
|
||||
const isDeleted =
|
||||
Number(await redis.sismember(REDIS_CLIENTS_DELETED_KEY, normalized)) === 1;
|
||||
if (cache) cache.set(normalized, isDeleted);
|
||||
if (isDeleted) state.deletedClientIds.add(normalized);
|
||||
return isDeleted;
|
||||
} catch {
|
||||
// Ignore transient Redis tombstone lookup error
|
||||
}
|
||||
}
|
||||
if (cache) cache.set(normalized, false);
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Marks a client deleted in the buffer layer and immediately strips queued local events.
|
||||
*/
|
||||
public static markClientDeleted(clientId: string): void {
|
||||
const normalized = normalizeBufferClientId(clientId);
|
||||
if (!normalized) return;
|
||||
const state = getState();
|
||||
state.deletedClientIds.add(normalized);
|
||||
state.buffer = state.buffer.filter(
|
||||
(evt) => normalizeBufferClientId(evt.clientId) !== normalized,
|
||||
);
|
||||
state.recentEvents = state.recentEvents.filter(
|
||||
(evt) => normalizeBufferClientId(evt.clientId) !== normalized,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears deleted tombstone when a client ID is explicitly re-registered.
|
||||
*/
|
||||
public static unmarkClientDeleted(clientId: string): void {
|
||||
const normalized = normalizeBufferClientId(clientId);
|
||||
if (!normalized) return;
|
||||
getState().deletedClientIds.delete(normalized);
|
||||
}
|
||||
|
||||
private static async filterRedisListExcludingClient(
|
||||
redis: RedisBufferClient,
|
||||
listKey: string,
|
||||
normalizedClientId: string,
|
||||
): Promise<number> {
|
||||
const removed = await redis.eval(
|
||||
PURGE_CLIENT_LIST_LUA_SCRIPT,
|
||||
1,
|
||||
listKey,
|
||||
normalizedClientId,
|
||||
);
|
||||
return Number(removed) || 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Purges all queued events for a deleted client from both local RAM and shared Redis queues (BUG-19).
|
||||
*/
|
||||
public static async purgeClientEventsAsync(
|
||||
clientId: string,
|
||||
): Promise<{ removedLocal: number; removedRedis: number }> {
|
||||
const normalized = normalizeBufferClientId(clientId);
|
||||
if (!normalized) return { removedLocal: 0, removedRedis: 0 };
|
||||
|
||||
const state = getState();
|
||||
state.deletedClientIds.add(normalized);
|
||||
|
||||
if (state.pendingRedisWrites.size > 0) {
|
||||
await Promise.allSettled([...state.pendingRedisWrites]);
|
||||
}
|
||||
if (state.isFlushing) {
|
||||
return { count: 0, files: [] };
|
||||
}
|
||||
|
||||
state.isFlushing = true;
|
||||
let totalWritten = 0;
|
||||
const filesSet = new Set<string>();
|
||||
|
||||
try {
|
||||
const redis = getRedisClient();
|
||||
if (redis) {
|
||||
const lockToken = crypto.randomUUID();
|
||||
let lockAcquired = false;
|
||||
const beforeLocal = state.buffer.length;
|
||||
state.buffer = state.buffer.filter(
|
||||
(evt) => normalizeBufferClientId(evt.clientId) !== normalized,
|
||||
);
|
||||
const removedLocal = beforeLocal - state.buffer.length;
|
||||
state.recentEvents = state.recentEvents.filter(
|
||||
(evt) => normalizeBufferClientId(evt.clientId) !== normalized,
|
||||
);
|
||||
|
||||
let removedRedis = 0;
|
||||
const redis = getRedisClient();
|
||||
if (redis) {
|
||||
const lockToken = crypto.randomUUID();
|
||||
let lockAcquired = false;
|
||||
const deadline = Date.now() + 1500;
|
||||
while (true) {
|
||||
try {
|
||||
const lockResult = await redis.set(
|
||||
const lockRes = await redis.set(
|
||||
REDIS_FLUSH_LOCK_KEY,
|
||||
lockToken,
|
||||
'PX',
|
||||
FLUSH_LOCK_TTL_MS,
|
||||
'NX',
|
||||
);
|
||||
lockAcquired = lockResult === 'OK';
|
||||
state.redisConnected = true;
|
||||
if (lockRes === 'OK') {
|
||||
lockAcquired = true;
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
state.redisConnected = false;
|
||||
lockAcquired = false;
|
||||
break;
|
||||
}
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining <= 0) break;
|
||||
await sleepMs(Math.min(FLUSH_LOCK_RETRY_MS, remaining));
|
||||
}
|
||||
|
||||
if (lockAcquired) {
|
||||
try {
|
||||
removedRedis += await this.filterRedisListExcludingClient(
|
||||
redis,
|
||||
REDIS_BUFFER_KEY,
|
||||
normalized,
|
||||
);
|
||||
await this.filterRedisListExcludingClient(
|
||||
redis,
|
||||
REDIS_RECENT_KEY,
|
||||
normalized,
|
||||
).catch(() => 0);
|
||||
} catch {
|
||||
// Flush-time deletedClientIds/tombstone filter remains active as secondary guard
|
||||
} finally {
|
||||
try {
|
||||
await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
|
||||
} catch {
|
||||
// Lock expires via PX TTL
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { removedLocal, removedRedis };
|
||||
}
|
||||
|
||||
/**
|
||||
* Flushes queued events (shared Redis list under distributed lock + local RAM fallback) to partitioned CSV files.
|
||||
* Redis LTRIM runs ONLY after CSVStorageManager.appendBatch succeeds and lock ownership is verified.
|
||||
*/
|
||||
public static async flush(options?: FlushOptions): Promise<FlushResult> {
|
||||
const state = getState();
|
||||
if (state.pendingRedisWrites.size > 0) {
|
||||
await Promise.allSettled([...state.pendingRedisWrites]);
|
||||
}
|
||||
if (state.activeFlushPromise) {
|
||||
return state.activeFlushPromise;
|
||||
}
|
||||
|
||||
const executeFlush = async (): Promise<FlushResult> => {
|
||||
state.isFlushing = true;
|
||||
let totalWritten = 0;
|
||||
let lockTimedOut = false;
|
||||
const filesSet = new Set<string>();
|
||||
const deletedCheckCache = new Map<string, boolean>();
|
||||
|
||||
try {
|
||||
if (state.pendingRedisWrites.size > 0) {
|
||||
await Promise.allSettled([...state.pendingRedisWrites]);
|
||||
}
|
||||
|
||||
if (lockAcquired) {
|
||||
try {
|
||||
const chunkSize = Math.max(500, state.batchFlushThreshold);
|
||||
while (true) {
|
||||
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, chunkSize - 1);
|
||||
if (!rawItems || rawItems.length === 0) {
|
||||
const redis = getRedisClient();
|
||||
if (redis) {
|
||||
const lockToken = crypto.randomUUID();
|
||||
const lockTtlMs = Math.max(50, Number(options?.lockTtlMs) || FLUSH_LOCK_TTL_MS);
|
||||
const waitMs = Math.max(
|
||||
0,
|
||||
options?.waitMs !== undefined ? Number(options.waitMs) : FLUSH_LOCK_WAIT_MS,
|
||||
);
|
||||
const deadline = Date.now() + waitMs;
|
||||
let lockAcquired = false;
|
||||
let redisError = false;
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
const lockResult = await redis.set(
|
||||
REDIS_FLUSH_LOCK_KEY,
|
||||
lockToken,
|
||||
'PX',
|
||||
lockTtlMs,
|
||||
'NX',
|
||||
);
|
||||
state.redisConnected = true;
|
||||
if (lockResult === 'OK') {
|
||||
lockAcquired = true;
|
||||
break;
|
||||
}
|
||||
} catch {
|
||||
state.redisConnected = false;
|
||||
redisError = true;
|
||||
break;
|
||||
}
|
||||
|
||||
const parsedBatch: DataEvent[] = [];
|
||||
for (const raw of rawItems) {
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining <= 0) {
|
||||
break;
|
||||
}
|
||||
await sleepMs(Math.min(FLUSH_LOCK_RETRY_MS, remaining));
|
||||
}
|
||||
|
||||
if (!lockAcquired && !redisError) {
|
||||
lockTimedOut = true;
|
||||
if (options?.throwOnLockTimeout === true) {
|
||||
throw new Error(
|
||||
'FLUSH_LOCK_TIMEOUT: Timed out waiting for distributed Redis flush lock',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (lockAcquired) {
|
||||
const renewIntervalMs = Math.max(20, Math.floor(lockTtlMs / 3));
|
||||
let lastRenewAt = Date.now();
|
||||
let lockLost = false;
|
||||
let renewInFlight: Promise<boolean> | null = null;
|
||||
|
||||
const renewLockOnce = (): Promise<boolean> => {
|
||||
if (renewInFlight) return renewInFlight;
|
||||
renewInFlight = (async () => {
|
||||
try {
|
||||
const evt = JSON.parse(raw) as DataEvent;
|
||||
if (evt && evt.eventId) {
|
||||
parsedBatch.push(evt);
|
||||
const renewed = await redis.eval(
|
||||
RENEW_LOCK_LUA_SCRIPT,
|
||||
1,
|
||||
REDIS_FLUSH_LOCK_KEY,
|
||||
lockToken,
|
||||
String(lockTtlMs),
|
||||
);
|
||||
if (Number(renewed) === 1) {
|
||||
lastRenewAt = Date.now();
|
||||
return true;
|
||||
}
|
||||
lockLost = true;
|
||||
return false;
|
||||
} catch {
|
||||
// Ignore malformed legacy entry while still trimming it
|
||||
return false;
|
||||
} finally {
|
||||
renewInFlight = null;
|
||||
}
|
||||
})();
|
||||
return renewInFlight;
|
||||
};
|
||||
|
||||
const renewTimer = setInterval(() => {
|
||||
void renewLockOnce();
|
||||
}, renewIntervalMs);
|
||||
if (typeof renewTimer.unref === 'function') {
|
||||
renewTimer.unref();
|
||||
}
|
||||
|
||||
try {
|
||||
const chunkSize = Math.max(500, state.batchFlushThreshold);
|
||||
while (true) {
|
||||
if (lockLost) break;
|
||||
|
||||
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, chunkSize - 1);
|
||||
if (!rawItems || rawItems.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
const parsedBatch: DataEvent[] = [];
|
||||
let malformedCount = 0;
|
||||
for (const raw of rawItems) {
|
||||
try {
|
||||
const evt = JSON.parse(raw) as DataEvent;
|
||||
if (evt && evt.eventId) {
|
||||
if (
|
||||
await this.isClientDeletedForBuffer(
|
||||
evt.clientId,
|
||||
redis,
|
||||
deletedCheckCache,
|
||||
)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
parsedBatch.push(evt);
|
||||
} else {
|
||||
malformedCount += 1;
|
||||
}
|
||||
} catch {
|
||||
malformedCount += 1;
|
||||
}
|
||||
}
|
||||
if (malformedCount > 0) {
|
||||
console.warn(
|
||||
`[DataBufferEngine] Skipped ${malformedCount} malformed buffered Redis event(s) during flush`,
|
||||
);
|
||||
}
|
||||
|
||||
if (Date.now() - lastRenewAt >= renewIntervalMs) {
|
||||
await renewLockOnce();
|
||||
}
|
||||
if (lockLost) break;
|
||||
|
||||
if (parsedBatch.length > 0) {
|
||||
const res = await CSVStorageManager.appendBatch(parsedBatch);
|
||||
totalWritten += res.written;
|
||||
for (const f of res.filesUpdated) filesSet.add(f);
|
||||
}
|
||||
|
||||
if (Date.now() - lastRenewAt >= renewIntervalMs) {
|
||||
await renewLockOnce();
|
||||
}
|
||||
if (lockLost) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Trim strictly AFTER CSVStorageManager.appendBatch resolves and lock ownership holds
|
||||
await redis.ltrim(REDIS_BUFFER_KEY, rawItems.length, -1);
|
||||
|
||||
if (rawItems.length < chunkSize) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (parsedBatch.length > 0) {
|
||||
const res = await CSVStorageManager.appendBatch(parsedBatch);
|
||||
totalWritten += res.written;
|
||||
for (const f of res.filesUpdated) filesSet.add(f);
|
||||
} finally {
|
||||
clearInterval(renewTimer);
|
||||
if (renewInFlight) {
|
||||
await renewInFlight.catch(() => false);
|
||||
}
|
||||
|
||||
// Trim strictly AFTER CSVStorageManager.appendBatch resolves safely
|
||||
await redis.ltrim(REDIS_BUFFER_KEY, rawItems.length, -1);
|
||||
|
||||
if (rawItems.length < chunkSize) {
|
||||
break;
|
||||
try {
|
||||
await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
|
||||
} catch {
|
||||
// Lock auto-expires after lockTtlMs
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
}
|
||||
}
|
||||
|
||||
if (state.buffer.length > 0) {
|
||||
const ramBatch = [...state.buffer];
|
||||
state.buffer = [];
|
||||
const validRamBatch: DataEvent[] = [];
|
||||
for (const evt of ramBatch) {
|
||||
if (
|
||||
await this.isClientDeletedForBuffer(
|
||||
evt.clientId,
|
||||
redis,
|
||||
deletedCheckCache,
|
||||
)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
validRamBatch.push(evt);
|
||||
}
|
||||
|
||||
if (validRamBatch.length > 0) {
|
||||
try {
|
||||
await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
|
||||
} catch {
|
||||
// Lock auto-expires after FLUSH_LOCK_TTL_MS
|
||||
const res = await CSVStorageManager.appendBatch(validRamBatch);
|
||||
totalWritten += res.written;
|
||||
for (const f of res.filesUpdated) filesSet.add(f);
|
||||
} catch (err) {
|
||||
state.buffer.unshift(...validRamBatch);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (state.buffer.length > 0) {
|
||||
const ramBatch = [...state.buffer];
|
||||
state.buffer = [];
|
||||
try {
|
||||
const res = await CSVStorageManager.appendBatch(ramBatch);
|
||||
totalWritten += res.written;
|
||||
for (const f of res.filesUpdated) filesSet.add(f);
|
||||
} catch (err) {
|
||||
state.buffer.unshift(...ramBatch);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
if (totalWritten > 0) {
|
||||
state.totalFlushed += totalWritten;
|
||||
const filesUpdated = Array.from(filesSet);
|
||||
for (const listener of state.flushListeners) {
|
||||
try {
|
||||
listener(totalWritten, filesUpdated);
|
||||
} catch {
|
||||
// ignore
|
||||
if (totalWritten > 0) {
|
||||
state.totalFlushed += totalWritten;
|
||||
const filesUpdated = Array.from(filesSet);
|
||||
for (const listener of state.flushListeners) {
|
||||
try {
|
||||
listener(totalWritten, filesUpdated);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { count: totalWritten, files: Array.from(filesSet) };
|
||||
} finally {
|
||||
state.isFlushing = false;
|
||||
return {
|
||||
count: totalWritten,
|
||||
files: Array.from(filesSet),
|
||||
...(lockTimedOut ? { lockTimedOut: true } : {}),
|
||||
};
|
||||
} finally {
|
||||
state.isFlushing = false;
|
||||
state.activeFlushPromise = null;
|
||||
}
|
||||
};
|
||||
|
||||
let flushDone = false;
|
||||
let promise: Promise<FlushResult> | null = null;
|
||||
promise = (async () => {
|
||||
try {
|
||||
return await executeFlush();
|
||||
} finally {
|
||||
flushDone = true;
|
||||
state.isFlushing = false;
|
||||
if (state.activeFlushPromise === promise) {
|
||||
state.activeFlushPromise = null;
|
||||
}
|
||||
}
|
||||
})();
|
||||
if (!flushDone) {
|
||||
state.activeFlushPromise = promise;
|
||||
}
|
||||
return promise;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -495,6 +916,15 @@ export class DataBufferEngine {
|
||||
};
|
||||
}
|
||||
|
||||
public static getLocalBufferSnapshot(limit?: number): DataEvent[] {
|
||||
const state = getState();
|
||||
if (typeof limit === 'number' && Number.isFinite(limit) && limit >= 0) {
|
||||
const cap = Math.floor(limit);
|
||||
return cap === 0 ? [] : state.buffer.slice(-cap);
|
||||
}
|
||||
return [...state.buffer];
|
||||
}
|
||||
|
||||
public static getRecentEvents(limit = 50, filterClientId?: string): DataEvent[] {
|
||||
const state = getState();
|
||||
if (filterClientId && filterClientId !== 'all') {
|
||||
@@ -514,7 +944,8 @@ export class DataBufferEngine {
|
||||
|
||||
if (redis) {
|
||||
try {
|
||||
const readCap = Math.min(REDIS_RECENT_LIMIT, Math.max(limit * 4, limit));
|
||||
const recentLimit = this.getRecentLimit();
|
||||
const readCap = Math.min(recentLimit, Math.max(limit * 4, limit));
|
||||
const rawItems = await redis.lrange(REDIS_RECENT_KEY, 0, readCap - 1);
|
||||
state.redisConnected = true;
|
||||
|
||||
@@ -523,6 +954,9 @@ export class DataBufferEngine {
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as DataEvent;
|
||||
if (!parsed || !parsed.eventId) continue;
|
||||
if (state.deletedClientIds.has(normalizeBufferClientId(parsed.clientId))) {
|
||||
continue;
|
||||
}
|
||||
if (filterClientId && filterClientId !== 'all' && parsed.clientId !== filterClientId) {
|
||||
continue;
|
||||
}
|
||||
|
||||
+520
-117
@@ -1,9 +1,21 @@
|
||||
import { existsSync, rmSync } from "node:fs";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { existsSync, lstatSync, readdirSync, rmSync } from "node:fs";
|
||||
import { resolve, sep } from "node:path";
|
||||
import { SettingsManager } from "../adminSettings/settingsManager";
|
||||
import { AuthService } from "../auth/jwt";
|
||||
import { DataBufferEngine } from "../data/buffer";
|
||||
import { DatabaseEngine } from "../Database/database";
|
||||
import { getInternalIngestionHeaders } from "../Security/security";
|
||||
import { CSVStorageManager } from "../Storage/csvStorage";
|
||||
|
||||
export interface BridgeSyncResult {
|
||||
synced: boolean;
|
||||
status?: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
const REDIS_CLIENTS_REGISTRY_KEY = "datastack:clients:registry";
|
||||
const REDIS_CLIENTS_DELETED_KEY = "datastack:clients:deleted";
|
||||
|
||||
export interface ClientSDK {
|
||||
clientId: string;
|
||||
@@ -17,13 +29,24 @@ export interface ClientSDK {
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
type PendingActivityEntry = {
|
||||
count: number;
|
||||
lastActive: string;
|
||||
};
|
||||
|
||||
type SDKDatabaseGlobals = typeof globalThis & {
|
||||
__dataStackClientsMap?: Map<string, ClientSDK>;
|
||||
__dataStackDeletedClients?: Set<string>;
|
||||
__dataStackClientsHydrated?: boolean;
|
||||
__dataStackSyncedSocketEvents?: Map<string, number>;
|
||||
__dataStackPendingClientActivity?: Map<string, PendingActivityEntry>;
|
||||
__dataStackActivityFlushTimer?: ReturnType<typeof setInterval> | null;
|
||||
__dataStackActivityFlushPromise?: Promise<number> | null;
|
||||
__dataStackActivityShutdownRegistered?: boolean;
|
||||
};
|
||||
|
||||
const CLIENT_ACTIVITY_FLUSH_INTERVAL_MS = 1000;
|
||||
const CLIENT_ACTIVITY_IMMEDIATE_BATCH_SIZE = 500;
|
||||
const globals = globalThis as SDKDatabaseGlobals;
|
||||
|
||||
async function ensureDbSettingsHydrated(): Promise<void> {
|
||||
@@ -53,6 +76,58 @@ function getSyncedSocketEventsMap(): Map<string, number> {
|
||||
return globals.__dataStackSyncedSocketEvents;
|
||||
}
|
||||
|
||||
function getPendingClientActivityMap(): Map<string, PendingActivityEntry> {
|
||||
if (!globals.__dataStackPendingClientActivity) {
|
||||
globals.__dataStackPendingClientActivity = new Map<
|
||||
string,
|
||||
PendingActivityEntry
|
||||
>();
|
||||
}
|
||||
return globals.__dataStackPendingClientActivity;
|
||||
}
|
||||
|
||||
function ensureActivityFlushTimer(): void {
|
||||
if (globals.__dataStackActivityFlushTimer) return;
|
||||
const timer = setInterval(() => {
|
||||
void SDKManager.flushPendingClientActivityAsync().catch(() => undefined);
|
||||
}, CLIENT_ACTIVITY_FLUSH_INTERVAL_MS);
|
||||
if (typeof timer.unref === "function") {
|
||||
timer.unref();
|
||||
}
|
||||
globals.__dataStackActivityFlushTimer = timer;
|
||||
}
|
||||
|
||||
interface RemoteInMemoryClient {
|
||||
clientId?: string;
|
||||
name?: string;
|
||||
status?: string;
|
||||
token?: string;
|
||||
rateLimit?: number;
|
||||
}
|
||||
|
||||
function hydrateLocalMapFromRemote(remoteClients: RemoteInMemoryClient[]): number {
|
||||
const map = getClientsMap();
|
||||
const deletedSet = getDeletedClientsSet();
|
||||
let hydrated = 0;
|
||||
for (const item of remoteClients) {
|
||||
if (!item || !item.clientId) continue;
|
||||
const cid = SDKManager.normalizeClientId(item.clientId);
|
||||
if (!cid || deletedSet.has(cid) || map.has(cid)) continue;
|
||||
map.set(cid, {
|
||||
clientId: cid,
|
||||
name: String(item.name || cid),
|
||||
secret: "",
|
||||
token: String(item.token || ""),
|
||||
rateLimit: Math.max(1, Number(item.rateLimit) || 100),
|
||||
status: item.status === "revoked" ? "revoked" : "active",
|
||||
totalEvents: 0,
|
||||
createdAt: new Date().toISOString(),
|
||||
});
|
||||
hydrated += 1;
|
||||
}
|
||||
return hydrated;
|
||||
}
|
||||
|
||||
function serializeSyncClient(client: ClientSDK) {
|
||||
return {
|
||||
clientId: client.clientId,
|
||||
@@ -63,18 +138,53 @@ function serializeSyncClient(client: ClientSDK) {
|
||||
};
|
||||
}
|
||||
|
||||
async function notifyIngestionClientSyncAsync(
|
||||
async function mirrorClientToRedis(
|
||||
clientId: string,
|
||||
action: "deleted" | "revoked" | "updated" | "registered",
|
||||
client?: ClientSDK,
|
||||
): Promise<void> {
|
||||
const redis = DataBufferEngine.getRedisClient();
|
||||
if (!redis) return;
|
||||
try {
|
||||
if (action === "deleted") {
|
||||
if (typeof redis.sadd === "function") {
|
||||
await redis.sadd(REDIS_CLIENTS_DELETED_KEY, clientId);
|
||||
}
|
||||
if (typeof redis.hdel === "function") {
|
||||
await redis.hdel(REDIS_CLIENTS_REGISTRY_KEY, clientId);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (action === "registered" && typeof redis.srem === "function") {
|
||||
await redis.srem(REDIS_CLIENTS_DELETED_KEY, clientId);
|
||||
}
|
||||
if (client && typeof redis.hset === "function") {
|
||||
await redis.hset(
|
||||
REDIS_CLIENTS_REGISTRY_KEY,
|
||||
clientId,
|
||||
JSON.stringify(serializeSyncClient(client)),
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
// Non-fatal; DB and direct HTTP bridge remain primary sources of truth
|
||||
}
|
||||
}
|
||||
|
||||
async function notifyIngestionClientSyncAsync(
|
||||
clientId: string,
|
||||
action: "deleted" | "revoked" | "updated" | "registered",
|
||||
client?: ClientSDK,
|
||||
): Promise<BridgeSyncResult> {
|
||||
await mirrorClientToRedis(clientId, action, client);
|
||||
const headers = getInternalIngestionHeaders();
|
||||
if (!headers["x-datastack-internal-secret"]) return;
|
||||
if (!headers["x-datastack-internal-secret"]) {
|
||||
return { synced: false, error: "Internal bridge secret is not configured" };
|
||||
}
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||
).replace(/\/$/, "");
|
||||
try {
|
||||
await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
|
||||
const res = await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
@@ -87,8 +197,21 @@ async function notifyIngestionClientSyncAsync(
|
||||
}),
|
||||
signal: AbortSignal.timeout(1000),
|
||||
});
|
||||
} catch {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
if (!res.ok) {
|
||||
const errorMsg = `Port 4000 client-sync (${action}:${clientId}) failed with HTTP ${res.status}`;
|
||||
console.warn(`[DataStack Bridge] ${errorMsg}`);
|
||||
return { synced: false, status: res.status, error: errorMsg };
|
||||
}
|
||||
return { synced: true, status: res.status };
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : "Bridge request failed";
|
||||
if (
|
||||
process.env.NODE_ENV === "production" ||
|
||||
Boolean(process.env.INGESTION_INTERNAL_URL)
|
||||
) {
|
||||
console.warn(`[DataStack Bridge] Port 4000 client-sync unreachable: ${message}`);
|
||||
}
|
||||
return { synced: false, error: message };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -127,6 +250,10 @@ export class SDKManager {
|
||||
const ready = await DatabaseEngine.ensureSchema();
|
||||
if (!ready) return null;
|
||||
|
||||
if (getPendingClientActivityMap().size > 0) {
|
||||
await this.flushPendingClientActivityAsync();
|
||||
}
|
||||
|
||||
const [result, deletedResult] = await Promise.all([
|
||||
pool.query(
|
||||
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
|
||||
@@ -166,16 +293,31 @@ export class SDKManager {
|
||||
return Array.from(getClientsMap().values());
|
||||
}
|
||||
|
||||
public static async syncAllClientsToIngestionAsync(): Promise<void> {
|
||||
public static async syncAllClientsToIngestionAsync(options?: {
|
||||
forceReplace?: boolean;
|
||||
}): Promise<BridgeSyncResult> {
|
||||
await ensureDbSettingsHydrated();
|
||||
const clients = Array.from(getClientsMap().values()).map(serializeSyncClient);
|
||||
const forceReplace = options?.forceReplace ?? Boolean(DatabaseEngine.getPool());
|
||||
const redis = DataBufferEngine.getRedisClient();
|
||||
if (redis && typeof redis.hset === "function") {
|
||||
try {
|
||||
for (const c of clients) {
|
||||
await redis.hset(REDIS_CLIENTS_REGISTRY_KEY, c.clientId, JSON.stringify(c));
|
||||
}
|
||||
} catch {
|
||||
// ignore Redis mirror failure
|
||||
}
|
||||
}
|
||||
const headers = getInternalIngestionHeaders();
|
||||
if (!headers["x-datastack-internal-secret"]) return;
|
||||
if (!headers["x-datastack-internal-secret"]) {
|
||||
return { synced: false, error: "Internal bridge secret is not configured" };
|
||||
}
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||
).replace(/\/$/, "");
|
||||
const clients = Array.from(getClientsMap().values()).map(serializeSyncClient);
|
||||
try {
|
||||
await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
|
||||
const res = await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
@@ -184,11 +326,43 @@ export class SDKManager {
|
||||
body: JSON.stringify({
|
||||
action: "bulk_sync",
|
||||
clients,
|
||||
forceReplace,
|
||||
}),
|
||||
signal: AbortSignal.timeout(1500),
|
||||
});
|
||||
} catch {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
if (!res.ok) {
|
||||
const errorMsg = `Port 4000 bulk_sync failed with HTTP ${res.status}`;
|
||||
console.warn(`[DataStack Bridge] ${errorMsg}`);
|
||||
return { synced: false, status: res.status, error: errorMsg };
|
||||
}
|
||||
// BUG-02: If Port 4000 preserved an existing no-DB registry because Port 3000 restarted empty,
|
||||
// hydrate Port 3000's local map non-destructively from the returned clients.
|
||||
try {
|
||||
const payload = (await res.json()) as {
|
||||
preserved?: boolean;
|
||||
clients?: RemoteInMemoryClient[];
|
||||
};
|
||||
if (
|
||||
payload?.preserved &&
|
||||
Array.isArray(payload.clients) &&
|
||||
payload.clients.length > 0 &&
|
||||
!DatabaseEngine.getPool()
|
||||
) {
|
||||
hydrateLocalMapFromRemote(payload.clients);
|
||||
}
|
||||
} catch {
|
||||
// Response body optional
|
||||
}
|
||||
return { synced: true, status: res.status };
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : "Bridge bulk_sync unreachable";
|
||||
if (
|
||||
process.env.NODE_ENV === "production" ||
|
||||
Boolean(process.env.INGESTION_INTERNAL_URL)
|
||||
) {
|
||||
console.warn(`[DataStack Bridge] ${message}`);
|
||||
}
|
||||
return { synced: false, error: message };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -214,12 +388,20 @@ export class SDKManager {
|
||||
if (res.ok) {
|
||||
const data = (await res.json()) as {
|
||||
hasSyncedInMemoryClients?: boolean;
|
||||
inMemoryClients?: RemoteInMemoryClient[];
|
||||
clientActivity?: Record<
|
||||
string,
|
||||
{ totalEvents?: number; lastActive?: string }
|
||||
>;
|
||||
};
|
||||
if (data.hasSyncedInMemoryClients === false && map.size > 0) {
|
||||
// BUG-02: Non-destructive hydration when Port 3000 restarted empty while Port 4000 still holds clients
|
||||
if (
|
||||
map.size === 0 &&
|
||||
Array.isArray(data.inMemoryClients) &&
|
||||
data.inMemoryClients.length > 0
|
||||
) {
|
||||
hydrateLocalMapFromRemote(data.inMemoryClients);
|
||||
} else if (data.hasSyncedInMemoryClients === false && map.size > 0) {
|
||||
await this.syncAllClientsToIngestionAsync();
|
||||
}
|
||||
if (data.clientActivity && typeof data.clientActivity === "object") {
|
||||
@@ -255,36 +437,63 @@ export class SDKManager {
|
||||
return Array.from(map.values());
|
||||
}
|
||||
|
||||
public static normalizeClientId(clientId: string): string {
|
||||
return String(clientId || "")
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9_-]/g, "_");
|
||||
}
|
||||
|
||||
public static getClient(clientId: string): ClientSDK | undefined {
|
||||
return getClientsMap().get(clientId);
|
||||
const normalizedId = this.normalizeClientId(clientId);
|
||||
if (!normalizedId) return undefined;
|
||||
return getClientsMap().get(normalizedId);
|
||||
}
|
||||
|
||||
public static async getClientAsync(
|
||||
clientId: string,
|
||||
): Promise<ClientSDK | undefined> {
|
||||
await ensureDbSettingsHydrated();
|
||||
const normalizedId = this.normalizeClientId(clientId);
|
||||
if (!normalizedId) return undefined;
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (pool) {
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
if (getPendingClientActivityMap().has(normalizedId)) {
|
||||
await this.flushPendingClientActivityAsync();
|
||||
}
|
||||
const result = await pool.query(
|
||||
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
|
||||
FROM client_sdks
|
||||
WHERE client_id = $1`,
|
||||
[clientId],
|
||||
[normalizedId],
|
||||
);
|
||||
if (result.rowCount && result.rowCount > 0) {
|
||||
const client = mapRowToClient(result.rows[0]);
|
||||
getClientsMap().set(client.clientId, client);
|
||||
return client;
|
||||
}
|
||||
getClientsMap().delete(clientId);
|
||||
getClientsMap().delete(normalizedId);
|
||||
return undefined;
|
||||
} catch {
|
||||
// Fall back to in-memory map
|
||||
}
|
||||
}
|
||||
return getClientsMap().get(clientId);
|
||||
const existing = getClientsMap().get(normalizedId);
|
||||
if (existing) {
|
||||
return existing;
|
||||
}
|
||||
// BUG-02: If Port 3000 restarted without DB and local map is empty, attempt non-destructive hydration from Port 4000
|
||||
if (getClientsMap().size === 0 && !getDeletedClientsSet().has(normalizedId)) {
|
||||
await this.listClientsAsync();
|
||||
return getClientsMap().get(normalizedId);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private static generateClientSecret(safeClientId: string): string {
|
||||
return `sec_${safeClientId}_${randomBytes(16).toString("hex")}`;
|
||||
}
|
||||
|
||||
public static registerClient(data: {
|
||||
@@ -292,11 +501,8 @@ export class SDKManager {
|
||||
name: string;
|
||||
rateLimit?: number;
|
||||
}): ClientSDK {
|
||||
const safeClientId = data.clientId
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9_-]/g, "_");
|
||||
const secret = `sec_${safeClientId}_${Math.random().toString(36).substring(2, 10)}`;
|
||||
const safeClientId = this.normalizeClientId(data.clientId);
|
||||
const secret = this.generateClientSecret(safeClientId);
|
||||
const rateLimit = Math.max(1, Number(data.rateLimit) || 100);
|
||||
|
||||
const token = AuthService.generateClientToken({
|
||||
@@ -317,8 +523,12 @@ export class SDKManager {
|
||||
};
|
||||
|
||||
getDeletedClientsSet().delete(safeClientId);
|
||||
DataBufferEngine.unmarkClientDeleted(safeClientId);
|
||||
getSyncedSocketEventsMap().delete(safeClientId);
|
||||
getClientsMap().set(safeClientId, newClient);
|
||||
void this.persistNewClient(newClient);
|
||||
void this.persistNewClient(newClient).catch((err) => {
|
||||
console.error("[DataStack SDK] Background client create DB persist failed:", err);
|
||||
});
|
||||
notifyIngestionClientSync(safeClientId, "registered", newClient);
|
||||
return newClient;
|
||||
}
|
||||
@@ -327,8 +537,12 @@ export class SDKManager {
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (!pool) return;
|
||||
|
||||
const ready = await DatabaseEngine.ensureSchema();
|
||||
if (!ready) {
|
||||
throw new Error("DATABASE_WRITE_FAILED: PostgreSQL schema unavailable for client registration.");
|
||||
}
|
||||
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
await pool.query(
|
||||
`DELETE FROM deleted_clients WHERE client_id = $1`,
|
||||
[client.clientId],
|
||||
@@ -354,8 +568,9 @@ export class SDKManager {
|
||||
client.createdAt,
|
||||
],
|
||||
);
|
||||
} catch {
|
||||
// In-memory map already updated
|
||||
} catch (err: unknown) {
|
||||
const detail = err instanceof Error ? err.message : "Unknown DB error";
|
||||
throw new Error(`DATABASE_WRITE_FAILED: Failed to register client (${detail})`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -365,11 +580,8 @@ export class SDKManager {
|
||||
rateLimit?: number;
|
||||
}): Promise<ClientSDK> {
|
||||
await ensureDbSettingsHydrated();
|
||||
const safeClientId = data.clientId
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9_-]/g, "_");
|
||||
const secret = `sec_${safeClientId}_${Math.random().toString(36).substring(2, 10)}`;
|
||||
const safeClientId = this.normalizeClientId(data.clientId);
|
||||
const secret = this.generateClientSecret(safeClientId);
|
||||
const rateLimit = Math.max(1, Number(data.rateLimit) || 100);
|
||||
|
||||
const token = AuthService.generateClientToken({
|
||||
@@ -389,20 +601,23 @@ export class SDKManager {
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
getDeletedClientsSet().delete(safeClientId);
|
||||
getClientsMap().set(safeClientId, newClient);
|
||||
await this.persistNewClient(newClient);
|
||||
getDeletedClientsSet().delete(safeClientId);
|
||||
DataBufferEngine.unmarkClientDeleted(safeClientId);
|
||||
getSyncedSocketEventsMap().delete(safeClientId);
|
||||
getClientsMap().set(safeClientId, newClient);
|
||||
await notifyIngestionClientSyncAsync(safeClientId, "registered", newClient);
|
||||
return newClient;
|
||||
}
|
||||
|
||||
public static regenerateToken(clientId: string): ClientSDK | null {
|
||||
const client = getClientsMap().get(clientId);
|
||||
const normalizedId = this.normalizeClientId(clientId);
|
||||
const client = getClientsMap().get(normalizedId);
|
||||
if (!client) return null;
|
||||
|
||||
const updated: ClientSDK = {
|
||||
...client,
|
||||
secret: `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`,
|
||||
secret: this.generateClientSecret(normalizedId),
|
||||
token: AuthService.generateClientToken({
|
||||
clientId: client.clientId,
|
||||
name: client.name,
|
||||
@@ -410,9 +625,11 @@ export class SDKManager {
|
||||
}),
|
||||
};
|
||||
|
||||
getClientsMap().set(clientId, updated);
|
||||
void this.persistClientCredentials(updated);
|
||||
notifyIngestionClientSync(clientId, "updated", updated);
|
||||
getClientsMap().set(normalizedId, updated);
|
||||
void this.persistClientCredentials(updated).catch((err) => {
|
||||
console.error("[DataStack SDK] Background token rotate DB persist failed:", err);
|
||||
});
|
||||
notifyIngestionClientSync(normalizedId, "updated", updated);
|
||||
return updated;
|
||||
}
|
||||
|
||||
@@ -422,16 +639,21 @@ export class SDKManager {
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (!pool) return;
|
||||
|
||||
const ready = await DatabaseEngine.ensureSchema();
|
||||
if (!ready) {
|
||||
throw new Error("DATABASE_WRITE_FAILED: PostgreSQL schema unavailable for credential update.");
|
||||
}
|
||||
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
await pool.query(
|
||||
`UPDATE client_sdks
|
||||
SET secret = $2, token = $3, updated_at = NOW()
|
||||
WHERE client_id = $1`,
|
||||
[client.clientId, client.secret, client.token],
|
||||
);
|
||||
} catch {
|
||||
// Ignore DB failure
|
||||
} catch (err: unknown) {
|
||||
const detail = err instanceof Error ? err.message : "Unknown DB error";
|
||||
throw new Error(`DATABASE_WRITE_FAILED: Failed to update client credentials (${detail})`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -439,12 +661,13 @@ export class SDKManager {
|
||||
clientId: string,
|
||||
): Promise<ClientSDK | null> {
|
||||
await ensureDbSettingsHydrated();
|
||||
const existing = await this.getClientAsync(clientId);
|
||||
const normalizedId = this.normalizeClientId(clientId);
|
||||
const existing = await this.getClientAsync(normalizedId);
|
||||
if (!existing) return null;
|
||||
|
||||
const updated: ClientSDK = {
|
||||
...existing,
|
||||
secret: `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`,
|
||||
secret: this.generateClientSecret(normalizedId),
|
||||
token: AuthService.generateClientToken({
|
||||
clientId: existing.clientId,
|
||||
name: existing.name,
|
||||
@@ -452,9 +675,9 @@ export class SDKManager {
|
||||
}),
|
||||
};
|
||||
|
||||
getClientsMap().set(clientId, updated);
|
||||
await this.persistClientCredentials(updated);
|
||||
await notifyIngestionClientSyncAsync(clientId, "updated", updated);
|
||||
getClientsMap().set(normalizedId, updated);
|
||||
await notifyIngestionClientSyncAsync(normalizedId, "updated", updated);
|
||||
return updated;
|
||||
}
|
||||
|
||||
@@ -462,7 +685,8 @@ export class SDKManager {
|
||||
clientId: string,
|
||||
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
|
||||
): ClientSDK | null {
|
||||
const client = getClientsMap().get(clientId);
|
||||
const normalizedId = this.normalizeClientId(clientId);
|
||||
const client = getClientsMap().get(normalizedId);
|
||||
if (!client) return null;
|
||||
|
||||
const updated: ClientSDK = {
|
||||
@@ -474,10 +698,12 @@ export class SDKManager {
|
||||
...(updates.status !== undefined ? { status: updates.status } : {}),
|
||||
};
|
||||
|
||||
getClientsMap().set(clientId, updated);
|
||||
void this.persistClientUpdates(updated);
|
||||
getClientsMap().set(normalizedId, updated);
|
||||
void this.persistClientUpdates(updated).catch((err) => {
|
||||
console.error("[DataStack SDK] Background client update DB persist failed:", err);
|
||||
});
|
||||
notifyIngestionClientSync(
|
||||
clientId,
|
||||
normalizedId,
|
||||
updated.status === "revoked" ? "revoked" : "updated",
|
||||
updated,
|
||||
);
|
||||
@@ -488,16 +714,21 @@ export class SDKManager {
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (!pool) return;
|
||||
|
||||
const ready = await DatabaseEngine.ensureSchema();
|
||||
if (!ready) {
|
||||
throw new Error("DATABASE_WRITE_FAILED: PostgreSQL schema unavailable for client update.");
|
||||
}
|
||||
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
await pool.query(
|
||||
`UPDATE client_sdks
|
||||
SET name = $2, rate_limit = $3, status = $4, updated_at = NOW()
|
||||
WHERE client_id = $1`,
|
||||
[client.clientId, client.name, client.rateLimit, client.status],
|
||||
);
|
||||
} catch {
|
||||
// Ignore DB failure
|
||||
} catch (err: unknown) {
|
||||
const detail = err instanceof Error ? err.message : "Unknown DB error";
|
||||
throw new Error(`DATABASE_WRITE_FAILED: Failed to update client (${detail})`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -506,7 +737,8 @@ export class SDKManager {
|
||||
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
|
||||
): Promise<ClientSDK | null> {
|
||||
await ensureDbSettingsHydrated();
|
||||
const client = await this.getClientAsync(clientId);
|
||||
const normalizedId = this.normalizeClientId(clientId);
|
||||
const client = await this.getClientAsync(normalizedId);
|
||||
if (!client) return null;
|
||||
|
||||
const updated: ClientSDK = {
|
||||
@@ -518,10 +750,10 @@ export class SDKManager {
|
||||
...(updates.status !== undefined ? { status: updates.status } : {}),
|
||||
};
|
||||
|
||||
getClientsMap().set(clientId, updated);
|
||||
await this.persistClientUpdates(updated);
|
||||
getClientsMap().set(normalizedId, updated);
|
||||
await notifyIngestionClientSyncAsync(
|
||||
clientId,
|
||||
normalizedId,
|
||||
updated.status === "revoked" ? "revoked" : "updated",
|
||||
updated,
|
||||
);
|
||||
@@ -529,16 +761,12 @@ export class SDKManager {
|
||||
}
|
||||
|
||||
public static isClientDeleted(clientId: string): boolean {
|
||||
const normalized = String(clientId || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const normalized = this.normalizeClientId(clientId);
|
||||
return getDeletedClientsSet().has(normalized);
|
||||
}
|
||||
|
||||
public static async isClientDeletedAsync(clientId: string): Promise<boolean> {
|
||||
const normalized = String(clientId || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const normalized = this.normalizeClientId(clientId);
|
||||
if (!normalized) return false;
|
||||
if (getDeletedClientsSet().has(normalized)) {
|
||||
return true;
|
||||
@@ -560,15 +788,18 @@ export class SDKManager {
|
||||
}
|
||||
|
||||
public static deleteClient(clientId: string): boolean {
|
||||
const normalized = String(clientId || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const normalized = this.normalizeClientId(clientId);
|
||||
const deleted = getClientsMap().delete(normalized);
|
||||
getSyncedSocketEventsMap().delete(normalized);
|
||||
getPendingClientActivityMap().delete(normalized);
|
||||
if (deleted) {
|
||||
getDeletedClientsSet().add(normalized);
|
||||
DataBufferEngine.markClientDeleted(normalized);
|
||||
void DataBufferEngine.purgeClientEventsAsync(normalized).catch(() => undefined);
|
||||
}
|
||||
void this.deleteClientFromDb(normalized);
|
||||
void this.deleteClientFromDb(normalized).catch((err) => {
|
||||
console.error("[DataStack SDK] Background client delete DB persist failed:", err);
|
||||
});
|
||||
notifyIngestionClientSync(normalized, "deleted");
|
||||
return deleted;
|
||||
}
|
||||
@@ -577,7 +808,10 @@ export class SDKManager {
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (!pool) return false;
|
||||
|
||||
await DatabaseEngine.ensureSchema();
|
||||
const ready = await DatabaseEngine.ensureSchema();
|
||||
if (!ready) {
|
||||
throw new Error("DATABASE_WRITE_FAILED: PostgreSQL schema unavailable for client deletion.");
|
||||
}
|
||||
const dbClient = await pool.connect();
|
||||
try {
|
||||
await dbClient.query("BEGIN");
|
||||
@@ -597,69 +831,226 @@ export class SDKManager {
|
||||
}
|
||||
await dbClient.query("ROLLBACK");
|
||||
return false;
|
||||
} catch {
|
||||
} catch (err: unknown) {
|
||||
await dbClient.query("ROLLBACK").catch(() => undefined);
|
||||
return false;
|
||||
const detail = err instanceof Error ? err.message : "Unknown DB error";
|
||||
throw new Error(`DATABASE_WRITE_FAILED: Failed to delete client (${detail})`);
|
||||
} finally {
|
||||
dbClient.release();
|
||||
}
|
||||
}
|
||||
|
||||
private static purgeClientStorageDir(clientId: string): void {
|
||||
private static purgeClientStorageDir(clientId: string): number {
|
||||
const safeId = clientId.replace(/[^a-z0-9_-]/gi, "_");
|
||||
if (!safeId) return;
|
||||
const storageRoot = resolve(process.cwd(), "storage", "data");
|
||||
if (!safeId) return 0;
|
||||
const storageRoot = resolve(CSVStorageManager.baseStorageDir);
|
||||
const targetDir = resolve(storageRoot, safeId);
|
||||
if (!targetDir.startsWith(`${storageRoot}${sep}`)) return;
|
||||
if (existsSync(targetDir)) {
|
||||
rmSync(targetDir, { recursive: true, force: true });
|
||||
if (!targetDir.startsWith(`${storageRoot}${sep}`)) return 0;
|
||||
if (!existsSync(targetDir)) return 0;
|
||||
|
||||
let purgedFiles = 0;
|
||||
try {
|
||||
const dirStat = lstatSync(targetDir);
|
||||
if (!dirStat.isSymbolicLink() && dirStat.isDirectory()) {
|
||||
for (const entry of readdirSync(targetDir)) {
|
||||
if (!entry.endsWith(".csv")) continue;
|
||||
const entryPath = resolve(targetDir, entry);
|
||||
try {
|
||||
const entryStat = lstatSync(entryPath);
|
||||
if (!entryStat.isSymbolicLink() && entryStat.isFile()) {
|
||||
purgedFiles += 1;
|
||||
}
|
||||
} catch {
|
||||
// Ignore unreadable file entry during pre-purge count
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
purgedFiles = 0;
|
||||
}
|
||||
|
||||
rmSync(targetDir, { recursive: true, force: true });
|
||||
CSVStorageManager.clearRowCountCache();
|
||||
return purgedFiles;
|
||||
}
|
||||
|
||||
public static async deleteClientWithStatsAsync(
|
||||
clientId: string,
|
||||
options?: { purgeFiles?: boolean },
|
||||
): Promise<{ deleted: boolean; purgedFiles: number }> {
|
||||
await ensureDbSettingsHydrated();
|
||||
const normalized = this.normalizeClientId(clientId);
|
||||
getPendingClientActivityMap().delete(normalized);
|
||||
const deletedFromDb = await this.deleteClientFromDb(normalized);
|
||||
const existedInMem = getClientsMap().delete(normalized);
|
||||
getSyncedSocketEventsMap().delete(normalized);
|
||||
let purgedFiles = 0;
|
||||
if (existedInMem || deletedFromDb) {
|
||||
getDeletedClientsSet().add(normalized);
|
||||
DataBufferEngine.markClientDeleted(normalized);
|
||||
await DataBufferEngine.purgeClientEventsAsync(normalized);
|
||||
await notifyIngestionClientSyncAsync(normalized, "deleted");
|
||||
if (options?.purgeFiles) {
|
||||
purgedFiles = this.purgeClientStorageDir(normalized);
|
||||
}
|
||||
}
|
||||
return { deleted: existedInMem || deletedFromDb, purgedFiles };
|
||||
}
|
||||
|
||||
public static async deleteClientAsync(
|
||||
clientId: string,
|
||||
options?: { purgeFiles?: boolean },
|
||||
): Promise<boolean> {
|
||||
await ensureDbSettingsHydrated();
|
||||
const normalized = String(clientId || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const existedInMem = getClientsMap().delete(normalized);
|
||||
getSyncedSocketEventsMap().delete(normalized);
|
||||
const deletedFromDb = await this.deleteClientFromDb(normalized);
|
||||
if (existedInMem || deletedFromDb) {
|
||||
getDeletedClientsSet().add(normalized);
|
||||
if (options?.purgeFiles) {
|
||||
this.purgeClientStorageDir(normalized);
|
||||
}
|
||||
await notifyIngestionClientSyncAsync(normalized, "deleted");
|
||||
}
|
||||
return existedInMem || deletedFromDb;
|
||||
const result = await this.deleteClientWithStatsAsync(clientId, options);
|
||||
return result.deleted;
|
||||
}
|
||||
|
||||
public static incrementEventCount(clientId: string, count = 1): void {
|
||||
const nowIso = new Date().toISOString();
|
||||
const client = getClientsMap().get(clientId);
|
||||
if (client) {
|
||||
client.totalEvents += count;
|
||||
client.lastActive = nowIso;
|
||||
getClientsMap().set(clientId, client);
|
||||
private static queuePendingActivityDelta(
|
||||
normalizedId: string,
|
||||
count: number,
|
||||
nowIso: string,
|
||||
): number {
|
||||
const pending = getPendingClientActivityMap();
|
||||
const existing = pending.get(normalizedId);
|
||||
if (existing) {
|
||||
pending.set(normalizedId, {
|
||||
count: existing.count + count,
|
||||
lastActive:
|
||||
nowIso > existing.lastActive ? nowIso : existing.lastActive,
|
||||
});
|
||||
} else {
|
||||
pending.set(normalizedId, {
|
||||
count,
|
||||
lastActive: nowIso,
|
||||
});
|
||||
}
|
||||
ensureActivityFlushTimer();
|
||||
let totalDelta = 0;
|
||||
for (const entry of pending.values()) {
|
||||
totalDelta += entry.count;
|
||||
}
|
||||
return totalDelta;
|
||||
}
|
||||
|
||||
public static async flushPendingClientActivityAsync(): Promise<number> {
|
||||
if (globals.__dataStackActivityFlushPromise) {
|
||||
await globals.__dataStackActivityFlushPromise;
|
||||
if (getPendingClientActivityMap().size === 0) return 0;
|
||||
}
|
||||
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (pool) {
|
||||
void DatabaseEngine.ensureSchema()
|
||||
.then(() =>
|
||||
pool.query(
|
||||
`UPDATE client_sdks
|
||||
SET total_events = total_events + $2,
|
||||
last_active = $3,
|
||||
updated_at = NOW()
|
||||
WHERE client_id = $1`,
|
||||
[clientId, count, nowIso],
|
||||
),
|
||||
)
|
||||
.catch(() => undefined);
|
||||
const pending = getPendingClientActivityMap();
|
||||
if (!pool || pending.size === 0) return 0;
|
||||
|
||||
const snapshot = Array.from(pending.entries());
|
||||
pending.clear();
|
||||
|
||||
const clientIds: string[] = [];
|
||||
const deltas: number[] = [];
|
||||
const lastActives: string[] = [];
|
||||
for (const [cid, entry] of snapshot) {
|
||||
clientIds.push(cid);
|
||||
deltas.push(entry.count);
|
||||
lastActives.push(entry.lastActive);
|
||||
}
|
||||
|
||||
const run = (async () => {
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
await pool.query(
|
||||
`UPDATE client_sdks AS c
|
||||
SET total_events = c.total_events + v.delta,
|
||||
last_active = GREATEST(COALESCE(c.last_active, v.last_active), v.last_active),
|
||||
updated_at = NOW()
|
||||
FROM (
|
||||
SELECT UNNEST($1::text[]) AS client_id,
|
||||
UNNEST($2::bigint[]) AS delta,
|
||||
UNNEST($3::timestamptz[]) AS last_active
|
||||
) AS v
|
||||
WHERE c.client_id = v.client_id`,
|
||||
[clientIds, deltas, lastActives],
|
||||
);
|
||||
return clientIds.length;
|
||||
} catch {
|
||||
// Re-merge deltas so transient DB errors do not drop event counts
|
||||
for (const [cid, entry] of snapshot) {
|
||||
const current = pending.get(cid);
|
||||
if (current) {
|
||||
pending.set(cid, {
|
||||
count: current.count + entry.count,
|
||||
lastActive:
|
||||
current.lastActive > entry.lastActive
|
||||
? current.lastActive
|
||||
: entry.lastActive,
|
||||
});
|
||||
} else {
|
||||
pending.set(cid, { ...entry });
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
} finally {
|
||||
globals.__dataStackActivityFlushPromise = null;
|
||||
}
|
||||
})();
|
||||
|
||||
globals.__dataStackActivityFlushPromise = run;
|
||||
return run;
|
||||
}
|
||||
|
||||
public static stopActivityFlushTimerForTests(): void {
|
||||
if (globals.__dataStackActivityFlushTimer) {
|
||||
clearInterval(globals.__dataStackActivityFlushTimer);
|
||||
globals.__dataStackActivityFlushTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
public static incrementEventCount(clientId: string, count = 1): void {
|
||||
const normalizedId = this.normalizeClientId(clientId);
|
||||
if (!normalizedId || count <= 0) return;
|
||||
const nowIso = new Date().toISOString();
|
||||
const client = getClientsMap().get(normalizedId);
|
||||
if (client) {
|
||||
client.totalEvents += count;
|
||||
client.lastActive = nowIso;
|
||||
getClientsMap().set(normalizedId, client);
|
||||
}
|
||||
|
||||
if (DatabaseEngine.getPool()) {
|
||||
const queuedTotal = this.queuePendingActivityDelta(
|
||||
normalizedId,
|
||||
count,
|
||||
nowIso,
|
||||
);
|
||||
if (queuedTotal >= CLIENT_ACTIVITY_IMMEDIATE_BATCH_SIZE) {
|
||||
void this.flushPendingClientActivityAsync().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static async incrementEventCountAsync(
|
||||
clientId: string,
|
||||
count = 1,
|
||||
): Promise<void> {
|
||||
const normalizedId = this.normalizeClientId(clientId);
|
||||
if (!normalizedId || count <= 0) return;
|
||||
const nowIso = new Date().toISOString();
|
||||
const client = getClientsMap().get(normalizedId);
|
||||
if (client) {
|
||||
client.totalEvents += count;
|
||||
client.lastActive = nowIso;
|
||||
getClientsMap().set(normalizedId, client);
|
||||
}
|
||||
|
||||
// BUG-22: Coalesce per-event DB updates into batched UNNEST writes instead of 1 SQL query/event
|
||||
if (DatabaseEngine.getPool()) {
|
||||
const queuedTotal = this.queuePendingActivityDelta(
|
||||
normalizedId,
|
||||
count,
|
||||
nowIso,
|
||||
);
|
||||
if (queuedTotal >= CLIENT_ACTIVITY_IMMEDIATE_BATCH_SIZE) {
|
||||
await this.flushPendingClientActivityAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -672,7 +1063,9 @@ export class SDKManager {
|
||||
rateLimit: client.rateLimit,
|
||||
});
|
||||
map.set(clientId, client);
|
||||
void this.persistClientCredentials(client);
|
||||
void this.persistClientCredentials(client).catch((err) => {
|
||||
console.error("[DataStack SDK] Background re-sign DB persist failed:", err);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -680,15 +1073,25 @@ export class SDKManager {
|
||||
const clients = await this.listClientsAsync();
|
||||
const map = getClientsMap();
|
||||
for (const client of clients) {
|
||||
client.token = AuthService.generateClientToken({
|
||||
clientId: client.clientId,
|
||||
name: client.name,
|
||||
rateLimit: client.rateLimit,
|
||||
});
|
||||
map.set(client.clientId, client);
|
||||
await this.persistClientCredentials(client);
|
||||
const updated: ClientSDK = {
|
||||
...client,
|
||||
token: AuthService.generateClientToken({
|
||||
clientId: client.clientId,
|
||||
name: client.name,
|
||||
rateLimit: client.rateLimit,
|
||||
}),
|
||||
};
|
||||
await this.persistClientCredentials(updated);
|
||||
map.set(updated.clientId, updated);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!globals.__dataStackActivityShutdownRegistered) {
|
||||
globals.__dataStackActivityShutdownRegistered = true;
|
||||
DataBufferEngine.registerShutdownHook(() =>
|
||||
SDKManager.flushPendingClientActivityAsync(),
|
||||
);
|
||||
}
|
||||
|
||||
export default SDKManager;
|
||||
Reference in new issue
Block a user