refactor(auth): standardize session management and security

- Centralize secret retrieval using `getAdminSessionSecret`
- Replace manual session parsing with `verifyRequestAdminSession` utility
- Tighten API validation and improve security headers handling
- Enforce environment variable security for production deployments
This commit is contained in:
Team Dark Devil committed 2026-11-18 05:18:55 +06:00
1 parent 34519c07df
commit 177ed4a844
78 files changed
+6748 -3157

No files matched your search

+25
View File
@@ -73,6 +73,31 @@ export class DatabaseEngine {
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`),
pool.query(`
CREATE TABLE IF NOT EXISTS deleted_clients (
client_id TEXT PRIMARY KEY,
deleted_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`),
pool.query(`
CREATE TABLE IF NOT EXISTS admin_users (
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
email TEXT NOT NULL UNIQUE,
password_salt BYTEA NOT NULL,
password_hash BYTEA NOT NULL,
token_version INTEGER NOT NULL DEFAULT 1,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`),
pool.query(`
CREATE TABLE IF NOT EXISTS admin_login_attempts (
key_hash TEXT PRIMARY KEY,
window_started_at TIMESTAMPTZ NOT NULL,
failed_attempts INTEGER NOT NULL,
blocked_until TIMESTAMPTZ,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`),
]).then(() => undefined);
try {
+48 -1
View File
@@ -1,3 +1,38 @@
import { createHash } from "node:crypto";
import { AuthService } from "../auth/jwt";
export function resolveInternalIngestionSecret(
jwtSecretOverride?: string,
): string {
const explicit = (process.env.DATASTACK_INTERNAL_SECRET || "").trim();
const isProd = process.env.NODE_ENV === "production";
if (isProd) {
if (explicit.length >= 32) return explicit;
if (explicit.length > 0) return "";
} else if (explicit.length > 0) {
return explicit;
}
const activeJwt = String(
jwtSecretOverride ?? AuthService.getSecret() ?? "",
).trim();
const minJwtLen = isProd ? 32 : 1;
if (activeJwt.length < minJwtLen) {
return "";
}
return createHash("sha256")
.update(`datastack-internal:${activeJwt}`)
.digest("hex");
}
export function getInternalIngestionHeaders(
jwtSecretOverride?: string,
): Record<string, string> {
const secret = resolveInternalIngestionSecret(jwtSecretOverride);
return secret ? { "x-datastack-internal-secret": secret } : {};
}
export interface ValidationResult {
allowed: boolean;
reason?: string;
@@ -130,7 +165,7 @@ export class SecurityEngine {
if (!withinLimit) {
return {
allowed: false,
reason: `Rate limit of ${clientMaxRate} events/sec exceeded for client ${clientId}`,
reason: `Rate limit of ${effectiveRateLimit} events/sec exceeded for client ${clientId}`,
code: "RATE_LIMIT_EXCEEDED",
};
}
@@ -144,6 +179,18 @@ export class SecurityEngine {
}
public static getStats() {
const now = Date.now();
for (const [k, timestamp] of this.idempotencySet.entries()) {
if (now - timestamp > this.IDEMPOTENCY_TTL_MS) {
this.idempotencySet.delete(k);
}
}
for (const [clientId, entry] of this.rateLimitMap.entries()) {
if (now - entry.windowStart > 10000) {
this.rateLimitMap.delete(clientId);
}
}
return {
activeIdempotencyKeys: this.idempotencySet.size,
rateLimitedClientsCount: this.rateLimitMap.size,
+392 -53
View File
@@ -19,10 +19,30 @@ export interface StorageFileMeta {
modifiedAt: Date;
}
interface RowCountCacheEntry {
sizeBytes: number;
mtimeMs: number;
linesCount: number;
}
const LOCK_RETRY_MIN_MS = 5;
const LOCK_RETRY_MAX_MS = 18;
const LOCK_TIMEOUT_MS = 10_000;
const STALE_LOCK_AGE_MS = 30_000;
function sleepMs(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
export class CSVStorageManager {
// Base storage directory: /storage/data
public static baseStorageDir = path.resolve(process.cwd(), 'storage', 'data');
public static maxFileSizeBytes = 100 * 1024 * 1024; // 100MB rotation limit
private static rowCountCache = new Map<string, RowCountCacheEntry>();
public static clearRowCountCache(): void {
this.rowCountCache.clear();
}
/**
* Ensures client directory exists
@@ -37,15 +57,194 @@ export class CSVStorageManager {
}
/**
* Resolves target CSV file path with daily and 100MB rotation rules
* Acquires an exclusive cooperative lock file for a client+date partition across Port 3000 & Port 4000
*/
public static async withPartitionLock<T>(
clientId: string,
dateStr: string,
fn: () => Promise<T> | T,
): Promise<T> {
const clientDir = this.ensureClientDir(clientId);
const date = this.extractSafePartitionDate(dateStr);
const lockPath = path.join(clientDir, `.${date}.csv.lock`);
const deadline = Date.now() + LOCK_TIMEOUT_MS;
let acquired = false;
while (Date.now() < deadline) {
try {
const fd = fs.openSync(lockPath, 'wx');
try {
fs.writeFileSync(fd, `${process.pid}:${Date.now()}`, 'utf8');
} finally {
fs.closeSync(fd);
}
acquired = true;
break;
} catch (err: unknown) {
const code = (err as NodeJS.ErrnoException)?.code;
if (code !== 'EEXIST') {
throw err;
}
try {
const lockStat = fs.statSync(lockPath);
if (Date.now() - lockStat.mtimeMs > STALE_LOCK_AGE_MS) {
fs.unlinkSync(lockPath);
continue;
}
} catch {
// Lock was released concurrently
}
const jitter =
LOCK_RETRY_MIN_MS +
Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS));
await sleepMs(jitter);
}
}
if (!acquired) {
throw new Error(`Timed out waiting for CSV partition lock: ${lockPath}`);
}
try {
return await fn();
} finally {
try {
if (fs.existsSync(lockPath)) {
fs.unlinkSync(lockPath);
}
} catch {
// Ignore cleanup failure if already unlinked
}
}
}
/**
* Normalizes and validates any timestamp or date string into a strict YYYY-MM-DD partition key
*/
public static extractSafePartitionDate(rawTimestamp?: string): string {
const fallback = new Date().toISOString().slice(0, 10);
if (!rawTimestamp || typeof rawTimestamp !== 'string') {
return fallback;
}
const trimmed = rawTimestamp.trim();
const prefix = trimmed.slice(0, 10);
if (/^\d{4}-\d{2}-\d{2}$/.test(prefix)) {
const parsedPrefix = new Date(`${prefix}T00:00:00.000Z`);
if (!Number.isNaN(parsedPrefix.getTime())) {
return prefix;
}
}
if (/^\d+(\.\d+)?$/.test(trimmed)) {
const num = Number(trimmed);
const ms = num < 1e11 ? num * 1000 : num;
const d = new Date(ms);
if (!Number.isNaN(d.getTime())) {
return d.toISOString().slice(0, 10);
}
}
const parsed = new Date(trimmed);
if (!Number.isNaN(parsed.getTime())) {
return parsed.toISOString().slice(0, 10);
}
return fallback;
}
/**
* Stateful RFC 4180 quote-aware chunked row counter so multi-100MB CSVs never load wholly into RAM
*/
private static createChunkRowCounter() {
let inQuotes = false;
let pendingQuote = false;
let currentLineHasChars = false;
let totalRows = 0;
return {
pushChunk(chunk: Buffer | string) {
const str = typeof chunk === 'string' ? chunk : chunk.toString('utf8');
for (let i = 0; i < str.length; i += 1) {
const ch = str[i];
if (pendingQuote) {
pendingQuote = false;
if (ch === '"') {
// Escaped quote "" across chunk boundary or inline
continue;
}
inQuotes = !inQuotes;
}
if (ch === '"') {
currentLineHasChars = true;
if (inQuotes) {
if (i + 1 < str.length) {
if (str[i + 1] === '"') {
i += 1;
} else {
inQuotes = false;
}
} else {
pendingQuote = true;
}
} else {
inQuotes = true;
}
} else if (ch === '\n' && !inQuotes) {
if (currentLineHasChars) {
totalRows += 1;
currentLineHasChars = false;
}
} else if (ch !== '\r' && ch !== ' ' && ch !== '\t') {
currentLineHasChars = true;
}
}
},
finish(): number {
if (currentLineHasChars) {
totalRows += 1;
}
return Math.max(0, totalRows - 1);
},
};
}
private static countCsvDataRowsChunkedSync(filePath: string): number {
const counter = this.createChunkRowCounter();
const buf = Buffer.allocUnsafe(64 * 1024);
const fd = fs.openSync(filePath, 'r');
try {
let bytesRead = 0;
while ((bytesRead = fs.readSync(fd, buf, 0, buf.length, null)) > 0) {
counter.pushChunk(buf.subarray(0, bytesRead));
}
return counter.finish();
} finally {
fs.closeSync(fd);
}
}
private static async countCsvDataRowsStreamAsync(
filePath: string,
): Promise<number> {
const counter = this.createChunkRowCounter();
return new Promise<number>((resolve, reject) => {
const stream = fs.createReadStream(filePath, {
highWaterMark: 64 * 1024,
});
stream.on('data', (chunk) => counter.pushChunk(chunk));
stream.on('end', () => resolve(counter.finish()));
stream.on('error', (err) => reject(err));
});
}
/**
* Resolves target CSV file path with daily and configured MB rotation rules
*/
public static getTargetFilePath(clientId: string, dateStr?: string): string {
const clientDir = this.ensureClientDir(clientId);
const date = dateStr || new Date().toISOString().split('T')[0];
const date = this.extractSafePartitionDate(dateStr);
let baseFileName = `${date}.csv`;
let targetPath = path.join(clientDir, baseFileName);
// Check for 100MB rotation
// Check for file rotation
let partIndex = 1;
while (fs.existsSync(targetPath)) {
try {
@@ -66,10 +265,12 @@ export class CSVStorageManager {
}
/**
* Batch append events into target CSV file.
* Batch append events into target CSV file under cooperative partition lock.
* If file doesn't exist, writes header: clientId,eventId,timestamp,payload
*/
public static async appendBatch(events: DataEvent[]): Promise<{ written: number; filesUpdated: string[] }> {
public static async appendBatch(
events: DataEvent[],
): Promise<{ written: number; filesUpdated: string[] }> {
if (!events || events.length === 0) {
return { written: 0, filesUpdated: [] };
}
@@ -77,7 +278,7 @@ export class CSVStorageManager {
// Group events by client + date
const grouped = new Map<string, DataEvent[]>();
for (const evt of events) {
const date = (evt.timestamp || new Date().toISOString()).split('T')[0];
const date = this.extractSafePartitionDate(evt.timestamp);
const key = `${evt.clientId}:::${date}`;
if (!grouped.has(key)) {
grouped.set(key, []);
@@ -90,31 +291,39 @@ export class CSVStorageManager {
for (const [key, batch] of grouped.entries()) {
const [clientId, date] = key.split(':::');
const targetPath = this.getTargetFilePath(clientId, date);
let hasContent = false;
try {
hasContent =
fs.existsSync(targetPath) && fs.statSync(targetPath).size > 0;
} catch {
hasContent = false;
}
const targetPath = await this.withPartitionLock(clientId, date, () => {
const resolvedPath = this.getTargetFilePath(clientId, date);
let hasContent = false;
try {
hasContent =
fs.existsSync(resolvedPath) && fs.statSync(resolvedPath).size > 0;
} catch {
hasContent = false;
}
// Format payload as JSON string if not already
const cleanBatch = batch.map((item) => ({
clientId: item.clientId,
eventId: item.eventId,
timestamp: item.timestamp,
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
}));
const cleanBatch = batch.map((item) => ({
clientId: item.clientId,
eventId: item.eventId,
timestamp: item.timestamp,
payload:
typeof item.payload === 'string'
? item.payload
: JSON.stringify(item.payload),
}));
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
const csvData = parser.parse(cleanBatch);
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
const csvData = parser.parse(cleanBatch);
if (hasContent) {
fs.appendFileSync(targetPath, '\n' + csvData, 'utf8');
} else {
fs.writeFileSync(targetPath, csvData, 'utf8');
}
if (hasContent) {
fs.appendFileSync(resolvedPath, '\n' + csvData, 'utf8');
} else {
fs.writeFileSync(resolvedPath, csvData, 'utf8');
}
// Invalidate row cache for modified file immediately
this.rowCountCache.delete(resolvedPath);
return resolvedPath;
});
filesUpdated.push(targetPath);
}
@@ -123,14 +332,16 @@ export class CSVStorageManager {
}
/**
* List all stored CSV files with sizes and dates
* List all stored CSV files with cached row counts (invalidated on mtime/size changes)
*/
public static listAllFiles(): StorageFileMeta[] {
const results: StorageFileMeta[] = [];
if (!fs.existsSync(this.baseStorageDir)) {
this.rowCountCache.clear();
return results;
}
const activePaths = new Set<string>();
try {
const clientDirs = fs.readdirSync(this.baseStorageDir);
for (const clientDir of clientDirs) {
@@ -142,15 +353,27 @@ export class CSVStorageManager {
if (!file.endsWith('.csv')) continue;
const fullFilePath = path.join(fullClientPath, file);
const stats = fs.statSync(fullFilePath);
activePaths.add(fullFilePath);
// Count lines approximately (or sample)
const cached = this.rowCountCache.get(fullFilePath);
let linesCount = 0;
try {
const content = fs.readFileSync(fullFilePath, 'utf8');
linesCount = content.split('\n').filter((l) => l.trim().length > 0).length - 1; // subtract header
if (linesCount < 0) linesCount = 0;
} catch {
linesCount = 0;
if (
cached &&
cached.sizeBytes === stats.size &&
cached.mtimeMs === stats.mtimeMs
) {
linesCount = cached.linesCount;
} else {
try {
linesCount = this.countCsvDataRowsChunkedSync(fullFilePath);
this.rowCountCache.set(fullFilePath, {
sizeBytes: stats.size,
mtimeMs: stats.mtimeMs,
linesCount,
});
} catch {
linesCount = 0;
}
}
const datePart = file.replace('.csv', '').split('-part')[0];
@@ -166,6 +389,12 @@ export class CSVStorageManager {
});
}
}
for (const key of this.rowCountCache.keys()) {
if (!activePaths.has(key)) {
this.rowCountCache.delete(key);
}
}
} catch {
// Return whatever collected
}
@@ -174,12 +403,88 @@ export class CSVStorageManager {
}
/**
* Prune CSV partitions older than autoPruneDays
* Non-blocking async listing using fs.promises + streaming row counting on cache misses
*/
public static async listAllFilesAsync(): Promise<StorageFileMeta[]> {
const results: StorageFileMeta[] = [];
if (!fs.existsSync(this.baseStorageDir)) {
this.rowCountCache.clear();
return results;
}
const activePaths = new Set<string>();
try {
const clientDirs = await fs.promises.readdir(this.baseStorageDir);
for (const clientDir of clientDirs) {
const fullClientPath = path.join(this.baseStorageDir, clientDir);
const dirStat = await fs.promises.stat(fullClientPath).catch(() => null);
if (!dirStat || !dirStat.isDirectory()) continue;
const files = await fs.promises.readdir(fullClientPath);
for (const file of files) {
if (!file.endsWith('.csv')) continue;
const fullFilePath = path.join(fullClientPath, file);
const stats = await fs.promises.stat(fullFilePath).catch(() => null);
if (!stats) continue;
activePaths.add(fullFilePath);
const cached = this.rowCountCache.get(fullFilePath);
let linesCount = 0;
if (
cached &&
cached.sizeBytes === stats.size &&
cached.mtimeMs === stats.mtimeMs
) {
linesCount = cached.linesCount;
} else {
try {
linesCount = await this.countCsvDataRowsStreamAsync(fullFilePath);
this.rowCountCache.set(fullFilePath, {
sizeBytes: stats.size,
mtimeMs: stats.mtimeMs,
linesCount,
});
} catch {
linesCount = 0;
}
}
const datePart = file.replace('.csv', '').split('-part')[0];
results.push({
clientId: clientDir,
date: datePart,
filename: file,
filePath: fullFilePath,
sizeBytes: stats.size,
linesCount,
modifiedAt: stats.mtime,
});
}
}
for (const key of this.rowCountCache.keys()) {
if (!activePaths.has(key)) {
this.rowCountCache.delete(key);
}
}
} catch {
// Return whatever collected
}
return results.sort((a, b) => b.modifiedAt.getTime() - a.modifiedAt.getTime());
}
/**
* Prune CSV partitions older than autoPruneDays without counting row contents
*/
public static pruneExpiredPartitions(autoPruneDays: number): number {
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
return 0;
}
if (!fs.existsSync(this.baseStorageDir)) {
return 0;
}
const cutoffDate = new Date(
Date.now() - Math.floor(autoPruneDays) * 86_400_000,
@@ -188,28 +493,38 @@ export class CSVStorageManager {
.split('T')[0];
let prunedCount = 0;
const files = this.listAllFiles();
for (const file of files) {
if (/^\d{4}-\d{2}-\d{2}$/.test(file.date) && file.date < cutoffDate) {
try {
if (fs.existsSync(file.filePath)) {
fs.unlinkSync(file.filePath);
prunedCount += 1;
try {
const clientDirs = fs.readdirSync(this.baseStorageDir);
for (const clientDir of clientDirs) {
const fullClientPath = path.join(this.baseStorageDir, clientDir);
if (!fs.statSync(fullClientPath).isDirectory()) continue;
const files = fs.readdirSync(fullClientPath);
for (const file of files) {
if (!file.endsWith('.csv')) continue;
const datePart = file.replace('.csv', '').split('-part')[0];
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
const fullPath = path.join(fullClientPath, file);
try {
if (fs.existsSync(fullPath)) {
fs.unlinkSync(fullPath);
this.rowCountCache.delete(fullPath);
prunedCount += 1;
}
} catch {
// Ignore file removal error
}
}
} catch {
// Ignore file removal error
}
}
} catch {
// Ignore directory scan failure
}
return prunedCount;
}
/**
* Get aggregate storage statistics
*/
public static getStorageStats() {
const files = this.listAllFiles();
private static formatStorageStats(files: StorageFileMeta[]) {
const totalBytes = files.reduce((acc, f) => acc + f.sizeBytes, 0);
const totalRows = files.reduce((acc, f) => acc + f.linesCount, 0);
const uniqueClients = new Set(files.map((f) => f.clientId)).size;
@@ -231,10 +546,34 @@ export class CSVStorageManager {
}
/**
* Reads file content for export / download
* Get aggregate storage statistics (sync cached)
*/
public static getStorageStats() {
return this.formatStorageStats(this.listAllFiles());
}
/**
* Get aggregate storage statistics (non-blocking async)
*/
public static async getStorageStatsAsync() {
const files = await this.listAllFilesAsync();
return this.formatStorageStats(files);
}
/**
* Reads file content for export / download with path traversal protection
*/
public static getFileContent(clientId: string, filename: string): string | null {
const fullPath = path.join(this.baseStorageDir, clientId, filename);
const safeClientId = String(clientId || '').replace(/[^a-zA-Z0-9_-]/g, '_');
const safeFilename = path.basename(String(filename || ''));
if (!safeClientId || !safeFilename || !safeFilename.endsWith('.csv')) {
return null;
}
const fullPath = path.resolve(this.baseStorageDir, safeClientId, safeFilename);
const resolvedBase = path.resolve(this.baseStorageDir);
if (!fullPath.startsWith(resolvedBase + path.sep)) {
return null;
}
if (fs.existsSync(fullPath)) {
return fs.readFileSync(fullPath, 'utf8');
}
+69 -11
View File
@@ -1,8 +1,11 @@
import { AuthService } from "../auth/jwt";
import { AuthService, resolveDefaultJwtSecret } from "../auth/jwt";
import { DataBufferEngine } from "../data/buffer";
import { DatabaseEngine } from "../Database/database";
import { SDKManager } from "../sdk/sdkManager";
import { SecurityEngine } from "../Security/security";
import {
SecurityEngine,
getInternalIngestionHeaders,
} from "../Security/security";
import { CSVStorageManager } from "../Storage/csvStorage";
export interface AdminSettings {
@@ -35,9 +38,7 @@ function getDefaultSettings(): AdminSettings {
batchFlushThreshold: 500,
batchFlushIntervalMs: 2000,
fileRotationMb: 100,
jwtSecret:
process.env.DATASTACK_JWT_SECRET ||
"datastack-jwt-default-secret-key-32-chars-long",
jwtSecret: resolveDefaultJwtSecret(),
enableIdempotency: true,
autoPruneDays: 30,
updatedAt: new Date().toISOString(),
@@ -62,7 +63,9 @@ export class SettingsManager {
AuthService.setSecret(settings.jwtSecret);
}
CSVStorageManager.maxFileSizeBytes = settings.fileRotationMb * 1024 * 1024;
CSVStorageManager.pruneExpiredPartitions(settings.autoPruneDays);
if (settings.autoPruneDays > 0) {
CSVStorageManager.pruneExpiredPartitions(settings.autoPruneDays);
}
}
private static async syncFromDatabase(): Promise<AdminSettings | null> {
@@ -83,6 +86,10 @@ export class SettingsManager {
if (result.rowCount && result.rowCount > 0) {
const row = result.rows[0];
const parsedPruneDays =
row.auto_prune_days !== undefined && row.auto_prune_days !== null
? Number(row.auto_prune_days)
: NaN;
const loaded: AdminSettings = {
serverPort: Number(row.server_port) || 4000,
webPort: Number(row.web_port) || 3000,
@@ -91,12 +98,9 @@ export class SettingsManager {
batchFlushThreshold: Number(row.batch_flush_threshold) || 500,
batchFlushIntervalMs: Number(row.batch_flush_interval_ms) || 2000,
fileRotationMb: Number(row.file_rotation_mb) || 100,
jwtSecret:
String(row.jwt_secret || "") ||
process.env.DATASTACK_JWT_SECRET ||
"datastack-jwt-default-secret-key-32-chars-long",
jwtSecret: String(row.jwt_secret || "").trim() || resolveDefaultJwtSecret(),
enableIdempotency: Boolean(row.enable_idempotency),
autoPruneDays: Number(row.auto_prune_days) || 30,
autoPruneDays: Number.isFinite(parsedPruneDays) ? parsedPruneDays : 30,
updatedAt: row.updated_at
? new Date(row.updated_at as string | Date).toISOString()
: new Date().toISOString(),
@@ -214,8 +218,46 @@ export class SettingsManager {
};
}
private static async notifyIngestionSettingsSync(
settings: AdminSettings,
authJwtSecret: string,
jwtSecretChanged: boolean,
): Promise<void> {
const headers = getInternalIngestionHeaders(authJwtSecret);
if (!headers["x-datastack-internal-secret"]) return;
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, "");
try {
await fetch(`${ingestionBaseUrl}/internal/settings-sync`, {
method: "POST",
headers: {
"Content-Type": "application/json",
...headers,
},
body: JSON.stringify({
maxEventsPerSec: settings.maxEventsPerSec,
maxPayloadSizeKb: settings.maxPayloadSizeKb,
batchFlushThreshold: settings.batchFlushThreshold,
batchFlushIntervalMs: settings.batchFlushIntervalMs,
fileRotationMb: settings.fileRotationMb,
enableIdempotency: settings.enableIdempotency,
autoPruneDays: settings.autoPruneDays,
...(jwtSecretChanged ? { jwtSecret: settings.jwtSecret } : {}),
}),
cache: "no-store",
signal: AbortSignal.timeout(1500),
});
} catch {
// Port 4000 may not be running in isolated single-process dev mode
}
}
public static updateSettings(updates: Partial<AdminSettings>): AdminSettings {
const current = getSettingsState();
const previousJwtSecret = current.jwtSecret;
const jwtSecretChanged =
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
const nextSettings: AdminSettings = {
@@ -229,6 +271,13 @@ export class SettingsManager {
SDKManager.resignAllClientTokens();
}
void this.persistSettingsToDb(nextSettings);
void this.notifyIngestionSettingsSync(
nextSettings,
previousJwtSecret,
jwtSecretChanged,
).then(() =>
jwtSecretChanged ? SDKManager.syncAllClientsToIngestionAsync() : undefined,
);
return { ...nextSettings };
}
@@ -236,6 +285,7 @@ export class SettingsManager {
updates: Partial<AdminSettings>,
): Promise<AdminSettings> {
const current = await this.getSettingsAsync();
const previousJwtSecret = current.jwtSecret;
const jwtSecretChanged =
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
const nextSettings: AdminSettings = {
@@ -249,6 +299,14 @@ export class SettingsManager {
await SDKManager.resignAllClientTokensAsync();
}
await this.persistSettingsToDb(nextSettings);
await this.notifyIngestionSettingsSync(
nextSettings,
previousJwtSecret,
jwtSecretChanged,
);
if (jwtSecretChanged) {
await SDKManager.syncAllClientsToIngestionAsync();
}
return { ...nextSettings };
}
}
+70 -13
View File
@@ -1,44 +1,93 @@
import { SettingsManager } from '../adminSettings/settingsManager';
import { SDKManager } from '../sdk/sdkManager';
import { CSVStorageManager } from '../Storage/csvStorage';
import { DataBufferEngine } from '../data/buffer';
import { SecurityEngine } from '../Security/security';
import {
SecurityEngine,
getInternalIngestionHeaders,
} from '../Security/security';
export class ApiEngine {
/**
* Retrieves overall telemetry and health metrics for the dashboard
*/
public static async getDashboardMetrics() {
const storageStats = CSVStorageManager.getStorageStats();
const localBufferStats = DataBufferEngine.getBufferStats();
const clients = await SDKManager.listClientsAsync();
const settings = await SettingsManager.getSettingsAsync();
const [storageStats, localBufferStats, clients] = await Promise.all([
CSVStorageManager.getStorageStatsAsync(),
DataBufferEngine.getBufferStatsAsync(),
SDKManager.listClientsAsync(),
]);
const activeClients = clients.filter((c) => c.status === 'active').length;
let activeSockets = 0;
let ingestionBufferSize = 0;
let ingestionLocalBufferSize = 0;
let ingestionRedisBufferSize = 0;
let ingestionRedisConnected = false;
let remoteSecurityStats: {
activeIdempotencyKeys?: number;
rateLimitedClientsCount?: number;
maxPayloadSizeBytes?: number;
} | null = null;
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
).replace(/\/$/, '');
const internalHeaders = getInternalIngestionHeaders();
try {
const res = await fetch(`${ingestionBaseUrl}/health`, {
const res = await fetch(`${ingestionBaseUrl}/internal/metrics`, {
headers: internalHeaders,
cache: 'no-store',
signal: AbortSignal.timeout(1000),
});
if (res.ok) {
const data = await res.json();
activeSockets = Number(data.activeSockets) || 0;
ingestionBufferSize = Number(data.bufferSize) || 0;
ingestionRedisConnected = Boolean(data.redisConnected);
if (data.localBufferSize !== undefined) {
ingestionLocalBufferSize = Number(data.localBufferSize) || 0;
ingestionRedisBufferSize = Number(data.redisBufferSize) || 0;
} else {
ingestionLocalBufferSize = Number(data.bufferSize) || 0;
}
if (data.securityStats && typeof data.securityStats === 'object') {
remoteSecurityStats = data.securityStats;
}
}
} catch {
// Ingestion server may not be running in single-process dev mode
activeSockets = 0;
ingestionBufferSize = 0;
ingestionLocalBufferSize = 0;
ingestionRedisBufferSize = 0;
}
const securityStats = SecurityEngine.getStats();
const localSecurityStats = SecurityEngine.getStats();
const securityStats = {
activeIdempotencyKeys:
localSecurityStats.activeIdempotencyKeys +
(Number(remoteSecurityStats?.activeIdempotencyKeys) || 0),
rateLimitedClientsCount:
localSecurityStats.rateLimitedClientsCount +
(Number(remoteSecurityStats?.rateLimitedClientsCount) || 0),
maxPayloadSizeBytes: Math.min(
localSecurityStats.maxPayloadSizeBytes,
Number(remoteSecurityStats?.maxPayloadSizeBytes) ||
localSecurityStats.maxPayloadSizeBytes,
),
};
const effectiveRedisBufferSize = Math.max(
localBufferStats.redisBufferSize || 0,
ingestionRedisBufferSize,
);
const bufferStats = {
...localBufferStats,
redisConnected: localBufferStats.redisConnected || ingestionRedisConnected,
redisBufferSize: effectiveRedisBufferSize,
currentBufferSize:
localBufferStats.currentBufferSize + ingestionBufferSize,
localBufferStats.localBufferSize +
ingestionLocalBufferSize +
effectiveRedisBufferSize,
};
return {
@@ -49,8 +98,8 @@ export class ApiEngine {
storageStats,
securityStats,
runtime: {
serverPort: 4000,
webPort: 3000,
serverPort: settings.serverPort || 4000,
webPort: settings.webPort || 3000,
uptimeSeconds: Math.floor(process.uptime()),
memoryUsageMb: (process.memoryUsage().rss / (1024 * 1024)).toFixed(1),
nodeVersion: process.version,
@@ -59,7 +108,7 @@ export class ApiEngine {
}
/**
* Filters and streams stored CSV data for export
* Filters and lists stored CSV files for export
*/
public static getExportFiles(clientId?: string) {
const allFiles = CSVStorageManager.listAllFiles();
@@ -68,6 +117,14 @@ export class ApiEngine {
}
return allFiles;
}
public static async getExportFilesAsync(clientId?: string) {
const allFiles = await CSVStorageManager.listAllFilesAsync();
if (clientId && clientId !== 'all') {
return allFiles.filter((f) => f.clientId === clientId);
}
return allFiles;
}
}
export default ApiEngine;
+27 -3
View File
@@ -1,8 +1,23 @@
const encoder = new TextEncoder();
interface AdminSessionPayload {
const DEV_DEFAULT_SESSION_SECRET =
"datastack-admin-session-default-secret-32-chars";
export interface AdminSessionPayload {
email: string;
expiresAt: number;
tokenVersion: number;
}
export function getAdminSessionSecret(): string | null {
const configured = process.env.DATASTACK_ADMIN_SESSION_SECRET?.trim() || "";
if (process.env.NODE_ENV === "production") {
return configured.length >= 32 ? configured : null;
}
if (configured.length > 0) {
return configured.length >= 32 ? configured : null;
}
return DEV_DEFAULT_SESSION_SECRET;
}
function encodeBase64Url(bytes: Uint8Array) {
@@ -34,9 +49,14 @@ export async function createAdminSessionToken(
email: string,
secret: string,
expiresAt: number,
tokenVersion = 1,
) {
const safeVersion =
Number.isInteger(tokenVersion) && tokenVersion >= 1 ? tokenVersion : 1;
const payload = encodeBase64Url(
encoder.encode(JSON.stringify({ email, expiresAt })),
encoder.encode(
JSON.stringify({ email, expiresAt, tokenVersion: safeVersion }),
),
);
const signature = await crypto.subtle.sign(
"HMAC",
@@ -50,6 +70,7 @@ export async function getAdminSessionPayload(
token: string,
secret: string,
): Promise<AdminSessionPayload | null> {
if (!secret || secret.length < 32) return null;
const [payload, signature, extra] = token.split(".");
if (!payload || !signature || extra) return null;
@@ -68,7 +89,10 @@ export async function getAdminSessionPayload(
if (
typeof session.email === "string" &&
typeof session.expiresAt === "number" &&
session.expiresAt > Date.now()
session.expiresAt > Date.now() &&
typeof session.tokenVersion === "number" &&
Number.isInteger(session.tokenVersion) &&
session.tokenVersion >= 1
) {
return session;
}
+279 -163
View File
@@ -6,15 +6,22 @@ import {
timingSafeEqual,
} from "node:crypto";
import { Pool, type PoolClient } from "pg";
import {
type AdminSessionPayload,
getAdminSessionPayload,
getAdminSessionSecret,
} from "./adminSession";
const PASSWORD_HASH_BYTES = 64;
const PASSWORD_SALT_BYTES = 16;
const DATABASE_LOCK_ID = "72025102501";
const LOGIN_ATTEMPT_LIMIT = 5;
const IP_LOGIN_ATTEMPT_LIMIT = 25;
const LOGIN_WINDOW_MS = 15 * 60 * 1000;
const LOGIN_BLOCK_MS = 15 * 60 * 1000;
const LOCAL_THROTTLE_MAX_ENTRIES = 5000;
const LOCAL_THROTTLE_CLEANUP_MS = 60 * 1000;
const DEV_DEFAULT_SETUP_KEY = "datastack-default-setup-key-32-chars";
interface LoginThrottleEntry {
windowStartedAt: number;
@@ -28,11 +35,17 @@ interface LoginThrottleDecision {
retryAfterSeconds: number;
}
export interface LoginThrottleBucket {
keyHash: string;
limit: number;
}
interface InMemoryAdmin {
id: number;
email: string;
passwordSalt: Buffer;
passwordHash: Buffer;
tokenVersion: number;
createdAt: Date;
}
@@ -47,8 +60,12 @@ const localLoginThrottle = new Map<string, LoginThrottleEntry>();
let lastLocalThrottleCleanup = 0;
let lastDatabaseThrottleCleanup = 0;
function isProductionEnv() {
return process.env.NODE_ENV === "production";
}
function getPool(): Pool | null {
const connectionString = process.env.DATABASE_URL;
const connectionString = process.env.DATABASE_URL?.trim();
if (!connectionString) {
return null;
}
@@ -74,17 +91,24 @@ async function ensureAdminTable() {
if (!pool) return;
globals.__dataStackAdminSchema ??= Promise.all([
pool.query(
`
pool
.query(
`
CREATE TABLE IF NOT EXISTS admin_users (
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
email TEXT NOT NULL UNIQUE,
password_salt BYTEA NOT NULL,
password_hash BYTEA NOT NULL,
token_version INTEGER NOT NULL DEFAULT 1,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`,
),
)
.then(() =>
pool.query(
`ALTER TABLE admin_users ADD COLUMN IF NOT EXISTS token_version INTEGER NOT NULL DEFAULT 1`,
),
),
pool.query(
`
CREATE TABLE IF NOT EXISTS admin_login_attempts (
@@ -121,8 +145,41 @@ function matchesSecret(candidate: string, expected: string) {
return timingSafeEqual(candidateHash, expectedHash);
}
function createLoginThrottleKeys(email: string, secret: string) {
return [createHmac("sha256", secret).update(`email:${email}`).digest("hex")];
export function getAdminSetupKey(): string | null {
const configured = process.env.DATASTACK_ADMIN_SETUP_KEY?.trim() || "";
if (isProductionEnv()) {
return configured.length >= 32 ? configured : null;
}
if (configured.length > 0) {
return configured.length >= 32 ? configured : null;
}
return DEV_DEFAULT_SETUP_KEY;
}
export function createLoginThrottleBuckets(
email: string,
secret: string,
ip = "unknown",
): LoginThrottleBucket[] {
const cleanEmail = email.trim().toLowerCase();
const cleanIp = ip.trim().toLowerCase() || "unknown";
const buckets: LoginThrottleBucket[] = [
{
keyHash: createHmac("sha256", secret)
.update(`ip_email:${cleanIp}:${cleanEmail}`)
.digest("hex"),
limit: LOGIN_ATTEMPT_LIMIT,
},
];
if (cleanIp !== "unknown") {
buckets.push({
keyHash: createHmac("sha256", secret)
.update(`ip:${cleanIp}`)
.digest("hex"),
limit: IP_LOGIN_ATTEMPT_LIMIT,
});
}
return buckets;
}
function getAdvisoryLockId(keyHash: string) {
@@ -157,14 +214,14 @@ function cleanupLocalLoginThrottle(now: number) {
}
function consumeLocalLoginAttempt(
keyHashes: string[],
buckets: LoginThrottleBucket[],
now: number,
): LoginThrottleDecision {
cleanupLocalLoginThrottle(now);
const blockedUntil = keyHashes.reduce(
(latest, keyHash) =>
Math.max(latest, localLoginThrottle.get(keyHash)?.blockedUntil ?? 0),
const blockedUntil = buckets.reduce(
(latest, bucket) =>
Math.max(latest, localLoginThrottle.get(bucket.keyHash)?.blockedUntil ?? 0),
0,
);
if (blockedUntil > now) {
@@ -174,16 +231,16 @@ function consumeLocalLoginAttempt(
};
}
for (const keyHash of keyHashes) {
const previous = localLoginThrottle.get(keyHash);
for (const bucket of buckets) {
const previous = localLoginThrottle.get(bucket.keyHash);
const windowExpired =
!previous || now - previous.windowStartedAt >= LOGIN_WINDOW_MS;
const failedAttempts = windowExpired ? 1 : previous.failedAttempts + 1;
localLoginThrottle.set(keyHash, {
localLoginThrottle.set(bucket.keyHash, {
windowStartedAt: windowExpired ? now : previous.windowStartedAt,
failedAttempts,
blockedUntil:
failedAttempts >= LOGIN_ATTEMPT_LIMIT ? now + LOGIN_BLOCK_MS : 0,
failedAttempts >= bucket.limit ? now + LOGIN_BLOCK_MS : 0,
updatedAt: now,
});
}
@@ -192,12 +249,13 @@ function consumeLocalLoginAttempt(
}
async function consumeDatabaseLoginAttempt(
keyHashes: string[],
buckets: LoginThrottleBucket[],
now: number,
): Promise<LoginThrottleDecision> {
const pool = getPool();
if (!pool) return consumeLocalLoginAttempt(keyHashes, now);
if (!pool) return consumeLocalLoginAttempt(buckets, now);
await ensureAdminTable();
const keyHashes = buckets.map((b) => b.keyHash);
const client = await pool.connect();
try {
@@ -229,8 +287,8 @@ async function consumeDatabaseLoginAttempt(
};
}
for (const keyHash of keyHashes) {
const previous = entries.get(keyHash);
for (const bucket of buckets) {
const previous = entries.get(bucket.keyHash);
const windowStartedAt = previous
? new Date(previous.window_started_at as string | Date).getTime()
: 0;
@@ -243,7 +301,7 @@ async function consumeDatabaseLoginAttempt(
? new Date(now)
: new Date(windowStartedAt);
const nextBlockedUntil =
failedAttempts >= LOGIN_ATTEMPT_LIMIT
failedAttempts >= bucket.limit
? new Date(now + LOGIN_BLOCK_MS)
: null;
@@ -256,7 +314,7 @@ async function consumeDatabaseLoginAttempt(
failed_attempts = EXCLUDED.failed_attempts,
blocked_until = EXCLUDED.blocked_until,
updated_at = NOW()`,
[keyHash, nextWindowStartedAt, failedAttempts, nextBlockedUntil],
[bucket.keyHash, nextWindowStartedAt, failedAttempts, nextBlockedUntil],
);
}
@@ -279,13 +337,20 @@ async function consumeDatabaseLoginAttempt(
async function getOrInitInMemoryAdmin(): Promise<InMemoryAdmin> {
if (!globals.__dataStackInMemoryAdmin) {
const envPassword = process.env.DATASTACK_ADMIN_PASSWORD;
const allowDemo =
!isProductionEnv() && process.env.DATASTACK_ALLOW_DEMO_ADMIN !== "false";
if (!envPassword && !allowDemo) {
throw new Error("Admin account has not completed initial setup.");
}
const defaultEmail = (
process.env.DATASTACK_ADMIN_EMAIL || "admin@datastack.dev"
)
.trim()
.toLowerCase();
const defaultPassword =
process.env.DATASTACK_ADMIN_PASSWORD || "admin12345678";
const defaultPassword = envPassword || "admin12345678";
const salt = randomBytes(PASSWORD_SALT_BYTES);
const hash = await derivePasswordHash(defaultPassword, salt);
globals.__dataStackInMemoryAdmin = {
@@ -293,6 +358,7 @@ async function getOrInitInMemoryAdmin(): Promise<InMemoryAdmin> {
email: defaultEmail,
passwordSalt: salt,
passwordHash: hash,
tokenVersion: 1,
createdAt: new Date(),
};
}
@@ -302,24 +368,26 @@ async function getOrInitInMemoryAdmin(): Promise<InMemoryAdmin> {
export async function consumeAdminLoginAttempt(
email: string,
secret: string,
ip = "unknown",
): Promise<LoginThrottleDecision> {
const keyHashes = createLoginThrottleKeys(email, secret);
const buckets = createLoginThrottleBuckets(email, secret, ip);
const now = Date.now();
const pool = getPool();
if (!pool) {
return consumeLocalLoginAttempt(keyHashes, now);
return consumeLocalLoginAttempt(buckets, now);
}
try {
return await consumeDatabaseLoginAttempt(keyHashes, now);
} catch {
return consumeLocalLoginAttempt(keyHashes, now);
}
return await consumeDatabaseLoginAttempt(buckets, now);
}
export async function clearAdminLoginAttempts(email: string, secret: string) {
const keyHashes = createLoginThrottleKeys(email, secret);
export async function clearAdminLoginAttempts(
email: string,
secret: string,
ip = "unknown",
) {
const buckets = createLoginThrottleBuckets(email, secret, ip);
const keyHashes = buckets.map((b) => b.keyHash);
const pool = getPool();
if (!pool) {
@@ -327,50 +395,45 @@ export async function clearAdminLoginAttempts(email: string, secret: string) {
return;
}
await ensureAdminTable();
const client = await pool.connect();
try {
await ensureAdminTable();
const client = await pool.connect();
try {
await client.query("BEGIN");
await lockLoginThrottleKeys(client, keyHashes);
await client.query(
"DELETE FROM admin_login_attempts WHERE key_hash = ANY($1::text[])",
[keyHashes],
);
await client.query("COMMIT");
} catch (error) {
await client.query("ROLLBACK").catch(() => undefined);
throw error;
} finally {
client.release();
}
} catch {
keyHashes.forEach((keyHash) => localLoginThrottle.delete(keyHash));
await client.query("BEGIN");
await lockLoginThrottleKeys(client, keyHashes);
await client.query(
"DELETE FROM admin_login_attempts WHERE key_hash = ANY($1::text[])",
[keyHashes],
);
await client.query("COMMIT");
} catch (error) {
await client.query("ROLLBACK").catch(() => undefined);
throw error;
} finally {
client.release();
}
}
export function verifyAdminSetupKey(candidate: string) {
const expected =
process.env.DATASTACK_ADMIN_SETUP_KEY ||
"datastack-default-setup-key-32-chars";
if (!expected || expected.length < 32) return false;
const expected = getAdminSetupKey();
if (!expected || expected.length < 32 || !candidate) return false;
return matchesSecret(candidate, expected);
}
export async function isAdminSetupComplete() {
const pool = getPool();
if (!pool) {
return true;
}
try {
await ensureAdminTable();
const result = await pool.query(
"SELECT 1 FROM admin_users WHERE id = 1",
);
return result.rowCount === 1;
} catch {
return true;
if (globals.__dataStackInMemoryAdmin) {
return true;
}
if (isProductionEnv()) {
return Boolean(process.env.DATASTACK_ADMIN_PASSWORD);
}
return process.env.DATASTACK_ALLOW_DEMO_ADMIN !== "false";
}
await ensureAdminTable();
const result = await pool.query("SELECT 1 FROM admin_users WHERE id = 1");
return result.rowCount === 1;
}
export async function createInitialAdmin(email: string, password: string) {
@@ -380,136 +443,176 @@ export async function createInitialAdmin(email: string, password: string) {
const pool = getPool();
if (!pool) {
if (globals.__dataStackInMemoryAdmin) {
return false;
}
globals.__dataStackInMemoryAdmin = {
id: 1,
email: cleanEmail,
passwordSalt: salt,
passwordHash,
tokenVersion: 1,
createdAt: new Date(),
};
return true;
}
await ensureAdminTable();
const client = await pool.connect();
try {
await ensureAdminTable();
const client = await pool.connect();
await client.query("BEGIN");
await client.query("SELECT pg_advisory_xact_lock($1::bigint)", [
DATABASE_LOCK_ID,
]);
try {
await client.query("BEGIN");
await client.query("SELECT pg_advisory_xact_lock($1::bigint)", [
DATABASE_LOCK_ID,
]);
const existing = await client.query(
"SELECT 1 FROM admin_users WHERE id = 1",
);
if (existing.rowCount) {
await client.query("ROLLBACK");
return false;
}
await client.query(
`INSERT INTO admin_users (id, email, password_salt, password_hash)
VALUES (1, $1, $2, $3)`,
[cleanEmail, salt, passwordHash],
);
await client.query("COMMIT");
return true;
} catch (error) {
await client.query("ROLLBACK").catch(() => undefined);
throw error;
} finally {
client.release();
const existing = await client.query(
"SELECT 1 FROM admin_users WHERE id = 1",
);
if (existing.rowCount) {
await client.query("ROLLBACK");
return false;
}
} catch {
globals.__dataStackInMemoryAdmin = {
id: 1,
email: cleanEmail,
passwordSalt: salt,
passwordHash,
createdAt: new Date(),
};
await client.query(
`INSERT INTO admin_users (id, email, password_salt, password_hash, token_version)
VALUES (1, $1, $2, $3, 1)`,
[cleanEmail, salt, passwordHash],
);
await client.query("COMMIT");
return true;
} catch (error) {
await client.query("ROLLBACK").catch(() => undefined);
throw error;
} finally {
client.release();
}
}
export async function verifyAdminCredentials(email: string, password: string) {
export async function authenticateAdminCredentials(
email: string,
password: string,
): Promise<{ authenticated: boolean; tokenVersion: number }> {
const cleanEmail = email.trim().toLowerCase();
const pool = getPool();
if (pool) {
try {
await ensureAdminTable();
const result = await pool.query(
`SELECT email, password_salt, password_hash
FROM admin_users
WHERE id = 1`,
);
if (result.rowCount && result.rowCount > 0) {
const admin = result.rows[0] as {
email: string;
password_salt: Buffer;
password_hash: Buffer;
};
const emailMatches = matchesSecret(
cleanEmail,
admin.email.trim().toLowerCase(),
);
const actualHash = await derivePasswordHash(
password,
admin.password_salt,
);
const passwordMatches = timingSafeEqual(
actualHash,
admin.password_hash,
);
return emailMatches && passwordMatches;
}
if (!process.env.DATASTACK_ADMIN_PASSWORD) {
return false;
}
} catch {
// Database connection error, fall back to in-memory admin
await ensureAdminTable();
const result = await pool.query(
`SELECT email, password_salt, password_hash, COALESCE(token_version, 1) AS token_version
FROM admin_users
WHERE id = 1`,
);
if (!result.rowCount || result.rowCount === 0) {
return { authenticated: false, tokenVersion: 0 };
}
const admin = result.rows[0] as {
email: string;
password_salt: Buffer;
password_hash: Buffer;
token_version: number;
};
const emailMatches = matchesSecret(
cleanEmail,
admin.email.trim().toLowerCase(),
);
const actualHash = await derivePasswordHash(password, admin.password_salt);
const passwordMatches = timingSafeEqual(actualHash, admin.password_hash);
return {
authenticated: emailMatches && passwordMatches,
tokenVersion: Number(admin.token_version) || 1,
};
}
const inMem = await getOrInitInMemoryAdmin();
if (!matchesSecret(cleanEmail, inMem.email)) {
return false;
return { authenticated: false, tokenVersion: 0 };
}
const actualHash = await derivePasswordHash(password, inMem.passwordSalt);
return timingSafeEqual(actualHash, inMem.passwordHash);
const authenticated = timingSafeEqual(actualHash, inMem.passwordHash);
return {
authenticated,
tokenVersion: authenticated ? inMem.tokenVersion : 0,
};
}
export async function verifyAdminCredentials(email: string, password: string) {
const result = await authenticateAdminCredentials(email, password);
return result.authenticated;
}
export async function getAdminAccountInfo(fallbackEmail?: string) {
const pool = getPool();
if (pool) {
try {
await ensureAdminTable();
const result = await pool.query(
`SELECT email, created_at FROM admin_users WHERE id = 1`,
);
if (result.rowCount && result.rowCount > 0) {
const row = result.rows[0] as { email: string; created_at: Date | string };
return {
email: row.email,
createdAt: new Date(row.created_at).toISOString(),
storageMode: "PostgreSQL" as const,
};
}
} catch {
// Fall back to in-memory admin
await ensureAdminTable();
const result = await pool.query(
`SELECT email, created_at, COALESCE(token_version, 1) AS token_version
FROM admin_users
WHERE id = 1`,
);
if (result.rowCount && result.rowCount > 0) {
const row = result.rows[0] as {
email: string;
created_at: Date | string;
token_version: number;
};
return {
email: row.email,
createdAt: new Date(row.created_at).toISOString(),
tokenVersion: Number(row.token_version) || 1,
storageMode: "PostgreSQL" as const,
};
}
throw new Error("Admin account is not initialized.");
}
const inMem = await getOrInitInMemoryAdmin();
return {
email: fallbackEmail || inMem.email,
createdAt: inMem.createdAt.toISOString(),
tokenVersion: inMem.tokenVersion,
storageMode: "In-Memory" as const,
};
}
export async function verifyRequestAdminSession(
cookieToken: string | undefined,
): Promise<
| { valid: true; session: AdminSessionPayload; sessionSecret: string }
| { valid: false; reason: "UNCONFIGURED" | "UNAUTHORIZED" | "UNAVAILABLE" }
> {
const sessionSecret = getAdminSessionSecret();
if (!sessionSecret) {
return { valid: false, reason: "UNCONFIGURED" };
}
if (!cookieToken) {
return { valid: false, reason: "UNAUTHORIZED" };
}
const session = await getAdminSessionPayload(cookieToken, sessionSecret);
if (!session) {
return { valid: false, reason: "UNAUTHORIZED" };
}
try {
const account = await getAdminAccountInfo();
const emailMatches = matchesSecret(
session.email.trim().toLowerCase(),
account.email.trim().toLowerCase(),
);
if (!emailMatches || session.tokenVersion !== account.tokenVersion) {
return { valid: false, reason: "UNAUTHORIZED" };
}
return { valid: true, session, sessionSecret };
} catch (error) {
if (getPool()) {
return { valid: false, reason: "UNAVAILABLE" };
}
void error;
return { valid: false, reason: "UNAUTHORIZED" };
}
}
export async function updateAdminAccount({
currentEmail,
currentPassword,
@@ -534,29 +637,42 @@ export async function updateAdminAccount({
const pool = getPool();
if (pool) {
try {
await ensureAdminTable();
const updated = await pool.query(
`UPDATE admin_users
SET email = $1, password_salt = $2, password_hash = $3
WHERE id = 1`,
[cleanEmail, salt, passwordHash],
);
if (updated.rowCount && updated.rowCount > 0) {
return { ok: true as const, email: cleanEmail };
}
} catch {
// Fall back to in-memory update
await ensureAdminTable();
const updated = await pool.query(
`UPDATE admin_users
SET email = $1,
password_salt = $2,
password_hash = $3,
token_version = COALESCE(token_version, 1) + 1
WHERE id = 1
RETURNING token_version`,
[cleanEmail, salt, passwordHash],
);
if (!updated.rowCount || updated.rowCount === 0) {
return { ok: false as const, error: "Admin account not found." };
}
return {
ok: true as const,
email: cleanEmail,
tokenVersion: Number(updated.rows[0].token_version) || 1,
};
}
const existing = await getOrInitInMemoryAdmin();
const nextVersion = existing.tokenVersion + 1;
globals.__dataStackInMemoryAdmin = {
id: 1,
email: cleanEmail,
passwordSalt: salt,
passwordHash,
tokenVersion: nextVersion,
createdAt: existing.createdAt,
};
return { ok: true as const, email: cleanEmail };
return { ok: true as const, email: cleanEmail, tokenVersion: nextVersion };
}
export function resetAdminStateForTests() {
globals.__dataStackInMemoryAdmin = undefined;
localLoginThrottle.clear();
lastLocalThrottleCleanup = 0;
}
+33 -10
View File
@@ -8,16 +8,31 @@ export interface ClientTokenPayload {
exp?: number;
}
const DEFAULT_SECRET =
process.env.DATASTACK_JWT_SECRET ||
const DEV_DEFAULT_JWT_SECRET =
"datastack-jwt-default-secret-key-32-chars-long";
export function resolveDefaultJwtSecret(): string {
const configured = process.env.DATASTACK_JWT_SECRET?.trim() || "";
if (process.env.NODE_ENV === "production") {
return configured.length >= 32 ? configured : "";
}
if (configured.length > 0) {
return configured.length >= 32 ? configured : "";
}
return DEV_DEFAULT_JWT_SECRET;
}
export class AuthService {
private static secret: string = DEFAULT_SECRET;
private static secret: string = resolveDefaultJwtSecret();
public static setSecret(newSecret: string) {
if (newSecret && newSecret.trim().length > 0) {
this.secret = newSecret.trim();
const trimmed = newSecret ? newSecret.trim() : "";
if (trimmed.length >= 32) {
this.secret = trimmed;
} else if (process.env.NODE_ENV !== "production" && trimmed.length > 0) {
this.secret = trimmed;
} else if (process.env.NODE_ENV === "production") {
this.secret = "";
}
}
@@ -25,6 +40,11 @@ export class AuthService {
return this.secret;
}
public static isConfigured(): boolean {
const minLen = process.env.NODE_ENV === "production" ? 32 : 1;
return this.secret.length >= minLen;
}
/**
* Generates a signed JWT for client SDK authentication.
*/
@@ -32,9 +52,9 @@ export class AuthService {
payload: { clientId: string; name?: string; rateLimit?: number },
expiresIn = "30d",
): string {
if (!this.secret) {
if (!this.isConfigured()) {
throw new Error(
"DATASTACK_JWT_SECRET must be configured before generating client tokens.",
"DATASTACK_JWT_SECRET (>= 32 characters) must be configured before generating client tokens.",
);
}
@@ -50,15 +70,18 @@ export class AuthService {
}
/**
* Verifies and decodes incoming token from Socket.IO handshake.
* Verifies and decodes incoming token from Socket.IO handshake or HTTP ingestion.
*/
public static verifyClientToken(token: string): {
valid: boolean;
payload?: ClientTokenPayload;
error?: string;
} {
if (!this.secret) {
return { valid: false, error: "DATASTACK_JWT_SECRET is not configured." };
if (!this.isConfigured()) {
return {
valid: false,
error: "DATASTACK_JWT_SECRET (>= 32 characters) is not configured.",
};
}
try {
+380 -28
View File
@@ -1,8 +1,37 @@
import crypto from 'node:crypto';
import Redis from 'ioredis';
import { CSVStorageManager, DataEvent } from '../Storage/csvStorage';
export type EventSubscriber = (event: DataEvent) => void;
export type FlushCallback = (count: number, files: string[]) => void;
export interface RedisBufferClient {
status?: string;
rpush(key: string, ...values: string[]): Promise<number>;
lpush(key: string, ...values: string[]): Promise<number>;
ltrim(key: string, start: number, stop: number): Promise<unknown>;
lrange(key: string, start: number, stop: number): Promise<string[]>;
llen(key: string): Promise<number>;
set(
key: string,
value: string,
pxMode: 'PX',
ttlMs: number,
nxMode: 'NX',
): Promise<string | null>;
eval(script: string, numKeys: number, ...args: string[]): Promise<unknown>;
quit?: () => Promise<unknown>;
disconnect?: () => void;
}
export const REDIS_BUFFER_KEY = 'datastack:buffer:events';
export const REDIS_RECENT_KEY = 'datastack:recent:events';
export const REDIS_FLUSH_LOCK_KEY = 'datastack:flush:lock';
const REDIS_RECENT_LIMIT = 500;
const FLUSH_LOCK_TTL_MS = 15000;
const UNLOCK_LUA_SCRIPT =
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end';
interface BufferGlobalState {
buffer: DataEvent[];
recentEvents: DataEvent[];
@@ -14,6 +43,11 @@ interface BufferGlobalState {
totalIngested: number;
totalFlushed: number;
isFlushing: boolean;
redisClient: RedisBufferClient | null | undefined;
redisUrlConfigured: string;
redisConnected: boolean;
redisTestOverride: RedisBufferClient | null | undefined;
pendingRedisWrites: Set<Promise<void>>;
}
type BufferGlobals = typeof globalThis & {
@@ -36,13 +70,86 @@ function getState(): BufferGlobalState {
totalIngested: 0,
totalFlushed: 0,
isFlushing: false,
redisClient: undefined,
redisUrlConfigured: '',
redisConnected: false,
redisTestOverride: undefined,
pendingRedisWrites: new Set(),
};
}
return globals.__dataStackBufferState;
}
function getRedisClient(): RedisBufferClient | null {
const state = getState();
if (state.redisTestOverride !== undefined) {
return state.redisTestOverride;
}
const configuredUrl = (process.env.REDIS_URL || '').trim();
if (!configuredUrl) {
if (state.redisClient && typeof state.redisClient.disconnect === 'function') {
try {
state.redisClient.disconnect();
} catch {
// ignore
}
}
state.redisClient = null;
state.redisUrlConfigured = '';
state.redisConnected = false;
return null;
}
if (state.redisClient && state.redisUrlConfigured === configuredUrl) {
return state.redisClient;
}
if (state.redisClient && typeof state.redisClient.disconnect === 'function') {
try {
state.redisClient.disconnect();
} catch {
// ignore
}
}
try {
const client = new Redis(configuredUrl, {
maxRetriesPerRequest: 2,
enableReadyCheck: true,
retryStrategy: (times: number) => Math.min(times * 250, 3000),
});
client.on('ready', () => {
state.redisConnected = true;
});
client.on('connect', () => {
state.redisConnected = true;
});
client.on('close', () => {
state.redisConnected = false;
});
client.on('end', () => {
state.redisConnected = false;
});
client.on('error', () => {
state.redisConnected = false;
});
state.redisClient = client as unknown as RedisBufferClient;
state.redisUrlConfigured = configuredUrl;
state.redisConnected = client.status === 'ready' || client.status === 'connect';
return state.redisClient;
} catch {
state.redisClient = null;
state.redisUrlConfigured = configuredUrl;
state.redisConnected = false;
return null;
}
}
export class DataBufferEngine {
private static maxRecentEvents = 200;
private static maxRecentEvents = REDIS_RECENT_LIMIT;
public static get batchFlushThreshold(): number {
return getState().batchFlushThreshold;
@@ -60,11 +167,46 @@ export class DataBufferEngine {
this.setFlushInterval(value);
}
public static setRedisClientForTests(client: RedisBufferClient | null | undefined): void {
const state = getState();
state.redisTestOverride = client;
state.redisConnected = Boolean(client);
}
public static async shutdown(): Promise<void> {
const state = getState();
if (state.flushTimer) {
clearInterval(state.flushTimer);
state.flushTimer = null;
}
if (state.pendingRedisWrites.size > 0) {
await Promise.allSettled([...state.pendingRedisWrites]);
}
const client = state.redisClient;
state.redisClient = undefined;
state.redisUrlConfigured = '';
state.redisConnected = false;
state.redisTestOverride = undefined;
if (client) {
try {
if (typeof client.quit === 'function') {
await client.quit();
} else if (typeof client.disconnect === 'function') {
client.disconnect();
}
} catch {
if (typeof client.disconnect === 'function') {
client.disconnect();
}
}
}
}
public static init() {
const state = getState();
if (!state.flushTimer) {
state.flushTimer = setInterval(() => {
void this.flush();
void this.flush().catch(() => undefined);
}, state.batchFlushIntervalMs);
}
@@ -96,26 +238,18 @@ export class DataBufferEngine {
}
state.flushTimer = setInterval(() => {
void this.flush();
void this.flush().catch(() => undefined);
}, state.batchFlushIntervalMs);
}
/**
* Push incoming event to in-memory buffer.
* If buffer hits threshold (e.g. 500 rows), triggers immediate batch flush.
*/
public static push(event: DataEvent): void {
private static recordLocalMetaAndNotify(event: DataEvent): void {
const state = getState();
state.buffer.push(event);
state.totalIngested += 1;
// Cache in recent events for live dashboard viewer
state.recentEvents.unshift(event);
if (state.recentEvents.length > this.maxRecentEvents) {
state.recentEvents.pop();
}
// Notify live subscribers
for (const sub of state.subscribers) {
try {
sub(event);
@@ -123,43 +257,188 @@ export class DataBufferEngine {
// Ignore subscriber error
}
}
}
// Check if buffer threshold reached
private static async writeEventsToRedisOrFallback(events: DataEvent[]): Promise<number> {
if (events.length === 0) return 0;
const state = getState();
const redis = getRedisClient();
if (redis) {
const serialized = events.map((e) => JSON.stringify(e));
try {
const bufferLen = await redis.rpush(REDIS_BUFFER_KEY, ...serialized);
await redis.lpush(REDIS_RECENT_KEY, ...serialized);
await redis.ltrim(REDIS_RECENT_KEY, 0, REDIS_RECENT_LIMIT - 1);
state.redisConnected = true;
return Number(bufferLen) + state.buffer.length;
} catch {
state.redisConnected = false;
}
}
state.buffer.push(...events);
return state.buffer.length;
}
/**
* Synchronous-signature push (queues to Redis asynchronously when configured, with automatic RAM fallback).
*/
public static push(event: DataEvent): void {
const state = getState();
this.recordLocalMetaAndNotify(event);
const redis = getRedisClient();
if (redis) {
const op = this.writeEventsToRedisOrFallback([event]).then((queuedCount) => {
if (queuedCount >= state.batchFlushThreshold) {
void this.flush().catch(() => undefined);
}
});
state.pendingRedisWrites.add(op);
void op.finally(() => {
state.pendingRedisWrites.delete(op);
});
return;
}
state.buffer.push(event);
if (state.buffer.length >= state.batchFlushThreshold) {
void this.flush();
void this.flush().catch(() => undefined);
}
}
/**
* Flushes current buffer to CSV files via CSVStorageManager
* Async single-event push (awaited by API routes so Redis persistence is guaranteed before HTTP response).
*/
public static async pushAsync(event: DataEvent): Promise<void> {
await this.pushBatchAsync([event]);
}
/**
* Async batch push with shared Redis durability and RAM fallback.
*/
public static async pushBatchAsync(events: DataEvent[]): Promise<void> {
if (!events || events.length === 0) return;
const state = getState();
for (const evt of events) {
this.recordLocalMetaAndNotify(evt);
}
const queuedCount = await this.writeEventsToRedisOrFallback(events);
if (queuedCount >= state.batchFlushThreshold) {
void this.flush().catch(() => undefined);
}
}
/**
* Flushes queued events (shared Redis list under distributed lock + local RAM fallback) to partitioned CSV files.
* Redis LTRIM runs ONLY after CSVStorageManager.appendBatch succeeds so crashes never lose data.
*/
public static async flush(): Promise<{ count: number; files: string[] }> {
const state = getState();
if (state.buffer.length === 0 || state.isFlushing) {
if (state.pendingRedisWrites.size > 0) {
await Promise.allSettled([...state.pendingRedisWrites]);
}
if (state.isFlushing) {
return { count: 0, files: [] };
}
state.isFlushing = true;
const batchToFlush = [...state.buffer];
state.buffer = [];
let totalWritten = 0;
const filesSet = new Set<string>();
try {
const result = await CSVStorageManager.appendBatch(batchToFlush);
state.totalFlushed += result.written;
const redis = getRedisClient();
if (redis) {
const lockToken = crypto.randomUUID();
let lockAcquired = false;
for (const listener of state.flushListeners) {
try {
listener(result.written, result.filesUpdated);
const lockResult = await redis.set(
REDIS_FLUSH_LOCK_KEY,
lockToken,
'PX',
FLUSH_LOCK_TTL_MS,
'NX',
);
lockAcquired = lockResult === 'OK';
state.redisConnected = true;
} catch {
// ignore
state.redisConnected = false;
lockAcquired = false;
}
if (lockAcquired) {
try {
const chunkSize = Math.max(500, state.batchFlushThreshold);
while (true) {
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, chunkSize - 1);
if (!rawItems || rawItems.length === 0) {
break;
}
const parsedBatch: DataEvent[] = [];
for (const raw of rawItems) {
try {
const evt = JSON.parse(raw) as DataEvent;
if (evt && evt.eventId) {
parsedBatch.push(evt);
}
} catch {
// Ignore malformed legacy entry while still trimming it
}
}
if (parsedBatch.length > 0) {
const res = await CSVStorageManager.appendBatch(parsedBatch);
totalWritten += res.written;
for (const f of res.filesUpdated) filesSet.add(f);
}
// Trim strictly AFTER CSVStorageManager.appendBatch resolves safely
await redis.ltrim(REDIS_BUFFER_KEY, rawItems.length, -1);
if (rawItems.length < chunkSize) {
break;
}
}
} finally {
try {
await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
} catch {
// Lock auto-expires after FLUSH_LOCK_TTL_MS
}
}
}
}
return { count: result.written, files: result.filesUpdated };
} catch (err) {
// On failure, re-queue to buffer front for retry
state.buffer.unshift(...batchToFlush);
throw err;
if (state.buffer.length > 0) {
const ramBatch = [...state.buffer];
state.buffer = [];
try {
const res = await CSVStorageManager.appendBatch(ramBatch);
totalWritten += res.written;
for (const f of res.filesUpdated) filesSet.add(f);
} catch (err) {
state.buffer.unshift(...ramBatch);
throw err;
}
}
if (totalWritten > 0) {
state.totalFlushed += totalWritten;
const filesUpdated = Array.from(filesSet);
for (const listener of state.flushListeners) {
try {
listener(totalWritten, filesUpdated);
} catch {
// ignore
}
}
}
return { count: totalWritten, files: Array.from(filesSet) };
} finally {
state.isFlushing = false;
}
@@ -194,10 +473,83 @@ export class DataBufferEngine {
return state.recentEvents.slice(0, limit);
}
public static async getRecentEventsAsync(
limit = 50,
filterClientId?: string,
): Promise<DataEvent[]> {
const state = getState();
const redis = getRedisClient();
if (redis) {
try {
const readCap = Math.min(REDIS_RECENT_LIMIT, Math.max(limit * 4, limit));
const rawItems = await redis.lrange(REDIS_RECENT_KEY, 0, readCap - 1);
state.redisConnected = true;
const redisEvents: DataEvent[] = [];
for (const raw of rawItems) {
try {
const parsed = JSON.parse(raw) as DataEvent;
if (!parsed || !parsed.eventId) continue;
if (filterClientId && filterClientId !== 'all' && parsed.clientId !== filterClientId) {
continue;
}
redisEvents.push(parsed);
if (redisEvents.length >= limit) break;
} catch {
// skip malformed JSON entry
}
}
if (redisEvents.length > 0 || state.recentEvents.length === 0) {
return redisEvents;
}
} catch {
state.redisConnected = false;
}
}
return this.getRecentEvents(limit, filterClientId);
}
public static getBufferStats() {
const state = getState();
return {
currentBufferSize: state.buffer.length,
localBufferSize: state.buffer.length,
redisBufferSize: 0,
redisEnabled: Boolean(getRedisClient()),
redisConnected: state.redisConnected,
batchFlushThreshold: state.batchFlushThreshold,
batchFlushIntervalMs: state.batchFlushIntervalMs,
totalIngested: state.totalIngested,
totalFlushed: state.totalFlushed,
recentEventsCached: state.recentEvents.length,
activeSubscribers: state.subscribers.size,
isFlushing: state.isFlushing,
};
}
public static async getBufferStatsAsync() {
const state = getState();
const redis = getRedisClient();
let redisBufferSize = 0;
if (redis) {
try {
redisBufferSize = Number(await redis.llen(REDIS_BUFFER_KEY)) || 0;
state.redisConnected = true;
} catch {
state.redisConnected = false;
redisBufferSize = 0;
}
}
return {
currentBufferSize: state.buffer.length + redisBufferSize,
localBufferSize: state.buffer.length,
redisBufferSize,
redisEnabled: Boolean(redis),
redisConnected: state.redisConnected,
batchFlushThreshold: state.batchFlushThreshold,
batchFlushIntervalMs: state.batchFlushIntervalMs,
totalIngested: state.totalIngested,
+315 -47
View File
@@ -1,5 +1,8 @@
import { existsSync, rmSync } from "node:fs";
import { resolve, sep } from "node:path";
import { AuthService } from "../auth/jwt";
import { DatabaseEngine } from "../Database/database";
import { getInternalIngestionHeaders } from "../Security/security";
export interface ClientSDK {
clientId: string;
@@ -15,7 +18,9 @@ export interface ClientSDK {
type SDKDatabaseGlobals = typeof globalThis & {
__dataStackClientsMap?: Map<string, ClientSDK>;
__dataStackDeletedClients?: Set<string>;
__dataStackClientsHydrated?: boolean;
__dataStackSyncedSocketEvents?: Map<string, number>;
};
const globals = globalThis as SDKDatabaseGlobals;
@@ -27,6 +32,67 @@ function getClientsMap(): Map<string, ClientSDK> {
return globals.__dataStackClientsMap;
}
function getDeletedClientsSet(): Set<string> {
if (!globals.__dataStackDeletedClients) {
globals.__dataStackDeletedClients = new Set<string>();
}
return globals.__dataStackDeletedClients;
}
function getSyncedSocketEventsMap(): Map<string, number> {
if (!globals.__dataStackSyncedSocketEvents) {
globals.__dataStackSyncedSocketEvents = new Map<string, number>();
}
return globals.__dataStackSyncedSocketEvents;
}
function serializeSyncClient(client: ClientSDK) {
return {
clientId: client.clientId,
name: client.name,
status: client.status,
token: client.token,
rateLimit: client.rateLimit,
};
}
async function notifyIngestionClientSyncAsync(
clientId: string,
action: "deleted" | "revoked" | "updated" | "registered",
client?: ClientSDK,
): Promise<void> {
const headers = getInternalIngestionHeaders();
if (!headers["x-datastack-internal-secret"]) return;
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, "");
try {
await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
method: "POST",
headers: {
"Content-Type": "application/json",
...headers,
},
body: JSON.stringify({
clientId,
action,
...(client ? { client: serializeSyncClient(client) } : {}),
}),
signal: AbortSignal.timeout(1000),
});
} catch {
// Port 4000 may not be running in isolated single-process dev mode
}
}
function notifyIngestionClientSync(
clientId: string,
action: "deleted" | "revoked" | "updated" | "registered",
client?: ClientSDK,
): void {
void notifyIngestionClientSyncAsync(clientId, action, client);
}
function mapRowToClient(row: Record<string, unknown>): ClientSDK {
return {
clientId: String(row.client_id),
@@ -54,11 +120,20 @@ export class SDKManager {
const ready = await DatabaseEngine.ensureSchema();
if (!ready) return null;
const result = await pool.query(
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
FROM client_sdks
ORDER BY created_at DESC`,
);
const [result, deletedResult] = await Promise.all([
pool.query(
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
FROM client_sdks
ORDER BY created_at DESC`,
),
pool.query(`SELECT client_id FROM deleted_clients`),
]);
const deletedSet = getDeletedClientsSet();
deletedSet.clear();
for (const row of deletedResult.rows) {
deletedSet.add(String(row.client_id).trim().toLowerCase());
}
const map = getClientsMap();
map.clear();
@@ -84,12 +159,91 @@ export class SDKManager {
return Array.from(getClientsMap().values());
}
public static async syncAllClientsToIngestionAsync(): Promise<void> {
const headers = getInternalIngestionHeaders();
if (!headers["x-datastack-internal-secret"]) return;
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, "");
const clients = Array.from(getClientsMap().values()).map(serializeSyncClient);
try {
await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
method: "POST",
headers: {
"Content-Type": "application/json",
...headers,
},
body: JSON.stringify({
action: "bulk_sync",
clients,
}),
signal: AbortSignal.timeout(1500),
});
} catch {
// Port 4000 may not be running in isolated single-process dev mode
}
}
public static async listClientsAsync(): Promise<ClientSDK[]> {
const fromDb = await this.syncFromDatabase();
if (fromDb !== null) {
return fromDb;
}
return Array.from(getClientsMap().values());
const map = getClientsMap();
const headers = getInternalIngestionHeaders();
if (headers["x-datastack-internal-secret"]) {
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, "");
try {
const res = await fetch(`${ingestionBaseUrl}/internal/metrics`, {
headers,
cache: "no-store",
signal: AbortSignal.timeout(1000),
});
if (res.ok) {
const data = (await res.json()) as {
hasSyncedInMemoryClients?: boolean;
clientActivity?: Record<
string,
{ totalEvents?: number; lastActive?: string }
>;
};
if (data.hasSyncedInMemoryClients === false && map.size > 0) {
await this.syncAllClientsToIngestionAsync();
}
if (data.clientActivity && typeof data.clientActivity === "object") {
const syncedMap = getSyncedSocketEventsMap();
for (const [cid, activity] of Object.entries(data.clientActivity)) {
const client = map.get(cid);
if (!client) continue;
const remoteTotal = Math.max(0, Number(activity?.totalEvents) || 0);
const prevSynced = syncedMap.get(cid) || 0;
const delta =
remoteTotal >= prevSynced
? remoteTotal - prevSynced
: remoteTotal;
if (delta > 0) {
client.totalEvents += delta;
}
syncedMap.set(cid, remoteTotal);
if (
activity?.lastActive &&
(!client.lastActive || activity.lastActive > client.lastActive)
) {
client.lastActive = activity.lastActive;
}
map.set(cid, client);
}
}
}
} catch {
// Port 4000 may not be running in isolated single-process dev mode
}
}
return Array.from(map.values());
}
public static getClient(clientId: string): ClientSDK | undefined {
@@ -152,8 +306,10 @@ export class SDKManager {
createdAt: new Date().toISOString(),
};
getDeletedClientsSet().delete(safeClientId);
getClientsMap().set(safeClientId, newClient);
void this.persistNewClient(newClient);
notifyIngestionClientSync(safeClientId, "registered", newClient);
return newClient;
}
@@ -163,6 +319,10 @@ export class SDKManager {
try {
await DatabaseEngine.ensureSchema();
await pool.query(
`DELETE FROM deleted_clients WHERE client_id = $1`,
[client.clientId],
);
await pool.query(
`INSERT INTO client_sdks (client_id, name, secret, token, rate_limit, status, total_events, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW())
@@ -218,8 +378,10 @@ export class SDKManager {
createdAt: new Date().toISOString(),
};
getDeletedClientsSet().delete(safeClientId);
getClientsMap().set(safeClientId, newClient);
await this.persistNewClient(newClient);
await notifyIngestionClientSyncAsync(safeClientId, "registered", newClient);
return newClient;
}
@@ -227,16 +389,20 @@ export class SDKManager {
const client = getClientsMap().get(clientId);
if (!client) return null;
client.secret = `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`;
client.token = AuthService.generateClientToken({
clientId: client.clientId,
name: client.name,
rateLimit: client.rateLimit,
});
const updated: ClientSDK = {
...client,
secret: `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`,
token: AuthService.generateClientToken({
clientId: client.clientId,
name: client.name,
rateLimit: client.rateLimit,
}),
};
getClientsMap().set(clientId, client);
void this.persistClientCredentials(client);
return client;
getClientsMap().set(clientId, updated);
void this.persistClientCredentials(updated);
notifyIngestionClientSync(clientId, "updated", updated);
return updated;
}
private static async persistClientCredentials(
@@ -264,16 +430,20 @@ export class SDKManager {
const existing = await this.getClientAsync(clientId);
if (!existing) return null;
existing.secret = `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`;
existing.token = AuthService.generateClientToken({
clientId: existing.clientId,
name: existing.name,
rateLimit: existing.rateLimit,
});
const updated: ClientSDK = {
...existing,
secret: `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`,
token: AuthService.generateClientToken({
clientId: existing.clientId,
name: existing.name,
rateLimit: existing.rateLimit,
}),
};
getClientsMap().set(clientId, existing);
await this.persistClientCredentials(existing);
return existing;
getClientsMap().set(clientId, updated);
await this.persistClientCredentials(updated);
await notifyIngestionClientSyncAsync(clientId, "updated", updated);
return updated;
}
public static updateClient(
@@ -283,14 +453,23 @@ export class SDKManager {
const client = getClientsMap().get(clientId);
if (!client) return null;
if (updates.name !== undefined) client.name = updates.name.trim();
if (updates.rateLimit !== undefined)
client.rateLimit = Math.max(1, Number(updates.rateLimit) || 100);
if (updates.status !== undefined) client.status = updates.status;
const updated: ClientSDK = {
...client,
...(updates.name !== undefined ? { name: updates.name.trim() } : {}),
...(updates.rateLimit !== undefined
? { rateLimit: Math.max(1, Number(updates.rateLimit) || 100) }
: {}),
...(updates.status !== undefined ? { status: updates.status } : {}),
};
getClientsMap().set(clientId, client);
void this.persistClientUpdates(client);
return client;
getClientsMap().set(clientId, updated);
void this.persistClientUpdates(updated);
notifyIngestionClientSync(
clientId,
updated.status === "revoked" ? "revoked" : "updated",
updated,
);
return updated;
}
private static async persistClientUpdates(client: ClientSDK): Promise<void> {
@@ -317,19 +496,67 @@ export class SDKManager {
const client = await this.getClientAsync(clientId);
if (!client) return null;
if (updates.name !== undefined) client.name = updates.name.trim();
if (updates.rateLimit !== undefined)
client.rateLimit = Math.max(1, Number(updates.rateLimit) || 100);
if (updates.status !== undefined) client.status = updates.status;
const updated: ClientSDK = {
...client,
...(updates.name !== undefined ? { name: updates.name.trim() } : {}),
...(updates.rateLimit !== undefined
? { rateLimit: Math.max(1, Number(updates.rateLimit) || 100) }
: {}),
...(updates.status !== undefined ? { status: updates.status } : {}),
};
getClientsMap().set(clientId, client);
await this.persistClientUpdates(client);
return client;
getClientsMap().set(clientId, updated);
await this.persistClientUpdates(updated);
await notifyIngestionClientSyncAsync(
clientId,
updated.status === "revoked" ? "revoked" : "updated",
updated,
);
return updated;
}
public static isClientDeleted(clientId: string): boolean {
const normalized = String(clientId || "")
.trim()
.toLowerCase();
return getDeletedClientsSet().has(normalized);
}
public static async isClientDeletedAsync(clientId: string): Promise<boolean> {
const normalized = String(clientId || "")
.trim()
.toLowerCase();
if (!normalized) return false;
if (getDeletedClientsSet().has(normalized)) {
return true;
}
const pool = DatabaseEngine.getPool();
if (!pool) return false;
await DatabaseEngine.ensureSchema();
const res = await pool.query(
`SELECT 1 FROM deleted_clients WHERE client_id = $1`,
[normalized],
);
if (res.rowCount && res.rowCount > 0) {
getDeletedClientsSet().add(normalized);
return true;
}
return false;
}
public static deleteClient(clientId: string): boolean {
const deleted = getClientsMap().delete(clientId);
void this.deleteClientFromDb(clientId);
const normalized = String(clientId || "")
.trim()
.toLowerCase();
const deleted = getClientsMap().delete(normalized);
getSyncedSocketEventsMap().delete(normalized);
if (deleted) {
getDeletedClientsSet().add(normalized);
}
void this.deleteClientFromDb(normalized);
notifyIngestionClientSync(normalized, "deleted");
return deleted;
}
@@ -337,21 +564,62 @@ export class SDKManager {
const pool = DatabaseEngine.getPool();
if (!pool) return false;
await DatabaseEngine.ensureSchema();
const dbClient = await pool.connect();
try {
await DatabaseEngine.ensureSchema();
const res = await pool.query(
await dbClient.query("BEGIN");
const res = await dbClient.query(
`DELETE FROM client_sdks WHERE client_id = $1`,
[clientId],
);
return Boolean(res.rowCount && res.rowCount > 0);
} catch {
if (res.rowCount && res.rowCount > 0) {
await dbClient.query(
`INSERT INTO deleted_clients (client_id, deleted_at)
VALUES ($1, NOW())
ON CONFLICT (client_id) DO UPDATE SET deleted_at = NOW()`,
[clientId],
);
await dbClient.query("COMMIT");
return true;
}
await dbClient.query("ROLLBACK");
return false;
} catch {
await dbClient.query("ROLLBACK").catch(() => undefined);
return false;
} finally {
dbClient.release();
}
}
public static async deleteClientAsync(clientId: string): Promise<boolean> {
const existedInMem = getClientsMap().delete(clientId);
const deletedFromDb = await this.deleteClientFromDb(clientId);
private static purgeClientStorageDir(clientId: string): void {
const safeId = clientId.replace(/[^a-z0-9_-]/gi, "_");
if (!safeId) return;
const storageRoot = resolve(process.cwd(), "storage", "data");
const targetDir = resolve(storageRoot, safeId);
if (!targetDir.startsWith(`${storageRoot}${sep}`)) return;
if (existsSync(targetDir)) {
rmSync(targetDir, { recursive: true, force: true });
}
}
public static async deleteClientAsync(
clientId: string,
options?: { purgeFiles?: boolean },
): Promise<boolean> {
const normalized = String(clientId || "")
.trim()
.toLowerCase();
const existedInMem = getClientsMap().delete(normalized);
getSyncedSocketEventsMap().delete(normalized);
const deletedFromDb = await this.deleteClientFromDb(normalized);
if (existedInMem || deletedFromDb) {
getDeletedClientsSet().add(normalized);
if (options?.purgeFiles) {
this.purgeClientStorageDir(normalized);
}
await notifyIngestionClientSyncAsync(normalized, "deleted");
}
return existedInMem || deletedFromDb;
}