refactor(auth): standardize session management and security
- Centralize secret retrieval using `getAdminSessionSecret` - Replace manual session parsing with `verifyRequestAdminSession` utility - Tighten API validation and improve security headers handling - Enforce environment variable security for production deployments
This commit is contained in:
1 parent
34519c07df
commit
177ed4a844
78 files changed
+6748
-3157
No files matched your search
@@ -73,6 +73,31 @@ export class DatabaseEngine {
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`),
|
||||
pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS deleted_clients (
|
||||
client_id TEXT PRIMARY KEY,
|
||||
deleted_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`),
|
||||
pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS admin_users (
|
||||
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
|
||||
email TEXT NOT NULL UNIQUE,
|
||||
password_salt BYTEA NOT NULL,
|
||||
password_hash BYTEA NOT NULL,
|
||||
token_version INTEGER NOT NULL DEFAULT 1,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`),
|
||||
pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS admin_login_attempts (
|
||||
key_hash TEXT PRIMARY KEY,
|
||||
window_started_at TIMESTAMPTZ NOT NULL,
|
||||
failed_attempts INTEGER NOT NULL,
|
||||
blocked_until TIMESTAMPTZ,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`),
|
||||
]).then(() => undefined);
|
||||
|
||||
try {
|
||||
|
||||
@@ -1,3 +1,38 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { AuthService } from "../auth/jwt";
|
||||
|
||||
export function resolveInternalIngestionSecret(
|
||||
jwtSecretOverride?: string,
|
||||
): string {
|
||||
const explicit = (process.env.DATASTACK_INTERNAL_SECRET || "").trim();
|
||||
const isProd = process.env.NODE_ENV === "production";
|
||||
if (isProd) {
|
||||
if (explicit.length >= 32) return explicit;
|
||||
if (explicit.length > 0) return "";
|
||||
} else if (explicit.length > 0) {
|
||||
return explicit;
|
||||
}
|
||||
|
||||
const activeJwt = String(
|
||||
jwtSecretOverride ?? AuthService.getSecret() ?? "",
|
||||
).trim();
|
||||
const minJwtLen = isProd ? 32 : 1;
|
||||
if (activeJwt.length < minJwtLen) {
|
||||
return "";
|
||||
}
|
||||
|
||||
return createHash("sha256")
|
||||
.update(`datastack-internal:${activeJwt}`)
|
||||
.digest("hex");
|
||||
}
|
||||
|
||||
export function getInternalIngestionHeaders(
|
||||
jwtSecretOverride?: string,
|
||||
): Record<string, string> {
|
||||
const secret = resolveInternalIngestionSecret(jwtSecretOverride);
|
||||
return secret ? { "x-datastack-internal-secret": secret } : {};
|
||||
}
|
||||
|
||||
export interface ValidationResult {
|
||||
allowed: boolean;
|
||||
reason?: string;
|
||||
@@ -130,7 +165,7 @@ export class SecurityEngine {
|
||||
if (!withinLimit) {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: `Rate limit of ${clientMaxRate} events/sec exceeded for client ${clientId}`,
|
||||
reason: `Rate limit of ${effectiveRateLimit} events/sec exceeded for client ${clientId}`,
|
||||
code: "RATE_LIMIT_EXCEEDED",
|
||||
};
|
||||
}
|
||||
@@ -144,6 +179,18 @@ export class SecurityEngine {
|
||||
}
|
||||
|
||||
public static getStats() {
|
||||
const now = Date.now();
|
||||
for (const [k, timestamp] of this.idempotencySet.entries()) {
|
||||
if (now - timestamp > this.IDEMPOTENCY_TTL_MS) {
|
||||
this.idempotencySet.delete(k);
|
||||
}
|
||||
}
|
||||
for (const [clientId, entry] of this.rateLimitMap.entries()) {
|
||||
if (now - entry.windowStart > 10000) {
|
||||
this.rateLimitMap.delete(clientId);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
activeIdempotencyKeys: this.idempotencySet.size,
|
||||
rateLimitedClientsCount: this.rateLimitMap.size,
|
||||
|
||||
+392
-53
@@ -19,10 +19,30 @@ export interface StorageFileMeta {
|
||||
modifiedAt: Date;
|
||||
}
|
||||
|
||||
interface RowCountCacheEntry {
|
||||
sizeBytes: number;
|
||||
mtimeMs: number;
|
||||
linesCount: number;
|
||||
}
|
||||
|
||||
const LOCK_RETRY_MIN_MS = 5;
|
||||
const LOCK_RETRY_MAX_MS = 18;
|
||||
const LOCK_TIMEOUT_MS = 10_000;
|
||||
const STALE_LOCK_AGE_MS = 30_000;
|
||||
|
||||
function sleepMs(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
export class CSVStorageManager {
|
||||
// Base storage directory: /storage/data
|
||||
public static baseStorageDir = path.resolve(process.cwd(), 'storage', 'data');
|
||||
public static maxFileSizeBytes = 100 * 1024 * 1024; // 100MB rotation limit
|
||||
private static rowCountCache = new Map<string, RowCountCacheEntry>();
|
||||
|
||||
public static clearRowCountCache(): void {
|
||||
this.rowCountCache.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensures client directory exists
|
||||
@@ -37,15 +57,194 @@ export class CSVStorageManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves target CSV file path with daily and 100MB rotation rules
|
||||
* Acquires an exclusive cooperative lock file for a client+date partition across Port 3000 & Port 4000
|
||||
*/
|
||||
public static async withPartitionLock<T>(
|
||||
clientId: string,
|
||||
dateStr: string,
|
||||
fn: () => Promise<T> | T,
|
||||
): Promise<T> {
|
||||
const clientDir = this.ensureClientDir(clientId);
|
||||
const date = this.extractSafePartitionDate(dateStr);
|
||||
const lockPath = path.join(clientDir, `.${date}.csv.lock`);
|
||||
const deadline = Date.now() + LOCK_TIMEOUT_MS;
|
||||
|
||||
let acquired = false;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const fd = fs.openSync(lockPath, 'wx');
|
||||
try {
|
||||
fs.writeFileSync(fd, `${process.pid}:${Date.now()}`, 'utf8');
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
acquired = true;
|
||||
break;
|
||||
} catch (err: unknown) {
|
||||
const code = (err as NodeJS.ErrnoException)?.code;
|
||||
if (code !== 'EEXIST') {
|
||||
throw err;
|
||||
}
|
||||
try {
|
||||
const lockStat = fs.statSync(lockPath);
|
||||
if (Date.now() - lockStat.mtimeMs > STALE_LOCK_AGE_MS) {
|
||||
fs.unlinkSync(lockPath);
|
||||
continue;
|
||||
}
|
||||
} catch {
|
||||
// Lock was released concurrently
|
||||
}
|
||||
const jitter =
|
||||
LOCK_RETRY_MIN_MS +
|
||||
Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS));
|
||||
await sleepMs(jitter);
|
||||
}
|
||||
}
|
||||
|
||||
if (!acquired) {
|
||||
throw new Error(`Timed out waiting for CSV partition lock: ${lockPath}`);
|
||||
}
|
||||
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
try {
|
||||
if (fs.existsSync(lockPath)) {
|
||||
fs.unlinkSync(lockPath);
|
||||
}
|
||||
} catch {
|
||||
// Ignore cleanup failure if already unlinked
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes and validates any timestamp or date string into a strict YYYY-MM-DD partition key
|
||||
*/
|
||||
public static extractSafePartitionDate(rawTimestamp?: string): string {
|
||||
const fallback = new Date().toISOString().slice(0, 10);
|
||||
if (!rawTimestamp || typeof rawTimestamp !== 'string') {
|
||||
return fallback;
|
||||
}
|
||||
const trimmed = rawTimestamp.trim();
|
||||
const prefix = trimmed.slice(0, 10);
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(prefix)) {
|
||||
const parsedPrefix = new Date(`${prefix}T00:00:00.000Z`);
|
||||
if (!Number.isNaN(parsedPrefix.getTime())) {
|
||||
return prefix;
|
||||
}
|
||||
}
|
||||
if (/^\d+(\.\d+)?$/.test(trimmed)) {
|
||||
const num = Number(trimmed);
|
||||
const ms = num < 1e11 ? num * 1000 : num;
|
||||
const d = new Date(ms);
|
||||
if (!Number.isNaN(d.getTime())) {
|
||||
return d.toISOString().slice(0, 10);
|
||||
}
|
||||
}
|
||||
const parsed = new Date(trimmed);
|
||||
if (!Number.isNaN(parsed.getTime())) {
|
||||
return parsed.toISOString().slice(0, 10);
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stateful RFC 4180 quote-aware chunked row counter so multi-100MB CSVs never load wholly into RAM
|
||||
*/
|
||||
private static createChunkRowCounter() {
|
||||
let inQuotes = false;
|
||||
let pendingQuote = false;
|
||||
let currentLineHasChars = false;
|
||||
let totalRows = 0;
|
||||
|
||||
return {
|
||||
pushChunk(chunk: Buffer | string) {
|
||||
const str = typeof chunk === 'string' ? chunk : chunk.toString('utf8');
|
||||
for (let i = 0; i < str.length; i += 1) {
|
||||
const ch = str[i];
|
||||
if (pendingQuote) {
|
||||
pendingQuote = false;
|
||||
if (ch === '"') {
|
||||
// Escaped quote "" across chunk boundary or inline
|
||||
continue;
|
||||
}
|
||||
inQuotes = !inQuotes;
|
||||
}
|
||||
|
||||
if (ch === '"') {
|
||||
currentLineHasChars = true;
|
||||
if (inQuotes) {
|
||||
if (i + 1 < str.length) {
|
||||
if (str[i + 1] === '"') {
|
||||
i += 1;
|
||||
} else {
|
||||
inQuotes = false;
|
||||
}
|
||||
} else {
|
||||
pendingQuote = true;
|
||||
}
|
||||
} else {
|
||||
inQuotes = true;
|
||||
}
|
||||
} else if (ch === '\n' && !inQuotes) {
|
||||
if (currentLineHasChars) {
|
||||
totalRows += 1;
|
||||
currentLineHasChars = false;
|
||||
}
|
||||
} else if (ch !== '\r' && ch !== ' ' && ch !== '\t') {
|
||||
currentLineHasChars = true;
|
||||
}
|
||||
}
|
||||
},
|
||||
finish(): number {
|
||||
if (currentLineHasChars) {
|
||||
totalRows += 1;
|
||||
}
|
||||
return Math.max(0, totalRows - 1);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private static countCsvDataRowsChunkedSync(filePath: string): number {
|
||||
const counter = this.createChunkRowCounter();
|
||||
const buf = Buffer.allocUnsafe(64 * 1024);
|
||||
const fd = fs.openSync(filePath, 'r');
|
||||
try {
|
||||
let bytesRead = 0;
|
||||
while ((bytesRead = fs.readSync(fd, buf, 0, buf.length, null)) > 0) {
|
||||
counter.pushChunk(buf.subarray(0, bytesRead));
|
||||
}
|
||||
return counter.finish();
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
}
|
||||
|
||||
private static async countCsvDataRowsStreamAsync(
|
||||
filePath: string,
|
||||
): Promise<number> {
|
||||
const counter = this.createChunkRowCounter();
|
||||
return new Promise<number>((resolve, reject) => {
|
||||
const stream = fs.createReadStream(filePath, {
|
||||
highWaterMark: 64 * 1024,
|
||||
});
|
||||
stream.on('data', (chunk) => counter.pushChunk(chunk));
|
||||
stream.on('end', () => resolve(counter.finish()));
|
||||
stream.on('error', (err) => reject(err));
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves target CSV file path with daily and configured MB rotation rules
|
||||
*/
|
||||
public static getTargetFilePath(clientId: string, dateStr?: string): string {
|
||||
const clientDir = this.ensureClientDir(clientId);
|
||||
const date = dateStr || new Date().toISOString().split('T')[0];
|
||||
const date = this.extractSafePartitionDate(dateStr);
|
||||
let baseFileName = `${date}.csv`;
|
||||
let targetPath = path.join(clientDir, baseFileName);
|
||||
|
||||
// Check for 100MB rotation
|
||||
// Check for file rotation
|
||||
let partIndex = 1;
|
||||
while (fs.existsSync(targetPath)) {
|
||||
try {
|
||||
@@ -66,10 +265,12 @@ export class CSVStorageManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch append events into target CSV file.
|
||||
* Batch append events into target CSV file under cooperative partition lock.
|
||||
* If file doesn't exist, writes header: clientId,eventId,timestamp,payload
|
||||
*/
|
||||
public static async appendBatch(events: DataEvent[]): Promise<{ written: number; filesUpdated: string[] }> {
|
||||
public static async appendBatch(
|
||||
events: DataEvent[],
|
||||
): Promise<{ written: number; filesUpdated: string[] }> {
|
||||
if (!events || events.length === 0) {
|
||||
return { written: 0, filesUpdated: [] };
|
||||
}
|
||||
@@ -77,7 +278,7 @@ export class CSVStorageManager {
|
||||
// Group events by client + date
|
||||
const grouped = new Map<string, DataEvent[]>();
|
||||
for (const evt of events) {
|
||||
const date = (evt.timestamp || new Date().toISOString()).split('T')[0];
|
||||
const date = this.extractSafePartitionDate(evt.timestamp);
|
||||
const key = `${evt.clientId}:::${date}`;
|
||||
if (!grouped.has(key)) {
|
||||
grouped.set(key, []);
|
||||
@@ -90,31 +291,39 @@ export class CSVStorageManager {
|
||||
|
||||
for (const [key, batch] of grouped.entries()) {
|
||||
const [clientId, date] = key.split(':::');
|
||||
const targetPath = this.getTargetFilePath(clientId, date);
|
||||
let hasContent = false;
|
||||
try {
|
||||
hasContent =
|
||||
fs.existsSync(targetPath) && fs.statSync(targetPath).size > 0;
|
||||
} catch {
|
||||
hasContent = false;
|
||||
}
|
||||
const targetPath = await this.withPartitionLock(clientId, date, () => {
|
||||
const resolvedPath = this.getTargetFilePath(clientId, date);
|
||||
let hasContent = false;
|
||||
try {
|
||||
hasContent =
|
||||
fs.existsSync(resolvedPath) && fs.statSync(resolvedPath).size > 0;
|
||||
} catch {
|
||||
hasContent = false;
|
||||
}
|
||||
|
||||
// Format payload as JSON string if not already
|
||||
const cleanBatch = batch.map((item) => ({
|
||||
clientId: item.clientId,
|
||||
eventId: item.eventId,
|
||||
timestamp: item.timestamp,
|
||||
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
|
||||
}));
|
||||
const cleanBatch = batch.map((item) => ({
|
||||
clientId: item.clientId,
|
||||
eventId: item.eventId,
|
||||
timestamp: item.timestamp,
|
||||
payload:
|
||||
typeof item.payload === 'string'
|
||||
? item.payload
|
||||
: JSON.stringify(item.payload),
|
||||
}));
|
||||
|
||||
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
|
||||
const csvData = parser.parse(cleanBatch);
|
||||
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
|
||||
const csvData = parser.parse(cleanBatch);
|
||||
|
||||
if (hasContent) {
|
||||
fs.appendFileSync(targetPath, '\n' + csvData, 'utf8');
|
||||
} else {
|
||||
fs.writeFileSync(targetPath, csvData, 'utf8');
|
||||
}
|
||||
if (hasContent) {
|
||||
fs.appendFileSync(resolvedPath, '\n' + csvData, 'utf8');
|
||||
} else {
|
||||
fs.writeFileSync(resolvedPath, csvData, 'utf8');
|
||||
}
|
||||
|
||||
// Invalidate row cache for modified file immediately
|
||||
this.rowCountCache.delete(resolvedPath);
|
||||
return resolvedPath;
|
||||
});
|
||||
|
||||
filesUpdated.push(targetPath);
|
||||
}
|
||||
@@ -123,14 +332,16 @@ export class CSVStorageManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* List all stored CSV files with sizes and dates
|
||||
* List all stored CSV files with cached row counts (invalidated on mtime/size changes)
|
||||
*/
|
||||
public static listAllFiles(): StorageFileMeta[] {
|
||||
const results: StorageFileMeta[] = [];
|
||||
if (!fs.existsSync(this.baseStorageDir)) {
|
||||
this.rowCountCache.clear();
|
||||
return results;
|
||||
}
|
||||
|
||||
const activePaths = new Set<string>();
|
||||
try {
|
||||
const clientDirs = fs.readdirSync(this.baseStorageDir);
|
||||
for (const clientDir of clientDirs) {
|
||||
@@ -142,15 +353,27 @@ export class CSVStorageManager {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
const fullFilePath = path.join(fullClientPath, file);
|
||||
const stats = fs.statSync(fullFilePath);
|
||||
activePaths.add(fullFilePath);
|
||||
|
||||
// Count lines approximately (or sample)
|
||||
const cached = this.rowCountCache.get(fullFilePath);
|
||||
let linesCount = 0;
|
||||
try {
|
||||
const content = fs.readFileSync(fullFilePath, 'utf8');
|
||||
linesCount = content.split('\n').filter((l) => l.trim().length > 0).length - 1; // subtract header
|
||||
if (linesCount < 0) linesCount = 0;
|
||||
} catch {
|
||||
linesCount = 0;
|
||||
if (
|
||||
cached &&
|
||||
cached.sizeBytes === stats.size &&
|
||||
cached.mtimeMs === stats.mtimeMs
|
||||
) {
|
||||
linesCount = cached.linesCount;
|
||||
} else {
|
||||
try {
|
||||
linesCount = this.countCsvDataRowsChunkedSync(fullFilePath);
|
||||
this.rowCountCache.set(fullFilePath, {
|
||||
sizeBytes: stats.size,
|
||||
mtimeMs: stats.mtimeMs,
|
||||
linesCount,
|
||||
});
|
||||
} catch {
|
||||
linesCount = 0;
|
||||
}
|
||||
}
|
||||
|
||||
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||
@@ -166,6 +389,12 @@ export class CSVStorageManager {
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const key of this.rowCountCache.keys()) {
|
||||
if (!activePaths.has(key)) {
|
||||
this.rowCountCache.delete(key);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Return whatever collected
|
||||
}
|
||||
@@ -174,12 +403,88 @@ export class CSVStorageManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Prune CSV partitions older than autoPruneDays
|
||||
* Non-blocking async listing using fs.promises + streaming row counting on cache misses
|
||||
*/
|
||||
public static async listAllFilesAsync(): Promise<StorageFileMeta[]> {
|
||||
const results: StorageFileMeta[] = [];
|
||||
if (!fs.existsSync(this.baseStorageDir)) {
|
||||
this.rowCountCache.clear();
|
||||
return results;
|
||||
}
|
||||
|
||||
const activePaths = new Set<string>();
|
||||
try {
|
||||
const clientDirs = await fs.promises.readdir(this.baseStorageDir);
|
||||
for (const clientDir of clientDirs) {
|
||||
const fullClientPath = path.join(this.baseStorageDir, clientDir);
|
||||
const dirStat = await fs.promises.stat(fullClientPath).catch(() => null);
|
||||
if (!dirStat || !dirStat.isDirectory()) continue;
|
||||
|
||||
const files = await fs.promises.readdir(fullClientPath);
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
const fullFilePath = path.join(fullClientPath, file);
|
||||
const stats = await fs.promises.stat(fullFilePath).catch(() => null);
|
||||
if (!stats) continue;
|
||||
activePaths.add(fullFilePath);
|
||||
|
||||
const cached = this.rowCountCache.get(fullFilePath);
|
||||
let linesCount = 0;
|
||||
if (
|
||||
cached &&
|
||||
cached.sizeBytes === stats.size &&
|
||||
cached.mtimeMs === stats.mtimeMs
|
||||
) {
|
||||
linesCount = cached.linesCount;
|
||||
} else {
|
||||
try {
|
||||
linesCount = await this.countCsvDataRowsStreamAsync(fullFilePath);
|
||||
this.rowCountCache.set(fullFilePath, {
|
||||
sizeBytes: stats.size,
|
||||
mtimeMs: stats.mtimeMs,
|
||||
linesCount,
|
||||
});
|
||||
} catch {
|
||||
linesCount = 0;
|
||||
}
|
||||
}
|
||||
|
||||
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||
|
||||
results.push({
|
||||
clientId: clientDir,
|
||||
date: datePart,
|
||||
filename: file,
|
||||
filePath: fullFilePath,
|
||||
sizeBytes: stats.size,
|
||||
linesCount,
|
||||
modifiedAt: stats.mtime,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const key of this.rowCountCache.keys()) {
|
||||
if (!activePaths.has(key)) {
|
||||
this.rowCountCache.delete(key);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Return whatever collected
|
||||
}
|
||||
|
||||
return results.sort((a, b) => b.modifiedAt.getTime() - a.modifiedAt.getTime());
|
||||
}
|
||||
|
||||
/**
|
||||
* Prune CSV partitions older than autoPruneDays without counting row contents
|
||||
*/
|
||||
public static pruneExpiredPartitions(autoPruneDays: number): number {
|
||||
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
|
||||
return 0;
|
||||
}
|
||||
if (!fs.existsSync(this.baseStorageDir)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const cutoffDate = new Date(
|
||||
Date.now() - Math.floor(autoPruneDays) * 86_400_000,
|
||||
@@ -188,28 +493,38 @@ export class CSVStorageManager {
|
||||
.split('T')[0];
|
||||
|
||||
let prunedCount = 0;
|
||||
const files = this.listAllFiles();
|
||||
for (const file of files) {
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(file.date) && file.date < cutoffDate) {
|
||||
try {
|
||||
if (fs.existsSync(file.filePath)) {
|
||||
fs.unlinkSync(file.filePath);
|
||||
prunedCount += 1;
|
||||
try {
|
||||
const clientDirs = fs.readdirSync(this.baseStorageDir);
|
||||
for (const clientDir of clientDirs) {
|
||||
const fullClientPath = path.join(this.baseStorageDir, clientDir);
|
||||
if (!fs.statSync(fullClientPath).isDirectory()) continue;
|
||||
|
||||
const files = fs.readdirSync(fullClientPath);
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
|
||||
const fullPath = path.join(fullClientPath, file);
|
||||
try {
|
||||
if (fs.existsSync(fullPath)) {
|
||||
fs.unlinkSync(fullPath);
|
||||
this.rowCountCache.delete(fullPath);
|
||||
prunedCount += 1;
|
||||
}
|
||||
} catch {
|
||||
// Ignore file removal error
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore file removal error
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore directory scan failure
|
||||
}
|
||||
|
||||
return prunedCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get aggregate storage statistics
|
||||
*/
|
||||
public static getStorageStats() {
|
||||
const files = this.listAllFiles();
|
||||
private static formatStorageStats(files: StorageFileMeta[]) {
|
||||
const totalBytes = files.reduce((acc, f) => acc + f.sizeBytes, 0);
|
||||
const totalRows = files.reduce((acc, f) => acc + f.linesCount, 0);
|
||||
const uniqueClients = new Set(files.map((f) => f.clientId)).size;
|
||||
@@ -231,10 +546,34 @@ export class CSVStorageManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads file content for export / download
|
||||
* Get aggregate storage statistics (sync cached)
|
||||
*/
|
||||
public static getStorageStats() {
|
||||
return this.formatStorageStats(this.listAllFiles());
|
||||
}
|
||||
|
||||
/**
|
||||
* Get aggregate storage statistics (non-blocking async)
|
||||
*/
|
||||
public static async getStorageStatsAsync() {
|
||||
const files = await this.listAllFilesAsync();
|
||||
return this.formatStorageStats(files);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads file content for export / download with path traversal protection
|
||||
*/
|
||||
public static getFileContent(clientId: string, filename: string): string | null {
|
||||
const fullPath = path.join(this.baseStorageDir, clientId, filename);
|
||||
const safeClientId = String(clientId || '').replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
const safeFilename = path.basename(String(filename || ''));
|
||||
if (!safeClientId || !safeFilename || !safeFilename.endsWith('.csv')) {
|
||||
return null;
|
||||
}
|
||||
const fullPath = path.resolve(this.baseStorageDir, safeClientId, safeFilename);
|
||||
const resolvedBase = path.resolve(this.baseStorageDir);
|
||||
if (!fullPath.startsWith(resolvedBase + path.sep)) {
|
||||
return null;
|
||||
}
|
||||
if (fs.existsSync(fullPath)) {
|
||||
return fs.readFileSync(fullPath, 'utf8');
|
||||
}
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
import { AuthService } from "../auth/jwt";
|
||||
import { AuthService, resolveDefaultJwtSecret } from "../auth/jwt";
|
||||
import { DataBufferEngine } from "../data/buffer";
|
||||
import { DatabaseEngine } from "../Database/database";
|
||||
import { SDKManager } from "../sdk/sdkManager";
|
||||
import { SecurityEngine } from "../Security/security";
|
||||
import {
|
||||
SecurityEngine,
|
||||
getInternalIngestionHeaders,
|
||||
} from "../Security/security";
|
||||
import { CSVStorageManager } from "../Storage/csvStorage";
|
||||
|
||||
export interface AdminSettings {
|
||||
@@ -35,9 +38,7 @@ function getDefaultSettings(): AdminSettings {
|
||||
batchFlushThreshold: 500,
|
||||
batchFlushIntervalMs: 2000,
|
||||
fileRotationMb: 100,
|
||||
jwtSecret:
|
||||
process.env.DATASTACK_JWT_SECRET ||
|
||||
"datastack-jwt-default-secret-key-32-chars-long",
|
||||
jwtSecret: resolveDefaultJwtSecret(),
|
||||
enableIdempotency: true,
|
||||
autoPruneDays: 30,
|
||||
updatedAt: new Date().toISOString(),
|
||||
@@ -62,7 +63,9 @@ export class SettingsManager {
|
||||
AuthService.setSecret(settings.jwtSecret);
|
||||
}
|
||||
CSVStorageManager.maxFileSizeBytes = settings.fileRotationMb * 1024 * 1024;
|
||||
CSVStorageManager.pruneExpiredPartitions(settings.autoPruneDays);
|
||||
if (settings.autoPruneDays > 0) {
|
||||
CSVStorageManager.pruneExpiredPartitions(settings.autoPruneDays);
|
||||
}
|
||||
}
|
||||
|
||||
private static async syncFromDatabase(): Promise<AdminSettings | null> {
|
||||
@@ -83,6 +86,10 @@ export class SettingsManager {
|
||||
|
||||
if (result.rowCount && result.rowCount > 0) {
|
||||
const row = result.rows[0];
|
||||
const parsedPruneDays =
|
||||
row.auto_prune_days !== undefined && row.auto_prune_days !== null
|
||||
? Number(row.auto_prune_days)
|
||||
: NaN;
|
||||
const loaded: AdminSettings = {
|
||||
serverPort: Number(row.server_port) || 4000,
|
||||
webPort: Number(row.web_port) || 3000,
|
||||
@@ -91,12 +98,9 @@ export class SettingsManager {
|
||||
batchFlushThreshold: Number(row.batch_flush_threshold) || 500,
|
||||
batchFlushIntervalMs: Number(row.batch_flush_interval_ms) || 2000,
|
||||
fileRotationMb: Number(row.file_rotation_mb) || 100,
|
||||
jwtSecret:
|
||||
String(row.jwt_secret || "") ||
|
||||
process.env.DATASTACK_JWT_SECRET ||
|
||||
"datastack-jwt-default-secret-key-32-chars-long",
|
||||
jwtSecret: String(row.jwt_secret || "").trim() || resolveDefaultJwtSecret(),
|
||||
enableIdempotency: Boolean(row.enable_idempotency),
|
||||
autoPruneDays: Number(row.auto_prune_days) || 30,
|
||||
autoPruneDays: Number.isFinite(parsedPruneDays) ? parsedPruneDays : 30,
|
||||
updatedAt: row.updated_at
|
||||
? new Date(row.updated_at as string | Date).toISOString()
|
||||
: new Date().toISOString(),
|
||||
@@ -214,8 +218,46 @@ export class SettingsManager {
|
||||
};
|
||||
}
|
||||
|
||||
private static async notifyIngestionSettingsSync(
|
||||
settings: AdminSettings,
|
||||
authJwtSecret: string,
|
||||
jwtSecretChanged: boolean,
|
||||
): Promise<void> {
|
||||
const headers = getInternalIngestionHeaders(authJwtSecret);
|
||||
if (!headers["x-datastack-internal-secret"]) return;
|
||||
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||
).replace(/\/$/, "");
|
||||
|
||||
try {
|
||||
await fetch(`${ingestionBaseUrl}/internal/settings-sync`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...headers,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
maxEventsPerSec: settings.maxEventsPerSec,
|
||||
maxPayloadSizeKb: settings.maxPayloadSizeKb,
|
||||
batchFlushThreshold: settings.batchFlushThreshold,
|
||||
batchFlushIntervalMs: settings.batchFlushIntervalMs,
|
||||
fileRotationMb: settings.fileRotationMb,
|
||||
enableIdempotency: settings.enableIdempotency,
|
||||
autoPruneDays: settings.autoPruneDays,
|
||||
...(jwtSecretChanged ? { jwtSecret: settings.jwtSecret } : {}),
|
||||
}),
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(1500),
|
||||
});
|
||||
} catch {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
}
|
||||
}
|
||||
|
||||
public static updateSettings(updates: Partial<AdminSettings>): AdminSettings {
|
||||
const current = getSettingsState();
|
||||
const previousJwtSecret = current.jwtSecret;
|
||||
const jwtSecretChanged =
|
||||
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
|
||||
const nextSettings: AdminSettings = {
|
||||
@@ -229,6 +271,13 @@ export class SettingsManager {
|
||||
SDKManager.resignAllClientTokens();
|
||||
}
|
||||
void this.persistSettingsToDb(nextSettings);
|
||||
void this.notifyIngestionSettingsSync(
|
||||
nextSettings,
|
||||
previousJwtSecret,
|
||||
jwtSecretChanged,
|
||||
).then(() =>
|
||||
jwtSecretChanged ? SDKManager.syncAllClientsToIngestionAsync() : undefined,
|
||||
);
|
||||
return { ...nextSettings };
|
||||
}
|
||||
|
||||
@@ -236,6 +285,7 @@ export class SettingsManager {
|
||||
updates: Partial<AdminSettings>,
|
||||
): Promise<AdminSettings> {
|
||||
const current = await this.getSettingsAsync();
|
||||
const previousJwtSecret = current.jwtSecret;
|
||||
const jwtSecretChanged =
|
||||
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
|
||||
const nextSettings: AdminSettings = {
|
||||
@@ -249,6 +299,14 @@ export class SettingsManager {
|
||||
await SDKManager.resignAllClientTokensAsync();
|
||||
}
|
||||
await this.persistSettingsToDb(nextSettings);
|
||||
await this.notifyIngestionSettingsSync(
|
||||
nextSettings,
|
||||
previousJwtSecret,
|
||||
jwtSecretChanged,
|
||||
);
|
||||
if (jwtSecretChanged) {
|
||||
await SDKManager.syncAllClientsToIngestionAsync();
|
||||
}
|
||||
return { ...nextSettings };
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,44 +1,93 @@
|
||||
import { SettingsManager } from '../adminSettings/settingsManager';
|
||||
import { SDKManager } from '../sdk/sdkManager';
|
||||
import { CSVStorageManager } from '../Storage/csvStorage';
|
||||
import { DataBufferEngine } from '../data/buffer';
|
||||
import { SecurityEngine } from '../Security/security';
|
||||
import {
|
||||
SecurityEngine,
|
||||
getInternalIngestionHeaders,
|
||||
} from '../Security/security';
|
||||
|
||||
export class ApiEngine {
|
||||
/**
|
||||
* Retrieves overall telemetry and health metrics for the dashboard
|
||||
*/
|
||||
public static async getDashboardMetrics() {
|
||||
const storageStats = CSVStorageManager.getStorageStats();
|
||||
const localBufferStats = DataBufferEngine.getBufferStats();
|
||||
const clients = await SDKManager.listClientsAsync();
|
||||
const settings = await SettingsManager.getSettingsAsync();
|
||||
const [storageStats, localBufferStats, clients] = await Promise.all([
|
||||
CSVStorageManager.getStorageStatsAsync(),
|
||||
DataBufferEngine.getBufferStatsAsync(),
|
||||
SDKManager.listClientsAsync(),
|
||||
]);
|
||||
const activeClients = clients.filter((c) => c.status === 'active').length;
|
||||
let activeSockets = 0;
|
||||
let ingestionBufferSize = 0;
|
||||
let ingestionLocalBufferSize = 0;
|
||||
let ingestionRedisBufferSize = 0;
|
||||
let ingestionRedisConnected = false;
|
||||
let remoteSecurityStats: {
|
||||
activeIdempotencyKeys?: number;
|
||||
rateLimitedClientsCount?: number;
|
||||
maxPayloadSizeBytes?: number;
|
||||
} | null = null;
|
||||
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
|
||||
).replace(/\/$/, '');
|
||||
const internalHeaders = getInternalIngestionHeaders();
|
||||
|
||||
try {
|
||||
const res = await fetch(`${ingestionBaseUrl}/health`, {
|
||||
const res = await fetch(`${ingestionBaseUrl}/internal/metrics`, {
|
||||
headers: internalHeaders,
|
||||
cache: 'no-store',
|
||||
signal: AbortSignal.timeout(1000),
|
||||
});
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
activeSockets = Number(data.activeSockets) || 0;
|
||||
ingestionBufferSize = Number(data.bufferSize) || 0;
|
||||
ingestionRedisConnected = Boolean(data.redisConnected);
|
||||
if (data.localBufferSize !== undefined) {
|
||||
ingestionLocalBufferSize = Number(data.localBufferSize) || 0;
|
||||
ingestionRedisBufferSize = Number(data.redisBufferSize) || 0;
|
||||
} else {
|
||||
ingestionLocalBufferSize = Number(data.bufferSize) || 0;
|
||||
}
|
||||
if (data.securityStats && typeof data.securityStats === 'object') {
|
||||
remoteSecurityStats = data.securityStats;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ingestion server may not be running in single-process dev mode
|
||||
activeSockets = 0;
|
||||
ingestionBufferSize = 0;
|
||||
ingestionLocalBufferSize = 0;
|
||||
ingestionRedisBufferSize = 0;
|
||||
}
|
||||
|
||||
const securityStats = SecurityEngine.getStats();
|
||||
const localSecurityStats = SecurityEngine.getStats();
|
||||
const securityStats = {
|
||||
activeIdempotencyKeys:
|
||||
localSecurityStats.activeIdempotencyKeys +
|
||||
(Number(remoteSecurityStats?.activeIdempotencyKeys) || 0),
|
||||
rateLimitedClientsCount:
|
||||
localSecurityStats.rateLimitedClientsCount +
|
||||
(Number(remoteSecurityStats?.rateLimitedClientsCount) || 0),
|
||||
maxPayloadSizeBytes: Math.min(
|
||||
localSecurityStats.maxPayloadSizeBytes,
|
||||
Number(remoteSecurityStats?.maxPayloadSizeBytes) ||
|
||||
localSecurityStats.maxPayloadSizeBytes,
|
||||
),
|
||||
};
|
||||
|
||||
const effectiveRedisBufferSize = Math.max(
|
||||
localBufferStats.redisBufferSize || 0,
|
||||
ingestionRedisBufferSize,
|
||||
);
|
||||
const bufferStats = {
|
||||
...localBufferStats,
|
||||
redisConnected: localBufferStats.redisConnected || ingestionRedisConnected,
|
||||
redisBufferSize: effectiveRedisBufferSize,
|
||||
currentBufferSize:
|
||||
localBufferStats.currentBufferSize + ingestionBufferSize,
|
||||
localBufferStats.localBufferSize +
|
||||
ingestionLocalBufferSize +
|
||||
effectiveRedisBufferSize,
|
||||
};
|
||||
|
||||
return {
|
||||
@@ -49,8 +98,8 @@ export class ApiEngine {
|
||||
storageStats,
|
||||
securityStats,
|
||||
runtime: {
|
||||
serverPort: 4000,
|
||||
webPort: 3000,
|
||||
serverPort: settings.serverPort || 4000,
|
||||
webPort: settings.webPort || 3000,
|
||||
uptimeSeconds: Math.floor(process.uptime()),
|
||||
memoryUsageMb: (process.memoryUsage().rss / (1024 * 1024)).toFixed(1),
|
||||
nodeVersion: process.version,
|
||||
@@ -59,7 +108,7 @@ export class ApiEngine {
|
||||
}
|
||||
|
||||
/**
|
||||
* Filters and streams stored CSV data for export
|
||||
* Filters and lists stored CSV files for export
|
||||
*/
|
||||
public static getExportFiles(clientId?: string) {
|
||||
const allFiles = CSVStorageManager.listAllFiles();
|
||||
@@ -68,6 +117,14 @@ export class ApiEngine {
|
||||
}
|
||||
return allFiles;
|
||||
}
|
||||
|
||||
public static async getExportFilesAsync(clientId?: string) {
|
||||
const allFiles = await CSVStorageManager.listAllFilesAsync();
|
||||
if (clientId && clientId !== 'all') {
|
||||
return allFiles.filter((f) => f.clientId === clientId);
|
||||
}
|
||||
return allFiles;
|
||||
}
|
||||
}
|
||||
|
||||
export default ApiEngine;
|
||||
@@ -1,8 +1,23 @@
|
||||
const encoder = new TextEncoder();
|
||||
|
||||
interface AdminSessionPayload {
|
||||
const DEV_DEFAULT_SESSION_SECRET =
|
||||
"datastack-admin-session-default-secret-32-chars";
|
||||
|
||||
export interface AdminSessionPayload {
|
||||
email: string;
|
||||
expiresAt: number;
|
||||
tokenVersion: number;
|
||||
}
|
||||
|
||||
export function getAdminSessionSecret(): string | null {
|
||||
const configured = process.env.DATASTACK_ADMIN_SESSION_SECRET?.trim() || "";
|
||||
if (process.env.NODE_ENV === "production") {
|
||||
return configured.length >= 32 ? configured : null;
|
||||
}
|
||||
if (configured.length > 0) {
|
||||
return configured.length >= 32 ? configured : null;
|
||||
}
|
||||
return DEV_DEFAULT_SESSION_SECRET;
|
||||
}
|
||||
|
||||
function encodeBase64Url(bytes: Uint8Array) {
|
||||
@@ -34,9 +49,14 @@ export async function createAdminSessionToken(
|
||||
email: string,
|
||||
secret: string,
|
||||
expiresAt: number,
|
||||
tokenVersion = 1,
|
||||
) {
|
||||
const safeVersion =
|
||||
Number.isInteger(tokenVersion) && tokenVersion >= 1 ? tokenVersion : 1;
|
||||
const payload = encodeBase64Url(
|
||||
encoder.encode(JSON.stringify({ email, expiresAt })),
|
||||
encoder.encode(
|
||||
JSON.stringify({ email, expiresAt, tokenVersion: safeVersion }),
|
||||
),
|
||||
);
|
||||
const signature = await crypto.subtle.sign(
|
||||
"HMAC",
|
||||
@@ -50,6 +70,7 @@ export async function getAdminSessionPayload(
|
||||
token: string,
|
||||
secret: string,
|
||||
): Promise<AdminSessionPayload | null> {
|
||||
if (!secret || secret.length < 32) return null;
|
||||
const [payload, signature, extra] = token.split(".");
|
||||
if (!payload || !signature || extra) return null;
|
||||
|
||||
@@ -68,7 +89,10 @@ export async function getAdminSessionPayload(
|
||||
if (
|
||||
typeof session.email === "string" &&
|
||||
typeof session.expiresAt === "number" &&
|
||||
session.expiresAt > Date.now()
|
||||
session.expiresAt > Date.now() &&
|
||||
typeof session.tokenVersion === "number" &&
|
||||
Number.isInteger(session.tokenVersion) &&
|
||||
session.tokenVersion >= 1
|
||||
) {
|
||||
return session;
|
||||
}
|
||||
|
||||
+279
-163
@@ -6,15 +6,22 @@ import {
|
||||
timingSafeEqual,
|
||||
} from "node:crypto";
|
||||
import { Pool, type PoolClient } from "pg";
|
||||
import {
|
||||
type AdminSessionPayload,
|
||||
getAdminSessionPayload,
|
||||
getAdminSessionSecret,
|
||||
} from "./adminSession";
|
||||
|
||||
const PASSWORD_HASH_BYTES = 64;
|
||||
const PASSWORD_SALT_BYTES = 16;
|
||||
const DATABASE_LOCK_ID = "72025102501";
|
||||
const LOGIN_ATTEMPT_LIMIT = 5;
|
||||
const IP_LOGIN_ATTEMPT_LIMIT = 25;
|
||||
const LOGIN_WINDOW_MS = 15 * 60 * 1000;
|
||||
const LOGIN_BLOCK_MS = 15 * 60 * 1000;
|
||||
const LOCAL_THROTTLE_MAX_ENTRIES = 5000;
|
||||
const LOCAL_THROTTLE_CLEANUP_MS = 60 * 1000;
|
||||
const DEV_DEFAULT_SETUP_KEY = "datastack-default-setup-key-32-chars";
|
||||
|
||||
interface LoginThrottleEntry {
|
||||
windowStartedAt: number;
|
||||
@@ -28,11 +35,17 @@ interface LoginThrottleDecision {
|
||||
retryAfterSeconds: number;
|
||||
}
|
||||
|
||||
export interface LoginThrottleBucket {
|
||||
keyHash: string;
|
||||
limit: number;
|
||||
}
|
||||
|
||||
interface InMemoryAdmin {
|
||||
id: number;
|
||||
email: string;
|
||||
passwordSalt: Buffer;
|
||||
passwordHash: Buffer;
|
||||
tokenVersion: number;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
@@ -47,8 +60,12 @@ const localLoginThrottle = new Map<string, LoginThrottleEntry>();
|
||||
let lastLocalThrottleCleanup = 0;
|
||||
let lastDatabaseThrottleCleanup = 0;
|
||||
|
||||
function isProductionEnv() {
|
||||
return process.env.NODE_ENV === "production";
|
||||
}
|
||||
|
||||
function getPool(): Pool | null {
|
||||
const connectionString = process.env.DATABASE_URL;
|
||||
const connectionString = process.env.DATABASE_URL?.trim();
|
||||
if (!connectionString) {
|
||||
return null;
|
||||
}
|
||||
@@ -74,17 +91,24 @@ async function ensureAdminTable() {
|
||||
if (!pool) return;
|
||||
|
||||
globals.__dataStackAdminSchema ??= Promise.all([
|
||||
pool.query(
|
||||
`
|
||||
pool
|
||||
.query(
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS admin_users (
|
||||
id SMALLINT PRIMARY KEY DEFAULT 1 CHECK (id = 1),
|
||||
email TEXT NOT NULL UNIQUE,
|
||||
password_salt BYTEA NOT NULL,
|
||||
password_hash BYTEA NOT NULL,
|
||||
token_version INTEGER NOT NULL DEFAULT 1,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`,
|
||||
),
|
||||
)
|
||||
.then(() =>
|
||||
pool.query(
|
||||
`ALTER TABLE admin_users ADD COLUMN IF NOT EXISTS token_version INTEGER NOT NULL DEFAULT 1`,
|
||||
),
|
||||
),
|
||||
pool.query(
|
||||
`
|
||||
CREATE TABLE IF NOT EXISTS admin_login_attempts (
|
||||
@@ -121,8 +145,41 @@ function matchesSecret(candidate: string, expected: string) {
|
||||
return timingSafeEqual(candidateHash, expectedHash);
|
||||
}
|
||||
|
||||
function createLoginThrottleKeys(email: string, secret: string) {
|
||||
return [createHmac("sha256", secret).update(`email:${email}`).digest("hex")];
|
||||
export function getAdminSetupKey(): string | null {
|
||||
const configured = process.env.DATASTACK_ADMIN_SETUP_KEY?.trim() || "";
|
||||
if (isProductionEnv()) {
|
||||
return configured.length >= 32 ? configured : null;
|
||||
}
|
||||
if (configured.length > 0) {
|
||||
return configured.length >= 32 ? configured : null;
|
||||
}
|
||||
return DEV_DEFAULT_SETUP_KEY;
|
||||
}
|
||||
|
||||
export function createLoginThrottleBuckets(
|
||||
email: string,
|
||||
secret: string,
|
||||
ip = "unknown",
|
||||
): LoginThrottleBucket[] {
|
||||
const cleanEmail = email.trim().toLowerCase();
|
||||
const cleanIp = ip.trim().toLowerCase() || "unknown";
|
||||
const buckets: LoginThrottleBucket[] = [
|
||||
{
|
||||
keyHash: createHmac("sha256", secret)
|
||||
.update(`ip_email:${cleanIp}:${cleanEmail}`)
|
||||
.digest("hex"),
|
||||
limit: LOGIN_ATTEMPT_LIMIT,
|
||||
},
|
||||
];
|
||||
if (cleanIp !== "unknown") {
|
||||
buckets.push({
|
||||
keyHash: createHmac("sha256", secret)
|
||||
.update(`ip:${cleanIp}`)
|
||||
.digest("hex"),
|
||||
limit: IP_LOGIN_ATTEMPT_LIMIT,
|
||||
});
|
||||
}
|
||||
return buckets;
|
||||
}
|
||||
|
||||
function getAdvisoryLockId(keyHash: string) {
|
||||
@@ -157,14 +214,14 @@ function cleanupLocalLoginThrottle(now: number) {
|
||||
}
|
||||
|
||||
function consumeLocalLoginAttempt(
|
||||
keyHashes: string[],
|
||||
buckets: LoginThrottleBucket[],
|
||||
now: number,
|
||||
): LoginThrottleDecision {
|
||||
cleanupLocalLoginThrottle(now);
|
||||
|
||||
const blockedUntil = keyHashes.reduce(
|
||||
(latest, keyHash) =>
|
||||
Math.max(latest, localLoginThrottle.get(keyHash)?.blockedUntil ?? 0),
|
||||
const blockedUntil = buckets.reduce(
|
||||
(latest, bucket) =>
|
||||
Math.max(latest, localLoginThrottle.get(bucket.keyHash)?.blockedUntil ?? 0),
|
||||
0,
|
||||
);
|
||||
if (blockedUntil > now) {
|
||||
@@ -174,16 +231,16 @@ function consumeLocalLoginAttempt(
|
||||
};
|
||||
}
|
||||
|
||||
for (const keyHash of keyHashes) {
|
||||
const previous = localLoginThrottle.get(keyHash);
|
||||
for (const bucket of buckets) {
|
||||
const previous = localLoginThrottle.get(bucket.keyHash);
|
||||
const windowExpired =
|
||||
!previous || now - previous.windowStartedAt >= LOGIN_WINDOW_MS;
|
||||
const failedAttempts = windowExpired ? 1 : previous.failedAttempts + 1;
|
||||
localLoginThrottle.set(keyHash, {
|
||||
localLoginThrottle.set(bucket.keyHash, {
|
||||
windowStartedAt: windowExpired ? now : previous.windowStartedAt,
|
||||
failedAttempts,
|
||||
blockedUntil:
|
||||
failedAttempts >= LOGIN_ATTEMPT_LIMIT ? now + LOGIN_BLOCK_MS : 0,
|
||||
failedAttempts >= bucket.limit ? now + LOGIN_BLOCK_MS : 0,
|
||||
updatedAt: now,
|
||||
});
|
||||
}
|
||||
@@ -192,12 +249,13 @@ function consumeLocalLoginAttempt(
|
||||
}
|
||||
|
||||
async function consumeDatabaseLoginAttempt(
|
||||
keyHashes: string[],
|
||||
buckets: LoginThrottleBucket[],
|
||||
now: number,
|
||||
): Promise<LoginThrottleDecision> {
|
||||
const pool = getPool();
|
||||
if (!pool) return consumeLocalLoginAttempt(keyHashes, now);
|
||||
if (!pool) return consumeLocalLoginAttempt(buckets, now);
|
||||
await ensureAdminTable();
|
||||
const keyHashes = buckets.map((b) => b.keyHash);
|
||||
const client = await pool.connect();
|
||||
|
||||
try {
|
||||
@@ -229,8 +287,8 @@ async function consumeDatabaseLoginAttempt(
|
||||
};
|
||||
}
|
||||
|
||||
for (const keyHash of keyHashes) {
|
||||
const previous = entries.get(keyHash);
|
||||
for (const bucket of buckets) {
|
||||
const previous = entries.get(bucket.keyHash);
|
||||
const windowStartedAt = previous
|
||||
? new Date(previous.window_started_at as string | Date).getTime()
|
||||
: 0;
|
||||
@@ -243,7 +301,7 @@ async function consumeDatabaseLoginAttempt(
|
||||
? new Date(now)
|
||||
: new Date(windowStartedAt);
|
||||
const nextBlockedUntil =
|
||||
failedAttempts >= LOGIN_ATTEMPT_LIMIT
|
||||
failedAttempts >= bucket.limit
|
||||
? new Date(now + LOGIN_BLOCK_MS)
|
||||
: null;
|
||||
|
||||
@@ -256,7 +314,7 @@ async function consumeDatabaseLoginAttempt(
|
||||
failed_attempts = EXCLUDED.failed_attempts,
|
||||
blocked_until = EXCLUDED.blocked_until,
|
||||
updated_at = NOW()`,
|
||||
[keyHash, nextWindowStartedAt, failedAttempts, nextBlockedUntil],
|
||||
[bucket.keyHash, nextWindowStartedAt, failedAttempts, nextBlockedUntil],
|
||||
);
|
||||
}
|
||||
|
||||
@@ -279,13 +337,20 @@ async function consumeDatabaseLoginAttempt(
|
||||
|
||||
async function getOrInitInMemoryAdmin(): Promise<InMemoryAdmin> {
|
||||
if (!globals.__dataStackInMemoryAdmin) {
|
||||
const envPassword = process.env.DATASTACK_ADMIN_PASSWORD;
|
||||
const allowDemo =
|
||||
!isProductionEnv() && process.env.DATASTACK_ALLOW_DEMO_ADMIN !== "false";
|
||||
|
||||
if (!envPassword && !allowDemo) {
|
||||
throw new Error("Admin account has not completed initial setup.");
|
||||
}
|
||||
|
||||
const defaultEmail = (
|
||||
process.env.DATASTACK_ADMIN_EMAIL || "admin@datastack.dev"
|
||||
)
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const defaultPassword =
|
||||
process.env.DATASTACK_ADMIN_PASSWORD || "admin12345678";
|
||||
const defaultPassword = envPassword || "admin12345678";
|
||||
const salt = randomBytes(PASSWORD_SALT_BYTES);
|
||||
const hash = await derivePasswordHash(defaultPassword, salt);
|
||||
globals.__dataStackInMemoryAdmin = {
|
||||
@@ -293,6 +358,7 @@ async function getOrInitInMemoryAdmin(): Promise<InMemoryAdmin> {
|
||||
email: defaultEmail,
|
||||
passwordSalt: salt,
|
||||
passwordHash: hash,
|
||||
tokenVersion: 1,
|
||||
createdAt: new Date(),
|
||||
};
|
||||
}
|
||||
@@ -302,24 +368,26 @@ async function getOrInitInMemoryAdmin(): Promise<InMemoryAdmin> {
|
||||
export async function consumeAdminLoginAttempt(
|
||||
email: string,
|
||||
secret: string,
|
||||
ip = "unknown",
|
||||
): Promise<LoginThrottleDecision> {
|
||||
const keyHashes = createLoginThrottleKeys(email, secret);
|
||||
const buckets = createLoginThrottleBuckets(email, secret, ip);
|
||||
const now = Date.now();
|
||||
|
||||
const pool = getPool();
|
||||
if (!pool) {
|
||||
return consumeLocalLoginAttempt(keyHashes, now);
|
||||
return consumeLocalLoginAttempt(buckets, now);
|
||||
}
|
||||
|
||||
try {
|
||||
return await consumeDatabaseLoginAttempt(keyHashes, now);
|
||||
} catch {
|
||||
return consumeLocalLoginAttempt(keyHashes, now);
|
||||
}
|
||||
return await consumeDatabaseLoginAttempt(buckets, now);
|
||||
}
|
||||
|
||||
export async function clearAdminLoginAttempts(email: string, secret: string) {
|
||||
const keyHashes = createLoginThrottleKeys(email, secret);
|
||||
export async function clearAdminLoginAttempts(
|
||||
email: string,
|
||||
secret: string,
|
||||
ip = "unknown",
|
||||
) {
|
||||
const buckets = createLoginThrottleBuckets(email, secret, ip);
|
||||
const keyHashes = buckets.map((b) => b.keyHash);
|
||||
|
||||
const pool = getPool();
|
||||
if (!pool) {
|
||||
@@ -327,50 +395,45 @@ export async function clearAdminLoginAttempts(email: string, secret: string) {
|
||||
return;
|
||||
}
|
||||
|
||||
await ensureAdminTable();
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await ensureAdminTable();
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
await lockLoginThrottleKeys(client, keyHashes);
|
||||
await client.query(
|
||||
"DELETE FROM admin_login_attempts WHERE key_hash = ANY($1::text[])",
|
||||
[keyHashes],
|
||||
);
|
||||
await client.query("COMMIT");
|
||||
} catch (error) {
|
||||
await client.query("ROLLBACK").catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
} catch {
|
||||
keyHashes.forEach((keyHash) => localLoginThrottle.delete(keyHash));
|
||||
await client.query("BEGIN");
|
||||
await lockLoginThrottleKeys(client, keyHashes);
|
||||
await client.query(
|
||||
"DELETE FROM admin_login_attempts WHERE key_hash = ANY($1::text[])",
|
||||
[keyHashes],
|
||||
);
|
||||
await client.query("COMMIT");
|
||||
} catch (error) {
|
||||
await client.query("ROLLBACK").catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
export function verifyAdminSetupKey(candidate: string) {
|
||||
const expected =
|
||||
process.env.DATASTACK_ADMIN_SETUP_KEY ||
|
||||
"datastack-default-setup-key-32-chars";
|
||||
if (!expected || expected.length < 32) return false;
|
||||
const expected = getAdminSetupKey();
|
||||
if (!expected || expected.length < 32 || !candidate) return false;
|
||||
return matchesSecret(candidate, expected);
|
||||
}
|
||||
|
||||
export async function isAdminSetupComplete() {
|
||||
const pool = getPool();
|
||||
if (!pool) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
await ensureAdminTable();
|
||||
const result = await pool.query(
|
||||
"SELECT 1 FROM admin_users WHERE id = 1",
|
||||
);
|
||||
return result.rowCount === 1;
|
||||
} catch {
|
||||
return true;
|
||||
if (globals.__dataStackInMemoryAdmin) {
|
||||
return true;
|
||||
}
|
||||
if (isProductionEnv()) {
|
||||
return Boolean(process.env.DATASTACK_ADMIN_PASSWORD);
|
||||
}
|
||||
return process.env.DATASTACK_ALLOW_DEMO_ADMIN !== "false";
|
||||
}
|
||||
|
||||
await ensureAdminTable();
|
||||
const result = await pool.query("SELECT 1 FROM admin_users WHERE id = 1");
|
||||
return result.rowCount === 1;
|
||||
}
|
||||
|
||||
export async function createInitialAdmin(email: string, password: string) {
|
||||
@@ -380,136 +443,176 @@ export async function createInitialAdmin(email: string, password: string) {
|
||||
const pool = getPool();
|
||||
|
||||
if (!pool) {
|
||||
if (globals.__dataStackInMemoryAdmin) {
|
||||
return false;
|
||||
}
|
||||
globals.__dataStackInMemoryAdmin = {
|
||||
id: 1,
|
||||
email: cleanEmail,
|
||||
passwordSalt: salt,
|
||||
passwordHash,
|
||||
tokenVersion: 1,
|
||||
createdAt: new Date(),
|
||||
};
|
||||
return true;
|
||||
}
|
||||
|
||||
await ensureAdminTable();
|
||||
const client = await pool.connect();
|
||||
|
||||
try {
|
||||
await ensureAdminTable();
|
||||
const client = await pool.connect();
|
||||
await client.query("BEGIN");
|
||||
await client.query("SELECT pg_advisory_xact_lock($1::bigint)", [
|
||||
DATABASE_LOCK_ID,
|
||||
]);
|
||||
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
await client.query("SELECT pg_advisory_xact_lock($1::bigint)", [
|
||||
DATABASE_LOCK_ID,
|
||||
]);
|
||||
|
||||
const existing = await client.query(
|
||||
"SELECT 1 FROM admin_users WHERE id = 1",
|
||||
);
|
||||
if (existing.rowCount) {
|
||||
await client.query("ROLLBACK");
|
||||
return false;
|
||||
}
|
||||
|
||||
await client.query(
|
||||
`INSERT INTO admin_users (id, email, password_salt, password_hash)
|
||||
VALUES (1, $1, $2, $3)`,
|
||||
[cleanEmail, salt, passwordHash],
|
||||
);
|
||||
await client.query("COMMIT");
|
||||
return true;
|
||||
} catch (error) {
|
||||
await client.query("ROLLBACK").catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
const existing = await client.query(
|
||||
"SELECT 1 FROM admin_users WHERE id = 1",
|
||||
);
|
||||
if (existing.rowCount) {
|
||||
await client.query("ROLLBACK");
|
||||
return false;
|
||||
}
|
||||
} catch {
|
||||
globals.__dataStackInMemoryAdmin = {
|
||||
id: 1,
|
||||
email: cleanEmail,
|
||||
passwordSalt: salt,
|
||||
passwordHash,
|
||||
createdAt: new Date(),
|
||||
};
|
||||
|
||||
await client.query(
|
||||
`INSERT INTO admin_users (id, email, password_salt, password_hash, token_version)
|
||||
VALUES (1, $1, $2, $3, 1)`,
|
||||
[cleanEmail, salt, passwordHash],
|
||||
);
|
||||
await client.query("COMMIT");
|
||||
return true;
|
||||
} catch (error) {
|
||||
await client.query("ROLLBACK").catch(() => undefined);
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
export async function verifyAdminCredentials(email: string, password: string) {
|
||||
export async function authenticateAdminCredentials(
|
||||
email: string,
|
||||
password: string,
|
||||
): Promise<{ authenticated: boolean; tokenVersion: number }> {
|
||||
const cleanEmail = email.trim().toLowerCase();
|
||||
const pool = getPool();
|
||||
|
||||
if (pool) {
|
||||
try {
|
||||
await ensureAdminTable();
|
||||
const result = await pool.query(
|
||||
`SELECT email, password_salt, password_hash
|
||||
FROM admin_users
|
||||
WHERE id = 1`,
|
||||
);
|
||||
if (result.rowCount && result.rowCount > 0) {
|
||||
const admin = result.rows[0] as {
|
||||
email: string;
|
||||
password_salt: Buffer;
|
||||
password_hash: Buffer;
|
||||
};
|
||||
const emailMatches = matchesSecret(
|
||||
cleanEmail,
|
||||
admin.email.trim().toLowerCase(),
|
||||
);
|
||||
const actualHash = await derivePasswordHash(
|
||||
password,
|
||||
admin.password_salt,
|
||||
);
|
||||
const passwordMatches = timingSafeEqual(
|
||||
actualHash,
|
||||
admin.password_hash,
|
||||
);
|
||||
return emailMatches && passwordMatches;
|
||||
}
|
||||
if (!process.env.DATASTACK_ADMIN_PASSWORD) {
|
||||
return false;
|
||||
}
|
||||
} catch {
|
||||
// Database connection error, fall back to in-memory admin
|
||||
await ensureAdminTable();
|
||||
const result = await pool.query(
|
||||
`SELECT email, password_salt, password_hash, COALESCE(token_version, 1) AS token_version
|
||||
FROM admin_users
|
||||
WHERE id = 1`,
|
||||
);
|
||||
if (!result.rowCount || result.rowCount === 0) {
|
||||
return { authenticated: false, tokenVersion: 0 };
|
||||
}
|
||||
const admin = result.rows[0] as {
|
||||
email: string;
|
||||
password_salt: Buffer;
|
||||
password_hash: Buffer;
|
||||
token_version: number;
|
||||
};
|
||||
const emailMatches = matchesSecret(
|
||||
cleanEmail,
|
||||
admin.email.trim().toLowerCase(),
|
||||
);
|
||||
const actualHash = await derivePasswordHash(password, admin.password_salt);
|
||||
const passwordMatches = timingSafeEqual(actualHash, admin.password_hash);
|
||||
return {
|
||||
authenticated: emailMatches && passwordMatches,
|
||||
tokenVersion: Number(admin.token_version) || 1,
|
||||
};
|
||||
}
|
||||
|
||||
const inMem = await getOrInitInMemoryAdmin();
|
||||
if (!matchesSecret(cleanEmail, inMem.email)) {
|
||||
return false;
|
||||
return { authenticated: false, tokenVersion: 0 };
|
||||
}
|
||||
const actualHash = await derivePasswordHash(password, inMem.passwordSalt);
|
||||
return timingSafeEqual(actualHash, inMem.passwordHash);
|
||||
const authenticated = timingSafeEqual(actualHash, inMem.passwordHash);
|
||||
return {
|
||||
authenticated,
|
||||
tokenVersion: authenticated ? inMem.tokenVersion : 0,
|
||||
};
|
||||
}
|
||||
|
||||
export async function verifyAdminCredentials(email: string, password: string) {
|
||||
const result = await authenticateAdminCredentials(email, password);
|
||||
return result.authenticated;
|
||||
}
|
||||
|
||||
export async function getAdminAccountInfo(fallbackEmail?: string) {
|
||||
const pool = getPool();
|
||||
if (pool) {
|
||||
try {
|
||||
await ensureAdminTable();
|
||||
const result = await pool.query(
|
||||
`SELECT email, created_at FROM admin_users WHERE id = 1`,
|
||||
);
|
||||
if (result.rowCount && result.rowCount > 0) {
|
||||
const row = result.rows[0] as { email: string; created_at: Date | string };
|
||||
return {
|
||||
email: row.email,
|
||||
createdAt: new Date(row.created_at).toISOString(),
|
||||
storageMode: "PostgreSQL" as const,
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
// Fall back to in-memory admin
|
||||
await ensureAdminTable();
|
||||
const result = await pool.query(
|
||||
`SELECT email, created_at, COALESCE(token_version, 1) AS token_version
|
||||
FROM admin_users
|
||||
WHERE id = 1`,
|
||||
);
|
||||
if (result.rowCount && result.rowCount > 0) {
|
||||
const row = result.rows[0] as {
|
||||
email: string;
|
||||
created_at: Date | string;
|
||||
token_version: number;
|
||||
};
|
||||
return {
|
||||
email: row.email,
|
||||
createdAt: new Date(row.created_at).toISOString(),
|
||||
tokenVersion: Number(row.token_version) || 1,
|
||||
storageMode: "PostgreSQL" as const,
|
||||
};
|
||||
}
|
||||
throw new Error("Admin account is not initialized.");
|
||||
}
|
||||
|
||||
const inMem = await getOrInitInMemoryAdmin();
|
||||
return {
|
||||
email: fallbackEmail || inMem.email,
|
||||
createdAt: inMem.createdAt.toISOString(),
|
||||
tokenVersion: inMem.tokenVersion,
|
||||
storageMode: "In-Memory" as const,
|
||||
};
|
||||
}
|
||||
|
||||
export async function verifyRequestAdminSession(
|
||||
cookieToken: string | undefined,
|
||||
): Promise<
|
||||
| { valid: true; session: AdminSessionPayload; sessionSecret: string }
|
||||
| { valid: false; reason: "UNCONFIGURED" | "UNAUTHORIZED" | "UNAVAILABLE" }
|
||||
> {
|
||||
const sessionSecret = getAdminSessionSecret();
|
||||
if (!sessionSecret) {
|
||||
return { valid: false, reason: "UNCONFIGURED" };
|
||||
}
|
||||
if (!cookieToken) {
|
||||
return { valid: false, reason: "UNAUTHORIZED" };
|
||||
}
|
||||
|
||||
const session = await getAdminSessionPayload(cookieToken, sessionSecret);
|
||||
if (!session) {
|
||||
return { valid: false, reason: "UNAUTHORIZED" };
|
||||
}
|
||||
|
||||
try {
|
||||
const account = await getAdminAccountInfo();
|
||||
const emailMatches = matchesSecret(
|
||||
session.email.trim().toLowerCase(),
|
||||
account.email.trim().toLowerCase(),
|
||||
);
|
||||
if (!emailMatches || session.tokenVersion !== account.tokenVersion) {
|
||||
return { valid: false, reason: "UNAUTHORIZED" };
|
||||
}
|
||||
return { valid: true, session, sessionSecret };
|
||||
} catch (error) {
|
||||
if (getPool()) {
|
||||
return { valid: false, reason: "UNAVAILABLE" };
|
||||
}
|
||||
void error;
|
||||
return { valid: false, reason: "UNAUTHORIZED" };
|
||||
}
|
||||
}
|
||||
|
||||
export async function updateAdminAccount({
|
||||
currentEmail,
|
||||
currentPassword,
|
||||
@@ -534,29 +637,42 @@ export async function updateAdminAccount({
|
||||
|
||||
const pool = getPool();
|
||||
if (pool) {
|
||||
try {
|
||||
await ensureAdminTable();
|
||||
const updated = await pool.query(
|
||||
`UPDATE admin_users
|
||||
SET email = $1, password_salt = $2, password_hash = $3
|
||||
WHERE id = 1`,
|
||||
[cleanEmail, salt, passwordHash],
|
||||
);
|
||||
if (updated.rowCount && updated.rowCount > 0) {
|
||||
return { ok: true as const, email: cleanEmail };
|
||||
}
|
||||
} catch {
|
||||
// Fall back to in-memory update
|
||||
await ensureAdminTable();
|
||||
const updated = await pool.query(
|
||||
`UPDATE admin_users
|
||||
SET email = $1,
|
||||
password_salt = $2,
|
||||
password_hash = $3,
|
||||
token_version = COALESCE(token_version, 1) + 1
|
||||
WHERE id = 1
|
||||
RETURNING token_version`,
|
||||
[cleanEmail, salt, passwordHash],
|
||||
);
|
||||
if (!updated.rowCount || updated.rowCount === 0) {
|
||||
return { ok: false as const, error: "Admin account not found." };
|
||||
}
|
||||
return {
|
||||
ok: true as const,
|
||||
email: cleanEmail,
|
||||
tokenVersion: Number(updated.rows[0].token_version) || 1,
|
||||
};
|
||||
}
|
||||
|
||||
const existing = await getOrInitInMemoryAdmin();
|
||||
const nextVersion = existing.tokenVersion + 1;
|
||||
globals.__dataStackInMemoryAdmin = {
|
||||
id: 1,
|
||||
email: cleanEmail,
|
||||
passwordSalt: salt,
|
||||
passwordHash,
|
||||
tokenVersion: nextVersion,
|
||||
createdAt: existing.createdAt,
|
||||
};
|
||||
return { ok: true as const, email: cleanEmail };
|
||||
return { ok: true as const, email: cleanEmail, tokenVersion: nextVersion };
|
||||
}
|
||||
|
||||
export function resetAdminStateForTests() {
|
||||
globals.__dataStackInMemoryAdmin = undefined;
|
||||
localLoginThrottle.clear();
|
||||
lastLocalThrottleCleanup = 0;
|
||||
}
|
||||
+33
-10
@@ -8,16 +8,31 @@ export interface ClientTokenPayload {
|
||||
exp?: number;
|
||||
}
|
||||
|
||||
const DEFAULT_SECRET =
|
||||
process.env.DATASTACK_JWT_SECRET ||
|
||||
const DEV_DEFAULT_JWT_SECRET =
|
||||
"datastack-jwt-default-secret-key-32-chars-long";
|
||||
|
||||
export function resolveDefaultJwtSecret(): string {
|
||||
const configured = process.env.DATASTACK_JWT_SECRET?.trim() || "";
|
||||
if (process.env.NODE_ENV === "production") {
|
||||
return configured.length >= 32 ? configured : "";
|
||||
}
|
||||
if (configured.length > 0) {
|
||||
return configured.length >= 32 ? configured : "";
|
||||
}
|
||||
return DEV_DEFAULT_JWT_SECRET;
|
||||
}
|
||||
|
||||
export class AuthService {
|
||||
private static secret: string = DEFAULT_SECRET;
|
||||
private static secret: string = resolveDefaultJwtSecret();
|
||||
|
||||
public static setSecret(newSecret: string) {
|
||||
if (newSecret && newSecret.trim().length > 0) {
|
||||
this.secret = newSecret.trim();
|
||||
const trimmed = newSecret ? newSecret.trim() : "";
|
||||
if (trimmed.length >= 32) {
|
||||
this.secret = trimmed;
|
||||
} else if (process.env.NODE_ENV !== "production" && trimmed.length > 0) {
|
||||
this.secret = trimmed;
|
||||
} else if (process.env.NODE_ENV === "production") {
|
||||
this.secret = "";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +40,11 @@ export class AuthService {
|
||||
return this.secret;
|
||||
}
|
||||
|
||||
public static isConfigured(): boolean {
|
||||
const minLen = process.env.NODE_ENV === "production" ? 32 : 1;
|
||||
return this.secret.length >= minLen;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates a signed JWT for client SDK authentication.
|
||||
*/
|
||||
@@ -32,9 +52,9 @@ export class AuthService {
|
||||
payload: { clientId: string; name?: string; rateLimit?: number },
|
||||
expiresIn = "30d",
|
||||
): string {
|
||||
if (!this.secret) {
|
||||
if (!this.isConfigured()) {
|
||||
throw new Error(
|
||||
"DATASTACK_JWT_SECRET must be configured before generating client tokens.",
|
||||
"DATASTACK_JWT_SECRET (>= 32 characters) must be configured before generating client tokens.",
|
||||
);
|
||||
}
|
||||
|
||||
@@ -50,15 +70,18 @@ export class AuthService {
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies and decodes incoming token from Socket.IO handshake.
|
||||
* Verifies and decodes incoming token from Socket.IO handshake or HTTP ingestion.
|
||||
*/
|
||||
public static verifyClientToken(token: string): {
|
||||
valid: boolean;
|
||||
payload?: ClientTokenPayload;
|
||||
error?: string;
|
||||
} {
|
||||
if (!this.secret) {
|
||||
return { valid: false, error: "DATASTACK_JWT_SECRET is not configured." };
|
||||
if (!this.isConfigured()) {
|
||||
return {
|
||||
valid: false,
|
||||
error: "DATASTACK_JWT_SECRET (>= 32 characters) is not configured.",
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
|
||||
+380
-28
@@ -1,8 +1,37 @@
|
||||
import crypto from 'node:crypto';
|
||||
import Redis from 'ioredis';
|
||||
import { CSVStorageManager, DataEvent } from '../Storage/csvStorage';
|
||||
|
||||
export type EventSubscriber = (event: DataEvent) => void;
|
||||
export type FlushCallback = (count: number, files: string[]) => void;
|
||||
|
||||
export interface RedisBufferClient {
|
||||
status?: string;
|
||||
rpush(key: string, ...values: string[]): Promise<number>;
|
||||
lpush(key: string, ...values: string[]): Promise<number>;
|
||||
ltrim(key: string, start: number, stop: number): Promise<unknown>;
|
||||
lrange(key: string, start: number, stop: number): Promise<string[]>;
|
||||
llen(key: string): Promise<number>;
|
||||
set(
|
||||
key: string,
|
||||
value: string,
|
||||
pxMode: 'PX',
|
||||
ttlMs: number,
|
||||
nxMode: 'NX',
|
||||
): Promise<string | null>;
|
||||
eval(script: string, numKeys: number, ...args: string[]): Promise<unknown>;
|
||||
quit?: () => Promise<unknown>;
|
||||
disconnect?: () => void;
|
||||
}
|
||||
|
||||
export const REDIS_BUFFER_KEY = 'datastack:buffer:events';
|
||||
export const REDIS_RECENT_KEY = 'datastack:recent:events';
|
||||
export const REDIS_FLUSH_LOCK_KEY = 'datastack:flush:lock';
|
||||
const REDIS_RECENT_LIMIT = 500;
|
||||
const FLUSH_LOCK_TTL_MS = 15000;
|
||||
const UNLOCK_LUA_SCRIPT =
|
||||
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end';
|
||||
|
||||
interface BufferGlobalState {
|
||||
buffer: DataEvent[];
|
||||
recentEvents: DataEvent[];
|
||||
@@ -14,6 +43,11 @@ interface BufferGlobalState {
|
||||
totalIngested: number;
|
||||
totalFlushed: number;
|
||||
isFlushing: boolean;
|
||||
redisClient: RedisBufferClient | null | undefined;
|
||||
redisUrlConfigured: string;
|
||||
redisConnected: boolean;
|
||||
redisTestOverride: RedisBufferClient | null | undefined;
|
||||
pendingRedisWrites: Set<Promise<void>>;
|
||||
}
|
||||
|
||||
type BufferGlobals = typeof globalThis & {
|
||||
@@ -36,13 +70,86 @@ function getState(): BufferGlobalState {
|
||||
totalIngested: 0,
|
||||
totalFlushed: 0,
|
||||
isFlushing: false,
|
||||
redisClient: undefined,
|
||||
redisUrlConfigured: '',
|
||||
redisConnected: false,
|
||||
redisTestOverride: undefined,
|
||||
pendingRedisWrites: new Set(),
|
||||
};
|
||||
}
|
||||
return globals.__dataStackBufferState;
|
||||
}
|
||||
|
||||
function getRedisClient(): RedisBufferClient | null {
|
||||
const state = getState();
|
||||
if (state.redisTestOverride !== undefined) {
|
||||
return state.redisTestOverride;
|
||||
}
|
||||
|
||||
const configuredUrl = (process.env.REDIS_URL || '').trim();
|
||||
if (!configuredUrl) {
|
||||
if (state.redisClient && typeof state.redisClient.disconnect === 'function') {
|
||||
try {
|
||||
state.redisClient.disconnect();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
state.redisClient = null;
|
||||
state.redisUrlConfigured = '';
|
||||
state.redisConnected = false;
|
||||
return null;
|
||||
}
|
||||
|
||||
if (state.redisClient && state.redisUrlConfigured === configuredUrl) {
|
||||
return state.redisClient;
|
||||
}
|
||||
|
||||
if (state.redisClient && typeof state.redisClient.disconnect === 'function') {
|
||||
try {
|
||||
state.redisClient.disconnect();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const client = new Redis(configuredUrl, {
|
||||
maxRetriesPerRequest: 2,
|
||||
enableReadyCheck: true,
|
||||
retryStrategy: (times: number) => Math.min(times * 250, 3000),
|
||||
});
|
||||
|
||||
client.on('ready', () => {
|
||||
state.redisConnected = true;
|
||||
});
|
||||
client.on('connect', () => {
|
||||
state.redisConnected = true;
|
||||
});
|
||||
client.on('close', () => {
|
||||
state.redisConnected = false;
|
||||
});
|
||||
client.on('end', () => {
|
||||
state.redisConnected = false;
|
||||
});
|
||||
client.on('error', () => {
|
||||
state.redisConnected = false;
|
||||
});
|
||||
|
||||
state.redisClient = client as unknown as RedisBufferClient;
|
||||
state.redisUrlConfigured = configuredUrl;
|
||||
state.redisConnected = client.status === 'ready' || client.status === 'connect';
|
||||
return state.redisClient;
|
||||
} catch {
|
||||
state.redisClient = null;
|
||||
state.redisUrlConfigured = configuredUrl;
|
||||
state.redisConnected = false;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export class DataBufferEngine {
|
||||
private static maxRecentEvents = 200;
|
||||
private static maxRecentEvents = REDIS_RECENT_LIMIT;
|
||||
|
||||
public static get batchFlushThreshold(): number {
|
||||
return getState().batchFlushThreshold;
|
||||
@@ -60,11 +167,46 @@ export class DataBufferEngine {
|
||||
this.setFlushInterval(value);
|
||||
}
|
||||
|
||||
public static setRedisClientForTests(client: RedisBufferClient | null | undefined): void {
|
||||
const state = getState();
|
||||
state.redisTestOverride = client;
|
||||
state.redisConnected = Boolean(client);
|
||||
}
|
||||
|
||||
public static async shutdown(): Promise<void> {
|
||||
const state = getState();
|
||||
if (state.flushTimer) {
|
||||
clearInterval(state.flushTimer);
|
||||
state.flushTimer = null;
|
||||
}
|
||||
if (state.pendingRedisWrites.size > 0) {
|
||||
await Promise.allSettled([...state.pendingRedisWrites]);
|
||||
}
|
||||
const client = state.redisClient;
|
||||
state.redisClient = undefined;
|
||||
state.redisUrlConfigured = '';
|
||||
state.redisConnected = false;
|
||||
state.redisTestOverride = undefined;
|
||||
if (client) {
|
||||
try {
|
||||
if (typeof client.quit === 'function') {
|
||||
await client.quit();
|
||||
} else if (typeof client.disconnect === 'function') {
|
||||
client.disconnect();
|
||||
}
|
||||
} catch {
|
||||
if (typeof client.disconnect === 'function') {
|
||||
client.disconnect();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static init() {
|
||||
const state = getState();
|
||||
if (!state.flushTimer) {
|
||||
state.flushTimer = setInterval(() => {
|
||||
void this.flush();
|
||||
void this.flush().catch(() => undefined);
|
||||
}, state.batchFlushIntervalMs);
|
||||
}
|
||||
|
||||
@@ -96,26 +238,18 @@ export class DataBufferEngine {
|
||||
}
|
||||
|
||||
state.flushTimer = setInterval(() => {
|
||||
void this.flush();
|
||||
void this.flush().catch(() => undefined);
|
||||
}, state.batchFlushIntervalMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Push incoming event to in-memory buffer.
|
||||
* If buffer hits threshold (e.g. 500 rows), triggers immediate batch flush.
|
||||
*/
|
||||
public static push(event: DataEvent): void {
|
||||
private static recordLocalMetaAndNotify(event: DataEvent): void {
|
||||
const state = getState();
|
||||
state.buffer.push(event);
|
||||
state.totalIngested += 1;
|
||||
|
||||
// Cache in recent events for live dashboard viewer
|
||||
state.recentEvents.unshift(event);
|
||||
if (state.recentEvents.length > this.maxRecentEvents) {
|
||||
state.recentEvents.pop();
|
||||
}
|
||||
|
||||
// Notify live subscribers
|
||||
for (const sub of state.subscribers) {
|
||||
try {
|
||||
sub(event);
|
||||
@@ -123,43 +257,188 @@ export class DataBufferEngine {
|
||||
// Ignore subscriber error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if buffer threshold reached
|
||||
private static async writeEventsToRedisOrFallback(events: DataEvent[]): Promise<number> {
|
||||
if (events.length === 0) return 0;
|
||||
const state = getState();
|
||||
const redis = getRedisClient();
|
||||
|
||||
if (redis) {
|
||||
const serialized = events.map((e) => JSON.stringify(e));
|
||||
try {
|
||||
const bufferLen = await redis.rpush(REDIS_BUFFER_KEY, ...serialized);
|
||||
await redis.lpush(REDIS_RECENT_KEY, ...serialized);
|
||||
await redis.ltrim(REDIS_RECENT_KEY, 0, REDIS_RECENT_LIMIT - 1);
|
||||
state.redisConnected = true;
|
||||
return Number(bufferLen) + state.buffer.length;
|
||||
} catch {
|
||||
state.redisConnected = false;
|
||||
}
|
||||
}
|
||||
|
||||
state.buffer.push(...events);
|
||||
return state.buffer.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Synchronous-signature push (queues to Redis asynchronously when configured, with automatic RAM fallback).
|
||||
*/
|
||||
public static push(event: DataEvent): void {
|
||||
const state = getState();
|
||||
this.recordLocalMetaAndNotify(event);
|
||||
|
||||
const redis = getRedisClient();
|
||||
if (redis) {
|
||||
const op = this.writeEventsToRedisOrFallback([event]).then((queuedCount) => {
|
||||
if (queuedCount >= state.batchFlushThreshold) {
|
||||
void this.flush().catch(() => undefined);
|
||||
}
|
||||
});
|
||||
state.pendingRedisWrites.add(op);
|
||||
void op.finally(() => {
|
||||
state.pendingRedisWrites.delete(op);
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
state.buffer.push(event);
|
||||
if (state.buffer.length >= state.batchFlushThreshold) {
|
||||
void this.flush();
|
||||
void this.flush().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Flushes current buffer to CSV files via CSVStorageManager
|
||||
* Async single-event push (awaited by API routes so Redis persistence is guaranteed before HTTP response).
|
||||
*/
|
||||
public static async pushAsync(event: DataEvent): Promise<void> {
|
||||
await this.pushBatchAsync([event]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Async batch push with shared Redis durability and RAM fallback.
|
||||
*/
|
||||
public static async pushBatchAsync(events: DataEvent[]): Promise<void> {
|
||||
if (!events || events.length === 0) return;
|
||||
const state = getState();
|
||||
for (const evt of events) {
|
||||
this.recordLocalMetaAndNotify(evt);
|
||||
}
|
||||
|
||||
const queuedCount = await this.writeEventsToRedisOrFallback(events);
|
||||
if (queuedCount >= state.batchFlushThreshold) {
|
||||
void this.flush().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Flushes queued events (shared Redis list under distributed lock + local RAM fallback) to partitioned CSV files.
|
||||
* Redis LTRIM runs ONLY after CSVStorageManager.appendBatch succeeds so crashes never lose data.
|
||||
*/
|
||||
public static async flush(): Promise<{ count: number; files: string[] }> {
|
||||
const state = getState();
|
||||
if (state.buffer.length === 0 || state.isFlushing) {
|
||||
if (state.pendingRedisWrites.size > 0) {
|
||||
await Promise.allSettled([...state.pendingRedisWrites]);
|
||||
}
|
||||
if (state.isFlushing) {
|
||||
return { count: 0, files: [] };
|
||||
}
|
||||
|
||||
state.isFlushing = true;
|
||||
const batchToFlush = [...state.buffer];
|
||||
state.buffer = [];
|
||||
let totalWritten = 0;
|
||||
const filesSet = new Set<string>();
|
||||
|
||||
try {
|
||||
const result = await CSVStorageManager.appendBatch(batchToFlush);
|
||||
state.totalFlushed += result.written;
|
||||
const redis = getRedisClient();
|
||||
if (redis) {
|
||||
const lockToken = crypto.randomUUID();
|
||||
let lockAcquired = false;
|
||||
|
||||
for (const listener of state.flushListeners) {
|
||||
try {
|
||||
listener(result.written, result.filesUpdated);
|
||||
const lockResult = await redis.set(
|
||||
REDIS_FLUSH_LOCK_KEY,
|
||||
lockToken,
|
||||
'PX',
|
||||
FLUSH_LOCK_TTL_MS,
|
||||
'NX',
|
||||
);
|
||||
lockAcquired = lockResult === 'OK';
|
||||
state.redisConnected = true;
|
||||
} catch {
|
||||
// ignore
|
||||
state.redisConnected = false;
|
||||
lockAcquired = false;
|
||||
}
|
||||
|
||||
if (lockAcquired) {
|
||||
try {
|
||||
const chunkSize = Math.max(500, state.batchFlushThreshold);
|
||||
while (true) {
|
||||
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, chunkSize - 1);
|
||||
if (!rawItems || rawItems.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
const parsedBatch: DataEvent[] = [];
|
||||
for (const raw of rawItems) {
|
||||
try {
|
||||
const evt = JSON.parse(raw) as DataEvent;
|
||||
if (evt && evt.eventId) {
|
||||
parsedBatch.push(evt);
|
||||
}
|
||||
} catch {
|
||||
// Ignore malformed legacy entry while still trimming it
|
||||
}
|
||||
}
|
||||
|
||||
if (parsedBatch.length > 0) {
|
||||
const res = await CSVStorageManager.appendBatch(parsedBatch);
|
||||
totalWritten += res.written;
|
||||
for (const f of res.filesUpdated) filesSet.add(f);
|
||||
}
|
||||
|
||||
// Trim strictly AFTER CSVStorageManager.appendBatch resolves safely
|
||||
await redis.ltrim(REDIS_BUFFER_KEY, rawItems.length, -1);
|
||||
|
||||
if (rawItems.length < chunkSize) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
await redis.eval(UNLOCK_LUA_SCRIPT, 1, REDIS_FLUSH_LOCK_KEY, lockToken);
|
||||
} catch {
|
||||
// Lock auto-expires after FLUSH_LOCK_TTL_MS
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { count: result.written, files: result.filesUpdated };
|
||||
} catch (err) {
|
||||
// On failure, re-queue to buffer front for retry
|
||||
state.buffer.unshift(...batchToFlush);
|
||||
throw err;
|
||||
if (state.buffer.length > 0) {
|
||||
const ramBatch = [...state.buffer];
|
||||
state.buffer = [];
|
||||
try {
|
||||
const res = await CSVStorageManager.appendBatch(ramBatch);
|
||||
totalWritten += res.written;
|
||||
for (const f of res.filesUpdated) filesSet.add(f);
|
||||
} catch (err) {
|
||||
state.buffer.unshift(...ramBatch);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
if (totalWritten > 0) {
|
||||
state.totalFlushed += totalWritten;
|
||||
const filesUpdated = Array.from(filesSet);
|
||||
for (const listener of state.flushListeners) {
|
||||
try {
|
||||
listener(totalWritten, filesUpdated);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { count: totalWritten, files: Array.from(filesSet) };
|
||||
} finally {
|
||||
state.isFlushing = false;
|
||||
}
|
||||
@@ -194,10 +473,83 @@ export class DataBufferEngine {
|
||||
return state.recentEvents.slice(0, limit);
|
||||
}
|
||||
|
||||
public static async getRecentEventsAsync(
|
||||
limit = 50,
|
||||
filterClientId?: string,
|
||||
): Promise<DataEvent[]> {
|
||||
const state = getState();
|
||||
const redis = getRedisClient();
|
||||
|
||||
if (redis) {
|
||||
try {
|
||||
const readCap = Math.min(REDIS_RECENT_LIMIT, Math.max(limit * 4, limit));
|
||||
const rawItems = await redis.lrange(REDIS_RECENT_KEY, 0, readCap - 1);
|
||||
state.redisConnected = true;
|
||||
|
||||
const redisEvents: DataEvent[] = [];
|
||||
for (const raw of rawItems) {
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as DataEvent;
|
||||
if (!parsed || !parsed.eventId) continue;
|
||||
if (filterClientId && filterClientId !== 'all' && parsed.clientId !== filterClientId) {
|
||||
continue;
|
||||
}
|
||||
redisEvents.push(parsed);
|
||||
if (redisEvents.length >= limit) break;
|
||||
} catch {
|
||||
// skip malformed JSON entry
|
||||
}
|
||||
}
|
||||
if (redisEvents.length > 0 || state.recentEvents.length === 0) {
|
||||
return redisEvents;
|
||||
}
|
||||
} catch {
|
||||
state.redisConnected = false;
|
||||
}
|
||||
}
|
||||
|
||||
return this.getRecentEvents(limit, filterClientId);
|
||||
}
|
||||
|
||||
public static getBufferStats() {
|
||||
const state = getState();
|
||||
return {
|
||||
currentBufferSize: state.buffer.length,
|
||||
localBufferSize: state.buffer.length,
|
||||
redisBufferSize: 0,
|
||||
redisEnabled: Boolean(getRedisClient()),
|
||||
redisConnected: state.redisConnected,
|
||||
batchFlushThreshold: state.batchFlushThreshold,
|
||||
batchFlushIntervalMs: state.batchFlushIntervalMs,
|
||||
totalIngested: state.totalIngested,
|
||||
totalFlushed: state.totalFlushed,
|
||||
recentEventsCached: state.recentEvents.length,
|
||||
activeSubscribers: state.subscribers.size,
|
||||
isFlushing: state.isFlushing,
|
||||
};
|
||||
}
|
||||
|
||||
public static async getBufferStatsAsync() {
|
||||
const state = getState();
|
||||
const redis = getRedisClient();
|
||||
let redisBufferSize = 0;
|
||||
|
||||
if (redis) {
|
||||
try {
|
||||
redisBufferSize = Number(await redis.llen(REDIS_BUFFER_KEY)) || 0;
|
||||
state.redisConnected = true;
|
||||
} catch {
|
||||
state.redisConnected = false;
|
||||
redisBufferSize = 0;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
currentBufferSize: state.buffer.length + redisBufferSize,
|
||||
localBufferSize: state.buffer.length,
|
||||
redisBufferSize,
|
||||
redisEnabled: Boolean(redis),
|
||||
redisConnected: state.redisConnected,
|
||||
batchFlushThreshold: state.batchFlushThreshold,
|
||||
batchFlushIntervalMs: state.batchFlushIntervalMs,
|
||||
totalIngested: state.totalIngested,
|
||||
|
||||
+315
-47
@@ -1,5 +1,8 @@
|
||||
import { existsSync, rmSync } from "node:fs";
|
||||
import { resolve, sep } from "node:path";
|
||||
import { AuthService } from "../auth/jwt";
|
||||
import { DatabaseEngine } from "../Database/database";
|
||||
import { getInternalIngestionHeaders } from "../Security/security";
|
||||
|
||||
export interface ClientSDK {
|
||||
clientId: string;
|
||||
@@ -15,7 +18,9 @@ export interface ClientSDK {
|
||||
|
||||
type SDKDatabaseGlobals = typeof globalThis & {
|
||||
__dataStackClientsMap?: Map<string, ClientSDK>;
|
||||
__dataStackDeletedClients?: Set<string>;
|
||||
__dataStackClientsHydrated?: boolean;
|
||||
__dataStackSyncedSocketEvents?: Map<string, number>;
|
||||
};
|
||||
|
||||
const globals = globalThis as SDKDatabaseGlobals;
|
||||
@@ -27,6 +32,67 @@ function getClientsMap(): Map<string, ClientSDK> {
|
||||
return globals.__dataStackClientsMap;
|
||||
}
|
||||
|
||||
function getDeletedClientsSet(): Set<string> {
|
||||
if (!globals.__dataStackDeletedClients) {
|
||||
globals.__dataStackDeletedClients = new Set<string>();
|
||||
}
|
||||
return globals.__dataStackDeletedClients;
|
||||
}
|
||||
|
||||
function getSyncedSocketEventsMap(): Map<string, number> {
|
||||
if (!globals.__dataStackSyncedSocketEvents) {
|
||||
globals.__dataStackSyncedSocketEvents = new Map<string, number>();
|
||||
}
|
||||
return globals.__dataStackSyncedSocketEvents;
|
||||
}
|
||||
|
||||
function serializeSyncClient(client: ClientSDK) {
|
||||
return {
|
||||
clientId: client.clientId,
|
||||
name: client.name,
|
||||
status: client.status,
|
||||
token: client.token,
|
||||
rateLimit: client.rateLimit,
|
||||
};
|
||||
}
|
||||
|
||||
async function notifyIngestionClientSyncAsync(
|
||||
clientId: string,
|
||||
action: "deleted" | "revoked" | "updated" | "registered",
|
||||
client?: ClientSDK,
|
||||
): Promise<void> {
|
||||
const headers = getInternalIngestionHeaders();
|
||||
if (!headers["x-datastack-internal-secret"]) return;
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||
).replace(/\/$/, "");
|
||||
try {
|
||||
await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...headers,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
clientId,
|
||||
action,
|
||||
...(client ? { client: serializeSyncClient(client) } : {}),
|
||||
}),
|
||||
signal: AbortSignal.timeout(1000),
|
||||
});
|
||||
} catch {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
}
|
||||
}
|
||||
|
||||
function notifyIngestionClientSync(
|
||||
clientId: string,
|
||||
action: "deleted" | "revoked" | "updated" | "registered",
|
||||
client?: ClientSDK,
|
||||
): void {
|
||||
void notifyIngestionClientSyncAsync(clientId, action, client);
|
||||
}
|
||||
|
||||
function mapRowToClient(row: Record<string, unknown>): ClientSDK {
|
||||
return {
|
||||
clientId: String(row.client_id),
|
||||
@@ -54,11 +120,20 @@ export class SDKManager {
|
||||
const ready = await DatabaseEngine.ensureSchema();
|
||||
if (!ready) return null;
|
||||
|
||||
const result = await pool.query(
|
||||
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
|
||||
FROM client_sdks
|
||||
ORDER BY created_at DESC`,
|
||||
);
|
||||
const [result, deletedResult] = await Promise.all([
|
||||
pool.query(
|
||||
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
|
||||
FROM client_sdks
|
||||
ORDER BY created_at DESC`,
|
||||
),
|
||||
pool.query(`SELECT client_id FROM deleted_clients`),
|
||||
]);
|
||||
|
||||
const deletedSet = getDeletedClientsSet();
|
||||
deletedSet.clear();
|
||||
for (const row of deletedResult.rows) {
|
||||
deletedSet.add(String(row.client_id).trim().toLowerCase());
|
||||
}
|
||||
|
||||
const map = getClientsMap();
|
||||
map.clear();
|
||||
@@ -84,12 +159,91 @@ export class SDKManager {
|
||||
return Array.from(getClientsMap().values());
|
||||
}
|
||||
|
||||
public static async syncAllClientsToIngestionAsync(): Promise<void> {
|
||||
const headers = getInternalIngestionHeaders();
|
||||
if (!headers["x-datastack-internal-secret"]) return;
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||
).replace(/\/$/, "");
|
||||
const clients = Array.from(getClientsMap().values()).map(serializeSyncClient);
|
||||
try {
|
||||
await fetch(`${ingestionBaseUrl}/internal/client-sync`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...headers,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
action: "bulk_sync",
|
||||
clients,
|
||||
}),
|
||||
signal: AbortSignal.timeout(1500),
|
||||
});
|
||||
} catch {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
}
|
||||
}
|
||||
|
||||
public static async listClientsAsync(): Promise<ClientSDK[]> {
|
||||
const fromDb = await this.syncFromDatabase();
|
||||
if (fromDb !== null) {
|
||||
return fromDb;
|
||||
}
|
||||
return Array.from(getClientsMap().values());
|
||||
|
||||
const map = getClientsMap();
|
||||
const headers = getInternalIngestionHeaders();
|
||||
if (headers["x-datastack-internal-secret"]) {
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||
).replace(/\/$/, "");
|
||||
try {
|
||||
const res = await fetch(`${ingestionBaseUrl}/internal/metrics`, {
|
||||
headers,
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(1000),
|
||||
});
|
||||
if (res.ok) {
|
||||
const data = (await res.json()) as {
|
||||
hasSyncedInMemoryClients?: boolean;
|
||||
clientActivity?: Record<
|
||||
string,
|
||||
{ totalEvents?: number; lastActive?: string }
|
||||
>;
|
||||
};
|
||||
if (data.hasSyncedInMemoryClients === false && map.size > 0) {
|
||||
await this.syncAllClientsToIngestionAsync();
|
||||
}
|
||||
if (data.clientActivity && typeof data.clientActivity === "object") {
|
||||
const syncedMap = getSyncedSocketEventsMap();
|
||||
for (const [cid, activity] of Object.entries(data.clientActivity)) {
|
||||
const client = map.get(cid);
|
||||
if (!client) continue;
|
||||
const remoteTotal = Math.max(0, Number(activity?.totalEvents) || 0);
|
||||
const prevSynced = syncedMap.get(cid) || 0;
|
||||
const delta =
|
||||
remoteTotal >= prevSynced
|
||||
? remoteTotal - prevSynced
|
||||
: remoteTotal;
|
||||
if (delta > 0) {
|
||||
client.totalEvents += delta;
|
||||
}
|
||||
syncedMap.set(cid, remoteTotal);
|
||||
if (
|
||||
activity?.lastActive &&
|
||||
(!client.lastActive || activity.lastActive > client.lastActive)
|
||||
) {
|
||||
client.lastActive = activity.lastActive;
|
||||
}
|
||||
map.set(cid, client);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(map.values());
|
||||
}
|
||||
|
||||
public static getClient(clientId: string): ClientSDK | undefined {
|
||||
@@ -152,8 +306,10 @@ export class SDKManager {
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
getDeletedClientsSet().delete(safeClientId);
|
||||
getClientsMap().set(safeClientId, newClient);
|
||||
void this.persistNewClient(newClient);
|
||||
notifyIngestionClientSync(safeClientId, "registered", newClient);
|
||||
return newClient;
|
||||
}
|
||||
|
||||
@@ -163,6 +319,10 @@ export class SDKManager {
|
||||
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
await pool.query(
|
||||
`DELETE FROM deleted_clients WHERE client_id = $1`,
|
||||
[client.clientId],
|
||||
);
|
||||
await pool.query(
|
||||
`INSERT INTO client_sdks (client_id, name, secret, token, rate_limit, status, total_events, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW())
|
||||
@@ -218,8 +378,10 @@ export class SDKManager {
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
getDeletedClientsSet().delete(safeClientId);
|
||||
getClientsMap().set(safeClientId, newClient);
|
||||
await this.persistNewClient(newClient);
|
||||
await notifyIngestionClientSyncAsync(safeClientId, "registered", newClient);
|
||||
return newClient;
|
||||
}
|
||||
|
||||
@@ -227,16 +389,20 @@ export class SDKManager {
|
||||
const client = getClientsMap().get(clientId);
|
||||
if (!client) return null;
|
||||
|
||||
client.secret = `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`;
|
||||
client.token = AuthService.generateClientToken({
|
||||
clientId: client.clientId,
|
||||
name: client.name,
|
||||
rateLimit: client.rateLimit,
|
||||
});
|
||||
const updated: ClientSDK = {
|
||||
...client,
|
||||
secret: `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`,
|
||||
token: AuthService.generateClientToken({
|
||||
clientId: client.clientId,
|
||||
name: client.name,
|
||||
rateLimit: client.rateLimit,
|
||||
}),
|
||||
};
|
||||
|
||||
getClientsMap().set(clientId, client);
|
||||
void this.persistClientCredentials(client);
|
||||
return client;
|
||||
getClientsMap().set(clientId, updated);
|
||||
void this.persistClientCredentials(updated);
|
||||
notifyIngestionClientSync(clientId, "updated", updated);
|
||||
return updated;
|
||||
}
|
||||
|
||||
private static async persistClientCredentials(
|
||||
@@ -264,16 +430,20 @@ export class SDKManager {
|
||||
const existing = await this.getClientAsync(clientId);
|
||||
if (!existing) return null;
|
||||
|
||||
existing.secret = `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`;
|
||||
existing.token = AuthService.generateClientToken({
|
||||
clientId: existing.clientId,
|
||||
name: existing.name,
|
||||
rateLimit: existing.rateLimit,
|
||||
});
|
||||
const updated: ClientSDK = {
|
||||
...existing,
|
||||
secret: `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`,
|
||||
token: AuthService.generateClientToken({
|
||||
clientId: existing.clientId,
|
||||
name: existing.name,
|
||||
rateLimit: existing.rateLimit,
|
||||
}),
|
||||
};
|
||||
|
||||
getClientsMap().set(clientId, existing);
|
||||
await this.persistClientCredentials(existing);
|
||||
return existing;
|
||||
getClientsMap().set(clientId, updated);
|
||||
await this.persistClientCredentials(updated);
|
||||
await notifyIngestionClientSyncAsync(clientId, "updated", updated);
|
||||
return updated;
|
||||
}
|
||||
|
||||
public static updateClient(
|
||||
@@ -283,14 +453,23 @@ export class SDKManager {
|
||||
const client = getClientsMap().get(clientId);
|
||||
if (!client) return null;
|
||||
|
||||
if (updates.name !== undefined) client.name = updates.name.trim();
|
||||
if (updates.rateLimit !== undefined)
|
||||
client.rateLimit = Math.max(1, Number(updates.rateLimit) || 100);
|
||||
if (updates.status !== undefined) client.status = updates.status;
|
||||
const updated: ClientSDK = {
|
||||
...client,
|
||||
...(updates.name !== undefined ? { name: updates.name.trim() } : {}),
|
||||
...(updates.rateLimit !== undefined
|
||||
? { rateLimit: Math.max(1, Number(updates.rateLimit) || 100) }
|
||||
: {}),
|
||||
...(updates.status !== undefined ? { status: updates.status } : {}),
|
||||
};
|
||||
|
||||
getClientsMap().set(clientId, client);
|
||||
void this.persistClientUpdates(client);
|
||||
return client;
|
||||
getClientsMap().set(clientId, updated);
|
||||
void this.persistClientUpdates(updated);
|
||||
notifyIngestionClientSync(
|
||||
clientId,
|
||||
updated.status === "revoked" ? "revoked" : "updated",
|
||||
updated,
|
||||
);
|
||||
return updated;
|
||||
}
|
||||
|
||||
private static async persistClientUpdates(client: ClientSDK): Promise<void> {
|
||||
@@ -317,19 +496,67 @@ export class SDKManager {
|
||||
const client = await this.getClientAsync(clientId);
|
||||
if (!client) return null;
|
||||
|
||||
if (updates.name !== undefined) client.name = updates.name.trim();
|
||||
if (updates.rateLimit !== undefined)
|
||||
client.rateLimit = Math.max(1, Number(updates.rateLimit) || 100);
|
||||
if (updates.status !== undefined) client.status = updates.status;
|
||||
const updated: ClientSDK = {
|
||||
...client,
|
||||
...(updates.name !== undefined ? { name: updates.name.trim() } : {}),
|
||||
...(updates.rateLimit !== undefined
|
||||
? { rateLimit: Math.max(1, Number(updates.rateLimit) || 100) }
|
||||
: {}),
|
||||
...(updates.status !== undefined ? { status: updates.status } : {}),
|
||||
};
|
||||
|
||||
getClientsMap().set(clientId, client);
|
||||
await this.persistClientUpdates(client);
|
||||
return client;
|
||||
getClientsMap().set(clientId, updated);
|
||||
await this.persistClientUpdates(updated);
|
||||
await notifyIngestionClientSyncAsync(
|
||||
clientId,
|
||||
updated.status === "revoked" ? "revoked" : "updated",
|
||||
updated,
|
||||
);
|
||||
return updated;
|
||||
}
|
||||
|
||||
public static isClientDeleted(clientId: string): boolean {
|
||||
const normalized = String(clientId || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
return getDeletedClientsSet().has(normalized);
|
||||
}
|
||||
|
||||
public static async isClientDeletedAsync(clientId: string): Promise<boolean> {
|
||||
const normalized = String(clientId || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
if (!normalized) return false;
|
||||
if (getDeletedClientsSet().has(normalized)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (!pool) return false;
|
||||
|
||||
await DatabaseEngine.ensureSchema();
|
||||
const res = await pool.query(
|
||||
`SELECT 1 FROM deleted_clients WHERE client_id = $1`,
|
||||
[normalized],
|
||||
);
|
||||
if (res.rowCount && res.rowCount > 0) {
|
||||
getDeletedClientsSet().add(normalized);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public static deleteClient(clientId: string): boolean {
|
||||
const deleted = getClientsMap().delete(clientId);
|
||||
void this.deleteClientFromDb(clientId);
|
||||
const normalized = String(clientId || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const deleted = getClientsMap().delete(normalized);
|
||||
getSyncedSocketEventsMap().delete(normalized);
|
||||
if (deleted) {
|
||||
getDeletedClientsSet().add(normalized);
|
||||
}
|
||||
void this.deleteClientFromDb(normalized);
|
||||
notifyIngestionClientSync(normalized, "deleted");
|
||||
return deleted;
|
||||
}
|
||||
|
||||
@@ -337,21 +564,62 @@ export class SDKManager {
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (!pool) return false;
|
||||
|
||||
await DatabaseEngine.ensureSchema();
|
||||
const dbClient = await pool.connect();
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
const res = await pool.query(
|
||||
await dbClient.query("BEGIN");
|
||||
const res = await dbClient.query(
|
||||
`DELETE FROM client_sdks WHERE client_id = $1`,
|
||||
[clientId],
|
||||
);
|
||||
return Boolean(res.rowCount && res.rowCount > 0);
|
||||
} catch {
|
||||
if (res.rowCount && res.rowCount > 0) {
|
||||
await dbClient.query(
|
||||
`INSERT INTO deleted_clients (client_id, deleted_at)
|
||||
VALUES ($1, NOW())
|
||||
ON CONFLICT (client_id) DO UPDATE SET deleted_at = NOW()`,
|
||||
[clientId],
|
||||
);
|
||||
await dbClient.query("COMMIT");
|
||||
return true;
|
||||
}
|
||||
await dbClient.query("ROLLBACK");
|
||||
return false;
|
||||
} catch {
|
||||
await dbClient.query("ROLLBACK").catch(() => undefined);
|
||||
return false;
|
||||
} finally {
|
||||
dbClient.release();
|
||||
}
|
||||
}
|
||||
|
||||
public static async deleteClientAsync(clientId: string): Promise<boolean> {
|
||||
const existedInMem = getClientsMap().delete(clientId);
|
||||
const deletedFromDb = await this.deleteClientFromDb(clientId);
|
||||
private static purgeClientStorageDir(clientId: string): void {
|
||||
const safeId = clientId.replace(/[^a-z0-9_-]/gi, "_");
|
||||
if (!safeId) return;
|
||||
const storageRoot = resolve(process.cwd(), "storage", "data");
|
||||
const targetDir = resolve(storageRoot, safeId);
|
||||
if (!targetDir.startsWith(`${storageRoot}${sep}`)) return;
|
||||
if (existsSync(targetDir)) {
|
||||
rmSync(targetDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
public static async deleteClientAsync(
|
||||
clientId: string,
|
||||
options?: { purgeFiles?: boolean },
|
||||
): Promise<boolean> {
|
||||
const normalized = String(clientId || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const existedInMem = getClientsMap().delete(normalized);
|
||||
getSyncedSocketEventsMap().delete(normalized);
|
||||
const deletedFromDb = await this.deleteClientFromDb(normalized);
|
||||
if (existedInMem || deletedFromDb) {
|
||||
getDeletedClientsSet().add(normalized);
|
||||
if (options?.purgeFiles) {
|
||||
this.purgeClientStorageDir(normalized);
|
||||
}
|
||||
await notifyIngestionClientSyncAsync(normalized, "deleted");
|
||||
}
|
||||
return existedInMem || deletedFromDb;
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user