refactor: switch SDK methods to asynchronous
Migrate SDK interactions to asynchronous calls, relax admin setup email constraints, and add event retrieval functionality.
This commit is contained in:
1 parent
81c5e4e2fd
commit
bb938c33c0
36 files changed
+1679
-754
No files matched your search
+117
-12
@@ -1,28 +1,133 @@
|
||||
import { Pool } from "pg";
|
||||
import { SettingsManager } from "../adminSettings/settingsManager";
|
||||
import { SDKManager } from "../sdk/sdkManager";
|
||||
|
||||
type CoreDatabaseGlobals = typeof globalThis & {
|
||||
__dataStackCorePool?: Pool | null;
|
||||
__dataStackCoreSchema?: Promise<void>;
|
||||
};
|
||||
|
||||
const globals = globalThis as CoreDatabaseGlobals;
|
||||
|
||||
/**
|
||||
* Database Module (PostgreSQL + Prisma abstraction)
|
||||
* Database Module (PostgreSQL Persistence Engine)
|
||||
* CRITICAL RULE: PostgreSQL stores strictly administrative metadata,
|
||||
* client registrations, and configuration. Collected data is NEVER stored in PostgreSQL.
|
||||
* client registrations (client_sdks), and configuration (admin_settings).
|
||||
* Collected telemetry streams are NEVER stored in PostgreSQL.
|
||||
*/
|
||||
export class DatabaseEngine {
|
||||
public static isConnected = true;
|
||||
public static getPool(): Pool | null {
|
||||
const connectionString = process.env.DATABASE_URL;
|
||||
if (!connectionString) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (globals.__dataStackCorePool === undefined) {
|
||||
try {
|
||||
globals.__dataStackCorePool = new Pool({
|
||||
connectionString,
|
||||
connectionTimeoutMillis: 3000,
|
||||
idleTimeoutMillis: 30000,
|
||||
max: 5,
|
||||
});
|
||||
} catch {
|
||||
globals.__dataStackCorePool = null;
|
||||
}
|
||||
}
|
||||
|
||||
return globals.__dataStackCorePool ?? null;
|
||||
}
|
||||
|
||||
public static async ensureSchema(): Promise<boolean> {
|
||||
const pool = this.getPool();
|
||||
if (!pool) return false;
|
||||
|
||||
globals.__dataStackCoreSchema ??= Promise.all([
|
||||
pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS client_sdks (
|
||||
client_id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
secret TEXT NOT NULL,
|
||||
token TEXT NOT NULL,
|
||||
rate_limit INTEGER NOT NULL DEFAULT 100,
|
||||
status TEXT NOT NULL DEFAULT 'active',
|
||||
total_events BIGINT NOT NULL DEFAULT 0,
|
||||
last_active TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`),
|
||||
pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS admin_settings (
|
||||
id TEXT PRIMARY KEY DEFAULT 'default',
|
||||
server_port INTEGER NOT NULL DEFAULT 4000,
|
||||
web_port INTEGER NOT NULL DEFAULT 3000,
|
||||
max_events_per_sec INTEGER NOT NULL DEFAULT 100,
|
||||
max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
|
||||
batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
|
||||
batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
|
||||
file_rotation_mb INTEGER NOT NULL DEFAULT 100,
|
||||
jwt_secret TEXT NOT NULL,
|
||||
enable_idempotency BOOLEAN NOT NULL DEFAULT true,
|
||||
auto_prune_days INTEGER NOT NULL DEFAULT 30,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`),
|
||||
]).then(() => undefined);
|
||||
|
||||
try {
|
||||
await globals.__dataStackCoreSchema;
|
||||
return true;
|
||||
} catch {
|
||||
globals.__dataStackCoreSchema = undefined;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public static async getHealth() {
|
||||
return {
|
||||
status: "healthy",
|
||||
type: "PostgreSQL + Prisma Schema Managed",
|
||||
database: "datastack_db",
|
||||
role: "Metadata & SDK Registrations Only",
|
||||
collectedDataStorage: "Strictly CSV (/storage/data)",
|
||||
};
|
||||
const pool = this.getPool();
|
||||
if (!pool) {
|
||||
return {
|
||||
status: "healthy" as const,
|
||||
storageMode: "In-Memory" as const,
|
||||
type: "In-Memory Fallback (DATABASE_URL not set)",
|
||||
database: "memory",
|
||||
role: "Metadata & SDK Registrations Only",
|
||||
collectedDataStorage: "Strictly CSV (/storage/data)",
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
await this.ensureSchema();
|
||||
await pool.query("SELECT 1");
|
||||
return {
|
||||
status: "healthy" as const,
|
||||
storageMode: "PostgreSQL" as const,
|
||||
type: "PostgreSQL Managed Schema",
|
||||
database: "datastack_db",
|
||||
role: "Metadata & SDK Registrations Only",
|
||||
collectedDataStorage: "Strictly CSV (/storage/data)",
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
status: "degraded" as const,
|
||||
storageMode: "In-Memory" as const,
|
||||
type: "PostgreSQL Unreachable (In-Memory Fallback Active)",
|
||||
database: "datastack_db",
|
||||
role: "Metadata & SDK Registrations Only",
|
||||
collectedDataStorage: "Strictly CSV (/storage/data)",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
public static async syncState() {
|
||||
const [clients, settings] = await Promise.all([
|
||||
SDKManager.listClientsAsync(),
|
||||
SettingsManager.getPublicSettingsAsync(),
|
||||
]);
|
||||
return {
|
||||
clientsCount: SDKManager.listClients().length,
|
||||
settings: SettingsManager.getPublicSettings(),
|
||||
clientsCount: clients.length,
|
||||
settings,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
import express from 'express';
|
||||
import http from 'http';
|
||||
import cors from 'cors';
|
||||
import { Server as SocketIOServer } from 'socket.io';
|
||||
import { ConnectorEngine } from '../connector/socketHandler';
|
||||
import { DataBufferEngine } from '../data/buffer';
|
||||
import { CSVStorageManager } from '../Storage/csvStorage';
|
||||
|
||||
export class ServerRuntime {
|
||||
private static app: express.Express | null = null;
|
||||
private static httpServer: http.Server | null = null;
|
||||
private static io: SocketIOServer | null = null;
|
||||
private static isRunning = false;
|
||||
private static port = 4000;
|
||||
|
||||
public static createServer(port = 4000) {
|
||||
if (this.isRunning && this.httpServer) {
|
||||
return { app: this.app, server: this.httpServer, io: this.io };
|
||||
}
|
||||
|
||||
this.port = port;
|
||||
const app = express();
|
||||
app.use(cors({ origin: '*' }));
|
||||
app.use(express.json({ limit: '10mb' }));
|
||||
|
||||
// Health route
|
||||
app.get('/health', (_req, res) => {
|
||||
res.json({
|
||||
status: 'online',
|
||||
service: 'DataStack Express + Socket.IO Intake',
|
||||
port: this.port,
|
||||
activeSockets: ConnectorEngine.getActiveConnectionsCount(),
|
||||
uptime: process.uptime(),
|
||||
});
|
||||
});
|
||||
|
||||
// HTTP fallback intake endpoint for non-socket clients
|
||||
app.post('/api/intake', (req, res) => {
|
||||
const authHeader = req.headers.authorization;
|
||||
if (!authHeader) {
|
||||
res.status(401).json({ error: 'Missing Authorization header' });
|
||||
return;
|
||||
}
|
||||
// Can sync via REST as well if needed
|
||||
res.json({ success: true, message: 'REST sync supported' });
|
||||
});
|
||||
|
||||
const httpServer = http.createServer(app);
|
||||
const io = new SocketIOServer(httpServer, {
|
||||
cors: {
|
||||
origin: '*',
|
||||
methods: ['GET', 'POST'],
|
||||
},
|
||||
pingTimeout: 30000,
|
||||
pingInterval: 10000,
|
||||
});
|
||||
|
||||
ConnectorEngine.initialize(io);
|
||||
|
||||
this.app = app;
|
||||
this.httpServer = httpServer;
|
||||
this.io = io;
|
||||
this.isRunning = true;
|
||||
|
||||
return { app, server: httpServer, io };
|
||||
}
|
||||
|
||||
public static start(port = 4000): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
const { server } = this.createServer(port);
|
||||
server.listen(port, () => {
|
||||
resolve(port);
|
||||
});
|
||||
server.on('error', (err: NodeJS.ErrnoException) => {
|
||||
if (err.code === 'EADDRINUSE') {
|
||||
// Already listening or address in use
|
||||
resolve(port);
|
||||
} else {
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
} catch (e) {
|
||||
reject(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public static getStatus() {
|
||||
return {
|
||||
isRunning: this.isRunning,
|
||||
port: this.port,
|
||||
activeSockets: ConnectorEngine.getActiveConnectionsCount(),
|
||||
bufferSize: DataBufferEngine.getBufferStats().currentBufferSize,
|
||||
storageStats: CSVStorageManager.getStorageStats(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export default ServerRuntime;
|
||||
@@ -167,6 +167,38 @@ export class CSVStorageManager {
|
||||
return results.sort((a, b) => b.modifiedAt.getTime() - a.modifiedAt.getTime());
|
||||
}
|
||||
|
||||
/**
|
||||
* Prune CSV partitions older than autoPruneDays
|
||||
*/
|
||||
public static pruneExpiredPartitions(autoPruneDays: number): number {
|
||||
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const cutoffDate = new Date(
|
||||
Date.now() - Math.floor(autoPruneDays) * 86_400_000,
|
||||
)
|
||||
.toISOString()
|
||||
.split('T')[0];
|
||||
|
||||
let prunedCount = 0;
|
||||
const files = this.listAllFiles();
|
||||
for (const file of files) {
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(file.date) && file.date < cutoffDate) {
|
||||
try {
|
||||
if (fs.existsSync(file.filePath)) {
|
||||
fs.unlinkSync(file.filePath);
|
||||
prunedCount += 1;
|
||||
}
|
||||
} catch {
|
||||
// Ignore file removal error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return prunedCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get aggregate storage statistics
|
||||
*/
|
||||
@@ -175,6 +207,10 @@ export class CSVStorageManager {
|
||||
const totalBytes = files.reduce((acc, f) => acc + f.sizeBytes, 0);
|
||||
const totalRows = files.reduce((acc, f) => acc + f.linesCount, 0);
|
||||
const uniqueClients = new Set(files.map((f) => f.clientId)).size;
|
||||
const rotationMb = Math.max(
|
||||
1,
|
||||
Math.round(this.maxFileSizeBytes / (1024 * 1024)),
|
||||
);
|
||||
|
||||
return {
|
||||
totalFiles: files.length,
|
||||
@@ -183,7 +219,8 @@ export class CSVStorageManager {
|
||||
totalRows,
|
||||
uniqueClients,
|
||||
storageDirectory: this.baseStorageDir,
|
||||
rotationRule: 'Per-day OR per-100MB rotation',
|
||||
rotationMb,
|
||||
rotationRule: `${rotationMb}MB / Daily`,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { AuthService } from "../auth/jwt";
|
||||
import { DataBufferEngine } from "../data/buffer";
|
||||
import { DatabaseEngine } from "../Database/database";
|
||||
import { SDKManager } from "../sdk/sdkManager";
|
||||
import { SecurityEngine } from "../Security/security";
|
||||
import { CSVStorageManager } from "../Storage/csvStorage";
|
||||
|
||||
@@ -17,10 +19,17 @@ export interface AdminSettings {
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export class SettingsManager {
|
||||
private static settings: AdminSettings = {
|
||||
serverPort: 4000,
|
||||
webPort: 3000,
|
||||
type SettingsGlobals = typeof globalThis & {
|
||||
__dataStackAdminSettings?: AdminSettings;
|
||||
__dataStackSettingsHydrated?: boolean;
|
||||
};
|
||||
|
||||
const globals = globalThis as SettingsGlobals;
|
||||
|
||||
function getDefaultSettings(): AdminSettings {
|
||||
return {
|
||||
serverPort: Number(process.env.PORT_SOCKET) || 4000,
|
||||
webPort: Number(process.env.PORT) || 3000,
|
||||
maxEventsPerSec: 100,
|
||||
maxPayloadSizeKb: 100,
|
||||
batchFlushThreshold: 500,
|
||||
@@ -33,13 +42,160 @@ export class SettingsManager {
|
||||
autoPruneDays: 30,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
function getSettingsState(): AdminSettings {
|
||||
if (!globals.__dataStackAdminSettings) {
|
||||
globals.__dataStackAdminSettings = getDefaultSettings();
|
||||
}
|
||||
return globals.__dataStackAdminSettings;
|
||||
}
|
||||
|
||||
export class SettingsManager {
|
||||
private static applyRuntimeSettings(settings: AdminSettings) {
|
||||
SecurityEngine.maxEventsPerSec = settings.maxEventsPerSec;
|
||||
SecurityEngine.enableIdempotency = settings.enableIdempotency;
|
||||
SecurityEngine.maxPayloadSizeBytes = settings.maxPayloadSizeKb * 1024;
|
||||
DataBufferEngine.batchFlushThreshold = settings.batchFlushThreshold;
|
||||
DataBufferEngine.setFlushInterval(settings.batchFlushIntervalMs);
|
||||
if (settings.jwtSecret) {
|
||||
AuthService.setSecret(settings.jwtSecret);
|
||||
}
|
||||
CSVStorageManager.maxFileSizeBytes = settings.fileRotationMb * 1024 * 1024;
|
||||
CSVStorageManager.pruneExpiredPartitions(settings.autoPruneDays);
|
||||
}
|
||||
|
||||
private static async syncFromDatabase(): Promise<AdminSettings | null> {
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (!pool) return null;
|
||||
|
||||
try {
|
||||
const ready = await DatabaseEngine.ensureSchema();
|
||||
if (!ready) return null;
|
||||
|
||||
const result = await pool.query(
|
||||
`SELECT server_port, web_port, max_events_per_sec, max_payload_size_kb,
|
||||
batch_flush_threshold, batch_flush_interval_ms, file_rotation_mb,
|
||||
jwt_secret, enable_idempotency, auto_prune_days, updated_at
|
||||
FROM admin_settings
|
||||
WHERE id = 'default'`,
|
||||
);
|
||||
|
||||
if (result.rowCount && result.rowCount > 0) {
|
||||
const row = result.rows[0];
|
||||
const loaded: AdminSettings = {
|
||||
serverPort: Number(row.server_port) || 4000,
|
||||
webPort: Number(row.web_port) || 3000,
|
||||
maxEventsPerSec: Number(row.max_events_per_sec) || 100,
|
||||
maxPayloadSizeKb: Number(row.max_payload_size_kb) || 100,
|
||||
batchFlushThreshold: Number(row.batch_flush_threshold) || 500,
|
||||
batchFlushIntervalMs: Number(row.batch_flush_interval_ms) || 2000,
|
||||
fileRotationMb: Number(row.file_rotation_mb) || 100,
|
||||
jwtSecret:
|
||||
String(row.jwt_secret || "") ||
|
||||
process.env.DATASTACK_JWT_SECRET ||
|
||||
"datastack-jwt-default-secret-key-32-chars-long",
|
||||
enableIdempotency: Boolean(row.enable_idempotency),
|
||||
autoPruneDays: Number(row.auto_prune_days) || 30,
|
||||
updatedAt: row.updated_at
|
||||
? new Date(row.updated_at as string | Date).toISOString()
|
||||
: new Date().toISOString(),
|
||||
};
|
||||
globals.__dataStackAdminSettings = loaded;
|
||||
globals.__dataStackSettingsHydrated = true;
|
||||
this.applyRuntimeSettings(loaded);
|
||||
return loaded;
|
||||
}
|
||||
|
||||
// Initialize default row in PostgreSQL if not present yet
|
||||
const defaults = getSettingsState();
|
||||
await this.persistSettingsToDb(defaults);
|
||||
globals.__dataStackSettingsHydrated = true;
|
||||
return defaults;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static async persistSettingsToDb(
|
||||
settings: AdminSettings,
|
||||
): Promise<void> {
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (!pool) return;
|
||||
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
await pool.query(
|
||||
`INSERT INTO admin_settings (
|
||||
id, server_port, web_port, max_events_per_sec, max_payload_size_kb,
|
||||
batch_flush_threshold, batch_flush_interval_ms, file_rotation_mb,
|
||||
jwt_secret, enable_idempotency, auto_prune_days, updated_at
|
||||
) VALUES ('default', $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, NOW())
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
server_port = EXCLUDED.server_port,
|
||||
web_port = EXCLUDED.web_port,
|
||||
max_events_per_sec = EXCLUDED.max_events_per_sec,
|
||||
max_payload_size_kb = EXCLUDED.max_payload_size_kb,
|
||||
batch_flush_threshold = EXCLUDED.batch_flush_threshold,
|
||||
batch_flush_interval_ms = EXCLUDED.batch_flush_interval_ms,
|
||||
file_rotation_mb = EXCLUDED.file_rotation_mb,
|
||||
jwt_secret = EXCLUDED.jwt_secret,
|
||||
enable_idempotency = EXCLUDED.enable_idempotency,
|
||||
auto_prune_days = EXCLUDED.auto_prune_days,
|
||||
updated_at = NOW()`,
|
||||
[
|
||||
settings.serverPort,
|
||||
settings.webPort,
|
||||
settings.maxEventsPerSec,
|
||||
settings.maxPayloadSizeKb,
|
||||
settings.batchFlushThreshold,
|
||||
settings.batchFlushIntervalMs,
|
||||
settings.fileRotationMb,
|
||||
settings.jwtSecret,
|
||||
settings.enableIdempotency,
|
||||
settings.autoPruneDays,
|
||||
],
|
||||
);
|
||||
} catch {
|
||||
// Ignore DB failure; in-memory state remains active
|
||||
}
|
||||
}
|
||||
|
||||
public static getSettings(): AdminSettings {
|
||||
return { ...this.settings };
|
||||
if (!globals.__dataStackSettingsHydrated) {
|
||||
void this.syncFromDatabase();
|
||||
}
|
||||
return { ...getSettingsState() };
|
||||
}
|
||||
|
||||
public static async getSettingsAsync(): Promise<AdminSettings> {
|
||||
const fromDb = await this.syncFromDatabase();
|
||||
if (fromDb) {
|
||||
return { ...fromDb };
|
||||
}
|
||||
return { ...getSettingsState() };
|
||||
}
|
||||
|
||||
public static getPublicSettings(): Omit<AdminSettings, "jwtSecret"> {
|
||||
const settings = this.settings;
|
||||
const settings = this.getSettings();
|
||||
return {
|
||||
serverPort: settings.serverPort,
|
||||
webPort: settings.webPort,
|
||||
maxEventsPerSec: settings.maxEventsPerSec,
|
||||
maxPayloadSizeKb: settings.maxPayloadSizeKb,
|
||||
batchFlushThreshold: settings.batchFlushThreshold,
|
||||
batchFlushIntervalMs: settings.batchFlushIntervalMs,
|
||||
fileRotationMb: settings.fileRotationMb,
|
||||
enableIdempotency: settings.enableIdempotency,
|
||||
autoPruneDays: settings.autoPruneDays,
|
||||
updatedAt: settings.updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
public static async getPublicSettingsAsync(): Promise<
|
||||
Omit<AdminSettings, "jwtSecret">
|
||||
> {
|
||||
const settings = await this.getSettingsAsync();
|
||||
return {
|
||||
serverPort: settings.serverPort,
|
||||
webPort: settings.webPort,
|
||||
@@ -55,36 +211,41 @@ export class SettingsManager {
|
||||
}
|
||||
|
||||
public static updateSettings(updates: Partial<AdminSettings>): AdminSettings {
|
||||
this.settings = {
|
||||
...this.settings,
|
||||
const current = getSettingsState();
|
||||
const jwtSecretChanged =
|
||||
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
|
||||
const nextSettings: AdminSettings = {
|
||||
...current,
|
||||
...updates,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
globals.__dataStackAdminSettings = nextSettings;
|
||||
this.applyRuntimeSettings(nextSettings);
|
||||
if (jwtSecretChanged) {
|
||||
SDKManager.resignAllClientTokens();
|
||||
}
|
||||
void this.persistSettingsToDb(nextSettings);
|
||||
return { ...nextSettings };
|
||||
}
|
||||
|
||||
// Apply live updates to subsystems
|
||||
if (updates.maxEventsPerSec !== undefined) {
|
||||
SecurityEngine.maxEventsPerSec = updates.maxEventsPerSec;
|
||||
public static async updateSettingsAsync(
|
||||
updates: Partial<AdminSettings>,
|
||||
): Promise<AdminSettings> {
|
||||
const current = await this.getSettingsAsync();
|
||||
const jwtSecretChanged =
|
||||
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
|
||||
const nextSettings: AdminSettings = {
|
||||
...current,
|
||||
...updates,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
globals.__dataStackAdminSettings = nextSettings;
|
||||
this.applyRuntimeSettings(nextSettings);
|
||||
if (jwtSecretChanged) {
|
||||
await SDKManager.resignAllClientTokensAsync();
|
||||
}
|
||||
if (updates.enableIdempotency !== undefined) {
|
||||
SecurityEngine.enableIdempotency = updates.enableIdempotency;
|
||||
}
|
||||
if (updates.maxPayloadSizeKb !== undefined) {
|
||||
SecurityEngine.maxPayloadSizeBytes = updates.maxPayloadSizeKb * 1024;
|
||||
}
|
||||
if (updates.batchFlushThreshold !== undefined) {
|
||||
DataBufferEngine.batchFlushThreshold = updates.batchFlushThreshold;
|
||||
}
|
||||
if (updates.batchFlushIntervalMs !== undefined) {
|
||||
DataBufferEngine.batchFlushIntervalMs = updates.batchFlushIntervalMs;
|
||||
}
|
||||
if (updates.jwtSecret !== undefined) {
|
||||
AuthService.setSecret(updates.jwtSecret);
|
||||
}
|
||||
if (updates.fileRotationMb !== undefined) {
|
||||
CSVStorageManager.maxFileSizeBytes = updates.fileRotationMb * 1024 * 1024;
|
||||
}
|
||||
|
||||
return this.settings;
|
||||
await this.persistSettingsToDb(nextSettings);
|
||||
return { ...nextSettings };
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { SDKManager } from '../sdk/sdkManager';
|
||||
import { CSVStorageManager } from '../Storage/csvStorage';
|
||||
import { DataBufferEngine } from '../data/buffer';
|
||||
import { SettingsManager } from '../adminSettings/settingsManager';
|
||||
import { SecurityEngine } from '../Security/security';
|
||||
|
||||
export class ApiEngine {
|
||||
@@ -10,23 +9,37 @@ export class ApiEngine {
|
||||
*/
|
||||
public static async getDashboardMetrics() {
|
||||
const storageStats = CSVStorageManager.getStorageStats();
|
||||
const bufferStats = DataBufferEngine.getBufferStats();
|
||||
const clients = SDKManager.listClients();
|
||||
const localBufferStats = DataBufferEngine.getBufferStats();
|
||||
const clients = await SDKManager.listClientsAsync();
|
||||
const activeClients = clients.filter((c) => c.status === 'active').length;
|
||||
let activeSockets = 0;
|
||||
let ingestionBufferSize = 0;
|
||||
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
|
||||
).replace(/\/$/, '');
|
||||
|
||||
try {
|
||||
const res = await fetch('http://localhost:4000/health', { cache: 'no-store' });
|
||||
const res = await fetch(`${ingestionBaseUrl}/health`, {
|
||||
cache: 'no-store',
|
||||
});
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
activeSockets = data.activeSockets ?? 0;
|
||||
activeSockets = Number(data.activeSockets) || 0;
|
||||
ingestionBufferSize = Number(data.bufferSize) || 0;
|
||||
}
|
||||
} catch {
|
||||
// Server may be starting
|
||||
// Ingestion server may not be running in single-process dev mode
|
||||
activeSockets = 0;
|
||||
ingestionBufferSize = 0;
|
||||
}
|
||||
|
||||
const securityStats = SecurityEngine.getStats();
|
||||
const bufferStats = {
|
||||
...localBufferStats,
|
||||
currentBufferSize:
|
||||
localBufferStats.currentBufferSize + ingestionBufferSize,
|
||||
};
|
||||
|
||||
return {
|
||||
activeSockets,
|
||||
|
||||
@@ -1,162 +0,0 @@
|
||||
import { Server as SocketIOServer, Socket } from 'socket.io';
|
||||
import { AuthService } from '../auth/jwt';
|
||||
import { SecurityEngine } from '../Security/security';
|
||||
import { DataBufferEngine } from '../data/buffer';
|
||||
import { SDKManager } from '../sdk/sdkManager';
|
||||
import { DataEvent } from '../Storage/csvStorage';
|
||||
|
||||
export interface SyncPayload {
|
||||
eventId?: string;
|
||||
timestamp?: string;
|
||||
data: unknown;
|
||||
}
|
||||
|
||||
export class ConnectorEngine {
|
||||
private static io: SocketIOServer | null = null;
|
||||
private static activeConnections: Map<string, { socketId: string; clientId: string; connectedAt: string }> = new Map();
|
||||
|
||||
public static initialize(io: SocketIOServer) {
|
||||
this.io = io;
|
||||
|
||||
// Authentication middleware
|
||||
io.use((socket: Socket, next) => {
|
||||
const authHeader =
|
||||
socket.handshake.auth?.token ||
|
||||
socket.handshake.headers?.authorization ||
|
||||
(socket.handshake.query?.token as string);
|
||||
|
||||
if (!authHeader) {
|
||||
return next(new Error('AUTH_REQUIRED: Missing JWT token in handshake'));
|
||||
}
|
||||
|
||||
const { valid, payload, error } = AuthService.verifyClientToken(authHeader);
|
||||
if (!valid || !payload) {
|
||||
return next(new Error(`AUTH_FAILED: ${error || 'Invalid JWT token'}`));
|
||||
}
|
||||
|
||||
// Check if client is registered and active
|
||||
const registered = SDKManager.getClient(payload.clientId);
|
||||
if (registered && registered.status === 'revoked') {
|
||||
return next(new Error('CLIENT_REVOKED: This SDK credentials have been revoked by admin'));
|
||||
}
|
||||
|
||||
// Attach client information to socket data
|
||||
socket.data.clientId = payload.clientId;
|
||||
socket.data.rateLimit = registered?.rateLimit || payload.rateLimit || 100;
|
||||
next();
|
||||
});
|
||||
|
||||
// Connection handler
|
||||
io.on('connection', (socket: Socket) => {
|
||||
const clientId = socket.data.clientId as string;
|
||||
const rateLimit = socket.data.rateLimit as number;
|
||||
|
||||
this.activeConnections.set(socket.id, {
|
||||
socketId: socket.id,
|
||||
clientId,
|
||||
connectedAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
// Join room for this client
|
||||
socket.join(`client:${clientId}`);
|
||||
|
||||
// Single Event Sync: "sync_data"
|
||||
socket.on('sync_data', (payload: SyncPayload, callback?: (res: { success: boolean; eventId?: string; error?: string }) => void) => {
|
||||
try {
|
||||
const eventId = payload.eventId || `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||
const timestamp = payload.timestamp || new Date().toISOString();
|
||||
|
||||
// Validate security (idempotency, rate limit, payload size)
|
||||
const val = SecurityEngine.validateEvent(clientId, eventId, payload.data, rateLimit);
|
||||
if (!val.allowed) {
|
||||
if (val.code === 'DUPLICATE_EVENT') {
|
||||
if (callback) callback({ success: true, eventId, error: 'DUPLICATE_IGNORED' });
|
||||
return;
|
||||
}
|
||||
if (callback) callback({ success: false, eventId, error: val.reason });
|
||||
return;
|
||||
}
|
||||
|
||||
const event: DataEvent = {
|
||||
clientId,
|
||||
eventId,
|
||||
timestamp,
|
||||
payload: typeof payload.data === 'string' ? payload.data : JSON.stringify(payload.data),
|
||||
};
|
||||
|
||||
DataBufferEngine.push(event);
|
||||
SDKManager.incrementEventCount(clientId, 1);
|
||||
|
||||
if (callback) {
|
||||
callback({ success: true, eventId });
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : 'Unknown sync error';
|
||||
if (callback) callback({ success: false, error: msg });
|
||||
}
|
||||
});
|
||||
|
||||
// Batch Event Sync: "batch_sync"
|
||||
socket.on('batch_sync', (batch: SyncPayload[], callback?: (res: { success: boolean; processed: number; rejected: number; errors?: string[] }) => void) => {
|
||||
try {
|
||||
if (!Array.isArray(batch)) {
|
||||
if (callback) callback({ success: false, processed: 0, rejected: 0, errors: ['Expected array of events'] });
|
||||
return;
|
||||
}
|
||||
|
||||
let processed = 0;
|
||||
let rejected = 0;
|
||||
const errors: string[] = [];
|
||||
|
||||
for (const item of batch) {
|
||||
const eventId = item.eventId || `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||
const timestamp = item.timestamp || new Date().toISOString();
|
||||
|
||||
const val = SecurityEngine.validateEvent(clientId, eventId, item.data, rateLimit);
|
||||
if (!val.allowed) {
|
||||
rejected += 1;
|
||||
if (val.code !== 'DUPLICATE_EVENT') {
|
||||
errors.push(`${eventId}: ${val.reason}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const event: DataEvent = {
|
||||
clientId,
|
||||
eventId,
|
||||
timestamp,
|
||||
payload: typeof item.data === 'string' ? item.data : JSON.stringify(item.data),
|
||||
};
|
||||
|
||||
DataBufferEngine.push(event);
|
||||
processed += 1;
|
||||
}
|
||||
|
||||
SDKManager.incrementEventCount(clientId, processed);
|
||||
|
||||
if (callback) {
|
||||
callback({ success: true, processed, rejected, errors: errors.length > 0 ? errors.slice(0, 5) : undefined });
|
||||
}
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : 'Batch error';
|
||||
if (callback) callback({ success: false, processed: 0, rejected: batch?.length || 0, errors: [msg] });
|
||||
}
|
||||
});
|
||||
|
||||
// Disconnect
|
||||
socket.on('disconnect', () => {
|
||||
this.activeConnections.delete(socket.id);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
public static getActiveConnections() {
|
||||
return Array.from(this.activeConnections.values());
|
||||
}
|
||||
|
||||
public static getActiveConnectionsCount(): number {
|
||||
return this.activeConnections.size;
|
||||
}
|
||||
}
|
||||
|
||||
export default ConnectorEngine;
|
||||
+110
-80
@@ -3,93 +3,116 @@ import { CSVStorageManager, DataEvent } from '../Storage/csvStorage';
|
||||
export type EventSubscriber = (event: DataEvent) => void;
|
||||
export type FlushCallback = (count: number, files: string[]) => void;
|
||||
|
||||
interface BufferGlobalState {
|
||||
buffer: DataEvent[];
|
||||
recentEvents: DataEvent[];
|
||||
batchFlushThreshold: number;
|
||||
batchFlushIntervalMs: number;
|
||||
flushTimer: NodeJS.Timeout | null;
|
||||
subscribers: Set<EventSubscriber>;
|
||||
flushListeners: Set<FlushCallback>;
|
||||
totalIngested: number;
|
||||
totalFlushed: number;
|
||||
isFlushing: boolean;
|
||||
}
|
||||
|
||||
type BufferGlobals = typeof globalThis & {
|
||||
__dataStackBufferState?: BufferGlobalState;
|
||||
};
|
||||
|
||||
const globals = globalThis as BufferGlobals;
|
||||
|
||||
function getState(): BufferGlobalState {
|
||||
if (!globals.__dataStackBufferState) {
|
||||
globals.__dataStackBufferState = {
|
||||
buffer: [],
|
||||
recentEvents: [],
|
||||
batchFlushThreshold: 500,
|
||||
batchFlushIntervalMs: 2000,
|
||||
flushTimer: null,
|
||||
subscribers: new Set(),
|
||||
flushListeners: new Set(),
|
||||
totalIngested: 0,
|
||||
totalFlushed: 0,
|
||||
isFlushing: false,
|
||||
};
|
||||
}
|
||||
return globals.__dataStackBufferState;
|
||||
}
|
||||
|
||||
export class DataBufferEngine {
|
||||
private static buffer: DataEvent[] = [];
|
||||
private static recentEvents: DataEvent[] = [];
|
||||
private static maxRecentEvents = 200;
|
||||
|
||||
// Thresholds
|
||||
public static batchFlushThreshold = 500; // rows
|
||||
public static batchFlushIntervalMs = 2000; // 2 seconds
|
||||
public static get batchFlushThreshold(): number {
|
||||
return getState().batchFlushThreshold;
|
||||
}
|
||||
|
||||
private static flushTimer: NodeJS.Timeout | null = null;
|
||||
private static subscribers: Set<EventSubscriber> = new Set();
|
||||
private static flushListeners: Set<FlushCallback> = new Set();
|
||||
public static set batchFlushThreshold(value: number) {
|
||||
getState().batchFlushThreshold = Math.max(1, Number(value) || 500);
|
||||
}
|
||||
|
||||
private static totalIngested = 5;
|
||||
private static totalFlushed = 1420;
|
||||
private static isFlushing = false;
|
||||
public static get batchFlushIntervalMs(): number {
|
||||
return getState().batchFlushIntervalMs;
|
||||
}
|
||||
|
||||
static {
|
||||
const now = Date.now();
|
||||
DataBufferEngine.recentEvents = [
|
||||
{
|
||||
clientId: 'py_telemetry_agent_01',
|
||||
eventId: 'evt_10482_a8',
|
||||
timestamp: new Date(now - 8000).toISOString(),
|
||||
payload: JSON.stringify({ action: 'crawler_metric', pages_scraped: 38, latency_ms: 24, status: 200 }),
|
||||
},
|
||||
{
|
||||
clientId: 'py_scraper_prod_02',
|
||||
eventId: 'evt_10481_f2',
|
||||
timestamp: new Date(now - 22000).toISOString(),
|
||||
payload: JSON.stringify({ spider: 'market_catalog', items: 114, parse_time_s: 1.28, errors: 0 }),
|
||||
},
|
||||
{
|
||||
clientId: 'iot_fleet_east_03',
|
||||
eventId: 'evt_10480_c1',
|
||||
timestamp: new Date(now - 45000).toISOString(),
|
||||
payload: JSON.stringify({ sensor_node: 'gw_alpha_09', temp_c: 24.3, humidity_pct: 58.1, battery_v: 3.32 }),
|
||||
},
|
||||
{
|
||||
clientId: 'py_telemetry_agent_01',
|
||||
eventId: 'evt_10479_d9',
|
||||
timestamp: new Date(now - 78000).toISOString(),
|
||||
payload: JSON.stringify({ action: 'heartbeat', memory_usage_mb: 284, cpu_load_pct: 12.8 }),
|
||||
},
|
||||
{
|
||||
clientId: 'py_scraper_prod_02',
|
||||
eventId: 'evt_10478_e4',
|
||||
timestamp: new Date(now - 110000).toISOString(),
|
||||
payload: JSON.stringify({ action: 'batch_sync', batch_size: 50, duration_ms: 82 }),
|
||||
},
|
||||
];
|
||||
public static set batchFlushIntervalMs(value: number) {
|
||||
this.setFlushInterval(value);
|
||||
}
|
||||
|
||||
public static init() {
|
||||
if (!this.flushTimer) {
|
||||
this.flushTimer = setInterval(() => {
|
||||
this.flush();
|
||||
}, this.batchFlushIntervalMs);
|
||||
const state = getState();
|
||||
if (!state.flushTimer) {
|
||||
state.flushTimer = setInterval(() => {
|
||||
void this.flush();
|
||||
}, state.batchFlushIntervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates flush interval and immediately restarts the active interval timer.
|
||||
*/
|
||||
public static setFlushInterval(intervalMs: number) {
|
||||
const state = getState();
|
||||
const nextInterval = Math.max(100, Number(intervalMs) || 2000);
|
||||
state.batchFlushIntervalMs = nextInterval;
|
||||
|
||||
if (state.flushTimer) {
|
||||
clearInterval(state.flushTimer);
|
||||
state.flushTimer = null;
|
||||
}
|
||||
|
||||
state.flushTimer = setInterval(() => {
|
||||
void this.flush();
|
||||
}, state.batchFlushIntervalMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Push incoming event to in-memory buffer.
|
||||
* If buffer hits threshold (e.g. 500 rows), triggers immediate batch flush.
|
||||
*/
|
||||
public static push(event: DataEvent): void {
|
||||
this.buffer.push(event);
|
||||
this.totalIngested += 1;
|
||||
const state = getState();
|
||||
state.buffer.push(event);
|
||||
state.totalIngested += 1;
|
||||
|
||||
// Cache in recent events for live dashboard viewer
|
||||
this.recentEvents.unshift(event);
|
||||
if (this.recentEvents.length > this.maxRecentEvents) {
|
||||
this.recentEvents.pop();
|
||||
state.recentEvents.unshift(event);
|
||||
if (state.recentEvents.length > this.maxRecentEvents) {
|
||||
state.recentEvents.pop();
|
||||
}
|
||||
|
||||
// Notify live subscribers
|
||||
for (const sub of this.subscribers) {
|
||||
for (const sub of state.subscribers) {
|
||||
try {
|
||||
sub(event);
|
||||
} catch {
|
||||
// Ignore subscriber err
|
||||
// Ignore subscriber error
|
||||
}
|
||||
}
|
||||
|
||||
// Check if buffer threshold reached
|
||||
if (this.buffer.length >= this.batchFlushThreshold) {
|
||||
this.flush();
|
||||
if (state.buffer.length >= state.batchFlushThreshold) {
|
||||
void this.flush();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,19 +120,20 @@ export class DataBufferEngine {
|
||||
* Flushes current buffer to CSV files via CSVStorageManager
|
||||
*/
|
||||
public static async flush(): Promise<{ count: number; files: string[] }> {
|
||||
if (this.buffer.length === 0 || this.isFlushing) {
|
||||
const state = getState();
|
||||
if (state.buffer.length === 0 || state.isFlushing) {
|
||||
return { count: 0, files: [] };
|
||||
}
|
||||
|
||||
this.isFlushing = true;
|
||||
const batchToFlush = [...this.buffer];
|
||||
this.buffer = [];
|
||||
state.isFlushing = true;
|
||||
const batchToFlush = [...state.buffer];
|
||||
state.buffer = [];
|
||||
|
||||
try {
|
||||
const result = await CSVStorageManager.appendBatch(batchToFlush);
|
||||
this.totalFlushed += result.written;
|
||||
state.totalFlushed += result.written;
|
||||
|
||||
for (const listener of this.flushListeners) {
|
||||
for (const listener of state.flushListeners) {
|
||||
try {
|
||||
listener(result.written, result.filesUpdated);
|
||||
} catch {
|
||||
@@ -120,10 +144,10 @@ export class DataBufferEngine {
|
||||
return { count: result.written, files: result.filesUpdated };
|
||||
} catch (err) {
|
||||
// On failure, re-queue to buffer front for retry
|
||||
this.buffer.unshift(...batchToFlush);
|
||||
state.buffer.unshift(...batchToFlush);
|
||||
throw err;
|
||||
} finally {
|
||||
this.isFlushing = false;
|
||||
state.isFlushing = false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -131,36 +155,42 @@ export class DataBufferEngine {
|
||||
* Subscribe to real-time events (for UI monitoring)
|
||||
*/
|
||||
public static subscribe(subscriber: EventSubscriber): () => void {
|
||||
this.subscribers.add(subscriber);
|
||||
const state = getState();
|
||||
state.subscribers.add(subscriber);
|
||||
return () => {
|
||||
this.subscribers.delete(subscriber);
|
||||
state.subscribers.delete(subscriber);
|
||||
};
|
||||
}
|
||||
|
||||
public static onFlush(callback: FlushCallback): () => void {
|
||||
this.flushListeners.add(callback);
|
||||
const state = getState();
|
||||
state.flushListeners.add(callback);
|
||||
return () => {
|
||||
this.flushListeners.delete(callback);
|
||||
state.flushListeners.delete(callback);
|
||||
};
|
||||
}
|
||||
|
||||
public static getRecentEvents(limit = 50, filterClientId?: string): DataEvent[] {
|
||||
const state = getState();
|
||||
if (filterClientId && filterClientId !== 'all') {
|
||||
return this.recentEvents.filter((e) => e.clientId === filterClientId).slice(0, limit);
|
||||
return state.recentEvents
|
||||
.filter((e) => e.clientId === filterClientId)
|
||||
.slice(0, limit);
|
||||
}
|
||||
return this.recentEvents.slice(0, limit);
|
||||
return state.recentEvents.slice(0, limit);
|
||||
}
|
||||
|
||||
public static getBufferStats() {
|
||||
const state = getState();
|
||||
return {
|
||||
currentBufferSize: this.buffer.length,
|
||||
batchFlushThreshold: this.batchFlushThreshold,
|
||||
batchFlushIntervalMs: this.batchFlushIntervalMs,
|
||||
totalIngested: this.totalIngested,
|
||||
totalFlushed: this.totalFlushed,
|
||||
recentEventsCached: this.recentEvents.length,
|
||||
activeSubscribers: this.subscribers.size,
|
||||
isFlushing: this.isFlushing,
|
||||
currentBufferSize: state.buffer.length,
|
||||
batchFlushThreshold: state.batchFlushThreshold,
|
||||
batchFlushIntervalMs: state.batchFlushIntervalMs,
|
||||
totalIngested: state.totalIngested,
|
||||
totalFlushed: state.totalFlushed,
|
||||
recentEventsCached: state.recentEvents.length,
|
||||
activeSubscribers: state.subscribers.size,
|
||||
isFlushing: state.isFlushing,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
export interface SystemNotification {
|
||||
id: string;
|
||||
type: 'info' | 'success' | 'warning' | 'error';
|
||||
title: string;
|
||||
message: string;
|
||||
source: 'socket' | 'storage' | 'security' | 'worker';
|
||||
timestamp: string;
|
||||
read: boolean;
|
||||
}
|
||||
|
||||
export class NotificationEngine {
|
||||
private static notifications: SystemNotification[] = [
|
||||
{
|
||||
id: 'notif-1',
|
||||
type: 'info',
|
||||
title: 'DataStack Engine Booted',
|
||||
message: 'Ingestion server listening on port 4000. CSV buffer worker initialized.',
|
||||
source: 'worker',
|
||||
timestamp: new Date().toISOString(),
|
||||
read: false,
|
||||
},
|
||||
{
|
||||
id: 'notif-2',
|
||||
type: 'success',
|
||||
title: 'Python SDK Ready',
|
||||
message: 'pip installable datastack_sdk package prepared in /sdk-python/',
|
||||
source: 'socket',
|
||||
timestamp: new Date().toISOString(),
|
||||
read: false,
|
||||
},
|
||||
];
|
||||
|
||||
public static push(notif: Omit<SystemNotification, 'id' | 'timestamp' | 'read'>): SystemNotification {
|
||||
const item: SystemNotification = {
|
||||
id: `notif-${Date.now()}-${Math.random().toString(36).substring(2, 6)}`,
|
||||
timestamp: new Date().toISOString(),
|
||||
read: false,
|
||||
...notif,
|
||||
};
|
||||
this.notifications.unshift(item);
|
||||
if (this.notifications.length > 100) {
|
||||
this.notifications.pop();
|
||||
}
|
||||
return item;
|
||||
}
|
||||
|
||||
public static list(): SystemNotification[] {
|
||||
return [...this.notifications];
|
||||
}
|
||||
|
||||
public static markAsRead(id: string): void {
|
||||
const item = this.notifications.find((n) => n.id === id);
|
||||
if (item) item.read = true;
|
||||
}
|
||||
|
||||
public static clearAll(): void {
|
||||
this.notifications = [];
|
||||
}
|
||||
}
|
||||
|
||||
export default NotificationEngine;
|
||||
+320
-55
@@ -1,4 +1,5 @@
|
||||
import { AuthService } from "../auth/jwt";
|
||||
import { DatabaseEngine } from "../Database/database";
|
||||
|
||||
export interface ClientSDK {
|
||||
clientId: string;
|
||||
@@ -12,59 +13,114 @@ export interface ClientSDK {
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
type SDKDatabaseGlobals = typeof globalThis & {
|
||||
__dataStackClientsMap?: Map<string, ClientSDK>;
|
||||
__dataStackClientsHydrated?: boolean;
|
||||
};
|
||||
|
||||
const globals = globalThis as SDKDatabaseGlobals;
|
||||
|
||||
function getClientsMap(): Map<string, ClientSDK> {
|
||||
if (!globals.__dataStackClientsMap) {
|
||||
globals.__dataStackClientsMap = new Map<string, ClientSDK>();
|
||||
}
|
||||
return globals.__dataStackClientsMap;
|
||||
}
|
||||
|
||||
function mapRowToClient(row: Record<string, unknown>): ClientSDK {
|
||||
return {
|
||||
clientId: String(row.client_id),
|
||||
name: String(row.name),
|
||||
secret: String(row.secret),
|
||||
token: String(row.token),
|
||||
rateLimit: Number(row.rate_limit) || 100,
|
||||
status: row.status === "revoked" ? "revoked" : "active",
|
||||
totalEvents: Number(row.total_events) || 0,
|
||||
lastActive: row.last_active
|
||||
? new Date(row.last_active as string | Date).toISOString()
|
||||
: undefined,
|
||||
createdAt: row.created_at
|
||||
? new Date(row.created_at as string | Date).toISOString()
|
||||
: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
export class SDKManager {
|
||||
private static clients: Map<string, ClientSDK> = new Map();
|
||||
private static async syncFromDatabase(): Promise<ClientSDK[] | null> {
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (!pool) return null;
|
||||
|
||||
// Initialize with production starter clients
|
||||
static {
|
||||
const seedClients = [
|
||||
{
|
||||
clientId: "py_telemetry_agent_01",
|
||||
name: "Python Telemetry Agent (US-East)",
|
||||
rateLimit: 150,
|
||||
},
|
||||
{
|
||||
clientId: "py_scraper_prod_02",
|
||||
name: "Python Web Scraper Engine",
|
||||
rateLimit: 200,
|
||||
},
|
||||
{
|
||||
clientId: "iot_fleet_east_03",
|
||||
name: "IoT Sensor Gateway Node",
|
||||
rateLimit: 50,
|
||||
},
|
||||
];
|
||||
try {
|
||||
const ready = await DatabaseEngine.ensureSchema();
|
||||
if (!ready) return null;
|
||||
|
||||
for (const c of seedClients) {
|
||||
const secret = `sec_${c.clientId}_${Math.random().toString(36).substring(2, 10)}`;
|
||||
const token = AuthService.getSecret()
|
||||
? AuthService.generateClientToken({
|
||||
clientId: c.clientId,
|
||||
name: c.name,
|
||||
rateLimit: c.rateLimit,
|
||||
})
|
||||
: "";
|
||||
const result = await pool.query(
|
||||
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
|
||||
FROM client_sdks
|
||||
ORDER BY created_at DESC`,
|
||||
);
|
||||
|
||||
SDKManager.clients.set(c.clientId, {
|
||||
clientId: c.clientId,
|
||||
name: c.name,
|
||||
secret,
|
||||
token,
|
||||
rateLimit: c.rateLimit,
|
||||
status: "active",
|
||||
totalEvents: 4280,
|
||||
lastActive: new Date().toISOString(),
|
||||
createdAt: "2026-10-20T10:00:00.000Z",
|
||||
});
|
||||
const map = getClientsMap();
|
||||
map.clear();
|
||||
const list: ClientSDK[] = [];
|
||||
|
||||
for (const row of result.rows) {
|
||||
const client = mapRowToClient(row);
|
||||
map.set(client.clientId, client);
|
||||
list.push(client);
|
||||
}
|
||||
|
||||
globals.__dataStackClientsHydrated = true;
|
||||
return list;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public static listClients(): ClientSDK[] {
|
||||
return Array.from(this.clients.values());
|
||||
if (!globals.__dataStackClientsHydrated) {
|
||||
void this.syncFromDatabase();
|
||||
}
|
||||
return Array.from(getClientsMap().values());
|
||||
}
|
||||
|
||||
public static async listClientsAsync(): Promise<ClientSDK[]> {
|
||||
const fromDb = await this.syncFromDatabase();
|
||||
if (fromDb !== null) {
|
||||
return fromDb;
|
||||
}
|
||||
return Array.from(getClientsMap().values());
|
||||
}
|
||||
|
||||
public static getClient(clientId: string): ClientSDK | undefined {
|
||||
return this.clients.get(clientId);
|
||||
return getClientsMap().get(clientId);
|
||||
}
|
||||
|
||||
public static async getClientAsync(
|
||||
clientId: string,
|
||||
): Promise<ClientSDK | undefined> {
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (pool) {
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
const result = await pool.query(
|
||||
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
|
||||
FROM client_sdks
|
||||
WHERE client_id = $1`,
|
||||
[clientId],
|
||||
);
|
||||
if (result.rowCount && result.rowCount > 0) {
|
||||
const client = mapRowToClient(result.rows[0]);
|
||||
getClientsMap().set(client.clientId, client);
|
||||
return client;
|
||||
}
|
||||
getClientsMap().delete(clientId);
|
||||
return undefined;
|
||||
} catch {
|
||||
// Fall back to in-memory map
|
||||
}
|
||||
}
|
||||
return getClientsMap().get(clientId);
|
||||
}
|
||||
|
||||
public static registerClient(data: {
|
||||
@@ -77,11 +133,11 @@ export class SDKManager {
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9_-]/g, "_");
|
||||
const secret = `sec_${safeClientId}_${Math.random().toString(36).substring(2, 10)}`;
|
||||
const rateLimit = data.rateLimit || 100;
|
||||
const rateLimit = Math.max(1, Number(data.rateLimit) || 100);
|
||||
|
||||
const token = AuthService.generateClientToken({
|
||||
clientId: safeClientId,
|
||||
name: data.name,
|
||||
name: data.name.trim(),
|
||||
rateLimit,
|
||||
});
|
||||
|
||||
@@ -96,12 +152,79 @@ export class SDKManager {
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
this.clients.set(safeClientId, newClient);
|
||||
getClientsMap().set(safeClientId, newClient);
|
||||
void this.persistNewClient(newClient);
|
||||
return newClient;
|
||||
}
|
||||
|
||||
private static async persistNewClient(client: ClientSDK): Promise<void> {
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (!pool) return;
|
||||
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
await pool.query(
|
||||
`INSERT INTO client_sdks (client_id, name, secret, token, rate_limit, status, total_events, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW())
|
||||
ON CONFLICT (client_id) DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
secret = EXCLUDED.secret,
|
||||
token = EXCLUDED.token,
|
||||
rate_limit = EXCLUDED.rate_limit,
|
||||
status = EXCLUDED.status,
|
||||
updated_at = NOW()`,
|
||||
[
|
||||
client.clientId,
|
||||
client.name,
|
||||
client.secret,
|
||||
client.token,
|
||||
client.rateLimit,
|
||||
client.status,
|
||||
client.totalEvents,
|
||||
client.createdAt,
|
||||
],
|
||||
);
|
||||
} catch {
|
||||
// In-memory map already updated
|
||||
}
|
||||
}
|
||||
|
||||
public static async registerClientAsync(data: {
|
||||
clientId: string;
|
||||
name: string;
|
||||
rateLimit?: number;
|
||||
}): Promise<ClientSDK> {
|
||||
const safeClientId = data.clientId
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9_-]/g, "_");
|
||||
const secret = `sec_${safeClientId}_${Math.random().toString(36).substring(2, 10)}`;
|
||||
const rateLimit = Math.max(1, Number(data.rateLimit) || 100);
|
||||
|
||||
const token = AuthService.generateClientToken({
|
||||
clientId: safeClientId,
|
||||
name: data.name.trim(),
|
||||
rateLimit,
|
||||
});
|
||||
|
||||
const newClient: ClientSDK = {
|
||||
clientId: safeClientId,
|
||||
name: data.name.trim(),
|
||||
secret,
|
||||
token,
|
||||
rateLimit,
|
||||
status: "active",
|
||||
totalEvents: 0,
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
getClientsMap().set(safeClientId, newClient);
|
||||
await this.persistNewClient(newClient);
|
||||
return newClient;
|
||||
}
|
||||
|
||||
public static regenerateToken(clientId: string): ClientSDK | null {
|
||||
const client = this.clients.get(clientId);
|
||||
const client = getClientsMap().get(clientId);
|
||||
if (!client) return null;
|
||||
|
||||
client.secret = `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`;
|
||||
@@ -111,35 +234,177 @@ export class SDKManager {
|
||||
rateLimit: client.rateLimit,
|
||||
});
|
||||
|
||||
this.clients.set(clientId, client);
|
||||
getClientsMap().set(clientId, client);
|
||||
void this.persistClientCredentials(client);
|
||||
return client;
|
||||
}
|
||||
|
||||
private static async persistClientCredentials(
|
||||
client: ClientSDK,
|
||||
): Promise<void> {
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (!pool) return;
|
||||
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
await pool.query(
|
||||
`UPDATE client_sdks
|
||||
SET secret = $2, token = $3, updated_at = NOW()
|
||||
WHERE client_id = $1`,
|
||||
[client.clientId, client.secret, client.token],
|
||||
);
|
||||
} catch {
|
||||
// Ignore DB failure
|
||||
}
|
||||
}
|
||||
|
||||
public static async regenerateTokenAsync(
|
||||
clientId: string,
|
||||
): Promise<ClientSDK | null> {
|
||||
const existing = await this.getClientAsync(clientId);
|
||||
if (!existing) return null;
|
||||
|
||||
existing.secret = `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`;
|
||||
existing.token = AuthService.generateClientToken({
|
||||
clientId: existing.clientId,
|
||||
name: existing.name,
|
||||
rateLimit: existing.rateLimit,
|
||||
});
|
||||
|
||||
getClientsMap().set(clientId, existing);
|
||||
await this.persistClientCredentials(existing);
|
||||
return existing;
|
||||
}
|
||||
|
||||
public static updateClient(
|
||||
clientId: string,
|
||||
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
|
||||
): ClientSDK | null {
|
||||
const client = this.clients.get(clientId);
|
||||
const client = getClientsMap().get(clientId);
|
||||
if (!client) return null;
|
||||
|
||||
if (updates.name !== undefined) client.name = updates.name;
|
||||
if (updates.rateLimit !== undefined) client.rateLimit = updates.rateLimit;
|
||||
if (updates.name !== undefined) client.name = updates.name.trim();
|
||||
if (updates.rateLimit !== undefined)
|
||||
client.rateLimit = Math.max(1, Number(updates.rateLimit) || 100);
|
||||
if (updates.status !== undefined) client.status = updates.status;
|
||||
|
||||
this.clients.set(clientId, client);
|
||||
getClientsMap().set(clientId, client);
|
||||
void this.persistClientUpdates(client);
|
||||
return client;
|
||||
}
|
||||
|
||||
private static async persistClientUpdates(client: ClientSDK): Promise<void> {
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (!pool) return;
|
||||
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
await pool.query(
|
||||
`UPDATE client_sdks
|
||||
SET name = $2, rate_limit = $3, status = $4, updated_at = NOW()
|
||||
WHERE client_id = $1`,
|
||||
[client.clientId, client.name, client.rateLimit, client.status],
|
||||
);
|
||||
} catch {
|
||||
// Ignore DB failure
|
||||
}
|
||||
}
|
||||
|
||||
public static async updateClientAsync(
|
||||
clientId: string,
|
||||
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
|
||||
): Promise<ClientSDK | null> {
|
||||
const client = await this.getClientAsync(clientId);
|
||||
if (!client) return null;
|
||||
|
||||
if (updates.name !== undefined) client.name = updates.name.trim();
|
||||
if (updates.rateLimit !== undefined)
|
||||
client.rateLimit = Math.max(1, Number(updates.rateLimit) || 100);
|
||||
if (updates.status !== undefined) client.status = updates.status;
|
||||
|
||||
getClientsMap().set(clientId, client);
|
||||
await this.persistClientUpdates(client);
|
||||
return client;
|
||||
}
|
||||
|
||||
public static deleteClient(clientId: string): boolean {
|
||||
return this.clients.delete(clientId);
|
||||
const deleted = getClientsMap().delete(clientId);
|
||||
void this.deleteClientFromDb(clientId);
|
||||
return deleted;
|
||||
}
|
||||
|
||||
private static async deleteClientFromDb(clientId: string): Promise<boolean> {
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (!pool) return false;
|
||||
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
const res = await pool.query(
|
||||
`DELETE FROM client_sdks WHERE client_id = $1`,
|
||||
[clientId],
|
||||
);
|
||||
return Boolean(res.rowCount && res.rowCount > 0);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public static async deleteClientAsync(clientId: string): Promise<boolean> {
|
||||
const existedInMem = getClientsMap().delete(clientId);
|
||||
const deletedFromDb = await this.deleteClientFromDb(clientId);
|
||||
return existedInMem || deletedFromDb;
|
||||
}
|
||||
|
||||
public static incrementEventCount(clientId: string, count = 1): void {
|
||||
const client = this.clients.get(clientId);
|
||||
const nowIso = new Date().toISOString();
|
||||
const client = getClientsMap().get(clientId);
|
||||
if (client) {
|
||||
client.totalEvents += count;
|
||||
client.lastActive = new Date().toISOString();
|
||||
this.clients.set(clientId, client);
|
||||
client.lastActive = nowIso;
|
||||
getClientsMap().set(clientId, client);
|
||||
}
|
||||
|
||||
const pool = DatabaseEngine.getPool();
|
||||
if (pool) {
|
||||
void DatabaseEngine.ensureSchema()
|
||||
.then(() =>
|
||||
pool.query(
|
||||
`UPDATE client_sdks
|
||||
SET total_events = total_events + $2,
|
||||
last_active = $3,
|
||||
updated_at = NOW()
|
||||
WHERE client_id = $1`,
|
||||
[clientId, count, nowIso],
|
||||
),
|
||||
)
|
||||
.catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
public static resignAllClientTokens(): void {
|
||||
const map = getClientsMap();
|
||||
for (const [clientId, client] of map.entries()) {
|
||||
client.token = AuthService.generateClientToken({
|
||||
clientId: client.clientId,
|
||||
name: client.name,
|
||||
rateLimit: client.rateLimit,
|
||||
});
|
||||
map.set(clientId, client);
|
||||
void this.persistClientCredentials(client);
|
||||
}
|
||||
}
|
||||
|
||||
public static async resignAllClientTokensAsync(): Promise<void> {
|
||||
const clients = await this.listClientsAsync();
|
||||
const map = getClientsMap();
|
||||
for (const client of clients) {
|
||||
client.token = AuthService.generateClientToken({
|
||||
clientId: client.clientId,
|
||||
name: client.name,
|
||||
rateLimit: client.rateLimit,
|
||||
});
|
||||
map.set(client.clientId, client);
|
||||
await this.persistClientCredentials(client);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user