refactor: switch SDK methods to asynchronous

Migrate SDK interactions to asynchronous calls, relax admin setup email constraints, and add event retrieval functionality.
This commit is contained in:
Team Dark Devil committed 2026-11-17 20:05:00 +06:00
1 parent 81c5e4e2fd
commit bb938c33c0
36 files changed
+1679 -754

No files matched your search

+117 -12
View File
@@ -1,28 +1,133 @@
import { Pool } from "pg";
import { SettingsManager } from "../adminSettings/settingsManager";
import { SDKManager } from "../sdk/sdkManager";
type CoreDatabaseGlobals = typeof globalThis & {
__dataStackCorePool?: Pool | null;
__dataStackCoreSchema?: Promise<void>;
};
const globals = globalThis as CoreDatabaseGlobals;
/**
* Database Module (PostgreSQL + Prisma abstraction)
* Database Module (PostgreSQL Persistence Engine)
* CRITICAL RULE: PostgreSQL stores strictly administrative metadata,
* client registrations, and configuration. Collected data is NEVER stored in PostgreSQL.
* client registrations (client_sdks), and configuration (admin_settings).
* Collected telemetry streams are NEVER stored in PostgreSQL.
*/
export class DatabaseEngine {
public static isConnected = true;
public static getPool(): Pool | null {
const connectionString = process.env.DATABASE_URL;
if (!connectionString) {
return null;
}
if (globals.__dataStackCorePool === undefined) {
try {
globals.__dataStackCorePool = new Pool({
connectionString,
connectionTimeoutMillis: 3000,
idleTimeoutMillis: 30000,
max: 5,
});
} catch {
globals.__dataStackCorePool = null;
}
}
return globals.__dataStackCorePool ?? null;
}
public static async ensureSchema(): Promise<boolean> {
const pool = this.getPool();
if (!pool) return false;
globals.__dataStackCoreSchema ??= Promise.all([
pool.query(`
CREATE TABLE IF NOT EXISTS client_sdks (
client_id TEXT PRIMARY KEY,
name TEXT NOT NULL,
secret TEXT NOT NULL,
token TEXT NOT NULL,
rate_limit INTEGER NOT NULL DEFAULT 100,
status TEXT NOT NULL DEFAULT 'active',
total_events BIGINT NOT NULL DEFAULT 0,
last_active TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`),
pool.query(`
CREATE TABLE IF NOT EXISTS admin_settings (
id TEXT PRIMARY KEY DEFAULT 'default',
server_port INTEGER NOT NULL DEFAULT 4000,
web_port INTEGER NOT NULL DEFAULT 3000,
max_events_per_sec INTEGER NOT NULL DEFAULT 100,
max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
file_rotation_mb INTEGER NOT NULL DEFAULT 100,
jwt_secret TEXT NOT NULL,
enable_idempotency BOOLEAN NOT NULL DEFAULT true,
auto_prune_days INTEGER NOT NULL DEFAULT 30,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`),
]).then(() => undefined);
try {
await globals.__dataStackCoreSchema;
return true;
} catch {
globals.__dataStackCoreSchema = undefined;
return false;
}
}
public static async getHealth() {
return {
status: "healthy",
type: "PostgreSQL + Prisma Schema Managed",
database: "datastack_db",
role: "Metadata & SDK Registrations Only",
collectedDataStorage: "Strictly CSV (/storage/data)",
};
const pool = this.getPool();
if (!pool) {
return {
status: "healthy" as const,
storageMode: "In-Memory" as const,
type: "In-Memory Fallback (DATABASE_URL not set)",
database: "memory",
role: "Metadata & SDK Registrations Only",
collectedDataStorage: "Strictly CSV (/storage/data)",
};
}
try {
await this.ensureSchema();
await pool.query("SELECT 1");
return {
status: "healthy" as const,
storageMode: "PostgreSQL" as const,
type: "PostgreSQL Managed Schema",
database: "datastack_db",
role: "Metadata & SDK Registrations Only",
collectedDataStorage: "Strictly CSV (/storage/data)",
};
} catch {
return {
status: "degraded" as const,
storageMode: "In-Memory" as const,
type: "PostgreSQL Unreachable (In-Memory Fallback Active)",
database: "datastack_db",
role: "Metadata & SDK Registrations Only",
collectedDataStorage: "Strictly CSV (/storage/data)",
};
}
}
public static async syncState() {
const [clients, settings] = await Promise.all([
SDKManager.listClientsAsync(),
SettingsManager.getPublicSettingsAsync(),
]);
return {
clientsCount: SDKManager.listClients().length,
settings: SettingsManager.getPublicSettings(),
clientsCount: clients.length,
settings,
};
}
}
-100
View File
@@ -1,100 +0,0 @@
import express from 'express';
import http from 'http';
import cors from 'cors';
import { Server as SocketIOServer } from 'socket.io';
import { ConnectorEngine } from '../connector/socketHandler';
import { DataBufferEngine } from '../data/buffer';
import { CSVStorageManager } from '../Storage/csvStorage';
export class ServerRuntime {
private static app: express.Express | null = null;
private static httpServer: http.Server | null = null;
private static io: SocketIOServer | null = null;
private static isRunning = false;
private static port = 4000;
public static createServer(port = 4000) {
if (this.isRunning && this.httpServer) {
return { app: this.app, server: this.httpServer, io: this.io };
}
this.port = port;
const app = express();
app.use(cors({ origin: '*' }));
app.use(express.json({ limit: '10mb' }));
// Health route
app.get('/health', (_req, res) => {
res.json({
status: 'online',
service: 'DataStack Express + Socket.IO Intake',
port: this.port,
activeSockets: ConnectorEngine.getActiveConnectionsCount(),
uptime: process.uptime(),
});
});
// HTTP fallback intake endpoint for non-socket clients
app.post('/api/intake', (req, res) => {
const authHeader = req.headers.authorization;
if (!authHeader) {
res.status(401).json({ error: 'Missing Authorization header' });
return;
}
// Can sync via REST as well if needed
res.json({ success: true, message: 'REST sync supported' });
});
const httpServer = http.createServer(app);
const io = new SocketIOServer(httpServer, {
cors: {
origin: '*',
methods: ['GET', 'POST'],
},
pingTimeout: 30000,
pingInterval: 10000,
});
ConnectorEngine.initialize(io);
this.app = app;
this.httpServer = httpServer;
this.io = io;
this.isRunning = true;
return { app, server: httpServer, io };
}
public static start(port = 4000): Promise<number> {
return new Promise((resolve, reject) => {
try {
const { server } = this.createServer(port);
server.listen(port, () => {
resolve(port);
});
server.on('error', (err: NodeJS.ErrnoException) => {
if (err.code === 'EADDRINUSE') {
// Already listening or address in use
resolve(port);
} else {
reject(err);
}
});
} catch (e) {
reject(e);
}
});
}
public static getStatus() {
return {
isRunning: this.isRunning,
port: this.port,
activeSockets: ConnectorEngine.getActiveConnectionsCount(),
bufferSize: DataBufferEngine.getBufferStats().currentBufferSize,
storageStats: CSVStorageManager.getStorageStats(),
};
}
}
export default ServerRuntime;
+38 -1
View File
@@ -167,6 +167,38 @@ export class CSVStorageManager {
return results.sort((a, b) => b.modifiedAt.getTime() - a.modifiedAt.getTime());
}
/**
* Prune CSV partitions older than autoPruneDays
*/
public static pruneExpiredPartitions(autoPruneDays: number): number {
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
return 0;
}
const cutoffDate = new Date(
Date.now() - Math.floor(autoPruneDays) * 86_400_000,
)
.toISOString()
.split('T')[0];
let prunedCount = 0;
const files = this.listAllFiles();
for (const file of files) {
if (/^\d{4}-\d{2}-\d{2}$/.test(file.date) && file.date < cutoffDate) {
try {
if (fs.existsSync(file.filePath)) {
fs.unlinkSync(file.filePath);
prunedCount += 1;
}
} catch {
// Ignore file removal error
}
}
}
return prunedCount;
}
/**
* Get aggregate storage statistics
*/
@@ -175,6 +207,10 @@ export class CSVStorageManager {
const totalBytes = files.reduce((acc, f) => acc + f.sizeBytes, 0);
const totalRows = files.reduce((acc, f) => acc + f.linesCount, 0);
const uniqueClients = new Set(files.map((f) => f.clientId)).size;
const rotationMb = Math.max(
1,
Math.round(this.maxFileSizeBytes / (1024 * 1024)),
);
return {
totalFiles: files.length,
@@ -183,7 +219,8 @@ export class CSVStorageManager {
totalRows,
uniqueClients,
storageDirectory: this.baseStorageDir,
rotationRule: 'Per-day OR per-100MB rotation',
rotationMb,
rotationRule: `${rotationMb}MB / Daily`,
};
}
+192 -31
View File
@@ -1,5 +1,7 @@
import { AuthService } from "../auth/jwt";
import { DataBufferEngine } from "../data/buffer";
import { DatabaseEngine } from "../Database/database";
import { SDKManager } from "../sdk/sdkManager";
import { SecurityEngine } from "../Security/security";
import { CSVStorageManager } from "../Storage/csvStorage";
@@ -17,10 +19,17 @@ export interface AdminSettings {
updatedAt: string;
}
export class SettingsManager {
private static settings: AdminSettings = {
serverPort: 4000,
webPort: 3000,
type SettingsGlobals = typeof globalThis & {
__dataStackAdminSettings?: AdminSettings;
__dataStackSettingsHydrated?: boolean;
};
const globals = globalThis as SettingsGlobals;
function getDefaultSettings(): AdminSettings {
return {
serverPort: Number(process.env.PORT_SOCKET) || 4000,
webPort: Number(process.env.PORT) || 3000,
maxEventsPerSec: 100,
maxPayloadSizeKb: 100,
batchFlushThreshold: 500,
@@ -33,13 +42,160 @@ export class SettingsManager {
autoPruneDays: 30,
updatedAt: new Date().toISOString(),
};
}
function getSettingsState(): AdminSettings {
if (!globals.__dataStackAdminSettings) {
globals.__dataStackAdminSettings = getDefaultSettings();
}
return globals.__dataStackAdminSettings;
}
export class SettingsManager {
private static applyRuntimeSettings(settings: AdminSettings) {
SecurityEngine.maxEventsPerSec = settings.maxEventsPerSec;
SecurityEngine.enableIdempotency = settings.enableIdempotency;
SecurityEngine.maxPayloadSizeBytes = settings.maxPayloadSizeKb * 1024;
DataBufferEngine.batchFlushThreshold = settings.batchFlushThreshold;
DataBufferEngine.setFlushInterval(settings.batchFlushIntervalMs);
if (settings.jwtSecret) {
AuthService.setSecret(settings.jwtSecret);
}
CSVStorageManager.maxFileSizeBytes = settings.fileRotationMb * 1024 * 1024;
CSVStorageManager.pruneExpiredPartitions(settings.autoPruneDays);
}
private static async syncFromDatabase(): Promise<AdminSettings | null> {
const pool = DatabaseEngine.getPool();
if (!pool) return null;
try {
const ready = await DatabaseEngine.ensureSchema();
if (!ready) return null;
const result = await pool.query(
`SELECT server_port, web_port, max_events_per_sec, max_payload_size_kb,
batch_flush_threshold, batch_flush_interval_ms, file_rotation_mb,
jwt_secret, enable_idempotency, auto_prune_days, updated_at
FROM admin_settings
WHERE id = 'default'`,
);
if (result.rowCount && result.rowCount > 0) {
const row = result.rows[0];
const loaded: AdminSettings = {
serverPort: Number(row.server_port) || 4000,
webPort: Number(row.web_port) || 3000,
maxEventsPerSec: Number(row.max_events_per_sec) || 100,
maxPayloadSizeKb: Number(row.max_payload_size_kb) || 100,
batchFlushThreshold: Number(row.batch_flush_threshold) || 500,
batchFlushIntervalMs: Number(row.batch_flush_interval_ms) || 2000,
fileRotationMb: Number(row.file_rotation_mb) || 100,
jwtSecret:
String(row.jwt_secret || "") ||
process.env.DATASTACK_JWT_SECRET ||
"datastack-jwt-default-secret-key-32-chars-long",
enableIdempotency: Boolean(row.enable_idempotency),
autoPruneDays: Number(row.auto_prune_days) || 30,
updatedAt: row.updated_at
? new Date(row.updated_at as string | Date).toISOString()
: new Date().toISOString(),
};
globals.__dataStackAdminSettings = loaded;
globals.__dataStackSettingsHydrated = true;
this.applyRuntimeSettings(loaded);
return loaded;
}
// Initialize default row in PostgreSQL if not present yet
const defaults = getSettingsState();
await this.persistSettingsToDb(defaults);
globals.__dataStackSettingsHydrated = true;
return defaults;
} catch {
return null;
}
}
private static async persistSettingsToDb(
settings: AdminSettings,
): Promise<void> {
const pool = DatabaseEngine.getPool();
if (!pool) return;
try {
await DatabaseEngine.ensureSchema();
await pool.query(
`INSERT INTO admin_settings (
id, server_port, web_port, max_events_per_sec, max_payload_size_kb,
batch_flush_threshold, batch_flush_interval_ms, file_rotation_mb,
jwt_secret, enable_idempotency, auto_prune_days, updated_at
) VALUES ('default', $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, NOW())
ON CONFLICT (id) DO UPDATE SET
server_port = EXCLUDED.server_port,
web_port = EXCLUDED.web_port,
max_events_per_sec = EXCLUDED.max_events_per_sec,
max_payload_size_kb = EXCLUDED.max_payload_size_kb,
batch_flush_threshold = EXCLUDED.batch_flush_threshold,
batch_flush_interval_ms = EXCLUDED.batch_flush_interval_ms,
file_rotation_mb = EXCLUDED.file_rotation_mb,
jwt_secret = EXCLUDED.jwt_secret,
enable_idempotency = EXCLUDED.enable_idempotency,
auto_prune_days = EXCLUDED.auto_prune_days,
updated_at = NOW()`,
[
settings.serverPort,
settings.webPort,
settings.maxEventsPerSec,
settings.maxPayloadSizeKb,
settings.batchFlushThreshold,
settings.batchFlushIntervalMs,
settings.fileRotationMb,
settings.jwtSecret,
settings.enableIdempotency,
settings.autoPruneDays,
],
);
} catch {
// Ignore DB failure; in-memory state remains active
}
}
public static getSettings(): AdminSettings {
return { ...this.settings };
if (!globals.__dataStackSettingsHydrated) {
void this.syncFromDatabase();
}
return { ...getSettingsState() };
}
public static async getSettingsAsync(): Promise<AdminSettings> {
const fromDb = await this.syncFromDatabase();
if (fromDb) {
return { ...fromDb };
}
return { ...getSettingsState() };
}
public static getPublicSettings(): Omit<AdminSettings, "jwtSecret"> {
const settings = this.settings;
const settings = this.getSettings();
return {
serverPort: settings.serverPort,
webPort: settings.webPort,
maxEventsPerSec: settings.maxEventsPerSec,
maxPayloadSizeKb: settings.maxPayloadSizeKb,
batchFlushThreshold: settings.batchFlushThreshold,
batchFlushIntervalMs: settings.batchFlushIntervalMs,
fileRotationMb: settings.fileRotationMb,
enableIdempotency: settings.enableIdempotency,
autoPruneDays: settings.autoPruneDays,
updatedAt: settings.updatedAt,
};
}
public static async getPublicSettingsAsync(): Promise<
Omit<AdminSettings, "jwtSecret">
> {
const settings = await this.getSettingsAsync();
return {
serverPort: settings.serverPort,
webPort: settings.webPort,
@@ -55,36 +211,41 @@ export class SettingsManager {
}
public static updateSettings(updates: Partial<AdminSettings>): AdminSettings {
this.settings = {
...this.settings,
const current = getSettingsState();
const jwtSecretChanged =
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
const nextSettings: AdminSettings = {
...current,
...updates,
updatedAt: new Date().toISOString(),
};
globals.__dataStackAdminSettings = nextSettings;
this.applyRuntimeSettings(nextSettings);
if (jwtSecretChanged) {
SDKManager.resignAllClientTokens();
}
void this.persistSettingsToDb(nextSettings);
return { ...nextSettings };
}
// Apply live updates to subsystems
if (updates.maxEventsPerSec !== undefined) {
SecurityEngine.maxEventsPerSec = updates.maxEventsPerSec;
public static async updateSettingsAsync(
updates: Partial<AdminSettings>,
): Promise<AdminSettings> {
const current = await this.getSettingsAsync();
const jwtSecretChanged =
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
const nextSettings: AdminSettings = {
...current,
...updates,
updatedAt: new Date().toISOString(),
};
globals.__dataStackAdminSettings = nextSettings;
this.applyRuntimeSettings(nextSettings);
if (jwtSecretChanged) {
await SDKManager.resignAllClientTokensAsync();
}
if (updates.enableIdempotency !== undefined) {
SecurityEngine.enableIdempotency = updates.enableIdempotency;
}
if (updates.maxPayloadSizeKb !== undefined) {
SecurityEngine.maxPayloadSizeBytes = updates.maxPayloadSizeKb * 1024;
}
if (updates.batchFlushThreshold !== undefined) {
DataBufferEngine.batchFlushThreshold = updates.batchFlushThreshold;
}
if (updates.batchFlushIntervalMs !== undefined) {
DataBufferEngine.batchFlushIntervalMs = updates.batchFlushIntervalMs;
}
if (updates.jwtSecret !== undefined) {
AuthService.setSecret(updates.jwtSecret);
}
if (updates.fileRotationMb !== undefined) {
CSVStorageManager.maxFileSizeBytes = updates.fileRotationMb * 1024 * 1024;
}
return this.settings;
await this.persistSettingsToDb(nextSettings);
return { ...nextSettings };
}
}
+19 -6
View File
@@ -1,7 +1,6 @@
import { SDKManager } from '../sdk/sdkManager';
import { CSVStorageManager } from '../Storage/csvStorage';
import { DataBufferEngine } from '../data/buffer';
import { SettingsManager } from '../adminSettings/settingsManager';
import { SecurityEngine } from '../Security/security';
export class ApiEngine {
@@ -10,23 +9,37 @@ export class ApiEngine {
*/
public static async getDashboardMetrics() {
const storageStats = CSVStorageManager.getStorageStats();
const bufferStats = DataBufferEngine.getBufferStats();
const clients = SDKManager.listClients();
const localBufferStats = DataBufferEngine.getBufferStats();
const clients = await SDKManager.listClientsAsync();
const activeClients = clients.filter((c) => c.status === 'active').length;
let activeSockets = 0;
let ingestionBufferSize = 0;
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
).replace(/\/$/, '');
try {
const res = await fetch('http://localhost:4000/health', { cache: 'no-store' });
const res = await fetch(`${ingestionBaseUrl}/health`, {
cache: 'no-store',
});
if (res.ok) {
const data = await res.json();
activeSockets = data.activeSockets ?? 0;
activeSockets = Number(data.activeSockets) || 0;
ingestionBufferSize = Number(data.bufferSize) || 0;
}
} catch {
// Server may be starting
// Ingestion server may not be running in single-process dev mode
activeSockets = 0;
ingestionBufferSize = 0;
}
const securityStats = SecurityEngine.getStats();
const bufferStats = {
...localBufferStats,
currentBufferSize:
localBufferStats.currentBufferSize + ingestionBufferSize,
};
return {
activeSockets,
-162
View File
@@ -1,162 +0,0 @@
import { Server as SocketIOServer, Socket } from 'socket.io';
import { AuthService } from '../auth/jwt';
import { SecurityEngine } from '../Security/security';
import { DataBufferEngine } from '../data/buffer';
import { SDKManager } from '../sdk/sdkManager';
import { DataEvent } from '../Storage/csvStorage';
export interface SyncPayload {
eventId?: string;
timestamp?: string;
data: unknown;
}
export class ConnectorEngine {
private static io: SocketIOServer | null = null;
private static activeConnections: Map<string, { socketId: string; clientId: string; connectedAt: string }> = new Map();
public static initialize(io: SocketIOServer) {
this.io = io;
// Authentication middleware
io.use((socket: Socket, next) => {
const authHeader =
socket.handshake.auth?.token ||
socket.handshake.headers?.authorization ||
(socket.handshake.query?.token as string);
if (!authHeader) {
return next(new Error('AUTH_REQUIRED: Missing JWT token in handshake'));
}
const { valid, payload, error } = AuthService.verifyClientToken(authHeader);
if (!valid || !payload) {
return next(new Error(`AUTH_FAILED: ${error || 'Invalid JWT token'}`));
}
// Check if client is registered and active
const registered = SDKManager.getClient(payload.clientId);
if (registered && registered.status === 'revoked') {
return next(new Error('CLIENT_REVOKED: This SDK credentials have been revoked by admin'));
}
// Attach client information to socket data
socket.data.clientId = payload.clientId;
socket.data.rateLimit = registered?.rateLimit || payload.rateLimit || 100;
next();
});
// Connection handler
io.on('connection', (socket: Socket) => {
const clientId = socket.data.clientId as string;
const rateLimit = socket.data.rateLimit as number;
this.activeConnections.set(socket.id, {
socketId: socket.id,
clientId,
connectedAt: new Date().toISOString(),
});
// Join room for this client
socket.join(`client:${clientId}`);
// Single Event Sync: "sync_data"
socket.on('sync_data', (payload: SyncPayload, callback?: (res: { success: boolean; eventId?: string; error?: string }) => void) => {
try {
const eventId = payload.eventId || `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
const timestamp = payload.timestamp || new Date().toISOString();
// Validate security (idempotency, rate limit, payload size)
const val = SecurityEngine.validateEvent(clientId, eventId, payload.data, rateLimit);
if (!val.allowed) {
if (val.code === 'DUPLICATE_EVENT') {
if (callback) callback({ success: true, eventId, error: 'DUPLICATE_IGNORED' });
return;
}
if (callback) callback({ success: false, eventId, error: val.reason });
return;
}
const event: DataEvent = {
clientId,
eventId,
timestamp,
payload: typeof payload.data === 'string' ? payload.data : JSON.stringify(payload.data),
};
DataBufferEngine.push(event);
SDKManager.incrementEventCount(clientId, 1);
if (callback) {
callback({ success: true, eventId });
}
} catch (err) {
const msg = err instanceof Error ? err.message : 'Unknown sync error';
if (callback) callback({ success: false, error: msg });
}
});
// Batch Event Sync: "batch_sync"
socket.on('batch_sync', (batch: SyncPayload[], callback?: (res: { success: boolean; processed: number; rejected: number; errors?: string[] }) => void) => {
try {
if (!Array.isArray(batch)) {
if (callback) callback({ success: false, processed: 0, rejected: 0, errors: ['Expected array of events'] });
return;
}
let processed = 0;
let rejected = 0;
const errors: string[] = [];
for (const item of batch) {
const eventId = item.eventId || `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
const timestamp = item.timestamp || new Date().toISOString();
const val = SecurityEngine.validateEvent(clientId, eventId, item.data, rateLimit);
if (!val.allowed) {
rejected += 1;
if (val.code !== 'DUPLICATE_EVENT') {
errors.push(`${eventId}: ${val.reason}`);
}
continue;
}
const event: DataEvent = {
clientId,
eventId,
timestamp,
payload: typeof item.data === 'string' ? item.data : JSON.stringify(item.data),
};
DataBufferEngine.push(event);
processed += 1;
}
SDKManager.incrementEventCount(clientId, processed);
if (callback) {
callback({ success: true, processed, rejected, errors: errors.length > 0 ? errors.slice(0, 5) : undefined });
}
} catch (err) {
const msg = err instanceof Error ? err.message : 'Batch error';
if (callback) callback({ success: false, processed: 0, rejected: batch?.length || 0, errors: [msg] });
}
});
// Disconnect
socket.on('disconnect', () => {
this.activeConnections.delete(socket.id);
});
});
}
public static getActiveConnections() {
return Array.from(this.activeConnections.values());
}
public static getActiveConnectionsCount(): number {
return this.activeConnections.size;
}
}
export default ConnectorEngine;
+110 -80
View File
@@ -3,93 +3,116 @@ import { CSVStorageManager, DataEvent } from '../Storage/csvStorage';
export type EventSubscriber = (event: DataEvent) => void;
export type FlushCallback = (count: number, files: string[]) => void;
interface BufferGlobalState {
buffer: DataEvent[];
recentEvents: DataEvent[];
batchFlushThreshold: number;
batchFlushIntervalMs: number;
flushTimer: NodeJS.Timeout | null;
subscribers: Set<EventSubscriber>;
flushListeners: Set<FlushCallback>;
totalIngested: number;
totalFlushed: number;
isFlushing: boolean;
}
type BufferGlobals = typeof globalThis & {
__dataStackBufferState?: BufferGlobalState;
};
const globals = globalThis as BufferGlobals;
function getState(): BufferGlobalState {
if (!globals.__dataStackBufferState) {
globals.__dataStackBufferState = {
buffer: [],
recentEvents: [],
batchFlushThreshold: 500,
batchFlushIntervalMs: 2000,
flushTimer: null,
subscribers: new Set(),
flushListeners: new Set(),
totalIngested: 0,
totalFlushed: 0,
isFlushing: false,
};
}
return globals.__dataStackBufferState;
}
export class DataBufferEngine {
private static buffer: DataEvent[] = [];
private static recentEvents: DataEvent[] = [];
private static maxRecentEvents = 200;
// Thresholds
public static batchFlushThreshold = 500; // rows
public static batchFlushIntervalMs = 2000; // 2 seconds
public static get batchFlushThreshold(): number {
return getState().batchFlushThreshold;
}
private static flushTimer: NodeJS.Timeout | null = null;
private static subscribers: Set<EventSubscriber> = new Set();
private static flushListeners: Set<FlushCallback> = new Set();
public static set batchFlushThreshold(value: number) {
getState().batchFlushThreshold = Math.max(1, Number(value) || 500);
}
private static totalIngested = 5;
private static totalFlushed = 1420;
private static isFlushing = false;
public static get batchFlushIntervalMs(): number {
return getState().batchFlushIntervalMs;
}
static {
const now = Date.now();
DataBufferEngine.recentEvents = [
{
clientId: 'py_telemetry_agent_01',
eventId: 'evt_10482_a8',
timestamp: new Date(now - 8000).toISOString(),
payload: JSON.stringify({ action: 'crawler_metric', pages_scraped: 38, latency_ms: 24, status: 200 }),
},
{
clientId: 'py_scraper_prod_02',
eventId: 'evt_10481_f2',
timestamp: new Date(now - 22000).toISOString(),
payload: JSON.stringify({ spider: 'market_catalog', items: 114, parse_time_s: 1.28, errors: 0 }),
},
{
clientId: 'iot_fleet_east_03',
eventId: 'evt_10480_c1',
timestamp: new Date(now - 45000).toISOString(),
payload: JSON.stringify({ sensor_node: 'gw_alpha_09', temp_c: 24.3, humidity_pct: 58.1, battery_v: 3.32 }),
},
{
clientId: 'py_telemetry_agent_01',
eventId: 'evt_10479_d9',
timestamp: new Date(now - 78000).toISOString(),
payload: JSON.stringify({ action: 'heartbeat', memory_usage_mb: 284, cpu_load_pct: 12.8 }),
},
{
clientId: 'py_scraper_prod_02',
eventId: 'evt_10478_e4',
timestamp: new Date(now - 110000).toISOString(),
payload: JSON.stringify({ action: 'batch_sync', batch_size: 50, duration_ms: 82 }),
},
];
public static set batchFlushIntervalMs(value: number) {
this.setFlushInterval(value);
}
public static init() {
if (!this.flushTimer) {
this.flushTimer = setInterval(() => {
this.flush();
}, this.batchFlushIntervalMs);
const state = getState();
if (!state.flushTimer) {
state.flushTimer = setInterval(() => {
void this.flush();
}, state.batchFlushIntervalMs);
}
}
/**
* Updates flush interval and immediately restarts the active interval timer.
*/
public static setFlushInterval(intervalMs: number) {
const state = getState();
const nextInterval = Math.max(100, Number(intervalMs) || 2000);
state.batchFlushIntervalMs = nextInterval;
if (state.flushTimer) {
clearInterval(state.flushTimer);
state.flushTimer = null;
}
state.flushTimer = setInterval(() => {
void this.flush();
}, state.batchFlushIntervalMs);
}
/**
* Push incoming event to in-memory buffer.
* If buffer hits threshold (e.g. 500 rows), triggers immediate batch flush.
*/
public static push(event: DataEvent): void {
this.buffer.push(event);
this.totalIngested += 1;
const state = getState();
state.buffer.push(event);
state.totalIngested += 1;
// Cache in recent events for live dashboard viewer
this.recentEvents.unshift(event);
if (this.recentEvents.length > this.maxRecentEvents) {
this.recentEvents.pop();
state.recentEvents.unshift(event);
if (state.recentEvents.length > this.maxRecentEvents) {
state.recentEvents.pop();
}
// Notify live subscribers
for (const sub of this.subscribers) {
for (const sub of state.subscribers) {
try {
sub(event);
} catch {
// Ignore subscriber err
// Ignore subscriber error
}
}
// Check if buffer threshold reached
if (this.buffer.length >= this.batchFlushThreshold) {
this.flush();
if (state.buffer.length >= state.batchFlushThreshold) {
void this.flush();
}
}
@@ -97,19 +120,20 @@ export class DataBufferEngine {
* Flushes current buffer to CSV files via CSVStorageManager
*/
public static async flush(): Promise<{ count: number; files: string[] }> {
if (this.buffer.length === 0 || this.isFlushing) {
const state = getState();
if (state.buffer.length === 0 || state.isFlushing) {
return { count: 0, files: [] };
}
this.isFlushing = true;
const batchToFlush = [...this.buffer];
this.buffer = [];
state.isFlushing = true;
const batchToFlush = [...state.buffer];
state.buffer = [];
try {
const result = await CSVStorageManager.appendBatch(batchToFlush);
this.totalFlushed += result.written;
state.totalFlushed += result.written;
for (const listener of this.flushListeners) {
for (const listener of state.flushListeners) {
try {
listener(result.written, result.filesUpdated);
} catch {
@@ -120,10 +144,10 @@ export class DataBufferEngine {
return { count: result.written, files: result.filesUpdated };
} catch (err) {
// On failure, re-queue to buffer front for retry
this.buffer.unshift(...batchToFlush);
state.buffer.unshift(...batchToFlush);
throw err;
} finally {
this.isFlushing = false;
state.isFlushing = false;
}
}
@@ -131,36 +155,42 @@ export class DataBufferEngine {
* Subscribe to real-time events (for UI monitoring)
*/
public static subscribe(subscriber: EventSubscriber): () => void {
this.subscribers.add(subscriber);
const state = getState();
state.subscribers.add(subscriber);
return () => {
this.subscribers.delete(subscriber);
state.subscribers.delete(subscriber);
};
}
public static onFlush(callback: FlushCallback): () => void {
this.flushListeners.add(callback);
const state = getState();
state.flushListeners.add(callback);
return () => {
this.flushListeners.delete(callback);
state.flushListeners.delete(callback);
};
}
public static getRecentEvents(limit = 50, filterClientId?: string): DataEvent[] {
const state = getState();
if (filterClientId && filterClientId !== 'all') {
return this.recentEvents.filter((e) => e.clientId === filterClientId).slice(0, limit);
return state.recentEvents
.filter((e) => e.clientId === filterClientId)
.slice(0, limit);
}
return this.recentEvents.slice(0, limit);
return state.recentEvents.slice(0, limit);
}
public static getBufferStats() {
const state = getState();
return {
currentBufferSize: this.buffer.length,
batchFlushThreshold: this.batchFlushThreshold,
batchFlushIntervalMs: this.batchFlushIntervalMs,
totalIngested: this.totalIngested,
totalFlushed: this.totalFlushed,
recentEventsCached: this.recentEvents.length,
activeSubscribers: this.subscribers.size,
isFlushing: this.isFlushing,
currentBufferSize: state.buffer.length,
batchFlushThreshold: state.batchFlushThreshold,
batchFlushIntervalMs: state.batchFlushIntervalMs,
totalIngested: state.totalIngested,
totalFlushed: state.totalFlushed,
recentEventsCached: state.recentEvents.length,
activeSubscribers: state.subscribers.size,
isFlushing: state.isFlushing,
};
}
}
-61
View File
@@ -1,61 +0,0 @@
export interface SystemNotification {
id: string;
type: 'info' | 'success' | 'warning' | 'error';
title: string;
message: string;
source: 'socket' | 'storage' | 'security' | 'worker';
timestamp: string;
read: boolean;
}
export class NotificationEngine {
private static notifications: SystemNotification[] = [
{
id: 'notif-1',
type: 'info',
title: 'DataStack Engine Booted',
message: 'Ingestion server listening on port 4000. CSV buffer worker initialized.',
source: 'worker',
timestamp: new Date().toISOString(),
read: false,
},
{
id: 'notif-2',
type: 'success',
title: 'Python SDK Ready',
message: 'pip installable datastack_sdk package prepared in /sdk-python/',
source: 'socket',
timestamp: new Date().toISOString(),
read: false,
},
];
public static push(notif: Omit<SystemNotification, 'id' | 'timestamp' | 'read'>): SystemNotification {
const item: SystemNotification = {
id: `notif-${Date.now()}-${Math.random().toString(36).substring(2, 6)}`,
timestamp: new Date().toISOString(),
read: false,
...notif,
};
this.notifications.unshift(item);
if (this.notifications.length > 100) {
this.notifications.pop();
}
return item;
}
public static list(): SystemNotification[] {
return [...this.notifications];
}
public static markAsRead(id: string): void {
const item = this.notifications.find((n) => n.id === id);
if (item) item.read = true;
}
public static clearAll(): void {
this.notifications = [];
}
}
export default NotificationEngine;
+320 -55
View File
@@ -1,4 +1,5 @@
import { AuthService } from "../auth/jwt";
import { DatabaseEngine } from "../Database/database";
export interface ClientSDK {
clientId: string;
@@ -12,59 +13,114 @@ export interface ClientSDK {
createdAt: string;
}
type SDKDatabaseGlobals = typeof globalThis & {
__dataStackClientsMap?: Map<string, ClientSDK>;
__dataStackClientsHydrated?: boolean;
};
const globals = globalThis as SDKDatabaseGlobals;
function getClientsMap(): Map<string, ClientSDK> {
if (!globals.__dataStackClientsMap) {
globals.__dataStackClientsMap = new Map<string, ClientSDK>();
}
return globals.__dataStackClientsMap;
}
function mapRowToClient(row: Record<string, unknown>): ClientSDK {
return {
clientId: String(row.client_id),
name: String(row.name),
secret: String(row.secret),
token: String(row.token),
rateLimit: Number(row.rate_limit) || 100,
status: row.status === "revoked" ? "revoked" : "active",
totalEvents: Number(row.total_events) || 0,
lastActive: row.last_active
? new Date(row.last_active as string | Date).toISOString()
: undefined,
createdAt: row.created_at
? new Date(row.created_at as string | Date).toISOString()
: new Date().toISOString(),
};
}
export class SDKManager {
private static clients: Map<string, ClientSDK> = new Map();
private static async syncFromDatabase(): Promise<ClientSDK[] | null> {
const pool = DatabaseEngine.getPool();
if (!pool) return null;
// Initialize with production starter clients
static {
const seedClients = [
{
clientId: "py_telemetry_agent_01",
name: "Python Telemetry Agent (US-East)",
rateLimit: 150,
},
{
clientId: "py_scraper_prod_02",
name: "Python Web Scraper Engine",
rateLimit: 200,
},
{
clientId: "iot_fleet_east_03",
name: "IoT Sensor Gateway Node",
rateLimit: 50,
},
];
try {
const ready = await DatabaseEngine.ensureSchema();
if (!ready) return null;
for (const c of seedClients) {
const secret = `sec_${c.clientId}_${Math.random().toString(36).substring(2, 10)}`;
const token = AuthService.getSecret()
? AuthService.generateClientToken({
clientId: c.clientId,
name: c.name,
rateLimit: c.rateLimit,
})
: "";
const result = await pool.query(
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
FROM client_sdks
ORDER BY created_at DESC`,
);
SDKManager.clients.set(c.clientId, {
clientId: c.clientId,
name: c.name,
secret,
token,
rateLimit: c.rateLimit,
status: "active",
totalEvents: 4280,
lastActive: new Date().toISOString(),
createdAt: "2026-10-20T10:00:00.000Z",
});
const map = getClientsMap();
map.clear();
const list: ClientSDK[] = [];
for (const row of result.rows) {
const client = mapRowToClient(row);
map.set(client.clientId, client);
list.push(client);
}
globals.__dataStackClientsHydrated = true;
return list;
} catch {
return null;
}
}
public static listClients(): ClientSDK[] {
return Array.from(this.clients.values());
if (!globals.__dataStackClientsHydrated) {
void this.syncFromDatabase();
}
return Array.from(getClientsMap().values());
}
public static async listClientsAsync(): Promise<ClientSDK[]> {
const fromDb = await this.syncFromDatabase();
if (fromDb !== null) {
return fromDb;
}
return Array.from(getClientsMap().values());
}
public static getClient(clientId: string): ClientSDK | undefined {
return this.clients.get(clientId);
return getClientsMap().get(clientId);
}
public static async getClientAsync(
clientId: string,
): Promise<ClientSDK | undefined> {
const pool = DatabaseEngine.getPool();
if (pool) {
try {
await DatabaseEngine.ensureSchema();
const result = await pool.query(
`SELECT client_id, name, secret, token, rate_limit, status, total_events, last_active, created_at
FROM client_sdks
WHERE client_id = $1`,
[clientId],
);
if (result.rowCount && result.rowCount > 0) {
const client = mapRowToClient(result.rows[0]);
getClientsMap().set(client.clientId, client);
return client;
}
getClientsMap().delete(clientId);
return undefined;
} catch {
// Fall back to in-memory map
}
}
return getClientsMap().get(clientId);
}
public static registerClient(data: {
@@ -77,11 +133,11 @@ export class SDKManager {
.toLowerCase()
.replace(/[^a-z0-9_-]/g, "_");
const secret = `sec_${safeClientId}_${Math.random().toString(36).substring(2, 10)}`;
const rateLimit = data.rateLimit || 100;
const rateLimit = Math.max(1, Number(data.rateLimit) || 100);
const token = AuthService.generateClientToken({
clientId: safeClientId,
name: data.name,
name: data.name.trim(),
rateLimit,
});
@@ -96,12 +152,79 @@ export class SDKManager {
createdAt: new Date().toISOString(),
};
this.clients.set(safeClientId, newClient);
getClientsMap().set(safeClientId, newClient);
void this.persistNewClient(newClient);
return newClient;
}
private static async persistNewClient(client: ClientSDK): Promise<void> {
const pool = DatabaseEngine.getPool();
if (!pool) return;
try {
await DatabaseEngine.ensureSchema();
await pool.query(
`INSERT INTO client_sdks (client_id, name, secret, token, rate_limit, status, total_events, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW())
ON CONFLICT (client_id) DO UPDATE SET
name = EXCLUDED.name,
secret = EXCLUDED.secret,
token = EXCLUDED.token,
rate_limit = EXCLUDED.rate_limit,
status = EXCLUDED.status,
updated_at = NOW()`,
[
client.clientId,
client.name,
client.secret,
client.token,
client.rateLimit,
client.status,
client.totalEvents,
client.createdAt,
],
);
} catch {
// In-memory map already updated
}
}
public static async registerClientAsync(data: {
clientId: string;
name: string;
rateLimit?: number;
}): Promise<ClientSDK> {
const safeClientId = data.clientId
.trim()
.toLowerCase()
.replace(/[^a-z0-9_-]/g, "_");
const secret = `sec_${safeClientId}_${Math.random().toString(36).substring(2, 10)}`;
const rateLimit = Math.max(1, Number(data.rateLimit) || 100);
const token = AuthService.generateClientToken({
clientId: safeClientId,
name: data.name.trim(),
rateLimit,
});
const newClient: ClientSDK = {
clientId: safeClientId,
name: data.name.trim(),
secret,
token,
rateLimit,
status: "active",
totalEvents: 0,
createdAt: new Date().toISOString(),
};
getClientsMap().set(safeClientId, newClient);
await this.persistNewClient(newClient);
return newClient;
}
public static regenerateToken(clientId: string): ClientSDK | null {
const client = this.clients.get(clientId);
const client = getClientsMap().get(clientId);
if (!client) return null;
client.secret = `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`;
@@ -111,35 +234,177 @@ export class SDKManager {
rateLimit: client.rateLimit,
});
this.clients.set(clientId, client);
getClientsMap().set(clientId, client);
void this.persistClientCredentials(client);
return client;
}
private static async persistClientCredentials(
client: ClientSDK,
): Promise<void> {
const pool = DatabaseEngine.getPool();
if (!pool) return;
try {
await DatabaseEngine.ensureSchema();
await pool.query(
`UPDATE client_sdks
SET secret = $2, token = $3, updated_at = NOW()
WHERE client_id = $1`,
[client.clientId, client.secret, client.token],
);
} catch {
// Ignore DB failure
}
}
public static async regenerateTokenAsync(
clientId: string,
): Promise<ClientSDK | null> {
const existing = await this.getClientAsync(clientId);
if (!existing) return null;
existing.secret = `sec_${clientId}_${Math.random().toString(36).substring(2, 10)}`;
existing.token = AuthService.generateClientToken({
clientId: existing.clientId,
name: existing.name,
rateLimit: existing.rateLimit,
});
getClientsMap().set(clientId, existing);
await this.persistClientCredentials(existing);
return existing;
}
public static updateClient(
clientId: string,
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
): ClientSDK | null {
const client = this.clients.get(clientId);
const client = getClientsMap().get(clientId);
if (!client) return null;
if (updates.name !== undefined) client.name = updates.name;
if (updates.rateLimit !== undefined) client.rateLimit = updates.rateLimit;
if (updates.name !== undefined) client.name = updates.name.trim();
if (updates.rateLimit !== undefined)
client.rateLimit = Math.max(1, Number(updates.rateLimit) || 100);
if (updates.status !== undefined) client.status = updates.status;
this.clients.set(clientId, client);
getClientsMap().set(clientId, client);
void this.persistClientUpdates(client);
return client;
}
private static async persistClientUpdates(client: ClientSDK): Promise<void> {
const pool = DatabaseEngine.getPool();
if (!pool) return;
try {
await DatabaseEngine.ensureSchema();
await pool.query(
`UPDATE client_sdks
SET name = $2, rate_limit = $3, status = $4, updated_at = NOW()
WHERE client_id = $1`,
[client.clientId, client.name, client.rateLimit, client.status],
);
} catch {
// Ignore DB failure
}
}
public static async updateClientAsync(
clientId: string,
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
): Promise<ClientSDK | null> {
const client = await this.getClientAsync(clientId);
if (!client) return null;
if (updates.name !== undefined) client.name = updates.name.trim();
if (updates.rateLimit !== undefined)
client.rateLimit = Math.max(1, Number(updates.rateLimit) || 100);
if (updates.status !== undefined) client.status = updates.status;
getClientsMap().set(clientId, client);
await this.persistClientUpdates(client);
return client;
}
public static deleteClient(clientId: string): boolean {
return this.clients.delete(clientId);
const deleted = getClientsMap().delete(clientId);
void this.deleteClientFromDb(clientId);
return deleted;
}
private static async deleteClientFromDb(clientId: string): Promise<boolean> {
const pool = DatabaseEngine.getPool();
if (!pool) return false;
try {
await DatabaseEngine.ensureSchema();
const res = await pool.query(
`DELETE FROM client_sdks WHERE client_id = $1`,
[clientId],
);
return Boolean(res.rowCount && res.rowCount > 0);
} catch {
return false;
}
}
public static async deleteClientAsync(clientId: string): Promise<boolean> {
const existedInMem = getClientsMap().delete(clientId);
const deletedFromDb = await this.deleteClientFromDb(clientId);
return existedInMem || deletedFromDb;
}
public static incrementEventCount(clientId: string, count = 1): void {
const client = this.clients.get(clientId);
const nowIso = new Date().toISOString();
const client = getClientsMap().get(clientId);
if (client) {
client.totalEvents += count;
client.lastActive = new Date().toISOString();
this.clients.set(clientId, client);
client.lastActive = nowIso;
getClientsMap().set(clientId, client);
}
const pool = DatabaseEngine.getPool();
if (pool) {
void DatabaseEngine.ensureSchema()
.then(() =>
pool.query(
`UPDATE client_sdks
SET total_events = total_events + $2,
last_active = $3,
updated_at = NOW()
WHERE client_id = $1`,
[clientId, count, nowIso],
),
)
.catch(() => undefined);
}
}
public static resignAllClientTokens(): void {
const map = getClientsMap();
for (const [clientId, client] of map.entries()) {
client.token = AuthService.generateClientToken({
clientId: client.clientId,
name: client.name,
rateLimit: client.rateLimit,
});
map.set(clientId, client);
void this.persistClientCredentials(client);
}
}
public static async resignAllClientTokensAsync(): Promise<void> {
const clients = await this.listClientsAsync();
const map = getClientsMap();
for (const client of clients) {
client.token = AuthService.generateClientToken({
clientId: client.clientId,
name: client.name,
rateLimit: client.rateLimit,
});
map.set(client.clientId, client);
await this.persistClientCredentials(client);
}
}
}