refactor: switch SDK methods to asynchronous

Migrate SDK interactions to asynchronous calls, relax admin setup email constraints, and add event retrieval functionality.
This commit is contained in:
Team Dark Devil committed 2026-11-17 20:05:00 +06:00
1 parent 81c5e4e2fd
commit bb938c33c0
36 files changed
+1679 -754

No files matched your search

+227 -24
View File
@@ -2,57 +2,260 @@ const express = require('express');
const http = require('http');
const cors = require('cors');
const { Server } = require('socket.io');
const { Pool } = require('pg');
const { setupSocketHandlers } = require('./socket/handlers');
const { CsvWriterWorker } = require('./workers/csvWriter');
const PORT = process.env.PORT_SOCKET || 4000;
const PORT = Number(process.env.PORT_SOCKET) || 4000;
const app = express();
app.use(cors({ origin: '*' }));
app.use(express.json());
app.use(express.json({ limit: '10mb' }));
const bufferQueue = [];
const writer = new CsvWriterWorker();
// Periodic Batch Flush: every 2000ms or when buffer hits 500
const FLUSH_INTERVAL_MS = 2000;
const FLUSH_THRESHOLD = 500;
const runtimeConfig = {
jwtSecret:
process.env.DATASTACK_JWT_SECRET ||
'datastack-jwt-default-secret-key-32-chars-long',
maxEventsPerSec: 100,
maxPayloadSizeBytes: 100 * 1024,
batchFlushThreshold: 500,
batchFlushIntervalMs: 2000,
fileRotationMb: 100,
enableIdempotency: true,
};
let dbPool = null;
if (process.env.DATABASE_URL) {
try {
dbPool = new Pool({
connectionString: process.env.DATABASE_URL,
connectionTimeoutMillis: 3000,
idleTimeoutMillis: 30000,
max: 5,
});
} catch {
dbPool = null;
}
}
// Short-lived cache (2s TTL) for client status & token checks on high-frequency events
const clientAuthCache = new Map();
const CLIENT_CACHE_TTL_MS = 2000;
async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
if (!dbPool || !clientId) {
return { allowed: true };
}
const now = Date.now();
if (!bypassCache) {
const cached = clientAuthCache.get(clientId);
if (cached && now - cached.checkedAt < CLIENT_CACHE_TTL_MS) {
if (cached.status === 'revoked') {
return {
allowed: false,
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
};
}
if (cached.token && presentedToken && cached.token !== presentedToken) {
return {
allowed: false,
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
};
}
return { allowed: true, rateLimit: cached.rateLimit };
}
}
try {
const res = await dbPool.query(
`SELECT status, token, rate_limit FROM client_sdks WHERE client_id = $1`,
[clientId],
);
if (!res.rowCount || res.rowCount === 0) {
// Allow token-authenticated client if not explicitly revoked in table
return { allowed: true };
}
const row = res.rows[0];
const entry = {
status: row.status,
token: row.token,
rateLimit: Number(row.rate_limit) || 100,
checkedAt: now,
};
clientAuthCache.set(clientId, entry);
if (entry.status === 'revoked') {
return {
allowed: false,
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
};
}
if (entry.token && presentedToken && entry.token !== presentedToken) {
return {
allowed: false,
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
};
}
return { allowed: true, rateLimit: entry.rateLimit };
} catch {
return { allowed: true };
}
}
function recordClientActivity(clientId, count = 1) {
if (!dbPool || !clientId || count <= 0) return;
const nowIso = new Date().toISOString();
dbPool
.query(
`UPDATE client_sdks
SET total_events = total_events + $2,
last_active = $3,
updated_at = NOW()
WHERE client_id = $1`,
[clientId, count, nowIso],
)
.catch(() => undefined);
}
let isFlushing = false;
let flushTimer = null;
async function flushBuffer() {
if (bufferQueue.length === 0) return;
if (bufferQueue.length === 0 || isFlushing) {
return { written: 0, files: [] };
}
isFlushing = true;
const batch = bufferQueue.splice(0, bufferQueue.length);
try {
const result = await writer.writeBatch(batch);
if (result.written > 0) {
console.log(`[CSV Worker] Flushed ${result.written} events to ${result.files.length} file(s)`);
console.log(
`[CSV Worker] Flushed ${result.written} events to ${result.files.length} file(s)`,
);
}
return result;
} catch (err) {
console.error('[CSV Worker] Flush error:', err);
bufferQueue.unshift(...batch); // Retry next cycle
bufferQueue.unshift(...batch);
return { written: 0, files: [] };
} finally {
isFlushing = false;
}
}
setInterval(flushBuffer, FLUSH_INTERVAL_MS);
function scheduleFlushTimer(intervalMs) {
const safeInterval = Math.max(100, Number(intervalMs) || 2000);
runtimeConfig.batchFlushIntervalMs = safeInterval;
if (flushTimer) {
clearInterval(flushTimer);
}
flushTimer = setInterval(() => {
void flushBuffer();
}, safeInterval);
}
// Health check endpoint
app.get('/health', (_req, res) => {
res.json({
status: 'online',
service: 'DataStack Express Ingestion',
port: PORT,
bufferSize: bufferQueue.length,
uptime: process.uptime(),
});
});
scheduleFlushTimer(runtimeConfig.batchFlushIntervalMs);
async function syncSettingsFromDatabase() {
if (!dbPool) return;
try {
const res = await dbPool.query(
`SELECT max_events_per_sec, max_payload_size_kb, batch_flush_threshold,
batch_flush_interval_ms, file_rotation_mb, jwt_secret, enable_idempotency
FROM admin_settings
WHERE id = 'default'`,
);
if (!res.rowCount || res.rowCount === 0) return;
const row = res.rows[0];
runtimeConfig.maxEventsPerSec = Number(row.max_events_per_sec) || 100;
runtimeConfig.maxPayloadSizeBytes =
(Number(row.max_payload_size_kb) || 100) * 1024;
runtimeConfig.batchFlushThreshold =
Math.max(1, Number(row.batch_flush_threshold) || 500);
runtimeConfig.enableIdempotency = Boolean(row.enable_idempotency);
if (row.jwt_secret && String(row.jwt_secret).trim().length > 0) {
runtimeConfig.jwtSecret = String(row.jwt_secret).trim();
}
const nextRotationMb = Number(row.file_rotation_mb) || 100;
if (nextRotationMb !== runtimeConfig.fileRotationMb) {
runtimeConfig.fileRotationMb = nextRotationMb;
writer.setMaxFileSizeMb(nextRotationMb);
}
const nextIntervalMs = Math.max(
100,
Number(row.batch_flush_interval_ms) || 2000,
);
if (nextIntervalMs !== runtimeConfig.batchFlushIntervalMs) {
scheduleFlushTimer(nextIntervalMs);
}
} catch {
// Keep existing runtimeConfig if table is not created yet
}
}
if (dbPool) {
void syncSettingsFromDatabase();
setInterval(() => {
void syncSettingsFromDatabase();
}, 3000);
}
const server = http.createServer(app);
const io = new Server(server, {
cors: { origin: '*', methods: ['GET', 'POST'] },
});
setupSocketHandlers(io, bufferQueue, writer);
server.listen(PORT, () => {
console.log(`[DataStack Ingestion Engine] Express + Socket.IO listening on port ${PORT}`);
// Health check endpoint with activeSockets count
app.get('/health', (_req, res) => {
res.json({
status: 'online',
service: 'DataStack Express Ingestion',
port: PORT,
activeSockets: io.engine ? io.engine.clientsCount : 0,
bufferSize: bufferQueue.length,
uptime: process.uptime(),
});
});
module.exports = { app, server, io, bufferQueue };
// Internal flush endpoint triggered by Next.js POST /api/storage
app.post('/internal/flush', async (_req, res) => {
try {
const result = await flushBuffer();
res.json({
success: true,
flushedCount: result.written,
filesUpdated: result.files,
});
} catch (err) {
res.status(500).json({
success: false,
error: err instanceof Error ? err.message : 'Flush failed',
});
}
});
setupSocketHandlers(io, bufferQueue, writer, {
getRuntimeConfig: () => runtimeConfig,
verifyClientInDb,
recordClientActivity,
onBufferPush: () => {
if (bufferQueue.length >= runtimeConfig.batchFlushThreshold) {
void flushBuffer();
}
},
});
server.listen(PORT, () => {
console.log(
`[DataStack Ingestion Engine] Express + Socket.IO listening on port ${PORT}`,
);
});
module.exports = { app, server, io, bufferQueue, flushBuffer };
+254 -62
View File
@@ -1,17 +1,120 @@
const jwt = require("jsonwebtoken");
const IDEMPOTENCY_TTL_MS = 5 * 60 * 1000; // 5 minutes
const IDEMPOTENCY_MAX_ENTRIES = 20000;
const CLEANUP_INTERVAL_MS = 60 * 1000;
function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
const jwtSecret = options.jwtSecret || process.env.DATASTACK_JWT_SECRET;
if (!jwtSecret) {
throw new Error(
"DATASTACK_JWT_SECRET must be configured before starting the socket server.",
);
}
const getRuntimeConfig =
typeof options.getRuntimeConfig === "function"
? options.getRuntimeConfig
: () => ({
jwtSecret:
options.jwtSecret ||
process.env.DATASTACK_JWT_SECRET ||
"datastack-jwt-default-secret-key-32-chars-long",
maxEventsPerSec: 100,
maxPayloadSizeBytes: 100 * 1024,
batchFlushThreshold: 500,
enableIdempotency: true,
});
const verifyClientInDb =
typeof options.verifyClientInDb === "function"
? options.verifyClientInDb
: async () => ({ allowed: true });
const recordClientActivity =
typeof options.recordClientActivity === "function"
? options.recordClientActivity
: () => undefined;
const onBufferPush =
typeof options.onBufferPush === "function"
? options.onBufferPush
: () => undefined;
const idempotencyMap = new Map();
const rateLimitMap = new Map();
let lastCleanupAt = Date.now();
function cleanupMaps(now) {
if (
now - lastCleanupAt < CLEANUP_INTERVAL_MS &&
idempotencyMap.size <= IDEMPOTENCY_MAX_ENTRIES
) {
return;
}
lastCleanupAt = now;
for (const [key, timestamp] of idempotencyMap.entries()) {
if (now - timestamp > IDEMPOTENCY_TTL_MS) {
idempotencyMap.delete(key);
}
}
for (const [clientId, rl] of rateLimitMap.entries()) {
if (now - rl.start > 10000) {
rateLimitMap.delete(clientId);
}
}
}
function validateSingleEvent(clientId, eventId, rawData, clientRateLimit) {
const config = getRuntimeConfig();
const now = Date.now();
cleanupMaps(now);
// 1. Idempotency check (with 5-minute TTL)
if (config.enableIdempotency && eventId) {
const key = `${clientId}:${eventId}`;
const existingTime = idempotencyMap.get(key);
if (existingTime && now - existingTime < IDEMPOTENCY_TTL_MS) {
return {
allowed: false,
duplicate: true,
reason: "Duplicate event dropped",
};
}
idempotencyMap.set(key, now);
}
// 2. Rate limit check
const effectiveLimit = Math.min(
Number(clientRateLimit) || 100,
Number(config.maxEventsPerSec) || 100,
);
const rl = rateLimitMap.get(clientId);
if (!rl || now - rl.start >= 1000) {
rateLimitMap.set(clientId, { count: 1, start: now });
} else if (rl.count >= effectiveLimit) {
return {
allowed: false,
duplicate: false,
reason: `RATE_LIMIT_EXCEEDED: Limit of ${effectiveLimit} events/sec exceeded`,
};
} else {
rl.count += 1;
}
// 3. Payload byte size check
const payloadStr =
typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {});
const byteLength = Buffer.byteLength(payloadStr, "utf8");
const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024;
if (byteLength > maxBytes) {
return {
allowed: false,
duplicate: false,
reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`,
};
}
return { allowed: true, payloadStr };
}
// Middleware: Handshake Auth
io.use((socket, next) => {
io.use(async (socket, next) => {
const authHeader =
socket.handshake.auth?.token ||
socket.handshake.headers?.authorization ||
@@ -22,12 +125,25 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
}
try {
const cleanToken = authHeader.startsWith("Bearer ")
? authHeader.slice(7)
: authHeader;
const decoded = jwt.verify(cleanToken, jwtSecret);
const cleanToken = String(authHeader).startsWith("Bearer ")
? String(authHeader).slice(7).trim()
: String(authHeader).trim();
const config = getRuntimeConfig();
const decoded = jwt.verify(cleanToken, config.jwtSecret);
const dbCheck = await verifyClientInDb(
decoded.clientId,
cleanToken,
true,
);
if (!dbCheck.allowed) {
return next(new Error(dbCheck.reason || "CLIENT_REVOKED"));
}
socket.data.clientId = decoded.clientId;
socket.data.rateLimit = decoded.rateLimit || 100;
socket.data.token = cleanToken;
socket.data.rateLimit =
dbCheck.rateLimit || decoded.rateLimit || config.maxEventsPerSec || 100;
next();
} catch (err) {
next(new Error(`AUTH_INVALID: ${err.message}`));
@@ -36,50 +152,64 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
io.on("connection", (socket) => {
const clientId = socket.data.clientId;
const rateLimit = socket.data.rateLimit;
socket.join(`client:${clientId}`);
// Single event
socket.on("sync_data", (payload, callback) => {
socket.on("sync_data", async (payload, callback) => {
try {
const eventId =
payload.eventId ||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
const timestamp = payload.timestamp || new Date().toISOString();
// Idempotency
const key = `${clientId}:${eventId}`;
if (idempotencyMap.has(key)) {
if (callback)
callback({ success: true, eventId, note: "DUPLICATE_DROPPED" });
const dbCheck = await verifyClientInDb(
clientId,
socket.data.token,
false,
);
if (!dbCheck.allowed) {
if (callback) {
callback({
success: false,
error: dbCheck.reason || "CLIENT_REVOKED",
});
}
socket.disconnect(true);
return;
}
idempotencyMap.set(key, Date.now());
// Rate limit
const now = Date.now();
const rl = rateLimitMap.get(clientId);
if (!rl || now - rl.start >= 1000) {
rateLimitMap.set(clientId, { count: 1, start: now });
} else if (rl.count >= rateLimit) {
if (callback)
callback({ success: false, error: "RATE_LIMIT_EXCEEDED" });
const effectiveClientRate =
dbCheck.rateLimit || socket.data.rateLimit || 100;
const safePayload = payload && typeof payload === "object" ? payload : {};
const eventId =
safePayload.eventId ||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
const timestamp = safePayload.timestamp || new Date().toISOString();
const check = validateSingleEvent(
clientId,
eventId,
safePayload.data,
effectiveClientRate,
);
if (!check.allowed) {
if (check.duplicate) {
if (callback) {
callback({ success: true, eventId, note: "DUPLICATE_DROPPED" });
}
return;
}
if (callback) {
callback({ success: false, eventId, error: check.reason });
}
return;
} else {
rl.count += 1;
}
const event = {
clientId,
eventId,
timestamp,
payload:
typeof payload.data === "string"
? payload.data
: JSON.stringify(payload.data),
payload: check.payloadStr,
};
bufferQueue.push(event);
recordClientActivity(clientId, 1);
onBufferPush();
if (callback) callback({ success: true, eventId });
} catch (err) {
@@ -88,32 +218,94 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
});
// Batch event
socket.on("batch_sync", (batch, callback) => {
if (!Array.isArray(batch)) {
if (callback) callback({ success: false, error: "Expected array" });
return;
}
socket.on("batch_sync", async (batch, callback) => {
try {
if (!Array.isArray(batch)) {
if (callback) callback({ success: false, error: "Expected array" });
return;
}
let count = 0;
for (const item of batch) {
const eventId =
item.eventId ||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
const timestamp = item.timestamp || new Date().toISOString();
bufferQueue.push({
const dbCheck = await verifyClientInDb(
clientId,
eventId,
timestamp,
payload:
typeof item.data === "string"
? item.data
: JSON.stringify(item.data),
});
count++;
}
socket.data.token,
false,
);
if (!dbCheck.allowed) {
if (callback) {
callback({
success: false,
processed: 0,
rejected: batch.length,
error: dbCheck.reason || "CLIENT_REVOKED",
});
}
socket.disconnect(true);
return;
}
if (callback) callback({ success: true, processed: count });
const effectiveClientRate =
dbCheck.rateLimit || socket.data.rateLimit || 100;
let processed = 0;
let rejected = 0;
const errors = [];
for (const item of batch) {
const safeItem = item && typeof item === "object" ? item : {};
const eventId =
safeItem.eventId ||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
const timestamp = safeItem.timestamp || new Date().toISOString();
const rawData =
safeItem.data !== undefined ? safeItem.data : safeItem;
const check = validateSingleEvent(
clientId,
eventId,
rawData,
effectiveClientRate,
);
if (!check.allowed) {
if (!check.duplicate) {
rejected += 1;
if (errors.length < 5) {
errors.push(`${eventId}: ${check.reason}`);
}
}
continue;
}
bufferQueue.push({
clientId,
eventId,
timestamp,
payload: check.payloadStr,
});
processed += 1;
}
if (processed > 0) {
recordClientActivity(clientId, processed);
onBufferPush();
}
if (callback) {
callback({
success: true,
processed,
rejected,
...(errors.length > 0 ? { errors } : {}),
});
}
} catch (err) {
if (callback) {
callback({
success: false,
processed: 0,
rejected: Array.isArray(batch) ? batch.length : 0,
error: err.message,
});
}
}
});
});
}
+8 -1
View File
@@ -14,6 +14,13 @@ class CsvWriterWorker {
this.ensureBaseDir();
}
setMaxFileSizeMb(megabytes) {
const mb = Number(megabytes);
if (Number.isFinite(mb) && mb >= 1) {
this.maxFileSize = Math.floor(mb * 1024 * 1024);
}
}
ensureBaseDir() {
if (!fs.existsSync(this.baseDir)) {
fs.mkdirSync(this.baseDir, { recursive: true });
@@ -21,7 +28,7 @@ class CsvWriterWorker {
}
getTargetFilePath(clientId, dateStr) {
const safeClientId = clientId.replace(/[^a-zA-Z0-9_-]/g, '_');
const safeClientId = String(clientId || 'unknown').replace(/[^a-zA-Z0-9_-]/g, '_');
const clientDir = path.join(this.baseDir, safeClientId);
if (!fs.existsSync(clientDir)) {
fs.mkdirSync(clientDir, { recursive: true });