refactor: switch SDK methods to asynchronous

Migrate SDK interactions to asynchronous calls, relax admin setup email constraints, and add event retrieval functionality.
This commit is contained in:
Team Dark Devil committed 2026-11-17 20:05:00 +06:00
1 parent 81c5e4e2fd
commit bb938c33c0
36 files changed
+1679 -754

No files matched your search

+254 -62
View File
@@ -1,17 +1,120 @@
const jwt = require("jsonwebtoken");
const IDEMPOTENCY_TTL_MS = 5 * 60 * 1000; // 5 minutes
const IDEMPOTENCY_MAX_ENTRIES = 20000;
const CLEANUP_INTERVAL_MS = 60 * 1000;
function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
const jwtSecret = options.jwtSecret || process.env.DATASTACK_JWT_SECRET;
if (!jwtSecret) {
throw new Error(
"DATASTACK_JWT_SECRET must be configured before starting the socket server.",
);
}
const getRuntimeConfig =
typeof options.getRuntimeConfig === "function"
? options.getRuntimeConfig
: () => ({
jwtSecret:
options.jwtSecret ||
process.env.DATASTACK_JWT_SECRET ||
"datastack-jwt-default-secret-key-32-chars-long",
maxEventsPerSec: 100,
maxPayloadSizeBytes: 100 * 1024,
batchFlushThreshold: 500,
enableIdempotency: true,
});
const verifyClientInDb =
typeof options.verifyClientInDb === "function"
? options.verifyClientInDb
: async () => ({ allowed: true });
const recordClientActivity =
typeof options.recordClientActivity === "function"
? options.recordClientActivity
: () => undefined;
const onBufferPush =
typeof options.onBufferPush === "function"
? options.onBufferPush
: () => undefined;
const idempotencyMap = new Map();
const rateLimitMap = new Map();
let lastCleanupAt = Date.now();
function cleanupMaps(now) {
if (
now - lastCleanupAt < CLEANUP_INTERVAL_MS &&
idempotencyMap.size <= IDEMPOTENCY_MAX_ENTRIES
) {
return;
}
lastCleanupAt = now;
for (const [key, timestamp] of idempotencyMap.entries()) {
if (now - timestamp > IDEMPOTENCY_TTL_MS) {
idempotencyMap.delete(key);
}
}
for (const [clientId, rl] of rateLimitMap.entries()) {
if (now - rl.start > 10000) {
rateLimitMap.delete(clientId);
}
}
}
function validateSingleEvent(clientId, eventId, rawData, clientRateLimit) {
const config = getRuntimeConfig();
const now = Date.now();
cleanupMaps(now);
// 1. Idempotency check (with 5-minute TTL)
if (config.enableIdempotency && eventId) {
const key = `${clientId}:${eventId}`;
const existingTime = idempotencyMap.get(key);
if (existingTime && now - existingTime < IDEMPOTENCY_TTL_MS) {
return {
allowed: false,
duplicate: true,
reason: "Duplicate event dropped",
};
}
idempotencyMap.set(key, now);
}
// 2. Rate limit check
const effectiveLimit = Math.min(
Number(clientRateLimit) || 100,
Number(config.maxEventsPerSec) || 100,
);
const rl = rateLimitMap.get(clientId);
if (!rl || now - rl.start >= 1000) {
rateLimitMap.set(clientId, { count: 1, start: now });
} else if (rl.count >= effectiveLimit) {
return {
allowed: false,
duplicate: false,
reason: `RATE_LIMIT_EXCEEDED: Limit of ${effectiveLimit} events/sec exceeded`,
};
} else {
rl.count += 1;
}
// 3. Payload byte size check
const payloadStr =
typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {});
const byteLength = Buffer.byteLength(payloadStr, "utf8");
const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024;
if (byteLength > maxBytes) {
return {
allowed: false,
duplicate: false,
reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`,
};
}
return { allowed: true, payloadStr };
}
// Middleware: Handshake Auth
io.use((socket, next) => {
io.use(async (socket, next) => {
const authHeader =
socket.handshake.auth?.token ||
socket.handshake.headers?.authorization ||
@@ -22,12 +125,25 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
}
try {
const cleanToken = authHeader.startsWith("Bearer ")
? authHeader.slice(7)
: authHeader;
const decoded = jwt.verify(cleanToken, jwtSecret);
const cleanToken = String(authHeader).startsWith("Bearer ")
? String(authHeader).slice(7).trim()
: String(authHeader).trim();
const config = getRuntimeConfig();
const decoded = jwt.verify(cleanToken, config.jwtSecret);
const dbCheck = await verifyClientInDb(
decoded.clientId,
cleanToken,
true,
);
if (!dbCheck.allowed) {
return next(new Error(dbCheck.reason || "CLIENT_REVOKED"));
}
socket.data.clientId = decoded.clientId;
socket.data.rateLimit = decoded.rateLimit || 100;
socket.data.token = cleanToken;
socket.data.rateLimit =
dbCheck.rateLimit || decoded.rateLimit || config.maxEventsPerSec || 100;
next();
} catch (err) {
next(new Error(`AUTH_INVALID: ${err.message}`));
@@ -36,50 +152,64 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
io.on("connection", (socket) => {
const clientId = socket.data.clientId;
const rateLimit = socket.data.rateLimit;
socket.join(`client:${clientId}`);
// Single event
socket.on("sync_data", (payload, callback) => {
socket.on("sync_data", async (payload, callback) => {
try {
const eventId =
payload.eventId ||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
const timestamp = payload.timestamp || new Date().toISOString();
// Idempotency
const key = `${clientId}:${eventId}`;
if (idempotencyMap.has(key)) {
if (callback)
callback({ success: true, eventId, note: "DUPLICATE_DROPPED" });
const dbCheck = await verifyClientInDb(
clientId,
socket.data.token,
false,
);
if (!dbCheck.allowed) {
if (callback) {
callback({
success: false,
error: dbCheck.reason || "CLIENT_REVOKED",
});
}
socket.disconnect(true);
return;
}
idempotencyMap.set(key, Date.now());
// Rate limit
const now = Date.now();
const rl = rateLimitMap.get(clientId);
if (!rl || now - rl.start >= 1000) {
rateLimitMap.set(clientId, { count: 1, start: now });
} else if (rl.count >= rateLimit) {
if (callback)
callback({ success: false, error: "RATE_LIMIT_EXCEEDED" });
const effectiveClientRate =
dbCheck.rateLimit || socket.data.rateLimit || 100;
const safePayload = payload && typeof payload === "object" ? payload : {};
const eventId =
safePayload.eventId ||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
const timestamp = safePayload.timestamp || new Date().toISOString();
const check = validateSingleEvent(
clientId,
eventId,
safePayload.data,
effectiveClientRate,
);
if (!check.allowed) {
if (check.duplicate) {
if (callback) {
callback({ success: true, eventId, note: "DUPLICATE_DROPPED" });
}
return;
}
if (callback) {
callback({ success: false, eventId, error: check.reason });
}
return;
} else {
rl.count += 1;
}
const event = {
clientId,
eventId,
timestamp,
payload:
typeof payload.data === "string"
? payload.data
: JSON.stringify(payload.data),
payload: check.payloadStr,
};
bufferQueue.push(event);
recordClientActivity(clientId, 1);
onBufferPush();
if (callback) callback({ success: true, eventId });
} catch (err) {
@@ -88,32 +218,94 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
});
// Batch event
socket.on("batch_sync", (batch, callback) => {
if (!Array.isArray(batch)) {
if (callback) callback({ success: false, error: "Expected array" });
return;
}
socket.on("batch_sync", async (batch, callback) => {
try {
if (!Array.isArray(batch)) {
if (callback) callback({ success: false, error: "Expected array" });
return;
}
let count = 0;
for (const item of batch) {
const eventId =
item.eventId ||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
const timestamp = item.timestamp || new Date().toISOString();
bufferQueue.push({
const dbCheck = await verifyClientInDb(
clientId,
eventId,
timestamp,
payload:
typeof item.data === "string"
? item.data
: JSON.stringify(item.data),
});
count++;
}
socket.data.token,
false,
);
if (!dbCheck.allowed) {
if (callback) {
callback({
success: false,
processed: 0,
rejected: batch.length,
error: dbCheck.reason || "CLIENT_REVOKED",
});
}
socket.disconnect(true);
return;
}
if (callback) callback({ success: true, processed: count });
const effectiveClientRate =
dbCheck.rateLimit || socket.data.rateLimit || 100;
let processed = 0;
let rejected = 0;
const errors = [];
for (const item of batch) {
const safeItem = item && typeof item === "object" ? item : {};
const eventId =
safeItem.eventId ||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
const timestamp = safeItem.timestamp || new Date().toISOString();
const rawData =
safeItem.data !== undefined ? safeItem.data : safeItem;
const check = validateSingleEvent(
clientId,
eventId,
rawData,
effectiveClientRate,
);
if (!check.allowed) {
if (!check.duplicate) {
rejected += 1;
if (errors.length < 5) {
errors.push(`${eventId}: ${check.reason}`);
}
}
continue;
}
bufferQueue.push({
clientId,
eventId,
timestamp,
payload: check.payloadStr,
});
processed += 1;
}
if (processed > 0) {
recordClientActivity(clientId, processed);
onBufferPush();
}
if (callback) {
callback({
success: true,
processed,
rejected,
...(errors.length > 0 ? { errors } : {}),
});
}
} catch (err) {
if (callback) {
callback({
success: false,
processed: 0,
rejected: Array.isArray(batch) ? batch.length : 0,
error: err.message,
});
}
}
});
});
}