refactor: switch SDK methods to asynchronous
Migrate SDK interactions to asynchronous calls, relax admin setup email constraints, and add event retrieval functionality.
This commit is contained in:
1 parent
81c5e4e2fd
commit
bb938c33c0
36 files changed
+1679
-754
No files matched your search
+227
-24
@@ -2,57 +2,260 @@ const express = require('express');
|
||||
const http = require('http');
|
||||
const cors = require('cors');
|
||||
const { Server } = require('socket.io');
|
||||
const { Pool } = require('pg');
|
||||
const { setupSocketHandlers } = require('./socket/handlers');
|
||||
const { CsvWriterWorker } = require('./workers/csvWriter');
|
||||
|
||||
const PORT = process.env.PORT_SOCKET || 4000;
|
||||
const PORT = Number(process.env.PORT_SOCKET) || 4000;
|
||||
const app = express();
|
||||
app.use(cors({ origin: '*' }));
|
||||
app.use(express.json());
|
||||
app.use(express.json({ limit: '10mb' }));
|
||||
|
||||
const bufferQueue = [];
|
||||
const writer = new CsvWriterWorker();
|
||||
|
||||
// Periodic Batch Flush: every 2000ms or when buffer hits 500
|
||||
const FLUSH_INTERVAL_MS = 2000;
|
||||
const FLUSH_THRESHOLD = 500;
|
||||
const runtimeConfig = {
|
||||
jwtSecret:
|
||||
process.env.DATASTACK_JWT_SECRET ||
|
||||
'datastack-jwt-default-secret-key-32-chars-long',
|
||||
maxEventsPerSec: 100,
|
||||
maxPayloadSizeBytes: 100 * 1024,
|
||||
batchFlushThreshold: 500,
|
||||
batchFlushIntervalMs: 2000,
|
||||
fileRotationMb: 100,
|
||||
enableIdempotency: true,
|
||||
};
|
||||
|
||||
let dbPool = null;
|
||||
if (process.env.DATABASE_URL) {
|
||||
try {
|
||||
dbPool = new Pool({
|
||||
connectionString: process.env.DATABASE_URL,
|
||||
connectionTimeoutMillis: 3000,
|
||||
idleTimeoutMillis: 30000,
|
||||
max: 5,
|
||||
});
|
||||
} catch {
|
||||
dbPool = null;
|
||||
}
|
||||
}
|
||||
|
||||
// Short-lived cache (2s TTL) for client status & token checks on high-frequency events
|
||||
const clientAuthCache = new Map();
|
||||
const CLIENT_CACHE_TTL_MS = 2000;
|
||||
|
||||
async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
|
||||
if (!dbPool || !clientId) {
|
||||
return { allowed: true };
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
if (!bypassCache) {
|
||||
const cached = clientAuthCache.get(clientId);
|
||||
if (cached && now - cached.checkedAt < CLIENT_CACHE_TTL_MS) {
|
||||
if (cached.status === 'revoked') {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
|
||||
};
|
||||
}
|
||||
if (cached.token && presentedToken && cached.token !== presentedToken) {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
|
||||
};
|
||||
}
|
||||
return { allowed: true, rateLimit: cached.rateLimit };
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const res = await dbPool.query(
|
||||
`SELECT status, token, rate_limit FROM client_sdks WHERE client_id = $1`,
|
||||
[clientId],
|
||||
);
|
||||
if (!res.rowCount || res.rowCount === 0) {
|
||||
// Allow token-authenticated client if not explicitly revoked in table
|
||||
return { allowed: true };
|
||||
}
|
||||
const row = res.rows[0];
|
||||
const entry = {
|
||||
status: row.status,
|
||||
token: row.token,
|
||||
rateLimit: Number(row.rate_limit) || 100,
|
||||
checkedAt: now,
|
||||
};
|
||||
clientAuthCache.set(clientId, entry);
|
||||
|
||||
if (entry.status === 'revoked') {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
|
||||
};
|
||||
}
|
||||
if (entry.token && presentedToken && entry.token !== presentedToken) {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
|
||||
};
|
||||
}
|
||||
return { allowed: true, rateLimit: entry.rateLimit };
|
||||
} catch {
|
||||
return { allowed: true };
|
||||
}
|
||||
}
|
||||
|
||||
function recordClientActivity(clientId, count = 1) {
|
||||
if (!dbPool || !clientId || count <= 0) return;
|
||||
const nowIso = new Date().toISOString();
|
||||
dbPool
|
||||
.query(
|
||||
`UPDATE client_sdks
|
||||
SET total_events = total_events + $2,
|
||||
last_active = $3,
|
||||
updated_at = NOW()
|
||||
WHERE client_id = $1`,
|
||||
[clientId, count, nowIso],
|
||||
)
|
||||
.catch(() => undefined);
|
||||
}
|
||||
|
||||
let isFlushing = false;
|
||||
let flushTimer = null;
|
||||
|
||||
async function flushBuffer() {
|
||||
if (bufferQueue.length === 0) return;
|
||||
if (bufferQueue.length === 0 || isFlushing) {
|
||||
return { written: 0, files: [] };
|
||||
}
|
||||
isFlushing = true;
|
||||
const batch = bufferQueue.splice(0, bufferQueue.length);
|
||||
try {
|
||||
const result = await writer.writeBatch(batch);
|
||||
if (result.written > 0) {
|
||||
console.log(`[CSV Worker] Flushed ${result.written} events to ${result.files.length} file(s)`);
|
||||
console.log(
|
||||
`[CSV Worker] Flushed ${result.written} events to ${result.files.length} file(s)`,
|
||||
);
|
||||
}
|
||||
return result;
|
||||
} catch (err) {
|
||||
console.error('[CSV Worker] Flush error:', err);
|
||||
bufferQueue.unshift(...batch); // Retry next cycle
|
||||
bufferQueue.unshift(...batch);
|
||||
return { written: 0, files: [] };
|
||||
} finally {
|
||||
isFlushing = false;
|
||||
}
|
||||
}
|
||||
|
||||
setInterval(flushBuffer, FLUSH_INTERVAL_MS);
|
||||
function scheduleFlushTimer(intervalMs) {
|
||||
const safeInterval = Math.max(100, Number(intervalMs) || 2000);
|
||||
runtimeConfig.batchFlushIntervalMs = safeInterval;
|
||||
if (flushTimer) {
|
||||
clearInterval(flushTimer);
|
||||
}
|
||||
flushTimer = setInterval(() => {
|
||||
void flushBuffer();
|
||||
}, safeInterval);
|
||||
}
|
||||
|
||||
// Health check endpoint
|
||||
app.get('/health', (_req, res) => {
|
||||
res.json({
|
||||
status: 'online',
|
||||
service: 'DataStack Express Ingestion',
|
||||
port: PORT,
|
||||
bufferSize: bufferQueue.length,
|
||||
uptime: process.uptime(),
|
||||
});
|
||||
});
|
||||
scheduleFlushTimer(runtimeConfig.batchFlushIntervalMs);
|
||||
|
||||
async function syncSettingsFromDatabase() {
|
||||
if (!dbPool) return;
|
||||
try {
|
||||
const res = await dbPool.query(
|
||||
`SELECT max_events_per_sec, max_payload_size_kb, batch_flush_threshold,
|
||||
batch_flush_interval_ms, file_rotation_mb, jwt_secret, enable_idempotency
|
||||
FROM admin_settings
|
||||
WHERE id = 'default'`,
|
||||
);
|
||||
if (!res.rowCount || res.rowCount === 0) return;
|
||||
const row = res.rows[0];
|
||||
|
||||
runtimeConfig.maxEventsPerSec = Number(row.max_events_per_sec) || 100;
|
||||
runtimeConfig.maxPayloadSizeBytes =
|
||||
(Number(row.max_payload_size_kb) || 100) * 1024;
|
||||
runtimeConfig.batchFlushThreshold =
|
||||
Math.max(1, Number(row.batch_flush_threshold) || 500);
|
||||
runtimeConfig.enableIdempotency = Boolean(row.enable_idempotency);
|
||||
|
||||
if (row.jwt_secret && String(row.jwt_secret).trim().length > 0) {
|
||||
runtimeConfig.jwtSecret = String(row.jwt_secret).trim();
|
||||
}
|
||||
|
||||
const nextRotationMb = Number(row.file_rotation_mb) || 100;
|
||||
if (nextRotationMb !== runtimeConfig.fileRotationMb) {
|
||||
runtimeConfig.fileRotationMb = nextRotationMb;
|
||||
writer.setMaxFileSizeMb(nextRotationMb);
|
||||
}
|
||||
|
||||
const nextIntervalMs = Math.max(
|
||||
100,
|
||||
Number(row.batch_flush_interval_ms) || 2000,
|
||||
);
|
||||
if (nextIntervalMs !== runtimeConfig.batchFlushIntervalMs) {
|
||||
scheduleFlushTimer(nextIntervalMs);
|
||||
}
|
||||
} catch {
|
||||
// Keep existing runtimeConfig if table is not created yet
|
||||
}
|
||||
}
|
||||
|
||||
if (dbPool) {
|
||||
void syncSettingsFromDatabase();
|
||||
setInterval(() => {
|
||||
void syncSettingsFromDatabase();
|
||||
}, 3000);
|
||||
}
|
||||
|
||||
const server = http.createServer(app);
|
||||
const io = new Server(server, {
|
||||
cors: { origin: '*', methods: ['GET', 'POST'] },
|
||||
});
|
||||
|
||||
setupSocketHandlers(io, bufferQueue, writer);
|
||||
|
||||
server.listen(PORT, () => {
|
||||
console.log(`[DataStack Ingestion Engine] Express + Socket.IO listening on port ${PORT}`);
|
||||
// Health check endpoint with activeSockets count
|
||||
app.get('/health', (_req, res) => {
|
||||
res.json({
|
||||
status: 'online',
|
||||
service: 'DataStack Express Ingestion',
|
||||
port: PORT,
|
||||
activeSockets: io.engine ? io.engine.clientsCount : 0,
|
||||
bufferSize: bufferQueue.length,
|
||||
uptime: process.uptime(),
|
||||
});
|
||||
});
|
||||
|
||||
module.exports = { app, server, io, bufferQueue };
|
||||
// Internal flush endpoint triggered by Next.js POST /api/storage
|
||||
app.post('/internal/flush', async (_req, res) => {
|
||||
try {
|
||||
const result = await flushBuffer();
|
||||
res.json({
|
||||
success: true,
|
||||
flushedCount: result.written,
|
||||
filesUpdated: result.files,
|
||||
});
|
||||
} catch (err) {
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: err instanceof Error ? err.message : 'Flush failed',
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
setupSocketHandlers(io, bufferQueue, writer, {
|
||||
getRuntimeConfig: () => runtimeConfig,
|
||||
verifyClientInDb,
|
||||
recordClientActivity,
|
||||
onBufferPush: () => {
|
||||
if (bufferQueue.length >= runtimeConfig.batchFlushThreshold) {
|
||||
void flushBuffer();
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
server.listen(PORT, () => {
|
||||
console.log(
|
||||
`[DataStack Ingestion Engine] Express + Socket.IO listening on port ${PORT}`,
|
||||
);
|
||||
});
|
||||
|
||||
module.exports = { app, server, io, bufferQueue, flushBuffer };
|
||||
+254
-62
@@ -1,17 +1,120 @@
|
||||
const jwt = require("jsonwebtoken");
|
||||
|
||||
const IDEMPOTENCY_TTL_MS = 5 * 60 * 1000; // 5 minutes
|
||||
const IDEMPOTENCY_MAX_ENTRIES = 20000;
|
||||
const CLEANUP_INTERVAL_MS = 60 * 1000;
|
||||
|
||||
function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
const jwtSecret = options.jwtSecret || process.env.DATASTACK_JWT_SECRET;
|
||||
if (!jwtSecret) {
|
||||
throw new Error(
|
||||
"DATASTACK_JWT_SECRET must be configured before starting the socket server.",
|
||||
);
|
||||
}
|
||||
const getRuntimeConfig =
|
||||
typeof options.getRuntimeConfig === "function"
|
||||
? options.getRuntimeConfig
|
||||
: () => ({
|
||||
jwtSecret:
|
||||
options.jwtSecret ||
|
||||
process.env.DATASTACK_JWT_SECRET ||
|
||||
"datastack-jwt-default-secret-key-32-chars-long",
|
||||
maxEventsPerSec: 100,
|
||||
maxPayloadSizeBytes: 100 * 1024,
|
||||
batchFlushThreshold: 500,
|
||||
enableIdempotency: true,
|
||||
});
|
||||
|
||||
const verifyClientInDb =
|
||||
typeof options.verifyClientInDb === "function"
|
||||
? options.verifyClientInDb
|
||||
: async () => ({ allowed: true });
|
||||
|
||||
const recordClientActivity =
|
||||
typeof options.recordClientActivity === "function"
|
||||
? options.recordClientActivity
|
||||
: () => undefined;
|
||||
|
||||
const onBufferPush =
|
||||
typeof options.onBufferPush === "function"
|
||||
? options.onBufferPush
|
||||
: () => undefined;
|
||||
|
||||
const idempotencyMap = new Map();
|
||||
const rateLimitMap = new Map();
|
||||
let lastCleanupAt = Date.now();
|
||||
|
||||
function cleanupMaps(now) {
|
||||
if (
|
||||
now - lastCleanupAt < CLEANUP_INTERVAL_MS &&
|
||||
idempotencyMap.size <= IDEMPOTENCY_MAX_ENTRIES
|
||||
) {
|
||||
return;
|
||||
}
|
||||
lastCleanupAt = now;
|
||||
|
||||
for (const [key, timestamp] of idempotencyMap.entries()) {
|
||||
if (now - timestamp > IDEMPOTENCY_TTL_MS) {
|
||||
idempotencyMap.delete(key);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [clientId, rl] of rateLimitMap.entries()) {
|
||||
if (now - rl.start > 10000) {
|
||||
rateLimitMap.delete(clientId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function validateSingleEvent(clientId, eventId, rawData, clientRateLimit) {
|
||||
const config = getRuntimeConfig();
|
||||
const now = Date.now();
|
||||
cleanupMaps(now);
|
||||
|
||||
// 1. Idempotency check (with 5-minute TTL)
|
||||
if (config.enableIdempotency && eventId) {
|
||||
const key = `${clientId}:${eventId}`;
|
||||
const existingTime = idempotencyMap.get(key);
|
||||
if (existingTime && now - existingTime < IDEMPOTENCY_TTL_MS) {
|
||||
return {
|
||||
allowed: false,
|
||||
duplicate: true,
|
||||
reason: "Duplicate event dropped",
|
||||
};
|
||||
}
|
||||
idempotencyMap.set(key, now);
|
||||
}
|
||||
|
||||
// 2. Rate limit check
|
||||
const effectiveLimit = Math.min(
|
||||
Number(clientRateLimit) || 100,
|
||||
Number(config.maxEventsPerSec) || 100,
|
||||
);
|
||||
const rl = rateLimitMap.get(clientId);
|
||||
if (!rl || now - rl.start >= 1000) {
|
||||
rateLimitMap.set(clientId, { count: 1, start: now });
|
||||
} else if (rl.count >= effectiveLimit) {
|
||||
return {
|
||||
allowed: false,
|
||||
duplicate: false,
|
||||
reason: `RATE_LIMIT_EXCEEDED: Limit of ${effectiveLimit} events/sec exceeded`,
|
||||
};
|
||||
} else {
|
||||
rl.count += 1;
|
||||
}
|
||||
|
||||
// 3. Payload byte size check
|
||||
const payloadStr =
|
||||
typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {});
|
||||
const byteLength = Buffer.byteLength(payloadStr, "utf8");
|
||||
const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024;
|
||||
if (byteLength > maxBytes) {
|
||||
return {
|
||||
allowed: false,
|
||||
duplicate: false,
|
||||
reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`,
|
||||
};
|
||||
}
|
||||
|
||||
return { allowed: true, payloadStr };
|
||||
}
|
||||
|
||||
// Middleware: Handshake Auth
|
||||
io.use((socket, next) => {
|
||||
io.use(async (socket, next) => {
|
||||
const authHeader =
|
||||
socket.handshake.auth?.token ||
|
||||
socket.handshake.headers?.authorization ||
|
||||
@@ -22,12 +125,25 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
}
|
||||
|
||||
try {
|
||||
const cleanToken = authHeader.startsWith("Bearer ")
|
||||
? authHeader.slice(7)
|
||||
: authHeader;
|
||||
const decoded = jwt.verify(cleanToken, jwtSecret);
|
||||
const cleanToken = String(authHeader).startsWith("Bearer ")
|
||||
? String(authHeader).slice(7).trim()
|
||||
: String(authHeader).trim();
|
||||
const config = getRuntimeConfig();
|
||||
const decoded = jwt.verify(cleanToken, config.jwtSecret);
|
||||
|
||||
const dbCheck = await verifyClientInDb(
|
||||
decoded.clientId,
|
||||
cleanToken,
|
||||
true,
|
||||
);
|
||||
if (!dbCheck.allowed) {
|
||||
return next(new Error(dbCheck.reason || "CLIENT_REVOKED"));
|
||||
}
|
||||
|
||||
socket.data.clientId = decoded.clientId;
|
||||
socket.data.rateLimit = decoded.rateLimit || 100;
|
||||
socket.data.token = cleanToken;
|
||||
socket.data.rateLimit =
|
||||
dbCheck.rateLimit || decoded.rateLimit || config.maxEventsPerSec || 100;
|
||||
next();
|
||||
} catch (err) {
|
||||
next(new Error(`AUTH_INVALID: ${err.message}`));
|
||||
@@ -36,50 +152,64 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
|
||||
io.on("connection", (socket) => {
|
||||
const clientId = socket.data.clientId;
|
||||
const rateLimit = socket.data.rateLimit;
|
||||
socket.join(`client:${clientId}`);
|
||||
|
||||
// Single event
|
||||
socket.on("sync_data", (payload, callback) => {
|
||||
socket.on("sync_data", async (payload, callback) => {
|
||||
try {
|
||||
const eventId =
|
||||
payload.eventId ||
|
||||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||
const timestamp = payload.timestamp || new Date().toISOString();
|
||||
|
||||
// Idempotency
|
||||
const key = `${clientId}:${eventId}`;
|
||||
if (idempotencyMap.has(key)) {
|
||||
if (callback)
|
||||
callback({ success: true, eventId, note: "DUPLICATE_DROPPED" });
|
||||
const dbCheck = await verifyClientInDb(
|
||||
clientId,
|
||||
socket.data.token,
|
||||
false,
|
||||
);
|
||||
if (!dbCheck.allowed) {
|
||||
if (callback) {
|
||||
callback({
|
||||
success: false,
|
||||
error: dbCheck.reason || "CLIENT_REVOKED",
|
||||
});
|
||||
}
|
||||
socket.disconnect(true);
|
||||
return;
|
||||
}
|
||||
idempotencyMap.set(key, Date.now());
|
||||
|
||||
// Rate limit
|
||||
const now = Date.now();
|
||||
const rl = rateLimitMap.get(clientId);
|
||||
if (!rl || now - rl.start >= 1000) {
|
||||
rateLimitMap.set(clientId, { count: 1, start: now });
|
||||
} else if (rl.count >= rateLimit) {
|
||||
if (callback)
|
||||
callback({ success: false, error: "RATE_LIMIT_EXCEEDED" });
|
||||
const effectiveClientRate =
|
||||
dbCheck.rateLimit || socket.data.rateLimit || 100;
|
||||
const safePayload = payload && typeof payload === "object" ? payload : {};
|
||||
const eventId =
|
||||
safePayload.eventId ||
|
||||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||
const timestamp = safePayload.timestamp || new Date().toISOString();
|
||||
|
||||
const check = validateSingleEvent(
|
||||
clientId,
|
||||
eventId,
|
||||
safePayload.data,
|
||||
effectiveClientRate,
|
||||
);
|
||||
if (!check.allowed) {
|
||||
if (check.duplicate) {
|
||||
if (callback) {
|
||||
callback({ success: true, eventId, note: "DUPLICATE_DROPPED" });
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (callback) {
|
||||
callback({ success: false, eventId, error: check.reason });
|
||||
}
|
||||
return;
|
||||
} else {
|
||||
rl.count += 1;
|
||||
}
|
||||
|
||||
const event = {
|
||||
clientId,
|
||||
eventId,
|
||||
timestamp,
|
||||
payload:
|
||||
typeof payload.data === "string"
|
||||
? payload.data
|
||||
: JSON.stringify(payload.data),
|
||||
payload: check.payloadStr,
|
||||
};
|
||||
|
||||
bufferQueue.push(event);
|
||||
recordClientActivity(clientId, 1);
|
||||
onBufferPush();
|
||||
|
||||
if (callback) callback({ success: true, eventId });
|
||||
} catch (err) {
|
||||
@@ -88,32 +218,94 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
});
|
||||
|
||||
// Batch event
|
||||
socket.on("batch_sync", (batch, callback) => {
|
||||
if (!Array.isArray(batch)) {
|
||||
if (callback) callback({ success: false, error: "Expected array" });
|
||||
return;
|
||||
}
|
||||
socket.on("batch_sync", async (batch, callback) => {
|
||||
try {
|
||||
if (!Array.isArray(batch)) {
|
||||
if (callback) callback({ success: false, error: "Expected array" });
|
||||
return;
|
||||
}
|
||||
|
||||
let count = 0;
|
||||
for (const item of batch) {
|
||||
const eventId =
|
||||
item.eventId ||
|
||||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||
const timestamp = item.timestamp || new Date().toISOString();
|
||||
|
||||
bufferQueue.push({
|
||||
const dbCheck = await verifyClientInDb(
|
||||
clientId,
|
||||
eventId,
|
||||
timestamp,
|
||||
payload:
|
||||
typeof item.data === "string"
|
||||
? item.data
|
||||
: JSON.stringify(item.data),
|
||||
});
|
||||
count++;
|
||||
}
|
||||
socket.data.token,
|
||||
false,
|
||||
);
|
||||
if (!dbCheck.allowed) {
|
||||
if (callback) {
|
||||
callback({
|
||||
success: false,
|
||||
processed: 0,
|
||||
rejected: batch.length,
|
||||
error: dbCheck.reason || "CLIENT_REVOKED",
|
||||
});
|
||||
}
|
||||
socket.disconnect(true);
|
||||
return;
|
||||
}
|
||||
|
||||
if (callback) callback({ success: true, processed: count });
|
||||
const effectiveClientRate =
|
||||
dbCheck.rateLimit || socket.data.rateLimit || 100;
|
||||
let processed = 0;
|
||||
let rejected = 0;
|
||||
const errors = [];
|
||||
|
||||
for (const item of batch) {
|
||||
const safeItem = item && typeof item === "object" ? item : {};
|
||||
const eventId =
|
||||
safeItem.eventId ||
|
||||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||
const timestamp = safeItem.timestamp || new Date().toISOString();
|
||||
const rawData =
|
||||
safeItem.data !== undefined ? safeItem.data : safeItem;
|
||||
|
||||
const check = validateSingleEvent(
|
||||
clientId,
|
||||
eventId,
|
||||
rawData,
|
||||
effectiveClientRate,
|
||||
);
|
||||
if (!check.allowed) {
|
||||
if (!check.duplicate) {
|
||||
rejected += 1;
|
||||
if (errors.length < 5) {
|
||||
errors.push(`${eventId}: ${check.reason}`);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
bufferQueue.push({
|
||||
clientId,
|
||||
eventId,
|
||||
timestamp,
|
||||
payload: check.payloadStr,
|
||||
});
|
||||
processed += 1;
|
||||
}
|
||||
|
||||
if (processed > 0) {
|
||||
recordClientActivity(clientId, processed);
|
||||
onBufferPush();
|
||||
}
|
||||
|
||||
if (callback) {
|
||||
callback({
|
||||
success: true,
|
||||
processed,
|
||||
rejected,
|
||||
...(errors.length > 0 ? { errors } : {}),
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
if (callback) {
|
||||
callback({
|
||||
success: false,
|
||||
processed: 0,
|
||||
rejected: Array.isArray(batch) ? batch.length : 0,
|
||||
error: err.message,
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@@ -14,6 +14,13 @@ class CsvWriterWorker {
|
||||
this.ensureBaseDir();
|
||||
}
|
||||
|
||||
setMaxFileSizeMb(megabytes) {
|
||||
const mb = Number(megabytes);
|
||||
if (Number.isFinite(mb) && mb >= 1) {
|
||||
this.maxFileSize = Math.floor(mb * 1024 * 1024);
|
||||
}
|
||||
}
|
||||
|
||||
ensureBaseDir() {
|
||||
if (!fs.existsSync(this.baseDir)) {
|
||||
fs.mkdirSync(this.baseDir, { recursive: true });
|
||||
@@ -21,7 +28,7 @@ class CsvWriterWorker {
|
||||
}
|
||||
|
||||
getTargetFilePath(clientId, dateStr) {
|
||||
const safeClientId = clientId.replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
const safeClientId = String(clientId || 'unknown').replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
const clientDir = path.join(this.baseDir, safeClientId);
|
||||
if (!fs.existsSync(clientDir)) {
|
||||
fs.mkdirSync(clientDir, { recursive: true });
|
||||
|
||||
Reference in new issue
Block a user