feat: aggregate external events and improve security

- Fetch events from external ingestion service in API routes
- Decouple idempotency check from registration in Security utility
- Refine CSV storage logic to verify file content for headers
- Replace Python SDK with comprehensive, feature-rich implementation
This commit is contained in:
Team Dark Devil committed 2026-11-18 03:20:00 +06:00
1 parent 043123dc21
commit 34519c07df
13 files changed
+2099 -451

No files matched your search

+12
View File
@@ -289,6 +289,18 @@ app.post('/internal/flush', async (_req, res) => {
}
});
// Internal endpoint to expose Port 4000 buffered events before CSV flush
app.get('/internal/recent', (req, res) => {
const clientId = req.query.clientId ? String(req.query.clientId) : '';
const limit = Math.min(200, Math.max(1, Number(req.query.limit) || 50));
const filtered =
clientId && clientId !== 'all'
? bufferQueue.filter((evt) => evt.clientId === clientId)
: bufferQueue;
const recent = filtered.slice(-limit).reverse();
res.json({ events: recent, count: recent.length });
});
setupSocketHandlers(io, bufferQueue, writer, {
getRuntimeConfig: () => runtimeConfig,
verifyClientInDb,
+21 -17
View File
@@ -65,10 +65,10 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
const now = Date.now();
cleanupMaps(now);
// 1. Idempotency check (with 5-minute TTL)
// 1. Idempotency check (read-only check first; record only after rate-limit & payload checks pass)
const idempotencyKey = `${clientId}:${eventId}`;
if (config.enableIdempotency && eventId) {
const key = `${clientId}:${eventId}`;
const existingTime = idempotencyMap.get(key);
const existingTime = idempotencyMap.get(idempotencyKey);
if (existingTime && now - existingTime < IDEMPOTENCY_TTL_MS) {
return {
allowed: false,
@@ -77,10 +77,23 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
reason: "Duplicate event dropped",
};
}
idempotencyMap.set(key, now);
}
// 2. Rate limit check
// 2. Payload byte size check (before consuming rate-limit quota)
const payloadStr =
typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {});
const byteLength = Buffer.byteLength(payloadStr, "utf8");
const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024;
if (byteLength > maxBytes) {
return {
allowed: false,
duplicate: false,
code: "PAYLOAD_TOO_LARGE",
reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`,
};
}
// 3. Rate limit check
const effectiveLimit = Math.min(
Number(clientRateLimit) || 100,
Number(config.maxEventsPerSec) || 100,
@@ -99,18 +112,9 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
rl.count += 1;
}
// 3. Payload byte size check
const payloadStr =
typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {});
const byteLength = Buffer.byteLength(payloadStr, "utf8");
const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024;
if (byteLength > maxBytes) {
return {
allowed: false,
duplicate: false,
code: "PAYLOAD_TOO_LARGE",
reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`,
};
// 4. Record idempotency key ONLY after event passes all checks
if (config.enableIdempotency && eventId) {
idempotencyMap.set(idempotencyKey, now);
}
return { allowed: true, payloadStr };
+9 -3
View File
@@ -74,7 +74,13 @@ class CsvWriterWorker {
for (const [key, batch] of grouped.entries()) {
const [clientId, date] = key.split(':::');
const targetPath = this.getTargetFilePath(clientId, date);
const fileExists = fs.existsSync(targetPath);
let hasContent = false;
try {
hasContent =
fs.existsSync(targetPath) && fs.statSync(targetPath).size > 0;
} catch {
hasContent = false;
}
const cleanBatch = batch.map((item) => ({
clientId: item.clientId,
@@ -83,10 +89,10 @@ class CsvWriterWorker {
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
}));
const parser = new Parser({ fields, header: !fileExists, eol: '\n' });
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
const csv = parser.parse(cleanBatch);
if (fileExists) {
if (hasContent) {
fs.appendFileSync(targetPath, '\n' + csv, 'utf8');
} else {
fs.writeFileSync(targetPath, csv, 'utf8');