feat: aggregate external events and improve security
- Fetch events from external ingestion service in API routes - Decouple idempotency check from registration in Security utility - Refine CSV storage logic to verify file content for headers - Replace Python SDK with comprehensive, feature-rich implementation
This commit is contained in:
1 parent
043123dc21
commit
34519c07df
13 files changed
+2099
-451
No files matched your search
@@ -289,6 +289,18 @@ app.post('/internal/flush', async (_req, res) => {
|
||||
}
|
||||
});
|
||||
|
||||
// Internal endpoint to expose Port 4000 buffered events before CSV flush
|
||||
app.get('/internal/recent', (req, res) => {
|
||||
const clientId = req.query.clientId ? String(req.query.clientId) : '';
|
||||
const limit = Math.min(200, Math.max(1, Number(req.query.limit) || 50));
|
||||
const filtered =
|
||||
clientId && clientId !== 'all'
|
||||
? bufferQueue.filter((evt) => evt.clientId === clientId)
|
||||
: bufferQueue;
|
||||
const recent = filtered.slice(-limit).reverse();
|
||||
res.json({ events: recent, count: recent.length });
|
||||
});
|
||||
|
||||
setupSocketHandlers(io, bufferQueue, writer, {
|
||||
getRuntimeConfig: () => runtimeConfig,
|
||||
verifyClientInDb,
|
||||
|
||||
+21
-17
@@ -65,10 +65,10 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
const now = Date.now();
|
||||
cleanupMaps(now);
|
||||
|
||||
// 1. Idempotency check (with 5-minute TTL)
|
||||
// 1. Idempotency check (read-only check first; record only after rate-limit & payload checks pass)
|
||||
const idempotencyKey = `${clientId}:${eventId}`;
|
||||
if (config.enableIdempotency && eventId) {
|
||||
const key = `${clientId}:${eventId}`;
|
||||
const existingTime = idempotencyMap.get(key);
|
||||
const existingTime = idempotencyMap.get(idempotencyKey);
|
||||
if (existingTime && now - existingTime < IDEMPOTENCY_TTL_MS) {
|
||||
return {
|
||||
allowed: false,
|
||||
@@ -77,10 +77,23 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
reason: "Duplicate event dropped",
|
||||
};
|
||||
}
|
||||
idempotencyMap.set(key, now);
|
||||
}
|
||||
|
||||
// 2. Rate limit check
|
||||
// 2. Payload byte size check (before consuming rate-limit quota)
|
||||
const payloadStr =
|
||||
typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {});
|
||||
const byteLength = Buffer.byteLength(payloadStr, "utf8");
|
||||
const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024;
|
||||
if (byteLength > maxBytes) {
|
||||
return {
|
||||
allowed: false,
|
||||
duplicate: false,
|
||||
code: "PAYLOAD_TOO_LARGE",
|
||||
reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`,
|
||||
};
|
||||
}
|
||||
|
||||
// 3. Rate limit check
|
||||
const effectiveLimit = Math.min(
|
||||
Number(clientRateLimit) || 100,
|
||||
Number(config.maxEventsPerSec) || 100,
|
||||
@@ -99,18 +112,9 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
rl.count += 1;
|
||||
}
|
||||
|
||||
// 3. Payload byte size check
|
||||
const payloadStr =
|
||||
typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {});
|
||||
const byteLength = Buffer.byteLength(payloadStr, "utf8");
|
||||
const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024;
|
||||
if (byteLength > maxBytes) {
|
||||
return {
|
||||
allowed: false,
|
||||
duplicate: false,
|
||||
code: "PAYLOAD_TOO_LARGE",
|
||||
reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`,
|
||||
};
|
||||
// 4. Record idempotency key ONLY after event passes all checks
|
||||
if (config.enableIdempotency && eventId) {
|
||||
idempotencyMap.set(idempotencyKey, now);
|
||||
}
|
||||
|
||||
return { allowed: true, payloadStr };
|
||||
|
||||
@@ -74,7 +74,13 @@ class CsvWriterWorker {
|
||||
for (const [key, batch] of grouped.entries()) {
|
||||
const [clientId, date] = key.split(':::');
|
||||
const targetPath = this.getTargetFilePath(clientId, date);
|
||||
const fileExists = fs.existsSync(targetPath);
|
||||
let hasContent = false;
|
||||
try {
|
||||
hasContent =
|
||||
fs.existsSync(targetPath) && fs.statSync(targetPath).size > 0;
|
||||
} catch {
|
||||
hasContent = false;
|
||||
}
|
||||
|
||||
const cleanBatch = batch.map((item) => ({
|
||||
clientId: item.clientId,
|
||||
@@ -83,10 +89,10 @@ class CsvWriterWorker {
|
||||
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
|
||||
}));
|
||||
|
||||
const parser = new Parser({ fields, header: !fileExists, eol: '\n' });
|
||||
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
|
||||
const csv = parser.parse(cleanBatch);
|
||||
|
||||
if (fileExists) {
|
||||
if (hasContent) {
|
||||
fs.appendFileSync(targetPath, '\n' + csv, 'utf8');
|
||||
} else {
|
||||
fs.writeFileSync(targetPath, csv, 'utf8');
|
||||
|
||||
Reference in new issue
Block a user