feat: aggregate external events and improve security

- Fetch events from external ingestion service in API routes
- Decouple idempotency check from registration in Security utility
- Refine CSV storage logic to verify file content for headers
- Replace Python SDK with comprehensive, feature-rich implementation
This commit is contained in:
Team Dark Devil committed 2026-11-18 03:20:00 +06:00
1 parent 043123dc21
commit 34519c07df
13 files changed
+2099 -451

No files matched your search

+38 -22
View File
@@ -30,8 +30,8 @@ export class SecurityEngine {
* ClientId + EventId must be unique.
* Duplicate events are dropped silently or tagged.
*/
public static checkIdempotency(clientId: string, eventId: string): boolean {
if (!eventId) return true; // If no eventId generated, pass through
public static hasDuplicate(clientId: string, eventId: string): boolean {
if (!eventId) return false;
const key = `${clientId}:${eventId}`;
const now = Date.now();
@@ -47,12 +47,24 @@ export class SecurityEngine {
if (this.idempotencySet.has(key)) {
const storedTime = this.idempotencySet.get(key) || 0;
if (now - storedTime < this.IDEMPOTENCY_TTL_MS) {
return false; // Duplicate detected
return true;
}
}
this.idempotencySet.set(key, now);
return true; // Unique, allowed
return false;
}
public static recordIdempotency(clientId: string, eventId: string): void {
if (!eventId) return;
this.idempotencySet.set(`${clientId}:${eventId}`, Date.now());
}
public static checkIdempotency(clientId: string, eventId: string): boolean {
if (this.hasDuplicate(clientId, eventId)) {
return false;
}
this.recordIdempotency(clientId, eventId);
return true;
}
/**
@@ -81,7 +93,8 @@ export class SecurityEngine {
/**
* Validates incoming event payload:
* Checks max byte size and JSON structure.
* Checks duplicate idempotency (read-only), max byte size, and rate limit,
* and records the idempotency key ONLY after all checks pass.
*/
public static validateEvent(
clientId: string,
@@ -89,10 +102,8 @@ export class SecurityEngine {
payload: unknown,
clientMaxRate = 100,
): ValidationResult {
// 1. Check idempotency
const isUnique =
!this.enableIdempotency || this.checkIdempotency(clientId, eventId);
if (!isUnique) {
// 1. Check idempotency (read-only so rate-limited retries are not burned)
if (this.enableIdempotency && this.hasDuplicate(clientId, eventId)) {
return {
allowed: false,
reason: "Duplicate event dropped silently",
@@ -100,18 +111,7 @@ export class SecurityEngine {
};
}
// 2. Check rate limit
const effectiveRateLimit = Math.min(clientMaxRate, this.maxEventsPerSec);
const withinLimit = this.checkRateLimit(clientId, effectiveRateLimit);
if (!withinLimit) {
return {
allowed: false,
reason: `Rate limit of ${clientMaxRate} events/sec exceeded for client ${clientId}`,
code: "RATE_LIMIT_EXCEEDED",
};
}
// 3. Check payload size
// 2. Check payload size before consuming rate-limit quota
const payloadStr =
typeof payload === "string" ? payload : JSON.stringify(payload);
const byteLength = Buffer.byteLength(payloadStr, "utf8");
@@ -124,6 +124,22 @@ export class SecurityEngine {
};
}
// 3. Check rate limit
const effectiveRateLimit = Math.min(clientMaxRate, this.maxEventsPerSec);
const withinLimit = this.checkRateLimit(clientId, effectiveRateLimit);
if (!withinLimit) {
return {
allowed: false,
reason: `Rate limit of ${clientMaxRate} events/sec exceeded for client ${clientId}`,
code: "RATE_LIMIT_EXCEEDED",
};
}
// 4. Record idempotency key ONLY after event is accepted
if (this.enableIdempotency && eventId) {
this.recordIdempotency(clientId, eventId);
}
return { allowed: true };
}
+9 -3
View File
@@ -91,7 +91,13 @@ export class CSVStorageManager {
for (const [key, batch] of grouped.entries()) {
const [clientId, date] = key.split(':::');
const targetPath = this.getTargetFilePath(clientId, date);
const fileExists = fs.existsSync(targetPath);
let hasContent = false;
try {
hasContent =
fs.existsSync(targetPath) && fs.statSync(targetPath).size > 0;
} catch {
hasContent = false;
}
// Format payload as JSON string if not already
const cleanBatch = batch.map((item) => ({
@@ -101,10 +107,10 @@ export class CSVStorageManager {
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
}));
const parser = new Parser({ fields, header: !fileExists, eol: '\n' });
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
const csvData = parser.parse(cleanBatch);
if (fileExists) {
if (hasContent) {
fs.appendFileSync(targetPath, '\n' + csvData, 'utf8');
} else {
fs.writeFileSync(targetPath, csvData, 'utf8');