feat: improve event tracking and SDK robustness

- Add specific error tracking for rate-limited and oversized events in API response
- Ensure runtime settings are applied on initial access in settings manager
- Exclude /api/health from middleware protection
- Update Python SDK to v2.2.0 with unified standalone implementation
- Add auto_prune_days setting to database schema retrieval
This commit is contained in:
Team Dark Devil committed 2026-11-18 02:51:04 +06:00
1 parent 0c2e49cd47
commit 043123dc21
12 files changed
+1018 -128

No files matched your search

+7 -1
View File
@@ -203,7 +203,8 @@ async function syncSettingsFromDatabase() {
await ensureDatabaseSchema();
const res = await dbPool.query(
`SELECT max_events_per_sec, max_payload_size_kb, batch_flush_threshold,
batch_flush_interval_ms, file_rotation_mb, jwt_secret, enable_idempotency
batch_flush_interval_ms, file_rotation_mb, jwt_secret, enable_idempotency,
auto_prune_days
FROM admin_settings
WHERE id = 'default'`,
);
@@ -227,6 +228,11 @@ async function syncSettingsFromDatabase() {
writer.setMaxFileSizeMb(nextRotationMb);
}
const nextPruneDays = Number(row.auto_prune_days) || 30;
if (nextPruneDays > 0) {
writer.pruneExpiredPartitions(nextPruneDays);
}
const nextIntervalMs = Math.max(
100,
Number(row.batch_flush_interval_ms) || 2000,
+56 -6
View File
@@ -73,6 +73,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
return {
allowed: false,
duplicate: true,
code: "DUPLICATE_EVENT",
reason: "Duplicate event dropped",
};
}
@@ -91,6 +92,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
return {
allowed: false,
duplicate: false,
code: "RATE_LIMIT_EXCEEDED",
reason: `RATE_LIMIT_EXCEEDED: Limit of ${effectiveLimit} events/sec exceeded`,
};
} else {
@@ -106,6 +108,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
return {
allowed: false,
duplicate: false,
code: "PAYLOAD_TOO_LARGE",
reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`,
};
}
@@ -166,6 +169,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
if (callback) {
callback({
success: false,
code: "CLIENT_REVOKED",
error: dbCheck.reason || "CLIENT_REVOKED",
});
}
@@ -195,7 +199,12 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
return;
}
if (callback) {
callback({ success: false, eventId, error: check.reason });
callback({
success: false,
eventId,
code: check.code,
error: check.reason,
});
}
return;
}
@@ -236,6 +245,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
success: false,
processed: 0,
rejected: batch.length,
code: "CLIENT_REVOKED",
error: dbCheck.reason || "CLIENT_REVOKED",
});
}
@@ -247,6 +257,11 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
dbCheck.rateLimit || socket.data.rateLimit || 100;
let processed = 0;
let rejected = 0;
let duplicates = 0;
let hitRateLimit = false;
let hitPayloadLimit = false;
const rateLimitedEventIds = [];
const oversizedEventIds = [];
const errors = [];
for (const item of batch) {
@@ -265,11 +280,21 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
effectiveClientRate,
);
if (!check.allowed) {
if (!check.duplicate) {
rejected += 1;
if (errors.length < 5) {
errors.push(`${eventId}: ${check.reason}`);
}
if (check.duplicate) {
duplicates += 1;
continue;
}
rejected += 1;
if (check.code === "RATE_LIMIT_EXCEEDED") {
hitRateLimit = true;
rateLimitedEventIds.push(eventId);
}
if (check.code === "PAYLOAD_TOO_LARGE") {
hitPayloadLimit = true;
oversizedEventIds.push(eventId);
}
if (errors.length < 5) {
errors.push(`${eventId}: ${check.reason}`);
}
continue;
}
@@ -288,11 +313,36 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
onBufferPush();
}
if (processed === 0 && duplicates === 0 && rejected > 0) {
const code = hitPayloadLimit
? "PAYLOAD_TOO_LARGE"
: hitRateLimit
? "RATE_LIMIT_EXCEEDED"
: "BATCH_REJECTED";
if (callback) {
callback({
success: false,
processed,
rejected,
duplicates,
code,
error: errors[0] || "Batch validation failed",
errors,
...(rateLimitedEventIds.length > 0 ? { rateLimitedEventIds } : {}),
...(oversizedEventIds.length > 0 ? { oversizedEventIds } : {}),
});
}
return;
}
if (callback) {
callback({
success: true,
processed,
rejected,
duplicates,
...(rateLimitedEventIds.length > 0 ? { rateLimitedEventIds } : {}),
...(oversizedEventIds.length > 0 ? { oversizedEventIds } : {}),
...(errors.length > 0 ? { errors } : {}),
});
}
+37
View File
@@ -97,6 +97,43 @@ class CsvWriterWorker {
return { written: events.length, files };
}
pruneExpiredPartitions(autoPruneDays) {
const days = Number(autoPruneDays);
if (!Number.isFinite(days) || days <= 0 || !fs.existsSync(this.baseDir)) {
return 0;
}
const cutoffDate = new Date(Date.now() - Math.floor(days) * 86400000)
.toISOString()
.split('T')[0];
let prunedCount = 0;
try {
const clientDirs = fs.readdirSync(this.baseDir);
for (const clientDir of clientDirs) {
const fullClientPath = path.join(this.baseDir, clientDir);
if (!fs.statSync(fullClientPath).isDirectory()) continue;
const files = fs.readdirSync(fullClientPath);
for (const file of files) {
if (!file.endsWith('.csv')) continue;
const datePart = file.replace('.csv', '').split('-part')[0];
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
try {
fs.unlinkSync(path.join(fullClientPath, file));
prunedCount += 1;
} catch {
// Ignore individual file unlink failure
}
}
}
}
} catch {
// Ignore directory scan failure
}
return prunedCount;
}
}
module.exports = { CsvWriterWorker };