feat: improve event tracking and SDK robustness
- Add specific error tracking for rate-limited and oversized events in API response - Ensure runtime settings are applied on initial access in settings manager - Exclude /api/health from middleware protection - Update Python SDK to v2.2.0 with unified standalone implementation - Add auto_prune_days setting to database schema retrieval
This commit is contained in:
1 parent
0c2e49cd47
commit
043123dc21
12 files changed
+1018
-128
No files matched your search
+7
-1
@@ -203,7 +203,8 @@ async function syncSettingsFromDatabase() {
|
||||
await ensureDatabaseSchema();
|
||||
const res = await dbPool.query(
|
||||
`SELECT max_events_per_sec, max_payload_size_kb, batch_flush_threshold,
|
||||
batch_flush_interval_ms, file_rotation_mb, jwt_secret, enable_idempotency
|
||||
batch_flush_interval_ms, file_rotation_mb, jwt_secret, enable_idempotency,
|
||||
auto_prune_days
|
||||
FROM admin_settings
|
||||
WHERE id = 'default'`,
|
||||
);
|
||||
@@ -227,6 +228,11 @@ async function syncSettingsFromDatabase() {
|
||||
writer.setMaxFileSizeMb(nextRotationMb);
|
||||
}
|
||||
|
||||
const nextPruneDays = Number(row.auto_prune_days) || 30;
|
||||
if (nextPruneDays > 0) {
|
||||
writer.pruneExpiredPartitions(nextPruneDays);
|
||||
}
|
||||
|
||||
const nextIntervalMs = Math.max(
|
||||
100,
|
||||
Number(row.batch_flush_interval_ms) || 2000,
|
||||
|
||||
@@ -73,6 +73,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
return {
|
||||
allowed: false,
|
||||
duplicate: true,
|
||||
code: "DUPLICATE_EVENT",
|
||||
reason: "Duplicate event dropped",
|
||||
};
|
||||
}
|
||||
@@ -91,6 +92,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
return {
|
||||
allowed: false,
|
||||
duplicate: false,
|
||||
code: "RATE_LIMIT_EXCEEDED",
|
||||
reason: `RATE_LIMIT_EXCEEDED: Limit of ${effectiveLimit} events/sec exceeded`,
|
||||
};
|
||||
} else {
|
||||
@@ -106,6 +108,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
return {
|
||||
allowed: false,
|
||||
duplicate: false,
|
||||
code: "PAYLOAD_TOO_LARGE",
|
||||
reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`,
|
||||
};
|
||||
}
|
||||
@@ -166,6 +169,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
if (callback) {
|
||||
callback({
|
||||
success: false,
|
||||
code: "CLIENT_REVOKED",
|
||||
error: dbCheck.reason || "CLIENT_REVOKED",
|
||||
});
|
||||
}
|
||||
@@ -195,7 +199,12 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
return;
|
||||
}
|
||||
if (callback) {
|
||||
callback({ success: false, eventId, error: check.reason });
|
||||
callback({
|
||||
success: false,
|
||||
eventId,
|
||||
code: check.code,
|
||||
error: check.reason,
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -236,6 +245,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
success: false,
|
||||
processed: 0,
|
||||
rejected: batch.length,
|
||||
code: "CLIENT_REVOKED",
|
||||
error: dbCheck.reason || "CLIENT_REVOKED",
|
||||
});
|
||||
}
|
||||
@@ -247,6 +257,11 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
dbCheck.rateLimit || socket.data.rateLimit || 100;
|
||||
let processed = 0;
|
||||
let rejected = 0;
|
||||
let duplicates = 0;
|
||||
let hitRateLimit = false;
|
||||
let hitPayloadLimit = false;
|
||||
const rateLimitedEventIds = [];
|
||||
const oversizedEventIds = [];
|
||||
const errors = [];
|
||||
|
||||
for (const item of batch) {
|
||||
@@ -265,11 +280,21 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
effectiveClientRate,
|
||||
);
|
||||
if (!check.allowed) {
|
||||
if (!check.duplicate) {
|
||||
rejected += 1;
|
||||
if (errors.length < 5) {
|
||||
errors.push(`${eventId}: ${check.reason}`);
|
||||
}
|
||||
if (check.duplicate) {
|
||||
duplicates += 1;
|
||||
continue;
|
||||
}
|
||||
rejected += 1;
|
||||
if (check.code === "RATE_LIMIT_EXCEEDED") {
|
||||
hitRateLimit = true;
|
||||
rateLimitedEventIds.push(eventId);
|
||||
}
|
||||
if (check.code === "PAYLOAD_TOO_LARGE") {
|
||||
hitPayloadLimit = true;
|
||||
oversizedEventIds.push(eventId);
|
||||
}
|
||||
if (errors.length < 5) {
|
||||
errors.push(`${eventId}: ${check.reason}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
@@ -288,11 +313,36 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
onBufferPush();
|
||||
}
|
||||
|
||||
if (processed === 0 && duplicates === 0 && rejected > 0) {
|
||||
const code = hitPayloadLimit
|
||||
? "PAYLOAD_TOO_LARGE"
|
||||
: hitRateLimit
|
||||
? "RATE_LIMIT_EXCEEDED"
|
||||
: "BATCH_REJECTED";
|
||||
if (callback) {
|
||||
callback({
|
||||
success: false,
|
||||
processed,
|
||||
rejected,
|
||||
duplicates,
|
||||
code,
|
||||
error: errors[0] || "Batch validation failed",
|
||||
errors,
|
||||
...(rateLimitedEventIds.length > 0 ? { rateLimitedEventIds } : {}),
|
||||
...(oversizedEventIds.length > 0 ? { oversizedEventIds } : {}),
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (callback) {
|
||||
callback({
|
||||
success: true,
|
||||
processed,
|
||||
rejected,
|
||||
duplicates,
|
||||
...(rateLimitedEventIds.length > 0 ? { rateLimitedEventIds } : {}),
|
||||
...(oversizedEventIds.length > 0 ? { oversizedEventIds } : {}),
|
||||
...(errors.length > 0 ? { errors } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -97,6 +97,43 @@ class CsvWriterWorker {
|
||||
|
||||
return { written: events.length, files };
|
||||
}
|
||||
|
||||
pruneExpiredPartitions(autoPruneDays) {
|
||||
const days = Number(autoPruneDays);
|
||||
if (!Number.isFinite(days) || days <= 0 || !fs.existsSync(this.baseDir)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const cutoffDate = new Date(Date.now() - Math.floor(days) * 86400000)
|
||||
.toISOString()
|
||||
.split('T')[0];
|
||||
|
||||
let prunedCount = 0;
|
||||
try {
|
||||
const clientDirs = fs.readdirSync(this.baseDir);
|
||||
for (const clientDir of clientDirs) {
|
||||
const fullClientPath = path.join(this.baseDir, clientDir);
|
||||
if (!fs.statSync(fullClientPath).isDirectory()) continue;
|
||||
|
||||
const files = fs.readdirSync(fullClientPath);
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
|
||||
try {
|
||||
fs.unlinkSync(path.join(fullClientPath, file));
|
||||
prunedCount += 1;
|
||||
} catch {
|
||||
// Ignore individual file unlink failure
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore directory scan failure
|
||||
}
|
||||
return prunedCount;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { CsvWriterWorker };
|
||||
Reference in new issue
Block a user