feat(client): implement per-client storage and security controls
- Added storage quota, IP allowlisting, and expiration logic to client management APIs. - Updated CSV storage manager to support auto-compression and improved partition file parsing. - Integrated security checks into event ingestion and data retrieval routes.
This commit is contained in:
1 parent
89e4d68690
commit
2873b9904a
32 files changed
+6075
-241
No files matched your search
+284
-8
@@ -199,6 +199,9 @@ const runtimeConfig = {
|
||||
fileRotationMb: 100,
|
||||
enableIdempotency: true,
|
||||
autoPruneDays: 30,
|
||||
compressAfterDays: 0,
|
||||
enableS3Backup: false,
|
||||
diskPauseThresholdPct: 95,
|
||||
};
|
||||
|
||||
let previousJwtSecret = '';
|
||||
@@ -287,17 +290,113 @@ const inMemoryClientActivity = new Map();
|
||||
let hasSyncedInMemoryClients = false;
|
||||
const CLIENT_CACHE_TTL_MS = 2000;
|
||||
|
||||
async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
|
||||
function stripIpv6MappedPrefix(rawIp) {
|
||||
let ip = String(rawIp || '').trim();
|
||||
if (!ip) return '';
|
||||
if (ip.startsWith('[') && ip.includes(']')) {
|
||||
ip = ip.slice(1, ip.indexOf(']'));
|
||||
} else if (/^\d{1,3}(?:\.\d{1,3}){3}:\d+$/.test(ip)) {
|
||||
ip = ip.slice(0, ip.lastIndexOf(':'));
|
||||
}
|
||||
const pctIndex = ip.indexOf('%');
|
||||
if (pctIndex !== -1) {
|
||||
ip = ip.slice(0, pctIndex);
|
||||
}
|
||||
if (/^::ffff:\d{1,3}(?:\.\d{1,3}){3}$/i.test(ip)) {
|
||||
return ip.slice(7);
|
||||
}
|
||||
return ip.toLowerCase();
|
||||
}
|
||||
|
||||
function ipv4ToUint32(ip) {
|
||||
const parts = ip.split('.');
|
||||
if (parts.length !== 4) return null;
|
||||
let acc = 0;
|
||||
for (const part of parts) {
|
||||
if (!/^\d{1,3}$/.test(part)) return null;
|
||||
if (part.length > 1 && part.startsWith('0')) return null;
|
||||
const num = Number(part);
|
||||
if (num < 0 || num > 255) return null;
|
||||
acc = ((acc << 8) | num) >>> 0;
|
||||
}
|
||||
return acc >>> 0;
|
||||
}
|
||||
|
||||
function matchesIpv4Cidr(clientIp, cidrPattern) {
|
||||
const [networkStr, prefixStr] = cidrPattern.split('/');
|
||||
if (!networkStr || prefixStr === undefined) return false;
|
||||
const prefix = Number(prefixStr);
|
||||
if (!Number.isInteger(prefix) || prefix < 0 || prefix > 32) return false;
|
||||
const clientNum = ipv4ToUint32(clientIp);
|
||||
const networkNum = ipv4ToUint32(networkStr);
|
||||
if (clientNum === null || networkNum === null) return false;
|
||||
if (prefix === 0) return true;
|
||||
const mask = (0xffffffff << (32 - prefix)) >>> 0;
|
||||
return (clientNum & mask) >>> 0 === (networkNum & mask) >>> 0;
|
||||
}
|
||||
|
||||
function isClientIpAllowed(clientIp, rawAllowedIps) {
|
||||
if (!rawAllowedIps || !String(rawAllowedIps).trim()) return true;
|
||||
const entries = String(rawAllowedIps)
|
||||
.split(/[\s,]+/)
|
||||
.map((entry) => entry.trim())
|
||||
.filter(Boolean);
|
||||
if (entries.length === 0) return true;
|
||||
if (entries.includes('*') || entries.includes('0.0.0.0/0') || entries.includes('::/0')) {
|
||||
return true;
|
||||
}
|
||||
const normalizedClientIp = stripIpv6MappedPrefix(clientIp);
|
||||
if (!normalizedClientIp || normalizedClientIp === 'unknown') {
|
||||
return false;
|
||||
}
|
||||
for (const entry of entries) {
|
||||
const normalizedEntry = stripIpv6MappedPrefix(entry);
|
||||
if (normalizedEntry.includes('/')) {
|
||||
if (matchesIpv4Cidr(normalizedClientIp, normalizedEntry)) {
|
||||
return true;
|
||||
}
|
||||
const [netAddr, prefixStr] = normalizedEntry.split('/');
|
||||
if (Number(prefixStr) === 128 && normalizedClientIp === stripIpv6MappedPrefix(netAddr)) {
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
if (normalizedClientIp === normalizedEntry) {
|
||||
return true;
|
||||
}
|
||||
if (
|
||||
(normalizedClientIp === '127.0.0.1' && normalizedEntry === '::1') ||
|
||||
(normalizedClientIp === '::1' && normalizedEntry === '127.0.0.1')
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function isClientTokenExpired(expiresAt, nowMs = Date.now()) {
|
||||
if (!expiresAt) return false;
|
||||
const ts =
|
||||
expiresAt instanceof Date
|
||||
? expiresAt.getTime()
|
||||
: new Date(String(expiresAt)).getTime();
|
||||
if (Number.isNaN(ts)) return false;
|
||||
return ts <= nowMs;
|
||||
}
|
||||
|
||||
async function verifyClientInDb(clientId, presentedToken, bypassCache = false, clientIp = '') {
|
||||
const normalizedId = normalizeClientId(clientId);
|
||||
if (!normalizedId) {
|
||||
return {
|
||||
allowed: false,
|
||||
code: 'CLIENT_REVOKED',
|
||||
reason: 'CLIENT_REVOKED: Missing client identifier',
|
||||
};
|
||||
}
|
||||
if (deletedClientsSet.has(normalizedId)) {
|
||||
return {
|
||||
allowed: false,
|
||||
code: 'CLIENT_REVOKED',
|
||||
reason: 'CLIENT_REVOKED: Client access has been deleted or revoked by admin',
|
||||
};
|
||||
}
|
||||
@@ -313,6 +412,7 @@ async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
|
||||
deletedClientsSet.add(normalizedId);
|
||||
return {
|
||||
allowed: false,
|
||||
code: 'CLIENT_REVOKED',
|
||||
reason: 'CLIENT_REVOKED: Client access has been deleted by admin',
|
||||
};
|
||||
}
|
||||
@@ -326,6 +426,9 @@ async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
|
||||
status: parsed.status === 'revoked' ? 'revoked' : 'active',
|
||||
token: String(parsed.token || ''),
|
||||
rateLimit: Math.max(1, Number(parsed.rateLimit) || 100),
|
||||
storageQuotaMb: Math.max(0, Math.floor(Number(parsed.storageQuotaMb) || 0)),
|
||||
allowedIps: typeof parsed.allowedIps === 'string' ? parsed.allowedIps : '',
|
||||
expiresAt: parsed.expiresAt ? String(parsed.expiresAt) : null,
|
||||
};
|
||||
inMemoryClientRegistry.set(normalizedId, memClient);
|
||||
hasSyncedInMemoryClients = true;
|
||||
@@ -341,20 +444,41 @@ async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
|
||||
if (memClient.status === 'revoked') {
|
||||
return {
|
||||
allowed: false,
|
||||
code: 'CLIENT_REVOKED',
|
||||
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
|
||||
};
|
||||
}
|
||||
if (memClient.token && presentedToken && memClient.token !== presentedToken) {
|
||||
return {
|
||||
allowed: false,
|
||||
code: 'TOKEN_ROTATED',
|
||||
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
|
||||
};
|
||||
}
|
||||
return { allowed: true, rateLimit: memClient.rateLimit || 100 };
|
||||
if (isClientTokenExpired(memClient.expiresAt)) {
|
||||
return {
|
||||
allowed: false,
|
||||
code: 'CLIENT_TOKEN_EXPIRED',
|
||||
reason: `CLIENT_TOKEN_EXPIRED: Client '${normalizedId}' token has expired`,
|
||||
};
|
||||
}
|
||||
if (memClient.allowedIps && memClient.allowedIps.trim() && !isClientIpAllowed(clientIp, memClient.allowedIps)) {
|
||||
return {
|
||||
allowed: false,
|
||||
code: 'CLIENT_IP_FORBIDDEN',
|
||||
reason: `CLIENT_IP_FORBIDDEN: Client IP '${clientIp || 'unknown'}' is not authorized`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
allowed: true,
|
||||
rateLimit: memClient.rateLimit || 100,
|
||||
storageQuotaMb: memClient.storageQuotaMb || 0,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
allowed: false,
|
||||
code: 'CLIENT_REVOKED',
|
||||
reason: 'CLIENT_REVOKED: Client registration not found or has been deleted',
|
||||
};
|
||||
}
|
||||
@@ -365,16 +489,36 @@ async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
|
||||
if (prevCached.status === 'revoked') {
|
||||
return {
|
||||
allowed: false,
|
||||
code: 'CLIENT_REVOKED',
|
||||
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
|
||||
};
|
||||
}
|
||||
if (prevCached.token && presentedToken && prevCached.token !== presentedToken) {
|
||||
return {
|
||||
allowed: false,
|
||||
code: 'TOKEN_ROTATED',
|
||||
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
|
||||
};
|
||||
}
|
||||
return { allowed: true, rateLimit: prevCached.rateLimit };
|
||||
if (isClientTokenExpired(prevCached.expiresAt, now)) {
|
||||
return {
|
||||
allowed: false,
|
||||
code: 'CLIENT_TOKEN_EXPIRED',
|
||||
reason: `CLIENT_TOKEN_EXPIRED: Client '${normalizedId}' token has expired`,
|
||||
};
|
||||
}
|
||||
if (prevCached.allowedIps && prevCached.allowedIps.trim() && !isClientIpAllowed(clientIp, prevCached.allowedIps)) {
|
||||
return {
|
||||
allowed: false,
|
||||
code: 'CLIENT_IP_FORBIDDEN',
|
||||
reason: `CLIENT_IP_FORBIDDEN: Client IP '${clientIp || 'unknown'}' is not authorized`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
allowed: true,
|
||||
rateLimit: prevCached.rateLimit,
|
||||
storageQuotaMb: prevCached.storageQuotaMb || 0,
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
@@ -387,26 +531,39 @@ async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
|
||||
deletedClientsSet.add(normalizedId);
|
||||
return {
|
||||
allowed: false,
|
||||
code: 'CLIENT_REVOKED',
|
||||
reason: 'CLIENT_REVOKED: Client access has been deleted by admin',
|
||||
};
|
||||
}
|
||||
|
||||
const res = await dbPool.query(
|
||||
`SELECT status, token, rate_limit FROM client_sdks WHERE client_id = $1`,
|
||||
`SELECT status, token, rate_limit, storage_quota_mb, allowed_ips, expires_at FROM client_sdks WHERE client_id = $1`,
|
||||
[normalizedId],
|
||||
);
|
||||
if (!res.rowCount || res.rowCount === 0) {
|
||||
clientAuthCache.delete(normalizedId);
|
||||
return {
|
||||
allowed: false,
|
||||
code: 'CLIENT_REVOKED',
|
||||
reason: 'CLIENT_REVOKED: Client registration not found or has been deleted',
|
||||
};
|
||||
}
|
||||
const row = res.rows[0];
|
||||
const parsedQuota = Number(row.storage_quota_mb);
|
||||
let expiresAtIso = null;
|
||||
if (row.expires_at) {
|
||||
const d = new Date(row.expires_at);
|
||||
if (!Number.isNaN(d.getTime())) {
|
||||
expiresAtIso = d.toISOString();
|
||||
}
|
||||
}
|
||||
const entry = {
|
||||
status: row.status,
|
||||
token: row.token,
|
||||
rateLimit: Number(row.rate_limit) || 100,
|
||||
storageQuotaMb: Number.isFinite(parsedQuota) && parsedQuota >= 0 ? Math.floor(parsedQuota) : 0,
|
||||
allowedIps: typeof row.allowed_ips === 'string' ? row.allowed_ips : '',
|
||||
expiresAt: expiresAtIso,
|
||||
checkedAt: now,
|
||||
};
|
||||
clientAuthCache.set(normalizedId, entry);
|
||||
@@ -414,19 +571,40 @@ async function verifyClientInDb(clientId, presentedToken, bypassCache = false) {
|
||||
if (entry.status === 'revoked') {
|
||||
return {
|
||||
allowed: false,
|
||||
code: 'CLIENT_REVOKED',
|
||||
reason: 'CLIENT_REVOKED: Client access has been revoked by admin',
|
||||
};
|
||||
}
|
||||
if (entry.token && presentedToken && entry.token !== presentedToken) {
|
||||
return {
|
||||
allowed: false,
|
||||
code: 'TOKEN_ROTATED',
|
||||
reason: 'TOKEN_ROTATED: Client credentials have been rotated',
|
||||
};
|
||||
}
|
||||
return { allowed: true, rateLimit: entry.rateLimit };
|
||||
if (isClientTokenExpired(entry.expiresAt, now)) {
|
||||
return {
|
||||
allowed: false,
|
||||
code: 'CLIENT_TOKEN_EXPIRED',
|
||||
reason: `CLIENT_TOKEN_EXPIRED: Client '${normalizedId}' token has expired`,
|
||||
};
|
||||
}
|
||||
if (entry.allowedIps && entry.allowedIps.trim() && !isClientIpAllowed(clientIp, entry.allowedIps)) {
|
||||
return {
|
||||
allowed: false,
|
||||
code: 'CLIENT_IP_FORBIDDEN',
|
||||
reason: `CLIENT_IP_FORBIDDEN: Client IP '${clientIp || 'unknown'}' is not authorized`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
allowed: true,
|
||||
rateLimit: entry.rateLimit,
|
||||
storageQuotaMb: entry.storageQuotaMb,
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
allowed: false,
|
||||
code: 'AUTH_UNAVAILABLE',
|
||||
reason: 'AUTH_UNAVAILABLE: Unable to verify client credentials against database',
|
||||
};
|
||||
}
|
||||
@@ -888,13 +1066,42 @@ function applyRuntimeSettingsObject(settingsObj) {
|
||||
}
|
||||
}
|
||||
|
||||
if (settingsObj.enableS3Backup !== undefined) {
|
||||
runtimeConfig.enableS3Backup = Boolean(settingsObj.enableS3Backup);
|
||||
}
|
||||
|
||||
if (
|
||||
settingsObj.diskPauseThresholdPct !== undefined &&
|
||||
settingsObj.diskPauseThresholdPct !== null
|
||||
) {
|
||||
const pct = Number(settingsObj.diskPauseThresholdPct);
|
||||
if (Number.isFinite(pct) && (pct === 0 || (pct >= 50 && pct <= 99))) {
|
||||
runtimeConfig.diskPauseThresholdPct = Math.floor(pct);
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
settingsObj.compressAfterDays !== undefined &&
|
||||
settingsObj.compressAfterDays !== null
|
||||
) {
|
||||
const parsedCompress = Number(settingsObj.compressAfterDays);
|
||||
if (Number.isFinite(parsedCompress) && parsedCompress >= 0) {
|
||||
const nextCompressDays = Math.floor(parsedCompress);
|
||||
const changed = nextCompressDays !== runtimeConfig.compressAfterDays;
|
||||
runtimeConfig.compressAfterDays = nextCompressDays;
|
||||
if (changed && nextCompressDays > 0) {
|
||||
void writer.compressOldPartitionsAsync(nextCompressDays).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (settingsObj.autoPruneDays !== undefined && settingsObj.autoPruneDays !== null) {
|
||||
const parsedPrune = Number(settingsObj.autoPruneDays);
|
||||
if (Number.isFinite(parsedPrune) && parsedPrune >= 0) {
|
||||
const nextPruneDays = Math.floor(parsedPrune);
|
||||
const changed = nextPruneDays !== runtimeConfig.autoPruneDays;
|
||||
runtimeConfig.autoPruneDays = nextPruneDays;
|
||||
if (changed && nextPruneDays > 0) {
|
||||
if (changed && nextPruneDays > 0 && !runtimeConfig.enableS3Backup) {
|
||||
void writer.pruneExpiredPartitionsAsync(nextPruneDays).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
@@ -948,6 +1155,9 @@ async function ensureDatabaseSchema() {
|
||||
secret TEXT NOT NULL,
|
||||
token TEXT NOT NULL,
|
||||
rate_limit INTEGER NOT NULL DEFAULT 100,
|
||||
storage_quota_mb INTEGER NOT NULL DEFAULT 0,
|
||||
allowed_ips TEXT NOT NULL DEFAULT '',
|
||||
expires_at TIMESTAMPTZ,
|
||||
status TEXT NOT NULL DEFAULT 'active',
|
||||
total_events BIGINT NOT NULL DEFAULT 0,
|
||||
last_active TIMESTAMPTZ,
|
||||
@@ -968,6 +1178,21 @@ async function ensureDatabaseSchema() {
|
||||
jwt_secret TEXT NOT NULL,
|
||||
enable_idempotency BOOLEAN NOT NULL DEFAULT true,
|
||||
auto_prune_days INTEGER NOT NULL DEFAULT 30,
|
||||
compress_after_days INTEGER NOT NULL DEFAULT 0,
|
||||
enable_s3_backup BOOLEAN NOT NULL DEFAULT false,
|
||||
s3_endpoint TEXT NOT NULL DEFAULT '',
|
||||
s3_region TEXT NOT NULL DEFAULT 'auto',
|
||||
s3_bucket TEXT NOT NULL DEFAULT '',
|
||||
s3_prefix TEXT NOT NULL DEFAULT 'datastack/',
|
||||
s3_access_key_id TEXT NOT NULL DEFAULT '',
|
||||
s3_secret_access_key_enc TEXT NOT NULL DEFAULT '',
|
||||
disk_pause_threshold_pct INTEGER NOT NULL DEFAULT 95,
|
||||
enable_webhook_alerts BOOLEAN NOT NULL DEFAULT false,
|
||||
webhook_url TEXT NOT NULL DEFAULT '',
|
||||
webhook_secret_enc TEXT NOT NULL DEFAULT '',
|
||||
webhook_cooldown_sec INTEGER NOT NULL DEFAULT 300,
|
||||
enable_prometheus_metrics BOOLEAN NOT NULL DEFAULT false,
|
||||
prometheus_metrics_token_enc TEXT NOT NULL DEFAULT '',
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`);
|
||||
@@ -999,6 +1224,9 @@ async function ensureDatabaseSchema() {
|
||||
await pool.query(`
|
||||
ALTER TABLE client_sdks
|
||||
ADD COLUMN IF NOT EXISTS rate_limit INTEGER NOT NULL DEFAULT 100,
|
||||
ADD COLUMN IF NOT EXISTS storage_quota_mb INTEGER NOT NULL DEFAULT 0,
|
||||
ADD COLUMN IF NOT EXISTS allowed_ips TEXT NOT NULL DEFAULT '',
|
||||
ADD COLUMN IF NOT EXISTS expires_at TIMESTAMPTZ,
|
||||
ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'active',
|
||||
ADD COLUMN IF NOT EXISTS total_events BIGINT NOT NULL DEFAULT 0,
|
||||
ADD COLUMN IF NOT EXISTS last_active TIMESTAMPTZ,
|
||||
@@ -1016,6 +1244,21 @@ async function ensureDatabaseSchema() {
|
||||
ADD COLUMN IF NOT EXISTS file_rotation_mb INTEGER NOT NULL DEFAULT 100,
|
||||
ADD COLUMN IF NOT EXISTS enable_idempotency BOOLEAN NOT NULL DEFAULT true,
|
||||
ADD COLUMN IF NOT EXISTS auto_prune_days INTEGER NOT NULL DEFAULT 30,
|
||||
ADD COLUMN IF NOT EXISTS compress_after_days INTEGER NOT NULL DEFAULT 0,
|
||||
ADD COLUMN IF NOT EXISTS enable_s3_backup BOOLEAN NOT NULL DEFAULT false,
|
||||
ADD COLUMN IF NOT EXISTS s3_endpoint TEXT NOT NULL DEFAULT '',
|
||||
ADD COLUMN IF NOT EXISTS s3_region TEXT NOT NULL DEFAULT 'auto',
|
||||
ADD COLUMN IF NOT EXISTS s3_bucket TEXT NOT NULL DEFAULT '',
|
||||
ADD COLUMN IF NOT EXISTS s3_prefix TEXT NOT NULL DEFAULT 'datastack/',
|
||||
ADD COLUMN IF NOT EXISTS s3_access_key_id TEXT NOT NULL DEFAULT '',
|
||||
ADD COLUMN IF NOT EXISTS s3_secret_access_key_enc TEXT NOT NULL DEFAULT '',
|
||||
ADD COLUMN IF NOT EXISTS disk_pause_threshold_pct INTEGER NOT NULL DEFAULT 95,
|
||||
ADD COLUMN IF NOT EXISTS enable_webhook_alerts BOOLEAN NOT NULL DEFAULT false,
|
||||
ADD COLUMN IF NOT EXISTS webhook_url TEXT NOT NULL DEFAULT '',
|
||||
ADD COLUMN IF NOT EXISTS webhook_secret_enc TEXT NOT NULL DEFAULT '',
|
||||
ADD COLUMN IF NOT EXISTS webhook_cooldown_sec INTEGER NOT NULL DEFAULT 300,
|
||||
ADD COLUMN IF NOT EXISTS enable_prometheus_metrics BOOLEAN NOT NULL DEFAULT false,
|
||||
ADD COLUMN IF NOT EXISTS prometheus_metrics_token_enc TEXT NOT NULL DEFAULT '',
|
||||
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
`);
|
||||
await pool.query(`
|
||||
@@ -1043,7 +1286,7 @@ async function syncSettingsFromDatabase() {
|
||||
const res = await dbPool.query(
|
||||
`SELECT max_events_per_sec, max_payload_size_kb, batch_flush_threshold,
|
||||
batch_flush_interval_ms, file_rotation_mb, jwt_secret, enable_idempotency,
|
||||
auto_prune_days
|
||||
auto_prune_days, compress_after_days, enable_s3_backup, disk_pause_threshold_pct
|
||||
FROM admin_settings
|
||||
WHERE id = 'default'`,
|
||||
);
|
||||
@@ -1053,6 +1296,14 @@ async function syncSettingsFromDatabase() {
|
||||
row.auto_prune_days !== undefined && row.auto_prune_days !== null
|
||||
? Number(row.auto_prune_days)
|
||||
: NaN;
|
||||
const parsedCompress =
|
||||
row.compress_after_days !== undefined && row.compress_after_days !== null
|
||||
? Number(row.compress_after_days)
|
||||
: NaN;
|
||||
const parsedDiskPct =
|
||||
row.disk_pause_threshold_pct !== undefined && row.disk_pause_threshold_pct !== null
|
||||
? Number(row.disk_pause_threshold_pct)
|
||||
: NaN;
|
||||
|
||||
applyRuntimeSettingsObject({
|
||||
maxEventsPerSec: Number(row.max_events_per_sec) || 100,
|
||||
@@ -1065,6 +1316,9 @@ async function syncSettingsFromDatabase() {
|
||||
fileRotationMb: Number(row.file_rotation_mb) || 100,
|
||||
enableIdempotency: Boolean(row.enable_idempotency),
|
||||
autoPruneDays: Number.isFinite(parsedPrune) ? parsedPrune : 30,
|
||||
compressAfterDays: Number.isFinite(parsedCompress) ? Math.max(0, parsedCompress) : 0,
|
||||
enableS3Backup: Boolean(row.enable_s3_backup),
|
||||
diskPauseThresholdPct: Number.isFinite(parsedDiskPct) ? parsedDiskPct : 95,
|
||||
jwtSecret: row.jwt_secret ? String(row.jwt_secret).trim() : undefined,
|
||||
});
|
||||
return true;
|
||||
@@ -1338,6 +1592,9 @@ app.post('/internal/client-sync', (req, res) => {
|
||||
status: nextStatus,
|
||||
token: nextToken,
|
||||
rateLimit: Math.max(1, Number(item.rateLimit) || 100),
|
||||
storageQuotaMb: Math.max(0, Math.floor(Number(item.storageQuotaMb) || 0)),
|
||||
allowedIps: typeof item.allowedIps === 'string' ? item.allowedIps : '',
|
||||
expiresAt: item.expiresAt ? String(item.expiresAt) : null,
|
||||
});
|
||||
if (nextStatus === 'revoked' || (prev && nextToken && prev.token !== nextToken)) {
|
||||
io.to(`client:${cid}`).disconnectSockets(true);
|
||||
@@ -1380,6 +1637,9 @@ app.post('/internal/client-sync', (req, res) => {
|
||||
status: 'revoked',
|
||||
token: String(clientPayload.token || ''),
|
||||
rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
|
||||
storageQuotaMb: Math.max(0, Math.floor(Number(clientPayload.storageQuotaMb) || 0)),
|
||||
allowedIps: typeof clientPayload.allowedIps === 'string' ? clientPayload.allowedIps : '',
|
||||
expiresAt: clientPayload.expiresAt ? String(clientPayload.expiresAt) : null,
|
||||
});
|
||||
} else if (existingMem) {
|
||||
existingMem.status = 'revoked';
|
||||
@@ -1390,6 +1650,9 @@ app.post('/internal/client-sync', (req, res) => {
|
||||
status: 'revoked',
|
||||
token: '',
|
||||
rateLimit: 100,
|
||||
storageQuotaMb: 0,
|
||||
allowedIps: '',
|
||||
expiresAt: null,
|
||||
});
|
||||
}
|
||||
io.to(`client:${normalizedId}`).disconnectSockets(true);
|
||||
@@ -1402,6 +1665,9 @@ app.post('/internal/client-sync', (req, res) => {
|
||||
status: clientPayload.status === 'revoked' ? 'revoked' : 'active',
|
||||
token: String(clientPayload.token || ''),
|
||||
rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
|
||||
storageQuotaMb: Math.max(0, Math.floor(Number(clientPayload.storageQuotaMb) || 0)),
|
||||
allowedIps: typeof clientPayload.allowedIps === 'string' ? clientPayload.allowedIps : '',
|
||||
expiresAt: clientPayload.expiresAt ? String(clientPayload.expiresAt) : null,
|
||||
});
|
||||
}
|
||||
} else if (action === 'updated') {
|
||||
@@ -1412,6 +1678,9 @@ app.post('/internal/client-sync', (req, res) => {
|
||||
status: clientPayload.status === 'revoked' ? 'revoked' : 'active',
|
||||
token: String(clientPayload.token || ''),
|
||||
rateLimit: Math.max(1, Number(clientPayload.rateLimit) || 100),
|
||||
storageQuotaMb: Math.max(0, Math.floor(Number(clientPayload.storageQuotaMb) || 0)),
|
||||
allowedIps: typeof clientPayload.allowedIps === 'string' ? clientPayload.allowedIps : '',
|
||||
expiresAt: clientPayload.expiresAt ? String(clientPayload.expiresAt) : null,
|
||||
});
|
||||
}
|
||||
io.to(`client:${normalizedId}`).disconnectSockets(true);
|
||||
@@ -1542,7 +1811,12 @@ async function startIngestionServer() {
|
||||
}
|
||||
|
||||
pruneTimer = setInterval(() => {
|
||||
if (runtimeConfig.autoPruneDays > 0) {
|
||||
if (runtimeConfig.compressAfterDays > 0) {
|
||||
void writer
|
||||
.compressOldPartitionsAsync(runtimeConfig.compressAfterDays)
|
||||
.catch(() => undefined);
|
||||
}
|
||||
if (runtimeConfig.autoPruneDays > 0 && !runtimeConfig.enableS3Backup) {
|
||||
void writer
|
||||
.pruneExpiredPartitionsAsync(runtimeConfig.autoPruneDays)
|
||||
.catch(() => undefined);
|
||||
@@ -1594,4 +1868,6 @@ module.exports = {
|
||||
parseAllowedCorsOrigins,
|
||||
isOriginAllowedByCors,
|
||||
corsOriginDelegate,
|
||||
isClientIpAllowed,
|
||||
isClientTokenExpired,
|
||||
};
|
||||
@@ -26,6 +26,34 @@ function normalizeClientId(clientId) {
|
||||
.replace(/[^a-z0-9_-]/g, "_");
|
||||
}
|
||||
|
||||
function extractSocketClientIp(socket) {
|
||||
const headers = (socket && socket.handshake && socket.handshake.headers) || {};
|
||||
const realIp =
|
||||
typeof headers["x-real-ip"] === "string" ? headers["x-real-ip"].trim() : "";
|
||||
if (realIp) {
|
||||
return realIp.slice(0, 128);
|
||||
}
|
||||
const forwardedFor =
|
||||
typeof headers["x-forwarded-for"] === "string"
|
||||
? headers["x-forwarded-for"]
|
||||
: "";
|
||||
if (forwardedFor) {
|
||||
const hops = forwardedFor
|
||||
.split(",")
|
||||
.map((part) => part.trim())
|
||||
.filter(Boolean);
|
||||
const rightmost = hops[hops.length - 1];
|
||||
if (rightmost) {
|
||||
return rightmost.slice(0, 128);
|
||||
}
|
||||
}
|
||||
const addr =
|
||||
(socket && socket.handshake && socket.handshake.address) ||
|
||||
(socket && socket.conn && socket.conn.remoteAddress) ||
|
||||
"";
|
||||
return String(addr).trim() || "127.0.0.1";
|
||||
}
|
||||
|
||||
function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
const getRuntimeConfig =
|
||||
typeof options.getRuntimeConfig === "function"
|
||||
@@ -340,17 +368,20 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
const decoded = jwt.verify(cleanToken, config.jwtSecret);
|
||||
|
||||
const normalizedClientId = normalizeClientId(decoded.clientId);
|
||||
const clientIp = extractSocketClientIp(socket);
|
||||
const dbCheck = await verifyClientInDb(
|
||||
normalizedClientId,
|
||||
cleanToken,
|
||||
true,
|
||||
clientIp,
|
||||
);
|
||||
if (!dbCheck.allowed) {
|
||||
return next(new Error(dbCheck.reason || "CLIENT_REVOKED"));
|
||||
return next(new Error(dbCheck.reason || dbCheck.code || "CLIENT_REVOKED"));
|
||||
}
|
||||
|
||||
socket.data.clientId = normalizedClientId;
|
||||
socket.data.token = cleanToken;
|
||||
socket.data.clientIp = clientIp;
|
||||
socket.data.rateLimit =
|
||||
dbCheck.rateLimit || decoded.rateLimit || config.maxEventsPerSec || 100;
|
||||
next();
|
||||
@@ -401,16 +432,18 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
// Single event
|
||||
socket.on("sync_data", async (payload, callback) => {
|
||||
try {
|
||||
const clientIp = socket.data.clientIp || extractSocketClientIp(socket);
|
||||
const dbCheck = await verifyClientInDb(
|
||||
clientId,
|
||||
socket.data.token,
|
||||
false,
|
||||
clientIp,
|
||||
);
|
||||
if (!dbCheck.allowed) {
|
||||
if (callback) {
|
||||
callback({
|
||||
success: false,
|
||||
code: "CLIENT_REVOKED",
|
||||
code: dbCheck.code || "CLIENT_REVOKED",
|
||||
error: dbCheck.reason || "CLIENT_REVOKED",
|
||||
});
|
||||
}
|
||||
@@ -418,6 +451,25 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (worker && typeof worker.checkStorageAdmission === "function") {
|
||||
const config = getRuntimeConfig();
|
||||
const admission = await worker.checkStorageAdmission(
|
||||
clientId,
|
||||
dbCheck.storageQuotaMb || 0,
|
||||
config.diskPauseThresholdPct ?? 95,
|
||||
);
|
||||
if (!admission.allowed) {
|
||||
if (callback) {
|
||||
callback({
|
||||
success: false,
|
||||
code: admission.code,
|
||||
error: admission.reason,
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const effectiveClientRate =
|
||||
dbCheck.rateLimit || socket.data.rateLimit || 100;
|
||||
const safePayload =
|
||||
@@ -499,10 +551,12 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
return;
|
||||
}
|
||||
|
||||
const clientIp = socket.data.clientIp || extractSocketClientIp(socket);
|
||||
const dbCheck = await verifyClientInDb(
|
||||
clientId,
|
||||
socket.data.token,
|
||||
false,
|
||||
clientIp,
|
||||
);
|
||||
if (!dbCheck.allowed) {
|
||||
if (callback) {
|
||||
@@ -510,7 +564,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
success: false,
|
||||
processed: 0,
|
||||
rejected: batch.length,
|
||||
code: "CLIENT_REVOKED",
|
||||
code: dbCheck.code || "CLIENT_REVOKED",
|
||||
error: dbCheck.reason || "CLIENT_REVOKED",
|
||||
});
|
||||
}
|
||||
@@ -518,6 +572,27 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (worker && typeof worker.checkStorageAdmission === "function") {
|
||||
const config = getRuntimeConfig();
|
||||
const admission = await worker.checkStorageAdmission(
|
||||
clientId,
|
||||
dbCheck.storageQuotaMb || 0,
|
||||
config.diskPauseThresholdPct ?? 95,
|
||||
);
|
||||
if (!admission.allowed) {
|
||||
if (callback) {
|
||||
callback({
|
||||
success: false,
|
||||
processed: 0,
|
||||
rejected: batch.length,
|
||||
code: admission.code,
|
||||
error: admission.reason,
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const effectiveClientRate =
|
||||
dbCheck.rateLimit || socket.data.rateLimit || 100;
|
||||
let processed = 0;
|
||||
@@ -646,16 +721,19 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
// Heartbeat & Reconnection Verification Event
|
||||
socket.on("ping_verify", async (_payload, callback) => {
|
||||
try {
|
||||
const clientIp = socket.data.clientIp || extractSocketClientIp(socket);
|
||||
const dbCheck = await verifyClientInDb(
|
||||
clientId,
|
||||
socket.data.token,
|
||||
true,
|
||||
clientIp,
|
||||
);
|
||||
if (!dbCheck.allowed) {
|
||||
if (callback) {
|
||||
callback({
|
||||
success: false,
|
||||
status: "REVOKED",
|
||||
code: dbCheck.code || "CLIENT_REVOKED",
|
||||
error: dbCheck.reason || "CLIENT_REVOKED",
|
||||
});
|
||||
}
|
||||
@@ -691,4 +769,4 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { setupSocketHandlers, normalizeClientId };
|
||||
module.exports = { setupSocketHandlers, normalizeClientId, extractSocketClientIp };
|
||||
+233
-5
@@ -1,12 +1,15 @@
|
||||
const crypto = require('node:crypto');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { pipeline } = require('node:stream/promises');
|
||||
const zlib = require('node:zlib');
|
||||
const { Parser } = require('json2csv');
|
||||
|
||||
const LOCK_RETRY_MIN_MS = 5;
|
||||
const LOCK_RETRY_MAX_MS = 18;
|
||||
const LOCK_TIMEOUT_MS = 10000;
|
||||
const STALE_LOCK_AGE_MS = 30000;
|
||||
const QUOTA_CACHE_TTL_MS = 30000;
|
||||
|
||||
function sleepMs(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
@@ -23,15 +26,42 @@ function resolveDefaultStorageDir() {
|
||||
/**
|
||||
* Background CSV Writer Worker
|
||||
* Flushes batches of real-time events to partitioned CSV files.
|
||||
* Path convention: /storage/data/{clientId}/{YYYY-MM-DD}.csv
|
||||
* Path convention: /storage/data/{clientId}/{YYYY-MM-DD}.csv(.gz)
|
||||
*/
|
||||
class CsvWriterWorker {
|
||||
constructor(baseDir = resolveDefaultStorageDir(), maxFileSize = 100 * 1024 * 1024) {
|
||||
this.baseDir = baseDir;
|
||||
this.maxFileSize = maxFileSize;
|
||||
this.diskUsageCache = null;
|
||||
this.clientBytesCache = new Map();
|
||||
this.statfsOverrideForTests = null;
|
||||
this.ensureBaseDir();
|
||||
}
|
||||
|
||||
isPartitionFilename(filename) {
|
||||
return typeof filename === 'string' && (filename.endsWith('.csv') || filename.endsWith('.csv.gz'));
|
||||
}
|
||||
|
||||
extractDateFromFilename(filename) {
|
||||
return String(filename || '')
|
||||
.replace(/\.csv(?:\.gz)?$/, '')
|
||||
.split('-part')[0];
|
||||
}
|
||||
|
||||
clearQuotaCaches(clientId) {
|
||||
this.diskUsageCache = null;
|
||||
if (clientId) {
|
||||
this.clientBytesCache.delete(this.sanitizeClientId(clientId));
|
||||
} else {
|
||||
this.clientBytesCache.clear();
|
||||
}
|
||||
}
|
||||
|
||||
setStatfsOverrideForTests(provider) {
|
||||
this.statfsOverrideForTests = typeof provider === 'function' ? provider : null;
|
||||
this.diskUsageCache = null;
|
||||
}
|
||||
|
||||
setMaxFileSizeMb(megabytes) {
|
||||
const mb = Number(megabytes);
|
||||
if (Number.isFinite(mb) && mb >= 1) {
|
||||
@@ -282,6 +312,7 @@ class CsvWriterWorker {
|
||||
fs.writeFileSync(resolvedPath, fullCsv, 'utf8');
|
||||
}
|
||||
|
||||
this.clearQuotaCaches(clientId);
|
||||
return resolvedPath;
|
||||
});
|
||||
|
||||
@@ -291,6 +322,101 @@ class CsvWriterWorker {
|
||||
return { written: events.length, files };
|
||||
}
|
||||
|
||||
async compressOldPartitionsAsync(compressAfterDays) {
|
||||
const days = Number(compressAfterDays);
|
||||
if (!Number.isFinite(days) || days <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const baseStat = await fs.promises.lstat(this.baseDir).catch(() => null);
|
||||
if (!baseStat || !baseStat.isDirectory()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const cutoffDate = new Date(Date.now() - Math.floor(days) * 86400000)
|
||||
.toISOString()
|
||||
.split('T')[0];
|
||||
|
||||
let compressedCount = 0;
|
||||
try {
|
||||
const clientDirs = await fs.promises.readdir(this.baseDir);
|
||||
for (const clientDir of clientDirs) {
|
||||
const fullClientPath = path.join(this.baseDir, clientDir);
|
||||
const dirLstat = await fs.promises.lstat(fullClientPath).catch(() => null);
|
||||
if (!dirLstat || dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
|
||||
|
||||
const files = await fs.promises.readdir(fullClientPath).catch(() => []);
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
const datePart = this.extractDateFromFilename(file);
|
||||
if (!(/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate)) continue;
|
||||
const fullPath = path.join(fullClientPath, file);
|
||||
const fileLstat = await fs.promises.lstat(fullPath).catch(() => null);
|
||||
if (!fileLstat || fileLstat.isSymbolicLink() || !fileLstat.isFile()) continue;
|
||||
|
||||
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
|
||||
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
|
||||
let acquired = false;
|
||||
const tryAcquireLock = async () => {
|
||||
const handle = await fs.promises.open(lockPath, 'wx');
|
||||
try {
|
||||
await handle.writeFile(lockValue, 'utf8');
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
try {
|
||||
acquired = await tryAcquireLock();
|
||||
} catch (err) {
|
||||
if (err && err.code === 'EEXIST') {
|
||||
await this.removeStaleLockAtomicallyAsync(lockPath);
|
||||
try {
|
||||
acquired = await tryAcquireLock();
|
||||
} catch {
|
||||
acquired = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!acquired) continue;
|
||||
|
||||
let targetGzPath = `${fullPath}.gz`;
|
||||
if (fs.existsSync(targetGzPath)) {
|
||||
const stem = fullPath.replace(/\.csv$/, '');
|
||||
let suffix = 2;
|
||||
while (fs.existsSync(`${stem}-gz${suffix}.csv.gz`)) {
|
||||
suffix += 1;
|
||||
}
|
||||
targetGzPath = `${stem}-gz${suffix}.csv.gz`;
|
||||
}
|
||||
const tmpGzPath = `${targetGzPath}.tmp.${crypto.randomUUID()}`;
|
||||
|
||||
try {
|
||||
await pipeline(
|
||||
fs.createReadStream(fullPath, { highWaterMark: 64 * 1024 }),
|
||||
zlib.createGzip({ level: 6 }),
|
||||
fs.createWriteStream(tmpGzPath),
|
||||
);
|
||||
await fs.promises.utimes(tmpGzPath, fileLstat.atime, fileLstat.mtime).catch(() => undefined);
|
||||
await fs.promises.rename(tmpGzPath, targetGzPath);
|
||||
await fs.promises.unlink(fullPath);
|
||||
this.clearQuotaCaches(clientDir);
|
||||
compressedCount += 1;
|
||||
} catch {
|
||||
await fs.promises.unlink(tmpGzPath).catch(() => undefined);
|
||||
} finally {
|
||||
await this.releaseLockIfOwnedAsync(lockPath, lockValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore directory scan failure
|
||||
}
|
||||
return compressedCount;
|
||||
}
|
||||
|
||||
pruneExpiredPartitions(autoPruneDays) {
|
||||
const days = Number(autoPruneDays);
|
||||
if (!Number.isFinite(days) || days <= 0 || !fs.existsSync(this.baseDir)) {
|
||||
@@ -316,8 +442,8 @@ class CsvWriterWorker {
|
||||
|
||||
const files = fs.readdirSync(fullClientPath);
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||
if (!this.isPartitionFilename(file)) continue;
|
||||
const datePart = this.extractDateFromFilename(file);
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
|
||||
const fullPath = path.join(fullClientPath, file);
|
||||
let fileLstat;
|
||||
@@ -361,6 +487,7 @@ class CsvWriterWorker {
|
||||
try {
|
||||
if (fs.existsSync(fullPath)) {
|
||||
fs.unlinkSync(fullPath);
|
||||
this.clearQuotaCaches(clientDir);
|
||||
prunedCount += 1;
|
||||
}
|
||||
} catch {
|
||||
@@ -402,8 +529,8 @@ class CsvWriterWorker {
|
||||
|
||||
const files = await fs.promises.readdir(fullClientPath).catch(() => []);
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||
if (!this.isPartitionFilename(file)) continue;
|
||||
const datePart = this.extractDateFromFilename(file);
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
|
||||
const fullPath = path.join(fullClientPath, file);
|
||||
const fileLstat = await fs.promises.lstat(fullPath).catch(() => null);
|
||||
@@ -439,6 +566,7 @@ class CsvWriterWorker {
|
||||
|
||||
try {
|
||||
await fs.promises.unlink(fullPath);
|
||||
this.clearQuotaCaches(clientDir);
|
||||
prunedCount += 1;
|
||||
} catch {
|
||||
// Ignore individual file unlink failure
|
||||
@@ -453,6 +581,106 @@ class CsvWriterWorker {
|
||||
}
|
||||
return prunedCount;
|
||||
}
|
||||
|
||||
async getDiskUsageAsync() {
|
||||
const now = Date.now();
|
||||
if (this.diskUsageCache && now - this.diskUsageCache.checkedAt < QUOTA_CACHE_TTL_MS) {
|
||||
return this.diskUsageCache;
|
||||
}
|
||||
|
||||
let probeDir = path.resolve(this.baseDir);
|
||||
while (!fs.existsSync(probeDir)) {
|
||||
const parent = path.dirname(probeDir);
|
||||
if (parent === probeDir) break;
|
||||
probeDir = parent;
|
||||
}
|
||||
|
||||
try {
|
||||
const statfsRes = this.statfsOverrideForTests
|
||||
? await this.statfsOverrideForTests(probeDir)
|
||||
: await fs.promises.statfs(probeDir);
|
||||
const bsize = Math.max(1, Number(statfsRes.bsize) || 4096);
|
||||
const totalBytes = Math.max(0, Number(statfsRes.blocks) * bsize);
|
||||
const availableBytes = Math.max(0, Number(statfsRes.bavail) * bsize);
|
||||
const usedBytes = Math.max(0, totalBytes - availableBytes);
|
||||
const usedPct =
|
||||
totalBytes > 0
|
||||
? Number(Math.min(100, Math.max(0, (usedBytes / totalBytes) * 100)).toFixed(2))
|
||||
: 0;
|
||||
const snapshot = { usedPct, availableBytes, totalBytes, checkedAt: now };
|
||||
this.diskUsageCache = snapshot;
|
||||
return snapshot;
|
||||
} catch {
|
||||
const fallback = { usedPct: 0, availableBytes: 0, totalBytes: 0, checkedAt: now };
|
||||
this.diskUsageCache = fallback;
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
async getClientStorageBytesAsync(clientId) {
|
||||
const safeClientId = this.sanitizeClientId(clientId);
|
||||
const now = Date.now();
|
||||
const cached = this.clientBytesCache.get(safeClientId);
|
||||
if (cached && now - cached.checkedAt < QUOTA_CACHE_TTL_MS) {
|
||||
return cached.bytes;
|
||||
}
|
||||
|
||||
const resolvedBase = path.resolve(this.baseDir);
|
||||
const clientDir = path.resolve(resolvedBase, safeClientId);
|
||||
if (!clientDir.startsWith(resolvedBase + path.sep)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let totalBytes = 0;
|
||||
try {
|
||||
const dirStat = await fs.promises.lstat(clientDir).catch(() => null);
|
||||
if (dirStat && !dirStat.isSymbolicLink() && dirStat.isDirectory()) {
|
||||
const entries = await fs.promises.readdir(clientDir).catch(() => []);
|
||||
for (const entry of entries) {
|
||||
if (!this.isPartitionFilename(entry)) continue;
|
||||
const filePath = path.join(clientDir, entry);
|
||||
const stats = await fs.promises.lstat(filePath).catch(() => null);
|
||||
if (stats && !stats.isSymbolicLink() && stats.isFile()) {
|
||||
totalBytes += stats.size;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
|
||||
this.clientBytesCache.set(safeClientId, { checkedAt: now, bytes: totalBytes });
|
||||
return totalBytes;
|
||||
}
|
||||
|
||||
async checkStorageAdmission(clientId, clientQuotaMb, diskPauseThresholdPct) {
|
||||
const threshold = Number(diskPauseThresholdPct);
|
||||
if (Number.isFinite(threshold) && threshold >= 50 && threshold <= 99) {
|
||||
const disk = await this.getDiskUsageAsync();
|
||||
if (disk.totalBytes > 0 && disk.usedPct >= threshold) {
|
||||
return {
|
||||
allowed: false,
|
||||
code: 'DISK_SPACE_LOW',
|
||||
reason: `DISK_SPACE_LOW: Disk utilization (${disk.usedPct.toFixed(1)}%) reached safety pause threshold (${threshold}%).`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const quotaMb = Number(clientQuotaMb);
|
||||
if (Number.isFinite(quotaMb) && quotaMb > 0) {
|
||||
const currentBytes = await this.getClientStorageBytesAsync(clientId);
|
||||
const quotaBytes = Math.floor(quotaMb) * 1024 * 1024;
|
||||
if (currentBytes >= quotaBytes) {
|
||||
return {
|
||||
allowed: false,
|
||||
code: 'STORAGE_QUOTA_EXCEEDED',
|
||||
reason: `STORAGE_QUOTA_EXCEEDED: Client '${clientId}' used ${(currentBytes / (1024 * 1024)).toFixed(2)} MB of ${quotaMb} MB quota.`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return { allowed: true };
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { CsvWriterWorker, resolveDefaultStorageDir };
|
||||
Reference in new issue
Block a user