feat(client): implement per-client storage and security controls
- Added storage quota, IP allowlisting, and expiration logic to client management APIs. - Updated CSV storage manager to support auto-compression and improved partition file parsing. - Integrated security checks into event ingestion and data retrieval routes.
This commit is contained in:
1 parent
89e4d68690
commit
2873b9904a
32 files changed
+6075
-241
No files matched your search
+444
-59
@@ -1,7 +1,10 @@
|
||||
import crypto from 'node:crypto';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { Readable } from 'node:stream';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import { StringDecoder } from 'node:string_decoder';
|
||||
import zlib from 'node:zlib';
|
||||
import { Parser } from 'json2csv';
|
||||
|
||||
export interface DataEvent {
|
||||
@@ -19,8 +22,28 @@ export interface StorageFileMeta {
|
||||
sizeBytes: number;
|
||||
linesCount: number;
|
||||
modifiedAt: Date;
|
||||
compressed?: boolean;
|
||||
}
|
||||
|
||||
export interface DiskUsageSnapshot {
|
||||
usedPct: number;
|
||||
availableBytes: number;
|
||||
totalBytes: number;
|
||||
checkedAt: number;
|
||||
}
|
||||
|
||||
export interface StorageAdmissionResult {
|
||||
allowed: boolean;
|
||||
code?: 'DISK_SPACE_LOW' | 'STORAGE_QUOTA_EXCEEDED';
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
export type StatfsProvider = (dir: string) => Promise<{
|
||||
blocks: number;
|
||||
bavail: number;
|
||||
bsize: number;
|
||||
}>;
|
||||
|
||||
interface RowCountCacheEntry {
|
||||
sizeBytes: number;
|
||||
mtimeMs: number;
|
||||
@@ -31,6 +54,7 @@ const LOCK_RETRY_MIN_MS = 5;
|
||||
const LOCK_RETRY_MAX_MS = 18;
|
||||
const LOCK_TIMEOUT_MS = 10_000;
|
||||
const STALE_LOCK_AGE_MS = 30_000;
|
||||
const QUOTA_CACHE_TTL_MS = 30_000;
|
||||
|
||||
function sleepMs(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
@@ -49,11 +73,39 @@ export class CSVStorageManager {
|
||||
public static baseStorageDir = resolveDefaultStorageDir();
|
||||
public static maxFileSizeBytes = 100 * 1024 * 1024; // 100MB rotation limit
|
||||
private static rowCountCache = new Map<string, RowCountCacheEntry>();
|
||||
private static diskUsageCache: DiskUsageSnapshot | null = null;
|
||||
private static clientBytesCache = new Map<
|
||||
string,
|
||||
{ checkedAt: number; bytes: number }
|
||||
>();
|
||||
private static statfsOverrideForTests: StatfsProvider | null = null;
|
||||
|
||||
public static isPartitionFilename(filename: string): boolean {
|
||||
return filename.endsWith('.csv') || filename.endsWith('.csv.gz');
|
||||
}
|
||||
|
||||
public static extractDateFromFilename(filename: string): string {
|
||||
return filename.replace(/\.csv(?:\.gz)?$/, '').split('-part')[0];
|
||||
}
|
||||
|
||||
public static clearRowCountCache(): void {
|
||||
this.rowCountCache.clear();
|
||||
}
|
||||
|
||||
public static clearQuotaCaches(clientId?: string): void {
|
||||
this.diskUsageCache = null;
|
||||
if (clientId) {
|
||||
this.clientBytesCache.delete(this.sanitizeClientId(clientId));
|
||||
} else {
|
||||
this.clientBytesCache.clear();
|
||||
}
|
||||
}
|
||||
|
||||
public static setStatfsOverrideForTests(provider: StatfsProvider | null): void {
|
||||
this.statfsOverrideForTests = provider;
|
||||
this.diskUsageCache = null;
|
||||
}
|
||||
|
||||
public static sanitizeClientId(clientId: string): string {
|
||||
const raw = String(clientId || 'unknown').trim().replace(/[^a-zA-Z0-9_-]/g, '_');
|
||||
return raw || 'unknown';
|
||||
@@ -299,6 +351,12 @@ export class CSVStorageManager {
|
||||
|
||||
private static countCsvDataRowsChunkedSync(filePath: string): number {
|
||||
const counter = this.createChunkRowCounter();
|
||||
if (filePath.endsWith('.gz')) {
|
||||
const compressed = fs.readFileSync(filePath);
|
||||
const decompressed = zlib.gunzipSync(compressed);
|
||||
counter.pushChunk(decompressed);
|
||||
return counter.finish();
|
||||
}
|
||||
const buf = Buffer.allocUnsafe(64 * 1024);
|
||||
const fd = fs.openSync(filePath, 'r');
|
||||
try {
|
||||
@@ -312,14 +370,34 @@ export class CSVStorageManager {
|
||||
}
|
||||
}
|
||||
|
||||
public static openPartitionTextStream(filePath: string): Readable {
|
||||
const fileStream = fs.createReadStream(filePath, {
|
||||
highWaterMark: 64 * 1024,
|
||||
});
|
||||
if (!filePath.endsWith('.gz')) {
|
||||
return fileStream;
|
||||
}
|
||||
const gunzip = zlib.createGunzip();
|
||||
fileStream.on('error', (err) => {
|
||||
if (!gunzip.destroyed) {
|
||||
gunzip.destroy(err);
|
||||
}
|
||||
});
|
||||
gunzip.on('close', () => {
|
||||
if (!fileStream.destroyed) {
|
||||
fileStream.destroy();
|
||||
}
|
||||
});
|
||||
fileStream.pipe(gunzip);
|
||||
return gunzip;
|
||||
}
|
||||
|
||||
private static async countCsvDataRowsStreamAsync(
|
||||
filePath: string,
|
||||
): Promise<number> {
|
||||
const counter = this.createChunkRowCounter();
|
||||
return new Promise<number>((resolve, reject) => {
|
||||
const stream = fs.createReadStream(filePath, {
|
||||
highWaterMark: 64 * 1024,
|
||||
});
|
||||
const stream = this.openPartitionTextStream(filePath);
|
||||
stream.on('data', (chunk) => counter.pushChunk(chunk));
|
||||
stream.on('end', () => resolve(counter.finish()));
|
||||
stream.on('error', (err) => reject(err));
|
||||
@@ -437,8 +515,9 @@ export class CSVStorageManager {
|
||||
fs.writeFileSync(resolvedPath, fullCsv, 'utf8');
|
||||
}
|
||||
|
||||
// Invalidate row cache for modified file immediately
|
||||
// Invalidate row & quota caches for modified file immediately
|
||||
this.rowCountCache.delete(resolvedPath);
|
||||
this.clearQuotaCaches(clientId);
|
||||
return resolvedPath;
|
||||
});
|
||||
|
||||
@@ -449,7 +528,7 @@ export class CSVStorageManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* List all stored CSV files with cached row counts (invalidated on mtime/size changes)
|
||||
* List all stored CSV and gzipped CSV (.csv / .csv.gz) files with cached row counts
|
||||
*/
|
||||
public static listAllFiles(): StorageFileMeta[] {
|
||||
const results: StorageFileMeta[] = [];
|
||||
@@ -473,7 +552,7 @@ export class CSVStorageManager {
|
||||
|
||||
const files = fs.readdirSync(fullClientPath);
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
if (!this.isPartitionFilename(file)) continue;
|
||||
const fullFilePath = path.join(fullClientPath, file);
|
||||
let stats: fs.Stats;
|
||||
try {
|
||||
@@ -505,7 +584,7 @@ export class CSVStorageManager {
|
||||
}
|
||||
}
|
||||
|
||||
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||
const datePart = this.extractDateFromFilename(file);
|
||||
|
||||
results.push({
|
||||
clientId: clientDir,
|
||||
@@ -515,6 +594,7 @@ export class CSVStorageManager {
|
||||
sizeBytes: stats.size,
|
||||
linesCount,
|
||||
modifiedAt: stats.mtime,
|
||||
compressed: file.endsWith('.csv.gz'),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -551,7 +631,7 @@ export class CSVStorageManager {
|
||||
|
||||
const files = await fs.promises.readdir(fullClientPath);
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
if (!this.isPartitionFilename(file)) continue;
|
||||
const fullFilePath = path.join(fullClientPath, file);
|
||||
const stats = await fs.promises.lstat(fullFilePath).catch(() => null);
|
||||
if (!stats || stats.isSymbolicLink() || !stats.isFile()) continue;
|
||||
@@ -578,7 +658,7 @@ export class CSVStorageManager {
|
||||
}
|
||||
}
|
||||
|
||||
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||
const datePart = this.extractDateFromFilename(file);
|
||||
|
||||
results.push({
|
||||
clientId: clientDir,
|
||||
@@ -588,6 +668,7 @@ export class CSVStorageManager {
|
||||
sizeBytes: stats.size,
|
||||
linesCount,
|
||||
modifiedAt: stats.mtime,
|
||||
compressed: file.endsWith('.csv.gz'),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -605,7 +686,121 @@ export class CSVStorageManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Prune CSV partitions older than autoPruneDays under cooperative partition locks
|
||||
* Compress uncompressed .csv partitions older than compressAfterDays into .csv.gz under cooperative partition locks
|
||||
*/
|
||||
public static async compressOldPartitionsAsync(
|
||||
compressAfterDays: number,
|
||||
): Promise<number> {
|
||||
if (!Number.isFinite(compressAfterDays) || compressAfterDays <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const baseStat = await fs.promises.lstat(this.baseStorageDir).catch(() => null);
|
||||
if (!baseStat || !baseStat.isDirectory()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const cutoffDate = new Date(
|
||||
Date.now() - Math.floor(compressAfterDays) * 86_400_000,
|
||||
)
|
||||
.toISOString()
|
||||
.split('T')[0];
|
||||
|
||||
let compressedCount = 0;
|
||||
try {
|
||||
const clientDirs = await fs.promises.readdir(this.baseStorageDir);
|
||||
for (const clientDir of clientDirs) {
|
||||
const fullClientPath = path.join(this.baseStorageDir, clientDir);
|
||||
const dirLstat = await fs.promises.lstat(fullClientPath).catch(() => null);
|
||||
if (!dirLstat || dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const files = await fs.promises
|
||||
.readdir(fullClientPath)
|
||||
.catch(() => [] as string[]);
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
const datePart = this.extractDateFromFilename(file);
|
||||
if (!(/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate)) {
|
||||
continue;
|
||||
}
|
||||
const fullPath = path.join(fullClientPath, file);
|
||||
const fileLstat = await fs.promises.lstat(fullPath).catch(() => null);
|
||||
if (!fileLstat || fileLstat.isSymbolicLink() || !fileLstat.isFile()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
|
||||
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
|
||||
let acquired = false;
|
||||
const tryAcquireLock = async (): Promise<boolean> => {
|
||||
const handle = await fs.promises.open(lockPath, 'wx');
|
||||
try {
|
||||
await handle.writeFile(lockValue, 'utf8');
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
try {
|
||||
acquired = await tryAcquireLock();
|
||||
} catch (err: unknown) {
|
||||
if ((err as NodeJS.ErrnoException)?.code === 'EEXIST') {
|
||||
await this.removeStaleLockAtomicallyAsync(lockPath);
|
||||
try {
|
||||
acquired = await tryAcquireLock();
|
||||
} catch {
|
||||
acquired = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!acquired) continue;
|
||||
|
||||
let targetGzPath = `${fullPath}.gz`;
|
||||
if (fs.existsSync(targetGzPath)) {
|
||||
const stem = fullPath.replace(/\.csv$/, '');
|
||||
let suffix = 2;
|
||||
while (fs.existsSync(`${stem}-gz${suffix}.csv.gz`)) {
|
||||
suffix += 1;
|
||||
}
|
||||
targetGzPath = `${stem}-gz${suffix}.csv.gz`;
|
||||
}
|
||||
const tmpGzPath = `${targetGzPath}.tmp.${crypto.randomUUID()}`;
|
||||
|
||||
try {
|
||||
await pipeline(
|
||||
fs.createReadStream(fullPath, { highWaterMark: 64 * 1024 }),
|
||||
zlib.createGzip({ level: 6 }),
|
||||
fs.createWriteStream(tmpGzPath),
|
||||
);
|
||||
await fs.promises
|
||||
.utimes(tmpGzPath, fileLstat.atime, fileLstat.mtime)
|
||||
.catch(() => undefined);
|
||||
await fs.promises.rename(tmpGzPath, targetGzPath);
|
||||
await fs.promises.unlink(fullPath);
|
||||
this.rowCountCache.delete(fullPath);
|
||||
this.rowCountCache.delete(targetGzPath);
|
||||
this.clearQuotaCaches(clientDir);
|
||||
compressedCount += 1;
|
||||
} catch {
|
||||
await fs.promises.unlink(tmpGzPath).catch(() => undefined);
|
||||
} finally {
|
||||
await this.releaseLockIfOwnedAsync(lockPath, lockValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore directory scan failure
|
||||
}
|
||||
|
||||
return compressedCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prune CSV / CSV.GZ partitions older than autoPruneDays under cooperative partition locks
|
||||
*/
|
||||
public static pruneExpiredPartitions(autoPruneDays: number): number {
|
||||
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
|
||||
@@ -636,8 +831,8 @@ export class CSVStorageManager {
|
||||
|
||||
const files = fs.readdirSync(fullClientPath);
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||
if (!this.isPartitionFilename(file)) continue;
|
||||
const datePart = this.extractDateFromFilename(file);
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
|
||||
const fullPath = path.join(fullClientPath, file);
|
||||
let fileLstat: fs.Stats;
|
||||
@@ -683,6 +878,7 @@ export class CSVStorageManager {
|
||||
if (fs.existsSync(fullPath)) {
|
||||
fs.unlinkSync(fullPath);
|
||||
this.rowCountCache.delete(fullPath);
|
||||
this.clearQuotaCaches(clientDir);
|
||||
prunedCount += 1;
|
||||
}
|
||||
} catch {
|
||||
@@ -701,27 +897,54 @@ export class CSVStorageManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Non-blocking async partition pruner using fs.promises under cooperative partition locks
|
||||
* Archive and/or prune expired partitions under cooperative partition locks.
|
||||
* Never deletes a local partition if options.beforeDeleteFile throws or rejects (V2-E02).
|
||||
*/
|
||||
public static async pruneExpiredPartitionsAsync(
|
||||
public static async pruneOrArchiveExpiredPartitionsAsync(
|
||||
autoPruneDays: number,
|
||||
): Promise<number> {
|
||||
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
|
||||
return 0;
|
||||
options?: {
|
||||
beforeDeleteFile?: (file: {
|
||||
clientId: string;
|
||||
filename: string;
|
||||
filePath: string;
|
||||
date: string;
|
||||
}) => Promise<void>;
|
||||
archiveOnlyWithoutDelete?: boolean;
|
||||
maxCutoffDays?: number;
|
||||
},
|
||||
): Promise<{
|
||||
scannedCount: number;
|
||||
archivedCount: number;
|
||||
prunedCount: number;
|
||||
errors: string[];
|
||||
}> {
|
||||
const targetDays =
|
||||
options?.maxCutoffDays !== undefined
|
||||
? options.maxCutoffDays
|
||||
: autoPruneDays;
|
||||
if (!Number.isFinite(targetDays) || targetDays < 0) {
|
||||
return { scannedCount: 0, archivedCount: 0, prunedCount: 0, errors: [] };
|
||||
}
|
||||
if (options?.maxCutoffDays === undefined && targetDays <= 0) {
|
||||
return { scannedCount: 0, archivedCount: 0, prunedCount: 0, errors: [] };
|
||||
}
|
||||
|
||||
const baseStat = await fs.promises.lstat(this.baseStorageDir).catch(() => null);
|
||||
if (!baseStat || !baseStat.isDirectory()) {
|
||||
return 0;
|
||||
return { scannedCount: 0, archivedCount: 0, prunedCount: 0, errors: [] };
|
||||
}
|
||||
|
||||
const cutoffDate = new Date(
|
||||
Date.now() - Math.floor(autoPruneDays) * 86_400_000,
|
||||
Date.now() - Math.floor(targetDays) * 86_400_000,
|
||||
)
|
||||
.toISOString()
|
||||
.split('T')[0];
|
||||
|
||||
let scannedCount = 0;
|
||||
let archivedCount = 0;
|
||||
let prunedCount = 0;
|
||||
const errors: string[] = [];
|
||||
|
||||
try {
|
||||
const clientDirs = await fs.promises.readdir(this.baseStorageDir);
|
||||
for (const clientDir of clientDirs) {
|
||||
@@ -731,54 +954,71 @@ export class CSVStorageManager {
|
||||
continue;
|
||||
}
|
||||
|
||||
const files = await fs.promises.readdir(fullClientPath).catch(() => [] as string[]);
|
||||
const files = await fs.promises
|
||||
.readdir(fullClientPath)
|
||||
.catch(() => [] as string[]);
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.csv')) continue;
|
||||
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
|
||||
const fullPath = path.join(fullClientPath, file);
|
||||
const fileLstat = await fs.promises.lstat(fullPath).catch(() => null);
|
||||
if (!fileLstat || fileLstat.isSymbolicLink() || !fileLstat.isFile()) {
|
||||
continue;
|
||||
}
|
||||
if (!this.isPartitionFilename(file)) continue;
|
||||
const datePart = this.extractDateFromFilename(file);
|
||||
if (!(/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate)) {
|
||||
continue;
|
||||
}
|
||||
const fullPath = path.join(fullClientPath, file);
|
||||
const fileLstat = await fs.promises.lstat(fullPath).catch(() => null);
|
||||
if (!fileLstat || fileLstat.isSymbolicLink() || !fileLstat.isFile()) {
|
||||
continue;
|
||||
}
|
||||
scannedCount += 1;
|
||||
|
||||
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
|
||||
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
|
||||
let acquired = false;
|
||||
const tryAcquireLock = async (): Promise<boolean> => {
|
||||
const handle = await fs.promises.open(lockPath, 'wx');
|
||||
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
|
||||
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
|
||||
let acquired = false;
|
||||
const tryAcquireLock = async (): Promise<boolean> => {
|
||||
const handle = await fs.promises.open(lockPath, 'wx');
|
||||
try {
|
||||
await handle.writeFile(lockValue, 'utf8');
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
try {
|
||||
acquired = await tryAcquireLock();
|
||||
} catch (err: unknown) {
|
||||
if ((err as NodeJS.ErrnoException)?.code === 'EEXIST') {
|
||||
await this.removeStaleLockAtomicallyAsync(lockPath);
|
||||
try {
|
||||
await handle.writeFile(lockValue, 'utf8');
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
try {
|
||||
acquired = await tryAcquireLock();
|
||||
} catch (err: unknown) {
|
||||
if ((err as NodeJS.ErrnoException)?.code === 'EEXIST') {
|
||||
await this.removeStaleLockAtomicallyAsync(lockPath);
|
||||
try {
|
||||
acquired = await tryAcquireLock();
|
||||
} catch {
|
||||
acquired = false;
|
||||
}
|
||||
acquired = await tryAcquireLock();
|
||||
} catch {
|
||||
acquired = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!acquired) continue;
|
||||
if (!acquired) continue;
|
||||
|
||||
try {
|
||||
try {
|
||||
if (options?.beforeDeleteFile) {
|
||||
await options.beforeDeleteFile({
|
||||
clientId: clientDir,
|
||||
filename: file,
|
||||
filePath: fullPath,
|
||||
date: datePart,
|
||||
});
|
||||
archivedCount += 1;
|
||||
}
|
||||
if (!options?.archiveOnlyWithoutDelete) {
|
||||
await fs.promises.unlink(fullPath);
|
||||
this.rowCountCache.delete(fullPath);
|
||||
this.clearQuotaCaches(clientDir);
|
||||
prunedCount += 1;
|
||||
} catch {
|
||||
// Ignore individual unlink error or concurrent removal
|
||||
} finally {
|
||||
await this.releaseLockIfOwnedAsync(lockPath, lockValue);
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
const msg = err instanceof Error ? err.message : 'Archive/prune error';
|
||||
errors.push(`${clientDir}/${file}: ${msg}`);
|
||||
} finally {
|
||||
await this.releaseLockIfOwnedAsync(lockPath, lockValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -786,7 +1026,149 @@ export class CSVStorageManager {
|
||||
// Ignore directory scan failure
|
||||
}
|
||||
|
||||
return prunedCount;
|
||||
return { scannedCount, archivedCount, prunedCount, errors };
|
||||
}
|
||||
|
||||
/**
|
||||
* Non-blocking async partition pruner using fs.promises under cooperative partition locks
|
||||
*/
|
||||
public static async pruneExpiredPartitionsAsync(
|
||||
autoPruneDays: number,
|
||||
): Promise<number> {
|
||||
const res = await this.pruneOrArchiveExpiredPartitionsAsync(autoPruneDays);
|
||||
return res.prunedCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cached VPS filesystem disk usage snapshot (30s TTL) for disk-full protection
|
||||
*/
|
||||
public static async getDiskUsageAsync(): Promise<DiskUsageSnapshot> {
|
||||
const now = Date.now();
|
||||
if (
|
||||
this.diskUsageCache &&
|
||||
now - this.diskUsageCache.checkedAt < QUOTA_CACHE_TTL_MS
|
||||
) {
|
||||
return this.diskUsageCache;
|
||||
}
|
||||
|
||||
let probeDir = path.resolve(this.baseStorageDir);
|
||||
while (!fs.existsSync(probeDir)) {
|
||||
const parent = path.dirname(probeDir);
|
||||
if (parent === probeDir) break;
|
||||
probeDir = parent;
|
||||
}
|
||||
|
||||
try {
|
||||
const statfsRes = this.statfsOverrideForTests
|
||||
? await this.statfsOverrideForTests(probeDir)
|
||||
: await fs.promises.statfs(probeDir);
|
||||
const bsize = Math.max(1, Number(statfsRes.bsize) || 4096);
|
||||
const totalBytes = Math.max(0, Number(statfsRes.blocks) * bsize);
|
||||
const availableBytes = Math.max(0, Number(statfsRes.bavail) * bsize);
|
||||
const usedBytes = Math.max(0, totalBytes - availableBytes);
|
||||
const usedPct =
|
||||
totalBytes > 0
|
||||
? Number(
|
||||
Math.min(100, Math.max(0, (usedBytes / totalBytes) * 100)).toFixed(2),
|
||||
)
|
||||
: 0;
|
||||
const snapshot: DiskUsageSnapshot = {
|
||||
usedPct,
|
||||
availableBytes,
|
||||
totalBytes,
|
||||
checkedAt: now,
|
||||
};
|
||||
this.diskUsageCache = snapshot;
|
||||
return snapshot;
|
||||
} catch {
|
||||
const fallback: DiskUsageSnapshot = {
|
||||
usedPct: 0,
|
||||
availableBytes: 0,
|
||||
totalBytes: 0,
|
||||
checkedAt: now,
|
||||
};
|
||||
this.diskUsageCache = fallback;
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Cached per-client directory byte count (30s TTL)
|
||||
*/
|
||||
public static async getClientStorageBytesAsync(
|
||||
clientId: string,
|
||||
): Promise<number> {
|
||||
const safeClientId = this.sanitizeClientId(clientId);
|
||||
const now = Date.now();
|
||||
const cached = this.clientBytesCache.get(safeClientId);
|
||||
if (cached && now - cached.checkedAt < QUOTA_CACHE_TTL_MS) {
|
||||
return cached.bytes;
|
||||
}
|
||||
|
||||
const resolvedBase = path.resolve(this.baseStorageDir);
|
||||
const clientDir = path.resolve(resolvedBase, safeClientId);
|
||||
if (!clientDir.startsWith(resolvedBase + path.sep)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let totalBytes = 0;
|
||||
try {
|
||||
const dirStat = await fs.promises.lstat(clientDir).catch(() => null);
|
||||
if (dirStat && !dirStat.isSymbolicLink() && dirStat.isDirectory()) {
|
||||
const entries = await fs.promises
|
||||
.readdir(clientDir)
|
||||
.catch(() => [] as string[]);
|
||||
for (const entry of entries) {
|
||||
if (!this.isPartitionFilename(entry)) continue;
|
||||
const filePath = path.join(clientDir, entry);
|
||||
const stats = await fs.promises.lstat(filePath).catch(() => null);
|
||||
if (stats && !stats.isSymbolicLink() && stats.isFile()) {
|
||||
totalBytes += stats.size;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Fall back to 0 on unreadable directory
|
||||
}
|
||||
|
||||
this.clientBytesCache.set(safeClientId, { checkedAt: now, bytes: totalBytes });
|
||||
return totalBytes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks global VPS disk threshold and per-client MB quota before admitting new events
|
||||
*/
|
||||
public static async checkStorageAdmissionAsync(
|
||||
clientId: string,
|
||||
clientQuotaMb: number,
|
||||
diskPauseThresholdPct: number,
|
||||
): Promise<StorageAdmissionResult> {
|
||||
const threshold = Number(diskPauseThresholdPct);
|
||||
if (Number.isFinite(threshold) && threshold >= 50 && threshold <= 99) {
|
||||
const disk = await this.getDiskUsageAsync();
|
||||
if (disk.totalBytes > 0 && disk.usedPct >= threshold) {
|
||||
return {
|
||||
allowed: false,
|
||||
code: 'DISK_SPACE_LOW',
|
||||
reason: `DISK_SPACE_LOW: Disk utilization (${disk.usedPct.toFixed(1)}%) reached safety pause threshold (${threshold}%).`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const quotaMb = Number(clientQuotaMb);
|
||||
if (Number.isFinite(quotaMb) && quotaMb > 0) {
|
||||
const currentBytes = await this.getClientStorageBytesAsync(clientId);
|
||||
const quotaBytes = Math.floor(quotaMb) * 1024 * 1024;
|
||||
if (currentBytes >= quotaBytes) {
|
||||
return {
|
||||
allowed: false,
|
||||
code: 'STORAGE_QUOTA_EXCEEDED',
|
||||
reason: `STORAGE_QUOTA_EXCEEDED: Client '${clientId}' used ${(currentBytes / (1024 * 1024)).toFixed(2)} MB of ${quotaMb} MB quota.`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return { allowed: true };
|
||||
}
|
||||
|
||||
public static formatStorageStats(files: StorageFileMeta[]) {
|
||||
@@ -826,12 +1208,12 @@ export class CSVStorageManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads file content for export / download with path traversal protection
|
||||
* Reads file content for export / download with path traversal protection (supports .csv & .csv.gz)
|
||||
*/
|
||||
public static getFileContent(clientId: string, filename: string): string | null {
|
||||
const safeClientId = this.sanitizeClientId(clientId);
|
||||
const safeFilename = path.basename(String(filename || ''));
|
||||
if (!safeClientId || !safeFilename || !safeFilename.endsWith('.csv')) {
|
||||
if (!safeClientId || !safeFilename || !this.isPartitionFilename(safeFilename)) {
|
||||
return null;
|
||||
}
|
||||
const resolvedBase = path.resolve(this.baseStorageDir);
|
||||
@@ -844,6 +1226,9 @@ export class CSVStorageManager {
|
||||
if (stats.isSymbolicLink() || !stats.isFile()) {
|
||||
return null;
|
||||
}
|
||||
if (safeFilename.endsWith('.gz')) {
|
||||
return zlib.gunzipSync(fs.readFileSync(fullPath)).toString('utf8');
|
||||
}
|
||||
return fs.readFileSync(fullPath, 'utf8');
|
||||
}
|
||||
return null;
|
||||
|
||||
@@ -0,0 +1,335 @@
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
|
||||
export interface S3ArchiveConfig {
|
||||
endpoint: string;
|
||||
region: string;
|
||||
bucket: string;
|
||||
prefix: string;
|
||||
accessKeyId: string;
|
||||
secretAccessKey: string;
|
||||
}
|
||||
|
||||
export interface S3OperationResult {
|
||||
ok: boolean;
|
||||
status?: number;
|
||||
endpoint?: string;
|
||||
bucket?: string;
|
||||
key?: string;
|
||||
etag?: string;
|
||||
code?: "S3_INVALID_CONFIG" | "S3_CONNECTION_FAILED" | "S3_ARCHIVE_FAILED";
|
||||
error?: string;
|
||||
}
|
||||
|
||||
function sha256Hex(data: Buffer | string): string {
|
||||
return crypto.createHash("sha256").update(data).digest("hex");
|
||||
}
|
||||
|
||||
function hmacSha256(key: Buffer | string, data: string): Buffer {
|
||||
return crypto.createHmac("sha256", key).update(data, "utf8").digest();
|
||||
}
|
||||
|
||||
function encodeRfc3986(value: string): string {
|
||||
return encodeURIComponent(value).replace(
|
||||
/[!'()*]/g,
|
||||
(ch) => `%${ch.charCodeAt(0).toString(16).toUpperCase()}`,
|
||||
);
|
||||
}
|
||||
|
||||
function encodeS3Path(rawPath: string): string {
|
||||
return rawPath
|
||||
.split("/")
|
||||
.map((segment) => encodeRfc3986(segment))
|
||||
.join("/");
|
||||
}
|
||||
|
||||
export function buildS3ObjectKey(
|
||||
prefix: string,
|
||||
clientId: string,
|
||||
filename: string,
|
||||
): string {
|
||||
const cleanPrefix = String(prefix || "")
|
||||
.trim()
|
||||
.replace(/^\/+|\/+$/g, "");
|
||||
const cleanClient = String(clientId || "unknown")
|
||||
.trim()
|
||||
.replace(/^\/+|\/+$/g, "");
|
||||
const cleanFile = String(filename || "")
|
||||
.trim()
|
||||
.replace(/^\/+/, "");
|
||||
return [cleanPrefix, cleanClient, cleanFile].filter(Boolean).join("/");
|
||||
}
|
||||
|
||||
export function normalizeS3EndpointUrl(
|
||||
endpoint: string,
|
||||
region: string,
|
||||
): { url: URL; region: string } {
|
||||
const trimmedRegion = String(region || "").trim() || "auto";
|
||||
const trimmedEndpoint = String(endpoint || "").trim().replace(/\/+$/, "");
|
||||
|
||||
if (trimmedEndpoint) {
|
||||
const withProtocol = /^https?:\/\//i.test(trimmedEndpoint)
|
||||
? trimmedEndpoint
|
||||
: `https://${trimmedEndpoint}`;
|
||||
const parsed = new URL(withProtocol);
|
||||
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
||||
throw new Error("S3 endpoint must use http:// or https://");
|
||||
}
|
||||
return { url: parsed, region: trimmedRegion };
|
||||
}
|
||||
|
||||
if (trimmedRegion === "auto") {
|
||||
throw new Error(
|
||||
"S3 Endpoint URL is required when Region is set to 'auto' (e.g. Cloudflare R2 or MinIO).",
|
||||
);
|
||||
}
|
||||
return {
|
||||
url: new URL(`https://s3.${trimmedRegion}.amazonaws.com`),
|
||||
region: trimmedRegion,
|
||||
};
|
||||
}
|
||||
|
||||
export function signS3Request(params: {
|
||||
method: "GET" | "HEAD" | "PUT";
|
||||
endpointUrl: URL;
|
||||
bucket: string;
|
||||
objectKey?: string;
|
||||
queryParams?: Record<string, string>;
|
||||
region: string;
|
||||
accessKeyId: string;
|
||||
secretAccessKey: string;
|
||||
payloadHashHex: string;
|
||||
contentType?: string;
|
||||
now?: Date;
|
||||
}): { targetUrl: string; headers: Record<string, string> } {
|
||||
const now = params.now ?? new Date();
|
||||
const amzDate = now.toISOString().replace(/[:-]|\.\d{3}/g, "");
|
||||
const dateStamp = amzDate.slice(0, 8);
|
||||
|
||||
const basePath = params.endpointUrl.pathname.replace(/\/+$/, "");
|
||||
const bucketPart = encodeRfc3986(params.bucket.trim());
|
||||
const keyPart = params.objectKey ? `/${encodeS3Path(params.objectKey)}` : "";
|
||||
const canonicalUri = `${basePath}/${bucketPart}${keyPart}`.replace(
|
||||
/\/{2,}/g,
|
||||
"/",
|
||||
);
|
||||
|
||||
const queryEntries = Object.entries(params.queryParams ?? {}).sort(([a], [b]) =>
|
||||
a < b ? -1 : a > b ? 1 : 0,
|
||||
);
|
||||
const canonicalQueryString = queryEntries
|
||||
.map(([k, v]) => `${encodeRfc3986(k)}=${encodeRfc3986(v)}`)
|
||||
.join("&");
|
||||
|
||||
const hostHeader = params.endpointUrl.host;
|
||||
const headerMap: Record<string, string> = {
|
||||
host: hostHeader,
|
||||
"x-amz-content-sha256": params.payloadHashHex,
|
||||
"x-amz-date": amzDate,
|
||||
};
|
||||
if (params.contentType) {
|
||||
headerMap["content-type"] = params.contentType;
|
||||
}
|
||||
|
||||
const sortedHeaderKeys = Object.keys(headerMap).sort();
|
||||
const canonicalHeaders = sortedHeaderKeys
|
||||
.map((key) => `${key}:${headerMap[key].trim()}\n`)
|
||||
.join("");
|
||||
const signedHeaders = sortedHeaderKeys.join(";");
|
||||
|
||||
const canonicalRequest = [
|
||||
params.method,
|
||||
canonicalUri,
|
||||
canonicalQueryString,
|
||||
canonicalHeaders,
|
||||
signedHeaders,
|
||||
params.payloadHashHex,
|
||||
].join("\n");
|
||||
|
||||
const credentialScope = `${dateStamp}/${params.region}/s3/aws4_request`;
|
||||
const stringToSign = [
|
||||
"AWS4-HMAC-SHA256",
|
||||
amzDate,
|
||||
credentialScope,
|
||||
sha256Hex(canonicalRequest),
|
||||
].join("\n");
|
||||
|
||||
const kDate = hmacSha256(`AWS4${params.secretAccessKey}`, dateStamp);
|
||||
const kRegion = hmacSha256(kDate, params.region);
|
||||
const kService = hmacSha256(kRegion, "s3");
|
||||
const kSigning = hmacSha256(kService, "aws4_request");
|
||||
const signature = crypto
|
||||
.createHmac("sha256", kSigning)
|
||||
.update(stringToSign, "utf8")
|
||||
.digest("hex");
|
||||
|
||||
const authorization = `AWS4-HMAC-SHA256 Credential=${params.accessKeyId}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`;
|
||||
|
||||
const targetUrl = `${params.endpointUrl.origin}${canonicalUri}${
|
||||
canonicalQueryString ? `?${canonicalQueryString}` : ""
|
||||
}`;
|
||||
|
||||
return {
|
||||
targetUrl,
|
||||
headers: {
|
||||
"x-amz-date": amzDate,
|
||||
"x-amz-content-sha256": params.payloadHashHex,
|
||||
...(params.contentType ? { "content-type": params.contentType } : {}),
|
||||
Authorization: authorization,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function validateS3Config(config: S3ArchiveConfig): string | null {
|
||||
if (!config.bucket?.trim()) return "S3 Bucket name is required.";
|
||||
if (!config.accessKeyId?.trim()) return "S3 Access Key ID is required.";
|
||||
if (!config.secretAccessKey?.trim()) return "S3 Secret Access Key is required.";
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function testS3Connection(
|
||||
config: S3ArchiveConfig,
|
||||
): Promise<S3OperationResult> {
|
||||
const validationError = validateS3Config(config);
|
||||
if (validationError) {
|
||||
return {
|
||||
ok: false,
|
||||
code: "S3_INVALID_CONFIG",
|
||||
error: validationError,
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const { url, region } = normalizeS3EndpointUrl(
|
||||
config.endpoint,
|
||||
config.region,
|
||||
);
|
||||
const signed = signS3Request({
|
||||
method: "GET",
|
||||
endpointUrl: url,
|
||||
bucket: config.bucket.trim(),
|
||||
queryParams: { "list-type": "2", "max-keys": "1" },
|
||||
region,
|
||||
accessKeyId: config.accessKeyId.trim(),
|
||||
secretAccessKey: config.secretAccessKey,
|
||||
payloadHashHex: sha256Hex(""),
|
||||
});
|
||||
|
||||
const response = await fetch(signed.targetUrl, {
|
||||
method: "GET",
|
||||
headers: signed.headers,
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(8000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const bodyPreview = await response.text().catch(() => "");
|
||||
return {
|
||||
ok: false,
|
||||
status: response.status,
|
||||
endpoint: url.origin,
|
||||
bucket: config.bucket.trim(),
|
||||
code: "S3_CONNECTION_FAILED",
|
||||
error: `S3 endpoint returned HTTP ${response.status}${
|
||||
bodyPreview ? `: ${bodyPreview.slice(0, 180)}` : ""
|
||||
}`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
status: response.status,
|
||||
endpoint: url.origin,
|
||||
bucket: config.bucket.trim(),
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
return {
|
||||
ok: false,
|
||||
code: "S3_CONNECTION_FAILED",
|
||||
error:
|
||||
err instanceof Error ? err.message : "Unable to reach S3 endpoint",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function uploadFileToS3(
|
||||
config: S3ArchiveConfig,
|
||||
localFilePath: string,
|
||||
objectKey: string,
|
||||
): Promise<S3OperationResult> {
|
||||
const validationError = validateS3Config(config);
|
||||
if (validationError) {
|
||||
return {
|
||||
ok: false,
|
||||
code: "S3_INVALID_CONFIG",
|
||||
error: validationError,
|
||||
key: objectKey,
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const { url, region } = normalizeS3EndpointUrl(
|
||||
config.endpoint,
|
||||
config.region,
|
||||
);
|
||||
const bodyBuffer = await fs.promises.readFile(localFilePath);
|
||||
const payloadHashHex = sha256Hex(bodyBuffer);
|
||||
const contentType = localFilePath.endsWith(".gz")
|
||||
? "application/gzip"
|
||||
: "text/csv; charset=utf-8";
|
||||
|
||||
const signed = signS3Request({
|
||||
method: "PUT",
|
||||
endpointUrl: url,
|
||||
bucket: config.bucket.trim(),
|
||||
objectKey,
|
||||
region,
|
||||
accessKeyId: config.accessKeyId.trim(),
|
||||
secretAccessKey: config.secretAccessKey,
|
||||
payloadHashHex,
|
||||
contentType,
|
||||
});
|
||||
|
||||
const response = await fetch(signed.targetUrl, {
|
||||
method: "PUT",
|
||||
headers: {
|
||||
...signed.headers,
|
||||
"content-length": String(bodyBuffer.byteLength),
|
||||
},
|
||||
body: new Uint8Array(bodyBuffer),
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(30000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const bodyPreview = await response.text().catch(() => "");
|
||||
return {
|
||||
ok: false,
|
||||
status: response.status,
|
||||
endpoint: url.origin,
|
||||
bucket: config.bucket.trim(),
|
||||
key: objectKey,
|
||||
code: "S3_ARCHIVE_FAILED",
|
||||
error: `S3 upload failed with HTTP ${response.status}${
|
||||
bodyPreview ? `: ${bodyPreview.slice(0, 180)}` : ""
|
||||
}`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
status: response.status,
|
||||
endpoint: url.origin,
|
||||
bucket: config.bucket.trim(),
|
||||
key: objectKey,
|
||||
etag: response.headers.get("etag") ?? undefined,
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
return {
|
||||
ok: false,
|
||||
key: objectKey,
|
||||
code: "S3_ARCHIVE_FAILED",
|
||||
error: err instanceof Error ? err.message : "S3 upload request failed",
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user