2026-11-19 20:22:01 +06:00
|
|
|
import crypto from 'node:crypto';
|
2026-10-23 11:43:49 -05:00
|
|
|
import fs from 'fs';
|
|
|
|
|
import path from 'path';
|
2026-11-19 23:02:27 +06:00
|
|
|
import { Readable } from 'node:stream';
|
|
|
|
|
import { pipeline } from 'node:stream/promises';
|
2026-11-19 20:22:01 +06:00
|
|
|
import { StringDecoder } from 'node:string_decoder';
|
2026-11-19 23:02:27 +06:00
|
|
|
import zlib from 'node:zlib';
|
2026-10-23 11:43:49 -05:00
|
|
|
import { Parser } from 'json2csv';
|
|
|
|
|
|
|
|
|
|
export interface DataEvent {
|
|
|
|
|
clientId: string;
|
|
|
|
|
eventId: string;
|
|
|
|
|
timestamp: string; // ISO 8601
|
|
|
|
|
payload: string; // JSON stringified
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export interface StorageFileMeta {
|
|
|
|
|
clientId: string;
|
|
|
|
|
date: string;
|
|
|
|
|
filename: string;
|
|
|
|
|
filePath: string;
|
|
|
|
|
sizeBytes: number;
|
|
|
|
|
linesCount: number;
|
|
|
|
|
modifiedAt: Date;
|
2026-11-19 23:02:27 +06:00
|
|
|
compressed?: boolean;
|
2026-10-23 11:43:49 -05:00
|
|
|
}
|
|
|
|
|
|
2026-11-19 23:02:27 +06:00
|
|
|
export interface DiskUsageSnapshot {
|
2026-11-21 00:06:55 +06:00
|
|
|
available: boolean;
|
2026-11-19 23:02:27 +06:00
|
|
|
usedPct: number;
|
|
|
|
|
availableBytes: number;
|
2026-11-21 00:06:55 +06:00
|
|
|
freeBytes: number;
|
2026-11-19 23:02:27 +06:00
|
|
|
totalBytes: number;
|
|
|
|
|
checkedAt: number;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export interface StorageAdmissionResult {
|
|
|
|
|
allowed: boolean;
|
|
|
|
|
code?: 'DISK_SPACE_LOW' | 'STORAGE_QUOTA_EXCEEDED';
|
|
|
|
|
reason?: string;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export type StatfsProvider = (dir: string) => Promise<{
|
|
|
|
|
blocks: number;
|
|
|
|
|
bavail: number;
|
|
|
|
|
bsize: number;
|
|
|
|
|
}>;
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
interface RowCountCacheEntry {
|
|
|
|
|
sizeBytes: number;
|
|
|
|
|
mtimeMs: number;
|
|
|
|
|
linesCount: number;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const LOCK_RETRY_MIN_MS = 5;
|
|
|
|
|
const LOCK_RETRY_MAX_MS = 18;
|
|
|
|
|
const LOCK_TIMEOUT_MS = 10_000;
|
|
|
|
|
const STALE_LOCK_AGE_MS = 30_000;
|
2026-11-19 23:02:27 +06:00
|
|
|
const QUOTA_CACHE_TTL_MS = 30_000;
|
2026-11-18 05:18:55 +06:00
|
|
|
|
|
|
|
|
function sleepMs(ms: number): Promise<void> {
|
|
|
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
export function resolveDefaultStorageDir(): string {
|
|
|
|
|
const configured = (process.env.DATASTACK_STORAGE_DIR || '').trim();
|
|
|
|
|
if (configured) {
|
|
|
|
|
return path.resolve(configured);
|
|
|
|
|
}
|
|
|
|
|
return path.resolve(process.cwd(), 'storage', 'data');
|
|
|
|
|
}
|
|
|
|
|
|
2026-10-23 11:43:49 -05:00
|
|
|
export class CSVStorageManager {
|
2026-11-19 20:22:01 +06:00
|
|
|
// Base storage directory: DATASTACK_STORAGE_DIR or /storage/data
|
|
|
|
|
public static baseStorageDir = resolveDefaultStorageDir();
|
2026-10-23 11:43:49 -05:00
|
|
|
public static maxFileSizeBytes = 100 * 1024 * 1024; // 100MB rotation limit
|
2026-11-18 05:18:55 +06:00
|
|
|
private static rowCountCache = new Map<string, RowCountCacheEntry>();
|
2026-11-19 23:02:27 +06:00
|
|
|
private static diskUsageCache: DiskUsageSnapshot | null = null;
|
|
|
|
|
private static clientBytesCache = new Map<
|
|
|
|
|
string,
|
|
|
|
|
{ checkedAt: number; bytes: number }
|
|
|
|
|
>();
|
|
|
|
|
private static statfsOverrideForTests: StatfsProvider | null = null;
|
|
|
|
|
|
|
|
|
|
public static isPartitionFilename(filename: string): boolean {
|
|
|
|
|
return filename.endsWith('.csv') || filename.endsWith('.csv.gz');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static extractDateFromFilename(filename: string): string {
|
2026-11-21 00:06:55 +06:00
|
|
|
return String(filename || '')
|
|
|
|
|
.replace(/\.csv(?:\.gz)?$/i, '')
|
|
|
|
|
.replace(/(?:-(?:part|gz)\d+)+$/i, '');
|
2026-11-19 23:02:27 +06:00
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
|
|
|
|
|
public static clearRowCountCache(): void {
|
|
|
|
|
this.rowCountCache.clear();
|
|
|
|
|
}
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-11-19 23:02:27 +06:00
|
|
|
public static clearQuotaCaches(clientId?: string): void {
|
|
|
|
|
this.diskUsageCache = null;
|
|
|
|
|
if (clientId) {
|
|
|
|
|
this.clientBytesCache.delete(this.sanitizeClientId(clientId));
|
|
|
|
|
} else {
|
|
|
|
|
this.clientBytesCache.clear();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static setStatfsOverrideForTests(provider: StatfsProvider | null): void {
|
|
|
|
|
this.statfsOverrideForTests = provider;
|
|
|
|
|
this.diskUsageCache = null;
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
public static sanitizeClientId(clientId: string): string {
|
|
|
|
|
const raw = String(clientId || 'unknown').trim().replace(/[^a-zA-Z0-9_-]/g, '_');
|
|
|
|
|
return raw || 'unknown';
|
|
|
|
|
}
|
|
|
|
|
|
2026-10-23 11:43:49 -05:00
|
|
|
/**
|
2026-11-19 20:22:01 +06:00
|
|
|
* Checks whether the last byte of a non-empty file is LF (\n)
|
|
|
|
|
*/
|
|
|
|
|
private static fileEndsWithNewline(filePath: string, sizeBytes: number): boolean {
|
|
|
|
|
if (sizeBytes <= 0) return false;
|
|
|
|
|
const fd = fs.openSync(filePath, 'r');
|
|
|
|
|
try {
|
|
|
|
|
const lastByte = Buffer.allocUnsafe(1);
|
|
|
|
|
const read = fs.readSync(fd, lastByte, 0, 1, sizeBytes - 1);
|
|
|
|
|
return read === 1 && lastByte[0] === 0x0a;
|
|
|
|
|
} finally {
|
|
|
|
|
fs.closeSync(fd);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static removeStaleLockAtomically(lockPath: string): void {
|
|
|
|
|
try {
|
|
|
|
|
const lockStat = fs.lstatSync(lockPath);
|
|
|
|
|
if (Date.now() - lockStat.mtimeMs <= STALE_LOCK_AGE_MS) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
const staleTmp = `${lockPath}.stale.${crypto.randomUUID()}`;
|
|
|
|
|
fs.renameSync(lockPath, staleTmp);
|
|
|
|
|
try {
|
|
|
|
|
fs.unlinkSync(staleTmp);
|
|
|
|
|
} catch {
|
|
|
|
|
// Ignore cleanup failure of already-renamed stale token
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
// Lock was released or claimed concurrently
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static releaseLockIfOwned(lockPath: string, lockValue: string): void {
|
|
|
|
|
try {
|
|
|
|
|
if (!fs.existsSync(lockPath)) return;
|
|
|
|
|
const current = fs.readFileSync(lockPath, 'utf8');
|
|
|
|
|
if (current === lockValue) {
|
|
|
|
|
fs.unlinkSync(lockPath);
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
// Ignore cleanup failure if already unlinked
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static async removeStaleLockAtomicallyAsync(lockPath: string): Promise<void> {
|
|
|
|
|
try {
|
|
|
|
|
const lockStat = await fs.promises.lstat(lockPath);
|
|
|
|
|
if (Date.now() - lockStat.mtimeMs <= STALE_LOCK_AGE_MS) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
const staleTmp = `${lockPath}.stale.${crypto.randomUUID()}`;
|
|
|
|
|
await fs.promises.rename(lockPath, staleTmp);
|
|
|
|
|
await fs.promises.unlink(staleTmp).catch(() => undefined);
|
|
|
|
|
} catch {
|
|
|
|
|
// Lock was released or claimed concurrently
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static async releaseLockIfOwnedAsync(
|
|
|
|
|
lockPath: string,
|
|
|
|
|
lockValue: string,
|
|
|
|
|
): Promise<void> {
|
|
|
|
|
try {
|
|
|
|
|
const current = await fs.promises.readFile(lockPath, 'utf8');
|
|
|
|
|
if (current === lockValue) {
|
|
|
|
|
await fs.promises.unlink(lockPath);
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
// Ignore cleanup failure if already unlinked
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Ensures client directory exists safely within baseStorageDir
|
2026-10-23 11:43:49 -05:00
|
|
|
*/
|
|
|
|
|
public static ensureClientDir(clientId: string): string {
|
2026-11-19 20:22:01 +06:00
|
|
|
const safeClientId = this.sanitizeClientId(clientId);
|
|
|
|
|
const resolvedBase = path.resolve(this.baseStorageDir);
|
|
|
|
|
const clientDir = path.resolve(resolvedBase, safeClientId);
|
|
|
|
|
if (!clientDir.startsWith(resolvedBase + path.sep)) {
|
|
|
|
|
throw new Error(`Invalid storage client directory: ${clientId}`);
|
|
|
|
|
}
|
2026-10-23 11:43:49 -05:00
|
|
|
if (!fs.existsSync(clientDir)) {
|
|
|
|
|
fs.mkdirSync(clientDir, { recursive: true });
|
|
|
|
|
}
|
|
|
|
|
return clientDir;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
2026-11-18 05:18:55 +06:00
|
|
|
* Acquires an exclusive cooperative lock file for a client+date partition across Port 3000 & Port 4000
|
|
|
|
|
*/
|
|
|
|
|
public static async withPartitionLock<T>(
|
|
|
|
|
clientId: string,
|
|
|
|
|
dateStr: string,
|
|
|
|
|
fn: () => Promise<T> | T,
|
|
|
|
|
): Promise<T> {
|
|
|
|
|
const clientDir = this.ensureClientDir(clientId);
|
|
|
|
|
const date = this.extractSafePartitionDate(dateStr);
|
|
|
|
|
const lockPath = path.join(clientDir, `.${date}.csv.lock`);
|
2026-11-19 20:22:01 +06:00
|
|
|
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
|
2026-11-18 05:18:55 +06:00
|
|
|
const deadline = Date.now() + LOCK_TIMEOUT_MS;
|
|
|
|
|
|
|
|
|
|
let acquired = false;
|
|
|
|
|
while (Date.now() < deadline) {
|
|
|
|
|
try {
|
|
|
|
|
const fd = fs.openSync(lockPath, 'wx');
|
|
|
|
|
try {
|
2026-11-19 20:22:01 +06:00
|
|
|
fs.writeFileSync(fd, lockValue, 'utf8');
|
2026-11-18 05:18:55 +06:00
|
|
|
} finally {
|
|
|
|
|
fs.closeSync(fd);
|
|
|
|
|
}
|
|
|
|
|
acquired = true;
|
|
|
|
|
break;
|
|
|
|
|
} catch (err: unknown) {
|
|
|
|
|
const code = (err as NodeJS.ErrnoException)?.code;
|
|
|
|
|
if (code !== 'EEXIST') {
|
|
|
|
|
throw err;
|
|
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
this.removeStaleLockAtomically(lockPath);
|
2026-11-18 05:18:55 +06:00
|
|
|
const jitter =
|
|
|
|
|
LOCK_RETRY_MIN_MS +
|
|
|
|
|
Math.floor(Math.random() * (LOCK_RETRY_MAX_MS - LOCK_RETRY_MIN_MS));
|
|
|
|
|
await sleepMs(jitter);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!acquired) {
|
|
|
|
|
throw new Error(`Timed out waiting for CSV partition lock: ${lockPath}`);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
return await fn();
|
|
|
|
|
} finally {
|
2026-11-19 20:22:01 +06:00
|
|
|
this.releaseLockIfOwned(lockPath, lockValue);
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
2026-11-19 00:42:38 +06:00
|
|
|
* Normalizes and validates any timestamp or date string into a strict YYYY-MM-DD partition key.
|
|
|
|
|
* Returns fallback (default: today's date) if unparseable, or null when fallback is null.
|
2026-11-18 05:18:55 +06:00
|
|
|
*/
|
2026-11-19 00:42:38 +06:00
|
|
|
public static extractSafePartitionDate(rawTimestamp?: string, fallback?: string): string;
|
|
|
|
|
public static extractSafePartitionDate(rawTimestamp: string | undefined, fallback: null): string | null;
|
|
|
|
|
public static extractSafePartitionDate(
|
|
|
|
|
rawTimestamp?: string,
|
|
|
|
|
fallback: string | null = new Date().toISOString().slice(0, 10),
|
|
|
|
|
): string | null {
|
2026-11-18 05:18:55 +06:00
|
|
|
if (!rawTimestamp || typeof rawTimestamp !== 'string') {
|
|
|
|
|
return fallback;
|
|
|
|
|
}
|
|
|
|
|
const trimmed = rawTimestamp.trim();
|
|
|
|
|
const prefix = trimmed.slice(0, 10);
|
|
|
|
|
if (/^\d{4}-\d{2}-\d{2}$/.test(prefix)) {
|
|
|
|
|
const parsedPrefix = new Date(`${prefix}T00:00:00.000Z`);
|
|
|
|
|
if (!Number.isNaN(parsedPrefix.getTime())) {
|
|
|
|
|
return prefix;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (/^\d+(\.\d+)?$/.test(trimmed)) {
|
|
|
|
|
const num = Number(trimmed);
|
|
|
|
|
const ms = num < 1e11 ? num * 1000 : num;
|
|
|
|
|
const d = new Date(ms);
|
|
|
|
|
if (!Number.isNaN(d.getTime())) {
|
|
|
|
|
return d.toISOString().slice(0, 10);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
const parsed = new Date(trimmed);
|
|
|
|
|
if (!Number.isNaN(parsed.getTime())) {
|
|
|
|
|
return parsed.toISOString().slice(0, 10);
|
|
|
|
|
}
|
|
|
|
|
return fallback;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Stateful RFC 4180 quote-aware chunked row counter so multi-100MB CSVs never load wholly into RAM
|
|
|
|
|
*/
|
|
|
|
|
private static createChunkRowCounter() {
|
2026-11-19 20:22:01 +06:00
|
|
|
const decoder = new StringDecoder('utf8');
|
2026-11-18 05:18:55 +06:00
|
|
|
let inQuotes = false;
|
|
|
|
|
let pendingQuote = false;
|
|
|
|
|
let currentLineHasChars = false;
|
|
|
|
|
let totalRows = 0;
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
const processStr = (str: string) => {
|
|
|
|
|
for (let i = 0; i < str.length; i += 1) {
|
|
|
|
|
const ch = str[i];
|
|
|
|
|
if (pendingQuote) {
|
|
|
|
|
pendingQuote = false;
|
2026-11-18 05:18:55 +06:00
|
|
|
if (ch === '"') {
|
2026-11-19 20:22:01 +06:00
|
|
|
// Escaped quote "" across chunk boundary or inline
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
inQuotes = !inQuotes;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (ch === '"') {
|
|
|
|
|
currentLineHasChars = true;
|
|
|
|
|
if (inQuotes) {
|
|
|
|
|
if (i + 1 < str.length) {
|
|
|
|
|
if (str[i + 1] === '"') {
|
|
|
|
|
i += 1;
|
2026-11-18 05:18:55 +06:00
|
|
|
} else {
|
2026-11-19 20:22:01 +06:00
|
|
|
inQuotes = false;
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
|
|
|
|
} else {
|
2026-11-19 20:22:01 +06:00
|
|
|
pendingQuote = true;
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
} else {
|
|
|
|
|
inQuotes = true;
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
} else if (ch === '\n' && !inQuotes) {
|
|
|
|
|
if (currentLineHasChars) {
|
|
|
|
|
totalRows += 1;
|
|
|
|
|
currentLineHasChars = false;
|
|
|
|
|
}
|
|
|
|
|
} else if (ch !== '\r' && ch !== ' ' && ch !== '\t') {
|
|
|
|
|
currentLineHasChars = true;
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
return {
|
|
|
|
|
pushChunk(chunk: Buffer | string) {
|
|
|
|
|
const str = typeof chunk === 'string' ? chunk : decoder.write(chunk);
|
|
|
|
|
if (str) processStr(str);
|
2026-11-18 05:18:55 +06:00
|
|
|
},
|
|
|
|
|
finish(): number {
|
2026-11-19 20:22:01 +06:00
|
|
|
const trailing = decoder.end();
|
|
|
|
|
if (trailing) processStr(trailing);
|
2026-11-18 05:18:55 +06:00
|
|
|
if (currentLineHasChars) {
|
|
|
|
|
totalRows += 1;
|
|
|
|
|
}
|
|
|
|
|
return Math.max(0, totalRows - 1);
|
|
|
|
|
},
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static countCsvDataRowsChunkedSync(filePath: string): number {
|
|
|
|
|
const counter = this.createChunkRowCounter();
|
2026-11-19 23:02:27 +06:00
|
|
|
if (filePath.endsWith('.gz')) {
|
|
|
|
|
const compressed = fs.readFileSync(filePath);
|
|
|
|
|
const decompressed = zlib.gunzipSync(compressed);
|
|
|
|
|
counter.pushChunk(decompressed);
|
|
|
|
|
return counter.finish();
|
|
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
const buf = Buffer.allocUnsafe(64 * 1024);
|
|
|
|
|
const fd = fs.openSync(filePath, 'r');
|
|
|
|
|
try {
|
|
|
|
|
let bytesRead = 0;
|
|
|
|
|
while ((bytesRead = fs.readSync(fd, buf, 0, buf.length, null)) > 0) {
|
|
|
|
|
counter.pushChunk(buf.subarray(0, bytesRead));
|
|
|
|
|
}
|
|
|
|
|
return counter.finish();
|
|
|
|
|
} finally {
|
|
|
|
|
fs.closeSync(fd);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 23:02:27 +06:00
|
|
|
public static openPartitionTextStream(filePath: string): Readable {
|
|
|
|
|
const fileStream = fs.createReadStream(filePath, {
|
|
|
|
|
highWaterMark: 64 * 1024,
|
|
|
|
|
});
|
|
|
|
|
if (!filePath.endsWith('.gz')) {
|
|
|
|
|
return fileStream;
|
|
|
|
|
}
|
|
|
|
|
const gunzip = zlib.createGunzip();
|
|
|
|
|
fileStream.on('error', (err) => {
|
|
|
|
|
if (!gunzip.destroyed) {
|
|
|
|
|
gunzip.destroy(err);
|
|
|
|
|
}
|
2026-11-21 22:15:32 +06:00
|
|
|
if (!fileStream.destroyed) {
|
|
|
|
|
fileStream.destroy();
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
gunzip.on('error', () => {
|
|
|
|
|
if (!fileStream.destroyed) {
|
|
|
|
|
fileStream.destroy();
|
|
|
|
|
}
|
2026-11-19 23:02:27 +06:00
|
|
|
});
|
|
|
|
|
gunzip.on('close', () => {
|
|
|
|
|
if (!fileStream.destroyed) {
|
|
|
|
|
fileStream.destroy();
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
fileStream.pipe(gunzip);
|
|
|
|
|
return gunzip;
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-21 22:15:32 +06:00
|
|
|
public static async countCsvDataRowsStreamAsync(
|
2026-11-18 05:18:55 +06:00
|
|
|
filePath: string,
|
|
|
|
|
): Promise<number> {
|
|
|
|
|
const counter = this.createChunkRowCounter();
|
2026-11-21 22:15:32 +06:00
|
|
|
const stream = this.openPartitionTextStream(filePath);
|
|
|
|
|
try {
|
|
|
|
|
for await (const chunk of stream) {
|
|
|
|
|
counter.pushChunk(chunk as Buffer | string);
|
|
|
|
|
}
|
|
|
|
|
return counter.finish();
|
|
|
|
|
} finally {
|
|
|
|
|
if (!stream.destroyed) {
|
|
|
|
|
stream.destroy();
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Resolves target CSV file path with daily and configured MB rotation rules
|
2026-10-23 11:43:49 -05:00
|
|
|
*/
|
2026-11-19 20:22:01 +06:00
|
|
|
public static getTargetFilePath(
|
|
|
|
|
clientId: string,
|
|
|
|
|
dateStr?: string,
|
|
|
|
|
incomingBytes = 0,
|
|
|
|
|
): string {
|
2026-10-23 11:43:49 -05:00
|
|
|
const clientDir = this.ensureClientDir(clientId);
|
2026-11-18 05:18:55 +06:00
|
|
|
const date = this.extractSafePartitionDate(dateStr);
|
2026-10-23 11:43:49 -05:00
|
|
|
let baseFileName = `${date}.csv`;
|
|
|
|
|
let targetPath = path.join(clientDir, baseFileName);
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
// Check for file rotation
|
2026-10-23 11:43:49 -05:00
|
|
|
let partIndex = 1;
|
|
|
|
|
while (fs.existsSync(targetPath)) {
|
2026-11-19 20:22:01 +06:00
|
|
|
let stats: fs.Stats;
|
2026-10-23 11:43:49 -05:00
|
|
|
try {
|
2026-11-19 20:22:01 +06:00
|
|
|
stats = fs.statSync(targetPath);
|
|
|
|
|
} catch (err: unknown) {
|
|
|
|
|
if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') {
|
2026-10-23 11:43:49 -05:00
|
|
|
break;
|
|
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
throw err;
|
|
|
|
|
}
|
|
|
|
|
const overflowsLimit =
|
|
|
|
|
stats.size > 0 &&
|
|
|
|
|
incomingBytes > 0 &&
|
|
|
|
|
stats.size + incomingBytes > this.maxFileSizeBytes;
|
|
|
|
|
if (stats.size >= this.maxFileSizeBytes || overflowsLimit) {
|
|
|
|
|
partIndex += 1;
|
|
|
|
|
baseFileName = `${date}-part${partIndex}.csv`;
|
|
|
|
|
targetPath = path.join(clientDir, baseFileName);
|
|
|
|
|
} else {
|
2026-10-23 11:43:49 -05:00
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return targetPath;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
2026-11-18 05:18:55 +06:00
|
|
|
* Batch append events into target CSV file under cooperative partition lock.
|
2026-10-23 11:43:49 -05:00
|
|
|
* If file doesn't exist, writes header: clientId,eventId,timestamp,payload
|
|
|
|
|
*/
|
2026-11-18 05:18:55 +06:00
|
|
|
public static async appendBatch(
|
|
|
|
|
events: DataEvent[],
|
|
|
|
|
): Promise<{ written: number; filesUpdated: string[] }> {
|
2026-10-23 11:43:49 -05:00
|
|
|
if (!events || events.length === 0) {
|
|
|
|
|
return { written: 0, filesUpdated: [] };
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
// Group events by client + date without ambiguous delimiter splitting
|
|
|
|
|
const grouped = new Map<string, { clientId: string; date: string; events: DataEvent[] }>();
|
2026-10-23 11:43:49 -05:00
|
|
|
for (const evt of events) {
|
2026-11-18 05:18:55 +06:00
|
|
|
const date = this.extractSafePartitionDate(evt.timestamp);
|
2026-11-19 20:22:01 +06:00
|
|
|
const clientId = String(evt.clientId || 'unknown');
|
|
|
|
|
const key = JSON.stringify([clientId, date]);
|
|
|
|
|
let bucket = grouped.get(key);
|
|
|
|
|
if (!bucket) {
|
|
|
|
|
bucket = { clientId, date, events: [] };
|
|
|
|
|
grouped.set(key, bucket);
|
2026-10-23 11:43:49 -05:00
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
bucket.events.push(evt);
|
2026-10-23 11:43:49 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const filesUpdated: string[] = [];
|
|
|
|
|
const fields = ['clientId', 'eventId', 'timestamp', 'payload'];
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
for (const { clientId, date, events: batch } of grouped.values()) {
|
|
|
|
|
const cleanBatch = batch.map((item) => ({
|
|
|
|
|
clientId: item.clientId,
|
|
|
|
|
eventId: item.eventId,
|
|
|
|
|
timestamp: item.timestamp,
|
|
|
|
|
payload:
|
|
|
|
|
typeof item.payload === 'string'
|
|
|
|
|
? item.payload
|
|
|
|
|
: JSON.stringify(item.payload),
|
|
|
|
|
}));
|
|
|
|
|
|
|
|
|
|
const bodyCsvNoHeader = new Parser({ fields, header: false, eol: '\n' }).parse(
|
|
|
|
|
cleanBatch,
|
|
|
|
|
);
|
|
|
|
|
const incomingBytes = Buffer.byteLength(bodyCsvNoHeader, 'utf8') + 1;
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
const targetPath = await this.withPartitionLock(clientId, date, () => {
|
2026-11-19 20:22:01 +06:00
|
|
|
const resolvedPath = this.getTargetFilePath(clientId, date, incomingBytes);
|
|
|
|
|
let sizeBytes = 0;
|
2026-11-18 05:18:55 +06:00
|
|
|
try {
|
2026-11-19 20:22:01 +06:00
|
|
|
if (fs.existsSync(resolvedPath)) {
|
|
|
|
|
sizeBytes = fs.statSync(resolvedPath).size;
|
|
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
} catch {
|
2026-11-19 20:22:01 +06:00
|
|
|
sizeBytes = 0;
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
const hasContent = sizeBytes > 0;
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
if (hasContent) {
|
2026-11-19 20:22:01 +06:00
|
|
|
const endsWithNl = this.fileEndsWithNewline(resolvedPath, sizeBytes);
|
|
|
|
|
fs.appendFileSync(
|
|
|
|
|
resolvedPath,
|
|
|
|
|
(endsWithNl ? '' : '\n') + bodyCsvNoHeader,
|
|
|
|
|
'utf8',
|
|
|
|
|
);
|
2026-11-18 05:18:55 +06:00
|
|
|
} else {
|
2026-11-19 20:22:01 +06:00
|
|
|
const fullCsv = new Parser({ fields, header: true, eol: '\n' }).parse(
|
|
|
|
|
cleanBatch,
|
|
|
|
|
);
|
|
|
|
|
fs.writeFileSync(resolvedPath, fullCsv, 'utf8');
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
|
|
|
|
|
2026-11-19 23:02:27 +06:00
|
|
|
// Invalidate row & quota caches for modified file immediately
|
2026-11-18 05:18:55 +06:00
|
|
|
this.rowCountCache.delete(resolvedPath);
|
2026-11-19 23:02:27 +06:00
|
|
|
this.clearQuotaCaches(clientId);
|
2026-11-18 05:18:55 +06:00
|
|
|
return resolvedPath;
|
|
|
|
|
});
|
2026-10-23 11:43:49 -05:00
|
|
|
|
|
|
|
|
filesUpdated.push(targetPath);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return { written: events.length, filesUpdated };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
2026-11-19 23:02:27 +06:00
|
|
|
* List all stored CSV and gzipped CSV (.csv / .csv.gz) files with cached row counts
|
2026-10-23 11:43:49 -05:00
|
|
|
*/
|
|
|
|
|
public static listAllFiles(): StorageFileMeta[] {
|
|
|
|
|
const results: StorageFileMeta[] = [];
|
|
|
|
|
if (!fs.existsSync(this.baseStorageDir)) {
|
2026-11-18 05:18:55 +06:00
|
|
|
this.rowCountCache.clear();
|
2026-10-23 11:43:49 -05:00
|
|
|
return results;
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
const activePaths = new Set<string>();
|
2026-10-23 11:43:49 -05:00
|
|
|
try {
|
|
|
|
|
const clientDirs = fs.readdirSync(this.baseStorageDir);
|
|
|
|
|
for (const clientDir of clientDirs) {
|
|
|
|
|
const fullClientPath = path.join(this.baseStorageDir, clientDir);
|
2026-11-19 20:22:01 +06:00
|
|
|
let dirLstat: fs.Stats;
|
|
|
|
|
try {
|
|
|
|
|
dirLstat = fs.lstatSync(fullClientPath);
|
|
|
|
|
} catch {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
if (dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
|
2026-10-23 11:43:49 -05:00
|
|
|
|
|
|
|
|
const files = fs.readdirSync(fullClientPath);
|
|
|
|
|
for (const file of files) {
|
2026-11-19 23:02:27 +06:00
|
|
|
if (!this.isPartitionFilename(file)) continue;
|
2026-10-23 11:43:49 -05:00
|
|
|
const fullFilePath = path.join(fullClientPath, file);
|
2026-11-19 20:22:01 +06:00
|
|
|
let stats: fs.Stats;
|
|
|
|
|
try {
|
|
|
|
|
stats = fs.lstatSync(fullFilePath);
|
|
|
|
|
} catch {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
if (stats.isSymbolicLink() || !stats.isFile()) continue;
|
2026-11-18 05:18:55 +06:00
|
|
|
activePaths.add(fullFilePath);
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
const cached = this.rowCountCache.get(fullFilePath);
|
2026-10-23 11:43:49 -05:00
|
|
|
let linesCount = 0;
|
2026-11-18 05:18:55 +06:00
|
|
|
if (
|
|
|
|
|
cached &&
|
|
|
|
|
cached.sizeBytes === stats.size &&
|
|
|
|
|
cached.mtimeMs === stats.mtimeMs
|
|
|
|
|
) {
|
|
|
|
|
linesCount = cached.linesCount;
|
|
|
|
|
} else {
|
|
|
|
|
try {
|
|
|
|
|
linesCount = this.countCsvDataRowsChunkedSync(fullFilePath);
|
|
|
|
|
this.rowCountCache.set(fullFilePath, {
|
|
|
|
|
sizeBytes: stats.size,
|
|
|
|
|
mtimeMs: stats.mtimeMs,
|
|
|
|
|
linesCount,
|
|
|
|
|
});
|
|
|
|
|
} catch {
|
|
|
|
|
linesCount = 0;
|
|
|
|
|
}
|
2026-10-23 11:43:49 -05:00
|
|
|
}
|
|
|
|
|
|
2026-11-19 23:02:27 +06:00
|
|
|
const datePart = this.extractDateFromFilename(file);
|
2026-10-23 11:43:49 -05:00
|
|
|
|
|
|
|
|
results.push({
|
|
|
|
|
clientId: clientDir,
|
|
|
|
|
date: datePart,
|
|
|
|
|
filename: file,
|
|
|
|
|
filePath: fullFilePath,
|
|
|
|
|
sizeBytes: stats.size,
|
|
|
|
|
linesCount,
|
|
|
|
|
modifiedAt: stats.mtime,
|
2026-11-19 23:02:27 +06:00
|
|
|
compressed: file.endsWith('.csv.gz'),
|
2026-10-23 11:43:49 -05:00
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
|
|
|
|
|
for (const key of this.rowCountCache.keys()) {
|
|
|
|
|
if (!activePaths.has(key)) {
|
|
|
|
|
this.rowCountCache.delete(key);
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-10-23 11:43:49 -05:00
|
|
|
} catch {
|
|
|
|
|
// Return whatever collected
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return results.sort((a, b) => b.modifiedAt.getTime() - a.modifiedAt.getTime());
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-17 20:05:00 +06:00
|
|
|
/**
|
2026-11-18 05:18:55 +06:00
|
|
|
* Non-blocking async listing using fs.promises + streaming row counting on cache misses
|
|
|
|
|
*/
|
|
|
|
|
public static async listAllFilesAsync(): Promise<StorageFileMeta[]> {
|
|
|
|
|
const results: StorageFileMeta[] = [];
|
|
|
|
|
if (!fs.existsSync(this.baseStorageDir)) {
|
|
|
|
|
this.rowCountCache.clear();
|
|
|
|
|
return results;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const activePaths = new Set<string>();
|
|
|
|
|
try {
|
|
|
|
|
const clientDirs = await fs.promises.readdir(this.baseStorageDir);
|
|
|
|
|
for (const clientDir of clientDirs) {
|
|
|
|
|
const fullClientPath = path.join(this.baseStorageDir, clientDir);
|
2026-11-19 20:22:01 +06:00
|
|
|
const dirLstat = await fs.promises.lstat(fullClientPath).catch(() => null);
|
|
|
|
|
if (!dirLstat || dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
|
2026-11-18 05:18:55 +06:00
|
|
|
|
|
|
|
|
const files = await fs.promises.readdir(fullClientPath);
|
|
|
|
|
for (const file of files) {
|
2026-11-19 23:02:27 +06:00
|
|
|
if (!this.isPartitionFilename(file)) continue;
|
2026-11-18 05:18:55 +06:00
|
|
|
const fullFilePath = path.join(fullClientPath, file);
|
2026-11-19 20:22:01 +06:00
|
|
|
const stats = await fs.promises.lstat(fullFilePath).catch(() => null);
|
|
|
|
|
if (!stats || stats.isSymbolicLink() || !stats.isFile()) continue;
|
2026-11-18 05:18:55 +06:00
|
|
|
activePaths.add(fullFilePath);
|
|
|
|
|
|
|
|
|
|
const cached = this.rowCountCache.get(fullFilePath);
|
|
|
|
|
let linesCount = 0;
|
|
|
|
|
if (
|
|
|
|
|
cached &&
|
|
|
|
|
cached.sizeBytes === stats.size &&
|
|
|
|
|
cached.mtimeMs === stats.mtimeMs
|
|
|
|
|
) {
|
|
|
|
|
linesCount = cached.linesCount;
|
|
|
|
|
} else {
|
|
|
|
|
try {
|
|
|
|
|
linesCount = await this.countCsvDataRowsStreamAsync(fullFilePath);
|
|
|
|
|
this.rowCountCache.set(fullFilePath, {
|
|
|
|
|
sizeBytes: stats.size,
|
|
|
|
|
mtimeMs: stats.mtimeMs,
|
|
|
|
|
linesCount,
|
|
|
|
|
});
|
|
|
|
|
} catch {
|
|
|
|
|
linesCount = 0;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 23:02:27 +06:00
|
|
|
const datePart = this.extractDateFromFilename(file);
|
2026-11-18 05:18:55 +06:00
|
|
|
|
|
|
|
|
results.push({
|
|
|
|
|
clientId: clientDir,
|
|
|
|
|
date: datePart,
|
|
|
|
|
filename: file,
|
|
|
|
|
filePath: fullFilePath,
|
|
|
|
|
sizeBytes: stats.size,
|
|
|
|
|
linesCount,
|
|
|
|
|
modifiedAt: stats.mtime,
|
2026-11-19 23:02:27 +06:00
|
|
|
compressed: file.endsWith('.csv.gz'),
|
2026-11-18 05:18:55 +06:00
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (const key of this.rowCountCache.keys()) {
|
|
|
|
|
if (!activePaths.has(key)) {
|
|
|
|
|
this.rowCountCache.delete(key);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
// Return whatever collected
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return results.sort((a, b) => b.modifiedAt.getTime() - a.modifiedAt.getTime());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
2026-11-19 23:02:27 +06:00
|
|
|
* Compress uncompressed .csv partitions older than compressAfterDays into .csv.gz under cooperative partition locks
|
|
|
|
|
*/
|
|
|
|
|
public static async compressOldPartitionsAsync(
|
|
|
|
|
compressAfterDays: number,
|
|
|
|
|
): Promise<number> {
|
|
|
|
|
if (!Number.isFinite(compressAfterDays) || compressAfterDays <= 0) {
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const baseStat = await fs.promises.lstat(this.baseStorageDir).catch(() => null);
|
|
|
|
|
if (!baseStat || !baseStat.isDirectory()) {
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const cutoffDate = new Date(
|
|
|
|
|
Date.now() - Math.floor(compressAfterDays) * 86_400_000,
|
|
|
|
|
)
|
|
|
|
|
.toISOString()
|
|
|
|
|
.split('T')[0];
|
|
|
|
|
|
|
|
|
|
let compressedCount = 0;
|
|
|
|
|
try {
|
|
|
|
|
const clientDirs = await fs.promises.readdir(this.baseStorageDir);
|
|
|
|
|
for (const clientDir of clientDirs) {
|
|
|
|
|
const fullClientPath = path.join(this.baseStorageDir, clientDir);
|
|
|
|
|
const dirLstat = await fs.promises.lstat(fullClientPath).catch(() => null);
|
|
|
|
|
if (!dirLstat || dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const files = await fs.promises
|
|
|
|
|
.readdir(fullClientPath)
|
|
|
|
|
.catch(() => [] as string[]);
|
|
|
|
|
for (const file of files) {
|
|
|
|
|
if (!file.endsWith('.csv')) continue;
|
|
|
|
|
const datePart = this.extractDateFromFilename(file);
|
|
|
|
|
if (!(/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate)) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
const fullPath = path.join(fullClientPath, file);
|
|
|
|
|
const fileLstat = await fs.promises.lstat(fullPath).catch(() => null);
|
|
|
|
|
if (!fileLstat || fileLstat.isSymbolicLink() || !fileLstat.isFile()) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
|
|
|
|
|
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
|
|
|
|
|
let acquired = false;
|
|
|
|
|
const tryAcquireLock = async (): Promise<boolean> => {
|
|
|
|
|
const handle = await fs.promises.open(lockPath, 'wx');
|
|
|
|
|
try {
|
|
|
|
|
await handle.writeFile(lockValue, 'utf8');
|
|
|
|
|
} finally {
|
|
|
|
|
await handle.close();
|
|
|
|
|
}
|
|
|
|
|
return true;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
acquired = await tryAcquireLock();
|
|
|
|
|
} catch (err: unknown) {
|
|
|
|
|
if ((err as NodeJS.ErrnoException)?.code === 'EEXIST') {
|
|
|
|
|
await this.removeStaleLockAtomicallyAsync(lockPath);
|
|
|
|
|
try {
|
|
|
|
|
acquired = await tryAcquireLock();
|
|
|
|
|
} catch {
|
|
|
|
|
acquired = false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!acquired) continue;
|
|
|
|
|
|
|
|
|
|
let targetGzPath = `${fullPath}.gz`;
|
|
|
|
|
if (fs.existsSync(targetGzPath)) {
|
|
|
|
|
const stem = fullPath.replace(/\.csv$/, '');
|
|
|
|
|
let suffix = 2;
|
|
|
|
|
while (fs.existsSync(`${stem}-gz${suffix}.csv.gz`)) {
|
|
|
|
|
suffix += 1;
|
|
|
|
|
}
|
|
|
|
|
targetGzPath = `${stem}-gz${suffix}.csv.gz`;
|
|
|
|
|
}
|
|
|
|
|
const tmpGzPath = `${targetGzPath}.tmp.${crypto.randomUUID()}`;
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
await pipeline(
|
|
|
|
|
fs.createReadStream(fullPath, { highWaterMark: 64 * 1024 }),
|
|
|
|
|
zlib.createGzip({ level: 6 }),
|
|
|
|
|
fs.createWriteStream(tmpGzPath),
|
|
|
|
|
);
|
|
|
|
|
await fs.promises
|
|
|
|
|
.utimes(tmpGzPath, fileLstat.atime, fileLstat.mtime)
|
|
|
|
|
.catch(() => undefined);
|
|
|
|
|
await fs.promises.rename(tmpGzPath, targetGzPath);
|
|
|
|
|
await fs.promises.unlink(fullPath);
|
|
|
|
|
this.rowCountCache.delete(fullPath);
|
|
|
|
|
this.rowCountCache.delete(targetGzPath);
|
|
|
|
|
this.clearQuotaCaches(clientDir);
|
|
|
|
|
compressedCount += 1;
|
|
|
|
|
} catch {
|
|
|
|
|
await fs.promises.unlink(tmpGzPath).catch(() => undefined);
|
|
|
|
|
} finally {
|
|
|
|
|
await this.releaseLockIfOwnedAsync(lockPath, lockValue);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
// Ignore directory scan failure
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return compressedCount;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Prune CSV / CSV.GZ partitions older than autoPruneDays under cooperative partition locks
|
2026-11-17 20:05:00 +06:00
|
|
|
*/
|
|
|
|
|
public static pruneExpiredPartitions(autoPruneDays: number): number {
|
|
|
|
|
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
if (!fs.existsSync(this.baseStorageDir)) {
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
2026-11-17 20:05:00 +06:00
|
|
|
|
|
|
|
|
const cutoffDate = new Date(
|
|
|
|
|
Date.now() - Math.floor(autoPruneDays) * 86_400_000,
|
|
|
|
|
)
|
|
|
|
|
.toISOString()
|
|
|
|
|
.split('T')[0];
|
|
|
|
|
|
|
|
|
|
let prunedCount = 0;
|
2026-11-18 05:18:55 +06:00
|
|
|
try {
|
|
|
|
|
const clientDirs = fs.readdirSync(this.baseStorageDir);
|
|
|
|
|
for (const clientDir of clientDirs) {
|
|
|
|
|
const fullClientPath = path.join(this.baseStorageDir, clientDir);
|
2026-11-19 20:22:01 +06:00
|
|
|
let dirLstat: fs.Stats;
|
|
|
|
|
try {
|
|
|
|
|
dirLstat = fs.lstatSync(fullClientPath);
|
|
|
|
|
} catch {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
if (dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
|
2026-11-18 05:18:55 +06:00
|
|
|
|
|
|
|
|
const files = fs.readdirSync(fullClientPath);
|
|
|
|
|
for (const file of files) {
|
2026-11-19 23:02:27 +06:00
|
|
|
if (!this.isPartitionFilename(file)) continue;
|
|
|
|
|
const datePart = this.extractDateFromFilename(file);
|
2026-11-18 05:18:55 +06:00
|
|
|
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
|
|
|
|
|
const fullPath = path.join(fullClientPath, file);
|
2026-11-19 20:22:01 +06:00
|
|
|
let fileLstat: fs.Stats;
|
|
|
|
|
try {
|
|
|
|
|
fileLstat = fs.lstatSync(fullPath);
|
|
|
|
|
} catch {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
if (fileLstat.isSymbolicLink() || !fileLstat.isFile()) continue;
|
|
|
|
|
|
|
|
|
|
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
|
|
|
|
|
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
|
|
|
|
|
let acquired = false;
|
|
|
|
|
try {
|
|
|
|
|
const fd = fs.openSync(lockPath, 'wx');
|
|
|
|
|
try {
|
|
|
|
|
fs.writeFileSync(fd, lockValue, 'utf8');
|
|
|
|
|
} finally {
|
|
|
|
|
fs.closeSync(fd);
|
|
|
|
|
}
|
|
|
|
|
acquired = true;
|
|
|
|
|
} catch (err: unknown) {
|
|
|
|
|
if ((err as NodeJS.ErrnoException)?.code === 'EEXIST') {
|
|
|
|
|
this.removeStaleLockAtomically(lockPath);
|
|
|
|
|
try {
|
|
|
|
|
const fd = fs.openSync(lockPath, 'wx');
|
|
|
|
|
try {
|
|
|
|
|
fs.writeFileSync(fd, lockValue, 'utf8');
|
|
|
|
|
} finally {
|
|
|
|
|
fs.closeSync(fd);
|
|
|
|
|
}
|
|
|
|
|
acquired = true;
|
|
|
|
|
} catch {
|
|
|
|
|
// Partition is currently locked by an active writer; skip safely
|
|
|
|
|
acquired = false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!acquired) continue;
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
try {
|
|
|
|
|
if (fs.existsSync(fullPath)) {
|
|
|
|
|
fs.unlinkSync(fullPath);
|
|
|
|
|
this.rowCountCache.delete(fullPath);
|
2026-11-19 23:02:27 +06:00
|
|
|
this.clearQuotaCaches(clientDir);
|
2026-11-18 05:18:55 +06:00
|
|
|
prunedCount += 1;
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
// Ignore file removal error
|
2026-11-19 20:22:01 +06:00
|
|
|
} finally {
|
|
|
|
|
this.releaseLockIfOwned(lockPath, lockValue);
|
2026-11-18 05:18:55 +06:00
|
|
|
}
|
2026-11-17 20:05:00 +06:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-11-18 05:18:55 +06:00
|
|
|
} catch {
|
|
|
|
|
// Ignore directory scan failure
|
2026-11-17 20:05:00 +06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return prunedCount;
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
/**
|
2026-11-19 23:02:27 +06:00
|
|
|
* Archive and/or prune expired partitions under cooperative partition locks.
|
|
|
|
|
* Never deletes a local partition if options.beforeDeleteFile throws or rejects (V2-E02).
|
2026-11-19 20:22:01 +06:00
|
|
|
*/
|
2026-11-19 23:02:27 +06:00
|
|
|
public static async pruneOrArchiveExpiredPartitionsAsync(
|
2026-11-19 20:22:01 +06:00
|
|
|
autoPruneDays: number,
|
2026-11-19 23:02:27 +06:00
|
|
|
options?: {
|
|
|
|
|
beforeDeleteFile?: (file: {
|
|
|
|
|
clientId: string;
|
|
|
|
|
filename: string;
|
|
|
|
|
filePath: string;
|
|
|
|
|
date: string;
|
|
|
|
|
}) => Promise<void>;
|
|
|
|
|
archiveOnlyWithoutDelete?: boolean;
|
|
|
|
|
maxCutoffDays?: number;
|
|
|
|
|
},
|
|
|
|
|
): Promise<{
|
|
|
|
|
scannedCount: number;
|
|
|
|
|
archivedCount: number;
|
|
|
|
|
prunedCount: number;
|
|
|
|
|
errors: string[];
|
|
|
|
|
}> {
|
|
|
|
|
const targetDays =
|
|
|
|
|
options?.maxCutoffDays !== undefined
|
|
|
|
|
? options.maxCutoffDays
|
|
|
|
|
: autoPruneDays;
|
|
|
|
|
if (!Number.isFinite(targetDays) || targetDays < 0) {
|
|
|
|
|
return { scannedCount: 0, archivedCount: 0, prunedCount: 0, errors: [] };
|
|
|
|
|
}
|
|
|
|
|
if (options?.maxCutoffDays === undefined && targetDays <= 0) {
|
|
|
|
|
return { scannedCount: 0, archivedCount: 0, prunedCount: 0, errors: [] };
|
2026-11-19 20:22:01 +06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const baseStat = await fs.promises.lstat(this.baseStorageDir).catch(() => null);
|
|
|
|
|
if (!baseStat || !baseStat.isDirectory()) {
|
2026-11-19 23:02:27 +06:00
|
|
|
return { scannedCount: 0, archivedCount: 0, prunedCount: 0, errors: [] };
|
2026-11-19 20:22:01 +06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const cutoffDate = new Date(
|
2026-11-19 23:02:27 +06:00
|
|
|
Date.now() - Math.floor(targetDays) * 86_400_000,
|
2026-11-19 20:22:01 +06:00
|
|
|
)
|
|
|
|
|
.toISOString()
|
|
|
|
|
.split('T')[0];
|
|
|
|
|
|
2026-11-19 23:02:27 +06:00
|
|
|
let scannedCount = 0;
|
|
|
|
|
let archivedCount = 0;
|
2026-11-19 20:22:01 +06:00
|
|
|
let prunedCount = 0;
|
2026-11-19 23:02:27 +06:00
|
|
|
const errors: string[] = [];
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
try {
|
|
|
|
|
const clientDirs = await fs.promises.readdir(this.baseStorageDir);
|
|
|
|
|
for (const clientDir of clientDirs) {
|
|
|
|
|
const fullClientPath = path.join(this.baseStorageDir, clientDir);
|
|
|
|
|
const dirLstat = await fs.promises.lstat(fullClientPath).catch(() => null);
|
|
|
|
|
if (!dirLstat || dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 23:02:27 +06:00
|
|
|
const files = await fs.promises
|
|
|
|
|
.readdir(fullClientPath)
|
|
|
|
|
.catch(() => [] as string[]);
|
2026-11-19 20:22:01 +06:00
|
|
|
for (const file of files) {
|
2026-11-19 23:02:27 +06:00
|
|
|
if (!this.isPartitionFilename(file)) continue;
|
|
|
|
|
const datePart = this.extractDateFromFilename(file);
|
|
|
|
|
if (!(/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate)) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
const fullPath = path.join(fullClientPath, file);
|
|
|
|
|
const fileLstat = await fs.promises.lstat(fullPath).catch(() => null);
|
|
|
|
|
if (!fileLstat || fileLstat.isSymbolicLink() || !fileLstat.isFile()) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
scannedCount += 1;
|
2026-11-19 20:22:01 +06:00
|
|
|
|
2026-11-19 23:02:27 +06:00
|
|
|
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
|
|
|
|
|
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
|
|
|
|
|
let acquired = false;
|
|
|
|
|
const tryAcquireLock = async (): Promise<boolean> => {
|
|
|
|
|
const handle = await fs.promises.open(lockPath, 'wx');
|
|
|
|
|
try {
|
|
|
|
|
await handle.writeFile(lockValue, 'utf8');
|
|
|
|
|
} finally {
|
|
|
|
|
await handle.close();
|
|
|
|
|
}
|
|
|
|
|
return true;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
acquired = await tryAcquireLock();
|
|
|
|
|
} catch (err: unknown) {
|
|
|
|
|
if ((err as NodeJS.ErrnoException)?.code === 'EEXIST') {
|
|
|
|
|
await this.removeStaleLockAtomicallyAsync(lockPath);
|
2026-11-19 20:22:01 +06:00
|
|
|
try {
|
2026-11-19 23:02:27 +06:00
|
|
|
acquired = await tryAcquireLock();
|
|
|
|
|
} catch {
|
|
|
|
|
acquired = false;
|
2026-11-19 20:22:01 +06:00
|
|
|
}
|
|
|
|
|
}
|
2026-11-19 23:02:27 +06:00
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
|
2026-11-19 23:02:27 +06:00
|
|
|
if (!acquired) continue;
|
2026-11-19 20:22:01 +06:00
|
|
|
|
2026-11-19 23:02:27 +06:00
|
|
|
try {
|
|
|
|
|
if (options?.beforeDeleteFile) {
|
|
|
|
|
await options.beforeDeleteFile({
|
|
|
|
|
clientId: clientDir,
|
|
|
|
|
filename: file,
|
|
|
|
|
filePath: fullPath,
|
|
|
|
|
date: datePart,
|
|
|
|
|
});
|
|
|
|
|
archivedCount += 1;
|
|
|
|
|
}
|
|
|
|
|
if (!options?.archiveOnlyWithoutDelete) {
|
2026-11-19 20:22:01 +06:00
|
|
|
await fs.promises.unlink(fullPath);
|
|
|
|
|
this.rowCountCache.delete(fullPath);
|
2026-11-19 23:02:27 +06:00
|
|
|
this.clearQuotaCaches(clientDir);
|
2026-11-19 20:22:01 +06:00
|
|
|
prunedCount += 1;
|
|
|
|
|
}
|
2026-11-19 23:02:27 +06:00
|
|
|
} catch (err: unknown) {
|
|
|
|
|
const msg = err instanceof Error ? err.message : 'Archive/prune error';
|
|
|
|
|
errors.push(`${clientDir}/${file}: ${msg}`);
|
|
|
|
|
} finally {
|
|
|
|
|
await this.releaseLockIfOwnedAsync(lockPath, lockValue);
|
2026-11-19 20:22:01 +06:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
// Ignore directory scan failure
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 23:02:27 +06:00
|
|
|
return { scannedCount, archivedCount, prunedCount, errors };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Non-blocking async partition pruner using fs.promises under cooperative partition locks
|
|
|
|
|
*/
|
|
|
|
|
public static async pruneExpiredPartitionsAsync(
|
|
|
|
|
autoPruneDays: number,
|
|
|
|
|
): Promise<number> {
|
|
|
|
|
const res = await this.pruneOrArchiveExpiredPartitionsAsync(autoPruneDays);
|
|
|
|
|
return res.prunedCount;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Cached VPS filesystem disk usage snapshot (30s TTL) for disk-full protection
|
|
|
|
|
*/
|
|
|
|
|
public static async getDiskUsageAsync(): Promise<DiskUsageSnapshot> {
|
|
|
|
|
const now = Date.now();
|
|
|
|
|
if (
|
|
|
|
|
this.diskUsageCache &&
|
|
|
|
|
now - this.diskUsageCache.checkedAt < QUOTA_CACHE_TTL_MS
|
|
|
|
|
) {
|
|
|
|
|
return this.diskUsageCache;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let probeDir = path.resolve(this.baseStorageDir);
|
|
|
|
|
while (!fs.existsSync(probeDir)) {
|
|
|
|
|
const parent = path.dirname(probeDir);
|
|
|
|
|
if (parent === probeDir) break;
|
|
|
|
|
probeDir = parent;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
const statfsRes = this.statfsOverrideForTests
|
|
|
|
|
? await this.statfsOverrideForTests(probeDir)
|
|
|
|
|
: await fs.promises.statfs(probeDir);
|
|
|
|
|
const bsize = Math.max(1, Number(statfsRes.bsize) || 4096);
|
|
|
|
|
const totalBytes = Math.max(0, Number(statfsRes.blocks) * bsize);
|
|
|
|
|
const availableBytes = Math.max(0, Number(statfsRes.bavail) * bsize);
|
|
|
|
|
const usedBytes = Math.max(0, totalBytes - availableBytes);
|
|
|
|
|
const usedPct =
|
|
|
|
|
totalBytes > 0
|
|
|
|
|
? Number(
|
|
|
|
|
Math.min(100, Math.max(0, (usedBytes / totalBytes) * 100)).toFixed(2),
|
|
|
|
|
)
|
|
|
|
|
: 0;
|
|
|
|
|
const snapshot: DiskUsageSnapshot = {
|
2026-11-21 00:06:55 +06:00
|
|
|
available: totalBytes > 0,
|
2026-11-19 23:02:27 +06:00
|
|
|
usedPct,
|
|
|
|
|
availableBytes,
|
2026-11-21 00:06:55 +06:00
|
|
|
freeBytes: availableBytes,
|
2026-11-19 23:02:27 +06:00
|
|
|
totalBytes,
|
|
|
|
|
checkedAt: now,
|
|
|
|
|
};
|
|
|
|
|
this.diskUsageCache = snapshot;
|
|
|
|
|
return snapshot;
|
|
|
|
|
} catch {
|
|
|
|
|
const fallback: DiskUsageSnapshot = {
|
2026-11-21 00:06:55 +06:00
|
|
|
available: false,
|
2026-11-19 23:02:27 +06:00
|
|
|
usedPct: 0,
|
|
|
|
|
availableBytes: 0,
|
2026-11-21 00:06:55 +06:00
|
|
|
freeBytes: 0,
|
2026-11-19 23:02:27 +06:00
|
|
|
totalBytes: 0,
|
|
|
|
|
checkedAt: now,
|
|
|
|
|
};
|
|
|
|
|
this.diskUsageCache = fallback;
|
|
|
|
|
return fallback;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Cached per-client directory byte count (30s TTL)
|
|
|
|
|
*/
|
|
|
|
|
public static async getClientStorageBytesAsync(
|
|
|
|
|
clientId: string,
|
|
|
|
|
): Promise<number> {
|
|
|
|
|
const safeClientId = this.sanitizeClientId(clientId);
|
|
|
|
|
const now = Date.now();
|
|
|
|
|
const cached = this.clientBytesCache.get(safeClientId);
|
|
|
|
|
if (cached && now - cached.checkedAt < QUOTA_CACHE_TTL_MS) {
|
|
|
|
|
return cached.bytes;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const resolvedBase = path.resolve(this.baseStorageDir);
|
|
|
|
|
const clientDir = path.resolve(resolvedBase, safeClientId);
|
|
|
|
|
if (!clientDir.startsWith(resolvedBase + path.sep)) {
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
let totalBytes = 0;
|
|
|
|
|
try {
|
|
|
|
|
const dirStat = await fs.promises.lstat(clientDir).catch(() => null);
|
|
|
|
|
if (dirStat && !dirStat.isSymbolicLink() && dirStat.isDirectory()) {
|
|
|
|
|
const entries = await fs.promises
|
|
|
|
|
.readdir(clientDir)
|
|
|
|
|
.catch(() => [] as string[]);
|
|
|
|
|
for (const entry of entries) {
|
|
|
|
|
if (!this.isPartitionFilename(entry)) continue;
|
|
|
|
|
const filePath = path.join(clientDir, entry);
|
|
|
|
|
const stats = await fs.promises.lstat(filePath).catch(() => null);
|
|
|
|
|
if (stats && !stats.isSymbolicLink() && stats.isFile()) {
|
|
|
|
|
totalBytes += stats.size;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
// Fall back to 0 on unreadable directory
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
this.clientBytesCache.set(safeClientId, { checkedAt: now, bytes: totalBytes });
|
|
|
|
|
return totalBytes;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Checks global VPS disk threshold and per-client MB quota before admitting new events
|
|
|
|
|
*/
|
|
|
|
|
public static async checkStorageAdmissionAsync(
|
|
|
|
|
clientId: string,
|
|
|
|
|
clientQuotaMb: number,
|
|
|
|
|
diskPauseThresholdPct: number,
|
|
|
|
|
): Promise<StorageAdmissionResult> {
|
|
|
|
|
const threshold = Number(diskPauseThresholdPct);
|
|
|
|
|
if (Number.isFinite(threshold) && threshold >= 50 && threshold <= 99) {
|
|
|
|
|
const disk = await this.getDiskUsageAsync();
|
|
|
|
|
if (disk.totalBytes > 0 && disk.usedPct >= threshold) {
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
|
|
|
|
code: 'DISK_SPACE_LOW',
|
|
|
|
|
reason: `DISK_SPACE_LOW: Disk utilization (${disk.usedPct.toFixed(1)}%) reached safety pause threshold (${threshold}%).`,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const quotaMb = Number(clientQuotaMb);
|
|
|
|
|
if (Number.isFinite(quotaMb) && quotaMb > 0) {
|
|
|
|
|
const currentBytes = await this.getClientStorageBytesAsync(clientId);
|
|
|
|
|
const quotaBytes = Math.floor(quotaMb) * 1024 * 1024;
|
|
|
|
|
if (currentBytes >= quotaBytes) {
|
|
|
|
|
return {
|
|
|
|
|
allowed: false,
|
|
|
|
|
code: 'STORAGE_QUOTA_EXCEEDED',
|
|
|
|
|
reason: `STORAGE_QUOTA_EXCEEDED: Client '${clientId}' used ${(currentBytes / (1024 * 1024)).toFixed(2)} MB of ${quotaMb} MB quota.`,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return { allowed: true };
|
2026-11-19 20:22:01 +06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static formatStorageStats(files: StorageFileMeta[]) {
|
2026-10-23 11:43:49 -05:00
|
|
|
const totalBytes = files.reduce((acc, f) => acc + f.sizeBytes, 0);
|
|
|
|
|
const totalRows = files.reduce((acc, f) => acc + f.linesCount, 0);
|
|
|
|
|
const uniqueClients = new Set(files.map((f) => f.clientId)).size;
|
2026-11-17 20:05:00 +06:00
|
|
|
const rotationMb = Math.max(
|
|
|
|
|
1,
|
|
|
|
|
Math.round(this.maxFileSizeBytes / (1024 * 1024)),
|
|
|
|
|
);
|
2026-10-23 11:43:49 -05:00
|
|
|
|
|
|
|
|
return {
|
|
|
|
|
totalFiles: files.length,
|
|
|
|
|
totalBytes,
|
|
|
|
|
totalMB: (totalBytes / (1024 * 1024)).toFixed(2),
|
|
|
|
|
totalRows,
|
|
|
|
|
uniqueClients,
|
|
|
|
|
storageDirectory: this.baseStorageDir,
|
2026-11-17 20:05:00 +06:00
|
|
|
rotationMb,
|
|
|
|
|
rotationRule: `${rotationMb}MB / Daily`,
|
2026-10-23 11:43:49 -05:00
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
2026-11-18 05:18:55 +06:00
|
|
|
* Get aggregate storage statistics (sync cached)
|
|
|
|
|
*/
|
|
|
|
|
public static getStorageStats() {
|
|
|
|
|
return this.formatStorageStats(this.listAllFiles());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Get aggregate storage statistics (non-blocking async)
|
|
|
|
|
*/
|
|
|
|
|
public static async getStorageStatsAsync() {
|
|
|
|
|
const files = await this.listAllFilesAsync();
|
|
|
|
|
return this.formatStorageStats(files);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
2026-11-19 23:02:27 +06:00
|
|
|
* Reads file content for export / download with path traversal protection (supports .csv & .csv.gz)
|
2026-10-23 11:43:49 -05:00
|
|
|
*/
|
|
|
|
|
public static getFileContent(clientId: string, filename: string): string | null {
|
2026-11-19 20:22:01 +06:00
|
|
|
const safeClientId = this.sanitizeClientId(clientId);
|
2026-11-18 05:18:55 +06:00
|
|
|
const safeFilename = path.basename(String(filename || ''));
|
2026-11-19 23:02:27 +06:00
|
|
|
if (!safeClientId || !safeFilename || !this.isPartitionFilename(safeFilename)) {
|
2026-11-18 05:18:55 +06:00
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
const resolvedBase = path.resolve(this.baseStorageDir);
|
2026-11-19 20:22:01 +06:00
|
|
|
const fullPath = path.resolve(resolvedBase, safeClientId, safeFilename);
|
2026-11-18 05:18:55 +06:00
|
|
|
if (!fullPath.startsWith(resolvedBase + path.sep)) {
|
|
|
|
|
return null;
|
|
|
|
|
}
|
2026-10-23 11:43:49 -05:00
|
|
|
if (fs.existsSync(fullPath)) {
|
2026-11-19 20:22:01 +06:00
|
|
|
const stats = fs.lstatSync(fullPath);
|
|
|
|
|
if (stats.isSymbolicLink() || !stats.isFile()) {
|
|
|
|
|
return null;
|
|
|
|
|
}
|
2026-11-19 23:02:27 +06:00
|
|
|
if (safeFilename.endsWith('.gz')) {
|
|
|
|
|
return zlib.gunzipSync(fs.readFileSync(fullPath)).toString('utf8');
|
|
|
|
|
}
|
2026-10-23 11:43:49 -05:00
|
|
|
return fs.readFileSync(fullPath, 'utf8');
|
|
|
|
|
}
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export default CSVStorageManager;
|