Files
DataStack/core/Storage/csvStorage.ts
T

239 lines
6.9 KiB
TypeScript
Raw Normal View History

2026-10-23 11:43:49 -05:00
import fs from 'fs';
import path from 'path';
import { Parser } from 'json2csv';
export interface DataEvent {
clientId: string;
eventId: string;
timestamp: string; // ISO 8601
payload: string; // JSON stringified
}
export interface StorageFileMeta {
clientId: string;
date: string;
filename: string;
filePath: string;
sizeBytes: number;
linesCount: number;
modifiedAt: Date;
}
export class CSVStorageManager {
// Base storage directory: /storage/data
public static baseStorageDir = path.resolve(process.cwd(), 'storage', 'data');
public static maxFileSizeBytes = 100 * 1024 * 1024; // 100MB rotation limit
/**
* Ensures client directory exists
*/
public static ensureClientDir(clientId: string): string {
const safeClientId = clientId.replace(/[^a-zA-Z0-9_-]/g, '_');
const clientDir = path.join(this.baseStorageDir, safeClientId);
if (!fs.existsSync(clientDir)) {
fs.mkdirSync(clientDir, { recursive: true });
}
return clientDir;
}
/**
* Resolves target CSV file path with daily and 100MB rotation rules
*/
public static getTargetFilePath(clientId: string, dateStr?: string): string {
const clientDir = this.ensureClientDir(clientId);
const date = dateStr || new Date().toISOString().split('T')[0];
let baseFileName = `${date}.csv`;
let targetPath = path.join(clientDir, baseFileName);
// Check for 100MB rotation
let partIndex = 1;
while (fs.existsSync(targetPath)) {
try {
const stats = fs.statSync(targetPath);
if (stats.size >= this.maxFileSizeBytes) {
partIndex += 1;
baseFileName = `${date}-part${partIndex}.csv`;
targetPath = path.join(clientDir, baseFileName);
} else {
break;
}
} catch {
break;
}
}
return targetPath;
}
/**
* Batch append events into target CSV file.
* If file doesn't exist, writes header: clientId,eventId,timestamp,payload
*/
public static async appendBatch(events: DataEvent[]): Promise<{ written: number; filesUpdated: string[] }> {
if (!events || events.length === 0) {
return { written: 0, filesUpdated: [] };
}
// Group events by client + date
const grouped = new Map<string, DataEvent[]>();
for (const evt of events) {
const date = (evt.timestamp || new Date().toISOString()).split('T')[0];
const key = `${evt.clientId}:::${date}`;
if (!grouped.has(key)) {
grouped.set(key, []);
}
grouped.get(key)!.push(evt);
}
const filesUpdated: string[] = [];
const fields = ['clientId', 'eventId', 'timestamp', 'payload'];
for (const [key, batch] of grouped.entries()) {
const [clientId, date] = key.split(':::');
const targetPath = this.getTargetFilePath(clientId, date);
const fileExists = fs.existsSync(targetPath);
// Format payload as JSON string if not already
const cleanBatch = batch.map((item) => ({
clientId: item.clientId,
eventId: item.eventId,
timestamp: item.timestamp,
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
}));
const parser = new Parser({ fields, header: !fileExists, eol: '\n' });
const csvData = parser.parse(cleanBatch);
if (fileExists) {
fs.appendFileSync(targetPath, '\n' + csvData, 'utf8');
} else {
fs.writeFileSync(targetPath, csvData, 'utf8');
}
filesUpdated.push(targetPath);
}
return { written: events.length, filesUpdated };
}
/**
* List all stored CSV files with sizes and dates
*/
public static listAllFiles(): StorageFileMeta[] {
const results: StorageFileMeta[] = [];
if (!fs.existsSync(this.baseStorageDir)) {
return results;
}
try {
const clientDirs = fs.readdirSync(this.baseStorageDir);
for (const clientDir of clientDirs) {
const fullClientPath = path.join(this.baseStorageDir, clientDir);
if (!fs.statSync(fullClientPath).isDirectory()) continue;
const files = fs.readdirSync(fullClientPath);
for (const file of files) {
if (!file.endsWith('.csv')) continue;
const fullFilePath = path.join(fullClientPath, file);
const stats = fs.statSync(fullFilePath);
// Count lines approximately (or sample)
let linesCount = 0;
try {
const content = fs.readFileSync(fullFilePath, 'utf8');
linesCount = content.split('\n').filter((l) => l.trim().length > 0).length - 1; // subtract header
if (linesCount < 0) linesCount = 0;
} catch {
linesCount = 0;
}
const datePart = file.replace('.csv', '').split('-part')[0];
results.push({
clientId: clientDir,
date: datePart,
filename: file,
filePath: fullFilePath,
sizeBytes: stats.size,
linesCount,
modifiedAt: stats.mtime,
});
}
}
} catch {
// Return whatever collected
}
return results.sort((a, b) => b.modifiedAt.getTime() - a.modifiedAt.getTime());
}
/**
* Prune CSV partitions older than autoPruneDays
*/
public static pruneExpiredPartitions(autoPruneDays: number): number {
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
return 0;
}
const cutoffDate = new Date(
Date.now() - Math.floor(autoPruneDays) * 86_400_000,
)
.toISOString()
.split('T')[0];
let prunedCount = 0;
const files = this.listAllFiles();
for (const file of files) {
if (/^\d{4}-\d{2}-\d{2}$/.test(file.date) && file.date < cutoffDate) {
try {
if (fs.existsSync(file.filePath)) {
fs.unlinkSync(file.filePath);
prunedCount += 1;
}
} catch {
// Ignore file removal error
}
}
}
return prunedCount;
}
2026-10-23 11:43:49 -05:00
/**
* Get aggregate storage statistics
*/
public static getStorageStats() {
const files = this.listAllFiles();
const totalBytes = files.reduce((acc, f) => acc + f.sizeBytes, 0);
const totalRows = files.reduce((acc, f) => acc + f.linesCount, 0);
const uniqueClients = new Set(files.map((f) => f.clientId)).size;
const rotationMb = Math.max(
1,
Math.round(this.maxFileSizeBytes / (1024 * 1024)),
);
2026-10-23 11:43:49 -05:00
return {
totalFiles: files.length,
totalBytes,
totalMB: (totalBytes / (1024 * 1024)).toFixed(2),
totalRows,
uniqueClients,
storageDirectory: this.baseStorageDir,
rotationMb,
rotationRule: `${rotationMb}MB / Daily`,
2026-10-23 11:43:49 -05:00
};
}
/**
* Reads file content for export / download
*/
public static getFileContent(clientId: string, filename: string): string | null {
const fullPath = path.join(this.baseStorageDir, clientId, filename);
if (fs.existsSync(fullPath)) {
return fs.readFileSync(fullPath, 'utf8');
}
return null;
}
}
export default CSVStorageManager;