feat(client): implement per-client storage and security controls

- Added storage quota, IP allowlisting, and expiration logic to client management APIs.
- Updated CSV storage manager to support auto-compression and improved partition file parsing.
- Integrated security checks into event ingestion and data retrieval routes.
This commit is contained in:
Team Dark Devil committed 2026-11-19 23:02:27 +06:00
1 parent 89e4d68690
commit 2873b9904a
32 files changed
+6075 -241

No files matched your search

+444 -59
View File
@@ -1,7 +1,10 @@
import crypto from 'node:crypto';
import fs from 'fs';
import path from 'path';
import { Readable } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import { StringDecoder } from 'node:string_decoder';
import zlib from 'node:zlib';
import { Parser } from 'json2csv';
export interface DataEvent {
@@ -19,8 +22,28 @@ export interface StorageFileMeta {
sizeBytes: number;
linesCount: number;
modifiedAt: Date;
compressed?: boolean;
}
export interface DiskUsageSnapshot {
usedPct: number;
availableBytes: number;
totalBytes: number;
checkedAt: number;
}
export interface StorageAdmissionResult {
allowed: boolean;
code?: 'DISK_SPACE_LOW' | 'STORAGE_QUOTA_EXCEEDED';
reason?: string;
}
export type StatfsProvider = (dir: string) => Promise<{
blocks: number;
bavail: number;
bsize: number;
}>;
interface RowCountCacheEntry {
sizeBytes: number;
mtimeMs: number;
@@ -31,6 +54,7 @@ const LOCK_RETRY_MIN_MS = 5;
const LOCK_RETRY_MAX_MS = 18;
const LOCK_TIMEOUT_MS = 10_000;
const STALE_LOCK_AGE_MS = 30_000;
const QUOTA_CACHE_TTL_MS = 30_000;
function sleepMs(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
@@ -49,11 +73,39 @@ export class CSVStorageManager {
public static baseStorageDir = resolveDefaultStorageDir();
public static maxFileSizeBytes = 100 * 1024 * 1024; // 100MB rotation limit
private static rowCountCache = new Map<string, RowCountCacheEntry>();
private static diskUsageCache: DiskUsageSnapshot | null = null;
private static clientBytesCache = new Map<
string,
{ checkedAt: number; bytes: number }
>();
private static statfsOverrideForTests: StatfsProvider | null = null;
public static isPartitionFilename(filename: string): boolean {
return filename.endsWith('.csv') || filename.endsWith('.csv.gz');
}
public static extractDateFromFilename(filename: string): string {
return filename.replace(/\.csv(?:\.gz)?$/, '').split('-part')[0];
}
public static clearRowCountCache(): void {
this.rowCountCache.clear();
}
public static clearQuotaCaches(clientId?: string): void {
this.diskUsageCache = null;
if (clientId) {
this.clientBytesCache.delete(this.sanitizeClientId(clientId));
} else {
this.clientBytesCache.clear();
}
}
public static setStatfsOverrideForTests(provider: StatfsProvider | null): void {
this.statfsOverrideForTests = provider;
this.diskUsageCache = null;
}
public static sanitizeClientId(clientId: string): string {
const raw = String(clientId || 'unknown').trim().replace(/[^a-zA-Z0-9_-]/g, '_');
return raw || 'unknown';
@@ -299,6 +351,12 @@ export class CSVStorageManager {
private static countCsvDataRowsChunkedSync(filePath: string): number {
const counter = this.createChunkRowCounter();
if (filePath.endsWith('.gz')) {
const compressed = fs.readFileSync(filePath);
const decompressed = zlib.gunzipSync(compressed);
counter.pushChunk(decompressed);
return counter.finish();
}
const buf = Buffer.allocUnsafe(64 * 1024);
const fd = fs.openSync(filePath, 'r');
try {
@@ -312,14 +370,34 @@ export class CSVStorageManager {
}
}
public static openPartitionTextStream(filePath: string): Readable {
const fileStream = fs.createReadStream(filePath, {
highWaterMark: 64 * 1024,
});
if (!filePath.endsWith('.gz')) {
return fileStream;
}
const gunzip = zlib.createGunzip();
fileStream.on('error', (err) => {
if (!gunzip.destroyed) {
gunzip.destroy(err);
}
});
gunzip.on('close', () => {
if (!fileStream.destroyed) {
fileStream.destroy();
}
});
fileStream.pipe(gunzip);
return gunzip;
}
private static async countCsvDataRowsStreamAsync(
filePath: string,
): Promise<number> {
const counter = this.createChunkRowCounter();
return new Promise<number>((resolve, reject) => {
const stream = fs.createReadStream(filePath, {
highWaterMark: 64 * 1024,
});
const stream = this.openPartitionTextStream(filePath);
stream.on('data', (chunk) => counter.pushChunk(chunk));
stream.on('end', () => resolve(counter.finish()));
stream.on('error', (err) => reject(err));
@@ -437,8 +515,9 @@ export class CSVStorageManager {
fs.writeFileSync(resolvedPath, fullCsv, 'utf8');
}
// Invalidate row cache for modified file immediately
// Invalidate row & quota caches for modified file immediately
this.rowCountCache.delete(resolvedPath);
this.clearQuotaCaches(clientId);
return resolvedPath;
});
@@ -449,7 +528,7 @@ export class CSVStorageManager {
}
/**
* List all stored CSV files with cached row counts (invalidated on mtime/size changes)
* List all stored CSV and gzipped CSV (.csv / .csv.gz) files with cached row counts
*/
public static listAllFiles(): StorageFileMeta[] {
const results: StorageFileMeta[] = [];
@@ -473,7 +552,7 @@ export class CSVStorageManager {
const files = fs.readdirSync(fullClientPath);
for (const file of files) {
if (!file.endsWith('.csv')) continue;
if (!this.isPartitionFilename(file)) continue;
const fullFilePath = path.join(fullClientPath, file);
let stats: fs.Stats;
try {
@@ -505,7 +584,7 @@ export class CSVStorageManager {
}
}
const datePart = file.replace('.csv', '').split('-part')[0];
const datePart = this.extractDateFromFilename(file);
results.push({
clientId: clientDir,
@@ -515,6 +594,7 @@ export class CSVStorageManager {
sizeBytes: stats.size,
linesCount,
modifiedAt: stats.mtime,
compressed: file.endsWith('.csv.gz'),
});
}
}
@@ -551,7 +631,7 @@ export class CSVStorageManager {
const files = await fs.promises.readdir(fullClientPath);
for (const file of files) {
if (!file.endsWith('.csv')) continue;
if (!this.isPartitionFilename(file)) continue;
const fullFilePath = path.join(fullClientPath, file);
const stats = await fs.promises.lstat(fullFilePath).catch(() => null);
if (!stats || stats.isSymbolicLink() || !stats.isFile()) continue;
@@ -578,7 +658,7 @@ export class CSVStorageManager {
}
}
const datePart = file.replace('.csv', '').split('-part')[0];
const datePart = this.extractDateFromFilename(file);
results.push({
clientId: clientDir,
@@ -588,6 +668,7 @@ export class CSVStorageManager {
sizeBytes: stats.size,
linesCount,
modifiedAt: stats.mtime,
compressed: file.endsWith('.csv.gz'),
});
}
}
@@ -605,7 +686,121 @@ export class CSVStorageManager {
}
/**
* Prune CSV partitions older than autoPruneDays under cooperative partition locks
* Compress uncompressed .csv partitions older than compressAfterDays into .csv.gz under cooperative partition locks
*/
public static async compressOldPartitionsAsync(
compressAfterDays: number,
): Promise<number> {
if (!Number.isFinite(compressAfterDays) || compressAfterDays <= 0) {
return 0;
}
const baseStat = await fs.promises.lstat(this.baseStorageDir).catch(() => null);
if (!baseStat || !baseStat.isDirectory()) {
return 0;
}
const cutoffDate = new Date(
Date.now() - Math.floor(compressAfterDays) * 86_400_000,
)
.toISOString()
.split('T')[0];
let compressedCount = 0;
try {
const clientDirs = await fs.promises.readdir(this.baseStorageDir);
for (const clientDir of clientDirs) {
const fullClientPath = path.join(this.baseStorageDir, clientDir);
const dirLstat = await fs.promises.lstat(fullClientPath).catch(() => null);
if (!dirLstat || dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) {
continue;
}
const files = await fs.promises
.readdir(fullClientPath)
.catch(() => [] as string[]);
for (const file of files) {
if (!file.endsWith('.csv')) continue;
const datePart = this.extractDateFromFilename(file);
if (!(/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate)) {
continue;
}
const fullPath = path.join(fullClientPath, file);
const fileLstat = await fs.promises.lstat(fullPath).catch(() => null);
if (!fileLstat || fileLstat.isSymbolicLink() || !fileLstat.isFile()) {
continue;
}
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
let acquired = false;
const tryAcquireLock = async (): Promise<boolean> => {
const handle = await fs.promises.open(lockPath, 'wx');
try {
await handle.writeFile(lockValue, 'utf8');
} finally {
await handle.close();
}
return true;
};
try {
acquired = await tryAcquireLock();
} catch (err: unknown) {
if ((err as NodeJS.ErrnoException)?.code === 'EEXIST') {
await this.removeStaleLockAtomicallyAsync(lockPath);
try {
acquired = await tryAcquireLock();
} catch {
acquired = false;
}
}
}
if (!acquired) continue;
let targetGzPath = `${fullPath}.gz`;
if (fs.existsSync(targetGzPath)) {
const stem = fullPath.replace(/\.csv$/, '');
let suffix = 2;
while (fs.existsSync(`${stem}-gz${suffix}.csv.gz`)) {
suffix += 1;
}
targetGzPath = `${stem}-gz${suffix}.csv.gz`;
}
const tmpGzPath = `${targetGzPath}.tmp.${crypto.randomUUID()}`;
try {
await pipeline(
fs.createReadStream(fullPath, { highWaterMark: 64 * 1024 }),
zlib.createGzip({ level: 6 }),
fs.createWriteStream(tmpGzPath),
);
await fs.promises
.utimes(tmpGzPath, fileLstat.atime, fileLstat.mtime)
.catch(() => undefined);
await fs.promises.rename(tmpGzPath, targetGzPath);
await fs.promises.unlink(fullPath);
this.rowCountCache.delete(fullPath);
this.rowCountCache.delete(targetGzPath);
this.clearQuotaCaches(clientDir);
compressedCount += 1;
} catch {
await fs.promises.unlink(tmpGzPath).catch(() => undefined);
} finally {
await this.releaseLockIfOwnedAsync(lockPath, lockValue);
}
}
}
} catch {
// Ignore directory scan failure
}
return compressedCount;
}
/**
* Prune CSV / CSV.GZ partitions older than autoPruneDays under cooperative partition locks
*/
public static pruneExpiredPartitions(autoPruneDays: number): number {
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
@@ -636,8 +831,8 @@ export class CSVStorageManager {
const files = fs.readdirSync(fullClientPath);
for (const file of files) {
if (!file.endsWith('.csv')) continue;
const datePart = file.replace('.csv', '').split('-part')[0];
if (!this.isPartitionFilename(file)) continue;
const datePart = this.extractDateFromFilename(file);
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
const fullPath = path.join(fullClientPath, file);
let fileLstat: fs.Stats;
@@ -683,6 +878,7 @@ export class CSVStorageManager {
if (fs.existsSync(fullPath)) {
fs.unlinkSync(fullPath);
this.rowCountCache.delete(fullPath);
this.clearQuotaCaches(clientDir);
prunedCount += 1;
}
} catch {
@@ -701,27 +897,54 @@ export class CSVStorageManager {
}
/**
* Non-blocking async partition pruner using fs.promises under cooperative partition locks
* Archive and/or prune expired partitions under cooperative partition locks.
* Never deletes a local partition if options.beforeDeleteFile throws or rejects (V2-E02).
*/
public static async pruneExpiredPartitionsAsync(
public static async pruneOrArchiveExpiredPartitionsAsync(
autoPruneDays: number,
): Promise<number> {
if (!Number.isFinite(autoPruneDays) || autoPruneDays <= 0) {
return 0;
options?: {
beforeDeleteFile?: (file: {
clientId: string;
filename: string;
filePath: string;
date: string;
}) => Promise<void>;
archiveOnlyWithoutDelete?: boolean;
maxCutoffDays?: number;
},
): Promise<{
scannedCount: number;
archivedCount: number;
prunedCount: number;
errors: string[];
}> {
const targetDays =
options?.maxCutoffDays !== undefined
? options.maxCutoffDays
: autoPruneDays;
if (!Number.isFinite(targetDays) || targetDays < 0) {
return { scannedCount: 0, archivedCount: 0, prunedCount: 0, errors: [] };
}
if (options?.maxCutoffDays === undefined && targetDays <= 0) {
return { scannedCount: 0, archivedCount: 0, prunedCount: 0, errors: [] };
}
const baseStat = await fs.promises.lstat(this.baseStorageDir).catch(() => null);
if (!baseStat || !baseStat.isDirectory()) {
return 0;
return { scannedCount: 0, archivedCount: 0, prunedCount: 0, errors: [] };
}
const cutoffDate = new Date(
Date.now() - Math.floor(autoPruneDays) * 86_400_000,
Date.now() - Math.floor(targetDays) * 86_400_000,
)
.toISOString()
.split('T')[0];
let scannedCount = 0;
let archivedCount = 0;
let prunedCount = 0;
const errors: string[] = [];
try {
const clientDirs = await fs.promises.readdir(this.baseStorageDir);
for (const clientDir of clientDirs) {
@@ -731,54 +954,71 @@ export class CSVStorageManager {
continue;
}
const files = await fs.promises.readdir(fullClientPath).catch(() => [] as string[]);
const files = await fs.promises
.readdir(fullClientPath)
.catch(() => [] as string[]);
for (const file of files) {
if (!file.endsWith('.csv')) continue;
const datePart = file.replace('.csv', '').split('-part')[0];
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
const fullPath = path.join(fullClientPath, file);
const fileLstat = await fs.promises.lstat(fullPath).catch(() => null);
if (!fileLstat || fileLstat.isSymbolicLink() || !fileLstat.isFile()) {
continue;
}
if (!this.isPartitionFilename(file)) continue;
const datePart = this.extractDateFromFilename(file);
if (!(/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate)) {
continue;
}
const fullPath = path.join(fullClientPath, file);
const fileLstat = await fs.promises.lstat(fullPath).catch(() => null);
if (!fileLstat || fileLstat.isSymbolicLink() || !fileLstat.isFile()) {
continue;
}
scannedCount += 1;
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
let acquired = false;
const tryAcquireLock = async (): Promise<boolean> => {
const handle = await fs.promises.open(lockPath, 'wx');
const lockPath = path.join(fullClientPath, `.${datePart}.csv.lock`);
const lockValue = `${process.pid}:${Date.now()}:${crypto.randomUUID()}`;
let acquired = false;
const tryAcquireLock = async (): Promise<boolean> => {
const handle = await fs.promises.open(lockPath, 'wx');
try {
await handle.writeFile(lockValue, 'utf8');
} finally {
await handle.close();
}
return true;
};
try {
acquired = await tryAcquireLock();
} catch (err: unknown) {
if ((err as NodeJS.ErrnoException)?.code === 'EEXIST') {
await this.removeStaleLockAtomicallyAsync(lockPath);
try {
await handle.writeFile(lockValue, 'utf8');
} finally {
await handle.close();
}
return true;
};
try {
acquired = await tryAcquireLock();
} catch (err: unknown) {
if ((err as NodeJS.ErrnoException)?.code === 'EEXIST') {
await this.removeStaleLockAtomicallyAsync(lockPath);
try {
acquired = await tryAcquireLock();
} catch {
acquired = false;
}
acquired = await tryAcquireLock();
} catch {
acquired = false;
}
}
}
if (!acquired) continue;
if (!acquired) continue;
try {
try {
if (options?.beforeDeleteFile) {
await options.beforeDeleteFile({
clientId: clientDir,
filename: file,
filePath: fullPath,
date: datePart,
});
archivedCount += 1;
}
if (!options?.archiveOnlyWithoutDelete) {
await fs.promises.unlink(fullPath);
this.rowCountCache.delete(fullPath);
this.clearQuotaCaches(clientDir);
prunedCount += 1;
} catch {
// Ignore individual unlink error or concurrent removal
} finally {
await this.releaseLockIfOwnedAsync(lockPath, lockValue);
}
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : 'Archive/prune error';
errors.push(`${clientDir}/${file}: ${msg}`);
} finally {
await this.releaseLockIfOwnedAsync(lockPath, lockValue);
}
}
}
@@ -786,7 +1026,149 @@ export class CSVStorageManager {
// Ignore directory scan failure
}
return prunedCount;
return { scannedCount, archivedCount, prunedCount, errors };
}
/**
* Non-blocking async partition pruner using fs.promises under cooperative partition locks
*/
public static async pruneExpiredPartitionsAsync(
autoPruneDays: number,
): Promise<number> {
const res = await this.pruneOrArchiveExpiredPartitionsAsync(autoPruneDays);
return res.prunedCount;
}
/**
* Cached VPS filesystem disk usage snapshot (30s TTL) for disk-full protection
*/
public static async getDiskUsageAsync(): Promise<DiskUsageSnapshot> {
const now = Date.now();
if (
this.diskUsageCache &&
now - this.diskUsageCache.checkedAt < QUOTA_CACHE_TTL_MS
) {
return this.diskUsageCache;
}
let probeDir = path.resolve(this.baseStorageDir);
while (!fs.existsSync(probeDir)) {
const parent = path.dirname(probeDir);
if (parent === probeDir) break;
probeDir = parent;
}
try {
const statfsRes = this.statfsOverrideForTests
? await this.statfsOverrideForTests(probeDir)
: await fs.promises.statfs(probeDir);
const bsize = Math.max(1, Number(statfsRes.bsize) || 4096);
const totalBytes = Math.max(0, Number(statfsRes.blocks) * bsize);
const availableBytes = Math.max(0, Number(statfsRes.bavail) * bsize);
const usedBytes = Math.max(0, totalBytes - availableBytes);
const usedPct =
totalBytes > 0
? Number(
Math.min(100, Math.max(0, (usedBytes / totalBytes) * 100)).toFixed(2),
)
: 0;
const snapshot: DiskUsageSnapshot = {
usedPct,
availableBytes,
totalBytes,
checkedAt: now,
};
this.diskUsageCache = snapshot;
return snapshot;
} catch {
const fallback: DiskUsageSnapshot = {
usedPct: 0,
availableBytes: 0,
totalBytes: 0,
checkedAt: now,
};
this.diskUsageCache = fallback;
return fallback;
}
}
/**
* Cached per-client directory byte count (30s TTL)
*/
public static async getClientStorageBytesAsync(
clientId: string,
): Promise<number> {
const safeClientId = this.sanitizeClientId(clientId);
const now = Date.now();
const cached = this.clientBytesCache.get(safeClientId);
if (cached && now - cached.checkedAt < QUOTA_CACHE_TTL_MS) {
return cached.bytes;
}
const resolvedBase = path.resolve(this.baseStorageDir);
const clientDir = path.resolve(resolvedBase, safeClientId);
if (!clientDir.startsWith(resolvedBase + path.sep)) {
return 0;
}
let totalBytes = 0;
try {
const dirStat = await fs.promises.lstat(clientDir).catch(() => null);
if (dirStat && !dirStat.isSymbolicLink() && dirStat.isDirectory()) {
const entries = await fs.promises
.readdir(clientDir)
.catch(() => [] as string[]);
for (const entry of entries) {
if (!this.isPartitionFilename(entry)) continue;
const filePath = path.join(clientDir, entry);
const stats = await fs.promises.lstat(filePath).catch(() => null);
if (stats && !stats.isSymbolicLink() && stats.isFile()) {
totalBytes += stats.size;
}
}
}
} catch {
// Fall back to 0 on unreadable directory
}
this.clientBytesCache.set(safeClientId, { checkedAt: now, bytes: totalBytes });
return totalBytes;
}
/**
* Checks global VPS disk threshold and per-client MB quota before admitting new events
*/
public static async checkStorageAdmissionAsync(
clientId: string,
clientQuotaMb: number,
diskPauseThresholdPct: number,
): Promise<StorageAdmissionResult> {
const threshold = Number(diskPauseThresholdPct);
if (Number.isFinite(threshold) && threshold >= 50 && threshold <= 99) {
const disk = await this.getDiskUsageAsync();
if (disk.totalBytes > 0 && disk.usedPct >= threshold) {
return {
allowed: false,
code: 'DISK_SPACE_LOW',
reason: `DISK_SPACE_LOW: Disk utilization (${disk.usedPct.toFixed(1)}%) reached safety pause threshold (${threshold}%).`,
};
}
}
const quotaMb = Number(clientQuotaMb);
if (Number.isFinite(quotaMb) && quotaMb > 0) {
const currentBytes = await this.getClientStorageBytesAsync(clientId);
const quotaBytes = Math.floor(quotaMb) * 1024 * 1024;
if (currentBytes >= quotaBytes) {
return {
allowed: false,
code: 'STORAGE_QUOTA_EXCEEDED',
reason: `STORAGE_QUOTA_EXCEEDED: Client '${clientId}' used ${(currentBytes / (1024 * 1024)).toFixed(2)} MB of ${quotaMb} MB quota.`,
};
}
}
return { allowed: true };
}
public static formatStorageStats(files: StorageFileMeta[]) {
@@ -826,12 +1208,12 @@ export class CSVStorageManager {
}
/**
* Reads file content for export / download with path traversal protection
* Reads file content for export / download with path traversal protection (supports .csv & .csv.gz)
*/
public static getFileContent(clientId: string, filename: string): string | null {
const safeClientId = this.sanitizeClientId(clientId);
const safeFilename = path.basename(String(filename || ''));
if (!safeClientId || !safeFilename || !safeFilename.endsWith('.csv')) {
if (!safeClientId || !safeFilename || !this.isPartitionFilename(safeFilename)) {
return null;
}
const resolvedBase = path.resolve(this.baseStorageDir);
@@ -844,6 +1226,9 @@ export class CSVStorageManager {
if (stats.isSymbolicLink() || !stats.isFile()) {
return null;
}
if (safeFilename.endsWith('.gz')) {
return zlib.gunzipSync(fs.readFileSync(fullPath)).toString('utf8');
}
return fs.readFileSync(fullPath, 'utf8');
}
return null;
+335
View File
@@ -0,0 +1,335 @@
import crypto from "node:crypto";
import fs from "node:fs";
export interface S3ArchiveConfig {
endpoint: string;
region: string;
bucket: string;
prefix: string;
accessKeyId: string;
secretAccessKey: string;
}
export interface S3OperationResult {
ok: boolean;
status?: number;
endpoint?: string;
bucket?: string;
key?: string;
etag?: string;
code?: "S3_INVALID_CONFIG" | "S3_CONNECTION_FAILED" | "S3_ARCHIVE_FAILED";
error?: string;
}
function sha256Hex(data: Buffer | string): string {
return crypto.createHash("sha256").update(data).digest("hex");
}
function hmacSha256(key: Buffer | string, data: string): Buffer {
return crypto.createHmac("sha256", key).update(data, "utf8").digest();
}
function encodeRfc3986(value: string): string {
return encodeURIComponent(value).replace(
/[!'()*]/g,
(ch) => `%${ch.charCodeAt(0).toString(16).toUpperCase()}`,
);
}
function encodeS3Path(rawPath: string): string {
return rawPath
.split("/")
.map((segment) => encodeRfc3986(segment))
.join("/");
}
export function buildS3ObjectKey(
prefix: string,
clientId: string,
filename: string,
): string {
const cleanPrefix = String(prefix || "")
.trim()
.replace(/^\/+|\/+$/g, "");
const cleanClient = String(clientId || "unknown")
.trim()
.replace(/^\/+|\/+$/g, "");
const cleanFile = String(filename || "")
.trim()
.replace(/^\/+/, "");
return [cleanPrefix, cleanClient, cleanFile].filter(Boolean).join("/");
}
export function normalizeS3EndpointUrl(
endpoint: string,
region: string,
): { url: URL; region: string } {
const trimmedRegion = String(region || "").trim() || "auto";
const trimmedEndpoint = String(endpoint || "").trim().replace(/\/+$/, "");
if (trimmedEndpoint) {
const withProtocol = /^https?:\/\//i.test(trimmedEndpoint)
? trimmedEndpoint
: `https://${trimmedEndpoint}`;
const parsed = new URL(withProtocol);
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
throw new Error("S3 endpoint must use http:// or https://");
}
return { url: parsed, region: trimmedRegion };
}
if (trimmedRegion === "auto") {
throw new Error(
"S3 Endpoint URL is required when Region is set to 'auto' (e.g. Cloudflare R2 or MinIO).",
);
}
return {
url: new URL(`https://s3.${trimmedRegion}.amazonaws.com`),
region: trimmedRegion,
};
}
export function signS3Request(params: {
method: "GET" | "HEAD" | "PUT";
endpointUrl: URL;
bucket: string;
objectKey?: string;
queryParams?: Record<string, string>;
region: string;
accessKeyId: string;
secretAccessKey: string;
payloadHashHex: string;
contentType?: string;
now?: Date;
}): { targetUrl: string; headers: Record<string, string> } {
const now = params.now ?? new Date();
const amzDate = now.toISOString().replace(/[:-]|\.\d{3}/g, "");
const dateStamp = amzDate.slice(0, 8);
const basePath = params.endpointUrl.pathname.replace(/\/+$/, "");
const bucketPart = encodeRfc3986(params.bucket.trim());
const keyPart = params.objectKey ? `/${encodeS3Path(params.objectKey)}` : "";
const canonicalUri = `${basePath}/${bucketPart}${keyPart}`.replace(
/\/{2,}/g,
"/",
);
const queryEntries = Object.entries(params.queryParams ?? {}).sort(([a], [b]) =>
a < b ? -1 : a > b ? 1 : 0,
);
const canonicalQueryString = queryEntries
.map(([k, v]) => `${encodeRfc3986(k)}=${encodeRfc3986(v)}`)
.join("&");
const hostHeader = params.endpointUrl.host;
const headerMap: Record<string, string> = {
host: hostHeader,
"x-amz-content-sha256": params.payloadHashHex,
"x-amz-date": amzDate,
};
if (params.contentType) {
headerMap["content-type"] = params.contentType;
}
const sortedHeaderKeys = Object.keys(headerMap).sort();
const canonicalHeaders = sortedHeaderKeys
.map((key) => `${key}:${headerMap[key].trim()}\n`)
.join("");
const signedHeaders = sortedHeaderKeys.join(";");
const canonicalRequest = [
params.method,
canonicalUri,
canonicalQueryString,
canonicalHeaders,
signedHeaders,
params.payloadHashHex,
].join("\n");
const credentialScope = `${dateStamp}/${params.region}/s3/aws4_request`;
const stringToSign = [
"AWS4-HMAC-SHA256",
amzDate,
credentialScope,
sha256Hex(canonicalRequest),
].join("\n");
const kDate = hmacSha256(`AWS4${params.secretAccessKey}`, dateStamp);
const kRegion = hmacSha256(kDate, params.region);
const kService = hmacSha256(kRegion, "s3");
const kSigning = hmacSha256(kService, "aws4_request");
const signature = crypto
.createHmac("sha256", kSigning)
.update(stringToSign, "utf8")
.digest("hex");
const authorization = `AWS4-HMAC-SHA256 Credential=${params.accessKeyId}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`;
const targetUrl = `${params.endpointUrl.origin}${canonicalUri}${
canonicalQueryString ? `?${canonicalQueryString}` : ""
}`;
return {
targetUrl,
headers: {
"x-amz-date": amzDate,
"x-amz-content-sha256": params.payloadHashHex,
...(params.contentType ? { "content-type": params.contentType } : {}),
Authorization: authorization,
},
};
}
function validateS3Config(config: S3ArchiveConfig): string | null {
if (!config.bucket?.trim()) return "S3 Bucket name is required.";
if (!config.accessKeyId?.trim()) return "S3 Access Key ID is required.";
if (!config.secretAccessKey?.trim()) return "S3 Secret Access Key is required.";
return null;
}
export async function testS3Connection(
config: S3ArchiveConfig,
): Promise<S3OperationResult> {
const validationError = validateS3Config(config);
if (validationError) {
return {
ok: false,
code: "S3_INVALID_CONFIG",
error: validationError,
};
}
try {
const { url, region } = normalizeS3EndpointUrl(
config.endpoint,
config.region,
);
const signed = signS3Request({
method: "GET",
endpointUrl: url,
bucket: config.bucket.trim(),
queryParams: { "list-type": "2", "max-keys": "1" },
region,
accessKeyId: config.accessKeyId.trim(),
secretAccessKey: config.secretAccessKey,
payloadHashHex: sha256Hex(""),
});
const response = await fetch(signed.targetUrl, {
method: "GET",
headers: signed.headers,
cache: "no-store",
signal: AbortSignal.timeout(8000),
});
if (!response.ok) {
const bodyPreview = await response.text().catch(() => "");
return {
ok: false,
status: response.status,
endpoint: url.origin,
bucket: config.bucket.trim(),
code: "S3_CONNECTION_FAILED",
error: `S3 endpoint returned HTTP ${response.status}${
bodyPreview ? `: ${bodyPreview.slice(0, 180)}` : ""
}`,
};
}
return {
ok: true,
status: response.status,
endpoint: url.origin,
bucket: config.bucket.trim(),
};
} catch (err: unknown) {
return {
ok: false,
code: "S3_CONNECTION_FAILED",
error:
err instanceof Error ? err.message : "Unable to reach S3 endpoint",
};
}
}
export async function uploadFileToS3(
config: S3ArchiveConfig,
localFilePath: string,
objectKey: string,
): Promise<S3OperationResult> {
const validationError = validateS3Config(config);
if (validationError) {
return {
ok: false,
code: "S3_INVALID_CONFIG",
error: validationError,
key: objectKey,
};
}
try {
const { url, region } = normalizeS3EndpointUrl(
config.endpoint,
config.region,
);
const bodyBuffer = await fs.promises.readFile(localFilePath);
const payloadHashHex = sha256Hex(bodyBuffer);
const contentType = localFilePath.endsWith(".gz")
? "application/gzip"
: "text/csv; charset=utf-8";
const signed = signS3Request({
method: "PUT",
endpointUrl: url,
bucket: config.bucket.trim(),
objectKey,
region,
accessKeyId: config.accessKeyId.trim(),
secretAccessKey: config.secretAccessKey,
payloadHashHex,
contentType,
});
const response = await fetch(signed.targetUrl, {
method: "PUT",
headers: {
...signed.headers,
"content-length": String(bodyBuffer.byteLength),
},
body: new Uint8Array(bodyBuffer),
cache: "no-store",
signal: AbortSignal.timeout(30000),
});
if (!response.ok) {
const bodyPreview = await response.text().catch(() => "");
return {
ok: false,
status: response.status,
endpoint: url.origin,
bucket: config.bucket.trim(),
key: objectKey,
code: "S3_ARCHIVE_FAILED",
error: `S3 upload failed with HTTP ${response.status}${
bodyPreview ? `: ${bodyPreview.slice(0, 180)}` : ""
}`,
};
}
return {
ok: true,
status: response.status,
endpoint: url.origin,
bucket: config.bucket.trim(),
key: objectKey,
etag: response.headers.get("etag") ?? undefined,
};
} catch (err: unknown) {
return {
ok: false,
key: objectKey,
code: "S3_ARCHIVE_FAILED",
error: err instanceof Error ? err.message : "S3 upload request failed",
};
}
}