feat(client): implement per-client storage and security controls

- Added storage quota, IP allowlisting, and expiration logic to client management APIs.
- Updated CSV storage manager to support auto-compression and improved partition file parsing.
- Integrated security checks into event ingestion and data retrieval routes.
This commit is contained in:
Team Dark Devil committed 2026-11-19 23:02:27 +06:00
1 parent 89e4d68690
commit 2873b9904a
32 files changed
+6075 -241

No files matched your search

+163
View File
@@ -0,0 +1,163 @@
import { SettingsManager } from "@/core/adminSettings/settingsManager";
import { SecretVaultError } from "@/core/Security/secretVault";
import { CSVStorageManager } from "@/core/Storage/csvStorage";
import {
buildS3ObjectKey,
normalizeS3EndpointUrl,
uploadFileToS3,
} from "@/core/Storage/s3Archive";
import { requireAdminApiSession } from "@/lib/adminAuth";
import { NextRequest, NextResponse } from "next/server";
export async function POST(req: NextRequest) {
const auth = await requireAdminApiSession(req);
if (!auth.valid) {
return auth.response;
}
let body: Record<string, unknown> = {};
try {
const text = await req.text();
if (text.trim()) {
const parsed = JSON.parse(text);
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
body = parsed as Record<string, unknown>;
}
}
} catch {
return NextResponse.json(
{ error: "Invalid JSON request payload.", code: "INVALID_JSON" },
{ status: 400 },
);
}
try {
const settings = await SettingsManager.getSettingsAsync();
const s3Config = await SettingsManager.getDecryptedS3ConfigAsync();
if (!s3Config.bucket.trim() || !s3Config.accessKeyId.trim() || !s3Config.secretAccessKey.trim()) {
return NextResponse.json(
{
success: false,
code: "S3_INVALID_CONFIG",
error: "Configure S3 Bucket, Access Key ID, and Secret Access Key before archiving.",
},
{ status: 400 },
);
}
try {
normalizeS3EndpointUrl(s3Config.endpoint, s3Config.region);
} catch (urlErr: unknown) {
return NextResponse.json(
{
success: false,
code: "S3_INVALID_CONFIG",
error: urlErr instanceof Error ? urlErr.message : "Invalid S3 endpoint URL.",
},
{ status: 400 },
);
}
const pruneAfterUpload = body.pruneAfterUpload === true;
const rawOlderThanDays =
typeof body.olderThanDays === "number" && Number.isFinite(body.olderThanDays)
? Math.max(0, Math.floor(body.olderThanDays))
: settings.autoPruneDays > 0
? settings.autoPruneDays
: 0;
const shouldCompressFirst =
body.compressFirst !== false && settings.compressAfterDays > 0;
let compressedCount = 0;
if (shouldCompressFirst) {
compressedCount = await CSVStorageManager.compressOldPartitionsAsync(
settings.compressAfterDays,
);
}
const archiveResult =
await CSVStorageManager.pruneOrArchiveExpiredPartitionsAsync(
settings.autoPruneDays,
{
archiveOnlyWithoutDelete: !pruneAfterUpload,
maxCutoffDays: rawOlderThanDays,
beforeDeleteFile: async (file) => {
const key = buildS3ObjectKey(
s3Config.prefix,
file.clientId,
file.filename,
);
const uploadRes = await uploadFileToS3(
s3Config,
file.filePath,
key,
);
if (!uploadRes.ok) {
throw new Error(
uploadRes.error || `Failed to upload ${file.filename}`,
);
}
},
},
);
if (
archiveResult.errors.length > 0 &&
archiveResult.archivedCount === 0
) {
SettingsManager.triggerWebhookAlert({
event: "S3_ARCHIVE_FAILED",
severity: "critical",
title: "S3 Cold Archive Upload Failed",
message: `Manual S3 archive failed: ${archiveResult.errors[0]}`,
details: { errorCount: archiveResult.errors.length },
});
return NextResponse.json(
{
success: false,
code: "S3_ARCHIVE_FAILED",
error: archiveResult.errors[0],
compressedCount,
...archiveResult,
},
{ status: 502 },
);
}
if (archiveResult.errors.length > 0) {
SettingsManager.triggerWebhookAlert({
event: "S3_ARCHIVE_FAILED",
severity: "warning",
title: "Partial S3 Cold Archive Errors",
message: `S3 archive encountered ${archiveResult.errors.length} error(s): ${archiveResult.errors[0]}`,
details: { errorCount: archiveResult.errors.length },
});
}
return NextResponse.json({
success: archiveResult.errors.length === 0,
compressedCount,
...archiveResult,
});
} catch (err: unknown) {
if (err instanceof SecretVaultError) {
return NextResponse.json(
{ success: false, code: err.code, error: err.message },
{ status: 400 },
);
}
const message =
err instanceof Error ? err.message : "Manual S3 archive failed";
SettingsManager.triggerWebhookAlert({
event: "S3_ARCHIVE_FAILED",
severity: "critical",
title: "S3 Cold Archive Upload Failed",
message,
});
return NextResponse.json(
{ success: false, code: "S3_ARCHIVE_FAILED", error: message },
{ status: 500 },
);
}
}
+5 -2
View File
@@ -23,7 +23,10 @@ export async function GET(req: NextRequest) {
? CSVStorageManager.sanitizeClientId(clientId)
: undefined;
const allFiles = await CSVStorageManager.listAllFilesAsync();
const [allFiles, diskUsage] = await Promise.all([
CSVStorageManager.listAllFilesAsync(),
CSVStorageManager.getDiskUsageAsync(),
]);
let files = allFiles;
if (clientId) {
@@ -39,7 +42,7 @@ export async function GET(req: NextRequest) {
const stats = CSVStorageManager.formatStorageStats(files);
return NextResponse.json({ stats, files });
return NextResponse.json({ stats, files, diskUsage });
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Storage stats failed';
return NextResponse.json({ error: message }, { status: 500 });
+105
View File
@@ -0,0 +1,105 @@
import { SettingsManager } from "@/core/adminSettings/settingsManager";
import {
decryptSecret,
SecretVaultError,
} from "@/core/Security/secretVault";
import {
testS3Connection,
type S3ArchiveConfig,
} from "@/core/Storage/s3Archive";
import { requireAdminApiSession } from "@/lib/adminAuth";
import { NextRequest, NextResponse } from "next/server";
export async function POST(req: NextRequest) {
const auth = await requireAdminApiSession(req);
if (!auth.valid) {
return auth.response;
}
let body: Record<string, unknown> = {};
try {
const text = await req.text();
if (text.trim()) {
const parsed = JSON.parse(text);
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
body = parsed as Record<string, unknown>;
}
}
} catch {
return NextResponse.json(
{ error: "Invalid JSON request payload.", code: "INVALID_JSON" },
{ status: 400 },
);
}
try {
const saved = await SettingsManager.getSettingsAsync();
const draftSecret =
typeof body.s3SecretAccessKey === "string" &&
body.s3SecretAccessKey.trim().length > 0
? body.s3SecretAccessKey.trim()
: "";
let effectiveSecret = draftSecret;
if (!effectiveSecret && !body.clearS3SecretKey) {
effectiveSecret = decryptSecret(saved.s3SecretAccessKeyEnc);
}
const config: S3ArchiveConfig = {
endpoint:
typeof body.s3Endpoint === "string"
? body.s3Endpoint.trim()
: saved.s3Endpoint,
region:
typeof body.s3Region === "string" && body.s3Region.trim()
? body.s3Region.trim()
: saved.s3Region || "auto",
bucket:
typeof body.s3Bucket === "string"
? body.s3Bucket.trim()
: saved.s3Bucket,
prefix:
typeof body.s3Prefix === "string"
? body.s3Prefix.trim()
: saved.s3Prefix,
accessKeyId:
typeof body.s3AccessKeyId === "string"
? body.s3AccessKeyId.trim()
: saved.s3AccessKeyId,
secretAccessKey: effectiveSecret,
};
const result = await testS3Connection(config);
if (!result.ok) {
return NextResponse.json(
{
success: false,
code: result.code || "S3_CONNECTION_FAILED",
error: result.error || "S3 connectivity check failed.",
...(result.status !== undefined ? { s3Status: result.status } : {}),
},
{ status: result.code === "S3_INVALID_CONFIG" ? 400 : 502 },
);
}
return NextResponse.json({
success: true,
endpoint: result.endpoint,
bucket: result.bucket,
status: result.status,
});
} catch (err: unknown) {
if (err instanceof SecretVaultError) {
return NextResponse.json(
{ success: false, code: err.code, error: err.message },
{ status: 400 },
);
}
const message =
err instanceof Error ? err.message : "S3 connectivity check failed";
return NextResponse.json(
{ success: false, code: "S3_CONNECTION_FAILED", error: message },
{ status: 500 },
);
}
}