feat(client): implement per-client storage and security controls
- Added storage quota, IP allowlisting, and expiration logic to client management APIs. - Updated CSV storage manager to support auto-compression and improved partition file parsing. - Integrated security checks into event ingestion and data retrieval routes.
This commit is contained in:
1 parent
89e4d68690
commit
2873b9904a
32 files changed
+6075
-241
No files matched your search
@@ -1,9 +1,21 @@
|
||||
import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
|
||||
import { SDKManager } from "@/core/sdk/sdkManager";
|
||||
import {
|
||||
validateAllowedIpsInput,
|
||||
validateClientExpiresAtInput,
|
||||
} from "@/core/Security/ipAllowlist";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
const SESSION_COOKIE = "datastack_admin_session";
|
||||
const ALLOWED_PATCH_KEYS = new Set(["action", "name", "rateLimit", "status"]);
|
||||
const ALLOWED_PATCH_KEYS = new Set([
|
||||
"action",
|
||||
"name",
|
||||
"rateLimit",
|
||||
"storageQuotaMb",
|
||||
"allowedIps",
|
||||
"expiresAt",
|
||||
"status",
|
||||
]);
|
||||
|
||||
export async function GET(
|
||||
req: NextRequest,
|
||||
@@ -125,7 +137,14 @@ export async function PATCH(
|
||||
}
|
||||
}
|
||||
|
||||
const updates: { name?: string; rateLimit?: number; status?: "active" | "revoked" } = {};
|
||||
const updates: {
|
||||
name?: string;
|
||||
rateLimit?: number;
|
||||
storageQuotaMb?: number;
|
||||
allowedIps?: string;
|
||||
expiresAt?: string | null;
|
||||
status?: "active" | "revoked";
|
||||
} = {};
|
||||
if (body.name !== undefined) {
|
||||
if (
|
||||
typeof body.name !== "string" ||
|
||||
@@ -153,6 +172,46 @@ export async function PATCH(
|
||||
}
|
||||
updates.rateLimit = body.rateLimit;
|
||||
}
|
||||
if (body.storageQuotaMb !== undefined) {
|
||||
if (
|
||||
typeof body.storageQuotaMb !== "number" ||
|
||||
!Number.isInteger(body.storageQuotaMb) ||
|
||||
body.storageQuotaMb < 0 ||
|
||||
body.storageQuotaMb > 1048576
|
||||
) {
|
||||
return NextResponse.json(
|
||||
{ error: "storageQuotaMb must be an integer between 0 and 1048576.", code: "VALIDATION_ERROR" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
updates.storageQuotaMb = body.storageQuotaMb;
|
||||
}
|
||||
if (body.allowedIps !== undefined) {
|
||||
if (typeof body.allowedIps !== "string") {
|
||||
return NextResponse.json(
|
||||
{ error: "allowedIps must be a string.", code: "VALIDATION_ERROR" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
const ipValidation = validateAllowedIpsInput(body.allowedIps);
|
||||
if (!ipValidation.valid) {
|
||||
return NextResponse.json(
|
||||
{ error: ipValidation.error || "Invalid allowedIps.", code: "VALIDATION_ERROR" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
updates.allowedIps = ipValidation.normalized;
|
||||
}
|
||||
if (body.expiresAt !== undefined) {
|
||||
const expValidation = validateClientExpiresAtInput(body.expiresAt);
|
||||
if (!expValidation.valid) {
|
||||
return NextResponse.json(
|
||||
{ error: expValidation.error || "Invalid expiresAt.", code: "VALIDATION_ERROR" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
updates.expiresAt = expValidation.normalized;
|
||||
}
|
||||
if (body.status !== undefined) {
|
||||
if (body.status !== "active" && body.status !== "revoked") {
|
||||
return NextResponse.json(
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
|
||||
import { SDKManager } from "@/core/sdk/sdkManager";
|
||||
import {
|
||||
validateAllowedIpsInput,
|
||||
validateClientExpiresAtInput,
|
||||
} from "@/core/Security/ipAllowlist";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
const SESSION_COOKIE = "datastack_admin_session";
|
||||
@@ -58,7 +62,14 @@ export async function POST(req: NextRequest) {
|
||||
}
|
||||
|
||||
try {
|
||||
const { clientId, name, rateLimit } = body;
|
||||
const {
|
||||
clientId,
|
||||
name,
|
||||
rateLimit,
|
||||
storageQuotaMb,
|
||||
allowedIps,
|
||||
expiresAt,
|
||||
} = body;
|
||||
|
||||
if (typeof clientId !== "string" || typeof name !== "string" || !clientId.trim() || !name.trim()) {
|
||||
return NextResponse.json(
|
||||
@@ -83,6 +94,44 @@ export async function POST(req: NextRequest) {
|
||||
);
|
||||
}
|
||||
|
||||
const parsedQuotaMb = storageQuotaMb === undefined ? 0 : Number(storageQuotaMb);
|
||||
if (!Number.isInteger(parsedQuotaMb) || parsedQuotaMb < 0 || parsedQuotaMb > 1048576) {
|
||||
return NextResponse.json(
|
||||
{ error: "storageQuotaMb must be an integer between 0 and 1048576.", code: "VALIDATION_ERROR" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
let normalizedAllowedIps = "";
|
||||
if (allowedIps !== undefined && allowedIps !== null) {
|
||||
if (typeof allowedIps !== "string") {
|
||||
return NextResponse.json(
|
||||
{ error: "allowedIps must be a string.", code: "VALIDATION_ERROR" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
const ipValidation = validateAllowedIpsInput(allowedIps);
|
||||
if (!ipValidation.valid) {
|
||||
return NextResponse.json(
|
||||
{ error: ipValidation.error || "Invalid allowedIps.", code: "VALIDATION_ERROR" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
normalizedAllowedIps = ipValidation.normalized;
|
||||
}
|
||||
|
||||
let normalizedExpiresAt: string | null = null;
|
||||
if (expiresAt !== undefined) {
|
||||
const expValidation = validateClientExpiresAtInput(expiresAt);
|
||||
if (!expValidation.valid) {
|
||||
return NextResponse.json(
|
||||
{ error: expValidation.error || "Invalid expiresAt.", code: "VALIDATION_ERROR" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
normalizedExpiresAt = expValidation.normalized;
|
||||
}
|
||||
|
||||
const existing = await SDKManager.getClientAsync(safeClientId);
|
||||
if (existing) {
|
||||
return NextResponse.json(
|
||||
@@ -95,6 +144,9 @@ export async function POST(req: NextRequest) {
|
||||
clientId: safeClientId,
|
||||
name: name.trim().slice(0, 120),
|
||||
rateLimit: parsedRateLimit,
|
||||
storageQuotaMb: parsedQuotaMb,
|
||||
allowedIps: normalizedAllowedIps,
|
||||
expiresAt: normalizedExpiresAt,
|
||||
});
|
||||
|
||||
return NextResponse.json({ client: newClient }, { status: 201 });
|
||||
|
||||
+12
-6
@@ -6,7 +6,6 @@ import { CSVStorageManager } from "@/core/Storage/csvStorage";
|
||||
import { requireAdminApiSession } from "@/lib/adminAuth";
|
||||
import { parse } from "csv-parse";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { createReadStream } from "node:fs";
|
||||
import { lstat, readdir } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
@@ -135,10 +134,10 @@ export async function GET(request: NextRequest) {
|
||||
try {
|
||||
filenames = (await readdir(clientDir))
|
||||
.filter((filename) => {
|
||||
if (!/^\d{4}-\d{2}-\d{2}(?:-part\d+)?\.csv$/.test(filename)) {
|
||||
if (!/^\d{4}-\d{2}-\d{2}(?:-part\d+)?(?:-gz\d+)?\.csv(?:\.gz)?$/.test(filename)) {
|
||||
return false;
|
||||
}
|
||||
const fileDate = filename.slice(0, 10);
|
||||
const fileDate = CSVStorageManager.extractDateFromFilename(filename);
|
||||
if ((from && fileDate < from) || (to && fileDate > to)) {
|
||||
return false;
|
||||
}
|
||||
@@ -150,7 +149,7 @@ export async function GET(request: NextRequest) {
|
||||
}
|
||||
|
||||
for (const filename of filenames) {
|
||||
const fileDate = filename.slice(0, 10);
|
||||
const fileDate = CSVStorageManager.extractDateFromFilename(filename);
|
||||
|
||||
const filePath = path.join(clientDir, filename);
|
||||
let fileLstat;
|
||||
@@ -161,10 +160,13 @@ export async function GET(request: NextRequest) {
|
||||
}
|
||||
if (fileLstat.isSymbolicLink() || !fileLstat.isFile()) continue;
|
||||
|
||||
const fileStream = createReadStream(filePath);
|
||||
const fileStream = CSVStorageManager.openPartitionTextStream(filePath);
|
||||
const parser = fileStream.pipe(
|
||||
parse({ columns: true, skip_empty_lines: true, bom: true }),
|
||||
);
|
||||
fileStream.on("error", (err) => {
|
||||
if (!parser.destroyed) parser.destroy(err);
|
||||
});
|
||||
|
||||
try {
|
||||
for await (const record of parser as AsyncIterable<
|
||||
@@ -180,7 +182,11 @@ export async function GET(request: NextRequest) {
|
||||
if (event.eventId) addIfMatching(event);
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn(`[API /data] Failed reading CSV partition ${filePath}:`, err);
|
||||
console.warn(`[API /data] Failed reading partition ${filePath}:`, err);
|
||||
if (filename.endsWith(".csv.gz")) {
|
||||
const detail = err instanceof Error ? err.message : String(err);
|
||||
throw new Error(`Corrupt .csv.gz partition (${filename}): ${detail}`);
|
||||
}
|
||||
} finally {
|
||||
if (!parser.destroyed) parser.destroy();
|
||||
if (!fileStream.destroyed) fileStream.destroy();
|
||||
|
||||
+81
-4
@@ -3,6 +3,10 @@ import { verifyRequestAdminSession } from '@/core/auth/adminUsers';
|
||||
import { AuthService } from '@/core/auth/jwt';
|
||||
import { DataBufferEngine, REDIS_BUFFER_KEY } from '@/core/data/buffer';
|
||||
import { SDKManager } from '@/core/sdk/sdkManager';
|
||||
import {
|
||||
isClientIpAllowed,
|
||||
isClientTokenExpired,
|
||||
} from '@/core/Security/ipAllowlist';
|
||||
import {
|
||||
SecurityEngine,
|
||||
getInternalIngestionHeaders,
|
||||
@@ -15,8 +19,7 @@ import {
|
||||
import { parse } from 'csv-parse';
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import crypto from 'node:crypto';
|
||||
import { createReadStream } from 'node:fs';
|
||||
import { requireAdminApiSession } from '@/lib/adminAuth';
|
||||
import { extractClientIp, requireAdminApiSession } from '@/lib/adminAuth';
|
||||
import type { BufferedBucket, BufferedSummary } from '@/lib/vconUiHelpers';
|
||||
|
||||
const SESSION_COOKIE = 'datastack_admin_session';
|
||||
@@ -233,10 +236,13 @@ async function getRecentStoredEvents(
|
||||
|
||||
for (const file of files) {
|
||||
const fileTail: DataEvent[] = [];
|
||||
const fileStream = createReadStream(file.filePath);
|
||||
const fileStream = CSVStorageManager.openPartitionTextStream(file.filePath);
|
||||
const parser = fileStream.pipe(
|
||||
parse({ columns: true, skip_empty_lines: true, bom: true }),
|
||||
);
|
||||
fileStream.on('error', (err) => {
|
||||
if (!parser.destroyed) parser.destroy(err);
|
||||
});
|
||||
|
||||
try {
|
||||
for await (const record of parser as AsyncIterable<
|
||||
@@ -254,7 +260,7 @@ async function getRecentStoredEvents(
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn(`[API /events] Failed reading CSV partition ${file.filePath}:`, err);
|
||||
console.warn(`[API /events] Failed reading partition ${file.filePath}:`, err);
|
||||
} finally {
|
||||
if (!parser.destroyed) parser.destroy();
|
||||
if (!fileStream.destroyed) fileStream.destroy();
|
||||
@@ -476,6 +482,77 @@ export async function POST(req: NextRequest) {
|
||||
{ status: 401 },
|
||||
);
|
||||
}
|
||||
if (isClientTokenExpired(registeredClient.expiresAt)) {
|
||||
SettingsManager.triggerWebhookAlert({
|
||||
event: 'CLIENT_TOKEN_EXPIRED',
|
||||
severity: 'warning',
|
||||
title: 'Expired Client Token Rejected',
|
||||
message: `Client '${clientId}' attempted HTTP ingestion with an expired token (expiresAt=${registeredClient.expiresAt}).`,
|
||||
clientId,
|
||||
details: { expiresAt: registeredClient.expiresAt },
|
||||
});
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: `CLIENT_TOKEN_EXPIRED: Client '${clientId}' token expired at ${registeredClient.expiresAt}. Rotate or extend client expiration.`,
|
||||
code: 'CLIENT_TOKEN_EXPIRED',
|
||||
},
|
||||
{ status: 401 },
|
||||
);
|
||||
}
|
||||
if (registeredClient.allowedIps && registeredClient.allowedIps.trim()) {
|
||||
const clientIp = extractClientIp(req);
|
||||
if (!isClientIpAllowed(clientIp, registeredClient.allowedIps)) {
|
||||
SettingsManager.triggerWebhookAlert({
|
||||
event: 'CLIENT_IP_FORBIDDEN',
|
||||
severity: 'warning',
|
||||
title: 'Unauthorized Client IP Blocked',
|
||||
message: `Client '${clientId}' request from IP '${clientIp}' was blocked by allowedIps policy.`,
|
||||
clientId,
|
||||
details: {
|
||||
clientIp,
|
||||
allowedIps: registeredClient.allowedIps,
|
||||
},
|
||||
});
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: `CLIENT_IP_FORBIDDEN: Request IP '${clientIp}' is not authorized for client '${clientId}'.`,
|
||||
code: 'CLIENT_IP_FORBIDDEN',
|
||||
},
|
||||
{ status: 403 },
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const storageAdmission = await CSVStorageManager.checkStorageAdmissionAsync(
|
||||
clientId,
|
||||
registeredClient.storageQuotaMb,
|
||||
settings.diskPauseThresholdPct,
|
||||
);
|
||||
if (!storageAdmission.allowed) {
|
||||
if (storageAdmission.code) {
|
||||
SettingsManager.triggerWebhookAlert({
|
||||
event: storageAdmission.code,
|
||||
severity:
|
||||
storageAdmission.code === 'DISK_SPACE_LOW' ? 'critical' : 'warning',
|
||||
title:
|
||||
storageAdmission.code === 'DISK_SPACE_LOW'
|
||||
? 'VPS Disk Pause Threshold Reached'
|
||||
: 'Client Storage Quota Exceeded',
|
||||
message:
|
||||
storageAdmission.reason ||
|
||||
`Storage admission paused (${storageAdmission.code}) for client '${clientId}'.`,
|
||||
clientId,
|
||||
});
|
||||
}
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: storageAdmission.reason,
|
||||
code: storageAdmission.code,
|
||||
},
|
||||
{ status: 507 },
|
||||
);
|
||||
}
|
||||
|
||||
const effectiveClientRate =
|
||||
|
||||
+44
-16
@@ -12,7 +12,7 @@ import { NextRequest, NextResponse } from "next/server";
|
||||
import { createReadStream } from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
type ExportFormat = "csv" | "json";
|
||||
type ExportFormat = "csv" | "json" | "ndjson";
|
||||
type CsvRecord = Record<string, string>;
|
||||
type StreamChunk = string | Uint8Array;
|
||||
|
||||
@@ -46,10 +46,13 @@ function toByteStream(chunks: AsyncIterable<StreamChunk>) {
|
||||
async function* readFileRecords(
|
||||
file: StorageFileMeta,
|
||||
): AsyncGenerator<CsvRecord> {
|
||||
const fileStream = createReadStream(file.filePath);
|
||||
const fileStream = CSVStorageManager.openPartitionTextStream(file.filePath);
|
||||
const parser = fileStream.pipe(
|
||||
parse({ columns: true, skip_empty_lines: true, bom: true }),
|
||||
);
|
||||
fileStream.on("error", (err) => {
|
||||
if (!parser.destroyed) parser.destroy(err);
|
||||
});
|
||||
|
||||
try {
|
||||
for await (const record of parser as AsyncIterable<CsvRecord>) {
|
||||
@@ -72,13 +75,13 @@ function makeExportStream(
|
||||
|
||||
if (format === "json") {
|
||||
yield "[";
|
||||
} else if (includeHeader) {
|
||||
} else if (format === "csv" && includeHeader) {
|
||||
yield `${fields.join(",")}\n`;
|
||||
}
|
||||
|
||||
for (const file of files) {
|
||||
for await (const record of readFileRecords(file)) {
|
||||
if (format === "json") {
|
||||
if (format === "json" || format === "ndjson") {
|
||||
const payload = record.payload ?? "";
|
||||
let parsedPayload: unknown = payload;
|
||||
try {
|
||||
@@ -86,7 +89,15 @@ function makeExportStream(
|
||||
} catch {
|
||||
// Keep non-JSON payloads as strings.
|
||||
}
|
||||
yield `${isFirst ? "" : ","}${JSON.stringify({ ...record, payload: parsedPayload })}`;
|
||||
const serialized = JSON.stringify({
|
||||
...record,
|
||||
payload: parsedPayload,
|
||||
});
|
||||
if (format === "ndjson") {
|
||||
yield `${serialized}\n`;
|
||||
} else {
|
||||
yield `${isFirst ? "" : ","}${serialized}`;
|
||||
}
|
||||
} else {
|
||||
const line = fields
|
||||
.map((field) => `"${String(record[field] ?? "").replaceAll('"', '""')}"`)
|
||||
@@ -224,9 +235,14 @@ export async function GET(request: NextRequest) {
|
||||
: SDKManager.normalizeClientId(rawClientId) || "all";
|
||||
const sanitizedClientId =
|
||||
clientId === "all" ? "all" : CSVStorageManager.sanitizeClientId(clientId);
|
||||
const filename = searchParams.get("filename");
|
||||
const rawFormat = searchParams.get("format");
|
||||
const format: ExportFormat = rawFormat === "json" ? "json" : "csv";
|
||||
const filename = searchParams.get("filename") || searchParams.get("file");
|
||||
const rawFormat = (searchParams.get("format") || "csv").toLowerCase();
|
||||
const format: ExportFormat =
|
||||
rawFormat === "json"
|
||||
? "json"
|
||||
: rawFormat === "ndjson"
|
||||
? "ndjson"
|
||||
: "csv";
|
||||
const includeHeader = searchParams.get("includeHeader") !== "false";
|
||||
|
||||
let files = await CSVStorageManager.listAllFilesAsync();
|
||||
@@ -259,21 +275,31 @@ export async function GET(request: NextRequest) {
|
||||
);
|
||||
}
|
||||
const file = matches[0];
|
||||
const baseStem = file.filename.replace(/\.csv(?:\.gz)?$/i, "");
|
||||
const downloadName =
|
||||
format === "json"
|
||||
? `${file.filename.replace(/\.csv$/i, "")}.json`
|
||||
: file.filename;
|
||||
? `${baseStem}.json`
|
||||
: format === "ndjson"
|
||||
? `${baseStem}.ndjson`
|
||||
: !includeHeader && file.compressed
|
||||
? `${baseStem}.csv`
|
||||
: file.filename;
|
||||
const bodyStream =
|
||||
format === "json" || !includeHeader
|
||||
format === "json" || format === "ndjson" || !includeHeader
|
||||
? makeExportStream([file], format, includeHeader)
|
||||
: streamFile(file);
|
||||
const contentType =
|
||||
format === "json"
|
||||
? "application/json; charset=utf-8"
|
||||
: format === "ndjson"
|
||||
? "application/x-ndjson; charset=utf-8"
|
||||
: file.compressed && includeHeader
|
||||
? "application/gzip"
|
||||
: "text/csv; charset=utf-8";
|
||||
return new NextResponse(bodyStream, {
|
||||
status: 200,
|
||||
headers: {
|
||||
"Content-Type":
|
||||
format === "json"
|
||||
? "application/json; charset=utf-8"
|
||||
: "text/csv; charset=utf-8",
|
||||
"Content-Type": contentType,
|
||||
"Content-Disposition": `attachment; filename="${downloadName.replace(/"/g, "")}"`,
|
||||
"Cache-Control": "no-store",
|
||||
},
|
||||
@@ -294,7 +320,9 @@ export async function GET(request: NextRequest) {
|
||||
"Content-Type":
|
||||
format === "json"
|
||||
? "application/json; charset=utf-8"
|
||||
: "text/csv; charset=utf-8",
|
||||
: format === "ndjson"
|
||||
? "application/x-ndjson; charset=utf-8"
|
||||
: "text/csv; charset=utf-8",
|
||||
"Content-Disposition": `attachment; filename="${baseName}_${datePart}.${format}"`,
|
||||
"Cache-Control": "no-store",
|
||||
},
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
import { SettingsManager } from "@/core/adminSettings/settingsManager";
|
||||
import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
|
||||
import { DataBufferEngine } from "@/core/data/buffer";
|
||||
import { SDKManager } from "@/core/sdk/sdkManager";
|
||||
import { getInternalIngestionHeaders } from "@/core/Security/security";
|
||||
import { CSVStorageManager } from "@/core/Storage/csvStorage";
|
||||
import { ADMIN_SESSION_COOKIE } from "@/lib/adminAuth";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { createHash, timingSafeEqual } from "node:crypto";
|
||||
|
||||
function verifyBearerTokenConstantTime(
|
||||
expectedToken: string,
|
||||
providedToken: string,
|
||||
): boolean {
|
||||
const expected = expectedToken.trim();
|
||||
const provided = providedToken.trim();
|
||||
if (!expected || !provided) return false;
|
||||
const expectedDigest = createHash("sha256")
|
||||
.update(expected, "utf8")
|
||||
.digest();
|
||||
const providedDigest = createHash("sha256")
|
||||
.update(provided, "utf8")
|
||||
.digest();
|
||||
return timingSafeEqual(expectedDigest, providedDigest);
|
||||
}
|
||||
|
||||
function escapePrometheusLabel(raw: string): string {
|
||||
return String(raw || "")
|
||||
.replace(/\\/g, "\\\\")
|
||||
.replace(/\n/g, "\\n")
|
||||
.replace(/"/g, '\\"');
|
||||
}
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
try {
|
||||
let authorized = false;
|
||||
|
||||
// 1. Check Bearer token (or ?token= query param) against encrypted Prometheus token
|
||||
const authHeader = req.headers.get("authorization") || "";
|
||||
const bearerCandidate = authHeader.toLowerCase().startsWith("bearer ")
|
||||
? authHeader.slice(7).trim()
|
||||
: req.nextUrl.searchParams.get("token")?.trim() || "";
|
||||
|
||||
const promConfig = await SettingsManager.getDecryptedPrometheusConfigAsync();
|
||||
if (
|
||||
promConfig.enabled &&
|
||||
promConfig.token.trim().length > 0 &&
|
||||
bearerCandidate.length > 0 &&
|
||||
verifyBearerTokenConstantTime(promConfig.token, bearerCandidate)
|
||||
) {
|
||||
authorized = true;
|
||||
}
|
||||
|
||||
// 2. Fallback: valid Admin Console session cookie
|
||||
if (!authorized) {
|
||||
const sessionCookie = req.cookies.get(ADMIN_SESSION_COOKIE)?.value;
|
||||
if (sessionCookie) {
|
||||
const adminAuth = await verifyRequestAdminSession(sessionCookie);
|
||||
if (adminAuth.valid) {
|
||||
authorized = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!authorized) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error:
|
||||
"PROMETHEUS_UNAUTHORIZED: Valid bearer scrape token or admin session required.",
|
||||
code: "PROMETHEUS_UNAUTHORIZED",
|
||||
},
|
||||
{ status: 401 },
|
||||
);
|
||||
}
|
||||
|
||||
const [clients, storageStats, diskHealth, bufferStats] = await Promise.all([
|
||||
SDKManager.listClientsAsync(),
|
||||
CSVStorageManager.getStatsAsync(),
|
||||
CSVStorageManager.getDiskSpaceHealthAsync(),
|
||||
DataBufferEngine.getBufferStatsAsync(),
|
||||
]);
|
||||
|
||||
let activeSockets = 0;
|
||||
let bridgeUp = 0;
|
||||
let remoteBufferSize = 0;
|
||||
|
||||
const headers = getInternalIngestionHeaders();
|
||||
if (headers["x-datastack-internal-secret"]) {
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||
).replace(/\/$/, "");
|
||||
try {
|
||||
const res = await fetch(`${ingestionBaseUrl}/internal/metrics`, {
|
||||
headers,
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(1000),
|
||||
});
|
||||
if (res.ok) {
|
||||
bridgeUp = 1;
|
||||
const data = (await res.json()) as {
|
||||
activeSockets?: number;
|
||||
bufferSize?: number;
|
||||
};
|
||||
activeSockets = Math.max(0, Number(data.activeSockets) || 0);
|
||||
remoteBufferSize = Math.max(0, Number(data.bufferSize) || 0);
|
||||
}
|
||||
} catch {
|
||||
bridgeUp = 0;
|
||||
}
|
||||
}
|
||||
|
||||
const totalBufferedEvents =
|
||||
bufferStats.localBufferSize +
|
||||
(bufferStats.redisEnabled && bufferStats.redisConnected
|
||||
? bufferStats.redisBufferSize
|
||||
: remoteBufferSize);
|
||||
|
||||
let activeClientsCount = 0;
|
||||
let revokedClientsCount = 0;
|
||||
for (const c of clients) {
|
||||
if (c.status === "revoked") {
|
||||
revokedClientsCount += 1;
|
||||
} else {
|
||||
activeClientsCount += 1;
|
||||
}
|
||||
}
|
||||
|
||||
const lines: string[] = [
|
||||
"# HELP datastack_up Whether the DataStack control plane is up (1 = healthy).",
|
||||
"# TYPE datastack_up gauge",
|
||||
"datastack_up 1",
|
||||
"# HELP datastack_ingestion_bridge_up Whether the Port 4000 Socket.IO ingestion engine is reachable (1 = reachable, 0 = unreachable).",
|
||||
"# TYPE datastack_ingestion_bridge_up gauge",
|
||||
`datastack_ingestion_bridge_up ${bridgeUp}`,
|
||||
"# HELP datastack_active_sockets Currently connected Socket.IO SDK clients on Port 4000.",
|
||||
"# TYPE datastack_active_sockets gauge",
|
||||
`datastack_active_sockets ${activeSockets}`,
|
||||
"# HELP datastack_buffer_events_total Total events currently queued in memory or Redis awaiting CSV flush.",
|
||||
"# TYPE datastack_buffer_events_total gauge",
|
||||
`datastack_buffer_events_total ${totalBufferedEvents}`,
|
||||
"# HELP datastack_storage_files_total Total CSV and gzipped CSV partition files on disk.",
|
||||
"# TYPE datastack_storage_files_total gauge",
|
||||
`datastack_storage_files_total ${storageStats.totalFiles}`,
|
||||
"# HELP datastack_storage_bytes_total Total bytes occupied by CSV and gzipped CSV partition files on disk.",
|
||||
"# TYPE datastack_storage_bytes_total gauge",
|
||||
`datastack_storage_bytes_total ${storageStats.totalSizeBytes}`,
|
||||
"# HELP datastack_storage_rows_total Total telemetry rows persisted across all partition files.",
|
||||
"# TYPE datastack_storage_rows_total gauge",
|
||||
`datastack_storage_rows_total ${storageStats.totalRows}`,
|
||||
"# HELP datastack_disk_used_percent Percentage of underlying filesystem capacity currently used.",
|
||||
"# TYPE datastack_disk_used_percent gauge",
|
||||
`datastack_disk_used_percent ${diskHealth.usedPct}`,
|
||||
"# HELP datastack_disk_free_bytes Free bytes available on the underlying storage volume.",
|
||||
"# TYPE datastack_disk_free_bytes gauge",
|
||||
`datastack_disk_free_bytes ${diskHealth.freeBytes}`,
|
||||
"# HELP datastack_disk_total_bytes Total capacity in bytes of the underlying storage volume.",
|
||||
"# TYPE datastack_disk_total_bytes gauge",
|
||||
`datastack_disk_total_bytes ${diskHealth.totalBytes}`,
|
||||
"# HELP datastack_clients_total Number of registered SDK clients partitioned by status.",
|
||||
"# TYPE datastack_clients_total gauge",
|
||||
`datastack_clients_total{status="active"} ${activeClientsCount}`,
|
||||
`datastack_clients_total{status="revoked"} ${revokedClientsCount}`,
|
||||
"# HELP datastack_client_events_total Cumulative ingested events per registered client.",
|
||||
"# TYPE datastack_client_events_total counter",
|
||||
];
|
||||
|
||||
for (const c of clients) {
|
||||
const safeId = escapePrometheusLabel(c.clientId);
|
||||
lines.push(
|
||||
`datastack_client_events_total{client_id="${safeId}"} ${Math.max(0, Number(c.totalEvents) || 0)}`,
|
||||
);
|
||||
}
|
||||
|
||||
lines.push(
|
||||
"# HELP datastack_client_storage_bytes Disk usage in bytes per registered client.",
|
||||
"# TYPE datastack_client_storage_bytes gauge",
|
||||
);
|
||||
|
||||
const perClientBytes = new Map<string, number>();
|
||||
for (const f of storageStats.files) {
|
||||
perClientBytes.set(
|
||||
f.clientId,
|
||||
(perClientBytes.get(f.clientId) || 0) + f.sizeBytes,
|
||||
);
|
||||
}
|
||||
for (const c of clients) {
|
||||
const safeId = escapePrometheusLabel(c.clientId);
|
||||
const usedBytes = perClientBytes.get(c.clientId) || 0;
|
||||
lines.push(
|
||||
`datastack_client_storage_bytes{client_id="${safeId}"} ${usedBytes}`,
|
||||
);
|
||||
}
|
||||
|
||||
const body = `${lines.join("\n")}\n`;
|
||||
return new NextResponse(body, {
|
||||
status: 200,
|
||||
headers: {
|
||||
"Content-Type": "text/plain; version=0.0.4; charset=utf-8",
|
||||
"Cache-Control": "no-store, no-cache, must-revalidate",
|
||||
},
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
const message =
|
||||
err instanceof Error ? err.message : "Failed to generate Prometheus metrics";
|
||||
return NextResponse.json({ error: message }, { status: 500 });
|
||||
}
|
||||
}
|
||||
+190
-3
@@ -1,8 +1,9 @@
|
||||
import {
|
||||
SettingsManager,
|
||||
type AdminSettings,
|
||||
type AdminSettingsUpdateInput,
|
||||
} from "@/core/adminSettings/settingsManager";
|
||||
import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
|
||||
import { SecretVaultError } from "@/core/Security/secretVault";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
const SESSION_COOKIE = "datastack_admin_session";
|
||||
@@ -16,6 +17,24 @@ const ALLOWED_SETTING_KEYS = new Set([
|
||||
"enableIdempotency",
|
||||
"autoPruneDays",
|
||||
"jwtSecret",
|
||||
"compressAfterDays",
|
||||
"enableS3Backup",
|
||||
"s3Endpoint",
|
||||
"s3Region",
|
||||
"s3Bucket",
|
||||
"s3Prefix",
|
||||
"s3AccessKeyId",
|
||||
"s3SecretAccessKey",
|
||||
"clearS3SecretKey",
|
||||
"diskPauseThresholdPct",
|
||||
"enableWebhookAlerts",
|
||||
"webhookUrl",
|
||||
"webhookSecret",
|
||||
"clearWebhookSecret",
|
||||
"webhookCooldownSec",
|
||||
"enablePrometheusMetrics",
|
||||
"prometheusMetricsToken",
|
||||
"clearPrometheusToken",
|
||||
]);
|
||||
|
||||
function parseBoundedInt(
|
||||
@@ -33,6 +52,21 @@ function parseBoundedInt(
|
||||
return raw;
|
||||
}
|
||||
|
||||
function parseOptionalString(
|
||||
raw: unknown,
|
||||
field: string,
|
||||
maxLength: number,
|
||||
): string {
|
||||
if (typeof raw !== "string") {
|
||||
throw new Error(`${field} must be a string.`);
|
||||
}
|
||||
const trimmed = raw.trim();
|
||||
if (trimmed.length > maxLength) {
|
||||
throw new Error(`${field} must be at most ${maxLength} characters.`);
|
||||
}
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
try {
|
||||
const auth = await verifyRequestAdminSession(
|
||||
@@ -109,7 +143,7 @@ export async function POST(req: NextRequest) {
|
||||
);
|
||||
}
|
||||
|
||||
const validated: Partial<AdminSettings> = {};
|
||||
const validated: AdminSettingsUpdateInput = {};
|
||||
try {
|
||||
if (body.maxEventsPerSec !== undefined) {
|
||||
validated.maxEventsPerSec = parseBoundedInt(
|
||||
@@ -159,14 +193,161 @@ export async function POST(req: NextRequest) {
|
||||
3650,
|
||||
);
|
||||
}
|
||||
if (body.compressAfterDays !== undefined) {
|
||||
validated.compressAfterDays = parseBoundedInt(
|
||||
body.compressAfterDays,
|
||||
"compressAfterDays",
|
||||
0,
|
||||
3650,
|
||||
);
|
||||
}
|
||||
if (body.diskPauseThresholdPct !== undefined) {
|
||||
const pct = parseBoundedInt(
|
||||
body.diskPauseThresholdPct,
|
||||
"diskPauseThresholdPct",
|
||||
0,
|
||||
99,
|
||||
);
|
||||
if (pct !== 0 && pct < 50) {
|
||||
throw new Error(
|
||||
"diskPauseThresholdPct must be 0 (disabled) or between 50 and 99.",
|
||||
);
|
||||
}
|
||||
validated.diskPauseThresholdPct = pct;
|
||||
}
|
||||
if (body.enableIdempotency !== undefined) {
|
||||
if (typeof body.enableIdempotency !== "boolean") {
|
||||
throw new Error("enableIdempotency must be a boolean.");
|
||||
}
|
||||
validated.enableIdempotency = body.enableIdempotency;
|
||||
}
|
||||
if (body.enableS3Backup !== undefined) {
|
||||
if (typeof body.enableS3Backup !== "boolean") {
|
||||
throw new Error("enableS3Backup must be a boolean.");
|
||||
}
|
||||
validated.enableS3Backup = body.enableS3Backup;
|
||||
}
|
||||
if (body.s3Endpoint !== undefined) {
|
||||
const ep = parseOptionalString(body.s3Endpoint, "s3Endpoint", 512);
|
||||
if (ep) {
|
||||
let parsedEp: URL;
|
||||
try {
|
||||
parsedEp = new URL(
|
||||
/^https?:\/\//i.test(ep) ? ep : `https://${ep}`,
|
||||
);
|
||||
} catch {
|
||||
throw new Error("s3Endpoint must be a valid http(s) URL.");
|
||||
}
|
||||
if (parsedEp.protocol !== "http:" && parsedEp.protocol !== "https:") {
|
||||
throw new Error("s3Endpoint must use http:// or https://.");
|
||||
}
|
||||
}
|
||||
validated.s3Endpoint = ep;
|
||||
}
|
||||
if (body.s3Region !== undefined) {
|
||||
validated.s3Region =
|
||||
parseOptionalString(body.s3Region, "s3Region", 64) || "auto";
|
||||
}
|
||||
if (body.s3Bucket !== undefined) {
|
||||
validated.s3Bucket = parseOptionalString(
|
||||
body.s3Bucket,
|
||||
"s3Bucket",
|
||||
128,
|
||||
);
|
||||
}
|
||||
if (body.s3Prefix !== undefined) {
|
||||
validated.s3Prefix = parseOptionalString(
|
||||
body.s3Prefix,
|
||||
"s3Prefix",
|
||||
256,
|
||||
);
|
||||
}
|
||||
if (body.s3AccessKeyId !== undefined) {
|
||||
validated.s3AccessKeyId = parseOptionalString(
|
||||
body.s3AccessKeyId,
|
||||
"s3AccessKeyId",
|
||||
256,
|
||||
);
|
||||
}
|
||||
if (body.s3SecretAccessKey !== undefined) {
|
||||
validated.s3SecretAccessKey = parseOptionalString(
|
||||
body.s3SecretAccessKey,
|
||||
"s3SecretAccessKey",
|
||||
512,
|
||||
);
|
||||
}
|
||||
if (body.clearS3SecretKey !== undefined) {
|
||||
if (typeof body.clearS3SecretKey !== "boolean") {
|
||||
throw new Error("clearS3SecretKey must be a boolean.");
|
||||
}
|
||||
validated.clearS3SecretKey = body.clearS3SecretKey;
|
||||
}
|
||||
if (body.enableWebhookAlerts !== undefined) {
|
||||
if (typeof body.enableWebhookAlerts !== "boolean") {
|
||||
throw new Error("enableWebhookAlerts must be a boolean.");
|
||||
}
|
||||
validated.enableWebhookAlerts = body.enableWebhookAlerts;
|
||||
}
|
||||
if (body.webhookUrl !== undefined) {
|
||||
const wh = parseOptionalString(body.webhookUrl, "webhookUrl", 1024);
|
||||
if (wh) {
|
||||
let parsedWh: URL;
|
||||
try {
|
||||
parsedWh = new URL(wh);
|
||||
} catch {
|
||||
throw new Error("webhookUrl must be a valid http(s) URL.");
|
||||
}
|
||||
if (parsedWh.protocol !== "http:" && parsedWh.protocol !== "https:") {
|
||||
throw new Error("webhookUrl must use http:// or https://.");
|
||||
}
|
||||
}
|
||||
validated.webhookUrl = wh;
|
||||
}
|
||||
if (body.webhookSecret !== undefined) {
|
||||
validated.webhookSecret = parseOptionalString(
|
||||
body.webhookSecret,
|
||||
"webhookSecret",
|
||||
512,
|
||||
);
|
||||
}
|
||||
if (body.clearWebhookSecret !== undefined) {
|
||||
if (typeof body.clearWebhookSecret !== "boolean") {
|
||||
throw new Error("clearWebhookSecret must be a boolean.");
|
||||
}
|
||||
validated.clearWebhookSecret = body.clearWebhookSecret;
|
||||
}
|
||||
if (body.webhookCooldownSec !== undefined) {
|
||||
validated.webhookCooldownSec = parseBoundedInt(
|
||||
body.webhookCooldownSec,
|
||||
"webhookCooldownSec",
|
||||
10,
|
||||
86400,
|
||||
);
|
||||
}
|
||||
if (body.enablePrometheusMetrics !== undefined) {
|
||||
if (typeof body.enablePrometheusMetrics !== "boolean") {
|
||||
throw new Error("enablePrometheusMetrics must be a boolean.");
|
||||
}
|
||||
validated.enablePrometheusMetrics = body.enablePrometheusMetrics;
|
||||
}
|
||||
if (body.prometheusMetricsToken !== undefined) {
|
||||
validated.prometheusMetricsToken = parseOptionalString(
|
||||
body.prometheusMetricsToken,
|
||||
"prometheusMetricsToken",
|
||||
512,
|
||||
);
|
||||
}
|
||||
if (body.clearPrometheusToken !== undefined) {
|
||||
if (typeof body.clearPrometheusToken !== "boolean") {
|
||||
throw new Error("clearPrometheusToken must be a boolean.");
|
||||
}
|
||||
validated.clearPrometheusToken = body.clearPrometheusToken;
|
||||
}
|
||||
if (body.jwtSecret !== undefined) {
|
||||
if (typeof body.jwtSecret !== "string" || body.jwtSecret.trim().length < 32) {
|
||||
if (
|
||||
typeof body.jwtSecret !== "string" ||
|
||||
body.jwtSecret.trim().length < 32
|
||||
) {
|
||||
throw new Error(
|
||||
"jwtSecret must be a string with at least 32 characters.",
|
||||
);
|
||||
@@ -208,6 +389,12 @@ export async function POST(req: NextRequest) {
|
||||
: {}),
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof SecretVaultError) {
|
||||
return NextResponse.json(
|
||||
{ error: err.message, code: err.code },
|
||||
{ status: 500 },
|
||||
);
|
||||
}
|
||||
const message =
|
||||
err instanceof Error ? err.message : "Settings update failed";
|
||||
return NextResponse.json({ error: message }, { status: 500 });
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
import { SettingsManager } from "@/core/adminSettings/settingsManager";
|
||||
import { dispatchWebhookAlertAsync } from "@/core/alerts/webhookNotifier";
|
||||
import { SecretVaultError } from "@/core/Security/secretVault";
|
||||
import { requireAdminApiSession } from "@/lib/adminAuth";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
const auth = await requireAdminApiSession(req);
|
||||
if (!auth.valid) {
|
||||
return auth.response;
|
||||
}
|
||||
|
||||
let body: Record<string, unknown> = {};
|
||||
try {
|
||||
const text = await req.text();
|
||||
if (text.trim()) {
|
||||
const parsed = JSON.parse(text);
|
||||
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
||||
body = parsed as Record<string, unknown>;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid JSON request payload.", code: "INVALID_JSON" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const savedConfig = await SettingsManager.getDecryptedWebhookConfigAsync();
|
||||
const webhookUrl =
|
||||
typeof body.webhookUrl === "string"
|
||||
? body.webhookUrl.trim()
|
||||
: savedConfig.webhookUrl.trim();
|
||||
const clearWebhookSecret = body.clearWebhookSecret === true;
|
||||
const webhookSecret = clearWebhookSecret
|
||||
? ""
|
||||
: typeof body.webhookSecret === "string" && body.webhookSecret.trim()
|
||||
? body.webhookSecret.trim()
|
||||
: savedConfig.webhookSecret;
|
||||
|
||||
if (!webhookUrl) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
code: "VALIDATION_ERROR",
|
||||
error: "Webhook URL is required to send a test alert.",
|
||||
},
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const parsedUrl = new URL(webhookUrl);
|
||||
if (parsedUrl.protocol !== "http:" && parsedUrl.protocol !== "https:") {
|
||||
throw new Error("Webhook URL must use http:// or https://.");
|
||||
}
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
code: "VALIDATION_ERROR",
|
||||
error: "Webhook URL must be a valid http:// or https:// endpoint.",
|
||||
},
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
const result = await dispatchWebhookAlertAsync(
|
||||
{
|
||||
event: "TEST_ALERT",
|
||||
severity: "info",
|
||||
title: "Webhook Verification",
|
||||
message: "Test alert dispatched from DataStack Admin Console.",
|
||||
details: {
|
||||
triggeredBy: auth.auth.user?.email || "admin",
|
||||
},
|
||||
},
|
||||
{
|
||||
enabled: true,
|
||||
webhookUrl,
|
||||
webhookSecret,
|
||||
cooldownSec: savedConfig.cooldownSec,
|
||||
},
|
||||
{ bypassCooldown: true },
|
||||
);
|
||||
|
||||
if (!result.ok) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
code: "WEBHOOK_DELIVERY_FAILED",
|
||||
error: result.error || "Webhook target failed to accept test alert.",
|
||||
status: result.status,
|
||||
},
|
||||
{ status: 502 },
|
||||
);
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
status: result.status,
|
||||
message: `Webhook test alert delivered (HTTP ${result.status ?? 200}).`,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof SecretVaultError) {
|
||||
return NextResponse.json(
|
||||
{ success: false, code: err.code, error: err.message },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
const message =
|
||||
err instanceof Error ? err.message : "Webhook test alert failed";
|
||||
return NextResponse.json(
|
||||
{ success: false, code: "WEBHOOK_DELIVERY_FAILED", error: message },
|
||||
{ status: 502 },
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
import { SettingsManager } from "@/core/adminSettings/settingsManager";
|
||||
import { SecretVaultError } from "@/core/Security/secretVault";
|
||||
import { CSVStorageManager } from "@/core/Storage/csvStorage";
|
||||
import {
|
||||
buildS3ObjectKey,
|
||||
normalizeS3EndpointUrl,
|
||||
uploadFileToS3,
|
||||
} from "@/core/Storage/s3Archive";
|
||||
import { requireAdminApiSession } from "@/lib/adminAuth";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
const auth = await requireAdminApiSession(req);
|
||||
if (!auth.valid) {
|
||||
return auth.response;
|
||||
}
|
||||
|
||||
let body: Record<string, unknown> = {};
|
||||
try {
|
||||
const text = await req.text();
|
||||
if (text.trim()) {
|
||||
const parsed = JSON.parse(text);
|
||||
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
||||
body = parsed as Record<string, unknown>;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid JSON request payload.", code: "INVALID_JSON" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const settings = await SettingsManager.getSettingsAsync();
|
||||
const s3Config = await SettingsManager.getDecryptedS3ConfigAsync();
|
||||
|
||||
if (!s3Config.bucket.trim() || !s3Config.accessKeyId.trim() || !s3Config.secretAccessKey.trim()) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
code: "S3_INVALID_CONFIG",
|
||||
error: "Configure S3 Bucket, Access Key ID, and Secret Access Key before archiving.",
|
||||
},
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
normalizeS3EndpointUrl(s3Config.endpoint, s3Config.region);
|
||||
} catch (urlErr: unknown) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
code: "S3_INVALID_CONFIG",
|
||||
error: urlErr instanceof Error ? urlErr.message : "Invalid S3 endpoint URL.",
|
||||
},
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
const pruneAfterUpload = body.pruneAfterUpload === true;
|
||||
const rawOlderThanDays =
|
||||
typeof body.olderThanDays === "number" && Number.isFinite(body.olderThanDays)
|
||||
? Math.max(0, Math.floor(body.olderThanDays))
|
||||
: settings.autoPruneDays > 0
|
||||
? settings.autoPruneDays
|
||||
: 0;
|
||||
const shouldCompressFirst =
|
||||
body.compressFirst !== false && settings.compressAfterDays > 0;
|
||||
|
||||
let compressedCount = 0;
|
||||
if (shouldCompressFirst) {
|
||||
compressedCount = await CSVStorageManager.compressOldPartitionsAsync(
|
||||
settings.compressAfterDays,
|
||||
);
|
||||
}
|
||||
|
||||
const archiveResult =
|
||||
await CSVStorageManager.pruneOrArchiveExpiredPartitionsAsync(
|
||||
settings.autoPruneDays,
|
||||
{
|
||||
archiveOnlyWithoutDelete: !pruneAfterUpload,
|
||||
maxCutoffDays: rawOlderThanDays,
|
||||
beforeDeleteFile: async (file) => {
|
||||
const key = buildS3ObjectKey(
|
||||
s3Config.prefix,
|
||||
file.clientId,
|
||||
file.filename,
|
||||
);
|
||||
const uploadRes = await uploadFileToS3(
|
||||
s3Config,
|
||||
file.filePath,
|
||||
key,
|
||||
);
|
||||
if (!uploadRes.ok) {
|
||||
throw new Error(
|
||||
uploadRes.error || `Failed to upload ${file.filename}`,
|
||||
);
|
||||
}
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
if (
|
||||
archiveResult.errors.length > 0 &&
|
||||
archiveResult.archivedCount === 0
|
||||
) {
|
||||
SettingsManager.triggerWebhookAlert({
|
||||
event: "S3_ARCHIVE_FAILED",
|
||||
severity: "critical",
|
||||
title: "S3 Cold Archive Upload Failed",
|
||||
message: `Manual S3 archive failed: ${archiveResult.errors[0]}`,
|
||||
details: { errorCount: archiveResult.errors.length },
|
||||
});
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
code: "S3_ARCHIVE_FAILED",
|
||||
error: archiveResult.errors[0],
|
||||
compressedCount,
|
||||
...archiveResult,
|
||||
},
|
||||
{ status: 502 },
|
||||
);
|
||||
}
|
||||
|
||||
if (archiveResult.errors.length > 0) {
|
||||
SettingsManager.triggerWebhookAlert({
|
||||
event: "S3_ARCHIVE_FAILED",
|
||||
severity: "warning",
|
||||
title: "Partial S3 Cold Archive Errors",
|
||||
message: `S3 archive encountered ${archiveResult.errors.length} error(s): ${archiveResult.errors[0]}`,
|
||||
details: { errorCount: archiveResult.errors.length },
|
||||
});
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
success: archiveResult.errors.length === 0,
|
||||
compressedCount,
|
||||
...archiveResult,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof SecretVaultError) {
|
||||
return NextResponse.json(
|
||||
{ success: false, code: err.code, error: err.message },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
const message =
|
||||
err instanceof Error ? err.message : "Manual S3 archive failed";
|
||||
SettingsManager.triggerWebhookAlert({
|
||||
event: "S3_ARCHIVE_FAILED",
|
||||
severity: "critical",
|
||||
title: "S3 Cold Archive Upload Failed",
|
||||
message,
|
||||
});
|
||||
return NextResponse.json(
|
||||
{ success: false, code: "S3_ARCHIVE_FAILED", error: message },
|
||||
{ status: 500 },
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -23,7 +23,10 @@ export async function GET(req: NextRequest) {
|
||||
? CSVStorageManager.sanitizeClientId(clientId)
|
||||
: undefined;
|
||||
|
||||
const allFiles = await CSVStorageManager.listAllFilesAsync();
|
||||
const [allFiles, diskUsage] = await Promise.all([
|
||||
CSVStorageManager.listAllFilesAsync(),
|
||||
CSVStorageManager.getDiskUsageAsync(),
|
||||
]);
|
||||
let files = allFiles;
|
||||
|
||||
if (clientId) {
|
||||
@@ -39,7 +42,7 @@ export async function GET(req: NextRequest) {
|
||||
|
||||
const stats = CSVStorageManager.formatStorageStats(files);
|
||||
|
||||
return NextResponse.json({ stats, files });
|
||||
return NextResponse.json({ stats, files, diskUsage });
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : 'Storage stats failed';
|
||||
return NextResponse.json({ error: message }, { status: 500 });
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
import { SettingsManager } from "@/core/adminSettings/settingsManager";
|
||||
import {
|
||||
decryptSecret,
|
||||
SecretVaultError,
|
||||
} from "@/core/Security/secretVault";
|
||||
import {
|
||||
testS3Connection,
|
||||
type S3ArchiveConfig,
|
||||
} from "@/core/Storage/s3Archive";
|
||||
import { requireAdminApiSession } from "@/lib/adminAuth";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
const auth = await requireAdminApiSession(req);
|
||||
if (!auth.valid) {
|
||||
return auth.response;
|
||||
}
|
||||
|
||||
let body: Record<string, unknown> = {};
|
||||
try {
|
||||
const text = await req.text();
|
||||
if (text.trim()) {
|
||||
const parsed = JSON.parse(text);
|
||||
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
||||
body = parsed as Record<string, unknown>;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid JSON request payload.", code: "INVALID_JSON" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const saved = await SettingsManager.getSettingsAsync();
|
||||
const draftSecret =
|
||||
typeof body.s3SecretAccessKey === "string" &&
|
||||
body.s3SecretAccessKey.trim().length > 0
|
||||
? body.s3SecretAccessKey.trim()
|
||||
: "";
|
||||
|
||||
let effectiveSecret = draftSecret;
|
||||
if (!effectiveSecret && !body.clearS3SecretKey) {
|
||||
effectiveSecret = decryptSecret(saved.s3SecretAccessKeyEnc);
|
||||
}
|
||||
|
||||
const config: S3ArchiveConfig = {
|
||||
endpoint:
|
||||
typeof body.s3Endpoint === "string"
|
||||
? body.s3Endpoint.trim()
|
||||
: saved.s3Endpoint,
|
||||
region:
|
||||
typeof body.s3Region === "string" && body.s3Region.trim()
|
||||
? body.s3Region.trim()
|
||||
: saved.s3Region || "auto",
|
||||
bucket:
|
||||
typeof body.s3Bucket === "string"
|
||||
? body.s3Bucket.trim()
|
||||
: saved.s3Bucket,
|
||||
prefix:
|
||||
typeof body.s3Prefix === "string"
|
||||
? body.s3Prefix.trim()
|
||||
: saved.s3Prefix,
|
||||
accessKeyId:
|
||||
typeof body.s3AccessKeyId === "string"
|
||||
? body.s3AccessKeyId.trim()
|
||||
: saved.s3AccessKeyId,
|
||||
secretAccessKey: effectiveSecret,
|
||||
};
|
||||
|
||||
const result = await testS3Connection(config);
|
||||
if (!result.ok) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
code: result.code || "S3_CONNECTION_FAILED",
|
||||
error: result.error || "S3 connectivity check failed.",
|
||||
...(result.status !== undefined ? { s3Status: result.status } : {}),
|
||||
},
|
||||
{ status: result.code === "S3_INVALID_CONFIG" ? 400 : 502 },
|
||||
);
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
endpoint: result.endpoint,
|
||||
bucket: result.bucket,
|
||||
status: result.status,
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
if (err instanceof SecretVaultError) {
|
||||
return NextResponse.json(
|
||||
{ success: false, code: err.code, error: err.message },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
const message =
|
||||
err instanceof Error ? err.message : "S3 connectivity check failed";
|
||||
return NextResponse.json(
|
||||
{ success: false, code: "S3_CONNECTION_FAILED", error: message },
|
||||
{ status: 500 },
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user