feat: implement R4 production hardening

- Add configurable ingestion batch item limits
- Improve client serialization and sanitization
- Add graceful handling for corrupt GZIP partitions in data API
- Expand admin session token scope
- Update project documentation for R4 status
This commit is contained in:
Team Dark Devil committed 2026-11-21 22:15:32 +06:00
1 parent 8132087085
commit ae9b554189
59 files changed
+16879 -1074

No files matched your search

+13 -5
View File
@@ -1,4 +1,4 @@
import { listAdminAuditLogsAsync } from "@/core/auth/auditLog";
import { listAdminAuditLogsWithStatusAsync } from "@/core/auth/auditLog";
import { requireAdminApiSession } from "@/lib/adminAuth";
import { NextRequest, NextResponse } from "next/server";
@@ -22,17 +22,25 @@ export async function GET(req: NextRequest) {
const search =
searchParams.get("q") || searchParams.get("search") || undefined;
const logs = await listAdminAuditLogsAsync({
const result = await listAdminAuditLogsWithStatusAsync({
limit,
action,
actorEmail,
search,
});
return NextResponse.json({
logs,
total: logs.length,
const response = NextResponse.json({
logs: result.logs,
total: result.logs.length,
storageMode: result.storageMode,
degraded: result.degraded,
...(result.code ? { code: result.code } : {}),
...(result.warning ? { warning: result.warning } : {}),
});
if (result.degraded) {
response.headers.set("X-DataStack-Audit-Degraded", "1");
}
return response;
} catch (err: unknown) {
const message =
err instanceof Error ? err.message : "Unable to fetch audit logs.";
+2
View File
@@ -145,6 +145,8 @@ export async function POST(request: NextRequest) {
sessionSecret,
Date.now() + SESSION_SECONDS * 1000,
1,
"owner",
"owner-1",
);
const response = NextResponse.json({ success: true });
response.cookies.set(
+19 -9
View File
@@ -17,6 +17,18 @@ const ALLOWED_PATCH_KEYS = new Set([
"status",
]);
function serializeClientResponse<
T extends { secret?: string; token?: string },
>(client: T, options: { includeToken: boolean }) {
const { secret, token, ...clientSummary } = client;
void secret;
return {
...clientSummary,
tokenPreview: token ? `${token.slice(0, 12)}...` : "",
...(options.includeToken && token ? { token } : {}),
};
}
export async function GET(
req: NextRequest,
{ params }: { params: Promise<{ id: string }> },
@@ -36,14 +48,8 @@ export async function GET(
if (!client) {
return NextResponse.json({ error: "Client not found" }, { status: 404 });
}
const { secret, token, ...clientSummary } = client;
void secret;
return NextResponse.json({
client: {
...clientSummary,
tokenPreview: token ? `${token.slice(0, 12)}...` : "",
...(includeToken ? { token } : {}),
},
client: serializeClientResponse(client, { includeToken }),
});
} catch (err: unknown) {
const message = err instanceof Error ? err.message : "Failed to load client";
@@ -132,7 +138,9 @@ export async function PATCH(
},
});
return NextResponse.json({ client: refreshed });
return NextResponse.json({
client: serializeClientResponse(refreshed, { includeToken: true }),
});
} catch (err: unknown) {
const message =
err instanceof Error ? err.message : "Token regeneration failed";
@@ -277,7 +285,9 @@ export async function PATCH(
metadata: updates,
});
return NextResponse.json({ client: updated });
return NextResponse.json({
client: serializeClientResponse(updated, { includeToken: false }),
});
} catch (err: unknown) {
const message = err instanceof Error ? err.message : "Update failed";
const isDbWrite = message.includes("DATABASE_WRITE_FAILED");
+16 -1
View File
@@ -17,6 +17,13 @@ interface DataEvent {
status: "stored" | "buffered";
}
interface CorruptPartitionWarning {
clientId: string;
filename: string;
code: "CORRUPT_GZIP_PARTITION";
error: string;
}
function parseTimestampMs(raw: string): number {
const trimmed = String(raw || "").trim();
if (/^\d+(\.\d+)?$/.test(trimmed)) {
@@ -62,6 +69,7 @@ export async function GET(request: NextRequest) {
const offset = (page - 1) * pageSize;
const selectedRows: DataEvent[] = [];
const eventIds = new Set<string>();
const corruptPartitions: CorruptPartitionWarning[] = [];
let total = 0;
const addIfMatching = (event: DataEvent) => {
@@ -185,7 +193,12 @@ export async function GET(request: NextRequest) {
console.warn(`[API /data] Failed reading partition ${filePath}:`, err);
if (filename.endsWith(".csv.gz")) {
const detail = err instanceof Error ? err.message : String(err);
throw new Error(`Corrupt .csv.gz partition (${filename}): ${detail}`);
corruptPartitions.push({
clientId: storedClientId,
filename,
code: "CORRUPT_GZIP_PARTITION",
error: `Corrupt .csv.gz partition (${filename}): ${detail}`,
});
}
} finally {
if (!parser.destroyed) parser.destroy();
@@ -249,6 +262,8 @@ export async function GET(request: NextRequest) {
page,
pageSize,
pageCount: Math.max(1, Math.ceil(total / pageSize)),
corruptPartitions,
warnings: corruptPartitions.length,
});
} catch (error) {
const message =
+46 -11
View File
@@ -25,6 +25,21 @@ const SINGLE_EVENT_ENVELOPE_KEYS = new Set(['eventId', 'timestamp', 'data', 'cli
const MAX_SINGLE_CLIENT_SCAN_FILES = 10;
const MAX_ALL_CLIENTS_PER_CLIENT_FILES = 2;
const MAX_ALL_CLIENTS_TOTAL_SCAN_FILES = 50;
const DEFAULT_MAX_BATCH_ITEMS = 1000;
const MIN_BATCH_ITEMS = 1;
const MAX_BATCH_ITEMS_CEILING = 10000;
function resolveMaxBatchItems(): number {
const raw = process.env.DATASTACK_MAX_BATCH_ITEMS;
if (raw === undefined || raw.trim() === '') {
return DEFAULT_MAX_BATCH_ITEMS;
}
const parsed = Math.floor(Number(raw));
if (!Number.isFinite(parsed)) {
return DEFAULT_MAX_BATCH_ITEMS;
}
return Math.min(MAX_BATCH_ITEMS_CEILING, Math.max(MIN_BATCH_ITEMS, parsed));
}
function generateFallbackEventId(): string {
return `evt_${Date.now()}_${crypto.randomBytes(6).toString('hex')}`;
@@ -513,6 +528,36 @@ export async function POST(req: NextRequest) {
}
}
// Detect if this is a batch array or direct file-records ingestion request
const batchItems: unknown[] | null = Array.isArray(body)
? body
: Array.isArray(bodyObj.batch)
? (bodyObj.batch as unknown[])
: Array.isArray(bodyObj.records)
? (bodyObj.records as unknown[])
: bodyObj.data === undefined && Array.isArray(bodyObj.events)
? (bodyObj.events as unknown[])
: null;
if (batchItems !== null) {
const maxBatchItems = resolveMaxBatchItems();
if (batchItems.length > maxBatchItems) {
return NextResponse.json(
{
success: false,
accepted: 0,
processed: 0,
rejected: batchItems.length,
duplicates: 0,
code: 'BATCH_LIMIT_EXCEEDED',
error: `BATCH_LIMIT_EXCEEDED: Batch size (${batchItems.length}) exceeds maximum allowed ${maxBatchItems} items per request.`,
maxBatchItems,
},
{ status: 400 },
);
}
}
const storageAdmission = await CSVStorageManager.checkStorageAdmissionAsync(
clientId,
registeredClient.storageQuotaMb,
@@ -550,18 +595,8 @@ export async function POST(req: NextRequest) {
settings.maxEventsPerSec ||
100;
// Detect if this is a batch array or direct file-records ingestion request
const batchItems: unknown[] | null = Array.isArray(body)
? body
: Array.isArray(bodyObj.batch)
? (bodyObj.batch as unknown[])
: Array.isArray(bodyObj.records)
? (bodyObj.records as unknown[])
: bodyObj.data === undefined && Array.isArray(bodyObj.events)
? (bodyObj.events as unknown[])
: null;
if (batchItems !== null) {
let processed = 0;
let rejected = 0;
let duplicates = 0;
+40 -2
View File
@@ -1,4 +1,5 @@
import { SettingsManager } from "@/core/adminSettings/settingsManager";
import { recordAdminAuditLogAsync } from "@/core/auth/auditLog";
import { DataBufferEngine } from "@/core/data/buffer";
import { SDKManager } from "@/core/sdk/sdkManager";
import { getInternalIngestionHeaders } from "@/core/Security/security";
@@ -6,7 +7,7 @@ import {
CSVStorageManager,
type StorageFileMeta,
} from "@/core/Storage/csvStorage";
import { requireAdminApiSession } from "@/lib/adminAuth";
import { extractClientIp, requireAdminApiSession } from "@/lib/adminAuth";
import { parse } from "csv-parse";
import { NextRequest, NextResponse } from "next/server";
import { createReadStream } from "node:fs";
@@ -296,6 +297,24 @@ export async function GET(request: NextRequest) {
: file.compressed && includeHeader
? "application/gzip"
: "text/csv; charset=utf-8";
await recordAdminAuditLogAsync({
actorEmail: auth.auth.session.email,
actorRole: auth.auth.session.role || "owner",
action: "DATA_EXPORT_DOWNLOADED",
target: file.clientId,
ipAddress: extractClientIp(request),
metadata: {
mode: "single_file",
clientId: file.clientId,
filename: file.filename,
downloadName,
format,
includeHeader,
fileCount: 1,
},
});
return new NextResponse(bodyStream, {
status: 200,
headers: {
@@ -313,6 +332,25 @@ export async function GET(request: NextRequest) {
const safeClientId = sanitizedClientId === "all" ? "all" : sanitizedClientId;
const baseName =
clientId === "all" ? "datastack_export" : `${safeClientId}_export`;
const downloadName = `${baseName}_${datePart}.${format}`;
await recordAdminAuditLogAsync({
actorEmail: auth.auth.session.email,
actorRole: auth.auth.session.role || "owner",
action: "DATA_EXPORT_DOWNLOADED",
target: clientId,
ipAddress: extractClientIp(request),
metadata: {
mode: "filtered_range",
clientId,
from: from || null,
to: to || null,
format,
includeHeader,
fileCount: files.length,
filename: downloadName,
},
});
return new NextResponse(makeExportStream(files, format, includeHeader), {
status: 200,
@@ -323,7 +361,7 @@ export async function GET(request: NextRequest) {
: format === "ndjson"
? "application/x-ndjson; charset=utf-8"
: "text/csv; charset=utf-8",
"Content-Disposition": `attachment; filename="${baseName}_${datePart}.${format}"`,
"Content-Disposition": `attachment; filename="${downloadName}"`,
"Cache-Control": "no-store",
},
});
+115
View File
@@ -0,0 +1,115 @@
import { requireAdminApiSession } from '@/lib/adminAuth';
import { NextRequest, NextResponse } from 'next/server';
export const dynamic = 'force-dynamic';
function resolveIngestionBaseUrl(): string {
return (process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000').replace(
/\/+$/,
'',
);
}
function resolveFilesProxyStatus(req: NextRequest): {
proxyMode: 'nginx' | 'standalone';
wsSupported: boolean;
wsHint: string | null;
} {
const configuredMode = process.env.DATASTACK_PROXY_MODE?.trim().toLowerCase();
const hasReverseProxyHeaders = Boolean(
req.headers.get('x-forwarded-host') ||
req.headers.get('x-real-ip') ||
req.headers.get('x-forwarded-proto'),
);
if (configuredMode === 'nginx' || hasReverseProxyHeaders) {
return {
proxyMode: 'nginx',
wsSupported: true,
wsHint: null,
};
}
return {
proxyMode: 'standalone',
wsSupported: false,
wsHint:
'Direct Next.js HTTP rewrite active without Nginx WebSocket proxying. Files Manager operates over HTTP polling.',
};
}
export async function GET(req: NextRequest) {
const auth = await requireAdminApiSession(req);
if (!auth.valid) {
return auth.response;
}
const role = auth.auth.session.role || 'owner';
const readOnly = role === 'viewer';
const proxyInfo = resolveFilesProxyStatus(req);
const targetUrl = `${resolveIngestionBaseUrl()}/vcon/files-engine/_status`;
try {
const cookieHeader = req.headers.get('cookie') || '';
const upstream = await fetch(targetUrl, {
method: 'GET',
headers: cookieHeader ? { cookie: cookieHeader } : undefined,
cache: 'no-store',
signal: AbortSignal.timeout(2500),
});
if (upstream.status === 401) {
return NextResponse.json(
{ ok: false, offline: false, error: 'Authentication required.' },
{ status: 401, headers: { 'Cache-Control': 'no-store, private' } },
);
}
const payload = (await upstream.json().catch(() => ({}))) as {
ok?: boolean;
engine?: string;
role?: string;
readOnly?: boolean;
error?: string;
};
if (upstream.ok && payload.ok) {
return NextResponse.json(
{
ok: true,
offline: false,
engine: payload.engine || 'cloudcmd',
role: payload.role || role,
readOnly: Boolean(payload.readOnly ?? readOnly),
...proxyInfo,
},
{ status: 200, headers: { 'Cache-Control': 'no-store, private' } },
);
}
return NextResponse.json(
{
ok: false,
offline: true,
role,
readOnly,
...proxyInfo,
error:
payload.error ||
`Files Manager engine returned HTTP ${upstream.status}. Ensure the Port 4000 ingestion server is running.`,
},
{ status: 200, headers: { 'Cache-Control': 'no-store, private' } },
);
} catch {
return NextResponse.json(
{
ok: false,
offline: true,
role,
readOnly,
...proxyInfo,
error:
'Files Manager engine on Port 4000 is not reachable. Start both services with "npm run dev" / "npm run dev:all" or Docker Compose.',
},
{ status: 200, headers: { 'Cache-Control': 'no-store, private' } },
);
}
}
+39 -17
View File
@@ -4,6 +4,10 @@ import {
} from "@/core/adminSettings/settingsManager";
import { recordAdminAuditLogAsync } from "@/core/auth/auditLog";
import { SecretVaultError } from "@/core/Security/secretVault";
import {
SSRFBlockedError,
validateOutboundUrlSync,
} from "@/core/Security/ssrfGuard";
import { extractClientIp, requireAdminApiSession } from "@/lib/adminAuth";
import { NextRequest, NextResponse } from "next/server";
@@ -209,16 +213,20 @@ export async function POST(req: NextRequest) {
if (body.s3Endpoint !== undefined) {
const ep = parseOptionalString(body.s3Endpoint, "s3Endpoint", 512);
if (ep) {
let parsedEp: URL;
try {
parsedEp = new URL(
/^https?:\/\//i.test(ep) ? ep : `https://${ep}`,
const normalizedEp = /^https?:\/\//i.test(ep) ? ep : `https://${ep}`;
const check = validateOutboundUrlSync(normalizedEp, {
fieldName: "s3Endpoint",
});
if (!check.allowed) {
if (check.code === "SSRF_BLOCKED_HOST") {
throw new SSRFBlockedError(
check.error || "s3Endpoint target host is blocked.",
check.hostname || "",
);
}
throw new Error(
check.error || "s3Endpoint must be a valid http(s) URL.",
);
} catch {
throw new Error("s3Endpoint must be a valid http(s) URL.");
}
if (parsedEp.protocol !== "http:" && parsedEp.protocol !== "https:") {
throw new Error("s3Endpoint must use http:// or https://.");
}
}
validated.s3Endpoint = ep;
@@ -270,14 +278,19 @@ export async function POST(req: NextRequest) {
if (body.webhookUrl !== undefined) {
const wh = parseOptionalString(body.webhookUrl, "webhookUrl", 1024);
if (wh) {
let parsedWh: URL;
try {
parsedWh = new URL(wh);
} catch {
throw new Error("webhookUrl must be a valid http(s) URL.");
}
if (parsedWh.protocol !== "http:" && parsedWh.protocol !== "https:") {
throw new Error("webhookUrl must use http:// or https://.");
const check = validateOutboundUrlSync(wh, {
fieldName: "webhookUrl",
});
if (!check.allowed) {
if (check.code === "SSRF_BLOCKED_HOST") {
throw new SSRFBlockedError(
check.error || "webhookUrl target host is blocked.",
check.hostname || "",
);
}
throw new Error(
check.error || "webhookUrl must be a valid http(s) URL.",
);
}
}
validated.webhookUrl = wh;
@@ -334,6 +347,15 @@ export async function POST(req: NextRequest) {
validated.jwtSecret = body.jwtSecret.trim();
}
} catch (validationErr: unknown) {
if (validationErr instanceof SSRFBlockedError) {
return NextResponse.json(
{
error: validationErr.message,
code: validationErr.code,
},
{ status: 400 },
);
}
return NextResponse.json(
{
error:
+15 -10
View File
@@ -5,6 +5,7 @@ import {
decryptSecret,
SecretVaultError,
} from "@/core/Security/secretVault";
import { validateOutboundUrlSync } from "@/core/Security/ssrfGuard";
import { extractClientIp, requireAdminApiSession } from "@/lib/adminAuth";
import { NextRequest, NextResponse } from "next/server";
@@ -57,17 +58,20 @@ export async function POST(req: NextRequest) {
);
}
try {
const parsedUrl = new URL(webhookUrl);
if (parsedUrl.protocol !== "http:" && parsedUrl.protocol !== "https:") {
throw new Error("Webhook URL must use http:// or https://.");
}
} catch {
const urlCheck = validateOutboundUrlSync(webhookUrl, {
fieldName: "Webhook URL",
});
if (!urlCheck.allowed) {
return NextResponse.json(
{
success: false,
code: "VALIDATION_ERROR",
error: "Webhook URL must be a valid http:// or https:// endpoint.",
code:
urlCheck.code === "SSRF_BLOCKED_HOST"
? "SSRF_BLOCKED_HOST"
: "VALIDATION_ERROR",
error:
urlCheck.error ||
"Webhook URL must be a valid http:// or https:// endpoint.",
},
{ status: 400 },
);
@@ -105,14 +109,15 @@ export async function POST(req: NextRequest) {
});
if (!result.ok) {
const isSsrfBlocked = result.code === "SSRF_BLOCKED_HOST";
return NextResponse.json(
{
success: false,
code: "WEBHOOK_DELIVERY_FAILED",
code: isSsrfBlocked ? "SSRF_BLOCKED_HOST" : "WEBHOOK_DELIVERY_FAILED",
error: result.error || "Webhook target failed to accept test alert.",
status: result.status,
},
{ status: 502 },
{ status: isSsrfBlocked ? 400 : 502 },
);
}
+5 -1
View File
@@ -71,6 +71,10 @@ export async function POST(req: NextRequest) {
const result = await testS3Connection(config);
if (!result.ok) {
const isClientError =
result.code === "S3_INVALID_CONFIG" ||
result.code === "SSRF_BLOCKED_HOST" ||
result.code === "INVALID_URL";
return NextResponse.json(
{
success: false,
@@ -78,7 +82,7 @@ export async function POST(req: NextRequest) {
error: result.error || "S3 connectivity check failed.",
...(result.status !== undefined ? { s3Status: result.status } : {}),
},
{ status: result.code === "S3_INVALID_CONFIG" ? 400 : 502 },
{ status: isClientError ? 400 : 502 },
);
}