feat(auth): harden session management and security

- Implement server-side session revocation on logout
- Add IP extraction utility and admin setup error handling
- Improve error codes and response consistency for auth endpoints
- Refine landing page redirect logic to handle transient DB outages
- Update deployment documentation and configuration defaults
This commit is contained in:
Team Dark Devil committed 2026-11-19 20:22:01 +06:00
1 parent 935f12a1e9
commit 89e4d68690
81 files changed
+11530 -6916

No files matched your search

+12 -12
View File
@@ -4,27 +4,18 @@ import {
getSessionCookieOptions,
} from "@/core/auth/adminSession";
import {
AdminSetupRequiredError,
authenticateAdminCredentials,
clearAdminLoginAttempts,
consumeAdminLoginAttempt,
} from "@/core/auth/adminUsers";
import { extractClientIp } from "@/lib/adminAuth";
import { NextRequest, NextResponse } from "next/server";
const SESSION_COOKIE = "datastack_admin_session";
const SESSION_SECONDS = 8 * 60 * 60;
const REMEMBERED_SESSION_SECONDS = 30 * 24 * 60 * 60;
function extractClientIp(request: NextRequest) {
const forwardedFor = request.headers.get("x-forwarded-for");
if (forwardedFor) {
const first = forwardedFor.split(",")[0]?.trim();
if (first) return first;
}
const realIp = request.headers.get("x-real-ip")?.trim();
if (realIp) return realIp;
return "unknown";
}
export async function POST(request: NextRequest) {
const sessionSecret = getAdminSessionSecret();
@@ -91,7 +82,16 @@ export async function POST(request: NextRequest) {
let authResult: { authenticated: boolean; tokenVersion: number };
try {
authResult = await authenticateAdminCredentials(email, password);
} catch {
} catch (err) {
if (err instanceof AdminSetupRequiredError) {
return NextResponse.json(
{
error: "Initial admin setup is required before signing in.",
code: err.code,
},
{ status: 409 },
);
}
return NextResponse.json(
{ error: "Admin authentication is temporarily unavailable." },
{ status: 503 },
+16
View File
@@ -1,9 +1,25 @@
import { getSessionCookieOptions } from "@/core/auth/adminSession";
import {
revokeAdminSessions,
verifyRequestAdminSession,
} from "@/core/auth/adminUsers";
import { NextRequest, NextResponse } from "next/server";
const SESSION_COOKIE = "datastack_admin_session";
export async function POST(request: NextRequest) {
const token = request.cookies.get(SESSION_COOKIE)?.value;
if (token) {
try {
const check = await verifyRequestAdminSession(token);
if (check.valid) {
await revokeAdminSessions();
}
} catch {
// Still clear client cookie even if server revocation encounters transient DB error
}
}
const response = NextResponse.json({ success: true });
response.cookies.set(SESSION_COOKIE, "", {
...getSessionCookieOptions(request, 0),
+10 -3
View File
@@ -22,11 +22,13 @@ export async function GET(request: NextRequest) {
if (!auth.valid) {
const status = auth.reason === "UNAUTHORIZED" ? 401 : 503;
const code =
auth.reason === "UNAUTHORIZED" ? "SESSION_EXPIRED" : "AUTH_UNAVAILABLE";
const message =
auth.reason === "UNAUTHORIZED"
? "Authentication required."
: "Admin authentication is temporarily unavailable.";
return NextResponse.json({ error: message }, { status });
return NextResponse.json({ error: message, code }, { status });
}
try {
@@ -54,11 +56,13 @@ export async function PUT(request: NextRequest) {
if (!auth.valid) {
const status = auth.reason === "UNAUTHORIZED" ? 401 : 503;
const code =
auth.reason === "UNAUTHORIZED" ? "SESSION_EXPIRED" : "AUTH_UNAVAILABLE";
const message =
auth.reason === "UNAUTHORIZED"
? "Authentication required."
: "Admin authentication is temporarily unavailable.";
return NextResponse.json({ error: message }, { status });
return NextResponse.json({ error: message, code }, { status });
}
let body: Record<string, unknown>;
@@ -115,7 +119,10 @@ export async function PUT(request: NextRequest) {
});
if (!result.ok) {
return NextResponse.json({ error: result.error }, { status: 401 });
return NextResponse.json(
{ error: result.error, code: "INVALID_PASSWORD" },
{ status: 401 },
);
}
// Preserve remaining session duration (e.g., 30-day "Remember me" session)
+59 -7
View File
@@ -4,14 +4,20 @@ import {
getSessionCookieOptions,
} from "@/core/auth/adminSession";
import {
AdminDatabaseRequiredError,
clearAdminLoginAttempts,
consumeAdminLoginAttempt,
createInitialAdmin,
isAdminSetupComplete,
requiresPersistentAdminDatabase,
verifyAdminSetupKey,
} from "@/core/auth/adminUsers";
import { extractClientIp } from "@/lib/adminAuth";
import { NextRequest, NextResponse } from "next/server";
const SESSION_COOKIE = "datastack_admin_session";
const SESSION_SECONDS = 8 * 60 * 60;
const SETUP_THROTTLE_BUCKET = "__admin_setup__";
export const runtime = "nodejs";
export const dynamic = "force-dynamic";
@@ -29,6 +35,17 @@ export async function POST(request: NextRequest) {
);
}
if (requiresPersistentAdminDatabase()) {
return NextResponse.json(
{
error:
"PostgreSQL DATABASE_URL must be configured in production before initial admin setup.",
code: "DATABASE_NOT_CONFIGURED",
},
{ status: 503 },
);
}
let body: Record<string, unknown>;
try {
const parsed = await request.json();
@@ -61,12 +78,7 @@ export async function POST(request: NextRequest) {
{ status: 400 },
);
}
if (!verifyAdminSetupKey(setupKey)) {
return NextResponse.json(
{ error: "The installation key is invalid." },
{ status: 403 },
);
}
if (await isAdminSetupComplete()) {
return NextResponse.json(
{ error: "Initial admin setup has already been completed." },
@@ -74,6 +86,34 @@ export async function POST(request: NextRequest) {
);
}
const clientIp = extractClientIp(request);
const throttleDecision = await consumeAdminLoginAttempt(
SETUP_THROTTLE_BUCKET,
sessionSecret,
clientIp,
);
if (!throttleDecision.allowed) {
return NextResponse.json(
{
error: "Too many setup attempts. Try again later.",
code: "TOO_MANY_ATTEMPTS",
},
{
status: 429,
headers: {
"Retry-After": String(throttleDecision.retryAfterSeconds),
},
},
);
}
if (!verifyAdminSetupKey(setupKey)) {
return NextResponse.json(
{ error: "The installation key is invalid." },
{ status: 403 },
);
}
const created = await createInitialAdmin(email, password);
if (!created) {
return NextResponse.json(
@@ -82,6 +122,12 @@ export async function POST(request: NextRequest) {
);
}
await clearAdminLoginAttempts(
SETUP_THROTTLE_BUCKET,
sessionSecret,
clientIp,
).catch(() => undefined);
const token = await createAdminSessionToken(
email,
sessionSecret,
@@ -95,7 +141,13 @@ export async function POST(request: NextRequest) {
getSessionCookieOptions(request, SESSION_SECONDS),
);
return response;
} catch {
} catch (err) {
if (err instanceof AdminDatabaseRequiredError) {
return NextResponse.json(
{ error: err.message, code: err.code },
{ status: 503 },
);
}
return NextResponse.json(
{ error: "Admin setup is temporarily unavailable." },
{ status: 503 },
+160 -28
View File
@@ -3,6 +3,7 @@ import { SDKManager } from "@/core/sdk/sdkManager";
import { NextRequest, NextResponse } from "next/server";
const SESSION_COOKIE = "datastack_admin_session";
const ALLOWED_PATCH_KEYS = new Set(["action", "name", "rateLimit", "status"]);
export async function GET(
req: NextRequest,
@@ -19,20 +20,33 @@ export async function GET(
}
const { id } = await params;
const client = await SDKManager.getClientAsync(id);
if (!client) {
return NextResponse.json({ error: "Client not found" }, { status: 404 });
const normalizedId = SDKManager.normalizeClientId(id);
try {
const client = await SDKManager.getClientAsync(normalizedId);
if (!client) {
return NextResponse.json({ error: "Client not found" }, { status: 404 });
}
const { secret, token, ...clientSummary } = client;
void secret;
const includeToken = req.nextUrl.searchParams.get("includeToken") === "true";
return NextResponse.json({
client: {
...clientSummary,
tokenPreview: token ? `${token.slice(0, 12)}...` : "",
...(includeToken ? { token } : {}),
},
});
} catch (err: unknown) {
const message = err instanceof Error ? err.message : "Failed to load client";
const isDbError = message.includes("DATABASE_");
return NextResponse.json(
{
error: message,
code: isDbError ? "DATABASE_READ_FAILED" : "CLIENT_FETCH_FAILED",
},
{ status: isDbError ? 503 : 500 },
);
}
const { secret, token, ...clientSummary } = client;
void secret;
const includeToken = req.nextUrl.searchParams.get("includeToken") === "true";
return NextResponse.json({
client: {
...clientSummary,
tokenPreview: token ? `${token.slice(0, 12)}...` : "",
...(includeToken ? { token } : {}),
},
});
}
export async function PATCH(
@@ -50,29 +64,128 @@ export async function PATCH(
}
const { id } = await params;
try {
const body = await req.json();
const normalizedId = SDKManager.normalizeClientId(id);
// Special action: regenerate token
if (body.action === "regenerate_token") {
const refreshed = await SDKManager.regenerateTokenAsync(id);
let rawBody: unknown;
try {
rawBody = await req.json();
} catch {
return NextResponse.json(
{ error: "Invalid JSON request payload.", code: "INVALID_JSON" },
{ status: 400 },
);
}
if (!rawBody || typeof rawBody !== "object" || Array.isArray(rawBody)) {
return NextResponse.json(
{ error: "Update payload must be a JSON object.", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
const body = rawBody as Record<string, unknown>;
const unknownKeys = Object.keys(body).filter(
(key) => !ALLOWED_PATCH_KEYS.has(key),
);
if (unknownKeys.length > 0) {
return NextResponse.json(
{
error: `Unsupported client field(s): ${unknownKeys.join(", ")}`,
code: "VALIDATION_ERROR",
},
{ status: 400 },
);
}
// Special action: regenerate token
if (body.action !== undefined) {
if (body.action !== "regenerate_token") {
return NextResponse.json(
{ error: "Unsupported client action.", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
try {
const refreshed = await SDKManager.regenerateTokenAsync(normalizedId);
if (!refreshed) {
return NextResponse.json(
{ error: "Client not found" },
{ status: 404 },
);
return NextResponse.json({ error: "Client not found" }, { status: 404 });
}
return NextResponse.json({ client: refreshed });
} catch (err: unknown) {
const message =
err instanceof Error ? err.message : "Token regeneration failed";
const isDbWrite = message.includes("DATABASE_WRITE_FAILED");
return NextResponse.json(
{
error: message,
code: isDbWrite ? "DATABASE_WRITE_FAILED" : "TOKEN_REGENERATION_FAILED",
},
{ status: isDbWrite ? 503 : 500 },
);
}
}
const updated = await SDKManager.updateClientAsync(id, body);
const updates: { name?: string; rateLimit?: number; status?: "active" | "revoked" } = {};
if (body.name !== undefined) {
if (
typeof body.name !== "string" ||
body.name.trim().length < 1 ||
body.name.trim().length > 120
) {
return NextResponse.json(
{ error: "Client name must be a non-empty string (max 120 characters).", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
updates.name = body.name.trim();
}
if (body.rateLimit !== undefined) {
if (
typeof body.rateLimit !== "number" ||
!Number.isInteger(body.rateLimit) ||
body.rateLimit < 1 ||
body.rateLimit > 100000
) {
return NextResponse.json(
{ error: "rateLimit must be an integer between 1 and 100000.", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
updates.rateLimit = body.rateLimit;
}
if (body.status !== undefined) {
if (body.status !== "active" && body.status !== "revoked") {
return NextResponse.json(
{ error: "status must be 'active' or 'revoked'.", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
updates.status = body.status;
}
if (Object.keys(updates).length === 0) {
return NextResponse.json(
{ error: "No valid fields provided for update.", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
try {
const updated = await SDKManager.updateClientAsync(normalizedId, updates);
if (!updated) {
return NextResponse.json({ error: "Client not found" }, { status: 404 });
}
return NextResponse.json({ client: updated });
} catch (err: unknown) {
const message = err instanceof Error ? err.message : "Update failed";
return NextResponse.json({ error: message }, { status: 500 });
const isDbWrite = message.includes("DATABASE_WRITE_FAILED");
return NextResponse.json(
{
error: message,
code: isDbWrite ? "DATABASE_WRITE_FAILED" : "CLIENT_UPDATE_FAILED",
},
{ status: isDbWrite ? 503 : 500 },
);
}
}
@@ -91,10 +204,29 @@ export async function DELETE(
}
const { id } = await params;
const normalizedId = SDKManager.normalizeClientId(id);
const purgeFiles = req.nextUrl.searchParams.get("purgeFiles") === "true";
const deleted = await SDKManager.deleteClientAsync(id, { purgeFiles });
if (!deleted) {
return NextResponse.json({ error: "Client not found" }, { status: 404 });
try {
const result = await SDKManager.deleteClientWithStatsAsync(normalizedId, {
purgeFiles,
});
if (!result.deleted) {
return NextResponse.json({ error: "Client not found" }, { status: 404 });
}
return NextResponse.json({
success: true,
message: `Client ${normalizedId} deleted`,
purgedFiles: result.purgedFiles,
});
} catch (err: unknown) {
const message = err instanceof Error ? err.message : "Client deletion failed";
const isDbWrite = message.includes("DATABASE_WRITE_FAILED");
return NextResponse.json(
{
error: message,
code: isDbWrite ? "DATABASE_WRITE_FAILED" : "CLIENT_DELETE_FAILED",
},
{ status: isDbWrite ? 503 : 500 },
);
}
return NextResponse.json({ success: true, message: `Client ${id} deleted` });
}
+20 -8
View File
@@ -60,17 +60,29 @@ export async function POST(req: NextRequest) {
try {
const { clientId, name, rateLimit } = body;
if (!clientId || !name) {
if (typeof clientId !== "string" || typeof name !== "string" || !clientId.trim() || !name.trim()) {
return NextResponse.json(
{ error: "clientId and name are required" },
{ error: "clientId and name are required strings", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
const safeClientId = SDKManager.normalizeClientId(clientId);
if (!safeClientId) {
return NextResponse.json(
{ error: "clientId is invalid", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
const parsedRateLimit = rateLimit === undefined ? 100 : Number(rateLimit);
if (!Number.isInteger(parsedRateLimit) || parsedRateLimit < 1 || parsedRateLimit > 100000) {
return NextResponse.json(
{ error: "rateLimit must be an integer between 1 and 100000.", code: "VALIDATION_ERROR" },
{ status: 400 },
);
}
const safeClientId = String(clientId)
.trim()
.toLowerCase()
.replace(/[^a-z0-9_-]/g, "_");
const existing = await SDKManager.getClientAsync(safeClientId);
if (existing) {
return NextResponse.json(
@@ -81,8 +93,8 @@ export async function POST(req: NextRequest) {
const newClient = await SDKManager.registerClientAsync({
clientId: safeClientId,
name: String(name),
rateLimit: rateLimit ? Number(rateLimit) : 100,
name: name.trim().slice(0, 120),
rateLimit: parsedRateLimit,
});
return NextResponse.json({ client: newClient }, { status: 201 });
+88 -20
View File
@@ -1,10 +1,13 @@
import { SettingsManager } from "@/core/adminSettings/settingsManager";
import { DataBufferEngine } from "@/core/data/buffer";
import { SDKManager } from "@/core/sdk/sdkManager";
import { getInternalIngestionHeaders } from "@/core/Security/security";
import { CSVStorageManager } from "@/core/Storage/csvStorage";
import { requireAdminApiSession } from "@/lib/adminAuth";
import { parse } from "csv-parse";
import { NextRequest, NextResponse } from "next/server";
import { createReadStream, readdirSync, statSync } from "node:fs";
import { createReadStream } from "node:fs";
import { lstat, readdir } from "node:fs/promises";
import path from "node:path";
interface DataEvent {
@@ -15,10 +18,40 @@ interface DataEvent {
status: "stored" | "buffered";
}
function parseTimestampMs(raw: string): number {
const trimmed = String(raw || "").trim();
if (/^\d+(\.\d+)?$/.test(trimmed)) {
const num = Number(trimmed);
return num < 1e11 ? num * 1000 : num;
}
const parsed = Date.parse(trimmed);
return Number.isNaN(parsed) ? Number.NEGATIVE_INFINITY : parsed;
}
function compareEventsDesc(a: DataEvent, b: DataEvent): number {
const aMs = parseTimestampMs(a.timestamp);
const bMs = parseTimestampMs(b.timestamp);
if (aMs !== bMs) {
return bMs - aMs;
}
return b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0;
}
export async function GET(request: NextRequest) {
const auth = await requireAdminApiSession(request);
if (!auth.valid) {
return auth.response;
}
try {
const { searchParams } = new URL(request.url);
const clientId = searchParams.get("clientId") || "all";
const rawClientId = searchParams.get("clientId") || "all";
const clientId =
rawClientId === "all"
? "all"
: SDKManager.normalizeClientId(rawClientId) || "all";
const sanitizedClientId =
clientId === "all" ? "all" : CSVStorageManager.sanitizeClientId(clientId);
const from = searchParams.get("from") || "";
const to = searchParams.get("to") || "";
const search = (searchParams.get("search") || "").toLowerCase();
@@ -34,7 +67,12 @@ export async function GET(request: NextRequest) {
const addIfMatching = (event: DataEvent) => {
const eventDate = CSVStorageManager.extractSafePartitionDate(event.timestamp, null);
const dedupeKey = `${event.clientId}:${event.eventId}`;
const normEventClient = SDKManager.normalizeClientId(event.clientId) || event.clientId;
const dedupeKey = `${normEventClient}:${event.eventId}`;
const matchesClient =
clientId === "all" ||
normEventClient === clientId ||
CSVStorageManager.sanitizeClientId(normEventClient) === sanitizedClientId;
const matchesSearch =
!search ||
event.payload.toLowerCase().includes(search) ||
@@ -42,7 +80,7 @@ export async function GET(request: NextRequest) {
event.clientId.toLowerCase().includes(search);
if (
(clientId !== "all" && event.clientId !== clientId) ||
!matchesClient ||
((from || to) && (!eventDate || (from && eventDate < from) || (to && eventDate > to))) ||
!matchesSearch ||
eventIds.has(dedupeKey)
@@ -57,7 +95,7 @@ export async function GET(request: NextRequest) {
let high = selectedRows.length;
while (low < high) {
const middle = (low + high) >>> 1;
if (selectedRows[middle].timestamp >= event.timestamp) low = middle + 1;
if (compareEventsDesc(selectedRows[middle], event) <= 0) low = middle + 1;
else high = middle;
}
@@ -70,31 +108,61 @@ export async function GET(request: NextRequest) {
const storageDir = CSVStorageManager.baseStorageDir;
let clientDirs: string[] = [];
try {
clientDirs = readdirSync(storageDir);
clientDirs = await readdir(storageDir);
} catch {
clientDirs = [];
}
for (const storedClientId of clientDirs) {
if (clientId !== "all" && storedClientId !== clientId) continue;
if (
clientId !== "all" &&
storedClientId !== clientId &&
storedClientId !== sanitizedClientId
) {
continue;
}
const clientDir = path.join(storageDir, storedClientId);
if (!statSync(clientDir).isDirectory()) continue;
let dirLstat;
try {
dirLstat = await lstat(clientDir);
} catch {
continue;
}
if (dirLstat.isSymbolicLink() || !dirLstat.isDirectory()) continue;
let filenames: string[];
try {
filenames = readdirSync(clientDir).filter((filename) =>
/^\d{4}-\d{2}-\d{2}(?:-part\d+)?\.csv$/.test(filename),
);
filenames = (await readdir(clientDir))
.filter((filename) => {
if (!/^\d{4}-\d{2}-\d{2}(?:-part\d+)?\.csv$/.test(filename)) {
return false;
}
const fileDate = filename.slice(0, 10);
if ((from && fileDate < from) || (to && fileDate > to)) {
return false;
}
return true;
})
.sort((a, b) => b.localeCompare(a));
} catch {
continue;
}
for (const filename of filenames) {
const fileDate = filename.slice(0, 10);
if ((from && fileDate < from) || (to && fileDate > to)) continue;
const parser = createReadStream(path.join(clientDir, filename)).pipe(
const filePath = path.join(clientDir, filename);
let fileLstat;
try {
fileLstat = await lstat(filePath);
} catch {
continue;
}
if (fileLstat.isSymbolicLink() || !fileLstat.isFile()) continue;
const fileStream = createReadStream(filePath);
const parser = fileStream.pipe(
parse({ columns: true, skip_empty_lines: true, bom: true }),
);
@@ -111,17 +179,17 @@ export async function GET(request: NextRequest) {
};
if (event.eventId) addIfMatching(event);
}
} catch {
parser.destroy();
} catch (err) {
console.warn(`[API /data] Failed reading CSV partition ${filePath}:`, err);
} finally {
if (!parser.destroyed) parser.destroy();
if (!fileStream.destroyed) fileStream.destroy();
}
}
}
const settings = await SettingsManager.getSettingsAsync();
const recentWindowLimit = Math.min(
2000,
Math.max(200, settings.batchFlushThreshold || 500),
);
await SettingsManager.getSettingsAsync();
const recentWindowLimit = DataBufferEngine.getRecentLimit();
for (const event of await DataBufferEngine.getRecentEventsAsync(
recentWindowLimit,
+296 -41
View File
@@ -1,33 +1,240 @@
import { SettingsManager } from '@/core/adminSettings/settingsManager';
import { verifyRequestAdminSession } from '@/core/auth/adminUsers';
import { AuthService } from '@/core/auth/jwt';
import { DataBufferEngine } from '@/core/data/buffer';
import { DataBufferEngine, REDIS_BUFFER_KEY } from '@/core/data/buffer';
import { SDKManager } from '@/core/sdk/sdkManager';
import {
SecurityEngine,
getInternalIngestionHeaders,
} from '@/core/Security/security';
import { CSVStorageManager, type DataEvent } from '@/core/Storage/csvStorage';
import {
CSVStorageManager,
type DataEvent,
type StorageFileMeta,
} from '@/core/Storage/csvStorage';
import { parse } from 'csv-parse';
import { NextRequest, NextResponse } from 'next/server';
import crypto from 'node:crypto';
import { createReadStream } from 'node:fs';
import { requireAdminApiSession } from '@/lib/adminAuth';
import type { BufferedBucket, BufferedSummary } from '@/lib/vconUiHelpers';
const SESSION_COOKIE = 'datastack_admin_session';
const SINGLE_EVENT_ENVELOPE_KEYS = new Set(['eventId', 'timestamp', 'data', 'clientId']);
const MAX_SINGLE_CLIENT_SCAN_FILES = 10;
const MAX_ALL_CLIENTS_PER_CLIENT_FILES = 2;
const MAX_ALL_CLIENTS_TOTAL_SCAN_FILES = 50;
function generateFallbackEventId(): string {
return `evt_${Date.now()}_${crypto.randomBytes(6).toString('hex')}`;
}
function extractSingleHttpEventData(rawBody: Record<string, unknown>): unknown {
if (rawBody.data !== undefined) {
const keys = Object.keys(rawBody);
if (keys.every((k) => SINGLE_EVENT_ENVELOPE_KEYS.has(k))) {
return rawBody.data;
}
return rawBody;
}
const stripped: Record<string, unknown> = {};
let hasNonEnvelopeKey = false;
for (const [k, v] of Object.entries(rawBody)) {
if (k === 'eventId' || k === 'timestamp' || k === 'clientId') {
continue;
}
hasNonEnvelopeKey = true;
stripped[k] = v;
}
return hasNonEnvelopeKey ? stripped : {};
}
function parseTimestampMs(raw: string): number {
const trimmed = String(raw || '').trim();
if (/^\d+(\.\d+)?$/.test(trimmed)) {
const num = Number(trimmed);
return num < 1e11 ? num * 1000 : num;
}
const parsed = Date.parse(trimmed);
return Number.isNaN(parsed) ? Number.NEGATIVE_INFINITY : parsed;
}
function resolveEventUtcDate(rawTimestamp: string): string {
const ms = parseTimestampMs(rawTimestamp);
if (Number.isFinite(ms)) {
const parsed = new Date(ms);
if (!Number.isNaN(parsed.getTime())) {
return parsed.toISOString().slice(0, 10);
}
}
return new Date().toISOString().slice(0, 10);
}
function selectRecentPartitionFiles(
allFiles: StorageFileMeta[],
normClientId?: string,
sanitizedDir?: string,
): StorageFileMeta[] {
const matching = allFiles
.filter(
(f) =>
!normClientId ||
f.clientId === normClientId ||
f.clientId === sanitizedDir,
)
.sort((a, b) => {
if (a.date !== b.date) return b.date.localeCompare(a.date);
return new Date(b.modifiedAt).getTime() - new Date(a.modifiedAt).getTime();
});
if (normClientId) {
return matching.slice(0, MAX_SINGLE_CLIENT_SCAN_FILES);
}
// Multi-client fair scan: take up to 2 newest partitions per active client (capped at 50 files total)
// so clients beyond the 5th most-recently-modified file are never silently hidden.
const perClientCounts = new Map<string, number>();
const selected: StorageFileMeta[] = [];
for (const file of matching) {
const currentCount = perClientCounts.get(file.clientId) ?? 0;
if (currentCount >= MAX_ALL_CLIENTS_PER_CLIENT_FILES) continue;
perClientCounts.set(file.clientId, currentCount + 1);
selected.push(file);
if (selected.length >= MAX_ALL_CLIENTS_TOTAL_SCAN_FILES) break;
}
return selected;
}
async function getPendingBufferedSummary(
ingestionBaseUrl: string,
filterClientId?: string,
): Promise<{ events: DataEvent[]; summary: BufferedSummary }> {
const bufferStats = await DataBufferEngine.getBufferStatsAsync();
const recentLimit = DataBufferEngine.getRecentLimit();
const pending: DataEvent[] = [];
let totalQueued = bufferStats.localBufferSize + bufferStats.redisBufferSize;
// 1. Read exact unflushed Redis queue (when Redis is enabled & connected)
const redis = DataBufferEngine.getRedisClient();
if (redis && bufferStats.redisBufferSize > 0) {
try {
const cap = Math.min(recentLimit, bufferStats.redisBufferSize);
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, cap - 1);
for (const raw of rawItems) {
try {
const parsed = JSON.parse(raw) as DataEvent;
if (!parsed || !parsed.eventId) continue;
pending.push(parsed);
} catch {
// Skip malformed queued item
}
}
} catch {
// Fall back to local buffer telemetry
}
}
// 2. Read Port 3000 exact unflushed RAM queue snapshot
if (bufferStats.localBufferSize > 0) {
const localPending = DataBufferEngine.getLocalBufferSnapshot(recentLimit);
pending.push(...localPending);
}
// 3. When Redis is not enabled or temporarily disconnected, Port 4000 `/internal/recent` serves its unflushed RAM `bufferQueue`
if (!bufferStats.redisEnabled || !bufferStats.redisConnected) {
try {
const qs = new URLSearchParams({ limit: String(recentLimit) });
const res = await fetch(`${ingestionBaseUrl}/internal/recent?${qs.toString()}`, {
headers: getInternalIngestionHeaders(),
cache: 'no-store',
signal: AbortSignal.timeout(1000),
});
if (res.ok) {
const data = await res.json();
if (Array.isArray(data.events) && data.events.length > 0) {
pending.push(...(data.events as DataEvent[]));
totalQueued += data.events.length;
}
}
} catch {
// Port 4000 may not be running in isolated single-process dev mode
}
}
const seen = new Set<string>();
const dedupedAll: DataEvent[] = [];
const filteredEvents: DataEvent[] = [];
const bucketMap = new Map<string, BufferedBucket>();
const sanitizedFilter = filterClientId
? CSVStorageManager.sanitizeClientId(filterClientId)
: undefined;
for (const evt of pending) {
if (!evt || !evt.eventId) continue;
const normEvtClient = SDKManager.normalizeClientId(evt.clientId) || evt.clientId;
const dedupeKey = `${normEvtClient}:${evt.eventId}`;
if (seen.has(dedupeKey)) continue;
seen.add(dedupeKey);
dedupedAll.push(evt);
const matchesFilter =
!filterClientId ||
normEvtClient === filterClientId ||
evt.clientId === filterClientId ||
evt.clientId === sanitizedFilter;
if (matchesFilter) {
filteredEvents.push(evt);
const utcDate = resolveEventUtcDate(evt.timestamp);
const bucketKey = `${normEvtClient}|${utcDate}`;
const existing = bucketMap.get(bucketKey);
if (existing) {
existing.count += 1;
} else {
bucketMap.set(bucketKey, {
clientId: normEvtClient,
date: utcDate,
count: 1,
});
}
}
}
const effectiveTotalQueued = Math.max(totalQueued, dedupedAll.length);
const unattributedCount = Math.max(0, effectiveTotalQueued - dedupedAll.length);
return {
events: filteredEvents,
summary: {
total: filteredEvents.length,
totalQueued: filterClientId
? filteredEvents.length + unattributedCount
: effectiveTotalQueued,
unattributedCount,
truncated: unattributedCount > 0,
buckets: Array.from(bucketMap.values()),
},
};
}
async function getRecentStoredEvents(
limit: number,
clientId?: string,
): Promise<DataEvent[]> {
const normClientId = clientId && clientId !== 'all' ? SDKManager.normalizeClientId(clientId) : undefined;
const sanitizedDir = normClientId ? CSVStorageManager.sanitizeClientId(normClientId) : undefined;
const allFiles = await CSVStorageManager.listAllFilesAsync();
const files = allFiles
.filter((f) => !clientId || clientId === 'all' || f.clientId === clientId)
.slice(0, 5);
const files = selectRecentPartitionFiles(allFiles, normClientId, sanitizedDir);
const collected: DataEvent[] = [];
for (const file of files) {
const fileTail: DataEvent[] = [];
const parser = createReadStream(file.filePath).pipe(
const fileStream = createReadStream(file.filePath);
const parser = fileStream.pipe(
parse({ columns: true, skip_empty_lines: true, bom: true }),
);
@@ -46,8 +253,11 @@ async function getRecentStoredEvents(
payload: record.payload || '',
});
}
} catch {
parser.destroy();
} catch (err) {
console.warn(`[API /events] Failed reading CSV partition ${file.filePath}:`, err);
} finally {
if (!parser.destroyed) parser.destroy();
if (!fileStream.destroyed) fileStream.destroy();
}
if (fileTail.length > 0) {
@@ -59,18 +269,47 @@ async function getRecentStoredEvents(
}
export async function GET(req: NextRequest) {
const auth = await requireAdminApiSession(req);
if (!auth.valid) {
return auth.response;
}
try {
const { searchParams } = new URL(req.url);
const clientId = searchParams.get('clientId') || undefined;
const rawClientId = searchParams.get('clientId');
const clientId =
rawClientId && rawClientId !== 'all'
? SDKManager.normalizeClientId(rawClientId) || undefined
: undefined;
const limit = Math.min(2000, Math.max(1, Number(searchParams.get('limit')) || 50));
const bufferedOnly = searchParams.get('bufferedOnly') === 'true';
const includeBufferedSummary =
bufferedOnly || searchParams.get('includeBufferedSummary') === 'true';
await SettingsManager.getSettingsAsync();
const bufferedEvents = await DataBufferEngine.getRecentEventsAsync(limit, clientId);
let socketBufferedEvents: DataEvent[] = [];
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
).replace(/\/$/, '');
if (bufferedOnly) {
const pendingResult = await getPendingBufferedSummary(ingestionBaseUrl, clientId);
const sortedPending = [...pendingResult.events].sort((a, b) => {
const aMs = parseTimestampMs(a.timestamp);
const bMs = parseTimestampMs(b.timestamp);
if (aMs !== bMs) return bMs - aMs;
return b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0;
});
const events = sortedPending.slice(0, limit);
return NextResponse.json({
events,
count: events.length,
bufferedSummary: pendingResult.summary,
});
}
const bufferedEvents = await DataBufferEngine.getRecentEventsAsync(limit, clientId);
let socketBufferedEvents: DataEvent[] = [];
try {
const qs = new URLSearchParams({ limit: String(limit) });
if (clientId) qs.set('clientId', clientId);
@@ -89,23 +328,38 @@ export async function GET(req: NextRequest) {
// Port 4000 may not be running in isolated single-process dev mode
}
const storedEvents = await getRecentStoredEvents(limit, clientId);
const [storedEvents, pendingResult] = await Promise.all([
getRecentStoredEvents(limit, clientId),
includeBufferedSummary
? getPendingBufferedSummary(ingestionBaseUrl, clientId)
: Promise.resolve(null),
]);
const seenIds = new Set<string>();
const merged: DataEvent[] = [];
for (const evt of [...bufferedEvents, ...socketBufferedEvents, ...storedEvents]) {
if (!evt.eventId) continue;
const dedupeKey = `${evt.clientId}:${evt.eventId}`;
const normEvtClient = SDKManager.normalizeClientId(evt.clientId) || evt.clientId;
const dedupeKey = `${normEvtClient}:${evt.eventId}`;
if (seenIds.has(dedupeKey)) continue;
seenIds.add(dedupeKey);
merged.push(evt);
}
merged.sort((a, b) => (b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0));
merged.sort((a, b) => {
const aMs = parseTimestampMs(a.timestamp);
const bMs = parseTimestampMs(b.timestamp);
if (aMs !== bMs) return bMs - aMs;
return b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0;
});
const events = merged.slice(0, limit);
return NextResponse.json({ events, count: events.length });
return NextResponse.json({
events,
count: events.length,
...(pendingResult ? { bufferedSummary: pendingResult.summary } : {}),
});
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Events retrieval failed';
return NextResponse.json({ error: message }, { status: 500 });
@@ -165,12 +419,18 @@ export async function POST(req: NextRequest) {
? (body as Record<string, unknown>)
: {};
const clientId = String(
tokenClientId || bodyObj.clientId || 'py_telemetry_agent_01',
)
.trim()
.toLowerCase()
.replace(/[^a-z0-9_-]/g, '_');
const clientId = SDKManager.normalizeClientId(
String(tokenClientId || bodyObj.clientId || ''),
);
if (!clientId) {
return NextResponse.json(
{
error: 'clientId is required and must belong to a registered client.',
code: 'VALIDATION_ERROR',
},
{ status: 400 },
);
}
if (await SDKManager.isClientDeletedAsync(clientId)) {
return NextResponse.json(
@@ -183,7 +443,7 @@ export async function POST(req: NextRequest) {
}
const registeredClient = await SDKManager.getClientAsync(clientId);
if (rawBearerToken && !registeredClient) {
if (!registeredClient) {
return NextResponse.json(
{
error: `CLIENT_REVOKED: Client '${clientId}' registration not found or has been deleted.`,
@@ -231,10 +491,11 @@ export async function POST(req: NextRequest) {
? (bodyObj.batch as unknown[])
: Array.isArray(bodyObj.records)
? (bodyObj.records as unknown[])
: null;
: bodyObj.data === undefined && Array.isArray(bodyObj.events)
? (bodyObj.events as unknown[])
: null;
if (batchItems !== null) {
const envelopeKeys = new Set(['eventId', 'timestamp', 'data', 'clientId']);
let processed = 0;
let rejected = 0;
let duplicates = 0;
@@ -255,19 +516,16 @@ export async function POST(req: NextRequest) {
const itemEventId =
typeof safeItem.eventId === 'string' && safeItem.eventId.trim()
? safeItem.eventId.trim()
: `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
: generateFallbackEventId();
const itemTimestamp =
typeof safeItem.timestamp === 'string' && safeItem.timestamp.trim()
? safeItem.timestamp.trim()
: new Date().toISOString();
const isEnvelope =
safeItem.data !== undefined &&
Object.keys(safeItem).every((k) => envelopeKeys.has(k));
const itemData = isEnvelope ? safeItem.data : safeItem;
const itemData = extractSingleHttpEventData(safeItem);
const val = SecurityEngine.validateEvent(
const val = await SecurityEngine.validateEventAsync(
clientId,
itemEventId,
itemData,
@@ -309,7 +567,7 @@ export async function POST(req: NextRequest) {
}
if (processed > 0) {
SDKManager.incrementEventCount(clientId, processed);
await SDKManager.incrementEventCountAsync(clientId, processed);
}
// If zero items were processed or deduplicated and all failed due to payload or rate limit, return corresponding status
@@ -323,6 +581,7 @@ export async function POST(req: NextRequest) {
return NextResponse.json(
{
success: false,
accepted: processed,
processed,
rejected,
duplicates,
@@ -338,6 +597,7 @@ export async function POST(req: NextRequest) {
return NextResponse.json({
success: true,
accepted: processed,
processed,
rejected,
duplicates,
@@ -350,24 +610,19 @@ export async function POST(req: NextRequest) {
});
}
// Single Event Ingestion
// Single Event Ingestion (exact envelope parity with Port 4000 WS sync_data)
const eventId =
typeof bodyObj.eventId === 'string' && bodyObj.eventId.trim()
? bodyObj.eventId.trim()
: `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
: generateFallbackEventId();
const timestamp =
typeof bodyObj.timestamp === 'string' && bodyObj.timestamp.trim()
? bodyObj.timestamp.trim()
: new Date().toISOString();
const data =
bodyObj.data !== undefined
? bodyObj.data
: Object.keys(bodyObj).length > 0
? bodyObj
: { sample: 'Realtime event via DataStack API', rand: Math.random() };
const data = extractSingleHttpEventData(bodyObj);
// Validate security (idempotency, rate limit, payload size)
const val = SecurityEngine.validateEvent(
const val = await SecurityEngine.validateEventAsync(
clientId,
eventId,
data,
@@ -393,7 +648,7 @@ export async function POST(req: NextRequest) {
};
await DataBufferEngine.pushAsync(event);
SDKManager.incrementEventCount(clientId, 1);
await SDKManager.incrementEventCountAsync(clientId, 1);
return NextResponse.json({ success: true, eventId, event });
} catch (err: unknown) {
+127 -22
View File
@@ -1,10 +1,12 @@
import { SettingsManager } from "@/core/adminSettings/settingsManager";
import { DataBufferEngine } from "@/core/data/buffer";
import { SDKManager } from "@/core/sdk/sdkManager";
import { getInternalIngestionHeaders } from "@/core/Security/security";
import {
CSVStorageManager,
type StorageFileMeta,
} from "@/core/Storage/csvStorage";
import { requireAdminApiSession } from "@/lib/adminAuth";
import { parse } from "csv-parse";
import { NextRequest, NextResponse } from "next/server";
import { createReadStream } from "node:fs";
@@ -44,7 +46,8 @@ function toByteStream(chunks: AsyncIterable<StreamChunk>) {
async function* readFileRecords(
file: StorageFileMeta,
): AsyncGenerator<CsvRecord> {
const parser = createReadStream(file.filePath).pipe(
const fileStream = createReadStream(file.filePath);
const parser = fileStream.pipe(
parse({ columns: true, skip_empty_lines: true, bom: true }),
);
@@ -52,8 +55,9 @@ async function* readFileRecords(
for await (const record of parser as AsyncIterable<CsvRecord>) {
yield record;
}
} catch {
parser.destroy();
} finally {
if (!parser.destroyed) parser.destroy();
if (!fileStream.destroyed) fileStream.destroy();
}
}
@@ -85,7 +89,7 @@ function makeExportStream(
yield `${isFirst ? "" : ","}${JSON.stringify({ ...record, payload: parsedPayload })}`;
} else {
const line = fields
.map((field) => `"${(record[field] ?? "").replaceAll('"', '""')}"`)
.map((field) => `"${String(record[field] ?? "").replaceAll('"', '""')}"`)
.join(",");
yield `${line}\n`;
}
@@ -101,8 +105,13 @@ function makeExportStream(
function streamFile(file: StorageFileMeta) {
async function* chunks() {
for await (const chunk of createReadStream(file.filePath)) {
yield new Uint8Array(chunk);
const fileStream = createReadStream(file.filePath);
try {
for await (const chunk of fileStream) {
yield new Uint8Array(chunk as Buffer);
}
} finally {
if (!fileStream.destroyed) fileStream.destroy();
}
}
@@ -110,38 +119,122 @@ function streamFile(file: StorageFileMeta) {
}
export async function GET(request: NextRequest) {
const auth = await requireAdminApiSession(request);
if (!auth.valid) {
return auth.response;
}
try {
const { searchParams } = new URL(request.url);
const from = (searchParams.get("from") || "").trim();
const to = (searchParams.get("to") || "").trim();
if (from && to && from > to) {
return NextResponse.json(
{
error: 'Start date ("from") must be on or before end date ("to").',
code: "INVALID_DATE_RANGE",
},
{ status: 400 },
);
}
const shouldFlush = searchParams.get("flush") !== "false";
await SettingsManager.getSettingsAsync();
if (shouldFlush) {
await DataBufferEngine.flush();
try {
await DataBufferEngine.flush({ waitMs: 2500, throwOnLockTimeout: true });
} catch (err: unknown) {
const message =
err instanceof Error ? err.message : "Pre-export flush failed";
if (message.includes("FLUSH_LOCK_TIMEOUT")) {
return NextResponse.json(
{
error: message,
code: "FLUSH_LOCK_TIMEOUT",
},
{ status: 503 },
);
}
throw err;
}
const explicitIngestionUrl = Boolean(process.env.INGESTION_INTERNAL_URL?.trim());
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, "");
try {
await fetch(`${ingestionBaseUrl}/internal/flush`, {
const flushRes = await fetch(`${ingestionBaseUrl}/internal/flush`, {
method: "POST",
headers: getInternalIngestionHeaders(),
signal: AbortSignal.timeout(1500),
headers: {
"Content-Type": "application/json",
...getInternalIngestionHeaders(),
},
body: JSON.stringify({ waitMs: 2500, throwOnLockTimeout: true }),
cache: "no-store",
signal: AbortSignal.timeout(4000),
});
} catch {
if (!flushRes.ok) {
const errPayload = await flushRes.json().catch(() => ({}));
const remoteErr =
typeof errPayload?.error === "string"
? errPayload.error
: `Ingestion flush returned HTTP ${flushRes.status}`;
if (
flushRes.status === 503 &&
errPayload?.code === "FLUSH_LOCK_TIMEOUT"
) {
return NextResponse.json(
{
error: remoteErr,
code: "FLUSH_LOCK_TIMEOUT",
ingestionStatus: flushRes.status,
},
{ status: 503 },
);
}
return NextResponse.json(
{
error: remoteErr,
code: "INGESTION_FLUSH_FAILED",
ingestionStatus: flushRes.status,
},
{ status: 502 },
);
}
} catch (err: unknown) {
if (explicitIngestionUrl) {
const errMsg =
err instanceof Error ? err.message : "Ingestion server unreachable";
return NextResponse.json(
{
error: `Ingestion server flush failed before export: ${errMsg}`,
code: "INGESTION_UNREACHABLE",
},
{ status: 502 },
);
}
// Port 4000 may not be running in isolated single-process dev mode
}
}
const clientId = searchParams.get("clientId") || "all";
const rawClientId = searchParams.get("clientId") || "all";
const clientId =
rawClientId === "all"
? "all"
: SDKManager.normalizeClientId(rawClientId) || "all";
const sanitizedClientId =
clientId === "all" ? "all" : CSVStorageManager.sanitizeClientId(clientId);
const filename = searchParams.get("filename");
const from = searchParams.get("from") || "";
const to = searchParams.get("to") || "";
const format: ExportFormat =
searchParams.get("format") === "json" ? "json" : "csv";
const rawFormat = searchParams.get("format");
const format: ExportFormat = rawFormat === "json" ? "json" : "csv";
const includeHeader = searchParams.get("includeHeader") !== "false";
let files = await CSVStorageManager.listAllFilesAsync();
if (clientId !== "all")
files = files.filter((file) => file.clientId === clientId);
if (clientId !== "all") {
files = files.filter(
(file) => file.clientId === clientId || file.clientId === sanitizedClientId,
);
}
if (filename) {
const safeFilename = path.basename(filename);
@@ -166,11 +259,23 @@ export async function GET(request: NextRequest) {
);
}
const file = matches[0];
return new NextResponse(streamFile(file), {
const downloadName =
format === "json"
? `${file.filename.replace(/\.csv$/i, "")}.json`
: file.filename;
const bodyStream =
format === "json" || !includeHeader
? makeExportStream([file], format, includeHeader)
: streamFile(file);
return new NextResponse(bodyStream, {
status: 200,
headers: {
"Content-Type": "text/csv; charset=utf-8",
"Content-Disposition": `attachment; filename="${file.filename}"`,
"Content-Type":
format === "json"
? "application/json; charset=utf-8"
: "text/csv; charset=utf-8",
"Content-Disposition": `attachment; filename="${downloadName.replace(/"/g, "")}"`,
"Cache-Control": "no-store",
},
});
}
@@ -179,7 +284,7 @@ export async function GET(request: NextRequest) {
(file) => (!from || file.date >= from) && (!to || file.date <= to),
);
const datePart = new Date().toISOString().slice(0, 10);
const safeClientId = clientId.replace(/[^a-zA-Z0-9_-]/g, "_");
const safeClientId = sanitizedClientId === "all" ? "all" : sanitizedClientId;
const baseName =
clientId === "all" ? "datastack_export" : `${safeClientId}_export`;
+33 -25
View File
@@ -23,10 +23,10 @@ export async function GET(req: NextRequest) {
);
}
const clientId = String(verified.payload.clientId || '')
.trim()
.toLowerCase();
if (SDKManager.isClientDeleted(clientId)) {
const clientId = SDKManager.normalizeClientId(
String(verified.payload.clientId || ''),
);
if (await SDKManager.isClientDeletedAsync(clientId)) {
return NextResponse.json(
{
status: 'error',
@@ -37,27 +37,35 @@ export async function GET(req: NextRequest) {
);
}
const registeredClient = await SDKManager.getClientAsync(clientId);
if (registeredClient) {
if (registeredClient.status === 'revoked') {
return NextResponse.json(
{
status: 'error',
code: 'CLIENT_REVOKED',
error: `CLIENT_REVOKED: Client '${clientId}' access has been revoked by administrator.`,
},
{ status: 403 },
);
}
if (registeredClient.token && registeredClient.token !== rawToken) {
return NextResponse.json(
{
status: 'error',
code: 'TOKEN_ROTATED',
error: 'TOKEN_ROTATED: Client credentials have been rotated. Use the latest token.',
},
{ status: 401 },
);
}
if (!registeredClient) {
return NextResponse.json(
{
status: 'error',
code: 'CLIENT_REVOKED',
error: `CLIENT_REVOKED: Client '${clientId}' registration not found or has been deleted.`,
},
{ status: 403 },
);
}
if (registeredClient.status === 'revoked') {
return NextResponse.json(
{
status: 'error',
code: 'CLIENT_REVOKED',
error: `CLIENT_REVOKED: Client '${clientId}' access has been revoked by administrator.`,
},
{ status: 403 },
);
}
if (registeredClient.token && registeredClient.token !== rawToken) {
return NextResponse.json(
{
status: 'error',
code: 'TOKEN_ROTATED',
error: 'TOKEN_ROTATED: Client credentials have been rotated. Use the latest token.',
},
{ status: 401 },
);
}
}
+8 -2
View File
@@ -1,7 +1,13 @@
import { NextResponse } from 'next/server';
import { NextRequest, NextResponse } from 'next/server';
import { ApiEngine } from '@/core/apiEngine/apiService';
import { requireAdminApiSession } from '@/lib/adminAuth';
export async function GET(req: NextRequest) {
const auth = await requireAdminApiSession(req);
if (!auth.valid) {
return auth.response;
}
export async function GET() {
try {
const metrics = await ApiEngine.getDashboardMetrics();
return NextResponse.json(metrics);
+15
View File
@@ -188,9 +188,24 @@ export async function POST(req: NextRequest) {
await SettingsManager.updateSettingsAsync(validated);
const settings = await SettingsManager.getPublicSettingsAsync();
const bridgeSync = SettingsManager.getLastBridgeSyncStatus();
return NextResponse.json({
settings,
success: true,
...(bridgeSync
? {
bridgeSync: {
attempted: bridgeSync.attempted ?? true,
ok: bridgeSync.ok ?? bridgeSync.synced,
synced: bridgeSync.synced,
...(bridgeSync.status !== undefined
? { status: bridgeSync.status }
: {}),
...(bridgeSync.error ? { error: bridgeSync.error } : {}),
checkedAt: bridgeSync.checkedAt,
},
}
: {}),
});
} catch (err: unknown) {
const message =
+70 -11
View File
@@ -1,24 +1,44 @@
import { NextRequest, NextResponse } from 'next/server';
import { SettingsManager } from '@/core/adminSettings/settingsManager';
import { SDKManager } from '@/core/sdk/sdkManager';
import { getInternalIngestionHeaders } from '@/core/Security/security';
import { CSVStorageManager } from '@/core/Storage/csvStorage';
import { DataBufferEngine } from '@/core/data/buffer';
import { requireAdminApiSession } from '@/lib/adminAuth';
export async function GET(req: NextRequest) {
const auth = await requireAdminApiSession(req);
if (!auth.valid) {
return auth.response;
}
try {
const { searchParams } = new URL(req.url);
const clientId = searchParams.get('clientId') || undefined;
const rawClientId = searchParams.get('clientId');
const clientId =
rawClientId && rawClientId !== 'all'
? SDKManager.normalizeClientId(rawClientId)
: undefined;
const sanitizedClientId = clientId
? CSVStorageManager.sanitizeClientId(clientId)
: undefined;
const [stats, allFiles] = await Promise.all([
CSVStorageManager.getStorageStatsAsync(),
CSVStorageManager.listAllFilesAsync(),
]);
const allFiles = await CSVStorageManager.listAllFilesAsync();
let files = allFiles;
if (clientId && clientId !== 'all') {
files = files.filter((f) => f.clientId === clientId);
if (clientId) {
files = allFiles.filter((f) => {
const normFolder = SDKManager.normalizeClientId(f.clientId);
return (
normFolder === clientId ||
f.clientId === clientId ||
f.clientId === sanitizedClientId
);
});
}
const stats = CSVStorageManager.formatStorageStats(files);
return NextResponse.json({ stats, files });
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Storage stats failed';
@@ -26,12 +46,20 @@ export async function GET(req: NextRequest) {
}
}
export async function POST(_req: NextRequest) {
export async function POST(req: NextRequest) {
const auth = await requireAdminApiSession(req);
if (!auth.valid) {
return auth.response;
}
try {
await SettingsManager.getSettingsAsync();
// 1. Flush local / shared Next.js DataBufferEngine
const localRes = await DataBufferEngine.flush();
const localRes = await DataBufferEngine.flush({
waitMs: 2500,
throwOnLockTimeout: true,
});
// 2. Also flush Port 4000 Socket.IO ingestion buffer
let remoteCount = 0;
@@ -46,9 +74,13 @@ export async function POST(_req: NextRequest) {
try {
const ingestRes = await fetch(`${ingestionBaseUrl}/internal/flush`, {
method: 'POST',
headers: getInternalIngestionHeaders(),
headers: {
'Content-Type': 'application/json',
...getInternalIngestionHeaders(),
},
body: JSON.stringify({ waitMs: 2500, throwOnLockTimeout: true }),
cache: 'no-store',
signal: AbortSignal.timeout(2500),
signal: AbortSignal.timeout(4000),
});
if (!ingestRes.ok) {
const errPayload = await ingestRes.json().catch(() => ({}));
@@ -56,6 +88,23 @@ export async function POST(_req: NextRequest) {
typeof errPayload?.error === 'string'
? errPayload.error
: `Ingestion flush returned HTTP ${ingestRes.status}`;
if (
ingestRes.status === 503 &&
errPayload?.code === 'FLUSH_LOCK_TIMEOUT'
) {
return NextResponse.json(
{
success: false,
error: remoteErr,
code: 'FLUSH_LOCK_TIMEOUT',
flushedCount: localRes.count,
filesUpdated: localRes.files,
ingestionFlushed: false,
ingestionStatus: ingestRes.status,
},
{ status: 503 },
);
}
return NextResponse.json(
{
success: false,
@@ -108,6 +157,16 @@ export async function POST(_req: NextRequest) {
});
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Manual flush failed';
if (message.includes('FLUSH_LOCK_TIMEOUT')) {
return NextResponse.json(
{
success: false,
error: message,
code: 'FLUSH_LOCK_TIMEOUT',
},
{ status: 503 },
);
}
return NextResponse.json({ error: message }, { status: 500 });
}
}