refactor(auth): standardize session management and security
- Centralize secret retrieval using `getAdminSessionSecret` - Replace manual session parsing with `verifyRequestAdminSession` utility - Tighten API validation and improve security headers handling - Enforce environment variable security for production deployments
This commit is contained in:
1 parent
34519c07df
commit
177ed4a844
78 files changed
+6748
-3157
No files matched your search
+102
-79
@@ -1,11 +1,12 @@
|
||||
import {
|
||||
createAdminSessionToken,
|
||||
getAdminSessionSecret,
|
||||
getSessionCookieOptions,
|
||||
} from "@/core/auth/adminSession";
|
||||
import {
|
||||
authenticateAdminCredentials,
|
||||
clearAdminLoginAttempts,
|
||||
consumeAdminLoginAttempt,
|
||||
verifyAdminCredentials,
|
||||
} from "@/core/auth/adminUsers";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
@@ -13,10 +14,19 @@ const SESSION_COOKIE = "datastack_admin_session";
|
||||
const SESSION_SECONDS = 8 * 60 * 60;
|
||||
const REMEMBERED_SESSION_SECONDS = 30 * 24 * 60 * 60;
|
||||
|
||||
function extractClientIp(request: NextRequest) {
|
||||
const forwardedFor = request.headers.get("x-forwarded-for");
|
||||
if (forwardedFor) {
|
||||
const first = forwardedFor.split(",")[0]?.trim();
|
||||
if (first) return first;
|
||||
}
|
||||
const realIp = request.headers.get("x-real-ip")?.trim();
|
||||
if (realIp) return realIp;
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
const sessionSecret =
|
||||
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
|
||||
"datastack-admin-session-default-secret-32-chars";
|
||||
const sessionSecret = getAdminSessionSecret();
|
||||
|
||||
if (!sessionSecret || sessionSecret.length < 32) {
|
||||
return NextResponse.json(
|
||||
@@ -27,86 +37,99 @@ export async function POST(request: NextRequest) {
|
||||
);
|
||||
}
|
||||
|
||||
let body: Record<string, unknown>;
|
||||
try {
|
||||
const body = await request.json();
|
||||
const email =
|
||||
typeof body.email === "string" ? body.email.trim().toLowerCase() : "";
|
||||
const password = typeof body.password === "string" ? body.password : "";
|
||||
if (!email || password.length === 0 || password.length > 1024) {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid email or password." },
|
||||
{ status: 401 },
|
||||
);
|
||||
}
|
||||
|
||||
let throttleDecision;
|
||||
try {
|
||||
throttleDecision = await consumeAdminLoginAttempt(email, sessionSecret);
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Admin authentication is temporarily unavailable." },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
if (!throttleDecision.allowed) {
|
||||
return NextResponse.json(
|
||||
{ error: "Too many sign-in attempts. Try again later." },
|
||||
{
|
||||
status: 429,
|
||||
headers: {
|
||||
"Retry-After": String(throttleDecision.retryAfterSeconds),
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let authenticated = false;
|
||||
try {
|
||||
authenticated = await verifyAdminCredentials(email, password);
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Admin authentication is temporarily unavailable." },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
if (!authenticated) {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid email or password." },
|
||||
{ status: 401 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
await clearAdminLoginAttempts(email, sessionSecret);
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Admin authentication is temporarily unavailable." },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
const remember = body.remember === true;
|
||||
const maxAge = remember ? REMEMBERED_SESSION_SECONDS : SESSION_SECONDS;
|
||||
const expiresAt = Date.now() + maxAge * 1000;
|
||||
const token = await createAdminSessionToken(
|
||||
email,
|
||||
sessionSecret,
|
||||
expiresAt,
|
||||
);
|
||||
const response = NextResponse.json({ success: true });
|
||||
|
||||
response.cookies.set(
|
||||
SESSION_COOKIE,
|
||||
token,
|
||||
getSessionCookieOptions(request, maxAge),
|
||||
);
|
||||
|
||||
return response;
|
||||
const parsed = await request.json();
|
||||
body =
|
||||
parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
||||
? (parsed as Record<string, unknown>)
|
||||
: {};
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid login request." },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
const email =
|
||||
typeof body.email === "string" ? body.email.trim().toLowerCase() : "";
|
||||
const password = typeof body.password === "string" ? body.password : "";
|
||||
if (!email || password.length === 0 || password.length > 1024) {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid email or password." },
|
||||
{ status: 401 },
|
||||
);
|
||||
}
|
||||
|
||||
const clientIp = extractClientIp(request);
|
||||
|
||||
let throttleDecision;
|
||||
try {
|
||||
throttleDecision = await consumeAdminLoginAttempt(
|
||||
email,
|
||||
sessionSecret,
|
||||
clientIp,
|
||||
);
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Admin authentication is temporarily unavailable." },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
if (!throttleDecision.allowed) {
|
||||
return NextResponse.json(
|
||||
{ error: "Too many sign-in attempts. Try again later." },
|
||||
{
|
||||
status: 429,
|
||||
headers: {
|
||||
"Retry-After": String(throttleDecision.retryAfterSeconds),
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
let authResult: { authenticated: boolean; tokenVersion: number };
|
||||
try {
|
||||
authResult = await authenticateAdminCredentials(email, password);
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Admin authentication is temporarily unavailable." },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
if (!authResult.authenticated) {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid email or password." },
|
||||
{ status: 401 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
await clearAdminLoginAttempts(email, sessionSecret, clientIp);
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Admin authentication is temporarily unavailable." },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
|
||||
const remember = body.remember === true;
|
||||
const maxAge = remember ? REMEMBERED_SESSION_SECONDS : SESSION_SECONDS;
|
||||
const expiresAt = Date.now() + maxAge * 1000;
|
||||
const token = await createAdminSessionToken(
|
||||
email,
|
||||
sessionSecret,
|
||||
expiresAt,
|
||||
authResult.tokenVersion,
|
||||
);
|
||||
const response = NextResponse.json({ success: true });
|
||||
|
||||
response.cookies.set(
|
||||
SESSION_COOKIE,
|
||||
token,
|
||||
getSessionCookieOptions(request, maxAge),
|
||||
);
|
||||
|
||||
return response;
|
||||
}
|
||||
@@ -1,11 +1,11 @@
|
||||
import {
|
||||
createAdminSessionToken,
|
||||
getAdminSessionPayload,
|
||||
getSessionCookieOptions,
|
||||
} from "@/core/auth/adminSession";
|
||||
import {
|
||||
getAdminAccountInfo,
|
||||
updateAdminAccount,
|
||||
verifyRequestAdminSession,
|
||||
} from "@/core/auth/adminUsers";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
@@ -17,51 +17,69 @@ function validEmail(email: string) {
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
const sessionSecret =
|
||||
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
|
||||
"datastack-admin-session-default-secret-32-chars";
|
||||
const token = request.cookies.get(SESSION_COOKIE)?.value || "";
|
||||
const session = await getAdminSessionPayload(token, sessionSecret);
|
||||
const token = request.cookies.get(SESSION_COOKIE)?.value;
|
||||
const auth = await verifyRequestAdminSession(token);
|
||||
|
||||
if (!session) {
|
||||
return NextResponse.json(
|
||||
{ error: "Authentication required." },
|
||||
{ status: 401 },
|
||||
);
|
||||
if (!auth.valid) {
|
||||
const status = auth.reason === "UNAUTHORIZED" ? 401 : 503;
|
||||
const message =
|
||||
auth.reason === "UNAUTHORIZED"
|
||||
? "Authentication required."
|
||||
: "Admin authentication is temporarily unavailable.";
|
||||
return NextResponse.json({ error: message }, { status });
|
||||
}
|
||||
|
||||
const account = await getAdminAccountInfo(session.email);
|
||||
return NextResponse.json({
|
||||
profile: {
|
||||
email: account.email,
|
||||
role: "Owner",
|
||||
storageMode: account.storageMode,
|
||||
createdAt: account.createdAt,
|
||||
sessionExpiresAt: new Date(session.expiresAt).toISOString(),
|
||||
},
|
||||
});
|
||||
try {
|
||||
const account = await getAdminAccountInfo(auth.session.email);
|
||||
return NextResponse.json({
|
||||
profile: {
|
||||
email: account.email,
|
||||
role: "Owner",
|
||||
storageMode: account.storageMode,
|
||||
createdAt: account.createdAt,
|
||||
sessionExpiresAt: new Date(auth.session.expiresAt).toISOString(),
|
||||
},
|
||||
});
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Unable to load admin profile." },
|
||||
{ status: 503 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function PUT(request: NextRequest) {
|
||||
const sessionSecret =
|
||||
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
|
||||
"datastack-admin-session-default-secret-32-chars";
|
||||
const token = request.cookies.get(SESSION_COOKIE)?.value || "";
|
||||
const session = await getAdminSessionPayload(token, sessionSecret);
|
||||
const token = request.cookies.get(SESSION_COOKIE)?.value;
|
||||
const auth = await verifyRequestAdminSession(token);
|
||||
|
||||
if (!session) {
|
||||
if (!auth.valid) {
|
||||
const status = auth.reason === "UNAUTHORIZED" ? 401 : 503;
|
||||
const message =
|
||||
auth.reason === "UNAUTHORIZED"
|
||||
? "Authentication required."
|
||||
: "Admin authentication is temporarily unavailable.";
|
||||
return NextResponse.json({ error: message }, { status });
|
||||
}
|
||||
|
||||
let body: Record<string, unknown>;
|
||||
try {
|
||||
const parsed = await request.json();
|
||||
body =
|
||||
parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
||||
? (parsed as Record<string, unknown>)
|
||||
: {};
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Authentication required." },
|
||||
{ status: 401 },
|
||||
{ error: "Invalid profile payload." },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const body = await request.json();
|
||||
const nextEmail =
|
||||
typeof body.email === "string"
|
||||
? body.email.trim().toLowerCase()
|
||||
: session.email;
|
||||
: auth.session.email;
|
||||
const currentPassword =
|
||||
typeof body.currentPassword === "string" ? body.currentPassword : "";
|
||||
const newPassword =
|
||||
@@ -88,7 +106,7 @@ export async function PUT(request: NextRequest) {
|
||||
);
|
||||
}
|
||||
|
||||
const account = await getAdminAccountInfo(session.email);
|
||||
const account = await getAdminAccountInfo(auth.session.email);
|
||||
const result = await updateAdminAccount({
|
||||
currentEmail: account.email,
|
||||
currentPassword,
|
||||
@@ -103,13 +121,14 @@ export async function PUT(request: NextRequest) {
|
||||
// Preserve remaining session duration (e.g., 30-day "Remember me" session)
|
||||
const remainingSeconds = Math.max(
|
||||
SESSION_SECONDS,
|
||||
Math.ceil((session.expiresAt - Date.now()) / 1000),
|
||||
Math.ceil((auth.session.expiresAt - Date.now()) / 1000),
|
||||
);
|
||||
const expiresAt = Date.now() + remainingSeconds * 1000;
|
||||
const nextToken = await createAdminSessionToken(
|
||||
result.email,
|
||||
sessionSecret,
|
||||
auth.sessionSecret,
|
||||
expiresAt,
|
||||
result.tokenVersion,
|
||||
);
|
||||
|
||||
const response = NextResponse.json({
|
||||
@@ -133,7 +152,7 @@ export async function PUT(request: NextRequest) {
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Unable to update profile." },
|
||||
{ status: 400 },
|
||||
{ status: 500 },
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import {
|
||||
createAdminSessionToken,
|
||||
getAdminSessionSecret,
|
||||
getSessionCookieOptions,
|
||||
} from "@/core/auth/adminSession";
|
||||
import {
|
||||
@@ -20,9 +21,7 @@ function validEmail(email: string) {
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
const sessionSecret =
|
||||
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
|
||||
"datastack-admin-session-default-secret-32-chars";
|
||||
const sessionSecret = getAdminSessionSecret();
|
||||
if (!sessionSecret || sessionSecret.length < 32) {
|
||||
return NextResponse.json(
|
||||
{ error: "Admin session configuration is unavailable." },
|
||||
@@ -30,8 +29,21 @@ export async function POST(request: NextRequest) {
|
||||
);
|
||||
}
|
||||
|
||||
let body: Record<string, unknown>;
|
||||
try {
|
||||
const parsed = await request.json();
|
||||
body =
|
||||
parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
||||
? (parsed as Record<string, unknown>)
|
||||
: {};
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid setup request payload." },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const body = await request.json();
|
||||
const email =
|
||||
typeof body.email === "string" ? body.email.trim().toLowerCase() : "";
|
||||
const password = typeof body.password === "string" ? body.password : "";
|
||||
@@ -74,6 +86,7 @@ export async function POST(request: NextRequest) {
|
||||
email,
|
||||
sessionSecret,
|
||||
Date.now() + SESSION_SECONDS * 1000,
|
||||
1,
|
||||
);
|
||||
const response = NextResponse.json({ success: true });
|
||||
response.cookies.set(
|
||||
|
||||
@@ -1,23 +1,54 @@
|
||||
import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
|
||||
import { SDKManager } from "@/core/sdk/sdkManager";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
const SESSION_COOKIE = "datastack_admin_session";
|
||||
|
||||
export async function GET(
|
||||
_req: NextRequest,
|
||||
req: NextRequest,
|
||||
{ params }: { params: Promise<{ id: string }> },
|
||||
) {
|
||||
const auth = await verifyRequestAdminSession(
|
||||
req.cookies.get(SESSION_COOKIE)?.value,
|
||||
);
|
||||
if (!auth.valid) {
|
||||
return NextResponse.json(
|
||||
{ error: "Authentication required." },
|
||||
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
|
||||
);
|
||||
}
|
||||
|
||||
const { id } = await params;
|
||||
const client = await SDKManager.getClientAsync(id);
|
||||
if (!client) {
|
||||
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
||||
}
|
||||
const { secret, token, ...clientSummary } = client;
|
||||
return NextResponse.json({ client: clientSummary });
|
||||
void secret;
|
||||
const includeToken = req.nextUrl.searchParams.get("includeToken") === "true";
|
||||
return NextResponse.json({
|
||||
client: {
|
||||
...clientSummary,
|
||||
tokenPreview: token ? `${token.slice(0, 12)}...` : "",
|
||||
...(includeToken ? { token } : {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function PATCH(
|
||||
req: NextRequest,
|
||||
{ params }: { params: Promise<{ id: string }> },
|
||||
) {
|
||||
const auth = await verifyRequestAdminSession(
|
||||
req.cookies.get(SESSION_COOKIE)?.value,
|
||||
);
|
||||
if (!auth.valid) {
|
||||
return NextResponse.json(
|
||||
{ error: "Authentication required." },
|
||||
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
|
||||
);
|
||||
}
|
||||
|
||||
const { id } = await params;
|
||||
try {
|
||||
const body = await req.json();
|
||||
@@ -46,11 +77,22 @@ export async function PATCH(
|
||||
}
|
||||
|
||||
export async function DELETE(
|
||||
_req: NextRequest,
|
||||
req: NextRequest,
|
||||
{ params }: { params: Promise<{ id: string }> },
|
||||
) {
|
||||
const auth = await verifyRequestAdminSession(
|
||||
req.cookies.get(SESSION_COOKIE)?.value,
|
||||
);
|
||||
if (!auth.valid) {
|
||||
return NextResponse.json(
|
||||
{ error: "Authentication required." },
|
||||
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
|
||||
);
|
||||
}
|
||||
|
||||
const { id } = await params;
|
||||
const deleted = await SDKManager.deleteClientAsync(id);
|
||||
const purgeFiles = req.nextUrl.searchParams.get("purgeFiles") === "true";
|
||||
const deleted = await SDKManager.deleteClientAsync(id, { purgeFiles });
|
||||
if (!deleted) {
|
||||
return NextResponse.json({ error: "Client not found" }, { status: 404 });
|
||||
}
|
||||
|
||||
@@ -1,12 +1,28 @@
|
||||
import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
|
||||
import { SDKManager } from "@/core/sdk/sdkManager";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
export async function GET() {
|
||||
const SESSION_COOKIE = "datastack_admin_session";
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
const auth = await verifyRequestAdminSession(
|
||||
req.cookies.get(SESSION_COOKIE)?.value,
|
||||
);
|
||||
if (!auth.valid) {
|
||||
return NextResponse.json(
|
||||
{ error: "Authentication required." },
|
||||
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const allClients = await SDKManager.listClientsAsync();
|
||||
const clients = allClients.map(({ secret, ...client }) => {
|
||||
const clients = allClients.map(({ secret, token, ...client }) => {
|
||||
void secret;
|
||||
return client;
|
||||
return {
|
||||
...client,
|
||||
tokenPreview: token ? `${token.slice(0, 12)}...` : "",
|
||||
};
|
||||
});
|
||||
return NextResponse.json({ clients, count: clients.length });
|
||||
} catch (err: unknown) {
|
||||
@@ -17,8 +33,31 @@ export async function GET() {
|
||||
}
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
const auth = await verifyRequestAdminSession(
|
||||
req.cookies.get(SESSION_COOKIE)?.value,
|
||||
);
|
||||
if (!auth.valid) {
|
||||
return NextResponse.json(
|
||||
{ error: "Authentication required." },
|
||||
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
|
||||
);
|
||||
}
|
||||
|
||||
let body: Record<string, unknown>;
|
||||
try {
|
||||
const parsed = await req.json();
|
||||
body =
|
||||
parsed && typeof parsed === "object" && !Array.isArray(parsed)
|
||||
? (parsed as Record<string, unknown>)
|
||||
: {};
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid request payload." },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const body = await req.json();
|
||||
const { clientId, name, rateLimit } = body;
|
||||
|
||||
if (!clientId || !name) {
|
||||
|
||||
+35
-9
@@ -1,4 +1,6 @@
|
||||
import { SettingsManager } from "@/core/adminSettings/settingsManager";
|
||||
import { DataBufferEngine } from "@/core/data/buffer";
|
||||
import { getInternalIngestionHeaders } from "@/core/Security/security";
|
||||
import { CSVStorageManager } from "@/core/Storage/csvStorage";
|
||||
import { parse } from "csv-parse";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
@@ -31,18 +33,25 @@ export async function GET(request: NextRequest) {
|
||||
let total = 0;
|
||||
|
||||
const addIfMatching = (event: DataEvent) => {
|
||||
const eventDate = event.timestamp.slice(0, 10);
|
||||
const eventDate = CSVStorageManager.extractSafePartitionDate(event.timestamp);
|
||||
const dedupeKey = `${event.clientId}:${event.eventId}`;
|
||||
const matchesSearch =
|
||||
!search ||
|
||||
event.payload.toLowerCase().includes(search) ||
|
||||
event.eventId.toLowerCase().includes(search) ||
|
||||
event.clientId.toLowerCase().includes(search);
|
||||
|
||||
if (
|
||||
(clientId !== "all" && event.clientId !== clientId) ||
|
||||
(from && eventDate < from) ||
|
||||
(to && eventDate > to) ||
|
||||
(search && !event.payload.toLowerCase().includes(search)) ||
|
||||
eventIds.has(event.eventId)
|
||||
!matchesSearch ||
|
||||
eventIds.has(dedupeKey)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
eventIds.add(event.eventId);
|
||||
eventIds.add(dedupeKey);
|
||||
total += 1;
|
||||
|
||||
let low = 0;
|
||||
@@ -109,7 +118,16 @@ export async function GET(request: NextRequest) {
|
||||
}
|
||||
}
|
||||
|
||||
for (const event of DataBufferEngine.getRecentEvents(200)) {
|
||||
const settings = await SettingsManager.getSettingsAsync();
|
||||
const recentWindowLimit = Math.min(
|
||||
2000,
|
||||
Math.max(200, settings.batchFlushThreshold || 500),
|
||||
);
|
||||
|
||||
for (const event of await DataBufferEngine.getRecentEventsAsync(
|
||||
recentWindowLimit,
|
||||
clientId,
|
||||
)) {
|
||||
addIfMatching({ ...event, status: "buffered" });
|
||||
}
|
||||
|
||||
@@ -117,10 +135,18 @@ export async function GET(request: NextRequest) {
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||
).replace(/\/$/, "");
|
||||
try {
|
||||
const res = await fetch(`${ingestionBaseUrl}/internal/recent?limit=200`, {
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(1000),
|
||||
});
|
||||
const qs = new URLSearchParams({ limit: String(recentWindowLimit) });
|
||||
if (clientId && clientId !== "all") {
|
||||
qs.set("clientId", clientId);
|
||||
}
|
||||
const res = await fetch(
|
||||
`${ingestionBaseUrl}/internal/recent?${qs.toString()}`,
|
||||
{
|
||||
headers: getInternalIngestionHeaders(),
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(1000),
|
||||
},
|
||||
);
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
if (Array.isArray(data.events)) {
|
||||
|
||||
+75
-30
@@ -1,9 +1,12 @@
|
||||
import { SettingsManager } from '@/core/adminSettings/settingsManager';
|
||||
import { verifyAdminSessionToken } from '@/core/auth/adminSession';
|
||||
import { verifyRequestAdminSession } from '@/core/auth/adminUsers';
|
||||
import { AuthService } from '@/core/auth/jwt';
|
||||
import { DataBufferEngine } from '@/core/data/buffer';
|
||||
import { SDKManager } from '@/core/sdk/sdkManager';
|
||||
import { SecurityEngine } from '@/core/Security/security';
|
||||
import {
|
||||
SecurityEngine,
|
||||
getInternalIngestionHeaders,
|
||||
} from '@/core/Security/security';
|
||||
import { CSVStorageManager, type DataEvent } from '@/core/Storage/csvStorage';
|
||||
import { parse } from 'csv-parse';
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
@@ -15,14 +18,15 @@ async function getRecentStoredEvents(
|
||||
limit: number,
|
||||
clientId?: string,
|
||||
): Promise<DataEvent[]> {
|
||||
const files = CSVStorageManager.listAllFiles()
|
||||
const allFiles = await CSVStorageManager.listAllFilesAsync();
|
||||
const files = allFiles
|
||||
.filter((f) => !clientId || clientId === 'all' || f.clientId === clientId)
|
||||
.slice(0, 5);
|
||||
|
||||
const collected: DataEvent[] = [];
|
||||
|
||||
for (const file of files) {
|
||||
const fileEvents: DataEvent[] = [];
|
||||
const fileTail: DataEvent[] = [];
|
||||
const parser = createReadStream(file.filePath).pipe(
|
||||
parse({ columns: true, skip_empty_lines: true, bom: true }),
|
||||
);
|
||||
@@ -32,7 +36,10 @@ async function getRecentStoredEvents(
|
||||
Record<string, string>
|
||||
>) {
|
||||
if (!record.eventId) continue;
|
||||
fileEvents.push({
|
||||
if (fileTail.length >= limit) {
|
||||
fileTail.shift();
|
||||
}
|
||||
fileTail.push({
|
||||
clientId: record.clientId || file.clientId,
|
||||
eventId: record.eventId,
|
||||
timestamp: record.timestamp || `${file.date}T00:00:00.000Z`,
|
||||
@@ -43,8 +50,8 @@ async function getRecentStoredEvents(
|
||||
parser.destroy();
|
||||
}
|
||||
|
||||
if (fileEvents.length > 0) {
|
||||
collected.push(...fileEvents.slice(-limit));
|
||||
if (fileTail.length > 0) {
|
||||
collected.push(...fileTail);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,9 +62,10 @@ export async function GET(req: NextRequest) {
|
||||
try {
|
||||
const { searchParams } = new URL(req.url);
|
||||
const clientId = searchParams.get('clientId') || undefined;
|
||||
const limit = Math.min(200, Math.max(1, Number(searchParams.get('limit')) || 50));
|
||||
const limit = Math.min(2000, Math.max(1, Number(searchParams.get('limit')) || 50));
|
||||
|
||||
const bufferedEvents = DataBufferEngine.getRecentEvents(limit, clientId);
|
||||
await SettingsManager.getSettingsAsync();
|
||||
const bufferedEvents = await DataBufferEngine.getRecentEventsAsync(limit, clientId);
|
||||
let socketBufferedEvents: DataEvent[] = [];
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
|
||||
@@ -67,6 +75,7 @@ export async function GET(req: NextRequest) {
|
||||
const qs = new URLSearchParams({ limit: String(limit) });
|
||||
if (clientId) qs.set('clientId', clientId);
|
||||
const res = await fetch(`${ingestionBaseUrl}/internal/recent?${qs.toString()}`, {
|
||||
headers: getInternalIngestionHeaders(),
|
||||
cache: 'no-store',
|
||||
signal: AbortSignal.timeout(1000),
|
||||
});
|
||||
@@ -86,8 +95,10 @@ export async function GET(req: NextRequest) {
|
||||
const merged: DataEvent[] = [];
|
||||
|
||||
for (const evt of [...bufferedEvents, ...socketBufferedEvents, ...storedEvents]) {
|
||||
if (!evt.eventId || seenIds.has(evt.eventId)) continue;
|
||||
seenIds.add(evt.eventId);
|
||||
if (!evt.eventId) continue;
|
||||
const dedupeKey = `${evt.clientId}:${evt.eventId}`;
|
||||
if (seenIds.has(dedupeKey)) continue;
|
||||
seenIds.add(dedupeKey);
|
||||
merged.push(evt);
|
||||
}
|
||||
|
||||
@@ -107,9 +118,6 @@ export async function POST(req: NextRequest) {
|
||||
|
||||
// Verify either Client Bearer JWT or Admin Session Cookie
|
||||
const authHeader = req.headers.get('authorization');
|
||||
const sessionSecret =
|
||||
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
|
||||
'datastack-admin-session-default-secret-32-chars';
|
||||
const sessionCookie = req.cookies.get(SESSION_COOKIE)?.value;
|
||||
|
||||
let tokenClientId: string | null = null;
|
||||
@@ -128,20 +136,30 @@ export async function POST(req: NextRequest) {
|
||||
tokenClientId = verified.payload.clientId;
|
||||
tokenRateLimit = verified.payload.rateLimit;
|
||||
} else {
|
||||
const isAdmin = Boolean(
|
||||
sessionSecret &&
|
||||
sessionCookie &&
|
||||
(await verifyAdminSessionToken(sessionCookie, sessionSecret)),
|
||||
);
|
||||
if (!isAdmin) {
|
||||
const auth = await verifyRequestAdminSession(sessionCookie);
|
||||
if (!auth.valid) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Authentication required. Provide a valid Client Bearer JWT or Admin Session.' },
|
||||
{ status: 401 },
|
||||
{
|
||||
error:
|
||||
auth.reason === 'UNAUTHORIZED'
|
||||
? 'Authentication required. Provide a valid Client Bearer JWT or Admin Session.'
|
||||
: 'Admin authentication is temporarily unavailable.',
|
||||
},
|
||||
{ status: auth.reason === 'UNAUTHORIZED' ? 401 : 503 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const body = await req.json();
|
||||
let body: unknown;
|
||||
try {
|
||||
body = await req.json();
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: 'Invalid JSON request payload.', code: 'INVALID_JSON' },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
const bodyObj =
|
||||
body && typeof body === 'object' && !Array.isArray(body)
|
||||
? (body as Record<string, unknown>)
|
||||
@@ -154,7 +172,27 @@ export async function POST(req: NextRequest) {
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9_-]/g, '_');
|
||||
|
||||
if (await SDKManager.isClientDeletedAsync(clientId)) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: `CLIENT_REVOKED: Client '${clientId}' has been deleted by administrator.`,
|
||||
code: 'CLIENT_REVOKED',
|
||||
},
|
||||
{ status: 403 },
|
||||
);
|
||||
}
|
||||
|
||||
const registeredClient = await SDKManager.getClientAsync(clientId);
|
||||
if (rawBearerToken && !registeredClient) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: `CLIENT_REVOKED: Client '${clientId}' registration not found or has been deleted.`,
|
||||
code: 'CLIENT_REVOKED',
|
||||
},
|
||||
{ status: 403 },
|
||||
);
|
||||
}
|
||||
|
||||
if (registeredClient) {
|
||||
if (registeredClient.status === 'revoked') {
|
||||
return NextResponse.json(
|
||||
@@ -196,6 +234,7 @@ export async function POST(req: NextRequest) {
|
||||
: null;
|
||||
|
||||
if (batchItems !== null) {
|
||||
const envelopeKeys = new Set(['eventId', 'timestamp', 'data', 'clientId']);
|
||||
let processed = 0;
|
||||
let rejected = 0;
|
||||
let duplicates = 0;
|
||||
@@ -205,6 +244,8 @@ export async function POST(req: NextRequest) {
|
||||
const oversizedEventIds: string[] = [];
|
||||
const errors: string[] = [];
|
||||
|
||||
const batchEvents: DataEvent[] = [];
|
||||
|
||||
for (const item of batchItems) {
|
||||
const safeItem =
|
||||
item && typeof item === 'object' && !Array.isArray(item)
|
||||
@@ -221,8 +262,10 @@ export async function POST(req: NextRequest) {
|
||||
? safeItem.timestamp.trim()
|
||||
: new Date().toISOString();
|
||||
|
||||
const itemData =
|
||||
safeItem.data !== undefined ? safeItem.data : safeItem;
|
||||
const isEnvelope =
|
||||
safeItem.data !== undefined &&
|
||||
Object.keys(safeItem).every((k) => envelopeKeys.has(k));
|
||||
const itemData = isEnvelope ? safeItem.data : safeItem;
|
||||
|
||||
const val = SecurityEngine.validateEvent(
|
||||
clientId,
|
||||
@@ -251,18 +294,20 @@ export async function POST(req: NextRequest) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const event: DataEvent = {
|
||||
batchEvents.push({
|
||||
clientId,
|
||||
eventId: itemEventId,
|
||||
timestamp: itemTimestamp,
|
||||
payload:
|
||||
typeof itemData === 'string' ? itemData : JSON.stringify(itemData),
|
||||
};
|
||||
|
||||
DataBufferEngine.push(event);
|
||||
});
|
||||
processed += 1;
|
||||
}
|
||||
|
||||
if (batchEvents.length > 0) {
|
||||
await DataBufferEngine.pushBatchAsync(batchEvents);
|
||||
}
|
||||
|
||||
if (processed > 0) {
|
||||
SDKManager.incrementEventCount(clientId, processed);
|
||||
}
|
||||
@@ -347,7 +392,7 @@ export async function POST(req: NextRequest) {
|
||||
payload: typeof data === 'string' ? data : JSON.stringify(data),
|
||||
};
|
||||
|
||||
DataBufferEngine.push(event);
|
||||
await DataBufferEngine.pushAsync(event);
|
||||
SDKManager.incrementEventCount(clientId, 1);
|
||||
|
||||
return NextResponse.json({ success: true, eventId, event });
|
||||
|
||||
+40
-14
@@ -1,4 +1,6 @@
|
||||
import { SettingsManager } from "@/core/adminSettings/settingsManager";
|
||||
import { DataBufferEngine } from "@/core/data/buffer";
|
||||
import { getInternalIngestionHeaders } from "@/core/Security/security";
|
||||
import {
|
||||
CSVStorageManager,
|
||||
type StorageFileMeta,
|
||||
@@ -6,6 +8,7 @@ import {
|
||||
import { parse } from "csv-parse";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { createReadStream } from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
type ExportFormat = "csv" | "json";
|
||||
type CsvRecord = Record<string, string>;
|
||||
@@ -108,19 +111,26 @@ function streamFile(file: StorageFileMeta) {
|
||||
|
||||
export async function GET(request: NextRequest) {
|
||||
try {
|
||||
await DataBufferEngine.flush();
|
||||
const ingestionBaseUrl =
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000";
|
||||
try {
|
||||
await fetch(`${ingestionBaseUrl}/internal/flush`, {
|
||||
method: "POST",
|
||||
signal: AbortSignal.timeout(1500),
|
||||
});
|
||||
} catch {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
const { searchParams } = new URL(request.url);
|
||||
const shouldFlush = searchParams.get("flush") !== "false";
|
||||
|
||||
await SettingsManager.getSettingsAsync();
|
||||
if (shouldFlush) {
|
||||
await DataBufferEngine.flush();
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||
).replace(/\/$/, "");
|
||||
try {
|
||||
await fetch(`${ingestionBaseUrl}/internal/flush`, {
|
||||
method: "POST",
|
||||
headers: getInternalIngestionHeaders(),
|
||||
signal: AbortSignal.timeout(1500),
|
||||
});
|
||||
} catch {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
}
|
||||
}
|
||||
|
||||
const { searchParams } = new URL(request.url);
|
||||
const clientId = searchParams.get("clientId") || "all";
|
||||
const filename = searchParams.get("filename");
|
||||
const from = searchParams.get("from") || "";
|
||||
@@ -129,17 +139,33 @@ export async function GET(request: NextRequest) {
|
||||
searchParams.get("format") === "json" ? "json" : "csv";
|
||||
const includeHeader = searchParams.get("includeHeader") !== "false";
|
||||
|
||||
let files = CSVStorageManager.listAllFiles();
|
||||
let files = await CSVStorageManager.listAllFilesAsync();
|
||||
if (clientId !== "all")
|
||||
files = files.filter((file) => file.clientId === clientId);
|
||||
|
||||
if (filename) {
|
||||
const file = files.find((candidate) => candidate.filename === filename);
|
||||
if (!file)
|
||||
const safeFilename = path.basename(filename);
|
||||
const matches = files.filter(
|
||||
(candidate) => candidate.filename === safeFilename,
|
||||
);
|
||||
if (matches.length === 0) {
|
||||
return NextResponse.json(
|
||||
{ error: "CSV file not found" },
|
||||
{ status: 404 },
|
||||
);
|
||||
}
|
||||
if (matches.length > 1) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error:
|
||||
"Ambiguous filename matches multiple clients. Specify clientId.",
|
||||
code: "AMBIGUOUS_FILENAME",
|
||||
matchingClients: matches.map((m) => m.clientId),
|
||||
},
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
const file = matches[0];
|
||||
return new NextResponse(streamFile(file), {
|
||||
status: 200,
|
||||
headers: {
|
||||
|
||||
+13
-1
@@ -23,7 +23,19 @@ export async function GET(req: NextRequest) {
|
||||
);
|
||||
}
|
||||
|
||||
const clientId = verified.payload.clientId;
|
||||
const clientId = String(verified.payload.clientId || '')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
if (SDKManager.isClientDeleted(clientId)) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
status: 'error',
|
||||
code: 'CLIENT_REVOKED',
|
||||
error: `CLIENT_REVOKED: Client '${clientId}' has been deleted by administrator.`,
|
||||
},
|
||||
{ status: 403 },
|
||||
);
|
||||
}
|
||||
const registeredClient = await SDKManager.getClientAsync(clientId);
|
||||
if (registeredClient) {
|
||||
if (registeredClient.status === 'revoked') {
|
||||
|
||||
+176
-4
@@ -1,8 +1,55 @@
|
||||
import { SettingsManager } from "@/core/adminSettings/settingsManager";
|
||||
import {
|
||||
SettingsManager,
|
||||
type AdminSettings,
|
||||
} from "@/core/adminSettings/settingsManager";
|
||||
import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
|
||||
export async function GET() {
|
||||
const SESSION_COOKIE = "datastack_admin_session";
|
||||
|
||||
const ALLOWED_SETTING_KEYS = new Set([
|
||||
"maxEventsPerSec",
|
||||
"maxPayloadSizeKb",
|
||||
"batchFlushThreshold",
|
||||
"batchFlushIntervalMs",
|
||||
"fileRotationMb",
|
||||
"enableIdempotency",
|
||||
"autoPruneDays",
|
||||
"jwtSecret",
|
||||
]);
|
||||
|
||||
function parseBoundedInt(
|
||||
raw: unknown,
|
||||
field: string,
|
||||
min: number,
|
||||
max: number,
|
||||
): number {
|
||||
if (typeof raw !== "number" || !Number.isInteger(raw)) {
|
||||
throw new Error(`${field} must be an integer between ${min} and ${max}.`);
|
||||
}
|
||||
if (raw < min || raw > max) {
|
||||
throw new Error(`${field} must be between ${min} and ${max}.`);
|
||||
}
|
||||
return raw;
|
||||
}
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
try {
|
||||
const auth = await verifyRequestAdminSession(
|
||||
req.cookies.get(SESSION_COOKIE)?.value,
|
||||
);
|
||||
if (!auth.valid) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error:
|
||||
auth.reason === "UNAUTHORIZED"
|
||||
? "Authentication required."
|
||||
: "Admin authentication is temporarily unavailable.",
|
||||
},
|
||||
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
|
||||
);
|
||||
}
|
||||
|
||||
const settings = await SettingsManager.getPublicSettingsAsync();
|
||||
return NextResponse.json({ settings });
|
||||
} catch (err: unknown) {
|
||||
@@ -13,8 +60,133 @@ export async function GET() {
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
try {
|
||||
const body = await req.json();
|
||||
await SettingsManager.updateSettingsAsync(body);
|
||||
const auth = await verifyRequestAdminSession(
|
||||
req.cookies.get(SESSION_COOKIE)?.value,
|
||||
);
|
||||
if (!auth.valid) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error:
|
||||
auth.reason === "UNAUTHORIZED"
|
||||
? "Authentication required."
|
||||
: "Admin authentication is temporarily unavailable.",
|
||||
},
|
||||
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
|
||||
);
|
||||
}
|
||||
|
||||
let rawBody: unknown;
|
||||
try {
|
||||
rawBody = await req.json();
|
||||
} catch {
|
||||
return NextResponse.json(
|
||||
{ error: "Invalid JSON request payload.", code: "INVALID_JSON" },
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
if (!rawBody || typeof rawBody !== "object" || Array.isArray(rawBody)) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: "Settings payload must be a JSON object.",
|
||||
code: "VALIDATION_ERROR",
|
||||
},
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
const body = rawBody as Record<string, unknown>;
|
||||
const unknownKeys = Object.keys(body).filter(
|
||||
(key) => !ALLOWED_SETTING_KEYS.has(key),
|
||||
);
|
||||
if (unknownKeys.length > 0) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: `Unsupported settings field(s): ${unknownKeys.join(", ")}`,
|
||||
code: "VALIDATION_ERROR",
|
||||
},
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
const validated: Partial<AdminSettings> = {};
|
||||
try {
|
||||
if (body.maxEventsPerSec !== undefined) {
|
||||
validated.maxEventsPerSec = parseBoundedInt(
|
||||
body.maxEventsPerSec,
|
||||
"maxEventsPerSec",
|
||||
1,
|
||||
100000,
|
||||
);
|
||||
}
|
||||
if (body.maxPayloadSizeKb !== undefined) {
|
||||
validated.maxPayloadSizeKb = parseBoundedInt(
|
||||
body.maxPayloadSizeKb,
|
||||
"maxPayloadSizeKb",
|
||||
1,
|
||||
10240,
|
||||
);
|
||||
}
|
||||
if (body.batchFlushThreshold !== undefined) {
|
||||
validated.batchFlushThreshold = parseBoundedInt(
|
||||
body.batchFlushThreshold,
|
||||
"batchFlushThreshold",
|
||||
1,
|
||||
50000,
|
||||
);
|
||||
}
|
||||
if (body.batchFlushIntervalMs !== undefined) {
|
||||
validated.batchFlushIntervalMs = parseBoundedInt(
|
||||
body.batchFlushIntervalMs,
|
||||
"batchFlushIntervalMs",
|
||||
100,
|
||||
60000,
|
||||
);
|
||||
}
|
||||
if (body.fileRotationMb !== undefined) {
|
||||
validated.fileRotationMb = parseBoundedInt(
|
||||
body.fileRotationMb,
|
||||
"fileRotationMb",
|
||||
1,
|
||||
2048,
|
||||
);
|
||||
}
|
||||
if (body.autoPruneDays !== undefined) {
|
||||
validated.autoPruneDays = parseBoundedInt(
|
||||
body.autoPruneDays,
|
||||
"autoPruneDays",
|
||||
0,
|
||||
3650,
|
||||
);
|
||||
}
|
||||
if (body.enableIdempotency !== undefined) {
|
||||
if (typeof body.enableIdempotency !== "boolean") {
|
||||
throw new Error("enableIdempotency must be a boolean.");
|
||||
}
|
||||
validated.enableIdempotency = body.enableIdempotency;
|
||||
}
|
||||
if (body.jwtSecret !== undefined) {
|
||||
if (typeof body.jwtSecret !== "string" || body.jwtSecret.trim().length < 32) {
|
||||
throw new Error(
|
||||
"jwtSecret must be a string with at least 32 characters.",
|
||||
);
|
||||
}
|
||||
validated.jwtSecret = body.jwtSecret.trim();
|
||||
}
|
||||
} catch (validationErr: unknown) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error:
|
||||
validationErr instanceof Error
|
||||
? validationErr.message
|
||||
: "Invalid settings value.",
|
||||
code: "VALIDATION_ERROR",
|
||||
},
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
await SettingsManager.updateSettingsAsync(validated);
|
||||
const settings = await SettingsManager.getPublicSettingsAsync();
|
||||
return NextResponse.json({
|
||||
settings,
|
||||
|
||||
+59
-11
@@ -1,4 +1,6 @@
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { SettingsManager } from '@/core/adminSettings/settingsManager';
|
||||
import { getInternalIngestionHeaders } from '@/core/Security/security';
|
||||
import { CSVStorageManager } from '@/core/Storage/csvStorage';
|
||||
import { DataBufferEngine } from '@/core/data/buffer';
|
||||
|
||||
@@ -7,8 +9,11 @@ export async function GET(req: NextRequest) {
|
||||
const { searchParams } = new URL(req.url);
|
||||
const clientId = searchParams.get('clientId') || undefined;
|
||||
|
||||
const stats = CSVStorageManager.getStorageStats();
|
||||
let files = CSVStorageManager.listAllFiles();
|
||||
const [stats, allFiles] = await Promise.all([
|
||||
CSVStorageManager.getStorageStatsAsync(),
|
||||
CSVStorageManager.listAllFilesAsync(),
|
||||
]);
|
||||
let files = allFiles;
|
||||
|
||||
if (clientId && clientId !== 'all') {
|
||||
files = files.filter((f) => f.clientId === clientId);
|
||||
@@ -23,12 +28,17 @@ export async function GET(req: NextRequest) {
|
||||
|
||||
export async function POST(_req: NextRequest) {
|
||||
try {
|
||||
// 1. Flush local Next.js DataBufferEngine
|
||||
await SettingsManager.getSettingsAsync();
|
||||
|
||||
// 1. Flush local / shared Next.js DataBufferEngine
|
||||
const localRes = await DataBufferEngine.flush();
|
||||
|
||||
// 2. Also flush Port 4000 Socket.IO ingestion buffer
|
||||
let remoteCount = 0;
|
||||
let remoteFiles: string[] = [];
|
||||
let ingestionFlushed = false;
|
||||
let warning: string | undefined;
|
||||
const explicitIngestionUrl = Boolean(process.env.INGESTION_INTERNAL_URL?.trim());
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
|
||||
).replace(/\/$/, '');
|
||||
@@ -36,17 +46,53 @@ export async function POST(_req: NextRequest) {
|
||||
try {
|
||||
const ingestRes = await fetch(`${ingestionBaseUrl}/internal/flush`, {
|
||||
method: 'POST',
|
||||
headers: getInternalIngestionHeaders(),
|
||||
cache: 'no-store',
|
||||
signal: AbortSignal.timeout(2500),
|
||||
});
|
||||
if (ingestRes.ok) {
|
||||
const ingestData = await ingestRes.json();
|
||||
remoteCount = Number(ingestData.flushedCount) || 0;
|
||||
if (Array.isArray(ingestData.filesUpdated)) {
|
||||
remoteFiles = ingestData.filesUpdated;
|
||||
}
|
||||
if (!ingestRes.ok) {
|
||||
const errPayload = await ingestRes.json().catch(() => ({}));
|
||||
const remoteErr =
|
||||
typeof errPayload?.error === 'string'
|
||||
? errPayload.error
|
||||
: `Ingestion flush returned HTTP ${ingestRes.status}`;
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: remoteErr,
|
||||
code: 'INGESTION_FLUSH_FAILED',
|
||||
flushedCount: localRes.count,
|
||||
filesUpdated: localRes.files,
|
||||
ingestionFlushed: false,
|
||||
ingestionStatus: ingestRes.status,
|
||||
},
|
||||
{ status: 502 },
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
// Port 4000 server may be offline in standalone Next.js dev mode
|
||||
|
||||
ingestionFlushed = true;
|
||||
const ingestData = await ingestRes.json();
|
||||
remoteCount = Number(ingestData.flushedCount) || 0;
|
||||
if (Array.isArray(ingestData.filesUpdated)) {
|
||||
remoteFiles = ingestData.filesUpdated;
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
const errMsg =
|
||||
err instanceof Error ? err.message : 'Ingestion server unreachable';
|
||||
if (explicitIngestionUrl) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: `Ingestion server flush failed: ${errMsg}`,
|
||||
code: 'INGESTION_UNREACHABLE',
|
||||
flushedCount: localRes.count,
|
||||
filesUpdated: localRes.files,
|
||||
ingestionFlushed: false,
|
||||
},
|
||||
{ status: 502 },
|
||||
);
|
||||
}
|
||||
warning = `Port 4000 ingestion server is unreachable (${errMsg}); flushed Next.js buffer only.`;
|
||||
}
|
||||
|
||||
const combinedFiles = Array.from(
|
||||
@@ -57,6 +103,8 @@ export async function POST(_req: NextRequest) {
|
||||
success: true,
|
||||
flushedCount: localRes.count + remoteCount,
|
||||
filesUpdated: combinedFiles,
|
||||
ingestionFlushed,
|
||||
...(warning ? { warning } : {}),
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : 'Manual flush failed';
|
||||
|
||||
Reference in new issue
Block a user