refactor(auth): standardize session management and security

- Centralize secret retrieval using `getAdminSessionSecret`
- Replace manual session parsing with `verifyRequestAdminSession` utility
- Tighten API validation and improve security headers handling
- Enforce environment variable security for production deployments
This commit is contained in:
Team Dark Devil committed 2026-11-18 05:18:55 +06:00
1 parent 34519c07df
commit 177ed4a844
78 files changed
+6748 -3157

No files matched your search

+102 -79
View File
@@ -1,11 +1,12 @@
import {
createAdminSessionToken,
getAdminSessionSecret,
getSessionCookieOptions,
} from "@/core/auth/adminSession";
import {
authenticateAdminCredentials,
clearAdminLoginAttempts,
consumeAdminLoginAttempt,
verifyAdminCredentials,
} from "@/core/auth/adminUsers";
import { NextRequest, NextResponse } from "next/server";
@@ -13,10 +14,19 @@ const SESSION_COOKIE = "datastack_admin_session";
const SESSION_SECONDS = 8 * 60 * 60;
const REMEMBERED_SESSION_SECONDS = 30 * 24 * 60 * 60;
function extractClientIp(request: NextRequest) {
const forwardedFor = request.headers.get("x-forwarded-for");
if (forwardedFor) {
const first = forwardedFor.split(",")[0]?.trim();
if (first) return first;
}
const realIp = request.headers.get("x-real-ip")?.trim();
if (realIp) return realIp;
return "unknown";
}
export async function POST(request: NextRequest) {
const sessionSecret =
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
"datastack-admin-session-default-secret-32-chars";
const sessionSecret = getAdminSessionSecret();
if (!sessionSecret || sessionSecret.length < 32) {
return NextResponse.json(
@@ -27,86 +37,99 @@ export async function POST(request: NextRequest) {
);
}
let body: Record<string, unknown>;
try {
const body = await request.json();
const email =
typeof body.email === "string" ? body.email.trim().toLowerCase() : "";
const password = typeof body.password === "string" ? body.password : "";
if (!email || password.length === 0 || password.length > 1024) {
return NextResponse.json(
{ error: "Invalid email or password." },
{ status: 401 },
);
}
let throttleDecision;
try {
throttleDecision = await consumeAdminLoginAttempt(email, sessionSecret);
} catch {
return NextResponse.json(
{ error: "Admin authentication is temporarily unavailable." },
{ status: 503 },
);
}
if (!throttleDecision.allowed) {
return NextResponse.json(
{ error: "Too many sign-in attempts. Try again later." },
{
status: 429,
headers: {
"Retry-After": String(throttleDecision.retryAfterSeconds),
},
},
);
}
let authenticated = false;
try {
authenticated = await verifyAdminCredentials(email, password);
} catch {
return NextResponse.json(
{ error: "Admin authentication is temporarily unavailable." },
{ status: 503 },
);
}
if (!authenticated) {
return NextResponse.json(
{ error: "Invalid email or password." },
{ status: 401 },
);
}
try {
await clearAdminLoginAttempts(email, sessionSecret);
} catch {
return NextResponse.json(
{ error: "Admin authentication is temporarily unavailable." },
{ status: 503 },
);
}
const remember = body.remember === true;
const maxAge = remember ? REMEMBERED_SESSION_SECONDS : SESSION_SECONDS;
const expiresAt = Date.now() + maxAge * 1000;
const token = await createAdminSessionToken(
email,
sessionSecret,
expiresAt,
);
const response = NextResponse.json({ success: true });
response.cookies.set(
SESSION_COOKIE,
token,
getSessionCookieOptions(request, maxAge),
);
return response;
const parsed = await request.json();
body =
parsed && typeof parsed === "object" && !Array.isArray(parsed)
? (parsed as Record<string, unknown>)
: {};
} catch {
return NextResponse.json(
{ error: "Invalid login request." },
{ status: 400 },
);
}
const email =
typeof body.email === "string" ? body.email.trim().toLowerCase() : "";
const password = typeof body.password === "string" ? body.password : "";
if (!email || password.length === 0 || password.length > 1024) {
return NextResponse.json(
{ error: "Invalid email or password." },
{ status: 401 },
);
}
const clientIp = extractClientIp(request);
let throttleDecision;
try {
throttleDecision = await consumeAdminLoginAttempt(
email,
sessionSecret,
clientIp,
);
} catch {
return NextResponse.json(
{ error: "Admin authentication is temporarily unavailable." },
{ status: 503 },
);
}
if (!throttleDecision.allowed) {
return NextResponse.json(
{ error: "Too many sign-in attempts. Try again later." },
{
status: 429,
headers: {
"Retry-After": String(throttleDecision.retryAfterSeconds),
},
},
);
}
let authResult: { authenticated: boolean; tokenVersion: number };
try {
authResult = await authenticateAdminCredentials(email, password);
} catch {
return NextResponse.json(
{ error: "Admin authentication is temporarily unavailable." },
{ status: 503 },
);
}
if (!authResult.authenticated) {
return NextResponse.json(
{ error: "Invalid email or password." },
{ status: 401 },
);
}
try {
await clearAdminLoginAttempts(email, sessionSecret, clientIp);
} catch {
return NextResponse.json(
{ error: "Admin authentication is temporarily unavailable." },
{ status: 503 },
);
}
const remember = body.remember === true;
const maxAge = remember ? REMEMBERED_SESSION_SECONDS : SESSION_SECONDS;
const expiresAt = Date.now() + maxAge * 1000;
const token = await createAdminSessionToken(
email,
sessionSecret,
expiresAt,
authResult.tokenVersion,
);
const response = NextResponse.json({ success: true });
response.cookies.set(
SESSION_COOKIE,
token,
getSessionCookieOptions(request, maxAge),
);
return response;
}
+54 -35
View File
@@ -1,11 +1,11 @@
import {
createAdminSessionToken,
getAdminSessionPayload,
getSessionCookieOptions,
} from "@/core/auth/adminSession";
import {
getAdminAccountInfo,
updateAdminAccount,
verifyRequestAdminSession,
} from "@/core/auth/adminUsers";
import { NextRequest, NextResponse } from "next/server";
@@ -17,51 +17,69 @@ function validEmail(email: string) {
}
export async function GET(request: NextRequest) {
const sessionSecret =
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
"datastack-admin-session-default-secret-32-chars";
const token = request.cookies.get(SESSION_COOKIE)?.value || "";
const session = await getAdminSessionPayload(token, sessionSecret);
const token = request.cookies.get(SESSION_COOKIE)?.value;
const auth = await verifyRequestAdminSession(token);
if (!session) {
return NextResponse.json(
{ error: "Authentication required." },
{ status: 401 },
);
if (!auth.valid) {
const status = auth.reason === "UNAUTHORIZED" ? 401 : 503;
const message =
auth.reason === "UNAUTHORIZED"
? "Authentication required."
: "Admin authentication is temporarily unavailable.";
return NextResponse.json({ error: message }, { status });
}
const account = await getAdminAccountInfo(session.email);
return NextResponse.json({
profile: {
email: account.email,
role: "Owner",
storageMode: account.storageMode,
createdAt: account.createdAt,
sessionExpiresAt: new Date(session.expiresAt).toISOString(),
},
});
try {
const account = await getAdminAccountInfo(auth.session.email);
return NextResponse.json({
profile: {
email: account.email,
role: "Owner",
storageMode: account.storageMode,
createdAt: account.createdAt,
sessionExpiresAt: new Date(auth.session.expiresAt).toISOString(),
},
});
} catch {
return NextResponse.json(
{ error: "Unable to load admin profile." },
{ status: 503 },
);
}
}
export async function PUT(request: NextRequest) {
const sessionSecret =
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
"datastack-admin-session-default-secret-32-chars";
const token = request.cookies.get(SESSION_COOKIE)?.value || "";
const session = await getAdminSessionPayload(token, sessionSecret);
const token = request.cookies.get(SESSION_COOKIE)?.value;
const auth = await verifyRequestAdminSession(token);
if (!session) {
if (!auth.valid) {
const status = auth.reason === "UNAUTHORIZED" ? 401 : 503;
const message =
auth.reason === "UNAUTHORIZED"
? "Authentication required."
: "Admin authentication is temporarily unavailable.";
return NextResponse.json({ error: message }, { status });
}
let body: Record<string, unknown>;
try {
const parsed = await request.json();
body =
parsed && typeof parsed === "object" && !Array.isArray(parsed)
? (parsed as Record<string, unknown>)
: {};
} catch {
return NextResponse.json(
{ error: "Authentication required." },
{ status: 401 },
{ error: "Invalid profile payload." },
{ status: 400 },
);
}
try {
const body = await request.json();
const nextEmail =
typeof body.email === "string"
? body.email.trim().toLowerCase()
: session.email;
: auth.session.email;
const currentPassword =
typeof body.currentPassword === "string" ? body.currentPassword : "";
const newPassword =
@@ -88,7 +106,7 @@ export async function PUT(request: NextRequest) {
);
}
const account = await getAdminAccountInfo(session.email);
const account = await getAdminAccountInfo(auth.session.email);
const result = await updateAdminAccount({
currentEmail: account.email,
currentPassword,
@@ -103,13 +121,14 @@ export async function PUT(request: NextRequest) {
// Preserve remaining session duration (e.g., 30-day "Remember me" session)
const remainingSeconds = Math.max(
SESSION_SECONDS,
Math.ceil((session.expiresAt - Date.now()) / 1000),
Math.ceil((auth.session.expiresAt - Date.now()) / 1000),
);
const expiresAt = Date.now() + remainingSeconds * 1000;
const nextToken = await createAdminSessionToken(
result.email,
sessionSecret,
auth.sessionSecret,
expiresAt,
result.tokenVersion,
);
const response = NextResponse.json({
@@ -133,7 +152,7 @@ export async function PUT(request: NextRequest) {
} catch {
return NextResponse.json(
{ error: "Unable to update profile." },
{ status: 400 },
{ status: 500 },
);
}
}
+17 -4
View File
@@ -1,5 +1,6 @@
import {
createAdminSessionToken,
getAdminSessionSecret,
getSessionCookieOptions,
} from "@/core/auth/adminSession";
import {
@@ -20,9 +21,7 @@ function validEmail(email: string) {
}
export async function POST(request: NextRequest) {
const sessionSecret =
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
"datastack-admin-session-default-secret-32-chars";
const sessionSecret = getAdminSessionSecret();
if (!sessionSecret || sessionSecret.length < 32) {
return NextResponse.json(
{ error: "Admin session configuration is unavailable." },
@@ -30,8 +29,21 @@ export async function POST(request: NextRequest) {
);
}
let body: Record<string, unknown>;
try {
const parsed = await request.json();
body =
parsed && typeof parsed === "object" && !Array.isArray(parsed)
? (parsed as Record<string, unknown>)
: {};
} catch {
return NextResponse.json(
{ error: "Invalid setup request payload." },
{ status: 400 },
);
}
try {
const body = await request.json();
const email =
typeof body.email === "string" ? body.email.trim().toLowerCase() : "";
const password = typeof body.password === "string" ? body.password : "";
@@ -74,6 +86,7 @@ export async function POST(request: NextRequest) {
email,
sessionSecret,
Date.now() + SESSION_SECONDS * 1000,
1,
);
const response = NextResponse.json({ success: true });
response.cookies.set(
+46 -4
View File
@@ -1,23 +1,54 @@
import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
import { SDKManager } from "@/core/sdk/sdkManager";
import { NextRequest, NextResponse } from "next/server";
const SESSION_COOKIE = "datastack_admin_session";
export async function GET(
_req: NextRequest,
req: NextRequest,
{ params }: { params: Promise<{ id: string }> },
) {
const auth = await verifyRequestAdminSession(
req.cookies.get(SESSION_COOKIE)?.value,
);
if (!auth.valid) {
return NextResponse.json(
{ error: "Authentication required." },
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
);
}
const { id } = await params;
const client = await SDKManager.getClientAsync(id);
if (!client) {
return NextResponse.json({ error: "Client not found" }, { status: 404 });
}
const { secret, token, ...clientSummary } = client;
return NextResponse.json({ client: clientSummary });
void secret;
const includeToken = req.nextUrl.searchParams.get("includeToken") === "true";
return NextResponse.json({
client: {
...clientSummary,
tokenPreview: token ? `${token.slice(0, 12)}...` : "",
...(includeToken ? { token } : {}),
},
});
}
export async function PATCH(
req: NextRequest,
{ params }: { params: Promise<{ id: string }> },
) {
const auth = await verifyRequestAdminSession(
req.cookies.get(SESSION_COOKIE)?.value,
);
if (!auth.valid) {
return NextResponse.json(
{ error: "Authentication required." },
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
);
}
const { id } = await params;
try {
const body = await req.json();
@@ -46,11 +77,22 @@ export async function PATCH(
}
export async function DELETE(
_req: NextRequest,
req: NextRequest,
{ params }: { params: Promise<{ id: string }> },
) {
const auth = await verifyRequestAdminSession(
req.cookies.get(SESSION_COOKIE)?.value,
);
if (!auth.valid) {
return NextResponse.json(
{ error: "Authentication required." },
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
);
}
const { id } = await params;
const deleted = await SDKManager.deleteClientAsync(id);
const purgeFiles = req.nextUrl.searchParams.get("purgeFiles") === "true";
const deleted = await SDKManager.deleteClientAsync(id, { purgeFiles });
if (!deleted) {
return NextResponse.json({ error: "Client not found" }, { status: 404 });
}
+43 -4
View File
@@ -1,12 +1,28 @@
import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
import { SDKManager } from "@/core/sdk/sdkManager";
import { NextRequest, NextResponse } from "next/server";
export async function GET() {
const SESSION_COOKIE = "datastack_admin_session";
export async function GET(req: NextRequest) {
const auth = await verifyRequestAdminSession(
req.cookies.get(SESSION_COOKIE)?.value,
);
if (!auth.valid) {
return NextResponse.json(
{ error: "Authentication required." },
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
);
}
try {
const allClients = await SDKManager.listClientsAsync();
const clients = allClients.map(({ secret, ...client }) => {
const clients = allClients.map(({ secret, token, ...client }) => {
void secret;
return client;
return {
...client,
tokenPreview: token ? `${token.slice(0, 12)}...` : "",
};
});
return NextResponse.json({ clients, count: clients.length });
} catch (err: unknown) {
@@ -17,8 +33,31 @@ export async function GET() {
}
export async function POST(req: NextRequest) {
const auth = await verifyRequestAdminSession(
req.cookies.get(SESSION_COOKIE)?.value,
);
if (!auth.valid) {
return NextResponse.json(
{ error: "Authentication required." },
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
);
}
let body: Record<string, unknown>;
try {
const parsed = await req.json();
body =
parsed && typeof parsed === "object" && !Array.isArray(parsed)
? (parsed as Record<string, unknown>)
: {};
} catch {
return NextResponse.json(
{ error: "Invalid request payload." },
{ status: 400 },
);
}
try {
const body = await req.json();
const { clientId, name, rateLimit } = body;
if (!clientId || !name) {
+35 -9
View File
@@ -1,4 +1,6 @@
import { SettingsManager } from "@/core/adminSettings/settingsManager";
import { DataBufferEngine } from "@/core/data/buffer";
import { getInternalIngestionHeaders } from "@/core/Security/security";
import { CSVStorageManager } from "@/core/Storage/csvStorage";
import { parse } from "csv-parse";
import { NextRequest, NextResponse } from "next/server";
@@ -31,18 +33,25 @@ export async function GET(request: NextRequest) {
let total = 0;
const addIfMatching = (event: DataEvent) => {
const eventDate = event.timestamp.slice(0, 10);
const eventDate = CSVStorageManager.extractSafePartitionDate(event.timestamp);
const dedupeKey = `${event.clientId}:${event.eventId}`;
const matchesSearch =
!search ||
event.payload.toLowerCase().includes(search) ||
event.eventId.toLowerCase().includes(search) ||
event.clientId.toLowerCase().includes(search);
if (
(clientId !== "all" && event.clientId !== clientId) ||
(from && eventDate < from) ||
(to && eventDate > to) ||
(search && !event.payload.toLowerCase().includes(search)) ||
eventIds.has(event.eventId)
!matchesSearch ||
eventIds.has(dedupeKey)
) {
return;
}
eventIds.add(event.eventId);
eventIds.add(dedupeKey);
total += 1;
let low = 0;
@@ -109,7 +118,16 @@ export async function GET(request: NextRequest) {
}
}
for (const event of DataBufferEngine.getRecentEvents(200)) {
const settings = await SettingsManager.getSettingsAsync();
const recentWindowLimit = Math.min(
2000,
Math.max(200, settings.batchFlushThreshold || 500),
);
for (const event of await DataBufferEngine.getRecentEventsAsync(
recentWindowLimit,
clientId,
)) {
addIfMatching({ ...event, status: "buffered" });
}
@@ -117,10 +135,18 @@ export async function GET(request: NextRequest) {
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, "");
try {
const res = await fetch(`${ingestionBaseUrl}/internal/recent?limit=200`, {
cache: "no-store",
signal: AbortSignal.timeout(1000),
});
const qs = new URLSearchParams({ limit: String(recentWindowLimit) });
if (clientId && clientId !== "all") {
qs.set("clientId", clientId);
}
const res = await fetch(
`${ingestionBaseUrl}/internal/recent?${qs.toString()}`,
{
headers: getInternalIngestionHeaders(),
cache: "no-store",
signal: AbortSignal.timeout(1000),
},
);
if (res.ok) {
const data = await res.json();
if (Array.isArray(data.events)) {
+75 -30
View File
@@ -1,9 +1,12 @@
import { SettingsManager } from '@/core/adminSettings/settingsManager';
import { verifyAdminSessionToken } from '@/core/auth/adminSession';
import { verifyRequestAdminSession } from '@/core/auth/adminUsers';
import { AuthService } from '@/core/auth/jwt';
import { DataBufferEngine } from '@/core/data/buffer';
import { SDKManager } from '@/core/sdk/sdkManager';
import { SecurityEngine } from '@/core/Security/security';
import {
SecurityEngine,
getInternalIngestionHeaders,
} from '@/core/Security/security';
import { CSVStorageManager, type DataEvent } from '@/core/Storage/csvStorage';
import { parse } from 'csv-parse';
import { NextRequest, NextResponse } from 'next/server';
@@ -15,14 +18,15 @@ async function getRecentStoredEvents(
limit: number,
clientId?: string,
): Promise<DataEvent[]> {
const files = CSVStorageManager.listAllFiles()
const allFiles = await CSVStorageManager.listAllFilesAsync();
const files = allFiles
.filter((f) => !clientId || clientId === 'all' || f.clientId === clientId)
.slice(0, 5);
const collected: DataEvent[] = [];
for (const file of files) {
const fileEvents: DataEvent[] = [];
const fileTail: DataEvent[] = [];
const parser = createReadStream(file.filePath).pipe(
parse({ columns: true, skip_empty_lines: true, bom: true }),
);
@@ -32,7 +36,10 @@ async function getRecentStoredEvents(
Record<string, string>
>) {
if (!record.eventId) continue;
fileEvents.push({
if (fileTail.length >= limit) {
fileTail.shift();
}
fileTail.push({
clientId: record.clientId || file.clientId,
eventId: record.eventId,
timestamp: record.timestamp || `${file.date}T00:00:00.000Z`,
@@ -43,8 +50,8 @@ async function getRecentStoredEvents(
parser.destroy();
}
if (fileEvents.length > 0) {
collected.push(...fileEvents.slice(-limit));
if (fileTail.length > 0) {
collected.push(...fileTail);
}
}
@@ -55,9 +62,10 @@ export async function GET(req: NextRequest) {
try {
const { searchParams } = new URL(req.url);
const clientId = searchParams.get('clientId') || undefined;
const limit = Math.min(200, Math.max(1, Number(searchParams.get('limit')) || 50));
const limit = Math.min(2000, Math.max(1, Number(searchParams.get('limit')) || 50));
const bufferedEvents = DataBufferEngine.getRecentEvents(limit, clientId);
await SettingsManager.getSettingsAsync();
const bufferedEvents = await DataBufferEngine.getRecentEventsAsync(limit, clientId);
let socketBufferedEvents: DataEvent[] = [];
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
@@ -67,6 +75,7 @@ export async function GET(req: NextRequest) {
const qs = new URLSearchParams({ limit: String(limit) });
if (clientId) qs.set('clientId', clientId);
const res = await fetch(`${ingestionBaseUrl}/internal/recent?${qs.toString()}`, {
headers: getInternalIngestionHeaders(),
cache: 'no-store',
signal: AbortSignal.timeout(1000),
});
@@ -86,8 +95,10 @@ export async function GET(req: NextRequest) {
const merged: DataEvent[] = [];
for (const evt of [...bufferedEvents, ...socketBufferedEvents, ...storedEvents]) {
if (!evt.eventId || seenIds.has(evt.eventId)) continue;
seenIds.add(evt.eventId);
if (!evt.eventId) continue;
const dedupeKey = `${evt.clientId}:${evt.eventId}`;
if (seenIds.has(dedupeKey)) continue;
seenIds.add(dedupeKey);
merged.push(evt);
}
@@ -107,9 +118,6 @@ export async function POST(req: NextRequest) {
// Verify either Client Bearer JWT or Admin Session Cookie
const authHeader = req.headers.get('authorization');
const sessionSecret =
process.env.DATASTACK_ADMIN_SESSION_SECRET ||
'datastack-admin-session-default-secret-32-chars';
const sessionCookie = req.cookies.get(SESSION_COOKIE)?.value;
let tokenClientId: string | null = null;
@@ -128,20 +136,30 @@ export async function POST(req: NextRequest) {
tokenClientId = verified.payload.clientId;
tokenRateLimit = verified.payload.rateLimit;
} else {
const isAdmin = Boolean(
sessionSecret &&
sessionCookie &&
(await verifyAdminSessionToken(sessionCookie, sessionSecret)),
);
if (!isAdmin) {
const auth = await verifyRequestAdminSession(sessionCookie);
if (!auth.valid) {
return NextResponse.json(
{ error: 'Authentication required. Provide a valid Client Bearer JWT or Admin Session.' },
{ status: 401 },
{
error:
auth.reason === 'UNAUTHORIZED'
? 'Authentication required. Provide a valid Client Bearer JWT or Admin Session.'
: 'Admin authentication is temporarily unavailable.',
},
{ status: auth.reason === 'UNAUTHORIZED' ? 401 : 503 },
);
}
}
const body = await req.json();
let body: unknown;
try {
body = await req.json();
} catch {
return NextResponse.json(
{ error: 'Invalid JSON request payload.', code: 'INVALID_JSON' },
{ status: 400 },
);
}
const bodyObj =
body && typeof body === 'object' && !Array.isArray(body)
? (body as Record<string, unknown>)
@@ -154,7 +172,27 @@ export async function POST(req: NextRequest) {
.toLowerCase()
.replace(/[^a-z0-9_-]/g, '_');
if (await SDKManager.isClientDeletedAsync(clientId)) {
return NextResponse.json(
{
error: `CLIENT_REVOKED: Client '${clientId}' has been deleted by administrator.`,
code: 'CLIENT_REVOKED',
},
{ status: 403 },
);
}
const registeredClient = await SDKManager.getClientAsync(clientId);
if (rawBearerToken && !registeredClient) {
return NextResponse.json(
{
error: `CLIENT_REVOKED: Client '${clientId}' registration not found or has been deleted.`,
code: 'CLIENT_REVOKED',
},
{ status: 403 },
);
}
if (registeredClient) {
if (registeredClient.status === 'revoked') {
return NextResponse.json(
@@ -196,6 +234,7 @@ export async function POST(req: NextRequest) {
: null;
if (batchItems !== null) {
const envelopeKeys = new Set(['eventId', 'timestamp', 'data', 'clientId']);
let processed = 0;
let rejected = 0;
let duplicates = 0;
@@ -205,6 +244,8 @@ export async function POST(req: NextRequest) {
const oversizedEventIds: string[] = [];
const errors: string[] = [];
const batchEvents: DataEvent[] = [];
for (const item of batchItems) {
const safeItem =
item && typeof item === 'object' && !Array.isArray(item)
@@ -221,8 +262,10 @@ export async function POST(req: NextRequest) {
? safeItem.timestamp.trim()
: new Date().toISOString();
const itemData =
safeItem.data !== undefined ? safeItem.data : safeItem;
const isEnvelope =
safeItem.data !== undefined &&
Object.keys(safeItem).every((k) => envelopeKeys.has(k));
const itemData = isEnvelope ? safeItem.data : safeItem;
const val = SecurityEngine.validateEvent(
clientId,
@@ -251,18 +294,20 @@ export async function POST(req: NextRequest) {
continue;
}
const event: DataEvent = {
batchEvents.push({
clientId,
eventId: itemEventId,
timestamp: itemTimestamp,
payload:
typeof itemData === 'string' ? itemData : JSON.stringify(itemData),
};
DataBufferEngine.push(event);
});
processed += 1;
}
if (batchEvents.length > 0) {
await DataBufferEngine.pushBatchAsync(batchEvents);
}
if (processed > 0) {
SDKManager.incrementEventCount(clientId, processed);
}
@@ -347,7 +392,7 @@ export async function POST(req: NextRequest) {
payload: typeof data === 'string' ? data : JSON.stringify(data),
};
DataBufferEngine.push(event);
await DataBufferEngine.pushAsync(event);
SDKManager.incrementEventCount(clientId, 1);
return NextResponse.json({ success: true, eventId, event });
+40 -14
View File
@@ -1,4 +1,6 @@
import { SettingsManager } from "@/core/adminSettings/settingsManager";
import { DataBufferEngine } from "@/core/data/buffer";
import { getInternalIngestionHeaders } from "@/core/Security/security";
import {
CSVStorageManager,
type StorageFileMeta,
@@ -6,6 +8,7 @@ import {
import { parse } from "csv-parse";
import { NextRequest, NextResponse } from "next/server";
import { createReadStream } from "node:fs";
import path from "node:path";
type ExportFormat = "csv" | "json";
type CsvRecord = Record<string, string>;
@@ -108,19 +111,26 @@ function streamFile(file: StorageFileMeta) {
export async function GET(request: NextRequest) {
try {
await DataBufferEngine.flush();
const ingestionBaseUrl =
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000";
try {
await fetch(`${ingestionBaseUrl}/internal/flush`, {
method: "POST",
signal: AbortSignal.timeout(1500),
});
} catch {
// Port 4000 may not be running in isolated single-process dev mode
const { searchParams } = new URL(request.url);
const shouldFlush = searchParams.get("flush") !== "false";
await SettingsManager.getSettingsAsync();
if (shouldFlush) {
await DataBufferEngine.flush();
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, "");
try {
await fetch(`${ingestionBaseUrl}/internal/flush`, {
method: "POST",
headers: getInternalIngestionHeaders(),
signal: AbortSignal.timeout(1500),
});
} catch {
// Port 4000 may not be running in isolated single-process dev mode
}
}
const { searchParams } = new URL(request.url);
const clientId = searchParams.get("clientId") || "all";
const filename = searchParams.get("filename");
const from = searchParams.get("from") || "";
@@ -129,17 +139,33 @@ export async function GET(request: NextRequest) {
searchParams.get("format") === "json" ? "json" : "csv";
const includeHeader = searchParams.get("includeHeader") !== "false";
let files = CSVStorageManager.listAllFiles();
let files = await CSVStorageManager.listAllFilesAsync();
if (clientId !== "all")
files = files.filter((file) => file.clientId === clientId);
if (filename) {
const file = files.find((candidate) => candidate.filename === filename);
if (!file)
const safeFilename = path.basename(filename);
const matches = files.filter(
(candidate) => candidate.filename === safeFilename,
);
if (matches.length === 0) {
return NextResponse.json(
{ error: "CSV file not found" },
{ status: 404 },
);
}
if (matches.length > 1) {
return NextResponse.json(
{
error:
"Ambiguous filename matches multiple clients. Specify clientId.",
code: "AMBIGUOUS_FILENAME",
matchingClients: matches.map((m) => m.clientId),
},
{ status: 400 },
);
}
const file = matches[0];
return new NextResponse(streamFile(file), {
status: 200,
headers: {
+13 -1
View File
@@ -23,7 +23,19 @@ export async function GET(req: NextRequest) {
);
}
const clientId = verified.payload.clientId;
const clientId = String(verified.payload.clientId || '')
.trim()
.toLowerCase();
if (SDKManager.isClientDeleted(clientId)) {
return NextResponse.json(
{
status: 'error',
code: 'CLIENT_REVOKED',
error: `CLIENT_REVOKED: Client '${clientId}' has been deleted by administrator.`,
},
{ status: 403 },
);
}
const registeredClient = await SDKManager.getClientAsync(clientId);
if (registeredClient) {
if (registeredClient.status === 'revoked') {
+176 -4
View File
@@ -1,8 +1,55 @@
import { SettingsManager } from "@/core/adminSettings/settingsManager";
import {
SettingsManager,
type AdminSettings,
} from "@/core/adminSettings/settingsManager";
import { verifyRequestAdminSession } from "@/core/auth/adminUsers";
import { NextRequest, NextResponse } from "next/server";
export async function GET() {
const SESSION_COOKIE = "datastack_admin_session";
const ALLOWED_SETTING_KEYS = new Set([
"maxEventsPerSec",
"maxPayloadSizeKb",
"batchFlushThreshold",
"batchFlushIntervalMs",
"fileRotationMb",
"enableIdempotency",
"autoPruneDays",
"jwtSecret",
]);
function parseBoundedInt(
raw: unknown,
field: string,
min: number,
max: number,
): number {
if (typeof raw !== "number" || !Number.isInteger(raw)) {
throw new Error(`${field} must be an integer between ${min} and ${max}.`);
}
if (raw < min || raw > max) {
throw new Error(`${field} must be between ${min} and ${max}.`);
}
return raw;
}
export async function GET(req: NextRequest) {
try {
const auth = await verifyRequestAdminSession(
req.cookies.get(SESSION_COOKIE)?.value,
);
if (!auth.valid) {
return NextResponse.json(
{
error:
auth.reason === "UNAUTHORIZED"
? "Authentication required."
: "Admin authentication is temporarily unavailable.",
},
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
);
}
const settings = await SettingsManager.getPublicSettingsAsync();
return NextResponse.json({ settings });
} catch (err: unknown) {
@@ -13,8 +60,133 @@ export async function GET() {
export async function POST(req: NextRequest) {
try {
const body = await req.json();
await SettingsManager.updateSettingsAsync(body);
const auth = await verifyRequestAdminSession(
req.cookies.get(SESSION_COOKIE)?.value,
);
if (!auth.valid) {
return NextResponse.json(
{
error:
auth.reason === "UNAUTHORIZED"
? "Authentication required."
: "Admin authentication is temporarily unavailable.",
},
{ status: auth.reason === "UNAUTHORIZED" ? 401 : 503 },
);
}
let rawBody: unknown;
try {
rawBody = await req.json();
} catch {
return NextResponse.json(
{ error: "Invalid JSON request payload.", code: "INVALID_JSON" },
{ status: 400 },
);
}
if (!rawBody || typeof rawBody !== "object" || Array.isArray(rawBody)) {
return NextResponse.json(
{
error: "Settings payload must be a JSON object.",
code: "VALIDATION_ERROR",
},
{ status: 400 },
);
}
const body = rawBody as Record<string, unknown>;
const unknownKeys = Object.keys(body).filter(
(key) => !ALLOWED_SETTING_KEYS.has(key),
);
if (unknownKeys.length > 0) {
return NextResponse.json(
{
error: `Unsupported settings field(s): ${unknownKeys.join(", ")}`,
code: "VALIDATION_ERROR",
},
{ status: 400 },
);
}
const validated: Partial<AdminSettings> = {};
try {
if (body.maxEventsPerSec !== undefined) {
validated.maxEventsPerSec = parseBoundedInt(
body.maxEventsPerSec,
"maxEventsPerSec",
1,
100000,
);
}
if (body.maxPayloadSizeKb !== undefined) {
validated.maxPayloadSizeKb = parseBoundedInt(
body.maxPayloadSizeKb,
"maxPayloadSizeKb",
1,
10240,
);
}
if (body.batchFlushThreshold !== undefined) {
validated.batchFlushThreshold = parseBoundedInt(
body.batchFlushThreshold,
"batchFlushThreshold",
1,
50000,
);
}
if (body.batchFlushIntervalMs !== undefined) {
validated.batchFlushIntervalMs = parseBoundedInt(
body.batchFlushIntervalMs,
"batchFlushIntervalMs",
100,
60000,
);
}
if (body.fileRotationMb !== undefined) {
validated.fileRotationMb = parseBoundedInt(
body.fileRotationMb,
"fileRotationMb",
1,
2048,
);
}
if (body.autoPruneDays !== undefined) {
validated.autoPruneDays = parseBoundedInt(
body.autoPruneDays,
"autoPruneDays",
0,
3650,
);
}
if (body.enableIdempotency !== undefined) {
if (typeof body.enableIdempotency !== "boolean") {
throw new Error("enableIdempotency must be a boolean.");
}
validated.enableIdempotency = body.enableIdempotency;
}
if (body.jwtSecret !== undefined) {
if (typeof body.jwtSecret !== "string" || body.jwtSecret.trim().length < 32) {
throw new Error(
"jwtSecret must be a string with at least 32 characters.",
);
}
validated.jwtSecret = body.jwtSecret.trim();
}
} catch (validationErr: unknown) {
return NextResponse.json(
{
error:
validationErr instanceof Error
? validationErr.message
: "Invalid settings value.",
code: "VALIDATION_ERROR",
},
{ status: 400 },
);
}
await SettingsManager.updateSettingsAsync(validated);
const settings = await SettingsManager.getPublicSettingsAsync();
return NextResponse.json({
settings,
+59 -11
View File
@@ -1,4 +1,6 @@
import { NextRequest, NextResponse } from 'next/server';
import { SettingsManager } from '@/core/adminSettings/settingsManager';
import { getInternalIngestionHeaders } from '@/core/Security/security';
import { CSVStorageManager } from '@/core/Storage/csvStorage';
import { DataBufferEngine } from '@/core/data/buffer';
@@ -7,8 +9,11 @@ export async function GET(req: NextRequest) {
const { searchParams } = new URL(req.url);
const clientId = searchParams.get('clientId') || undefined;
const stats = CSVStorageManager.getStorageStats();
let files = CSVStorageManager.listAllFiles();
const [stats, allFiles] = await Promise.all([
CSVStorageManager.getStorageStatsAsync(),
CSVStorageManager.listAllFilesAsync(),
]);
let files = allFiles;
if (clientId && clientId !== 'all') {
files = files.filter((f) => f.clientId === clientId);
@@ -23,12 +28,17 @@ export async function GET(req: NextRequest) {
export async function POST(_req: NextRequest) {
try {
// 1. Flush local Next.js DataBufferEngine
await SettingsManager.getSettingsAsync();
// 1. Flush local / shared Next.js DataBufferEngine
const localRes = await DataBufferEngine.flush();
// 2. Also flush Port 4000 Socket.IO ingestion buffer
let remoteCount = 0;
let remoteFiles: string[] = [];
let ingestionFlushed = false;
let warning: string | undefined;
const explicitIngestionUrl = Boolean(process.env.INGESTION_INTERNAL_URL?.trim());
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
).replace(/\/$/, '');
@@ -36,17 +46,53 @@ export async function POST(_req: NextRequest) {
try {
const ingestRes = await fetch(`${ingestionBaseUrl}/internal/flush`, {
method: 'POST',
headers: getInternalIngestionHeaders(),
cache: 'no-store',
signal: AbortSignal.timeout(2500),
});
if (ingestRes.ok) {
const ingestData = await ingestRes.json();
remoteCount = Number(ingestData.flushedCount) || 0;
if (Array.isArray(ingestData.filesUpdated)) {
remoteFiles = ingestData.filesUpdated;
}
if (!ingestRes.ok) {
const errPayload = await ingestRes.json().catch(() => ({}));
const remoteErr =
typeof errPayload?.error === 'string'
? errPayload.error
: `Ingestion flush returned HTTP ${ingestRes.status}`;
return NextResponse.json(
{
success: false,
error: remoteErr,
code: 'INGESTION_FLUSH_FAILED',
flushedCount: localRes.count,
filesUpdated: localRes.files,
ingestionFlushed: false,
ingestionStatus: ingestRes.status,
},
{ status: 502 },
);
}
} catch {
// Port 4000 server may be offline in standalone Next.js dev mode
ingestionFlushed = true;
const ingestData = await ingestRes.json();
remoteCount = Number(ingestData.flushedCount) || 0;
if (Array.isArray(ingestData.filesUpdated)) {
remoteFiles = ingestData.filesUpdated;
}
} catch (err: unknown) {
const errMsg =
err instanceof Error ? err.message : 'Ingestion server unreachable';
if (explicitIngestionUrl) {
return NextResponse.json(
{
success: false,
error: `Ingestion server flush failed: ${errMsg}`,
code: 'INGESTION_UNREACHABLE',
flushedCount: localRes.count,
filesUpdated: localRes.files,
ingestionFlushed: false,
},
{ status: 502 },
);
}
warning = `Port 4000 ingestion server is unreachable (${errMsg}); flushed Next.js buffer only.`;
}
const combinedFiles = Array.from(
@@ -57,6 +103,8 @@ export async function POST(_req: NextRequest) {
success: true,
flushedCount: localRes.count + remoteCount,
filesUpdated: combinedFiles,
ingestionFlushed,
...(warning ? { warning } : {}),
});
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Manual flush failed';