2026-11-17 20:05:00 +06:00
|
|
|
import { SettingsManager } from '@/core/adminSettings/settingsManager';
|
|
|
|
|
import { AuthService } from '@/core/auth/jwt';
|
2026-11-19 20:22:01 +06:00
|
|
|
import { DataBufferEngine, REDIS_BUFFER_KEY } from '@/core/data/buffer';
|
2026-10-23 11:43:49 -05:00
|
|
|
import { SDKManager } from '@/core/sdk/sdkManager';
|
2026-11-19 23:02:27 +06:00
|
|
|
import {
|
|
|
|
|
isClientIpAllowed,
|
|
|
|
|
isClientTokenExpired,
|
|
|
|
|
} from '@/core/Security/ipAllowlist';
|
2026-11-18 05:18:55 +06:00
|
|
|
import {
|
|
|
|
|
SecurityEngine,
|
|
|
|
|
getInternalIngestionHeaders,
|
|
|
|
|
} from '@/core/Security/security';
|
2026-11-19 20:22:01 +06:00
|
|
|
import {
|
|
|
|
|
CSVStorageManager,
|
|
|
|
|
type DataEvent,
|
|
|
|
|
type StorageFileMeta,
|
|
|
|
|
} from '@/core/Storage/csvStorage';
|
2026-11-17 20:05:00 +06:00
|
|
|
import { parse } from 'csv-parse';
|
|
|
|
|
import { NextRequest, NextResponse } from 'next/server';
|
2026-11-19 20:22:01 +06:00
|
|
|
import crypto from 'node:crypto';
|
2026-11-19 23:02:27 +06:00
|
|
|
import { extractClientIp, requireAdminApiSession } from '@/lib/adminAuth';
|
2026-11-19 20:22:01 +06:00
|
|
|
import type { BufferedBucket, BufferedSummary } from '@/lib/vconUiHelpers';
|
2026-11-17 20:05:00 +06:00
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
const SINGLE_EVENT_ENVELOPE_KEYS = new Set(['eventId', 'timestamp', 'data', 'clientId']);
|
|
|
|
|
const MAX_SINGLE_CLIENT_SCAN_FILES = 10;
|
|
|
|
|
const MAX_ALL_CLIENTS_PER_CLIENT_FILES = 2;
|
|
|
|
|
const MAX_ALL_CLIENTS_TOTAL_SCAN_FILES = 50;
|
|
|
|
|
|
|
|
|
|
function generateFallbackEventId(): string {
|
|
|
|
|
return `evt_${Date.now()}_${crypto.randomBytes(6).toString('hex')}`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function extractSingleHttpEventData(rawBody: Record<string, unknown>): unknown {
|
|
|
|
|
if (rawBody.data !== undefined) {
|
|
|
|
|
const keys = Object.keys(rawBody);
|
|
|
|
|
if (keys.every((k) => SINGLE_EVENT_ENVELOPE_KEYS.has(k))) {
|
|
|
|
|
return rawBody.data;
|
|
|
|
|
}
|
|
|
|
|
return rawBody;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const stripped: Record<string, unknown> = {};
|
|
|
|
|
let hasNonEnvelopeKey = false;
|
|
|
|
|
for (const [k, v] of Object.entries(rawBody)) {
|
|
|
|
|
if (k === 'eventId' || k === 'timestamp' || k === 'clientId') {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
hasNonEnvelopeKey = true;
|
|
|
|
|
stripped[k] = v;
|
|
|
|
|
}
|
|
|
|
|
return hasNonEnvelopeKey ? stripped : {};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function parseTimestampMs(raw: string): number {
|
|
|
|
|
const trimmed = String(raw || '').trim();
|
|
|
|
|
if (/^\d+(\.\d+)?$/.test(trimmed)) {
|
|
|
|
|
const num = Number(trimmed);
|
|
|
|
|
return num < 1e11 ? num * 1000 : num;
|
|
|
|
|
}
|
|
|
|
|
const parsed = Date.parse(trimmed);
|
|
|
|
|
return Number.isNaN(parsed) ? Number.NEGATIVE_INFINITY : parsed;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function resolveEventUtcDate(rawTimestamp: string): string {
|
|
|
|
|
const ms = parseTimestampMs(rawTimestamp);
|
|
|
|
|
if (Number.isFinite(ms)) {
|
|
|
|
|
const parsed = new Date(ms);
|
|
|
|
|
if (!Number.isNaN(parsed.getTime())) {
|
|
|
|
|
return parsed.toISOString().slice(0, 10);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return new Date().toISOString().slice(0, 10);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function selectRecentPartitionFiles(
|
|
|
|
|
allFiles: StorageFileMeta[],
|
|
|
|
|
normClientId?: string,
|
|
|
|
|
sanitizedDir?: string,
|
|
|
|
|
): StorageFileMeta[] {
|
|
|
|
|
const matching = allFiles
|
|
|
|
|
.filter(
|
|
|
|
|
(f) =>
|
|
|
|
|
!normClientId ||
|
|
|
|
|
f.clientId === normClientId ||
|
|
|
|
|
f.clientId === sanitizedDir,
|
|
|
|
|
)
|
|
|
|
|
.sort((a, b) => {
|
|
|
|
|
if (a.date !== b.date) return b.date.localeCompare(a.date);
|
|
|
|
|
return new Date(b.modifiedAt).getTime() - new Date(a.modifiedAt).getTime();
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
if (normClientId) {
|
|
|
|
|
return matching.slice(0, MAX_SINGLE_CLIENT_SCAN_FILES);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Multi-client fair scan: take up to 2 newest partitions per active client (capped at 50 files total)
|
|
|
|
|
// so clients beyond the 5th most-recently-modified file are never silently hidden.
|
|
|
|
|
const perClientCounts = new Map<string, number>();
|
|
|
|
|
const selected: StorageFileMeta[] = [];
|
|
|
|
|
|
|
|
|
|
for (const file of matching) {
|
|
|
|
|
const currentCount = perClientCounts.get(file.clientId) ?? 0;
|
|
|
|
|
if (currentCount >= MAX_ALL_CLIENTS_PER_CLIENT_FILES) continue;
|
|
|
|
|
perClientCounts.set(file.clientId, currentCount + 1);
|
|
|
|
|
selected.push(file);
|
|
|
|
|
if (selected.length >= MAX_ALL_CLIENTS_TOTAL_SCAN_FILES) break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return selected;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function getPendingBufferedSummary(
|
|
|
|
|
ingestionBaseUrl: string,
|
|
|
|
|
filterClientId?: string,
|
|
|
|
|
): Promise<{ events: DataEvent[]; summary: BufferedSummary }> {
|
|
|
|
|
const bufferStats = await DataBufferEngine.getBufferStatsAsync();
|
|
|
|
|
const recentLimit = DataBufferEngine.getRecentLimit();
|
|
|
|
|
const pending: DataEvent[] = [];
|
|
|
|
|
let totalQueued = bufferStats.localBufferSize + bufferStats.redisBufferSize;
|
|
|
|
|
|
|
|
|
|
// 1. Read exact unflushed Redis queue (when Redis is enabled & connected)
|
|
|
|
|
const redis = DataBufferEngine.getRedisClient();
|
|
|
|
|
if (redis && bufferStats.redisBufferSize > 0) {
|
|
|
|
|
try {
|
|
|
|
|
const cap = Math.min(recentLimit, bufferStats.redisBufferSize);
|
|
|
|
|
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, cap - 1);
|
|
|
|
|
for (const raw of rawItems) {
|
|
|
|
|
try {
|
|
|
|
|
const parsed = JSON.parse(raw) as DataEvent;
|
|
|
|
|
if (!parsed || !parsed.eventId) continue;
|
|
|
|
|
pending.push(parsed);
|
|
|
|
|
} catch {
|
|
|
|
|
// Skip malformed queued item
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
// Fall back to local buffer telemetry
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 2. Read Port 3000 exact unflushed RAM queue snapshot
|
|
|
|
|
if (bufferStats.localBufferSize > 0) {
|
|
|
|
|
const localPending = DataBufferEngine.getLocalBufferSnapshot(recentLimit);
|
|
|
|
|
pending.push(...localPending);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 3. When Redis is not enabled or temporarily disconnected, Port 4000 `/internal/recent` serves its unflushed RAM `bufferQueue`
|
|
|
|
|
if (!bufferStats.redisEnabled || !bufferStats.redisConnected) {
|
|
|
|
|
try {
|
|
|
|
|
const qs = new URLSearchParams({ limit: String(recentLimit) });
|
|
|
|
|
const res = await fetch(`${ingestionBaseUrl}/internal/recent?${qs.toString()}`, {
|
|
|
|
|
headers: getInternalIngestionHeaders(),
|
|
|
|
|
cache: 'no-store',
|
|
|
|
|
signal: AbortSignal.timeout(1000),
|
|
|
|
|
});
|
|
|
|
|
if (res.ok) {
|
|
|
|
|
const data = await res.json();
|
|
|
|
|
if (Array.isArray(data.events) && data.events.length > 0) {
|
|
|
|
|
pending.push(...(data.events as DataEvent[]));
|
|
|
|
|
totalQueued += data.events.length;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
// Port 4000 may not be running in isolated single-process dev mode
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const seen = new Set<string>();
|
|
|
|
|
const dedupedAll: DataEvent[] = [];
|
|
|
|
|
const filteredEvents: DataEvent[] = [];
|
|
|
|
|
const bucketMap = new Map<string, BufferedBucket>();
|
|
|
|
|
const sanitizedFilter = filterClientId
|
|
|
|
|
? CSVStorageManager.sanitizeClientId(filterClientId)
|
|
|
|
|
: undefined;
|
|
|
|
|
|
|
|
|
|
for (const evt of pending) {
|
|
|
|
|
if (!evt || !evt.eventId) continue;
|
|
|
|
|
const normEvtClient = SDKManager.normalizeClientId(evt.clientId) || evt.clientId;
|
|
|
|
|
const dedupeKey = `${normEvtClient}:${evt.eventId}`;
|
|
|
|
|
if (seen.has(dedupeKey)) continue;
|
|
|
|
|
seen.add(dedupeKey);
|
|
|
|
|
dedupedAll.push(evt);
|
|
|
|
|
|
|
|
|
|
const matchesFilter =
|
|
|
|
|
!filterClientId ||
|
|
|
|
|
normEvtClient === filterClientId ||
|
|
|
|
|
evt.clientId === filterClientId ||
|
|
|
|
|
evt.clientId === sanitizedFilter;
|
|
|
|
|
|
|
|
|
|
if (matchesFilter) {
|
|
|
|
|
filteredEvents.push(evt);
|
|
|
|
|
const utcDate = resolveEventUtcDate(evt.timestamp);
|
|
|
|
|
const bucketKey = `${normEvtClient}|${utcDate}`;
|
|
|
|
|
const existing = bucketMap.get(bucketKey);
|
|
|
|
|
if (existing) {
|
|
|
|
|
existing.count += 1;
|
|
|
|
|
} else {
|
|
|
|
|
bucketMap.set(bucketKey, {
|
|
|
|
|
clientId: normEvtClient,
|
|
|
|
|
date: utcDate,
|
|
|
|
|
count: 1,
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const effectiveTotalQueued = Math.max(totalQueued, dedupedAll.length);
|
|
|
|
|
const unattributedCount = Math.max(0, effectiveTotalQueued - dedupedAll.length);
|
|
|
|
|
|
|
|
|
|
return {
|
|
|
|
|
events: filteredEvents,
|
|
|
|
|
summary: {
|
|
|
|
|
total: filteredEvents.length,
|
|
|
|
|
totalQueued: filterClientId
|
|
|
|
|
? filteredEvents.length + unattributedCount
|
|
|
|
|
: effectiveTotalQueued,
|
|
|
|
|
unattributedCount,
|
|
|
|
|
truncated: unattributedCount > 0,
|
|
|
|
|
buckets: Array.from(bucketMap.values()),
|
|
|
|
|
},
|
|
|
|
|
};
|
|
|
|
|
}
|
2026-11-17 20:05:00 +06:00
|
|
|
|
|
|
|
|
async function getRecentStoredEvents(
|
|
|
|
|
limit: number,
|
|
|
|
|
clientId?: string,
|
|
|
|
|
): Promise<DataEvent[]> {
|
2026-11-19 20:22:01 +06:00
|
|
|
const normClientId = clientId && clientId !== 'all' ? SDKManager.normalizeClientId(clientId) : undefined;
|
|
|
|
|
const sanitizedDir = normClientId ? CSVStorageManager.sanitizeClientId(normClientId) : undefined;
|
2026-11-18 05:18:55 +06:00
|
|
|
const allFiles = await CSVStorageManager.listAllFilesAsync();
|
2026-11-19 20:22:01 +06:00
|
|
|
const files = selectRecentPartitionFiles(allFiles, normClientId, sanitizedDir);
|
2026-11-17 20:05:00 +06:00
|
|
|
|
|
|
|
|
const collected: DataEvent[] = [];
|
|
|
|
|
|
|
|
|
|
for (const file of files) {
|
2026-11-18 05:18:55 +06:00
|
|
|
const fileTail: DataEvent[] = [];
|
2026-11-19 23:02:27 +06:00
|
|
|
const fileStream = CSVStorageManager.openPartitionTextStream(file.filePath);
|
2026-11-19 20:22:01 +06:00
|
|
|
const parser = fileStream.pipe(
|
2026-11-17 20:05:00 +06:00
|
|
|
parse({ columns: true, skip_empty_lines: true, bom: true }),
|
|
|
|
|
);
|
2026-11-19 23:02:27 +06:00
|
|
|
fileStream.on('error', (err) => {
|
|
|
|
|
if (!parser.destroyed) parser.destroy(err);
|
|
|
|
|
});
|
2026-11-17 20:05:00 +06:00
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
for await (const record of parser as AsyncIterable<
|
|
|
|
|
Record<string, string>
|
|
|
|
|
>) {
|
|
|
|
|
if (!record.eventId) continue;
|
2026-11-18 05:18:55 +06:00
|
|
|
if (fileTail.length >= limit) {
|
|
|
|
|
fileTail.shift();
|
|
|
|
|
}
|
|
|
|
|
fileTail.push({
|
2026-11-17 20:05:00 +06:00
|
|
|
clientId: record.clientId || file.clientId,
|
|
|
|
|
eventId: record.eventId,
|
|
|
|
|
timestamp: record.timestamp || `${file.date}T00:00:00.000Z`,
|
|
|
|
|
payload: record.payload || '',
|
|
|
|
|
});
|
|
|
|
|
}
|
2026-11-19 20:22:01 +06:00
|
|
|
} catch (err) {
|
2026-11-19 23:02:27 +06:00
|
|
|
console.warn(`[API /events] Failed reading partition ${file.filePath}:`, err);
|
2026-11-19 20:22:01 +06:00
|
|
|
} finally {
|
|
|
|
|
if (!parser.destroyed) parser.destroy();
|
|
|
|
|
if (!fileStream.destroyed) fileStream.destroy();
|
2026-11-17 20:05:00 +06:00
|
|
|
}
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
if (fileTail.length > 0) {
|
|
|
|
|
collected.push(...fileTail);
|
2026-11-17 20:05:00 +06:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return collected;
|
|
|
|
|
}
|
2026-10-23 11:43:49 -05:00
|
|
|
|
|
|
|
|
export async function GET(req: NextRequest) {
|
2026-11-19 20:22:01 +06:00
|
|
|
const auth = await requireAdminApiSession(req);
|
|
|
|
|
if (!auth.valid) {
|
|
|
|
|
return auth.response;
|
|
|
|
|
}
|
|
|
|
|
|
2026-10-23 11:43:49 -05:00
|
|
|
try {
|
|
|
|
|
const { searchParams } = new URL(req.url);
|
2026-11-19 20:22:01 +06:00
|
|
|
const rawClientId = searchParams.get('clientId');
|
|
|
|
|
const clientId =
|
|
|
|
|
rawClientId && rawClientId !== 'all'
|
|
|
|
|
? SDKManager.normalizeClientId(rawClientId) || undefined
|
|
|
|
|
: undefined;
|
2026-11-18 05:18:55 +06:00
|
|
|
const limit = Math.min(2000, Math.max(1, Number(searchParams.get('limit')) || 50));
|
2026-11-19 20:22:01 +06:00
|
|
|
const bufferedOnly = searchParams.get('bufferedOnly') === 'true';
|
|
|
|
|
const includeBufferedSummary =
|
|
|
|
|
bufferedOnly || searchParams.get('includeBufferedSummary') === 'true';
|
2026-11-17 20:05:00 +06:00
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
await SettingsManager.getSettingsAsync();
|
2026-11-18 03:20:00 +06:00
|
|
|
const ingestionBaseUrl = (
|
|
|
|
|
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
|
|
|
|
|
).replace(/\/$/, '');
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
if (bufferedOnly) {
|
|
|
|
|
const pendingResult = await getPendingBufferedSummary(ingestionBaseUrl, clientId);
|
|
|
|
|
const sortedPending = [...pendingResult.events].sort((a, b) => {
|
|
|
|
|
const aMs = parseTimestampMs(a.timestamp);
|
|
|
|
|
const bMs = parseTimestampMs(b.timestamp);
|
|
|
|
|
if (aMs !== bMs) return bMs - aMs;
|
|
|
|
|
return b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0;
|
|
|
|
|
});
|
|
|
|
|
const events = sortedPending.slice(0, limit);
|
|
|
|
|
return NextResponse.json({
|
|
|
|
|
events,
|
|
|
|
|
count: events.length,
|
|
|
|
|
bufferedSummary: pendingResult.summary,
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const bufferedEvents = await DataBufferEngine.getRecentEventsAsync(limit, clientId);
|
|
|
|
|
let socketBufferedEvents: DataEvent[] = [];
|
|
|
|
|
|
2026-11-18 03:20:00 +06:00
|
|
|
try {
|
|
|
|
|
const qs = new URLSearchParams({ limit: String(limit) });
|
|
|
|
|
if (clientId) qs.set('clientId', clientId);
|
|
|
|
|
const res = await fetch(`${ingestionBaseUrl}/internal/recent?${qs.toString()}`, {
|
2026-11-18 05:18:55 +06:00
|
|
|
headers: getInternalIngestionHeaders(),
|
2026-11-18 03:20:00 +06:00
|
|
|
cache: 'no-store',
|
|
|
|
|
signal: AbortSignal.timeout(1000),
|
|
|
|
|
});
|
|
|
|
|
if (res.ok) {
|
|
|
|
|
const data = await res.json();
|
|
|
|
|
if (Array.isArray(data.events)) {
|
|
|
|
|
socketBufferedEvents = data.events;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
// Port 4000 may not be running in isolated single-process dev mode
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
const [storedEvents, pendingResult] = await Promise.all([
|
|
|
|
|
getRecentStoredEvents(limit, clientId),
|
|
|
|
|
includeBufferedSummary
|
|
|
|
|
? getPendingBufferedSummary(ingestionBaseUrl, clientId)
|
|
|
|
|
: Promise.resolve(null),
|
|
|
|
|
]);
|
2026-11-17 20:05:00 +06:00
|
|
|
|
|
|
|
|
const seenIds = new Set<string>();
|
|
|
|
|
const merged: DataEvent[] = [];
|
|
|
|
|
|
2026-11-18 03:20:00 +06:00
|
|
|
for (const evt of [...bufferedEvents, ...socketBufferedEvents, ...storedEvents]) {
|
2026-11-18 05:18:55 +06:00
|
|
|
if (!evt.eventId) continue;
|
2026-11-19 20:22:01 +06:00
|
|
|
const normEvtClient = SDKManager.normalizeClientId(evt.clientId) || evt.clientId;
|
|
|
|
|
const dedupeKey = `${normEvtClient}:${evt.eventId}`;
|
2026-11-18 05:18:55 +06:00
|
|
|
if (seenIds.has(dedupeKey)) continue;
|
|
|
|
|
seenIds.add(dedupeKey);
|
2026-11-17 20:05:00 +06:00
|
|
|
merged.push(evt);
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
merged.sort((a, b) => {
|
|
|
|
|
const aMs = parseTimestampMs(a.timestamp);
|
|
|
|
|
const bMs = parseTimestampMs(b.timestamp);
|
|
|
|
|
if (aMs !== bMs) return bMs - aMs;
|
|
|
|
|
return b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0;
|
|
|
|
|
});
|
2026-11-17 20:05:00 +06:00
|
|
|
const events = merged.slice(0, limit);
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
return NextResponse.json({
|
|
|
|
|
events,
|
|
|
|
|
count: events.length,
|
|
|
|
|
...(pendingResult ? { bufferedSummary: pendingResult.summary } : {}),
|
|
|
|
|
});
|
2026-10-23 11:43:49 -05:00
|
|
|
} catch (err: unknown) {
|
|
|
|
|
const message = err instanceof Error ? err.message : 'Events retrieval failed';
|
|
|
|
|
return NextResponse.json({ error: message }, { status: 500 });
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export async function POST(req: NextRequest) {
|
|
|
|
|
try {
|
2026-11-17 20:05:00 +06:00
|
|
|
const settings = await SettingsManager.getSettingsAsync();
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-11-21 00:06:55 +06:00
|
|
|
// Verify either Client Bearer JWT or Admin Session Cookie (minRole: operator)
|
2026-11-17 20:05:00 +06:00
|
|
|
const authHeader = req.headers.get('authorization');
|
|
|
|
|
|
|
|
|
|
let tokenClientId: string | null = null;
|
|
|
|
|
let tokenRateLimit: number | undefined;
|
|
|
|
|
let rawBearerToken: string | null = null;
|
|
|
|
|
|
|
|
|
|
if (authHeader && authHeader.toLowerCase().startsWith('bearer ')) {
|
|
|
|
|
rawBearerToken = authHeader.slice(7).trim();
|
|
|
|
|
const verified = AuthService.verifyClientToken(rawBearerToken);
|
|
|
|
|
if (!verified.valid || !verified.payload) {
|
|
|
|
|
return NextResponse.json(
|
|
|
|
|
{ error: verified.error || 'Invalid client JWT token.' },
|
|
|
|
|
{ status: 401 },
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
tokenClientId = verified.payload.clientId;
|
|
|
|
|
tokenRateLimit = verified.payload.rateLimit;
|
|
|
|
|
} else {
|
2026-11-21 00:06:55 +06:00
|
|
|
const auth = await requireAdminApiSession(req, { minRole: 'operator' });
|
2026-11-18 05:18:55 +06:00
|
|
|
if (!auth.valid) {
|
2026-11-21 00:06:55 +06:00
|
|
|
return auth.response;
|
2026-11-17 20:05:00 +06:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
let body: unknown;
|
|
|
|
|
try {
|
|
|
|
|
body = await req.json();
|
|
|
|
|
} catch {
|
|
|
|
|
return NextResponse.json(
|
|
|
|
|
{ error: 'Invalid JSON request payload.', code: 'INVALID_JSON' },
|
|
|
|
|
{ status: 400 },
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-18 02:23:01 +06:00
|
|
|
const bodyObj =
|
|
|
|
|
body && typeof body === 'object' && !Array.isArray(body)
|
|
|
|
|
? (body as Record<string, unknown>)
|
|
|
|
|
: {};
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
const clientId = SDKManager.normalizeClientId(
|
|
|
|
|
String(tokenClientId || bodyObj.clientId || ''),
|
|
|
|
|
);
|
|
|
|
|
if (!clientId) {
|
|
|
|
|
return NextResponse.json(
|
|
|
|
|
{
|
|
|
|
|
error: 'clientId is required and must belong to a registered client.',
|
|
|
|
|
code: 'VALIDATION_ERROR',
|
|
|
|
|
},
|
|
|
|
|
{ status: 400 },
|
|
|
|
|
);
|
|
|
|
|
}
|
2026-11-17 20:05:00 +06:00
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
if (await SDKManager.isClientDeletedAsync(clientId)) {
|
|
|
|
|
return NextResponse.json(
|
|
|
|
|
{
|
|
|
|
|
error: `CLIENT_REVOKED: Client '${clientId}' has been deleted by administrator.`,
|
|
|
|
|
code: 'CLIENT_REVOKED',
|
|
|
|
|
},
|
|
|
|
|
{ status: 403 },
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-17 20:05:00 +06:00
|
|
|
const registeredClient = await SDKManager.getClientAsync(clientId);
|
2026-11-19 20:22:01 +06:00
|
|
|
if (!registeredClient) {
|
2026-11-18 05:18:55 +06:00
|
|
|
return NextResponse.json(
|
|
|
|
|
{
|
|
|
|
|
error: `CLIENT_REVOKED: Client '${clientId}' registration not found or has been deleted.`,
|
|
|
|
|
code: 'CLIENT_REVOKED',
|
|
|
|
|
},
|
|
|
|
|
{ status: 403 },
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-17 20:05:00 +06:00
|
|
|
if (registeredClient) {
|
|
|
|
|
if (registeredClient.status === 'revoked') {
|
|
|
|
|
return NextResponse.json(
|
2026-11-18 02:23:01 +06:00
|
|
|
{
|
|
|
|
|
error: `CLIENT_REVOKED: Client '${clientId}' access has been revoked by administrator.`,
|
|
|
|
|
code: 'CLIENT_REVOKED',
|
|
|
|
|
},
|
2026-11-17 20:05:00 +06:00
|
|
|
{ status: 403 },
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
if (
|
|
|
|
|
rawBearerToken &&
|
|
|
|
|
registeredClient.token &&
|
|
|
|
|
registeredClient.token !== rawBearerToken
|
|
|
|
|
) {
|
|
|
|
|
return NextResponse.json(
|
2026-11-18 02:23:01 +06:00
|
|
|
{
|
|
|
|
|
error: 'TOKEN_ROTATED: Client credentials have been rotated. Use the latest token.',
|
|
|
|
|
code: 'TOKEN_ROTATED',
|
|
|
|
|
},
|
2026-11-17 20:05:00 +06:00
|
|
|
{ status: 401 },
|
|
|
|
|
);
|
|
|
|
|
}
|
2026-11-19 23:02:27 +06:00
|
|
|
if (isClientTokenExpired(registeredClient.expiresAt)) {
|
|
|
|
|
SettingsManager.triggerWebhookAlert({
|
|
|
|
|
event: 'CLIENT_TOKEN_EXPIRED',
|
|
|
|
|
severity: 'warning',
|
|
|
|
|
title: 'Expired Client Token Rejected',
|
|
|
|
|
message: `Client '${clientId}' attempted HTTP ingestion with an expired token (expiresAt=${registeredClient.expiresAt}).`,
|
|
|
|
|
clientId,
|
|
|
|
|
details: { expiresAt: registeredClient.expiresAt },
|
|
|
|
|
});
|
|
|
|
|
return NextResponse.json(
|
|
|
|
|
{
|
|
|
|
|
error: `CLIENT_TOKEN_EXPIRED: Client '${clientId}' token expired at ${registeredClient.expiresAt}. Rotate or extend client expiration.`,
|
|
|
|
|
code: 'CLIENT_TOKEN_EXPIRED',
|
|
|
|
|
},
|
|
|
|
|
{ status: 401 },
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
if (registeredClient.allowedIps && registeredClient.allowedIps.trim()) {
|
|
|
|
|
const clientIp = extractClientIp(req);
|
|
|
|
|
if (!isClientIpAllowed(clientIp, registeredClient.allowedIps)) {
|
|
|
|
|
SettingsManager.triggerWebhookAlert({
|
|
|
|
|
event: 'CLIENT_IP_FORBIDDEN',
|
|
|
|
|
severity: 'warning',
|
|
|
|
|
title: 'Unauthorized Client IP Blocked',
|
|
|
|
|
message: `Client '${clientId}' request from IP '${clientIp}' was blocked by allowedIps policy.`,
|
|
|
|
|
clientId,
|
|
|
|
|
details: {
|
|
|
|
|
clientIp,
|
|
|
|
|
allowedIps: registeredClient.allowedIps,
|
|
|
|
|
},
|
|
|
|
|
});
|
|
|
|
|
return NextResponse.json(
|
|
|
|
|
{
|
|
|
|
|
error: `CLIENT_IP_FORBIDDEN: Request IP '${clientIp}' is not authorized for client '${clientId}'.`,
|
|
|
|
|
code: 'CLIENT_IP_FORBIDDEN',
|
|
|
|
|
},
|
|
|
|
|
{ status: 403 },
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const storageAdmission = await CSVStorageManager.checkStorageAdmissionAsync(
|
|
|
|
|
clientId,
|
|
|
|
|
registeredClient.storageQuotaMb,
|
|
|
|
|
settings.diskPauseThresholdPct,
|
|
|
|
|
);
|
|
|
|
|
if (!storageAdmission.allowed) {
|
|
|
|
|
if (storageAdmission.code) {
|
|
|
|
|
SettingsManager.triggerWebhookAlert({
|
|
|
|
|
event: storageAdmission.code,
|
|
|
|
|
severity:
|
|
|
|
|
storageAdmission.code === 'DISK_SPACE_LOW' ? 'critical' : 'warning',
|
|
|
|
|
title:
|
|
|
|
|
storageAdmission.code === 'DISK_SPACE_LOW'
|
|
|
|
|
? 'VPS Disk Pause Threshold Reached'
|
|
|
|
|
: 'Client Storage Quota Exceeded',
|
|
|
|
|
message:
|
|
|
|
|
storageAdmission.reason ||
|
|
|
|
|
`Storage admission paused (${storageAdmission.code}) for client '${clientId}'.`,
|
|
|
|
|
clientId,
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
return NextResponse.json(
|
|
|
|
|
{
|
|
|
|
|
success: false,
|
|
|
|
|
error: storageAdmission.reason,
|
|
|
|
|
code: storageAdmission.code,
|
|
|
|
|
},
|
|
|
|
|
{ status: 507 },
|
|
|
|
|
);
|
2026-11-17 20:05:00 +06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const effectiveClientRate =
|
|
|
|
|
registeredClient?.rateLimit ||
|
|
|
|
|
tokenRateLimit ||
|
|
|
|
|
settings.maxEventsPerSec ||
|
|
|
|
|
100;
|
|
|
|
|
|
2026-11-18 02:23:01 +06:00
|
|
|
// Detect if this is a batch array or direct file-records ingestion request
|
|
|
|
|
const batchItems: unknown[] | null = Array.isArray(body)
|
|
|
|
|
? body
|
|
|
|
|
: Array.isArray(bodyObj.batch)
|
|
|
|
|
? (bodyObj.batch as unknown[])
|
|
|
|
|
: Array.isArray(bodyObj.records)
|
|
|
|
|
? (bodyObj.records as unknown[])
|
2026-11-19 20:22:01 +06:00
|
|
|
: bodyObj.data === undefined && Array.isArray(bodyObj.events)
|
|
|
|
|
? (bodyObj.events as unknown[])
|
|
|
|
|
: null;
|
2026-11-18 02:23:01 +06:00
|
|
|
|
|
|
|
|
if (batchItems !== null) {
|
|
|
|
|
let processed = 0;
|
|
|
|
|
let rejected = 0;
|
|
|
|
|
let duplicates = 0;
|
|
|
|
|
let hitRateLimit = false;
|
|
|
|
|
let hitPayloadLimit = false;
|
2026-11-18 02:51:04 +06:00
|
|
|
const rateLimitedEventIds: string[] = [];
|
|
|
|
|
const oversizedEventIds: string[] = [];
|
2026-11-18 02:23:01 +06:00
|
|
|
const errors: string[] = [];
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
const batchEvents: DataEvent[] = [];
|
|
|
|
|
|
2026-11-18 02:23:01 +06:00
|
|
|
for (const item of batchItems) {
|
|
|
|
|
const safeItem =
|
|
|
|
|
item && typeof item === 'object' && !Array.isArray(item)
|
|
|
|
|
? (item as Record<string, unknown>)
|
|
|
|
|
: { value: item };
|
|
|
|
|
|
|
|
|
|
const itemEventId =
|
|
|
|
|
typeof safeItem.eventId === 'string' && safeItem.eventId.trim()
|
|
|
|
|
? safeItem.eventId.trim()
|
2026-11-19 20:22:01 +06:00
|
|
|
: generateFallbackEventId();
|
2026-11-18 02:23:01 +06:00
|
|
|
|
|
|
|
|
const itemTimestamp =
|
|
|
|
|
typeof safeItem.timestamp === 'string' && safeItem.timestamp.trim()
|
|
|
|
|
? safeItem.timestamp.trim()
|
|
|
|
|
: new Date().toISOString();
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
const itemData = extractSingleHttpEventData(safeItem);
|
2026-11-18 02:23:01 +06:00
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
const val = await SecurityEngine.validateEventAsync(
|
2026-11-18 02:23:01 +06:00
|
|
|
clientId,
|
|
|
|
|
itemEventId,
|
|
|
|
|
itemData,
|
|
|
|
|
effectiveClientRate,
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
if (!val.allowed) {
|
|
|
|
|
if (val.code === 'DUPLICATE_EVENT') {
|
|
|
|
|
duplicates += 1;
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
rejected += 1;
|
2026-11-18 02:51:04 +06:00
|
|
|
if (val.code === 'RATE_LIMIT_EXCEEDED') {
|
|
|
|
|
hitRateLimit = true;
|
|
|
|
|
rateLimitedEventIds.push(itemEventId);
|
|
|
|
|
}
|
|
|
|
|
if (val.code === 'PAYLOAD_TOO_LARGE') {
|
|
|
|
|
hitPayloadLimit = true;
|
|
|
|
|
oversizedEventIds.push(itemEventId);
|
|
|
|
|
}
|
2026-11-18 02:23:01 +06:00
|
|
|
if (errors.length < 5) {
|
|
|
|
|
errors.push(`${itemEventId}: ${val.code || 'REJECTED'} - ${val.reason}`);
|
|
|
|
|
}
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
batchEvents.push({
|
2026-11-18 02:23:01 +06:00
|
|
|
clientId,
|
|
|
|
|
eventId: itemEventId,
|
|
|
|
|
timestamp: itemTimestamp,
|
|
|
|
|
payload:
|
|
|
|
|
typeof itemData === 'string' ? itemData : JSON.stringify(itemData),
|
2026-11-18 05:18:55 +06:00
|
|
|
});
|
2026-11-18 02:23:01 +06:00
|
|
|
processed += 1;
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
if (batchEvents.length > 0) {
|
|
|
|
|
await DataBufferEngine.pushBatchAsync(batchEvents);
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-18 02:23:01 +06:00
|
|
|
if (processed > 0) {
|
2026-11-19 20:22:01 +06:00
|
|
|
await SDKManager.incrementEventCountAsync(clientId, processed);
|
2026-11-18 02:23:01 +06:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// If zero items were processed or deduplicated and all failed due to payload or rate limit, return corresponding status
|
|
|
|
|
if (processed === 0 && duplicates === 0 && rejected > 0) {
|
|
|
|
|
const status = hitPayloadLimit ? 413 : hitRateLimit ? 429 : 400;
|
|
|
|
|
const code = hitPayloadLimit
|
|
|
|
|
? 'PAYLOAD_TOO_LARGE'
|
|
|
|
|
: hitRateLimit
|
|
|
|
|
? 'RATE_LIMIT_EXCEEDED'
|
|
|
|
|
: 'BATCH_REJECTED';
|
|
|
|
|
return NextResponse.json(
|
|
|
|
|
{
|
|
|
|
|
success: false,
|
2026-11-19 20:22:01 +06:00
|
|
|
accepted: processed,
|
2026-11-18 02:23:01 +06:00
|
|
|
processed,
|
|
|
|
|
rejected,
|
|
|
|
|
duplicates,
|
|
|
|
|
code,
|
|
|
|
|
error: errors[0] || 'Batch validation failed',
|
|
|
|
|
errors,
|
2026-11-18 02:51:04 +06:00
|
|
|
...(rateLimitedEventIds.length > 0 ? { rateLimitedEventIds } : {}),
|
|
|
|
|
...(oversizedEventIds.length > 0 ? { oversizedEventIds } : {}),
|
2026-11-18 02:23:01 +06:00
|
|
|
},
|
|
|
|
|
{ status },
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return NextResponse.json({
|
|
|
|
|
success: true,
|
2026-11-19 20:22:01 +06:00
|
|
|
accepted: processed,
|
2026-11-18 02:23:01 +06:00
|
|
|
processed,
|
|
|
|
|
rejected,
|
|
|
|
|
duplicates,
|
|
|
|
|
...(typeof bodyObj.fileName === 'string'
|
|
|
|
|
? { fileName: bodyObj.fileName }
|
|
|
|
|
: {}),
|
2026-11-18 02:51:04 +06:00
|
|
|
...(rateLimitedEventIds.length > 0 ? { rateLimitedEventIds } : {}),
|
|
|
|
|
...(oversizedEventIds.length > 0 ? { oversizedEventIds } : {}),
|
2026-11-18 02:23:01 +06:00
|
|
|
...(errors.length > 0 ? { errors } : {}),
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-19 20:22:01 +06:00
|
|
|
// Single Event Ingestion (exact envelope parity with Port 4000 WS sync_data)
|
2026-11-18 02:23:01 +06:00
|
|
|
const eventId =
|
|
|
|
|
typeof bodyObj.eventId === 'string' && bodyObj.eventId.trim()
|
|
|
|
|
? bodyObj.eventId.trim()
|
2026-11-19 20:22:01 +06:00
|
|
|
: generateFallbackEventId();
|
2026-11-18 02:23:01 +06:00
|
|
|
const timestamp =
|
|
|
|
|
typeof bodyObj.timestamp === 'string' && bodyObj.timestamp.trim()
|
|
|
|
|
? bodyObj.timestamp.trim()
|
|
|
|
|
: new Date().toISOString();
|
2026-11-19 20:22:01 +06:00
|
|
|
const data = extractSingleHttpEventData(bodyObj);
|
2026-11-18 02:23:01 +06:00
|
|
|
|
2026-11-17 20:05:00 +06:00
|
|
|
// Validate security (idempotency, rate limit, payload size)
|
2026-11-19 20:22:01 +06:00
|
|
|
const val = await SecurityEngine.validateEventAsync(
|
2026-11-17 20:05:00 +06:00
|
|
|
clientId,
|
|
|
|
|
eventId,
|
|
|
|
|
data,
|
|
|
|
|
effectiveClientRate,
|
|
|
|
|
);
|
2026-10-23 11:43:49 -05:00
|
|
|
if (!val.allowed) {
|
2026-11-18 02:23:01 +06:00
|
|
|
if (val.code === 'DUPLICATE_EVENT') {
|
|
|
|
|
return NextResponse.json({
|
|
|
|
|
success: true,
|
|
|
|
|
eventId,
|
|
|
|
|
note: 'DUPLICATE_DROPPED',
|
|
|
|
|
});
|
|
|
|
|
}
|
2026-11-17 20:05:00 +06:00
|
|
|
const status = val.code === 'PAYLOAD_TOO_LARGE' ? 413 : 429;
|
|
|
|
|
return NextResponse.json({ error: val.reason, code: val.code }, { status });
|
2026-10-23 11:43:49 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const event = {
|
|
|
|
|
clientId,
|
|
|
|
|
eventId,
|
|
|
|
|
timestamp,
|
|
|
|
|
payload: typeof data === 'string' ? data : JSON.stringify(data),
|
|
|
|
|
};
|
|
|
|
|
|
2026-11-18 05:18:55 +06:00
|
|
|
await DataBufferEngine.pushAsync(event);
|
2026-11-19 20:22:01 +06:00
|
|
|
await SDKManager.incrementEventCountAsync(clientId, 1);
|
2026-10-23 11:43:49 -05:00
|
|
|
|
2026-11-18 02:23:01 +06:00
|
|
|
return NextResponse.json({ success: true, eventId, event });
|
2026-10-23 11:43:49 -05:00
|
|
|
} catch (err: unknown) {
|
2026-11-17 20:05:00 +06:00
|
|
|
const message = err instanceof Error ? err.message : 'Event ingestion failed';
|
2026-10-23 11:43:49 -05:00
|
|
|
return NextResponse.json({ error: message }, { status: 500 });
|
|
|
|
|
}
|
|
|
|
|
}
|