feat(auth): harden session management and security
- Implement server-side session revocation on logout - Add IP extraction utility and admin setup error handling - Improve error codes and response consistency for auth endpoints - Refine landing page redirect logic to handle transient DB outages - Update deployment documentation and configuration defaults
This commit is contained in:
1 parent
935f12a1e9
commit
89e4d68690
81 files changed
+11530
-6916
No files matched your search
+296
-41
@@ -1,33 +1,240 @@
|
||||
import { SettingsManager } from '@/core/adminSettings/settingsManager';
|
||||
import { verifyRequestAdminSession } from '@/core/auth/adminUsers';
|
||||
import { AuthService } from '@/core/auth/jwt';
|
||||
import { DataBufferEngine } from '@/core/data/buffer';
|
||||
import { DataBufferEngine, REDIS_BUFFER_KEY } from '@/core/data/buffer';
|
||||
import { SDKManager } from '@/core/sdk/sdkManager';
|
||||
import {
|
||||
SecurityEngine,
|
||||
getInternalIngestionHeaders,
|
||||
} from '@/core/Security/security';
|
||||
import { CSVStorageManager, type DataEvent } from '@/core/Storage/csvStorage';
|
||||
import {
|
||||
CSVStorageManager,
|
||||
type DataEvent,
|
||||
type StorageFileMeta,
|
||||
} from '@/core/Storage/csvStorage';
|
||||
import { parse } from 'csv-parse';
|
||||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import crypto from 'node:crypto';
|
||||
import { createReadStream } from 'node:fs';
|
||||
import { requireAdminApiSession } from '@/lib/adminAuth';
|
||||
import type { BufferedBucket, BufferedSummary } from '@/lib/vconUiHelpers';
|
||||
|
||||
const SESSION_COOKIE = 'datastack_admin_session';
|
||||
const SINGLE_EVENT_ENVELOPE_KEYS = new Set(['eventId', 'timestamp', 'data', 'clientId']);
|
||||
const MAX_SINGLE_CLIENT_SCAN_FILES = 10;
|
||||
const MAX_ALL_CLIENTS_PER_CLIENT_FILES = 2;
|
||||
const MAX_ALL_CLIENTS_TOTAL_SCAN_FILES = 50;
|
||||
|
||||
function generateFallbackEventId(): string {
|
||||
return `evt_${Date.now()}_${crypto.randomBytes(6).toString('hex')}`;
|
||||
}
|
||||
|
||||
function extractSingleHttpEventData(rawBody: Record<string, unknown>): unknown {
|
||||
if (rawBody.data !== undefined) {
|
||||
const keys = Object.keys(rawBody);
|
||||
if (keys.every((k) => SINGLE_EVENT_ENVELOPE_KEYS.has(k))) {
|
||||
return rawBody.data;
|
||||
}
|
||||
return rawBody;
|
||||
}
|
||||
|
||||
const stripped: Record<string, unknown> = {};
|
||||
let hasNonEnvelopeKey = false;
|
||||
for (const [k, v] of Object.entries(rawBody)) {
|
||||
if (k === 'eventId' || k === 'timestamp' || k === 'clientId') {
|
||||
continue;
|
||||
}
|
||||
hasNonEnvelopeKey = true;
|
||||
stripped[k] = v;
|
||||
}
|
||||
return hasNonEnvelopeKey ? stripped : {};
|
||||
}
|
||||
|
||||
function parseTimestampMs(raw: string): number {
|
||||
const trimmed = String(raw || '').trim();
|
||||
if (/^\d+(\.\d+)?$/.test(trimmed)) {
|
||||
const num = Number(trimmed);
|
||||
return num < 1e11 ? num * 1000 : num;
|
||||
}
|
||||
const parsed = Date.parse(trimmed);
|
||||
return Number.isNaN(parsed) ? Number.NEGATIVE_INFINITY : parsed;
|
||||
}
|
||||
|
||||
function resolveEventUtcDate(rawTimestamp: string): string {
|
||||
const ms = parseTimestampMs(rawTimestamp);
|
||||
if (Number.isFinite(ms)) {
|
||||
const parsed = new Date(ms);
|
||||
if (!Number.isNaN(parsed.getTime())) {
|
||||
return parsed.toISOString().slice(0, 10);
|
||||
}
|
||||
}
|
||||
return new Date().toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function selectRecentPartitionFiles(
|
||||
allFiles: StorageFileMeta[],
|
||||
normClientId?: string,
|
||||
sanitizedDir?: string,
|
||||
): StorageFileMeta[] {
|
||||
const matching = allFiles
|
||||
.filter(
|
||||
(f) =>
|
||||
!normClientId ||
|
||||
f.clientId === normClientId ||
|
||||
f.clientId === sanitizedDir,
|
||||
)
|
||||
.sort((a, b) => {
|
||||
if (a.date !== b.date) return b.date.localeCompare(a.date);
|
||||
return new Date(b.modifiedAt).getTime() - new Date(a.modifiedAt).getTime();
|
||||
});
|
||||
|
||||
if (normClientId) {
|
||||
return matching.slice(0, MAX_SINGLE_CLIENT_SCAN_FILES);
|
||||
}
|
||||
|
||||
// Multi-client fair scan: take up to 2 newest partitions per active client (capped at 50 files total)
|
||||
// so clients beyond the 5th most-recently-modified file are never silently hidden.
|
||||
const perClientCounts = new Map<string, number>();
|
||||
const selected: StorageFileMeta[] = [];
|
||||
|
||||
for (const file of matching) {
|
||||
const currentCount = perClientCounts.get(file.clientId) ?? 0;
|
||||
if (currentCount >= MAX_ALL_CLIENTS_PER_CLIENT_FILES) continue;
|
||||
perClientCounts.set(file.clientId, currentCount + 1);
|
||||
selected.push(file);
|
||||
if (selected.length >= MAX_ALL_CLIENTS_TOTAL_SCAN_FILES) break;
|
||||
}
|
||||
|
||||
return selected;
|
||||
}
|
||||
|
||||
async function getPendingBufferedSummary(
|
||||
ingestionBaseUrl: string,
|
||||
filterClientId?: string,
|
||||
): Promise<{ events: DataEvent[]; summary: BufferedSummary }> {
|
||||
const bufferStats = await DataBufferEngine.getBufferStatsAsync();
|
||||
const recentLimit = DataBufferEngine.getRecentLimit();
|
||||
const pending: DataEvent[] = [];
|
||||
let totalQueued = bufferStats.localBufferSize + bufferStats.redisBufferSize;
|
||||
|
||||
// 1. Read exact unflushed Redis queue (when Redis is enabled & connected)
|
||||
const redis = DataBufferEngine.getRedisClient();
|
||||
if (redis && bufferStats.redisBufferSize > 0) {
|
||||
try {
|
||||
const cap = Math.min(recentLimit, bufferStats.redisBufferSize);
|
||||
const rawItems = await redis.lrange(REDIS_BUFFER_KEY, 0, cap - 1);
|
||||
for (const raw of rawItems) {
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as DataEvent;
|
||||
if (!parsed || !parsed.eventId) continue;
|
||||
pending.push(parsed);
|
||||
} catch {
|
||||
// Skip malformed queued item
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Fall back to local buffer telemetry
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Read Port 3000 exact unflushed RAM queue snapshot
|
||||
if (bufferStats.localBufferSize > 0) {
|
||||
const localPending = DataBufferEngine.getLocalBufferSnapshot(recentLimit);
|
||||
pending.push(...localPending);
|
||||
}
|
||||
|
||||
// 3. When Redis is not enabled or temporarily disconnected, Port 4000 `/internal/recent` serves its unflushed RAM `bufferQueue`
|
||||
if (!bufferStats.redisEnabled || !bufferStats.redisConnected) {
|
||||
try {
|
||||
const qs = new URLSearchParams({ limit: String(recentLimit) });
|
||||
const res = await fetch(`${ingestionBaseUrl}/internal/recent?${qs.toString()}`, {
|
||||
headers: getInternalIngestionHeaders(),
|
||||
cache: 'no-store',
|
||||
signal: AbortSignal.timeout(1000),
|
||||
});
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
if (Array.isArray(data.events) && data.events.length > 0) {
|
||||
pending.push(...(data.events as DataEvent[]));
|
||||
totalQueued += data.events.length;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
}
|
||||
}
|
||||
|
||||
const seen = new Set<string>();
|
||||
const dedupedAll: DataEvent[] = [];
|
||||
const filteredEvents: DataEvent[] = [];
|
||||
const bucketMap = new Map<string, BufferedBucket>();
|
||||
const sanitizedFilter = filterClientId
|
||||
? CSVStorageManager.sanitizeClientId(filterClientId)
|
||||
: undefined;
|
||||
|
||||
for (const evt of pending) {
|
||||
if (!evt || !evt.eventId) continue;
|
||||
const normEvtClient = SDKManager.normalizeClientId(evt.clientId) || evt.clientId;
|
||||
const dedupeKey = `${normEvtClient}:${evt.eventId}`;
|
||||
if (seen.has(dedupeKey)) continue;
|
||||
seen.add(dedupeKey);
|
||||
dedupedAll.push(evt);
|
||||
|
||||
const matchesFilter =
|
||||
!filterClientId ||
|
||||
normEvtClient === filterClientId ||
|
||||
evt.clientId === filterClientId ||
|
||||
evt.clientId === sanitizedFilter;
|
||||
|
||||
if (matchesFilter) {
|
||||
filteredEvents.push(evt);
|
||||
const utcDate = resolveEventUtcDate(evt.timestamp);
|
||||
const bucketKey = `${normEvtClient}|${utcDate}`;
|
||||
const existing = bucketMap.get(bucketKey);
|
||||
if (existing) {
|
||||
existing.count += 1;
|
||||
} else {
|
||||
bucketMap.set(bucketKey, {
|
||||
clientId: normEvtClient,
|
||||
date: utcDate,
|
||||
count: 1,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const effectiveTotalQueued = Math.max(totalQueued, dedupedAll.length);
|
||||
const unattributedCount = Math.max(0, effectiveTotalQueued - dedupedAll.length);
|
||||
|
||||
return {
|
||||
events: filteredEvents,
|
||||
summary: {
|
||||
total: filteredEvents.length,
|
||||
totalQueued: filterClientId
|
||||
? filteredEvents.length + unattributedCount
|
||||
: effectiveTotalQueued,
|
||||
unattributedCount,
|
||||
truncated: unattributedCount > 0,
|
||||
buckets: Array.from(bucketMap.values()),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function getRecentStoredEvents(
|
||||
limit: number,
|
||||
clientId?: string,
|
||||
): Promise<DataEvent[]> {
|
||||
const normClientId = clientId && clientId !== 'all' ? SDKManager.normalizeClientId(clientId) : undefined;
|
||||
const sanitizedDir = normClientId ? CSVStorageManager.sanitizeClientId(normClientId) : undefined;
|
||||
const allFiles = await CSVStorageManager.listAllFilesAsync();
|
||||
const files = allFiles
|
||||
.filter((f) => !clientId || clientId === 'all' || f.clientId === clientId)
|
||||
.slice(0, 5);
|
||||
const files = selectRecentPartitionFiles(allFiles, normClientId, sanitizedDir);
|
||||
|
||||
const collected: DataEvent[] = [];
|
||||
|
||||
for (const file of files) {
|
||||
const fileTail: DataEvent[] = [];
|
||||
const parser = createReadStream(file.filePath).pipe(
|
||||
const fileStream = createReadStream(file.filePath);
|
||||
const parser = fileStream.pipe(
|
||||
parse({ columns: true, skip_empty_lines: true, bom: true }),
|
||||
);
|
||||
|
||||
@@ -46,8 +253,11 @@ async function getRecentStoredEvents(
|
||||
payload: record.payload || '',
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
parser.destroy();
|
||||
} catch (err) {
|
||||
console.warn(`[API /events] Failed reading CSV partition ${file.filePath}:`, err);
|
||||
} finally {
|
||||
if (!parser.destroyed) parser.destroy();
|
||||
if (!fileStream.destroyed) fileStream.destroy();
|
||||
}
|
||||
|
||||
if (fileTail.length > 0) {
|
||||
@@ -59,18 +269,47 @@ async function getRecentStoredEvents(
|
||||
}
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
const auth = await requireAdminApiSession(req);
|
||||
if (!auth.valid) {
|
||||
return auth.response;
|
||||
}
|
||||
|
||||
try {
|
||||
const { searchParams } = new URL(req.url);
|
||||
const clientId = searchParams.get('clientId') || undefined;
|
||||
const rawClientId = searchParams.get('clientId');
|
||||
const clientId =
|
||||
rawClientId && rawClientId !== 'all'
|
||||
? SDKManager.normalizeClientId(rawClientId) || undefined
|
||||
: undefined;
|
||||
const limit = Math.min(2000, Math.max(1, Number(searchParams.get('limit')) || 50));
|
||||
const bufferedOnly = searchParams.get('bufferedOnly') === 'true';
|
||||
const includeBufferedSummary =
|
||||
bufferedOnly || searchParams.get('includeBufferedSummary') === 'true';
|
||||
|
||||
await SettingsManager.getSettingsAsync();
|
||||
const bufferedEvents = await DataBufferEngine.getRecentEventsAsync(limit, clientId);
|
||||
let socketBufferedEvents: DataEvent[] = [];
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
|
||||
).replace(/\/$/, '');
|
||||
|
||||
if (bufferedOnly) {
|
||||
const pendingResult = await getPendingBufferedSummary(ingestionBaseUrl, clientId);
|
||||
const sortedPending = [...pendingResult.events].sort((a, b) => {
|
||||
const aMs = parseTimestampMs(a.timestamp);
|
||||
const bMs = parseTimestampMs(b.timestamp);
|
||||
if (aMs !== bMs) return bMs - aMs;
|
||||
return b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0;
|
||||
});
|
||||
const events = sortedPending.slice(0, limit);
|
||||
return NextResponse.json({
|
||||
events,
|
||||
count: events.length,
|
||||
bufferedSummary: pendingResult.summary,
|
||||
});
|
||||
}
|
||||
|
||||
const bufferedEvents = await DataBufferEngine.getRecentEventsAsync(limit, clientId);
|
||||
let socketBufferedEvents: DataEvent[] = [];
|
||||
|
||||
try {
|
||||
const qs = new URLSearchParams({ limit: String(limit) });
|
||||
if (clientId) qs.set('clientId', clientId);
|
||||
@@ -89,23 +328,38 @@ export async function GET(req: NextRequest) {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
}
|
||||
|
||||
const storedEvents = await getRecentStoredEvents(limit, clientId);
|
||||
const [storedEvents, pendingResult] = await Promise.all([
|
||||
getRecentStoredEvents(limit, clientId),
|
||||
includeBufferedSummary
|
||||
? getPendingBufferedSummary(ingestionBaseUrl, clientId)
|
||||
: Promise.resolve(null),
|
||||
]);
|
||||
|
||||
const seenIds = new Set<string>();
|
||||
const merged: DataEvent[] = [];
|
||||
|
||||
for (const evt of [...bufferedEvents, ...socketBufferedEvents, ...storedEvents]) {
|
||||
if (!evt.eventId) continue;
|
||||
const dedupeKey = `${evt.clientId}:${evt.eventId}`;
|
||||
const normEvtClient = SDKManager.normalizeClientId(evt.clientId) || evt.clientId;
|
||||
const dedupeKey = `${normEvtClient}:${evt.eventId}`;
|
||||
if (seenIds.has(dedupeKey)) continue;
|
||||
seenIds.add(dedupeKey);
|
||||
merged.push(evt);
|
||||
}
|
||||
|
||||
merged.sort((a, b) => (b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0));
|
||||
merged.sort((a, b) => {
|
||||
const aMs = parseTimestampMs(a.timestamp);
|
||||
const bMs = parseTimestampMs(b.timestamp);
|
||||
if (aMs !== bMs) return bMs - aMs;
|
||||
return b.timestamp > a.timestamp ? 1 : b.timestamp < a.timestamp ? -1 : 0;
|
||||
});
|
||||
const events = merged.slice(0, limit);
|
||||
|
||||
return NextResponse.json({ events, count: events.length });
|
||||
return NextResponse.json({
|
||||
events,
|
||||
count: events.length,
|
||||
...(pendingResult ? { bufferedSummary: pendingResult.summary } : {}),
|
||||
});
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : 'Events retrieval failed';
|
||||
return NextResponse.json({ error: message }, { status: 500 });
|
||||
@@ -165,12 +419,18 @@ export async function POST(req: NextRequest) {
|
||||
? (body as Record<string, unknown>)
|
||||
: {};
|
||||
|
||||
const clientId = String(
|
||||
tokenClientId || bodyObj.clientId || 'py_telemetry_agent_01',
|
||||
)
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9_-]/g, '_');
|
||||
const clientId = SDKManager.normalizeClientId(
|
||||
String(tokenClientId || bodyObj.clientId || ''),
|
||||
);
|
||||
if (!clientId) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: 'clientId is required and must belong to a registered client.',
|
||||
code: 'VALIDATION_ERROR',
|
||||
},
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
if (await SDKManager.isClientDeletedAsync(clientId)) {
|
||||
return NextResponse.json(
|
||||
@@ -183,7 +443,7 @@ export async function POST(req: NextRequest) {
|
||||
}
|
||||
|
||||
const registeredClient = await SDKManager.getClientAsync(clientId);
|
||||
if (rawBearerToken && !registeredClient) {
|
||||
if (!registeredClient) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: `CLIENT_REVOKED: Client '${clientId}' registration not found or has been deleted.`,
|
||||
@@ -231,10 +491,11 @@ export async function POST(req: NextRequest) {
|
||||
? (bodyObj.batch as unknown[])
|
||||
: Array.isArray(bodyObj.records)
|
||||
? (bodyObj.records as unknown[])
|
||||
: null;
|
||||
: bodyObj.data === undefined && Array.isArray(bodyObj.events)
|
||||
? (bodyObj.events as unknown[])
|
||||
: null;
|
||||
|
||||
if (batchItems !== null) {
|
||||
const envelopeKeys = new Set(['eventId', 'timestamp', 'data', 'clientId']);
|
||||
let processed = 0;
|
||||
let rejected = 0;
|
||||
let duplicates = 0;
|
||||
@@ -255,19 +516,16 @@ export async function POST(req: NextRequest) {
|
||||
const itemEventId =
|
||||
typeof safeItem.eventId === 'string' && safeItem.eventId.trim()
|
||||
? safeItem.eventId.trim()
|
||||
: `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||
: generateFallbackEventId();
|
||||
|
||||
const itemTimestamp =
|
||||
typeof safeItem.timestamp === 'string' && safeItem.timestamp.trim()
|
||||
? safeItem.timestamp.trim()
|
||||
: new Date().toISOString();
|
||||
|
||||
const isEnvelope =
|
||||
safeItem.data !== undefined &&
|
||||
Object.keys(safeItem).every((k) => envelopeKeys.has(k));
|
||||
const itemData = isEnvelope ? safeItem.data : safeItem;
|
||||
const itemData = extractSingleHttpEventData(safeItem);
|
||||
|
||||
const val = SecurityEngine.validateEvent(
|
||||
const val = await SecurityEngine.validateEventAsync(
|
||||
clientId,
|
||||
itemEventId,
|
||||
itemData,
|
||||
@@ -309,7 +567,7 @@ export async function POST(req: NextRequest) {
|
||||
}
|
||||
|
||||
if (processed > 0) {
|
||||
SDKManager.incrementEventCount(clientId, processed);
|
||||
await SDKManager.incrementEventCountAsync(clientId, processed);
|
||||
}
|
||||
|
||||
// If zero items were processed or deduplicated and all failed due to payload or rate limit, return corresponding status
|
||||
@@ -323,6 +581,7 @@ export async function POST(req: NextRequest) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
accepted: processed,
|
||||
processed,
|
||||
rejected,
|
||||
duplicates,
|
||||
@@ -338,6 +597,7 @@ export async function POST(req: NextRequest) {
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
accepted: processed,
|
||||
processed,
|
||||
rejected,
|
||||
duplicates,
|
||||
@@ -350,24 +610,19 @@ export async function POST(req: NextRequest) {
|
||||
});
|
||||
}
|
||||
|
||||
// Single Event Ingestion
|
||||
// Single Event Ingestion (exact envelope parity with Port 4000 WS sync_data)
|
||||
const eventId =
|
||||
typeof bodyObj.eventId === 'string' && bodyObj.eventId.trim()
|
||||
? bodyObj.eventId.trim()
|
||||
: `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||
: generateFallbackEventId();
|
||||
const timestamp =
|
||||
typeof bodyObj.timestamp === 'string' && bodyObj.timestamp.trim()
|
||||
? bodyObj.timestamp.trim()
|
||||
: new Date().toISOString();
|
||||
const data =
|
||||
bodyObj.data !== undefined
|
||||
? bodyObj.data
|
||||
: Object.keys(bodyObj).length > 0
|
||||
? bodyObj
|
||||
: { sample: 'Realtime event via DataStack API', rand: Math.random() };
|
||||
const data = extractSingleHttpEventData(bodyObj);
|
||||
|
||||
// Validate security (idempotency, rate limit, payload size)
|
||||
const val = SecurityEngine.validateEvent(
|
||||
const val = await SecurityEngine.validateEventAsync(
|
||||
clientId,
|
||||
eventId,
|
||||
data,
|
||||
@@ -393,7 +648,7 @@ export async function POST(req: NextRequest) {
|
||||
};
|
||||
|
||||
await DataBufferEngine.pushAsync(event);
|
||||
SDKManager.incrementEventCount(clientId, 1);
|
||||
await SDKManager.incrementEventCountAsync(clientId, 1);
|
||||
|
||||
return NextResponse.json({ success: true, eventId, event });
|
||||
} catch (err: unknown) {
|
||||
|
||||
Reference in new issue
Block a user