feat: aggregate external events and improve security
- Fetch events from external ingestion service in API routes - Decouple idempotency check from registration in Security utility - Refine CSV storage logic to verify file content for headers - Replace Python SDK with comprehensive, feature-rich implementation
This commit is contained in:
1 parent
043123dc21
commit
34519c07df
13 files changed
+2099
-451
No files matched your search
@@ -113,6 +113,37 @@ export async function GET(request: NextRequest) {
|
||||
addIfMatching({ ...event, status: "buffered" });
|
||||
}
|
||||
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||
).replace(/\/$/, "");
|
||||
try {
|
||||
const res = await fetch(`${ingestionBaseUrl}/internal/recent?limit=200`, {
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(1000),
|
||||
});
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
if (Array.isArray(data.events)) {
|
||||
for (const evt of data.events) {
|
||||
if (evt && evt.eventId) {
|
||||
addIfMatching({
|
||||
clientId: String(evt.clientId || ""),
|
||||
eventId: String(evt.eventId),
|
||||
timestamp: String(evt.timestamp || new Date().toISOString()),
|
||||
payload:
|
||||
typeof evt.payload === "string"
|
||||
? evt.payload
|
||||
: JSON.stringify(evt.payload ?? {}),
|
||||
status: "buffered",
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
events: selectedRows.slice(offset),
|
||||
total,
|
||||
|
||||
+23
-1
@@ -58,12 +58,34 @@ export async function GET(req: NextRequest) {
|
||||
const limit = Math.min(200, Math.max(1, Number(searchParams.get('limit')) || 50));
|
||||
|
||||
const bufferedEvents = DataBufferEngine.getRecentEvents(limit, clientId);
|
||||
let socketBufferedEvents: DataEvent[] = [];
|
||||
const ingestionBaseUrl = (
|
||||
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
|
||||
).replace(/\/$/, '');
|
||||
|
||||
try {
|
||||
const qs = new URLSearchParams({ limit: String(limit) });
|
||||
if (clientId) qs.set('clientId', clientId);
|
||||
const res = await fetch(`${ingestionBaseUrl}/internal/recent?${qs.toString()}`, {
|
||||
cache: 'no-store',
|
||||
signal: AbortSignal.timeout(1000),
|
||||
});
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
if (Array.isArray(data.events)) {
|
||||
socketBufferedEvents = data.events;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Port 4000 may not be running in isolated single-process dev mode
|
||||
}
|
||||
|
||||
const storedEvents = await getRecentStoredEvents(limit, clientId);
|
||||
|
||||
const seenIds = new Set<string>();
|
||||
const merged: DataEvent[] = [];
|
||||
|
||||
for (const evt of [...bufferedEvents, ...storedEvents]) {
|
||||
for (const evt of [...bufferedEvents, ...socketBufferedEvents, ...storedEvents]) {
|
||||
if (!evt.eventId || seenIds.has(evt.eventId)) continue;
|
||||
seenIds.add(evt.eventId);
|
||||
merged.push(evt);
|
||||
|
||||
Reference in new issue
Block a user