feat: refactor Python SDK and enhance API error reporting

- Relocated Python SDK to `sdk/python/datastack` with new auto-configuration support via `config.datastack.py`
- Updated SDK installation and usage documentation
- Enhanced `/api/events` response format to include specific error codes for revoked and rotated credentials
- Updated SDK dashboard snippet generator to support new configuration pattern
This commit is contained in:
Team Dark Devil committed 2026-11-18 02:23:01 +06:00
1 parent 367dd819f8
commit 0c2e49cd47
18 files changed
+2288 -249

No files matched your search

+145 -13
View File
@@ -120,8 +120,13 @@ export async function POST(req: NextRequest) {
}
const body = await req.json();
const bodyObj =
body && typeof body === 'object' && !Array.isArray(body)
? (body as Record<string, unknown>)
: {};
const clientId = String(
tokenClientId || body.clientId || 'py_telemetry_agent_01',
tokenClientId || bodyObj.clientId || 'py_telemetry_agent_01',
)
.trim()
.toLowerCase()
@@ -131,7 +136,10 @@ export async function POST(req: NextRequest) {
if (registeredClient) {
if (registeredClient.status === 'revoked') {
return NextResponse.json(
{ error: `Client '${clientId}' access has been revoked by administrator.` },
{
error: `CLIENT_REVOKED: Client '${clientId}' access has been revoked by administrator.`,
code: 'CLIENT_REVOKED',
},
{ status: 403 },
);
}
@@ -141,27 +149,144 @@ export async function POST(req: NextRequest) {
registeredClient.token !== rawBearerToken
) {
return NextResponse.json(
{ error: 'Client credentials have been rotated. Use the latest token.' },
{
error: 'TOKEN_ROTATED: Client credentials have been rotated. Use the latest token.',
code: 'TOKEN_ROTATED',
},
{ status: 401 },
);
}
}
const eventId =
body.eventId ||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
const timestamp = body.timestamp || new Date().toISOString();
const data =
body.data !== undefined
? body.data
: { sample: 'Realtime event via DataStack API', rand: Math.random() };
const effectiveClientRate =
registeredClient?.rateLimit ||
tokenRateLimit ||
settings.maxEventsPerSec ||
100;
// Detect if this is a batch array or direct file-records ingestion request
const batchItems: unknown[] | null = Array.isArray(body)
? body
: Array.isArray(bodyObj.batch)
? (bodyObj.batch as unknown[])
: Array.isArray(bodyObj.records)
? (bodyObj.records as unknown[])
: null;
if (batchItems !== null) {
let processed = 0;
let rejected = 0;
let duplicates = 0;
let hitRateLimit = false;
let hitPayloadLimit = false;
const errors: string[] = [];
for (const item of batchItems) {
const safeItem =
item && typeof item === 'object' && !Array.isArray(item)
? (item as Record<string, unknown>)
: { value: item };
const itemEventId =
typeof safeItem.eventId === 'string' && safeItem.eventId.trim()
? safeItem.eventId.trim()
: `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
const itemTimestamp =
typeof safeItem.timestamp === 'string' && safeItem.timestamp.trim()
? safeItem.timestamp.trim()
: new Date().toISOString();
const itemData =
safeItem.data !== undefined ? safeItem.data : safeItem;
const val = SecurityEngine.validateEvent(
clientId,
itemEventId,
itemData,
effectiveClientRate,
);
if (!val.allowed) {
if (val.code === 'DUPLICATE_EVENT') {
duplicates += 1;
continue;
}
rejected += 1;
if (val.code === 'RATE_LIMIT_EXCEEDED') hitRateLimit = true;
if (val.code === 'PAYLOAD_TOO_LARGE') hitPayloadLimit = true;
if (errors.length < 5) {
errors.push(`${itemEventId}: ${val.code || 'REJECTED'} - ${val.reason}`);
}
continue;
}
const event: DataEvent = {
clientId,
eventId: itemEventId,
timestamp: itemTimestamp,
payload:
typeof itemData === 'string' ? itemData : JSON.stringify(itemData),
};
DataBufferEngine.push(event);
processed += 1;
}
if (processed > 0) {
SDKManager.incrementEventCount(clientId, processed);
}
// If zero items were processed or deduplicated and all failed due to payload or rate limit, return corresponding status
if (processed === 0 && duplicates === 0 && rejected > 0) {
const status = hitPayloadLimit ? 413 : hitRateLimit ? 429 : 400;
const code = hitPayloadLimit
? 'PAYLOAD_TOO_LARGE'
: hitRateLimit
? 'RATE_LIMIT_EXCEEDED'
: 'BATCH_REJECTED';
return NextResponse.json(
{
success: false,
processed,
rejected,
duplicates,
code,
error: errors[0] || 'Batch validation failed',
errors,
},
{ status },
);
}
return NextResponse.json({
success: true,
processed,
rejected,
duplicates,
...(typeof bodyObj.fileName === 'string'
? { fileName: bodyObj.fileName }
: {}),
...(errors.length > 0 ? { errors } : {}),
});
}
// Single Event Ingestion
const eventId =
typeof bodyObj.eventId === 'string' && bodyObj.eventId.trim()
? bodyObj.eventId.trim()
: `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
const timestamp =
typeof bodyObj.timestamp === 'string' && bodyObj.timestamp.trim()
? bodyObj.timestamp.trim()
: new Date().toISOString();
const data =
bodyObj.data !== undefined
? bodyObj.data
: Object.keys(bodyObj).length > 0
? bodyObj
: { sample: 'Realtime event via DataStack API', rand: Math.random() };
// Validate security (idempotency, rate limit, payload size)
const val = SecurityEngine.validateEvent(
clientId,
@@ -170,6 +295,13 @@ export async function POST(req: NextRequest) {
effectiveClientRate,
);
if (!val.allowed) {
if (val.code === 'DUPLICATE_EVENT') {
return NextResponse.json({
success: true,
eventId,
note: 'DUPLICATE_DROPPED',
});
}
const status = val.code === 'PAYLOAD_TOO_LARGE' ? 413 : 429;
return NextResponse.json({ error: val.reason, code: val.code }, { status });
}
@@ -184,7 +316,7 @@ export async function POST(req: NextRequest) {
DataBufferEngine.push(event);
SDKManager.incrementEventCount(clientId, 1);
return NextResponse.json({ success: true, event });
return NextResponse.json({ success: true, eventId, event });
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Event ingestion failed';
return NextResponse.json({ error: message }, { status: 500 });