feat: refactor Python SDK and enhance API error reporting
- Relocated Python SDK to `sdk/python/datastack` with new auto-configuration support via `config.datastack.py` - Updated SDK installation and usage documentation - Enhanced `/api/events` response format to include specific error codes for revoked and rotated credentials - Updated SDK dashboard snippet generator to support new configuration pattern
This commit is contained in:
1 parent
367dd819f8
commit
0c2e49cd47
18 files changed
+2288
-249
No files matched your search
+145
-13
@@ -120,8 +120,13 @@ export async function POST(req: NextRequest) {
|
||||
}
|
||||
|
||||
const body = await req.json();
|
||||
const bodyObj =
|
||||
body && typeof body === 'object' && !Array.isArray(body)
|
||||
? (body as Record<string, unknown>)
|
||||
: {};
|
||||
|
||||
const clientId = String(
|
||||
tokenClientId || body.clientId || 'py_telemetry_agent_01',
|
||||
tokenClientId || bodyObj.clientId || 'py_telemetry_agent_01',
|
||||
)
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
@@ -131,7 +136,10 @@ export async function POST(req: NextRequest) {
|
||||
if (registeredClient) {
|
||||
if (registeredClient.status === 'revoked') {
|
||||
return NextResponse.json(
|
||||
{ error: `Client '${clientId}' access has been revoked by administrator.` },
|
||||
{
|
||||
error: `CLIENT_REVOKED: Client '${clientId}' access has been revoked by administrator.`,
|
||||
code: 'CLIENT_REVOKED',
|
||||
},
|
||||
{ status: 403 },
|
||||
);
|
||||
}
|
||||
@@ -141,27 +149,144 @@ export async function POST(req: NextRequest) {
|
||||
registeredClient.token !== rawBearerToken
|
||||
) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Client credentials have been rotated. Use the latest token.' },
|
||||
{
|
||||
error: 'TOKEN_ROTATED: Client credentials have been rotated. Use the latest token.',
|
||||
code: 'TOKEN_ROTATED',
|
||||
},
|
||||
{ status: 401 },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const eventId =
|
||||
body.eventId ||
|
||||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||
const timestamp = body.timestamp || new Date().toISOString();
|
||||
const data =
|
||||
body.data !== undefined
|
||||
? body.data
|
||||
: { sample: 'Realtime event via DataStack API', rand: Math.random() };
|
||||
|
||||
const effectiveClientRate =
|
||||
registeredClient?.rateLimit ||
|
||||
tokenRateLimit ||
|
||||
settings.maxEventsPerSec ||
|
||||
100;
|
||||
|
||||
// Detect if this is a batch array or direct file-records ingestion request
|
||||
const batchItems: unknown[] | null = Array.isArray(body)
|
||||
? body
|
||||
: Array.isArray(bodyObj.batch)
|
||||
? (bodyObj.batch as unknown[])
|
||||
: Array.isArray(bodyObj.records)
|
||||
? (bodyObj.records as unknown[])
|
||||
: null;
|
||||
|
||||
if (batchItems !== null) {
|
||||
let processed = 0;
|
||||
let rejected = 0;
|
||||
let duplicates = 0;
|
||||
let hitRateLimit = false;
|
||||
let hitPayloadLimit = false;
|
||||
const errors: string[] = [];
|
||||
|
||||
for (const item of batchItems) {
|
||||
const safeItem =
|
||||
item && typeof item === 'object' && !Array.isArray(item)
|
||||
? (item as Record<string, unknown>)
|
||||
: { value: item };
|
||||
|
||||
const itemEventId =
|
||||
typeof safeItem.eventId === 'string' && safeItem.eventId.trim()
|
||||
? safeItem.eventId.trim()
|
||||
: `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||
|
||||
const itemTimestamp =
|
||||
typeof safeItem.timestamp === 'string' && safeItem.timestamp.trim()
|
||||
? safeItem.timestamp.trim()
|
||||
: new Date().toISOString();
|
||||
|
||||
const itemData =
|
||||
safeItem.data !== undefined ? safeItem.data : safeItem;
|
||||
|
||||
const val = SecurityEngine.validateEvent(
|
||||
clientId,
|
||||
itemEventId,
|
||||
itemData,
|
||||
effectiveClientRate,
|
||||
);
|
||||
|
||||
if (!val.allowed) {
|
||||
if (val.code === 'DUPLICATE_EVENT') {
|
||||
duplicates += 1;
|
||||
continue;
|
||||
}
|
||||
rejected += 1;
|
||||
if (val.code === 'RATE_LIMIT_EXCEEDED') hitRateLimit = true;
|
||||
if (val.code === 'PAYLOAD_TOO_LARGE') hitPayloadLimit = true;
|
||||
if (errors.length < 5) {
|
||||
errors.push(`${itemEventId}: ${val.code || 'REJECTED'} - ${val.reason}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const event: DataEvent = {
|
||||
clientId,
|
||||
eventId: itemEventId,
|
||||
timestamp: itemTimestamp,
|
||||
payload:
|
||||
typeof itemData === 'string' ? itemData : JSON.stringify(itemData),
|
||||
};
|
||||
|
||||
DataBufferEngine.push(event);
|
||||
processed += 1;
|
||||
}
|
||||
|
||||
if (processed > 0) {
|
||||
SDKManager.incrementEventCount(clientId, processed);
|
||||
}
|
||||
|
||||
// If zero items were processed or deduplicated and all failed due to payload or rate limit, return corresponding status
|
||||
if (processed === 0 && duplicates === 0 && rejected > 0) {
|
||||
const status = hitPayloadLimit ? 413 : hitRateLimit ? 429 : 400;
|
||||
const code = hitPayloadLimit
|
||||
? 'PAYLOAD_TOO_LARGE'
|
||||
: hitRateLimit
|
||||
? 'RATE_LIMIT_EXCEEDED'
|
||||
: 'BATCH_REJECTED';
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
processed,
|
||||
rejected,
|
||||
duplicates,
|
||||
code,
|
||||
error: errors[0] || 'Batch validation failed',
|
||||
errors,
|
||||
},
|
||||
{ status },
|
||||
);
|
||||
}
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
processed,
|
||||
rejected,
|
||||
duplicates,
|
||||
...(typeof bodyObj.fileName === 'string'
|
||||
? { fileName: bodyObj.fileName }
|
||||
: {}),
|
||||
...(errors.length > 0 ? { errors } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
// Single Event Ingestion
|
||||
const eventId =
|
||||
typeof bodyObj.eventId === 'string' && bodyObj.eventId.trim()
|
||||
? bodyObj.eventId.trim()
|
||||
: `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||
const timestamp =
|
||||
typeof bodyObj.timestamp === 'string' && bodyObj.timestamp.trim()
|
||||
? bodyObj.timestamp.trim()
|
||||
: new Date().toISOString();
|
||||
const data =
|
||||
bodyObj.data !== undefined
|
||||
? bodyObj.data
|
||||
: Object.keys(bodyObj).length > 0
|
||||
? bodyObj
|
||||
: { sample: 'Realtime event via DataStack API', rand: Math.random() };
|
||||
|
||||
// Validate security (idempotency, rate limit, payload size)
|
||||
const val = SecurityEngine.validateEvent(
|
||||
clientId,
|
||||
@@ -170,6 +295,13 @@ export async function POST(req: NextRequest) {
|
||||
effectiveClientRate,
|
||||
);
|
||||
if (!val.allowed) {
|
||||
if (val.code === 'DUPLICATE_EVENT') {
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
eventId,
|
||||
note: 'DUPLICATE_DROPPED',
|
||||
});
|
||||
}
|
||||
const status = val.code === 'PAYLOAD_TOO_LARGE' ? 413 : 429;
|
||||
return NextResponse.json({ error: val.reason, code: val.code }, { status });
|
||||
}
|
||||
@@ -184,7 +316,7 @@ export async function POST(req: NextRequest) {
|
||||
DataBufferEngine.push(event);
|
||||
SDKManager.incrementEventCount(clientId, 1);
|
||||
|
||||
return NextResponse.json({ success: true, event });
|
||||
return NextResponse.json({ success: true, eventId, event });
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : 'Event ingestion failed';
|
||||
return NextResponse.json({ error: message }, { status: 500 });
|
||||
|
||||
+96
-39
@@ -21,6 +21,7 @@ export default function SdkGuidePage() {
|
||||
return cached[0]?.clientId ?? '';
|
||||
});
|
||||
const [serverUrl, setServerUrl] = useState<string>('http://localhost:4000');
|
||||
const [apiUrl, setApiUrl] = useState<string>('http://localhost:3000');
|
||||
|
||||
useEffect(() => {
|
||||
const isLocalDev = window.location.port === '3000';
|
||||
@@ -29,6 +30,7 @@ export default function SdkGuidePage() {
|
||||
? `${window.location.protocol}//${window.location.hostname}:4000`
|
||||
: window.location.origin
|
||||
);
|
||||
setApiUrl(window.location.origin);
|
||||
|
||||
async function loadClients() {
|
||||
try {
|
||||
@@ -54,28 +56,48 @@ export default function SdkGuidePage() {
|
||||
clients.find((c) => c.clientId === selectedClientId) || clients[0];
|
||||
const sampleToken = activeClient?.token || 'YOUR_SIGNED_CLIENT_JWT_TOKEN';
|
||||
|
||||
const pythonSnippet = `from datastack_sdk import DataStackClient
|
||||
const configSnippet = `# config.datastack.py — Place in your Python project root
|
||||
DATASTACK_CONFIG = {
|
||||
"SERVER_URL": "${serverUrl}",
|
||||
"API_URL": "${apiUrl}",
|
||||
"CLIENT_TOKEN": "${sampleToken}",
|
||||
"AUTO_CONNECT": True,
|
||||
"ENABLE_HTTP_FALLBACK": True,
|
||||
"HEARTBEAT_INTERVAL_SEC": 5.0,
|
||||
"MAX_RETRIES": 3,
|
||||
"BATCH_CHUNK_SIZE": 250,
|
||||
"OFFLINE_SPOOL_ENABLED": True,
|
||||
"OFFLINE_SPOOL_DIR": ".datastack_spool",
|
||||
}`;
|
||||
|
||||
client = DataStackClient(
|
||||
server_url="${serverUrl}",
|
||||
token="${sampleToken}"
|
||||
)
|
||||
const pythonSnippet = `from datastack import DataStack
|
||||
|
||||
if client.connect():
|
||||
res = client.sync({
|
||||
"event_type": "page_view",
|
||||
"url": "/products/wireless-headphones",
|
||||
"user_id": "usr_9942"
|
||||
})
|
||||
print("Sync response:", res)
|
||||
# Automatically discovers config.datastack.py and connects
|
||||
client = DataStack()
|
||||
|
||||
batch_res = client.batch_sync([
|
||||
{"sensor": "temp", "value": 24.5, "unit": "C"},
|
||||
{"sensor": "humidity", "value": 61.2, "unit": "%"}
|
||||
])
|
||||
print("Batch response:", batch_res)
|
||||
# 1. Single Event (WebSocket + Auto HTTP Fallback + Offline Spool)
|
||||
res = client.send({
|
||||
"event_type": "page_view",
|
||||
"url": "/products/wireless-headphones",
|
||||
"user_id": "usr_9942"
|
||||
})
|
||||
print("Event response:", res)
|
||||
|
||||
client.disconnect()`;
|
||||
# 2. Batch Events
|
||||
batch_res = client.send_batch([
|
||||
{"sensor": "temp", "value": 24.5, "unit": "C"},
|
||||
{"sensor": "humidity", "value": 61.2, "unit": "%"}
|
||||
])
|
||||
print("Batch response:", batch_res)
|
||||
|
||||
# 3. Direct File Ingestion (.csv, .json, .jsonl)
|
||||
file_res = client.send_file("events_dataset.csv", chunk_size=250)
|
||||
print("File ingestion:", file_res)
|
||||
|
||||
# 4. Inspect Connection & Heartbeat IDLE Status
|
||||
print("SDK Status:", client.status())
|
||||
|
||||
client.disconnect()`;
|
||||
|
||||
const copySnippet = (text: string, label: string) => {
|
||||
navigator.clipboard.writeText(text);
|
||||
@@ -97,11 +119,19 @@ if client.connect():
|
||||
Copy Token
|
||||
</Button>
|
||||
)}
|
||||
<Button
|
||||
size="xs"
|
||||
variant="secondary"
|
||||
icon="FileCode"
|
||||
onClick={() => copySnippet(configSnippet, 'config.datastack.py')}
|
||||
>
|
||||
Copy Config
|
||||
</Button>
|
||||
<Button
|
||||
size="xs"
|
||||
variant="secondary"
|
||||
icon="Copy"
|
||||
onClick={() => copySnippet(pythonSnippet, 'Snippet')}
|
||||
onClick={() => copySnippet(pythonSnippet, 'Python SDK snippet')}
|
||||
>
|
||||
Copy Code
|
||||
</Button>
|
||||
@@ -110,8 +140,8 @@ if client.connect():
|
||||
>
|
||||
<StatStrip
|
||||
items={[
|
||||
{ label: 'SDK package', value: 'datastack_sdk', dot: 'ok' },
|
||||
{ label: 'Transport', value: 'Socket.IO / JWT' },
|
||||
{ label: 'SDK package', value: 'datastack (v2.2.0)', dot: 'ok' },
|
||||
{ label: 'Transport', value: 'WS + HTTP Fallback + Offline Spool' },
|
||||
{ label: 'Server endpoint', value: serverUrl },
|
||||
{
|
||||
label: 'Active client',
|
||||
@@ -124,34 +154,61 @@ if client.connect():
|
||||
compact
|
||||
className="flex w-full items-center justify-between bg-[var(--surface)] px-3 py-2 font-mono text-[11.5px]"
|
||||
>
|
||||
<span className="text-[var(--accent)]">pip install ./sdk-python</span>
|
||||
<span className="text-[var(--accent)]">
|
||||
pip install ./sdk/python/datastack
|
||||
</span>
|
||||
<Button
|
||||
size="xs"
|
||||
variant="ghost"
|
||||
icon="Copy"
|
||||
onClick={() => copySnippet('pip install ./sdk-python', 'Pip command')}
|
||||
onClick={() =>
|
||||
copySnippet('pip install ./sdk/python/datastack', 'Pip command')
|
||||
}
|
||||
/>
|
||||
</Card>
|
||||
|
||||
<Panel
|
||||
title="Client Snippet"
|
||||
title="Step 1 — config.datastack.py"
|
||||
className="w-full"
|
||||
action={
|
||||
clients.length > 0 ? (
|
||||
<div className="w-[190px]">
|
||||
<Select
|
||||
compact
|
||||
value={selectedClientId}
|
||||
onChange={(e) => setSelectedClientId(e.target.value)}
|
||||
>
|
||||
{clients.map((c) => (
|
||||
<option key={c.clientId} value={c.clientId}>
|
||||
{c.clientId}
|
||||
</option>
|
||||
))}
|
||||
</Select>
|
||||
</div>
|
||||
) : undefined
|
||||
<div className="flex items-center gap-2">
|
||||
{clients.length > 0 && (
|
||||
<div className="w-[190px]">
|
||||
<Select
|
||||
compact
|
||||
value={selectedClientId}
|
||||
onChange={(e) => setSelectedClientId(e.target.value)}
|
||||
>
|
||||
{clients.map((c) => (
|
||||
<option key={c.clientId} value={c.clientId}>
|
||||
{c.clientId}
|
||||
</option>
|
||||
))}
|
||||
</Select>
|
||||
</div>
|
||||
)}
|
||||
<Button
|
||||
size="xs"
|
||||
variant="ghost"
|
||||
icon="Copy"
|
||||
onClick={() => copySnippet(configSnippet, 'config.datastack.py')}
|
||||
/>
|
||||
</div>
|
||||
}
|
||||
>
|
||||
<CodeBlock>{configSnippet}</CodeBlock>
|
||||
</Panel>
|
||||
|
||||
<Panel
|
||||
title="Step 2 — Application Usage (Events, Batches & Direct CSV/JSON Files)"
|
||||
className="w-full"
|
||||
action={
|
||||
<Button
|
||||
size="xs"
|
||||
variant="ghost"
|
||||
icon="Copy"
|
||||
onClick={() => copySnippet(pythonSnippet, 'Python SDK snippet')}
|
||||
/>
|
||||
}
|
||||
>
|
||||
<CodeBlock>{pythonSnippet}</CodeBlock>
|
||||
|
||||
Reference in new issue
Block a user