From 0c2e49cd47c00ac109cbb3ef6a4b4fa5150652a9 Mon Sep 17 00:00:00 2001 From: Team Dark Devil Date: Wed, 18 Nov 2026 02:23:01 +0600 Subject: [PATCH] feat: refactor Python SDK and enhance API error reporting - Relocated Python SDK to `sdk/python/datastack` with new auto-configuration support via `config.datastack.py` - Updated SDK installation and usage documentation - Enhanced `/api/events` response format to include specific error codes for revoked and rotated credentials - Updated SDK dashboard snippet generator to support new configuration pattern --- README.md | 87 +- app/api/events/route.ts | 158 +++- app/vcon/sdk/page.tsx | 135 ++- components/sections/SdkSection.tsx | 23 +- manifest.md | 3 +- sdk-python/datastack_sdk/client.py | 195 +--- sdk/python/datastack/README.md | 110 +++ .../datastack/config.datastack.example.py | 51 ++ sdk/python/datastack/datastack/__init__.py | 43 + sdk/python/datastack/datastack/client.py | 846 ++++++++++++++++++ sdk/python/datastack/datastack/config.py | 321 +++++++ sdk/python/datastack/datastack/exceptions.py | 72 ++ sdk/python/datastack/datastack/heartbeat.py | 186 ++++ sdk/python/datastack/datastack/spool.py | 164 ++++ sdk/python/datastack/pyproject.toml | 54 ++ sdk/python/datastack/setup.py | 44 + server/index.js | 4 +- server/socket/handlers.js | 41 + 18 files changed, 2288 insertions(+), 249 deletions(-) create mode 100644 sdk/python/datastack/README.md create mode 100644 sdk/python/datastack/config.datastack.example.py create mode 100644 sdk/python/datastack/datastack/__init__.py create mode 100644 sdk/python/datastack/datastack/client.py create mode 100644 sdk/python/datastack/datastack/config.py create mode 100644 sdk/python/datastack/datastack/exceptions.py create mode 100644 sdk/python/datastack/datastack/heartbeat.py create mode 100644 sdk/python/datastack/datastack/spool.py create mode 100644 sdk/python/datastack/pyproject.toml create mode 100644 sdk/python/datastack/setup.py diff --git a/README.md b/README.md index 1caa829..d83a789 100644 --- a/README.md +++ b/README.md @@ -182,50 +182,72 @@ All environment variables are documented in [`.env.example`](./.env.example): --- -## Python SDK (`datastack-sdk`) +## Python SDK (`sdk/python/datastack/`) -The official Python SDK is located in [`./sdk-python`](./sdk-python) and uses `python-socketio` with `threading.Event` synchronization for reliable acknowledgments over both local and WAN connections. +The official production Python SDK is located in [`./sdk/python/datastack`](./sdk/python/datastack) (with full backward compatibility in [`./sdk-python`](./sdk-python)). It provides zero-boilerplate `config.datastack.py` auto-discovery, low-latency Socket.IO streaming, automatic Heartbeat HTTP fallback (`POST /api/events`), offline disk spooling (`.datastack_spool/spool.jsonl`), and direct `.csv`, `.json`, and `.jsonl` file ingestion. ### Installation ```bash -pip install ./sdk-python +# Install from local repository +pip install ./sdk/python/datastack + +# Or install from PyPI release +pip install datastack ``` -### Usage Example +### 1. Drop `config.datastack.py` into Your Project Root + +You can copy a pre-populated `config.datastack.py` directly from **`/vcon/sdk`**: ```python -from datastack_sdk import DataStackClient +# config.datastack.py +DATASTACK_CONFIG = { + "SERVER_URL": "http://localhost:4000", # Port 4000 Socket.IO Ingestion Service + "API_URL": "http://localhost:3000", # Port 3000 HTTP Fallback & File Ingestion API + "CLIENT_TOKEN": "YOUR_SIGNED_CLIENT_JWT_TOKEN", + "AUTO_CONNECT": True, + "ENABLE_HTTP_FALLBACK": True, + "HEARTBEAT_INTERVAL_SEC": 5.0, + "MAX_RETRIES": 3, + "BATCH_CHUNK_SIZE": 250, + "OFFLINE_SPOOL_ENABLED": True, + "OFFLINE_SPOOL_DIR": ".datastack_spool", +} +``` -client = DataStackClient( - server_url="http://localhost:4000", - token="YOUR_SIGNED_CLIENT_JWT_TOKEN", -) +### 2. Stream Events, Batches, and `.csv` / `.json` Files -if client.connect(): - # 1. Send a single event and wait for server acknowledgment - ack = client.sync( - data={ - "event_type": "sensor_reading", - "device_id": "env-node-04", - "temperature_c": 24.2, - "humidity_pct": 56.8, - }, - timeout=5.0, - ) - print("Single event ack:", ack) +```python +from datastack import DataStack - # 2. Send a batch of events in a single round-trip - batch_ack = client.batch_sync( - records=[ - {"metric": "cpu_usage", "value": 38.4, "unit": "%"}, - {"metric": "memory_used_mb", "value": 2048, "unit": "MB"}, - ], - timeout=10.0, - ) - print("Batch ack:", batch_ack) +# Automatically discovers config.datastack.py and connects +client = DataStack() - client.disconnect() +# 1. Send a single event (WebSocket primary -> Auto HTTP Fallback -> Offline Spool) +ack = client.send({ + "event_type": "sensor_reading", + "device_id": "env-node-04", + "temperature_c": 24.2, + "humidity_pct": 56.8, +}) +print("Single event ack:", ack) + +# 2. Send a batch of events +batch_ack = client.send_batch([ + {"metric": "cpu_usage", "value": 38.4, "unit": "%"}, + {"metric": "memory_used_mb", "value": 2048, "unit": "MB"}, +]) +print("Batch ack:", batch_ack) + +# 3. Directly stream a .csv, .json, or .jsonl file in chunks +file_ack = client.send_file("monthly_telemetry.csv", chunk_size=250) +print("File ingestion ack:", file_ack) + +# 4. Inspect connection state & Heartbeat IDLE status +print("SDK status:", client.status()) + +client.disconnect() ``` --- @@ -237,12 +259,13 @@ Connect with Socket.IO handshake auth: `auth: { token: "" }`. - **`sync_data`**: Emit `{ eventId?, timestamp?, data }` -> Callback returns `{ success: true, eventId }` or `{ success: false, error }`. - **`batch_sync`**: Emit `[{ eventId?, timestamp?, data }, ...]` -> Callback returns `{ success: true, processed, rejected, errors? }`. +- **`ping_verify`**: Emit `{}` -> Verifies live client status in PostgreSQL and returns `{ success: true, status: "VERIFIED", clientId, rateLimit, maxPayloadSizeBytes, serverTime }`. ### REST Endpoints (Port 3000) | Method & Endpoint | Auth Required | Description | | :--- | :--- | :--- | -| `POST /api/events` | Client Bearer JWT or Admin Cookie | Ingest a single event or `{ batch: [...] }` over HTTP | +| `POST /api/events` | Client Bearer JWT or Admin Cookie | Ingest a single event, `{ batch: [...] }`, or `{ fileName, records: [...] }` over HTTP | | `GET /api/events` | Admin Session Cookie | List recent buffered and persisted events | | `GET /api/clients` | Admin Session Cookie | List registered SDK clients and their signed JWT tokens | | `POST /api/clients` | Admin Session Cookie | Register a new SDK client (`clientId`, `name`, `rateLimit`) | diff --git a/app/api/events/route.ts b/app/api/events/route.ts index 07aaeca..9aa2148 100644 --- a/app/api/events/route.ts +++ b/app/api/events/route.ts @@ -120,8 +120,13 @@ export async function POST(req: NextRequest) { } const body = await req.json(); + const bodyObj = + body && typeof body === 'object' && !Array.isArray(body) + ? (body as Record) + : {}; + const clientId = String( - tokenClientId || body.clientId || 'py_telemetry_agent_01', + tokenClientId || bodyObj.clientId || 'py_telemetry_agent_01', ) .trim() .toLowerCase() @@ -131,7 +136,10 @@ export async function POST(req: NextRequest) { if (registeredClient) { if (registeredClient.status === 'revoked') { return NextResponse.json( - { error: `Client '${clientId}' access has been revoked by administrator.` }, + { + error: `CLIENT_REVOKED: Client '${clientId}' access has been revoked by administrator.`, + code: 'CLIENT_REVOKED', + }, { status: 403 }, ); } @@ -141,27 +149,144 @@ export async function POST(req: NextRequest) { registeredClient.token !== rawBearerToken ) { return NextResponse.json( - { error: 'Client credentials have been rotated. Use the latest token.' }, + { + error: 'TOKEN_ROTATED: Client credentials have been rotated. Use the latest token.', + code: 'TOKEN_ROTATED', + }, { status: 401 }, ); } } - const eventId = - body.eventId || - `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`; - const timestamp = body.timestamp || new Date().toISOString(); - const data = - body.data !== undefined - ? body.data - : { sample: 'Realtime event via DataStack API', rand: Math.random() }; - const effectiveClientRate = registeredClient?.rateLimit || tokenRateLimit || settings.maxEventsPerSec || 100; + // Detect if this is a batch array or direct file-records ingestion request + const batchItems: unknown[] | null = Array.isArray(body) + ? body + : Array.isArray(bodyObj.batch) + ? (bodyObj.batch as unknown[]) + : Array.isArray(bodyObj.records) + ? (bodyObj.records as unknown[]) + : null; + + if (batchItems !== null) { + let processed = 0; + let rejected = 0; + let duplicates = 0; + let hitRateLimit = false; + let hitPayloadLimit = false; + const errors: string[] = []; + + for (const item of batchItems) { + const safeItem = + item && typeof item === 'object' && !Array.isArray(item) + ? (item as Record) + : { value: item }; + + const itemEventId = + typeof safeItem.eventId === 'string' && safeItem.eventId.trim() + ? safeItem.eventId.trim() + : `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`; + + const itemTimestamp = + typeof safeItem.timestamp === 'string' && safeItem.timestamp.trim() + ? safeItem.timestamp.trim() + : new Date().toISOString(); + + const itemData = + safeItem.data !== undefined ? safeItem.data : safeItem; + + const val = SecurityEngine.validateEvent( + clientId, + itemEventId, + itemData, + effectiveClientRate, + ); + + if (!val.allowed) { + if (val.code === 'DUPLICATE_EVENT') { + duplicates += 1; + continue; + } + rejected += 1; + if (val.code === 'RATE_LIMIT_EXCEEDED') hitRateLimit = true; + if (val.code === 'PAYLOAD_TOO_LARGE') hitPayloadLimit = true; + if (errors.length < 5) { + errors.push(`${itemEventId}: ${val.code || 'REJECTED'} - ${val.reason}`); + } + continue; + } + + const event: DataEvent = { + clientId, + eventId: itemEventId, + timestamp: itemTimestamp, + payload: + typeof itemData === 'string' ? itemData : JSON.stringify(itemData), + }; + + DataBufferEngine.push(event); + processed += 1; + } + + if (processed > 0) { + SDKManager.incrementEventCount(clientId, processed); + } + + // If zero items were processed or deduplicated and all failed due to payload or rate limit, return corresponding status + if (processed === 0 && duplicates === 0 && rejected > 0) { + const status = hitPayloadLimit ? 413 : hitRateLimit ? 429 : 400; + const code = hitPayloadLimit + ? 'PAYLOAD_TOO_LARGE' + : hitRateLimit + ? 'RATE_LIMIT_EXCEEDED' + : 'BATCH_REJECTED'; + return NextResponse.json( + { + success: false, + processed, + rejected, + duplicates, + code, + error: errors[0] || 'Batch validation failed', + errors, + }, + { status }, + ); + } + + return NextResponse.json({ + success: true, + processed, + rejected, + duplicates, + ...(typeof bodyObj.fileName === 'string' + ? { fileName: bodyObj.fileName } + : {}), + ...(errors.length > 0 ? { errors } : {}), + }); + } + + // Single Event Ingestion + const eventId = + typeof bodyObj.eventId === 'string' && bodyObj.eventId.trim() + ? bodyObj.eventId.trim() + : `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`; + const timestamp = + typeof bodyObj.timestamp === 'string' && bodyObj.timestamp.trim() + ? bodyObj.timestamp.trim() + : new Date().toISOString(); + const data = + bodyObj.data !== undefined + ? bodyObj.data + : Object.keys(bodyObj).length > 0 + ? bodyObj + : { sample: 'Realtime event via DataStack API', rand: Math.random() }; + // Validate security (idempotency, rate limit, payload size) const val = SecurityEngine.validateEvent( clientId, @@ -170,6 +295,13 @@ export async function POST(req: NextRequest) { effectiveClientRate, ); if (!val.allowed) { + if (val.code === 'DUPLICATE_EVENT') { + return NextResponse.json({ + success: true, + eventId, + note: 'DUPLICATE_DROPPED', + }); + } const status = val.code === 'PAYLOAD_TOO_LARGE' ? 413 : 429; return NextResponse.json({ error: val.reason, code: val.code }, { status }); } @@ -184,7 +316,7 @@ export async function POST(req: NextRequest) { DataBufferEngine.push(event); SDKManager.incrementEventCount(clientId, 1); - return NextResponse.json({ success: true, event }); + return NextResponse.json({ success: true, eventId, event }); } catch (err: unknown) { const message = err instanceof Error ? err.message : 'Event ingestion failed'; return NextResponse.json({ error: message }, { status: 500 }); diff --git a/app/vcon/sdk/page.tsx b/app/vcon/sdk/page.tsx index 1ad0422..dce2a0c 100644 --- a/app/vcon/sdk/page.tsx +++ b/app/vcon/sdk/page.tsx @@ -21,6 +21,7 @@ export default function SdkGuidePage() { return cached[0]?.clientId ?? ''; }); const [serverUrl, setServerUrl] = useState('http://localhost:4000'); + const [apiUrl, setApiUrl] = useState('http://localhost:3000'); useEffect(() => { const isLocalDev = window.location.port === '3000'; @@ -29,6 +30,7 @@ export default function SdkGuidePage() { ? `${window.location.protocol}//${window.location.hostname}:4000` : window.location.origin ); + setApiUrl(window.location.origin); async function loadClients() { try { @@ -54,28 +56,48 @@ export default function SdkGuidePage() { clients.find((c) => c.clientId === selectedClientId) || clients[0]; const sampleToken = activeClient?.token || 'YOUR_SIGNED_CLIENT_JWT_TOKEN'; - const pythonSnippet = `from datastack_sdk import DataStackClient + const configSnippet = `# config.datastack.py — Place in your Python project root +DATASTACK_CONFIG = { + "SERVER_URL": "${serverUrl}", + "API_URL": "${apiUrl}", + "CLIENT_TOKEN": "${sampleToken}", + "AUTO_CONNECT": True, + "ENABLE_HTTP_FALLBACK": True, + "HEARTBEAT_INTERVAL_SEC": 5.0, + "MAX_RETRIES": 3, + "BATCH_CHUNK_SIZE": 250, + "OFFLINE_SPOOL_ENABLED": True, + "OFFLINE_SPOOL_DIR": ".datastack_spool", +}`; -client = DataStackClient( - server_url="${serverUrl}", - token="${sampleToken}" -) + const pythonSnippet = `from datastack import DataStack -if client.connect(): - res = client.sync({ - "event_type": "page_view", - "url": "/products/wireless-headphones", - "user_id": "usr_9942" - }) - print("Sync response:", res) +# Automatically discovers config.datastack.py and connects +client = DataStack() - batch_res = client.batch_sync([ - {"sensor": "temp", "value": 24.5, "unit": "C"}, - {"sensor": "humidity", "value": 61.2, "unit": "%"} - ]) - print("Batch response:", batch_res) +# 1. Single Event (WebSocket + Auto HTTP Fallback + Offline Spool) +res = client.send({ + "event_type": "page_view", + "url": "/products/wireless-headphones", + "user_id": "usr_9942" +}) +print("Event response:", res) - client.disconnect()`; +# 2. Batch Events +batch_res = client.send_batch([ + {"sensor": "temp", "value": 24.5, "unit": "C"}, + {"sensor": "humidity", "value": 61.2, "unit": "%"} +]) +print("Batch response:", batch_res) + +# 3. Direct File Ingestion (.csv, .json, .jsonl) +file_res = client.send_file("events_dataset.csv", chunk_size=250) +print("File ingestion:", file_res) + +# 4. Inspect Connection & Heartbeat IDLE Status +print("SDK Status:", client.status()) + +client.disconnect()`; const copySnippet = (text: string, label: string) => { navigator.clipboard.writeText(text); @@ -97,11 +119,19 @@ if client.connect(): Copy Token )} + @@ -110,8 +140,8 @@ if client.connect(): > - pip install ./sdk-python + + pip install ./sdk/python/datastack +