feat: refactor Python SDK and enhance API error reporting
- Relocated Python SDK to `sdk/python/datastack` with new auto-configuration support via `config.datastack.py` - Updated SDK installation and usage documentation - Enhanced `/api/events` response format to include specific error codes for revoked and rotated credentials - Updated SDK dashboard snippet generator to support new configuration pattern
This commit is contained in:
1 parent
367dd819f8
commit
0c2e49cd47
18 files changed
+2257
-218
No files matched your search
@@ -182,49 +182,71 @@ All environment variables are documented in [`.env.example`](./.env.example):
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Python SDK (`datastack-sdk`)
|
## Python SDK (`sdk/python/datastack/`)
|
||||||
|
|
||||||
The official Python SDK is located in [`./sdk-python`](./sdk-python) and uses `python-socketio` with `threading.Event` synchronization for reliable acknowledgments over both local and WAN connections.
|
The official production Python SDK is located in [`./sdk/python/datastack`](./sdk/python/datastack) (with full backward compatibility in [`./sdk-python`](./sdk-python)). It provides zero-boilerplate `config.datastack.py` auto-discovery, low-latency Socket.IO streaming, automatic Heartbeat HTTP fallback (`POST /api/events`), offline disk spooling (`.datastack_spool/spool.jsonl`), and direct `.csv`, `.json`, and `.jsonl` file ingestion.
|
||||||
|
|
||||||
### Installation
|
### Installation
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
pip install ./sdk-python
|
# Install from local repository
|
||||||
|
pip install ./sdk/python/datastack
|
||||||
|
|
||||||
|
# Or install from PyPI release
|
||||||
|
pip install datastack
|
||||||
```
|
```
|
||||||
|
|
||||||
### Usage Example
|
### 1. Drop `config.datastack.py` into Your Project Root
|
||||||
|
|
||||||
|
You can copy a pre-populated `config.datastack.py` directly from **`/vcon/sdk`**:
|
||||||
|
|
||||||
```python
|
```python
|
||||||
from datastack_sdk import DataStackClient
|
# config.datastack.py
|
||||||
|
DATASTACK_CONFIG = {
|
||||||
|
"SERVER_URL": "http://localhost:4000", # Port 4000 Socket.IO Ingestion Service
|
||||||
|
"API_URL": "http://localhost:3000", # Port 3000 HTTP Fallback & File Ingestion API
|
||||||
|
"CLIENT_TOKEN": "YOUR_SIGNED_CLIENT_JWT_TOKEN",
|
||||||
|
"AUTO_CONNECT": True,
|
||||||
|
"ENABLE_HTTP_FALLBACK": True,
|
||||||
|
"HEARTBEAT_INTERVAL_SEC": 5.0,
|
||||||
|
"MAX_RETRIES": 3,
|
||||||
|
"BATCH_CHUNK_SIZE": 250,
|
||||||
|
"OFFLINE_SPOOL_ENABLED": True,
|
||||||
|
"OFFLINE_SPOOL_DIR": ".datastack_spool",
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
client = DataStackClient(
|
### 2. Stream Events, Batches, and `.csv` / `.json` Files
|
||||||
server_url="http://localhost:4000",
|
|
||||||
token="YOUR_SIGNED_CLIENT_JWT_TOKEN",
|
|
||||||
)
|
|
||||||
|
|
||||||
if client.connect():
|
```python
|
||||||
# 1. Send a single event and wait for server acknowledgment
|
from datastack import DataStack
|
||||||
ack = client.sync(
|
|
||||||
data={
|
# Automatically discovers config.datastack.py and connects
|
||||||
|
client = DataStack()
|
||||||
|
|
||||||
|
# 1. Send a single event (WebSocket primary -> Auto HTTP Fallback -> Offline Spool)
|
||||||
|
ack = client.send({
|
||||||
"event_type": "sensor_reading",
|
"event_type": "sensor_reading",
|
||||||
"device_id": "env-node-04",
|
"device_id": "env-node-04",
|
||||||
"temperature_c": 24.2,
|
"temperature_c": 24.2,
|
||||||
"humidity_pct": 56.8,
|
"humidity_pct": 56.8,
|
||||||
},
|
})
|
||||||
timeout=5.0,
|
|
||||||
)
|
|
||||||
print("Single event ack:", ack)
|
print("Single event ack:", ack)
|
||||||
|
|
||||||
# 2. Send a batch of events in a single round-trip
|
# 2. Send a batch of events
|
||||||
batch_ack = client.batch_sync(
|
batch_ack = client.send_batch([
|
||||||
records=[
|
|
||||||
{"metric": "cpu_usage", "value": 38.4, "unit": "%"},
|
{"metric": "cpu_usage", "value": 38.4, "unit": "%"},
|
||||||
{"metric": "memory_used_mb", "value": 2048, "unit": "MB"},
|
{"metric": "memory_used_mb", "value": 2048, "unit": "MB"},
|
||||||
],
|
])
|
||||||
timeout=10.0,
|
|
||||||
)
|
|
||||||
print("Batch ack:", batch_ack)
|
print("Batch ack:", batch_ack)
|
||||||
|
|
||||||
|
# 3. Directly stream a .csv, .json, or .jsonl file in chunks
|
||||||
|
file_ack = client.send_file("monthly_telemetry.csv", chunk_size=250)
|
||||||
|
print("File ingestion ack:", file_ack)
|
||||||
|
|
||||||
|
# 4. Inspect connection state & Heartbeat IDLE status
|
||||||
|
print("SDK status:", client.status())
|
||||||
|
|
||||||
client.disconnect()
|
client.disconnect()
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -237,12 +259,13 @@ Connect with Socket.IO handshake auth: `auth: { token: "<CLIENT_JWT_TOKEN>" }`.
|
|||||||
|
|
||||||
- **`sync_data`**: Emit `{ eventId?, timestamp?, data }` -> Callback returns `{ success: true, eventId }` or `{ success: false, error }`.
|
- **`sync_data`**: Emit `{ eventId?, timestamp?, data }` -> Callback returns `{ success: true, eventId }` or `{ success: false, error }`.
|
||||||
- **`batch_sync`**: Emit `[{ eventId?, timestamp?, data }, ...]` -> Callback returns `{ success: true, processed, rejected, errors? }`.
|
- **`batch_sync`**: Emit `[{ eventId?, timestamp?, data }, ...]` -> Callback returns `{ success: true, processed, rejected, errors? }`.
|
||||||
|
- **`ping_verify`**: Emit `{}` -> Verifies live client status in PostgreSQL and returns `{ success: true, status: "VERIFIED", clientId, rateLimit, maxPayloadSizeBytes, serverTime }`.
|
||||||
|
|
||||||
### REST Endpoints (Port 3000)
|
### REST Endpoints (Port 3000)
|
||||||
|
|
||||||
| Method & Endpoint | Auth Required | Description |
|
| Method & Endpoint | Auth Required | Description |
|
||||||
| :--- | :--- | :--- |
|
| :--- | :--- | :--- |
|
||||||
| `POST /api/events` | Client Bearer JWT or Admin Cookie | Ingest a single event or `{ batch: [...] }` over HTTP |
|
| `POST /api/events` | Client Bearer JWT or Admin Cookie | Ingest a single event, `{ batch: [...] }`, or `{ fileName, records: [...] }` over HTTP |
|
||||||
| `GET /api/events` | Admin Session Cookie | List recent buffered and persisted events |
|
| `GET /api/events` | Admin Session Cookie | List recent buffered and persisted events |
|
||||||
| `GET /api/clients` | Admin Session Cookie | List registered SDK clients and their signed JWT tokens |
|
| `GET /api/clients` | Admin Session Cookie | List registered SDK clients and their signed JWT tokens |
|
||||||
| `POST /api/clients` | Admin Session Cookie | Register a new SDK client (`clientId`, `name`, `rateLimit`) |
|
| `POST /api/clients` | Admin Session Cookie | Register a new SDK client (`clientId`, `name`, `rateLimit`) |
|
||||||
|
|||||||
+145
-13
@@ -120,8 +120,13 @@ export async function POST(req: NextRequest) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const body = await req.json();
|
const body = await req.json();
|
||||||
|
const bodyObj =
|
||||||
|
body && typeof body === 'object' && !Array.isArray(body)
|
||||||
|
? (body as Record<string, unknown>)
|
||||||
|
: {};
|
||||||
|
|
||||||
const clientId = String(
|
const clientId = String(
|
||||||
tokenClientId || body.clientId || 'py_telemetry_agent_01',
|
tokenClientId || bodyObj.clientId || 'py_telemetry_agent_01',
|
||||||
)
|
)
|
||||||
.trim()
|
.trim()
|
||||||
.toLowerCase()
|
.toLowerCase()
|
||||||
@@ -131,7 +136,10 @@ export async function POST(req: NextRequest) {
|
|||||||
if (registeredClient) {
|
if (registeredClient) {
|
||||||
if (registeredClient.status === 'revoked') {
|
if (registeredClient.status === 'revoked') {
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{ error: `Client '${clientId}' access has been revoked by administrator.` },
|
{
|
||||||
|
error: `CLIENT_REVOKED: Client '${clientId}' access has been revoked by administrator.`,
|
||||||
|
code: 'CLIENT_REVOKED',
|
||||||
|
},
|
||||||
{ status: 403 },
|
{ status: 403 },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -141,27 +149,144 @@ export async function POST(req: NextRequest) {
|
|||||||
registeredClient.token !== rawBearerToken
|
registeredClient.token !== rawBearerToken
|
||||||
) {
|
) {
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
{ error: 'Client credentials have been rotated. Use the latest token.' },
|
{
|
||||||
|
error: 'TOKEN_ROTATED: Client credentials have been rotated. Use the latest token.',
|
||||||
|
code: 'TOKEN_ROTATED',
|
||||||
|
},
|
||||||
{ status: 401 },
|
{ status: 401 },
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const eventId =
|
|
||||||
body.eventId ||
|
|
||||||
`evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
|
||||||
const timestamp = body.timestamp || new Date().toISOString();
|
|
||||||
const data =
|
|
||||||
body.data !== undefined
|
|
||||||
? body.data
|
|
||||||
: { sample: 'Realtime event via DataStack API', rand: Math.random() };
|
|
||||||
|
|
||||||
const effectiveClientRate =
|
const effectiveClientRate =
|
||||||
registeredClient?.rateLimit ||
|
registeredClient?.rateLimit ||
|
||||||
tokenRateLimit ||
|
tokenRateLimit ||
|
||||||
settings.maxEventsPerSec ||
|
settings.maxEventsPerSec ||
|
||||||
100;
|
100;
|
||||||
|
|
||||||
|
// Detect if this is a batch array or direct file-records ingestion request
|
||||||
|
const batchItems: unknown[] | null = Array.isArray(body)
|
||||||
|
? body
|
||||||
|
: Array.isArray(bodyObj.batch)
|
||||||
|
? (bodyObj.batch as unknown[])
|
||||||
|
: Array.isArray(bodyObj.records)
|
||||||
|
? (bodyObj.records as unknown[])
|
||||||
|
: null;
|
||||||
|
|
||||||
|
if (batchItems !== null) {
|
||||||
|
let processed = 0;
|
||||||
|
let rejected = 0;
|
||||||
|
let duplicates = 0;
|
||||||
|
let hitRateLimit = false;
|
||||||
|
let hitPayloadLimit = false;
|
||||||
|
const errors: string[] = [];
|
||||||
|
|
||||||
|
for (const item of batchItems) {
|
||||||
|
const safeItem =
|
||||||
|
item && typeof item === 'object' && !Array.isArray(item)
|
||||||
|
? (item as Record<string, unknown>)
|
||||||
|
: { value: item };
|
||||||
|
|
||||||
|
const itemEventId =
|
||||||
|
typeof safeItem.eventId === 'string' && safeItem.eventId.trim()
|
||||||
|
? safeItem.eventId.trim()
|
||||||
|
: `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||||
|
|
||||||
|
const itemTimestamp =
|
||||||
|
typeof safeItem.timestamp === 'string' && safeItem.timestamp.trim()
|
||||||
|
? safeItem.timestamp.trim()
|
||||||
|
: new Date().toISOString();
|
||||||
|
|
||||||
|
const itemData =
|
||||||
|
safeItem.data !== undefined ? safeItem.data : safeItem;
|
||||||
|
|
||||||
|
const val = SecurityEngine.validateEvent(
|
||||||
|
clientId,
|
||||||
|
itemEventId,
|
||||||
|
itemData,
|
||||||
|
effectiveClientRate,
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!val.allowed) {
|
||||||
|
if (val.code === 'DUPLICATE_EVENT') {
|
||||||
|
duplicates += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
rejected += 1;
|
||||||
|
if (val.code === 'RATE_LIMIT_EXCEEDED') hitRateLimit = true;
|
||||||
|
if (val.code === 'PAYLOAD_TOO_LARGE') hitPayloadLimit = true;
|
||||||
|
if (errors.length < 5) {
|
||||||
|
errors.push(`${itemEventId}: ${val.code || 'REJECTED'} - ${val.reason}`);
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const event: DataEvent = {
|
||||||
|
clientId,
|
||||||
|
eventId: itemEventId,
|
||||||
|
timestamp: itemTimestamp,
|
||||||
|
payload:
|
||||||
|
typeof itemData === 'string' ? itemData : JSON.stringify(itemData),
|
||||||
|
};
|
||||||
|
|
||||||
|
DataBufferEngine.push(event);
|
||||||
|
processed += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (processed > 0) {
|
||||||
|
SDKManager.incrementEventCount(clientId, processed);
|
||||||
|
}
|
||||||
|
|
||||||
|
// If zero items were processed or deduplicated and all failed due to payload or rate limit, return corresponding status
|
||||||
|
if (processed === 0 && duplicates === 0 && rejected > 0) {
|
||||||
|
const status = hitPayloadLimit ? 413 : hitRateLimit ? 429 : 400;
|
||||||
|
const code = hitPayloadLimit
|
||||||
|
? 'PAYLOAD_TOO_LARGE'
|
||||||
|
: hitRateLimit
|
||||||
|
? 'RATE_LIMIT_EXCEEDED'
|
||||||
|
: 'BATCH_REJECTED';
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
success: false,
|
||||||
|
processed,
|
||||||
|
rejected,
|
||||||
|
duplicates,
|
||||||
|
code,
|
||||||
|
error: errors[0] || 'Batch validation failed',
|
||||||
|
errors,
|
||||||
|
},
|
||||||
|
{ status },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return NextResponse.json({
|
||||||
|
success: true,
|
||||||
|
processed,
|
||||||
|
rejected,
|
||||||
|
duplicates,
|
||||||
|
...(typeof bodyObj.fileName === 'string'
|
||||||
|
? { fileName: bodyObj.fileName }
|
||||||
|
: {}),
|
||||||
|
...(errors.length > 0 ? { errors } : {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Single Event Ingestion
|
||||||
|
const eventId =
|
||||||
|
typeof bodyObj.eventId === 'string' && bodyObj.eventId.trim()
|
||||||
|
? bodyObj.eventId.trim()
|
||||||
|
: `evt_${Date.now()}_${Math.random().toString(36).substring(2, 7)}`;
|
||||||
|
const timestamp =
|
||||||
|
typeof bodyObj.timestamp === 'string' && bodyObj.timestamp.trim()
|
||||||
|
? bodyObj.timestamp.trim()
|
||||||
|
: new Date().toISOString();
|
||||||
|
const data =
|
||||||
|
bodyObj.data !== undefined
|
||||||
|
? bodyObj.data
|
||||||
|
: Object.keys(bodyObj).length > 0
|
||||||
|
? bodyObj
|
||||||
|
: { sample: 'Realtime event via DataStack API', rand: Math.random() };
|
||||||
|
|
||||||
// Validate security (idempotency, rate limit, payload size)
|
// Validate security (idempotency, rate limit, payload size)
|
||||||
const val = SecurityEngine.validateEvent(
|
const val = SecurityEngine.validateEvent(
|
||||||
clientId,
|
clientId,
|
||||||
@@ -170,6 +295,13 @@ export async function POST(req: NextRequest) {
|
|||||||
effectiveClientRate,
|
effectiveClientRate,
|
||||||
);
|
);
|
||||||
if (!val.allowed) {
|
if (!val.allowed) {
|
||||||
|
if (val.code === 'DUPLICATE_EVENT') {
|
||||||
|
return NextResponse.json({
|
||||||
|
success: true,
|
||||||
|
eventId,
|
||||||
|
note: 'DUPLICATE_DROPPED',
|
||||||
|
});
|
||||||
|
}
|
||||||
const status = val.code === 'PAYLOAD_TOO_LARGE' ? 413 : 429;
|
const status = val.code === 'PAYLOAD_TOO_LARGE' ? 413 : 429;
|
||||||
return NextResponse.json({ error: val.reason, code: val.code }, { status });
|
return NextResponse.json({ error: val.reason, code: val.code }, { status });
|
||||||
}
|
}
|
||||||
@@ -184,7 +316,7 @@ export async function POST(req: NextRequest) {
|
|||||||
DataBufferEngine.push(event);
|
DataBufferEngine.push(event);
|
||||||
SDKManager.incrementEventCount(clientId, 1);
|
SDKManager.incrementEventCount(clientId, 1);
|
||||||
|
|
||||||
return NextResponse.json({ success: true, event });
|
return NextResponse.json({ success: true, eventId, event });
|
||||||
} catch (err: unknown) {
|
} catch (err: unknown) {
|
||||||
const message = err instanceof Error ? err.message : 'Event ingestion failed';
|
const message = err instanceof Error ? err.message : 'Event ingestion failed';
|
||||||
return NextResponse.json({ error: message }, { status: 500 });
|
return NextResponse.json({ error: message }, { status: 500 });
|
||||||
|
|||||||
+74
-17
@@ -21,6 +21,7 @@ export default function SdkGuidePage() {
|
|||||||
return cached[0]?.clientId ?? '';
|
return cached[0]?.clientId ?? '';
|
||||||
});
|
});
|
||||||
const [serverUrl, setServerUrl] = useState<string>('http://localhost:4000');
|
const [serverUrl, setServerUrl] = useState<string>('http://localhost:4000');
|
||||||
|
const [apiUrl, setApiUrl] = useState<string>('http://localhost:3000');
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const isLocalDev = window.location.port === '3000';
|
const isLocalDev = window.location.port === '3000';
|
||||||
@@ -29,6 +30,7 @@ export default function SdkGuidePage() {
|
|||||||
? `${window.location.protocol}//${window.location.hostname}:4000`
|
? `${window.location.protocol}//${window.location.hostname}:4000`
|
||||||
: window.location.origin
|
: window.location.origin
|
||||||
);
|
);
|
||||||
|
setApiUrl(window.location.origin);
|
||||||
|
|
||||||
async function loadClients() {
|
async function loadClients() {
|
||||||
try {
|
try {
|
||||||
@@ -54,27 +56,47 @@ export default function SdkGuidePage() {
|
|||||||
clients.find((c) => c.clientId === selectedClientId) || clients[0];
|
clients.find((c) => c.clientId === selectedClientId) || clients[0];
|
||||||
const sampleToken = activeClient?.token || 'YOUR_SIGNED_CLIENT_JWT_TOKEN';
|
const sampleToken = activeClient?.token || 'YOUR_SIGNED_CLIENT_JWT_TOKEN';
|
||||||
|
|
||||||
const pythonSnippet = `from datastack_sdk import DataStackClient
|
const configSnippet = `# config.datastack.py — Place in your Python project root
|
||||||
|
DATASTACK_CONFIG = {
|
||||||
|
"SERVER_URL": "${serverUrl}",
|
||||||
|
"API_URL": "${apiUrl}",
|
||||||
|
"CLIENT_TOKEN": "${sampleToken}",
|
||||||
|
"AUTO_CONNECT": True,
|
||||||
|
"ENABLE_HTTP_FALLBACK": True,
|
||||||
|
"HEARTBEAT_INTERVAL_SEC": 5.0,
|
||||||
|
"MAX_RETRIES": 3,
|
||||||
|
"BATCH_CHUNK_SIZE": 250,
|
||||||
|
"OFFLINE_SPOOL_ENABLED": True,
|
||||||
|
"OFFLINE_SPOOL_DIR": ".datastack_spool",
|
||||||
|
}`;
|
||||||
|
|
||||||
client = DataStackClient(
|
const pythonSnippet = `from datastack import DataStack
|
||||||
server_url="${serverUrl}",
|
|
||||||
token="${sampleToken}"
|
|
||||||
)
|
|
||||||
|
|
||||||
if client.connect():
|
# Automatically discovers config.datastack.py and connects
|
||||||
res = client.sync({
|
client = DataStack()
|
||||||
|
|
||||||
|
# 1. Single Event (WebSocket + Auto HTTP Fallback + Offline Spool)
|
||||||
|
res = client.send({
|
||||||
"event_type": "page_view",
|
"event_type": "page_view",
|
||||||
"url": "/products/wireless-headphones",
|
"url": "/products/wireless-headphones",
|
||||||
"user_id": "usr_9942"
|
"user_id": "usr_9942"
|
||||||
})
|
})
|
||||||
print("Sync response:", res)
|
print("Event response:", res)
|
||||||
|
|
||||||
batch_res = client.batch_sync([
|
# 2. Batch Events
|
||||||
|
batch_res = client.send_batch([
|
||||||
{"sensor": "temp", "value": 24.5, "unit": "C"},
|
{"sensor": "temp", "value": 24.5, "unit": "C"},
|
||||||
{"sensor": "humidity", "value": 61.2, "unit": "%"}
|
{"sensor": "humidity", "value": 61.2, "unit": "%"}
|
||||||
])
|
])
|
||||||
print("Batch response:", batch_res)
|
print("Batch response:", batch_res)
|
||||||
|
|
||||||
|
# 3. Direct File Ingestion (.csv, .json, .jsonl)
|
||||||
|
file_res = client.send_file("events_dataset.csv", chunk_size=250)
|
||||||
|
print("File ingestion:", file_res)
|
||||||
|
|
||||||
|
# 4. Inspect Connection & Heartbeat IDLE Status
|
||||||
|
print("SDK Status:", client.status())
|
||||||
|
|
||||||
client.disconnect()`;
|
client.disconnect()`;
|
||||||
|
|
||||||
const copySnippet = (text: string, label: string) => {
|
const copySnippet = (text: string, label: string) => {
|
||||||
@@ -97,11 +119,19 @@ if client.connect():
|
|||||||
Copy Token
|
Copy Token
|
||||||
</Button>
|
</Button>
|
||||||
)}
|
)}
|
||||||
|
<Button
|
||||||
|
size="xs"
|
||||||
|
variant="secondary"
|
||||||
|
icon="FileCode"
|
||||||
|
onClick={() => copySnippet(configSnippet, 'config.datastack.py')}
|
||||||
|
>
|
||||||
|
Copy Config
|
||||||
|
</Button>
|
||||||
<Button
|
<Button
|
||||||
size="xs"
|
size="xs"
|
||||||
variant="secondary"
|
variant="secondary"
|
||||||
icon="Copy"
|
icon="Copy"
|
||||||
onClick={() => copySnippet(pythonSnippet, 'Snippet')}
|
onClick={() => copySnippet(pythonSnippet, 'Python SDK snippet')}
|
||||||
>
|
>
|
||||||
Copy Code
|
Copy Code
|
||||||
</Button>
|
</Button>
|
||||||
@@ -110,8 +140,8 @@ if client.connect():
|
|||||||
>
|
>
|
||||||
<StatStrip
|
<StatStrip
|
||||||
items={[
|
items={[
|
||||||
{ label: 'SDK package', value: 'datastack_sdk', dot: 'ok' },
|
{ label: 'SDK package', value: 'datastack (v2.2.0)', dot: 'ok' },
|
||||||
{ label: 'Transport', value: 'Socket.IO / JWT' },
|
{ label: 'Transport', value: 'WS + HTTP Fallback + Offline Spool' },
|
||||||
{ label: 'Server endpoint', value: serverUrl },
|
{ label: 'Server endpoint', value: serverUrl },
|
||||||
{
|
{
|
||||||
label: 'Active client',
|
label: 'Active client',
|
||||||
@@ -124,20 +154,25 @@ if client.connect():
|
|||||||
compact
|
compact
|
||||||
className="flex w-full items-center justify-between bg-[var(--surface)] px-3 py-2 font-mono text-[11.5px]"
|
className="flex w-full items-center justify-between bg-[var(--surface)] px-3 py-2 font-mono text-[11.5px]"
|
||||||
>
|
>
|
||||||
<span className="text-[var(--accent)]">pip install ./sdk-python</span>
|
<span className="text-[var(--accent)]">
|
||||||
|
pip install ./sdk/python/datastack
|
||||||
|
</span>
|
||||||
<Button
|
<Button
|
||||||
size="xs"
|
size="xs"
|
||||||
variant="ghost"
|
variant="ghost"
|
||||||
icon="Copy"
|
icon="Copy"
|
||||||
onClick={() => copySnippet('pip install ./sdk-python', 'Pip command')}
|
onClick={() =>
|
||||||
|
copySnippet('pip install ./sdk/python/datastack', 'Pip command')
|
||||||
|
}
|
||||||
/>
|
/>
|
||||||
</Card>
|
</Card>
|
||||||
|
|
||||||
<Panel
|
<Panel
|
||||||
title="Client Snippet"
|
title="Step 1 — config.datastack.py"
|
||||||
className="w-full"
|
className="w-full"
|
||||||
action={
|
action={
|
||||||
clients.length > 0 ? (
|
<div className="flex items-center gap-2">
|
||||||
|
{clients.length > 0 && (
|
||||||
<div className="w-[190px]">
|
<div className="w-[190px]">
|
||||||
<Select
|
<Select
|
||||||
compact
|
compact
|
||||||
@@ -151,7 +186,29 @@ if client.connect():
|
|||||||
))}
|
))}
|
||||||
</Select>
|
</Select>
|
||||||
</div>
|
</div>
|
||||||
) : undefined
|
)}
|
||||||
|
<Button
|
||||||
|
size="xs"
|
||||||
|
variant="ghost"
|
||||||
|
icon="Copy"
|
||||||
|
onClick={() => copySnippet(configSnippet, 'config.datastack.py')}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<CodeBlock>{configSnippet}</CodeBlock>
|
||||||
|
</Panel>
|
||||||
|
|
||||||
|
<Panel
|
||||||
|
title="Step 2 — Application Usage (Events, Batches & Direct CSV/JSON Files)"
|
||||||
|
className="w-full"
|
||||||
|
action={
|
||||||
|
<Button
|
||||||
|
size="xs"
|
||||||
|
variant="ghost"
|
||||||
|
icon="Copy"
|
||||||
|
onClick={() => copySnippet(pythonSnippet, 'Python SDK snippet')}
|
||||||
|
/>
|
||||||
}
|
}
|
||||||
>
|
>
|
||||||
<CodeBlock>{pythonSnippet}</CodeBlock>
|
<CodeBlock>{pythonSnippet}</CodeBlock>
|
||||||
|
|||||||
@@ -7,23 +7,20 @@ export const SdkSection: React.FC = () => (
|
|||||||
<SectionHeading index="02" label="Python SDK" title="Drop in the SDK. Start streaming." />
|
<SectionHeading index="02" label="Python SDK" title="Drop in the SDK. Start streaming." />
|
||||||
<div className="grid grid-cols-1 gap-px border border-[var(--border)] bg-[var(--border)] md:grid-cols-2">
|
<div className="grid grid-cols-1 gap-px border border-[var(--border)] bg-[var(--border)] md:grid-cols-2">
|
||||||
<CodeBlock>
|
<CodeBlock>
|
||||||
<span className="text-[var(--text-faint)]"># Install the client</span>{'\n'}
|
<span className="text-[var(--text-faint)]"># Install the official Python SDK</span>{'\n'}
|
||||||
<span className="text-[var(--text)]">pip install ./sdk-python</span>{'\n\n'}
|
<span className="text-[var(--text)]">pip install ./sdk/python/datastack</span>{'\n\n'}
|
||||||
<span className="text-[var(--text-faint)]"># Send an event</span>{'\n'}
|
<span className="text-[var(--text-faint)]"># Auto-loads config.datastack.py from project root</span>{'\n'}
|
||||||
<span className="text-[var(--text)]">from datastack_sdk import DataStackClient</span>{'\n\n'}
|
<span className="text-[var(--text)]">from datastack import DataStack</span>{'\n\n'}
|
||||||
<span className="text-[var(--text)]">client = DataStackClient(</span>{'\n'}
|
<span className="text-[var(--text)]">client = DataStack()</span>{'\n'}
|
||||||
{' '}server_url=<span className="text-[var(--accent)]">"YOUR_SERVER_URL"</span>,{'\n'}
|
<span className="text-[var(--text)]">client.send({'{'}<span className="text-[var(--accent)]">"event"</span>: <span className="text-[var(--accent)]">"package_received"</span>{'}'})</span>{'\n'}
|
||||||
{' '}token=<span className="text-[var(--accent)]">"YOUR_CLIENT_TOKEN"</span>,{'\n'}
|
<span className="text-[var(--text)]">client.send_file(<span className="text-[var(--accent)]">"telemetry_batch.csv"</span>)</span>{'\n'}
|
||||||
<span className="text-[var(--text)]">)</span>{'\n\n'}
|
<span className="text-[var(--text)]">client.disconnect()</span>
|
||||||
<span className="text-[var(--text)]">if client.connect():</span>{'\n'}
|
|
||||||
{' '}client.sync({'{'}<span className="text-[var(--accent)]">"event"</span>: <span className="text-[var(--accent)]">"package_received"</span>{'}'}){'\n'}
|
|
||||||
{' '}client.disconnect()
|
|
||||||
</CodeBlock>
|
</CodeBlock>
|
||||||
<div className="bg-[var(--surface)] p-5 sm:p-6">
|
<div className="bg-[var(--surface)] p-5 sm:p-6">
|
||||||
<p className="mb-2 text-[11px] uppercase text-[var(--text-faint)]">How it works</p>
|
<p className="mb-2 text-[11px] uppercase text-[var(--text-faint)]">How it works</p>
|
||||||
<h3 className="mb-2 text-[14px] font-medium text-[var(--text)]">A small client. A clear workflow.</h3>
|
<h3 className="mb-2 text-[14px] font-medium text-[var(--text)]">Zero-boilerplate config. Auto-recovery built in.</h3>
|
||||||
<p className="text-[12px] leading-relaxed text-[var(--text-muted)]">
|
<p className="text-[12px] leading-relaxed text-[var(--text-muted)]">
|
||||||
Connect a Python source, send individual or batched events, then review and export activity from one workspace.
|
Drop <code className="text-[var(--text)]">config.datastack.py</code> into your Python project to stream single events, batches, or <code className="text-[var(--text)]">.csv</code> / <code className="text-[var(--text)]">.json</code> files directly—with automatic HTTP fallback, offline disk spooling, and Heartbeat IDLE recovery.
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
+2
-1
@@ -9,7 +9,8 @@ Root directory manifest for **DataStack**, an open-source, self-hosted real-time
|
|||||||
- `core/`: Backend TypeScript modules (`auth`, `data`, `Database`, `Storage`, `Security`, `sdk`, `adminSettings`, `apiEngine`).
|
- `core/`: Backend TypeScript modules (`auth`, `data`, `Database`, `Storage`, `Security`, `sdk`, `adminSettings`, `apiEngine`).
|
||||||
- `server/`: Dedicated Express + Socket.IO ingestion service on Port 4000 (`index.js`, `socket/handlers.js`, `workers/csvWriter.js`).
|
- `server/`: Dedicated Express + Socket.IO ingestion service on Port 4000 (`index.js`, `socket/handlers.js`, `workers/csvWriter.js`).
|
||||||
- `storage/`: Partitioned CSV data directory (`/storage/data/{clientId}/{YYYY-MM-DD}.csv`).
|
- `storage/`: Partitioned CSV data directory (`/storage/data/{clientId}/{YYYY-MM-DD}.csv`).
|
||||||
- `sdk-python/`: Official pip-installable Python client library (`datastack-sdk`).
|
- `sdk/python/datastack/`: Official production Python SDK (`datastack`) featuring `config.datastack.py` auto-discovery, Heartbeat HTTP fallback, offline disk spooling, and direct `.csv`/`.json`/`.jsonl` file ingestion.
|
||||||
|
- `sdk-python/`: Backward-compatible Python client package (`datastack-sdk`).
|
||||||
- `docker/`: Production Dockerfiles (`Dockerfile.web`, `Dockerfile.server`, `Dockerfile.proxy`), Nginx config, and `docker-compose.yml`.
|
- `docker/`: Production Dockerfiles (`Dockerfile.web`, `Dockerfile.server`, `Dockerfile.proxy`), Nginx config, and `docker-compose.yml`.
|
||||||
- `nginx/`: Local Nginx reverse proxy configuration (`default.conf`).
|
- `nginx/`: Local Nginx reverse proxy configuration (`default.conf`).
|
||||||
- `README.md`: Comprehensive A–Z project documentation, quickstart, architecture, and deployment guide.
|
- `README.md`: Comprehensive A–Z project documentation, quickstart, architecture, and deployment guide.
|
||||||
|
|||||||
@@ -1,154 +1,49 @@
|
|||||||
import time
|
|
||||||
import uuid
|
|
||||||
import logging
|
|
||||||
import threading
|
|
||||||
from typing import Any, Dict, List, Optional
|
|
||||||
import socketio
|
|
||||||
|
|
||||||
logging.basicConfig(level=logging.INFO)
|
|
||||||
logger = logging.getLogger("DataStackClient")
|
|
||||||
|
|
||||||
class DataStackClient:
|
|
||||||
"""
|
"""
|
||||||
DataStack Universal Realtime Data Collection Client
|
Backward-Compatible `datastack_sdk.client` Wrapper.
|
||||||
Syncs telemetry and event payloads via Socket.IO to port 4000
|
Delegates to the unified production SDK implementation in `sdk/python/datastack/datastack`
|
||||||
|
so existing applications importing `from datastack_sdk import DataStackClient` gain
|
||||||
|
automatic `config.datastack.py` discovery, Heartbeat HTTP fallback, offline spooling,
|
||||||
|
and `send_file()` support.
|
||||||
"""
|
"""
|
||||||
def __init__(
|
|
||||||
self,
|
import sys
|
||||||
server_url: str = "http://localhost:4000",
|
from pathlib import Path
|
||||||
token: str = "",
|
|
||||||
reconnection_attempts: int = 5,
|
# Resolve `sdk/python/datastack` from the repository root if not installed in site-packages
|
||||||
reconnection_delay: int = 1,
|
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||||
):
|
_NEW_SDK_PATH = _REPO_ROOT / "sdk" / "python" / "datastack"
|
||||||
self.server_url = server_url.rstrip("/")
|
if _NEW_SDK_PATH.exists() and str(_NEW_SDK_PATH) not in sys.path:
|
||||||
self.token = token
|
sys.path.insert(0, str(_NEW_SDK_PATH))
|
||||||
self.sio = socketio.Client(
|
|
||||||
reconnection=True,
|
from datastack.client import DataStack, DataStackClient # noqa: E402
|
||||||
reconnection_attempts=reconnection_attempts,
|
from datastack.config import DataStackConfig, load_config # noqa: E402
|
||||||
reconnection_delay=reconnection_delay,
|
from datastack.exceptions import ( # noqa: E402
|
||||||
logger=False,
|
DataStackAuthError,
|
||||||
engineio_logger=False,
|
DataStackConfigError,
|
||||||
|
DataStackConnectionError,
|
||||||
|
DataStackError,
|
||||||
|
DataStackFileError,
|
||||||
|
DataStackOfflineSpoolError,
|
||||||
|
DataStackPayloadTooLargeError,
|
||||||
|
DataStackRateLimitError,
|
||||||
)
|
)
|
||||||
self._register_handlers()
|
from datastack.heartbeat import ConnectionState, HeartbeatMonitor # noqa: E402
|
||||||
self.is_connected = False
|
from datastack.spool import DiskSpooler # noqa: E402
|
||||||
|
|
||||||
def _register_handlers(self):
|
__all__ = [
|
||||||
@self.sio.event
|
"DataStack",
|
||||||
def connect():
|
"DataStackClient",
|
||||||
self.is_connected = True
|
"DataStackConfig",
|
||||||
logger.info(f"Connected to DataStack Engine at {self.server_url}")
|
"load_config",
|
||||||
|
"ConnectionState",
|
||||||
@self.sio.event
|
"HeartbeatMonitor",
|
||||||
def disconnect():
|
"DiskSpooler",
|
||||||
self.is_connected = False
|
"DataStackError",
|
||||||
logger.warning("Disconnected from DataStack Engine")
|
"DataStackConfigError",
|
||||||
|
"DataStackAuthError",
|
||||||
@self.sio.event
|
"DataStackConnectionError",
|
||||||
def connect_error(data):
|
"DataStackRateLimitError",
|
||||||
self.is_connected = False
|
"DataStackPayloadTooLargeError",
|
||||||
logger.error(f"DataStack Connection Error: {data}")
|
"DataStackFileError",
|
||||||
|
"DataStackOfflineSpoolError",
|
||||||
def connect(self, wait_timeout: int = 5) -> bool:
|
]
|
||||||
"""
|
|
||||||
Connects to DataStack Socket.IO server with JWT token in handshake auth
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
self.sio.connect(
|
|
||||||
self.server_url,
|
|
||||||
auth={"token": self.token},
|
|
||||||
wait_timeout=wait_timeout,
|
|
||||||
transports=["websocket", "polling"],
|
|
||||||
)
|
|
||||||
return True
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Failed to connect to DataStack: {e}")
|
|
||||||
return False
|
|
||||||
|
|
||||||
def sync(
|
|
||||||
self,
|
|
||||||
data: Any,
|
|
||||||
event_id: Optional[str] = None,
|
|
||||||
timeout: float = 5.0,
|
|
||||||
) -> Dict[str, Any]:
|
|
||||||
"""
|
|
||||||
Sync single event payload in realtime.
|
|
||||||
Blocks until the server ack callback fires or timeout expires.
|
|
||||||
"""
|
|
||||||
if not self.is_connected:
|
|
||||||
raise ConnectionError("DataStack client is not connected. Call connect() first.")
|
|
||||||
|
|
||||||
evt_id = event_id or f"evt_{uuid.uuid4().hex[:12]}"
|
|
||||||
iso_timestamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
||||||
|
|
||||||
payload = {
|
|
||||||
"eventId": evt_id,
|
|
||||||
"timestamp": iso_timestamp,
|
|
||||||
"data": data,
|
|
||||||
}
|
|
||||||
|
|
||||||
ack_event = threading.Event()
|
|
||||||
response: Dict[str, Any] = {
|
|
||||||
"success": False,
|
|
||||||
"eventId": evt_id,
|
|
||||||
"error": "Timeout waiting for server acknowledgment",
|
|
||||||
}
|
|
||||||
|
|
||||||
def on_ack(res):
|
|
||||||
nonlocal response
|
|
||||||
if isinstance(res, dict):
|
|
||||||
response = res
|
|
||||||
ack_event.set()
|
|
||||||
|
|
||||||
try:
|
|
||||||
self.sio.emit("sync_data", payload, callback=on_ack)
|
|
||||||
ack_event.wait(timeout=timeout)
|
|
||||||
return response
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Sync error for event {evt_id}: {e}")
|
|
||||||
return {"success": False, "eventId": evt_id, "error": str(e)}
|
|
||||||
|
|
||||||
def batch_sync(
|
|
||||||
self,
|
|
||||||
records: List[Dict[str, Any]],
|
|
||||||
timeout: float = 10.0,
|
|
||||||
) -> Dict[str, Any]:
|
|
||||||
"""
|
|
||||||
Sync list of records in a single batch.
|
|
||||||
Blocks until the server ack callback fires or timeout expires.
|
|
||||||
"""
|
|
||||||
if not self.is_connected:
|
|
||||||
raise ConnectionError("DataStack client is not connected.")
|
|
||||||
|
|
||||||
batch_payload = []
|
|
||||||
for r in records:
|
|
||||||
batch_payload.append({
|
|
||||||
"eventId": r.get("eventId") or f"evt_{uuid.uuid4().hex[:12]}",
|
|
||||||
"timestamp": r.get("timestamp") or time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
||||||
"data": r.get("data", r),
|
|
||||||
})
|
|
||||||
|
|
||||||
ack_event = threading.Event()
|
|
||||||
response: Dict[str, Any] = {
|
|
||||||
"success": False,
|
|
||||||
"processed": 0,
|
|
||||||
"error": "Timeout waiting for server batch acknowledgment",
|
|
||||||
}
|
|
||||||
|
|
||||||
def on_ack(res):
|
|
||||||
nonlocal response
|
|
||||||
if isinstance(res, dict):
|
|
||||||
response = res
|
|
||||||
ack_event.set()
|
|
||||||
|
|
||||||
try:
|
|
||||||
self.sio.emit("batch_sync", batch_payload, callback=on_ack)
|
|
||||||
ack_event.wait(timeout=timeout)
|
|
||||||
return response
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Batch sync error: {e}")
|
|
||||||
return {"success": False, "error": str(e)}
|
|
||||||
|
|
||||||
def disconnect(self):
|
|
||||||
"""Disconnects socket connection"""
|
|
||||||
if self.is_connected:
|
|
||||||
self.sio.disconnect()
|
|
||||||
@@ -0,0 +1,110 @@
|
|||||||
|
# DataStack Official Python SDK (`datastack`)
|
||||||
|
|
||||||
|
> **Production-Grade Python Client for DataStack** — featuring automatic `config.datastack.py` configuration discovery, real-time Socket.IO streaming (`Port 4000`), automatic Heartbeat HTTP fallback (`Port 3000`), offline disk spooling, and direct `.csv`, `.json`, and `.jsonl` file ingestion.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Installation
|
||||||
|
|
||||||
|
### Install from Local Repository
|
||||||
|
```bash
|
||||||
|
pip install ./sdk/python/datastack
|
||||||
|
```
|
||||||
|
|
||||||
|
### Editable Development Mode
|
||||||
|
```bash
|
||||||
|
pip install -e ./sdk/python/datastack
|
||||||
|
```
|
||||||
|
|
||||||
|
### PyPI Package Installation
|
||||||
|
```bash
|
||||||
|
pip install datastack
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Zero-Boilerplate Setup with `config.datastack.py`
|
||||||
|
|
||||||
|
Create a file named **`config.datastack.py`** in your Python project's root directory (you can also copy a pre-filled version directly from the DataStack Admin Console at **`/vcon/sdk`**):
|
||||||
|
|
||||||
|
```python
|
||||||
|
# config.datastack.py
|
||||||
|
DATASTACK_CONFIG = {
|
||||||
|
"SERVER_URL": "http://localhost:4000", # Port 4000 Socket.IO Ingestion Service
|
||||||
|
"API_URL": "http://localhost:3000", # Port 3000 HTTP Fallback & File Ingestion API
|
||||||
|
"CLIENT_TOKEN": "YOUR_CLIENT_JWT_TOKEN", # Provisioned in /vcon/clients
|
||||||
|
"AUTO_CONNECT": True,
|
||||||
|
"ENABLE_HTTP_FALLBACK": True,
|
||||||
|
"HEARTBEAT_INTERVAL_SEC": 5.0,
|
||||||
|
"MAX_RETRIES": 3,
|
||||||
|
"RETRY_BACKOFF_BASE_SEC": 0.5,
|
||||||
|
"BATCH_CHUNK_SIZE": 250,
|
||||||
|
"OFFLINE_SPOOL_ENABLED": True,
|
||||||
|
"OFFLINE_SPOOL_DIR": ".datastack_spool",
|
||||||
|
"MAX_SPOOL_SIZE_MB": 50,
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Quickstart Usage
|
||||||
|
|
||||||
|
```python
|
||||||
|
from datastack import DataStack
|
||||||
|
|
||||||
|
# Automatically discovers and loads config.datastack.py and connects!
|
||||||
|
client = DataStack()
|
||||||
|
|
||||||
|
# 1. Push a Single Event
|
||||||
|
res = client.send({
|
||||||
|
"service": "checkout-worker",
|
||||||
|
"order_id": "ORD-9921",
|
||||||
|
"amount_usd": 149.99,
|
||||||
|
"status": "completed",
|
||||||
|
})
|
||||||
|
print("Single Event:", res)
|
||||||
|
|
||||||
|
# 2. Push a Batch of Events
|
||||||
|
batch_res = client.send_batch([
|
||||||
|
{"data": {"metric": "cpu_load", "value": 0.42}},
|
||||||
|
{"data": {"metric": "mem_used_mb", "value": 2048}},
|
||||||
|
])
|
||||||
|
print("Batch Result:", batch_res)
|
||||||
|
|
||||||
|
# 3. Directly Stream .csv, .json, or .jsonl Files
|
||||||
|
file_res = client.send_file("monthly_records.csv", chunk_size=250)
|
||||||
|
print("File Ingestion:", file_res)
|
||||||
|
|
||||||
|
# 4. Inspect Connection & Heartbeat Status
|
||||||
|
print("SDK Status:", client.status())
|
||||||
|
|
||||||
|
client.disconnect()
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Smart Connection State Machine & Auto-Recovery
|
||||||
|
|
||||||
|
The DataStack Python SDK maintains a 4-state resilience lifecycle managed by `HeartbeatMonitor`:
|
||||||
|
|
||||||
|
1. **`CONNECTED_IDLE` (Primary WebSocket Active — Heartbeat IDLE)**:
|
||||||
|
- Events stream over Socket.IO (`Port 4000`) with sub-millisecond overhead.
|
||||||
|
- The Heartbeat Monitor sleeps in **IDLE mode** (`heartbeatMode: "IDLE"`), consuming zero network or CPU resources.
|
||||||
|
2. **`FALLBACK_HTTP` (WebSocket Disconnected — Auto HTTP Fallback Active)**:
|
||||||
|
- If the WebSocket connection drops or closes, `send()`, `send_batch()`, and `send_file()` automatically switch to `POST {API_URL}/api/events` so zero application calls fail.
|
||||||
|
- Simultaneously, the Heartbeat Monitor wakes up (`heartbeatMode: "ACTIVE_RECOVERY"`), probes and reconnects the WebSocket in the background, verifies the handshake (`ping_verify`), and transitions back to **`CONNECTED_IDLE` (`IDLE` mode)**.
|
||||||
|
3. **`OFFLINE_SPOOL` (Device Offline — Local Disk Spooling)**:
|
||||||
|
- If the client machine loses internet connectivity completely, events are safely appended to `.datastack_spool/spool.jsonl` with deterministic `eventId`s.
|
||||||
|
- When the device comes back online, the Heartbeat Monitor detects connectivity, reconnects the WebSocket, replays all spooled events without duplicates, and returns to **`CONNECTED_IDLE` (`IDLE` mode)**.
|
||||||
|
4. **Structured Error Handling & Auto-Chunking**:
|
||||||
|
- **`413 PAYLOAD_TOO_LARGE`**: Batch/file payloads automatically split in half (`chunk // 2`) recursively and retry.
|
||||||
|
- **`429 RATE_LIMIT_EXCEEDED`**: Automatically retries with exponential backoff and random jitter.
|
||||||
|
- **`401 / 403` (`CLIENT_REVOKED`, `TOKEN_ROTATED`)**: Immediately halts retries and raises `DataStackAuthError`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Author & License
|
||||||
|
|
||||||
|
- **Organization**: [Vib Tools](https://vibtools.top/)
|
||||||
|
- **Author**: Md Anik (`ANIK.D.ART`) — `info@vibtools.top`
|
||||||
|
- **License**: MIT License
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
"""
|
||||||
|
config.datastack.py — Official DataStack Python SDK Configuration File
|
||||||
|
======================================================================
|
||||||
|
Place a copy of this file named `config.datastack.py` in the root directory
|
||||||
|
of your Python application. When you initialize `DataStack()`, the SDK will
|
||||||
|
automatically discover and load these settings.
|
||||||
|
"""
|
||||||
|
|
||||||
|
DATASTACK_CONFIG = {
|
||||||
|
# Port 4000 Dedicated Socket.IO Ingestion Service URL
|
||||||
|
"SERVER_URL": "http://localhost:4000",
|
||||||
|
|
||||||
|
# Port 3000 Next.js Web & HTTP Fallback Ingestion API URL
|
||||||
|
"API_URL": "http://localhost:3000",
|
||||||
|
|
||||||
|
# Client JWT Token generated from the DataStack Admin Console (/vcon/clients)
|
||||||
|
"CLIENT_TOKEN": "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN",
|
||||||
|
|
||||||
|
# Optional explicit client identifier override (defaults to JWT sub/clientId)
|
||||||
|
"CLIENT_ID": "",
|
||||||
|
|
||||||
|
# Automatically connect on DataStack() instantiation
|
||||||
|
"AUTO_CONNECT": True,
|
||||||
|
|
||||||
|
# Automatically switch to HTTP POST /api/events when WebSocket disconnects
|
||||||
|
"ENABLE_HTTP_FALLBACK": True,
|
||||||
|
|
||||||
|
# Heartbeat interval (in seconds) when probing to repair WebSocket or recover from offline state
|
||||||
|
"HEARTBEAT_INTERVAL_SEC": 5.0,
|
||||||
|
|
||||||
|
# Socket & HTTP request timeout in seconds
|
||||||
|
"REQUEST_TIMEOUT_SEC": 8.0,
|
||||||
|
|
||||||
|
# Maximum retry attempts on transient network errors or 429 RATE_LIMIT_EXCEEDED
|
||||||
|
"MAX_RETRIES": 3,
|
||||||
|
|
||||||
|
# Base delay (in seconds) for exponential backoff with jitter
|
||||||
|
"RETRY_BACKOFF_BASE_SEC": 0.5,
|
||||||
|
|
||||||
|
# Default records per batch when streaming .csv / .json / .jsonl files via send_file()
|
||||||
|
"BATCH_CHUNK_SIZE": 250,
|
||||||
|
|
||||||
|
# Offline Disk Spooling: safely persist events to disk when device is completely offline
|
||||||
|
"OFFLINE_SPOOL_ENABLED": True,
|
||||||
|
|
||||||
|
# Local directory path for offline event spooling
|
||||||
|
"OFFLINE_SPOOL_DIR": ".datastack_spool",
|
||||||
|
|
||||||
|
# Maximum disk spool file size in megabytes (MB)
|
||||||
|
"MAX_SPOOL_SIZE_MB": 50,
|
||||||
|
}
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
"""
|
||||||
|
DataStack Official Python SDK (`datastack`).
|
||||||
|
Supports automatic `config.datastack.py` configuration discovery, real-time
|
||||||
|
Socket.IO ingestion (Port 4000), automatic Heartbeat HTTP fallback (Port 3000),
|
||||||
|
offline disk spooling, and direct `.csv` / `.json` / `.jsonl` file streaming.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from .config import DataStackConfig, load_config
|
||||||
|
from .exceptions import (
|
||||||
|
DataStackAuthError,
|
||||||
|
DataStackConfigError,
|
||||||
|
DataStackConnectionError,
|
||||||
|
DataStackError,
|
||||||
|
DataStackFileError,
|
||||||
|
DataStackOfflineSpoolError,
|
||||||
|
DataStackPayloadTooLargeError,
|
||||||
|
DataStackRateLimitError,
|
||||||
|
)
|
||||||
|
from .spool import DiskSpooler
|
||||||
|
|
||||||
|
try:
|
||||||
|
from .client import DataStack, DataStackClient
|
||||||
|
except ImportError:
|
||||||
|
DataStack = None # type: ignore
|
||||||
|
DataStackClient = None # type: ignore
|
||||||
|
|
||||||
|
__version__ = "2.2.0"
|
||||||
|
|
||||||
|
__all__ = [
|
||||||
|
"DataStack",
|
||||||
|
"DataStackClient",
|
||||||
|
"DataStackConfig",
|
||||||
|
"load_config",
|
||||||
|
"DiskSpooler",
|
||||||
|
"DataStackError",
|
||||||
|
"DataStackConfigError",
|
||||||
|
"DataStackAuthError",
|
||||||
|
"DataStackConnectionError",
|
||||||
|
"DataStackRateLimitError",
|
||||||
|
"DataStackPayloadTooLargeError",
|
||||||
|
"DataStackFileError",
|
||||||
|
"DataStackOfflineSpoolError",
|
||||||
|
]
|
||||||
@@ -0,0 +1,846 @@
|
|||||||
|
"""
|
||||||
|
Official DataStack Python SDK Client (`DataStack` / `DataStackClient`).
|
||||||
|
|
||||||
|
Features:
|
||||||
|
- Automatic discovery of `config.datastack.py` in the client app's source/working directory.
|
||||||
|
- Primary real-time Socket.IO telemetry streaming (`sync_data`, `batch_sync`, `ping_verify`).
|
||||||
|
- Automatic Heartbeat HTTP Fallback (`POST /api/events`) when WebSocket closes or disconnects.
|
||||||
|
- Automatic WebSocket reconnection, handshake verification (`ping_verify`), and return to IDLE mode.
|
||||||
|
- Offline Disk Spooling (`.datastack_spool/spool.jsonl`) when the client device loses internet,
|
||||||
|
with automatic replay when connectivity returns.
|
||||||
|
- Direct `.csv`, `.json`, and `.jsonl` file ingestion (`send_file()`) with automatic batch chunking
|
||||||
|
(`413 PAYLOAD_TOO_LARGE` auto-splitting) and exponential backoff retries (`429 RATE_LIMIT_EXCEEDED`).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import csv
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import random
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, Iterable, List, Optional, Union
|
||||||
|
|
||||||
|
try:
|
||||||
|
import socketio # type: ignore
|
||||||
|
except ImportError:
|
||||||
|
socketio = None # type: ignore
|
||||||
|
|
||||||
|
try:
|
||||||
|
import requests # type: ignore
|
||||||
|
except ImportError:
|
||||||
|
requests = None # type: ignore
|
||||||
|
|
||||||
|
from .config import DataStackConfig, load_config
|
||||||
|
from .exceptions import (
|
||||||
|
DataStackAuthError,
|
||||||
|
DataStackConnectionError,
|
||||||
|
DataStackError,
|
||||||
|
DataStackFileError,
|
||||||
|
DataStackPayloadTooLargeError,
|
||||||
|
DataStackRateLimitError,
|
||||||
|
)
|
||||||
|
from .heartbeat import ConnectionState, HeartbeatMonitor
|
||||||
|
from .spool import DiskSpooler
|
||||||
|
|
||||||
|
logger = logging.getLogger("datastack.client")
|
||||||
|
|
||||||
|
|
||||||
|
def _utc_now_iso() -> str:
|
||||||
|
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||||
|
|
||||||
|
|
||||||
|
def _make_event_id() -> str:
|
||||||
|
return f"evt_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}"
|
||||||
|
|
||||||
|
|
||||||
|
class DataStackClient:
|
||||||
|
"""
|
||||||
|
Production-grade DataStack Python Client with `config.datastack.py` auto-loading,
|
||||||
|
WebSocket + Heartbeat HTTP Fallback + Offline Spooling + Direct File Streaming.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
server_url: Optional[str] = None,
|
||||||
|
token: Optional[str] = None,
|
||||||
|
api_url: Optional[str] = None,
|
||||||
|
config_path: Optional[str] = None,
|
||||||
|
auto_connect: Optional[bool] = None,
|
||||||
|
**kwargs: Any,
|
||||||
|
) -> None:
|
||||||
|
self.config: DataStackConfig = load_config(
|
||||||
|
config_path=config_path,
|
||||||
|
require_token=False,
|
||||||
|
SERVER_URL=server_url,
|
||||||
|
CLIENT_TOKEN=token,
|
||||||
|
API_URL=api_url,
|
||||||
|
AUTO_CONNECT=auto_connect,
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.server_url = self.config.server_url
|
||||||
|
self.api_url = self.config.api_url
|
||||||
|
self.token = self.config.client_token
|
||||||
|
self.is_connected = False
|
||||||
|
self._lock = threading.RLock()
|
||||||
|
|
||||||
|
# Offline Disk Spooler
|
||||||
|
self.spooler = DiskSpooler(
|
||||||
|
spool_dir=self.config.offline_spool_dir,
|
||||||
|
max_size_mb=self.config.max_spool_size_mb,
|
||||||
|
enabled=self.config.offline_spool_enabled,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Heartbeat Monitor (starts in IDLE once connected, wakes on disconnect/offline)
|
||||||
|
self.heartbeat = HeartbeatMonitor(
|
||||||
|
interval_sec=self.config.heartbeat_interval_sec,
|
||||||
|
recovery_callback=self._heartbeat_recovery_step,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Socket.IO Client (disable built-in infinite loop so HeartbeatMonitor coordinates fallback + IDLE)
|
||||||
|
self.sio = (
|
||||||
|
socketio.Client(
|
||||||
|
reconnection=False,
|
||||||
|
logger=False,
|
||||||
|
engineio_logger=False,
|
||||||
|
)
|
||||||
|
if socketio is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
if self.sio is not None:
|
||||||
|
self._register_socket_events()
|
||||||
|
|
||||||
|
if self.config.auto_connect and self.token and (
|
||||||
|
self.token != "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN"
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
self.connect()
|
||||||
|
except DataStackAuthError:
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning(
|
||||||
|
"[DataStack] Initial connect entered fallback/recovery mode: %s",
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _register_socket_events(self) -> None:
|
||||||
|
if self.sio is None:
|
||||||
|
return
|
||||||
|
|
||||||
|
@self.sio.event
|
||||||
|
def connect() -> None:
|
||||||
|
with self._lock:
|
||||||
|
self.is_connected = True
|
||||||
|
logger.info("[DataStack] WebSocket connected to %s", self.server_url)
|
||||||
|
|
||||||
|
@self.sio.event
|
||||||
|
def connect_error(data: Any) -> None:
|
||||||
|
with self._lock:
|
||||||
|
self.is_connected = False
|
||||||
|
msg = str(data)
|
||||||
|
if self._is_non_retryable_auth_error(msg):
|
||||||
|
self.heartbeat.stop(reason=msg)
|
||||||
|
else:
|
||||||
|
self.heartbeat.trigger_fallback(reason=f"Socket connect_error: {msg}")
|
||||||
|
|
||||||
|
@self.sio.event
|
||||||
|
def disconnect() -> None:
|
||||||
|
with self._lock:
|
||||||
|
was_connected = self.is_connected
|
||||||
|
self.is_connected = False
|
||||||
|
if was_connected and self.heartbeat.state != ConnectionState.STOPPED:
|
||||||
|
logger.warning(
|
||||||
|
"[DataStack] WebSocket disconnected. Activating Heartbeat HTTP Fallback."
|
||||||
|
)
|
||||||
|
self.heartbeat.trigger_fallback(reason="WebSocket connection closed")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _is_non_retryable_auth_error(message: str) -> bool:
|
||||||
|
upper = (message or "").upper()
|
||||||
|
return any(
|
||||||
|
code in upper
|
||||||
|
for code in (
|
||||||
|
"AUTH_REQUIRED",
|
||||||
|
"AUTH_INVALID",
|
||||||
|
"CLIENT_REVOKED",
|
||||||
|
"TOKEN_ROTATED",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def _ensure_token_present(self) -> None:
|
||||||
|
if not self.token or self.token == "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN":
|
||||||
|
from .exceptions import DataStackConfigError
|
||||||
|
|
||||||
|
raise DataStackConfigError(
|
||||||
|
"Missing CLIENT_TOKEN. Set CLIENT_TOKEN in `config.datastack.py` or pass token='...'."
|
||||||
|
)
|
||||||
|
|
||||||
|
def connect(self) -> bool:
|
||||||
|
"""
|
||||||
|
Establishes the primary WebSocket connection on Port 4000 and verifies the
|
||||||
|
handshake via `ping_verify`. If WebSocket is unavailable and `ENABLE_HTTP_FALLBACK`
|
||||||
|
is True, automatically enters `FALLBACK_HTTP` (or `OFFLINE_SPOOL` if offline)
|
||||||
|
and starts the background Heartbeat Monitor.
|
||||||
|
"""
|
||||||
|
self._ensure_token_present()
|
||||||
|
self.heartbeat.start()
|
||||||
|
|
||||||
|
try:
|
||||||
|
if self._connect_and_verify_websocket():
|
||||||
|
self._drain_offline_spool()
|
||||||
|
self.heartbeat.set_idle_connected()
|
||||||
|
return True
|
||||||
|
except DataStackAuthError as auth_err:
|
||||||
|
self.heartbeat.stop(reason=str(auth_err))
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("[DataStack] WebSocket connect failed: %s", exc)
|
||||||
|
|
||||||
|
# WebSocket was not reachable; check if HTTP fallback API is reachable
|
||||||
|
if self.config.enable_http_fallback:
|
||||||
|
if self._check_http_health():
|
||||||
|
self.heartbeat.trigger_fallback(
|
||||||
|
reason="WebSocket unreachable on connect; using HTTP fallback"
|
||||||
|
)
|
||||||
|
self._drain_offline_spool()
|
||||||
|
return True
|
||||||
|
|
||||||
|
if self.config.offline_spool_enabled:
|
||||||
|
self.heartbeat.trigger_offline(
|
||||||
|
reason="Both WebSocket and HTTP endpoints unreachable on connect"
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
|
raise DataStackConnectionError(
|
||||||
|
f"Unable to connect to DataStack WebSocket ({self.server_url}) or HTTP API ({self.api_url})."
|
||||||
|
)
|
||||||
|
|
||||||
|
def disconnect(self) -> None:
|
||||||
|
"""Gracefully stops the Heartbeat Monitor and disconnects the WebSocket."""
|
||||||
|
self.heartbeat.stop(reason="Client closed by application")
|
||||||
|
with self._lock:
|
||||||
|
self.is_connected = False
|
||||||
|
if self.sio is not None and self.sio.connected:
|
||||||
|
try:
|
||||||
|
self.sio.disconnect()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _connect_and_verify_websocket(self) -> bool:
|
||||||
|
if self.sio is None:
|
||||||
|
return False
|
||||||
|
|
||||||
|
with self._lock:
|
||||||
|
if self.sio.connected:
|
||||||
|
try:
|
||||||
|
self.sio.disconnect()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
self.is_connected = False
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.sio.connect(
|
||||||
|
self.server_url,
|
||||||
|
auth={"token": self.token},
|
||||||
|
headers={"Authorization": f"Bearer {self.token}"},
|
||||||
|
transports=["websocket", "polling"],
|
||||||
|
wait_timeout=self.config.request_timeout_sec,
|
||||||
|
)
|
||||||
|
self.is_connected = True
|
||||||
|
except Exception as exc:
|
||||||
|
self.is_connected = False
|
||||||
|
msg = str(exc)
|
||||||
|
if self._is_non_retryable_auth_error(msg):
|
||||||
|
raise DataStackAuthError(msg) from exc
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Verify handshake & PostgreSQL client status via `ping_verify`
|
||||||
|
return self._verify_websocket_handshake()
|
||||||
|
|
||||||
|
def _verify_websocket_handshake(self) -> bool:
|
||||||
|
if self.sio is None or not self.sio.connected:
|
||||||
|
return False
|
||||||
|
|
||||||
|
ack_event = threading.Event()
|
||||||
|
result_holder: Dict[str, Any] = {}
|
||||||
|
|
||||||
|
def on_verify(resp: Dict[str, Any]) -> None:
|
||||||
|
if isinstance(resp, dict):
|
||||||
|
result_holder.update(resp)
|
||||||
|
ack_event.set()
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.sio.emit("ping_verify", {}, callback=on_verify)
|
||||||
|
if not ack_event.wait(timeout=self.config.request_timeout_sec):
|
||||||
|
return False
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
if not result_holder.get("success"):
|
||||||
|
err_msg = str(result_holder.get("error") or "Handshake verification failed")
|
||||||
|
if self._is_non_retryable_auth_error(err_msg):
|
||||||
|
raise DataStackAuthError(err_msg)
|
||||||
|
return False
|
||||||
|
|
||||||
|
return result_holder.get("status") == "VERIFIED"
|
||||||
|
|
||||||
|
def _heartbeat_recovery_step(self) -> bool:
|
||||||
|
"""
|
||||||
|
Invoked periodically by `HeartbeatMonitor` ONLY when not in `CONNECTED_IDLE` mode.
|
||||||
|
1. Attempts to reconnect and verify (`ping_verify`) the primary WebSocket.
|
||||||
|
2. If WebSocket is verified, drains any offline spooled events and returns True
|
||||||
|
so `HeartbeatMonitor` transitions back to `CONNECTED_IDLE` (IDLE mode).
|
||||||
|
3. If WebSocket is still down, checks if HTTP fallback (`Port 3000`) is online:
|
||||||
|
- If HTTP is online, drains offline spooled events over HTTP and remains in `FALLBACK_HTTP`.
|
||||||
|
- If HTTP is also unreachable, sets state to `OFFLINE_SPOOL`.
|
||||||
|
"""
|
||||||
|
self.heartbeat.set_reconnecting()
|
||||||
|
try:
|
||||||
|
if self._connect_and_verify_websocket():
|
||||||
|
self._drain_offline_spool()
|
||||||
|
return True
|
||||||
|
except DataStackAuthError as auth_err:
|
||||||
|
self.heartbeat.stop(reason=str(auth_err))
|
||||||
|
return False
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# WebSocket still unavailable; check HTTP availability
|
||||||
|
if self.config.enable_http_fallback and self._check_http_health():
|
||||||
|
self.heartbeat.trigger_fallback(
|
||||||
|
reason="WebSocket still unavailable; HTTP fallback active"
|
||||||
|
)
|
||||||
|
self._drain_offline_spool()
|
||||||
|
return False
|
||||||
|
|
||||||
|
self.heartbeat.trigger_offline(
|
||||||
|
reason="Device offline: WebSocket and HTTP endpoints unreachable"
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _check_http_health(self) -> bool:
|
||||||
|
url = f"{self.api_url}/api/health"
|
||||||
|
try:
|
||||||
|
status_code, _ = self._http_request("GET", url, payload=None, timeout=3.0)
|
||||||
|
return 200 <= status_code < 500
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _http_request(
|
||||||
|
self,
|
||||||
|
method: str,
|
||||||
|
url: str,
|
||||||
|
payload: Optional[Dict[str, Any]] = None,
|
||||||
|
timeout: Optional[float] = None,
|
||||||
|
) -> tuple[int, Dict[str, Any]]:
|
||||||
|
req_timeout = timeout or self.config.request_timeout_sec
|
||||||
|
headers = {
|
||||||
|
"Authorization": f"Bearer {self.token}",
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
"Accept": "application/json",
|
||||||
|
}
|
||||||
|
|
||||||
|
if requests is not None:
|
||||||
|
resp = requests.request(
|
||||||
|
method=method,
|
||||||
|
url=url,
|
||||||
|
json=payload,
|
||||||
|
headers=headers,
|
||||||
|
timeout=req_timeout,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
body = resp.json() if resp.text else {}
|
||||||
|
except Exception:
|
||||||
|
body = {"error": resp.text}
|
||||||
|
return resp.status_code, body if isinstance(body, dict) else {"data": body}
|
||||||
|
|
||||||
|
# Standard library urllib fallback when `requests` is not installed
|
||||||
|
data_bytes = (
|
||||||
|
json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
||||||
|
if payload is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
req = urllib.request.Request(
|
||||||
|
url=url, data=data_bytes, headers=headers, method=method
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=req_timeout) as response:
|
||||||
|
raw = response.read().decode("utf-8")
|
||||||
|
parsed = json.loads(raw) if raw else {}
|
||||||
|
return (
|
||||||
|
response.status,
|
||||||
|
parsed if isinstance(parsed, dict) else {"data": parsed},
|
||||||
|
)
|
||||||
|
except urllib.error.HTTPError as http_err:
|
||||||
|
raw_err = http_err.read().decode("utf-8", errors="replace")
|
||||||
|
try:
|
||||||
|
parsed_err = json.loads(raw_err) if raw_err else {}
|
||||||
|
except Exception:
|
||||||
|
parsed_err = {"error": raw_err or str(http_err)}
|
||||||
|
return (
|
||||||
|
http_err.code,
|
||||||
|
parsed_err if isinstance(parsed_err, dict) else {"error": str(parsed_err)},
|
||||||
|
)
|
||||||
|
|
||||||
|
def _send_via_http_fallback(
|
||||||
|
self, payload: Dict[str, Any]
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Sends a single event, batch array (`{ batch: [...] }`), or file records
|
||||||
|
(`{ fileName, records: [...] }`) over HTTP `POST {API_URL}/api/events`.
|
||||||
|
"""
|
||||||
|
url = f"{self.api_url}/api/events"
|
||||||
|
status_code, body = self._http_request("POST", url, payload=payload)
|
||||||
|
|
||||||
|
if 200 <= status_code < 300 and body.get("success", True):
|
||||||
|
return {
|
||||||
|
**body,
|
||||||
|
"success": True,
|
||||||
|
"transport": "HTTP_FALLBACK",
|
||||||
|
}
|
||||||
|
|
||||||
|
err_msg = str(body.get("error") or f"HTTP {status_code}")
|
||||||
|
err_code = str(body.get("code") or "")
|
||||||
|
|
||||||
|
if status_code in (401, 403) or self._is_non_retryable_auth_error(
|
||||||
|
f"{err_code} {err_msg}"
|
||||||
|
):
|
||||||
|
self.heartbeat.stop(reason=err_msg)
|
||||||
|
raise DataStackAuthError(err_msg, code=err_code or "AUTH_ERROR")
|
||||||
|
|
||||||
|
if status_code == 413 or "PAYLOAD_TOO_LARGE" in f"{err_code} {err_msg}":
|
||||||
|
raise DataStackPayloadTooLargeError(err_msg)
|
||||||
|
|
||||||
|
if status_code == 429 or "RATE_LIMIT_EXCEEDED" in f"{err_code} {err_msg}":
|
||||||
|
raise DataStackRateLimitError(err_msg)
|
||||||
|
|
||||||
|
raise DataStackConnectionError(f"HTTP fallback error ({status_code}): {err_msg}")
|
||||||
|
|
||||||
|
def _emit_via_websocket(
|
||||||
|
self,
|
||||||
|
event_name: str,
|
||||||
|
payload: Any,
|
||||||
|
timeout: Optional[float] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
if self.sio is None or not self.is_connected or not self.sio.connected:
|
||||||
|
raise DataStackConnectionError("WebSocket is not currently connected")
|
||||||
|
|
||||||
|
ack_event = threading.Event()
|
||||||
|
response_data: Dict[str, Any] = {}
|
||||||
|
|
||||||
|
def ack_callback(res: Dict[str, Any]) -> None:
|
||||||
|
if isinstance(res, dict):
|
||||||
|
response_data.update(res)
|
||||||
|
ack_event.set()
|
||||||
|
|
||||||
|
try:
|
||||||
|
self.sio.emit(event_name, payload, callback=ack_callback)
|
||||||
|
except Exception as exc:
|
||||||
|
self.is_connected = False
|
||||||
|
self.heartbeat.trigger_fallback(reason=f"WebSocket emit error: {exc}")
|
||||||
|
raise DataStackConnectionError(f"WebSocket emit failed: {exc}") from exc
|
||||||
|
|
||||||
|
wait_sec = timeout or self.config.request_timeout_sec
|
||||||
|
if not ack_event.wait(timeout=wait_sec):
|
||||||
|
self.is_connected = False
|
||||||
|
self.heartbeat.trigger_fallback(
|
||||||
|
reason=f"WebSocket ACK timeout after {wait_sec}s"
|
||||||
|
)
|
||||||
|
raise DataStackConnectionError(
|
||||||
|
f"WebSocket ACK timed out after {wait_sec}s"
|
||||||
|
)
|
||||||
|
|
||||||
|
if not response_data.get("success", False):
|
||||||
|
err_msg = str(response_data.get("error") or "WebSocket event rejected")
|
||||||
|
if self._is_non_retryable_auth_error(err_msg):
|
||||||
|
self.heartbeat.stop(reason=err_msg)
|
||||||
|
raise DataStackAuthError(err_msg)
|
||||||
|
if "PAYLOAD_TOO_LARGE" in err_msg:
|
||||||
|
raise DataStackPayloadTooLargeError(err_msg)
|
||||||
|
if "RATE_LIMIT_EXCEEDED" in err_msg:
|
||||||
|
raise DataStackRateLimitError(err_msg)
|
||||||
|
raise DataStackError(err_msg)
|
||||||
|
|
||||||
|
return {
|
||||||
|
**response_data,
|
||||||
|
"transport": "WEBSOCKET",
|
||||||
|
}
|
||||||
|
|
||||||
|
def _execute_with_retry_and_fallback(
|
||||||
|
self,
|
||||||
|
ws_event: str,
|
||||||
|
ws_payload: Any,
|
||||||
|
http_payload: Dict[str, Any],
|
||||||
|
spool_items: List[Dict[str, Any]],
|
||||||
|
timeout: Optional[float] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Core resilient dispatcher:
|
||||||
|
1. Attempts WebSocket if connected (`CONNECTED_IDLE`).
|
||||||
|
2. On WebSocket failure/disconnect, triggers `HeartbeatMonitor` (`FALLBACK_HTTP`)
|
||||||
|
and automatically sends via HTTP `POST /api/events`.
|
||||||
|
3. Applies exponential backoff with jitter on `RATE_LIMIT_EXCEEDED` or transient errors.
|
||||||
|
4. If both WebSocket and HTTP are unreachable (device offline), spools events to
|
||||||
|
`.datastack_spool/spool.jsonl` and triggers `OFFLINE_SPOOL`.
|
||||||
|
"""
|
||||||
|
self._ensure_token_present()
|
||||||
|
max_attempts = max(1, self.config.max_retries + 1)
|
||||||
|
last_exception: Optional[Exception] = None
|
||||||
|
|
||||||
|
for attempt in range(max_attempts):
|
||||||
|
if attempt > 0:
|
||||||
|
delay = self.config.retry_backoff_base_sec * (2 ** (attempt - 1))
|
||||||
|
jitter = random.uniform(0.0, 0.25 * delay)
|
||||||
|
time.sleep(delay + jitter)
|
||||||
|
|
||||||
|
# 1. Try Primary WebSocket if connected
|
||||||
|
if self.is_connected and self.sio is not None and self.sio.connected:
|
||||||
|
try:
|
||||||
|
return self._emit_via_websocket(
|
||||||
|
ws_event, ws_payload, timeout=timeout
|
||||||
|
)
|
||||||
|
except (DataStackAuthError, DataStackPayloadTooLargeError):
|
||||||
|
raise
|
||||||
|
except DataStackRateLimitError as rl_err:
|
||||||
|
last_exception = rl_err
|
||||||
|
continue
|
||||||
|
except Exception as ws_err:
|
||||||
|
last_exception = ws_err
|
||||||
|
self.heartbeat.trigger_fallback(reason=str(ws_err))
|
||||||
|
|
||||||
|
# 2. Try HTTP Fallback (`POST /api/events`)
|
||||||
|
if self.config.enable_http_fallback:
|
||||||
|
try:
|
||||||
|
if self.heartbeat.state == ConnectionState.CONNECTED_IDLE:
|
||||||
|
self.heartbeat.trigger_fallback(
|
||||||
|
reason="WebSocket not connected; routing via HTTP fallback"
|
||||||
|
)
|
||||||
|
return self._send_via_http_fallback(http_payload)
|
||||||
|
except (DataStackAuthError, DataStackPayloadTooLargeError):
|
||||||
|
raise
|
||||||
|
except DataStackRateLimitError as rl_err:
|
||||||
|
last_exception = rl_err
|
||||||
|
continue
|
||||||
|
except Exception as http_err:
|
||||||
|
last_exception = http_err
|
||||||
|
|
||||||
|
# 3. If rate-limited or payload-rejected after all retries, raise directly
|
||||||
|
if isinstance(
|
||||||
|
last_exception, (DataStackRateLimitError, DataStackPayloadTooLargeError)
|
||||||
|
):
|
||||||
|
raise last_exception
|
||||||
|
|
||||||
|
# 4. Device is offline (both WebSocket and HTTP failed): Spool to local disk
|
||||||
|
if self.config.offline_spool_enabled and spool_items:
|
||||||
|
spooled_count = self.spooler.enqueue(spool_items)
|
||||||
|
self.heartbeat.trigger_offline(
|
||||||
|
reason=str(last_exception or "Network unreachable")
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"success": True,
|
||||||
|
"spooled": True,
|
||||||
|
"spooledCount": spooled_count,
|
||||||
|
"pendingSpoolTotal": self.spooler.pending_count(),
|
||||||
|
"transport": "OFFLINE_SPOOL",
|
||||||
|
}
|
||||||
|
|
||||||
|
raise DataStackConnectionError(
|
||||||
|
f"DataStack delivery failed after {max_attempts} attempts: {last_exception}"
|
||||||
|
)
|
||||||
|
|
||||||
|
def _drain_offline_spool(self) -> int:
|
||||||
|
"""
|
||||||
|
Replays any offline spooled events from `.datastack_spool/spool.jsonl`
|
||||||
|
once connectivity (WebSocket or HTTP fallback) is restored.
|
||||||
|
"""
|
||||||
|
if not self.config.offline_spool_enabled:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
total_flushed = 0
|
||||||
|
while True:
|
||||||
|
batch = self.spooler.peek_batch(limit=self.config.batch_chunk_size)
|
||||||
|
if not batch:
|
||||||
|
break
|
||||||
|
|
||||||
|
try:
|
||||||
|
if self.is_connected and self.sio is not None and self.sio.connected:
|
||||||
|
self._emit_via_websocket("batch_sync", batch)
|
||||||
|
elif self.config.enable_http_fallback:
|
||||||
|
self._send_via_http_fallback({"batch": batch})
|
||||||
|
else:
|
||||||
|
break
|
||||||
|
|
||||||
|
self.spooler.ack_batch(len(batch))
|
||||||
|
total_flushed += len(batch)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.debug("[DataStack] Spool drain paused: %s", exc)
|
||||||
|
break
|
||||||
|
|
||||||
|
return total_flushed
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# Public Telemetry Ingestion API
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
def send(
|
||||||
|
self,
|
||||||
|
data: Dict[str, Any],
|
||||||
|
event_id: Optional[str] = None,
|
||||||
|
timestamp: Optional[str] = None,
|
||||||
|
timeout: Optional[float] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Sends a single telemetry event. Automatically falls back to HTTP if WebSocket
|
||||||
|
is disconnected, or spools to disk if the device is offline.
|
||||||
|
"""
|
||||||
|
resolved_id = (event_id or "").strip() or _make_event_id()
|
||||||
|
resolved_ts = (timestamp or "").strip() or _utc_now_iso()
|
||||||
|
envelope = {
|
||||||
|
"eventId": resolved_id,
|
||||||
|
"timestamp": resolved_ts,
|
||||||
|
"data": data,
|
||||||
|
}
|
||||||
|
return self._execute_with_retry_and_fallback(
|
||||||
|
ws_event="sync_data",
|
||||||
|
ws_payload=envelope,
|
||||||
|
http_payload=envelope,
|
||||||
|
spool_items=[envelope],
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
|
||||||
|
def sync(
|
||||||
|
self,
|
||||||
|
data: Dict[str, Any],
|
||||||
|
event_id: Optional[str] = None,
|
||||||
|
timeout: float = 5.0,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Alias for `send()` preserving backward compatibility."""
|
||||||
|
return self.send(data=data, event_id=event_id, timeout=timeout)
|
||||||
|
|
||||||
|
def send_batch(
|
||||||
|
self,
|
||||||
|
events: List[Dict[str, Any]],
|
||||||
|
timeout: Optional[float] = None,
|
||||||
|
file_name: Optional[str] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Sends a batch of events. If the batch triggers `413 PAYLOAD_TOO_LARGE`,
|
||||||
|
automatically splits the batch into smaller halves and retries.
|
||||||
|
"""
|
||||||
|
if not isinstance(events, list) or len(events) == 0:
|
||||||
|
return {"success": True, "processed": 0, "rejected": 0}
|
||||||
|
|
||||||
|
formatted: List[Dict[str, Any]] = []
|
||||||
|
for item in events:
|
||||||
|
safe_item = item if isinstance(item, dict) else {"value": item}
|
||||||
|
evt_id = str(safe_item.get("eventId") or "").strip() or _make_event_id()
|
||||||
|
evt_ts = str(safe_item.get("timestamp") or "").strip() or _utc_now_iso()
|
||||||
|
evt_data = safe_item.get("data", safe_item)
|
||||||
|
formatted.append(
|
||||||
|
{
|
||||||
|
"eventId": evt_id,
|
||||||
|
"timestamp": evt_ts,
|
||||||
|
"data": evt_data,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
return self._send_formatted_batch_recursive(
|
||||||
|
formatted, timeout=timeout, file_name=file_name
|
||||||
|
)
|
||||||
|
|
||||||
|
def batch_sync(
|
||||||
|
self,
|
||||||
|
events: List[Dict[str, Any]],
|
||||||
|
timeout: float = 10.0,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""Alias for `send_batch()` preserving backward compatibility."""
|
||||||
|
return self.send_batch(events=events, timeout=timeout)
|
||||||
|
|
||||||
|
def _send_formatted_batch_recursive(
|
||||||
|
self,
|
||||||
|
formatted_batch: List[Dict[str, Any]],
|
||||||
|
timeout: Optional[float] = None,
|
||||||
|
file_name: Optional[str] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
http_payload: Dict[str, Any] = (
|
||||||
|
{"fileName": file_name, "records": formatted_batch}
|
||||||
|
if file_name
|
||||||
|
else {"batch": formatted_batch}
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
return self._execute_with_retry_and_fallback(
|
||||||
|
ws_event="batch_sync",
|
||||||
|
ws_payload=formatted_batch,
|
||||||
|
http_payload=http_payload,
|
||||||
|
spool_items=formatted_batch,
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
except DataStackPayloadTooLargeError:
|
||||||
|
if len(formatted_batch) <= 1:
|
||||||
|
raise
|
||||||
|
mid = len(formatted_batch) // 2
|
||||||
|
left_res = self._send_formatted_batch_recursive(
|
||||||
|
formatted_batch[:mid], timeout=timeout, file_name=file_name
|
||||||
|
)
|
||||||
|
right_res = self._send_formatted_batch_recursive(
|
||||||
|
formatted_batch[mid:], timeout=timeout, file_name=file_name
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"success": bool(
|
||||||
|
left_res.get("success", True) and right_res.get("success", True)
|
||||||
|
),
|
||||||
|
"processed": int(left_res.get("processed", 0))
|
||||||
|
+ int(right_res.get("processed", 0)),
|
||||||
|
"rejected": int(left_res.get("rejected", 0))
|
||||||
|
+ int(right_res.get("rejected", 0)),
|
||||||
|
"spooledCount": int(left_res.get("spooledCount", 0))
|
||||||
|
+ int(right_res.get("spooledCount", 0)),
|
||||||
|
"transport": right_res.get("transport") or left_res.get("transport"),
|
||||||
|
}
|
||||||
|
|
||||||
|
# =========================================================================
|
||||||
|
# Direct File Streaming (.csv, .json, .jsonl / .ndjson)
|
||||||
|
# =========================================================================
|
||||||
|
|
||||||
|
def send_file(
|
||||||
|
self,
|
||||||
|
file_path: Union[str, Path],
|
||||||
|
chunk_size: Optional[int] = None,
|
||||||
|
extra_metadata: Optional[Dict[str, Any]] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Streams a `.csv`, `.json`, or `.jsonl` (`.ndjson`) file directly to DataStack
|
||||||
|
in safe chunks with deterministic idempotency IDs, automatic HTTP fallback,
|
||||||
|
and offline spooling support.
|
||||||
|
"""
|
||||||
|
path = Path(file_path).expanduser().resolve()
|
||||||
|
if not path.exists() or not path.is_file():
|
||||||
|
raise DataStackFileError(f"File not found: {path}")
|
||||||
|
|
||||||
|
ext = path.suffix.lower()
|
||||||
|
if ext not in (".csv", ".json", ".jsonl", ".ndjson"):
|
||||||
|
raise DataStackFileError(
|
||||||
|
f"Unsupported file extension '{ext}'. Supported formats: .csv, .json, .jsonl, .ndjson"
|
||||||
|
)
|
||||||
|
|
||||||
|
effective_chunk = max(1, int(chunk_size or self.config.batch_chunk_size))
|
||||||
|
total_processed = 0
|
||||||
|
total_rejected = 0
|
||||||
|
total_spooled = 0
|
||||||
|
chunks_sent = 0
|
||||||
|
last_transport = "WEBSOCKET"
|
||||||
|
|
||||||
|
buffer: List[Dict[str, Any]] = []
|
||||||
|
for idx, record in enumerate(self._iter_file_records(path, ext), start=1):
|
||||||
|
enriched = (
|
||||||
|
{**extra_metadata, **record}
|
||||||
|
if extra_metadata and isinstance(record, dict)
|
||||||
|
else record
|
||||||
|
)
|
||||||
|
buffer.append(
|
||||||
|
{
|
||||||
|
"eventId": f"file_{path.stem}_{idx}_{uuid.uuid4().hex[:6]}",
|
||||||
|
"timestamp": _utc_now_iso(),
|
||||||
|
"data": enriched,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if len(buffer) >= effective_chunk:
|
||||||
|
res = self.send_batch(buffer, file_name=path.name)
|
||||||
|
total_processed += int(res.get("processed", 0))
|
||||||
|
total_rejected += int(res.get("rejected", 0))
|
||||||
|
total_spooled += int(res.get("spooledCount", 0))
|
||||||
|
last_transport = str(res.get("transport", last_transport))
|
||||||
|
chunks_sent += 1
|
||||||
|
buffer = []
|
||||||
|
|
||||||
|
if buffer:
|
||||||
|
res = self.send_batch(buffer, file_name=path.name)
|
||||||
|
total_processed += int(res.get("processed", 0))
|
||||||
|
total_rejected += int(res.get("rejected", 0))
|
||||||
|
total_spooled += int(res.get("spooledCount", 0))
|
||||||
|
last_transport = str(res.get("transport", last_transport))
|
||||||
|
chunks_sent += 1
|
||||||
|
|
||||||
|
return {
|
||||||
|
"success": True,
|
||||||
|
"fileName": path.name,
|
||||||
|
"fileType": ext.lstrip("."),
|
||||||
|
"chunksSent": chunks_sent,
|
||||||
|
"processed": total_processed,
|
||||||
|
"rejected": total_rejected,
|
||||||
|
"spooledCount": total_spooled,
|
||||||
|
"transport": last_transport,
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _iter_file_records(path: Path, ext: str) -> Iterable[Dict[str, Any]]:
|
||||||
|
try:
|
||||||
|
if ext == ".csv":
|
||||||
|
with path.open("r", encoding="utf-8-sig", newline="") as fh:
|
||||||
|
reader = csv.DictReader(fh)
|
||||||
|
for row in reader:
|
||||||
|
yield dict(row)
|
||||||
|
elif ext in (".jsonl", ".ndjson"):
|
||||||
|
with path.open("r", encoding="utf-8-sig") as fh:
|
||||||
|
for line_no, line in enumerate(fh, start=1):
|
||||||
|
stripped = line.strip()
|
||||||
|
if not stripped:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
parsed = json.loads(stripped)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise DataStackFileError(
|
||||||
|
f"Malformed JSONL at line {line_no} in {path.name}: {exc}"
|
||||||
|
) from exc
|
||||||
|
yield parsed if isinstance(parsed, dict) else {"value": parsed}
|
||||||
|
elif ext == ".json":
|
||||||
|
with path.open("r", encoding="utf-8-sig") as fh:
|
||||||
|
parsed = json.load(fh)
|
||||||
|
if isinstance(parsed, list):
|
||||||
|
for item in parsed:
|
||||||
|
yield item if isinstance(item, dict) else {"value": item}
|
||||||
|
elif isinstance(parsed, dict):
|
||||||
|
if isinstance(parsed.get("records"), list):
|
||||||
|
for item in parsed["records"]:
|
||||||
|
yield item if isinstance(item, dict) else {"value": item}
|
||||||
|
elif isinstance(parsed.get("data"), list):
|
||||||
|
for item in parsed["data"]:
|
||||||
|
yield item if isinstance(item, dict) else {"value": item}
|
||||||
|
else:
|
||||||
|
yield parsed
|
||||||
|
else:
|
||||||
|
yield {"value": parsed}
|
||||||
|
except DataStackFileError:
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
raise DataStackFileError(
|
||||||
|
f"Failed reading file '{path.name}': {exc}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
def status(self) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
Returns a complete diagnostic snapshot of the SDK connection state,
|
||||||
|
Heartbeat Monitor mode (`IDLE` vs `ACTIVE_RECOVERY`), and offline spool queue.
|
||||||
|
"""
|
||||||
|
hb = self.heartbeat.status_snapshot()
|
||||||
|
return {
|
||||||
|
"connectedWebSocket": self.is_connected,
|
||||||
|
"serverUrl": self.server_url,
|
||||||
|
"apiUrl": self.api_url,
|
||||||
|
"configSource": self.config.config_source_path,
|
||||||
|
"connectionState": hb["state"],
|
||||||
|
"heartbeatMode": hb["heartbeatMode"],
|
||||||
|
"recoveryAttempts": hb["recoveryAttempts"],
|
||||||
|
"lastError": hb["lastError"],
|
||||||
|
"offlineSpoolPending": self.spooler.pending_count(),
|
||||||
|
"offlineSpoolBytes": self.spooler.size_bytes(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# Convenient alias so developers can write `from datastack import DataStack`
|
||||||
|
DataStack = DataStackClient
|
||||||
@@ -0,0 +1,321 @@
|
|||||||
|
"""
|
||||||
|
Auto-Discovery Configuration Loader for `config.datastack.py`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
from .exceptions import DataStackConfigError
|
||||||
|
|
||||||
|
CONFIG_FILENAME = "config.datastack.py"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class DataStackConfig:
|
||||||
|
server_url: str = "http://localhost:4000"
|
||||||
|
api_url: str = "http://localhost:3000"
|
||||||
|
client_token: str = ""
|
||||||
|
client_id: str = ""
|
||||||
|
auto_connect: bool = True
|
||||||
|
enable_http_fallback: bool = True
|
||||||
|
heartbeat_interval_sec: float = 5.0
|
||||||
|
request_timeout_sec: float = 8.0
|
||||||
|
max_retries: int = 3
|
||||||
|
retry_backoff_base_sec: float = 0.5
|
||||||
|
batch_chunk_size: int = 250
|
||||||
|
offline_spool_enabled: bool = True
|
||||||
|
offline_spool_dir: str = ".datastack_spool"
|
||||||
|
max_spool_size_mb: int = 50
|
||||||
|
config_source_path: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
def _derive_default_api_url(server_url: str) -> str:
|
||||||
|
"""
|
||||||
|
If API_URL is not explicitly set, infer Port 3000 on the same host when SERVER_URL
|
||||||
|
uses Port 4000, or fall back to http://localhost:3000.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
parsed = urlparse(server_url)
|
||||||
|
if parsed.scheme and parsed.hostname:
|
||||||
|
if parsed.port == 4000:
|
||||||
|
return f"{parsed.scheme}://{parsed.hostname}:3000"
|
||||||
|
return f"{parsed.scheme}://{parsed.netloc}"
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return "http://localhost:3000"
|
||||||
|
|
||||||
|
|
||||||
|
def _candidate_config_paths(explicit_path: Optional[str] = None) -> List[Path]:
|
||||||
|
candidates: List[Path] = []
|
||||||
|
|
||||||
|
if explicit_path:
|
||||||
|
p = Path(explicit_path).expanduser().resolve()
|
||||||
|
if p.is_dir():
|
||||||
|
candidates.append(p / CONFIG_FILENAME)
|
||||||
|
else:
|
||||||
|
candidates.append(p)
|
||||||
|
|
||||||
|
env_path = os.environ.get("DATASTACK_CONFIG_PATH", "").strip()
|
||||||
|
if env_path:
|
||||||
|
ep = Path(env_path).expanduser().resolve()
|
||||||
|
if ep.is_dir():
|
||||||
|
candidates.append(ep / CONFIG_FILENAME)
|
||||||
|
else:
|
||||||
|
candidates.append(ep)
|
||||||
|
|
||||||
|
# Search current working directory and up to 3 parent levels
|
||||||
|
try:
|
||||||
|
cwd = Path.cwd().resolve()
|
||||||
|
candidates.append(cwd / CONFIG_FILENAME)
|
||||||
|
for parent in list(cwd.parents)[:3]:
|
||||||
|
candidates.append(parent / CONFIG_FILENAME)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Search main entry script directory
|
||||||
|
try:
|
||||||
|
if sys.argv and sys.argv[0]:
|
||||||
|
script_dir = Path(sys.argv[0]).resolve().parent
|
||||||
|
candidates.append(script_dir / CONFIG_FILENAME)
|
||||||
|
for parent in list(script_dir.parents)[:2]:
|
||||||
|
candidates.append(parent / CONFIG_FILENAME)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Deduplicate while preserving order
|
||||||
|
seen = set()
|
||||||
|
unique: List[Path] = []
|
||||||
|
for c in candidates:
|
||||||
|
key = str(c)
|
||||||
|
if key not in seen:
|
||||||
|
seen.add(key)
|
||||||
|
unique.append(c)
|
||||||
|
return unique
|
||||||
|
|
||||||
|
|
||||||
|
def _load_python_config_file(file_path: Path) -> Dict[str, Any]:
|
||||||
|
try:
|
||||||
|
spec = importlib.util.spec_from_file_location(
|
||||||
|
"datastack_user_config", str(file_path)
|
||||||
|
)
|
||||||
|
if spec is None or spec.loader is None:
|
||||||
|
raise DataStackConfigError(
|
||||||
|
f"Unable to load configuration specification from {file_path}"
|
||||||
|
)
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
|
||||||
|
raw: Dict[str, Any] = {}
|
||||||
|
if hasattr(module, "DATASTACK_CONFIG") and isinstance(
|
||||||
|
getattr(module, "DATASTACK_CONFIG"), dict
|
||||||
|
):
|
||||||
|
raw.update(getattr(module, "DATASTACK_CONFIG"))
|
||||||
|
|
||||||
|
# Also support top-level module variables (e.g., SERVER_URL = "...", CLIENT_TOKEN = "...")
|
||||||
|
for attr in dir(module):
|
||||||
|
if attr.isupper() and attr != "DATASTACK_CONFIG":
|
||||||
|
raw[attr] = getattr(module, attr)
|
||||||
|
|
||||||
|
return raw
|
||||||
|
except DataStackConfigError:
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
raise DataStackConfigError(
|
||||||
|
f"Failed to parse {file_path}: {exc}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def load_config(
|
||||||
|
config_path: Optional[str] = None,
|
||||||
|
require_token: bool = False,
|
||||||
|
**overrides: Any,
|
||||||
|
) -> DataStackConfig:
|
||||||
|
"""
|
||||||
|
Discovers and loads `config.datastack.py` from the client application's
|
||||||
|
directory, merges environment variables (`DATASTACK_*`), and applies
|
||||||
|
any explicit keyword overrides.
|
||||||
|
"""
|
||||||
|
file_values: Dict[str, Any] = {}
|
||||||
|
resolved_source: Optional[str] = None
|
||||||
|
|
||||||
|
for candidate in _candidate_config_paths(config_path):
|
||||||
|
if candidate.is_file():
|
||||||
|
file_values = _load_python_config_file(candidate)
|
||||||
|
resolved_source = str(candidate)
|
||||||
|
break
|
||||||
|
|
||||||
|
if config_path and not resolved_source:
|
||||||
|
raise DataStackConfigError(
|
||||||
|
f"Explicitly requested config file was not found: {config_path}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Normalize keys to uppercase for lookup
|
||||||
|
normalized_file = {str(k).upper(): v for k, v in file_values.items()}
|
||||||
|
normalized_overrides = {
|
||||||
|
str(k).upper(): v for k, v in overrides.items() if v is not None
|
||||||
|
}
|
||||||
|
|
||||||
|
def pick(keys: List[str], env_keys: List[str], default: Any) -> Any:
|
||||||
|
for k in keys:
|
||||||
|
if k in normalized_overrides:
|
||||||
|
return normalized_overrides[k]
|
||||||
|
for ek in env_keys:
|
||||||
|
env_val = os.environ.get(ek)
|
||||||
|
if env_val is not None and str(env_val).strip() != "":
|
||||||
|
return env_val
|
||||||
|
for k in keys:
|
||||||
|
if k in normalized_file and normalized_file[k] is not None:
|
||||||
|
return normalized_file[k]
|
||||||
|
return default
|
||||||
|
|
||||||
|
def to_bool(val: Any, default: bool) -> bool:
|
||||||
|
if isinstance(val, bool):
|
||||||
|
return val
|
||||||
|
if isinstance(val, str):
|
||||||
|
return val.strip().lower() in ("1", "true", "yes", "on")
|
||||||
|
if val is None:
|
||||||
|
return default
|
||||||
|
return bool(val)
|
||||||
|
|
||||||
|
server_url = str(
|
||||||
|
pick(
|
||||||
|
["SERVER_URL", "SOCKET_URL", "WS_URL"],
|
||||||
|
["DATASTACK_SERVER_URL", "DATASTACK_SOCKET_URL"],
|
||||||
|
"http://localhost:4000",
|
||||||
|
)
|
||||||
|
).rstrip("/")
|
||||||
|
|
||||||
|
default_api = _derive_default_api_url(server_url)
|
||||||
|
api_url = str(
|
||||||
|
pick(
|
||||||
|
["API_URL", "HTTP_URL", "WEB_URL"],
|
||||||
|
["DATASTACK_API_URL", "DATASTACK_HTTP_URL"],
|
||||||
|
default_api,
|
||||||
|
)
|
||||||
|
).rstrip("/")
|
||||||
|
|
||||||
|
client_token = str(
|
||||||
|
pick(
|
||||||
|
["CLIENT_TOKEN", "TOKEN", "JWT_TOKEN", "API_TOKEN"],
|
||||||
|
["DATASTACK_CLIENT_TOKEN", "DATASTACK_TOKEN"],
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
).strip()
|
||||||
|
|
||||||
|
client_id = str(
|
||||||
|
pick(
|
||||||
|
["CLIENT_ID"],
|
||||||
|
["DATASTACK_CLIENT_ID"],
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
).strip()
|
||||||
|
|
||||||
|
cfg = DataStackConfig(
|
||||||
|
server_url=server_url,
|
||||||
|
api_url=api_url,
|
||||||
|
client_token=client_token,
|
||||||
|
client_id=client_id,
|
||||||
|
auto_connect=to_bool(
|
||||||
|
pick(["AUTO_CONNECT"], ["DATASTACK_AUTO_CONNECT"], True), True
|
||||||
|
),
|
||||||
|
enable_http_fallback=to_bool(
|
||||||
|
pick(
|
||||||
|
["ENABLE_HTTP_FALLBACK", "HTTP_FALLBACK"],
|
||||||
|
["DATASTACK_ENABLE_HTTP_FALLBACK"],
|
||||||
|
True,
|
||||||
|
),
|
||||||
|
True,
|
||||||
|
),
|
||||||
|
heartbeat_interval_sec=max(
|
||||||
|
1.0,
|
||||||
|
float(
|
||||||
|
pick(
|
||||||
|
["HEARTBEAT_INTERVAL_SEC", "HEARTBEAT_INTERVAL"],
|
||||||
|
["DATASTACK_HEARTBEAT_INTERVAL_SEC"],
|
||||||
|
5.0,
|
||||||
|
)
|
||||||
|
),
|
||||||
|
),
|
||||||
|
request_timeout_sec=max(
|
||||||
|
1.0,
|
||||||
|
float(
|
||||||
|
pick(
|
||||||
|
["REQUEST_TIMEOUT_SEC", "TIMEOUT"],
|
||||||
|
["DATASTACK_REQUEST_TIMEOUT_SEC"],
|
||||||
|
8.0,
|
||||||
|
)
|
||||||
|
),
|
||||||
|
),
|
||||||
|
max_retries=max(
|
||||||
|
0,
|
||||||
|
int(
|
||||||
|
pick(
|
||||||
|
["MAX_RETRIES", "RETRIES"],
|
||||||
|
["DATASTACK_MAX_RETRIES"],
|
||||||
|
3,
|
||||||
|
)
|
||||||
|
),
|
||||||
|
),
|
||||||
|
retry_backoff_base_sec=max(
|
||||||
|
0.1,
|
||||||
|
float(
|
||||||
|
pick(
|
||||||
|
["RETRY_BACKOFF_BASE_SEC"],
|
||||||
|
["DATASTACK_RETRY_BACKOFF_BASE_SEC"],
|
||||||
|
0.5,
|
||||||
|
)
|
||||||
|
),
|
||||||
|
),
|
||||||
|
batch_chunk_size=max(
|
||||||
|
1,
|
||||||
|
int(
|
||||||
|
pick(
|
||||||
|
["BATCH_CHUNK_SIZE", "CHUNK_SIZE"],
|
||||||
|
["DATASTACK_BATCH_CHUNK_SIZE"],
|
||||||
|
250,
|
||||||
|
)
|
||||||
|
),
|
||||||
|
),
|
||||||
|
offline_spool_enabled=to_bool(
|
||||||
|
pick(
|
||||||
|
["OFFLINE_SPOOL_ENABLED"],
|
||||||
|
["DATASTACK_OFFLINE_SPOOL_ENABLED"],
|
||||||
|
True,
|
||||||
|
),
|
||||||
|
True,
|
||||||
|
),
|
||||||
|
offline_spool_dir=str(
|
||||||
|
pick(
|
||||||
|
["OFFLINE_SPOOL_DIR", "SPOOL_DIR"],
|
||||||
|
["DATASTACK_OFFLINE_SPOOL_DIR"],
|
||||||
|
".datastack_spool",
|
||||||
|
)
|
||||||
|
),
|
||||||
|
max_spool_size_mb=max(
|
||||||
|
1,
|
||||||
|
int(
|
||||||
|
pick(
|
||||||
|
["MAX_SPOOL_SIZE_MB"],
|
||||||
|
["DATASTACK_MAX_SPOOL_SIZE_MB"],
|
||||||
|
50,
|
||||||
|
)
|
||||||
|
),
|
||||||
|
),
|
||||||
|
config_source_path=resolved_source,
|
||||||
|
)
|
||||||
|
|
||||||
|
if require_token and (
|
||||||
|
not cfg.client_token
|
||||||
|
or cfg.client_token == "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN"
|
||||||
|
):
|
||||||
|
raise DataStackConfigError(
|
||||||
|
"DataStack CLIENT_TOKEN is missing. Add your token to `config.datastack.py`, "
|
||||||
|
"set DATASTACK_TOKEN in your environment, or pass token='...' to DataStack()."
|
||||||
|
)
|
||||||
|
|
||||||
|
return cfg
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
"""
|
||||||
|
Structured Exception Hierarchy for the DataStack Python SDK.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
|
||||||
|
class DataStackError(Exception):
|
||||||
|
"""Base exception for all DataStack SDK errors."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
message: str,
|
||||||
|
code: Optional[str] = None,
|
||||||
|
retryable: bool = False,
|
||||||
|
) -> None:
|
||||||
|
super().__init__(message)
|
||||||
|
self.message = message
|
||||||
|
self.code = code or "DATASTACK_ERROR"
|
||||||
|
self.retryable = retryable
|
||||||
|
|
||||||
|
|
||||||
|
class DataStackConfigError(DataStackError):
|
||||||
|
"""Raised when `config.datastack.py` or required credentials are missing or invalid."""
|
||||||
|
|
||||||
|
def __init__(self, message: str) -> None:
|
||||||
|
super().__init__(message, code="CONFIG_ERROR", retryable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class DataStackAuthError(DataStackError):
|
||||||
|
"""
|
||||||
|
Raised when authentication fails (`AUTH_REQUIRED`, `AUTH_INVALID`,
|
||||||
|
`CLIENT_REVOKED`, or `TOKEN_ROTATED`). Non-retryable until credentials are updated.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, message: str, code: str = "AUTH_ERROR") -> None:
|
||||||
|
super().__init__(message, code=code, retryable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class DataStackRateLimitError(DataStackError):
|
||||||
|
"""Raised when `RATE_LIMIT_EXCEEDED` persists after all backoff retries."""
|
||||||
|
|
||||||
|
def __init__(self, message: str) -> None:
|
||||||
|
super().__init__(message, code="RATE_LIMIT_EXCEEDED", retryable=True)
|
||||||
|
|
||||||
|
|
||||||
|
class DataStackPayloadTooLargeError(DataStackError):
|
||||||
|
"""Raised when a single event exceeds the server's `maxPayloadSizeKb` limit."""
|
||||||
|
|
||||||
|
def __init__(self, message: str) -> None:
|
||||||
|
super().__init__(message, code="PAYLOAD_TOO_LARGE", retryable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class DataStackConnectionError(DataStackError):
|
||||||
|
"""Raised when both WebSocket and HTTP fallback are unreachable and offline spooling is disabled."""
|
||||||
|
|
||||||
|
def __init__(self, message: str) -> None:
|
||||||
|
super().__init__(message, code="CONNECTION_ERROR", retryable=True)
|
||||||
|
|
||||||
|
|
||||||
|
class DataStackFileError(DataStackError):
|
||||||
|
"""Raised when a file passed to `send_file()` is missing, unsupported, or malformed."""
|
||||||
|
|
||||||
|
def __init__(self, message: str) -> None:
|
||||||
|
super().__init__(message, code="FILE_ERROR", retryable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class DataStackOfflineSpoolError(DataStackError):
|
||||||
|
"""Raised when the local offline disk spool directory is full or unwriteable."""
|
||||||
|
|
||||||
|
def __init__(self, message: str) -> None:
|
||||||
|
super().__init__(message, code="OFFLINE_SPOOL_ERROR", retryable=False)
|
||||||
@@ -0,0 +1,186 @@
|
|||||||
|
"""
|
||||||
|
Smart Heartbeat Monitor & Connection State Machine (`HeartbeatMonitor`).
|
||||||
|
|
||||||
|
Lifecycle States:
|
||||||
|
- `CONNECTED_IDLE`: WebSocket (Port 4000) is connected and verified. Heartbeat sleeps in IDLE mode.
|
||||||
|
- `FALLBACK_HTTP`: WebSocket disconnected. Client automatically sends events via HTTP (`POST /api/events`)
|
||||||
|
while Heartbeat actively probes to reconnect and verify WebSocket, then returns to `CONNECTED_IDLE`.
|
||||||
|
- `OFFLINE_SPOOL`: Device is completely offline. Events spool to disk (`.datastack_spool/spool.jsonl`).
|
||||||
|
Heartbeat monitors network restoration, reconnects, replays spooled events, and returns to `CONNECTED_IDLE`.
|
||||||
|
- `RECONNECTING_VERIFY`: Heartbeat is verifying handshake (`ping_verify`) and draining spooled events.
|
||||||
|
- `STOPPED`: Client was closed intentionally or halted due to non-retryable auth revocation.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from typing import Any, Callable, Dict, Optional
|
||||||
|
|
||||||
|
logger = logging.getLogger("datastack.heartbeat")
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectionState:
|
||||||
|
CONNECTED_IDLE = "CONNECTED_IDLE"
|
||||||
|
FALLBACK_HTTP = "FALLBACK_HTTP"
|
||||||
|
OFFLINE_SPOOL = "OFFLINE_SPOOL"
|
||||||
|
RECONNECTING_VERIFY = "RECONNECTING_VERIFY"
|
||||||
|
STOPPED = "STOPPED"
|
||||||
|
|
||||||
|
|
||||||
|
class HeartbeatMonitor:
|
||||||
|
"""
|
||||||
|
Background recovery thread that sleeps in zero-CPU IDLE mode while WebSocket is
|
||||||
|
healthy (`CONNECTED_IDLE`) and wakes up automatically when WebSocket disconnects
|
||||||
|
or when the client device goes offline.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
interval_sec: float,
|
||||||
|
recovery_callback: Callable[[], bool],
|
||||||
|
) -> None:
|
||||||
|
self.interval_sec = max(1.0, float(interval_sec))
|
||||||
|
self._recovery_callback = recovery_callback
|
||||||
|
self._state = ConnectionState.CONNECTED_IDLE
|
||||||
|
self._closed = False
|
||||||
|
self._last_error: Optional[str] = None
|
||||||
|
self._recovery_attempts = 0
|
||||||
|
self._last_verified_at: Optional[float] = None
|
||||||
|
|
||||||
|
self._lock = threading.RLock()
|
||||||
|
self._wake_event = threading.Event()
|
||||||
|
self._stop_event = threading.Event()
|
||||||
|
self._thread: Optional[threading.Thread] = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def state(self) -> str:
|
||||||
|
with self._lock:
|
||||||
|
return self._state
|
||||||
|
|
||||||
|
@property
|
||||||
|
def is_idle(self) -> bool:
|
||||||
|
with self._lock:
|
||||||
|
return self._state == ConnectionState.CONNECTED_IDLE
|
||||||
|
|
||||||
|
def set_idle_connected(self) -> None:
|
||||||
|
"""
|
||||||
|
Marks the connection as healthy (`CONNECTED_IDLE`) and puts the Heartbeat
|
||||||
|
Monitor into IDLE mode (`_wake_event.clear()`).
|
||||||
|
"""
|
||||||
|
with self._lock:
|
||||||
|
self._closed = False
|
||||||
|
self._state = ConnectionState.CONNECTED_IDLE
|
||||||
|
self._recovery_attempts = 0
|
||||||
|
self._last_error = None
|
||||||
|
self._last_verified_at = time.time()
|
||||||
|
self._wake_event.clear()
|
||||||
|
|
||||||
|
def trigger_fallback(self, reason: str = "WebSocket disconnected") -> None:
|
||||||
|
"""
|
||||||
|
Transitions to `FALLBACK_HTTP` and wakes up the Heartbeat Monitor thread
|
||||||
|
to repair the WebSocket connection in the background.
|
||||||
|
"""
|
||||||
|
with self._lock:
|
||||||
|
if self._closed:
|
||||||
|
return
|
||||||
|
self._state = ConnectionState.FALLBACK_HTTP
|
||||||
|
self._last_error = reason
|
||||||
|
self._ensure_thread_started()
|
||||||
|
self._wake_event.set()
|
||||||
|
|
||||||
|
def trigger_offline(self, reason: str = "Device offline") -> None:
|
||||||
|
"""
|
||||||
|
Transitions to `OFFLINE_SPOOL` when both WebSocket and HTTP fallback are
|
||||||
|
unreachable, waking the Heartbeat Monitor to watch for network return.
|
||||||
|
"""
|
||||||
|
with self._lock:
|
||||||
|
if self._closed:
|
||||||
|
return
|
||||||
|
self._state = ConnectionState.OFFLINE_SPOOL
|
||||||
|
self._last_error = reason
|
||||||
|
self._ensure_thread_started()
|
||||||
|
self._wake_event.set()
|
||||||
|
|
||||||
|
def set_reconnecting(self) -> None:
|
||||||
|
with self._lock:
|
||||||
|
if not self._closed:
|
||||||
|
self._state = ConnectionState.RECONNECTING_VERIFY
|
||||||
|
|
||||||
|
def start(self) -> None:
|
||||||
|
with self._lock:
|
||||||
|
self._closed = False
|
||||||
|
self._stop_event.clear()
|
||||||
|
self._ensure_thread_started()
|
||||||
|
|
||||||
|
def stop(self, reason: Optional[str] = None) -> None:
|
||||||
|
with self._lock:
|
||||||
|
self._closed = True
|
||||||
|
self._state = ConnectionState.STOPPED
|
||||||
|
if reason:
|
||||||
|
self._last_error = reason
|
||||||
|
self._stop_event.set()
|
||||||
|
self._wake_event.set()
|
||||||
|
|
||||||
|
def status_snapshot(self) -> Dict[str, Any]:
|
||||||
|
with self._lock:
|
||||||
|
return {
|
||||||
|
"state": self._state,
|
||||||
|
"heartbeatMode": (
|
||||||
|
"IDLE"
|
||||||
|
if self._state == ConnectionState.CONNECTED_IDLE
|
||||||
|
else "ACTIVE_RECOVERY"
|
||||||
|
if self._state
|
||||||
|
in (
|
||||||
|
ConnectionState.FALLBACK_HTTP,
|
||||||
|
ConnectionState.OFFLINE_SPOOL,
|
||||||
|
ConnectionState.RECONNECTING_VERIFY,
|
||||||
|
)
|
||||||
|
else "STOPPED"
|
||||||
|
),
|
||||||
|
"recoveryAttempts": self._recovery_attempts,
|
||||||
|
"lastError": self._last_error,
|
||||||
|
"lastVerifiedAt": self._last_verified_at,
|
||||||
|
"intervalSec": self.interval_sec,
|
||||||
|
}
|
||||||
|
|
||||||
|
def _ensure_thread_started(self) -> None:
|
||||||
|
if self._thread is not None and self._thread.is_alive():
|
||||||
|
return
|
||||||
|
self._stop_event.clear()
|
||||||
|
self._thread = threading.Thread(
|
||||||
|
target=self._run_loop,
|
||||||
|
name="DataStack-HeartbeatMonitor",
|
||||||
|
daemon=True,
|
||||||
|
)
|
||||||
|
self._thread.start()
|
||||||
|
|
||||||
|
def _run_loop(self) -> None:
|
||||||
|
while not self._stop_event.is_set():
|
||||||
|
# Wait in zero-CPU IDLE mode until woken by a disconnect or offline trigger
|
||||||
|
self._wake_event.wait()
|
||||||
|
if self._stop_event.is_set():
|
||||||
|
break
|
||||||
|
|
||||||
|
# Wait for the heartbeat interval before probing
|
||||||
|
if self._stop_event.wait(self.interval_sec):
|
||||||
|
break
|
||||||
|
|
||||||
|
if not self._wake_event.is_set():
|
||||||
|
continue
|
||||||
|
|
||||||
|
with self._lock:
|
||||||
|
if self._state == ConnectionState.STOPPED:
|
||||||
|
break
|
||||||
|
self._recovery_attempts += 1
|
||||||
|
|
||||||
|
try:
|
||||||
|
recovered_to_idle = self._recovery_callback()
|
||||||
|
if recovered_to_idle:
|
||||||
|
self.set_idle_connected()
|
||||||
|
logger.info(
|
||||||
|
"[DataStack Heartbeat] WebSocket reconnected & verified. Returning to IDLE mode."
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
with self._lock:
|
||||||
|
self._last_error = str(exc)
|
||||||
|
logger.debug("[DataStack Heartbeat] Recovery probe failed: %s", exc)
|
||||||
@@ -0,0 +1,164 @@
|
|||||||
|
"""
|
||||||
|
Thread-Safe Offline Disk Spooler (`DiskSpooler`).
|
||||||
|
Persists unsent telemetry events to `.datastack_spool/spool.jsonl` when a client
|
||||||
|
device loses network connectivity completely, and drains them in deterministic
|
||||||
|
batches once the Heartbeat Monitor restores connectivity.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import threading
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
|
from .exceptions import DataStackOfflineSpoolError
|
||||||
|
|
||||||
|
|
||||||
|
class DiskSpooler:
|
||||||
|
"""
|
||||||
|
Append-only JSONL disk queue with atomic batch drainage and size cap protection.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
spool_dir: str = ".datastack_spool",
|
||||||
|
max_size_mb: int = 50,
|
||||||
|
enabled: bool = True,
|
||||||
|
) -> None:
|
||||||
|
self.enabled = enabled
|
||||||
|
self.spool_dir = Path(spool_dir)
|
||||||
|
self.spool_file = self.spool_dir / "spool.jsonl"
|
||||||
|
self.max_bytes = max(1, int(max_size_mb)) * 1024 * 1024
|
||||||
|
self._lock = threading.RLock()
|
||||||
|
|
||||||
|
def _ensure_dir(self) -> None:
|
||||||
|
try:
|
||||||
|
self.spool_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
except Exception as exc:
|
||||||
|
raise DataStackOfflineSpoolError(
|
||||||
|
f"Cannot create offline spool directory '{self.spool_dir}': {exc}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
def enqueue(self, events: List[Dict[str, Any]]) -> int:
|
||||||
|
"""
|
||||||
|
Appends one or more formatted events (`{eventId, timestamp, data}`)
|
||||||
|
to `.datastack_spool/spool.jsonl`. Returns count of spooled events.
|
||||||
|
"""
|
||||||
|
if not self.enabled or not events:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
with self._lock:
|
||||||
|
self._ensure_dir()
|
||||||
|
current_size = (
|
||||||
|
self.spool_file.stat().st_size if self.spool_file.exists() else 0
|
||||||
|
)
|
||||||
|
|
||||||
|
lines: List[str] = []
|
||||||
|
added_bytes = 0
|
||||||
|
for evt in events:
|
||||||
|
serialized = json.dumps(evt, ensure_ascii=False) + "\n"
|
||||||
|
encoded_len = len(serialized.encode("utf-8"))
|
||||||
|
if current_size + added_bytes + encoded_len > self.max_bytes:
|
||||||
|
raise DataStackOfflineSpoolError(
|
||||||
|
f"Offline spool limit ({self.max_bytes // (1024 * 1024)} MB) "
|
||||||
|
f"reached at '{self.spool_file}'."
|
||||||
|
)
|
||||||
|
lines.append(serialized)
|
||||||
|
added_bytes += encoded_len
|
||||||
|
|
||||||
|
try:
|
||||||
|
with self.spool_file.open("a", encoding="utf-8") as fh:
|
||||||
|
fh.writelines(lines)
|
||||||
|
fh.flush()
|
||||||
|
except Exception as exc:
|
||||||
|
raise DataStackOfflineSpoolError(
|
||||||
|
f"Failed writing to offline spool file '{self.spool_file}': {exc}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
return len(lines)
|
||||||
|
|
||||||
|
def peek_batch(self, limit: int = 250) -> List[Dict[str, Any]]:
|
||||||
|
"""
|
||||||
|
Reads up to `limit` spooled events from the front of the spool file
|
||||||
|
without removing them until `ack_batch()` confirms delivery.
|
||||||
|
"""
|
||||||
|
if not self.enabled:
|
||||||
|
return []
|
||||||
|
|
||||||
|
with self._lock:
|
||||||
|
if not self.spool_file.exists():
|
||||||
|
return []
|
||||||
|
|
||||||
|
batch: List[Dict[str, Any]] = []
|
||||||
|
try:
|
||||||
|
with self.spool_file.open("r", encoding="utf-8") as fh:
|
||||||
|
for line in fh:
|
||||||
|
if len(batch) >= limit:
|
||||||
|
break
|
||||||
|
stripped = line.strip()
|
||||||
|
if not stripped:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
parsed = json.loads(stripped)
|
||||||
|
if isinstance(parsed, dict):
|
||||||
|
batch.append(parsed)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
continue
|
||||||
|
except Exception:
|
||||||
|
return []
|
||||||
|
|
||||||
|
return batch
|
||||||
|
|
||||||
|
def ack_batch(self, count: int) -> int:
|
||||||
|
"""
|
||||||
|
Removes the first `count` non-empty lines from the spool file atomically
|
||||||
|
after the Heartbeat Monitor confirms they have been ingested by DataStack.
|
||||||
|
"""
|
||||||
|
if not self.enabled or count <= 0:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
with self._lock:
|
||||||
|
if not self.spool_file.exists():
|
||||||
|
return 0
|
||||||
|
|
||||||
|
try:
|
||||||
|
with self.spool_file.open("r", encoding="utf-8") as fh:
|
||||||
|
all_lines = [ln for ln in fh if ln.strip()]
|
||||||
|
|
||||||
|
if count >= len(all_lines):
|
||||||
|
self.spool_file.unlink(missing_ok=True)
|
||||||
|
return len(all_lines)
|
||||||
|
|
||||||
|
remaining = all_lines[count:]
|
||||||
|
tmp_file = self.spool_dir / "spool.jsonl.tmp"
|
||||||
|
with tmp_file.open("w", encoding="utf-8") as fh:
|
||||||
|
fh.writelines(remaining)
|
||||||
|
fh.flush()
|
||||||
|
os.replace(str(tmp_file), str(self.spool_file))
|
||||||
|
return count
|
||||||
|
except Exception:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def pending_count(self) -> int:
|
||||||
|
"""Returns the number of queued events waiting in the disk spool."""
|
||||||
|
if not self.enabled:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
with self._lock:
|
||||||
|
if not self.spool_file.exists():
|
||||||
|
return 0
|
||||||
|
try:
|
||||||
|
with self.spool_file.open("r", encoding="utf-8") as fh:
|
||||||
|
return sum(1 for ln in fh if ln.strip())
|
||||||
|
except Exception:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def size_bytes(self) -> int:
|
||||||
|
"""Returns current disk usage of the spool file in bytes."""
|
||||||
|
with self._lock:
|
||||||
|
if not self.spool_file.exists():
|
||||||
|
return 0
|
||||||
|
try:
|
||||||
|
return self.spool_file.stat().st_size
|
||||||
|
except Exception:
|
||||||
|
return 0
|
||||||
@@ -0,0 +1,54 @@
|
|||||||
|
[build-system]
|
||||||
|
requires = ["setuptools>=61.0", "wheel"]
|
||||||
|
build-backend = "setuptools.build_meta"
|
||||||
|
|
||||||
|
[project]
|
||||||
|
name = "datastack"
|
||||||
|
version = "2.2.0"
|
||||||
|
description = "Official Python SDK for DataStack — Real-Time WebSocket & HTTP Fallback Telemetry Ingestion Engine with Auto Heartbeat Recovery, Offline Spooling, and Direct CSV/JSON File Streaming"
|
||||||
|
readme = "README.md"
|
||||||
|
license = { text = "MIT" }
|
||||||
|
requires-python = ">=3.8"
|
||||||
|
authors = [
|
||||||
|
{ name = "Md Anik (Vib Tools)", email = "info@vibtools.top" }
|
||||||
|
]
|
||||||
|
keywords = [
|
||||||
|
"datastack",
|
||||||
|
"telemetry",
|
||||||
|
"event-ingestion",
|
||||||
|
"websocket",
|
||||||
|
"socketio",
|
||||||
|
"csv-streaming",
|
||||||
|
"offline-spool",
|
||||||
|
"vib-tools"
|
||||||
|
]
|
||||||
|
classifiers = [
|
||||||
|
"Development Status :: 5 - Production/Stable",
|
||||||
|
"Intended Audience :: Developers",
|
||||||
|
"License :: OSI Approved :: MIT License",
|
||||||
|
"Operating System :: OS Independent",
|
||||||
|
"Programming Language :: Python :: 3",
|
||||||
|
"Programming Language :: Python :: 3.8",
|
||||||
|
"Programming Language :: Python :: 3.9",
|
||||||
|
"Programming Language :: Python :: 3.10",
|
||||||
|
"Programming Language :: Python :: 3.11",
|
||||||
|
"Programming Language :: Python :: 3.12",
|
||||||
|
"Topic :: Software Development :: Libraries :: Python Modules",
|
||||||
|
"Topic :: System :: Logging",
|
||||||
|
"Topic :: System :: Monitoring"
|
||||||
|
]
|
||||||
|
dependencies = [
|
||||||
|
"python-socketio[client]>=5.8.0",
|
||||||
|
"requests>=2.28.0",
|
||||||
|
"websocket-client>=1.5.0"
|
||||||
|
]
|
||||||
|
|
||||||
|
[project.urls]
|
||||||
|
Homepage = "https://vibtools.top/"
|
||||||
|
Documentation = "https://github.com/anik-d-art/datastack"
|
||||||
|
Repository = "https://github.com/anik-d-art/datastack"
|
||||||
|
Issues = "https://github.com/anik-d-art/datastack/issues"
|
||||||
|
|
||||||
|
[tool.setuptools.packages.find]
|
||||||
|
where = ["."]
|
||||||
|
include = ["datastack*"]
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
from pathlib import Path
|
||||||
|
from setuptools import find_packages, setup
|
||||||
|
|
||||||
|
ROOT_DIR = Path(__file__).resolve().parent
|
||||||
|
README_PATH = ROOT_DIR / "README.md"
|
||||||
|
long_description = (
|
||||||
|
README_PATH.read_text(encoding="utf-8")
|
||||||
|
if README_PATH.exists()
|
||||||
|
else "Official Python SDK for DataStack Real-Time Event Ingestion Platform."
|
||||||
|
)
|
||||||
|
|
||||||
|
setup(
|
||||||
|
name="datastack",
|
||||||
|
version="2.2.0",
|
||||||
|
description=(
|
||||||
|
"Official Python SDK for DataStack — Real-Time WebSocket & HTTP Fallback "
|
||||||
|
"Telemetry Ingestion Engine with Auto Heartbeat Recovery, Offline Spooling, "
|
||||||
|
"and Direct CSV/JSON File Streaming"
|
||||||
|
),
|
||||||
|
long_description=long_description,
|
||||||
|
long_description_content_type="text/markdown",
|
||||||
|
author="Md Anik (Vib Tools)",
|
||||||
|
author_email="info@vibtools.top",
|
||||||
|
url="https://vibtools.top/",
|
||||||
|
license="MIT",
|
||||||
|
packages=find_packages(include=["datastack", "datastack.*"]),
|
||||||
|
install_requires=[
|
||||||
|
"python-socketio[client]>=5.8.0",
|
||||||
|
"requests>=2.28.0",
|
||||||
|
"websocket-client>=1.5.0",
|
||||||
|
],
|
||||||
|
python_requires=">=3.8",
|
||||||
|
classifiers=[
|
||||||
|
"Development Status :: 5 - Production/Stable",
|
||||||
|
"Intended Audience :: Developers",
|
||||||
|
"License :: OSI Approved :: MIT License",
|
||||||
|
"Programming Language :: Python :: 3",
|
||||||
|
"Programming Language :: Python :: 3.8",
|
||||||
|
"Programming Language :: Python :: 3.9",
|
||||||
|
"Programming Language :: Python :: 3.10",
|
||||||
|
"Programming Language :: Python :: 3.11",
|
||||||
|
"Programming Language :: Python :: 3.12",
|
||||||
|
],
|
||||||
|
)
|
||||||
+3
-1
@@ -252,7 +252,7 @@ const io = new Server(server, {
|
|||||||
cors: { origin: '*', methods: ['GET', 'POST'] },
|
cors: { origin: '*', methods: ['GET', 'POST'] },
|
||||||
});
|
});
|
||||||
|
|
||||||
// Health check endpoint with activeSockets count
|
// Health check endpoint with activeSockets count and runtime capabilities
|
||||||
app.get('/health', (_req, res) => {
|
app.get('/health', (_req, res) => {
|
||||||
res.json({
|
res.json({
|
||||||
status: 'online',
|
status: 'online',
|
||||||
@@ -260,6 +260,8 @@ app.get('/health', (_req, res) => {
|
|||||||
port: PORT,
|
port: PORT,
|
||||||
activeSockets: io.engine ? io.engine.clientsCount : 0,
|
activeSockets: io.engine ? io.engine.clientsCount : 0,
|
||||||
bufferSize: bufferQueue.length,
|
bufferSize: bufferQueue.length,
|
||||||
|
maxEventsPerSec: runtimeConfig.maxEventsPerSec,
|
||||||
|
maxPayloadSizeBytes: runtimeConfig.maxPayloadSizeBytes,
|
||||||
uptime: process.uptime(),
|
uptime: process.uptime(),
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -307,6 +307,47 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Heartbeat & Reconnection Verification Event
|
||||||
|
socket.on("ping_verify", async (_payload, callback) => {
|
||||||
|
try {
|
||||||
|
const dbCheck = await verifyClientInDb(
|
||||||
|
clientId,
|
||||||
|
socket.data.token,
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
if (!dbCheck.allowed) {
|
||||||
|
if (callback) {
|
||||||
|
callback({
|
||||||
|
success: false,
|
||||||
|
status: "REVOKED",
|
||||||
|
error: dbCheck.reason || "CLIENT_REVOKED",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
socket.disconnect(true);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const config = getRuntimeConfig();
|
||||||
|
if (callback) {
|
||||||
|
callback({
|
||||||
|
success: true,
|
||||||
|
status: "VERIFIED",
|
||||||
|
clientId,
|
||||||
|
rateLimit: dbCheck.rateLimit || socket.data.rateLimit || 100,
|
||||||
|
maxPayloadSizeBytes: config.maxPayloadSizeBytes,
|
||||||
|
serverTime: new Date().toISOString(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
if (callback) {
|
||||||
|
callback({
|
||||||
|
success: false,
|
||||||
|
status: "ERROR",
|
||||||
|
error: err.message,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user