- Fetch events from external ingestion service in API routes - Decouple idempotency check from registration in Security utility - Refine CSV storage logic to verify file content for headers - Replace Python SDK with comprehensive, feature-rich implementation
1194 lines
44 KiB
Python
1194 lines
44 KiB
Python
"""
|
|
Official DataStack Python SDK Client (`DataStack` / `DataStackClient`) for `datastack_sdk`.
|
|
|
|
Self-contained implementation with 100% feature parity across both `datastack`
|
|
and `datastack_sdk` packages:
|
|
- Automatic discovery of `config.datastack.py` in the client app's source/working directory.
|
|
- Primary real-time Socket.IO telemetry streaming (`sync_data`, `batch_sync`, `ping_verify`).
|
|
- Automatic Heartbeat HTTP Fallback (`POST /api/events`) when WebSocket closes or disconnects.
|
|
- Automatic WebSocket reconnection, handshake verification (`ping_verify`), and return to IDLE mode.
|
|
- Offline Disk Spooling (`.datastack_spool/spool.jsonl`) when the client device loses internet,
|
|
with automatic paced replay when connectivity returns.
|
|
- Direct `.csv`, `.json`, and `.jsonl` file ingestion (`send_file()`) with deterministic SHA-256
|
|
`eventId` deduplication, automatic batch chunking (`413 PAYLOAD_TOO_LARGE` auto-splitting),
|
|
and lossless partial-batch rate-limit retry (`429 RATE_LIMIT_EXCEEDED`).
|
|
"""
|
|
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import random
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Dict, Iterable, List, Optional, Union
|
|
|
|
try:
|
|
import socketio # type: ignore
|
|
except ImportError:
|
|
socketio = None # type: ignore
|
|
|
|
try:
|
|
import requests # type: ignore
|
|
except ImportError:
|
|
requests = None # type: ignore
|
|
|
|
from .config import DataStackConfig, load_config
|
|
from .exceptions import (
|
|
DataStackAuthError,
|
|
DataStackConfigError,
|
|
DataStackConnectionError,
|
|
DataStackError,
|
|
DataStackFileError,
|
|
DataStackOfflineSpoolError,
|
|
DataStackPayloadTooLargeError,
|
|
DataStackRateLimitError,
|
|
)
|
|
from .heartbeat import ConnectionState, HeartbeatMonitor
|
|
from .spool import DiskSpooler
|
|
|
|
logger = logging.getLogger("datastack.client")
|
|
|
|
|
|
def _utc_now_iso() -> str:
|
|
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def _make_event_id() -> str:
|
|
return f"evt_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}"
|
|
|
|
|
|
def _make_deterministic_file_event_id(
|
|
file_stem: str, row_index: int, record: Any
|
|
) -> str:
|
|
raw = f"{file_stem}:{row_index}:{json.dumps(record, sort_keys=True, default=str, ensure_ascii=False)}"
|
|
digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:12]
|
|
safe_stem = "".join(
|
|
ch if ch.isalnum() or ch in ("_", "-") else "_" for ch in file_stem
|
|
)[:32]
|
|
return f"file_{safe_stem}_{row_index}_{digest}"
|
|
|
|
|
|
class DataStackClient:
|
|
"""
|
|
Production-grade DataStack Python Client with `config.datastack.py` auto-loading,
|
|
WebSocket + Heartbeat HTTP Fallback + Offline Spooling + Direct File Streaming.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
server_url: Optional[str] = None,
|
|
token: Optional[str] = None,
|
|
api_url: Optional[str] = None,
|
|
config_path: Optional[str] = None,
|
|
auto_connect: Optional[bool] = None,
|
|
reconnection_attempts: Optional[int] = None,
|
|
reconnection_delay: Optional[float] = None,
|
|
**kwargs: Any,
|
|
) -> None:
|
|
if reconnection_attempts is not None and "MAX_RETRIES" not in kwargs:
|
|
kwargs["MAX_RETRIES"] = reconnection_attempts
|
|
if (
|
|
reconnection_delay is not None
|
|
and "HEARTBEAT_INTERVAL_SEC" not in kwargs
|
|
):
|
|
kwargs["HEARTBEAT_INTERVAL_SEC"] = reconnection_delay
|
|
|
|
self.config: DataStackConfig = load_config(
|
|
config_path=config_path,
|
|
require_token=False,
|
|
SERVER_URL=server_url,
|
|
CLIENT_TOKEN=token,
|
|
API_URL=api_url,
|
|
AUTO_CONNECT=auto_connect,
|
|
**kwargs,
|
|
)
|
|
|
|
self.server_url = self.config.server_url
|
|
self.api_url = self.config.api_url
|
|
self.token = self.config.client_token
|
|
self.is_connected = False
|
|
self._lock = threading.RLock()
|
|
self._is_draining_spool = False
|
|
|
|
# Offline Disk Spooler
|
|
self.spooler = DiskSpooler(
|
|
spool_dir=self.config.offline_spool_dir,
|
|
max_size_mb=self.config.max_spool_size_mb,
|
|
enabled=self.config.offline_spool_enabled,
|
|
)
|
|
|
|
# Heartbeat Monitor (starts in IDLE once connected, wakes on disconnect/offline)
|
|
self.heartbeat = HeartbeatMonitor(
|
|
interval_sec=self.config.heartbeat_interval_sec,
|
|
recovery_callback=self._heartbeat_recovery_step,
|
|
)
|
|
|
|
# Socket.IO Client (disable built-in infinite loop so HeartbeatMonitor coordinates fallback + IDLE)
|
|
self.sio = (
|
|
socketio.Client(
|
|
reconnection=False,
|
|
logger=False,
|
|
engineio_logger=False,
|
|
)
|
|
if socketio is not None
|
|
else None
|
|
)
|
|
if self.sio is not None:
|
|
self._register_socket_events()
|
|
|
|
if (
|
|
self.config.auto_connect
|
|
and self.token
|
|
and self.token != "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN"
|
|
):
|
|
try:
|
|
self.connect()
|
|
except DataStackAuthError:
|
|
raise
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"[DataStack] Initial connect entered fallback/recovery mode: %s",
|
|
exc,
|
|
)
|
|
|
|
def __enter__(self) -> "DataStackClient":
|
|
if (
|
|
not self.is_connected
|
|
and self.token
|
|
and self.token != "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN"
|
|
):
|
|
try:
|
|
self.connect()
|
|
except DataStackAuthError:
|
|
raise
|
|
except Exception:
|
|
pass
|
|
return self
|
|
|
|
def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
|
|
self.disconnect()
|
|
|
|
def _register_socket_events(self) -> None:
|
|
if self.sio is None:
|
|
return
|
|
|
|
@self.sio.event
|
|
def connect() -> None:
|
|
with self._lock:
|
|
self.is_connected = True
|
|
logger.info("[DataStack] WebSocket connected to %s", self.server_url)
|
|
|
|
@self.sio.event
|
|
def connect_error(data: Any) -> None:
|
|
with self._lock:
|
|
self.is_connected = False
|
|
msg = str(data)
|
|
if self._is_non_retryable_auth_error(msg):
|
|
self.heartbeat.stop(reason=msg)
|
|
else:
|
|
self.heartbeat.trigger_fallback(reason=f"Socket connect_error: {msg}")
|
|
|
|
@self.sio.event
|
|
def disconnect() -> None:
|
|
with self._lock:
|
|
was_connected = self.is_connected
|
|
self.is_connected = False
|
|
if was_connected and self.heartbeat.state != ConnectionState.STOPPED:
|
|
logger.warning(
|
|
"[DataStack] WebSocket disconnected. Activating Heartbeat HTTP Fallback."
|
|
)
|
|
self.heartbeat.trigger_fallback(reason="WebSocket connection closed")
|
|
|
|
@staticmethod
|
|
def _is_non_retryable_auth_error(message: str) -> bool:
|
|
upper = (message or "").upper()
|
|
return any(
|
|
code in upper
|
|
for code in (
|
|
"AUTH_REQUIRED",
|
|
"AUTH_INVALID",
|
|
"CLIENT_REVOKED",
|
|
"TOKEN_ROTATED",
|
|
)
|
|
)
|
|
|
|
def _ensure_token_present(self) -> None:
|
|
if not self.token or self.token == "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN":
|
|
raise DataStackConfigError(
|
|
"Missing CLIENT_TOKEN. Set CLIENT_TOKEN in `config.datastack.py` or pass token='...'."
|
|
)
|
|
|
|
def connect(self, wait_timeout: Optional[float] = None) -> bool:
|
|
"""
|
|
Establishes the primary WebSocket connection on Port 4000 and verifies the
|
|
handshake via `ping_verify`. If WebSocket is unavailable and `ENABLE_HTTP_FALLBACK`
|
|
is True, automatically enters `FALLBACK_HTTP` (or `OFFLINE_SPOOL` if offline)
|
|
and starts the background Heartbeat Monitor.
|
|
"""
|
|
self._ensure_token_present()
|
|
self.heartbeat.start()
|
|
|
|
try:
|
|
if self._connect_and_verify_websocket(wait_timeout=wait_timeout):
|
|
self._drain_offline_spool()
|
|
self.heartbeat.set_idle_connected()
|
|
return True
|
|
except DataStackAuthError as auth_err:
|
|
self.heartbeat.stop(reason=str(auth_err))
|
|
raise
|
|
except Exception as exc:
|
|
logger.warning("[DataStack] WebSocket connect failed: %s", exc)
|
|
|
|
# WebSocket was not reachable; check if HTTP fallback API is reachable
|
|
if self.config.enable_http_fallback:
|
|
if self._check_http_health():
|
|
self.heartbeat.trigger_fallback(
|
|
reason="WebSocket unreachable on connect; using HTTP fallback"
|
|
)
|
|
self._drain_offline_spool()
|
|
return True
|
|
|
|
if self.config.offline_spool_enabled:
|
|
self.heartbeat.trigger_offline(
|
|
reason="Both WebSocket and HTTP endpoints unreachable on connect"
|
|
)
|
|
return False
|
|
|
|
raise DataStackConnectionError(
|
|
f"Unable to connect to DataStack WebSocket ({self.server_url}) or HTTP API ({self.api_url})."
|
|
)
|
|
|
|
def disconnect(self) -> None:
|
|
"""Gracefully stops the Heartbeat Monitor and disconnects the WebSocket."""
|
|
self.heartbeat.stop(reason="Client closed by application")
|
|
with self._lock:
|
|
self.is_connected = False
|
|
if self.sio is not None and self.sio.connected:
|
|
try:
|
|
self.sio.disconnect()
|
|
except Exception:
|
|
pass
|
|
|
|
def _connect_and_verify_websocket(
|
|
self, wait_timeout: Optional[float] = None
|
|
) -> bool:
|
|
if self.sio is None:
|
|
return False
|
|
|
|
effective_timeout = (
|
|
float(wait_timeout)
|
|
if wait_timeout is not None
|
|
else self.config.request_timeout_sec
|
|
)
|
|
|
|
with self._lock:
|
|
self.is_connected = False
|
|
if self.sio.connected:
|
|
try:
|
|
self.sio.disconnect()
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
self.sio.connect(
|
|
self.server_url,
|
|
auth={"token": self.token},
|
|
headers={"Authorization": f"Bearer {self.token}"},
|
|
transports=["websocket", "polling"],
|
|
wait_timeout=effective_timeout,
|
|
)
|
|
self.is_connected = True
|
|
except Exception as exc:
|
|
self.is_connected = False
|
|
msg = str(exc)
|
|
if self._is_non_retryable_auth_error(msg):
|
|
raise DataStackAuthError(msg) from exc
|
|
return False
|
|
|
|
# Verify handshake & PostgreSQL client status via `ping_verify`
|
|
return self._verify_websocket_handshake(timeout=effective_timeout)
|
|
|
|
def _verify_websocket_handshake(
|
|
self, timeout: Optional[float] = None
|
|
) -> bool:
|
|
if self.sio is None or not self.sio.connected:
|
|
return False
|
|
|
|
ack_event = threading.Event()
|
|
result_holder: Dict[str, Any] = {}
|
|
|
|
def on_verify(resp: Dict[str, Any]) -> None:
|
|
if isinstance(resp, dict):
|
|
result_holder.update(resp)
|
|
ack_event.set()
|
|
|
|
wait_sec = timeout or self.config.request_timeout_sec
|
|
try:
|
|
self.sio.emit("ping_verify", {}, callback=on_verify)
|
|
if not ack_event.wait(timeout=wait_sec):
|
|
return False
|
|
except Exception:
|
|
return False
|
|
|
|
if not result_holder.get("success"):
|
|
err_msg = str(
|
|
result_holder.get("error") or "Handshake verification failed"
|
|
)
|
|
if self._is_non_retryable_auth_error(err_msg):
|
|
raise DataStackAuthError(err_msg)
|
|
return False
|
|
|
|
return result_holder.get("status") == "VERIFIED"
|
|
|
|
def _heartbeat_recovery_step(self) -> bool:
|
|
"""
|
|
Invoked periodically by `HeartbeatMonitor` ONLY when not in `CONNECTED_IDLE` mode.
|
|
1. Attempts to reconnect and verify (`ping_verify`) the primary WebSocket.
|
|
2. If WebSocket is verified, drains any offline spooled events and returns True
|
|
so `HeartbeatMonitor` transitions back to `CONNECTED_IDLE` (IDLE mode).
|
|
3. If WebSocket is still down, checks if HTTP fallback (`Port 3000`) is online:
|
|
- If HTTP is online, drains offline spooled events over HTTP and remains in `FALLBACK_HTTP`.
|
|
- If HTTP returns `401/403` auth revocation, stops Heartbeat immediately.
|
|
- If HTTP is also unreachable, sets state to `OFFLINE_SPOOL`.
|
|
"""
|
|
self.heartbeat.set_reconnecting()
|
|
try:
|
|
if self._connect_and_verify_websocket():
|
|
self._drain_offline_spool()
|
|
return True
|
|
except DataStackAuthError as auth_err:
|
|
self.heartbeat.stop(reason=str(auth_err))
|
|
return False
|
|
except Exception:
|
|
pass
|
|
|
|
# WebSocket still unavailable; check HTTP availability
|
|
try:
|
|
if self.config.enable_http_fallback and self._check_http_health():
|
|
self.heartbeat.trigger_fallback(
|
|
reason="WebSocket still unavailable; HTTP fallback active"
|
|
)
|
|
self._drain_offline_spool()
|
|
return False
|
|
except DataStackAuthError as auth_err:
|
|
self.heartbeat.stop(reason=str(auth_err))
|
|
return False
|
|
|
|
self.heartbeat.trigger_offline(
|
|
reason="Device offline: WebSocket and HTTP endpoints unreachable"
|
|
)
|
|
return False
|
|
|
|
def _check_http_health(self) -> bool:
|
|
"""
|
|
Probes `GET {API_URL}/api/health` with the client's Bearer JWT token.
|
|
Raises `DataStackAuthError` and halts Heartbeat if client credentials were
|
|
revoked or rotated (`401` / `403`). Returns `True` ONLY on `2xx` with
|
|
`status == 'online'`.
|
|
"""
|
|
url = f"{self.api_url}/api/health"
|
|
try:
|
|
status_code, body = self._http_request(
|
|
"GET", url, payload=None, timeout=3.0
|
|
)
|
|
except Exception:
|
|
return False
|
|
|
|
err_code = str(body.get("code") or "")
|
|
err_msg = str(body.get("error") or f"HTTP {status_code}")
|
|
|
|
if status_code in (401, 403) or self._is_non_retryable_auth_error(
|
|
f"{err_code} {err_msg}"
|
|
):
|
|
self.heartbeat.stop(reason=err_msg)
|
|
raise DataStackAuthError(err_msg, code=err_code or "AUTH_ERROR")
|
|
|
|
return 200 <= status_code < 300 and body.get("status") == "online"
|
|
|
|
def _http_request(
|
|
self,
|
|
method: str,
|
|
url: str,
|
|
payload: Optional[Dict[str, Any]] = None,
|
|
timeout: Optional[float] = None,
|
|
) -> tuple[int, Dict[str, Any]]:
|
|
req_timeout = timeout or self.config.request_timeout_sec
|
|
headers = {
|
|
"Authorization": f"Bearer {self.token}",
|
|
"Content-Type": "application/json",
|
|
"Accept": "application/json",
|
|
}
|
|
|
|
if requests is not None:
|
|
resp = requests.request(
|
|
method=method,
|
|
url=url,
|
|
json=payload,
|
|
headers=headers,
|
|
timeout=req_timeout,
|
|
)
|
|
try:
|
|
body = resp.json() if resp.text else {}
|
|
except Exception:
|
|
body = {"error": resp.text}
|
|
return resp.status_code, (
|
|
body if isinstance(body, dict) else {"data": body}
|
|
)
|
|
|
|
# Standard library urllib fallback when `requests` is not installed
|
|
data_bytes = (
|
|
json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
|
if payload is not None
|
|
else None
|
|
)
|
|
req = urllib.request.Request(
|
|
url=url, data=data_bytes, headers=headers, method=method
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=req_timeout) as response:
|
|
raw = response.read().decode("utf-8")
|
|
parsed = json.loads(raw) if raw else {}
|
|
return (
|
|
response.status,
|
|
parsed if isinstance(parsed, dict) else {"data": parsed},
|
|
)
|
|
except urllib.error.HTTPError as http_err:
|
|
raw_err = http_err.read().decode("utf-8", errors="replace")
|
|
try:
|
|
parsed_err = json.loads(raw_err) if raw_err else {}
|
|
except Exception:
|
|
parsed_err = {"error": raw_err or str(http_err)}
|
|
return (
|
|
http_err.code,
|
|
parsed_err
|
|
if isinstance(parsed_err, dict)
|
|
else {"error": str(parsed_err)},
|
|
)
|
|
|
|
def _send_via_http_fallback(
|
|
self, payload: Dict[str, Any]
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Sends a single event, batch array (`{ batch: [...] }`), or file records
|
|
(`{ fileName, records: [...] }`) over HTTP `POST {API_URL}/api/events`.
|
|
"""
|
|
url = f"{self.api_url}/api/events"
|
|
status_code, body = self._http_request("POST", url, payload=payload)
|
|
|
|
if 200 <= status_code < 300 and body.get("success", True):
|
|
return {
|
|
**body,
|
|
"success": True,
|
|
"transport": "HTTP_FALLBACK",
|
|
}
|
|
|
|
errors_list = body.get("errors")
|
|
first_err = (
|
|
errors_list[0]
|
|
if isinstance(errors_list, list) and len(errors_list) > 0
|
|
else None
|
|
)
|
|
err_msg = str(body.get("error") or first_err or f"HTTP {status_code}")
|
|
err_code = str(body.get("code") or "")
|
|
combined = f"{err_code} {err_msg}"
|
|
|
|
if status_code in (401, 403) or self._is_non_retryable_auth_error(
|
|
combined
|
|
):
|
|
self.heartbeat.stop(reason=err_msg)
|
|
raise DataStackAuthError(err_msg, code=err_code or "AUTH_ERROR")
|
|
|
|
if status_code == 413 or "PAYLOAD_TOO_LARGE" in combined:
|
|
raise DataStackPayloadTooLargeError(err_msg)
|
|
|
|
if status_code == 429 or "RATE_LIMIT_EXCEEDED" in combined:
|
|
raise DataStackRateLimitError(err_msg)
|
|
|
|
raise DataStackConnectionError(
|
|
f"HTTP fallback error ({status_code}): {err_msg}"
|
|
)
|
|
|
|
def _emit_via_websocket(
|
|
self,
|
|
event_name: str,
|
|
payload: Any,
|
|
timeout: Optional[float] = None,
|
|
) -> Dict[str, Any]:
|
|
if self.sio is None or not self.is_connected or not self.sio.connected:
|
|
raise DataStackConnectionError(
|
|
"WebSocket is not currently connected"
|
|
)
|
|
|
|
ack_event = threading.Event()
|
|
response_data: Dict[str, Any] = {}
|
|
|
|
def ack_callback(res: Dict[str, Any]) -> None:
|
|
if isinstance(res, dict):
|
|
response_data.update(res)
|
|
ack_event.set()
|
|
|
|
try:
|
|
self.sio.emit(event_name, payload, callback=ack_callback)
|
|
except Exception as exc:
|
|
self.is_connected = False
|
|
self.heartbeat.trigger_fallback(
|
|
reason=f"WebSocket emit error: {exc}"
|
|
)
|
|
raise DataStackConnectionError(
|
|
f"WebSocket emit failed: {exc}"
|
|
) from exc
|
|
|
|
wait_sec = timeout or self.config.request_timeout_sec
|
|
if not ack_event.wait(timeout=wait_sec):
|
|
self.is_connected = False
|
|
self.heartbeat.trigger_fallback(
|
|
reason=f"WebSocket ACK timeout after {wait_sec}s"
|
|
)
|
|
raise DataStackConnectionError(
|
|
f"WebSocket ACK timed out after {wait_sec}s"
|
|
)
|
|
|
|
if not response_data.get("success", False):
|
|
errors_list = response_data.get("errors")
|
|
first_err = (
|
|
errors_list[0]
|
|
if isinstance(errors_list, list) and len(errors_list) > 0
|
|
else None
|
|
)
|
|
err_msg = str(
|
|
response_data.get("error")
|
|
or first_err
|
|
or "WebSocket event rejected"
|
|
)
|
|
err_code = str(response_data.get("code") or "")
|
|
combined = f"{err_code} {err_msg}"
|
|
|
|
if self._is_non_retryable_auth_error(combined):
|
|
self.heartbeat.stop(reason=err_msg)
|
|
raise DataStackAuthError(err_msg, code=err_code or "AUTH_ERROR")
|
|
if "PAYLOAD_TOO_LARGE" in combined:
|
|
raise DataStackPayloadTooLargeError(err_msg)
|
|
if "RATE_LIMIT_EXCEEDED" in combined:
|
|
raise DataStackRateLimitError(err_msg)
|
|
raise DataStackError(err_msg, code=err_code or "WS_REJECTED")
|
|
|
|
return {
|
|
**response_data,
|
|
"transport": "WEBSOCKET",
|
|
}
|
|
|
|
def _execute_with_retry_and_fallback(
|
|
self,
|
|
ws_event: str,
|
|
ws_payload: Any,
|
|
http_payload: Dict[str, Any],
|
|
spool_items: List[Dict[str, Any]],
|
|
timeout: Optional[float] = None,
|
|
allow_spool: bool = True,
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Core resilient dispatcher:
|
|
1. Attempts WebSocket if connected (`CONNECTED_IDLE`).
|
|
2. On WebSocket failure/disconnect, triggers `HeartbeatMonitor` (`FALLBACK_HTTP`)
|
|
and automatically sends via HTTP `POST /api/events`.
|
|
3. Applies exponential backoff with jitter on `RATE_LIMIT_EXCEEDED` or transient errors.
|
|
4. If both WebSocket and HTTP are unreachable (device offline) and `allow_spool` is True,
|
|
spools events to `.datastack_spool/spool.jsonl` and triggers `OFFLINE_SPOOL`.
|
|
"""
|
|
self._ensure_token_present()
|
|
max_attempts = max(1, self.config.max_retries + 1)
|
|
last_exception: Optional[Exception] = None
|
|
|
|
for attempt in range(max_attempts):
|
|
if attempt > 0:
|
|
base_delay = (
|
|
max(1.0, self.config.retry_backoff_base_sec)
|
|
if isinstance(last_exception, DataStackRateLimitError)
|
|
else self.config.retry_backoff_base_sec
|
|
)
|
|
delay = base_delay * (2 ** (attempt - 1))
|
|
jitter = random.uniform(0.0, 0.2 * delay)
|
|
time.sleep(delay + jitter)
|
|
|
|
# 1. Try Primary WebSocket if connected
|
|
if self.is_connected and self.sio is not None and self.sio.connected:
|
|
try:
|
|
res = self._emit_via_websocket(
|
|
ws_event, ws_payload, timeout=timeout
|
|
)
|
|
if (
|
|
allow_spool
|
|
and not self._is_draining_spool
|
|
and self.spooler.pending_count() > 0
|
|
):
|
|
self._drain_offline_spool()
|
|
return res
|
|
except (DataStackAuthError, DataStackPayloadTooLargeError):
|
|
raise
|
|
except DataStackRateLimitError as rl_err:
|
|
last_exception = rl_err
|
|
continue
|
|
except Exception as ws_err:
|
|
last_exception = ws_err
|
|
self.heartbeat.trigger_fallback(reason=str(ws_err))
|
|
|
|
# 2. Try HTTP Fallback (`POST /api/events`)
|
|
if self.config.enable_http_fallback:
|
|
try:
|
|
res = self._send_via_http_fallback(http_payload)
|
|
if self.heartbeat.state in (
|
|
ConnectionState.CONNECTED_IDLE,
|
|
ConnectionState.OFFLINE_SPOOL,
|
|
):
|
|
self.heartbeat.trigger_fallback(
|
|
reason="WebSocket not connected; routing via HTTP fallback"
|
|
)
|
|
if (
|
|
allow_spool
|
|
and not self._is_draining_spool
|
|
and self.spooler.pending_count() > 0
|
|
):
|
|
self._drain_offline_spool()
|
|
return res
|
|
except (DataStackAuthError, DataStackPayloadTooLargeError):
|
|
raise
|
|
except DataStackRateLimitError as rl_err:
|
|
last_exception = rl_err
|
|
continue
|
|
except Exception as http_err:
|
|
last_exception = http_err
|
|
|
|
# 3. If rate-limited or payload-rejected after all retries, raise directly
|
|
if isinstance(
|
|
last_exception,
|
|
(DataStackRateLimitError, DataStackPayloadTooLargeError),
|
|
):
|
|
raise last_exception
|
|
|
|
# 4. Device is offline (both WebSocket and HTTP failed): Spool to local disk
|
|
if (
|
|
allow_spool
|
|
and self.config.offline_spool_enabled
|
|
and spool_items
|
|
):
|
|
spooled_count = self.spooler.enqueue(spool_items)
|
|
self.heartbeat.trigger_offline(
|
|
reason=str(last_exception or "Network unreachable")
|
|
)
|
|
return {
|
|
"success": True,
|
|
"spooled": True,
|
|
"spooledCount": spooled_count,
|
|
"pendingSpoolTotal": self.spooler.pending_count(),
|
|
"transport": "OFFLINE_SPOOL",
|
|
}
|
|
|
|
raise DataStackConnectionError(
|
|
f"DataStack delivery failed after {max_attempts} attempts: {last_exception}"
|
|
)
|
|
|
|
def _drain_offline_spool(self) -> int:
|
|
"""
|
|
Replays any offline spooled events from `.datastack_spool/spool.jsonl`
|
|
once connectivity (WebSocket or HTTP fallback) is restored.
|
|
Routes batches through `_send_formatted_batch_recursive(..., allow_spool=False)`
|
|
and preserves any `unprocessedItems` so zero spooled events are ever lost.
|
|
"""
|
|
if not self.config.offline_spool_enabled:
|
|
return 0
|
|
|
|
with self._lock:
|
|
if self._is_draining_spool:
|
|
return 0
|
|
self._is_draining_spool = True
|
|
|
|
total_flushed = 0
|
|
try:
|
|
while True:
|
|
batch = self.spooler.peek_batch(
|
|
limit=self.config.batch_chunk_size
|
|
)
|
|
if not batch:
|
|
break
|
|
|
|
res = self._send_formatted_batch_recursive(
|
|
batch,
|
|
timeout=self.config.request_timeout_sec,
|
|
file_name=None,
|
|
allow_spool=False,
|
|
)
|
|
processed_now = int(res.get("processed", 0))
|
|
duplicates_now = int(res.get("duplicates", 0))
|
|
unprocessed = res.get("unprocessedItems")
|
|
|
|
if (
|
|
not res.get("success", False)
|
|
and processed_now == 0
|
|
and duplicates_now == 0
|
|
):
|
|
break
|
|
|
|
self.spooler.ack_batch(len(batch))
|
|
if isinstance(unprocessed, list) and len(unprocessed) > 0:
|
|
self.spooler.enqueue(unprocessed)
|
|
total_flushed += max(0, len(batch) - len(unprocessed))
|
|
break
|
|
|
|
total_flushed += len(batch)
|
|
except Exception as exc:
|
|
logger.debug("[DataStack] Spool drain paused: %s", exc)
|
|
finally:
|
|
with self._lock:
|
|
self._is_draining_spool = False
|
|
|
|
return total_flushed
|
|
|
|
# =========================================================================
|
|
# Public Telemetry Ingestion API
|
|
# =========================================================================
|
|
|
|
def send(
|
|
self,
|
|
data: Dict[str, Any],
|
|
event_id: Optional[str] = None,
|
|
timestamp: Optional[str] = None,
|
|
timeout: Optional[float] = None,
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Sends a single telemetry event. Automatically falls back to HTTP if WebSocket
|
|
is disconnected, or spools to disk if the device is offline.
|
|
"""
|
|
resolved_id = (event_id or "").strip() or _make_event_id()
|
|
resolved_ts = (timestamp or "").strip() or _utc_now_iso()
|
|
envelope = {
|
|
"eventId": resolved_id,
|
|
"timestamp": resolved_ts,
|
|
"data": data,
|
|
}
|
|
return self._execute_with_retry_and_fallback(
|
|
ws_event="sync_data",
|
|
ws_payload=envelope,
|
|
http_payload=envelope,
|
|
spool_items=[envelope],
|
|
timeout=timeout,
|
|
allow_spool=True,
|
|
)
|
|
|
|
def sync(
|
|
self,
|
|
data: Dict[str, Any],
|
|
event_id: Optional[str] = None,
|
|
timeout: float = 5.0,
|
|
) -> Dict[str, Any]:
|
|
"""Alias for `send()` preserving backward compatibility."""
|
|
return self.send(data=data, event_id=event_id, timeout=timeout)
|
|
|
|
def send_batch(
|
|
self,
|
|
events: Optional[List[Dict[str, Any]]] = None,
|
|
timeout: Optional[float] = None,
|
|
file_name: Optional[str] = None,
|
|
*,
|
|
records: Optional[List[Dict[str, Any]]] = None,
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Sends a batch of events (accepts either `events=[...]` or `records=[...]`).
|
|
- If the batch triggers `413 PAYLOAD_TOO_LARGE`, automatically splits the batch
|
|
in half recursively and retries.
|
|
- If a subset of events in the batch hits `429 RATE_LIMIT_EXCEEDED`
|
|
(`rateLimitedEventIds`), automatically backs off and retries the remaining
|
|
rate-limited events so zero records are dropped.
|
|
"""
|
|
source_list = events if events is not None else records
|
|
if not isinstance(source_list, list) or len(source_list) == 0:
|
|
return {"success": True, "processed": 0, "rejected": 0}
|
|
|
|
formatted: List[Dict[str, Any]] = []
|
|
for item in source_list:
|
|
safe_item = item if isinstance(item, dict) else {"value": item}
|
|
evt_id = (
|
|
str(safe_item.get("eventId") or "").strip() or _make_event_id()
|
|
)
|
|
evt_ts = (
|
|
str(safe_item.get("timestamp") or "").strip() or _utc_now_iso()
|
|
)
|
|
evt_data = safe_item.get("data", safe_item)
|
|
formatted.append(
|
|
{
|
|
"eventId": evt_id,
|
|
"timestamp": evt_ts,
|
|
"data": evt_data,
|
|
}
|
|
)
|
|
|
|
return self._send_formatted_batch_recursive(
|
|
formatted,
|
|
timeout=timeout,
|
|
file_name=file_name,
|
|
allow_spool=True,
|
|
)
|
|
|
|
def batch_sync(
|
|
self,
|
|
events: Optional[List[Dict[str, Any]]] = None,
|
|
timeout: float = 10.0,
|
|
*,
|
|
records: Optional[List[Dict[str, Any]]] = None,
|
|
) -> Dict[str, Any]:
|
|
"""Alias for `send_batch()` supporting both `events` and `records` kwargs."""
|
|
return self.send_batch(events=events, records=records, timeout=timeout)
|
|
|
|
def _send_formatted_batch_recursive(
|
|
self,
|
|
formatted_batch: List[Dict[str, Any]],
|
|
timeout: Optional[float] = None,
|
|
file_name: Optional[str] = None,
|
|
allow_spool: bool = True,
|
|
) -> Dict[str, Any]:
|
|
http_payload: Dict[str, Any] = (
|
|
{"fileName": file_name, "records": formatted_batch}
|
|
if file_name
|
|
else {"batch": formatted_batch}
|
|
)
|
|
try:
|
|
res = self._execute_with_retry_and_fallback(
|
|
ws_event="batch_sync",
|
|
ws_payload=formatted_batch,
|
|
http_payload=http_payload,
|
|
spool_items=formatted_batch,
|
|
timeout=timeout,
|
|
allow_spool=allow_spool,
|
|
)
|
|
except DataStackPayloadTooLargeError:
|
|
if len(formatted_batch) <= 1:
|
|
raise
|
|
mid = len(formatted_batch) // 2
|
|
left_res = self._send_formatted_batch_recursive(
|
|
formatted_batch[:mid],
|
|
timeout=timeout,
|
|
file_name=file_name,
|
|
allow_spool=allow_spool,
|
|
)
|
|
right_res = self._send_formatted_batch_recursive(
|
|
formatted_batch[mid:],
|
|
timeout=timeout,
|
|
file_name=file_name,
|
|
allow_spool=allow_spool,
|
|
)
|
|
left_unprocessed = left_res.get("unprocessedItems") or []
|
|
right_unprocessed = right_res.get("unprocessedItems") or []
|
|
combined_unprocessed = [
|
|
*(left_unprocessed if isinstance(left_unprocessed, list) else []),
|
|
*(right_unprocessed if isinstance(right_unprocessed, list) else []),
|
|
]
|
|
return {
|
|
"success": bool(
|
|
left_res.get("success", True)
|
|
and right_res.get("success", True)
|
|
),
|
|
"processed": int(left_res.get("processed", 0))
|
|
+ int(right_res.get("processed", 0)),
|
|
"rejected": int(left_res.get("rejected", 0))
|
|
+ int(right_res.get("rejected", 0)),
|
|
"duplicates": int(left_res.get("duplicates", 0))
|
|
+ int(right_res.get("duplicates", 0)),
|
|
"spooledCount": int(left_res.get("spooledCount", 0))
|
|
+ int(right_res.get("spooledCount", 0)),
|
|
"transport": right_res.get("transport")
|
|
or left_res.get("transport"),
|
|
**(
|
|
{"unprocessedItems": combined_unprocessed}
|
|
if len(combined_unprocessed) > 0
|
|
else {}
|
|
),
|
|
}
|
|
|
|
# Handle partial-batch rate limiting (`rateLimitedEventIds`) so zero rows are dropped
|
|
rate_limited_ids = res.get("rateLimitedEventIds")
|
|
if isinstance(rate_limited_ids, list) and len(rate_limited_ids) > 0:
|
|
pending_id_set = {str(eid) for eid in rate_limited_ids}
|
|
remaining_items = [
|
|
item
|
|
for item in formatted_batch
|
|
if str(item.get("eventId")) in pending_id_set
|
|
]
|
|
total_processed = int(res.get("processed", 0))
|
|
total_duplicates = int(res.get("duplicates", 0))
|
|
total_spooled = int(res.get("spooledCount", 0))
|
|
last_transport = res.get("transport")
|
|
|
|
if self.config.max_retries > 0:
|
|
for _retry_round in range(1, self.config.max_retries + 2):
|
|
if not remaining_items:
|
|
break
|
|
wait_delay = max(
|
|
1.05, self.config.retry_backoff_base_sec
|
|
) + random.uniform(0.0, 0.15)
|
|
time.sleep(wait_delay)
|
|
|
|
sub_http_payload: Dict[str, Any] = (
|
|
{"fileName": file_name, "records": remaining_items}
|
|
if file_name
|
|
else {"batch": remaining_items}
|
|
)
|
|
sub_res = self._execute_with_retry_and_fallback(
|
|
ws_event="batch_sync",
|
|
ws_payload=remaining_items,
|
|
http_payload=sub_http_payload,
|
|
spool_items=remaining_items,
|
|
timeout=timeout,
|
|
allow_spool=allow_spool,
|
|
)
|
|
total_processed += int(sub_res.get("processed", 0))
|
|
total_duplicates += int(sub_res.get("duplicates", 0))
|
|
total_spooled += int(sub_res.get("spooledCount", 0))
|
|
last_transport = sub_res.get("transport") or last_transport
|
|
|
|
if sub_res.get("spooled"):
|
|
remaining_items = []
|
|
break
|
|
|
|
next_rl_ids = sub_res.get("rateLimitedEventIds")
|
|
if (
|
|
not isinstance(next_rl_ids, list)
|
|
or len(next_rl_ids) == 0
|
|
):
|
|
remaining_items = []
|
|
break
|
|
next_set = {str(eid) for eid in next_rl_ids}
|
|
remaining_items = [
|
|
item
|
|
for item in remaining_items
|
|
if str(item.get("eventId")) in next_set
|
|
]
|
|
|
|
# If any items still remain rate-limited after retries and spooling is allowed, spool them so zero data is lost
|
|
if (
|
|
remaining_items
|
|
and allow_spool
|
|
and self.config.offline_spool_enabled
|
|
):
|
|
spooled_now = self.spooler.enqueue(remaining_items)
|
|
total_spooled += spooled_now
|
|
remaining_items = []
|
|
|
|
oversized_ids = res.get("oversizedEventIds")
|
|
final_rejected = len(remaining_items) + (
|
|
len(oversized_ids) if isinstance(oversized_ids, list) else 0
|
|
)
|
|
out: Dict[str, Any] = {
|
|
"success": final_rejected == 0
|
|
or total_processed > 0
|
|
or total_spooled > 0,
|
|
"processed": total_processed,
|
|
"rejected": final_rejected,
|
|
"duplicates": total_duplicates,
|
|
"spooledCount": total_spooled,
|
|
"transport": last_transport,
|
|
}
|
|
if remaining_items:
|
|
out["unprocessedItems"] = remaining_items
|
|
return out
|
|
|
|
return res
|
|
|
|
# =========================================================================
|
|
# Direct File Streaming (.csv, .json, .jsonl / .ndjson)
|
|
# =========================================================================
|
|
|
|
def send_file(
|
|
self,
|
|
file_path: Union[str, Path],
|
|
chunk_size: Optional[int] = None,
|
|
extra_metadata: Optional[Dict[str, Any]] = None,
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Streams a `.csv`, `.json`, or `.jsonl` (`.ndjson`) file directly to DataStack
|
|
in safe chunks with deterministic SHA-256 idempotency IDs, automatic HTTP fallback,
|
|
partial-batch rate-limit pacing, and offline spooling support.
|
|
"""
|
|
path = Path(file_path).expanduser().resolve()
|
|
if not path.exists() or not path.is_file():
|
|
raise DataStackFileError(f"File not found: {path}")
|
|
|
|
ext = path.suffix.lower()
|
|
if ext not in (".csv", ".json", ".jsonl", ".ndjson"):
|
|
raise DataStackFileError(
|
|
f"Unsupported file extension '{ext}'. Supported formats: .csv, .json, .jsonl, .ndjson"
|
|
)
|
|
|
|
effective_chunk = max(
|
|
1, int(chunk_size or self.config.batch_chunk_size)
|
|
)
|
|
total_processed = 0
|
|
total_rejected = 0
|
|
total_duplicates = 0
|
|
total_spooled = 0
|
|
chunks_sent = 0
|
|
last_transport = "WEBSOCKET"
|
|
|
|
buffer: List[Dict[str, Any]] = []
|
|
for idx, record in enumerate(self._iter_file_records(path, ext), start=1):
|
|
enriched = (
|
|
{**extra_metadata, **record}
|
|
if extra_metadata and isinstance(record, dict)
|
|
else record
|
|
)
|
|
existing_id = (
|
|
str(enriched.get("eventId") or "").strip()
|
|
if isinstance(enriched, dict)
|
|
else ""
|
|
)
|
|
evt_id = existing_id or _make_deterministic_file_event_id(
|
|
path.stem, idx, enriched
|
|
)
|
|
existing_ts = (
|
|
str(enriched.get("timestamp") or "").strip()
|
|
if isinstance(enriched, dict)
|
|
else ""
|
|
)
|
|
evt_ts = existing_ts or _utc_now_iso()
|
|
|
|
buffer.append(
|
|
{
|
|
"eventId": evt_id,
|
|
"timestamp": evt_ts,
|
|
"data": enriched,
|
|
}
|
|
)
|
|
if len(buffer) >= effective_chunk:
|
|
res = self.send_batch(buffer, file_name=path.name)
|
|
total_processed += int(res.get("processed", 0))
|
|
total_rejected += int(res.get("rejected", 0))
|
|
total_duplicates += int(res.get("duplicates", 0))
|
|
total_spooled += int(res.get("spooledCount", 0))
|
|
last_transport = str(res.get("transport", last_transport))
|
|
chunks_sent += 1
|
|
buffer = []
|
|
|
|
if buffer:
|
|
res = self.send_batch(buffer, file_name=path.name)
|
|
total_processed += int(res.get("processed", 0))
|
|
total_rejected += int(res.get("rejected", 0))
|
|
total_duplicates += int(res.get("duplicates", 0))
|
|
total_spooled += int(res.get("spooledCount", 0))
|
|
last_transport = str(res.get("transport", last_transport))
|
|
chunks_sent += 1
|
|
|
|
return {
|
|
"success": True,
|
|
"fileName": path.name,
|
|
"fileType": ext.lstrip("."),
|
|
"chunksSent": chunks_sent,
|
|
"processed": total_processed,
|
|
"rejected": total_rejected,
|
|
"duplicates": total_duplicates,
|
|
"spooledCount": total_spooled,
|
|
"transport": last_transport,
|
|
}
|
|
|
|
@staticmethod
|
|
def _iter_file_records(path: Path, ext: str) -> Iterable[Dict[str, Any]]:
|
|
try:
|
|
if ext == ".csv":
|
|
with path.open("r", encoding="utf-8-sig", newline="") as fh:
|
|
reader = csv.DictReader(fh)
|
|
for row in reader:
|
|
yield dict(row)
|
|
elif ext in (".jsonl", ".ndjson"):
|
|
with path.open("r", encoding="utf-8-sig") as fh:
|
|
for line_no, line in enumerate(fh, start=1):
|
|
stripped = line.strip()
|
|
if not stripped:
|
|
continue
|
|
try:
|
|
parsed = json.loads(stripped)
|
|
except json.JSONDecodeError as exc:
|
|
raise DataStackFileError(
|
|
f"Malformed JSONL at line {line_no} in {path.name}: {exc}"
|
|
) from exc
|
|
yield (
|
|
parsed
|
|
if isinstance(parsed, dict)
|
|
else {"value": parsed}
|
|
)
|
|
elif ext == ".json":
|
|
with path.open("r", encoding="utf-8-sig") as fh:
|
|
parsed = json.load(fh)
|
|
if isinstance(parsed, list):
|
|
for item in parsed:
|
|
yield (
|
|
item if isinstance(item, dict) else {"value": item}
|
|
)
|
|
elif isinstance(parsed, dict):
|
|
if isinstance(parsed.get("records"), list):
|
|
for item in parsed["records"]:
|
|
yield (
|
|
item
|
|
if isinstance(item, dict)
|
|
else {"value": item}
|
|
)
|
|
elif isinstance(parsed.get("data"), list):
|
|
for item in parsed["data"]:
|
|
yield (
|
|
item
|
|
if isinstance(item, dict)
|
|
else {"value": item}
|
|
)
|
|
else:
|
|
yield parsed
|
|
else:
|
|
yield {"value": parsed}
|
|
except DataStackFileError:
|
|
raise
|
|
except Exception as exc:
|
|
raise DataStackFileError(
|
|
f"Failed reading file '{path.name}': {exc}"
|
|
) from exc
|
|
|
|
def status(self) -> Dict[str, Any]:
|
|
"""
|
|
Returns a complete diagnostic snapshot of the SDK connection state,
|
|
Heartbeat Monitor mode (`IDLE` vs `ACTIVE_RECOVERY`), and offline spool queue.
|
|
"""
|
|
hb = self.heartbeat.status_snapshot()
|
|
return {
|
|
"connectedWebSocket": self.is_connected,
|
|
"serverUrl": self.server_url,
|
|
"apiUrl": self.api_url,
|
|
"configSource": self.config.config_source_path,
|
|
"connectionState": hb["state"],
|
|
"heartbeatMode": hb["heartbeatMode"],
|
|
"recoveryAttempts": hb["recoveryAttempts"],
|
|
"lastError": hb["lastError"],
|
|
"offlineSpoolPending": self.spooler.pending_count(),
|
|
"offlineSpoolBytes": self.spooler.size_bytes(),
|
|
}
|
|
|
|
|
|
DataStack = DataStackClient
|
|
|
|
__all__ = [
|
|
"DataStack",
|
|
"DataStackClient",
|
|
"DataStackConfig",
|
|
"load_config",
|
|
"ConnectionState",
|
|
"HeartbeatMonitor",
|
|
"DiskSpooler",
|
|
"DataStackError",
|
|
"DataStackConfigError",
|
|
"DataStackAuthError",
|
|
"DataStackConnectionError",
|
|
"DataStackRateLimitError",
|
|
"DataStackPayloadTooLargeError",
|
|
"DataStackFileError",
|
|
"DataStackOfflineSpoolError",
|
|
]
|