diff --git a/app/api/data/route.ts b/app/api/data/route.ts index 7a59b9f..bdb6f32 100644 --- a/app/api/data/route.ts +++ b/app/api/data/route.ts @@ -113,6 +113,37 @@ export async function GET(request: NextRequest) { addIfMatching({ ...event, status: "buffered" }); } + const ingestionBaseUrl = ( + process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000" + ).replace(/\/$/, ""); + try { + const res = await fetch(`${ingestionBaseUrl}/internal/recent?limit=200`, { + cache: "no-store", + signal: AbortSignal.timeout(1000), + }); + if (res.ok) { + const data = await res.json(); + if (Array.isArray(data.events)) { + for (const evt of data.events) { + if (evt && evt.eventId) { + addIfMatching({ + clientId: String(evt.clientId || ""), + eventId: String(evt.eventId), + timestamp: String(evt.timestamp || new Date().toISOString()), + payload: + typeof evt.payload === "string" + ? evt.payload + : JSON.stringify(evt.payload ?? {}), + status: "buffered", + }); + } + } + } + } + } catch { + // Port 4000 may not be running in isolated single-process dev mode + } + return NextResponse.json({ events: selectedRows.slice(offset), total, diff --git a/app/api/events/route.ts b/app/api/events/route.ts index 50200b4..a93d99f 100644 --- a/app/api/events/route.ts +++ b/app/api/events/route.ts @@ -58,12 +58,34 @@ export async function GET(req: NextRequest) { const limit = Math.min(200, Math.max(1, Number(searchParams.get('limit')) || 50)); const bufferedEvents = DataBufferEngine.getRecentEvents(limit, clientId); + let socketBufferedEvents: DataEvent[] = []; + const ingestionBaseUrl = ( + process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000' + ).replace(/\/$/, ''); + + try { + const qs = new URLSearchParams({ limit: String(limit) }); + if (clientId) qs.set('clientId', clientId); + const res = await fetch(`${ingestionBaseUrl}/internal/recent?${qs.toString()}`, { + cache: 'no-store', + signal: AbortSignal.timeout(1000), + }); + if (res.ok) { + const data = await res.json(); + if (Array.isArray(data.events)) { + socketBufferedEvents = data.events; + } + } + } catch { + // Port 4000 may not be running in isolated single-process dev mode + } + const storedEvents = await getRecentStoredEvents(limit, clientId); const seenIds = new Set(); const merged: DataEvent[] = []; - for (const evt of [...bufferedEvents, ...storedEvents]) { + for (const evt of [...bufferedEvents, ...socketBufferedEvents, ...storedEvents]) { if (!evt.eventId || seenIds.has(evt.eventId)) continue; seenIds.add(evt.eventId); merged.push(evt); diff --git a/core/Security/security.ts b/core/Security/security.ts index b0aee37..452f24c 100644 --- a/core/Security/security.ts +++ b/core/Security/security.ts @@ -30,8 +30,8 @@ export class SecurityEngine { * ClientId + EventId must be unique. * Duplicate events are dropped silently or tagged. */ - public static checkIdempotency(clientId: string, eventId: string): boolean { - if (!eventId) return true; // If no eventId generated, pass through + public static hasDuplicate(clientId: string, eventId: string): boolean { + if (!eventId) return false; const key = `${clientId}:${eventId}`; const now = Date.now(); @@ -47,12 +47,24 @@ export class SecurityEngine { if (this.idempotencySet.has(key)) { const storedTime = this.idempotencySet.get(key) || 0; if (now - storedTime < this.IDEMPOTENCY_TTL_MS) { - return false; // Duplicate detected + return true; } } - this.idempotencySet.set(key, now); - return true; // Unique, allowed + return false; + } + + public static recordIdempotency(clientId: string, eventId: string): void { + if (!eventId) return; + this.idempotencySet.set(`${clientId}:${eventId}`, Date.now()); + } + + public static checkIdempotency(clientId: string, eventId: string): boolean { + if (this.hasDuplicate(clientId, eventId)) { + return false; + } + this.recordIdempotency(clientId, eventId); + return true; } /** @@ -81,7 +93,8 @@ export class SecurityEngine { /** * Validates incoming event payload: - * Checks max byte size and JSON structure. + * Checks duplicate idempotency (read-only), max byte size, and rate limit, + * and records the idempotency key ONLY after all checks pass. */ public static validateEvent( clientId: string, @@ -89,10 +102,8 @@ export class SecurityEngine { payload: unknown, clientMaxRate = 100, ): ValidationResult { - // 1. Check idempotency - const isUnique = - !this.enableIdempotency || this.checkIdempotency(clientId, eventId); - if (!isUnique) { + // 1. Check idempotency (read-only so rate-limited retries are not burned) + if (this.enableIdempotency && this.hasDuplicate(clientId, eventId)) { return { allowed: false, reason: "Duplicate event dropped silently", @@ -100,18 +111,7 @@ export class SecurityEngine { }; } - // 2. Check rate limit - const effectiveRateLimit = Math.min(clientMaxRate, this.maxEventsPerSec); - const withinLimit = this.checkRateLimit(clientId, effectiveRateLimit); - if (!withinLimit) { - return { - allowed: false, - reason: `Rate limit of ${clientMaxRate} events/sec exceeded for client ${clientId}`, - code: "RATE_LIMIT_EXCEEDED", - }; - } - - // 3. Check payload size + // 2. Check payload size before consuming rate-limit quota const payloadStr = typeof payload === "string" ? payload : JSON.stringify(payload); const byteLength = Buffer.byteLength(payloadStr, "utf8"); @@ -124,6 +124,22 @@ export class SecurityEngine { }; } + // 3. Check rate limit + const effectiveRateLimit = Math.min(clientMaxRate, this.maxEventsPerSec); + const withinLimit = this.checkRateLimit(clientId, effectiveRateLimit); + if (!withinLimit) { + return { + allowed: false, + reason: `Rate limit of ${clientMaxRate} events/sec exceeded for client ${clientId}`, + code: "RATE_LIMIT_EXCEEDED", + }; + } + + // 4. Record idempotency key ONLY after event is accepted + if (this.enableIdempotency && eventId) { + this.recordIdempotency(clientId, eventId); + } + return { allowed: true }; } diff --git a/core/Storage/csvStorage.ts b/core/Storage/csvStorage.ts index 5ef47ed..badf20b 100644 --- a/core/Storage/csvStorage.ts +++ b/core/Storage/csvStorage.ts @@ -91,7 +91,13 @@ export class CSVStorageManager { for (const [key, batch] of grouped.entries()) { const [clientId, date] = key.split(':::'); const targetPath = this.getTargetFilePath(clientId, date); - const fileExists = fs.existsSync(targetPath); + let hasContent = false; + try { + hasContent = + fs.existsSync(targetPath) && fs.statSync(targetPath).size > 0; + } catch { + hasContent = false; + } // Format payload as JSON string if not already const cleanBatch = batch.map((item) => ({ @@ -101,10 +107,10 @@ export class CSVStorageManager { payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload), })); - const parser = new Parser({ fields, header: !fileExists, eol: '\n' }); + const parser = new Parser({ fields, header: !hasContent, eol: '\n' }); const csvData = parser.parse(cleanBatch); - if (fileExists) { + if (hasContent) { fs.appendFileSync(targetPath, '\n' + csvData, 'utf8'); } else { fs.writeFileSync(targetPath, csvData, 'utf8'); diff --git a/sdk-python/datastack_sdk/client.py b/sdk-python/datastack_sdk/client.py index 1b526e9..2758060 100644 --- a/sdk-python/datastack_sdk/client.py +++ b/sdk-python/datastack_sdk/client.py @@ -1,413 +1,1178 @@ """ -Self-Contained & Unified `datastack_sdk.client` Module. -When running inside the repository or when `datastack` is installed in site-packages, -re-exports the canonical classes from `datastack`. If `sdk-python/datastack_sdk` is -copied or installed standalone without `datastack`, provides the complete embedded -implementation so `from datastack_sdk import DataStackClient, DataStack` never fails -with `ModuleNotFoundError`. +Official DataStack Python SDK Client (`DataStack` / `DataStackClient`) for `datastack_sdk`. + +Self-contained implementation with 100% feature parity across both `datastack` +and `datastack_sdk` packages: +- Automatic discovery of `config.datastack.py` in the client app's source/working directory. +- Primary real-time Socket.IO telemetry streaming (`sync_data`, `batch_sync`, `ping_verify`). +- Automatic Heartbeat HTTP Fallback (`POST /api/events`) when WebSocket closes or disconnects. +- Automatic WebSocket reconnection, handshake verification (`ping_verify`), and return to IDLE mode. +- Offline Disk Spooling (`.datastack_spool/spool.jsonl`) when the client device loses internet, + with automatic paced replay when connectivity returns. +- Direct `.csv`, `.json`, and `.jsonl` file ingestion (`send_file()`) with deterministic SHA-256 + `eventId` deduplication, automatic batch chunking (`413 PAYLOAD_TOO_LARGE` auto-splitting), + and lossless partial-batch rate-limit retry (`429 RATE_LIMIT_EXCEEDED`). """ -import sys +import csv +import hashlib +import json +import logging +import random +import threading +import time +import urllib.error +import urllib.request +import uuid +from datetime import datetime, timezone from pathlib import Path - -_REPO_ROOT = Path(__file__).resolve().parents[2] -_NEW_SDK_PATH = _REPO_ROOT / "sdk" / "python" / "datastack" -if _NEW_SDK_PATH.exists() and str(_NEW_SDK_PATH) not in sys.path: - sys.path.insert(0, str(_NEW_SDK_PATH)) +from typing import Any, Dict, Iterable, List, Optional, Union try: - from datastack.client import DataStack, DataStackClient # type: ignore - from datastack.config import DataStackConfig, load_config # type: ignore - from datastack.exceptions import ( # type: ignore - DataStackAuthError, - DataStackConfigError, - DataStackConnectionError, - DataStackError, - DataStackFileError, - DataStackOfflineSpoolError, - DataStackPayloadTooLargeError, - DataStackRateLimitError, - ) - from datastack.heartbeat import ConnectionState, HeartbeatMonitor # type: ignore - from datastack.spool import DiskSpooler # type: ignore + import socketio # type: ignore except ImportError: - # Standalone fallback when `datastack_sdk` is copied/installed outside the monorepo - import csv - import hashlib - import importlib.util - import json - import logging - import os - import random - import threading - import time - import urllib.error - import urllib.request - import uuid - from dataclasses import dataclass - from datetime import datetime, timezone - from typing import Any, Callable, Dict, Iterable, List, Optional, Union - from urllib.parse import urlparse + socketio = None # type: ignore - try: - import socketio # type: ignore - except ImportError: - socketio = None # type: ignore +try: + import requests # type: ignore +except ImportError: + requests = None # type: ignore - try: - import requests # type: ignore - except ImportError: - requests = None # type: ignore +from .config import DataStackConfig, load_config +from .exceptions import ( + DataStackAuthError, + DataStackConfigError, + DataStackConnectionError, + DataStackError, + DataStackFileError, + DataStackOfflineSpoolError, + DataStackPayloadTooLargeError, + DataStackRateLimitError, +) +from .heartbeat import ConnectionState, HeartbeatMonitor +from .spool import DiskSpooler - logger = logging.getLogger("datastack_sdk") +logger = logging.getLogger("datastack.client") - class DataStackError(Exception): - def __init__( - self, - message: str, - code: Optional[str] = None, - retryable: bool = False, - ) -> None: - super().__init__(message) - self.message = message - self.code = code or "DATASTACK_ERROR" - self.retryable = retryable - class DataStackConfigError(DataStackError): - def __init__(self, message: str) -> None: - super().__init__(message, code="CONFIG_ERROR", retryable=False) +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") - class DataStackAuthError(DataStackError): - def __init__(self, message: str, code: str = "AUTH_ERROR") -> None: - super().__init__(message, code=code, retryable=False) - class DataStackRateLimitError(DataStackError): - def __init__(self, message: str) -> None: - super().__init__( - message, code="RATE_LIMIT_EXCEEDED", retryable=True - ) +def _make_event_id() -> str: + return f"evt_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}" - class DataStackPayloadTooLargeError(DataStackError): - def __init__(self, message: str) -> None: - super().__init__( - message, code="PAYLOAD_TOO_LARGE", retryable=False - ) - class DataStackConnectionError(DataStackError): - def __init__(self, message: str) -> None: - super().__init__(message, code="CONNECTION_ERROR", retryable=True) +def _make_deterministic_file_event_id( + file_stem: str, row_index: int, record: Any +) -> str: + raw = f"{file_stem}:{row_index}:{json.dumps(record, sort_keys=True, default=str, ensure_ascii=False)}" + digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:12] + safe_stem = "".join( + ch if ch.isalnum() or ch in ("_", "-") else "_" for ch in file_stem + )[:32] + return f"file_{safe_stem}_{row_index}_{digest}" - class DataStackFileError(DataStackError): - def __init__(self, message: str) -> None: - super().__init__(message, code="FILE_ERROR", retryable=False) - class DataStackOfflineSpoolError(DataStackError): - def __init__(self, message: str) -> None: - super().__init__( - message, code="OFFLINE_SPOOL_ERROR", retryable=False - ) +class DataStackClient: + """ + Production-grade DataStack Python Client with `config.datastack.py` auto-loading, + WebSocket + Heartbeat HTTP Fallback + Offline Spooling + Direct File Streaming. + """ - @dataclass - class DataStackConfig: - server_url: str = "http://localhost:4000" - api_url: str = "http://localhost:3000" - client_token: str = "" - client_id: str = "" - auto_connect: bool = True - enable_http_fallback: bool = True - heartbeat_interval_sec: float = 5.0 - request_timeout_sec: float = 8.0 - max_retries: int = 3 - retry_backoff_base_sec: float = 0.5 - batch_chunk_size: int = 250 - offline_spool_enabled: bool = True - offline_spool_dir: str = ".datastack_spool" - max_spool_size_mb: int = 50 - config_source_path: Optional[str] = None - - def load_config( + def __init__( + self, + server_url: Optional[str] = None, + token: Optional[str] = None, + api_url: Optional[str] = None, config_path: Optional[str] = None, - require_token: bool = False, - **overrides: Any, - ) -> DataStackConfig: - server_url = str( - overrides.get("SERVER_URL") - or os.environ.get("DATASTACK_SERVER_URL") - or "http://localhost:4000" - ).rstrip("/") - parsed = urlparse(server_url) - default_api = ( - f"{parsed.scheme}://{parsed.hostname}:3000" - if parsed.scheme and parsed.hostname and parsed.port == 4000 - else "http://localhost:3000" + auto_connect: Optional[bool] = None, + reconnection_attempts: Optional[int] = None, + reconnection_delay: Optional[float] = None, + **kwargs: Any, + ) -> None: + if reconnection_attempts is not None and "MAX_RETRIES" not in kwargs: + kwargs["MAX_RETRIES"] = reconnection_attempts + if ( + reconnection_delay is not None + and "HEARTBEAT_INTERVAL_SEC" not in kwargs + ): + kwargs["HEARTBEAT_INTERVAL_SEC"] = reconnection_delay + + self.config: DataStackConfig = load_config( + config_path=config_path, + require_token=False, + SERVER_URL=server_url, + CLIENT_TOKEN=token, + API_URL=api_url, + AUTO_CONNECT=auto_connect, + **kwargs, ) - api_url = str( - overrides.get("API_URL") - or os.environ.get("DATASTACK_API_URL") - or default_api - ).rstrip("/") - token = str( - overrides.get("CLIENT_TOKEN") - or os.environ.get("DATASTACK_TOKEN") - or "" - ).strip() - auto_connect = overrides.get("AUTO_CONNECT") - cfg = DataStackConfig( - server_url=server_url, - api_url=api_url, - client_token=token, - auto_connect=True if auto_connect is None else bool(auto_connect), - config_source_path=config_path, + + self.server_url = self.config.server_url + self.api_url = self.config.api_url + self.token = self.config.client_token + self.is_connected = False + self._lock = threading.RLock() + self._is_draining_spool = False + + # Offline Disk Spooler + self.spooler = DiskSpooler( + spool_dir=self.config.offline_spool_dir, + max_size_mb=self.config.max_spool_size_mb, + enabled=self.config.offline_spool_enabled, ) - if require_token and not cfg.client_token: - raise DataStackConfigError("Missing CLIENT_TOKEN") - return cfg - class ConnectionState: - CONNECTED_IDLE = "CONNECTED_IDLE" - FALLBACK_HTTP = "FALLBACK_HTTP" - OFFLINE_SPOOL = "OFFLINE_SPOOL" - RECONNECTING_VERIFY = "RECONNECTING_VERIFY" - STOPPED = "STOPPED" + # Heartbeat Monitor (starts in IDLE once connected, wakes on disconnect/offline) + self.heartbeat = HeartbeatMonitor( + interval_sec=self.config.heartbeat_interval_sec, + recovery_callback=self._heartbeat_recovery_step, + ) - class DiskSpooler: - def __init__( - self, - spool_dir: str = ".datastack_spool", - max_size_mb: int = 50, - enabled: bool = True, - ) -> None: - self.enabled = enabled - self.spool_dir = Path(spool_dir) - self.spool_file = self.spool_dir / "spool.jsonl" - self.max_bytes = max(1, int(max_size_mb)) * 1024 * 1024 - self._lock = threading.RLock() + # Socket.IO Client (disable built-in infinite loop so HeartbeatMonitor coordinates fallback + IDLE) + self.sio = ( + socketio.Client( + reconnection=False, + logger=False, + engineio_logger=False, + ) + if socketio is not None + else None + ) + if self.sio is not None: + self._register_socket_events() - def enqueue(self, events: List[Dict[str, Any]]) -> int: - if not self.enabled or not events: - return 0 - with self._lock: - self.spool_dir.mkdir(parents=True, exist_ok=True) - with self.spool_file.open("a", encoding="utf-8") as fh: - for evt in events: - fh.write(json.dumps(evt, ensure_ascii=False) + "\n") - return len(events) - - def peek_batch(self, limit: int = 250) -> List[Dict[str, Any]]: - if not self.enabled or not self.spool_file.exists(): - return [] - with self._lock: - out: List[Dict[str, Any]] = [] - with self.spool_file.open("r", encoding="utf-8") as fh: - for line in fh: - if len(out) >= limit: - break - if line.strip(): - out.append(json.loads(line)) - return out - - def ack_batch(self, count: int) -> int: - if not self.enabled or count <= 0 or not self.spool_file.exists(): - return 0 - with self._lock: - lines = [ - ln - for ln in self.spool_file.read_text( - encoding="utf-8" - ).splitlines(True) - if ln.strip() - ] - if count >= len(lines): - self.spool_file.unlink(missing_ok=True) - return len(lines) - self.spool_file.write_text( - "".join(lines[count:]), encoding="utf-8" - ) - return count - - def pending_count(self) -> int: - if not self.enabled or not self.spool_file.exists(): - return 0 - with self._lock: - return sum( - 1 - for ln in self.spool_file.read_text( - encoding="utf-8" - ).splitlines() - if ln.strip() + if ( + self.config.auto_connect + and self.token + and self.token != "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN" + ): + try: + self.connect() + except DataStackAuthError: + raise + except Exception as exc: + logger.warning( + "[DataStack] Initial connect entered fallback/recovery mode: %s", + exc, ) - def size_bytes(self) -> int: + def __enter__(self) -> "DataStackClient": + if ( + not self.is_connected + and self.token + and self.token != "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN" + ): + try: + self.connect() + except DataStackAuthError: + raise + except Exception: + pass + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + self.disconnect() + + def _register_socket_events(self) -> None: + if self.sio is None: + return + + @self.sio.event + def connect() -> None: + with self._lock: + self.is_connected = True + logger.info("[DataStack] WebSocket connected to %s", self.server_url) + + @self.sio.event + def connect_error(data: Any) -> None: + with self._lock: + self.is_connected = False + msg = str(data) + if self._is_non_retryable_auth_error(msg): + self.heartbeat.stop(reason=msg) + else: + self.heartbeat.trigger_fallback(reason=f"Socket connect_error: {msg}") + + @self.sio.event + def disconnect() -> None: + with self._lock: + was_connected = self.is_connected + self.is_connected = False + if was_connected and self.heartbeat.state != ConnectionState.STOPPED: + logger.warning( + "[DataStack] WebSocket disconnected. Activating Heartbeat HTTP Fallback." + ) + self.heartbeat.trigger_fallback(reason="WebSocket connection closed") + + @staticmethod + def _is_non_retryable_auth_error(message: str) -> bool: + upper = (message or "").upper() + return any( + code in upper + for code in ( + "AUTH_REQUIRED", + "AUTH_INVALID", + "CLIENT_REVOKED", + "TOKEN_ROTATED", + ) + ) + + def _ensure_token_present(self) -> None: + if not self.token or self.token == "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN": + raise DataStackConfigError( + "Missing CLIENT_TOKEN. Set CLIENT_TOKEN in `config.datastack.py` or pass token='...'." + ) + + def connect(self, wait_timeout: Optional[float] = None) -> bool: + """ + Establishes the primary WebSocket connection on Port 4000 and verifies the + handshake via `ping_verify`. If WebSocket is unavailable and `ENABLE_HTTP_FALLBACK` + is True, automatically enters `FALLBACK_HTTP` (or `OFFLINE_SPOOL` if offline) + and starts the background Heartbeat Monitor. + """ + self._ensure_token_present() + self.heartbeat.start() + + try: + if self._connect_and_verify_websocket(wait_timeout=wait_timeout): + self._drain_offline_spool() + self.heartbeat.set_idle_connected() + return True + except DataStackAuthError as auth_err: + self.heartbeat.stop(reason=str(auth_err)) + raise + except Exception as exc: + logger.warning("[DataStack] WebSocket connect failed: %s", exc) + + # WebSocket was not reachable; check if HTTP fallback API is reachable + if self.config.enable_http_fallback: + if self._check_http_health(): + self.heartbeat.trigger_fallback( + reason="WebSocket unreachable on connect; using HTTP fallback" + ) + self._drain_offline_spool() + return True + + if self.config.offline_spool_enabled: + self.heartbeat.trigger_offline( + reason="Both WebSocket and HTTP endpoints unreachable on connect" + ) + return False + + raise DataStackConnectionError( + f"Unable to connect to DataStack WebSocket ({self.server_url}) or HTTP API ({self.api_url})." + ) + + def disconnect(self) -> None: + """Gracefully stops the Heartbeat Monitor and disconnects the WebSocket.""" + self.heartbeat.stop(reason="Client closed by application") + with self._lock: + self.is_connected = False + if self.sio is not None and self.sio.connected: + try: + self.sio.disconnect() + except Exception: + pass + + def _connect_and_verify_websocket( + self, wait_timeout: Optional[float] = None + ) -> bool: + if self.sio is None: + return False + + effective_timeout = ( + float(wait_timeout) + if wait_timeout is not None + else self.config.request_timeout_sec + ) + + with self._lock: + self.is_connected = False + if self.sio.connected: + try: + self.sio.disconnect() + except Exception: + pass + + try: + self.sio.connect( + self.server_url, + auth={"token": self.token}, + headers={"Authorization": f"Bearer {self.token}"}, + transports=["websocket", "polling"], + wait_timeout=effective_timeout, + ) + self.is_connected = True + except Exception as exc: + self.is_connected = False + msg = str(exc) + if self._is_non_retryable_auth_error(msg): + raise DataStackAuthError(msg) from exc + return False + + # Verify handshake & PostgreSQL client status via `ping_verify` + return self._verify_websocket_handshake(timeout=effective_timeout) + + def _verify_websocket_handshake( + self, timeout: Optional[float] = None + ) -> bool: + if self.sio is None or not self.sio.connected: + return False + + ack_event = threading.Event() + result_holder: Dict[str, Any] = {} + + def on_verify(resp: Dict[str, Any]) -> None: + if isinstance(resp, dict): + result_holder.update(resp) + ack_event.set() + + wait_sec = timeout or self.config.request_timeout_sec + try: + self.sio.emit("ping_verify", {}, callback=on_verify) + if not ack_event.wait(timeout=wait_sec): + return False + except Exception: + return False + + if not result_holder.get("success"): + err_msg = str( + result_holder.get("error") or "Handshake verification failed" + ) + if self._is_non_retryable_auth_error(err_msg): + raise DataStackAuthError(err_msg) + return False + + return result_holder.get("status") == "VERIFIED" + + def _heartbeat_recovery_step(self) -> bool: + """ + Invoked periodically by `HeartbeatMonitor` ONLY when not in `CONNECTED_IDLE` mode. + 1. Attempts to reconnect and verify (`ping_verify`) the primary WebSocket. + 2. If WebSocket is verified, drains any offline spooled events and returns True + so `HeartbeatMonitor` transitions back to `CONNECTED_IDLE` (IDLE mode). + 3. If WebSocket is still down, checks if HTTP fallback (`Port 3000`) is online: + - If HTTP is online, drains offline spooled events over HTTP and remains in `FALLBACK_HTTP`. + - If HTTP returns `401/403` auth revocation, stops Heartbeat immediately. + - If HTTP is also unreachable, sets state to `OFFLINE_SPOOL`. + """ + self.heartbeat.set_reconnecting() + try: + if self._connect_and_verify_websocket(): + self._drain_offline_spool() + return True + except DataStackAuthError as auth_err: + self.heartbeat.stop(reason=str(auth_err)) + return False + except Exception: + pass + + # WebSocket still unavailable; check HTTP availability + try: + if self.config.enable_http_fallback and self._check_http_health(): + self.heartbeat.trigger_fallback( + reason="WebSocket still unavailable; HTTP fallback active" + ) + self._drain_offline_spool() + return False + except DataStackAuthError as auth_err: + self.heartbeat.stop(reason=str(auth_err)) + return False + + self.heartbeat.trigger_offline( + reason="Device offline: WebSocket and HTTP endpoints unreachable" + ) + return False + + def _check_http_health(self) -> bool: + """ + Probes `GET {API_URL}/api/health` with the client's Bearer JWT token. + Raises `DataStackAuthError` and halts Heartbeat if client credentials were + revoked or rotated (`401` / `403`). Returns `True` ONLY on `2xx` with + `status == 'online'`. + """ + url = f"{self.api_url}/api/health" + try: + status_code, body = self._http_request( + "GET", url, payload=None, timeout=3.0 + ) + except Exception: + return False + + err_code = str(body.get("code") or "") + err_msg = str(body.get("error") or f"HTTP {status_code}") + + if status_code in (401, 403) or self._is_non_retryable_auth_error( + f"{err_code} {err_msg}" + ): + self.heartbeat.stop(reason=err_msg) + raise DataStackAuthError(err_msg, code=err_code or "AUTH_ERROR") + + return 200 <= status_code < 300 and body.get("status") == "online" + + def _http_request( + self, + method: str, + url: str, + payload: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + ) -> tuple[int, Dict[str, Any]]: + req_timeout = timeout or self.config.request_timeout_sec + headers = { + "Authorization": f"Bearer {self.token}", + "Content-Type": "application/json", + "Accept": "application/json", + } + + if requests is not None: + resp = requests.request( + method=method, + url=url, + json=payload, + headers=headers, + timeout=req_timeout, + ) + try: + body = resp.json() if resp.text else {} + except Exception: + body = {"error": resp.text} + return resp.status_code, ( + body if isinstance(body, dict) else {"data": body} + ) + + # Standard library urllib fallback when `requests` is not installed + data_bytes = ( + json.dumps(payload, ensure_ascii=False).encode("utf-8") + if payload is not None + else None + ) + req = urllib.request.Request( + url=url, data=data_bytes, headers=headers, method=method + ) + try: + with urllib.request.urlopen(req, timeout=req_timeout) as response: + raw = response.read().decode("utf-8") + parsed = json.loads(raw) if raw else {} + return ( + response.status, + parsed if isinstance(parsed, dict) else {"data": parsed}, + ) + except urllib.error.HTTPError as http_err: + raw_err = http_err.read().decode("utf-8", errors="replace") + try: + parsed_err = json.loads(raw_err) if raw_err else {} + except Exception: + parsed_err = {"error": raw_err or str(http_err)} return ( - self.spool_file.stat().st_size - if self.spool_file.exists() - else 0 + http_err.code, + parsed_err + if isinstance(parsed_err, dict) + else {"error": str(parsed_err)}, ) - class HeartbeatMonitor: - def __init__( - self, interval_sec: float, recovery_callback: Callable[[], bool] - ) -> None: - self.interval_sec = max(0.2, float(interval_sec)) - self._recovery_callback = recovery_callback - self._state = ConnectionState.CONNECTED_IDLE - self._closed = False - self._last_error: Optional[str] = None - self._recovery_attempts = 0 - self._last_verified_at: Optional[float] = None - self._lock = threading.RLock() + def _send_via_http_fallback( + self, payload: Dict[str, Any] + ) -> Dict[str, Any]: + """ + Sends a single event, batch array (`{ batch: [...] }`), or file records + (`{ fileName, records: [...] }`) over HTTP `POST {API_URL}/api/events`. + """ + url = f"{self.api_url}/api/events" + status_code, body = self._http_request("POST", url, payload=payload) - @property - def state(self) -> str: - return self._state - - def set_idle_connected(self) -> None: - self._closed = False - self._state = ConnectionState.CONNECTED_IDLE - - def trigger_fallback(self, reason: str = "") -> None: - if not self._closed: - self._state = ConnectionState.FALLBACK_HTTP - self._last_error = reason - - def trigger_offline(self, reason: str = "") -> None: - if not self._closed: - self._state = ConnectionState.OFFLINE_SPOOL - self._last_error = reason - - def set_reconnecting(self) -> None: - if not self._closed: - self._state = ConnectionState.RECONNECTING_VERIFY - - def start(self) -> None: - self._closed = False - - def stop(self, reason: Optional[str] = None) -> None: - self._closed = True - self._state = ConnectionState.STOPPED - if reason: - self._last_error = reason - - def status_snapshot(self) -> Dict[str, Any]: + if 200 <= status_code < 300 and body.get("success", True): return { - "state": self._state, - "heartbeatMode": ( - "IDLE" - if self._state == ConnectionState.CONNECTED_IDLE - else "ACTIVE_RECOVERY" - ), - "recoveryAttempts": self._recovery_attempts, - "lastError": self._last_error, - "lastVerifiedAt": self._last_verified_at, - "intervalSec": self.interval_sec, + **body, + "success": True, + "transport": "HTTP_FALLBACK", } - class DataStackClient: # type: ignore[no-redef] - def __init__( - self, - server_url: Optional[str] = None, - token: Optional[str] = None, - api_url: Optional[str] = None, - config_path: Optional[str] = None, - auto_connect: Optional[bool] = None, - **kwargs: Any, - ) -> None: - self.config = load_config( - config_path=config_path, - SERVER_URL=server_url, - CLIENT_TOKEN=token, - API_URL=api_url, - AUTO_CONNECT=auto_connect, - **kwargs, + errors_list = body.get("errors") + first_err = ( + errors_list[0] + if isinstance(errors_list, list) and len(errors_list) > 0 + else None + ) + err_msg = str(body.get("error") or first_err or f"HTTP {status_code}") + err_code = str(body.get("code") or "") + combined = f"{err_code} {err_msg}" + + if status_code in (401, 403) or self._is_non_retryable_auth_error( + combined + ): + self.heartbeat.stop(reason=err_msg) + raise DataStackAuthError(err_msg, code=err_code or "AUTH_ERROR") + + if status_code == 413 or "PAYLOAD_TOO_LARGE" in combined: + raise DataStackPayloadTooLargeError(err_msg) + + if status_code == 429 or "RATE_LIMIT_EXCEEDED" in combined: + raise DataStackRateLimitError(err_msg) + + raise DataStackConnectionError( + f"HTTP fallback error ({status_code}): {err_msg}" + ) + + def _emit_via_websocket( + self, + event_name: str, + payload: Any, + timeout: Optional[float] = None, + ) -> Dict[str, Any]: + if self.sio is None or not self.is_connected or not self.sio.connected: + raise DataStackConnectionError( + "WebSocket is not currently connected" ) - self.server_url = self.config.server_url - self.api_url = self.config.api_url - self.token = self.config.client_token + + ack_event = threading.Event() + response_data: Dict[str, Any] = {} + + def ack_callback(res: Dict[str, Any]) -> None: + if isinstance(res, dict): + response_data.update(res) + ack_event.set() + + try: + self.sio.emit(event_name, payload, callback=ack_callback) + except Exception as exc: self.is_connected = False - self.spooler = DiskSpooler( - spool_dir=self.config.offline_spool_dir, - max_size_mb=self.config.max_spool_size_mb, - enabled=self.config.offline_spool_enabled, - ) - self.heartbeat = HeartbeatMonitor( - self.config.heartbeat_interval_sec, lambda: False + self.heartbeat.trigger_fallback( + reason=f"WebSocket emit error: {exc}" ) + raise DataStackConnectionError( + f"WebSocket emit failed: {exc}" + ) from exc - def __enter__(self) -> "DataStackClient": - return self - - def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: - self.disconnect() - - def connect(self, wait_timeout: Optional[float] = None) -> bool: - self.is_connected = True - self.heartbeat.set_idle_connected() - return True - - def disconnect(self) -> None: + wait_sec = timeout or self.config.request_timeout_sec + if not ack_event.wait(timeout=wait_sec): self.is_connected = False - self.heartbeat.stop() + self.heartbeat.trigger_fallback( + reason=f"WebSocket ACK timeout after {wait_sec}s" + ) + raise DataStackConnectionError( + f"WebSocket ACK timed out after {wait_sec}s" + ) - def send( - self, - data: Dict[str, Any], - event_id: Optional[str] = None, - timestamp: Optional[str] = None, - timeout: Optional[float] = None, - ) -> Dict[str, Any]: - evt = { - "eventId": event_id or f"evt_{uuid.uuid4().hex[:8]}", - "timestamp": timestamp - or datetime.now(timezone.utc) - .isoformat() - .replace("+00:00", "Z"), - "data": data, - } - self.spooler.enqueue([evt]) + if not response_data.get("success", False): + errors_list = response_data.get("errors") + first_err = ( + errors_list[0] + if isinstance(errors_list, list) and len(errors_list) > 0 + else None + ) + err_msg = str( + response_data.get("error") + or first_err + or "WebSocket event rejected" + ) + err_code = str(response_data.get("code") or "") + combined = f"{err_code} {err_msg}" + + if self._is_non_retryable_auth_error(combined): + self.heartbeat.stop(reason=err_msg) + raise DataStackAuthError(err_msg, code=err_code or "AUTH_ERROR") + if "PAYLOAD_TOO_LARGE" in combined: + raise DataStackPayloadTooLargeError(err_msg) + if "RATE_LIMIT_EXCEEDED" in combined: + raise DataStackRateLimitError(err_msg) + raise DataStackError(err_msg, code=err_code or "WS_REJECTED") + + return { + **response_data, + "transport": "WEBSOCKET", + } + + def _execute_with_retry_and_fallback( + self, + ws_event: str, + ws_payload: Any, + http_payload: Dict[str, Any], + spool_items: List[Dict[str, Any]], + timeout: Optional[float] = None, + allow_spool: bool = True, + ) -> Dict[str, Any]: + """ + Core resilient dispatcher: + 1. Attempts WebSocket if connected (`CONNECTED_IDLE`). + 2. On WebSocket failure/disconnect, triggers `HeartbeatMonitor` (`FALLBACK_HTTP`) + and automatically sends via HTTP `POST /api/events`. + 3. Applies exponential backoff with jitter on `RATE_LIMIT_EXCEEDED` or transient errors. + 4. If both WebSocket and HTTP are unreachable (device offline) and `allow_spool` is True, + spools events to `.datastack_spool/spool.jsonl` and triggers `OFFLINE_SPOOL`. + """ + self._ensure_token_present() + max_attempts = max(1, self.config.max_retries + 1) + last_exception: Optional[Exception] = None + + for attempt in range(max_attempts): + if attempt > 0: + base_delay = ( + max(1.0, self.config.retry_backoff_base_sec) + if isinstance(last_exception, DataStackRateLimitError) + else self.config.retry_backoff_base_sec + ) + delay = base_delay * (2 ** (attempt - 1)) + jitter = random.uniform(0.0, 0.2 * delay) + time.sleep(delay + jitter) + + # 1. Try Primary WebSocket if connected + if self.is_connected and self.sio is not None and self.sio.connected: + try: + res = self._emit_via_websocket( + ws_event, ws_payload, timeout=timeout + ) + if ( + allow_spool + and not self._is_draining_spool + and self.spooler.pending_count() > 0 + ): + self._drain_offline_spool() + return res + except (DataStackAuthError, DataStackPayloadTooLargeError): + raise + except DataStackRateLimitError as rl_err: + last_exception = rl_err + continue + except Exception as ws_err: + last_exception = ws_err + self.heartbeat.trigger_fallback(reason=str(ws_err)) + + # 2. Try HTTP Fallback (`POST /api/events`) + if self.config.enable_http_fallback: + try: + res = self._send_via_http_fallback(http_payload) + if self.heartbeat.state in ( + ConnectionState.CONNECTED_IDLE, + ConnectionState.OFFLINE_SPOOL, + ): + self.heartbeat.trigger_fallback( + reason="WebSocket not connected; routing via HTTP fallback" + ) + if ( + allow_spool + and not self._is_draining_spool + and self.spooler.pending_count() > 0 + ): + self._drain_offline_spool() + return res + except (DataStackAuthError, DataStackPayloadTooLargeError): + raise + except DataStackRateLimitError as rl_err: + last_exception = rl_err + continue + except Exception as http_err: + last_exception = http_err + + # 3. If rate-limited or payload-rejected after all retries, raise directly + if isinstance( + last_exception, + (DataStackRateLimitError, DataStackPayloadTooLargeError), + ): + raise last_exception + + # 4. Device is offline (both WebSocket and HTTP failed): Spool to local disk + if ( + allow_spool + and self.config.offline_spool_enabled + and spool_items + ): + spooled_count = self.spooler.enqueue(spool_items) + self.heartbeat.trigger_offline( + reason=str(last_exception or "Network unreachable") + ) return { "success": True, "spooled": True, - "spooledCount": 1, + "spooledCount": spooled_count, + "pendingSpoolTotal": self.spooler.pending_count(), "transport": "OFFLINE_SPOOL", } - def sync( - self, - data: Dict[str, Any], - event_id: Optional[str] = None, - timeout: float = 5.0, - ) -> Dict[str, Any]: - return self.send(data=data, event_id=event_id, timeout=timeout) + raise DataStackConnectionError( + f"DataStack delivery failed after {max_attempts} attempts: {last_exception}" + ) - def send_batch( - self, - events: Optional[List[Dict[str, Any]]] = None, - timeout: Optional[float] = None, - file_name: Optional[str] = None, - *, - records: Optional[List[Dict[str, Any]]] = None, - ) -> Dict[str, Any]: - items = events if events is not None else (records or []) - self.spooler.enqueue(items) - return { - "success": True, - "processed": 0, - "spooledCount": len(items), - "transport": "OFFLINE_SPOOL", - } + def _drain_offline_spool(self) -> int: + """ + Replays any offline spooled events from `.datastack_spool/spool.jsonl` + once connectivity (WebSocket or HTTP fallback) is restored. + Routes batches through `_send_formatted_batch_recursive(..., allow_spool=False)` + and preserves any `unprocessedItems` so zero spooled events are ever lost. + """ + if not self.config.offline_spool_enabled: + return 0 - def batch_sync( - self, - events: Optional[List[Dict[str, Any]]] = None, - timeout: float = 10.0, - *, - records: Optional[List[Dict[str, Any]]] = None, - ) -> Dict[str, Any]: - return self.send_batch( - events=events, records=records, timeout=timeout + with self._lock: + if self._is_draining_spool: + return 0 + self._is_draining_spool = True + + total_flushed = 0 + try: + while True: + batch = self.spooler.peek_batch( + limit=self.config.batch_chunk_size + ) + if not batch: + break + + res = self._send_formatted_batch_recursive( + batch, + timeout=self.config.request_timeout_sec, + file_name=None, + allow_spool=False, + ) + processed_now = int(res.get("processed", 0)) + duplicates_now = int(res.get("duplicates", 0)) + unprocessed = res.get("unprocessedItems") + + if ( + not res.get("success", False) + and processed_now == 0 + and duplicates_now == 0 + ): + break + + self.spooler.ack_batch(len(batch)) + if isinstance(unprocessed, list) and len(unprocessed) > 0: + self.spooler.enqueue(unprocessed) + total_flushed += max(0, len(batch) - len(unprocessed)) + break + + total_flushed += len(batch) + except Exception as exc: + logger.debug("[DataStack] Spool drain paused: %s", exc) + finally: + with self._lock: + self._is_draining_spool = False + + return total_flushed + + # ========================================================================= + # Public Telemetry Ingestion API + # ========================================================================= + + def send( + self, + data: Dict[str, Any], + event_id: Optional[str] = None, + timestamp: Optional[str] = None, + timeout: Optional[float] = None, + ) -> Dict[str, Any]: + """ + Sends a single telemetry event. Automatically falls back to HTTP if WebSocket + is disconnected, or spools to disk if the device is offline. + """ + resolved_id = (event_id or "").strip() or _make_event_id() + resolved_ts = (timestamp or "").strip() or _utc_now_iso() + envelope = { + "eventId": resolved_id, + "timestamp": resolved_ts, + "data": data, + } + return self._execute_with_retry_and_fallback( + ws_event="sync_data", + ws_payload=envelope, + http_payload=envelope, + spool_items=[envelope], + timeout=timeout, + allow_spool=True, + ) + + def sync( + self, + data: Dict[str, Any], + event_id: Optional[str] = None, + timeout: float = 5.0, + ) -> Dict[str, Any]: + """Alias for `send()` preserving backward compatibility.""" + return self.send(data=data, event_id=event_id, timeout=timeout) + + def send_batch( + self, + events: Optional[List[Dict[str, Any]]] = None, + timeout: Optional[float] = None, + file_name: Optional[str] = None, + *, + records: Optional[List[Dict[str, Any]]] = None, + ) -> Dict[str, Any]: + """ + Sends a batch of events (accepts either `events=[...]` or `records=[...]`). + - If the batch triggers `413 PAYLOAD_TOO_LARGE`, automatically splits the batch + in half recursively and retries. + - If a subset of events in the batch hits `429 RATE_LIMIT_EXCEEDED` + (`rateLimitedEventIds`), automatically backs off and retries the remaining + rate-limited events so zero records are dropped. + """ + source_list = events if events is not None else records + if not isinstance(source_list, list) or len(source_list) == 0: + return {"success": True, "processed": 0, "rejected": 0} + + formatted: List[Dict[str, Any]] = [] + for item in source_list: + safe_item = item if isinstance(item, dict) else {"value": item} + evt_id = ( + str(safe_item.get("eventId") or "").strip() or _make_event_id() + ) + evt_ts = ( + str(safe_item.get("timestamp") or "").strip() or _utc_now_iso() + ) + evt_data = safe_item.get("data", safe_item) + formatted.append( + { + "eventId": evt_id, + "timestamp": evt_ts, + "data": evt_data, + } ) - DataStack = DataStackClient + return self._send_formatted_batch_recursive( + formatted, + timeout=timeout, + file_name=file_name, + allow_spool=True, + ) + + def batch_sync( + self, + events: Optional[List[Dict[str, Any]]] = None, + timeout: float = 10.0, + *, + records: Optional[List[Dict[str, Any]]] = None, + ) -> Dict[str, Any]: + """Alias for `send_batch()` supporting both `events` and `records` kwargs.""" + return self.send_batch(events=events, records=records, timeout=timeout) + + def _send_formatted_batch_recursive( + self, + formatted_batch: List[Dict[str, Any]], + timeout: Optional[float] = None, + file_name: Optional[str] = None, + allow_spool: bool = True, + ) -> Dict[str, Any]: + http_payload: Dict[str, Any] = ( + {"fileName": file_name, "records": formatted_batch} + if file_name + else {"batch": formatted_batch} + ) + try: + res = self._execute_with_retry_and_fallback( + ws_event="batch_sync", + ws_payload=formatted_batch, + http_payload=http_payload, + spool_items=formatted_batch, + timeout=timeout, + allow_spool=allow_spool, + ) + except DataStackPayloadTooLargeError: + if len(formatted_batch) <= 1: + raise + mid = len(formatted_batch) // 2 + left_res = self._send_formatted_batch_recursive( + formatted_batch[:mid], + timeout=timeout, + file_name=file_name, + allow_spool=allow_spool, + ) + right_res = self._send_formatted_batch_recursive( + formatted_batch[mid:], + timeout=timeout, + file_name=file_name, + allow_spool=allow_spool, + ) + left_unprocessed = left_res.get("unprocessedItems") or [] + right_unprocessed = right_res.get("unprocessedItems") or [] + combined_unprocessed = [ + *(left_unprocessed if isinstance(left_unprocessed, list) else []), + *(right_unprocessed if isinstance(right_unprocessed, list) else []), + ] + return { + "success": bool( + left_res.get("success", True) + and right_res.get("success", True) + ), + "processed": int(left_res.get("processed", 0)) + + int(right_res.get("processed", 0)), + "rejected": int(left_res.get("rejected", 0)) + + int(right_res.get("rejected", 0)), + "duplicates": int(left_res.get("duplicates", 0)) + + int(right_res.get("duplicates", 0)), + "spooledCount": int(left_res.get("spooledCount", 0)) + + int(right_res.get("spooledCount", 0)), + "transport": right_res.get("transport") + or left_res.get("transport"), + **( + {"unprocessedItems": combined_unprocessed} + if len(combined_unprocessed) > 0 + else {} + ), + } + + # Handle partial-batch rate limiting (`rateLimitedEventIds`) so zero rows are dropped + rate_limited_ids = res.get("rateLimitedEventIds") + if isinstance(rate_limited_ids, list) and len(rate_limited_ids) > 0: + pending_id_set = {str(eid) for eid in rate_limited_ids} + remaining_items = [ + item + for item in formatted_batch + if str(item.get("eventId")) in pending_id_set + ] + total_processed = int(res.get("processed", 0)) + total_duplicates = int(res.get("duplicates", 0)) + total_spooled = int(res.get("spooledCount", 0)) + last_transport = res.get("transport") + + if self.config.max_retries > 0: + for _retry_round in range(1, self.config.max_retries + 2): + if not remaining_items: + break + wait_delay = max( + 1.05, self.config.retry_backoff_base_sec + ) + random.uniform(0.0, 0.15) + time.sleep(wait_delay) + + sub_http_payload: Dict[str, Any] = ( + {"fileName": file_name, "records": remaining_items} + if file_name + else {"batch": remaining_items} + ) + sub_res = self._execute_with_retry_and_fallback( + ws_event="batch_sync", + ws_payload=remaining_items, + http_payload=sub_http_payload, + spool_items=remaining_items, + timeout=timeout, + allow_spool=allow_spool, + ) + total_processed += int(sub_res.get("processed", 0)) + total_duplicates += int(sub_res.get("duplicates", 0)) + total_spooled += int(sub_res.get("spooledCount", 0)) + last_transport = sub_res.get("transport") or last_transport + + if sub_res.get("spooled"): + remaining_items = [] + break + + next_rl_ids = sub_res.get("rateLimitedEventIds") + if ( + not isinstance(next_rl_ids, list) + or len(next_rl_ids) == 0 + ): + remaining_items = [] + break + next_set = {str(eid) for eid in next_rl_ids} + remaining_items = [ + item + for item in remaining_items + if str(item.get("eventId")) in next_set + ] + + # If any items still remain rate-limited after retries and spooling is allowed, spool them so zero data is lost + if ( + remaining_items + and allow_spool + and self.config.offline_spool_enabled + ): + spooled_now = self.spooler.enqueue(remaining_items) + total_spooled += spooled_now + remaining_items = [] + + oversized_ids = res.get("oversizedEventIds") + final_rejected = len(remaining_items) + ( + len(oversized_ids) if isinstance(oversized_ids, list) else 0 + ) + out: Dict[str, Any] = { + "success": final_rejected == 0 + or total_processed > 0 + or total_spooled > 0, + "processed": total_processed, + "rejected": final_rejected, + "duplicates": total_duplicates, + "spooledCount": total_spooled, + "transport": last_transport, + } + if remaining_items: + out["unprocessedItems"] = remaining_items + return out + + return res + + # ========================================================================= + # Direct File Streaming (.csv, .json, .jsonl / .ndjson) + # ========================================================================= + + def send_file( + self, + file_path: Union[str, Path], + chunk_size: Optional[int] = None, + extra_metadata: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + """ + Streams a `.csv`, `.json`, or `.jsonl` (`.ndjson`) file directly to DataStack + in safe chunks with deterministic SHA-256 idempotency IDs, automatic HTTP fallback, + partial-batch rate-limit pacing, and offline spooling support. + """ + path = Path(file_path).expanduser().resolve() + if not path.exists() or not path.is_file(): + raise DataStackFileError(f"File not found: {path}") + + ext = path.suffix.lower() + if ext not in (".csv", ".json", ".jsonl", ".ndjson"): + raise DataStackFileError( + f"Unsupported file extension '{ext}'. Supported formats: .csv, .json, .jsonl, .ndjson" + ) + + effective_chunk = max( + 1, int(chunk_size or self.config.batch_chunk_size) + ) + total_processed = 0 + total_rejected = 0 + total_duplicates = 0 + total_spooled = 0 + chunks_sent = 0 + last_transport = "WEBSOCKET" + + buffer: List[Dict[str, Any]] = [] + for idx, record in enumerate(self._iter_file_records(path, ext), start=1): + enriched = ( + {**extra_metadata, **record} + if extra_metadata and isinstance(record, dict) + else record + ) + existing_id = ( + str(enriched.get("eventId") or "").strip() + if isinstance(enriched, dict) + else "" + ) + evt_id = existing_id or _make_deterministic_file_event_id( + path.stem, idx, enriched + ) + existing_ts = ( + str(enriched.get("timestamp") or "").strip() + if isinstance(enriched, dict) + else "" + ) + evt_ts = existing_ts or _utc_now_iso() + + buffer.append( + { + "eventId": evt_id, + "timestamp": evt_ts, + "data": enriched, + } + ) + if len(buffer) >= effective_chunk: + res = self.send_batch(buffer, file_name=path.name) + total_processed += int(res.get("processed", 0)) + total_rejected += int(res.get("rejected", 0)) + total_duplicates += int(res.get("duplicates", 0)) + total_spooled += int(res.get("spooledCount", 0)) + last_transport = str(res.get("transport", last_transport)) + chunks_sent += 1 + buffer = [] + + if buffer: + res = self.send_batch(buffer, file_name=path.name) + total_processed += int(res.get("processed", 0)) + total_rejected += int(res.get("rejected", 0)) + total_duplicates += int(res.get("duplicates", 0)) + total_spooled += int(res.get("spooledCount", 0)) + last_transport = str(res.get("transport", last_transport)) + chunks_sent += 1 + + return { + "success": True, + "fileName": path.name, + "fileType": ext.lstrip("."), + "chunksSent": chunks_sent, + "processed": total_processed, + "rejected": total_rejected, + "duplicates": total_duplicates, + "spooledCount": total_spooled, + "transport": last_transport, + } + + @staticmethod + def _iter_file_records(path: Path, ext: str) -> Iterable[Dict[str, Any]]: + try: + if ext == ".csv": + with path.open("r", encoding="utf-8-sig", newline="") as fh: + reader = csv.DictReader(fh) + for row in reader: + yield dict(row) + elif ext in (".jsonl", ".ndjson"): + with path.open("r", encoding="utf-8-sig") as fh: + for line_no, line in enumerate(fh, start=1): + stripped = line.strip() + if not stripped: + continue + try: + parsed = json.loads(stripped) + except json.JSONDecodeError as exc: + raise DataStackFileError( + f"Malformed JSONL at line {line_no} in {path.name}: {exc}" + ) from exc + yield ( + parsed + if isinstance(parsed, dict) + else {"value": parsed} + ) + elif ext == ".json": + with path.open("r", encoding="utf-8-sig") as fh: + parsed = json.load(fh) + if isinstance(parsed, list): + for item in parsed: + yield ( + item if isinstance(item, dict) else {"value": item} + ) + elif isinstance(parsed, dict): + if isinstance(parsed.get("records"), list): + for item in parsed["records"]: + yield ( + item + if isinstance(item, dict) + else {"value": item} + ) + elif isinstance(parsed.get("data"), list): + for item in parsed["data"]: + yield ( + item + if isinstance(item, dict) + else {"value": item} + ) + else: + yield parsed + else: + yield {"value": parsed} + except DataStackFileError: + raise + except Exception as exc: + raise DataStackFileError( + f"Failed reading file '{path.name}': {exc}" + ) from exc + + def status(self) -> Dict[str, Any]: + """ + Returns a complete diagnostic snapshot of the SDK connection state, + Heartbeat Monitor mode (`IDLE` vs `ACTIVE_RECOVERY`), and offline spool queue. + """ + hb = self.heartbeat.status_snapshot() + return { + "connectedWebSocket": self.is_connected, + "serverUrl": self.server_url, + "apiUrl": self.api_url, + "configSource": self.config.config_source_path, + "connectionState": hb["state"], + "heartbeatMode": hb["heartbeatMode"], + "recoveryAttempts": hb["recoveryAttempts"], + "lastError": hb["lastError"], + "offlineSpoolPending": self.spooler.pending_count(), + "offlineSpoolBytes": self.spooler.size_bytes(), + } + + +DataStack = DataStackClient __all__ = [ "DataStack", diff --git a/sdk-python/datastack_sdk/config.py b/sdk-python/datastack_sdk/config.py new file mode 100644 index 0000000..76c272f --- /dev/null +++ b/sdk-python/datastack_sdk/config.py @@ -0,0 +1,321 @@ +""" +Auto-Discovery Configuration Loader for `config.datastack.py`. +""" + +import importlib.util +import os +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List, Optional +from urllib.parse import urlparse + +from .exceptions import DataStackConfigError + +CONFIG_FILENAME = "config.datastack.py" + + +@dataclass +class DataStackConfig: + server_url: str = "http://localhost:4000" + api_url: str = "http://localhost:3000" + client_token: str = "" + client_id: str = "" + auto_connect: bool = True + enable_http_fallback: bool = True + heartbeat_interval_sec: float = 5.0 + request_timeout_sec: float = 8.0 + max_retries: int = 3 + retry_backoff_base_sec: float = 0.5 + batch_chunk_size: int = 250 + offline_spool_enabled: bool = True + offline_spool_dir: str = ".datastack_spool" + max_spool_size_mb: int = 50 + config_source_path: Optional[str] = None + + +def _derive_default_api_url(server_url: str) -> str: + """ + If API_URL is not explicitly set, infer Port 3000 on the same host when SERVER_URL + uses Port 4000, or fall back to http://localhost:3000. + """ + try: + parsed = urlparse(server_url) + if parsed.scheme and parsed.hostname: + if parsed.port == 4000: + return f"{parsed.scheme}://{parsed.hostname}:3000" + return f"{parsed.scheme}://{parsed.netloc}" + except Exception: + pass + return "http://localhost:3000" + + +def _candidate_config_paths(explicit_path: Optional[str] = None) -> List[Path]: + candidates: List[Path] = [] + + if explicit_path: + p = Path(explicit_path).expanduser().resolve() + if p.is_dir(): + candidates.append(p / CONFIG_FILENAME) + else: + candidates.append(p) + + env_path = os.environ.get("DATASTACK_CONFIG_PATH", "").strip() + if env_path: + ep = Path(env_path).expanduser().resolve() + if ep.is_dir(): + candidates.append(ep / CONFIG_FILENAME) + else: + candidates.append(ep) + + # Search current working directory and up to 3 parent levels + try: + cwd = Path.cwd().resolve() + candidates.append(cwd / CONFIG_FILENAME) + for parent in list(cwd.parents)[:3]: + candidates.append(parent / CONFIG_FILENAME) + except Exception: + pass + + # Search main entry script directory + try: + if sys.argv and sys.argv[0]: + script_dir = Path(sys.argv[0]).resolve().parent + candidates.append(script_dir / CONFIG_FILENAME) + for parent in list(script_dir.parents)[:2]: + candidates.append(parent / CONFIG_FILENAME) + except Exception: + pass + + # Deduplicate while preserving order + seen = set() + unique: List[Path] = [] + for c in candidates: + key = str(c) + if key not in seen: + seen.add(key) + unique.append(c) + return unique + + +def _load_python_config_file(file_path: Path) -> Dict[str, Any]: + try: + spec = importlib.util.spec_from_file_location( + "datastack_user_config", str(file_path) + ) + if spec is None or spec.loader is None: + raise DataStackConfigError( + f"Unable to load configuration specification from {file_path}" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + raw: Dict[str, Any] = {} + if hasattr(module, "DATASTACK_CONFIG") and isinstance( + getattr(module, "DATASTACK_CONFIG"), dict + ): + raw.update(getattr(module, "DATASTACK_CONFIG")) + + # Also support top-level module variables (e.g., SERVER_URL = "...", CLIENT_TOKEN = "...") + for attr in dir(module): + if attr.isupper() and attr != "DATASTACK_CONFIG": + raw[attr] = getattr(module, attr) + + return raw + except DataStackConfigError: + raise + except Exception as exc: + raise DataStackConfigError( + f"Failed to parse {file_path}: {exc}" + ) from exc + + +def load_config( + config_path: Optional[str] = None, + require_token: bool = False, + **overrides: Any, +) -> DataStackConfig: + """ + Discovers and loads `config.datastack.py` from the client application's + directory, merges environment variables (`DATASTACK_*`), and applies + any explicit keyword overrides. + """ + file_values: Dict[str, Any] = {} + resolved_source: Optional[str] = None + + for candidate in _candidate_config_paths(config_path): + if candidate.is_file(): + file_values = _load_python_config_file(candidate) + resolved_source = str(candidate) + break + + if config_path and not resolved_source: + raise DataStackConfigError( + f"Explicitly requested config file was not found: {config_path}" + ) + + # Normalize keys to uppercase for lookup + normalized_file = {str(k).upper(): v for k, v in file_values.items()} + normalized_overrides = { + str(k).upper(): v for k, v in overrides.items() if v is not None + } + + def pick(keys: List[str], env_keys: List[str], default: Any) -> Any: + for k in keys: + if k in normalized_overrides: + return normalized_overrides[k] + for ek in env_keys: + env_val = os.environ.get(ek) + if env_val is not None and str(env_val).strip() != "": + return env_val + for k in keys: + if k in normalized_file and normalized_file[k] is not None: + return normalized_file[k] + return default + + def to_bool(val: Any, default: bool) -> bool: + if isinstance(val, bool): + return val + if isinstance(val, str): + return val.strip().lower() in ("1", "true", "yes", "on") + if val is None: + return default + return bool(val) + + server_url = str( + pick( + ["SERVER_URL", "SOCKET_URL", "WS_URL"], + ["DATASTACK_SERVER_URL", "DATASTACK_SOCKET_URL"], + "http://localhost:4000", + ) + ).rstrip("/") + + default_api = _derive_default_api_url(server_url) + api_url = str( + pick( + ["API_URL", "HTTP_URL", "WEB_URL"], + ["DATASTACK_API_URL", "DATASTACK_HTTP_URL"], + default_api, + ) + ).rstrip("/") + + client_token = str( + pick( + ["CLIENT_TOKEN", "TOKEN", "JWT_TOKEN", "API_TOKEN"], + ["DATASTACK_CLIENT_TOKEN", "DATASTACK_TOKEN"], + "", + ) + ).strip() + + client_id = str( + pick( + ["CLIENT_ID"], + ["DATASTACK_CLIENT_ID"], + "", + ) + ).strip() + + cfg = DataStackConfig( + server_url=server_url, + api_url=api_url, + client_token=client_token, + client_id=client_id, + auto_connect=to_bool( + pick(["AUTO_CONNECT"], ["DATASTACK_AUTO_CONNECT"], True), True + ), + enable_http_fallback=to_bool( + pick( + ["ENABLE_HTTP_FALLBACK", "HTTP_FALLBACK"], + ["DATASTACK_ENABLE_HTTP_FALLBACK"], + True, + ), + True, + ), + heartbeat_interval_sec=max( + 1.0, + float( + pick( + ["HEARTBEAT_INTERVAL_SEC", "HEARTBEAT_INTERVAL"], + ["DATASTACK_HEARTBEAT_INTERVAL_SEC"], + 5.0, + ) + ), + ), + request_timeout_sec=max( + 1.0, + float( + pick( + ["REQUEST_TIMEOUT_SEC", "TIMEOUT"], + ["DATASTACK_REQUEST_TIMEOUT_SEC"], + 8.0, + ) + ), + ), + max_retries=max( + 0, + int( + pick( + ["MAX_RETRIES", "RETRIES"], + ["DATASTACK_MAX_RETRIES"], + 3, + ) + ), + ), + retry_backoff_base_sec=max( + 0.1, + float( + pick( + ["RETRY_BACKOFF_BASE_SEC"], + ["DATASTACK_RETRY_BACKOFF_BASE_SEC"], + 0.5, + ) + ), + ), + batch_chunk_size=max( + 1, + int( + pick( + ["BATCH_CHUNK_SIZE", "CHUNK_SIZE"], + ["DATASTACK_BATCH_CHUNK_SIZE"], + 250, + ) + ), + ), + offline_spool_enabled=to_bool( + pick( + ["OFFLINE_SPOOL_ENABLED"], + ["DATASTACK_OFFLINE_SPOOL_ENABLED"], + True, + ), + True, + ), + offline_spool_dir=str( + pick( + ["OFFLINE_SPOOL_DIR", "SPOOL_DIR"], + ["DATASTACK_OFFLINE_SPOOL_DIR"], + ".datastack_spool", + ) + ), + max_spool_size_mb=max( + 1, + int( + pick( + ["MAX_SPOOL_SIZE_MB"], + ["DATASTACK_MAX_SPOOL_SIZE_MB"], + 50, + ) + ), + ), + config_source_path=resolved_source, + ) + + if require_token and ( + not cfg.client_token + or cfg.client_token == "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN" + ): + raise DataStackConfigError( + "DataStack CLIENT_TOKEN is missing. Add your token to `config.datastack.py`, " + "set DATASTACK_TOKEN in your environment, or pass token='...' to DataStack()." + ) + + return cfg diff --git a/sdk-python/datastack_sdk/exceptions.py b/sdk-python/datastack_sdk/exceptions.py new file mode 100644 index 0000000..1aa255d --- /dev/null +++ b/sdk-python/datastack_sdk/exceptions.py @@ -0,0 +1,72 @@ +""" +Structured Exception Hierarchy for the DataStack Python SDK. +""" + +from typing import Optional + + +class DataStackError(Exception): + """Base exception for all DataStack SDK errors.""" + + def __init__( + self, + message: str, + code: Optional[str] = None, + retryable: bool = False, + ) -> None: + super().__init__(message) + self.message = message + self.code = code or "DATASTACK_ERROR" + self.retryable = retryable + + +class DataStackConfigError(DataStackError): + """Raised when `config.datastack.py` or required credentials are missing or invalid.""" + + def __init__(self, message: str) -> None: + super().__init__(message, code="CONFIG_ERROR", retryable=False) + + +class DataStackAuthError(DataStackError): + """ + Raised when authentication fails (`AUTH_REQUIRED`, `AUTH_INVALID`, + `CLIENT_REVOKED`, or `TOKEN_ROTATED`). Non-retryable until credentials are updated. + """ + + def __init__(self, message: str, code: str = "AUTH_ERROR") -> None: + super().__init__(message, code=code, retryable=False) + + +class DataStackRateLimitError(DataStackError): + """Raised when `RATE_LIMIT_EXCEEDED` persists after all backoff retries.""" + + def __init__(self, message: str) -> None: + super().__init__(message, code="RATE_LIMIT_EXCEEDED", retryable=True) + + +class DataStackPayloadTooLargeError(DataStackError): + """Raised when a single event exceeds the server's `maxPayloadSizeKb` limit.""" + + def __init__(self, message: str) -> None: + super().__init__(message, code="PAYLOAD_TOO_LARGE", retryable=False) + + +class DataStackConnectionError(DataStackError): + """Raised when both WebSocket and HTTP fallback are unreachable and offline spooling is disabled.""" + + def __init__(self, message: str) -> None: + super().__init__(message, code="CONNECTION_ERROR", retryable=True) + + +class DataStackFileError(DataStackError): + """Raised when a file passed to `send_file()` is missing, unsupported, or malformed.""" + + def __init__(self, message: str) -> None: + super().__init__(message, code="FILE_ERROR", retryable=False) + + +class DataStackOfflineSpoolError(DataStackError): + """Raised when the local offline disk spool directory is full or unwriteable.""" + + def __init__(self, message: str) -> None: + super().__init__(message, code="OFFLINE_SPOOL_ERROR", retryable=False) diff --git a/sdk-python/datastack_sdk/heartbeat.py b/sdk-python/datastack_sdk/heartbeat.py new file mode 100644 index 0000000..34e9893 --- /dev/null +++ b/sdk-python/datastack_sdk/heartbeat.py @@ -0,0 +1,186 @@ +""" +Smart Heartbeat Monitor & Connection State Machine (`HeartbeatMonitor`). + +Lifecycle States: +- `CONNECTED_IDLE`: WebSocket (Port 4000) is connected and verified. Heartbeat sleeps in IDLE mode. +- `FALLBACK_HTTP`: WebSocket disconnected. Client automatically sends events via HTTP (`POST /api/events`) + while Heartbeat actively probes to reconnect and verify WebSocket, then returns to `CONNECTED_IDLE`. +- `OFFLINE_SPOOL`: Device is completely offline. Events spool to disk (`.datastack_spool/spool.jsonl`). + Heartbeat monitors network restoration, reconnects, replays spooled events, and returns to `CONNECTED_IDLE`. +- `RECONNECTING_VERIFY`: Heartbeat is verifying handshake (`ping_verify`) and draining spooled events. +- `STOPPED`: Client was closed intentionally or halted due to non-retryable auth revocation. +""" + +import logging +import threading +import time +from typing import Any, Callable, Dict, Optional + +logger = logging.getLogger("datastack.heartbeat") + + +class ConnectionState: + CONNECTED_IDLE = "CONNECTED_IDLE" + FALLBACK_HTTP = "FALLBACK_HTTP" + OFFLINE_SPOOL = "OFFLINE_SPOOL" + RECONNECTING_VERIFY = "RECONNECTING_VERIFY" + STOPPED = "STOPPED" + + +class HeartbeatMonitor: + """ + Background recovery thread that sleeps in zero-CPU IDLE mode while WebSocket is + healthy (`CONNECTED_IDLE`) and wakes up automatically when WebSocket disconnects + or when the client device goes offline. + """ + + def __init__( + self, + interval_sec: float, + recovery_callback: Callable[[], bool], + ) -> None: + self.interval_sec = max(0.2, float(interval_sec)) + self._recovery_callback = recovery_callback + self._state = ConnectionState.CONNECTED_IDLE + self._closed = False + self._last_error: Optional[str] = None + self._recovery_attempts = 0 + self._last_verified_at: Optional[float] = None + + self._lock = threading.RLock() + self._wake_event = threading.Event() + self._stop_event = threading.Event() + self._thread: Optional[threading.Thread] = None + + @property + def state(self) -> str: + with self._lock: + return self._state + + @property + def is_idle(self) -> bool: + with self._lock: + return self._state == ConnectionState.CONNECTED_IDLE + + def set_idle_connected(self) -> None: + """ + Marks the connection as healthy (`CONNECTED_IDLE`) and puts the Heartbeat + Monitor into IDLE mode (`_wake_event.clear()`). + """ + with self._lock: + self._closed = False + self._state = ConnectionState.CONNECTED_IDLE + self._recovery_attempts = 0 + self._last_error = None + self._last_verified_at = time.time() + self._wake_event.clear() + + def trigger_fallback(self, reason: str = "WebSocket disconnected") -> None: + """ + Transitions to `FALLBACK_HTTP` and wakes up the Heartbeat Monitor thread + to repair the WebSocket connection in the background. + """ + with self._lock: + if self._closed: + return + self._state = ConnectionState.FALLBACK_HTTP + self._last_error = reason + self._ensure_thread_started() + self._wake_event.set() + + def trigger_offline(self, reason: str = "Device offline") -> None: + """ + Transitions to `OFFLINE_SPOOL` when both WebSocket and HTTP fallback are + unreachable, waking the Heartbeat Monitor to watch for network return. + """ + with self._lock: + if self._closed: + return + self._state = ConnectionState.OFFLINE_SPOOL + self._last_error = reason + self._ensure_thread_started() + self._wake_event.set() + + def set_reconnecting(self) -> None: + with self._lock: + if not self._closed: + self._state = ConnectionState.RECONNECTING_VERIFY + + def start(self) -> None: + with self._lock: + self._closed = False + self._stop_event.clear() + self._ensure_thread_started() + + def stop(self, reason: Optional[str] = None) -> None: + with self._lock: + self._closed = True + self._state = ConnectionState.STOPPED + if reason: + self._last_error = reason + self._stop_event.set() + self._wake_event.set() + + def status_snapshot(self) -> Dict[str, Any]: + with self._lock: + return { + "state": self._state, + "heartbeatMode": ( + "IDLE" + if self._state == ConnectionState.CONNECTED_IDLE + else "ACTIVE_RECOVERY" + if self._state + in ( + ConnectionState.FALLBACK_HTTP, + ConnectionState.OFFLINE_SPOOL, + ConnectionState.RECONNECTING_VERIFY, + ) + else "STOPPED" + ), + "recoveryAttempts": self._recovery_attempts, + "lastError": self._last_error, + "lastVerifiedAt": self._last_verified_at, + "intervalSec": self.interval_sec, + } + + def _ensure_thread_started(self) -> None: + if self._thread is not None and self._thread.is_alive(): + return + self._stop_event.clear() + self._thread = threading.Thread( + target=self._run_loop, + name="DataStack-HeartbeatMonitor", + daemon=True, + ) + self._thread.start() + + def _run_loop(self) -> None: + while not self._stop_event.is_set(): + # Wait in zero-CPU IDLE mode until woken by a disconnect or offline trigger + self._wake_event.wait() + if self._stop_event.is_set(): + break + + # Wait for the heartbeat interval before probing + if self._stop_event.wait(self.interval_sec): + break + + if not self._wake_event.is_set(): + continue + + with self._lock: + if self._closed or self._state == ConnectionState.STOPPED: + break + self._recovery_attempts += 1 + + try: + recovered_to_idle = self._recovery_callback() + if recovered_to_idle: + self.set_idle_connected() + logger.info( + "[DataStack Heartbeat] WebSocket reconnected & verified. Returning to IDLE mode." + ) + except Exception as exc: + with self._lock: + self._last_error = str(exc) + logger.debug("[DataStack Heartbeat] Recovery probe failed: %s", exc) diff --git a/sdk-python/datastack_sdk/spool.py b/sdk-python/datastack_sdk/spool.py new file mode 100644 index 0000000..e98bdad --- /dev/null +++ b/sdk-python/datastack_sdk/spool.py @@ -0,0 +1,164 @@ +""" +Thread-Safe Offline Disk Spooler (`DiskSpooler`). +Persists unsent telemetry events to `.datastack_spool/spool.jsonl` when a client +device loses network connectivity completely, and drains them in deterministic +batches once the Heartbeat Monitor restores connectivity. +""" + +import json +import os +import threading +from pathlib import Path +from typing import Any, Dict, List + +from .exceptions import DataStackOfflineSpoolError + + +class DiskSpooler: + """ + Append-only JSONL disk queue with atomic batch drainage and size cap protection. + """ + + def __init__( + self, + spool_dir: str = ".datastack_spool", + max_size_mb: int = 50, + enabled: bool = True, + ) -> None: + self.enabled = enabled + self.spool_dir = Path(spool_dir) + self.spool_file = self.spool_dir / "spool.jsonl" + self.max_bytes = max(1, int(max_size_mb)) * 1024 * 1024 + self._lock = threading.RLock() + + def _ensure_dir(self) -> None: + try: + self.spool_dir.mkdir(parents=True, exist_ok=True) + except Exception as exc: + raise DataStackOfflineSpoolError( + f"Cannot create offline spool directory '{self.spool_dir}': {exc}" + ) from exc + + def enqueue(self, events: List[Dict[str, Any]]) -> int: + """ + Appends one or more formatted events (`{eventId, timestamp, data}`) + to `.datastack_spool/spool.jsonl`. Returns count of spooled events. + """ + if not self.enabled or not events: + return 0 + + with self._lock: + self._ensure_dir() + current_size = ( + self.spool_file.stat().st_size if self.spool_file.exists() else 0 + ) + + lines: List[str] = [] + added_bytes = 0 + for evt in events: + serialized = json.dumps(evt, ensure_ascii=False) + "\n" + encoded_len = len(serialized.encode("utf-8")) + if current_size + added_bytes + encoded_len > self.max_bytes: + raise DataStackOfflineSpoolError( + f"Offline spool limit ({self.max_bytes // (1024 * 1024)} MB) " + f"reached at '{self.spool_file}'." + ) + lines.append(serialized) + added_bytes += encoded_len + + try: + with self.spool_file.open("a", encoding="utf-8") as fh: + fh.writelines(lines) + fh.flush() + except Exception as exc: + raise DataStackOfflineSpoolError( + f"Failed writing to offline spool file '{self.spool_file}': {exc}" + ) from exc + + return len(lines) + + def peek_batch(self, limit: int = 250) -> List[Dict[str, Any]]: + """ + Reads up to `limit` spooled events from the front of the spool file + without removing them until `ack_batch()` confirms delivery. + """ + if not self.enabled: + return [] + + with self._lock: + if not self.spool_file.exists(): + return [] + + batch: List[Dict[str, Any]] = [] + try: + with self.spool_file.open("r", encoding="utf-8") as fh: + for line in fh: + if len(batch) >= limit: + break + stripped = line.strip() + if not stripped: + continue + try: + parsed = json.loads(stripped) + if isinstance(parsed, dict): + batch.append(parsed) + except json.JSONDecodeError: + continue + except Exception: + return [] + + return batch + + def ack_batch(self, count: int) -> int: + """ + Removes the first `count` non-empty lines from the spool file atomically + after the Heartbeat Monitor confirms they have been ingested by DataStack. + """ + if not self.enabled or count <= 0: + return 0 + + with self._lock: + if not self.spool_file.exists(): + return 0 + + try: + with self.spool_file.open("r", encoding="utf-8") as fh: + all_lines = [ln for ln in fh if ln.strip()] + + if count >= len(all_lines): + self.spool_file.unlink(missing_ok=True) + return len(all_lines) + + remaining = all_lines[count:] + tmp_file = self.spool_dir / "spool.jsonl.tmp" + with tmp_file.open("w", encoding="utf-8") as fh: + fh.writelines(remaining) + fh.flush() + os.replace(str(tmp_file), str(self.spool_file)) + return count + except Exception: + return 0 + + def pending_count(self) -> int: + """Returns the number of queued events waiting in the disk spool.""" + if not self.enabled: + return 0 + + with self._lock: + if not self.spool_file.exists(): + return 0 + try: + with self.spool_file.open("r", encoding="utf-8") as fh: + return sum(1 for ln in fh if ln.strip()) + except Exception: + return 0 + + def size_bytes(self) -> int: + """Returns current disk usage of the spool file in bytes.""" + with self._lock: + if not self.spool_file.exists(): + return 0 + try: + return self.spool_file.stat().st_size + except Exception: + return 0 diff --git a/sdk/python/datastack/datastack/client.py b/sdk/python/datastack/datastack/client.py index a5ab105..959fed0 100644 --- a/sdk/python/datastack/datastack/client.py +++ b/sdk/python/datastack/datastack/client.py @@ -288,12 +288,12 @@ class DataStackClient: ) with self._lock: + self.is_connected = False if self.sio.connected: try: self.sio.disconnect() except Exception: pass - self.is_connected = False try: self.sio.connect( @@ -699,7 +699,7 @@ class DataStackClient: Replays any offline spooled events from `.datastack_spool/spool.jsonl` once connectivity (WebSocket or HTTP fallback) is restored. Routes batches through `_send_formatted_batch_recursive(..., allow_spool=False)` - so oversized or rate-limited spool batches automatically split and pace. + and preserves any `unprocessedItems` so zero spooled events are ever lost. """ if not self.config.offline_spool_enabled: return 0 @@ -724,10 +724,23 @@ class DataStackClient: file_name=None, allow_spool=False, ) - if not res.get("success", False): + processed_now = int(res.get("processed", 0)) + duplicates_now = int(res.get("duplicates", 0)) + unprocessed = res.get("unprocessedItems") + + if ( + not res.get("success", False) + and processed_now == 0 + and duplicates_now == 0 + ): break self.spooler.ack_batch(len(batch)) + if isinstance(unprocessed, list) and len(unprocessed) > 0: + self.spooler.enqueue(unprocessed) + total_flushed += max(0, len(batch) - len(unprocessed)) + break + total_flushed += len(batch) except Exception as exc: logger.debug("[DataStack] Spool drain paused: %s", exc) @@ -869,6 +882,12 @@ class DataStackClient: file_name=file_name, allow_spool=allow_spool, ) + left_unprocessed = left_res.get("unprocessedItems") or [] + right_unprocessed = right_res.get("unprocessedItems") or [] + combined_unprocessed = [ + *(left_unprocessed if isinstance(left_unprocessed, list) else []), + *(right_unprocessed if isinstance(right_unprocessed, list) else []), + ] return { "success": bool( left_res.get("success", True) @@ -884,15 +903,16 @@ class DataStackClient: + int(right_res.get("spooledCount", 0)), "transport": right_res.get("transport") or left_res.get("transport"), + **( + {"unprocessedItems": combined_unprocessed} + if len(combined_unprocessed) > 0 + else {} + ), } # Handle partial-batch rate limiting (`rateLimitedEventIds`) so zero rows are dropped rate_limited_ids = res.get("rateLimitedEventIds") - if ( - isinstance(rate_limited_ids, list) - and len(rate_limited_ids) > 0 - and self.config.max_retries > 0 - ): + if isinstance(rate_limited_ids, list) and len(rate_limited_ids) > 0: pending_id_set = {str(eid) for eid in rate_limited_ids} remaining_items = [ item @@ -904,56 +924,79 @@ class DataStackClient: total_spooled = int(res.get("spooledCount", 0)) last_transport = res.get("transport") - for retry_round in range(1, self.config.max_retries + 2): - if not remaining_items: - break - # Wait slightly over 1.0s so the server's 1-second rate-limit window resets - wait_delay = max(1.05, self.config.retry_backoff_base_sec) + random.uniform( - 0.0, 0.15 - ) - time.sleep(wait_delay) + if self.config.max_retries > 0: + for _retry_round in range(1, self.config.max_retries + 2): + if not remaining_items: + break + # Wait slightly over 1.0s so the server's 1-second rate-limit window resets + wait_delay = max( + 1.05, self.config.retry_backoff_base_sec + ) + random.uniform(0.0, 0.15) + time.sleep(wait_delay) - sub_http_payload: Dict[str, Any] = ( - {"fileName": file_name, "records": remaining_items} - if file_name - else {"batch": remaining_items} - ) - sub_res = self._execute_with_retry_and_fallback( - ws_event="batch_sync", - ws_payload=remaining_items, - http_payload=sub_http_payload, - spool_items=remaining_items, - timeout=timeout, - allow_spool=allow_spool, - ) - total_processed += int(sub_res.get("processed", 0)) - total_duplicates += int(sub_res.get("duplicates", 0)) - total_spooled += int(sub_res.get("spooledCount", 0)) - last_transport = sub_res.get("transport") or last_transport + sub_http_payload: Dict[str, Any] = ( + {"fileName": file_name, "records": remaining_items} + if file_name + else {"batch": remaining_items} + ) + sub_res = self._execute_with_retry_and_fallback( + ws_event="batch_sync", + ws_payload=remaining_items, + http_payload=sub_http_payload, + spool_items=remaining_items, + timeout=timeout, + allow_spool=allow_spool, + ) + total_processed += int(sub_res.get("processed", 0)) + total_duplicates += int(sub_res.get("duplicates", 0)) + total_spooled += int(sub_res.get("spooledCount", 0)) + last_transport = sub_res.get("transport") or last_transport - next_rl_ids = sub_res.get("rateLimitedEventIds") - if not isinstance(next_rl_ids, list) or len(next_rl_ids) == 0: - remaining_items = [] - break - next_set = {str(eid) for eid in next_rl_ids} - remaining_items = [ - item - for item in remaining_items - if str(item.get("eventId")) in next_set - ] + if sub_res.get("spooled"): + remaining_items = [] + break + + next_rl_ids = sub_res.get("rateLimitedEventIds") + if ( + not isinstance(next_rl_ids, list) + or len(next_rl_ids) == 0 + ): + remaining_items = [] + break + next_set = {str(eid) for eid in next_rl_ids} + remaining_items = [ + item + for item in remaining_items + if str(item.get("eventId")) in next_set + ] + + # If any items still remain rate-limited after retries and spooling is allowed, spool them so zero data is lost + if ( + remaining_items + and allow_spool + and self.config.offline_spool_enabled + ): + spooled_now = self.spooler.enqueue(remaining_items) + total_spooled += spooled_now + remaining_items = [] oversized_ids = res.get("oversizedEventIds") final_rejected = len(remaining_items) + ( len(oversized_ids) if isinstance(oversized_ids, list) else 0 ) - return { - "success": final_rejected == 0 or total_processed > 0, + out: Dict[str, Any] = { + "success": final_rejected == 0 + or total_processed > 0 + or total_spooled > 0, "processed": total_processed, "rejected": final_rejected, "duplicates": total_duplicates, "spooledCount": total_spooled, "transport": last_transport, } + if remaining_items: + out["unprocessedItems"] = remaining_items + return out return res diff --git a/server/index.js b/server/index.js index 0f5d5c2..bad4e29 100644 --- a/server/index.js +++ b/server/index.js @@ -289,6 +289,18 @@ app.post('/internal/flush', async (_req, res) => { } }); +// Internal endpoint to expose Port 4000 buffered events before CSV flush +app.get('/internal/recent', (req, res) => { + const clientId = req.query.clientId ? String(req.query.clientId) : ''; + const limit = Math.min(200, Math.max(1, Number(req.query.limit) || 50)); + const filtered = + clientId && clientId !== 'all' + ? bufferQueue.filter((evt) => evt.clientId === clientId) + : bufferQueue; + const recent = filtered.slice(-limit).reverse(); + res.json({ events: recent, count: recent.length }); +}); + setupSocketHandlers(io, bufferQueue, writer, { getRuntimeConfig: () => runtimeConfig, verifyClientInDb, diff --git a/server/socket/handlers.js b/server/socket/handlers.js index 7f8c1c7..9f1922b 100644 --- a/server/socket/handlers.js +++ b/server/socket/handlers.js @@ -65,10 +65,10 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { const now = Date.now(); cleanupMaps(now); - // 1. Idempotency check (with 5-minute TTL) + // 1. Idempotency check (read-only check first; record only after rate-limit & payload checks pass) + const idempotencyKey = `${clientId}:${eventId}`; if (config.enableIdempotency && eventId) { - const key = `${clientId}:${eventId}`; - const existingTime = idempotencyMap.get(key); + const existingTime = idempotencyMap.get(idempotencyKey); if (existingTime && now - existingTime < IDEMPOTENCY_TTL_MS) { return { allowed: false, @@ -77,10 +77,23 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { reason: "Duplicate event dropped", }; } - idempotencyMap.set(key, now); } - // 2. Rate limit check + // 2. Payload byte size check (before consuming rate-limit quota) + const payloadStr = + typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {}); + const byteLength = Buffer.byteLength(payloadStr, "utf8"); + const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024; + if (byteLength > maxBytes) { + return { + allowed: false, + duplicate: false, + code: "PAYLOAD_TOO_LARGE", + reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`, + }; + } + + // 3. Rate limit check const effectiveLimit = Math.min( Number(clientRateLimit) || 100, Number(config.maxEventsPerSec) || 100, @@ -99,18 +112,9 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { rl.count += 1; } - // 3. Payload byte size check - const payloadStr = - typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {}); - const byteLength = Buffer.byteLength(payloadStr, "utf8"); - const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024; - if (byteLength > maxBytes) { - return { - allowed: false, - duplicate: false, - code: "PAYLOAD_TOO_LARGE", - reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`, - }; + // 4. Record idempotency key ONLY after event passes all checks + if (config.enableIdempotency && eventId) { + idempotencyMap.set(idempotencyKey, now); } return { allowed: true, payloadStr }; diff --git a/server/workers/csvWriter.js b/server/workers/csvWriter.js index bd75f18..4cb5359 100644 --- a/server/workers/csvWriter.js +++ b/server/workers/csvWriter.js @@ -74,7 +74,13 @@ class CsvWriterWorker { for (const [key, batch] of grouped.entries()) { const [clientId, date] = key.split(':::'); const targetPath = this.getTargetFilePath(clientId, date); - const fileExists = fs.existsSync(targetPath); + let hasContent = false; + try { + hasContent = + fs.existsSync(targetPath) && fs.statSync(targetPath).size > 0; + } catch { + hasContent = false; + } const cleanBatch = batch.map((item) => ({ clientId: item.clientId, @@ -83,10 +89,10 @@ class CsvWriterWorker { payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload), })); - const parser = new Parser({ fields, header: !fileExists, eol: '\n' }); + const parser = new Parser({ fields, header: !hasContent, eol: '\n' }); const csv = parser.parse(cleanBatch); - if (fileExists) { + if (hasContent) { fs.appendFileSync(targetPath, '\n' + csv, 'utf8'); } else { fs.writeFileSync(targetPath, csv, 'utf8');