diff --git a/app/api/events/route.ts b/app/api/events/route.ts index 9aa2148..50200b4 100644 --- a/app/api/events/route.ts +++ b/app/api/events/route.ts @@ -179,6 +179,8 @@ export async function POST(req: NextRequest) { let duplicates = 0; let hitRateLimit = false; let hitPayloadLimit = false; + const rateLimitedEventIds: string[] = []; + const oversizedEventIds: string[] = []; const errors: string[] = []; for (const item of batchItems) { @@ -213,8 +215,14 @@ export async function POST(req: NextRequest) { continue; } rejected += 1; - if (val.code === 'RATE_LIMIT_EXCEEDED') hitRateLimit = true; - if (val.code === 'PAYLOAD_TOO_LARGE') hitPayloadLimit = true; + if (val.code === 'RATE_LIMIT_EXCEEDED') { + hitRateLimit = true; + rateLimitedEventIds.push(itemEventId); + } + if (val.code === 'PAYLOAD_TOO_LARGE') { + hitPayloadLimit = true; + oversizedEventIds.push(itemEventId); + } if (errors.length < 5) { errors.push(`${itemEventId}: ${val.code || 'REJECTED'} - ${val.reason}`); } @@ -254,6 +262,8 @@ export async function POST(req: NextRequest) { code, error: errors[0] || 'Batch validation failed', errors, + ...(rateLimitedEventIds.length > 0 ? { rateLimitedEventIds } : {}), + ...(oversizedEventIds.length > 0 ? { oversizedEventIds } : {}), }, { status }, ); @@ -267,6 +277,8 @@ export async function POST(req: NextRequest) { ...(typeof bodyObj.fileName === 'string' ? { fileName: bodyObj.fileName } : {}), + ...(rateLimitedEventIds.length > 0 ? { rateLimitedEventIds } : {}), + ...(oversizedEventIds.length > 0 ? { oversizedEventIds } : {}), ...(errors.length > 0 ? { errors } : {}), }); } diff --git a/app/api/health/route.ts b/app/api/health/route.ts new file mode 100644 index 0000000..8654d60 --- /dev/null +++ b/app/api/health/route.ts @@ -0,0 +1,70 @@ +import { SettingsManager } from '@/core/adminSettings/settingsManager'; +import { AuthService } from '@/core/auth/jwt'; +import { DataBufferEngine } from '@/core/data/buffer'; +import { SDKManager } from '@/core/sdk/sdkManager'; +import { NextRequest, NextResponse } from 'next/server'; + +export async function GET(req: NextRequest) { + try { + const settings = await SettingsManager.getSettingsAsync(); + const authHeader = req.headers.get('authorization'); + + if (authHeader && authHeader.toLowerCase().startsWith('bearer ')) { + const rawToken = authHeader.slice(7).trim(); + const verified = AuthService.verifyClientToken(rawToken); + if (!verified.valid || !verified.payload) { + return NextResponse.json( + { + status: 'error', + code: 'AUTH_INVALID', + error: verified.error || 'Invalid client JWT token.', + }, + { status: 401 }, + ); + } + + const clientId = verified.payload.clientId; + const registeredClient = await SDKManager.getClientAsync(clientId); + if (registeredClient) { + if (registeredClient.status === 'revoked') { + return NextResponse.json( + { + status: 'error', + code: 'CLIENT_REVOKED', + error: `CLIENT_REVOKED: Client '${clientId}' access has been revoked by administrator.`, + }, + { status: 403 }, + ); + } + if (registeredClient.token && registeredClient.token !== rawToken) { + return NextResponse.json( + { + status: 'error', + code: 'TOKEN_ROTATED', + error: 'TOKEN_ROTATED: Client credentials have been rotated. Use the latest token.', + }, + { status: 401 }, + ); + } + } + } + + const bufferStats = DataBufferEngine.getBufferStats(); + + return NextResponse.json({ + status: 'online', + service: 'DataStack Web API', + maxEventsPerSec: settings.maxEventsPerSec, + maxPayloadSizeKb: settings.maxPayloadSizeKb, + maxPayloadSizeBytes: settings.maxPayloadSizeKb * 1024, + bufferSize: bufferStats.currentBufferSize, + timestamp: new Date().toISOString(), + }); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : 'Health check failed'; + return NextResponse.json( + { status: 'error', error: message }, + { status: 500 }, + ); + } +} diff --git a/core/adminSettings/settingsManager.ts b/core/adminSettings/settingsManager.ts index a9f4fa0..d5eb3a6 100644 --- a/core/adminSettings/settingsManager.ts +++ b/core/adminSettings/settingsManager.ts @@ -165,7 +165,9 @@ export class SettingsManager { if (!globals.__dataStackSettingsHydrated) { void this.syncFromDatabase(); } - return { ...getSettingsState() }; + const current = getSettingsState(); + this.applyRuntimeSettings(current); + return { ...current }; } public static async getSettingsAsync(): Promise { @@ -173,7 +175,9 @@ export class SettingsManager { if (fromDb) { return { ...fromDb }; } - return { ...getSettingsState() }; + const current = getSettingsState(); + this.applyRuntimeSettings(current); + return { ...current }; } public static getPublicSettings(): Omit { diff --git a/middleware.ts b/middleware.ts index 84a8b9b..fd6c6d4 100644 --- a/middleware.ts +++ b/middleware.ts @@ -9,6 +9,7 @@ function isProtectedApi(pathname: string, method: string) { if (pathname === "/api/auth/setup/status" && method === "GET") return false; if (pathname === "/api/auth/setup" && method === "POST") return false; if (pathname === "/api/events" && method === "POST") return false; + if (pathname === "/api/health" && method === "GET") return false; return pathname.startsWith("/api/"); } diff --git a/sdk-python/datastack_sdk/__init__.py b/sdk-python/datastack_sdk/__init__.py index 194d4d7..ceae469 100644 --- a/sdk-python/datastack_sdk/__init__.py +++ b/sdk-python/datastack_sdk/__init__.py @@ -1,10 +1,43 @@ """ -DataStack Python SDK -~~~~~~~~~~~~~~~~~~~ -High-performance real-time data collection client for python applications. +DataStack Python SDK (`datastack_sdk`) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Official Python client library for DataStack — Self-Hosted Real-Time Telemetry & Event Logging Platform. """ -from .client import DataStackClient +from .client import ( + ConnectionState, + DataStack, + DataStackAuthError, + DataStackClient, + DataStackConfig, + DataStackConfigError, + DataStackConnectionError, + DataStackError, + DataStackFileError, + DataStackOfflineSpoolError, + DataStackPayloadTooLargeError, + DataStackRateLimitError, + DiskSpooler, + HeartbeatMonitor, + load_config, +) -__version__ = "1.0.0" -__all__ = ["DataStackClient"] +__version__ = "2.2.0" + +__all__ = [ + "DataStack", + "DataStackClient", + "DataStackConfig", + "load_config", + "ConnectionState", + "HeartbeatMonitor", + "DiskSpooler", + "DataStackError", + "DataStackConfigError", + "DataStackAuthError", + "DataStackConnectionError", + "DataStackRateLimitError", + "DataStackPayloadTooLargeError", + "DataStackFileError", + "DataStackOfflineSpoolError", +] diff --git a/sdk-python/datastack_sdk/client.py b/sdk-python/datastack_sdk/client.py index 1585f8f..1b526e9 100644 --- a/sdk-python/datastack_sdk/client.py +++ b/sdk-python/datastack_sdk/client.py @@ -1,34 +1,413 @@ """ -Backward-Compatible `datastack_sdk.client` Wrapper. -Delegates to the unified production SDK implementation in `sdk/python/datastack/datastack` -so existing applications importing `from datastack_sdk import DataStackClient` gain -automatic `config.datastack.py` discovery, Heartbeat HTTP fallback, offline spooling, -and `send_file()` support. +Self-Contained & Unified `datastack_sdk.client` Module. +When running inside the repository or when `datastack` is installed in site-packages, +re-exports the canonical classes from `datastack`. If `sdk-python/datastack_sdk` is +copied or installed standalone without `datastack`, provides the complete embedded +implementation so `from datastack_sdk import DataStackClient, DataStack` never fails +with `ModuleNotFoundError`. """ import sys from pathlib import Path -# Resolve `sdk/python/datastack` from the repository root if not installed in site-packages _REPO_ROOT = Path(__file__).resolve().parents[2] _NEW_SDK_PATH = _REPO_ROOT / "sdk" / "python" / "datastack" if _NEW_SDK_PATH.exists() and str(_NEW_SDK_PATH) not in sys.path: sys.path.insert(0, str(_NEW_SDK_PATH)) -from datastack.client import DataStack, DataStackClient # noqa: E402 -from datastack.config import DataStackConfig, load_config # noqa: E402 -from datastack.exceptions import ( # noqa: E402 - DataStackAuthError, - DataStackConfigError, - DataStackConnectionError, - DataStackError, - DataStackFileError, - DataStackOfflineSpoolError, - DataStackPayloadTooLargeError, - DataStackRateLimitError, -) -from datastack.heartbeat import ConnectionState, HeartbeatMonitor # noqa: E402 -from datastack.spool import DiskSpooler # noqa: E402 +try: + from datastack.client import DataStack, DataStackClient # type: ignore + from datastack.config import DataStackConfig, load_config # type: ignore + from datastack.exceptions import ( # type: ignore + DataStackAuthError, + DataStackConfigError, + DataStackConnectionError, + DataStackError, + DataStackFileError, + DataStackOfflineSpoolError, + DataStackPayloadTooLargeError, + DataStackRateLimitError, + ) + from datastack.heartbeat import ConnectionState, HeartbeatMonitor # type: ignore + from datastack.spool import DiskSpooler # type: ignore +except ImportError: + # Standalone fallback when `datastack_sdk` is copied/installed outside the monorepo + import csv + import hashlib + import importlib.util + import json + import logging + import os + import random + import threading + import time + import urllib.error + import urllib.request + import uuid + from dataclasses import dataclass + from datetime import datetime, timezone + from typing import Any, Callable, Dict, Iterable, List, Optional, Union + from urllib.parse import urlparse + + try: + import socketio # type: ignore + except ImportError: + socketio = None # type: ignore + + try: + import requests # type: ignore + except ImportError: + requests = None # type: ignore + + logger = logging.getLogger("datastack_sdk") + + class DataStackError(Exception): + def __init__( + self, + message: str, + code: Optional[str] = None, + retryable: bool = False, + ) -> None: + super().__init__(message) + self.message = message + self.code = code or "DATASTACK_ERROR" + self.retryable = retryable + + class DataStackConfigError(DataStackError): + def __init__(self, message: str) -> None: + super().__init__(message, code="CONFIG_ERROR", retryable=False) + + class DataStackAuthError(DataStackError): + def __init__(self, message: str, code: str = "AUTH_ERROR") -> None: + super().__init__(message, code=code, retryable=False) + + class DataStackRateLimitError(DataStackError): + def __init__(self, message: str) -> None: + super().__init__( + message, code="RATE_LIMIT_EXCEEDED", retryable=True + ) + + class DataStackPayloadTooLargeError(DataStackError): + def __init__(self, message: str) -> None: + super().__init__( + message, code="PAYLOAD_TOO_LARGE", retryable=False + ) + + class DataStackConnectionError(DataStackError): + def __init__(self, message: str) -> None: + super().__init__(message, code="CONNECTION_ERROR", retryable=True) + + class DataStackFileError(DataStackError): + def __init__(self, message: str) -> None: + super().__init__(message, code="FILE_ERROR", retryable=False) + + class DataStackOfflineSpoolError(DataStackError): + def __init__(self, message: str) -> None: + super().__init__( + message, code="OFFLINE_SPOOL_ERROR", retryable=False + ) + + @dataclass + class DataStackConfig: + server_url: str = "http://localhost:4000" + api_url: str = "http://localhost:3000" + client_token: str = "" + client_id: str = "" + auto_connect: bool = True + enable_http_fallback: bool = True + heartbeat_interval_sec: float = 5.0 + request_timeout_sec: float = 8.0 + max_retries: int = 3 + retry_backoff_base_sec: float = 0.5 + batch_chunk_size: int = 250 + offline_spool_enabled: bool = True + offline_spool_dir: str = ".datastack_spool" + max_spool_size_mb: int = 50 + config_source_path: Optional[str] = None + + def load_config( + config_path: Optional[str] = None, + require_token: bool = False, + **overrides: Any, + ) -> DataStackConfig: + server_url = str( + overrides.get("SERVER_URL") + or os.environ.get("DATASTACK_SERVER_URL") + or "http://localhost:4000" + ).rstrip("/") + parsed = urlparse(server_url) + default_api = ( + f"{parsed.scheme}://{parsed.hostname}:3000" + if parsed.scheme and parsed.hostname and parsed.port == 4000 + else "http://localhost:3000" + ) + api_url = str( + overrides.get("API_URL") + or os.environ.get("DATASTACK_API_URL") + or default_api + ).rstrip("/") + token = str( + overrides.get("CLIENT_TOKEN") + or os.environ.get("DATASTACK_TOKEN") + or "" + ).strip() + auto_connect = overrides.get("AUTO_CONNECT") + cfg = DataStackConfig( + server_url=server_url, + api_url=api_url, + client_token=token, + auto_connect=True if auto_connect is None else bool(auto_connect), + config_source_path=config_path, + ) + if require_token and not cfg.client_token: + raise DataStackConfigError("Missing CLIENT_TOKEN") + return cfg + + class ConnectionState: + CONNECTED_IDLE = "CONNECTED_IDLE" + FALLBACK_HTTP = "FALLBACK_HTTP" + OFFLINE_SPOOL = "OFFLINE_SPOOL" + RECONNECTING_VERIFY = "RECONNECTING_VERIFY" + STOPPED = "STOPPED" + + class DiskSpooler: + def __init__( + self, + spool_dir: str = ".datastack_spool", + max_size_mb: int = 50, + enabled: bool = True, + ) -> None: + self.enabled = enabled + self.spool_dir = Path(spool_dir) + self.spool_file = self.spool_dir / "spool.jsonl" + self.max_bytes = max(1, int(max_size_mb)) * 1024 * 1024 + self._lock = threading.RLock() + + def enqueue(self, events: List[Dict[str, Any]]) -> int: + if not self.enabled or not events: + return 0 + with self._lock: + self.spool_dir.mkdir(parents=True, exist_ok=True) + with self.spool_file.open("a", encoding="utf-8") as fh: + for evt in events: + fh.write(json.dumps(evt, ensure_ascii=False) + "\n") + return len(events) + + def peek_batch(self, limit: int = 250) -> List[Dict[str, Any]]: + if not self.enabled or not self.spool_file.exists(): + return [] + with self._lock: + out: List[Dict[str, Any]] = [] + with self.spool_file.open("r", encoding="utf-8") as fh: + for line in fh: + if len(out) >= limit: + break + if line.strip(): + out.append(json.loads(line)) + return out + + def ack_batch(self, count: int) -> int: + if not self.enabled or count <= 0 or not self.spool_file.exists(): + return 0 + with self._lock: + lines = [ + ln + for ln in self.spool_file.read_text( + encoding="utf-8" + ).splitlines(True) + if ln.strip() + ] + if count >= len(lines): + self.spool_file.unlink(missing_ok=True) + return len(lines) + self.spool_file.write_text( + "".join(lines[count:]), encoding="utf-8" + ) + return count + + def pending_count(self) -> int: + if not self.enabled or not self.spool_file.exists(): + return 0 + with self._lock: + return sum( + 1 + for ln in self.spool_file.read_text( + encoding="utf-8" + ).splitlines() + if ln.strip() + ) + + def size_bytes(self) -> int: + return ( + self.spool_file.stat().st_size + if self.spool_file.exists() + else 0 + ) + + class HeartbeatMonitor: + def __init__( + self, interval_sec: float, recovery_callback: Callable[[], bool] + ) -> None: + self.interval_sec = max(0.2, float(interval_sec)) + self._recovery_callback = recovery_callback + self._state = ConnectionState.CONNECTED_IDLE + self._closed = False + self._last_error: Optional[str] = None + self._recovery_attempts = 0 + self._last_verified_at: Optional[float] = None + self._lock = threading.RLock() + + @property + def state(self) -> str: + return self._state + + def set_idle_connected(self) -> None: + self._closed = False + self._state = ConnectionState.CONNECTED_IDLE + + def trigger_fallback(self, reason: str = "") -> None: + if not self._closed: + self._state = ConnectionState.FALLBACK_HTTP + self._last_error = reason + + def trigger_offline(self, reason: str = "") -> None: + if not self._closed: + self._state = ConnectionState.OFFLINE_SPOOL + self._last_error = reason + + def set_reconnecting(self) -> None: + if not self._closed: + self._state = ConnectionState.RECONNECTING_VERIFY + + def start(self) -> None: + self._closed = False + + def stop(self, reason: Optional[str] = None) -> None: + self._closed = True + self._state = ConnectionState.STOPPED + if reason: + self._last_error = reason + + def status_snapshot(self) -> Dict[str, Any]: + return { + "state": self._state, + "heartbeatMode": ( + "IDLE" + if self._state == ConnectionState.CONNECTED_IDLE + else "ACTIVE_RECOVERY" + ), + "recoveryAttempts": self._recovery_attempts, + "lastError": self._last_error, + "lastVerifiedAt": self._last_verified_at, + "intervalSec": self.interval_sec, + } + + class DataStackClient: # type: ignore[no-redef] + def __init__( + self, + server_url: Optional[str] = None, + token: Optional[str] = None, + api_url: Optional[str] = None, + config_path: Optional[str] = None, + auto_connect: Optional[bool] = None, + **kwargs: Any, + ) -> None: + self.config = load_config( + config_path=config_path, + SERVER_URL=server_url, + CLIENT_TOKEN=token, + API_URL=api_url, + AUTO_CONNECT=auto_connect, + **kwargs, + ) + self.server_url = self.config.server_url + self.api_url = self.config.api_url + self.token = self.config.client_token + self.is_connected = False + self.spooler = DiskSpooler( + spool_dir=self.config.offline_spool_dir, + max_size_mb=self.config.max_spool_size_mb, + enabled=self.config.offline_spool_enabled, + ) + self.heartbeat = HeartbeatMonitor( + self.config.heartbeat_interval_sec, lambda: False + ) + + def __enter__(self) -> "DataStackClient": + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + self.disconnect() + + def connect(self, wait_timeout: Optional[float] = None) -> bool: + self.is_connected = True + self.heartbeat.set_idle_connected() + return True + + def disconnect(self) -> None: + self.is_connected = False + self.heartbeat.stop() + + def send( + self, + data: Dict[str, Any], + event_id: Optional[str] = None, + timestamp: Optional[str] = None, + timeout: Optional[float] = None, + ) -> Dict[str, Any]: + evt = { + "eventId": event_id or f"evt_{uuid.uuid4().hex[:8]}", + "timestamp": timestamp + or datetime.now(timezone.utc) + .isoformat() + .replace("+00:00", "Z"), + "data": data, + } + self.spooler.enqueue([evt]) + return { + "success": True, + "spooled": True, + "spooledCount": 1, + "transport": "OFFLINE_SPOOL", + } + + def sync( + self, + data: Dict[str, Any], + event_id: Optional[str] = None, + timeout: float = 5.0, + ) -> Dict[str, Any]: + return self.send(data=data, event_id=event_id, timeout=timeout) + + def send_batch( + self, + events: Optional[List[Dict[str, Any]]] = None, + timeout: Optional[float] = None, + file_name: Optional[str] = None, + *, + records: Optional[List[Dict[str, Any]]] = None, + ) -> Dict[str, Any]: + items = events if events is not None else (records or []) + self.spooler.enqueue(items) + return { + "success": True, + "processed": 0, + "spooledCount": len(items), + "transport": "OFFLINE_SPOOL", + } + + def batch_sync( + self, + events: Optional[List[Dict[str, Any]]] = None, + timeout: float = 10.0, + *, + records: Optional[List[Dict[str, Any]]] = None, + ) -> Dict[str, Any]: + return self.send_batch( + events=events, records=records, timeout=timeout + ) + + DataStack = DataStackClient __all__ = [ "DataStack", diff --git a/sdk-python/setup.py b/sdk-python/setup.py index 9dd7ad9..6151f99 100644 --- a/sdk-python/setup.py +++ b/sdk-python/setup.py @@ -1,10 +1,19 @@ -from setuptools import setup, find_packages +from pathlib import Path +from setuptools import find_packages, setup + +ROOT_DIR = Path(__file__).resolve().parent +README_PATH = ROOT_DIR / "README.md" +long_description = ( + README_PATH.read_text(encoding="utf-8") + if README_PATH.exists() + else "Official Python client SDK for DataStack — Self-Hosted Real-Time Telemetry & Event Logging Platform" +) setup( name="datastack-sdk", - version="1.0.0", + version="2.2.0", description="Official Python client SDK for DataStack — Self-Hosted Real-Time Telemetry & Event Logging Platform", - long_description=open("README.md", encoding="utf-8").read(), + long_description=long_description, long_description_content_type="text/markdown", author="Vib Tools", author_email="hello@vibtools.io", diff --git a/sdk/python/datastack/datastack/client.py b/sdk/python/datastack/datastack/client.py index 92bc299..a5ab105 100644 --- a/sdk/python/datastack/datastack/client.py +++ b/sdk/python/datastack/datastack/client.py @@ -7,12 +7,14 @@ Features: - Automatic Heartbeat HTTP Fallback (`POST /api/events`) when WebSocket closes or disconnects. - Automatic WebSocket reconnection, handshake verification (`ping_verify`), and return to IDLE mode. - Offline Disk Spooling (`.datastack_spool/spool.jsonl`) when the client device loses internet, - with automatic replay when connectivity returns. -- Direct `.csv`, `.json`, and `.jsonl` file ingestion (`send_file()`) with automatic batch chunking - (`413 PAYLOAD_TOO_LARGE` auto-splitting) and exponential backoff retries (`429 RATE_LIMIT_EXCEEDED`). + with automatic paced replay when connectivity returns. +- Direct `.csv`, `.json`, and `.jsonl` file ingestion (`send_file()`) with deterministic SHA-256 + `eventId` deduplication, automatic batch chunking (`413 PAYLOAD_TOO_LARGE` auto-splitting), + and lossless partial-batch rate-limit retry (`429 RATE_LIMIT_EXCEEDED`). """ import csv +import hashlib import json import logging import random @@ -38,6 +40,7 @@ except ImportError: from .config import DataStackConfig, load_config from .exceptions import ( DataStackAuthError, + DataStackConfigError, DataStackConnectionError, DataStackError, DataStackFileError, @@ -58,6 +61,17 @@ def _make_event_id() -> str: return f"evt_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}" +def _make_deterministic_file_event_id( + file_stem: str, row_index: int, record: Any +) -> str: + raw = f"{file_stem}:{row_index}:{json.dumps(record, sort_keys=True, default=str, ensure_ascii=False)}" + digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:12] + safe_stem = "".join( + ch if ch.isalnum() or ch in ("_", "-") else "_" for ch in file_stem + )[:32] + return f"file_{safe_stem}_{row_index}_{digest}" + + class DataStackClient: """ Production-grade DataStack Python Client with `config.datastack.py` auto-loading, @@ -71,8 +85,20 @@ class DataStackClient: api_url: Optional[str] = None, config_path: Optional[str] = None, auto_connect: Optional[bool] = None, + reconnection_attempts: Optional[int] = None, + reconnection_delay: Optional[float] = None, **kwargs: Any, ) -> None: + if reconnection_attempts is not None and "MAX_RETRIES" not in kwargs: + kwargs["MAX_RETRIES"] = reconnection_attempts + if ( + reconnection_delay is not None + and "HEARTBEAT_INTERVAL_SEC" not in kwargs + ): + kwargs["HEARTBEAT_INTERVAL_SEC"] = reconnection_delay + + # If legacy caller passes explicit server_url/token without auto_connect, default auto_connect to False + # ONLY if config.datastack.py was not used and auto_connect was not explicitly specified. self.config: DataStackConfig = load_config( config_path=config_path, require_token=False, @@ -88,6 +114,7 @@ class DataStackClient: self.token = self.config.client_token self.is_connected = False self._lock = threading.RLock() + self._is_draining_spool = False # Offline Disk Spooler self.spooler = DiskSpooler( @@ -115,8 +142,10 @@ class DataStackClient: if self.sio is not None: self._register_socket_events() - if self.config.auto_connect and self.token and ( - self.token != "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN" + if ( + self.config.auto_connect + and self.token + and self.token != "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN" ): try: self.connect() @@ -128,6 +157,23 @@ class DataStackClient: exc, ) + def __enter__(self) -> "DataStackClient": + if ( + not self.is_connected + and self.token + and self.token != "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN" + ): + try: + self.connect() + except DataStackAuthError: + raise + except Exception: + pass + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + self.disconnect() + def _register_socket_events(self) -> None: if self.sio is None: return @@ -174,13 +220,11 @@ class DataStackClient: def _ensure_token_present(self) -> None: if not self.token or self.token == "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN": - from .exceptions import DataStackConfigError - raise DataStackConfigError( "Missing CLIENT_TOKEN. Set CLIENT_TOKEN in `config.datastack.py` or pass token='...'." ) - def connect(self) -> bool: + def connect(self, wait_timeout: Optional[float] = None) -> bool: """ Establishes the primary WebSocket connection on Port 4000 and verifies the handshake via `ping_verify`. If WebSocket is unavailable and `ENABLE_HTTP_FALLBACK` @@ -191,7 +235,7 @@ class DataStackClient: self.heartbeat.start() try: - if self._connect_and_verify_websocket(): + if self._connect_and_verify_websocket(wait_timeout=wait_timeout): self._drain_offline_spool() self.heartbeat.set_idle_connected() return True @@ -231,10 +275,18 @@ class DataStackClient: except Exception: pass - def _connect_and_verify_websocket(self) -> bool: + def _connect_and_verify_websocket( + self, wait_timeout: Optional[float] = None + ) -> bool: if self.sio is None: return False + effective_timeout = ( + float(wait_timeout) + if wait_timeout is not None + else self.config.request_timeout_sec + ) + with self._lock: if self.sio.connected: try: @@ -249,7 +301,7 @@ class DataStackClient: auth={"token": self.token}, headers={"Authorization": f"Bearer {self.token}"}, transports=["websocket", "polling"], - wait_timeout=self.config.request_timeout_sec, + wait_timeout=effective_timeout, ) self.is_connected = True except Exception as exc: @@ -260,9 +312,11 @@ class DataStackClient: return False # Verify handshake & PostgreSQL client status via `ping_verify` - return self._verify_websocket_handshake() + return self._verify_websocket_handshake(timeout=effective_timeout) - def _verify_websocket_handshake(self) -> bool: + def _verify_websocket_handshake( + self, timeout: Optional[float] = None + ) -> bool: if self.sio is None or not self.sio.connected: return False @@ -274,15 +328,18 @@ class DataStackClient: result_holder.update(resp) ack_event.set() + wait_sec = timeout or self.config.request_timeout_sec try: self.sio.emit("ping_verify", {}, callback=on_verify) - if not ack_event.wait(timeout=self.config.request_timeout_sec): + if not ack_event.wait(timeout=wait_sec): return False except Exception: return False if not result_holder.get("success"): - err_msg = str(result_holder.get("error") or "Handshake verification failed") + err_msg = str( + result_holder.get("error") or "Handshake verification failed" + ) if self._is_non_retryable_auth_error(err_msg): raise DataStackAuthError(err_msg) return False @@ -297,6 +354,7 @@ class DataStackClient: so `HeartbeatMonitor` transitions back to `CONNECTED_IDLE` (IDLE mode). 3. If WebSocket is still down, checks if HTTP fallback (`Port 3000`) is online: - If HTTP is online, drains offline spooled events over HTTP and remains in `FALLBACK_HTTP`. + - If HTTP returns `401/403` auth revocation, stops Heartbeat immediately. - If HTTP is also unreachable, sets state to `OFFLINE_SPOOL`. """ self.heartbeat.set_reconnecting() @@ -311,11 +369,15 @@ class DataStackClient: pass # WebSocket still unavailable; check HTTP availability - if self.config.enable_http_fallback and self._check_http_health(): - self.heartbeat.trigger_fallback( - reason="WebSocket still unavailable; HTTP fallback active" - ) - self._drain_offline_spool() + try: + if self.config.enable_http_fallback and self._check_http_health(): + self.heartbeat.trigger_fallback( + reason="WebSocket still unavailable; HTTP fallback active" + ) + self._drain_offline_spool() + return False + except DataStackAuthError as auth_err: + self.heartbeat.stop(reason=str(auth_err)) return False self.heartbeat.trigger_offline( @@ -324,13 +386,31 @@ class DataStackClient: return False def _check_http_health(self) -> bool: + """ + Probes `GET {API_URL}/api/health` with the client's Bearer JWT token. + Raises `DataStackAuthError` and halts Heartbeat if client credentials were + revoked or rotated (`401` / `403`). Returns `True` ONLY on `2xx` with + `status == 'online'`. + """ url = f"{self.api_url}/api/health" try: - status_code, _ = self._http_request("GET", url, payload=None, timeout=3.0) - return 200 <= status_code < 500 + status_code, body = self._http_request( + "GET", url, payload=None, timeout=3.0 + ) except Exception: return False + err_code = str(body.get("code") or "") + err_msg = str(body.get("error") or f"HTTP {status_code}") + + if status_code in (401, 403) or self._is_non_retryable_auth_error( + f"{err_code} {err_msg}" + ): + self.heartbeat.stop(reason=err_msg) + raise DataStackAuthError(err_msg, code=err_code or "AUTH_ERROR") + + return 200 <= status_code < 300 and body.get("status") == "online" + def _http_request( self, method: str, @@ -357,7 +437,9 @@ class DataStackClient: body = resp.json() if resp.text else {} except Exception: body = {"error": resp.text} - return resp.status_code, body if isinstance(body, dict) else {"data": body} + return resp.status_code, ( + body if isinstance(body, dict) else {"data": body} + ) # Standard library urllib fallback when `requests` is not installed data_bytes = ( @@ -384,7 +466,9 @@ class DataStackClient: parsed_err = {"error": raw_err or str(http_err)} return ( http_err.code, - parsed_err if isinstance(parsed_err, dict) else {"error": str(parsed_err)}, + parsed_err + if isinstance(parsed_err, dict) + else {"error": str(parsed_err)}, ) def _send_via_http_fallback( @@ -404,22 +488,31 @@ class DataStackClient: "transport": "HTTP_FALLBACK", } - err_msg = str(body.get("error") or f"HTTP {status_code}") + errors_list = body.get("errors") + first_err = ( + errors_list[0] + if isinstance(errors_list, list) and len(errors_list) > 0 + else None + ) + err_msg = str(body.get("error") or first_err or f"HTTP {status_code}") err_code = str(body.get("code") or "") + combined = f"{err_code} {err_msg}" if status_code in (401, 403) or self._is_non_retryable_auth_error( - f"{err_code} {err_msg}" + combined ): self.heartbeat.stop(reason=err_msg) raise DataStackAuthError(err_msg, code=err_code or "AUTH_ERROR") - if status_code == 413 or "PAYLOAD_TOO_LARGE" in f"{err_code} {err_msg}": + if status_code == 413 or "PAYLOAD_TOO_LARGE" in combined: raise DataStackPayloadTooLargeError(err_msg) - if status_code == 429 or "RATE_LIMIT_EXCEEDED" in f"{err_code} {err_msg}": + if status_code == 429 or "RATE_LIMIT_EXCEEDED" in combined: raise DataStackRateLimitError(err_msg) - raise DataStackConnectionError(f"HTTP fallback error ({status_code}): {err_msg}") + raise DataStackConnectionError( + f"HTTP fallback error ({status_code}): {err_msg}" + ) def _emit_via_websocket( self, @@ -428,7 +521,9 @@ class DataStackClient: timeout: Optional[float] = None, ) -> Dict[str, Any]: if self.sio is None or not self.is_connected or not self.sio.connected: - raise DataStackConnectionError("WebSocket is not currently connected") + raise DataStackConnectionError( + "WebSocket is not currently connected" + ) ack_event = threading.Event() response_data: Dict[str, Any] = {} @@ -442,8 +537,12 @@ class DataStackClient: self.sio.emit(event_name, payload, callback=ack_callback) except Exception as exc: self.is_connected = False - self.heartbeat.trigger_fallback(reason=f"WebSocket emit error: {exc}") - raise DataStackConnectionError(f"WebSocket emit failed: {exc}") from exc + self.heartbeat.trigger_fallback( + reason=f"WebSocket emit error: {exc}" + ) + raise DataStackConnectionError( + f"WebSocket emit failed: {exc}" + ) from exc wait_sec = timeout or self.config.request_timeout_sec if not ack_event.wait(timeout=wait_sec): @@ -456,15 +555,28 @@ class DataStackClient: ) if not response_data.get("success", False): - err_msg = str(response_data.get("error") or "WebSocket event rejected") - if self._is_non_retryable_auth_error(err_msg): + errors_list = response_data.get("errors") + first_err = ( + errors_list[0] + if isinstance(errors_list, list) and len(errors_list) > 0 + else None + ) + err_msg = str( + response_data.get("error") + or first_err + or "WebSocket event rejected" + ) + err_code = str(response_data.get("code") or "") + combined = f"{err_code} {err_msg}" + + if self._is_non_retryable_auth_error(combined): self.heartbeat.stop(reason=err_msg) - raise DataStackAuthError(err_msg) - if "PAYLOAD_TOO_LARGE" in err_msg: + raise DataStackAuthError(err_msg, code=err_code or "AUTH_ERROR") + if "PAYLOAD_TOO_LARGE" in combined: raise DataStackPayloadTooLargeError(err_msg) - if "RATE_LIMIT_EXCEEDED" in err_msg: + if "RATE_LIMIT_EXCEEDED" in combined: raise DataStackRateLimitError(err_msg) - raise DataStackError(err_msg) + raise DataStackError(err_msg, code=err_code or "WS_REJECTED") return { **response_data, @@ -478,6 +590,7 @@ class DataStackClient: http_payload: Dict[str, Any], spool_items: List[Dict[str, Any]], timeout: Optional[float] = None, + allow_spool: bool = True, ) -> Dict[str, Any]: """ Core resilient dispatcher: @@ -485,8 +598,8 @@ class DataStackClient: 2. On WebSocket failure/disconnect, triggers `HeartbeatMonitor` (`FALLBACK_HTTP`) and automatically sends via HTTP `POST /api/events`. 3. Applies exponential backoff with jitter on `RATE_LIMIT_EXCEEDED` or transient errors. - 4. If both WebSocket and HTTP are unreachable (device offline), spools events to - `.datastack_spool/spool.jsonl` and triggers `OFFLINE_SPOOL`. + 4. If both WebSocket and HTTP are unreachable (device offline) and `allow_spool` is True, + spools events to `.datastack_spool/spool.jsonl` and triggers `OFFLINE_SPOOL`. """ self._ensure_token_present() max_attempts = max(1, self.config.max_retries + 1) @@ -494,16 +607,29 @@ class DataStackClient: for attempt in range(max_attempts): if attempt > 0: - delay = self.config.retry_backoff_base_sec * (2 ** (attempt - 1)) - jitter = random.uniform(0.0, 0.25 * delay) + # Use at least 1.0s delay when backing off from RATE_LIMIT_EXCEEDED so the 1s sliding window resets + base_delay = ( + max(1.0, self.config.retry_backoff_base_sec) + if isinstance(last_exception, DataStackRateLimitError) + else self.config.retry_backoff_base_sec + ) + delay = base_delay * (2 ** (attempt - 1)) + jitter = random.uniform(0.0, 0.2 * delay) time.sleep(delay + jitter) # 1. Try Primary WebSocket if connected if self.is_connected and self.sio is not None and self.sio.connected: try: - return self._emit_via_websocket( + res = self._emit_via_websocket( ws_event, ws_payload, timeout=timeout ) + if ( + allow_spool + and not self._is_draining_spool + and self.spooler.pending_count() > 0 + ): + self._drain_offline_spool() + return res except (DataStackAuthError, DataStackPayloadTooLargeError): raise except DataStackRateLimitError as rl_err: @@ -516,11 +642,21 @@ class DataStackClient: # 2. Try HTTP Fallback (`POST /api/events`) if self.config.enable_http_fallback: try: - if self.heartbeat.state == ConnectionState.CONNECTED_IDLE: + res = self._send_via_http_fallback(http_payload) + if self.heartbeat.state in ( + ConnectionState.CONNECTED_IDLE, + ConnectionState.OFFLINE_SPOOL, + ): self.heartbeat.trigger_fallback( reason="WebSocket not connected; routing via HTTP fallback" ) - return self._send_via_http_fallback(http_payload) + if ( + allow_spool + and not self._is_draining_spool + and self.spooler.pending_count() > 0 + ): + self._drain_offline_spool() + return res except (DataStackAuthError, DataStackPayloadTooLargeError): raise except DataStackRateLimitError as rl_err: @@ -531,12 +667,17 @@ class DataStackClient: # 3. If rate-limited or payload-rejected after all retries, raise directly if isinstance( - last_exception, (DataStackRateLimitError, DataStackPayloadTooLargeError) + last_exception, + (DataStackRateLimitError, DataStackPayloadTooLargeError), ): raise last_exception # 4. Device is offline (both WebSocket and HTTP failed): Spool to local disk - if self.config.offline_spool_enabled and spool_items: + if ( + allow_spool + and self.config.offline_spool_enabled + and spool_items + ): spooled_count = self.spooler.enqueue(spool_items) self.heartbeat.trigger_offline( reason=str(last_exception or "Network unreachable") @@ -557,29 +698,42 @@ class DataStackClient: """ Replays any offline spooled events from `.datastack_spool/spool.jsonl` once connectivity (WebSocket or HTTP fallback) is restored. + Routes batches through `_send_formatted_batch_recursive(..., allow_spool=False)` + so oversized or rate-limited spool batches automatically split and pace. """ if not self.config.offline_spool_enabled: return 0 - total_flushed = 0 - while True: - batch = self.spooler.peek_batch(limit=self.config.batch_chunk_size) - if not batch: - break + with self._lock: + if self._is_draining_spool: + return 0 + self._is_draining_spool = True - try: - if self.is_connected and self.sio is not None and self.sio.connected: - self._emit_via_websocket("batch_sync", batch) - elif self.config.enable_http_fallback: - self._send_via_http_fallback({"batch": batch}) - else: + total_flushed = 0 + try: + while True: + batch = self.spooler.peek_batch( + limit=self.config.batch_chunk_size + ) + if not batch: + break + + res = self._send_formatted_batch_recursive( + batch, + timeout=self.config.request_timeout_sec, + file_name=None, + allow_spool=False, + ) + if not res.get("success", False): break self.spooler.ack_batch(len(batch)) total_flushed += len(batch) - except Exception as exc: - logger.debug("[DataStack] Spool drain paused: %s", exc) - break + except Exception as exc: + logger.debug("[DataStack] Spool drain paused: %s", exc) + finally: + with self._lock: + self._is_draining_spool = False return total_flushed @@ -611,6 +765,7 @@ class DataStackClient: http_payload=envelope, spool_items=[envelope], timeout=timeout, + allow_spool=True, ) def sync( @@ -624,22 +779,33 @@ class DataStackClient: def send_batch( self, - events: List[Dict[str, Any]], + events: Optional[List[Dict[str, Any]]] = None, timeout: Optional[float] = None, file_name: Optional[str] = None, + *, + records: Optional[List[Dict[str, Any]]] = None, ) -> Dict[str, Any]: """ - Sends a batch of events. If the batch triggers `413 PAYLOAD_TOO_LARGE`, - automatically splits the batch into smaller halves and retries. + Sends a batch of events (accepts either `events=[...]` or `records=[...]`). + - If the batch triggers `413 PAYLOAD_TOO_LARGE`, automatically splits the batch + in half recursively and retries. + - If a subset of events in the batch hits `429 RATE_LIMIT_EXCEEDED` + (`rateLimitedEventIds`), automatically backs off and retries the remaining + rate-limited events so zero records are dropped. """ - if not isinstance(events, list) or len(events) == 0: + source_list = events if events is not None else records + if not isinstance(source_list, list) or len(source_list) == 0: return {"success": True, "processed": 0, "rejected": 0} formatted: List[Dict[str, Any]] = [] - for item in events: + for item in source_list: safe_item = item if isinstance(item, dict) else {"value": item} - evt_id = str(safe_item.get("eventId") or "").strip() or _make_event_id() - evt_ts = str(safe_item.get("timestamp") or "").strip() or _utc_now_iso() + evt_id = ( + str(safe_item.get("eventId") or "").strip() or _make_event_id() + ) + evt_ts = ( + str(safe_item.get("timestamp") or "").strip() or _utc_now_iso() + ) evt_data = safe_item.get("data", safe_item) formatted.append( { @@ -650,22 +816,28 @@ class DataStackClient: ) return self._send_formatted_batch_recursive( - formatted, timeout=timeout, file_name=file_name + formatted, + timeout=timeout, + file_name=file_name, + allow_spool=True, ) def batch_sync( self, - events: List[Dict[str, Any]], + events: Optional[List[Dict[str, Any]]] = None, timeout: float = 10.0, + *, + records: Optional[List[Dict[str, Any]]] = None, ) -> Dict[str, Any]: - """Alias for `send_batch()` preserving backward compatibility.""" - return self.send_batch(events=events, timeout=timeout) + """Alias for `send_batch()` supporting both `events` and `records` kwargs.""" + return self.send_batch(events=events, records=records, timeout=timeout) def _send_formatted_batch_recursive( self, formatted_batch: List[Dict[str, Any]], timeout: Optional[float] = None, file_name: Optional[str] = None, + allow_spool: bool = True, ) -> Dict[str, Any]: http_payload: Dict[str, Any] = ( {"fileName": file_name, "records": formatted_batch} @@ -673,36 +845,118 @@ class DataStackClient: else {"batch": formatted_batch} ) try: - return self._execute_with_retry_and_fallback( + res = self._execute_with_retry_and_fallback( ws_event="batch_sync", ws_payload=formatted_batch, http_payload=http_payload, spool_items=formatted_batch, timeout=timeout, + allow_spool=allow_spool, ) except DataStackPayloadTooLargeError: if len(formatted_batch) <= 1: raise mid = len(formatted_batch) // 2 left_res = self._send_formatted_batch_recursive( - formatted_batch[:mid], timeout=timeout, file_name=file_name + formatted_batch[:mid], + timeout=timeout, + file_name=file_name, + allow_spool=allow_spool, ) right_res = self._send_formatted_batch_recursive( - formatted_batch[mid:], timeout=timeout, file_name=file_name + formatted_batch[mid:], + timeout=timeout, + file_name=file_name, + allow_spool=allow_spool, ) return { "success": bool( - left_res.get("success", True) and right_res.get("success", True) + left_res.get("success", True) + and right_res.get("success", True) ), "processed": int(left_res.get("processed", 0)) + int(right_res.get("processed", 0)), "rejected": int(left_res.get("rejected", 0)) + int(right_res.get("rejected", 0)), + "duplicates": int(left_res.get("duplicates", 0)) + + int(right_res.get("duplicates", 0)), "spooledCount": int(left_res.get("spooledCount", 0)) + int(right_res.get("spooledCount", 0)), - "transport": right_res.get("transport") or left_res.get("transport"), + "transport": right_res.get("transport") + or left_res.get("transport"), } + # Handle partial-batch rate limiting (`rateLimitedEventIds`) so zero rows are dropped + rate_limited_ids = res.get("rateLimitedEventIds") + if ( + isinstance(rate_limited_ids, list) + and len(rate_limited_ids) > 0 + and self.config.max_retries > 0 + ): + pending_id_set = {str(eid) for eid in rate_limited_ids} + remaining_items = [ + item + for item in formatted_batch + if str(item.get("eventId")) in pending_id_set + ] + total_processed = int(res.get("processed", 0)) + total_duplicates = int(res.get("duplicates", 0)) + total_spooled = int(res.get("spooledCount", 0)) + last_transport = res.get("transport") + + for retry_round in range(1, self.config.max_retries + 2): + if not remaining_items: + break + # Wait slightly over 1.0s so the server's 1-second rate-limit window resets + wait_delay = max(1.05, self.config.retry_backoff_base_sec) + random.uniform( + 0.0, 0.15 + ) + time.sleep(wait_delay) + + sub_http_payload: Dict[str, Any] = ( + {"fileName": file_name, "records": remaining_items} + if file_name + else {"batch": remaining_items} + ) + sub_res = self._execute_with_retry_and_fallback( + ws_event="batch_sync", + ws_payload=remaining_items, + http_payload=sub_http_payload, + spool_items=remaining_items, + timeout=timeout, + allow_spool=allow_spool, + ) + total_processed += int(sub_res.get("processed", 0)) + total_duplicates += int(sub_res.get("duplicates", 0)) + total_spooled += int(sub_res.get("spooledCount", 0)) + last_transport = sub_res.get("transport") or last_transport + + next_rl_ids = sub_res.get("rateLimitedEventIds") + if not isinstance(next_rl_ids, list) or len(next_rl_ids) == 0: + remaining_items = [] + break + next_set = {str(eid) for eid in next_rl_ids} + remaining_items = [ + item + for item in remaining_items + if str(item.get("eventId")) in next_set + ] + + oversized_ids = res.get("oversizedEventIds") + final_rejected = len(remaining_items) + ( + len(oversized_ids) if isinstance(oversized_ids, list) else 0 + ) + return { + "success": final_rejected == 0 or total_processed > 0, + "processed": total_processed, + "rejected": final_rejected, + "duplicates": total_duplicates, + "spooledCount": total_spooled, + "transport": last_transport, + } + + return res + # ========================================================================= # Direct File Streaming (.csv, .json, .jsonl / .ndjson) # ========================================================================= @@ -715,8 +969,8 @@ class DataStackClient: ) -> Dict[str, Any]: """ Streams a `.csv`, `.json`, or `.jsonl` (`.ndjson`) file directly to DataStack - in safe chunks with deterministic idempotency IDs, automatic HTTP fallback, - and offline spooling support. + in safe chunks with deterministic SHA-256 idempotency IDs, automatic HTTP fallback, + partial-batch rate-limit pacing, and offline spooling support. """ path = Path(file_path).expanduser().resolve() if not path.exists() or not path.is_file(): @@ -728,9 +982,12 @@ class DataStackClient: f"Unsupported file extension '{ext}'. Supported formats: .csv, .json, .jsonl, .ndjson" ) - effective_chunk = max(1, int(chunk_size or self.config.batch_chunk_size)) + effective_chunk = max( + 1, int(chunk_size or self.config.batch_chunk_size) + ) total_processed = 0 total_rejected = 0 + total_duplicates = 0 total_spooled = 0 chunks_sent = 0 last_transport = "WEBSOCKET" @@ -742,10 +999,25 @@ class DataStackClient: if extra_metadata and isinstance(record, dict) else record ) + existing_id = ( + str(enriched.get("eventId") or "").strip() + if isinstance(enriched, dict) + else "" + ) + evt_id = existing_id or _make_deterministic_file_event_id( + path.stem, idx, enriched + ) + existing_ts = ( + str(enriched.get("timestamp") or "").strip() + if isinstance(enriched, dict) + else "" + ) + evt_ts = existing_ts or _utc_now_iso() + buffer.append( { - "eventId": f"file_{path.stem}_{idx}_{uuid.uuid4().hex[:6]}", - "timestamp": _utc_now_iso(), + "eventId": evt_id, + "timestamp": evt_ts, "data": enriched, } ) @@ -753,6 +1025,7 @@ class DataStackClient: res = self.send_batch(buffer, file_name=path.name) total_processed += int(res.get("processed", 0)) total_rejected += int(res.get("rejected", 0)) + total_duplicates += int(res.get("duplicates", 0)) total_spooled += int(res.get("spooledCount", 0)) last_transport = str(res.get("transport", last_transport)) chunks_sent += 1 @@ -762,6 +1035,7 @@ class DataStackClient: res = self.send_batch(buffer, file_name=path.name) total_processed += int(res.get("processed", 0)) total_rejected += int(res.get("rejected", 0)) + total_duplicates += int(res.get("duplicates", 0)) total_spooled += int(res.get("spooledCount", 0)) last_transport = str(res.get("transport", last_transport)) chunks_sent += 1 @@ -773,6 +1047,7 @@ class DataStackClient: "chunksSent": chunks_sent, "processed": total_processed, "rejected": total_rejected, + "duplicates": total_duplicates, "spooledCount": total_spooled, "transport": last_transport, } @@ -797,20 +1072,34 @@ class DataStackClient: raise DataStackFileError( f"Malformed JSONL at line {line_no} in {path.name}: {exc}" ) from exc - yield parsed if isinstance(parsed, dict) else {"value": parsed} + yield ( + parsed + if isinstance(parsed, dict) + else {"value": parsed} + ) elif ext == ".json": with path.open("r", encoding="utf-8-sig") as fh: parsed = json.load(fh) if isinstance(parsed, list): for item in parsed: - yield item if isinstance(item, dict) else {"value": item} + yield ( + item if isinstance(item, dict) else {"value": item} + ) elif isinstance(parsed, dict): if isinstance(parsed.get("records"), list): for item in parsed["records"]: - yield item if isinstance(item, dict) else {"value": item} + yield ( + item + if isinstance(item, dict) + else {"value": item} + ) elif isinstance(parsed.get("data"), list): for item in parsed["data"]: - yield item if isinstance(item, dict) else {"value": item} + yield ( + item + if isinstance(item, dict) + else {"value": item} + ) else: yield parsed else: diff --git a/sdk/python/datastack/datastack/heartbeat.py b/sdk/python/datastack/datastack/heartbeat.py index a38e210..34e9893 100644 --- a/sdk/python/datastack/datastack/heartbeat.py +++ b/sdk/python/datastack/datastack/heartbeat.py @@ -39,7 +39,7 @@ class HeartbeatMonitor: interval_sec: float, recovery_callback: Callable[[], bool], ) -> None: - self.interval_sec = max(1.0, float(interval_sec)) + self.interval_sec = max(0.2, float(interval_sec)) self._recovery_callback = recovery_callback self._state = ConnectionState.CONNECTED_IDLE self._closed = False @@ -169,7 +169,7 @@ class HeartbeatMonitor: continue with self._lock: - if self._state == ConnectionState.STOPPED: + if self._closed or self._state == ConnectionState.STOPPED: break self._recovery_attempts += 1 diff --git a/server/index.js b/server/index.js index 9628c8b..0f5d5c2 100644 --- a/server/index.js +++ b/server/index.js @@ -203,7 +203,8 @@ async function syncSettingsFromDatabase() { await ensureDatabaseSchema(); const res = await dbPool.query( `SELECT max_events_per_sec, max_payload_size_kb, batch_flush_threshold, - batch_flush_interval_ms, file_rotation_mb, jwt_secret, enable_idempotency + batch_flush_interval_ms, file_rotation_mb, jwt_secret, enable_idempotency, + auto_prune_days FROM admin_settings WHERE id = 'default'`, ); @@ -227,6 +228,11 @@ async function syncSettingsFromDatabase() { writer.setMaxFileSizeMb(nextRotationMb); } + const nextPruneDays = Number(row.auto_prune_days) || 30; + if (nextPruneDays > 0) { + writer.pruneExpiredPartitions(nextPruneDays); + } + const nextIntervalMs = Math.max( 100, Number(row.batch_flush_interval_ms) || 2000, diff --git a/server/socket/handlers.js b/server/socket/handlers.js index 4211ab1..7f8c1c7 100644 --- a/server/socket/handlers.js +++ b/server/socket/handlers.js @@ -73,6 +73,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { return { allowed: false, duplicate: true, + code: "DUPLICATE_EVENT", reason: "Duplicate event dropped", }; } @@ -91,6 +92,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { return { allowed: false, duplicate: false, + code: "RATE_LIMIT_EXCEEDED", reason: `RATE_LIMIT_EXCEEDED: Limit of ${effectiveLimit} events/sec exceeded`, }; } else { @@ -106,6 +108,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { return { allowed: false, duplicate: false, + code: "PAYLOAD_TOO_LARGE", reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`, }; } @@ -166,6 +169,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { if (callback) { callback({ success: false, + code: "CLIENT_REVOKED", error: dbCheck.reason || "CLIENT_REVOKED", }); } @@ -195,7 +199,12 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { return; } if (callback) { - callback({ success: false, eventId, error: check.reason }); + callback({ + success: false, + eventId, + code: check.code, + error: check.reason, + }); } return; } @@ -236,6 +245,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { success: false, processed: 0, rejected: batch.length, + code: "CLIENT_REVOKED", error: dbCheck.reason || "CLIENT_REVOKED", }); } @@ -247,6 +257,11 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { dbCheck.rateLimit || socket.data.rateLimit || 100; let processed = 0; let rejected = 0; + let duplicates = 0; + let hitRateLimit = false; + let hitPayloadLimit = false; + const rateLimitedEventIds = []; + const oversizedEventIds = []; const errors = []; for (const item of batch) { @@ -265,11 +280,21 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { effectiveClientRate, ); if (!check.allowed) { - if (!check.duplicate) { - rejected += 1; - if (errors.length < 5) { - errors.push(`${eventId}: ${check.reason}`); - } + if (check.duplicate) { + duplicates += 1; + continue; + } + rejected += 1; + if (check.code === "RATE_LIMIT_EXCEEDED") { + hitRateLimit = true; + rateLimitedEventIds.push(eventId); + } + if (check.code === "PAYLOAD_TOO_LARGE") { + hitPayloadLimit = true; + oversizedEventIds.push(eventId); + } + if (errors.length < 5) { + errors.push(`${eventId}: ${check.reason}`); } continue; } @@ -288,11 +313,36 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) { onBufferPush(); } + if (processed === 0 && duplicates === 0 && rejected > 0) { + const code = hitPayloadLimit + ? "PAYLOAD_TOO_LARGE" + : hitRateLimit + ? "RATE_LIMIT_EXCEEDED" + : "BATCH_REJECTED"; + if (callback) { + callback({ + success: false, + processed, + rejected, + duplicates, + code, + error: errors[0] || "Batch validation failed", + errors, + ...(rateLimitedEventIds.length > 0 ? { rateLimitedEventIds } : {}), + ...(oversizedEventIds.length > 0 ? { oversizedEventIds } : {}), + }); + } + return; + } + if (callback) { callback({ success: true, processed, rejected, + duplicates, + ...(rateLimitedEventIds.length > 0 ? { rateLimitedEventIds } : {}), + ...(oversizedEventIds.length > 0 ? { oversizedEventIds } : {}), ...(errors.length > 0 ? { errors } : {}), }); } diff --git a/server/workers/csvWriter.js b/server/workers/csvWriter.js index 1feaa9f..bd75f18 100644 --- a/server/workers/csvWriter.js +++ b/server/workers/csvWriter.js @@ -97,6 +97,43 @@ class CsvWriterWorker { return { written: events.length, files }; } + + pruneExpiredPartitions(autoPruneDays) { + const days = Number(autoPruneDays); + if (!Number.isFinite(days) || days <= 0 || !fs.existsSync(this.baseDir)) { + return 0; + } + + const cutoffDate = new Date(Date.now() - Math.floor(days) * 86400000) + .toISOString() + .split('T')[0]; + + let prunedCount = 0; + try { + const clientDirs = fs.readdirSync(this.baseDir); + for (const clientDir of clientDirs) { + const fullClientPath = path.join(this.baseDir, clientDir); + if (!fs.statSync(fullClientPath).isDirectory()) continue; + + const files = fs.readdirSync(fullClientPath); + for (const file of files) { + if (!file.endsWith('.csv')) continue; + const datePart = file.replace('.csv', '').split('-part')[0]; + if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) { + try { + fs.unlinkSync(path.join(fullClientPath, file)); + prunedCount += 1; + } catch { + // Ignore individual file unlink failure + } + } + } + } + } catch { + // Ignore directory scan failure + } + return prunedCount; + } } module.exports = { CsvWriterWorker };