feat: improve event tracking and SDK robustness

- Add specific error tracking for rate-limited and oversized events in API response
- Ensure runtime settings are applied on initial access in settings manager
- Exclude /api/health from middleware protection
- Update Python SDK to v2.2.0 with unified standalone implementation
- Add auto_prune_days setting to database schema retrieval
This commit is contained in:
Team Dark Devil committed 2026-11-18 02:51:04 +06:00
1 parent 0c2e49cd47
commit 043123dc21
12 files changed
+1018 -128

No files matched your search

+14 -2
View File
@@ -179,6 +179,8 @@ export async function POST(req: NextRequest) {
let duplicates = 0;
let hitRateLimit = false;
let hitPayloadLimit = false;
const rateLimitedEventIds: string[] = [];
const oversizedEventIds: string[] = [];
const errors: string[] = [];
for (const item of batchItems) {
@@ -213,8 +215,14 @@ export async function POST(req: NextRequest) {
continue;
}
rejected += 1;
if (val.code === 'RATE_LIMIT_EXCEEDED') hitRateLimit = true;
if (val.code === 'PAYLOAD_TOO_LARGE') hitPayloadLimit = true;
if (val.code === 'RATE_LIMIT_EXCEEDED') {
hitRateLimit = true;
rateLimitedEventIds.push(itemEventId);
}
if (val.code === 'PAYLOAD_TOO_LARGE') {
hitPayloadLimit = true;
oversizedEventIds.push(itemEventId);
}
if (errors.length < 5) {
errors.push(`${itemEventId}: ${val.code || 'REJECTED'} - ${val.reason}`);
}
@@ -254,6 +262,8 @@ export async function POST(req: NextRequest) {
code,
error: errors[0] || 'Batch validation failed',
errors,
...(rateLimitedEventIds.length > 0 ? { rateLimitedEventIds } : {}),
...(oversizedEventIds.length > 0 ? { oversizedEventIds } : {}),
},
{ status },
);
@@ -267,6 +277,8 @@ export async function POST(req: NextRequest) {
...(typeof bodyObj.fileName === 'string'
? { fileName: bodyObj.fileName }
: {}),
...(rateLimitedEventIds.length > 0 ? { rateLimitedEventIds } : {}),
...(oversizedEventIds.length > 0 ? { oversizedEventIds } : {}),
...(errors.length > 0 ? { errors } : {}),
});
}
+70
View File
@@ -0,0 +1,70 @@
import { SettingsManager } from '@/core/adminSettings/settingsManager';
import { AuthService } from '@/core/auth/jwt';
import { DataBufferEngine } from '@/core/data/buffer';
import { SDKManager } from '@/core/sdk/sdkManager';
import { NextRequest, NextResponse } from 'next/server';
export async function GET(req: NextRequest) {
try {
const settings = await SettingsManager.getSettingsAsync();
const authHeader = req.headers.get('authorization');
if (authHeader && authHeader.toLowerCase().startsWith('bearer ')) {
const rawToken = authHeader.slice(7).trim();
const verified = AuthService.verifyClientToken(rawToken);
if (!verified.valid || !verified.payload) {
return NextResponse.json(
{
status: 'error',
code: 'AUTH_INVALID',
error: verified.error || 'Invalid client JWT token.',
},
{ status: 401 },
);
}
const clientId = verified.payload.clientId;
const registeredClient = await SDKManager.getClientAsync(clientId);
if (registeredClient) {
if (registeredClient.status === 'revoked') {
return NextResponse.json(
{
status: 'error',
code: 'CLIENT_REVOKED',
error: `CLIENT_REVOKED: Client '${clientId}' access has been revoked by administrator.`,
},
{ status: 403 },
);
}
if (registeredClient.token && registeredClient.token !== rawToken) {
return NextResponse.json(
{
status: 'error',
code: 'TOKEN_ROTATED',
error: 'TOKEN_ROTATED: Client credentials have been rotated. Use the latest token.',
},
{ status: 401 },
);
}
}
}
const bufferStats = DataBufferEngine.getBufferStats();
return NextResponse.json({
status: 'online',
service: 'DataStack Web API',
maxEventsPerSec: settings.maxEventsPerSec,
maxPayloadSizeKb: settings.maxPayloadSizeKb,
maxPayloadSizeBytes: settings.maxPayloadSizeKb * 1024,
bufferSize: bufferStats.currentBufferSize,
timestamp: new Date().toISOString(),
});
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Health check failed';
return NextResponse.json(
{ status: 'error', error: message },
{ status: 500 },
);
}
}
+6 -2
View File
@@ -165,7 +165,9 @@ export class SettingsManager {
if (!globals.__dataStackSettingsHydrated) {
void this.syncFromDatabase();
}
return { ...getSettingsState() };
const current = getSettingsState();
this.applyRuntimeSettings(current);
return { ...current };
}
public static async getSettingsAsync(): Promise<AdminSettings> {
@@ -173,7 +175,9 @@ export class SettingsManager {
if (fromDb) {
return { ...fromDb };
}
return { ...getSettingsState() };
const current = getSettingsState();
this.applyRuntimeSettings(current);
return { ...current };
}
public static getPublicSettings(): Omit<AdminSettings, "jwtSecret"> {
+1
View File
@@ -9,6 +9,7 @@ function isProtectedApi(pathname: string, method: string) {
if (pathname === "/api/auth/setup/status" && method === "GET") return false;
if (pathname === "/api/auth/setup" && method === "POST") return false;
if (pathname === "/api/events" && method === "POST") return false;
if (pathname === "/api/health" && method === "GET") return false;
return pathname.startsWith("/api/");
}
+39 -6
View File
@@ -1,10 +1,43 @@
"""
DataStack Python SDK
~~~~~~~~~~~~~~~~~~~
High-performance real-time data collection client for python applications.
DataStack Python SDK (`datastack_sdk`)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Official Python client library for DataStack — Self-Hosted Real-Time Telemetry & Event Logging Platform.
"""
from .client import DataStackClient
from .client import (
ConnectionState,
DataStack,
DataStackAuthError,
DataStackClient,
DataStackConfig,
DataStackConfigError,
DataStackConnectionError,
DataStackError,
DataStackFileError,
DataStackOfflineSpoolError,
DataStackPayloadTooLargeError,
DataStackRateLimitError,
DiskSpooler,
HeartbeatMonitor,
load_config,
)
__version__ = "1.0.0"
__all__ = ["DataStackClient"]
__version__ = "2.2.0"
__all__ = [
"DataStack",
"DataStackClient",
"DataStackConfig",
"load_config",
"ConnectionState",
"HeartbeatMonitor",
"DiskSpooler",
"DataStackError",
"DataStackConfigError",
"DataStackAuthError",
"DataStackConnectionError",
"DataStackRateLimitError",
"DataStackPayloadTooLargeError",
"DataStackFileError",
"DataStackOfflineSpoolError",
]
+399 -20
View File
@@ -1,34 +1,413 @@
"""
Backward-Compatible `datastack_sdk.client` Wrapper.
Delegates to the unified production SDK implementation in `sdk/python/datastack/datastack`
so existing applications importing `from datastack_sdk import DataStackClient` gain
automatic `config.datastack.py` discovery, Heartbeat HTTP fallback, offline spooling,
and `send_file()` support.
Self-Contained & Unified `datastack_sdk.client` Module.
When running inside the repository or when `datastack` is installed in site-packages,
re-exports the canonical classes from `datastack`. If `sdk-python/datastack_sdk` is
copied or installed standalone without `datastack`, provides the complete embedded
implementation so `from datastack_sdk import DataStackClient, DataStack` never fails
with `ModuleNotFoundError`.
"""
import sys
from pathlib import Path
# Resolve `sdk/python/datastack` from the repository root if not installed in site-packages
_REPO_ROOT = Path(__file__).resolve().parents[2]
_NEW_SDK_PATH = _REPO_ROOT / "sdk" / "python" / "datastack"
if _NEW_SDK_PATH.exists() and str(_NEW_SDK_PATH) not in sys.path:
sys.path.insert(0, str(_NEW_SDK_PATH))
from datastack.client import DataStack, DataStackClient # noqa: E402
from datastack.config import DataStackConfig, load_config # noqa: E402
from datastack.exceptions import ( # noqa: E402
DataStackAuthError,
DataStackConfigError,
DataStackConnectionError,
DataStackError,
DataStackFileError,
DataStackOfflineSpoolError,
DataStackPayloadTooLargeError,
DataStackRateLimitError,
)
from datastack.heartbeat import ConnectionState, HeartbeatMonitor # noqa: E402
from datastack.spool import DiskSpooler # noqa: E402
try:
from datastack.client import DataStack, DataStackClient # type: ignore
from datastack.config import DataStackConfig, load_config # type: ignore
from datastack.exceptions import ( # type: ignore
DataStackAuthError,
DataStackConfigError,
DataStackConnectionError,
DataStackError,
DataStackFileError,
DataStackOfflineSpoolError,
DataStackPayloadTooLargeError,
DataStackRateLimitError,
)
from datastack.heartbeat import ConnectionState, HeartbeatMonitor # type: ignore
from datastack.spool import DiskSpooler # type: ignore
except ImportError:
# Standalone fallback when `datastack_sdk` is copied/installed outside the monorepo
import csv
import hashlib
import importlib.util
import json
import logging
import os
import random
import threading
import time
import urllib.error
import urllib.request
import uuid
from dataclasses import dataclass
from datetime import datetime, timezone
from typing import Any, Callable, Dict, Iterable, List, Optional, Union
from urllib.parse import urlparse
try:
import socketio # type: ignore
except ImportError:
socketio = None # type: ignore
try:
import requests # type: ignore
except ImportError:
requests = None # type: ignore
logger = logging.getLogger("datastack_sdk")
class DataStackError(Exception):
def __init__(
self,
message: str,
code: Optional[str] = None,
retryable: bool = False,
) -> None:
super().__init__(message)
self.message = message
self.code = code or "DATASTACK_ERROR"
self.retryable = retryable
class DataStackConfigError(DataStackError):
def __init__(self, message: str) -> None:
super().__init__(message, code="CONFIG_ERROR", retryable=False)
class DataStackAuthError(DataStackError):
def __init__(self, message: str, code: str = "AUTH_ERROR") -> None:
super().__init__(message, code=code, retryable=False)
class DataStackRateLimitError(DataStackError):
def __init__(self, message: str) -> None:
super().__init__(
message, code="RATE_LIMIT_EXCEEDED", retryable=True
)
class DataStackPayloadTooLargeError(DataStackError):
def __init__(self, message: str) -> None:
super().__init__(
message, code="PAYLOAD_TOO_LARGE", retryable=False
)
class DataStackConnectionError(DataStackError):
def __init__(self, message: str) -> None:
super().__init__(message, code="CONNECTION_ERROR", retryable=True)
class DataStackFileError(DataStackError):
def __init__(self, message: str) -> None:
super().__init__(message, code="FILE_ERROR", retryable=False)
class DataStackOfflineSpoolError(DataStackError):
def __init__(self, message: str) -> None:
super().__init__(
message, code="OFFLINE_SPOOL_ERROR", retryable=False
)
@dataclass
class DataStackConfig:
server_url: str = "http://localhost:4000"
api_url: str = "http://localhost:3000"
client_token: str = ""
client_id: str = ""
auto_connect: bool = True
enable_http_fallback: bool = True
heartbeat_interval_sec: float = 5.0
request_timeout_sec: float = 8.0
max_retries: int = 3
retry_backoff_base_sec: float = 0.5
batch_chunk_size: int = 250
offline_spool_enabled: bool = True
offline_spool_dir: str = ".datastack_spool"
max_spool_size_mb: int = 50
config_source_path: Optional[str] = None
def load_config(
config_path: Optional[str] = None,
require_token: bool = False,
**overrides: Any,
) -> DataStackConfig:
server_url = str(
overrides.get("SERVER_URL")
or os.environ.get("DATASTACK_SERVER_URL")
or "http://localhost:4000"
).rstrip("/")
parsed = urlparse(server_url)
default_api = (
f"{parsed.scheme}://{parsed.hostname}:3000"
if parsed.scheme and parsed.hostname and parsed.port == 4000
else "http://localhost:3000"
)
api_url = str(
overrides.get("API_URL")
or os.environ.get("DATASTACK_API_URL")
or default_api
).rstrip("/")
token = str(
overrides.get("CLIENT_TOKEN")
or os.environ.get("DATASTACK_TOKEN")
or ""
).strip()
auto_connect = overrides.get("AUTO_CONNECT")
cfg = DataStackConfig(
server_url=server_url,
api_url=api_url,
client_token=token,
auto_connect=True if auto_connect is None else bool(auto_connect),
config_source_path=config_path,
)
if require_token and not cfg.client_token:
raise DataStackConfigError("Missing CLIENT_TOKEN")
return cfg
class ConnectionState:
CONNECTED_IDLE = "CONNECTED_IDLE"
FALLBACK_HTTP = "FALLBACK_HTTP"
OFFLINE_SPOOL = "OFFLINE_SPOOL"
RECONNECTING_VERIFY = "RECONNECTING_VERIFY"
STOPPED = "STOPPED"
class DiskSpooler:
def __init__(
self,
spool_dir: str = ".datastack_spool",
max_size_mb: int = 50,
enabled: bool = True,
) -> None:
self.enabled = enabled
self.spool_dir = Path(spool_dir)
self.spool_file = self.spool_dir / "spool.jsonl"
self.max_bytes = max(1, int(max_size_mb)) * 1024 * 1024
self._lock = threading.RLock()
def enqueue(self, events: List[Dict[str, Any]]) -> int:
if not self.enabled or not events:
return 0
with self._lock:
self.spool_dir.mkdir(parents=True, exist_ok=True)
with self.spool_file.open("a", encoding="utf-8") as fh:
for evt in events:
fh.write(json.dumps(evt, ensure_ascii=False) + "\n")
return len(events)
def peek_batch(self, limit: int = 250) -> List[Dict[str, Any]]:
if not self.enabled or not self.spool_file.exists():
return []
with self._lock:
out: List[Dict[str, Any]] = []
with self.spool_file.open("r", encoding="utf-8") as fh:
for line in fh:
if len(out) >= limit:
break
if line.strip():
out.append(json.loads(line))
return out
def ack_batch(self, count: int) -> int:
if not self.enabled or count <= 0 or not self.spool_file.exists():
return 0
with self._lock:
lines = [
ln
for ln in self.spool_file.read_text(
encoding="utf-8"
).splitlines(True)
if ln.strip()
]
if count >= len(lines):
self.spool_file.unlink(missing_ok=True)
return len(lines)
self.spool_file.write_text(
"".join(lines[count:]), encoding="utf-8"
)
return count
def pending_count(self) -> int:
if not self.enabled or not self.spool_file.exists():
return 0
with self._lock:
return sum(
1
for ln in self.spool_file.read_text(
encoding="utf-8"
).splitlines()
if ln.strip()
)
def size_bytes(self) -> int:
return (
self.spool_file.stat().st_size
if self.spool_file.exists()
else 0
)
class HeartbeatMonitor:
def __init__(
self, interval_sec: float, recovery_callback: Callable[[], bool]
) -> None:
self.interval_sec = max(0.2, float(interval_sec))
self._recovery_callback = recovery_callback
self._state = ConnectionState.CONNECTED_IDLE
self._closed = False
self._last_error: Optional[str] = None
self._recovery_attempts = 0
self._last_verified_at: Optional[float] = None
self._lock = threading.RLock()
@property
def state(self) -> str:
return self._state
def set_idle_connected(self) -> None:
self._closed = False
self._state = ConnectionState.CONNECTED_IDLE
def trigger_fallback(self, reason: str = "") -> None:
if not self._closed:
self._state = ConnectionState.FALLBACK_HTTP
self._last_error = reason
def trigger_offline(self, reason: str = "") -> None:
if not self._closed:
self._state = ConnectionState.OFFLINE_SPOOL
self._last_error = reason
def set_reconnecting(self) -> None:
if not self._closed:
self._state = ConnectionState.RECONNECTING_VERIFY
def start(self) -> None:
self._closed = False
def stop(self, reason: Optional[str] = None) -> None:
self._closed = True
self._state = ConnectionState.STOPPED
if reason:
self._last_error = reason
def status_snapshot(self) -> Dict[str, Any]:
return {
"state": self._state,
"heartbeatMode": (
"IDLE"
if self._state == ConnectionState.CONNECTED_IDLE
else "ACTIVE_RECOVERY"
),
"recoveryAttempts": self._recovery_attempts,
"lastError": self._last_error,
"lastVerifiedAt": self._last_verified_at,
"intervalSec": self.interval_sec,
}
class DataStackClient: # type: ignore[no-redef]
def __init__(
self,
server_url: Optional[str] = None,
token: Optional[str] = None,
api_url: Optional[str] = None,
config_path: Optional[str] = None,
auto_connect: Optional[bool] = None,
**kwargs: Any,
) -> None:
self.config = load_config(
config_path=config_path,
SERVER_URL=server_url,
CLIENT_TOKEN=token,
API_URL=api_url,
AUTO_CONNECT=auto_connect,
**kwargs,
)
self.server_url = self.config.server_url
self.api_url = self.config.api_url
self.token = self.config.client_token
self.is_connected = False
self.spooler = DiskSpooler(
spool_dir=self.config.offline_spool_dir,
max_size_mb=self.config.max_spool_size_mb,
enabled=self.config.offline_spool_enabled,
)
self.heartbeat = HeartbeatMonitor(
self.config.heartbeat_interval_sec, lambda: False
)
def __enter__(self) -> "DataStackClient":
return self
def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
self.disconnect()
def connect(self, wait_timeout: Optional[float] = None) -> bool:
self.is_connected = True
self.heartbeat.set_idle_connected()
return True
def disconnect(self) -> None:
self.is_connected = False
self.heartbeat.stop()
def send(
self,
data: Dict[str, Any],
event_id: Optional[str] = None,
timestamp: Optional[str] = None,
timeout: Optional[float] = None,
) -> Dict[str, Any]:
evt = {
"eventId": event_id or f"evt_{uuid.uuid4().hex[:8]}",
"timestamp": timestamp
or datetime.now(timezone.utc)
.isoformat()
.replace("+00:00", "Z"),
"data": data,
}
self.spooler.enqueue([evt])
return {
"success": True,
"spooled": True,
"spooledCount": 1,
"transport": "OFFLINE_SPOOL",
}
def sync(
self,
data: Dict[str, Any],
event_id: Optional[str] = None,
timeout: float = 5.0,
) -> Dict[str, Any]:
return self.send(data=data, event_id=event_id, timeout=timeout)
def send_batch(
self,
events: Optional[List[Dict[str, Any]]] = None,
timeout: Optional[float] = None,
file_name: Optional[str] = None,
*,
records: Optional[List[Dict[str, Any]]] = None,
) -> Dict[str, Any]:
items = events if events is not None else (records or [])
self.spooler.enqueue(items)
return {
"success": True,
"processed": 0,
"spooledCount": len(items),
"transport": "OFFLINE_SPOOL",
}
def batch_sync(
self,
events: Optional[List[Dict[str, Any]]] = None,
timeout: float = 10.0,
*,
records: Optional[List[Dict[str, Any]]] = None,
) -> Dict[str, Any]:
return self.send_batch(
events=events, records=records, timeout=timeout
)
DataStack = DataStackClient
__all__ = [
"DataStack",
+12 -3
View File
@@ -1,10 +1,19 @@
from setuptools import setup, find_packages
from pathlib import Path
from setuptools import find_packages, setup
ROOT_DIR = Path(__file__).resolve().parent
README_PATH = ROOT_DIR / "README.md"
long_description = (
README_PATH.read_text(encoding="utf-8")
if README_PATH.exists()
else "Official Python client SDK for DataStack — Self-Hosted Real-Time Telemetry & Event Logging Platform"
)
setup(
name="datastack-sdk",
version="1.0.0",
version="2.2.0",
description="Official Python client SDK for DataStack — Self-Hosted Real-Time Telemetry & Event Logging Platform",
long_description=open("README.md", encoding="utf-8").read(),
long_description=long_description,
long_description_content_type="text/markdown",
author="Vib Tools",
author_email="hello@vibtools.io",
+375 -86
View File
@@ -7,12 +7,14 @@ Features:
- Automatic Heartbeat HTTP Fallback (`POST /api/events`) when WebSocket closes or disconnects.
- Automatic WebSocket reconnection, handshake verification (`ping_verify`), and return to IDLE mode.
- Offline Disk Spooling (`.datastack_spool/spool.jsonl`) when the client device loses internet,
with automatic replay when connectivity returns.
- Direct `.csv`, `.json`, and `.jsonl` file ingestion (`send_file()`) with automatic batch chunking
(`413 PAYLOAD_TOO_LARGE` auto-splitting) and exponential backoff retries (`429 RATE_LIMIT_EXCEEDED`).
with automatic paced replay when connectivity returns.
- Direct `.csv`, `.json`, and `.jsonl` file ingestion (`send_file()`) with deterministic SHA-256
`eventId` deduplication, automatic batch chunking (`413 PAYLOAD_TOO_LARGE` auto-splitting),
and lossless partial-batch rate-limit retry (`429 RATE_LIMIT_EXCEEDED`).
"""
import csv
import hashlib
import json
import logging
import random
@@ -38,6 +40,7 @@ except ImportError:
from .config import DataStackConfig, load_config
from .exceptions import (
DataStackAuthError,
DataStackConfigError,
DataStackConnectionError,
DataStackError,
DataStackFileError,
@@ -58,6 +61,17 @@ def _make_event_id() -> str:
return f"evt_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}"
def _make_deterministic_file_event_id(
file_stem: str, row_index: int, record: Any
) -> str:
raw = f"{file_stem}:{row_index}:{json.dumps(record, sort_keys=True, default=str, ensure_ascii=False)}"
digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:12]
safe_stem = "".join(
ch if ch.isalnum() or ch in ("_", "-") else "_" for ch in file_stem
)[:32]
return f"file_{safe_stem}_{row_index}_{digest}"
class DataStackClient:
"""
Production-grade DataStack Python Client with `config.datastack.py` auto-loading,
@@ -71,8 +85,20 @@ class DataStackClient:
api_url: Optional[str] = None,
config_path: Optional[str] = None,
auto_connect: Optional[bool] = None,
reconnection_attempts: Optional[int] = None,
reconnection_delay: Optional[float] = None,
**kwargs: Any,
) -> None:
if reconnection_attempts is not None and "MAX_RETRIES" not in kwargs:
kwargs["MAX_RETRIES"] = reconnection_attempts
if (
reconnection_delay is not None
and "HEARTBEAT_INTERVAL_SEC" not in kwargs
):
kwargs["HEARTBEAT_INTERVAL_SEC"] = reconnection_delay
# If legacy caller passes explicit server_url/token without auto_connect, default auto_connect to False
# ONLY if config.datastack.py was not used and auto_connect was not explicitly specified.
self.config: DataStackConfig = load_config(
config_path=config_path,
require_token=False,
@@ -88,6 +114,7 @@ class DataStackClient:
self.token = self.config.client_token
self.is_connected = False
self._lock = threading.RLock()
self._is_draining_spool = False
# Offline Disk Spooler
self.spooler = DiskSpooler(
@@ -115,8 +142,10 @@ class DataStackClient:
if self.sio is not None:
self._register_socket_events()
if self.config.auto_connect and self.token and (
self.token != "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN"
if (
self.config.auto_connect
and self.token
and self.token != "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN"
):
try:
self.connect()
@@ -128,6 +157,23 @@ class DataStackClient:
exc,
)
def __enter__(self) -> "DataStackClient":
if (
not self.is_connected
and self.token
and self.token != "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN"
):
try:
self.connect()
except DataStackAuthError:
raise
except Exception:
pass
return self
def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
self.disconnect()
def _register_socket_events(self) -> None:
if self.sio is None:
return
@@ -174,13 +220,11 @@ class DataStackClient:
def _ensure_token_present(self) -> None:
if not self.token or self.token == "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN":
from .exceptions import DataStackConfigError
raise DataStackConfigError(
"Missing CLIENT_TOKEN. Set CLIENT_TOKEN in `config.datastack.py` or pass token='...'."
)
def connect(self) -> bool:
def connect(self, wait_timeout: Optional[float] = None) -> bool:
"""
Establishes the primary WebSocket connection on Port 4000 and verifies the
handshake via `ping_verify`. If WebSocket is unavailable and `ENABLE_HTTP_FALLBACK`
@@ -191,7 +235,7 @@ class DataStackClient:
self.heartbeat.start()
try:
if self._connect_and_verify_websocket():
if self._connect_and_verify_websocket(wait_timeout=wait_timeout):
self._drain_offline_spool()
self.heartbeat.set_idle_connected()
return True
@@ -231,10 +275,18 @@ class DataStackClient:
except Exception:
pass
def _connect_and_verify_websocket(self) -> bool:
def _connect_and_verify_websocket(
self, wait_timeout: Optional[float] = None
) -> bool:
if self.sio is None:
return False
effective_timeout = (
float(wait_timeout)
if wait_timeout is not None
else self.config.request_timeout_sec
)
with self._lock:
if self.sio.connected:
try:
@@ -249,7 +301,7 @@ class DataStackClient:
auth={"token": self.token},
headers={"Authorization": f"Bearer {self.token}"},
transports=["websocket", "polling"],
wait_timeout=self.config.request_timeout_sec,
wait_timeout=effective_timeout,
)
self.is_connected = True
except Exception as exc:
@@ -260,9 +312,11 @@ class DataStackClient:
return False
# Verify handshake & PostgreSQL client status via `ping_verify`
return self._verify_websocket_handshake()
return self._verify_websocket_handshake(timeout=effective_timeout)
def _verify_websocket_handshake(self) -> bool:
def _verify_websocket_handshake(
self, timeout: Optional[float] = None
) -> bool:
if self.sio is None or not self.sio.connected:
return False
@@ -274,15 +328,18 @@ class DataStackClient:
result_holder.update(resp)
ack_event.set()
wait_sec = timeout or self.config.request_timeout_sec
try:
self.sio.emit("ping_verify", {}, callback=on_verify)
if not ack_event.wait(timeout=self.config.request_timeout_sec):
if not ack_event.wait(timeout=wait_sec):
return False
except Exception:
return False
if not result_holder.get("success"):
err_msg = str(result_holder.get("error") or "Handshake verification failed")
err_msg = str(
result_holder.get("error") or "Handshake verification failed"
)
if self._is_non_retryable_auth_error(err_msg):
raise DataStackAuthError(err_msg)
return False
@@ -297,6 +354,7 @@ class DataStackClient:
so `HeartbeatMonitor` transitions back to `CONNECTED_IDLE` (IDLE mode).
3. If WebSocket is still down, checks if HTTP fallback (`Port 3000`) is online:
- If HTTP is online, drains offline spooled events over HTTP and remains in `FALLBACK_HTTP`.
- If HTTP returns `401/403` auth revocation, stops Heartbeat immediately.
- If HTTP is also unreachable, sets state to `OFFLINE_SPOOL`.
"""
self.heartbeat.set_reconnecting()
@@ -311,11 +369,15 @@ class DataStackClient:
pass
# WebSocket still unavailable; check HTTP availability
if self.config.enable_http_fallback and self._check_http_health():
self.heartbeat.trigger_fallback(
reason="WebSocket still unavailable; HTTP fallback active"
)
self._drain_offline_spool()
try:
if self.config.enable_http_fallback and self._check_http_health():
self.heartbeat.trigger_fallback(
reason="WebSocket still unavailable; HTTP fallback active"
)
self._drain_offline_spool()
return False
except DataStackAuthError as auth_err:
self.heartbeat.stop(reason=str(auth_err))
return False
self.heartbeat.trigger_offline(
@@ -324,13 +386,31 @@ class DataStackClient:
return False
def _check_http_health(self) -> bool:
"""
Probes `GET {API_URL}/api/health` with the client's Bearer JWT token.
Raises `DataStackAuthError` and halts Heartbeat if client credentials were
revoked or rotated (`401` / `403`). Returns `True` ONLY on `2xx` with
`status == 'online'`.
"""
url = f"{self.api_url}/api/health"
try:
status_code, _ = self._http_request("GET", url, payload=None, timeout=3.0)
return 200 <= status_code < 500
status_code, body = self._http_request(
"GET", url, payload=None, timeout=3.0
)
except Exception:
return False
err_code = str(body.get("code") or "")
err_msg = str(body.get("error") or f"HTTP {status_code}")
if status_code in (401, 403) or self._is_non_retryable_auth_error(
f"{err_code} {err_msg}"
):
self.heartbeat.stop(reason=err_msg)
raise DataStackAuthError(err_msg, code=err_code or "AUTH_ERROR")
return 200 <= status_code < 300 and body.get("status") == "online"
def _http_request(
self,
method: str,
@@ -357,7 +437,9 @@ class DataStackClient:
body = resp.json() if resp.text else {}
except Exception:
body = {"error": resp.text}
return resp.status_code, body if isinstance(body, dict) else {"data": body}
return resp.status_code, (
body if isinstance(body, dict) else {"data": body}
)
# Standard library urllib fallback when `requests` is not installed
data_bytes = (
@@ -384,7 +466,9 @@ class DataStackClient:
parsed_err = {"error": raw_err or str(http_err)}
return (
http_err.code,
parsed_err if isinstance(parsed_err, dict) else {"error": str(parsed_err)},
parsed_err
if isinstance(parsed_err, dict)
else {"error": str(parsed_err)},
)
def _send_via_http_fallback(
@@ -404,22 +488,31 @@ class DataStackClient:
"transport": "HTTP_FALLBACK",
}
err_msg = str(body.get("error") or f"HTTP {status_code}")
errors_list = body.get("errors")
first_err = (
errors_list[0]
if isinstance(errors_list, list) and len(errors_list) > 0
else None
)
err_msg = str(body.get("error") or first_err or f"HTTP {status_code}")
err_code = str(body.get("code") or "")
combined = f"{err_code} {err_msg}"
if status_code in (401, 403) or self._is_non_retryable_auth_error(
f"{err_code} {err_msg}"
combined
):
self.heartbeat.stop(reason=err_msg)
raise DataStackAuthError(err_msg, code=err_code or "AUTH_ERROR")
if status_code == 413 or "PAYLOAD_TOO_LARGE" in f"{err_code} {err_msg}":
if status_code == 413 or "PAYLOAD_TOO_LARGE" in combined:
raise DataStackPayloadTooLargeError(err_msg)
if status_code == 429 or "RATE_LIMIT_EXCEEDED" in f"{err_code} {err_msg}":
if status_code == 429 or "RATE_LIMIT_EXCEEDED" in combined:
raise DataStackRateLimitError(err_msg)
raise DataStackConnectionError(f"HTTP fallback error ({status_code}): {err_msg}")
raise DataStackConnectionError(
f"HTTP fallback error ({status_code}): {err_msg}"
)
def _emit_via_websocket(
self,
@@ -428,7 +521,9 @@ class DataStackClient:
timeout: Optional[float] = None,
) -> Dict[str, Any]:
if self.sio is None or not self.is_connected or not self.sio.connected:
raise DataStackConnectionError("WebSocket is not currently connected")
raise DataStackConnectionError(
"WebSocket is not currently connected"
)
ack_event = threading.Event()
response_data: Dict[str, Any] = {}
@@ -442,8 +537,12 @@ class DataStackClient:
self.sio.emit(event_name, payload, callback=ack_callback)
except Exception as exc:
self.is_connected = False
self.heartbeat.trigger_fallback(reason=f"WebSocket emit error: {exc}")
raise DataStackConnectionError(f"WebSocket emit failed: {exc}") from exc
self.heartbeat.trigger_fallback(
reason=f"WebSocket emit error: {exc}"
)
raise DataStackConnectionError(
f"WebSocket emit failed: {exc}"
) from exc
wait_sec = timeout or self.config.request_timeout_sec
if not ack_event.wait(timeout=wait_sec):
@@ -456,15 +555,28 @@ class DataStackClient:
)
if not response_data.get("success", False):
err_msg = str(response_data.get("error") or "WebSocket event rejected")
if self._is_non_retryable_auth_error(err_msg):
errors_list = response_data.get("errors")
first_err = (
errors_list[0]
if isinstance(errors_list, list) and len(errors_list) > 0
else None
)
err_msg = str(
response_data.get("error")
or first_err
or "WebSocket event rejected"
)
err_code = str(response_data.get("code") or "")
combined = f"{err_code} {err_msg}"
if self._is_non_retryable_auth_error(combined):
self.heartbeat.stop(reason=err_msg)
raise DataStackAuthError(err_msg)
if "PAYLOAD_TOO_LARGE" in err_msg:
raise DataStackAuthError(err_msg, code=err_code or "AUTH_ERROR")
if "PAYLOAD_TOO_LARGE" in combined:
raise DataStackPayloadTooLargeError(err_msg)
if "RATE_LIMIT_EXCEEDED" in err_msg:
if "RATE_LIMIT_EXCEEDED" in combined:
raise DataStackRateLimitError(err_msg)
raise DataStackError(err_msg)
raise DataStackError(err_msg, code=err_code or "WS_REJECTED")
return {
**response_data,
@@ -478,6 +590,7 @@ class DataStackClient:
http_payload: Dict[str, Any],
spool_items: List[Dict[str, Any]],
timeout: Optional[float] = None,
allow_spool: bool = True,
) -> Dict[str, Any]:
"""
Core resilient dispatcher:
@@ -485,8 +598,8 @@ class DataStackClient:
2. On WebSocket failure/disconnect, triggers `HeartbeatMonitor` (`FALLBACK_HTTP`)
and automatically sends via HTTP `POST /api/events`.
3. Applies exponential backoff with jitter on `RATE_LIMIT_EXCEEDED` or transient errors.
4. If both WebSocket and HTTP are unreachable (device offline), spools events to
`.datastack_spool/spool.jsonl` and triggers `OFFLINE_SPOOL`.
4. If both WebSocket and HTTP are unreachable (device offline) and `allow_spool` is True,
spools events to `.datastack_spool/spool.jsonl` and triggers `OFFLINE_SPOOL`.
"""
self._ensure_token_present()
max_attempts = max(1, self.config.max_retries + 1)
@@ -494,16 +607,29 @@ class DataStackClient:
for attempt in range(max_attempts):
if attempt > 0:
delay = self.config.retry_backoff_base_sec * (2 ** (attempt - 1))
jitter = random.uniform(0.0, 0.25 * delay)
# Use at least 1.0s delay when backing off from RATE_LIMIT_EXCEEDED so the 1s sliding window resets
base_delay = (
max(1.0, self.config.retry_backoff_base_sec)
if isinstance(last_exception, DataStackRateLimitError)
else self.config.retry_backoff_base_sec
)
delay = base_delay * (2 ** (attempt - 1))
jitter = random.uniform(0.0, 0.2 * delay)
time.sleep(delay + jitter)
# 1. Try Primary WebSocket if connected
if self.is_connected and self.sio is not None and self.sio.connected:
try:
return self._emit_via_websocket(
res = self._emit_via_websocket(
ws_event, ws_payload, timeout=timeout
)
if (
allow_spool
and not self._is_draining_spool
and self.spooler.pending_count() > 0
):
self._drain_offline_spool()
return res
except (DataStackAuthError, DataStackPayloadTooLargeError):
raise
except DataStackRateLimitError as rl_err:
@@ -516,11 +642,21 @@ class DataStackClient:
# 2. Try HTTP Fallback (`POST /api/events`)
if self.config.enable_http_fallback:
try:
if self.heartbeat.state == ConnectionState.CONNECTED_IDLE:
res = self._send_via_http_fallback(http_payload)
if self.heartbeat.state in (
ConnectionState.CONNECTED_IDLE,
ConnectionState.OFFLINE_SPOOL,
):
self.heartbeat.trigger_fallback(
reason="WebSocket not connected; routing via HTTP fallback"
)
return self._send_via_http_fallback(http_payload)
if (
allow_spool
and not self._is_draining_spool
and self.spooler.pending_count() > 0
):
self._drain_offline_spool()
return res
except (DataStackAuthError, DataStackPayloadTooLargeError):
raise
except DataStackRateLimitError as rl_err:
@@ -531,12 +667,17 @@ class DataStackClient:
# 3. If rate-limited or payload-rejected after all retries, raise directly
if isinstance(
last_exception, (DataStackRateLimitError, DataStackPayloadTooLargeError)
last_exception,
(DataStackRateLimitError, DataStackPayloadTooLargeError),
):
raise last_exception
# 4. Device is offline (both WebSocket and HTTP failed): Spool to local disk
if self.config.offline_spool_enabled and spool_items:
if (
allow_spool
and self.config.offline_spool_enabled
and spool_items
):
spooled_count = self.spooler.enqueue(spool_items)
self.heartbeat.trigger_offline(
reason=str(last_exception or "Network unreachable")
@@ -557,29 +698,42 @@ class DataStackClient:
"""
Replays any offline spooled events from `.datastack_spool/spool.jsonl`
once connectivity (WebSocket or HTTP fallback) is restored.
Routes batches through `_send_formatted_batch_recursive(..., allow_spool=False)`
so oversized or rate-limited spool batches automatically split and pace.
"""
if not self.config.offline_spool_enabled:
return 0
total_flushed = 0
while True:
batch = self.spooler.peek_batch(limit=self.config.batch_chunk_size)
if not batch:
break
with self._lock:
if self._is_draining_spool:
return 0
self._is_draining_spool = True
try:
if self.is_connected and self.sio is not None and self.sio.connected:
self._emit_via_websocket("batch_sync", batch)
elif self.config.enable_http_fallback:
self._send_via_http_fallback({"batch": batch})
else:
total_flushed = 0
try:
while True:
batch = self.spooler.peek_batch(
limit=self.config.batch_chunk_size
)
if not batch:
break
res = self._send_formatted_batch_recursive(
batch,
timeout=self.config.request_timeout_sec,
file_name=None,
allow_spool=False,
)
if not res.get("success", False):
break
self.spooler.ack_batch(len(batch))
total_flushed += len(batch)
except Exception as exc:
logger.debug("[DataStack] Spool drain paused: %s", exc)
break
except Exception as exc:
logger.debug("[DataStack] Spool drain paused: %s", exc)
finally:
with self._lock:
self._is_draining_spool = False
return total_flushed
@@ -611,6 +765,7 @@ class DataStackClient:
http_payload=envelope,
spool_items=[envelope],
timeout=timeout,
allow_spool=True,
)
def sync(
@@ -624,22 +779,33 @@ class DataStackClient:
def send_batch(
self,
events: List[Dict[str, Any]],
events: Optional[List[Dict[str, Any]]] = None,
timeout: Optional[float] = None,
file_name: Optional[str] = None,
*,
records: Optional[List[Dict[str, Any]]] = None,
) -> Dict[str, Any]:
"""
Sends a batch of events. If the batch triggers `413 PAYLOAD_TOO_LARGE`,
automatically splits the batch into smaller halves and retries.
Sends a batch of events (accepts either `events=[...]` or `records=[...]`).
- If the batch triggers `413 PAYLOAD_TOO_LARGE`, automatically splits the batch
in half recursively and retries.
- If a subset of events in the batch hits `429 RATE_LIMIT_EXCEEDED`
(`rateLimitedEventIds`), automatically backs off and retries the remaining
rate-limited events so zero records are dropped.
"""
if not isinstance(events, list) or len(events) == 0:
source_list = events if events is not None else records
if not isinstance(source_list, list) or len(source_list) == 0:
return {"success": True, "processed": 0, "rejected": 0}
formatted: List[Dict[str, Any]] = []
for item in events:
for item in source_list:
safe_item = item if isinstance(item, dict) else {"value": item}
evt_id = str(safe_item.get("eventId") or "").strip() or _make_event_id()
evt_ts = str(safe_item.get("timestamp") or "").strip() or _utc_now_iso()
evt_id = (
str(safe_item.get("eventId") or "").strip() or _make_event_id()
)
evt_ts = (
str(safe_item.get("timestamp") or "").strip() or _utc_now_iso()
)
evt_data = safe_item.get("data", safe_item)
formatted.append(
{
@@ -650,22 +816,28 @@ class DataStackClient:
)
return self._send_formatted_batch_recursive(
formatted, timeout=timeout, file_name=file_name
formatted,
timeout=timeout,
file_name=file_name,
allow_spool=True,
)
def batch_sync(
self,
events: List[Dict[str, Any]],
events: Optional[List[Dict[str, Any]]] = None,
timeout: float = 10.0,
*,
records: Optional[List[Dict[str, Any]]] = None,
) -> Dict[str, Any]:
"""Alias for `send_batch()` preserving backward compatibility."""
return self.send_batch(events=events, timeout=timeout)
"""Alias for `send_batch()` supporting both `events` and `records` kwargs."""
return self.send_batch(events=events, records=records, timeout=timeout)
def _send_formatted_batch_recursive(
self,
formatted_batch: List[Dict[str, Any]],
timeout: Optional[float] = None,
file_name: Optional[str] = None,
allow_spool: bool = True,
) -> Dict[str, Any]:
http_payload: Dict[str, Any] = (
{"fileName": file_name, "records": formatted_batch}
@@ -673,36 +845,118 @@ class DataStackClient:
else {"batch": formatted_batch}
)
try:
return self._execute_with_retry_and_fallback(
res = self._execute_with_retry_and_fallback(
ws_event="batch_sync",
ws_payload=formatted_batch,
http_payload=http_payload,
spool_items=formatted_batch,
timeout=timeout,
allow_spool=allow_spool,
)
except DataStackPayloadTooLargeError:
if len(formatted_batch) <= 1:
raise
mid = len(formatted_batch) // 2
left_res = self._send_formatted_batch_recursive(
formatted_batch[:mid], timeout=timeout, file_name=file_name
formatted_batch[:mid],
timeout=timeout,
file_name=file_name,
allow_spool=allow_spool,
)
right_res = self._send_formatted_batch_recursive(
formatted_batch[mid:], timeout=timeout, file_name=file_name
formatted_batch[mid:],
timeout=timeout,
file_name=file_name,
allow_spool=allow_spool,
)
return {
"success": bool(
left_res.get("success", True) and right_res.get("success", True)
left_res.get("success", True)
and right_res.get("success", True)
),
"processed": int(left_res.get("processed", 0))
+ int(right_res.get("processed", 0)),
"rejected": int(left_res.get("rejected", 0))
+ int(right_res.get("rejected", 0)),
"duplicates": int(left_res.get("duplicates", 0))
+ int(right_res.get("duplicates", 0)),
"spooledCount": int(left_res.get("spooledCount", 0))
+ int(right_res.get("spooledCount", 0)),
"transport": right_res.get("transport") or left_res.get("transport"),
"transport": right_res.get("transport")
or left_res.get("transport"),
}
# Handle partial-batch rate limiting (`rateLimitedEventIds`) so zero rows are dropped
rate_limited_ids = res.get("rateLimitedEventIds")
if (
isinstance(rate_limited_ids, list)
and len(rate_limited_ids) > 0
and self.config.max_retries > 0
):
pending_id_set = {str(eid) for eid in rate_limited_ids}
remaining_items = [
item
for item in formatted_batch
if str(item.get("eventId")) in pending_id_set
]
total_processed = int(res.get("processed", 0))
total_duplicates = int(res.get("duplicates", 0))
total_spooled = int(res.get("spooledCount", 0))
last_transport = res.get("transport")
for retry_round in range(1, self.config.max_retries + 2):
if not remaining_items:
break
# Wait slightly over 1.0s so the server's 1-second rate-limit window resets
wait_delay = max(1.05, self.config.retry_backoff_base_sec) + random.uniform(
0.0, 0.15
)
time.sleep(wait_delay)
sub_http_payload: Dict[str, Any] = (
{"fileName": file_name, "records": remaining_items}
if file_name
else {"batch": remaining_items}
)
sub_res = self._execute_with_retry_and_fallback(
ws_event="batch_sync",
ws_payload=remaining_items,
http_payload=sub_http_payload,
spool_items=remaining_items,
timeout=timeout,
allow_spool=allow_spool,
)
total_processed += int(sub_res.get("processed", 0))
total_duplicates += int(sub_res.get("duplicates", 0))
total_spooled += int(sub_res.get("spooledCount", 0))
last_transport = sub_res.get("transport") or last_transport
next_rl_ids = sub_res.get("rateLimitedEventIds")
if not isinstance(next_rl_ids, list) or len(next_rl_ids) == 0:
remaining_items = []
break
next_set = {str(eid) for eid in next_rl_ids}
remaining_items = [
item
for item in remaining_items
if str(item.get("eventId")) in next_set
]
oversized_ids = res.get("oversizedEventIds")
final_rejected = len(remaining_items) + (
len(oversized_ids) if isinstance(oversized_ids, list) else 0
)
return {
"success": final_rejected == 0 or total_processed > 0,
"processed": total_processed,
"rejected": final_rejected,
"duplicates": total_duplicates,
"spooledCount": total_spooled,
"transport": last_transport,
}
return res
# =========================================================================
# Direct File Streaming (.csv, .json, .jsonl / .ndjson)
# =========================================================================
@@ -715,8 +969,8 @@ class DataStackClient:
) -> Dict[str, Any]:
"""
Streams a `.csv`, `.json`, or `.jsonl` (`.ndjson`) file directly to DataStack
in safe chunks with deterministic idempotency IDs, automatic HTTP fallback,
and offline spooling support.
in safe chunks with deterministic SHA-256 idempotency IDs, automatic HTTP fallback,
partial-batch rate-limit pacing, and offline spooling support.
"""
path = Path(file_path).expanduser().resolve()
if not path.exists() or not path.is_file():
@@ -728,9 +982,12 @@ class DataStackClient:
f"Unsupported file extension '{ext}'. Supported formats: .csv, .json, .jsonl, .ndjson"
)
effective_chunk = max(1, int(chunk_size or self.config.batch_chunk_size))
effective_chunk = max(
1, int(chunk_size or self.config.batch_chunk_size)
)
total_processed = 0
total_rejected = 0
total_duplicates = 0
total_spooled = 0
chunks_sent = 0
last_transport = "WEBSOCKET"
@@ -742,10 +999,25 @@ class DataStackClient:
if extra_metadata and isinstance(record, dict)
else record
)
existing_id = (
str(enriched.get("eventId") or "").strip()
if isinstance(enriched, dict)
else ""
)
evt_id = existing_id or _make_deterministic_file_event_id(
path.stem, idx, enriched
)
existing_ts = (
str(enriched.get("timestamp") or "").strip()
if isinstance(enriched, dict)
else ""
)
evt_ts = existing_ts or _utc_now_iso()
buffer.append(
{
"eventId": f"file_{path.stem}_{idx}_{uuid.uuid4().hex[:6]}",
"timestamp": _utc_now_iso(),
"eventId": evt_id,
"timestamp": evt_ts,
"data": enriched,
}
)
@@ -753,6 +1025,7 @@ class DataStackClient:
res = self.send_batch(buffer, file_name=path.name)
total_processed += int(res.get("processed", 0))
total_rejected += int(res.get("rejected", 0))
total_duplicates += int(res.get("duplicates", 0))
total_spooled += int(res.get("spooledCount", 0))
last_transport = str(res.get("transport", last_transport))
chunks_sent += 1
@@ -762,6 +1035,7 @@ class DataStackClient:
res = self.send_batch(buffer, file_name=path.name)
total_processed += int(res.get("processed", 0))
total_rejected += int(res.get("rejected", 0))
total_duplicates += int(res.get("duplicates", 0))
total_spooled += int(res.get("spooledCount", 0))
last_transport = str(res.get("transport", last_transport))
chunks_sent += 1
@@ -773,6 +1047,7 @@ class DataStackClient:
"chunksSent": chunks_sent,
"processed": total_processed,
"rejected": total_rejected,
"duplicates": total_duplicates,
"spooledCount": total_spooled,
"transport": last_transport,
}
@@ -797,20 +1072,34 @@ class DataStackClient:
raise DataStackFileError(
f"Malformed JSONL at line {line_no} in {path.name}: {exc}"
) from exc
yield parsed if isinstance(parsed, dict) else {"value": parsed}
yield (
parsed
if isinstance(parsed, dict)
else {"value": parsed}
)
elif ext == ".json":
with path.open("r", encoding="utf-8-sig") as fh:
parsed = json.load(fh)
if isinstance(parsed, list):
for item in parsed:
yield item if isinstance(item, dict) else {"value": item}
yield (
item if isinstance(item, dict) else {"value": item}
)
elif isinstance(parsed, dict):
if isinstance(parsed.get("records"), list):
for item in parsed["records"]:
yield item if isinstance(item, dict) else {"value": item}
yield (
item
if isinstance(item, dict)
else {"value": item}
)
elif isinstance(parsed.get("data"), list):
for item in parsed["data"]:
yield item if isinstance(item, dict) else {"value": item}
yield (
item
if isinstance(item, dict)
else {"value": item}
)
else:
yield parsed
else:
+2 -2
View File
@@ -39,7 +39,7 @@ class HeartbeatMonitor:
interval_sec: float,
recovery_callback: Callable[[], bool],
) -> None:
self.interval_sec = max(1.0, float(interval_sec))
self.interval_sec = max(0.2, float(interval_sec))
self._recovery_callback = recovery_callback
self._state = ConnectionState.CONNECTED_IDLE
self._closed = False
@@ -169,7 +169,7 @@ class HeartbeatMonitor:
continue
with self._lock:
if self._state == ConnectionState.STOPPED:
if self._closed or self._state == ConnectionState.STOPPED:
break
self._recovery_attempts += 1
+7 -1
View File
@@ -203,7 +203,8 @@ async function syncSettingsFromDatabase() {
await ensureDatabaseSchema();
const res = await dbPool.query(
`SELECT max_events_per_sec, max_payload_size_kb, batch_flush_threshold,
batch_flush_interval_ms, file_rotation_mb, jwt_secret, enable_idempotency
batch_flush_interval_ms, file_rotation_mb, jwt_secret, enable_idempotency,
auto_prune_days
FROM admin_settings
WHERE id = 'default'`,
);
@@ -227,6 +228,11 @@ async function syncSettingsFromDatabase() {
writer.setMaxFileSizeMb(nextRotationMb);
}
const nextPruneDays = Number(row.auto_prune_days) || 30;
if (nextPruneDays > 0) {
writer.pruneExpiredPartitions(nextPruneDays);
}
const nextIntervalMs = Math.max(
100,
Number(row.batch_flush_interval_ms) || 2000,
+56 -6
View File
@@ -73,6 +73,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
return {
allowed: false,
duplicate: true,
code: "DUPLICATE_EVENT",
reason: "Duplicate event dropped",
};
}
@@ -91,6 +92,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
return {
allowed: false,
duplicate: false,
code: "RATE_LIMIT_EXCEEDED",
reason: `RATE_LIMIT_EXCEEDED: Limit of ${effectiveLimit} events/sec exceeded`,
};
} else {
@@ -106,6 +108,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
return {
allowed: false,
duplicate: false,
code: "PAYLOAD_TOO_LARGE",
reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`,
};
}
@@ -166,6 +169,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
if (callback) {
callback({
success: false,
code: "CLIENT_REVOKED",
error: dbCheck.reason || "CLIENT_REVOKED",
});
}
@@ -195,7 +199,12 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
return;
}
if (callback) {
callback({ success: false, eventId, error: check.reason });
callback({
success: false,
eventId,
code: check.code,
error: check.reason,
});
}
return;
}
@@ -236,6 +245,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
success: false,
processed: 0,
rejected: batch.length,
code: "CLIENT_REVOKED",
error: dbCheck.reason || "CLIENT_REVOKED",
});
}
@@ -247,6 +257,11 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
dbCheck.rateLimit || socket.data.rateLimit || 100;
let processed = 0;
let rejected = 0;
let duplicates = 0;
let hitRateLimit = false;
let hitPayloadLimit = false;
const rateLimitedEventIds = [];
const oversizedEventIds = [];
const errors = [];
for (const item of batch) {
@@ -265,11 +280,21 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
effectiveClientRate,
);
if (!check.allowed) {
if (!check.duplicate) {
rejected += 1;
if (errors.length < 5) {
errors.push(`${eventId}: ${check.reason}`);
}
if (check.duplicate) {
duplicates += 1;
continue;
}
rejected += 1;
if (check.code === "RATE_LIMIT_EXCEEDED") {
hitRateLimit = true;
rateLimitedEventIds.push(eventId);
}
if (check.code === "PAYLOAD_TOO_LARGE") {
hitPayloadLimit = true;
oversizedEventIds.push(eventId);
}
if (errors.length < 5) {
errors.push(`${eventId}: ${check.reason}`);
}
continue;
}
@@ -288,11 +313,36 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
onBufferPush();
}
if (processed === 0 && duplicates === 0 && rejected > 0) {
const code = hitPayloadLimit
? "PAYLOAD_TOO_LARGE"
: hitRateLimit
? "RATE_LIMIT_EXCEEDED"
: "BATCH_REJECTED";
if (callback) {
callback({
success: false,
processed,
rejected,
duplicates,
code,
error: errors[0] || "Batch validation failed",
errors,
...(rateLimitedEventIds.length > 0 ? { rateLimitedEventIds } : {}),
...(oversizedEventIds.length > 0 ? { oversizedEventIds } : {}),
});
}
return;
}
if (callback) {
callback({
success: true,
processed,
rejected,
duplicates,
...(rateLimitedEventIds.length > 0 ? { rateLimitedEventIds } : {}),
...(oversizedEventIds.length > 0 ? { oversizedEventIds } : {}),
...(errors.length > 0 ? { errors } : {}),
});
}
+37
View File
@@ -97,6 +97,43 @@ class CsvWriterWorker {
return { written: events.length, files };
}
pruneExpiredPartitions(autoPruneDays) {
const days = Number(autoPruneDays);
if (!Number.isFinite(days) || days <= 0 || !fs.existsSync(this.baseDir)) {
return 0;
}
const cutoffDate = new Date(Date.now() - Math.floor(days) * 86400000)
.toISOString()
.split('T')[0];
let prunedCount = 0;
try {
const clientDirs = fs.readdirSync(this.baseDir);
for (const clientDir of clientDirs) {
const fullClientPath = path.join(this.baseDir, clientDir);
if (!fs.statSync(fullClientPath).isDirectory()) continue;
const files = fs.readdirSync(fullClientPath);
for (const file of files) {
if (!file.endsWith('.csv')) continue;
const datePart = file.replace('.csv', '').split('-part')[0];
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
try {
fs.unlinkSync(path.join(fullClientPath, file));
prunedCount += 1;
} catch {
// Ignore individual file unlink failure
}
}
}
}
} catch {
// Ignore directory scan failure
}
return prunedCount;
}
}
module.exports = { CsvWriterWorker };