feat: improve event tracking and SDK robustness
- Add specific error tracking for rate-limited and oversized events in API response - Ensure runtime settings are applied on initial access in settings manager - Exclude /api/health from middleware protection - Update Python SDK to v2.2.0 with unified standalone implementation - Add auto_prune_days setting to database schema retrieval
This commit is contained in:
1 parent
0c2e49cd47
commit
043123dc21
12 files changed
+1018
-128
No files matched your search
+14
-2
@@ -179,6 +179,8 @@ export async function POST(req: NextRequest) {
|
|||||||
let duplicates = 0;
|
let duplicates = 0;
|
||||||
let hitRateLimit = false;
|
let hitRateLimit = false;
|
||||||
let hitPayloadLimit = false;
|
let hitPayloadLimit = false;
|
||||||
|
const rateLimitedEventIds: string[] = [];
|
||||||
|
const oversizedEventIds: string[] = [];
|
||||||
const errors: string[] = [];
|
const errors: string[] = [];
|
||||||
|
|
||||||
for (const item of batchItems) {
|
for (const item of batchItems) {
|
||||||
@@ -213,8 +215,14 @@ export async function POST(req: NextRequest) {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
rejected += 1;
|
rejected += 1;
|
||||||
if (val.code === 'RATE_LIMIT_EXCEEDED') hitRateLimit = true;
|
if (val.code === 'RATE_LIMIT_EXCEEDED') {
|
||||||
if (val.code === 'PAYLOAD_TOO_LARGE') hitPayloadLimit = true;
|
hitRateLimit = true;
|
||||||
|
rateLimitedEventIds.push(itemEventId);
|
||||||
|
}
|
||||||
|
if (val.code === 'PAYLOAD_TOO_LARGE') {
|
||||||
|
hitPayloadLimit = true;
|
||||||
|
oversizedEventIds.push(itemEventId);
|
||||||
|
}
|
||||||
if (errors.length < 5) {
|
if (errors.length < 5) {
|
||||||
errors.push(`${itemEventId}: ${val.code || 'REJECTED'} - ${val.reason}`);
|
errors.push(`${itemEventId}: ${val.code || 'REJECTED'} - ${val.reason}`);
|
||||||
}
|
}
|
||||||
@@ -254,6 +262,8 @@ export async function POST(req: NextRequest) {
|
|||||||
code,
|
code,
|
||||||
error: errors[0] || 'Batch validation failed',
|
error: errors[0] || 'Batch validation failed',
|
||||||
errors,
|
errors,
|
||||||
|
...(rateLimitedEventIds.length > 0 ? { rateLimitedEventIds } : {}),
|
||||||
|
...(oversizedEventIds.length > 0 ? { oversizedEventIds } : {}),
|
||||||
},
|
},
|
||||||
{ status },
|
{ status },
|
||||||
);
|
);
|
||||||
@@ -267,6 +277,8 @@ export async function POST(req: NextRequest) {
|
|||||||
...(typeof bodyObj.fileName === 'string'
|
...(typeof bodyObj.fileName === 'string'
|
||||||
? { fileName: bodyObj.fileName }
|
? { fileName: bodyObj.fileName }
|
||||||
: {}),
|
: {}),
|
||||||
|
...(rateLimitedEventIds.length > 0 ? { rateLimitedEventIds } : {}),
|
||||||
|
...(oversizedEventIds.length > 0 ? { oversizedEventIds } : {}),
|
||||||
...(errors.length > 0 ? { errors } : {}),
|
...(errors.length > 0 ? { errors } : {}),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,70 @@
|
|||||||
|
import { SettingsManager } from '@/core/adminSettings/settingsManager';
|
||||||
|
import { AuthService } from '@/core/auth/jwt';
|
||||||
|
import { DataBufferEngine } from '@/core/data/buffer';
|
||||||
|
import { SDKManager } from '@/core/sdk/sdkManager';
|
||||||
|
import { NextRequest, NextResponse } from 'next/server';
|
||||||
|
|
||||||
|
export async function GET(req: NextRequest) {
|
||||||
|
try {
|
||||||
|
const settings = await SettingsManager.getSettingsAsync();
|
||||||
|
const authHeader = req.headers.get('authorization');
|
||||||
|
|
||||||
|
if (authHeader && authHeader.toLowerCase().startsWith('bearer ')) {
|
||||||
|
const rawToken = authHeader.slice(7).trim();
|
||||||
|
const verified = AuthService.verifyClientToken(rawToken);
|
||||||
|
if (!verified.valid || !verified.payload) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
status: 'error',
|
||||||
|
code: 'AUTH_INVALID',
|
||||||
|
error: verified.error || 'Invalid client JWT token.',
|
||||||
|
},
|
||||||
|
{ status: 401 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const clientId = verified.payload.clientId;
|
||||||
|
const registeredClient = await SDKManager.getClientAsync(clientId);
|
||||||
|
if (registeredClient) {
|
||||||
|
if (registeredClient.status === 'revoked') {
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
status: 'error',
|
||||||
|
code: 'CLIENT_REVOKED',
|
||||||
|
error: `CLIENT_REVOKED: Client '${clientId}' access has been revoked by administrator.`,
|
||||||
|
},
|
||||||
|
{ status: 403 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (registeredClient.token && registeredClient.token !== rawToken) {
|
||||||
|
return NextResponse.json(
|
||||||
|
{
|
||||||
|
status: 'error',
|
||||||
|
code: 'TOKEN_ROTATED',
|
||||||
|
error: 'TOKEN_ROTATED: Client credentials have been rotated. Use the latest token.',
|
||||||
|
},
|
||||||
|
{ status: 401 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const bufferStats = DataBufferEngine.getBufferStats();
|
||||||
|
|
||||||
|
return NextResponse.json({
|
||||||
|
status: 'online',
|
||||||
|
service: 'DataStack Web API',
|
||||||
|
maxEventsPerSec: settings.maxEventsPerSec,
|
||||||
|
maxPayloadSizeKb: settings.maxPayloadSizeKb,
|
||||||
|
maxPayloadSizeBytes: settings.maxPayloadSizeKb * 1024,
|
||||||
|
bufferSize: bufferStats.currentBufferSize,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
});
|
||||||
|
} catch (err: unknown) {
|
||||||
|
const message = err instanceof Error ? err.message : 'Health check failed';
|
||||||
|
return NextResponse.json(
|
||||||
|
{ status: 'error', error: message },
|
||||||
|
{ status: 500 },
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -165,7 +165,9 @@ export class SettingsManager {
|
|||||||
if (!globals.__dataStackSettingsHydrated) {
|
if (!globals.__dataStackSettingsHydrated) {
|
||||||
void this.syncFromDatabase();
|
void this.syncFromDatabase();
|
||||||
}
|
}
|
||||||
return { ...getSettingsState() };
|
const current = getSettingsState();
|
||||||
|
this.applyRuntimeSettings(current);
|
||||||
|
return { ...current };
|
||||||
}
|
}
|
||||||
|
|
||||||
public static async getSettingsAsync(): Promise<AdminSettings> {
|
public static async getSettingsAsync(): Promise<AdminSettings> {
|
||||||
@@ -173,7 +175,9 @@ export class SettingsManager {
|
|||||||
if (fromDb) {
|
if (fromDb) {
|
||||||
return { ...fromDb };
|
return { ...fromDb };
|
||||||
}
|
}
|
||||||
return { ...getSettingsState() };
|
const current = getSettingsState();
|
||||||
|
this.applyRuntimeSettings(current);
|
||||||
|
return { ...current };
|
||||||
}
|
}
|
||||||
|
|
||||||
public static getPublicSettings(): Omit<AdminSettings, "jwtSecret"> {
|
public static getPublicSettings(): Omit<AdminSettings, "jwtSecret"> {
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ function isProtectedApi(pathname: string, method: string) {
|
|||||||
if (pathname === "/api/auth/setup/status" && method === "GET") return false;
|
if (pathname === "/api/auth/setup/status" && method === "GET") return false;
|
||||||
if (pathname === "/api/auth/setup" && method === "POST") return false;
|
if (pathname === "/api/auth/setup" && method === "POST") return false;
|
||||||
if (pathname === "/api/events" && method === "POST") return false;
|
if (pathname === "/api/events" && method === "POST") return false;
|
||||||
|
if (pathname === "/api/health" && method === "GET") return false;
|
||||||
return pathname.startsWith("/api/");
|
return pathname.startsWith("/api/");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,10 +1,43 @@
|
|||||||
"""
|
"""
|
||||||
DataStack Python SDK
|
DataStack Python SDK (`datastack_sdk`)
|
||||||
~~~~~~~~~~~~~~~~~~~
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||||
High-performance real-time data collection client for python applications.
|
Official Python client library for DataStack — Self-Hosted Real-Time Telemetry & Event Logging Platform.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from .client import DataStackClient
|
from .client import (
|
||||||
|
ConnectionState,
|
||||||
|
DataStack,
|
||||||
|
DataStackAuthError,
|
||||||
|
DataStackClient,
|
||||||
|
DataStackConfig,
|
||||||
|
DataStackConfigError,
|
||||||
|
DataStackConnectionError,
|
||||||
|
DataStackError,
|
||||||
|
DataStackFileError,
|
||||||
|
DataStackOfflineSpoolError,
|
||||||
|
DataStackPayloadTooLargeError,
|
||||||
|
DataStackRateLimitError,
|
||||||
|
DiskSpooler,
|
||||||
|
HeartbeatMonitor,
|
||||||
|
load_config,
|
||||||
|
)
|
||||||
|
|
||||||
__version__ = "1.0.0"
|
__version__ = "2.2.0"
|
||||||
__all__ = ["DataStackClient"]
|
|
||||||
|
__all__ = [
|
||||||
|
"DataStack",
|
||||||
|
"DataStackClient",
|
||||||
|
"DataStackConfig",
|
||||||
|
"load_config",
|
||||||
|
"ConnectionState",
|
||||||
|
"HeartbeatMonitor",
|
||||||
|
"DiskSpooler",
|
||||||
|
"DataStackError",
|
||||||
|
"DataStackConfigError",
|
||||||
|
"DataStackAuthError",
|
||||||
|
"DataStackConnectionError",
|
||||||
|
"DataStackRateLimitError",
|
||||||
|
"DataStackPayloadTooLargeError",
|
||||||
|
"DataStackFileError",
|
||||||
|
"DataStackOfflineSpoolError",
|
||||||
|
]
|
||||||
@@ -1,34 +1,413 @@
|
|||||||
"""
|
"""
|
||||||
Backward-Compatible `datastack_sdk.client` Wrapper.
|
Self-Contained & Unified `datastack_sdk.client` Module.
|
||||||
Delegates to the unified production SDK implementation in `sdk/python/datastack/datastack`
|
When running inside the repository or when `datastack` is installed in site-packages,
|
||||||
so existing applications importing `from datastack_sdk import DataStackClient` gain
|
re-exports the canonical classes from `datastack`. If `sdk-python/datastack_sdk` is
|
||||||
automatic `config.datastack.py` discovery, Heartbeat HTTP fallback, offline spooling,
|
copied or installed standalone without `datastack`, provides the complete embedded
|
||||||
and `send_file()` support.
|
implementation so `from datastack_sdk import DataStackClient, DataStack` never fails
|
||||||
|
with `ModuleNotFoundError`.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
# Resolve `sdk/python/datastack` from the repository root if not installed in site-packages
|
|
||||||
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||||
_NEW_SDK_PATH = _REPO_ROOT / "sdk" / "python" / "datastack"
|
_NEW_SDK_PATH = _REPO_ROOT / "sdk" / "python" / "datastack"
|
||||||
if _NEW_SDK_PATH.exists() and str(_NEW_SDK_PATH) not in sys.path:
|
if _NEW_SDK_PATH.exists() and str(_NEW_SDK_PATH) not in sys.path:
|
||||||
sys.path.insert(0, str(_NEW_SDK_PATH))
|
sys.path.insert(0, str(_NEW_SDK_PATH))
|
||||||
|
|
||||||
from datastack.client import DataStack, DataStackClient # noqa: E402
|
try:
|
||||||
from datastack.config import DataStackConfig, load_config # noqa: E402
|
from datastack.client import DataStack, DataStackClient # type: ignore
|
||||||
from datastack.exceptions import ( # noqa: E402
|
from datastack.config import DataStackConfig, load_config # type: ignore
|
||||||
DataStackAuthError,
|
from datastack.exceptions import ( # type: ignore
|
||||||
DataStackConfigError,
|
DataStackAuthError,
|
||||||
DataStackConnectionError,
|
DataStackConfigError,
|
||||||
DataStackError,
|
DataStackConnectionError,
|
||||||
DataStackFileError,
|
DataStackError,
|
||||||
DataStackOfflineSpoolError,
|
DataStackFileError,
|
||||||
DataStackPayloadTooLargeError,
|
DataStackOfflineSpoolError,
|
||||||
DataStackRateLimitError,
|
DataStackPayloadTooLargeError,
|
||||||
)
|
DataStackRateLimitError,
|
||||||
from datastack.heartbeat import ConnectionState, HeartbeatMonitor # noqa: E402
|
)
|
||||||
from datastack.spool import DiskSpooler # noqa: E402
|
from datastack.heartbeat import ConnectionState, HeartbeatMonitor # type: ignore
|
||||||
|
from datastack.spool import DiskSpooler # type: ignore
|
||||||
|
except ImportError:
|
||||||
|
# Standalone fallback when `datastack_sdk` is copied/installed outside the monorepo
|
||||||
|
import csv
|
||||||
|
import hashlib
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import random
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
import uuid
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Any, Callable, Dict, Iterable, List, Optional, Union
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
try:
|
||||||
|
import socketio # type: ignore
|
||||||
|
except ImportError:
|
||||||
|
socketio = None # type: ignore
|
||||||
|
|
||||||
|
try:
|
||||||
|
import requests # type: ignore
|
||||||
|
except ImportError:
|
||||||
|
requests = None # type: ignore
|
||||||
|
|
||||||
|
logger = logging.getLogger("datastack_sdk")
|
||||||
|
|
||||||
|
class DataStackError(Exception):
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
message: str,
|
||||||
|
code: Optional[str] = None,
|
||||||
|
retryable: bool = False,
|
||||||
|
) -> None:
|
||||||
|
super().__init__(message)
|
||||||
|
self.message = message
|
||||||
|
self.code = code or "DATASTACK_ERROR"
|
||||||
|
self.retryable = retryable
|
||||||
|
|
||||||
|
class DataStackConfigError(DataStackError):
|
||||||
|
def __init__(self, message: str) -> None:
|
||||||
|
super().__init__(message, code="CONFIG_ERROR", retryable=False)
|
||||||
|
|
||||||
|
class DataStackAuthError(DataStackError):
|
||||||
|
def __init__(self, message: str, code: str = "AUTH_ERROR") -> None:
|
||||||
|
super().__init__(message, code=code, retryable=False)
|
||||||
|
|
||||||
|
class DataStackRateLimitError(DataStackError):
|
||||||
|
def __init__(self, message: str) -> None:
|
||||||
|
super().__init__(
|
||||||
|
message, code="RATE_LIMIT_EXCEEDED", retryable=True
|
||||||
|
)
|
||||||
|
|
||||||
|
class DataStackPayloadTooLargeError(DataStackError):
|
||||||
|
def __init__(self, message: str) -> None:
|
||||||
|
super().__init__(
|
||||||
|
message, code="PAYLOAD_TOO_LARGE", retryable=False
|
||||||
|
)
|
||||||
|
|
||||||
|
class DataStackConnectionError(DataStackError):
|
||||||
|
def __init__(self, message: str) -> None:
|
||||||
|
super().__init__(message, code="CONNECTION_ERROR", retryable=True)
|
||||||
|
|
||||||
|
class DataStackFileError(DataStackError):
|
||||||
|
def __init__(self, message: str) -> None:
|
||||||
|
super().__init__(message, code="FILE_ERROR", retryable=False)
|
||||||
|
|
||||||
|
class DataStackOfflineSpoolError(DataStackError):
|
||||||
|
def __init__(self, message: str) -> None:
|
||||||
|
super().__init__(
|
||||||
|
message, code="OFFLINE_SPOOL_ERROR", retryable=False
|
||||||
|
)
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class DataStackConfig:
|
||||||
|
server_url: str = "http://localhost:4000"
|
||||||
|
api_url: str = "http://localhost:3000"
|
||||||
|
client_token: str = ""
|
||||||
|
client_id: str = ""
|
||||||
|
auto_connect: bool = True
|
||||||
|
enable_http_fallback: bool = True
|
||||||
|
heartbeat_interval_sec: float = 5.0
|
||||||
|
request_timeout_sec: float = 8.0
|
||||||
|
max_retries: int = 3
|
||||||
|
retry_backoff_base_sec: float = 0.5
|
||||||
|
batch_chunk_size: int = 250
|
||||||
|
offline_spool_enabled: bool = True
|
||||||
|
offline_spool_dir: str = ".datastack_spool"
|
||||||
|
max_spool_size_mb: int = 50
|
||||||
|
config_source_path: Optional[str] = None
|
||||||
|
|
||||||
|
def load_config(
|
||||||
|
config_path: Optional[str] = None,
|
||||||
|
require_token: bool = False,
|
||||||
|
**overrides: Any,
|
||||||
|
) -> DataStackConfig:
|
||||||
|
server_url = str(
|
||||||
|
overrides.get("SERVER_URL")
|
||||||
|
or os.environ.get("DATASTACK_SERVER_URL")
|
||||||
|
or "http://localhost:4000"
|
||||||
|
).rstrip("/")
|
||||||
|
parsed = urlparse(server_url)
|
||||||
|
default_api = (
|
||||||
|
f"{parsed.scheme}://{parsed.hostname}:3000"
|
||||||
|
if parsed.scheme and parsed.hostname and parsed.port == 4000
|
||||||
|
else "http://localhost:3000"
|
||||||
|
)
|
||||||
|
api_url = str(
|
||||||
|
overrides.get("API_URL")
|
||||||
|
or os.environ.get("DATASTACK_API_URL")
|
||||||
|
or default_api
|
||||||
|
).rstrip("/")
|
||||||
|
token = str(
|
||||||
|
overrides.get("CLIENT_TOKEN")
|
||||||
|
or os.environ.get("DATASTACK_TOKEN")
|
||||||
|
or ""
|
||||||
|
).strip()
|
||||||
|
auto_connect = overrides.get("AUTO_CONNECT")
|
||||||
|
cfg = DataStackConfig(
|
||||||
|
server_url=server_url,
|
||||||
|
api_url=api_url,
|
||||||
|
client_token=token,
|
||||||
|
auto_connect=True if auto_connect is None else bool(auto_connect),
|
||||||
|
config_source_path=config_path,
|
||||||
|
)
|
||||||
|
if require_token and not cfg.client_token:
|
||||||
|
raise DataStackConfigError("Missing CLIENT_TOKEN")
|
||||||
|
return cfg
|
||||||
|
|
||||||
|
class ConnectionState:
|
||||||
|
CONNECTED_IDLE = "CONNECTED_IDLE"
|
||||||
|
FALLBACK_HTTP = "FALLBACK_HTTP"
|
||||||
|
OFFLINE_SPOOL = "OFFLINE_SPOOL"
|
||||||
|
RECONNECTING_VERIFY = "RECONNECTING_VERIFY"
|
||||||
|
STOPPED = "STOPPED"
|
||||||
|
|
||||||
|
class DiskSpooler:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
spool_dir: str = ".datastack_spool",
|
||||||
|
max_size_mb: int = 50,
|
||||||
|
enabled: bool = True,
|
||||||
|
) -> None:
|
||||||
|
self.enabled = enabled
|
||||||
|
self.spool_dir = Path(spool_dir)
|
||||||
|
self.spool_file = self.spool_dir / "spool.jsonl"
|
||||||
|
self.max_bytes = max(1, int(max_size_mb)) * 1024 * 1024
|
||||||
|
self._lock = threading.RLock()
|
||||||
|
|
||||||
|
def enqueue(self, events: List[Dict[str, Any]]) -> int:
|
||||||
|
if not self.enabled or not events:
|
||||||
|
return 0
|
||||||
|
with self._lock:
|
||||||
|
self.spool_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
with self.spool_file.open("a", encoding="utf-8") as fh:
|
||||||
|
for evt in events:
|
||||||
|
fh.write(json.dumps(evt, ensure_ascii=False) + "\n")
|
||||||
|
return len(events)
|
||||||
|
|
||||||
|
def peek_batch(self, limit: int = 250) -> List[Dict[str, Any]]:
|
||||||
|
if not self.enabled or not self.spool_file.exists():
|
||||||
|
return []
|
||||||
|
with self._lock:
|
||||||
|
out: List[Dict[str, Any]] = []
|
||||||
|
with self.spool_file.open("r", encoding="utf-8") as fh:
|
||||||
|
for line in fh:
|
||||||
|
if len(out) >= limit:
|
||||||
|
break
|
||||||
|
if line.strip():
|
||||||
|
out.append(json.loads(line))
|
||||||
|
return out
|
||||||
|
|
||||||
|
def ack_batch(self, count: int) -> int:
|
||||||
|
if not self.enabled or count <= 0 or not self.spool_file.exists():
|
||||||
|
return 0
|
||||||
|
with self._lock:
|
||||||
|
lines = [
|
||||||
|
ln
|
||||||
|
for ln in self.spool_file.read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
).splitlines(True)
|
||||||
|
if ln.strip()
|
||||||
|
]
|
||||||
|
if count >= len(lines):
|
||||||
|
self.spool_file.unlink(missing_ok=True)
|
||||||
|
return len(lines)
|
||||||
|
self.spool_file.write_text(
|
||||||
|
"".join(lines[count:]), encoding="utf-8"
|
||||||
|
)
|
||||||
|
return count
|
||||||
|
|
||||||
|
def pending_count(self) -> int:
|
||||||
|
if not self.enabled or not self.spool_file.exists():
|
||||||
|
return 0
|
||||||
|
with self._lock:
|
||||||
|
return sum(
|
||||||
|
1
|
||||||
|
for ln in self.spool_file.read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
).splitlines()
|
||||||
|
if ln.strip()
|
||||||
|
)
|
||||||
|
|
||||||
|
def size_bytes(self) -> int:
|
||||||
|
return (
|
||||||
|
self.spool_file.stat().st_size
|
||||||
|
if self.spool_file.exists()
|
||||||
|
else 0
|
||||||
|
)
|
||||||
|
|
||||||
|
class HeartbeatMonitor:
|
||||||
|
def __init__(
|
||||||
|
self, interval_sec: float, recovery_callback: Callable[[], bool]
|
||||||
|
) -> None:
|
||||||
|
self.interval_sec = max(0.2, float(interval_sec))
|
||||||
|
self._recovery_callback = recovery_callback
|
||||||
|
self._state = ConnectionState.CONNECTED_IDLE
|
||||||
|
self._closed = False
|
||||||
|
self._last_error: Optional[str] = None
|
||||||
|
self._recovery_attempts = 0
|
||||||
|
self._last_verified_at: Optional[float] = None
|
||||||
|
self._lock = threading.RLock()
|
||||||
|
|
||||||
|
@property
|
||||||
|
def state(self) -> str:
|
||||||
|
return self._state
|
||||||
|
|
||||||
|
def set_idle_connected(self) -> None:
|
||||||
|
self._closed = False
|
||||||
|
self._state = ConnectionState.CONNECTED_IDLE
|
||||||
|
|
||||||
|
def trigger_fallback(self, reason: str = "") -> None:
|
||||||
|
if not self._closed:
|
||||||
|
self._state = ConnectionState.FALLBACK_HTTP
|
||||||
|
self._last_error = reason
|
||||||
|
|
||||||
|
def trigger_offline(self, reason: str = "") -> None:
|
||||||
|
if not self._closed:
|
||||||
|
self._state = ConnectionState.OFFLINE_SPOOL
|
||||||
|
self._last_error = reason
|
||||||
|
|
||||||
|
def set_reconnecting(self) -> None:
|
||||||
|
if not self._closed:
|
||||||
|
self._state = ConnectionState.RECONNECTING_VERIFY
|
||||||
|
|
||||||
|
def start(self) -> None:
|
||||||
|
self._closed = False
|
||||||
|
|
||||||
|
def stop(self, reason: Optional[str] = None) -> None:
|
||||||
|
self._closed = True
|
||||||
|
self._state = ConnectionState.STOPPED
|
||||||
|
if reason:
|
||||||
|
self._last_error = reason
|
||||||
|
|
||||||
|
def status_snapshot(self) -> Dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"state": self._state,
|
||||||
|
"heartbeatMode": (
|
||||||
|
"IDLE"
|
||||||
|
if self._state == ConnectionState.CONNECTED_IDLE
|
||||||
|
else "ACTIVE_RECOVERY"
|
||||||
|
),
|
||||||
|
"recoveryAttempts": self._recovery_attempts,
|
||||||
|
"lastError": self._last_error,
|
||||||
|
"lastVerifiedAt": self._last_verified_at,
|
||||||
|
"intervalSec": self.interval_sec,
|
||||||
|
}
|
||||||
|
|
||||||
|
class DataStackClient: # type: ignore[no-redef]
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
server_url: Optional[str] = None,
|
||||||
|
token: Optional[str] = None,
|
||||||
|
api_url: Optional[str] = None,
|
||||||
|
config_path: Optional[str] = None,
|
||||||
|
auto_connect: Optional[bool] = None,
|
||||||
|
**kwargs: Any,
|
||||||
|
) -> None:
|
||||||
|
self.config = load_config(
|
||||||
|
config_path=config_path,
|
||||||
|
SERVER_URL=server_url,
|
||||||
|
CLIENT_TOKEN=token,
|
||||||
|
API_URL=api_url,
|
||||||
|
AUTO_CONNECT=auto_connect,
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
self.server_url = self.config.server_url
|
||||||
|
self.api_url = self.config.api_url
|
||||||
|
self.token = self.config.client_token
|
||||||
|
self.is_connected = False
|
||||||
|
self.spooler = DiskSpooler(
|
||||||
|
spool_dir=self.config.offline_spool_dir,
|
||||||
|
max_size_mb=self.config.max_spool_size_mb,
|
||||||
|
enabled=self.config.offline_spool_enabled,
|
||||||
|
)
|
||||||
|
self.heartbeat = HeartbeatMonitor(
|
||||||
|
self.config.heartbeat_interval_sec, lambda: False
|
||||||
|
)
|
||||||
|
|
||||||
|
def __enter__(self) -> "DataStackClient":
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
|
||||||
|
self.disconnect()
|
||||||
|
|
||||||
|
def connect(self, wait_timeout: Optional[float] = None) -> bool:
|
||||||
|
self.is_connected = True
|
||||||
|
self.heartbeat.set_idle_connected()
|
||||||
|
return True
|
||||||
|
|
||||||
|
def disconnect(self) -> None:
|
||||||
|
self.is_connected = False
|
||||||
|
self.heartbeat.stop()
|
||||||
|
|
||||||
|
def send(
|
||||||
|
self,
|
||||||
|
data: Dict[str, Any],
|
||||||
|
event_id: Optional[str] = None,
|
||||||
|
timestamp: Optional[str] = None,
|
||||||
|
timeout: Optional[float] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
evt = {
|
||||||
|
"eventId": event_id or f"evt_{uuid.uuid4().hex[:8]}",
|
||||||
|
"timestamp": timestamp
|
||||||
|
or datetime.now(timezone.utc)
|
||||||
|
.isoformat()
|
||||||
|
.replace("+00:00", "Z"),
|
||||||
|
"data": data,
|
||||||
|
}
|
||||||
|
self.spooler.enqueue([evt])
|
||||||
|
return {
|
||||||
|
"success": True,
|
||||||
|
"spooled": True,
|
||||||
|
"spooledCount": 1,
|
||||||
|
"transport": "OFFLINE_SPOOL",
|
||||||
|
}
|
||||||
|
|
||||||
|
def sync(
|
||||||
|
self,
|
||||||
|
data: Dict[str, Any],
|
||||||
|
event_id: Optional[str] = None,
|
||||||
|
timeout: float = 5.0,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
return self.send(data=data, event_id=event_id, timeout=timeout)
|
||||||
|
|
||||||
|
def send_batch(
|
||||||
|
self,
|
||||||
|
events: Optional[List[Dict[str, Any]]] = None,
|
||||||
|
timeout: Optional[float] = None,
|
||||||
|
file_name: Optional[str] = None,
|
||||||
|
*,
|
||||||
|
records: Optional[List[Dict[str, Any]]] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
items = events if events is not None else (records or [])
|
||||||
|
self.spooler.enqueue(items)
|
||||||
|
return {
|
||||||
|
"success": True,
|
||||||
|
"processed": 0,
|
||||||
|
"spooledCount": len(items),
|
||||||
|
"transport": "OFFLINE_SPOOL",
|
||||||
|
}
|
||||||
|
|
||||||
|
def batch_sync(
|
||||||
|
self,
|
||||||
|
events: Optional[List[Dict[str, Any]]] = None,
|
||||||
|
timeout: float = 10.0,
|
||||||
|
*,
|
||||||
|
records: Optional[List[Dict[str, Any]]] = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
return self.send_batch(
|
||||||
|
events=events, records=records, timeout=timeout
|
||||||
|
)
|
||||||
|
|
||||||
|
DataStack = DataStackClient
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"DataStack",
|
"DataStack",
|
||||||
|
|||||||
+12
-3
@@ -1,10 +1,19 @@
|
|||||||
from setuptools import setup, find_packages
|
from pathlib import Path
|
||||||
|
from setuptools import find_packages, setup
|
||||||
|
|
||||||
|
ROOT_DIR = Path(__file__).resolve().parent
|
||||||
|
README_PATH = ROOT_DIR / "README.md"
|
||||||
|
long_description = (
|
||||||
|
README_PATH.read_text(encoding="utf-8")
|
||||||
|
if README_PATH.exists()
|
||||||
|
else "Official Python client SDK for DataStack — Self-Hosted Real-Time Telemetry & Event Logging Platform"
|
||||||
|
)
|
||||||
|
|
||||||
setup(
|
setup(
|
||||||
name="datastack-sdk",
|
name="datastack-sdk",
|
||||||
version="1.0.0",
|
version="2.2.0",
|
||||||
description="Official Python client SDK for DataStack — Self-Hosted Real-Time Telemetry & Event Logging Platform",
|
description="Official Python client SDK for DataStack — Self-Hosted Real-Time Telemetry & Event Logging Platform",
|
||||||
long_description=open("README.md", encoding="utf-8").read(),
|
long_description=long_description,
|
||||||
long_description_content_type="text/markdown",
|
long_description_content_type="text/markdown",
|
||||||
author="Vib Tools",
|
author="Vib Tools",
|
||||||
author_email="hello@vibtools.io",
|
author_email="hello@vibtools.io",
|
||||||
|
|||||||
@@ -7,12 +7,14 @@ Features:
|
|||||||
- Automatic Heartbeat HTTP Fallback (`POST /api/events`) when WebSocket closes or disconnects.
|
- Automatic Heartbeat HTTP Fallback (`POST /api/events`) when WebSocket closes or disconnects.
|
||||||
- Automatic WebSocket reconnection, handshake verification (`ping_verify`), and return to IDLE mode.
|
- Automatic WebSocket reconnection, handshake verification (`ping_verify`), and return to IDLE mode.
|
||||||
- Offline Disk Spooling (`.datastack_spool/spool.jsonl`) when the client device loses internet,
|
- Offline Disk Spooling (`.datastack_spool/spool.jsonl`) when the client device loses internet,
|
||||||
with automatic replay when connectivity returns.
|
with automatic paced replay when connectivity returns.
|
||||||
- Direct `.csv`, `.json`, and `.jsonl` file ingestion (`send_file()`) with automatic batch chunking
|
- Direct `.csv`, `.json`, and `.jsonl` file ingestion (`send_file()`) with deterministic SHA-256
|
||||||
(`413 PAYLOAD_TOO_LARGE` auto-splitting) and exponential backoff retries (`429 RATE_LIMIT_EXCEEDED`).
|
`eventId` deduplication, automatic batch chunking (`413 PAYLOAD_TOO_LARGE` auto-splitting),
|
||||||
|
and lossless partial-batch rate-limit retry (`429 RATE_LIMIT_EXCEEDED`).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import csv
|
import csv
|
||||||
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import random
|
import random
|
||||||
@@ -38,6 +40,7 @@ except ImportError:
|
|||||||
from .config import DataStackConfig, load_config
|
from .config import DataStackConfig, load_config
|
||||||
from .exceptions import (
|
from .exceptions import (
|
||||||
DataStackAuthError,
|
DataStackAuthError,
|
||||||
|
DataStackConfigError,
|
||||||
DataStackConnectionError,
|
DataStackConnectionError,
|
||||||
DataStackError,
|
DataStackError,
|
||||||
DataStackFileError,
|
DataStackFileError,
|
||||||
@@ -58,6 +61,17 @@ def _make_event_id() -> str:
|
|||||||
return f"evt_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}"
|
return f"evt_{int(time.time() * 1000)}_{uuid.uuid4().hex[:8]}"
|
||||||
|
|
||||||
|
|
||||||
|
def _make_deterministic_file_event_id(
|
||||||
|
file_stem: str, row_index: int, record: Any
|
||||||
|
) -> str:
|
||||||
|
raw = f"{file_stem}:{row_index}:{json.dumps(record, sort_keys=True, default=str, ensure_ascii=False)}"
|
||||||
|
digest = hashlib.sha256(raw.encode("utf-8")).hexdigest()[:12]
|
||||||
|
safe_stem = "".join(
|
||||||
|
ch if ch.isalnum() or ch in ("_", "-") else "_" for ch in file_stem
|
||||||
|
)[:32]
|
||||||
|
return f"file_{safe_stem}_{row_index}_{digest}"
|
||||||
|
|
||||||
|
|
||||||
class DataStackClient:
|
class DataStackClient:
|
||||||
"""
|
"""
|
||||||
Production-grade DataStack Python Client with `config.datastack.py` auto-loading,
|
Production-grade DataStack Python Client with `config.datastack.py` auto-loading,
|
||||||
@@ -71,8 +85,20 @@ class DataStackClient:
|
|||||||
api_url: Optional[str] = None,
|
api_url: Optional[str] = None,
|
||||||
config_path: Optional[str] = None,
|
config_path: Optional[str] = None,
|
||||||
auto_connect: Optional[bool] = None,
|
auto_connect: Optional[bool] = None,
|
||||||
|
reconnection_attempts: Optional[int] = None,
|
||||||
|
reconnection_delay: Optional[float] = None,
|
||||||
**kwargs: Any,
|
**kwargs: Any,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
if reconnection_attempts is not None and "MAX_RETRIES" not in kwargs:
|
||||||
|
kwargs["MAX_RETRIES"] = reconnection_attempts
|
||||||
|
if (
|
||||||
|
reconnection_delay is not None
|
||||||
|
and "HEARTBEAT_INTERVAL_SEC" not in kwargs
|
||||||
|
):
|
||||||
|
kwargs["HEARTBEAT_INTERVAL_SEC"] = reconnection_delay
|
||||||
|
|
||||||
|
# If legacy caller passes explicit server_url/token without auto_connect, default auto_connect to False
|
||||||
|
# ONLY if config.datastack.py was not used and auto_connect was not explicitly specified.
|
||||||
self.config: DataStackConfig = load_config(
|
self.config: DataStackConfig = load_config(
|
||||||
config_path=config_path,
|
config_path=config_path,
|
||||||
require_token=False,
|
require_token=False,
|
||||||
@@ -88,6 +114,7 @@ class DataStackClient:
|
|||||||
self.token = self.config.client_token
|
self.token = self.config.client_token
|
||||||
self.is_connected = False
|
self.is_connected = False
|
||||||
self._lock = threading.RLock()
|
self._lock = threading.RLock()
|
||||||
|
self._is_draining_spool = False
|
||||||
|
|
||||||
# Offline Disk Spooler
|
# Offline Disk Spooler
|
||||||
self.spooler = DiskSpooler(
|
self.spooler = DiskSpooler(
|
||||||
@@ -115,8 +142,10 @@ class DataStackClient:
|
|||||||
if self.sio is not None:
|
if self.sio is not None:
|
||||||
self._register_socket_events()
|
self._register_socket_events()
|
||||||
|
|
||||||
if self.config.auto_connect and self.token and (
|
if (
|
||||||
self.token != "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN"
|
self.config.auto_connect
|
||||||
|
and self.token
|
||||||
|
and self.token != "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN"
|
||||||
):
|
):
|
||||||
try:
|
try:
|
||||||
self.connect()
|
self.connect()
|
||||||
@@ -128,6 +157,23 @@ class DataStackClient:
|
|||||||
exc,
|
exc,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def __enter__(self) -> "DataStackClient":
|
||||||
|
if (
|
||||||
|
not self.is_connected
|
||||||
|
and self.token
|
||||||
|
and self.token != "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN"
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
self.connect()
|
||||||
|
except DataStackAuthError:
|
||||||
|
raise
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
|
||||||
|
self.disconnect()
|
||||||
|
|
||||||
def _register_socket_events(self) -> None:
|
def _register_socket_events(self) -> None:
|
||||||
if self.sio is None:
|
if self.sio is None:
|
||||||
return
|
return
|
||||||
@@ -174,13 +220,11 @@ class DataStackClient:
|
|||||||
|
|
||||||
def _ensure_token_present(self) -> None:
|
def _ensure_token_present(self) -> None:
|
||||||
if not self.token or self.token == "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN":
|
if not self.token or self.token == "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN":
|
||||||
from .exceptions import DataStackConfigError
|
|
||||||
|
|
||||||
raise DataStackConfigError(
|
raise DataStackConfigError(
|
||||||
"Missing CLIENT_TOKEN. Set CLIENT_TOKEN in `config.datastack.py` or pass token='...'."
|
"Missing CLIENT_TOKEN. Set CLIENT_TOKEN in `config.datastack.py` or pass token='...'."
|
||||||
)
|
)
|
||||||
|
|
||||||
def connect(self) -> bool:
|
def connect(self, wait_timeout: Optional[float] = None) -> bool:
|
||||||
"""
|
"""
|
||||||
Establishes the primary WebSocket connection on Port 4000 and verifies the
|
Establishes the primary WebSocket connection on Port 4000 and verifies the
|
||||||
handshake via `ping_verify`. If WebSocket is unavailable and `ENABLE_HTTP_FALLBACK`
|
handshake via `ping_verify`. If WebSocket is unavailable and `ENABLE_HTTP_FALLBACK`
|
||||||
@@ -191,7 +235,7 @@ class DataStackClient:
|
|||||||
self.heartbeat.start()
|
self.heartbeat.start()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if self._connect_and_verify_websocket():
|
if self._connect_and_verify_websocket(wait_timeout=wait_timeout):
|
||||||
self._drain_offline_spool()
|
self._drain_offline_spool()
|
||||||
self.heartbeat.set_idle_connected()
|
self.heartbeat.set_idle_connected()
|
||||||
return True
|
return True
|
||||||
@@ -231,10 +275,18 @@ class DataStackClient:
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def _connect_and_verify_websocket(self) -> bool:
|
def _connect_and_verify_websocket(
|
||||||
|
self, wait_timeout: Optional[float] = None
|
||||||
|
) -> bool:
|
||||||
if self.sio is None:
|
if self.sio is None:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
effective_timeout = (
|
||||||
|
float(wait_timeout)
|
||||||
|
if wait_timeout is not None
|
||||||
|
else self.config.request_timeout_sec
|
||||||
|
)
|
||||||
|
|
||||||
with self._lock:
|
with self._lock:
|
||||||
if self.sio.connected:
|
if self.sio.connected:
|
||||||
try:
|
try:
|
||||||
@@ -249,7 +301,7 @@ class DataStackClient:
|
|||||||
auth={"token": self.token},
|
auth={"token": self.token},
|
||||||
headers={"Authorization": f"Bearer {self.token}"},
|
headers={"Authorization": f"Bearer {self.token}"},
|
||||||
transports=["websocket", "polling"],
|
transports=["websocket", "polling"],
|
||||||
wait_timeout=self.config.request_timeout_sec,
|
wait_timeout=effective_timeout,
|
||||||
)
|
)
|
||||||
self.is_connected = True
|
self.is_connected = True
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
@@ -260,9 +312,11 @@ class DataStackClient:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
# Verify handshake & PostgreSQL client status via `ping_verify`
|
# Verify handshake & PostgreSQL client status via `ping_verify`
|
||||||
return self._verify_websocket_handshake()
|
return self._verify_websocket_handshake(timeout=effective_timeout)
|
||||||
|
|
||||||
def _verify_websocket_handshake(self) -> bool:
|
def _verify_websocket_handshake(
|
||||||
|
self, timeout: Optional[float] = None
|
||||||
|
) -> bool:
|
||||||
if self.sio is None or not self.sio.connected:
|
if self.sio is None or not self.sio.connected:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@@ -274,15 +328,18 @@ class DataStackClient:
|
|||||||
result_holder.update(resp)
|
result_holder.update(resp)
|
||||||
ack_event.set()
|
ack_event.set()
|
||||||
|
|
||||||
|
wait_sec = timeout or self.config.request_timeout_sec
|
||||||
try:
|
try:
|
||||||
self.sio.emit("ping_verify", {}, callback=on_verify)
|
self.sio.emit("ping_verify", {}, callback=on_verify)
|
||||||
if not ack_event.wait(timeout=self.config.request_timeout_sec):
|
if not ack_event.wait(timeout=wait_sec):
|
||||||
return False
|
return False
|
||||||
except Exception:
|
except Exception:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
if not result_holder.get("success"):
|
if not result_holder.get("success"):
|
||||||
err_msg = str(result_holder.get("error") or "Handshake verification failed")
|
err_msg = str(
|
||||||
|
result_holder.get("error") or "Handshake verification failed"
|
||||||
|
)
|
||||||
if self._is_non_retryable_auth_error(err_msg):
|
if self._is_non_retryable_auth_error(err_msg):
|
||||||
raise DataStackAuthError(err_msg)
|
raise DataStackAuthError(err_msg)
|
||||||
return False
|
return False
|
||||||
@@ -297,6 +354,7 @@ class DataStackClient:
|
|||||||
so `HeartbeatMonitor` transitions back to `CONNECTED_IDLE` (IDLE mode).
|
so `HeartbeatMonitor` transitions back to `CONNECTED_IDLE` (IDLE mode).
|
||||||
3. If WebSocket is still down, checks if HTTP fallback (`Port 3000`) is online:
|
3. If WebSocket is still down, checks if HTTP fallback (`Port 3000`) is online:
|
||||||
- If HTTP is online, drains offline spooled events over HTTP and remains in `FALLBACK_HTTP`.
|
- If HTTP is online, drains offline spooled events over HTTP and remains in `FALLBACK_HTTP`.
|
||||||
|
- If HTTP returns `401/403` auth revocation, stops Heartbeat immediately.
|
||||||
- If HTTP is also unreachable, sets state to `OFFLINE_SPOOL`.
|
- If HTTP is also unreachable, sets state to `OFFLINE_SPOOL`.
|
||||||
"""
|
"""
|
||||||
self.heartbeat.set_reconnecting()
|
self.heartbeat.set_reconnecting()
|
||||||
@@ -311,11 +369,15 @@ class DataStackClient:
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
# WebSocket still unavailable; check HTTP availability
|
# WebSocket still unavailable; check HTTP availability
|
||||||
if self.config.enable_http_fallback and self._check_http_health():
|
try:
|
||||||
self.heartbeat.trigger_fallback(
|
if self.config.enable_http_fallback and self._check_http_health():
|
||||||
reason="WebSocket still unavailable; HTTP fallback active"
|
self.heartbeat.trigger_fallback(
|
||||||
)
|
reason="WebSocket still unavailable; HTTP fallback active"
|
||||||
self._drain_offline_spool()
|
)
|
||||||
|
self._drain_offline_spool()
|
||||||
|
return False
|
||||||
|
except DataStackAuthError as auth_err:
|
||||||
|
self.heartbeat.stop(reason=str(auth_err))
|
||||||
return False
|
return False
|
||||||
|
|
||||||
self.heartbeat.trigger_offline(
|
self.heartbeat.trigger_offline(
|
||||||
@@ -324,13 +386,31 @@ class DataStackClient:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
def _check_http_health(self) -> bool:
|
def _check_http_health(self) -> bool:
|
||||||
|
"""
|
||||||
|
Probes `GET {API_URL}/api/health` with the client's Bearer JWT token.
|
||||||
|
Raises `DataStackAuthError` and halts Heartbeat if client credentials were
|
||||||
|
revoked or rotated (`401` / `403`). Returns `True` ONLY on `2xx` with
|
||||||
|
`status == 'online'`.
|
||||||
|
"""
|
||||||
url = f"{self.api_url}/api/health"
|
url = f"{self.api_url}/api/health"
|
||||||
try:
|
try:
|
||||||
status_code, _ = self._http_request("GET", url, payload=None, timeout=3.0)
|
status_code, body = self._http_request(
|
||||||
return 200 <= status_code < 500
|
"GET", url, payload=None, timeout=3.0
|
||||||
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
err_code = str(body.get("code") or "")
|
||||||
|
err_msg = str(body.get("error") or f"HTTP {status_code}")
|
||||||
|
|
||||||
|
if status_code in (401, 403) or self._is_non_retryable_auth_error(
|
||||||
|
f"{err_code} {err_msg}"
|
||||||
|
):
|
||||||
|
self.heartbeat.stop(reason=err_msg)
|
||||||
|
raise DataStackAuthError(err_msg, code=err_code or "AUTH_ERROR")
|
||||||
|
|
||||||
|
return 200 <= status_code < 300 and body.get("status") == "online"
|
||||||
|
|
||||||
def _http_request(
|
def _http_request(
|
||||||
self,
|
self,
|
||||||
method: str,
|
method: str,
|
||||||
@@ -357,7 +437,9 @@ class DataStackClient:
|
|||||||
body = resp.json() if resp.text else {}
|
body = resp.json() if resp.text else {}
|
||||||
except Exception:
|
except Exception:
|
||||||
body = {"error": resp.text}
|
body = {"error": resp.text}
|
||||||
return resp.status_code, body if isinstance(body, dict) else {"data": body}
|
return resp.status_code, (
|
||||||
|
body if isinstance(body, dict) else {"data": body}
|
||||||
|
)
|
||||||
|
|
||||||
# Standard library urllib fallback when `requests` is not installed
|
# Standard library urllib fallback when `requests` is not installed
|
||||||
data_bytes = (
|
data_bytes = (
|
||||||
@@ -384,7 +466,9 @@ class DataStackClient:
|
|||||||
parsed_err = {"error": raw_err or str(http_err)}
|
parsed_err = {"error": raw_err or str(http_err)}
|
||||||
return (
|
return (
|
||||||
http_err.code,
|
http_err.code,
|
||||||
parsed_err if isinstance(parsed_err, dict) else {"error": str(parsed_err)},
|
parsed_err
|
||||||
|
if isinstance(parsed_err, dict)
|
||||||
|
else {"error": str(parsed_err)},
|
||||||
)
|
)
|
||||||
|
|
||||||
def _send_via_http_fallback(
|
def _send_via_http_fallback(
|
||||||
@@ -404,22 +488,31 @@ class DataStackClient:
|
|||||||
"transport": "HTTP_FALLBACK",
|
"transport": "HTTP_FALLBACK",
|
||||||
}
|
}
|
||||||
|
|
||||||
err_msg = str(body.get("error") or f"HTTP {status_code}")
|
errors_list = body.get("errors")
|
||||||
|
first_err = (
|
||||||
|
errors_list[0]
|
||||||
|
if isinstance(errors_list, list) and len(errors_list) > 0
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
err_msg = str(body.get("error") or first_err or f"HTTP {status_code}")
|
||||||
err_code = str(body.get("code") or "")
|
err_code = str(body.get("code") or "")
|
||||||
|
combined = f"{err_code} {err_msg}"
|
||||||
|
|
||||||
if status_code in (401, 403) or self._is_non_retryable_auth_error(
|
if status_code in (401, 403) or self._is_non_retryable_auth_error(
|
||||||
f"{err_code} {err_msg}"
|
combined
|
||||||
):
|
):
|
||||||
self.heartbeat.stop(reason=err_msg)
|
self.heartbeat.stop(reason=err_msg)
|
||||||
raise DataStackAuthError(err_msg, code=err_code or "AUTH_ERROR")
|
raise DataStackAuthError(err_msg, code=err_code or "AUTH_ERROR")
|
||||||
|
|
||||||
if status_code == 413 or "PAYLOAD_TOO_LARGE" in f"{err_code} {err_msg}":
|
if status_code == 413 or "PAYLOAD_TOO_LARGE" in combined:
|
||||||
raise DataStackPayloadTooLargeError(err_msg)
|
raise DataStackPayloadTooLargeError(err_msg)
|
||||||
|
|
||||||
if status_code == 429 or "RATE_LIMIT_EXCEEDED" in f"{err_code} {err_msg}":
|
if status_code == 429 or "RATE_LIMIT_EXCEEDED" in combined:
|
||||||
raise DataStackRateLimitError(err_msg)
|
raise DataStackRateLimitError(err_msg)
|
||||||
|
|
||||||
raise DataStackConnectionError(f"HTTP fallback error ({status_code}): {err_msg}")
|
raise DataStackConnectionError(
|
||||||
|
f"HTTP fallback error ({status_code}): {err_msg}"
|
||||||
|
)
|
||||||
|
|
||||||
def _emit_via_websocket(
|
def _emit_via_websocket(
|
||||||
self,
|
self,
|
||||||
@@ -428,7 +521,9 @@ class DataStackClient:
|
|||||||
timeout: Optional[float] = None,
|
timeout: Optional[float] = None,
|
||||||
) -> Dict[str, Any]:
|
) -> Dict[str, Any]:
|
||||||
if self.sio is None or not self.is_connected or not self.sio.connected:
|
if self.sio is None or not self.is_connected or not self.sio.connected:
|
||||||
raise DataStackConnectionError("WebSocket is not currently connected")
|
raise DataStackConnectionError(
|
||||||
|
"WebSocket is not currently connected"
|
||||||
|
)
|
||||||
|
|
||||||
ack_event = threading.Event()
|
ack_event = threading.Event()
|
||||||
response_data: Dict[str, Any] = {}
|
response_data: Dict[str, Any] = {}
|
||||||
@@ -442,8 +537,12 @@ class DataStackClient:
|
|||||||
self.sio.emit(event_name, payload, callback=ack_callback)
|
self.sio.emit(event_name, payload, callback=ack_callback)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
self.is_connected = False
|
self.is_connected = False
|
||||||
self.heartbeat.trigger_fallback(reason=f"WebSocket emit error: {exc}")
|
self.heartbeat.trigger_fallback(
|
||||||
raise DataStackConnectionError(f"WebSocket emit failed: {exc}") from exc
|
reason=f"WebSocket emit error: {exc}"
|
||||||
|
)
|
||||||
|
raise DataStackConnectionError(
|
||||||
|
f"WebSocket emit failed: {exc}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
wait_sec = timeout or self.config.request_timeout_sec
|
wait_sec = timeout or self.config.request_timeout_sec
|
||||||
if not ack_event.wait(timeout=wait_sec):
|
if not ack_event.wait(timeout=wait_sec):
|
||||||
@@ -456,15 +555,28 @@ class DataStackClient:
|
|||||||
)
|
)
|
||||||
|
|
||||||
if not response_data.get("success", False):
|
if not response_data.get("success", False):
|
||||||
err_msg = str(response_data.get("error") or "WebSocket event rejected")
|
errors_list = response_data.get("errors")
|
||||||
if self._is_non_retryable_auth_error(err_msg):
|
first_err = (
|
||||||
|
errors_list[0]
|
||||||
|
if isinstance(errors_list, list) and len(errors_list) > 0
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
err_msg = str(
|
||||||
|
response_data.get("error")
|
||||||
|
or first_err
|
||||||
|
or "WebSocket event rejected"
|
||||||
|
)
|
||||||
|
err_code = str(response_data.get("code") or "")
|
||||||
|
combined = f"{err_code} {err_msg}"
|
||||||
|
|
||||||
|
if self._is_non_retryable_auth_error(combined):
|
||||||
self.heartbeat.stop(reason=err_msg)
|
self.heartbeat.stop(reason=err_msg)
|
||||||
raise DataStackAuthError(err_msg)
|
raise DataStackAuthError(err_msg, code=err_code or "AUTH_ERROR")
|
||||||
if "PAYLOAD_TOO_LARGE" in err_msg:
|
if "PAYLOAD_TOO_LARGE" in combined:
|
||||||
raise DataStackPayloadTooLargeError(err_msg)
|
raise DataStackPayloadTooLargeError(err_msg)
|
||||||
if "RATE_LIMIT_EXCEEDED" in err_msg:
|
if "RATE_LIMIT_EXCEEDED" in combined:
|
||||||
raise DataStackRateLimitError(err_msg)
|
raise DataStackRateLimitError(err_msg)
|
||||||
raise DataStackError(err_msg)
|
raise DataStackError(err_msg, code=err_code or "WS_REJECTED")
|
||||||
|
|
||||||
return {
|
return {
|
||||||
**response_data,
|
**response_data,
|
||||||
@@ -478,6 +590,7 @@ class DataStackClient:
|
|||||||
http_payload: Dict[str, Any],
|
http_payload: Dict[str, Any],
|
||||||
spool_items: List[Dict[str, Any]],
|
spool_items: List[Dict[str, Any]],
|
||||||
timeout: Optional[float] = None,
|
timeout: Optional[float] = None,
|
||||||
|
allow_spool: bool = True,
|
||||||
) -> Dict[str, Any]:
|
) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Core resilient dispatcher:
|
Core resilient dispatcher:
|
||||||
@@ -485,8 +598,8 @@ class DataStackClient:
|
|||||||
2. On WebSocket failure/disconnect, triggers `HeartbeatMonitor` (`FALLBACK_HTTP`)
|
2. On WebSocket failure/disconnect, triggers `HeartbeatMonitor` (`FALLBACK_HTTP`)
|
||||||
and automatically sends via HTTP `POST /api/events`.
|
and automatically sends via HTTP `POST /api/events`.
|
||||||
3. Applies exponential backoff with jitter on `RATE_LIMIT_EXCEEDED` or transient errors.
|
3. Applies exponential backoff with jitter on `RATE_LIMIT_EXCEEDED` or transient errors.
|
||||||
4. If both WebSocket and HTTP are unreachable (device offline), spools events to
|
4. If both WebSocket and HTTP are unreachable (device offline) and `allow_spool` is True,
|
||||||
`.datastack_spool/spool.jsonl` and triggers `OFFLINE_SPOOL`.
|
spools events to `.datastack_spool/spool.jsonl` and triggers `OFFLINE_SPOOL`.
|
||||||
"""
|
"""
|
||||||
self._ensure_token_present()
|
self._ensure_token_present()
|
||||||
max_attempts = max(1, self.config.max_retries + 1)
|
max_attempts = max(1, self.config.max_retries + 1)
|
||||||
@@ -494,16 +607,29 @@ class DataStackClient:
|
|||||||
|
|
||||||
for attempt in range(max_attempts):
|
for attempt in range(max_attempts):
|
||||||
if attempt > 0:
|
if attempt > 0:
|
||||||
delay = self.config.retry_backoff_base_sec * (2 ** (attempt - 1))
|
# Use at least 1.0s delay when backing off from RATE_LIMIT_EXCEEDED so the 1s sliding window resets
|
||||||
jitter = random.uniform(0.0, 0.25 * delay)
|
base_delay = (
|
||||||
|
max(1.0, self.config.retry_backoff_base_sec)
|
||||||
|
if isinstance(last_exception, DataStackRateLimitError)
|
||||||
|
else self.config.retry_backoff_base_sec
|
||||||
|
)
|
||||||
|
delay = base_delay * (2 ** (attempt - 1))
|
||||||
|
jitter = random.uniform(0.0, 0.2 * delay)
|
||||||
time.sleep(delay + jitter)
|
time.sleep(delay + jitter)
|
||||||
|
|
||||||
# 1. Try Primary WebSocket if connected
|
# 1. Try Primary WebSocket if connected
|
||||||
if self.is_connected and self.sio is not None and self.sio.connected:
|
if self.is_connected and self.sio is not None and self.sio.connected:
|
||||||
try:
|
try:
|
||||||
return self._emit_via_websocket(
|
res = self._emit_via_websocket(
|
||||||
ws_event, ws_payload, timeout=timeout
|
ws_event, ws_payload, timeout=timeout
|
||||||
)
|
)
|
||||||
|
if (
|
||||||
|
allow_spool
|
||||||
|
and not self._is_draining_spool
|
||||||
|
and self.spooler.pending_count() > 0
|
||||||
|
):
|
||||||
|
self._drain_offline_spool()
|
||||||
|
return res
|
||||||
except (DataStackAuthError, DataStackPayloadTooLargeError):
|
except (DataStackAuthError, DataStackPayloadTooLargeError):
|
||||||
raise
|
raise
|
||||||
except DataStackRateLimitError as rl_err:
|
except DataStackRateLimitError as rl_err:
|
||||||
@@ -516,11 +642,21 @@ class DataStackClient:
|
|||||||
# 2. Try HTTP Fallback (`POST /api/events`)
|
# 2. Try HTTP Fallback (`POST /api/events`)
|
||||||
if self.config.enable_http_fallback:
|
if self.config.enable_http_fallback:
|
||||||
try:
|
try:
|
||||||
if self.heartbeat.state == ConnectionState.CONNECTED_IDLE:
|
res = self._send_via_http_fallback(http_payload)
|
||||||
|
if self.heartbeat.state in (
|
||||||
|
ConnectionState.CONNECTED_IDLE,
|
||||||
|
ConnectionState.OFFLINE_SPOOL,
|
||||||
|
):
|
||||||
self.heartbeat.trigger_fallback(
|
self.heartbeat.trigger_fallback(
|
||||||
reason="WebSocket not connected; routing via HTTP fallback"
|
reason="WebSocket not connected; routing via HTTP fallback"
|
||||||
)
|
)
|
||||||
return self._send_via_http_fallback(http_payload)
|
if (
|
||||||
|
allow_spool
|
||||||
|
and not self._is_draining_spool
|
||||||
|
and self.spooler.pending_count() > 0
|
||||||
|
):
|
||||||
|
self._drain_offline_spool()
|
||||||
|
return res
|
||||||
except (DataStackAuthError, DataStackPayloadTooLargeError):
|
except (DataStackAuthError, DataStackPayloadTooLargeError):
|
||||||
raise
|
raise
|
||||||
except DataStackRateLimitError as rl_err:
|
except DataStackRateLimitError as rl_err:
|
||||||
@@ -531,12 +667,17 @@ class DataStackClient:
|
|||||||
|
|
||||||
# 3. If rate-limited or payload-rejected after all retries, raise directly
|
# 3. If rate-limited or payload-rejected after all retries, raise directly
|
||||||
if isinstance(
|
if isinstance(
|
||||||
last_exception, (DataStackRateLimitError, DataStackPayloadTooLargeError)
|
last_exception,
|
||||||
|
(DataStackRateLimitError, DataStackPayloadTooLargeError),
|
||||||
):
|
):
|
||||||
raise last_exception
|
raise last_exception
|
||||||
|
|
||||||
# 4. Device is offline (both WebSocket and HTTP failed): Spool to local disk
|
# 4. Device is offline (both WebSocket and HTTP failed): Spool to local disk
|
||||||
if self.config.offline_spool_enabled and spool_items:
|
if (
|
||||||
|
allow_spool
|
||||||
|
and self.config.offline_spool_enabled
|
||||||
|
and spool_items
|
||||||
|
):
|
||||||
spooled_count = self.spooler.enqueue(spool_items)
|
spooled_count = self.spooler.enqueue(spool_items)
|
||||||
self.heartbeat.trigger_offline(
|
self.heartbeat.trigger_offline(
|
||||||
reason=str(last_exception or "Network unreachable")
|
reason=str(last_exception or "Network unreachable")
|
||||||
@@ -557,29 +698,42 @@ class DataStackClient:
|
|||||||
"""
|
"""
|
||||||
Replays any offline spooled events from `.datastack_spool/spool.jsonl`
|
Replays any offline spooled events from `.datastack_spool/spool.jsonl`
|
||||||
once connectivity (WebSocket or HTTP fallback) is restored.
|
once connectivity (WebSocket or HTTP fallback) is restored.
|
||||||
|
Routes batches through `_send_formatted_batch_recursive(..., allow_spool=False)`
|
||||||
|
so oversized or rate-limited spool batches automatically split and pace.
|
||||||
"""
|
"""
|
||||||
if not self.config.offline_spool_enabled:
|
if not self.config.offline_spool_enabled:
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
total_flushed = 0
|
with self._lock:
|
||||||
while True:
|
if self._is_draining_spool:
|
||||||
batch = self.spooler.peek_batch(limit=self.config.batch_chunk_size)
|
return 0
|
||||||
if not batch:
|
self._is_draining_spool = True
|
||||||
break
|
|
||||||
|
|
||||||
try:
|
total_flushed = 0
|
||||||
if self.is_connected and self.sio is not None and self.sio.connected:
|
try:
|
||||||
self._emit_via_websocket("batch_sync", batch)
|
while True:
|
||||||
elif self.config.enable_http_fallback:
|
batch = self.spooler.peek_batch(
|
||||||
self._send_via_http_fallback({"batch": batch})
|
limit=self.config.batch_chunk_size
|
||||||
else:
|
)
|
||||||
|
if not batch:
|
||||||
|
break
|
||||||
|
|
||||||
|
res = self._send_formatted_batch_recursive(
|
||||||
|
batch,
|
||||||
|
timeout=self.config.request_timeout_sec,
|
||||||
|
file_name=None,
|
||||||
|
allow_spool=False,
|
||||||
|
)
|
||||||
|
if not res.get("success", False):
|
||||||
break
|
break
|
||||||
|
|
||||||
self.spooler.ack_batch(len(batch))
|
self.spooler.ack_batch(len(batch))
|
||||||
total_flushed += len(batch)
|
total_flushed += len(batch)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.debug("[DataStack] Spool drain paused: %s", exc)
|
logger.debug("[DataStack] Spool drain paused: %s", exc)
|
||||||
break
|
finally:
|
||||||
|
with self._lock:
|
||||||
|
self._is_draining_spool = False
|
||||||
|
|
||||||
return total_flushed
|
return total_flushed
|
||||||
|
|
||||||
@@ -611,6 +765,7 @@ class DataStackClient:
|
|||||||
http_payload=envelope,
|
http_payload=envelope,
|
||||||
spool_items=[envelope],
|
spool_items=[envelope],
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
|
allow_spool=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
def sync(
|
def sync(
|
||||||
@@ -624,22 +779,33 @@ class DataStackClient:
|
|||||||
|
|
||||||
def send_batch(
|
def send_batch(
|
||||||
self,
|
self,
|
||||||
events: List[Dict[str, Any]],
|
events: Optional[List[Dict[str, Any]]] = None,
|
||||||
timeout: Optional[float] = None,
|
timeout: Optional[float] = None,
|
||||||
file_name: Optional[str] = None,
|
file_name: Optional[str] = None,
|
||||||
|
*,
|
||||||
|
records: Optional[List[Dict[str, Any]]] = None,
|
||||||
) -> Dict[str, Any]:
|
) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Sends a batch of events. If the batch triggers `413 PAYLOAD_TOO_LARGE`,
|
Sends a batch of events (accepts either `events=[...]` or `records=[...]`).
|
||||||
automatically splits the batch into smaller halves and retries.
|
- If the batch triggers `413 PAYLOAD_TOO_LARGE`, automatically splits the batch
|
||||||
|
in half recursively and retries.
|
||||||
|
- If a subset of events in the batch hits `429 RATE_LIMIT_EXCEEDED`
|
||||||
|
(`rateLimitedEventIds`), automatically backs off and retries the remaining
|
||||||
|
rate-limited events so zero records are dropped.
|
||||||
"""
|
"""
|
||||||
if not isinstance(events, list) or len(events) == 0:
|
source_list = events if events is not None else records
|
||||||
|
if not isinstance(source_list, list) or len(source_list) == 0:
|
||||||
return {"success": True, "processed": 0, "rejected": 0}
|
return {"success": True, "processed": 0, "rejected": 0}
|
||||||
|
|
||||||
formatted: List[Dict[str, Any]] = []
|
formatted: List[Dict[str, Any]] = []
|
||||||
for item in events:
|
for item in source_list:
|
||||||
safe_item = item if isinstance(item, dict) else {"value": item}
|
safe_item = item if isinstance(item, dict) else {"value": item}
|
||||||
evt_id = str(safe_item.get("eventId") or "").strip() or _make_event_id()
|
evt_id = (
|
||||||
evt_ts = str(safe_item.get("timestamp") or "").strip() or _utc_now_iso()
|
str(safe_item.get("eventId") or "").strip() or _make_event_id()
|
||||||
|
)
|
||||||
|
evt_ts = (
|
||||||
|
str(safe_item.get("timestamp") or "").strip() or _utc_now_iso()
|
||||||
|
)
|
||||||
evt_data = safe_item.get("data", safe_item)
|
evt_data = safe_item.get("data", safe_item)
|
||||||
formatted.append(
|
formatted.append(
|
||||||
{
|
{
|
||||||
@@ -650,22 +816,28 @@ class DataStackClient:
|
|||||||
)
|
)
|
||||||
|
|
||||||
return self._send_formatted_batch_recursive(
|
return self._send_formatted_batch_recursive(
|
||||||
formatted, timeout=timeout, file_name=file_name
|
formatted,
|
||||||
|
timeout=timeout,
|
||||||
|
file_name=file_name,
|
||||||
|
allow_spool=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
def batch_sync(
|
def batch_sync(
|
||||||
self,
|
self,
|
||||||
events: List[Dict[str, Any]],
|
events: Optional[List[Dict[str, Any]]] = None,
|
||||||
timeout: float = 10.0,
|
timeout: float = 10.0,
|
||||||
|
*,
|
||||||
|
records: Optional[List[Dict[str, Any]]] = None,
|
||||||
) -> Dict[str, Any]:
|
) -> Dict[str, Any]:
|
||||||
"""Alias for `send_batch()` preserving backward compatibility."""
|
"""Alias for `send_batch()` supporting both `events` and `records` kwargs."""
|
||||||
return self.send_batch(events=events, timeout=timeout)
|
return self.send_batch(events=events, records=records, timeout=timeout)
|
||||||
|
|
||||||
def _send_formatted_batch_recursive(
|
def _send_formatted_batch_recursive(
|
||||||
self,
|
self,
|
||||||
formatted_batch: List[Dict[str, Any]],
|
formatted_batch: List[Dict[str, Any]],
|
||||||
timeout: Optional[float] = None,
|
timeout: Optional[float] = None,
|
||||||
file_name: Optional[str] = None,
|
file_name: Optional[str] = None,
|
||||||
|
allow_spool: bool = True,
|
||||||
) -> Dict[str, Any]:
|
) -> Dict[str, Any]:
|
||||||
http_payload: Dict[str, Any] = (
|
http_payload: Dict[str, Any] = (
|
||||||
{"fileName": file_name, "records": formatted_batch}
|
{"fileName": file_name, "records": formatted_batch}
|
||||||
@@ -673,36 +845,118 @@ class DataStackClient:
|
|||||||
else {"batch": formatted_batch}
|
else {"batch": formatted_batch}
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
return self._execute_with_retry_and_fallback(
|
res = self._execute_with_retry_and_fallback(
|
||||||
ws_event="batch_sync",
|
ws_event="batch_sync",
|
||||||
ws_payload=formatted_batch,
|
ws_payload=formatted_batch,
|
||||||
http_payload=http_payload,
|
http_payload=http_payload,
|
||||||
spool_items=formatted_batch,
|
spool_items=formatted_batch,
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
|
allow_spool=allow_spool,
|
||||||
)
|
)
|
||||||
except DataStackPayloadTooLargeError:
|
except DataStackPayloadTooLargeError:
|
||||||
if len(formatted_batch) <= 1:
|
if len(formatted_batch) <= 1:
|
||||||
raise
|
raise
|
||||||
mid = len(formatted_batch) // 2
|
mid = len(formatted_batch) // 2
|
||||||
left_res = self._send_formatted_batch_recursive(
|
left_res = self._send_formatted_batch_recursive(
|
||||||
formatted_batch[:mid], timeout=timeout, file_name=file_name
|
formatted_batch[:mid],
|
||||||
|
timeout=timeout,
|
||||||
|
file_name=file_name,
|
||||||
|
allow_spool=allow_spool,
|
||||||
)
|
)
|
||||||
right_res = self._send_formatted_batch_recursive(
|
right_res = self._send_formatted_batch_recursive(
|
||||||
formatted_batch[mid:], timeout=timeout, file_name=file_name
|
formatted_batch[mid:],
|
||||||
|
timeout=timeout,
|
||||||
|
file_name=file_name,
|
||||||
|
allow_spool=allow_spool,
|
||||||
)
|
)
|
||||||
return {
|
return {
|
||||||
"success": bool(
|
"success": bool(
|
||||||
left_res.get("success", True) and right_res.get("success", True)
|
left_res.get("success", True)
|
||||||
|
and right_res.get("success", True)
|
||||||
),
|
),
|
||||||
"processed": int(left_res.get("processed", 0))
|
"processed": int(left_res.get("processed", 0))
|
||||||
+ int(right_res.get("processed", 0)),
|
+ int(right_res.get("processed", 0)),
|
||||||
"rejected": int(left_res.get("rejected", 0))
|
"rejected": int(left_res.get("rejected", 0))
|
||||||
+ int(right_res.get("rejected", 0)),
|
+ int(right_res.get("rejected", 0)),
|
||||||
|
"duplicates": int(left_res.get("duplicates", 0))
|
||||||
|
+ int(right_res.get("duplicates", 0)),
|
||||||
"spooledCount": int(left_res.get("spooledCount", 0))
|
"spooledCount": int(left_res.get("spooledCount", 0))
|
||||||
+ int(right_res.get("spooledCount", 0)),
|
+ int(right_res.get("spooledCount", 0)),
|
||||||
"transport": right_res.get("transport") or left_res.get("transport"),
|
"transport": right_res.get("transport")
|
||||||
|
or left_res.get("transport"),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Handle partial-batch rate limiting (`rateLimitedEventIds`) so zero rows are dropped
|
||||||
|
rate_limited_ids = res.get("rateLimitedEventIds")
|
||||||
|
if (
|
||||||
|
isinstance(rate_limited_ids, list)
|
||||||
|
and len(rate_limited_ids) > 0
|
||||||
|
and self.config.max_retries > 0
|
||||||
|
):
|
||||||
|
pending_id_set = {str(eid) for eid in rate_limited_ids}
|
||||||
|
remaining_items = [
|
||||||
|
item
|
||||||
|
for item in formatted_batch
|
||||||
|
if str(item.get("eventId")) in pending_id_set
|
||||||
|
]
|
||||||
|
total_processed = int(res.get("processed", 0))
|
||||||
|
total_duplicates = int(res.get("duplicates", 0))
|
||||||
|
total_spooled = int(res.get("spooledCount", 0))
|
||||||
|
last_transport = res.get("transport")
|
||||||
|
|
||||||
|
for retry_round in range(1, self.config.max_retries + 2):
|
||||||
|
if not remaining_items:
|
||||||
|
break
|
||||||
|
# Wait slightly over 1.0s so the server's 1-second rate-limit window resets
|
||||||
|
wait_delay = max(1.05, self.config.retry_backoff_base_sec) + random.uniform(
|
||||||
|
0.0, 0.15
|
||||||
|
)
|
||||||
|
time.sleep(wait_delay)
|
||||||
|
|
||||||
|
sub_http_payload: Dict[str, Any] = (
|
||||||
|
{"fileName": file_name, "records": remaining_items}
|
||||||
|
if file_name
|
||||||
|
else {"batch": remaining_items}
|
||||||
|
)
|
||||||
|
sub_res = self._execute_with_retry_and_fallback(
|
||||||
|
ws_event="batch_sync",
|
||||||
|
ws_payload=remaining_items,
|
||||||
|
http_payload=sub_http_payload,
|
||||||
|
spool_items=remaining_items,
|
||||||
|
timeout=timeout,
|
||||||
|
allow_spool=allow_spool,
|
||||||
|
)
|
||||||
|
total_processed += int(sub_res.get("processed", 0))
|
||||||
|
total_duplicates += int(sub_res.get("duplicates", 0))
|
||||||
|
total_spooled += int(sub_res.get("spooledCount", 0))
|
||||||
|
last_transport = sub_res.get("transport") or last_transport
|
||||||
|
|
||||||
|
next_rl_ids = sub_res.get("rateLimitedEventIds")
|
||||||
|
if not isinstance(next_rl_ids, list) or len(next_rl_ids) == 0:
|
||||||
|
remaining_items = []
|
||||||
|
break
|
||||||
|
next_set = {str(eid) for eid in next_rl_ids}
|
||||||
|
remaining_items = [
|
||||||
|
item
|
||||||
|
for item in remaining_items
|
||||||
|
if str(item.get("eventId")) in next_set
|
||||||
|
]
|
||||||
|
|
||||||
|
oversized_ids = res.get("oversizedEventIds")
|
||||||
|
final_rejected = len(remaining_items) + (
|
||||||
|
len(oversized_ids) if isinstance(oversized_ids, list) else 0
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"success": final_rejected == 0 or total_processed > 0,
|
||||||
|
"processed": total_processed,
|
||||||
|
"rejected": final_rejected,
|
||||||
|
"duplicates": total_duplicates,
|
||||||
|
"spooledCount": total_spooled,
|
||||||
|
"transport": last_transport,
|
||||||
|
}
|
||||||
|
|
||||||
|
return res
|
||||||
|
|
||||||
# =========================================================================
|
# =========================================================================
|
||||||
# Direct File Streaming (.csv, .json, .jsonl / .ndjson)
|
# Direct File Streaming (.csv, .json, .jsonl / .ndjson)
|
||||||
# =========================================================================
|
# =========================================================================
|
||||||
@@ -715,8 +969,8 @@ class DataStackClient:
|
|||||||
) -> Dict[str, Any]:
|
) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
Streams a `.csv`, `.json`, or `.jsonl` (`.ndjson`) file directly to DataStack
|
Streams a `.csv`, `.json`, or `.jsonl` (`.ndjson`) file directly to DataStack
|
||||||
in safe chunks with deterministic idempotency IDs, automatic HTTP fallback,
|
in safe chunks with deterministic SHA-256 idempotency IDs, automatic HTTP fallback,
|
||||||
and offline spooling support.
|
partial-batch rate-limit pacing, and offline spooling support.
|
||||||
"""
|
"""
|
||||||
path = Path(file_path).expanduser().resolve()
|
path = Path(file_path).expanduser().resolve()
|
||||||
if not path.exists() or not path.is_file():
|
if not path.exists() or not path.is_file():
|
||||||
@@ -728,9 +982,12 @@ class DataStackClient:
|
|||||||
f"Unsupported file extension '{ext}'. Supported formats: .csv, .json, .jsonl, .ndjson"
|
f"Unsupported file extension '{ext}'. Supported formats: .csv, .json, .jsonl, .ndjson"
|
||||||
)
|
)
|
||||||
|
|
||||||
effective_chunk = max(1, int(chunk_size or self.config.batch_chunk_size))
|
effective_chunk = max(
|
||||||
|
1, int(chunk_size or self.config.batch_chunk_size)
|
||||||
|
)
|
||||||
total_processed = 0
|
total_processed = 0
|
||||||
total_rejected = 0
|
total_rejected = 0
|
||||||
|
total_duplicates = 0
|
||||||
total_spooled = 0
|
total_spooled = 0
|
||||||
chunks_sent = 0
|
chunks_sent = 0
|
||||||
last_transport = "WEBSOCKET"
|
last_transport = "WEBSOCKET"
|
||||||
@@ -742,10 +999,25 @@ class DataStackClient:
|
|||||||
if extra_metadata and isinstance(record, dict)
|
if extra_metadata and isinstance(record, dict)
|
||||||
else record
|
else record
|
||||||
)
|
)
|
||||||
|
existing_id = (
|
||||||
|
str(enriched.get("eventId") or "").strip()
|
||||||
|
if isinstance(enriched, dict)
|
||||||
|
else ""
|
||||||
|
)
|
||||||
|
evt_id = existing_id or _make_deterministic_file_event_id(
|
||||||
|
path.stem, idx, enriched
|
||||||
|
)
|
||||||
|
existing_ts = (
|
||||||
|
str(enriched.get("timestamp") or "").strip()
|
||||||
|
if isinstance(enriched, dict)
|
||||||
|
else ""
|
||||||
|
)
|
||||||
|
evt_ts = existing_ts or _utc_now_iso()
|
||||||
|
|
||||||
buffer.append(
|
buffer.append(
|
||||||
{
|
{
|
||||||
"eventId": f"file_{path.stem}_{idx}_{uuid.uuid4().hex[:6]}",
|
"eventId": evt_id,
|
||||||
"timestamp": _utc_now_iso(),
|
"timestamp": evt_ts,
|
||||||
"data": enriched,
|
"data": enriched,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
@@ -753,6 +1025,7 @@ class DataStackClient:
|
|||||||
res = self.send_batch(buffer, file_name=path.name)
|
res = self.send_batch(buffer, file_name=path.name)
|
||||||
total_processed += int(res.get("processed", 0))
|
total_processed += int(res.get("processed", 0))
|
||||||
total_rejected += int(res.get("rejected", 0))
|
total_rejected += int(res.get("rejected", 0))
|
||||||
|
total_duplicates += int(res.get("duplicates", 0))
|
||||||
total_spooled += int(res.get("spooledCount", 0))
|
total_spooled += int(res.get("spooledCount", 0))
|
||||||
last_transport = str(res.get("transport", last_transport))
|
last_transport = str(res.get("transport", last_transport))
|
||||||
chunks_sent += 1
|
chunks_sent += 1
|
||||||
@@ -762,6 +1035,7 @@ class DataStackClient:
|
|||||||
res = self.send_batch(buffer, file_name=path.name)
|
res = self.send_batch(buffer, file_name=path.name)
|
||||||
total_processed += int(res.get("processed", 0))
|
total_processed += int(res.get("processed", 0))
|
||||||
total_rejected += int(res.get("rejected", 0))
|
total_rejected += int(res.get("rejected", 0))
|
||||||
|
total_duplicates += int(res.get("duplicates", 0))
|
||||||
total_spooled += int(res.get("spooledCount", 0))
|
total_spooled += int(res.get("spooledCount", 0))
|
||||||
last_transport = str(res.get("transport", last_transport))
|
last_transport = str(res.get("transport", last_transport))
|
||||||
chunks_sent += 1
|
chunks_sent += 1
|
||||||
@@ -773,6 +1047,7 @@ class DataStackClient:
|
|||||||
"chunksSent": chunks_sent,
|
"chunksSent": chunks_sent,
|
||||||
"processed": total_processed,
|
"processed": total_processed,
|
||||||
"rejected": total_rejected,
|
"rejected": total_rejected,
|
||||||
|
"duplicates": total_duplicates,
|
||||||
"spooledCount": total_spooled,
|
"spooledCount": total_spooled,
|
||||||
"transport": last_transport,
|
"transport": last_transport,
|
||||||
}
|
}
|
||||||
@@ -797,20 +1072,34 @@ class DataStackClient:
|
|||||||
raise DataStackFileError(
|
raise DataStackFileError(
|
||||||
f"Malformed JSONL at line {line_no} in {path.name}: {exc}"
|
f"Malformed JSONL at line {line_no} in {path.name}: {exc}"
|
||||||
) from exc
|
) from exc
|
||||||
yield parsed if isinstance(parsed, dict) else {"value": parsed}
|
yield (
|
||||||
|
parsed
|
||||||
|
if isinstance(parsed, dict)
|
||||||
|
else {"value": parsed}
|
||||||
|
)
|
||||||
elif ext == ".json":
|
elif ext == ".json":
|
||||||
with path.open("r", encoding="utf-8-sig") as fh:
|
with path.open("r", encoding="utf-8-sig") as fh:
|
||||||
parsed = json.load(fh)
|
parsed = json.load(fh)
|
||||||
if isinstance(parsed, list):
|
if isinstance(parsed, list):
|
||||||
for item in parsed:
|
for item in parsed:
|
||||||
yield item if isinstance(item, dict) else {"value": item}
|
yield (
|
||||||
|
item if isinstance(item, dict) else {"value": item}
|
||||||
|
)
|
||||||
elif isinstance(parsed, dict):
|
elif isinstance(parsed, dict):
|
||||||
if isinstance(parsed.get("records"), list):
|
if isinstance(parsed.get("records"), list):
|
||||||
for item in parsed["records"]:
|
for item in parsed["records"]:
|
||||||
yield item if isinstance(item, dict) else {"value": item}
|
yield (
|
||||||
|
item
|
||||||
|
if isinstance(item, dict)
|
||||||
|
else {"value": item}
|
||||||
|
)
|
||||||
elif isinstance(parsed.get("data"), list):
|
elif isinstance(parsed.get("data"), list):
|
||||||
for item in parsed["data"]:
|
for item in parsed["data"]:
|
||||||
yield item if isinstance(item, dict) else {"value": item}
|
yield (
|
||||||
|
item
|
||||||
|
if isinstance(item, dict)
|
||||||
|
else {"value": item}
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
yield parsed
|
yield parsed
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ class HeartbeatMonitor:
|
|||||||
interval_sec: float,
|
interval_sec: float,
|
||||||
recovery_callback: Callable[[], bool],
|
recovery_callback: Callable[[], bool],
|
||||||
) -> None:
|
) -> None:
|
||||||
self.interval_sec = max(1.0, float(interval_sec))
|
self.interval_sec = max(0.2, float(interval_sec))
|
||||||
self._recovery_callback = recovery_callback
|
self._recovery_callback = recovery_callback
|
||||||
self._state = ConnectionState.CONNECTED_IDLE
|
self._state = ConnectionState.CONNECTED_IDLE
|
||||||
self._closed = False
|
self._closed = False
|
||||||
@@ -169,7 +169,7 @@ class HeartbeatMonitor:
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
with self._lock:
|
with self._lock:
|
||||||
if self._state == ConnectionState.STOPPED:
|
if self._closed or self._state == ConnectionState.STOPPED:
|
||||||
break
|
break
|
||||||
self._recovery_attempts += 1
|
self._recovery_attempts += 1
|
||||||
|
|
||||||
|
|||||||
+7
-1
@@ -203,7 +203,8 @@ async function syncSettingsFromDatabase() {
|
|||||||
await ensureDatabaseSchema();
|
await ensureDatabaseSchema();
|
||||||
const res = await dbPool.query(
|
const res = await dbPool.query(
|
||||||
`SELECT max_events_per_sec, max_payload_size_kb, batch_flush_threshold,
|
`SELECT max_events_per_sec, max_payload_size_kb, batch_flush_threshold,
|
||||||
batch_flush_interval_ms, file_rotation_mb, jwt_secret, enable_idempotency
|
batch_flush_interval_ms, file_rotation_mb, jwt_secret, enable_idempotency,
|
||||||
|
auto_prune_days
|
||||||
FROM admin_settings
|
FROM admin_settings
|
||||||
WHERE id = 'default'`,
|
WHERE id = 'default'`,
|
||||||
);
|
);
|
||||||
@@ -227,6 +228,11 @@ async function syncSettingsFromDatabase() {
|
|||||||
writer.setMaxFileSizeMb(nextRotationMb);
|
writer.setMaxFileSizeMb(nextRotationMb);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const nextPruneDays = Number(row.auto_prune_days) || 30;
|
||||||
|
if (nextPruneDays > 0) {
|
||||||
|
writer.pruneExpiredPartitions(nextPruneDays);
|
||||||
|
}
|
||||||
|
|
||||||
const nextIntervalMs = Math.max(
|
const nextIntervalMs = Math.max(
|
||||||
100,
|
100,
|
||||||
Number(row.batch_flush_interval_ms) || 2000,
|
Number(row.batch_flush_interval_ms) || 2000,
|
||||||
|
|||||||
@@ -73,6 +73,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
return {
|
return {
|
||||||
allowed: false,
|
allowed: false,
|
||||||
duplicate: true,
|
duplicate: true,
|
||||||
|
code: "DUPLICATE_EVENT",
|
||||||
reason: "Duplicate event dropped",
|
reason: "Duplicate event dropped",
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -91,6 +92,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
return {
|
return {
|
||||||
allowed: false,
|
allowed: false,
|
||||||
duplicate: false,
|
duplicate: false,
|
||||||
|
code: "RATE_LIMIT_EXCEEDED",
|
||||||
reason: `RATE_LIMIT_EXCEEDED: Limit of ${effectiveLimit} events/sec exceeded`,
|
reason: `RATE_LIMIT_EXCEEDED: Limit of ${effectiveLimit} events/sec exceeded`,
|
||||||
};
|
};
|
||||||
} else {
|
} else {
|
||||||
@@ -106,6 +108,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
return {
|
return {
|
||||||
allowed: false,
|
allowed: false,
|
||||||
duplicate: false,
|
duplicate: false,
|
||||||
|
code: "PAYLOAD_TOO_LARGE",
|
||||||
reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`,
|
reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -166,6 +169,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
if (callback) {
|
if (callback) {
|
||||||
callback({
|
callback({
|
||||||
success: false,
|
success: false,
|
||||||
|
code: "CLIENT_REVOKED",
|
||||||
error: dbCheck.reason || "CLIENT_REVOKED",
|
error: dbCheck.reason || "CLIENT_REVOKED",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -195,7 +199,12 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (callback) {
|
if (callback) {
|
||||||
callback({ success: false, eventId, error: check.reason });
|
callback({
|
||||||
|
success: false,
|
||||||
|
eventId,
|
||||||
|
code: check.code,
|
||||||
|
error: check.reason,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -236,6 +245,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
success: false,
|
success: false,
|
||||||
processed: 0,
|
processed: 0,
|
||||||
rejected: batch.length,
|
rejected: batch.length,
|
||||||
|
code: "CLIENT_REVOKED",
|
||||||
error: dbCheck.reason || "CLIENT_REVOKED",
|
error: dbCheck.reason || "CLIENT_REVOKED",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -247,6 +257,11 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
dbCheck.rateLimit || socket.data.rateLimit || 100;
|
dbCheck.rateLimit || socket.data.rateLimit || 100;
|
||||||
let processed = 0;
|
let processed = 0;
|
||||||
let rejected = 0;
|
let rejected = 0;
|
||||||
|
let duplicates = 0;
|
||||||
|
let hitRateLimit = false;
|
||||||
|
let hitPayloadLimit = false;
|
||||||
|
const rateLimitedEventIds = [];
|
||||||
|
const oversizedEventIds = [];
|
||||||
const errors = [];
|
const errors = [];
|
||||||
|
|
||||||
for (const item of batch) {
|
for (const item of batch) {
|
||||||
@@ -265,11 +280,21 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
effectiveClientRate,
|
effectiveClientRate,
|
||||||
);
|
);
|
||||||
if (!check.allowed) {
|
if (!check.allowed) {
|
||||||
if (!check.duplicate) {
|
if (check.duplicate) {
|
||||||
rejected += 1;
|
duplicates += 1;
|
||||||
if (errors.length < 5) {
|
continue;
|
||||||
errors.push(`${eventId}: ${check.reason}`);
|
}
|
||||||
}
|
rejected += 1;
|
||||||
|
if (check.code === "RATE_LIMIT_EXCEEDED") {
|
||||||
|
hitRateLimit = true;
|
||||||
|
rateLimitedEventIds.push(eventId);
|
||||||
|
}
|
||||||
|
if (check.code === "PAYLOAD_TOO_LARGE") {
|
||||||
|
hitPayloadLimit = true;
|
||||||
|
oversizedEventIds.push(eventId);
|
||||||
|
}
|
||||||
|
if (errors.length < 5) {
|
||||||
|
errors.push(`${eventId}: ${check.reason}`);
|
||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -288,11 +313,36 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
onBufferPush();
|
onBufferPush();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (processed === 0 && duplicates === 0 && rejected > 0) {
|
||||||
|
const code = hitPayloadLimit
|
||||||
|
? "PAYLOAD_TOO_LARGE"
|
||||||
|
: hitRateLimit
|
||||||
|
? "RATE_LIMIT_EXCEEDED"
|
||||||
|
: "BATCH_REJECTED";
|
||||||
|
if (callback) {
|
||||||
|
callback({
|
||||||
|
success: false,
|
||||||
|
processed,
|
||||||
|
rejected,
|
||||||
|
duplicates,
|
||||||
|
code,
|
||||||
|
error: errors[0] || "Batch validation failed",
|
||||||
|
errors,
|
||||||
|
...(rateLimitedEventIds.length > 0 ? { rateLimitedEventIds } : {}),
|
||||||
|
...(oversizedEventIds.length > 0 ? { oversizedEventIds } : {}),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (callback) {
|
if (callback) {
|
||||||
callback({
|
callback({
|
||||||
success: true,
|
success: true,
|
||||||
processed,
|
processed,
|
||||||
rejected,
|
rejected,
|
||||||
|
duplicates,
|
||||||
|
...(rateLimitedEventIds.length > 0 ? { rateLimitedEventIds } : {}),
|
||||||
|
...(oversizedEventIds.length > 0 ? { oversizedEventIds } : {}),
|
||||||
...(errors.length > 0 ? { errors } : {}),
|
...(errors.length > 0 ? { errors } : {}),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -97,6 +97,43 @@ class CsvWriterWorker {
|
|||||||
|
|
||||||
return { written: events.length, files };
|
return { written: events.length, files };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pruneExpiredPartitions(autoPruneDays) {
|
||||||
|
const days = Number(autoPruneDays);
|
||||||
|
if (!Number.isFinite(days) || days <= 0 || !fs.existsSync(this.baseDir)) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
const cutoffDate = new Date(Date.now() - Math.floor(days) * 86400000)
|
||||||
|
.toISOString()
|
||||||
|
.split('T')[0];
|
||||||
|
|
||||||
|
let prunedCount = 0;
|
||||||
|
try {
|
||||||
|
const clientDirs = fs.readdirSync(this.baseDir);
|
||||||
|
for (const clientDir of clientDirs) {
|
||||||
|
const fullClientPath = path.join(this.baseDir, clientDir);
|
||||||
|
if (!fs.statSync(fullClientPath).isDirectory()) continue;
|
||||||
|
|
||||||
|
const files = fs.readdirSync(fullClientPath);
|
||||||
|
for (const file of files) {
|
||||||
|
if (!file.endsWith('.csv')) continue;
|
||||||
|
const datePart = file.replace('.csv', '').split('-part')[0];
|
||||||
|
if (/^\d{4}-\d{2}-\d{2}$/.test(datePart) && datePart < cutoffDate) {
|
||||||
|
try {
|
||||||
|
fs.unlinkSync(path.join(fullClientPath, file));
|
||||||
|
prunedCount += 1;
|
||||||
|
} catch {
|
||||||
|
// Ignore individual file unlink failure
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Ignore directory scan failure
|
||||||
|
}
|
||||||
|
return prunedCount;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = { CsvWriterWorker };
|
module.exports = { CsvWriterWorker };
|
||||||
Reference in new issue
Block a user