feat: aggregate external events and improve security

- Fetch events from external ingestion service in API routes
- Decouple idempotency check from registration in Security utility
- Refine CSV storage logic to verify file content for headers
- Replace Python SDK with comprehensive, feature-rich implementation
This commit is contained in:
Team Dark Devil committed 2026-11-18 03:20:00 +06:00
1 parent 043123dc21
commit 34519c07df
13 files changed
+2099 -451

No files matched your search

+31
View File
@@ -113,6 +113,37 @@ export async function GET(request: NextRequest) {
addIfMatching({ ...event, status: "buffered" });
}
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
).replace(/\/$/, "");
try {
const res = await fetch(`${ingestionBaseUrl}/internal/recent?limit=200`, {
cache: "no-store",
signal: AbortSignal.timeout(1000),
});
if (res.ok) {
const data = await res.json();
if (Array.isArray(data.events)) {
for (const evt of data.events) {
if (evt && evt.eventId) {
addIfMatching({
clientId: String(evt.clientId || ""),
eventId: String(evt.eventId),
timestamp: String(evt.timestamp || new Date().toISOString()),
payload:
typeof evt.payload === "string"
? evt.payload
: JSON.stringify(evt.payload ?? {}),
status: "buffered",
});
}
}
}
}
} catch {
// Port 4000 may not be running in isolated single-process dev mode
}
return NextResponse.json({
events: selectedRows.slice(offset),
total,
+23 -1
View File
@@ -58,12 +58,34 @@ export async function GET(req: NextRequest) {
const limit = Math.min(200, Math.max(1, Number(searchParams.get('limit')) || 50));
const bufferedEvents = DataBufferEngine.getRecentEvents(limit, clientId);
let socketBufferedEvents: DataEvent[] = [];
const ingestionBaseUrl = (
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
).replace(/\/$/, '');
try {
const qs = new URLSearchParams({ limit: String(limit) });
if (clientId) qs.set('clientId', clientId);
const res = await fetch(`${ingestionBaseUrl}/internal/recent?${qs.toString()}`, {
cache: 'no-store',
signal: AbortSignal.timeout(1000),
});
if (res.ok) {
const data = await res.json();
if (Array.isArray(data.events)) {
socketBufferedEvents = data.events;
}
}
} catch {
// Port 4000 may not be running in isolated single-process dev mode
}
const storedEvents = await getRecentStoredEvents(limit, clientId);
const seenIds = new Set<string>();
const merged: DataEvent[] = [];
for (const evt of [...bufferedEvents, ...storedEvents]) {
for (const evt of [...bufferedEvents, ...socketBufferedEvents, ...storedEvents]) {
if (!evt.eventId || seenIds.has(evt.eventId)) continue;
seenIds.add(evt.eventId);
merged.push(evt);
+38 -22
View File
@@ -30,8 +30,8 @@ export class SecurityEngine {
* ClientId + EventId must be unique.
* Duplicate events are dropped silently or tagged.
*/
public static checkIdempotency(clientId: string, eventId: string): boolean {
if (!eventId) return true; // If no eventId generated, pass through
public static hasDuplicate(clientId: string, eventId: string): boolean {
if (!eventId) return false;
const key = `${clientId}:${eventId}`;
const now = Date.now();
@@ -47,12 +47,24 @@ export class SecurityEngine {
if (this.idempotencySet.has(key)) {
const storedTime = this.idempotencySet.get(key) || 0;
if (now - storedTime < this.IDEMPOTENCY_TTL_MS) {
return false; // Duplicate detected
return true;
}
}
this.idempotencySet.set(key, now);
return true; // Unique, allowed
return false;
}
public static recordIdempotency(clientId: string, eventId: string): void {
if (!eventId) return;
this.idempotencySet.set(`${clientId}:${eventId}`, Date.now());
}
public static checkIdempotency(clientId: string, eventId: string): boolean {
if (this.hasDuplicate(clientId, eventId)) {
return false;
}
this.recordIdempotency(clientId, eventId);
return true;
}
/**
@@ -81,7 +93,8 @@ export class SecurityEngine {
/**
* Validates incoming event payload:
* Checks max byte size and JSON structure.
* Checks duplicate idempotency (read-only), max byte size, and rate limit,
* and records the idempotency key ONLY after all checks pass.
*/
public static validateEvent(
clientId: string,
@@ -89,10 +102,8 @@ export class SecurityEngine {
payload: unknown,
clientMaxRate = 100,
): ValidationResult {
// 1. Check idempotency
const isUnique =
!this.enableIdempotency || this.checkIdempotency(clientId, eventId);
if (!isUnique) {
// 1. Check idempotency (read-only so rate-limited retries are not burned)
if (this.enableIdempotency && this.hasDuplicate(clientId, eventId)) {
return {
allowed: false,
reason: "Duplicate event dropped silently",
@@ -100,18 +111,7 @@ export class SecurityEngine {
};
}
// 2. Check rate limit
const effectiveRateLimit = Math.min(clientMaxRate, this.maxEventsPerSec);
const withinLimit = this.checkRateLimit(clientId, effectiveRateLimit);
if (!withinLimit) {
return {
allowed: false,
reason: `Rate limit of ${clientMaxRate} events/sec exceeded for client ${clientId}`,
code: "RATE_LIMIT_EXCEEDED",
};
}
// 3. Check payload size
// 2. Check payload size before consuming rate-limit quota
const payloadStr =
typeof payload === "string" ? payload : JSON.stringify(payload);
const byteLength = Buffer.byteLength(payloadStr, "utf8");
@@ -124,6 +124,22 @@ export class SecurityEngine {
};
}
// 3. Check rate limit
const effectiveRateLimit = Math.min(clientMaxRate, this.maxEventsPerSec);
const withinLimit = this.checkRateLimit(clientId, effectiveRateLimit);
if (!withinLimit) {
return {
allowed: false,
reason: `Rate limit of ${clientMaxRate} events/sec exceeded for client ${clientId}`,
code: "RATE_LIMIT_EXCEEDED",
};
}
// 4. Record idempotency key ONLY after event is accepted
if (this.enableIdempotency && eventId) {
this.recordIdempotency(clientId, eventId);
}
return { allowed: true };
}
+9 -3
View File
@@ -91,7 +91,13 @@ export class CSVStorageManager {
for (const [key, batch] of grouped.entries()) {
const [clientId, date] = key.split(':::');
const targetPath = this.getTargetFilePath(clientId, date);
const fileExists = fs.existsSync(targetPath);
let hasContent = false;
try {
hasContent =
fs.existsSync(targetPath) && fs.statSync(targetPath).size > 0;
} catch {
hasContent = false;
}
// Format payload as JSON string if not already
const cleanBatch = batch.map((item) => ({
@@ -101,10 +107,10 @@ export class CSVStorageManager {
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
}));
const parser = new Parser({ fields, header: !fileExists, eol: '\n' });
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
const csvData = parser.parse(cleanBatch);
if (fileExists) {
if (hasContent) {
fs.appendFileSync(targetPath, '\n' + csvData, 'utf8');
} else {
fs.writeFileSync(targetPath, csvData, 'utf8');
File diff suppressed because it is too large. Load diff
+321
View File
@@ -0,0 +1,321 @@
"""
Auto-Discovery Configuration Loader for `config.datastack.py`.
"""
import importlib.util
import os
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Dict, List, Optional
from urllib.parse import urlparse
from .exceptions import DataStackConfigError
CONFIG_FILENAME = "config.datastack.py"
@dataclass
class DataStackConfig:
server_url: str = "http://localhost:4000"
api_url: str = "http://localhost:3000"
client_token: str = ""
client_id: str = ""
auto_connect: bool = True
enable_http_fallback: bool = True
heartbeat_interval_sec: float = 5.0
request_timeout_sec: float = 8.0
max_retries: int = 3
retry_backoff_base_sec: float = 0.5
batch_chunk_size: int = 250
offline_spool_enabled: bool = True
offline_spool_dir: str = ".datastack_spool"
max_spool_size_mb: int = 50
config_source_path: Optional[str] = None
def _derive_default_api_url(server_url: str) -> str:
"""
If API_URL is not explicitly set, infer Port 3000 on the same host when SERVER_URL
uses Port 4000, or fall back to http://localhost:3000.
"""
try:
parsed = urlparse(server_url)
if parsed.scheme and parsed.hostname:
if parsed.port == 4000:
return f"{parsed.scheme}://{parsed.hostname}:3000"
return f"{parsed.scheme}://{parsed.netloc}"
except Exception:
pass
return "http://localhost:3000"
def _candidate_config_paths(explicit_path: Optional[str] = None) -> List[Path]:
candidates: List[Path] = []
if explicit_path:
p = Path(explicit_path).expanduser().resolve()
if p.is_dir():
candidates.append(p / CONFIG_FILENAME)
else:
candidates.append(p)
env_path = os.environ.get("DATASTACK_CONFIG_PATH", "").strip()
if env_path:
ep = Path(env_path).expanduser().resolve()
if ep.is_dir():
candidates.append(ep / CONFIG_FILENAME)
else:
candidates.append(ep)
# Search current working directory and up to 3 parent levels
try:
cwd = Path.cwd().resolve()
candidates.append(cwd / CONFIG_FILENAME)
for parent in list(cwd.parents)[:3]:
candidates.append(parent / CONFIG_FILENAME)
except Exception:
pass
# Search main entry script directory
try:
if sys.argv and sys.argv[0]:
script_dir = Path(sys.argv[0]).resolve().parent
candidates.append(script_dir / CONFIG_FILENAME)
for parent in list(script_dir.parents)[:2]:
candidates.append(parent / CONFIG_FILENAME)
except Exception:
pass
# Deduplicate while preserving order
seen = set()
unique: List[Path] = []
for c in candidates:
key = str(c)
if key not in seen:
seen.add(key)
unique.append(c)
return unique
def _load_python_config_file(file_path: Path) -> Dict[str, Any]:
try:
spec = importlib.util.spec_from_file_location(
"datastack_user_config", str(file_path)
)
if spec is None or spec.loader is None:
raise DataStackConfigError(
f"Unable to load configuration specification from {file_path}"
)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
raw: Dict[str, Any] = {}
if hasattr(module, "DATASTACK_CONFIG") and isinstance(
getattr(module, "DATASTACK_CONFIG"), dict
):
raw.update(getattr(module, "DATASTACK_CONFIG"))
# Also support top-level module variables (e.g., SERVER_URL = "...", CLIENT_TOKEN = "...")
for attr in dir(module):
if attr.isupper() and attr != "DATASTACK_CONFIG":
raw[attr] = getattr(module, attr)
return raw
except DataStackConfigError:
raise
except Exception as exc:
raise DataStackConfigError(
f"Failed to parse {file_path}: {exc}"
) from exc
def load_config(
config_path: Optional[str] = None,
require_token: bool = False,
**overrides: Any,
) -> DataStackConfig:
"""
Discovers and loads `config.datastack.py` from the client application's
directory, merges environment variables (`DATASTACK_*`), and applies
any explicit keyword overrides.
"""
file_values: Dict[str, Any] = {}
resolved_source: Optional[str] = None
for candidate in _candidate_config_paths(config_path):
if candidate.is_file():
file_values = _load_python_config_file(candidate)
resolved_source = str(candidate)
break
if config_path and not resolved_source:
raise DataStackConfigError(
f"Explicitly requested config file was not found: {config_path}"
)
# Normalize keys to uppercase for lookup
normalized_file = {str(k).upper(): v for k, v in file_values.items()}
normalized_overrides = {
str(k).upper(): v for k, v in overrides.items() if v is not None
}
def pick(keys: List[str], env_keys: List[str], default: Any) -> Any:
for k in keys:
if k in normalized_overrides:
return normalized_overrides[k]
for ek in env_keys:
env_val = os.environ.get(ek)
if env_val is not None and str(env_val).strip() != "":
return env_val
for k in keys:
if k in normalized_file and normalized_file[k] is not None:
return normalized_file[k]
return default
def to_bool(val: Any, default: bool) -> bool:
if isinstance(val, bool):
return val
if isinstance(val, str):
return val.strip().lower() in ("1", "true", "yes", "on")
if val is None:
return default
return bool(val)
server_url = str(
pick(
["SERVER_URL", "SOCKET_URL", "WS_URL"],
["DATASTACK_SERVER_URL", "DATASTACK_SOCKET_URL"],
"http://localhost:4000",
)
).rstrip("/")
default_api = _derive_default_api_url(server_url)
api_url = str(
pick(
["API_URL", "HTTP_URL", "WEB_URL"],
["DATASTACK_API_URL", "DATASTACK_HTTP_URL"],
default_api,
)
).rstrip("/")
client_token = str(
pick(
["CLIENT_TOKEN", "TOKEN", "JWT_TOKEN", "API_TOKEN"],
["DATASTACK_CLIENT_TOKEN", "DATASTACK_TOKEN"],
"",
)
).strip()
client_id = str(
pick(
["CLIENT_ID"],
["DATASTACK_CLIENT_ID"],
"",
)
).strip()
cfg = DataStackConfig(
server_url=server_url,
api_url=api_url,
client_token=client_token,
client_id=client_id,
auto_connect=to_bool(
pick(["AUTO_CONNECT"], ["DATASTACK_AUTO_CONNECT"], True), True
),
enable_http_fallback=to_bool(
pick(
["ENABLE_HTTP_FALLBACK", "HTTP_FALLBACK"],
["DATASTACK_ENABLE_HTTP_FALLBACK"],
True,
),
True,
),
heartbeat_interval_sec=max(
1.0,
float(
pick(
["HEARTBEAT_INTERVAL_SEC", "HEARTBEAT_INTERVAL"],
["DATASTACK_HEARTBEAT_INTERVAL_SEC"],
5.0,
)
),
),
request_timeout_sec=max(
1.0,
float(
pick(
["REQUEST_TIMEOUT_SEC", "TIMEOUT"],
["DATASTACK_REQUEST_TIMEOUT_SEC"],
8.0,
)
),
),
max_retries=max(
0,
int(
pick(
["MAX_RETRIES", "RETRIES"],
["DATASTACK_MAX_RETRIES"],
3,
)
),
),
retry_backoff_base_sec=max(
0.1,
float(
pick(
["RETRY_BACKOFF_BASE_SEC"],
["DATASTACK_RETRY_BACKOFF_BASE_SEC"],
0.5,
)
),
),
batch_chunk_size=max(
1,
int(
pick(
["BATCH_CHUNK_SIZE", "CHUNK_SIZE"],
["DATASTACK_BATCH_CHUNK_SIZE"],
250,
)
),
),
offline_spool_enabled=to_bool(
pick(
["OFFLINE_SPOOL_ENABLED"],
["DATASTACK_OFFLINE_SPOOL_ENABLED"],
True,
),
True,
),
offline_spool_dir=str(
pick(
["OFFLINE_SPOOL_DIR", "SPOOL_DIR"],
["DATASTACK_OFFLINE_SPOOL_DIR"],
".datastack_spool",
)
),
max_spool_size_mb=max(
1,
int(
pick(
["MAX_SPOOL_SIZE_MB"],
["DATASTACK_MAX_SPOOL_SIZE_MB"],
50,
)
),
),
config_source_path=resolved_source,
)
if require_token and (
not cfg.client_token
or cfg.client_token == "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN"
):
raise DataStackConfigError(
"DataStack CLIENT_TOKEN is missing. Add your token to `config.datastack.py`, "
"set DATASTACK_TOKEN in your environment, or pass token='...' to DataStack()."
)
return cfg
+72
View File
@@ -0,0 +1,72 @@
"""
Structured Exception Hierarchy for the DataStack Python SDK.
"""
from typing import Optional
class DataStackError(Exception):
"""Base exception for all DataStack SDK errors."""
def __init__(
self,
message: str,
code: Optional[str] = None,
retryable: bool = False,
) -> None:
super().__init__(message)
self.message = message
self.code = code or "DATASTACK_ERROR"
self.retryable = retryable
class DataStackConfigError(DataStackError):
"""Raised when `config.datastack.py` or required credentials are missing or invalid."""
def __init__(self, message: str) -> None:
super().__init__(message, code="CONFIG_ERROR", retryable=False)
class DataStackAuthError(DataStackError):
"""
Raised when authentication fails (`AUTH_REQUIRED`, `AUTH_INVALID`,
`CLIENT_REVOKED`, or `TOKEN_ROTATED`). Non-retryable until credentials are updated.
"""
def __init__(self, message: str, code: str = "AUTH_ERROR") -> None:
super().__init__(message, code=code, retryable=False)
class DataStackRateLimitError(DataStackError):
"""Raised when `RATE_LIMIT_EXCEEDED` persists after all backoff retries."""
def __init__(self, message: str) -> None:
super().__init__(message, code="RATE_LIMIT_EXCEEDED", retryable=True)
class DataStackPayloadTooLargeError(DataStackError):
"""Raised when a single event exceeds the server's `maxPayloadSizeKb` limit."""
def __init__(self, message: str) -> None:
super().__init__(message, code="PAYLOAD_TOO_LARGE", retryable=False)
class DataStackConnectionError(DataStackError):
"""Raised when both WebSocket and HTTP fallback are unreachable and offline spooling is disabled."""
def __init__(self, message: str) -> None:
super().__init__(message, code="CONNECTION_ERROR", retryable=True)
class DataStackFileError(DataStackError):
"""Raised when a file passed to `send_file()` is missing, unsupported, or malformed."""
def __init__(self, message: str) -> None:
super().__init__(message, code="FILE_ERROR", retryable=False)
class DataStackOfflineSpoolError(DataStackError):
"""Raised when the local offline disk spool directory is full or unwriteable."""
def __init__(self, message: str) -> None:
super().__init__(message, code="OFFLINE_SPOOL_ERROR", retryable=False)
+186
View File
@@ -0,0 +1,186 @@
"""
Smart Heartbeat Monitor & Connection State Machine (`HeartbeatMonitor`).
Lifecycle States:
- `CONNECTED_IDLE`: WebSocket (Port 4000) is connected and verified. Heartbeat sleeps in IDLE mode.
- `FALLBACK_HTTP`: WebSocket disconnected. Client automatically sends events via HTTP (`POST /api/events`)
while Heartbeat actively probes to reconnect and verify WebSocket, then returns to `CONNECTED_IDLE`.
- `OFFLINE_SPOOL`: Device is completely offline. Events spool to disk (`.datastack_spool/spool.jsonl`).
Heartbeat monitors network restoration, reconnects, replays spooled events, and returns to `CONNECTED_IDLE`.
- `RECONNECTING_VERIFY`: Heartbeat is verifying handshake (`ping_verify`) and draining spooled events.
- `STOPPED`: Client was closed intentionally or halted due to non-retryable auth revocation.
"""
import logging
import threading
import time
from typing import Any, Callable, Dict, Optional
logger = logging.getLogger("datastack.heartbeat")
class ConnectionState:
CONNECTED_IDLE = "CONNECTED_IDLE"
FALLBACK_HTTP = "FALLBACK_HTTP"
OFFLINE_SPOOL = "OFFLINE_SPOOL"
RECONNECTING_VERIFY = "RECONNECTING_VERIFY"
STOPPED = "STOPPED"
class HeartbeatMonitor:
"""
Background recovery thread that sleeps in zero-CPU IDLE mode while WebSocket is
healthy (`CONNECTED_IDLE`) and wakes up automatically when WebSocket disconnects
or when the client device goes offline.
"""
def __init__(
self,
interval_sec: float,
recovery_callback: Callable[[], bool],
) -> None:
self.interval_sec = max(0.2, float(interval_sec))
self._recovery_callback = recovery_callback
self._state = ConnectionState.CONNECTED_IDLE
self._closed = False
self._last_error: Optional[str] = None
self._recovery_attempts = 0
self._last_verified_at: Optional[float] = None
self._lock = threading.RLock()
self._wake_event = threading.Event()
self._stop_event = threading.Event()
self._thread: Optional[threading.Thread] = None
@property
def state(self) -> str:
with self._lock:
return self._state
@property
def is_idle(self) -> bool:
with self._lock:
return self._state == ConnectionState.CONNECTED_IDLE
def set_idle_connected(self) -> None:
"""
Marks the connection as healthy (`CONNECTED_IDLE`) and puts the Heartbeat
Monitor into IDLE mode (`_wake_event.clear()`).
"""
with self._lock:
self._closed = False
self._state = ConnectionState.CONNECTED_IDLE
self._recovery_attempts = 0
self._last_error = None
self._last_verified_at = time.time()
self._wake_event.clear()
def trigger_fallback(self, reason: str = "WebSocket disconnected") -> None:
"""
Transitions to `FALLBACK_HTTP` and wakes up the Heartbeat Monitor thread
to repair the WebSocket connection in the background.
"""
with self._lock:
if self._closed:
return
self._state = ConnectionState.FALLBACK_HTTP
self._last_error = reason
self._ensure_thread_started()
self._wake_event.set()
def trigger_offline(self, reason: str = "Device offline") -> None:
"""
Transitions to `OFFLINE_SPOOL` when both WebSocket and HTTP fallback are
unreachable, waking the Heartbeat Monitor to watch for network return.
"""
with self._lock:
if self._closed:
return
self._state = ConnectionState.OFFLINE_SPOOL
self._last_error = reason
self._ensure_thread_started()
self._wake_event.set()
def set_reconnecting(self) -> None:
with self._lock:
if not self._closed:
self._state = ConnectionState.RECONNECTING_VERIFY
def start(self) -> None:
with self._lock:
self._closed = False
self._stop_event.clear()
self._ensure_thread_started()
def stop(self, reason: Optional[str] = None) -> None:
with self._lock:
self._closed = True
self._state = ConnectionState.STOPPED
if reason:
self._last_error = reason
self._stop_event.set()
self._wake_event.set()
def status_snapshot(self) -> Dict[str, Any]:
with self._lock:
return {
"state": self._state,
"heartbeatMode": (
"IDLE"
if self._state == ConnectionState.CONNECTED_IDLE
else "ACTIVE_RECOVERY"
if self._state
in (
ConnectionState.FALLBACK_HTTP,
ConnectionState.OFFLINE_SPOOL,
ConnectionState.RECONNECTING_VERIFY,
)
else "STOPPED"
),
"recoveryAttempts": self._recovery_attempts,
"lastError": self._last_error,
"lastVerifiedAt": self._last_verified_at,
"intervalSec": self.interval_sec,
}
def _ensure_thread_started(self) -> None:
if self._thread is not None and self._thread.is_alive():
return
self._stop_event.clear()
self._thread = threading.Thread(
target=self._run_loop,
name="DataStack-HeartbeatMonitor",
daemon=True,
)
self._thread.start()
def _run_loop(self) -> None:
while not self._stop_event.is_set():
# Wait in zero-CPU IDLE mode until woken by a disconnect or offline trigger
self._wake_event.wait()
if self._stop_event.is_set():
break
# Wait for the heartbeat interval before probing
if self._stop_event.wait(self.interval_sec):
break
if not self._wake_event.is_set():
continue
with self._lock:
if self._closed or self._state == ConnectionState.STOPPED:
break
self._recovery_attempts += 1
try:
recovered_to_idle = self._recovery_callback()
if recovered_to_idle:
self.set_idle_connected()
logger.info(
"[DataStack Heartbeat] WebSocket reconnected & verified. Returning to IDLE mode."
)
except Exception as exc:
with self._lock:
self._last_error = str(exc)
logger.debug("[DataStack Heartbeat] Recovery probe failed: %s", exc)
+164
View File
@@ -0,0 +1,164 @@
"""
Thread-Safe Offline Disk Spooler (`DiskSpooler`).
Persists unsent telemetry events to `.datastack_spool/spool.jsonl` when a client
device loses network connectivity completely, and drains them in deterministic
batches once the Heartbeat Monitor restores connectivity.
"""
import json
import os
import threading
from pathlib import Path
from typing import Any, Dict, List
from .exceptions import DataStackOfflineSpoolError
class DiskSpooler:
"""
Append-only JSONL disk queue with atomic batch drainage and size cap protection.
"""
def __init__(
self,
spool_dir: str = ".datastack_spool",
max_size_mb: int = 50,
enabled: bool = True,
) -> None:
self.enabled = enabled
self.spool_dir = Path(spool_dir)
self.spool_file = self.spool_dir / "spool.jsonl"
self.max_bytes = max(1, int(max_size_mb)) * 1024 * 1024
self._lock = threading.RLock()
def _ensure_dir(self) -> None:
try:
self.spool_dir.mkdir(parents=True, exist_ok=True)
except Exception as exc:
raise DataStackOfflineSpoolError(
f"Cannot create offline spool directory '{self.spool_dir}': {exc}"
) from exc
def enqueue(self, events: List[Dict[str, Any]]) -> int:
"""
Appends one or more formatted events (`{eventId, timestamp, data}`)
to `.datastack_spool/spool.jsonl`. Returns count of spooled events.
"""
if not self.enabled or not events:
return 0
with self._lock:
self._ensure_dir()
current_size = (
self.spool_file.stat().st_size if self.spool_file.exists() else 0
)
lines: List[str] = []
added_bytes = 0
for evt in events:
serialized = json.dumps(evt, ensure_ascii=False) + "\n"
encoded_len = len(serialized.encode("utf-8"))
if current_size + added_bytes + encoded_len > self.max_bytes:
raise DataStackOfflineSpoolError(
f"Offline spool limit ({self.max_bytes // (1024 * 1024)} MB) "
f"reached at '{self.spool_file}'."
)
lines.append(serialized)
added_bytes += encoded_len
try:
with self.spool_file.open("a", encoding="utf-8") as fh:
fh.writelines(lines)
fh.flush()
except Exception as exc:
raise DataStackOfflineSpoolError(
f"Failed writing to offline spool file '{self.spool_file}': {exc}"
) from exc
return len(lines)
def peek_batch(self, limit: int = 250) -> List[Dict[str, Any]]:
"""
Reads up to `limit` spooled events from the front of the spool file
without removing them until `ack_batch()` confirms delivery.
"""
if not self.enabled:
return []
with self._lock:
if not self.spool_file.exists():
return []
batch: List[Dict[str, Any]] = []
try:
with self.spool_file.open("r", encoding="utf-8") as fh:
for line in fh:
if len(batch) >= limit:
break
stripped = line.strip()
if not stripped:
continue
try:
parsed = json.loads(stripped)
if isinstance(parsed, dict):
batch.append(parsed)
except json.JSONDecodeError:
continue
except Exception:
return []
return batch
def ack_batch(self, count: int) -> int:
"""
Removes the first `count` non-empty lines from the spool file atomically
after the Heartbeat Monitor confirms they have been ingested by DataStack.
"""
if not self.enabled or count <= 0:
return 0
with self._lock:
if not self.spool_file.exists():
return 0
try:
with self.spool_file.open("r", encoding="utf-8") as fh:
all_lines = [ln for ln in fh if ln.strip()]
if count >= len(all_lines):
self.spool_file.unlink(missing_ok=True)
return len(all_lines)
remaining = all_lines[count:]
tmp_file = self.spool_dir / "spool.jsonl.tmp"
with tmp_file.open("w", encoding="utf-8") as fh:
fh.writelines(remaining)
fh.flush()
os.replace(str(tmp_file), str(self.spool_file))
return count
except Exception:
return 0
def pending_count(self) -> int:
"""Returns the number of queued events waiting in the disk spool."""
if not self.enabled:
return 0
with self._lock:
if not self.spool_file.exists():
return 0
try:
with self.spool_file.open("r", encoding="utf-8") as fh:
return sum(1 for ln in fh if ln.strip())
except Exception:
return 0
def size_bytes(self) -> int:
"""Returns current disk usage of the spool file in bytes."""
with self._lock:
if not self.spool_file.exists():
return 0
try:
return self.spool_file.stat().st_size
except Exception:
return 0
+88 -45
View File
@@ -288,12 +288,12 @@ class DataStackClient:
)
with self._lock:
self.is_connected = False
if self.sio.connected:
try:
self.sio.disconnect()
except Exception:
pass
self.is_connected = False
try:
self.sio.connect(
@@ -699,7 +699,7 @@ class DataStackClient:
Replays any offline spooled events from `.datastack_spool/spool.jsonl`
once connectivity (WebSocket or HTTP fallback) is restored.
Routes batches through `_send_formatted_batch_recursive(..., allow_spool=False)`
so oversized or rate-limited spool batches automatically split and pace.
and preserves any `unprocessedItems` so zero spooled events are ever lost.
"""
if not self.config.offline_spool_enabled:
return 0
@@ -724,10 +724,23 @@ class DataStackClient:
file_name=None,
allow_spool=False,
)
if not res.get("success", False):
processed_now = int(res.get("processed", 0))
duplicates_now = int(res.get("duplicates", 0))
unprocessed = res.get("unprocessedItems")
if (
not res.get("success", False)
and processed_now == 0
and duplicates_now == 0
):
break
self.spooler.ack_batch(len(batch))
if isinstance(unprocessed, list) and len(unprocessed) > 0:
self.spooler.enqueue(unprocessed)
total_flushed += max(0, len(batch) - len(unprocessed))
break
total_flushed += len(batch)
except Exception as exc:
logger.debug("[DataStack] Spool drain paused: %s", exc)
@@ -869,6 +882,12 @@ class DataStackClient:
file_name=file_name,
allow_spool=allow_spool,
)
left_unprocessed = left_res.get("unprocessedItems") or []
right_unprocessed = right_res.get("unprocessedItems") or []
combined_unprocessed = [
*(left_unprocessed if isinstance(left_unprocessed, list) else []),
*(right_unprocessed if isinstance(right_unprocessed, list) else []),
]
return {
"success": bool(
left_res.get("success", True)
@@ -884,15 +903,16 @@ class DataStackClient:
+ int(right_res.get("spooledCount", 0)),
"transport": right_res.get("transport")
or left_res.get("transport"),
**(
{"unprocessedItems": combined_unprocessed}
if len(combined_unprocessed) > 0
else {}
),
}
# Handle partial-batch rate limiting (`rateLimitedEventIds`) so zero rows are dropped
rate_limited_ids = res.get("rateLimitedEventIds")
if (
isinstance(rate_limited_ids, list)
and len(rate_limited_ids) > 0
and self.config.max_retries > 0
):
if isinstance(rate_limited_ids, list) and len(rate_limited_ids) > 0:
pending_id_set = {str(eid) for eid in rate_limited_ids}
remaining_items = [
item
@@ -904,56 +924,79 @@ class DataStackClient:
total_spooled = int(res.get("spooledCount", 0))
last_transport = res.get("transport")
for retry_round in range(1, self.config.max_retries + 2):
if not remaining_items:
break
# Wait slightly over 1.0s so the server's 1-second rate-limit window resets
wait_delay = max(1.05, self.config.retry_backoff_base_sec) + random.uniform(
0.0, 0.15
)
time.sleep(wait_delay)
if self.config.max_retries > 0:
for _retry_round in range(1, self.config.max_retries + 2):
if not remaining_items:
break
# Wait slightly over 1.0s so the server's 1-second rate-limit window resets
wait_delay = max(
1.05, self.config.retry_backoff_base_sec
) + random.uniform(0.0, 0.15)
time.sleep(wait_delay)
sub_http_payload: Dict[str, Any] = (
{"fileName": file_name, "records": remaining_items}
if file_name
else {"batch": remaining_items}
)
sub_res = self._execute_with_retry_and_fallback(
ws_event="batch_sync",
ws_payload=remaining_items,
http_payload=sub_http_payload,
spool_items=remaining_items,
timeout=timeout,
allow_spool=allow_spool,
)
total_processed += int(sub_res.get("processed", 0))
total_duplicates += int(sub_res.get("duplicates", 0))
total_spooled += int(sub_res.get("spooledCount", 0))
last_transport = sub_res.get("transport") or last_transport
sub_http_payload: Dict[str, Any] = (
{"fileName": file_name, "records": remaining_items}
if file_name
else {"batch": remaining_items}
)
sub_res = self._execute_with_retry_and_fallback(
ws_event="batch_sync",
ws_payload=remaining_items,
http_payload=sub_http_payload,
spool_items=remaining_items,
timeout=timeout,
allow_spool=allow_spool,
)
total_processed += int(sub_res.get("processed", 0))
total_duplicates += int(sub_res.get("duplicates", 0))
total_spooled += int(sub_res.get("spooledCount", 0))
last_transport = sub_res.get("transport") or last_transport
next_rl_ids = sub_res.get("rateLimitedEventIds")
if not isinstance(next_rl_ids, list) or len(next_rl_ids) == 0:
remaining_items = []
break
next_set = {str(eid) for eid in next_rl_ids}
remaining_items = [
item
for item in remaining_items
if str(item.get("eventId")) in next_set
]
if sub_res.get("spooled"):
remaining_items = []
break
next_rl_ids = sub_res.get("rateLimitedEventIds")
if (
not isinstance(next_rl_ids, list)
or len(next_rl_ids) == 0
):
remaining_items = []
break
next_set = {str(eid) for eid in next_rl_ids}
remaining_items = [
item
for item in remaining_items
if str(item.get("eventId")) in next_set
]
# If any items still remain rate-limited after retries and spooling is allowed, spool them so zero data is lost
if (
remaining_items
and allow_spool
and self.config.offline_spool_enabled
):
spooled_now = self.spooler.enqueue(remaining_items)
total_spooled += spooled_now
remaining_items = []
oversized_ids = res.get("oversizedEventIds")
final_rejected = len(remaining_items) + (
len(oversized_ids) if isinstance(oversized_ids, list) else 0
)
return {
"success": final_rejected == 0 or total_processed > 0,
out: Dict[str, Any] = {
"success": final_rejected == 0
or total_processed > 0
or total_spooled > 0,
"processed": total_processed,
"rejected": final_rejected,
"duplicates": total_duplicates,
"spooledCount": total_spooled,
"transport": last_transport,
}
if remaining_items:
out["unprocessedItems"] = remaining_items
return out
return res
+12
View File
@@ -289,6 +289,18 @@ app.post('/internal/flush', async (_req, res) => {
}
});
// Internal endpoint to expose Port 4000 buffered events before CSV flush
app.get('/internal/recent', (req, res) => {
const clientId = req.query.clientId ? String(req.query.clientId) : '';
const limit = Math.min(200, Math.max(1, Number(req.query.limit) || 50));
const filtered =
clientId && clientId !== 'all'
? bufferQueue.filter((evt) => evt.clientId === clientId)
: bufferQueue;
const recent = filtered.slice(-limit).reverse();
res.json({ events: recent, count: recent.length });
});
setupSocketHandlers(io, bufferQueue, writer, {
getRuntimeConfig: () => runtimeConfig,
verifyClientInDb,
+21 -17
View File
@@ -65,10 +65,10 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
const now = Date.now();
cleanupMaps(now);
// 1. Idempotency check (with 5-minute TTL)
// 1. Idempotency check (read-only check first; record only after rate-limit & payload checks pass)
const idempotencyKey = `${clientId}:${eventId}`;
if (config.enableIdempotency && eventId) {
const key = `${clientId}:${eventId}`;
const existingTime = idempotencyMap.get(key);
const existingTime = idempotencyMap.get(idempotencyKey);
if (existingTime && now - existingTime < IDEMPOTENCY_TTL_MS) {
return {
allowed: false,
@@ -77,10 +77,23 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
reason: "Duplicate event dropped",
};
}
idempotencyMap.set(key, now);
}
// 2. Rate limit check
// 2. Payload byte size check (before consuming rate-limit quota)
const payloadStr =
typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {});
const byteLength = Buffer.byteLength(payloadStr, "utf8");
const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024;
if (byteLength > maxBytes) {
return {
allowed: false,
duplicate: false,
code: "PAYLOAD_TOO_LARGE",
reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`,
};
}
// 3. Rate limit check
const effectiveLimit = Math.min(
Number(clientRateLimit) || 100,
Number(config.maxEventsPerSec) || 100,
@@ -99,18 +112,9 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
rl.count += 1;
}
// 3. Payload byte size check
const payloadStr =
typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {});
const byteLength = Buffer.byteLength(payloadStr, "utf8");
const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024;
if (byteLength > maxBytes) {
return {
allowed: false,
duplicate: false,
code: "PAYLOAD_TOO_LARGE",
reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`,
};
// 4. Record idempotency key ONLY after event passes all checks
if (config.enableIdempotency && eventId) {
idempotencyMap.set(idempotencyKey, now);
}
return { allowed: true, payloadStr };
+9 -3
View File
@@ -74,7 +74,13 @@ class CsvWriterWorker {
for (const [key, batch] of grouped.entries()) {
const [clientId, date] = key.split(':::');
const targetPath = this.getTargetFilePath(clientId, date);
const fileExists = fs.existsSync(targetPath);
let hasContent = false;
try {
hasContent =
fs.existsSync(targetPath) && fs.statSync(targetPath).size > 0;
} catch {
hasContent = false;
}
const cleanBatch = batch.map((item) => ({
clientId: item.clientId,
@@ -83,10 +89,10 @@ class CsvWriterWorker {
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
}));
const parser = new Parser({ fields, header: !fileExists, eol: '\n' });
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
const csv = parser.parse(cleanBatch);
if (fileExists) {
if (hasContent) {
fs.appendFileSync(targetPath, '\n' + csv, 'utf8');
} else {
fs.writeFileSync(targetPath, csv, 'utf8');