feat: aggregate external events and improve security

- Fetch events from external ingestion service in API routes
- Decouple idempotency check from registration in Security utility
- Refine CSV storage logic to verify file content for headers
- Replace Python SDK with comprehensive, feature-rich implementation
This commit is contained in:
Team Dark Devil committed 2026-11-18 03:20:00 +06:00
1 parent 043123dc21
commit 34519c07df
13 files changed
+2099 -451

No files matched your search

File diff suppressed because it is too large. Load diff
+321
View File
@@ -0,0 +1,321 @@
"""
Auto-Discovery Configuration Loader for `config.datastack.py`.
"""
import importlib.util
import os
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Dict, List, Optional
from urllib.parse import urlparse
from .exceptions import DataStackConfigError
CONFIG_FILENAME = "config.datastack.py"
@dataclass
class DataStackConfig:
server_url: str = "http://localhost:4000"
api_url: str = "http://localhost:3000"
client_token: str = ""
client_id: str = ""
auto_connect: bool = True
enable_http_fallback: bool = True
heartbeat_interval_sec: float = 5.0
request_timeout_sec: float = 8.0
max_retries: int = 3
retry_backoff_base_sec: float = 0.5
batch_chunk_size: int = 250
offline_spool_enabled: bool = True
offline_spool_dir: str = ".datastack_spool"
max_spool_size_mb: int = 50
config_source_path: Optional[str] = None
def _derive_default_api_url(server_url: str) -> str:
"""
If API_URL is not explicitly set, infer Port 3000 on the same host when SERVER_URL
uses Port 4000, or fall back to http://localhost:3000.
"""
try:
parsed = urlparse(server_url)
if parsed.scheme and parsed.hostname:
if parsed.port == 4000:
return f"{parsed.scheme}://{parsed.hostname}:3000"
return f"{parsed.scheme}://{parsed.netloc}"
except Exception:
pass
return "http://localhost:3000"
def _candidate_config_paths(explicit_path: Optional[str] = None) -> List[Path]:
candidates: List[Path] = []
if explicit_path:
p = Path(explicit_path).expanduser().resolve()
if p.is_dir():
candidates.append(p / CONFIG_FILENAME)
else:
candidates.append(p)
env_path = os.environ.get("DATASTACK_CONFIG_PATH", "").strip()
if env_path:
ep = Path(env_path).expanduser().resolve()
if ep.is_dir():
candidates.append(ep / CONFIG_FILENAME)
else:
candidates.append(ep)
# Search current working directory and up to 3 parent levels
try:
cwd = Path.cwd().resolve()
candidates.append(cwd / CONFIG_FILENAME)
for parent in list(cwd.parents)[:3]:
candidates.append(parent / CONFIG_FILENAME)
except Exception:
pass
# Search main entry script directory
try:
if sys.argv and sys.argv[0]:
script_dir = Path(sys.argv[0]).resolve().parent
candidates.append(script_dir / CONFIG_FILENAME)
for parent in list(script_dir.parents)[:2]:
candidates.append(parent / CONFIG_FILENAME)
except Exception:
pass
# Deduplicate while preserving order
seen = set()
unique: List[Path] = []
for c in candidates:
key = str(c)
if key not in seen:
seen.add(key)
unique.append(c)
return unique
def _load_python_config_file(file_path: Path) -> Dict[str, Any]:
try:
spec = importlib.util.spec_from_file_location(
"datastack_user_config", str(file_path)
)
if spec is None or spec.loader is None:
raise DataStackConfigError(
f"Unable to load configuration specification from {file_path}"
)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
raw: Dict[str, Any] = {}
if hasattr(module, "DATASTACK_CONFIG") and isinstance(
getattr(module, "DATASTACK_CONFIG"), dict
):
raw.update(getattr(module, "DATASTACK_CONFIG"))
# Also support top-level module variables (e.g., SERVER_URL = "...", CLIENT_TOKEN = "...")
for attr in dir(module):
if attr.isupper() and attr != "DATASTACK_CONFIG":
raw[attr] = getattr(module, attr)
return raw
except DataStackConfigError:
raise
except Exception as exc:
raise DataStackConfigError(
f"Failed to parse {file_path}: {exc}"
) from exc
def load_config(
config_path: Optional[str] = None,
require_token: bool = False,
**overrides: Any,
) -> DataStackConfig:
"""
Discovers and loads `config.datastack.py` from the client application's
directory, merges environment variables (`DATASTACK_*`), and applies
any explicit keyword overrides.
"""
file_values: Dict[str, Any] = {}
resolved_source: Optional[str] = None
for candidate in _candidate_config_paths(config_path):
if candidate.is_file():
file_values = _load_python_config_file(candidate)
resolved_source = str(candidate)
break
if config_path and not resolved_source:
raise DataStackConfigError(
f"Explicitly requested config file was not found: {config_path}"
)
# Normalize keys to uppercase for lookup
normalized_file = {str(k).upper(): v for k, v in file_values.items()}
normalized_overrides = {
str(k).upper(): v for k, v in overrides.items() if v is not None
}
def pick(keys: List[str], env_keys: List[str], default: Any) -> Any:
for k in keys:
if k in normalized_overrides:
return normalized_overrides[k]
for ek in env_keys:
env_val = os.environ.get(ek)
if env_val is not None and str(env_val).strip() != "":
return env_val
for k in keys:
if k in normalized_file and normalized_file[k] is not None:
return normalized_file[k]
return default
def to_bool(val: Any, default: bool) -> bool:
if isinstance(val, bool):
return val
if isinstance(val, str):
return val.strip().lower() in ("1", "true", "yes", "on")
if val is None:
return default
return bool(val)
server_url = str(
pick(
["SERVER_URL", "SOCKET_URL", "WS_URL"],
["DATASTACK_SERVER_URL", "DATASTACK_SOCKET_URL"],
"http://localhost:4000",
)
).rstrip("/")
default_api = _derive_default_api_url(server_url)
api_url = str(
pick(
["API_URL", "HTTP_URL", "WEB_URL"],
["DATASTACK_API_URL", "DATASTACK_HTTP_URL"],
default_api,
)
).rstrip("/")
client_token = str(
pick(
["CLIENT_TOKEN", "TOKEN", "JWT_TOKEN", "API_TOKEN"],
["DATASTACK_CLIENT_TOKEN", "DATASTACK_TOKEN"],
"",
)
).strip()
client_id = str(
pick(
["CLIENT_ID"],
["DATASTACK_CLIENT_ID"],
"",
)
).strip()
cfg = DataStackConfig(
server_url=server_url,
api_url=api_url,
client_token=client_token,
client_id=client_id,
auto_connect=to_bool(
pick(["AUTO_CONNECT"], ["DATASTACK_AUTO_CONNECT"], True), True
),
enable_http_fallback=to_bool(
pick(
["ENABLE_HTTP_FALLBACK", "HTTP_FALLBACK"],
["DATASTACK_ENABLE_HTTP_FALLBACK"],
True,
),
True,
),
heartbeat_interval_sec=max(
1.0,
float(
pick(
["HEARTBEAT_INTERVAL_SEC", "HEARTBEAT_INTERVAL"],
["DATASTACK_HEARTBEAT_INTERVAL_SEC"],
5.0,
)
),
),
request_timeout_sec=max(
1.0,
float(
pick(
["REQUEST_TIMEOUT_SEC", "TIMEOUT"],
["DATASTACK_REQUEST_TIMEOUT_SEC"],
8.0,
)
),
),
max_retries=max(
0,
int(
pick(
["MAX_RETRIES", "RETRIES"],
["DATASTACK_MAX_RETRIES"],
3,
)
),
),
retry_backoff_base_sec=max(
0.1,
float(
pick(
["RETRY_BACKOFF_BASE_SEC"],
["DATASTACK_RETRY_BACKOFF_BASE_SEC"],
0.5,
)
),
),
batch_chunk_size=max(
1,
int(
pick(
["BATCH_CHUNK_SIZE", "CHUNK_SIZE"],
["DATASTACK_BATCH_CHUNK_SIZE"],
250,
)
),
),
offline_spool_enabled=to_bool(
pick(
["OFFLINE_SPOOL_ENABLED"],
["DATASTACK_OFFLINE_SPOOL_ENABLED"],
True,
),
True,
),
offline_spool_dir=str(
pick(
["OFFLINE_SPOOL_DIR", "SPOOL_DIR"],
["DATASTACK_OFFLINE_SPOOL_DIR"],
".datastack_spool",
)
),
max_spool_size_mb=max(
1,
int(
pick(
["MAX_SPOOL_SIZE_MB"],
["DATASTACK_MAX_SPOOL_SIZE_MB"],
50,
)
),
),
config_source_path=resolved_source,
)
if require_token and (
not cfg.client_token
or cfg.client_token == "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN"
):
raise DataStackConfigError(
"DataStack CLIENT_TOKEN is missing. Add your token to `config.datastack.py`, "
"set DATASTACK_TOKEN in your environment, or pass token='...' to DataStack()."
)
return cfg
+72
View File
@@ -0,0 +1,72 @@
"""
Structured Exception Hierarchy for the DataStack Python SDK.
"""
from typing import Optional
class DataStackError(Exception):
"""Base exception for all DataStack SDK errors."""
def __init__(
self,
message: str,
code: Optional[str] = None,
retryable: bool = False,
) -> None:
super().__init__(message)
self.message = message
self.code = code or "DATASTACK_ERROR"
self.retryable = retryable
class DataStackConfigError(DataStackError):
"""Raised when `config.datastack.py` or required credentials are missing or invalid."""
def __init__(self, message: str) -> None:
super().__init__(message, code="CONFIG_ERROR", retryable=False)
class DataStackAuthError(DataStackError):
"""
Raised when authentication fails (`AUTH_REQUIRED`, `AUTH_INVALID`,
`CLIENT_REVOKED`, or `TOKEN_ROTATED`). Non-retryable until credentials are updated.
"""
def __init__(self, message: str, code: str = "AUTH_ERROR") -> None:
super().__init__(message, code=code, retryable=False)
class DataStackRateLimitError(DataStackError):
"""Raised when `RATE_LIMIT_EXCEEDED` persists after all backoff retries."""
def __init__(self, message: str) -> None:
super().__init__(message, code="RATE_LIMIT_EXCEEDED", retryable=True)
class DataStackPayloadTooLargeError(DataStackError):
"""Raised when a single event exceeds the server's `maxPayloadSizeKb` limit."""
def __init__(self, message: str) -> None:
super().__init__(message, code="PAYLOAD_TOO_LARGE", retryable=False)
class DataStackConnectionError(DataStackError):
"""Raised when both WebSocket and HTTP fallback are unreachable and offline spooling is disabled."""
def __init__(self, message: str) -> None:
super().__init__(message, code="CONNECTION_ERROR", retryable=True)
class DataStackFileError(DataStackError):
"""Raised when a file passed to `send_file()` is missing, unsupported, or malformed."""
def __init__(self, message: str) -> None:
super().__init__(message, code="FILE_ERROR", retryable=False)
class DataStackOfflineSpoolError(DataStackError):
"""Raised when the local offline disk spool directory is full or unwriteable."""
def __init__(self, message: str) -> None:
super().__init__(message, code="OFFLINE_SPOOL_ERROR", retryable=False)
+186
View File
@@ -0,0 +1,186 @@
"""
Smart Heartbeat Monitor & Connection State Machine (`HeartbeatMonitor`).
Lifecycle States:
- `CONNECTED_IDLE`: WebSocket (Port 4000) is connected and verified. Heartbeat sleeps in IDLE mode.
- `FALLBACK_HTTP`: WebSocket disconnected. Client automatically sends events via HTTP (`POST /api/events`)
while Heartbeat actively probes to reconnect and verify WebSocket, then returns to `CONNECTED_IDLE`.
- `OFFLINE_SPOOL`: Device is completely offline. Events spool to disk (`.datastack_spool/spool.jsonl`).
Heartbeat monitors network restoration, reconnects, replays spooled events, and returns to `CONNECTED_IDLE`.
- `RECONNECTING_VERIFY`: Heartbeat is verifying handshake (`ping_verify`) and draining spooled events.
- `STOPPED`: Client was closed intentionally or halted due to non-retryable auth revocation.
"""
import logging
import threading
import time
from typing import Any, Callable, Dict, Optional
logger = logging.getLogger("datastack.heartbeat")
class ConnectionState:
CONNECTED_IDLE = "CONNECTED_IDLE"
FALLBACK_HTTP = "FALLBACK_HTTP"
OFFLINE_SPOOL = "OFFLINE_SPOOL"
RECONNECTING_VERIFY = "RECONNECTING_VERIFY"
STOPPED = "STOPPED"
class HeartbeatMonitor:
"""
Background recovery thread that sleeps in zero-CPU IDLE mode while WebSocket is
healthy (`CONNECTED_IDLE`) and wakes up automatically when WebSocket disconnects
or when the client device goes offline.
"""
def __init__(
self,
interval_sec: float,
recovery_callback: Callable[[], bool],
) -> None:
self.interval_sec = max(0.2, float(interval_sec))
self._recovery_callback = recovery_callback
self._state = ConnectionState.CONNECTED_IDLE
self._closed = False
self._last_error: Optional[str] = None
self._recovery_attempts = 0
self._last_verified_at: Optional[float] = None
self._lock = threading.RLock()
self._wake_event = threading.Event()
self._stop_event = threading.Event()
self._thread: Optional[threading.Thread] = None
@property
def state(self) -> str:
with self._lock:
return self._state
@property
def is_idle(self) -> bool:
with self._lock:
return self._state == ConnectionState.CONNECTED_IDLE
def set_idle_connected(self) -> None:
"""
Marks the connection as healthy (`CONNECTED_IDLE`) and puts the Heartbeat
Monitor into IDLE mode (`_wake_event.clear()`).
"""
with self._lock:
self._closed = False
self._state = ConnectionState.CONNECTED_IDLE
self._recovery_attempts = 0
self._last_error = None
self._last_verified_at = time.time()
self._wake_event.clear()
def trigger_fallback(self, reason: str = "WebSocket disconnected") -> None:
"""
Transitions to `FALLBACK_HTTP` and wakes up the Heartbeat Monitor thread
to repair the WebSocket connection in the background.
"""
with self._lock:
if self._closed:
return
self._state = ConnectionState.FALLBACK_HTTP
self._last_error = reason
self._ensure_thread_started()
self._wake_event.set()
def trigger_offline(self, reason: str = "Device offline") -> None:
"""
Transitions to `OFFLINE_SPOOL` when both WebSocket and HTTP fallback are
unreachable, waking the Heartbeat Monitor to watch for network return.
"""
with self._lock:
if self._closed:
return
self._state = ConnectionState.OFFLINE_SPOOL
self._last_error = reason
self._ensure_thread_started()
self._wake_event.set()
def set_reconnecting(self) -> None:
with self._lock:
if not self._closed:
self._state = ConnectionState.RECONNECTING_VERIFY
def start(self) -> None:
with self._lock:
self._closed = False
self._stop_event.clear()
self._ensure_thread_started()
def stop(self, reason: Optional[str] = None) -> None:
with self._lock:
self._closed = True
self._state = ConnectionState.STOPPED
if reason:
self._last_error = reason
self._stop_event.set()
self._wake_event.set()
def status_snapshot(self) -> Dict[str, Any]:
with self._lock:
return {
"state": self._state,
"heartbeatMode": (
"IDLE"
if self._state == ConnectionState.CONNECTED_IDLE
else "ACTIVE_RECOVERY"
if self._state
in (
ConnectionState.FALLBACK_HTTP,
ConnectionState.OFFLINE_SPOOL,
ConnectionState.RECONNECTING_VERIFY,
)
else "STOPPED"
),
"recoveryAttempts": self._recovery_attempts,
"lastError": self._last_error,
"lastVerifiedAt": self._last_verified_at,
"intervalSec": self.interval_sec,
}
def _ensure_thread_started(self) -> None:
if self._thread is not None and self._thread.is_alive():
return
self._stop_event.clear()
self._thread = threading.Thread(
target=self._run_loop,
name="DataStack-HeartbeatMonitor",
daemon=True,
)
self._thread.start()
def _run_loop(self) -> None:
while not self._stop_event.is_set():
# Wait in zero-CPU IDLE mode until woken by a disconnect or offline trigger
self._wake_event.wait()
if self._stop_event.is_set():
break
# Wait for the heartbeat interval before probing
if self._stop_event.wait(self.interval_sec):
break
if not self._wake_event.is_set():
continue
with self._lock:
if self._closed or self._state == ConnectionState.STOPPED:
break
self._recovery_attempts += 1
try:
recovered_to_idle = self._recovery_callback()
if recovered_to_idle:
self.set_idle_connected()
logger.info(
"[DataStack Heartbeat] WebSocket reconnected & verified. Returning to IDLE mode."
)
except Exception as exc:
with self._lock:
self._last_error = str(exc)
logger.debug("[DataStack Heartbeat] Recovery probe failed: %s", exc)
+164
View File
@@ -0,0 +1,164 @@
"""
Thread-Safe Offline Disk Spooler (`DiskSpooler`).
Persists unsent telemetry events to `.datastack_spool/spool.jsonl` when a client
device loses network connectivity completely, and drains them in deterministic
batches once the Heartbeat Monitor restores connectivity.
"""
import json
import os
import threading
from pathlib import Path
from typing import Any, Dict, List
from .exceptions import DataStackOfflineSpoolError
class DiskSpooler:
"""
Append-only JSONL disk queue with atomic batch drainage and size cap protection.
"""
def __init__(
self,
spool_dir: str = ".datastack_spool",
max_size_mb: int = 50,
enabled: bool = True,
) -> None:
self.enabled = enabled
self.spool_dir = Path(spool_dir)
self.spool_file = self.spool_dir / "spool.jsonl"
self.max_bytes = max(1, int(max_size_mb)) * 1024 * 1024
self._lock = threading.RLock()
def _ensure_dir(self) -> None:
try:
self.spool_dir.mkdir(parents=True, exist_ok=True)
except Exception as exc:
raise DataStackOfflineSpoolError(
f"Cannot create offline spool directory '{self.spool_dir}': {exc}"
) from exc
def enqueue(self, events: List[Dict[str, Any]]) -> int:
"""
Appends one or more formatted events (`{eventId, timestamp, data}`)
to `.datastack_spool/spool.jsonl`. Returns count of spooled events.
"""
if not self.enabled or not events:
return 0
with self._lock:
self._ensure_dir()
current_size = (
self.spool_file.stat().st_size if self.spool_file.exists() else 0
)
lines: List[str] = []
added_bytes = 0
for evt in events:
serialized = json.dumps(evt, ensure_ascii=False) + "\n"
encoded_len = len(serialized.encode("utf-8"))
if current_size + added_bytes + encoded_len > self.max_bytes:
raise DataStackOfflineSpoolError(
f"Offline spool limit ({self.max_bytes // (1024 * 1024)} MB) "
f"reached at '{self.spool_file}'."
)
lines.append(serialized)
added_bytes += encoded_len
try:
with self.spool_file.open("a", encoding="utf-8") as fh:
fh.writelines(lines)
fh.flush()
except Exception as exc:
raise DataStackOfflineSpoolError(
f"Failed writing to offline spool file '{self.spool_file}': {exc}"
) from exc
return len(lines)
def peek_batch(self, limit: int = 250) -> List[Dict[str, Any]]:
"""
Reads up to `limit` spooled events from the front of the spool file
without removing them until `ack_batch()` confirms delivery.
"""
if not self.enabled:
return []
with self._lock:
if not self.spool_file.exists():
return []
batch: List[Dict[str, Any]] = []
try:
with self.spool_file.open("r", encoding="utf-8") as fh:
for line in fh:
if len(batch) >= limit:
break
stripped = line.strip()
if not stripped:
continue
try:
parsed = json.loads(stripped)
if isinstance(parsed, dict):
batch.append(parsed)
except json.JSONDecodeError:
continue
except Exception:
return []
return batch
def ack_batch(self, count: int) -> int:
"""
Removes the first `count` non-empty lines from the spool file atomically
after the Heartbeat Monitor confirms they have been ingested by DataStack.
"""
if not self.enabled or count <= 0:
return 0
with self._lock:
if not self.spool_file.exists():
return 0
try:
with self.spool_file.open("r", encoding="utf-8") as fh:
all_lines = [ln for ln in fh if ln.strip()]
if count >= len(all_lines):
self.spool_file.unlink(missing_ok=True)
return len(all_lines)
remaining = all_lines[count:]
tmp_file = self.spool_dir / "spool.jsonl.tmp"
with tmp_file.open("w", encoding="utf-8") as fh:
fh.writelines(remaining)
fh.flush()
os.replace(str(tmp_file), str(self.spool_file))
return count
except Exception:
return 0
def pending_count(self) -> int:
"""Returns the number of queued events waiting in the disk spool."""
if not self.enabled:
return 0
with self._lock:
if not self.spool_file.exists():
return 0
try:
with self.spool_file.open("r", encoding="utf-8") as fh:
return sum(1 for ln in fh if ln.strip())
except Exception:
return 0
def size_bytes(self) -> int:
"""Returns current disk usage of the spool file in bytes."""
with self._lock:
if not self.spool_file.exists():
return 0
try:
return self.spool_file.stat().st_size
except Exception:
return 0