refactor(auth): standardize session management and security
- Centralize secret retrieval using `getAdminSessionSecret` - Replace manual session parsing with `verifyRequestAdminSession` utility - Tighten API validation and improve security headers handling - Enforce environment variable security for production deployments
This commit is contained in:
1 parent
34519c07df
commit
177ed4a844
78 files changed
+6748
-3157
No files matched your search
@@ -33,6 +33,7 @@ DATASTACK_CONFIG = {
|
||||
"SERVER_URL": "http://localhost:4000", # Port 4000 Socket.IO Ingestion Service
|
||||
"API_URL": "http://localhost:3000", # Port 3000 HTTP Fallback & File Ingestion API
|
||||
"CLIENT_TOKEN": "YOUR_CLIENT_JWT_TOKEN", # Provisioned in /vcon/clients
|
||||
"CLIENT_ID": "", # Optional override (defaults to JWT sub/clientId)
|
||||
"AUTO_CONNECT": True,
|
||||
"ENABLE_HTTP_FALLBACK": True,
|
||||
"HEARTBEAT_INTERVAL_SEC": 5.0,
|
||||
|
||||
@@ -5,6 +5,7 @@ Socket.IO ingestion (Port 4000), automatic Heartbeat HTTP fallback (Port 3000),
|
||||
offline disk spooling, and direct `.csv` / `.json` / `.jsonl` file streaming.
|
||||
"""
|
||||
|
||||
from .client import DataStack, DataStackClient
|
||||
from .config import DataStackConfig, load_config
|
||||
from .exceptions import (
|
||||
DataStackAuthError,
|
||||
@@ -16,14 +17,9 @@ from .exceptions import (
|
||||
DataStackPayloadTooLargeError,
|
||||
DataStackRateLimitError,
|
||||
)
|
||||
from .heartbeat import ConnectionState, HeartbeatMonitor
|
||||
from .spool import DiskSpooler
|
||||
|
||||
try:
|
||||
from .client import DataStack, DataStackClient
|
||||
except ImportError:
|
||||
DataStack = None # type: ignore
|
||||
DataStackClient = None # type: ignore
|
||||
|
||||
__version__ = "2.2.0"
|
||||
|
||||
__all__ = [
|
||||
@@ -31,6 +27,8 @@ __all__ = [
|
||||
"DataStackClient",
|
||||
"DataStackConfig",
|
||||
"load_config",
|
||||
"ConnectionState",
|
||||
"HeartbeatMonitor",
|
||||
"DiskSpooler",
|
||||
"DataStackError",
|
||||
"DataStackConfigError",
|
||||
|
||||
@@ -13,6 +13,7 @@ Features:
|
||||
and lossless partial-batch rate-limit retry (`429 RATE_LIMIT_EXCEEDED`).
|
||||
"""
|
||||
|
||||
import base64
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
@@ -44,6 +45,7 @@ from .exceptions import (
|
||||
DataStackConnectionError,
|
||||
DataStackError,
|
||||
DataStackFileError,
|
||||
DataStackOfflineSpoolError,
|
||||
DataStackPayloadTooLargeError,
|
||||
DataStackRateLimitError,
|
||||
)
|
||||
@@ -72,6 +74,23 @@ def _make_deterministic_file_event_id(
|
||||
return f"file_{safe_stem}_{row_index}_{digest}"
|
||||
|
||||
|
||||
def _decode_jwt_client_id(token: str) -> str:
|
||||
"""Safely extracts `clientId` or `sub` claim from an unverified JWT payload string."""
|
||||
try:
|
||||
parts = (token or "").split(".")
|
||||
if len(parts) != 3:
|
||||
return ""
|
||||
payload_b64 = parts[1]
|
||||
padding = "=" * (-len(payload_b64) % 4)
|
||||
raw = base64.urlsafe_b64decode(payload_b64 + padding).decode("utf-8")
|
||||
claims = json.loads(raw)
|
||||
if isinstance(claims, dict):
|
||||
return str(claims.get("clientId") or claims.get("sub") or "").strip()
|
||||
except Exception:
|
||||
pass
|
||||
return ""
|
||||
|
||||
|
||||
class DataStackClient:
|
||||
"""
|
||||
Production-grade DataStack Python Client with `config.datastack.py` auto-loading,
|
||||
@@ -87,6 +106,7 @@ class DataStackClient:
|
||||
auto_connect: Optional[bool] = None,
|
||||
reconnection_attempts: Optional[int] = None,
|
||||
reconnection_delay: Optional[float] = None,
|
||||
client_id: Optional[str] = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
if reconnection_attempts is not None and "MAX_RETRIES" not in kwargs:
|
||||
@@ -105,6 +125,7 @@ class DataStackClient:
|
||||
SERVER_URL=server_url,
|
||||
CLIENT_TOKEN=token,
|
||||
API_URL=api_url,
|
||||
CLIENT_ID=client_id,
|
||||
AUTO_CONNECT=auto_connect,
|
||||
**kwargs,
|
||||
)
|
||||
@@ -112,6 +133,20 @@ class DataStackClient:
|
||||
self.server_url = self.config.server_url
|
||||
self.api_url = self.config.api_url
|
||||
self.token = self.config.client_token
|
||||
|
||||
configured_client_id = (self.config.client_id or "").strip()
|
||||
token_client_id = _decode_jwt_client_id(self.token)
|
||||
if (
|
||||
configured_client_id
|
||||
and token_client_id
|
||||
and configured_client_id != token_client_id
|
||||
):
|
||||
raise DataStackConfigError(
|
||||
f"Configured CLIENT_ID '{configured_client_id}' does not match "
|
||||
f"CLIENT_TOKEN subject '{token_client_id}'."
|
||||
)
|
||||
self.client_id: str = configured_client_id or token_client_id
|
||||
|
||||
self.is_connected = False
|
||||
self._lock = threading.RLock()
|
||||
self._is_draining_spool = False
|
||||
@@ -311,8 +346,11 @@ class DataStackClient:
|
||||
raise DataStackAuthError(msg) from exc
|
||||
return False
|
||||
|
||||
# Verify handshake & PostgreSQL client status via `ping_verify`
|
||||
return self._verify_websocket_handshake(timeout=effective_timeout)
|
||||
# Verify handshake & PostgreSQL client status via `ping_verify` while still serialized under RLock
|
||||
verified = self._verify_websocket_handshake(timeout=effective_timeout)
|
||||
if not verified:
|
||||
self.is_connected = False
|
||||
return verified
|
||||
|
||||
def _verify_websocket_handshake(
|
||||
self, timeout: Optional[float] = None
|
||||
@@ -424,6 +462,8 @@ class DataStackClient:
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
if self.client_id:
|
||||
headers["X-DataStack-Client-Id"] = self.client_id
|
||||
|
||||
if requests is not None:
|
||||
resp = requests.request(
|
||||
@@ -735,10 +775,14 @@ class DataStackClient:
|
||||
):
|
||||
break
|
||||
|
||||
self.spooler.ack_batch(len(batch))
|
||||
if isinstance(unprocessed, list) and len(unprocessed) > 0:
|
||||
self.spooler.enqueue(unprocessed)
|
||||
total_flushed += max(0, len(batch) - len(unprocessed))
|
||||
unprocessed_list = (
|
||||
unprocessed if isinstance(unprocessed, list) else []
|
||||
)
|
||||
self.spooler.ack_and_requeue_front(
|
||||
len(batch), unprocessed_list
|
||||
)
|
||||
if unprocessed_list:
|
||||
total_flushed += max(0, len(batch) - len(unprocessed_list))
|
||||
break
|
||||
|
||||
total_flushed += len(batch)
|
||||
@@ -767,11 +811,13 @@ class DataStackClient:
|
||||
"""
|
||||
resolved_id = (event_id or "").strip() or _make_event_id()
|
||||
resolved_ts = (timestamp or "").strip() or _utc_now_iso()
|
||||
envelope = {
|
||||
envelope: Dict[str, Any] = {
|
||||
"eventId": resolved_id,
|
||||
"timestamp": resolved_ts,
|
||||
"data": data,
|
||||
}
|
||||
if self.client_id:
|
||||
envelope["clientId"] = self.client_id
|
||||
return self._execute_with_retry_and_fallback(
|
||||
ws_event="sync_data",
|
||||
ws_payload=envelope,
|
||||
@@ -810,6 +856,7 @@ class DataStackClient:
|
||||
if not isinstance(source_list, list) or len(source_list) == 0:
|
||||
return {"success": True, "processed": 0, "rejected": 0}
|
||||
|
||||
envelope_keys = {"eventId", "timestamp", "data", "clientId"}
|
||||
formatted: List[Dict[str, Any]] = []
|
||||
for item in source_list:
|
||||
safe_item = item if isinstance(item, dict) else {"value": item}
|
||||
@@ -819,14 +866,21 @@ class DataStackClient:
|
||||
evt_ts = (
|
||||
str(safe_item.get("timestamp") or "").strip() or _utc_now_iso()
|
||||
)
|
||||
evt_data = safe_item.get("data", safe_item)
|
||||
formatted.append(
|
||||
{
|
||||
"eventId": evt_id,
|
||||
"timestamp": evt_ts,
|
||||
"data": evt_data,
|
||||
}
|
||||
is_envelope = "data" in safe_item and set(safe_item.keys()).issubset(
|
||||
envelope_keys
|
||||
)
|
||||
evt_data = safe_item["data"] if is_envelope else safe_item
|
||||
evt_client_id = (
|
||||
str(safe_item.get("clientId") or "").strip() or self.client_id
|
||||
)
|
||||
entry: Dict[str, Any] = {
|
||||
"eventId": evt_id,
|
||||
"timestamp": evt_ts,
|
||||
"data": evt_data,
|
||||
}
|
||||
if evt_client_id:
|
||||
entry["clientId"] = evt_client_id
|
||||
formatted.append(entry)
|
||||
|
||||
return self._send_formatted_batch_recursive(
|
||||
formatted,
|
||||
@@ -1057,13 +1111,14 @@ class DataStackClient:
|
||||
)
|
||||
evt_ts = existing_ts or _utc_now_iso()
|
||||
|
||||
buffer.append(
|
||||
{
|
||||
"eventId": evt_id,
|
||||
"timestamp": evt_ts,
|
||||
"data": enriched,
|
||||
}
|
||||
)
|
||||
file_entry: Dict[str, Any] = {
|
||||
"eventId": evt_id,
|
||||
"timestamp": evt_ts,
|
||||
"data": enriched,
|
||||
}
|
||||
if self.client_id:
|
||||
file_entry["clientId"] = self.client_id
|
||||
buffer.append(file_entry)
|
||||
if len(buffer) >= effective_chunk:
|
||||
res = self.send_batch(buffer, file_name=path.name)
|
||||
total_processed += int(res.get("processed", 0))
|
||||
@@ -1162,6 +1217,7 @@ class DataStackClient:
|
||||
hb = self.heartbeat.status_snapshot()
|
||||
return {
|
||||
"connectedWebSocket": self.is_connected,
|
||||
"clientId": self.client_id or None,
|
||||
"serverUrl": self.server_url,
|
||||
"apiUrl": self.api_url,
|
||||
"configSource": self.config.config_source_path,
|
||||
@@ -1176,3 +1232,21 @@ class DataStackClient:
|
||||
|
||||
# Convenient alias so developers can write `from datastack import DataStack`
|
||||
DataStack = DataStackClient
|
||||
|
||||
__all__ = [
|
||||
"DataStack",
|
||||
"DataStackClient",
|
||||
"DataStackConfig",
|
||||
"load_config",
|
||||
"ConnectionState",
|
||||
"HeartbeatMonitor",
|
||||
"DiskSpooler",
|
||||
"DataStackError",
|
||||
"DataStackConfigError",
|
||||
"DataStackAuthError",
|
||||
"DataStackConnectionError",
|
||||
"DataStackRateLimitError",
|
||||
"DataStackPayloadTooLargeError",
|
||||
"DataStackFileError",
|
||||
"DataStackOfflineSpoolError",
|
||||
]
|
||||
@@ -171,6 +171,11 @@ class HeartbeatMonitor:
|
||||
with self._lock:
|
||||
if self._closed or self._state == ConnectionState.STOPPED:
|
||||
break
|
||||
if (
|
||||
self._state == ConnectionState.CONNECTED_IDLE
|
||||
or not self._wake_event.is_set()
|
||||
):
|
||||
continue
|
||||
self._recovery_attempts += 1
|
||||
|
||||
try:
|
||||
|
||||
@@ -9,7 +9,7 @@ import json
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from .exceptions import DataStackOfflineSpoolError
|
||||
|
||||
@@ -139,6 +139,49 @@ class DiskSpooler:
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
def ack_and_requeue_front(
|
||||
self,
|
||||
acked_count: int,
|
||||
unprocessed: Optional[List[Dict[str, Any]]] = None,
|
||||
) -> int:
|
||||
"""
|
||||
Atomically removes `acked_count` head lines from `spool.jsonl` while
|
||||
preserving any `unprocessed` events at the FRONT of the queue in a single
|
||||
atomic `os.replace()` write (FIFO safe, zero crash-window data loss).
|
||||
"""
|
||||
if not self.enabled or acked_count <= 0:
|
||||
return 0
|
||||
|
||||
with self._lock:
|
||||
if not self.spool_file.exists():
|
||||
return 0
|
||||
|
||||
try:
|
||||
with self.spool_file.open("r", encoding="utf-8") as fh:
|
||||
all_lines = [ln for ln in fh if ln.strip()]
|
||||
|
||||
consumed = min(acked_count, len(all_lines))
|
||||
tail_lines = all_lines[consumed:]
|
||||
head_lines = [
|
||||
json.dumps(evt, ensure_ascii=False) + "\n"
|
||||
for evt in (unprocessed or [])
|
||||
]
|
||||
combined = head_lines + tail_lines
|
||||
|
||||
if not combined:
|
||||
self.spool_file.unlink(missing_ok=True)
|
||||
return consumed
|
||||
|
||||
self._ensure_dir()
|
||||
tmp_file = self.spool_dir / "spool.jsonl.tmp"
|
||||
with tmp_file.open("w", encoding="utf-8") as fh:
|
||||
fh.writelines(combined)
|
||||
fh.flush()
|
||||
os.replace(str(tmp_file), str(self.spool_file))
|
||||
return max(0, consumed - len(head_lines))
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
def pending_count(self) -> int:
|
||||
"""Returns the number of queued events waiting in the disk spool."""
|
||||
if not self.enabled:
|
||||
|
||||
Reference in new issue
Block a user