refactor(auth): standardize session management and security

- Centralize secret retrieval using `getAdminSessionSecret`
- Replace manual session parsing with `verifyRequestAdminSession` utility
- Tighten API validation and improve security headers handling
- Enforce environment variable security for production deployments
This commit is contained in:
Team Dark Devil committed 2026-11-18 05:18:55 +06:00
1 parent 34519c07df
commit 177ed4a844
78 files changed
+6748 -3157

No files matched your search

+1
View File
@@ -33,6 +33,7 @@ DATASTACK_CONFIG = {
"SERVER_URL": "http://localhost:4000", # Port 4000 Socket.IO Ingestion Service
"API_URL": "http://localhost:3000", # Port 3000 HTTP Fallback & File Ingestion API
"CLIENT_TOKEN": "YOUR_CLIENT_JWT_TOKEN", # Provisioned in /vcon/clients
"CLIENT_ID": "", # Optional override (defaults to JWT sub/clientId)
"AUTO_CONNECT": True,
"ENABLE_HTTP_FALLBACK": True,
"HEARTBEAT_INTERVAL_SEC": 5.0,
+4 -6
View File
@@ -5,6 +5,7 @@ Socket.IO ingestion (Port 4000), automatic Heartbeat HTTP fallback (Port 3000),
offline disk spooling, and direct `.csv` / `.json` / `.jsonl` file streaming.
"""
from .client import DataStack, DataStackClient
from .config import DataStackConfig, load_config
from .exceptions import (
DataStackAuthError,
@@ -16,14 +17,9 @@ from .exceptions import (
DataStackPayloadTooLargeError,
DataStackRateLimitError,
)
from .heartbeat import ConnectionState, HeartbeatMonitor
from .spool import DiskSpooler
try:
from .client import DataStack, DataStackClient
except ImportError:
DataStack = None # type: ignore
DataStackClient = None # type: ignore
__version__ = "2.2.0"
__all__ = [
@@ -31,6 +27,8 @@ __all__ = [
"DataStackClient",
"DataStackConfig",
"load_config",
"ConnectionState",
"HeartbeatMonitor",
"DiskSpooler",
"DataStackError",
"DataStackConfigError",
+95 -21
View File
@@ -13,6 +13,7 @@ Features:
and lossless partial-batch rate-limit retry (`429 RATE_LIMIT_EXCEEDED`).
"""
import base64
import csv
import hashlib
import json
@@ -44,6 +45,7 @@ from .exceptions import (
DataStackConnectionError,
DataStackError,
DataStackFileError,
DataStackOfflineSpoolError,
DataStackPayloadTooLargeError,
DataStackRateLimitError,
)
@@ -72,6 +74,23 @@ def _make_deterministic_file_event_id(
return f"file_{safe_stem}_{row_index}_{digest}"
def _decode_jwt_client_id(token: str) -> str:
"""Safely extracts `clientId` or `sub` claim from an unverified JWT payload string."""
try:
parts = (token or "").split(".")
if len(parts) != 3:
return ""
payload_b64 = parts[1]
padding = "=" * (-len(payload_b64) % 4)
raw = base64.urlsafe_b64decode(payload_b64 + padding).decode("utf-8")
claims = json.loads(raw)
if isinstance(claims, dict):
return str(claims.get("clientId") or claims.get("sub") or "").strip()
except Exception:
pass
return ""
class DataStackClient:
"""
Production-grade DataStack Python Client with `config.datastack.py` auto-loading,
@@ -87,6 +106,7 @@ class DataStackClient:
auto_connect: Optional[bool] = None,
reconnection_attempts: Optional[int] = None,
reconnection_delay: Optional[float] = None,
client_id: Optional[str] = None,
**kwargs: Any,
) -> None:
if reconnection_attempts is not None and "MAX_RETRIES" not in kwargs:
@@ -105,6 +125,7 @@ class DataStackClient:
SERVER_URL=server_url,
CLIENT_TOKEN=token,
API_URL=api_url,
CLIENT_ID=client_id,
AUTO_CONNECT=auto_connect,
**kwargs,
)
@@ -112,6 +133,20 @@ class DataStackClient:
self.server_url = self.config.server_url
self.api_url = self.config.api_url
self.token = self.config.client_token
configured_client_id = (self.config.client_id or "").strip()
token_client_id = _decode_jwt_client_id(self.token)
if (
configured_client_id
and token_client_id
and configured_client_id != token_client_id
):
raise DataStackConfigError(
f"Configured CLIENT_ID '{configured_client_id}' does not match "
f"CLIENT_TOKEN subject '{token_client_id}'."
)
self.client_id: str = configured_client_id or token_client_id
self.is_connected = False
self._lock = threading.RLock()
self._is_draining_spool = False
@@ -311,8 +346,11 @@ class DataStackClient:
raise DataStackAuthError(msg) from exc
return False
# Verify handshake & PostgreSQL client status via `ping_verify`
return self._verify_websocket_handshake(timeout=effective_timeout)
# Verify handshake & PostgreSQL client status via `ping_verify` while still serialized under RLock
verified = self._verify_websocket_handshake(timeout=effective_timeout)
if not verified:
self.is_connected = False
return verified
def _verify_websocket_handshake(
self, timeout: Optional[float] = None
@@ -424,6 +462,8 @@ class DataStackClient:
"Content-Type": "application/json",
"Accept": "application/json",
}
if self.client_id:
headers["X-DataStack-Client-Id"] = self.client_id
if requests is not None:
resp = requests.request(
@@ -735,10 +775,14 @@ class DataStackClient:
):
break
self.spooler.ack_batch(len(batch))
if isinstance(unprocessed, list) and len(unprocessed) > 0:
self.spooler.enqueue(unprocessed)
total_flushed += max(0, len(batch) - len(unprocessed))
unprocessed_list = (
unprocessed if isinstance(unprocessed, list) else []
)
self.spooler.ack_and_requeue_front(
len(batch), unprocessed_list
)
if unprocessed_list:
total_flushed += max(0, len(batch) - len(unprocessed_list))
break
total_flushed += len(batch)
@@ -767,11 +811,13 @@ class DataStackClient:
"""
resolved_id = (event_id or "").strip() or _make_event_id()
resolved_ts = (timestamp or "").strip() or _utc_now_iso()
envelope = {
envelope: Dict[str, Any] = {
"eventId": resolved_id,
"timestamp": resolved_ts,
"data": data,
}
if self.client_id:
envelope["clientId"] = self.client_id
return self._execute_with_retry_and_fallback(
ws_event="sync_data",
ws_payload=envelope,
@@ -810,6 +856,7 @@ class DataStackClient:
if not isinstance(source_list, list) or len(source_list) == 0:
return {"success": True, "processed": 0, "rejected": 0}
envelope_keys = {"eventId", "timestamp", "data", "clientId"}
formatted: List[Dict[str, Any]] = []
for item in source_list:
safe_item = item if isinstance(item, dict) else {"value": item}
@@ -819,14 +866,21 @@ class DataStackClient:
evt_ts = (
str(safe_item.get("timestamp") or "").strip() or _utc_now_iso()
)
evt_data = safe_item.get("data", safe_item)
formatted.append(
{
"eventId": evt_id,
"timestamp": evt_ts,
"data": evt_data,
}
is_envelope = "data" in safe_item and set(safe_item.keys()).issubset(
envelope_keys
)
evt_data = safe_item["data"] if is_envelope else safe_item
evt_client_id = (
str(safe_item.get("clientId") or "").strip() or self.client_id
)
entry: Dict[str, Any] = {
"eventId": evt_id,
"timestamp": evt_ts,
"data": evt_data,
}
if evt_client_id:
entry["clientId"] = evt_client_id
formatted.append(entry)
return self._send_formatted_batch_recursive(
formatted,
@@ -1057,13 +1111,14 @@ class DataStackClient:
)
evt_ts = existing_ts or _utc_now_iso()
buffer.append(
{
"eventId": evt_id,
"timestamp": evt_ts,
"data": enriched,
}
)
file_entry: Dict[str, Any] = {
"eventId": evt_id,
"timestamp": evt_ts,
"data": enriched,
}
if self.client_id:
file_entry["clientId"] = self.client_id
buffer.append(file_entry)
if len(buffer) >= effective_chunk:
res = self.send_batch(buffer, file_name=path.name)
total_processed += int(res.get("processed", 0))
@@ -1162,6 +1217,7 @@ class DataStackClient:
hb = self.heartbeat.status_snapshot()
return {
"connectedWebSocket": self.is_connected,
"clientId": self.client_id or None,
"serverUrl": self.server_url,
"apiUrl": self.api_url,
"configSource": self.config.config_source_path,
@@ -1176,3 +1232,21 @@ class DataStackClient:
# Convenient alias so developers can write `from datastack import DataStack`
DataStack = DataStackClient
__all__ = [
"DataStack",
"DataStackClient",
"DataStackConfig",
"load_config",
"ConnectionState",
"HeartbeatMonitor",
"DiskSpooler",
"DataStackError",
"DataStackConfigError",
"DataStackAuthError",
"DataStackConnectionError",
"DataStackRateLimitError",
"DataStackPayloadTooLargeError",
"DataStackFileError",
"DataStackOfflineSpoolError",
]
@@ -171,6 +171,11 @@ class HeartbeatMonitor:
with self._lock:
if self._closed or self._state == ConnectionState.STOPPED:
break
if (
self._state == ConnectionState.CONNECTED_IDLE
or not self._wake_event.is_set()
):
continue
self._recovery_attempts += 1
try:
+44 -1
View File
@@ -9,7 +9,7 @@ import json
import os
import threading
from pathlib import Path
from typing import Any, Dict, List
from typing import Any, Dict, List, Optional
from .exceptions import DataStackOfflineSpoolError
@@ -139,6 +139,49 @@ class DiskSpooler:
except Exception:
return 0
def ack_and_requeue_front(
self,
acked_count: int,
unprocessed: Optional[List[Dict[str, Any]]] = None,
) -> int:
"""
Atomically removes `acked_count` head lines from `spool.jsonl` while
preserving any `unprocessed` events at the FRONT of the queue in a single
atomic `os.replace()` write (FIFO safe, zero crash-window data loss).
"""
if not self.enabled or acked_count <= 0:
return 0
with self._lock:
if not self.spool_file.exists():
return 0
try:
with self.spool_file.open("r", encoding="utf-8") as fh:
all_lines = [ln for ln in fh if ln.strip()]
consumed = min(acked_count, len(all_lines))
tail_lines = all_lines[consumed:]
head_lines = [
json.dumps(evt, ensure_ascii=False) + "\n"
for evt in (unprocessed or [])
]
combined = head_lines + tail_lines
if not combined:
self.spool_file.unlink(missing_ok=True)
return consumed
self._ensure_dir()
tmp_file = self.spool_dir / "spool.jsonl.tmp"
with tmp_file.open("w", encoding="utf-8") as fh:
fh.writelines(combined)
fh.flush()
os.replace(str(tmp_file), str(self.spool_file))
return max(0, consumed - len(head_lines))
except Exception:
return 0
def pending_count(self) -> int:
"""Returns the number of queued events waiting in the disk spool."""
if not self.enabled: