feat(auth): harden session management and security

- Implement server-side session revocation on logout
- Add IP extraction utility and admin setup error handling
- Improve error codes and response consistency for auth endpoints
- Refine landing page redirect logic to handle transient DB outages
- Update deployment documentation and configuration defaults
This commit is contained in:
Team Dark Devil committed 2026-11-19 20:22:01 +06:00
1 parent 935f12a1e9
commit 89e4d68690
81 files changed
+11530 -6916

No files matched your search

+131 -36
View File
@@ -13,12 +13,15 @@ Features:
and lossless partial-batch rate-limit retry (`429 RATE_LIMIT_EXCEEDED`).
"""
from __future__ import annotations
import base64
import csv
import hashlib
import json
import logging
import random
import re
import threading
import time
import urllib.error
@@ -26,7 +29,7 @@ import urllib.request
import uuid
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional, Union
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
try:
import socketio # type: ignore
@@ -74,6 +77,14 @@ def _make_deterministic_file_event_id(
return f"file_{safe_stem}_{row_index}_{digest}"
def _normalize_client_id(value: Optional[str]) -> str:
"""Matches server SDKManager.normalizeClientId() normalization rules."""
raw = str(value or "").strip().lower()
if not raw:
return ""
return re.sub(r"[^a-z0-9_-]", "_", raw)
def _decode_jwt_client_id(token: str) -> str:
"""Safely extracts `clientId` or `sub` claim from an unverified JWT payload string."""
try:
@@ -136,16 +147,18 @@ class DataStackClient:
configured_client_id = (self.config.client_id or "").strip()
token_client_id = _decode_jwt_client_id(self.token)
norm_configured = _normalize_client_id(configured_client_id)
norm_token = _normalize_client_id(token_client_id)
if (
configured_client_id
and token_client_id
and configured_client_id != token_client_id
norm_configured
and norm_token
and norm_configured != norm_token
):
raise DataStackConfigError(
f"Configured CLIENT_ID '{configured_client_id}' does not match "
f"CLIENT_TOKEN subject '{token_client_id}'."
)
self.client_id: str = configured_client_id or token_client_id
self.client_id: str = norm_token or norm_configured
self.is_connected = False
self._lock = threading.RLock()
@@ -186,6 +199,18 @@ class DataStackClient:
self.connect()
except DataStackAuthError:
raise
except DataStackConnectionError:
if (
not self.config.enable_http_fallback
and not self.config.offline_spool_enabled
):
self.heartbeat.stop(
reason="Initial connect failed with fallback and spool disabled"
)
raise
logger.warning(
"[DataStack] Initial connect entered fallback/recovery mode.",
)
except Exception as exc:
logger.warning(
"[DataStack] Initial connect entered fallback/recovery mode: %s",
@@ -202,6 +227,15 @@ class DataStackClient:
self.connect()
except DataStackAuthError:
raise
except DataStackConnectionError:
if (
not self.config.enable_http_fallback
and not self.config.offline_spool_enabled
):
self.heartbeat.stop(
reason="Context connect failed with fallback and spool disabled"
)
raise
except Exception:
pass
return self
@@ -295,13 +329,16 @@ class DataStackClient:
)
return False
self.heartbeat.stop(
reason="WebSocket and HTTP unreachable with fallback and spool disabled"
)
raise DataStackConnectionError(
f"Unable to connect to DataStack WebSocket ({self.server_url}) or HTTP API ({self.api_url})."
)
def disconnect(self) -> None:
"""Gracefully stops the Heartbeat Monitor and disconnects the WebSocket."""
self.heartbeat.stop(reason="Client closed by application")
self.heartbeat.stop(reason="Client closed by application", join_timeout=0.5)
with self._lock:
self.is_connected = False
if self.sio is not None and self.sio.connected:
@@ -347,9 +384,24 @@ class DataStackClient:
return False
# Verify handshake & PostgreSQL client status via `ping_verify` while still serialized under RLock
verified = self._verify_websocket_handshake(timeout=effective_timeout)
try:
verified = self._verify_websocket_handshake(timeout=effective_timeout)
except Exception:
self.is_connected = False
if self.sio.connected:
try:
self.sio.disconnect()
except Exception:
pass
raise
if not verified:
self.is_connected = False
if self.sio.connected:
try:
self.sio.disconnect()
except Exception:
pass
return verified
def _verify_websocket_handshake(
@@ -455,7 +507,7 @@ class DataStackClient:
url: str,
payload: Optional[Dict[str, Any]] = None,
timeout: Optional[float] = None,
) -> tuple[int, Dict[str, Any]]:
) -> Tuple[int, Dict[str, Any]]:
req_timeout = timeout or self.config.request_timeout_sec
headers = {
"Authorization": f"Bearer {self.token}",
@@ -869,7 +921,16 @@ class DataStackClient:
is_envelope = "data" in safe_item and set(safe_item.keys()).issubset(
envelope_keys
)
evt_data = safe_item["data"] if is_envelope else safe_item
if is_envelope:
evt_data = safe_item["data"]
elif "data" in safe_item:
evt_data = safe_item
else:
evt_data = {
k: v
for k, v in safe_item.items()
if k not in ("eventId", "timestamp", "clientId")
}
evt_client_id = (
str(safe_item.get("clientId") or "").strip() or self.client_id
)
@@ -993,14 +1054,18 @@ class DataStackClient:
if file_name
else {"batch": remaining_items}
)
sub_res = self._execute_with_retry_and_fallback(
ws_event="batch_sync",
ws_payload=remaining_items,
http_payload=sub_http_payload,
spool_items=remaining_items,
timeout=timeout,
allow_spool=allow_spool,
)
try:
sub_res = self._execute_with_retry_and_fallback(
ws_event="batch_sync",
ws_payload=remaining_items,
http_payload=sub_http_payload,
spool_items=remaining_items,
timeout=timeout,
allow_spool=allow_spool,
)
except (DataStackRateLimitError, DataStackConnectionError):
break
total_processed += int(sub_res.get("processed", 0))
total_duplicates += int(sub_res.get("duplicates", 0))
total_spooled += int(sub_res.get("spooledCount", 0))
@@ -1088,6 +1153,20 @@ class DataStackClient:
total_spooled = 0
chunks_sent = 0
last_transport = "WEBSOCKET"
unprocessed_items: List[Dict[str, Any]] = []
def _flush_chunk(chunk_items: List[Dict[str, Any]]) -> None:
nonlocal total_processed, total_rejected, total_duplicates, total_spooled, chunks_sent, last_transport
res = self.send_batch(chunk_items, file_name=path.name)
total_processed += int(res.get("processed", 0))
total_rejected += int(res.get("rejected", 0))
total_duplicates += int(res.get("duplicates", 0))
total_spooled += int(res.get("spooledCount", 0))
last_transport = str(res.get("transport", last_transport))
chunk_unprocessed = res.get("unprocessedItems")
if isinstance(chunk_unprocessed, list) and chunk_unprocessed:
unprocessed_items.extend(chunk_unprocessed)
chunks_sent += 1
buffer: List[Dict[str, Any]] = []
for idx, record in enumerate(self._iter_file_records(path, ext), start=1):
@@ -1111,35 +1190,44 @@ class DataStackClient:
)
evt_ts = existing_ts or _utc_now_iso()
if isinstance(enriched, dict):
is_file_envelope = "data" in enriched and set(
enriched.keys()
).issubset({"eventId", "timestamp", "data", "clientId"})
if is_file_envelope:
file_data: Any = enriched["data"]
elif "data" in enriched:
file_data = enriched
else:
file_data = {
k: v
for k, v in enriched.items()
if k not in ("eventId", "timestamp", "clientId")
}
else:
file_data = enriched
file_entry: Dict[str, Any] = {
"eventId": evt_id,
"timestamp": evt_ts,
"data": enriched,
"data": file_data,
}
if self.client_id:
file_entry["clientId"] = self.client_id
buffer.append(file_entry)
if len(buffer) >= effective_chunk:
res = self.send_batch(buffer, file_name=path.name)
total_processed += int(res.get("processed", 0))
total_rejected += int(res.get("rejected", 0))
total_duplicates += int(res.get("duplicates", 0))
total_spooled += int(res.get("spooledCount", 0))
last_transport = str(res.get("transport", last_transport))
chunks_sent += 1
_flush_chunk(buffer)
buffer = []
if buffer:
res = self.send_batch(buffer, file_name=path.name)
total_processed += int(res.get("processed", 0))
total_rejected += int(res.get("rejected", 0))
total_duplicates += int(res.get("duplicates", 0))
total_spooled += int(res.get("spooledCount", 0))
last_transport = str(res.get("transport", last_transport))
chunks_sent += 1
_flush_chunk(buffer)
return {
"success": True,
summary: Dict[str, Any] = {
"success": (
total_rejected == 0
or total_processed > 0
or total_spooled > 0
),
"fileName": path.name,
"fileType": ext.lstrip("."),
"chunksSent": chunks_sent,
@@ -1149,15 +1237,22 @@ class DataStackClient:
"spooledCount": total_spooled,
"transport": last_transport,
}
if unprocessed_items:
summary["unprocessedItems"] = unprocessed_items
return summary
@staticmethod
def _iter_file_records(path: Path, ext: str) -> Iterable[Dict[str, Any]]:
try:
if ext == ".csv":
with path.open("r", encoding="utf-8-sig", newline="") as fh:
reader = csv.DictReader(fh)
reader = csv.DictReader(fh, restkey="_extra")
for row in reader:
yield dict(row)
clean_row = {
("_extra" if k is None else str(k)): v
for k, v in row.items()
}
yield clean_row
elif ext in (".jsonl", ".ndjson"):
with path.open("r", encoding="utf-8-sig") as fh:
for line_no, line in enumerate(fh, start=1):
+34 -12
View File
@@ -182,6 +182,22 @@ def load_config(
return default
return bool(val)
def to_float(val: Any, name: str) -> float:
try:
return float(val)
except (TypeError, ValueError) as exc:
raise DataStackConfigError(
f"Invalid numeric configuration for '{name}': {val!r}"
) from exc
def to_int(val: Any, name: str) -> int:
try:
return int(val)
except (TypeError, ValueError) as exc:
raise DataStackConfigError(
f"Invalid integer configuration for '{name}': {val!r}"
) from exc
server_url = str(
pick(
["SERVER_URL", "SOCKET_URL", "WS_URL"],
@@ -233,52 +249,57 @@ def load_config(
),
heartbeat_interval_sec=max(
1.0,
float(
to_float(
pick(
["HEARTBEAT_INTERVAL_SEC", "HEARTBEAT_INTERVAL"],
["DATASTACK_HEARTBEAT_INTERVAL_SEC"],
5.0,
)
),
"HEARTBEAT_INTERVAL_SEC",
),
),
request_timeout_sec=max(
1.0,
float(
to_float(
pick(
["REQUEST_TIMEOUT_SEC", "TIMEOUT"],
["DATASTACK_REQUEST_TIMEOUT_SEC"],
8.0,
)
),
"REQUEST_TIMEOUT_SEC",
),
),
max_retries=max(
0,
int(
to_int(
pick(
["MAX_RETRIES", "RETRIES"],
["DATASTACK_MAX_RETRIES"],
3,
)
),
"MAX_RETRIES",
),
),
retry_backoff_base_sec=max(
0.1,
float(
to_float(
pick(
["RETRY_BACKOFF_BASE_SEC"],
["DATASTACK_RETRY_BACKOFF_BASE_SEC"],
0.5,
)
),
"RETRY_BACKOFF_BASE_SEC",
),
),
batch_chunk_size=max(
1,
int(
to_int(
pick(
["BATCH_CHUNK_SIZE", "CHUNK_SIZE"],
["DATASTACK_BATCH_CHUNK_SIZE"],
250,
)
),
"BATCH_CHUNK_SIZE",
),
),
offline_spool_enabled=to_bool(
@@ -298,12 +319,13 @@ def load_config(
),
max_spool_size_mb=max(
1,
int(
to_int(
pick(
["MAX_SPOOL_SIZE_MB"],
["DATASTACK_MAX_SPOOL_SIZE_MB"],
50,
)
),
"MAX_SPOOL_SIZE_MB",
),
),
config_source_path=resolved_source,
+23 -6
View File
@@ -68,7 +68,8 @@ class HeartbeatMonitor:
Monitor into IDLE mode (`_wake_event.clear()`).
"""
with self._lock:
self._closed = False
if self._closed or self._stop_event.is_set():
return
self._state = ConnectionState.CONNECTED_IDLE
self._recovery_attempts = 0
self._last_error = None
@@ -112,7 +113,8 @@ class HeartbeatMonitor:
self._stop_event.clear()
self._ensure_thread_started()
def stop(self, reason: Optional[str] = None) -> None:
def stop(self, reason: Optional[str] = None, join_timeout: float = 0.0) -> None:
thread_to_join: Optional[threading.Thread] = None
with self._lock:
self._closed = True
self._state = ConnectionState.STOPPED
@@ -120,6 +122,14 @@ class HeartbeatMonitor:
self._last_error = reason
self._stop_event.set()
self._wake_event.set()
thread_to_join = self._thread
if (
join_timeout > 0
and thread_to_join is not None
and thread_to_join.is_alive()
and threading.current_thread() is not thread_to_join
):
thread_to_join.join(timeout=join_timeout)
def status_snapshot(self) -> Dict[str, Any]:
with self._lock:
@@ -181,10 +191,17 @@ class HeartbeatMonitor:
try:
recovered_to_idle = self._recovery_callback()
if recovered_to_idle:
self.set_idle_connected()
logger.info(
"[DataStack Heartbeat] WebSocket reconnected & verified. Returning to IDLE mode."
)
with self._lock:
still_active = (
not self._closed
and not self._stop_event.is_set()
and self._state != ConnectionState.STOPPED
)
if still_active:
self.set_idle_connected()
logger.info(
"[DataStack Heartbeat] WebSocket reconnected & verified. Returning to IDLE mode."
)
except Exception as exc:
with self._lock:
self._last_error = str(exc)
Whitespace-only changes.
+15 -4
View File
@@ -6,6 +6,7 @@ batches once the Heartbeat Monitor restores connectivity.
"""
import json
import logging
import os
import threading
from pathlib import Path
@@ -13,6 +14,8 @@ from typing import Any, Dict, List, Optional
from .exceptions import DataStackOfflineSpoolError
logger = logging.getLogger("datastack.spool")
class DiskSpooler:
"""
@@ -121,6 +124,7 @@ class DiskSpooler:
if not self.spool_file.exists():
return 0
tmp_file = self.spool_dir / "spool.jsonl.tmp"
try:
with self.spool_file.open("r", encoding="utf-8") as fh:
all_lines = [ln for ln in fh if ln.strip()]
@@ -130,13 +134,15 @@ class DiskSpooler:
return len(all_lines)
remaining = all_lines[count:]
tmp_file = self.spool_dir / "spool.jsonl.tmp"
with tmp_file.open("w", encoding="utf-8") as fh:
fh.writelines(remaining)
fh.flush()
os.fsync(fh.fileno())
os.replace(str(tmp_file), str(self.spool_file))
return count
except Exception:
except Exception as exc:
tmp_file.unlink(missing_ok=True)
logger.warning("[DataStack Spool] ack_batch failed: %s", exc)
return 0
def ack_and_requeue_front(
@@ -156,6 +162,7 @@ class DiskSpooler:
if not self.spool_file.exists():
return 0
tmp_file = self.spool_dir / "spool.jsonl.tmp"
try:
with self.spool_file.open("r", encoding="utf-8") as fh:
all_lines = [ln for ln in fh if ln.strip()]
@@ -173,13 +180,17 @@ class DiskSpooler:
return consumed
self._ensure_dir()
tmp_file = self.spool_dir / "spool.jsonl.tmp"
with tmp_file.open("w", encoding="utf-8") as fh:
fh.writelines(combined)
fh.flush()
os.fsync(fh.fileno())
os.replace(str(tmp_file), str(self.spool_file))
return max(0, consumed - len(head_lines))
except Exception:
except Exception as exc:
tmp_file.unlink(missing_ok=True)
logger.warning(
"[DataStack Spool] ack_and_requeue_front failed: %s", exc
)
return 0
def pending_count(self) -> int:
+4
View File
@@ -33,6 +33,7 @@ classifiers = [
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Topic :: Software Development :: Libraries :: Python Modules",
"Topic :: System :: Logging",
"Topic :: System :: Monitoring"
@@ -52,3 +53,6 @@ Issues = "https://github.com/anik-d-art/datastack/issues"
[tool.setuptools.packages.find]
where = ["."]
include = ["datastack*"]
[tool.setuptools.package-data]
datastack = ["py.typed"]
+3
View File
@@ -24,6 +24,8 @@ setup(
url="https://vibtools.top/",
license="MIT",
packages=find_packages(include=["datastack", "datastack.*"]),
package_data={"datastack": ["py.typed"]},
include_package_data=True,
install_requires=[
"python-socketio[client]>=5.8.0",
"requests>=2.28.0",
@@ -40,5 +42,6 @@ setup(
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
],
)