feat(auth): harden session management and security
- Implement server-side session revocation on logout - Add IP extraction utility and admin setup error handling - Improve error codes and response consistency for auth endpoints - Refine landing page redirect logic to handle transient DB outages - Update deployment documentation and configuration defaults
This commit is contained in:
1 parent
935f12a1e9
commit
89e4d68690
81 files changed
+11530
-6916
No files matched your search
@@ -13,12 +13,15 @@ Features:
|
||||
and lossless partial-batch rate-limit retry (`429 RATE_LIMIT_EXCEEDED`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
@@ -26,7 +29,7 @@ import urllib.request
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Optional, Union
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
|
||||
|
||||
try:
|
||||
import socketio # type: ignore
|
||||
@@ -74,6 +77,14 @@ def _make_deterministic_file_event_id(
|
||||
return f"file_{safe_stem}_{row_index}_{digest}"
|
||||
|
||||
|
||||
def _normalize_client_id(value: Optional[str]) -> str:
|
||||
"""Matches server SDKManager.normalizeClientId() normalization rules."""
|
||||
raw = str(value or "").strip().lower()
|
||||
if not raw:
|
||||
return ""
|
||||
return re.sub(r"[^a-z0-9_-]", "_", raw)
|
||||
|
||||
|
||||
def _decode_jwt_client_id(token: str) -> str:
|
||||
"""Safely extracts `clientId` or `sub` claim from an unverified JWT payload string."""
|
||||
try:
|
||||
@@ -136,16 +147,18 @@ class DataStackClient:
|
||||
|
||||
configured_client_id = (self.config.client_id or "").strip()
|
||||
token_client_id = _decode_jwt_client_id(self.token)
|
||||
norm_configured = _normalize_client_id(configured_client_id)
|
||||
norm_token = _normalize_client_id(token_client_id)
|
||||
if (
|
||||
configured_client_id
|
||||
and token_client_id
|
||||
and configured_client_id != token_client_id
|
||||
norm_configured
|
||||
and norm_token
|
||||
and norm_configured != norm_token
|
||||
):
|
||||
raise DataStackConfigError(
|
||||
f"Configured CLIENT_ID '{configured_client_id}' does not match "
|
||||
f"CLIENT_TOKEN subject '{token_client_id}'."
|
||||
)
|
||||
self.client_id: str = configured_client_id or token_client_id
|
||||
self.client_id: str = norm_token or norm_configured
|
||||
|
||||
self.is_connected = False
|
||||
self._lock = threading.RLock()
|
||||
@@ -186,6 +199,18 @@ class DataStackClient:
|
||||
self.connect()
|
||||
except DataStackAuthError:
|
||||
raise
|
||||
except DataStackConnectionError:
|
||||
if (
|
||||
not self.config.enable_http_fallback
|
||||
and not self.config.offline_spool_enabled
|
||||
):
|
||||
self.heartbeat.stop(
|
||||
reason="Initial connect failed with fallback and spool disabled"
|
||||
)
|
||||
raise
|
||||
logger.warning(
|
||||
"[DataStack] Initial connect entered fallback/recovery mode.",
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"[DataStack] Initial connect entered fallback/recovery mode: %s",
|
||||
@@ -202,6 +227,15 @@ class DataStackClient:
|
||||
self.connect()
|
||||
except DataStackAuthError:
|
||||
raise
|
||||
except DataStackConnectionError:
|
||||
if (
|
||||
not self.config.enable_http_fallback
|
||||
and not self.config.offline_spool_enabled
|
||||
):
|
||||
self.heartbeat.stop(
|
||||
reason="Context connect failed with fallback and spool disabled"
|
||||
)
|
||||
raise
|
||||
except Exception:
|
||||
pass
|
||||
return self
|
||||
@@ -295,13 +329,16 @@ class DataStackClient:
|
||||
)
|
||||
return False
|
||||
|
||||
self.heartbeat.stop(
|
||||
reason="WebSocket and HTTP unreachable with fallback and spool disabled"
|
||||
)
|
||||
raise DataStackConnectionError(
|
||||
f"Unable to connect to DataStack WebSocket ({self.server_url}) or HTTP API ({self.api_url})."
|
||||
)
|
||||
|
||||
def disconnect(self) -> None:
|
||||
"""Gracefully stops the Heartbeat Monitor and disconnects the WebSocket."""
|
||||
self.heartbeat.stop(reason="Client closed by application")
|
||||
self.heartbeat.stop(reason="Client closed by application", join_timeout=0.5)
|
||||
with self._lock:
|
||||
self.is_connected = False
|
||||
if self.sio is not None and self.sio.connected:
|
||||
@@ -347,9 +384,24 @@ class DataStackClient:
|
||||
return False
|
||||
|
||||
# Verify handshake & PostgreSQL client status via `ping_verify` while still serialized under RLock
|
||||
verified = self._verify_websocket_handshake(timeout=effective_timeout)
|
||||
try:
|
||||
verified = self._verify_websocket_handshake(timeout=effective_timeout)
|
||||
except Exception:
|
||||
self.is_connected = False
|
||||
if self.sio.connected:
|
||||
try:
|
||||
self.sio.disconnect()
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
|
||||
if not verified:
|
||||
self.is_connected = False
|
||||
if self.sio.connected:
|
||||
try:
|
||||
self.sio.disconnect()
|
||||
except Exception:
|
||||
pass
|
||||
return verified
|
||||
|
||||
def _verify_websocket_handshake(
|
||||
@@ -455,7 +507,7 @@ class DataStackClient:
|
||||
url: str,
|
||||
payload: Optional[Dict[str, Any]] = None,
|
||||
timeout: Optional[float] = None,
|
||||
) -> tuple[int, Dict[str, Any]]:
|
||||
) -> Tuple[int, Dict[str, Any]]:
|
||||
req_timeout = timeout or self.config.request_timeout_sec
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.token}",
|
||||
@@ -869,7 +921,16 @@ class DataStackClient:
|
||||
is_envelope = "data" in safe_item and set(safe_item.keys()).issubset(
|
||||
envelope_keys
|
||||
)
|
||||
evt_data = safe_item["data"] if is_envelope else safe_item
|
||||
if is_envelope:
|
||||
evt_data = safe_item["data"]
|
||||
elif "data" in safe_item:
|
||||
evt_data = safe_item
|
||||
else:
|
||||
evt_data = {
|
||||
k: v
|
||||
for k, v in safe_item.items()
|
||||
if k not in ("eventId", "timestamp", "clientId")
|
||||
}
|
||||
evt_client_id = (
|
||||
str(safe_item.get("clientId") or "").strip() or self.client_id
|
||||
)
|
||||
@@ -993,14 +1054,18 @@ class DataStackClient:
|
||||
if file_name
|
||||
else {"batch": remaining_items}
|
||||
)
|
||||
sub_res = self._execute_with_retry_and_fallback(
|
||||
ws_event="batch_sync",
|
||||
ws_payload=remaining_items,
|
||||
http_payload=sub_http_payload,
|
||||
spool_items=remaining_items,
|
||||
timeout=timeout,
|
||||
allow_spool=allow_spool,
|
||||
)
|
||||
try:
|
||||
sub_res = self._execute_with_retry_and_fallback(
|
||||
ws_event="batch_sync",
|
||||
ws_payload=remaining_items,
|
||||
http_payload=sub_http_payload,
|
||||
spool_items=remaining_items,
|
||||
timeout=timeout,
|
||||
allow_spool=allow_spool,
|
||||
)
|
||||
except (DataStackRateLimitError, DataStackConnectionError):
|
||||
break
|
||||
|
||||
total_processed += int(sub_res.get("processed", 0))
|
||||
total_duplicates += int(sub_res.get("duplicates", 0))
|
||||
total_spooled += int(sub_res.get("spooledCount", 0))
|
||||
@@ -1088,6 +1153,20 @@ class DataStackClient:
|
||||
total_spooled = 0
|
||||
chunks_sent = 0
|
||||
last_transport = "WEBSOCKET"
|
||||
unprocessed_items: List[Dict[str, Any]] = []
|
||||
|
||||
def _flush_chunk(chunk_items: List[Dict[str, Any]]) -> None:
|
||||
nonlocal total_processed, total_rejected, total_duplicates, total_spooled, chunks_sent, last_transport
|
||||
res = self.send_batch(chunk_items, file_name=path.name)
|
||||
total_processed += int(res.get("processed", 0))
|
||||
total_rejected += int(res.get("rejected", 0))
|
||||
total_duplicates += int(res.get("duplicates", 0))
|
||||
total_spooled += int(res.get("spooledCount", 0))
|
||||
last_transport = str(res.get("transport", last_transport))
|
||||
chunk_unprocessed = res.get("unprocessedItems")
|
||||
if isinstance(chunk_unprocessed, list) and chunk_unprocessed:
|
||||
unprocessed_items.extend(chunk_unprocessed)
|
||||
chunks_sent += 1
|
||||
|
||||
buffer: List[Dict[str, Any]] = []
|
||||
for idx, record in enumerate(self._iter_file_records(path, ext), start=1):
|
||||
@@ -1111,35 +1190,44 @@ class DataStackClient:
|
||||
)
|
||||
evt_ts = existing_ts or _utc_now_iso()
|
||||
|
||||
if isinstance(enriched, dict):
|
||||
is_file_envelope = "data" in enriched and set(
|
||||
enriched.keys()
|
||||
).issubset({"eventId", "timestamp", "data", "clientId"})
|
||||
if is_file_envelope:
|
||||
file_data: Any = enriched["data"]
|
||||
elif "data" in enriched:
|
||||
file_data = enriched
|
||||
else:
|
||||
file_data = {
|
||||
k: v
|
||||
for k, v in enriched.items()
|
||||
if k not in ("eventId", "timestamp", "clientId")
|
||||
}
|
||||
else:
|
||||
file_data = enriched
|
||||
|
||||
file_entry: Dict[str, Any] = {
|
||||
"eventId": evt_id,
|
||||
"timestamp": evt_ts,
|
||||
"data": enriched,
|
||||
"data": file_data,
|
||||
}
|
||||
if self.client_id:
|
||||
file_entry["clientId"] = self.client_id
|
||||
buffer.append(file_entry)
|
||||
if len(buffer) >= effective_chunk:
|
||||
res = self.send_batch(buffer, file_name=path.name)
|
||||
total_processed += int(res.get("processed", 0))
|
||||
total_rejected += int(res.get("rejected", 0))
|
||||
total_duplicates += int(res.get("duplicates", 0))
|
||||
total_spooled += int(res.get("spooledCount", 0))
|
||||
last_transport = str(res.get("transport", last_transport))
|
||||
chunks_sent += 1
|
||||
_flush_chunk(buffer)
|
||||
buffer = []
|
||||
|
||||
if buffer:
|
||||
res = self.send_batch(buffer, file_name=path.name)
|
||||
total_processed += int(res.get("processed", 0))
|
||||
total_rejected += int(res.get("rejected", 0))
|
||||
total_duplicates += int(res.get("duplicates", 0))
|
||||
total_spooled += int(res.get("spooledCount", 0))
|
||||
last_transport = str(res.get("transport", last_transport))
|
||||
chunks_sent += 1
|
||||
_flush_chunk(buffer)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
summary: Dict[str, Any] = {
|
||||
"success": (
|
||||
total_rejected == 0
|
||||
or total_processed > 0
|
||||
or total_spooled > 0
|
||||
),
|
||||
"fileName": path.name,
|
||||
"fileType": ext.lstrip("."),
|
||||
"chunksSent": chunks_sent,
|
||||
@@ -1149,15 +1237,22 @@ class DataStackClient:
|
||||
"spooledCount": total_spooled,
|
||||
"transport": last_transport,
|
||||
}
|
||||
if unprocessed_items:
|
||||
summary["unprocessedItems"] = unprocessed_items
|
||||
return summary
|
||||
|
||||
@staticmethod
|
||||
def _iter_file_records(path: Path, ext: str) -> Iterable[Dict[str, Any]]:
|
||||
try:
|
||||
if ext == ".csv":
|
||||
with path.open("r", encoding="utf-8-sig", newline="") as fh:
|
||||
reader = csv.DictReader(fh)
|
||||
reader = csv.DictReader(fh, restkey="_extra")
|
||||
for row in reader:
|
||||
yield dict(row)
|
||||
clean_row = {
|
||||
("_extra" if k is None else str(k)): v
|
||||
for k, v in row.items()
|
||||
}
|
||||
yield clean_row
|
||||
elif ext in (".jsonl", ".ndjson"):
|
||||
with path.open("r", encoding="utf-8-sig") as fh:
|
||||
for line_no, line in enumerate(fh, start=1):
|
||||
|
||||
@@ -182,6 +182,22 @@ def load_config(
|
||||
return default
|
||||
return bool(val)
|
||||
|
||||
def to_float(val: Any, name: str) -> float:
|
||||
try:
|
||||
return float(val)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise DataStackConfigError(
|
||||
f"Invalid numeric configuration for '{name}': {val!r}"
|
||||
) from exc
|
||||
|
||||
def to_int(val: Any, name: str) -> int:
|
||||
try:
|
||||
return int(val)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise DataStackConfigError(
|
||||
f"Invalid integer configuration for '{name}': {val!r}"
|
||||
) from exc
|
||||
|
||||
server_url = str(
|
||||
pick(
|
||||
["SERVER_URL", "SOCKET_URL", "WS_URL"],
|
||||
@@ -233,52 +249,57 @@ def load_config(
|
||||
),
|
||||
heartbeat_interval_sec=max(
|
||||
1.0,
|
||||
float(
|
||||
to_float(
|
||||
pick(
|
||||
["HEARTBEAT_INTERVAL_SEC", "HEARTBEAT_INTERVAL"],
|
||||
["DATASTACK_HEARTBEAT_INTERVAL_SEC"],
|
||||
5.0,
|
||||
)
|
||||
),
|
||||
"HEARTBEAT_INTERVAL_SEC",
|
||||
),
|
||||
),
|
||||
request_timeout_sec=max(
|
||||
1.0,
|
||||
float(
|
||||
to_float(
|
||||
pick(
|
||||
["REQUEST_TIMEOUT_SEC", "TIMEOUT"],
|
||||
["DATASTACK_REQUEST_TIMEOUT_SEC"],
|
||||
8.0,
|
||||
)
|
||||
),
|
||||
"REQUEST_TIMEOUT_SEC",
|
||||
),
|
||||
),
|
||||
max_retries=max(
|
||||
0,
|
||||
int(
|
||||
to_int(
|
||||
pick(
|
||||
["MAX_RETRIES", "RETRIES"],
|
||||
["DATASTACK_MAX_RETRIES"],
|
||||
3,
|
||||
)
|
||||
),
|
||||
"MAX_RETRIES",
|
||||
),
|
||||
),
|
||||
retry_backoff_base_sec=max(
|
||||
0.1,
|
||||
float(
|
||||
to_float(
|
||||
pick(
|
||||
["RETRY_BACKOFF_BASE_SEC"],
|
||||
["DATASTACK_RETRY_BACKOFF_BASE_SEC"],
|
||||
0.5,
|
||||
)
|
||||
),
|
||||
"RETRY_BACKOFF_BASE_SEC",
|
||||
),
|
||||
),
|
||||
batch_chunk_size=max(
|
||||
1,
|
||||
int(
|
||||
to_int(
|
||||
pick(
|
||||
["BATCH_CHUNK_SIZE", "CHUNK_SIZE"],
|
||||
["DATASTACK_BATCH_CHUNK_SIZE"],
|
||||
250,
|
||||
)
|
||||
),
|
||||
"BATCH_CHUNK_SIZE",
|
||||
),
|
||||
),
|
||||
offline_spool_enabled=to_bool(
|
||||
@@ -298,12 +319,13 @@ def load_config(
|
||||
),
|
||||
max_spool_size_mb=max(
|
||||
1,
|
||||
int(
|
||||
to_int(
|
||||
pick(
|
||||
["MAX_SPOOL_SIZE_MB"],
|
||||
["DATASTACK_MAX_SPOOL_SIZE_MB"],
|
||||
50,
|
||||
)
|
||||
),
|
||||
"MAX_SPOOL_SIZE_MB",
|
||||
),
|
||||
),
|
||||
config_source_path=resolved_source,
|
||||
|
||||
@@ -68,7 +68,8 @@ class HeartbeatMonitor:
|
||||
Monitor into IDLE mode (`_wake_event.clear()`).
|
||||
"""
|
||||
with self._lock:
|
||||
self._closed = False
|
||||
if self._closed or self._stop_event.is_set():
|
||||
return
|
||||
self._state = ConnectionState.CONNECTED_IDLE
|
||||
self._recovery_attempts = 0
|
||||
self._last_error = None
|
||||
@@ -112,7 +113,8 @@ class HeartbeatMonitor:
|
||||
self._stop_event.clear()
|
||||
self._ensure_thread_started()
|
||||
|
||||
def stop(self, reason: Optional[str] = None) -> None:
|
||||
def stop(self, reason: Optional[str] = None, join_timeout: float = 0.0) -> None:
|
||||
thread_to_join: Optional[threading.Thread] = None
|
||||
with self._lock:
|
||||
self._closed = True
|
||||
self._state = ConnectionState.STOPPED
|
||||
@@ -120,6 +122,14 @@ class HeartbeatMonitor:
|
||||
self._last_error = reason
|
||||
self._stop_event.set()
|
||||
self._wake_event.set()
|
||||
thread_to_join = self._thread
|
||||
if (
|
||||
join_timeout > 0
|
||||
and thread_to_join is not None
|
||||
and thread_to_join.is_alive()
|
||||
and threading.current_thread() is not thread_to_join
|
||||
):
|
||||
thread_to_join.join(timeout=join_timeout)
|
||||
|
||||
def status_snapshot(self) -> Dict[str, Any]:
|
||||
with self._lock:
|
||||
@@ -181,10 +191,17 @@ class HeartbeatMonitor:
|
||||
try:
|
||||
recovered_to_idle = self._recovery_callback()
|
||||
if recovered_to_idle:
|
||||
self.set_idle_connected()
|
||||
logger.info(
|
||||
"[DataStack Heartbeat] WebSocket reconnected & verified. Returning to IDLE mode."
|
||||
)
|
||||
with self._lock:
|
||||
still_active = (
|
||||
not self._closed
|
||||
and not self._stop_event.is_set()
|
||||
and self._state != ConnectionState.STOPPED
|
||||
)
|
||||
if still_active:
|
||||
self.set_idle_connected()
|
||||
logger.info(
|
||||
"[DataStack Heartbeat] WebSocket reconnected & verified. Returning to IDLE mode."
|
||||
)
|
||||
except Exception as exc:
|
||||
with self._lock:
|
||||
self._last_error = str(exc)
|
||||
|
||||
Whitespace-only changes.
@@ -6,6 +6,7 @@ batches once the Heartbeat Monitor restores connectivity.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from pathlib import Path
|
||||
@@ -13,6 +14,8 @@ from typing import Any, Dict, List, Optional
|
||||
|
||||
from .exceptions import DataStackOfflineSpoolError
|
||||
|
||||
logger = logging.getLogger("datastack.spool")
|
||||
|
||||
|
||||
class DiskSpooler:
|
||||
"""
|
||||
@@ -121,6 +124,7 @@ class DiskSpooler:
|
||||
if not self.spool_file.exists():
|
||||
return 0
|
||||
|
||||
tmp_file = self.spool_dir / "spool.jsonl.tmp"
|
||||
try:
|
||||
with self.spool_file.open("r", encoding="utf-8") as fh:
|
||||
all_lines = [ln for ln in fh if ln.strip()]
|
||||
@@ -130,13 +134,15 @@ class DiskSpooler:
|
||||
return len(all_lines)
|
||||
|
||||
remaining = all_lines[count:]
|
||||
tmp_file = self.spool_dir / "spool.jsonl.tmp"
|
||||
with tmp_file.open("w", encoding="utf-8") as fh:
|
||||
fh.writelines(remaining)
|
||||
fh.flush()
|
||||
os.fsync(fh.fileno())
|
||||
os.replace(str(tmp_file), str(self.spool_file))
|
||||
return count
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
tmp_file.unlink(missing_ok=True)
|
||||
logger.warning("[DataStack Spool] ack_batch failed: %s", exc)
|
||||
return 0
|
||||
|
||||
def ack_and_requeue_front(
|
||||
@@ -156,6 +162,7 @@ class DiskSpooler:
|
||||
if not self.spool_file.exists():
|
||||
return 0
|
||||
|
||||
tmp_file = self.spool_dir / "spool.jsonl.tmp"
|
||||
try:
|
||||
with self.spool_file.open("r", encoding="utf-8") as fh:
|
||||
all_lines = [ln for ln in fh if ln.strip()]
|
||||
@@ -173,13 +180,17 @@ class DiskSpooler:
|
||||
return consumed
|
||||
|
||||
self._ensure_dir()
|
||||
tmp_file = self.spool_dir / "spool.jsonl.tmp"
|
||||
with tmp_file.open("w", encoding="utf-8") as fh:
|
||||
fh.writelines(combined)
|
||||
fh.flush()
|
||||
os.fsync(fh.fileno())
|
||||
os.replace(str(tmp_file), str(self.spool_file))
|
||||
return max(0, consumed - len(head_lines))
|
||||
except Exception:
|
||||
except Exception as exc:
|
||||
tmp_file.unlink(missing_ok=True)
|
||||
logger.warning(
|
||||
"[DataStack Spool] ack_and_requeue_front failed: %s", exc
|
||||
)
|
||||
return 0
|
||||
|
||||
def pending_count(self) -> int:
|
||||
|
||||
@@ -33,6 +33,7 @@ classifiers = [
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
"Topic :: Software Development :: Libraries :: Python Modules",
|
||||
"Topic :: System :: Logging",
|
||||
"Topic :: System :: Monitoring"
|
||||
@@ -52,3 +53,6 @@ Issues = "https://github.com/anik-d-art/datastack/issues"
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["."]
|
||||
include = ["datastack*"]
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
datastack = ["py.typed"]
|
||||
@@ -24,6 +24,8 @@ setup(
|
||||
url="https://vibtools.top/",
|
||||
license="MIT",
|
||||
packages=find_packages(include=["datastack", "datastack.*"]),
|
||||
package_data={"datastack": ["py.typed"]},
|
||||
include_package_data=True,
|
||||
install_requires=[
|
||||
"python-socketio[client]>=5.8.0",
|
||||
"requests>=2.28.0",
|
||||
@@ -40,5 +42,6 @@ setup(
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
],
|
||||
)
|
||||
Reference in new issue
Block a user