Files
DataStack/tests/phase05_sdk_test.py
T
Team Dark Devil 89e4d68690 feat(auth): harden session management and security
- Implement server-side session revocation on logout
- Add IP extraction utility and admin setup error handling
- Improve error codes and response consistency for auth endpoints
- Refine landing page redirect logic to handle transient DB outages
- Update deployment documentation and configuration defaults
2026-11-19 20:22:01 +06:00

608 lines
25 KiB
Python

"""
Phase-05 Verification Tests (BUG-18):
- Single Source of Truth between canonical `datastack` (`sdk/python/datastack`)
and compatibility alias `datastack_sdk` (`sdk-python`).
- `CLIENT_ID` config loading (`config.datastack.py`, `DATASTACK_CLIENT_ID`, kwarg),
JWT `clientId`/`sub` fallback extraction, and mismatch validation.
- End-to-end propagation of `clientId` & `X-DataStack-Client-Id` into HTTP
fallback ingestion (`send`, `send_batch`, `send_file`), status snapshot, and
offline disk spool replay against a local HTTP test server.
"""
import ast
import base64
import json
import os
import sys
import tempfile
import threading
import unittest
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import Any, Dict, List
REPO_ROOT = Path(__file__).resolve().parents[1]
CANONICAL_PKG_ROOT = REPO_ROOT / "sdk" / "python" / "datastack"
COMPAT_PKG_ROOT = REPO_ROOT / "sdk-python"
for p in (str(CANONICAL_PKG_ROOT), str(COMPAT_PKG_ROOT)):
if p not in sys.path:
sys.path.insert(0, p)
import datastack # noqa: E402
import datastack_sdk # noqa: E402
from datastack import ( # noqa: E402
DataStack,
DataStackAuthError,
DataStackClient,
DataStackConfig,
DataStackConfigError,
DataStackConnectionError,
DataStackOfflineSpoolError,
DiskSpooler,
HeartbeatMonitor,
load_config,
)
def _make_jwt(payload: Dict[str, Any]) -> str:
header = (
base64.urlsafe_b64encode(json.dumps({"alg": "HS256", "typ": "JWT"}).encode())
.decode()
.rstrip("=")
)
body = (
base64.urlsafe_b64encode(json.dumps(payload).encode())
.decode()
.rstrip("=")
)
return f"{header}.{body}.testsig"
class Phase05PythonSdkTests(unittest.TestCase):
def test_single_source_of_truth_identity(self) -> None:
self.assertIs(datastack_sdk.DataStack, datastack.DataStack)
self.assertIs(datastack_sdk.DataStackClient, datastack.DataStackClient)
self.assertIs(datastack_sdk.DataStackConfig, datastack.DataStackConfig)
self.assertIs(datastack_sdk.load_config, datastack.load_config)
self.assertIs(
datastack_sdk.DataStackConfigError, datastack.DataStackConfigError
)
self.assertIs(
datastack_sdk.DataStackOfflineSpoolError,
datastack.DataStackOfflineSpoolError,
)
self.assertIs(datastack_sdk.DiskSpooler, datastack.DiskSpooler)
self.assertIs(datastack_sdk.HeartbeatMonitor, datastack.HeartbeatMonitor)
self.assertEqual(datastack_sdk.__version__, datastack.__version__)
# Submodule shims also resolve to canonical classes
from datastack_sdk.client import DataStackClient as ShimClient
from datastack_sdk.config import load_config as shim_load_config
from datastack_sdk.exceptions import (
DataStackOfflineSpoolError as ShimSpoolErr,
)
from datastack_sdk.heartbeat import HeartbeatMonitor as ShimHeartbeat
from datastack_sdk.spool import DiskSpooler as ShimSpooler
self.assertIs(ShimClient, DataStackClient)
self.assertIs(shim_load_config, load_config)
self.assertIs(ShimSpoolErr, DataStackOfflineSpoolError)
self.assertIs(ShimHeartbeat, HeartbeatMonitor)
self.assertIs(ShimSpooler, DiskSpooler)
def test_client_id_config_env_and_jwt_validation(self) -> None:
tok_alpha = _make_jwt({"clientId": "alpha_svc", "sub": "alpha_svc"})
with tempfile.TemporaryDirectory() as tmpdir:
cfg_file = Path(tmpdir) / "config.datastack.py"
cfg_file.write_text(
f'DATASTACK_CONFIG = {{\n'
f' "SERVER_URL": "http://127.0.0.1:49999",\n'
f' "API_URL": "http://127.0.0.1:39999",\n'
f' "CLIENT_TOKEN": "{tok_alpha}",\n'
f' "CLIENT_ID": "alpha_svc",\n'
f' "AUTO_CONNECT": False,\n'
f' "OFFLINE_SPOOL_DIR": "{Path(tmpdir) / "spool"}",\n'
f'}}\n',
encoding="utf-8",
)
cfg = load_config(config_path=str(cfg_file))
self.assertIsInstance(cfg, DataStackConfig)
self.assertEqual(cfg.client_id, "alpha_svc")
# Matching explicit client_id succeeds
client = DataStack(config_path=str(cfg_file), auto_connect=False)
self.assertEqual(client.client_id, "alpha_svc")
self.assertEqual(client.status()["clientId"], "alpha_svc")
client.disconnect()
# Env override matching JWT succeeds; mismatching raises DataStackConfigError
old_env = os.environ.get("DATASTACK_CLIENT_ID")
try:
os.environ["DATASTACK_CLIENT_ID"] = "wrong_svc"
with self.assertRaises(DataStackConfigError):
DataStack(config_path=str(cfg_file), auto_connect=False)
finally:
if old_env is None:
os.environ.pop("DATASTACK_CLIENT_ID", None)
else:
os.environ["DATASTACK_CLIENT_ID"] = old_env
# Mismatched kwarg client_id raises DataStackConfigError
with self.assertRaises(DataStackConfigError):
DataStack(
config_path=str(cfg_file),
client_id="mismatch_svc",
auto_connect=False,
)
# Auto-derives client_id from JWT when CLIENT_ID is omitted
tok_derived = _make_jwt({"sub": "derived_worker"})
derived_client = DataStack(
server_url="http://127.0.0.1:49999",
api_url="http://127.0.0.1:39999",
token=tok_derived,
auto_connect=False,
)
self.assertEqual(derived_client.client_id, "derived_worker")
self.assertEqual(derived_client.status()["clientId"], "derived_worker")
derived_client.disconnect()
def test_http_fallback_and_offline_spool_carry_client_id(self) -> None:
recorded_requests: List[Dict[str, Any]] = []
tok = _make_jwt({"clientId": "sensor_01", "sub": "sensor_01"})
class MockHandler(BaseHTTPRequestHandler):
def do_GET(self) -> None: # noqa: N802
if self.path == "/api/health":
body = json.dumps({"status": "online"}).encode("utf-8")
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
return
self.send_response(404)
self.end_headers()
def do_POST(self) -> None: # noqa: N802
length = int(self.headers.get("Content-Length", "0"))
raw = self.rfile.read(length).decode("utf-8")
parsed = json.loads(raw) if raw else {}
recorded_requests.append(
{
"path": self.path,
"client_id_header": self.headers.get("X-DataStack-Client-Id"),
"auth": self.headers.get("Authorization"),
"body": parsed,
}
)
resp_payload: Dict[str, Any] = {"success": True}
if isinstance(parsed, dict) and "batch" in parsed:
resp_payload["processed"] = len(parsed["batch"])
resp_payload["rejected"] = 0
elif isinstance(parsed, dict) and "records" in parsed:
resp_payload["processed"] = len(parsed["records"])
resp_payload["rejected"] = 0
body = json.dumps(resp_payload).encode("utf-8")
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, format: str, *args: Any) -> None:
return
server = ThreadingHTTPServer(("127.0.0.1", 0), MockHandler)
port = server.server_address[1]
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
with tempfile.TemporaryDirectory() as tmpdir:
spool_dir = Path(tmpdir) / "spool"
csv_file = Path(tmpdir) / "sample.csv"
csv_file.write_text("temp,unit\n21.5,C\n22.0,C\n", encoding="utf-8")
client = DataStack(
server_url="http://127.0.0.1:1", # unreachable WS -> HTTP fallback
api_url=f"http://127.0.0.1:{port}",
token=tok,
client_id="sensor_01",
auto_connect=False,
request_timeout_sec=1.0,
max_retries=0,
offline_spool_dir=str(spool_dir),
)
# Single event via HTTP fallback
res_single = client.send({"reading": 42})
self.assertTrue(res_single.get("success"))
self.assertEqual(res_single.get("transport"), "HTTP_FALLBACK")
# Batch events via HTTP fallback
res_batch = client.send_batch([{"k": "a"}, {"k": "b"}])
self.assertEqual(res_batch.get("processed"), 2)
# File ingestion via HTTP fallback
res_file = client.send_file(csv_file)
self.assertEqual(res_file.get("processed"), 2)
client.disconnect()
self.assertEqual(len(recorded_requests), 3)
for req in recorded_requests:
self.assertEqual(req["path"], "/api/events")
self.assertEqual(req["client_id_header"], "sensor_01")
self.assertEqual(req["auth"], f"Bearer {tok}")
self.assertEqual(recorded_requests[0]["body"]["clientId"], "sensor_01")
self.assertEqual(
[r["clientId"] for r in recorded_requests[1]["body"]["batch"]],
["sensor_01", "sensor_01"],
)
self.assertEqual(
[r["clientId"] for r in recorded_requests[2]["body"]["records"]],
["sensor_01", "sensor_01"],
)
# Offline spool check when HTTP is also unreachable
offline_client = datastack_sdk.DataStackClient(
server_url="http://127.0.0.1:1",
api_url="http://127.0.0.1:1",
token=tok,
auto_connect=False,
request_timeout_sec=1.0,
max_retries=0,
offline_spool_dir=str(spool_dir),
)
spool_res = offline_client.send({"offline_metric": 99})
offline_client.disconnect()
self.assertTrue(spool_res.get("spooled"))
self.assertEqual(spool_res.get("transport"), "OFFLINE_SPOOL")
self.assertGreaterEqual(spool_res.get("spooledCount", 0), 1)
finally:
server.shutdown()
server.server_close()
def test_client_id_normalization_and_invalid_numeric_env(self) -> None:
tok_norm = _make_jwt({"clientId": "sensor_01", "sub": "sensor_01"})
# Case/punctuation variations matching server SDKManager.normalizeClientId() must succeed
client = DataStack(
server_url="http://127.0.0.1:49999",
api_url="http://127.0.0.1:39999",
token=tok_norm,
client_id="Sensor.01",
auto_connect=False,
)
self.assertEqual(client.client_id, "sensor_01")
client.disconnect()
# Invalid numeric env var must raise DataStackConfigError instead of raw ValueError
old_retries = os.environ.get("DATASTACK_MAX_RETRIES")
try:
os.environ["DATASTACK_MAX_RETRIES"] = "not-an-int"
with self.assertRaises(DataStackConfigError):
load_config()
finally:
if old_retries is None:
os.environ.pop("DATASTACK_MAX_RETRIES", None)
else:
os.environ["DATASTACK_MAX_RETRIES"] = old_retries
def test_ragged_csv_all_rejected_send_file_and_partial_rl_recovery(self) -> None:
tok = _make_jwt({"clientId": "csv_worker", "sub": "csv_worker"})
mode = {"behavior": "reject_oversized"}
seen_records: List[Dict[str, Any]] = []
class EdgeHandler(BaseHTTPRequestHandler):
def do_POST(self) -> None: # noqa: N802
length = int(self.headers.get("Content-Length", "0"))
raw = self.rfile.read(length).decode("utf-8")
parsed = json.loads(raw) if raw else {}
items = parsed.get("records") or parsed.get("batch") or []
seen_records.extend(items)
if mode["behavior"] == "reject_oversized":
resp = {
"success": True,
"processed": 0,
"rejected": len(items),
"duplicates": 0,
"oversizedEventIds": [it.get("eventId") for it in items],
}
status = 200
else:
# First round processes first item, rate-limits second; second round disconnects/429
if len(items) > 1:
resp = {
"success": True,
"processed": 1,
"rejected": 1,
"duplicates": 0,
"rateLimitedEventIds": [items[1]["eventId"]],
}
status = 200
else:
resp = {
"success": False,
"code": "RATE_LIMIT_EXCEEDED",
"error": "Still limited",
}
status = 429
body = json.dumps(resp).encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def log_message(self, format: str, *args: Any) -> None:
return
server = ThreadingHTTPServer(("127.0.0.1", 0), EdgeHandler)
port = server.server_address[1]
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
with tempfile.TemporaryDirectory() as tmpdir:
# Ragged CSV with extra trailing column (used to crash sort_keys=True with None key)
ragged_csv = Path(tmpdir) / "ragged.csv"
ragged_csv.write_text(
"sensor,val\ns1,10,extra_trailing_col\n",
encoding="utf-8",
)
client = DataStack(
server_url="http://127.0.0.1:1",
api_url=f"http://127.0.0.1:{port}",
token=tok,
auto_connect=False,
request_timeout_sec=1.0,
max_retries=0,
retry_backoff_base_sec=0.1,
offline_spool_enabled=False,
)
res_file = client.send_file(ragged_csv)
self.assertFalse(res_file["success"])
self.assertEqual(res_file["processed"], 0)
self.assertEqual(res_file["rejected"], 1)
self.assertEqual(seen_records[0]["data"]["_extra"], ["extra_trailing_col"])
# Partial rate-limit where follow-up retry exhausts retries must preserve round-1 processed count
mode["behavior"] = "partial_rl"
spool_dir = Path(tmpdir) / "spool"
rl_client = DataStack(
server_url="http://127.0.0.1:1",
api_url=f"http://127.0.0.1:{port}",
token=tok,
auto_connect=False,
request_timeout_sec=1.0,
max_retries=1,
retry_backoff_base_sec=0.1,
offline_spool_dir=str(spool_dir),
)
rl_res = rl_client.send_batch(
[
{"eventId": "evt_ok_1", "data": {"n": 1}},
{"eventId": "evt_rl_2", "data": {"n": 2}},
]
)
self.assertTrue(rl_res["success"])
self.assertEqual(rl_res["processed"], 1)
self.assertEqual(rl_res["spooledCount"], 1)
self.assertEqual(rl_client.spooler.pending_count(), 1)
client.disconnect()
rl_client.disconnect()
finally:
server.shutdown()
server.server_close()
def test_heartbeat_stop_race_and_spooler_clean_ops(self) -> None:
hb = HeartbeatMonitor(interval_sec=5.0, recovery_callback=lambda: True)
hb.start()
hb.trigger_fallback("disconnect")
hb.stop("shutdown", join_timeout=0.5)
# Calling set_idle_connected after stop() must NOT resurrect IDLE state
hb.set_idle_connected()
self.assertEqual(hb.state, "STOPPED")
with tempfile.TemporaryDirectory() as tmpdir:
spool = DiskSpooler(spool_dir=tmpdir, max_size_mb=5, enabled=True)
spool.enqueue([{"eventId": "a"}, {"eventId": "b"}, {"eventId": "c"}])
self.assertEqual(spool.pending_count(), 3)
spool.ack_and_requeue_front(2, [{"eventId": "b_retry"}])
batch = spool.peek_batch(limit=10)
self.assertEqual([item["eventId"] for item in batch], ["b_retry", "c"])
self.assertFalse((Path(tmpdir) / "spool.jsonl.tmp").exists())
spool.ack_batch(2)
self.assertEqual(spool.pending_count(), 0)
def test_bug25_disconnects_leaked_socket_on_failed_ping_verify(self) -> None:
tok = _make_jwt({"clientId": "cli_bug25_sock"})
client = DataStack(
server_url="http://127.0.0.1:4000",
api_url="http://127.0.0.1:3000",
token=tok,
auto_connect=False,
enable_http_fallback=False,
offline_spool_enabled=False,
request_timeout_sec=0.5,
)
class FakeSocket:
def __init__(self) -> None:
self.connected = False
self.disconnect_calls = 0
def connect(self, *args: Any, **kwargs: Any) -> None:
self.connected = True
def disconnect(self) -> None:
self.disconnect_calls += 1
self.connected = False
fake_sio = FakeSocket()
client.sio = fake_sio
# Case 1: ping_verify returns False -> socket must be disconnected
client._verify_websocket_handshake = lambda timeout=None: False # type: ignore[method-assign]
self.assertFalse(client._connect_and_verify_websocket(wait_timeout=0.5))
self.assertFalse(client.is_connected)
self.assertFalse(fake_sio.connected)
self.assertEqual(fake_sio.disconnect_calls, 1)
# Case 2: ping_verify raises DataStackAuthError -> socket must be disconnected before re-raising
def raise_auth(timeout: Any = None) -> bool:
raise DataStackAuthError("CLIENT_REVOKED")
client._verify_websocket_handshake = raise_auth # type: ignore[method-assign]
with self.assertRaises(DataStackAuthError):
client._connect_and_verify_websocket(wait_timeout=0.5)
self.assertFalse(client.is_connected)
self.assertFalse(fake_sio.connected)
self.assertEqual(fake_sio.disconnect_calls, 2)
client.disconnect()
def test_bug25_strict_auto_connect_and_context_raise_connection_error(self) -> None:
tok = _make_jwt({"clientId": "cli_bug25_strict"})
# auto_connect=True with both fallback and offline spool disabled must raise DataStackConnectionError
with self.assertRaises(DataStackConnectionError):
DataStack(
server_url="http://127.0.0.1:1",
api_url="http://127.0.0.1:1",
token=tok,
auto_connect=True,
enable_http_fallback=False,
offline_spool_enabled=False,
request_timeout_sec=0.2,
)
# Context manager __enter__ with both fallback and offline spool disabled must also raise
strict_client = DataStack(
server_url="http://127.0.0.1:1",
api_url="http://127.0.0.1:1",
token=tok,
auto_connect=False,
enable_http_fallback=False,
offline_spool_enabled=False,
request_timeout_sec=0.2,
)
with self.assertRaises(DataStackConnectionError):
with strict_client:
pass
self.assertEqual(strict_client.heartbeat.state, "STOPPED")
def test_r2_phase03_bug11_flat_batch_and_file_strip_envelope_keys(self) -> None:
tok = _make_jwt({"clientId": "cli_bug11_strip"})
captured_batches: List[List[Dict[str, Any]]] = []
with tempfile.TemporaryDirectory() as tmpdir:
client = DataStack(
server_url="http://127.0.0.1:1",
api_url="http://127.0.0.1:1",
token=tok,
auto_connect=False,
offline_spool_enabled=False,
)
def fake_execute(**kwargs: Any) -> Dict[str, Any]:
ws_payload = kwargs.get("ws_payload")
if isinstance(ws_payload, list):
captured_batches.append(ws_payload)
return {
"success": True,
"processed": len(ws_payload),
"rejected": 0,
"transport": "HTTP",
}
return {"success": True, "processed": 0, "rejected": 0}
client._execute_with_retry_and_fallback = fake_execute # type: ignore[method-assign]
res_batch = client.send_batch(
[
{
"eventId": "py-flat-1",
"timestamp": "2026-11-19T12:00:00Z",
"clientId": "cli_bug11_strip",
"sensor": "temp",
"value": 24.5,
},
{
"eventId": "py-env-2",
"timestamp": "2026-11-19T12:00:01Z",
"data": {"sensor": "humidity", "value": 55},
},
]
)
self.assertTrue(res_batch["success"])
self.assertEqual(len(captured_batches), 1)
self.assertEqual(captured_batches[0][0]["eventId"], "py-flat-1")
self.assertEqual(
captured_batches[0][0]["data"],
{"sensor": "temp", "value": 24.5},
)
self.assertEqual(
captured_batches[0][1]["data"],
{"sensor": "humidity", "value": 55},
)
csv_path = Path(tmpdir) / "flat_records.csv"
csv_path.write_text(
"eventId,timestamp,clientId,sensor,reading\n"
"file-flat-1,2026-11-19T13:00:00Z,cli_bug11_strip,pressure,101.3\n",
encoding="utf-8",
)
res_file = client.send_file(csv_path)
self.assertTrue(res_file["success"])
self.assertEqual(len(captured_batches), 2)
self.assertEqual(captured_batches[1][0]["eventId"], "file-flat-1")
self.assertEqual(
captured_batches[1][0]["data"],
{"sensor": "pressure", "reading": "101.3"},
)
def test_r2_phase04_bug14_python38_ast_typing_compatibility(self) -> None:
pkg_dir = CANONICAL_PKG_ROOT / "datastack"
py_files = sorted(pkg_dir.glob("*.py"))
self.assertGreater(len(py_files), 0)
pep585_builtins = {"tuple", "list", "dict", "set", "frozenset", "type"}
for py_file in py_files:
src = py_file.read_text(encoding="utf-8")
tree = ast.parse(src, filename=str(py_file), feature_version=(3, 8))
has_future_annotations = any(
isinstance(node, ast.ImportFrom)
and node.module == "__future__"
and any(alias.name == "annotations" for alias in node.names)
for node in tree.body
)
if py_file.name == "client.py":
self.assertTrue(
has_future_annotations,
"client.py must declare `from __future__ import annotations`",
)
for node in ast.walk(tree):
if (
isinstance(node, ast.Subscript)
and isinstance(node.value, ast.Name)
and node.value.id in pep585_builtins
):
self.fail(
f"{py_file.name}:{node.lineno} uses Python 3.9+ PEP 585 generic `{node.value.id}[...]`"
)
if __name__ == "__main__":
unittest.main(verbosity=2)