feat(auth): harden session management and security
- Implement server-side session revocation on logout - Add IP extraction utility and admin setup error handling - Improve error codes and response consistency for auth endpoints - Refine landing page redirect logic to handle transient DB outages - Update deployment documentation and configuration defaults
This commit is contained in:
1 parent
935f12a1e9
commit
89e4d68690
81 files changed
+11530
-6916
No files matched your search
@@ -9,6 +9,7 @@ Phase-05 Verification Tests (BUG-18):
|
||||
offline disk spool replay against a local HTTP test server.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
@@ -32,9 +33,11 @@ import datastack # noqa: E402
|
||||
import datastack_sdk # noqa: E402
|
||||
from datastack import ( # noqa: E402
|
||||
DataStack,
|
||||
DataStackAuthError,
|
||||
DataStackClient,
|
||||
DataStackConfig,
|
||||
DataStackConfigError,
|
||||
DataStackConnectionError,
|
||||
DataStackOfflineSpoolError,
|
||||
DiskSpooler,
|
||||
HeartbeatMonitor,
|
||||
@@ -265,6 +268,340 @@ class Phase05PythonSdkTests(unittest.TestCase):
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
def test_client_id_normalization_and_invalid_numeric_env(self) -> None:
|
||||
tok_norm = _make_jwt({"clientId": "sensor_01", "sub": "sensor_01"})
|
||||
# Case/punctuation variations matching server SDKManager.normalizeClientId() must succeed
|
||||
client = DataStack(
|
||||
server_url="http://127.0.0.1:49999",
|
||||
api_url="http://127.0.0.1:39999",
|
||||
token=tok_norm,
|
||||
client_id="Sensor.01",
|
||||
auto_connect=False,
|
||||
)
|
||||
self.assertEqual(client.client_id, "sensor_01")
|
||||
client.disconnect()
|
||||
|
||||
# Invalid numeric env var must raise DataStackConfigError instead of raw ValueError
|
||||
old_retries = os.environ.get("DATASTACK_MAX_RETRIES")
|
||||
try:
|
||||
os.environ["DATASTACK_MAX_RETRIES"] = "not-an-int"
|
||||
with self.assertRaises(DataStackConfigError):
|
||||
load_config()
|
||||
finally:
|
||||
if old_retries is None:
|
||||
os.environ.pop("DATASTACK_MAX_RETRIES", None)
|
||||
else:
|
||||
os.environ["DATASTACK_MAX_RETRIES"] = old_retries
|
||||
|
||||
def test_ragged_csv_all_rejected_send_file_and_partial_rl_recovery(self) -> None:
|
||||
tok = _make_jwt({"clientId": "csv_worker", "sub": "csv_worker"})
|
||||
mode = {"behavior": "reject_oversized"}
|
||||
seen_records: List[Dict[str, Any]] = []
|
||||
|
||||
class EdgeHandler(BaseHTTPRequestHandler):
|
||||
def do_POST(self) -> None: # noqa: N802
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
raw = self.rfile.read(length).decode("utf-8")
|
||||
parsed = json.loads(raw) if raw else {}
|
||||
items = parsed.get("records") or parsed.get("batch") or []
|
||||
seen_records.extend(items)
|
||||
|
||||
if mode["behavior"] == "reject_oversized":
|
||||
resp = {
|
||||
"success": True,
|
||||
"processed": 0,
|
||||
"rejected": len(items),
|
||||
"duplicates": 0,
|
||||
"oversizedEventIds": [it.get("eventId") for it in items],
|
||||
}
|
||||
status = 200
|
||||
else:
|
||||
# First round processes first item, rate-limits second; second round disconnects/429
|
||||
if len(items) > 1:
|
||||
resp = {
|
||||
"success": True,
|
||||
"processed": 1,
|
||||
"rejected": 1,
|
||||
"duplicates": 0,
|
||||
"rateLimitedEventIds": [items[1]["eventId"]],
|
||||
}
|
||||
status = 200
|
||||
else:
|
||||
resp = {
|
||||
"success": False,
|
||||
"code": "RATE_LIMIT_EXCEEDED",
|
||||
"error": "Still limited",
|
||||
}
|
||||
status = 429
|
||||
|
||||
body = json.dumps(resp).encode("utf-8")
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, format: str, *args: Any) -> None:
|
||||
return
|
||||
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), EdgeHandler)
|
||||
port = server.server_address[1]
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# Ragged CSV with extra trailing column (used to crash sort_keys=True with None key)
|
||||
ragged_csv = Path(tmpdir) / "ragged.csv"
|
||||
ragged_csv.write_text(
|
||||
"sensor,val\ns1,10,extra_trailing_col\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
client = DataStack(
|
||||
server_url="http://127.0.0.1:1",
|
||||
api_url=f"http://127.0.0.1:{port}",
|
||||
token=tok,
|
||||
auto_connect=False,
|
||||
request_timeout_sec=1.0,
|
||||
max_retries=0,
|
||||
retry_backoff_base_sec=0.1,
|
||||
offline_spool_enabled=False,
|
||||
)
|
||||
|
||||
res_file = client.send_file(ragged_csv)
|
||||
self.assertFalse(res_file["success"])
|
||||
self.assertEqual(res_file["processed"], 0)
|
||||
self.assertEqual(res_file["rejected"], 1)
|
||||
self.assertEqual(seen_records[0]["data"]["_extra"], ["extra_trailing_col"])
|
||||
|
||||
# Partial rate-limit where follow-up retry exhausts retries must preserve round-1 processed count
|
||||
mode["behavior"] = "partial_rl"
|
||||
spool_dir = Path(tmpdir) / "spool"
|
||||
rl_client = DataStack(
|
||||
server_url="http://127.0.0.1:1",
|
||||
api_url=f"http://127.0.0.1:{port}",
|
||||
token=tok,
|
||||
auto_connect=False,
|
||||
request_timeout_sec=1.0,
|
||||
max_retries=1,
|
||||
retry_backoff_base_sec=0.1,
|
||||
offline_spool_dir=str(spool_dir),
|
||||
)
|
||||
rl_res = rl_client.send_batch(
|
||||
[
|
||||
{"eventId": "evt_ok_1", "data": {"n": 1}},
|
||||
{"eventId": "evt_rl_2", "data": {"n": 2}},
|
||||
]
|
||||
)
|
||||
self.assertTrue(rl_res["success"])
|
||||
self.assertEqual(rl_res["processed"], 1)
|
||||
self.assertEqual(rl_res["spooledCount"], 1)
|
||||
self.assertEqual(rl_client.spooler.pending_count(), 1)
|
||||
|
||||
client.disconnect()
|
||||
rl_client.disconnect()
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
def test_heartbeat_stop_race_and_spooler_clean_ops(self) -> None:
|
||||
hb = HeartbeatMonitor(interval_sec=5.0, recovery_callback=lambda: True)
|
||||
hb.start()
|
||||
hb.trigger_fallback("disconnect")
|
||||
hb.stop("shutdown", join_timeout=0.5)
|
||||
# Calling set_idle_connected after stop() must NOT resurrect IDLE state
|
||||
hb.set_idle_connected()
|
||||
self.assertEqual(hb.state, "STOPPED")
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
spool = DiskSpooler(spool_dir=tmpdir, max_size_mb=5, enabled=True)
|
||||
spool.enqueue([{"eventId": "a"}, {"eventId": "b"}, {"eventId": "c"}])
|
||||
self.assertEqual(spool.pending_count(), 3)
|
||||
spool.ack_and_requeue_front(2, [{"eventId": "b_retry"}])
|
||||
batch = spool.peek_batch(limit=10)
|
||||
self.assertEqual([item["eventId"] for item in batch], ["b_retry", "c"])
|
||||
self.assertFalse((Path(tmpdir) / "spool.jsonl.tmp").exists())
|
||||
spool.ack_batch(2)
|
||||
self.assertEqual(spool.pending_count(), 0)
|
||||
|
||||
def test_bug25_disconnects_leaked_socket_on_failed_ping_verify(self) -> None:
|
||||
tok = _make_jwt({"clientId": "cli_bug25_sock"})
|
||||
client = DataStack(
|
||||
server_url="http://127.0.0.1:4000",
|
||||
api_url="http://127.0.0.1:3000",
|
||||
token=tok,
|
||||
auto_connect=False,
|
||||
enable_http_fallback=False,
|
||||
offline_spool_enabled=False,
|
||||
request_timeout_sec=0.5,
|
||||
)
|
||||
|
||||
class FakeSocket:
|
||||
def __init__(self) -> None:
|
||||
self.connected = False
|
||||
self.disconnect_calls = 0
|
||||
|
||||
def connect(self, *args: Any, **kwargs: Any) -> None:
|
||||
self.connected = True
|
||||
|
||||
def disconnect(self) -> None:
|
||||
self.disconnect_calls += 1
|
||||
self.connected = False
|
||||
|
||||
fake_sio = FakeSocket()
|
||||
client.sio = fake_sio
|
||||
|
||||
# Case 1: ping_verify returns False -> socket must be disconnected
|
||||
client._verify_websocket_handshake = lambda timeout=None: False # type: ignore[method-assign]
|
||||
self.assertFalse(client._connect_and_verify_websocket(wait_timeout=0.5))
|
||||
self.assertFalse(client.is_connected)
|
||||
self.assertFalse(fake_sio.connected)
|
||||
self.assertEqual(fake_sio.disconnect_calls, 1)
|
||||
|
||||
# Case 2: ping_verify raises DataStackAuthError -> socket must be disconnected before re-raising
|
||||
def raise_auth(timeout: Any = None) -> bool:
|
||||
raise DataStackAuthError("CLIENT_REVOKED")
|
||||
|
||||
client._verify_websocket_handshake = raise_auth # type: ignore[method-assign]
|
||||
with self.assertRaises(DataStackAuthError):
|
||||
client._connect_and_verify_websocket(wait_timeout=0.5)
|
||||
self.assertFalse(client.is_connected)
|
||||
self.assertFalse(fake_sio.connected)
|
||||
self.assertEqual(fake_sio.disconnect_calls, 2)
|
||||
client.disconnect()
|
||||
|
||||
def test_bug25_strict_auto_connect_and_context_raise_connection_error(self) -> None:
|
||||
tok = _make_jwt({"clientId": "cli_bug25_strict"})
|
||||
|
||||
# auto_connect=True with both fallback and offline spool disabled must raise DataStackConnectionError
|
||||
with self.assertRaises(DataStackConnectionError):
|
||||
DataStack(
|
||||
server_url="http://127.0.0.1:1",
|
||||
api_url="http://127.0.0.1:1",
|
||||
token=tok,
|
||||
auto_connect=True,
|
||||
enable_http_fallback=False,
|
||||
offline_spool_enabled=False,
|
||||
request_timeout_sec=0.2,
|
||||
)
|
||||
|
||||
# Context manager __enter__ with both fallback and offline spool disabled must also raise
|
||||
strict_client = DataStack(
|
||||
server_url="http://127.0.0.1:1",
|
||||
api_url="http://127.0.0.1:1",
|
||||
token=tok,
|
||||
auto_connect=False,
|
||||
enable_http_fallback=False,
|
||||
offline_spool_enabled=False,
|
||||
request_timeout_sec=0.2,
|
||||
)
|
||||
with self.assertRaises(DataStackConnectionError):
|
||||
with strict_client:
|
||||
pass
|
||||
self.assertEqual(strict_client.heartbeat.state, "STOPPED")
|
||||
|
||||
def test_r2_phase03_bug11_flat_batch_and_file_strip_envelope_keys(self) -> None:
|
||||
tok = _make_jwt({"clientId": "cli_bug11_strip"})
|
||||
captured_batches: List[List[Dict[str, Any]]] = []
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
client = DataStack(
|
||||
server_url="http://127.0.0.1:1",
|
||||
api_url="http://127.0.0.1:1",
|
||||
token=tok,
|
||||
auto_connect=False,
|
||||
offline_spool_enabled=False,
|
||||
)
|
||||
|
||||
def fake_execute(**kwargs: Any) -> Dict[str, Any]:
|
||||
ws_payload = kwargs.get("ws_payload")
|
||||
if isinstance(ws_payload, list):
|
||||
captured_batches.append(ws_payload)
|
||||
return {
|
||||
"success": True,
|
||||
"processed": len(ws_payload),
|
||||
"rejected": 0,
|
||||
"transport": "HTTP",
|
||||
}
|
||||
return {"success": True, "processed": 0, "rejected": 0}
|
||||
|
||||
client._execute_with_retry_and_fallback = fake_execute # type: ignore[method-assign]
|
||||
|
||||
res_batch = client.send_batch(
|
||||
[
|
||||
{
|
||||
"eventId": "py-flat-1",
|
||||
"timestamp": "2026-11-19T12:00:00Z",
|
||||
"clientId": "cli_bug11_strip",
|
||||
"sensor": "temp",
|
||||
"value": 24.5,
|
||||
},
|
||||
{
|
||||
"eventId": "py-env-2",
|
||||
"timestamp": "2026-11-19T12:00:01Z",
|
||||
"data": {"sensor": "humidity", "value": 55},
|
||||
},
|
||||
]
|
||||
)
|
||||
self.assertTrue(res_batch["success"])
|
||||
self.assertEqual(len(captured_batches), 1)
|
||||
self.assertEqual(captured_batches[0][0]["eventId"], "py-flat-1")
|
||||
self.assertEqual(
|
||||
captured_batches[0][0]["data"],
|
||||
{"sensor": "temp", "value": 24.5},
|
||||
)
|
||||
self.assertEqual(
|
||||
captured_batches[0][1]["data"],
|
||||
{"sensor": "humidity", "value": 55},
|
||||
)
|
||||
|
||||
csv_path = Path(tmpdir) / "flat_records.csv"
|
||||
csv_path.write_text(
|
||||
"eventId,timestamp,clientId,sensor,reading\n"
|
||||
"file-flat-1,2026-11-19T13:00:00Z,cli_bug11_strip,pressure,101.3\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
res_file = client.send_file(csv_path)
|
||||
self.assertTrue(res_file["success"])
|
||||
self.assertEqual(len(captured_batches), 2)
|
||||
self.assertEqual(captured_batches[1][0]["eventId"], "file-flat-1")
|
||||
self.assertEqual(
|
||||
captured_batches[1][0]["data"],
|
||||
{"sensor": "pressure", "reading": "101.3"},
|
||||
)
|
||||
|
||||
def test_r2_phase04_bug14_python38_ast_typing_compatibility(self) -> None:
|
||||
pkg_dir = CANONICAL_PKG_ROOT / "datastack"
|
||||
py_files = sorted(pkg_dir.glob("*.py"))
|
||||
self.assertGreater(len(py_files), 0)
|
||||
|
||||
pep585_builtins = {"tuple", "list", "dict", "set", "frozenset", "type"}
|
||||
for py_file in py_files:
|
||||
src = py_file.read_text(encoding="utf-8")
|
||||
tree = ast.parse(src, filename=str(py_file), feature_version=(3, 8))
|
||||
has_future_annotations = any(
|
||||
isinstance(node, ast.ImportFrom)
|
||||
and node.module == "__future__"
|
||||
and any(alias.name == "annotations" for alias in node.names)
|
||||
for node in tree.body
|
||||
)
|
||||
if py_file.name == "client.py":
|
||||
self.assertTrue(
|
||||
has_future_annotations,
|
||||
"client.py must declare `from __future__ import annotations`",
|
||||
)
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if (
|
||||
isinstance(node, ast.Subscript)
|
||||
and isinstance(node.value, ast.Name)
|
||||
and node.value.id in pep585_builtins
|
||||
):
|
||||
self.fail(
|
||||
f"{py_file.name}:{node.lineno} uses Python 3.9+ PEP 585 generic `{node.value.id}[...]`"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
Reference in new issue
Block a user