feat: aggregate external events and improve security

- Fetch events from external ingestion service in API routes
- Decouple idempotency check from registration in Security utility
- Refine CSV storage logic to verify file content for headers
- Replace Python SDK with comprehensive, feature-rich implementation
This commit is contained in:
Team Dark Devil committed 2026-11-18 03:20:00 +06:00
1 parent 043123dc21
commit 34519c07df
13 files changed
+2099 -451

No files matched your search

+164
View File
@@ -0,0 +1,164 @@
"""
Thread-Safe Offline Disk Spooler (`DiskSpooler`).
Persists unsent telemetry events to `.datastack_spool/spool.jsonl` when a client
device loses network connectivity completely, and drains them in deterministic
batches once the Heartbeat Monitor restores connectivity.
"""
import json
import os
import threading
from pathlib import Path
from typing import Any, Dict, List
from .exceptions import DataStackOfflineSpoolError
class DiskSpooler:
"""
Append-only JSONL disk queue with atomic batch drainage and size cap protection.
"""
def __init__(
self,
spool_dir: str = ".datastack_spool",
max_size_mb: int = 50,
enabled: bool = True,
) -> None:
self.enabled = enabled
self.spool_dir = Path(spool_dir)
self.spool_file = self.spool_dir / "spool.jsonl"
self.max_bytes = max(1, int(max_size_mb)) * 1024 * 1024
self._lock = threading.RLock()
def _ensure_dir(self) -> None:
try:
self.spool_dir.mkdir(parents=True, exist_ok=True)
except Exception as exc:
raise DataStackOfflineSpoolError(
f"Cannot create offline spool directory '{self.spool_dir}': {exc}"
) from exc
def enqueue(self, events: List[Dict[str, Any]]) -> int:
"""
Appends one or more formatted events (`{eventId, timestamp, data}`)
to `.datastack_spool/spool.jsonl`. Returns count of spooled events.
"""
if not self.enabled or not events:
return 0
with self._lock:
self._ensure_dir()
current_size = (
self.spool_file.stat().st_size if self.spool_file.exists() else 0
)
lines: List[str] = []
added_bytes = 0
for evt in events:
serialized = json.dumps(evt, ensure_ascii=False) + "\n"
encoded_len = len(serialized.encode("utf-8"))
if current_size + added_bytes + encoded_len > self.max_bytes:
raise DataStackOfflineSpoolError(
f"Offline spool limit ({self.max_bytes // (1024 * 1024)} MB) "
f"reached at '{self.spool_file}'."
)
lines.append(serialized)
added_bytes += encoded_len
try:
with self.spool_file.open("a", encoding="utf-8") as fh:
fh.writelines(lines)
fh.flush()
except Exception as exc:
raise DataStackOfflineSpoolError(
f"Failed writing to offline spool file '{self.spool_file}': {exc}"
) from exc
return len(lines)
def peek_batch(self, limit: int = 250) -> List[Dict[str, Any]]:
"""
Reads up to `limit` spooled events from the front of the spool file
without removing them until `ack_batch()` confirms delivery.
"""
if not self.enabled:
return []
with self._lock:
if not self.spool_file.exists():
return []
batch: List[Dict[str, Any]] = []
try:
with self.spool_file.open("r", encoding="utf-8") as fh:
for line in fh:
if len(batch) >= limit:
break
stripped = line.strip()
if not stripped:
continue
try:
parsed = json.loads(stripped)
if isinstance(parsed, dict):
batch.append(parsed)
except json.JSONDecodeError:
continue
except Exception:
return []
return batch
def ack_batch(self, count: int) -> int:
"""
Removes the first `count` non-empty lines from the spool file atomically
after the Heartbeat Monitor confirms they have been ingested by DataStack.
"""
if not self.enabled or count <= 0:
return 0
with self._lock:
if not self.spool_file.exists():
return 0
try:
with self.spool_file.open("r", encoding="utf-8") as fh:
all_lines = [ln for ln in fh if ln.strip()]
if count >= len(all_lines):
self.spool_file.unlink(missing_ok=True)
return len(all_lines)
remaining = all_lines[count:]
tmp_file = self.spool_dir / "spool.jsonl.tmp"
with tmp_file.open("w", encoding="utf-8") as fh:
fh.writelines(remaining)
fh.flush()
os.replace(str(tmp_file), str(self.spool_file))
return count
except Exception:
return 0
def pending_count(self) -> int:
"""Returns the number of queued events waiting in the disk spool."""
if not self.enabled:
return 0
with self._lock:
if not self.spool_file.exists():
return 0
try:
with self.spool_file.open("r", encoding="utf-8") as fh:
return sum(1 for ln in fh if ln.strip())
except Exception:
return 0
def size_bytes(self) -> int:
"""Returns current disk usage of the spool file in bytes."""
with self._lock:
if not self.spool_file.exists():
return 0
try:
return self.spool_file.stat().st_size
except Exception:
return 0