feat: aggregate external events and improve security
- Fetch events from external ingestion service in API routes - Decouple idempotency check from registration in Security utility - Refine CSV storage logic to verify file content for headers - Replace Python SDK with comprehensive, feature-rich implementation
This commit is contained in:
1 parent
043123dc21
commit
34519c07df
13 files changed
+2099
-451
No files matched your search
@@ -113,6 +113,37 @@ export async function GET(request: NextRequest) {
|
|||||||
addIfMatching({ ...event, status: "buffered" });
|
addIfMatching({ ...event, status: "buffered" });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const ingestionBaseUrl = (
|
||||||
|
process.env.INGESTION_INTERNAL_URL || "http://127.0.0.1:4000"
|
||||||
|
).replace(/\/$/, "");
|
||||||
|
try {
|
||||||
|
const res = await fetch(`${ingestionBaseUrl}/internal/recent?limit=200`, {
|
||||||
|
cache: "no-store",
|
||||||
|
signal: AbortSignal.timeout(1000),
|
||||||
|
});
|
||||||
|
if (res.ok) {
|
||||||
|
const data = await res.json();
|
||||||
|
if (Array.isArray(data.events)) {
|
||||||
|
for (const evt of data.events) {
|
||||||
|
if (evt && evt.eventId) {
|
||||||
|
addIfMatching({
|
||||||
|
clientId: String(evt.clientId || ""),
|
||||||
|
eventId: String(evt.eventId),
|
||||||
|
timestamp: String(evt.timestamp || new Date().toISOString()),
|
||||||
|
payload:
|
||||||
|
typeof evt.payload === "string"
|
||||||
|
? evt.payload
|
||||||
|
: JSON.stringify(evt.payload ?? {}),
|
||||||
|
status: "buffered",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Port 4000 may not be running in isolated single-process dev mode
|
||||||
|
}
|
||||||
|
|
||||||
return NextResponse.json({
|
return NextResponse.json({
|
||||||
events: selectedRows.slice(offset),
|
events: selectedRows.slice(offset),
|
||||||
total,
|
total,
|
||||||
|
|||||||
+23
-1
@@ -58,12 +58,34 @@ export async function GET(req: NextRequest) {
|
|||||||
const limit = Math.min(200, Math.max(1, Number(searchParams.get('limit')) || 50));
|
const limit = Math.min(200, Math.max(1, Number(searchParams.get('limit')) || 50));
|
||||||
|
|
||||||
const bufferedEvents = DataBufferEngine.getRecentEvents(limit, clientId);
|
const bufferedEvents = DataBufferEngine.getRecentEvents(limit, clientId);
|
||||||
|
let socketBufferedEvents: DataEvent[] = [];
|
||||||
|
const ingestionBaseUrl = (
|
||||||
|
process.env.INGESTION_INTERNAL_URL || 'http://127.0.0.1:4000'
|
||||||
|
).replace(/\/$/, '');
|
||||||
|
|
||||||
|
try {
|
||||||
|
const qs = new URLSearchParams({ limit: String(limit) });
|
||||||
|
if (clientId) qs.set('clientId', clientId);
|
||||||
|
const res = await fetch(`${ingestionBaseUrl}/internal/recent?${qs.toString()}`, {
|
||||||
|
cache: 'no-store',
|
||||||
|
signal: AbortSignal.timeout(1000),
|
||||||
|
});
|
||||||
|
if (res.ok) {
|
||||||
|
const data = await res.json();
|
||||||
|
if (Array.isArray(data.events)) {
|
||||||
|
socketBufferedEvents = data.events;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch {
|
||||||
|
// Port 4000 may not be running in isolated single-process dev mode
|
||||||
|
}
|
||||||
|
|
||||||
const storedEvents = await getRecentStoredEvents(limit, clientId);
|
const storedEvents = await getRecentStoredEvents(limit, clientId);
|
||||||
|
|
||||||
const seenIds = new Set<string>();
|
const seenIds = new Set<string>();
|
||||||
const merged: DataEvent[] = [];
|
const merged: DataEvent[] = [];
|
||||||
|
|
||||||
for (const evt of [...bufferedEvents, ...storedEvents]) {
|
for (const evt of [...bufferedEvents, ...socketBufferedEvents, ...storedEvents]) {
|
||||||
if (!evt.eventId || seenIds.has(evt.eventId)) continue;
|
if (!evt.eventId || seenIds.has(evt.eventId)) continue;
|
||||||
seenIds.add(evt.eventId);
|
seenIds.add(evt.eventId);
|
||||||
merged.push(evt);
|
merged.push(evt);
|
||||||
|
|||||||
+38
-22
@@ -30,8 +30,8 @@ export class SecurityEngine {
|
|||||||
* ClientId + EventId must be unique.
|
* ClientId + EventId must be unique.
|
||||||
* Duplicate events are dropped silently or tagged.
|
* Duplicate events are dropped silently or tagged.
|
||||||
*/
|
*/
|
||||||
public static checkIdempotency(clientId: string, eventId: string): boolean {
|
public static hasDuplicate(clientId: string, eventId: string): boolean {
|
||||||
if (!eventId) return true; // If no eventId generated, pass through
|
if (!eventId) return false;
|
||||||
const key = `${clientId}:${eventId}`;
|
const key = `${clientId}:${eventId}`;
|
||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
|
|
||||||
@@ -47,12 +47,24 @@ export class SecurityEngine {
|
|||||||
if (this.idempotencySet.has(key)) {
|
if (this.idempotencySet.has(key)) {
|
||||||
const storedTime = this.idempotencySet.get(key) || 0;
|
const storedTime = this.idempotencySet.get(key) || 0;
|
||||||
if (now - storedTime < this.IDEMPOTENCY_TTL_MS) {
|
if (now - storedTime < this.IDEMPOTENCY_TTL_MS) {
|
||||||
return false; // Duplicate detected
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
this.idempotencySet.set(key, now);
|
return false;
|
||||||
return true; // Unique, allowed
|
}
|
||||||
|
|
||||||
|
public static recordIdempotency(clientId: string, eventId: string): void {
|
||||||
|
if (!eventId) return;
|
||||||
|
this.idempotencySet.set(`${clientId}:${eventId}`, Date.now());
|
||||||
|
}
|
||||||
|
|
||||||
|
public static checkIdempotency(clientId: string, eventId: string): boolean {
|
||||||
|
if (this.hasDuplicate(clientId, eventId)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
this.recordIdempotency(clientId, eventId);
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -81,7 +93,8 @@ export class SecurityEngine {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Validates incoming event payload:
|
* Validates incoming event payload:
|
||||||
* Checks max byte size and JSON structure.
|
* Checks duplicate idempotency (read-only), max byte size, and rate limit,
|
||||||
|
* and records the idempotency key ONLY after all checks pass.
|
||||||
*/
|
*/
|
||||||
public static validateEvent(
|
public static validateEvent(
|
||||||
clientId: string,
|
clientId: string,
|
||||||
@@ -89,10 +102,8 @@ export class SecurityEngine {
|
|||||||
payload: unknown,
|
payload: unknown,
|
||||||
clientMaxRate = 100,
|
clientMaxRate = 100,
|
||||||
): ValidationResult {
|
): ValidationResult {
|
||||||
// 1. Check idempotency
|
// 1. Check idempotency (read-only so rate-limited retries are not burned)
|
||||||
const isUnique =
|
if (this.enableIdempotency && this.hasDuplicate(clientId, eventId)) {
|
||||||
!this.enableIdempotency || this.checkIdempotency(clientId, eventId);
|
|
||||||
if (!isUnique) {
|
|
||||||
return {
|
return {
|
||||||
allowed: false,
|
allowed: false,
|
||||||
reason: "Duplicate event dropped silently",
|
reason: "Duplicate event dropped silently",
|
||||||
@@ -100,18 +111,7 @@ export class SecurityEngine {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Check rate limit
|
// 2. Check payload size before consuming rate-limit quota
|
||||||
const effectiveRateLimit = Math.min(clientMaxRate, this.maxEventsPerSec);
|
|
||||||
const withinLimit = this.checkRateLimit(clientId, effectiveRateLimit);
|
|
||||||
if (!withinLimit) {
|
|
||||||
return {
|
|
||||||
allowed: false,
|
|
||||||
reason: `Rate limit of ${clientMaxRate} events/sec exceeded for client ${clientId}`,
|
|
||||||
code: "RATE_LIMIT_EXCEEDED",
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
// 3. Check payload size
|
|
||||||
const payloadStr =
|
const payloadStr =
|
||||||
typeof payload === "string" ? payload : JSON.stringify(payload);
|
typeof payload === "string" ? payload : JSON.stringify(payload);
|
||||||
const byteLength = Buffer.byteLength(payloadStr, "utf8");
|
const byteLength = Buffer.byteLength(payloadStr, "utf8");
|
||||||
@@ -124,6 +124,22 @@ export class SecurityEngine {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 3. Check rate limit
|
||||||
|
const effectiveRateLimit = Math.min(clientMaxRate, this.maxEventsPerSec);
|
||||||
|
const withinLimit = this.checkRateLimit(clientId, effectiveRateLimit);
|
||||||
|
if (!withinLimit) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
reason: `Rate limit of ${clientMaxRate} events/sec exceeded for client ${clientId}`,
|
||||||
|
code: "RATE_LIMIT_EXCEEDED",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Record idempotency key ONLY after event is accepted
|
||||||
|
if (this.enableIdempotency && eventId) {
|
||||||
|
this.recordIdempotency(clientId, eventId);
|
||||||
|
}
|
||||||
|
|
||||||
return { allowed: true };
|
return { allowed: true };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -91,7 +91,13 @@ export class CSVStorageManager {
|
|||||||
for (const [key, batch] of grouped.entries()) {
|
for (const [key, batch] of grouped.entries()) {
|
||||||
const [clientId, date] = key.split(':::');
|
const [clientId, date] = key.split(':::');
|
||||||
const targetPath = this.getTargetFilePath(clientId, date);
|
const targetPath = this.getTargetFilePath(clientId, date);
|
||||||
const fileExists = fs.existsSync(targetPath);
|
let hasContent = false;
|
||||||
|
try {
|
||||||
|
hasContent =
|
||||||
|
fs.existsSync(targetPath) && fs.statSync(targetPath).size > 0;
|
||||||
|
} catch {
|
||||||
|
hasContent = false;
|
||||||
|
}
|
||||||
|
|
||||||
// Format payload as JSON string if not already
|
// Format payload as JSON string if not already
|
||||||
const cleanBatch = batch.map((item) => ({
|
const cleanBatch = batch.map((item) => ({
|
||||||
@@ -101,10 +107,10 @@ export class CSVStorageManager {
|
|||||||
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
|
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
const parser = new Parser({ fields, header: !fileExists, eol: '\n' });
|
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
|
||||||
const csvData = parser.parse(cleanBatch);
|
const csvData = parser.parse(cleanBatch);
|
||||||
|
|
||||||
if (fileExists) {
|
if (hasContent) {
|
||||||
fs.appendFileSync(targetPath, '\n' + csvData, 'utf8');
|
fs.appendFileSync(targetPath, '\n' + csvData, 'utf8');
|
||||||
} else {
|
} else {
|
||||||
fs.writeFileSync(targetPath, csvData, 'utf8');
|
fs.writeFileSync(targetPath, csvData, 'utf8');
|
||||||
|
|||||||
+1125
-360
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,321 @@
|
|||||||
|
"""
|
||||||
|
Auto-Discovery Configuration Loader for `config.datastack.py`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, List, Optional
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
from .exceptions import DataStackConfigError
|
||||||
|
|
||||||
|
CONFIG_FILENAME = "config.datastack.py"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class DataStackConfig:
|
||||||
|
server_url: str = "http://localhost:4000"
|
||||||
|
api_url: str = "http://localhost:3000"
|
||||||
|
client_token: str = ""
|
||||||
|
client_id: str = ""
|
||||||
|
auto_connect: bool = True
|
||||||
|
enable_http_fallback: bool = True
|
||||||
|
heartbeat_interval_sec: float = 5.0
|
||||||
|
request_timeout_sec: float = 8.0
|
||||||
|
max_retries: int = 3
|
||||||
|
retry_backoff_base_sec: float = 0.5
|
||||||
|
batch_chunk_size: int = 250
|
||||||
|
offline_spool_enabled: bool = True
|
||||||
|
offline_spool_dir: str = ".datastack_spool"
|
||||||
|
max_spool_size_mb: int = 50
|
||||||
|
config_source_path: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
def _derive_default_api_url(server_url: str) -> str:
|
||||||
|
"""
|
||||||
|
If API_URL is not explicitly set, infer Port 3000 on the same host when SERVER_URL
|
||||||
|
uses Port 4000, or fall back to http://localhost:3000.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
parsed = urlparse(server_url)
|
||||||
|
if parsed.scheme and parsed.hostname:
|
||||||
|
if parsed.port == 4000:
|
||||||
|
return f"{parsed.scheme}://{parsed.hostname}:3000"
|
||||||
|
return f"{parsed.scheme}://{parsed.netloc}"
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return "http://localhost:3000"
|
||||||
|
|
||||||
|
|
||||||
|
def _candidate_config_paths(explicit_path: Optional[str] = None) -> List[Path]:
|
||||||
|
candidates: List[Path] = []
|
||||||
|
|
||||||
|
if explicit_path:
|
||||||
|
p = Path(explicit_path).expanduser().resolve()
|
||||||
|
if p.is_dir():
|
||||||
|
candidates.append(p / CONFIG_FILENAME)
|
||||||
|
else:
|
||||||
|
candidates.append(p)
|
||||||
|
|
||||||
|
env_path = os.environ.get("DATASTACK_CONFIG_PATH", "").strip()
|
||||||
|
if env_path:
|
||||||
|
ep = Path(env_path).expanduser().resolve()
|
||||||
|
if ep.is_dir():
|
||||||
|
candidates.append(ep / CONFIG_FILENAME)
|
||||||
|
else:
|
||||||
|
candidates.append(ep)
|
||||||
|
|
||||||
|
# Search current working directory and up to 3 parent levels
|
||||||
|
try:
|
||||||
|
cwd = Path.cwd().resolve()
|
||||||
|
candidates.append(cwd / CONFIG_FILENAME)
|
||||||
|
for parent in list(cwd.parents)[:3]:
|
||||||
|
candidates.append(parent / CONFIG_FILENAME)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Search main entry script directory
|
||||||
|
try:
|
||||||
|
if sys.argv and sys.argv[0]:
|
||||||
|
script_dir = Path(sys.argv[0]).resolve().parent
|
||||||
|
candidates.append(script_dir / CONFIG_FILENAME)
|
||||||
|
for parent in list(script_dir.parents)[:2]:
|
||||||
|
candidates.append(parent / CONFIG_FILENAME)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Deduplicate while preserving order
|
||||||
|
seen = set()
|
||||||
|
unique: List[Path] = []
|
||||||
|
for c in candidates:
|
||||||
|
key = str(c)
|
||||||
|
if key not in seen:
|
||||||
|
seen.add(key)
|
||||||
|
unique.append(c)
|
||||||
|
return unique
|
||||||
|
|
||||||
|
|
||||||
|
def _load_python_config_file(file_path: Path) -> Dict[str, Any]:
|
||||||
|
try:
|
||||||
|
spec = importlib.util.spec_from_file_location(
|
||||||
|
"datastack_user_config", str(file_path)
|
||||||
|
)
|
||||||
|
if spec is None or spec.loader is None:
|
||||||
|
raise DataStackConfigError(
|
||||||
|
f"Unable to load configuration specification from {file_path}"
|
||||||
|
)
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
|
||||||
|
raw: Dict[str, Any] = {}
|
||||||
|
if hasattr(module, "DATASTACK_CONFIG") and isinstance(
|
||||||
|
getattr(module, "DATASTACK_CONFIG"), dict
|
||||||
|
):
|
||||||
|
raw.update(getattr(module, "DATASTACK_CONFIG"))
|
||||||
|
|
||||||
|
# Also support top-level module variables (e.g., SERVER_URL = "...", CLIENT_TOKEN = "...")
|
||||||
|
for attr in dir(module):
|
||||||
|
if attr.isupper() and attr != "DATASTACK_CONFIG":
|
||||||
|
raw[attr] = getattr(module, attr)
|
||||||
|
|
||||||
|
return raw
|
||||||
|
except DataStackConfigError:
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
raise DataStackConfigError(
|
||||||
|
f"Failed to parse {file_path}: {exc}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def load_config(
|
||||||
|
config_path: Optional[str] = None,
|
||||||
|
require_token: bool = False,
|
||||||
|
**overrides: Any,
|
||||||
|
) -> DataStackConfig:
|
||||||
|
"""
|
||||||
|
Discovers and loads `config.datastack.py` from the client application's
|
||||||
|
directory, merges environment variables (`DATASTACK_*`), and applies
|
||||||
|
any explicit keyword overrides.
|
||||||
|
"""
|
||||||
|
file_values: Dict[str, Any] = {}
|
||||||
|
resolved_source: Optional[str] = None
|
||||||
|
|
||||||
|
for candidate in _candidate_config_paths(config_path):
|
||||||
|
if candidate.is_file():
|
||||||
|
file_values = _load_python_config_file(candidate)
|
||||||
|
resolved_source = str(candidate)
|
||||||
|
break
|
||||||
|
|
||||||
|
if config_path and not resolved_source:
|
||||||
|
raise DataStackConfigError(
|
||||||
|
f"Explicitly requested config file was not found: {config_path}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Normalize keys to uppercase for lookup
|
||||||
|
normalized_file = {str(k).upper(): v for k, v in file_values.items()}
|
||||||
|
normalized_overrides = {
|
||||||
|
str(k).upper(): v for k, v in overrides.items() if v is not None
|
||||||
|
}
|
||||||
|
|
||||||
|
def pick(keys: List[str], env_keys: List[str], default: Any) -> Any:
|
||||||
|
for k in keys:
|
||||||
|
if k in normalized_overrides:
|
||||||
|
return normalized_overrides[k]
|
||||||
|
for ek in env_keys:
|
||||||
|
env_val = os.environ.get(ek)
|
||||||
|
if env_val is not None and str(env_val).strip() != "":
|
||||||
|
return env_val
|
||||||
|
for k in keys:
|
||||||
|
if k in normalized_file and normalized_file[k] is not None:
|
||||||
|
return normalized_file[k]
|
||||||
|
return default
|
||||||
|
|
||||||
|
def to_bool(val: Any, default: bool) -> bool:
|
||||||
|
if isinstance(val, bool):
|
||||||
|
return val
|
||||||
|
if isinstance(val, str):
|
||||||
|
return val.strip().lower() in ("1", "true", "yes", "on")
|
||||||
|
if val is None:
|
||||||
|
return default
|
||||||
|
return bool(val)
|
||||||
|
|
||||||
|
server_url = str(
|
||||||
|
pick(
|
||||||
|
["SERVER_URL", "SOCKET_URL", "WS_URL"],
|
||||||
|
["DATASTACK_SERVER_URL", "DATASTACK_SOCKET_URL"],
|
||||||
|
"http://localhost:4000",
|
||||||
|
)
|
||||||
|
).rstrip("/")
|
||||||
|
|
||||||
|
default_api = _derive_default_api_url(server_url)
|
||||||
|
api_url = str(
|
||||||
|
pick(
|
||||||
|
["API_URL", "HTTP_URL", "WEB_URL"],
|
||||||
|
["DATASTACK_API_URL", "DATASTACK_HTTP_URL"],
|
||||||
|
default_api,
|
||||||
|
)
|
||||||
|
).rstrip("/")
|
||||||
|
|
||||||
|
client_token = str(
|
||||||
|
pick(
|
||||||
|
["CLIENT_TOKEN", "TOKEN", "JWT_TOKEN", "API_TOKEN"],
|
||||||
|
["DATASTACK_CLIENT_TOKEN", "DATASTACK_TOKEN"],
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
).strip()
|
||||||
|
|
||||||
|
client_id = str(
|
||||||
|
pick(
|
||||||
|
["CLIENT_ID"],
|
||||||
|
["DATASTACK_CLIENT_ID"],
|
||||||
|
"",
|
||||||
|
)
|
||||||
|
).strip()
|
||||||
|
|
||||||
|
cfg = DataStackConfig(
|
||||||
|
server_url=server_url,
|
||||||
|
api_url=api_url,
|
||||||
|
client_token=client_token,
|
||||||
|
client_id=client_id,
|
||||||
|
auto_connect=to_bool(
|
||||||
|
pick(["AUTO_CONNECT"], ["DATASTACK_AUTO_CONNECT"], True), True
|
||||||
|
),
|
||||||
|
enable_http_fallback=to_bool(
|
||||||
|
pick(
|
||||||
|
["ENABLE_HTTP_FALLBACK", "HTTP_FALLBACK"],
|
||||||
|
["DATASTACK_ENABLE_HTTP_FALLBACK"],
|
||||||
|
True,
|
||||||
|
),
|
||||||
|
True,
|
||||||
|
),
|
||||||
|
heartbeat_interval_sec=max(
|
||||||
|
1.0,
|
||||||
|
float(
|
||||||
|
pick(
|
||||||
|
["HEARTBEAT_INTERVAL_SEC", "HEARTBEAT_INTERVAL"],
|
||||||
|
["DATASTACK_HEARTBEAT_INTERVAL_SEC"],
|
||||||
|
5.0,
|
||||||
|
)
|
||||||
|
),
|
||||||
|
),
|
||||||
|
request_timeout_sec=max(
|
||||||
|
1.0,
|
||||||
|
float(
|
||||||
|
pick(
|
||||||
|
["REQUEST_TIMEOUT_SEC", "TIMEOUT"],
|
||||||
|
["DATASTACK_REQUEST_TIMEOUT_SEC"],
|
||||||
|
8.0,
|
||||||
|
)
|
||||||
|
),
|
||||||
|
),
|
||||||
|
max_retries=max(
|
||||||
|
0,
|
||||||
|
int(
|
||||||
|
pick(
|
||||||
|
["MAX_RETRIES", "RETRIES"],
|
||||||
|
["DATASTACK_MAX_RETRIES"],
|
||||||
|
3,
|
||||||
|
)
|
||||||
|
),
|
||||||
|
),
|
||||||
|
retry_backoff_base_sec=max(
|
||||||
|
0.1,
|
||||||
|
float(
|
||||||
|
pick(
|
||||||
|
["RETRY_BACKOFF_BASE_SEC"],
|
||||||
|
["DATASTACK_RETRY_BACKOFF_BASE_SEC"],
|
||||||
|
0.5,
|
||||||
|
)
|
||||||
|
),
|
||||||
|
),
|
||||||
|
batch_chunk_size=max(
|
||||||
|
1,
|
||||||
|
int(
|
||||||
|
pick(
|
||||||
|
["BATCH_CHUNK_SIZE", "CHUNK_SIZE"],
|
||||||
|
["DATASTACK_BATCH_CHUNK_SIZE"],
|
||||||
|
250,
|
||||||
|
)
|
||||||
|
),
|
||||||
|
),
|
||||||
|
offline_spool_enabled=to_bool(
|
||||||
|
pick(
|
||||||
|
["OFFLINE_SPOOL_ENABLED"],
|
||||||
|
["DATASTACK_OFFLINE_SPOOL_ENABLED"],
|
||||||
|
True,
|
||||||
|
),
|
||||||
|
True,
|
||||||
|
),
|
||||||
|
offline_spool_dir=str(
|
||||||
|
pick(
|
||||||
|
["OFFLINE_SPOOL_DIR", "SPOOL_DIR"],
|
||||||
|
["DATASTACK_OFFLINE_SPOOL_DIR"],
|
||||||
|
".datastack_spool",
|
||||||
|
)
|
||||||
|
),
|
||||||
|
max_spool_size_mb=max(
|
||||||
|
1,
|
||||||
|
int(
|
||||||
|
pick(
|
||||||
|
["MAX_SPOOL_SIZE_MB"],
|
||||||
|
["DATASTACK_MAX_SPOOL_SIZE_MB"],
|
||||||
|
50,
|
||||||
|
)
|
||||||
|
),
|
||||||
|
),
|
||||||
|
config_source_path=resolved_source,
|
||||||
|
)
|
||||||
|
|
||||||
|
if require_token and (
|
||||||
|
not cfg.client_token
|
||||||
|
or cfg.client_token == "REPLACE_WITH_YOUR_CLIENT_JWT_TOKEN"
|
||||||
|
):
|
||||||
|
raise DataStackConfigError(
|
||||||
|
"DataStack CLIENT_TOKEN is missing. Add your token to `config.datastack.py`, "
|
||||||
|
"set DATASTACK_TOKEN in your environment, or pass token='...' to DataStack()."
|
||||||
|
)
|
||||||
|
|
||||||
|
return cfg
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
"""
|
||||||
|
Structured Exception Hierarchy for the DataStack Python SDK.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
|
||||||
|
class DataStackError(Exception):
|
||||||
|
"""Base exception for all DataStack SDK errors."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
message: str,
|
||||||
|
code: Optional[str] = None,
|
||||||
|
retryable: bool = False,
|
||||||
|
) -> None:
|
||||||
|
super().__init__(message)
|
||||||
|
self.message = message
|
||||||
|
self.code = code or "DATASTACK_ERROR"
|
||||||
|
self.retryable = retryable
|
||||||
|
|
||||||
|
|
||||||
|
class DataStackConfigError(DataStackError):
|
||||||
|
"""Raised when `config.datastack.py` or required credentials are missing or invalid."""
|
||||||
|
|
||||||
|
def __init__(self, message: str) -> None:
|
||||||
|
super().__init__(message, code="CONFIG_ERROR", retryable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class DataStackAuthError(DataStackError):
|
||||||
|
"""
|
||||||
|
Raised when authentication fails (`AUTH_REQUIRED`, `AUTH_INVALID`,
|
||||||
|
`CLIENT_REVOKED`, or `TOKEN_ROTATED`). Non-retryable until credentials are updated.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, message: str, code: str = "AUTH_ERROR") -> None:
|
||||||
|
super().__init__(message, code=code, retryable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class DataStackRateLimitError(DataStackError):
|
||||||
|
"""Raised when `RATE_LIMIT_EXCEEDED` persists after all backoff retries."""
|
||||||
|
|
||||||
|
def __init__(self, message: str) -> None:
|
||||||
|
super().__init__(message, code="RATE_LIMIT_EXCEEDED", retryable=True)
|
||||||
|
|
||||||
|
|
||||||
|
class DataStackPayloadTooLargeError(DataStackError):
|
||||||
|
"""Raised when a single event exceeds the server's `maxPayloadSizeKb` limit."""
|
||||||
|
|
||||||
|
def __init__(self, message: str) -> None:
|
||||||
|
super().__init__(message, code="PAYLOAD_TOO_LARGE", retryable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class DataStackConnectionError(DataStackError):
|
||||||
|
"""Raised when both WebSocket and HTTP fallback are unreachable and offline spooling is disabled."""
|
||||||
|
|
||||||
|
def __init__(self, message: str) -> None:
|
||||||
|
super().__init__(message, code="CONNECTION_ERROR", retryable=True)
|
||||||
|
|
||||||
|
|
||||||
|
class DataStackFileError(DataStackError):
|
||||||
|
"""Raised when a file passed to `send_file()` is missing, unsupported, or malformed."""
|
||||||
|
|
||||||
|
def __init__(self, message: str) -> None:
|
||||||
|
super().__init__(message, code="FILE_ERROR", retryable=False)
|
||||||
|
|
||||||
|
|
||||||
|
class DataStackOfflineSpoolError(DataStackError):
|
||||||
|
"""Raised when the local offline disk spool directory is full or unwriteable."""
|
||||||
|
|
||||||
|
def __init__(self, message: str) -> None:
|
||||||
|
super().__init__(message, code="OFFLINE_SPOOL_ERROR", retryable=False)
|
||||||
@@ -0,0 +1,186 @@
|
|||||||
|
"""
|
||||||
|
Smart Heartbeat Monitor & Connection State Machine (`HeartbeatMonitor`).
|
||||||
|
|
||||||
|
Lifecycle States:
|
||||||
|
- `CONNECTED_IDLE`: WebSocket (Port 4000) is connected and verified. Heartbeat sleeps in IDLE mode.
|
||||||
|
- `FALLBACK_HTTP`: WebSocket disconnected. Client automatically sends events via HTTP (`POST /api/events`)
|
||||||
|
while Heartbeat actively probes to reconnect and verify WebSocket, then returns to `CONNECTED_IDLE`.
|
||||||
|
- `OFFLINE_SPOOL`: Device is completely offline. Events spool to disk (`.datastack_spool/spool.jsonl`).
|
||||||
|
Heartbeat monitors network restoration, reconnects, replays spooled events, and returns to `CONNECTED_IDLE`.
|
||||||
|
- `RECONNECTING_VERIFY`: Heartbeat is verifying handshake (`ping_verify`) and draining spooled events.
|
||||||
|
- `STOPPED`: Client was closed intentionally or halted due to non-retryable auth revocation.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from typing import Any, Callable, Dict, Optional
|
||||||
|
|
||||||
|
logger = logging.getLogger("datastack.heartbeat")
|
||||||
|
|
||||||
|
|
||||||
|
class ConnectionState:
|
||||||
|
CONNECTED_IDLE = "CONNECTED_IDLE"
|
||||||
|
FALLBACK_HTTP = "FALLBACK_HTTP"
|
||||||
|
OFFLINE_SPOOL = "OFFLINE_SPOOL"
|
||||||
|
RECONNECTING_VERIFY = "RECONNECTING_VERIFY"
|
||||||
|
STOPPED = "STOPPED"
|
||||||
|
|
||||||
|
|
||||||
|
class HeartbeatMonitor:
|
||||||
|
"""
|
||||||
|
Background recovery thread that sleeps in zero-CPU IDLE mode while WebSocket is
|
||||||
|
healthy (`CONNECTED_IDLE`) and wakes up automatically when WebSocket disconnects
|
||||||
|
or when the client device goes offline.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
interval_sec: float,
|
||||||
|
recovery_callback: Callable[[], bool],
|
||||||
|
) -> None:
|
||||||
|
self.interval_sec = max(0.2, float(interval_sec))
|
||||||
|
self._recovery_callback = recovery_callback
|
||||||
|
self._state = ConnectionState.CONNECTED_IDLE
|
||||||
|
self._closed = False
|
||||||
|
self._last_error: Optional[str] = None
|
||||||
|
self._recovery_attempts = 0
|
||||||
|
self._last_verified_at: Optional[float] = None
|
||||||
|
|
||||||
|
self._lock = threading.RLock()
|
||||||
|
self._wake_event = threading.Event()
|
||||||
|
self._stop_event = threading.Event()
|
||||||
|
self._thread: Optional[threading.Thread] = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def state(self) -> str:
|
||||||
|
with self._lock:
|
||||||
|
return self._state
|
||||||
|
|
||||||
|
@property
|
||||||
|
def is_idle(self) -> bool:
|
||||||
|
with self._lock:
|
||||||
|
return self._state == ConnectionState.CONNECTED_IDLE
|
||||||
|
|
||||||
|
def set_idle_connected(self) -> None:
|
||||||
|
"""
|
||||||
|
Marks the connection as healthy (`CONNECTED_IDLE`) and puts the Heartbeat
|
||||||
|
Monitor into IDLE mode (`_wake_event.clear()`).
|
||||||
|
"""
|
||||||
|
with self._lock:
|
||||||
|
self._closed = False
|
||||||
|
self._state = ConnectionState.CONNECTED_IDLE
|
||||||
|
self._recovery_attempts = 0
|
||||||
|
self._last_error = None
|
||||||
|
self._last_verified_at = time.time()
|
||||||
|
self._wake_event.clear()
|
||||||
|
|
||||||
|
def trigger_fallback(self, reason: str = "WebSocket disconnected") -> None:
|
||||||
|
"""
|
||||||
|
Transitions to `FALLBACK_HTTP` and wakes up the Heartbeat Monitor thread
|
||||||
|
to repair the WebSocket connection in the background.
|
||||||
|
"""
|
||||||
|
with self._lock:
|
||||||
|
if self._closed:
|
||||||
|
return
|
||||||
|
self._state = ConnectionState.FALLBACK_HTTP
|
||||||
|
self._last_error = reason
|
||||||
|
self._ensure_thread_started()
|
||||||
|
self._wake_event.set()
|
||||||
|
|
||||||
|
def trigger_offline(self, reason: str = "Device offline") -> None:
|
||||||
|
"""
|
||||||
|
Transitions to `OFFLINE_SPOOL` when both WebSocket and HTTP fallback are
|
||||||
|
unreachable, waking the Heartbeat Monitor to watch for network return.
|
||||||
|
"""
|
||||||
|
with self._lock:
|
||||||
|
if self._closed:
|
||||||
|
return
|
||||||
|
self._state = ConnectionState.OFFLINE_SPOOL
|
||||||
|
self._last_error = reason
|
||||||
|
self._ensure_thread_started()
|
||||||
|
self._wake_event.set()
|
||||||
|
|
||||||
|
def set_reconnecting(self) -> None:
|
||||||
|
with self._lock:
|
||||||
|
if not self._closed:
|
||||||
|
self._state = ConnectionState.RECONNECTING_VERIFY
|
||||||
|
|
||||||
|
def start(self) -> None:
|
||||||
|
with self._lock:
|
||||||
|
self._closed = False
|
||||||
|
self._stop_event.clear()
|
||||||
|
self._ensure_thread_started()
|
||||||
|
|
||||||
|
def stop(self, reason: Optional[str] = None) -> None:
|
||||||
|
with self._lock:
|
||||||
|
self._closed = True
|
||||||
|
self._state = ConnectionState.STOPPED
|
||||||
|
if reason:
|
||||||
|
self._last_error = reason
|
||||||
|
self._stop_event.set()
|
||||||
|
self._wake_event.set()
|
||||||
|
|
||||||
|
def status_snapshot(self) -> Dict[str, Any]:
|
||||||
|
with self._lock:
|
||||||
|
return {
|
||||||
|
"state": self._state,
|
||||||
|
"heartbeatMode": (
|
||||||
|
"IDLE"
|
||||||
|
if self._state == ConnectionState.CONNECTED_IDLE
|
||||||
|
else "ACTIVE_RECOVERY"
|
||||||
|
if self._state
|
||||||
|
in (
|
||||||
|
ConnectionState.FALLBACK_HTTP,
|
||||||
|
ConnectionState.OFFLINE_SPOOL,
|
||||||
|
ConnectionState.RECONNECTING_VERIFY,
|
||||||
|
)
|
||||||
|
else "STOPPED"
|
||||||
|
),
|
||||||
|
"recoveryAttempts": self._recovery_attempts,
|
||||||
|
"lastError": self._last_error,
|
||||||
|
"lastVerifiedAt": self._last_verified_at,
|
||||||
|
"intervalSec": self.interval_sec,
|
||||||
|
}
|
||||||
|
|
||||||
|
def _ensure_thread_started(self) -> None:
|
||||||
|
if self._thread is not None and self._thread.is_alive():
|
||||||
|
return
|
||||||
|
self._stop_event.clear()
|
||||||
|
self._thread = threading.Thread(
|
||||||
|
target=self._run_loop,
|
||||||
|
name="DataStack-HeartbeatMonitor",
|
||||||
|
daemon=True,
|
||||||
|
)
|
||||||
|
self._thread.start()
|
||||||
|
|
||||||
|
def _run_loop(self) -> None:
|
||||||
|
while not self._stop_event.is_set():
|
||||||
|
# Wait in zero-CPU IDLE mode until woken by a disconnect or offline trigger
|
||||||
|
self._wake_event.wait()
|
||||||
|
if self._stop_event.is_set():
|
||||||
|
break
|
||||||
|
|
||||||
|
# Wait for the heartbeat interval before probing
|
||||||
|
if self._stop_event.wait(self.interval_sec):
|
||||||
|
break
|
||||||
|
|
||||||
|
if not self._wake_event.is_set():
|
||||||
|
continue
|
||||||
|
|
||||||
|
with self._lock:
|
||||||
|
if self._closed or self._state == ConnectionState.STOPPED:
|
||||||
|
break
|
||||||
|
self._recovery_attempts += 1
|
||||||
|
|
||||||
|
try:
|
||||||
|
recovered_to_idle = self._recovery_callback()
|
||||||
|
if recovered_to_idle:
|
||||||
|
self.set_idle_connected()
|
||||||
|
logger.info(
|
||||||
|
"[DataStack Heartbeat] WebSocket reconnected & verified. Returning to IDLE mode."
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
with self._lock:
|
||||||
|
self._last_error = str(exc)
|
||||||
|
logger.debug("[DataStack Heartbeat] Recovery probe failed: %s", exc)
|
||||||
@@ -0,0 +1,164 @@
|
|||||||
|
"""
|
||||||
|
Thread-Safe Offline Disk Spooler (`DiskSpooler`).
|
||||||
|
Persists unsent telemetry events to `.datastack_spool/spool.jsonl` when a client
|
||||||
|
device loses network connectivity completely, and drains them in deterministic
|
||||||
|
batches once the Heartbeat Monitor restores connectivity.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import threading
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
|
from .exceptions import DataStackOfflineSpoolError
|
||||||
|
|
||||||
|
|
||||||
|
class DiskSpooler:
|
||||||
|
"""
|
||||||
|
Append-only JSONL disk queue with atomic batch drainage and size cap protection.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
spool_dir: str = ".datastack_spool",
|
||||||
|
max_size_mb: int = 50,
|
||||||
|
enabled: bool = True,
|
||||||
|
) -> None:
|
||||||
|
self.enabled = enabled
|
||||||
|
self.spool_dir = Path(spool_dir)
|
||||||
|
self.spool_file = self.spool_dir / "spool.jsonl"
|
||||||
|
self.max_bytes = max(1, int(max_size_mb)) * 1024 * 1024
|
||||||
|
self._lock = threading.RLock()
|
||||||
|
|
||||||
|
def _ensure_dir(self) -> None:
|
||||||
|
try:
|
||||||
|
self.spool_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
except Exception as exc:
|
||||||
|
raise DataStackOfflineSpoolError(
|
||||||
|
f"Cannot create offline spool directory '{self.spool_dir}': {exc}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
def enqueue(self, events: List[Dict[str, Any]]) -> int:
|
||||||
|
"""
|
||||||
|
Appends one or more formatted events (`{eventId, timestamp, data}`)
|
||||||
|
to `.datastack_spool/spool.jsonl`. Returns count of spooled events.
|
||||||
|
"""
|
||||||
|
if not self.enabled or not events:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
with self._lock:
|
||||||
|
self._ensure_dir()
|
||||||
|
current_size = (
|
||||||
|
self.spool_file.stat().st_size if self.spool_file.exists() else 0
|
||||||
|
)
|
||||||
|
|
||||||
|
lines: List[str] = []
|
||||||
|
added_bytes = 0
|
||||||
|
for evt in events:
|
||||||
|
serialized = json.dumps(evt, ensure_ascii=False) + "\n"
|
||||||
|
encoded_len = len(serialized.encode("utf-8"))
|
||||||
|
if current_size + added_bytes + encoded_len > self.max_bytes:
|
||||||
|
raise DataStackOfflineSpoolError(
|
||||||
|
f"Offline spool limit ({self.max_bytes // (1024 * 1024)} MB) "
|
||||||
|
f"reached at '{self.spool_file}'."
|
||||||
|
)
|
||||||
|
lines.append(serialized)
|
||||||
|
added_bytes += encoded_len
|
||||||
|
|
||||||
|
try:
|
||||||
|
with self.spool_file.open("a", encoding="utf-8") as fh:
|
||||||
|
fh.writelines(lines)
|
||||||
|
fh.flush()
|
||||||
|
except Exception as exc:
|
||||||
|
raise DataStackOfflineSpoolError(
|
||||||
|
f"Failed writing to offline spool file '{self.spool_file}': {exc}"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
return len(lines)
|
||||||
|
|
||||||
|
def peek_batch(self, limit: int = 250) -> List[Dict[str, Any]]:
|
||||||
|
"""
|
||||||
|
Reads up to `limit` spooled events from the front of the spool file
|
||||||
|
without removing them until `ack_batch()` confirms delivery.
|
||||||
|
"""
|
||||||
|
if not self.enabled:
|
||||||
|
return []
|
||||||
|
|
||||||
|
with self._lock:
|
||||||
|
if not self.spool_file.exists():
|
||||||
|
return []
|
||||||
|
|
||||||
|
batch: List[Dict[str, Any]] = []
|
||||||
|
try:
|
||||||
|
with self.spool_file.open("r", encoding="utf-8") as fh:
|
||||||
|
for line in fh:
|
||||||
|
if len(batch) >= limit:
|
||||||
|
break
|
||||||
|
stripped = line.strip()
|
||||||
|
if not stripped:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
parsed = json.loads(stripped)
|
||||||
|
if isinstance(parsed, dict):
|
||||||
|
batch.append(parsed)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
continue
|
||||||
|
except Exception:
|
||||||
|
return []
|
||||||
|
|
||||||
|
return batch
|
||||||
|
|
||||||
|
def ack_batch(self, count: int) -> int:
|
||||||
|
"""
|
||||||
|
Removes the first `count` non-empty lines from the spool file atomically
|
||||||
|
after the Heartbeat Monitor confirms they have been ingested by DataStack.
|
||||||
|
"""
|
||||||
|
if not self.enabled or count <= 0:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
with self._lock:
|
||||||
|
if not self.spool_file.exists():
|
||||||
|
return 0
|
||||||
|
|
||||||
|
try:
|
||||||
|
with self.spool_file.open("r", encoding="utf-8") as fh:
|
||||||
|
all_lines = [ln for ln in fh if ln.strip()]
|
||||||
|
|
||||||
|
if count >= len(all_lines):
|
||||||
|
self.spool_file.unlink(missing_ok=True)
|
||||||
|
return len(all_lines)
|
||||||
|
|
||||||
|
remaining = all_lines[count:]
|
||||||
|
tmp_file = self.spool_dir / "spool.jsonl.tmp"
|
||||||
|
with tmp_file.open("w", encoding="utf-8") as fh:
|
||||||
|
fh.writelines(remaining)
|
||||||
|
fh.flush()
|
||||||
|
os.replace(str(tmp_file), str(self.spool_file))
|
||||||
|
return count
|
||||||
|
except Exception:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def pending_count(self) -> int:
|
||||||
|
"""Returns the number of queued events waiting in the disk spool."""
|
||||||
|
if not self.enabled:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
with self._lock:
|
||||||
|
if not self.spool_file.exists():
|
||||||
|
return 0
|
||||||
|
try:
|
||||||
|
with self.spool_file.open("r", encoding="utf-8") as fh:
|
||||||
|
return sum(1 for ln in fh if ln.strip())
|
||||||
|
except Exception:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def size_bytes(self) -> int:
|
||||||
|
"""Returns current disk usage of the spool file in bytes."""
|
||||||
|
with self._lock:
|
||||||
|
if not self.spool_file.exists():
|
||||||
|
return 0
|
||||||
|
try:
|
||||||
|
return self.spool_file.stat().st_size
|
||||||
|
except Exception:
|
||||||
|
return 0
|
||||||
@@ -288,12 +288,12 @@ class DataStackClient:
|
|||||||
)
|
)
|
||||||
|
|
||||||
with self._lock:
|
with self._lock:
|
||||||
|
self.is_connected = False
|
||||||
if self.sio.connected:
|
if self.sio.connected:
|
||||||
try:
|
try:
|
||||||
self.sio.disconnect()
|
self.sio.disconnect()
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
self.is_connected = False
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
self.sio.connect(
|
self.sio.connect(
|
||||||
@@ -699,7 +699,7 @@ class DataStackClient:
|
|||||||
Replays any offline spooled events from `.datastack_spool/spool.jsonl`
|
Replays any offline spooled events from `.datastack_spool/spool.jsonl`
|
||||||
once connectivity (WebSocket or HTTP fallback) is restored.
|
once connectivity (WebSocket or HTTP fallback) is restored.
|
||||||
Routes batches through `_send_formatted_batch_recursive(..., allow_spool=False)`
|
Routes batches through `_send_formatted_batch_recursive(..., allow_spool=False)`
|
||||||
so oversized or rate-limited spool batches automatically split and pace.
|
and preserves any `unprocessedItems` so zero spooled events are ever lost.
|
||||||
"""
|
"""
|
||||||
if not self.config.offline_spool_enabled:
|
if not self.config.offline_spool_enabled:
|
||||||
return 0
|
return 0
|
||||||
@@ -724,10 +724,23 @@ class DataStackClient:
|
|||||||
file_name=None,
|
file_name=None,
|
||||||
allow_spool=False,
|
allow_spool=False,
|
||||||
)
|
)
|
||||||
if not res.get("success", False):
|
processed_now = int(res.get("processed", 0))
|
||||||
|
duplicates_now = int(res.get("duplicates", 0))
|
||||||
|
unprocessed = res.get("unprocessedItems")
|
||||||
|
|
||||||
|
if (
|
||||||
|
not res.get("success", False)
|
||||||
|
and processed_now == 0
|
||||||
|
and duplicates_now == 0
|
||||||
|
):
|
||||||
break
|
break
|
||||||
|
|
||||||
self.spooler.ack_batch(len(batch))
|
self.spooler.ack_batch(len(batch))
|
||||||
|
if isinstance(unprocessed, list) and len(unprocessed) > 0:
|
||||||
|
self.spooler.enqueue(unprocessed)
|
||||||
|
total_flushed += max(0, len(batch) - len(unprocessed))
|
||||||
|
break
|
||||||
|
|
||||||
total_flushed += len(batch)
|
total_flushed += len(batch)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.debug("[DataStack] Spool drain paused: %s", exc)
|
logger.debug("[DataStack] Spool drain paused: %s", exc)
|
||||||
@@ -869,6 +882,12 @@ class DataStackClient:
|
|||||||
file_name=file_name,
|
file_name=file_name,
|
||||||
allow_spool=allow_spool,
|
allow_spool=allow_spool,
|
||||||
)
|
)
|
||||||
|
left_unprocessed = left_res.get("unprocessedItems") or []
|
||||||
|
right_unprocessed = right_res.get("unprocessedItems") or []
|
||||||
|
combined_unprocessed = [
|
||||||
|
*(left_unprocessed if isinstance(left_unprocessed, list) else []),
|
||||||
|
*(right_unprocessed if isinstance(right_unprocessed, list) else []),
|
||||||
|
]
|
||||||
return {
|
return {
|
||||||
"success": bool(
|
"success": bool(
|
||||||
left_res.get("success", True)
|
left_res.get("success", True)
|
||||||
@@ -884,15 +903,16 @@ class DataStackClient:
|
|||||||
+ int(right_res.get("spooledCount", 0)),
|
+ int(right_res.get("spooledCount", 0)),
|
||||||
"transport": right_res.get("transport")
|
"transport": right_res.get("transport")
|
||||||
or left_res.get("transport"),
|
or left_res.get("transport"),
|
||||||
|
**(
|
||||||
|
{"unprocessedItems": combined_unprocessed}
|
||||||
|
if len(combined_unprocessed) > 0
|
||||||
|
else {}
|
||||||
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
# Handle partial-batch rate limiting (`rateLimitedEventIds`) so zero rows are dropped
|
# Handle partial-batch rate limiting (`rateLimitedEventIds`) so zero rows are dropped
|
||||||
rate_limited_ids = res.get("rateLimitedEventIds")
|
rate_limited_ids = res.get("rateLimitedEventIds")
|
||||||
if (
|
if isinstance(rate_limited_ids, list) and len(rate_limited_ids) > 0:
|
||||||
isinstance(rate_limited_ids, list)
|
|
||||||
and len(rate_limited_ids) > 0
|
|
||||||
and self.config.max_retries > 0
|
|
||||||
):
|
|
||||||
pending_id_set = {str(eid) for eid in rate_limited_ids}
|
pending_id_set = {str(eid) for eid in rate_limited_ids}
|
||||||
remaining_items = [
|
remaining_items = [
|
||||||
item
|
item
|
||||||
@@ -904,56 +924,79 @@ class DataStackClient:
|
|||||||
total_spooled = int(res.get("spooledCount", 0))
|
total_spooled = int(res.get("spooledCount", 0))
|
||||||
last_transport = res.get("transport")
|
last_transport = res.get("transport")
|
||||||
|
|
||||||
for retry_round in range(1, self.config.max_retries + 2):
|
if self.config.max_retries > 0:
|
||||||
if not remaining_items:
|
for _retry_round in range(1, self.config.max_retries + 2):
|
||||||
break
|
if not remaining_items:
|
||||||
# Wait slightly over 1.0s so the server's 1-second rate-limit window resets
|
break
|
||||||
wait_delay = max(1.05, self.config.retry_backoff_base_sec) + random.uniform(
|
# Wait slightly over 1.0s so the server's 1-second rate-limit window resets
|
||||||
0.0, 0.15
|
wait_delay = max(
|
||||||
)
|
1.05, self.config.retry_backoff_base_sec
|
||||||
time.sleep(wait_delay)
|
) + random.uniform(0.0, 0.15)
|
||||||
|
time.sleep(wait_delay)
|
||||||
|
|
||||||
sub_http_payload: Dict[str, Any] = (
|
sub_http_payload: Dict[str, Any] = (
|
||||||
{"fileName": file_name, "records": remaining_items}
|
{"fileName": file_name, "records": remaining_items}
|
||||||
if file_name
|
if file_name
|
||||||
else {"batch": remaining_items}
|
else {"batch": remaining_items}
|
||||||
)
|
)
|
||||||
sub_res = self._execute_with_retry_and_fallback(
|
sub_res = self._execute_with_retry_and_fallback(
|
||||||
ws_event="batch_sync",
|
ws_event="batch_sync",
|
||||||
ws_payload=remaining_items,
|
ws_payload=remaining_items,
|
||||||
http_payload=sub_http_payload,
|
http_payload=sub_http_payload,
|
||||||
spool_items=remaining_items,
|
spool_items=remaining_items,
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
allow_spool=allow_spool,
|
allow_spool=allow_spool,
|
||||||
)
|
)
|
||||||
total_processed += int(sub_res.get("processed", 0))
|
total_processed += int(sub_res.get("processed", 0))
|
||||||
total_duplicates += int(sub_res.get("duplicates", 0))
|
total_duplicates += int(sub_res.get("duplicates", 0))
|
||||||
total_spooled += int(sub_res.get("spooledCount", 0))
|
total_spooled += int(sub_res.get("spooledCount", 0))
|
||||||
last_transport = sub_res.get("transport") or last_transport
|
last_transport = sub_res.get("transport") or last_transport
|
||||||
|
|
||||||
next_rl_ids = sub_res.get("rateLimitedEventIds")
|
if sub_res.get("spooled"):
|
||||||
if not isinstance(next_rl_ids, list) or len(next_rl_ids) == 0:
|
remaining_items = []
|
||||||
remaining_items = []
|
break
|
||||||
break
|
|
||||||
next_set = {str(eid) for eid in next_rl_ids}
|
next_rl_ids = sub_res.get("rateLimitedEventIds")
|
||||||
remaining_items = [
|
if (
|
||||||
item
|
not isinstance(next_rl_ids, list)
|
||||||
for item in remaining_items
|
or len(next_rl_ids) == 0
|
||||||
if str(item.get("eventId")) in next_set
|
):
|
||||||
]
|
remaining_items = []
|
||||||
|
break
|
||||||
|
next_set = {str(eid) for eid in next_rl_ids}
|
||||||
|
remaining_items = [
|
||||||
|
item
|
||||||
|
for item in remaining_items
|
||||||
|
if str(item.get("eventId")) in next_set
|
||||||
|
]
|
||||||
|
|
||||||
|
# If any items still remain rate-limited after retries and spooling is allowed, spool them so zero data is lost
|
||||||
|
if (
|
||||||
|
remaining_items
|
||||||
|
and allow_spool
|
||||||
|
and self.config.offline_spool_enabled
|
||||||
|
):
|
||||||
|
spooled_now = self.spooler.enqueue(remaining_items)
|
||||||
|
total_spooled += spooled_now
|
||||||
|
remaining_items = []
|
||||||
|
|
||||||
oversized_ids = res.get("oversizedEventIds")
|
oversized_ids = res.get("oversizedEventIds")
|
||||||
final_rejected = len(remaining_items) + (
|
final_rejected = len(remaining_items) + (
|
||||||
len(oversized_ids) if isinstance(oversized_ids, list) else 0
|
len(oversized_ids) if isinstance(oversized_ids, list) else 0
|
||||||
)
|
)
|
||||||
return {
|
out: Dict[str, Any] = {
|
||||||
"success": final_rejected == 0 or total_processed > 0,
|
"success": final_rejected == 0
|
||||||
|
or total_processed > 0
|
||||||
|
or total_spooled > 0,
|
||||||
"processed": total_processed,
|
"processed": total_processed,
|
||||||
"rejected": final_rejected,
|
"rejected": final_rejected,
|
||||||
"duplicates": total_duplicates,
|
"duplicates": total_duplicates,
|
||||||
"spooledCount": total_spooled,
|
"spooledCount": total_spooled,
|
||||||
"transport": last_transport,
|
"transport": last_transport,
|
||||||
}
|
}
|
||||||
|
if remaining_items:
|
||||||
|
out["unprocessedItems"] = remaining_items
|
||||||
|
return out
|
||||||
|
|
||||||
return res
|
return res
|
||||||
|
|
||||||
|
|||||||
@@ -289,6 +289,18 @@ app.post('/internal/flush', async (_req, res) => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Internal endpoint to expose Port 4000 buffered events before CSV flush
|
||||||
|
app.get('/internal/recent', (req, res) => {
|
||||||
|
const clientId = req.query.clientId ? String(req.query.clientId) : '';
|
||||||
|
const limit = Math.min(200, Math.max(1, Number(req.query.limit) || 50));
|
||||||
|
const filtered =
|
||||||
|
clientId && clientId !== 'all'
|
||||||
|
? bufferQueue.filter((evt) => evt.clientId === clientId)
|
||||||
|
: bufferQueue;
|
||||||
|
const recent = filtered.slice(-limit).reverse();
|
||||||
|
res.json({ events: recent, count: recent.length });
|
||||||
|
});
|
||||||
|
|
||||||
setupSocketHandlers(io, bufferQueue, writer, {
|
setupSocketHandlers(io, bufferQueue, writer, {
|
||||||
getRuntimeConfig: () => runtimeConfig,
|
getRuntimeConfig: () => runtimeConfig,
|
||||||
verifyClientInDb,
|
verifyClientInDb,
|
||||||
|
|||||||
+21
-17
@@ -65,10 +65,10 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
const now = Date.now();
|
const now = Date.now();
|
||||||
cleanupMaps(now);
|
cleanupMaps(now);
|
||||||
|
|
||||||
// 1. Idempotency check (with 5-minute TTL)
|
// 1. Idempotency check (read-only check first; record only after rate-limit & payload checks pass)
|
||||||
|
const idempotencyKey = `${clientId}:${eventId}`;
|
||||||
if (config.enableIdempotency && eventId) {
|
if (config.enableIdempotency && eventId) {
|
||||||
const key = `${clientId}:${eventId}`;
|
const existingTime = idempotencyMap.get(idempotencyKey);
|
||||||
const existingTime = idempotencyMap.get(key);
|
|
||||||
if (existingTime && now - existingTime < IDEMPOTENCY_TTL_MS) {
|
if (existingTime && now - existingTime < IDEMPOTENCY_TTL_MS) {
|
||||||
return {
|
return {
|
||||||
allowed: false,
|
allowed: false,
|
||||||
@@ -77,10 +77,23 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
reason: "Duplicate event dropped",
|
reason: "Duplicate event dropped",
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
idempotencyMap.set(key, now);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// 2. Rate limit check
|
// 2. Payload byte size check (before consuming rate-limit quota)
|
||||||
|
const payloadStr =
|
||||||
|
typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {});
|
||||||
|
const byteLength = Buffer.byteLength(payloadStr, "utf8");
|
||||||
|
const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024;
|
||||||
|
if (byteLength > maxBytes) {
|
||||||
|
return {
|
||||||
|
allowed: false,
|
||||||
|
duplicate: false,
|
||||||
|
code: "PAYLOAD_TOO_LARGE",
|
||||||
|
reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Rate limit check
|
||||||
const effectiveLimit = Math.min(
|
const effectiveLimit = Math.min(
|
||||||
Number(clientRateLimit) || 100,
|
Number(clientRateLimit) || 100,
|
||||||
Number(config.maxEventsPerSec) || 100,
|
Number(config.maxEventsPerSec) || 100,
|
||||||
@@ -99,18 +112,9 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
rl.count += 1;
|
rl.count += 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3. Payload byte size check
|
// 4. Record idempotency key ONLY after event passes all checks
|
||||||
const payloadStr =
|
if (config.enableIdempotency && eventId) {
|
||||||
typeof rawData === "string" ? rawData : JSON.stringify(rawData ?? {});
|
idempotencyMap.set(idempotencyKey, now);
|
||||||
const byteLength = Buffer.byteLength(payloadStr, "utf8");
|
|
||||||
const maxBytes = Number(config.maxPayloadSizeBytes) || 100 * 1024;
|
|
||||||
if (byteLength > maxBytes) {
|
|
||||||
return {
|
|
||||||
allowed: false,
|
|
||||||
duplicate: false,
|
|
||||||
code: "PAYLOAD_TOO_LARGE",
|
|
||||||
reason: `PAYLOAD_TOO_LARGE: Payload (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${(maxBytes / 1024).toFixed(0)} KB`,
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return { allowed: true, payloadStr };
|
return { allowed: true, payloadStr };
|
||||||
|
|||||||
@@ -74,7 +74,13 @@ class CsvWriterWorker {
|
|||||||
for (const [key, batch] of grouped.entries()) {
|
for (const [key, batch] of grouped.entries()) {
|
||||||
const [clientId, date] = key.split(':::');
|
const [clientId, date] = key.split(':::');
|
||||||
const targetPath = this.getTargetFilePath(clientId, date);
|
const targetPath = this.getTargetFilePath(clientId, date);
|
||||||
const fileExists = fs.existsSync(targetPath);
|
let hasContent = false;
|
||||||
|
try {
|
||||||
|
hasContent =
|
||||||
|
fs.existsSync(targetPath) && fs.statSync(targetPath).size > 0;
|
||||||
|
} catch {
|
||||||
|
hasContent = false;
|
||||||
|
}
|
||||||
|
|
||||||
const cleanBatch = batch.map((item) => ({
|
const cleanBatch = batch.map((item) => ({
|
||||||
clientId: item.clientId,
|
clientId: item.clientId,
|
||||||
@@ -83,10 +89,10 @@ class CsvWriterWorker {
|
|||||||
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
|
payload: typeof item.payload === 'string' ? item.payload : JSON.stringify(item.payload),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
const parser = new Parser({ fields, header: !fileExists, eol: '\n' });
|
const parser = new Parser({ fields, header: !hasContent, eol: '\n' });
|
||||||
const csv = parser.parse(cleanBatch);
|
const csv = parser.parse(cleanBatch);
|
||||||
|
|
||||||
if (fileExists) {
|
if (hasContent) {
|
||||||
fs.appendFileSync(targetPath, '\n' + csv, 'utf8');
|
fs.appendFileSync(targetPath, '\n' + csv, 'utf8');
|
||||||
} else {
|
} else {
|
||||||
fs.writeFileSync(targetPath, csv, 'utf8');
|
fs.writeFileSync(targetPath, csv, 'utf8');
|
||||||
|
|||||||
Reference in new issue
Block a user