import { parse as parseCsvSync } from 'csv-parse/sync'; import crypto from 'node:crypto'; import fs from 'node:fs'; import path from 'node:path'; import { io, type Socket } from 'socket.io-client'; import { DiskSpooler } from './spool'; import { DataStackAuthError, DataStackConfigError, DataStackError, DataStackFileError, DataStackPayloadTooLargeError, DataStackQuotaExceededError, DataStackRateLimitError, DataStackShutdownTimeoutError, type DataStackConfigOptions, type DataStackConnectionState, type DataStackEnvelope, type DataStackEventPayload, type DataStackFileSendResult, type DataStackSendResult, type DataStackStatusSnapshot, type ResolvedDataStackConfig, } from './types'; export * from './spool'; export * from './types'; export const SDK_VERSION = '2.2.0'; /** * Matches server `SDKManager.normalizeClientId()` and Python SDK `_normalize_client_id()`. */ export function normalizeClientId(value?: string | null): string { const raw = String(value ?? '') .trim() .toLowerCase(); if (!raw) return ''; return raw.replace(/[^a-z0-9_-]/g, '_'); } /** * Extracts `clientId` or `sub` claim from an unverified JWT payload string. */ export function decodeJwtClientId(token?: string | null): string { try { const parts = String(token ?? '').split('.'); if (parts.length !== 3) return ''; const json = Buffer.from(parts[1], 'base64url').toString('utf8'); const claims = JSON.parse(json) as Record; if (claims && typeof claims === 'object') { return String(claims.clientId || claims.sub || '').trim(); } } catch { // Ignore malformed JWT } return ''; } export function makeEventId(): string { return `evt_${Date.now()}_${crypto.randomBytes(6).toString('hex')}`; } /** * Generates a deterministic SHA-256 `eventId` for file rows (1:1 parity with Python SDK) * so re-ingesting the same `.csv`, `.json`, or `.jsonl` file is idempotently deduplicated. */ export function makeDeterministicFileEventId( fileStem: string, rowIndex: number, record: DataStackEventPayload ): string { const sortedKeys = Object.keys(record).sort(); const canonicalObj: Record = {}; for (const key of sortedKeys) { canonicalObj[key] = record[key]; } const raw = `${fileStem}:${rowIndex}:${JSON.stringify(canonicalObj)}`; const digest = crypto.createHash('sha256').update(raw, 'utf8').digest('hex').slice(0, 12); const safeStem = fileStem.replace(/[^a-zA-Z0-9_-]/g, '_').slice(0, 32); return `file_${safeStem}_${rowIndex}_${digest}`; } function loadJsonConfigFile(configPath?: string): Partial { const target = configPath ? path.resolve(configPath) : path.resolve(process.cwd(), 'datastack.config.json'); if (!fs.existsSync(target)) { if (configPath) { throw new DataStackConfigError(`Configuration file not found: ${target}`); } return {}; } try { const parsed = JSON.parse(fs.readFileSync(target, 'utf8')) as Record; return { serverUrl: typeof parsed.serverUrl === 'string' ? parsed.serverUrl : typeof parsed.SERVER_URL === 'string' ? parsed.SERVER_URL : undefined, apiUrl: typeof parsed.apiUrl === 'string' ? parsed.apiUrl : typeof parsed.API_URL === 'string' ? parsed.API_URL : undefined, clientToken: typeof parsed.clientToken === 'string' ? parsed.clientToken : typeof parsed.CLIENT_TOKEN === 'string' ? parsed.CLIENT_TOKEN : typeof parsed.token === 'string' ? parsed.token : undefined, clientId: typeof parsed.clientId === 'string' ? parsed.clientId : typeof parsed.CLIENT_ID === 'string' ? parsed.CLIENT_ID : undefined, autoConnect: typeof parsed.autoConnect === 'boolean' ? parsed.autoConnect : typeof parsed.AUTO_CONNECT === 'boolean' ? parsed.AUTO_CONNECT : undefined, enableHttpFallback: typeof parsed.enableHttpFallback === 'boolean' ? parsed.enableHttpFallback : typeof parsed.ENABLE_HTTP_FALLBACK === 'boolean' ? parsed.ENABLE_HTTP_FALLBACK : undefined, heartbeatIntervalMs: typeof parsed.heartbeatIntervalMs === 'number' ? parsed.heartbeatIntervalMs : undefined, maxRetries: typeof parsed.maxRetries === 'number' ? parsed.maxRetries : typeof parsed.MAX_RETRIES === 'number' ? parsed.MAX_RETRIES : undefined, batchChunkSize: typeof parsed.batchChunkSize === 'number' ? parsed.batchChunkSize : typeof parsed.BATCH_CHUNK_SIZE === 'number' ? parsed.BATCH_CHUNK_SIZE : undefined, offlineSpoolEnabled: typeof parsed.offlineSpoolEnabled === 'boolean' ? parsed.offlineSpoolEnabled : typeof parsed.OFFLINE_SPOOL_ENABLED === 'boolean' ? parsed.OFFLINE_SPOOL_ENABLED : undefined, offlineSpoolDir: typeof parsed.offlineSpoolDir === 'string' ? parsed.offlineSpoolDir : typeof parsed.OFFLINE_SPOOL_DIR === 'string' ? parsed.OFFLINE_SPOOL_DIR : undefined, maxSpoolSizeMb: typeof parsed.maxSpoolSizeMb === 'number' ? parsed.maxSpoolSizeMb : undefined, requestTimeoutMs: typeof parsed.requestTimeoutMs === 'number' ? parsed.requestTimeoutMs : undefined, }; } catch (err) { throw new DataStackConfigError( `Failed to parse datastack.config.json (${target}): ${err instanceof Error ? err.message : String(err)}` ); } } export class DataStackClient { public readonly config: ResolvedDataStackConfig; public readonly clientId: string; public readonly spooler: DiskSpooler; private socket: Socket | null = null; private isConnected = false; private transportState: DataStackConnectionState = 'DISCONNECTED'; private inFlightSet = new Set>(); private isDrainingSpool = false; private readonly fetchFn: typeof fetch; constructor(options: DataStackConfigOptions = {}) { const fileCfg = loadJsonConfigFile(options.configPath); const serverUrl = String( options.serverUrl ?? fileCfg.serverUrl ?? process.env.DATASTACK_SERVER_URL ?? 'http://localhost:4000' ) .trim() .replace(/\/+$/, ''); const apiUrl = String( options.apiUrl ?? fileCfg.apiUrl ?? process.env.DATASTACK_API_URL ?? 'http://localhost:3000' ) .trim() .replace(/\/+$/, ''); const clientToken = String( options.token ?? options.clientToken ?? fileCfg.clientToken ?? process.env.DATASTACK_CLIENT_TOKEN ?? '' ).trim(); if (!/^https?:\/\//i.test(serverUrl)) { throw new DataStackConfigError( `Invalid serverUrl '${serverUrl}'. Expected http:// or https:// URL.` ); } if (!/^https?:\/\//i.test(apiUrl)) { throw new DataStackConfigError( `Invalid apiUrl '${apiUrl}'. Expected http:// or https:// URL.` ); } const configuredClientId = String( options.clientId ?? fileCfg.clientId ?? process.env.DATASTACK_CLIENT_ID ?? '' ).trim(); const tokenClientId = decodeJwtClientId(clientToken); const normConfigured = normalizeClientId(configuredClientId); const normToken = normalizeClientId(tokenClientId); if (normConfigured && normToken && normConfigured !== normToken) { throw new DataStackConfigError( `Configured clientId '${configuredClientId}' does not match JWT token subject '${tokenClientId}'.` ); } const batchChunkSize = Number( options.batchChunkSize ?? fileCfg.batchChunkSize ?? 250 ); if (!Number.isFinite(batchChunkSize) || batchChunkSize < 1 || batchChunkSize > 1000) { throw new DataStackConfigError( `Invalid batchChunkSize '${batchChunkSize}'. Must be between 1 and 1000.` ); } const maxRetries = Number(options.maxRetries ?? fileCfg.maxRetries ?? 3); if (!Number.isFinite(maxRetries) || maxRetries < 0 || maxRetries > 20) { throw new DataStackConfigError( `Invalid maxRetries '${maxRetries}'. Must be between 0 and 20.` ); } const requestTimeoutMs = Number( options.requestTimeoutMs ?? fileCfg.requestTimeoutMs ?? 5000 ); if (!Number.isFinite(requestTimeoutMs) || requestTimeoutMs < 100) { throw new DataStackConfigError( `Invalid requestTimeoutMs '${requestTimeoutMs}'. Must be >= 100ms.` ); } this.clientId = normToken || normConfigured; this.config = { serverUrl, apiUrl, clientToken, clientId: this.clientId, autoConnect: Boolean(options.autoConnect ?? fileCfg.autoConnect ?? false), enableHttpFallback: options.enableHttpFallback ?? fileCfg.enableHttpFallback ?? true, heartbeatIntervalMs: Number( options.heartbeatIntervalMs ?? fileCfg.heartbeatIntervalMs ?? 5000 ), maxRetries, batchChunkSize: Math.floor(batchChunkSize), offlineSpoolEnabled: options.offlineSpoolEnabled ?? fileCfg.offlineSpoolEnabled ?? true, offlineSpoolDir: String( options.offlineSpoolDir ?? fileCfg.offlineSpoolDir ?? '.datastack_spool' ), maxSpoolSizeMb: Number( options.maxSpoolSizeMb ?? fileCfg.maxSpoolSizeMb ?? 50 ), requestTimeoutMs, }; this.fetchFn = options.fetchImpl ?? globalThis.fetch; this.spooler = new DiskSpooler({ spoolDir: this.config.offlineSpoolDir, maxSizeMb: this.config.maxSpoolSizeMb, enabled: this.config.offlineSpoolEnabled, }); if (this.config.autoConnect && this.config.clientToken) { void this.connect().catch(() => { // Defer connection error handling to send/fallback }); } } private trackInFlight(promise: Promise): Promise { this.inFlightSet.add(promise); return promise.finally(() => { this.inFlightSet.delete(promise); }); } /** * Establishes a real-time Socket.IO connection to Port 4000 (`serverUrl`). */ public async connect(): Promise { if (!this.config.clientToken) { throw new DataStackConfigError( 'Cannot connect: clientToken is required.' ); } if (this.isConnected && this.socket?.connected) { return true; } if (this.socket) { this.socket.removeAllListeners(); this.socket.disconnect(); this.socket = null; } return new Promise((resolve, reject) => { const sock = io(this.config.serverUrl, { auth: { token: this.config.clientToken, clientId: this.clientId || undefined, }, transports: ['websocket', 'polling'], reconnection: false, timeout: this.config.requestTimeoutMs, }); this.socket = sock; let settled = false; const timer = setTimeout(() => { if (settled) return; settled = true; sock.disconnect(); this.isConnected = false; this.transportState = this.config.enableHttpFallback ? 'HTTP_FALLBACK' : 'DISCONNECTED'; resolve(false); }, this.config.requestTimeoutMs); sock.once('connect', () => { if (settled) return; settled = true; clearTimeout(timer); this.isConnected = true; this.transportState = 'IDLE_WS'; void this.flushSpool().catch(() => {}); resolve(true); }); sock.once('connect_error', (err: Error & { data?: Record }) => { if (settled) return; settled = true; clearTimeout(timer); this.isConnected = false; sock.disconnect(); const msg = String(err?.message || ''); const code = String(err?.data?.code || ''); if ( code === 'CLIENT_TOKEN_EXPIRED' || msg.includes('CLIENT_TOKEN_EXPIRED') ) { reject( new DataStackAuthError( msg || 'Client JWT token has expired.', 'CLIENT_TOKEN_EXPIRED', 401 ) ); return; } if ( code === 'CLIENT_IP_FORBIDDEN' || msg.includes('CLIENT_IP_FORBIDDEN') ) { reject( new DataStackAuthError( msg || 'Client IP is not permitted by allowlist.', 'CLIENT_IP_FORBIDDEN', 403 ) ); return; } if ( code === 'AUTH_FAILED' || msg.toLowerCase().includes('authentication') || msg.toLowerCase().includes('invalid token') || msg.toLowerCase().includes('revoked') ) { reject( new DataStackAuthError( msg || 'Socket authentication failed.', 'AUTH_FAILED', 401 ) ); return; } this.transportState = this.config.enableHttpFallback ? 'HTTP_FALLBACK' : 'DISCONNECTED'; resolve(false); }); sock.on('disconnect', () => { this.isConnected = false; if (this.transportState === 'IDLE_WS') { this.transportState = this.config.enableHttpFallback ? 'HTTP_FALLBACK' : 'DISCONNECTED'; } }); }); } public disconnect(): void { if (this.socket) { this.socket.removeAllListeners(); this.socket.disconnect(); this.socket = null; } this.isConnected = false; this.transportState = 'DISCONNECTED'; } private normalizeSingleEnvelope( payload: DataStackEventPayload, explicitEventId?: string, explicitTimestamp?: string ): DataStackEnvelope { if (!payload || typeof payload !== 'object' || Array.isArray(payload)) { throw new DataStackConfigError( 'Event payload must be a non-null plain object.' ); } const rawCopy: Record = { ...payload }; const eventId = String( explicitEventId || (typeof rawCopy.eventId === 'string' && rawCopy.eventId.trim() ? rawCopy.eventId.trim() : makeEventId()) ); const timestamp = String( explicitTimestamp || (typeof rawCopy.timestamp === 'string' && rawCopy.timestamp.trim() ? rawCopy.timestamp.trim() : new Date().toISOString()) ); let innerData: Record; if ( rawCopy.data && typeof rawCopy.data === 'object' && !Array.isArray(rawCopy.data) ) { innerData = { ...(rawCopy.data as Record) }; } else { delete rawCopy.eventId; delete rawCopy.timestamp; delete rawCopy.clientId; innerData = rawCopy; } if (Object.keys(innerData).length === 0) { throw new DataStackConfigError( 'Event payload cannot be empty after stripping envelope metadata.' ); } return { clientId: this.clientId, eventId, timestamp, data: innerData, }; } private async emitSocketSingle( envelope: DataStackEnvelope ): Promise { if (!this.isConnected || !this.socket || !this.socket.connected) { return null; } return new Promise((resolve, reject) => { const timer = setTimeout(() => { resolve(null); }, this.config.requestTimeoutMs); this.socket!.emit( 'sync_data', { clientId: envelope.clientId, eventId: envelope.eventId, timestamp: envelope.timestamp, data: envelope.data, }, (ack: Record | undefined) => { clearTimeout(timer); if (!ack || typeof ack !== 'object') { resolve(null); return; } if (ack.success === true) { this.transportState = 'IDLE_WS'; resolve({ success: true, transport: 'websocket', eventId: String(ack.eventId || envelope.eventId), accepted: 1, }); return; } const code = String(ack.code || 'SOCKET_REJECTED'); const msg = String(ack.error || 'Socket event rejected'); if (code === 'CLIENT_TOKEN_EXPIRED' || code === 'AUTH_FAILED' || code === 'CLIENT_REVOKED') { reject(new DataStackAuthError(msg, code as 'CLIENT_TOKEN_EXPIRED' | 'AUTH_FAILED', 401)); return; } if (code === 'CLIENT_IP_FORBIDDEN') { reject(new DataStackAuthError(msg, 'CLIENT_IP_FORBIDDEN', 403)); return; } if (code === 'STORAGE_QUOTA_EXCEEDED' || code === 'DISK_SPACE_LOW') { reject(new DataStackQuotaExceededError(msg, code)); return; } if (code === 'PAYLOAD_TOO_LARGE') { reject(new DataStackPayloadTooLargeError(msg)); return; } if (code === 'RATE_LIMIT_EXCEEDED') { reject(new DataStackRateLimitError(msg, [envelope.eventId])); return; } resolve(null); } ); }); } private async sendHttpSingle( envelope: DataStackEnvelope ): Promise { if (!this.config.clientToken) { throw new DataStackConfigError( 'clientToken is required for HTTP event ingestion.' ); } const controller = new AbortController(); const timer = setTimeout( () => controller.abort(), this.config.requestTimeoutMs ); let res: Response; try { res = await this.fetchFn(`${this.config.apiUrl}/api/events`, { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${this.config.clientToken}`, }, body: JSON.stringify({ clientId: envelope.clientId || undefined, eventId: envelope.eventId, timestamp: envelope.timestamp, data: envelope.data, }), signal: controller.signal, }); } finally { clearTimeout(timer); } const body = (await res.json().catch(() => ({}))) as Record; if (res.ok && body.success !== false) { this.transportState = 'HTTP_FALLBACK'; return { success: true, transport: 'http', eventId: String(body.eventId || envelope.eventId), accepted: 1, duplicates: body.duplicate ? 1 : 0, }; } const code = String(body.code || `HTTP_${res.status}`); const errMsg = String(body.error || `HTTP ${res.status} error`); if (res.status === 401) { throw new DataStackAuthError( errMsg, code === 'CLIENT_TOKEN_EXPIRED' ? 'CLIENT_TOKEN_EXPIRED' : 'AUTH_FAILED', 401, body ); } if (res.status === 403) { throw new DataStackAuthError( errMsg, code === 'CLIENT_IP_FORBIDDEN' ? 'CLIENT_IP_FORBIDDEN' : 'CLIENT_REVOKED', 403, body ); } if (res.status === 413 || code === 'PAYLOAD_TOO_LARGE') { throw new DataStackPayloadTooLargeError(errMsg, body); } if (res.status === 429 || code === 'RATE_LIMIT_EXCEEDED') { throw new DataStackRateLimitError(errMsg, [envelope.eventId], body); } if ( res.status === 507 || code === 'STORAGE_QUOTA_EXCEEDED' || code === 'DISK_SPACE_LOW' ) { throw new DataStackQuotaExceededError( errMsg, code === 'DISK_SPACE_LOW' ? 'DISK_SPACE_LOW' : 'STORAGE_QUOTA_EXCEEDED', body ); } throw new DataStackError(errMsg, code, res.status, body); } /** * Sends a single telemetry event via WebSocket primary transport, falling back * automatically to HTTP `POST /api/events` and offline disk spooling. */ public async send( payload: DataStackEventPayload ): Promise { const task = (async (): Promise => { const envelope = this.normalizeSingleEnvelope(payload); // 1. Try active WebSocket const wsResult = await this.emitSocketSingle(envelope); if (wsResult) { if (this.spooler.count() > 0) { void this.flushSpool().catch(() => {}); } return wsResult; } // 2. Try HTTP Fallback with bounded retry on 429 / transient 5xx if (this.config.enableHttpFallback) { let attempt = 0; while (attempt <= this.config.maxRetries) { try { const httpRes = await this.sendHttpSingle(envelope); if (this.spooler.count() > 0) { void this.flushSpool().catch(() => {}); } return httpRes; } catch (err) { if ( err instanceof DataStackAuthError || err instanceof DataStackQuotaExceededError || err instanceof DataStackPayloadTooLargeError || err instanceof DataStackConfigError ) { throw err; } if ( err instanceof DataStackRateLimitError && attempt < this.config.maxRetries ) { attempt += 1; await new Promise((r) => setTimeout(r, 50 * attempt)); continue; } if (err instanceof DataStackRateLimitError) { throw err; } break; } } } // 3. Offline Disk Spool fallback if (this.spooler.enabled) { const spooled = this.spooler.enqueue(envelope); if (spooled > 0) { this.transportState = 'OFFLINE_SPOOL'; return { success: true, transport: 'spool', eventId: envelope.eventId, accepted: 0, spooled, }; } } throw new DataStackError( 'Failed to deliver event via WebSocket, HTTP fallback, or offline spool.', 'TRANSPORT_UNAVAILABLE', 503 ); })(); return this.trackInFlight(task); } /** * Alias for `send()` for 1:1 naming parity with Python SDK (`client.sync()`). */ public async sync( payload: DataStackEventPayload ): Promise { return this.send(payload); } private async sendBatchChunkInternal( chunk: DataStackEnvelope[] ): Promise { // 1. Try WebSocket batch_sync if connected if (this.isConnected && this.socket && this.socket.connected) { const wsBatchRes = await new Promise( (resolve, reject) => { const timer = setTimeout( () => resolve(null), this.config.requestTimeoutMs ); this.socket!.emit( 'batch_sync', { clientId: this.clientId, batch: chunk.map((item) => ({ eventId: item.eventId, timestamp: item.timestamp, data: item.data, })), }, (ack: Record | undefined) => { clearTimeout(timer); if (!ack || typeof ack !== 'object') { resolve(null); return; } if (ack.success === true) { const rateLimited = Array.isArray(ack.rateLimitedEventIds) ? (ack.rateLimitedEventIds as string[]) : []; resolve({ success: true, transport: 'websocket', accepted: Number( ack.processedCount ?? ack.accepted ?? chunk.length - rateLimited.length ), rateLimitedEventIds: rateLimited, }); return; } const code = String(ack.code || 'BATCH_REJECTED'); const msg = String(ack.error || 'Batch rejected'); if (code === 'CLIENT_TOKEN_EXPIRED' || code === 'AUTH_FAILED') { reject(new DataStackAuthError(msg, code as 'CLIENT_TOKEN_EXPIRED' | 'AUTH_FAILED', 401)); return; } if (code === 'CLIENT_IP_FORBIDDEN') { reject(new DataStackAuthError(msg, 'CLIENT_IP_FORBIDDEN', 403)); return; } if (code === 'STORAGE_QUOTA_EXCEEDED' || code === 'DISK_SPACE_LOW') { reject(new DataStackQuotaExceededError(msg, code)); return; } if (code === 'PAYLOAD_TOO_LARGE') { reject(new DataStackPayloadTooLargeError(msg)); return; } if (code === 'RATE_LIMIT_EXCEEDED') { const rateLimited = Array.isArray(ack.rateLimitedEventIds) ? (ack.rateLimitedEventIds as string[]) : chunk.map((c) => c.eventId); reject(new DataStackRateLimitError(msg, rateLimited)); return; } resolve(null); } ); } ); if (wsBatchRes) { return wsBatchRes; } } // 2. Try HTTP POST /api/events batch if (!this.config.enableHttpFallback) { throw new DataStackError( 'WebSocket disconnected and HTTP fallback is disabled.', 'TRANSPORT_UNAVAILABLE', 503 ); } if (!this.config.clientToken) { throw new DataStackConfigError( 'clientToken is required for HTTP batch ingestion.' ); } const controller = new AbortController(); const timer = setTimeout( () => controller.abort(), this.config.requestTimeoutMs ); let res: Response; try { res = await this.fetchFn(`${this.config.apiUrl}/api/events`, { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${this.config.clientToken}`, }, body: JSON.stringify({ clientId: this.clientId || undefined, batch: chunk.map((item) => ({ eventId: item.eventId, timestamp: item.timestamp, data: item.data, })), }), signal: controller.signal, }); } finally { clearTimeout(timer); } const body = (await res.json().catch(() => ({}))) as Record; const rateLimited = Array.isArray(body.rateLimitedEventIds) ? (body.rateLimitedEventIds as string[]) : []; if (res.ok && body.success !== false) { this.transportState = 'HTTP_FALLBACK'; return { success: true, transport: 'http', accepted: Number( body.accepted ?? body.processedCount ?? chunk.length - rateLimited.length ), duplicates: Number(body.duplicates ?? 0), rateLimitedEventIds: rateLimited, }; } const code = String(body.code || `HTTP_${res.status}`); const errMsg = String(body.error || `HTTP ${res.status} batch error`); if (res.status === 401) { throw new DataStackAuthError( errMsg, code === 'CLIENT_TOKEN_EXPIRED' ? 'CLIENT_TOKEN_EXPIRED' : 'AUTH_FAILED', 401, body ); } if (res.status === 403) { throw new DataStackAuthError( errMsg, code === 'CLIENT_IP_FORBIDDEN' ? 'CLIENT_IP_FORBIDDEN' : 'CLIENT_REVOKED', 403, body ); } if (res.status === 413 || code === 'PAYLOAD_TOO_LARGE') { throw new DataStackPayloadTooLargeError(errMsg, body); } if (res.status === 429 || code === 'RATE_LIMIT_EXCEEDED') { throw new DataStackRateLimitError( errMsg, rateLimited.length > 0 ? rateLimited : chunk.map((c) => c.eventId), body ); } if ( res.status === 507 || code === 'STORAGE_QUOTA_EXCEEDED' || code === 'DISK_SPACE_LOW' ) { throw new DataStackQuotaExceededError( errMsg, code === 'DISK_SPACE_LOW' ? 'DISK_SPACE_LOW' : 'STORAGE_QUOTA_EXCEEDED', body ); } throw new DataStackError(errMsg, code, res.status, body); } private async deliverChunkWithSplitAndRetry( chunk: DataStackEnvelope[], allowSpoolFallback = true ): Promise { let pending = [...chunk]; let totalAccepted = 0; let totalSpooled = 0; let lastTransport: 'websocket' | 'http' | 'spool' = 'http'; let attempt = 0; while (pending.length > 0) { try { const res = await this.sendBatchChunkInternal(pending); lastTransport = res.transport; const rateLimitedSet = new Set(res.rateLimitedEventIds || []); if (rateLimitedSet.size === 0) { totalAccepted += res.accepted ?? pending.length; pending = []; break; } const acceptedNow = res.accepted ?? Math.max(0, pending.length - rateLimitedSet.size); totalAccepted += acceptedNow; pending = pending.filter((item) => rateLimitedSet.has(item.eventId)); if (attempt >= this.config.maxRetries) { throw new DataStackRateLimitError( `Rate limit exceeded for ${pending.length} event(s) after ${this.config.maxRetries} retries.`, pending.map((p) => p.eventId) ); } attempt += 1; await new Promise((r) => setTimeout(r, 50 * attempt)); } catch (err) { if ( err instanceof DataStackPayloadTooLargeError && pending.length > 1 ) { const mid = Math.floor(pending.length / 2); const left = await this.deliverChunkWithSplitAndRetry( pending.slice(0, mid), allowSpoolFallback ); const right = await this.deliverChunkWithSplitAndRetry( pending.slice(mid), allowSpoolFallback ); return { success: left.success && right.success, transport: right.transport, accepted: (left.accepted || 0) + (right.accepted || 0), spooled: (left.spooled || 0) + (right.spooled || 0), }; } if ( err instanceof DataStackRateLimitError && attempt < this.config.maxRetries ) { const rlSet = new Set(err.rateLimitedEventIds); if (rlSet.size > 0 && rlSet.size < pending.length) { totalAccepted += pending.length - rlSet.size; pending = pending.filter((p) => rlSet.has(p.eventId)); } attempt += 1; await new Promise((r) => setTimeout(r, 50 * attempt)); continue; } if ( err instanceof DataStackAuthError || err instanceof DataStackQuotaExceededError || err instanceof DataStackPayloadTooLargeError || err instanceof DataStackRateLimitError || err instanceof DataStackConfigError ) { throw err; } if (allowSpoolFallback && this.spooler.enabled) { const spooled = this.spooler.enqueue(pending); if (spooled > 0) { totalSpooled += spooled; lastTransport = 'spool'; this.transportState = 'OFFLINE_SPOOL'; pending = []; break; } } throw err; } } return { success: pending.length === 0, transport: lastTransport, accepted: totalAccepted, spooled: totalSpooled, unprocessedItems: pending.map((p) => p.data), }; } /** * Sends a batch of telemetry events with automatic chunking (`batchChunkSize`), * recursive `413 PAYLOAD_TOO_LARGE` splitting, partial-batch `429` retry, and * offline disk spooling. */ public async sendBatch( events: DataStackEventPayload[], options?: { chunkSize?: number } ): Promise { const task = (async (): Promise => { if (!Array.isArray(events) || events.length === 0) { throw new DataStackConfigError( 'sendBatch requires a non-empty array of event objects.' ); } const envelopes = events.map((ev) => this.normalizeSingleEnvelope(ev)); const chunkSize = Math.max( 1, Math.min(1000, Number(options?.chunkSize ?? this.config.batchChunkSize)) ); let totalAccepted = 0; let totalSpooled = 0; let lastTransport: 'websocket' | 'http' | 'spool' = 'http'; for (let i = 0; i < envelopes.length; i += chunkSize) { const slice = envelopes.slice(i, i + chunkSize); const res = await this.deliverChunkWithSplitAndRetry(slice, true); totalAccepted += res.accepted || 0; totalSpooled += res.spooled || 0; lastTransport = res.transport; } return { success: true, transport: lastTransport, accepted: totalAccepted, spooled: totalSpooled, }; })(); return this.trackInFlight(task); } /** * Alias for `sendBatch()` for 1:1 parity with Python SDK (`client.batch_sync()`). */ public async batchSync( events: DataStackEventPayload[], options?: { chunkSize?: number } ): Promise { return this.sendBatch(events, options); } /** * Ingests a `.csv`, `.json`, `.jsonl`, or `.ndjson` file directly with * deterministic SHA-256 `eventId` deduplication (`file___`). */ public async sendFile( filePath: string, options?: { chunkSize?: number } ): Promise { const task = (async (): Promise => { const resolvedPath = path.resolve(filePath); if (!fs.existsSync(resolvedPath)) { throw new DataStackFileError(`File not found: ${resolvedPath}`); } const ext = path.extname(resolvedPath).toLowerCase(); const fileStem = path.basename(resolvedPath, ext); const rawContent = fs.readFileSync(resolvedPath, 'utf8'); let records: DataStackEventPayload[] = []; if (ext === '.csv') { const parsed = parseCsvSync(rawContent, { columns: true, skip_empty_lines: true, relax_column_count: true, trim: true, }) as DataStackEventPayload[]; records = parsed; } else if (ext === '.json') { try { const parsed = JSON.parse(rawContent) as unknown; if (Array.isArray(parsed)) { records = parsed as DataStackEventPayload[]; } else if (parsed && typeof parsed === 'object') { records = [parsed as DataStackEventPayload]; } else { throw new DataStackFileError( 'JSON file must contain an object or array of objects.' ); } } catch (err) { if (err instanceof DataStackFileError) throw err; throw new DataStackFileError( `Invalid JSON file (${resolvedPath}): ${err instanceof Error ? err.message : String(err)}` ); } } else if (ext === '.jsonl' || ext === '.ndjson') { const lines = rawContent .split('\n') .map((l) => l.trim()) .filter(Boolean); records = lines.map((line, idx) => { try { const obj = JSON.parse(line) as unknown; if (!obj || typeof obj !== 'object' || Array.isArray(obj)) { throw new Error('Line is not a JSON object'); } return obj as DataStackEventPayload; } catch (err) { throw new DataStackFileError( `Invalid JSONL at line ${idx + 1}: ${err instanceof Error ? err.message : String(err)}` ); } }); } else { throw new DataStackFileError( `Unsupported file extension '${ext}'. Supported extensions: .csv, .json, .jsonl, .ndjson` ); } if (records.length === 0) { return { success: true, transport: this.isConnected ? 'websocket' : 'http', filePath: resolvedPath, totalRows: 0, accepted: 0, acceptedRows: 0, spooled: 0, spooledRows: 0, chunksSent: 0, }; } const envelopes: DataStackEnvelope[] = records.map((rec, idx) => { const deterministicId = typeof rec.eventId === 'string' && rec.eventId.trim() ? rec.eventId.trim() : makeDeterministicFileEventId(fileStem, idx, rec); return this.normalizeSingleEnvelope(rec, deterministicId); }); const chunkSize = Math.max( 1, Math.min(1000, Number(options?.chunkSize ?? this.config.batchChunkSize)) ); let acceptedRows = 0; let spooledRows = 0; let chunksSent = 0; let lastTransport: 'websocket' | 'http' | 'spool' = 'http'; for (let i = 0; i < envelopes.length; i += chunkSize) { const slice = envelopes.slice(i, i + chunkSize); const res = await this.deliverChunkWithSplitAndRetry(slice, true); acceptedRows += res.accepted || 0; spooledRows += res.spooled || 0; chunksSent += 1; lastTransport = res.transport; } return { success: acceptedRows + spooledRows === envelopes.length, transport: lastTransport, filePath: resolvedPath, totalRows: envelopes.length, accepted: acceptedRows, acceptedRows, spooled: spooledRows, spooledRows, chunksSent, }; })(); return this.trackInFlight(task); } /** * Replays locally spooled events from `.datastack_spool/spool.jsonl` to the server. */ public async flushSpool(): Promise<{ drained: number; remaining: number }> { if (!this.spooler.enabled || this.isDrainingSpool) { return { drained: 0, remaining: this.spooler.count() }; } this.isDrainingSpool = true; let drained = 0; try { while (this.spooler.count() > 0) { const batch = this.spooler.peekBatch(this.config.batchChunkSize); if (batch.length === 0) break; const res = await this.deliverChunkWithSplitAndRetry(batch, false); if (res.success) { this.spooler.removeFirst(batch.length); drained += batch.length; } else { break; } } } finally { this.isDrainingSpool = false; } return { drained, remaining: this.spooler.count() }; } /** * Gracefully awaits all in-flight telemetry requests, attempts to drain any * spooled events within `timeoutMs`, and closes the Socket.IO connection. */ public async shutdown( timeoutMs = 5000, options?: { throwOnTimeout?: boolean } ): Promise<{ drained: boolean; remainingInFlight: number; remainingSpooled: number }> { const deadline = Date.now() + Math.max(50, timeoutMs); const waitWork = (async () => { while (this.inFlightSet.size > 0 && Date.now() < deadline) { await Promise.race([ Promise.allSettled(Array.from(this.inFlightSet)), new Promise((r) => setTimeout(r, 25)), ]); } if (this.spooler.count() > 0 && Date.now() < deadline) { try { await this.flushSpool(); } catch { // Best-effort spool drain before shutdown } } })(); const remainingMs = Math.max(10, deadline - Date.now()); let timedOut = false; await Promise.race([ waitWork, new Promise((resolve) => setTimeout(() => { timedOut = true; resolve(); }, remainingMs) ), ]); this.disconnect(); const remainingInFlight = this.inFlightSet.size; const remainingSpooled = this.spooler.count(); if (timedOut && remainingInFlight > 0 && options?.throwOnTimeout) { throw new DataStackShutdownTimeoutError( `Graceful shutdown timed out after ${timeoutMs}ms with ${remainingInFlight} in-flight request(s).` ); } return { drained: !timedOut && remainingInFlight === 0, remainingInFlight, remainingSpooled, }; } public status(): DataStackStatusSnapshot { return { connected: this.isConnected && Boolean(this.socket?.connected), transportState: this.transportState, clientId: this.clientId, serverUrl: this.config.serverUrl, apiUrl: this.config.apiUrl, spooledCount: this.spooler.count(), spooledBytes: this.spooler.sizeBytes(), inFlightRequests: this.inFlightSet.size, }; } } export { DataStackClient as DataStack }; export default DataStackClient;