2026-11-20 18:48:54 +06:00
|
|
|
import { parse as parseCsvSync } from 'csv-parse/sync';
|
|
|
|
|
import crypto from 'node:crypto';
|
|
|
|
|
import fs from 'node:fs';
|
|
|
|
|
import path from 'node:path';
|
|
|
|
|
import { io, type Socket } from 'socket.io-client';
|
2026-11-21 00:06:55 +06:00
|
|
|
import { DiskSpooler, type DiskSpoolerOptions } from './spool';
|
2026-11-20 18:48:54 +06:00
|
|
|
import {
|
|
|
|
|
DataStackAuthError,
|
|
|
|
|
DataStackConfigError,
|
|
|
|
|
DataStackError,
|
|
|
|
|
DataStackFileError,
|
|
|
|
|
DataStackPayloadTooLargeError,
|
|
|
|
|
DataStackQuotaExceededError,
|
|
|
|
|
DataStackRateLimitError,
|
|
|
|
|
DataStackShutdownTimeoutError,
|
2026-11-21 00:06:55 +06:00
|
|
|
type DataStackAuthErrorCode,
|
2026-11-20 18:48:54 +06:00
|
|
|
type DataStackConfigOptions,
|
|
|
|
|
type DataStackConnectionState,
|
|
|
|
|
type DataStackEnvelope,
|
|
|
|
|
type DataStackEventPayload,
|
|
|
|
|
type DataStackFileSendResult,
|
|
|
|
|
type DataStackSendResult,
|
|
|
|
|
type DataStackStatusSnapshot,
|
2026-11-21 00:06:55 +06:00
|
|
|
type DataStackTransportMode,
|
2026-11-20 18:48:54 +06:00
|
|
|
type ResolvedDataStackConfig,
|
|
|
|
|
} from './types';
|
|
|
|
|
|
2026-11-21 00:06:55 +06:00
|
|
|
export {
|
|
|
|
|
DiskSpooler,
|
|
|
|
|
DataStackAuthError,
|
|
|
|
|
DataStackConfigError,
|
|
|
|
|
DataStackError,
|
|
|
|
|
DataStackFileError,
|
|
|
|
|
DataStackPayloadTooLargeError,
|
|
|
|
|
DataStackQuotaExceededError,
|
|
|
|
|
DataStackRateLimitError,
|
|
|
|
|
DataStackShutdownTimeoutError,
|
|
|
|
|
type DiskSpoolerOptions,
|
|
|
|
|
type DataStackAuthErrorCode,
|
|
|
|
|
type DataStackConfigOptions,
|
|
|
|
|
type DataStackConnectionState,
|
|
|
|
|
type DataStackEnvelope,
|
|
|
|
|
type DataStackEventPayload,
|
|
|
|
|
type DataStackFileSendResult,
|
|
|
|
|
type DataStackSendResult,
|
|
|
|
|
type DataStackStatusSnapshot,
|
|
|
|
|
type DataStackTransportMode,
|
|
|
|
|
type ResolvedDataStackConfig,
|
|
|
|
|
};
|
2026-11-20 18:48:54 +06:00
|
|
|
|
|
|
|
|
export const SDK_VERSION = '2.2.0';
|
|
|
|
|
|
2026-11-21 00:06:55 +06:00
|
|
|
function classifyAuthRejection(
|
|
|
|
|
rawCode: string,
|
|
|
|
|
rawMsg: string
|
|
|
|
|
): { code: DataStackAuthErrorCode; statusCode: number } | null {
|
|
|
|
|
const code = String(rawCode || '').toUpperCase();
|
|
|
|
|
const upperMsg = String(rawMsg || '').toUpperCase();
|
|
|
|
|
const combined = `${code} ${upperMsg}`;
|
|
|
|
|
|
|
|
|
|
if (combined.includes('CLIENT_TOKEN_EXPIRED')) {
|
|
|
|
|
return { code: 'CLIENT_TOKEN_EXPIRED', statusCode: 401 };
|
|
|
|
|
}
|
|
|
|
|
if (combined.includes('TOKEN_ROTATED')) {
|
|
|
|
|
return { code: 'TOKEN_ROTATED', statusCode: 401 };
|
|
|
|
|
}
|
|
|
|
|
if (combined.includes('CLIENT_IP_FORBIDDEN')) {
|
|
|
|
|
return { code: 'CLIENT_IP_FORBIDDEN', statusCode: 403 };
|
|
|
|
|
}
|
|
|
|
|
if (combined.includes('CLIENT_REVOKED') || upperMsg.includes('REVOKED')) {
|
|
|
|
|
return { code: 'CLIENT_REVOKED', statusCode: 403 };
|
|
|
|
|
}
|
|
|
|
|
if (
|
|
|
|
|
code === 'AUTH_FAILED' ||
|
|
|
|
|
code === 'AUTH_INVALID' ||
|
|
|
|
|
code === 'AUTH_REQUIRED' ||
|
|
|
|
|
upperMsg.includes('AUTHENTICATION') ||
|
|
|
|
|
upperMsg.includes('INVALID TOKEN')
|
|
|
|
|
) {
|
|
|
|
|
return { code: 'AUTH_FAILED', statusCode: 401 };
|
|
|
|
|
}
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-20 18:48:54 +06:00
|
|
|
/**
|
|
|
|
|
* Matches server `SDKManager.normalizeClientId()` and Python SDK `_normalize_client_id()`.
|
|
|
|
|
*/
|
|
|
|
|
export function normalizeClientId(value?: string | null): string {
|
|
|
|
|
const raw = String(value ?? '')
|
|
|
|
|
.trim()
|
|
|
|
|
.toLowerCase();
|
|
|
|
|
if (!raw) return '';
|
|
|
|
|
return raw.replace(/[^a-z0-9_-]/g, '_');
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Extracts `clientId` or `sub` claim from an unverified JWT payload string.
|
|
|
|
|
*/
|
|
|
|
|
export function decodeJwtClientId(token?: string | null): string {
|
|
|
|
|
try {
|
|
|
|
|
const parts = String(token ?? '').split('.');
|
|
|
|
|
if (parts.length !== 3) return '';
|
|
|
|
|
const json = Buffer.from(parts[1], 'base64url').toString('utf8');
|
|
|
|
|
const claims = JSON.parse(json) as Record<string, unknown>;
|
|
|
|
|
if (claims && typeof claims === 'object') {
|
|
|
|
|
return String(claims.clientId || claims.sub || '').trim();
|
|
|
|
|
}
|
|
|
|
|
} catch {
|
|
|
|
|
// Ignore malformed JWT
|
|
|
|
|
}
|
|
|
|
|
return '';
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function makeEventId(): string {
|
|
|
|
|
return `evt_${Date.now()}_${crypto.randomBytes(6).toString('hex')}`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Generates a deterministic SHA-256 `eventId` for file rows (1:1 parity with Python SDK)
|
|
|
|
|
* so re-ingesting the same `.csv`, `.json`, or `.jsonl` file is idempotently deduplicated.
|
|
|
|
|
*/
|
|
|
|
|
export function makeDeterministicFileEventId(
|
|
|
|
|
fileStem: string,
|
|
|
|
|
rowIndex: number,
|
|
|
|
|
record: DataStackEventPayload
|
|
|
|
|
): string {
|
|
|
|
|
const sortedKeys = Object.keys(record).sort();
|
|
|
|
|
const canonicalObj: Record<string, unknown> = {};
|
|
|
|
|
for (const key of sortedKeys) {
|
|
|
|
|
canonicalObj[key] = record[key];
|
|
|
|
|
}
|
|
|
|
|
const raw = `${fileStem}:${rowIndex}:${JSON.stringify(canonicalObj)}`;
|
|
|
|
|
const digest = crypto.createHash('sha256').update(raw, 'utf8').digest('hex').slice(0, 12);
|
|
|
|
|
const safeStem = fileStem.replace(/[^a-zA-Z0-9_-]/g, '_').slice(0, 32);
|
|
|
|
|
return `file_${safeStem}_${rowIndex}_${digest}`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function loadJsonConfigFile(configPath?: string): Partial<DataStackConfigOptions> {
|
|
|
|
|
const target = configPath
|
|
|
|
|
? path.resolve(configPath)
|
|
|
|
|
: path.resolve(process.cwd(), 'datastack.config.json');
|
|
|
|
|
if (!fs.existsSync(target)) {
|
|
|
|
|
if (configPath) {
|
|
|
|
|
throw new DataStackConfigError(`Configuration file not found: ${target}`);
|
|
|
|
|
}
|
|
|
|
|
return {};
|
|
|
|
|
}
|
|
|
|
|
try {
|
|
|
|
|
const parsed = JSON.parse(fs.readFileSync(target, 'utf8')) as Record<string, unknown>;
|
|
|
|
|
return {
|
|
|
|
|
serverUrl: typeof parsed.serverUrl === 'string' ? parsed.serverUrl : typeof parsed.SERVER_URL === 'string' ? parsed.SERVER_URL : undefined,
|
|
|
|
|
apiUrl: typeof parsed.apiUrl === 'string' ? parsed.apiUrl : typeof parsed.API_URL === 'string' ? parsed.API_URL : undefined,
|
|
|
|
|
clientToken: typeof parsed.clientToken === 'string' ? parsed.clientToken : typeof parsed.CLIENT_TOKEN === 'string' ? parsed.CLIENT_TOKEN : typeof parsed.token === 'string' ? parsed.token : undefined,
|
|
|
|
|
clientId: typeof parsed.clientId === 'string' ? parsed.clientId : typeof parsed.CLIENT_ID === 'string' ? parsed.CLIENT_ID : undefined,
|
|
|
|
|
autoConnect: typeof parsed.autoConnect === 'boolean' ? parsed.autoConnect : typeof parsed.AUTO_CONNECT === 'boolean' ? parsed.AUTO_CONNECT : undefined,
|
|
|
|
|
enableHttpFallback: typeof parsed.enableHttpFallback === 'boolean' ? parsed.enableHttpFallback : typeof parsed.ENABLE_HTTP_FALLBACK === 'boolean' ? parsed.ENABLE_HTTP_FALLBACK : undefined,
|
|
|
|
|
heartbeatIntervalMs: typeof parsed.heartbeatIntervalMs === 'number' ? parsed.heartbeatIntervalMs : undefined,
|
|
|
|
|
maxRetries: typeof parsed.maxRetries === 'number' ? parsed.maxRetries : typeof parsed.MAX_RETRIES === 'number' ? parsed.MAX_RETRIES : undefined,
|
|
|
|
|
batchChunkSize: typeof parsed.batchChunkSize === 'number' ? parsed.batchChunkSize : typeof parsed.BATCH_CHUNK_SIZE === 'number' ? parsed.BATCH_CHUNK_SIZE : undefined,
|
|
|
|
|
offlineSpoolEnabled: typeof parsed.offlineSpoolEnabled === 'boolean' ? parsed.offlineSpoolEnabled : typeof parsed.OFFLINE_SPOOL_ENABLED === 'boolean' ? parsed.OFFLINE_SPOOL_ENABLED : undefined,
|
|
|
|
|
offlineSpoolDir: typeof parsed.offlineSpoolDir === 'string' ? parsed.offlineSpoolDir : typeof parsed.OFFLINE_SPOOL_DIR === 'string' ? parsed.OFFLINE_SPOOL_DIR : undefined,
|
|
|
|
|
maxSpoolSizeMb: typeof parsed.maxSpoolSizeMb === 'number' ? parsed.maxSpoolSizeMb : undefined,
|
|
|
|
|
requestTimeoutMs: typeof parsed.requestTimeoutMs === 'number' ? parsed.requestTimeoutMs : undefined,
|
|
|
|
|
};
|
|
|
|
|
} catch (err) {
|
|
|
|
|
throw new DataStackConfigError(
|
|
|
|
|
`Failed to parse datastack.config.json (${target}): ${err instanceof Error ? err.message : String(err)}`
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export class DataStackClient {
|
|
|
|
|
public readonly config: ResolvedDataStackConfig;
|
|
|
|
|
public readonly clientId: string;
|
|
|
|
|
public readonly spooler: DiskSpooler;
|
|
|
|
|
private socket: Socket | null = null;
|
|
|
|
|
private isConnected = false;
|
|
|
|
|
private transportState: DataStackConnectionState = 'DISCONNECTED';
|
|
|
|
|
private inFlightSet = new Set<Promise<unknown>>();
|
|
|
|
|
private isDrainingSpool = false;
|
2026-11-21 00:06:55 +06:00
|
|
|
private heartbeatTimer: ReturnType<typeof setInterval> | null = null;
|
|
|
|
|
private heartbeatInFlight = false;
|
|
|
|
|
private heartbeatStopped = false;
|
2026-11-20 18:48:54 +06:00
|
|
|
private readonly fetchFn: typeof fetch;
|
|
|
|
|
|
|
|
|
|
constructor(options: DataStackConfigOptions = {}) {
|
|
|
|
|
const fileCfg = loadJsonConfigFile(options.configPath);
|
|
|
|
|
const serverUrl = String(
|
|
|
|
|
options.serverUrl ??
|
|
|
|
|
fileCfg.serverUrl ??
|
|
|
|
|
process.env.DATASTACK_SERVER_URL ??
|
|
|
|
|
'http://localhost:4000'
|
|
|
|
|
)
|
|
|
|
|
.trim()
|
|
|
|
|
.replace(/\/+$/, '');
|
|
|
|
|
const apiUrl = String(
|
|
|
|
|
options.apiUrl ??
|
|
|
|
|
fileCfg.apiUrl ??
|
|
|
|
|
process.env.DATASTACK_API_URL ??
|
|
|
|
|
'http://localhost:3000'
|
|
|
|
|
)
|
|
|
|
|
.trim()
|
|
|
|
|
.replace(/\/+$/, '');
|
|
|
|
|
const clientToken = String(
|
|
|
|
|
options.token ??
|
|
|
|
|
options.clientToken ??
|
|
|
|
|
fileCfg.clientToken ??
|
|
|
|
|
process.env.DATASTACK_CLIENT_TOKEN ??
|
|
|
|
|
''
|
|
|
|
|
).trim();
|
|
|
|
|
|
|
|
|
|
if (!/^https?:\/\//i.test(serverUrl)) {
|
|
|
|
|
throw new DataStackConfigError(
|
|
|
|
|
`Invalid serverUrl '${serverUrl}'. Expected http:// or https:// URL.`
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
if (!/^https?:\/\//i.test(apiUrl)) {
|
|
|
|
|
throw new DataStackConfigError(
|
|
|
|
|
`Invalid apiUrl '${apiUrl}'. Expected http:// or https:// URL.`
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const configuredClientId = String(
|
|
|
|
|
options.clientId ??
|
|
|
|
|
fileCfg.clientId ??
|
|
|
|
|
process.env.DATASTACK_CLIENT_ID ??
|
|
|
|
|
''
|
|
|
|
|
).trim();
|
|
|
|
|
const tokenClientId = decodeJwtClientId(clientToken);
|
|
|
|
|
const normConfigured = normalizeClientId(configuredClientId);
|
|
|
|
|
const normToken = normalizeClientId(tokenClientId);
|
|
|
|
|
|
|
|
|
|
if (normConfigured && normToken && normConfigured !== normToken) {
|
|
|
|
|
throw new DataStackConfigError(
|
|
|
|
|
`Configured clientId '${configuredClientId}' does not match JWT token subject '${tokenClientId}'.`
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const batchChunkSize = Number(
|
|
|
|
|
options.batchChunkSize ?? fileCfg.batchChunkSize ?? 250
|
|
|
|
|
);
|
|
|
|
|
if (!Number.isFinite(batchChunkSize) || batchChunkSize < 1 || batchChunkSize > 1000) {
|
|
|
|
|
throw new DataStackConfigError(
|
|
|
|
|
`Invalid batchChunkSize '${batchChunkSize}'. Must be between 1 and 1000.`
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const maxRetries = Number(options.maxRetries ?? fileCfg.maxRetries ?? 3);
|
|
|
|
|
if (!Number.isFinite(maxRetries) || maxRetries < 0 || maxRetries > 20) {
|
|
|
|
|
throw new DataStackConfigError(
|
|
|
|
|
`Invalid maxRetries '${maxRetries}'. Must be between 0 and 20.`
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const requestTimeoutMs = Number(
|
|
|
|
|
options.requestTimeoutMs ?? fileCfg.requestTimeoutMs ?? 5000
|
|
|
|
|
);
|
|
|
|
|
if (!Number.isFinite(requestTimeoutMs) || requestTimeoutMs < 100) {
|
|
|
|
|
throw new DataStackConfigError(
|
|
|
|
|
`Invalid requestTimeoutMs '${requestTimeoutMs}'. Must be >= 100ms.`
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-21 00:06:55 +06:00
|
|
|
const rawHeartbeat = Number(
|
|
|
|
|
options.heartbeatIntervalMs ?? fileCfg.heartbeatIntervalMs ?? 5000
|
|
|
|
|
);
|
|
|
|
|
const heartbeatIntervalMs =
|
|
|
|
|
Number.isFinite(rawHeartbeat) && rawHeartbeat > 0
|
|
|
|
|
? Math.max(50, Math.floor(rawHeartbeat))
|
|
|
|
|
: 0;
|
|
|
|
|
|
2026-11-20 18:48:54 +06:00
|
|
|
this.clientId = normToken || normConfigured;
|
|
|
|
|
this.config = {
|
|
|
|
|
serverUrl,
|
|
|
|
|
apiUrl,
|
|
|
|
|
clientToken,
|
|
|
|
|
clientId: this.clientId,
|
|
|
|
|
autoConnect: Boolean(options.autoConnect ?? fileCfg.autoConnect ?? false),
|
|
|
|
|
enableHttpFallback:
|
|
|
|
|
options.enableHttpFallback ?? fileCfg.enableHttpFallback ?? true,
|
2026-11-21 00:06:55 +06:00
|
|
|
heartbeatIntervalMs,
|
2026-11-20 18:48:54 +06:00
|
|
|
maxRetries,
|
|
|
|
|
batchChunkSize: Math.floor(batchChunkSize),
|
|
|
|
|
offlineSpoolEnabled:
|
|
|
|
|
options.offlineSpoolEnabled ?? fileCfg.offlineSpoolEnabled ?? true,
|
|
|
|
|
offlineSpoolDir: String(
|
|
|
|
|
options.offlineSpoolDir ?? fileCfg.offlineSpoolDir ?? '.datastack_spool'
|
|
|
|
|
),
|
|
|
|
|
maxSpoolSizeMb: Number(
|
|
|
|
|
options.maxSpoolSizeMb ?? fileCfg.maxSpoolSizeMb ?? 50
|
|
|
|
|
),
|
|
|
|
|
requestTimeoutMs,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
this.fetchFn = options.fetchImpl ?? globalThis.fetch;
|
|
|
|
|
this.spooler = new DiskSpooler({
|
|
|
|
|
spoolDir: this.config.offlineSpoolDir,
|
|
|
|
|
maxSizeMb: this.config.maxSpoolSizeMb,
|
|
|
|
|
enabled: this.config.offlineSpoolEnabled,
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
if (this.config.autoConnect && this.config.clientToken) {
|
|
|
|
|
void this.connect().catch(() => {
|
|
|
|
|
// Defer connection error handling to send/fallback
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private trackInFlight<T>(promise: Promise<T>): Promise<T> {
|
|
|
|
|
this.inFlightSet.add(promise);
|
|
|
|
|
return promise.finally(() => {
|
|
|
|
|
this.inFlightSet.delete(promise);
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-21 00:06:55 +06:00
|
|
|
private startHeartbeatMonitor(): void {
|
|
|
|
|
if (
|
|
|
|
|
this.heartbeatStopped ||
|
|
|
|
|
this.config.heartbeatIntervalMs <= 0 ||
|
|
|
|
|
this.heartbeatTimer !== null
|
|
|
|
|
) {
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
const timer = setInterval(() => {
|
|
|
|
|
void this.runHeartbeatStep();
|
|
|
|
|
}, this.config.heartbeatIntervalMs);
|
|
|
|
|
timer.unref?.();
|
|
|
|
|
this.heartbeatTimer = timer;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private stopHeartbeatMonitor(): void {
|
|
|
|
|
if (this.heartbeatTimer !== null) {
|
|
|
|
|
clearInterval(this.heartbeatTimer);
|
|
|
|
|
this.heartbeatTimer = null;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Verifies the active WebSocket handshake with Port 4000 via `ping_verify`.
|
|
|
|
|
* Throws `DataStackAuthError` immediately on terminal auth rejections.
|
|
|
|
|
*/
|
|
|
|
|
public async pingVerify(timeoutMs?: number): Promise<boolean> {
|
|
|
|
|
if (!this.isConnected || !this.socket || !this.socket.connected) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
const waitMs = Math.max(
|
|
|
|
|
100,
|
|
|
|
|
Math.min(timeoutMs ?? this.config.requestTimeoutMs, 5000)
|
|
|
|
|
);
|
|
|
|
|
return new Promise<boolean>((resolve, reject) => {
|
|
|
|
|
const timer = setTimeout(() => resolve(false), waitMs);
|
|
|
|
|
this.socket!.emit(
|
|
|
|
|
'ping_verify',
|
|
|
|
|
{},
|
|
|
|
|
(ack: Record<string, unknown> | undefined) => {
|
|
|
|
|
clearTimeout(timer);
|
|
|
|
|
if (!ack || typeof ack !== 'object') {
|
|
|
|
|
resolve(false);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
if (ack.success === true) {
|
|
|
|
|
resolve(ack.status === 'VERIFIED' || ack.status === undefined);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
const code = String(ack.code || '');
|
|
|
|
|
const msg = String(ack.error || 'Handshake verification failed');
|
|
|
|
|
const authErr = classifyAuthRejection(code, msg);
|
|
|
|
|
if (authErr) {
|
|
|
|
|
this.heartbeatStopped = true;
|
|
|
|
|
this.stopHeartbeatMonitor();
|
|
|
|
|
reject(
|
|
|
|
|
new DataStackAuthError(msg, authErr.code, authErr.statusCode, ack)
|
|
|
|
|
);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
resolve(false);
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Executes one zero-CPU IDLE / active recovery heartbeat step:
|
|
|
|
|
* - If WebSocket is already connected (`IDLE_WS`), stops the active recovery timer and returns `true`.
|
|
|
|
|
* - If disconnected/fallback, attempts WS reconnection + `ping_verify` verification, drains spool,
|
|
|
|
|
* and returns to zero-CPU `IDLE_WS` upon recovery.
|
|
|
|
|
*/
|
|
|
|
|
public async runHeartbeatStep(): Promise<boolean> {
|
|
|
|
|
if (this.heartbeatStopped || this.heartbeatInFlight) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
if (this.isConnected && this.socket?.connected) {
|
|
|
|
|
this.transportState = 'IDLE_WS';
|
|
|
|
|
this.stopHeartbeatMonitor();
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
this.heartbeatInFlight = true;
|
|
|
|
|
try {
|
|
|
|
|
const reconnected = await this.connect();
|
|
|
|
|
if (!reconnected) {
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
const verified = await this.pingVerify();
|
|
|
|
|
if (!verified) {
|
|
|
|
|
if (this.socket) {
|
|
|
|
|
this.socket.removeAllListeners();
|
|
|
|
|
this.socket.disconnect();
|
|
|
|
|
this.socket = null;
|
|
|
|
|
}
|
|
|
|
|
this.isConnected = false;
|
|
|
|
|
this.transportState = this.config.enableHttpFallback
|
|
|
|
|
? 'HTTP_FALLBACK'
|
|
|
|
|
: 'DISCONNECTED';
|
|
|
|
|
this.startHeartbeatMonitor();
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
this.transportState = 'IDLE_WS';
|
|
|
|
|
this.stopHeartbeatMonitor();
|
|
|
|
|
if (this.spooler.count() > 0) {
|
|
|
|
|
void this.flushSpool().catch(() => {});
|
|
|
|
|
}
|
|
|
|
|
return true;
|
|
|
|
|
} catch (err) {
|
|
|
|
|
if (err instanceof DataStackAuthError) {
|
|
|
|
|
this.heartbeatStopped = true;
|
|
|
|
|
this.stopHeartbeatMonitor();
|
|
|
|
|
}
|
|
|
|
|
return false;
|
|
|
|
|
} finally {
|
|
|
|
|
this.heartbeatInFlight = false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-11-20 18:48:54 +06:00
|
|
|
/**
|
|
|
|
|
* Establishes a real-time Socket.IO connection to Port 4000 (`serverUrl`).
|
|
|
|
|
*/
|
|
|
|
|
public async connect(): Promise<boolean> {
|
|
|
|
|
if (!this.config.clientToken) {
|
|
|
|
|
throw new DataStackConfigError(
|
|
|
|
|
'Cannot connect: clientToken is required.'
|
|
|
|
|
);
|
|
|
|
|
}
|
2026-11-21 00:06:55 +06:00
|
|
|
this.heartbeatStopped = false;
|
2026-11-20 18:48:54 +06:00
|
|
|
if (this.isConnected && this.socket?.connected) {
|
2026-11-21 00:06:55 +06:00
|
|
|
this.stopHeartbeatMonitor();
|
2026-11-20 18:48:54 +06:00
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (this.socket) {
|
|
|
|
|
this.socket.removeAllListeners();
|
|
|
|
|
this.socket.disconnect();
|
|
|
|
|
this.socket = null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return new Promise<boolean>((resolve, reject) => {
|
|
|
|
|
const sock = io(this.config.serverUrl, {
|
|
|
|
|
auth: {
|
|
|
|
|
token: this.config.clientToken,
|
|
|
|
|
clientId: this.clientId || undefined,
|
|
|
|
|
},
|
|
|
|
|
transports: ['websocket', 'polling'],
|
|
|
|
|
reconnection: false,
|
|
|
|
|
timeout: this.config.requestTimeoutMs,
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
this.socket = sock;
|
|
|
|
|
let settled = false;
|
|
|
|
|
|
|
|
|
|
const timer = setTimeout(() => {
|
|
|
|
|
if (settled) return;
|
|
|
|
|
settled = true;
|
|
|
|
|
sock.disconnect();
|
|
|
|
|
this.isConnected = false;
|
|
|
|
|
this.transportState = this.config.enableHttpFallback
|
|
|
|
|
? 'HTTP_FALLBACK'
|
|
|
|
|
: 'DISCONNECTED';
|
2026-11-21 00:06:55 +06:00
|
|
|
this.startHeartbeatMonitor();
|
2026-11-20 18:48:54 +06:00
|
|
|
resolve(false);
|
|
|
|
|
}, this.config.requestTimeoutMs);
|
|
|
|
|
|
|
|
|
|
sock.once('connect', () => {
|
|
|
|
|
if (settled) return;
|
|
|
|
|
settled = true;
|
|
|
|
|
clearTimeout(timer);
|
|
|
|
|
this.isConnected = true;
|
|
|
|
|
this.transportState = 'IDLE_WS';
|
2026-11-21 00:06:55 +06:00
|
|
|
this.stopHeartbeatMonitor();
|
2026-11-20 18:48:54 +06:00
|
|
|
void this.flushSpool().catch(() => {});
|
|
|
|
|
resolve(true);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
sock.once('connect_error', (err: Error & { data?: Record<string, unknown> }) => {
|
|
|
|
|
if (settled) return;
|
|
|
|
|
settled = true;
|
|
|
|
|
clearTimeout(timer);
|
|
|
|
|
this.isConnected = false;
|
|
|
|
|
sock.disconnect();
|
|
|
|
|
|
|
|
|
|
const msg = String(err?.message || '');
|
|
|
|
|
const code = String(err?.data?.code || '');
|
2026-11-21 00:06:55 +06:00
|
|
|
const authErr = classifyAuthRejection(code, msg);
|
|
|
|
|
if (authErr) {
|
|
|
|
|
this.heartbeatStopped = true;
|
|
|
|
|
this.stopHeartbeatMonitor();
|
2026-11-20 18:48:54 +06:00
|
|
|
reject(
|
|
|
|
|
new DataStackAuthError(
|
|
|
|
|
msg || 'Socket authentication failed.',
|
2026-11-21 00:06:55 +06:00
|
|
|
authErr.code,
|
|
|
|
|
authErr.statusCode,
|
|
|
|
|
err?.data
|
2026-11-20 18:48:54 +06:00
|
|
|
)
|
|
|
|
|
);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
this.transportState = this.config.enableHttpFallback
|
|
|
|
|
? 'HTTP_FALLBACK'
|
|
|
|
|
: 'DISCONNECTED';
|
2026-11-21 00:06:55 +06:00
|
|
|
this.startHeartbeatMonitor();
|
2026-11-20 18:48:54 +06:00
|
|
|
resolve(false);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
sock.on('disconnect', () => {
|
|
|
|
|
this.isConnected = false;
|
|
|
|
|
if (this.transportState === 'IDLE_WS') {
|
|
|
|
|
this.transportState = this.config.enableHttpFallback
|
|
|
|
|
? 'HTTP_FALLBACK'
|
|
|
|
|
: 'DISCONNECTED';
|
|
|
|
|
}
|
2026-11-21 00:06:55 +06:00
|
|
|
if (!this.heartbeatStopped) {
|
|
|
|
|
this.startHeartbeatMonitor();
|
|
|
|
|
}
|
2026-11-20 18:48:54 +06:00
|
|
|
});
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public disconnect(): void {
|
2026-11-21 00:06:55 +06:00
|
|
|
this.heartbeatStopped = true;
|
|
|
|
|
this.stopHeartbeatMonitor();
|
2026-11-20 18:48:54 +06:00
|
|
|
if (this.socket) {
|
|
|
|
|
this.socket.removeAllListeners();
|
|
|
|
|
this.socket.disconnect();
|
|
|
|
|
this.socket = null;
|
|
|
|
|
}
|
|
|
|
|
this.isConnected = false;
|
|
|
|
|
this.transportState = 'DISCONNECTED';
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private normalizeSingleEnvelope(
|
|
|
|
|
payload: DataStackEventPayload,
|
|
|
|
|
explicitEventId?: string,
|
|
|
|
|
explicitTimestamp?: string
|
|
|
|
|
): DataStackEnvelope {
|
|
|
|
|
if (!payload || typeof payload !== 'object' || Array.isArray(payload)) {
|
|
|
|
|
throw new DataStackConfigError(
|
|
|
|
|
'Event payload must be a non-null plain object.'
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const rawCopy: Record<string, unknown> = { ...payload };
|
|
|
|
|
const eventId = String(
|
|
|
|
|
explicitEventId ||
|
|
|
|
|
(typeof rawCopy.eventId === 'string' && rawCopy.eventId.trim()
|
|
|
|
|
? rawCopy.eventId.trim()
|
|
|
|
|
: makeEventId())
|
|
|
|
|
);
|
|
|
|
|
const timestamp = String(
|
|
|
|
|
explicitTimestamp ||
|
|
|
|
|
(typeof rawCopy.timestamp === 'string' && rawCopy.timestamp.trim()
|
|
|
|
|
? rawCopy.timestamp.trim()
|
|
|
|
|
: new Date().toISOString())
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
let innerData: Record<string, unknown>;
|
|
|
|
|
if (
|
|
|
|
|
rawCopy.data &&
|
|
|
|
|
typeof rawCopy.data === 'object' &&
|
|
|
|
|
!Array.isArray(rawCopy.data)
|
|
|
|
|
) {
|
|
|
|
|
innerData = { ...(rawCopy.data as Record<string, unknown>) };
|
|
|
|
|
} else {
|
|
|
|
|
delete rawCopy.eventId;
|
|
|
|
|
delete rawCopy.timestamp;
|
|
|
|
|
delete rawCopy.clientId;
|
|
|
|
|
innerData = rawCopy;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (Object.keys(innerData).length === 0) {
|
|
|
|
|
throw new DataStackConfigError(
|
|
|
|
|
'Event payload cannot be empty after stripping envelope metadata.'
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return {
|
|
|
|
|
clientId: this.clientId,
|
|
|
|
|
eventId,
|
|
|
|
|
timestamp,
|
|
|
|
|
data: innerData,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private async emitSocketSingle(
|
|
|
|
|
envelope: DataStackEnvelope
|
|
|
|
|
): Promise<DataStackSendResult | null> {
|
|
|
|
|
if (!this.isConnected || !this.socket || !this.socket.connected) {
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return new Promise<DataStackSendResult | null>((resolve, reject) => {
|
|
|
|
|
const timer = setTimeout(() => {
|
|
|
|
|
resolve(null);
|
|
|
|
|
}, this.config.requestTimeoutMs);
|
|
|
|
|
|
|
|
|
|
this.socket!.emit(
|
|
|
|
|
'sync_data',
|
|
|
|
|
{
|
|
|
|
|
clientId: envelope.clientId,
|
|
|
|
|
eventId: envelope.eventId,
|
|
|
|
|
timestamp: envelope.timestamp,
|
|
|
|
|
data: envelope.data,
|
|
|
|
|
},
|
|
|
|
|
(ack: Record<string, unknown> | undefined) => {
|
|
|
|
|
clearTimeout(timer);
|
|
|
|
|
if (!ack || typeof ack !== 'object') {
|
|
|
|
|
resolve(null);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
if (ack.success === true) {
|
|
|
|
|
this.transportState = 'IDLE_WS';
|
2026-11-21 00:06:55 +06:00
|
|
|
const isDuplicate =
|
|
|
|
|
ack.note === 'DUPLICATE_DROPPED' ||
|
|
|
|
|
ack.note === 'DUPLICATE_EVENT' ||
|
|
|
|
|
ack.duplicate === true;
|
2026-11-20 18:48:54 +06:00
|
|
|
resolve({
|
|
|
|
|
success: true,
|
|
|
|
|
transport: 'websocket',
|
|
|
|
|
eventId: String(ack.eventId || envelope.eventId),
|
2026-11-21 00:06:55 +06:00
|
|
|
accepted: isDuplicate ? 0 : 1,
|
|
|
|
|
duplicates: isDuplicate ? 1 : 0,
|
2026-11-20 18:48:54 +06:00
|
|
|
});
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
const code = String(ack.code || 'SOCKET_REJECTED');
|
|
|
|
|
const msg = String(ack.error || 'Socket event rejected');
|
2026-11-21 00:06:55 +06:00
|
|
|
const authErr = classifyAuthRejection(code, msg);
|
|
|
|
|
if (authErr) {
|
|
|
|
|
this.heartbeatStopped = true;
|
|
|
|
|
this.stopHeartbeatMonitor();
|
|
|
|
|
reject(
|
|
|
|
|
new DataStackAuthError(msg, authErr.code, authErr.statusCode, ack)
|
|
|
|
|
);
|
2026-11-20 18:48:54 +06:00
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
if (code === 'STORAGE_QUOTA_EXCEEDED' || code === 'DISK_SPACE_LOW') {
|
|
|
|
|
reject(new DataStackQuotaExceededError(msg, code));
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
if (code === 'PAYLOAD_TOO_LARGE') {
|
|
|
|
|
reject(new DataStackPayloadTooLargeError(msg));
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
if (code === 'RATE_LIMIT_EXCEEDED') {
|
|
|
|
|
reject(new DataStackRateLimitError(msg, [envelope.eventId]));
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
resolve(null);
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private async sendHttpSingle(
|
|
|
|
|
envelope: DataStackEnvelope
|
|
|
|
|
): Promise<DataStackSendResult> {
|
|
|
|
|
if (!this.config.clientToken) {
|
|
|
|
|
throw new DataStackConfigError(
|
|
|
|
|
'clientToken is required for HTTP event ingestion.'
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const controller = new AbortController();
|
|
|
|
|
const timer = setTimeout(
|
|
|
|
|
() => controller.abort(),
|
|
|
|
|
this.config.requestTimeoutMs
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
let res: Response;
|
|
|
|
|
try {
|
|
|
|
|
res = await this.fetchFn(`${this.config.apiUrl}/api/events`, {
|
|
|
|
|
method: 'POST',
|
|
|
|
|
headers: {
|
|
|
|
|
'Content-Type': 'application/json',
|
|
|
|
|
Authorization: `Bearer ${this.config.clientToken}`,
|
|
|
|
|
},
|
|
|
|
|
body: JSON.stringify({
|
|
|
|
|
clientId: envelope.clientId || undefined,
|
|
|
|
|
eventId: envelope.eventId,
|
|
|
|
|
timestamp: envelope.timestamp,
|
|
|
|
|
data: envelope.data,
|
|
|
|
|
}),
|
|
|
|
|
signal: controller.signal,
|
|
|
|
|
});
|
|
|
|
|
} finally {
|
|
|
|
|
clearTimeout(timer);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const body = (await res.json().catch(() => ({}))) as Record<string, unknown>;
|
|
|
|
|
if (res.ok && body.success !== false) {
|
|
|
|
|
this.transportState = 'HTTP_FALLBACK';
|
2026-11-21 00:06:55 +06:00
|
|
|
const isDuplicate =
|
|
|
|
|
body.note === 'DUPLICATE_DROPPED' ||
|
|
|
|
|
body.note === 'DUPLICATE_EVENT' ||
|
|
|
|
|
body.duplicate === true;
|
2026-11-20 18:48:54 +06:00
|
|
|
return {
|
|
|
|
|
success: true,
|
|
|
|
|
transport: 'http',
|
|
|
|
|
eventId: String(body.eventId || envelope.eventId),
|
2026-11-21 00:06:55 +06:00
|
|
|
accepted: isDuplicate ? 0 : 1,
|
|
|
|
|
duplicates: isDuplicate ? 1 : 0,
|
2026-11-20 18:48:54 +06:00
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const code = String(body.code || `HTTP_${res.status}`);
|
|
|
|
|
const errMsg = String(body.error || `HTTP ${res.status} error`);
|
|
|
|
|
|
|
|
|
|
if (res.status === 401) {
|
2026-11-21 00:06:55 +06:00
|
|
|
this.heartbeatStopped = true;
|
|
|
|
|
this.stopHeartbeatMonitor();
|
|
|
|
|
const authCode: DataStackAuthErrorCode =
|
|
|
|
|
code === 'CLIENT_TOKEN_EXPIRED'
|
|
|
|
|
? 'CLIENT_TOKEN_EXPIRED'
|
|
|
|
|
: code === 'TOKEN_ROTATED'
|
|
|
|
|
? 'TOKEN_ROTATED'
|
|
|
|
|
: 'AUTH_FAILED';
|
|
|
|
|
throw new DataStackAuthError(errMsg, authCode, 401, body);
|
2026-11-20 18:48:54 +06:00
|
|
|
}
|
|
|
|
|
if (res.status === 403) {
|
2026-11-21 00:06:55 +06:00
|
|
|
this.heartbeatStopped = true;
|
|
|
|
|
this.stopHeartbeatMonitor();
|
2026-11-20 18:48:54 +06:00
|
|
|
throw new DataStackAuthError(
|
|
|
|
|
errMsg,
|
|
|
|
|
code === 'CLIENT_IP_FORBIDDEN' ? 'CLIENT_IP_FORBIDDEN' : 'CLIENT_REVOKED',
|
|
|
|
|
403,
|
|
|
|
|
body
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
if (res.status === 413 || code === 'PAYLOAD_TOO_LARGE') {
|
|
|
|
|
throw new DataStackPayloadTooLargeError(errMsg, body);
|
|
|
|
|
}
|
|
|
|
|
if (res.status === 429 || code === 'RATE_LIMIT_EXCEEDED') {
|
|
|
|
|
throw new DataStackRateLimitError(errMsg, [envelope.eventId], body);
|
|
|
|
|
}
|
|
|
|
|
if (
|
|
|
|
|
res.status === 507 ||
|
|
|
|
|
code === 'STORAGE_QUOTA_EXCEEDED' ||
|
|
|
|
|
code === 'DISK_SPACE_LOW'
|
|
|
|
|
) {
|
|
|
|
|
throw new DataStackQuotaExceededError(
|
|
|
|
|
errMsg,
|
|
|
|
|
code === 'DISK_SPACE_LOW' ? 'DISK_SPACE_LOW' : 'STORAGE_QUOTA_EXCEEDED',
|
|
|
|
|
body
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
throw new DataStackError(errMsg, code, res.status, body);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Sends a single telemetry event via WebSocket primary transport, falling back
|
|
|
|
|
* automatically to HTTP `POST /api/events` and offline disk spooling.
|
|
|
|
|
*/
|
|
|
|
|
public async send(
|
|
|
|
|
payload: DataStackEventPayload
|
|
|
|
|
): Promise<DataStackSendResult> {
|
|
|
|
|
const task = (async (): Promise<DataStackSendResult> => {
|
|
|
|
|
const envelope = this.normalizeSingleEnvelope(payload);
|
|
|
|
|
|
|
|
|
|
// 1. Try active WebSocket
|
|
|
|
|
const wsResult = await this.emitSocketSingle(envelope);
|
|
|
|
|
if (wsResult) {
|
|
|
|
|
if (this.spooler.count() > 0) {
|
|
|
|
|
void this.flushSpool().catch(() => {});
|
|
|
|
|
}
|
|
|
|
|
return wsResult;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 2. Try HTTP Fallback with bounded retry on 429 / transient 5xx
|
|
|
|
|
if (this.config.enableHttpFallback) {
|
|
|
|
|
let attempt = 0;
|
|
|
|
|
while (attempt <= this.config.maxRetries) {
|
|
|
|
|
try {
|
|
|
|
|
const httpRes = await this.sendHttpSingle(envelope);
|
|
|
|
|
if (this.spooler.count() > 0) {
|
|
|
|
|
void this.flushSpool().catch(() => {});
|
|
|
|
|
}
|
|
|
|
|
return httpRes;
|
|
|
|
|
} catch (err) {
|
|
|
|
|
if (
|
|
|
|
|
err instanceof DataStackAuthError ||
|
|
|
|
|
err instanceof DataStackQuotaExceededError ||
|
|
|
|
|
err instanceof DataStackPayloadTooLargeError ||
|
|
|
|
|
err instanceof DataStackConfigError
|
|
|
|
|
) {
|
|
|
|
|
throw err;
|
|
|
|
|
}
|
|
|
|
|
if (
|
|
|
|
|
err instanceof DataStackRateLimitError &&
|
|
|
|
|
attempt < this.config.maxRetries
|
|
|
|
|
) {
|
|
|
|
|
attempt += 1;
|
|
|
|
|
await new Promise((r) => setTimeout(r, 50 * attempt));
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
if (err instanceof DataStackRateLimitError) {
|
|
|
|
|
throw err;
|
|
|
|
|
}
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 3. Offline Disk Spool fallback
|
|
|
|
|
if (this.spooler.enabled) {
|
|
|
|
|
const spooled = this.spooler.enqueue(envelope);
|
|
|
|
|
if (spooled > 0) {
|
|
|
|
|
this.transportState = 'OFFLINE_SPOOL';
|
|
|
|
|
return {
|
|
|
|
|
success: true,
|
|
|
|
|
transport: 'spool',
|
|
|
|
|
eventId: envelope.eventId,
|
|
|
|
|
accepted: 0,
|
|
|
|
|
spooled,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
throw new DataStackError(
|
|
|
|
|
'Failed to deliver event via WebSocket, HTTP fallback, or offline spool.',
|
|
|
|
|
'TRANSPORT_UNAVAILABLE',
|
|
|
|
|
503
|
|
|
|
|
);
|
|
|
|
|
})();
|
|
|
|
|
|
|
|
|
|
return this.trackInFlight(task);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Alias for `send()` for 1:1 naming parity with Python SDK (`client.sync()`).
|
|
|
|
|
*/
|
|
|
|
|
public async sync(
|
|
|
|
|
payload: DataStackEventPayload
|
|
|
|
|
): Promise<DataStackSendResult> {
|
|
|
|
|
return this.send(payload);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private async sendBatchChunkInternal(
|
|
|
|
|
chunk: DataStackEnvelope[]
|
|
|
|
|
): Promise<DataStackSendResult> {
|
|
|
|
|
// 1. Try WebSocket batch_sync if connected
|
|
|
|
|
if (this.isConnected && this.socket && this.socket.connected) {
|
|
|
|
|
const wsBatchRes = await new Promise<DataStackSendResult | null>(
|
|
|
|
|
(resolve, reject) => {
|
|
|
|
|
const timer = setTimeout(
|
|
|
|
|
() => resolve(null),
|
|
|
|
|
this.config.requestTimeoutMs
|
|
|
|
|
);
|
|
|
|
|
this.socket!.emit(
|
|
|
|
|
'batch_sync',
|
2026-11-21 00:06:55 +06:00
|
|
|
chunk.map((item) => ({
|
|
|
|
|
clientId: item.clientId || this.clientId || undefined,
|
|
|
|
|
eventId: item.eventId,
|
|
|
|
|
timestamp: item.timestamp,
|
|
|
|
|
data: item.data,
|
|
|
|
|
})),
|
2026-11-20 18:48:54 +06:00
|
|
|
(ack: Record<string, unknown> | undefined) => {
|
|
|
|
|
clearTimeout(timer);
|
|
|
|
|
if (!ack || typeof ack !== 'object') {
|
|
|
|
|
resolve(null);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
if (ack.success === true) {
|
|
|
|
|
const rateLimited = Array.isArray(ack.rateLimitedEventIds)
|
|
|
|
|
? (ack.rateLimitedEventIds as string[])
|
|
|
|
|
: [];
|
2026-11-21 00:06:55 +06:00
|
|
|
const duplicates = Number(ack.duplicates ?? 0);
|
|
|
|
|
const acceptedVal =
|
|
|
|
|
ack.processed ??
|
|
|
|
|
ack.processedCount ??
|
|
|
|
|
ack.accepted ??
|
|
|
|
|
Math.max(0, chunk.length - rateLimited.length - duplicates);
|
2026-11-20 18:48:54 +06:00
|
|
|
resolve({
|
|
|
|
|
success: true,
|
|
|
|
|
transport: 'websocket',
|
2026-11-21 00:06:55 +06:00
|
|
|
accepted: Number(acceptedVal),
|
|
|
|
|
duplicates,
|
2026-11-20 18:48:54 +06:00
|
|
|
rateLimitedEventIds: rateLimited,
|
|
|
|
|
});
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
const code = String(ack.code || 'BATCH_REJECTED');
|
|
|
|
|
const msg = String(ack.error || 'Batch rejected');
|
2026-11-21 00:06:55 +06:00
|
|
|
const authErr = classifyAuthRejection(code, msg);
|
|
|
|
|
if (authErr) {
|
|
|
|
|
this.heartbeatStopped = true;
|
|
|
|
|
this.stopHeartbeatMonitor();
|
|
|
|
|
reject(
|
|
|
|
|
new DataStackAuthError(
|
|
|
|
|
msg,
|
|
|
|
|
authErr.code,
|
|
|
|
|
authErr.statusCode,
|
|
|
|
|
ack
|
|
|
|
|
)
|
|
|
|
|
);
|
2026-11-20 18:48:54 +06:00
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
if (code === 'STORAGE_QUOTA_EXCEEDED' || code === 'DISK_SPACE_LOW') {
|
|
|
|
|
reject(new DataStackQuotaExceededError(msg, code));
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
if (code === 'PAYLOAD_TOO_LARGE') {
|
|
|
|
|
reject(new DataStackPayloadTooLargeError(msg));
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
if (code === 'RATE_LIMIT_EXCEEDED') {
|
|
|
|
|
const rateLimited = Array.isArray(ack.rateLimitedEventIds)
|
|
|
|
|
? (ack.rateLimitedEventIds as string[])
|
|
|
|
|
: chunk.map((c) => c.eventId);
|
|
|
|
|
reject(new DataStackRateLimitError(msg, rateLimited));
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
resolve(null);
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
if (wsBatchRes) {
|
|
|
|
|
return wsBatchRes;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// 2. Try HTTP POST /api/events batch
|
|
|
|
|
if (!this.config.enableHttpFallback) {
|
|
|
|
|
throw new DataStackError(
|
|
|
|
|
'WebSocket disconnected and HTTP fallback is disabled.',
|
|
|
|
|
'TRANSPORT_UNAVAILABLE',
|
|
|
|
|
503
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
if (!this.config.clientToken) {
|
|
|
|
|
throw new DataStackConfigError(
|
|
|
|
|
'clientToken is required for HTTP batch ingestion.'
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const controller = new AbortController();
|
|
|
|
|
const timer = setTimeout(
|
|
|
|
|
() => controller.abort(),
|
|
|
|
|
this.config.requestTimeoutMs
|
|
|
|
|
);
|
|
|
|
|
let res: Response;
|
|
|
|
|
try {
|
|
|
|
|
res = await this.fetchFn(`${this.config.apiUrl}/api/events`, {
|
|
|
|
|
method: 'POST',
|
|
|
|
|
headers: {
|
|
|
|
|
'Content-Type': 'application/json',
|
|
|
|
|
Authorization: `Bearer ${this.config.clientToken}`,
|
|
|
|
|
},
|
|
|
|
|
body: JSON.stringify({
|
|
|
|
|
clientId: this.clientId || undefined,
|
|
|
|
|
batch: chunk.map((item) => ({
|
|
|
|
|
eventId: item.eventId,
|
|
|
|
|
timestamp: item.timestamp,
|
|
|
|
|
data: item.data,
|
|
|
|
|
})),
|
|
|
|
|
}),
|
|
|
|
|
signal: controller.signal,
|
|
|
|
|
});
|
|
|
|
|
} finally {
|
|
|
|
|
clearTimeout(timer);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const body = (await res.json().catch(() => ({}))) as Record<string, unknown>;
|
|
|
|
|
const rateLimited = Array.isArray(body.rateLimitedEventIds)
|
|
|
|
|
? (body.rateLimitedEventIds as string[])
|
|
|
|
|
: [];
|
|
|
|
|
|
|
|
|
|
if (res.ok && body.success !== false) {
|
|
|
|
|
this.transportState = 'HTTP_FALLBACK';
|
2026-11-21 00:06:55 +06:00
|
|
|
const duplicates = Number(body.duplicates ?? 0);
|
|
|
|
|
const acceptedVal =
|
|
|
|
|
body.processed ??
|
|
|
|
|
body.processedCount ??
|
|
|
|
|
body.accepted ??
|
|
|
|
|
Math.max(0, chunk.length - rateLimited.length - duplicates);
|
2026-11-20 18:48:54 +06:00
|
|
|
return {
|
|
|
|
|
success: true,
|
|
|
|
|
transport: 'http',
|
2026-11-21 00:06:55 +06:00
|
|
|
accepted: Number(acceptedVal),
|
|
|
|
|
duplicates,
|
2026-11-20 18:48:54 +06:00
|
|
|
rateLimitedEventIds: rateLimited,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const code = String(body.code || `HTTP_${res.status}`);
|
|
|
|
|
const errMsg = String(body.error || `HTTP ${res.status} batch error`);
|
|
|
|
|
|
|
|
|
|
if (res.status === 401) {
|
2026-11-21 00:06:55 +06:00
|
|
|
this.heartbeatStopped = true;
|
|
|
|
|
this.stopHeartbeatMonitor();
|
|
|
|
|
const authCode: DataStackAuthErrorCode =
|
|
|
|
|
code === 'CLIENT_TOKEN_EXPIRED'
|
|
|
|
|
? 'CLIENT_TOKEN_EXPIRED'
|
|
|
|
|
: code === 'TOKEN_ROTATED'
|
|
|
|
|
? 'TOKEN_ROTATED'
|
|
|
|
|
: 'AUTH_FAILED';
|
|
|
|
|
throw new DataStackAuthError(errMsg, authCode, 401, body);
|
2026-11-20 18:48:54 +06:00
|
|
|
}
|
|
|
|
|
if (res.status === 403) {
|
2026-11-21 00:06:55 +06:00
|
|
|
this.heartbeatStopped = true;
|
|
|
|
|
this.stopHeartbeatMonitor();
|
2026-11-20 18:48:54 +06:00
|
|
|
throw new DataStackAuthError(
|
|
|
|
|
errMsg,
|
|
|
|
|
code === 'CLIENT_IP_FORBIDDEN' ? 'CLIENT_IP_FORBIDDEN' : 'CLIENT_REVOKED',
|
|
|
|
|
403,
|
|
|
|
|
body
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
if (res.status === 413 || code === 'PAYLOAD_TOO_LARGE') {
|
|
|
|
|
throw new DataStackPayloadTooLargeError(errMsg, body);
|
|
|
|
|
}
|
|
|
|
|
if (res.status === 429 || code === 'RATE_LIMIT_EXCEEDED') {
|
|
|
|
|
throw new DataStackRateLimitError(
|
|
|
|
|
errMsg,
|
|
|
|
|
rateLimited.length > 0 ? rateLimited : chunk.map((c) => c.eventId),
|
|
|
|
|
body
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
if (
|
|
|
|
|
res.status === 507 ||
|
|
|
|
|
code === 'STORAGE_QUOTA_EXCEEDED' ||
|
|
|
|
|
code === 'DISK_SPACE_LOW'
|
|
|
|
|
) {
|
|
|
|
|
throw new DataStackQuotaExceededError(
|
|
|
|
|
errMsg,
|
|
|
|
|
code === 'DISK_SPACE_LOW' ? 'DISK_SPACE_LOW' : 'STORAGE_QUOTA_EXCEEDED',
|
|
|
|
|
body
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
throw new DataStackError(errMsg, code, res.status, body);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private async deliverChunkWithSplitAndRetry(
|
|
|
|
|
chunk: DataStackEnvelope[],
|
|
|
|
|
allowSpoolFallback = true
|
|
|
|
|
): Promise<DataStackSendResult> {
|
|
|
|
|
let pending = [...chunk];
|
|
|
|
|
let totalAccepted = 0;
|
2026-11-21 00:06:55 +06:00
|
|
|
let totalDuplicates = 0;
|
2026-11-20 18:48:54 +06:00
|
|
|
let totalSpooled = 0;
|
|
|
|
|
let lastTransport: 'websocket' | 'http' | 'spool' = 'http';
|
|
|
|
|
let attempt = 0;
|
|
|
|
|
|
|
|
|
|
while (pending.length > 0) {
|
|
|
|
|
try {
|
|
|
|
|
const res = await this.sendBatchChunkInternal(pending);
|
|
|
|
|
lastTransport = res.transport;
|
|
|
|
|
const rateLimitedSet = new Set(res.rateLimitedEventIds || []);
|
2026-11-21 00:06:55 +06:00
|
|
|
const duplicatesNow = res.duplicates || 0;
|
|
|
|
|
totalDuplicates += duplicatesNow;
|
2026-11-20 18:48:54 +06:00
|
|
|
|
|
|
|
|
if (rateLimitedSet.size === 0) {
|
2026-11-21 00:06:55 +06:00
|
|
|
totalAccepted += res.accepted ?? Math.max(0, pending.length - duplicatesNow);
|
2026-11-20 18:48:54 +06:00
|
|
|
pending = [];
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const acceptedNow =
|
2026-11-21 00:06:55 +06:00
|
|
|
res.accepted ??
|
|
|
|
|
Math.max(0, pending.length - rateLimitedSet.size - duplicatesNow);
|
2026-11-20 18:48:54 +06:00
|
|
|
totalAccepted += acceptedNow;
|
|
|
|
|
pending = pending.filter((item) => rateLimitedSet.has(item.eventId));
|
|
|
|
|
|
|
|
|
|
if (attempt >= this.config.maxRetries) {
|
|
|
|
|
throw new DataStackRateLimitError(
|
|
|
|
|
`Rate limit exceeded for ${pending.length} event(s) after ${this.config.maxRetries} retries.`,
|
|
|
|
|
pending.map((p) => p.eventId)
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
attempt += 1;
|
|
|
|
|
await new Promise((r) => setTimeout(r, 50 * attempt));
|
|
|
|
|
} catch (err) {
|
|
|
|
|
if (
|
|
|
|
|
err instanceof DataStackPayloadTooLargeError &&
|
|
|
|
|
pending.length > 1
|
|
|
|
|
) {
|
|
|
|
|
const mid = Math.floor(pending.length / 2);
|
|
|
|
|
const left = await this.deliverChunkWithSplitAndRetry(
|
|
|
|
|
pending.slice(0, mid),
|
|
|
|
|
allowSpoolFallback
|
|
|
|
|
);
|
|
|
|
|
const right = await this.deliverChunkWithSplitAndRetry(
|
|
|
|
|
pending.slice(mid),
|
|
|
|
|
allowSpoolFallback
|
|
|
|
|
);
|
|
|
|
|
return {
|
|
|
|
|
success: left.success && right.success,
|
|
|
|
|
transport: right.transport,
|
|
|
|
|
accepted: (left.accepted || 0) + (right.accepted || 0),
|
2026-11-21 00:06:55 +06:00
|
|
|
duplicates: (left.duplicates || 0) + (right.duplicates || 0),
|
2026-11-20 18:48:54 +06:00
|
|
|
spooled: (left.spooled || 0) + (right.spooled || 0),
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (
|
|
|
|
|
err instanceof DataStackRateLimitError &&
|
|
|
|
|
attempt < this.config.maxRetries
|
|
|
|
|
) {
|
|
|
|
|
const rlSet = new Set(err.rateLimitedEventIds);
|
|
|
|
|
if (rlSet.size > 0 && rlSet.size < pending.length) {
|
|
|
|
|
totalAccepted += pending.length - rlSet.size;
|
|
|
|
|
pending = pending.filter((p) => rlSet.has(p.eventId));
|
|
|
|
|
}
|
|
|
|
|
attempt += 1;
|
|
|
|
|
await new Promise((r) => setTimeout(r, 50 * attempt));
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (
|
|
|
|
|
err instanceof DataStackAuthError ||
|
|
|
|
|
err instanceof DataStackQuotaExceededError ||
|
|
|
|
|
err instanceof DataStackPayloadTooLargeError ||
|
|
|
|
|
err instanceof DataStackRateLimitError ||
|
|
|
|
|
err instanceof DataStackConfigError
|
|
|
|
|
) {
|
|
|
|
|
throw err;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (allowSpoolFallback && this.spooler.enabled) {
|
|
|
|
|
const spooled = this.spooler.enqueue(pending);
|
|
|
|
|
if (spooled > 0) {
|
|
|
|
|
totalSpooled += spooled;
|
|
|
|
|
lastTransport = 'spool';
|
|
|
|
|
this.transportState = 'OFFLINE_SPOOL';
|
|
|
|
|
pending = [];
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
throw err;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return {
|
|
|
|
|
success: pending.length === 0,
|
|
|
|
|
transport: lastTransport,
|
|
|
|
|
accepted: totalAccepted,
|
2026-11-21 00:06:55 +06:00
|
|
|
duplicates: totalDuplicates,
|
2026-11-20 18:48:54 +06:00
|
|
|
spooled: totalSpooled,
|
|
|
|
|
unprocessedItems: pending.map((p) => p.data),
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Sends a batch of telemetry events with automatic chunking (`batchChunkSize`),
|
|
|
|
|
* recursive `413 PAYLOAD_TOO_LARGE` splitting, partial-batch `429` retry, and
|
|
|
|
|
* offline disk spooling.
|
|
|
|
|
*/
|
|
|
|
|
public async sendBatch(
|
|
|
|
|
events: DataStackEventPayload[],
|
|
|
|
|
options?: { chunkSize?: number }
|
|
|
|
|
): Promise<DataStackSendResult> {
|
|
|
|
|
const task = (async (): Promise<DataStackSendResult> => {
|
|
|
|
|
if (!Array.isArray(events) || events.length === 0) {
|
|
|
|
|
throw new DataStackConfigError(
|
|
|
|
|
'sendBatch requires a non-empty array of event objects.'
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const envelopes = events.map((ev) => this.normalizeSingleEnvelope(ev));
|
|
|
|
|
const chunkSize = Math.max(
|
|
|
|
|
1,
|
|
|
|
|
Math.min(1000, Number(options?.chunkSize ?? this.config.batchChunkSize))
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
let totalAccepted = 0;
|
2026-11-21 00:06:55 +06:00
|
|
|
let totalDuplicates = 0;
|
2026-11-20 18:48:54 +06:00
|
|
|
let totalSpooled = 0;
|
|
|
|
|
let lastTransport: 'websocket' | 'http' | 'spool' = 'http';
|
|
|
|
|
|
|
|
|
|
for (let i = 0; i < envelopes.length; i += chunkSize) {
|
|
|
|
|
const slice = envelopes.slice(i, i + chunkSize);
|
|
|
|
|
const res = await this.deliverChunkWithSplitAndRetry(slice, true);
|
|
|
|
|
totalAccepted += res.accepted || 0;
|
2026-11-21 00:06:55 +06:00
|
|
|
totalDuplicates += res.duplicates || 0;
|
2026-11-20 18:48:54 +06:00
|
|
|
totalSpooled += res.spooled || 0;
|
|
|
|
|
lastTransport = res.transport;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return {
|
|
|
|
|
success: true,
|
|
|
|
|
transport: lastTransport,
|
|
|
|
|
accepted: totalAccepted,
|
2026-11-21 00:06:55 +06:00
|
|
|
duplicates: totalDuplicates,
|
2026-11-20 18:48:54 +06:00
|
|
|
spooled: totalSpooled,
|
|
|
|
|
};
|
|
|
|
|
})();
|
|
|
|
|
|
|
|
|
|
return this.trackInFlight(task);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Alias for `sendBatch()` for 1:1 parity with Python SDK (`client.batch_sync()`).
|
|
|
|
|
*/
|
|
|
|
|
public async batchSync(
|
|
|
|
|
events: DataStackEventPayload[],
|
|
|
|
|
options?: { chunkSize?: number }
|
|
|
|
|
): Promise<DataStackSendResult> {
|
|
|
|
|
return this.sendBatch(events, options);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Ingests a `.csv`, `.json`, `.jsonl`, or `.ndjson` file directly with
|
|
|
|
|
* deterministic SHA-256 `eventId` deduplication (`file_<stem>_<row>_<hash>`).
|
|
|
|
|
*/
|
|
|
|
|
public async sendFile(
|
|
|
|
|
filePath: string,
|
|
|
|
|
options?: { chunkSize?: number }
|
|
|
|
|
): Promise<DataStackFileSendResult> {
|
|
|
|
|
const task = (async (): Promise<DataStackFileSendResult> => {
|
|
|
|
|
const resolvedPath = path.resolve(filePath);
|
|
|
|
|
if (!fs.existsSync(resolvedPath)) {
|
|
|
|
|
throw new DataStackFileError(`File not found: ${resolvedPath}`);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const ext = path.extname(resolvedPath).toLowerCase();
|
|
|
|
|
const fileStem = path.basename(resolvedPath, ext);
|
|
|
|
|
const rawContent = fs.readFileSync(resolvedPath, 'utf8');
|
|
|
|
|
|
|
|
|
|
let records: DataStackEventPayload[] = [];
|
|
|
|
|
|
|
|
|
|
if (ext === '.csv') {
|
|
|
|
|
const parsed = parseCsvSync(rawContent, {
|
|
|
|
|
columns: true,
|
|
|
|
|
skip_empty_lines: true,
|
|
|
|
|
relax_column_count: true,
|
|
|
|
|
trim: true,
|
|
|
|
|
}) as DataStackEventPayload[];
|
|
|
|
|
records = parsed;
|
|
|
|
|
} else if (ext === '.json') {
|
|
|
|
|
try {
|
|
|
|
|
const parsed = JSON.parse(rawContent) as unknown;
|
|
|
|
|
if (Array.isArray(parsed)) {
|
|
|
|
|
records = parsed as DataStackEventPayload[];
|
|
|
|
|
} else if (parsed && typeof parsed === 'object') {
|
|
|
|
|
records = [parsed as DataStackEventPayload];
|
|
|
|
|
} else {
|
|
|
|
|
throw new DataStackFileError(
|
|
|
|
|
'JSON file must contain an object or array of objects.'
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
} catch (err) {
|
|
|
|
|
if (err instanceof DataStackFileError) throw err;
|
|
|
|
|
throw new DataStackFileError(
|
|
|
|
|
`Invalid JSON file (${resolvedPath}): ${err instanceof Error ? err.message : String(err)}`
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
} else if (ext === '.jsonl' || ext === '.ndjson') {
|
|
|
|
|
const lines = rawContent
|
|
|
|
|
.split('\n')
|
|
|
|
|
.map((l) => l.trim())
|
|
|
|
|
.filter(Boolean);
|
|
|
|
|
records = lines.map((line, idx) => {
|
|
|
|
|
try {
|
|
|
|
|
const obj = JSON.parse(line) as unknown;
|
|
|
|
|
if (!obj || typeof obj !== 'object' || Array.isArray(obj)) {
|
|
|
|
|
throw new Error('Line is not a JSON object');
|
|
|
|
|
}
|
|
|
|
|
return obj as DataStackEventPayload;
|
|
|
|
|
} catch (err) {
|
|
|
|
|
throw new DataStackFileError(
|
|
|
|
|
`Invalid JSONL at line ${idx + 1}: ${err instanceof Error ? err.message : String(err)}`
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
} else {
|
|
|
|
|
throw new DataStackFileError(
|
|
|
|
|
`Unsupported file extension '${ext}'. Supported extensions: .csv, .json, .jsonl, .ndjson`
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (records.length === 0) {
|
|
|
|
|
return {
|
|
|
|
|
success: true,
|
|
|
|
|
transport: this.isConnected ? 'websocket' : 'http',
|
|
|
|
|
filePath: resolvedPath,
|
|
|
|
|
totalRows: 0,
|
|
|
|
|
accepted: 0,
|
|
|
|
|
acceptedRows: 0,
|
|
|
|
|
spooled: 0,
|
|
|
|
|
spooledRows: 0,
|
|
|
|
|
chunksSent: 0,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const envelopes: DataStackEnvelope[] = records.map((rec, idx) => {
|
|
|
|
|
const deterministicId =
|
|
|
|
|
typeof rec.eventId === 'string' && rec.eventId.trim()
|
|
|
|
|
? rec.eventId.trim()
|
|
|
|
|
: makeDeterministicFileEventId(fileStem, idx, rec);
|
|
|
|
|
return this.normalizeSingleEnvelope(rec, deterministicId);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
const chunkSize = Math.max(
|
|
|
|
|
1,
|
|
|
|
|
Math.min(1000, Number(options?.chunkSize ?? this.config.batchChunkSize))
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
let acceptedRows = 0;
|
2026-11-21 00:06:55 +06:00
|
|
|
let duplicateRows = 0;
|
2026-11-20 18:48:54 +06:00
|
|
|
let spooledRows = 0;
|
|
|
|
|
let chunksSent = 0;
|
|
|
|
|
let lastTransport: 'websocket' | 'http' | 'spool' = 'http';
|
|
|
|
|
|
|
|
|
|
for (let i = 0; i < envelopes.length; i += chunkSize) {
|
|
|
|
|
const slice = envelopes.slice(i, i + chunkSize);
|
|
|
|
|
const res = await this.deliverChunkWithSplitAndRetry(slice, true);
|
|
|
|
|
acceptedRows += res.accepted || 0;
|
2026-11-21 00:06:55 +06:00
|
|
|
duplicateRows += res.duplicates || 0;
|
2026-11-20 18:48:54 +06:00
|
|
|
spooledRows += res.spooled || 0;
|
|
|
|
|
chunksSent += 1;
|
|
|
|
|
lastTransport = res.transport;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return {
|
2026-11-21 00:06:55 +06:00
|
|
|
success: acceptedRows + duplicateRows + spooledRows === envelopes.length,
|
2026-11-20 18:48:54 +06:00
|
|
|
transport: lastTransport,
|
|
|
|
|
filePath: resolvedPath,
|
|
|
|
|
totalRows: envelopes.length,
|
|
|
|
|
accepted: acceptedRows,
|
|
|
|
|
acceptedRows,
|
2026-11-21 00:06:55 +06:00
|
|
|
duplicates: duplicateRows,
|
2026-11-20 18:48:54 +06:00
|
|
|
spooled: spooledRows,
|
|
|
|
|
spooledRows,
|
|
|
|
|
chunksSent,
|
|
|
|
|
};
|
|
|
|
|
})();
|
|
|
|
|
|
|
|
|
|
return this.trackInFlight(task);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Replays locally spooled events from `.datastack_spool/spool.jsonl` to the server.
|
|
|
|
|
*/
|
|
|
|
|
public async flushSpool(): Promise<{ drained: number; remaining: number }> {
|
|
|
|
|
if (!this.spooler.enabled || this.isDrainingSpool) {
|
|
|
|
|
return { drained: 0, remaining: this.spooler.count() };
|
|
|
|
|
}
|
|
|
|
|
this.isDrainingSpool = true;
|
|
|
|
|
let drained = 0;
|
|
|
|
|
try {
|
|
|
|
|
while (this.spooler.count() > 0) {
|
|
|
|
|
const batch = this.spooler.peekBatch(this.config.batchChunkSize);
|
|
|
|
|
if (batch.length === 0) break;
|
|
|
|
|
const res = await this.deliverChunkWithSplitAndRetry(batch, false);
|
|
|
|
|
if (res.success) {
|
|
|
|
|
this.spooler.removeFirst(batch.length);
|
|
|
|
|
drained += batch.length;
|
|
|
|
|
} else {
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
} finally {
|
|
|
|
|
this.isDrainingSpool = false;
|
|
|
|
|
}
|
|
|
|
|
return { drained, remaining: this.spooler.count() };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Gracefully awaits all in-flight telemetry requests, attempts to drain any
|
|
|
|
|
* spooled events within `timeoutMs`, and closes the Socket.IO connection.
|
|
|
|
|
*/
|
|
|
|
|
public async shutdown(
|
|
|
|
|
timeoutMs = 5000,
|
|
|
|
|
options?: { throwOnTimeout?: boolean }
|
|
|
|
|
): Promise<{ drained: boolean; remainingInFlight: number; remainingSpooled: number }> {
|
|
|
|
|
const deadline = Date.now() + Math.max(50, timeoutMs);
|
|
|
|
|
|
|
|
|
|
const waitWork = (async () => {
|
|
|
|
|
while (this.inFlightSet.size > 0 && Date.now() < deadline) {
|
|
|
|
|
await Promise.race([
|
|
|
|
|
Promise.allSettled(Array.from(this.inFlightSet)),
|
|
|
|
|
new Promise((r) => setTimeout(r, 25)),
|
|
|
|
|
]);
|
|
|
|
|
}
|
|
|
|
|
if (this.spooler.count() > 0 && Date.now() < deadline) {
|
|
|
|
|
try {
|
|
|
|
|
await this.flushSpool();
|
|
|
|
|
} catch {
|
|
|
|
|
// Best-effort spool drain before shutdown
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
})();
|
|
|
|
|
|
|
|
|
|
const remainingMs = Math.max(10, deadline - Date.now());
|
|
|
|
|
let timedOut = false;
|
|
|
|
|
await Promise.race([
|
|
|
|
|
waitWork,
|
|
|
|
|
new Promise<void>((resolve) =>
|
|
|
|
|
setTimeout(() => {
|
|
|
|
|
timedOut = true;
|
|
|
|
|
resolve();
|
|
|
|
|
}, remainingMs)
|
|
|
|
|
),
|
|
|
|
|
]);
|
|
|
|
|
|
|
|
|
|
this.disconnect();
|
|
|
|
|
const remainingInFlight = this.inFlightSet.size;
|
|
|
|
|
const remainingSpooled = this.spooler.count();
|
|
|
|
|
|
|
|
|
|
if (timedOut && remainingInFlight > 0 && options?.throwOnTimeout) {
|
|
|
|
|
throw new DataStackShutdownTimeoutError(
|
|
|
|
|
`Graceful shutdown timed out after ${timeoutMs}ms with ${remainingInFlight} in-flight request(s).`
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return {
|
|
|
|
|
drained: !timedOut && remainingInFlight === 0,
|
|
|
|
|
remainingInFlight,
|
|
|
|
|
remainingSpooled,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public status(): DataStackStatusSnapshot {
|
|
|
|
|
return {
|
|
|
|
|
connected: this.isConnected && Boolean(this.socket?.connected),
|
|
|
|
|
transportState: this.transportState,
|
|
|
|
|
clientId: this.clientId,
|
|
|
|
|
serverUrl: this.config.serverUrl,
|
|
|
|
|
apiUrl: this.config.apiUrl,
|
|
|
|
|
spooledCount: this.spooler.count(),
|
|
|
|
|
spooledBytes: this.spooler.sizeBytes(),
|
|
|
|
|
inFlightRequests: this.inFlightSet.size,
|
2026-11-21 00:06:55 +06:00
|
|
|
heartbeatActive: this.heartbeatTimer !== null,
|
2026-11-20 18:48:54 +06:00
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export { DataStackClient as DataStack };
|
|
|
|
|
export default DataStackClient;
|