feat(auth): implement RBAC audit logging and session security
Integrate granular audit logging for admin actions, transition to role-based API access via `requireAdminApiSession`, and update session tokens to persist user roles and IDs for consistent multi-admin management.
This commit is contained in:
1 parent
2873b9904a
commit
5c977d6977
37 files changed
+5939
-349
No files matched your search
@@ -0,0 +1,84 @@
|
||||
# DataStack Node.js & TypeScript SDK (`@datastack/sdk` v2.2.0)
|
||||
|
||||
Official server-side Node.js and TypeScript SDK for **DataStack**. Provides 1:1 transport, fallback, deduplication, and offline spooling parity with the official Python SDK (`sdk/python/datastack`).
|
||||
|
||||
---
|
||||
|
||||
## Features
|
||||
|
||||
- **Primary Real-Time WebSocket Transport (`socket.io-client`)**: Streams single events (`sync_data`) and batches (`batch_sync`) to Port `4000` with explicit server persistence acknowledgments.
|
||||
- **Automatic HTTP Fallback (`POST /api/events`)**: Seamlessly falls back to Port `3000` REST ingestion whenever the WebSocket transport is disconnected or unreachable.
|
||||
- **Offline Disk Spooling (`.datastack_spool/spool.jsonl`)**: Persists undelivered events to local disk when both WebSocket and HTTP endpoints are offline, and automatically drains the spool via `flushSpool()` when connectivity recovers.
|
||||
- **Direct File Ingestion (`sendFile()`)**: Streams `.csv`, `.json`, `.jsonl`, and `.ndjson` files with deterministic SHA-256 `eventId` generation (`file_<stem>_<row>_<hash>`) so retries never duplicate rows.
|
||||
- **Typed Error Hierarchy (`V2-E12`)**: Distinguishes terminal errors (`DataStackAuthError` for `401 CLIENT_TOKEN_EXPIRED` / `403 CLIENT_IP_FORBIDDEN`, `DataStackQuotaExceededError` for `507 STORAGE_QUOTA_EXCEEDED` / `DISK_SPACE_LOW`, `DataStackConfigError`) from retryable `429 RATE_LIMIT_EXCEEDED` (`DataStackRateLimitError`) and auto-splittable `413 PAYLOAD_TOO_LARGE` (`DataStackPayloadTooLargeError`).
|
||||
- **Graceful Shutdown Drain (`await client.shutdown(timeoutMs)`)**: Waits for in-flight requests and drains pending spooled records before closing the socket.
|
||||
|
||||
---
|
||||
|
||||
## Installation
|
||||
|
||||
Install from the self-hosted repository path:
|
||||
|
||||
```bash
|
||||
npm install /path/to/datastack/sdk/node/datastack
|
||||
```
|
||||
|
||||
Or import directly inside a monorepo / TypeScript workspace:
|
||||
|
||||
```ts
|
||||
import { DataStack } from './sdk/node/datastack';
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Configuration (`datastack.config.json` or Constructor Options)
|
||||
|
||||
Create `datastack.config.json` in your Node.js project root (or pass options directly to `new DataStack({...})`):
|
||||
|
||||
```json
|
||||
{
|
||||
"serverUrl": "http://localhost:4000",
|
||||
"apiUrl": "http://localhost:3000",
|
||||
"clientToken": "YOUR_SIGNED_CLIENT_JWT_TOKEN",
|
||||
"clientId": "my_service_prod",
|
||||
"autoConnect": true,
|
||||
"enableHttpFallback": true,
|
||||
"batchChunkSize": 250,
|
||||
"offlineSpoolEnabled": true,
|
||||
"offlineSpoolDir": ".datastack_spool"
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Quickstart
|
||||
|
||||
```ts
|
||||
import { DataStack } from '@datastack/sdk';
|
||||
|
||||
const client = new DataStack({
|
||||
serverUrl: process.env.DATASTACK_SERVER_URL || 'http://localhost:4000',
|
||||
apiUrl: process.env.DATASTACK_API_URL || 'http://localhost:3000',
|
||||
token: process.env.DATASTACK_CLIENT_TOKEN,
|
||||
clientId: 'checkout_api',
|
||||
});
|
||||
|
||||
// 1. Single Event
|
||||
const res = await client.send({
|
||||
event_type: 'order_completed',
|
||||
order_id: 'ord_9481',
|
||||
amount_usd: 129.99,
|
||||
});
|
||||
|
||||
// 2. Batch Events
|
||||
const batchRes = await client.sendBatch([
|
||||
{ metric: 'cpu_load', value: 0.42, host: 'worker-01' },
|
||||
{ metric: 'mem_used_mb', value: 1840, host: 'worker-01' },
|
||||
]);
|
||||
|
||||
// 3. Direct File Ingestion (.csv, .json, .jsonl, .ndjson)
|
||||
const fileRes = await client.sendFile('./backfill_events.csv', { chunkSize: 250 });
|
||||
|
||||
// 4. Graceful Shutdown Drain
|
||||
await client.shutdown(5000);
|
||||
```
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"name": "@datastack/sdk",
|
||||
"version": "2.2.0",
|
||||
"description": "Official DataStack Node.js & TypeScript Client SDK — Real-time Socket.IO telemetry streaming with automatic HTTP fallback, deterministic file ingestion, and offline disk spooling.",
|
||||
"main": "./index.ts",
|
||||
"types": "./index.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./index.ts",
|
||||
"default": "./index.ts"
|
||||
}
|
||||
},
|
||||
"files": [
|
||||
"index.ts",
|
||||
"types.ts",
|
||||
"spool.ts",
|
||||
"README.md"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"csv-parse": "^7.0.3",
|
||||
"socket.io-client": "^4.8.4"
|
||||
},
|
||||
"keywords": [
|
||||
"datastack",
|
||||
"telemetry",
|
||||
"socketio",
|
||||
"csv-partitioning",
|
||||
"nodejs-sdk",
|
||||
"typescript-sdk"
|
||||
],
|
||||
"license": "MIT"
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import type { DataStackEnvelope } from './types';
|
||||
|
||||
export interface DiskSpoolerOptions {
|
||||
spoolDir?: string;
|
||||
maxSizeMb?: number;
|
||||
enabled?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Durable newline-delimited JSON (`.datastack_spool/spool.jsonl`) local disk spooler
|
||||
* for offline telemetry persistence when both Port 4000 WebSocket and Port 3000 HTTP
|
||||
* endpoints are temporarily unreachable.
|
||||
*/
|
||||
export class DiskSpooler {
|
||||
public readonly spoolDir: string;
|
||||
public readonly spoolFile: string;
|
||||
public readonly maxBytes: number;
|
||||
public readonly enabled: boolean;
|
||||
|
||||
constructor(options: DiskSpoolerOptions = {}) {
|
||||
this.spoolDir = path.resolve(options.spoolDir || '.datastack_spool');
|
||||
this.spoolFile = path.join(this.spoolDir, 'spool.jsonl');
|
||||
const maxSizeMb = Number(options.maxSizeMb ?? 50);
|
||||
this.maxBytes = Math.max(1, Number.isFinite(maxSizeMb) ? maxSizeMb : 50) * 1024 * 1024;
|
||||
this.enabled = options.enabled !== false;
|
||||
}
|
||||
|
||||
private ensureDir(): void {
|
||||
if (!this.enabled) return;
|
||||
if (!fs.existsSync(this.spoolDir)) {
|
||||
fs.mkdirSync(this.spoolDir, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Appends one or more event envelopes to `spool.jsonl`.
|
||||
* Returns the number of events persisted to disk.
|
||||
*/
|
||||
public enqueue(items: DataStackEnvelope | DataStackEnvelope[]): number {
|
||||
if (!this.enabled) return 0;
|
||||
const list = Array.isArray(items) ? items : [items];
|
||||
if (list.length === 0) return 0;
|
||||
|
||||
this.ensureDir();
|
||||
const currentSize = this.sizeBytes();
|
||||
const lines = list.map((item) => JSON.stringify(item)).join('\n') + '\n';
|
||||
const byteLength = Buffer.byteLength(lines, 'utf8');
|
||||
|
||||
if (currentSize + byteLength > this.maxBytes) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
const fd = fs.openSync(this.spoolFile, 'a');
|
||||
try {
|
||||
fs.writeFileSync(fd, lines, 'utf8');
|
||||
fs.fsyncSync(fd);
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
return list.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads up to `limit` spooled event envelopes from the head of `spool.jsonl`
|
||||
* without removing them until `removeFirst(count)` is called after server ACK.
|
||||
*/
|
||||
public peekBatch(limit = 250): DataStackEnvelope[] {
|
||||
if (!this.enabled || !fs.existsSync(this.spoolFile)) {
|
||||
return [];
|
||||
}
|
||||
const raw = fs.readFileSync(this.spoolFile, 'utf8');
|
||||
const lines = raw
|
||||
.split('\n')
|
||||
.map((l) => l.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
const results: DataStackEnvelope[] = [];
|
||||
for (const line of lines.slice(0, Math.max(1, limit))) {
|
||||
try {
|
||||
const parsed = JSON.parse(line) as DataStackEnvelope;
|
||||
if (parsed && typeof parsed === 'object' && parsed.eventId && parsed.data) {
|
||||
results.push(parsed);
|
||||
}
|
||||
} catch {
|
||||
// Skip malformed line
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Atomically removes the first `removeCount` lines from `spool.jsonl`
|
||||
* via a temporary file + `fsync` + `renameSync`.
|
||||
*/
|
||||
public removeFirst(removeCount: number): void {
|
||||
if (!this.enabled || removeCount <= 0 || !fs.existsSync(this.spoolFile)) {
|
||||
return;
|
||||
}
|
||||
const raw = fs.readFileSync(this.spoolFile, 'utf8');
|
||||
const lines = raw
|
||||
.split('\n')
|
||||
.map((l) => l.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
const remaining = lines.slice(removeCount);
|
||||
if (remaining.length === 0) {
|
||||
try {
|
||||
fs.unlinkSync(this.spoolFile);
|
||||
} catch {
|
||||
// Ignore if already removed
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
this.ensureDir();
|
||||
const tmpFile = `${this.spoolFile}.${process.pid}.${Date.now()}.tmp`;
|
||||
const payload = remaining.join('\n') + '\n';
|
||||
try {
|
||||
const fd = fs.openSync(tmpFile, 'w');
|
||||
try {
|
||||
fs.writeFileSync(fd, payload, 'utf8');
|
||||
fs.fsyncSync(fd);
|
||||
} finally {
|
||||
fs.closeSync(fd);
|
||||
}
|
||||
fs.renameSync(tmpFile, this.spoolFile);
|
||||
} catch (err) {
|
||||
try {
|
||||
if (fs.existsSync(tmpFile)) {
|
||||
fs.unlinkSync(tmpFile);
|
||||
}
|
||||
} catch {
|
||||
// Ignore cleanup failure
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
public count(): number {
|
||||
if (!this.enabled || !fs.existsSync(this.spoolFile)) {
|
||||
return 0;
|
||||
}
|
||||
try {
|
||||
const raw = fs.readFileSync(this.spoolFile, 'utf8');
|
||||
return raw
|
||||
.split('\n')
|
||||
.map((l) => l.trim())
|
||||
.filter(Boolean).length;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
public sizeBytes(): number {
|
||||
if (!this.enabled || !fs.existsSync(this.spoolFile)) {
|
||||
return 0;
|
||||
}
|
||||
try {
|
||||
return fs.statSync(this.spoolFile).size;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
public clear(): void {
|
||||
if (fs.existsSync(this.spoolFile)) {
|
||||
try {
|
||||
fs.unlinkSync(this.spoolFile);
|
||||
} catch {
|
||||
// Ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
export type DataStackTransportMode = 'websocket' | 'http' | 'spool';
|
||||
|
||||
export type DataStackConnectionState =
|
||||
| 'IDLE_WS'
|
||||
| 'HTTP_FALLBACK'
|
||||
| 'OFFLINE_SPOOL'
|
||||
| 'DISCONNECTED';
|
||||
|
||||
export type DataStackEventPayload = Record<string, unknown>;
|
||||
|
||||
export interface DataStackConfigOptions {
|
||||
/** Port 4000 Socket.IO ingestion server URL (e.g. http://localhost:4000) */
|
||||
serverUrl?: string;
|
||||
/** Port 3000 Next.js HTTP API URL (e.g. http://localhost:3000) */
|
||||
apiUrl?: string;
|
||||
/** Signed client JWT token issued from /vcon/clients */
|
||||
token?: string;
|
||||
/** Alias for token */
|
||||
clientToken?: string;
|
||||
/** Expected normalized clientId (validated against token subject if both present) */
|
||||
clientId?: string;
|
||||
/** Optional path to datastack.config.json */
|
||||
configPath?: string;
|
||||
/** Automatically connect WebSocket on instantiation (default: false in Node tests, true when configured) */
|
||||
autoConnect?: boolean;
|
||||
/** Fall back to HTTP POST /api/events when WebSocket is disconnected (default: true) */
|
||||
enableHttpFallback?: boolean;
|
||||
/** Background heartbeat recovery interval in milliseconds (default: 5000) */
|
||||
heartbeatIntervalMs?: number;
|
||||
/** Maximum retry attempts for transient or 429 rate-limited chunks (default: 3) */
|
||||
maxRetries?: number;
|
||||
/** Maximum records per batch chunk (default: 250, max: 1000) */
|
||||
batchChunkSize?: number;
|
||||
/** Enable local disk spooling when both WebSocket and HTTP are unreachable (default: true) */
|
||||
offlineSpoolEnabled?: boolean;
|
||||
/** Directory path for offline spool JSONL file (default: ".datastack_spool") */
|
||||
offlineSpoolDir?: string;
|
||||
/** Maximum offline spool file size in megabytes (default: 50) */
|
||||
maxSpoolSizeMb?: number;
|
||||
/** Per-request timeout in milliseconds for WebSocket ACKs and HTTP requests (default: 5000) */
|
||||
requestTimeoutMs?: number;
|
||||
/** Custom fetch implementation for testing or custom agents */
|
||||
fetchImpl?: typeof fetch;
|
||||
}
|
||||
|
||||
export interface ResolvedDataStackConfig {
|
||||
serverUrl: string;
|
||||
apiUrl: string;
|
||||
clientToken: string;
|
||||
clientId: string;
|
||||
autoConnect: boolean;
|
||||
enableHttpFallback: boolean;
|
||||
heartbeatIntervalMs: number;
|
||||
maxRetries: number;
|
||||
batchChunkSize: number;
|
||||
offlineSpoolEnabled: boolean;
|
||||
offlineSpoolDir: string;
|
||||
maxSpoolSizeMb: number;
|
||||
requestTimeoutMs: number;
|
||||
}
|
||||
|
||||
export interface DataStackEnvelope {
|
||||
clientId: string;
|
||||
eventId: string;
|
||||
timestamp: string;
|
||||
data: DataStackEventPayload;
|
||||
}
|
||||
|
||||
export interface DataStackSendResult {
|
||||
success: boolean;
|
||||
transport: DataStackTransportMode;
|
||||
eventId?: string;
|
||||
accepted?: number;
|
||||
spooled?: number;
|
||||
duplicates?: number;
|
||||
code?: string;
|
||||
error?: string;
|
||||
rateLimitedEventIds?: string[];
|
||||
unprocessedItems?: DataStackEventPayload[];
|
||||
}
|
||||
|
||||
export interface DataStackFileSendResult extends DataStackSendResult {
|
||||
filePath: string;
|
||||
totalRows: number;
|
||||
acceptedRows: number;
|
||||
spooledRows: number;
|
||||
chunksSent: number;
|
||||
}
|
||||
|
||||
export interface DataStackStatusSnapshot {
|
||||
connected: boolean;
|
||||
transportState: DataStackConnectionState;
|
||||
clientId: string;
|
||||
serverUrl: string;
|
||||
apiUrl: string;
|
||||
spooledCount: number;
|
||||
spooledBytes: number;
|
||||
inFlightRequests: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Base error class for all DataStack Node.js / TypeScript SDK errors (V2-E12).
|
||||
*/
|
||||
export class DataStackError extends Error {
|
||||
public readonly code: string;
|
||||
public readonly statusCode?: number;
|
||||
public readonly details?: Record<string, unknown>;
|
||||
|
||||
constructor(
|
||||
message: string,
|
||||
code = 'DATASTACK_ERROR',
|
||||
statusCode?: number,
|
||||
details?: Record<string, unknown>
|
||||
) {
|
||||
super(message);
|
||||
this.name = 'DataStackError';
|
||||
this.code = code;
|
||||
this.statusCode = statusCode;
|
||||
this.details = details;
|
||||
}
|
||||
}
|
||||
|
||||
export class DataStackConfigError extends DataStackError {
|
||||
constructor(message: string, details?: Record<string, unknown>) {
|
||||
super(message, 'INVALID_SDK_CONFIG', 400, details);
|
||||
this.name = 'DataStackConfigError';
|
||||
}
|
||||
}
|
||||
|
||||
export class DataStackAuthError extends DataStackError {
|
||||
constructor(
|
||||
message: string,
|
||||
code:
|
||||
| 'AUTH_FAILED'
|
||||
| 'CLIENT_TOKEN_EXPIRED'
|
||||
| 'CLIENT_IP_FORBIDDEN'
|
||||
| 'CLIENT_REVOKED' = 'AUTH_FAILED',
|
||||
statusCode = 401,
|
||||
details?: Record<string, unknown>
|
||||
) {
|
||||
super(message, code, statusCode, details);
|
||||
this.name = 'DataStackAuthError';
|
||||
}
|
||||
}
|
||||
|
||||
export class DataStackRateLimitError extends DataStackError {
|
||||
public readonly rateLimitedEventIds: string[];
|
||||
|
||||
constructor(
|
||||
message: string,
|
||||
rateLimitedEventIds: string[] = [],
|
||||
details?: Record<string, unknown>
|
||||
) {
|
||||
super(message, 'RATE_LIMIT_EXCEEDED', 429, details);
|
||||
this.name = 'DataStackRateLimitError';
|
||||
this.rateLimitedEventIds = rateLimitedEventIds;
|
||||
}
|
||||
}
|
||||
|
||||
export class DataStackQuotaExceededError extends DataStackError {
|
||||
constructor(
|
||||
message: string,
|
||||
code: 'STORAGE_QUOTA_EXCEEDED' | 'DISK_SPACE_LOW' = 'STORAGE_QUOTA_EXCEEDED',
|
||||
details?: Record<string, unknown>
|
||||
) {
|
||||
super(message, code, 507, details);
|
||||
this.name = 'DataStackQuotaExceededError';
|
||||
}
|
||||
}
|
||||
|
||||
export class DataStackPayloadTooLargeError extends DataStackError {
|
||||
constructor(message: string, details?: Record<string, unknown>) {
|
||||
super(message, 'PAYLOAD_TOO_LARGE', 413, details);
|
||||
this.name = 'DataStackPayloadTooLargeError';
|
||||
}
|
||||
}
|
||||
|
||||
export class DataStackFileError extends DataStackError {
|
||||
constructor(message: string, details?: Record<string, unknown>) {
|
||||
super(message, 'FILE_INGESTION_ERROR', 400, details);
|
||||
this.name = 'DataStackFileError';
|
||||
}
|
||||
}
|
||||
|
||||
export class DataStackShutdownTimeoutError extends DataStackError {
|
||||
constructor(message: string, details?: Record<string, unknown>) {
|
||||
super(message, 'SHUTDOWN_FLUSH_TIMEOUT', 408, details);
|
||||
this.name = 'DataStackShutdownTimeoutError';
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user