feat(auth): implement RBAC audit logging and session security

Integrate granular audit logging for admin actions, transition to role-based API access via `requireAdminApiSession`, and update session tokens to persist user roles and IDs for consistent multi-admin management.
This commit is contained in:
Team Dark Devil committed 2026-11-20 18:48:54 +06:00
1 parent 2873b9904a
commit 5c977d6977
37 files changed
+5939 -349

No files matched your search

+84
View File
@@ -0,0 +1,84 @@
# DataStack Node.js & TypeScript SDK (`@datastack/sdk` v2.2.0)
Official server-side Node.js and TypeScript SDK for **DataStack**. Provides 1:1 transport, fallback, deduplication, and offline spooling parity with the official Python SDK (`sdk/python/datastack`).
---
## Features
- **Primary Real-Time WebSocket Transport (`socket.io-client`)**: Streams single events (`sync_data`) and batches (`batch_sync`) to Port `4000` with explicit server persistence acknowledgments.
- **Automatic HTTP Fallback (`POST /api/events`)**: Seamlessly falls back to Port `3000` REST ingestion whenever the WebSocket transport is disconnected or unreachable.
- **Offline Disk Spooling (`.datastack_spool/spool.jsonl`)**: Persists undelivered events to local disk when both WebSocket and HTTP endpoints are offline, and automatically drains the spool via `flushSpool()` when connectivity recovers.
- **Direct File Ingestion (`sendFile()`)**: Streams `.csv`, `.json`, `.jsonl`, and `.ndjson` files with deterministic SHA-256 `eventId` generation (`file_<stem>_<row>_<hash>`) so retries never duplicate rows.
- **Typed Error Hierarchy (`V2-E12`)**: Distinguishes terminal errors (`DataStackAuthError` for `401 CLIENT_TOKEN_EXPIRED` / `403 CLIENT_IP_FORBIDDEN`, `DataStackQuotaExceededError` for `507 STORAGE_QUOTA_EXCEEDED` / `DISK_SPACE_LOW`, `DataStackConfigError`) from retryable `429 RATE_LIMIT_EXCEEDED` (`DataStackRateLimitError`) and auto-splittable `413 PAYLOAD_TOO_LARGE` (`DataStackPayloadTooLargeError`).
- **Graceful Shutdown Drain (`await client.shutdown(timeoutMs)`)**: Waits for in-flight requests and drains pending spooled records before closing the socket.
---
## Installation
Install from the self-hosted repository path:
```bash
npm install /path/to/datastack/sdk/node/datastack
```
Or import directly inside a monorepo / TypeScript workspace:
```ts
import { DataStack } from './sdk/node/datastack';
```
---
## Configuration (`datastack.config.json` or Constructor Options)
Create `datastack.config.json` in your Node.js project root (or pass options directly to `new DataStack({...})`):
```json
{
"serverUrl": "http://localhost:4000",
"apiUrl": "http://localhost:3000",
"clientToken": "YOUR_SIGNED_CLIENT_JWT_TOKEN",
"clientId": "my_service_prod",
"autoConnect": true,
"enableHttpFallback": true,
"batchChunkSize": 250,
"offlineSpoolEnabled": true,
"offlineSpoolDir": ".datastack_spool"
}
```
---
## Quickstart
```ts
import { DataStack } from '@datastack/sdk';
const client = new DataStack({
serverUrl: process.env.DATASTACK_SERVER_URL || 'http://localhost:4000',
apiUrl: process.env.DATASTACK_API_URL || 'http://localhost:3000',
token: process.env.DATASTACK_CLIENT_TOKEN,
clientId: 'checkout_api',
});
// 1. Single Event
const res = await client.send({
event_type: 'order_completed',
order_id: 'ord_9481',
amount_usd: 129.99,
});
// 2. Batch Events
const batchRes = await client.sendBatch([
{ metric: 'cpu_load', value: 0.42, host: 'worker-01' },
{ metric: 'mem_used_mb', value: 1840, host: 'worker-01' },
]);
// 3. Direct File Ingestion (.csv, .json, .jsonl, .ndjson)
const fileRes = await client.sendFile('./backfill_events.csv', { chunkSize: 250 });
// 4. Graceful Shutdown Drain
await client.shutdown(5000);
```
File diff suppressed because it is too large. Load diff
+35
View File
@@ -0,0 +1,35 @@
{
"name": "@datastack/sdk",
"version": "2.2.0",
"description": "Official DataStack Node.js & TypeScript Client SDK — Real-time Socket.IO telemetry streaming with automatic HTTP fallback, deterministic file ingestion, and offline disk spooling.",
"main": "./index.ts",
"types": "./index.ts",
"exports": {
".": {
"types": "./index.ts",
"default": "./index.ts"
}
},
"files": [
"index.ts",
"types.ts",
"spool.ts",
"README.md"
],
"engines": {
"node": ">=20.0.0"
},
"dependencies": {
"csv-parse": "^7.0.3",
"socket.io-client": "^4.8.4"
},
"keywords": [
"datastack",
"telemetry",
"socketio",
"csv-partitioning",
"nodejs-sdk",
"typescript-sdk"
],
"license": "MIT"
}
+176
View File
@@ -0,0 +1,176 @@
import fs from 'node:fs';
import path from 'node:path';
import type { DataStackEnvelope } from './types';
export interface DiskSpoolerOptions {
spoolDir?: string;
maxSizeMb?: number;
enabled?: boolean;
}
/**
* Durable newline-delimited JSON (`.datastack_spool/spool.jsonl`) local disk spooler
* for offline telemetry persistence when both Port 4000 WebSocket and Port 3000 HTTP
* endpoints are temporarily unreachable.
*/
export class DiskSpooler {
public readonly spoolDir: string;
public readonly spoolFile: string;
public readonly maxBytes: number;
public readonly enabled: boolean;
constructor(options: DiskSpoolerOptions = {}) {
this.spoolDir = path.resolve(options.spoolDir || '.datastack_spool');
this.spoolFile = path.join(this.spoolDir, 'spool.jsonl');
const maxSizeMb = Number(options.maxSizeMb ?? 50);
this.maxBytes = Math.max(1, Number.isFinite(maxSizeMb) ? maxSizeMb : 50) * 1024 * 1024;
this.enabled = options.enabled !== false;
}
private ensureDir(): void {
if (!this.enabled) return;
if (!fs.existsSync(this.spoolDir)) {
fs.mkdirSync(this.spoolDir, { recursive: true });
}
}
/**
* Appends one or more event envelopes to `spool.jsonl`.
* Returns the number of events persisted to disk.
*/
public enqueue(items: DataStackEnvelope | DataStackEnvelope[]): number {
if (!this.enabled) return 0;
const list = Array.isArray(items) ? items : [items];
if (list.length === 0) return 0;
this.ensureDir();
const currentSize = this.sizeBytes();
const lines = list.map((item) => JSON.stringify(item)).join('\n') + '\n';
const byteLength = Buffer.byteLength(lines, 'utf8');
if (currentSize + byteLength > this.maxBytes) {
return 0;
}
const fd = fs.openSync(this.spoolFile, 'a');
try {
fs.writeFileSync(fd, lines, 'utf8');
fs.fsyncSync(fd);
} finally {
fs.closeSync(fd);
}
return list.length;
}
/**
* Reads up to `limit` spooled event envelopes from the head of `spool.jsonl`
* without removing them until `removeFirst(count)` is called after server ACK.
*/
public peekBatch(limit = 250): DataStackEnvelope[] {
if (!this.enabled || !fs.existsSync(this.spoolFile)) {
return [];
}
const raw = fs.readFileSync(this.spoolFile, 'utf8');
const lines = raw
.split('\n')
.map((l) => l.trim())
.filter(Boolean);
const results: DataStackEnvelope[] = [];
for (const line of lines.slice(0, Math.max(1, limit))) {
try {
const parsed = JSON.parse(line) as DataStackEnvelope;
if (parsed && typeof parsed === 'object' && parsed.eventId && parsed.data) {
results.push(parsed);
}
} catch {
// Skip malformed line
}
}
return results;
}
/**
* Atomically removes the first `removeCount` lines from `spool.jsonl`
* via a temporary file + `fsync` + `renameSync`.
*/
public removeFirst(removeCount: number): void {
if (!this.enabled || removeCount <= 0 || !fs.existsSync(this.spoolFile)) {
return;
}
const raw = fs.readFileSync(this.spoolFile, 'utf8');
const lines = raw
.split('\n')
.map((l) => l.trim())
.filter(Boolean);
const remaining = lines.slice(removeCount);
if (remaining.length === 0) {
try {
fs.unlinkSync(this.spoolFile);
} catch {
// Ignore if already removed
}
return;
}
this.ensureDir();
const tmpFile = `${this.spoolFile}.${process.pid}.${Date.now()}.tmp`;
const payload = remaining.join('\n') + '\n';
try {
const fd = fs.openSync(tmpFile, 'w');
try {
fs.writeFileSync(fd, payload, 'utf8');
fs.fsyncSync(fd);
} finally {
fs.closeSync(fd);
}
fs.renameSync(tmpFile, this.spoolFile);
} catch (err) {
try {
if (fs.existsSync(tmpFile)) {
fs.unlinkSync(tmpFile);
}
} catch {
// Ignore cleanup failure
}
throw err;
}
}
public count(): number {
if (!this.enabled || !fs.existsSync(this.spoolFile)) {
return 0;
}
try {
const raw = fs.readFileSync(this.spoolFile, 'utf8');
return raw
.split('\n')
.map((l) => l.trim())
.filter(Boolean).length;
} catch {
return 0;
}
}
public sizeBytes(): number {
if (!this.enabled || !fs.existsSync(this.spoolFile)) {
return 0;
}
try {
return fs.statSync(this.spoolFile).size;
} catch {
return 0;
}
}
public clear(): void {
if (fs.existsSync(this.spoolFile)) {
try {
fs.unlinkSync(this.spoolFile);
} catch {
// Ignore
}
}
}
}
+190
View File
@@ -0,0 +1,190 @@
export type DataStackTransportMode = 'websocket' | 'http' | 'spool';
export type DataStackConnectionState =
| 'IDLE_WS'
| 'HTTP_FALLBACK'
| 'OFFLINE_SPOOL'
| 'DISCONNECTED';
export type DataStackEventPayload = Record<string, unknown>;
export interface DataStackConfigOptions {
/** Port 4000 Socket.IO ingestion server URL (e.g. http://localhost:4000) */
serverUrl?: string;
/** Port 3000 Next.js HTTP API URL (e.g. http://localhost:3000) */
apiUrl?: string;
/** Signed client JWT token issued from /vcon/clients */
token?: string;
/** Alias for token */
clientToken?: string;
/** Expected normalized clientId (validated against token subject if both present) */
clientId?: string;
/** Optional path to datastack.config.json */
configPath?: string;
/** Automatically connect WebSocket on instantiation (default: false in Node tests, true when configured) */
autoConnect?: boolean;
/** Fall back to HTTP POST /api/events when WebSocket is disconnected (default: true) */
enableHttpFallback?: boolean;
/** Background heartbeat recovery interval in milliseconds (default: 5000) */
heartbeatIntervalMs?: number;
/** Maximum retry attempts for transient or 429 rate-limited chunks (default: 3) */
maxRetries?: number;
/** Maximum records per batch chunk (default: 250, max: 1000) */
batchChunkSize?: number;
/** Enable local disk spooling when both WebSocket and HTTP are unreachable (default: true) */
offlineSpoolEnabled?: boolean;
/** Directory path for offline spool JSONL file (default: ".datastack_spool") */
offlineSpoolDir?: string;
/** Maximum offline spool file size in megabytes (default: 50) */
maxSpoolSizeMb?: number;
/** Per-request timeout in milliseconds for WebSocket ACKs and HTTP requests (default: 5000) */
requestTimeoutMs?: number;
/** Custom fetch implementation for testing or custom agents */
fetchImpl?: typeof fetch;
}
export interface ResolvedDataStackConfig {
serverUrl: string;
apiUrl: string;
clientToken: string;
clientId: string;
autoConnect: boolean;
enableHttpFallback: boolean;
heartbeatIntervalMs: number;
maxRetries: number;
batchChunkSize: number;
offlineSpoolEnabled: boolean;
offlineSpoolDir: string;
maxSpoolSizeMb: number;
requestTimeoutMs: number;
}
export interface DataStackEnvelope {
clientId: string;
eventId: string;
timestamp: string;
data: DataStackEventPayload;
}
export interface DataStackSendResult {
success: boolean;
transport: DataStackTransportMode;
eventId?: string;
accepted?: number;
spooled?: number;
duplicates?: number;
code?: string;
error?: string;
rateLimitedEventIds?: string[];
unprocessedItems?: DataStackEventPayload[];
}
export interface DataStackFileSendResult extends DataStackSendResult {
filePath: string;
totalRows: number;
acceptedRows: number;
spooledRows: number;
chunksSent: number;
}
export interface DataStackStatusSnapshot {
connected: boolean;
transportState: DataStackConnectionState;
clientId: string;
serverUrl: string;
apiUrl: string;
spooledCount: number;
spooledBytes: number;
inFlightRequests: number;
}
/**
* Base error class for all DataStack Node.js / TypeScript SDK errors (V2-E12).
*/
export class DataStackError extends Error {
public readonly code: string;
public readonly statusCode?: number;
public readonly details?: Record<string, unknown>;
constructor(
message: string,
code = 'DATASTACK_ERROR',
statusCode?: number,
details?: Record<string, unknown>
) {
super(message);
this.name = 'DataStackError';
this.code = code;
this.statusCode = statusCode;
this.details = details;
}
}
export class DataStackConfigError extends DataStackError {
constructor(message: string, details?: Record<string, unknown>) {
super(message, 'INVALID_SDK_CONFIG', 400, details);
this.name = 'DataStackConfigError';
}
}
export class DataStackAuthError extends DataStackError {
constructor(
message: string,
code:
| 'AUTH_FAILED'
| 'CLIENT_TOKEN_EXPIRED'
| 'CLIENT_IP_FORBIDDEN'
| 'CLIENT_REVOKED' = 'AUTH_FAILED',
statusCode = 401,
details?: Record<string, unknown>
) {
super(message, code, statusCode, details);
this.name = 'DataStackAuthError';
}
}
export class DataStackRateLimitError extends DataStackError {
public readonly rateLimitedEventIds: string[];
constructor(
message: string,
rateLimitedEventIds: string[] = [],
details?: Record<string, unknown>
) {
super(message, 'RATE_LIMIT_EXCEEDED', 429, details);
this.name = 'DataStackRateLimitError';
this.rateLimitedEventIds = rateLimitedEventIds;
}
}
export class DataStackQuotaExceededError extends DataStackError {
constructor(
message: string,
code: 'STORAGE_QUOTA_EXCEEDED' | 'DISK_SPACE_LOW' = 'STORAGE_QUOTA_EXCEEDED',
details?: Record<string, unknown>
) {
super(message, code, 507, details);
this.name = 'DataStackQuotaExceededError';
}
}
export class DataStackPayloadTooLargeError extends DataStackError {
constructor(message: string, details?: Record<string, unknown>) {
super(message, 'PAYLOAD_TOO_LARGE', 413, details);
this.name = 'DataStackPayloadTooLargeError';
}
}
export class DataStackFileError extends DataStackError {
constructor(message: string, details?: Record<string, unknown>) {
super(message, 'FILE_INGESTION_ERROR', 400, details);
this.name = 'DataStackFileError';
}
}
export class DataStackShutdownTimeoutError extends DataStackError {
constructor(message: string, details?: Record<string, unknown>) {
super(message, 'SHUTDOWN_FLUSH_TIMEOUT', 408, details);
this.name = 'DataStackShutdownTimeoutError';
}
}