chore: publish current DataStack updates

Publish the current application, design, and Phase 1 deployment updates. Excludes ignored personal notes, local MCP config, and generated TypeScript build info.
This commit is contained in:
i vib committed 2026-10-23 19:56:38 -07:00
1 parent d39d9f92f9
commit fcc4f4db2d
88 files changed
+21248 -2213

No files matched your search

+8 -8
View File
@@ -1,5 +1,5 @@
import { SDKManager } from '../sdk/sdkManager';
import { SettingsManager } from '../adminSettings/settingsManager';
import { SettingsManager } from "../adminSettings/settingsManager";
import { SDKManager } from "../sdk/sdkManager";
/**
* Database Module (PostgreSQL + Prisma abstraction)
@@ -11,18 +11,18 @@ export class DatabaseEngine {
public static async getHealth() {
return {
status: 'healthy',
type: 'PostgreSQL + Prisma Schema Managed',
database: 'datastack_db',
role: 'Metadata & SDK Registrations Only',
collectedDataStorage: 'Strictly CSV (/storage/data)',
status: "healthy",
type: "PostgreSQL + Prisma Schema Managed",
database: "datastack_db",
role: "Metadata & SDK Registrations Only",
collectedDataStorage: "Strictly CSV (/storage/data)",
};
}
public static async syncState() {
return {
clientsCount: SDKManager.listClients().length,
settings: SettingsManager.getSettings(),
settings: SettingsManager.getPublicSettings(),
};
}
}
+31 -11
View File
@@ -1,16 +1,26 @@
export interface ValidationResult {
allowed: boolean;
reason?: string;
code?: 'RATE_LIMIT_EXCEEDED' | 'PAYLOAD_TOO_LARGE' | 'DUPLICATE_EVENT' | 'MALFORMED_PAYLOAD';
code?:
| "RATE_LIMIT_EXCEEDED"
| "PAYLOAD_TOO_LARGE"
| "DUPLICATE_EVENT"
| "MALFORMED_PAYLOAD";
}
export class SecurityEngine {
public static maxEventsPerSec = 100;
public static enableIdempotency = true;
// Idempotency cache: key -> timestamp (expires after TTL)
private static idempotencySet = new Map<string, number>();
private static readonly IDEMPOTENCY_TTL_MS = 5 * 60 * 1000; // 5 minutes
// Rate limiter: clientId -> { count, windowStart }
private static rateLimitMap = new Map<string, { count: number; windowStart: number }>();
private static rateLimitMap = new Map<
string,
{ count: number; windowStart: number }
>();
// Default Max payload size (100KB)
public static maxPayloadSizeBytes = 100 * 1024;
@@ -49,7 +59,10 @@ export class SecurityEngine {
* Rate Limit Check:
* Enforces max events per second per client.
*/
public static checkRateLimit(clientId: string, maxEventsPerSec = 100): boolean {
public static checkRateLimit(
clientId: string,
maxEventsPerSec = this.maxEventsPerSec,
): boolean {
const now = Date.now();
const current = this.rateLimitMap.get(clientId);
@@ -74,33 +87,40 @@ export class SecurityEngine {
clientId: string,
eventId: string,
payload: unknown,
clientMaxRate = 100
clientMaxRate = 100,
): ValidationResult {
// 1. Check idempotency
const isUnique = this.checkIdempotency(clientId, eventId);
const isUnique =
!this.enableIdempotency || this.checkIdempotency(clientId, eventId);
if (!isUnique) {
return { allowed: false, reason: 'Duplicate event dropped silently', code: 'DUPLICATE_EVENT' };
return {
allowed: false,
reason: "Duplicate event dropped silently",
code: "DUPLICATE_EVENT",
};
}
// 2. Check rate limit
const withinLimit = this.checkRateLimit(clientId, clientMaxRate);
const effectiveRateLimit = Math.min(clientMaxRate, this.maxEventsPerSec);
const withinLimit = this.checkRateLimit(clientId, effectiveRateLimit);
if (!withinLimit) {
return {
allowed: false,
reason: `Rate limit of ${clientMaxRate} events/sec exceeded for client ${clientId}`,
code: 'RATE_LIMIT_EXCEEDED',
code: "RATE_LIMIT_EXCEEDED",
};
}
// 3. Check payload size
const payloadStr = typeof payload === 'string' ? payload : JSON.stringify(payload);
const byteLength = Buffer.byteLength(payloadStr, 'utf8');
const payloadStr =
typeof payload === "string" ? payload : JSON.stringify(payload);
const byteLength = Buffer.byteLength(payloadStr, "utf8");
if (byteLength > this.maxPayloadSizeBytes) {
return {
allowed: false,
reason: `Payload size (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${this.maxPayloadSizeBytes / 1024} KB`,
code: 'PAYLOAD_TOO_LARGE',
code: "PAYLOAD_TOO_LARGE",
};
}
+27 -5
View File
@@ -1,7 +1,7 @@
import { SecurityEngine } from '../Security/security';
import { DataBufferEngine } from '../data/buffer';
import { AuthService } from '../auth/jwt';
import { CSVStorageManager } from '../Storage/csvStorage';
import { AuthService } from "../auth/jwt";
import { DataBufferEngine } from "../data/buffer";
import { SecurityEngine } from "../Security/security";
import { CSVStorageManager } from "../Storage/csvStorage";
export interface AdminSettings {
serverPort: number;
@@ -26,7 +26,7 @@ export class SettingsManager {
batchFlushThreshold: 500,
batchFlushIntervalMs: 2000,
fileRotationMb: 100,
jwtSecret: 'datastack-universal-secret-key-2026',
jwtSecret: process.env.DATASTACK_JWT_SECRET || "",
enableIdempotency: true,
autoPruneDays: 30,
updatedAt: new Date().toISOString(),
@@ -36,6 +36,22 @@ export class SettingsManager {
return { ...this.settings };
}
public static getPublicSettings(): Omit<AdminSettings, "jwtSecret"> {
const settings = this.settings;
return {
serverPort: settings.serverPort,
webPort: settings.webPort,
maxEventsPerSec: settings.maxEventsPerSec,
maxPayloadSizeKb: settings.maxPayloadSizeKb,
batchFlushThreshold: settings.batchFlushThreshold,
batchFlushIntervalMs: settings.batchFlushIntervalMs,
fileRotationMb: settings.fileRotationMb,
enableIdempotency: settings.enableIdempotency,
autoPruneDays: settings.autoPruneDays,
updatedAt: settings.updatedAt,
};
}
public static updateSettings(updates: Partial<AdminSettings>): AdminSettings {
this.settings = {
...this.settings,
@@ -44,6 +60,12 @@ export class SettingsManager {
};
// Apply live updates to subsystems
if (updates.maxEventsPerSec !== undefined) {
SecurityEngine.maxEventsPerSec = updates.maxEventsPerSec;
}
if (updates.enableIdempotency !== undefined) {
SecurityEngine.enableIdempotency = updates.enableIdempotency;
}
if (updates.maxPayloadSizeKb !== undefined) {
SecurityEngine.maxPayloadSizeBytes = updates.maxPayloadSizeKb * 1024;
}
+73
View File
@@ -0,0 +1,73 @@
const encoder = new TextEncoder();
interface AdminSessionPayload {
email: string;
expiresAt: number;
}
function encodeBase64Url(bytes: Uint8Array) {
let binary = "";
for (let index = 0; index < bytes.length; index += 1) {
binary += String.fromCharCode(bytes[index]);
}
return btoa(binary).replace(/=/g, "").replace(/\+/g, "-").replace(/\//g, "_");
}
function decodeBase64Url(value: string) {
const base64 = value.replace(/-/g, "+").replace(/_/g, "/");
const padded = base64.padEnd(Math.ceil(base64.length / 4) * 4, "=");
const binary = atob(padded);
return Uint8Array.from(binary, (character) => character.charCodeAt(0));
}
async function importKey(secret: string) {
return crypto.subtle.importKey(
"raw",
encoder.encode(secret),
{ name: "HMAC", hash: "SHA-256" },
false,
["sign", "verify"],
);
}
export async function createAdminSessionToken(
email: string,
secret: string,
expiresAt: number,
) {
const payload = encodeBase64Url(
encoder.encode(JSON.stringify({ email, expiresAt })),
);
const signature = await crypto.subtle.sign(
"HMAC",
await importKey(secret),
encoder.encode(payload),
);
return `${payload}.${encodeBase64Url(new Uint8Array(signature))}`;
}
export async function verifyAdminSessionToken(token: string, secret: string) {
const [payload, signature, extra] = token.split(".");
if (!payload || !signature || extra) return false;
try {
const valid = await crypto.subtle.verify(
"HMAC",
await importKey(secret),
decodeBase64Url(signature),
encoder.encode(payload),
);
if (!valid) return false;
const session = JSON.parse(
new TextDecoder().decode(decodeBase64Url(payload)),
) as AdminSessionPayload;
return (
typeof session.email === "string" &&
typeof session.expiresAt === "number" &&
session.expiresAt > Date.now()
);
} catch {
return false;
}
}
+26 -7
View File
@@ -1,4 +1,4 @@
import jwt from 'jsonwebtoken';
import jwt from "jsonwebtoken";
export interface ClientTokenPayload {
clientId: string;
@@ -8,7 +8,7 @@ export interface ClientTokenPayload {
exp?: number;
}
const DEFAULT_SECRET = process.env.DATASTACK_JWT_SECRET || 'datastack-universal-secret-key-2026';
const DEFAULT_SECRET = process.env.DATASTACK_JWT_SECRET || "";
export class AuthService {
private static secret: string = DEFAULT_SECRET;
@@ -26,7 +26,16 @@ export class AuthService {
/**
* Generates a signed JWT for client SDK authentication.
*/
public static generateClientToken(payload: { clientId: string; name?: string; rateLimit?: number }, expiresIn = '30d'): string {
public static generateClientToken(
payload: { clientId: string; name?: string; rateLimit?: number },
expiresIn = "30d",
): string {
if (!this.secret) {
throw new Error(
"DATASTACK_JWT_SECRET must be configured before generating client tokens.",
);
}
return jwt.sign(
{
clientId: payload.clientId,
@@ -34,20 +43,30 @@ export class AuthService {
rateLimit: payload.rateLimit || 100,
},
this.secret,
{ expiresIn: expiresIn as jwt.SignOptions['expiresIn'] }
{ expiresIn: expiresIn as jwt.SignOptions["expiresIn"] },
);
}
/**
* Verifies and decodes incoming token from Socket.IO handshake.
*/
public static verifyClientToken(token: string): { valid: boolean; payload?: ClientTokenPayload; error?: string } {
public static verifyClientToken(token: string): {
valid: boolean;
payload?: ClientTokenPayload;
error?: string;
} {
if (!this.secret) {
return { valid: false, error: "DATASTACK_JWT_SECRET is not configured." };
}
try {
const cleanToken = token.startsWith('Bearer ') ? token.slice(7).trim() : token.trim();
const cleanToken = token.startsWith("Bearer ")
? token.slice(7).trim()
: token.trim();
const decoded = jwt.verify(cleanToken, this.secret) as ClientTokenPayload;
return { valid: true, payload: decoded };
} catch (err: unknown) {
const message = err instanceof Error ? err.message : 'Invalid token';
const message = err instanceof Error ? err.message : "Invalid token";
return { valid: false, error: message };
}
}
+31 -19
View File
@@ -1,4 +1,4 @@
import { AuthService } from '../auth/jwt';
import { AuthService } from "../auth/jwt";
export interface ClientSDK {
clientId: string;
@@ -6,7 +6,7 @@ export interface ClientSDK {
secret: string;
token: string;
rateLimit: number; // max events per sec
status: 'active' | 'revoked';
status: "active" | "revoked";
totalEvents: number;
lastActive?: string;
createdAt: string;
@@ -19,29 +19,31 @@ export class SDKManager {
static {
const seedClients = [
{
clientId: 'py_telemetry_agent_01',
name: 'Python Telemetry Agent (US-East)',
clientId: "py_telemetry_agent_01",
name: "Python Telemetry Agent (US-East)",
rateLimit: 150,
},
{
clientId: 'py_scraper_prod_02',
name: 'Python Web Scraper Engine',
clientId: "py_scraper_prod_02",
name: "Python Web Scraper Engine",
rateLimit: 200,
},
{
clientId: 'iot_fleet_east_03',
name: 'IoT Sensor Gateway Node',
clientId: "iot_fleet_east_03",
name: "IoT Sensor Gateway Node",
rateLimit: 50,
},
];
for (const c of seedClients) {
const secret = `sec_${c.clientId}_${Math.random().toString(36).substring(2, 10)}`;
const token = AuthService.generateClientToken({
clientId: c.clientId,
name: c.name,
rateLimit: c.rateLimit,
});
const token = AuthService.getSecret()
? AuthService.generateClientToken({
clientId: c.clientId,
name: c.name,
rateLimit: c.rateLimit,
})
: "";
SDKManager.clients.set(c.clientId, {
clientId: c.clientId,
@@ -49,10 +51,10 @@ export class SDKManager {
secret,
token,
rateLimit: c.rateLimit,
status: 'active',
status: "active",
totalEvents: 4280,
lastActive: new Date().toISOString(),
createdAt: '2026-10-20T10:00:00.000Z',
createdAt: "2026-10-20T10:00:00.000Z",
});
}
}
@@ -65,8 +67,15 @@ export class SDKManager {
return this.clients.get(clientId);
}
public static registerClient(data: { clientId: string; name: string; rateLimit?: number }): ClientSDK {
const safeClientId = data.clientId.trim().toLowerCase().replace(/[^a-z0-9_-]/g, '_');
public static registerClient(data: {
clientId: string;
name: string;
rateLimit?: number;
}): ClientSDK {
const safeClientId = data.clientId
.trim()
.toLowerCase()
.replace(/[^a-z0-9_-]/g, "_");
const secret = `sec_${safeClientId}_${Math.random().toString(36).substring(2, 10)}`;
const rateLimit = data.rateLimit || 100;
@@ -82,7 +91,7 @@ export class SDKManager {
secret,
token,
rateLimit,
status: 'active',
status: "active",
totalEvents: 0,
createdAt: new Date().toISOString(),
};
@@ -106,7 +115,10 @@ export class SDKManager {
return client;
}
public static updateClient(clientId: string, updates: Partial<Pick<ClientSDK, 'name' | 'rateLimit' | 'status'>>): ClientSDK | null {
public static updateClient(
clientId: string,
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
): ClientSDK | null {
const client = this.clients.get(clientId);
if (!client) return null;