chore: publish current DataStack updates
Publish the current application, design, and Phase 1 deployment updates. Excludes ignored personal notes, local MCP config, and generated TypeScript build info.
This commit is contained in:
1 parent
d39d9f92f9
commit
fcc4f4db2d
88 files changed
+21248
-2213
No files matched your search
@@ -1,5 +1,5 @@
|
||||
import { SDKManager } from '../sdk/sdkManager';
|
||||
import { SettingsManager } from '../adminSettings/settingsManager';
|
||||
import { SettingsManager } from "../adminSettings/settingsManager";
|
||||
import { SDKManager } from "../sdk/sdkManager";
|
||||
|
||||
/**
|
||||
* Database Module (PostgreSQL + Prisma abstraction)
|
||||
@@ -11,18 +11,18 @@ export class DatabaseEngine {
|
||||
|
||||
public static async getHealth() {
|
||||
return {
|
||||
status: 'healthy',
|
||||
type: 'PostgreSQL + Prisma Schema Managed',
|
||||
database: 'datastack_db',
|
||||
role: 'Metadata & SDK Registrations Only',
|
||||
collectedDataStorage: 'Strictly CSV (/storage/data)',
|
||||
status: "healthy",
|
||||
type: "PostgreSQL + Prisma Schema Managed",
|
||||
database: "datastack_db",
|
||||
role: "Metadata & SDK Registrations Only",
|
||||
collectedDataStorage: "Strictly CSV (/storage/data)",
|
||||
};
|
||||
}
|
||||
|
||||
public static async syncState() {
|
||||
return {
|
||||
clientsCount: SDKManager.listClients().length,
|
||||
settings: SettingsManager.getSettings(),
|
||||
settings: SettingsManager.getPublicSettings(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
+31
-11
@@ -1,16 +1,26 @@
|
||||
export interface ValidationResult {
|
||||
allowed: boolean;
|
||||
reason?: string;
|
||||
code?: 'RATE_LIMIT_EXCEEDED' | 'PAYLOAD_TOO_LARGE' | 'DUPLICATE_EVENT' | 'MALFORMED_PAYLOAD';
|
||||
code?:
|
||||
| "RATE_LIMIT_EXCEEDED"
|
||||
| "PAYLOAD_TOO_LARGE"
|
||||
| "DUPLICATE_EVENT"
|
||||
| "MALFORMED_PAYLOAD";
|
||||
}
|
||||
|
||||
export class SecurityEngine {
|
||||
public static maxEventsPerSec = 100;
|
||||
public static enableIdempotency = true;
|
||||
|
||||
// Idempotency cache: key -> timestamp (expires after TTL)
|
||||
private static idempotencySet = new Map<string, number>();
|
||||
private static readonly IDEMPOTENCY_TTL_MS = 5 * 60 * 1000; // 5 minutes
|
||||
|
||||
// Rate limiter: clientId -> { count, windowStart }
|
||||
private static rateLimitMap = new Map<string, { count: number; windowStart: number }>();
|
||||
private static rateLimitMap = new Map<
|
||||
string,
|
||||
{ count: number; windowStart: number }
|
||||
>();
|
||||
|
||||
// Default Max payload size (100KB)
|
||||
public static maxPayloadSizeBytes = 100 * 1024;
|
||||
@@ -49,7 +59,10 @@ export class SecurityEngine {
|
||||
* Rate Limit Check:
|
||||
* Enforces max events per second per client.
|
||||
*/
|
||||
public static checkRateLimit(clientId: string, maxEventsPerSec = 100): boolean {
|
||||
public static checkRateLimit(
|
||||
clientId: string,
|
||||
maxEventsPerSec = this.maxEventsPerSec,
|
||||
): boolean {
|
||||
const now = Date.now();
|
||||
const current = this.rateLimitMap.get(clientId);
|
||||
|
||||
@@ -74,33 +87,40 @@ export class SecurityEngine {
|
||||
clientId: string,
|
||||
eventId: string,
|
||||
payload: unknown,
|
||||
clientMaxRate = 100
|
||||
clientMaxRate = 100,
|
||||
): ValidationResult {
|
||||
// 1. Check idempotency
|
||||
const isUnique = this.checkIdempotency(clientId, eventId);
|
||||
const isUnique =
|
||||
!this.enableIdempotency || this.checkIdempotency(clientId, eventId);
|
||||
if (!isUnique) {
|
||||
return { allowed: false, reason: 'Duplicate event dropped silently', code: 'DUPLICATE_EVENT' };
|
||||
return {
|
||||
allowed: false,
|
||||
reason: "Duplicate event dropped silently",
|
||||
code: "DUPLICATE_EVENT",
|
||||
};
|
||||
}
|
||||
|
||||
// 2. Check rate limit
|
||||
const withinLimit = this.checkRateLimit(clientId, clientMaxRate);
|
||||
const effectiveRateLimit = Math.min(clientMaxRate, this.maxEventsPerSec);
|
||||
const withinLimit = this.checkRateLimit(clientId, effectiveRateLimit);
|
||||
if (!withinLimit) {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: `Rate limit of ${clientMaxRate} events/sec exceeded for client ${clientId}`,
|
||||
code: 'RATE_LIMIT_EXCEEDED',
|
||||
code: "RATE_LIMIT_EXCEEDED",
|
||||
};
|
||||
}
|
||||
|
||||
// 3. Check payload size
|
||||
const payloadStr = typeof payload === 'string' ? payload : JSON.stringify(payload);
|
||||
const byteLength = Buffer.byteLength(payloadStr, 'utf8');
|
||||
const payloadStr =
|
||||
typeof payload === "string" ? payload : JSON.stringify(payload);
|
||||
const byteLength = Buffer.byteLength(payloadStr, "utf8");
|
||||
|
||||
if (byteLength > this.maxPayloadSizeBytes) {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: `Payload size (${(byteLength / 1024).toFixed(1)} KB) exceeds limit of ${this.maxPayloadSizeBytes / 1024} KB`,
|
||||
code: 'PAYLOAD_TOO_LARGE',
|
||||
code: "PAYLOAD_TOO_LARGE",
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { SecurityEngine } from '../Security/security';
|
||||
import { DataBufferEngine } from '../data/buffer';
|
||||
import { AuthService } from '../auth/jwt';
|
||||
import { CSVStorageManager } from '../Storage/csvStorage';
|
||||
import { AuthService } from "../auth/jwt";
|
||||
import { DataBufferEngine } from "../data/buffer";
|
||||
import { SecurityEngine } from "../Security/security";
|
||||
import { CSVStorageManager } from "../Storage/csvStorage";
|
||||
|
||||
export interface AdminSettings {
|
||||
serverPort: number;
|
||||
@@ -26,7 +26,7 @@ export class SettingsManager {
|
||||
batchFlushThreshold: 500,
|
||||
batchFlushIntervalMs: 2000,
|
||||
fileRotationMb: 100,
|
||||
jwtSecret: 'datastack-universal-secret-key-2026',
|
||||
jwtSecret: process.env.DATASTACK_JWT_SECRET || "",
|
||||
enableIdempotency: true,
|
||||
autoPruneDays: 30,
|
||||
updatedAt: new Date().toISOString(),
|
||||
@@ -36,6 +36,22 @@ export class SettingsManager {
|
||||
return { ...this.settings };
|
||||
}
|
||||
|
||||
public static getPublicSettings(): Omit<AdminSettings, "jwtSecret"> {
|
||||
const settings = this.settings;
|
||||
return {
|
||||
serverPort: settings.serverPort,
|
||||
webPort: settings.webPort,
|
||||
maxEventsPerSec: settings.maxEventsPerSec,
|
||||
maxPayloadSizeKb: settings.maxPayloadSizeKb,
|
||||
batchFlushThreshold: settings.batchFlushThreshold,
|
||||
batchFlushIntervalMs: settings.batchFlushIntervalMs,
|
||||
fileRotationMb: settings.fileRotationMb,
|
||||
enableIdempotency: settings.enableIdempotency,
|
||||
autoPruneDays: settings.autoPruneDays,
|
||||
updatedAt: settings.updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
public static updateSettings(updates: Partial<AdminSettings>): AdminSettings {
|
||||
this.settings = {
|
||||
...this.settings,
|
||||
@@ -44,6 +60,12 @@ export class SettingsManager {
|
||||
};
|
||||
|
||||
// Apply live updates to subsystems
|
||||
if (updates.maxEventsPerSec !== undefined) {
|
||||
SecurityEngine.maxEventsPerSec = updates.maxEventsPerSec;
|
||||
}
|
||||
if (updates.enableIdempotency !== undefined) {
|
||||
SecurityEngine.enableIdempotency = updates.enableIdempotency;
|
||||
}
|
||||
if (updates.maxPayloadSizeKb !== undefined) {
|
||||
SecurityEngine.maxPayloadSizeBytes = updates.maxPayloadSizeKb * 1024;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
const encoder = new TextEncoder();
|
||||
|
||||
interface AdminSessionPayload {
|
||||
email: string;
|
||||
expiresAt: number;
|
||||
}
|
||||
|
||||
function encodeBase64Url(bytes: Uint8Array) {
|
||||
let binary = "";
|
||||
for (let index = 0; index < bytes.length; index += 1) {
|
||||
binary += String.fromCharCode(bytes[index]);
|
||||
}
|
||||
return btoa(binary).replace(/=/g, "").replace(/\+/g, "-").replace(/\//g, "_");
|
||||
}
|
||||
|
||||
function decodeBase64Url(value: string) {
|
||||
const base64 = value.replace(/-/g, "+").replace(/_/g, "/");
|
||||
const padded = base64.padEnd(Math.ceil(base64.length / 4) * 4, "=");
|
||||
const binary = atob(padded);
|
||||
return Uint8Array.from(binary, (character) => character.charCodeAt(0));
|
||||
}
|
||||
|
||||
async function importKey(secret: string) {
|
||||
return crypto.subtle.importKey(
|
||||
"raw",
|
||||
encoder.encode(secret),
|
||||
{ name: "HMAC", hash: "SHA-256" },
|
||||
false,
|
||||
["sign", "verify"],
|
||||
);
|
||||
}
|
||||
|
||||
export async function createAdminSessionToken(
|
||||
email: string,
|
||||
secret: string,
|
||||
expiresAt: number,
|
||||
) {
|
||||
const payload = encodeBase64Url(
|
||||
encoder.encode(JSON.stringify({ email, expiresAt })),
|
||||
);
|
||||
const signature = await crypto.subtle.sign(
|
||||
"HMAC",
|
||||
await importKey(secret),
|
||||
encoder.encode(payload),
|
||||
);
|
||||
return `${payload}.${encodeBase64Url(new Uint8Array(signature))}`;
|
||||
}
|
||||
|
||||
export async function verifyAdminSessionToken(token: string, secret: string) {
|
||||
const [payload, signature, extra] = token.split(".");
|
||||
if (!payload || !signature || extra) return false;
|
||||
|
||||
try {
|
||||
const valid = await crypto.subtle.verify(
|
||||
"HMAC",
|
||||
await importKey(secret),
|
||||
decodeBase64Url(signature),
|
||||
encoder.encode(payload),
|
||||
);
|
||||
if (!valid) return false;
|
||||
|
||||
const session = JSON.parse(
|
||||
new TextDecoder().decode(decodeBase64Url(payload)),
|
||||
) as AdminSessionPayload;
|
||||
return (
|
||||
typeof session.email === "string" &&
|
||||
typeof session.expiresAt === "number" &&
|
||||
session.expiresAt > Date.now()
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
+26
-7
@@ -1,4 +1,4 @@
|
||||
import jwt from 'jsonwebtoken';
|
||||
import jwt from "jsonwebtoken";
|
||||
|
||||
export interface ClientTokenPayload {
|
||||
clientId: string;
|
||||
@@ -8,7 +8,7 @@ export interface ClientTokenPayload {
|
||||
exp?: number;
|
||||
}
|
||||
|
||||
const DEFAULT_SECRET = process.env.DATASTACK_JWT_SECRET || 'datastack-universal-secret-key-2026';
|
||||
const DEFAULT_SECRET = process.env.DATASTACK_JWT_SECRET || "";
|
||||
|
||||
export class AuthService {
|
||||
private static secret: string = DEFAULT_SECRET;
|
||||
@@ -26,7 +26,16 @@ export class AuthService {
|
||||
/**
|
||||
* Generates a signed JWT for client SDK authentication.
|
||||
*/
|
||||
public static generateClientToken(payload: { clientId: string; name?: string; rateLimit?: number }, expiresIn = '30d'): string {
|
||||
public static generateClientToken(
|
||||
payload: { clientId: string; name?: string; rateLimit?: number },
|
||||
expiresIn = "30d",
|
||||
): string {
|
||||
if (!this.secret) {
|
||||
throw new Error(
|
||||
"DATASTACK_JWT_SECRET must be configured before generating client tokens.",
|
||||
);
|
||||
}
|
||||
|
||||
return jwt.sign(
|
||||
{
|
||||
clientId: payload.clientId,
|
||||
@@ -34,20 +43,30 @@ export class AuthService {
|
||||
rateLimit: payload.rateLimit || 100,
|
||||
},
|
||||
this.secret,
|
||||
{ expiresIn: expiresIn as jwt.SignOptions['expiresIn'] }
|
||||
{ expiresIn: expiresIn as jwt.SignOptions["expiresIn"] },
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies and decodes incoming token from Socket.IO handshake.
|
||||
*/
|
||||
public static verifyClientToken(token: string): { valid: boolean; payload?: ClientTokenPayload; error?: string } {
|
||||
public static verifyClientToken(token: string): {
|
||||
valid: boolean;
|
||||
payload?: ClientTokenPayload;
|
||||
error?: string;
|
||||
} {
|
||||
if (!this.secret) {
|
||||
return { valid: false, error: "DATASTACK_JWT_SECRET is not configured." };
|
||||
}
|
||||
|
||||
try {
|
||||
const cleanToken = token.startsWith('Bearer ') ? token.slice(7).trim() : token.trim();
|
||||
const cleanToken = token.startsWith("Bearer ")
|
||||
? token.slice(7).trim()
|
||||
: token.trim();
|
||||
const decoded = jwt.verify(cleanToken, this.secret) as ClientTokenPayload;
|
||||
return { valid: true, payload: decoded };
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : 'Invalid token';
|
||||
const message = err instanceof Error ? err.message : "Invalid token";
|
||||
return { valid: false, error: message };
|
||||
}
|
||||
}
|
||||
|
||||
+31
-19
@@ -1,4 +1,4 @@
|
||||
import { AuthService } from '../auth/jwt';
|
||||
import { AuthService } from "../auth/jwt";
|
||||
|
||||
export interface ClientSDK {
|
||||
clientId: string;
|
||||
@@ -6,7 +6,7 @@ export interface ClientSDK {
|
||||
secret: string;
|
||||
token: string;
|
||||
rateLimit: number; // max events per sec
|
||||
status: 'active' | 'revoked';
|
||||
status: "active" | "revoked";
|
||||
totalEvents: number;
|
||||
lastActive?: string;
|
||||
createdAt: string;
|
||||
@@ -19,29 +19,31 @@ export class SDKManager {
|
||||
static {
|
||||
const seedClients = [
|
||||
{
|
||||
clientId: 'py_telemetry_agent_01',
|
||||
name: 'Python Telemetry Agent (US-East)',
|
||||
clientId: "py_telemetry_agent_01",
|
||||
name: "Python Telemetry Agent (US-East)",
|
||||
rateLimit: 150,
|
||||
},
|
||||
{
|
||||
clientId: 'py_scraper_prod_02',
|
||||
name: 'Python Web Scraper Engine',
|
||||
clientId: "py_scraper_prod_02",
|
||||
name: "Python Web Scraper Engine",
|
||||
rateLimit: 200,
|
||||
},
|
||||
{
|
||||
clientId: 'iot_fleet_east_03',
|
||||
name: 'IoT Sensor Gateway Node',
|
||||
clientId: "iot_fleet_east_03",
|
||||
name: "IoT Sensor Gateway Node",
|
||||
rateLimit: 50,
|
||||
},
|
||||
];
|
||||
|
||||
for (const c of seedClients) {
|
||||
const secret = `sec_${c.clientId}_${Math.random().toString(36).substring(2, 10)}`;
|
||||
const token = AuthService.generateClientToken({
|
||||
clientId: c.clientId,
|
||||
name: c.name,
|
||||
rateLimit: c.rateLimit,
|
||||
});
|
||||
const token = AuthService.getSecret()
|
||||
? AuthService.generateClientToken({
|
||||
clientId: c.clientId,
|
||||
name: c.name,
|
||||
rateLimit: c.rateLimit,
|
||||
})
|
||||
: "";
|
||||
|
||||
SDKManager.clients.set(c.clientId, {
|
||||
clientId: c.clientId,
|
||||
@@ -49,10 +51,10 @@ export class SDKManager {
|
||||
secret,
|
||||
token,
|
||||
rateLimit: c.rateLimit,
|
||||
status: 'active',
|
||||
status: "active",
|
||||
totalEvents: 4280,
|
||||
lastActive: new Date().toISOString(),
|
||||
createdAt: '2026-10-20T10:00:00.000Z',
|
||||
createdAt: "2026-10-20T10:00:00.000Z",
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -65,8 +67,15 @@ export class SDKManager {
|
||||
return this.clients.get(clientId);
|
||||
}
|
||||
|
||||
public static registerClient(data: { clientId: string; name: string; rateLimit?: number }): ClientSDK {
|
||||
const safeClientId = data.clientId.trim().toLowerCase().replace(/[^a-z0-9_-]/g, '_');
|
||||
public static registerClient(data: {
|
||||
clientId: string;
|
||||
name: string;
|
||||
rateLimit?: number;
|
||||
}): ClientSDK {
|
||||
const safeClientId = data.clientId
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9_-]/g, "_");
|
||||
const secret = `sec_${safeClientId}_${Math.random().toString(36).substring(2, 10)}`;
|
||||
const rateLimit = data.rateLimit || 100;
|
||||
|
||||
@@ -82,7 +91,7 @@ export class SDKManager {
|
||||
secret,
|
||||
token,
|
||||
rateLimit,
|
||||
status: 'active',
|
||||
status: "active",
|
||||
totalEvents: 0,
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
@@ -106,7 +115,10 @@ export class SDKManager {
|
||||
return client;
|
||||
}
|
||||
|
||||
public static updateClient(clientId: string, updates: Partial<Pick<ClientSDK, 'name' | 'rateLimit' | 'status'>>): ClientSDK | null {
|
||||
public static updateClient(
|
||||
clientId: string,
|
||||
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
|
||||
): ClientSDK | null {
|
||||
const client = this.clients.get(clientId);
|
||||
if (!client) return null;
|
||||
|
||||
|
||||
Reference in new issue
Block a user