docs: update architecture documentation and manifests
Refine technical documentation to reflect current architecture, component structure, and routing conventions for the vcon console.
This commit is contained in:
1 parent
bb938c33c0
commit
367dd819f8
64 files changed
+910
-2661
No files matched your search
+83
-2
@@ -159,9 +159,48 @@ function scheduleFlushTimer(intervalMs) {
|
||||
|
||||
scheduleFlushTimer(runtimeConfig.batchFlushIntervalMs);
|
||||
|
||||
async function ensureDatabaseSchema() {
|
||||
if (!dbPool) return;
|
||||
try {
|
||||
await dbPool.query(`
|
||||
CREATE TABLE IF NOT EXISTS client_sdks (
|
||||
client_id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
secret TEXT NOT NULL,
|
||||
token TEXT NOT NULL,
|
||||
rate_limit INTEGER NOT NULL DEFAULT 100,
|
||||
status TEXT NOT NULL DEFAULT 'active',
|
||||
total_events BIGINT NOT NULL DEFAULT 0,
|
||||
last_active TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`);
|
||||
await dbPool.query(`
|
||||
CREATE TABLE IF NOT EXISTS admin_settings (
|
||||
id TEXT PRIMARY KEY DEFAULT 'default',
|
||||
server_port INTEGER NOT NULL DEFAULT 4000,
|
||||
web_port INTEGER NOT NULL DEFAULT 3000,
|
||||
max_events_per_sec INTEGER NOT NULL DEFAULT 100,
|
||||
max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
|
||||
batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
|
||||
batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
|
||||
file_rotation_mb INTEGER NOT NULL DEFAULT 100,
|
||||
jwt_secret TEXT NOT NULL,
|
||||
enable_idempotency BOOLEAN NOT NULL DEFAULT true,
|
||||
auto_prune_days INTEGER NOT NULL DEFAULT 30,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
)
|
||||
`);
|
||||
} catch {
|
||||
// Ignore transient startup DB error; retried on next sync cycle
|
||||
}
|
||||
}
|
||||
|
||||
async function syncSettingsFromDatabase() {
|
||||
if (!dbPool) return;
|
||||
try {
|
||||
await ensureDatabaseSchema();
|
||||
const res = await dbPool.query(
|
||||
`SELECT max_events_per_sec, max_payload_size_kb, batch_flush_threshold,
|
||||
batch_flush_interval_ms, file_rotation_mb, jwt_secret, enable_idempotency
|
||||
@@ -196,13 +235,14 @@ async function syncSettingsFromDatabase() {
|
||||
scheduleFlushTimer(nextIntervalMs);
|
||||
}
|
||||
} catch {
|
||||
// Keep existing runtimeConfig if table is not created yet
|
||||
// Keep existing runtimeConfig if table is not ready yet
|
||||
}
|
||||
}
|
||||
|
||||
let settingsSyncTimer = null;
|
||||
if (dbPool) {
|
||||
void syncSettingsFromDatabase();
|
||||
setInterval(() => {
|
||||
settingsSyncTimer = setInterval(() => {
|
||||
void syncSettingsFromDatabase();
|
||||
}, 3000);
|
||||
}
|
||||
@@ -252,6 +292,47 @@ setupSocketHandlers(io, bufferQueue, writer, {
|
||||
},
|
||||
});
|
||||
|
||||
let isShuttingDown = false;
|
||||
async function gracefulShutdown(signal) {
|
||||
if (isShuttingDown) return;
|
||||
isShuttingDown = true;
|
||||
console.log(`[DataStack Ingestion Engine] Received ${signal}, flushing buffer before exit...`);
|
||||
|
||||
if (flushTimer) {
|
||||
clearInterval(flushTimer);
|
||||
flushTimer = null;
|
||||
}
|
||||
if (settingsSyncTimer) {
|
||||
clearInterval(settingsSyncTimer);
|
||||
settingsSyncTimer = null;
|
||||
}
|
||||
|
||||
try {
|
||||
await flushBuffer();
|
||||
} catch {
|
||||
// Best-effort flush on exit
|
||||
}
|
||||
|
||||
try {
|
||||
io.close();
|
||||
server.close();
|
||||
if (dbPool) {
|
||||
await dbPool.end();
|
||||
}
|
||||
} catch {
|
||||
// Ignore close errors
|
||||
}
|
||||
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
process.on('SIGTERM', () => {
|
||||
void gracefulShutdown('SIGTERM');
|
||||
});
|
||||
process.on('SIGINT', () => {
|
||||
void gracefulShutdown('SIGINT');
|
||||
});
|
||||
|
||||
server.listen(PORT, () => {
|
||||
console.log(
|
||||
`[DataStack Ingestion Engine] Express + Socket.IO listening on port ${PORT}`,
|
||||
|
||||
+5
-13
@@ -1,17 +1,9 @@
|
||||
# Server Manifest
|
||||
# Socket.IO Ingestion Server Manifest (`server/`)
|
||||
|
||||
## Purpose
|
||||
Independent Express.js and Socket.IO ingestion service operating on Port 4000.
|
||||
Standalone Express.js and Socket.IO ingestion service running on Port `4000` (`server/index.js`).
|
||||
|
||||
## Architecture
|
||||
- **Port**: 4000 (Decoupled from Next.js on port 3000)
|
||||
- **Role**: High-throughput WebSocket handshake, JWT verify, event intake
|
||||
- **Worker**: `workers/csvWriter.js` for async batch appending to `/storage/data/{clientId}/{YYYY-MM-DD}.csv`
|
||||
- **Socket Handlers**: `socket/handlers.js` with client room partitioning and rate limiting.
|
||||
|
||||
## Dependencies
|
||||
- `express`
|
||||
- `socket.io`
|
||||
- `cors`
|
||||
- `jsonwebtoken`
|
||||
- `json2csv`
|
||||
- `index.js`: Initializes the HTTP/Socket.IO server, synchronizes runtime settings and client status from PostgreSQL (`admin_settings`, `client_sdks`), schedules periodic and threshold-based buffer flushes, and exposes `GET /health` and `POST /internal/flush`.
|
||||
- `socket/handlers.js`: Authenticates Socket.IO handshakes via JWT, blocks revoked clients or rotated tokens in real time, validates rate limits, payload sizes, and 5-minute TTL idempotency on `sync_data` and `batch_sync`, and updates client event counters in PostgreSQL.
|
||||
- `workers/csvWriter.js`: Asynchronous batch CSV writer appending queued events to `/storage/data/{clientId}/{YYYY-MM-DD}.csv` with automatic size-based file rotation (`-part2.csv`).
|
||||
Reference in new issue
Block a user