docs: update architecture documentation and manifests

Refine technical documentation to reflect current architecture, component structure, and routing conventions for the vcon console.
This commit is contained in:
Team Dark Devil committed 2026-11-17 21:22:55 +06:00
1 parent bb938c33c0
commit 367dd819f8
64 files changed
+910 -2661

No files matched your search

+83 -2
View File
@@ -159,9 +159,48 @@ function scheduleFlushTimer(intervalMs) {
scheduleFlushTimer(runtimeConfig.batchFlushIntervalMs);
async function ensureDatabaseSchema() {
if (!dbPool) return;
try {
await dbPool.query(`
CREATE TABLE IF NOT EXISTS client_sdks (
client_id TEXT PRIMARY KEY,
name TEXT NOT NULL,
secret TEXT NOT NULL,
token TEXT NOT NULL,
rate_limit INTEGER NOT NULL DEFAULT 100,
status TEXT NOT NULL DEFAULT 'active',
total_events BIGINT NOT NULL DEFAULT 0,
last_active TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`);
await dbPool.query(`
CREATE TABLE IF NOT EXISTS admin_settings (
id TEXT PRIMARY KEY DEFAULT 'default',
server_port INTEGER NOT NULL DEFAULT 4000,
web_port INTEGER NOT NULL DEFAULT 3000,
max_events_per_sec INTEGER NOT NULL DEFAULT 100,
max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
file_rotation_mb INTEGER NOT NULL DEFAULT 100,
jwt_secret TEXT NOT NULL,
enable_idempotency BOOLEAN NOT NULL DEFAULT true,
auto_prune_days INTEGER NOT NULL DEFAULT 30,
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`);
} catch {
// Ignore transient startup DB error; retried on next sync cycle
}
}
async function syncSettingsFromDatabase() {
if (!dbPool) return;
try {
await ensureDatabaseSchema();
const res = await dbPool.query(
`SELECT max_events_per_sec, max_payload_size_kb, batch_flush_threshold,
batch_flush_interval_ms, file_rotation_mb, jwt_secret, enable_idempotency
@@ -196,13 +235,14 @@ async function syncSettingsFromDatabase() {
scheduleFlushTimer(nextIntervalMs);
}
} catch {
// Keep existing runtimeConfig if table is not created yet
// Keep existing runtimeConfig if table is not ready yet
}
}
let settingsSyncTimer = null;
if (dbPool) {
void syncSettingsFromDatabase();
setInterval(() => {
settingsSyncTimer = setInterval(() => {
void syncSettingsFromDatabase();
}, 3000);
}
@@ -252,6 +292,47 @@ setupSocketHandlers(io, bufferQueue, writer, {
},
});
let isShuttingDown = false;
async function gracefulShutdown(signal) {
if (isShuttingDown) return;
isShuttingDown = true;
console.log(`[DataStack Ingestion Engine] Received ${signal}, flushing buffer before exit...`);
if (flushTimer) {
clearInterval(flushTimer);
flushTimer = null;
}
if (settingsSyncTimer) {
clearInterval(settingsSyncTimer);
settingsSyncTimer = null;
}
try {
await flushBuffer();
} catch {
// Best-effort flush on exit
}
try {
io.close();
server.close();
if (dbPool) {
await dbPool.end();
}
} catch {
// Ignore close errors
}
process.exit(0);
}
process.on('SIGTERM', () => {
void gracefulShutdown('SIGTERM');
});
process.on('SIGINT', () => {
void gracefulShutdown('SIGINT');
});
server.listen(PORT, () => {
console.log(
`[DataStack Ingestion Engine] Express + Socket.IO listening on port ${PORT}`,
+5 -13
View File
@@ -1,17 +1,9 @@
# Server Manifest
# Socket.IO Ingestion Server Manifest (`server/`)
## Purpose
Independent Express.js and Socket.IO ingestion service operating on Port 4000.
Standalone Express.js and Socket.IO ingestion service running on Port `4000` (`server/index.js`).
## Architecture
- **Port**: 4000 (Decoupled from Next.js on port 3000)
- **Role**: High-throughput WebSocket handshake, JWT verify, event intake
- **Worker**: `workers/csvWriter.js` for async batch appending to `/storage/data/{clientId}/{YYYY-MM-DD}.csv`
- **Socket Handlers**: `socket/handlers.js` with client room partitioning and rate limiting.
## Dependencies
- `express`
- `socket.io`
- `cors`
- `jsonwebtoken`
- `json2csv`
- `index.js`: Initializes the HTTP/Socket.IO server, synchronizes runtime settings and client status from PostgreSQL (`admin_settings`, `client_sdks`), schedules periodic and threshold-based buffer flushes, and exposes `GET /health` and `POST /internal/flush`.
- `socket/handlers.js`: Authenticates Socket.IO handshakes via JWT, blocks revoked clients or rotated tokens in real time, validates rate limits, payload sizes, and 5-minute TTL idempotency on `sync_data` and `batch_sync`, and updates client event counters in PostgreSQL.
- `workers/csvWriter.js`: Asynchronous batch CSV writer appending queued events to `/storage/data/{clientId}/{YYYY-MM-DD}.csv` with automatic size-based file rotation (`-part2.csv`).