feat: update schema, improve shutdown and SDK logic

- Add columns to `client_sdks` and `admin_settings` tables
- Introduce `flushAndShutdownOnSignal` for graceful buffer shutdown
- Ensure database settings are hydrated before syncing SDKs
- Update documentation for SDK structure and volume mounts
This commit is contained in:
Team Dark Devil committed 2026-11-18 05:42:19 +06:00
1 parent 177ed4a844
commit 05407fd338
15 files changed
+1985 -3582

No files matched your search

+26 -1
View File
@@ -98,7 +98,32 @@ export class DatabaseEngine {
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
`),
]).then(() => undefined);
]).then(() =>
Promise.all([
pool.query(`
ALTER TABLE client_sdks
ADD COLUMN IF NOT EXISTS rate_limit INTEGER NOT NULL DEFAULT 100,
ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'active',
ADD COLUMN IF NOT EXISTS total_events BIGINT NOT NULL DEFAULT 0,
ADD COLUMN IF NOT EXISTS last_active TIMESTAMPTZ,
ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
`),
pool.query(`
ALTER TABLE admin_settings
ADD COLUMN IF NOT EXISTS server_port INTEGER NOT NULL DEFAULT 4000,
ADD COLUMN IF NOT EXISTS web_port INTEGER NOT NULL DEFAULT 3000,
ADD COLUMN IF NOT EXISTS max_events_per_sec INTEGER NOT NULL DEFAULT 100,
ADD COLUMN IF NOT EXISTS max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
ADD COLUMN IF NOT EXISTS batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
ADD COLUMN IF NOT EXISTS batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
ADD COLUMN IF NOT EXISTS file_rotation_mb INTEGER NOT NULL DEFAULT 100,
ADD COLUMN IF NOT EXISTS enable_idempotency BOOLEAN NOT NULL DEFAULT true,
ADD COLUMN IF NOT EXISTS auto_prune_days INTEGER NOT NULL DEFAULT 30,
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
`),
]),
).then(() => undefined);
try {
await globals.__dataStackCoreSchema;
+5 -1
View File
@@ -63,7 +63,11 @@ export class SettingsManager {
AuthService.setSecret(settings.jwtSecret);
}
CSVStorageManager.maxFileSizeBytes = settings.fileRotationMb * 1024 * 1024;
if (settings.autoPruneDays > 0) {
const hasConfiguredDb = Boolean(process.env.DATABASE_URL?.trim());
if (
settings.autoPruneDays > 0 &&
(!hasConfiguredDb || globals.__dataStackSettingsHydrated === true)
) {
CSVStorageManager.pruneExpiredPartitions(settings.autoPruneDays);
}
}
+36 -4
View File
@@ -53,6 +53,7 @@ interface BufferGlobalState {
type BufferGlobals = typeof globalThis & {
__dataStackBufferState?: BufferGlobalState;
__dataStackBufferShutdownRegistered?: boolean;
__dataStackBufferShuttingDown?: Promise<void> | null;
};
const globals = globalThis as BufferGlobals;
@@ -202,6 +203,30 @@ export class DataBufferEngine {
}
}
public static async flushAndShutdownOnSignal(): Promise<void> {
if (globals.__dataStackBufferShuttingDown) {
return globals.__dataStackBufferShuttingDown;
}
const run = (async () => {
try {
await Promise.race([
this.flush().catch(() => undefined),
new Promise<void>((resolve) => setTimeout(resolve, 4000)),
]);
} finally {
await Promise.race([
this.shutdown().catch(() => undefined),
new Promise<void>((resolve) => setTimeout(resolve, 1500)),
]);
globals.__dataStackBufferShuttingDown = null;
}
})();
globals.__dataStackBufferShuttingDown = run;
return run;
}
public static init() {
const state = getState();
if (!state.flushTimer) {
@@ -212,15 +237,22 @@ export class DataBufferEngine {
if (
typeof process !== 'undefined' &&
typeof process.on === 'function' &&
typeof process.once === 'function' &&
!globals.__dataStackBufferShutdownRegistered
) {
globals.__dataStackBufferShutdownRegistered = true;
const flushOnExit = () => {
void this.flush().catch(() => undefined);
void this.flushAndShutdownOnSignal().finally(() => {
if (
process.env.NODE_ENV === 'production' &&
!process.execArgv.includes('--test')
) {
process.exit(0);
}
});
};
process.on('SIGTERM', flushOnExit);
process.on('SIGINT', flushOnExit);
process.once('SIGTERM', flushOnExit);
process.once('SIGINT', flushOnExit);
}
}
+14
View File
@@ -1,5 +1,6 @@
import { existsSync, rmSync } from "node:fs";
import { resolve, sep } from "node:path";
import { SettingsManager } from "../adminSettings/settingsManager";
import { AuthService } from "../auth/jwt";
import { DatabaseEngine } from "../Database/database";
import { getInternalIngestionHeaders } from "../Security/security";
@@ -25,6 +26,12 @@ type SDKDatabaseGlobals = typeof globalThis & {
const globals = globalThis as SDKDatabaseGlobals;
async function ensureDbSettingsHydrated(): Promise<void> {
if (DatabaseEngine.getPool()) {
await SettingsManager.getSettingsAsync();
}
}
function getClientsMap(): Map<string, ClientSDK> {
if (!globals.__dataStackClientsMap) {
globals.__dataStackClientsMap = new Map<string, ClientSDK>();
@@ -160,6 +167,7 @@ export class SDKManager {
}
public static async syncAllClientsToIngestionAsync(): Promise<void> {
await ensureDbSettingsHydrated();
const headers = getInternalIngestionHeaders();
if (!headers["x-datastack-internal-secret"]) return;
const ingestionBaseUrl = (
@@ -185,6 +193,7 @@ export class SDKManager {
}
public static async listClientsAsync(): Promise<ClientSDK[]> {
await ensureDbSettingsHydrated();
const fromDb = await this.syncFromDatabase();
if (fromDb !== null) {
return fromDb;
@@ -253,6 +262,7 @@ export class SDKManager {
public static async getClientAsync(
clientId: string,
): Promise<ClientSDK | undefined> {
await ensureDbSettingsHydrated();
const pool = DatabaseEngine.getPool();
if (pool) {
try {
@@ -354,6 +364,7 @@ export class SDKManager {
name: string;
rateLimit?: number;
}): Promise<ClientSDK> {
await ensureDbSettingsHydrated();
const safeClientId = data.clientId
.trim()
.toLowerCase()
@@ -427,6 +438,7 @@ export class SDKManager {
public static async regenerateTokenAsync(
clientId: string,
): Promise<ClientSDK | null> {
await ensureDbSettingsHydrated();
const existing = await this.getClientAsync(clientId);
if (!existing) return null;
@@ -493,6 +505,7 @@ export class SDKManager {
clientId: string,
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
): Promise<ClientSDK | null> {
await ensureDbSettingsHydrated();
const client = await this.getClientAsync(clientId);
if (!client) return null;
@@ -607,6 +620,7 @@ export class SDKManager {
clientId: string,
options?: { purgeFiles?: boolean },
): Promise<boolean> {
await ensureDbSettingsHydrated();
const normalized = String(clientId || "")
.trim()
.toLowerCase();