feat: update schema, improve shutdown and SDK logic
- Add columns to `client_sdks` and `admin_settings` tables - Introduce `flushAndShutdownOnSignal` for graceful buffer shutdown - Ensure database settings are hydrated before syncing SDKs - Update documentation for SDK structure and volume mounts
This commit is contained in:
1 parent
177ed4a844
commit
05407fd338
15 files changed
+1985
-3582
No files matched your search
@@ -289,7 +289,7 @@ DataStack includes a production-ready Docker Compose stack in [`docker/docker-co
|
|||||||
- **`ingestion`**: Express + Socket.IO container on port `4000` (runs as non-root user).
|
- **`ingestion`**: Express + Socket.IO container on port `4000` (runs as non-root user).
|
||||||
- **`postgres`**: PostgreSQL 16 with persistent `pgdata` volume.
|
- **`postgres`**: PostgreSQL 16 with persistent `pgdata` volume.
|
||||||
- **`redis`**: Redis 7 with persistent `redisdata` volume.
|
- **`redis`**: Redis 7 with persistent `redisdata` volume.
|
||||||
- **`datastack-data` Volume**: Mounted at `/app/storage/data` in both `web` and `ingestion` so CSV partitions are shared seamlessly across containers.
|
- **`datastack-data` Volume**: Mounted at `/app/storage` (persisting `/app/storage/data/{clientId}/{YYYY-MM-DD}.csv`) in both `web` and `ingestion` so CSV partitions are shared seamlessly across containers.
|
||||||
|
|
||||||
### Coolify Deployment Steps
|
### Coolify Deployment Steps
|
||||||
1. Create a **Docker Compose** resource in Coolify pointing to your repository.
|
1. Create a **Docker Compose** resource in Coolify pointing to your repository.
|
||||||
@@ -310,7 +310,8 @@ datastack/
|
|||||||
│ └── ui/ # Single-source-of-truth UI primitives (Button, Input, Table, Modal, StatStrip)
|
│ └── ui/ # Single-source-of-truth UI primitives (Button, Input, Table, Modal, StatStrip)
|
||||||
├── core/ # Backend TypeScript modules (auth, Database, sdk, adminSettings, data, Storage, Security)
|
├── core/ # Backend TypeScript modules (auth, Database, sdk, adminSettings, data, Storage, Security)
|
||||||
├── server/ # Dedicated Port 4000 Express + Socket.IO service & CSV batch worker
|
├── server/ # Dedicated Port 4000 Express + Socket.IO service & CSV batch worker
|
||||||
├── sdk-python/ # Official pip-installable Python client SDK (datastack-sdk)
|
├── sdk/ # Official pip-installable Python SDK (`./sdk/python/datastack`)
|
||||||
|
├── sdk-python/ # Backward-compatible Python client package (`datastack-sdk`)
|
||||||
├── storage/ # Partitioned CSV storage root (/storage/data/{clientId}/{YYYY-MM-DD}.csv)
|
├── storage/ # Partitioned CSV storage root (/storage/data/{clientId}/{YYYY-MM-DD}.csv)
|
||||||
├── docker/ # Production Dockerfiles, Nginx config, and docker-compose.yml
|
├── docker/ # Production Dockerfiles, Nginx config, and docker-compose.yml
|
||||||
├── nginx/ # Local Nginx reverse proxy configuration
|
├── nginx/ # Local Nginx reverse proxy configuration
|
||||||
|
|||||||
@@ -61,7 +61,7 @@ export async function GET(req: NextRequest) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const bufferStats = DataBufferEngine.getBufferStats();
|
const bufferStats = await DataBufferEngine.getBufferStatsAsync();
|
||||||
|
|
||||||
return NextResponse.json({
|
return NextResponse.json({
|
||||||
status: 'online',
|
status: 'online',
|
||||||
|
|||||||
@@ -98,7 +98,32 @@ export class DatabaseEngine {
|
|||||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||||
)
|
)
|
||||||
`),
|
`),
|
||||||
]).then(() => undefined);
|
]).then(() =>
|
||||||
|
Promise.all([
|
||||||
|
pool.query(`
|
||||||
|
ALTER TABLE client_sdks
|
||||||
|
ADD COLUMN IF NOT EXISTS rate_limit INTEGER NOT NULL DEFAULT 100,
|
||||||
|
ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'active',
|
||||||
|
ADD COLUMN IF NOT EXISTS total_events BIGINT NOT NULL DEFAULT 0,
|
||||||
|
ADD COLUMN IF NOT EXISTS last_active TIMESTAMPTZ,
|
||||||
|
ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||||
|
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||||
|
`),
|
||||||
|
pool.query(`
|
||||||
|
ALTER TABLE admin_settings
|
||||||
|
ADD COLUMN IF NOT EXISTS server_port INTEGER NOT NULL DEFAULT 4000,
|
||||||
|
ADD COLUMN IF NOT EXISTS web_port INTEGER NOT NULL DEFAULT 3000,
|
||||||
|
ADD COLUMN IF NOT EXISTS max_events_per_sec INTEGER NOT NULL DEFAULT 100,
|
||||||
|
ADD COLUMN IF NOT EXISTS max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
|
||||||
|
ADD COLUMN IF NOT EXISTS batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
|
||||||
|
ADD COLUMN IF NOT EXISTS batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
|
||||||
|
ADD COLUMN IF NOT EXISTS file_rotation_mb INTEGER NOT NULL DEFAULT 100,
|
||||||
|
ADD COLUMN IF NOT EXISTS enable_idempotency BOOLEAN NOT NULL DEFAULT true,
|
||||||
|
ADD COLUMN IF NOT EXISTS auto_prune_days INTEGER NOT NULL DEFAULT 30,
|
||||||
|
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||||
|
`),
|
||||||
|
]),
|
||||||
|
).then(() => undefined);
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await globals.__dataStackCoreSchema;
|
await globals.__dataStackCoreSchema;
|
||||||
|
|||||||
@@ -63,7 +63,11 @@ export class SettingsManager {
|
|||||||
AuthService.setSecret(settings.jwtSecret);
|
AuthService.setSecret(settings.jwtSecret);
|
||||||
}
|
}
|
||||||
CSVStorageManager.maxFileSizeBytes = settings.fileRotationMb * 1024 * 1024;
|
CSVStorageManager.maxFileSizeBytes = settings.fileRotationMb * 1024 * 1024;
|
||||||
if (settings.autoPruneDays > 0) {
|
const hasConfiguredDb = Boolean(process.env.DATABASE_URL?.trim());
|
||||||
|
if (
|
||||||
|
settings.autoPruneDays > 0 &&
|
||||||
|
(!hasConfiguredDb || globals.__dataStackSettingsHydrated === true)
|
||||||
|
) {
|
||||||
CSVStorageManager.pruneExpiredPartitions(settings.autoPruneDays);
|
CSVStorageManager.pruneExpiredPartitions(settings.autoPruneDays);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+36
-4
@@ -53,6 +53,7 @@ interface BufferGlobalState {
|
|||||||
type BufferGlobals = typeof globalThis & {
|
type BufferGlobals = typeof globalThis & {
|
||||||
__dataStackBufferState?: BufferGlobalState;
|
__dataStackBufferState?: BufferGlobalState;
|
||||||
__dataStackBufferShutdownRegistered?: boolean;
|
__dataStackBufferShutdownRegistered?: boolean;
|
||||||
|
__dataStackBufferShuttingDown?: Promise<void> | null;
|
||||||
};
|
};
|
||||||
|
|
||||||
const globals = globalThis as BufferGlobals;
|
const globals = globalThis as BufferGlobals;
|
||||||
@@ -202,6 +203,30 @@ export class DataBufferEngine {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static async flushAndShutdownOnSignal(): Promise<void> {
|
||||||
|
if (globals.__dataStackBufferShuttingDown) {
|
||||||
|
return globals.__dataStackBufferShuttingDown;
|
||||||
|
}
|
||||||
|
|
||||||
|
const run = (async () => {
|
||||||
|
try {
|
||||||
|
await Promise.race([
|
||||||
|
this.flush().catch(() => undefined),
|
||||||
|
new Promise<void>((resolve) => setTimeout(resolve, 4000)),
|
||||||
|
]);
|
||||||
|
} finally {
|
||||||
|
await Promise.race([
|
||||||
|
this.shutdown().catch(() => undefined),
|
||||||
|
new Promise<void>((resolve) => setTimeout(resolve, 1500)),
|
||||||
|
]);
|
||||||
|
globals.__dataStackBufferShuttingDown = null;
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
|
||||||
|
globals.__dataStackBufferShuttingDown = run;
|
||||||
|
return run;
|
||||||
|
}
|
||||||
|
|
||||||
public static init() {
|
public static init() {
|
||||||
const state = getState();
|
const state = getState();
|
||||||
if (!state.flushTimer) {
|
if (!state.flushTimer) {
|
||||||
@@ -212,15 +237,22 @@ export class DataBufferEngine {
|
|||||||
|
|
||||||
if (
|
if (
|
||||||
typeof process !== 'undefined' &&
|
typeof process !== 'undefined' &&
|
||||||
typeof process.on === 'function' &&
|
typeof process.once === 'function' &&
|
||||||
!globals.__dataStackBufferShutdownRegistered
|
!globals.__dataStackBufferShutdownRegistered
|
||||||
) {
|
) {
|
||||||
globals.__dataStackBufferShutdownRegistered = true;
|
globals.__dataStackBufferShutdownRegistered = true;
|
||||||
const flushOnExit = () => {
|
const flushOnExit = () => {
|
||||||
void this.flush().catch(() => undefined);
|
void this.flushAndShutdownOnSignal().finally(() => {
|
||||||
|
if (
|
||||||
|
process.env.NODE_ENV === 'production' &&
|
||||||
|
!process.execArgv.includes('--test')
|
||||||
|
) {
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
});
|
||||||
};
|
};
|
||||||
process.on('SIGTERM', flushOnExit);
|
process.once('SIGTERM', flushOnExit);
|
||||||
process.on('SIGINT', flushOnExit);
|
process.once('SIGINT', flushOnExit);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import { existsSync, rmSync } from "node:fs";
|
import { existsSync, rmSync } from "node:fs";
|
||||||
import { resolve, sep } from "node:path";
|
import { resolve, sep } from "node:path";
|
||||||
|
import { SettingsManager } from "../adminSettings/settingsManager";
|
||||||
import { AuthService } from "../auth/jwt";
|
import { AuthService } from "../auth/jwt";
|
||||||
import { DatabaseEngine } from "../Database/database";
|
import { DatabaseEngine } from "../Database/database";
|
||||||
import { getInternalIngestionHeaders } from "../Security/security";
|
import { getInternalIngestionHeaders } from "../Security/security";
|
||||||
@@ -25,6 +26,12 @@ type SDKDatabaseGlobals = typeof globalThis & {
|
|||||||
|
|
||||||
const globals = globalThis as SDKDatabaseGlobals;
|
const globals = globalThis as SDKDatabaseGlobals;
|
||||||
|
|
||||||
|
async function ensureDbSettingsHydrated(): Promise<void> {
|
||||||
|
if (DatabaseEngine.getPool()) {
|
||||||
|
await SettingsManager.getSettingsAsync();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function getClientsMap(): Map<string, ClientSDK> {
|
function getClientsMap(): Map<string, ClientSDK> {
|
||||||
if (!globals.__dataStackClientsMap) {
|
if (!globals.__dataStackClientsMap) {
|
||||||
globals.__dataStackClientsMap = new Map<string, ClientSDK>();
|
globals.__dataStackClientsMap = new Map<string, ClientSDK>();
|
||||||
@@ -160,6 +167,7 @@ export class SDKManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public static async syncAllClientsToIngestionAsync(): Promise<void> {
|
public static async syncAllClientsToIngestionAsync(): Promise<void> {
|
||||||
|
await ensureDbSettingsHydrated();
|
||||||
const headers = getInternalIngestionHeaders();
|
const headers = getInternalIngestionHeaders();
|
||||||
if (!headers["x-datastack-internal-secret"]) return;
|
if (!headers["x-datastack-internal-secret"]) return;
|
||||||
const ingestionBaseUrl = (
|
const ingestionBaseUrl = (
|
||||||
@@ -185,6 +193,7 @@ export class SDKManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public static async listClientsAsync(): Promise<ClientSDK[]> {
|
public static async listClientsAsync(): Promise<ClientSDK[]> {
|
||||||
|
await ensureDbSettingsHydrated();
|
||||||
const fromDb = await this.syncFromDatabase();
|
const fromDb = await this.syncFromDatabase();
|
||||||
if (fromDb !== null) {
|
if (fromDb !== null) {
|
||||||
return fromDb;
|
return fromDb;
|
||||||
@@ -253,6 +262,7 @@ export class SDKManager {
|
|||||||
public static async getClientAsync(
|
public static async getClientAsync(
|
||||||
clientId: string,
|
clientId: string,
|
||||||
): Promise<ClientSDK | undefined> {
|
): Promise<ClientSDK | undefined> {
|
||||||
|
await ensureDbSettingsHydrated();
|
||||||
const pool = DatabaseEngine.getPool();
|
const pool = DatabaseEngine.getPool();
|
||||||
if (pool) {
|
if (pool) {
|
||||||
try {
|
try {
|
||||||
@@ -354,6 +364,7 @@ export class SDKManager {
|
|||||||
name: string;
|
name: string;
|
||||||
rateLimit?: number;
|
rateLimit?: number;
|
||||||
}): Promise<ClientSDK> {
|
}): Promise<ClientSDK> {
|
||||||
|
await ensureDbSettingsHydrated();
|
||||||
const safeClientId = data.clientId
|
const safeClientId = data.clientId
|
||||||
.trim()
|
.trim()
|
||||||
.toLowerCase()
|
.toLowerCase()
|
||||||
@@ -427,6 +438,7 @@ export class SDKManager {
|
|||||||
public static async regenerateTokenAsync(
|
public static async regenerateTokenAsync(
|
||||||
clientId: string,
|
clientId: string,
|
||||||
): Promise<ClientSDK | null> {
|
): Promise<ClientSDK | null> {
|
||||||
|
await ensureDbSettingsHydrated();
|
||||||
const existing = await this.getClientAsync(clientId);
|
const existing = await this.getClientAsync(clientId);
|
||||||
if (!existing) return null;
|
if (!existing) return null;
|
||||||
|
|
||||||
@@ -493,6 +505,7 @@ export class SDKManager {
|
|||||||
clientId: string,
|
clientId: string,
|
||||||
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
|
updates: Partial<Pick<ClientSDK, "name" | "rateLimit" | "status">>,
|
||||||
): Promise<ClientSDK | null> {
|
): Promise<ClientSDK | null> {
|
||||||
|
await ensureDbSettingsHydrated();
|
||||||
const client = await this.getClientAsync(clientId);
|
const client = await this.getClientAsync(clientId);
|
||||||
if (!client) return null;
|
if (!client) return null;
|
||||||
|
|
||||||
@@ -607,6 +620,7 @@ export class SDKManager {
|
|||||||
clientId: string,
|
clientId: string,
|
||||||
options?: { purgeFiles?: boolean },
|
options?: { purgeFiles?: boolean },
|
||||||
): Promise<boolean> {
|
): Promise<boolean> {
|
||||||
|
await ensureDbSettingsHydrated();
|
||||||
const normalized = String(clientId || "")
|
const normalized = String(clientId || "")
|
||||||
.trim()
|
.trim()
|
||||||
.toLowerCase();
|
.toLowerCase();
|
||||||
|
|||||||
@@ -13,4 +13,4 @@ RUN apk add --no-cache su-exec \
|
|||||||
COPY --from=dependencies --chown=node:node /app/node_modules ./node_modules
|
COPY --from=dependencies --chown=node:node /app/node_modules ./node_modules
|
||||||
COPY --chown=node:node server ./server
|
COPY --chown=node:node server ./server
|
||||||
EXPOSE 4000
|
EXPOSE 4000
|
||||||
CMD ["sh", "-c", "mkdir -p /app/storage/data && chown -R node:node /app/storage && exec su-exec node:node node server/index.js"]
|
CMD ["sh", "-c", "mkdir -p /app/storage/data && NODE_UG=\"$(id -u node):$(id -g node)\" && if [ \"$(stat -c '%u:%g' /app/storage/data)\" != \"$NODE_UG\" ]; then chown -R node:node /app/storage; else chown node:node /app/storage /app/storage/data; fi && exec su-exec node:node node server/index.js"]
|
||||||
@@ -25,4 +25,4 @@ COPY --from=builder --chown=node:node /app/public ./public
|
|||||||
COPY --from=builder --chown=node:node /app/.next/standalone ./
|
COPY --from=builder --chown=node:node /app/.next/standalone ./
|
||||||
COPY --from=builder --chown=node:node /app/.next/static ./.next/static
|
COPY --from=builder --chown=node:node /app/.next/static ./.next/static
|
||||||
EXPOSE 3000
|
EXPOSE 3000
|
||||||
CMD ["sh", "-c", "mkdir -p /app/storage/data && chown -R node:node /app/storage && exec su-exec node:node node server.js"]
|
CMD ["sh", "-c", "mkdir -p /app/storage/data && NODE_UG=\"$(id -u node):$(id -g node)\" && if [ \"$(stat -c '%u:%g' /app/storage/data)\" != \"$NODE_UG\" ]; then chown -R node:node /app/storage; else chown node:node /app/storage /app/storage/data; fi && exec su-exec node:node node server.js"]
|
||||||
@@ -27,6 +27,16 @@ server {
|
|||||||
|
|
||||||
client_max_body_size 50M;
|
client_max_body_size 50M;
|
||||||
|
|
||||||
|
location = /health {
|
||||||
|
proxy_pass http://socketio_upstream;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Forwarded-Host $forwarded_host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $forwarded_proto;
|
||||||
|
}
|
||||||
|
|
||||||
location /socket.io/ {
|
location /socket.io/ {
|
||||||
proxy_pass http://socketio_upstream;
|
proxy_pass http://socketio_upstream;
|
||||||
proxy_http_version 1.1;
|
proxy_http_version 1.1;
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ services:
|
|||||||
PORT: "3000"
|
PORT: "3000"
|
||||||
INGESTION_INTERNAL_URL: "http://ingestion:4000"
|
INGESTION_INTERNAL_URL: "http://ingestion:4000"
|
||||||
DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}"
|
DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}"
|
||||||
|
DATASTACK_INTERNAL_SECRET: "${DATASTACK_INTERNAL_SECRET:-${SERVICE_REALBASE64_64_INTERNAL:-}}"
|
||||||
DATASTACK_ADMIN_EMAIL: "${DATASTACK_ADMIN_EMAIL:-admin@datastack.dev}"
|
DATASTACK_ADMIN_EMAIL: "${DATASTACK_ADMIN_EMAIL:-admin@datastack.dev}"
|
||||||
DATASTACK_ADMIN_SESSION_SECRET: "${SERVICE_REALBASE64_64_ADMINSESSION:?Coolify must generate SERVICE_REALBASE64_64_ADMINSESSION}"
|
DATASTACK_ADMIN_SESSION_SECRET: "${SERVICE_REALBASE64_64_ADMINSESSION:?Coolify must generate SERVICE_REALBASE64_64_ADMINSESSION}"
|
||||||
DATASTACK_ADMIN_SETUP_KEY: "${SERVICE_REALBASE64_64_ADMININSTALL:?Coolify must generate SERVICE_REALBASE64_64_ADMININSTALL}"
|
DATASTACK_ADMIN_SETUP_KEY: "${SERVICE_REALBASE64_64_ADMININSTALL:?Coolify must generate SERVICE_REALBASE64_64_ADMININSTALL}"
|
||||||
@@ -30,7 +31,7 @@ services:
|
|||||||
"CMD",
|
"CMD",
|
||||||
"node",
|
"node",
|
||||||
"-e",
|
"-e",
|
||||||
"fetch('http://127.0.0.1:3000/').then((response) => process.exit(response.ok ? 0 : 1)).catch(() => process.exit(1))",
|
"fetch('http://127.0.0.1:3000/api/health').then((response) => process.exit(response.ok ? 0 : 1)).catch(() => process.exit(1))",
|
||||||
]
|
]
|
||||||
interval: 30s
|
interval: 30s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
@@ -50,6 +51,7 @@ services:
|
|||||||
NODE_ENV: production
|
NODE_ENV: production
|
||||||
PORT_SOCKET: "4000"
|
PORT_SOCKET: "4000"
|
||||||
DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}"
|
DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}"
|
||||||
|
DATASTACK_INTERNAL_SECRET: "${DATASTACK_INTERNAL_SECRET:-${SERVICE_REALBASE64_64_INTERNAL:-}}"
|
||||||
DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}"
|
DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}"
|
||||||
REDIS_URL: "redis://redis:6379"
|
REDIS_URL: "redis://redis:6379"
|
||||||
depends_on:
|
depends_on:
|
||||||
@@ -85,7 +87,7 @@ services:
|
|||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1/ || exit 1"]
|
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1/api/health || exit 1"]
|
||||||
interval: 30s
|
interval: 30s
|
||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 3
|
retries: 3
|
||||||
@@ -94,6 +96,9 @@ services:
|
|||||||
# Shared Redis ingestion buffer & distributed flush coordination
|
# Shared Redis ingestion buffer & distributed flush coordination
|
||||||
redis:
|
redis:
|
||||||
image: redis:7-alpine
|
image: redis:7-alpine
|
||||||
|
command: ["redis-server", "--appendonly", "yes"]
|
||||||
|
volumes:
|
||||||
|
- redisdata:/data
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "redis-cli", "ping"]
|
test: ["CMD", "redis-cli", "ping"]
|
||||||
@@ -120,3 +125,4 @@ services:
|
|||||||
volumes:
|
volumes:
|
||||||
datastack-data:
|
datastack-data:
|
||||||
pgdata:
|
pgdata:
|
||||||
|
redisdata:
|
||||||
+2
-2
@@ -7,9 +7,9 @@ Production container definitions and orchestration for DataStack's multi-service
|
|||||||
- `ingestion` (`Dockerfile.server`): Express + Socket.IO real-time ingestion server on port `4000` (runs as non-root user).
|
- `ingestion` (`Dockerfile.server`): Express + Socket.IO real-time ingestion server on port `4000` (runs as non-root user).
|
||||||
- `proxy` (`Dockerfile.proxy` + `coolify-nginx.conf`): Nginx reverse proxy on internal port `80`, forwarding `/socket.io/` to `ingestion:4000` and web traffic to `web:3000`.
|
- `proxy` (`Dockerfile.proxy` + `coolify-nginx.conf`): Nginx reverse proxy on internal port `80`, forwarding `/socket.io/` to `ingestion:4000` and web traffic to `web:3000`.
|
||||||
- `postgres`: PostgreSQL 16 service with persistent `pgdata` volume storing `admin_users`, `admin_login_attempts`, `client_sdks`, and `admin_settings`.
|
- `postgres`: PostgreSQL 16 service with persistent `pgdata` volume storing `admin_users`, `admin_login_attempts`, `client_sdks`, and `admin_settings`.
|
||||||
- `redis`: Redis 7 service with persistent `redisdata` volume.
|
- `redis`: Redis 7 service with persistent `redisdata` volume (`--appendonly yes`).
|
||||||
|
|
||||||
## Shared Volumes & Networking
|
## Shared Volumes & Networking
|
||||||
- Both `web` and `ingestion` mount the `datastack-data` volume at `/app/storage/data`, ensuring CSV partitions written by either process are immediately visible across the platform.
|
- Both `web` and `ingestion` mount the `datastack-data` volume at `/app/storage` (persisting `/app/storage/data/{clientId}/{YYYY-MM-DD}.csv`), ensuring CSV partitions written by either process are immediately visible across the platform.
|
||||||
- Both `web` and `ingestion` connect to `postgres` via `DATABASE_URL` over the private Compose network, synchronizing client registrations, revocation status, and runtime settings automatically.
|
- Both `web` and `ingestion` connect to `postgres` via `DATABASE_URL` over the private Compose network, synchronizing client registrations, revocation status, and runtime settings automatically.
|
||||||
- `web` communicates with `ingestion` via `INGESTION_INTERNAL_URL="http://ingestion:4000"` for live health metrics and manual buffer flushes.
|
- `web` communicates with `ingestion` via `INGESTION_INTERNAL_URL="http://ingestion:4000"` for live health metrics and manual buffer flushes.
|
||||||
@@ -27,6 +27,17 @@ server {
|
|||||||
|
|
||||||
client_max_body_size 50M;
|
client_max_body_size 50M;
|
||||||
|
|
||||||
|
# Express + Socket.IO health endpoint on port 4000
|
||||||
|
location = /health {
|
||||||
|
proxy_pass http://socketio_upstream;
|
||||||
|
proxy_http_version 1.1;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Forwarded-Host $forwarded_host;
|
||||||
|
proxy_set_header X-Real-IP $remote_addr;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $forwarded_proto;
|
||||||
|
}
|
||||||
|
|
||||||
# Socket.IO WebSocket reverse proxy to port 4000
|
# Socket.IO WebSocket reverse proxy to port 4000
|
||||||
location /socket.io/ {
|
location /socket.io/ {
|
||||||
proxy_pass http://socketio_upstream;
|
proxy_pass http://socketio_upstream;
|
||||||
|
|||||||
Generated
+1646
-3567
File diff suppressed because it is too large.
Load diff
@@ -599,6 +599,28 @@ async function ensureDatabaseSchema() {
|
|||||||
deleted_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
deleted_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||||
)
|
)
|
||||||
`);
|
`);
|
||||||
|
await dbPool.query(`
|
||||||
|
ALTER TABLE client_sdks
|
||||||
|
ADD COLUMN IF NOT EXISTS rate_limit INTEGER NOT NULL DEFAULT 100,
|
||||||
|
ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'active',
|
||||||
|
ADD COLUMN IF NOT EXISTS total_events BIGINT NOT NULL DEFAULT 0,
|
||||||
|
ADD COLUMN IF NOT EXISTS last_active TIMESTAMPTZ,
|
||||||
|
ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||||
|
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||||
|
`);
|
||||||
|
await dbPool.query(`
|
||||||
|
ALTER TABLE admin_settings
|
||||||
|
ADD COLUMN IF NOT EXISTS server_port INTEGER NOT NULL DEFAULT 4000,
|
||||||
|
ADD COLUMN IF NOT EXISTS web_port INTEGER NOT NULL DEFAULT 3000,
|
||||||
|
ADD COLUMN IF NOT EXISTS max_events_per_sec INTEGER NOT NULL DEFAULT 100,
|
||||||
|
ADD COLUMN IF NOT EXISTS max_payload_size_kb INTEGER NOT NULL DEFAULT 100,
|
||||||
|
ADD COLUMN IF NOT EXISTS batch_flush_threshold INTEGER NOT NULL DEFAULT 500,
|
||||||
|
ADD COLUMN IF NOT EXISTS batch_flush_interval_ms INTEGER NOT NULL DEFAULT 2000,
|
||||||
|
ADD COLUMN IF NOT EXISTS file_rotation_mb INTEGER NOT NULL DEFAULT 100,
|
||||||
|
ADD COLUMN IF NOT EXISTS enable_idempotency BOOLEAN NOT NULL DEFAULT true,
|
||||||
|
ADD COLUMN IF NOT EXISTS auto_prune_days INTEGER NOT NULL DEFAULT 30,
|
||||||
|
ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||||
|
`);
|
||||||
} catch {
|
} catch {
|
||||||
// Ignore transient startup DB error; retried on next sync cycle
|
// Ignore transient startup DB error; retried on next sync cycle
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,199 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { existsSync, mkdirSync, readFileSync, rmSync, utimesSync, writeFileSync } from "node:fs";
|
||||||
|
import { resolve } from "node:path";
|
||||||
|
import test, { after } from "node:test";
|
||||||
|
import { NextRequest } from "next/server";
|
||||||
|
import { GET as getHealthRoute } from "../app/api/health/route";
|
||||||
|
import { SettingsManager } from "../core/adminSettings/settingsManager";
|
||||||
|
import { DataBufferEngine, type RedisBufferClient } from "../core/data/buffer";
|
||||||
|
|
||||||
|
const g = globalThis as typeof globalThis & {
|
||||||
|
__dataStackAdminSettings?: unknown;
|
||||||
|
__dataStackSettingsHydrated?: boolean;
|
||||||
|
__dataStackCorePool?: unknown;
|
||||||
|
__dataStackBufferState?: {
|
||||||
|
buffer: unknown[];
|
||||||
|
flushTimer: ReturnType<typeof setInterval> | null;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
const prevDbUrl = process.env.DATABASE_URL;
|
||||||
|
const prevRedisUrl = process.env.REDIS_URL;
|
||||||
|
|
||||||
|
after(async () => {
|
||||||
|
DataBufferEngine.setRedisClientForTests(undefined);
|
||||||
|
await DataBufferEngine.shutdown();
|
||||||
|
|
||||||
|
if (g.__dataStackBufferState?.flushTimer) {
|
||||||
|
clearInterval(g.__dataStackBufferState.flushTimer);
|
||||||
|
g.__dataStackBufferState.flushTimer = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
delete g.__dataStackAdminSettings;
|
||||||
|
delete g.__dataStackSettingsHydrated;
|
||||||
|
delete g.__dataStackCorePool;
|
||||||
|
|
||||||
|
if (prevDbUrl === undefined) delete process.env.DATABASE_URL;
|
||||||
|
else process.env.DATABASE_URL = prevDbUrl;
|
||||||
|
|
||||||
|
if (prevRedisUrl === undefined) delete process.env.REDIS_URL;
|
||||||
|
else process.env.REDIS_URL = prevRedisUrl;
|
||||||
|
|
||||||
|
const testDirs = [
|
||||||
|
resolve(process.cwd(), "storage", "data", "phase07_prune_guard"),
|
||||||
|
resolve(process.cwd(), "storage", "data", "phase07_shutdown_flush"),
|
||||||
|
];
|
||||||
|
for (const dir of testDirs) {
|
||||||
|
if (existsSync(dir)) {
|
||||||
|
rmSync(dir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("DEPLOY-BLK-01 & 02 & 06: docker-compose.yml persists Redis AOF, configures internal bridge secret, and probes /api/health", () => {
|
||||||
|
const compose = readFileSync(resolve(process.cwd(), "docker", "docker-compose.yml"), "utf8");
|
||||||
|
|
||||||
|
assert.match(compose, /command:\s*\["redis-server",\s*"--appendonly",\s*"yes"\]/);
|
||||||
|
assert.match(compose, /-\s*redisdata:\/data/);
|
||||||
|
assert.match(compose, /^volumes:\r?\n\s+datastack-data:\r?\n\s+pgdata:\r?\n\s+redisdata:/m);
|
||||||
|
|
||||||
|
const internalSecretMatches = compose.match(/^\s+DATASTACK_INTERNAL_SECRET:/gm) ?? [];
|
||||||
|
assert.equal(
|
||||||
|
internalSecretMatches.length,
|
||||||
|
2,
|
||||||
|
"Both web and ingestion services must define DATASTACK_INTERNAL_SECRET",
|
||||||
|
);
|
||||||
|
|
||||||
|
assert.match(compose, /http:\/\/127\.0\.0\.1:3000\/api\/health/);
|
||||||
|
assert.match(compose, /http:\/\/127\.0\.0\.1\/api\/health/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("DEPLOY-BLK-03 & 05: Dockerfiles guard recursive chown on /app/storage/data and package-lock.json exists for deterministic npm ci", () => {
|
||||||
|
const webDockerfile = readFileSync(resolve(process.cwd(), "docker", "Dockerfile.web"), "utf8");
|
||||||
|
const serverDockerfile = readFileSync(resolve(process.cwd(), "docker", "Dockerfile.server"), "utf8");
|
||||||
|
|
||||||
|
for (const df of [webDockerfile, serverDockerfile]) {
|
||||||
|
assert.equal(df.includes('NODE_UG=\\"$(id -u node):$(id -g node)\\"'), true);
|
||||||
|
assert.match(df, /stat -c '%u:%g' \/app\/storage\/data/);
|
||||||
|
}
|
||||||
|
|
||||||
|
const lockPath = resolve(process.cwd(), "package-lock.json");
|
||||||
|
assert.equal(existsSync(lockPath), true, "package-lock.json must exist");
|
||||||
|
const lockJson = JSON.parse(readFileSync(lockPath, "utf8")) as {
|
||||||
|
lockfileVersion?: number;
|
||||||
|
packages?: Record<string, unknown>;
|
||||||
|
};
|
||||||
|
assert.equal(lockJson.lockfileVersion, 3);
|
||||||
|
assert.ok(lockJson.packages && Object.keys(lockJson.packages).length > 10);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("DEPLOY-BLK-04: Both Nginx configs proxy = /health to socketio_upstream", () => {
|
||||||
|
for (const relPath of ["docker/coolify-nginx.conf", "nginx/default.conf"]) {
|
||||||
|
const conf = readFileSync(resolve(process.cwd(), relPath), "utf8");
|
||||||
|
assert.match(
|
||||||
|
conf,
|
||||||
|
/location\s*=\s*\/health\s*\{[\s\S]*?proxy_pass\s+http:\/\/socketio_upstream;/,
|
||||||
|
`${relPath} must route location = /health to socketio_upstream`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("DEPLOY-BLK-03 docs: manifest.md and README.md align volume mount path (/app/storage) and include sdk/python/datastack", () => {
|
||||||
|
const manifest = readFileSync(resolve(process.cwd(), "docker", "manifest.md"), "utf8");
|
||||||
|
const readme = readFileSync(resolve(process.cwd(), "README.md"), "utf8");
|
||||||
|
|
||||||
|
assert.match(manifest, /`datastack-data` volume at `\/app\/storage`/);
|
||||||
|
assert.match(readme, /`datastack-data` Volume\*\*:\s*Mounted at `\/app\/storage`/);
|
||||||
|
assert.match(readme, /sdk\/python\/datastack/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("DEPLOY-BLK-02 & 06 schema/hydration: DDL has additive ADD COLUMN IF NOT EXISTS guards and SDKManager hydrates DB settings", () => {
|
||||||
|
const databaseTs = readFileSync(resolve(process.cwd(), "core", "Database", "database.ts"), "utf8");
|
||||||
|
const serverJs = readFileSync(resolve(process.cwd(), "server", "index.js"), "utf8");
|
||||||
|
const sdkManagerTs = readFileSync(resolve(process.cwd(), "core", "sdk", "sdkManager.ts"), "utf8");
|
||||||
|
|
||||||
|
for (const src of [databaseTs, serverJs]) {
|
||||||
|
assert.match(src, /ALTER TABLE client_sdks\s+ADD COLUMN IF NOT EXISTS/);
|
||||||
|
assert.match(src, /ALTER TABLE admin_settings\s+ADD COLUMN IF NOT EXISTS/);
|
||||||
|
}
|
||||||
|
|
||||||
|
assert.match(sdkManagerTs, /await ensureDbSettingsHydrated\(\);/);
|
||||||
|
assert.match(sdkManagerTs, /SettingsManager\.getSettingsAsync\(\)/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("DEPLOY-BLK-06 prune guard: SettingsManager skips default 30d auto-prune when DATABASE_URL is configured but unhydrated", async () => {
|
||||||
|
const clientDir = resolve(process.cwd(), "storage", "data", "phase07_prune_guard");
|
||||||
|
mkdirSync(clientDir, { recursive: true });
|
||||||
|
const oldCsvPath = resolve(clientDir, "2020-01-01.csv");
|
||||||
|
writeFileSync(oldCsvPath, "eventId,timestamp,payload\nold-1,2020-01-01T00:00:00Z,{}\n", "utf8");
|
||||||
|
const ancient = new Date("2020-01-02T00:00:00Z");
|
||||||
|
utimesSync(oldCsvPath, ancient, ancient);
|
||||||
|
|
||||||
|
delete g.__dataStackAdminSettings;
|
||||||
|
g.__dataStackSettingsHydrated = false;
|
||||||
|
g.__dataStackCorePool = null;
|
||||||
|
process.env.DATABASE_URL = "postgresql://datastack:invalid@127.0.0.1:1/datastack";
|
||||||
|
|
||||||
|
const settings = await SettingsManager.getSettingsAsync();
|
||||||
|
assert.equal(settings.autoPruneDays, 30);
|
||||||
|
assert.equal(
|
||||||
|
existsSync(oldCsvPath),
|
||||||
|
true,
|
||||||
|
"Historical CSV must NOT be pruned during transient unhydrated DB state",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("DEPLOY-BLK-06 shutdown & health: DataBufferEngine.flushAndShutdownOnSignal flushes RAM events and /api/health reports async buffer stats", async () => {
|
||||||
|
delete process.env.DATABASE_URL;
|
||||||
|
g.__dataStackCorePool = null;
|
||||||
|
g.__dataStackSettingsHydrated = false;
|
||||||
|
|
||||||
|
const clientId = "phase07_shutdown_flush";
|
||||||
|
const evtId = `evt-p07-${Date.now()}`;
|
||||||
|
DataBufferEngine.setRedisClientForTests(null);
|
||||||
|
|
||||||
|
await DataBufferEngine.pushAsync({
|
||||||
|
clientId,
|
||||||
|
eventId: evtId,
|
||||||
|
timestamp: new Date().toISOString(),
|
||||||
|
payload: JSON.stringify({ phase: 7 }),
|
||||||
|
});
|
||||||
|
|
||||||
|
await DataBufferEngine.flushAndShutdownOnSignal();
|
||||||
|
|
||||||
|
const clientDir = resolve(process.cwd(), "storage", "data", clientId);
|
||||||
|
assert.equal(existsSync(clientDir), true, "Expected CSV directory after shutdown flush");
|
||||||
|
|
||||||
|
const mockRedis: RedisBufferClient = {
|
||||||
|
status: "ready",
|
||||||
|
async rpush() {
|
||||||
|
return 0;
|
||||||
|
},
|
||||||
|
async lpush() {
|
||||||
|
return 0;
|
||||||
|
},
|
||||||
|
async ltrim() {
|
||||||
|
return "OK";
|
||||||
|
},
|
||||||
|
async lrange() {
|
||||||
|
return [];
|
||||||
|
},
|
||||||
|
async llen() {
|
||||||
|
return 7;
|
||||||
|
},
|
||||||
|
async set() {
|
||||||
|
return "OK";
|
||||||
|
},
|
||||||
|
async eval() {
|
||||||
|
return 1;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
DataBufferEngine.setRedisClientForTests(mockRedis);
|
||||||
|
|
||||||
|
const req = new NextRequest("http://localhost:3000/api/health");
|
||||||
|
const res = await getHealthRoute(req);
|
||||||
|
assert.equal(res.status, 200);
|
||||||
|
const json = (await res.json()) as { status: string; bufferSize: number };
|
||||||
|
assert.equal(json.status, "online");
|
||||||
|
assert.equal(json.bufferSize, 7);
|
||||||
|
});
|
||||||
Reference in new issue
Block a user