feat: update schema, improve shutdown and SDK logic

- Add columns to `client_sdks` and `admin_settings` tables
- Introduce `flushAndShutdownOnSignal` for graceful buffer shutdown
- Ensure database settings are hydrated before syncing SDKs
- Update documentation for SDK structure and volume mounts
This commit is contained in:
Team Dark Devil committed 2026-11-18 05:42:19 +06:00
1 parent 177ed4a844
commit 05407fd338
15 files changed
+1985 -3582

No files matched your search

+1 -1
View File
@@ -13,4 +13,4 @@ RUN apk add --no-cache su-exec \
COPY --from=dependencies --chown=node:node /app/node_modules ./node_modules
COPY --chown=node:node server ./server
EXPOSE 4000
CMD ["sh", "-c", "mkdir -p /app/storage/data && chown -R node:node /app/storage && exec su-exec node:node node server/index.js"]
CMD ["sh", "-c", "mkdir -p /app/storage/data && NODE_UG=\"$(id -u node):$(id -g node)\" && if [ \"$(stat -c '%u:%g' /app/storage/data)\" != \"$NODE_UG\" ]; then chown -R node:node /app/storage; else chown node:node /app/storage /app/storage/data; fi && exec su-exec node:node node server/index.js"]
+1 -1
View File
@@ -25,4 +25,4 @@ COPY --from=builder --chown=node:node /app/public ./public
COPY --from=builder --chown=node:node /app/.next/standalone ./
COPY --from=builder --chown=node:node /app/.next/static ./.next/static
EXPOSE 3000
CMD ["sh", "-c", "mkdir -p /app/storage/data && chown -R node:node /app/storage && exec su-exec node:node node server.js"]
CMD ["sh", "-c", "mkdir -p /app/storage/data && NODE_UG=\"$(id -u node):$(id -g node)\" && if [ \"$(stat -c '%u:%g' /app/storage/data)\" != \"$NODE_UG\" ]; then chown -R node:node /app/storage; else chown node:node /app/storage /app/storage/data; fi && exec su-exec node:node node server.js"]
+10
View File
@@ -27,6 +27,16 @@ server {
client_max_body_size 50M;
location = /health {
proxy_pass http://socketio_upstream;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-Host $forwarded_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $forwarded_proto;
}
location /socket.io/ {
proxy_pass http://socketio_upstream;
proxy_http_version 1.1;
+8 -2
View File
@@ -13,6 +13,7 @@ services:
PORT: "3000"
INGESTION_INTERNAL_URL: "http://ingestion:4000"
DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}"
DATASTACK_INTERNAL_SECRET: "${DATASTACK_INTERNAL_SECRET:-${SERVICE_REALBASE64_64_INTERNAL:-}}"
DATASTACK_ADMIN_EMAIL: "${DATASTACK_ADMIN_EMAIL:-admin@datastack.dev}"
DATASTACK_ADMIN_SESSION_SECRET: "${SERVICE_REALBASE64_64_ADMINSESSION:?Coolify must generate SERVICE_REALBASE64_64_ADMINSESSION}"
DATASTACK_ADMIN_SETUP_KEY: "${SERVICE_REALBASE64_64_ADMININSTALL:?Coolify must generate SERVICE_REALBASE64_64_ADMININSTALL}"
@@ -30,7 +31,7 @@ services:
"CMD",
"node",
"-e",
"fetch('http://127.0.0.1:3000/').then((response) => process.exit(response.ok ? 0 : 1)).catch(() => process.exit(1))",
"fetch('http://127.0.0.1:3000/api/health').then((response) => process.exit(response.ok ? 0 : 1)).catch(() => process.exit(1))",
]
interval: 30s
timeout: 5s
@@ -50,6 +51,7 @@ services:
NODE_ENV: production
PORT_SOCKET: "4000"
DATASTACK_JWT_SECRET: "${SERVICE_REALBASE64_64_JWT:?Coolify must generate SERVICE_REALBASE64_64_JWT}"
DATASTACK_INTERNAL_SECRET: "${DATASTACK_INTERNAL_SECRET:-${SERVICE_REALBASE64_64_INTERNAL:-}}"
DATABASE_URL: "postgresql://datastack:${SERVICE_PASSWORD_64_POSTGRES:?Coolify must generate SERVICE_PASSWORD_64_POSTGRES}@postgres:5432/${SERVICE_HEX_32_DATABASENAME:?Coolify must generate SERVICE_HEX_32_DATABASENAME}"
REDIS_URL: "redis://redis:6379"
depends_on:
@@ -85,7 +87,7 @@ services:
condition: service_healthy
restart: unless-stopped
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1/ || exit 1"]
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1/api/health || exit 1"]
interval: 30s
timeout: 5s
retries: 3
@@ -94,6 +96,9 @@ services:
# Shared Redis ingestion buffer & distributed flush coordination
redis:
image: redis:7-alpine
command: ["redis-server", "--appendonly", "yes"]
volumes:
- redisdata:/data
restart: unless-stopped
healthcheck:
test: ["CMD", "redis-cli", "ping"]
@@ -120,3 +125,4 @@ services:
volumes:
datastack-data:
pgdata:
redisdata:
+2 -2
View File
@@ -7,9 +7,9 @@ Production container definitions and orchestration for DataStack's multi-service
- `ingestion` (`Dockerfile.server`): Express + Socket.IO real-time ingestion server on port `4000` (runs as non-root user).
- `proxy` (`Dockerfile.proxy` + `coolify-nginx.conf`): Nginx reverse proxy on internal port `80`, forwarding `/socket.io/` to `ingestion:4000` and web traffic to `web:3000`.
- `postgres`: PostgreSQL 16 service with persistent `pgdata` volume storing `admin_users`, `admin_login_attempts`, `client_sdks`, and `admin_settings`.
- `redis`: Redis 7 service with persistent `redisdata` volume.
- `redis`: Redis 7 service with persistent `redisdata` volume (`--appendonly yes`).
## Shared Volumes & Networking
- Both `web` and `ingestion` mount the `datastack-data` volume at `/app/storage/data`, ensuring CSV partitions written by either process are immediately visible across the platform.
- Both `web` and `ingestion` mount the `datastack-data` volume at `/app/storage` (persisting `/app/storage/data/{clientId}/{YYYY-MM-DD}.csv`), ensuring CSV partitions written by either process are immediately visible across the platform.
- Both `web` and `ingestion` connect to `postgres` via `DATABASE_URL` over the private Compose network, synchronizing client registrations, revocation status, and runtime settings automatically.
- `web` communicates with `ingestion` via `INGESTION_INTERNAL_URL="http://ingestion:4000"` for live health metrics and manual buffer flushes.