feat(client): implement per-client storage and security controls
- Added storage quota, IP allowlisting, and expiration logic to client management APIs. - Updated CSV storage manager to support auto-compression and improved partition file parsing. - Integrated security checks into event ingestion and data retrieval routes.
This commit is contained in:
1 parent
89e4d68690
commit
2873b9904a
32 files changed
+6075
-241
No files matched your search
@@ -1,12 +1,27 @@
|
||||
import {
|
||||
dispatchWebhookAlertAsync,
|
||||
type WebhookAlertPayload,
|
||||
type WebhookRuntimeConfig,
|
||||
} from "../alerts/webhookNotifier";
|
||||
import { AuthService, resolveDefaultJwtSecret } from "../auth/jwt";
|
||||
import { DataBufferEngine } from "../data/buffer";
|
||||
import { DatabaseEngine } from "../Database/database";
|
||||
import { SDKManager } from "../sdk/sdkManager";
|
||||
import {
|
||||
decryptSecret,
|
||||
encryptSecret,
|
||||
getMaskedHintFromEncrypted,
|
||||
} from "../Security/secretVault";
|
||||
import {
|
||||
SecurityEngine,
|
||||
getInternalIngestionHeaders,
|
||||
} from "../Security/security";
|
||||
import { CSVStorageManager } from "../Storage/csvStorage";
|
||||
import {
|
||||
buildS3ObjectKey,
|
||||
type S3ArchiveConfig,
|
||||
uploadFileToS3,
|
||||
} from "../Storage/s3Archive";
|
||||
|
||||
export interface AdminSettings {
|
||||
serverPort: number;
|
||||
@@ -19,9 +34,49 @@ export interface AdminSettings {
|
||||
jwtSecret: string;
|
||||
enableIdempotency: boolean;
|
||||
autoPruneDays: number;
|
||||
compressAfterDays: number;
|
||||
enableS3Backup: boolean;
|
||||
s3Endpoint: string;
|
||||
s3Region: string;
|
||||
s3Bucket: string;
|
||||
s3Prefix: string;
|
||||
s3AccessKeyId: string;
|
||||
s3SecretAccessKeyEnc: string;
|
||||
diskPauseThresholdPct: number;
|
||||
enableWebhookAlerts: boolean;
|
||||
webhookUrl: string;
|
||||
webhookSecretEnc: string;
|
||||
webhookCooldownSec: number;
|
||||
enablePrometheusMetrics: boolean;
|
||||
prometheusMetricsTokenEnc: string;
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface PublicAdminSettings
|
||||
extends Omit<
|
||||
AdminSettings,
|
||||
| "jwtSecret"
|
||||
| "s3SecretAccessKeyEnc"
|
||||
| "webhookSecretEnc"
|
||||
| "prometheusMetricsTokenEnc"
|
||||
> {
|
||||
hasS3SecretKey: boolean;
|
||||
s3SecretKeyMask: string;
|
||||
hasWebhookSecret: boolean;
|
||||
webhookSecretMask: string;
|
||||
hasPrometheusToken: boolean;
|
||||
prometheusTokenMask: string;
|
||||
}
|
||||
|
||||
export type AdminSettingsUpdateInput = Partial<AdminSettings> & {
|
||||
s3SecretAccessKey?: string;
|
||||
clearS3SecretKey?: boolean;
|
||||
webhookSecret?: string;
|
||||
clearWebhookSecret?: boolean;
|
||||
prometheusMetricsToken?: string;
|
||||
clearPrometheusToken?: boolean;
|
||||
};
|
||||
|
||||
export interface SettingsBridgeSyncStatus {
|
||||
attempted: boolean;
|
||||
ok: boolean;
|
||||
@@ -36,9 +91,11 @@ type SettingsGlobals = typeof globalThis & {
|
||||
__dataStackSettingsHydrated?: boolean;
|
||||
__dataStackLastSettingsBridgeSync?: SettingsBridgeSyncStatus;
|
||||
__dataStackLastPruneMeta?: { days: number; ranAtMs: number };
|
||||
__dataStackLastCompressMeta?: { days: number; ranAtMs: number };
|
||||
};
|
||||
|
||||
const PRUNE_COOLDOWN_MS = 60 * 60 * 1000;
|
||||
const COMPRESS_COOLDOWN_MS = 60 * 60 * 1000;
|
||||
const globals = globalThis as SettingsGlobals;
|
||||
|
||||
function getDefaultSettings(): AdminSettings {
|
||||
@@ -53,6 +110,21 @@ function getDefaultSettings(): AdminSettings {
|
||||
jwtSecret: resolveDefaultJwtSecret(),
|
||||
enableIdempotency: true,
|
||||
autoPruneDays: 30,
|
||||
compressAfterDays: 0,
|
||||
enableS3Backup: false,
|
||||
s3Endpoint: "",
|
||||
s3Region: "auto",
|
||||
s3Bucket: "",
|
||||
s3Prefix: "datastack/",
|
||||
s3AccessKeyId: "",
|
||||
s3SecretAccessKeyEnc: "",
|
||||
diskPauseThresholdPct: 95,
|
||||
enableWebhookAlerts: false,
|
||||
webhookUrl: "",
|
||||
webhookSecretEnc: "",
|
||||
webhookCooldownSec: 300,
|
||||
enablePrometheusMetrics: false,
|
||||
prometheusMetricsTokenEnc: "",
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
@@ -64,8 +136,132 @@ function getSettingsState(): AdminSettings {
|
||||
return globals.__dataStackAdminSettings;
|
||||
}
|
||||
|
||||
function toPublicSettings(settings: AdminSettings): PublicAdminSettings {
|
||||
const s3SecretKeyMask = getMaskedHintFromEncrypted(
|
||||
settings.s3SecretAccessKeyEnc,
|
||||
);
|
||||
const webhookSecretMask = getMaskedHintFromEncrypted(
|
||||
settings.webhookSecretEnc,
|
||||
);
|
||||
const prometheusTokenMask = getMaskedHintFromEncrypted(
|
||||
settings.prometheusMetricsTokenEnc,
|
||||
);
|
||||
return {
|
||||
serverPort: settings.serverPort,
|
||||
webPort: settings.webPort,
|
||||
maxEventsPerSec: settings.maxEventsPerSec,
|
||||
maxPayloadSizeKb: settings.maxPayloadSizeKb,
|
||||
batchFlushThreshold: settings.batchFlushThreshold,
|
||||
batchFlushIntervalMs: settings.batchFlushIntervalMs,
|
||||
fileRotationMb: settings.fileRotationMb,
|
||||
enableIdempotency: settings.enableIdempotency,
|
||||
autoPruneDays: settings.autoPruneDays,
|
||||
compressAfterDays: settings.compressAfterDays,
|
||||
enableS3Backup: settings.enableS3Backup,
|
||||
s3Endpoint: settings.s3Endpoint,
|
||||
s3Region: settings.s3Region,
|
||||
s3Bucket: settings.s3Bucket,
|
||||
s3Prefix: settings.s3Prefix,
|
||||
s3AccessKeyId: settings.s3AccessKeyId,
|
||||
hasS3SecretKey: Boolean(settings.s3SecretAccessKeyEnc),
|
||||
s3SecretKeyMask,
|
||||
diskPauseThresholdPct: settings.diskPauseThresholdPct,
|
||||
enableWebhookAlerts: settings.enableWebhookAlerts,
|
||||
webhookUrl: settings.webhookUrl,
|
||||
hasWebhookSecret: Boolean(settings.webhookSecretEnc),
|
||||
webhookSecretMask,
|
||||
webhookCooldownSec: settings.webhookCooldownSec,
|
||||
enablePrometheusMetrics: settings.enablePrometheusMetrics,
|
||||
hasPrometheusToken: Boolean(settings.prometheusMetricsTokenEnc),
|
||||
prometheusTokenMask,
|
||||
updatedAt: settings.updatedAt,
|
||||
};
|
||||
}
|
||||
|
||||
export class SettingsManager {
|
||||
private static applyRuntimeSettings(settings: AdminSettings, forcePrune = false) {
|
||||
public static triggerWebhookAlert(payload: WebhookAlertPayload): void {
|
||||
const current = getSettingsState();
|
||||
if (!current.enableWebhookAlerts || !current.webhookUrl.trim()) {
|
||||
return;
|
||||
}
|
||||
void (async () => {
|
||||
try {
|
||||
let secret = "";
|
||||
if (current.webhookSecretEnc) {
|
||||
try {
|
||||
secret = decryptSecret(current.webhookSecretEnc);
|
||||
} catch {
|
||||
secret = "";
|
||||
}
|
||||
}
|
||||
await dispatchWebhookAlertAsync(payload, {
|
||||
enabled: current.enableWebhookAlerts,
|
||||
webhookUrl: current.webhookUrl,
|
||||
webhookSecret: secret,
|
||||
cooldownSec: current.webhookCooldownSec,
|
||||
});
|
||||
} catch {
|
||||
// Never allow background webhook dispatch to throw into callers
|
||||
}
|
||||
})();
|
||||
}
|
||||
|
||||
private static async executeArchiveAndPrune(
|
||||
settings: AdminSettings,
|
||||
): Promise<void> {
|
||||
if (!settings.enableS3Backup) {
|
||||
await CSVStorageManager.pruneExpiredPartitionsAsync(
|
||||
settings.autoPruneDays,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const s3Config: S3ArchiveConfig = {
|
||||
endpoint: settings.s3Endpoint,
|
||||
region: settings.s3Region,
|
||||
bucket: settings.s3Bucket,
|
||||
prefix: settings.s3Prefix,
|
||||
accessKeyId: settings.s3AccessKeyId,
|
||||
secretAccessKey: decryptSecret(settings.s3SecretAccessKeyEnc),
|
||||
};
|
||||
|
||||
await CSVStorageManager.pruneOrArchiveExpiredPartitionsAsync(
|
||||
settings.autoPruneDays,
|
||||
{
|
||||
beforeDeleteFile: async (file) => {
|
||||
const objectKey = buildS3ObjectKey(
|
||||
s3Config.prefix,
|
||||
file.clientId,
|
||||
file.filename,
|
||||
);
|
||||
const result = await uploadFileToS3(
|
||||
s3Config,
|
||||
file.filePath,
|
||||
objectKey,
|
||||
);
|
||||
if (!result.ok) {
|
||||
const errMsg =
|
||||
result.error || `Failed to upload ${file.filename} to S3`;
|
||||
this.triggerWebhookAlert({
|
||||
event: "S3_ARCHIVE_FAILED",
|
||||
severity: "critical",
|
||||
title: "Off-Site S3 Archive Upload Failed",
|
||||
message: errMsg,
|
||||
clientId: file.clientId,
|
||||
details: { filename: file.filename, bucket: s3Config.bucket },
|
||||
});
|
||||
throw new Error(errMsg);
|
||||
}
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
private static applyRuntimeSettings(
|
||||
settings: AdminSettings,
|
||||
forcePrune = false,
|
||||
forceCompress = false,
|
||||
) {
|
||||
SecurityEngine.maxEventsPerSec = settings.maxEventsPerSec;
|
||||
SecurityEngine.enableIdempotency = settings.enableIdempotency;
|
||||
SecurityEngine.maxPayloadSizeBytes = settings.maxPayloadSizeKb * 1024;
|
||||
@@ -76,10 +272,34 @@ export class SettingsManager {
|
||||
}
|
||||
CSVStorageManager.maxFileSizeBytes = settings.fileRotationMb * 1024 * 1024;
|
||||
const hasConfiguredDb = Boolean(process.env.DATABASE_URL?.trim());
|
||||
if (
|
||||
settings.autoPruneDays > 0 &&
|
||||
(!hasConfiguredDb || globals.__dataStackSettingsHydrated === true)
|
||||
) {
|
||||
const dbReadyForMaintenance =
|
||||
!hasConfiguredDb || globals.__dataStackSettingsHydrated === true;
|
||||
|
||||
if (settings.compressAfterDays > 0 && dbReadyForMaintenance) {
|
||||
const prevComp = globals.__dataStackLastCompressMeta;
|
||||
const now = Date.now();
|
||||
const shouldCompress =
|
||||
forceCompress ||
|
||||
!prevComp ||
|
||||
prevComp.days !== settings.compressAfterDays ||
|
||||
now - prevComp.ranAtMs >= COMPRESS_COOLDOWN_MS;
|
||||
if (shouldCompress) {
|
||||
globals.__dataStackLastCompressMeta = {
|
||||
days: settings.compressAfterDays,
|
||||
ranAtMs: now,
|
||||
};
|
||||
void CSVStorageManager.compressOldPartitionsAsync(
|
||||
settings.compressAfterDays,
|
||||
).catch((err) => {
|
||||
console.warn(
|
||||
"[DataStack Storage] Background partition compression failed:",
|
||||
err,
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (settings.autoPruneDays > 0 && dbReadyForMaintenance) {
|
||||
const prev = globals.__dataStackLastPruneMeta;
|
||||
const now = Date.now();
|
||||
const shouldPrune =
|
||||
@@ -92,11 +312,9 @@ export class SettingsManager {
|
||||
days: settings.autoPruneDays,
|
||||
ranAtMs: now,
|
||||
};
|
||||
void CSVStorageManager.pruneExpiredPartitionsAsync(
|
||||
settings.autoPruneDays,
|
||||
).catch((err) => {
|
||||
void this.executeArchiveAndPrune(settings).catch((err) => {
|
||||
console.warn(
|
||||
"[DataStack Storage] Background async partition prune failed:",
|
||||
"[DataStack Storage] Background async partition prune/archive failed:",
|
||||
err,
|
||||
);
|
||||
});
|
||||
@@ -115,7 +333,12 @@ export class SettingsManager {
|
||||
const result = await pool.query(
|
||||
`SELECT server_port, web_port, max_events_per_sec, max_payload_size_kb,
|
||||
batch_flush_threshold, batch_flush_interval_ms, file_rotation_mb,
|
||||
jwt_secret, enable_idempotency, auto_prune_days, updated_at
|
||||
jwt_secret, enable_idempotency, auto_prune_days,
|
||||
compress_after_days, enable_s3_backup, s3_endpoint, s3_region,
|
||||
s3_bucket, s3_prefix, s3_access_key_id, s3_secret_access_key_enc,
|
||||
disk_pause_threshold_pct, enable_webhook_alerts, webhook_url,
|
||||
webhook_secret_enc, webhook_cooldown_sec, enable_prometheus_metrics,
|
||||
prometheus_metrics_token_enc, updated_at
|
||||
FROM admin_settings
|
||||
WHERE id = 'default'`,
|
||||
);
|
||||
@@ -126,6 +349,21 @@ export class SettingsManager {
|
||||
row.auto_prune_days !== undefined && row.auto_prune_days !== null
|
||||
? Number(row.auto_prune_days)
|
||||
: NaN;
|
||||
const parsedCompressDays =
|
||||
row.compress_after_days !== undefined &&
|
||||
row.compress_after_days !== null
|
||||
? Number(row.compress_after_days)
|
||||
: NaN;
|
||||
const parsedDiskThreshold =
|
||||
row.disk_pause_threshold_pct !== undefined &&
|
||||
row.disk_pause_threshold_pct !== null
|
||||
? Number(row.disk_pause_threshold_pct)
|
||||
: NaN;
|
||||
const parsedWebhookCooldown =
|
||||
row.webhook_cooldown_sec !== undefined &&
|
||||
row.webhook_cooldown_sec !== null
|
||||
? Number(row.webhook_cooldown_sec)
|
||||
: NaN;
|
||||
const envSocketPort = Number(process.env.PORT_SOCKET);
|
||||
const envWebPort = Number(process.env.PORT);
|
||||
const loaded: AdminSettings = {
|
||||
@@ -142,9 +380,34 @@ export class SettingsManager {
|
||||
batchFlushThreshold: Number(row.batch_flush_threshold) || 500,
|
||||
batchFlushIntervalMs: Number(row.batch_flush_interval_ms) || 2000,
|
||||
fileRotationMb: Number(row.file_rotation_mb) || 100,
|
||||
jwtSecret: String(row.jwt_secret || "").trim() || resolveDefaultJwtSecret(),
|
||||
jwtSecret:
|
||||
String(row.jwt_secret || "").trim() || resolveDefaultJwtSecret(),
|
||||
enableIdempotency: Boolean(row.enable_idempotency),
|
||||
autoPruneDays: Number.isFinite(parsedPruneDays) ? parsedPruneDays : 30,
|
||||
compressAfterDays: Number.isFinite(parsedCompressDays)
|
||||
? Math.max(0, parsedCompressDays)
|
||||
: 0,
|
||||
enableS3Backup: Boolean(row.enable_s3_backup),
|
||||
s3Endpoint: String(row.s3_endpoint ?? ""),
|
||||
s3Region: String(row.s3_region || "auto"),
|
||||
s3Bucket: String(row.s3_bucket ?? ""),
|
||||
s3Prefix: String(row.s3_prefix ?? "datastack/"),
|
||||
s3AccessKeyId: String(row.s3_access_key_id ?? ""),
|
||||
s3SecretAccessKeyEnc: String(row.s3_secret_access_key_enc ?? ""),
|
||||
diskPauseThresholdPct: Number.isFinite(parsedDiskThreshold)
|
||||
? parsedDiskThreshold
|
||||
: 95,
|
||||
enableWebhookAlerts: Boolean(row.enable_webhook_alerts),
|
||||
webhookUrl: String(row.webhook_url ?? ""),
|
||||
webhookSecretEnc: String(row.webhook_secret_enc ?? ""),
|
||||
webhookCooldownSec:
|
||||
Number.isFinite(parsedWebhookCooldown) && parsedWebhookCooldown >= 10
|
||||
? Math.floor(parsedWebhookCooldown)
|
||||
: 300,
|
||||
enablePrometheusMetrics: Boolean(row.enable_prometheus_metrics),
|
||||
prometheusMetricsTokenEnc: String(
|
||||
row.prometheus_metrics_token_enc ?? "",
|
||||
),
|
||||
updatedAt: row.updated_at
|
||||
? new Date(row.updated_at as string | Date).toISOString()
|
||||
: new Date().toISOString(),
|
||||
@@ -198,8 +461,17 @@ export class SettingsManager {
|
||||
`INSERT INTO admin_settings (
|
||||
id, server_port, web_port, max_events_per_sec, max_payload_size_kb,
|
||||
batch_flush_threshold, batch_flush_interval_ms, file_rotation_mb,
|
||||
jwt_secret, enable_idempotency, auto_prune_days, updated_at
|
||||
) VALUES ('default', $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, NOW())
|
||||
jwt_secret, enable_idempotency, auto_prune_days,
|
||||
compress_after_days, enable_s3_backup, s3_endpoint, s3_region,
|
||||
s3_bucket, s3_prefix, s3_access_key_id, s3_secret_access_key_enc,
|
||||
disk_pause_threshold_pct, enable_webhook_alerts, webhook_url,
|
||||
webhook_secret_enc, webhook_cooldown_sec, enable_prometheus_metrics,
|
||||
prometheus_metrics_token_enc, updated_at
|
||||
) VALUES (
|
||||
'default', $1, $2, $3, $4, $5, $6, $7, $8, $9, $10,
|
||||
$11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21,
|
||||
$22, $23, $24, $25, NOW()
|
||||
)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
server_port = EXCLUDED.server_port,
|
||||
web_port = EXCLUDED.web_port,
|
||||
@@ -211,6 +483,21 @@ export class SettingsManager {
|
||||
jwt_secret = EXCLUDED.jwt_secret,
|
||||
enable_idempotency = EXCLUDED.enable_idempotency,
|
||||
auto_prune_days = EXCLUDED.auto_prune_days,
|
||||
compress_after_days = EXCLUDED.compress_after_days,
|
||||
enable_s3_backup = EXCLUDED.enable_s3_backup,
|
||||
s3_endpoint = EXCLUDED.s3_endpoint,
|
||||
s3_region = EXCLUDED.s3_region,
|
||||
s3_bucket = EXCLUDED.s3_bucket,
|
||||
s3_prefix = EXCLUDED.s3_prefix,
|
||||
s3_access_key_id = EXCLUDED.s3_access_key_id,
|
||||
s3_secret_access_key_enc = EXCLUDED.s3_secret_access_key_enc,
|
||||
disk_pause_threshold_pct = EXCLUDED.disk_pause_threshold_pct,
|
||||
enable_webhook_alerts = EXCLUDED.enable_webhook_alerts,
|
||||
webhook_url = EXCLUDED.webhook_url,
|
||||
webhook_secret_enc = EXCLUDED.webhook_secret_enc,
|
||||
webhook_cooldown_sec = EXCLUDED.webhook_cooldown_sec,
|
||||
enable_prometheus_metrics = EXCLUDED.enable_prometheus_metrics,
|
||||
prometheus_metrics_token_enc = EXCLUDED.prometheus_metrics_token_enc,
|
||||
updated_at = NOW()`,
|
||||
[
|
||||
settings.serverPort,
|
||||
@@ -223,6 +510,21 @@ export class SettingsManager {
|
||||
settings.jwtSecret,
|
||||
settings.enableIdempotency,
|
||||
settings.autoPruneDays,
|
||||
settings.compressAfterDays,
|
||||
settings.enableS3Backup,
|
||||
settings.s3Endpoint,
|
||||
settings.s3Region,
|
||||
settings.s3Bucket,
|
||||
settings.s3Prefix,
|
||||
settings.s3AccessKeyId,
|
||||
settings.s3SecretAccessKeyEnc,
|
||||
settings.diskPauseThresholdPct,
|
||||
settings.enableWebhookAlerts,
|
||||
settings.webhookUrl,
|
||||
settings.webhookSecretEnc,
|
||||
settings.webhookCooldownSec,
|
||||
settings.enablePrometheusMetrics,
|
||||
settings.prometheusMetricsTokenEnc,
|
||||
],
|
||||
);
|
||||
} catch (err: unknown) {
|
||||
@@ -250,37 +552,49 @@ export class SettingsManager {
|
||||
return { ...current };
|
||||
}
|
||||
|
||||
public static getPublicSettings(): Omit<AdminSettings, "jwtSecret"> {
|
||||
const settings = this.getSettings();
|
||||
public static getPublicSettings(): PublicAdminSettings {
|
||||
return toPublicSettings(this.getSettings());
|
||||
}
|
||||
|
||||
public static async getPublicSettingsAsync(): Promise<PublicAdminSettings> {
|
||||
const settings = await this.getSettingsAsync();
|
||||
return toPublicSettings(settings);
|
||||
}
|
||||
|
||||
public static async getDecryptedS3ConfigAsync(): Promise<S3ArchiveConfig> {
|
||||
const settings = await this.getSettingsAsync();
|
||||
return {
|
||||
serverPort: settings.serverPort,
|
||||
webPort: settings.webPort,
|
||||
maxEventsPerSec: settings.maxEventsPerSec,
|
||||
maxPayloadSizeKb: settings.maxPayloadSizeKb,
|
||||
batchFlushThreshold: settings.batchFlushThreshold,
|
||||
batchFlushIntervalMs: settings.batchFlushIntervalMs,
|
||||
fileRotationMb: settings.fileRotationMb,
|
||||
enableIdempotency: settings.enableIdempotency,
|
||||
autoPruneDays: settings.autoPruneDays,
|
||||
updatedAt: settings.updatedAt,
|
||||
endpoint: settings.s3Endpoint,
|
||||
region: settings.s3Region,
|
||||
bucket: settings.s3Bucket,
|
||||
prefix: settings.s3Prefix,
|
||||
accessKeyId: settings.s3AccessKeyId,
|
||||
secretAccessKey: decryptSecret(settings.s3SecretAccessKeyEnc),
|
||||
};
|
||||
}
|
||||
|
||||
public static async getPublicSettingsAsync(): Promise<
|
||||
Omit<AdminSettings, "jwtSecret">
|
||||
> {
|
||||
public static async getDecryptedWebhookConfigAsync(): Promise<WebhookRuntimeConfig> {
|
||||
const settings = await this.getSettingsAsync();
|
||||
return {
|
||||
serverPort: settings.serverPort,
|
||||
webPort: settings.webPort,
|
||||
maxEventsPerSec: settings.maxEventsPerSec,
|
||||
maxPayloadSizeKb: settings.maxPayloadSizeKb,
|
||||
batchFlushThreshold: settings.batchFlushThreshold,
|
||||
batchFlushIntervalMs: settings.batchFlushIntervalMs,
|
||||
fileRotationMb: settings.fileRotationMb,
|
||||
enableIdempotency: settings.enableIdempotency,
|
||||
autoPruneDays: settings.autoPruneDays,
|
||||
updatedAt: settings.updatedAt,
|
||||
enabled: settings.enableWebhookAlerts,
|
||||
webhookUrl: settings.webhookUrl,
|
||||
webhookSecret: settings.webhookSecretEnc
|
||||
? decryptSecret(settings.webhookSecretEnc)
|
||||
: "",
|
||||
cooldownSec: settings.webhookCooldownSec,
|
||||
};
|
||||
}
|
||||
|
||||
public static async getDecryptedPrometheusConfigAsync(): Promise<{
|
||||
enabled: boolean;
|
||||
token: string;
|
||||
}> {
|
||||
const settings = await this.getSettingsAsync();
|
||||
return {
|
||||
enabled: settings.enablePrometheusMetrics,
|
||||
token: settings.prometheusMetricsTokenEnc
|
||||
? decryptSecret(settings.prometheusMetricsTokenEnc)
|
||||
: "",
|
||||
};
|
||||
}
|
||||
|
||||
@@ -326,6 +640,12 @@ export class SettingsManager {
|
||||
fileRotationMb: settings.fileRotationMb,
|
||||
enableIdempotency: settings.enableIdempotency,
|
||||
autoPruneDays: settings.autoPruneDays,
|
||||
compressAfterDays: settings.compressAfterDays,
|
||||
enableS3Backup: settings.enableS3Backup,
|
||||
diskPauseThresholdPct: settings.diskPauseThresholdPct,
|
||||
enableWebhookAlerts: settings.enableWebhookAlerts,
|
||||
webhookUrl: settings.webhookUrl,
|
||||
webhookCooldownSec: settings.webhookCooldownSec,
|
||||
...(jwtSecretChanged ? { jwtSecret: settings.jwtSecret } : {}),
|
||||
}),
|
||||
cache: "no-store",
|
||||
@@ -355,7 +675,10 @@ export class SettingsManager {
|
||||
globals.__dataStackLastSettingsBridgeSync = result;
|
||||
return result;
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : "Port 4000 settings-sync unreachable";
|
||||
const message =
|
||||
err instanceof Error
|
||||
? err.message
|
||||
: "Port 4000 settings-sync unreachable";
|
||||
if (
|
||||
process.env.NODE_ENV === "production" ||
|
||||
Boolean(process.env.INGESTION_INTERNAL_URL)
|
||||
@@ -374,18 +697,79 @@ export class SettingsManager {
|
||||
}
|
||||
}
|
||||
|
||||
public static updateSettings(updates: Partial<AdminSettings>): AdminSettings {
|
||||
private static mergeSettingsUpdate(
|
||||
current: AdminSettings,
|
||||
updates: AdminSettingsUpdateInput,
|
||||
): AdminSettings {
|
||||
const {
|
||||
s3SecretAccessKey,
|
||||
clearS3SecretKey,
|
||||
webhookSecret,
|
||||
clearWebhookSecret,
|
||||
prometheusMetricsToken,
|
||||
clearPrometheusToken,
|
||||
...rest
|
||||
} = updates;
|
||||
let nextEncryptedSecret = current.s3SecretAccessKeyEnc;
|
||||
if (clearS3SecretKey) {
|
||||
nextEncryptedSecret = "";
|
||||
} else if (
|
||||
typeof s3SecretAccessKey === "string" &&
|
||||
s3SecretAccessKey.trim().length > 0
|
||||
) {
|
||||
nextEncryptedSecret = encryptSecret(s3SecretAccessKey.trim());
|
||||
} else if (rest.s3SecretAccessKeyEnc !== undefined) {
|
||||
nextEncryptedSecret = rest.s3SecretAccessKeyEnc;
|
||||
}
|
||||
|
||||
let nextWebhookSecretEnc = current.webhookSecretEnc;
|
||||
if (clearWebhookSecret) {
|
||||
nextWebhookSecretEnc = "";
|
||||
} else if (
|
||||
typeof webhookSecret === "string" &&
|
||||
webhookSecret.trim().length > 0
|
||||
) {
|
||||
nextWebhookSecretEnc = encryptSecret(webhookSecret.trim());
|
||||
} else if (rest.webhookSecretEnc !== undefined) {
|
||||
nextWebhookSecretEnc = rest.webhookSecretEnc;
|
||||
}
|
||||
|
||||
let nextPrometheusTokenEnc = current.prometheusMetricsTokenEnc;
|
||||
if (clearPrometheusToken) {
|
||||
nextPrometheusTokenEnc = "";
|
||||
} else if (
|
||||
typeof prometheusMetricsToken === "string" &&
|
||||
prometheusMetricsToken.trim().length > 0
|
||||
) {
|
||||
nextPrometheusTokenEnc = encryptSecret(prometheusMetricsToken.trim());
|
||||
} else if (rest.prometheusMetricsTokenEnc !== undefined) {
|
||||
nextPrometheusTokenEnc = rest.prometheusMetricsTokenEnc;
|
||||
}
|
||||
|
||||
return {
|
||||
...current,
|
||||
...rest,
|
||||
s3SecretAccessKeyEnc: nextEncryptedSecret,
|
||||
webhookSecretEnc: nextWebhookSecretEnc,
|
||||
prometheusMetricsTokenEnc: nextPrometheusTokenEnc,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
public static updateSettings(
|
||||
updates: AdminSettingsUpdateInput,
|
||||
): AdminSettings {
|
||||
const current = getSettingsState();
|
||||
const previousJwtSecret = current.jwtSecret;
|
||||
const jwtSecretChanged =
|
||||
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
|
||||
const nextSettings: AdminSettings = {
|
||||
...current,
|
||||
...updates,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
const nextSettings = this.mergeSettingsUpdate(current, updates);
|
||||
globals.__dataStackAdminSettings = nextSettings;
|
||||
this.applyRuntimeSettings(nextSettings, updates.autoPruneDays !== undefined);
|
||||
this.applyRuntimeSettings(
|
||||
nextSettings,
|
||||
updates.autoPruneDays !== undefined,
|
||||
updates.compressAfterDays !== undefined,
|
||||
);
|
||||
if (jwtSecretChanged) {
|
||||
SDKManager.resignAllClientTokens();
|
||||
}
|
||||
@@ -403,17 +787,13 @@ export class SettingsManager {
|
||||
}
|
||||
|
||||
public static async updateSettingsAsync(
|
||||
updates: Partial<AdminSettings>,
|
||||
updates: AdminSettingsUpdateInput,
|
||||
): Promise<AdminSettings> {
|
||||
const current = await this.getSettingsAsync();
|
||||
const previousJwtSecret = current.jwtSecret;
|
||||
const jwtSecretChanged =
|
||||
Boolean(updates.jwtSecret) && updates.jwtSecret !== current.jwtSecret;
|
||||
const nextSettings: AdminSettings = {
|
||||
...current,
|
||||
...updates,
|
||||
updatedAt: new Date().toISOString(),
|
||||
};
|
||||
const nextSettings = this.mergeSettingsUpdate(current, updates);
|
||||
|
||||
if (
|
||||
jwtSecretChanged &&
|
||||
@@ -450,11 +830,15 @@ export class SettingsManager {
|
||||
try {
|
||||
await this.persistSettingsToDb(nextSettings);
|
||||
globals.__dataStackAdminSettings = nextSettings;
|
||||
this.applyRuntimeSettings(nextSettings, updates.autoPruneDays !== undefined);
|
||||
this.applyRuntimeSettings(
|
||||
nextSettings,
|
||||
updates.autoPruneDays !== undefined,
|
||||
updates.compressAfterDays !== undefined,
|
||||
);
|
||||
await SDKManager.resignAllClientTokensAsync();
|
||||
} catch (err) {
|
||||
globals.__dataStackAdminSettings = current;
|
||||
this.applyRuntimeSettings(current, false);
|
||||
this.applyRuntimeSettings(current, false, false);
|
||||
await this.persistSettingsToDb(current).catch(() => undefined);
|
||||
if (preSyncStatus.synced) {
|
||||
await this.notifyIngestionSettingsSync(
|
||||
@@ -482,7 +866,11 @@ export class SettingsManager {
|
||||
|
||||
await this.persistSettingsToDb(nextSettings);
|
||||
globals.__dataStackAdminSettings = nextSettings;
|
||||
this.applyRuntimeSettings(nextSettings, updates.autoPruneDays !== undefined);
|
||||
this.applyRuntimeSettings(
|
||||
nextSettings,
|
||||
updates.autoPruneDays !== undefined,
|
||||
updates.compressAfterDays !== undefined,
|
||||
);
|
||||
await this.notifyIngestionSettingsSync(
|
||||
nextSettings,
|
||||
previousJwtSecret,
|
||||
|
||||
Reference in new issue
Block a user