feat: implement R4 production hardening
- Add configurable ingestion batch item limits - Improve client serialization and sanitization - Add graceful handling for corrupt GZIP partitions in data API - Expand admin session token scope - Update project documentation for R4 status
This commit is contained in:
1 parent
8132087085
commit
ae9b554189
59 files changed
+16879
-1074
No files matched your search
@@ -0,0 +1,311 @@
|
||||
import dns from "node:dns/promises";
|
||||
import net from "node:net";
|
||||
|
||||
export class SSRFBlockedError extends Error {
|
||||
public readonly code = "SSRF_BLOCKED_HOST" as const;
|
||||
public readonly hostname: string;
|
||||
|
||||
constructor(message: string, hostname: string) {
|
||||
super(message);
|
||||
this.name = "SSRFBlockedError";
|
||||
this.hostname = hostname;
|
||||
}
|
||||
}
|
||||
|
||||
export interface OutboundUrlValidationOptions {
|
||||
allowPrivate?: boolean;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
dnsLookup?: (
|
||||
hostname: string,
|
||||
) => Promise<Array<{ address: string; family: number }>>;
|
||||
fieldName?: string;
|
||||
}
|
||||
|
||||
export interface OutboundUrlValidationResult {
|
||||
allowed: boolean;
|
||||
url?: URL;
|
||||
code?: "SSRF_BLOCKED_HOST" | "INVALID_URL";
|
||||
error?: string;
|
||||
hostname?: string;
|
||||
}
|
||||
|
||||
const ALWAYS_BLOCKED_HOSTNAMES = new Set([
|
||||
"metadata.google.internal",
|
||||
"metadata.goog",
|
||||
"metadata",
|
||||
"instance-data",
|
||||
"instance-data.ec2.internal",
|
||||
]);
|
||||
|
||||
const PRIVATE_HOSTNAMES = new Set([
|
||||
"localhost",
|
||||
"localhost.localdomain",
|
||||
"ip6-localhost",
|
||||
"ip6-loopback",
|
||||
]);
|
||||
|
||||
function buildAlwaysBlockedBlockList(): net.BlockList {
|
||||
const list = new net.BlockList();
|
||||
// Unspecified / zero network
|
||||
list.addSubnet("0.0.0.0", 8, "ipv4");
|
||||
// Link-local & cloud metadata (AWS/GCP/Azure/Oracle 169.254.169.254)
|
||||
list.addSubnet("169.254.0.0", 16, "ipv4");
|
||||
// Multicast & reserved
|
||||
list.addSubnet("224.0.0.0", 4, "ipv4");
|
||||
list.addSubnet("240.0.0.0", 4, "ipv4");
|
||||
// IPv6 unspecified, link-local, multicast
|
||||
list.addAddress("::", "ipv6");
|
||||
list.addSubnet("fe80::", 10, "ipv6");
|
||||
list.addSubnet("ff00::", 8, "ipv6");
|
||||
return list;
|
||||
}
|
||||
|
||||
function buildPrivateBlockList(): net.BlockList {
|
||||
const list = new net.BlockList();
|
||||
// Loopback
|
||||
list.addSubnet("127.0.0.0", 8, "ipv4");
|
||||
// RFC 1918 private networks
|
||||
list.addSubnet("10.0.0.0", 8, "ipv4");
|
||||
list.addSubnet("172.16.0.0", 12, "ipv4");
|
||||
list.addSubnet("192.168.0.0", 16, "ipv4");
|
||||
// Carrier-grade NAT (RFC 6598)
|
||||
list.addSubnet("100.64.0.0", 10, "ipv4");
|
||||
// Benchmarking / documentation
|
||||
list.addSubnet("198.18.0.0", 15, "ipv4");
|
||||
// IPv6 loopback & Unique Local Addresses (ULA)
|
||||
list.addAddress("::1", "ipv6");
|
||||
list.addSubnet("fc00::", 7, "ipv6");
|
||||
return list;
|
||||
}
|
||||
|
||||
const ALWAYS_BLOCKED_IPS = buildAlwaysBlockedBlockList();
|
||||
const PRIVATE_IPS = buildPrivateBlockList();
|
||||
|
||||
export function isPrivateOutboundUrlAllowed(
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): boolean {
|
||||
const raw = String(env.DATASTACK_ALLOW_PRIVATE_URLS || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
return raw === "true" || raw === "1" || raw === "yes";
|
||||
}
|
||||
|
||||
function stripBracketsAndZone(rawHost: string): string {
|
||||
let host = String(rawHost || "").trim().toLowerCase();
|
||||
if (host.startsWith("[") && host.endsWith("]")) {
|
||||
host = host.slice(1, -1);
|
||||
}
|
||||
const zoneIdx = host.indexOf("%");
|
||||
if (zoneIdx >= 0) {
|
||||
host = host.slice(0, zoneIdx);
|
||||
}
|
||||
// Remove trailing FQDN dot (e.g. "localhost.")
|
||||
return host.replace(/\.+$/, "");
|
||||
}
|
||||
|
||||
function normalizeIpForCheck(rawIp: string): { ip: string; family: 4 | 6 } | null {
|
||||
const clean = stripBracketsAndZone(rawIp);
|
||||
if (!clean) return null;
|
||||
|
||||
// Handle IPv4-mapped IPv6 addresses (e.g. ::ffff:127.0.0.1 or ::ffff:7f00:1)
|
||||
if (clean.startsWith("::ffff:")) {
|
||||
const mapped = clean.slice("::ffff:".length);
|
||||
if (net.isIP(mapped) === 4) {
|
||||
return { ip: mapped, family: 4 };
|
||||
}
|
||||
const hexParts = mapped.split(":");
|
||||
if (hexParts.length === 2) {
|
||||
const hi = Number.parseInt(hexParts[0], 16);
|
||||
const lo = Number.parseInt(hexParts[1], 16);
|
||||
if (
|
||||
Number.isInteger(hi) &&
|
||||
Number.isInteger(lo) &&
|
||||
hi >= 0 &&
|
||||
hi <= 0xffff &&
|
||||
lo >= 0 &&
|
||||
lo <= 0xffff
|
||||
) {
|
||||
const ipv4 = `${(hi >> 8) & 0xff}.${hi & 0xff}.${(lo >> 8) & 0xff}.${lo & 0xff}`;
|
||||
if (net.isIP(ipv4) === 4) {
|
||||
return { ip: ipv4, family: 4 };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const family = net.isIP(clean);
|
||||
if (family === 4 || family === 6) {
|
||||
return { ip: clean, family };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function checkIpAddressAllowed(
|
||||
rawIp: string,
|
||||
allowPrivate: boolean,
|
||||
): { allowed: boolean; reason?: string } {
|
||||
const parsed = normalizeIpForCheck(rawIp);
|
||||
if (!parsed) {
|
||||
return { allowed: false, reason: `Invalid IP address '${rawIp}'.` };
|
||||
}
|
||||
const ipType = parsed.family === 4 ? "ipv4" : "ipv6";
|
||||
if (ALWAYS_BLOCKED_IPS.check(parsed.ip, ipType)) {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: `Address '${parsed.ip}' is in a restricted link-local/metadata/reserved network range.`,
|
||||
};
|
||||
}
|
||||
if (!allowPrivate && PRIVATE_IPS.check(parsed.ip, ipType)) {
|
||||
return {
|
||||
allowed: false,
|
||||
reason: `Address '${parsed.ip}' is a private or loopback network address (set DATASTACK_ALLOW_PRIVATE_URLS=true only if intentionally targeting a private host).`,
|
||||
};
|
||||
}
|
||||
return { allowed: true };
|
||||
}
|
||||
|
||||
export function validateOutboundUrlSync(
|
||||
rawUrl: string,
|
||||
options: OutboundUrlValidationOptions = {},
|
||||
): OutboundUrlValidationResult {
|
||||
const fieldName = options.fieldName || "URL";
|
||||
const allowPrivate =
|
||||
options.allowPrivate !== undefined
|
||||
? options.allowPrivate
|
||||
: isPrivateOutboundUrlAllowed(options.env);
|
||||
|
||||
const trimmed = String(rawUrl || "").trim();
|
||||
if (!trimmed) {
|
||||
return {
|
||||
allowed: false,
|
||||
code: "INVALID_URL",
|
||||
error: `${fieldName} is required.`,
|
||||
};
|
||||
}
|
||||
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(trimmed);
|
||||
} catch {
|
||||
return {
|
||||
allowed: false,
|
||||
code: "INVALID_URL",
|
||||
error: `${fieldName} must be a valid http:// or https:// URL.`,
|
||||
};
|
||||
}
|
||||
|
||||
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
||||
return {
|
||||
allowed: false,
|
||||
code: "SSRF_BLOCKED_HOST",
|
||||
hostname: parsed.hostname,
|
||||
error: `${fieldName} must use http:// or https://.`,
|
||||
};
|
||||
}
|
||||
|
||||
if (parsed.username || parsed.password) {
|
||||
return {
|
||||
allowed: false,
|
||||
code: "SSRF_BLOCKED_HOST",
|
||||
hostname: parsed.hostname,
|
||||
error: `${fieldName} must not embed URL credentials.`,
|
||||
};
|
||||
}
|
||||
|
||||
const host = stripBracketsAndZone(parsed.hostname);
|
||||
if (!host) {
|
||||
return {
|
||||
allowed: false,
|
||||
code: "INVALID_URL",
|
||||
error: `${fieldName} must include a valid hostname.`,
|
||||
};
|
||||
}
|
||||
|
||||
if (ALWAYS_BLOCKED_HOSTNAMES.has(host)) {
|
||||
return {
|
||||
allowed: false,
|
||||
code: "SSRF_BLOCKED_HOST",
|
||||
hostname: host,
|
||||
error: `${fieldName} target '${host}' is blocked (cloud metadata/internal endpoint).`,
|
||||
};
|
||||
}
|
||||
|
||||
// Literal IP check (covers IPv4, IPv6, and IPv4-mapped IPv6)
|
||||
const ipParsed = normalizeIpForCheck(host);
|
||||
if (ipParsed) {
|
||||
const ipCheck = checkIpAddressAllowed(host, allowPrivate);
|
||||
if (!ipCheck.allowed) {
|
||||
return {
|
||||
allowed: false,
|
||||
code: "SSRF_BLOCKED_HOST",
|
||||
hostname: host,
|
||||
error: `${fieldName} blocked: ${ipCheck.reason}`,
|
||||
};
|
||||
}
|
||||
return { allowed: true, url: parsed, hostname: host };
|
||||
}
|
||||
|
||||
// Non-IP hostname checks
|
||||
if (!allowPrivate) {
|
||||
if (
|
||||
PRIVATE_HOSTNAMES.has(host) ||
|
||||
host.endsWith(".localhost") ||
|
||||
host.endsWith(".local") ||
|
||||
host.endsWith(".internal") ||
|
||||
!host.includes(".")
|
||||
) {
|
||||
return {
|
||||
allowed: false,
|
||||
code: "SSRF_BLOCKED_HOST",
|
||||
hostname: host,
|
||||
error: `${fieldName} target '${host}' is a private/internal hostname (set DATASTACK_ALLOW_PRIVATE_URLS=true only for trusted private hosts).`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return { allowed: true, url: parsed, hostname: host };
|
||||
}
|
||||
|
||||
export async function validateOutboundUrlAsync(
|
||||
rawUrl: string,
|
||||
options: OutboundUrlValidationOptions = {},
|
||||
): Promise<OutboundUrlValidationResult> {
|
||||
const syncResult = validateOutboundUrlSync(rawUrl, options);
|
||||
if (!syncResult.allowed || !syncResult.url || !syncResult.hostname) {
|
||||
return syncResult;
|
||||
}
|
||||
|
||||
const host = syncResult.hostname;
|
||||
const fieldName = options.fieldName || "URL";
|
||||
const allowPrivate =
|
||||
options.allowPrivate !== undefined
|
||||
? options.allowPrivate
|
||||
: isPrivateOutboundUrlAllowed(options.env);
|
||||
|
||||
// Skip DNS resolution if already validated as a literal IP
|
||||
if (normalizeIpForCheck(host)) {
|
||||
return syncResult;
|
||||
}
|
||||
|
||||
try {
|
||||
const lookupFn =
|
||||
options.dnsLookup ||
|
||||
((h: string) => dns.lookup(h, { all: true, verbatim: true }));
|
||||
const records = await lookupFn(host);
|
||||
for (const rec of records) {
|
||||
const check = checkIpAddressAllowed(rec.address, allowPrivate);
|
||||
if (!check.allowed) {
|
||||
return {
|
||||
allowed: false,
|
||||
code: "SSRF_BLOCKED_HOST",
|
||||
hostname: host,
|
||||
error: `${fieldName} hostname '${host}' resolved to a blocked address: ${check.reason}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// DNS resolution failure (e.g. ENOTFOUND/EAI_AGAIN) will be handled cleanly by the caller or fetch()
|
||||
}
|
||||
|
||||
return syncResult;
|
||||
}
|
||||
@@ -386,6 +386,14 @@ export class CSVStorageManager {
|
||||
if (!gunzip.destroyed) {
|
||||
gunzip.destroy(err);
|
||||
}
|
||||
if (!fileStream.destroyed) {
|
||||
fileStream.destroy();
|
||||
}
|
||||
});
|
||||
gunzip.on('error', () => {
|
||||
if (!fileStream.destroyed) {
|
||||
fileStream.destroy();
|
||||
}
|
||||
});
|
||||
gunzip.on('close', () => {
|
||||
if (!fileStream.destroyed) {
|
||||
@@ -396,16 +404,21 @@ export class CSVStorageManager {
|
||||
return gunzip;
|
||||
}
|
||||
|
||||
private static async countCsvDataRowsStreamAsync(
|
||||
public static async countCsvDataRowsStreamAsync(
|
||||
filePath: string,
|
||||
): Promise<number> {
|
||||
const counter = this.createChunkRowCounter();
|
||||
return new Promise<number>((resolve, reject) => {
|
||||
const stream = this.openPartitionTextStream(filePath);
|
||||
stream.on('data', (chunk) => counter.pushChunk(chunk));
|
||||
stream.on('end', () => resolve(counter.finish()));
|
||||
stream.on('error', (err) => reject(err));
|
||||
});
|
||||
const stream = this.openPartitionTextStream(filePath);
|
||||
try {
|
||||
for await (const chunk of stream) {
|
||||
counter.pushChunk(chunk as Buffer | string);
|
||||
}
|
||||
return counter.finish();
|
||||
} finally {
|
||||
if (!stream.destroyed) {
|
||||
stream.destroy();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
+98
-12
@@ -1,5 +1,7 @@
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import { Readable } from "node:stream";
|
||||
import { validateOutboundUrlAsync } from "@/core/Security/ssrfGuard";
|
||||
|
||||
export interface S3ArchiveConfig {
|
||||
endpoint: string;
|
||||
@@ -17,7 +19,12 @@ export interface S3OperationResult {
|
||||
bucket?: string;
|
||||
key?: string;
|
||||
etag?: string;
|
||||
code?: "S3_INVALID_CONFIG" | "S3_CONNECTION_FAILED" | "S3_ARCHIVE_FAILED";
|
||||
code?:
|
||||
| "S3_INVALID_CONFIG"
|
||||
| "S3_CONNECTION_FAILED"
|
||||
| "S3_ARCHIVE_FAILED"
|
||||
| "SSRF_BLOCKED_HOST"
|
||||
| "INVALID_URL";
|
||||
error?: string;
|
||||
}
|
||||
|
||||
@@ -25,6 +32,27 @@ function sha256Hex(data: Buffer | string): string {
|
||||
return crypto.createHash("sha256").update(data).digest("hex");
|
||||
}
|
||||
|
||||
async function computeFileSha256AndSize(
|
||||
filePath: string,
|
||||
): Promise<{ sha256Hex: string; sizeBytes: number }> {
|
||||
const stat = await fs.promises.stat(filePath);
|
||||
const hash = crypto.createHash("sha256");
|
||||
const readStream = fs.createReadStream(filePath, { highWaterMark: 64 * 1024 });
|
||||
try {
|
||||
for await (const chunk of readStream) {
|
||||
hash.update(chunk as Buffer | string);
|
||||
}
|
||||
return {
|
||||
sha256Hex: hash.digest("hex"),
|
||||
sizeBytes: stat.size,
|
||||
};
|
||||
} finally {
|
||||
if (!readStream.destroyed) {
|
||||
readStream.destroy();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function hmacSha256(key: Buffer | string, data: string): Buffer {
|
||||
return crypto.createHmac("sha256", key).update(data, "utf8").digest();
|
||||
}
|
||||
@@ -204,6 +232,16 @@ export async function testS3Connection(
|
||||
config.endpoint,
|
||||
config.region,
|
||||
);
|
||||
const ssrfCheck = await validateOutboundUrlAsync(url.toString(), {
|
||||
fieldName: "S3 Endpoint URL",
|
||||
});
|
||||
if (!ssrfCheck.allowed) {
|
||||
return {
|
||||
ok: false,
|
||||
code: ssrfCheck.code || "SSRF_BLOCKED_HOST",
|
||||
error: ssrfCheck.error || "S3 Endpoint URL is blocked.",
|
||||
};
|
||||
}
|
||||
const signed = signS3Request({
|
||||
method: "GET",
|
||||
endpointUrl: url,
|
||||
@@ -219,9 +257,21 @@ export async function testS3Connection(
|
||||
method: "GET",
|
||||
headers: signed.headers,
|
||||
cache: "no-store",
|
||||
redirect: "manual",
|
||||
signal: AbortSignal.timeout(8000),
|
||||
});
|
||||
|
||||
if (response.status >= 300 && response.status < 400) {
|
||||
return {
|
||||
ok: false,
|
||||
status: response.status,
|
||||
endpoint: url.origin,
|
||||
bucket: config.bucket.trim(),
|
||||
code: "SSRF_BLOCKED_HOST",
|
||||
error: `S3 endpoint returned an unsupported HTTP ${response.status} redirect.`,
|
||||
};
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const bodyPreview = await response.text().catch(() => "");
|
||||
return {
|
||||
@@ -272,8 +322,19 @@ export async function uploadFileToS3(
|
||||
config.endpoint,
|
||||
config.region,
|
||||
);
|
||||
const bodyBuffer = await fs.promises.readFile(localFilePath);
|
||||
const payloadHashHex = sha256Hex(bodyBuffer);
|
||||
const ssrfCheck = await validateOutboundUrlAsync(url.toString(), {
|
||||
fieldName: "S3 Endpoint URL",
|
||||
});
|
||||
if (!ssrfCheck.allowed) {
|
||||
return {
|
||||
ok: false,
|
||||
code: ssrfCheck.code || "SSRF_BLOCKED_HOST",
|
||||
error: ssrfCheck.error || "S3 Endpoint URL is blocked.",
|
||||
key: objectKey,
|
||||
};
|
||||
}
|
||||
const { sha256Hex: payloadHashHex, sizeBytes } =
|
||||
await computeFileSha256AndSize(localFilePath);
|
||||
const contentType = localFilePath.endsWith(".gz")
|
||||
? "application/gzip"
|
||||
: "text/csv; charset=utf-8";
|
||||
@@ -290,16 +351,41 @@ export async function uploadFileToS3(
|
||||
contentType,
|
||||
});
|
||||
|
||||
const response = await fetch(signed.targetUrl, {
|
||||
method: "PUT",
|
||||
headers: {
|
||||
...signed.headers,
|
||||
"content-length": String(bodyBuffer.byteLength),
|
||||
},
|
||||
body: new Uint8Array(bodyBuffer),
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(30000),
|
||||
const uploadStream = fs.createReadStream(localFilePath, {
|
||||
highWaterMark: 64 * 1024,
|
||||
});
|
||||
let response: Response;
|
||||
try {
|
||||
const requestInit: RequestInit & { duplex: "half" } = {
|
||||
method: "PUT",
|
||||
headers: {
|
||||
...signed.headers,
|
||||
"content-length": String(sizeBytes),
|
||||
},
|
||||
body: Readable.toWeb(uploadStream) as unknown as BodyInit,
|
||||
duplex: "half",
|
||||
cache: "no-store",
|
||||
redirect: "manual",
|
||||
signal: AbortSignal.timeout(30000),
|
||||
};
|
||||
response = await fetch(signed.targetUrl, requestInit);
|
||||
} finally {
|
||||
if (!uploadStream.destroyed) {
|
||||
uploadStream.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
if (response.status >= 300 && response.status < 400) {
|
||||
return {
|
||||
ok: false,
|
||||
status: response.status,
|
||||
endpoint: url.origin,
|
||||
bucket: config.bucket.trim(),
|
||||
key: objectKey,
|
||||
code: "SSRF_BLOCKED_HOST",
|
||||
error: `S3 upload returned an unsupported HTTP ${response.status} redirect.`,
|
||||
};
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const bodyPreview = await response.text().catch(() => "");
|
||||
|
||||
@@ -309,6 +309,34 @@ export class SettingsManager {
|
||||
return true;
|
||||
}
|
||||
|
||||
public static async bootstrapMaintenanceSchedulerOnBoot(): Promise<{
|
||||
schedulerActive: boolean;
|
||||
hydrated: boolean;
|
||||
error?: string;
|
||||
}> {
|
||||
this.ensureMaintenanceScheduler();
|
||||
try {
|
||||
await this.getSettingsAsync();
|
||||
return {
|
||||
schedulerActive: this.isMaintenanceSchedulerActive(),
|
||||
hydrated:
|
||||
!process.env.DATABASE_URL?.trim() ||
|
||||
globals.__dataStackSettingsHydrated === true,
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.warn(
|
||||
"[DataStack Boot] Maintenance scheduler init deferred/failed:",
|
||||
message,
|
||||
);
|
||||
return {
|
||||
schedulerActive: this.isMaintenanceSchedulerActive(),
|
||||
hydrated: false,
|
||||
error: message,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
public static stopMaintenanceScheduler(): void {
|
||||
if (globals.__dataStackMaintenanceTimer) {
|
||||
clearInterval(globals.__dataStackMaintenanceTimer);
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import crypto from "node:crypto";
|
||||
import { validateOutboundUrlAsync } from "@/core/Security/ssrfGuard";
|
||||
|
||||
export type WebhookAlertEventType =
|
||||
| "DISK_SPACE_LOW"
|
||||
@@ -30,6 +31,7 @@ export interface WebhookDispatchResult {
|
||||
ok: boolean;
|
||||
skipped?: boolean;
|
||||
reason?: string;
|
||||
code?: string;
|
||||
status?: number;
|
||||
error?: string;
|
||||
}
|
||||
@@ -106,21 +108,17 @@ export async function dispatchWebhookAlertAsync(
|
||||
};
|
||||
}
|
||||
|
||||
let parsedUrl: URL;
|
||||
try {
|
||||
parsedUrl = new URL(webhookUrl);
|
||||
if (parsedUrl.protocol !== "http:" && parsedUrl.protocol !== "https:") {
|
||||
return {
|
||||
ok: false,
|
||||
error: "Webhook URL must use http:// or https://.",
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
const ssrfCheck = await validateOutboundUrlAsync(webhookUrl, {
|
||||
fieldName: "Webhook URL",
|
||||
});
|
||||
if (!ssrfCheck.allowed || !ssrfCheck.url) {
|
||||
return {
|
||||
ok: false,
|
||||
error: "Webhook URL is not a valid URL.",
|
||||
code: ssrfCheck.code || "SSRF_BLOCKED_HOST",
|
||||
error: ssrfCheck.error || "Webhook URL is not allowed.",
|
||||
};
|
||||
}
|
||||
const parsedUrl = ssrfCheck.url;
|
||||
|
||||
const dedupeKey = `${payload.event}:${payload.clientId || "global"}`;
|
||||
const cooldownMs = Math.max(0, (Number(config.cooldownSec) || 300) * 1000);
|
||||
@@ -156,9 +154,19 @@ export async function dispatchWebhookAlertAsync(
|
||||
headers,
|
||||
body: bodyStr,
|
||||
cache: "no-store",
|
||||
redirect: "manual",
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
});
|
||||
|
||||
if (res.status >= 300 && res.status < 400) {
|
||||
return {
|
||||
ok: false,
|
||||
code: "SSRF_BLOCKED_HOST",
|
||||
status: res.status,
|
||||
error: `Webhook endpoint attempted an unsupported HTTP ${res.status} redirect.`,
|
||||
};
|
||||
}
|
||||
|
||||
if (!res.ok) {
|
||||
if (!options?.bypassCooldown) {
|
||||
// Keep cooldown entry so failing webhooks don't DDOS a broken upstream endpoint
|
||||
|
||||
@@ -15,6 +15,40 @@ import {
|
||||
} from "./adminSession";
|
||||
import { clearAuditLogsForTests } from "./auditLog";
|
||||
import { DatabaseEngine, ensureCoreSchemaOnPool } from "../Database/database";
|
||||
import { getInternalIngestionHeaders } from "../Security/security";
|
||||
|
||||
async function notifyIngestionAdminSessionSync(payload: {
|
||||
memberId?: string;
|
||||
email?: string;
|
||||
previousEmail?: string;
|
||||
minTokenVersion?: number;
|
||||
disabled?: boolean;
|
||||
deleted?: boolean;
|
||||
}): Promise<void> {
|
||||
const configuredUrl = process.env.INGESTION_INTERNAL_URL?.trim();
|
||||
if (process.env.DATASTACK_DISABLE_LISTEN === "1" && !configuredUrl) {
|
||||
return;
|
||||
}
|
||||
const baseUrl = (configuredUrl || "http://127.0.0.1:4000").replace(
|
||||
/\/+$/,
|
||||
"",
|
||||
);
|
||||
try {
|
||||
await fetch(`${baseUrl}/internal/admin-session-sync`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...getInternalIngestionHeaders(),
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(1200),
|
||||
});
|
||||
} catch {
|
||||
// Best-effort bridge sync when Port 4000 is offline or not running
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
const PASSWORD_HASH_BYTES = 64;
|
||||
const PASSWORD_SALT_BYTES = 16;
|
||||
@@ -1040,6 +1074,13 @@ export async function updateAdminAccount({
|
||||
tokenVersion: nextVersion,
|
||||
createdAt: existing.createdAt,
|
||||
};
|
||||
await notifyIngestionAdminSessionSync({
|
||||
memberId: "owner-1",
|
||||
email: cleanEmail,
|
||||
previousEmail: existing.email,
|
||||
minTokenVersion: nextVersion,
|
||||
disabled: false,
|
||||
});
|
||||
return {
|
||||
ok: true as const,
|
||||
email: cleanEmail,
|
||||
@@ -1085,6 +1126,13 @@ export async function updateAdminAccount({
|
||||
tokenVersion: nextVersion,
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
await notifyIngestionAdminSessionSync({
|
||||
memberId: member.id,
|
||||
email: cleanEmail,
|
||||
previousEmail: member.email,
|
||||
minTokenVersion: nextVersion,
|
||||
disabled: member.status === "disabled",
|
||||
});
|
||||
return {
|
||||
ok: true as const,
|
||||
email: cleanEmail,
|
||||
@@ -1131,6 +1179,12 @@ export async function revokeAdminSessions(
|
||||
member.tokenVersion += 1;
|
||||
member.updatedAt = new Date();
|
||||
teamMap.set(member.id, member);
|
||||
await notifyIngestionAdminSessionSync({
|
||||
memberId: member.id,
|
||||
email: member.email,
|
||||
minTokenVersion: member.tokenVersion,
|
||||
disabled: member.status === "disabled",
|
||||
});
|
||||
return member.tokenVersion;
|
||||
}
|
||||
}
|
||||
@@ -1157,6 +1211,12 @@ export async function revokeAdminSessions(
|
||||
member.tokenVersion += 1;
|
||||
member.updatedAt = new Date();
|
||||
teamMap.set(member.id, member);
|
||||
await notifyIngestionAdminSessionSync({
|
||||
memberId: member.id,
|
||||
email: member.email,
|
||||
minTokenVersion: member.tokenVersion,
|
||||
disabled: member.status === "disabled",
|
||||
});
|
||||
return member.tokenVersion;
|
||||
}
|
||||
}
|
||||
@@ -1181,6 +1241,12 @@ export async function revokeAdminSessions(
|
||||
globals.__dataStackInMemoryAdmin ?? (await getOrInitInMemoryAdmin());
|
||||
existing.tokenVersion += 1;
|
||||
globals.__dataStackInMemoryAdmin = existing;
|
||||
await notifyIngestionAdminSessionSync({
|
||||
memberId: "owner-1",
|
||||
email: existing.email,
|
||||
minTokenVersion: existing.tokenVersion,
|
||||
disabled: false,
|
||||
});
|
||||
return existing.tokenVersion;
|
||||
}
|
||||
|
||||
@@ -1588,6 +1654,12 @@ export async function updateAdminTeamMemberAsync(
|
||||
updatedAt: new Date(),
|
||||
};
|
||||
teamMap.set(id, updated);
|
||||
await notifyIngestionAdminSessionSync({
|
||||
memberId: updated.id,
|
||||
email: updated.email,
|
||||
minTokenVersion: updated.tokenVersion,
|
||||
disabled: updated.status === "disabled",
|
||||
});
|
||||
|
||||
return {
|
||||
id: updated.id,
|
||||
@@ -1654,6 +1726,13 @@ export async function deleteAdminTeamMemberAsync(
|
||||
);
|
||||
}
|
||||
teamMap.delete(id);
|
||||
await notifyIngestionAdminSessionSync({
|
||||
memberId: existing.id,
|
||||
email: existing.email,
|
||||
minTokenVersion: existing.tokenVersion + 1,
|
||||
disabled: true,
|
||||
deleted: true,
|
||||
});
|
||||
return { id: existing.id, email: existing.email };
|
||||
}
|
||||
|
||||
|
||||
+65
-8
@@ -20,6 +20,8 @@ export type AuditActionType =
|
||||
| "STORAGE_FLUSH_TRIGGERED"
|
||||
| "S3_ARCHIVE_TRIGGERED"
|
||||
| "WEBHOOK_TEST_TRIGGERED"
|
||||
| "FILES_MANAGER_MUTATION"
|
||||
| "DATA_EXPORT_DOWNLOADED"
|
||||
| string;
|
||||
|
||||
export interface AuditLogEntry {
|
||||
@@ -49,10 +51,19 @@ export interface ListAuditLogsQuery {
|
||||
search?: string;
|
||||
}
|
||||
|
||||
export interface AuditLogListResult {
|
||||
logs: AuditLogEntry[];
|
||||
storageMode: "postgresql" | "memory";
|
||||
degraded: boolean;
|
||||
code?: "AUDIT_STORAGE_DEGRADED";
|
||||
warning?: string;
|
||||
}
|
||||
|
||||
const MAX_IN_MEMORY_AUDIT_LOGS = 1000;
|
||||
|
||||
type AuditLogGlobals = typeof globalThis & {
|
||||
__dataStackAuditLogs?: AuditLogEntry[];
|
||||
__dataStackAuditLastWriteError?: string;
|
||||
};
|
||||
|
||||
const globals = globalThis as AuditLogGlobals;
|
||||
@@ -149,8 +160,13 @@ export async function recordAdminAuditLogAsync(
|
||||
entry.createdAt,
|
||||
],
|
||||
);
|
||||
} catch {
|
||||
// Never block or crash administrative actions if audit table write experiences transient DB failure
|
||||
globals.__dataStackAuditLastWriteError = undefined;
|
||||
} catch (err: unknown) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
globals.__dataStackAuditLastWriteError = msg;
|
||||
console.warn(
|
||||
`[DataStack Audit] PostgreSQL audit insert failed for action=${entry.action} (buffered in memory only): ${msg}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -161,9 +177,9 @@ export function recordAdminAuditLog(input: RecordAuditLogInput): void {
|
||||
void recordAdminAuditLogAsync(input);
|
||||
}
|
||||
|
||||
export async function listAdminAuditLogsAsync(
|
||||
export async function listAdminAuditLogsWithStatusAsync(
|
||||
query: ListAuditLogsQuery = {},
|
||||
): Promise<AuditLogEntry[]> {
|
||||
): Promise<AuditLogListResult> {
|
||||
const rawLimit =
|
||||
typeof query.limit === "number" && Number.isFinite(query.limit)
|
||||
? Math.floor(query.limit)
|
||||
@@ -174,6 +190,7 @@ export async function listAdminAuditLogsAsync(
|
||||
const searchFilter = query.search?.trim().toLowerCase() || "";
|
||||
|
||||
const pool = DatabaseEngine.getPool();
|
||||
let dbReadError: string | null = null;
|
||||
if (pool) {
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
@@ -227,7 +244,7 @@ export async function listAdminAuditLogsAsync(
|
||||
params,
|
||||
);
|
||||
|
||||
return result.rows.map((row) => {
|
||||
const logs = result.rows.map((row) => {
|
||||
let parsedMeta: Record<string, unknown> = {};
|
||||
try {
|
||||
const parsed = JSON.parse(row.metadata_json || "{}");
|
||||
@@ -251,12 +268,28 @@ export async function listAdminAuditLogsAsync(
|
||||
: new Date(row.created_at).toISOString(),
|
||||
};
|
||||
});
|
||||
} catch {
|
||||
// Fallback to in-memory log buffer if PostgreSQL query fails
|
||||
|
||||
const writeErr = globals.__dataStackAuditLastWriteError;
|
||||
return {
|
||||
logs,
|
||||
storageMode: "postgresql",
|
||||
degraded: Boolean(writeErr),
|
||||
...(writeErr
|
||||
? {
|
||||
code: "AUDIT_STORAGE_DEGRADED" as const,
|
||||
warning: `Recent audit log insert failed in PostgreSQL: ${writeErr}`,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
dbReadError = err instanceof Error ? err.message : String(err);
|
||||
console.warn(
|
||||
`[DataStack Audit] PostgreSQL audit query failed (serving ephemeral in-memory fallback): ${dbReadError}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return getInMemoryAuditLogs()
|
||||
const fallbackLogs = getInMemoryAuditLogs()
|
||||
.filter((entry) => {
|
||||
if (
|
||||
actionFilter &&
|
||||
@@ -278,8 +311,32 @@ export async function listAdminAuditLogsAsync(
|
||||
return true;
|
||||
})
|
||||
.slice(0, limit);
|
||||
|
||||
if (dbReadError) {
|
||||
return {
|
||||
logs: fallbackLogs,
|
||||
storageMode: "memory",
|
||||
degraded: true,
|
||||
code: "AUDIT_STORAGE_DEGRADED",
|
||||
warning: `PostgreSQL audit query failed (${dbReadError}); serving ephemeral in-memory logs.`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
logs: fallbackLogs,
|
||||
storageMode: "memory",
|
||||
degraded: false,
|
||||
};
|
||||
}
|
||||
|
||||
export async function listAdminAuditLogsAsync(
|
||||
query: ListAuditLogsQuery = {},
|
||||
): Promise<AuditLogEntry[]> {
|
||||
const result = await listAdminAuditLogsWithStatusAsync(query);
|
||||
return result.logs;
|
||||
}
|
||||
|
||||
export function clearAuditLogsForTests(): void {
|
||||
globals.__dataStackAuditLogs = [];
|
||||
globals.__dataStackAuditLastWriteError = undefined;
|
||||
}
|
||||
Reference in new issue
Block a user