feat: implement R4 production hardening

- Add configurable ingestion batch item limits
- Improve client serialization and sanitization
- Add graceful handling for corrupt GZIP partitions in data API
- Expand admin session token scope
- Update project documentation for R4 status
This commit is contained in:
Team Dark Devil committed 2026-11-21 22:15:32 +06:00
1 parent 8132087085
commit ae9b554189
59 files changed
+16879 -1074

No files matched your search

+98 -12
View File
@@ -1,5 +1,7 @@
import crypto from "node:crypto";
import fs from "node:fs";
import { Readable } from "node:stream";
import { validateOutboundUrlAsync } from "@/core/Security/ssrfGuard";
export interface S3ArchiveConfig {
endpoint: string;
@@ -17,7 +19,12 @@ export interface S3OperationResult {
bucket?: string;
key?: string;
etag?: string;
code?: "S3_INVALID_CONFIG" | "S3_CONNECTION_FAILED" | "S3_ARCHIVE_FAILED";
code?:
| "S3_INVALID_CONFIG"
| "S3_CONNECTION_FAILED"
| "S3_ARCHIVE_FAILED"
| "SSRF_BLOCKED_HOST"
| "INVALID_URL";
error?: string;
}
@@ -25,6 +32,27 @@ function sha256Hex(data: Buffer | string): string {
return crypto.createHash("sha256").update(data).digest("hex");
}
async function computeFileSha256AndSize(
filePath: string,
): Promise<{ sha256Hex: string; sizeBytes: number }> {
const stat = await fs.promises.stat(filePath);
const hash = crypto.createHash("sha256");
const readStream = fs.createReadStream(filePath, { highWaterMark: 64 * 1024 });
try {
for await (const chunk of readStream) {
hash.update(chunk as Buffer | string);
}
return {
sha256Hex: hash.digest("hex"),
sizeBytes: stat.size,
};
} finally {
if (!readStream.destroyed) {
readStream.destroy();
}
}
}
function hmacSha256(key: Buffer | string, data: string): Buffer {
return crypto.createHmac("sha256", key).update(data, "utf8").digest();
}
@@ -204,6 +232,16 @@ export async function testS3Connection(
config.endpoint,
config.region,
);
const ssrfCheck = await validateOutboundUrlAsync(url.toString(), {
fieldName: "S3 Endpoint URL",
});
if (!ssrfCheck.allowed) {
return {
ok: false,
code: ssrfCheck.code || "SSRF_BLOCKED_HOST",
error: ssrfCheck.error || "S3 Endpoint URL is blocked.",
};
}
const signed = signS3Request({
method: "GET",
endpointUrl: url,
@@ -219,9 +257,21 @@ export async function testS3Connection(
method: "GET",
headers: signed.headers,
cache: "no-store",
redirect: "manual",
signal: AbortSignal.timeout(8000),
});
if (response.status >= 300 && response.status < 400) {
return {
ok: false,
status: response.status,
endpoint: url.origin,
bucket: config.bucket.trim(),
code: "SSRF_BLOCKED_HOST",
error: `S3 endpoint returned an unsupported HTTP ${response.status} redirect.`,
};
}
if (!response.ok) {
const bodyPreview = await response.text().catch(() => "");
return {
@@ -272,8 +322,19 @@ export async function uploadFileToS3(
config.endpoint,
config.region,
);
const bodyBuffer = await fs.promises.readFile(localFilePath);
const payloadHashHex = sha256Hex(bodyBuffer);
const ssrfCheck = await validateOutboundUrlAsync(url.toString(), {
fieldName: "S3 Endpoint URL",
});
if (!ssrfCheck.allowed) {
return {
ok: false,
code: ssrfCheck.code || "SSRF_BLOCKED_HOST",
error: ssrfCheck.error || "S3 Endpoint URL is blocked.",
key: objectKey,
};
}
const { sha256Hex: payloadHashHex, sizeBytes } =
await computeFileSha256AndSize(localFilePath);
const contentType = localFilePath.endsWith(".gz")
? "application/gzip"
: "text/csv; charset=utf-8";
@@ -290,16 +351,41 @@ export async function uploadFileToS3(
contentType,
});
const response = await fetch(signed.targetUrl, {
method: "PUT",
headers: {
...signed.headers,
"content-length": String(bodyBuffer.byteLength),
},
body: new Uint8Array(bodyBuffer),
cache: "no-store",
signal: AbortSignal.timeout(30000),
const uploadStream = fs.createReadStream(localFilePath, {
highWaterMark: 64 * 1024,
});
let response: Response;
try {
const requestInit: RequestInit & { duplex: "half" } = {
method: "PUT",
headers: {
...signed.headers,
"content-length": String(sizeBytes),
},
body: Readable.toWeb(uploadStream) as unknown as BodyInit,
duplex: "half",
cache: "no-store",
redirect: "manual",
signal: AbortSignal.timeout(30000),
};
response = await fetch(signed.targetUrl, requestInit);
} finally {
if (!uploadStream.destroyed) {
uploadStream.destroy();
}
}
if (response.status >= 300 && response.status < 400) {
return {
ok: false,
status: response.status,
endpoint: url.origin,
bucket: config.bucket.trim(),
key: objectKey,
code: "SSRF_BLOCKED_HOST",
error: `S3 upload returned an unsupported HTTP ${response.status} redirect.`,
};
}
if (!response.ok) {
const bodyPreview = await response.text().catch(() => "");