feat: implement R4 production hardening

- Add configurable ingestion batch item limits
- Improve client serialization and sanitization
- Add graceful handling for corrupt GZIP partitions in data API
- Expand admin session token scope
- Update project documentation for R4 status
This commit is contained in:
Team Dark Devil committed 2026-11-21 22:15:32 +06:00
1 parent 8132087085
commit ae9b554189
59 files changed
+16879 -1074

No files matched your search

+20 -7
View File
@@ -386,6 +386,14 @@ export class CSVStorageManager {
if (!gunzip.destroyed) {
gunzip.destroy(err);
}
if (!fileStream.destroyed) {
fileStream.destroy();
}
});
gunzip.on('error', () => {
if (!fileStream.destroyed) {
fileStream.destroy();
}
});
gunzip.on('close', () => {
if (!fileStream.destroyed) {
@@ -396,16 +404,21 @@ export class CSVStorageManager {
return gunzip;
}
private static async countCsvDataRowsStreamAsync(
public static async countCsvDataRowsStreamAsync(
filePath: string,
): Promise<number> {
const counter = this.createChunkRowCounter();
return new Promise<number>((resolve, reject) => {
const stream = this.openPartitionTextStream(filePath);
stream.on('data', (chunk) => counter.pushChunk(chunk));
stream.on('end', () => resolve(counter.finish()));
stream.on('error', (err) => reject(err));
});
const stream = this.openPartitionTextStream(filePath);
try {
for await (const chunk of stream) {
counter.pushChunk(chunk as Buffer | string);
}
return counter.finish();
} finally {
if (!stream.destroyed) {
stream.destroy();
}
}
}
/**