feat: implement R4 production hardening

- Add configurable ingestion batch item limits
- Improve client serialization and sanitization
- Add graceful handling for corrupt GZIP partitions in data API
- Expand admin session token scope
- Update project documentation for R4 status
This commit is contained in:
Team Dark Devil committed 2026-11-21 22:15:32 +06:00
1 parent 8132087085
commit ae9b554189
59 files changed
+16879 -1074

No files matched your search

+79
View File
@@ -15,6 +15,40 @@ import {
} from "./adminSession";
import { clearAuditLogsForTests } from "./auditLog";
import { DatabaseEngine, ensureCoreSchemaOnPool } from "../Database/database";
import { getInternalIngestionHeaders } from "../Security/security";
async function notifyIngestionAdminSessionSync(payload: {
memberId?: string;
email?: string;
previousEmail?: string;
minTokenVersion?: number;
disabled?: boolean;
deleted?: boolean;
}): Promise<void> {
const configuredUrl = process.env.INGESTION_INTERNAL_URL?.trim();
if (process.env.DATASTACK_DISABLE_LISTEN === "1" && !configuredUrl) {
return;
}
const baseUrl = (configuredUrl || "http://127.0.0.1:4000").replace(
/\/+$/,
"",
);
try {
await fetch(`${baseUrl}/internal/admin-session-sync`, {
method: "POST",
headers: {
"Content-Type": "application/json",
...getInternalIngestionHeaders(),
},
body: JSON.stringify(payload),
cache: "no-store",
signal: AbortSignal.timeout(1200),
});
} catch {
// Best-effort bridge sync when Port 4000 is offline or not running
}
}
const PASSWORD_HASH_BYTES = 64;
const PASSWORD_SALT_BYTES = 16;
@@ -1040,6 +1074,13 @@ export async function updateAdminAccount({
tokenVersion: nextVersion,
createdAt: existing.createdAt,
};
await notifyIngestionAdminSessionSync({
memberId: "owner-1",
email: cleanEmail,
previousEmail: existing.email,
minTokenVersion: nextVersion,
disabled: false,
});
return {
ok: true as const,
email: cleanEmail,
@@ -1085,6 +1126,13 @@ export async function updateAdminAccount({
tokenVersion: nextVersion,
updatedAt: new Date(),
});
await notifyIngestionAdminSessionSync({
memberId: member.id,
email: cleanEmail,
previousEmail: member.email,
minTokenVersion: nextVersion,
disabled: member.status === "disabled",
});
return {
ok: true as const,
email: cleanEmail,
@@ -1131,6 +1179,12 @@ export async function revokeAdminSessions(
member.tokenVersion += 1;
member.updatedAt = new Date();
teamMap.set(member.id, member);
await notifyIngestionAdminSessionSync({
memberId: member.id,
email: member.email,
minTokenVersion: member.tokenVersion,
disabled: member.status === "disabled",
});
return member.tokenVersion;
}
}
@@ -1157,6 +1211,12 @@ export async function revokeAdminSessions(
member.tokenVersion += 1;
member.updatedAt = new Date();
teamMap.set(member.id, member);
await notifyIngestionAdminSessionSync({
memberId: member.id,
email: member.email,
minTokenVersion: member.tokenVersion,
disabled: member.status === "disabled",
});
return member.tokenVersion;
}
}
@@ -1181,6 +1241,12 @@ export async function revokeAdminSessions(
globals.__dataStackInMemoryAdmin ?? (await getOrInitInMemoryAdmin());
existing.tokenVersion += 1;
globals.__dataStackInMemoryAdmin = existing;
await notifyIngestionAdminSessionSync({
memberId: "owner-1",
email: existing.email,
minTokenVersion: existing.tokenVersion,
disabled: false,
});
return existing.tokenVersion;
}
@@ -1588,6 +1654,12 @@ export async function updateAdminTeamMemberAsync(
updatedAt: new Date(),
};
teamMap.set(id, updated);
await notifyIngestionAdminSessionSync({
memberId: updated.id,
email: updated.email,
minTokenVersion: updated.tokenVersion,
disabled: updated.status === "disabled",
});
return {
id: updated.id,
@@ -1654,6 +1726,13 @@ export async function deleteAdminTeamMemberAsync(
);
}
teamMap.delete(id);
await notifyIngestionAdminSessionSync({
memberId: existing.id,
email: existing.email,
minTokenVersion: existing.tokenVersion + 1,
disabled: true,
deleted: true,
});
return { id: existing.id, email: existing.email };
}
+65 -8
View File
@@ -20,6 +20,8 @@ export type AuditActionType =
| "STORAGE_FLUSH_TRIGGERED"
| "S3_ARCHIVE_TRIGGERED"
| "WEBHOOK_TEST_TRIGGERED"
| "FILES_MANAGER_MUTATION"
| "DATA_EXPORT_DOWNLOADED"
| string;
export interface AuditLogEntry {
@@ -49,10 +51,19 @@ export interface ListAuditLogsQuery {
search?: string;
}
export interface AuditLogListResult {
logs: AuditLogEntry[];
storageMode: "postgresql" | "memory";
degraded: boolean;
code?: "AUDIT_STORAGE_DEGRADED";
warning?: string;
}
const MAX_IN_MEMORY_AUDIT_LOGS = 1000;
type AuditLogGlobals = typeof globalThis & {
__dataStackAuditLogs?: AuditLogEntry[];
__dataStackAuditLastWriteError?: string;
};
const globals = globalThis as AuditLogGlobals;
@@ -149,8 +160,13 @@ export async function recordAdminAuditLogAsync(
entry.createdAt,
],
);
} catch {
// Never block or crash administrative actions if audit table write experiences transient DB failure
globals.__dataStackAuditLastWriteError = undefined;
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
globals.__dataStackAuditLastWriteError = msg;
console.warn(
`[DataStack Audit] PostgreSQL audit insert failed for action=${entry.action} (buffered in memory only): ${msg}`,
);
}
}
@@ -161,9 +177,9 @@ export function recordAdminAuditLog(input: RecordAuditLogInput): void {
void recordAdminAuditLogAsync(input);
}
export async function listAdminAuditLogsAsync(
export async function listAdminAuditLogsWithStatusAsync(
query: ListAuditLogsQuery = {},
): Promise<AuditLogEntry[]> {
): Promise<AuditLogListResult> {
const rawLimit =
typeof query.limit === "number" && Number.isFinite(query.limit)
? Math.floor(query.limit)
@@ -174,6 +190,7 @@ export async function listAdminAuditLogsAsync(
const searchFilter = query.search?.trim().toLowerCase() || "";
const pool = DatabaseEngine.getPool();
let dbReadError: string | null = null;
if (pool) {
try {
await DatabaseEngine.ensureSchema();
@@ -227,7 +244,7 @@ export async function listAdminAuditLogsAsync(
params,
);
return result.rows.map((row) => {
const logs = result.rows.map((row) => {
let parsedMeta: Record<string, unknown> = {};
try {
const parsed = JSON.parse(row.metadata_json || "{}");
@@ -251,12 +268,28 @@ export async function listAdminAuditLogsAsync(
: new Date(row.created_at).toISOString(),
};
});
} catch {
// Fallback to in-memory log buffer if PostgreSQL query fails
const writeErr = globals.__dataStackAuditLastWriteError;
return {
logs,
storageMode: "postgresql",
degraded: Boolean(writeErr),
...(writeErr
? {
code: "AUDIT_STORAGE_DEGRADED" as const,
warning: `Recent audit log insert failed in PostgreSQL: ${writeErr}`,
}
: {}),
};
} catch (err: unknown) {
dbReadError = err instanceof Error ? err.message : String(err);
console.warn(
`[DataStack Audit] PostgreSQL audit query failed (serving ephemeral in-memory fallback): ${dbReadError}`,
);
}
}
return getInMemoryAuditLogs()
const fallbackLogs = getInMemoryAuditLogs()
.filter((entry) => {
if (
actionFilter &&
@@ -278,8 +311,32 @@ export async function listAdminAuditLogsAsync(
return true;
})
.slice(0, limit);
if (dbReadError) {
return {
logs: fallbackLogs,
storageMode: "memory",
degraded: true,
code: "AUDIT_STORAGE_DEGRADED",
warning: `PostgreSQL audit query failed (${dbReadError}); serving ephemeral in-memory logs.`,
};
}
return {
logs: fallbackLogs,
storageMode: "memory",
degraded: false,
};
}
export async function listAdminAuditLogsAsync(
query: ListAuditLogsQuery = {},
): Promise<AuditLogEntry[]> {
const result = await listAdminAuditLogsWithStatusAsync(query);
return result.logs;
}
export function clearAuditLogsForTests(): void {
globals.__dataStackAuditLogs = [];
globals.__dataStackAuditLastWriteError = undefined;
}