feat: implement R4 production hardening
- Add configurable ingestion batch item limits - Improve client serialization and sanitization - Add graceful handling for corrupt GZIP partitions in data API - Expand admin session token scope - Update project documentation for R4 status
This commit is contained in:
1 parent
8132087085
commit
ae9b554189
59 files changed
+16879
-1074
No files matched your search
@@ -15,6 +15,40 @@ import {
|
||||
} from "./adminSession";
|
||||
import { clearAuditLogsForTests } from "./auditLog";
|
||||
import { DatabaseEngine, ensureCoreSchemaOnPool } from "../Database/database";
|
||||
import { getInternalIngestionHeaders } from "../Security/security";
|
||||
|
||||
async function notifyIngestionAdminSessionSync(payload: {
|
||||
memberId?: string;
|
||||
email?: string;
|
||||
previousEmail?: string;
|
||||
minTokenVersion?: number;
|
||||
disabled?: boolean;
|
||||
deleted?: boolean;
|
||||
}): Promise<void> {
|
||||
const configuredUrl = process.env.INGESTION_INTERNAL_URL?.trim();
|
||||
if (process.env.DATASTACK_DISABLE_LISTEN === "1" && !configuredUrl) {
|
||||
return;
|
||||
}
|
||||
const baseUrl = (configuredUrl || "http://127.0.0.1:4000").replace(
|
||||
/\/+$/,
|
||||
"",
|
||||
);
|
||||
try {
|
||||
await fetch(`${baseUrl}/internal/admin-session-sync`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...getInternalIngestionHeaders(),
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
cache: "no-store",
|
||||
signal: AbortSignal.timeout(1200),
|
||||
});
|
||||
} catch {
|
||||
// Best-effort bridge sync when Port 4000 is offline or not running
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
const PASSWORD_HASH_BYTES = 64;
|
||||
const PASSWORD_SALT_BYTES = 16;
|
||||
@@ -1040,6 +1074,13 @@ export async function updateAdminAccount({
|
||||
tokenVersion: nextVersion,
|
||||
createdAt: existing.createdAt,
|
||||
};
|
||||
await notifyIngestionAdminSessionSync({
|
||||
memberId: "owner-1",
|
||||
email: cleanEmail,
|
||||
previousEmail: existing.email,
|
||||
minTokenVersion: nextVersion,
|
||||
disabled: false,
|
||||
});
|
||||
return {
|
||||
ok: true as const,
|
||||
email: cleanEmail,
|
||||
@@ -1085,6 +1126,13 @@ export async function updateAdminAccount({
|
||||
tokenVersion: nextVersion,
|
||||
updatedAt: new Date(),
|
||||
});
|
||||
await notifyIngestionAdminSessionSync({
|
||||
memberId: member.id,
|
||||
email: cleanEmail,
|
||||
previousEmail: member.email,
|
||||
minTokenVersion: nextVersion,
|
||||
disabled: member.status === "disabled",
|
||||
});
|
||||
return {
|
||||
ok: true as const,
|
||||
email: cleanEmail,
|
||||
@@ -1131,6 +1179,12 @@ export async function revokeAdminSessions(
|
||||
member.tokenVersion += 1;
|
||||
member.updatedAt = new Date();
|
||||
teamMap.set(member.id, member);
|
||||
await notifyIngestionAdminSessionSync({
|
||||
memberId: member.id,
|
||||
email: member.email,
|
||||
minTokenVersion: member.tokenVersion,
|
||||
disabled: member.status === "disabled",
|
||||
});
|
||||
return member.tokenVersion;
|
||||
}
|
||||
}
|
||||
@@ -1157,6 +1211,12 @@ export async function revokeAdminSessions(
|
||||
member.tokenVersion += 1;
|
||||
member.updatedAt = new Date();
|
||||
teamMap.set(member.id, member);
|
||||
await notifyIngestionAdminSessionSync({
|
||||
memberId: member.id,
|
||||
email: member.email,
|
||||
minTokenVersion: member.tokenVersion,
|
||||
disabled: member.status === "disabled",
|
||||
});
|
||||
return member.tokenVersion;
|
||||
}
|
||||
}
|
||||
@@ -1181,6 +1241,12 @@ export async function revokeAdminSessions(
|
||||
globals.__dataStackInMemoryAdmin ?? (await getOrInitInMemoryAdmin());
|
||||
existing.tokenVersion += 1;
|
||||
globals.__dataStackInMemoryAdmin = existing;
|
||||
await notifyIngestionAdminSessionSync({
|
||||
memberId: "owner-1",
|
||||
email: existing.email,
|
||||
minTokenVersion: existing.tokenVersion,
|
||||
disabled: false,
|
||||
});
|
||||
return existing.tokenVersion;
|
||||
}
|
||||
|
||||
@@ -1588,6 +1654,12 @@ export async function updateAdminTeamMemberAsync(
|
||||
updatedAt: new Date(),
|
||||
};
|
||||
teamMap.set(id, updated);
|
||||
await notifyIngestionAdminSessionSync({
|
||||
memberId: updated.id,
|
||||
email: updated.email,
|
||||
minTokenVersion: updated.tokenVersion,
|
||||
disabled: updated.status === "disabled",
|
||||
});
|
||||
|
||||
return {
|
||||
id: updated.id,
|
||||
@@ -1654,6 +1726,13 @@ export async function deleteAdminTeamMemberAsync(
|
||||
);
|
||||
}
|
||||
teamMap.delete(id);
|
||||
await notifyIngestionAdminSessionSync({
|
||||
memberId: existing.id,
|
||||
email: existing.email,
|
||||
minTokenVersion: existing.tokenVersion + 1,
|
||||
disabled: true,
|
||||
deleted: true,
|
||||
});
|
||||
return { id: existing.id, email: existing.email };
|
||||
}
|
||||
|
||||
|
||||
+65
-8
@@ -20,6 +20,8 @@ export type AuditActionType =
|
||||
| "STORAGE_FLUSH_TRIGGERED"
|
||||
| "S3_ARCHIVE_TRIGGERED"
|
||||
| "WEBHOOK_TEST_TRIGGERED"
|
||||
| "FILES_MANAGER_MUTATION"
|
||||
| "DATA_EXPORT_DOWNLOADED"
|
||||
| string;
|
||||
|
||||
export interface AuditLogEntry {
|
||||
@@ -49,10 +51,19 @@ export interface ListAuditLogsQuery {
|
||||
search?: string;
|
||||
}
|
||||
|
||||
export interface AuditLogListResult {
|
||||
logs: AuditLogEntry[];
|
||||
storageMode: "postgresql" | "memory";
|
||||
degraded: boolean;
|
||||
code?: "AUDIT_STORAGE_DEGRADED";
|
||||
warning?: string;
|
||||
}
|
||||
|
||||
const MAX_IN_MEMORY_AUDIT_LOGS = 1000;
|
||||
|
||||
type AuditLogGlobals = typeof globalThis & {
|
||||
__dataStackAuditLogs?: AuditLogEntry[];
|
||||
__dataStackAuditLastWriteError?: string;
|
||||
};
|
||||
|
||||
const globals = globalThis as AuditLogGlobals;
|
||||
@@ -149,8 +160,13 @@ export async function recordAdminAuditLogAsync(
|
||||
entry.createdAt,
|
||||
],
|
||||
);
|
||||
} catch {
|
||||
// Never block or crash administrative actions if audit table write experiences transient DB failure
|
||||
globals.__dataStackAuditLastWriteError = undefined;
|
||||
} catch (err: unknown) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
globals.__dataStackAuditLastWriteError = msg;
|
||||
console.warn(
|
||||
`[DataStack Audit] PostgreSQL audit insert failed for action=${entry.action} (buffered in memory only): ${msg}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -161,9 +177,9 @@ export function recordAdminAuditLog(input: RecordAuditLogInput): void {
|
||||
void recordAdminAuditLogAsync(input);
|
||||
}
|
||||
|
||||
export async function listAdminAuditLogsAsync(
|
||||
export async function listAdminAuditLogsWithStatusAsync(
|
||||
query: ListAuditLogsQuery = {},
|
||||
): Promise<AuditLogEntry[]> {
|
||||
): Promise<AuditLogListResult> {
|
||||
const rawLimit =
|
||||
typeof query.limit === "number" && Number.isFinite(query.limit)
|
||||
? Math.floor(query.limit)
|
||||
@@ -174,6 +190,7 @@ export async function listAdminAuditLogsAsync(
|
||||
const searchFilter = query.search?.trim().toLowerCase() || "";
|
||||
|
||||
const pool = DatabaseEngine.getPool();
|
||||
let dbReadError: string | null = null;
|
||||
if (pool) {
|
||||
try {
|
||||
await DatabaseEngine.ensureSchema();
|
||||
@@ -227,7 +244,7 @@ export async function listAdminAuditLogsAsync(
|
||||
params,
|
||||
);
|
||||
|
||||
return result.rows.map((row) => {
|
||||
const logs = result.rows.map((row) => {
|
||||
let parsedMeta: Record<string, unknown> = {};
|
||||
try {
|
||||
const parsed = JSON.parse(row.metadata_json || "{}");
|
||||
@@ -251,12 +268,28 @@ export async function listAdminAuditLogsAsync(
|
||||
: new Date(row.created_at).toISOString(),
|
||||
};
|
||||
});
|
||||
} catch {
|
||||
// Fallback to in-memory log buffer if PostgreSQL query fails
|
||||
|
||||
const writeErr = globals.__dataStackAuditLastWriteError;
|
||||
return {
|
||||
logs,
|
||||
storageMode: "postgresql",
|
||||
degraded: Boolean(writeErr),
|
||||
...(writeErr
|
||||
? {
|
||||
code: "AUDIT_STORAGE_DEGRADED" as const,
|
||||
warning: `Recent audit log insert failed in PostgreSQL: ${writeErr}`,
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
} catch (err: unknown) {
|
||||
dbReadError = err instanceof Error ? err.message : String(err);
|
||||
console.warn(
|
||||
`[DataStack Audit] PostgreSQL audit query failed (serving ephemeral in-memory fallback): ${dbReadError}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return getInMemoryAuditLogs()
|
||||
const fallbackLogs = getInMemoryAuditLogs()
|
||||
.filter((entry) => {
|
||||
if (
|
||||
actionFilter &&
|
||||
@@ -278,8 +311,32 @@ export async function listAdminAuditLogsAsync(
|
||||
return true;
|
||||
})
|
||||
.slice(0, limit);
|
||||
|
||||
if (dbReadError) {
|
||||
return {
|
||||
logs: fallbackLogs,
|
||||
storageMode: "memory",
|
||||
degraded: true,
|
||||
code: "AUDIT_STORAGE_DEGRADED",
|
||||
warning: `PostgreSQL audit query failed (${dbReadError}); serving ephemeral in-memory logs.`,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
logs: fallbackLogs,
|
||||
storageMode: "memory",
|
||||
degraded: false,
|
||||
};
|
||||
}
|
||||
|
||||
export async function listAdminAuditLogsAsync(
|
||||
query: ListAuditLogsQuery = {},
|
||||
): Promise<AuditLogEntry[]> {
|
||||
const result = await listAdminAuditLogsWithStatusAsync(query);
|
||||
return result.logs;
|
||||
}
|
||||
|
||||
export function clearAuditLogsForTests(): void {
|
||||
globals.__dataStackAuditLogs = [];
|
||||
globals.__dataStackAuditLastWriteError = undefined;
|
||||
}
|
||||
Reference in new issue
Block a user