feat: implement R4 production hardening

- Add configurable ingestion batch item limits
- Improve client serialization and sanitization
- Add graceful handling for corrupt GZIP partitions in data API
- Expand admin session token scope
- Update project documentation for R4 status
This commit is contained in:
Team Dark Devil committed 2026-11-21 22:15:32 +06:00
1 parent 8132087085
commit ae9b554189
59 files changed
+16879 -1074

No files matched your search

+2
View File
@@ -50,6 +50,8 @@ const ACTION_FILTER_OPTIONS = [
{ value: 'STORAGE_FLUSH_TRIGGERED', label: 'STORAGE_FLUSH_TRIGGERED' },
{ value: 'S3_ARCHIVE_TRIGGERED', label: 'S3_ARCHIVE_TRIGGERED' },
{ value: 'WEBHOOK_TEST_TRIGGERED', label: 'WEBHOOK_TEST_TRIGGERED' },
{ value: 'FILES_MANAGER_MUTATION', label: 'FILES_MANAGER_MUTATION' },
{ value: 'DATA_EXPORT_DOWNLOADED', label: 'DATA_EXPORT_DOWNLOADED' },
];
export function AuditLogViewer() {
+34
View File
@@ -36,6 +36,13 @@ interface Client {
name: string;
}
interface CorruptPartitionWarning {
clientId: string;
filename: string;
code?: string;
error: string;
}
const PAGE_SIZE = 12;
function formatEventPayload(raw: string): string {
@@ -70,6 +77,7 @@ export function DataBrowser() {
const [filtersReady, setFiltersReady] = useState(false);
const [refreshing, setRefreshing] = useState(false);
const [errorMessage, setErrorMessage] = useState('');
const [corruptPartitions, setCorruptPartitions] = useState<CorruptPartitionWarning[]>([]);
const [selectedEvent, setSelectedEvent] = useState<StreamEvent | null>(null);
const fetchEvents = useCallback(async () => {
@@ -94,6 +102,9 @@ export function DataBrowser() {
setEvents(data.events || []);
setTotal(data.total || 0);
setPageCount(data.pageCount || 1);
setCorruptPartitions(
Array.isArray(data.corruptPartitions) ? data.corruptPartitions : []
);
setErrorMessage('');
} catch (error) {
setErrorMessage(error instanceof Error ? error.message : 'Unable to load events');
@@ -271,6 +282,29 @@ export function DataBrowser() {
)}
</Card>
{corruptPartitions.length > 0 && (
<div
role="status"
data-testid="data-corrupt-partitions-warning"
className="border border-[var(--warning)]/40 bg-[var(--warning)]/10 px-3 py-2 text-[11.5px] text-[var(--text)]"
>
<div className="font-medium">
{corruptPartitions.length} corrupt .csv.gz partition
{corruptPartitions.length === 1 ? '' : 's'} skipped while loading healthy events.
</div>
<ul className="mt-1 flex flex-col gap-0.5 font-mono text-[10.5px] text-[var(--text-muted)]">
{corruptPartitions.slice(0, 5).map((item) => (
<li key={`${item.clientId}:${item.filename}`}>
{item.clientId}/{item.filename} — {item.error}
</li>
))}
{corruptPartitions.length > 5 && (
<li>+{corruptPartitions.length - 5} more unreadable archive(s)</li>
)}
</ul>
</div>
)}
<Panel
title="Events"
meta={`${total.toLocaleString()} · ${page}/${pageCount}`}
@@ -0,0 +1,210 @@
'use client';
import { PageShell } from '@/components/layout/PageShell';
import { useVconRole } from '@/components/layout/VconShell';
import { Button } from '@/components/ui/Button';
import { Icon } from '@/components/ui/Icon';
import { getVconRoleCapabilities } from '@/lib/vconUiHelpers';
import React, { useCallback, useEffect, useState } from 'react';
type EngineStatus = 'checking' | 'online' | 'offline';
interface FilesEngineStatusResponse {
ok?: boolean;
engine?: string;
role?: string;
readOnly?: boolean;
proxyMode?: 'nginx' | 'standalone';
wsSupported?: boolean;
wsHint?: string | null;
error?: string;
code?: string;
}
const FILES_ENGINE_STATUS_URL = '/api/files/status';
const FILES_ENGINE_FRAME_URL = '/vcon/files-engine';
export function FilesManagerWorkspace() {
const currentRole = useVconRole();
const { canOperate } = getVconRoleCapabilities(currentRole);
const [status, setStatus] = useState<EngineStatus>('checking');
const [statusError, setStatusError] = useState<string>('');
const [wsHint, setWsHint] = useState<string | null>(null);
const [engineReadOnly, setEngineReadOnly] = useState<boolean>(!canOperate);
const [frameKey, setFrameKey] = useState<number>(1);
const [probing, setProbing] = useState<boolean>(false);
const checkEngineStatus = useCallback(async (reloadFrame = false) => {
setProbing(true);
setStatusError('');
try {
const response = await fetch(FILES_ENGINE_STATUS_URL, {
method: 'GET',
cache: 'no-store',
credentials: 'same-origin',
});
if (response.status === 401) {
window.location.assign('/vcon/login?next=%2Fvcon%2Ffiles');
return;
}
const data = (await response.json().catch(() => ({}))) as FilesEngineStatusResponse;
setWsHint(typeof data.wsHint === 'string' && data.wsHint ? data.wsHint : null);
if (response.ok && data.ok) {
setEngineReadOnly(Boolean(data.readOnly));
setStatus('online');
if (reloadFrame) {
setFrameKey((prev) => prev + 1);
}
return;
}
setStatus('offline');
setStatusError(
data.error ||
`Files Manager engine returned HTTP ${response.status}. Ensure the Port 4000 ingestion server is running.`
);
} catch {
setStatus('offline');
setStatusError(
'Cannot reach the Port 4000 Files Manager engine. Start both services with "npm run dev:all" or Docker Compose.'
);
} finally {
setProbing(false);
}
}, []);
useEffect(() => {
void checkEngineStatus(false);
}, [checkEngineStatus]);
const effectiveReadOnly = !canOperate || engineReadOnly;
return (
<PageShell
title="Files Manager"
actions={
<>
<span
className="border border-[var(--border)] bg-[var(--surface-2)] px-2 py-1 font-mono text-[10.5px] uppercase tracking-wider text-[var(--text-muted)]"
data-testid="files-manager-access-badge"
>
{effectiveReadOnly
? `Read-only (${currentRole})`
: `Full Access (${currentRole})`}
</span>
<Button
size="xs"
variant="secondary"
icon="RotateCw"
loading={probing}
disabled={probing}
onClick={() => void checkEngineStatus(true)}
>
Reload
</Button>
{status === 'online' && (
<a
href={FILES_ENGINE_FRAME_URL}
target="_blank"
rel="noopener noreferrer"
className="inline-flex h-6 items-center gap-1.5 border border-[var(--border)] bg-[var(--surface)] px-2 text-[11px] font-medium text-[var(--text)] transition-colors hover:border-[var(--border-strong)] hover:bg-[var(--surface-2)]"
>
<Icon name="ExternalLink" size={12} />
<span>Full View</span>
</a>
)}
</>
}
>
<div className="flex flex-wrap items-center justify-between gap-2 border border-[var(--border)] bg-[var(--surface)] px-3 py-1.5 text-[11.5px] text-[var(--text-muted)]">
<div className="flex items-center gap-2">
<Icon name="FolderOpen" size={13} className="shrink-0 text-[var(--accent)]" />
<span>
Root: <code className="font-mono text-[var(--text)]">/storage/data</code> — browse, edit, rename, upload, download, and ZIP/UNZIP client partitions.
</span>
</div>
<span className="font-mono text-[10.5px] text-[var(--text-dim)]">
{effectiveReadOnly
? 'Viewer mode: browse, view & download only'
: 'Active *.lock files are protected; avoid editing live UTC-today CSVs during peak flush'}
</span>
</div>
{status === 'online' && wsHint && (
<div
className="flex items-center gap-2 border border-[var(--border)] bg-[var(--surface-2)] px-3 py-1.5 text-[11px] text-[var(--text-muted)]"
data-testid="files-manager-proxy-hint"
>
<Icon name="Info" size={12} className="shrink-0 text-[var(--accent)]" />
<span>{wsHint}</span>
</div>
)}
<div
className="w-full overflow-hidden border border-[var(--border)] bg-[var(--surface)]"
data-testid="files-manager-container"
>
{status === 'checking' && (
<div className="flex h-[calc(100dvh-180px)] min-h-[480px] flex-col items-center justify-center gap-2.5 p-6 text-center">
<Icon name="RotateCw" size={18} className="animate-spin text-[var(--accent)]" />
<p className="text-[12.5px] font-medium text-[var(--text)]">
Connecting to Files Manager engine...
</p>
<p className="text-[11.5px] text-[var(--text-dim)]">
Verifying admin session and storage root on Port 4000
</p>
</div>
)}
{status === 'offline' && (
<div
className="flex h-[calc(100dvh-180px)] min-h-[480px] flex-col items-center justify-center gap-3 p-6 text-center"
data-testid="files-manager-offline"
>
<div className="flex h-9 w-9 items-center justify-center border border-[var(--border-strong)] bg-[var(--surface-2)] text-[var(--text-muted)]">
<Icon name="HardDrive" size={18} />
</div>
<div className="max-w-md space-y-1">
<h2 className="text-[13px] font-medium text-[var(--text)]">
Files Manager Engine Unavailable
</h2>
<p className="text-[12px] text-[var(--text-muted)]">
{statusError ||
'The Port 4000 ingestion and files engine is currently offline.'}
</p>
</div>
<div className="border border-[var(--border)] bg-[var(--surface-2)] px-3 py-2 font-mono text-[11px] text-[var(--text-dim)]">
Local: npm run dev:all &nbsp;|&nbsp; Production: docker compose up
</div>
<Button
size="sm"
variant="primary"
icon="RotateCw"
loading={probing}
disabled={probing}
onClick={() => void checkEngineStatus(true)}
>
Retry Connection
</Button>
</div>
)}
{status === 'online' && (
<iframe
key={frameKey}
src={FILES_ENGINE_FRAME_URL}
title="DataStack Files Manager"
data-testid="files-manager-iframe"
className="block h-[calc(100dvh-180px)] min-h-[540px] w-full border-0 bg-[var(--surface)]"
/>
)}
</div>
</PageShell>
);
}
export default FilesManagerWorkspace;
+1 -1
View File
@@ -21,7 +21,7 @@ export const Hero: React.FC = () => (
Explore features
</Link>
<Link href="/#sdk" className={buttonClassNames({ variant: 'secondary', size: 'sm' })}>
Python SDK
Client SDKs
</Link>
</div>
<ActivityPreview />
+1 -1
View File
@@ -9,7 +9,7 @@ export const LandingAbout: React.FC = () => (
<p className="text-[13px] text-[var(--text-muted)]">Start with the SDK guide and make the workspace your own.</p>
</div>
<Link href="/#sdk" className={buttonClassNames({ variant: 'primary', size: 'sm' })}>
View Python SDK
View Client SDKs
</Link>
</section>
);
+26 -11
View File
@@ -27,10 +27,10 @@ interface AdminProfileData {
sessionExpiresAt: string;
}
const DEFAULT_PROFILE: AdminProfileData = {
email: 'admin@datastack.dev',
role: 'Owner',
storageMode: 'In-Memory',
const EMPTY_PROFILE: AdminProfileData = {
email: '',
role: '',
storageMode: '',
createdAt: '',
sessionExpiresAt: '',
};
@@ -38,10 +38,10 @@ const DEFAULT_PROFILE: AdminProfileData = {
export function ProfileWorkspace() {
const { success, error } = useToast();
const [profile, setProfile] = useState<AdminProfileData>(() =>
getCached<AdminProfileData>('vcon:profile', DEFAULT_PROFILE)
getCached<AdminProfileData>('vcon:profile', EMPTY_PROFILE)
);
const [email, setEmail] = useState(
() => getCached<AdminProfileData>('vcon:profile', DEFAULT_PROFILE).email
() => getCached<AdminProfileData>('vcon:profile', EMPTY_PROFILE).email
);
const [currentPassword, setCurrentPassword] = useState('');
const [newPassword, setNewPassword] = useState('');
@@ -183,12 +183,26 @@ export function ProfileWorkspace() {
>
<StatStrip
items={[
{ label: 'Administrator', value: profile.email },
{ label: 'Role', value: profile.role, dot: 'ok' },
{ label: 'Auth Store', value: profile.storageMode },
{
label: 'Administrator',
value: profile.email || (loading ? 'Loading...' : '—'),
},
{
label: 'Role',
value: profile.role || (loading ? 'Loading...' : '—'),
dot: profile.role ? 'ok' : undefined,
},
{
label: 'Auth Store',
value: profile.storageMode || (loading ? 'Loading...' : '—'),
},
{
label: 'Session Expires',
value: formatSessionExpiryLabel(profile.sessionExpiresAt),
value: profile.sessionExpiresAt
? formatSessionExpiryLabel(profile.sessionExpiresAt)
: loading
? 'Loading...'
: 'Active',
},
]}
/>
@@ -200,6 +214,7 @@ export function ProfileWorkspace() {
<Input
compact
type="email"
placeholder={loading ? 'Loading account email...' : 'admin@company.com'}
value={email}
onChange={(event) => setEmail(event.target.value)}
required
@@ -208,7 +223,7 @@ export function ProfileWorkspace() {
<SettingsField label="Access role">
<Input
compact
value={profile.role}
value={profile.role || (loading ? 'Loading...' : '—')}
readOnly
className="border-[var(--border)] bg-[var(--surface-2)] text-[var(--text-muted)]"
/>
+18 -11
View File
@@ -4,25 +4,32 @@ import React from 'react';
export const SdkSection: React.FC = () => (
<section id="sdk" className="border-b border-[var(--border)] py-10">
<SectionHeading index="02" label="Python SDK" title="Drop in the SDK. Start streaming." />
<SectionHeading index="02" label="Client SDKs" title="Drop in the SDK. Start streaming." />
<div className="grid grid-cols-1 gap-px border border-[var(--border)] bg-[var(--border)] md:grid-cols-2">
<CodeBlock>
<span className="text-[var(--text-faint)]"># Install the bundled Python SDK (from repo or local wheel/path)</span>{'\n'}
<span className="text-[var(--text)]">pip install /path/to/datastack/sdk/python/datastack</span>{'\n\n'}
<span className="text-[var(--text-faint)]"># Auto-loads config.datastack.py from project root</span>{'\n'}
<span className="text-[var(--text-faint)]"># Python SDK (datastack v2.2.0) — auto-loads config.datastack.py</span>{'\n'}
<span className="text-[var(--text)]">pip install ./sdk/python/datastack</span>{'\n\n'}
<span className="text-[var(--text)]">from datastack import DataStack</span>{'\n\n'}
<span className="text-[var(--text)]">client = DataStack()</span>{'\n'}
<span className="text-[var(--text)]">client.send({'{'}<span className="text-[var(--accent)]">&quot;event_type&quot;</span>: <span className="text-[var(--accent)]">&quot;sensor_telemetry&quot;</span>, <span className="text-[var(--accent)]">&quot;temperature&quot;</span>: 24.8, <span className="text-[var(--accent)]">&quot;status&quot;</span>: <span className="text-[var(--accent)]">&quot;NOMINAL&quot;</span>{'}'})</span>{'\n'}
<span className="text-[var(--text)]">client.send_file(<span className="text-[var(--accent)]">&quot;telemetry_batch.csv&quot;</span>)</span>{'\n'}
<span className="text-[var(--text)]">client.disconnect()</span>
</CodeBlock>
<div className="bg-[var(--surface)] p-5 sm:p-6">
<p className="mb-2 text-[11px] uppercase text-[var(--text-faint)]">How it works</p>
<h3 className="mb-2 text-[14px] font-medium text-[var(--text)]">Zero-boilerplate config. Auto-recovery built in.</h3>
<p className="text-[12px] leading-relaxed text-[var(--text-muted)]">
Install the self-hosted SDK from <code className="text-[var(--text)]">sdk/python/datastack</code> (or build a wheel with <code className="text-[var(--text)]">pip wheel ./sdk/python/datastack -w dist/</code>) and drop <code className="text-[var(--text)]">config.datastack.py</code> into your Python project to stream single events, batches, or <code className="text-[var(--text)]">.csv</code> / <code className="text-[var(--text)]">.json</code> files directly—with automatic HTTP fallback, offline disk spooling, and Heartbeat IDLE recovery.
</p>
</div>
<CodeBlock>
<span className="text-[var(--text-faint)]">{'// Node.js / TypeScript SDK (@datastack/sdk v2.2.0)'}</span>{'\n'}
<span className="text-[var(--text)]">npm install ./sdk/node/datastack</span>{'\n\n'}
<span className="text-[var(--text)]">import {'{'} DataStack {'}'} from <span className="text-[var(--accent)]">&apos;@datastack/sdk&apos;</span>;</span>{'\n\n'}
<span className="text-[var(--text)]">const client = new DataStack();</span>{'\n'}
<span className="text-[var(--text)]">await client.send({'{'} <span className="text-[var(--accent)]">event_type</span>: <span className="text-[var(--accent)]">&apos;sensor_telemetry&apos;</span>, <span className="text-[var(--accent)]">temperature</span>: 24.8 {'}'});</span>{'\n'}
<span className="text-[var(--text)]">await client.sendFile(<span className="text-[var(--accent)]">&apos;telemetry_batch.csv&apos;</span>);</span>{'\n'}
<span className="text-[var(--text)]">await client.shutdown(5000);</span>
</CodeBlock>
</div>
<div className="border-x border-b border-[var(--border)] bg-[var(--surface)] px-5 py-4 sm:px-6">
<p className="mb-1 text-[11px] uppercase text-[var(--text-faint)]">Zero-boilerplate config · Auto-recovery built in</p>
<p className="text-[12px] leading-relaxed text-[var(--text-muted)]">
Install either <code className="text-[var(--text)]">sdk/python/datastack</code> (<code className="text-[var(--text)]">config.datastack.py</code>) or <code className="text-[var(--text)]">@datastack/sdk</code> in <code className="text-[var(--text)]">sdk/node/datastack</code> (<code className="text-[var(--text)]">datastack.config.json</code>) to stream single events, batches, or <code className="text-[var(--text)]">.csv</code> / <code className="text-[var(--text)]">.json</code> / <code className="text-[var(--text)]">.jsonl</code> files—with automatic HTTP fallback, offline disk spooling, and graceful shutdown drain.
</p>
</div>
</section>
);
+59 -6
View File
@@ -1,6 +1,7 @@
'use client';
import { PageShell } from '@/components/layout/PageShell';
import { useVconRole } from '@/components/layout/VconShell';
import { Button } from '@/components/ui/Button';
import { Input, Select } from '@/components/ui/Input';
import { Modal } from '@/components/ui/Modal';
@@ -34,8 +35,9 @@ interface TeamMemberItem {
export function TeamWorkspace() {
const { success, error } = useToast();
const shellRole = useVconRole();
const [members, setMembers] = useState<TeamMemberItem[]>([]);
const [currentRole, setCurrentRole] = useState<string>('owner');
const [currentRole, setCurrentRole] = useState<string>(() => shellRole || 'viewer');
const [loading, setLoading] = useState<boolean>(true);
const [createOpen, setCreateOpen] = useState<boolean>(false);
@@ -43,6 +45,7 @@ export function TeamWorkspace() {
const [newPassword, setNewPassword] = useState<string>('');
const [newRole, setNewRole] = useState<string>('viewer');
const [creating, setCreating] = useState<boolean>(false);
const [deleteTarget, setDeleteTarget] = useState<TeamMemberItem | null>(null);
const [busyId, setBusyId] = useState<string | null>(null);
@@ -186,6 +189,7 @@ export function TeamWorkspace() {
throw new Error(data?.error || 'Delete failed.');
}
success(`Removed ${member.email} from team.`);
setDeleteTarget(null);
await fetchMembers();
} catch (err: unknown) {
error(err instanceof Error ? err.message : 'Unable to delete team member.');
@@ -246,7 +250,7 @@ export function TeamWorkspace() {
value: String(ownerCount),
},
{
label: 'Admins / Operators',
label: 'Operators / Admins',
value: String(operatorCount),
},
{
@@ -306,14 +310,15 @@ export function TeamWorkspace() {
<Select
compact
aria-label={`Role for ${row.email}`}
value={row.role === 'operator' ? 'admin' : row.role}
value={row.role}
disabled={busyId === row.id}
onChange={(e) =>
void handleRoleChange(row, e.target.value)
}
options={[
{ value: 'viewer', label: 'Viewer (Read-Only)' },
{ value: 'admin', label: 'Admin / Operator' },
{ value: 'operator', label: 'Operator (Manage Clients)' },
{ value: 'admin', label: 'Admin (Manage Clients)' },
{ value: 'owner', label: 'Owner (Full Control)' },
]}
/>
@@ -359,7 +364,7 @@ export function TeamWorkspace() {
variant="danger"
size="sm"
disabled={busyId === row.id}
onClick={() => void handleDeleteMember(row)}
onClick={() => setDeleteTarget(row)}
>
Remove
</Button>
@@ -410,9 +415,13 @@ export function TeamWorkspace() {
value: 'viewer',
label: 'Viewer — Read-only dashboards, events, clients & exports',
},
{
value: 'operator',
label: 'Operator — Manage SDK clients, tokens & manual flushes',
},
{
value: 'admin',
label: 'Admin / Operator — Manage SDK clients, tokens & flushes',
label: 'Admin — Operational role (manage SDK clients & flushes)',
},
{
value: 'owner',
@@ -440,6 +449,50 @@ export function TeamWorkspace() {
</div>
</form>
</Modal>
<Modal
isOpen={Boolean(deleteTarget)}
onClose={() => {
if (!busyId) setDeleteTarget(null);
}}
title="Confirm Team Member Removal"
maxWidth="sm"
>
{deleteTarget && (
<div className="flex flex-col gap-3">
<p className="text-[12px] text-[var(--text-muted)] leading-relaxed">
Remove{' '}
<span className="font-mono font-medium text-[var(--text)]">
{deleteTarget.email}
</span>{' '}
(<span className="font-mono uppercase">{deleteTarget.role}</span>) from
administrator access? Active sessions will stop working immediately.
</p>
<div className="flex justify-end gap-2 pt-1">
<Button
type="button"
variant="secondary"
size="sm"
disabled={busyId === deleteTarget.id}
onClick={() => setDeleteTarget(null)}
>
Cancel
</Button>
<Button
type="button"
variant="danger"
size="sm"
data-testid="confirm-remove-member"
loading={busyId === deleteTarget.id}
disabled={busyId === deleteTarget.id}
onClick={() => void handleDeleteMember(deleteTarget)}
>
Confirm Remove
</Button>
</div>
</div>
)}
</Modal>
</PageShell>
);
}