27 changed files with 2319 additions and 2871 deletions

No files matched your search

+111 -27
View File
@@ -5,9 +5,26 @@ on:
paths:
- '.gitea/workflows/datastack-ci.yml'
workflow_dispatch:
inputs:
p01_only:
description: Run P01 tests, typecheck, build and Windows clean-install only
required: false
type: boolean
default: false
quality_only:
description: Run only TypeScript and ESLint after a focused P01 test type fix
required: false
type: boolean
default: false
python_only:
description: Narrow CI-06 Python job rerun after a proven runner setup failure
required: false
type: boolean
default: false
jobs:
node-regression:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.quality_only != 'true' && (github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true') }}
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
@@ -38,26 +55,38 @@ jobs:
test "$failed" -eq 0
python-regression:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.p01_only != 'true' }}
runs-on: ubuntu-latest
timeout-minutes: 15
container: node:22-bookworm-slim
timeout-minutes: 20
steps:
- uses: actions/checkout@v4
with:
github-server-url: https://dev.ivib.cc
- name: Install Python toolchain
- name: Install Bookworm Python and checkout prerequisites
run: |
set -eu
apt-get update -qq
apt-get install -y python3 python3-venv python3-pip
python3 -m venv /tmp/datastack-ci-venv
/tmp/datastack-ci-venv/bin/python -m pip install -e ./sdk/python/datastack -e ./sdk-python
- name: Run all Python SDK tests
apt-get install -y --no-install-recommends git ca-certificates python3 python3-venv python3-pip
- uses: actions/checkout@v4
with:
github-server-url: https://dev.ivib.cc
- name: CI-06 Python 3.11 isolated virtual environment
run: |
set -eu
/tmp/datastack-ci-venv/bin/python -m compileall -q ./sdk/python/datastack ./sdk-python
/tmp/datastack-ci-venv/bin/python -m unittest discover -s tests -p '*_test.py' -v
python3 --version
python3 -m venv /tmp/datastack-ci-venv
/tmp/datastack-ci-venv/bin/python -m pip install --disable-pip-version-check -e ./sdk/python/datastack -e ./sdk-python
- name: CI-06 compile and assert 14 Python SDK regressions
run: |
set -eu
py=/tmp/datastack-ci-venv/bin/python
"$py" -m compileall -q ./sdk/python/datastack ./sdk-python
"$py" -m unittest discover -s tests -p '*_test.py' -v > /tmp/datastack-python-tests.log 2>&1 || { cat /tmp/datastack-python-tests.log; exit 1; }
cat /tmp/datastack-python-tests.log
grep -Eq '^Ran 14 tests in ' /tmp/datastack-python-tests.log
grep -Eq '^OK$' /tmp/datastack-python-tests.log
echo DATASTACK_PYTHON_14_OF_14_PASS
quality:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true' }}
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
@@ -72,6 +101,7 @@ jobs:
run: npm run lint
production-build:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.quality_only != 'true' && (github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true') }}
runs-on: ubuntu-latest
timeout-minutes: 25
steps:
@@ -87,28 +117,45 @@ jobs:
test -f .next/standalone/server.js
docker-image-build:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.p01_only != 'true' && (github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true') }}
runs-on: ubuntu-latest
timeout-minutes: 35
container: node:22-alpine
timeout-minutes: 45
steps:
- name: CI-07 provision Docker Buildx CLI
run: |
set -eu
apk add --no-cache git docker-cli docker-cli-buildx docker-cli-compose
docker --version
docker buildx version
docker compose version
- uses: actions/checkout@v4
with:
github-server-url: https://dev.ivib.cc
- name: Docker and Compose availability
- name: CI-07 inspect daemon and validate Compose without starting services
run: |
set -eu
docker --version
docker compose version
docker info
- name: Validate Compose structure
run: docker compose -f docker/docker-compose.yml config --no-interpolate --quiet
- name: Build Docker images without running services
docker info --format 'Driver={{.Driver}} Root={{.DockerRootDir}}'
df -h .
docker compose -f docker/docker-compose.yml config --no-interpolate --quiet
echo DATASTACK_COMPOSE_CONFIG_PASS
- name: CI-07 isolated BuildKit all three images (no deployment)
run: |
set -eu
docker build -f docker/Dockerfile.server -t datastack-ci-ingestion .
docker build -f docker/Dockerfile.web -t datastack-ci-web .
docker build -f docker/Dockerfile.proxy -t datastack-ci-proxy .
builder="datastack-p03-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}"
tag="datastack-p03-${GITHUB_SHA:-local}"
cleanup() { docker buildx rm --force "$builder" >/dev/null 2>&1 || true; }
trap cleanup EXIT
docker buildx create --name "$builder" --driver docker-container --use
docker buildx inspect "$builder" --bootstrap
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.server -t "$tag-ingestion" .
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.web -t "$tag-web" .
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.proxy -t "$tag-proxy" .
docker image inspect "$tag-ingestion" "$tag-web" "$tag-proxy" >/dev/null
echo DATASTACK_THREE_IMAGES_BUILD_PASS
windows-compatibility:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.quality_only != 'true' && (github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true') }}
runs-on: windows-latest
timeout-minutes: 25
steps:
@@ -127,13 +174,50 @@ jobs:
shell: powershell
run: |
$ErrorActionPreference = 'Stop'
# Diagnostic only: regenerate the lock in the disposable runner checkout.
# The tracked repository lock remains unchanged for the later bug-fix phase.
npm install --package-lock-only --no-audit --no-fund
if ($LASTEXITCODE -ne 0) { throw 'Temporary lock regeneration failed' }
# Verify the committed cross-platform lock without disposable regeneration.
npm ci --no-audit --no-fund
if ($LASTEXITCODE -ne 0) { throw 'Diagnostic npm install failed' }
if ($LASTEXITCODE -ne 0) { throw 'Committed-lock npm ci failed' }
npm --prefix sdk/node/datastack install --no-audit --no-fund
if ($LASTEXITCODE -ne 0) { throw 'SDK install failed' }
npm --prefix sdk/node/datastack run build
if ($LASTEXITCODE -ne 0) { throw 'SDK compilation failed' }
security-audit:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.p01_only != 'true' && (github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true') }}
runs-on: ubuntu-latest
container: node:22-alpine
timeout-minutes: 15
steps:
- name: Install checkout prerequisite
run: apk add --no-cache git
- uses: actions/checkout@v4
with:
github-server-url: https://dev.ivib.cc
- name: CI-08 read-only full and runtime npm advisory inventory
run: |
set -eu
if npm audit --json > /tmp/datastack-audit-full.json; then :; else test "$?" -eq 1; fi
if npm audit --omit=dev --json > /tmp/datastack-audit-runtime.json; then :; else test "$?" -eq 1; fi
node <<'NODE'
const fs = require("node:fs");
for (const scope of ["full", "runtime"]) {
const audit = JSON.parse(fs.readFileSync("/tmp/datastack-audit-" + scope + ".json", "utf8"));
if (!audit.metadata || !audit.metadata.vulnerabilities || !audit.vulnerabilities) {
throw Error("Incomplete npm audit response for " + scope);
}
console.log("AUDIT_COUNTS " + scope + " " + JSON.stringify(audit.metadata.vulnerabilities));
for (const [name, item] of Object.entries(audit.vulnerabilities)) {
const advisories = (item.via || []).filter(v => typeof v === "object").map(v => ({
id: v.source, url: v.url, range: v.range, severity: v.severity
}));
console.log("AUDIT_PACKAGE " + scope + " " + JSON.stringify({
package: name, severity: item.severity, direct: item.isDirect,
range: item.range, via: advisories,
viaPackages: (item.via || []).filter(v => typeof v === "string"),
fixAvailable: item.fixAvailable
}));
}
}
NODE
git diff --exit-code -- package.json package-lock.json
echo DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS
+19 -17
View File
@@ -1,39 +1,41 @@
name: DataStack Docker Build CI
on:
push:
branches: [main]
paths:
- '.gitea/workflows/datastack-docker-ci.yml'
workflow_dispatch:
jobs:
docker-build:
runs-on: ubuntu-latest
container: node:22-alpine
timeout-minutes: 40
timeout-minutes: 45
steps:
- name: Provision isolated Docker CLI and Git tools
- name: CI-07 provision Docker Buildx CLI
run: |
set -eu
apk add --no-cache git docker-cli docker-cli-compose
apk add --no-cache git docker-cli docker-cli-buildx docker-cli-compose
docker --version
docker buildx version
docker compose version
- uses: actions/checkout@v4
with:
github-server-url: https://dev.ivib.cc
- name: Validate actual Compose configuration
- name: CI-07 inspect daemon and validate Compose without starting services
run: |
set -eu
docker info --format 'Driver={{.Driver}} Root={{.DockerRootDir}}'
df -h .
docker compose -f docker/docker-compose.yml config --no-interpolate --quiet
echo DATASTACK_COMPOSE_CONFIG_PASS
- name: Check CI-only Docker daemon
- name: CI-07 isolated BuildKit all three images (no deployment)
run: |
set -eu
docker info >/dev/null || { echo DOCKER_DAEMON_UNAVAILABLE; exit 3; }
- name: Build all DataStack container images without deployment
run: |
set -eu
docker build -f docker/Dockerfile.server -t datastack-ci-ingestion .
docker build -f docker/Dockerfile.web -t datastack-ci-web .
docker build -f docker/Dockerfile.proxy -t datastack-ci-proxy .
echo DATASTACK_DOCKER_IMAGES_PASS
builder="datastack-p03-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}"
tag="datastack-p03-${GITHUB_SHA:-local}"
cleanup() { docker buildx rm --force "$builder" >/dev/null 2>&1 || true; }
trap cleanup EXIT
docker buildx create --name "$builder" --driver docker-container --use
docker buildx inspect "$builder" --bootstrap
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.server -t "$tag-ingestion" .
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.web -t "$tag-web" .
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.proxy -t "$tag-proxy" .
docker image inspect "$tag-ingestion" "$tag-web" "$tag-proxy" >/dev/null
echo DATASTACK_THREE_IMAGES_BUILD_PASS
-2747
View File
File diff suppressed because it is too large. Load diff
+72 -6
View File
@@ -650,6 +650,66 @@ export class DataBufferEngine {
}
const executeFlush = async (): Promise<FlushResult> => {
// P02 R-02: examine CSV partitions before replay of an untrimmed Redis
// batch. Streaming parsing limits memory and handles escaped/newline CSV.
const excludeAlreadyPersisted = async (events: DataEvent[]): Promise<DataEvent[]> => {
if (!events.length) return events;
const fs = await import('node:fs');
const path = await import('node:path');
const zlib = await import('node:zlib');
const { pipeline } = await import('node:stream/promises');
const { Writable } = await import('node:stream');
const { parse } = await import('csv-parse');
const groups = new Map<string, {
clientId: string; date: string; ids: Set<string>;
}>();
for (const event of events) {
const clientId = String(event.clientId || 'unknown');
const date = CSVStorageManager.extractSafePartitionDate(event.timestamp);
const key = JSON.stringify([clientId, date]);
if (!groups.has(key)) groups.set(key, { clientId, date, ids: new Set() });
groups.get(key)!.ids.add(String(event.eventId));
}
const persisted = new Set<string>();
for (const [groupKey, group] of groups) {
const dir = CSVStorageManager.ensureClientDir(group.clientId);
const entries = await fs.promises.readdir(dir, { withFileTypes: true });
for (const entry of entries) {
if (!entry.isFile() ||
!/\.csv(?:\.gz)?$/.test(entry.name) ||
CSVStorageManager.extractSafePartitionDate(
entry.name.replace(/\.csv(?:\.gz)?$/, '').replace(/(?:-(?:part|gz)\d+)+$/, ''),
) !== group.date) continue;
const reader = fs.createReadStream(path.join(dir, entry.name));
const parser = parse({ columns: true, bom: true, skip_empty_lines: true });
const sink = new Writable({
objectMode: true,
write(row: { clientId?: string; eventId?: string }, _enc, done) {
if (row.clientId === group.clientId && group.ids.has(String(row.eventId))) {
persisted.add(JSON.stringify([groupKey, String(row.eventId)]));
}
done();
},
});
if (entry.name.endsWith('.gz')) {
await pipeline(reader, zlib.createGunzip(), parser, sink);
} else {
await pipeline(reader, parser, sink);
}
}
}
const batchSeen = new Set<string>();
return events.filter((event) => {
const key = JSON.stringify([String(event.clientId || 'unknown'),
CSVStorageManager.extractSafePartitionDate(event.timestamp)]);
const rowKey = JSON.stringify([key, String(event.eventId)]);
if (persisted.has(rowKey) || batchSeen.has(rowKey)) return false;
batchSeen.add(rowKey);
return true;
});
};
state.isFlushing = true;
let totalWritten = 0;
let lockTimedOut = false;
@@ -793,9 +853,12 @@ export class DataBufferEngine {
if (lockLost) break;
if (parsedBatch.length > 0) {
const res = await CSVStorageManager.appendBatch(parsedBatch);
totalWritten += res.written;
for (const f of res.filesUpdated) filesSet.add(f);
const unwritten = await excludeAlreadyPersisted(parsedBatch);
if (unwritten.length > 0) {
const res = await CSVStorageManager.appendBatch(unwritten);
totalWritten += res.written;
for (const f of res.filesUpdated) filesSet.add(f);
}
}
if (Date.now() - lastRenewAt >= renewIntervalMs) {
@@ -845,9 +908,12 @@ export class DataBufferEngine {
if (validRamBatch.length > 0) {
try {
const res = await CSVStorageManager.appendBatch(validRamBatch);
totalWritten += res.written;
for (const f of res.filesUpdated) filesSet.add(f);
const unwritten = await excludeAlreadyPersisted(validRamBatch);
if (unwritten.length > 0) {
const res = await CSVStorageManager.appendBatch(unwritten);
totalWritten += res.written;
for (const f of res.filesUpdated) filesSet.add(f);
}
} catch (err) {
state.buffer.unshift(...validRamBatch);
throw err;
+58 -1
View File
@@ -1,6 +1,6 @@
# DataStack — Production-Readiness Forensic Audit (Documentation Baseline)
Status: **DOCUMENTED / NO BUG FIX APPLIED**. Date: 2026-11-22 (Asia/Dhaka).
Status: **BASELINE FORENSIC SNAPSHOT; P01 VERIFIED AND CLOSED (see closure appendix)**. Date: 2026-11-22 (Asia/Dhaka).
Frozen source HEAD: `d725841ebc828d58cbe90f517da9e600f23c4630`. Local, Gitea `vibtools/DataStack`, GitLab `ivibtools/datastack`, and GitHub `JILLU-Games/DataStack` confirmed at this exact HEAD before writing these notes.
Scope of this document: **evidence-backed CI and source review**, not proof that every production feature works. No Coolify live write, rollback, restart or deployment was conducted. Public GET `https://data.ivib.cc/api/health` returned `status: online`; full DB/Redis/MCP/S3 transaction health is not established.
@@ -64,3 +64,60 @@ Optional configuration review (not counted among eight): `server/index.js:1086-1
## Release acceptance rule
NOT production-update-ready until P01, conditional P02 gates, and P03 finish, 24/24 TypeScript suites exit 0, 14/14 Python tests pass, full strict `tsc --noEmit` and ESLint pass, Linux/Windows clean install is reproducible, standalone Next build passes, all 3 Docker images build, security advisories are triaged, and approved **non-mutating** live readiness checks pass. Production deploy is a separate operator-approved action; never automatic from this documentation phase.
## P01 Closure Appendix — 2026-11-22
**Implementation outcome: CI-01…CI-05 resolved within P01 source allowlist; no production app/runtime/UI/SDK changes.** Evidence: Gitea [#59](https://dev.ivib.cc/vibtools/DataStack/actions/runs/59) and focused quality retry [#60](https://dev.ivib.cc/vibtools/DataStack/actions/runs/60). Run #59 overall FAILED because a single test-only `stopTimers` type declaration remained; it was corrected in `tests/r5-phase01-mcp-foundation-logs.test.ts` and strict `tsc --noEmit` plus ESLint then PASSED in #60. The two run results are complementary; neither should be misreported as a single all-green run.
| Finding | Resolution and proof |
|---|---|
| CI-01 | `package-lock.json` gained 68 registry-integrity-verified **optional** entries including nested `@napi-rs/wasm-runtime/node_modules/@tybys/wasm-util@0.10.5`; 1293 original package entries unchanged, no removals or version bumps. Committed-lock `npm ci` PASS on Linux (job 90) and Windows (job 95). |
| CI-02 | Removed obsolete `bun.lock`; `tests/phase07-deployment.test.ts` exit 0. |
| CI-03 | Added suite-specific teardown hooks in four previously hanging suites, without changing core timer/runtime implementation; each exits 0. |
| CI-04 | Replaced stale client-expiry inline-function regex with shared classifier/count behavioral assertions; `tests/v2-phase01-storage-quotas.test.ts` exit 0. |
| CI-05 | 43 historical test-only TypeScript diagnostics corrected; one additional teardown declaration fixed after #59 typecheck failure. Final #60 strict TypeScript PASS (zero errors) and ESLint PASS. |
**Acceptance evidence:** #59 Node SDK build PASS, 24/24 isolated TypeScript suites exit 0 (no timeouts), Windows SDK build PASS, Next.js 15.5.29 standalone production build PASS with 22/22 pages. #60 targeted `tsc --noEmit` and `npm run lint` both PASS. Prior canonical Windows Python SDK 14/14 PASS from [#48](https://dev.ivib.cc/vibtools/DataStack/actions/runs/48); Python source unchanged and not rerun in P01. Docker images and live production behavior NOT verified in P01; P02 runtime reliability risks and P03 infrastructure/security gates remain open.
## P02 Forensic Fault-Injection Update — 2026-11-22
Source freeze for P02: Gitea/GitHub main `667a140c4e9e75382bfdcdbe1d3e9eb96f6e808a`, matching GitLab tracked tree `d53b9e6dd7ff6d291b8bab7b841d8684ef89c703`. R-01/R-02/R-03 originally **risks**, not established production incidents. The diagnostic-only P02 patch adds disposable fixture tests in `tests/phase02-sync.test.ts` (R-01/R-03) and `tests/phase03-storage-buffer.test.ts` (R-02); CI: https://dev.ivib.cc/vibtools/DataStack/actions/runs/63. **Do not interpret diagnostic assertion success as a durability fix.** No runtime code, API, UI, data store, SDK, security policy, deployment or production host has been modified.
- **R-01 durable ACK:** `server/index.js:165-193,1895-1923` catches a failed Redis RPUSH and pushes accepted events into volatile RAM. `server/socket/handlers.js:583-602` then sends `{success:true,eventId}` if `onBufferPush` resolves. In the injected-RPUSH-failure fixture, simulated process-memory reset models loss of the acknowledged event; no actual server is killed. **Potential release blocker: acknowledged data is not necessarily durable.**
- **R-02 CSV/replay:** `server/index.js:981-993` and `core/data/buffer.ts:795-809` write CSV before Redis LTRIM. The fixture injects an LTRIM failure after successful CSV append and retries the same queue; a duplicate eventId write would demonstrate at-least-once rather than exactly-once delivery. This fixture is an analogous web-buffer implementation, **not direct Port-4000 writer fault injection**. **A release decision and exact remediation require independent targeted evidence; do not extrapolate no-duplicates.**
- **R-03 health:** `server/index.js:1926-1957` returns HTTP 200 `status:online` when Redis LLEN fails, while surfacing `redisConnected:false`; the historical route is **liveness**, not dependency readiness. `app/api/health/route.ts:95-111` likewise reports web liveness/buffer and does not prove database/Redis write readiness. A new readiness endpoint or altered public response requires separate approval.
Do not auto-fix source based solely on tests. Runtime code is READ-ONLY under the current approval. Any proven R-01/R-02 risk requiring behavior change needs a new exact method/file approval before application changes. P03 remains independent and unstarted.
## P02 Runtime Remediation Candidate — 2026-11-22 (NOT YET VERIFIED)
Authorized follow-up baseline: Gitea/GitHub `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898` (GitLab same tree). Local isolated branch `p02-runtime-remediation-20261122`, implementation commit `44eee8877e40d1683f44502c2a0ba3d437a03b88`. Scope: **server/index.js** (`pushEventsToRedisOrFallback`, `flushBuffer`), **server/socket/handlers.js** (sync_data/batch_sync ACK and claim rollback), **core/data/buffer.ts** (`flush`), **tests/phase02-sync.test.ts** and **tests/phase03-storage-buffer.test.ts** only, plus notes/001–006. All other source is unchanged.
**Candidate R-01 behavior:** when a configured Redis client returns an RPUSH error, `pushEventsToRedisOrFallback` retains RAM fallback but throws a sanitized `BUFFER_PERSIST_FAILED` instead of returning a durable-looking buffer count. Socket single/batch ACK returns `success:false`; best-effort Redis idempotency-key removal and bounded in-process negative-ACK marker handle ordinary retries. This is a **candidate** pending tests. When Redis is intentionally absent, historical RAM fallback is deliberately unchanged; it is not a universal durable-ACK guarantee. If Redis claim deletion fails and the server restarts, stale Redis idempotency claims could still misclassify a retry; a full crash-atomic claim/queue protocol is **not established** by this patch.
**Candidate R-02 behavior:** Port 4000 `flushBuffer` and Port 3000 `DataBufferEngine.flush` stream-parse existing CSV/CSV.gz partitions for same-client/date/event IDs before replaying any buffered Redis or RAM batch, dropping already-persisted IDs while retaining Redis queue until the write/check completes. Scan/read/parse exceptions fail closed before LTRIM. In-scope test adds a direct Port 4000 writer + Redis LTRIM-failure/recovery test and updates the existing web-buffer replay fixture. **Not proven globally exactly-once:** cross-process CSV writers outside the Redis flush lock and unknown archival/partial-corruption cases remain; scanning partitions on each flush also has real disk-I/O cost, requiring throughput evaluation before production promotion.
**Candidate acceptance:** one disposable Gitea CI with 24/24 suites, strict TS, lint, Linux/Windows clean npm CI and production build; direct Port4000 replay tests; exact allowlist/diff audit and three-remote tree parity. The Gitea MCP/host was unavailable on the implementation attempt, so this is **NOT PASSED** and the P02 phase remains **BLOCKED / NOT CLOSED**. No live deployment or runtime production mutation.
## P02 Final Acceptance Verification — 2026-11-22 (current superseding evidence)
Baseline published diagnosis: `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`. Runtime candidate `76cd972bdeea9ef07b4b38e9d8b5b5ed3312ca5b` passed Gitea CI #66 (24/24 Node regression suites, Node SDK, strict TypeScript, ESLint, Next production build, Windows SDK). New final narrow patch `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532`: `server/socket/handlers.js` plus two previously scoped TS tests only; final disposable Gitea CI #69 dispatched, terminal result to be recorded before phase closure.
Confirmed exact root cause in candidate: `server/socket/handlers.js` stores failed Redis persistence in `idempotencyMap` at `-Date.now()` while `cleanupMaps` deletes entries when `now - timestamp > 300000`; after the 60000ms cleanup window a failed marker can disappear despite an uncommitted Redis idempotency claim, permitting false-positive `DUPLICATE_DROPPED`. Arithmetic reproduction before code patch: at +61000ms, cleanup deletes a marker although Redis claim TTL (300000ms) is not over. Final patch uses a separate positive-timestamp failed-persistence marker and preserves negative ACK while the outstanding claim remains. A mixed batch with a failed duplicate must not return overall success while distinct records persist.
R-02: `server/index.js:843-897,1043-1058` and `core/data/buffer.ts:652-711,855-872` replay-scan existing same-client/day CSV, preserving Redis queue until a successful write/dedupe; fault tests include direct Port 4000 post-CSV LTRIM outage, rotated compressed .csv.gz containing quoted multiline payload and unrelated-date corrupt partition; malformed matching-date CSV must reject and leave Redis queue intact, with recovery after disposable fixture repair. Streaming parsing bounds memory, **not total disk I/O**; scan latency grows with matching partition size. No universal exactly-once guarantee, no live throughput claim and no claim of finite hard disk-I/O cap.
Public R-03 health remains historical liveness. Intentionally Redis-free RAM fallback and its historical successful ACK stay unchanged. No app/UI/auth/API schema/SDK/Docker/production-data/deployment changes. Work only in a new clean temp worktree; original PC repository's staged 16 P01 files and a separate pre-existing dirty candidate worktree preserved. No production readiness claim until CI #69 and final audit.
### P02 Final Focused Correction — CI #69 to #72
Gitea [CI #69](https://dev.ivib.cc/vibtools/DataStack/actions/runs/69) was run against `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532`: the new Redis stale-claim test timed out because it globally replaced `Date.now` during live Socket.IO ACK scheduling. This is a **test-harness failure**, not a proven backend regression; other new CSV.gz, partial-corruption, mixed-new/durable and Redis-free regression cases passed. An additional source-grounded edge case was found: `batch_sync` previously failed to reject a batch containing a non-durable claim PLUS an already accepted duplicate when `processed=0, duplicates>0`. Under exact approved scope, the `server/socket/handlers.js` branch changed from `if (volatileDuplicate && processed > 0)` to `if (volatileDuplicate)`, ensuring false overall ACK is never sent for any such mixture. Test now uses local simulated TTL arithmetic and real-clock Socket.IO retries; no global clock manipulation. Corrected commit `acca8be01a31f8d3972564cbaec8fd3117a0b551` sent to [focused CI #72](https://dev.ivib.cc/vibtools/DataStack/actions/runs/72); still no P02 closure or production deployment until PASS.
### P02 definitive CI acceptance (2026-11-22)
Source-verified P02 final correction is `acca8be01a31f8d3972564cbaec8fd3117a0b551`. [Gitea CI #72](https://dev.ivib.cc/vibtools/DataStack/actions/runs/72) terminal **SUCCESS** (Linux 24/24 TypeScript suites, explicit R-01/R-02 fault cases, strict tsc, ESLint, Windows Node SDK, Next.js standalone build). Earlier #69 was cancelled after a test-only global `Date.now` Socket.IO ACK timeout; fixed by removing global clock override and adding direct arithmetic proof without changing timers. #72 proves the corrected test passes. Final source-path audit: exactly 11 approved paths (3 runtime, 2 tests, 6 notes). No keys/secrets, auth/SDK/UI/public health, dependency pins, CI workflows, deployment or database migrations changed. ACK remains **best-effort in intentionally Redis-free RAM mode**; Redis-configured RPUSH failure must return negative ACK; CSV deduplication guarantees only tested recovery windows, not globally atomic exactly-once. Partition scans are memory-streamed but disk reads scale with matching partition size, and production throughput is NOT certified. Release/installation still requires separate authorization.
+58 -3
View File
@@ -1,7 +1,7 @@
# DataStack — Production Update Roadmap / Exact Scope Lock
Planning baseline: `d725841ebc828d58cbe90f517da9e600f23c4630` (frozen; documentation commits will advance Git HEAD but NOT this source baseline).
Current state: **3 implementation phases P01–P03 planned; 0 implemented; approvals not yet granted**.
Current state: **P01 CLOSED; P02 CI ACCEPTED (Gitea #72 terminal SUCCESS), final three-remote publication gate; P03 NOT STARTED and requires separate explicit approval**. See latest P02 final acceptance appendix and publication report.
Prerequisite for any phase: compare current HEAD and changed paths with this baseline and previously committed notes. For new commits since source baseline, inspect exact diff; no implicit rebase or silent discard.
## Global non-negotiable Scope Lock (applies to ALL phases)
@@ -65,8 +65,63 @@ Prerequisite for any phase: compare current HEAD and changed paths with this bas
## Sequence and approval gate
P01 -> P02 -> P03, strictly in this order. **No P01 implementation has begun** in this documentation-only update.
P01 **CLOSED** -> P02 **approved diagnostic completed / BLOCKED on R-01 and R-02 remediation** -> P03 **must await P02 closure**. P01 acceptance evidence: Gitea CI #59 (Windows + Linux clean install, 24/24 tests, Node SDK, Next standalone) and #60 (strict TypeScript + ESLint), with historical #59 single test-type error corrected before #60. No actual app/runtime/UI/SDK change or live deployment.
Approval command for next turn:
`APPROVED — DataStack Production Readiness P01. Baseline: d725841ebc828d58cbe90f517da9e600f23c4630. Implement ONLY notes/002 P01 exact Scope Lock; preserve all app, UI, SDK, backend, live data, deployment and unrelated files. One focused disposable Gitea CI, exact diff/security audit, notes/003–005 update, one commit, push GitHub/GitLab/Gitea and verify all HEADs. No live deployment, tags, releases or scope expansion.`
`APPROVED — DataStack Production Readiness P02. Follow notes/002 P02 Exact Scope Lock. Verify R-01 Redis ACK durability, R-02 CSV crash/replay duplication, and R-03 liveness/readiness with disposable fault-injection tests. App/runtime source READ-ONLY unless a reproducible bug and separate exact patch approval. Preserve all existing UI, API, auth, SDK, data, and production. One focused CI, forensic diff, notes update, commit, three-remote sync. No deploy, releases, migrations or scope expansion.`
## P02 Execution Scope Addendum — 2026-11-22
P02 approved as **diagnostic-only**. Published P01 baseline: `667a140c4e9e75382bfdcdbe1d3e9eb96f6e808a`. Isolated local worktree: `p02-readiness-test-20261122`, preserving previously staged local P01 source. The **only code edits** are `tests/phase02-sync.test.ts` and `tests/phase03-storage-buffer.test.ts`; six `notes/001–006` documents may be amended. CI #63 uses the existing `.gitea/workflows/datastack-ci.yml`; changing workflow files is **outside P02 Scope Lock**. No runtime/source/API/UI/SDK change is approved. If loss or unbounded replay is reproduced, classify as **BLOCKED pending an independently authorized narrow runtime patch**, not as fixed; do not advance P03 as if P02's production-readiness gate passed. Approval after evidence must name precise methods, event ACK contract, deduplication/CSV recovery strategy and allowed tests before editing runtime.
### P02 Conditional Remediation Approval (NOT CURRENTLY APPROVED)
`APPROVED — DataStack P02-RUNTIME-REMEDIATION. Baseline: use the verified P02 diagnostic commit/remote tree. Fix reproduced R-01 and R-02 only, following exact notes/001–006 proof and scoped server/index.js pushEventsToRedisOrFallback/onBufferPush and flushBuffer, server/socket/handlers.js sync_data/batch_sync ACK, and their three existing P02 test files. First document safe ACK/retry semantics, exactly-once versus at-least-once CSV contract and crash recovery. Add disposable fault tests proving no positive ACK for non-durable data and no unbounded duplicate replay. Runtime sources core/data/buffer.ts and server/workers/csvWriter.js remain READ-ONLY unless a separately justified exact file/line amendment is approved. Preserve all UI/auth/SDK/public health behavior and existing error codes, production data, security, other features and Docker. One focused CI, diff/security audit, notes update, commit and 3-remote parity. No deployment, migrations, tags/releases or scope expansion. If a safe fix cannot be achieved within these exact methods, stop and report the required additional Scope Lock rather than broadening automatically.`
After separate approved remediation and verified P02 closure, **P03 plan** remains CI-06 Linux Python runner provisioning, CI-07 isolated Docker/BuildKit failure remediation and 3-image build, CI-08 supply-chain advisory triage, and non-mutating readiness evidence. P03 should not modify unrelated UI/runtime/API code by default.
## P02 Runtime Remediation Execution Handoff — 2026-11-22
The user explicitly approved **only** R-01 Redis failure ACK and R-02 Redis-to-CSV replay corrections in these methods: `server/index.js` `pushEventsToRedisOrFallback` / `flushBuffer` / existing `onBufferPush` integration; `server/socket/handlers.js` `sync_data` / `batch_sync` ACK handling; `core/data/buffer.ts` `flush` if needed; the three earlier P02 test files; `notes/001–006`. No other application method/UI/API/auth/SDK/Docker/CI-workflow edit. Public R-03 `/health` semantics remain unchanged.
Local candidate `44eee8877e40d1683f44502c2a0ba3d437a03b88` is **not release-ready** until Gitea acceptance. Before closure, prove: negative ACK on Redis failure with no sensitive error leakage; ordinary successful Redis ACK preserved; both single and batch retries don't convert volatile failures to success; Redis queued event replays after crash without writing duplicate CSV rows in the Port 4000 worker and Port 3000 web buffer; CSV read/parse failure never causes premature Redis trim; validate bounded scan overhead and edge cases for rotated/gzipped partitions; all historical 401/403/429/507/503 tests unchanged.
**Stop rule:** if required fixes need touching `validateSingleEventAsync`, `csvWriter.js` or separate journal/storage scheme, do not broaden the implementation scope; record the residual and request new explicit approval. Existing unrelated files including `MEMORY.md` remain unchanged due exact Scope Lock; this note is the cross-IDE handoff. P03 must wait for P02 CLOSED, not merely implementation committed.
## P02-FINAL-ACCEPTANCE-01 Execution / P03 Gate
User approved strict P02 final acceptance after `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898` published diagnosis and `76cd972bdeea9ef07b4b38e9d8b5b5ed3312ca5b` runtime candidate. Previously failed claim is negative timestamp cleaned with positive-timestamp TTL arithmetic; final patch changes only `server/socket/handlers.js` sync_data/batch_sync marker and mixed-batch handling, plus `tests/phase02-sync.test.ts` and `tests/phase03-storage-buffer.test.ts`. It does not edit production startup, public health, schema, CSV format, SDK or dependency files. Candidate CI #66 **terminal SUCCESS**: all 24/24 TypeScript suites, strict tsc, lint, Windows Node SDK and Next build PASS. Final focused CI #69 on `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532` was dispatched; confirm terminal results before any P02 CLOSED claim. No repeated broad CI runs unless this run demonstrates a scoped failure.
**Acceptance contract:** ACK durability applies to Redis-backed events when Redis configured. Intentionally Redis-free RAM mode retains its historical best-effort ACK. CSV replay is at-least-once with per-partition event-id duplicate suppression under tested crash windows, not a global transactional exactly-once guarantee. Existing scan uses streaming parsing (bounded memory) and scans entire matching partitions (disk I/O scales with data size; monitor before production promotion). Corrupt matching CSV fails closed. If a hard upper disk-scan budget is required for production traffic, obtain a dedicated storage/index design approval rather than silently breaking ingestion.
**Next P03 (ONLY after P02 closure):** CI-06 Python runner environment and 14/14 Python SDK; CI-07 three Docker image builds/isolated BuildKit with no live prune; CI-08 read-only npm advisory IDs/direct-transitive audit, compatible dependency pins only if justified; Compose/standalone/health gate evidence and manual release decision. Exact P03 allowlist remains in this roadmap. No live deploy, migrations, tags or releases without separate approval.
### P02 Final CI Exception / Narrow Retry
The final acceptance test-only clock override in #69 interfered with Socket.IO ACK timing. The source-grounded mixed `batch_sync` all-duplicate edge also required a single conditional correction within the previously approved handler. Commit `acca8be01a31f8d3972564cbaec8fd3117a0b551` changes only `server/socket/handlers.js` and `tests/phase02-sync.test.ts` after initial final candidate; one narrow Gitea CI #72 requested because CI #69 node-regression failed one fixture. Other scope/production invariants remain unchanged. P02 closure requires terminal #72 and exact patch/security evidence.
## P02 Final Acceptance Closure Gate / P03 approval handoff — 2026-11-22
P01 CLOSED; **P02 tests/quality/build ACCEPTED via terminal [Gitea CI #72](https://dev.ivib.cc/vibtools/DataStack/actions/runs/72) SUCCESS** on `acca8be01a31f8d3972564cbaec8fd3117a0b551`; P02 phase transitions to CLOSED only when the verified code plus these documentation-only notes are merged to Gitea main and published to GitHub/GitLab with identical tracked Git trees. Terminal jobs: node-regression 24/24 PASS; quality strict TypeScript and ESLint PASS; Windows compatibility SDK PASS; production Next standalone PASS; Python and Docker intentionally skipped (P03). #69 historical test-only clock failure is resolved in #72. A production deployment and all live services remain **NOT VERIFIED** and NOT authorized.
**P03 next / exact future Scope Lock (requires separate user approval):** CI-06 Linux/Python environment: `.gitea/workflows/datastack-ci.yml` and `.gitea/workflows/datastack-python-ci.yml` only; Python SDK 14/14 isolated. CI-07 Docker/BuildKit 3-image verification: `.gitea/workflows/datastack-ci.yml`, `.gitea/workflows/datastack-docker-ci.yml` only, runner evidence read-only; Dockerfiles can be edited ONLY after independently approved exact exception, no shared runner prune. CI-08 advisory triage: read-only `npm audit --json`, actual CVE/advisory and direct/transitive dependency graph; only minimal justified `package.json`/`package-lock.json` pins under this phase scope, never forced audit repair. `.gitea/workflows/datastack-eslint-ci.yml` quality harness only if source-proven CI-08 need. `notes/001–006` for evidence. Existing UI/API/auth/SDK/runtime logic, Compose, volumes, DB data, secrets, real host, and release configuration remain unchanged. One focused acceptance / one exact patch/security audit / one main commit and 3-way tree sync. Do not deploy, run live ingestion, migrate, prune shared Docker, tag or release.
**P03 approval command (not authorization until user explicitly sends):**
```text
APPROVED — DataStack Production Readiness P03.
Use notes/002 P03 CI-06/CI-07/CI-08 Exact Scope Lock and the verified published P02 main baseline.
CI-06: fix Python runner/toolchain only in approved .gitea workflows; verify Python SDK 14/14.
CI-07: isolate Docker runner/BuildKit and build all 3 images without deploying or pruning shared caches; Dockerfiles READ-ONLY unless a separately approved exact exception.
CI-08: read-only npm audit advisory inventory, minimal compatible package pins only for proven security need; no npm audit fix --force.
Preserve UI/UX, app/API/auth/SDK/runtime, Compose, production data, secrets and existing features unchanged.
Use one focused Gitea CI, strict patch/security diff, notes/001–006 update, Git commit and Gitea/GitHub/GitLab tree parity.
No live deployment, migrations, tags or releases.
Close P03 only on verified acceptance; provide production release readiness report.
```
+88 -3
View File
@@ -2,13 +2,13 @@
Frozen source baseline: `d725841ebc828d58cbe90f517da9e600f23c4630`. Generated: 2026-11-22 Asia/Dhaka.
Scope: documentation-only kickoff. Any future phase must update this log **after** verification, not on intention.
**Completed implementation phases: 0 / 3. Remaining: 3.** Documentation baseline preparation is a separate non-implementation milestone.
**P01 CLOSED; P02 verified in Gitea CI #72 and awaiting recorded three-remote publication; P03 NOT STARTED.** Publication/tree-parity proof and final phase-close status are recorded in the latest acceptance appendix and operator report.
| Phase | Status | IDs / function | Changed source | Verified CI evidence | Commit / remote sync |
|---|---|---|---|---|---|
| P00 Notes / freeze | COMPLETED (documentation only) | `001–006` docs, source baseline and exact scope | `notes/**` only | 6 Markdown files; balanced fences; staged diff `--check` PASS | Documentation commit containing this file; remote SHA verification recorded in session handoff |
| P01 Tests & lockfile | NOT STARTED | CI-01,02,03,04,05 | None | Not run | — |
| P02 Ingestion reliability | NOT STARTED | Conditional R-01,02,03 | None | Not run | — |
| P01 Tests & lockfile | **CLOSED — VERIFIED** | CI-01…CI-05, no runtime feature changes | 16 exact P01 code/test/CI paths, plus `notes/001–006` records | #59 Linux/Windows clean npm/SDK, 24/24 TS suites, Next standalone PASS; #60 strict TypeScript and ESLint PASS; historical #59 quality mismatch corrected | One permanent P01 main commit containing these notes; confirm remote SHA/tree parity in phase handoff |
| P02 Ingestion reliability | **CI VERIFIED #72 — final publish gate** | R-01 ACK/idempotency safety; R-02 CSV replay; R-03 liveness verified unchanged | Exactly 3 runtime files, 2 tests, notes/001-006 | #72 terminal SUCCESS: 24/24 TS, TS typecheck, lint, Next standalone, Windows SDK | Final source/test acca8be01a31f8d3972564cbaec8fd3117a0b551; three-remote main parity recorded on publication |
| P03 CI/Docker/security | NOT STARTED | CI-06,07,08; release gates | None | Not run | — |
## Existing diagnostic CI snapshot (NOT implementation phase completions)
@@ -43,6 +43,15 @@ Historical baseline code stayed frozen through these CI workflow changes. Curren
- MEMORY.md cross-IDE handoff text (copyable):
`[DataStack P0X] Baseline=<SHA>; findings=<IDs>; scope=<exact paths>; CI=<run + exit>; commit=<SHA>; remote sync=<all 3>; phase=CLOSED; next=P0Y; production deployment=unchanged.`
## P01 attempted implementation & gate outcome (2026-11-22)
- Frozen source baseline remains `d725841ebc828d58cbe90f517da9e600f23c4630`; latest source HEAD before P01 was documentation commit `d8ac96e39dad4ea3b241c1d2f16f959c5215701e` on all three remotes.
- Disposable CI commit `90fce0629a752dd2916d8d3a2afeaf2c52ed6cbb` contains **16 P01-allowed paths**: `bun.lock` deletion, `package-lock.json` optional/native metadata closure (+67 entries, no existing version or root-manifest change), 14 scoped test files, and `.gitea/workflows/datastack-ci.yml` clean-install/phase-filter changes. No application runtime source/UI/SDK/Docker changes.
- Gitea run: https://dev.ivib.cc/vibtools/DataStack/actions/runs/58; overall **FAIL**, terminal. `windows-compatibility`, `node-regression`, `quality`, `production-build` all failed during clean `npm ci`: `Missing: @tybys/wasm-util@0.10.5 from lock file`. The Python and Docker jobs were intentionally skipped for P01. **No current TypeScript test, strict typecheck, ESLint or build PASS may be claimed**.
- Root issue likely parallel incompatible transitive requirement of `@tybys/wasm-util`; inspect `node_modules/@napi-rs/wasm-runtime` vs `node_modules/@tailwindcss/oxide-wasm32-wasi` in lock and install exact requested transitive version under correct nested path without changing existing pinned entries. Then verify npm clean-install on both platforms and rerun focused P01 acceptance once after repair; do not blindly override lock integrity.
- Local original Windows device's VARUA connection reached monthly call quota; local changes were staged via safe squash on `main` but **no P01 main commit/push** occurred. A separate connected desktop did not have this workspace. Do not claim local/3-remote post-P01 sync until verified.
- **Historical #58 status:** 0/3, P01 BLOCKED before nested lockfile correction. **Superseded by closure evidence below**; no force push or deploy was used.
## Status transition rules
- OPEN -> IN PROGRESS only with explicit approval and baseline SHA match.
@@ -50,3 +59,79 @@ Historical baseline code stayed frozen through these CI workflow changes. Curren
- Failed test -> BLOCKED with log excerpt and scope decision; never log as complete.
- Unexpected file change -> STOP and revert only own documented changes safely; never touch pre-existing work.
- At any time, `actual production deployment status` is a separate tracked dimension (do not infer deployed SHA from Git HEAD).
## P01 Final Closure — CI-01…CI-05
- **Status:** CLOSED after one P01 acceptance CI (#59) plus one narrow quality-only correction/retry (#60); completed implementation phases **1/3**, remaining **2** (P02 and P03).
- **Baseline:** frozen source `d725841ebc828d58cbe90f517da9e600f23c4630`; pre-P01 documentation HEAD `d8ac96e39dad4ea3b241c1d2f16f959c5215701e`.
- **Validated implementation code snapshot:** `7d8eebd3dc867e82113343403b63ce1d4ed20584` on disposable Gitea P01 branch. Permanent main commit is the **single squash commit containing this closure log**; remote SHAs and content-tree equality must be checked at publish, not predicted here.
- **Exact P01 patch:** deletion `bun.lock`; append-only 68 optional/native lock entries with zero original package-version changes; 13 in-scope test files with typed mocks, teardown, and shared-helper assertions; narrow `.gitea/workflows/datastack-ci.yml` clean-install and focused verification inputs. `package.json`, `tsconfig.json`, all app/UI/backend/SDK/Docker/production files unchanged.
- **#58 failed before tests** (`@tybys/wasm-util@0.10.5` missing); fixed by nested, registry-hash-verified optional dependency. **#59 overall FAIL** due exactly one TS2339 test teardown type declaration; the same run separately proved 24/24 TS suite exits 0, Linux/Windows original committed-lock `npm ci`, both Node SDK builds, and Next standalone production build (22/22 pages). Fixed only the test declaration; **#60 quality-only PASS** for `npx tsc --noEmit` and `npm run lint` with no error.
- Prior independent Python SDK 14/14 PASS: #48 (SDK unmodified); not rerun during P01. P03 Docker and security/advisory findings unresolved; no Docker builds, live ingestion, network fault injection, migrations, releases or production deployment occurred.
- **Diff audit:** only allowlisted 16 P01 code/test/CI paths and 6 `notes/*.md` files; `git diff --check` PASS; no unrelated features, API behavior, UI or persistent data changed. 1293 original lock entries remained byte-equivalent; 68 optional entries added with integrity hashes.
- **Working tree caveat:** original local Windows VARUA device reached its monthly quota during P01; its current local branch/working tree could not be rechecked and should be reconciled against published `main` before new local work. A separate authorized temporary Windows clone was used for diff and Git push verification. Never claim original PC is already updated.
- **Next:** P02 read-only initial source audit plus targeted disposable fault-injection tests for R-01 (Redis ACK), R-02 (CSV replay), R-03 (health readiness). Runtime-source change requires reproduced issue and a separate exact method/file approval. P03 only after P02 closure.
- **P02 implementation approval command:** `APPROVED — DataStack P02. Implement only notes/002 P02 diagnostic/test write scope for R-01, R-02 and R-03. Keep app/UI/API/auth/SDK/runtime/data unchanged. Runtime files read-only unless a proven failure and separately approved narrow code patch. One focused disposable Gitea CI, exact diff, notes/001–006 update, commit and remote verification. No production deployment or scope expansion.`
## P02 Execution Log — 2026-11-22 (supersedes P02: pending until acceptance)
- **Status: IN PROGRESS / diagnostic CI pending.** Not CLOSED and not a production-readiness certificate.
- P01 published baseline `667a140c4e9e75382bfdcdbe1d3e9eb96f6e808a`. Original PC local `main` still at prior docs HEAD with **16 user-visible staged P01 paths**, preserved untouched. New isolated `p02-readiness-test-20261122` worktree based on published baseline; no reset/stash/force update.
- P02 disposable test commit: `bbe73d91666641743cbec6cf7203f1694094caef`; **only two files**: `tests/phase02-sync.test.ts` (Redis negative-path ACK and health fallback), `tests/phase03-storage-buffer.test.ts` (failure between CSV append and Redis LTRIM, replay). Source contracts and auth flows unchanged.
- Focused evidence: https://dev.ivib.cc/vibtools/DataStack/actions/runs/63. Record terminal suite/typecheck/build results after completion. Test assertions expose existing behavior; they do not implement durability or readiness fixes.
- Scope audited: `server/index.js`, `server/socket/handlers.js`, `server/workers/csvWriter.js`, `core/data/buffer.ts`, `app/api/health/route.ts` all READ-ONLY; `app/**`, `components/**`, `lib/**`, `sdk/**`, Docker, settings, production data and deployments unchanged.
- P01 remains CLOSED (1/3). If R-01 or R-02 are confirmed unsafe, P02 must remain BLOCKED for runtime remediation approval. No P03 implementation on this approval.
### P02 Verified Diagnostic Gate (R-01/R-02/R-03)
- Gitea #63 (test-only commit `bbe73d91666641743cbec6cf7203f1694094caef`): `P02 R-01 diagnostic` PASS, `P02 R-03 diagnostic` PASS, `P02 R-02 diagnostic` PASS. Each PASS validates reproduction of existing behavior, **not a successful reliability remediation**.
- `tests/phase02-sync.test.ts` suite exit 0; `tests/phase03-storage-buffer.test.ts` suite exit 0. Other CI jobs may have independent status; do not infer full job or complete workflow PASS from three cases.
- **BLOCKING RELEASE FINDINGS:** R-01 success ACK issued for non-durable RAM fallback when Redis RPUSH fails; R-02 identical eventId can append twice on post-CSV Redis-trim failure (web-side representative test). R-03 health remains online on Redis loss; liveness-only behavior is backward-compatible, protected dependency-readiness verification remains missing.
- **P02 PHASE = BLOCKED**, diagnostic implementation done, release reliability acceptance NOT satisfied. Runtime remediation requires separate exact approval. 1/3 phases still CLOSED (P01 only); P02 and P03 remain unclosed. No P03 start authorization from this diagnostic completion.
- Next approval must explicitly authorize exact narrow server/core patch for ACK durability and CSV replay deduplication, plus concrete no-loss/no-duplicate tests. Any readiness endpoint change needs separate explicit approval. User's initial P02 approval is insufficient for runtime mutation.
## P02-RUNTIME-REMEDIATION — Candidate implementation checkpoint (2026-11-22)
- **Current P02: BLOCKED / NOT CLOSED.** One previous diagnostic Gitea CI #63 reproduced R-01/R-02/R-03; runtime remediation was separately approved after that. Verified completed implementation phases remain **1/3** (P01 only). Do not approve P03 yet.
- Published P02 diagnosis baseline `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`; isolated local candidate commit `44eee8877e40d1683f44502c2a0ba3d437a03b88` on `p02-runtime-remediation-20261122`.
- Exactly five code/test files changed in this candidate: `server/index.js`, `server/socket/handlers.js`, `core/data/buffer.ts`, `tests/phase02-sync.test.ts`, `tests/phase03-storage-buffer.test.ts`. No `server/workers/csvWriter.js`, `app/**`, `components/**`, SDK, public R-03 health, CI workflow, database, migration, production data, secrets or deployment touched.
- JS `node --check` on two changed JS files PASS; `git diff --check` PASS and allowlist audit PASS. Targeted TS/Node regression and Gitea CI **NOT YET RUN TO TERMINAL PASS**. The Gitea connector/host was unavailable when dispatch/push was attempted. Local temporary dependency installation also not yet complete.
- The original user PC root `D:\VibTools_Workspace\01_Products\14_DataStack` still has 16 pre-existing staged P01 paths on older `d8ac96e`; deliberately preserved without reset/stash/discard. All P02 implementation is in a separate temporary worktree. Do not assume the old local root is up to date.
- R-01 and R-02 implementations are candidates; practical reliability limitations and I/O cost recorded in `notes/001` and `notes/004`. No false claim of globally exactly-once CSV or universally durable ACK.
- **Next gates:** restore Gitea access; run one focused disposable acceptance; fix any in-scope test/TS defects; record exact CI terminal results; only after required acceptance and security audit, squash single phase main commit and verify GitHub/GitLab/Gitea tree parity; then, and only then, decide P02 CLOSED versus residual-risk blocker.
### Verification environment blocker — local dependency install
The authorized Windows PC attempted a **local disposable** `npm.cmd ci --ignore-scripts --no-audit --no-fund --prefer-offline` in the isolated P02 worktree only; it failed with `npm ERR! ETARGET No matching version found for yaml@2.9.1` plus cleanup EPERM warnings in the temp worktree's `node_modules`. Exact output captured in `%TEMP%\DataStack-P02-npm-ci.log` on the authorized PC. This is **not a passed code test nor proof of a repository package defect**, and `package-lock.json` / `package.json` cannot be edited under P02 scope. Gitea MCP remained UNAVAILABLE and the Git remote/HTTP service was not responding when CI was attempted, so the mandatory P02 focused CI was **not dispatched**. Phase remains BLOCKED pending actual CI evidence; no runtime change was pushed to production/main.
## P02-FINAL-ACCEPTANCE-01 Current Verification (latest record)
- **At this checkpoint: P02 acceptance PENDING CI #69**. P01 CLOSED; P03 NOT STARTED. Do not reinterpret earlier P02 BLOCKED audit as a completed release.
- Gitea #66 on `76cd972bdeea9ef07b4b38e9d8b5b5ed3312ca5b` **COMPLETED SUCCESS**: node-regression 24/24 suites PASS including direct Port4000 replay; quality (strict tsc+ESLint) PASS; production-build PASS; windows-compatibility SDK PASS; Python/Docker skipped by scoped `p01_only` input.
- Final patch `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532` in clean isolated `p02-final-acceptance-20261122`: resolved source-grounded negative-marker TTL bug and mixed-batch success ACK, plus test-only stale-claim time jump (+61s), failed Redis claim, mixed batch, intentionally Redis-free compatibility, rotated .csv.gz+multiline replay and partial-corrupt CSV fail-closed/recovery. No source outside `server/socket/handlers.js` changed since candidate.
- One focused Gitea CI #69 dispatched for final patch; job terminal outcome and any corrective action must be recorded. **Do not merge until this passes and exact patch/security audit is reviewed.**
- Original `D:\VibTools_Workspace\01_Products\14_DataStack` at old `d8ac96e` with 16 staged P01 changes untouched. Separate pre-existing `p02-runtime-remediation-20261122` had three dirty files; left untouched. The new isolated worktree was created clean from candidate SHA.
- Source behavior: no Redis configured = historical RAM best-effort acceptance preserved; configured Redis failure = negative ACK; public /health unchanged. Per-CSV replay scanning bounded-memory but linear in stored bytes; no global exactly-once/throughput guarantee. P03 cannot begin until P02 acceptance/remote parity close.
### P02 Focused Retry — #69 fixture failure / #72 correction
- Final acceptance CI #69 on `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532`: node-regression **FAIL** because `phase02-sync` stale-marker test globally mocked `Date.now` and Socket.IO ACK timed out. Other new tests for gzip/rotated CSV, partial CSV corruption, Redis-free RAM and mixed batch with a new persisted event PASS. Strict TypeScript, lint and Windows SDK jobs PASS; full workflow conclusion must be read at terminal.
- New source-grounded all-duplicate mixed batch path (`processed=0`, `duplicates=1`, `rejected>0`, volatile claim present) could return overall success; added focused test and changed handler conditional to reject all `volatileDuplicate` combinations.
- Test correction removes process-wide Date.now override, verifies 61000ms/300000ms cleanup math with static source marker assertion, and checks real Socket.IO negative retries. No unrelated files, CI workflows, SDK, auth or public health modified.
- Commit `acca8be01a31f8d3972564cbaec8fd3117a0b551` submitted to disposable CI #72; **P02 remains pending / NOT CLOSED** until terminal acceptance and 3-way mirror parity.
## P02 Verified Acceptance and Conditional Publication — 2026-11-22
- **Code acceptance VERIFIED — Gitea CI #72 terminal SUCCESS.** Code SHA `acca8be01a31f8d3972564cbaec8fd3117a0b551`. Node regression exactly `CI_SUITE_SUMMARY total=24 failed=0`, individual `tests/phase02-sync.test.ts` and `tests/phase03-storage-buffer.test.ts` suites exit 0. Quality strict TS+ESLint, Next standalone build and Windows Node SDK jobs all SUCCESS. Python/Docker intentionally skipped in this focused P02 workflow.
- **Root causes fixed:** configured Redis RPUSH outage returns `BUFFER_PERSIST_FAILED` negative ACK rather than volatile RAM success; stale failed idempotency claim retains correct TTL and remains negative through retries; `batch_sync` with old durable duplicates plus failed volatile claim returns `success:false`; CSV-before-Redis-LTRIM replay suppresses previously written same-client/date/eventId across plain/rotated/gz CSV with multiline quoted payload; corrupt matching CSV prevents Redis trim and can be recovered after disposable repair.
- **Explicit limits:** Redis intentionally unconfigured retains historical best-effort RAM ACK; no globally atomic exactly-once promise; scan I/O proportional to matching partition bytes even though memory streaming; public /health continues liveness not dependency readiness; live production unknown. No deployment, migration, secret access, user-data mutation, tag, release or build of production images.
- **Historical testing:** #66 PASS candidate; #69 suite failure (Socket.IO ACK timeout caused by test-only global `Date.now` override) and cancellation; #72 PASS corrected source/test. No false claim that #69 succeeded.
- Final code/diff safety audit: 11/11 approved paths only: `server/index.js`, `server/socket/handlers.js`, `core/data/buffer.ts`, `tests/phase02-sync.test.ts`, `tests/phase03-storage-buffer.test.ts`, plus `notes/001–006`; `git diff --check` and JS syntax PASS. Original PC root old staged P01 (16 paths) and separate dirty earlier temporary worktree preserved.
- **Publication gate:** Gitea PR #3 final verified source + documentation notes; squash-merge with `[skip ci]`, publish to GitHub and GitLab, verify identical tracked Git trees despite potentially different GitLab commit SHA. P02 marked CLOSED only after three-remote content parity; evidence SHA/tree in operator's final publication report. P03 NOT STARTED and requires separate approval.
+57
View File
@@ -42,6 +42,63 @@ Source baseline: `d725841ebc828d58cbe90f517da9e600f23c4630`. This is an **implem
- MCP: disabled by default, read vs write permissions, bearer hashing, IP allowlist, sandbox traversal and symlink reject, no secret emission.
- Do not run these failure injections against `https://data.ivib.cc/`. Use disposable CI fixtures/data directories/Redis containers only.
## P01 dependency-install blocker note (2026-11-22)
Gitea run #58 failed before application error-handling tests because clean npm install found an unresolved transitive lock entry: `@tybys/wasm-util@0.10.5`. No runtime error-handling paths were changed or re-verified in this run. Existing source-level handler inventory remains unchanged; P02 fault-injection gates are **NOT STARTED**.
## Change control
Any new error code, status, SDK public exception or alert flow is **feature/API behavior change**: STOP and request explicit new scope approval. This register is documentation for deciding whether a new handler is warranted, not blanket authorization to add handlers.
## P01 Closure Verification (2026-11-22)
The previous #58 package-lock error was resolved by adding the exact nested optional `@tybys/wasm-util@0.10.5` lock entry without changing the 1293 original locked packages. Gitea #59 clean installs and all **24/24** TypeScript regression suites passed; #59 had a single test-only TS2339 mock-type declaration error, corrected in scope and verified by **#60 strict TypeScript + ESLint PASS**. P01 made **zero changes** to runtime error-handler implementation, response codes, auth, SDK error classification, telemetry behavior or UI. Existing handler inventory remains valid as source observations. P02 R-01…R-03 fault-injection and P03 infrastructure/security remain unverified; do not claim production fault tolerance from P01.
## P02 R-01–R-03 Diagnostic Addendum — 2026-11-22
| Gate | Injected failure / expected observation | Scope / remediation gate |
|---|---|---|
| R-01 Redis durability | Mock `rpush` rejects; in-process websocket `sync_data` ACK may still return `success:true` while the event exists only in RAM. A simulated RAM reset (not killing live services) shows unrecoverable ACKed event if Redis remained empty. | `server/index.js:165-193,1897-1923`, `server/socket/handlers.js:583-602` **READ ONLY**. A correction to ACK/persist/spooling semantics requires explicit approval. |
| R-02 CSV replay | Mock Redis LTRIM fails **after** CSV append. Retrying buffered event may write same `eventId` twice, exposing duplicate amplification under persistent failures. | Test the web-side `DataBufferEngine.flush` CSV-before-trim pattern; direct ingestion writer proof remains separate. Deduplication scheme or exactly-once guarantees require explicit approval. |
| R-03 availability | Redis `llen` rejection leaves Express `/health` online with `redisConnected:false`. Recovery flips the flag while HTTP 200 remains unchanged. | Public 200 is liveness by existing contract; a protected dependency readiness probe/503 is **not implemented** and needs explicit approval. |
Security invariants: do not expose secrets in ACK/errors, keep JWT/auth and 401/403/429/503/507 contracts unchanged, and never run failure-injection against actual Redis, Postgres, production API, live events, or user CSV. Disposable fake Redis and temporary files only. CI #63: https://dev.ivib.cc/vibtools/DataStack/actions/runs/63; results tracked in notes/003.
## P02 Runtime Candidate — Error-path change inventory (pending CI)
| Path | Candidate result | Explicit limitation |
|---|---|---|
| Redis `RPUSH` rejects with configured Redis client | Retains event in volatile RAM as preexisting fallback but returns `BUFFER_PERSIST_FAILED` to Socket.IO via thrown error; **no successful ACK** for that failure; error text does not expose connection details | No-Redis intentional legacy RAM mode unchanged, still not a durable guarantee |
| `sync_data` and `batch_sync` failures | `success:false`, `code:BUFFER_PERSIST_FAILED` and best-effort rollback of previously claimed idempotency keys, negative failure marker for in-process retries | If claim rollback cannot reach Redis and process crashes, durable claim/queue atomicity remains unsolved |
| Redis buffer `LTRIM` rejects after successful CSV append | Subsequent flush checks physically stored CSV IDs and avoids appending matching rows again; Redis is trimmed only after successful check/write | Concurrent independent CSV appenders, compressed archived CSV, corrupt partial rows, and per-flush CSV scan cost require separate proof |
| Legacy public health | **UNCHANGED** HTTP 200 liveness contract and Redis-connected diagnostic field; R-03 readiness is not introduced in this approval | Does not prove full DB/Redis transactional readiness |
Security/API invariants: leave token/JWT/auth/rate limits/quotas/webhooks/MCP/sandbox/UI/SDK unchanged. No production Redis/Postgres/API/CSV mutation. No lockfile/dependency versions or CI workflow changes. CI acceptance remains pending; these are code-level observations, not release certification.
## P02 Final Fault / Recovery Matrix
| Case | Final behavior target and evidence |
|---|---|
| Redis RPUSH rejects with Redis configured | `BUFFER_PERSIST_FAILED` negative ACK, no false durable success; candidate CI #66 verified single/batch, final CI #69 for marker TTL |
| Redis idempotency claim survives failed queue write | Positive-time separate failed marker survives cleanup after +61000ms (below five-minute TTL); two retries remain negative rather than `DUPLICATE_DROPPED` success. Final disposable fixture `tests/phase02-sync.test.ts` |
| Mixed `batch_sync` contains stale claim and a new durable record | Overall response `success:false` with `BUFFER_PERSIST_FAILED`, even if some new records committed; preserve counts and avoid success cover-up |
| Redis deliberately disabled | Historical RAM mode unchanged; distinguish this existing best-effort contract from configured Redis durability |
| CSV append success but Redis LTRIM failure | Replay scans written event IDs and does not duplicate physical row before trim; direct Port4000 and web-buffer regression |
| Rotated / gzip CSV, quoted multiline payload | Streaming CSV parser, exact client/date/eventId match, no duplicate append or data-format change |
| Matching CSV partial/corrupt | Fail closed (no Redis trim); disposable repair allows retry; do not touch real user CSV |
| Public /health & 401/403/429/503/507 | Preserve existing behavior; no new endpoint or error contract beyond already approved BUFFER_PERSIST_FAILED |
**Limitations:** no globally atomic Redis+CSV transaction, stale Redis claim after process crash may require TTL/retry policy, and per-partition scan has data-size-proportional I/O even though memory is streamed. No live production reliability or throughput certification. CI #66 SUCCESS; final CI #69 pending.
### Additional P02 final-acceptance negative ACK case
If a `batch_sync` contains only (a) an uncommitted Redis-claim duplicate and (b) an ordinary already-durable duplicate, the old `processed===0 && duplicates===0 && rejected>0` test did not trigger; success ACK incorrectly masked volatile claim. Narrow correction uses `if (volatileDuplicate)` before the generic aggregate success path, independent of `processed` or `duplicates`. Specific test in `tests/phase02-sync.test.ts` injects Redis RPUSH failure and fake stale claim then includes the existing durable duplicate. All pending-claim ACKs must remain `BUFFER_PERSIST_FAILED`, even if some other events already committed. Terminal CI #72 pending; no global 5-minute clock advancement inside sockets.
### P02 terminal CI #72 error handling acceptance
Gitea #72 **SUCCESS** at verified final code SHA `acca8be01a31f8d3972564cbaec8fd3117a0b551`; all 24 TS suites exit zero including stale claim, mixed batch (both with new durable event and old durable duplicate), Redis-free compatibility, gzip/rotated CSV and corrupt matching CSV fail-closed. `BUFFER_PERSIST_FAILED` is negative-ACK response on configured Redis write failure, rather than success for RAM fallback. Redis-free RAM mode old semantics unchanged. Existing 401/403/429/503/507 tests unaffected and pass within 24 suites. Public /health liveness remains HTTP 200 with `redisConnected:false` during mocked outage; separate readiness endpoint not approved. Remaining risk: no atomic Redis+CSV commit, no global exactly-once guarantee, CSV partition scan disk I/O grows with volume. No live traffic test.
+36 -2
View File
@@ -41,6 +41,40 @@ After EACH P01/P02/P03 close, amend only affected table rows:
Do not mark "actual working" simply because a patch compiled. Distinguish source, CI, and actual deployment.
## Change log
- P00: No feature added/modified. `notes/**` documentation only. 0/3 implementation phases completed.
- P01: pending.
- P02: pending.
- P01: **CLOSED — 1/3 implementation phases**. CI-01…05 corrected exclusively in package-lock, obsolete bun.lock removal, scoped regression/type tests and Gitea CI harness. **No actual application, UI, backend, API, SDK or production functionality was added, removed or altered.** Gitea #59: committed-lock Linux/Windows clean install PASS; Node SDK builds PASS; all 24 TS suites exit0 with zero timeouts; Next standalone 22/22 pages PASS. One historical #59 test-only declaration TS error fixed; #60 strict TS zero errors and full ESLint PASS. Previous canonical Python SDK 14/14 (#48) preserved but not rerun. No live endpoint mutation or deployment; live data/Redis/S3/MCP actual operational status is not established.
- P02: **BLOCKED pending separate approved runtime remediation**. Three disposable fault-injection tests demonstrated existing behavior: positive ACK on Redis write failure with RAM-only data (R-01), duplicate appended event ID after Redis trim failure and retry in web-side buffer (R-02), and public health 200/online when Redis is down (R-03). These are **diagnostic tests, not production fixes**. Zero app/API/UI/SDK/runtime changes, no live deploy. CI #63. P01 remains closed (1/3 phases complete).
- P03: pending.
## P02 Diagnostic Feature Record — 2026-11-22
- **New production features: NONE.** This phase adds only three disposable failure-injection scenarios across two existing test suites; tests are not production handlers. Existing source/UI/auth/backend/SDK/MCP, Redis/PG/CSV services, Next/Express health interfaces and deployment remain unmodified.
- R-01: existing volatile RAM fallback and WS ACK behavior audited, *not corrected*. Do not mark ACK durability as working merely because a diagnostic test proves current behavior.
- R-02: existing CSV-before-Redis-trim order audited, *not corrected*. Exactly-once replay safety is **NOT VERIFIED**.
- R-03: current public Express health 200 plus `redisConnected:false` is liveness, not dependency readiness. Separate readiness remains **NOT IMPLEMENTED**.
- Gitea disposable CI #63: https://dev.ivib.cc/vibtools/DataStack/actions/runs/63. The previous P01 working results remain valid; no live read/write test or production deploy is included in P02.
## P02 Runtime Candidate Feature Delta — 2026-11-22 (NOT RELEASED)
- **Intended change:** failure cases on socket ingestion report `BUFFER_PERSIST_FAILED` instead of positive ACK when Redis RPUSH rejects; single and batch code paths tested by new disposable assertions. Ordinary success ACK, auth, limits and SDK public interfaces intended unchanged.
- **Intended change:** Redis replay flush on both ingestion/Next paths inspects existing client CSV partitions and filters already-stored event IDs before reappend; safe failure leaves Redis queue untrimmed. This prevents the tested post-append/LTRIM replay duplication but does **not** prove transactionally exactly-once delivery or zero performance impact.
- **No changes:** application UI/UX, auth, SDK, database schema, Docker, environment, public R-03 health, other features and production runtime. No deployment or live verification.
- **Status:** P02 code candidate locally committed, acceptance CI blocked by Gitea unavailability; **NOT CLOSED**, not production-ready. P01 remains only completed phase.
## P02 Final Feature/Compatibility Register — 2026-11-22
Existing UI/UX, admin auth, SDK public signatures, health route, database schema, Docker and deployment: **UNCHANGED**. Only existing Socket.IO single/batch persistence failure ACK/idempotency cleanup, Redis->CSV replay duplicate filtering and disposable tests changed during P02. No new production feature, public endpoint, release or migration. CI #66 terminal SUCCESS (24/24 TS suites, strict types, ESLint, Windows Node SDK, Next production build); final CI #69 pending. Client-facing `BUFFER_PERSIST_FAILED` error is used only for previously incorrectly acknowledged Redis-backed failure; intentionally Redis-free RAM mode is preserved. CSV write content/format unchanged, gzip and rotation replay acceptance covered in final test fixture. All live status values are **NOT VERIFIED / NO DEPLOYMENT**. No guarantee of global exactly-once delivery or hard bound on disk scans; future throughput benchmark/design approval if required. P03 still gated on final P02 close.
### P02 final acceptance correction checkpoint
Gitea #69 node-regression observed one **fixture timeout**, not a proven new production feature defect, caused by altering global `Date.now`. A separate source-proven mixed batch all-duplicate ACK bug was corrected only in `server/socket/handlers.js`, test case added in existing allowed TS file. Revised `acca8be01a31f8d3972564cbaec8fd3117a0b551` pending focused Gitea CI #72. Existing UI/SDK/auth/public health remain unchanged. This is not evidence that production has been deployed or live-tested.
### P02 verified feature-state reconciliation — Gitea #72 SUCCESS
- **P02 R-01: FIXED in source and CI-PASS, conditional scope.** Configured Redis queue RPUSH rejects => `BUFFER_PERSIST_FAILED` negative ACK for single/mixed batches, stale Redis idempotency claim cannot be reported as durable success; clock arithmetic and actual negative ACK retries verified. Intentionally Redis-free RAM mode historical best-effort ACK **preserved**, not a lossless guarantee.
- **P02 R-02: FIXED for verified crash/replay windows**, not global transactionality. Port4000 and Port3000 CSV replay suppresses duplicate same-client/day/eventId across plain, rotated and gz; failed reads prevent trimming; dedicated regression tests PASS. Streaming parser bounded memory, disk read throughput/latency not yet load-tested.
- **P02 R-03: UNCHANGED**: public /health remains backwards-compatible liveness, not a dependency-readiness endpoint. Production live health **NOT VERIFIED**.
- All other runtime features/UI/auth/API/SDK/database/Compose and package dependencies unchanged. **P02 CI ACCEPTED at #72**; code SHA `acca8be01a31f8d3972564cbaec8fd3117a0b551`. Phase closed only after verified 3-way content sync. No P03 implementation or production deployment.
+136
View File
@@ -151,3 +151,139 @@ return NextResponse.json({ status: 'online', service: 'DataStack Web API', /* ..
## C. Documentation baseline update record
This file **freezes the source baseline**, not the future documentation-commit SHA. After adding `notes/**`, record commit SHA and three remote SHAs in `notes/003` and the user-facing result. Subsequent P01 implementation should inspect `git diff d725841ebc828d58cbe90f517da9e600f23c4630..HEAD` and permit only these notes as intervening changes; any unexpected source edits require a new review.
## D. P01 Actual Patch & CI Closure Proof — 2026-11-22
Frozen source baseline remains `d725841ebc828d58cbe90f517da9e600f23c4630`, docs baseline `d8ac96e39dad4ea3b241c1d2f16f959c5215701e`. CI-verified implementation snapshot `7d8eebd3dc867e82113343403b63ce1d4ed20584` (Gitea disposable P01 branch). Permanent main closure is single squash commit containing this note; verify post-push three remote refs/content trees.
### Lock and package integrity evidence (CI-01 / CI-02)
```json
{
"node_modules/@napi-rs/wasm-runtime/node_modules/@tybys/wasm-util": {
"version": "0.10.5",
"resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.5.tgz",
"integrity": "sha512-dOGOPxlaNV+G046mZzN+9C0dAcQZeSKcXiVaM0TYy/LM4ttTmgSBDxYfovcYI8Em3+eAbiAYpvWLOv8+ZVRJKA==",
"optional": true,
"dependencies": {"tslib": "^2.4.0"}
}
}
```
The root `@tybys/wasm-util@0.9.0` remains unchanged; 68 optional entries added, 1293 old entries unchanged, none removed/repinned, all added entries with registry tarball and integrity, `package.json` unchanged. `bun.lock` removed after checking no active consumers. Windows and Linux `npm ci --no-audit --no-fund` PASS using committed lock without temporary regeneration.
### Test-only teardown correction (CI-03 / CI-05)
`tests/r5-phase01-mcp-foundation-logs.test.ts` local CommonJS require declaration now explicitly includes `stopTimers`, matching the existing runtime export; `test.after` releases ingestion timers and `DataBufferEngine.shutdown()` without any changes to runtime source:
```ts
const ingestionServer = require("../server/index.js") as {
setDbPoolForTests: (pool: unknown) => void;
syncSettingsFromDatabase: () => Promise<boolean>;
stopTimers: () => void;
};
test.after(async () => {
ingestionServer.stopTimers();
SDKManager.stopActivityFlushTimerForTests();
await DataBufferEngine.shutdown();
});
```
Remaining CI-05 corrections were confined to 9 test file type contracts; CI-04 replaced old `isClientExpired` page regex expectations with `classifyUiClientStatus` + `summarizeUiClientCounts` behavioral assertions. No TS strictness flags disabled, no global `any` blanket suppression, no SDK/API runtime signature changed.
### Exact gate results
- [Run #58](https://dev.ivib.cc/vibtools/DataStack/actions/runs/58): failed at lockfile nested dependency before tests; historical and superseded.
- [Run #59](https://dev.ivib.cc/vibtools/DataStack/actions/runs/59): overall failed because exactly one test mock TypeScript declaration lacked `stopTimers`, but separate job results independently verified **24/24 TypeScript suites exit0**, no timeouts, Linux/Windows committed-lock clean npm installs, Node SDK compilation and Next.js standalone build + 22/22 pages all PASS.
- [Run #60](https://dev.ivib.cc/vibtools/DataStack/actions/runs/60): focused quality-only retry after exact test-type correction, **strict `npx tsc --noEmit` PASS 0 errors** and **`npm run lint` PASS**.
- [Run #48](https://dev.ivib.cc/vibtools/DataStack/actions/runs/48): previous Python SDK 14/14 PASS, no Python source edits or P01 rerun.
Targeted path/diff audit from prior documentation HEAD: 16 scoped code/test/CI paths plus notes `001–006`; `git diff --check` PASS. P02 remaining R-01…R-03 are conditional reliability risks, not confirmed application bugs. P03 Docker/Python-runner/advisory issues still open. No live app, deployment, database, security defaults or user features changed.
## E. P02 Concrete Code/CI Evidence — 2026-11-22
**Frozen previous phase:** P01 Gitea/GitHub commit `667a140c4e9e75382bfdcdbe1d3e9eb96f6e808a`. P02 test-only commit `bbe73d91666641743cbec6cf7203f1694094caef` adds 103 lines across exactly two approved test paths. Gitea CI #63: https://dev.ivib.cc/vibtools/DataStack/actions/runs/63. No application source modification.
### R-01 ACK after Redis RPUSH failure
`server/index.js:171-192`:
```js
try { bufferLen = await redis.rpush(REDIS_BUFFER_KEY, ...serialized); }
catch { redisConnected = false; }
// ... Redis failure path
bufferQueue.push(...events);
return bufferQueue.length;
```
`server/socket/handlers.js:583-602` invokes `await onBufferPush()` and then calls `callback({success:true,eventId})` unless an exception escapes. Test: `tests/phase02-sync.test.ts`, `P02 R-01 diagnostic`, overriding fake Redis `rpush` to reject; check ACK, absent Redis persistence, volatile RAM, and simulated reset. No real process exit.
### R-02 replay after append-before-trim
`core/data/buffer.ts:795-809` and Port-4000 `server/index.js:981-993` order `appendBatch()/writeBatch()` before `redis.ltrim()`. Test: `tests/phase03-storage-buffer.test.ts`, `P02 R-02 diagnostic`; inject the *first* LTRIM failure only after successful append, verify still queued, retry flush and count duplicate `eventId` in appended rows. Web-side representative fault-injection; not a complete ingestion-crash certificate.
### R-03 public status versus dependency readiness
`server/index.js:1926-1957` catches Redis LLEN error, sets `redisConnected=false`, and returns HTTP 200 `status:online`. Test `P02 R-03 diagnostic` mocks LLEN failure and recovery. This preserves existing public shape rather than implementing a readiness endpoint. PG startup fail-closed coverage remains source-only/NOT VERIFIED.
**Patch scope:** modified only `tests/phase02-sync.test.ts` and `tests/phase03-storage-buffer.test.ts` (plus notes/001–006). Original local user-staged P01 work preserved. Runtime files examined read-only. No production host, Redis, DB, CSV, API, service, CI workflow, release, or security setting changed.
### CI #63 observed P02 results
- `P02 R-01 diagnostic: Redis RPUSH failure yields positive WS ACK for volatile RAM-only event` — **ok 15**, test suite `phase02-sync` exit 0. Diagnostic indicates acknowledged event loss on simulated RAM reset when Redis has no record.
- `P02 R-03 diagnostic: Redis query failure leaves public ingestion health 200/online` — **ok 16**, same suite exit 0, Redis flag false while liveness stays online.
- `P02 R-02 diagnostic: post-CSV Redis trim failure replays identical event on retry` — **ok 18**, `phase03-storage-buffer` suite exit 0; duplicate append after retry reproduced on representative web buffer path. **Direct ingestion Port 4000 CSV worker crash coverage remains outstanding** and should be requested in remediation approval before guarantee changes.
- These three diagnostic tests **prove findings, do not close production-readiness**. See Gitea https://dev.ivib.cc/vibtools/DataStack/actions/runs/63; never call run successful until terminal conclusion is confirmed. No server-side source fix applied.
## F. P02 Remediation Candidate Patch Proof — 2026-11-22
Base: Gitea/GitHub published P02 diagnostic tree `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`; GitLab distinct commit, same previous tree. Local candidate commit `44eee8877e40d1683f44502c2a0ba3d437a03b88` on isolated worktree. Initial exact delta: 5 approved source/test paths, 310 insertions and 22 deletions, no application UI, Python SDK, auth, public health route, CI or deployment edit. `git diff --check` and `node --check` on JS files PASS.
- `server/index.js:pushEventsToRedisOrFallback`: catch RPUSH failure, keep previous RAM buffering, throw sanitized `BUFFER_PERSIST_FAILED`; no positive ACK for that branch.
- `server/socket/handlers.js:sync_data/batch_sync`: negative ACK and best-effort claimed idempotency-key rollback with negative retry guard, no change to legitimate duplicate success path or existing auth/limits/error codes.
- `server/index.js:flushBuffer` and `core/data/buffer.ts:flush`: scoped streaming CSV/CSV.gz read-before-reappend (parsing quoted/multiline rows), exact client/date/event ID match; failures abort before Redis LTRIM. No changes to `server/workers/csvWriter.js`.
- `tests/phase02-sync.test.ts`: R-01 single/batch outage and duplicate claim ACK negative, successful recovery ACK, and unchanged R-03 liveness.
- `tests/phase03-storage-buffer.test.ts`: web-side replay verifies exactly one physically stored row; new direct Port 4000 writer Redis-trim failure/recovery proof. All tests on fake Redis and disposable temp CSV paths.
**Unverified:** Gitea P02 implementation acceptance was unavailable due inaccessible host/connector. Local `npm ci` not confirmed successful. Do not invent a CI run ID or claim full PASS. Full cross-process transactional exactly-once and all Redis claim crash windows are out of proof scope and may require exact additional approval. The streaming dedupe scan changes I/O cost per flush; performance validation before production is required.
### Blocked acceptance evidence — actual tool outcomes
- JS syntax: local `node --check server/index.js` and `node --check server/socket/handlers.js` PASS.
- Source scope: local `git diff --check` PASS; only the five allowed P02 runtime/test paths changed before notes updates.
- Gitea: connection returned `UNAVAILABLE: McpServerError: Connection failed`; Git remote `dev.ivib.cc` unresponsive. No Gitea CI run ID exists for candidate `44eee8877e40d1683f44502c2a0ba3d437a03b88`.
- Local Windows npm: `npm ci` failed `ETARGET yaml@2.9.1` and cleanup `EPERM` in the isolated worktree, **not** a TypeScript/test result. Dependency changes are out of P02 scope.
- Therefore neither TS typecheck nor regression-suite remediation acceptance is established; keep P02 BLOCKED and do not advance P03 or alter any protected runtime/source beyond this approval.
## F. P02 Final Acceptance Patch Index (current source/CI proof)
- Published diagnosis Gitea/GitHub main: `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`; previous GitLab main same tracked tree.
- Runtime candidate: `76cd972bdeea9ef07b4b38e9d8b5b5ed3312ca5b`; Gitea [CI #66](https://dev.ivib.cc/vibtools/DataStack/actions/runs/66) **terminal SUCCESS** across node 24/24, strict TS, lint, Next standalone and Windows Node SDK.
- Exact supplemental commit: `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532` (3 changed files: `server/socket/handlers.js`, `tests/phase02-sync.test.ts`, `tests/phase03-storage-buffer.test.ts`). Compared with published diagnosis, final candidate also includes runtime `server/index.js`, `core/data/buffer.ts`, and previously modified tests, all authorized, plus `notes/001–006`.
- **R-01 arithmetic reproduction** before patch: `const marker=-t; const now=t+61000; now-marker > 300000` is true, while `now-t > 300000` is false. `server/socket/handlers.js:142-158` `cleanupMaps` expires old negative marker; `validateSingleEventAsync:293-317` sees retained Redis idempotency claim; `sync_data` returned duplicate success. Final marker is `\u0000failed-persist:<clientId>:<eventId>` with positive timestamp, corrected age check, cleanup after true TTL, clear marker after durable ACK. Mixed batches reject overall positive ACK while a failed claim remains.
- **R-02:** `server/index.js:843-897,1043-1058`, `core/data/buffer.ts:652-711,855-872`: stream-parsed same-day client CSV/CSV.gz replay suppression, then Redis trim. Focused tests inject direct Port4000 LTRIM crash and compressed part2 quoted/multiline replay; malformed matching partition causes safe no-trim, fixture repair recovers. Memory streamed, partition disk-I/O remains proportional; not globally exactly-once.
- **Compat:** Redis-free RAM ACK and old public /health unchanged; other HTTP/Socket auth, SDK and quota routes unaffected.
- Final [Gitea CI #69](https://dev.ivib.cc/vibtools/DataStack/actions/runs/69) dispatched against exact supplemental commit; until terminal result, phase stays pending and no remote publish.
### Final P02 CI #69 failure and exact retry patch
- CI #69: `phase02-sync.test.ts` subtest `stale Redis claim never becomes positive ACK after cleanup window` `ERR_TEST_FAILURE: operation has timed out` after replacing the **global** `Date.now` during Socket.IO ACK. This cannot support a server-failure claim; it was a test harness flaw. Other new replay/compatibility assertions PASS and quality/Windows jobs separately passed.
- Discovered source branch in `server/socket/handlers.js`: `if (volatileDuplicate && processed > 0)` allowed an overall `success:true` response with pending non-durable claim and already-durable duplicate when `processed=0,duplicates>0`. Revised `if (volatileDuplicate)` blocks every mixed response with an uncommitted claim; new test reproduces the exact combination.
- Test no longer overrides Date.now; independent simulated arithmetic proves the original negative-marker expiry mistake; live ACK checks run on real clock. Supplemental commit `acca8be01a31f8d3972564cbaec8fd3117a0b551`; [CI #72](https://dev.ivib.cc/vibtools/DataStack/actions/runs/72) must pass before P02 closure. No modification of package/dependency files, test CI workflow, public API, UI or production.
## G. P02 Final Acceptance — authoritative terminal CI / Scope Lock proof
- Source baseline published diagnosis `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`. Fixed runtime candidate `76cd972bdeea9ef07b4b38e9d8b5b5ed3312ca5b`, previous #66 SUCCESS. Final corrected **source/test** SHA `acca8be01a31f8d3972564cbaec8fd3117a0b551` fully verified in [Gitea #72](https://dev.ivib.cc/vibtools/DataStack/actions/runs/72), **terminal COMPLETED SUCCESS**, Node `CI_SUITE_SUMMARY total=24 failed=0`. CI job conclusions: node-regression SUCCESS, quality SUCCESS, windows-compatibility SUCCESS, production-build SUCCESS; python-regression and docker-image-build SKIPPED by narrow dispatch.
- Exact reproducible Redis idempotency bug: old `-Date.now()` failed timestamp ages incorrectly at +61000ms, while 300000ms Redis claim TTL has not elapsed. New `\u0000failed-persist:${claimKey}` positive timestamp marker and corrected TTL comparison isolate the failed claim. Live Socket.IO retries remain negative ACK; no process-wide clock monkeypatch. Mixed all-duplicate batch also rejects pending claim instead of sending global positive ACK.
- CSV fault evidence: `tests/phase03-storage-buffer.test.ts` after CSV append but before Redis LTRIM, direct Port4000 writer, rotated `.csv.gz`, JSON multiline-quoted payload, corrupt matching partition no-trim/recovery all PASS. Test assertions cannot prove a global exactly-once protocol or live throughput cap. Full-partition scan is streaming bounded-memory, but data-size-linear disk I/O.
- Historical #69 failed one test-only global Date.now-induced Socket.IO timeout, **not** a demonstrated runtime regression; corrected #72 PASS. No secret or credential introduction, no production data access.
- Final audit `git diff --check` PASS and source JS syntax checks PASS; exact tracked source diff from diagnosis = 11 allowlisted files (3 runtime + 2 test + 6 notes); no unrelated UI/auth/SDK/public health, DB schema, package metadata, Docker, CI workflow or live deployment.
- Git release gate: final docs-only commits after tested source SHA add **no source/test differences**. Gitea PR #3 squashed into one `[skip ci]` main commit; GitHub/GitLab tracked Git trees must match Gitea before final CLOSED. If GitLab commit SHA differs because of parent history, compare tree SHA not commit SHA. Store exact remote SHAs/trees in phase close operator report; do not invent them before push.
+1016
View File
File diff suppressed because it is too large. Load diff
+74 -6
View File
@@ -165,6 +165,7 @@ function setRedisClientForTests(client) {
async function pushEventsToRedisOrFallback(events) {
if (!events || events.length === 0) return bufferQueue.length;
const redis = getRedisClient();
let persistFailed = false;
if (redis) {
const serialized = events.map((e) => JSON.stringify(e));
let bufferLen = null;
@@ -173,6 +174,7 @@ async function pushEventsToRedisOrFallback(events) {
redisConnected = true;
} catch {
redisConnected = false;
persistFailed = true;
}
if (bufferLen !== null) {
@@ -189,6 +191,11 @@ async function pushEventsToRedisOrFallback(events) {
}
bufferQueue.push(...events);
if (persistFailed) {
// Keep the volatile fallback for later recovery, but never ACK it as durable.
// Suppress sensitive Redis transport details in the client-facing error.
throw new Error('BUFFER_PERSIST_FAILED: Redis unavailable; event not durably queued');
}
return bufferQueue.length;
}
@@ -834,6 +841,61 @@ async function isClientDeletedForFlush(clientId, redis, cache) {
}
async function flushBuffer(options = {}) {
// P02 R-02: Redis and CSV cannot be committed atomically. Before replaying an
// untrimmed Redis batch, check actual CSV partitions for previously written IDs.
// Streaming parsing bounds memory and preserves multiline/quoted CSV fields.
const excludeAlreadyPersisted = async (events) => {
if (!events.length) return events;
const fs = require('node:fs');
const path = require('node:path');
const zlib = require('node:zlib');
const { pipeline } = require('node:stream/promises');
const { Writable } = require('node:stream');
const { parse } = require('csv-parse');
const groups = new Map();
for (const event of events) {
const clientId = String(event.clientId || 'unknown');
const date = writer.extractSafePartitionDate(event.timestamp);
const key = JSON.stringify([clientId, date]);
if (!groups.has(key)) groups.set(key, { clientId, date, ids: new Set() });
groups.get(key).ids.add(String(event.eventId));
}
const persisted = new Set();
for (const [groupKey, group] of groups) {
const dir = writer.ensureClientDir(group.clientId);
const entries = await fs.promises.readdir(dir, { withFileTypes: true });
for (const entry of entries) {
if (!entry.isFile() || !writer.isPartitionFilename(entry.name) ||
writer.extractDateFromFilename(entry.name) !== group.date) continue;
const reader = fs.createReadStream(path.join(dir, entry.name));
const streams = [reader];
if (entry.name.endsWith('.gz')) streams.push(zlib.createGunzip());
streams.push(parse({ columns: true, bom: true, skip_empty_lines: true }));
streams.push(new Writable({
objectMode: true,
write(row, _enc, done) {
if (row.clientId === group.clientId && group.ids.has(String(row.eventId))) {
persisted.add(JSON.stringify([groupKey, String(row.eventId)]));
}
done();
},
}));
// On read/parse failure, abort without LTRIM: never discard uncertain rows.
await pipeline(...streams);
}
}
const batchSeen = new Set();
return events.filter((event) => {
const key = JSON.stringify([String(event.clientId || 'unknown'),
writer.extractSafePartitionDate(event.timestamp)]);
const rowKey = JSON.stringify([key, String(event.eventId)]);
if (persisted.has(rowKey) || batchSeen.has(rowKey)) return false;
batchSeen.add(rowKey);
return true;
});
};
if (pendingRedisWrites.size > 0) {
await Promise.allSettled([...pendingRedisWrites]);
}
@@ -979,9 +1041,12 @@ async function flushBuffer(options = {}) {
if (lockLost) break;
if (parsedBatch.length > 0) {
const res = await writer.writeBatch(parsedBatch);
totalWritten += res.written;
for (const f of res.files) filesSet.add(f);
const unwritten = await excludeAlreadyPersisted(parsedBatch);
if (unwritten.length > 0) {
const res = await writer.writeBatch(unwritten);
totalWritten += res.written;
for (const f of res.files) filesSet.add(f);
}
}
if (Date.now() - lastRenewAt >= renewIntervalMs) {
@@ -1022,9 +1087,12 @@ async function flushBuffer(options = {}) {
if (validBatch.length > 0) {
try {
const res = await writer.writeBatch(validBatch);
totalWritten += res.written;
for (const f of res.files) filesSet.add(f);
const unwritten = await excludeAlreadyPersisted(validBatch);
if (unwritten.length > 0) {
const res = await writer.writeBatch(unwritten);
totalWritten += res.written;
for (const f of res.files) filesSet.add(f);
}
} catch (err) {
bufferQueue.unshift(...validBatch);
throw err;
+75 -7
View File
@@ -549,6 +549,9 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
: new Date().toISOString();
const eventData = extractSingleEventData(payload);
// A previous Redis write failure can leave an uncommitted idempotency
// claim. Never report that claim as a successful durable duplicate.
const failedAt = idempotencyMap.get(`\u0000failed-persist:${clientId}:${eventId}`);
const check = await validateSingleEventAsync(
clientId,
eventId,
@@ -557,7 +560,13 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
);
if (!check.allowed) {
if (check.duplicate) {
if (callback) {
if (typeof failedAt === "number" &&
Date.now() - failedAt < IDEMPOTENCY_TTL_MS) {
if (callback) callback({
success: false, eventId, code: "BUFFER_PERSIST_FAILED",
error: "BUFFER_PERSIST_FAILED: Event not durably queued",
});
} else if (callback) {
callback({ success: true, eventId, note: "DUPLICATE_DROPPED" });
}
return;
@@ -585,6 +594,20 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
try {
await onBufferPush();
} catch (persistErr) {
// Invalidate the pre-persistence idempotency claim so SDK retry can
// resubmit after Redis recovers. Keep the volatile RAM event pending.
const claimKey = `${clientId}:${eventId}`;
idempotencyMap.delete(claimKey);
idempotencyMap.set(`\u0000failed-persist:${claimKey}`, Date.now());
const claimRedis = getRedisClient();
if (claimRedis && typeof claimRedis.del === "function") {
try {
await claimRedis.del(`${REDIS_IDEMPOTENCY_PREFIX}${claimKey}`);
idempotencyMap.delete(claimKey);
} catch {
// Claim may remain: negative ACK must be retained for retries.
}
}
if (callback) {
callback({
success: false,
@@ -599,6 +622,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
return;
}
idempotencyMap.delete(`\u0000failed-persist:${clientId}:${eventId}`);
if (callback) callback({ success: true, eventId });
} catch (err) {
if (callback) callback({ success: false, error: err.message });
@@ -689,6 +713,8 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
const rateLimitedEventIds = [];
const oversizedEventIds = [];
const errors = [];
const acceptedEventIds = [];
let volatileDuplicate = false;
for (const item of batch) {
const safeItem = item && typeof item === "object" ? item : {};
@@ -702,6 +728,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
: new Date().toISOString();
const rawData = extractBatchItemData(safeItem);
const failedAt = idempotencyMap.get(`\u0000failed-persist:${clientId}:${eventId}`);
const check = await validateSingleEventAsync(
clientId,
eventId,
@@ -710,7 +737,15 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
);
if (!check.allowed) {
if (check.duplicate) {
duplicates += 1;
if (typeof failedAt === "number" &&
Date.now() - failedAt < IDEMPOTENCY_TTL_MS) {
rejected += 1;
volatileDuplicate = true;
if (errors.length < 5) errors.push(
`${eventId}: BUFFER_PERSIST_FAILED: Event not durably queued`);
} else {
duplicates += 1;
}
continue;
}
rejected += 1;
@@ -735,6 +770,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
payload: check.payloadStr,
});
processed += 1;
acceptedEventIds.push(eventId);
}
if (processed > 0) {
@@ -742,6 +778,20 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
try {
await onBufferPush();
} catch (persistErr) {
const claimRedis = getRedisClient();
for (const eventId of acceptedEventIds) {
const claimKey = `${clientId}:${eventId}`;
idempotencyMap.delete(claimKey);
idempotencyMap.set(`\u0000failed-persist:${claimKey}`, Date.now());
if (claimRedis && typeof claimRedis.del === "function") {
try {
await claimRedis.del(`${REDIS_IDEMPOTENCY_PREFIX}${claimKey}`);
idempotencyMap.delete(claimKey);
} catch {
// Preserve negative-ACK retry guard if claim removal fails.
}
}
}
if (callback) {
callback({
success: false,
@@ -759,12 +809,30 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
}
}
// Only durable new writes clear pending failure markers.
for (const persistedId of acceptedEventIds) {
idempotencyMap.delete(`\u0000failed-persist:${clientId}:${persistedId}`);
}
// A mixed batch must never receive success while one event still has
// an uncommitted Redis claim, even if other items were persisted.
if (volatileDuplicate) {
if (callback) callback({
success: false, processed, rejected, duplicates,
code: "BUFFER_PERSIST_FAILED",
error: "BUFFER_PERSIST_FAILED: Batch contains non-durable event",
errors,
});
return;
}
if (processed === 0 && duplicates === 0 && rejected > 0) {
const code = hitPayloadLimit
? "PAYLOAD_TOO_LARGE"
: hitRateLimit
? "RATE_LIMIT_EXCEEDED"
: "BATCH_REJECTED";
const code = volatileDuplicate
? "BUFFER_PERSIST_FAILED"
: hitPayloadLimit
? "PAYLOAD_TOO_LARGE"
: hitRateLimit
? "RATE_LIMIT_EXCEEDED"
: "BATCH_REJECTED";
if (callback) {
callback({
success: false,
+4 -4
View File
@@ -85,7 +85,7 @@ describe("Phase-01 Auth, Secrets, Revocation & Proxy Hardening", () => {
});
it("BUG-01 & BUG-02: fails closed in production when secrets or DB/setup are missing", async () => {
process.env.NODE_ENV = "production";
(process.env as Record<string, string | undefined>).NODE_ENV = "production";
delete process.env.DATABASE_URL;
delete process.env.DATASTACK_ADMIN_SESSION_SECRET;
delete process.env.DATASTACK_ADMIN_SETUP_KEY;
@@ -557,7 +557,7 @@ describe("Phase-01 Auth, Secrets, Revocation & Proxy Hardening", () => {
});
it("FORENSIC BUG-01: blocks volatile RAM admin setup in production without DATABASE_URL and returns 409 SETUP_REQUIRED on pre-setup login", async () => {
process.env.NODE_ENV = "production";
(process.env as Record<string, string | undefined>).NODE_ENV = "production";
delete process.env.DATABASE_URL;
delete process.env.DATASTACK_ADMIN_PASSWORD;
process.env.DATASTACK_ADMIN_SESSION_SECRET =
@@ -770,7 +770,7 @@ describe("Phase-01 Auth, Secrets, Revocation & Proxy Hardening", () => {
try {
// 1. Production with empty jwtSecret must not persist empty jwt_secret row into PG
process.env.NODE_ENV = "production";
(process.env as Record<string, string | undefined>).NODE_ENV = "production";
delete process.env.DATASTACK_JWT_SECRET;
settingsGlobals.__dataStackAdminSettings = undefined;
settingsGlobals.__dataStackSettingsHydrated = false;
@@ -791,7 +791,7 @@ describe("Phase-01 Auth, Secrets, Revocation & Proxy Hardening", () => {
);
// 2. Failing PG write must reject (not silently return 200) and must not mutate RAM
process.env.NODE_ENV = "test";
(process.env as Record<string, string | undefined>).NODE_ENV = "test";
process.env.DATASTACK_JWT_SECRET =
"valid-jwt-secret-at-least-32-characters-long";
AuthService.setSecret(process.env.DATASTACK_JWT_SECRET);
+275
View File
@@ -1220,3 +1220,278 @@ test("R2 PHASE-02 BUG-10: GET, PATCH (regenerate_token), and DELETE /api/clients
SDKManager.deleteClientWithStatsAsync = origDeleteWithStats;
}
});
test("P02 R-01 diagnostic: Redis RPUSH failure yields positive WS ACK for volatile RAM-only event", async () => {
// Disposable test-only fixture: no production Redis, service, or storage state.
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
const previousRam = serverMod.bufferQueue.slice();
const failingRedis = createSharedMockRedis();
failingRedis.rpush = async () => {
throw new Error("P02 injected Redis RPUSH outage");
};
serverMod.setRedisClientForTests(failingRedis);
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
serverMod.bufferQueue.length = 0;
let socket: Socket | undefined;
try {
const client = await SDKManager.registerClientAsync({
clientId: "p02_redis_rpush_failure",
name: "P02 Redis RPUSH Failure",
rateLimit: 100,
});
socket = await connectSocket(client.token);
const ack = await emitWithAck<{ success: boolean; eventId?: string; code?: string }>(
socket, "sync_data",
{ eventId: "p02-ack-before-crash-001", data: { value: 42 } },
);
const committedRows = await failingRedis.lrange("datastack:buffer:events", 0, -1);
const volatileRows = serverMod.bufferQueue.filter((e) => e.eventId === "p02-ack-before-crash-001");
assert.equal(ack.success, false, "P02 remediation must reject non-durable ACK");
assert.equal(ack.code, "BUFFER_PERSIST_FAILED");
const sameIdRetry = await emitWithAck<{ success: boolean; code?: string }>(
socket, "sync_data",
{ eventId: "p02-ack-before-crash-001", data: { value: 42 } },
);
assert.equal(sameIdRetry.success, false,
"Pending idempotency claims must not return false positive duplicate ACK");
assert.equal(sameIdRetry.code, "BUFFER_PERSIST_FAILED");
assert.equal(committedRows.length, 0, "Fake Redis has no durable event");
assert.equal(volatileRows.length, 1, "Acknowledged event is RAM-only");
// Crash model: restart loses RAM but fake Redis survives. No actual process is killed.
serverMod.bufferQueue.length = 0;
assert.equal((await failingRedis.lrange("datastack:buffer:events", 0, -1)).length, 0);
// A new event may ACK only after Redis actually resumes persistence.
const healthyRedis = createSharedMockRedis();
serverMod.setRedisClientForTests(healthyRedis);
const recovered = await emitWithAck<{ success: boolean; eventId?: string }>(
socket, "sync_data",
{ eventId: "p02-ack-recovered-002", data: { value: 43 } },
);
assert.equal(recovered.success, true);
assert.equal((await healthyRedis.lrange("datastack:buffer:events", 0, -1)).length, 1);
} finally {
socket?.disconnect();
serverMod.setRedisClientForTests(undefined);
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousRam);
}
});
test("P02 R-03 diagnostic: Redis query failure leaves public ingestion health 200/online", async () => {
const redis = createSharedMockRedis();
redis.llen = async () => { throw new Error("P02 simulated Redis health outage"); };
serverMod.setRedisClientForTests(redis);
try {
const res = await fetch(`${ingestionUrl}/health`);
assert.equal(res.status, 200, "Legacy liveness status remains backwards-compatible");
const body = await res.json() as { status: string; redisConnected: boolean; redisBufferSize: number };
assert.equal(body.status, "online");
assert.equal(body.redisConnected, false, "Redis failure is represented only in the payload");
assert.equal(body.redisBufferSize, 0);
const healthyRedis = createSharedMockRedis();
serverMod.setRedisClientForTests(healthyRedis);
const recovered = await fetch(`${ingestionUrl}/health`);
const recoveredBody = await recovered.json() as { redisConnected: boolean };
assert.equal(recovered.status, 200);
assert.equal(recoveredBody.redisConnected, true);
} finally {
serverMod.setRedisClientForTests(undefined);
}
});
test("P02 R-01: batch_sync rejects non-durable Redis writes and pending duplicate claims", async () => {
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
const previousQueue = serverMod.bufferQueue.slice();
const failingRedis = createSharedMockRedis();
failingRedis.rpush = async () => { throw new Error("P02 injected batch Redis outage"); };
serverMod.setRedisClientForTests(failingRedis);
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
serverMod.bufferQueue.length = 0;
let socket: Socket | undefined;
const payload = [
{ eventId: "p02-batch-volatile-1", data: { value: 1 } },
{ eventId: "p02-batch-volatile-2", data: { value: 2 } },
];
try {
const client = await SDKManager.registerClientAsync({
clientId: "p02_redis_batch_failure",
name: "P02 Redis Batch Failure",
rateLimit: 100,
});
socket = await connectSocket(client.token);
const ack = await emitWithAck<{ success: boolean; code?: string; processed?: number }>(
socket, "batch_sync", payload,
);
assert.equal(ack.success, false);
assert.equal(ack.code, "BUFFER_PERSIST_FAILED");
assert.equal(ack.processed, 0);
assert.equal((await failingRedis.lrange("datastack:buffer:events", 0, -1)).length, 0);
const retry = await emitWithAck<{ success: boolean; code?: string }>(
socket, "batch_sync", payload,
);
assert.equal(retry.success, false);
assert.equal(retry.code, "BUFFER_PERSIST_FAILED");
} finally {
socket?.disconnect();
serverMod.setRedisClientForTests(undefined);
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
}
});
test("P02 FINAL R-01: stale Redis claim never becomes positive ACK after cleanup window", async () => {
// Source proof at candidate 76cd972: a failure marker is -Date.now();
// cleanupMaps compares now - negativeTimestamp > TTL, which expires immediately
// on the first scheduled cleanup even though a Redis idempotency claim remains.
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
const previousQueue = serverMod.bufferQueue.slice();
const redis = createSharedMockRedis(); // intentionally no DEL: simulate stuck Redis claim
redis.rpush = async () => { throw new Error("Injected Redis queue outage"); };
serverMod.setRedisClientForTests(redis);
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
serverMod.bufferQueue.length = 0;
let socket: Socket | undefined;
try {
const client = await SDKManager.registerClientAsync({
clientId: "p02_stale_claim_cleanup", name: "P02 Stale Claim", rateLimit: 100,
});
socket = await connectSocket(client.token);
const payload = { eventId: "p02-stale-claim-1", data: { value: 1 } };
const first = await emitWithAck<{ success: boolean; code?: string }>(socket, "sync_data", payload);
assert.equal(first.success, false);
assert.equal(first.code, "BUFFER_PERSIST_FAILED");
// Verify TTL arithmetic without overriding the global clock (Socket.IO timeouts).
// Also keep two real socket retries to prove pending claims stay negative.
// Do not mutate process timestamps or wait for wall-clock TTL.
const stamped = Date.now();
const cleanupClock = stamped + 61_000;
assert.equal(cleanupClock - (-stamped) > 300_000, true, "Old negative marker expired prematurely");
assert.equal(cleanupClock - stamped < 300_000, true, "Five-minute TTL should still be active");
const handlerSource = readFileSync(require.resolve("../server/socket/handlers.js"), "utf8");
assert.ok(handlerSource.includes('idempotencyMap.set(`\\u0000failed-persist:${claimKey}`, Date.now());'));
assert.equal(handlerSource.includes("idempotencyMap.set(claimKey, -Date.now());"), false);
const retry1 = await emitWithAck<{ success: boolean; code?: string }>(socket, "sync_data", payload);
const retry2 = await emitWithAck<{ success: boolean; code?: string }>(socket, "sync_data", payload);
assert.equal(retry1.success, false);
assert.equal(retry1.code, "BUFFER_PERSIST_FAILED");
assert.equal(retry2.success, false, "Cleanup must not convert a volatile claim into DUPLICATE_DROPPED success");
assert.equal(retry2.code, "BUFFER_PERSIST_FAILED");
assert.equal((await redis.lrange("datastack:buffer:events", 0, -1)).length, 0);
} finally {
socket?.disconnect();
serverMod.setRedisClientForTests(undefined);
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
}
});
test("P02 FINAL R-01: mixed batch with stale volatile claim never ACKs success", async () => {
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
const previousQueue = serverMod.bufferQueue.slice();
const redis = createSharedMockRedis();
const originalRpush = redis.rpush.bind(redis);
redis.rpush = async () => { throw new Error("Injected Redis outage before batch recovery"); };
serverMod.setRedisClientForTests(redis);
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
serverMod.bufferQueue.length = 0;
let socket: Socket | undefined;
try {
const client = await SDKManager.registerClientAsync({
clientId: "p02_mixed_batch_claim", name: "P02 Mixed Batch Claim", rateLimit: 100,
});
socket = await connectSocket(client.token);
const pending = { eventId: "p02-mixed-pending-01", data: { v: 1 } };
const first = await emitWithAck<{ success: boolean; code?: string }>(
socket, "sync_data", pending,
);
assert.equal(first.success, false);
// Simulated process RAM loss: Redis idempotency key survives; queue does not.
serverMod.bufferQueue.length = 0;
redis.rpush = originalRpush;
const mixed = await emitWithAck<{ success: boolean; code?: string; processed?: number }>(
socket, "batch_sync", [pending, { eventId: "p02-mixed-durable-02", data: { v: 2 } }],
);
assert.equal(mixed.success, false, "Mixed batch cannot falsely ACK the volatile duplicate");
assert.equal(mixed.code, "BUFFER_PERSIST_FAILED");
assert.equal(mixed.processed, 1, "A distinct new event can commit without masking the failed claim");
const rows = await redis.lrange("datastack:buffer:events", 0, -1);
assert.equal(rows.length, 1);
assert.equal((JSON.parse(rows[0]) as { eventId: string }).eventId, "p02-mixed-durable-02");
} finally {
socket?.disconnect();
serverMod.setRedisClientForTests(undefined);
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
}
});
test("P02 FINAL R-01 compatibility: deliberately Redis-free RAM mode remains unchanged", async () => {
const threshold = serverMod.runtimeConfig.batchFlushThreshold;
const queue = serverMod.bufferQueue.slice();
serverMod.setRedisClientForTests(null);
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
serverMod.bufferQueue.length = 0;
let socket: Socket | undefined;
try {
const client = await SDKManager.registerClientAsync({
clientId: "p02_redis_free_compat", name: "P02 Redis-free Compatibility", rateLimit: 100,
});
socket = await connectSocket(client.token);
const ack = await emitWithAck<{ success: boolean; eventId?: string }>(
socket, "sync_data", { eventId: "p02-ram-compat-001", data: { value: 1 } },
);
assert.equal(ack.success, true, "Preserve historical RAM-mode acceptance contract");
assert.equal(serverMod.bufferQueue.some((e) => e.eventId === "p02-ram-compat-001"), true);
} finally {
socket?.disconnect();
serverMod.setRedisClientForTests(undefined);
serverMod.runtimeConfig.batchFlushThreshold = threshold;
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...queue);
}
});
test("P02 FINAL R-01: pending claim plus durable duplicate must reject whole batch", async () => {
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
const previousQueue = serverMod.bufferQueue.slice();
const redis = createSharedMockRedis();
const healthyRpush = redis.rpush.bind(redis);
serverMod.setRedisClientForTests(redis);
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
serverMod.bufferQueue.length = 0;
let socket: Socket | undefined;
try {
const client = await SDKManager.registerClientAsync({
clientId: "p02_mixed_only_duplicates", name: "P02 Failed and Durable Duplicate", rateLimit: 100,
});
socket = await connectSocket(client.token);
const durable = { eventId: "p02-durable-duplicate", data: { v: 10 } };
const pending = { eventId: "p02-failed-duplicate", data: { v: 20 } };
const first = await emitWithAck<{ success: boolean }>(socket, "sync_data", durable);
assert.equal(first.success, true);
redis.rpush = async () => { throw new Error("Injected Redis failure for second event"); };
const rejected = await emitWithAck<{ success: boolean; code?: string }>(
socket, "sync_data", pending,
);
assert.equal(rejected.success, false);
assert.equal(rejected.code, "BUFFER_PERSIST_FAILED");
serverMod.bufferQueue.length = 0; // simulate volatile queue lost across restart
redis.rpush = healthyRpush;
const mixed = await emitWithAck<{
success: boolean; code?: string; processed?: number; duplicates?: number;
}>(socket, "batch_sync", [pending, durable]);
assert.equal(mixed.processed, 0);
assert.equal(mixed.duplicates, 1);
assert.equal(mixed.success, false,
"Existing durable duplicate cannot mask a non-durable pending claim");
assert.equal(mixed.code, "BUFFER_PERSIST_FAILED");
assert.equal((await redis.lrange("datastack:buffer:events", 0, -1)).length, 1);
} finally {
socket?.disconnect();
serverMod.setRedisClientForTests(undefined);
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
}
});
+171 -5
View File
@@ -1085,10 +1085,10 @@ test("FORENSIC BUG-19: Deleting a client purges RAM, Redis, and Port 4000 buffer
test("FORENSIC BUG-20: Socket.IO sync_data and batch_sync ACKs wait until async Redis RPUSH completes", async () => {
const fakeRedis = new FakeRedisClient();
let rpushGateResolve: (() => void) | null = null;
const rpushGateControl: { resolve?: () => void } = {};
let rpushEnteredResolve: (() => void) | null = null;
let rpushGate = new Promise<void>((resolve) => {
rpushGateResolve = resolve;
rpushGateControl.resolve = resolve;
});
let rpushEntered = new Promise<void>((resolve) => {
rpushEnteredResolve = resolve;
@@ -1155,14 +1155,14 @@ test("FORENSIC BUG-20: Socket.IO sync_data and batch_sync ACKs wait until async
"sync_data ACK must NOT fire before Redis RPUSH finishes",
);
rpushGateResolve?.();
rpushGateControl.resolve?.();
const syncAck = await syncAckPromise;
assert.equal(syncAck.success, true);
assert.equal(await fakeRedis.llen(REDIS_BUFFER_KEY), 1);
// 2. batch_sync must also wait for Redis RPUSH before ACK
rpushGate = new Promise<void>((resolve) => {
rpushGateResolve = resolve;
rpushGateControl.resolve = resolve;
});
rpushEntered = new Promise<void>((resolve) => {
rpushEnteredResolve = resolve;
@@ -1199,7 +1199,7 @@ test("FORENSIC BUG-20: Socket.IO sync_data and batch_sync ACKs wait until async
"batch_sync ACK must NOT fire before Redis RPUSH finishes",
);
rpushGateResolve?.();
rpushGateControl.resolve?.();
const batchAck = await batchAckPromise;
assert.equal(batchAck.success, true);
assert.equal(batchAck.processed, 2);
@@ -1549,3 +1549,169 @@ test("R2 PHASE-03 BUG-15: GET /api/export rejects inverted date range with 400 a
assert.match(serverIndexSrc, /const filterClientId =\s*rawClientId && rawClientId !== 'all' \? normalizeClientId\(rawClientId\) : ''/);
});
test("P02 R-02 diagnostic: post-CSV Redis trim failure replays identical event on retry", async () => {
const redis = new FakeRedisClient();
const realTrim = redis.ltrim.bind(redis);
let injected = false;
redis.ltrim = async (key: string, start: number, stop: number): Promise<"OK"> => {
if (key === REDIS_BUFFER_KEY && !injected) {
injected = true;
throw new Error("P02 simulated crash boundary after CSV append, before Redis trim");
}
return realTrim(key, start, stop);
};
const event: DataEvent = {
clientId: "p02_csv_replay",
eventId: "p02-replay-001",
timestamp: "2026-11-22T01:00:00.000Z",
payload: JSON.stringify({ marker: "p02" }),
};
DataBufferEngine.setRedisClientForTests(redis);
const originalAppend = CSVStorageManager.appendBatch;
const writtenIds: string[] = [];
CSVStorageManager.appendBatch = async (events) => {
const result = await originalAppend.call(CSVStorageManager, events);
writtenIds.push(...events.map((row) => row.eventId));
return result;
};
try {
await DataBufferEngine.pushBatchAsync([event]);
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 1);
await assert.rejects(() => DataBufferEngine.flush(), /P02 simulated crash boundary/);
assert.equal(injected, true);
assert.equal(writtenIds.filter((id) => id === event.eventId).length, 1);
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 1, "Redis still queues previously appended CSV event");
await DataBufferEngine.flush();
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0);
assert.equal(writtenIds.filter((id) => id === event.eventId).length, 1,
"Replay must detect the persisted CSV eventId before attempting another append");
const csvPath = path.join(tmpStorageDir, event.clientId, "2026-11-22.csv");
const csvContents = fs.readFileSync(csvPath, "utf8");
assert.equal(csvContents.split(event.eventId).length - 1, 1,
"Exactly one physical CSV row must exist after a failed LTRIM and retry");
} finally {
CSVStorageManager.appendBatch = originalAppend;
DataBufferEngine.setRedisClientForTests(undefined);
}
});
test("P02 R-02: Port 4000 CSV worker suppresses a crash-window replay after Redis LTRIM error", async () => {
const mockRedis = new FakeRedisClient();
const originalTrim = mockRedis.ltrim.bind(mockRedis);
let failedOnce = false;
mockRedis.ltrim = async (key: string, start: number, stop: number): Promise<"OK"> => {
if (key === REDIS_BUFFER_KEY && !failedOnce) {
failedOnce = true;
throw new Error("P02 Port4000 injected post-CSV trim failure");
}
return originalTrim(key, start, stop);
};
const ingestion = require("../server/index.js") as IngestionServerModule;
const previousDir = ingestion.writer.baseDir;
const previousQueue = ingestion.bufferQueue.slice();
ingestion.writer.baseDir = tmpStorageDir;
ingestion.bufferQueue.length = 0;
ingestion.setRedisClientForTests(mockRedis);
const evt: DataEvent = {
clientId: "p02_ingestion_replay",
eventId: "p02-port4000-001",
timestamp: "2026-11-22T13:00:00.000Z",
payload: JSON.stringify({ stage: "before-ltrim" }),
};
try {
await mockRedis.rpush(REDIS_BUFFER_KEY, JSON.stringify(evt));
await assert.rejects(
() => ingestion.flushBuffer(),
/P02 Port4000 injected post-CSV trim failure/,
);
assert.equal(failedOnce, true);
assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 1);
const csvPath = path.join(tmpStorageDir, evt.clientId, "2026-11-22.csv");
assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1);
const recovered = await ingestion.flushBuffer();
assert.equal(recovered.written, 0, "Recovered replay should not append the event again");
assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 0);
assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1);
} finally {
ingestion.writer.baseDir = previousDir;
ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...previousQueue);
ingestion.setRedisClientForTests(undefined);
}
});
test("P02 FINAL R-02: Port 4000 scans rotated compressed CSV with quoted multiline payload", async () => {
const zlib = await import("node:zlib");
const redis = new FakeRedisClient();
const ingestion = require("../server/index.js") as IngestionServerModule;
const oldDir = ingestion.writer.baseDir;
const oldQueue = ingestion.bufferQueue.slice();
ingestion.writer.baseDir = tmpStorageDir;
ingestion.bufferQueue.length = 0;
ingestion.setRedisClientForTests(redis);
const event: DataEvent = {
clientId: "p02_rotated_gzip",
eventId: "p02-gzip-quoted-001",
timestamp: "2026-11-22T12:45:00.000Z",
payload: JSON.stringify({ body: "Line one, \"quoted\"\nLine two" }),
};
try {
const diskWriter = new CsvWriterWorker(tmpStorageDir);
await diskWriter.writeBatch([event]);
const dir = path.join(tmpStorageDir, event.clientId);
const csvPath = path.join(dir, "2026-11-22.csv");
const rotatedPath = path.join(dir, "2026-11-22-part2.csv");
const gzPath = rotatedPath + ".gz";
fs.renameSync(csvPath, rotatedPath);
fs.writeFileSync(gzPath, zlib.gzipSync(fs.readFileSync(rotatedPath)));
fs.unlinkSync(rotatedPath);
// A different date partition must not be parsed as part of this replay.
fs.writeFileSync(path.join(dir, "2026-11-21.csv"), 'invalid,"unterminated');
await redis.rpush(REDIS_BUFFER_KEY, JSON.stringify(event));
const replay = await ingestion.flushBuffer();
assert.equal(replay.written, 0, "Already written rotated/gzipped event must not append again");
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0, "Successful dedupe permits Redis trim");
assert.equal(fs.existsSync(csvPath), false, "Do not recreate a duplicate plain CSV");
assert.equal(zlib.gunzipSync(fs.readFileSync(gzPath)).toString("utf8").split(event.eventId).length - 1, 1);
} finally {
ingestion.writer.baseDir = oldDir;
ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...oldQueue);
ingestion.setRedisClientForTests(undefined);
}
});
test("P02 FINAL R-02: partial corrupted CSV blocks Redis trim and recovers after fixture repair", async () => {
const redis = new FakeRedisClient();
const ingestion = require("../server/index.js") as IngestionServerModule;
const oldDir = ingestion.writer.baseDir;
const oldQueue = ingestion.bufferQueue.slice();
ingestion.writer.baseDir = tmpStorageDir;
ingestion.bufferQueue.length = 0;
ingestion.setRedisClientForTests(redis);
const event: DataEvent = {
clientId: "p02_partial_corrupt",
eventId: "p02-corrupt-recovery-001",
timestamp: "2026-11-22T13:15:00.000Z",
payload: JSON.stringify({ value: "safe" }),
};
try {
const dir = path.join(tmpStorageDir, event.clientId);
fs.mkdirSync(dir, { recursive: true });
const badPath = path.join(dir, "2026-11-22.csv");
fs.writeFileSync(badPath, 'clientId,eventId,timestamp,payload\n"unterminated');
await redis.rpush(REDIS_BUFFER_KEY, JSON.stringify(event));
await assert.rejects(() => ingestion.flushBuffer(),
/quote|CSV_|invalid|record|unterminated/i, "Malformed existing CSV must fail closed");
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 1, "Never trim after failed CSV replay check");
// Disposable fixture repair only; does not alter user or production files.
fs.unlinkSync(badPath);
const recovered = await ingestion.flushBuffer();
assert.equal(recovered.written, 1);
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0);
assert.equal(fs.readFileSync(badPath, "utf8").split(event.eventId).length - 1, 1);
} finally {
ingestion.writer.baseDir = oldDir;
ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...oldQueue);
ingestion.setRedisClientForTests(undefined);
}
});
@@ -172,7 +172,7 @@ test("R3-Phase-01 [BUG-03]: Client SDK JWTs omit hardcoded 30d exp so Never-Expi
test("R3-Phase-01 [BUG-04 & BUG-15]: Viewer blocked (403) on POST /api/events and Operator blocked (403) on POST /api/storage/s3-test", async () => {
resetAdminStateForTests();
await createInitialAdmin("r3-owner@datastack.dev", "OwnerPassword123!");
const sessionSecret = getAdminSessionSecret();
const sessionSecret = getAdminSessionSecret()!;
const suffix = Date.now();
const viewerEmail = `r3viewer_${suffix}@datastack.dev`;
const operatorEmail = `r3operator_${suffix}@datastack.dev`;
@@ -182,13 +182,11 @@ test("R3-Phase-01 [BUG-04 & BUG-15]: Viewer blocked (403) on POST /api/events an
email: viewerEmail,
password: "ViewerPassword123!",
role: "viewer",
displayName: "R3 Viewer",
});
const operator = await createAdminTeamMemberAsync({
email: operatorEmail,
password: "OperatorPassword123!",
role: "operator",
displayName: "R3 Operator",
});
await SDKManager.registerClientAsync({
clientId,
@@ -292,7 +290,7 @@ test("R3-Phase-01 [BUG-06]: Locked webhook secret does not break unrelated setti
const ownerToken = await createAdminSessionToken(
owner.email,
getAdminSessionSecret(),
getAdminSessionSecret()!,
Date.now() + 3600_000,
owner.tokenVersion,
owner.role,
@@ -427,7 +425,7 @@ test("R3-Phase-01 [BUG-10]: Brute-force 429 admin login lockout dispatches ADMIN
assert.equal(dispatchedBodies[0].instance, "DataStack");
} finally {
global.fetch = origFetch;
await clearAdminLoginAttempts(email, getAdminSessionSecret(), testIp);
await clearAdminLoginAttempts(email, getAdminSessionSecret()!, testIp);
await SettingsManager.updateSettingsAsync({
enableWebhookAlerts: false,
webhookUrl: "",
@@ -40,7 +40,7 @@ function writeGzipCsv(filePath: string, content: string): void {
async function createOwnerCookie(): Promise<string> {
delete process.env.DATABASE_URL;
const sessionSecret = getAdminSessionSecret();
const sessionSecret = getAdminSessionSecret()!;
resetAdminStateForTests();
const ownerEmail = "owner-r3p2@datastack.local";
await createInitialAdmin(ownerEmail, "StrongPassword123!");
+6
View File
@@ -5,6 +5,7 @@ import test from "node:test";
import { NextRequest } from "next/server";
import { GET as prometheusGet } from "../app/api/prometheus/route";
import { SettingsManager } from "../core/adminSettings/settingsManager";
import { DataBufferEngine } from "../core/data/buffer";
import { ApiEngine } from "../core/apiEngine/apiService";
import {
createAdminSessionToken,
@@ -24,6 +25,11 @@ import {
summarizeUiClientCounts,
} from "../lib/vconUiHelpers";
test.after(async () => {
SDKManager.stopActivityFlushTimerForTests();
await DataBufferEngine.shutdown();
});
async function makeOwnerCookie(): Promise<string> {
resetAdminStateForTests();
await createInitialAdmin("r3-p4-owner@datastack.dev", "R3Phase04Pass!");
+2 -2
View File
@@ -96,7 +96,7 @@ function restoreEnv(): void {
async function createOwnerCookie(email = "r4p1-owner@datastack.dev"): Promise<string> {
resetAdminStateForTests();
await createInitialAdmin(email, "OwnerPassword123!");
const secret = getAdminSessionSecret();
const secret = getAdminSessionSecret()!;
const token = await createAdminSessionToken(
email,
secret,
@@ -531,7 +531,7 @@ test("R4-Phase-01 [AUDIT-09 / ERR-R4-04 & ERR-R4-05]: Fail-loud audit DB degrade
assert.equal(unauthSync.status, 401);
// Owner session v1 works before logout/revocation
const secret = getAdminSessionSecret();
const secret = getAdminSessionSecret()!;
const ownerTokenV1 = await createAdminSessionToken(
"r4-audit-owner@datastack.dev",
secret,
@@ -5,6 +5,7 @@ import fsp from "node:fs/promises";
import http from "node:http";
import { createRequire } from "node:module";
import type { AddressInfo } from "node:net";
import type { Request, Response } from "express";
import os from "node:os";
import path from "node:path";
import test from "node:test";
@@ -81,7 +82,7 @@ async function createOwnerCookie(
): Promise<string> {
resetAdminStateForTests();
await createInitialAdmin(email, "OwnerPassword123!");
const secret = getAdminSessionSecret();
const secret = getAdminSessionSecret()!;
const token = await createAdminSessionToken(
email,
secret,
@@ -329,13 +330,13 @@ test("R4-Phase-02 AUDIT-06: Governance audit logging for /api/export, Files Mana
getDbPool: () => null,
}),
);
ramApp.put("/vcon/files-engine/api/v1/fs/sample.csv", (_req: express.Request, res: express.Response) => {
ramApp.put("/vcon/files-engine/api/v1/fs/sample.csv", (_req: Request, res: Response) => {
res.status(200).json({ ok: true });
});
ramApp.put("/vcon/files-engine/api/v1/rename", (_req: express.Request, res: express.Response) => {
ramApp.put("/vcon/files-engine/api/v1/rename", (_req: Request, res: Response) => {
res.status(200).json({ ok: true });
});
ramApp.delete("/vcon/files-engine/api/v1/fs/sample.csv", (_req: express.Request, res: express.Response) => {
ramApp.delete("/vcon/files-engine/api/v1/fs/sample.csv", (_req: Request, res: Response) => {
res.status(403).json({ ok: false }); // non-2xx should NOT emit mutation audit
});
@@ -404,7 +405,7 @@ test("R4-Phase-02 AUDIT-06: Governance audit logging for /api/export, Files Mana
getDbPool: () => mockDbPool,
}),
);
pgApp.delete("/vcon/files-engine/api/v1/fs/obsolete.csv", (_req: express.Request, res: express.Response) => {
pgApp.delete("/vcon/files-engine/api/v1/fs/obsolete.csv", (_req: Request, res: Response) => {
res.status(200).json({ ok: true });
});
+6 -1
View File
@@ -3,7 +3,7 @@ import fs from 'node:fs';
import { createRequire } from 'node:module';
import os from 'node:os';
import path from 'node:path';
import { afterEach, beforeEach, describe, it } from 'node:test';
import { after, afterEach, beforeEach, describe, it } from 'node:test';
import { NextRequest } from 'next/server';
import { POST as setupPost } from '../app/api/auth/setup/route';
import { GET as filesStatusGet } from '../app/api/files/status/route';
@@ -13,9 +13,14 @@ import {
getAdminSessionSecret,
} from '../core/auth/adminSession';
import { resetAdminStateForTests } from '../core/auth/adminUsers';
import { DataBufferEngine } from '../core/data/buffer';
import { middleware } from '../middleware';
import { DataStack, DiskSpooler } from '../sdk/node/datastack';
after(async () => {
await DataBufferEngine.shutdown();
});
const require = createRequire(import.meta.url);
const ORIGINAL_ENV = { ...process.env };
const ORIGINAL_FETCH = globalThis.fetch;
@@ -35,6 +35,7 @@ import { SDKManager } from "../core/sdk/sdkManager";
import { getInternalIngestionHeaders } from "../core/Security/security";
import { SSRFBlockedError } from "../core/Security/ssrfGuard";
import { CSVStorageManager } from "../core/Storage/csvStorage";
import { DataBufferEngine } from "../core/data/buffer";
import { middleware } from "../middleware";
const require = createRequire(import.meta.url);
@@ -78,8 +79,15 @@ const cjsRuntimeLogCapture = require("../server/runtimeLogCapture.js") as {
const ingestionServer = require("../server/index.js") as {
setDbPoolForTests: (pool: unknown) => void;
syncSettingsFromDatabase: () => Promise<boolean>;
stopTimers: () => void;
};
test.after(async () => {
ingestionServer.stopTimers();
SDKManager.stopActivityFlushTimerForTests();
await DataBufferEngine.shutdown();
});
test("R5-Phase-01 [MCP-AUDIT-02 / ERR-R5-02]: Runtime log capture scrubs secrets, rotates at 2MB, filters/tails safely, and handles missing dirs gracefully", async () => {
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "datastack-r5-logs-"));
const logsDir = path.join(tmpDir, ".runtime-logs");
+12 -12
View File
@@ -507,7 +507,7 @@ test("R5-Phase-02 [MCP-AUDIT-01/06/07 & ERR-R5-03/04/05/06]: Port 4000 MCP serve
});
assert.equal(deniedClientCreate.isError, true);
const deniedPayload = JSON.parse(
(deniedClientCreate.content[0] as { text: string }).text,
((deniedClientCreate.content as Array<{ text: string }>)[0]).text,
) as { code: string; permission: string };
assert.equal(deniedPayload.code, "MCP_PERMISSION_DENIED");
assert.equal(deniedPayload.permission, "clients_create");
@@ -521,7 +521,7 @@ test("R5-Phase-02 [MCP-AUDIT-01/06/07 & ERR-R5-03/04/05/06]: Port 4000 MCP serve
});
assert.notEqual(createdToolRes.isError, true);
const createdPayload = JSON.parse(
(createdToolRes.content[0] as { text: string }).text,
((createdToolRes.content as Array<{ text: string }>)[0]).text,
) as { success: boolean; client: { clientId: string; rateLimit: number; token?: string } };
assert.equal(createdPayload.success, true);
assert.equal(createdPayload.client.clientId, testClientId);
@@ -535,7 +535,7 @@ test("R5-Phase-02 [MCP-AUDIT-01/06/07 & ERR-R5-03/04/05/06]: Port 4000 MCP serve
});
assert.equal(traversalRes.isError, true);
assert.equal(
(JSON.parse((traversalRes.content[0] as { text: string }).text) as { code: string }).code,
(JSON.parse(((traversalRes.content as Array<{ text: string }>)[0]).text) as { code: string }).code,
"PATH_OUTSIDE_ROOT",
);
@@ -546,7 +546,7 @@ test("R5-Phase-02 [MCP-AUDIT-01/06/07 & ERR-R5-03/04/05/06]: Port 4000 MCP serve
});
assert.equal(symlinkRes.isError, true);
assert.equal(
(JSON.parse((symlinkRes.content[0] as { text: string }).text) as { code: string }).code,
(JSON.parse(((symlinkRes.content as Array<{ text: string }>)[0]).text) as { code: string }).code,
"SYMLINK_FORBIDDEN",
);
}
@@ -557,7 +557,7 @@ test("R5-Phase-02 [MCP-AUDIT-01/06/07 & ERR-R5-03/04/05/06]: Port 4000 MCP serve
});
assert.equal(lockReadRes.isError, true);
assert.equal(
(JSON.parse((lockReadRes.content[0] as { text: string }).text) as { code: string }).code,
(JSON.parse(((lockReadRes.content as Array<{ text: string }>)[0]).text) as { code: string }).code,
"LOCKFILE_PROTECTED",
);
@@ -567,7 +567,7 @@ test("R5-Phase-02 [MCP-AUDIT-01/06/07 & ERR-R5-03/04/05/06]: Port 4000 MCP serve
});
assert.equal(lockedFileUploadRes.isError, true);
assert.equal(
(JSON.parse((lockedFileUploadRes.content[0] as { text: string }).text) as { code: string })
(JSON.parse(((lockedFileUploadRes.content as Array<{ text: string }>)[0]).text) as { code: string })
.code,
"LOCKFILE_PROTECTED",
);
@@ -578,7 +578,7 @@ test("R5-Phase-02 [MCP-AUDIT-01/06/07 & ERR-R5-03/04/05/06]: Port 4000 MCP serve
});
assert.equal(logWriteRes.isError, true);
assert.equal(
(JSON.parse((logWriteRes.content[0] as { text: string }).text) as { code: string }).code,
(JSON.parse(((logWriteRes.content as Array<{ text: string }>)[0]).text) as { code: string }).code,
"PROTECTED_LOG_DIR",
);
@@ -601,7 +601,7 @@ test("R5-Phase-02 [MCP-AUDIT-01/06/07 & ERR-R5-03/04/05/06]: Port 4000 MCP serve
});
assert.notEqual(dlRes.isError, true);
assert.equal(
(JSON.parse((dlRes.content[0] as { text: string }).text) as { content: string }).content,
(JSON.parse(((dlRes.content as Array<{ text: string }>)[0]).text) as { content: string }).content,
"alpha gamma",
);
@@ -610,7 +610,7 @@ test("R5-Phase-02 [MCP-AUDIT-01/06/07 & ERR-R5-03/04/05/06]: Port 4000 MCP serve
arguments: { path: "data" },
});
assert.notEqual(listRes.isError, true);
const listParsed = JSON.parse((listRes.content[0] as { text: string }).text) as {
const listParsed = JSON.parse(((listRes.content as Array<{ text: string }>)[0]).text) as {
entries: Array<{ name: string }>;
};
assert.ok(listParsed.entries.some((e) => e.name === "active.csv"));
@@ -632,7 +632,7 @@ test("R5-Phase-02 [MCP-AUDIT-01/06/07 & ERR-R5-03/04/05/06]: Port 4000 MCP serve
});
assert.notEqual(metricsRes.isError, true);
assert.equal(
(JSON.parse((metricsRes.content[0] as { text: string }).text) as { success: boolean }).success,
(JSON.parse(((metricsRes.content as Array<{ text: string }>)[0]).text) as { success: boolean }).success,
true,
);
@@ -641,7 +641,7 @@ test("R5-Phase-02 [MCP-AUDIT-01/06/07 & ERR-R5-03/04/05/06]: Port 4000 MCP serve
arguments: { action: "flush" },
});
assert.notEqual(flushRes.isError, true);
const flushPayload = JSON.parse((flushRes.content[0] as { text: string }).text) as {
const flushPayload = JSON.parse(((flushRes.content as Array<{ text: string }>)[0]).text) as {
success: boolean;
ingestionFlushed: number;
};
@@ -654,7 +654,7 @@ test("R5-Phase-02 [MCP-AUDIT-01/06/07 & ERR-R5-03/04/05/06]: Port 4000 MCP serve
});
assert.equal(coolifySsrfRes.isError, true);
assert.equal(
(JSON.parse((coolifySsrfRes.content[0] as { text: string }).text) as { code: string }).code,
(JSON.parse(((coolifySsrfRes.content as Array<{ text: string }>)[0]).text) as { code: string }).code,
"SSRF_BLOCKED_HOST",
);
+5
View File
@@ -4,6 +4,7 @@ import path from "node:path";
import test from "node:test";
import { MCP_PERMISSION_DEFINITIONS } from "../core/mcp/mcpSettings";
import { DataBufferEngine } from "../core/data/buffer";
import {
applyCategoryPermissionToggle,
applyMcpPermissionPreset,
@@ -23,6 +24,10 @@ import {
const DEFINITIONS = MCP_PERMISSION_DEFINITIONS as readonly McpPermissionDefinitionView[];
test.after(async () => {
await DataBufferEngine.shutdown();
});
test("R5-Phase-03 [lib/mcpUiHelpers.ts]: URL builder, AI client JSON snippets, permission grouping/presets, bridge status, and diff payload", () => {
// 1. Endpoint URL builder
assert.equal(
+19 -16
View File
@@ -43,6 +43,8 @@ import {
uploadFileToS3,
} from "../core/Storage/s3Archive";
import {
classifyUiClientStatus,
summarizeUiClientCounts,
DEFAULT_EDITABLE_SETTINGS,
settingsToNumericDrafts,
validateEditableSettingsDraft,
@@ -759,7 +761,7 @@ test("V2-Audit-Phase-03 [BUG-V2-AUDIT-01, BUG-V2-AUDIT-07, BUG-V2-AUDIT-10]: Dis
bsize: 4096,
}));
const snapshot = await CSVStorageManager.getDiskUsageAsync(true);
const snapshot = await CSVStorageManager.getDiskUsageAsync();
assert.equal(snapshot.available, true);
assert.equal(snapshot.usedPct, 25);
assert.equal(snapshot.totalBytes, 2000 * 4096);
@@ -780,29 +782,30 @@ test("V2-Audit-Phase-03 [BUG-V2-AUDIT-01, BUG-V2-AUDIT-07, BUG-V2-AUDIT-10]: Dis
CSVStorageManager.setStatfsOverrideForTests(async () => {
throw new Error("Simulated statfs failure");
});
const fallbackSnapshot = await CSVStorageManager.getDiskUsageAsync(true);
const fallbackSnapshot = await CSVStorageManager.getDiskUsageAsync();
assert.equal(fallbackSnapshot.available, false);
assert.equal(fallbackSnapshot.freeBytes, 0);
assert.equal(fallbackSnapshot.availableBytes, 0);
// 2. BUG-V2-AUDIT-07: app/vcon/clients/page.tsx defines isClientExpired and excludes expired tokens from onlineCount, pendingCount, and online/pending filter tabs
// 2. BUG-V2-AUDIT-07: shared status classifier excludes expired clients from active counters.
const clientsPageSrc = await fs.readFile(
path.join(process.cwd(), "app/vcon/clients/page.tsx"),
"utf8",
);
assert.match(clientsPageSrc, /function isClientExpired\(/);
assert.match(
clientsPageSrc,
/const expired = !revoked && isClientExpired\(client\.expiresAt,\s*now\)/,
);
assert.match(
clientsPageSrc,
/!isClientExpired\(client\.expiresAt,\s*now\)/,
);
assert.match(
clientsPageSrc,
/const pending =\s*client\.status === 'active' && !expired && !client\.lastActive/,
);
assert.match(clientsPageSrc, /summarizeUiClientCounts\(clients, now\)/);
assert.match(clientsPageSrc, /classifyUiClientStatus\(client, now\)/);
const now = Date.parse("2026-11-21T10:00:00.000Z");
const expiredClient = { status: "active" as const, expiresAt: "2020-01-01T00:00:00.000Z", lastActive: "2026-11-21T09:59:50.000Z" };
const pendingClient = { status: "active" as const, expiresAt: null, lastActive: undefined };
const onlineClient = { status: "active" as const, expiresAt: null, lastActive: "2026-11-21T09:59:50.000Z" };
assert.equal(classifyUiClientStatus(expiredClient, now), "expired");
assert.equal(classifyUiClientStatus(pendingClient, now), "pending");
assert.equal(classifyUiClientStatus(onlineClient, now), "online");
const summary = summarizeUiClientCounts([expiredClient, pendingClient, onlineClient], now);
assert.equal(summary.expired, 1);
assert.equal(summary.online, 1);
assert.equal(summary.pending, 1);
assert.equal(summary.active, 2);
// 3. BUG-V2-AUDIT-10: ExportWorkspace.tsx and app/vcon/storage/page.tsx render CSV.GZ badge for compressed .csv.gz partitions
const exportWorkspaceSrc = await fs.readFile(
@@ -331,7 +331,7 @@ test("V2-Phase-02 [HTTP & Socket Ingestion Security]: Per-client IP allowlist (4
token: string,
bypassCache?: boolean,
clientIp?: string,
) => Promise<{ ok: boolean; reason?: string }>;
) => Promise<{ allowed: boolean; code?: string; reason?: string }>;
setDbPoolForTests: (pool: unknown) => void;
stopTimers: () => void;
};