4 Commits
15 changed files with 391 additions and 810 deletions

No files matched your search

+23 -104
View File
@@ -16,15 +16,10 @@ on:
required: false required: false
type: boolean type: boolean
default: false default: false
python_only:
description: Narrow CI-06 Python job rerun after a proven runner setup failure
required: false
type: boolean
default: false
jobs: jobs:
node-regression: node-regression:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.quality_only != 'true' && (github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true') }} if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.quality_only != 'true' }}
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 30 timeout-minutes: 30
steps: steps:
@@ -57,36 +52,25 @@ jobs:
python-regression: python-regression:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.p01_only != 'true' }} if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.p01_only != 'true' }}
runs-on: ubuntu-latest runs-on: ubuntu-latest
container: node:22-bookworm-slim timeout-minutes: 15
timeout-minutes: 20
steps: steps:
- name: Install Bookworm Python and checkout prerequisites
run: |
set -eu
apt-get update -qq
apt-get install -y --no-install-recommends git ca-certificates python3 python3-venv python3-pip
- uses: actions/checkout@v4 - uses: actions/checkout@v4
with: with:
github-server-url: https://dev.ivib.cc github-server-url: https://dev.ivib.cc
- name: CI-06 Python 3.11 isolated virtual environment - name: Install Python toolchain
run: | run: |
set -eu set -eu
python3 --version apt-get update -qq
apt-get install -y python3 python3-venv python3-pip
python3 -m venv /tmp/datastack-ci-venv python3 -m venv /tmp/datastack-ci-venv
/tmp/datastack-ci-venv/bin/python -m pip install --disable-pip-version-check -e ./sdk/python/datastack -e ./sdk-python /tmp/datastack-ci-venv/bin/python -m pip install -e ./sdk/python/datastack -e ./sdk-python
- name: CI-06 compile and assert 14 Python SDK regressions - name: Run all Python SDK tests
run: | run: |
set -eu set -eu
py=/tmp/datastack-ci-venv/bin/python /tmp/datastack-ci-venv/bin/python -m compileall -q ./sdk/python/datastack ./sdk-python
"$py" -m compileall -q ./sdk/python/datastack ./sdk-python /tmp/datastack-ci-venv/bin/python -m unittest discover -s tests -p '*_test.py' -v
"$py" -m unittest discover -s tests -p '*_test.py' -v > /tmp/datastack-python-tests.log 2>&1 || { cat /tmp/datastack-python-tests.log; exit 1; }
cat /tmp/datastack-python-tests.log
grep -Eq '^Ran 14 tests in ' /tmp/datastack-python-tests.log
grep -Eq '^OK$' /tmp/datastack-python-tests.log
echo DATASTACK_PYTHON_14_OF_14_PASS
quality: quality:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true' }}
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 20 timeout-minutes: 20
steps: steps:
@@ -101,7 +85,7 @@ jobs:
run: npm run lint run: npm run lint
production-build: production-build:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.quality_only != 'true' && (github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true') }} if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.quality_only != 'true' }}
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 25 timeout-minutes: 25
steps: steps:
@@ -117,45 +101,30 @@ jobs:
test -f .next/standalone/server.js test -f .next/standalone/server.js
docker-image-build: docker-image-build:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.p01_only != 'true' && (github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true') }} if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.p01_only != 'true' }}
runs-on: ubuntu-latest runs-on: ubuntu-latest
container: node:22-alpine timeout-minutes: 35
timeout-minutes: 45
steps: steps:
- name: CI-07 provision Docker Buildx CLI
run: |
set -eu
apk add --no-cache git docker-cli docker-cli-buildx docker-cli-compose
docker --version
docker buildx version
docker compose version
- uses: actions/checkout@v4 - uses: actions/checkout@v4
with: with:
github-server-url: https://dev.ivib.cc github-server-url: https://dev.ivib.cc
- name: CI-07 inspect daemon and validate Compose without starting services - name: Docker and Compose availability
run: | run: |
set -eu set -eu
docker info --format 'Driver={{.Driver}} Root={{.DockerRootDir}}' docker --version
df -h . docker compose version
docker compose -f docker/docker-compose.yml config --no-interpolate --quiet docker info
echo DATASTACK_COMPOSE_CONFIG_PASS - name: Validate Compose structure
- name: CI-07 isolated BuildKit all three images (no deployment) run: docker compose -f docker/docker-compose.yml config --no-interpolate --quiet
- name: Build Docker images without running services
run: | run: |
set -eu set -eu
builder="datastack-p03-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}" docker build -f docker/Dockerfile.server -t datastack-ci-ingestion .
tag="datastack-p03-${GITHUB_SHA:-local}" docker build -f docker/Dockerfile.web -t datastack-ci-web .
cleanup() { docker buildx rm --force "$builder" >/dev/null 2>&1 || true; } docker build -f docker/Dockerfile.proxy -t datastack-ci-proxy .
trap cleanup EXIT
docker buildx create --name "$builder" --driver docker-container --use
docker buildx inspect "$builder" --bootstrap
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.server -t "$tag-ingestion" .
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.web -t "$tag-web" .
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.proxy -t "$tag-proxy" .
docker image inspect "$tag-ingestion" "$tag-web" "$tag-proxy" >/dev/null
echo DATASTACK_THREE_IMAGES_BUILD_PASS
windows-compatibility: windows-compatibility:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.quality_only != 'true' && (github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true') }} if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.quality_only != 'true' }}
runs-on: windows-latest runs-on: windows-latest
timeout-minutes: 25 timeout-minutes: 25
steps: steps:
@@ -181,53 +150,3 @@ jobs:
if ($LASTEXITCODE -ne 0) { throw 'SDK install failed' } if ($LASTEXITCODE -ne 0) { throw 'SDK install failed' }
npm --prefix sdk/node/datastack run build npm --prefix sdk/node/datastack run build
if ($LASTEXITCODE -ne 0) { throw 'SDK compilation failed' } if ($LASTEXITCODE -ne 0) { throw 'SDK compilation failed' }
security-audit:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.p01_only != 'true' && (github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true') }}
runs-on: ubuntu-latest
container: node:22-alpine
timeout-minutes: 15
steps:
- name: Install checkout prerequisite
run: apk add --no-cache git
- uses: actions/checkout@v4
with:
github-server-url: https://dev.ivib.cc
- name: CI-08 read-only full and runtime npm advisory inventory
run: |
set -eu
if npm audit --json > /tmp/datastack-audit-full.json; then :; else test "$?" -eq 1; fi
if npm audit --omit=dev --json > /tmp/datastack-audit-runtime.json; then :; else test "$?" -eq 1; fi
node <<'NODE'
const fs = require("node:fs");
for (const scope of ["full", "runtime"]) {
const audit = JSON.parse(fs.readFileSync("/tmp/datastack-audit-" + scope + ".json", "utf8"));
if (!audit.metadata || !audit.metadata.vulnerabilities || !audit.vulnerabilities) {
throw Error("Incomplete npm audit response for " + scope);
}
console.log("AUDIT_COUNTS " + scope + " " + JSON.stringify(audit.metadata.vulnerabilities));
for (const [name, item] of Object.entries(audit.vulnerabilities)) {
const advisories = (item.via || []).filter(v => typeof v === "object").map(v => ({
id: v.source, url: v.url, range: v.range, severity: v.severity
}));
console.log("AUDIT_PACKAGE " + scope + " " + JSON.stringify({
package: name, severity: item.severity, direct: item.isDirect,
range: item.range, via: advisories,
viaPackages: (item.via || []).filter(v => typeof v === "string"),
fixAvailable: item.fixAvailable
}));
}
if (scope === "runtime") {
const counts = audit.metadata.vulnerabilities;
if (!Number.isInteger(counts.high) || !Number.isInteger(counts.critical)) {
throw Error("Incomplete runtime npm advisory severity counts");
}
if (counts.high > 0 || counts.critical > 0) {
throw Error("DATASTACK_RUNTIME_SECURITY_GATE_FAILED high=" + counts.high + " critical=" + counts.critical);
}
console.log("DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO");
}
}
NODE
git diff --exit-code -- package.json package-lock.json
echo DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS
+17 -19
View File
@@ -1,41 +1,39 @@
name: DataStack Docker Build CI name: DataStack Docker Build CI
on: on:
push:
branches: [main]
paths:
- '.gitea/workflows/datastack-docker-ci.yml'
workflow_dispatch: workflow_dispatch:
jobs: jobs:
docker-build: docker-build:
runs-on: ubuntu-latest runs-on: ubuntu-latest
container: node:22-alpine container: node:22-alpine
timeout-minutes: 45 timeout-minutes: 40
steps: steps:
- name: CI-07 provision Docker Buildx CLI - name: Provision isolated Docker CLI and Git tools
run: | run: |
set -eu set -eu
apk add --no-cache git docker-cli docker-cli-buildx docker-cli-compose apk add --no-cache git docker-cli docker-cli-compose
docker --version docker --version
docker buildx version
docker compose version docker compose version
- uses: actions/checkout@v4 - uses: actions/checkout@v4
with: with:
github-server-url: https://dev.ivib.cc github-server-url: https://dev.ivib.cc
- name: CI-07 inspect daemon and validate Compose without starting services - name: Validate actual Compose configuration
run: | run: |
set -eu set -eu
docker info --format 'Driver={{.Driver}} Root={{.DockerRootDir}}'
df -h .
docker compose -f docker/docker-compose.yml config --no-interpolate --quiet docker compose -f docker/docker-compose.yml config --no-interpolate --quiet
echo DATASTACK_COMPOSE_CONFIG_PASS echo DATASTACK_COMPOSE_CONFIG_PASS
- name: CI-07 isolated BuildKit all three images (no deployment) - name: Check CI-only Docker daemon
run: | run: |
set -eu set -eu
builder="datastack-p03-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}" docker info >/dev/null || { echo DOCKER_DAEMON_UNAVAILABLE; exit 3; }
tag="datastack-p03-${GITHUB_SHA:-local}" - name: Build all DataStack container images without deployment
cleanup() { docker buildx rm --force "$builder" >/dev/null 2>&1 || true; } run: |
trap cleanup EXIT set -eu
docker buildx create --name "$builder" --driver docker-container --use docker build -f docker/Dockerfile.server -t datastack-ci-ingestion .
docker buildx inspect "$builder" --bootstrap docker build -f docker/Dockerfile.web -t datastack-ci-web .
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.server -t "$tag-ingestion" . docker build -f docker/Dockerfile.proxy -t datastack-ci-proxy .
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.web -t "$tag-web" . echo DATASTACK_DOCKER_IMAGES_PASS
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.proxy -t "$tag-proxy" .
docker image inspect "$tag-ingestion" "$tag-web" "$tag-proxy" >/dev/null
echo DATASTACK_THREE_IMAGES_BUILD_PASS
+10 -60
View File
@@ -101,69 +101,19 @@ Authorized follow-up baseline: Gitea/GitHub `fd57a3518d14bcb8367ce9ab9c3ec306bcf
**Candidate acceptance:** one disposable Gitea CI with 24/24 suites, strict TS, lint, Linux/Windows clean npm CI and production build; direct Port4000 replay tests; exact allowlist/diff audit and three-remote tree parity. The Gitea MCP/host was unavailable on the implementation attempt, so this is **NOT PASSED** and the P02 phase remains **BLOCKED / NOT CLOSED**. No live deployment or runtime production mutation. **Candidate acceptance:** one disposable Gitea CI with 24/24 suites, strict TS, lint, Linux/Windows clean npm CI and production build; direct Port4000 replay tests; exact allowlist/diff audit and three-remote tree parity. The Gitea MCP/host was unavailable on the implementation attempt, so this is **NOT PASSED** and the P02 phase remains **BLOCKED / NOT CLOSED**. No live deployment or runtime production mutation.
## P02-RUNTIME-REMEDIATION-02 — Scope-locked post-patch forensic audit (2026-11-22)
## P02 Final Acceptance Verification — 2026-11-22 (current superseding evidence) **Source baseline:** P02 diagnostic `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`. **Tested patch branch:** `p02-runtime-remediation-20261122`, final remediation source snapshot `5c880885378ae0f4fa8dd3a145f9c7055dada635`. This appendix supersedes the earlier "Gitea unavailable" candidate note. **Do not claim P02 CLOSED**: a separate HTTP fallback code path remains in scope-lock-excluded files.
Baseline published diagnosis: `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`. Runtime candidate `76cd972bdeea9ef07b4b38e9d8b5b5ed3312ca5b` passed Gitea CI #66 (24/24 Node regression suites, Node SDK, strict TypeScript, ESLint, Next production build, Windows SDK). New final narrow patch `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532`: `server/socket/handlers.js` plus two previously scoped TS tests only; final disposable Gitea CI #69 dispatched, terminal result to be recorded before phase closure. **R-01 Port 4000 improvements implemented (not live-deployed):**
- `server/index.js:pushEventsToRedisOrFallback` fails closed with sanitized `BUFFER_PERSIST_FAILED` on Redis RPUSH rejection while retaining RAM for recovery; also deduplicates same-client/eventId retries inside a single push.
- The `onBufferPush` integration synchronously flushes to CSV before a successful ACK when Redis is intentionally absent; failed CSV append returns a negative ACK. No-Redis mode is therefore not an automatic RAM-only success ACK.
- `server/socket/handlers.js` single/batch ACK logic clears failed local idempotency claims, never calls an uncertain Redis claim "durable duplicate", verifies a claimed duplicate against a bounded Redis buffer read or read-only CSV/.csv.gz scan, and denies uncertain mixed-batch ACKs. The proof uses a 512-item Redis window and a bounded CSV scan (256 MiB, 1.8 seconds); lack of proof yields a negative ACK, **not proof of data loss**.
Confirmed exact root cause in candidate: `server/socket/handlers.js` stores failed Redis persistence in `idempotencyMap` at `-Date.now()` while `cleanupMaps` deletes entries when `now - timestamp > 300000`; after the 60000ms cleanup window a failed marker can disappear despite an uncommitted Redis idempotency claim, permitting false-positive `DUPLICATE_DROPPED`. Arithmetic reproduction before code patch: at +61000ms, cleanup deletes a marker although Redis claim TTL (300000ms) is not over. Final patch uses a separate positive-timestamp failed-persistence marker and preserves negative ACK while the outstanding claim remains. A mixed batch with a failed duplicate must not return overall success while distinct records persist. **R-02 Port 4000 + web-buffer improvements implemented:** Both allowed flush methods check same-client/partition/eventId in physically persisted CSV/CSV.gz (including rotation) before retrying after Redis trim failure, and only trim when the disk append/check succeeds. Direct disposable Port 4000 LTRIM failure and gzip-partition replay tests exist. **Not a formal globally atomic/exactly-once guarantee:** concurrent independent writers, data corruption and repeated full-partition scans may introduce performance or recovery limitations.
R-02: `server/index.js:843-897,1043-1058` and `core/data/buffer.ts:652-711,855-872` replay-scan existing same-client/day CSV, preserving Redis queue until a successful write/dedupe; fault tests include direct Port 4000 post-CSV LTRIM outage, rotated compressed .csv.gz containing quoted multiline payload and unrelated-date corrupt partition; malformed matching-date CSV must reject and leave Redis queue intact, with recovery after disposable fixture repair. Streaming parsing bounds memory, **not total disk I/O**; scan latency grows with matching partition size. No universal exactly-once guarantee, no live throughput claim and no claim of finite hard disk-I/O cap. **Critical residual R-01 HTTP path (outside this approval):** `core/data/buffer.ts:412-442` silently falls back to RAM on Redis RPUSH error; `app/api/events/route.ts:744-754` returns HTTP success after `pushAsync`. `sdk/node/datastack/index.ts:643-666,916-942` routes non-auth WebSocket rejections into HTTP fallback. Thus a negative Socket.IO ACK does **not establish end-to-end SDK/HTTP durability**. A disposable HTTP fault diagnostic was added to the P02 test allowlist; it is not a runtime fix. Fixing HTTP `POST` and the buffer push/idempotency contract needs separate explicit approval, including `core/Security/security.ts:230-375` only if reproduced claim rollback demands it.
Public R-03 health remains historical liveness. Intentionally Redis-free RAM fallback and its historical successful ACK stay unchanged. No app/UI/auth/API schema/SDK/Docker/production-data/deployment changes. Work only in a new clean temp worktree; original PC repository's staged 16 P01 files and a separate pre-existing dirty candidate worktree preserved. No production readiness claim until CI #69 and final audit. **R-03:** public `/health` remains HTTP 200 liveness; no readiness endpoint, UI or auth change.
**CI facts:** Gitea [#66](https://dev.ivib.cc/vibtools/DataStack/actions/runs/66) terminal SUCCESS for initial patch; [#68](https://dev.ivib.cc/vibtools/DataStack/actions/runs/68) diagnosed a same-ID retry assertion regression; the exact local idempotency-claim cleanup was fixed in scope. [#70](https://dev.ivib.cc/vibtools/DataStack/actions/runs/70) is the latest disposable source acceptance (check terminal outcome in notes/003). **No code was deployed to production.**
### P02 Final Focused Correction — CI #69 to #72
Gitea [CI #69](https://dev.ivib.cc/vibtools/DataStack/actions/runs/69) was run against `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532`: the new Redis stale-claim test timed out because it globally replaced `Date.now` during live Socket.IO ACK scheduling. This is a **test-harness failure**, not a proven backend regression; other new CSV.gz, partial-corruption, mixed-new/durable and Redis-free regression cases passed. An additional source-grounded edge case was found: `batch_sync` previously failed to reject a batch containing a non-durable claim PLUS an already accepted duplicate when `processed=0, duplicates>0`. Under exact approved scope, the `server/socket/handlers.js` branch changed from `if (volatileDuplicate && processed > 0)` to `if (volatileDuplicate)`, ensuring false overall ACK is never sent for any such mixture. Test now uses local simulated TTL arithmetic and real-clock Socket.IO retries; no global clock manipulation. Corrected commit `acca8be01a31f8d3972564cbaec8fd3117a0b551` sent to [focused CI #72](https://dev.ivib.cc/vibtools/DataStack/actions/runs/72); still no P02 closure or production deployment until PASS.
### P02 definitive CI acceptance (2026-11-22)
Source-verified P02 final correction is `acca8be01a31f8d3972564cbaec8fd3117a0b551`. [Gitea CI #72](https://dev.ivib.cc/vibtools/DataStack/actions/runs/72) terminal **SUCCESS** (Linux 24/24 TypeScript suites, explicit R-01/R-02 fault cases, strict tsc, ESLint, Windows Node SDK, Next.js standalone build). Earlier #69 was cancelled after a test-only global `Date.now` Socket.IO ACK timeout; fixed by removing global clock override and adding direct arithmetic proof without changing timers. #72 proves the corrected test passes. Final source-path audit: exactly 11 approved paths (3 runtime, 2 tests, 6 notes). No keys/secrets, auth/SDK/UI/public health, dependency pins, CI workflows, deployment or database migrations changed. ACK remains **best-effort in intentionally Redis-free RAM mode**; Redis-configured RPUSH failure must return negative ACK; CSV deduplication guarantees only tested recovery windows, not globally atomic exactly-once. Partition scans are memory-streamed but disk reads scale with matching partition size, and production throughput is NOT certified. Release/installation still requires separate authorization.
## P03 production forensic acceptance — 2026-11-22
Baseline Gitea/GitHub `48fb277079b380fdbf1e58cd2da40afeed0b03ee`; GitLab baseline same tracked tree `41a169eec0c05ce19bc4acc29d7a8cfe1f8dd90d` (317/317 blobs). Final P03 workflow candidate `549d575653b68c29f97e2f78036646f2e2035719`. [Gitea CI #77](https://dev.ivib.cc/vibtools/DataStack/actions/runs/77): overall **FAILURE** because the initial Python-only checkout lacked Node (exit 127), **not** an SDK regression. Its other six jobs terminal **SUCCESS**: Node `CI_SUITE_SUMMARY total=24 failed=0`, strict TS+ESLint, Next standalone build, Windows SDK, Compose + isolated BuildKit ingestion/web/proxy image build `DATASTACK_THREE_IMAGES_BUILD_PASS`, read-only advisory inventory `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Narrow [Python-only CI #78](https://dev.ivib.cc/vibtools/DataStack/actions/runs/78) terminal **SUCCESS**: Node-capable Bookworm container, Python 3.11 isolated venv, `Ran 14 tests in 3.959s`, `OK`, `DATASTACK_PYTHON_14_OF_14_PASS`. The other six jobs were deliberately skipped in #78.
**CI-08 security disposition:** `npm audit --json` full=26 affected packages (25 moderate, 1 high); runtime=24 (23 moderate, 1 high), **not counts of distinct advisories**. High-severity advisories [GHSA-6g55-p6wh-862q](https://github.com/advisories/GHSA-6g55-p6wh-862q), [GHSA-r28c-9q8g-f849](https://github.com/advisories/GHSA-r28c-9q8g-f849) affect Next 15.5.29's nested `postcss@8.4.31` despite patched root `postcss@8.5.33`; npm recommends major `next@16.4.6`. Direct `cloudcmd@19.24.0` carries moderate transitive issues (`dompurify`, `file-type`, `glob`, `minimatch`, `got`); npm recommends `cloudcmd@17.0.2` downgrade, not a verified safe pin. No speculative package pin, major upgrade or force fix; a separately tested compatible remediation is needed before release.
**Scope / release decision:** only `.gitea/workflows/datastack-ci.yml`, `.gitea/workflows/datastack-docker-ci.yml`, and `notes/001–006` changed. No `package.json`/lock, Dockerfile, application UI/UX, runtime, auth, SDK, database, Compose, production data, secret, shared Docker prune, deployment, migration, tag or release. P03 technical CI and advisory-triage acceptance verified by #77 non-Python jobs plus #78 Python; do **not** mislabel #77 overall as successful. Final phase publication requires one main commit and Gitea/GitHub/GitLab exact tracked tree parity. **Production release NO-GO** while runtime high advisory is unresolved; live production health, throughput and deployment remain unverified.
## P04-S01 blocked security reconciliation report
P04-S01 SEC-01 BLOCKED — 2026-11-22. Approved scoped `overrides.next.postcss=8.5.33` trialed in disposable Windows npm 10.9.8 workspace only. `npm install --package-lock-only` EXIT 0, but `npm ls --package-lock-only postcss --all` EXIT 1 (`ELSPROBLEMS`: Next nested `postcss@8.4.31` invalid; expected 8.5.33); npm dedupe also kept invalid dependency. Read-only `npm audit --omit=dev --json` EXIT 1, runtime high 1/moderate 23/critical 0. Lockfile regeneration caused unrelated churn (91 package entries added, 106 removed, 69 changed, including two lightningcss optional 1.33.0 -> 1.30.1). Fresh minimal Next 15.5.29 fixture with identical override resolves PostCSS 8.5.33, isolating failure to DataStack committed lock graph. `npm ci --dry-run` exit 0 does not prove real clean install. Exact stop rule applied: no dependency patch, no CI acceptance, no main commit, no GitHub/GitLab sync, no release/deploy; production NO-GO. Need separate explicit narrowly scoped lock reconciliation approval.
Root proof: `package-lock.json:10552-10560` Next pins 8.4.31; `:10602-10603` nested 8.4.31; `:11427-11428` root 8.5.33; advisories GHSA-6g55-p6wh-862q and GHSA-r28c-9q8g-f849.
## P04-S01B actual trial / release blocker — 2026-11-22
User explicitly approved P04-S01B stale-copy removal exception. In a clean disposable Windows npm 10.9.8 probe based on Gitea main `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3`, remove ONLY the stale nested PostCSS 8.4.31 entry from the disposable lock copy and set `overrides.next.postcss=8.5.33`. `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline` exited 0; `npm ls --package-lock-only postcss --all --depth=4` exited 0 and Next 15.5.29 used deduped 8.5.33. Read-only runtime audit on the **unpublished candidate** showed high=0, critical=0, moderate=22 (22 affected package entries); SEC-01 candidate mitigation confirmed but NOT accepted. The regenerated lock changed **67 unrelated package metadata entries** (36 `dev` and 29 `libc` field changes plus dependency order-only differences), despite zero unrelated version changes, and removed only the stale node. This violates explicit metadata preservation. STOP before real `npm ci`, Gitea CI, security-gate edit or main/mirror publication. No production release or security-clean claim for published baseline.
## P04-S01C verified candidate / Windows acceptance outstanding — 2026-11-22
Approved deterministic lockfile reconciliation fixed the **SEC-01 Next 15.5.29 nested PostCSS 8.4.31** candidate without touching any unrelated package metadata, versions, hashes, URLs or feature/runtime sources. Scoping: `package.json` scoped `overrides.next.postcss=8.5.33`; `package-lock.json` only 26-line removal of obsolete `node_modules/next/node_modules/postcss`; existing Gitea `security-audit` job adds runtime high/critical failure. Generated lockfile with deterministic baseline metadata restoration had SHA-256 `e6b5b362877627085dc04c9ded03c019971e07ef9f4af1e78057d58961a1c091` and Gitea branch bytes matched exactly. No unrelated package entry/metadata delta (0 added, 1 removed, 0 changed). Gitea [CI #85](https://dev.ivib.cc/vibtools/DataStack/actions/runs/85) candidate `c759be990c032c2474d2d0bad519c643dec11a3a`: Node 24/24, Python 14/14, strict TypeScript + ESLint, Next standalone, Docker 3/3 and runtime advisory gate all SUCCESS. Runtime audit **22 moderate, 0 high, 0 critical**; full audit **24 moderate, 0 high, 0 critical**. **Windows compatibility job still QUEUED at checkpoint; overall CI not yet terminal accepted, main and mirrors unchanged.** Production remains NO-GO pending Windows CI, 3-remote sync, remaining moderate advisories/live health.
## P04-S01C FINAL CI acceptance (7/7) — 2026-11-22
Gitea CI #85 https://dev.ivib.cc/vibtools/DataStack/actions/runs/85 COMPLETED/SUCCESS; tested implementation SHA c759be990c032c2474d2d0bad519c643dec11a3a; later changes to candidate are documentation only. All seven jobs terminal SUCCESS: node-regression (Linux committed-lock npm ci and exactly 24/24 TypeScript suites); python-regression (Ran 14 tests, OK); quality (strict tsc and ESLint); production-build (Next 15.5.29 standalone, 22/22 pages); docker-image-build (three isolated BuildKit ingestion/web/proxy images); windows-compatibility (Windows committed-lock npm ci and Node SDK build, job #182, terminal SUCCESS); security-audit (runtime moderate=22 high=0 critical=0; full moderate=24 high=0 critical=0; DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO). Single Gitea acceptance run used, no additional broad reruns.
Scope: exactly nine approved paths: package.json scoped next->postcss 8.5.33 override; package-lock.json removes only old nested postcss@8.4.31 entry; existing .gitea/workflows/datastack-ci.yml security-audit job high/critical rejection; notes/001–006 evidence. Other package metadata, versions, integrity and resolved URLs match baseline. Next.js 15.5.29 and Cloudcmd 19.24.0 preserved. Final candidate package-lock SHA256 e6b5b362877627085dc04c9ded03c019971e07ef9f4af1e78057d58961a1c091 (verified byte-for-byte with Gitea). No API, auth, SDK, UI, application runtime, database, Compose, Dockerfile, tests or production data changes, deployment, migration, prune, tag, release.
CI fully accepted; publication not yet asserted in these pre-merge notes. Final phase CLOSED requires one squash merge of PR #5 into Gitea main and verified identical tracked content across Gitea/GitHub/GitLab; record actual new SHAs in terminal report. Production release remains NO-GO pending P04-S02 moderate vulnerability triage and separately authorized live readiness verification.
## P04-S02 scoped candidate — 2026-11-22
Baseline Gitea main `2d467ff1494858164a37814f7dbc2b321f510291`, tree `648c76ca496d2c9bf22a16f558c9ba347a45d4f7`. Cloudcmd 19.24.0 -> deepword 11.0.1 -> monaco-editor 0.55.1 pins DOMPurify 3.2.7, vulnerable according to npm advisory inventory. Disposable Windows npm 10.9.8 `npm view dompurify@3.4.18` verified, scoped npm override `overrides.cloudcmd.dompurify=3.4.18` resolves `dompurify@3.4.18` via `npm ls --package-lock-only dompurify --all --depth=5` (exit 0). `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline` exit 0. npm graph initially changed 67 unrelated metadata entries; deterministically restored exact baseline values after validating unrelated versions, URLs, integrity and dependencies. Final lock has **one changed package entry**, DOMPurify; same-major replacement plus verified registry `resolved` and `integrity`, no other packages added/deleted/changed. Candidate runtime audit affected-package counts **moderate 20/high 0/critical 0** versus baseline **22/0/0**; full candidate **moderate 22/high 0/critical 0** versus baseline **24/0/0**. DOMPurify and Monaco Editor dropped from vulnerable-package inventory. Other moderate packages (got, glob, minimatch, nested file-type, Cloudcmd transitive chain) remain; they require major package changes under current advisory constraints, prohibited by Scope Lock. This is a **pre-acceptance candidate**: do not claim application compatibility, CI PASS, publication, or phase CLOSED before full focused Gitea CI and exact three-remote tree parity.
Security forensic disposition: only DOMPurify's same-major compatible override supported by the disposable registry/audit proof; other moderate transitive packages not force-fixed.
## P04-S02 final CI acceptance — 2026-11-22
Gitea CI #92 (https://dev.ivib.cc/vibtools/DataStack/actions/runs/92) **COMPLETED / SUCCESS**, all 7/7 jobs terminal success on source commit `a88bc5f4498ea1b11acc0acfeb66b5aa9d6531be`: node-regression real Linux npm ci + 24/24 tests including `tests/files-manager.test.ts` exit 0; Python SDK 14/14; quality strict TypeScript/ESLint; Next.js standalone (22/22 static pages); isolated Docker/BuildKit 3-image verification `DATASTACK_THREE_IMAGES_BUILD_PASS`; Windows committed-lock npm ci + Node SDK compatibility; security-audit `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO` and `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Audit full=22 moderate/high=0/critical=0 (baseline 24), runtime=20 moderate/high=0/critical=0 (baseline 22). These are affected package entries, not distinct advisories. No new CI workflow or extra broad CI dispatched.
Exact approved patch: `package.json` scoped `overrides.cloudcmd.dompurify=3.4.18`, preserving `overrides.next.postcss=8.5.33`; npm-generated `package-lock.json` changes solely `node_modules/dompurify` version 3.2.7 -> 3.4.18 and verified `resolved`/SHA512 integrity. Baseline Cloudcmd 19.24.0, Monaco Editor 0.55.1, Next.js 15.5.29, all unrelated locked package versions, URL, hash, dependency and metadata preserved. Gitea published candidate lock bytes match disposable SHA-256 `57c5ea9df9f28be496f8c3202fbc280b0066bf8c216b8d8a3ca98b6f7b942854`. Exact candidate tree diff 420 entries, 8 approved changed files only (`package.json`, `package-lock.json`, notes/001–006), 0 unauthorized paths. No runtime/UI/auth/API/SDK/tests/CI/Dockerfiles/Compose/db/schema/data modifications.
Security forensic result. **Residual risk:** 20 moderate runtime affected package entries remain, including Cloudcmd transitive dependencies `got`, `glob`, `minimatch`, nested `file-type` and their parent chains. Current npm audit suggests `cloudcmd@17.0.2` (a major downgrade), explicitly out-of-scope; other transitive fixes need major-version dependency-contract changes. No force/downgrade/unverified override. Security improvement verified but NOT zero vulnerabilities. Production release stays NO-GO; remaining advisory disposition and live readiness need independent scope approval.
**Merge gate:** CI #92 7/7 PASS, approved 8-file patch integrity PASS; main squash merge and Gitea/GitHub/GitLab exact Git content-tree verification required before P04-S02 CLOSED. This note is written before that publication, so final remote commit SHAs shall be appended to the PR closure report after verification. No deployment, migrations, prune, tag, release.
+6 -83
View File
@@ -1,7 +1,7 @@
# DataStack — Production Update Roadmap / Exact Scope Lock # DataStack — Production Update Roadmap / Exact Scope Lock
Planning baseline: `d725841ebc828d58cbe90f517da9e600f23c4630` (frozen; documentation commits will advance Git HEAD but NOT this source baseline). Planning baseline: `d725841ebc828d58cbe90f517da9e600f23c4630` (frozen; documentation commits will advance Git HEAD but NOT this source baseline).
Current state: **P01 CLOSED; P02 CI ACCEPTED (Gitea #72 terminal SUCCESS), final three-remote publication gate; P03 NOT STARTED and requires separate explicit approval**. See latest P02 final acceptance appendix and publication report. Current state: **P01 CLOSED and CI-verified (1 of 3 phases complete); P02 and P03 NOT STARTED; P02 requires new explicit approval**.
Prerequisite for any phase: compare current HEAD and changed paths with this baseline and previously committed notes. For new commits since source baseline, inspect exact diff; no implicit rebase or silent discard. Prerequisite for any phase: compare current HEAD and changed paths with this baseline and previously committed notes. For new commits since source baseline, inspect exact diff; no implicit rebase or silent discard.
## Global non-negotiable Scope Lock (applies to ALL phases) ## Global non-negotiable Scope Lock (applies to ALL phases)
@@ -91,88 +91,11 @@ Local candidate `44eee8877e40d1683f44502c2a0ba3d437a03b88` is **not release-read
**Stop rule:** if required fixes need touching `validateSingleEventAsync`, `csvWriter.js` or separate journal/storage scheme, do not broaden the implementation scope; record the residual and request new explicit approval. Existing unrelated files including `MEMORY.md` remain unchanged due exact Scope Lock; this note is the cross-IDE handoff. P03 must wait for P02 CLOSED, not merely implementation committed. **Stop rule:** if required fixes need touching `validateSingleEventAsync`, `csvWriter.js` or separate journal/storage scheme, do not broaden the implementation scope; record the residual and request new explicit approval. Existing unrelated files including `MEMORY.md` remain unchanged due exact Scope Lock; this note is the cross-IDE handoff. P03 must wait for P02 CLOSED, not merely implementation committed.
## P02 Post-implementation approval boundary — 2026-11-22
## P02-FINAL-ACCEPTANCE-01 Execution / P03 Gate The separately approved R-01/R-02 **Socket.IO and CSV flush** hardening is implemented on an isolated branch, with all allowed runtime sources/tests only. Existing candidate approval text above is historic; the newest evidence is the appendix in notes/001 and notes/003. P02 must **NOT** be closed solely on P02 branch CI success: the HTTP `POST /api/events` RAM-fallback-success path is source-observed outside the current runtime method allowlist. P03 remains NOT STARTED until P02 acceptance and a new explicit approval.
User approved strict P02 final acceptance after `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898` published diagnosis and `76cd972bdeea9ef07b4b38e9d8b5b5ed3312ca5b` runtime candidate. Previously failed claim is negative timestamp cleaned with positive-timestamp TTL arithmetic; final patch changes only `server/socket/handlers.js` sync_data/batch_sync marker and mixed-batch handling, plus `tests/phase02-sync.test.ts` and `tests/phase03-storage-buffer.test.ts`. It does not edit production startup, public health, schema, CSV format, SDK or dependency files. Candidate CI #66 **terminal SUCCESS**: all 24/24 TypeScript suites, strict tsc, lint, Windows Node SDK and Next build PASS. Final focused CI #69 on `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532` was dispatched; confirm terminal results before any P02 CLOSED claim. No repeated broad CI runs unless this run demonstrates a scoped failure. **Proposed separate Scope Lock approval — P02-HTTP-DURABILITY-01 (NOT YET APPROVED):**
`APPROVED — DataStack P02-HTTP-DURABILITY-01. Use latest verified P02 runtime branch and notes/001–006. Default WRITE only app/api/events/route.ts POST; core/data/buffer.ts writeEventsToRedisOrFallback, pushAsync and pushBatchAsync; tests/phase02-sync.test.ts, tests/phase03-storage-buffer.test.ts, tests/r4-phase02-ingestion-audit-streaming.test.ts; notes/001–006. Allow core/Security/security.ts validateEventAsync/idempotency rollback ONLY if a focused failing test proves stale claims cannot be handled within listed methods, and record exact lines before any edit. Fix HTTP success on volatile RAM after Redis write failure; preserve legitimate Redis-free CSV durable fallback, cross-transport duplicate correctness, current 401/403/429/507 contracts, SDK wire format, auth and user data. Use no production Redis/DB, no live requests, no UI, SDK, Docker, migrations, releases, R-03 health change or unrelated files. One focused disposable Gitea CI, exact patch/security audit, notes update, one Git commit, three-remote content parity. Then close P02 only if the entire acceptance matrix is demonstrated; provide P03 approval command.`
**Acceptance contract:** ACK durability applies to Redis-backed events when Redis configured. Intentionally Redis-free RAM mode retains its historical best-effort ACK. CSV replay is at-least-once with per-partition event-id duplicate suppression under tested crash windows, not a global transactional exactly-once guarantee. Existing scan uses streaming parsing (bounded memory) and scans entire matching partitions (disk I/O scales with data size; monitor before production promotion). Corrupt matching CSV fails closed. If a hard upper disk-scan budget is required for production traffic, obtain a dedicated storage/index design approval rather than silently breaking ingestion.
**Next P03 (ONLY after P02 closure):** CI-06 Python runner environment and 14/14 Python SDK; CI-07 three Docker image builds/isolated BuildKit with no live prune; CI-08 read-only npm advisory IDs/direct-transitive audit, compatible dependency pins only if justified; Compose/standalone/health gate evidence and manual release decision. Exact P03 allowlist remains in this roadmap. No live deploy, migrations, tags or releases without separate approval.
### P02 Final CI Exception / Narrow Retry
The final acceptance test-only clock override in #69 interfered with Socket.IO ACK timing. The source-grounded mixed `batch_sync` all-duplicate edge also required a single conditional correction within the previously approved handler. Commit `acca8be01a31f8d3972564cbaec8fd3117a0b551` changes only `server/socket/handlers.js` and `tests/phase02-sync.test.ts` after initial final candidate; one narrow Gitea CI #72 requested because CI #69 node-regression failed one fixture. Other scope/production invariants remain unchanged. P02 closure requires terminal #72 and exact patch/security evidence.
## P02 Final Acceptance Closure Gate / P03 approval handoff — 2026-11-22
P01 CLOSED; **P02 tests/quality/build ACCEPTED via terminal [Gitea CI #72](https://dev.ivib.cc/vibtools/DataStack/actions/runs/72) SUCCESS** on `acca8be01a31f8d3972564cbaec8fd3117a0b551`; P02 phase transitions to CLOSED only when the verified code plus these documentation-only notes are merged to Gitea main and published to GitHub/GitLab with identical tracked Git trees. Terminal jobs: node-regression 24/24 PASS; quality strict TypeScript and ESLint PASS; Windows compatibility SDK PASS; production Next standalone PASS; Python and Docker intentionally skipped (P03). #69 historical test-only clock failure is resolved in #72. A production deployment and all live services remain **NOT VERIFIED** and NOT authorized.
**P03 next / exact future Scope Lock (requires separate user approval):** CI-06 Linux/Python environment: `.gitea/workflows/datastack-ci.yml` and `.gitea/workflows/datastack-python-ci.yml` only; Python SDK 14/14 isolated. CI-07 Docker/BuildKit 3-image verification: `.gitea/workflows/datastack-ci.yml`, `.gitea/workflows/datastack-docker-ci.yml` only, runner evidence read-only; Dockerfiles can be edited ONLY after independently approved exact exception, no shared runner prune. CI-08 advisory triage: read-only `npm audit --json`, actual CVE/advisory and direct/transitive dependency graph; only minimal justified `package.json`/`package-lock.json` pins under this phase scope, never forced audit repair. `.gitea/workflows/datastack-eslint-ci.yml` quality harness only if source-proven CI-08 need. `notes/001–006` for evidence. Existing UI/API/auth/SDK/runtime logic, Compose, volumes, DB data, secrets, real host, and release configuration remain unchanged. One focused acceptance / one exact patch/security audit / one main commit and 3-way tree sync. Do not deploy, run live ingestion, migrate, prune shared Docker, tag or release.
**P03 approval command (not authorization until user explicitly sends):**
```text
APPROVED — DataStack Production Readiness P03.
Use notes/002 P03 CI-06/CI-07/CI-08 Exact Scope Lock and the verified published P02 main baseline.
CI-06: fix Python runner/toolchain only in approved .gitea workflows; verify Python SDK 14/14.
CI-07: isolate Docker runner/BuildKit and build all 3 images without deploying or pruning shared caches; Dockerfiles READ-ONLY unless a separately approved exact exception.
CI-08: read-only npm audit advisory inventory, minimal compatible package pins only for proven security need; no npm audit fix --force.
Preserve UI/UX, app/API/auth/SDK/runtime, Compose, production data, secrets and existing features unchanged.
Use one focused Gitea CI, strict patch/security diff, notes/001–006 update, Git commit and Gitea/GitHub/GitLab tree parity.
No live deployment, migrations, tags or releases.
Close P03 only on verified acceptance; provide production release readiness report.
```
## P03 scoped execution and release gate — 2026-11-22
Baseline Gitea/GitHub `48fb277079b380fdbf1e58cd2da40afeed0b03ee`; GitLab baseline same tracked tree `41a169eec0c05ce19bc4acc29d7a8cfe1f8dd90d` (317/317 blobs). Final P03 workflow candidate `549d575653b68c29f97e2f78036646f2e2035719`. [Gitea CI #77](https://dev.ivib.cc/vibtools/DataStack/actions/runs/77): overall **FAILURE** because the initial Python-only checkout lacked Node (exit 127), **not** an SDK regression. Its other six jobs terminal **SUCCESS**: Node `CI_SUITE_SUMMARY total=24 failed=0`, strict TS+ESLint, Next standalone build, Windows SDK, Compose + isolated BuildKit ingestion/web/proxy image build `DATASTACK_THREE_IMAGES_BUILD_PASS`, read-only advisory inventory `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Narrow [Python-only CI #78](https://dev.ivib.cc/vibtools/DataStack/actions/runs/78) terminal **SUCCESS**: Node-capable Bookworm container, Python 3.11 isolated venv, `Ran 14 tests in 3.959s`, `OK`, `DATASTACK_PYTHON_14_OF_14_PASS`. The other six jobs were deliberately skipped in #78.
**CI-08 security disposition:** `npm audit --json` full=26 affected packages (25 moderate, 1 high); runtime=24 (23 moderate, 1 high), **not counts of distinct advisories**. High-severity advisories [GHSA-6g55-p6wh-862q](https://github.com/advisories/GHSA-6g55-p6wh-862q), [GHSA-r28c-9q8g-f849](https://github.com/advisories/GHSA-r28c-9q8g-f849) affect Next 15.5.29's nested `postcss@8.4.31` despite patched root `postcss@8.5.33`; npm recommends major `next@16.4.6`. Direct `cloudcmd@19.24.0` carries moderate transitive issues (`dompurify`, `file-type`, `glob`, `minimatch`, `got`); npm recommends `cloudcmd@17.0.2` downgrade, not a verified safe pin. No speculative package pin, major upgrade or force fix; a separately tested compatible remediation is needed before release.
**Scope / release decision:** only `.gitea/workflows/datastack-ci.yml`, `.gitea/workflows/datastack-docker-ci.yml`, and `notes/001–006` changed. No `package.json`/lock, Dockerfile, application UI/UX, runtime, auth, SDK, database, Compose, production data, secret, shared Docker prune, deployment, migration, tag or release. P03 technical CI and advisory-triage acceptance verified by #77 non-Python jobs plus #78 Python; do **not** mislabel #77 overall as successful. Final phase publication requires one main commit and Gitea/GitHub/GitLab exact tracked tree parity. **Production release NO-GO** while runtime high advisory is unresolved; live production health, throughput and deployment remain unverified.
## P04-S01 approved security remediation scope — 2026-11-22
Baseline Gitea `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3`, tree `12818ba732f5abdc42af46d83278b13b6be4eafd`. SEC-01: Next `15.5.29` nested `postcss@8.4.31` is the sole P03 high runtime affected package despite top-level `postcss@8.5.33`; see CI #77 `AUDIT_PACKAGE runtime postcss`, GHSA-6g55-p6wh-862q / GHSA-r28c-9q8g-f849. Source `package-lock.json:10552-10560`, `10602-10603`, `11427-11428`; `package.json:35,37`. Approval authorizes only scoped override in `package.json`, npm-generated `package-lock.json`, **existing** `.gitea/workflows/datastack-ci.yml` security-audit job and `notes/001–006`. All application, test, SDK, Docker/Compose and deployment files READ-ONLY. Never force next@16, downgrade cloudcmd, `npm audit fix --force`, prune shared Docker, deploy, migrate or tag. Acceptance: reproducible lockfile, clean npm ci, Node 24/24, Python 14/14, tsc/lint, Windows SDK, Next standalone, 3 Docker images, runtime high/critical zero and exact Git diff. One focused Gitea CI, one accepted main commit, three-remote content parity; unresolved moderates/live checks retain production NO-GO. If override incompatible, STOP as BLOCKED and request exact exception.
## P04-S01 blocked security reconciliation report
P04-S01 SEC-01 BLOCKED — 2026-11-22. Approved scoped `overrides.next.postcss=8.5.33` trialed in disposable Windows npm 10.9.8 workspace only. `npm install --package-lock-only` EXIT 0, but `npm ls --package-lock-only postcss --all` EXIT 1 (`ELSPROBLEMS`: Next nested `postcss@8.4.31` invalid; expected 8.5.33); npm dedupe also kept invalid dependency. Read-only `npm audit --omit=dev --json` EXIT 1, runtime high 1/moderate 23/critical 0. Lockfile regeneration caused unrelated churn (91 package entries added, 106 removed, 69 changed, including two lightningcss optional 1.33.0 -> 1.30.1). Fresh minimal Next 15.5.29 fixture with identical override resolves PostCSS 8.5.33, isolating failure to DataStack committed lock graph. `npm ci --dry-run` exit 0 does not prove real clean install. Exact stop rule applied: no dependency patch, no CI acceptance, no main commit, no GitHub/GitLab sync, no release/deploy; production NO-GO. Need separate explicit narrowly scoped lock reconciliation approval.
Approved scope remains `package.json` scoped override, npm-generated `package-lock.json`, current Gitea `security-audit` job, notes/001–006. Do not add manual lock edits, major Next 16, Cloudcmd downgrade, npm audit fix --force or unrelated source files. Phase BLOCKED until amended approval.
## P04-S01B correction gate — 2026-11-22
The targeted removal-and-regenerate method fixes SEC-01 in an isolated copy: valid `npm ls` exit 0 and `npm audit --omit=dev` high=0/critical=0/moderate=22, while Next 15.5.29, Cloudcmd 19.24.0 and all unrelated versions remain unchanged. **But** npm regenerated 67 unrelated package metadata entries (36 `dev` flags, 29 `libc` fields, 2 dependency map order changes); that is outside the expressly approved preservation contract. P04-S01B **BLOCKED by exact scope lock**, not by unresolved PostCSS resolution. Do not commit the unacceptable lock or quietly normalize metadata. Require separate approved exact metadata-reconciliation strategy/exception and content-preserving proof. Until then no acceptance CI, commit to main, remote parity, release or deployment.
## P04-S01C implementation checkpoint and exact release gate — 2026-11-22
Approved P04-S01C changes are strictly `package.json`, `package-lock.json`, existing `.gitea/workflows/datastack-ci.yml:security-audit`, notes/001–006. Applied scoped Next→PostCSS 8.5.33 override, npm regenerated disposable lock after stale entry deletion, deterministically restored **all 67** unrelated npm metadata modifications against baseline; exact final diff only stale nested package removal. No version, integrity, peer or optional metadata drift. Gitea branch patch `c759be990c032c2474d2d0bad519c643dec11a3a`; lock SHA-256 `e6b5b362877627085dc04c9ded03c019971e07ef9f4af1e78057d58961a1c091`. Focused [CI #85](https://dev.ivib.cc/vibtools/DataStack/actions/runs/85) six jobs terminal SUCCESS including Docker 3/3, Python 14/14, Node 24/24, Next/tsc/lint, runtime high/critical zero; Windows compatibility **QUEUED**, so P04-S01C **PENDING**. Do not merge/mark CLOSED until Windows job succeeds, overall #85 SUCCESS, exact file-scope audit and Gitea/GitHub/GitLab tracked-tree parity; no deployment, migration, release/tag/prune. After closure, independently approve P04-S02 Cloudcmd moderate advisory triage and later production readiness.
## P04-S01C FINAL CI acceptance (7/7) — 2026-11-22
Gitea CI #85 https://dev.ivib.cc/vibtools/DataStack/actions/runs/85 COMPLETED/SUCCESS; tested implementation SHA c759be990c032c2474d2d0bad519c643dec11a3a; later changes to candidate are documentation only. All seven jobs terminal SUCCESS: node-regression (Linux committed-lock npm ci and exactly 24/24 TypeScript suites); python-regression (Ran 14 tests, OK); quality (strict tsc and ESLint); production-build (Next 15.5.29 standalone, 22/22 pages); docker-image-build (three isolated BuildKit ingestion/web/proxy images); windows-compatibility (Windows committed-lock npm ci and Node SDK build, job #182, terminal SUCCESS); security-audit (runtime moderate=22 high=0 critical=0; full moderate=24 high=0 critical=0; DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO). Single Gitea acceptance run used, no additional broad reruns.
Scope: exactly nine approved paths: package.json scoped next->postcss 8.5.33 override; package-lock.json removes only old nested postcss@8.4.31 entry; existing .gitea/workflows/datastack-ci.yml security-audit job high/critical rejection; notes/001–006 evidence. Other package metadata, versions, integrity and resolved URLs match baseline. Next.js 15.5.29 and Cloudcmd 19.24.0 preserved. Final candidate package-lock SHA256 e6b5b362877627085dc04c9ded03c019971e07ef9f4af1e78057d58961a1c091 (verified byte-for-byte with Gitea). No API, auth, SDK, UI, application runtime, database, Compose, Dockerfile, tests or production data changes, deployment, migration, prune, tag, release.
Next roadmap: P04-S02 Cloudcmd moderate-advisory triage, followed by separate live readiness approval. No forced major upgrade/downgrade. CI fully accepted; publication not yet asserted in these pre-merge notes. Final phase CLOSED requires one squash merge of PR #5 into Gitea main and verified identical tracked content across Gitea/GitHub/GitLab; record actual new SHAs in terminal report. Production release remains NO-GO pending P04-S02 moderate vulnerability triage and separately authorized live readiness verification.
## P04-S02 scoped candidate — 2026-11-22
Baseline Gitea main `2d467ff1494858164a37814f7dbc2b321f510291`, tree `648c76ca496d2c9bf22a16f558c9ba347a45d4f7`. Cloudcmd 19.24.0 -> deepword 11.0.1 -> monaco-editor 0.55.1 pins DOMPurify 3.2.7, vulnerable according to npm advisory inventory. Disposable Windows npm 10.9.8 `npm view dompurify@3.4.18` verified, scoped npm override `overrides.cloudcmd.dompurify=3.4.18` resolves `dompurify@3.4.18` via `npm ls --package-lock-only dompurify --all --depth=5` (exit 0). `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline` exit 0. npm graph initially changed 67 unrelated metadata entries; deterministically restored exact baseline values after validating unrelated versions, URLs, integrity and dependencies. Final lock has **one changed package entry**, DOMPurify; same-major replacement plus verified registry `resolved` and `integrity`, no other packages added/deleted/changed. Candidate runtime audit affected-package counts **moderate 20/high 0/critical 0** versus baseline **22/0/0**; full candidate **moderate 22/high 0/critical 0** versus baseline **24/0/0**. DOMPurify and Monaco Editor dropped from vulnerable-package inventory. Other moderate packages (got, glob, minimatch, nested file-type, Cloudcmd transitive chain) remain; they require major package changes under current advisory constraints, prohibited by Scope Lock. This is a **pre-acceptance candidate**: do not claim application compatibility, CI PASS, publication, or phase CLOSED before full focused Gitea CI and exact three-remote tree parity.
Roadmap: finish one focused Gitea acceptance, exact 8-file scope and three-remote parity. Remaining moderate findings deferred to independently approved scope, never silently upgraded.
## P04-S02 final CI acceptance — 2026-11-22
Gitea CI #92 (https://dev.ivib.cc/vibtools/DataStack/actions/runs/92) **COMPLETED / SUCCESS**, all 7/7 jobs terminal success on source commit `a88bc5f4498ea1b11acc0acfeb66b5aa9d6531be`: node-regression real Linux npm ci + 24/24 tests including `tests/files-manager.test.ts` exit 0; Python SDK 14/14; quality strict TypeScript/ESLint; Next.js standalone (22/22 static pages); isolated Docker/BuildKit 3-image verification `DATASTACK_THREE_IMAGES_BUILD_PASS`; Windows committed-lock npm ci + Node SDK compatibility; security-audit `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO` and `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Audit full=22 moderate/high=0/critical=0 (baseline 24), runtime=20 moderate/high=0/critical=0 (baseline 22). These are affected package entries, not distinct advisories. No new CI workflow or extra broad CI dispatched.
Exact approved patch: `package.json` scoped `overrides.cloudcmd.dompurify=3.4.18`, preserving `overrides.next.postcss=8.5.33`; npm-generated `package-lock.json` changes solely `node_modules/dompurify` version 3.2.7 -> 3.4.18 and verified `resolved`/SHA512 integrity. Baseline Cloudcmd 19.24.0, Monaco Editor 0.55.1, Next.js 15.5.29, all unrelated locked package versions, URL, hash, dependency and metadata preserved. Gitea published candidate lock bytes match disposable SHA-256 `57c5ea9df9f28be496f8c3202fbc280b0066bf8c216b8d8a3ca98b6f7b942854`. Exact candidate tree diff 420 entries, 8 approved changed files only (`package.json`, `package-lock.json`, notes/001–006), 0 unauthorized paths. No runtime/UI/auth/API/SDK/tests/CI/Dockerfiles/Compose/db/schema/data modifications.
Roadmap: after P04-S02 accepted mirror parity, next separately approved P04-S03 remaining moderate advisory dependency-contract triage (read-only / disposable compatibility first), then P05 live readiness. **Residual risk:** 20 moderate runtime affected package entries remain, including Cloudcmd transitive dependencies `got`, `glob`, `minimatch`, nested `file-type` and their parent chains. Current npm audit suggests `cloudcmd@17.0.2` (a major downgrade), explicitly out-of-scope; other transitive fixes need major-version dependency-contract changes. No force/downgrade/unverified override. Security improvement verified but NOT zero vulnerabilities. Production release stays NO-GO; remaining advisory disposition and live readiness need independent scope approval.
**Merge gate:** CI #92 7/7 PASS, approved 8-file patch integrity PASS; main squash merge and Gitea/GitHub/GitLab exact Git content-tree verification required before P04-S02 CLOSED. This note is written before that publication, so final remote commit SHAs shall be appended to the PR closure report after verification. No deployment, migrations, prune, tag, release.
**P03 outline once P02 is truly CLOSED:** CI-06 restore reliable Python CI runner provisioning; CI-07 diagnose Docker/containerd Builder snapshot error and build all three images in disposable CI; CI-08 trace exact npm advisories and apply only proven compatible security upgrades. Preserve UI, auth, backend feature semantics and live data. No automatic deployment.
+10 -91
View File
@@ -2,13 +2,13 @@
Frozen source baseline: `d725841ebc828d58cbe90f517da9e600f23c4630`. Generated: 2026-11-22 Asia/Dhaka. Frozen source baseline: `d725841ebc828d58cbe90f517da9e600f23c4630`. Generated: 2026-11-22 Asia/Dhaka.
Scope: documentation-only kickoff. Any future phase must update this log **after** verification, not on intention. Scope: documentation-only kickoff. Any future phase must update this log **after** verification, not on intention.
**P01 CLOSED; P02 verified in Gitea CI #72 and awaiting recorded three-remote publication; P03 NOT STARTED.** Publication/tree-parity proof and final phase-close status are recorded in the latest acceptance appendix and operator report. **Completed implementation phases: 1 / 3. Remaining: 2.** Documentation baseline preparation is a separate non-implementation milestone.
| Phase | Status | IDs / function | Changed source | Verified CI evidence | Commit / remote sync | | Phase | Status | IDs / function | Changed source | Verified CI evidence | Commit / remote sync |
|---|---|---|---|---|---| |---|---|---|---|---|---|
| P00 Notes / freeze | COMPLETED (documentation only) | `001–006` docs, source baseline and exact scope | `notes/**` only | 6 Markdown files; balanced fences; staged diff `--check` PASS | Documentation commit containing this file; remote SHA verification recorded in session handoff | | P00 Notes / freeze | COMPLETED (documentation only) | `001–006` docs, source baseline and exact scope | `notes/**` only | 6 Markdown files; balanced fences; staged diff `--check` PASS | Documentation commit containing this file; remote SHA verification recorded in session handoff |
| P01 Tests & lockfile | **CLOSED — VERIFIED** | CI-01…CI-05, no runtime feature changes | 16 exact P01 code/test/CI paths, plus `notes/001–006` records | #59 Linux/Windows clean npm/SDK, 24/24 TS suites, Next standalone PASS; #60 strict TypeScript and ESLint PASS; historical #59 quality mismatch corrected | One permanent P01 main commit containing these notes; confirm remote SHA/tree parity in phase handoff | | P01 Tests & lockfile | **CLOSED — VERIFIED** | CI-01…CI-05, no runtime feature changes | 16 exact P01 code/test/CI paths, plus `notes/001–006` records | #59 Linux/Windows clean npm/SDK, 24/24 TS suites, Next standalone PASS; #60 strict TypeScript and ESLint PASS; historical #59 quality mismatch corrected | One permanent P01 main commit containing these notes; confirm remote SHA/tree parity in phase handoff |
| P02 Ingestion reliability | **CI VERIFIED #72 — final publish gate** | R-01 ACK/idempotency safety; R-02 CSV replay; R-03 liveness verified unchanged | Exactly 3 runtime files, 2 tests, notes/001-006 | #72 terminal SUCCESS: 24/24 TS, TS typecheck, lint, Next standalone, Windows SDK | Final source/test acca8be01a31f8d3972564cbaec8fd3117a0b551; three-remote main parity recorded on publication | | P02 Ingestion reliability | **BLOCKED — three diagnostic cases reproduced; runtime fix requires separate approval** | R-01 ACK/RAM loss; R-02 CSV duplicate replay; R-03 health liveness vs readiness | Only `tests/phase02-sync.test.ts` and `tests/phase03-storage-buffer.test.ts`, plus notes/001–006 | Gitea #63: all three P02 subtests PASS to reproduce existing behavior; full matrix status recorded in appendix | Diagnostic branch `p02-readiness-test-20261122`, no production runtime changes |
| P03 CI/Docker/security | NOT STARTED | CI-06,07,08; release gates | None | Not run | — | | P03 CI/Docker/security | NOT STARTED | CI-06,07,08; release gates | None | Not run | — |
## Existing diagnostic CI snapshot (NOT implementation phase completions) ## Existing diagnostic CI snapshot (NOT implementation phase completions)
@@ -108,93 +108,12 @@ Historical baseline code stayed frozen through these CI workflow changes. Curren
The authorized Windows PC attempted a **local disposable** `npm.cmd ci --ignore-scripts --no-audit --no-fund --prefer-offline` in the isolated P02 worktree only; it failed with `npm ERR! ETARGET No matching version found for yaml@2.9.1` plus cleanup EPERM warnings in the temp worktree's `node_modules`. Exact output captured in `%TEMP%\DataStack-P02-npm-ci.log` on the authorized PC. This is **not a passed code test nor proof of a repository package defect**, and `package-lock.json` / `package.json` cannot be edited under P02 scope. Gitea MCP remained UNAVAILABLE and the Git remote/HTTP service was not responding when CI was attempted, so the mandatory P02 focused CI was **not dispatched**. Phase remains BLOCKED pending actual CI evidence; no runtime change was pushed to production/main. The authorized Windows PC attempted a **local disposable** `npm.cmd ci --ignore-scripts --no-audit --no-fund --prefer-offline` in the isolated P02 worktree only; it failed with `npm ERR! ETARGET No matching version found for yaml@2.9.1` plus cleanup EPERM warnings in the temp worktree's `node_modules`. Exact output captured in `%TEMP%\DataStack-P02-npm-ci.log` on the authorized PC. This is **not a passed code test nor proof of a repository package defect**, and `package-lock.json` / `package.json` cannot be edited under P02 scope. Gitea MCP remained UNAVAILABLE and the Git remote/HTTP service was not responding when CI was attempted, so the mandatory P02 focused CI was **not dispatched**. Phase remains BLOCKED pending actual CI evidence; no runtime change was pushed to production/main.
## P02-RUNTIME-REMEDIATION-02 — Post-patch evidence/status (2026-11-22)
## P02-FINAL-ACCEPTANCE-01 Current Verification (latest record) - **P02 current status: BLOCKED / NOT CLOSED** by a separate Port 3000 HTTP RAM-ACK path (and until terminal CI evidence). Completed implementation phases remain **1/3** (P01 only); P03 not authorized.
- Baseline `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`; isolated Gitea branch `p02-runtime-remediation-20261122`; inspected code commit `5c880885378ae0f4fa8dd3a145f9c7055dada635`. Original PC root `main` at earlier docs SHA with 16 user-staged P01 paths remains untouched. No force-reset, stash, production write or deployment.
- **At this checkpoint: P02 acceptance PENDING CI #69**. P01 CLOSED; P03 NOT STARTED. Do not reinterpret earlier P02 BLOCKED audit as a completed release. - **Verified earlier focused CI #66:** 24/24 TS suites, strict TypeScript, ESLint, Windows/npm SDK and Next.js standalone production build success; Python/Docker jobs skipped as separate P03 work.
- Gitea #66 on `76cd972bdeea9ef07b4b38e9d8b5b5ed3312ca5b` **COMPLETED SUCCESS**: node-regression 24/24 suites PASS including direct Port4000 replay; quality (strict tsc+ESLint) PASS; production-build PASS; windows-compatibility SDK PASS; Python/Docker skipped by scoped `p01_only` input. - **CI #68:** new same-ID retry test failed because an unproven duplicate had left a local claim set; this was corrected in handlers on the subsequent scoped code commit. #68 was cancelled after failure (logs preserved). **CI #70:** revised focused acceptance and added HTTP failure diagnostic; confirm actual terminal job conclusions before claiming PASS. The HTTP diagnostic is intentionally expected to expose an unresolved behavior, not fix it.
- Final patch `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532` in clean isolated `p02-final-acceptance-20261122`: resolved source-grounded negative-marker TTL bug and mixed-batch success ACK, plus test-only stale-claim time jump (+61s), failed Redis claim, mixed batch, intentionally Redis-free compatibility, rotated .csv.gz+multiline replay and partial-corrupt CSV fail-closed/recovery. No source outside `server/socket/handlers.js` changed since candidate. - P02 code changes confined to `server/index.js` selected ACK/buffer/flush methods, `server/socket/handlers.js` persistence ACK/duplicate proof methods, `core/data/buffer.ts` flush, and three approved test files; notes/001–006 only. R-03 public health unchanged. All other app/UI/auth/SDK/CI workflow/lockfile/Docker/runtime source unchanged.
- One focused Gitea CI #69 dispatched for final patch; job terminal outcome and any corrective action must be recorded. **Do not merge until this passes and exact patch/security audit is reviewed.** - **Residual blocking source evidence:** `app/api/events/route.ts:744-754` returns success following `core/data/buffer.ts:412-442` silent Redis-error RAM fallback; Node SDK may switch to HTTP after negative WS ACK. A separate exact scope exception is required for end-to-end no-positive-ACK-on-volatile-data acceptance. Do not merge the blocked candidate into three `main` refs or assert P02 production ready.
- Original `D:\VibTools_Workspace\01_Products\14_DataStack` at old `d8ac96e` with 16 staged P01 changes untouched. Separate pre-existing `p02-runtime-remediation-20261122` had three dirty files; left untouched. The new isolated worktree was created clean from candidate SHA. - Performance boundary: replay checking can stream entire CSV partitions per flush (not throughput benchmarked); duplicate proof is capped at last 512 Redis queue items and 256 MiB/1.8s CSV proof (negative ACK on uncertainty). This is safer than a false positive, but compatibility under heavy ingestion must be reviewed before release.
- Source behavior: no Redis configured = historical RAM best-effort acceptance preserved; configured Redis failure = negative ACK; public /health unchanged. Per-CSV replay scanning bounded-memory but linear in stored bytes; no global exactly-once/throughput guarantee. P03 cannot begin until P02 acceptance/remote parity close.
### P02 Focused Retry — #69 fixture failure / #72 correction
- Final acceptance CI #69 on `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532`: node-regression **FAIL** because `phase02-sync` stale-marker test globally mocked `Date.now` and Socket.IO ACK timed out. Other new tests for gzip/rotated CSV, partial CSV corruption, Redis-free RAM and mixed batch with a new persisted event PASS. Strict TypeScript, lint and Windows SDK jobs PASS; full workflow conclusion must be read at terminal.
- New source-grounded all-duplicate mixed batch path (`processed=0`, `duplicates=1`, `rejected>0`, volatile claim present) could return overall success; added focused test and changed handler conditional to reject all `volatileDuplicate` combinations.
- Test correction removes process-wide Date.now override, verifies 61000ms/300000ms cleanup math with static source marker assertion, and checks real Socket.IO negative retries. No unrelated files, CI workflows, SDK, auth or public health modified.
- Commit `acca8be01a31f8d3972564cbaec8fd3117a0b551` submitted to disposable CI #72; **P02 remains pending / NOT CLOSED** until terminal acceptance and 3-way mirror parity.
## P02 Verified Acceptance and Conditional Publication — 2026-11-22
- **Code acceptance VERIFIED — Gitea CI #72 terminal SUCCESS.** Code SHA `acca8be01a31f8d3972564cbaec8fd3117a0b551`. Node regression exactly `CI_SUITE_SUMMARY total=24 failed=0`, individual `tests/phase02-sync.test.ts` and `tests/phase03-storage-buffer.test.ts` suites exit 0. Quality strict TS+ESLint, Next standalone build and Windows Node SDK jobs all SUCCESS. Python/Docker intentionally skipped in this focused P02 workflow.
- **Root causes fixed:** configured Redis RPUSH outage returns `BUFFER_PERSIST_FAILED` negative ACK rather than volatile RAM success; stale failed idempotency claim retains correct TTL and remains negative through retries; `batch_sync` with old durable duplicates plus failed volatile claim returns `success:false`; CSV-before-Redis-LTRIM replay suppresses previously written same-client/date/eventId across plain/rotated/gz CSV with multiline quoted payload; corrupt matching CSV prevents Redis trim and can be recovered after disposable repair.
- **Explicit limits:** Redis intentionally unconfigured retains historical best-effort RAM ACK; no globally atomic exactly-once promise; scan I/O proportional to matching partition bytes even though memory streaming; public /health continues liveness not dependency readiness; live production unknown. No deployment, migration, secret access, user-data mutation, tag, release or build of production images.
- **Historical testing:** #66 PASS candidate; #69 suite failure (Socket.IO ACK timeout caused by test-only global `Date.now` override) and cancellation; #72 PASS corrected source/test. No false claim that #69 succeeded.
- Final code/diff safety audit: 11/11 approved paths only: `server/index.js`, `server/socket/handlers.js`, `core/data/buffer.ts`, `tests/phase02-sync.test.ts`, `tests/phase03-storage-buffer.test.ts`, plus `notes/001–006`; `git diff --check` and JS syntax PASS. Original PC root old staged P01 (16 paths) and separate dirty earlier temporary worktree preserved.
- **Publication gate:** Gitea PR #3 final verified source + documentation notes; squash-merge with `[skip ci]`, publish to GitHub and GitLab, verify identical tracked Git trees despite potentially different GitLab commit SHA. P02 marked CLOSED only after three-remote content parity; evidence SHA/tree in operator's final publication report. P03 NOT STARTED and requires separate approval.
## P03 final CI acceptance ledger — 2026-11-22
| Gate | Terminal outcome |
| --- | --- |
| #77 Node | SUCCESS, 24/24 |
| #77 TypeScript / ESLint | SUCCESS |
| #77 Next standalone build | SUCCESS |
| #77 Windows Node SDK | SUCCESS |
| #77 isolated three-image BuildKit | SUCCESS |
| #77 read-only npm audit inventory | SUCCESS with 1 high runtime finding |
| #77 Python | FAILED before tests: checkout requires Node |
| #78 corrected Python-only | SUCCESS, 14/14; other jobs SKIPPED |
**CI-08 security disposition:** `npm audit --json` full=26 affected packages (25 moderate, 1 high); runtime=24 (23 moderate, 1 high), **not counts of distinct advisories**. High-severity advisories [GHSA-6g55-p6wh-862q](https://github.com/advisories/GHSA-6g55-p6wh-862q), [GHSA-r28c-9q8g-f849](https://github.com/advisories/GHSA-r28c-9q8g-f849) affect Next 15.5.29's nested `postcss@8.4.31` despite patched root `postcss@8.5.33`; npm recommends major `next@16.4.6`. Direct `cloudcmd@19.24.0` carries moderate transitive issues (`dompurify`, `file-type`, `glob`, `minimatch`, `got`); npm recommends `cloudcmd@17.0.2` downgrade, not a verified safe pin. No speculative package pin, major upgrade or force fix; a separately tested compatible remediation is needed before release.
**Scope / release decision:** only `.gitea/workflows/datastack-ci.yml`, `.gitea/workflows/datastack-docker-ci.yml`, and `notes/001–006` changed. No `package.json`/lock, Dockerfile, application UI/UX, runtime, auth, SDK, database, Compose, production data, secret, shared Docker prune, deployment, migration, tag or release. P03 technical CI and advisory-triage acceptance verified by #77 non-Python jobs plus #78 Python; do **not** mislabel #77 overall as successful. Final phase publication requires one main commit and Gitea/GitHub/GitLab exact tracked tree parity. **Production release NO-GO** while runtime high advisory is unresolved; live production health, throughput and deployment remain unverified.
## P04-S01 security remediation kickoff — 2026-11-22
**Status: IN PROGRESS / NOT ACCEPTED.** User-approved baseline `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3`. Scope `package.json`, npm-generated `package-lock.json`, existing Gitea CI `security-audit` job, `notes/001–006`. Root cause `node_modules/next` locks `postcss@8.4.31` while root is `8.5.33`; P03 runtime audit 1 high and 23 moderate package entries. Plan minimal nested npm override, lockfile-only regeneration, high/critical audit gate; preserve all runtime/UI/API/auth/SDK/DB/Compose/production. One focused CI acceptance and exact 3-remote tree equality required; DO NOT label closed before terminal results, publish or claim release-ready.
## P04-S01 blocked security reconciliation report
P04-S01 SEC-01 BLOCKED — 2026-11-22. Approved scoped `overrides.next.postcss=8.5.33` trialed in disposable Windows npm 10.9.8 workspace only. `npm install --package-lock-only` EXIT 0, but `npm ls --package-lock-only postcss --all` EXIT 1 (`ELSPROBLEMS`: Next nested `postcss@8.4.31` invalid; expected 8.5.33); npm dedupe also kept invalid dependency. Read-only `npm audit --omit=dev --json` EXIT 1, runtime high 1/moderate 23/critical 0. Lockfile regeneration caused unrelated churn (91 package entries added, 106 removed, 69 changed, including two lightningcss optional 1.33.0 -> 1.30.1). Fresh minimal Next 15.5.29 fixture with identical override resolves PostCSS 8.5.33, isolating failure to DataStack committed lock graph. `npm ci --dry-run` exit 0 does not prove real clean install. Exact stop rule applied: no dependency patch, no CI acceptance, no main commit, no GitHub/GitLab sync, no release/deploy; production NO-GO. Need separate explicit narrowly scoped lock reconciliation approval.
Candidate branch `p04-s01-postcss-security-20261122` contains notes only. Gitea main stays `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3`; no Gitea CI was dispatched and no phase acceptance established.
## P04-S01B run ledger — 2026-11-22
**P04-S01B = BLOCKED / stop rule.** Disposable Windows npm 10.9.8 only (not user project workspace). Verified baseline Gitea `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3` and candidate notes branch `4abcc55fee927b5ae2c1193cea2b4585b8d00158` before test. With approved stale node deletion in disposable lock plus scoped manifest override, `npm install --package-lock-only` 0, `npm ls --package-lock-only postcss --all --depth=4` 0, audit high 0 / critical 0 / moderate 22. Next 15.5.29, Cloudcmd 19.24.0 and every unrelated package version unchanged; package entries added 0, removed 1 (stale `node_modules/next/node_modules/postcss`), existing entries with changed metadata 67 (36 dev, 29 libc, 2 dependency ordering overlaps). **Reject generated candidate due preservation of unrelated metadata requirement.** No real npm ci, regression, Docker, focused Gitea CI, security gate edit, application changes, main commit, GitHub/GitLab mirror or release. P03 CLOSED, P04-S01B NOT CLOSED, published production still NO-GO.
## P04-S01C verified six-job acceptance checkpoint — 2026-11-22
- User-approved baseline Gitea main `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3` still unchanged; candidate source commit `c759be990c032c2474d2d0bad519c643dec11a3a`, PR #5 draft pending acceptance.
- Exact source diff: only `package.json` scoped override, `package-lock.json` delete stale nested PostCSS entry (one package), existing `.gitea/workflows/datastack-ci.yml` security-audit threshold. Updated notes six files only; no runtime/SDK/UI/test/Dockerfile/Compose source changes.
- [Gitea CI #85](https://dev.ivib.cc/vibtools/DataStack/actions/runs/85) results as of checkpoint: node-regression **SUCCESS** `CI_SUITE_SUMMARY total=24 failed=0` plus real Linux `npm ci`; python-regression **SUCCESS** `Ran 14 tests`, `OK`; quality tsc/ESLint **SUCCESS**; production Next standalone **SUCCESS**; Docker/BuildKit 3 images **SUCCESS** `DATASTACK_THREE_IMAGES_BUILD_PASS`; security-audit **SUCCESS** runtime high 0/critical 0, `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO`, full moderate 24/runtime moderate 22. Windows-compatibility **QUEUED** at checkpoint: no terminal proof; entire run must not be mislabelled success.
- **P04-S01C PENDING** terminal Windows acceptance, main squash commit, GitHub/GitLab parity. Production NO-GO; 22 moderate runtime package entries and live readiness remain unresolved.
## P04-S01C FINAL CI acceptance (7/7) — 2026-11-22
Gitea CI #85 https://dev.ivib.cc/vibtools/DataStack/actions/runs/85 COMPLETED/SUCCESS; tested implementation SHA c759be990c032c2474d2d0bad519c643dec11a3a; later changes to candidate are documentation only. All seven jobs terminal SUCCESS: node-regression (Linux committed-lock npm ci and exactly 24/24 TypeScript suites); python-regression (Ran 14 tests, OK); quality (strict tsc and ESLint); production-build (Next 15.5.29 standalone, 22/22 pages); docker-image-build (three isolated BuildKit ingestion/web/proxy images); windows-compatibility (Windows committed-lock npm ci and Node SDK build, job #182, terminal SUCCESS); security-audit (runtime moderate=22 high=0 critical=0; full moderate=24 high=0 critical=0; DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO). Single Gitea acceptance run used, no additional broad reruns.
Scope: exactly nine approved paths: package.json scoped next->postcss 8.5.33 override; package-lock.json removes only old nested postcss@8.4.31 entry; existing .gitea/workflows/datastack-ci.yml security-audit job high/critical rejection; notes/001–006 evidence. Other package metadata, versions, integrity and resolved URLs match baseline. Next.js 15.5.29 and Cloudcmd 19.24.0 preserved. Final candidate package-lock SHA256 e6b5b362877627085dc04c9ded03c019971e07ef9f4af1e78057d58961a1c091 (verified byte-for-byte with Gitea). No API, auth, SDK, UI, application runtime, database, Compose, Dockerfile, tests or production data changes, deployment, migration, prune, tag, release.
CI fully accepted; publication not yet asserted in these pre-merge notes. Final phase CLOSED requires one squash merge of PR #5 into Gitea main and verified identical tracked content across Gitea/GitHub/GitLab; record actual new SHAs in terminal report. Production release remains NO-GO pending P04-S02 moderate vulnerability triage and separately authorized live readiness verification.
## P04-S02 scoped candidate — 2026-11-22
Baseline Gitea main `2d467ff1494858164a37814f7dbc2b321f510291`, tree `648c76ca496d2c9bf22a16f558c9ba347a45d4f7`. Cloudcmd 19.24.0 -> deepword 11.0.1 -> monaco-editor 0.55.1 pins DOMPurify 3.2.7, vulnerable according to npm advisory inventory. Disposable Windows npm 10.9.8 `npm view dompurify@3.4.18` verified, scoped npm override `overrides.cloudcmd.dompurify=3.4.18` resolves `dompurify@3.4.18` via `npm ls --package-lock-only dompurify --all --depth=5` (exit 0). `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline` exit 0. npm graph initially changed 67 unrelated metadata entries; deterministically restored exact baseline values after validating unrelated versions, URLs, integrity and dependencies. Final lock has **one changed package entry**, DOMPurify; same-major replacement plus verified registry `resolved` and `integrity`, no other packages added/deleted/changed. Candidate runtime audit affected-package counts **moderate 20/high 0/critical 0** versus baseline **22/0/0**; full candidate **moderate 22/high 0/critical 0** versus baseline **24/0/0**. DOMPurify and Monaco Editor dropped from vulnerable-package inventory. Other moderate packages (got, glob, minimatch, nested file-type, Cloudcmd transitive chain) remain; they require major package changes under current advisory constraints, prohibited by Scope Lock. This is a **pre-acceptance candidate**: do not claim application compatibility, CI PASS, publication, or phase CLOSED before full focused Gitea CI and exact three-remote tree parity.
Phase ledger: P04-S02 candidate patch under test, not yet accepted. One focused CI to verify Linux/Windows npm ci, Cloudcmd File Manager existing tests, Node 24/24, Python 14/14, TS/lint, Next standalone and Docker 3 images.
## P04-S02 final CI acceptance — 2026-11-22
Gitea CI #92 (https://dev.ivib.cc/vibtools/DataStack/actions/runs/92) **COMPLETED / SUCCESS**, all 7/7 jobs terminal success on source commit `a88bc5f4498ea1b11acc0acfeb66b5aa9d6531be`: node-regression real Linux npm ci + 24/24 tests including `tests/files-manager.test.ts` exit 0; Python SDK 14/14; quality strict TypeScript/ESLint; Next.js standalone (22/22 static pages); isolated Docker/BuildKit 3-image verification `DATASTACK_THREE_IMAGES_BUILD_PASS`; Windows committed-lock npm ci + Node SDK compatibility; security-audit `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO` and `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Audit full=22 moderate/high=0/critical=0 (baseline 24), runtime=20 moderate/high=0/critical=0 (baseline 22). These are affected package entries, not distinct advisories. No new CI workflow or extra broad CI dispatched.
Exact approved patch: `package.json` scoped `overrides.cloudcmd.dompurify=3.4.18`, preserving `overrides.next.postcss=8.5.33`; npm-generated `package-lock.json` changes solely `node_modules/dompurify` version 3.2.7 -> 3.4.18 and verified `resolved`/SHA512 integrity. Baseline Cloudcmd 19.24.0, Monaco Editor 0.55.1, Next.js 15.5.29, all unrelated locked package versions, URL, hash, dependency and metadata preserved. Gitea published candidate lock bytes match disposable SHA-256 `57c5ea9df9f28be496f8c3202fbc280b0066bf8c216b8d8a3ca98b6f7b942854`. Exact candidate tree diff 420 entries, 8 approved changed files only (`package.json`, `package-lock.json`, notes/001–006), 0 unauthorized paths. No runtime/UI/auth/API/SDK/tests/CI/Dockerfiles/Compose/db/schema/data modifications.
Phase ledger: P04-S02 fully CI-accepted, publication gate pending. **Residual risk:** 20 moderate runtime affected package entries remain, including Cloudcmd transitive dependencies `got`, `glob`, `minimatch`, nested `file-type` and their parent chains. Current npm audit suggests `cloudcmd@17.0.2` (a major downgrade), explicitly out-of-scope; other transitive fixes need major-version dependency-contract changes. No force/downgrade/unverified override. Security improvement verified but NOT zero vulnerabilities. Production release stays NO-GO; remaining advisory disposition and live readiness need independent scope approval.
**Merge gate:** CI #92 7/7 PASS, approved 8-file patch integrity PASS; main squash merge and Gitea/GitHub/GitLab exact Git content-tree verification required before P04-S02 CLOSED. This note is written before that publication, so final remote commit SHAs shall be appended to the PR closure report after verification. No deployment, migrations, prune, tag, release.
+11 -68
View File
@@ -77,73 +77,16 @@ Security invariants: do not expose secrets in ACK/errors, keep JWT/auth and 401/
Security/API invariants: leave token/JWT/auth/rate limits/quotas/webhooks/MCP/sandbox/UI/SDK unchanged. No production Redis/Postgres/API/CSV mutation. No lockfile/dependency versions or CI workflow changes. CI acceptance remains pending; these are code-level observations, not release certification. Security/API invariants: leave token/JWT/auth/rate limits/quotas/webhooks/MCP/sandbox/UI/SDK unchanged. No production Redis/Postgres/API/CSV mutation. No lockfile/dependency versions or CI workflow changes. CI acceptance remains pending; these are code-level observations, not release certification.
## P02-RUNTIME-REMEDIATION-02 — Updated safety contract evidence
## P02 Final Fault / Recovery Matrix | Scenario | In-scope behavior | Residual |
|---|---|---|
| Port 4000 Redis `RPUSH` failure | `BUFFER_PERSIST_FAILED`, `success:false`; no Redis transport diagnostics disclosed in client ACK | SDK can retry via HTTP; Port 3000 volatile RAM success path remains unpatched |
| Port 4000 without Redis | Await actual CSV flush before positive ACK; disk error yields negative ACK and queue retained | Disk API does not provide fsync-level power-loss guarantees |
| Stale idempotency after restart | Never treat a claim alone as persisted; only accept a duplicate after in-memory acknowledged proof, Redis queued ID or read-only CSV partition proof | Bounded scans may cause temporary negative ACK for otherwise durable old events, rather than unsafe success |
| Mixed `batch_sync` with uncertain claim | Overall negative `BUFFER_PERSIST_FAILED`; newer items may already be persisted, repeat dedupe resolves them | Preserve existing response fields and rate/quota/auth validations |
| Redis `LTRIM` fails after CSV write | On retry, scan CSV/CSV.gz by client/date/event ID and skip already written event; trim Redis only after safe check | Cross-process exact-once not proven; I/O cost needs throughput validation |
| Port 3000 HTTP `pushAsync` / `pushBatchAsync` | **UNCHANGED** (excluded from current approval); Redis error can return HTTP success with data only in volatile RAM | **OPEN R-01 blocker**, requires separate approval; `core/Security/security.ts` precommit idempotency claims may also need scoped rollback |
| Public `/health` | Unchanged 200 liveness, `redisConnected` diagnostic | No dependency readiness probe under P02-R01/R02 authorization |
| Case | Final behavior target and evidence | Do not silently remap 401/403/429/507 errors; no SDK edits or live fault injection. Tests use mock Redis and temp CSV only. Relevant CI #66/#68/#70 in notes/003.
|---|---|
| Redis RPUSH rejects with Redis configured | `BUFFER_PERSIST_FAILED` negative ACK, no false durable success; candidate CI #66 verified single/batch, final CI #69 for marker TTL |
| Redis idempotency claim survives failed queue write | Positive-time separate failed marker survives cleanup after +61000ms (below five-minute TTL); two retries remain negative rather than `DUPLICATE_DROPPED` success. Final disposable fixture `tests/phase02-sync.test.ts` |
| Mixed `batch_sync` contains stale claim and a new durable record | Overall response `success:false` with `BUFFER_PERSIST_FAILED`, even if some new records committed; preserve counts and avoid success cover-up |
| Redis deliberately disabled | Historical RAM mode unchanged; distinguish this existing best-effort contract from configured Redis durability |
| CSV append success but Redis LTRIM failure | Replay scans written event IDs and does not duplicate physical row before trim; direct Port4000 and web-buffer regression |
| Rotated / gzip CSV, quoted multiline payload | Streaming CSV parser, exact client/date/eventId match, no duplicate append or data-format change |
| Matching CSV partial/corrupt | Fail closed (no Redis trim); disposable repair allows retry; do not touch real user CSV |
| Public /health & 401/403/429/503/507 | Preserve existing behavior; no new endpoint or error contract beyond already approved BUFFER_PERSIST_FAILED |
**Limitations:** no globally atomic Redis+CSV transaction, stale Redis claim after process crash may require TTL/retry policy, and per-partition scan has data-size-proportional I/O even though memory is streamed. No live production reliability or throughput certification. CI #66 SUCCESS; final CI #69 pending.
### Additional P02 final-acceptance negative ACK case
If a `batch_sync` contains only (a) an uncommitted Redis-claim duplicate and (b) an ordinary already-durable duplicate, the old `processed===0 && duplicates===0 && rejected>0` test did not trigger; success ACK incorrectly masked volatile claim. Narrow correction uses `if (volatileDuplicate)` before the generic aggregate success path, independent of `processed` or `duplicates`. Specific test in `tests/phase02-sync.test.ts` injects Redis RPUSH failure and fake stale claim then includes the existing durable duplicate. All pending-claim ACKs must remain `BUFFER_PERSIST_FAILED`, even if some other events already committed. Terminal CI #72 pending; no global 5-minute clock advancement inside sockets.
### P02 terminal CI #72 error handling acceptance
Gitea #72 **SUCCESS** at verified final code SHA `acca8be01a31f8d3972564cbaec8fd3117a0b551`; all 24 TS suites exit zero including stale claim, mixed batch (both with new durable event and old durable duplicate), Redis-free compatibility, gzip/rotated CSV and corrupt matching CSV fail-closed. `BUFFER_PERSIST_FAILED` is negative-ACK response on configured Redis write failure, rather than success for RAM fallback. Redis-free RAM mode old semantics unchanged. Existing 401/403/429/503/507 tests unaffected and pass within 24 suites. Public /health liveness remains HTTP 200 with `redisConnected:false` during mocked outage; separate readiness endpoint not approved. Remaining risk: no atomic Redis+CSV commit, no global exactly-once guarantee, CSV partition scan disk I/O grows with volume. No live traffic test.
## P03 runner/security defect register — 2026-11-22
- **CI-06:** `python:3.12-slim-bookworm` lacked Node for `actions/checkout@v4` (127). Approved workflow-only correction: `node:22-bookworm-slim` plus Debian Python 3.11 venv and prerequisites; narrow CI #78 **14/14 PASS**.
- **CI-07:** Historical classic Docker `COPY node_modules/@uiw` layer-export failure was not reproduced using disposable BuildKit builder; #77 Compose and 3/3 builds plus image inspect **PASS**; no Dockerfile modifications or shared prune.
- **CI-08:** **CI-08 security disposition:** `npm audit --json` full=26 affected packages (25 moderate, 1 high); runtime=24 (23 moderate, 1 high), **not counts of distinct advisories**. High-severity advisories [GHSA-6g55-p6wh-862q](https://github.com/advisories/GHSA-6g55-p6wh-862q), [GHSA-r28c-9q8g-f849](https://github.com/advisories/GHSA-r28c-9q8g-f849) affect Next 15.5.29's nested `postcss@8.4.31` despite patched root `postcss@8.5.33`; npm recommends major `next@16.4.6`. Direct `cloudcmd@19.24.0` carries moderate transitive issues (`dompurify`, `file-type`, `glob`, `minimatch`, `got`); npm recommends `cloudcmd@17.0.2` downgrade, not a verified safe pin. No speculative package pin, major upgrade or force fix; a separately tested compatible remediation is needed before release.
- **Runtime API errors:** P02's 401/403/429/503/507, `BUFFER_PERSIST_FAILED` negative ACK, Redis/CSV replay behavior and public `/health` shape unchanged; no live production fault claims.
**Scope / release decision:** only `.gitea/workflows/datastack-ci.yml`, `.gitea/workflows/datastack-docker-ci.yml`, and `notes/001–006` changed. No `package.json`/lock, Dockerfile, application UI/UX, runtime, auth, SDK, database, Compose, production data, secret, shared Docker prune, deployment, migration, tag or release. P03 technical CI and advisory-triage acceptance verified by #77 non-Python jobs plus #78 Python; do **not** mislabel #77 overall as successful. Final phase publication requires one main commit and Gitea/GitHub/GitLab exact tracked tree parity. **Production release NO-GO** while runtime high advisory is unresolved; live production health, throughput and deployment remain unverified.
## P04-S01 blocked security reconciliation report
P04-S01 SEC-01 BLOCKED — 2026-11-22. Approved scoped `overrides.next.postcss=8.5.33` trialed in disposable Windows npm 10.9.8 workspace only. `npm install --package-lock-only` EXIT 0, but `npm ls --package-lock-only postcss --all` EXIT 1 (`ELSPROBLEMS`: Next nested `postcss@8.4.31` invalid; expected 8.5.33); npm dedupe also kept invalid dependency. Read-only `npm audit --omit=dev --json` EXIT 1, runtime high 1/moderate 23/critical 0. Lockfile regeneration caused unrelated churn (91 package entries added, 106 removed, 69 changed, including two lightningcss optional 1.33.0 -> 1.30.1). Fresh minimal Next 15.5.29 fixture with identical override resolves PostCSS 8.5.33, isolating failure to DataStack committed lock graph. `npm ci --dry-run` exit 0 does not prove real clean install. Exact stop rule applied: no dependency patch, no CI acceptance, no main commit, no GitHub/GitLab sync, no release/deploy; production NO-GO. Need separate explicit narrowly scoped lock reconciliation approval.
Do not treat successful npm install/dedupe/dry-run as security or compatibility validation. High PostCSS remains reachable through lock graph, no runtime handlers or error codes touched.
## P04-S01B exact failed acceptance condition — 2026-11-22
PostCSS resolution fixed within disposable test copy, with audit high/critical 0, but lockfile metadata integrity requirement **FAIL**: npm wrote 67 unrelated package-entry metadata differences. No API error/ACK behavior touched. Existing peer dependency warning for `@putout/eslint@6.1.1` versus ESLint 9 is non-blocking in the isolated lock generation and not permission to upgrade it. Stop before acceptance CI or source edit; production/main lock and security-audit workflow remain unchanged.
## P04-S01C security gate and error contract — 2026-11-22
SEC-01 candidate remediation verified: Next 15.5.29/Cloudcmd 19.24.0 unchanged, PostCSS 8.5.33 deduped, nested vulnerable PostCSS removed without unrelated dependency integrity/metadata drift. Focused CI #85 read-only audit: full affected packages moderate=24/high=0/critical=0; runtime moderate=22/high=0/critical=0, and `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO` succeeded. New gate fails on incomplete severity counts and any runtime high/critical. Moderate Cloudcmd transitive issues intentionally remain P04-S02. No JWT/auth, HTTP/socket error codes, Redis durability, CSV replay, SDK/API/UI or production health changes; Windows CI still queued and publication pending.
## P04-S01C FINAL CI acceptance (7/7) — 2026-11-22
Gitea CI #85 https://dev.ivib.cc/vibtools/DataStack/actions/runs/85 COMPLETED/SUCCESS; tested implementation SHA c759be990c032c2474d2d0bad519c643dec11a3a; later changes to candidate are documentation only. All seven jobs terminal SUCCESS: node-regression (Linux committed-lock npm ci and exactly 24/24 TypeScript suites); python-regression (Ran 14 tests, OK); quality (strict tsc and ESLint); production-build (Next 15.5.29 standalone, 22/22 pages); docker-image-build (three isolated BuildKit ingestion/web/proxy images); windows-compatibility (Windows committed-lock npm ci and Node SDK build, job #182, terminal SUCCESS); security-audit (runtime moderate=22 high=0 critical=0; full moderate=24 high=0 critical=0; DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO). Single Gitea acceptance run used, no additional broad reruns.
Security improvement: remove Next nested vulnerable PostCSS and enforce runtime high/critical fail gate. Existing 22 moderate runtime affected-package entries remain, not a security-clean claim for all severities. No change to runtime error codes/auth/API contracts. CI fully accepted; publication not yet asserted in these pre-merge notes. Final phase CLOSED requires one squash merge of PR #5 into Gitea main and verified identical tracked content across Gitea/GitHub/GitLab; record actual new SHAs in terminal report. Production release remains NO-GO pending P04-S02 moderate vulnerability triage and separately authorized live readiness verification.
## P04-S02 scoped candidate — 2026-11-22
Baseline Gitea main `2d467ff1494858164a37814f7dbc2b321f510291`, tree `648c76ca496d2c9bf22a16f558c9ba347a45d4f7`. Cloudcmd 19.24.0 -> deepword 11.0.1 -> monaco-editor 0.55.1 pins DOMPurify 3.2.7, vulnerable according to npm advisory inventory. Disposable Windows npm 10.9.8 `npm view dompurify@3.4.18` verified, scoped npm override `overrides.cloudcmd.dompurify=3.4.18` resolves `dompurify@3.4.18` via `npm ls --package-lock-only dompurify --all --depth=5` (exit 0). `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline` exit 0. npm graph initially changed 67 unrelated metadata entries; deterministically restored exact baseline values after validating unrelated versions, URLs, integrity and dependencies. Final lock has **one changed package entry**, DOMPurify; same-major replacement plus verified registry `resolved` and `integrity`, no other packages added/deleted/changed. Candidate runtime audit affected-package counts **moderate 20/high 0/critical 0** versus baseline **22/0/0**; full candidate **moderate 22/high 0/critical 0** versus baseline **24/0/0**. DOMPurify and Monaco Editor dropped from vulnerable-package inventory. Other moderate packages (got, glob, minimatch, nested file-type, Cloudcmd transitive chain) remain; they require major package changes under current advisory constraints, prohibited by Scope Lock. This is a **pre-acceptance candidate**: do not claim application compatibility, CI PASS, publication, or phase CLOSED before full focused Gitea CI and exact three-remote tree parity.
Error contract: no user-facing error codes or handler changes. Security audit improvement from 22 to 20 moderate runtime affected packages, high/critical stays zero in disposable audit. No claim of eliminating all moderate advisories.
## P04-S02 final CI acceptance — 2026-11-22
Gitea CI #92 (https://dev.ivib.cc/vibtools/DataStack/actions/runs/92) **COMPLETED / SUCCESS**, all 7/7 jobs terminal success on source commit `a88bc5f4498ea1b11acc0acfeb66b5aa9d6531be`: node-regression real Linux npm ci + 24/24 tests including `tests/files-manager.test.ts` exit 0; Python SDK 14/14; quality strict TypeScript/ESLint; Next.js standalone (22/22 static pages); isolated Docker/BuildKit 3-image verification `DATASTACK_THREE_IMAGES_BUILD_PASS`; Windows committed-lock npm ci + Node SDK compatibility; security-audit `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO` and `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Audit full=22 moderate/high=0/critical=0 (baseline 24), runtime=20 moderate/high=0/critical=0 (baseline 22). These are affected package entries, not distinct advisories. No new CI workflow or extra broad CI dispatched.
Exact approved patch: `package.json` scoped `overrides.cloudcmd.dompurify=3.4.18`, preserving `overrides.next.postcss=8.5.33`; npm-generated `package-lock.json` changes solely `node_modules/dompurify` version 3.2.7 -> 3.4.18 and verified `resolved`/SHA512 integrity. Baseline Cloudcmd 19.24.0, Monaco Editor 0.55.1, Next.js 15.5.29, all unrelated locked package versions, URL, hash, dependency and metadata preserved. Gitea published candidate lock bytes match disposable SHA-256 `57c5ea9df9f28be496f8c3202fbc280b0066bf8c216b8d8a3ca98b6f7b942854`. Exact candidate tree diff 420 entries, 8 approved changed files only (`package.json`, `package-lock.json`, notes/001–006), 0 unauthorized paths. No runtime/UI/auth/API/SDK/tests/CI/Dockerfiles/Compose/db/schema/data modifications.
Error/security semantics: no API error handler, permission, middleware, auth or HTTP/WebSocket changes. Upgrade only same-major DOMPurify. **Residual risk:** 20 moderate runtime affected package entries remain, including Cloudcmd transitive dependencies `got`, `glob`, `minimatch`, nested `file-type` and their parent chains. Current npm audit suggests `cloudcmd@17.0.2` (a major downgrade), explicitly out-of-scope; other transitive fixes need major-version dependency-contract changes. No force/downgrade/unverified override. Security improvement verified but NOT zero vulnerabilities. Production release stays NO-GO; remaining advisory disposition and live readiness need independent scope approval.
**Merge gate:** CI #92 7/7 PASS, approved 8-file patch integrity PASS; main squash merge and Gitea/GitHub/GitLab exact Git content-tree verification required before P04-S02 CLOSED. This note is written before that publication, so final remote commit SHAs shall be appended to the PR closure report after verification. No deployment, migrations, prune, tag, release.
+6 -60
View File
@@ -61,64 +61,10 @@ Do not mark "actual working" simply because a patch compiled. Distinguish source
- **No changes:** application UI/UX, auth, SDK, database schema, Docker, environment, public R-03 health, other features and production runtime. No deployment or live verification. - **No changes:** application UI/UX, auth, SDK, database schema, Docker, environment, public R-03 health, other features and production runtime. No deployment or live verification.
- **Status:** P02 code candidate locally committed, acceptance CI blocked by Gitea unavailability; **NOT CLOSED**, not production-ready. P01 remains only completed phase. - **Status:** P02 code candidate locally committed, acceptance CI blocked by Gitea unavailability; **NOT CLOSED**, not production-ready. P01 remains only completed phase.
## P02 Remediation Hardening Register — 2026-11-22
## P02 Final Feature/Compatibility Register — 2026-11-22 - **Changed in source, not deployed:** Port 4000 Socket.IO error ACK behavior on Redis write failure; no-Redis mode now waits for CSV flush before success; duplicate ACK checks actual Redis/CSV durability; mixed batch uncertainty gives negative ACK. EventId retry handling is made fail-closed with in-scope handler logic.
- **Changed in source, not deployed:** Port 4000 and web-buffer CSV replay after Redis LTRIM failure now checks persisted CSV/CSV.gz event IDs. Existing file rotation/compression and disk errors need focused tests; scanner I/O impacts high-volume performance and does not certify global exactly-once semantics.
Existing UI/UX, admin auth, SDK public signatures, health route, database schema, Docker and deployment: **UNCHANGED**. Only existing Socket.IO single/batch persistence failure ACK/idempotency cleanup, Redis->CSV replay duplicate filtering and disposable tests changed during P02. No new production feature, public endpoint, release or migration. CI #66 terminal SUCCESS (24/24 TS suites, strict types, ESLint, Windows Node SDK, Next production build); final CI #69 pending. Client-facing `BUFFER_PERSIST_FAILED` error is used only for previously incorrectly acknowledged Redis-backed failure; intentionally Redis-free RAM mode is preserved. CSV write content/format unchanged, gzip and rotation replay acceptance covered in final test fixture. All live status values are **NOT VERIFIED / NO DEPLOYMENT**. No guarantee of global exactly-once delivery or hard bound on disk scans; future throughput benchmark/design approval if required. P03 still gated on final P02 close. - **Unchanged:** UI/UX/pages, public R-03 health shape/status, auth/JWT/SDK implementation, database/Prisma, Docker/Compose, environment, unrelated features, production data, live server and deployment.
- **Still NOT working as end-to-end guarantee:** HTTP `POST /api/events` currently may send success after Redis RPUSH fails while only RAM contains the event. This is an additional source-observed release blocker, not something fixed by WS error ACK. **P02 NOT CLOSED; P03 NOT STARTED.**
- **Evidence:** Gitea #66 initial remediation PASS; #68 exposed same-ID retry bug fixed in current branch; #70 final focused run awaiting terminal confirmation. The original PC main workspace's 16 staged P01 paths remain untouched.
### P02 final acceptance correction checkpoint
Gitea #69 node-regression observed one **fixture timeout**, not a proven new production feature defect, caused by altering global `Date.now`. A separate source-proven mixed batch all-duplicate ACK bug was corrected only in `server/socket/handlers.js`, test case added in existing allowed TS file. Revised `acca8be01a31f8d3972564cbaec8fd3117a0b551` pending focused Gitea CI #72. Existing UI/SDK/auth/public health remain unchanged. This is not evidence that production has been deployed or live-tested.
### P02 verified feature-state reconciliation — Gitea #72 SUCCESS
- **P02 R-01: FIXED in source and CI-PASS, conditional scope.** Configured Redis queue RPUSH rejects => `BUFFER_PERSIST_FAILED` negative ACK for single/mixed batches, stale Redis idempotency claim cannot be reported as durable success; clock arithmetic and actual negative ACK retries verified. Intentionally Redis-free RAM mode historical best-effort ACK **preserved**, not a lossless guarantee.
- **P02 R-02: FIXED for verified crash/replay windows**, not global transactionality. Port4000 and Port3000 CSV replay suppresses duplicate same-client/day/eventId across plain, rotated and gz; failed reads prevent trimming; dedicated regression tests PASS. Streaming parser bounded memory, disk read throughput/latency not yet load-tested.
- **P02 R-03: UNCHANGED**: public /health remains backwards-compatible liveness, not a dependency-readiness endpoint. Production live health **NOT VERIFIED**.
- All other runtime features/UI/auth/API/SDK/database/Compose and package dependencies unchanged. **P02 CI ACCEPTED at #72**; code SHA `acca8be01a31f8d3972564cbaec8fd3117a0b551`. Phase closed only after verified 3-way content sync. No P03 implementation or production deployment.
## P03 feature invariance — 2026-11-22
P03 modifies workflow verification and Markdown evidence only, not a product feature. No UI, API, auth, SDK, DB, CSV, Redis, Dockerfile, Compose or deployment change. The P02 regression contract remains 24/24 PASS; corrected Python SDK passes 14/14; all three Docker images build without running them. Read-only npm advisory triage is complete but **not security-clean**.
**CI-08 security disposition:** `npm audit --json` full=26 affected packages (25 moderate, 1 high); runtime=24 (23 moderate, 1 high), **not counts of distinct advisories**. High-severity advisories [GHSA-6g55-p6wh-862q](https://github.com/advisories/GHSA-6g55-p6wh-862q), [GHSA-r28c-9q8g-f849](https://github.com/advisories/GHSA-r28c-9q8g-f849) affect Next 15.5.29's nested `postcss@8.4.31` despite patched root `postcss@8.5.33`; npm recommends major `next@16.4.6`. Direct `cloudcmd@19.24.0` carries moderate transitive issues (`dompurify`, `file-type`, `glob`, `minimatch`, `got`); npm recommends `cloudcmd@17.0.2` downgrade, not a verified safe pin. No speculative package pin, major upgrade or force fix; a separately tested compatible remediation is needed before release.
**Scope / release decision:** only `.gitea/workflows/datastack-ci.yml`, `.gitea/workflows/datastack-docker-ci.yml`, and `notes/001–006` changed. No `package.json`/lock, Dockerfile, application UI/UX, runtime, auth, SDK, database, Compose, production data, secret, shared Docker prune, deployment, migration, tag or release. P03 technical CI and advisory-triage acceptance verified by #77 non-Python jobs plus #78 Python; do **not** mislabel #77 overall as successful. Final phase publication requires one main commit and Gitea/GitHub/GitLab exact tracked tree parity. **Production release NO-GO** while runtime high advisory is unresolved; live production health, throughput and deployment remain unverified.
## P04-S01 blocked security reconciliation report
P04-S01 SEC-01 BLOCKED — 2026-11-22. Approved scoped `overrides.next.postcss=8.5.33` trialed in disposable Windows npm 10.9.8 workspace only. `npm install --package-lock-only` EXIT 0, but `npm ls --package-lock-only postcss --all` EXIT 1 (`ELSPROBLEMS`: Next nested `postcss@8.4.31` invalid; expected 8.5.33); npm dedupe also kept invalid dependency. Read-only `npm audit --omit=dev --json` EXIT 1, runtime high 1/moderate 23/critical 0. Lockfile regeneration caused unrelated churn (91 package entries added, 106 removed, 69 changed, including two lightningcss optional 1.33.0 -> 1.30.1). Fresh minimal Next 15.5.29 fixture with identical override resolves PostCSS 8.5.33, isolating failure to DataStack committed lock graph. `npm ci --dry-run` exit 0 does not prove real clean install. Exact stop rule applied: no dependency patch, no CI acceptance, no main commit, no GitHub/GitLab sync, no release/deploy; production NO-GO. Need separate explicit narrowly scoped lock reconciliation approval.
No change to UI, API, SDK, auth, file manager, CSV, Redis, storage, feature set, Docker, deployment or production data. P03 baseline regressions preserved without claiming retest.
## P04-S01B no-feature-change / gate state — 2026-11-22
Candidate scope proof: PostCSS root 8.5.33 can satisfy Next 15.5.29 when stale nested 8.4.31 removed in a disposable lock, with moderate=22/high=0/critical=0 read-only runtime audit. Not published: regenerated metadata edits exceeded approved lock scope. Existing UI/UX, auth, API, SDK, CSV, Redis, Cloudcmd, Docker and production service behavior remain untouched; **do not claim regression acceptance or security remediation of published code**. Next approval must address exact npm-generated metadata drift and integrity preservation, not feature work.
## P04-S01C feature preservation and verification — 2026-11-22
No feature, UI, backend runtime, public API, auth, database/schema, SDK, CSV storage, Dockerfile or Compose source change. Package manifest scoped override and lockfile one-entry deletion only; Next 15.5.29 and Cloudcmd 19.24.0 preserved. Gitea CI #85 node 24/24, Python 14/14, strict types/lint, Next standalone and 3 Docker images completed SUCCESS. Security runtime audit high/critical 0; moderate advisories still pending separate triage. Windows compatibility QUEUED at checkpoint; no production release, deployment, migration or tag and no accepted main/mirror publication yet.
## P04-S01C FINAL CI acceptance (7/7) — 2026-11-22
Gitea CI #85 https://dev.ivib.cc/vibtools/DataStack/actions/runs/85 COMPLETED/SUCCESS; tested implementation SHA c759be990c032c2474d2d0bad519c643dec11a3a; later changes to candidate are documentation only. All seven jobs terminal SUCCESS: node-regression (Linux committed-lock npm ci and exactly 24/24 TypeScript suites); python-regression (Ran 14 tests, OK); quality (strict tsc and ESLint); production-build (Next 15.5.29 standalone, 22/22 pages); docker-image-build (three isolated BuildKit ingestion/web/proxy images); windows-compatibility (Windows committed-lock npm ci and Node SDK build, job #182, terminal SUCCESS); security-audit (runtime moderate=22 high=0 critical=0; full moderate=24 high=0 critical=0; DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO). Single Gitea acceptance run used, no additional broad reruns.
No product feature, UI, runtime, API, auth, SDK, data format, Docker or deployment change. Only resolved PostCSS dependency and CI security gate. CI fully accepted; publication not yet asserted in these pre-merge notes. Final phase CLOSED requires one squash merge of PR #5 into Gitea main and verified identical tracked content across Gitea/GitHub/GitLab; record actual new SHAs in terminal report. Production release remains NO-GO pending P04-S02 moderate vulnerability triage and separately authorized live readiness verification.
## P04-S02 scoped candidate — 2026-11-22
Baseline Gitea main `2d467ff1494858164a37814f7dbc2b321f510291`, tree `648c76ca496d2c9bf22a16f558c9ba347a45d4f7`. Cloudcmd 19.24.0 -> deepword 11.0.1 -> monaco-editor 0.55.1 pins DOMPurify 3.2.7, vulnerable according to npm advisory inventory. Disposable Windows npm 10.9.8 `npm view dompurify@3.4.18` verified, scoped npm override `overrides.cloudcmd.dompurify=3.4.18` resolves `dompurify@3.4.18` via `npm ls --package-lock-only dompurify --all --depth=5` (exit 0). `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline` exit 0. npm graph initially changed 67 unrelated metadata entries; deterministically restored exact baseline values after validating unrelated versions, URLs, integrity and dependencies. Final lock has **one changed package entry**, DOMPurify; same-major replacement plus verified registry `resolved` and `integrity`, no other packages added/deleted/changed. Candidate runtime audit affected-package counts **moderate 20/high 0/critical 0** versus baseline **22/0/0**; full candidate **moderate 22/high 0/critical 0** versus baseline **24/0/0**. DOMPurify and Monaco Editor dropped from vulnerable-package inventory. Other moderate packages (got, glob, minimatch, nested file-type, Cloudcmd transitive chain) remain; they require major package changes under current advisory constraints, prohibited by Scope Lock. This is a **pre-acceptance candidate**: do not claim application compatibility, CI PASS, publication, or phase CLOSED before full focused Gitea CI and exact three-remote tree parity.
Feature preservation: Cloudcmd major 19.24.0, Monaco Editor 0.55.1, Next 15.5.29, PostCSS override 8.5.33, runtime APIs/UI/auth/SDK/Redis/CSV/storage and Dockerfiles unchanged. Existing File Manager behavior must be verified by read-only tests.
## P04-S02 final CI acceptance — 2026-11-22
Gitea CI #92 (https://dev.ivib.cc/vibtools/DataStack/actions/runs/92) **COMPLETED / SUCCESS**, all 7/7 jobs terminal success on source commit `a88bc5f4498ea1b11acc0acfeb66b5aa9d6531be`: node-regression real Linux npm ci + 24/24 tests including `tests/files-manager.test.ts` exit 0; Python SDK 14/14; quality strict TypeScript/ESLint; Next.js standalone (22/22 static pages); isolated Docker/BuildKit 3-image verification `DATASTACK_THREE_IMAGES_BUILD_PASS`; Windows committed-lock npm ci + Node SDK compatibility; security-audit `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO` and `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Audit full=22 moderate/high=0/critical=0 (baseline 24), runtime=20 moderate/high=0/critical=0 (baseline 22). These are affected package entries, not distinct advisories. No new CI workflow or extra broad CI dispatched.
Exact approved patch: `package.json` scoped `overrides.cloudcmd.dompurify=3.4.18`, preserving `overrides.next.postcss=8.5.33`; npm-generated `package-lock.json` changes solely `node_modules/dompurify` version 3.2.7 -> 3.4.18 and verified `resolved`/SHA512 integrity. Baseline Cloudcmd 19.24.0, Monaco Editor 0.55.1, Next.js 15.5.29, all unrelated locked package versions, URL, hash, dependency and metadata preserved. Gitea published candidate lock bytes match disposable SHA-256 `57c5ea9df9f28be496f8c3202fbc280b0066bf8c216b8d8a3ca98b6f7b942854`. Exact candidate tree diff 420 entries, 8 approved changed files only (`package.json`, `package-lock.json`, notes/001–006), 0 unauthorized paths. No runtime/UI/auth/API/SDK/tests/CI/Dockerfiles/Compose/db/schema/data modifications.
Feature preservation: File Manager test and Monaco-related Next production build PASS; no UI features removed, no editor behavior modified in source. **Residual risk:** 20 moderate runtime affected package entries remain, including Cloudcmd transitive dependencies `got`, `glob`, `minimatch`, nested `file-type` and their parent chains. Current npm audit suggests `cloudcmd@17.0.2` (a major downgrade), explicitly out-of-scope; other transitive fixes need major-version dependency-contract changes. No force/downgrade/unverified override. Security improvement verified but NOT zero vulnerabilities. Production release stays NO-GO; remaining advisory disposition and live readiness need independent scope approval.
**Merge gate:** CI #92 7/7 PASS, approved 8-file patch integrity PASS; main squash merge and Gitea/GitHub/GitLab exact Git content-tree verification required before P04-S02 CLOSED. This note is written before that publication, so final remote commit SHAs shall be appended to the PR closure report after verification. No deployment, migrations, prune, tag, release.
+6 -80
View File
@@ -260,88 +260,14 @@ Base: Gitea/GitHub published P02 diagnostic tree `fd57a3518d14bcb8367ce9ab9c3ec3
- Local Windows npm: `npm ci` failed `ETARGET yaml@2.9.1` and cleanup `EPERM` in the isolated worktree, **not** a TypeScript/test result. Dependency changes are out of P02 scope. - Local Windows npm: `npm ci` failed `ETARGET yaml@2.9.1` and cleanup `EPERM` in the isolated worktree, **not** a TypeScript/test result. Dependency changes are out of P02 scope.
- Therefore neither TS typecheck nor regression-suite remediation acceptance is established; keep P02 BLOCKED and do not advance P03 or alter any protected runtime/source beyond this approval. - Therefore neither TS typecheck nor regression-suite remediation acceptance is established; keep P02 BLOCKED and do not advance P03 or alter any protected runtime/source beyond this approval.
## F. P02-RUNTIME-REMEDIATION-02 — Exact edge-case proof and scope audit (2026-11-22)
## F. P02 Final Acceptance Patch Index (current source/CI proof) **Baseline** `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`; isolated branch `p02-runtime-remediation-20261122`; patch revision `5c880885378ae0f4fa8dd3a145f9c7055dada635`. Scope: `server/index.js` (RPUSH fallback/onBufferPush/flush), `server/socket/handlers.js` (single/batch ACK + durability verification helper), `core/data/buffer.ts` (flush), `tests/{phase02-sync,phase03-storage-buffer,r4-phase02-ingestion-audit-streaming}.test.ts` and `notes/001–006` only. No other source file/CI workflow/dependency/SDK/UI change.
- Published diagnosis Gitea/GitHub main: `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`; previous GitLab main same tracked tree. **Runtime contract and tests:** configured Redis RPUSH rejection -> negative `BUFFER_PERSIST_FAILED`; Redis-free Socket.IO ACK -> synchronous local CSV flush, with negative ACK on disk error; fail closed on previous Redis idempotency claim unless persisted to Redis buffer or matching physical CSV; recover same eventId after Redis reconnection; mixed batch uncertain duplicate remains negative until persistence proven. R-02 test injects failure after Port 4000 CSV append and before Redis trim, verifies subsequent retry does not add another row, including moved/rotated `.csv.gz`. R-03 `/health` remains unchanged.
- Runtime candidate: `76cd972bdeea9ef07b4b38e9d8b5b5ed3312ca5b`; Gitea [CI #66](https://dev.ivib.cc/vibtools/DataStack/actions/runs/66) **terminal SUCCESS** across node 24/24, strict TS, lint, Next standalone and Windows Node SDK.
- Exact supplemental commit: `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532` (3 changed files: `server/socket/handlers.js`, `tests/phase02-sync.test.ts`, `tests/phase03-storage-buffer.test.ts`). Compared with published diagnosis, final candidate also includes runtime `server/index.js`, `core/data/buffer.ts`, and previously modified tests, all authorized, plus `notes/001–006`.
- **R-01 arithmetic reproduction** before patch: `const marker=-t; const now=t+61000; now-marker > 300000` is true, while `now-t > 300000` is false. `server/socket/handlers.js:142-158` `cleanupMaps` expires old negative marker; `validateSingleEventAsync:293-317` sees retained Redis idempotency claim; `sync_data` returned duplicate success. Final marker is `\u0000failed-persist:<clientId>:<eventId>` with positive timestamp, corrected age check, cleanup after true TTL, clear marker after durable ACK. Mixed batches reject overall positive ACK while a failed claim remains.
- **R-02:** `server/index.js:843-897,1043-1058`, `core/data/buffer.ts:652-711,855-872`: stream-parsed same-day client CSV/CSV.gz replay suppression, then Redis trim. Focused tests inject direct Port4000 LTRIM crash and compressed part2 quoted/multiline replay; malformed matching partition causes safe no-trim, fixture repair recovers. Memory streamed, partition disk-I/O remains proportional; not globally exactly-once.
- **Compat:** Redis-free RAM ACK and old public /health unchanged; other HTTP/Socket auth, SDK and quota routes unaffected.
- Final [Gitea CI #69](https://dev.ivib.cc/vibtools/DataStack/actions/runs/69) dispatched against exact supplemental commit; until terminal result, phase stays pending and no remote publish.
**Test-only HTTP residual diagnostic:** `tests/phase02-sync.test.ts` adds Redis RPUSH failure against `POST /api/events` on a fake Redis. Expected prior behavior: HTTP 200 `success:true` while `DataBufferEngine.getLocalBufferSnapshot()` has the event and Redis buffered list is empty, then simulated RAM loss. This *diagnostic PASS* would confirm a remaining risk, not a completed fix. Source: `app/api/events/route.ts:744-754`, `core/data/buffer.ts:412-442,481-491`, `core/Security/security.ts:230-375`, Node SDK `sdk/node/datastack/index.ts:643-666,916-942`. Those runtime methods are **READ ONLY under current approval**.
### Final P02 CI #69 failure and exact retry patch **CI ledger:** [#66](https://dev.ivib.cc/vibtools/DataStack/actions/runs/66) initial in-scope implementation overall SUCCESS; [#68](https://dev.ivib.cc/vibtools/DataStack/actions/runs/68) discovered same-ID retry local claim leak, corrected in subsequent branch commit; [#70](https://dev.ivib.cc/vibtools/DataStack/actions/runs/70) revised source/test acceptance queued/running as of this addendum. Do not assert it PASS without terminal proof.
- CI #69: `phase02-sync.test.ts` subtest `stale Redis claim never becomes positive ACK after cleanup window` `ERR_TEST_FAILURE: operation has timed out` after replacing the **global** `Date.now` during Socket.IO ACK. This cannot support a server-failure claim; it was a test harness flaw. Other new replay/compatibility assertions PASS and quality/Windows jobs separately passed.
- Discovered source branch in `server/socket/handlers.js`: `if (volatileDuplicate && processed > 0)` allowed an overall `success:true` response with pending non-durable claim and already-durable duplicate when `processed=0,duplicates>0`. Revised `if (volatileDuplicate)` blocks every mixed response with an uncommitted claim; new test reproduces the exact combination.
- Test no longer overrides Date.now; independent simulated arithmetic proves the original negative-marker expiry mistake; live ACK checks run on real clock. Supplemental commit `acca8be01a31f8d3972564cbaec8fd3117a0b551`; [CI #72](https://dev.ivib.cc/vibtools/DataStack/actions/runs/72) must pass before P02 closure. No modification of package/dependency files, test CI workflow, public API, UI or production.
## G. P02 Final Acceptance — authoritative terminal CI / Scope Lock proof
- Source baseline published diagnosis `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`. Fixed runtime candidate `76cd972bdeea9ef07b4b38e9d8b5b5ed3312ca5b`, previous #66 SUCCESS. Final corrected **source/test** SHA `acca8be01a31f8d3972564cbaec8fd3117a0b551` fully verified in [Gitea #72](https://dev.ivib.cc/vibtools/DataStack/actions/runs/72), **terminal COMPLETED SUCCESS**, Node `CI_SUITE_SUMMARY total=24 failed=0`. CI job conclusions: node-regression SUCCESS, quality SUCCESS, windows-compatibility SUCCESS, production-build SUCCESS; python-regression and docker-image-build SKIPPED by narrow dispatch.
- Exact reproducible Redis idempotency bug: old `-Date.now()` failed timestamp ages incorrectly at +61000ms, while 300000ms Redis claim TTL has not elapsed. New `\u0000failed-persist:${claimKey}` positive timestamp marker and corrected TTL comparison isolate the failed claim. Live Socket.IO retries remain negative ACK; no process-wide clock monkeypatch. Mixed all-duplicate batch also rejects pending claim instead of sending global positive ACK.
- CSV fault evidence: `tests/phase03-storage-buffer.test.ts` after CSV append but before Redis LTRIM, direct Port4000 writer, rotated `.csv.gz`, JSON multiline-quoted payload, corrupt matching partition no-trim/recovery all PASS. Test assertions cannot prove a global exactly-once protocol or live throughput cap. Full-partition scan is streaming bounded-memory, but data-size-linear disk I/O.
- Historical #69 failed one test-only global Date.now-induced Socket.IO timeout, **not** a demonstrated runtime regression; corrected #72 PASS. No secret or credential introduction, no production data access.
- Final audit `git diff --check` PASS and source JS syntax checks PASS; exact tracked source diff from diagnosis = 11 allowlisted files (3 runtime + 2 test + 6 notes); no unrelated UI/auth/SDK/public health, DB schema, package metadata, Docker, CI workflow or live deployment.
- Git release gate: final docs-only commits after tested source SHA add **no source/test differences**. Gitea PR #3 squashed into one `[skip ci]` main commit; GitHub/GitLab tracked Git trees must match Gitea before final CLOSED. If GitLab commit SHA differs because of parent history, compare tree SHA not commit SHA. Store exact remote SHAs/trees in phase close operator report; do not invent them before push.
## H. P03 exact audit and terminal evidence — 2026-11-22
Baseline Gitea/GitHub `48fb277079b380fdbf1e58cd2da40afeed0b03ee`; GitLab baseline same tracked tree `41a169eec0c05ce19bc4acc29d7a8cfe1f8dd90d` (317/317 blobs). Final P03 workflow candidate `549d575653b68c29f97e2f78036646f2e2035719`. [Gitea CI #77](https://dev.ivib.cc/vibtools/DataStack/actions/runs/77): overall **FAILURE** because the initial Python-only checkout lacked Node (exit 127), **not** an SDK regression. Its other six jobs terminal **SUCCESS**: Node `CI_SUITE_SUMMARY total=24 failed=0`, strict TS+ESLint, Next standalone build, Windows SDK, Compose + isolated BuildKit ingestion/web/proxy image build `DATASTACK_THREE_IMAGES_BUILD_PASS`, read-only advisory inventory `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Narrow [Python-only CI #78](https://dev.ivib.cc/vibtools/DataStack/actions/runs/78) terminal **SUCCESS**: Node-capable Bookworm container, Python 3.11 isolated venv, `Ran 14 tests in 3.959s`, `OK`, `DATASTACK_PYTHON_14_OF_14_PASS`. The other six jobs were deliberately skipped in #78.
**Docker evidence:** separate BuildKit `docker-container` builder with owned-builder cleanup, `--no-cache --load`, actual Compose `config --no-interpolate --quiet`, ingestion/web/proxy Dockerfiles *read-only*, three tags inspected; old `@uiw` error absent. **Python evidence:** only Node-capable image change within workflow, Python 3.11 venv with all 14 tests; Python SDK source unchanged.
**CI-08 security disposition:** `npm audit --json` full=26 affected packages (25 moderate, 1 high); runtime=24 (23 moderate, 1 high), **not counts of distinct advisories**. High-severity advisories [GHSA-6g55-p6wh-862q](https://github.com/advisories/GHSA-6g55-p6wh-862q), [GHSA-r28c-9q8g-f849](https://github.com/advisories/GHSA-r28c-9q8g-f849) affect Next 15.5.29's nested `postcss@8.4.31` despite patched root `postcss@8.5.33`; npm recommends major `next@16.4.6`. Direct `cloudcmd@19.24.0` carries moderate transitive issues (`dompurify`, `file-type`, `glob`, `minimatch`, `got`); npm recommends `cloudcmd@17.0.2` downgrade, not a verified safe pin. No speculative package pin, major upgrade or force fix; a separately tested compatible remediation is needed before release.
**Exact allowlist audit:** pre-documentation candidate diff had only two approved workflow files; final publication includes six approved `notes/001–006` updates. Workflow checks: no trailing whitespace, no `docker ... prune`, no `npm audit fix --force`, all 3 Dockerfile paths present, no runtime or dependency lock changes. Check three-remote tracked tree parity before marking phase CLOSED.
**Scope / release decision:** only `.gitea/workflows/datastack-ci.yml`, `.gitea/workflows/datastack-docker-ci.yml`, and `notes/001–006` changed. No `package.json`/lock, Dockerfile, application UI/UX, runtime, auth, SDK, database, Compose, production data, secret, shared Docker prune, deployment, migration, tag or release. P03 technical CI and advisory-triage acceptance verified by #77 non-Python jobs plus #78 Python; do **not** mislabel #77 overall as successful. Final phase publication requires one main commit and Gitea/GitHub/GitLab exact tracked tree parity. **Production release NO-GO** while runtime high advisory is unresolved; live production health, throughput and deployment remain unverified.
## I. P04-S01 approved baseline and source proof — 2026-11-22
Baseline commit `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3`, content tree `12818ba732f5abdc42af46d83278b13b6be4eafd`. P03 CI #77 `AUDIT_COUNTS runtime {moderate:23,high:1,critical:0,total:24}` (affected packages). Lockfile line proof `package-lock.json:10552-10560` Next `15.5.29` pinned nested postcss `8.4.31`; `10602-10603` resolved nested copy `8.4.31`; `11427-11428` root `8.5.33`. Gitea candidate branch `p04-s01-postcss-security-20261122`, no runtime changes. Approved exact write boundaries are only `package.json` scoped PostCSS override, npm-generated lockfile, `.gitea/workflows/datastack-ci.yml` security-audit block and notes/001–006. Acceptance pending.
## P04-S01 blocked security reconciliation report
P04-S01 SEC-01 BLOCKED — 2026-11-22. Approved scoped `overrides.next.postcss=8.5.33` trialed in disposable Windows npm 10.9.8 workspace only. `npm install --package-lock-only` EXIT 0, but `npm ls --package-lock-only postcss --all` EXIT 1 (`ELSPROBLEMS`: Next nested `postcss@8.4.31` invalid; expected 8.5.33); npm dedupe also kept invalid dependency. Read-only `npm audit --omit=dev --json` EXIT 1, runtime high 1/moderate 23/critical 0. Lockfile regeneration caused unrelated churn (91 package entries added, 106 removed, 69 changed, including two lightningcss optional 1.33.0 -> 1.30.1). Fresh minimal Next 15.5.29 fixture with identical override resolves PostCSS 8.5.33, isolating failure to DataStack committed lock graph. `npm ci --dry-run` exit 0 does not prove real clean install. Exact stop rule applied: no dependency patch, no CI acceptance, no main commit, no GitHub/GitLab sync, no release/deploy; production NO-GO. Need separate explicit narrowly scoped lock reconciliation approval.
Baseline tree `12818ba732f5abdc42af46d83278b13b6be4eafd`; npm `audit --omit=dev` includes affected node `node_modules/next/node_modules/postcss` and high GHSAs above. This proof lives in notes, not a change to repository dependencies; no patch is accepted.
## K. P04-S01B reproducible candidate proof / unaccepted patch — 2026-11-22
**Baseline:** Gitea main `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3`, tree `12818ba732f5abdc42af46d83278b13b6be4eafd`. Gitea source path `package-lock.json:10552-10603` includes Next 15.5.29 and nested PostCSS 8.4.31; root PostCSS 8.5.33. Connected Windows npm 10.9.8 temp probe downloads baseline manifest/lock; exact approved exception deletes `node_modules/next/node_modules/postcss` entry **in disposable copy only** and adds manifest `overrides.next.postcss=8.5.33`. `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline`: exit 0. `npm ls --package-lock-only postcss --all --depth=4`: exit 0; Next uses PostCSS 8.5.33 deduped and no nested 8.4.31. `npm audit --omit=dev --json` exits 1 for remaining **moderates**; counts `{moderate:22,high:0,critical:0,total:22}`; `postcss` no longer appears as vulnerable package. Zero unrelated version changes and exactly 1 removed stale nested package; **67 other package entries changed metadata**: `dev` 36, `libc` 29, 2 `dependencies` JSON order differences (field frequencies can overlap). Must preserve unrelated version AND metadata per user's approval. **STOP**: candidate not committed to repo; no real clean npm ci, no Gitea CI acceptance or workflow gate change, no main merge or 3-mirror sync. P04-S01B remains BLOCKED, released baseline still vulnerable/high=1 and production NO-GO. A future separate exact approved strategy for metadata preservation or bounded exceptions is necessary.
## L. P04-S01C candidate exact patch, CI and scope proof — 2026-11-22
- Baseline: Gitea main `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3`, tree `12818ba732f5abdc42af46d83278b13b6be4eafd`; P04 notes branch `p04-s01-postcss-security-20261122`. Approved patch commit `c759be990c032c2474d2d0bad519c643dec11a3a`. PR [#5](https://dev.ivib.cc/vibtools/DataStack/pulls/5) draft pending full acceptance.
- Exact patch diff from Gitea source commit: `package.json` append only `overrides.next.postcss = 8.5.33`; `package-lock.json` delete only 26 lines for `node_modules/next/node_modules/postcss@8.4.31`; `.gitea/workflows/datastack-ci.yml:security-audit` insert 10-line runtime high/critical failure/verification gate. No changes outside 3 source/config files plus notes/001–006.
- **Generated lock preservation proof:** remove stale entry in disposable npm lock copy; regenerate using npm 10.9.8; restore 67 unrelated metadata entries byte-exactly to baseline. Structural comparison final versus baseline `added=0 removed=1 changedExistingEntries=0`; all unrelated versions, resolved URLs, integrity hashes and dependency metadata identical. `npm ls --package-lock-only postcss --all --depth=4` exit 0, Next 15.5.29 resolves root PostCSS 8.5.33. Final lockfile SHA-256 `e6b5b362877627085dc04c9ded03c019971e07ef9f4af1e78057d58961a1c091`; downloaded Gitea branch lock bytes match. Linux Gitea runner actual `npm ci` SUCCESS, followed by 24 suites. Connected Windows scratch `npm ci` completion not independently verified; Windows Gitea job outstanding.
- Gitea CI [#85](https://dev.ivib.cc/vibtools/DataStack/actions/runs/85) on source commit `c759be99`: node-regression 24/24 + SDK SUCCESS; Python 14/14 SUCCESS; quality strict TS/ESLint SUCCESS; Next 22/22 static pages + standalone SUCCESS; isolated BuildKit 3-image build SUCCESS; security-audit SUCCESS `AUDIT_COUNTS full {moderate:24,high:0,critical:0}`, `AUDIT_COUNTS runtime {moderate:22,high:0,critical:0}`, `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO`, `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Windows-compatibility QUEUED at checkpoint: no full terminal acceptance yet. No main or mirror changes, no deployment, migration, shared Docker prune, tag or release. Production NO-GO pending 7/7 and remote parity, moderate advisories/live verification.
## P04-S01C FINAL CI acceptance (7/7) — 2026-11-22
Gitea CI #85 https://dev.ivib.cc/vibtools/DataStack/actions/runs/85 COMPLETED/SUCCESS; tested implementation SHA c759be990c032c2474d2d0bad519c643dec11a3a; later changes to candidate are documentation only. All seven jobs terminal SUCCESS: node-regression (Linux committed-lock npm ci and exactly 24/24 TypeScript suites); python-regression (Ran 14 tests, OK); quality (strict tsc and ESLint); production-build (Next 15.5.29 standalone, 22/22 pages); docker-image-build (three isolated BuildKit ingestion/web/proxy images); windows-compatibility (Windows committed-lock npm ci and Node SDK build, job #182, terminal SUCCESS); security-audit (runtime moderate=22 high=0 critical=0; full moderate=24 high=0 critical=0; DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO). Single Gitea acceptance run used, no additional broad reruns.
Scope: exactly nine approved paths: package.json scoped next->postcss 8.5.33 override; package-lock.json removes only old nested postcss@8.4.31 entry; existing .gitea/workflows/datastack-ci.yml security-audit job high/critical rejection; notes/001–006 evidence. Other package metadata, versions, integrity and resolved URLs match baseline. Next.js 15.5.29 and Cloudcmd 19.24.0 preserved. Final candidate package-lock SHA256 e6b5b362877627085dc04c9ded03c019971e07ef9f4af1e78057d58961a1c091 (verified byte-for-byte with Gitea). No API, auth, SDK, UI, application runtime, database, Compose, Dockerfile, tests or production data changes, deployment, migration, prune, tag, release.
Exact job IDs: 177 Node, 178 Python, 179 quality, 180 Next production, 181 Docker three images, 182 Windows compatibility, 183 security-audit; all seven completed/success, run #85 overall completed/success. CI fully accepted; publication not yet asserted in these pre-merge notes. Final phase CLOSED requires one squash merge of PR #5 into Gitea main and verified identical tracked content across Gitea/GitHub/GitLab; record actual new SHAs in terminal report. Production release remains NO-GO pending P04-S02 moderate vulnerability triage and separately authorized live readiness verification.
## P04-S02 scoped candidate — 2026-11-22
Baseline Gitea main `2d467ff1494858164a37814f7dbc2b321f510291`, tree `648c76ca496d2c9bf22a16f558c9ba347a45d4f7`. Cloudcmd 19.24.0 -> deepword 11.0.1 -> monaco-editor 0.55.1 pins DOMPurify 3.2.7, vulnerable according to npm advisory inventory. Disposable Windows npm 10.9.8 `npm view dompurify@3.4.18` verified, scoped npm override `overrides.cloudcmd.dompurify=3.4.18` resolves `dompurify@3.4.18` via `npm ls --package-lock-only dompurify --all --depth=5` (exit 0). `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline` exit 0. npm graph initially changed 67 unrelated metadata entries; deterministically restored exact baseline values after validating unrelated versions, URLs, integrity and dependencies. Final lock has **one changed package entry**, DOMPurify; same-major replacement plus verified registry `resolved` and `integrity`, no other packages added/deleted/changed. Candidate runtime audit affected-package counts **moderate 20/high 0/critical 0** versus baseline **22/0/0**; full candidate **moderate 22/high 0/critical 0** versus baseline **24/0/0**. DOMPurify and Monaco Editor dropped from vulnerable-package inventory. Other moderate packages (got, glob, minimatch, nested file-type, Cloudcmd transitive chain) remain; they require major package changes under current advisory constraints, prohibited by Scope Lock. This is a **pre-acceptance candidate**: do not claim application compatibility, CI PASS, publication, or phase CLOSED before full focused Gitea CI and exact three-remote tree parity.
Patch provenance: npm registry DOMPurify 3.4.18 tarball URL and integrity SHA512 in npm-generated entry, unchanged other package metadata after deterministic baseline restoration. New graph root overrides change only Cloudcmd-specific DOMPurify. One focused acceptance run pending.
## P04-S02 final CI acceptance — 2026-11-22
Gitea CI #92 (https://dev.ivib.cc/vibtools/DataStack/actions/runs/92) **COMPLETED / SUCCESS**, all 7/7 jobs terminal success on source commit `a88bc5f4498ea1b11acc0acfeb66b5aa9d6531be`: node-regression real Linux npm ci + 24/24 tests including `tests/files-manager.test.ts` exit 0; Python SDK 14/14; quality strict TypeScript/ESLint; Next.js standalone (22/22 static pages); isolated Docker/BuildKit 3-image verification `DATASTACK_THREE_IMAGES_BUILD_PASS`; Windows committed-lock npm ci + Node SDK compatibility; security-audit `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO` and `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Audit full=22 moderate/high=0/critical=0 (baseline 24), runtime=20 moderate/high=0/critical=0 (baseline 22). These are affected package entries, not distinct advisories. No new CI workflow or extra broad CI dispatched.
Exact approved patch: `package.json` scoped `overrides.cloudcmd.dompurify=3.4.18`, preserving `overrides.next.postcss=8.5.33`; npm-generated `package-lock.json` changes solely `node_modules/dompurify` version 3.2.7 -> 3.4.18 and verified `resolved`/SHA512 integrity. Baseline Cloudcmd 19.24.0, Monaco Editor 0.55.1, Next.js 15.5.29, all unrelated locked package versions, URL, hash, dependency and metadata preserved. Gitea published candidate lock bytes match disposable SHA-256 `57c5ea9df9f28be496f8c3202fbc280b0066bf8c216b8d8a3ca98b6f7b942854`. Exact candidate tree diff 420 entries, 8 approved changed files only (`package.json`, `package-lock.json`, notes/001–006), 0 unauthorized paths. No runtime/UI/auth/API/SDK/tests/CI/Dockerfiles/Compose/db/schema/data modifications.
Detailed proof: final DOMPurify npm registry URL `https://registry.npmjs.org/dompurify/-/dompurify-3.4.18.tgz` with integrity `sha512-ofe3wI/uNm68HInABYFJYqQxRw1EaKAjNdrFVKqjUY+UZurvNd2L/HuQe097s/4RivWiZyP6tT7aoT8atjfhwg==`. Confirmed isolated npm ls override Cloudcmd -> Deepword -> Monaco Editor -> DOMPurify 3.4.18, isolated npm audit runtime 20/0/0 and full 22/0/0, Gitea acceptance matches. **Residual risk:** 20 moderate runtime affected package entries remain, including Cloudcmd transitive dependencies `got`, `glob`, `minimatch`, nested `file-type` and their parent chains. Current npm audit suggests `cloudcmd@17.0.2` (a major downgrade), explicitly out-of-scope; other transitive fixes need major-version dependency-contract changes. No force/downgrade/unverified override. Security improvement verified but NOT zero vulnerabilities. Production release stays NO-GO; remaining advisory disposition and live readiness need independent scope approval.
**Merge gate:** CI #92 7/7 PASS, approved 8-file patch integrity PASS; main squash merge and Gitea/GitHub/GitLab exact Git content-tree verification required before P04-S02 CLOSED. This note is written before that publication, so final remote commit SHAs shall be appended to the PR closure report after verification. No deployment, migrations, prune, tag, release.
**Security/limitations:** never expose Redis backend errors or credentials in ACKs; preserve auth, quotas, rate limits and SDK response fields. Duplicate proof reads no more than 512 recent Redis buffered records and scans up to 256 MiB of CSV partition data with an abort deadline of 1800 ms, refusing uncertain duplicate-success. R-02 streaming CSV scan per flush has unmeasured throughput cost; no globally atomic multi-process exactly-once claim. Unmodified production data, no migrations, no live events, no releases/deploys; original local root staged changes untouched. P02 remains BLOCKED until a separate narrow HTTP durability approval or an equivalent tested safe contract.
+27 -3
View File
@@ -5843,9 +5843,7 @@
} }
}, },
"node_modules/dompurify": { "node_modules/dompurify": {
"version": "3.4.18", "version": "3.2.7",
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.18.tgz",
"integrity": "sha512-ofe3wI/uNm68HInABYFJYqQxRw1EaKAjNdrFVKqjUY+UZurvNd2L/HuQe097s/4RivWiZyP6tT7aoT8atjfhwg==",
"license": "(MPL-2.0 OR Apache-2.0)", "license": "(MPL-2.0 OR Apache-2.0)",
"optionalDependencies": { "optionalDependencies": {
"@types/trusted-types": "^2.0.7" "@types/trusted-types": "^2.0.7"
@@ -10601,6 +10599,32 @@
} }
} }
}, },
"node_modules/next/node_modules/postcss": {
"version": "8.4.31",
"funding": [
{
"type": "opencollective",
"url": "https://opencollective.com/postcss/"
},
{
"type": "tidelift",
"url": "https://tidelift.com/funding/github/npm/postcss"
},
{
"type": "github",
"url": "https://github.com/sponsors/ai"
}
],
"license": "MIT",
"dependencies": {
"nanoid": "^3.3.6",
"picocolors": "^1.0.0",
"source-map-js": "^1.0.2"
},
"engines": {
"node": "^10 || ^12 || >=14"
}
},
"node_modules/nicki": { "node_modules/nicki": {
"version": "6.1.1", "version": "6.1.1",
"license": "MIT", "license": "MIT",
-8
View File
@@ -61,13 +61,5 @@
"optionalDependencies": { "optionalDependencies": {
"lightningcss-linux-arm64-musl": "^1.30.1", "lightningcss-linux-arm64-musl": "^1.30.1",
"lightningcss-linux-x64-musl": "^1.30.1" "lightningcss-linux-x64-musl": "^1.30.1"
},
"overrides": {
"next": {
"postcss": "8.5.33"
},
"cloudcmd": {
"dompurify": "3.4.18"
}
} }
} }
+18 -2
View File
@@ -164,6 +164,16 @@ function setRedisClientForTests(client) {
async function pushEventsToRedisOrFallback(events) { async function pushEventsToRedisOrFallback(events) {
if (!events || events.length === 0) return bufferQueue.length; if (!events || events.length === 0) return bufferQueue.length;
// Failed persistence may leave an unacknowledged RAM event beside an SDK
// retry with the same ID. Persist each client/event ID at most once per push.
const seen = new Set();
events = events.filter((evt) => {
if (!evt || !evt.clientId || !evt.eventId) return true;
const key = JSON.stringify([evt.clientId, evt.eventId]);
if (seen.has(key)) return false;
seen.add(key);
return true;
});
const redis = getRedisClient(); const redis = getRedisClient();
let persistFailed = false; let persistFailed = false;
if (redis) { if (redis) {
@@ -1985,8 +1995,14 @@ const socketHandlersApi = setupSocketHandlers(io, bufferQueue, writer, {
return; return;
} }
if (bufferQueue.length >= runtimeConfig.batchFlushThreshold) { // Redis is optional. Without it, a positive Socket.IO ACK requires a
void flushBuffer().catch(() => undefined); // completed CSV write, not merely an event sitting in volatile RAM.
// Keep existing RAM retry state if the disk write fails.
if (bufferQueue.length > 0) {
const saved = await flushBuffer();
if (saved.lockTimedOut || bufferQueue.length > 0) {
throw new Error('BUFFER_PERSIST_FAILED: Event not durably queued');
}
} }
}, },
}); });
+97 -16
View File
@@ -478,6 +478,84 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
return hasNonEnvelopeKey ? stripped : {}; return hasNonEnvelopeKey ? stripped : {};
} }
// P02 R-01: an idempotency claim is created before persistence. After a
// crash that claim may survive without its event. Prove the event reached
// Redis's buffer or a CSV partition before issuing a duplicate-success ACK.
async function hasDurableEvent(clientId, eventId, timestamp) {
const ackKey = `\u0000durably-acked:${clientId}:${eventId}`;
const previousAck = idempotencyMap.get(ackKey);
if (typeof previousAck === 'number' && Date.now() - previousAck < IDEMPOTENCY_TTL_MS) {
return true;
}
const redis = getRedisClient();
if (redis && typeof redis.lrange === 'function') {
try {
// Bounded read: reject uncertain old claims instead of loading a huge Redis queue.
const queued = await redis.lrange('datastack:buffer:events', -512, -1);
if (Array.isArray(queued) && queued.some((raw) => {
try {
const evt = JSON.parse(raw);
return evt && evt.clientId === clientId && evt.eventId === eventId;
} catch { return false; }
})) {
idempotencyMap.set(ackKey, Date.now());
return true;
}
} catch {
// Redis cannot prove durability; fall through to read-only CSV proof.
}
}
if (!worker || typeof worker.sanitizeClientId !== 'function' ||
typeof worker.extractSafePartitionDate !== 'function' ||
typeof worker.isPartitionFilename !== 'function' ||
typeof worker.extractDateFromFilename !== 'function') return false;
const fs = require('node:fs');
const path = require('node:path');
const { pipeline } = require('node:stream/promises');
const { Writable } = require('node:stream');
const { parse } = require('csv-parse');
const zlib = require('node:zlib');
const aborter = new AbortController();
const deadline = setTimeout(() => aborter.abort(), 1800);
deadline.unref?.();
try {
const base = await fs.promises.realpath(worker.baseDir);
const directory = path.resolve(base, worker.sanitizeClientId(clientId));
const realDir = await fs.promises.realpath(directory);
if (!realDir.startsWith(base + path.sep)) return false;
const date = worker.extractSafePartitionDate(timestamp);
const files = await fs.promises.readdir(realDir, { withFileTypes: true });
let scannedBytes = 0;
for (const item of files) {
if (!item.isFile() || !worker.isPartitionFilename(item.name) ||
worker.extractDateFromFilename(item.name) !== date) continue;
const filename = path.join(realDir, item.name);
scannedBytes += (await fs.promises.stat(filename)).size;
if (scannedBytes > 256 * 1024 * 1024) return false;
let found = false;
const stages = [fs.createReadStream(filename)];
if (item.name.endsWith('.gz')) stages.push(zlib.createGunzip());
stages.push(parse({ columns: true, bom: true, skip_empty_lines: true }));
stages.push(new Writable({
objectMode: true,
write(row, _encoding, done) {
if (row.clientId === clientId && row.eventId === eventId) found = true;
done();
},
}));
await pipeline(...stages, { signal: aborter.signal });
if (found) {
idempotencyMap.set(ackKey, Date.now());
return true;
}
}
} catch {
// Missing, corrupt, unreadable or slow CSV cannot justify positive ACK.
} finally {
clearTimeout(deadline);
}
return false;
}
io.on("connection", (socket) => { io.on("connection", (socket) => {
const clientId = socket.data.clientId; const clientId = socket.data.clientId;
socket.join(`client:${clientId}`); socket.join(`client:${clientId}`);
@@ -551,7 +629,6 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
// A previous Redis write failure can leave an uncommitted idempotency // A previous Redis write failure can leave an uncommitted idempotency
// claim. Never report that claim as a successful durable duplicate. // claim. Never report that claim as a successful durable duplicate.
const failedAt = idempotencyMap.get(`\u0000failed-persist:${clientId}:${eventId}`);
const check = await validateSingleEventAsync( const check = await validateSingleEventAsync(
clientId, clientId,
eventId, eventId,
@@ -560,14 +637,17 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
); );
if (!check.allowed) { if (!check.allowed) {
if (check.duplicate) { if (check.duplicate) {
if (typeof failedAt === "number" && if (await hasDurableEvent(clientId, eventId, timestamp)) {
Date.now() - failedAt < IDEMPOTENCY_TTL_MS) { idempotencyMap.delete(`\u0000failed-persist:${clientId}:${eventId}`);
if (callback) callback({ success: true, eventId, note: "DUPLICATE_DROPPED" });
} else {
// A stale local claim must not block a valid same-ID SDK retry
// after Redis recovers; the Redis claim still guards other workers.
idempotencyMap.delete(`${clientId}:${eventId}`);
if (callback) callback({ if (callback) callback({
success: false, eventId, code: "BUFFER_PERSIST_FAILED", success: false, eventId, code: "BUFFER_PERSIST_FAILED",
error: "BUFFER_PERSIST_FAILED: Event not durably queued", error: "BUFFER_PERSIST_FAILED: Idempotency claim has no durable event",
}); });
} else if (callback) {
callback({ success: true, eventId, note: "DUPLICATE_DROPPED" });
} }
return; return;
} }
@@ -622,7 +702,9 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
return; return;
} }
// Clear any failed local persistence marker only after a durable write.
idempotencyMap.delete(`\u0000failed-persist:${clientId}:${eventId}`); idempotencyMap.delete(`\u0000failed-persist:${clientId}:${eventId}`);
idempotencyMap.set(`\u0000durably-acked:${clientId}:${eventId}`, Date.now());
if (callback) callback({ success: true, eventId }); if (callback) callback({ success: true, eventId });
} catch (err) { } catch (err) {
if (callback) callback({ success: false, error: err.message }); if (callback) callback({ success: false, error: err.message });
@@ -728,8 +810,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
: new Date().toISOString(); : new Date().toISOString();
const rawData = extractBatchItemData(safeItem); const rawData = extractBatchItemData(safeItem);
const failedAt = idempotencyMap.get(`\u0000failed-persist:${clientId}:${eventId}`); const check = await validateSingleEventAsync(
const check = await validateSingleEventAsync(
clientId, clientId,
eventId, eventId,
rawData, rawData,
@@ -737,14 +818,15 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
); );
if (!check.allowed) { if (!check.allowed) {
if (check.duplicate) { if (check.duplicate) {
if (typeof failedAt === "number" && if (await hasDurableEvent(clientId, eventId, timestamp)) {
Date.now() - failedAt < IDEMPOTENCY_TTL_MS) { idempotencyMap.delete(`\u0000failed-persist:${clientId}:${eventId}`);
duplicates += 1;
} else {
idempotencyMap.delete(`${clientId}:${eventId}`);
rejected += 1; rejected += 1;
volatileDuplicate = true; volatileDuplicate = true;
if (errors.length < 5) errors.push( if (errors.length < 5) errors.push(
`${eventId}: BUFFER_PERSIST_FAILED: Event not durably queued`); `${eventId}: BUFFER_PERSIST_FAILED: Idempotency claim has no durable event`);
} else {
duplicates += 1;
} }
continue; continue;
} }
@@ -809,12 +891,11 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
} }
} }
// Only durable new writes clear pending failure markers.
for (const persistedId of acceptedEventIds) { for (const persistedId of acceptedEventIds) {
idempotencyMap.delete(`\u0000failed-persist:${clientId}:${persistedId}`); idempotencyMap.delete(`\u0000failed-persist:${clientId}:${persistedId}`);
idempotencyMap.set(`\u0000durably-acked:${clientId}:${persistedId}`, Date.now());
} }
// A mixed batch must never receive success while one event still has // Never ACK a mixed batch as successful while any event remains volatile.
// an uncommitted Redis claim, even if other items were persisted.
if (volatileDuplicate) { if (volatileDuplicate) {
if (callback) callback({ if (callback) callback({
success: false, processed, rejected, duplicates, success: false, processed, rejected, duplicates,
+93 -138
View File
@@ -36,6 +36,7 @@ import {
const require = createRequire(import.meta.url); const require = createRequire(import.meta.url);
interface IngestionServerModule { interface IngestionServerModule {
writer: { baseDir: string };
server: import("node:http").Server; server: import("node:http").Server;
io: import("socket.io").Server; io: import("socket.io").Server;
bufferQueue: Array<{ clientId: string; eventId: string; timestamp: string; payload: string }>; bufferQueue: Array<{ clientId: string; eventId: string; timestamp: string; payload: string }>;
@@ -1221,7 +1222,7 @@ test("R2 PHASE-02 BUG-10: GET, PATCH (regenerate_token), and DELETE /api/clients
} }
}); });
test("P02 R-01 diagnostic: Redis RPUSH failure yields positive WS ACK for volatile RAM-only event", async () => { test("P02 R-01 remediation: Redis RPUSH failure rejects non-durable WS ACK", async () => {
// Disposable test-only fixture: no production Redis, service, or storage state. // Disposable test-only fixture: no production Redis, service, or storage state.
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold; const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
const previousRam = serverMod.bufferQueue.slice(); const previousRam = serverMod.bufferQueue.slice();
@@ -1255,6 +1256,12 @@ test("P02 R-01 diagnostic: Redis RPUSH failure yields positive WS ACK for volati
assert.equal(sameIdRetry.success, false, assert.equal(sameIdRetry.success, false,
"Pending idempotency claims must not return false positive duplicate ACK"); "Pending idempotency claims must not return false positive duplicate ACK");
assert.equal(sameIdRetry.code, "BUFFER_PERSIST_FAILED"); assert.equal(sameIdRetry.code, "BUFFER_PERSIST_FAILED");
const secondRetry = await emitWithAck<{ success: boolean; code?: string }>(
socket, "sync_data",
{ eventId: "p02-ack-before-crash-001", data: { value: 42 } },
);
assert.equal(secondRetry.success, false, "A repeated pending claim must remain negative");
assert.equal(secondRetry.code, "BUFFER_PERSIST_FAILED");
assert.equal(committedRows.length, 0, "Fake Redis has no durable event"); assert.equal(committedRows.length, 0, "Fake Redis has no durable event");
assert.equal(volatileRows.length, 1, "Acknowledged event is RAM-only"); assert.equal(volatileRows.length, 1, "Acknowledged event is RAM-only");
// Crash model: restart loses RAM but fake Redis survives. No actual process is killed. // Crash model: restart loses RAM but fake Redis survives. No actual process is killed.
@@ -1268,7 +1275,12 @@ test("P02 R-01 diagnostic: Redis RPUSH failure yields positive WS ACK for volati
{ eventId: "p02-ack-recovered-002", data: { value: 43 } }, { eventId: "p02-ack-recovered-002", data: { value: 43 } },
); );
assert.equal(recovered.success, true); assert.equal(recovered.success, true);
assert.equal((await healthyRedis.lrange("datastack:buffer:events", 0, -1)).length, 1); const recoveredSameId = await emitWithAck<{ success: boolean; code?: string }>(
socket, "sync_data",
{ eventId: "p02-ack-before-crash-001", data: { value: 42 } },
);
assert.equal(recoveredSameId.success, true, "Original eventId must be retryable after Redis recovery");
assert.equal((await healthyRedis.lrange("datastack:buffer:events", 0, -1)).length, 2);
} finally { } finally {
socket?.disconnect(); socket?.disconnect();
serverMod.setRedisClientForTests(undefined); serverMod.setRedisClientForTests(undefined);
@@ -1303,6 +1315,7 @@ test("P02 R-01: batch_sync rejects non-durable Redis writes and pending duplicat
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold; const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
const previousQueue = serverMod.bufferQueue.slice(); const previousQueue = serverMod.bufferQueue.slice();
const failingRedis = createSharedMockRedis(); const failingRedis = createSharedMockRedis();
const recoveredRpush = failingRedis.rpush.bind(failingRedis);
failingRedis.rpush = async () => { throw new Error("P02 injected batch Redis outage"); }; failingRedis.rpush = async () => { throw new Error("P02 injected batch Redis outage"); };
serverMod.setRedisClientForTests(failingRedis); serverMod.setRedisClientForTests(failingRedis);
serverMod.runtimeConfig.batchFlushThreshold = 100_000; serverMod.runtimeConfig.batchFlushThreshold = 100_000;
@@ -1331,95 +1344,18 @@ test("P02 R-01: batch_sync rejects non-durable Redis writes and pending duplicat
); );
assert.equal(retry.success, false); assert.equal(retry.success, false);
assert.equal(retry.code, "BUFFER_PERSIST_FAILED"); assert.equal(retry.code, "BUFFER_PERSIST_FAILED");
} finally { failingRedis.rpush = recoveredRpush;
socket?.disconnect();
serverMod.setRedisClientForTests(undefined);
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
}
});
test("P02 FINAL R-01: stale Redis claim never becomes positive ACK after cleanup window", async () => {
// Source proof at candidate 76cd972: a failure marker is -Date.now();
// cleanupMaps compares now - negativeTimestamp > TTL, which expires immediately
// on the first scheduled cleanup even though a Redis idempotency claim remains.
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
const previousQueue = serverMod.bufferQueue.slice();
const redis = createSharedMockRedis(); // intentionally no DEL: simulate stuck Redis claim
redis.rpush = async () => { throw new Error("Injected Redis queue outage"); };
serverMod.setRedisClientForTests(redis);
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
serverMod.bufferQueue.length = 0;
let socket: Socket | undefined;
try {
const client = await SDKManager.registerClientAsync({
clientId: "p02_stale_claim_cleanup", name: "P02 Stale Claim", rateLimit: 100,
});
socket = await connectSocket(client.token);
const payload = { eventId: "p02-stale-claim-1", data: { value: 1 } };
const first = await emitWithAck<{ success: boolean; code?: string }>(socket, "sync_data", payload);
assert.equal(first.success, false);
assert.equal(first.code, "BUFFER_PERSIST_FAILED");
// Verify TTL arithmetic without overriding the global clock (Socket.IO timeouts).
// Also keep two real socket retries to prove pending claims stay negative.
// Do not mutate process timestamps or wait for wall-clock TTL.
const stamped = Date.now();
const cleanupClock = stamped + 61_000;
assert.equal(cleanupClock - (-stamped) > 300_000, true, "Old negative marker expired prematurely");
assert.equal(cleanupClock - stamped < 300_000, true, "Five-minute TTL should still be active");
const handlerSource = readFileSync(require.resolve("../server/socket/handlers.js"), "utf8");
assert.ok(handlerSource.includes('idempotencyMap.set(`\\u0000failed-persist:${claimKey}`, Date.now());'));
assert.equal(handlerSource.includes("idempotencyMap.set(claimKey, -Date.now());"), false);
const retry1 = await emitWithAck<{ success: boolean; code?: string }>(socket, "sync_data", payload);
const retry2 = await emitWithAck<{ success: boolean; code?: string }>(socket, "sync_data", payload);
assert.equal(retry1.success, false);
assert.equal(retry1.code, "BUFFER_PERSIST_FAILED");
assert.equal(retry2.success, false, "Cleanup must not convert a volatile claim into DUPLICATE_DROPPED success");
assert.equal(retry2.code, "BUFFER_PERSIST_FAILED");
assert.equal((await redis.lrange("datastack:buffer:events", 0, -1)).length, 0);
} finally {
socket?.disconnect();
serverMod.setRedisClientForTests(undefined);
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
}
});
test("P02 FINAL R-01: mixed batch with stale volatile claim never ACKs success", async () => {
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
const previousQueue = serverMod.bufferQueue.slice();
const redis = createSharedMockRedis();
const originalRpush = redis.rpush.bind(redis);
redis.rpush = async () => { throw new Error("Injected Redis outage before batch recovery"); };
serverMod.setRedisClientForTests(redis);
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
serverMod.bufferQueue.length = 0;
let socket: Socket | undefined;
try {
const client = await SDKManager.registerClientAsync({
clientId: "p02_mixed_batch_claim", name: "P02 Mixed Batch Claim", rateLimit: 100,
});
socket = await connectSocket(client.token);
const pending = { eventId: "p02-mixed-pending-01", data: { v: 1 } };
const first = await emitWithAck<{ success: boolean; code?: string }>(
socket, "sync_data", pending,
);
assert.equal(first.success, false);
// Simulated process RAM loss: Redis idempotency key survives; queue does not.
serverMod.bufferQueue.length = 0;
redis.rpush = originalRpush;
const mixed = await emitWithAck<{ success: boolean; code?: string; processed?: number }>( const mixed = await emitWithAck<{ success: boolean; code?: string; processed?: number }>(
socket, "batch_sync", [pending, { eventId: "p02-mixed-durable-02", data: { v: 2 } }], socket, "batch_sync", [payload[0], { eventId: "p02-batch-recovered-3", data: { value: 3 } }],
); );
assert.equal(mixed.success, false, "Mixed batch cannot falsely ACK the volatile duplicate"); assert.equal(mixed.success, false, "A mixed recovered/volatile batch cannot ACK success");
assert.equal(mixed.code, "BUFFER_PERSIST_FAILED"); assert.equal(mixed.code, "BUFFER_PERSIST_FAILED");
assert.equal(mixed.processed, 1, "A distinct new event can commit without masking the failed claim"); assert.equal(mixed.processed, 1, "New item is durably queued, old idempotency claim is uncertain");
const rows = await redis.lrange("datastack:buffer:events", 0, -1); const confirmed = await emitWithAck<{ success: boolean; duplicates?: number }>(
assert.equal(rows.length, 1); socket, "batch_sync", payload,
assert.equal((JSON.parse(rows[0]) as { eventId: string }).eventId, "p02-mixed-durable-02"); );
assert.equal(confirmed.success, true, "Once old events are persisted, duplicates are safely acknowledged");
assert.equal(confirmed.duplicates, 2);
} finally { } finally {
socket?.disconnect(); socket?.disconnect();
serverMod.setRedisClientForTests(undefined); serverMod.setRedisClientForTests(undefined);
@@ -1428,70 +1364,89 @@ test("P02 FINAL R-01: mixed batch with stale volatile claim never ACKs success",
} }
}); });
test("P02 FINAL R-01 compatibility: deliberately Redis-free RAM mode remains unchanged", async () => { test("P02 R-01: Redis-free Socket.IO ACK waits for CSV durability and denies disk errors", async () => {
const threshold = serverMod.runtimeConfig.batchFlushThreshold; const fs = await import("node:fs");
const queue = serverMod.bufferQueue.slice(); const os = await import("node:os");
serverMod.setRedisClientForTests(null); const path = await import("node:path");
serverMod.runtimeConfig.batchFlushThreshold = 100_000; const baseDir = fs.mkdtempSync(path.join(os.tmpdir(), "datastack-p02-ack-disk-"));
serverMod.bufferQueue.length = 0; const previousDir = serverMod.writer.baseDir;
let socket: Socket | undefined;
try {
const client = await SDKManager.registerClientAsync({
clientId: "p02_redis_free_compat", name: "P02 Redis-free Compatibility", rateLimit: 100,
});
socket = await connectSocket(client.token);
const ack = await emitWithAck<{ success: boolean; eventId?: string }>(
socket, "sync_data", { eventId: "p02-ram-compat-001", data: { value: 1 } },
);
assert.equal(ack.success, true, "Preserve historical RAM-mode acceptance contract");
assert.equal(serverMod.bufferQueue.some((e) => e.eventId === "p02-ram-compat-001"), true);
} finally {
socket?.disconnect();
serverMod.setRedisClientForTests(undefined);
serverMod.runtimeConfig.batchFlushThreshold = threshold;
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...queue);
}
});
test("P02 FINAL R-01: pending claim plus durable duplicate must reject whole batch", async () => {
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold; const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
const previousQueue = serverMod.bufferQueue.slice(); const previousQueue = serverMod.bufferQueue.slice();
const redis = createSharedMockRedis(); serverMod.writer.baseDir = baseDir;
const healthyRpush = redis.rpush.bind(redis);
serverMod.setRedisClientForTests(redis);
serverMod.runtimeConfig.batchFlushThreshold = 100_000; serverMod.runtimeConfig.batchFlushThreshold = 100_000;
serverMod.setRedisClientForTests(null);
serverMod.bufferQueue.length = 0; serverMod.bufferQueue.length = 0;
let socket: Socket | undefined; let socket: Socket | undefined;
try { try {
const client = await SDKManager.registerClientAsync({ const client = await SDKManager.registerClientAsync({
clientId: "p02_mixed_only_duplicates", name: "P02 Failed and Durable Duplicate", rateLimit: 100, clientId: "p02_disk_durability", name: "P02 Disk Durability", rateLimit: 100,
}); });
socket = await connectSocket(client.token); socket = await connectSocket(client.token);
const durable = { eventId: "p02-durable-duplicate", data: { v: 10 } }; const accepted = await emitWithAck<{ success: boolean; code?: string }>(
const pending = { eventId: "p02-failed-duplicate", data: { v: 20 } }; socket, "sync_data", { eventId: "p02-disk-ack-001", data: { value: 11 } },
const first = await emitWithAck<{ success: boolean }>(socket, "sync_data", durable);
assert.equal(first.success, true);
redis.rpush = async () => { throw new Error("Injected Redis failure for second event"); };
const rejected = await emitWithAck<{ success: boolean; code?: string }>(
socket, "sync_data", pending,
); );
assert.equal(rejected.success, false); assert.equal(accepted.success, true);
assert.equal(rejected.code, "BUFFER_PERSIST_FAILED"); const csvPath = path.join(baseDir, client.clientId, new Date().toISOString().slice(0, 10) + ".csv");
serverMod.bufferQueue.length = 0; // simulate volatile queue lost across restart assert.equal(fs.readFileSync(csvPath, "utf8").includes("p02-disk-ack-001"), true,
redis.rpush = healthyRpush; "A successful ACK requires a completed CSV append when Redis is absent");
const mixed = await emitWithAck<{ const blockedBase = path.join(baseDir, "not-a-directory");
success: boolean; code?: string; processed?: number; duplicates?: number; fs.writeFileSync(blockedBase, "fixture");
}>(socket, "batch_sync", [pending, durable]); serverMod.writer.baseDir = blockedBase;
assert.equal(mixed.processed, 0); const failed = await emitWithAck<{ success: boolean; code?: string }>(
assert.equal(mixed.duplicates, 1); socket, "sync_data", { eventId: "p02-disk-ack-002", data: { value: 12 } },
assert.equal(mixed.success, false, );
"Existing durable duplicate cannot mask a non-durable pending claim"); assert.equal(failed.success, false, "CSV failure must never receive a positive ACK");
assert.equal(mixed.code, "BUFFER_PERSIST_FAILED"); assert.equal(failed.code, "BUFFER_PERSIST_FAILED");
assert.equal((await redis.lrange("datastack:buffer:events", 0, -1)).length, 1); serverMod.writer.baseDir = baseDir;
const retry = await emitWithAck<{ success: boolean; code?: string }>(
socket, "sync_data", { eventId: "p02-disk-ack-002", data: { value: 12 } },
);
assert.equal(retry.success, true, "The same eventId must recover after disk restoration");
const csv = fs.readFileSync(csvPath, "utf8");
assert.equal(csv.split("p02-disk-ack-002").length - 1, 1);
} finally { } finally {
socket?.disconnect(); socket?.disconnect();
serverMod.setRedisClientForTests(undefined); serverMod.setRedisClientForTests(undefined);
serverMod.writer.baseDir = previousDir;
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold; serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue); serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
fs.rmSync(baseDir, { recursive: true, force: true });
}
});
test("P02 R-01 diagnostic: HTTP fallback still ACKs RAM-only data after Redis outage (scope blocker)", async () => {
// P02 remediation only authorizes Socket.IO ACK and buffer *flush* changes.
// Record the separate Port 3000 write/ACK risk; do NOT alter HTTP runtime here.
const redis = createSharedMockRedis();
redis.rpush = async () => { throw new Error("P02 injected HTTP Redis RPUSH outage"); };
const client = await SDKManager.registerClientAsync({
clientId: "p02_http_redis_outage", name: "P02 HTTP Redis Outage", rateLimit: 100,
});
const eventId = "p02-http-ram-only-001";
DataBufferEngine.getLocalBufferSnapshot(1);
const state = (globalThis as unknown as {
__dataStackBufferState?: { buffer: Array<{ eventId: string }>; batchFlushThreshold: number };
}).__dataStackBufferState;
const previousThreshold = state?.batchFlushThreshold;
if (state) state.batchFlushThreshold = 100_000;
DataBufferEngine.setRedisClientForTests(redis);
try {
const response = await postEventsRoute(new NextRequest("http://localhost:3000/api/events", {
method: "POST",
headers: { "content-type": "application/json", authorization: `Bearer ${client.token}` },
body: JSON.stringify({ eventId, data: { source: "p02-http-diagnostic" } }),
}));
const body = await response.json() as { success?: boolean; code?: string };
assert.equal(response.status, 200, "Existing HTTP endpoint reports accepted despite failed Redis write");
assert.equal(body.success, true);
const queued = DataBufferEngine.getLocalBufferSnapshot(1000).filter((evt) => evt.eventId === eventId);
assert.equal(queued.length, 1, "Accepted HTTP event is volatile RAM-only");
assert.equal((await redis.lrange("datastack:buffer:events", 0, -1)).length, 0);
// Simulated process-loss proof: dropping RAM removes the accepted event.
if (state) state.buffer = state.buffer.filter((evt) => evt.eventId !== eventId);
assert.equal(DataBufferEngine.getLocalBufferSnapshot(1000).some((evt) => evt.eventId === eventId), false);
} finally {
if (state && previousThreshold !== undefined) state.batchFlushThreshold = previousThreshold;
DataBufferEngine.setRedisClientForTests(undefined);
} }
}); });
+10 -78
View File
@@ -1629,89 +1629,21 @@ test("P02 R-02: Port 4000 CSV worker suppresses a crash-window replay after Redi
assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 1); assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 1);
const csvPath = path.join(tmpStorageDir, evt.clientId, "2026-11-22.csv"); const csvPath = path.join(tmpStorageDir, evt.clientId, "2026-11-22.csv");
assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1); assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1);
// Crash/restart may occur after daily part rotation and archive compression.
// Verify replay detection against an actual rotated .csv.gz partition.
const zlib = await import("node:zlib");
const archivedPath = path.join(tmpStorageDir, evt.clientId, "2026-11-22-part2.csv.gz");
fs.writeFileSync(archivedPath, zlib.gzipSync(fs.readFileSync(csvPath)));
fs.unlinkSync(csvPath);
const recovered = await ingestion.flushBuffer(); const recovered = await ingestion.flushBuffer();
assert.equal(recovered.written, 0, "Recovered replay should not append the event again"); assert.equal(recovered.written, 0, "Recovered compressed replay must not append again");
assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 0); assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 0);
assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1); assert.equal(fs.existsSync(csvPath), false, "No duplicate active CSV partition should be created");
const archivedRows = zlib.gunzipSync(fs.readFileSync(archivedPath)).toString("utf8");
assert.equal(archivedRows.split(evt.eventId).length - 1, 1);
} finally { } finally {
ingestion.writer.baseDir = previousDir; ingestion.writer.baseDir = previousDir;
ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...previousQueue); ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...previousQueue);
ingestion.setRedisClientForTests(undefined); ingestion.setRedisClientForTests(undefined);
} }
}); });
test("P02 FINAL R-02: Port 4000 scans rotated compressed CSV with quoted multiline payload", async () => {
const zlib = await import("node:zlib");
const redis = new FakeRedisClient();
const ingestion = require("../server/index.js") as IngestionServerModule;
const oldDir = ingestion.writer.baseDir;
const oldQueue = ingestion.bufferQueue.slice();
ingestion.writer.baseDir = tmpStorageDir;
ingestion.bufferQueue.length = 0;
ingestion.setRedisClientForTests(redis);
const event: DataEvent = {
clientId: "p02_rotated_gzip",
eventId: "p02-gzip-quoted-001",
timestamp: "2026-11-22T12:45:00.000Z",
payload: JSON.stringify({ body: "Line one, \"quoted\"\nLine two" }),
};
try {
const diskWriter = new CsvWriterWorker(tmpStorageDir);
await diskWriter.writeBatch([event]);
const dir = path.join(tmpStorageDir, event.clientId);
const csvPath = path.join(dir, "2026-11-22.csv");
const rotatedPath = path.join(dir, "2026-11-22-part2.csv");
const gzPath = rotatedPath + ".gz";
fs.renameSync(csvPath, rotatedPath);
fs.writeFileSync(gzPath, zlib.gzipSync(fs.readFileSync(rotatedPath)));
fs.unlinkSync(rotatedPath);
// A different date partition must not be parsed as part of this replay.
fs.writeFileSync(path.join(dir, "2026-11-21.csv"), 'invalid,"unterminated');
await redis.rpush(REDIS_BUFFER_KEY, JSON.stringify(event));
const replay = await ingestion.flushBuffer();
assert.equal(replay.written, 0, "Already written rotated/gzipped event must not append again");
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0, "Successful dedupe permits Redis trim");
assert.equal(fs.existsSync(csvPath), false, "Do not recreate a duplicate plain CSV");
assert.equal(zlib.gunzipSync(fs.readFileSync(gzPath)).toString("utf8").split(event.eventId).length - 1, 1);
} finally {
ingestion.writer.baseDir = oldDir;
ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...oldQueue);
ingestion.setRedisClientForTests(undefined);
}
});
test("P02 FINAL R-02: partial corrupted CSV blocks Redis trim and recovers after fixture repair", async () => {
const redis = new FakeRedisClient();
const ingestion = require("../server/index.js") as IngestionServerModule;
const oldDir = ingestion.writer.baseDir;
const oldQueue = ingestion.bufferQueue.slice();
ingestion.writer.baseDir = tmpStorageDir;
ingestion.bufferQueue.length = 0;
ingestion.setRedisClientForTests(redis);
const event: DataEvent = {
clientId: "p02_partial_corrupt",
eventId: "p02-corrupt-recovery-001",
timestamp: "2026-11-22T13:15:00.000Z",
payload: JSON.stringify({ value: "safe" }),
};
try {
const dir = path.join(tmpStorageDir, event.clientId);
fs.mkdirSync(dir, { recursive: true });
const badPath = path.join(dir, "2026-11-22.csv");
fs.writeFileSync(badPath, 'clientId,eventId,timestamp,payload\n"unterminated');
await redis.rpush(REDIS_BUFFER_KEY, JSON.stringify(event));
await assert.rejects(() => ingestion.flushBuffer(),
/quote|CSV_|invalid|record|unterminated/i, "Malformed existing CSV must fail closed");
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 1, "Never trim after failed CSV replay check");
// Disposable fixture repair only; does not alter user or production files.
fs.unlinkSync(badPath);
const recovered = await ingestion.flushBuffer();
assert.equal(recovered.written, 1);
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0);
assert.equal(fs.readFileSync(badPath, "utf8").split(event.eventId).length - 1, 1);
} finally {
ingestion.writer.baseDir = oldDir;
ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...oldQueue);
ingestion.setRedisClientForTests(undefined);
}
});
@@ -776,3 +776,60 @@ test("R4-Phase-02 AUDIT-12 / ERR-R4-08: Streaming S3 upload & streaming .csv.gz
assert.equal(fs.existsSync(tmpDir), false); assert.equal(fs.existsSync(tmpDir), false);
} }
}); });
test("P02 R-01: stale pre-persistence Redis claim after restart never yields a false duplicate ACK", async () => {
// A fresh Socket.IO handler has no in-process ACK cache, but Redis retained
// the idempotency claim from a hypothetical crash before Redis RPUSH.
const client = await SDKManager.registerClientAsync({
clientId: "p02_stale_claim", name: "P02 Stale Claim", rateLimit: 200,
});
const eventId = "p02-not-yet-persisted-001";
const stored = [] as string[];
const redis = {
status: "ready",
async get(key: string) {
return key === `datastack:idem:${client.clientId}:${eventId}` ? "1" : null;
},
async set() { return null; },
async incr() { return 1; },
async pexpire() { return 1; },
async lrange() { return stored; },
};
const httpServer = http.createServer();
const io = new Server(httpServer);
setupSocketHandlers(io, [], null, {
getRuntimeConfig: () => ({
jwtSecret: "datastack-jwt-default-secret-key-32-chars-long",
maxEventsPerSec: 200, maxPayloadSizeBytes: 100 * 1024, enableIdempotency: true,
}),
verifyClientInDb: async () => ({ allowed: true, rateLimit: 200 }),
getRedisClient: () => redis,
onBufferPush: async () => undefined,
});
let socket: ReturnType<typeof ioClient> | null = null;
try {
await new Promise<void>((resolve) => httpServer.listen(0, "127.0.0.1", resolve));
const port = (httpServer.address() as AddressInfo).port;
socket = ioClient(`http://127.0.0.1:${port}`, { auth: { token: client.token }, transports: ["websocket"] });
await new Promise<void>((resolve, reject) => {
socket!.once("connect", resolve);
socket!.once("connect_error", reject);
});
const sendDuplicate = () => new Promise<{ success: boolean; code?: string; note?: string }>((resolve, reject) => {
socket!.timeout(3000).emit("sync_data", { eventId, data: { seq: 1 } },
(err: Error | null, ack: { success: boolean; code?: string; note?: string }) => {
if (err) reject(err); else resolve(ack);
});
});
const uncertain = await sendDuplicate();
assert.equal(uncertain.success, false, "Redis idempotency claim alone is not a persistence proof");
assert.equal(uncertain.code, "BUFFER_PERSIST_FAILED");
stored.push(JSON.stringify({ clientId: client.clientId, eventId }));
const durable = await sendDuplicate();
assert.equal(durable.success, true, "Same claim is a valid duplicate only after its event is durable");
assert.equal(durable.note, "DUPLICATE_DROPPED");
} finally {
socket?.disconnect();
await new Promise<void>((resolve) => io.close(() => resolve()));
}
});