Author SHA1 Message Date
victorsteele428 a9ad770063 P04-S02: accepted Cloudcmd DOMPurify security remediation — Gitea CI #92 7/7 PASS (#6)
fix(p04-s02): accept scoped Cloudcmd DOMPurify 3.4.18 remediation, CI #92 PASS [skip ci]

Preserve UI/runtime/SDK/Cloudcmd major and unrelated lock metadata. Security runtime moderate 22->20, high/critical 0. Production remains NO-GO.
2026-11-22 18:37:43 +00:00
victorsteele428 2d467ff149 P04-S01C: verified PostCSS security lock reconciliation and runtime advisory gate (#5)
fix(p04-s01c): accept scoped PostCSS reconciliation with CI #85 7/7 PASS [skip ci]

Preserve dependency graph and unrelated metadata; add runtime high/critical CI gate. Notes/001–006 document acceptance. Production release remains NO-GO.
2026-11-22 17:19:06 +00:00
victorsteele428 fac0934a00 ci(p03): close CI verification and document advisory release block [skip ci] 2026-11-22 12:24:38 +00:00
victorsteele428 48fb277079 P02 final acceptance: CI #72 verified Redis durability and CSV replay safety (#3)
fix(p02): close verified Redis ACK and CSV replay acceptance [skip ci]
2026-11-22 11:49:06 +00:00
15 changed files with 1230 additions and 82 deletions

No files matched your search

+107 -26
View File
@@ -16,10 +16,15 @@ on:
required: false required: false
type: boolean type: boolean
default: false default: false
python_only:
description: Narrow CI-06 Python job rerun after a proven runner setup failure
required: false
type: boolean
default: false
jobs: jobs:
node-regression: node-regression:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.quality_only != 'true' }} if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.quality_only != 'true' && (github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true') }}
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 30 timeout-minutes: 30
steps: steps:
@@ -52,25 +57,36 @@ jobs:
python-regression: python-regression:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.p01_only != 'true' }} if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.p01_only != 'true' }}
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 15 container: node:22-bookworm-slim
timeout-minutes: 20
steps: steps:
- uses: actions/checkout@v4 - name: Install Bookworm Python and checkout prerequisites
with:
github-server-url: https://dev.ivib.cc
- name: Install Python toolchain
run: | run: |
set -eu set -eu
apt-get update -qq apt-get update -qq
apt-get install -y python3 python3-venv python3-pip apt-get install -y --no-install-recommends git ca-certificates python3 python3-venv python3-pip
python3 -m venv /tmp/datastack-ci-venv - uses: actions/checkout@v4
/tmp/datastack-ci-venv/bin/python -m pip install -e ./sdk/python/datastack -e ./sdk-python with:
- name: Run all Python SDK tests github-server-url: https://dev.ivib.cc
- name: CI-06 Python 3.11 isolated virtual environment
run: | run: |
set -eu set -eu
/tmp/datastack-ci-venv/bin/python -m compileall -q ./sdk/python/datastack ./sdk-python python3 --version
/tmp/datastack-ci-venv/bin/python -m unittest discover -s tests -p '*_test.py' -v python3 -m venv /tmp/datastack-ci-venv
/tmp/datastack-ci-venv/bin/python -m pip install --disable-pip-version-check -e ./sdk/python/datastack -e ./sdk-python
- name: CI-06 compile and assert 14 Python SDK regressions
run: |
set -eu
py=/tmp/datastack-ci-venv/bin/python
"$py" -m compileall -q ./sdk/python/datastack ./sdk-python
"$py" -m unittest discover -s tests -p '*_test.py' -v > /tmp/datastack-python-tests.log 2>&1 || { cat /tmp/datastack-python-tests.log; exit 1; }
cat /tmp/datastack-python-tests.log
grep -Eq '^Ran 14 tests in ' /tmp/datastack-python-tests.log
grep -Eq '^OK$' /tmp/datastack-python-tests.log
echo DATASTACK_PYTHON_14_OF_14_PASS
quality: quality:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true' }}
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 20 timeout-minutes: 20
steps: steps:
@@ -85,7 +101,7 @@ jobs:
run: npm run lint run: npm run lint
production-build: production-build:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.quality_only != 'true' }} if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.quality_only != 'true' && (github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true') }}
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 25 timeout-minutes: 25
steps: steps:
@@ -101,30 +117,45 @@ jobs:
test -f .next/standalone/server.js test -f .next/standalone/server.js
docker-image-build: docker-image-build:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.p01_only != 'true' }} if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.p01_only != 'true' && (github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true') }}
runs-on: ubuntu-latest runs-on: ubuntu-latest
timeout-minutes: 35 container: node:22-alpine
timeout-minutes: 45
steps: steps:
- name: CI-07 provision Docker Buildx CLI
run: |
set -eu
apk add --no-cache git docker-cli docker-cli-buildx docker-cli-compose
docker --version
docker buildx version
docker compose version
- uses: actions/checkout@v4 - uses: actions/checkout@v4
with: with:
github-server-url: https://dev.ivib.cc github-server-url: https://dev.ivib.cc
- name: Docker and Compose availability - name: CI-07 inspect daemon and validate Compose without starting services
run: | run: |
set -eu set -eu
docker --version docker info --format 'Driver={{.Driver}} Root={{.DockerRootDir}}'
docker compose version df -h .
docker info docker compose -f docker/docker-compose.yml config --no-interpolate --quiet
- name: Validate Compose structure echo DATASTACK_COMPOSE_CONFIG_PASS
run: docker compose -f docker/docker-compose.yml config --no-interpolate --quiet - name: CI-07 isolated BuildKit all three images (no deployment)
- name: Build Docker images without running services
run: | run: |
set -eu set -eu
docker build -f docker/Dockerfile.server -t datastack-ci-ingestion . builder="datastack-p03-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}"
docker build -f docker/Dockerfile.web -t datastack-ci-web . tag="datastack-p03-${GITHUB_SHA:-local}"
docker build -f docker/Dockerfile.proxy -t datastack-ci-proxy . cleanup() { docker buildx rm --force "$builder" >/dev/null 2>&1 || true; }
trap cleanup EXIT
docker buildx create --name "$builder" --driver docker-container --use
docker buildx inspect "$builder" --bootstrap
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.server -t "$tag-ingestion" .
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.web -t "$tag-web" .
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.proxy -t "$tag-proxy" .
docker image inspect "$tag-ingestion" "$tag-web" "$tag-proxy" >/dev/null
echo DATASTACK_THREE_IMAGES_BUILD_PASS
windows-compatibility: windows-compatibility:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.quality_only != 'true' }} if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.quality_only != 'true' && (github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true') }}
runs-on: windows-latest runs-on: windows-latest
timeout-minutes: 25 timeout-minutes: 25
steps: steps:
@@ -150,3 +181,53 @@ jobs:
if ($LASTEXITCODE -ne 0) { throw 'SDK install failed' } if ($LASTEXITCODE -ne 0) { throw 'SDK install failed' }
npm --prefix sdk/node/datastack run build npm --prefix sdk/node/datastack run build
if ($LASTEXITCODE -ne 0) { throw 'SDK compilation failed' } if ($LASTEXITCODE -ne 0) { throw 'SDK compilation failed' }
security-audit:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.p01_only != 'true' && (github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true') }}
runs-on: ubuntu-latest
container: node:22-alpine
timeout-minutes: 15
steps:
- name: Install checkout prerequisite
run: apk add --no-cache git
- uses: actions/checkout@v4
with:
github-server-url: https://dev.ivib.cc
- name: CI-08 read-only full and runtime npm advisory inventory
run: |
set -eu
if npm audit --json > /tmp/datastack-audit-full.json; then :; else test "$?" -eq 1; fi
if npm audit --omit=dev --json > /tmp/datastack-audit-runtime.json; then :; else test "$?" -eq 1; fi
node <<'NODE'
const fs = require("node:fs");
for (const scope of ["full", "runtime"]) {
const audit = JSON.parse(fs.readFileSync("/tmp/datastack-audit-" + scope + ".json", "utf8"));
if (!audit.metadata || !audit.metadata.vulnerabilities || !audit.vulnerabilities) {
throw Error("Incomplete npm audit response for " + scope);
}
console.log("AUDIT_COUNTS " + scope + " " + JSON.stringify(audit.metadata.vulnerabilities));
for (const [name, item] of Object.entries(audit.vulnerabilities)) {
const advisories = (item.via || []).filter(v => typeof v === "object").map(v => ({
id: v.source, url: v.url, range: v.range, severity: v.severity
}));
console.log("AUDIT_PACKAGE " + scope + " " + JSON.stringify({
package: name, severity: item.severity, direct: item.isDirect,
range: item.range, via: advisories,
viaPackages: (item.via || []).filter(v => typeof v === "string"),
fixAvailable: item.fixAvailable
}));
}
if (scope === "runtime") {
const counts = audit.metadata.vulnerabilities;
if (!Number.isInteger(counts.high) || !Number.isInteger(counts.critical)) {
throw Error("Incomplete runtime npm advisory severity counts");
}
if (counts.high > 0 || counts.critical > 0) {
throw Error("DATASTACK_RUNTIME_SECURITY_GATE_FAILED high=" + counts.high + " critical=" + counts.critical);
}
console.log("DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO");
}
}
NODE
git diff --exit-code -- package.json package-lock.json
echo DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS
+19 -17
View File
@@ -1,39 +1,41 @@
name: DataStack Docker Build CI name: DataStack Docker Build CI
on: on:
push:
branches: [main]
paths:
- '.gitea/workflows/datastack-docker-ci.yml'
workflow_dispatch: workflow_dispatch:
jobs: jobs:
docker-build: docker-build:
runs-on: ubuntu-latest runs-on: ubuntu-latest
container: node:22-alpine container: node:22-alpine
timeout-minutes: 40 timeout-minutes: 45
steps: steps:
- name: Provision isolated Docker CLI and Git tools - name: CI-07 provision Docker Buildx CLI
run: | run: |
set -eu set -eu
apk add --no-cache git docker-cli docker-cli-compose apk add --no-cache git docker-cli docker-cli-buildx docker-cli-compose
docker --version docker --version
docker buildx version
docker compose version docker compose version
- uses: actions/checkout@v4 - uses: actions/checkout@v4
with: with:
github-server-url: https://dev.ivib.cc github-server-url: https://dev.ivib.cc
- name: Validate actual Compose configuration - name: CI-07 inspect daemon and validate Compose without starting services
run: | run: |
set -eu set -eu
docker info --format 'Driver={{.Driver}} Root={{.DockerRootDir}}'
df -h .
docker compose -f docker/docker-compose.yml config --no-interpolate --quiet docker compose -f docker/docker-compose.yml config --no-interpolate --quiet
echo DATASTACK_COMPOSE_CONFIG_PASS echo DATASTACK_COMPOSE_CONFIG_PASS
- name: Check CI-only Docker daemon - name: CI-07 isolated BuildKit all three images (no deployment)
run: | run: |
set -eu set -eu
docker info >/dev/null || { echo DOCKER_DAEMON_UNAVAILABLE; exit 3; } builder="datastack-p03-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}"
- name: Build all DataStack container images without deployment tag="datastack-p03-${GITHUB_SHA:-local}"
run: | cleanup() { docker buildx rm --force "$builder" >/dev/null 2>&1 || true; }
set -eu trap cleanup EXIT
docker build -f docker/Dockerfile.server -t datastack-ci-ingestion . docker buildx create --name "$builder" --driver docker-container --use
docker build -f docker/Dockerfile.web -t datastack-ci-web . docker buildx inspect "$builder" --bootstrap
docker build -f docker/Dockerfile.proxy -t datastack-ci-proxy . docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.server -t "$tag-ingestion" .
echo DATASTACK_DOCKER_IMAGES_PASS docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.web -t "$tag-web" .
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.proxy -t "$tag-proxy" .
docker image inspect "$tag-ingestion" "$tag-web" "$tag-proxy" >/dev/null
echo DATASTACK_THREE_IMAGES_BUILD_PASS
+68 -2
View File
@@ -650,6 +650,66 @@ export class DataBufferEngine {
} }
const executeFlush = async (): Promise<FlushResult> => { const executeFlush = async (): Promise<FlushResult> => {
// P02 R-02: examine CSV partitions before replay of an untrimmed Redis
// batch. Streaming parsing limits memory and handles escaped/newline CSV.
const excludeAlreadyPersisted = async (events: DataEvent[]): Promise<DataEvent[]> => {
if (!events.length) return events;
const fs = await import('node:fs');
const path = await import('node:path');
const zlib = await import('node:zlib');
const { pipeline } = await import('node:stream/promises');
const { Writable } = await import('node:stream');
const { parse } = await import('csv-parse');
const groups = new Map<string, {
clientId: string; date: string; ids: Set<string>;
}>();
for (const event of events) {
const clientId = String(event.clientId || 'unknown');
const date = CSVStorageManager.extractSafePartitionDate(event.timestamp);
const key = JSON.stringify([clientId, date]);
if (!groups.has(key)) groups.set(key, { clientId, date, ids: new Set() });
groups.get(key)!.ids.add(String(event.eventId));
}
const persisted = new Set<string>();
for (const [groupKey, group] of groups) {
const dir = CSVStorageManager.ensureClientDir(group.clientId);
const entries = await fs.promises.readdir(dir, { withFileTypes: true });
for (const entry of entries) {
if (!entry.isFile() ||
!/\.csv(?:\.gz)?$/.test(entry.name) ||
CSVStorageManager.extractSafePartitionDate(
entry.name.replace(/\.csv(?:\.gz)?$/, '').replace(/(?:-(?:part|gz)\d+)+$/, ''),
) !== group.date) continue;
const reader = fs.createReadStream(path.join(dir, entry.name));
const parser = parse({ columns: true, bom: true, skip_empty_lines: true });
const sink = new Writable({
objectMode: true,
write(row: { clientId?: string; eventId?: string }, _enc, done) {
if (row.clientId === group.clientId && group.ids.has(String(row.eventId))) {
persisted.add(JSON.stringify([groupKey, String(row.eventId)]));
}
done();
},
});
if (entry.name.endsWith('.gz')) {
await pipeline(reader, zlib.createGunzip(), parser, sink);
} else {
await pipeline(reader, parser, sink);
}
}
}
const batchSeen = new Set<string>();
return events.filter((event) => {
const key = JSON.stringify([String(event.clientId || 'unknown'),
CSVStorageManager.extractSafePartitionDate(event.timestamp)]);
const rowKey = JSON.stringify([key, String(event.eventId)]);
if (persisted.has(rowKey) || batchSeen.has(rowKey)) return false;
batchSeen.add(rowKey);
return true;
});
};
state.isFlushing = true; state.isFlushing = true;
let totalWritten = 0; let totalWritten = 0;
let lockTimedOut = false; let lockTimedOut = false;
@@ -793,10 +853,13 @@ export class DataBufferEngine {
if (lockLost) break; if (lockLost) break;
if (parsedBatch.length > 0) { if (parsedBatch.length > 0) {
const res = await CSVStorageManager.appendBatch(parsedBatch); const unwritten = await excludeAlreadyPersisted(parsedBatch);
if (unwritten.length > 0) {
const res = await CSVStorageManager.appendBatch(unwritten);
totalWritten += res.written; totalWritten += res.written;
for (const f of res.filesUpdated) filesSet.add(f); for (const f of res.filesUpdated) filesSet.add(f);
} }
}
if (Date.now() - lastRenewAt >= renewIntervalMs) { if (Date.now() - lastRenewAt >= renewIntervalMs) {
await renewLockOnce(); await renewLockOnce();
@@ -845,9 +908,12 @@ export class DataBufferEngine {
if (validRamBatch.length > 0) { if (validRamBatch.length > 0) {
try { try {
const res = await CSVStorageManager.appendBatch(validRamBatch); const unwritten = await excludeAlreadyPersisted(validRamBatch);
if (unwritten.length > 0) {
const res = await CSVStorageManager.appendBatch(unwritten);
totalWritten += res.written; totalWritten += res.written;
for (const f of res.filesUpdated) filesSet.add(f); for (const f of res.filesUpdated) filesSet.add(f);
}
} catch (err) { } catch (err) {
state.buffer.unshift(...validRamBatch); state.buffer.unshift(...validRamBatch);
throw err; throw err;
+78
View File
@@ -89,3 +89,81 @@ Source freeze for P02: Gitea/GitHub main `667a140c4e9e75382bfdcdbe1d3e9eb96f6e80
- **R-03 health:** `server/index.js:1926-1957` returns HTTP 200 `status:online` when Redis LLEN fails, while surfacing `redisConnected:false`; the historical route is **liveness**, not dependency readiness. `app/api/health/route.ts:95-111` likewise reports web liveness/buffer and does not prove database/Redis write readiness. A new readiness endpoint or altered public response requires separate approval. - **R-03 health:** `server/index.js:1926-1957` returns HTTP 200 `status:online` when Redis LLEN fails, while surfacing `redisConnected:false`; the historical route is **liveness**, not dependency readiness. `app/api/health/route.ts:95-111` likewise reports web liveness/buffer and does not prove database/Redis write readiness. A new readiness endpoint or altered public response requires separate approval.
Do not auto-fix source based solely on tests. Runtime code is READ-ONLY under the current approval. Any proven R-01/R-02 risk requiring behavior change needs a new exact method/file approval before application changes. P03 remains independent and unstarted. Do not auto-fix source based solely on tests. Runtime code is READ-ONLY under the current approval. Any proven R-01/R-02 risk requiring behavior change needs a new exact method/file approval before application changes. P03 remains independent and unstarted.
## P02 Runtime Remediation Candidate — 2026-11-22 (NOT YET VERIFIED)
Authorized follow-up baseline: Gitea/GitHub `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898` (GitLab same tree). Local isolated branch `p02-runtime-remediation-20261122`, implementation commit `44eee8877e40d1683f44502c2a0ba3d437a03b88`. Scope: **server/index.js** (`pushEventsToRedisOrFallback`, `flushBuffer`), **server/socket/handlers.js** (sync_data/batch_sync ACK and claim rollback), **core/data/buffer.ts** (`flush`), **tests/phase02-sync.test.ts** and **tests/phase03-storage-buffer.test.ts** only, plus notes/001–006. All other source is unchanged.
**Candidate R-01 behavior:** when a configured Redis client returns an RPUSH error, `pushEventsToRedisOrFallback` retains RAM fallback but throws a sanitized `BUFFER_PERSIST_FAILED` instead of returning a durable-looking buffer count. Socket single/batch ACK returns `success:false`; best-effort Redis idempotency-key removal and bounded in-process negative-ACK marker handle ordinary retries. This is a **candidate** pending tests. When Redis is intentionally absent, historical RAM fallback is deliberately unchanged; it is not a universal durable-ACK guarantee. If Redis claim deletion fails and the server restarts, stale Redis idempotency claims could still misclassify a retry; a full crash-atomic claim/queue protocol is **not established** by this patch.
**Candidate R-02 behavior:** Port 4000 `flushBuffer` and Port 3000 `DataBufferEngine.flush` stream-parse existing CSV/CSV.gz partitions for same-client/date/event IDs before replaying any buffered Redis or RAM batch, dropping already-persisted IDs while retaining Redis queue until the write/check completes. Scan/read/parse exceptions fail closed before LTRIM. In-scope test adds a direct Port 4000 writer + Redis LTRIM-failure/recovery test and updates the existing web-buffer replay fixture. **Not proven globally exactly-once:** cross-process CSV writers outside the Redis flush lock and unknown archival/partial-corruption cases remain; scanning partitions on each flush also has real disk-I/O cost, requiring throughput evaluation before production promotion.
**Candidate acceptance:** one disposable Gitea CI with 24/24 suites, strict TS, lint, Linux/Windows clean npm CI and production build; direct Port4000 replay tests; exact allowlist/diff audit and three-remote tree parity. The Gitea MCP/host was unavailable on the implementation attempt, so this is **NOT PASSED** and the P02 phase remains **BLOCKED / NOT CLOSED**. No live deployment or runtime production mutation.
## P02 Final Acceptance Verification — 2026-11-22 (current superseding evidence)
Baseline published diagnosis: `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`. Runtime candidate `76cd972bdeea9ef07b4b38e9d8b5b5ed3312ca5b` passed Gitea CI #66 (24/24 Node regression suites, Node SDK, strict TypeScript, ESLint, Next production build, Windows SDK). New final narrow patch `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532`: `server/socket/handlers.js` plus two previously scoped TS tests only; final disposable Gitea CI #69 dispatched, terminal result to be recorded before phase closure.
Confirmed exact root cause in candidate: `server/socket/handlers.js` stores failed Redis persistence in `idempotencyMap` at `-Date.now()` while `cleanupMaps` deletes entries when `now - timestamp > 300000`; after the 60000ms cleanup window a failed marker can disappear despite an uncommitted Redis idempotency claim, permitting false-positive `DUPLICATE_DROPPED`. Arithmetic reproduction before code patch: at +61000ms, cleanup deletes a marker although Redis claim TTL (300000ms) is not over. Final patch uses a separate positive-timestamp failed-persistence marker and preserves negative ACK while the outstanding claim remains. A mixed batch with a failed duplicate must not return overall success while distinct records persist.
R-02: `server/index.js:843-897,1043-1058` and `core/data/buffer.ts:652-711,855-872` replay-scan existing same-client/day CSV, preserving Redis queue until a successful write/dedupe; fault tests include direct Port 4000 post-CSV LTRIM outage, rotated compressed .csv.gz containing quoted multiline payload and unrelated-date corrupt partition; malformed matching-date CSV must reject and leave Redis queue intact, with recovery after disposable fixture repair. Streaming parsing bounds memory, **not total disk I/O**; scan latency grows with matching partition size. No universal exactly-once guarantee, no live throughput claim and no claim of finite hard disk-I/O cap.
Public R-03 health remains historical liveness. Intentionally Redis-free RAM fallback and its historical successful ACK stay unchanged. No app/UI/auth/API schema/SDK/Docker/production-data/deployment changes. Work only in a new clean temp worktree; original PC repository's staged 16 P01 files and a separate pre-existing dirty candidate worktree preserved. No production readiness claim until CI #69 and final audit.
### P02 Final Focused Correction — CI #69 to #72
Gitea [CI #69](https://dev.ivib.cc/vibtools/DataStack/actions/runs/69) was run against `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532`: the new Redis stale-claim test timed out because it globally replaced `Date.now` during live Socket.IO ACK scheduling. This is a **test-harness failure**, not a proven backend regression; other new CSV.gz, partial-corruption, mixed-new/durable and Redis-free regression cases passed. An additional source-grounded edge case was found: `batch_sync` previously failed to reject a batch containing a non-durable claim PLUS an already accepted duplicate when `processed=0, duplicates>0`. Under exact approved scope, the `server/socket/handlers.js` branch changed from `if (volatileDuplicate && processed > 0)` to `if (volatileDuplicate)`, ensuring false overall ACK is never sent for any such mixture. Test now uses local simulated TTL arithmetic and real-clock Socket.IO retries; no global clock manipulation. Corrected commit `acca8be01a31f8d3972564cbaec8fd3117a0b551` sent to [focused CI #72](https://dev.ivib.cc/vibtools/DataStack/actions/runs/72); still no P02 closure or production deployment until PASS.
### P02 definitive CI acceptance (2026-11-22)
Source-verified P02 final correction is `acca8be01a31f8d3972564cbaec8fd3117a0b551`. [Gitea CI #72](https://dev.ivib.cc/vibtools/DataStack/actions/runs/72) terminal **SUCCESS** (Linux 24/24 TypeScript suites, explicit R-01/R-02 fault cases, strict tsc, ESLint, Windows Node SDK, Next.js standalone build). Earlier #69 was cancelled after a test-only global `Date.now` Socket.IO ACK timeout; fixed by removing global clock override and adding direct arithmetic proof without changing timers. #72 proves the corrected test passes. Final source-path audit: exactly 11 approved paths (3 runtime, 2 tests, 6 notes). No keys/secrets, auth/SDK/UI/public health, dependency pins, CI workflows, deployment or database migrations changed. ACK remains **best-effort in intentionally Redis-free RAM mode**; Redis-configured RPUSH failure must return negative ACK; CSV deduplication guarantees only tested recovery windows, not globally atomic exactly-once. Partition scans are memory-streamed but disk reads scale with matching partition size, and production throughput is NOT certified. Release/installation still requires separate authorization.
## P03 production forensic acceptance — 2026-11-22
Baseline Gitea/GitHub `48fb277079b380fdbf1e58cd2da40afeed0b03ee`; GitLab baseline same tracked tree `41a169eec0c05ce19bc4acc29d7a8cfe1f8dd90d` (317/317 blobs). Final P03 workflow candidate `549d575653b68c29f97e2f78036646f2e2035719`. [Gitea CI #77](https://dev.ivib.cc/vibtools/DataStack/actions/runs/77): overall **FAILURE** because the initial Python-only checkout lacked Node (exit 127), **not** an SDK regression. Its other six jobs terminal **SUCCESS**: Node `CI_SUITE_SUMMARY total=24 failed=0`, strict TS+ESLint, Next standalone build, Windows SDK, Compose + isolated BuildKit ingestion/web/proxy image build `DATASTACK_THREE_IMAGES_BUILD_PASS`, read-only advisory inventory `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Narrow [Python-only CI #78](https://dev.ivib.cc/vibtools/DataStack/actions/runs/78) terminal **SUCCESS**: Node-capable Bookworm container, Python 3.11 isolated venv, `Ran 14 tests in 3.959s`, `OK`, `DATASTACK_PYTHON_14_OF_14_PASS`. The other six jobs were deliberately skipped in #78.
**CI-08 security disposition:** `npm audit --json` full=26 affected packages (25 moderate, 1 high); runtime=24 (23 moderate, 1 high), **not counts of distinct advisories**. High-severity advisories [GHSA-6g55-p6wh-862q](https://github.com/advisories/GHSA-6g55-p6wh-862q), [GHSA-r28c-9q8g-f849](https://github.com/advisories/GHSA-r28c-9q8g-f849) affect Next 15.5.29's nested `postcss@8.4.31` despite patched root `postcss@8.5.33`; npm recommends major `next@16.4.6`. Direct `cloudcmd@19.24.0` carries moderate transitive issues (`dompurify`, `file-type`, `glob`, `minimatch`, `got`); npm recommends `cloudcmd@17.0.2` downgrade, not a verified safe pin. No speculative package pin, major upgrade or force fix; a separately tested compatible remediation is needed before release.
**Scope / release decision:** only `.gitea/workflows/datastack-ci.yml`, `.gitea/workflows/datastack-docker-ci.yml`, and `notes/001–006` changed. No `package.json`/lock, Dockerfile, application UI/UX, runtime, auth, SDK, database, Compose, production data, secret, shared Docker prune, deployment, migration, tag or release. P03 technical CI and advisory-triage acceptance verified by #77 non-Python jobs plus #78 Python; do **not** mislabel #77 overall as successful. Final phase publication requires one main commit and Gitea/GitHub/GitLab exact tracked tree parity. **Production release NO-GO** while runtime high advisory is unresolved; live production health, throughput and deployment remain unverified.
## P04-S01 blocked security reconciliation report
P04-S01 SEC-01 BLOCKED — 2026-11-22. Approved scoped `overrides.next.postcss=8.5.33` trialed in disposable Windows npm 10.9.8 workspace only. `npm install --package-lock-only` EXIT 0, but `npm ls --package-lock-only postcss --all` EXIT 1 (`ELSPROBLEMS`: Next nested `postcss@8.4.31` invalid; expected 8.5.33); npm dedupe also kept invalid dependency. Read-only `npm audit --omit=dev --json` EXIT 1, runtime high 1/moderate 23/critical 0. Lockfile regeneration caused unrelated churn (91 package entries added, 106 removed, 69 changed, including two lightningcss optional 1.33.0 -> 1.30.1). Fresh minimal Next 15.5.29 fixture with identical override resolves PostCSS 8.5.33, isolating failure to DataStack committed lock graph. `npm ci --dry-run` exit 0 does not prove real clean install. Exact stop rule applied: no dependency patch, no CI acceptance, no main commit, no GitHub/GitLab sync, no release/deploy; production NO-GO. Need separate explicit narrowly scoped lock reconciliation approval.
Root proof: `package-lock.json:10552-10560` Next pins 8.4.31; `:10602-10603` nested 8.4.31; `:11427-11428` root 8.5.33; advisories GHSA-6g55-p6wh-862q and GHSA-r28c-9q8g-f849.
## P04-S01B actual trial / release blocker — 2026-11-22
User explicitly approved P04-S01B stale-copy removal exception. In a clean disposable Windows npm 10.9.8 probe based on Gitea main `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3`, remove ONLY the stale nested PostCSS 8.4.31 entry from the disposable lock copy and set `overrides.next.postcss=8.5.33`. `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline` exited 0; `npm ls --package-lock-only postcss --all --depth=4` exited 0 and Next 15.5.29 used deduped 8.5.33. Read-only runtime audit on the **unpublished candidate** showed high=0, critical=0, moderate=22 (22 affected package entries); SEC-01 candidate mitigation confirmed but NOT accepted. The regenerated lock changed **67 unrelated package metadata entries** (36 `dev` and 29 `libc` field changes plus dependency order-only differences), despite zero unrelated version changes, and removed only the stale node. This violates explicit metadata preservation. STOP before real `npm ci`, Gitea CI, security-gate edit or main/mirror publication. No production release or security-clean claim for published baseline.
## P04-S01C verified candidate / Windows acceptance outstanding — 2026-11-22
Approved deterministic lockfile reconciliation fixed the **SEC-01 Next 15.5.29 nested PostCSS 8.4.31** candidate without touching any unrelated package metadata, versions, hashes, URLs or feature/runtime sources. Scoping: `package.json` scoped `overrides.next.postcss=8.5.33`; `package-lock.json` only 26-line removal of obsolete `node_modules/next/node_modules/postcss`; existing Gitea `security-audit` job adds runtime high/critical failure. Generated lockfile with deterministic baseline metadata restoration had SHA-256 `e6b5b362877627085dc04c9ded03c019971e07ef9f4af1e78057d58961a1c091` and Gitea branch bytes matched exactly. No unrelated package entry/metadata delta (0 added, 1 removed, 0 changed). Gitea [CI #85](https://dev.ivib.cc/vibtools/DataStack/actions/runs/85) candidate `c759be990c032c2474d2d0bad519c643dec11a3a`: Node 24/24, Python 14/14, strict TypeScript + ESLint, Next standalone, Docker 3/3 and runtime advisory gate all SUCCESS. Runtime audit **22 moderate, 0 high, 0 critical**; full audit **24 moderate, 0 high, 0 critical**. **Windows compatibility job still QUEUED at checkpoint; overall CI not yet terminal accepted, main and mirrors unchanged.** Production remains NO-GO pending Windows CI, 3-remote sync, remaining moderate advisories/live health.
## P04-S01C FINAL CI acceptance (7/7) — 2026-11-22
Gitea CI #85 https://dev.ivib.cc/vibtools/DataStack/actions/runs/85 COMPLETED/SUCCESS; tested implementation SHA c759be990c032c2474d2d0bad519c643dec11a3a; later changes to candidate are documentation only. All seven jobs terminal SUCCESS: node-regression (Linux committed-lock npm ci and exactly 24/24 TypeScript suites); python-regression (Ran 14 tests, OK); quality (strict tsc and ESLint); production-build (Next 15.5.29 standalone, 22/22 pages); docker-image-build (three isolated BuildKit ingestion/web/proxy images); windows-compatibility (Windows committed-lock npm ci and Node SDK build, job #182, terminal SUCCESS); security-audit (runtime moderate=22 high=0 critical=0; full moderate=24 high=0 critical=0; DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO). Single Gitea acceptance run used, no additional broad reruns.
Scope: exactly nine approved paths: package.json scoped next->postcss 8.5.33 override; package-lock.json removes only old nested postcss@8.4.31 entry; existing .gitea/workflows/datastack-ci.yml security-audit job high/critical rejection; notes/001–006 evidence. Other package metadata, versions, integrity and resolved URLs match baseline. Next.js 15.5.29 and Cloudcmd 19.24.0 preserved. Final candidate package-lock SHA256 e6b5b362877627085dc04c9ded03c019971e07ef9f4af1e78057d58961a1c091 (verified byte-for-byte with Gitea). No API, auth, SDK, UI, application runtime, database, Compose, Dockerfile, tests or production data changes, deployment, migration, prune, tag, release.
CI fully accepted; publication not yet asserted in these pre-merge notes. Final phase CLOSED requires one squash merge of PR #5 into Gitea main and verified identical tracked content across Gitea/GitHub/GitLab; record actual new SHAs in terminal report. Production release remains NO-GO pending P04-S02 moderate vulnerability triage and separately authorized live readiness verification.
## P04-S02 scoped candidate — 2026-11-22
Baseline Gitea main `2d467ff1494858164a37814f7dbc2b321f510291`, tree `648c76ca496d2c9bf22a16f558c9ba347a45d4f7`. Cloudcmd 19.24.0 -> deepword 11.0.1 -> monaco-editor 0.55.1 pins DOMPurify 3.2.7, vulnerable according to npm advisory inventory. Disposable Windows npm 10.9.8 `npm view dompurify@3.4.18` verified, scoped npm override `overrides.cloudcmd.dompurify=3.4.18` resolves `dompurify@3.4.18` via `npm ls --package-lock-only dompurify --all --depth=5` (exit 0). `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline` exit 0. npm graph initially changed 67 unrelated metadata entries; deterministically restored exact baseline values after validating unrelated versions, URLs, integrity and dependencies. Final lock has **one changed package entry**, DOMPurify; same-major replacement plus verified registry `resolved` and `integrity`, no other packages added/deleted/changed. Candidate runtime audit affected-package counts **moderate 20/high 0/critical 0** versus baseline **22/0/0**; full candidate **moderate 22/high 0/critical 0** versus baseline **24/0/0**. DOMPurify and Monaco Editor dropped from vulnerable-package inventory. Other moderate packages (got, glob, minimatch, nested file-type, Cloudcmd transitive chain) remain; they require major package changes under current advisory constraints, prohibited by Scope Lock. This is a **pre-acceptance candidate**: do not claim application compatibility, CI PASS, publication, or phase CLOSED before full focused Gitea CI and exact three-remote tree parity.
Security forensic disposition: only DOMPurify's same-major compatible override supported by the disposable registry/audit proof; other moderate transitive packages not force-fixed.
## P04-S02 final CI acceptance — 2026-11-22
Gitea CI #92 (https://dev.ivib.cc/vibtools/DataStack/actions/runs/92) **COMPLETED / SUCCESS**, all 7/7 jobs terminal success on source commit `a88bc5f4498ea1b11acc0acfeb66b5aa9d6531be`: node-regression real Linux npm ci + 24/24 tests including `tests/files-manager.test.ts` exit 0; Python SDK 14/14; quality strict TypeScript/ESLint; Next.js standalone (22/22 static pages); isolated Docker/BuildKit 3-image verification `DATASTACK_THREE_IMAGES_BUILD_PASS`; Windows committed-lock npm ci + Node SDK compatibility; security-audit `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO` and `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Audit full=22 moderate/high=0/critical=0 (baseline 24), runtime=20 moderate/high=0/critical=0 (baseline 22). These are affected package entries, not distinct advisories. No new CI workflow or extra broad CI dispatched.
Exact approved patch: `package.json` scoped `overrides.cloudcmd.dompurify=3.4.18`, preserving `overrides.next.postcss=8.5.33`; npm-generated `package-lock.json` changes solely `node_modules/dompurify` version 3.2.7 -> 3.4.18 and verified `resolved`/SHA512 integrity. Baseline Cloudcmd 19.24.0, Monaco Editor 0.55.1, Next.js 15.5.29, all unrelated locked package versions, URL, hash, dependency and metadata preserved. Gitea published candidate lock bytes match disposable SHA-256 `57c5ea9df9f28be496f8c3202fbc280b0066bf8c216b8d8a3ca98b6f7b942854`. Exact candidate tree diff 420 entries, 8 approved changed files only (`package.json`, `package-lock.json`, notes/001–006), 0 unauthorized paths. No runtime/UI/auth/API/SDK/tests/CI/Dockerfiles/Compose/db/schema/data modifications.
Security forensic result. **Residual risk:** 20 moderate runtime affected package entries remain, including Cloudcmd transitive dependencies `got`, `glob`, `minimatch`, nested `file-type` and their parent chains. Current npm audit suggests `cloudcmd@17.0.2` (a major downgrade), explicitly out-of-scope; other transitive fixes need major-version dependency-contract changes. No force/downgrade/unverified override. Security improvement verified but NOT zero vulnerabilities. Production release stays NO-GO; remaining advisory disposition and live readiness need independent scope approval.
**Merge gate:** CI #92 7/7 PASS, approved 8-file patch integrity PASS; main squash merge and Gitea/GitHub/GitLab exact Git content-tree verification required before P04-S02 CLOSED. This note is written before that publication, so final remote commit SHAs shall be appended to the PR closure report after verification. No deployment, migrations, prune, tag, release.
+96 -1
View File
@@ -1,7 +1,7 @@
# DataStack — Production Update Roadmap / Exact Scope Lock # DataStack — Production Update Roadmap / Exact Scope Lock
Planning baseline: `d725841ebc828d58cbe90f517da9e600f23c4630` (frozen; documentation commits will advance Git HEAD but NOT this source baseline). Planning baseline: `d725841ebc828d58cbe90f517da9e600f23c4630` (frozen; documentation commits will advance Git HEAD but NOT this source baseline).
Current state: **P01 CLOSED and CI-verified (1 of 3 phases complete); P02 and P03 NOT STARTED; P02 requires new explicit approval**. Current state: **P01 CLOSED; P02 CI ACCEPTED (Gitea #72 terminal SUCCESS), final three-remote publication gate; P03 NOT STARTED and requires separate explicit approval**. See latest P02 final acceptance appendix and publication report.
Prerequisite for any phase: compare current HEAD and changed paths with this baseline and previously committed notes. For new commits since source baseline, inspect exact diff; no implicit rebase or silent discard. Prerequisite for any phase: compare current HEAD and changed paths with this baseline and previously committed notes. For new commits since source baseline, inspect exact diff; no implicit rebase or silent discard.
## Global non-negotiable Scope Lock (applies to ALL phases) ## Global non-negotiable Scope Lock (applies to ALL phases)
@@ -81,3 +81,98 @@ P02 approved as **diagnostic-only**. Published P01 baseline: `667a140c4e9e75382b
`APPROVED — DataStack P02-RUNTIME-REMEDIATION. Baseline: use the verified P02 diagnostic commit/remote tree. Fix reproduced R-01 and R-02 only, following exact notes/001–006 proof and scoped server/index.js pushEventsToRedisOrFallback/onBufferPush and flushBuffer, server/socket/handlers.js sync_data/batch_sync ACK, and their three existing P02 test files. First document safe ACK/retry semantics, exactly-once versus at-least-once CSV contract and crash recovery. Add disposable fault tests proving no positive ACK for non-durable data and no unbounded duplicate replay. Runtime sources core/data/buffer.ts and server/workers/csvWriter.js remain READ-ONLY unless a separately justified exact file/line amendment is approved. Preserve all UI/auth/SDK/public health behavior and existing error codes, production data, security, other features and Docker. One focused CI, diff/security audit, notes update, commit and 3-remote parity. No deployment, migrations, tags/releases or scope expansion. If a safe fix cannot be achieved within these exact methods, stop and report the required additional Scope Lock rather than broadening automatically.` `APPROVED — DataStack P02-RUNTIME-REMEDIATION. Baseline: use the verified P02 diagnostic commit/remote tree. Fix reproduced R-01 and R-02 only, following exact notes/001–006 proof and scoped server/index.js pushEventsToRedisOrFallback/onBufferPush and flushBuffer, server/socket/handlers.js sync_data/batch_sync ACK, and their three existing P02 test files. First document safe ACK/retry semantics, exactly-once versus at-least-once CSV contract and crash recovery. Add disposable fault tests proving no positive ACK for non-durable data and no unbounded duplicate replay. Runtime sources core/data/buffer.ts and server/workers/csvWriter.js remain READ-ONLY unless a separately justified exact file/line amendment is approved. Preserve all UI/auth/SDK/public health behavior and existing error codes, production data, security, other features and Docker. One focused CI, diff/security audit, notes update, commit and 3-remote parity. No deployment, migrations, tags/releases or scope expansion. If a safe fix cannot be achieved within these exact methods, stop and report the required additional Scope Lock rather than broadening automatically.`
After separate approved remediation and verified P02 closure, **P03 plan** remains CI-06 Linux Python runner provisioning, CI-07 isolated Docker/BuildKit failure remediation and 3-image build, CI-08 supply-chain advisory triage, and non-mutating readiness evidence. P03 should not modify unrelated UI/runtime/API code by default. After separate approved remediation and verified P02 closure, **P03 plan** remains CI-06 Linux Python runner provisioning, CI-07 isolated Docker/BuildKit failure remediation and 3-image build, CI-08 supply-chain advisory triage, and non-mutating readiness evidence. P03 should not modify unrelated UI/runtime/API code by default.
## P02 Runtime Remediation Execution Handoff — 2026-11-22
The user explicitly approved **only** R-01 Redis failure ACK and R-02 Redis-to-CSV replay corrections in these methods: `server/index.js` `pushEventsToRedisOrFallback` / `flushBuffer` / existing `onBufferPush` integration; `server/socket/handlers.js` `sync_data` / `batch_sync` ACK handling; `core/data/buffer.ts` `flush` if needed; the three earlier P02 test files; `notes/001–006`. No other application method/UI/API/auth/SDK/Docker/CI-workflow edit. Public R-03 `/health` semantics remain unchanged.
Local candidate `44eee8877e40d1683f44502c2a0ba3d437a03b88` is **not release-ready** until Gitea acceptance. Before closure, prove: negative ACK on Redis failure with no sensitive error leakage; ordinary successful Redis ACK preserved; both single and batch retries don't convert volatile failures to success; Redis queued event replays after crash without writing duplicate CSV rows in the Port 4000 worker and Port 3000 web buffer; CSV read/parse failure never causes premature Redis trim; validate bounded scan overhead and edge cases for rotated/gzipped partitions; all historical 401/403/429/507/503 tests unchanged.
**Stop rule:** if required fixes need touching `validateSingleEventAsync`, `csvWriter.js` or separate journal/storage scheme, do not broaden the implementation scope; record the residual and request new explicit approval. Existing unrelated files including `MEMORY.md` remain unchanged due exact Scope Lock; this note is the cross-IDE handoff. P03 must wait for P02 CLOSED, not merely implementation committed.
## P02-FINAL-ACCEPTANCE-01 Execution / P03 Gate
User approved strict P02 final acceptance after `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898` published diagnosis and `76cd972bdeea9ef07b4b38e9d8b5b5ed3312ca5b` runtime candidate. Previously failed claim is negative timestamp cleaned with positive-timestamp TTL arithmetic; final patch changes only `server/socket/handlers.js` sync_data/batch_sync marker and mixed-batch handling, plus `tests/phase02-sync.test.ts` and `tests/phase03-storage-buffer.test.ts`. It does not edit production startup, public health, schema, CSV format, SDK or dependency files. Candidate CI #66 **terminal SUCCESS**: all 24/24 TypeScript suites, strict tsc, lint, Windows Node SDK and Next build PASS. Final focused CI #69 on `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532` was dispatched; confirm terminal results before any P02 CLOSED claim. No repeated broad CI runs unless this run demonstrates a scoped failure.
**Acceptance contract:** ACK durability applies to Redis-backed events when Redis configured. Intentionally Redis-free RAM mode retains its historical best-effort ACK. CSV replay is at-least-once with per-partition event-id duplicate suppression under tested crash windows, not a global transactional exactly-once guarantee. Existing scan uses streaming parsing (bounded memory) and scans entire matching partitions (disk I/O scales with data size; monitor before production promotion). Corrupt matching CSV fails closed. If a hard upper disk-scan budget is required for production traffic, obtain a dedicated storage/index design approval rather than silently breaking ingestion.
**Next P03 (ONLY after P02 closure):** CI-06 Python runner environment and 14/14 Python SDK; CI-07 three Docker image builds/isolated BuildKit with no live prune; CI-08 read-only npm advisory IDs/direct-transitive audit, compatible dependency pins only if justified; Compose/standalone/health gate evidence and manual release decision. Exact P03 allowlist remains in this roadmap. No live deploy, migrations, tags or releases without separate approval.
### P02 Final CI Exception / Narrow Retry
The final acceptance test-only clock override in #69 interfered with Socket.IO ACK timing. The source-grounded mixed `batch_sync` all-duplicate edge also required a single conditional correction within the previously approved handler. Commit `acca8be01a31f8d3972564cbaec8fd3117a0b551` changes only `server/socket/handlers.js` and `tests/phase02-sync.test.ts` after initial final candidate; one narrow Gitea CI #72 requested because CI #69 node-regression failed one fixture. Other scope/production invariants remain unchanged. P02 closure requires terminal #72 and exact patch/security evidence.
## P02 Final Acceptance Closure Gate / P03 approval handoff — 2026-11-22
P01 CLOSED; **P02 tests/quality/build ACCEPTED via terminal [Gitea CI #72](https://dev.ivib.cc/vibtools/DataStack/actions/runs/72) SUCCESS** on `acca8be01a31f8d3972564cbaec8fd3117a0b551`; P02 phase transitions to CLOSED only when the verified code plus these documentation-only notes are merged to Gitea main and published to GitHub/GitLab with identical tracked Git trees. Terminal jobs: node-regression 24/24 PASS; quality strict TypeScript and ESLint PASS; Windows compatibility SDK PASS; production Next standalone PASS; Python and Docker intentionally skipped (P03). #69 historical test-only clock failure is resolved in #72. A production deployment and all live services remain **NOT VERIFIED** and NOT authorized.
**P03 next / exact future Scope Lock (requires separate user approval):** CI-06 Linux/Python environment: `.gitea/workflows/datastack-ci.yml` and `.gitea/workflows/datastack-python-ci.yml` only; Python SDK 14/14 isolated. CI-07 Docker/BuildKit 3-image verification: `.gitea/workflows/datastack-ci.yml`, `.gitea/workflows/datastack-docker-ci.yml` only, runner evidence read-only; Dockerfiles can be edited ONLY after independently approved exact exception, no shared runner prune. CI-08 advisory triage: read-only `npm audit --json`, actual CVE/advisory and direct/transitive dependency graph; only minimal justified `package.json`/`package-lock.json` pins under this phase scope, never forced audit repair. `.gitea/workflows/datastack-eslint-ci.yml` quality harness only if source-proven CI-08 need. `notes/001–006` for evidence. Existing UI/API/auth/SDK/runtime logic, Compose, volumes, DB data, secrets, real host, and release configuration remain unchanged. One focused acceptance / one exact patch/security audit / one main commit and 3-way tree sync. Do not deploy, run live ingestion, migrate, prune shared Docker, tag or release.
**P03 approval command (not authorization until user explicitly sends):**
```text
APPROVED — DataStack Production Readiness P03.
Use notes/002 P03 CI-06/CI-07/CI-08 Exact Scope Lock and the verified published P02 main baseline.
CI-06: fix Python runner/toolchain only in approved .gitea workflows; verify Python SDK 14/14.
CI-07: isolate Docker runner/BuildKit and build all 3 images without deploying or pruning shared caches; Dockerfiles READ-ONLY unless a separately approved exact exception.
CI-08: read-only npm audit advisory inventory, minimal compatible package pins only for proven security need; no npm audit fix --force.
Preserve UI/UX, app/API/auth/SDK/runtime, Compose, production data, secrets and existing features unchanged.
Use one focused Gitea CI, strict patch/security diff, notes/001–006 update, Git commit and Gitea/GitHub/GitLab tree parity.
No live deployment, migrations, tags or releases.
Close P03 only on verified acceptance; provide production release readiness report.
```
## P03 scoped execution and release gate — 2026-11-22
Baseline Gitea/GitHub `48fb277079b380fdbf1e58cd2da40afeed0b03ee`; GitLab baseline same tracked tree `41a169eec0c05ce19bc4acc29d7a8cfe1f8dd90d` (317/317 blobs). Final P03 workflow candidate `549d575653b68c29f97e2f78036646f2e2035719`. [Gitea CI #77](https://dev.ivib.cc/vibtools/DataStack/actions/runs/77): overall **FAILURE** because the initial Python-only checkout lacked Node (exit 127), **not** an SDK regression. Its other six jobs terminal **SUCCESS**: Node `CI_SUITE_SUMMARY total=24 failed=0`, strict TS+ESLint, Next standalone build, Windows SDK, Compose + isolated BuildKit ingestion/web/proxy image build `DATASTACK_THREE_IMAGES_BUILD_PASS`, read-only advisory inventory `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Narrow [Python-only CI #78](https://dev.ivib.cc/vibtools/DataStack/actions/runs/78) terminal **SUCCESS**: Node-capable Bookworm container, Python 3.11 isolated venv, `Ran 14 tests in 3.959s`, `OK`, `DATASTACK_PYTHON_14_OF_14_PASS`. The other six jobs were deliberately skipped in #78.
**CI-08 security disposition:** `npm audit --json` full=26 affected packages (25 moderate, 1 high); runtime=24 (23 moderate, 1 high), **not counts of distinct advisories**. High-severity advisories [GHSA-6g55-p6wh-862q](https://github.com/advisories/GHSA-6g55-p6wh-862q), [GHSA-r28c-9q8g-f849](https://github.com/advisories/GHSA-r28c-9q8g-f849) affect Next 15.5.29's nested `postcss@8.4.31` despite patched root `postcss@8.5.33`; npm recommends major `next@16.4.6`. Direct `cloudcmd@19.24.0` carries moderate transitive issues (`dompurify`, `file-type`, `glob`, `minimatch`, `got`); npm recommends `cloudcmd@17.0.2` downgrade, not a verified safe pin. No speculative package pin, major upgrade or force fix; a separately tested compatible remediation is needed before release.
**Scope / release decision:** only `.gitea/workflows/datastack-ci.yml`, `.gitea/workflows/datastack-docker-ci.yml`, and `notes/001–006` changed. No `package.json`/lock, Dockerfile, application UI/UX, runtime, auth, SDK, database, Compose, production data, secret, shared Docker prune, deployment, migration, tag or release. P03 technical CI and advisory-triage acceptance verified by #77 non-Python jobs plus #78 Python; do **not** mislabel #77 overall as successful. Final phase publication requires one main commit and Gitea/GitHub/GitLab exact tracked tree parity. **Production release NO-GO** while runtime high advisory is unresolved; live production health, throughput and deployment remain unverified.
## P04-S01 approved security remediation scope — 2026-11-22
Baseline Gitea `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3`, tree `12818ba732f5abdc42af46d83278b13b6be4eafd`. SEC-01: Next `15.5.29` nested `postcss@8.4.31` is the sole P03 high runtime affected package despite top-level `postcss@8.5.33`; see CI #77 `AUDIT_PACKAGE runtime postcss`, GHSA-6g55-p6wh-862q / GHSA-r28c-9q8g-f849. Source `package-lock.json:10552-10560`, `10602-10603`, `11427-11428`; `package.json:35,37`. Approval authorizes only scoped override in `package.json`, npm-generated `package-lock.json`, **existing** `.gitea/workflows/datastack-ci.yml` security-audit job and `notes/001–006`. All application, test, SDK, Docker/Compose and deployment files READ-ONLY. Never force next@16, downgrade cloudcmd, `npm audit fix --force`, prune shared Docker, deploy, migrate or tag. Acceptance: reproducible lockfile, clean npm ci, Node 24/24, Python 14/14, tsc/lint, Windows SDK, Next standalone, 3 Docker images, runtime high/critical zero and exact Git diff. One focused Gitea CI, one accepted main commit, three-remote content parity; unresolved moderates/live checks retain production NO-GO. If override incompatible, STOP as BLOCKED and request exact exception.
## P04-S01 blocked security reconciliation report
P04-S01 SEC-01 BLOCKED — 2026-11-22. Approved scoped `overrides.next.postcss=8.5.33` trialed in disposable Windows npm 10.9.8 workspace only. `npm install --package-lock-only` EXIT 0, but `npm ls --package-lock-only postcss --all` EXIT 1 (`ELSPROBLEMS`: Next nested `postcss@8.4.31` invalid; expected 8.5.33); npm dedupe also kept invalid dependency. Read-only `npm audit --omit=dev --json` EXIT 1, runtime high 1/moderate 23/critical 0. Lockfile regeneration caused unrelated churn (91 package entries added, 106 removed, 69 changed, including two lightningcss optional 1.33.0 -> 1.30.1). Fresh minimal Next 15.5.29 fixture with identical override resolves PostCSS 8.5.33, isolating failure to DataStack committed lock graph. `npm ci --dry-run` exit 0 does not prove real clean install. Exact stop rule applied: no dependency patch, no CI acceptance, no main commit, no GitHub/GitLab sync, no release/deploy; production NO-GO. Need separate explicit narrowly scoped lock reconciliation approval.
Approved scope remains `package.json` scoped override, npm-generated `package-lock.json`, current Gitea `security-audit` job, notes/001–006. Do not add manual lock edits, major Next 16, Cloudcmd downgrade, npm audit fix --force or unrelated source files. Phase BLOCKED until amended approval.
## P04-S01B correction gate — 2026-11-22
The targeted removal-and-regenerate method fixes SEC-01 in an isolated copy: valid `npm ls` exit 0 and `npm audit --omit=dev` high=0/critical=0/moderate=22, while Next 15.5.29, Cloudcmd 19.24.0 and all unrelated versions remain unchanged. **But** npm regenerated 67 unrelated package metadata entries (36 `dev` flags, 29 `libc` fields, 2 dependency map order changes); that is outside the expressly approved preservation contract. P04-S01B **BLOCKED by exact scope lock**, not by unresolved PostCSS resolution. Do not commit the unacceptable lock or quietly normalize metadata. Require separate approved exact metadata-reconciliation strategy/exception and content-preserving proof. Until then no acceptance CI, commit to main, remote parity, release or deployment.
## P04-S01C implementation checkpoint and exact release gate — 2026-11-22
Approved P04-S01C changes are strictly `package.json`, `package-lock.json`, existing `.gitea/workflows/datastack-ci.yml:security-audit`, notes/001–006. Applied scoped Next→PostCSS 8.5.33 override, npm regenerated disposable lock after stale entry deletion, deterministically restored **all 67** unrelated npm metadata modifications against baseline; exact final diff only stale nested package removal. No version, integrity, peer or optional metadata drift. Gitea branch patch `c759be990c032c2474d2d0bad519c643dec11a3a`; lock SHA-256 `e6b5b362877627085dc04c9ded03c019971e07ef9f4af1e78057d58961a1c091`. Focused [CI #85](https://dev.ivib.cc/vibtools/DataStack/actions/runs/85) six jobs terminal SUCCESS including Docker 3/3, Python 14/14, Node 24/24, Next/tsc/lint, runtime high/critical zero; Windows compatibility **QUEUED**, so P04-S01C **PENDING**. Do not merge/mark CLOSED until Windows job succeeds, overall #85 SUCCESS, exact file-scope audit and Gitea/GitHub/GitLab tracked-tree parity; no deployment, migration, release/tag/prune. After closure, independently approve P04-S02 Cloudcmd moderate advisory triage and later production readiness.
## P04-S01C FINAL CI acceptance (7/7) — 2026-11-22
Gitea CI #85 https://dev.ivib.cc/vibtools/DataStack/actions/runs/85 COMPLETED/SUCCESS; tested implementation SHA c759be990c032c2474d2d0bad519c643dec11a3a; later changes to candidate are documentation only. All seven jobs terminal SUCCESS: node-regression (Linux committed-lock npm ci and exactly 24/24 TypeScript suites); python-regression (Ran 14 tests, OK); quality (strict tsc and ESLint); production-build (Next 15.5.29 standalone, 22/22 pages); docker-image-build (three isolated BuildKit ingestion/web/proxy images); windows-compatibility (Windows committed-lock npm ci and Node SDK build, job #182, terminal SUCCESS); security-audit (runtime moderate=22 high=0 critical=0; full moderate=24 high=0 critical=0; DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO). Single Gitea acceptance run used, no additional broad reruns.
Scope: exactly nine approved paths: package.json scoped next->postcss 8.5.33 override; package-lock.json removes only old nested postcss@8.4.31 entry; existing .gitea/workflows/datastack-ci.yml security-audit job high/critical rejection; notes/001–006 evidence. Other package metadata, versions, integrity and resolved URLs match baseline. Next.js 15.5.29 and Cloudcmd 19.24.0 preserved. Final candidate package-lock SHA256 e6b5b362877627085dc04c9ded03c019971e07ef9f4af1e78057d58961a1c091 (verified byte-for-byte with Gitea). No API, auth, SDK, UI, application runtime, database, Compose, Dockerfile, tests or production data changes, deployment, migration, prune, tag, release.
Next roadmap: P04-S02 Cloudcmd moderate-advisory triage, followed by separate live readiness approval. No forced major upgrade/downgrade. CI fully accepted; publication not yet asserted in these pre-merge notes. Final phase CLOSED requires one squash merge of PR #5 into Gitea main and verified identical tracked content across Gitea/GitHub/GitLab; record actual new SHAs in terminal report. Production release remains NO-GO pending P04-S02 moderate vulnerability triage and separately authorized live readiness verification.
## P04-S02 scoped candidate — 2026-11-22
Baseline Gitea main `2d467ff1494858164a37814f7dbc2b321f510291`, tree `648c76ca496d2c9bf22a16f558c9ba347a45d4f7`. Cloudcmd 19.24.0 -> deepword 11.0.1 -> monaco-editor 0.55.1 pins DOMPurify 3.2.7, vulnerable according to npm advisory inventory. Disposable Windows npm 10.9.8 `npm view dompurify@3.4.18` verified, scoped npm override `overrides.cloudcmd.dompurify=3.4.18` resolves `dompurify@3.4.18` via `npm ls --package-lock-only dompurify --all --depth=5` (exit 0). `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline` exit 0. npm graph initially changed 67 unrelated metadata entries; deterministically restored exact baseline values after validating unrelated versions, URLs, integrity and dependencies. Final lock has **one changed package entry**, DOMPurify; same-major replacement plus verified registry `resolved` and `integrity`, no other packages added/deleted/changed. Candidate runtime audit affected-package counts **moderate 20/high 0/critical 0** versus baseline **22/0/0**; full candidate **moderate 22/high 0/critical 0** versus baseline **24/0/0**. DOMPurify and Monaco Editor dropped from vulnerable-package inventory. Other moderate packages (got, glob, minimatch, nested file-type, Cloudcmd transitive chain) remain; they require major package changes under current advisory constraints, prohibited by Scope Lock. This is a **pre-acceptance candidate**: do not claim application compatibility, CI PASS, publication, or phase CLOSED before full focused Gitea CI and exact three-remote tree parity.
Roadmap: finish one focused Gitea acceptance, exact 8-file scope and three-remote parity. Remaining moderate findings deferred to independently approved scope, never silently upgraded.
## P04-S02 final CI acceptance — 2026-11-22
Gitea CI #92 (https://dev.ivib.cc/vibtools/DataStack/actions/runs/92) **COMPLETED / SUCCESS**, all 7/7 jobs terminal success on source commit `a88bc5f4498ea1b11acc0acfeb66b5aa9d6531be`: node-regression real Linux npm ci + 24/24 tests including `tests/files-manager.test.ts` exit 0; Python SDK 14/14; quality strict TypeScript/ESLint; Next.js standalone (22/22 static pages); isolated Docker/BuildKit 3-image verification `DATASTACK_THREE_IMAGES_BUILD_PASS`; Windows committed-lock npm ci + Node SDK compatibility; security-audit `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO` and `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Audit full=22 moderate/high=0/critical=0 (baseline 24), runtime=20 moderate/high=0/critical=0 (baseline 22). These are affected package entries, not distinct advisories. No new CI workflow or extra broad CI dispatched.
Exact approved patch: `package.json` scoped `overrides.cloudcmd.dompurify=3.4.18`, preserving `overrides.next.postcss=8.5.33`; npm-generated `package-lock.json` changes solely `node_modules/dompurify` version 3.2.7 -> 3.4.18 and verified `resolved`/SHA512 integrity. Baseline Cloudcmd 19.24.0, Monaco Editor 0.55.1, Next.js 15.5.29, all unrelated locked package versions, URL, hash, dependency and metadata preserved. Gitea published candidate lock bytes match disposable SHA-256 `57c5ea9df9f28be496f8c3202fbc280b0066bf8c216b8d8a3ca98b6f7b942854`. Exact candidate tree diff 420 entries, 8 approved changed files only (`package.json`, `package-lock.json`, notes/001–006), 0 unauthorized paths. No runtime/UI/auth/API/SDK/tests/CI/Dockerfiles/Compose/db/schema/data modifications.
Roadmap: after P04-S02 accepted mirror parity, next separately approved P04-S03 remaining moderate advisory dependency-contract triage (read-only / disposable compatibility first), then P05 live readiness. **Residual risk:** 20 moderate runtime affected package entries remain, including Cloudcmd transitive dependencies `got`, `glob`, `minimatch`, nested `file-type` and their parent chains. Current npm audit suggests `cloudcmd@17.0.2` (a major downgrade), explicitly out-of-scope; other transitive fixes need major-version dependency-contract changes. No force/downgrade/unverified override. Security improvement verified but NOT zero vulnerabilities. Production release stays NO-GO; remaining advisory disposition and live readiness need independent scope approval.
**Merge gate:** CI #92 7/7 PASS, approved 8-file patch integrity PASS; main squash merge and Gitea/GitHub/GitLab exact Git content-tree verification required before P04-S02 CLOSED. This note is written before that publication, so final remote commit SHAs shall be appended to the PR closure report after verification. No deployment, migrations, prune, tag, release.
+109 -2
View File
@@ -2,13 +2,13 @@
Frozen source baseline: `d725841ebc828d58cbe90f517da9e600f23c4630`. Generated: 2026-11-22 Asia/Dhaka. Frozen source baseline: `d725841ebc828d58cbe90f517da9e600f23c4630`. Generated: 2026-11-22 Asia/Dhaka.
Scope: documentation-only kickoff. Any future phase must update this log **after** verification, not on intention. Scope: documentation-only kickoff. Any future phase must update this log **after** verification, not on intention.
**Completed implementation phases: 1 / 3. Remaining: 2.** Documentation baseline preparation is a separate non-implementation milestone. **P01 CLOSED; P02 verified in Gitea CI #72 and awaiting recorded three-remote publication; P03 NOT STARTED.** Publication/tree-parity proof and final phase-close status are recorded in the latest acceptance appendix and operator report.
| Phase | Status | IDs / function | Changed source | Verified CI evidence | Commit / remote sync | | Phase | Status | IDs / function | Changed source | Verified CI evidence | Commit / remote sync |
|---|---|---|---|---|---| |---|---|---|---|---|---|
| P00 Notes / freeze | COMPLETED (documentation only) | `001–006` docs, source baseline and exact scope | `notes/**` only | 6 Markdown files; balanced fences; staged diff `--check` PASS | Documentation commit containing this file; remote SHA verification recorded in session handoff | | P00 Notes / freeze | COMPLETED (documentation only) | `001–006` docs, source baseline and exact scope | `notes/**` only | 6 Markdown files; balanced fences; staged diff `--check` PASS | Documentation commit containing this file; remote SHA verification recorded in session handoff |
| P01 Tests & lockfile | **CLOSED — VERIFIED** | CI-01…CI-05, no runtime feature changes | 16 exact P01 code/test/CI paths, plus `notes/001–006` records | #59 Linux/Windows clean npm/SDK, 24/24 TS suites, Next standalone PASS; #60 strict TypeScript and ESLint PASS; historical #59 quality mismatch corrected | One permanent P01 main commit containing these notes; confirm remote SHA/tree parity in phase handoff | | P01 Tests & lockfile | **CLOSED — VERIFIED** | CI-01…CI-05, no runtime feature changes | 16 exact P01 code/test/CI paths, plus `notes/001–006` records | #59 Linux/Windows clean npm/SDK, 24/24 TS suites, Next standalone PASS; #60 strict TypeScript and ESLint PASS; historical #59 quality mismatch corrected | One permanent P01 main commit containing these notes; confirm remote SHA/tree parity in phase handoff |
| P02 Ingestion reliability | **BLOCKED — three diagnostic cases reproduced; runtime fix requires separate approval** | R-01 ACK/RAM loss; R-02 CSV duplicate replay; R-03 health liveness vs readiness | Only `tests/phase02-sync.test.ts` and `tests/phase03-storage-buffer.test.ts`, plus notes/001–006 | Gitea #63: all three P02 subtests PASS to reproduce existing behavior; full matrix status recorded in appendix | Diagnostic branch `p02-readiness-test-20261122`, no production runtime changes | | P02 Ingestion reliability | **CI VERIFIED #72 — final publish gate** | R-01 ACK/idempotency safety; R-02 CSV replay; R-03 liveness verified unchanged | Exactly 3 runtime files, 2 tests, notes/001-006 | #72 terminal SUCCESS: 24/24 TS, TS typecheck, lint, Next standalone, Windows SDK | Final source/test acca8be01a31f8d3972564cbaec8fd3117a0b551; three-remote main parity recorded on publication |
| P03 CI/Docker/security | NOT STARTED | CI-06,07,08; release gates | None | Not run | — | | P03 CI/Docker/security | NOT STARTED | CI-06,07,08; release gates | None | Not run | — |
## Existing diagnostic CI snapshot (NOT implementation phase completions) ## Existing diagnostic CI snapshot (NOT implementation phase completions)
@@ -91,3 +91,110 @@ Historical baseline code stayed frozen through these CI workflow changes. Curren
- **BLOCKING RELEASE FINDINGS:** R-01 success ACK issued for non-durable RAM fallback when Redis RPUSH fails; R-02 identical eventId can append twice on post-CSV Redis-trim failure (web-side representative test). R-03 health remains online on Redis loss; liveness-only behavior is backward-compatible, protected dependency-readiness verification remains missing. - **BLOCKING RELEASE FINDINGS:** R-01 success ACK issued for non-durable RAM fallback when Redis RPUSH fails; R-02 identical eventId can append twice on post-CSV Redis-trim failure (web-side representative test). R-03 health remains online on Redis loss; liveness-only behavior is backward-compatible, protected dependency-readiness verification remains missing.
- **P02 PHASE = BLOCKED**, diagnostic implementation done, release reliability acceptance NOT satisfied. Runtime remediation requires separate exact approval. 1/3 phases still CLOSED (P01 only); P02 and P03 remain unclosed. No P03 start authorization from this diagnostic completion. - **P02 PHASE = BLOCKED**, diagnostic implementation done, release reliability acceptance NOT satisfied. Runtime remediation requires separate exact approval. 1/3 phases still CLOSED (P01 only); P02 and P03 remain unclosed. No P03 start authorization from this diagnostic completion.
- Next approval must explicitly authorize exact narrow server/core patch for ACK durability and CSV replay deduplication, plus concrete no-loss/no-duplicate tests. Any readiness endpoint change needs separate explicit approval. User's initial P02 approval is insufficient for runtime mutation. - Next approval must explicitly authorize exact narrow server/core patch for ACK durability and CSV replay deduplication, plus concrete no-loss/no-duplicate tests. Any readiness endpoint change needs separate explicit approval. User's initial P02 approval is insufficient for runtime mutation.
## P02-RUNTIME-REMEDIATION — Candidate implementation checkpoint (2026-11-22)
- **Current P02: BLOCKED / NOT CLOSED.** One previous diagnostic Gitea CI #63 reproduced R-01/R-02/R-03; runtime remediation was separately approved after that. Verified completed implementation phases remain **1/3** (P01 only). Do not approve P03 yet.
- Published P02 diagnosis baseline `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`; isolated local candidate commit `44eee8877e40d1683f44502c2a0ba3d437a03b88` on `p02-runtime-remediation-20261122`.
- Exactly five code/test files changed in this candidate: `server/index.js`, `server/socket/handlers.js`, `core/data/buffer.ts`, `tests/phase02-sync.test.ts`, `tests/phase03-storage-buffer.test.ts`. No `server/workers/csvWriter.js`, `app/**`, `components/**`, SDK, public R-03 health, CI workflow, database, migration, production data, secrets or deployment touched.
- JS `node --check` on two changed JS files PASS; `git diff --check` PASS and allowlist audit PASS. Targeted TS/Node regression and Gitea CI **NOT YET RUN TO TERMINAL PASS**. The Gitea connector/host was unavailable when dispatch/push was attempted. Local temporary dependency installation also not yet complete.
- The original user PC root `D:\VibTools_Workspace\01_Products\14_DataStack` still has 16 pre-existing staged P01 paths on older `d8ac96e`; deliberately preserved without reset/stash/discard. All P02 implementation is in a separate temporary worktree. Do not assume the old local root is up to date.
- R-01 and R-02 implementations are candidates; practical reliability limitations and I/O cost recorded in `notes/001` and `notes/004`. No false claim of globally exactly-once CSV or universally durable ACK.
- **Next gates:** restore Gitea access; run one focused disposable acceptance; fix any in-scope test/TS defects; record exact CI terminal results; only after required acceptance and security audit, squash single phase main commit and verify GitHub/GitLab/Gitea tree parity; then, and only then, decide P02 CLOSED versus residual-risk blocker.
### Verification environment blocker — local dependency install
The authorized Windows PC attempted a **local disposable** `npm.cmd ci --ignore-scripts --no-audit --no-fund --prefer-offline` in the isolated P02 worktree only; it failed with `npm ERR! ETARGET No matching version found for yaml@2.9.1` plus cleanup EPERM warnings in the temp worktree's `node_modules`. Exact output captured in `%TEMP%\DataStack-P02-npm-ci.log` on the authorized PC. This is **not a passed code test nor proof of a repository package defect**, and `package-lock.json` / `package.json` cannot be edited under P02 scope. Gitea MCP remained UNAVAILABLE and the Git remote/HTTP service was not responding when CI was attempted, so the mandatory P02 focused CI was **not dispatched**. Phase remains BLOCKED pending actual CI evidence; no runtime change was pushed to production/main.
## P02-FINAL-ACCEPTANCE-01 Current Verification (latest record)
- **At this checkpoint: P02 acceptance PENDING CI #69**. P01 CLOSED; P03 NOT STARTED. Do not reinterpret earlier P02 BLOCKED audit as a completed release.
- Gitea #66 on `76cd972bdeea9ef07b4b38e9d8b5b5ed3312ca5b` **COMPLETED SUCCESS**: node-regression 24/24 suites PASS including direct Port4000 replay; quality (strict tsc+ESLint) PASS; production-build PASS; windows-compatibility SDK PASS; Python/Docker skipped by scoped `p01_only` input.
- Final patch `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532` in clean isolated `p02-final-acceptance-20261122`: resolved source-grounded negative-marker TTL bug and mixed-batch success ACK, plus test-only stale-claim time jump (+61s), failed Redis claim, mixed batch, intentionally Redis-free compatibility, rotated .csv.gz+multiline replay and partial-corrupt CSV fail-closed/recovery. No source outside `server/socket/handlers.js` changed since candidate.
- One focused Gitea CI #69 dispatched for final patch; job terminal outcome and any corrective action must be recorded. **Do not merge until this passes and exact patch/security audit is reviewed.**
- Original `D:\VibTools_Workspace\01_Products\14_DataStack` at old `d8ac96e` with 16 staged P01 changes untouched. Separate pre-existing `p02-runtime-remediation-20261122` had three dirty files; left untouched. The new isolated worktree was created clean from candidate SHA.
- Source behavior: no Redis configured = historical RAM best-effort acceptance preserved; configured Redis failure = negative ACK; public /health unchanged. Per-CSV replay scanning bounded-memory but linear in stored bytes; no global exactly-once/throughput guarantee. P03 cannot begin until P02 acceptance/remote parity close.
### P02 Focused Retry — #69 fixture failure / #72 correction
- Final acceptance CI #69 on `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532`: node-regression **FAIL** because `phase02-sync` stale-marker test globally mocked `Date.now` and Socket.IO ACK timed out. Other new tests for gzip/rotated CSV, partial CSV corruption, Redis-free RAM and mixed batch with a new persisted event PASS. Strict TypeScript, lint and Windows SDK jobs PASS; full workflow conclusion must be read at terminal.
- New source-grounded all-duplicate mixed batch path (`processed=0`, `duplicates=1`, `rejected>0`, volatile claim present) could return overall success; added focused test and changed handler conditional to reject all `volatileDuplicate` combinations.
- Test correction removes process-wide Date.now override, verifies 61000ms/300000ms cleanup math with static source marker assertion, and checks real Socket.IO negative retries. No unrelated files, CI workflows, SDK, auth or public health modified.
- Commit `acca8be01a31f8d3972564cbaec8fd3117a0b551` submitted to disposable CI #72; **P02 remains pending / NOT CLOSED** until terminal acceptance and 3-way mirror parity.
## P02 Verified Acceptance and Conditional Publication — 2026-11-22
- **Code acceptance VERIFIED — Gitea CI #72 terminal SUCCESS.** Code SHA `acca8be01a31f8d3972564cbaec8fd3117a0b551`. Node regression exactly `CI_SUITE_SUMMARY total=24 failed=0`, individual `tests/phase02-sync.test.ts` and `tests/phase03-storage-buffer.test.ts` suites exit 0. Quality strict TS+ESLint, Next standalone build and Windows Node SDK jobs all SUCCESS. Python/Docker intentionally skipped in this focused P02 workflow.
- **Root causes fixed:** configured Redis RPUSH outage returns `BUFFER_PERSIST_FAILED` negative ACK rather than volatile RAM success; stale failed idempotency claim retains correct TTL and remains negative through retries; `batch_sync` with old durable duplicates plus failed volatile claim returns `success:false`; CSV-before-Redis-LTRIM replay suppresses previously written same-client/date/eventId across plain/rotated/gz CSV with multiline quoted payload; corrupt matching CSV prevents Redis trim and can be recovered after disposable repair.
- **Explicit limits:** Redis intentionally unconfigured retains historical best-effort RAM ACK; no globally atomic exactly-once promise; scan I/O proportional to matching partition bytes even though memory streaming; public /health continues liveness not dependency readiness; live production unknown. No deployment, migration, secret access, user-data mutation, tag, release or build of production images.
- **Historical testing:** #66 PASS candidate; #69 suite failure (Socket.IO ACK timeout caused by test-only global `Date.now` override) and cancellation; #72 PASS corrected source/test. No false claim that #69 succeeded.
- Final code/diff safety audit: 11/11 approved paths only: `server/index.js`, `server/socket/handlers.js`, `core/data/buffer.ts`, `tests/phase02-sync.test.ts`, `tests/phase03-storage-buffer.test.ts`, plus `notes/001–006`; `git diff --check` and JS syntax PASS. Original PC root old staged P01 (16 paths) and separate dirty earlier temporary worktree preserved.
- **Publication gate:** Gitea PR #3 final verified source + documentation notes; squash-merge with `[skip ci]`, publish to GitHub and GitLab, verify identical tracked Git trees despite potentially different GitLab commit SHA. P02 marked CLOSED only after three-remote content parity; evidence SHA/tree in operator's final publication report. P03 NOT STARTED and requires separate approval.
## P03 final CI acceptance ledger — 2026-11-22
| Gate | Terminal outcome |
| --- | --- |
| #77 Node | SUCCESS, 24/24 |
| #77 TypeScript / ESLint | SUCCESS |
| #77 Next standalone build | SUCCESS |
| #77 Windows Node SDK | SUCCESS |
| #77 isolated three-image BuildKit | SUCCESS |
| #77 read-only npm audit inventory | SUCCESS with 1 high runtime finding |
| #77 Python | FAILED before tests: checkout requires Node |
| #78 corrected Python-only | SUCCESS, 14/14; other jobs SKIPPED |
**CI-08 security disposition:** `npm audit --json` full=26 affected packages (25 moderate, 1 high); runtime=24 (23 moderate, 1 high), **not counts of distinct advisories**. High-severity advisories [GHSA-6g55-p6wh-862q](https://github.com/advisories/GHSA-6g55-p6wh-862q), [GHSA-r28c-9q8g-f849](https://github.com/advisories/GHSA-r28c-9q8g-f849) affect Next 15.5.29's nested `postcss@8.4.31` despite patched root `postcss@8.5.33`; npm recommends major `next@16.4.6`. Direct `cloudcmd@19.24.0` carries moderate transitive issues (`dompurify`, `file-type`, `glob`, `minimatch`, `got`); npm recommends `cloudcmd@17.0.2` downgrade, not a verified safe pin. No speculative package pin, major upgrade or force fix; a separately tested compatible remediation is needed before release.
**Scope / release decision:** only `.gitea/workflows/datastack-ci.yml`, `.gitea/workflows/datastack-docker-ci.yml`, and `notes/001–006` changed. No `package.json`/lock, Dockerfile, application UI/UX, runtime, auth, SDK, database, Compose, production data, secret, shared Docker prune, deployment, migration, tag or release. P03 technical CI and advisory-triage acceptance verified by #77 non-Python jobs plus #78 Python; do **not** mislabel #77 overall as successful. Final phase publication requires one main commit and Gitea/GitHub/GitLab exact tracked tree parity. **Production release NO-GO** while runtime high advisory is unresolved; live production health, throughput and deployment remain unverified.
## P04-S01 security remediation kickoff — 2026-11-22
**Status: IN PROGRESS / NOT ACCEPTED.** User-approved baseline `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3`. Scope `package.json`, npm-generated `package-lock.json`, existing Gitea CI `security-audit` job, `notes/001–006`. Root cause `node_modules/next` locks `postcss@8.4.31` while root is `8.5.33`; P03 runtime audit 1 high and 23 moderate package entries. Plan minimal nested npm override, lockfile-only regeneration, high/critical audit gate; preserve all runtime/UI/API/auth/SDK/DB/Compose/production. One focused CI acceptance and exact 3-remote tree equality required; DO NOT label closed before terminal results, publish or claim release-ready.
## P04-S01 blocked security reconciliation report
P04-S01 SEC-01 BLOCKED — 2026-11-22. Approved scoped `overrides.next.postcss=8.5.33` trialed in disposable Windows npm 10.9.8 workspace only. `npm install --package-lock-only` EXIT 0, but `npm ls --package-lock-only postcss --all` EXIT 1 (`ELSPROBLEMS`: Next nested `postcss@8.4.31` invalid; expected 8.5.33); npm dedupe also kept invalid dependency. Read-only `npm audit --omit=dev --json` EXIT 1, runtime high 1/moderate 23/critical 0. Lockfile regeneration caused unrelated churn (91 package entries added, 106 removed, 69 changed, including two lightningcss optional 1.33.0 -> 1.30.1). Fresh minimal Next 15.5.29 fixture with identical override resolves PostCSS 8.5.33, isolating failure to DataStack committed lock graph. `npm ci --dry-run` exit 0 does not prove real clean install. Exact stop rule applied: no dependency patch, no CI acceptance, no main commit, no GitHub/GitLab sync, no release/deploy; production NO-GO. Need separate explicit narrowly scoped lock reconciliation approval.
Candidate branch `p04-s01-postcss-security-20261122` contains notes only. Gitea main stays `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3`; no Gitea CI was dispatched and no phase acceptance established.
## P04-S01B run ledger — 2026-11-22
**P04-S01B = BLOCKED / stop rule.** Disposable Windows npm 10.9.8 only (not user project workspace). Verified baseline Gitea `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3` and candidate notes branch `4abcc55fee927b5ae2c1193cea2b4585b8d00158` before test. With approved stale node deletion in disposable lock plus scoped manifest override, `npm install --package-lock-only` 0, `npm ls --package-lock-only postcss --all --depth=4` 0, audit high 0 / critical 0 / moderate 22. Next 15.5.29, Cloudcmd 19.24.0 and every unrelated package version unchanged; package entries added 0, removed 1 (stale `node_modules/next/node_modules/postcss`), existing entries with changed metadata 67 (36 dev, 29 libc, 2 dependency ordering overlaps). **Reject generated candidate due preservation of unrelated metadata requirement.** No real npm ci, regression, Docker, focused Gitea CI, security gate edit, application changes, main commit, GitHub/GitLab mirror or release. P03 CLOSED, P04-S01B NOT CLOSED, published production still NO-GO.
## P04-S01C verified six-job acceptance checkpoint — 2026-11-22
- User-approved baseline Gitea main `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3` still unchanged; candidate source commit `c759be990c032c2474d2d0bad519c643dec11a3a`, PR #5 draft pending acceptance.
- Exact source diff: only `package.json` scoped override, `package-lock.json` delete stale nested PostCSS entry (one package), existing `.gitea/workflows/datastack-ci.yml` security-audit threshold. Updated notes six files only; no runtime/SDK/UI/test/Dockerfile/Compose source changes.
- [Gitea CI #85](https://dev.ivib.cc/vibtools/DataStack/actions/runs/85) results as of checkpoint: node-regression **SUCCESS** `CI_SUITE_SUMMARY total=24 failed=0` plus real Linux `npm ci`; python-regression **SUCCESS** `Ran 14 tests`, `OK`; quality tsc/ESLint **SUCCESS**; production Next standalone **SUCCESS**; Docker/BuildKit 3 images **SUCCESS** `DATASTACK_THREE_IMAGES_BUILD_PASS`; security-audit **SUCCESS** runtime high 0/critical 0, `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO`, full moderate 24/runtime moderate 22. Windows-compatibility **QUEUED** at checkpoint: no terminal proof; entire run must not be mislabelled success.
- **P04-S01C PENDING** terminal Windows acceptance, main squash commit, GitHub/GitLab parity. Production NO-GO; 22 moderate runtime package entries and live readiness remain unresolved.
## P04-S01C FINAL CI acceptance (7/7) — 2026-11-22
Gitea CI #85 https://dev.ivib.cc/vibtools/DataStack/actions/runs/85 COMPLETED/SUCCESS; tested implementation SHA c759be990c032c2474d2d0bad519c643dec11a3a; later changes to candidate are documentation only. All seven jobs terminal SUCCESS: node-regression (Linux committed-lock npm ci and exactly 24/24 TypeScript suites); python-regression (Ran 14 tests, OK); quality (strict tsc and ESLint); production-build (Next 15.5.29 standalone, 22/22 pages); docker-image-build (three isolated BuildKit ingestion/web/proxy images); windows-compatibility (Windows committed-lock npm ci and Node SDK build, job #182, terminal SUCCESS); security-audit (runtime moderate=22 high=0 critical=0; full moderate=24 high=0 critical=0; DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO). Single Gitea acceptance run used, no additional broad reruns.
Scope: exactly nine approved paths: package.json scoped next->postcss 8.5.33 override; package-lock.json removes only old nested postcss@8.4.31 entry; existing .gitea/workflows/datastack-ci.yml security-audit job high/critical rejection; notes/001–006 evidence. Other package metadata, versions, integrity and resolved URLs match baseline. Next.js 15.5.29 and Cloudcmd 19.24.0 preserved. Final candidate package-lock SHA256 e6b5b362877627085dc04c9ded03c019971e07ef9f4af1e78057d58961a1c091 (verified byte-for-byte with Gitea). No API, auth, SDK, UI, application runtime, database, Compose, Dockerfile, tests or production data changes, deployment, migration, prune, tag, release.
CI fully accepted; publication not yet asserted in these pre-merge notes. Final phase CLOSED requires one squash merge of PR #5 into Gitea main and verified identical tracked content across Gitea/GitHub/GitLab; record actual new SHAs in terminal report. Production release remains NO-GO pending P04-S02 moderate vulnerability triage and separately authorized live readiness verification.
## P04-S02 scoped candidate — 2026-11-22
Baseline Gitea main `2d467ff1494858164a37814f7dbc2b321f510291`, tree `648c76ca496d2c9bf22a16f558c9ba347a45d4f7`. Cloudcmd 19.24.0 -> deepword 11.0.1 -> monaco-editor 0.55.1 pins DOMPurify 3.2.7, vulnerable according to npm advisory inventory. Disposable Windows npm 10.9.8 `npm view dompurify@3.4.18` verified, scoped npm override `overrides.cloudcmd.dompurify=3.4.18` resolves `dompurify@3.4.18` via `npm ls --package-lock-only dompurify --all --depth=5` (exit 0). `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline` exit 0. npm graph initially changed 67 unrelated metadata entries; deterministically restored exact baseline values after validating unrelated versions, URLs, integrity and dependencies. Final lock has **one changed package entry**, DOMPurify; same-major replacement plus verified registry `resolved` and `integrity`, no other packages added/deleted/changed. Candidate runtime audit affected-package counts **moderate 20/high 0/critical 0** versus baseline **22/0/0**; full candidate **moderate 22/high 0/critical 0** versus baseline **24/0/0**. DOMPurify and Monaco Editor dropped from vulnerable-package inventory. Other moderate packages (got, glob, minimatch, nested file-type, Cloudcmd transitive chain) remain; they require major package changes under current advisory constraints, prohibited by Scope Lock. This is a **pre-acceptance candidate**: do not claim application compatibility, CI PASS, publication, or phase CLOSED before full focused Gitea CI and exact three-remote tree parity.
Phase ledger: P04-S02 candidate patch under test, not yet accepted. One focused CI to verify Linux/Windows npm ci, Cloudcmd File Manager existing tests, Node 24/24, Python 14/14, TS/lint, Next standalone and Docker 3 images.
## P04-S02 final CI acceptance — 2026-11-22
Gitea CI #92 (https://dev.ivib.cc/vibtools/DataStack/actions/runs/92) **COMPLETED / SUCCESS**, all 7/7 jobs terminal success on source commit `a88bc5f4498ea1b11acc0acfeb66b5aa9d6531be`: node-regression real Linux npm ci + 24/24 tests including `tests/files-manager.test.ts` exit 0; Python SDK 14/14; quality strict TypeScript/ESLint; Next.js standalone (22/22 static pages); isolated Docker/BuildKit 3-image verification `DATASTACK_THREE_IMAGES_BUILD_PASS`; Windows committed-lock npm ci + Node SDK compatibility; security-audit `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO` and `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Audit full=22 moderate/high=0/critical=0 (baseline 24), runtime=20 moderate/high=0/critical=0 (baseline 22). These are affected package entries, not distinct advisories. No new CI workflow or extra broad CI dispatched.
Exact approved patch: `package.json` scoped `overrides.cloudcmd.dompurify=3.4.18`, preserving `overrides.next.postcss=8.5.33`; npm-generated `package-lock.json` changes solely `node_modules/dompurify` version 3.2.7 -> 3.4.18 and verified `resolved`/SHA512 integrity. Baseline Cloudcmd 19.24.0, Monaco Editor 0.55.1, Next.js 15.5.29, all unrelated locked package versions, URL, hash, dependency and metadata preserved. Gitea published candidate lock bytes match disposable SHA-256 `57c5ea9df9f28be496f8c3202fbc280b0066bf8c216b8d8a3ca98b6f7b942854`. Exact candidate tree diff 420 entries, 8 approved changed files only (`package.json`, `package-lock.json`, notes/001–006), 0 unauthorized paths. No runtime/UI/auth/API/SDK/tests/CI/Dockerfiles/Compose/db/schema/data modifications.
Phase ledger: P04-S02 fully CI-accepted, publication gate pending. **Residual risk:** 20 moderate runtime affected package entries remain, including Cloudcmd transitive dependencies `got`, `glob`, `minimatch`, nested `file-type` and their parent chains. Current npm audit suggests `cloudcmd@17.0.2` (a major downgrade), explicitly out-of-scope; other transitive fixes need major-version dependency-contract changes. No force/downgrade/unverified override. Security improvement verified but NOT zero vulnerabilities. Production release stays NO-GO; remaining advisory disposition and live readiness need independent scope approval.
**Merge gate:** CI #92 7/7 PASS, approved 8-file patch integrity PASS; main squash merge and Gitea/GitHub/GitLab exact Git content-tree verification required before P04-S02 CLOSED. This note is written before that publication, so final remote commit SHAs shall be appended to the PR closure report after verification. No deployment, migrations, prune, tag, release.
+83
View File
@@ -64,3 +64,86 @@ The previous #58 package-lock error was resolved by adding the exact nested opti
| R-03 availability | Redis `llen` rejection leaves Express `/health` online with `redisConnected:false`. Recovery flips the flag while HTTP 200 remains unchanged. | Public 200 is liveness by existing contract; a protected dependency readiness probe/503 is **not implemented** and needs explicit approval. | | R-03 availability | Redis `llen` rejection leaves Express `/health` online with `redisConnected:false`. Recovery flips the flag while HTTP 200 remains unchanged. | Public 200 is liveness by existing contract; a protected dependency readiness probe/503 is **not implemented** and needs explicit approval. |
Security invariants: do not expose secrets in ACK/errors, keep JWT/auth and 401/403/429/503/507 contracts unchanged, and never run failure-injection against actual Redis, Postgres, production API, live events, or user CSV. Disposable fake Redis and temporary files only. CI #63: https://dev.ivib.cc/vibtools/DataStack/actions/runs/63; results tracked in notes/003. Security invariants: do not expose secrets in ACK/errors, keep JWT/auth and 401/403/429/503/507 contracts unchanged, and never run failure-injection against actual Redis, Postgres, production API, live events, or user CSV. Disposable fake Redis and temporary files only. CI #63: https://dev.ivib.cc/vibtools/DataStack/actions/runs/63; results tracked in notes/003.
## P02 Runtime Candidate — Error-path change inventory (pending CI)
| Path | Candidate result | Explicit limitation |
|---|---|---|
| Redis `RPUSH` rejects with configured Redis client | Retains event in volatile RAM as preexisting fallback but returns `BUFFER_PERSIST_FAILED` to Socket.IO via thrown error; **no successful ACK** for that failure; error text does not expose connection details | No-Redis intentional legacy RAM mode unchanged, still not a durable guarantee |
| `sync_data` and `batch_sync` failures | `success:false`, `code:BUFFER_PERSIST_FAILED` and best-effort rollback of previously claimed idempotency keys, negative failure marker for in-process retries | If claim rollback cannot reach Redis and process crashes, durable claim/queue atomicity remains unsolved |
| Redis buffer `LTRIM` rejects after successful CSV append | Subsequent flush checks physically stored CSV IDs and avoids appending matching rows again; Redis is trimmed only after successful check/write | Concurrent independent CSV appenders, compressed archived CSV, corrupt partial rows, and per-flush CSV scan cost require separate proof |
| Legacy public health | **UNCHANGED** HTTP 200 liveness contract and Redis-connected diagnostic field; R-03 readiness is not introduced in this approval | Does not prove full DB/Redis transactional readiness |
Security/API invariants: leave token/JWT/auth/rate limits/quotas/webhooks/MCP/sandbox/UI/SDK unchanged. No production Redis/Postgres/API/CSV mutation. No lockfile/dependency versions or CI workflow changes. CI acceptance remains pending; these are code-level observations, not release certification.
## P02 Final Fault / Recovery Matrix
| Case | Final behavior target and evidence |
|---|---|
| Redis RPUSH rejects with Redis configured | `BUFFER_PERSIST_FAILED` negative ACK, no false durable success; candidate CI #66 verified single/batch, final CI #69 for marker TTL |
| Redis idempotency claim survives failed queue write | Positive-time separate failed marker survives cleanup after +61000ms (below five-minute TTL); two retries remain negative rather than `DUPLICATE_DROPPED` success. Final disposable fixture `tests/phase02-sync.test.ts` |
| Mixed `batch_sync` contains stale claim and a new durable record | Overall response `success:false` with `BUFFER_PERSIST_FAILED`, even if some new records committed; preserve counts and avoid success cover-up |
| Redis deliberately disabled | Historical RAM mode unchanged; distinguish this existing best-effort contract from configured Redis durability |
| CSV append success but Redis LTRIM failure | Replay scans written event IDs and does not duplicate physical row before trim; direct Port4000 and web-buffer regression |
| Rotated / gzip CSV, quoted multiline payload | Streaming CSV parser, exact client/date/eventId match, no duplicate append or data-format change |
| Matching CSV partial/corrupt | Fail closed (no Redis trim); disposable repair allows retry; do not touch real user CSV |
| Public /health & 401/403/429/503/507 | Preserve existing behavior; no new endpoint or error contract beyond already approved BUFFER_PERSIST_FAILED |
**Limitations:** no globally atomic Redis+CSV transaction, stale Redis claim after process crash may require TTL/retry policy, and per-partition scan has data-size-proportional I/O even though memory is streamed. No live production reliability or throughput certification. CI #66 SUCCESS; final CI #69 pending.
### Additional P02 final-acceptance negative ACK case
If a `batch_sync` contains only (a) an uncommitted Redis-claim duplicate and (b) an ordinary already-durable duplicate, the old `processed===0 && duplicates===0 && rejected>0` test did not trigger; success ACK incorrectly masked volatile claim. Narrow correction uses `if (volatileDuplicate)` before the generic aggregate success path, independent of `processed` or `duplicates`. Specific test in `tests/phase02-sync.test.ts` injects Redis RPUSH failure and fake stale claim then includes the existing durable duplicate. All pending-claim ACKs must remain `BUFFER_PERSIST_FAILED`, even if some other events already committed. Terminal CI #72 pending; no global 5-minute clock advancement inside sockets.
### P02 terminal CI #72 error handling acceptance
Gitea #72 **SUCCESS** at verified final code SHA `acca8be01a31f8d3972564cbaec8fd3117a0b551`; all 24 TS suites exit zero including stale claim, mixed batch (both with new durable event and old durable duplicate), Redis-free compatibility, gzip/rotated CSV and corrupt matching CSV fail-closed. `BUFFER_PERSIST_FAILED` is negative-ACK response on configured Redis write failure, rather than success for RAM fallback. Redis-free RAM mode old semantics unchanged. Existing 401/403/429/503/507 tests unaffected and pass within 24 suites. Public /health liveness remains HTTP 200 with `redisConnected:false` during mocked outage; separate readiness endpoint not approved. Remaining risk: no atomic Redis+CSV commit, no global exactly-once guarantee, CSV partition scan disk I/O grows with volume. No live traffic test.
## P03 runner/security defect register — 2026-11-22
- **CI-06:** `python:3.12-slim-bookworm` lacked Node for `actions/checkout@v4` (127). Approved workflow-only correction: `node:22-bookworm-slim` plus Debian Python 3.11 venv and prerequisites; narrow CI #78 **14/14 PASS**.
- **CI-07:** Historical classic Docker `COPY node_modules/@uiw` layer-export failure was not reproduced using disposable BuildKit builder; #77 Compose and 3/3 builds plus image inspect **PASS**; no Dockerfile modifications or shared prune.
- **CI-08:** **CI-08 security disposition:** `npm audit --json` full=26 affected packages (25 moderate, 1 high); runtime=24 (23 moderate, 1 high), **not counts of distinct advisories**. High-severity advisories [GHSA-6g55-p6wh-862q](https://github.com/advisories/GHSA-6g55-p6wh-862q), [GHSA-r28c-9q8g-f849](https://github.com/advisories/GHSA-r28c-9q8g-f849) affect Next 15.5.29's nested `postcss@8.4.31` despite patched root `postcss@8.5.33`; npm recommends major `next@16.4.6`. Direct `cloudcmd@19.24.0` carries moderate transitive issues (`dompurify`, `file-type`, `glob`, `minimatch`, `got`); npm recommends `cloudcmd@17.0.2` downgrade, not a verified safe pin. No speculative package pin, major upgrade or force fix; a separately tested compatible remediation is needed before release.
- **Runtime API errors:** P02's 401/403/429/503/507, `BUFFER_PERSIST_FAILED` negative ACK, Redis/CSV replay behavior and public `/health` shape unchanged; no live production fault claims.
**Scope / release decision:** only `.gitea/workflows/datastack-ci.yml`, `.gitea/workflows/datastack-docker-ci.yml`, and `notes/001–006` changed. No `package.json`/lock, Dockerfile, application UI/UX, runtime, auth, SDK, database, Compose, production data, secret, shared Docker prune, deployment, migration, tag or release. P03 technical CI and advisory-triage acceptance verified by #77 non-Python jobs plus #78 Python; do **not** mislabel #77 overall as successful. Final phase publication requires one main commit and Gitea/GitHub/GitLab exact tracked tree parity. **Production release NO-GO** while runtime high advisory is unresolved; live production health, throughput and deployment remain unverified.
## P04-S01 blocked security reconciliation report
P04-S01 SEC-01 BLOCKED — 2026-11-22. Approved scoped `overrides.next.postcss=8.5.33` trialed in disposable Windows npm 10.9.8 workspace only. `npm install --package-lock-only` EXIT 0, but `npm ls --package-lock-only postcss --all` EXIT 1 (`ELSPROBLEMS`: Next nested `postcss@8.4.31` invalid; expected 8.5.33); npm dedupe also kept invalid dependency. Read-only `npm audit --omit=dev --json` EXIT 1, runtime high 1/moderate 23/critical 0. Lockfile regeneration caused unrelated churn (91 package entries added, 106 removed, 69 changed, including two lightningcss optional 1.33.0 -> 1.30.1). Fresh minimal Next 15.5.29 fixture with identical override resolves PostCSS 8.5.33, isolating failure to DataStack committed lock graph. `npm ci --dry-run` exit 0 does not prove real clean install. Exact stop rule applied: no dependency patch, no CI acceptance, no main commit, no GitHub/GitLab sync, no release/deploy; production NO-GO. Need separate explicit narrowly scoped lock reconciliation approval.
Do not treat successful npm install/dedupe/dry-run as security or compatibility validation. High PostCSS remains reachable through lock graph, no runtime handlers or error codes touched.
## P04-S01B exact failed acceptance condition — 2026-11-22
PostCSS resolution fixed within disposable test copy, with audit high/critical 0, but lockfile metadata integrity requirement **FAIL**: npm wrote 67 unrelated package-entry metadata differences. No API error/ACK behavior touched. Existing peer dependency warning for `@putout/eslint@6.1.1` versus ESLint 9 is non-blocking in the isolated lock generation and not permission to upgrade it. Stop before acceptance CI or source edit; production/main lock and security-audit workflow remain unchanged.
## P04-S01C security gate and error contract — 2026-11-22
SEC-01 candidate remediation verified: Next 15.5.29/Cloudcmd 19.24.0 unchanged, PostCSS 8.5.33 deduped, nested vulnerable PostCSS removed without unrelated dependency integrity/metadata drift. Focused CI #85 read-only audit: full affected packages moderate=24/high=0/critical=0; runtime moderate=22/high=0/critical=0, and `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO` succeeded. New gate fails on incomplete severity counts and any runtime high/critical. Moderate Cloudcmd transitive issues intentionally remain P04-S02. No JWT/auth, HTTP/socket error codes, Redis durability, CSV replay, SDK/API/UI or production health changes; Windows CI still queued and publication pending.
## P04-S01C FINAL CI acceptance (7/7) — 2026-11-22
Gitea CI #85 https://dev.ivib.cc/vibtools/DataStack/actions/runs/85 COMPLETED/SUCCESS; tested implementation SHA c759be990c032c2474d2d0bad519c643dec11a3a; later changes to candidate are documentation only. All seven jobs terminal SUCCESS: node-regression (Linux committed-lock npm ci and exactly 24/24 TypeScript suites); python-regression (Ran 14 tests, OK); quality (strict tsc and ESLint); production-build (Next 15.5.29 standalone, 22/22 pages); docker-image-build (three isolated BuildKit ingestion/web/proxy images); windows-compatibility (Windows committed-lock npm ci and Node SDK build, job #182, terminal SUCCESS); security-audit (runtime moderate=22 high=0 critical=0; full moderate=24 high=0 critical=0; DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO). Single Gitea acceptance run used, no additional broad reruns.
Security improvement: remove Next nested vulnerable PostCSS and enforce runtime high/critical fail gate. Existing 22 moderate runtime affected-package entries remain, not a security-clean claim for all severities. No change to runtime error codes/auth/API contracts. CI fully accepted; publication not yet asserted in these pre-merge notes. Final phase CLOSED requires one squash merge of PR #5 into Gitea main and verified identical tracked content across Gitea/GitHub/GitLab; record actual new SHAs in terminal report. Production release remains NO-GO pending P04-S02 moderate vulnerability triage and separately authorized live readiness verification.
## P04-S02 scoped candidate — 2026-11-22
Baseline Gitea main `2d467ff1494858164a37814f7dbc2b321f510291`, tree `648c76ca496d2c9bf22a16f558c9ba347a45d4f7`. Cloudcmd 19.24.0 -> deepword 11.0.1 -> monaco-editor 0.55.1 pins DOMPurify 3.2.7, vulnerable according to npm advisory inventory. Disposable Windows npm 10.9.8 `npm view dompurify@3.4.18` verified, scoped npm override `overrides.cloudcmd.dompurify=3.4.18` resolves `dompurify@3.4.18` via `npm ls --package-lock-only dompurify --all --depth=5` (exit 0). `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline` exit 0. npm graph initially changed 67 unrelated metadata entries; deterministically restored exact baseline values after validating unrelated versions, URLs, integrity and dependencies. Final lock has **one changed package entry**, DOMPurify; same-major replacement plus verified registry `resolved` and `integrity`, no other packages added/deleted/changed. Candidate runtime audit affected-package counts **moderate 20/high 0/critical 0** versus baseline **22/0/0**; full candidate **moderate 22/high 0/critical 0** versus baseline **24/0/0**. DOMPurify and Monaco Editor dropped from vulnerable-package inventory. Other moderate packages (got, glob, minimatch, nested file-type, Cloudcmd transitive chain) remain; they require major package changes under current advisory constraints, prohibited by Scope Lock. This is a **pre-acceptance candidate**: do not claim application compatibility, CI PASS, publication, or phase CLOSED before full focused Gitea CI and exact three-remote tree parity.
Error contract: no user-facing error codes or handler changes. Security audit improvement from 22 to 20 moderate runtime affected packages, high/critical stays zero in disposable audit. No claim of eliminating all moderate advisories.
## P04-S02 final CI acceptance — 2026-11-22
Gitea CI #92 (https://dev.ivib.cc/vibtools/DataStack/actions/runs/92) **COMPLETED / SUCCESS**, all 7/7 jobs terminal success on source commit `a88bc5f4498ea1b11acc0acfeb66b5aa9d6531be`: node-regression real Linux npm ci + 24/24 tests including `tests/files-manager.test.ts` exit 0; Python SDK 14/14; quality strict TypeScript/ESLint; Next.js standalone (22/22 static pages); isolated Docker/BuildKit 3-image verification `DATASTACK_THREE_IMAGES_BUILD_PASS`; Windows committed-lock npm ci + Node SDK compatibility; security-audit `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO` and `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Audit full=22 moderate/high=0/critical=0 (baseline 24), runtime=20 moderate/high=0/critical=0 (baseline 22). These are affected package entries, not distinct advisories. No new CI workflow or extra broad CI dispatched.
Exact approved patch: `package.json` scoped `overrides.cloudcmd.dompurify=3.4.18`, preserving `overrides.next.postcss=8.5.33`; npm-generated `package-lock.json` changes solely `node_modules/dompurify` version 3.2.7 -> 3.4.18 and verified `resolved`/SHA512 integrity. Baseline Cloudcmd 19.24.0, Monaco Editor 0.55.1, Next.js 15.5.29, all unrelated locked package versions, URL, hash, dependency and metadata preserved. Gitea published candidate lock bytes match disposable SHA-256 `57c5ea9df9f28be496f8c3202fbc280b0066bf8c216b8d8a3ca98b6f7b942854`. Exact candidate tree diff 420 entries, 8 approved changed files only (`package.json`, `package-lock.json`, notes/001–006), 0 unauthorized paths. No runtime/UI/auth/API/SDK/tests/CI/Dockerfiles/Compose/db/schema/data modifications.
Error/security semantics: no API error handler, permission, middleware, auth or HTTP/WebSocket changes. Upgrade only same-major DOMPurify. **Residual risk:** 20 moderate runtime affected package entries remain, including Cloudcmd transitive dependencies `got`, `glob`, `minimatch`, nested `file-type` and their parent chains. Current npm audit suggests `cloudcmd@17.0.2` (a major downgrade), explicitly out-of-scope; other transitive fixes need major-version dependency-contract changes. No force/downgrade/unverified override. Security improvement verified but NOT zero vulnerabilities. Production release stays NO-GO; remaining advisory disposition and live readiness need independent scope approval.
**Merge gate:** CI #92 7/7 PASS, approved 8-file patch integrity PASS; main squash merge and Gitea/GitHub/GitLab exact Git content-tree verification required before P04-S02 CLOSED. This note is written before that publication, so final remote commit SHAs shall be appended to the PR closure report after verification. No deployment, migrations, prune, tag, release.
+70
View File
@@ -52,3 +52,73 @@ Do not mark "actual working" simply because a patch compiled. Distinguish source
- R-02: existing CSV-before-Redis-trim order audited, *not corrected*. Exactly-once replay safety is **NOT VERIFIED**. - R-02: existing CSV-before-Redis-trim order audited, *not corrected*. Exactly-once replay safety is **NOT VERIFIED**.
- R-03: current public Express health 200 plus `redisConnected:false` is liveness, not dependency readiness. Separate readiness remains **NOT IMPLEMENTED**. - R-03: current public Express health 200 plus `redisConnected:false` is liveness, not dependency readiness. Separate readiness remains **NOT IMPLEMENTED**.
- Gitea disposable CI #63: https://dev.ivib.cc/vibtools/DataStack/actions/runs/63. The previous P01 working results remain valid; no live read/write test or production deploy is included in P02. - Gitea disposable CI #63: https://dev.ivib.cc/vibtools/DataStack/actions/runs/63. The previous P01 working results remain valid; no live read/write test or production deploy is included in P02.
## P02 Runtime Candidate Feature Delta — 2026-11-22 (NOT RELEASED)
- **Intended change:** failure cases on socket ingestion report `BUFFER_PERSIST_FAILED` instead of positive ACK when Redis RPUSH rejects; single and batch code paths tested by new disposable assertions. Ordinary success ACK, auth, limits and SDK public interfaces intended unchanged.
- **Intended change:** Redis replay flush on both ingestion/Next paths inspects existing client CSV partitions and filters already-stored event IDs before reappend; safe failure leaves Redis queue untrimmed. This prevents the tested post-append/LTRIM replay duplication but does **not** prove transactionally exactly-once delivery or zero performance impact.
- **No changes:** application UI/UX, auth, SDK, database schema, Docker, environment, public R-03 health, other features and production runtime. No deployment or live verification.
- **Status:** P02 code candidate locally committed, acceptance CI blocked by Gitea unavailability; **NOT CLOSED**, not production-ready. P01 remains only completed phase.
## P02 Final Feature/Compatibility Register — 2026-11-22
Existing UI/UX, admin auth, SDK public signatures, health route, database schema, Docker and deployment: **UNCHANGED**. Only existing Socket.IO single/batch persistence failure ACK/idempotency cleanup, Redis->CSV replay duplicate filtering and disposable tests changed during P02. No new production feature, public endpoint, release or migration. CI #66 terminal SUCCESS (24/24 TS suites, strict types, ESLint, Windows Node SDK, Next production build); final CI #69 pending. Client-facing `BUFFER_PERSIST_FAILED` error is used only for previously incorrectly acknowledged Redis-backed failure; intentionally Redis-free RAM mode is preserved. CSV write content/format unchanged, gzip and rotation replay acceptance covered in final test fixture. All live status values are **NOT VERIFIED / NO DEPLOYMENT**. No guarantee of global exactly-once delivery or hard bound on disk scans; future throughput benchmark/design approval if required. P03 still gated on final P02 close.
### P02 final acceptance correction checkpoint
Gitea #69 node-regression observed one **fixture timeout**, not a proven new production feature defect, caused by altering global `Date.now`. A separate source-proven mixed batch all-duplicate ACK bug was corrected only in `server/socket/handlers.js`, test case added in existing allowed TS file. Revised `acca8be01a31f8d3972564cbaec8fd3117a0b551` pending focused Gitea CI #72. Existing UI/SDK/auth/public health remain unchanged. This is not evidence that production has been deployed or live-tested.
### P02 verified feature-state reconciliation — Gitea #72 SUCCESS
- **P02 R-01: FIXED in source and CI-PASS, conditional scope.** Configured Redis queue RPUSH rejects => `BUFFER_PERSIST_FAILED` negative ACK for single/mixed batches, stale Redis idempotency claim cannot be reported as durable success; clock arithmetic and actual negative ACK retries verified. Intentionally Redis-free RAM mode historical best-effort ACK **preserved**, not a lossless guarantee.
- **P02 R-02: FIXED for verified crash/replay windows**, not global transactionality. Port4000 and Port3000 CSV replay suppresses duplicate same-client/day/eventId across plain, rotated and gz; failed reads prevent trimming; dedicated regression tests PASS. Streaming parser bounded memory, disk read throughput/latency not yet load-tested.
- **P02 R-03: UNCHANGED**: public /health remains backwards-compatible liveness, not a dependency-readiness endpoint. Production live health **NOT VERIFIED**.
- All other runtime features/UI/auth/API/SDK/database/Compose and package dependencies unchanged. **P02 CI ACCEPTED at #72**; code SHA `acca8be01a31f8d3972564cbaec8fd3117a0b551`. Phase closed only after verified 3-way content sync. No P03 implementation or production deployment.
## P03 feature invariance — 2026-11-22
P03 modifies workflow verification and Markdown evidence only, not a product feature. No UI, API, auth, SDK, DB, CSV, Redis, Dockerfile, Compose or deployment change. The P02 regression contract remains 24/24 PASS; corrected Python SDK passes 14/14; all three Docker images build without running them. Read-only npm advisory triage is complete but **not security-clean**.
**CI-08 security disposition:** `npm audit --json` full=26 affected packages (25 moderate, 1 high); runtime=24 (23 moderate, 1 high), **not counts of distinct advisories**. High-severity advisories [GHSA-6g55-p6wh-862q](https://github.com/advisories/GHSA-6g55-p6wh-862q), [GHSA-r28c-9q8g-f849](https://github.com/advisories/GHSA-r28c-9q8g-f849) affect Next 15.5.29's nested `postcss@8.4.31` despite patched root `postcss@8.5.33`; npm recommends major `next@16.4.6`. Direct `cloudcmd@19.24.0` carries moderate transitive issues (`dompurify`, `file-type`, `glob`, `minimatch`, `got`); npm recommends `cloudcmd@17.0.2` downgrade, not a verified safe pin. No speculative package pin, major upgrade or force fix; a separately tested compatible remediation is needed before release.
**Scope / release decision:** only `.gitea/workflows/datastack-ci.yml`, `.gitea/workflows/datastack-docker-ci.yml`, and `notes/001–006` changed. No `package.json`/lock, Dockerfile, application UI/UX, runtime, auth, SDK, database, Compose, production data, secret, shared Docker prune, deployment, migration, tag or release. P03 technical CI and advisory-triage acceptance verified by #77 non-Python jobs plus #78 Python; do **not** mislabel #77 overall as successful. Final phase publication requires one main commit and Gitea/GitHub/GitLab exact tracked tree parity. **Production release NO-GO** while runtime high advisory is unresolved; live production health, throughput and deployment remain unverified.
## P04-S01 blocked security reconciliation report
P04-S01 SEC-01 BLOCKED — 2026-11-22. Approved scoped `overrides.next.postcss=8.5.33` trialed in disposable Windows npm 10.9.8 workspace only. `npm install --package-lock-only` EXIT 0, but `npm ls --package-lock-only postcss --all` EXIT 1 (`ELSPROBLEMS`: Next nested `postcss@8.4.31` invalid; expected 8.5.33); npm dedupe also kept invalid dependency. Read-only `npm audit --omit=dev --json` EXIT 1, runtime high 1/moderate 23/critical 0. Lockfile regeneration caused unrelated churn (91 package entries added, 106 removed, 69 changed, including two lightningcss optional 1.33.0 -> 1.30.1). Fresh minimal Next 15.5.29 fixture with identical override resolves PostCSS 8.5.33, isolating failure to DataStack committed lock graph. `npm ci --dry-run` exit 0 does not prove real clean install. Exact stop rule applied: no dependency patch, no CI acceptance, no main commit, no GitHub/GitLab sync, no release/deploy; production NO-GO. Need separate explicit narrowly scoped lock reconciliation approval.
No change to UI, API, SDK, auth, file manager, CSV, Redis, storage, feature set, Docker, deployment or production data. P03 baseline regressions preserved without claiming retest.
## P04-S01B no-feature-change / gate state — 2026-11-22
Candidate scope proof: PostCSS root 8.5.33 can satisfy Next 15.5.29 when stale nested 8.4.31 removed in a disposable lock, with moderate=22/high=0/critical=0 read-only runtime audit. Not published: regenerated metadata edits exceeded approved lock scope. Existing UI/UX, auth, API, SDK, CSV, Redis, Cloudcmd, Docker and production service behavior remain untouched; **do not claim regression acceptance or security remediation of published code**. Next approval must address exact npm-generated metadata drift and integrity preservation, not feature work.
## P04-S01C feature preservation and verification — 2026-11-22
No feature, UI, backend runtime, public API, auth, database/schema, SDK, CSV storage, Dockerfile or Compose source change. Package manifest scoped override and lockfile one-entry deletion only; Next 15.5.29 and Cloudcmd 19.24.0 preserved. Gitea CI #85 node 24/24, Python 14/14, strict types/lint, Next standalone and 3 Docker images completed SUCCESS. Security runtime audit high/critical 0; moderate advisories still pending separate triage. Windows compatibility QUEUED at checkpoint; no production release, deployment, migration or tag and no accepted main/mirror publication yet.
## P04-S01C FINAL CI acceptance (7/7) — 2026-11-22
Gitea CI #85 https://dev.ivib.cc/vibtools/DataStack/actions/runs/85 COMPLETED/SUCCESS; tested implementation SHA c759be990c032c2474d2d0bad519c643dec11a3a; later changes to candidate are documentation only. All seven jobs terminal SUCCESS: node-regression (Linux committed-lock npm ci and exactly 24/24 TypeScript suites); python-regression (Ran 14 tests, OK); quality (strict tsc and ESLint); production-build (Next 15.5.29 standalone, 22/22 pages); docker-image-build (three isolated BuildKit ingestion/web/proxy images); windows-compatibility (Windows committed-lock npm ci and Node SDK build, job #182, terminal SUCCESS); security-audit (runtime moderate=22 high=0 critical=0; full moderate=24 high=0 critical=0; DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO). Single Gitea acceptance run used, no additional broad reruns.
No product feature, UI, runtime, API, auth, SDK, data format, Docker or deployment change. Only resolved PostCSS dependency and CI security gate. CI fully accepted; publication not yet asserted in these pre-merge notes. Final phase CLOSED requires one squash merge of PR #5 into Gitea main and verified identical tracked content across Gitea/GitHub/GitLab; record actual new SHAs in terminal report. Production release remains NO-GO pending P04-S02 moderate vulnerability triage and separately authorized live readiness verification.
## P04-S02 scoped candidate — 2026-11-22
Baseline Gitea main `2d467ff1494858164a37814f7dbc2b321f510291`, tree `648c76ca496d2c9bf22a16f558c9ba347a45d4f7`. Cloudcmd 19.24.0 -> deepword 11.0.1 -> monaco-editor 0.55.1 pins DOMPurify 3.2.7, vulnerable according to npm advisory inventory. Disposable Windows npm 10.9.8 `npm view dompurify@3.4.18` verified, scoped npm override `overrides.cloudcmd.dompurify=3.4.18` resolves `dompurify@3.4.18` via `npm ls --package-lock-only dompurify --all --depth=5` (exit 0). `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline` exit 0. npm graph initially changed 67 unrelated metadata entries; deterministically restored exact baseline values after validating unrelated versions, URLs, integrity and dependencies. Final lock has **one changed package entry**, DOMPurify; same-major replacement plus verified registry `resolved` and `integrity`, no other packages added/deleted/changed. Candidate runtime audit affected-package counts **moderate 20/high 0/critical 0** versus baseline **22/0/0**; full candidate **moderate 22/high 0/critical 0** versus baseline **24/0/0**. DOMPurify and Monaco Editor dropped from vulnerable-package inventory. Other moderate packages (got, glob, minimatch, nested file-type, Cloudcmd transitive chain) remain; they require major package changes under current advisory constraints, prohibited by Scope Lock. This is a **pre-acceptance candidate**: do not claim application compatibility, CI PASS, publication, or phase CLOSED before full focused Gitea CI and exact three-remote tree parity.
Feature preservation: Cloudcmd major 19.24.0, Monaco Editor 0.55.1, Next 15.5.29, PostCSS override 8.5.33, runtime APIs/UI/auth/SDK/Redis/CSV/storage and Dockerfiles unchanged. Existing File Manager behavior must be verified by read-only tests.
## P04-S02 final CI acceptance — 2026-11-22
Gitea CI #92 (https://dev.ivib.cc/vibtools/DataStack/actions/runs/92) **COMPLETED / SUCCESS**, all 7/7 jobs terminal success on source commit `a88bc5f4498ea1b11acc0acfeb66b5aa9d6531be`: node-regression real Linux npm ci + 24/24 tests including `tests/files-manager.test.ts` exit 0; Python SDK 14/14; quality strict TypeScript/ESLint; Next.js standalone (22/22 static pages); isolated Docker/BuildKit 3-image verification `DATASTACK_THREE_IMAGES_BUILD_PASS`; Windows committed-lock npm ci + Node SDK compatibility; security-audit `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO` and `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Audit full=22 moderate/high=0/critical=0 (baseline 24), runtime=20 moderate/high=0/critical=0 (baseline 22). These are affected package entries, not distinct advisories. No new CI workflow or extra broad CI dispatched.
Exact approved patch: `package.json` scoped `overrides.cloudcmd.dompurify=3.4.18`, preserving `overrides.next.postcss=8.5.33`; npm-generated `package-lock.json` changes solely `node_modules/dompurify` version 3.2.7 -> 3.4.18 and verified `resolved`/SHA512 integrity. Baseline Cloudcmd 19.24.0, Monaco Editor 0.55.1, Next.js 15.5.29, all unrelated locked package versions, URL, hash, dependency and metadata preserved. Gitea published candidate lock bytes match disposable SHA-256 `57c5ea9df9f28be496f8c3202fbc280b0066bf8c216b8d8a3ca98b6f7b942854`. Exact candidate tree diff 420 entries, 8 approved changed files only (`package.json`, `package-lock.json`, notes/001–006), 0 unauthorized paths. No runtime/UI/auth/API/SDK/tests/CI/Dockerfiles/Compose/db/schema/data modifications.
Feature preservation: File Manager test and Monaco-related Next production build PASS; no UI features removed, no editor behavior modified in source. **Residual risk:** 20 moderate runtime affected package entries remain, including Cloudcmd transitive dependencies `got`, `glob`, `minimatch`, nested `file-type` and their parent chains. Current npm audit suggests `cloudcmd@17.0.2` (a major downgrade), explicitly out-of-scope; other transitive fixes need major-version dependency-contract changes. No force/downgrade/unverified override. Security improvement verified but NOT zero vulnerabilities. Production release stays NO-GO; remaining advisory disposition and live readiness need independent scope approval.
**Merge gate:** CI #92 7/7 PASS, approved 8-file patch integrity PASS; main squash merge and Gitea/GitHub/GitLab exact Git content-tree verification required before P04-S02 CLOSED. This note is written before that publication, so final remote commit SHAs shall be appended to the PR closure report after verification. No deployment, migrations, prune, tag, release.
+108
View File
@@ -237,3 +237,111 @@ return bufferQueue.length;
- `P02 R-03 diagnostic: Redis query failure leaves public ingestion health 200/online` — **ok 16**, same suite exit 0, Redis flag false while liveness stays online. - `P02 R-03 diagnostic: Redis query failure leaves public ingestion health 200/online` — **ok 16**, same suite exit 0, Redis flag false while liveness stays online.
- `P02 R-02 diagnostic: post-CSV Redis trim failure replays identical event on retry` — **ok 18**, `phase03-storage-buffer` suite exit 0; duplicate append after retry reproduced on representative web buffer path. **Direct ingestion Port 4000 CSV worker crash coverage remains outstanding** and should be requested in remediation approval before guarantee changes. - `P02 R-02 diagnostic: post-CSV Redis trim failure replays identical event on retry` — **ok 18**, `phase03-storage-buffer` suite exit 0; duplicate append after retry reproduced on representative web buffer path. **Direct ingestion Port 4000 CSV worker crash coverage remains outstanding** and should be requested in remediation approval before guarantee changes.
- These three diagnostic tests **prove findings, do not close production-readiness**. See Gitea https://dev.ivib.cc/vibtools/DataStack/actions/runs/63; never call run successful until terminal conclusion is confirmed. No server-side source fix applied. - These three diagnostic tests **prove findings, do not close production-readiness**. See Gitea https://dev.ivib.cc/vibtools/DataStack/actions/runs/63; never call run successful until terminal conclusion is confirmed. No server-side source fix applied.
## F. P02 Remediation Candidate Patch Proof — 2026-11-22
Base: Gitea/GitHub published P02 diagnostic tree `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`; GitLab distinct commit, same previous tree. Local candidate commit `44eee8877e40d1683f44502c2a0ba3d437a03b88` on isolated worktree. Initial exact delta: 5 approved source/test paths, 310 insertions and 22 deletions, no application UI, Python SDK, auth, public health route, CI or deployment edit. `git diff --check` and `node --check` on JS files PASS.
- `server/index.js:pushEventsToRedisOrFallback`: catch RPUSH failure, keep previous RAM buffering, throw sanitized `BUFFER_PERSIST_FAILED`; no positive ACK for that branch.
- `server/socket/handlers.js:sync_data/batch_sync`: negative ACK and best-effort claimed idempotency-key rollback with negative retry guard, no change to legitimate duplicate success path or existing auth/limits/error codes.
- `server/index.js:flushBuffer` and `core/data/buffer.ts:flush`: scoped streaming CSV/CSV.gz read-before-reappend (parsing quoted/multiline rows), exact client/date/event ID match; failures abort before Redis LTRIM. No changes to `server/workers/csvWriter.js`.
- `tests/phase02-sync.test.ts`: R-01 single/batch outage and duplicate claim ACK negative, successful recovery ACK, and unchanged R-03 liveness.
- `tests/phase03-storage-buffer.test.ts`: web-side replay verifies exactly one physically stored row; new direct Port 4000 writer Redis-trim failure/recovery proof. All tests on fake Redis and disposable temp CSV paths.
**Unverified:** Gitea P02 implementation acceptance was unavailable due inaccessible host/connector. Local `npm ci` not confirmed successful. Do not invent a CI run ID or claim full PASS. Full cross-process transactional exactly-once and all Redis claim crash windows are out of proof scope and may require exact additional approval. The streaming dedupe scan changes I/O cost per flush; performance validation before production is required.
### Blocked acceptance evidence — actual tool outcomes
- JS syntax: local `node --check server/index.js` and `node --check server/socket/handlers.js` PASS.
- Source scope: local `git diff --check` PASS; only the five allowed P02 runtime/test paths changed before notes updates.
- Gitea: connection returned `UNAVAILABLE: McpServerError: Connection failed`; Git remote `dev.ivib.cc` unresponsive. No Gitea CI run ID exists for candidate `44eee8877e40d1683f44502c2a0ba3d437a03b88`.
- Local Windows npm: `npm ci` failed `ETARGET yaml@2.9.1` and cleanup `EPERM` in the isolated worktree, **not** a TypeScript/test result. Dependency changes are out of P02 scope.
- Therefore neither TS typecheck nor regression-suite remediation acceptance is established; keep P02 BLOCKED and do not advance P03 or alter any protected runtime/source beyond this approval.
## F. P02 Final Acceptance Patch Index (current source/CI proof)
- Published diagnosis Gitea/GitHub main: `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`; previous GitLab main same tracked tree.
- Runtime candidate: `76cd972bdeea9ef07b4b38e9d8b5b5ed3312ca5b`; Gitea [CI #66](https://dev.ivib.cc/vibtools/DataStack/actions/runs/66) **terminal SUCCESS** across node 24/24, strict TS, lint, Next standalone and Windows Node SDK.
- Exact supplemental commit: `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532` (3 changed files: `server/socket/handlers.js`, `tests/phase02-sync.test.ts`, `tests/phase03-storage-buffer.test.ts`). Compared with published diagnosis, final candidate also includes runtime `server/index.js`, `core/data/buffer.ts`, and previously modified tests, all authorized, plus `notes/001–006`.
- **R-01 arithmetic reproduction** before patch: `const marker=-t; const now=t+61000; now-marker > 300000` is true, while `now-t > 300000` is false. `server/socket/handlers.js:142-158` `cleanupMaps` expires old negative marker; `validateSingleEventAsync:293-317` sees retained Redis idempotency claim; `sync_data` returned duplicate success. Final marker is `\u0000failed-persist:<clientId>:<eventId>` with positive timestamp, corrected age check, cleanup after true TTL, clear marker after durable ACK. Mixed batches reject overall positive ACK while a failed claim remains.
- **R-02:** `server/index.js:843-897,1043-1058`, `core/data/buffer.ts:652-711,855-872`: stream-parsed same-day client CSV/CSV.gz replay suppression, then Redis trim. Focused tests inject direct Port4000 LTRIM crash and compressed part2 quoted/multiline replay; malformed matching partition causes safe no-trim, fixture repair recovers. Memory streamed, partition disk-I/O remains proportional; not globally exactly-once.
- **Compat:** Redis-free RAM ACK and old public /health unchanged; other HTTP/Socket auth, SDK and quota routes unaffected.
- Final [Gitea CI #69](https://dev.ivib.cc/vibtools/DataStack/actions/runs/69) dispatched against exact supplemental commit; until terminal result, phase stays pending and no remote publish.
### Final P02 CI #69 failure and exact retry patch
- CI #69: `phase02-sync.test.ts` subtest `stale Redis claim never becomes positive ACK after cleanup window` `ERR_TEST_FAILURE: operation has timed out` after replacing the **global** `Date.now` during Socket.IO ACK. This cannot support a server-failure claim; it was a test harness flaw. Other new replay/compatibility assertions PASS and quality/Windows jobs separately passed.
- Discovered source branch in `server/socket/handlers.js`: `if (volatileDuplicate && processed > 0)` allowed an overall `success:true` response with pending non-durable claim and already-durable duplicate when `processed=0,duplicates>0`. Revised `if (volatileDuplicate)` blocks every mixed response with an uncommitted claim; new test reproduces the exact combination.
- Test no longer overrides Date.now; independent simulated arithmetic proves the original negative-marker expiry mistake; live ACK checks run on real clock. Supplemental commit `acca8be01a31f8d3972564cbaec8fd3117a0b551`; [CI #72](https://dev.ivib.cc/vibtools/DataStack/actions/runs/72) must pass before P02 closure. No modification of package/dependency files, test CI workflow, public API, UI or production.
## G. P02 Final Acceptance — authoritative terminal CI / Scope Lock proof
- Source baseline published diagnosis `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`. Fixed runtime candidate `76cd972bdeea9ef07b4b38e9d8b5b5ed3312ca5b`, previous #66 SUCCESS. Final corrected **source/test** SHA `acca8be01a31f8d3972564cbaec8fd3117a0b551` fully verified in [Gitea #72](https://dev.ivib.cc/vibtools/DataStack/actions/runs/72), **terminal COMPLETED SUCCESS**, Node `CI_SUITE_SUMMARY total=24 failed=0`. CI job conclusions: node-regression SUCCESS, quality SUCCESS, windows-compatibility SUCCESS, production-build SUCCESS; python-regression and docker-image-build SKIPPED by narrow dispatch.
- Exact reproducible Redis idempotency bug: old `-Date.now()` failed timestamp ages incorrectly at +61000ms, while 300000ms Redis claim TTL has not elapsed. New `\u0000failed-persist:${claimKey}` positive timestamp marker and corrected TTL comparison isolate the failed claim. Live Socket.IO retries remain negative ACK; no process-wide clock monkeypatch. Mixed all-duplicate batch also rejects pending claim instead of sending global positive ACK.
- CSV fault evidence: `tests/phase03-storage-buffer.test.ts` after CSV append but before Redis LTRIM, direct Port4000 writer, rotated `.csv.gz`, JSON multiline-quoted payload, corrupt matching partition no-trim/recovery all PASS. Test assertions cannot prove a global exactly-once protocol or live throughput cap. Full-partition scan is streaming bounded-memory, but data-size-linear disk I/O.
- Historical #69 failed one test-only global Date.now-induced Socket.IO timeout, **not** a demonstrated runtime regression; corrected #72 PASS. No secret or credential introduction, no production data access.
- Final audit `git diff --check` PASS and source JS syntax checks PASS; exact tracked source diff from diagnosis = 11 allowlisted files (3 runtime + 2 test + 6 notes); no unrelated UI/auth/SDK/public health, DB schema, package metadata, Docker, CI workflow or live deployment.
- Git release gate: final docs-only commits after tested source SHA add **no source/test differences**. Gitea PR #3 squashed into one `[skip ci]` main commit; GitHub/GitLab tracked Git trees must match Gitea before final CLOSED. If GitLab commit SHA differs because of parent history, compare tree SHA not commit SHA. Store exact remote SHAs/trees in phase close operator report; do not invent them before push.
## H. P03 exact audit and terminal evidence — 2026-11-22
Baseline Gitea/GitHub `48fb277079b380fdbf1e58cd2da40afeed0b03ee`; GitLab baseline same tracked tree `41a169eec0c05ce19bc4acc29d7a8cfe1f8dd90d` (317/317 blobs). Final P03 workflow candidate `549d575653b68c29f97e2f78036646f2e2035719`. [Gitea CI #77](https://dev.ivib.cc/vibtools/DataStack/actions/runs/77): overall **FAILURE** because the initial Python-only checkout lacked Node (exit 127), **not** an SDK regression. Its other six jobs terminal **SUCCESS**: Node `CI_SUITE_SUMMARY total=24 failed=0`, strict TS+ESLint, Next standalone build, Windows SDK, Compose + isolated BuildKit ingestion/web/proxy image build `DATASTACK_THREE_IMAGES_BUILD_PASS`, read-only advisory inventory `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Narrow [Python-only CI #78](https://dev.ivib.cc/vibtools/DataStack/actions/runs/78) terminal **SUCCESS**: Node-capable Bookworm container, Python 3.11 isolated venv, `Ran 14 tests in 3.959s`, `OK`, `DATASTACK_PYTHON_14_OF_14_PASS`. The other six jobs were deliberately skipped in #78.
**Docker evidence:** separate BuildKit `docker-container` builder with owned-builder cleanup, `--no-cache --load`, actual Compose `config --no-interpolate --quiet`, ingestion/web/proxy Dockerfiles *read-only*, three tags inspected; old `@uiw` error absent. **Python evidence:** only Node-capable image change within workflow, Python 3.11 venv with all 14 tests; Python SDK source unchanged.
**CI-08 security disposition:** `npm audit --json` full=26 affected packages (25 moderate, 1 high); runtime=24 (23 moderate, 1 high), **not counts of distinct advisories**. High-severity advisories [GHSA-6g55-p6wh-862q](https://github.com/advisories/GHSA-6g55-p6wh-862q), [GHSA-r28c-9q8g-f849](https://github.com/advisories/GHSA-r28c-9q8g-f849) affect Next 15.5.29's nested `postcss@8.4.31` despite patched root `postcss@8.5.33`; npm recommends major `next@16.4.6`. Direct `cloudcmd@19.24.0` carries moderate transitive issues (`dompurify`, `file-type`, `glob`, `minimatch`, `got`); npm recommends `cloudcmd@17.0.2` downgrade, not a verified safe pin. No speculative package pin, major upgrade or force fix; a separately tested compatible remediation is needed before release.
**Exact allowlist audit:** pre-documentation candidate diff had only two approved workflow files; final publication includes six approved `notes/001–006` updates. Workflow checks: no trailing whitespace, no `docker ... prune`, no `npm audit fix --force`, all 3 Dockerfile paths present, no runtime or dependency lock changes. Check three-remote tracked tree parity before marking phase CLOSED.
**Scope / release decision:** only `.gitea/workflows/datastack-ci.yml`, `.gitea/workflows/datastack-docker-ci.yml`, and `notes/001–006` changed. No `package.json`/lock, Dockerfile, application UI/UX, runtime, auth, SDK, database, Compose, production data, secret, shared Docker prune, deployment, migration, tag or release. P03 technical CI and advisory-triage acceptance verified by #77 non-Python jobs plus #78 Python; do **not** mislabel #77 overall as successful. Final phase publication requires one main commit and Gitea/GitHub/GitLab exact tracked tree parity. **Production release NO-GO** while runtime high advisory is unresolved; live production health, throughput and deployment remain unverified.
## I. P04-S01 approved baseline and source proof — 2026-11-22
Baseline commit `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3`, content tree `12818ba732f5abdc42af46d83278b13b6be4eafd`. P03 CI #77 `AUDIT_COUNTS runtime {moderate:23,high:1,critical:0,total:24}` (affected packages). Lockfile line proof `package-lock.json:10552-10560` Next `15.5.29` pinned nested postcss `8.4.31`; `10602-10603` resolved nested copy `8.4.31`; `11427-11428` root `8.5.33`. Gitea candidate branch `p04-s01-postcss-security-20261122`, no runtime changes. Approved exact write boundaries are only `package.json` scoped PostCSS override, npm-generated lockfile, `.gitea/workflows/datastack-ci.yml` security-audit block and notes/001–006. Acceptance pending.
## P04-S01 blocked security reconciliation report
P04-S01 SEC-01 BLOCKED — 2026-11-22. Approved scoped `overrides.next.postcss=8.5.33` trialed in disposable Windows npm 10.9.8 workspace only. `npm install --package-lock-only` EXIT 0, but `npm ls --package-lock-only postcss --all` EXIT 1 (`ELSPROBLEMS`: Next nested `postcss@8.4.31` invalid; expected 8.5.33); npm dedupe also kept invalid dependency. Read-only `npm audit --omit=dev --json` EXIT 1, runtime high 1/moderate 23/critical 0. Lockfile regeneration caused unrelated churn (91 package entries added, 106 removed, 69 changed, including two lightningcss optional 1.33.0 -> 1.30.1). Fresh minimal Next 15.5.29 fixture with identical override resolves PostCSS 8.5.33, isolating failure to DataStack committed lock graph. `npm ci --dry-run` exit 0 does not prove real clean install. Exact stop rule applied: no dependency patch, no CI acceptance, no main commit, no GitHub/GitLab sync, no release/deploy; production NO-GO. Need separate explicit narrowly scoped lock reconciliation approval.
Baseline tree `12818ba732f5abdc42af46d83278b13b6be4eafd`; npm `audit --omit=dev` includes affected node `node_modules/next/node_modules/postcss` and high GHSAs above. This proof lives in notes, not a change to repository dependencies; no patch is accepted.
## K. P04-S01B reproducible candidate proof / unaccepted patch — 2026-11-22
**Baseline:** Gitea main `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3`, tree `12818ba732f5abdc42af46d83278b13b6be4eafd`. Gitea source path `package-lock.json:10552-10603` includes Next 15.5.29 and nested PostCSS 8.4.31; root PostCSS 8.5.33. Connected Windows npm 10.9.8 temp probe downloads baseline manifest/lock; exact approved exception deletes `node_modules/next/node_modules/postcss` entry **in disposable copy only** and adds manifest `overrides.next.postcss=8.5.33`. `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline`: exit 0. `npm ls --package-lock-only postcss --all --depth=4`: exit 0; Next uses PostCSS 8.5.33 deduped and no nested 8.4.31. `npm audit --omit=dev --json` exits 1 for remaining **moderates**; counts `{moderate:22,high:0,critical:0,total:22}`; `postcss` no longer appears as vulnerable package. Zero unrelated version changes and exactly 1 removed stale nested package; **67 other package entries changed metadata**: `dev` 36, `libc` 29, 2 `dependencies` JSON order differences (field frequencies can overlap). Must preserve unrelated version AND metadata per user's approval. **STOP**: candidate not committed to repo; no real clean npm ci, no Gitea CI acceptance or workflow gate change, no main merge or 3-mirror sync. P04-S01B remains BLOCKED, released baseline still vulnerable/high=1 and production NO-GO. A future separate exact approved strategy for metadata preservation or bounded exceptions is necessary.
## L. P04-S01C candidate exact patch, CI and scope proof — 2026-11-22
- Baseline: Gitea main `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3`, tree `12818ba732f5abdc42af46d83278b13b6be4eafd`; P04 notes branch `p04-s01-postcss-security-20261122`. Approved patch commit `c759be990c032c2474d2d0bad519c643dec11a3a`. PR [#5](https://dev.ivib.cc/vibtools/DataStack/pulls/5) draft pending full acceptance.
- Exact patch diff from Gitea source commit: `package.json` append only `overrides.next.postcss = 8.5.33`; `package-lock.json` delete only 26 lines for `node_modules/next/node_modules/postcss@8.4.31`; `.gitea/workflows/datastack-ci.yml:security-audit` insert 10-line runtime high/critical failure/verification gate. No changes outside 3 source/config files plus notes/001–006.
- **Generated lock preservation proof:** remove stale entry in disposable npm lock copy; regenerate using npm 10.9.8; restore 67 unrelated metadata entries byte-exactly to baseline. Structural comparison final versus baseline `added=0 removed=1 changedExistingEntries=0`; all unrelated versions, resolved URLs, integrity hashes and dependency metadata identical. `npm ls --package-lock-only postcss --all --depth=4` exit 0, Next 15.5.29 resolves root PostCSS 8.5.33. Final lockfile SHA-256 `e6b5b362877627085dc04c9ded03c019971e07ef9f4af1e78057d58961a1c091`; downloaded Gitea branch lock bytes match. Linux Gitea runner actual `npm ci` SUCCESS, followed by 24 suites. Connected Windows scratch `npm ci` completion not independently verified; Windows Gitea job outstanding.
- Gitea CI [#85](https://dev.ivib.cc/vibtools/DataStack/actions/runs/85) on source commit `c759be99`: node-regression 24/24 + SDK SUCCESS; Python 14/14 SUCCESS; quality strict TS/ESLint SUCCESS; Next 22/22 static pages + standalone SUCCESS; isolated BuildKit 3-image build SUCCESS; security-audit SUCCESS `AUDIT_COUNTS full {moderate:24,high:0,critical:0}`, `AUDIT_COUNTS runtime {moderate:22,high:0,critical:0}`, `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO`, `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Windows-compatibility QUEUED at checkpoint: no full terminal acceptance yet. No main or mirror changes, no deployment, migration, shared Docker prune, tag or release. Production NO-GO pending 7/7 and remote parity, moderate advisories/live verification.
## P04-S01C FINAL CI acceptance (7/7) — 2026-11-22
Gitea CI #85 https://dev.ivib.cc/vibtools/DataStack/actions/runs/85 COMPLETED/SUCCESS; tested implementation SHA c759be990c032c2474d2d0bad519c643dec11a3a; later changes to candidate are documentation only. All seven jobs terminal SUCCESS: node-regression (Linux committed-lock npm ci and exactly 24/24 TypeScript suites); python-regression (Ran 14 tests, OK); quality (strict tsc and ESLint); production-build (Next 15.5.29 standalone, 22/22 pages); docker-image-build (three isolated BuildKit ingestion/web/proxy images); windows-compatibility (Windows committed-lock npm ci and Node SDK build, job #182, terminal SUCCESS); security-audit (runtime moderate=22 high=0 critical=0; full moderate=24 high=0 critical=0; DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO). Single Gitea acceptance run used, no additional broad reruns.
Scope: exactly nine approved paths: package.json scoped next->postcss 8.5.33 override; package-lock.json removes only old nested postcss@8.4.31 entry; existing .gitea/workflows/datastack-ci.yml security-audit job high/critical rejection; notes/001–006 evidence. Other package metadata, versions, integrity and resolved URLs match baseline. Next.js 15.5.29 and Cloudcmd 19.24.0 preserved. Final candidate package-lock SHA256 e6b5b362877627085dc04c9ded03c019971e07ef9f4af1e78057d58961a1c091 (verified byte-for-byte with Gitea). No API, auth, SDK, UI, application runtime, database, Compose, Dockerfile, tests or production data changes, deployment, migration, prune, tag, release.
Exact job IDs: 177 Node, 178 Python, 179 quality, 180 Next production, 181 Docker three images, 182 Windows compatibility, 183 security-audit; all seven completed/success, run #85 overall completed/success. CI fully accepted; publication not yet asserted in these pre-merge notes. Final phase CLOSED requires one squash merge of PR #5 into Gitea main and verified identical tracked content across Gitea/GitHub/GitLab; record actual new SHAs in terminal report. Production release remains NO-GO pending P04-S02 moderate vulnerability triage and separately authorized live readiness verification.
## P04-S02 scoped candidate — 2026-11-22
Baseline Gitea main `2d467ff1494858164a37814f7dbc2b321f510291`, tree `648c76ca496d2c9bf22a16f558c9ba347a45d4f7`. Cloudcmd 19.24.0 -> deepword 11.0.1 -> monaco-editor 0.55.1 pins DOMPurify 3.2.7, vulnerable according to npm advisory inventory. Disposable Windows npm 10.9.8 `npm view dompurify@3.4.18` verified, scoped npm override `overrides.cloudcmd.dompurify=3.4.18` resolves `dompurify@3.4.18` via `npm ls --package-lock-only dompurify --all --depth=5` (exit 0). `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline` exit 0. npm graph initially changed 67 unrelated metadata entries; deterministically restored exact baseline values after validating unrelated versions, URLs, integrity and dependencies. Final lock has **one changed package entry**, DOMPurify; same-major replacement plus verified registry `resolved` and `integrity`, no other packages added/deleted/changed. Candidate runtime audit affected-package counts **moderate 20/high 0/critical 0** versus baseline **22/0/0**; full candidate **moderate 22/high 0/critical 0** versus baseline **24/0/0**. DOMPurify and Monaco Editor dropped from vulnerable-package inventory. Other moderate packages (got, glob, minimatch, nested file-type, Cloudcmd transitive chain) remain; they require major package changes under current advisory constraints, prohibited by Scope Lock. This is a **pre-acceptance candidate**: do not claim application compatibility, CI PASS, publication, or phase CLOSED before full focused Gitea CI and exact three-remote tree parity.
Patch provenance: npm registry DOMPurify 3.4.18 tarball URL and integrity SHA512 in npm-generated entry, unchanged other package metadata after deterministic baseline restoration. New graph root overrides change only Cloudcmd-specific DOMPurify. One focused acceptance run pending.
## P04-S02 final CI acceptance — 2026-11-22
Gitea CI #92 (https://dev.ivib.cc/vibtools/DataStack/actions/runs/92) **COMPLETED / SUCCESS**, all 7/7 jobs terminal success on source commit `a88bc5f4498ea1b11acc0acfeb66b5aa9d6531be`: node-regression real Linux npm ci + 24/24 tests including `tests/files-manager.test.ts` exit 0; Python SDK 14/14; quality strict TypeScript/ESLint; Next.js standalone (22/22 static pages); isolated Docker/BuildKit 3-image verification `DATASTACK_THREE_IMAGES_BUILD_PASS`; Windows committed-lock npm ci + Node SDK compatibility; security-audit `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO` and `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Audit full=22 moderate/high=0/critical=0 (baseline 24), runtime=20 moderate/high=0/critical=0 (baseline 22). These are affected package entries, not distinct advisories. No new CI workflow or extra broad CI dispatched.
Exact approved patch: `package.json` scoped `overrides.cloudcmd.dompurify=3.4.18`, preserving `overrides.next.postcss=8.5.33`; npm-generated `package-lock.json` changes solely `node_modules/dompurify` version 3.2.7 -> 3.4.18 and verified `resolved`/SHA512 integrity. Baseline Cloudcmd 19.24.0, Monaco Editor 0.55.1, Next.js 15.5.29, all unrelated locked package versions, URL, hash, dependency and metadata preserved. Gitea published candidate lock bytes match disposable SHA-256 `57c5ea9df9f28be496f8c3202fbc280b0066bf8c216b8d8a3ca98b6f7b942854`. Exact candidate tree diff 420 entries, 8 approved changed files only (`package.json`, `package-lock.json`, notes/001–006), 0 unauthorized paths. No runtime/UI/auth/API/SDK/tests/CI/Dockerfiles/Compose/db/schema/data modifications.
Detailed proof: final DOMPurify npm registry URL `https://registry.npmjs.org/dompurify/-/dompurify-3.4.18.tgz` with integrity `sha512-ofe3wI/uNm68HInABYFJYqQxRw1EaKAjNdrFVKqjUY+UZurvNd2L/HuQe097s/4RivWiZyP6tT7aoT8atjfhwg==`. Confirmed isolated npm ls override Cloudcmd -> Deepword -> Monaco Editor -> DOMPurify 3.4.18, isolated npm audit runtime 20/0/0 and full 22/0/0, Gitea acceptance matches. **Residual risk:** 20 moderate runtime affected package entries remain, including Cloudcmd transitive dependencies `got`, `glob`, `minimatch`, nested `file-type` and their parent chains. Current npm audit suggests `cloudcmd@17.0.2` (a major downgrade), explicitly out-of-scope; other transitive fixes need major-version dependency-contract changes. No force/downgrade/unverified override. Security improvement verified but NOT zero vulnerabilities. Production release stays NO-GO; remaining advisory disposition and live readiness need independent scope approval.
**Merge gate:** CI #92 7/7 PASS, approved 8-file patch integrity PASS; main squash merge and Gitea/GitHub/GitLab exact Git content-tree verification required before P04-S02 CLOSED. This note is written before that publication, so final remote commit SHAs shall be appended to the PR closure report after verification. No deployment, migrations, prune, tag, release.
+3 -27
View File
@@ -5843,7 +5843,9 @@
} }
}, },
"node_modules/dompurify": { "node_modules/dompurify": {
"version": "3.2.7", "version": "3.4.18",
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.18.tgz",
"integrity": "sha512-ofe3wI/uNm68HInABYFJYqQxRw1EaKAjNdrFVKqjUY+UZurvNd2L/HuQe097s/4RivWiZyP6tT7aoT8atjfhwg==",
"license": "(MPL-2.0 OR Apache-2.0)", "license": "(MPL-2.0 OR Apache-2.0)",
"optionalDependencies": { "optionalDependencies": {
"@types/trusted-types": "^2.0.7" "@types/trusted-types": "^2.0.7"
@@ -10599,32 +10601,6 @@
} }
} }
}, },
"node_modules/next/node_modules/postcss": {
"version": "8.4.31",
"funding": [
{
"type": "opencollective",
"url": "https://opencollective.com/postcss/"
},
{
"type": "tidelift",
"url": "https://tidelift.com/funding/github/npm/postcss"
},
{
"type": "github",
"url": "https://github.com/sponsors/ai"
}
],
"license": "MIT",
"dependencies": {
"nanoid": "^3.3.6",
"picocolors": "^1.0.0",
"source-map-js": "^1.0.2"
},
"engines": {
"node": "^10 || ^12 || >=14"
}
},
"node_modules/nicki": { "node_modules/nicki": {
"version": "6.1.1", "version": "6.1.1",
"license": "MIT", "license": "MIT",
+8
View File
@@ -61,5 +61,13 @@
"optionalDependencies": { "optionalDependencies": {
"lightningcss-linux-arm64-musl": "^1.30.1", "lightningcss-linux-arm64-musl": "^1.30.1",
"lightningcss-linux-x64-musl": "^1.30.1" "lightningcss-linux-x64-musl": "^1.30.1"
},
"overrides": {
"next": {
"postcss": "8.5.33"
},
"cloudcmd": {
"dompurify": "3.4.18"
}
} }
} }
+70 -2
View File
@@ -165,6 +165,7 @@ function setRedisClientForTests(client) {
async function pushEventsToRedisOrFallback(events) { async function pushEventsToRedisOrFallback(events) {
if (!events || events.length === 0) return bufferQueue.length; if (!events || events.length === 0) return bufferQueue.length;
const redis = getRedisClient(); const redis = getRedisClient();
let persistFailed = false;
if (redis) { if (redis) {
const serialized = events.map((e) => JSON.stringify(e)); const serialized = events.map((e) => JSON.stringify(e));
let bufferLen = null; let bufferLen = null;
@@ -173,6 +174,7 @@ async function pushEventsToRedisOrFallback(events) {
redisConnected = true; redisConnected = true;
} catch { } catch {
redisConnected = false; redisConnected = false;
persistFailed = true;
} }
if (bufferLen !== null) { if (bufferLen !== null) {
@@ -189,6 +191,11 @@ async function pushEventsToRedisOrFallback(events) {
} }
bufferQueue.push(...events); bufferQueue.push(...events);
if (persistFailed) {
// Keep the volatile fallback for later recovery, but never ACK it as durable.
// Suppress sensitive Redis transport details in the client-facing error.
throw new Error('BUFFER_PERSIST_FAILED: Redis unavailable; event not durably queued');
}
return bufferQueue.length; return bufferQueue.length;
} }
@@ -834,6 +841,61 @@ async function isClientDeletedForFlush(clientId, redis, cache) {
} }
async function flushBuffer(options = {}) { async function flushBuffer(options = {}) {
// P02 R-02: Redis and CSV cannot be committed atomically. Before replaying an
// untrimmed Redis batch, check actual CSV partitions for previously written IDs.
// Streaming parsing bounds memory and preserves multiline/quoted CSV fields.
const excludeAlreadyPersisted = async (events) => {
if (!events.length) return events;
const fs = require('node:fs');
const path = require('node:path');
const zlib = require('node:zlib');
const { pipeline } = require('node:stream/promises');
const { Writable } = require('node:stream');
const { parse } = require('csv-parse');
const groups = new Map();
for (const event of events) {
const clientId = String(event.clientId || 'unknown');
const date = writer.extractSafePartitionDate(event.timestamp);
const key = JSON.stringify([clientId, date]);
if (!groups.has(key)) groups.set(key, { clientId, date, ids: new Set() });
groups.get(key).ids.add(String(event.eventId));
}
const persisted = new Set();
for (const [groupKey, group] of groups) {
const dir = writer.ensureClientDir(group.clientId);
const entries = await fs.promises.readdir(dir, { withFileTypes: true });
for (const entry of entries) {
if (!entry.isFile() || !writer.isPartitionFilename(entry.name) ||
writer.extractDateFromFilename(entry.name) !== group.date) continue;
const reader = fs.createReadStream(path.join(dir, entry.name));
const streams = [reader];
if (entry.name.endsWith('.gz')) streams.push(zlib.createGunzip());
streams.push(parse({ columns: true, bom: true, skip_empty_lines: true }));
streams.push(new Writable({
objectMode: true,
write(row, _enc, done) {
if (row.clientId === group.clientId && group.ids.has(String(row.eventId))) {
persisted.add(JSON.stringify([groupKey, String(row.eventId)]));
}
done();
},
}));
// On read/parse failure, abort without LTRIM: never discard uncertain rows.
await pipeline(...streams);
}
}
const batchSeen = new Set();
return events.filter((event) => {
const key = JSON.stringify([String(event.clientId || 'unknown'),
writer.extractSafePartitionDate(event.timestamp)]);
const rowKey = JSON.stringify([key, String(event.eventId)]);
if (persisted.has(rowKey) || batchSeen.has(rowKey)) return false;
batchSeen.add(rowKey);
return true;
});
};
if (pendingRedisWrites.size > 0) { if (pendingRedisWrites.size > 0) {
await Promise.allSettled([...pendingRedisWrites]); await Promise.allSettled([...pendingRedisWrites]);
} }
@@ -979,10 +1041,13 @@ async function flushBuffer(options = {}) {
if (lockLost) break; if (lockLost) break;
if (parsedBatch.length > 0) { if (parsedBatch.length > 0) {
const res = await writer.writeBatch(parsedBatch); const unwritten = await excludeAlreadyPersisted(parsedBatch);
if (unwritten.length > 0) {
const res = await writer.writeBatch(unwritten);
totalWritten += res.written; totalWritten += res.written;
for (const f of res.files) filesSet.add(f); for (const f of res.files) filesSet.add(f);
} }
}
if (Date.now() - lastRenewAt >= renewIntervalMs) { if (Date.now() - lastRenewAt >= renewIntervalMs) {
await renewLockOnce(); await renewLockOnce();
@@ -1022,9 +1087,12 @@ async function flushBuffer(options = {}) {
if (validBatch.length > 0) { if (validBatch.length > 0) {
try { try {
const res = await writer.writeBatch(validBatch); const unwritten = await excludeAlreadyPersisted(validBatch);
if (unwritten.length > 0) {
const res = await writer.writeBatch(unwritten);
totalWritten += res.written; totalWritten += res.written;
for (const f of res.files) filesSet.add(f); for (const f of res.files) filesSet.add(f);
}
} catch (err) { } catch (err) {
bufferQueue.unshift(...validBatch); bufferQueue.unshift(...validBatch);
throw err; throw err;
+70 -2
View File
@@ -549,6 +549,9 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
: new Date().toISOString(); : new Date().toISOString();
const eventData = extractSingleEventData(payload); const eventData = extractSingleEventData(payload);
// A previous Redis write failure can leave an uncommitted idempotency
// claim. Never report that claim as a successful durable duplicate.
const failedAt = idempotencyMap.get(`\u0000failed-persist:${clientId}:${eventId}`);
const check = await validateSingleEventAsync( const check = await validateSingleEventAsync(
clientId, clientId,
eventId, eventId,
@@ -557,7 +560,13 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
); );
if (!check.allowed) { if (!check.allowed) {
if (check.duplicate) { if (check.duplicate) {
if (callback) { if (typeof failedAt === "number" &&
Date.now() - failedAt < IDEMPOTENCY_TTL_MS) {
if (callback) callback({
success: false, eventId, code: "BUFFER_PERSIST_FAILED",
error: "BUFFER_PERSIST_FAILED: Event not durably queued",
});
} else if (callback) {
callback({ success: true, eventId, note: "DUPLICATE_DROPPED" }); callback({ success: true, eventId, note: "DUPLICATE_DROPPED" });
} }
return; return;
@@ -585,6 +594,20 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
try { try {
await onBufferPush(); await onBufferPush();
} catch (persistErr) { } catch (persistErr) {
// Invalidate the pre-persistence idempotency claim so SDK retry can
// resubmit after Redis recovers. Keep the volatile RAM event pending.
const claimKey = `${clientId}:${eventId}`;
idempotencyMap.delete(claimKey);
idempotencyMap.set(`\u0000failed-persist:${claimKey}`, Date.now());
const claimRedis = getRedisClient();
if (claimRedis && typeof claimRedis.del === "function") {
try {
await claimRedis.del(`${REDIS_IDEMPOTENCY_PREFIX}${claimKey}`);
idempotencyMap.delete(claimKey);
} catch {
// Claim may remain: negative ACK must be retained for retries.
}
}
if (callback) { if (callback) {
callback({ callback({
success: false, success: false,
@@ -599,6 +622,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
return; return;
} }
idempotencyMap.delete(`\u0000failed-persist:${clientId}:${eventId}`);
if (callback) callback({ success: true, eventId }); if (callback) callback({ success: true, eventId });
} catch (err) { } catch (err) {
if (callback) callback({ success: false, error: err.message }); if (callback) callback({ success: false, error: err.message });
@@ -689,6 +713,8 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
const rateLimitedEventIds = []; const rateLimitedEventIds = [];
const oversizedEventIds = []; const oversizedEventIds = [];
const errors = []; const errors = [];
const acceptedEventIds = [];
let volatileDuplicate = false;
for (const item of batch) { for (const item of batch) {
const safeItem = item && typeof item === "object" ? item : {}; const safeItem = item && typeof item === "object" ? item : {};
@@ -702,6 +728,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
: new Date().toISOString(); : new Date().toISOString();
const rawData = extractBatchItemData(safeItem); const rawData = extractBatchItemData(safeItem);
const failedAt = idempotencyMap.get(`\u0000failed-persist:${clientId}:${eventId}`);
const check = await validateSingleEventAsync( const check = await validateSingleEventAsync(
clientId, clientId,
eventId, eventId,
@@ -710,7 +737,15 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
); );
if (!check.allowed) { if (!check.allowed) {
if (check.duplicate) { if (check.duplicate) {
if (typeof failedAt === "number" &&
Date.now() - failedAt < IDEMPOTENCY_TTL_MS) {
rejected += 1;
volatileDuplicate = true;
if (errors.length < 5) errors.push(
`${eventId}: BUFFER_PERSIST_FAILED: Event not durably queued`);
} else {
duplicates += 1; duplicates += 1;
}
continue; continue;
} }
rejected += 1; rejected += 1;
@@ -735,6 +770,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
payload: check.payloadStr, payload: check.payloadStr,
}); });
processed += 1; processed += 1;
acceptedEventIds.push(eventId);
} }
if (processed > 0) { if (processed > 0) {
@@ -742,6 +778,20 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
try { try {
await onBufferPush(); await onBufferPush();
} catch (persistErr) { } catch (persistErr) {
const claimRedis = getRedisClient();
for (const eventId of acceptedEventIds) {
const claimKey = `${clientId}:${eventId}`;
idempotencyMap.delete(claimKey);
idempotencyMap.set(`\u0000failed-persist:${claimKey}`, Date.now());
if (claimRedis && typeof claimRedis.del === "function") {
try {
await claimRedis.del(`${REDIS_IDEMPOTENCY_PREFIX}${claimKey}`);
idempotencyMap.delete(claimKey);
} catch {
// Preserve negative-ACK retry guard if claim removal fails.
}
}
}
if (callback) { if (callback) {
callback({ callback({
success: false, success: false,
@@ -759,8 +809,26 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
} }
} }
// Only durable new writes clear pending failure markers.
for (const persistedId of acceptedEventIds) {
idempotencyMap.delete(`\u0000failed-persist:${clientId}:${persistedId}`);
}
// A mixed batch must never receive success while one event still has
// an uncommitted Redis claim, even if other items were persisted.
if (volatileDuplicate) {
if (callback) callback({
success: false, processed, rejected, duplicates,
code: "BUFFER_PERSIST_FAILED",
error: "BUFFER_PERSIST_FAILED: Batch contains non-durable event",
errors,
});
return;
}
if (processed === 0 && duplicates === 0 && rejected > 0) { if (processed === 0 && duplicates === 0 && rejected > 0) {
const code = hitPayloadLimit const code = volatileDuplicate
? "BUFFER_PERSIST_FAILED"
: hitPayloadLimit
? "PAYLOAD_TOO_LARGE" ? "PAYLOAD_TOO_LARGE"
: hitRateLimit : hitRateLimit
? "RATE_LIMIT_EXCEEDED" ? "RATE_LIMIT_EXCEEDED"
+215 -1
View File
@@ -1246,12 +1246,29 @@ test("P02 R-01 diagnostic: Redis RPUSH failure yields positive WS ACK for volati
); );
const committedRows = await failingRedis.lrange("datastack:buffer:events", 0, -1); const committedRows = await failingRedis.lrange("datastack:buffer:events", 0, -1);
const volatileRows = serverMod.bufferQueue.filter((e) => e.eventId === "p02-ack-before-crash-001"); const volatileRows = serverMod.bufferQueue.filter((e) => e.eventId === "p02-ack-before-crash-001");
assert.equal(ack.success, true, "Existing handler currently sends a positive ACK after Redis failure"); assert.equal(ack.success, false, "P02 remediation must reject non-durable ACK");
assert.equal(ack.code, "BUFFER_PERSIST_FAILED");
const sameIdRetry = await emitWithAck<{ success: boolean; code?: string }>(
socket, "sync_data",
{ eventId: "p02-ack-before-crash-001", data: { value: 42 } },
);
assert.equal(sameIdRetry.success, false,
"Pending idempotency claims must not return false positive duplicate ACK");
assert.equal(sameIdRetry.code, "BUFFER_PERSIST_FAILED");
assert.equal(committedRows.length, 0, "Fake Redis has no durable event"); assert.equal(committedRows.length, 0, "Fake Redis has no durable event");
assert.equal(volatileRows.length, 1, "Acknowledged event is RAM-only"); assert.equal(volatileRows.length, 1, "Acknowledged event is RAM-only");
// Crash model: restart loses RAM but fake Redis survives. No actual process is killed. // Crash model: restart loses RAM but fake Redis survives. No actual process is killed.
serverMod.bufferQueue.length = 0; serverMod.bufferQueue.length = 0;
assert.equal((await failingRedis.lrange("datastack:buffer:events", 0, -1)).length, 0); assert.equal((await failingRedis.lrange("datastack:buffer:events", 0, -1)).length, 0);
// A new event may ACK only after Redis actually resumes persistence.
const healthyRedis = createSharedMockRedis();
serverMod.setRedisClientForTests(healthyRedis);
const recovered = await emitWithAck<{ success: boolean; eventId?: string }>(
socket, "sync_data",
{ eventId: "p02-ack-recovered-002", data: { value: 43 } },
);
assert.equal(recovered.success, true);
assert.equal((await healthyRedis.lrange("datastack:buffer:events", 0, -1)).length, 1);
} finally { } finally {
socket?.disconnect(); socket?.disconnect();
serverMod.setRedisClientForTests(undefined); serverMod.setRedisClientForTests(undefined);
@@ -1281,3 +1298,200 @@ test("P02 R-03 diagnostic: Redis query failure leaves public ingestion health 20
serverMod.setRedisClientForTests(undefined); serverMod.setRedisClientForTests(undefined);
} }
}); });
test("P02 R-01: batch_sync rejects non-durable Redis writes and pending duplicate claims", async () => {
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
const previousQueue = serverMod.bufferQueue.slice();
const failingRedis = createSharedMockRedis();
failingRedis.rpush = async () => { throw new Error("P02 injected batch Redis outage"); };
serverMod.setRedisClientForTests(failingRedis);
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
serverMod.bufferQueue.length = 0;
let socket: Socket | undefined;
const payload = [
{ eventId: "p02-batch-volatile-1", data: { value: 1 } },
{ eventId: "p02-batch-volatile-2", data: { value: 2 } },
];
try {
const client = await SDKManager.registerClientAsync({
clientId: "p02_redis_batch_failure",
name: "P02 Redis Batch Failure",
rateLimit: 100,
});
socket = await connectSocket(client.token);
const ack = await emitWithAck<{ success: boolean; code?: string; processed?: number }>(
socket, "batch_sync", payload,
);
assert.equal(ack.success, false);
assert.equal(ack.code, "BUFFER_PERSIST_FAILED");
assert.equal(ack.processed, 0);
assert.equal((await failingRedis.lrange("datastack:buffer:events", 0, -1)).length, 0);
const retry = await emitWithAck<{ success: boolean; code?: string }>(
socket, "batch_sync", payload,
);
assert.equal(retry.success, false);
assert.equal(retry.code, "BUFFER_PERSIST_FAILED");
} finally {
socket?.disconnect();
serverMod.setRedisClientForTests(undefined);
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
}
});
test("P02 FINAL R-01: stale Redis claim never becomes positive ACK after cleanup window", async () => {
// Source proof at candidate 76cd972: a failure marker is -Date.now();
// cleanupMaps compares now - negativeTimestamp > TTL, which expires immediately
// on the first scheduled cleanup even though a Redis idempotency claim remains.
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
const previousQueue = serverMod.bufferQueue.slice();
const redis = createSharedMockRedis(); // intentionally no DEL: simulate stuck Redis claim
redis.rpush = async () => { throw new Error("Injected Redis queue outage"); };
serverMod.setRedisClientForTests(redis);
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
serverMod.bufferQueue.length = 0;
let socket: Socket | undefined;
try {
const client = await SDKManager.registerClientAsync({
clientId: "p02_stale_claim_cleanup", name: "P02 Stale Claim", rateLimit: 100,
});
socket = await connectSocket(client.token);
const payload = { eventId: "p02-stale-claim-1", data: { value: 1 } };
const first = await emitWithAck<{ success: boolean; code?: string }>(socket, "sync_data", payload);
assert.equal(first.success, false);
assert.equal(first.code, "BUFFER_PERSIST_FAILED");
// Verify TTL arithmetic without overriding the global clock (Socket.IO timeouts).
// Also keep two real socket retries to prove pending claims stay negative.
// Do not mutate process timestamps or wait for wall-clock TTL.
const stamped = Date.now();
const cleanupClock = stamped + 61_000;
assert.equal(cleanupClock - (-stamped) > 300_000, true, "Old negative marker expired prematurely");
assert.equal(cleanupClock - stamped < 300_000, true, "Five-minute TTL should still be active");
const handlerSource = readFileSync(require.resolve("../server/socket/handlers.js"), "utf8");
assert.ok(handlerSource.includes('idempotencyMap.set(`\\u0000failed-persist:${claimKey}`, Date.now());'));
assert.equal(handlerSource.includes("idempotencyMap.set(claimKey, -Date.now());"), false);
const retry1 = await emitWithAck<{ success: boolean; code?: string }>(socket, "sync_data", payload);
const retry2 = await emitWithAck<{ success: boolean; code?: string }>(socket, "sync_data", payload);
assert.equal(retry1.success, false);
assert.equal(retry1.code, "BUFFER_PERSIST_FAILED");
assert.equal(retry2.success, false, "Cleanup must not convert a volatile claim into DUPLICATE_DROPPED success");
assert.equal(retry2.code, "BUFFER_PERSIST_FAILED");
assert.equal((await redis.lrange("datastack:buffer:events", 0, -1)).length, 0);
} finally {
socket?.disconnect();
serverMod.setRedisClientForTests(undefined);
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
}
});
test("P02 FINAL R-01: mixed batch with stale volatile claim never ACKs success", async () => {
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
const previousQueue = serverMod.bufferQueue.slice();
const redis = createSharedMockRedis();
const originalRpush = redis.rpush.bind(redis);
redis.rpush = async () => { throw new Error("Injected Redis outage before batch recovery"); };
serverMod.setRedisClientForTests(redis);
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
serverMod.bufferQueue.length = 0;
let socket: Socket | undefined;
try {
const client = await SDKManager.registerClientAsync({
clientId: "p02_mixed_batch_claim", name: "P02 Mixed Batch Claim", rateLimit: 100,
});
socket = await connectSocket(client.token);
const pending = { eventId: "p02-mixed-pending-01", data: { v: 1 } };
const first = await emitWithAck<{ success: boolean; code?: string }>(
socket, "sync_data", pending,
);
assert.equal(first.success, false);
// Simulated process RAM loss: Redis idempotency key survives; queue does not.
serverMod.bufferQueue.length = 0;
redis.rpush = originalRpush;
const mixed = await emitWithAck<{ success: boolean; code?: string; processed?: number }>(
socket, "batch_sync", [pending, { eventId: "p02-mixed-durable-02", data: { v: 2 } }],
);
assert.equal(mixed.success, false, "Mixed batch cannot falsely ACK the volatile duplicate");
assert.equal(mixed.code, "BUFFER_PERSIST_FAILED");
assert.equal(mixed.processed, 1, "A distinct new event can commit without masking the failed claim");
const rows = await redis.lrange("datastack:buffer:events", 0, -1);
assert.equal(rows.length, 1);
assert.equal((JSON.parse(rows[0]) as { eventId: string }).eventId, "p02-mixed-durable-02");
} finally {
socket?.disconnect();
serverMod.setRedisClientForTests(undefined);
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
}
});
test("P02 FINAL R-01 compatibility: deliberately Redis-free RAM mode remains unchanged", async () => {
const threshold = serverMod.runtimeConfig.batchFlushThreshold;
const queue = serverMod.bufferQueue.slice();
serverMod.setRedisClientForTests(null);
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
serverMod.bufferQueue.length = 0;
let socket: Socket | undefined;
try {
const client = await SDKManager.registerClientAsync({
clientId: "p02_redis_free_compat", name: "P02 Redis-free Compatibility", rateLimit: 100,
});
socket = await connectSocket(client.token);
const ack = await emitWithAck<{ success: boolean; eventId?: string }>(
socket, "sync_data", { eventId: "p02-ram-compat-001", data: { value: 1 } },
);
assert.equal(ack.success, true, "Preserve historical RAM-mode acceptance contract");
assert.equal(serverMod.bufferQueue.some((e) => e.eventId === "p02-ram-compat-001"), true);
} finally {
socket?.disconnect();
serverMod.setRedisClientForTests(undefined);
serverMod.runtimeConfig.batchFlushThreshold = threshold;
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...queue);
}
});
test("P02 FINAL R-01: pending claim plus durable duplicate must reject whole batch", async () => {
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
const previousQueue = serverMod.bufferQueue.slice();
const redis = createSharedMockRedis();
const healthyRpush = redis.rpush.bind(redis);
serverMod.setRedisClientForTests(redis);
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
serverMod.bufferQueue.length = 0;
let socket: Socket | undefined;
try {
const client = await SDKManager.registerClientAsync({
clientId: "p02_mixed_only_duplicates", name: "P02 Failed and Durable Duplicate", rateLimit: 100,
});
socket = await connectSocket(client.token);
const durable = { eventId: "p02-durable-duplicate", data: { v: 10 } };
const pending = { eventId: "p02-failed-duplicate", data: { v: 20 } };
const first = await emitWithAck<{ success: boolean }>(socket, "sync_data", durable);
assert.equal(first.success, true);
redis.rpush = async () => { throw new Error("Injected Redis failure for second event"); };
const rejected = await emitWithAck<{ success: boolean; code?: string }>(
socket, "sync_data", pending,
);
assert.equal(rejected.success, false);
assert.equal(rejected.code, "BUFFER_PERSIST_FAILED");
serverMod.bufferQueue.length = 0; // simulate volatile queue lost across restart
redis.rpush = healthyRpush;
const mixed = await emitWithAck<{
success: boolean; code?: string; processed?: number; duplicates?: number;
}>(socket, "batch_sync", [pending, durable]);
assert.equal(mixed.processed, 0);
assert.equal(mixed.duplicates, 1);
assert.equal(mixed.success, false,
"Existing durable duplicate cannot mask a non-durable pending claim");
assert.equal(mixed.code, "BUFFER_PERSIST_FAILED");
assert.equal((await redis.lrange("datastack:buffer:events", 0, -1)).length, 1);
} finally {
socket?.disconnect();
serverMod.setRedisClientForTests(undefined);
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
}
});
+126 -2
View File
@@ -1584,10 +1584,134 @@ test("P02 R-02 diagnostic: post-CSV Redis trim failure replays identical event o
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 1, "Redis still queues previously appended CSV event"); assert.equal(await redis.llen(REDIS_BUFFER_KEY), 1, "Redis still queues previously appended CSV event");
await DataBufferEngine.flush(); await DataBufferEngine.flush();
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0); assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0);
assert.equal(writtenIds.filter((id) => id === event.eventId).length, 2, assert.equal(writtenIds.filter((id) => id === event.eventId).length, 1,
"A second flush appends the same eventId again after failed trim: at-least-once, not exactly-once"); "Replay must detect the persisted CSV eventId before attempting another append");
const csvPath = path.join(tmpStorageDir, event.clientId, "2026-11-22.csv");
const csvContents = fs.readFileSync(csvPath, "utf8");
assert.equal(csvContents.split(event.eventId).length - 1, 1,
"Exactly one physical CSV row must exist after a failed LTRIM and retry");
} finally { } finally {
CSVStorageManager.appendBatch = originalAppend; CSVStorageManager.appendBatch = originalAppend;
DataBufferEngine.setRedisClientForTests(undefined); DataBufferEngine.setRedisClientForTests(undefined);
} }
}); });
test("P02 R-02: Port 4000 CSV worker suppresses a crash-window replay after Redis LTRIM error", async () => {
const mockRedis = new FakeRedisClient();
const originalTrim = mockRedis.ltrim.bind(mockRedis);
let failedOnce = false;
mockRedis.ltrim = async (key: string, start: number, stop: number): Promise<"OK"> => {
if (key === REDIS_BUFFER_KEY && !failedOnce) {
failedOnce = true;
throw new Error("P02 Port4000 injected post-CSV trim failure");
}
return originalTrim(key, start, stop);
};
const ingestion = require("../server/index.js") as IngestionServerModule;
const previousDir = ingestion.writer.baseDir;
const previousQueue = ingestion.bufferQueue.slice();
ingestion.writer.baseDir = tmpStorageDir;
ingestion.bufferQueue.length = 0;
ingestion.setRedisClientForTests(mockRedis);
const evt: DataEvent = {
clientId: "p02_ingestion_replay",
eventId: "p02-port4000-001",
timestamp: "2026-11-22T13:00:00.000Z",
payload: JSON.stringify({ stage: "before-ltrim" }),
};
try {
await mockRedis.rpush(REDIS_BUFFER_KEY, JSON.stringify(evt));
await assert.rejects(
() => ingestion.flushBuffer(),
/P02 Port4000 injected post-CSV trim failure/,
);
assert.equal(failedOnce, true);
assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 1);
const csvPath = path.join(tmpStorageDir, evt.clientId, "2026-11-22.csv");
assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1);
const recovered = await ingestion.flushBuffer();
assert.equal(recovered.written, 0, "Recovered replay should not append the event again");
assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 0);
assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1);
} finally {
ingestion.writer.baseDir = previousDir;
ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...previousQueue);
ingestion.setRedisClientForTests(undefined);
}
});
test("P02 FINAL R-02: Port 4000 scans rotated compressed CSV with quoted multiline payload", async () => {
const zlib = await import("node:zlib");
const redis = new FakeRedisClient();
const ingestion = require("../server/index.js") as IngestionServerModule;
const oldDir = ingestion.writer.baseDir;
const oldQueue = ingestion.bufferQueue.slice();
ingestion.writer.baseDir = tmpStorageDir;
ingestion.bufferQueue.length = 0;
ingestion.setRedisClientForTests(redis);
const event: DataEvent = {
clientId: "p02_rotated_gzip",
eventId: "p02-gzip-quoted-001",
timestamp: "2026-11-22T12:45:00.000Z",
payload: JSON.stringify({ body: "Line one, \"quoted\"\nLine two" }),
};
try {
const diskWriter = new CsvWriterWorker(tmpStorageDir);
await diskWriter.writeBatch([event]);
const dir = path.join(tmpStorageDir, event.clientId);
const csvPath = path.join(dir, "2026-11-22.csv");
const rotatedPath = path.join(dir, "2026-11-22-part2.csv");
const gzPath = rotatedPath + ".gz";
fs.renameSync(csvPath, rotatedPath);
fs.writeFileSync(gzPath, zlib.gzipSync(fs.readFileSync(rotatedPath)));
fs.unlinkSync(rotatedPath);
// A different date partition must not be parsed as part of this replay.
fs.writeFileSync(path.join(dir, "2026-11-21.csv"), 'invalid,"unterminated');
await redis.rpush(REDIS_BUFFER_KEY, JSON.stringify(event));
const replay = await ingestion.flushBuffer();
assert.equal(replay.written, 0, "Already written rotated/gzipped event must not append again");
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0, "Successful dedupe permits Redis trim");
assert.equal(fs.existsSync(csvPath), false, "Do not recreate a duplicate plain CSV");
assert.equal(zlib.gunzipSync(fs.readFileSync(gzPath)).toString("utf8").split(event.eventId).length - 1, 1);
} finally {
ingestion.writer.baseDir = oldDir;
ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...oldQueue);
ingestion.setRedisClientForTests(undefined);
}
});
test("P02 FINAL R-02: partial corrupted CSV blocks Redis trim and recovers after fixture repair", async () => {
const redis = new FakeRedisClient();
const ingestion = require("../server/index.js") as IngestionServerModule;
const oldDir = ingestion.writer.baseDir;
const oldQueue = ingestion.bufferQueue.slice();
ingestion.writer.baseDir = tmpStorageDir;
ingestion.bufferQueue.length = 0;
ingestion.setRedisClientForTests(redis);
const event: DataEvent = {
clientId: "p02_partial_corrupt",
eventId: "p02-corrupt-recovery-001",
timestamp: "2026-11-22T13:15:00.000Z",
payload: JSON.stringify({ value: "safe" }),
};
try {
const dir = path.join(tmpStorageDir, event.clientId);
fs.mkdirSync(dir, { recursive: true });
const badPath = path.join(dir, "2026-11-22.csv");
fs.writeFileSync(badPath, 'clientId,eventId,timestamp,payload\n"unterminated');
await redis.rpush(REDIS_BUFFER_KEY, JSON.stringify(event));
await assert.rejects(() => ingestion.flushBuffer(),
/quote|CSV_|invalid|record|unterminated/i, "Malformed existing CSV must fail closed");
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 1, "Never trim after failed CSV replay check");
// Disposable fixture repair only; does not alter user or production files.
fs.unlinkSync(badPath);
const recovered = await ingestion.flushBuffer();
assert.equal(recovered.written, 1);
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0);
assert.equal(fs.readFileSync(badPath, "utf8").split(event.eventId).length - 1, 1);
} finally {
ingestion.writer.baseDir = oldDir;
ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...oldQueue);
ingestion.setRedisClientForTests(undefined);
}
});