Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a9ad770063 | ||
|
|
2d467ff149 | ||
|
|
fac0934a00 | ||
|
|
48fb277079 | ||
|
|
fd57a3518d |
No files matched your search
@@ -16,10 +16,15 @@ on:
|
|||||||
required: false
|
required: false
|
||||||
type: boolean
|
type: boolean
|
||||||
default: false
|
default: false
|
||||||
|
python_only:
|
||||||
|
description: Narrow CI-06 Python job rerun after a proven runner setup failure
|
||||||
|
required: false
|
||||||
|
type: boolean
|
||||||
|
default: false
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
node-regression:
|
node-regression:
|
||||||
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.quality_only != 'true' }}
|
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.quality_only != 'true' && (github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true') }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 30
|
timeout-minutes: 30
|
||||||
steps:
|
steps:
|
||||||
@@ -52,25 +57,36 @@ jobs:
|
|||||||
python-regression:
|
python-regression:
|
||||||
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.p01_only != 'true' }}
|
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.p01_only != 'true' }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 15
|
container: node:22-bookworm-slim
|
||||||
|
timeout-minutes: 20
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- name: Install Bookworm Python and checkout prerequisites
|
||||||
with:
|
|
||||||
github-server-url: https://dev.ivib.cc
|
|
||||||
- name: Install Python toolchain
|
|
||||||
run: |
|
run: |
|
||||||
set -eu
|
set -eu
|
||||||
apt-get update -qq
|
apt-get update -qq
|
||||||
apt-get install -y python3 python3-venv python3-pip
|
apt-get install -y --no-install-recommends git ca-certificates python3 python3-venv python3-pip
|
||||||
python3 -m venv /tmp/datastack-ci-venv
|
- uses: actions/checkout@v4
|
||||||
/tmp/datastack-ci-venv/bin/python -m pip install -e ./sdk/python/datastack -e ./sdk-python
|
with:
|
||||||
- name: Run all Python SDK tests
|
github-server-url: https://dev.ivib.cc
|
||||||
|
- name: CI-06 Python 3.11 isolated virtual environment
|
||||||
run: |
|
run: |
|
||||||
set -eu
|
set -eu
|
||||||
/tmp/datastack-ci-venv/bin/python -m compileall -q ./sdk/python/datastack ./sdk-python
|
python3 --version
|
||||||
/tmp/datastack-ci-venv/bin/python -m unittest discover -s tests -p '*_test.py' -v
|
python3 -m venv /tmp/datastack-ci-venv
|
||||||
|
/tmp/datastack-ci-venv/bin/python -m pip install --disable-pip-version-check -e ./sdk/python/datastack -e ./sdk-python
|
||||||
|
- name: CI-06 compile and assert 14 Python SDK regressions
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
py=/tmp/datastack-ci-venv/bin/python
|
||||||
|
"$py" -m compileall -q ./sdk/python/datastack ./sdk-python
|
||||||
|
"$py" -m unittest discover -s tests -p '*_test.py' -v > /tmp/datastack-python-tests.log 2>&1 || { cat /tmp/datastack-python-tests.log; exit 1; }
|
||||||
|
cat /tmp/datastack-python-tests.log
|
||||||
|
grep -Eq '^Ran 14 tests in ' /tmp/datastack-python-tests.log
|
||||||
|
grep -Eq '^OK$' /tmp/datastack-python-tests.log
|
||||||
|
echo DATASTACK_PYTHON_14_OF_14_PASS
|
||||||
|
|
||||||
quality:
|
quality:
|
||||||
|
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true' }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 20
|
timeout-minutes: 20
|
||||||
steps:
|
steps:
|
||||||
@@ -85,7 +101,7 @@ jobs:
|
|||||||
run: npm run lint
|
run: npm run lint
|
||||||
|
|
||||||
production-build:
|
production-build:
|
||||||
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.quality_only != 'true' }}
|
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.quality_only != 'true' && (github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true') }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 25
|
timeout-minutes: 25
|
||||||
steps:
|
steps:
|
||||||
@@ -101,30 +117,45 @@ jobs:
|
|||||||
test -f .next/standalone/server.js
|
test -f .next/standalone/server.js
|
||||||
|
|
||||||
docker-image-build:
|
docker-image-build:
|
||||||
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.p01_only != 'true' }}
|
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.p01_only != 'true' && (github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true') }}
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 35
|
container: node:22-alpine
|
||||||
|
timeout-minutes: 45
|
||||||
steps:
|
steps:
|
||||||
|
- name: CI-07 provision Docker Buildx CLI
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
apk add --no-cache git docker-cli docker-cli-buildx docker-cli-compose
|
||||||
|
docker --version
|
||||||
|
docker buildx version
|
||||||
|
docker compose version
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
github-server-url: https://dev.ivib.cc
|
github-server-url: https://dev.ivib.cc
|
||||||
- name: Docker and Compose availability
|
- name: CI-07 inspect daemon and validate Compose without starting services
|
||||||
run: |
|
run: |
|
||||||
set -eu
|
set -eu
|
||||||
docker --version
|
docker info --format 'Driver={{.Driver}} Root={{.DockerRootDir}}'
|
||||||
docker compose version
|
df -h .
|
||||||
docker info
|
docker compose -f docker/docker-compose.yml config --no-interpolate --quiet
|
||||||
- name: Validate Compose structure
|
echo DATASTACK_COMPOSE_CONFIG_PASS
|
||||||
run: docker compose -f docker/docker-compose.yml config --no-interpolate --quiet
|
- name: CI-07 isolated BuildKit all three images (no deployment)
|
||||||
- name: Build Docker images without running services
|
|
||||||
run: |
|
run: |
|
||||||
set -eu
|
set -eu
|
||||||
docker build -f docker/Dockerfile.server -t datastack-ci-ingestion .
|
builder="datastack-p03-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}"
|
||||||
docker build -f docker/Dockerfile.web -t datastack-ci-web .
|
tag="datastack-p03-${GITHUB_SHA:-local}"
|
||||||
docker build -f docker/Dockerfile.proxy -t datastack-ci-proxy .
|
cleanup() { docker buildx rm --force "$builder" >/dev/null 2>&1 || true; }
|
||||||
|
trap cleanup EXIT
|
||||||
|
docker buildx create --name "$builder" --driver docker-container --use
|
||||||
|
docker buildx inspect "$builder" --bootstrap
|
||||||
|
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.server -t "$tag-ingestion" .
|
||||||
|
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.web -t "$tag-web" .
|
||||||
|
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.proxy -t "$tag-proxy" .
|
||||||
|
docker image inspect "$tag-ingestion" "$tag-web" "$tag-proxy" >/dev/null
|
||||||
|
echo DATASTACK_THREE_IMAGES_BUILD_PASS
|
||||||
|
|
||||||
windows-compatibility:
|
windows-compatibility:
|
||||||
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.quality_only != 'true' }}
|
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.quality_only != 'true' && (github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true') }}
|
||||||
runs-on: windows-latest
|
runs-on: windows-latest
|
||||||
timeout-minutes: 25
|
timeout-minutes: 25
|
||||||
steps:
|
steps:
|
||||||
@@ -150,3 +181,53 @@ jobs:
|
|||||||
if ($LASTEXITCODE -ne 0) { throw 'SDK install failed' }
|
if ($LASTEXITCODE -ne 0) { throw 'SDK install failed' }
|
||||||
npm --prefix sdk/node/datastack run build
|
npm --prefix sdk/node/datastack run build
|
||||||
if ($LASTEXITCODE -ne 0) { throw 'SDK compilation failed' }
|
if ($LASTEXITCODE -ne 0) { throw 'SDK compilation failed' }
|
||||||
|
|
||||||
|
security-audit:
|
||||||
|
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.p01_only != 'true' && (github.event_name != 'workflow_dispatch' || github.event.inputs.python_only != 'true') }}
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
container: node:22-alpine
|
||||||
|
timeout-minutes: 15
|
||||||
|
steps:
|
||||||
|
- name: Install checkout prerequisite
|
||||||
|
run: apk add --no-cache git
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
github-server-url: https://dev.ivib.cc
|
||||||
|
- name: CI-08 read-only full and runtime npm advisory inventory
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
if npm audit --json > /tmp/datastack-audit-full.json; then :; else test "$?" -eq 1; fi
|
||||||
|
if npm audit --omit=dev --json > /tmp/datastack-audit-runtime.json; then :; else test "$?" -eq 1; fi
|
||||||
|
node <<'NODE'
|
||||||
|
const fs = require("node:fs");
|
||||||
|
for (const scope of ["full", "runtime"]) {
|
||||||
|
const audit = JSON.parse(fs.readFileSync("/tmp/datastack-audit-" + scope + ".json", "utf8"));
|
||||||
|
if (!audit.metadata || !audit.metadata.vulnerabilities || !audit.vulnerabilities) {
|
||||||
|
throw Error("Incomplete npm audit response for " + scope);
|
||||||
|
}
|
||||||
|
console.log("AUDIT_COUNTS " + scope + " " + JSON.stringify(audit.metadata.vulnerabilities));
|
||||||
|
for (const [name, item] of Object.entries(audit.vulnerabilities)) {
|
||||||
|
const advisories = (item.via || []).filter(v => typeof v === "object").map(v => ({
|
||||||
|
id: v.source, url: v.url, range: v.range, severity: v.severity
|
||||||
|
}));
|
||||||
|
console.log("AUDIT_PACKAGE " + scope + " " + JSON.stringify({
|
||||||
|
package: name, severity: item.severity, direct: item.isDirect,
|
||||||
|
range: item.range, via: advisories,
|
||||||
|
viaPackages: (item.via || []).filter(v => typeof v === "string"),
|
||||||
|
fixAvailable: item.fixAvailable
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
if (scope === "runtime") {
|
||||||
|
const counts = audit.metadata.vulnerabilities;
|
||||||
|
if (!Number.isInteger(counts.high) || !Number.isInteger(counts.critical)) {
|
||||||
|
throw Error("Incomplete runtime npm advisory severity counts");
|
||||||
|
}
|
||||||
|
if (counts.high > 0 || counts.critical > 0) {
|
||||||
|
throw Error("DATASTACK_RUNTIME_SECURITY_GATE_FAILED high=" + counts.high + " critical=" + counts.critical);
|
||||||
|
}
|
||||||
|
console.log("DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
NODE
|
||||||
|
git diff --exit-code -- package.json package-lock.json
|
||||||
|
echo DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS
|
||||||
@@ -1,39 +1,41 @@
|
|||||||
name: DataStack Docker Build CI
|
name: DataStack Docker Build CI
|
||||||
on:
|
on:
|
||||||
push:
|
|
||||||
branches: [main]
|
|
||||||
paths:
|
|
||||||
- '.gitea/workflows/datastack-docker-ci.yml'
|
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
docker-build:
|
docker-build:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
container: node:22-alpine
|
container: node:22-alpine
|
||||||
timeout-minutes: 40
|
timeout-minutes: 45
|
||||||
steps:
|
steps:
|
||||||
- name: Provision isolated Docker CLI and Git tools
|
- name: CI-07 provision Docker Buildx CLI
|
||||||
run: |
|
run: |
|
||||||
set -eu
|
set -eu
|
||||||
apk add --no-cache git docker-cli docker-cli-compose
|
apk add --no-cache git docker-cli docker-cli-buildx docker-cli-compose
|
||||||
docker --version
|
docker --version
|
||||||
|
docker buildx version
|
||||||
docker compose version
|
docker compose version
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
github-server-url: https://dev.ivib.cc
|
github-server-url: https://dev.ivib.cc
|
||||||
- name: Validate actual Compose configuration
|
- name: CI-07 inspect daemon and validate Compose without starting services
|
||||||
run: |
|
run: |
|
||||||
set -eu
|
set -eu
|
||||||
|
docker info --format 'Driver={{.Driver}} Root={{.DockerRootDir}}'
|
||||||
|
df -h .
|
||||||
docker compose -f docker/docker-compose.yml config --no-interpolate --quiet
|
docker compose -f docker/docker-compose.yml config --no-interpolate --quiet
|
||||||
echo DATASTACK_COMPOSE_CONFIG_PASS
|
echo DATASTACK_COMPOSE_CONFIG_PASS
|
||||||
- name: Check CI-only Docker daemon
|
- name: CI-07 isolated BuildKit all three images (no deployment)
|
||||||
run: |
|
run: |
|
||||||
set -eu
|
set -eu
|
||||||
docker info >/dev/null || { echo DOCKER_DAEMON_UNAVAILABLE; exit 3; }
|
builder="datastack-p03-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}"
|
||||||
- name: Build all DataStack container images without deployment
|
tag="datastack-p03-${GITHUB_SHA:-local}"
|
||||||
run: |
|
cleanup() { docker buildx rm --force "$builder" >/dev/null 2>&1 || true; }
|
||||||
set -eu
|
trap cleanup EXIT
|
||||||
docker build -f docker/Dockerfile.server -t datastack-ci-ingestion .
|
docker buildx create --name "$builder" --driver docker-container --use
|
||||||
docker build -f docker/Dockerfile.web -t datastack-ci-web .
|
docker buildx inspect "$builder" --bootstrap
|
||||||
docker build -f docker/Dockerfile.proxy -t datastack-ci-proxy .
|
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.server -t "$tag-ingestion" .
|
||||||
echo DATASTACK_DOCKER_IMAGES_PASS
|
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.web -t "$tag-web" .
|
||||||
|
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.proxy -t "$tag-proxy" .
|
||||||
|
docker image inspect "$tag-ingestion" "$tag-web" "$tag-proxy" >/dev/null
|
||||||
|
echo DATASTACK_THREE_IMAGES_BUILD_PASS
|
||||||
+68
-2
@@ -650,6 +650,66 @@ export class DataBufferEngine {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const executeFlush = async (): Promise<FlushResult> => {
|
const executeFlush = async (): Promise<FlushResult> => {
|
||||||
|
|
||||||
|
// P02 R-02: examine CSV partitions before replay of an untrimmed Redis
|
||||||
|
// batch. Streaming parsing limits memory and handles escaped/newline CSV.
|
||||||
|
const excludeAlreadyPersisted = async (events: DataEvent[]): Promise<DataEvent[]> => {
|
||||||
|
if (!events.length) return events;
|
||||||
|
const fs = await import('node:fs');
|
||||||
|
const path = await import('node:path');
|
||||||
|
const zlib = await import('node:zlib');
|
||||||
|
const { pipeline } = await import('node:stream/promises');
|
||||||
|
const { Writable } = await import('node:stream');
|
||||||
|
const { parse } = await import('csv-parse');
|
||||||
|
const groups = new Map<string, {
|
||||||
|
clientId: string; date: string; ids: Set<string>;
|
||||||
|
}>();
|
||||||
|
for (const event of events) {
|
||||||
|
const clientId = String(event.clientId || 'unknown');
|
||||||
|
const date = CSVStorageManager.extractSafePartitionDate(event.timestamp);
|
||||||
|
const key = JSON.stringify([clientId, date]);
|
||||||
|
if (!groups.has(key)) groups.set(key, { clientId, date, ids: new Set() });
|
||||||
|
groups.get(key)!.ids.add(String(event.eventId));
|
||||||
|
}
|
||||||
|
const persisted = new Set<string>();
|
||||||
|
for (const [groupKey, group] of groups) {
|
||||||
|
const dir = CSVStorageManager.ensureClientDir(group.clientId);
|
||||||
|
const entries = await fs.promises.readdir(dir, { withFileTypes: true });
|
||||||
|
for (const entry of entries) {
|
||||||
|
if (!entry.isFile() ||
|
||||||
|
!/\.csv(?:\.gz)?$/.test(entry.name) ||
|
||||||
|
CSVStorageManager.extractSafePartitionDate(
|
||||||
|
entry.name.replace(/\.csv(?:\.gz)?$/, '').replace(/(?:-(?:part|gz)\d+)+$/, ''),
|
||||||
|
) !== group.date) continue;
|
||||||
|
const reader = fs.createReadStream(path.join(dir, entry.name));
|
||||||
|
const parser = parse({ columns: true, bom: true, skip_empty_lines: true });
|
||||||
|
const sink = new Writable({
|
||||||
|
objectMode: true,
|
||||||
|
write(row: { clientId?: string; eventId?: string }, _enc, done) {
|
||||||
|
if (row.clientId === group.clientId && group.ids.has(String(row.eventId))) {
|
||||||
|
persisted.add(JSON.stringify([groupKey, String(row.eventId)]));
|
||||||
|
}
|
||||||
|
done();
|
||||||
|
},
|
||||||
|
});
|
||||||
|
if (entry.name.endsWith('.gz')) {
|
||||||
|
await pipeline(reader, zlib.createGunzip(), parser, sink);
|
||||||
|
} else {
|
||||||
|
await pipeline(reader, parser, sink);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const batchSeen = new Set<string>();
|
||||||
|
return events.filter((event) => {
|
||||||
|
const key = JSON.stringify([String(event.clientId || 'unknown'),
|
||||||
|
CSVStorageManager.extractSafePartitionDate(event.timestamp)]);
|
||||||
|
const rowKey = JSON.stringify([key, String(event.eventId)]);
|
||||||
|
if (persisted.has(rowKey) || batchSeen.has(rowKey)) return false;
|
||||||
|
batchSeen.add(rowKey);
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
state.isFlushing = true;
|
state.isFlushing = true;
|
||||||
let totalWritten = 0;
|
let totalWritten = 0;
|
||||||
let lockTimedOut = false;
|
let lockTimedOut = false;
|
||||||
@@ -793,10 +853,13 @@ export class DataBufferEngine {
|
|||||||
if (lockLost) break;
|
if (lockLost) break;
|
||||||
|
|
||||||
if (parsedBatch.length > 0) {
|
if (parsedBatch.length > 0) {
|
||||||
const res = await CSVStorageManager.appendBatch(parsedBatch);
|
const unwritten = await excludeAlreadyPersisted(parsedBatch);
|
||||||
|
if (unwritten.length > 0) {
|
||||||
|
const res = await CSVStorageManager.appendBatch(unwritten);
|
||||||
totalWritten += res.written;
|
totalWritten += res.written;
|
||||||
for (const f of res.filesUpdated) filesSet.add(f);
|
for (const f of res.filesUpdated) filesSet.add(f);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (Date.now() - lastRenewAt >= renewIntervalMs) {
|
if (Date.now() - lastRenewAt >= renewIntervalMs) {
|
||||||
await renewLockOnce();
|
await renewLockOnce();
|
||||||
@@ -845,9 +908,12 @@ export class DataBufferEngine {
|
|||||||
|
|
||||||
if (validRamBatch.length > 0) {
|
if (validRamBatch.length > 0) {
|
||||||
try {
|
try {
|
||||||
const res = await CSVStorageManager.appendBatch(validRamBatch);
|
const unwritten = await excludeAlreadyPersisted(validRamBatch);
|
||||||
|
if (unwritten.length > 0) {
|
||||||
|
const res = await CSVStorageManager.appendBatch(unwritten);
|
||||||
totalWritten += res.written;
|
totalWritten += res.written;
|
||||||
for (const f of res.filesUpdated) filesSet.add(f);
|
for (const f of res.filesUpdated) filesSet.add(f);
|
||||||
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
state.buffer.unshift(...validRamBatch);
|
state.buffer.unshift(...validRamBatch);
|
||||||
throw err;
|
throw err;
|
||||||
|
|||||||
@@ -89,3 +89,81 @@ Source freeze for P02: Gitea/GitHub main `667a140c4e9e75382bfdcdbe1d3e9eb96f6e80
|
|||||||
- **R-03 health:** `server/index.js:1926-1957` returns HTTP 200 `status:online` when Redis LLEN fails, while surfacing `redisConnected:false`; the historical route is **liveness**, not dependency readiness. `app/api/health/route.ts:95-111` likewise reports web liveness/buffer and does not prove database/Redis write readiness. A new readiness endpoint or altered public response requires separate approval.
|
- **R-03 health:** `server/index.js:1926-1957` returns HTTP 200 `status:online` when Redis LLEN fails, while surfacing `redisConnected:false`; the historical route is **liveness**, not dependency readiness. `app/api/health/route.ts:95-111` likewise reports web liveness/buffer and does not prove database/Redis write readiness. A new readiness endpoint or altered public response requires separate approval.
|
||||||
|
|
||||||
Do not auto-fix source based solely on tests. Runtime code is READ-ONLY under the current approval. Any proven R-01/R-02 risk requiring behavior change needs a new exact method/file approval before application changes. P03 remains independent and unstarted.
|
Do not auto-fix source based solely on tests. Runtime code is READ-ONLY under the current approval. Any proven R-01/R-02 risk requiring behavior change needs a new exact method/file approval before application changes. P03 remains independent and unstarted.
|
||||||
|
|
||||||
|
|
||||||
|
## P02 Runtime Remediation Candidate — 2026-11-22 (NOT YET VERIFIED)
|
||||||
|
|
||||||
|
Authorized follow-up baseline: Gitea/GitHub `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898` (GitLab same tree). Local isolated branch `p02-runtime-remediation-20261122`, implementation commit `44eee8877e40d1683f44502c2a0ba3d437a03b88`. Scope: **server/index.js** (`pushEventsToRedisOrFallback`, `flushBuffer`), **server/socket/handlers.js** (sync_data/batch_sync ACK and claim rollback), **core/data/buffer.ts** (`flush`), **tests/phase02-sync.test.ts** and **tests/phase03-storage-buffer.test.ts** only, plus notes/001–006. All other source is unchanged.
|
||||||
|
|
||||||
|
**Candidate R-01 behavior:** when a configured Redis client returns an RPUSH error, `pushEventsToRedisOrFallback` retains RAM fallback but throws a sanitized `BUFFER_PERSIST_FAILED` instead of returning a durable-looking buffer count. Socket single/batch ACK returns `success:false`; best-effort Redis idempotency-key removal and bounded in-process negative-ACK marker handle ordinary retries. This is a **candidate** pending tests. When Redis is intentionally absent, historical RAM fallback is deliberately unchanged; it is not a universal durable-ACK guarantee. If Redis claim deletion fails and the server restarts, stale Redis idempotency claims could still misclassify a retry; a full crash-atomic claim/queue protocol is **not established** by this patch.
|
||||||
|
|
||||||
|
**Candidate R-02 behavior:** Port 4000 `flushBuffer` and Port 3000 `DataBufferEngine.flush` stream-parse existing CSV/CSV.gz partitions for same-client/date/event IDs before replaying any buffered Redis or RAM batch, dropping already-persisted IDs while retaining Redis queue until the write/check completes. Scan/read/parse exceptions fail closed before LTRIM. In-scope test adds a direct Port 4000 writer + Redis LTRIM-failure/recovery test and updates the existing web-buffer replay fixture. **Not proven globally exactly-once:** cross-process CSV writers outside the Redis flush lock and unknown archival/partial-corruption cases remain; scanning partitions on each flush also has real disk-I/O cost, requiring throughput evaluation before production promotion.
|
||||||
|
|
||||||
|
**Candidate acceptance:** one disposable Gitea CI with 24/24 suites, strict TS, lint, Linux/Windows clean npm CI and production build; direct Port4000 replay tests; exact allowlist/diff audit and three-remote tree parity. The Gitea MCP/host was unavailable on the implementation attempt, so this is **NOT PASSED** and the P02 phase remains **BLOCKED / NOT CLOSED**. No live deployment or runtime production mutation.
|
||||||
|
|
||||||
|
|
||||||
|
## P02 Final Acceptance Verification — 2026-11-22 (current superseding evidence)
|
||||||
|
|
||||||
|
Baseline published diagnosis: `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`. Runtime candidate `76cd972bdeea9ef07b4b38e9d8b5b5ed3312ca5b` passed Gitea CI #66 (24/24 Node regression suites, Node SDK, strict TypeScript, ESLint, Next production build, Windows SDK). New final narrow patch `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532`: `server/socket/handlers.js` plus two previously scoped TS tests only; final disposable Gitea CI #69 dispatched, terminal result to be recorded before phase closure.
|
||||||
|
|
||||||
|
Confirmed exact root cause in candidate: `server/socket/handlers.js` stores failed Redis persistence in `idempotencyMap` at `-Date.now()` while `cleanupMaps` deletes entries when `now - timestamp > 300000`; after the 60000ms cleanup window a failed marker can disappear despite an uncommitted Redis idempotency claim, permitting false-positive `DUPLICATE_DROPPED`. Arithmetic reproduction before code patch: at +61000ms, cleanup deletes a marker although Redis claim TTL (300000ms) is not over. Final patch uses a separate positive-timestamp failed-persistence marker and preserves negative ACK while the outstanding claim remains. A mixed batch with a failed duplicate must not return overall success while distinct records persist.
|
||||||
|
|
||||||
|
R-02: `server/index.js:843-897,1043-1058` and `core/data/buffer.ts:652-711,855-872` replay-scan existing same-client/day CSV, preserving Redis queue until a successful write/dedupe; fault tests include direct Port 4000 post-CSV LTRIM outage, rotated compressed .csv.gz containing quoted multiline payload and unrelated-date corrupt partition; malformed matching-date CSV must reject and leave Redis queue intact, with recovery after disposable fixture repair. Streaming parsing bounds memory, **not total disk I/O**; scan latency grows with matching partition size. No universal exactly-once guarantee, no live throughput claim and no claim of finite hard disk-I/O cap.
|
||||||
|
|
||||||
|
Public R-03 health remains historical liveness. Intentionally Redis-free RAM fallback and its historical successful ACK stay unchanged. No app/UI/auth/API schema/SDK/Docker/production-data/deployment changes. Work only in a new clean temp worktree; original PC repository's staged 16 P01 files and a separate pre-existing dirty candidate worktree preserved. No production readiness claim until CI #69 and final audit.
|
||||||
|
|
||||||
|
|
||||||
|
### P02 Final Focused Correction — CI #69 to #72
|
||||||
|
|
||||||
|
Gitea [CI #69](https://dev.ivib.cc/vibtools/DataStack/actions/runs/69) was run against `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532`: the new Redis stale-claim test timed out because it globally replaced `Date.now` during live Socket.IO ACK scheduling. This is a **test-harness failure**, not a proven backend regression; other new CSV.gz, partial-corruption, mixed-new/durable and Redis-free regression cases passed. An additional source-grounded edge case was found: `batch_sync` previously failed to reject a batch containing a non-durable claim PLUS an already accepted duplicate when `processed=0, duplicates>0`. Under exact approved scope, the `server/socket/handlers.js` branch changed from `if (volatileDuplicate && processed > 0)` to `if (volatileDuplicate)`, ensuring false overall ACK is never sent for any such mixture. Test now uses local simulated TTL arithmetic and real-clock Socket.IO retries; no global clock manipulation. Corrected commit `acca8be01a31f8d3972564cbaec8fd3117a0b551` sent to [focused CI #72](https://dev.ivib.cc/vibtools/DataStack/actions/runs/72); still no P02 closure or production deployment until PASS.
|
||||||
|
|
||||||
|
|
||||||
|
### P02 definitive CI acceptance (2026-11-22)
|
||||||
|
|
||||||
|
Source-verified P02 final correction is `acca8be01a31f8d3972564cbaec8fd3117a0b551`. [Gitea CI #72](https://dev.ivib.cc/vibtools/DataStack/actions/runs/72) terminal **SUCCESS** (Linux 24/24 TypeScript suites, explicit R-01/R-02 fault cases, strict tsc, ESLint, Windows Node SDK, Next.js standalone build). Earlier #69 was cancelled after a test-only global `Date.now` Socket.IO ACK timeout; fixed by removing global clock override and adding direct arithmetic proof without changing timers. #72 proves the corrected test passes. Final source-path audit: exactly 11 approved paths (3 runtime, 2 tests, 6 notes). No keys/secrets, auth/SDK/UI/public health, dependency pins, CI workflows, deployment or database migrations changed. ACK remains **best-effort in intentionally Redis-free RAM mode**; Redis-configured RPUSH failure must return negative ACK; CSV deduplication guarantees only tested recovery windows, not globally atomic exactly-once. Partition scans are memory-streamed but disk reads scale with matching partition size, and production throughput is NOT certified. Release/installation still requires separate authorization.
|
||||||
|
|
||||||
|
## P03 production forensic acceptance — 2026-11-22
|
||||||
|
|
||||||
|
Baseline Gitea/GitHub `48fb277079b380fdbf1e58cd2da40afeed0b03ee`; GitLab baseline same tracked tree `41a169eec0c05ce19bc4acc29d7a8cfe1f8dd90d` (317/317 blobs). Final P03 workflow candidate `549d575653b68c29f97e2f78036646f2e2035719`. [Gitea CI #77](https://dev.ivib.cc/vibtools/DataStack/actions/runs/77): overall **FAILURE** because the initial Python-only checkout lacked Node (exit 127), **not** an SDK regression. Its other six jobs terminal **SUCCESS**: Node `CI_SUITE_SUMMARY total=24 failed=0`, strict TS+ESLint, Next standalone build, Windows SDK, Compose + isolated BuildKit ingestion/web/proxy image build `DATASTACK_THREE_IMAGES_BUILD_PASS`, read-only advisory inventory `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Narrow [Python-only CI #78](https://dev.ivib.cc/vibtools/DataStack/actions/runs/78) terminal **SUCCESS**: Node-capable Bookworm container, Python 3.11 isolated venv, `Ran 14 tests in 3.959s`, `OK`, `DATASTACK_PYTHON_14_OF_14_PASS`. The other six jobs were deliberately skipped in #78.
|
||||||
|
|
||||||
|
**CI-08 security disposition:** `npm audit --json` full=26 affected packages (25 moderate, 1 high); runtime=24 (23 moderate, 1 high), **not counts of distinct advisories**. High-severity advisories [GHSA-6g55-p6wh-862q](https://github.com/advisories/GHSA-6g55-p6wh-862q), [GHSA-r28c-9q8g-f849](https://github.com/advisories/GHSA-r28c-9q8g-f849) affect Next 15.5.29's nested `postcss@8.4.31` despite patched root `postcss@8.5.33`; npm recommends major `next@16.4.6`. Direct `cloudcmd@19.24.0` carries moderate transitive issues (`dompurify`, `file-type`, `glob`, `minimatch`, `got`); npm recommends `cloudcmd@17.0.2` downgrade, not a verified safe pin. No speculative package pin, major upgrade or force fix; a separately tested compatible remediation is needed before release.
|
||||||
|
|
||||||
|
**Scope / release decision:** only `.gitea/workflows/datastack-ci.yml`, `.gitea/workflows/datastack-docker-ci.yml`, and `notes/001–006` changed. No `package.json`/lock, Dockerfile, application UI/UX, runtime, auth, SDK, database, Compose, production data, secret, shared Docker prune, deployment, migration, tag or release. P03 technical CI and advisory-triage acceptance verified by #77 non-Python jobs plus #78 Python; do **not** mislabel #77 overall as successful. Final phase publication requires one main commit and Gitea/GitHub/GitLab exact tracked tree parity. **Production release NO-GO** while runtime high advisory is unresolved; live production health, throughput and deployment remain unverified.
|
||||||
|
|
||||||
|
## P04-S01 blocked security reconciliation report
|
||||||
|
|
||||||
|
P04-S01 SEC-01 BLOCKED — 2026-11-22. Approved scoped `overrides.next.postcss=8.5.33` trialed in disposable Windows npm 10.9.8 workspace only. `npm install --package-lock-only` EXIT 0, but `npm ls --package-lock-only postcss --all` EXIT 1 (`ELSPROBLEMS`: Next nested `postcss@8.4.31` invalid; expected 8.5.33); npm dedupe also kept invalid dependency. Read-only `npm audit --omit=dev --json` EXIT 1, runtime high 1/moderate 23/critical 0. Lockfile regeneration caused unrelated churn (91 package entries added, 106 removed, 69 changed, including two lightningcss optional 1.33.0 -> 1.30.1). Fresh minimal Next 15.5.29 fixture with identical override resolves PostCSS 8.5.33, isolating failure to DataStack committed lock graph. `npm ci --dry-run` exit 0 does not prove real clean install. Exact stop rule applied: no dependency patch, no CI acceptance, no main commit, no GitHub/GitLab sync, no release/deploy; production NO-GO. Need separate explicit narrowly scoped lock reconciliation approval.
|
||||||
|
|
||||||
|
Root proof: `package-lock.json:10552-10560` Next pins 8.4.31; `:10602-10603` nested 8.4.31; `:11427-11428` root 8.5.33; advisories GHSA-6g55-p6wh-862q and GHSA-r28c-9q8g-f849.
|
||||||
|
|
||||||
|
## P04-S01B actual trial / release blocker — 2026-11-22
|
||||||
|
|
||||||
|
User explicitly approved P04-S01B stale-copy removal exception. In a clean disposable Windows npm 10.9.8 probe based on Gitea main `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3`, remove ONLY the stale nested PostCSS 8.4.31 entry from the disposable lock copy and set `overrides.next.postcss=8.5.33`. `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline` exited 0; `npm ls --package-lock-only postcss --all --depth=4` exited 0 and Next 15.5.29 used deduped 8.5.33. Read-only runtime audit on the **unpublished candidate** showed high=0, critical=0, moderate=22 (22 affected package entries); SEC-01 candidate mitigation confirmed but NOT accepted. The regenerated lock changed **67 unrelated package metadata entries** (36 `dev` and 29 `libc` field changes plus dependency order-only differences), despite zero unrelated version changes, and removed only the stale node. This violates explicit metadata preservation. STOP before real `npm ci`, Gitea CI, security-gate edit or main/mirror publication. No production release or security-clean claim for published baseline.
|
||||||
|
|
||||||
|
## P04-S01C verified candidate / Windows acceptance outstanding — 2026-11-22
|
||||||
|
|
||||||
|
Approved deterministic lockfile reconciliation fixed the **SEC-01 Next 15.5.29 nested PostCSS 8.4.31** candidate without touching any unrelated package metadata, versions, hashes, URLs or feature/runtime sources. Scoping: `package.json` scoped `overrides.next.postcss=8.5.33`; `package-lock.json` only 26-line removal of obsolete `node_modules/next/node_modules/postcss`; existing Gitea `security-audit` job adds runtime high/critical failure. Generated lockfile with deterministic baseline metadata restoration had SHA-256 `e6b5b362877627085dc04c9ded03c019971e07ef9f4af1e78057d58961a1c091` and Gitea branch bytes matched exactly. No unrelated package entry/metadata delta (0 added, 1 removed, 0 changed). Gitea [CI #85](https://dev.ivib.cc/vibtools/DataStack/actions/runs/85) candidate `c759be990c032c2474d2d0bad519c643dec11a3a`: Node 24/24, Python 14/14, strict TypeScript + ESLint, Next standalone, Docker 3/3 and runtime advisory gate all SUCCESS. Runtime audit **22 moderate, 0 high, 0 critical**; full audit **24 moderate, 0 high, 0 critical**. **Windows compatibility job still QUEUED at checkpoint; overall CI not yet terminal accepted, main and mirrors unchanged.** Production remains NO-GO pending Windows CI, 3-remote sync, remaining moderate advisories/live health.
|
||||||
|
|
||||||
|
## P04-S01C FINAL CI acceptance (7/7) — 2026-11-22
|
||||||
|
|
||||||
|
Gitea CI #85 https://dev.ivib.cc/vibtools/DataStack/actions/runs/85 COMPLETED/SUCCESS; tested implementation SHA c759be990c032c2474d2d0bad519c643dec11a3a; later changes to candidate are documentation only. All seven jobs terminal SUCCESS: node-regression (Linux committed-lock npm ci and exactly 24/24 TypeScript suites); python-regression (Ran 14 tests, OK); quality (strict tsc and ESLint); production-build (Next 15.5.29 standalone, 22/22 pages); docker-image-build (three isolated BuildKit ingestion/web/proxy images); windows-compatibility (Windows committed-lock npm ci and Node SDK build, job #182, terminal SUCCESS); security-audit (runtime moderate=22 high=0 critical=0; full moderate=24 high=0 critical=0; DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO). Single Gitea acceptance run used, no additional broad reruns.
|
||||||
|
|
||||||
|
Scope: exactly nine approved paths: package.json scoped next->postcss 8.5.33 override; package-lock.json removes only old nested postcss@8.4.31 entry; existing .gitea/workflows/datastack-ci.yml security-audit job high/critical rejection; notes/001–006 evidence. Other package metadata, versions, integrity and resolved URLs match baseline. Next.js 15.5.29 and Cloudcmd 19.24.0 preserved. Final candidate package-lock SHA256 e6b5b362877627085dc04c9ded03c019971e07ef9f4af1e78057d58961a1c091 (verified byte-for-byte with Gitea). No API, auth, SDK, UI, application runtime, database, Compose, Dockerfile, tests or production data changes, deployment, migration, prune, tag, release.
|
||||||
|
|
||||||
|
CI fully accepted; publication not yet asserted in these pre-merge notes. Final phase CLOSED requires one squash merge of PR #5 into Gitea main and verified identical tracked content across Gitea/GitHub/GitLab; record actual new SHAs in terminal report. Production release remains NO-GO pending P04-S02 moderate vulnerability triage and separately authorized live readiness verification.
|
||||||
|
|
||||||
|
## P04-S02 scoped candidate — 2026-11-22
|
||||||
|
|
||||||
|
Baseline Gitea main `2d467ff1494858164a37814f7dbc2b321f510291`, tree `648c76ca496d2c9bf22a16f558c9ba347a45d4f7`. Cloudcmd 19.24.0 -> deepword 11.0.1 -> monaco-editor 0.55.1 pins DOMPurify 3.2.7, vulnerable according to npm advisory inventory. Disposable Windows npm 10.9.8 `npm view dompurify@3.4.18` verified, scoped npm override `overrides.cloudcmd.dompurify=3.4.18` resolves `dompurify@3.4.18` via `npm ls --package-lock-only dompurify --all --depth=5` (exit 0). `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline` exit 0. npm graph initially changed 67 unrelated metadata entries; deterministically restored exact baseline values after validating unrelated versions, URLs, integrity and dependencies. Final lock has **one changed package entry**, DOMPurify; same-major replacement plus verified registry `resolved` and `integrity`, no other packages added/deleted/changed. Candidate runtime audit affected-package counts **moderate 20/high 0/critical 0** versus baseline **22/0/0**; full candidate **moderate 22/high 0/critical 0** versus baseline **24/0/0**. DOMPurify and Monaco Editor dropped from vulnerable-package inventory. Other moderate packages (got, glob, minimatch, nested file-type, Cloudcmd transitive chain) remain; they require major package changes under current advisory constraints, prohibited by Scope Lock. This is a **pre-acceptance candidate**: do not claim application compatibility, CI PASS, publication, or phase CLOSED before full focused Gitea CI and exact three-remote tree parity.
|
||||||
|
|
||||||
|
Security forensic disposition: only DOMPurify's same-major compatible override supported by the disposable registry/audit proof; other moderate transitive packages not force-fixed.
|
||||||
|
|
||||||
|
## P04-S02 final CI acceptance — 2026-11-22
|
||||||
|
|
||||||
|
Gitea CI #92 (https://dev.ivib.cc/vibtools/DataStack/actions/runs/92) **COMPLETED / SUCCESS**, all 7/7 jobs terminal success on source commit `a88bc5f4498ea1b11acc0acfeb66b5aa9d6531be`: node-regression real Linux npm ci + 24/24 tests including `tests/files-manager.test.ts` exit 0; Python SDK 14/14; quality strict TypeScript/ESLint; Next.js standalone (22/22 static pages); isolated Docker/BuildKit 3-image verification `DATASTACK_THREE_IMAGES_BUILD_PASS`; Windows committed-lock npm ci + Node SDK compatibility; security-audit `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO` and `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Audit full=22 moderate/high=0/critical=0 (baseline 24), runtime=20 moderate/high=0/critical=0 (baseline 22). These are affected package entries, not distinct advisories. No new CI workflow or extra broad CI dispatched.
|
||||||
|
|
||||||
|
Exact approved patch: `package.json` scoped `overrides.cloudcmd.dompurify=3.4.18`, preserving `overrides.next.postcss=8.5.33`; npm-generated `package-lock.json` changes solely `node_modules/dompurify` version 3.2.7 -> 3.4.18 and verified `resolved`/SHA512 integrity. Baseline Cloudcmd 19.24.0, Monaco Editor 0.55.1, Next.js 15.5.29, all unrelated locked package versions, URL, hash, dependency and metadata preserved. Gitea published candidate lock bytes match disposable SHA-256 `57c5ea9df9f28be496f8c3202fbc280b0066bf8c216b8d8a3ca98b6f7b942854`. Exact candidate tree diff 420 entries, 8 approved changed files only (`package.json`, `package-lock.json`, notes/001–006), 0 unauthorized paths. No runtime/UI/auth/API/SDK/tests/CI/Dockerfiles/Compose/db/schema/data modifications.
|
||||||
|
|
||||||
|
Security forensic result. **Residual risk:** 20 moderate runtime affected package entries remain, including Cloudcmd transitive dependencies `got`, `glob`, `minimatch`, nested `file-type` and their parent chains. Current npm audit suggests `cloudcmd@17.0.2` (a major downgrade), explicitly out-of-scope; other transitive fixes need major-version dependency-contract changes. No force/downgrade/unverified override. Security improvement verified but NOT zero vulnerabilities. Production release stays NO-GO; remaining advisory disposition and live readiness need independent scope approval.
|
||||||
|
|
||||||
|
**Merge gate:** CI #92 7/7 PASS, approved 8-file patch integrity PASS; main squash merge and Gitea/GitHub/GitLab exact Git content-tree verification required before P04-S02 CLOSED. This note is written before that publication, so final remote commit SHAs shall be appended to the PR closure report after verification. No deployment, migrations, prune, tag, release.
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
# DataStack — Production Update Roadmap / Exact Scope Lock
|
# DataStack — Production Update Roadmap / Exact Scope Lock
|
||||||
|
|
||||||
Planning baseline: `d725841ebc828d58cbe90f517da9e600f23c4630` (frozen; documentation commits will advance Git HEAD but NOT this source baseline).
|
Planning baseline: `d725841ebc828d58cbe90f517da9e600f23c4630` (frozen; documentation commits will advance Git HEAD but NOT this source baseline).
|
||||||
Current state: **P01 CLOSED and CI-verified (1 of 3 phases complete); P02 and P03 NOT STARTED; P02 requires new explicit approval**.
|
Current state: **P01 CLOSED; P02 CI ACCEPTED (Gitea #72 terminal SUCCESS), final three-remote publication gate; P03 NOT STARTED and requires separate explicit approval**. See latest P02 final acceptance appendix and publication report.
|
||||||
Prerequisite for any phase: compare current HEAD and changed paths with this baseline and previously committed notes. For new commits since source baseline, inspect exact diff; no implicit rebase or silent discard.
|
Prerequisite for any phase: compare current HEAD and changed paths with this baseline and previously committed notes. For new commits since source baseline, inspect exact diff; no implicit rebase or silent discard.
|
||||||
|
|
||||||
## Global non-negotiable Scope Lock (applies to ALL phases)
|
## Global non-negotiable Scope Lock (applies to ALL phases)
|
||||||
@@ -81,3 +81,98 @@ P02 approved as **diagnostic-only**. Published P01 baseline: `667a140c4e9e75382b
|
|||||||
`APPROVED — DataStack P02-RUNTIME-REMEDIATION. Baseline: use the verified P02 diagnostic commit/remote tree. Fix reproduced R-01 and R-02 only, following exact notes/001–006 proof and scoped server/index.js pushEventsToRedisOrFallback/onBufferPush and flushBuffer, server/socket/handlers.js sync_data/batch_sync ACK, and their three existing P02 test files. First document safe ACK/retry semantics, exactly-once versus at-least-once CSV contract and crash recovery. Add disposable fault tests proving no positive ACK for non-durable data and no unbounded duplicate replay. Runtime sources core/data/buffer.ts and server/workers/csvWriter.js remain READ-ONLY unless a separately justified exact file/line amendment is approved. Preserve all UI/auth/SDK/public health behavior and existing error codes, production data, security, other features and Docker. One focused CI, diff/security audit, notes update, commit and 3-remote parity. No deployment, migrations, tags/releases or scope expansion. If a safe fix cannot be achieved within these exact methods, stop and report the required additional Scope Lock rather than broadening automatically.`
|
`APPROVED — DataStack P02-RUNTIME-REMEDIATION. Baseline: use the verified P02 diagnostic commit/remote tree. Fix reproduced R-01 and R-02 only, following exact notes/001–006 proof and scoped server/index.js pushEventsToRedisOrFallback/onBufferPush and flushBuffer, server/socket/handlers.js sync_data/batch_sync ACK, and their three existing P02 test files. First document safe ACK/retry semantics, exactly-once versus at-least-once CSV contract and crash recovery. Add disposable fault tests proving no positive ACK for non-durable data and no unbounded duplicate replay. Runtime sources core/data/buffer.ts and server/workers/csvWriter.js remain READ-ONLY unless a separately justified exact file/line amendment is approved. Preserve all UI/auth/SDK/public health behavior and existing error codes, production data, security, other features and Docker. One focused CI, diff/security audit, notes update, commit and 3-remote parity. No deployment, migrations, tags/releases or scope expansion. If a safe fix cannot be achieved within these exact methods, stop and report the required additional Scope Lock rather than broadening automatically.`
|
||||||
|
|
||||||
After separate approved remediation and verified P02 closure, **P03 plan** remains CI-06 Linux Python runner provisioning, CI-07 isolated Docker/BuildKit failure remediation and 3-image build, CI-08 supply-chain advisory triage, and non-mutating readiness evidence. P03 should not modify unrelated UI/runtime/API code by default.
|
After separate approved remediation and verified P02 closure, **P03 plan** remains CI-06 Linux Python runner provisioning, CI-07 isolated Docker/BuildKit failure remediation and 3-image build, CI-08 supply-chain advisory triage, and non-mutating readiness evidence. P03 should not modify unrelated UI/runtime/API code by default.
|
||||||
|
|
||||||
|
|
||||||
|
## P02 Runtime Remediation Execution Handoff — 2026-11-22
|
||||||
|
|
||||||
|
The user explicitly approved **only** R-01 Redis failure ACK and R-02 Redis-to-CSV replay corrections in these methods: `server/index.js` `pushEventsToRedisOrFallback` / `flushBuffer` / existing `onBufferPush` integration; `server/socket/handlers.js` `sync_data` / `batch_sync` ACK handling; `core/data/buffer.ts` `flush` if needed; the three earlier P02 test files; `notes/001–006`. No other application method/UI/API/auth/SDK/Docker/CI-workflow edit. Public R-03 `/health` semantics remain unchanged.
|
||||||
|
|
||||||
|
Local candidate `44eee8877e40d1683f44502c2a0ba3d437a03b88` is **not release-ready** until Gitea acceptance. Before closure, prove: negative ACK on Redis failure with no sensitive error leakage; ordinary successful Redis ACK preserved; both single and batch retries don't convert volatile failures to success; Redis queued event replays after crash without writing duplicate CSV rows in the Port 4000 worker and Port 3000 web buffer; CSV read/parse failure never causes premature Redis trim; validate bounded scan overhead and edge cases for rotated/gzipped partitions; all historical 401/403/429/507/503 tests unchanged.
|
||||||
|
|
||||||
|
**Stop rule:** if required fixes need touching `validateSingleEventAsync`, `csvWriter.js` or separate journal/storage scheme, do not broaden the implementation scope; record the residual and request new explicit approval. Existing unrelated files including `MEMORY.md` remain unchanged due exact Scope Lock; this note is the cross-IDE handoff. P03 must wait for P02 CLOSED, not merely implementation committed.
|
||||||
|
|
||||||
|
|
||||||
|
## P02-FINAL-ACCEPTANCE-01 Execution / P03 Gate
|
||||||
|
|
||||||
|
User approved strict P02 final acceptance after `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898` published diagnosis and `76cd972bdeea9ef07b4b38e9d8b5b5ed3312ca5b` runtime candidate. Previously failed claim is negative timestamp cleaned with positive-timestamp TTL arithmetic; final patch changes only `server/socket/handlers.js` sync_data/batch_sync marker and mixed-batch handling, plus `tests/phase02-sync.test.ts` and `tests/phase03-storage-buffer.test.ts`. It does not edit production startup, public health, schema, CSV format, SDK or dependency files. Candidate CI #66 **terminal SUCCESS**: all 24/24 TypeScript suites, strict tsc, lint, Windows Node SDK and Next build PASS. Final focused CI #69 on `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532` was dispatched; confirm terminal results before any P02 CLOSED claim. No repeated broad CI runs unless this run demonstrates a scoped failure.
|
||||||
|
|
||||||
|
**Acceptance contract:** ACK durability applies to Redis-backed events when Redis configured. Intentionally Redis-free RAM mode retains its historical best-effort ACK. CSV replay is at-least-once with per-partition event-id duplicate suppression under tested crash windows, not a global transactional exactly-once guarantee. Existing scan uses streaming parsing (bounded memory) and scans entire matching partitions (disk I/O scales with data size; monitor before production promotion). Corrupt matching CSV fails closed. If a hard upper disk-scan budget is required for production traffic, obtain a dedicated storage/index design approval rather than silently breaking ingestion.
|
||||||
|
|
||||||
|
**Next P03 (ONLY after P02 closure):** CI-06 Python runner environment and 14/14 Python SDK; CI-07 three Docker image builds/isolated BuildKit with no live prune; CI-08 read-only npm advisory IDs/direct-transitive audit, compatible dependency pins only if justified; Compose/standalone/health gate evidence and manual release decision. Exact P03 allowlist remains in this roadmap. No live deploy, migrations, tags or releases without separate approval.
|
||||||
|
|
||||||
|
|
||||||
|
### P02 Final CI Exception / Narrow Retry
|
||||||
|
|
||||||
|
The final acceptance test-only clock override in #69 interfered with Socket.IO ACK timing. The source-grounded mixed `batch_sync` all-duplicate edge also required a single conditional correction within the previously approved handler. Commit `acca8be01a31f8d3972564cbaec8fd3117a0b551` changes only `server/socket/handlers.js` and `tests/phase02-sync.test.ts` after initial final candidate; one narrow Gitea CI #72 requested because CI #69 node-regression failed one fixture. Other scope/production invariants remain unchanged. P02 closure requires terminal #72 and exact patch/security evidence.
|
||||||
|
|
||||||
|
|
||||||
|
## P02 Final Acceptance Closure Gate / P03 approval handoff — 2026-11-22
|
||||||
|
|
||||||
|
P01 CLOSED; **P02 tests/quality/build ACCEPTED via terminal [Gitea CI #72](https://dev.ivib.cc/vibtools/DataStack/actions/runs/72) SUCCESS** on `acca8be01a31f8d3972564cbaec8fd3117a0b551`; P02 phase transitions to CLOSED only when the verified code plus these documentation-only notes are merged to Gitea main and published to GitHub/GitLab with identical tracked Git trees. Terminal jobs: node-regression 24/24 PASS; quality strict TypeScript and ESLint PASS; Windows compatibility SDK PASS; production Next standalone PASS; Python and Docker intentionally skipped (P03). #69 historical test-only clock failure is resolved in #72. A production deployment and all live services remain **NOT VERIFIED** and NOT authorized.
|
||||||
|
|
||||||
|
**P03 next / exact future Scope Lock (requires separate user approval):** CI-06 Linux/Python environment: `.gitea/workflows/datastack-ci.yml` and `.gitea/workflows/datastack-python-ci.yml` only; Python SDK 14/14 isolated. CI-07 Docker/BuildKit 3-image verification: `.gitea/workflows/datastack-ci.yml`, `.gitea/workflows/datastack-docker-ci.yml` only, runner evidence read-only; Dockerfiles can be edited ONLY after independently approved exact exception, no shared runner prune. CI-08 advisory triage: read-only `npm audit --json`, actual CVE/advisory and direct/transitive dependency graph; only minimal justified `package.json`/`package-lock.json` pins under this phase scope, never forced audit repair. `.gitea/workflows/datastack-eslint-ci.yml` quality harness only if source-proven CI-08 need. `notes/001–006` for evidence. Existing UI/API/auth/SDK/runtime logic, Compose, volumes, DB data, secrets, real host, and release configuration remain unchanged. One focused acceptance / one exact patch/security audit / one main commit and 3-way tree sync. Do not deploy, run live ingestion, migrate, prune shared Docker, tag or release.
|
||||||
|
|
||||||
|
**P03 approval command (not authorization until user explicitly sends):**
|
||||||
|
|
||||||
|
```text
|
||||||
|
APPROVED — DataStack Production Readiness P03.
|
||||||
|
Use notes/002 P03 CI-06/CI-07/CI-08 Exact Scope Lock and the verified published P02 main baseline.
|
||||||
|
CI-06: fix Python runner/toolchain only in approved .gitea workflows; verify Python SDK 14/14.
|
||||||
|
CI-07: isolate Docker runner/BuildKit and build all 3 images without deploying or pruning shared caches; Dockerfiles READ-ONLY unless a separately approved exact exception.
|
||||||
|
CI-08: read-only npm audit advisory inventory, minimal compatible package pins only for proven security need; no npm audit fix --force.
|
||||||
|
Preserve UI/UX, app/API/auth/SDK/runtime, Compose, production data, secrets and existing features unchanged.
|
||||||
|
Use one focused Gitea CI, strict patch/security diff, notes/001–006 update, Git commit and Gitea/GitHub/GitLab tree parity.
|
||||||
|
No live deployment, migrations, tags or releases.
|
||||||
|
Close P03 only on verified acceptance; provide production release readiness report.
|
||||||
|
```
|
||||||
|
|
||||||
|
## P03 scoped execution and release gate — 2026-11-22
|
||||||
|
|
||||||
|
Baseline Gitea/GitHub `48fb277079b380fdbf1e58cd2da40afeed0b03ee`; GitLab baseline same tracked tree `41a169eec0c05ce19bc4acc29d7a8cfe1f8dd90d` (317/317 blobs). Final P03 workflow candidate `549d575653b68c29f97e2f78036646f2e2035719`. [Gitea CI #77](https://dev.ivib.cc/vibtools/DataStack/actions/runs/77): overall **FAILURE** because the initial Python-only checkout lacked Node (exit 127), **not** an SDK regression. Its other six jobs terminal **SUCCESS**: Node `CI_SUITE_SUMMARY total=24 failed=0`, strict TS+ESLint, Next standalone build, Windows SDK, Compose + isolated BuildKit ingestion/web/proxy image build `DATASTACK_THREE_IMAGES_BUILD_PASS`, read-only advisory inventory `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Narrow [Python-only CI #78](https://dev.ivib.cc/vibtools/DataStack/actions/runs/78) terminal **SUCCESS**: Node-capable Bookworm container, Python 3.11 isolated venv, `Ran 14 tests in 3.959s`, `OK`, `DATASTACK_PYTHON_14_OF_14_PASS`. The other six jobs were deliberately skipped in #78.
|
||||||
|
|
||||||
|
**CI-08 security disposition:** `npm audit --json` full=26 affected packages (25 moderate, 1 high); runtime=24 (23 moderate, 1 high), **not counts of distinct advisories**. High-severity advisories [GHSA-6g55-p6wh-862q](https://github.com/advisories/GHSA-6g55-p6wh-862q), [GHSA-r28c-9q8g-f849](https://github.com/advisories/GHSA-r28c-9q8g-f849) affect Next 15.5.29's nested `postcss@8.4.31` despite patched root `postcss@8.5.33`; npm recommends major `next@16.4.6`. Direct `cloudcmd@19.24.0` carries moderate transitive issues (`dompurify`, `file-type`, `glob`, `minimatch`, `got`); npm recommends `cloudcmd@17.0.2` downgrade, not a verified safe pin. No speculative package pin, major upgrade or force fix; a separately tested compatible remediation is needed before release.
|
||||||
|
|
||||||
|
**Scope / release decision:** only `.gitea/workflows/datastack-ci.yml`, `.gitea/workflows/datastack-docker-ci.yml`, and `notes/001–006` changed. No `package.json`/lock, Dockerfile, application UI/UX, runtime, auth, SDK, database, Compose, production data, secret, shared Docker prune, deployment, migration, tag or release. P03 technical CI and advisory-triage acceptance verified by #77 non-Python jobs plus #78 Python; do **not** mislabel #77 overall as successful. Final phase publication requires one main commit and Gitea/GitHub/GitLab exact tracked tree parity. **Production release NO-GO** while runtime high advisory is unresolved; live production health, throughput and deployment remain unverified.
|
||||||
|
|
||||||
|
## P04-S01 approved security remediation scope — 2026-11-22
|
||||||
|
|
||||||
|
Baseline Gitea `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3`, tree `12818ba732f5abdc42af46d83278b13b6be4eafd`. SEC-01: Next `15.5.29` nested `postcss@8.4.31` is the sole P03 high runtime affected package despite top-level `postcss@8.5.33`; see CI #77 `AUDIT_PACKAGE runtime postcss`, GHSA-6g55-p6wh-862q / GHSA-r28c-9q8g-f849. Source `package-lock.json:10552-10560`, `10602-10603`, `11427-11428`; `package.json:35,37`. Approval authorizes only scoped override in `package.json`, npm-generated `package-lock.json`, **existing** `.gitea/workflows/datastack-ci.yml` security-audit job and `notes/001–006`. All application, test, SDK, Docker/Compose and deployment files READ-ONLY. Never force next@16, downgrade cloudcmd, `npm audit fix --force`, prune shared Docker, deploy, migrate or tag. Acceptance: reproducible lockfile, clean npm ci, Node 24/24, Python 14/14, tsc/lint, Windows SDK, Next standalone, 3 Docker images, runtime high/critical zero and exact Git diff. One focused Gitea CI, one accepted main commit, three-remote content parity; unresolved moderates/live checks retain production NO-GO. If override incompatible, STOP as BLOCKED and request exact exception.
|
||||||
|
|
||||||
|
## P04-S01 blocked security reconciliation report
|
||||||
|
|
||||||
|
P04-S01 SEC-01 BLOCKED — 2026-11-22. Approved scoped `overrides.next.postcss=8.5.33` trialed in disposable Windows npm 10.9.8 workspace only. `npm install --package-lock-only` EXIT 0, but `npm ls --package-lock-only postcss --all` EXIT 1 (`ELSPROBLEMS`: Next nested `postcss@8.4.31` invalid; expected 8.5.33); npm dedupe also kept invalid dependency. Read-only `npm audit --omit=dev --json` EXIT 1, runtime high 1/moderate 23/critical 0. Lockfile regeneration caused unrelated churn (91 package entries added, 106 removed, 69 changed, including two lightningcss optional 1.33.0 -> 1.30.1). Fresh minimal Next 15.5.29 fixture with identical override resolves PostCSS 8.5.33, isolating failure to DataStack committed lock graph. `npm ci --dry-run` exit 0 does not prove real clean install. Exact stop rule applied: no dependency patch, no CI acceptance, no main commit, no GitHub/GitLab sync, no release/deploy; production NO-GO. Need separate explicit narrowly scoped lock reconciliation approval.
|
||||||
|
|
||||||
|
Approved scope remains `package.json` scoped override, npm-generated `package-lock.json`, current Gitea `security-audit` job, notes/001–006. Do not add manual lock edits, major Next 16, Cloudcmd downgrade, npm audit fix --force or unrelated source files. Phase BLOCKED until amended approval.
|
||||||
|
|
||||||
|
## P04-S01B correction gate — 2026-11-22
|
||||||
|
|
||||||
|
The targeted removal-and-regenerate method fixes SEC-01 in an isolated copy: valid `npm ls` exit 0 and `npm audit --omit=dev` high=0/critical=0/moderate=22, while Next 15.5.29, Cloudcmd 19.24.0 and all unrelated versions remain unchanged. **But** npm regenerated 67 unrelated package metadata entries (36 `dev` flags, 29 `libc` fields, 2 dependency map order changes); that is outside the expressly approved preservation contract. P04-S01B **BLOCKED by exact scope lock**, not by unresolved PostCSS resolution. Do not commit the unacceptable lock or quietly normalize metadata. Require separate approved exact metadata-reconciliation strategy/exception and content-preserving proof. Until then no acceptance CI, commit to main, remote parity, release or deployment.
|
||||||
|
|
||||||
|
## P04-S01C implementation checkpoint and exact release gate — 2026-11-22
|
||||||
|
|
||||||
|
Approved P04-S01C changes are strictly `package.json`, `package-lock.json`, existing `.gitea/workflows/datastack-ci.yml:security-audit`, notes/001–006. Applied scoped Next→PostCSS 8.5.33 override, npm regenerated disposable lock after stale entry deletion, deterministically restored **all 67** unrelated npm metadata modifications against baseline; exact final diff only stale nested package removal. No version, integrity, peer or optional metadata drift. Gitea branch patch `c759be990c032c2474d2d0bad519c643dec11a3a`; lock SHA-256 `e6b5b362877627085dc04c9ded03c019971e07ef9f4af1e78057d58961a1c091`. Focused [CI #85](https://dev.ivib.cc/vibtools/DataStack/actions/runs/85) six jobs terminal SUCCESS including Docker 3/3, Python 14/14, Node 24/24, Next/tsc/lint, runtime high/critical zero; Windows compatibility **QUEUED**, so P04-S01C **PENDING**. Do not merge/mark CLOSED until Windows job succeeds, overall #85 SUCCESS, exact file-scope audit and Gitea/GitHub/GitLab tracked-tree parity; no deployment, migration, release/tag/prune. After closure, independently approve P04-S02 Cloudcmd moderate advisory triage and later production readiness.
|
||||||
|
|
||||||
|
## P04-S01C FINAL CI acceptance (7/7) — 2026-11-22
|
||||||
|
|
||||||
|
Gitea CI #85 https://dev.ivib.cc/vibtools/DataStack/actions/runs/85 COMPLETED/SUCCESS; tested implementation SHA c759be990c032c2474d2d0bad519c643dec11a3a; later changes to candidate are documentation only. All seven jobs terminal SUCCESS: node-regression (Linux committed-lock npm ci and exactly 24/24 TypeScript suites); python-regression (Ran 14 tests, OK); quality (strict tsc and ESLint); production-build (Next 15.5.29 standalone, 22/22 pages); docker-image-build (three isolated BuildKit ingestion/web/proxy images); windows-compatibility (Windows committed-lock npm ci and Node SDK build, job #182, terminal SUCCESS); security-audit (runtime moderate=22 high=0 critical=0; full moderate=24 high=0 critical=0; DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO). Single Gitea acceptance run used, no additional broad reruns.
|
||||||
|
|
||||||
|
Scope: exactly nine approved paths: package.json scoped next->postcss 8.5.33 override; package-lock.json removes only old nested postcss@8.4.31 entry; existing .gitea/workflows/datastack-ci.yml security-audit job high/critical rejection; notes/001–006 evidence. Other package metadata, versions, integrity and resolved URLs match baseline. Next.js 15.5.29 and Cloudcmd 19.24.0 preserved. Final candidate package-lock SHA256 e6b5b362877627085dc04c9ded03c019971e07ef9f4af1e78057d58961a1c091 (verified byte-for-byte with Gitea). No API, auth, SDK, UI, application runtime, database, Compose, Dockerfile, tests or production data changes, deployment, migration, prune, tag, release.
|
||||||
|
|
||||||
|
Next roadmap: P04-S02 Cloudcmd moderate-advisory triage, followed by separate live readiness approval. No forced major upgrade/downgrade. CI fully accepted; publication not yet asserted in these pre-merge notes. Final phase CLOSED requires one squash merge of PR #5 into Gitea main and verified identical tracked content across Gitea/GitHub/GitLab; record actual new SHAs in terminal report. Production release remains NO-GO pending P04-S02 moderate vulnerability triage and separately authorized live readiness verification.
|
||||||
|
|
||||||
|
## P04-S02 scoped candidate — 2026-11-22
|
||||||
|
|
||||||
|
Baseline Gitea main `2d467ff1494858164a37814f7dbc2b321f510291`, tree `648c76ca496d2c9bf22a16f558c9ba347a45d4f7`. Cloudcmd 19.24.0 -> deepword 11.0.1 -> monaco-editor 0.55.1 pins DOMPurify 3.2.7, vulnerable according to npm advisory inventory. Disposable Windows npm 10.9.8 `npm view dompurify@3.4.18` verified, scoped npm override `overrides.cloudcmd.dompurify=3.4.18` resolves `dompurify@3.4.18` via `npm ls --package-lock-only dompurify --all --depth=5` (exit 0). `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline` exit 0. npm graph initially changed 67 unrelated metadata entries; deterministically restored exact baseline values after validating unrelated versions, URLs, integrity and dependencies. Final lock has **one changed package entry**, DOMPurify; same-major replacement plus verified registry `resolved` and `integrity`, no other packages added/deleted/changed. Candidate runtime audit affected-package counts **moderate 20/high 0/critical 0** versus baseline **22/0/0**; full candidate **moderate 22/high 0/critical 0** versus baseline **24/0/0**. DOMPurify and Monaco Editor dropped from vulnerable-package inventory. Other moderate packages (got, glob, minimatch, nested file-type, Cloudcmd transitive chain) remain; they require major package changes under current advisory constraints, prohibited by Scope Lock. This is a **pre-acceptance candidate**: do not claim application compatibility, CI PASS, publication, or phase CLOSED before full focused Gitea CI and exact three-remote tree parity.
|
||||||
|
|
||||||
|
Roadmap: finish one focused Gitea acceptance, exact 8-file scope and three-remote parity. Remaining moderate findings deferred to independently approved scope, never silently upgraded.
|
||||||
|
|
||||||
|
## P04-S02 final CI acceptance — 2026-11-22
|
||||||
|
|
||||||
|
Gitea CI #92 (https://dev.ivib.cc/vibtools/DataStack/actions/runs/92) **COMPLETED / SUCCESS**, all 7/7 jobs terminal success on source commit `a88bc5f4498ea1b11acc0acfeb66b5aa9d6531be`: node-regression real Linux npm ci + 24/24 tests including `tests/files-manager.test.ts` exit 0; Python SDK 14/14; quality strict TypeScript/ESLint; Next.js standalone (22/22 static pages); isolated Docker/BuildKit 3-image verification `DATASTACK_THREE_IMAGES_BUILD_PASS`; Windows committed-lock npm ci + Node SDK compatibility; security-audit `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO` and `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Audit full=22 moderate/high=0/critical=0 (baseline 24), runtime=20 moderate/high=0/critical=0 (baseline 22). These are affected package entries, not distinct advisories. No new CI workflow or extra broad CI dispatched.
|
||||||
|
|
||||||
|
Exact approved patch: `package.json` scoped `overrides.cloudcmd.dompurify=3.4.18`, preserving `overrides.next.postcss=8.5.33`; npm-generated `package-lock.json` changes solely `node_modules/dompurify` version 3.2.7 -> 3.4.18 and verified `resolved`/SHA512 integrity. Baseline Cloudcmd 19.24.0, Monaco Editor 0.55.1, Next.js 15.5.29, all unrelated locked package versions, URL, hash, dependency and metadata preserved. Gitea published candidate lock bytes match disposable SHA-256 `57c5ea9df9f28be496f8c3202fbc280b0066bf8c216b8d8a3ca98b6f7b942854`. Exact candidate tree diff 420 entries, 8 approved changed files only (`package.json`, `package-lock.json`, notes/001–006), 0 unauthorized paths. No runtime/UI/auth/API/SDK/tests/CI/Dockerfiles/Compose/db/schema/data modifications.
|
||||||
|
|
||||||
|
Roadmap: after P04-S02 accepted mirror parity, next separately approved P04-S03 remaining moderate advisory dependency-contract triage (read-only / disposable compatibility first), then P05 live readiness. **Residual risk:** 20 moderate runtime affected package entries remain, including Cloudcmd transitive dependencies `got`, `glob`, `minimatch`, nested `file-type` and their parent chains. Current npm audit suggests `cloudcmd@17.0.2` (a major downgrade), explicitly out-of-scope; other transitive fixes need major-version dependency-contract changes. No force/downgrade/unverified override. Security improvement verified but NOT zero vulnerabilities. Production release stays NO-GO; remaining advisory disposition and live readiness need independent scope approval.
|
||||||
|
|
||||||
|
**Merge gate:** CI #92 7/7 PASS, approved 8-file patch integrity PASS; main squash merge and Gitea/GitHub/GitLab exact Git content-tree verification required before P04-S02 CLOSED. This note is written before that publication, so final remote commit SHAs shall be appended to the PR closure report after verification. No deployment, migrations, prune, tag, release.
|
||||||
|
|
||||||
@@ -2,13 +2,13 @@
|
|||||||
|
|
||||||
Frozen source baseline: `d725841ebc828d58cbe90f517da9e600f23c4630`. Generated: 2026-11-22 Asia/Dhaka.
|
Frozen source baseline: `d725841ebc828d58cbe90f517da9e600f23c4630`. Generated: 2026-11-22 Asia/Dhaka.
|
||||||
Scope: documentation-only kickoff. Any future phase must update this log **after** verification, not on intention.
|
Scope: documentation-only kickoff. Any future phase must update this log **after** verification, not on intention.
|
||||||
**Completed implementation phases: 1 / 3. Remaining: 2.** Documentation baseline preparation is a separate non-implementation milestone.
|
**P01 CLOSED; P02 verified in Gitea CI #72 and awaiting recorded three-remote publication; P03 NOT STARTED.** Publication/tree-parity proof and final phase-close status are recorded in the latest acceptance appendix and operator report.
|
||||||
|
|
||||||
| Phase | Status | IDs / function | Changed source | Verified CI evidence | Commit / remote sync |
|
| Phase | Status | IDs / function | Changed source | Verified CI evidence | Commit / remote sync |
|
||||||
|---|---|---|---|---|---|
|
|---|---|---|---|---|---|
|
||||||
| P00 Notes / freeze | COMPLETED (documentation only) | `001–006` docs, source baseline and exact scope | `notes/**` only | 6 Markdown files; balanced fences; staged diff `--check` PASS | Documentation commit containing this file; remote SHA verification recorded in session handoff |
|
| P00 Notes / freeze | COMPLETED (documentation only) | `001–006` docs, source baseline and exact scope | `notes/**` only | 6 Markdown files; balanced fences; staged diff `--check` PASS | Documentation commit containing this file; remote SHA verification recorded in session handoff |
|
||||||
| P01 Tests & lockfile | **CLOSED — VERIFIED** | CI-01…CI-05, no runtime feature changes | 16 exact P01 code/test/CI paths, plus `notes/001–006` records | #59 Linux/Windows clean npm/SDK, 24/24 TS suites, Next standalone PASS; #60 strict TypeScript and ESLint PASS; historical #59 quality mismatch corrected | One permanent P01 main commit containing these notes; confirm remote SHA/tree parity in phase handoff |
|
| P01 Tests & lockfile | **CLOSED — VERIFIED** | CI-01…CI-05, no runtime feature changes | 16 exact P01 code/test/CI paths, plus `notes/001–006` records | #59 Linux/Windows clean npm/SDK, 24/24 TS suites, Next standalone PASS; #60 strict TypeScript and ESLint PASS; historical #59 quality mismatch corrected | One permanent P01 main commit containing these notes; confirm remote SHA/tree parity in phase handoff |
|
||||||
| P02 Ingestion reliability | **BLOCKED — three diagnostic cases reproduced; runtime fix requires separate approval** | R-01 ACK/RAM loss; R-02 CSV duplicate replay; R-03 health liveness vs readiness | Only `tests/phase02-sync.test.ts` and `tests/phase03-storage-buffer.test.ts`, plus notes/001–006 | Gitea #63: all three P02 subtests PASS to reproduce existing behavior; full matrix status recorded in appendix | Diagnostic branch `p02-readiness-test-20261122`, no production runtime changes |
|
| P02 Ingestion reliability | **CI VERIFIED #72 — final publish gate** | R-01 ACK/idempotency safety; R-02 CSV replay; R-03 liveness verified unchanged | Exactly 3 runtime files, 2 tests, notes/001-006 | #72 terminal SUCCESS: 24/24 TS, TS typecheck, lint, Next standalone, Windows SDK | Final source/test acca8be01a31f8d3972564cbaec8fd3117a0b551; three-remote main parity recorded on publication |
|
||||||
| P03 CI/Docker/security | NOT STARTED | CI-06,07,08; release gates | None | Not run | — |
|
| P03 CI/Docker/security | NOT STARTED | CI-06,07,08; release gates | None | Not run | — |
|
||||||
|
|
||||||
## Existing diagnostic CI snapshot (NOT implementation phase completions)
|
## Existing diagnostic CI snapshot (NOT implementation phase completions)
|
||||||
@@ -91,3 +91,110 @@ Historical baseline code stayed frozen through these CI workflow changes. Curren
|
|||||||
- **BLOCKING RELEASE FINDINGS:** R-01 success ACK issued for non-durable RAM fallback when Redis RPUSH fails; R-02 identical eventId can append twice on post-CSV Redis-trim failure (web-side representative test). R-03 health remains online on Redis loss; liveness-only behavior is backward-compatible, protected dependency-readiness verification remains missing.
|
- **BLOCKING RELEASE FINDINGS:** R-01 success ACK issued for non-durable RAM fallback when Redis RPUSH fails; R-02 identical eventId can append twice on post-CSV Redis-trim failure (web-side representative test). R-03 health remains online on Redis loss; liveness-only behavior is backward-compatible, protected dependency-readiness verification remains missing.
|
||||||
- **P02 PHASE = BLOCKED**, diagnostic implementation done, release reliability acceptance NOT satisfied. Runtime remediation requires separate exact approval. 1/3 phases still CLOSED (P01 only); P02 and P03 remain unclosed. No P03 start authorization from this diagnostic completion.
|
- **P02 PHASE = BLOCKED**, diagnostic implementation done, release reliability acceptance NOT satisfied. Runtime remediation requires separate exact approval. 1/3 phases still CLOSED (P01 only); P02 and P03 remain unclosed. No P03 start authorization from this diagnostic completion.
|
||||||
- Next approval must explicitly authorize exact narrow server/core patch for ACK durability and CSV replay deduplication, plus concrete no-loss/no-duplicate tests. Any readiness endpoint change needs separate explicit approval. User's initial P02 approval is insufficient for runtime mutation.
|
- Next approval must explicitly authorize exact narrow server/core patch for ACK durability and CSV replay deduplication, plus concrete no-loss/no-duplicate tests. Any readiness endpoint change needs separate explicit approval. User's initial P02 approval is insufficient for runtime mutation.
|
||||||
|
|
||||||
|
|
||||||
|
## P02-RUNTIME-REMEDIATION — Candidate implementation checkpoint (2026-11-22)
|
||||||
|
|
||||||
|
- **Current P02: BLOCKED / NOT CLOSED.** One previous diagnostic Gitea CI #63 reproduced R-01/R-02/R-03; runtime remediation was separately approved after that. Verified completed implementation phases remain **1/3** (P01 only). Do not approve P03 yet.
|
||||||
|
- Published P02 diagnosis baseline `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`; isolated local candidate commit `44eee8877e40d1683f44502c2a0ba3d437a03b88` on `p02-runtime-remediation-20261122`.
|
||||||
|
- Exactly five code/test files changed in this candidate: `server/index.js`, `server/socket/handlers.js`, `core/data/buffer.ts`, `tests/phase02-sync.test.ts`, `tests/phase03-storage-buffer.test.ts`. No `server/workers/csvWriter.js`, `app/**`, `components/**`, SDK, public R-03 health, CI workflow, database, migration, production data, secrets or deployment touched.
|
||||||
|
- JS `node --check` on two changed JS files PASS; `git diff --check` PASS and allowlist audit PASS. Targeted TS/Node regression and Gitea CI **NOT YET RUN TO TERMINAL PASS**. The Gitea connector/host was unavailable when dispatch/push was attempted. Local temporary dependency installation also not yet complete.
|
||||||
|
- The original user PC root `D:\VibTools_Workspace\01_Products\14_DataStack` still has 16 pre-existing staged P01 paths on older `d8ac96e`; deliberately preserved without reset/stash/discard. All P02 implementation is in a separate temporary worktree. Do not assume the old local root is up to date.
|
||||||
|
- R-01 and R-02 implementations are candidates; practical reliability limitations and I/O cost recorded in `notes/001` and `notes/004`. No false claim of globally exactly-once CSV or universally durable ACK.
|
||||||
|
- **Next gates:** restore Gitea access; run one focused disposable acceptance; fix any in-scope test/TS defects; record exact CI terminal results; only after required acceptance and security audit, squash single phase main commit and verify GitHub/GitLab/Gitea tree parity; then, and only then, decide P02 CLOSED versus residual-risk blocker.
|
||||||
|
|
||||||
|
|
||||||
|
### Verification environment blocker — local dependency install
|
||||||
|
|
||||||
|
The authorized Windows PC attempted a **local disposable** `npm.cmd ci --ignore-scripts --no-audit --no-fund --prefer-offline` in the isolated P02 worktree only; it failed with `npm ERR! ETARGET No matching version found for yaml@2.9.1` plus cleanup EPERM warnings in the temp worktree's `node_modules`. Exact output captured in `%TEMP%\DataStack-P02-npm-ci.log` on the authorized PC. This is **not a passed code test nor proof of a repository package defect**, and `package-lock.json` / `package.json` cannot be edited under P02 scope. Gitea MCP remained UNAVAILABLE and the Git remote/HTTP service was not responding when CI was attempted, so the mandatory P02 focused CI was **not dispatched**. Phase remains BLOCKED pending actual CI evidence; no runtime change was pushed to production/main.
|
||||||
|
|
||||||
|
|
||||||
|
## P02-FINAL-ACCEPTANCE-01 Current Verification (latest record)
|
||||||
|
|
||||||
|
- **At this checkpoint: P02 acceptance PENDING CI #69**. P01 CLOSED; P03 NOT STARTED. Do not reinterpret earlier P02 BLOCKED audit as a completed release.
|
||||||
|
- Gitea #66 on `76cd972bdeea9ef07b4b38e9d8b5b5ed3312ca5b` **COMPLETED SUCCESS**: node-regression 24/24 suites PASS including direct Port4000 replay; quality (strict tsc+ESLint) PASS; production-build PASS; windows-compatibility SDK PASS; Python/Docker skipped by scoped `p01_only` input.
|
||||||
|
- Final patch `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532` in clean isolated `p02-final-acceptance-20261122`: resolved source-grounded negative-marker TTL bug and mixed-batch success ACK, plus test-only stale-claim time jump (+61s), failed Redis claim, mixed batch, intentionally Redis-free compatibility, rotated .csv.gz+multiline replay and partial-corrupt CSV fail-closed/recovery. No source outside `server/socket/handlers.js` changed since candidate.
|
||||||
|
- One focused Gitea CI #69 dispatched for final patch; job terminal outcome and any corrective action must be recorded. **Do not merge until this passes and exact patch/security audit is reviewed.**
|
||||||
|
- Original `D:\VibTools_Workspace\01_Products\14_DataStack` at old `d8ac96e` with 16 staged P01 changes untouched. Separate pre-existing `p02-runtime-remediation-20261122` had three dirty files; left untouched. The new isolated worktree was created clean from candidate SHA.
|
||||||
|
- Source behavior: no Redis configured = historical RAM best-effort acceptance preserved; configured Redis failure = negative ACK; public /health unchanged. Per-CSV replay scanning bounded-memory but linear in stored bytes; no global exactly-once/throughput guarantee. P03 cannot begin until P02 acceptance/remote parity close.
|
||||||
|
|
||||||
|
|
||||||
|
### P02 Focused Retry — #69 fixture failure / #72 correction
|
||||||
|
|
||||||
|
- Final acceptance CI #69 on `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532`: node-regression **FAIL** because `phase02-sync` stale-marker test globally mocked `Date.now` and Socket.IO ACK timed out. Other new tests for gzip/rotated CSV, partial CSV corruption, Redis-free RAM and mixed batch with a new persisted event PASS. Strict TypeScript, lint and Windows SDK jobs PASS; full workflow conclusion must be read at terminal.
|
||||||
|
- New source-grounded all-duplicate mixed batch path (`processed=0`, `duplicates=1`, `rejected>0`, volatile claim present) could return overall success; added focused test and changed handler conditional to reject all `volatileDuplicate` combinations.
|
||||||
|
- Test correction removes process-wide Date.now override, verifies 61000ms/300000ms cleanup math with static source marker assertion, and checks real Socket.IO negative retries. No unrelated files, CI workflows, SDK, auth or public health modified.
|
||||||
|
- Commit `acca8be01a31f8d3972564cbaec8fd3117a0b551` submitted to disposable CI #72; **P02 remains pending / NOT CLOSED** until terminal acceptance and 3-way mirror parity.
|
||||||
|
|
||||||
|
|
||||||
|
## P02 Verified Acceptance and Conditional Publication — 2026-11-22
|
||||||
|
|
||||||
|
- **Code acceptance VERIFIED — Gitea CI #72 terminal SUCCESS.** Code SHA `acca8be01a31f8d3972564cbaec8fd3117a0b551`. Node regression exactly `CI_SUITE_SUMMARY total=24 failed=0`, individual `tests/phase02-sync.test.ts` and `tests/phase03-storage-buffer.test.ts` suites exit 0. Quality strict TS+ESLint, Next standalone build and Windows Node SDK jobs all SUCCESS. Python/Docker intentionally skipped in this focused P02 workflow.
|
||||||
|
- **Root causes fixed:** configured Redis RPUSH outage returns `BUFFER_PERSIST_FAILED` negative ACK rather than volatile RAM success; stale failed idempotency claim retains correct TTL and remains negative through retries; `batch_sync` with old durable duplicates plus failed volatile claim returns `success:false`; CSV-before-Redis-LTRIM replay suppresses previously written same-client/date/eventId across plain/rotated/gz CSV with multiline quoted payload; corrupt matching CSV prevents Redis trim and can be recovered after disposable repair.
|
||||||
|
- **Explicit limits:** Redis intentionally unconfigured retains historical best-effort RAM ACK; no globally atomic exactly-once promise; scan I/O proportional to matching partition bytes even though memory streaming; public /health continues liveness not dependency readiness; live production unknown. No deployment, migration, secret access, user-data mutation, tag, release or build of production images.
|
||||||
|
- **Historical testing:** #66 PASS candidate; #69 suite failure (Socket.IO ACK timeout caused by test-only global `Date.now` override) and cancellation; #72 PASS corrected source/test. No false claim that #69 succeeded.
|
||||||
|
- Final code/diff safety audit: 11/11 approved paths only: `server/index.js`, `server/socket/handlers.js`, `core/data/buffer.ts`, `tests/phase02-sync.test.ts`, `tests/phase03-storage-buffer.test.ts`, plus `notes/001–006`; `git diff --check` and JS syntax PASS. Original PC root old staged P01 (16 paths) and separate dirty earlier temporary worktree preserved.
|
||||||
|
- **Publication gate:** Gitea PR #3 final verified source + documentation notes; squash-merge with `[skip ci]`, publish to GitHub and GitLab, verify identical tracked Git trees despite potentially different GitLab commit SHA. P02 marked CLOSED only after three-remote content parity; evidence SHA/tree in operator's final publication report. P03 NOT STARTED and requires separate approval.
|
||||||
|
|
||||||
|
## P03 final CI acceptance ledger — 2026-11-22
|
||||||
|
|
||||||
|
| Gate | Terminal outcome |
|
||||||
|
| --- | --- |
|
||||||
|
| #77 Node | SUCCESS, 24/24 |
|
||||||
|
| #77 TypeScript / ESLint | SUCCESS |
|
||||||
|
| #77 Next standalone build | SUCCESS |
|
||||||
|
| #77 Windows Node SDK | SUCCESS |
|
||||||
|
| #77 isolated three-image BuildKit | SUCCESS |
|
||||||
|
| #77 read-only npm audit inventory | SUCCESS with 1 high runtime finding |
|
||||||
|
| #77 Python | FAILED before tests: checkout requires Node |
|
||||||
|
| #78 corrected Python-only | SUCCESS, 14/14; other jobs SKIPPED |
|
||||||
|
|
||||||
|
**CI-08 security disposition:** `npm audit --json` full=26 affected packages (25 moderate, 1 high); runtime=24 (23 moderate, 1 high), **not counts of distinct advisories**. High-severity advisories [GHSA-6g55-p6wh-862q](https://github.com/advisories/GHSA-6g55-p6wh-862q), [GHSA-r28c-9q8g-f849](https://github.com/advisories/GHSA-r28c-9q8g-f849) affect Next 15.5.29's nested `postcss@8.4.31` despite patched root `postcss@8.5.33`; npm recommends major `next@16.4.6`. Direct `cloudcmd@19.24.0` carries moderate transitive issues (`dompurify`, `file-type`, `glob`, `minimatch`, `got`); npm recommends `cloudcmd@17.0.2` downgrade, not a verified safe pin. No speculative package pin, major upgrade or force fix; a separately tested compatible remediation is needed before release.
|
||||||
|
|
||||||
|
**Scope / release decision:** only `.gitea/workflows/datastack-ci.yml`, `.gitea/workflows/datastack-docker-ci.yml`, and `notes/001–006` changed. No `package.json`/lock, Dockerfile, application UI/UX, runtime, auth, SDK, database, Compose, production data, secret, shared Docker prune, deployment, migration, tag or release. P03 technical CI and advisory-triage acceptance verified by #77 non-Python jobs plus #78 Python; do **not** mislabel #77 overall as successful. Final phase publication requires one main commit and Gitea/GitHub/GitLab exact tracked tree parity. **Production release NO-GO** while runtime high advisory is unresolved; live production health, throughput and deployment remain unverified.
|
||||||
|
|
||||||
|
## P04-S01 security remediation kickoff — 2026-11-22
|
||||||
|
|
||||||
|
**Status: IN PROGRESS / NOT ACCEPTED.** User-approved baseline `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3`. Scope `package.json`, npm-generated `package-lock.json`, existing Gitea CI `security-audit` job, `notes/001–006`. Root cause `node_modules/next` locks `postcss@8.4.31` while root is `8.5.33`; P03 runtime audit 1 high and 23 moderate package entries. Plan minimal nested npm override, lockfile-only regeneration, high/critical audit gate; preserve all runtime/UI/API/auth/SDK/DB/Compose/production. One focused CI acceptance and exact 3-remote tree equality required; DO NOT label closed before terminal results, publish or claim release-ready.
|
||||||
|
|
||||||
|
## P04-S01 blocked security reconciliation report
|
||||||
|
|
||||||
|
P04-S01 SEC-01 BLOCKED — 2026-11-22. Approved scoped `overrides.next.postcss=8.5.33` trialed in disposable Windows npm 10.9.8 workspace only. `npm install --package-lock-only` EXIT 0, but `npm ls --package-lock-only postcss --all` EXIT 1 (`ELSPROBLEMS`: Next nested `postcss@8.4.31` invalid; expected 8.5.33); npm dedupe also kept invalid dependency. Read-only `npm audit --omit=dev --json` EXIT 1, runtime high 1/moderate 23/critical 0. Lockfile regeneration caused unrelated churn (91 package entries added, 106 removed, 69 changed, including two lightningcss optional 1.33.0 -> 1.30.1). Fresh minimal Next 15.5.29 fixture with identical override resolves PostCSS 8.5.33, isolating failure to DataStack committed lock graph. `npm ci --dry-run` exit 0 does not prove real clean install. Exact stop rule applied: no dependency patch, no CI acceptance, no main commit, no GitHub/GitLab sync, no release/deploy; production NO-GO. Need separate explicit narrowly scoped lock reconciliation approval.
|
||||||
|
|
||||||
|
Candidate branch `p04-s01-postcss-security-20261122` contains notes only. Gitea main stays `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3`; no Gitea CI was dispatched and no phase acceptance established.
|
||||||
|
|
||||||
|
## P04-S01B run ledger — 2026-11-22
|
||||||
|
|
||||||
|
**P04-S01B = BLOCKED / stop rule.** Disposable Windows npm 10.9.8 only (not user project workspace). Verified baseline Gitea `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3` and candidate notes branch `4abcc55fee927b5ae2c1193cea2b4585b8d00158` before test. With approved stale node deletion in disposable lock plus scoped manifest override, `npm install --package-lock-only` 0, `npm ls --package-lock-only postcss --all --depth=4` 0, audit high 0 / critical 0 / moderate 22. Next 15.5.29, Cloudcmd 19.24.0 and every unrelated package version unchanged; package entries added 0, removed 1 (stale `node_modules/next/node_modules/postcss`), existing entries with changed metadata 67 (36 dev, 29 libc, 2 dependency ordering overlaps). **Reject generated candidate due preservation of unrelated metadata requirement.** No real npm ci, regression, Docker, focused Gitea CI, security gate edit, application changes, main commit, GitHub/GitLab mirror or release. P03 CLOSED, P04-S01B NOT CLOSED, published production still NO-GO.
|
||||||
|
|
||||||
|
## P04-S01C verified six-job acceptance checkpoint — 2026-11-22
|
||||||
|
|
||||||
|
- User-approved baseline Gitea main `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3` still unchanged; candidate source commit `c759be990c032c2474d2d0bad519c643dec11a3a`, PR #5 draft pending acceptance.
|
||||||
|
- Exact source diff: only `package.json` scoped override, `package-lock.json` delete stale nested PostCSS entry (one package), existing `.gitea/workflows/datastack-ci.yml` security-audit threshold. Updated notes six files only; no runtime/SDK/UI/test/Dockerfile/Compose source changes.
|
||||||
|
- [Gitea CI #85](https://dev.ivib.cc/vibtools/DataStack/actions/runs/85) results as of checkpoint: node-regression **SUCCESS** `CI_SUITE_SUMMARY total=24 failed=0` plus real Linux `npm ci`; python-regression **SUCCESS** `Ran 14 tests`, `OK`; quality tsc/ESLint **SUCCESS**; production Next standalone **SUCCESS**; Docker/BuildKit 3 images **SUCCESS** `DATASTACK_THREE_IMAGES_BUILD_PASS`; security-audit **SUCCESS** runtime high 0/critical 0, `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO`, full moderate 24/runtime moderate 22. Windows-compatibility **QUEUED** at checkpoint: no terminal proof; entire run must not be mislabelled success.
|
||||||
|
- **P04-S01C PENDING** terminal Windows acceptance, main squash commit, GitHub/GitLab parity. Production NO-GO; 22 moderate runtime package entries and live readiness remain unresolved.
|
||||||
|
|
||||||
|
## P04-S01C FINAL CI acceptance (7/7) — 2026-11-22
|
||||||
|
|
||||||
|
Gitea CI #85 https://dev.ivib.cc/vibtools/DataStack/actions/runs/85 COMPLETED/SUCCESS; tested implementation SHA c759be990c032c2474d2d0bad519c643dec11a3a; later changes to candidate are documentation only. All seven jobs terminal SUCCESS: node-regression (Linux committed-lock npm ci and exactly 24/24 TypeScript suites); python-regression (Ran 14 tests, OK); quality (strict tsc and ESLint); production-build (Next 15.5.29 standalone, 22/22 pages); docker-image-build (three isolated BuildKit ingestion/web/proxy images); windows-compatibility (Windows committed-lock npm ci and Node SDK build, job #182, terminal SUCCESS); security-audit (runtime moderate=22 high=0 critical=0; full moderate=24 high=0 critical=0; DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO). Single Gitea acceptance run used, no additional broad reruns.
|
||||||
|
|
||||||
|
Scope: exactly nine approved paths: package.json scoped next->postcss 8.5.33 override; package-lock.json removes only old nested postcss@8.4.31 entry; existing .gitea/workflows/datastack-ci.yml security-audit job high/critical rejection; notes/001–006 evidence. Other package metadata, versions, integrity and resolved URLs match baseline. Next.js 15.5.29 and Cloudcmd 19.24.0 preserved. Final candidate package-lock SHA256 e6b5b362877627085dc04c9ded03c019971e07ef9f4af1e78057d58961a1c091 (verified byte-for-byte with Gitea). No API, auth, SDK, UI, application runtime, database, Compose, Dockerfile, tests or production data changes, deployment, migration, prune, tag, release.
|
||||||
|
|
||||||
|
CI fully accepted; publication not yet asserted in these pre-merge notes. Final phase CLOSED requires one squash merge of PR #5 into Gitea main and verified identical tracked content across Gitea/GitHub/GitLab; record actual new SHAs in terminal report. Production release remains NO-GO pending P04-S02 moderate vulnerability triage and separately authorized live readiness verification.
|
||||||
|
|
||||||
|
## P04-S02 scoped candidate — 2026-11-22
|
||||||
|
|
||||||
|
Baseline Gitea main `2d467ff1494858164a37814f7dbc2b321f510291`, tree `648c76ca496d2c9bf22a16f558c9ba347a45d4f7`. Cloudcmd 19.24.0 -> deepword 11.0.1 -> monaco-editor 0.55.1 pins DOMPurify 3.2.7, vulnerable according to npm advisory inventory. Disposable Windows npm 10.9.8 `npm view dompurify@3.4.18` verified, scoped npm override `overrides.cloudcmd.dompurify=3.4.18` resolves `dompurify@3.4.18` via `npm ls --package-lock-only dompurify --all --depth=5` (exit 0). `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline` exit 0. npm graph initially changed 67 unrelated metadata entries; deterministically restored exact baseline values after validating unrelated versions, URLs, integrity and dependencies. Final lock has **one changed package entry**, DOMPurify; same-major replacement plus verified registry `resolved` and `integrity`, no other packages added/deleted/changed. Candidate runtime audit affected-package counts **moderate 20/high 0/critical 0** versus baseline **22/0/0**; full candidate **moderate 22/high 0/critical 0** versus baseline **24/0/0**. DOMPurify and Monaco Editor dropped from vulnerable-package inventory. Other moderate packages (got, glob, minimatch, nested file-type, Cloudcmd transitive chain) remain; they require major package changes under current advisory constraints, prohibited by Scope Lock. This is a **pre-acceptance candidate**: do not claim application compatibility, CI PASS, publication, or phase CLOSED before full focused Gitea CI and exact three-remote tree parity.
|
||||||
|
|
||||||
|
Phase ledger: P04-S02 candidate patch under test, not yet accepted. One focused CI to verify Linux/Windows npm ci, Cloudcmd File Manager existing tests, Node 24/24, Python 14/14, TS/lint, Next standalone and Docker 3 images.
|
||||||
|
|
||||||
|
## P04-S02 final CI acceptance — 2026-11-22
|
||||||
|
|
||||||
|
Gitea CI #92 (https://dev.ivib.cc/vibtools/DataStack/actions/runs/92) **COMPLETED / SUCCESS**, all 7/7 jobs terminal success on source commit `a88bc5f4498ea1b11acc0acfeb66b5aa9d6531be`: node-regression real Linux npm ci + 24/24 tests including `tests/files-manager.test.ts` exit 0; Python SDK 14/14; quality strict TypeScript/ESLint; Next.js standalone (22/22 static pages); isolated Docker/BuildKit 3-image verification `DATASTACK_THREE_IMAGES_BUILD_PASS`; Windows committed-lock npm ci + Node SDK compatibility; security-audit `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO` and `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Audit full=22 moderate/high=0/critical=0 (baseline 24), runtime=20 moderate/high=0/critical=0 (baseline 22). These are affected package entries, not distinct advisories. No new CI workflow or extra broad CI dispatched.
|
||||||
|
|
||||||
|
Exact approved patch: `package.json` scoped `overrides.cloudcmd.dompurify=3.4.18`, preserving `overrides.next.postcss=8.5.33`; npm-generated `package-lock.json` changes solely `node_modules/dompurify` version 3.2.7 -> 3.4.18 and verified `resolved`/SHA512 integrity. Baseline Cloudcmd 19.24.0, Monaco Editor 0.55.1, Next.js 15.5.29, all unrelated locked package versions, URL, hash, dependency and metadata preserved. Gitea published candidate lock bytes match disposable SHA-256 `57c5ea9df9f28be496f8c3202fbc280b0066bf8c216b8d8a3ca98b6f7b942854`. Exact candidate tree diff 420 entries, 8 approved changed files only (`package.json`, `package-lock.json`, notes/001–006), 0 unauthorized paths. No runtime/UI/auth/API/SDK/tests/CI/Dockerfiles/Compose/db/schema/data modifications.
|
||||||
|
|
||||||
|
Phase ledger: P04-S02 fully CI-accepted, publication gate pending. **Residual risk:** 20 moderate runtime affected package entries remain, including Cloudcmd transitive dependencies `got`, `glob`, `minimatch`, nested `file-type` and their parent chains. Current npm audit suggests `cloudcmd@17.0.2` (a major downgrade), explicitly out-of-scope; other transitive fixes need major-version dependency-contract changes. No force/downgrade/unverified override. Security improvement verified but NOT zero vulnerabilities. Production release stays NO-GO; remaining advisory disposition and live readiness need independent scope approval.
|
||||||
|
|
||||||
|
**Merge gate:** CI #92 7/7 PASS, approved 8-file patch integrity PASS; main squash merge and Gitea/GitHub/GitLab exact Git content-tree verification required before P04-S02 CLOSED. This note is written before that publication, so final remote commit SHAs shall be appended to the PR closure report after verification. No deployment, migrations, prune, tag, release.
|
||||||
|
|
||||||
@@ -64,3 +64,86 @@ The previous #58 package-lock error was resolved by adding the exact nested opti
|
|||||||
| R-03 availability | Redis `llen` rejection leaves Express `/health` online with `redisConnected:false`. Recovery flips the flag while HTTP 200 remains unchanged. | Public 200 is liveness by existing contract; a protected dependency readiness probe/503 is **not implemented** and needs explicit approval. |
|
| R-03 availability | Redis `llen` rejection leaves Express `/health` online with `redisConnected:false`. Recovery flips the flag while HTTP 200 remains unchanged. | Public 200 is liveness by existing contract; a protected dependency readiness probe/503 is **not implemented** and needs explicit approval. |
|
||||||
|
|
||||||
Security invariants: do not expose secrets in ACK/errors, keep JWT/auth and 401/403/429/503/507 contracts unchanged, and never run failure-injection against actual Redis, Postgres, production API, live events, or user CSV. Disposable fake Redis and temporary files only. CI #63: https://dev.ivib.cc/vibtools/DataStack/actions/runs/63; results tracked in notes/003.
|
Security invariants: do not expose secrets in ACK/errors, keep JWT/auth and 401/403/429/503/507 contracts unchanged, and never run failure-injection against actual Redis, Postgres, production API, live events, or user CSV. Disposable fake Redis and temporary files only. CI #63: https://dev.ivib.cc/vibtools/DataStack/actions/runs/63; results tracked in notes/003.
|
||||||
|
|
||||||
|
|
||||||
|
## P02 Runtime Candidate — Error-path change inventory (pending CI)
|
||||||
|
|
||||||
|
| Path | Candidate result | Explicit limitation |
|
||||||
|
|---|---|---|
|
||||||
|
| Redis `RPUSH` rejects with configured Redis client | Retains event in volatile RAM as preexisting fallback but returns `BUFFER_PERSIST_FAILED` to Socket.IO via thrown error; **no successful ACK** for that failure; error text does not expose connection details | No-Redis intentional legacy RAM mode unchanged, still not a durable guarantee |
|
||||||
|
| `sync_data` and `batch_sync` failures | `success:false`, `code:BUFFER_PERSIST_FAILED` and best-effort rollback of previously claimed idempotency keys, negative failure marker for in-process retries | If claim rollback cannot reach Redis and process crashes, durable claim/queue atomicity remains unsolved |
|
||||||
|
| Redis buffer `LTRIM` rejects after successful CSV append | Subsequent flush checks physically stored CSV IDs and avoids appending matching rows again; Redis is trimmed only after successful check/write | Concurrent independent CSV appenders, compressed archived CSV, corrupt partial rows, and per-flush CSV scan cost require separate proof |
|
||||||
|
| Legacy public health | **UNCHANGED** HTTP 200 liveness contract and Redis-connected diagnostic field; R-03 readiness is not introduced in this approval | Does not prove full DB/Redis transactional readiness |
|
||||||
|
|
||||||
|
Security/API invariants: leave token/JWT/auth/rate limits/quotas/webhooks/MCP/sandbox/UI/SDK unchanged. No production Redis/Postgres/API/CSV mutation. No lockfile/dependency versions or CI workflow changes. CI acceptance remains pending; these are code-level observations, not release certification.
|
||||||
|
|
||||||
|
|
||||||
|
## P02 Final Fault / Recovery Matrix
|
||||||
|
|
||||||
|
| Case | Final behavior target and evidence |
|
||||||
|
|---|---|
|
||||||
|
| Redis RPUSH rejects with Redis configured | `BUFFER_PERSIST_FAILED` negative ACK, no false durable success; candidate CI #66 verified single/batch, final CI #69 for marker TTL |
|
||||||
|
| Redis idempotency claim survives failed queue write | Positive-time separate failed marker survives cleanup after +61000ms (below five-minute TTL); two retries remain negative rather than `DUPLICATE_DROPPED` success. Final disposable fixture `tests/phase02-sync.test.ts` |
|
||||||
|
| Mixed `batch_sync` contains stale claim and a new durable record | Overall response `success:false` with `BUFFER_PERSIST_FAILED`, even if some new records committed; preserve counts and avoid success cover-up |
|
||||||
|
| Redis deliberately disabled | Historical RAM mode unchanged; distinguish this existing best-effort contract from configured Redis durability |
|
||||||
|
| CSV append success but Redis LTRIM failure | Replay scans written event IDs and does not duplicate physical row before trim; direct Port4000 and web-buffer regression |
|
||||||
|
| Rotated / gzip CSV, quoted multiline payload | Streaming CSV parser, exact client/date/eventId match, no duplicate append or data-format change |
|
||||||
|
| Matching CSV partial/corrupt | Fail closed (no Redis trim); disposable repair allows retry; do not touch real user CSV |
|
||||||
|
| Public /health & 401/403/429/503/507 | Preserve existing behavior; no new endpoint or error contract beyond already approved BUFFER_PERSIST_FAILED |
|
||||||
|
|
||||||
|
**Limitations:** no globally atomic Redis+CSV transaction, stale Redis claim after process crash may require TTL/retry policy, and per-partition scan has data-size-proportional I/O even though memory is streamed. No live production reliability or throughput certification. CI #66 SUCCESS; final CI #69 pending.
|
||||||
|
|
||||||
|
|
||||||
|
### Additional P02 final-acceptance negative ACK case
|
||||||
|
|
||||||
|
If a `batch_sync` contains only (a) an uncommitted Redis-claim duplicate and (b) an ordinary already-durable duplicate, the old `processed===0 && duplicates===0 && rejected>0` test did not trigger; success ACK incorrectly masked volatile claim. Narrow correction uses `if (volatileDuplicate)` before the generic aggregate success path, independent of `processed` or `duplicates`. Specific test in `tests/phase02-sync.test.ts` injects Redis RPUSH failure and fake stale claim then includes the existing durable duplicate. All pending-claim ACKs must remain `BUFFER_PERSIST_FAILED`, even if some other events already committed. Terminal CI #72 pending; no global 5-minute clock advancement inside sockets.
|
||||||
|
|
||||||
|
|
||||||
|
### P02 terminal CI #72 error handling acceptance
|
||||||
|
|
||||||
|
Gitea #72 **SUCCESS** at verified final code SHA `acca8be01a31f8d3972564cbaec8fd3117a0b551`; all 24 TS suites exit zero including stale claim, mixed batch (both with new durable event and old durable duplicate), Redis-free compatibility, gzip/rotated CSV and corrupt matching CSV fail-closed. `BUFFER_PERSIST_FAILED` is negative-ACK response on configured Redis write failure, rather than success for RAM fallback. Redis-free RAM mode old semantics unchanged. Existing 401/403/429/503/507 tests unaffected and pass within 24 suites. Public /health liveness remains HTTP 200 with `redisConnected:false` during mocked outage; separate readiness endpoint not approved. Remaining risk: no atomic Redis+CSV commit, no global exactly-once guarantee, CSV partition scan disk I/O grows with volume. No live traffic test.
|
||||||
|
|
||||||
|
## P03 runner/security defect register — 2026-11-22
|
||||||
|
|
||||||
|
- **CI-06:** `python:3.12-slim-bookworm` lacked Node for `actions/checkout@v4` (127). Approved workflow-only correction: `node:22-bookworm-slim` plus Debian Python 3.11 venv and prerequisites; narrow CI #78 **14/14 PASS**.
|
||||||
|
- **CI-07:** Historical classic Docker `COPY node_modules/@uiw` layer-export failure was not reproduced using disposable BuildKit builder; #77 Compose and 3/3 builds plus image inspect **PASS**; no Dockerfile modifications or shared prune.
|
||||||
|
- **CI-08:** **CI-08 security disposition:** `npm audit --json` full=26 affected packages (25 moderate, 1 high); runtime=24 (23 moderate, 1 high), **not counts of distinct advisories**. High-severity advisories [GHSA-6g55-p6wh-862q](https://github.com/advisories/GHSA-6g55-p6wh-862q), [GHSA-r28c-9q8g-f849](https://github.com/advisories/GHSA-r28c-9q8g-f849) affect Next 15.5.29's nested `postcss@8.4.31` despite patched root `postcss@8.5.33`; npm recommends major `next@16.4.6`. Direct `cloudcmd@19.24.0` carries moderate transitive issues (`dompurify`, `file-type`, `glob`, `minimatch`, `got`); npm recommends `cloudcmd@17.0.2` downgrade, not a verified safe pin. No speculative package pin, major upgrade or force fix; a separately tested compatible remediation is needed before release.
|
||||||
|
- **Runtime API errors:** P02's 401/403/429/503/507, `BUFFER_PERSIST_FAILED` negative ACK, Redis/CSV replay behavior and public `/health` shape unchanged; no live production fault claims.
|
||||||
|
|
||||||
|
**Scope / release decision:** only `.gitea/workflows/datastack-ci.yml`, `.gitea/workflows/datastack-docker-ci.yml`, and `notes/001–006` changed. No `package.json`/lock, Dockerfile, application UI/UX, runtime, auth, SDK, database, Compose, production data, secret, shared Docker prune, deployment, migration, tag or release. P03 technical CI and advisory-triage acceptance verified by #77 non-Python jobs plus #78 Python; do **not** mislabel #77 overall as successful. Final phase publication requires one main commit and Gitea/GitHub/GitLab exact tracked tree parity. **Production release NO-GO** while runtime high advisory is unresolved; live production health, throughput and deployment remain unverified.
|
||||||
|
|
||||||
|
## P04-S01 blocked security reconciliation report
|
||||||
|
|
||||||
|
P04-S01 SEC-01 BLOCKED — 2026-11-22. Approved scoped `overrides.next.postcss=8.5.33` trialed in disposable Windows npm 10.9.8 workspace only. `npm install --package-lock-only` EXIT 0, but `npm ls --package-lock-only postcss --all` EXIT 1 (`ELSPROBLEMS`: Next nested `postcss@8.4.31` invalid; expected 8.5.33); npm dedupe also kept invalid dependency. Read-only `npm audit --omit=dev --json` EXIT 1, runtime high 1/moderate 23/critical 0. Lockfile regeneration caused unrelated churn (91 package entries added, 106 removed, 69 changed, including two lightningcss optional 1.33.0 -> 1.30.1). Fresh minimal Next 15.5.29 fixture with identical override resolves PostCSS 8.5.33, isolating failure to DataStack committed lock graph. `npm ci --dry-run` exit 0 does not prove real clean install. Exact stop rule applied: no dependency patch, no CI acceptance, no main commit, no GitHub/GitLab sync, no release/deploy; production NO-GO. Need separate explicit narrowly scoped lock reconciliation approval.
|
||||||
|
|
||||||
|
Do not treat successful npm install/dedupe/dry-run as security or compatibility validation. High PostCSS remains reachable through lock graph, no runtime handlers or error codes touched.
|
||||||
|
|
||||||
|
## P04-S01B exact failed acceptance condition — 2026-11-22
|
||||||
|
|
||||||
|
PostCSS resolution fixed within disposable test copy, with audit high/critical 0, but lockfile metadata integrity requirement **FAIL**: npm wrote 67 unrelated package-entry metadata differences. No API error/ACK behavior touched. Existing peer dependency warning for `@putout/eslint@6.1.1` versus ESLint 9 is non-blocking in the isolated lock generation and not permission to upgrade it. Stop before acceptance CI or source edit; production/main lock and security-audit workflow remain unchanged.
|
||||||
|
|
||||||
|
## P04-S01C security gate and error contract — 2026-11-22
|
||||||
|
|
||||||
|
SEC-01 candidate remediation verified: Next 15.5.29/Cloudcmd 19.24.0 unchanged, PostCSS 8.5.33 deduped, nested vulnerable PostCSS removed without unrelated dependency integrity/metadata drift. Focused CI #85 read-only audit: full affected packages moderate=24/high=0/critical=0; runtime moderate=22/high=0/critical=0, and `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO` succeeded. New gate fails on incomplete severity counts and any runtime high/critical. Moderate Cloudcmd transitive issues intentionally remain P04-S02. No JWT/auth, HTTP/socket error codes, Redis durability, CSV replay, SDK/API/UI or production health changes; Windows CI still queued and publication pending.
|
||||||
|
|
||||||
|
## P04-S01C FINAL CI acceptance (7/7) — 2026-11-22
|
||||||
|
|
||||||
|
Gitea CI #85 https://dev.ivib.cc/vibtools/DataStack/actions/runs/85 COMPLETED/SUCCESS; tested implementation SHA c759be990c032c2474d2d0bad519c643dec11a3a; later changes to candidate are documentation only. All seven jobs terminal SUCCESS: node-regression (Linux committed-lock npm ci and exactly 24/24 TypeScript suites); python-regression (Ran 14 tests, OK); quality (strict tsc and ESLint); production-build (Next 15.5.29 standalone, 22/22 pages); docker-image-build (three isolated BuildKit ingestion/web/proxy images); windows-compatibility (Windows committed-lock npm ci and Node SDK build, job #182, terminal SUCCESS); security-audit (runtime moderate=22 high=0 critical=0; full moderate=24 high=0 critical=0; DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO). Single Gitea acceptance run used, no additional broad reruns.
|
||||||
|
|
||||||
|
Security improvement: remove Next nested vulnerable PostCSS and enforce runtime high/critical fail gate. Existing 22 moderate runtime affected-package entries remain, not a security-clean claim for all severities. No change to runtime error codes/auth/API contracts. CI fully accepted; publication not yet asserted in these pre-merge notes. Final phase CLOSED requires one squash merge of PR #5 into Gitea main and verified identical tracked content across Gitea/GitHub/GitLab; record actual new SHAs in terminal report. Production release remains NO-GO pending P04-S02 moderate vulnerability triage and separately authorized live readiness verification.
|
||||||
|
|
||||||
|
## P04-S02 scoped candidate — 2026-11-22
|
||||||
|
|
||||||
|
Baseline Gitea main `2d467ff1494858164a37814f7dbc2b321f510291`, tree `648c76ca496d2c9bf22a16f558c9ba347a45d4f7`. Cloudcmd 19.24.0 -> deepword 11.0.1 -> monaco-editor 0.55.1 pins DOMPurify 3.2.7, vulnerable according to npm advisory inventory. Disposable Windows npm 10.9.8 `npm view dompurify@3.4.18` verified, scoped npm override `overrides.cloudcmd.dompurify=3.4.18` resolves `dompurify@3.4.18` via `npm ls --package-lock-only dompurify --all --depth=5` (exit 0). `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline` exit 0. npm graph initially changed 67 unrelated metadata entries; deterministically restored exact baseline values after validating unrelated versions, URLs, integrity and dependencies. Final lock has **one changed package entry**, DOMPurify; same-major replacement plus verified registry `resolved` and `integrity`, no other packages added/deleted/changed. Candidate runtime audit affected-package counts **moderate 20/high 0/critical 0** versus baseline **22/0/0**; full candidate **moderate 22/high 0/critical 0** versus baseline **24/0/0**. DOMPurify and Monaco Editor dropped from vulnerable-package inventory. Other moderate packages (got, glob, minimatch, nested file-type, Cloudcmd transitive chain) remain; they require major package changes under current advisory constraints, prohibited by Scope Lock. This is a **pre-acceptance candidate**: do not claim application compatibility, CI PASS, publication, or phase CLOSED before full focused Gitea CI and exact three-remote tree parity.
|
||||||
|
|
||||||
|
Error contract: no user-facing error codes or handler changes. Security audit improvement from 22 to 20 moderate runtime affected packages, high/critical stays zero in disposable audit. No claim of eliminating all moderate advisories.
|
||||||
|
|
||||||
|
## P04-S02 final CI acceptance — 2026-11-22
|
||||||
|
|
||||||
|
Gitea CI #92 (https://dev.ivib.cc/vibtools/DataStack/actions/runs/92) **COMPLETED / SUCCESS**, all 7/7 jobs terminal success on source commit `a88bc5f4498ea1b11acc0acfeb66b5aa9d6531be`: node-regression real Linux npm ci + 24/24 tests including `tests/files-manager.test.ts` exit 0; Python SDK 14/14; quality strict TypeScript/ESLint; Next.js standalone (22/22 static pages); isolated Docker/BuildKit 3-image verification `DATASTACK_THREE_IMAGES_BUILD_PASS`; Windows committed-lock npm ci + Node SDK compatibility; security-audit `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO` and `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Audit full=22 moderate/high=0/critical=0 (baseline 24), runtime=20 moderate/high=0/critical=0 (baseline 22). These are affected package entries, not distinct advisories. No new CI workflow or extra broad CI dispatched.
|
||||||
|
|
||||||
|
Exact approved patch: `package.json` scoped `overrides.cloudcmd.dompurify=3.4.18`, preserving `overrides.next.postcss=8.5.33`; npm-generated `package-lock.json` changes solely `node_modules/dompurify` version 3.2.7 -> 3.4.18 and verified `resolved`/SHA512 integrity. Baseline Cloudcmd 19.24.0, Monaco Editor 0.55.1, Next.js 15.5.29, all unrelated locked package versions, URL, hash, dependency and metadata preserved. Gitea published candidate lock bytes match disposable SHA-256 `57c5ea9df9f28be496f8c3202fbc280b0066bf8c216b8d8a3ca98b6f7b942854`. Exact candidate tree diff 420 entries, 8 approved changed files only (`package.json`, `package-lock.json`, notes/001–006), 0 unauthorized paths. No runtime/UI/auth/API/SDK/tests/CI/Dockerfiles/Compose/db/schema/data modifications.
|
||||||
|
|
||||||
|
Error/security semantics: no API error handler, permission, middleware, auth or HTTP/WebSocket changes. Upgrade only same-major DOMPurify. **Residual risk:** 20 moderate runtime affected package entries remain, including Cloudcmd transitive dependencies `got`, `glob`, `minimatch`, nested `file-type` and their parent chains. Current npm audit suggests `cloudcmd@17.0.2` (a major downgrade), explicitly out-of-scope; other transitive fixes need major-version dependency-contract changes. No force/downgrade/unverified override. Security improvement verified but NOT zero vulnerabilities. Production release stays NO-GO; remaining advisory disposition and live readiness need independent scope approval.
|
||||||
|
|
||||||
|
**Merge gate:** CI #92 7/7 PASS, approved 8-file patch integrity PASS; main squash merge and Gitea/GitHub/GitLab exact Git content-tree verification required before P04-S02 CLOSED. This note is written before that publication, so final remote commit SHAs shall be appended to the PR closure report after verification. No deployment, migrations, prune, tag, release.
|
||||||
@@ -52,3 +52,73 @@ Do not mark "actual working" simply because a patch compiled. Distinguish source
|
|||||||
- R-02: existing CSV-before-Redis-trim order audited, *not corrected*. Exactly-once replay safety is **NOT VERIFIED**.
|
- R-02: existing CSV-before-Redis-trim order audited, *not corrected*. Exactly-once replay safety is **NOT VERIFIED**.
|
||||||
- R-03: current public Express health 200 plus `redisConnected:false` is liveness, not dependency readiness. Separate readiness remains **NOT IMPLEMENTED**.
|
- R-03: current public Express health 200 plus `redisConnected:false` is liveness, not dependency readiness. Separate readiness remains **NOT IMPLEMENTED**.
|
||||||
- Gitea disposable CI #63: https://dev.ivib.cc/vibtools/DataStack/actions/runs/63. The previous P01 working results remain valid; no live read/write test or production deploy is included in P02.
|
- Gitea disposable CI #63: https://dev.ivib.cc/vibtools/DataStack/actions/runs/63. The previous P01 working results remain valid; no live read/write test or production deploy is included in P02.
|
||||||
|
|
||||||
|
|
||||||
|
## P02 Runtime Candidate Feature Delta — 2026-11-22 (NOT RELEASED)
|
||||||
|
|
||||||
|
- **Intended change:** failure cases on socket ingestion report `BUFFER_PERSIST_FAILED` instead of positive ACK when Redis RPUSH rejects; single and batch code paths tested by new disposable assertions. Ordinary success ACK, auth, limits and SDK public interfaces intended unchanged.
|
||||||
|
- **Intended change:** Redis replay flush on both ingestion/Next paths inspects existing client CSV partitions and filters already-stored event IDs before reappend; safe failure leaves Redis queue untrimmed. This prevents the tested post-append/LTRIM replay duplication but does **not** prove transactionally exactly-once delivery or zero performance impact.
|
||||||
|
- **No changes:** application UI/UX, auth, SDK, database schema, Docker, environment, public R-03 health, other features and production runtime. No deployment or live verification.
|
||||||
|
- **Status:** P02 code candidate locally committed, acceptance CI blocked by Gitea unavailability; **NOT CLOSED**, not production-ready. P01 remains only completed phase.
|
||||||
|
|
||||||
|
|
||||||
|
## P02 Final Feature/Compatibility Register — 2026-11-22
|
||||||
|
|
||||||
|
Existing UI/UX, admin auth, SDK public signatures, health route, database schema, Docker and deployment: **UNCHANGED**. Only existing Socket.IO single/batch persistence failure ACK/idempotency cleanup, Redis->CSV replay duplicate filtering and disposable tests changed during P02. No new production feature, public endpoint, release or migration. CI #66 terminal SUCCESS (24/24 TS suites, strict types, ESLint, Windows Node SDK, Next production build); final CI #69 pending. Client-facing `BUFFER_PERSIST_FAILED` error is used only for previously incorrectly acknowledged Redis-backed failure; intentionally Redis-free RAM mode is preserved. CSV write content/format unchanged, gzip and rotation replay acceptance covered in final test fixture. All live status values are **NOT VERIFIED / NO DEPLOYMENT**. No guarantee of global exactly-once delivery or hard bound on disk scans; future throughput benchmark/design approval if required. P03 still gated on final P02 close.
|
||||||
|
|
||||||
|
|
||||||
|
### P02 final acceptance correction checkpoint
|
||||||
|
|
||||||
|
Gitea #69 node-regression observed one **fixture timeout**, not a proven new production feature defect, caused by altering global `Date.now`. A separate source-proven mixed batch all-duplicate ACK bug was corrected only in `server/socket/handlers.js`, test case added in existing allowed TS file. Revised `acca8be01a31f8d3972564cbaec8fd3117a0b551` pending focused Gitea CI #72. Existing UI/SDK/auth/public health remain unchanged. This is not evidence that production has been deployed or live-tested.
|
||||||
|
|
||||||
|
|
||||||
|
### P02 verified feature-state reconciliation — Gitea #72 SUCCESS
|
||||||
|
|
||||||
|
- **P02 R-01: FIXED in source and CI-PASS, conditional scope.** Configured Redis queue RPUSH rejects => `BUFFER_PERSIST_FAILED` negative ACK for single/mixed batches, stale Redis idempotency claim cannot be reported as durable success; clock arithmetic and actual negative ACK retries verified. Intentionally Redis-free RAM mode historical best-effort ACK **preserved**, not a lossless guarantee.
|
||||||
|
- **P02 R-02: FIXED for verified crash/replay windows**, not global transactionality. Port4000 and Port3000 CSV replay suppresses duplicate same-client/day/eventId across plain, rotated and gz; failed reads prevent trimming; dedicated regression tests PASS. Streaming parser bounded memory, disk read throughput/latency not yet load-tested.
|
||||||
|
- **P02 R-03: UNCHANGED**: public /health remains backwards-compatible liveness, not a dependency-readiness endpoint. Production live health **NOT VERIFIED**.
|
||||||
|
- All other runtime features/UI/auth/API/SDK/database/Compose and package dependencies unchanged. **P02 CI ACCEPTED at #72**; code SHA `acca8be01a31f8d3972564cbaec8fd3117a0b551`. Phase closed only after verified 3-way content sync. No P03 implementation or production deployment.
|
||||||
|
|
||||||
|
## P03 feature invariance — 2026-11-22
|
||||||
|
|
||||||
|
P03 modifies workflow verification and Markdown evidence only, not a product feature. No UI, API, auth, SDK, DB, CSV, Redis, Dockerfile, Compose or deployment change. The P02 regression contract remains 24/24 PASS; corrected Python SDK passes 14/14; all three Docker images build without running them. Read-only npm advisory triage is complete but **not security-clean**.
|
||||||
|
|
||||||
|
**CI-08 security disposition:** `npm audit --json` full=26 affected packages (25 moderate, 1 high); runtime=24 (23 moderate, 1 high), **not counts of distinct advisories**. High-severity advisories [GHSA-6g55-p6wh-862q](https://github.com/advisories/GHSA-6g55-p6wh-862q), [GHSA-r28c-9q8g-f849](https://github.com/advisories/GHSA-r28c-9q8g-f849) affect Next 15.5.29's nested `postcss@8.4.31` despite patched root `postcss@8.5.33`; npm recommends major `next@16.4.6`. Direct `cloudcmd@19.24.0` carries moderate transitive issues (`dompurify`, `file-type`, `glob`, `minimatch`, `got`); npm recommends `cloudcmd@17.0.2` downgrade, not a verified safe pin. No speculative package pin, major upgrade or force fix; a separately tested compatible remediation is needed before release.
|
||||||
|
|
||||||
|
**Scope / release decision:** only `.gitea/workflows/datastack-ci.yml`, `.gitea/workflows/datastack-docker-ci.yml`, and `notes/001–006` changed. No `package.json`/lock, Dockerfile, application UI/UX, runtime, auth, SDK, database, Compose, production data, secret, shared Docker prune, deployment, migration, tag or release. P03 technical CI and advisory-triage acceptance verified by #77 non-Python jobs plus #78 Python; do **not** mislabel #77 overall as successful. Final phase publication requires one main commit and Gitea/GitHub/GitLab exact tracked tree parity. **Production release NO-GO** while runtime high advisory is unresolved; live production health, throughput and deployment remain unverified.
|
||||||
|
|
||||||
|
## P04-S01 blocked security reconciliation report
|
||||||
|
|
||||||
|
P04-S01 SEC-01 BLOCKED — 2026-11-22. Approved scoped `overrides.next.postcss=8.5.33` trialed in disposable Windows npm 10.9.8 workspace only. `npm install --package-lock-only` EXIT 0, but `npm ls --package-lock-only postcss --all` EXIT 1 (`ELSPROBLEMS`: Next nested `postcss@8.4.31` invalid; expected 8.5.33); npm dedupe also kept invalid dependency. Read-only `npm audit --omit=dev --json` EXIT 1, runtime high 1/moderate 23/critical 0. Lockfile regeneration caused unrelated churn (91 package entries added, 106 removed, 69 changed, including two lightningcss optional 1.33.0 -> 1.30.1). Fresh minimal Next 15.5.29 fixture with identical override resolves PostCSS 8.5.33, isolating failure to DataStack committed lock graph. `npm ci --dry-run` exit 0 does not prove real clean install. Exact stop rule applied: no dependency patch, no CI acceptance, no main commit, no GitHub/GitLab sync, no release/deploy; production NO-GO. Need separate explicit narrowly scoped lock reconciliation approval.
|
||||||
|
|
||||||
|
No change to UI, API, SDK, auth, file manager, CSV, Redis, storage, feature set, Docker, deployment or production data. P03 baseline regressions preserved without claiming retest.
|
||||||
|
|
||||||
|
## P04-S01B no-feature-change / gate state — 2026-11-22
|
||||||
|
|
||||||
|
Candidate scope proof: PostCSS root 8.5.33 can satisfy Next 15.5.29 when stale nested 8.4.31 removed in a disposable lock, with moderate=22/high=0/critical=0 read-only runtime audit. Not published: regenerated metadata edits exceeded approved lock scope. Existing UI/UX, auth, API, SDK, CSV, Redis, Cloudcmd, Docker and production service behavior remain untouched; **do not claim regression acceptance or security remediation of published code**. Next approval must address exact npm-generated metadata drift and integrity preservation, not feature work.
|
||||||
|
|
||||||
|
## P04-S01C feature preservation and verification — 2026-11-22
|
||||||
|
|
||||||
|
No feature, UI, backend runtime, public API, auth, database/schema, SDK, CSV storage, Dockerfile or Compose source change. Package manifest scoped override and lockfile one-entry deletion only; Next 15.5.29 and Cloudcmd 19.24.0 preserved. Gitea CI #85 node 24/24, Python 14/14, strict types/lint, Next standalone and 3 Docker images completed SUCCESS. Security runtime audit high/critical 0; moderate advisories still pending separate triage. Windows compatibility QUEUED at checkpoint; no production release, deployment, migration or tag and no accepted main/mirror publication yet.
|
||||||
|
|
||||||
|
## P04-S01C FINAL CI acceptance (7/7) — 2026-11-22
|
||||||
|
|
||||||
|
Gitea CI #85 https://dev.ivib.cc/vibtools/DataStack/actions/runs/85 COMPLETED/SUCCESS; tested implementation SHA c759be990c032c2474d2d0bad519c643dec11a3a; later changes to candidate are documentation only. All seven jobs terminal SUCCESS: node-regression (Linux committed-lock npm ci and exactly 24/24 TypeScript suites); python-regression (Ran 14 tests, OK); quality (strict tsc and ESLint); production-build (Next 15.5.29 standalone, 22/22 pages); docker-image-build (three isolated BuildKit ingestion/web/proxy images); windows-compatibility (Windows committed-lock npm ci and Node SDK build, job #182, terminal SUCCESS); security-audit (runtime moderate=22 high=0 critical=0; full moderate=24 high=0 critical=0; DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO). Single Gitea acceptance run used, no additional broad reruns.
|
||||||
|
|
||||||
|
No product feature, UI, runtime, API, auth, SDK, data format, Docker or deployment change. Only resolved PostCSS dependency and CI security gate. CI fully accepted; publication not yet asserted in these pre-merge notes. Final phase CLOSED requires one squash merge of PR #5 into Gitea main and verified identical tracked content across Gitea/GitHub/GitLab; record actual new SHAs in terminal report. Production release remains NO-GO pending P04-S02 moderate vulnerability triage and separately authorized live readiness verification.
|
||||||
|
|
||||||
|
## P04-S02 scoped candidate — 2026-11-22
|
||||||
|
|
||||||
|
Baseline Gitea main `2d467ff1494858164a37814f7dbc2b321f510291`, tree `648c76ca496d2c9bf22a16f558c9ba347a45d4f7`. Cloudcmd 19.24.0 -> deepword 11.0.1 -> monaco-editor 0.55.1 pins DOMPurify 3.2.7, vulnerable according to npm advisory inventory. Disposable Windows npm 10.9.8 `npm view dompurify@3.4.18` verified, scoped npm override `overrides.cloudcmd.dompurify=3.4.18` resolves `dompurify@3.4.18` via `npm ls --package-lock-only dompurify --all --depth=5` (exit 0). `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline` exit 0. npm graph initially changed 67 unrelated metadata entries; deterministically restored exact baseline values after validating unrelated versions, URLs, integrity and dependencies. Final lock has **one changed package entry**, DOMPurify; same-major replacement plus verified registry `resolved` and `integrity`, no other packages added/deleted/changed. Candidate runtime audit affected-package counts **moderate 20/high 0/critical 0** versus baseline **22/0/0**; full candidate **moderate 22/high 0/critical 0** versus baseline **24/0/0**. DOMPurify and Monaco Editor dropped from vulnerable-package inventory. Other moderate packages (got, glob, minimatch, nested file-type, Cloudcmd transitive chain) remain; they require major package changes under current advisory constraints, prohibited by Scope Lock. This is a **pre-acceptance candidate**: do not claim application compatibility, CI PASS, publication, or phase CLOSED before full focused Gitea CI and exact three-remote tree parity.
|
||||||
|
|
||||||
|
Feature preservation: Cloudcmd major 19.24.0, Monaco Editor 0.55.1, Next 15.5.29, PostCSS override 8.5.33, runtime APIs/UI/auth/SDK/Redis/CSV/storage and Dockerfiles unchanged. Existing File Manager behavior must be verified by read-only tests.
|
||||||
|
|
||||||
|
## P04-S02 final CI acceptance — 2026-11-22
|
||||||
|
|
||||||
|
Gitea CI #92 (https://dev.ivib.cc/vibtools/DataStack/actions/runs/92) **COMPLETED / SUCCESS**, all 7/7 jobs terminal success on source commit `a88bc5f4498ea1b11acc0acfeb66b5aa9d6531be`: node-regression real Linux npm ci + 24/24 tests including `tests/files-manager.test.ts` exit 0; Python SDK 14/14; quality strict TypeScript/ESLint; Next.js standalone (22/22 static pages); isolated Docker/BuildKit 3-image verification `DATASTACK_THREE_IMAGES_BUILD_PASS`; Windows committed-lock npm ci + Node SDK compatibility; security-audit `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO` and `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Audit full=22 moderate/high=0/critical=0 (baseline 24), runtime=20 moderate/high=0/critical=0 (baseline 22). These are affected package entries, not distinct advisories. No new CI workflow or extra broad CI dispatched.
|
||||||
|
|
||||||
|
Exact approved patch: `package.json` scoped `overrides.cloudcmd.dompurify=3.4.18`, preserving `overrides.next.postcss=8.5.33`; npm-generated `package-lock.json` changes solely `node_modules/dompurify` version 3.2.7 -> 3.4.18 and verified `resolved`/SHA512 integrity. Baseline Cloudcmd 19.24.0, Monaco Editor 0.55.1, Next.js 15.5.29, all unrelated locked package versions, URL, hash, dependency and metadata preserved. Gitea published candidate lock bytes match disposable SHA-256 `57c5ea9df9f28be496f8c3202fbc280b0066bf8c216b8d8a3ca98b6f7b942854`. Exact candidate tree diff 420 entries, 8 approved changed files only (`package.json`, `package-lock.json`, notes/001–006), 0 unauthorized paths. No runtime/UI/auth/API/SDK/tests/CI/Dockerfiles/Compose/db/schema/data modifications.
|
||||||
|
|
||||||
|
Feature preservation: File Manager test and Monaco-related Next production build PASS; no UI features removed, no editor behavior modified in source. **Residual risk:** 20 moderate runtime affected package entries remain, including Cloudcmd transitive dependencies `got`, `glob`, `minimatch`, nested `file-type` and their parent chains. Current npm audit suggests `cloudcmd@17.0.2` (a major downgrade), explicitly out-of-scope; other transitive fixes need major-version dependency-contract changes. No force/downgrade/unverified override. Security improvement verified but NOT zero vulnerabilities. Production release stays NO-GO; remaining advisory disposition and live readiness need independent scope approval.
|
||||||
|
|
||||||
|
**Merge gate:** CI #92 7/7 PASS, approved 8-file patch integrity PASS; main squash merge and Gitea/GitHub/GitLab exact Git content-tree verification required before P04-S02 CLOSED. This note is written before that publication, so final remote commit SHAs shall be appended to the PR closure report after verification. No deployment, migrations, prune, tag, release.
|
||||||
@@ -237,3 +237,111 @@ return bufferQueue.length;
|
|||||||
- `P02 R-03 diagnostic: Redis query failure leaves public ingestion health 200/online` — **ok 16**, same suite exit 0, Redis flag false while liveness stays online.
|
- `P02 R-03 diagnostic: Redis query failure leaves public ingestion health 200/online` — **ok 16**, same suite exit 0, Redis flag false while liveness stays online.
|
||||||
- `P02 R-02 diagnostic: post-CSV Redis trim failure replays identical event on retry` — **ok 18**, `phase03-storage-buffer` suite exit 0; duplicate append after retry reproduced on representative web buffer path. **Direct ingestion Port 4000 CSV worker crash coverage remains outstanding** and should be requested in remediation approval before guarantee changes.
|
- `P02 R-02 diagnostic: post-CSV Redis trim failure replays identical event on retry` — **ok 18**, `phase03-storage-buffer` suite exit 0; duplicate append after retry reproduced on representative web buffer path. **Direct ingestion Port 4000 CSV worker crash coverage remains outstanding** and should be requested in remediation approval before guarantee changes.
|
||||||
- These three diagnostic tests **prove findings, do not close production-readiness**. See Gitea https://dev.ivib.cc/vibtools/DataStack/actions/runs/63; never call run successful until terminal conclusion is confirmed. No server-side source fix applied.
|
- These three diagnostic tests **prove findings, do not close production-readiness**. See Gitea https://dev.ivib.cc/vibtools/DataStack/actions/runs/63; never call run successful until terminal conclusion is confirmed. No server-side source fix applied.
|
||||||
|
|
||||||
|
|
||||||
|
## F. P02 Remediation Candidate Patch Proof — 2026-11-22
|
||||||
|
|
||||||
|
Base: Gitea/GitHub published P02 diagnostic tree `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`; GitLab distinct commit, same previous tree. Local candidate commit `44eee8877e40d1683f44502c2a0ba3d437a03b88` on isolated worktree. Initial exact delta: 5 approved source/test paths, 310 insertions and 22 deletions, no application UI, Python SDK, auth, public health route, CI or deployment edit. `git diff --check` and `node --check` on JS files PASS.
|
||||||
|
|
||||||
|
- `server/index.js:pushEventsToRedisOrFallback`: catch RPUSH failure, keep previous RAM buffering, throw sanitized `BUFFER_PERSIST_FAILED`; no positive ACK for that branch.
|
||||||
|
- `server/socket/handlers.js:sync_data/batch_sync`: negative ACK and best-effort claimed idempotency-key rollback with negative retry guard, no change to legitimate duplicate success path or existing auth/limits/error codes.
|
||||||
|
- `server/index.js:flushBuffer` and `core/data/buffer.ts:flush`: scoped streaming CSV/CSV.gz read-before-reappend (parsing quoted/multiline rows), exact client/date/event ID match; failures abort before Redis LTRIM. No changes to `server/workers/csvWriter.js`.
|
||||||
|
- `tests/phase02-sync.test.ts`: R-01 single/batch outage and duplicate claim ACK negative, successful recovery ACK, and unchanged R-03 liveness.
|
||||||
|
- `tests/phase03-storage-buffer.test.ts`: web-side replay verifies exactly one physically stored row; new direct Port 4000 writer Redis-trim failure/recovery proof. All tests on fake Redis and disposable temp CSV paths.
|
||||||
|
|
||||||
|
**Unverified:** Gitea P02 implementation acceptance was unavailable due inaccessible host/connector. Local `npm ci` not confirmed successful. Do not invent a CI run ID or claim full PASS. Full cross-process transactional exactly-once and all Redis claim crash windows are out of proof scope and may require exact additional approval. The streaming dedupe scan changes I/O cost per flush; performance validation before production is required.
|
||||||
|
|
||||||
|
|
||||||
|
### Blocked acceptance evidence — actual tool outcomes
|
||||||
|
|
||||||
|
- JS syntax: local `node --check server/index.js` and `node --check server/socket/handlers.js` PASS.
|
||||||
|
- Source scope: local `git diff --check` PASS; only the five allowed P02 runtime/test paths changed before notes updates.
|
||||||
|
- Gitea: connection returned `UNAVAILABLE: McpServerError: Connection failed`; Git remote `dev.ivib.cc` unresponsive. No Gitea CI run ID exists for candidate `44eee8877e40d1683f44502c2a0ba3d437a03b88`.
|
||||||
|
- Local Windows npm: `npm ci` failed `ETARGET yaml@2.9.1` and cleanup `EPERM` in the isolated worktree, **not** a TypeScript/test result. Dependency changes are out of P02 scope.
|
||||||
|
- Therefore neither TS typecheck nor regression-suite remediation acceptance is established; keep P02 BLOCKED and do not advance P03 or alter any protected runtime/source beyond this approval.
|
||||||
|
|
||||||
|
|
||||||
|
## F. P02 Final Acceptance Patch Index (current source/CI proof)
|
||||||
|
|
||||||
|
- Published diagnosis Gitea/GitHub main: `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`; previous GitLab main same tracked tree.
|
||||||
|
- Runtime candidate: `76cd972bdeea9ef07b4b38e9d8b5b5ed3312ca5b`; Gitea [CI #66](https://dev.ivib.cc/vibtools/DataStack/actions/runs/66) **terminal SUCCESS** across node 24/24, strict TS, lint, Next standalone and Windows Node SDK.
|
||||||
|
- Exact supplemental commit: `bf525f0d4e5e4781de7fddd2f5e7d3d602a80532` (3 changed files: `server/socket/handlers.js`, `tests/phase02-sync.test.ts`, `tests/phase03-storage-buffer.test.ts`). Compared with published diagnosis, final candidate also includes runtime `server/index.js`, `core/data/buffer.ts`, and previously modified tests, all authorized, plus `notes/001–006`.
|
||||||
|
- **R-01 arithmetic reproduction** before patch: `const marker=-t; const now=t+61000; now-marker > 300000` is true, while `now-t > 300000` is false. `server/socket/handlers.js:142-158` `cleanupMaps` expires old negative marker; `validateSingleEventAsync:293-317` sees retained Redis idempotency claim; `sync_data` returned duplicate success. Final marker is `\u0000failed-persist:<clientId>:<eventId>` with positive timestamp, corrected age check, cleanup after true TTL, clear marker after durable ACK. Mixed batches reject overall positive ACK while a failed claim remains.
|
||||||
|
- **R-02:** `server/index.js:843-897,1043-1058`, `core/data/buffer.ts:652-711,855-872`: stream-parsed same-day client CSV/CSV.gz replay suppression, then Redis trim. Focused tests inject direct Port4000 LTRIM crash and compressed part2 quoted/multiline replay; malformed matching partition causes safe no-trim, fixture repair recovers. Memory streamed, partition disk-I/O remains proportional; not globally exactly-once.
|
||||||
|
- **Compat:** Redis-free RAM ACK and old public /health unchanged; other HTTP/Socket auth, SDK and quota routes unaffected.
|
||||||
|
- Final [Gitea CI #69](https://dev.ivib.cc/vibtools/DataStack/actions/runs/69) dispatched against exact supplemental commit; until terminal result, phase stays pending and no remote publish.
|
||||||
|
|
||||||
|
|
||||||
|
### Final P02 CI #69 failure and exact retry patch
|
||||||
|
|
||||||
|
- CI #69: `phase02-sync.test.ts` subtest `stale Redis claim never becomes positive ACK after cleanup window` `ERR_TEST_FAILURE: operation has timed out` after replacing the **global** `Date.now` during Socket.IO ACK. This cannot support a server-failure claim; it was a test harness flaw. Other new replay/compatibility assertions PASS and quality/Windows jobs separately passed.
|
||||||
|
- Discovered source branch in `server/socket/handlers.js`: `if (volatileDuplicate && processed > 0)` allowed an overall `success:true` response with pending non-durable claim and already-durable duplicate when `processed=0,duplicates>0`. Revised `if (volatileDuplicate)` blocks every mixed response with an uncommitted claim; new test reproduces the exact combination.
|
||||||
|
- Test no longer overrides Date.now; independent simulated arithmetic proves the original negative-marker expiry mistake; live ACK checks run on real clock. Supplemental commit `acca8be01a31f8d3972564cbaec8fd3117a0b551`; [CI #72](https://dev.ivib.cc/vibtools/DataStack/actions/runs/72) must pass before P02 closure. No modification of package/dependency files, test CI workflow, public API, UI or production.
|
||||||
|
|
||||||
|
|
||||||
|
## G. P02 Final Acceptance — authoritative terminal CI / Scope Lock proof
|
||||||
|
|
||||||
|
- Source baseline published diagnosis `fd57a3518d14bcb8367ce9ab9c3ec306bcfc9898`. Fixed runtime candidate `76cd972bdeea9ef07b4b38e9d8b5b5ed3312ca5b`, previous #66 SUCCESS. Final corrected **source/test** SHA `acca8be01a31f8d3972564cbaec8fd3117a0b551` fully verified in [Gitea #72](https://dev.ivib.cc/vibtools/DataStack/actions/runs/72), **terminal COMPLETED SUCCESS**, Node `CI_SUITE_SUMMARY total=24 failed=0`. CI job conclusions: node-regression SUCCESS, quality SUCCESS, windows-compatibility SUCCESS, production-build SUCCESS; python-regression and docker-image-build SKIPPED by narrow dispatch.
|
||||||
|
- Exact reproducible Redis idempotency bug: old `-Date.now()` failed timestamp ages incorrectly at +61000ms, while 300000ms Redis claim TTL has not elapsed. New `\u0000failed-persist:${claimKey}` positive timestamp marker and corrected TTL comparison isolate the failed claim. Live Socket.IO retries remain negative ACK; no process-wide clock monkeypatch. Mixed all-duplicate batch also rejects pending claim instead of sending global positive ACK.
|
||||||
|
- CSV fault evidence: `tests/phase03-storage-buffer.test.ts` after CSV append but before Redis LTRIM, direct Port4000 writer, rotated `.csv.gz`, JSON multiline-quoted payload, corrupt matching partition no-trim/recovery all PASS. Test assertions cannot prove a global exactly-once protocol or live throughput cap. Full-partition scan is streaming bounded-memory, but data-size-linear disk I/O.
|
||||||
|
- Historical #69 failed one test-only global Date.now-induced Socket.IO timeout, **not** a demonstrated runtime regression; corrected #72 PASS. No secret or credential introduction, no production data access.
|
||||||
|
- Final audit `git diff --check` PASS and source JS syntax checks PASS; exact tracked source diff from diagnosis = 11 allowlisted files (3 runtime + 2 test + 6 notes); no unrelated UI/auth/SDK/public health, DB schema, package metadata, Docker, CI workflow or live deployment.
|
||||||
|
- Git release gate: final docs-only commits after tested source SHA add **no source/test differences**. Gitea PR #3 squashed into one `[skip ci]` main commit; GitHub/GitLab tracked Git trees must match Gitea before final CLOSED. If GitLab commit SHA differs because of parent history, compare tree SHA not commit SHA. Store exact remote SHAs/trees in phase close operator report; do not invent them before push.
|
||||||
|
|
||||||
|
## H. P03 exact audit and terminal evidence — 2026-11-22
|
||||||
|
|
||||||
|
Baseline Gitea/GitHub `48fb277079b380fdbf1e58cd2da40afeed0b03ee`; GitLab baseline same tracked tree `41a169eec0c05ce19bc4acc29d7a8cfe1f8dd90d` (317/317 blobs). Final P03 workflow candidate `549d575653b68c29f97e2f78036646f2e2035719`. [Gitea CI #77](https://dev.ivib.cc/vibtools/DataStack/actions/runs/77): overall **FAILURE** because the initial Python-only checkout lacked Node (exit 127), **not** an SDK regression. Its other six jobs terminal **SUCCESS**: Node `CI_SUITE_SUMMARY total=24 failed=0`, strict TS+ESLint, Next standalone build, Windows SDK, Compose + isolated BuildKit ingestion/web/proxy image build `DATASTACK_THREE_IMAGES_BUILD_PASS`, read-only advisory inventory `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Narrow [Python-only CI #78](https://dev.ivib.cc/vibtools/DataStack/actions/runs/78) terminal **SUCCESS**: Node-capable Bookworm container, Python 3.11 isolated venv, `Ran 14 tests in 3.959s`, `OK`, `DATASTACK_PYTHON_14_OF_14_PASS`. The other six jobs were deliberately skipped in #78.
|
||||||
|
|
||||||
|
**Docker evidence:** separate BuildKit `docker-container` builder with owned-builder cleanup, `--no-cache --load`, actual Compose `config --no-interpolate --quiet`, ingestion/web/proxy Dockerfiles *read-only*, three tags inspected; old `@uiw` error absent. **Python evidence:** only Node-capable image change within workflow, Python 3.11 venv with all 14 tests; Python SDK source unchanged.
|
||||||
|
|
||||||
|
**CI-08 security disposition:** `npm audit --json` full=26 affected packages (25 moderate, 1 high); runtime=24 (23 moderate, 1 high), **not counts of distinct advisories**. High-severity advisories [GHSA-6g55-p6wh-862q](https://github.com/advisories/GHSA-6g55-p6wh-862q), [GHSA-r28c-9q8g-f849](https://github.com/advisories/GHSA-r28c-9q8g-f849) affect Next 15.5.29's nested `postcss@8.4.31` despite patched root `postcss@8.5.33`; npm recommends major `next@16.4.6`. Direct `cloudcmd@19.24.0` carries moderate transitive issues (`dompurify`, `file-type`, `glob`, `minimatch`, `got`); npm recommends `cloudcmd@17.0.2` downgrade, not a verified safe pin. No speculative package pin, major upgrade or force fix; a separately tested compatible remediation is needed before release.
|
||||||
|
|
||||||
|
**Exact allowlist audit:** pre-documentation candidate diff had only two approved workflow files; final publication includes six approved `notes/001–006` updates. Workflow checks: no trailing whitespace, no `docker ... prune`, no `npm audit fix --force`, all 3 Dockerfile paths present, no runtime or dependency lock changes. Check three-remote tracked tree parity before marking phase CLOSED.
|
||||||
|
|
||||||
|
**Scope / release decision:** only `.gitea/workflows/datastack-ci.yml`, `.gitea/workflows/datastack-docker-ci.yml`, and `notes/001–006` changed. No `package.json`/lock, Dockerfile, application UI/UX, runtime, auth, SDK, database, Compose, production data, secret, shared Docker prune, deployment, migration, tag or release. P03 technical CI and advisory-triage acceptance verified by #77 non-Python jobs plus #78 Python; do **not** mislabel #77 overall as successful. Final phase publication requires one main commit and Gitea/GitHub/GitLab exact tracked tree parity. **Production release NO-GO** while runtime high advisory is unresolved; live production health, throughput and deployment remain unverified.
|
||||||
|
|
||||||
|
## I. P04-S01 approved baseline and source proof — 2026-11-22
|
||||||
|
|
||||||
|
Baseline commit `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3`, content tree `12818ba732f5abdc42af46d83278b13b6be4eafd`. P03 CI #77 `AUDIT_COUNTS runtime {moderate:23,high:1,critical:0,total:24}` (affected packages). Lockfile line proof `package-lock.json:10552-10560` Next `15.5.29` pinned nested postcss `8.4.31`; `10602-10603` resolved nested copy `8.4.31`; `11427-11428` root `8.5.33`. Gitea candidate branch `p04-s01-postcss-security-20261122`, no runtime changes. Approved exact write boundaries are only `package.json` scoped PostCSS override, npm-generated lockfile, `.gitea/workflows/datastack-ci.yml` security-audit block and notes/001–006. Acceptance pending.
|
||||||
|
|
||||||
|
## P04-S01 blocked security reconciliation report
|
||||||
|
|
||||||
|
P04-S01 SEC-01 BLOCKED — 2026-11-22. Approved scoped `overrides.next.postcss=8.5.33` trialed in disposable Windows npm 10.9.8 workspace only. `npm install --package-lock-only` EXIT 0, but `npm ls --package-lock-only postcss --all` EXIT 1 (`ELSPROBLEMS`: Next nested `postcss@8.4.31` invalid; expected 8.5.33); npm dedupe also kept invalid dependency. Read-only `npm audit --omit=dev --json` EXIT 1, runtime high 1/moderate 23/critical 0. Lockfile regeneration caused unrelated churn (91 package entries added, 106 removed, 69 changed, including two lightningcss optional 1.33.0 -> 1.30.1). Fresh minimal Next 15.5.29 fixture with identical override resolves PostCSS 8.5.33, isolating failure to DataStack committed lock graph. `npm ci --dry-run` exit 0 does not prove real clean install. Exact stop rule applied: no dependency patch, no CI acceptance, no main commit, no GitHub/GitLab sync, no release/deploy; production NO-GO. Need separate explicit narrowly scoped lock reconciliation approval.
|
||||||
|
|
||||||
|
Baseline tree `12818ba732f5abdc42af46d83278b13b6be4eafd`; npm `audit --omit=dev` includes affected node `node_modules/next/node_modules/postcss` and high GHSAs above. This proof lives in notes, not a change to repository dependencies; no patch is accepted.
|
||||||
|
|
||||||
|
## K. P04-S01B reproducible candidate proof / unaccepted patch — 2026-11-22
|
||||||
|
|
||||||
|
**Baseline:** Gitea main `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3`, tree `12818ba732f5abdc42af46d83278b13b6be4eafd`. Gitea source path `package-lock.json:10552-10603` includes Next 15.5.29 and nested PostCSS 8.4.31; root PostCSS 8.5.33. Connected Windows npm 10.9.8 temp probe downloads baseline manifest/lock; exact approved exception deletes `node_modules/next/node_modules/postcss` entry **in disposable copy only** and adds manifest `overrides.next.postcss=8.5.33`. `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline`: exit 0. `npm ls --package-lock-only postcss --all --depth=4`: exit 0; Next uses PostCSS 8.5.33 deduped and no nested 8.4.31. `npm audit --omit=dev --json` exits 1 for remaining **moderates**; counts `{moderate:22,high:0,critical:0,total:22}`; `postcss` no longer appears as vulnerable package. Zero unrelated version changes and exactly 1 removed stale nested package; **67 other package entries changed metadata**: `dev` 36, `libc` 29, 2 `dependencies` JSON order differences (field frequencies can overlap). Must preserve unrelated version AND metadata per user's approval. **STOP**: candidate not committed to repo; no real clean npm ci, no Gitea CI acceptance or workflow gate change, no main merge or 3-mirror sync. P04-S01B remains BLOCKED, released baseline still vulnerable/high=1 and production NO-GO. A future separate exact approved strategy for metadata preservation or bounded exceptions is necessary.
|
||||||
|
|
||||||
|
## L. P04-S01C candidate exact patch, CI and scope proof — 2026-11-22
|
||||||
|
|
||||||
|
- Baseline: Gitea main `fac0934a00e6c15902738a3ffafb11b5ec2f0ac3`, tree `12818ba732f5abdc42af46d83278b13b6be4eafd`; P04 notes branch `p04-s01-postcss-security-20261122`. Approved patch commit `c759be990c032c2474d2d0bad519c643dec11a3a`. PR [#5](https://dev.ivib.cc/vibtools/DataStack/pulls/5) draft pending full acceptance.
|
||||||
|
- Exact patch diff from Gitea source commit: `package.json` append only `overrides.next.postcss = 8.5.33`; `package-lock.json` delete only 26 lines for `node_modules/next/node_modules/postcss@8.4.31`; `.gitea/workflows/datastack-ci.yml:security-audit` insert 10-line runtime high/critical failure/verification gate. No changes outside 3 source/config files plus notes/001–006.
|
||||||
|
- **Generated lock preservation proof:** remove stale entry in disposable npm lock copy; regenerate using npm 10.9.8; restore 67 unrelated metadata entries byte-exactly to baseline. Structural comparison final versus baseline `added=0 removed=1 changedExistingEntries=0`; all unrelated versions, resolved URLs, integrity hashes and dependency metadata identical. `npm ls --package-lock-only postcss --all --depth=4` exit 0, Next 15.5.29 resolves root PostCSS 8.5.33. Final lockfile SHA-256 `e6b5b362877627085dc04c9ded03c019971e07ef9f4af1e78057d58961a1c091`; downloaded Gitea branch lock bytes match. Linux Gitea runner actual `npm ci` SUCCESS, followed by 24 suites. Connected Windows scratch `npm ci` completion not independently verified; Windows Gitea job outstanding.
|
||||||
|
- Gitea CI [#85](https://dev.ivib.cc/vibtools/DataStack/actions/runs/85) on source commit `c759be99`: node-regression 24/24 + SDK SUCCESS; Python 14/14 SUCCESS; quality strict TS/ESLint SUCCESS; Next 22/22 static pages + standalone SUCCESS; isolated BuildKit 3-image build SUCCESS; security-audit SUCCESS `AUDIT_COUNTS full {moderate:24,high:0,critical:0}`, `AUDIT_COUNTS runtime {moderate:22,high:0,critical:0}`, `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO`, `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Windows-compatibility QUEUED at checkpoint: no full terminal acceptance yet. No main or mirror changes, no deployment, migration, shared Docker prune, tag or release. Production NO-GO pending 7/7 and remote parity, moderate advisories/live verification.
|
||||||
|
|
||||||
|
## P04-S01C FINAL CI acceptance (7/7) — 2026-11-22
|
||||||
|
|
||||||
|
Gitea CI #85 https://dev.ivib.cc/vibtools/DataStack/actions/runs/85 COMPLETED/SUCCESS; tested implementation SHA c759be990c032c2474d2d0bad519c643dec11a3a; later changes to candidate are documentation only. All seven jobs terminal SUCCESS: node-regression (Linux committed-lock npm ci and exactly 24/24 TypeScript suites); python-regression (Ran 14 tests, OK); quality (strict tsc and ESLint); production-build (Next 15.5.29 standalone, 22/22 pages); docker-image-build (three isolated BuildKit ingestion/web/proxy images); windows-compatibility (Windows committed-lock npm ci and Node SDK build, job #182, terminal SUCCESS); security-audit (runtime moderate=22 high=0 critical=0; full moderate=24 high=0 critical=0; DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO). Single Gitea acceptance run used, no additional broad reruns.
|
||||||
|
|
||||||
|
Scope: exactly nine approved paths: package.json scoped next->postcss 8.5.33 override; package-lock.json removes only old nested postcss@8.4.31 entry; existing .gitea/workflows/datastack-ci.yml security-audit job high/critical rejection; notes/001–006 evidence. Other package metadata, versions, integrity and resolved URLs match baseline. Next.js 15.5.29 and Cloudcmd 19.24.0 preserved. Final candidate package-lock SHA256 e6b5b362877627085dc04c9ded03c019971e07ef9f4af1e78057d58961a1c091 (verified byte-for-byte with Gitea). No API, auth, SDK, UI, application runtime, database, Compose, Dockerfile, tests or production data changes, deployment, migration, prune, tag, release.
|
||||||
|
|
||||||
|
Exact job IDs: 177 Node, 178 Python, 179 quality, 180 Next production, 181 Docker three images, 182 Windows compatibility, 183 security-audit; all seven completed/success, run #85 overall completed/success. CI fully accepted; publication not yet asserted in these pre-merge notes. Final phase CLOSED requires one squash merge of PR #5 into Gitea main and verified identical tracked content across Gitea/GitHub/GitLab; record actual new SHAs in terminal report. Production release remains NO-GO pending P04-S02 moderate vulnerability triage and separately authorized live readiness verification.
|
||||||
|
|
||||||
|
## P04-S02 scoped candidate — 2026-11-22
|
||||||
|
|
||||||
|
Baseline Gitea main `2d467ff1494858164a37814f7dbc2b321f510291`, tree `648c76ca496d2c9bf22a16f558c9ba347a45d4f7`. Cloudcmd 19.24.0 -> deepword 11.0.1 -> monaco-editor 0.55.1 pins DOMPurify 3.2.7, vulnerable according to npm advisory inventory. Disposable Windows npm 10.9.8 `npm view dompurify@3.4.18` verified, scoped npm override `overrides.cloudcmd.dompurify=3.4.18` resolves `dompurify@3.4.18` via `npm ls --package-lock-only dompurify --all --depth=5` (exit 0). `npm install --package-lock-only --ignore-scripts --no-audit --no-fund --prefer-offline` exit 0. npm graph initially changed 67 unrelated metadata entries; deterministically restored exact baseline values after validating unrelated versions, URLs, integrity and dependencies. Final lock has **one changed package entry**, DOMPurify; same-major replacement plus verified registry `resolved` and `integrity`, no other packages added/deleted/changed. Candidate runtime audit affected-package counts **moderate 20/high 0/critical 0** versus baseline **22/0/0**; full candidate **moderate 22/high 0/critical 0** versus baseline **24/0/0**. DOMPurify and Monaco Editor dropped from vulnerable-package inventory. Other moderate packages (got, glob, minimatch, nested file-type, Cloudcmd transitive chain) remain; they require major package changes under current advisory constraints, prohibited by Scope Lock. This is a **pre-acceptance candidate**: do not claim application compatibility, CI PASS, publication, or phase CLOSED before full focused Gitea CI and exact three-remote tree parity.
|
||||||
|
|
||||||
|
Patch provenance: npm registry DOMPurify 3.4.18 tarball URL and integrity SHA512 in npm-generated entry, unchanged other package metadata after deterministic baseline restoration. New graph root overrides change only Cloudcmd-specific DOMPurify. One focused acceptance run pending.
|
||||||
|
|
||||||
|
## P04-S02 final CI acceptance — 2026-11-22
|
||||||
|
|
||||||
|
Gitea CI #92 (https://dev.ivib.cc/vibtools/DataStack/actions/runs/92) **COMPLETED / SUCCESS**, all 7/7 jobs terminal success on source commit `a88bc5f4498ea1b11acc0acfeb66b5aa9d6531be`: node-regression real Linux npm ci + 24/24 tests including `tests/files-manager.test.ts` exit 0; Python SDK 14/14; quality strict TypeScript/ESLint; Next.js standalone (22/22 static pages); isolated Docker/BuildKit 3-image verification `DATASTACK_THREE_IMAGES_BUILD_PASS`; Windows committed-lock npm ci + Node SDK compatibility; security-audit `DATASTACK_RUNTIME_SECURITY_HIGH_CRITICAL_ZERO` and `DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS`. Audit full=22 moderate/high=0/critical=0 (baseline 24), runtime=20 moderate/high=0/critical=0 (baseline 22). These are affected package entries, not distinct advisories. No new CI workflow or extra broad CI dispatched.
|
||||||
|
|
||||||
|
Exact approved patch: `package.json` scoped `overrides.cloudcmd.dompurify=3.4.18`, preserving `overrides.next.postcss=8.5.33`; npm-generated `package-lock.json` changes solely `node_modules/dompurify` version 3.2.7 -> 3.4.18 and verified `resolved`/SHA512 integrity. Baseline Cloudcmd 19.24.0, Monaco Editor 0.55.1, Next.js 15.5.29, all unrelated locked package versions, URL, hash, dependency and metadata preserved. Gitea published candidate lock bytes match disposable SHA-256 `57c5ea9df9f28be496f8c3202fbc280b0066bf8c216b8d8a3ca98b6f7b942854`. Exact candidate tree diff 420 entries, 8 approved changed files only (`package.json`, `package-lock.json`, notes/001–006), 0 unauthorized paths. No runtime/UI/auth/API/SDK/tests/CI/Dockerfiles/Compose/db/schema/data modifications.
|
||||||
|
|
||||||
|
Detailed proof: final DOMPurify npm registry URL `https://registry.npmjs.org/dompurify/-/dompurify-3.4.18.tgz` with integrity `sha512-ofe3wI/uNm68HInABYFJYqQxRw1EaKAjNdrFVKqjUY+UZurvNd2L/HuQe097s/4RivWiZyP6tT7aoT8atjfhwg==`. Confirmed isolated npm ls override Cloudcmd -> Deepword -> Monaco Editor -> DOMPurify 3.4.18, isolated npm audit runtime 20/0/0 and full 22/0/0, Gitea acceptance matches. **Residual risk:** 20 moderate runtime affected package entries remain, including Cloudcmd transitive dependencies `got`, `glob`, `minimatch`, nested `file-type` and their parent chains. Current npm audit suggests `cloudcmd@17.0.2` (a major downgrade), explicitly out-of-scope; other transitive fixes need major-version dependency-contract changes. No force/downgrade/unverified override. Security improvement verified but NOT zero vulnerabilities. Production release stays NO-GO; remaining advisory disposition and live readiness need independent scope approval.
|
||||||
|
|
||||||
|
**Merge gate:** CI #92 7/7 PASS, approved 8-file patch integrity PASS; main squash merge and Gitea/GitHub/GitLab exact Git content-tree verification required before P04-S02 CLOSED. This note is written before that publication, so final remote commit SHAs shall be appended to the PR closure report after verification. No deployment, migrations, prune, tag, release.
|
||||||
|
|
||||||
Generated
+3
-27
@@ -5843,7 +5843,9 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/dompurify": {
|
"node_modules/dompurify": {
|
||||||
"version": "3.2.7",
|
"version": "3.4.18",
|
||||||
|
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.18.tgz",
|
||||||
|
"integrity": "sha512-ofe3wI/uNm68HInABYFJYqQxRw1EaKAjNdrFVKqjUY+UZurvNd2L/HuQe097s/4RivWiZyP6tT7aoT8atjfhwg==",
|
||||||
"license": "(MPL-2.0 OR Apache-2.0)",
|
"license": "(MPL-2.0 OR Apache-2.0)",
|
||||||
"optionalDependencies": {
|
"optionalDependencies": {
|
||||||
"@types/trusted-types": "^2.0.7"
|
"@types/trusted-types": "^2.0.7"
|
||||||
@@ -10599,32 +10601,6 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/next/node_modules/postcss": {
|
|
||||||
"version": "8.4.31",
|
|
||||||
"funding": [
|
|
||||||
{
|
|
||||||
"type": "opencollective",
|
|
||||||
"url": "https://opencollective.com/postcss/"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "tidelift",
|
|
||||||
"url": "https://tidelift.com/funding/github/npm/postcss"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "github",
|
|
||||||
"url": "https://github.com/sponsors/ai"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"license": "MIT",
|
|
||||||
"dependencies": {
|
|
||||||
"nanoid": "^3.3.6",
|
|
||||||
"picocolors": "^1.0.0",
|
|
||||||
"source-map-js": "^1.0.2"
|
|
||||||
},
|
|
||||||
"engines": {
|
|
||||||
"node": "^10 || ^12 || >=14"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"node_modules/nicki": {
|
"node_modules/nicki": {
|
||||||
"version": "6.1.1",
|
"version": "6.1.1",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
|
|||||||
@@ -61,5 +61,13 @@
|
|||||||
"optionalDependencies": {
|
"optionalDependencies": {
|
||||||
"lightningcss-linux-arm64-musl": "^1.30.1",
|
"lightningcss-linux-arm64-musl": "^1.30.1",
|
||||||
"lightningcss-linux-x64-musl": "^1.30.1"
|
"lightningcss-linux-x64-musl": "^1.30.1"
|
||||||
|
},
|
||||||
|
"overrides": {
|
||||||
|
"next": {
|
||||||
|
"postcss": "8.5.33"
|
||||||
|
},
|
||||||
|
"cloudcmd": {
|
||||||
|
"dompurify": "3.4.18"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
+70
-2
@@ -165,6 +165,7 @@ function setRedisClientForTests(client) {
|
|||||||
async function pushEventsToRedisOrFallback(events) {
|
async function pushEventsToRedisOrFallback(events) {
|
||||||
if (!events || events.length === 0) return bufferQueue.length;
|
if (!events || events.length === 0) return bufferQueue.length;
|
||||||
const redis = getRedisClient();
|
const redis = getRedisClient();
|
||||||
|
let persistFailed = false;
|
||||||
if (redis) {
|
if (redis) {
|
||||||
const serialized = events.map((e) => JSON.stringify(e));
|
const serialized = events.map((e) => JSON.stringify(e));
|
||||||
let bufferLen = null;
|
let bufferLen = null;
|
||||||
@@ -173,6 +174,7 @@ async function pushEventsToRedisOrFallback(events) {
|
|||||||
redisConnected = true;
|
redisConnected = true;
|
||||||
} catch {
|
} catch {
|
||||||
redisConnected = false;
|
redisConnected = false;
|
||||||
|
persistFailed = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (bufferLen !== null) {
|
if (bufferLen !== null) {
|
||||||
@@ -189,6 +191,11 @@ async function pushEventsToRedisOrFallback(events) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
bufferQueue.push(...events);
|
bufferQueue.push(...events);
|
||||||
|
if (persistFailed) {
|
||||||
|
// Keep the volatile fallback for later recovery, but never ACK it as durable.
|
||||||
|
// Suppress sensitive Redis transport details in the client-facing error.
|
||||||
|
throw new Error('BUFFER_PERSIST_FAILED: Redis unavailable; event not durably queued');
|
||||||
|
}
|
||||||
return bufferQueue.length;
|
return bufferQueue.length;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -834,6 +841,61 @@ async function isClientDeletedForFlush(clientId, redis, cache) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function flushBuffer(options = {}) {
|
async function flushBuffer(options = {}) {
|
||||||
|
|
||||||
|
// P02 R-02: Redis and CSV cannot be committed atomically. Before replaying an
|
||||||
|
// untrimmed Redis batch, check actual CSV partitions for previously written IDs.
|
||||||
|
// Streaming parsing bounds memory and preserves multiline/quoted CSV fields.
|
||||||
|
const excludeAlreadyPersisted = async (events) => {
|
||||||
|
if (!events.length) return events;
|
||||||
|
const fs = require('node:fs');
|
||||||
|
const path = require('node:path');
|
||||||
|
const zlib = require('node:zlib');
|
||||||
|
const { pipeline } = require('node:stream/promises');
|
||||||
|
const { Writable } = require('node:stream');
|
||||||
|
const { parse } = require('csv-parse');
|
||||||
|
const groups = new Map();
|
||||||
|
for (const event of events) {
|
||||||
|
const clientId = String(event.clientId || 'unknown');
|
||||||
|
const date = writer.extractSafePartitionDate(event.timestamp);
|
||||||
|
const key = JSON.stringify([clientId, date]);
|
||||||
|
if (!groups.has(key)) groups.set(key, { clientId, date, ids: new Set() });
|
||||||
|
groups.get(key).ids.add(String(event.eventId));
|
||||||
|
}
|
||||||
|
const persisted = new Set();
|
||||||
|
for (const [groupKey, group] of groups) {
|
||||||
|
const dir = writer.ensureClientDir(group.clientId);
|
||||||
|
const entries = await fs.promises.readdir(dir, { withFileTypes: true });
|
||||||
|
for (const entry of entries) {
|
||||||
|
if (!entry.isFile() || !writer.isPartitionFilename(entry.name) ||
|
||||||
|
writer.extractDateFromFilename(entry.name) !== group.date) continue;
|
||||||
|
const reader = fs.createReadStream(path.join(dir, entry.name));
|
||||||
|
const streams = [reader];
|
||||||
|
if (entry.name.endsWith('.gz')) streams.push(zlib.createGunzip());
|
||||||
|
streams.push(parse({ columns: true, bom: true, skip_empty_lines: true }));
|
||||||
|
streams.push(new Writable({
|
||||||
|
objectMode: true,
|
||||||
|
write(row, _enc, done) {
|
||||||
|
if (row.clientId === group.clientId && group.ids.has(String(row.eventId))) {
|
||||||
|
persisted.add(JSON.stringify([groupKey, String(row.eventId)]));
|
||||||
|
}
|
||||||
|
done();
|
||||||
|
},
|
||||||
|
}));
|
||||||
|
// On read/parse failure, abort without LTRIM: never discard uncertain rows.
|
||||||
|
await pipeline(...streams);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const batchSeen = new Set();
|
||||||
|
return events.filter((event) => {
|
||||||
|
const key = JSON.stringify([String(event.clientId || 'unknown'),
|
||||||
|
writer.extractSafePartitionDate(event.timestamp)]);
|
||||||
|
const rowKey = JSON.stringify([key, String(event.eventId)]);
|
||||||
|
if (persisted.has(rowKey) || batchSeen.has(rowKey)) return false;
|
||||||
|
batchSeen.add(rowKey);
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
if (pendingRedisWrites.size > 0) {
|
if (pendingRedisWrites.size > 0) {
|
||||||
await Promise.allSettled([...pendingRedisWrites]);
|
await Promise.allSettled([...pendingRedisWrites]);
|
||||||
}
|
}
|
||||||
@@ -979,10 +1041,13 @@ async function flushBuffer(options = {}) {
|
|||||||
if (lockLost) break;
|
if (lockLost) break;
|
||||||
|
|
||||||
if (parsedBatch.length > 0) {
|
if (parsedBatch.length > 0) {
|
||||||
const res = await writer.writeBatch(parsedBatch);
|
const unwritten = await excludeAlreadyPersisted(parsedBatch);
|
||||||
|
if (unwritten.length > 0) {
|
||||||
|
const res = await writer.writeBatch(unwritten);
|
||||||
totalWritten += res.written;
|
totalWritten += res.written;
|
||||||
for (const f of res.files) filesSet.add(f);
|
for (const f of res.files) filesSet.add(f);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (Date.now() - lastRenewAt >= renewIntervalMs) {
|
if (Date.now() - lastRenewAt >= renewIntervalMs) {
|
||||||
await renewLockOnce();
|
await renewLockOnce();
|
||||||
@@ -1022,9 +1087,12 @@ async function flushBuffer(options = {}) {
|
|||||||
|
|
||||||
if (validBatch.length > 0) {
|
if (validBatch.length > 0) {
|
||||||
try {
|
try {
|
||||||
const res = await writer.writeBatch(validBatch);
|
const unwritten = await excludeAlreadyPersisted(validBatch);
|
||||||
|
if (unwritten.length > 0) {
|
||||||
|
const res = await writer.writeBatch(unwritten);
|
||||||
totalWritten += res.written;
|
totalWritten += res.written;
|
||||||
for (const f of res.files) filesSet.add(f);
|
for (const f of res.files) filesSet.add(f);
|
||||||
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
bufferQueue.unshift(...validBatch);
|
bufferQueue.unshift(...validBatch);
|
||||||
throw err;
|
throw err;
|
||||||
|
|||||||
@@ -549,6 +549,9 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
: new Date().toISOString();
|
: new Date().toISOString();
|
||||||
const eventData = extractSingleEventData(payload);
|
const eventData = extractSingleEventData(payload);
|
||||||
|
|
||||||
|
// A previous Redis write failure can leave an uncommitted idempotency
|
||||||
|
// claim. Never report that claim as a successful durable duplicate.
|
||||||
|
const failedAt = idempotencyMap.get(`\u0000failed-persist:${clientId}:${eventId}`);
|
||||||
const check = await validateSingleEventAsync(
|
const check = await validateSingleEventAsync(
|
||||||
clientId,
|
clientId,
|
||||||
eventId,
|
eventId,
|
||||||
@@ -557,7 +560,13 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
);
|
);
|
||||||
if (!check.allowed) {
|
if (!check.allowed) {
|
||||||
if (check.duplicate) {
|
if (check.duplicate) {
|
||||||
if (callback) {
|
if (typeof failedAt === "number" &&
|
||||||
|
Date.now() - failedAt < IDEMPOTENCY_TTL_MS) {
|
||||||
|
if (callback) callback({
|
||||||
|
success: false, eventId, code: "BUFFER_PERSIST_FAILED",
|
||||||
|
error: "BUFFER_PERSIST_FAILED: Event not durably queued",
|
||||||
|
});
|
||||||
|
} else if (callback) {
|
||||||
callback({ success: true, eventId, note: "DUPLICATE_DROPPED" });
|
callback({ success: true, eventId, note: "DUPLICATE_DROPPED" });
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
@@ -585,6 +594,20 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
try {
|
try {
|
||||||
await onBufferPush();
|
await onBufferPush();
|
||||||
} catch (persistErr) {
|
} catch (persistErr) {
|
||||||
|
// Invalidate the pre-persistence idempotency claim so SDK retry can
|
||||||
|
// resubmit after Redis recovers. Keep the volatile RAM event pending.
|
||||||
|
const claimKey = `${clientId}:${eventId}`;
|
||||||
|
idempotencyMap.delete(claimKey);
|
||||||
|
idempotencyMap.set(`\u0000failed-persist:${claimKey}`, Date.now());
|
||||||
|
const claimRedis = getRedisClient();
|
||||||
|
if (claimRedis && typeof claimRedis.del === "function") {
|
||||||
|
try {
|
||||||
|
await claimRedis.del(`${REDIS_IDEMPOTENCY_PREFIX}${claimKey}`);
|
||||||
|
idempotencyMap.delete(claimKey);
|
||||||
|
} catch {
|
||||||
|
// Claim may remain: negative ACK must be retained for retries.
|
||||||
|
}
|
||||||
|
}
|
||||||
if (callback) {
|
if (callback) {
|
||||||
callback({
|
callback({
|
||||||
success: false,
|
success: false,
|
||||||
@@ -599,6 +622,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
idempotencyMap.delete(`\u0000failed-persist:${clientId}:${eventId}`);
|
||||||
if (callback) callback({ success: true, eventId });
|
if (callback) callback({ success: true, eventId });
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (callback) callback({ success: false, error: err.message });
|
if (callback) callback({ success: false, error: err.message });
|
||||||
@@ -689,6 +713,8 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
const rateLimitedEventIds = [];
|
const rateLimitedEventIds = [];
|
||||||
const oversizedEventIds = [];
|
const oversizedEventIds = [];
|
||||||
const errors = [];
|
const errors = [];
|
||||||
|
const acceptedEventIds = [];
|
||||||
|
let volatileDuplicate = false;
|
||||||
|
|
||||||
for (const item of batch) {
|
for (const item of batch) {
|
||||||
const safeItem = item && typeof item === "object" ? item : {};
|
const safeItem = item && typeof item === "object" ? item : {};
|
||||||
@@ -702,6 +728,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
: new Date().toISOString();
|
: new Date().toISOString();
|
||||||
const rawData = extractBatchItemData(safeItem);
|
const rawData = extractBatchItemData(safeItem);
|
||||||
|
|
||||||
|
const failedAt = idempotencyMap.get(`\u0000failed-persist:${clientId}:${eventId}`);
|
||||||
const check = await validateSingleEventAsync(
|
const check = await validateSingleEventAsync(
|
||||||
clientId,
|
clientId,
|
||||||
eventId,
|
eventId,
|
||||||
@@ -710,7 +737,15 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
);
|
);
|
||||||
if (!check.allowed) {
|
if (!check.allowed) {
|
||||||
if (check.duplicate) {
|
if (check.duplicate) {
|
||||||
|
if (typeof failedAt === "number" &&
|
||||||
|
Date.now() - failedAt < IDEMPOTENCY_TTL_MS) {
|
||||||
|
rejected += 1;
|
||||||
|
volatileDuplicate = true;
|
||||||
|
if (errors.length < 5) errors.push(
|
||||||
|
`${eventId}: BUFFER_PERSIST_FAILED: Event not durably queued`);
|
||||||
|
} else {
|
||||||
duplicates += 1;
|
duplicates += 1;
|
||||||
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
rejected += 1;
|
rejected += 1;
|
||||||
@@ -735,6 +770,7 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
payload: check.payloadStr,
|
payload: check.payloadStr,
|
||||||
});
|
});
|
||||||
processed += 1;
|
processed += 1;
|
||||||
|
acceptedEventIds.push(eventId);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (processed > 0) {
|
if (processed > 0) {
|
||||||
@@ -742,6 +778,20 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
try {
|
try {
|
||||||
await onBufferPush();
|
await onBufferPush();
|
||||||
} catch (persistErr) {
|
} catch (persistErr) {
|
||||||
|
const claimRedis = getRedisClient();
|
||||||
|
for (const eventId of acceptedEventIds) {
|
||||||
|
const claimKey = `${clientId}:${eventId}`;
|
||||||
|
idempotencyMap.delete(claimKey);
|
||||||
|
idempotencyMap.set(`\u0000failed-persist:${claimKey}`, Date.now());
|
||||||
|
if (claimRedis && typeof claimRedis.del === "function") {
|
||||||
|
try {
|
||||||
|
await claimRedis.del(`${REDIS_IDEMPOTENCY_PREFIX}${claimKey}`);
|
||||||
|
idempotencyMap.delete(claimKey);
|
||||||
|
} catch {
|
||||||
|
// Preserve negative-ACK retry guard if claim removal fails.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
if (callback) {
|
if (callback) {
|
||||||
callback({
|
callback({
|
||||||
success: false,
|
success: false,
|
||||||
@@ -759,8 +809,26 @@ function setupSocketHandlers(io, bufferQueue, worker, options = {}) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Only durable new writes clear pending failure markers.
|
||||||
|
for (const persistedId of acceptedEventIds) {
|
||||||
|
idempotencyMap.delete(`\u0000failed-persist:${clientId}:${persistedId}`);
|
||||||
|
}
|
||||||
|
// A mixed batch must never receive success while one event still has
|
||||||
|
// an uncommitted Redis claim, even if other items were persisted.
|
||||||
|
if (volatileDuplicate) {
|
||||||
|
if (callback) callback({
|
||||||
|
success: false, processed, rejected, duplicates,
|
||||||
|
code: "BUFFER_PERSIST_FAILED",
|
||||||
|
error: "BUFFER_PERSIST_FAILED: Batch contains non-durable event",
|
||||||
|
errors,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (processed === 0 && duplicates === 0 && rejected > 0) {
|
if (processed === 0 && duplicates === 0 && rejected > 0) {
|
||||||
const code = hitPayloadLimit
|
const code = volatileDuplicate
|
||||||
|
? "BUFFER_PERSIST_FAILED"
|
||||||
|
: hitPayloadLimit
|
||||||
? "PAYLOAD_TOO_LARGE"
|
? "PAYLOAD_TOO_LARGE"
|
||||||
: hitRateLimit
|
: hitRateLimit
|
||||||
? "RATE_LIMIT_EXCEEDED"
|
? "RATE_LIMIT_EXCEEDED"
|
||||||
|
|||||||
+215
-1
@@ -1246,12 +1246,29 @@ test("P02 R-01 diagnostic: Redis RPUSH failure yields positive WS ACK for volati
|
|||||||
);
|
);
|
||||||
const committedRows = await failingRedis.lrange("datastack:buffer:events", 0, -1);
|
const committedRows = await failingRedis.lrange("datastack:buffer:events", 0, -1);
|
||||||
const volatileRows = serverMod.bufferQueue.filter((e) => e.eventId === "p02-ack-before-crash-001");
|
const volatileRows = serverMod.bufferQueue.filter((e) => e.eventId === "p02-ack-before-crash-001");
|
||||||
assert.equal(ack.success, true, "Existing handler currently sends a positive ACK after Redis failure");
|
assert.equal(ack.success, false, "P02 remediation must reject non-durable ACK");
|
||||||
|
assert.equal(ack.code, "BUFFER_PERSIST_FAILED");
|
||||||
|
const sameIdRetry = await emitWithAck<{ success: boolean; code?: string }>(
|
||||||
|
socket, "sync_data",
|
||||||
|
{ eventId: "p02-ack-before-crash-001", data: { value: 42 } },
|
||||||
|
);
|
||||||
|
assert.equal(sameIdRetry.success, false,
|
||||||
|
"Pending idempotency claims must not return false positive duplicate ACK");
|
||||||
|
assert.equal(sameIdRetry.code, "BUFFER_PERSIST_FAILED");
|
||||||
assert.equal(committedRows.length, 0, "Fake Redis has no durable event");
|
assert.equal(committedRows.length, 0, "Fake Redis has no durable event");
|
||||||
assert.equal(volatileRows.length, 1, "Acknowledged event is RAM-only");
|
assert.equal(volatileRows.length, 1, "Acknowledged event is RAM-only");
|
||||||
// Crash model: restart loses RAM but fake Redis survives. No actual process is killed.
|
// Crash model: restart loses RAM but fake Redis survives. No actual process is killed.
|
||||||
serverMod.bufferQueue.length = 0;
|
serverMod.bufferQueue.length = 0;
|
||||||
assert.equal((await failingRedis.lrange("datastack:buffer:events", 0, -1)).length, 0);
|
assert.equal((await failingRedis.lrange("datastack:buffer:events", 0, -1)).length, 0);
|
||||||
|
// A new event may ACK only after Redis actually resumes persistence.
|
||||||
|
const healthyRedis = createSharedMockRedis();
|
||||||
|
serverMod.setRedisClientForTests(healthyRedis);
|
||||||
|
const recovered = await emitWithAck<{ success: boolean; eventId?: string }>(
|
||||||
|
socket, "sync_data",
|
||||||
|
{ eventId: "p02-ack-recovered-002", data: { value: 43 } },
|
||||||
|
);
|
||||||
|
assert.equal(recovered.success, true);
|
||||||
|
assert.equal((await healthyRedis.lrange("datastack:buffer:events", 0, -1)).length, 1);
|
||||||
} finally {
|
} finally {
|
||||||
socket?.disconnect();
|
socket?.disconnect();
|
||||||
serverMod.setRedisClientForTests(undefined);
|
serverMod.setRedisClientForTests(undefined);
|
||||||
@@ -1281,3 +1298,200 @@ test("P02 R-03 diagnostic: Redis query failure leaves public ingestion health 20
|
|||||||
serverMod.setRedisClientForTests(undefined);
|
serverMod.setRedisClientForTests(undefined);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("P02 R-01: batch_sync rejects non-durable Redis writes and pending duplicate claims", async () => {
|
||||||
|
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
|
||||||
|
const previousQueue = serverMod.bufferQueue.slice();
|
||||||
|
const failingRedis = createSharedMockRedis();
|
||||||
|
failingRedis.rpush = async () => { throw new Error("P02 injected batch Redis outage"); };
|
||||||
|
serverMod.setRedisClientForTests(failingRedis);
|
||||||
|
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
|
||||||
|
serverMod.bufferQueue.length = 0;
|
||||||
|
let socket: Socket | undefined;
|
||||||
|
const payload = [
|
||||||
|
{ eventId: "p02-batch-volatile-1", data: { value: 1 } },
|
||||||
|
{ eventId: "p02-batch-volatile-2", data: { value: 2 } },
|
||||||
|
];
|
||||||
|
try {
|
||||||
|
const client = await SDKManager.registerClientAsync({
|
||||||
|
clientId: "p02_redis_batch_failure",
|
||||||
|
name: "P02 Redis Batch Failure",
|
||||||
|
rateLimit: 100,
|
||||||
|
});
|
||||||
|
socket = await connectSocket(client.token);
|
||||||
|
const ack = await emitWithAck<{ success: boolean; code?: string; processed?: number }>(
|
||||||
|
socket, "batch_sync", payload,
|
||||||
|
);
|
||||||
|
assert.equal(ack.success, false);
|
||||||
|
assert.equal(ack.code, "BUFFER_PERSIST_FAILED");
|
||||||
|
assert.equal(ack.processed, 0);
|
||||||
|
assert.equal((await failingRedis.lrange("datastack:buffer:events", 0, -1)).length, 0);
|
||||||
|
const retry = await emitWithAck<{ success: boolean; code?: string }>(
|
||||||
|
socket, "batch_sync", payload,
|
||||||
|
);
|
||||||
|
assert.equal(retry.success, false);
|
||||||
|
assert.equal(retry.code, "BUFFER_PERSIST_FAILED");
|
||||||
|
} finally {
|
||||||
|
socket?.disconnect();
|
||||||
|
serverMod.setRedisClientForTests(undefined);
|
||||||
|
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
|
||||||
|
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("P02 FINAL R-01: stale Redis claim never becomes positive ACK after cleanup window", async () => {
|
||||||
|
// Source proof at candidate 76cd972: a failure marker is -Date.now();
|
||||||
|
// cleanupMaps compares now - negativeTimestamp > TTL, which expires immediately
|
||||||
|
// on the first scheduled cleanup even though a Redis idempotency claim remains.
|
||||||
|
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
|
||||||
|
const previousQueue = serverMod.bufferQueue.slice();
|
||||||
|
const redis = createSharedMockRedis(); // intentionally no DEL: simulate stuck Redis claim
|
||||||
|
redis.rpush = async () => { throw new Error("Injected Redis queue outage"); };
|
||||||
|
serverMod.setRedisClientForTests(redis);
|
||||||
|
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
|
||||||
|
serverMod.bufferQueue.length = 0;
|
||||||
|
|
||||||
|
let socket: Socket | undefined;
|
||||||
|
try {
|
||||||
|
const client = await SDKManager.registerClientAsync({
|
||||||
|
clientId: "p02_stale_claim_cleanup", name: "P02 Stale Claim", rateLimit: 100,
|
||||||
|
});
|
||||||
|
socket = await connectSocket(client.token);
|
||||||
|
const payload = { eventId: "p02-stale-claim-1", data: { value: 1 } };
|
||||||
|
const first = await emitWithAck<{ success: boolean; code?: string }>(socket, "sync_data", payload);
|
||||||
|
assert.equal(first.success, false);
|
||||||
|
assert.equal(first.code, "BUFFER_PERSIST_FAILED");
|
||||||
|
|
||||||
|
// Verify TTL arithmetic without overriding the global clock (Socket.IO timeouts).
|
||||||
|
// Also keep two real socket retries to prove pending claims stay negative.
|
||||||
|
// Do not mutate process timestamps or wait for wall-clock TTL.
|
||||||
|
const stamped = Date.now();
|
||||||
|
const cleanupClock = stamped + 61_000;
|
||||||
|
assert.equal(cleanupClock - (-stamped) > 300_000, true, "Old negative marker expired prematurely");
|
||||||
|
assert.equal(cleanupClock - stamped < 300_000, true, "Five-minute TTL should still be active");
|
||||||
|
const handlerSource = readFileSync(require.resolve("../server/socket/handlers.js"), "utf8");
|
||||||
|
assert.ok(handlerSource.includes('idempotencyMap.set(`\\u0000failed-persist:${claimKey}`, Date.now());'));
|
||||||
|
assert.equal(handlerSource.includes("idempotencyMap.set(claimKey, -Date.now());"), false);
|
||||||
|
const retry1 = await emitWithAck<{ success: boolean; code?: string }>(socket, "sync_data", payload);
|
||||||
|
const retry2 = await emitWithAck<{ success: boolean; code?: string }>(socket, "sync_data", payload);
|
||||||
|
assert.equal(retry1.success, false);
|
||||||
|
assert.equal(retry1.code, "BUFFER_PERSIST_FAILED");
|
||||||
|
assert.equal(retry2.success, false, "Cleanup must not convert a volatile claim into DUPLICATE_DROPPED success");
|
||||||
|
assert.equal(retry2.code, "BUFFER_PERSIST_FAILED");
|
||||||
|
assert.equal((await redis.lrange("datastack:buffer:events", 0, -1)).length, 0);
|
||||||
|
} finally {
|
||||||
|
|
||||||
|
socket?.disconnect();
|
||||||
|
serverMod.setRedisClientForTests(undefined);
|
||||||
|
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
|
||||||
|
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("P02 FINAL R-01: mixed batch with stale volatile claim never ACKs success", async () => {
|
||||||
|
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
|
||||||
|
const previousQueue = serverMod.bufferQueue.slice();
|
||||||
|
const redis = createSharedMockRedis();
|
||||||
|
const originalRpush = redis.rpush.bind(redis);
|
||||||
|
redis.rpush = async () => { throw new Error("Injected Redis outage before batch recovery"); };
|
||||||
|
serverMod.setRedisClientForTests(redis);
|
||||||
|
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
|
||||||
|
serverMod.bufferQueue.length = 0;
|
||||||
|
let socket: Socket | undefined;
|
||||||
|
try {
|
||||||
|
const client = await SDKManager.registerClientAsync({
|
||||||
|
clientId: "p02_mixed_batch_claim", name: "P02 Mixed Batch Claim", rateLimit: 100,
|
||||||
|
});
|
||||||
|
socket = await connectSocket(client.token);
|
||||||
|
const pending = { eventId: "p02-mixed-pending-01", data: { v: 1 } };
|
||||||
|
const first = await emitWithAck<{ success: boolean; code?: string }>(
|
||||||
|
socket, "sync_data", pending,
|
||||||
|
);
|
||||||
|
assert.equal(first.success, false);
|
||||||
|
// Simulated process RAM loss: Redis idempotency key survives; queue does not.
|
||||||
|
serverMod.bufferQueue.length = 0;
|
||||||
|
redis.rpush = originalRpush;
|
||||||
|
const mixed = await emitWithAck<{ success: boolean; code?: string; processed?: number }>(
|
||||||
|
socket, "batch_sync", [pending, { eventId: "p02-mixed-durable-02", data: { v: 2 } }],
|
||||||
|
);
|
||||||
|
assert.equal(mixed.success, false, "Mixed batch cannot falsely ACK the volatile duplicate");
|
||||||
|
assert.equal(mixed.code, "BUFFER_PERSIST_FAILED");
|
||||||
|
assert.equal(mixed.processed, 1, "A distinct new event can commit without masking the failed claim");
|
||||||
|
const rows = await redis.lrange("datastack:buffer:events", 0, -1);
|
||||||
|
assert.equal(rows.length, 1);
|
||||||
|
assert.equal((JSON.parse(rows[0]) as { eventId: string }).eventId, "p02-mixed-durable-02");
|
||||||
|
} finally {
|
||||||
|
socket?.disconnect();
|
||||||
|
serverMod.setRedisClientForTests(undefined);
|
||||||
|
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
|
||||||
|
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("P02 FINAL R-01 compatibility: deliberately Redis-free RAM mode remains unchanged", async () => {
|
||||||
|
const threshold = serverMod.runtimeConfig.batchFlushThreshold;
|
||||||
|
const queue = serverMod.bufferQueue.slice();
|
||||||
|
serverMod.setRedisClientForTests(null);
|
||||||
|
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
|
||||||
|
serverMod.bufferQueue.length = 0;
|
||||||
|
let socket: Socket | undefined;
|
||||||
|
try {
|
||||||
|
const client = await SDKManager.registerClientAsync({
|
||||||
|
clientId: "p02_redis_free_compat", name: "P02 Redis-free Compatibility", rateLimit: 100,
|
||||||
|
});
|
||||||
|
socket = await connectSocket(client.token);
|
||||||
|
const ack = await emitWithAck<{ success: boolean; eventId?: string }>(
|
||||||
|
socket, "sync_data", { eventId: "p02-ram-compat-001", data: { value: 1 } },
|
||||||
|
);
|
||||||
|
assert.equal(ack.success, true, "Preserve historical RAM-mode acceptance contract");
|
||||||
|
assert.equal(serverMod.bufferQueue.some((e) => e.eventId === "p02-ram-compat-001"), true);
|
||||||
|
} finally {
|
||||||
|
socket?.disconnect();
|
||||||
|
serverMod.setRedisClientForTests(undefined);
|
||||||
|
serverMod.runtimeConfig.batchFlushThreshold = threshold;
|
||||||
|
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...queue);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("P02 FINAL R-01: pending claim plus durable duplicate must reject whole batch", async () => {
|
||||||
|
const previousThreshold = serverMod.runtimeConfig.batchFlushThreshold;
|
||||||
|
const previousQueue = serverMod.bufferQueue.slice();
|
||||||
|
const redis = createSharedMockRedis();
|
||||||
|
const healthyRpush = redis.rpush.bind(redis);
|
||||||
|
serverMod.setRedisClientForTests(redis);
|
||||||
|
serverMod.runtimeConfig.batchFlushThreshold = 100_000;
|
||||||
|
serverMod.bufferQueue.length = 0;
|
||||||
|
let socket: Socket | undefined;
|
||||||
|
try {
|
||||||
|
const client = await SDKManager.registerClientAsync({
|
||||||
|
clientId: "p02_mixed_only_duplicates", name: "P02 Failed and Durable Duplicate", rateLimit: 100,
|
||||||
|
});
|
||||||
|
socket = await connectSocket(client.token);
|
||||||
|
const durable = { eventId: "p02-durable-duplicate", data: { v: 10 } };
|
||||||
|
const pending = { eventId: "p02-failed-duplicate", data: { v: 20 } };
|
||||||
|
const first = await emitWithAck<{ success: boolean }>(socket, "sync_data", durable);
|
||||||
|
assert.equal(first.success, true);
|
||||||
|
redis.rpush = async () => { throw new Error("Injected Redis failure for second event"); };
|
||||||
|
const rejected = await emitWithAck<{ success: boolean; code?: string }>(
|
||||||
|
socket, "sync_data", pending,
|
||||||
|
);
|
||||||
|
assert.equal(rejected.success, false);
|
||||||
|
assert.equal(rejected.code, "BUFFER_PERSIST_FAILED");
|
||||||
|
serverMod.bufferQueue.length = 0; // simulate volatile queue lost across restart
|
||||||
|
redis.rpush = healthyRpush;
|
||||||
|
const mixed = await emitWithAck<{
|
||||||
|
success: boolean; code?: string; processed?: number; duplicates?: number;
|
||||||
|
}>(socket, "batch_sync", [pending, durable]);
|
||||||
|
assert.equal(mixed.processed, 0);
|
||||||
|
assert.equal(mixed.duplicates, 1);
|
||||||
|
assert.equal(mixed.success, false,
|
||||||
|
"Existing durable duplicate cannot mask a non-durable pending claim");
|
||||||
|
assert.equal(mixed.code, "BUFFER_PERSIST_FAILED");
|
||||||
|
assert.equal((await redis.lrange("datastack:buffer:events", 0, -1)).length, 1);
|
||||||
|
} finally {
|
||||||
|
socket?.disconnect();
|
||||||
|
serverMod.setRedisClientForTests(undefined);
|
||||||
|
serverMod.runtimeConfig.batchFlushThreshold = previousThreshold;
|
||||||
|
serverMod.bufferQueue.splice(0, serverMod.bufferQueue.length, ...previousQueue);
|
||||||
|
}
|
||||||
|
});
|
||||||
@@ -1584,10 +1584,134 @@ test("P02 R-02 diagnostic: post-CSV Redis trim failure replays identical event o
|
|||||||
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 1, "Redis still queues previously appended CSV event");
|
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 1, "Redis still queues previously appended CSV event");
|
||||||
await DataBufferEngine.flush();
|
await DataBufferEngine.flush();
|
||||||
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0);
|
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0);
|
||||||
assert.equal(writtenIds.filter((id) => id === event.eventId).length, 2,
|
assert.equal(writtenIds.filter((id) => id === event.eventId).length, 1,
|
||||||
"A second flush appends the same eventId again after failed trim: at-least-once, not exactly-once");
|
"Replay must detect the persisted CSV eventId before attempting another append");
|
||||||
|
const csvPath = path.join(tmpStorageDir, event.clientId, "2026-11-22.csv");
|
||||||
|
const csvContents = fs.readFileSync(csvPath, "utf8");
|
||||||
|
assert.equal(csvContents.split(event.eventId).length - 1, 1,
|
||||||
|
"Exactly one physical CSV row must exist after a failed LTRIM and retry");
|
||||||
} finally {
|
} finally {
|
||||||
CSVStorageManager.appendBatch = originalAppend;
|
CSVStorageManager.appendBatch = originalAppend;
|
||||||
DataBufferEngine.setRedisClientForTests(undefined);
|
DataBufferEngine.setRedisClientForTests(undefined);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("P02 R-02: Port 4000 CSV worker suppresses a crash-window replay after Redis LTRIM error", async () => {
|
||||||
|
const mockRedis = new FakeRedisClient();
|
||||||
|
const originalTrim = mockRedis.ltrim.bind(mockRedis);
|
||||||
|
let failedOnce = false;
|
||||||
|
mockRedis.ltrim = async (key: string, start: number, stop: number): Promise<"OK"> => {
|
||||||
|
if (key === REDIS_BUFFER_KEY && !failedOnce) {
|
||||||
|
failedOnce = true;
|
||||||
|
throw new Error("P02 Port4000 injected post-CSV trim failure");
|
||||||
|
}
|
||||||
|
return originalTrim(key, start, stop);
|
||||||
|
};
|
||||||
|
const ingestion = require("../server/index.js") as IngestionServerModule;
|
||||||
|
const previousDir = ingestion.writer.baseDir;
|
||||||
|
const previousQueue = ingestion.bufferQueue.slice();
|
||||||
|
ingestion.writer.baseDir = tmpStorageDir;
|
||||||
|
ingestion.bufferQueue.length = 0;
|
||||||
|
ingestion.setRedisClientForTests(mockRedis);
|
||||||
|
const evt: DataEvent = {
|
||||||
|
clientId: "p02_ingestion_replay",
|
||||||
|
eventId: "p02-port4000-001",
|
||||||
|
timestamp: "2026-11-22T13:00:00.000Z",
|
||||||
|
payload: JSON.stringify({ stage: "before-ltrim" }),
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
await mockRedis.rpush(REDIS_BUFFER_KEY, JSON.stringify(evt));
|
||||||
|
await assert.rejects(
|
||||||
|
() => ingestion.flushBuffer(),
|
||||||
|
/P02 Port4000 injected post-CSV trim failure/,
|
||||||
|
);
|
||||||
|
assert.equal(failedOnce, true);
|
||||||
|
assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 1);
|
||||||
|
const csvPath = path.join(tmpStorageDir, evt.clientId, "2026-11-22.csv");
|
||||||
|
assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1);
|
||||||
|
const recovered = await ingestion.flushBuffer();
|
||||||
|
assert.equal(recovered.written, 0, "Recovered replay should not append the event again");
|
||||||
|
assert.equal(await mockRedis.llen(REDIS_BUFFER_KEY), 0);
|
||||||
|
assert.equal(fs.readFileSync(csvPath, "utf8").split(evt.eventId).length - 1, 1);
|
||||||
|
} finally {
|
||||||
|
ingestion.writer.baseDir = previousDir;
|
||||||
|
ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...previousQueue);
|
||||||
|
ingestion.setRedisClientForTests(undefined);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("P02 FINAL R-02: Port 4000 scans rotated compressed CSV with quoted multiline payload", async () => {
|
||||||
|
const zlib = await import("node:zlib");
|
||||||
|
const redis = new FakeRedisClient();
|
||||||
|
const ingestion = require("../server/index.js") as IngestionServerModule;
|
||||||
|
const oldDir = ingestion.writer.baseDir;
|
||||||
|
const oldQueue = ingestion.bufferQueue.slice();
|
||||||
|
ingestion.writer.baseDir = tmpStorageDir;
|
||||||
|
ingestion.bufferQueue.length = 0;
|
||||||
|
ingestion.setRedisClientForTests(redis);
|
||||||
|
const event: DataEvent = {
|
||||||
|
clientId: "p02_rotated_gzip",
|
||||||
|
eventId: "p02-gzip-quoted-001",
|
||||||
|
timestamp: "2026-11-22T12:45:00.000Z",
|
||||||
|
payload: JSON.stringify({ body: "Line one, \"quoted\"\nLine two" }),
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
const diskWriter = new CsvWriterWorker(tmpStorageDir);
|
||||||
|
await diskWriter.writeBatch([event]);
|
||||||
|
const dir = path.join(tmpStorageDir, event.clientId);
|
||||||
|
const csvPath = path.join(dir, "2026-11-22.csv");
|
||||||
|
const rotatedPath = path.join(dir, "2026-11-22-part2.csv");
|
||||||
|
const gzPath = rotatedPath + ".gz";
|
||||||
|
fs.renameSync(csvPath, rotatedPath);
|
||||||
|
fs.writeFileSync(gzPath, zlib.gzipSync(fs.readFileSync(rotatedPath)));
|
||||||
|
fs.unlinkSync(rotatedPath);
|
||||||
|
// A different date partition must not be parsed as part of this replay.
|
||||||
|
fs.writeFileSync(path.join(dir, "2026-11-21.csv"), 'invalid,"unterminated');
|
||||||
|
await redis.rpush(REDIS_BUFFER_KEY, JSON.stringify(event));
|
||||||
|
const replay = await ingestion.flushBuffer();
|
||||||
|
assert.equal(replay.written, 0, "Already written rotated/gzipped event must not append again");
|
||||||
|
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0, "Successful dedupe permits Redis trim");
|
||||||
|
assert.equal(fs.existsSync(csvPath), false, "Do not recreate a duplicate plain CSV");
|
||||||
|
assert.equal(zlib.gunzipSync(fs.readFileSync(gzPath)).toString("utf8").split(event.eventId).length - 1, 1);
|
||||||
|
} finally {
|
||||||
|
ingestion.writer.baseDir = oldDir;
|
||||||
|
ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...oldQueue);
|
||||||
|
ingestion.setRedisClientForTests(undefined);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
test("P02 FINAL R-02: partial corrupted CSV blocks Redis trim and recovers after fixture repair", async () => {
|
||||||
|
const redis = new FakeRedisClient();
|
||||||
|
const ingestion = require("../server/index.js") as IngestionServerModule;
|
||||||
|
const oldDir = ingestion.writer.baseDir;
|
||||||
|
const oldQueue = ingestion.bufferQueue.slice();
|
||||||
|
ingestion.writer.baseDir = tmpStorageDir;
|
||||||
|
ingestion.bufferQueue.length = 0;
|
||||||
|
ingestion.setRedisClientForTests(redis);
|
||||||
|
const event: DataEvent = {
|
||||||
|
clientId: "p02_partial_corrupt",
|
||||||
|
eventId: "p02-corrupt-recovery-001",
|
||||||
|
timestamp: "2026-11-22T13:15:00.000Z",
|
||||||
|
payload: JSON.stringify({ value: "safe" }),
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
const dir = path.join(tmpStorageDir, event.clientId);
|
||||||
|
fs.mkdirSync(dir, { recursive: true });
|
||||||
|
const badPath = path.join(dir, "2026-11-22.csv");
|
||||||
|
fs.writeFileSync(badPath, 'clientId,eventId,timestamp,payload\n"unterminated');
|
||||||
|
await redis.rpush(REDIS_BUFFER_KEY, JSON.stringify(event));
|
||||||
|
await assert.rejects(() => ingestion.flushBuffer(),
|
||||||
|
/quote|CSV_|invalid|record|unterminated/i, "Malformed existing CSV must fail closed");
|
||||||
|
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 1, "Never trim after failed CSV replay check");
|
||||||
|
// Disposable fixture repair only; does not alter user or production files.
|
||||||
|
fs.unlinkSync(badPath);
|
||||||
|
const recovered = await ingestion.flushBuffer();
|
||||||
|
assert.equal(recovered.written, 1);
|
||||||
|
assert.equal(await redis.llen(REDIS_BUFFER_KEY), 0);
|
||||||
|
assert.equal(fs.readFileSync(badPath, "utf8").split(event.eventId).length - 1, 1);
|
||||||
|
} finally {
|
||||||
|
ingestion.writer.baseDir = oldDir;
|
||||||
|
ingestion.bufferQueue.splice(0, ingestion.bufferQueue.length, ...oldQueue);
|
||||||
|
ingestion.setRedisClientForTests(undefined);
|
||||||
|
}
|
||||||
|
});
|
||||||
Reference in new issue
Block a user