ci(p03): isolate Python 3.12 and BuildKit; inventory npm advisories

This commit is contained in:
victorsteele428 committed 2026-11-22 12:01:15 +00:00
1 parent 48fb277079
commit 59346dc2d5
2 files changed
+99 -37

No files matched your search

+80 -20
View File
@@ -52,23 +52,28 @@ jobs:
python-regression:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.p01_only != 'true' }}
runs-on: ubuntu-latest
timeout-minutes: 15
container: python:3.12-slim-bookworm
timeout-minutes: 20
steps:
- uses: actions/checkout@v4
with:
github-server-url: https://dev.ivib.cc
- name: Install Python toolchain
- name: CI-06 Python 3.12 isolated virtual environment
run: |
set -eu
apt-get update -qq
apt-get install -y python3 python3-venv python3-pip
python3 -m venv /tmp/datastack-ci-venv
/tmp/datastack-ci-venv/bin/python -m pip install -e ./sdk/python/datastack -e ./sdk-python
- name: Run all Python SDK tests
python --version
python -m venv /tmp/datastack-ci-venv
/tmp/datastack-ci-venv/bin/python -m pip install --disable-pip-version-check -e ./sdk/python/datastack -e ./sdk-python
- name: CI-06 compile and assert 14 Python SDK regressions
run: |
set -eu
/tmp/datastack-ci-venv/bin/python -m compileall -q ./sdk/python/datastack ./sdk-python
/tmp/datastack-ci-venv/bin/python -m unittest discover -s tests -p '*_test.py' -v
py=/tmp/datastack-ci-venv/bin/python
"$py" -m compileall -q ./sdk/python/datastack ./sdk-python
"$py" -m unittest discover -s tests -p '*_test.py' -v > /tmp/datastack-python-tests.log 2>&1 || { cat /tmp/datastack-python-tests.log; exit 1; }
cat /tmp/datastack-python-tests.log
grep -Eq '^Ran 14 tests in ' /tmp/datastack-python-tests.log
grep -Eq '^OK$' /tmp/datastack-python-tests.log
echo DATASTACK_PYTHON_14_OF_14_PASS
quality:
runs-on: ubuntu-latest
@@ -103,25 +108,40 @@ jobs:
docker-image-build:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.p01_only != 'true' }}
runs-on: ubuntu-latest
timeout-minutes: 35
container: node:22-alpine
timeout-minutes: 45
steps:
- name: CI-07 provision Docker Buildx CLI
run: |
set -eu
apk add --no-cache git docker-cli docker-cli-buildx docker-cli-compose
docker --version
docker buildx version
docker compose version
- uses: actions/checkout@v4
with:
github-server-url: https://dev.ivib.cc
- name: Docker and Compose availability
- name: CI-07 inspect daemon and validate Compose without starting services
run: |
set -eu
docker --version
docker compose version
docker info
- name: Validate Compose structure
run: docker compose -f docker/docker-compose.yml config --no-interpolate --quiet
- name: Build Docker images without running services
docker info --format 'Driver={{.Driver}} Root={{.DockerRootDir}}'
df -h .
docker compose -f docker/docker-compose.yml config --no-interpolate --quiet
echo DATASTACK_COMPOSE_CONFIG_PASS
- name: CI-07 isolated BuildKit all three images (no deployment)
run: |
set -eu
docker build -f docker/Dockerfile.server -t datastack-ci-ingestion .
docker build -f docker/Dockerfile.web -t datastack-ci-web .
docker build -f docker/Dockerfile.proxy -t datastack-ci-proxy .
builder="datastack-p03-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}"
tag="datastack-p03-${GITHUB_SHA:-local}"
cleanup() { docker buildx rm --force "$builder" >/dev/null 2>&1 || true; }
trap cleanup EXIT
docker buildx create --name "$builder" --driver docker-container --use
docker buildx inspect "$builder" --bootstrap
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.server -t "$tag-ingestion" .
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.web -t "$tag-web" .
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.proxy -t "$tag-proxy" .
docker image inspect "$tag-ingestion" "$tag-web" "$tag-proxy" >/dev/null
echo DATASTACK_THREE_IMAGES_BUILD_PASS
windows-compatibility:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.quality_only != 'true' }}
@@ -150,3 +170,43 @@ jobs:
if ($LASTEXITCODE -ne 0) { throw 'SDK install failed' }
npm --prefix sdk/node/datastack run build
if ($LASTEXITCODE -ne 0) { throw 'SDK compilation failed' }
security-audit:
if: ${{ github.event_name != 'workflow_dispatch' || github.event.inputs.p01_only != 'true' }}
runs-on: ubuntu-latest
container: node:22-alpine
timeout-minutes: 15
steps:
- name: Install checkout prerequisite
run: apk add --no-cache git
- uses: actions/checkout@v4
with:
github-server-url: https://dev.ivib.cc
- name: CI-08 read-only full and runtime npm advisory inventory
run: |
set -eu
if npm audit --json > /tmp/datastack-audit-full.json; then :; else test "$?" -eq 1; fi
if npm audit --omit=dev --json > /tmp/datastack-audit-runtime.json; then :; else test "$?" -eq 1; fi
node <<'NODE'
const fs = require("node:fs");
for (const scope of ["full", "runtime"]) {
const audit = JSON.parse(fs.readFileSync("/tmp/datastack-audit-" + scope + ".json", "utf8"));
if (!audit.metadata || !audit.metadata.vulnerabilities || !audit.vulnerabilities) {
throw Error("Incomplete npm audit response for " + scope);
}
console.log("AUDIT_COUNTS " + scope + " " + JSON.stringify(audit.metadata.vulnerabilities));
for (const [name, item] of Object.entries(audit.vulnerabilities)) {
const advisories = (item.via || []).filter(v => typeof v === "object").map(v => ({
id: v.source, url: v.url, range: v.range, severity: v.severity
}));
console.log("AUDIT_PACKAGE " + scope + " " + JSON.stringify({
package: name, severity: item.severity, direct: item.isDirect,
range: item.range, via: advisories,
viaPackages: (item.via || []).filter(v => typeof v === "string"),
fixAvailable: item.fixAvailable
}));
}
}
NODE
git diff --exit-code -- package.json package-lock.json
echo DATASTACK_SECURITY_TRIAGE_INVENTORY_PASS
+19 -17
View File
@@ -1,39 +1,41 @@
name: DataStack Docker Build CI
on:
push:
branches: [main]
paths:
- '.gitea/workflows/datastack-docker-ci.yml'
workflow_dispatch:
jobs:
docker-build:
runs-on: ubuntu-latest
container: node:22-alpine
timeout-minutes: 40
timeout-minutes: 45
steps:
- name: Provision isolated Docker CLI and Git tools
- name: CI-07 provision Docker Buildx CLI
run: |
set -eu
apk add --no-cache git docker-cli docker-cli-compose
apk add --no-cache git docker-cli docker-cli-buildx docker-cli-compose
docker --version
docker buildx version
docker compose version
- uses: actions/checkout@v4
with:
github-server-url: https://dev.ivib.cc
- name: Validate actual Compose configuration
- name: CI-07 inspect daemon and validate Compose without starting services
run: |
set -eu
docker info --format 'Driver={{.Driver}} Root={{.DockerRootDir}}'
df -h .
docker compose -f docker/docker-compose.yml config --no-interpolate --quiet
echo DATASTACK_COMPOSE_CONFIG_PASS
- name: Check CI-only Docker daemon
- name: CI-07 isolated BuildKit all three images (no deployment)
run: |
set -eu
docker info >/dev/null || { echo DOCKER_DAEMON_UNAVAILABLE; exit 3; }
- name: Build all DataStack container images without deployment
run: |
set -eu
docker build -f docker/Dockerfile.server -t datastack-ci-ingestion .
docker build -f docker/Dockerfile.web -t datastack-ci-web .
docker build -f docker/Dockerfile.proxy -t datastack-ci-proxy .
echo DATASTACK_DOCKER_IMAGES_PASS
builder="datastack-p03-${GITHUB_RUN_ID:-0}-${GITHUB_RUN_ATTEMPT:-1}"
tag="datastack-p03-${GITHUB_SHA:-local}"
cleanup() { docker buildx rm --force "$builder" >/dev/null 2>&1 || true; }
trap cleanup EXIT
docker buildx create --name "$builder" --driver docker-container --use
docker buildx inspect "$builder" --bootstrap
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.server -t "$tag-ingestion" .
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.web -t "$tag-web" .
docker buildx build --builder "$builder" --progress=plain --no-cache --load -f docker/Dockerfile.proxy -t "$tag-proxy" .
docker image inspect "$tag-ingestion" "$tag-web" "$tag-proxy" >/dev/null
echo DATASTACK_THREE_IMAGES_BUILD_PASS