From 41c7d318e953e54019d44843d5942e7185d29398 Mon Sep 17 00:00:00 2001 From: kert Date: Tue, 27 Oct 2026 11:04:54 -0400 Subject: [PATCH] fix(ci): make image and package scanning see again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Harden has failed every Sunday since 2026-09-27: the mc build (dead quay.io/minio/mc) aborted the job before a single trivy step ran, so for a month no image was scanned and nothing said so beyond an auto-filed issue per week. Harden (gen_config -> .gitea/workflows/harden.yml) - Each image is built -> scanned -> pushed on its own. A failed build is continue-on-error and skips only that image's scan and push; a gate step after the vuln report fails the job, so the breakage is still filed, never swallowed. - Builds use --pull --no-cache: --no-cache alone reused whatever stale base the runner had cached. - mc is scanned now too (it was built but excluded). Tooling pins (harden, deploy, rebuild-all, pkg-supply-chain) - trivy 0.76.0 via the tagged installer (it checks the release sha256) instead of install.sh from `main`. - crane 0.22.1 with its sha256 checked against the release checksums.txt, instead of an unverified `releases/latest` tarball. - Both are Renovate-tracked constants in backends/gitea.py. Package supply chain - docs/pnpm-lock.yaml is scanned nightly into data/pkg-vulns-docs.json and filed like uv.lock findings. - pkg_issues.py read only the first 50 open `ci` issues, so once CI failures piled up every known CVE past page one was invisible: re-filed nightly (174 open issues for 91 titles) and never closed. It now walks every page, closes all duplicates of a title, and runs the close pass even when a scan comes back clean — it used to return early there, i.e. exactly when the most issues resolve. Dry run against the live tracker with the new lock: closes all 174, files none. Refs #1365 --- .gitea/workflows/deploy.yml | 64 ++++++++++++--- .gitea/workflows/harden.yml | 95 +++++++++++++++------ .gitea/workflows/pkg-supply-chain.yml | 4 +- .gitea/workflows/rebuild-all.yml | 15 +++- dev/scripts/backends/gitea.py | 96 +++++++++++++++++----- dev/scripts/pkg_issues.py | 114 ++++++++++++++++---------- stack.toml | 1 - 7 files changed, 284 insertions(+), 105 deletions(-) diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 7e7e594..5897390 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -71,7 +71,13 @@ jobs: uses: https://github.com/actions/checkout@v4 - name: Install crane - run: curl -sL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /usr/local/bin crane + run: | + cd /tmp + base=https://github.com/google/go-containerregistry/releases/download/v0.22.1 + tgz=go-containerregistry_Linux_x86_64.tar.gz + curl -fsSLO "$base/$tgz" + curl -fsSL "$base/checksums.txt" | grep -F " $tgz" | sha256sum -c - + tar xzf "$tgz" -C /usr/local/bin crane - name: Log in to registry run: crane auth login git:3000 -u "${{ secrets.REGISTRY_USER }}" -p "${{ secrets.REGISTRY_TOKEN }}" @@ -80,7 +86,7 @@ jobs: - name: Install trivy - run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin + run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/v0.76.0/contrib/install.sh | sh -s -- -b /usr/local/bin v0.76.0 - name: Scan notebooks run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o notebooks-scan.json local/notebooks:build @@ -123,7 +129,13 @@ jobs: uses: https://github.com/actions/checkout@v4 - name: Install crane - run: curl -sL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /usr/local/bin crane + run: | + cd /tmp + base=https://github.com/google/go-containerregistry/releases/download/v0.22.1 + tgz=go-containerregistry_Linux_x86_64.tar.gz + curl -fsSLO "$base/$tgz" + curl -fsSL "$base/checksums.txt" | grep -F " $tgz" | sha256sum -c - + tar xzf "$tgz" -C /usr/local/bin crane - name: Log in to registry run: crane auth login git:3000 -u "${{ secrets.REGISTRY_USER }}" -p "${{ secrets.REGISTRY_TOKEN }}" @@ -132,7 +144,7 @@ jobs: - name: Install trivy - run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin + run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/v0.76.0/contrib/install.sh | sh -s -- -b /usr/local/bin v0.76.0 - name: Scan zotero run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o zotero-scan.json local/zotero:build @@ -175,7 +187,13 @@ jobs: uses: https://github.com/actions/checkout@v4 - name: Install crane - run: curl -sL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /usr/local/bin crane + run: | + cd /tmp + base=https://github.com/google/go-containerregistry/releases/download/v0.22.1 + tgz=go-containerregistry_Linux_x86_64.tar.gz + curl -fsSLO "$base/$tgz" + curl -fsSL "$base/checksums.txt" | grep -F " $tgz" | sha256sum -c - + tar xzf "$tgz" -C /usr/local/bin crane - name: Log in to registry run: crane auth login git:3000 -u "${{ secrets.REGISTRY_USER }}" -p "${{ secrets.REGISTRY_TOKEN }}" @@ -184,7 +202,7 @@ jobs: - name: Install trivy - run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin + run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/v0.76.0/contrib/install.sh | sh -s -- -b /usr/local/bin v0.76.0 - name: Scan docs run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o docs-scan.json local/docs:build @@ -227,7 +245,13 @@ jobs: uses: https://github.com/actions/checkout@v4 - name: Install crane - run: curl -sL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /usr/local/bin crane + run: | + cd /tmp + base=https://github.com/google/go-containerregistry/releases/download/v0.22.1 + tgz=go-containerregistry_Linux_x86_64.tar.gz + curl -fsSLO "$base/$tgz" + curl -fsSL "$base/checksums.txt" | grep -F " $tgz" | sha256sum -c - + tar xzf "$tgz" -C /usr/local/bin crane - name: Log in to registry run: crane auth login git:3000 -u "${{ secrets.REGISTRY_USER }}" -p "${{ secrets.REGISTRY_TOKEN }}" @@ -236,7 +260,7 @@ jobs: - name: Install trivy - run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin + run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/v0.76.0/contrib/install.sh | sh -s -- -b /usr/local/bin v0.76.0 - name: Scan api run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o api-scan.json local/api:build @@ -279,7 +303,13 @@ jobs: uses: https://github.com/actions/checkout@v4 - name: Install crane - run: curl -sL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /usr/local/bin crane + run: | + cd /tmp + base=https://github.com/google/go-containerregistry/releases/download/v0.22.1 + tgz=go-containerregistry_Linux_x86_64.tar.gz + curl -fsSLO "$base/$tgz" + curl -fsSL "$base/checksums.txt" | grep -F " $tgz" | sha256sum -c - + tar xzf "$tgz" -C /usr/local/bin crane - name: Log in to registry run: crane auth login git:3000 -u "${{ secrets.REGISTRY_USER }}" -p "${{ secrets.REGISTRY_TOKEN }}" @@ -288,7 +318,7 @@ jobs: - name: Install trivy - run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin + run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/v0.76.0/contrib/install.sh | sh -s -- -b /usr/local/bin v0.76.0 - name: Scan llm run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o llm-scan.json local/llm:build @@ -331,7 +361,13 @@ jobs: uses: https://github.com/actions/checkout@v4 - name: Install crane - run: curl -sL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /usr/local/bin crane + run: | + cd /tmp + base=https://github.com/google/go-containerregistry/releases/download/v0.22.1 + tgz=go-containerregistry_Linux_x86_64.tar.gz + curl -fsSLO "$base/$tgz" + curl -fsSL "$base/checksums.txt" | grep -F " $tgz" | sha256sum -c - + tar xzf "$tgz" -C /usr/local/bin crane - name: Log in to registry run: crane auth login git:3000 -u "${{ secrets.REGISTRY_USER }}" -p "${{ secrets.REGISTRY_TOKEN }}" @@ -339,6 +375,12 @@ jobs: CRANE_INSECURE: "true" + - name: Install trivy + run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/v0.76.0/contrib/install.sh | sh -s -- -b /usr/local/bin v0.76.0 + + - name: Scan mc + run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o mc-scan.json local/mc:build + - name: Compute short SHA run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV" diff --git a/.gitea/workflows/harden.yml b/.gitea/workflows/harden.yml index 993a480..2a5d9d3 100644 --- a/.gitea/workflows/harden.yml +++ b/.gitea/workflows/harden.yml @@ -16,7 +16,13 @@ jobs: uses: https://github.com/actions/checkout@v4 - name: Install crane - run: curl -sL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /usr/local/bin crane + run: | + cd /tmp + base=https://github.com/google/go-containerregistry/releases/download/v0.22.1 + tgz=go-containerregistry_Linux_x86_64.tar.gz + curl -fsSLO "$base/$tgz" + curl -fsSL "$base/checksums.txt" | grep -F " $tgz" | sha256sum -c - + tar xzf "$tgz" -C /usr/local/bin crane - name: Log in to registry run: crane auth login git:3000 -u "${{ secrets.REGISTRY_USER }}" -p "${{ secrets.REGISTRY_TOKEN }}" @@ -24,7 +30,7 @@ jobs: CRANE_INSECURE: "true" - name: Install trivy - run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin + run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/v0.76.0/contrib/install.sh | sh -s -- -b /usr/local/bin v0.76.0 - name: Set up uv run: curl -LsSf https://astral.sh/uv/install.sh | sh @@ -32,86 +38,127 @@ jobs: UV_INSTALL_DIR: /usr/local/bin - name: Build notebooks - run: docker build --no-cache -f infra/images/notebooks.Dockerfile -t local/notebooks:build . + id: build_notebooks + continue-on-error: true + run: docker build --pull --no-cache -f infra/images/notebooks.Dockerfile -t local/notebooks:build . + + - name: Scan notebooks + if: ${{ steps.build_notebooks.outcome == 'success' }} + run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o notebooks-scan.json local/notebooks:build - name: Push notebooks + if: ${{ steps.build_notebooks.outcome == 'success' }} run: | docker save local/notebooks:build -o /tmp/notebooks.tar crane push /tmp/notebooks.tar git:3000/homelab/stack/notebooks:hardened --insecure crane push /tmp/notebooks.tar git:3000/homelab/stack/notebooks:latest --insecure - name: Build zotero - run: docker build --no-cache -f infra/images/zotero.Dockerfile -t local/zotero:build . + id: build_zotero + continue-on-error: true + run: docker build --pull --no-cache -f infra/images/zotero.Dockerfile -t local/zotero:build . + + - name: Scan zotero + if: ${{ steps.build_zotero.outcome == 'success' }} + run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o zotero-scan.json local/zotero:build - name: Push zotero + if: ${{ steps.build_zotero.outcome == 'success' }} run: | docker save local/zotero:build -o /tmp/zotero.tar crane push /tmp/zotero.tar git:3000/homelab/stack/zotero:hardened --insecure crane push /tmp/zotero.tar git:3000/homelab/stack/zotero:latest --insecure - name: Build docs - run: docker build --no-cache -f infra/images/docs.Dockerfile -t local/docs:build . + id: build_docs + continue-on-error: true + run: docker build --pull --no-cache -f infra/images/docs.Dockerfile -t local/docs:build . + + - name: Scan docs + if: ${{ steps.build_docs.outcome == 'success' }} + run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o docs-scan.json local/docs:build - name: Push docs + if: ${{ steps.build_docs.outcome == 'success' }} run: | docker save local/docs:build -o /tmp/docs.tar crane push /tmp/docs.tar git:3000/homelab/stack/docs:hardened --insecure crane push /tmp/docs.tar git:3000/homelab/stack/docs:latest --insecure - name: Build api - run: docker build --no-cache -f infra/images/api.Dockerfile -t local/api:build . + id: build_api + continue-on-error: true + run: docker build --pull --no-cache -f infra/images/api.Dockerfile -t local/api:build . + + - name: Scan api + if: ${{ steps.build_api.outcome == 'success' }} + run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o api-scan.json local/api:build - name: Push api + if: ${{ steps.build_api.outcome == 'success' }} run: | docker save local/api:build -o /tmp/api.tar crane push /tmp/api.tar git:3000/homelab/stack/api:hardened --insecure crane push /tmp/api.tar git:3000/homelab/stack/api:latest --insecure - name: Build llm - run: docker build --no-cache -f infra/images/llm.Dockerfile -t local/llm:build . + id: build_llm + continue-on-error: true + run: docker build --pull --no-cache -f infra/images/llm.Dockerfile -t local/llm:build . + + - name: Scan llm + if: ${{ steps.build_llm.outcome == 'success' }} + run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o llm-scan.json local/llm:build - name: Push llm + if: ${{ steps.build_llm.outcome == 'success' }} run: | docker save local/llm:build -o /tmp/llm.tar crane push /tmp/llm.tar git:3000/homelab/stack/llm:hardened --insecure crane push /tmp/llm.tar git:3000/homelab/stack/llm:latest --insecure - name: Build mc - run: docker build --no-cache -f infra/images/mc.Dockerfile -t local/mc:build infra/rustfs/ + id: build_mc + continue-on-error: true + run: docker build --pull --no-cache -f infra/images/mc.Dockerfile -t local/mc:build infra/rustfs/ + + - name: Scan mc + if: ${{ steps.build_mc.outcome == 'success' }} + run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o mc-scan.json local/mc:build - name: Push mc + if: ${{ steps.build_mc.outcome == 'success' }} run: | docker save local/mc:build -o /tmp/mc.tar crane push /tmp/mc.tar git:3000/homelab/stack/mc:hardened --insecure crane push /tmp/mc.tar git:3000/homelab/stack/mc:latest --insecure - - name: Scan notebooks - run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o notebooks-scan.json local/notebooks:build - - - name: Scan zotero - run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o zotero-scan.json local/zotero:build - - - name: Scan docs - run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o docs-scan.json local/docs:build - - - name: Scan api - run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o api-scan.json local/api:build - - - name: Scan llm - run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o llm-scan.json local/llm:build - - name: Close resolved or file new vuln issues env: GITEA_TOKEN: ${{ secrets.DEPLOY_TOKEN }} run: | uv sync --no-dev - for f in notebooks-scan.json zotero-scan.json docs-scan.json api-scan.json llm-scan.json; do + for f in notebooks-scan.json zotero-scan.json docs-scan.json api-scan.json llm-scan.json mc-scan.json; do if [ -f "$f" ]; then uv run python -m api.diag.vuln --close "$f" || \ uv run python -m api.diag.vuln "$f" || true fi done + - name: Fail if any image failed to build + run: | + failed="" + [ "${{ steps.build_notebooks.outcome }}" = success ] || failed="$failed notebooks" + [ "${{ steps.build_zotero.outcome }}" = success ] || failed="$failed zotero" + [ "${{ steps.build_docs.outcome }}" = success ] || failed="$failed docs" + [ "${{ steps.build_api.outcome }}" = success ] || failed="$failed api" + [ "${{ steps.build_llm.outcome }}" = success ] || failed="$failed llm" + [ "${{ steps.build_mc.outcome }}" = success ] || failed="$failed mc" + if [ -n "$failed" ]; then + echo "Images that failed to build (not scanned, not pushed):$failed" + exit 1 + fi + - name: File failure issue if: failure() env: diff --git a/.gitea/workflows/pkg-supply-chain.yml b/.gitea/workflows/pkg-supply-chain.yml index dc92e13..8b055a3 100644 --- a/.gitea/workflows/pkg-supply-chain.yml +++ b/.gitea/workflows/pkg-supply-chain.yml @@ -45,13 +45,15 @@ jobs: run: uv run python dev/scripts/pkg_drift.py - name: Install trivy - run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin + run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/v0.76.0/contrib/install.sh | sh -s -- -b /usr/local/bin v0.76.0 - name: Vulnerability scan run: | set -euo pipefail trivy fs --scanners vuln --format json -o data/pkg-vulns.json uv.lock trivy fs --scanners vuln --format json -o data/pkg-vulns-pyproject.json pyproject.toml + # The docs site's build toolchain — never scanned before (116 CVEs, 2026-10-27). + trivy fs --scanners vuln --format json -o data/pkg-vulns-docs.json docs/pnpm-lock.yaml - name: Auto-create issues for drift and vulns env: diff --git a/.gitea/workflows/rebuild-all.yml b/.gitea/workflows/rebuild-all.yml index 7eb6a93..a60d8e3 100644 --- a/.gitea/workflows/rebuild-all.yml +++ b/.gitea/workflows/rebuild-all.yml @@ -14,7 +14,13 @@ jobs: uses: https://github.com/actions/checkout@v4 - name: Install crane - run: curl -sL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /usr/local/bin crane + run: | + cd /tmp + base=https://github.com/google/go-containerregistry/releases/download/v0.22.1 + tgz=go-containerregistry_Linux_x86_64.tar.gz + curl -fsSLO "$base/$tgz" + curl -fsSL "$base/checksums.txt" | grep -F " $tgz" | sha256sum -c - + tar xzf "$tgz" -C /usr/local/bin crane - name: Log in to registry run: crane auth login git:3000 -u "${{ secrets.REGISTRY_USER }}" -p "${{ secrets.REGISTRY_TOKEN }}" @@ -22,7 +28,7 @@ jobs: CRANE_INSECURE: "true" - name: Install trivy - run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin + run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/v0.76.0/contrib/install.sh | sh -s -- -b /usr/local/bin v0.76.0 - name: Set up uv run: curl -LsSf https://astral.sh/uv/install.sh | sh @@ -101,12 +107,15 @@ jobs: - name: Scan llm run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o llm-scan.json local/llm:build + - name: Scan mc + run: trivy image --severity HIGH,CRITICAL --exit-code 0 --format json -o mc-scan.json local/mc:build + - name: Report vulnerabilities env: GITEA_TOKEN: ${{ secrets.DEPLOY_TOKEN }} run: | uv sync --no-dev - for f in notebooks-scan.json zotero-scan.json docs-scan.json api-scan.json llm-scan.json; do + for f in notebooks-scan.json zotero-scan.json docs-scan.json api-scan.json llm-scan.json mc-scan.json; do [ -f "$f" ] && uv run python -m api.diag.vuln "$f" || true done diff --git a/dev/scripts/backends/gitea.py b/dev/scripts/backends/gitea.py index f2e6b21..ec97ea5 100644 --- a/dev/scripts/backends/gitea.py +++ b/dev/scripts/backends/gitea.py @@ -14,6 +14,15 @@ from backends.github import ( _HEADER, ) +# Pinned scanner/registry tooling. A `latest`/`main` install would make the +# tools that guard the supply chain an unpinned dependency themselves. +# Renovate bumps these via the regex manager in renovate.json; re-run +# gen_config.py after a bump. +# renovate: datasource=github-releases depName=aquasecurity/trivy +TRIVY_VERSION = "0.76.0" +# renovate: datasource=github-releases depName=google/go-containerregistry +CRANE_VERSION = "0.22.1" + # ── Override action refs with full GitHub URLs ─────────────────── # Gitea Actions resolves short refs (e.g. actions/checkout@v4) against # the local Gitea instance. Use full URLs to pull from GitHub. @@ -57,7 +66,9 @@ def _build_push_step( name = img["name"] tags = [t.strip() for t in tags_expr.split(",") if t.strip()] local_tag = f"local/{name}:build" - cache_flag = " --no-cache" if no_cache else "" + # A from-scratch rebuild must also re-pull base images — --no-cache alone + # reuses whatever stale base the runner already has. + cache_flag = " --pull --no-cache" if no_cache else "" lines = f"""\ - name: Build {name} run: docker build{cache_flag} -f {img["dockerfile"]} -t {local_tag} {img["context"]}""" @@ -76,10 +87,15 @@ def _build_push_step( def _install_trivy_step() -> str: - """Install trivy as a native binary — no Docker-in-Docker needed.""" - return """\ + """Install a pinned trivy as a native binary — no Docker-in-Docker needed. + + The installer from the release tag verifies the tarball's sha256 against + the release checksums. + """ + v = TRIVY_VERSION + return f"""\ - name: Install trivy - run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin""" + run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/v{v}/contrib/install.sh | sh -s -- -b /usr/local/bin v{v}""" def _trivy_step( @@ -129,7 +145,13 @@ def _docker_login_step(registry: str) -> str: """Install crane and authenticate — daemonless push tool.""" return f"""\ - name: Install crane - run: curl -sL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /usr/local/bin crane + run: | + cd /tmp + base=https://github.com/google/go-containerregistry/releases/download/v{CRANE_VERSION} + tgz=go-containerregistry_Linux_x86_64.tar.gz + curl -fsSLO "$base/$tgz" + curl -fsSL "$base/checksums.txt" | grep -F " $tgz" | sha256sum -c - + tar xzf "$tgz" -C /usr/local/bin crane - name: Log in to registry run: crane auth login {registry} -u "${{{{ secrets.REGISTRY_USER }}}}" -p "${{{{ secrets.REGISTRY_TOKEN }}}}" @@ -412,21 +434,46 @@ def _gen_harden( uv_version: str, **_kw: object, ) -> tuple[str, str]: - build_steps = [] + # Each image is built, scanned and pushed on its own, and a failed build + # only skips that image: one dead upstream base (the archived minio/mc, + # Sept–Oct 2026) used to abort the job before any scan ran, leaving every + # image unscanned for weeks. The gate step at the end still fails the job + # — after the vuln report — so the breakage is filed, not swallowed. + scan_names = {i["name"] for i in scans} + image_steps = [] for img in images: - tags = ( - f"{registry}/{owner_repo}/{img['name']}:hardened," - f"{registry}/{owner_repo}/{img['name']}:latest" - ) - build_steps.append( - _build_push_step(img, tags, registry, owner_repo, no_cache=True) - ) - build_block = "\n\n".join(build_steps) + name = img["name"] + step_id = f"build_{name.replace('-', '_')}" + built = f"steps.{step_id}.outcome == 'success'" + steps = [ + f"""\ + - name: Build {name} + id: {step_id} + continue-on-error: true + run: docker build --pull --no-cache -f {img["dockerfile"]} -t local/{name}:build {img["context"]}""" + ] + if name in scan_names: + sev = img.get("trivy_severity", "HIGH,CRITICAL") + ec = img.get("trivy_exit_code", 0) + steps.append(f"""\ + - name: Scan {name} + if: ${{{{ {built} }}}} + run: trivy image --severity {sev} --exit-code {ec} --format json -o {name}-scan.json local/{name}:build""") + steps.append(f"""\ + - name: Push {name} + if: ${{{{ {built} }}}} + run: | + docker save local/{name}:build -o /tmp/{name}.tar + crane push /tmp/{name}.tar {registry}/{owner_repo}/{name}:hardened --insecure + crane push /tmp/{name}.tar {registry}/{owner_repo}/{name}:latest --insecure""") + image_steps.append("\n\n".join(steps)) + image_block = "\n\n".join(image_steps) - scan_steps = [] - for img in scans: - scan_steps.append(_trivy_step(img, "hardened", registry, owner_repo)) - scan_block = "\n\n".join(scan_steps) + gate_lines = "\n ".join( + f'[ "${{{{ steps.build_{i["name"].replace("-", "_")}.outcome }}}}" = success ]' + f' || failed="$failed {i["name"]}"' + for i in images + ) vuln_files = " ".join(f"{i['name']}-scan.json" for i in scans) @@ -451,9 +498,7 @@ jobs: {_setup_uv_step(uv_version)} -{build_block} - -{scan_block} +{image_block} - name: Close resolved or file new vuln issues env: @@ -467,6 +512,15 @@ jobs: fi done + - name: Fail if any image failed to build + run: | + failed="" + {gate_lines} + if [ -n "$failed" ]; then + echo "Images that failed to build (not scanned, not pushed):$failed" + exit 1 + fi + {_failure_step("Harden", "build-scan-report")} """ return (".gitea/workflows/harden.yml", content) diff --git a/dev/scripts/pkg_issues.py b/dev/scripts/pkg_issues.py index c763197..bcb0385 100644 --- a/dev/scripts/pkg_issues.py +++ b/dev/scripts/pkg_issues.py @@ -20,7 +20,11 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[2] MANIFEST_PATH = ROOT / "data" / "pkg-manifest.json" -VULN_PATH = ROOT / "data" / "pkg-vulns.json" +# trivy JSON reports, one per lockfile (uv.lock, docs/pnpm-lock.yaml) +VULN_PATHS = ( + ROOT / "data" / "pkg-vulns.json", + ROOT / "data" / "pkg-vulns-docs.json", +) GITEA_URL = os.environ.get("GITEA_URL", "http://localhost:3000") GITEA_TOKEN = os.environ.get("GITEA_TOKEN", "") @@ -99,15 +103,41 @@ def _get_label_ids(names: list[str]) -> list[int]: return [name_to_id[n] for n in names if n in name_to_id] -def _find_existing_issues(prefix: str) -> dict[str, dict]: - """Find open issues with titles starting with prefix.""" - issues = _api( - "GET", - f"repos/{REPO}/issues?state=open&type=issues&limit=50&labels={LABEL_CI}", - ) - if not issues: - return {} - return {i["title"]: i for i in issues if i["title"].startswith(prefix)} +def _find_existing_issues(prefix: str) -> dict[str, list[dict]]: + """Find open issues with titles starting with prefix, grouped by title. + + Walks every page: a single 50-issue page hid most of the tracker once + CI-failure issues piled up, so known CVEs were re-filed nightly (174 + issues for 91 titles) and resolved ones were never closed. + """ + found: dict[str, list[dict]] = {} + page = 1 + while True: + issues = _api( + "GET", + f"repos/{REPO}/issues?state=open&type=issues&limit=50" + f"&page={page}&labels={LABEL_CI}", + ) + if not issues: + return found + for i in issues: + if i["title"].startswith(prefix): + found.setdefault(i["title"], []).append(i) + page += 1 + + +def _close_all(title: str, issues: list[dict], *, dry_run: bool) -> None: + """Close every open issue carrying ``title`` (duplicates included).""" + for issue in issues: + if dry_run: + print(f" would close #{issue['number']}: {title}") + else: + _api( + "PATCH", + f"repos/{REPO}/issues/{issue['number']}", + {"state": "closed"}, + ) + print(f" closed #{issue['number']}: {title}") def handle_drift(*, dry_run: bool = False) -> int: @@ -172,7 +202,7 @@ def handle_drift(*, dry_run: bool = False) -> int: created += 1 # Close issues for packages that are no longer missing - for title, issue in existing.items(): + for title, issues in existing.items(): # Extract package name from title m = re.match(r"\[pkg-missing\] (\w+)/(.+?)(?:\s|$)", title) if not m: @@ -186,40 +216,36 @@ def handle_drift(*, dry_run: bool = False) -> int: if pkg_name in sub_d.get("untracked", []): still_missing = True break - if not still_missing: - if dry_run: - print(f" would close: {title}") - else: - _api( - "PATCH", - f"repos/{REPO}/issues/{issue['number']}", - {"state": "closed"}, - ) - print(f" closed #{issue['number']}: {title}") + issues = sorted(issues, key=lambda i: i.get("number", 0)) + _close_all(title, issues[1:] if still_missing else issues, dry_run=dry_run) return created def handle_vulns(*, dry_run: bool = False) -> int: - """Create issues for vulnerabilities found in package scans.""" - if not VULN_PATH.exists(): - print(" vulns: no scan results found (data/pkg-vulns.json)") - return 0 + """Create issues for vulnerabilities found in package scans. - data = json.loads(VULN_PATH.read_text()) + Also closes issues whose CVE no longer appears — including when a scan + comes back clean, which is exactly when the most issues resolve. + """ + reports = [p for p in VULN_PATHS if p.exists()] + if not reports: + print(" vulns: no scan results found (data/pkg-vulns*.json)") + return 0 # trivy JSON format: {"Results": [{"Vulnerabilities": [...]}]} vulns: list[dict] = [] - for result in data.get("Results", []): - for v in result.get("Vulnerabilities", []): - vulns.append(v) + for path in reports: + data = json.loads(path.read_text()) + for result in data.get("Results", []): + for v in result.get("Vulnerabilities") or []: + vulns.append(v) if not vulns: print(" vulns: no vulnerabilities found") - return 0 - milestone_id = _get_milestone_id() - label_ids = _get_label_ids([LABEL_CI, LABEL_QUALITY]) + milestone_id = _get_milestone_id() if vulns else None + label_ids = _get_label_ids([LABEL_CI, LABEL_QUALITY]) if vulns else [] existing = _find_existing_issues("[pkg-vuln]") created = 0 @@ -245,6 +271,7 @@ def handle_vulns(*, dry_run: bool = False) -> int: if dry_run: print(f" would create: {title}") + existing[title] = [] else: issue_data = {"title": title, "body": body, "labels": label_ids} if milestone_id: @@ -253,24 +280,23 @@ def handle_vulns(*, dry_run: bool = False) -> int: if result: print(f" created #{result['number']}: {title}") created += 1 + # the same CVE can surface twice (two lockfiles, two paths) + existing[title] = [result] - # Close issues for CVEs that are no longer present + # Close issues for CVEs that are no longer present, and duplicates of + # ones that are (keep the oldest open). active_cves = { f"[pkg-vuln] {v.get('VulnerabilityID', '')} in " f"{v.get('PkgName', '')}@{v.get('InstalledVersion', '')}" for v in vulns } - for title, issue in existing.items(): - if title not in active_cves: - if dry_run: - print(f" would close: {title}") - else: - _api( - "PATCH", - f"repos/{REPO}/issues/{issue['number']}", - {"state": "closed"}, - ) - print(f" closed #{issue['number']}: {title}") + for title, issues in existing.items(): + issues = sorted(issues, key=lambda i: i.get("number", 0)) + _close_all( + title, + issues[1:] if title in active_cves else issues, + dry_run=dry_run, + ) return created diff --git a/stack.toml b/stack.toml index bf5ac17..f283404 100644 --- a/stack.toml +++ b/stack.toml @@ -320,4 +320,3 @@ path_filter = ["infra/images/llm.Dockerfile", "src/**", "pyproject.toml"] dockerfile = "infra/images/mc.Dockerfile" context = "infra/rustfs/" path_filter = ["infra/rustfs/**", "infra/images/mc.Dockerfile"] -scan = false