41c7d318e9
Harden has failed every Sunday since 2026-09-27: the mc build (dead quay.io/minio/mc) aborted the job before a single trivy step ran, so for a month no image was scanned and nothing said so beyond an auto-filed issue per week. Harden (gen_config -> .gitea/workflows/harden.yml) - Each image is built -> scanned -> pushed on its own. A failed build is continue-on-error and skips only that image's scan and push; a gate step after the vuln report fails the job, so the breakage is still filed, never swallowed. - Builds use --pull --no-cache: --no-cache alone reused whatever stale base the runner had cached. - mc is scanned now too (it was built but excluded). Tooling pins (harden, deploy, rebuild-all, pkg-supply-chain) - trivy 0.76.0 via the tagged installer (it checks the release sha256) instead of install.sh from `main`. - crane 0.22.1 with its sha256 checked against the release checksums.txt, instead of an unverified `releases/latest` tarball. - Both are Renovate-tracked constants in backends/gitea.py. Package supply chain - docs/pnpm-lock.yaml is scanned nightly into data/pkg-vulns-docs.json and filed like uv.lock findings. - pkg_issues.py read only the first 50 open `ci` issues, so once CI failures piled up every known CVE past page one was invisible: re-filed nightly (174 open issues for 91 titles) and never closed. It now walks every page, closes all duplicates of a title, and runs the close pass even when a scan comes back clean — it used to return early there, i.e. exactly when the most issues resolve. Dry run against the live tracker with the new lock: closes all 174, files none. Refs #1365
916 lines
28 KiB
Python
916 lines
28 KiB
Python
"""Gitea Actions workflow emitter.
|
||
|
||
Thin adapter over the GitHub backend — Gitea Actions uses the same
|
||
YAML syntax but with:
|
||
- Output to .gitea/workflows/ (not .github/workflows/)
|
||
- Registry auth via PAT secret (GITEA_TOKEN lacks package scope)
|
||
- Platform registry (not ghcr.io)
|
||
- Full URL form for actions (for air-gapped installs)
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from backends.github import (
|
||
_HEADER,
|
||
)
|
||
|
||
# Pinned scanner/registry tooling. A `latest`/`main` install would make the
|
||
# tools that guard the supply chain an unpinned dependency themselves.
|
||
# Renovate bumps these via the regex manager in renovate.json; re-run
|
||
# gen_config.py after a bump.
|
||
# renovate: datasource=github-releases depName=aquasecurity/trivy
|
||
TRIVY_VERSION = "0.76.0"
|
||
# renovate: datasource=github-releases depName=google/go-containerregistry
|
||
CRANE_VERSION = "0.22.1"
|
||
|
||
# ── Override action refs with full GitHub URLs ───────────────────
|
||
# Gitea Actions resolves short refs (e.g. actions/checkout@v4) against
|
||
# the local Gitea instance. Use full URLs to pull from GitHub.
|
||
|
||
|
||
def _checkout_step() -> str:
|
||
return """\
|
||
- name: Checkout
|
||
uses: https://github.com/actions/checkout@v4"""
|
||
|
||
|
||
def _setup_buildx_step() -> str:
|
||
return "" # Not needed — we use plain docker build/push
|
||
|
||
|
||
def _setup_uv_step(uv_version: str) -> str:
|
||
# astral-sh/setup-uv is a composite action that calls the Gitea API
|
||
# internally, which fails because 'astral-sh' isn't a local user.
|
||
# Install uv directly via the official installer script instead.
|
||
return """\
|
||
- name: Set up uv
|
||
run: curl -LsSf https://astral.sh/uv/install.sh | sh
|
||
env:
|
||
UV_INSTALL_DIR: /usr/local/bin"""
|
||
|
||
|
||
def _build_push_step(
|
||
img: dict,
|
||
tags_expr: str,
|
||
registry: str,
|
||
owner_repo: str,
|
||
*,
|
||
no_cache: bool = False,
|
||
load_only: bool = False,
|
||
) -> str:
|
||
"""docker build (via socket) + crane push (daemonless, uses container DNS).
|
||
|
||
docker push goes through the host daemon which can't resolve 'gitea'.
|
||
crane pushes directly from the job container's network stack.
|
||
"""
|
||
name = img["name"]
|
||
tags = [t.strip() for t in tags_expr.split(",") if t.strip()]
|
||
local_tag = f"local/{name}:build"
|
||
# A from-scratch rebuild must also re-pull base images — --no-cache alone
|
||
# reuses whatever stale base the runner already has.
|
||
cache_flag = " --pull --no-cache" if no_cache else ""
|
||
lines = f"""\
|
||
- name: Build {name}
|
||
run: docker build{cache_flag} -f {img["dockerfile"]} -t {local_tag} {img["context"]}"""
|
||
if not load_only and tags:
|
||
# Save to tarball, then crane push each tag (daemonless)
|
||
push_cmds = "\n ".join(
|
||
f"crane push /tmp/{name}.tar {t} --insecure" for t in tags
|
||
)
|
||
lines += f"""
|
||
|
||
- name: Push {name}
|
||
run: |
|
||
docker save {local_tag} -o /tmp/{name}.tar
|
||
{push_cmds}"""
|
||
return lines
|
||
|
||
|
||
def _install_trivy_step() -> str:
|
||
"""Install a pinned trivy as a native binary — no Docker-in-Docker needed.
|
||
|
||
The installer from the release tag verifies the tarball's sha256 against
|
||
the release checksums.
|
||
"""
|
||
v = TRIVY_VERSION
|
||
return f"""\
|
||
- name: Install trivy
|
||
run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/v{v}/contrib/install.sh | sh -s -- -b /usr/local/bin v{v}"""
|
||
|
||
|
||
def _trivy_step(
|
||
img: dict,
|
||
tag: str,
|
||
registry: str,
|
||
owner_repo: str,
|
||
) -> str:
|
||
"""Scan local Docker image — no registry pull, no DNS resolution."""
|
||
name = img["name"]
|
||
local_tag = f"local/{name}:build"
|
||
sev = img.get("trivy_severity", "HIGH,CRITICAL")
|
||
ec = img.get("trivy_exit_code", 0)
|
||
return f"""\
|
||
- name: Scan {name}
|
||
run: trivy image --severity {sev} --exit-code {ec} --format json -o {name}-scan.json {local_tag}"""
|
||
|
||
|
||
# ── Failure reporting ─────────────────────────────────────────────
|
||
|
||
|
||
def _failure_step(workflow_name: str, job_name: str) -> str:
|
||
"""Emit an if:failure() step that files a Gitea issue."""
|
||
return f"""\
|
||
- name: File failure issue
|
||
if: failure()
|
||
env:
|
||
GITEA_TOKEN: ${{{{ secrets.DEPLOY_TOKEN }}}}
|
||
run: |
|
||
# Jobs that never install uv (image builds, infra checks) still
|
||
# need it here — otherwise the filer dies with "uv: command not
|
||
# found" and the failure is never filed.
|
||
command -v uv >/dev/null 2>&1 || curl -LsSf https://astral.sh/uv/install.sh | sh
|
||
export PATH="$HOME/.local/bin:$PATH"
|
||
uv sync --no-dev --quiet 2>/dev/null || true
|
||
uv run python -m api.diag.ci \\
|
||
--workflow "{workflow_name}" --job "{job_name}" \\
|
||
--run "${{{{ github.run_number }}}}" \\
|
||
--sha "${{{{ github.sha }}}}" \\
|
||
--ref "${{{{ github.ref }}}}" || true"""
|
||
|
||
|
||
# ── Gitea-specific helpers ───────────────────────────────────────
|
||
|
||
|
||
def _docker_login_step(registry: str) -> str:
|
||
"""Install crane and authenticate — daemonless push tool."""
|
||
return f"""\
|
||
- name: Install crane
|
||
run: |
|
||
cd /tmp
|
||
base=https://github.com/google/go-containerregistry/releases/download/v{CRANE_VERSION}
|
||
tgz=go-containerregistry_Linux_x86_64.tar.gz
|
||
curl -fsSLO "$base/$tgz"
|
||
curl -fsSL "$base/checksums.txt" | grep -F " $tgz" | sha256sum -c -
|
||
tar xzf "$tgz" -C /usr/local/bin crane
|
||
|
||
- name: Log in to registry
|
||
run: crane auth login {registry} -u "${{{{ secrets.REGISTRY_USER }}}}" -p "${{{{ secrets.REGISTRY_TOKEN }}}}"
|
||
env:
|
||
CRANE_INSECURE: "true\""""
|
||
|
||
|
||
# ── Workflow generators ──────────────────────────────────────────
|
||
|
||
|
||
def _gen_ci(
|
||
runner: str, uv_version: str, coverage_threshold: int = 99, **_kw: object
|
||
) -> tuple[str, str]:
|
||
content = f"""\
|
||
{_HEADER}
|
||
name: CI
|
||
|
||
on:
|
||
push:
|
||
branches: ["**"]
|
||
pull_request:
|
||
|
||
jobs:
|
||
lint:
|
||
runs-on: {runner}
|
||
steps:
|
||
{_checkout_step()}
|
||
|
||
{_setup_uv_step(uv_version)}
|
||
|
||
- name: Install dependencies
|
||
run: uv sync --dev
|
||
|
||
- name: Ruff check
|
||
run: uv run ruff check src/ tests/ --output-format=concise
|
||
|
||
- name: Ruff format
|
||
run: uv run ruff format --check src/ tests/
|
||
|
||
- name: Validate generated config
|
||
run: uv run python dev/scripts/gen_config.py --check
|
||
|
||
test:
|
||
runs-on: {runner}
|
||
steps:
|
||
{_checkout_step()}
|
||
|
||
{_setup_uv_step(uv_version)}
|
||
|
||
- name: Install dependencies
|
||
run: uv sync --dev
|
||
|
||
- name: Preinstall duckdb extensions
|
||
# tests/zot/test_duck.py runs INSTALL sqlite from every xdist
|
||
# worker; concurrent installs race the extension-file rename in
|
||
# ~/.duckdb ("Could not remove file ... sqlite_scanner", #515).
|
||
# Installing once up front makes the in-test INSTALL a no-op.
|
||
# ducklake: same race class, used by tests/aco DuckLakeContext.
|
||
run: uv run python -c "import duckdb; duckdb.connect().execute('INSTALL sqlite; INSTALL ducklake')"
|
||
|
||
- name: Pytest
|
||
# -n auto parallelizes across runner cores. Coverage combining
|
||
# is configured via [tool.coverage.run] parallel=true in
|
||
# pyproject.toml — without that the per-worker .coverage.* files
|
||
# don't merge reliably (see #388 for the prior revert).
|
||
run: uv run pytest tests/ -x --cov=src --cov-report=term-missing --cov-fail-under={coverage_threshold} -q -n auto
|
||
|
||
{_failure_step("CI", "test")}
|
||
|
||
notebooks-smoke:
|
||
runs-on: {runner}
|
||
steps:
|
||
{_checkout_step()}
|
||
|
||
{_setup_uv_step(uv_version)}
|
||
|
||
- name: Install dependencies
|
||
run: uv sync --dev
|
||
|
||
- name: Stub aco.duckdb
|
||
# The checkout has no data volume; conf.connect.duckdb() opens
|
||
# data/aco.duckdb read-only and fails if it doesn't exist. An
|
||
# empty-but-valid database is enough for the ci-smoke notebooks
|
||
# (schema browsing over information_schema returns zero rows).
|
||
run: |
|
||
[ -f data/aco.duckdb ] || uv run python -c \\
|
||
"import duckdb; duckdb.connect('data/aco.duckdb').close()"
|
||
|
||
- name: Run data-independent notebooks headless
|
||
# Executes the [ci_smoke] set from infra/marimo/nb-tests.toml via
|
||
# `marimo export session` and fails on any cell error. Runs on
|
||
# every push (not path-gated): notebooks import src/ modules, so
|
||
# src changes can break them too.
|
||
run: uv run python dev/scripts/nb_integration.py --set ci-smoke
|
||
|
||
{_failure_step("CI", "notebooks-smoke")}
|
||
"""
|
||
return (".gitea/workflows/ci.yml", content)
|
||
|
||
|
||
_CHANGES_JOB = "changes"
|
||
_CHANGES_STEP_ID = "filter"
|
||
_ALL_CHANGED = "__all__"
|
||
_NULL_SHA = "0" * 40
|
||
|
||
# Shell body of the change-detection step; the braces of the `match`
|
||
# function are doubled so they survive the f-string.
|
||
_CHANGES_SCRIPT = f"""\
|
||
# `github.event.head_commit.modified` only describes the tip commit
|
||
# and is empty for merge commits, so a PR merged with a merge commit
|
||
# never matched any path filter. Diff the pushed range instead:
|
||
# the branch tip before the push against the new one. When that
|
||
# previous tip is unknown or unfetchable (first push, history
|
||
# rewrite) build everything rather than nothing.
|
||
changed={_ALL_CHANGED}
|
||
if [ -n "$BEFORE" ] && [ "$BEFORE" != "{_NULL_SHA}" ] \\
|
||
&& git fetch --quiet --depth=1 origin "$BEFORE"; then
|
||
changed="$(git diff --name-only "$BEFORE" "$GITHUB_SHA")" || changed={_ALL_CHANGED}
|
||
fi
|
||
echo "changed since $BEFORE:"
|
||
printf '%s\\n' "$changed"
|
||
match() {{
|
||
name="$1"; shift
|
||
hit=false
|
||
if [ "$changed" = {_ALL_CHANGED} ]; then
|
||
hit=true
|
||
else
|
||
while IFS= read -r f; do
|
||
[ -n "$f" ] || continue
|
||
for g in "$@"; do
|
||
case "$f" in $g) hit=true ;; esac
|
||
done
|
||
done <<< "$changed"
|
||
fi
|
||
echo "$name=$hit" >> "$GITHUB_OUTPUT"
|
||
echo "$name: $hit"
|
||
}}"""
|
||
|
||
|
||
def _case_glob(pattern: str) -> str:
|
||
"""Translate a ``path_filter`` glob into a bash ``case`` pattern.
|
||
|
||
``*`` already crosses ``/`` in ``case``, so ``src/**`` becomes ``src/*``.
|
||
"""
|
||
while "**" in pattern:
|
||
pattern = pattern.replace("**", "*")
|
||
return pattern
|
||
|
||
|
||
def _changes_job(images: list[dict], runner: str) -> str:
|
||
"""Emit the job that turns the pushed diff into one boolean per image."""
|
||
outputs = "\n".join(
|
||
f" {img['name']}: "
|
||
f"${{{{ steps.{_CHANGES_STEP_ID}.outputs.{img['name']} }}}}"
|
||
for img in images
|
||
)
|
||
matches = []
|
||
for img in images:
|
||
globs = [_case_glob(p) for p in img.get("path_filter", [])]
|
||
if globs:
|
||
quoted = " ".join(f"'{g}'" for g in globs)
|
||
matches.append(f" match {img['name']} {quoted}")
|
||
else:
|
||
# No filter means the image follows every push.
|
||
matches.append(f' echo "{img["name"]}=true" >> "$GITHUB_OUTPUT"')
|
||
match_block = "\n".join(matches)
|
||
return f"""\
|
||
{_CHANGES_JOB}:
|
||
runs-on: {runner}
|
||
outputs:
|
||
{outputs}
|
||
steps:
|
||
{_checkout_step()}
|
||
|
||
- name: Detect changed paths
|
||
id: {_CHANGES_STEP_ID}
|
||
env:
|
||
BEFORE: ${{{{ github.event.before }}}}
|
||
run: |
|
||
{_CHANGES_SCRIPT}
|
||
{match_block}"""
|
||
|
||
|
||
def _gen_deploy(
|
||
images: list[dict],
|
||
scans: list[dict],
|
||
registry: str,
|
||
owner_repo: str,
|
||
runner: str,
|
||
uv_version: str,
|
||
**_kw: object,
|
||
) -> tuple[str, str]:
|
||
scan_set = {s["name"] for s in scans}
|
||
|
||
# Gitea Actions doesn't support `paths:` filters on push, so one job
|
||
# computes which images the pushed range touches and the per-image
|
||
# jobs gate on its outputs.
|
||
job_blocks = [_changes_job(images, runner)]
|
||
job_names = [_CHANGES_JOB]
|
||
for img in images:
|
||
name = img["name"]
|
||
job_names.append(name)
|
||
tags = (
|
||
f"{registry}/{owner_repo}/{name}:${{{{ env.SHORT_SHA }}}},"
|
||
f"{registry}/{owner_repo}/{name}:latest"
|
||
)
|
||
scan_step = ""
|
||
if name in scan_set:
|
||
scan_step = f"""
|
||
|
||
{_install_trivy_step()}
|
||
|
||
{_trivy_step(img, "${{{{ env.SHORT_SHA }}}}", registry, owner_repo)}"""
|
||
|
||
job_blocks.append(f"""\
|
||
{name}:
|
||
runs-on: {runner}
|
||
needs: [{_CHANGES_JOB}]
|
||
if: needs.{_CHANGES_JOB}.outputs.{name} == 'true'
|
||
steps:
|
||
{_checkout_step()}
|
||
|
||
{_docker_login_step(registry)}
|
||
{scan_step}
|
||
|
||
- name: Compute short SHA
|
||
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
||
|
||
{_build_push_step(img, tags, registry, owner_repo)}
|
||
|
||
{_failure_step("Deploy", name)}""")
|
||
|
||
jobs_block = "\n\n".join(job_blocks)
|
||
needs_list = ", ".join(job_names)
|
||
|
||
content = f"""\
|
||
{_HEADER}
|
||
name: Deploy
|
||
|
||
on:
|
||
push:
|
||
branches: [main]
|
||
|
||
jobs:
|
||
{jobs_block}
|
||
|
||
report:
|
||
runs-on: {runner}
|
||
needs: [{needs_list}]
|
||
if: always() && !cancelled()
|
||
steps:
|
||
{_checkout_step()}
|
||
|
||
{_setup_uv_step(uv_version)}
|
||
|
||
- name: Report
|
||
# Only file an issue if at least one upstream build job failed.
|
||
# Without this gate api.diag.ci runs unconditionally and files
|
||
# a fake "failed" issue on every successful Deploy run.
|
||
if: contains(needs.*.result, 'failure')
|
||
env:
|
||
GITEA_TOKEN: ${{{{ secrets.DEPLOY_TOKEN }}}}
|
||
run: |
|
||
uv sync --no-dev --quiet 2>/dev/null || true
|
||
uv run python -m api.diag.ci \\
|
||
--workflow "Deploy" --job "report" \\
|
||
--run "${{{{ github.run_number }}}}" \\
|
||
--sha "${{{{ github.sha }}}}" \\
|
||
--ref "${{{{ github.ref }}}}" || true
|
||
"""
|
||
return (".gitea/workflows/deploy.yml", content)
|
||
|
||
|
||
def _gen_harden(
|
||
images: list[dict],
|
||
scans: list[dict],
|
||
registry: str,
|
||
owner_repo: str,
|
||
runner: str,
|
||
uv_version: str,
|
||
**_kw: object,
|
||
) -> tuple[str, str]:
|
||
# Each image is built, scanned and pushed on its own, and a failed build
|
||
# only skips that image: one dead upstream base (the archived minio/mc,
|
||
# Sept–Oct 2026) used to abort the job before any scan ran, leaving every
|
||
# image unscanned for weeks. The gate step at the end still fails the job
|
||
# — after the vuln report — so the breakage is filed, not swallowed.
|
||
scan_names = {i["name"] for i in scans}
|
||
image_steps = []
|
||
for img in images:
|
||
name = img["name"]
|
||
step_id = f"build_{name.replace('-', '_')}"
|
||
built = f"steps.{step_id}.outcome == 'success'"
|
||
steps = [
|
||
f"""\
|
||
- name: Build {name}
|
||
id: {step_id}
|
||
continue-on-error: true
|
||
run: docker build --pull --no-cache -f {img["dockerfile"]} -t local/{name}:build {img["context"]}"""
|
||
]
|
||
if name in scan_names:
|
||
sev = img.get("trivy_severity", "HIGH,CRITICAL")
|
||
ec = img.get("trivy_exit_code", 0)
|
||
steps.append(f"""\
|
||
- name: Scan {name}
|
||
if: ${{{{ {built} }}}}
|
||
run: trivy image --severity {sev} --exit-code {ec} --format json -o {name}-scan.json local/{name}:build""")
|
||
steps.append(f"""\
|
||
- name: Push {name}
|
||
if: ${{{{ {built} }}}}
|
||
run: |
|
||
docker save local/{name}:build -o /tmp/{name}.tar
|
||
crane push /tmp/{name}.tar {registry}/{owner_repo}/{name}:hardened --insecure
|
||
crane push /tmp/{name}.tar {registry}/{owner_repo}/{name}:latest --insecure""")
|
||
image_steps.append("\n\n".join(steps))
|
||
image_block = "\n\n".join(image_steps)
|
||
|
||
gate_lines = "\n ".join(
|
||
f'[ "${{{{ steps.build_{i["name"].replace("-", "_")}.outcome }}}}" = success ]'
|
||
f' || failed="$failed {i["name"]}"'
|
||
for i in images
|
||
)
|
||
|
||
vuln_files = " ".join(f"{i['name']}-scan.json" for i in scans)
|
||
|
||
content = f"""\
|
||
{_HEADER}
|
||
name: Harden
|
||
|
||
on:
|
||
workflow_dispatch:
|
||
schedule:
|
||
- cron: "0 2 * * 0"
|
||
|
||
jobs:
|
||
build-scan-report:
|
||
runs-on: {runner}
|
||
steps:
|
||
{_checkout_step()}
|
||
|
||
{_docker_login_step(registry)}
|
||
|
||
{_install_trivy_step()}
|
||
|
||
{_setup_uv_step(uv_version)}
|
||
|
||
{image_block}
|
||
|
||
- name: Close resolved or file new vuln issues
|
||
env:
|
||
GITEA_TOKEN: ${{{{ secrets.DEPLOY_TOKEN }}}}
|
||
run: |
|
||
uv sync --no-dev
|
||
for f in {vuln_files}; do
|
||
if [ -f "$f" ]; then
|
||
uv run python -m api.diag.vuln --close "$f" || \\
|
||
uv run python -m api.diag.vuln "$f" || true
|
||
fi
|
||
done
|
||
|
||
- name: Fail if any image failed to build
|
||
run: |
|
||
failed=""
|
||
{gate_lines}
|
||
if [ -n "$failed" ]; then
|
||
echo "Images that failed to build (not scanned, not pushed):$failed"
|
||
exit 1
|
||
fi
|
||
|
||
{_failure_step("Harden", "build-scan-report")}
|
||
"""
|
||
return (".gitea/workflows/harden.yml", content)
|
||
|
||
|
||
def _gen_rebuild_all(
|
||
images: list[dict],
|
||
scans: list[dict],
|
||
registry: str,
|
||
owner_repo: str,
|
||
runner: str,
|
||
uv_version: str,
|
||
**_kw: object,
|
||
) -> tuple[str, str]:
|
||
build_steps = []
|
||
for img in images:
|
||
tags = (
|
||
f"{registry}/{owner_repo}/{img['name']}:${{{{ env.SHORT_SHA }}}},"
|
||
f"{registry}/{owner_repo}/{img['name']}:latest"
|
||
)
|
||
build_steps.append(_build_push_step(img, tags, registry, owner_repo))
|
||
build_block = "\n\n".join(build_steps)
|
||
|
||
scan_steps = []
|
||
for img in scans:
|
||
scan_steps.append(
|
||
_trivy_step(img, "${{ env.SHORT_SHA }}", registry, owner_repo)
|
||
)
|
||
scan_block = "\n\n".join(scan_steps)
|
||
|
||
vuln_files = " ".join(f"{i['name']}-scan.json" for i in scans)
|
||
|
||
content = f"""\
|
||
{_HEADER}
|
||
name: Rebuild All
|
||
|
||
on:
|
||
workflow_dispatch:
|
||
|
||
jobs:
|
||
build-scan-report:
|
||
runs-on: {runner}
|
||
steps:
|
||
{_checkout_step()}
|
||
|
||
{_docker_login_step(registry)}
|
||
|
||
{_install_trivy_step()}
|
||
|
||
{_setup_uv_step(uv_version)}
|
||
|
||
- name: Compute short SHA
|
||
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
||
|
||
{build_block}
|
||
|
||
{scan_block}
|
||
|
||
- name: Report vulnerabilities
|
||
env:
|
||
GITEA_TOKEN: ${{{{ secrets.DEPLOY_TOKEN }}}}
|
||
run: |
|
||
uv sync --no-dev
|
||
for f in {vuln_files}; do
|
||
[ -f "$f" ] && uv run python -m api.diag.vuln "$f" || true
|
||
done
|
||
|
||
{_failure_step("Rebuild All", "build-scan-report")}
|
||
"""
|
||
return (".gitea/workflows/rebuild-all.yml", content)
|
||
|
||
|
||
def _gen_infra_ci(
|
||
images: list[dict],
|
||
runner: str,
|
||
**_kw: object,
|
||
) -> tuple[str, str]:
|
||
all_paths: list[str] = []
|
||
for img in images:
|
||
all_paths.extend(img.get("path_filter", []))
|
||
paths_block = "\n".join(f" - {p!r}" for p in all_paths)
|
||
|
||
jobs_parts = []
|
||
for img in images:
|
||
if not img.get("hadolint", True):
|
||
continue
|
||
name = img["name"]
|
||
# The notebooks image gets a frontend smoke gate: boot the freshly
|
||
# built image and load the editor in a headless browser, failing on
|
||
# any console/page error. Guards against runtime-broken frontend
|
||
# bundles that compile cleanly (the 2026-07-09 "d is not a
|
||
# constructor" incident shipped through a green build).
|
||
fe_smoke_step = ""
|
||
if name == "notebooks":
|
||
fe_smoke_step = f"""
|
||
|
||
- name: Frontend smoke gate
|
||
run: python3 dev/scripts/nb_fe_smoke.py --image local/{name}:build --network ci"""
|
||
jobs_parts.append(f"""\
|
||
{name}:
|
||
runs-on: {runner}
|
||
steps:
|
||
{_checkout_step()}
|
||
|
||
- name: Hadolint {name}
|
||
uses: https://github.com/hadolint/hadolint-action@v3.1.0
|
||
with:
|
||
dockerfile: {img["dockerfile"]}
|
||
|
||
{_setup_buildx_step()}
|
||
|
||
{_build_push_step(img, f"ci-test-{name}", "", "", load_only=True)}{fe_smoke_step}
|
||
|
||
{_failure_step("Infra CI", name)}""")
|
||
|
||
jobs_block = "\n\n".join(jobs_parts)
|
||
|
||
content = f"""\
|
||
{_HEADER}
|
||
name: Infra CI
|
||
|
||
on:
|
||
push:
|
||
paths:
|
||
{paths_block}
|
||
pull_request:
|
||
paths:
|
||
{paths_block}
|
||
|
||
jobs:
|
||
{jobs_block}
|
||
"""
|
||
return (".gitea/workflows/infra-ci.yml", content)
|
||
|
||
|
||
def _gen_notebooks_integration(
|
||
runner: str, uv_version: str, **_kw: object
|
||
) -> tuple[str, str]:
|
||
"""Nightly full-set notebook run inside the production container.
|
||
|
||
The full notebook set needs real data (aco.duckdb, ./data mounts) that
|
||
only the prod container has, so the scripts are docker-cp'd in and run
|
||
there. Failures are routed through nb_issue_filer (dedup + auto-close),
|
||
not exit codes — a red nightly run should page via the issue tracker,
|
||
not accumulate api.diag.ci duplicates. 03:30 sits before the 06:00
|
||
pkg-supply-chain run and outside interactive hours (duckdb lock, #508).
|
||
"""
|
||
content = f"""\
|
||
{_HEADER}
|
||
name: Notebooks Integration
|
||
|
||
on:
|
||
workflow_dispatch:
|
||
schedule:
|
||
- cron: "30 3 * * *"
|
||
|
||
jobs:
|
||
notebooks-integration:
|
||
runs-on: {runner}
|
||
steps:
|
||
{_checkout_step()}
|
||
|
||
{_setup_uv_step(uv_version)}
|
||
|
||
- name: Run full notebook set in prod container
|
||
env:
|
||
GITEA_TOKEN: ${{{{ secrets.DEPLOY_TOKEN }}}}
|
||
run: |
|
||
set -euo pipefail
|
||
docker exec notebooks mkdir -p /tmp/nbtest
|
||
docker cp dev/scripts/nb_integration.py notebooks:/tmp/nbtest/
|
||
docker cp dev/scripts/nb_issue_filer.py notebooks:/tmp/nbtest/
|
||
docker cp infra/marimo/nb-tests.toml notebooks:/tmp/nbtest/
|
||
docker exec \\
|
||
-e GITEA_TOKEN \\
|
||
-e GITEA_API_BASE=http://git:3000/api/v1 \\
|
||
notebooks \\
|
||
uv run --project /home/kert/workspace python /tmp/nbtest/nb_integration.py \\
|
||
--set all --file-issues \\
|
||
--nb-dir /home/kert/notebooks \\
|
||
--config /tmp/nbtest/nb-tests.toml \\
|
||
--report /tmp/nbtest/report.json
|
||
docker cp notebooks:/tmp/nbtest/report.json nb-integration-report.json
|
||
cat nb-integration-report.json
|
||
|
||
- name: Frontend smoke against live service
|
||
run: python3 dev/scripts/nb_fe_smoke.py --url http://notebooks:2718 --network gateway
|
||
|
||
{_failure_step("Notebooks Integration", "notebooks-integration")}
|
||
"""
|
||
return (".gitea/workflows/notebooks-integration.yml", content)
|
||
|
||
|
||
def _gen_zotero_sync(runner: str, uv_version: str, **_kw: object) -> tuple[str, str]:
|
||
"""Nightly bib → Zotero routing for mail-ingested items.
|
||
|
||
mail-poller ingests CMS mail into bib continuously, but Zotero only
|
||
sees those items when `stack bib sync-zotero` pushes them — and that
|
||
had no scheduler, so the Cmsupdates collection silently froze for two
|
||
months (Apr 29 → Jul 10, 212-item backlog). sync writes directly into
|
||
zotero.sqlite, so the running Zotero app must be stopped for the
|
||
duration; the stop/start lives in workflow steps (visible, and
|
||
restart is if:always()) rather than the CLI's --hold-zotero, because
|
||
the sync runs docker-exec'd inside the api container, which has no
|
||
docker socket. 04:15 sits after the 03:30 notebooks-integration run
|
||
so the two never contend for zotero.sqlite.
|
||
"""
|
||
content = f"""\
|
||
{_HEADER}
|
||
name: Zotero Sync
|
||
|
||
on:
|
||
workflow_dispatch:
|
||
schedule:
|
||
- cron: "15 4 * * *"
|
||
|
||
jobs:
|
||
zotero-sync:
|
||
runs-on: {runner}
|
||
steps:
|
||
{_checkout_step()}
|
||
|
||
{_setup_uv_step(uv_version)}
|
||
|
||
- name: Stop zotero (release SQLite write lock)
|
||
run: docker stop zotero
|
||
|
||
- name: Sync mail-ingested bib items into Zotero
|
||
# --merge-dupes: listserv re-sends are now aliased at ingest
|
||
# (Task 3), and any same-URL twins that still land in Zotero are
|
||
# merged (union tags/collections/notes/attachments, keep-earliest)
|
||
# rather than deleted.
|
||
run: |
|
||
docker exec api uv run --no-sync \\
|
||
stack bib sync-zotero --tag source:email --no-hold \\
|
||
--merge-dupes
|
||
|
||
- name: Sync Federal Register rule items into Zotero
|
||
# Rule items only ever reached Zotero via one-off manual syncs
|
||
# (P36 step 5); P37 skipped that step and the CY2027 NPRM never
|
||
# arrived (#626/#627). Tag-scoped: a full unscoped sync would
|
||
# hydrate ~180k bib items.
|
||
run: |
|
||
docker exec api uv run --no-sync \\
|
||
stack bib sync-zotero --tag source:federal-register --no-hold
|
||
|
||
- name: Restart zotero
|
||
if: always()
|
||
run: docker start zotero
|
||
|
||
{_failure_step("Zotero Sync", "zotero-sync")}
|
||
"""
|
||
return (".gitea/workflows/zotero-sync.yml", content)
|
||
|
||
|
||
def _gen_llm_golden(runner: str, uv_version: str, **_kw: object) -> tuple[str, str]:
|
||
"""Nightly golden longitudinal evaluation of the chat (P49 Task 7).
|
||
|
||
Mirrors ``_gen_notebooks_integration``: the golden set needs a live
|
||
chat over the real pgvector/DuckDB/Ollama stack, which only the
|
||
``llm`` container (``uvicorn llm.api:app``, port 8000, no published
|
||
port) has — so the runner and the golden set are docker-cp'd in and
|
||
run there against its own ``http://localhost:8000``. Regressions
|
||
are filed through ``nb_issue_filer`` (dedup + auto-close) under the
|
||
``llm`` label, not ``notebooks``. 03:40 sits after the 03:30
|
||
notebooks-integration run and before the 04:15 zotero-sync run.
|
||
"""
|
||
content = f"""\
|
||
{_HEADER}
|
||
name: LLM Golden
|
||
|
||
on:
|
||
workflow_dispatch:
|
||
schedule:
|
||
- cron: "40 3 * * *"
|
||
|
||
jobs:
|
||
llm-golden:
|
||
runs-on: {runner}
|
||
steps:
|
||
{_checkout_step()}
|
||
|
||
{_setup_uv_step(uv_version)}
|
||
|
||
- name: Run golden longitudinal evaluation against the live chat
|
||
# golden_rc is captured (not left to `set -e`) so a failing golden
|
||
# run still docker-cps the report back and cats it — llm_golden.py
|
||
# itself exits 0 once --file-issues has filed the regressions
|
||
# (nb_issue_filer is the failure channel here), but this still
|
||
# guards the case where it exits nonzero before reaching that
|
||
# point (e.g. a crash while loading the golden set).
|
||
env:
|
||
GITEA_TOKEN: ${{{{ secrets.DEPLOY_TOKEN }}}}
|
||
run: |
|
||
set -euo pipefail
|
||
docker exec llm mkdir -p /tmp/golden
|
||
docker cp dev/scripts/llm_golden.py llm:/tmp/golden/
|
||
docker cp dev/scripts/nb_issue_filer.py llm:/tmp/golden/
|
||
docker cp tests/llm/golden_lineage.yaml llm:/tmp/golden/
|
||
docker exec \\
|
||
-e GITEA_TOKEN \\
|
||
-e GITEA_API_BASE=http://git:3000/api/v1 \\
|
||
-e NB_ISSUE_LABEL=llm \\
|
||
llm \\
|
||
uv run --no-sync --project /app python /tmp/golden/llm_golden.py run \\
|
||
--url http://localhost:8000 \\
|
||
--set /tmp/golden/golden_lineage.yaml \\
|
||
--report /tmp/golden/report.json \\
|
||
--file-issues --source nightly-llm-golden || golden_rc=$?
|
||
docker cp llm:/tmp/golden/report.json llm-golden-report.json || true
|
||
cat llm-golden-report.json || true
|
||
exit "${{golden_rc:-0}}"
|
||
|
||
{_failure_step("LLM Golden", "llm-golden")}
|
||
"""
|
||
return (".gitea/workflows/llm-golden.yml", content)
|
||
|
||
|
||
def _gen_release(runner: str, uv_version: str, **_kw: object) -> tuple[str, str]:
|
||
content = f"""\
|
||
{_HEADER}
|
||
name: Release
|
||
|
||
on:
|
||
push:
|
||
tags: ["v*"]
|
||
|
||
jobs:
|
||
release:
|
||
runs-on: {runner}
|
||
steps:
|
||
{_checkout_step()}
|
||
|
||
{_setup_uv_step(uv_version)}
|
||
|
||
- name: Build package
|
||
run: uv build --out-dir dist/
|
||
|
||
- name: Create release
|
||
uses: https://github.com/softprops/action-gh-release@v2
|
||
with:
|
||
files: |
|
||
dist/*.whl
|
||
dist/*.tar.gz
|
||
|
||
{_failure_step("Release", "release")}
|
||
"""
|
||
return (".gitea/workflows/release.yml", content)
|
||
|
||
|
||
# ── Public API ────────────────────────────────────────────────────
|
||
|
||
|
||
def emit(
|
||
images: list[dict],
|
||
scans: list[dict],
|
||
platform: dict,
|
||
ci_cfg: dict,
|
||
) -> dict[str, str]:
|
||
"""Return {relative_path: content} for all Gitea Actions workflows."""
|
||
# ci_registry is the internal HTTP endpoint used for push (avoids TLS)
|
||
registry = ci_cfg.get("ci_registry", platform["registry"])
|
||
owner_repo = platform["repo"]
|
||
runner = ci_cfg.get("runner_labels", ["ubuntu-latest"])[0]
|
||
uv_version = ci_cfg.get("uv_version", "latest")
|
||
|
||
common = dict(
|
||
images=images,
|
||
scans=scans,
|
||
registry=registry,
|
||
owner_repo=owner_repo,
|
||
runner=runner,
|
||
uv_version=uv_version,
|
||
coverage_threshold=ci_cfg.get("coverage_threshold", 99),
|
||
)
|
||
|
||
files: dict[str, str] = {}
|
||
for gen_fn in (
|
||
_gen_ci,
|
||
_gen_deploy,
|
||
_gen_harden,
|
||
_gen_rebuild_all,
|
||
_gen_infra_ci,
|
||
_gen_notebooks_integration,
|
||
_gen_zotero_sync,
|
||
_gen_llm_golden,
|
||
_gen_release,
|
||
):
|
||
path, content = gen_fn(**common) # type: ignore[arg-type]
|
||
files[path] = content
|
||
|
||
return files
|