Some checks failed
CI / lint (push) Successful in 38s
CI / notebooks-smoke (push) Successful in 2m9s
Deploy / changes (push) Successful in 16s
Infra CI / notebooks (push) Successful in 59s
CI / test (push) Failing after 4m13s
Infra CI / zotero (push) Successful in 18s
Infra CI / docs (push) Successful in 1m12s
Infra CI / api (push) Successful in 1m17s
Infra CI / llm (push) Successful in 1m9s
Infra CI / mc (push) Successful in 14s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Infra CI / bcda (push) Successful in 1m58s
Deploy / api (push) Successful in 1m23s
Deploy / bcda (push) Successful in 1m37s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 1m5s
Deploy / report (push) Successful in 14s
api.diag.ci opened a new issue on every red run and never closed it. A repeat now comments on the open issue and closes older copies. A green run closes that issue. The workflow step no longer syncs the repo just to file or close.
679 lines
16 KiB
Python
679 lines
16 KiB
Python
"""GitHub Actions workflow emitter.
|
|
|
|
Generates .github/workflows/*.yml from the same image definitions
|
|
used by the Gitea Actions backend. All output is valid GitHub Actions YAML.
|
|
|
|
Usage (called by gen_config.py)::
|
|
|
|
from backends.github import emit
|
|
files = emit(images, scans, platform, ci_cfg)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
# ── Constants ─────────────────────────────────────────────────────
|
|
|
|
_HEADER = (
|
|
"# DO NOT EDIT — generated by gen_config.py from stack.toml\n"
|
|
"# Re-generate: uv run python dev/scripts/gen_config.py\n"
|
|
)
|
|
|
|
_SHA_SHORT = 'echo "${GITHUB_SHA::8}"'
|
|
|
|
|
|
# ── Helpers ───────────────────────────────────────────────────────
|
|
|
|
|
|
def _indent(text: str, spaces: int = 0) -> str:
|
|
"""Indent every line of *text* by *spaces*."""
|
|
prefix = " " * spaces
|
|
return "\n".join(prefix + line if line else "" for line in text.splitlines())
|
|
|
|
|
|
def _trivy_step(
|
|
img: dict,
|
|
tag: str,
|
|
registry: str,
|
|
owner_repo: str,
|
|
) -> str:
|
|
"""aquasecurity/trivy-action step for one image."""
|
|
name = img["name"]
|
|
sev = img.get("trivy_severity", "HIGH,CRITICAL")
|
|
ec = img.get("trivy_exit_code", 0)
|
|
return f"""\
|
|
- name: Scan {name}
|
|
uses: aquasecurity/trivy-action@master
|
|
with:
|
|
image-ref: {registry}/{owner_repo}/{name}:{tag}
|
|
severity: {sev}
|
|
exit-code: "{ec}"
|
|
format: json
|
|
output: {name}-scan.json"""
|
|
|
|
|
|
def _docker_login_step(registry: str) -> str:
|
|
return f"""\
|
|
- name: Log in to container registry
|
|
uses: docker/login-action@v3
|
|
with:
|
|
registry: {registry}
|
|
username: ${{{{ github.actor }}}}
|
|
password: ${{{{ secrets.GITHUB_TOKEN }}}}"""
|
|
|
|
|
|
def _checkout_step() -> str:
|
|
return """\
|
|
- name: Checkout
|
|
uses: actions/checkout@v4"""
|
|
|
|
|
|
def _setup_buildx_step() -> str:
|
|
return """\
|
|
- name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v3"""
|
|
|
|
|
|
def _setup_uv_step(uv_version: str) -> str:
|
|
return f"""\
|
|
- name: Set up uv
|
|
uses: astral-sh/setup-uv@v4
|
|
with:
|
|
version: {uv_version}"""
|
|
|
|
|
|
def _build_push_step(
|
|
img: dict,
|
|
tags_expr: str,
|
|
registry: str,
|
|
owner_repo: str,
|
|
*,
|
|
no_cache: bool = False,
|
|
load_only: bool = False,
|
|
) -> str:
|
|
name = img["name"]
|
|
push = "false" if load_only else "true"
|
|
lines = f"""\
|
|
- name: Build {name}
|
|
uses: docker/build-push-action@v6
|
|
with:
|
|
context: {img["context"]}
|
|
file: {img["dockerfile"]}
|
|
push: {push}
|
|
tags: {tags_expr}"""
|
|
if load_only:
|
|
lines += "\n load: true"
|
|
if no_cache:
|
|
lines += "\n no-cache: true"
|
|
return lines
|
|
|
|
|
|
# ── Failure reporting ─────────────────────────────────────────────
|
|
|
|
|
|
def _filer_run(workflow_name: str, job_name: str, *, close: bool = False) -> str:
|
|
"""Shell that runs the filer. python3 is on the runner; no uv sync."""
|
|
flag = "--close \\\n " if close else ""
|
|
return f"""\
|
|
py=python3
|
|
command -v python3 >/dev/null 2>&1 || {{ command -v uv >/dev/null 2>&1 || curl -LsSf https://astral.sh/uv/install.sh | sh; export PATH="$HOME/.local/bin:$PATH"; py="uv run --no-project python"; }}
|
|
$py src/api/diag/ci.py \\
|
|
{flag}--workflow "{workflow_name}" --job "{job_name}" \\
|
|
--run "${{{{ github.run_number }}}}" \\
|
|
--sha "${{{{ github.sha }}}}" \\
|
|
--ref "${{{{ github.ref }}}}" || true"""
|
|
|
|
|
|
def _failure_step(workflow_name: str, job_name: str) -> str:
|
|
"""Emit file-on-failure and close-on-success steps."""
|
|
return f"""\
|
|
- name: File failure issue
|
|
if: failure()
|
|
env:
|
|
GITEA_TOKEN: ${{{{ secrets.GITEA_TOKEN }}}}
|
|
run: |
|
|
{_filer_run(workflow_name, job_name)}
|
|
- name: Close resolved issue
|
|
if: success()
|
|
env:
|
|
GITEA_TOKEN: ${{{{ secrets.GITEA_TOKEN }}}}
|
|
run: |
|
|
{_filer_run(workflow_name, job_name, close=True)}"""
|
|
|
|
|
|
# ── Workflow generators ───────────────────────────────────────────
|
|
|
|
|
|
def _gen_ci(
|
|
runner: str, uv_version: str, coverage_threshold: int = 99, **_kw: object
|
|
) -> tuple[str, str]:
|
|
content = f"""\
|
|
{_HEADER}
|
|
name: CI
|
|
|
|
on:
|
|
push:
|
|
branches: ["**"]
|
|
pull_request:
|
|
|
|
jobs:
|
|
lint-test:
|
|
runs-on: {runner}
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
{_setup_uv_step(uv_version)}
|
|
|
|
- name: Install dependencies
|
|
run: uv sync --dev
|
|
|
|
- name: Ruff check
|
|
run: uv run ruff check src/ tests/ --output-format=concise
|
|
|
|
- name: Ruff format
|
|
run: uv run ruff format --check src/ tests/
|
|
|
|
- name: Pytest
|
|
# -n auto parallelizes across runner cores; pytest-cov combines
|
|
# results via [tool.coverage.run] parallel=true in pyproject.toml.
|
|
run: uv run pytest tests/ --cov=src --cov-report=term-missing --cov-fail-under={coverage_threshold} -q -n auto
|
|
|
|
- name: Validate generated config
|
|
run: uv run python dev/scripts/gen_config.py --check
|
|
|
|
{_failure_step("CI", "lint-test")}
|
|
|
|
skinny-install:
|
|
runs-on: {runner}
|
|
strategy:
|
|
matrix:
|
|
extra: [conf, aco, api, bcda, bib, bls, ccw, cli, cms, opps, perf, pfs, rex]
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
{_setup_uv_step(uv_version)}
|
|
|
|
- name: Install stack[${{{{ matrix.extra }}}}]
|
|
run: uv sync --no-dev --extra ${{{{ matrix.extra }}}}
|
|
|
|
- name: Verify import
|
|
run: uv run python -c "import ${{{{ matrix.extra }}}}"
|
|
|
|
- name: Run module tests
|
|
run: |
|
|
if [ -d "tests/${{{{ matrix.extra }}}}" ]; then
|
|
uv run pytest "tests/${{{{ matrix.extra }}}}/" -x -q || true
|
|
fi
|
|
|
|
{_failure_step("CI", "skinny-install")}
|
|
"""
|
|
return (".github/workflows/ci.yml", content)
|
|
|
|
|
|
def _gen_deploy(
|
|
images: list[dict],
|
|
scans: list[dict],
|
|
registry: str,
|
|
owner_repo: str,
|
|
runner: str,
|
|
uv_version: str,
|
|
**_kw: object,
|
|
) -> tuple[str, str]:
|
|
# Matrix include list
|
|
matrix_entries = []
|
|
for img in images:
|
|
matrix_entries.append(
|
|
f" - name: {img['name']}\n"
|
|
f" dockerfile: {img['dockerfile']}\n"
|
|
f" context: {img['context']}"
|
|
)
|
|
matrix_block = "\n".join(matrix_entries)
|
|
|
|
# Scan steps (only for scannable images)
|
|
scan_steps = []
|
|
for img in scans:
|
|
scan_steps.append(
|
|
_trivy_step(
|
|
img,
|
|
"${{ env.SHORT_SHA }}",
|
|
registry,
|
|
owner_repo,
|
|
)
|
|
)
|
|
scan_block = "\n\n".join(scan_steps)
|
|
|
|
# Report job vuln loop
|
|
vuln_loop_files = " ".join(f"{i['name']}-scan.json" for i in scans)
|
|
|
|
content = f"""\
|
|
{_HEADER}
|
|
name: Deploy
|
|
|
|
on:
|
|
push:
|
|
branches: [main]
|
|
|
|
jobs:
|
|
build-scan:
|
|
runs-on: {runner}
|
|
strategy:
|
|
matrix:
|
|
include:
|
|
{matrix_block}
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
{_setup_buildx_step()}
|
|
|
|
{_docker_login_step(registry)}
|
|
|
|
- name: Compute short SHA
|
|
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
|
|
|
{
|
|
_build_push_step(
|
|
{
|
|
"name": "${{ matrix.name }}",
|
|
"dockerfile": "${{ matrix.dockerfile }}",
|
|
"context": "${{ matrix.context }}",
|
|
},
|
|
"{registry}/{owner_repo}/${{{{ matrix.name }}}}:${{{{ env.SHORT_SHA }}}},{registry}/{owner_repo}/${{{{ matrix.name }}}}:latest".format(
|
|
registry=registry, owner_repo=owner_repo
|
|
),
|
|
registry,
|
|
owner_repo,
|
|
)
|
|
}
|
|
|
|
{_failure_step("Deploy", "build-scan")}
|
|
|
|
scan:
|
|
runs-on: {runner}
|
|
needs: build-scan
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
- name: Compute short SHA
|
|
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
|
|
|
{scan_block}
|
|
|
|
- name: Upload scan results
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: trivy-scans
|
|
path: "*-scan.json"
|
|
|
|
{_failure_step("Deploy", "scan")}
|
|
|
|
report-vulns:
|
|
runs-on: {runner}
|
|
needs: scan
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
{_setup_uv_step(uv_version)}
|
|
|
|
- name: Download scan results
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: trivy-scans
|
|
|
|
- name: Report vulnerabilities
|
|
env:
|
|
GITEA_TOKEN: ${{{{ secrets.GITEA_TOKEN }}}}
|
|
run: |
|
|
uv sync --no-dev
|
|
for f in {vuln_loop_files}; do
|
|
[ -f "$f" ] && uv run python -m api.diag.vuln "$f" || true
|
|
done
|
|
|
|
{_failure_step("Deploy", "report-vulns")}
|
|
"""
|
|
return (".github/workflows/deploy.yml", content)
|
|
|
|
|
|
def _gen_harden(
|
|
images: list[dict],
|
|
scans: list[dict],
|
|
registry: str,
|
|
owner_repo: str,
|
|
runner: str,
|
|
uv_version: str,
|
|
**_kw: object,
|
|
) -> tuple[str, str]:
|
|
# Build steps
|
|
build_steps = []
|
|
for img in images:
|
|
tags = f"{registry}/{owner_repo}/{img['name']}:hardened,{registry}/{owner_repo}/{img['name']}:latest"
|
|
build_steps.append(
|
|
_build_push_step(
|
|
img,
|
|
tags,
|
|
registry,
|
|
owner_repo,
|
|
no_cache=True,
|
|
)
|
|
)
|
|
build_block = "\n\n".join(build_steps)
|
|
|
|
# Scan steps
|
|
scan_steps = []
|
|
for img in scans:
|
|
scan_steps.append(_trivy_step(img, "hardened", registry, owner_repo))
|
|
scan_block = "\n\n".join(scan_steps)
|
|
|
|
vuln_loop_files = " ".join(f"{i['name']}-scan.json" for i in scans)
|
|
|
|
content = f"""\
|
|
{_HEADER}
|
|
name: Harden
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
schedule:
|
|
- cron: "0 2 * * 0"
|
|
|
|
jobs:
|
|
build-scan:
|
|
runs-on: {runner}
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
{_setup_buildx_step()}
|
|
|
|
{_docker_login_step(registry)}
|
|
|
|
{build_block}
|
|
|
|
{_failure_step("Harden", "build-scan")}
|
|
|
|
scan:
|
|
runs-on: {runner}
|
|
needs: build-scan
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
{scan_block}
|
|
|
|
- name: Upload scan results
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: trivy-scans-harden
|
|
path: "*-scan.json"
|
|
|
|
{_failure_step("Harden", "scan")}
|
|
|
|
close-or-report-vulns:
|
|
runs-on: {runner}
|
|
needs: scan
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
{_setup_uv_step(uv_version)}
|
|
|
|
- name: Download scan results
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: trivy-scans-harden
|
|
|
|
- name: Close resolved or file new vuln issues
|
|
env:
|
|
GITEA_TOKEN: ${{{{ secrets.GITEA_TOKEN }}}}
|
|
run: |
|
|
uv sync --no-dev
|
|
for f in {vuln_loop_files}; do
|
|
if [ -f "$f" ]; then
|
|
uv run python -m api.diag.vuln --close "$f" || \\
|
|
uv run python -m api.diag.vuln "$f" || true
|
|
fi
|
|
done
|
|
|
|
{_failure_step("Harden", "close-or-report-vulns")}
|
|
"""
|
|
return (".github/workflows/harden.yml", content)
|
|
|
|
|
|
def _gen_rebuild_all(
|
|
images: list[dict],
|
|
scans: list[dict],
|
|
registry: str,
|
|
owner_repo: str,
|
|
runner: str,
|
|
uv_version: str,
|
|
**_kw: object,
|
|
) -> tuple[str, str]:
|
|
# Build steps
|
|
build_steps = []
|
|
for img in images:
|
|
tags = f"{registry}/{owner_repo}/{img['name']}:${{{{ env.SHORT_SHA }}}},{registry}/{owner_repo}/{img['name']}:latest"
|
|
build_steps.append(_build_push_step(img, tags, registry, owner_repo))
|
|
build_block = "\n\n".join(build_steps)
|
|
|
|
# Scan steps
|
|
scan_steps = []
|
|
for img in scans:
|
|
scan_steps.append(
|
|
_trivy_step(
|
|
img,
|
|
"${{ env.SHORT_SHA }}",
|
|
registry,
|
|
owner_repo,
|
|
)
|
|
)
|
|
scan_block = "\n\n".join(scan_steps)
|
|
|
|
vuln_loop_files = " ".join(f"{i['name']}-scan.json" for i in scans)
|
|
|
|
content = f"""\
|
|
{_HEADER}
|
|
name: Rebuild All
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
|
|
jobs:
|
|
build:
|
|
runs-on: {runner}
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
{_setup_buildx_step()}
|
|
|
|
{_docker_login_step(registry)}
|
|
|
|
- name: Compute short SHA
|
|
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
|
|
|
{build_block}
|
|
|
|
{_failure_step("Rebuild All", "build")}
|
|
|
|
scan:
|
|
runs-on: {runner}
|
|
needs: build
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
- name: Compute short SHA
|
|
run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"
|
|
|
|
{scan_block}
|
|
|
|
- name: Upload scan results
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: trivy-scans-rebuild
|
|
path: "*-scan.json"
|
|
|
|
{_failure_step("Rebuild All", "scan")}
|
|
|
|
report-vulns:
|
|
runs-on: {runner}
|
|
needs: scan
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
{_setup_uv_step(uv_version)}
|
|
|
|
- name: Download scan results
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: trivy-scans-rebuild
|
|
|
|
- name: Report vulnerabilities
|
|
env:
|
|
GITEA_TOKEN: ${{{{ secrets.GITEA_TOKEN }}}}
|
|
run: |
|
|
uv sync --no-dev
|
|
for f in {vuln_loop_files}; do
|
|
[ -f "$f" ] && uv run python -m api.diag.vuln "$f" || true
|
|
done
|
|
|
|
{_failure_step("Rebuild All", "report-vulns")}
|
|
"""
|
|
return (".github/workflows/rebuild-all.yml", content)
|
|
|
|
|
|
def _gen_infra_ci(
|
|
images: list[dict],
|
|
runner: str,
|
|
**_kw: object,
|
|
) -> tuple[str, str]:
|
|
# Collect all path filters for global trigger
|
|
all_paths: list[str] = []
|
|
for img in images:
|
|
all_paths.extend(img.get("path_filter", []))
|
|
|
|
paths_block = "\n".join(f" - {p!r}" for p in all_paths) if all_paths else ""
|
|
|
|
# Per-image hadolint + dry-run build jobs
|
|
jobs_block_parts = []
|
|
for img in images:
|
|
if not img.get("hadolint", True):
|
|
continue
|
|
name = img["name"]
|
|
pf = img.get("path_filter", []) # noqa: F841 — reserved for future per-job filtering
|
|
|
|
jobs_block_parts.append(f"""\
|
|
{name}:
|
|
runs-on: {runner}
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
- name: Hadolint {name}
|
|
uses: hadolint/hadolint-action@v3.1.0
|
|
with:
|
|
dockerfile: {img["dockerfile"]}
|
|
|
|
{_setup_buildx_step()}
|
|
|
|
{_build_push_step(img, f"ci-test-{name}", "", "", load_only=True)}
|
|
|
|
{_failure_step("Infra CI", name)}""")
|
|
|
|
jobs_block = "\n\n".join(jobs_block_parts)
|
|
|
|
content = f"""\
|
|
{_HEADER}
|
|
name: Infra CI
|
|
|
|
on:
|
|
push:
|
|
paths:
|
|
{paths_block}
|
|
pull_request:
|
|
paths:
|
|
{paths_block}
|
|
|
|
jobs:
|
|
{jobs_block}
|
|
"""
|
|
return (".github/workflows/infra-ci.yml", content)
|
|
|
|
|
|
def _gen_release(runner: str, uv_version: str, **_kw: object) -> tuple[str, str]:
|
|
content = f"""\
|
|
{_HEADER}
|
|
name: Release
|
|
|
|
on:
|
|
push:
|
|
tags: ["v*"]
|
|
|
|
jobs:
|
|
release:
|
|
runs-on: {runner}
|
|
permissions:
|
|
contents: write
|
|
steps:
|
|
{_checkout_step()}
|
|
|
|
{_setup_uv_step(uv_version)}
|
|
|
|
- name: Build package
|
|
run: uv build --out-dir dist/
|
|
|
|
- name: Create GitHub release
|
|
uses: softprops/action-gh-release@v2
|
|
with:
|
|
files: |
|
|
dist/*.whl
|
|
dist/*.tar.gz
|
|
|
|
{_failure_step("Release", "release")}
|
|
"""
|
|
return (".github/workflows/release.yml", content)
|
|
|
|
|
|
# ── Public API ────────────────────────────────────────────────────
|
|
|
|
|
|
def emit(
|
|
images: list[dict],
|
|
scans: list[dict],
|
|
platform: dict,
|
|
ci_cfg: dict,
|
|
) -> dict[str, str]:
|
|
"""Return {relative_path: content} for all GitHub Actions workflows.
|
|
|
|
Parameters
|
|
----------
|
|
images:
|
|
Merged image dicts from ``load_images()``.
|
|
scans:
|
|
Scannable subset (``scan=True``).
|
|
platform:
|
|
Dict with keys *registry*, *domain*, *host_ip*, *image_prefix*,
|
|
*repo* (``"org/name"`` form).
|
|
ci_cfg:
|
|
Dict from ``cfg.ci.github`` — *runner*, *registry*, *uv_version*.
|
|
"""
|
|
registry = ci_cfg["registry"]
|
|
owner_repo = platform["repo"]
|
|
runner = ci_cfg["runner"]
|
|
uv_version = ci_cfg["uv_version"]
|
|
|
|
common = dict(
|
|
images=images,
|
|
scans=scans,
|
|
registry=registry,
|
|
owner_repo=owner_repo,
|
|
runner=runner,
|
|
uv_version=uv_version,
|
|
coverage_threshold=ci_cfg.get("coverage_threshold", 99),
|
|
)
|
|
|
|
files: dict[str, str] = {}
|
|
for gen_fn in (
|
|
_gen_ci,
|
|
_gen_deploy,
|
|
_gen_harden,
|
|
_gen_rebuild_all,
|
|
_gen_infra_ci,
|
|
_gen_release,
|
|
):
|
|
path, content = gen_fn(**common) # type: ignore[arg-type]
|
|
files[path] = content
|
|
|
|
return files
|