stack/dev/scripts/backends/gitea.py

"""Gitea Actions workflow emitter.

Thin adapter over the GitHub backend — Gitea Actions uses the same
YAML syntax but with:
  - Output to .gitea/workflows/ (not .github/workflows/)
  - Registry auth via PAT secret (GITEA_TOKEN lacks package scope)
  - Platform registry (not ghcr.io)
  - Full URL form for actions (for air-gapped installs)
"""

from __future__ import annotations

from backends.github import (
    _HEADER,
)

# ── Override action refs with full GitHub URLs ───────────────────
# Gitea Actions resolves short refs (e.g. actions/checkout@v4) against
# the local Gitea instance. Use full URLs to pull from GitHub.


def _checkout_step() -> str:
    return """\
      - name: Checkout
        uses: https://github.com/actions/checkout@v4"""


def _setup_buildx_step() -> str:
    return ""  # Not needed — we use plain docker build/push


def _setup_uv_step(uv_version: str) -> str:
    # astral-sh/setup-uv is a composite action that calls the Gitea API
    # internally, which fails because 'astral-sh' isn't a local user.
    # Install uv directly via the official installer script instead.
    return """\
      - name: Set up uv
        run: curl -LsSf https://astral.sh/uv/install.sh | sh
        env:
          UV_INSTALL_DIR: /usr/local/bin"""


def _build_push_step(
    img: dict,
    tags_expr: str,
    registry: str,
    owner_repo: str,
    *,
    no_cache: bool = False,
    load_only: bool = False,
) -> str:
    """docker build (via socket) + crane push (daemonless, uses container DNS).

    docker push goes through the host daemon which can't resolve 'gitea'.
    crane pushes directly from the job container's network stack.
    """
    name = img["name"]
    tags = [t.strip() for t in tags_expr.split(",") if t.strip()]
    local_tag = f"local/{name}:build"
    cache_flag = " --no-cache" if no_cache else ""
    lines = f"""\
      - name: Build {name}
        run: docker build{cache_flag} -f {img["dockerfile"]} -t {local_tag} {img["context"]}"""
    if not load_only and tags:
        # Save to tarball, then crane push each tag (daemonless)
        push_cmds = "\n          ".join(
            f"crane push /tmp/{name}.tar {t} --insecure" for t in tags
        )
        lines += f"""

      - name: Push {name}
        run: |
          docker save {local_tag} -o /tmp/{name}.tar
          {push_cmds}"""
    return lines


def _install_trivy_step() -> str:
    """Install trivy as a native binary — no Docker-in-Docker needed."""
    return """\
      - name: Install trivy
        run: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin"""


def _trivy_step(
    img: dict,
    tag: str,
    registry: str,
    owner_repo: str,
) -> str:
    """Scan local Docker image — no registry pull, no DNS resolution."""
    name = img["name"]
    local_tag = f"local/{name}:build"
    sev = img.get("trivy_severity", "HIGH,CRITICAL")
    ec = img.get("trivy_exit_code", 0)
    return f"""\
      - name: Scan {name}
        run: trivy image --severity {sev} --exit-code {ec} --format json -o {name}-scan.json {local_tag}"""


# ── Failure reporting ─────────────────────────────────────────────


def _failure_step(workflow_name: str, job_name: str) -> str:
    """Emit an if:failure() step that files a Gitea issue."""
    return f"""\
      - name: File failure issue
        if: failure()
        env:
          GITEA_TOKEN: ${{{{ secrets.DEPLOY_TOKEN }}}}
        run: |
          uv sync --no-dev --quiet 2>/dev/null || true
          uv run python -m api.diag.ci \\
            --workflow "{workflow_name}" --job "{job_name}" \\
            --run "${{{{ github.run_number }}}}" \\
            --sha "${{{{ github.sha }}}}" \\
            --ref "${{{{ github.ref }}}}" || true"""


# ── Gitea-specific helpers ───────────────────────────────────────


def _docker_login_step(registry: str) -> str:
    """Install crane and authenticate — daemonless push tool."""
    return f"""\
      - name: Install crane
        run: curl -sL https://github.com/google/go-containerregistry/releases/latest/download/go-containerregistry_Linux_x86_64.tar.gz | tar xz -C /usr/local/bin crane

      - name: Log in to registry
        run: crane auth login {registry} -u "${{{{ secrets.REGISTRY_USER }}}}" -p "${{{{ secrets.REGISTRY_TOKEN }}}}"
        env:
          CRANE_INSECURE: "true\""""


# ── Workflow generators ──────────────────────────────────────────


def _gen_ci(
    runner: str, uv_version: str, coverage_threshold: int = 99, **_kw: object
) -> tuple[str, str]:
    content = f"""\
{_HEADER}
name: CI

on:
  push:
    branches: ["**"]
  pull_request:

jobs:
  lint:
    runs-on: {runner}
    steps:
{_checkout_step()}

{_setup_uv_step(uv_version)}

      - name: Install dependencies
        run: uv sync --dev

      - name: Ruff check
        run: uv run ruff check src/ tests/ --output-format=concise

      - name: Ruff format
        run: uv run ruff format --check src/ tests/

      - name: Validate generated config
        run: uv run python dev/scripts/gen_config.py --check

  test:
    runs-on: {runner}
    steps:
{_checkout_step()}

{_setup_uv_step(uv_version)}

      - name: Install dependencies
        run: uv sync --dev

      - name: Preinstall duckdb extensions
        # tests/zot/test_duck.py runs INSTALL sqlite from every xdist
        # worker; concurrent installs race the extension-file rename in
        # ~/.duckdb ("Could not remove file ... sqlite_scanner", #515).
        # Installing once up front makes the in-test INSTALL a no-op.
        # ducklake: same race class, used by tests/aco DuckLakeContext.
        run: uv run python -c "import duckdb; duckdb.connect().execute('INSTALL sqlite; INSTALL ducklake')"

      - name: Pytest
        # -n auto parallelizes across runner cores. Coverage combining
        # is configured via [tool.coverage.run] parallel=true in
        # pyproject.toml — without that the per-worker .coverage.* files
        # don't merge reliably (see #388 for the prior revert).
        run: uv run pytest tests/ -x --cov=src --cov-report=term-missing --cov-fail-under={coverage_threshold} -q -n auto

{_failure_step("CI", "test")}

  notebooks-smoke:
    runs-on: {runner}
    steps:
{_checkout_step()}

{_setup_uv_step(uv_version)}

      - name: Install dependencies
        run: uv sync --dev

      - name: Stub aco.duckdb
        # The checkout has no data volume; conf.connect.duckdb() opens
        # data/aco.duckdb read-only and fails if it doesn't exist. An
        # empty-but-valid database is enough for the ci-smoke notebooks
        # (schema browsing over information_schema returns zero rows).
        run: |
          [ -f data/aco.duckdb ] || uv run python -c \\
            "import duckdb; duckdb.connect('data/aco.duckdb').close()"

      - name: Run data-independent notebooks headless
        # Executes the [ci_smoke] set from infra/marimo/nb-tests.toml via
        # `marimo export session` and fails on any cell error. Runs on
        # every push (not path-gated): notebooks import src/ modules, so
        # src changes can break them too.
        run: uv run python dev/scripts/nb_integration.py --set ci-smoke

{_failure_step("CI", "notebooks-smoke")}
"""
    return (".gitea/workflows/ci.yml", content)


def _gen_deploy(
    images: list[dict],
    scans: list[dict],
    registry: str,
    owner_repo: str,
    runner: str,
    uv_version: str,
    **_kw: object,
) -> tuple[str, str]:
    scan_set = {s["name"] for s in scans}

    # Generate one job per image with path filters
    job_blocks = []
    job_names = []
    for img in images:
        name = img["name"]
        job_names.append(name)
        paths = img.get("path_filter", [])

        # Gitea Actions doesn't support `paths:` on push without branches,
        # so use `if: contains(...)` on the modified files list.
        def _clean_path(p: str) -> str:
            return p.rstrip("*").rstrip("/")

        path_checks = " ||\n      ".join(
            f"contains(github.event.head_commit.modified, '{_clean_path(p)}')"
            for p in paths
        )
        tags = (
            f"{registry}/{owner_repo}/{name}:${{{{ env.SHORT_SHA }}}},"
            f"{registry}/{owner_repo}/{name}:latest"
        )
        scan_step = ""
        if name in scan_set:
            scan_step = f"""

{_install_trivy_step()}

{_trivy_step(img, "${{{{ env.SHORT_SHA }}}}", registry, owner_repo)}"""

        job_blocks.append(f"""\
  {name}:
    runs-on: {runner}
    if: >-
      {path_checks}
    steps:
{_checkout_step()}

{_docker_login_step(registry)}
{scan_step}

      - name: Compute short SHA
        run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"

{_build_push_step(img, tags, registry, owner_repo)}

{_failure_step("Deploy", name)}""")

    jobs_block = "\n\n".join(job_blocks)
    needs_list = ", ".join(job_names)

    content = f"""\
{_HEADER}
name: Deploy

on:
  push:
    branches: [main]

jobs:
{jobs_block}

  report:
    runs-on: {runner}
    needs: [{needs_list}]
    if: always() && !cancelled()
    steps:
{_checkout_step()}

{_setup_uv_step(uv_version)}

      - name: Report
        # Only file an issue if at least one upstream build job failed.
        # Without this gate api.diag.ci runs unconditionally and files
        # a fake "failed" issue on every successful Deploy run.
        if: contains(needs.*.result, 'failure')
        env:
          GITEA_TOKEN: ${{{{ secrets.DEPLOY_TOKEN }}}}
        run: |
          uv sync --no-dev --quiet 2>/dev/null || true
          uv run python -m api.diag.ci \\
            --workflow "Deploy" --job "report" \\
            --run "${{{{ github.run_number }}}}" \\
            --sha "${{{{ github.sha }}}}" \\
            --ref "${{{{ github.ref }}}}" || true
"""
    return (".gitea/workflows/deploy.yml", content)


def _gen_harden(
    images: list[dict],
    scans: list[dict],
    registry: str,
    owner_repo: str,
    runner: str,
    uv_version: str,
    **_kw: object,
) -> tuple[str, str]:
    build_steps = []
    for img in images:
        tags = (
            f"{registry}/{owner_repo}/{img['name']}:hardened,"
            f"{registry}/{owner_repo}/{img['name']}:latest"
        )
        build_steps.append(
            _build_push_step(img, tags, registry, owner_repo, no_cache=True)
        )
    build_block = "\n\n".join(build_steps)

    scan_steps = []
    for img in scans:
        scan_steps.append(_trivy_step(img, "hardened", registry, owner_repo))
    scan_block = "\n\n".join(scan_steps)

    vuln_files = " ".join(f"{i['name']}-scan.json" for i in scans)

    content = f"""\
{_HEADER}
name: Harden

on:
  workflow_dispatch:
  schedule:
    - cron: "0 2 * * 0"

jobs:
  build-scan-report:
    runs-on: {runner}
    steps:
{_checkout_step()}

{_docker_login_step(registry)}

{_install_trivy_step()}

{_setup_uv_step(uv_version)}

{build_block}

{scan_block}

      - name: Close resolved or file new vuln issues
        env:
          GITEA_TOKEN: ${{{{ secrets.DEPLOY_TOKEN }}}}
        run: |
          uv sync --no-dev
          for f in {vuln_files}; do
            if [ -f "$f" ]; then
              uv run python -m api.diag.vuln --close "$f" || \\
              uv run python -m api.diag.vuln "$f" || true
            fi
          done

{_failure_step("Harden", "build-scan-report")}
"""
    return (".gitea/workflows/harden.yml", content)


def _gen_rebuild_all(
    images: list[dict],
    scans: list[dict],
    registry: str,
    owner_repo: str,
    runner: str,
    uv_version: str,
    **_kw: object,
) -> tuple[str, str]:
    build_steps = []
    for img in images:
        tags = (
            f"{registry}/{owner_repo}/{img['name']}:${{{{ env.SHORT_SHA }}}},"
            f"{registry}/{owner_repo}/{img['name']}:latest"
        )
        build_steps.append(_build_push_step(img, tags, registry, owner_repo))
    build_block = "\n\n".join(build_steps)

    scan_steps = []
    for img in scans:
        scan_steps.append(
            _trivy_step(img, "${{ env.SHORT_SHA }}", registry, owner_repo)
        )
    scan_block = "\n\n".join(scan_steps)

    vuln_files = " ".join(f"{i['name']}-scan.json" for i in scans)

    content = f"""\
{_HEADER}
name: Rebuild All

on:
  workflow_dispatch:

jobs:
  build-scan-report:
    runs-on: {runner}
    steps:
{_checkout_step()}

{_docker_login_step(registry)}

{_install_trivy_step()}

{_setup_uv_step(uv_version)}

      - name: Compute short SHA
        run: echo "SHORT_SHA=$(echo $GITHUB_SHA | head -c 8)" >> "$GITHUB_ENV"

{build_block}

{scan_block}

      - name: Report vulnerabilities
        env:
          GITEA_TOKEN: ${{{{ secrets.DEPLOY_TOKEN }}}}
        run: |
          uv sync --no-dev
          for f in {vuln_files}; do
            [ -f "$f" ] && uv run python -m api.diag.vuln "$f" || true
          done

{_failure_step("Rebuild All", "build-scan-report")}
"""
    return (".gitea/workflows/rebuild-all.yml", content)


def _gen_infra_ci(
    images: list[dict],
    runner: str,
    **_kw: object,
) -> tuple[str, str]:
    all_paths: list[str] = []
    for img in images:
        all_paths.extend(img.get("path_filter", []))
    paths_block = "\n".join(f"      - {p!r}" for p in all_paths)

    jobs_parts = []
    for img in images:
        if not img.get("hadolint", True):
            continue
        name = img["name"]
        # The notebooks image gets a frontend smoke gate: boot the freshly
        # built image and load the editor in a headless browser, failing on
        # any console/page error. Guards against runtime-broken frontend
        # bundles that compile cleanly (the 2026-07-09 "d is not a
        # constructor" incident shipped through a green build).
        fe_smoke_step = ""
        if name == "notebooks":
            fe_smoke_step = f"""

      - name: Frontend smoke gate
        run: python3 dev/scripts/nb_fe_smoke.py --image local/{name}:build --network ci"""
        jobs_parts.append(f"""\
  {name}:
    runs-on: {runner}
    steps:
{_checkout_step()}

      - name: Hadolint {name}
        uses: https://github.com/hadolint/hadolint-action@v3.1.0
        with:
          dockerfile: {img["dockerfile"]}

{_setup_buildx_step()}

{_build_push_step(img, f"ci-test-{name}", "", "", load_only=True)}{fe_smoke_step}

{_failure_step("Infra CI", name)}""")

    jobs_block = "\n\n".join(jobs_parts)

    content = f"""\
{_HEADER}
name: Infra CI

on:
  push:
    paths:
{paths_block}
  pull_request:
    paths:
{paths_block}

jobs:
{jobs_block}
"""
    return (".gitea/workflows/infra-ci.yml", content)


def _gen_notebooks_integration(
    runner: str, uv_version: str, **_kw: object
) -> tuple[str, str]:
    """Nightly full-set notebook run inside the production container.

    The full notebook set needs real data (aco.duckdb, ./data mounts) that
    only the prod container has, so the scripts are docker-cp'd in and run
    there. Failures are routed through nb_issue_filer (dedup + auto-close),
    not exit codes — a red nightly run should page via the issue tracker,
    not accumulate api.diag.ci duplicates. 03:30 sits before the 06:00
    pkg-supply-chain run and outside interactive hours (duckdb lock, #508).
    """
    content = f"""\
{_HEADER}
name: Notebooks Integration

on:
  workflow_dispatch:
  schedule:
    - cron: "30 3 * * *"

jobs:
  notebooks-integration:
    runs-on: {runner}
    steps:
{_checkout_step()}

{_setup_uv_step(uv_version)}

      - name: Run full notebook set in prod container
        env:
          GITEA_TOKEN: ${{{{ secrets.DEPLOY_TOKEN }}}}
        run: |
          set -euo pipefail
          docker exec notebooks mkdir -p /tmp/nbtest
          docker cp dev/scripts/nb_integration.py notebooks:/tmp/nbtest/
          docker cp dev/scripts/nb_issue_filer.py notebooks:/tmp/nbtest/
          docker cp infra/marimo/nb-tests.toml notebooks:/tmp/nbtest/
          docker exec \\
            -e GITEA_TOKEN \\
            -e GITEA_API_BASE=http://git:3000/api/v1 \\
            notebooks \\
            uv run --project /home/kert/workspace python /tmp/nbtest/nb_integration.py \\
              --set all --file-issues \\
              --nb-dir /home/kert/notebooks \\
              --config /tmp/nbtest/nb-tests.toml \\
              --report /tmp/nbtest/report.json
          docker cp notebooks:/tmp/nbtest/report.json nb-integration-report.json
          cat nb-integration-report.json

      - name: Frontend smoke against live service
        run: python3 dev/scripts/nb_fe_smoke.py --url http://notebooks:2718 --network gateway

{_failure_step("Notebooks Integration", "notebooks-integration")}
"""
    return (".gitea/workflows/notebooks-integration.yml", content)


def _gen_zotero_sync(runner: str, uv_version: str, **_kw: object) -> tuple[str, str]:
    """Nightly bib → Zotero routing for mail-ingested items.

    mail-poller ingests CMS mail into bib continuously, but Zotero only
    sees those items when `stack bib sync-zotero` pushes them — and that
    had no scheduler, so the Cmsupdates collection silently froze for two
    months (Apr 29 → Jul 10, 212-item backlog). sync writes directly into
    zotero.sqlite, so the running Zotero app must be stopped for the
    duration; the stop/start lives in workflow steps (visible, and
    restart is if:always()) rather than the CLI's --hold-zotero, because
    the sync runs docker-exec'd inside the api container, which has no
    docker socket. 04:15 sits after the 03:30 notebooks-integration run
    so the two never contend for zotero.sqlite.
    """
    content = f"""\
{_HEADER}
name: Zotero Sync

on:
  workflow_dispatch:
  schedule:
    - cron: "15 4 * * *"

jobs:
  zotero-sync:
    runs-on: {runner}
    steps:
{_checkout_step()}

{_setup_uv_step(uv_version)}

      - name: Stop zotero (release SQLite write lock)
        run: docker stop zotero

      - name: Sync mail-ingested bib items into Zotero
        # --merge-dupes: listserv re-sends are now aliased at ingest
        # (Task 3), and any same-URL twins that still land in Zotero are
        # merged (union tags/collections/notes/attachments, keep-earliest)
        # rather than deleted.
        run: |
          docker exec api uv run --no-sync \\
            stack bib sync-zotero --tag source:email --no-hold \\
              --merge-dupes

      - name: Sync Federal Register rule items into Zotero
        # Rule items only ever reached Zotero via one-off manual syncs
        # (P36 step 5); P37 skipped that step and the CY2027 NPRM never
        # arrived (#626/#627). Tag-scoped: a full unscoped sync would
        # hydrate ~180k bib items.
        run: |
          docker exec api uv run --no-sync \\
            stack bib sync-zotero --tag source:federal-register --no-hold

      - name: Restart zotero
        if: always()
        run: docker start zotero

{_failure_step("Zotero Sync", "zotero-sync")}
"""
    return (".gitea/workflows/zotero-sync.yml", content)


def _gen_llm_golden(runner: str, uv_version: str, **_kw: object) -> tuple[str, str]:
    """Nightly golden longitudinal evaluation of the chat (P49 Task 7).

    Mirrors ``_gen_notebooks_integration``: the golden set needs a live
    chat over the real pgvector/DuckDB/Ollama stack, which only the
    ``llm`` container (``uvicorn llm.api:app``, port 8000, no published
    port) has — so the runner and the golden set are docker-cp'd in and
    run there against its own ``http://localhost:8000``. Regressions
    are filed through ``nb_issue_filer`` (dedup + auto-close) under the
    ``llm`` label, not ``notebooks``. 03:40 sits after the 03:30
    notebooks-integration run and before the 04:15 zotero-sync run.
    """
    content = f"""\
{_HEADER}
name: LLM Golden

on:
  workflow_dispatch:
  schedule:
    - cron: "40 3 * * *"

jobs:
  llm-golden:
    runs-on: {runner}
    steps:
{_checkout_step()}

{_setup_uv_step(uv_version)}

      - name: Run golden longitudinal evaluation against the live chat
        # golden_rc is captured (not left to `set -e`) so a failing golden
        # run still docker-cps the report back and cats it — llm_golden.py
        # itself exits 0 once --file-issues has filed the regressions
        # (nb_issue_filer is the failure channel here), but this still
        # guards the case where it exits nonzero before reaching that
        # point (e.g. a crash while loading the golden set).
        env:
          GITEA_TOKEN: ${{{{ secrets.DEPLOY_TOKEN }}}}
        run: |
          set -euo pipefail
          docker exec llm mkdir -p /tmp/golden
          docker cp dev/scripts/llm_golden.py llm:/tmp/golden/
          docker cp dev/scripts/nb_issue_filer.py llm:/tmp/golden/
          docker cp tests/llm/golden_lineage.yaml llm:/tmp/golden/
          docker exec \\
            -e GITEA_TOKEN \\
            -e GITEA_API_BASE=http://git:3000/api/v1 \\
            -e NB_ISSUE_LABEL=llm \\
            llm \\
            uv run --project /app python /tmp/golden/llm_golden.py run \\
              --url http://localhost:8000 \\
              --set /tmp/golden/golden_lineage.yaml \\
              --report /tmp/golden/report.json \\
              --file-issues --source nightly-llm-golden || golden_rc=$?
          docker cp llm:/tmp/golden/report.json llm-golden-report.json
          cat llm-golden-report.json
          exit "${{golden_rc:-0}}"

{_failure_step("LLM Golden", "llm-golden")}
"""
    return (".gitea/workflows/llm-golden.yml", content)


def _gen_release(runner: str, uv_version: str, **_kw: object) -> tuple[str, str]:
    content = f"""\
{_HEADER}
name: Release

on:
  push:
    tags: ["v*"]

jobs:
  release:
    runs-on: {runner}
    steps:
{_checkout_step()}

{_setup_uv_step(uv_version)}

      - name: Build package
        run: uv build --out-dir dist/

      - name: Create release
        uses: https://github.com/softprops/action-gh-release@v2
        with:
          files: |
            dist/*.whl
            dist/*.tar.gz

{_failure_step("Release", "release")}
"""
    return (".gitea/workflows/release.yml", content)


# ── Public API ────────────────────────────────────────────────────


def emit(
    images: list[dict],
    scans: list[dict],
    platform: dict,
    ci_cfg: dict,
) -> dict[str, str]:
    """Return {relative_path: content} for all Gitea Actions workflows."""
    # ci_registry is the internal HTTP endpoint used for push (avoids TLS)
    registry = ci_cfg.get("ci_registry", platform["registry"])
    owner_repo = platform["repo"]
    runner = ci_cfg.get("runner_labels", ["ubuntu-latest"])[0]
    uv_version = ci_cfg.get("uv_version", "latest")

    common = dict(
        images=images,
        scans=scans,
        registry=registry,
        owner_repo=owner_repo,
        runner=runner,
        uv_version=uv_version,
        coverage_threshold=ci_cfg.get("coverage_threshold", 99),
    )

    files: dict[str, str] = {}
    for gen_fn in (
        _gen_ci,
        _gen_deploy,
        _gen_harden,
        _gen_rebuild_all,
        _gen_infra_ci,
        _gen_notebooks_integration,
        _gen_zotero_sync,
        _gen_llm_golden,
        _gen_release,
    ):
        path, content = gen_fn(**common)  # type: ignore[arg-type]
        files[path] = content

    return files