feat: auto-file Gitea issues on any CI workflow step failure

Add backend-agnostic CI failure reporter (api.diag.ci) and wire
`if: failure()` steps into every job across all generated workflows.

- src/api/diag/ci.py: CLI tool that files Gitea issues with workflow,
  job, run, commit context — works with Gitea Actions, GitHub Actions,
  and Woodpecker (takes args, not env-specific)
- dev/scripts/backends/gitea.py: _failure_step() added to all 6
  workflows (11 jobs total)
- dev/scripts/backends/github.py: same _failure_step() pattern for
  GitHub Actions backend
- .gitea/workflows/pkg-supply-chain.yml: manual failure step added
  (stale generated file)
- 6 new tests, 12,061 total passing

Now ANY step failure in ANY CI workflow on ANY backend will auto-create
a Gitea issue with the workflow name, job, run number, and commit SHA.

.gitea/workflows/ci.yml

@@ -35,6 +35,18 @@ jobs:
       - name: Validate generated config
         run: uv run python dev/scripts/gen_config.py --check
 
+      - name: File failure issue
+        if: failure()
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+        run: |
+          uv sync --no-dev --quiet 2>/dev/null || true
+          uv run python -m api.diag.ci \
+            --workflow "CI" --job "lint-test" \
+            --run "${{ github.run_number }}" \
+            --sha "${{ github.sha }}" \
+            --ref "${{ github.ref }}" || true
+
   skinny-install:
     runs-on: ubuntu-latest
     strategy:
@@ -60,3 +72,15 @@ jobs:
           if [ -d "tests/${{ matrix.extra }}" ]; then
             uv run pytest "tests/${{ matrix.extra }}/" -x -q || true
           fi
+
+      - name: File failure issue
+        if: failure()
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+        run: |
+          uv sync --no-dev --quiet 2>/dev/null || true
+          uv run python -m api.diag.ci \
+            --workflow "CI" --job "skinny-install" \
+            --run "${{ github.run_number }}" \
+            --sha "${{ github.sha }}" \
+            --ref "${{ github.ref }}" || true

.gitea/workflows/deploy.yml

@@ -98,3 +98,15 @@ jobs:
           for f in notebooks-scan.json zotero-scan.json docs-scan.json api-scan.json; do
             [ -f "$f" ] && uv run python -m api.diag.vuln "$f" || true
           done
+
+      - name: File failure issue
+        if: failure()
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+        run: |
+          uv sync --no-dev --quiet 2>/dev/null || true
+          uv run python -m api.diag.ci \
+            --workflow "Deploy" --job "build-scan-report" \
+            --run "${{ github.run_number }}" \
+            --sha "${{ github.sha }}" \
+            --ref "${{ github.ref }}" || true

.gitea/workflows/harden.yml

@@ -99,3 +99,15 @@ jobs:
               uv run python -m api.diag.vuln "$f" || true
             fi
           done
+
+      - name: File failure issue
+        if: failure()
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+        run: |
+          uv sync --no-dev --quiet 2>/dev/null || true
+          uv run python -m api.diag.ci \
+            --workflow "Harden" --job "build-scan-report" \
+            --run "${{ github.run_number }}" \
+            --sha "${{ github.sha }}" \
+            --ref "${{ github.ref }}" || true

.gitea/workflows/infra-ci.yml

@@ -50,6 +50,18 @@ jobs:
       - name: Build notebooks
         run: docker build -f infra/images/notebooks.Dockerfile -t local/notebooks:build notebooks/
 
+      - name: File failure issue
+        if: failure()
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+        run: |
+          uv sync --no-dev --quiet 2>/dev/null || true
+          uv run python -m api.diag.ci \
+            --workflow "Infra CI" --job "notebooks" \
+            --run "${{ github.run_number }}" \
+            --sha "${{ github.sha }}" \
+            --ref "${{ github.ref }}" || true
+
   zotero:
     runs-on: ubuntu-latest
     steps:
@@ -66,6 +78,18 @@ jobs:
       - name: Build zotero
         run: docker build -f infra/images/zotero.Dockerfile -t local/zotero:build data/zotero/
 
+      - name: File failure issue
+        if: failure()
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+        run: |
+          uv sync --no-dev --quiet 2>/dev/null || true
+          uv run python -m api.diag.ci \
+            --workflow "Infra CI" --job "zotero" \
+            --run "${{ github.run_number }}" \
+            --sha "${{ github.sha }}" \
+            --ref "${{ github.ref }}" || true
+
   docs:
     runs-on: ubuntu-latest
     steps:
@@ -82,6 +106,18 @@ jobs:
       - name: Build docs
         run: docker build -f infra/images/docs.Dockerfile -t local/docs:build .
 
+      - name: File failure issue
+        if: failure()
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+        run: |
+          uv sync --no-dev --quiet 2>/dev/null || true
+          uv run python -m api.diag.ci \
+            --workflow "Infra CI" --job "docs" \
+            --run "${{ github.run_number }}" \
+            --sha "${{ github.sha }}" \
+            --ref "${{ github.ref }}" || true
+
   api:
     runs-on: ubuntu-latest
     steps:
@@ -98,6 +134,18 @@ jobs:
       - name: Build api
         run: docker build -f infra/images/api.Dockerfile -t local/api:build .
 
+      - name: File failure issue
+        if: failure()
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+        run: |
+          uv sync --no-dev --quiet 2>/dev/null || true
+          uv run python -m api.diag.ci \
+            --workflow "Infra CI" --job "api" \
+            --run "${{ github.run_number }}" \
+            --sha "${{ github.sha }}" \
+            --ref "${{ github.ref }}" || true
+
   mc:
     runs-on: ubuntu-latest
     steps:
@@ -113,3 +161,15 @@ jobs:
 
       - name: Build mc
         run: docker build -f infra/images/mc.Dockerfile -t local/mc:build infra/rustfs/
+
+      - name: File failure issue
+        if: failure()
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+        run: |
+          uv sync --no-dev --quiet 2>/dev/null || true
+          uv run python -m api.diag.ci \
+            --workflow "Infra CI" --job "mc" \
+            --run "${{ github.run_number }}" \
+            --sha "${{ github.sha }}" \
+            --ref "${{ github.ref }}" || true

.gitea/workflows/pkg-supply-chain.yml

@@ -56,3 +56,15 @@ jobs:
         env:
           GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
         run: uv run python dev/scripts/pkg_issues.py
+
+      - name: File failure issue
+        if: failure()
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+        run: |
+          uv sync --no-dev --quiet 2>/dev/null || true
+          uv run python -m api.diag.ci \
+            --workflow "Package Supply Chain" --job "pkg-supply-chain" \
+            --run "${{ github.run_number }}" \
+            --sha "${{ github.sha }}" \
+            --ref "${{ github.ref }}" || true

.gitea/workflows/rebuild-all.yml

@@ -97,3 +97,15 @@ jobs:
           for f in notebooks-scan.json zotero-scan.json docs-scan.json api-scan.json; do
             [ -f "$f" ] && uv run python -m api.diag.vuln "$f" || true
           done
+
+      - name: File failure issue
+        if: failure()
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+        run: |
+          uv sync --no-dev --quiet 2>/dev/null || true
+          uv run python -m api.diag.ci \
+            --workflow "Rebuild All" --job "build-scan-report" \
+            --run "${{ github.run_number }}" \
+            --sha "${{ github.sha }}" \
+            --ref "${{ github.ref }}" || true

.gitea/workflows/release.yml

@@ -28,3 +28,15 @@ jobs:
           files: |
             dist/*.whl
             dist/*.tar.gz
+
+      - name: File failure issue
+        if: failure()
+        env:
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+        run: |
+          uv sync --no-dev --quiet 2>/dev/null || true
+          uv run python -m api.diag.ci \
+            --workflow "Release" --job "release" \
+            --run "${{ github.run_number }}" \
+            --sha "${{ github.sha }}" \
+            --ref "${{ github.ref }}" || true

dev/scripts/backends/gitea.py

@@ -98,6 +98,25 @@ def _trivy_step(
         run: trivy image --severity {sev} --exit-code {ec} --format json -o {name}-scan.json {local_tag}"""
 
 
+# ── Failure reporting ─────────────────────────────────────────────
+
+
+def _failure_step(workflow_name: str, job_name: str) -> str:
+    """Emit an if:failure() step that files a Gitea issue."""
+    return f"""\
+      - name: File failure issue
+        if: failure()
+        env:
+          GITEA_TOKEN: ${{{{ secrets.GITEA_TOKEN }}}}
+        run: |
+          uv sync --no-dev --quiet 2>/dev/null || true
+          uv run python -m api.diag.ci \\
+            --workflow "{workflow_name}" --job "{job_name}" \\
+            --run "${{{{ github.run_number }}}}" \\
+            --sha "${{{{ github.sha }}}}" \\
+            --ref "${{{{ github.ref }}}}" || true"""
+
+
 # ── Gitea-specific helpers ───────────────────────────────────────
 
 
@@ -149,6 +168,8 @@ jobs:
       - name: Validate generated config
         run: uv run python dev/scripts/gen_config.py --check
 
+{_failure_step("CI", "lint-test")}
+
   skinny-install:
     runs-on: {runner}
     strategy:
@@ -170,6 +191,8 @@ jobs:
           if [ -d "tests/${{{{ matrix.extra }}}}" ]; then
             uv run pytest "tests/${{{{ matrix.extra }}}}/" -x -q || true
           fi
+
+{_failure_step("CI", "skinny-install")}
 """
     return (".gitea/workflows/ci.yml", content)
 
@@ -237,6 +260,8 @@ jobs:
           for f in {vuln_files}; do
             [ -f "$f" ] && uv run python -m api.diag.vuln "$f" || true
           done
+
+{_failure_step("Deploy", "build-scan-report")}
 """
     return (".gitea/workflows/deploy.yml", content)
 
@@ -304,6 +329,8 @@ jobs:
               uv run python -m api.diag.vuln "$f" || true
             fi
           done
+
+{_failure_step("Harden", "build-scan-report")}
 """
     return (".gitea/workflows/harden.yml", content)
 
@@ -369,6 +396,8 @@ jobs:
           for f in {vuln_files}; do
             [ -f "$f" ] && uv run python -m api.diag.vuln "$f" || true
           done
+
+{_failure_step("Rebuild All", "build-scan-report")}
 """
     return (".gitea/workflows/rebuild-all.yml", content)
 
@@ -401,7 +430,9 @@ def _gen_infra_ci(
 
 {_setup_buildx_step()}
 
-{_build_push_step(img, f"ci-test-{name}", "", "", load_only=True)}""")
+{_build_push_step(img, f"ci-test-{name}", "", "", load_only=True)}
+
+{_failure_step("Infra CI", name)}""")
 
     jobs_block = "\n\n".join(jobs_parts)
 
@@ -449,6 +480,8 @@ jobs:
           files: |
             dist/*.whl
             dist/*.tar.gz
+
+{_failure_step("Release", "release")}
 """
     return (".gitea/workflows/release.yml", content)
 

dev/scripts/backends/github.py

@@ -107,6 +107,25 @@ def _build_push_step(
     return lines
 
 
+# ── Failure reporting ─────────────────────────────────────────────
+
+
+def _failure_step(workflow_name: str, job_name: str) -> str:
+    """Emit an if:failure() step that files a Gitea issue."""
+    return f"""\
+      - name: File failure issue
+        if: failure()
+        env:
+          GITEA_TOKEN: ${{{{ secrets.GITEA_TOKEN }}}}
+        run: |
+          uv sync --no-dev --quiet 2>/dev/null || true
+          uv run python -m api.diag.ci \\
+            --workflow "{workflow_name}" --job "{job_name}" \\
+            --run "${{{{ github.run_number }}}}" \\
+            --sha "${{{{ github.sha }}}}" \\
+            --ref "${{{{ github.ref }}}}" || true"""
+
+
 # ── Workflow generators ───────────────────────────────────────────
 
 
@@ -142,6 +161,8 @@ jobs:
 
       - name: Validate generated config
         run: uv run python dev/scripts/gen_config.py --check
+
+{_failure_step("CI", "lint-test")}
 """
     return (".github/workflows/ci.yml", content)
 
@@ -221,6 +242,8 @@ jobs:
         )
     }
 
+{_failure_step("Deploy", "build-scan")}
+
   scan:
     runs-on: {runner}
     needs: build-scan
@@ -238,6 +261,8 @@ jobs:
           name: trivy-scans
           path: "*-scan.json"
 
+{_failure_step("Deploy", "scan")}
+
   report-vulns:
     runs-on: {runner}
     needs: scan
@@ -259,6 +284,8 @@ jobs:
           for f in {vuln_loop_files}; do
             [ -f "$f" ] && uv run python -m api.diag.vuln "$f" || true
           done
+
+{_failure_step("Deploy", "report-vulns")}
 """
     return (".github/workflows/deploy.yml", content)
 
@@ -316,6 +343,8 @@ jobs:
 
 {build_block}
 
+{_failure_step("Harden", "build-scan")}
+
   scan:
     runs-on: {runner}
     needs: build-scan
@@ -330,6 +359,8 @@ jobs:
           name: trivy-scans-harden
           path: "*-scan.json"
 
+{_failure_step("Harden", "scan")}
+
   close-or-report-vulns:
     runs-on: {runner}
     needs: scan
@@ -354,6 +385,8 @@ jobs:
               uv run python -m api.diag.vuln "$f" || true
             fi
           done
+
+{_failure_step("Harden", "close-or-report-vulns")}
 """
     return (".github/workflows/harden.yml", content)
 
@@ -411,6 +444,8 @@ jobs:
 
 {build_block}
 
+{_failure_step("Rebuild All", "build")}
+
   scan:
     runs-on: {runner}
     needs: build
@@ -428,6 +463,8 @@ jobs:
           name: trivy-scans-rebuild
           path: "*-scan.json"
 
+{_failure_step("Rebuild All", "scan")}
+
   report-vulns:
     runs-on: {runner}
     needs: scan
@@ -449,6 +486,8 @@ jobs:
           for f in {vuln_loop_files}; do
             [ -f "$f" ] && uv run python -m api.diag.vuln "$f" || true
           done
+
+{_failure_step("Rebuild All", "report-vulns")}
 """
     return (".github/workflows/rebuild-all.yml", content)
 
@@ -486,7 +525,9 @@ def _gen_infra_ci(
 
 {_setup_buildx_step()}
 
-{_build_push_step(img, f"ci-test-{name}", "", "", load_only=True)}""")
+{_build_push_step(img, f"ci-test-{name}", "", "", load_only=True)}
+
+{_failure_step("Infra CI", name)}""")
 
     jobs_block = "\n\n".join(jobs_block_parts)
 
@@ -536,6 +577,8 @@ jobs:
           files: |
             dist/*.whl
             dist/*.tar.gz
+
+{_failure_step("Release", "release")}
 """
     return (".github/workflows/release.yml", content)
 

src/api/diag/ci.py

@@ -0,0 +1,131 @@
+"""Backend-agnostic CI failure reporter.
+
+Files a Gitea issue when any CI workflow step fails, regardless of
+whether the CI backend is Gitea Actions, GitHub Actions, or Woodpecker.
+
+Usage from workflow YAML::
+
+    uv run python -m api.diag.ci \\
+        --workflow "Deploy" --job "build-scan-report" \\
+        --run "42" --sha "abc1234" --ref "refs/heads/main"
+
+All arguments are passed explicitly by the workflow step so the module
+doesn't need to know which CI system is running.
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import os
+import sys
+
+log = logging.getLogger(__name__)
+
+
+def _get_token() -> str:
+    token = os.environ.get("GITEA_TOKEN", "")
+    if token:
+        return token
+    try:
+        from conf import secret
+
+        return secret("gitea.token", "GITEA_TOKEN")
+    except (ImportError, KeyError):
+        return ""
+
+
+def _truncate(s: str, n: int) -> str:
+    return s if len(s) <= n else s[: n - 3] + "..."
+
+
+def file_ci_failure(
+    *,
+    workflow: str,
+    job: str,
+    run: str,
+    sha: str,
+    ref: str,
+    owner: str = "homelab",
+    repo: str = "stack",
+) -> dict | None:
+    """File a Gitea issue for a CI workflow failure."""
+    token = _get_token()
+    if not token:
+        log.error("No GITEA_TOKEN — cannot file issue")
+        return None
+
+    sha_short = sha[:8] if sha else "unknown"
+
+    title = f"ci/{workflow}/{job} failed (#{run} @ {sha_short})"
+    title = _truncate(title, 120)
+
+    body_parts = [
+        f"**Workflow:** `{workflow}`",
+        f"**Job:** `{job}`",
+        f"**Run:** `#{run}`",
+        f"**Commit:** `{sha_short}`",
+        f"**Ref:** `{ref}`",
+        "",
+        "A step in this CI job failed. Check the workflow run logs for details.",
+    ]
+
+    # Try to detect which step failed from env (Gitea/GitHub set these)
+    step_name = os.environ.get("GITHUB_ACTION", "")
+    if step_name:
+        body_parts.insert(5, f"**Step:** `{step_name}`")
+
+    body = "\n".join(body_parts)
+
+    # Resolve labels
+    label_names = ["ci"]
+    wf_lower = workflow.lower()
+    for keyword in ("deploy", "harden", "ci", "infra", "release", "rebuild"):
+        if keyword in wf_lower:
+            label_names.append(f"pipeline:{keyword}")
+            break
+
+    try:
+        from api.clients.gitea import GiteaClient
+
+        client = GiteaClient(token)
+        label_ids = client.resolve_labels(owner, repo, label_names)
+        issue_body: dict = {"title": title, "body": body}
+        if label_ids:
+            issue_body["labels"] = label_ids
+        result = client.create_issue(owner, repo, issue_body)
+        log.info("Filed issue #%s: %s", result.get("number"), title)
+        client.close()
+        return result
+    except Exception:
+        log.exception("Failed to file Gitea issue")
+        return None
+
+
+def main() -> int:
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+
+    parser = argparse.ArgumentParser(description="File a Gitea issue for CI failure")
+    parser.add_argument("--workflow", required=True, help="Workflow name")
+    parser.add_argument("--job", required=True, help="Job name")
+    parser.add_argument("--run", required=True, help="Run number")
+    parser.add_argument("--sha", required=True, help="Commit SHA")
+    parser.add_argument("--ref", default="", help="Git ref (branch/tag)")
+    parser.add_argument("--owner", default="homelab", help="Repo owner")
+    parser.add_argument("--repo", default="stack", help="Repo name")
+    args = parser.parse_args()
+
+    result = file_ci_failure(
+        workflow=args.workflow,
+        job=args.job,
+        run=args.run,
+        sha=args.sha,
+        ref=args.ref,
+        owner=args.owner,
+        repo=args.repo,
+    )
+    return 0 if result else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/api/test_diag_ci.py

@@ -0,0 +1,149 @@
+"""Tests for the backend-agnostic CI failure reporter."""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+
+class TestFileCiFailure:
+    """file_ci_failure builds correct issue title and body."""
+
+    @patch("api.diag.ci._get_token", return_value="fake-token")
+    @patch("api.clients.gitea.GiteaClient")
+    def test_files_issue_with_correct_title(self, mock_cls, mock_token):
+        mock_client = MagicMock()
+        mock_client.create_issue.return_value = {"number": 999}
+        mock_client.resolve_labels.return_value = [1]
+        mock_cls.return_value = mock_client
+
+        from api.diag.ci import file_ci_failure
+
+        result = file_ci_failure(
+            workflow="Deploy",
+            job="build-scan-report",
+            run="42",
+            sha="abc12345def67890",
+            ref="refs/heads/main",
+        )
+
+        assert result is not None
+        assert result["number"] == 999
+
+        call_args = mock_client.create_issue.call_args
+        issue_body = call_args[0][2]
+        assert "Deploy" in issue_body["title"]
+        assert "build-scan-report" in issue_body["title"]
+        assert "abc12345" in issue_body["title"]
+        assert "#42" in issue_body["title"]
+
+    @patch("api.diag.ci._get_token", return_value="fake-token")
+    @patch("api.clients.gitea.GiteaClient")
+    def test_body_includes_metadata(self, mock_cls, mock_token):
+        mock_client = MagicMock()
+        mock_client.create_issue.return_value = {"number": 1}
+        mock_client.resolve_labels.return_value = []
+        mock_cls.return_value = mock_client
+
+        from api.diag.ci import file_ci_failure
+
+        file_ci_failure(
+            workflow="CI",
+            job="lint-test",
+            run="99",
+            sha="deadbeef",
+            ref="refs/heads/feature",
+        )
+
+        issue_body = mock_client.create_issue.call_args[0][2]
+        body = issue_body["body"]
+        assert "CI" in body
+        assert "lint-test" in body
+        assert "#99" in body
+        assert "deadbeef" in body
+        assert "refs/heads/feature" in body
+
+    @patch("api.diag.ci._get_token", return_value="fake-token")
+    @patch("api.clients.gitea.GiteaClient")
+    def test_resolves_pipeline_label(self, mock_cls, mock_token):
+        mock_client = MagicMock()
+        mock_client.create_issue.return_value = {"number": 2}
+        mock_client.resolve_labels.return_value = [5, 10]
+        mock_cls.return_value = mock_client
+
+        from api.diag.ci import file_ci_failure
+
+        file_ci_failure(
+            workflow="Deploy",
+            job="build",
+            run="1",
+            sha="aaa",
+            ref="",
+        )
+
+        label_call = mock_client.resolve_labels.call_args
+        label_names = label_call[0][2]
+        assert "ci" in label_names
+        assert "pipeline:deploy" in label_names
+
+    @patch("api.diag.ci._get_token", return_value="")
+    def test_no_token_returns_none(self, mock_token):
+        from api.diag.ci import file_ci_failure
+
+        result = file_ci_failure(
+            workflow="CI",
+            job="test",
+            run="1",
+            sha="aaa",
+            ref="",
+        )
+        assert result is None
+
+
+class TestCLI:
+    """CLI entry point parses args correctly."""
+
+    @patch("api.diag.ci.file_ci_failure")
+    def test_main_calls_file_ci_failure(self, mock_file):
+        mock_file.return_value = {"number": 1}
+        import sys
+
+        from api.diag.ci import main
+
+        old_argv = sys.argv
+        sys.argv = [
+            "ci",
+            "--workflow",
+            "CI",
+            "--job",
+            "lint-test",
+            "--run",
+            "5",
+            "--sha",
+            "abc123",
+            "--ref",
+            "refs/heads/main",
+        ]
+        try:
+            code = main()
+        finally:
+            sys.argv = old_argv
+
+        assert code == 0
+        mock_file.assert_called_once()
+        kwargs = mock_file.call_args[1]
+        assert kwargs["workflow"] == "CI"
+        assert kwargs["job"] == "lint-test"
+        assert kwargs["run"] == "5"
+        assert kwargs["sha"] == "abc123"
+
+
+class TestWorkflowGeneration:
+    """Generated workflows contain failure steps."""
+
+    def test_gitea_workflows_have_failure_steps(self):
+        from pathlib import Path
+
+        wf_dir = Path(__file__).resolve().parents[2] / ".gitea" / "workflows"
+        for yml in wf_dir.glob("*.yml"):
+            content = yml.read_text()
+            assert "File failure issue" in content, f"{yml.name} missing failure step"