diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index 191e98e..d43361b 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -4,6 +4,7 @@ name: Deploy on: + workflow_dispatch: push: branches: [main] @@ -22,17 +23,44 @@ jobs: - name: Checkout uses: https://github.com/actions/checkout@v4 + - name: Find the last successful deploy + id: base + env: + GITEA_TOKEN: ${{ secrets.DEPLOY_TOKEN }} + API: http://git:3000/api/v1 + REPO: ${{ github.repository }} + run: | + # Diff from the last *successful* Deploy, not from the previous + # push: a re-push cancels the run in flight, and diffing only + # github.event.before then dropped the cancelled push's paths for + # good (2026-10-27: the notebooks image never rebuilt for 9cba4fd + # or 409fc6f). A failed run likewise stays in the next run's range + # until it lands. The runner image has no python or jq; Gitea + # writes each run's "path" before its "head_sha", so grep + awk + # pair them (awk reads to the end, so grep never takes a SIGPIPE + # under pipefail). Nothing found leaves sha empty: build everything. + sha="$(curl -sf -H "Authorization: token $GITEA_TOKEN" \ + "$API/repos/$REPO/actions/runs?status=success&branch=main&limit=50" \ + | grep -o '"path":"[^"]*"\|"head_sha":"[0-9a-f]*"' \ + | awk -F'"' '$2 == "path" {p = $4} + $2 == "head_sha" && !sha && index(p, "deploy.yml@") == 1 {sha = $4} + END {print sha}')" \ + || sha="" + echo "last successful deploy: ${sha:-none}" + echo "sha=$sha" >> "$GITHUB_OUTPUT" + - name: Detect changed paths id: filter env: - BEFORE: ${{ github.event.before }} + BEFORE: ${{ steps.base.outputs.sha }} run: | # `github.event.head_commit.modified` only describes the tip commit # and is empty for merge commits, so a PR merged with a merge commit - # never matched any path filter. Diff the pushed range instead: - # the branch tip before the push against the new one. When that - # previous tip is unknown or unfetchable (first push, history - # rewrite) build everything rather than nothing. + # never matched any path filter. Diff a range instead: from the + # last successfully deployed commit (the base step) to the new tip. + # When that commit is unknown or unfetchable (no successful run, + # API unreachable, history rewrite) build everything rather than + # nothing. changed=__all__ if [ -n "$BEFORE" ] && [ "$BEFORE" != "0000000000000000000000000000000000000000" ] \ && git fetch --quiet --depth=1 origin "$BEFORE"; then diff --git a/dev/scripts/backends/gitea.py b/dev/scripts/backends/gitea.py index d15137b..977d626 100644 --- a/dev/scripts/backends/gitea.py +++ b/dev/scripts/backends/gitea.py @@ -254,18 +254,42 @@ jobs: _CHANGES_JOB = "changes" _CHANGES_STEP_ID = "filter" +_BASE_STEP_ID = "base" +_DEPLOY_WORKFLOW = "deploy.yml" _ALL_CHANGED = "__all__" _NULL_SHA = "0" * 40 +# Shell body of the step that finds the last successful Deploy's commit. +_BASE_SCRIPT = f"""\ + # Diff from the last *successful* Deploy, not from the previous + # push: a re-push cancels the run in flight, and diffing only + # github.event.before then dropped the cancelled push's paths for + # good (2026-10-27: the notebooks image never rebuilt for 9cba4fd + # or 409fc6f). A failed run likewise stays in the next run's range + # until it lands. The runner image has no python or jq; Gitea + # writes each run's "path" before its "head_sha", so grep + awk + # pair them (awk reads to the end, so grep never takes a SIGPIPE + # under pipefail). Nothing found leaves sha empty: build everything. + sha="$(curl -sf -H "Authorization: token $GITEA_TOKEN" \\ + "$API/repos/$REPO/actions/runs?status=success&branch=main&limit=50" \\ + | grep -o '"path":"[^"]*"\\|"head_sha":"[0-9a-f]*"' \\ + | awk -F'"' '$2 == "path" {{p = $4}} + $2 == "head_sha" && !sha && index(p, "{_DEPLOY_WORKFLOW}@") == 1 {{sha = $4}} + END {{print sha}}')" \\ + || sha="" + echo "last successful deploy: ${{sha:-none}}" + echo "sha=$sha" >> "$GITHUB_OUTPUT\"""" + # Shell body of the change-detection step; the braces of the `match` # function are doubled so they survive the f-string. _CHANGES_SCRIPT = f"""\ # `github.event.head_commit.modified` only describes the tip commit # and is empty for merge commits, so a PR merged with a merge commit - # never matched any path filter. Diff the pushed range instead: - # the branch tip before the push against the new one. When that - # previous tip is unknown or unfetchable (first push, history - # rewrite) build everything rather than nothing. + # never matched any path filter. Diff a range instead: from the + # last successfully deployed commit (the base step) to the new tip. + # When that commit is unknown or unfetchable (no successful run, + # API unreachable, history rewrite) build everything rather than + # nothing. changed={_ALL_CHANGED} if [ -n "$BEFORE" ] && [ "$BEFORE" != "{_NULL_SHA}" ] \\ && git fetch --quiet --depth=1 origin "$BEFORE"; then @@ -326,10 +350,19 @@ def _changes_job(images: list[dict], runner: str) -> str: steps: {_checkout_step()} + - name: Find the last successful deploy + id: {_BASE_STEP_ID} + env: + GITEA_TOKEN: ${{{{ secrets.DEPLOY_TOKEN }}}} + API: http://git:3000/api/v1 + REPO: ${{{{ github.repository }}}} + run: | +{_BASE_SCRIPT} + - name: Detect changed paths id: {_CHANGES_STEP_ID} env: - BEFORE: ${{{{ github.event.before }}}} + BEFORE: ${{{{ steps.{_BASE_STEP_ID}.outputs.sha }}}} run: | {_CHANGES_SCRIPT} {match_block}""" @@ -347,8 +380,9 @@ def _gen_deploy( scan_set = {s["name"] for s in scans} # Gitea Actions doesn't support `paths:` filters on push, so one job - # computes which images the pushed range touches and the per-image - # jobs gate on its outputs. + # computes which images changed since the last successful deploy and + # the per-image jobs gate on its outputs. workflow_dispatch runs the + # same catch-up without a push. job_blocks = [_changes_job(images, runner)] job_names = [_CHANGES_JOB] for img in images: @@ -392,6 +426,7 @@ def _gen_deploy( name: Deploy on: + workflow_dispatch: push: branches: [main] diff --git a/tests/dev/test_gen_config_deploy.py b/tests/dev/test_gen_config_deploy.py index 54e5691..7ead6b7 100644 --- a/tests/dev/test_gen_config_deploy.py +++ b/tests/dev/test_gen_config_deploy.py @@ -78,7 +78,8 @@ def test_changes_job_exposes_one_output_per_image(): "always": "${{ steps.filter.outputs.always }}", } step = next(s for s in changes["steps"] if s.get("id") == "filter") - assert step["env"] == {"BEFORE": "${{ github.event.before }}"} + # The range starts at the last successful deploy, not the previous push. + assert step["env"] == {"BEFORE": "${{ steps.base.outputs.sha }}"} assert ( "match api 'infra/images/api.Dockerfile' 'src/*' 'pyproject.toml'" in (step["run"]) @@ -88,6 +89,26 @@ def test_changes_job_exposes_one_output_per_image(): assert 'echo "always=true" >> "$GITHUB_OUTPUT"' in step["run"] +def test_base_step_runs_before_the_detector_and_reads_gitea(): + _, wf = _gen() + ids = [st.get("id") for st in wf["jobs"]["changes"]["steps"]] + assert ids.index("base") < ids.index("filter") + base = wf["jobs"]["changes"]["steps"][ids.index("base")] + assert base["env"] == { + "GITEA_TOKEN": "${{ secrets.DEPLOY_TOKEN }}", + "API": "http://git:3000/api/v1", + "REPO": "${{ github.repository }}", + } + assert "actions/runs?status=success&branch=main" in base["run"] + + +def test_deploy_can_be_dispatched_to_catch_up(): + _, wf = _gen() + # PyYAML reads the bare `on:` key as True. + assert "workflow_dispatch" in wf[True] + assert wf[True]["push"] == {"branches": ["main"]} + + def test_image_jobs_gate_on_changes_outputs_not_head_commit(): _, wf = _gen() for name in ("api", "mc", "always"): @@ -219,3 +240,95 @@ def test_unknown_previous_tip_builds_everything(history, tmp_path): for before in ("", "0" * 40, "f" * 40): res = _run_detector(history["shallow"], before, history["merge"], tmp_path) assert res["outputs"] == {"api": "true", "mc": "true", "always": "true"}, before + + +# ── Behavioural: the last-successful-deploy lookup ────────────────────── + + +def _gitea_run(run_id: int, path: str, sha: str, conclusion: str) -> str: + """A run object in Gitea 1.25's field order: nested actor/repository + objects carry their own ids, and `path` precedes `head_sha`.""" + return ( + f'{{"id":{run_id},"url":"u","html_url":"h","display_title":"t",' + f'"path":"{path}@refs/heads/main","event":"push","run_attempt":1,' + f'"run_number":{run_id},"repository_id":1,"head_sha":"{sha}",' + f'"head_branch":"main","status":"completed",' + f'"actor":{{"id":2,"login":"kert"}},"trigger_actor":{{"id":2}},' + f'"repository":{{"id":1,"full_name":"homelab/stack"}},' + f'"conclusion":"{conclusion}"}}' + ) + + +def _run_base_step(tmp_path: Path, body: str | None) -> str: + """Run the generated base step with a fake `curl` on PATH that prints + `body` (or fails like `curl -f` on an HTTP error when body is None).""" + _, wf = _gen() + step = next(s for s in wf["jobs"]["changes"]["steps"] if s.get("id") == "base") + bindir = tmp_path / "bin" + bindir.mkdir(parents=True) + payload = tmp_path / "runs.json" + if body is None: + script = "#!/bin/sh\nexit 22\n" + else: + payload.write_text(body) + script = f"#!/bin/sh\ncat '{payload}'\n" + curl = bindir / "curl" + curl.write_text(script) + curl.chmod(0o755) + out_file = tmp_path / "github_output" + out_file.write_text("") + proc = subprocess.run( + ["bash", "-e", "-c", step["run"]], + cwd=tmp_path, + env={ + **_clean_env(), + "PATH": f"{bindir}:{os.environ['PATH']}", + "GITEA_TOKEN": "t", + "API": "http://git:3000/api/v1", + "REPO": "homelab/stack", + "GITHUB_OUTPUT": str(out_file), + }, + capture_output=True, + text=True, + ) + assert proc.returncode == 0, proc.stdout + proc.stderr + outputs = dict( + line.split("=", 1) for line in out_file.read_text().splitlines() if line + ) + return outputs["sha"] + + +def test_base_is_the_newest_successful_deploy_not_other_workflows(tmp_path): + a, b = "a" * 40, "b" * 40 + runs = ",".join( + [ + _gitea_run(9, "ci.yml", "c" * 40, "success"), # newer, other workflow + _gitea_run(8, "deploy.yml", a, "success"), + _gitea_run(5, "deploy.yml", b, "success"), + ] + ) + body = f'{{"workflow_runs":[{runs}],"total_count":3}}' + assert _run_base_step(tmp_path, body) == a + + +def test_no_successful_deploy_or_api_error_leaves_base_empty(tmp_path): + only_ci = f'{{"workflow_runs":[{_gitea_run(9, "ci.yml", "c" * 40, "success")}]}}' + assert _run_base_step(tmp_path / "1", only_ci) == "" + assert _run_base_step(tmp_path / "2", None) == "" # curl -f on a 401/500 + + +def test_cancelled_push_paths_carry_into_the_next_run(history, tmp_path): + """The cancelled-run gap: the push that touched infra/rustfs was + cancelled, so the last success is still `base`. The next push's own + range (direct → merge) never touched mc, but diffing from the last + successful deploy still rebuilds it.""" + for d in ("p", "s"): + (tmp_path / d).mkdir() + from_push = _run_detector( + history["shallow"], history["direct"], history["merge"], tmp_path / "p" + ) + assert from_push["outputs"]["mc"] == "false" # the old behaviour's miss + from_success = _run_detector( + history["shallow"], history["base"], history["merge"], tmp_path / "s" + ) + assert from_success["outputs"] == {"api": "true", "mc": "true", "always": "true"}