From 45110f35f36a7f09a6adac05e2b0c8a10098daf8 Mon Sep 17 00:00:00 2001 From: kert Date: Wed, 11 Nov 2026 15:14:28 -0500 Subject: [PATCH] feat(lead): stack lead release publish, served at reach /releases behind SSO publish refuses a bag that does not verify or a version already out, then copies it under data/lead/releases (served read-only by the landing nginx at reach./releases/ only, through git-sso like the rest of reach), rewrites index.json/index.html, archives a tarball to rustfs bucket lead-releases via the mc container's rc, creates the Gitea release lead-review-v at the bag's commit, and adds a bib item with the RO-Crate attached. The bags hold the stack's source, so they stay behind SSO until a decision to publish them openly. --- compose.yml | 1 + infra/nginx/nginx.conf | 10 ++ infra/traefik/dynamic/services.yml | 21 +++ src/cli/lead.py | 49 ++++++ src/lead/release/publish.py | 235 +++++++++++++++++++++++++++++ tests/lead/test_release_publish.py | 191 +++++++++++++++++++++++ 6 files changed, 507 insertions(+) create mode 100644 src/lead/release/publish.py create mode 100644 tests/lead/test_release_publish.py diff --git a/compose.yml b/compose.yml index 6dd7a68..d1c350c 100644 --- a/compose.yml +++ b/compose.yml @@ -609,6 +609,7 @@ services: volumes: - ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro - ./infra/nginx/healthz-headers.conf:/etc/nginx/healthz-headers.conf:ro + - ./data/lead/releases:/usr/share/nginx/releases:ro - ./infra/nginx/index.html:/usr/share/nginx/html/index.html:ro - ./assets/css/dashboard.css:/usr/share/nginx/html/dashboard.css:ro - ./assets/css/inject.css:/usr/share/nginx/html/fhirworx.css:ro diff --git a/infra/nginx/nginx.conf b/infra/nginx/nginx.conf index 2a888d7..ee1fa4c 100644 --- a/infra/nginx/nginx.conf +++ b/infra/nginx/nginx.conf @@ -24,6 +24,16 @@ http { try_files $uri =404; } + # LEAD living-review releases (BagIt bags). Only on the reach host: + # traefik routes reach./releases here behind git-sso; the + # public dashboard host never serves them. + location /releases/ { + if ($host !~* "^reach\.") { return 404; } + alias /usr/share/nginx/releases/; + autoindex on; + add_header Cache-Control "no-cache"; + } + # Service health for the landing panels. The Gitea home fetches these # cross-origin, so the status code has to be readable from any host. # Each name is one fixed upstream. Nothing here is an open proxy. diff --git a/infra/traefik/dynamic/services.yml b/infra/traefik/dynamic/services.yml index b9f424f..88196f9 100644 --- a/infra/traefik/dynamic/services.yml +++ b/infra/traefik/dynamic/services.yml @@ -97,6 +97,27 @@ http: middlewares: - projects-ratelimit - secure-headers + # LEAD living-review releases: reach./releases is served by the + # landing nginx from data/lead/releases, behind the same SSO as reach. + reach-releases: + rule: "Host(`reach.{{ $domain }}`) && PathPrefix(`/releases`)" + priority: 1000 + service: dashboard + entryPoints: + - web + middlewares: + - git-sso + - secure-headers + reach-releases-tls: + rule: "Host(`reach.{{ $domain }}`) && PathPrefix(`/releases`)" + priority: 1000 + service: dashboard + entryPoints: + - websecure + tls: {} + middlewares: + - git-sso + - secure-headers {{- range $name, $svc := $reef }} {{ $name }}: rule: "Host(`{{ get $svc "subdomain" | default $name }}.{{ $domain }}`){{ range get $svc "extra_hosts" | default list }} || Host(`{{ . }}`){{ end }}" diff --git a/src/cli/lead.py b/src/cli/lead.py index 8333fba..112de30 100644 --- a/src/cli/lead.py +++ b/src/cli/lead.py @@ -257,6 +257,55 @@ def release_build( typer.echo(f"commit\t{result.commit}") +@release_app.command("publish") +def release_publish( + path: str = typer.Argument(..., help="Path to a built release bag."), + serve_root: str = typer.Option( + "data/lead/releases", + "--serve-root", + help="Directory reach serves /releases from.", + ), + base_url: str = typer.Option( + "https://reach.fhirworx.io/releases", + "--base-url", + help="Public URL of serve-root.", + ), + archive: bool = typer.Option( + True, "--archive/--no-archive", help="Tarball to rustfs." + ), + gitea: bool = typer.Option(True, "--gitea/--no-gitea", help="Gitea release + tag."), + bib: bool = typer.Option( + True, "--bib/--no-bib", help="Bib item with the RO-Crate." + ), +) -> None: + """Publish a verified bag: serve it at reach /releases (same SSO as + reach), archive it to rustfs, create the Gitea release, add a bib item. + Refuses a bag that does not verify or a version already published.""" + from pathlib import Path + + from lead.release.publish import ( + bib_release_item, + gitea_release_via_api, + publish, + rc_archive, + ) + + try: + out = publish( + Path(path), + serve_root=Path(serve_root), + base_url=base_url, + archive=rc_archive if archive else None, + gitea_release=gitea_release_via_api if gitea else None, + bib_item=bib_release_item if bib else None, + ) + except (ValueError, FileExistsError) as e: + typer.echo(f"refused: {e}", err=True) + raise typer.Exit(1) from e + for k, v in out.items(): + typer.echo(f"{k}\t{v}") + + @release_app.command("verify") def release_verify( path: str = typer.Argument(..., help="Path to a release bag directory."), diff --git a/src/lead/release/publish.py b/src/lead/release/publish.py new file mode 100644 index 0000000..fa1e1f5 --- /dev/null +++ b/src/lead/release/publish.py @@ -0,0 +1,235 @@ +"""Publish a verified release bag: serve it, archive it, announce it. + +``publish`` copies the bag under the serving root (``data/lead/releases``, +served read-only at ``reach./releases/`` behind the same SSO as +reach), rewrites the release index, archives a tarball to rustfs, creates +the Gitea release ``lead-review-v`` at the bag's commit, and adds a +bib item. A published version is never replaced. Every outward step is a +callable so tests and dry runs stay hermetic. +""" + +from __future__ import annotations + +import html +import json +import shutil +from pathlib import Path +from typing import Callable + +from lead.release.build import verify_release + +Archive = Callable[[Path, str], None] + + +def read_bag_info(bag: Path) -> dict[str, str]: + """``bag-info.txt`` as a dict; folded continuation lines are joined.""" + info: dict[str, str] = {} + last = "" + for line in (bag / "bag-info.txt").read_text(encoding="utf-8").splitlines(): + if line[:1] in (" ", "\t") and last: + info[last] = f"{info[last]} {line.strip()}" + elif ":" in line: + label, value = line.split(":", 1) + last = label.strip() + info[last] = value.strip() + return info + + +def _release_rows(serve_root: Path, base_url: str) -> list[dict]: + rows = [] + for bag in sorted(serve_root.glob("lead-review-v*")): + if not (bag / "bag-info.txt").is_file(): + continue + info = read_bag_info(bag) + rows.append( + { + "version": info.get("Version", ""), + "name": bag.name, + "date": info.get("Bagging-Date", ""), + "commit": info.get("Git-Commit", ""), + "size": info.get("Bag-Size", ""), + "payload_oxum": info.get("Payload-Oxum", ""), + "note": info.get("Internal-Sender-Description", ""), + "url": f"{base_url.rstrip('/')}/{bag.name}/", + } + ) + + def key(row: dict) -> tuple: + return tuple(int(p) if p.isdigit() else 0 for p in row["version"].split(".")) + + return sorted(rows, key=key, reverse=True) + + +def write_index(serve_root: Path, *, base_url: str) -> list[dict]: + """``index.json`` and a plain ``index.html`` listing every served + release, newest first.""" + rows = _release_rows(serve_root, base_url) + (serve_root / "index.json").write_text( + json.dumps({"releases": rows}, indent=1) + "\n", encoding="utf-8" + ) + items = "\n".join( + f'
  • {html.escape(r["name"])} — ' + f"{html.escape(r['date'])}, {html.escape(r['size'])}, commit " + f"{html.escape(r['commit'][:12])}. {html.escape(r['note'])}
    • " + for r in rows + ) + (serve_root / "index.html").write_text( + "LEAD living review releases" + "

    LEAD living review releases

    " + "

    Each release is a BagIt bag with RO-Crate metadata. Protocol: " + 'OSF 8WQ3M.

    ' + f"
      \n{items}\n
    \n", + encoding="utf-8", + ) + return rows + + +def publish( + bag: Path, + *, + serve_root: Path, + base_url: str, + archive: Archive | None, + gitea_release: Callable[..., str] | None, + bib_item: Callable[..., str] | None, +) -> dict[str, str]: + """Publish *bag*; raises before any outward step when it does not + verify or its version is already published.""" + problems = verify_release(bag) + if problems: + raise ValueError(f"{bag} does not verify: " + "; ".join(problems[:5])) + info = read_bag_info(bag) + version = info.get("Version", "") + if not version: + raise ValueError(f"{bag}: bag-info.txt has no Version") + serve_root.mkdir(parents=True, exist_ok=True) + served = serve_root / bag.name + if served.resolve() != bag.resolve(): + if served.exists(): + raise FileExistsError(f"{served} is already published") + shutil.copytree(bag, served) + url = f"{base_url.rstrip('/')}/{bag.name}/" + write_index(serve_root, base_url=base_url) + out = {"version": version, "url": url, "served": str(served)} + if archive is not None: + archive(served, bag.name) + out["archived"] = f"lead-releases/{bag.name}.tar" + if gitea_release is not None: + out["gitea"] = gitea_release( + tag=f"lead-review-v{version}", + commit=info.get("Git-Commit", ""), + name=f"LEAD living review {version}", + body=( + f"{info.get('Internal-Sender-Description', '')}\n\n" + f"Bag: {url}\nPayload-Oxum: {info.get('Payload-Oxum', '')}\n" + "Protocol: https://doi.org/10.17605/OSF.IO/8WQ3M" + ), + ) + if bib_item is not None: + out["bib"] = bib_item( + title=f"LEAD living review release {version}", + url=url, + version=version, + date=info.get("Bagging-Date", ""), + note=info.get("Internal-Sender-Description", ""), + crate=served / "data" / "ro-crate-metadata.json", + ) + return out + + +# ── live adapters (rustfs via the mc container, Gitea API, bib) ───────── + + +def rc_archive(served: Path, name: str) -> None: # pragma: no cover — live rustfs + """Archive *served* as one tarball in rustfs bucket ``lead-releases`` + through the ``mc`` container's authenticated ``rc`` client.""" + import subprocess + import tarfile + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + tar_path = Path(tmp) / f"{name}.tar" + with tarfile.open(tar_path, "w") as tar: + tar.add(served, arcname=name) + steps = [ + ["docker", "cp", str(tar_path), f"mc:/tmp/{name}.tar"], + [ + "docker", + "exec", + "mc", + "rc", + "bucket", + "create", + "--ignore-existing", + "local/lead-releases", + ], + [ + "docker", + "exec", + "mc", + "rc", + "object", + "copy", + f"/tmp/{name}.tar", + f"local/lead-releases/{name}.tar", + ], + ["docker", "exec", "mc", "rm", "-f", f"/tmp/{name}.tar"], + ] + for cmd in steps: + subprocess.run(cmd, check=True, capture_output=True, timeout=600) + + +def gitea_release_via_api( # pragma: no cover — live Gitea + *, + tag: str, + commit: str, + name: str, + body: str, + owner: str = "homelab", + repo: str = "stack", +) -> str: + """Create the Gitea release (and its tag) at *commit*; an existing + release for *tag* is returned as is.""" + import os + + import httpx + + from api.clients.gitea.client import GiteaClient + + client = GiteaClient(os.environ["GITEA_TOKEN"]) + try: + found = client.get(f"/repos/{owner}/{repo}/releases/tags/{tag}") + return found.json().get("html_url", "") + except Exception: # noqa: BLE001 — 404: not released yet + pass + resp = client.post( + f"/repos/{owner}/{repo}/releases", + json={"tag_name": tag, "target_commitish": commit, "name": name, "body": body}, + ) + if isinstance(resp, httpx.Response): + return resp.json().get("html_url", "") + return "" + + +def bib_release_item( # pragma: no cover — live bib + *, title: str, url: str, version: str, date: str, note: str, crate: Path +) -> str: + """A bib ``download`` item for the release, with its RO-Crate attached.""" + from bib import connect + from bib.item import Download + + store = connect() + item = Download( + title=title, + url=url, + date_published=date, + institution="LEAD living review", + page_type="lead_release", + file_urls=[url], + extra=note, + ) + for tag in ("source:lead-release", "project:lead", f"lead-release:{version}"): + item.add_tag(tag) + key = store.upsert(item) + store.attach_file(key, crate, title=f"{title} — RO-Crate metadata") + return key diff --git a/tests/lead/test_release_publish.py b/tests/lead/test_release_publish.py new file mode 100644 index 0000000..6f720df --- /dev/null +++ b/tests/lead/test_release_publish.py @@ -0,0 +1,191 @@ +"""stack lead release publish — serve, archive, announce a verified bag.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from lead.release.bagit import write_bag +from lead.release.publish import publish, read_bag_info, write_index + + +def _bag(root: Path, version: str = "0.1") -> Path: + bag = root / f"lead-review-v{version}" + (bag / "data").mkdir(parents=True) + (bag / "data" / "protocol.md").write_text("# protocol\n") + (bag / "data" / "ro-crate-metadata.json").write_text( + json.dumps( + { + "@context": "https://w3id.org/ro/crate/1.2/context", + "@graph": [ + { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "about": {"@id": "./"}, + }, + { + "@id": "./", + "@type": "Dataset", + "name": "LEAD living review", + "description": "test release", + "datePublished": "2026-11-11", + "license": { + "@id": "https://creativecommons.org/licenses/by/4.0/" + }, + "hasPart": [{"@id": "protocol.md"}], + }, + { + "@id": "protocol.md", + "@type": "File", + "sha256": __import__("hashlib") + .sha256(b"# protocol\n") + .hexdigest(), + "contentSize": "11", + }, + ], + } + ) + ) + write_bag( + bag, + [ + ("Version", version), + ("Git-Commit", "a" * 40), + ("Bagging-Date", "2026-11-11"), + ("Internal-Sender-Description", "first release"), + ], + ) + return bag + + +class Calls: + def __init__(self) -> None: + self.archived: list[tuple[Path, str]] = [] + self.releases: list[dict] = [] + self.bib: list[dict] = [] + + def archive(self, bag: Path, name: str) -> None: + self.archived.append((bag, name)) + + def gitea_release(self, **kw) -> str: + self.releases.append(kw) + return "https://git.test/rel/1" + + def bib_item(self, **kw) -> str: + self.bib.append(kw) + return "KEY12345" + + +def test_read_bag_info_joins_continuation_lines(tmp_path): + bag = _bag(tmp_path) + info = read_bag_info(bag) + assert info["Version"] == "0.1" and info["Git-Commit"] == "a" * 40 + assert info["Internal-Sender-Description"] == "first release" + + +def test_publish_serves_archives_and_announces(tmp_path): + bag = _bag(tmp_path / "build") + serve = tmp_path / "serve" + calls = Calls() + out = publish( + bag, + serve_root=serve, + base_url="https://reach.test/releases", + archive=calls.archive, + gitea_release=calls.gitea_release, + bib_item=calls.bib_item, + ) + served = serve / bag.name + assert (served / "bagit.txt").exists() + assert calls.archived == [(served, bag.name)] + assert calls.releases[0]["tag"] == "lead-review-v0.1" + assert calls.releases[0]["commit"] == "a" * 40 + assert calls.bib[0]["url"] == "https://reach.test/releases/lead-review-v0.1/" + index = json.loads((serve / "index.json").read_text()) + assert index["releases"][0]["version"] == "0.1" + assert out["url"] == "https://reach.test/releases/lead-review-v0.1/" + + +def test_publish_refuses_a_bag_that_does_not_verify(tmp_path): + bag = _bag(tmp_path / "build") + (bag / "data" / "protocol.md").write_text("tampered\n") + calls = Calls() + with pytest.raises(ValueError, match="does not verify"): + publish( + bag, + serve_root=tmp_path / "serve", + base_url="https://reach.test/releases", + archive=calls.archive, + gitea_release=calls.gitea_release, + bib_item=calls.bib_item, + ) + assert not calls.archived and not calls.releases + + +def test_publish_never_overwrites_a_published_version(tmp_path): + calls = Calls() + kw = dict( + serve_root=tmp_path / "serve", + base_url="https://reach.test/releases", + archive=calls.archive, + gitea_release=calls.gitea_release, + bib_item=calls.bib_item, + ) + publish(_bag(tmp_path / "a"), **kw) + other = _bag(tmp_path / "b") + (other / "data" / "protocol.md").write_text("# protocol\n") # same name + with pytest.raises(FileExistsError): + publish(other, **kw) + + +def test_index_lists_every_served_release_newest_first(tmp_path): + serve = tmp_path / "serve" + for v in ("0.1", "0.2"): + bag = _bag(tmp_path / f"b{v}", v) + (serve).mkdir(exist_ok=True) + bag.rename(serve / bag.name) + write_index(serve, base_url="https://reach.test/releases") + index = json.loads((serve / "index.json").read_text()) + assert [r["version"] for r in index["releases"]] == ["0.2", "0.1"] + assert (serve / "index.html").read_text().count("lead-review-v") >= 2 + + +def test_cli_publish_offline(tmp_path): + from typer.testing import CliRunner + + from cli.lead import app + + bag = _bag(tmp_path / "build") + res = CliRunner().invoke( + app, + [ + "release", + "publish", + str(bag), + "--serve-root", + str(tmp_path / "serve"), + "--base-url", + "https://reach.test/releases", + "--no-archive", + "--no-gitea", + "--no-bib", + ], + ) + assert res.exit_code == 0, res.output + assert "url\thttps://reach.test/releases/lead-review-v0.1/" in res.output + again = CliRunner().invoke( + app, + [ + "release", + "publish", + str(bag), + "--serve-root", + str(tmp_path / "serve"), + "--no-archive", + "--no-gitea", + "--no-bib", + ], + ) + assert again.exit_code == 1 and "already published" in again.output