From bf9485e6342c81978b2ff36b6faf6a320a48e58f Mon Sep 17 00:00:00 2001 From: kert Date: Wed, 11 Nov 2026 15:09:46 -0500 Subject: [PATCH] =?UTF-8?q?feat(lead):=20stack=20lead=20release=20build/ve?= =?UTF-8?q?rify=20=E2=80=94=20BagIt=20+=20RO-Crate=20review=20releases?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each LEAD living-review update becomes a BagIt 1.0 bag (sha512 manifests, Payload-Oxum, bag-info with the OSF registration DOI and git commit) holding the protocol (OSF registration 8wq3m, verbatim and rendered), the evidence ledger tables, human screening decisions and PRISMA flow counts, included/screened bibliography as CSL-JSON, the theme vocabulary, and a source-only code archive (src, config, dev/scripts, specs — no third-party documents). RO-Crate 1.2 metadata describes every payload file with its sha256 and records the build. Reviewer email addresses are replaced with reviewer-N. Builds are deterministic for the same inputs; verify checks manifests, Oxum and the crate. Stdlib only, no new dependencies. --- src/cli/lead.py | 112 ++++++ src/lead/release/__init__.py | 14 + src/lead/release/bagit.py | 269 +++++++++++++ src/lead/release/build.py | 321 ++++++++++++++++ src/lead/release/content.py | 452 ++++++++++++++++++++++ src/lead/release/crate.py | 234 +++++++++++ src/lead/release/protocol.py | 161 ++++++++ tests/lead/test_release_bagit.py | 150 ++++++++ tests/lead/test_release_build.py | 639 +++++++++++++++++++++++++++++++ tests/lead/test_release_crate.py | 307 +++++++++++++++ 10 files changed, 2659 insertions(+) create mode 100644 src/lead/release/__init__.py create mode 100644 src/lead/release/bagit.py create mode 100644 src/lead/release/build.py create mode 100644 src/lead/release/content.py create mode 100644 src/lead/release/crate.py create mode 100644 src/lead/release/protocol.py create mode 100644 tests/lead/test_release_bagit.py create mode 100644 tests/lead/test_release_build.py create mode 100644 tests/lead/test_release_crate.py diff --git a/src/cli/lead.py b/src/cli/lead.py index ceff1ab..8333fba 100644 --- a/src/cli/lead.py +++ b/src/cli/lead.py @@ -16,8 +16,12 @@ from lead.targets import TARGETS app = typer.Typer(no_args_is_help=True) ledger_app = typer.Typer(no_args_is_help=True) manifest_app = typer.Typer(no_args_is_help=True) +release_app = typer.Typer(no_args_is_help=True) app.add_typer(ledger_app, name="ledger", help="Evidence ledger.") app.add_typer(manifest_app, name="manifest", help="Public-release manifest.") +app.add_typer( + release_app, name="release", help="Review releases (BagIt bag + RO-Crate)." +) @app.command(name="targets") @@ -164,3 +168,111 @@ def load_data( ): if counts[status]: typer.echo(f"{status}\t{counts[status]}") + + +@release_app.command("build") +def release_build( + version: str = typer.Option(..., "--version", help="Release version, MAJOR.MINOR."), + out: str = typer.Option( + "", "--out", help="Directory for the bag (default data/lead/releases)." + ), + date: str = typer.Option( + "", "--date", help="Bagging-Date and datePublished, YYYY-MM-DD (default today)." + ), + note: str = typer.Option( + "", "--note", help="One line on what changed (Internal-Sender-Description)." + ), + registration: str = typer.Option( + "", + "--registration", + help="Cached OSF registration JSON; default fetches 8wq3m from api.osf.io.", + ), + ledger: str = typer.Option( + "", "--ledger", help="DuckDB ledger (default data/lead/ledger.duckdb)." + ), + labels: str = typer.Option( + "", + "--labels", + help="Screening labels (default data/lead/review/labels.sqlite).", + ), + zotero: str = typer.Option( + "", "--zotero", help="Zotero sqlite, read-only (default [db] zotero)." + ), + vocab: str = typer.Option( + "", "--vocab", help="Theme vocabulary (default src/llm/vocab/themes.yaml)." + ), + repo: str = typer.Option("", "--repo", help="Git repo to archive (default stack)."), + source_org: str = typer.Option( + "", + "--source-org", + help="Source-Organization for bag-info.txt (omitted if blank).", + ), + force: bool = typer.Option(False, "--force", help="Replace an existing bag."), +) -> None: + """Write a release bag. Reads the ledger, labels and Zotero read-only.""" + from pathlib import Path + + from conf import ROOT + from conf import path as conf_path + from lead.release import Inputs, build_release + from lead.release.protocol import USER_AGENT, fetch_registration, load_registration + + if registration: + reg = load_registration(registration) + else: + import httpx + + with httpx.Client( + headers={"User-Agent": USER_AGENT}, timeout=60, follow_redirects=True + ) as client: + reg = fetch_registration(client) + inputs = Inputs( + ledger=Path(ledger) if ledger else ROOT / "data" / "lead" / "ledger.duckdb", + labels=Path(labels) + if labels + else ROOT / "data" / "lead" / "review" / "labels.sqlite", + zotero=Path(zotero) if zotero else conf_path("db.zotero"), + vocab=Path(vocab) if vocab else ROOT / "src" / "llm" / "vocab" / "themes.yaml", + repo=Path(repo) if repo else ROOT, + registration=reg, + ) + try: + result = build_release( + version=version, + out=Path(out) if out else ROOT / "data" / "lead" / "releases", + inputs=inputs, + date=date, + note=note, + source_org=source_org, + force=force, + ) + except (ValueError, FileExistsError, FileNotFoundError) as exc: + typer.echo(f"error: {exc}", err=True) + raise typer.Exit(2) from exc + for warning in result.warnings: + typer.echo(f"warning: {warning}", err=True) + typer.echo(f"built\t{result.path}") + typer.echo(f"files\t{result.files}") + typer.echo(f"bytes\t{result.bytes}") + typer.echo(f"commit\t{result.commit}") + + +@release_app.command("verify") +def release_verify( + path: str = typer.Argument(..., help="Path to a release bag directory."), +) -> None: + """Check a bag: BagIt manifests and Payload-Oxum, then the RO-Crate.""" + from pathlib import Path + + from lead.release import verify_release + from lead.release.bagit import payload_files + + problems = verify_release(Path(path)) + if problems: + for problem in problems: + typer.echo(problem, err=True) + typer.echo(f"{len(problems)} problem(s) in {path}", err=True) + raise typer.Exit(1) + files = payload_files(Path(path)) + size = sum((Path(path) / rel).stat().st_size for rel in files) + typer.echo(f"ok\t{path}\t{len(files)} payload files\t{size} bytes") diff --git a/src/lead/release/__init__.py b/src/lead/release/__init__.py new file mode 100644 index 0000000..0df2bbc --- /dev/null +++ b/src/lead/release/__init__.py @@ -0,0 +1,14 @@ +"""LEAD review releases: one BagIt 1.0 bag per update, RO-Crate 1.2 inside. + +``stack lead release build --version X.Y`` writes the bag from the +evidence ledger, the screening labels, the Zotero library and the frozen +OSF registration (all read-only); ``stack lead release verify PATH`` +checks its manifests, Payload-Oxum and crate. Design: +docs/superpowers/specs/2026-11-11-lead-review-releases-and-tagging-design.md +""" + +from __future__ import annotations + +from lead.release.build import BuildResult, Inputs, build_release, verify_release + +__all__ = ["BuildResult", "Inputs", "build_release", "verify_release"] diff --git a/src/lead/release/bagit.py b/src/lead/release/bagit.py new file mode 100644 index 0000000..88d5c93 --- /dev/null +++ b/src/lead/release/bagit.py @@ -0,0 +1,269 @@ +"""BagIt 1.0 (RFC 8493) writer and validator, stdlib only. + +A bag is a directory with ``bagit.txt``, a ``data/`` payload, a payload +manifest per algorithm (``manifest-.txt``), ``bag-info.txt`` and a +tag manifest (``tagmanifest-.txt``) over the other tag files. The +writer uses sha512; the validator checks every manifest it finds. +""" + +from __future__ import annotations + +import hashlib +import re +import textwrap +from collections.abc import Sequence +from pathlib import Path, PurePosixPath + +ALGORITHM = "sha512" +BAGIT_TXT = "BagIt-Version: 1.0\nTag-File-Character-Encoding: UTF-8\n" +_CHUNK = 1 << 20 +_WIDTH = 79 +_MANIFEST = re.compile(r"^(?:tag)?manifest-([a-z0-9]+)\.txt$") +_LINE = re.compile(r"^([0-9a-fA-F]+)[ \t]+(.+)$") + + +def encode_path(path: str) -> str: + """Percent-encode the three characters a manifest line cannot hold.""" + return path.replace("%", "%25").replace("\n", "%0A").replace("\r", "%0D") + + +def decode_path(path: str) -> str: + """Inverse of :func:`encode_path`.""" + out = re.sub(r"%0[aA]", "\n", path) + out = re.sub(r"%0[dD]", "\r", out) + return out.replace("%25", "%") + + +def file_digest(path: Path, algorithm: str = ALGORITHM) -> str: + h = hashlib.new(algorithm) + with path.open("rb") as fh: + while chunk := fh.read(_CHUNK): + h.update(chunk) + return h.hexdigest() + + +def payload_files(bag: Path) -> list[str]: + """Every regular file under ``data/``, as sorted bag-relative paths.""" + data = bag / "data" + if not data.is_dir(): + return [] + return sorted( + p.relative_to(bag).as_posix() + for p in data.rglob("*") + if p.is_file() or p.is_symlink() + ) + + +def human_size(octets: int) -> str: + """``Bag-Size`` text: decimal units, one decimal place above bytes.""" + if octets < 1000: + return f"{octets} B" + size = float(octets) + for unit in ("KB", "MB", "GB", "TB"): + size /= 1000 + if size < 1000 or unit == "TB": + return f"{size:.1f} {unit}" + raise AssertionError("unreachable") + + +def _manifest_text(bag: Path, files: Sequence[str], algorithm: str) -> str: + return "".join( + f"{file_digest(bag / rel, algorithm)} {encode_path(rel)}\n" for rel in files + ) + + +def _info_text(info: Sequence[tuple[str, str]]) -> str: + lines: list[str] = [] + for label, value in info: + if ":" in label or label != label.strip() or not label: + raise ValueError(f"bad bag-info label: {label!r}") + text = " ".join(str(value).split()) + lines.extend( + textwrap.wrap( + f"{label}: {text}", + width=_WIDTH, + subsequent_indent=" ", + break_long_words=False, + break_on_hyphens=False, + ) + or [f"{label}:"] + ) + return "".join(f"{line}\n" for line in lines) + + +def write_bag(bag: Path, info: Sequence[tuple[str, str]]) -> tuple[int, int]: + """Turn a directory holding ``data/`` into a bag. Returns (octets, files). + + ``info`` is written to ``bag-info.txt`` in order, with ``Bag-Size`` and + ``Payload-Oxum`` inserted after ``Bagging-Date`` (or appended when there + is none). Values are whitespace-normalised and folded at 79 columns. + """ + files = payload_files(bag) + octets = sum((bag / rel).stat().st_size for rel in files) + computed = [ + ("Bag-Size", human_size(octets)), + ("Payload-Oxum", f"{octets}.{len(files)}"), + ] + labels = [label for label, _value in info] + if "Bag-Size" in labels or "Payload-Oxum" in labels: + raise ValueError("Bag-Size and Payload-Oxum are computed") + full = list(info) + at = labels.index("Bagging-Date") + 1 if "Bagging-Date" in labels else len(full) + full[at:at] = computed + + (bag / "bagit.txt").write_text(BAGIT_TXT, encoding="utf-8", newline="\n") + manifest = f"manifest-{ALGORITHM}.txt" + (bag / manifest).write_text( + _manifest_text(bag, files, ALGORITHM), encoding="utf-8", newline="\n" + ) + (bag / "bag-info.txt").write_text(_info_text(full), encoding="utf-8", newline="\n") + tag_files = sorted(["bagit.txt", "bag-info.txt", manifest]) + (bag / f"tagmanifest-{ALGORITHM}.txt").write_text( + _manifest_text(bag, tag_files, ALGORITHM), encoding="utf-8", newline="\n" + ) + return octets, len(files) + + +def read_bag_info(bag: Path) -> list[tuple[str, str]]: + """``bag-info.txt`` as (label, value) pairs; folded lines are joined.""" + path = bag / "bag-info.txt" + pairs: list[tuple[str, str]] = [] + for raw in path.read_text(encoding="utf-8").splitlines(): + if not raw.strip(): + continue + if raw[0] in " \t": + if not pairs: + raise ValueError("bag-info.txt starts with a continuation line") + label, value = pairs[-1] + pairs[-1] = (label, f"{value} {raw.strip()}".strip()) + continue + label, sep, value = raw.partition(":") + if not sep: + raise ValueError(f"bag-info.txt line has no label: {raw!r}") + pairs.append((label.strip(), value.strip())) + return pairs + + +def _check_bagit_txt(bag: Path) -> list[str]: + path = bag / "bagit.txt" + if not path.is_file(): + return ["bagit.txt is missing"] + try: + lines = path.read_text(encoding="utf-8").splitlines() + except UnicodeDecodeError: + return ["bagit.txt is not UTF-8"] + problems = [] + if len(lines) != 2: + problems.append(f"bagit.txt has {len(lines)} lines, expected 2") + if not lines or not re.fullmatch(r"BagIt-Version: \d+\.\d+", lines[0]): + problems.append("bagit.txt: first line is not 'BagIt-Version: M.N'") + if len(lines) < 2 or lines[1] != "Tag-File-Character-Encoding: UTF-8": + problems.append( + "bagit.txt: second line is not 'Tag-File-Character-Encoding: UTF-8'" + ) + return problems + + +def _read_manifest(bag: Path, name: str, problems: list[str]) -> dict[str, str]: + entries: dict[str, str] = {} + text = (bag / name).read_text(encoding="utf-8") + for n, raw in enumerate(text.splitlines(), start=1): + if not raw.strip(): + continue + match = _LINE.match(raw) + if match is None: + problems.append(f"{name} line {n} is not ' '") + continue + rel = decode_path(match.group(2).strip()) + parts = PurePosixPath(rel).parts + if rel.startswith("/") or ".." in parts or "\\" in rel: + problems.append(f"{name} line {n}: unsafe path {rel!r}") + continue + if rel in entries: + problems.append(f"{name}: {rel} is listed twice") + entries[rel] = match.group(1).lower() + return entries + + +def _check_entries( + bag: Path, + name: str, + algorithm: str, + entries: dict[str, str], + problems: list[str], +) -> None: + for rel, expected in sorted(entries.items()): + path = bag / rel + if not path.is_file(): + problems.append(f"{rel}: listed in {name} but missing") + continue + if file_digest(path, algorithm) != expected: + problems.append(f"{rel}: {algorithm} in {name} does not match") + + +def validate_bag(bag: Path) -> list[str]: + """Every problem that makes ``bag`` incomplete or invalid. Empty is valid. + + Checks ``bagit.txt``; that each payload manifest lists every payload + file once, lists nothing outside ``data/`` and every digest matches; + that ``Payload-Oxum`` matches; and that the tag manifests cover + ``bagit.txt``, ``bag-info.txt`` and the payload manifests with + matching digests. + """ + if not bag.is_dir(): + return [f"{bag} is not a directory"] + problems = _check_bagit_txt(bag) + if not (bag / "data").is_dir(): + problems.append("data/ is missing") + if not (bag / f"manifest-{ALGORITHM}.txt").is_file(): + problems.append(f"manifest-{ALGORITHM}.txt is missing") + if not (bag / f"tagmanifest-{ALGORITHM}.txt").is_file(): + problems.append(f"tagmanifest-{ALGORITHM}.txt is missing") + + payload = payload_files(bag) + payload_set = set(payload) + payload_manifests: list[str] = [] + tag_manifests: list[str] = [] + for child in sorted(p.name for p in bag.iterdir() if p.is_file()): + match = _MANIFEST.match(child) + if match is None: + continue + algorithm = match.group(1) + if algorithm not in hashlib.algorithms_available: + problems.append(f"{child}: unsupported algorithm {algorithm}") + continue + entries = _read_manifest(bag, child, problems) + if child.startswith("tag"): + tag_manifests.append(child) + for rel in entries: + if rel.startswith("data/") or rel.startswith("tagmanifest-"): + problems.append(f"{child}: {rel} is not a tag file it can list") + else: + payload_manifests.append(child) + for rel in sorted(set(entries) - payload_set): + if not rel.startswith("data/"): + problems.append(f"{child}: {rel} is outside data/") + for rel in sorted(payload_set - set(entries)): + problems.append(f"{rel}: not in {child}") + _check_entries(bag, child, algorithm, entries, problems) + + for child in tag_manifests: + listed = set(_read_manifest(bag, child, [])) + for required in ["bagit.txt", "bag-info.txt", *payload_manifests]: + if (bag / required).is_file() and required not in listed: + problems.append(f"{required}: not in {child}") + + if (bag / "bag-info.txt").is_file(): + try: + info = dict(read_bag_info(bag)) + except (ValueError, UnicodeDecodeError) as exc: + problems.append(f"bag-info.txt: {exc}") + info = {} + oxum = info.get("Payload-Oxum") + if oxum is not None: + actual = ( + f"{sum((bag / rel).stat().st_size for rel in payload)}.{len(payload)}" + ) + if oxum != actual: + problems.append(f"Payload-Oxum is {oxum}, payload is {actual}") + return problems diff --git a/src/lead/release/build.py b/src/lead/release/build.py new file mode 100644 index 0000000..ff48977 --- /dev/null +++ b/src/lead/release/build.py @@ -0,0 +1,321 @@ +"""Build and verify a LEAD review release: a BagIt bag with RO-Crate inside. + + lead-review-v/ + bagit.txt bag-info.txt manifest-sha512.txt tagmanifest-sha512.txt + data/ + ro-crate-metadata.json + protocol/registration-8wq3m.json protocol/protocol.md + ledger/.parquet (every lead_evidence table) + screening/decisions.parquet screening/prisma-flow.json + bibliography/included.csl.json bibliography/screened.csl.json + tags/vocab-v.yaml + code/stack-.tar.gz + +Two builds from the same commit and inputs are byte-identical apart from +``Bagging-Date`` and ``datePublished``/``endTime`` (the ``date``). +""" + +from __future__ import annotations + +import json +import platform +import re +import shutil +from dataclasses import dataclass +from datetime import date as _date +from pathlib import Path + +from lead.release import content +from lead.release.bagit import validate_bag, write_bag +from lead.release.crate import build_crate, check_crate, write_crate +from lead.release.protocol import DOI, DOI_URL, render_protocol + +GROUP = "lead-living-review" +_VERSION = re.compile(r"^\d+\.\d+$") + + +@dataclass(frozen=True) +class Inputs: + """Where a build reads from. ``registration`` is the normalized OSF record.""" + + ledger: Path + labels: Path + zotero: Path + vocab: Path + repo: Path + registration: dict + + +@dataclass(frozen=True) +class BuildResult: + path: Path + files: int + bytes: int + commit: str + dirty: bool + warnings: tuple[str, ...] + + +def bag_name(version: str) -> str: + return f"lead-review-v{version}" + + +def _json(path: Path, value: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(value, indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + newline="\n", + ) + + +def _shown(path: Path, repo: Path) -> str: + """A repo-relative path, or just the file name; never a home directory.""" + try: + return path.resolve().relative_to(repo.resolve()).as_posix() + except ValueError: + return path.name + + +def _tool_versions() -> dict[str, str]: + import duckdb + import polars + + return { + "python": platform.python_version(), + "duckdb": duckdb.__version__, + "polars": polars.__version__, + } + + +def _context( + *, + version: str, + date: str, + inputs: Inputs, + commit: str, + tables: list[str], +) -> list[dict]: + """The build's CreateAction, the software it ran and the inputs it read.""" + repo = inputs.repo + tools = _tool_versions() + software = [ + { + "@id": "#stack", + "@type": "SoftwareApplication", + "name": "stack", + "version": commit, + "description": "The monorepo that built this release; its source is " + f"code/stack-{commit}.tar.gz.", + }, + *( + { + "@id": f"#{name}", + "@type": "SoftwareApplication", + "name": name, + "version": value, + } + for name, value in tools.items() + ), + ] + sources = [ + { + "@id": "#input-ledger", + "@type": "Dataset", + "name": "lead_evidence ledger (DuckDB)", + "description": f"Every table in schema lead_evidence of " + f"{_shown(inputs.ledger, repo)}: {', '.join(tables)}.", + }, + { + "@id": "#input-labels", + "@type": "Dataset", + "name": "LEAD screening labels (SQLite)", + "description": f"Table label of {_shown(inputs.labels, repo)}, every " + "row; e-mail actors replaced by reviewer-N.", + }, + { + "@id": "#input-zotero", + "@type": "Dataset", + "name": "Zotero library (SQLite)", + "description": f"{_shown(inputs.zotero, repo)}, read-only. Records " + "tagged project:lead- (not attachments, notes or " + "module:prisma anchors); screened = stage:2-4 tags plus labelled " + "keys, included = stage:4-included; flow counts from " + "prisma.flow.count.", + }, + { + "@id": "#input-vocab", + "@type": "CreativeWork", + "name": "Theme vocabulary", + "description": _shown(inputs.vocab, repo), + }, + ] + action = { + "@id": "#build", + "@type": "CreateAction", + "name": f"stack lead release build --version {version}", + "endTime": date, + "instrument": [{"@id": s["@id"]} for s in software], + "object": [{"@id": s["@id"]} for s in sources] + [{"@id": DOI_URL}], + "result": {"@id": "./"}, + } + return [action, *software, *sources] + + +def build_release( + *, + version: str, + out: Path, + inputs: Inputs, + date: str = "", + note: str = "", + source_org: str = "", + force: bool = False, +) -> BuildResult: + """Write ``out/lead-review-v/`` and return what went in. + + Raises ``ValueError`` for a version that is not ``MAJOR.MINOR`` or a + date that is not ``YYYY-MM-DD``, and ``FileExistsError`` when the bag + exists and ``force`` is not set. The bag is assembled beside its final + path and moved into place only once it is complete. + """ + if not _VERSION.fullmatch(version): + raise ValueError(f"version must be MAJOR.MINOR, got {version!r}") + date = date or _date.today().isoformat() + if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", date): + raise ValueError(f"date must be YYYY-MM-DD, got {date!r}") + _date.fromisoformat(date) + out = Path(out) + dest = out / bag_name(version) + if dest.exists() and not force: + raise FileExistsError(f"{dest} exists; pass --force to replace it") + for source in (inputs.ledger, inputs.labels, inputs.zotero, inputs.vocab): + if not Path(source).is_file(): + raise FileNotFoundError(source) + + out.mkdir(parents=True, exist_ok=True) + stage = out / f".{bag_name(version)}.partial" + if stage.exists(): + shutil.rmtree(stage) + data = stage / "data" + data.mkdir(parents=True) + warnings: list[str] = [] + try: + reg = inputs.registration + guid = reg.get("guid", "8wq3m") + _json(data / "protocol" / f"registration-{guid}.json", reg) + (data / "protocol" / "protocol.md").write_text( + render_protocol(reg), encoding="utf-8", newline="\n" + ) + + ledger = content.read_ledger(Path(inputs.ledger)) + labels = content.read_labels(Path(inputs.labels)) + actors = [*labels["actor"].to_list()] + for frame in ledger.values(): + if "actor" in frame.columns: + actors.extend(frame["actor"].to_list()) + aliases = content.actor_aliases(actors) + for table, frame in ledger.items(): + content.write_parquet( + content.alias_actor(frame, aliases), + data / "ledger" / f"{table}.parquet", + ) + content.write_parquet( + content.alias_actor(labels, aliases), + data / "screening" / "decisions.parquet", + ) + + zotero = content.zotero_connect(Path(inputs.zotero)) + try: + _json(data / "screening" / "prisma-flow.json", content.flow_counts(zotero)) + bib = content.bibliography(zotero, labels["zotero_key"].to_list()) + finally: + zotero.close() + _json(data / "bibliography" / "included.csl.json", bib.included) + _json(data / "bibliography" / "screened.csl.json", bib.screened) + if bib.missing: + warnings.append( + f"{len(bib.missing)} labelled key(s) not in the Zotero library: " + + ", ".join(bib.missing) + ) + + content.copy_vocab(Path(inputs.vocab), data / "tags") + code = content.archive_code(Path(inputs.repo), data / "code") + if code.dirty: + warnings.append( + "the repo has uncommitted changes; the code archive is HEAD " + f"{code.commit[:12]} without them" + ) + + based_on = { + "@id": DOI_URL, + "@type": "CreativeWork", + "name": reg.get("title") or "OSF registration", + "identifier": f"doi:{DOI}", + "url": reg.get("url") or f"https://osf.io/{guid}/", + } + if reg.get("date_registered"): + based_on["dateCreated"] = str(reg["date_registered"])[:10] + description = ( + f"Release {version} of the LEAD living review registered at OSF " + f"({DOI_URL}): the protocol, the evidence ledger, screening " + "decisions and PRISMA flow counts, the bibliography of screened " + "and included records, the theme vocabulary, and the code that " + "built it." + ) + crate = build_crate( + data, + name=f"LEAD living review, release {version}", + description=description, + version=version, + date_published=date, + based_on=based_on, + entities=_context( + version=version, + date=date, + inputs=inputs, + commit=code.commit, + tables=sorted(ledger), + ), + ) + write_crate(data, crate) + + info = [] + if source_org.strip(): + info.append(("Source-Organization", source_org.strip())) + info += [ + ("External-Identifier", f"doi:{DOI}"), + ("Bag-Group-Identifier", GROUP), + ("Bagging-Date", date), + ("Version", version), + ("Git-Commit", code.commit), + ] + if note.strip(): + info.append(("Internal-Sender-Description", note.strip())) + octets, files = write_bag(stage, info) + + if dest.exists(): + shutil.rmtree(dest) + stage.rename(dest) + except BaseException: + shutil.rmtree(stage, ignore_errors=True) + raise + return BuildResult( + path=dest, + files=files, + bytes=octets, + commit=code.commit, + dirty=code.dirty, + warnings=tuple(warnings), + ) + + +def verify_release(bag: Path) -> list[str]: + """BagIt validation plus RO-Crate checks; every problem, or empty.""" + bag = Path(bag) + if not bag.is_dir(): + return [f"{bag} is not a directory"] + problems = validate_bag(bag) + if (bag / "data").is_dir(): + problems += [f"ro-crate: {p}" for p in check_crate(bag / "data")] + return problems diff --git a/src/lead/release/content.py b/src/lead/release/content.py new file mode 100644 index 0000000..1a5e885 --- /dev/null +++ b/src/lead/release/content.py @@ -0,0 +1,452 @@ +"""Read the review's state for a release. Every source is opened read-only. + +- ledger: every table in the ``lead_evidence`` schema of the DuckDB ledger +- screening: the human labels (``lead.review``) and PRISMA flow counts + (``prisma.flow`` over the Zotero tags) +- bibliography: CSL-JSON for the screened and included Zotero records +- tags: the theme vocabulary file +- code: ``git archive`` of HEAD +""" + +from __future__ import annotations + +import dataclasses +import gzip +import os +import re +import shutil +import sqlite3 +import subprocess +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from pathlib import Path +from types import SimpleNamespace + +import polars as pl + +from lead.review.codes import PROJECTS + +LEDGER_SCHEMA = "lead_evidence" +_EMAIL = re.compile(r"[^@\s]+@[^@\s]+\.[^@\s]+") + + +# ── actors ─────────────────────────────────────────────────────── + + +def actor_aliases(actors: Iterable[str | None]) -> dict[str, str]: + """``reviewer-N`` for each e-mail address, numbered in sorted order. + + A release is public; a reviewer's address is not published with it. + Non-address actors (``stack``, a model name) are kept as they are. + """ + emails = sorted({a for a in actors if a and _EMAIL.fullmatch(a.strip())}) + return {email: f"reviewer-{n}" for n, email in enumerate(emails, start=1)} + + +def alias_actor(frame: pl.DataFrame, aliases: Mapping[str, str]) -> pl.DataFrame: + if "actor" not in frame.columns or not aliases: + return frame + return frame.with_columns(pl.col("actor").replace(dict(aliases))) + + +def write_parquet(frame: pl.DataFrame, dest: Path) -> None: + dest.parent.mkdir(parents=True, exist_ok=True) + frame.write_parquet(dest, compression="zstd", statistics=True) + + +# ── ledger ─────────────────────────────────────────────────────── + + +def read_ledger(path: Path) -> dict[str, pl.DataFrame]: + """Each ``lead_evidence`` table, rows sorted on every column.""" + import duckdb + + con = duckdb.connect(str(path), read_only=True) + try: + tables = [ + row[0] + for row in con.execute( + """SELECT table_name FROM information_schema.tables + WHERE table_schema = ? ORDER BY table_name""", + [LEDGER_SCHEMA], + ).fetchall() + ] + return { + table: con.execute( + f'SELECT * FROM {LEDGER_SCHEMA}."{table}" ORDER BY ALL' + ).pl() + for table in tables + } + finally: + con.close() + + +# ── screening ──────────────────────────────────────────────────── + +_LABEL_SCHEMA = { + "label_id": pl.Int64, + "zotero_key": pl.String, + "project": pl.String, + "stage": pl.String, + "pass": pl.Int64, + "decision": pl.String, + "reason": pl.String, + "rationale": pl.String, + "actor": pl.String, + "created_at": pl.String, +} + + +def _sqlite_ro(path: Path) -> sqlite3.Connection: + """Read-only, immutable: no lock taken, no -wal/-shm file written.""" + if not path.is_file(): + raise FileNotFoundError(path) + return sqlite3.connect(f"file:{path}?mode=ro&immutable=1", uri=True) + + +def read_labels(path: Path) -> pl.DataFrame: + """Every human screening label, oldest first (the label file is append-only).""" + con = _sqlite_ro(path) + try: + rows = con.execute( + """SELECT id, zotero_key, project, stage, pass, decision, reason, + rationale, actor, created_at + FROM label ORDER BY id""" + ).fetchall() + finally: + con.close() + return pl.DataFrame(rows, schema=_LABEL_SCHEMA, orient="row") + + +def flow_counts(zotero: sqlite3.Connection) -> dict: + """PRISMA 2020 flow counts per LEAD strand and per review, from Zotero tags.""" + from prisma import flow + + db = SimpleNamespace(con=zotero) + per = {project: flow.count(db, project) for project in sorted(PROJECTS)} # type: ignore[arg-type] + scoping = flow.combined([per[p] for p in flow.SCOPING_PROJECTS]) + return { + "source": "prisma.flow.count over the Zotero tags project:, " + "stage:*, screen:*, dedup:duplicate and found-in:*", + "note": "Stage and screen tags are not scoped to a project: a record " + "shared with another review in the library is counted with that " + "review's stage tags. A record on two scoping strands is counted " + "twice in the scoping total.", + "projects": {p: dataclasses.asdict(c) for p, c in per.items()}, + "reviews": { + "scoping": { + "projects": list(flow.SCOPING_PROJECTS), + **dataclasses.asdict(scoping), + }, + "systematic": { + "projects": [flow.SYSTEMATIC_PROJECT], + **dataclasses.asdict(per[flow.SYSTEMATIC_PROJECT]), + }, + }, + } + + +# ── bibliography ───────────────────────────────────────────────── + +_SKIP_TYPES = ("attachment", "note", "annotation") +_SCREENED_STAGES = ("stage:2-title-abstract", "stage:3-full-text", "stage:4-included") +_INCLUDED = "stage:4-included" + +# Zotero item type -> CSL type. +CSL_TYPES = { + "artwork": "graphic", + "audioRecording": "song", + "bill": "bill", + "blogPost": "post-weblog", + "book": "book", + "bookSection": "chapter", + "case": "legal_case", + "computerProgram": "software", + "conferencePaper": "paper-conference", + "dataset": "dataset", + "dictionaryEntry": "entry-dictionary", + "document": "document", + "email": "personal_communication", + "encyclopediaArticle": "entry-encyclopedia", + "film": "motion_picture", + "forumPost": "post", + "hearing": "hearing", + "instantMessage": "personal_communication", + "interview": "interview", + "journalArticle": "article-journal", + "letter": "personal_communication", + "magazineArticle": "article-magazine", + "manuscript": "manuscript", + "map": "map", + "newspaperArticle": "article-newspaper", + "patent": "patent", + "podcast": "song", + "preprint": "article", + "presentation": "speech", + "radioBroadcast": "broadcast", + "report": "report", + "standard": "standard", + "statute": "legislation", + "thesis": "thesis", + "tvBroadcast": "broadcast", + "videoRecording": "motion_picture", + "webpage": "webpage", +} +_FIELDS = ("title", "nameOfAct", "caseName", "date", "dateEnacted", "DOI", "url") +_SQL_DATE = re.compile(r"^(\d{4})-(\d{2})-(\d{2})") +_YEAR = re.compile(r"\b(\d{4})\b") + + +@dataclass(frozen=True) +class Bibliography: + included: list[dict] + screened: list[dict] + missing: tuple[str, ...] + + +def issued(date: str) -> dict | None: + """CSL ``issued`` from a Zotero date (``YYYY-MM-DD original``).""" + match = _SQL_DATE.match(date.strip()) + if match: + parts = [int(p) for p in match.groups()] + while parts and parts[-1] == 0: + parts.pop() + return {"date-parts": [parts]} if parts else None + year = _YEAR.search(date) + return {"date-parts": [[int(year.group(1))]]} if year else None + + +def _lead_items(con: sqlite3.Connection) -> dict[int, tuple[str, str]]: + """itemID -> (key, type) for records in a LEAD strand (no anchors).""" + tags = [f"project:{p}" for p in PROJECTS] + rows = con.execute( + f"""SELECT DISTINCT i.itemID, i.key, ty.typeName + FROM items i + JOIN itemTypes ty ON ty.itemTypeID = i.itemTypeID + JOIN itemTags it ON it.itemID = i.itemID + JOIN tags t ON t.tagID = it.tagID + WHERE t.name IN ({",".join("?" * len(tags))}) + AND ty.typeName NOT IN (?, ?, ?) + AND i.itemID NOT IN ( + SELECT it2.itemID FROM itemTags it2 + JOIN tags t2 ON t2.tagID = it2.tagID + WHERE t2.name = 'module:prisma')""", + (*tags, *_SKIP_TYPES), + ).fetchall() + return {row[0]: (row[1], row[2]) for row in rows} + + +def _tagged( + con: sqlite3.Connection, ids: Iterable[int], names: Iterable[str] +) -> set[int]: + wanted = list(names) + ids = list(ids) + if not ids or not wanted: + return set() + found: set[int] = set() + for start in range(0, len(ids), 500): + chunk = ids[start : start + 500] + rows = con.execute( + f"""SELECT DISTINCT it.itemID FROM itemTags it + JOIN tags t ON t.tagID = it.tagID + WHERE t.name IN ({",".join("?" * len(wanted))}) + AND it.itemID IN ({",".join("?" * len(chunk))})""", + (*wanted, *chunk), + ).fetchall() + found.update(row[0] for row in rows) + return found + + +def _by_key(con: sqlite3.Connection, keys: Iterable[str]) -> dict[str, tuple[int, str]]: + keys = sorted(set(keys)) + out: dict[str, tuple[int, str]] = {} + for start in range(0, len(keys), 500): + chunk = keys[start : start + 500] + rows = con.execute( + f"""SELECT i.itemID, i.key, ty.typeName FROM items i + JOIN itemTypes ty ON ty.itemTypeID = i.itemTypeID + WHERE i.key IN ({",".join("?" * len(chunk))}) + AND ty.typeName NOT IN (?, ?, ?)""", + (*chunk, *_SKIP_TYPES), + ).fetchall() + out.update({row[1]: (row[0], row[2]) for row in rows}) + return out + + +def csl_item(con: sqlite3.Connection, item_id: int, key: str, type_name: str) -> dict: + """One record as CSL-JSON: id, type, title, author, issued, DOI, URL.""" + fields = dict( + con.execute( + f"""SELECT f.fieldName, v.value FROM itemData d + JOIN fields f ON f.fieldID = d.fieldID + JOIN itemDataValues v ON v.valueID = d.valueID + WHERE d.itemID = ? AND f.fieldName IN ({",".join("?" * len(_FIELDS))})""", + (item_id, *_FIELDS), + ).fetchall() + ) + authors = [] + for first, last, mode in con.execute( + """SELECT c.firstName, c.lastName, c.fieldMode FROM itemCreators ic + JOIN creators c ON c.creatorID = ic.creatorID + JOIN creatorTypes ct ON ct.creatorTypeID = ic.creatorTypeID + WHERE ic.itemID = ? AND ct.creatorType = 'author' + ORDER BY ic.orderIndex""", + (item_id,), + ).fetchall(): + if mode == 1 or not first: + if last: + authors.append({"literal": last}) + elif last: + authors.append({"family": last, "given": first}) + else: + authors.append({"literal": first}) + entry: dict = {"id": key, "type": CSL_TYPES.get(type_name, "document")} + title = fields.get("title") or fields.get("nameOfAct") or fields.get("caseName") + if title: + entry["title"] = str(title) + if authors: + entry["author"] = authors + when = issued(str(fields.get("date") or fields.get("dateEnacted") or "")) + if when: + entry["issued"] = when + if fields.get("DOI"): + entry["DOI"] = str(fields["DOI"]) + if fields.get("url"): + entry["URL"] = str(fields["url"]) + return entry + + +def bibliography(con: sqlite3.Connection, labelled: Iterable[str]) -> Bibliography: + """CSL-JSON for the LEAD records, sorted by Zotero key. + + Screened: every record with a human screening label, plus every LEAD + record that carries a stage 2-4 tag (the PRISMA ``screened`` box). + Included: LEAD records tagged ``stage:4-included``. Labelled keys that + are not in the library come back in ``missing``. + """ + lead = _lead_items(con) + staged = _tagged(con, lead, _SCREENED_STAGES) + included_ids = _tagged(con, lead, [_INCLUDED]) + labelled = set(labelled) + known = _by_key(con, labelled) + screened: dict[str, tuple[int, str]] = {lead[i][0]: (i, lead[i][1]) for i in staged} + screened.update(known) + missing = tuple(sorted(labelled - set(known))) + entries = { + key: csl_item(con, item_id, key, type_name) + for key, (item_id, type_name) in screened.items() + } + included_keys = {lead[i][0] for i in included_ids} + return Bibliography( + included=[entries[k] for k in sorted(included_keys)], + screened=[entries[k] for k in sorted(entries)], + missing=missing, + ) + + +def zotero_connect(path: Path) -> sqlite3.Connection: + return _sqlite_ro(path) + + +# ── tags ───────────────────────────────────────────────────────── + + +def vocab_version(path: Path) -> str: + import yaml + + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + version = data.get("version") if isinstance(data, dict) else None + if version in (None, ""): + raise ValueError(f"{path} has no version") + return str(version) + + +def copy_vocab(path: Path, dest_dir: Path) -> Path: + dest = dest_dir / f"vocab-v{vocab_version(path)}.yaml" + dest_dir.mkdir(parents=True, exist_ok=True) + shutil.copyfile(path, dest) + return dest + + +# ── code ───────────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class Code: + commit: str + dirty: bool + path: Path + + +def _git_env() -> dict[str, str]: + # A pre-commit hook exports GIT_DIR / GIT_INDEX_FILE; drop them so the + # command reads the repo it is pointed at. + return {k: v for k, v in os.environ.items() if not k.startswith("GIT_")} + + +def _git(repo: Path, *args: str) -> str: + return subprocess.run( + ["git", "-C", str(repo), *args], + env=_git_env(), + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +#: What a release's code archive carries: the code and config that build +#: it. Everything else tracked in the repo (vendor manuals under hw/docs, +#: books under dev/seeds, BCDA material) is third-party and stays out of a +#: public bag. +CODE_PATHS = ( + "src", + "pyproject.toml", + "uv.lock", + "stack.toml", + "dev/scripts", + "docs/superpowers", +) + + +def archive_code(repo: Path, dest_dir: Path) -> Code: + """``git archive`` of HEAD's :data:`CODE_PATHS` as + ``stack-.tar.gz``; gzip carries no mtime. + + Uncommitted changes are not in the archive; ``dirty`` says whether + there were any to tracked files. + """ + commit = _git(repo, "rev-parse", "HEAD") + tracked = set(_git(repo, "ls-tree", "--name-only", "HEAD").split()) + paths = [p for p in CODE_PATHS if p.split("/")[0] in tracked] + dirty = bool(_git(repo, "status", "--porcelain", "--untracked-files=no")) + dest_dir.mkdir(parents=True, exist_ok=True) + dest = dest_dir / f"stack-{commit}.tar.gz" + proc = subprocess.Popen( + [ + "git", + "-C", + str(repo), + "archive", + "--format=tar", + f"--prefix=stack-{commit[:12]}/", + commit, + "--", + *paths, + ], + env=_git_env(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert proc.stdout is not None + with dest.open("wb") as raw: + with gzip.GzipFile( + filename="", mode="wb", fileobj=raw, compresslevel=6, mtime=0 + ) as gz: + shutil.copyfileobj(proc.stdout, gz, 1 << 20) + proc.stdout.close() + err = proc.stderr.read() if proc.stderr else b"" + if proc.stderr: + proc.stderr.close() + if proc.wait() != 0: + raise RuntimeError(f"git archive failed: {err.decode(errors='replace')}") + return Code(commit=commit, dirty=dirty, path=dest) diff --git a/src/lead/release/crate.py b/src/lead/release/crate.py new file mode 100644 index 0000000..27241f6 --- /dev/null +++ b/src/lead/release/crate.py @@ -0,0 +1,234 @@ +"""RO-Crate 1.2 metadata for a release payload, stdlib only. + +The crate root is the bag's ``data/`` directory. The root ``Dataset`` +lists one ``File`` per payload file (``encodingFormat``, ``contentSize``, +``sha256``); the metadata file itself is the descriptor, not a part. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Sequence +from pathlib import Path, PurePosixPath +from urllib.parse import quote, unquote + +CONTEXT = "https://w3id.org/ro/crate/1.2/context" +PROFILE = "https://w3id.org/ro/crate/1.2" +METADATA_FILE = "ro-crate-metadata.json" +LICENSE = "https://creativecommons.org/licenses/by/4.0/" +LICENSE_NAME = "Creative Commons Attribution 4.0 International" + +# The OSF tags on registration 8wq3m. dev/scripts/osf_metadata.py writes +# them there; a test pins the two lists together. +KEYWORDS = ( + "LEAD model", + "ACO", + "accountable care", + "Medicare", + "CMS Innovation Center", + "living review", + "scoping review", + "systematic review", + "prognostic factors", + "health policy", + "PRISMA-ScR", + "performance year 2027", +) + +# Longest suffix first: ``.csl.json`` before ``.json``. +_FORMATS = ( + (".csl.json", "application/vnd.citationstyles.csl+json"), + (".tar.gz", "application/gzip"), + (".parquet", "application/vnd.apache.parquet"), + (".json", "application/json"), + (".md", "text/markdown"), + (".yaml", "application/yaml"), + (".yml", "application/yaml"), + (".gz", "application/gzip"), + (".txt", "text/plain"), +) +_REQUIRED_ROOT = ("name", "description", "datePublished", "license") + + +def encoding_format(rel: str) -> str: + """IANA media type for a payload path, by suffix.""" + name = PurePosixPath(rel).name.lower() + for suffix, media in _FORMATS: + if name.endswith(suffix): + return media + return "application/octet-stream" + + +def _sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as fh: + while chunk := fh.read(1 << 20): + h.update(chunk) + return h.hexdigest() + + +def _files(root: Path) -> list[str]: + return sorted( + p.relative_to(root).as_posix() + for p in root.rglob("*") + if (p.is_file() or p.is_symlink()) + and p.relative_to(root).as_posix() != METADATA_FILE + ) + + +def file_entity(root: Path, rel: str) -> dict: + path = root / rel + return { + "@id": quote(rel, safe="/"), + "@type": "File", + "name": PurePosixPath(rel).name, + "encodingFormat": encoding_format(rel), + "contentSize": str(path.stat().st_size), + "sha256": _sha256(path), + } + + +def build_crate( + root: Path, + *, + name: str, + description: str, + version: str, + date_published: str, + based_on: dict, + entities: Sequence[dict] = (), +) -> dict: + """The crate for every file now under ``root`` (except the metadata file). + + ``based_on`` is the entity the release derives from (the registration); + ``entities`` are further contextual entities, such as the build's + ``CreateAction`` and its inputs. + """ + files = [file_entity(root, rel) for rel in _files(root)] + dataset = { + "@id": "./", + "@type": "Dataset", + "name": name, + "description": description, + "version": version, + "datePublished": date_published, + "license": {"@id": LICENSE}, + "keywords": list(KEYWORDS), + "isBasedOn": {"@id": based_on["@id"]}, + "hasPart": [{"@id": entity["@id"]} for entity in files], + } + graph = [ + { + "@id": METADATA_FILE, + "@type": "CreativeWork", + "conformsTo": {"@id": PROFILE}, + "about": {"@id": "./"}, + }, + dataset, + {"@id": LICENSE, "@type": "CreativeWork", "name": LICENSE_NAME}, + based_on, + *files, + *entities, + ] + return {"@context": CONTEXT, "@graph": graph} + + +def write_crate(root: Path, crate: dict) -> Path: + path = root / METADATA_FILE + path.write_text( + json.dumps(crate, indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + newline="\n", + ) + return path + + +def _is_local(entity_id: str) -> bool: + return not ( + entity_id.startswith("#") + or entity_id == "./" + or "://" in entity_id + or entity_id.startswith(("mailto:", "doi:", "urn:")) + ) + + +def _types(entity: dict) -> list[str]: + kind = entity.get("@type", []) + return [kind] if isinstance(kind, str) else list(kind) + + +def check_crate(root: Path) -> list[str]: + """Every problem with the crate under ``root``. Empty means it checks out. + + The descriptor and root ``Dataset`` must exist with the required + properties; every file under ``root`` (bar the metadata file) must be + a ``File`` in ``hasPart``; every described file must exist with the + recorded ``contentSize`` and ``sha256``. + """ + path = root / METADATA_FILE + if not path.is_file(): + return [f"{METADATA_FILE} is missing"] + try: + crate = json.loads(path.read_text(encoding="utf-8")) + except (ValueError, UnicodeDecodeError) as exc: + return [f"{METADATA_FILE} is not JSON: {exc}"] + graph = crate.get("@graph") if isinstance(crate, dict) else None + if not isinstance(graph, list): + return [f"{METADATA_FILE} has no @graph"] + by_id = {e.get("@id"): e for e in graph if isinstance(e, dict)} + problems: list[str] = [] + if crate.get("@context") != CONTEXT: + problems.append(f"@context is not {CONTEXT}") + descriptor = by_id.get(METADATA_FILE) + if descriptor is None: + problems.append(f"no {METADATA_FILE} descriptor entity") + elif descriptor.get("about") != {"@id": "./"}: + problems.append("the descriptor is not about ./") + dataset = by_id.get("./") + if dataset is None or "Dataset" not in _types(dataset): + return [*problems, "no root Dataset ./"] + for prop in _REQUIRED_ROOT: + if not dataset.get(prop): + problems.append(f"root Dataset has no {prop}") + + parts = dataset.get("hasPart", []) + if isinstance(parts, dict): + parts = [parts] + described = {unquote(p.get("@id", "")) for p in parts if isinstance(p, dict)} + for rel in _files(root): + if rel not in described: + problems.append(f"{rel}: not described in hasPart") + + resolved = root.resolve() + file_ids = { + entity_id + for entity_id, entity in by_id.items() + if isinstance(entity_id, str) + and _is_local(entity_id) + and "File" in _types(entity) + } + for entity_id in sorted(file_ids | {quote(rel, safe="/") for rel in described}): + rel = unquote(entity_id) + if rel == METADATA_FILE: + continue + entity = by_id.get(entity_id) + target = (root / rel).resolve() + if not target.is_relative_to(resolved): + problems.append(f"{rel}: path leaves the crate") + continue + if entity is None: + problems.append(f"{rel}: in hasPart but has no entity") + continue + if not target.is_file(): + problems.append(f"{rel}: described but missing") + continue + size = entity.get("contentSize") + if size is not None and str(size) != str(target.stat().st_size): + problems.append(f"{rel}: contentSize {size} does not match") + digest = entity.get("sha256") + if not digest: + problems.append(f"{rel}: no sha256") + elif _sha256(target) != digest: + problems.append(f"{rel}: sha256 does not match") + return problems diff --git a/src/lead/release/protocol.py b/src/lead/release/protocol.py new file mode 100644 index 0000000..f9dd40c --- /dev/null +++ b/src/lead/release/protocol.py @@ -0,0 +1,161 @@ +"""The frozen OSF registration (8wq3m) that every release is based on. + +The registration's form answers (``registration_responses``) are kept +verbatim, with the question labels and page headings from its schema +blocks so the answers can be rendered as a readable protocol. +""" + +from __future__ import annotations + +import json +from collections.abc import Sequence +from pathlib import Path +from typing import Any + +GUID = "8wq3m" +DOI = "10.17605/OSF.IO/8WQ3M" +DOI_URL = f"https://doi.org/{DOI}" +API = "https://api.osf.io/v2" +USER_AGENT = "stack-lead-release/1.0" + + +def normalize(data: dict, blocks: Sequence[dict]) -> dict: + """The release's registration record from an API ``data`` object. + + ``blocks`` are the registration schema's blocks (``schema_blocks``); + each answer key gets the label of the question in its block group + and the page heading it sits under. + """ + attrs = data.get("attributes", {}) + guid = data.get("id", GUID) + labels: dict[str, str] = {} + ordered = sorted( + (b.get("attributes", b) for b in blocks), key=lambda a: a.get("index", 0) + ) + for attr in ordered: + if attr.get("block_type") == "question-label": + labels[attr.get("schema_block_group_key", "")] = " ".join( + str(attr.get("display_text", "")).split() + ) + questions: list[dict[str, str]] = [] + page = "" + for attr in ordered: + kind = attr.get("block_type") + if kind == "page-heading": + page = " ".join(str(attr.get("display_text", "")).split()) + key = attr.get("registration_response_key") + if key: + questions.append( + { + "key": key, + "page": page, + "label": labels.get(attr.get("schema_block_group_key", ""), ""), + } + ) + return { + "guid": guid, + "doi": DOI if guid == GUID else "", + "url": f"https://osf.io/{guid}/", + "title": attrs.get("title", ""), + "description": attrs.get("description", ""), + "date_registered": attrs.get("date_registered", ""), + "registration_supplement": attrs.get("registration_supplement", ""), + "registration_responses": attrs.get("registration_responses") or {}, + "questions": questions, + } + + +def _get(client: Any, url: str) -> dict: + response = client.get(url) + response.raise_for_status() + return response.json() + + +def fetch_registration(client: Any, guid: str = GUID) -> dict: + """Read the public registration and its schema blocks from the OSF API. + + ``client`` is an ``httpx.Client``; the caller sets the User-Agent. + """ + data = _get(client, f"{API}/registrations/{guid}/")["data"] + schema = ( + data.get("relationships", {}).get("registration_schema", {}).get("data") or {} + ) + blocks: list[dict] = [] + if schema.get("id"): + url: str | None = ( + f"{API}/schemas/registrations/{schema['id']}/schema_blocks/?page[size]=100" + ) + while url: + page = _get(client, url) + blocks.extend(page.get("data", [])) + url = (page.get("links") or {}).get("next") + return normalize(data, blocks) + + +def load_registration(path: str | Path) -> dict: + """A cached registration: a release's own JSON or a raw API response.""" + payload = json.loads(Path(path).read_text(encoding="utf-8")) + if "registration_responses" in payload: + return payload + if isinstance(payload.get("data"), dict): + return normalize(payload["data"], []) + raise ValueError(f"{path} is not an OSF registration") + + +def _answer(value: Any) -> str: + if isinstance(value, str): + return value.strip() or "(no response)" + if not value: + return "(no response)" + if isinstance(value, list): + lines = [] + for item in value: + if isinstance(item, dict): + lines.append(f"- {item.get('file_name') or item.get('name') or item}") + else: + lines.append(f"- {item}") + return "\n".join(lines) + return json.dumps(value, ensure_ascii=False, sort_keys=True) + + +def render_protocol(reg: dict) -> str: + """Markdown protocol: one section per form page, one per question.""" + responses = reg.get("registration_responses", {}) + lines = [f"# {reg.get('title', '')}", ""] + if reg.get("url"): + lines.append(f"- Registration: {reg['url']}") + if reg.get("doi"): + lines.append(f"- DOI: https://doi.org/{reg['doi']}") + if reg.get("date_registered"): + lines.append(f"- Registered: {str(reg['date_registered'])[:10]}") + if reg.get("registration_supplement"): + lines.append(f"- Form: {reg['registration_supplement']}") + lines.append("") + if reg.get("description"): + lines.extend([reg["description"].strip(), ""]) + lines.extend( + [ + f"Rendered from the frozen OSF registration; the answers are verbatim " + f"in registration-{reg.get('guid', GUID)}.json.", + "", + ] + ) + page = None + seen: set[str] = set() + for question in reg.get("questions", []): + key = question["key"] + if key not in responses or key in seen: + continue + seen.add(key) + if question.get("page") != page: + page = question.get("page") + lines.extend([f"## {page or 'Responses'}", ""]) + lines.extend( + [f"### {question.get('label') or key}", "", _answer(responses[key]), ""] + ) + rest = [key for key in responses if key not in seen] + if rest: + lines.extend(["## Other responses", ""]) + for key in rest: + lines.extend([f"### {key}", "", _answer(responses[key]), ""]) + return "\n".join(lines).rstrip("\n") + "\n" diff --git a/tests/lead/test_release_bagit.py b/tests/lead/test_release_bagit.py new file mode 100644 index 0000000..10b3930 --- /dev/null +++ b/tests/lead/test_release_bagit.py @@ -0,0 +1,150 @@ +"""BagIt 1.0 writer and validator for LEAD review releases (RFC 8493).""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +from lead.release.bagit import ( + decode_path, + encode_path, + read_bag_info, + validate_bag, + write_bag, +) + +INFO = [ + ("External-Identifier", "doi:10.17605/OSF.IO/8WQ3M"), + ("Bag-Group-Identifier", "lead-living-review"), + ("Bagging-Date", "2026-11-11"), + ("Version", "0.1"), +] + + +def _payload(bag: Path) -> None: + data = bag / "data" + (data / "ledger").mkdir(parents=True) + (data / "a.txt").write_text("alpha\n") + (data / "ledger" / "claim.parquet").write_bytes(b"\x00\x01\x02") + + +def test_write_bag_writes_the_four_tag_files(tmp_path): + bag = tmp_path / "bag" + _payload(bag) + octets, streams = write_bag(bag, INFO) + assert (octets, streams) == (9, 2) + assert (bag / "bagit.txt").read_text() == ( + "BagIt-Version: 1.0\nTag-File-Character-Encoding: UTF-8\n" + ) + manifest = (bag / "manifest-sha512.txt").read_text().splitlines() + digest = hashlib.sha512(b"alpha\n").hexdigest() + assert manifest == [ + f"{digest} data/a.txt", + f"{hashlib.sha512(bytes([0, 1, 2])).hexdigest()} data/ledger/claim.parquet", + ] + tags = [ + line.split()[1] + for line in (bag / "tagmanifest-sha512.txt").read_text().splitlines() + ] + assert tags == ["bag-info.txt", "bagit.txt", "manifest-sha512.txt"] + + +def test_bag_info_adds_size_and_oxum_after_the_bagging_date(tmp_path): + bag = tmp_path / "bag" + _payload(bag) + write_bag(bag, INFO) + labels = [label for label, _value in read_bag_info(bag)] + assert labels == [ + "External-Identifier", + "Bag-Group-Identifier", + "Bagging-Date", + "Bag-Size", + "Payload-Oxum", + "Version", + ] + assert dict(read_bag_info(bag))["Payload-Oxum"] == "9.2" + assert dict(read_bag_info(bag))["Bag-Size"] == "9 B" + + +def test_a_long_bag_info_value_folds_and_reads_back_whole(tmp_path): + bag = tmp_path / "bag" + _payload(bag) + note = " ".join(["the ledger gained claims and the vocabulary moved"] * 4) + write_bag(bag, [*INFO, ("Internal-Sender-Description", note)]) + text = (bag / "bag-info.txt").read_text() + assert all(len(line) <= 79 for line in text.splitlines()) + assert dict(read_bag_info(bag))["Internal-Sender-Description"] == note + assert validate_bag(bag) == [] + + +def test_a_fresh_bag_is_valid(tmp_path): + bag = tmp_path / "bag" + _payload(bag) + write_bag(bag, INFO) + assert validate_bag(bag) == [] + + +def test_validate_reports_a_changed_payload_file(tmp_path): + bag = tmp_path / "bag" + _payload(bag) + write_bag(bag, INFO) + (bag / "data" / "a.txt").write_text("alpha!\n") + problems = validate_bag(bag) + assert any("data/a.txt" in p and "sha512" in p for p in problems) + assert any("Payload-Oxum" in p for p in problems) + + +def test_validate_reports_unlisted_and_missing_payload_files(tmp_path): + bag = tmp_path / "bag" + _payload(bag) + write_bag(bag, INFO) + (bag / "data" / "extra.txt").write_text("x") + (bag / "data" / "ledger" / "claim.parquet").unlink() + problems = validate_bag(bag) + assert any( + "data/extra.txt" in p and "not in manifest-sha512.txt" in p for p in problems + ) + assert any("data/ledger/claim.parquet" in p and "missing" in p for p in problems) + + +def test_validate_reports_a_bad_bagit_txt(tmp_path): + bag = tmp_path / "bag" + _payload(bag) + write_bag(bag, INFO) + (bag / "bagit.txt").write_text("BagIt-Version: one\n") + problems = validate_bag(bag) + assert any(p.startswith("bagit.txt") for p in problems) + + +def test_validate_reports_a_missing_bagit_txt_and_manifest(tmp_path): + bag = tmp_path / "bag" + _payload(bag) + problems = validate_bag(bag) + assert any("bagit.txt is missing" in p for p in problems) + assert any("manifest-sha512.txt is missing" in p for p in problems) + + +def test_validate_reports_an_edited_tag_file(tmp_path): + bag = tmp_path / "bag" + _payload(bag) + write_bag(bag, INFO) + info = bag / "bag-info.txt" + info.write_text(info.read_text().replace("0.1", "0.2")) + problems = validate_bag(bag) + assert problems == ["bag-info.txt: sha512 in tagmanifest-sha512.txt does not match"] + + +def test_validate_reports_a_manifest_path_outside_data(tmp_path): + bag = tmp_path / "bag" + _payload(bag) + write_bag(bag, INFO) + manifest = bag / "manifest-sha512.txt" + manifest.write_text(manifest.read_text() + f"{'0' * 128} ../escape.txt\n") + problems = validate_bag(bag) + assert any("../escape.txt" in p for p in problems) + + +def test_path_encoding_round_trips_percent_and_newlines(): + raw = "data/100%\nodd\r.txt" + assert encode_path(raw) == "data/100%25%0Aodd%0D.txt" + assert decode_path(encode_path(raw)) == raw diff --git a/tests/lead/test_release_build.py b/tests/lead/test_release_build.py new file mode 100644 index 0000000..3d2e710 --- /dev/null +++ b/tests/lead/test_release_build.py @@ -0,0 +1,639 @@ +"""`stack lead release build` / `verify`: a BagIt bag with RO-Crate metadata.""" + +from __future__ import annotations + +import dataclasses +import hashlib +import json +import os +import sqlite3 +import subprocess +import tarfile +from pathlib import Path + +import duckdb +import polars as pl +import pytest +from typer.testing import CliRunner + +from cli import app +from lead.ledger.store import append, open_ledger +from lead.release import Inputs, build_release, verify_release +from lead.release.bagit import read_bag_info +from lead.release.protocol import normalize +from lead.review.store import Store + +runner = CliRunner() + +EMAIL = "reviewer@example.org" + +REGISTRATION = normalize( + { + "id": "8wq3m", + "attributes": { + "title": "LEAD evidence chassis", + "description": "Protocol for a living review.", + "date_registered": "2026-11-01T03:36:24.703072", + "registration_supplement": "Generalized Systematic Review Registration", + "registration_responses": {"376-8": "Living review."}, + }, + }, + [ + { + "attributes": { + "index": 0, + "block_type": "page-heading", + "display_text": "Methods", + } + }, + { + "attributes": { + "index": 1, + "block_type": "question-label", + "display_text": "Type of review", + "schema_block_group_key": "g", + } + }, + { + "attributes": { + "index": 2, + "block_type": "long-text-input", + "registration_response_key": "376-8", + "schema_block_group_key": "g", + } + }, + ], +) + + +# ── fixtures ───────────────────────────────────────────────────── + + +def _git(*args: str, cwd: Path) -> str: + # A pre-commit hook exports GIT_DIR / GIT_INDEX_FILE; without dropping + # them every scratch-repo command would land in the real repo. + env = {k: v for k, v in os.environ.items() if not k.startswith("GIT_")} + env.update( + GIT_AUTHOR_NAME="t", + GIT_AUTHOR_EMAIL="t@example.invalid", + GIT_COMMITTER_NAME="t", + GIT_COMMITTER_EMAIL="t@example.invalid", + GIT_AUTHOR_DATE="2026-11-01T00:00:00Z", + GIT_COMMITTER_DATE="2026-11-01T00:00:00Z", + ) + return subprocess.run( + ["git", "-c", "core.fsync=none", *args], + cwd=cwd, + env=env, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def _repo(root: Path) -> Path: + repo = root / "repo" + repo.mkdir() + _git("init", "-q", "-b", "main", cwd=repo) + (repo / "README.md").write_text("stack\n") + (repo / "src").mkdir() + (repo / "src" / "app.py").write_text("print('lead')\n") + (repo / "hw" / "docs").mkdir(parents=True) + (repo / "hw" / "docs" / "vendor-manual.pdf").write_bytes(b"%PDF third-party") + _git("add", "-A", cwd=repo) + _git("commit", "-q", "-m", "base", cwd=repo) + return repo + + +def _ledger(root: Path) -> Path: + path = root / "ledger.duckdb" + con = open_ledger(str(path)) + try: + append( + con, + "claim", + { + "kind": "precursor", + "subject": "readmission", + "target_id": "Q.ACR", + "tiers": "literature", + "content_hash": "c1", + "actor": EMAIL, + "created_at": "2026-11-01T00:00:00Z", + }, + ) + append( + con, + "search", + { + "project": "lead-history", + "database": "pubmed", + "query": "LEAD[tiab]", + "hit_count": 3, + "content_hash": "s1", + "actor": "stack", + "created_at": "2026-11-01T00:00:00Z", + }, + ) + finally: + con.close() + return path + + +def _labels(root: Path) -> Path: + # Store creates the label table; rows go in one unsynced transaction + # (Store commits, and so fsyncs, per row). + path = root / "review" / "labels.sqlite" + Store(path) + con = sqlite3.connect(path) + try: + con.execute("PRAGMA synchronous=OFF") + con.executemany( + """INSERT INTO label (zotero_key, project, stage, pass, decision, + reason, rationale, actor, created_at) + VALUES (?, 'lead-history', 'title_abstract', 1, 'include', '', + 'CMS methodology', ?, '2026-11-04T23:59:36Z')""", + [(key, EMAIL) for key in ("AAAA2222", "BBBB3333", "GGGG8888")], + ) + con.commit() + finally: + con.close() + return path + + +_ZOTERO_DDL = """ +CREATE TABLE itemTypes (itemTypeID INTEGER PRIMARY KEY, typeName TEXT); +CREATE TABLE items (itemID INTEGER PRIMARY KEY, itemTypeID INT, key TEXT); +CREATE TABLE tags (tagID INTEGER PRIMARY KEY, name TEXT); +CREATE TABLE itemTags (itemID INT, tagID INT, type INT DEFAULT 0); +CREATE TABLE fields (fieldID INTEGER PRIMARY KEY, fieldName TEXT); +CREATE TABLE itemDataValues (valueID INTEGER PRIMARY KEY, value); +CREATE TABLE itemData (itemID INT, fieldID INT, valueID INT); +CREATE TABLE creators (creatorID INTEGER PRIMARY KEY, firstName TEXT, + lastName TEXT, fieldMode INT); +CREATE TABLE creatorTypes (creatorTypeID INTEGER PRIMARY KEY, creatorType TEXT); +CREATE TABLE itemCreators (itemID INT, creatorID INT, creatorTypeID INT, + orderIndex INT); +""" + + +def _zotero(root: Path) -> Path: + path = root / "zotero.sqlite" + con = sqlite3.connect(path) + try: + con.execute("PRAGMA synchronous=OFF") + con.execute("PRAGMA journal_mode=MEMORY") + con.executescript(_ZOTERO_DDL) + types = ["journalArticle", "report", "note", "statute"] + con.executemany( + "INSERT INTO itemTypes VALUES (?, ?)", list(enumerate(types, start=1)) + ) + fields = ["title", "date", "DOI", "url", "nameOfAct", "dateEnacted"] + con.executemany( + "INSERT INTO fields VALUES (?, ?)", list(enumerate(fields, start=1)) + ) + con.executemany( + "INSERT INTO creatorTypes VALUES (?, ?)", [(1, "author"), (2, "editor")] + ) + tag_ids: dict[str, int] = {} + value_ids: dict[str, int] = {} + + def item(item_id, key, type_name, tags, data, creators=()): + con.execute( + "INSERT INTO items VALUES (?, ?, ?)", + (item_id, types.index(type_name) + 1, key), + ) + for name in tags: + tag_id = tag_ids.setdefault(name, len(tag_ids) + 1) + con.execute("INSERT OR IGNORE INTO tags VALUES (?, ?)", (tag_id, name)) + con.execute("INSERT INTO itemTags VALUES (?, ?, 0)", (item_id, tag_id)) + for field, value in data.items(): + value_id = value_ids.setdefault(value, len(value_ids) + 1) + con.execute( + "INSERT OR IGNORE INTO itemDataValues VALUES (?, ?)", + (value_id, value), + ) + con.execute( + "INSERT INTO itemData VALUES (?, ?, ?)", + (item_id, fields.index(field) + 1, value_id), + ) + for order, (first, last, mode, ctype) in enumerate(creators): + cur = con.execute( + "INSERT INTO creators (firstName, lastName, fieldMode) " + "VALUES (?, ?, ?)", + (first, last, mode), + ) + con.execute( + "INSERT INTO itemCreators VALUES (?, ?, ?, ?)", + (item_id, cur.lastrowid, ctype, order), + ) + + item( + 1, + "AAAA2222", + "journalArticle", + ["project:lead-history", "stage:4-included", "found-in:pubmed"], + {"title": "Alpha study", "date": "2020-03-00 March 2020", "DOI": "10.1/a"}, + [("Jane", "Smith", 0, 1), ("Ed", "Itor", 0, 2)], + ) + item( + 2, + "BBBB3333", + "report", + ["project:lead-history"], + {"title": "Beta report", "url": "https://example.org/b", "date": "2019"}, + [("", "CMS", 1, 1)], + ) + item( + 3, + "CCCC4444", + "statute", + [ + "project:lead-limits", + "stage:2-title-abstract", + "screen:exclude", + "screen:reason:off-topic", + ], + {"nameOfAct": "Gamma Act", "dateEnacted": "2010-03-23 2010-03-23"}, + ) + item(4, "DDDD5555", "journalArticle", ["project:lead-scan"], {"title": "Delta"}) + item(5, "EEEE6666", "note", ["project:lead-history", "module:prisma"], {}) + item( + 6, + "FFFF7777", + "journalArticle", + ["project:skin-subs", "stage:4-included"], + {"title": "Not LEAD"}, + ) + con.commit() + finally: + con.close() + return path + + +def _vocab(root: Path) -> Path: + path = root / "themes.yaml" + path.write_text("# vocab\nversion: 3\nthemes:\n - slug: work-rvu\n") + return path + + +@pytest.fixture(scope="module") +def inputs(tmp_path_factory: pytest.TempPathFactory) -> Inputs: + """Built once per module: the build only reads them.""" + src = tmp_path_factory.mktemp("src") + return Inputs( + ledger=_ledger(src), + labels=_labels(src), + zotero=_zotero(src), + vocab=_vocab(src), + repo=_repo(src), + registration=REGISTRATION, + ) + + +def _build(inputs: Inputs, out: Path, **kw): + kw.setdefault("version", "0.1") + kw.setdefault("date", "2026-11-11") + kw.setdefault("note", "First release.") + return build_release(out=out, inputs=inputs, **kw) + + +def _sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +# ── build ──────────────────────────────────────────────────────── + + +def test_build_writes_a_valid_bag(inputs, tmp_path): + result = _build(inputs, tmp_path / "out") + bag = tmp_path / "out" / "lead-review-v0.1" + assert result.path == bag + assert verify_release(bag) == [] + payload = sorted( + p.relative_to(bag).as_posix() for p in (bag / "data").rglob("*") if p.is_file() + ) + sha = _git("rev-parse", "HEAD", cwd=inputs.repo) + assert result.commit == sha + assert payload == sorted( + [ + "data/bibliography/included.csl.json", + "data/bibliography/screened.csl.json", + f"data/code/stack-{sha}.tar.gz", + *[ + f"data/ledger/{t}.parquet" + for t in ( + "claim", + "decision", + "exogenous", + "extraction", + "feature", + "finding", + "run", + "search", + "source", + ) + ], + "data/protocol/protocol.md", + "data/protocol/registration-8wq3m.json", + "data/ro-crate-metadata.json", + "data/screening/decisions.parquet", + "data/screening/prisma-flow.json", + "data/tags/vocab-v3.yaml", + ] + ) + assert result.files == len(payload) + assert result.bytes == sum((bag / p).stat().st_size for p in payload) + assert sorted(p.name for p in bag.iterdir()) == [ + "bag-info.txt", + "bagit.txt", + "data", + "manifest-sha512.txt", + "tagmanifest-sha512.txt", + ] + + +def test_build_does_not_touch_its_inputs(inputs, tmp_path): + files = [inputs.ledger, inputs.labels, inputs.zotero, inputs.vocab] + before = {p: (_sha(p), p.stat().st_mtime_ns) for p in files} + siblings = {p.parent: sorted(x.name for x in p.parent.iterdir()) for p in files} + _build(inputs, tmp_path / "out") + assert {p: (_sha(p), p.stat().st_mtime_ns) for p in files} == before + assert {d: sorted(x.name for x in d.iterdir()) for d in siblings} == siblings + + +def test_bag_info_fields(inputs, tmp_path): + result = _build(inputs, tmp_path / "out", source_org="LEAD review team") + info = dict(read_bag_info(result.path)) + assert info["Source-Organization"] == "LEAD review team" + assert info["External-Identifier"] == "doi:10.17605/OSF.IO/8WQ3M" + assert info["Bag-Group-Identifier"] == "lead-living-review" + assert info["Bagging-Date"] == "2026-11-11" + assert info["Version"] == "0.1" + assert info["Git-Commit"] == result.commit + assert info["Internal-Sender-Description"] == "First release." + assert info["Payload-Oxum"] == f"{result.bytes}.{result.files}" + assert "Bag-Size" in info + + +def test_build_is_deterministic_except_for_the_date(inputs, tmp_path): + a = _build(inputs, tmp_path / "a").path + b = _build(inputs, tmp_path / "b").path + assert (a / "manifest-sha512.txt").read_text() == ( + b / "manifest-sha512.txt" + ).read_text() + assert (a / "tagmanifest-sha512.txt").read_text() == ( + b / "tagmanifest-sha512.txt" + ).read_text() + c = _build(inputs, tmp_path / "c", date="2026-12-01").path + left = (a / "manifest-sha512.txt").read_text().splitlines() + right = (c / "manifest-sha512.txt").read_text().splitlines() + changed = sorted({line.split()[1] for line in set(left) ^ set(right)}) + assert changed == ["data/ro-crate-metadata.json"] + + +def test_protocol_files(inputs, tmp_path): + bag = _build(inputs, tmp_path / "out").path + reg = json.loads((bag / "data/protocol/registration-8wq3m.json").read_text()) + assert reg == REGISTRATION + text = (bag / "data/protocol/protocol.md").read_text() + assert "### Type of review\n\nLiving review.\n" in text + + +def test_ledger_tables_are_exported_with_email_actors_aliased(inputs, tmp_path): + bag = _build(inputs, tmp_path / "out").path + claim = pl.read_parquet(bag / "data/ledger/claim.parquet") + assert claim["subject"].to_list() == ["readmission"] + assert claim["actor"].to_list() == ["reviewer-1"] + search = pl.read_parquet(bag / "data/ledger/search.parquet") + assert search["actor"].to_list() == ["stack"] + assert search["hit_count"].to_list() == [3] + empty = pl.read_parquet(bag / "data/ledger/finding.parquet") + assert empty.height == 0 and "claim_id" in empty.columns + + +def test_screening_decisions_and_flow(inputs, tmp_path): + bag = _build(inputs, tmp_path / "out").path + decisions = pl.read_parquet(bag / "data/screening/decisions.parquet") + assert decisions.columns == [ + "label_id", + "zotero_key", + "project", + "stage", + "pass", + "decision", + "reason", + "rationale", + "actor", + "created_at", + ] + assert decisions["zotero_key"].to_list() == ["AAAA2222", "BBBB3333", "GGGG8888"] + assert set(decisions["actor"]) == {"reviewer-1"} + raw = b"".join(p.read_bytes() for p in (bag / "data").rglob("*") if p.is_file()) + assert EMAIL.encode() not in raw + flow = json.loads((bag / "data/screening/prisma-flow.json").read_text()) + history = flow["projects"]["lead-history"] + assert history["identified"] == 2 + assert history["screened"] == 1 + assert history["included"] == 1 + assert history["identified_by_source"] == {"pubmed": 1} + limits = flow["projects"]["lead-limits"] + assert limits["excluded_stage2"] == 1 + assert limits["excluded_stage2_reasons"] == {"off-topic": 1} + assert sorted(flow["projects"]) == [ + "lead-external", + "lead-history", + "lead-limits", + "lead-precursors", + "lead-scan", + ] + assert flow["reviews"]["scoping"]["identified"] == 4 + assert flow["reviews"]["systematic"]["identified"] == 0 + + +def test_bibliography_is_csl_json(inputs, tmp_path): + result = _build(inputs, tmp_path / "out") + bag = result.path + included = json.loads((bag / "data/bibliography/included.csl.json").read_text()) + assert included == [ + { + "id": "AAAA2222", + "type": "article-journal", + "title": "Alpha study", + "author": [{"family": "Smith", "given": "Jane"}], + "issued": {"date-parts": [[2020, 3]]}, + "DOI": "10.1/a", + } + ] + screened = json.loads((bag / "data/bibliography/screened.csl.json").read_text()) + assert [item["id"] for item in screened] == ["AAAA2222", "BBBB3333", "CCCC4444"] + beta = screened[1] + assert beta["type"] == "report" + assert beta["author"] == [{"literal": "CMS"}] + assert beta["URL"] == "https://example.org/b" + assert beta["issued"] == {"date-parts": [[2019]]} + gamma = screened[2] + assert gamma["type"] == "legislation" + assert gamma["title"] == "Gamma Act" + assert gamma["issued"] == {"date-parts": [[2010, 3, 23]]} + assert any("GGGG8888" in w for w in result.warnings) + + +def test_tags_and_code(inputs, tmp_path): + inputs = dataclasses.replace(inputs, repo=_repo(tmp_path)) + result = _build(inputs, tmp_path / "out") + bag = result.path + assert (bag / "data/tags/vocab-v3.yaml").read_bytes() == inputs.vocab.read_bytes() + archive = bag / f"data/code/stack-{result.commit}.tar.gz" + with tarfile.open(archive) as tar: + names = tar.getnames() + # Only the code that builds a release; third-party documents tracked + # elsewhere in the repo (hw/docs manuals, books) never ship in a bag. + assert f"stack-{result.commit[:12]}/src/app.py" in names + assert not any("vendor-manual.pdf" in n for n in names) + assert result.dirty is False + (inputs.repo / "src" / "app.py").write_text("changed\n") + assert _build(inputs, tmp_path / "dirty").dirty is True + + +def test_crate_records_the_build(inputs, tmp_path): + result = _build(inputs, tmp_path / "out") + crate = json.loads((result.path / "data/ro-crate-metadata.json").read_text()) + graph = {e["@id"]: e for e in crate["@graph"]} + root = graph["./"] + assert root["version"] == "0.1" + assert root["datePublished"] == "2026-11-11" + assert root["isBasedOn"] == {"@id": "https://doi.org/10.17605/OSF.IO/8WQ3M"} + assert {"@id": "screening/decisions.parquet"} in root["hasPart"] + action = next(e for e in crate["@graph"] if e["@type"] == "CreateAction") + assert action["name"] == "stack lead release build --version 0.1" + assert action["result"] == {"@id": "./"} + stack = graph[action["instrument"][0]["@id"]] + assert stack["version"] == result.commit + inputs_described = [graph[o["@id"]]["name"] for o in action["object"]] + assert any("ledger" in name for name in inputs_described) + assert any("Zotero" in name for name in inputs_described) + + +def test_build_refuses_to_overwrite_without_force(inputs, tmp_path): + _build(inputs, tmp_path / "out") + with pytest.raises(FileExistsError): + _build(inputs, tmp_path / "out") + again = _build(inputs, tmp_path / "out", force=True, note="Rebuilt.") + assert dict(read_bag_info(again.path))["Internal-Sender-Description"] == "Rebuilt." + assert [p.name for p in (tmp_path / "out").iterdir()] == ["lead-review-v0.1"] + + +@pytest.mark.parametrize("version", ["1", "v0.1", "0.1.2", ""]) +def test_build_rejects_a_version_that_is_not_major_minor(inputs, tmp_path, version): + with pytest.raises(ValueError): + _build(inputs, tmp_path / "out", version=version) + + +# ── verify ─────────────────────────────────────────────────────── + + +def test_verify_reports_bag_and_crate_problems(inputs, tmp_path): + bag = _build(inputs, tmp_path / "out").path + (bag / "data/tags/vocab-v3.yaml").write_text("version: 4\n") + (bag / "data/notes.txt").write_text("stray\n") + problems = verify_release(bag) + assert any("data/tags/vocab-v3.yaml" in p and "sha512" in p for p in problems) + assert any( + p.startswith("ro-crate:") and "tags/vocab-v3.yaml" in p and "sha256" in p + for p in problems + ) + assert any("data/notes.txt" in p and "manifest" in p for p in problems) + assert any(p.startswith("ro-crate:") and "notes.txt" in p for p in problems) + + +def test_verify_reports_a_bag_without_a_crate(inputs, tmp_path): + bag = _build(inputs, tmp_path / "out").path + (bag / "data/ro-crate-metadata.json").unlink() + problems = verify_release(bag) + assert "ro-crate: ro-crate-metadata.json is missing" in problems + + +def test_verify_reports_a_path_that_is_not_a_bag(tmp_path): + assert verify_release(tmp_path / "nope") == [ + f"{tmp_path / 'nope'} is not a directory" + ] + + +# ── CLI ────────────────────────────────────────────────────────── + + +def _cli_args(inputs: Inputs, out: Path, reg: Path) -> list[str]: + return [ + "lead", + "release", + "build", + "--version", + "0.1", + "--out", + str(out), + "--date", + "2026-11-11", + "--note", + "First release.", + "--registration", + str(reg), + "--ledger", + str(inputs.ledger), + "--labels", + str(inputs.labels), + "--zotero", + str(inputs.zotero), + "--vocab", + str(inputs.vocab), + "--repo", + str(inputs.repo), + ] + + +def test_cli_build_then_verify(inputs, tmp_path): + reg = tmp_path / "registration.json" + reg.write_text(json.dumps(REGISTRATION)) + out = tmp_path / "out" + result = runner.invoke(app, _cli_args(inputs, out, reg)) + assert result.exit_code == 0, result.output + bag = out / "lead-review-v0.1" + assert str(bag) in result.output + assert "GGGG8888" in result.output + ok = runner.invoke(app, ["lead", "release", "verify", str(bag)]) + assert ok.exit_code == 0, ok.output + assert "ok" in ok.output + again = runner.invoke(app, _cli_args(inputs, out, reg)) + assert again.exit_code != 0 + assert "--force" in again.output + + +def test_cli_verify_lists_problems_and_fails(inputs, tmp_path): + bag = _build(inputs, tmp_path / "out").path + (bag / "data/protocol/protocol.md").write_text("edited\n") + result = runner.invoke(app, ["lead", "release", "verify", str(bag)]) + assert result.exit_code == 1 + assert "data/protocol/protocol.md" in result.output + assert "problem" in result.output + + +def test_cli_build_rejects_a_bad_date(inputs, tmp_path): + reg = tmp_path / "registration.json" + reg.write_text(json.dumps(REGISTRATION)) + args = _cli_args(inputs, tmp_path / "out", reg) + args[args.index("2026-11-11")] = "11/11/2026" + result = runner.invoke(app, args) + assert result.exit_code != 0 + + +def test_parquet_reads_back_in_duckdb(inputs, tmp_path): + bag = _build(inputs, tmp_path / "out").path + con = duckdb.connect() + try: + n = con.execute( + "SELECT count(*) FROM read_parquet(?)", + [str(bag / "data/screening/decisions.parquet")], + ).fetchone()[0] + finally: + con.close() + assert n == 3 diff --git a/tests/lead/test_release_crate.py b/tests/lead/test_release_crate.py new file mode 100644 index 0000000..44d346c --- /dev/null +++ b/tests/lead/test_release_crate.py @@ -0,0 +1,307 @@ +"""RO-Crate 1.2 metadata and the OSF protocol record for LEAD releases.""" + +from __future__ import annotations + +import ast +import hashlib +import json +from pathlib import Path + +import httpx + +from lead.release.crate import ( + CONTEXT, + KEYWORDS, + LICENSE, + METADATA_FILE, + build_crate, + check_crate, + encoding_format, + write_crate, +) +from lead.release.protocol import ( + DOI_URL, + USER_AGENT, + fetch_registration, + load_registration, + normalize, + render_protocol, +) + +REPO = Path(__file__).resolve().parents[2] + +BASED_ON = {"@id": DOI_URL, "@type": "CreativeWork", "name": "LEAD protocol"} + + +def _root(tmp_path: Path) -> Path: + root = tmp_path / "data" + (root / "ledger").mkdir(parents=True) + (root / "ledger" / "claim.parquet").write_bytes(b"PAR1") + (root / "protocol").mkdir() + (root / "protocol" / "protocol.md").write_text("# Protocol\n") + return root + + +def _crate(root: Path) -> dict: + crate = build_crate( + root, + name="LEAD living review, release 0.1", + description="Release 0.1.", + version="0.1", + date_published="2026-11-11", + based_on=BASED_ON, + entities=[{"@id": "#build", "@type": "CreateAction", "name": "build"}], + ) + write_crate(root, crate) + return crate + + +def _entity(crate: dict, entity_id: str) -> dict: + return next(e for e in crate["@graph"] if e["@id"] == entity_id) + + +def test_keywords_are_the_osf_tags(): + script = REPO / "dev" / "scripts" / "osf_metadata.py" + tree = ast.parse(script.read_text()) + tags = next( + ast.literal_eval(node.value) + for node in tree.body + if isinstance(node, ast.Assign) + and any(getattr(t, "id", "") == "TAGS" for t in node.targets) + ) + assert list(KEYWORDS) == tags + + +def test_root_dataset_carries_the_release_metadata(tmp_path): + crate = _crate(_root(tmp_path)) + assert crate["@context"] == CONTEXT + descriptor = _entity(crate, METADATA_FILE) + assert descriptor["conformsTo"] == {"@id": "https://w3id.org/ro/crate/1.2"} + assert descriptor["about"] == {"@id": "./"} + root = _entity(crate, "./") + assert root["@type"] == "Dataset" + assert root["version"] == "0.1" + assert root["datePublished"] == "2026-11-11" + assert root["license"] == {"@id": LICENSE} + assert LICENSE == "https://creativecommons.org/licenses/by/4.0/" + assert root["keywords"] == list(KEYWORDS) + assert root["isBasedOn"] == {"@id": DOI_URL} + assert root["hasPart"] == [ + {"@id": "ledger/claim.parquet"}, + {"@id": "protocol/protocol.md"}, + ] + assert _entity(crate, DOI_URL)["name"] == "LEAD protocol" + assert _entity(crate, "#build")["@type"] == "CreateAction" + + +def test_file_entities_carry_format_size_and_sha256(tmp_path): + crate = _crate(_root(tmp_path)) + entity = _entity(crate, "protocol/protocol.md") + assert entity["@type"] == "File" + assert entity["encodingFormat"] == "text/markdown" + assert entity["contentSize"] == str(len(b"# Protocol\n")) + assert entity["sha256"] == hashlib.sha256(b"# Protocol\n").hexdigest() + + +def test_encoding_formats(): + assert encoding_format("ledger/claim.parquet") == "application/vnd.apache.parquet" + assert ( + encoding_format("bibliography/included.csl.json") + == "application/vnd.citationstyles.csl+json" + ) + assert encoding_format("protocol/registration-8wq3m.json") == "application/json" + assert encoding_format("tags/vocab-v2.yaml") == "application/yaml" + assert encoding_format("code/stack-abc.tar.gz") == "application/gzip" + + +def test_check_crate_passes_a_fresh_crate(tmp_path): + root = _root(tmp_path) + _crate(root) + assert check_crate(root) == [] + + +def test_check_crate_reports_undescribed_missing_and_changed_files(tmp_path): + root = _root(tmp_path) + _crate(root) + (root / "extra.json").write_text("{}") + (root / "ledger" / "claim.parquet").unlink() + (root / "protocol" / "protocol.md").write_text("# Changed\n") + problems = check_crate(root) + assert any("extra.json" in p and "not described" in p for p in problems) + assert any("ledger/claim.parquet" in p and "missing" in p for p in problems) + assert any("protocol/protocol.md" in p and "sha256" in p for p in problems) + + +def test_check_crate_reports_a_missing_or_broken_metadata_file(tmp_path): + root = _root(tmp_path) + assert check_crate(root) == [f"{METADATA_FILE} is missing"] + (root / METADATA_FILE).write_text("{not json") + assert check_crate(root)[0].startswith(f"{METADATA_FILE} is not JSON") + + +def test_check_crate_reports_a_root_without_required_properties(tmp_path): + root = _root(tmp_path) + crate = _crate(root) + dataset = _entity(crate, "./") + del dataset["license"] + write_crate(root, crate) + assert check_crate(root) == ["root Dataset has no license"] + + +# ── OSF registration ───────────────────────────────────────────── + + +RAW = { + "data": { + "id": "8wq3m", + "attributes": { + "title": "LEAD evidence chassis", + "description": "Protocol for a living review.", + "date_registered": "2026-11-01T03:36:24.703072", + "registration_supplement": "Generalized Systematic Review Registration", + "registration_responses": { + "376-8": "Living review.", + "376-10": "Preparation; search.", + "376-76": [], + "376-99": "Answer with no label.", + }, + "tags": [], + }, + "relationships": { + "registration_schema": {"data": {"id": "schema1", "type": "x"}} + }, + } +} + +BLOCKS = [ + { + "attributes": { + "index": 0, + "block_type": "page-heading", + "display_text": "Landing Page", + } + }, + {"attributes": {"index": 1, "block_type": "paragraph", "display_text": "Intro."}}, + { + "attributes": { + "index": 2, + "block_type": "page-heading", + "display_text": "Review Methods", + } + }, + { + "attributes": { + "index": 3, + "block_type": "question-label", + "display_text": "Type of review ", + "schema_block_group_key": "g8", + } + }, + { + "attributes": { + "index": 4, + "block_type": "long-text-input", + "registration_response_key": "376-8", + "schema_block_group_key": "g8", + } + }, + { + "attributes": { + "index": 5, + "block_type": "question-label", + "display_text": "Review stages", + "schema_block_group_key": "g10", + } + }, + { + "attributes": { + "index": 6, + "block_type": "long-text-input", + "registration_response_key": "376-10", + "schema_block_group_key": "g10", + } + }, + {"attributes": {"index": 7, "block_type": "page-heading", "display_text": "Data"}}, + { + "attributes": { + "index": 8, + "block_type": "question-label", + "display_text": "Attachments", + "schema_block_group_key": "g76", + } + }, + { + "attributes": { + "index": 9, + "block_type": "file-input", + "registration_response_key": "376-76", + "schema_block_group_key": "g76", + } + }, +] + + +def test_normalize_keeps_responses_verbatim_and_labels_questions(): + reg = normalize(RAW["data"], BLOCKS) + assert reg["guid"] == "8wq3m" + assert reg["doi"] == "10.17605/OSF.IO/8WQ3M" + assert reg["title"] == "LEAD evidence chassis" + assert reg["date_registered"] == "2026-11-01T03:36:24.703072" + assert ( + reg["registration_responses"] + == RAW["data"]["attributes"]["registration_responses"] + ) + assert reg["questions"] == [ + {"key": "376-8", "page": "Review Methods", "label": "Type of review"}, + {"key": "376-10", "page": "Review Methods", "label": "Review stages"}, + {"key": "376-76", "page": "Data", "label": "Attachments"}, + ] + + +def test_render_protocol_groups_answers_by_page(): + text = render_protocol(normalize(RAW["data"], BLOCKS)) + assert text.startswith("# LEAD evidence chassis\n") + assert "https://doi.org/10.17605/OSF.IO/8WQ3M" in text + assert "## Review Methods\n\n### Type of review\n\nLiving review.\n" in text + assert "### Attachments\n\n(no response)\n" in text + assert "### 376-99\n\nAnswer with no label.\n" in text + assert "Landing Page" not in text + + +def test_load_registration_reads_a_normalized_or_raw_file(tmp_path): + reg = normalize(RAW["data"], BLOCKS) + cached = tmp_path / "registration-8wq3m.json" + cached.write_text(json.dumps(reg)) + assert load_registration(cached) == reg + raw = tmp_path / "raw.json" + raw.write_text(json.dumps(RAW)) + loaded = load_registration(raw) + assert loaded["registration_responses"] == reg["registration_responses"] + assert loaded["questions"] == [] + + +def test_fetch_registration_reads_the_record_and_the_schema_blocks(): + seen: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + if request.url.path == "/v2/registrations/8wq3m/": + return httpx.Response(200, json=RAW) + if request.url.path == "/v2/schemas/registrations/schema1/schema_blocks/": + if request.url.params.get("page") == "2": + return httpx.Response( + 200, json={"data": BLOCKS[5:], "links": {"next": None}} + ) + nxt = str(request.url.copy_merge_params({"page": "2"})) + return httpx.Response( + 200, json={"data": BLOCKS[:5], "links": {"next": nxt}} + ) + return httpx.Response(404) + + transport = httpx.MockTransport(handler) + with httpx.Client( + transport=transport, headers={"User-Agent": USER_AGENT} + ) as client: + reg = fetch_registration(client) + assert reg == normalize(RAW["data"], BLOCKS) + assert all(r.headers["User-Agent"] == "stack-lead-release/1.0" for r in seen)