stack/src/lead/release/build.py

"""Build and verify a LEAD review release: a BagIt bag with RO-Crate inside.

    lead-review-v<X.Y>/
      bagit.txt  bag-info.txt  manifest-sha512.txt  tagmanifest-sha512.txt
      data/
        ro-crate-metadata.json
        protocol/registration-8wq3m.json   protocol/protocol.md
        ledger/<table>.parquet             (every lead_evidence table)
        screening/decisions.parquet        screening/prisma-flow.json
        bibliography/included.csl.json     bibliography/screened.csl.json
        tags/vocab-v<N>.yaml
        code/stack-<sha>.tar.gz

Two builds from the same commit and inputs are byte-identical apart from
``Bagging-Date`` and ``datePublished``/``endTime`` (the ``date``).
"""

from __future__ import annotations

import json
import platform
import re
import shutil
from dataclasses import dataclass
from datetime import date as _date
from pathlib import Path

from lead.release import content
from lead.release.bagit import validate_bag, write_bag
from lead.release.crate import build_crate, check_crate, write_crate
from lead.release.protocol import DOI, DOI_URL, render_protocol

GROUP = "lead-living-review"
_VERSION = re.compile(r"^\d+\.\d+$")


@dataclass(frozen=True)
class Inputs:
    """Where a build reads from. ``registration`` is the normalized OSF record."""

    ledger: Path
    labels: Path
    zotero: Path
    vocab: Path
    repo: Path
    registration: dict


@dataclass(frozen=True)
class BuildResult:
    path: Path
    files: int
    bytes: int
    commit: str
    dirty: bool
    warnings: tuple[str, ...]


def bag_name(version: str) -> str:
    return f"lead-review-v{version}"


def _json(path: Path, value: object) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(
        json.dumps(value, indent=2, ensure_ascii=False) + "\n",
        encoding="utf-8",
        newline="\n",
    )


def _shown(path: Path, repo: Path) -> str:
    """A repo-relative path, or just the file name; never a home directory."""
    try:
        return path.resolve().relative_to(repo.resolve()).as_posix()
    except ValueError:
        return path.name


def _tool_versions() -> dict[str, str]:
    import duckdb
    import polars

    return {
        "python": platform.python_version(),
        "duckdb": duckdb.__version__,
        "polars": polars.__version__,
    }


def _context(
    *,
    version: str,
    date: str,
    inputs: Inputs,
    commit: str,
    tables: list[str],
) -> list[dict]:
    """The build's CreateAction, the software it ran and the inputs it read."""
    repo = inputs.repo
    tools = _tool_versions()
    software = [
        {
            "@id": "#stack",
            "@type": "SoftwareApplication",
            "name": "stack",
            "version": commit,
            "description": "The monorepo that built this release; its source is "
            f"code/stack-{commit}.tar.gz.",
        },
        *(
            {
                "@id": f"#{name}",
                "@type": "SoftwareApplication",
                "name": name,
                "version": value,
            }
            for name, value in tools.items()
        ),
    ]
    sources = [
        {
            "@id": "#input-ledger",
            "@type": "Dataset",
            "name": "lead_evidence ledger (DuckDB)",
            "description": f"Every table in schema lead_evidence of "
            f"{_shown(inputs.ledger, repo)}: {', '.join(tables)}.",
        },
        {
            "@id": "#input-labels",
            "@type": "Dataset",
            "name": "LEAD screening labels (SQLite)",
            "description": f"Table label of {_shown(inputs.labels, repo)}, every "
            "row; e-mail actors replaced by reviewer-N.",
        },
        {
            "@id": "#input-zotero",
            "@type": "Dataset",
            "name": "Zotero library (SQLite)",
            "description": f"{_shown(inputs.zotero, repo)}, read-only. Records "
            "tagged project:lead-<strand> (not attachments, notes or "
            "module:prisma anchors); screened = stage:2-4 tags plus labelled "
            "keys, included = stage:4-included; flow counts from "
            "prisma.flow.count.",
        },
        {
            "@id": "#input-vocab",
            "@type": "CreativeWork",
            "name": "Theme vocabulary",
            "description": _shown(inputs.vocab, repo),
        },
    ]
    action = {
        "@id": "#build",
        "@type": "CreateAction",
        "name": f"stack lead release build --version {version}",
        "endTime": date,
        "instrument": [{"@id": s["@id"]} for s in software],
        "object": [{"@id": s["@id"]} for s in sources] + [{"@id": DOI_URL}],
        "result": {"@id": "./"},
    }
    return [action, *software, *sources]


def build_release(
    *,
    version: str,
    out: Path,
    inputs: Inputs,
    date: str = "",
    note: str = "",
    source_org: str = "",
    force: bool = False,
) -> BuildResult:
    """Write ``out/lead-review-v<version>/`` and return what went in.

    Raises ``ValueError`` for a version that is not ``MAJOR.MINOR`` or a
    date that is not ``YYYY-MM-DD``, and ``FileExistsError`` when the bag
    exists and ``force`` is not set. The bag is assembled beside its final
    path and moved into place only once it is complete.
    """
    if not _VERSION.fullmatch(version):
        raise ValueError(f"version must be MAJOR.MINOR, got {version!r}")
    date = date or _date.today().isoformat()
    if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", date):
        raise ValueError(f"date must be YYYY-MM-DD, got {date!r}")
    _date.fromisoformat(date)
    out = Path(out)
    dest = out / bag_name(version)
    if dest.exists() and not force:
        raise FileExistsError(f"{dest} exists; pass --force to replace it")
    for source in (inputs.ledger, inputs.labels, inputs.zotero, inputs.vocab):
        if not Path(source).is_file():
            raise FileNotFoundError(source)

    out.mkdir(parents=True, exist_ok=True)
    stage = out / f".{bag_name(version)}.partial"
    if stage.exists():
        shutil.rmtree(stage)
    data = stage / "data"
    data.mkdir(parents=True)
    warnings: list[str] = []
    try:
        reg = inputs.registration
        guid = reg.get("guid", "8wq3m")
        _json(data / "protocol" / f"registration-{guid}.json", reg)
        (data / "protocol" / "protocol.md").write_text(
            render_protocol(reg), encoding="utf-8", newline="\n"
        )

        ledger = content.read_ledger(Path(inputs.ledger))
        labels = content.read_labels(Path(inputs.labels))
        actors = [*labels["actor"].to_list()]
        for frame in ledger.values():
            if "actor" in frame.columns:
                actors.extend(frame["actor"].to_list())
        aliases = content.actor_aliases(actors)
        for table, frame in ledger.items():
            content.write_parquet(
                content.alias_actor(frame, aliases),
                data / "ledger" / f"{table}.parquet",
            )
        content.write_parquet(
            content.alias_actor(labels, aliases),
            data / "screening" / "decisions.parquet",
        )

        zotero = content.zotero_connect(Path(inputs.zotero))
        try:
            _json(data / "screening" / "prisma-flow.json", content.flow_counts(zotero))
            bib = content.bibliography(zotero, labels["zotero_key"].to_list())
        finally:
            zotero.close()
        _json(data / "bibliography" / "included.csl.json", bib.included)
        _json(data / "bibliography" / "screened.csl.json", bib.screened)
        if bib.missing:
            warnings.append(
                f"{len(bib.missing)} labelled key(s) not in the Zotero library: "
                + ", ".join(bib.missing)
            )

        content.copy_vocab(Path(inputs.vocab), data / "tags")
        code = content.archive_code(Path(inputs.repo), data / "code")
        if code.dirty:
            warnings.append(
                "the repo has uncommitted changes; the code archive is HEAD "
                f"{code.commit[:12]} without them"
            )

        based_on = {
            "@id": DOI_URL,
            "@type": "CreativeWork",
            "name": reg.get("title") or "OSF registration",
            "identifier": f"doi:{DOI}",
            "url": reg.get("url") or f"https://osf.io/{guid}/",
        }
        if reg.get("date_registered"):
            based_on["dateCreated"] = str(reg["date_registered"])[:10]
        description = (
            f"Release {version} of the LEAD living review registered at OSF "
            f"({DOI_URL}): the protocol, the evidence ledger, screening "
            "decisions and PRISMA flow counts, the bibliography of screened "
            "and included records, the theme vocabulary, and the code that "
            "built it."
        )
        crate = build_crate(
            data,
            name=f"LEAD living review, release {version}",
            description=description,
            version=version,
            date_published=date,
            based_on=based_on,
            entities=_context(
                version=version,
                date=date,
                inputs=inputs,
                commit=code.commit,
                tables=sorted(ledger),
            ),
        )
        write_crate(data, crate)

        info = []
        if source_org.strip():
            info.append(("Source-Organization", source_org.strip()))
        info += [
            ("External-Identifier", f"doi:{DOI}"),
            ("Bag-Group-Identifier", GROUP),
            ("Bagging-Date", date),
            ("Version", version),
            ("Git-Commit", code.commit),
        ]
        if note.strip():
            info.append(("Internal-Sender-Description", note.strip()))
        octets, files = write_bag(stage, info)

        if dest.exists():
            shutil.rmtree(dest)
        stage.rename(dest)
    except BaseException:
        shutil.rmtree(stage, ignore_errors=True)
        raise
    return BuildResult(
        path=dest,
        files=files,
        bytes=octets,
        commit=code.commit,
        dirty=code.dirty,
        warnings=tuple(warnings),
    )


def verify_release(bag: Path) -> list[str]:
    """BagIt validation plus RO-Crate checks; every problem, or empty."""
    bag = Path(bag)
    if not bag.is_dir():
        return [f"{bag} is not a directory"]
    problems = validate_bag(bag)
    if (bag / "data").is_dir():
        problems += [f"ro-crate: {p}" for p in check_crate(bag / "data")]
    return problems