Files
stack/src/lead/release/content.py
kert bf9485e634
Some checks failed
CI / lint (push) Successful in 1m25s
CI / notebooks-smoke (push) Successful in 2m12s
Deploy / changes (push) Successful in 22s
CI / test (push) Has been cancelled
Deploy / report (push) Has been cancelled
Deploy / notebooks (push) Has been cancelled
Deploy / zotero (push) Has been cancelled
Deploy / docs (push) Has been cancelled
Deploy / api (push) Has been cancelled
Deploy / bcda (push) Has been cancelled
Deploy / llm (push) Has been cancelled
Deploy / mc (push) Has been cancelled
Infra CI / api (push) Has been cancelled
Infra CI / bcda (push) Has been cancelled
Infra CI / zotero (push) Has been cancelled
Infra CI / docs (push) Has been cancelled
Infra CI / llm (push) Has been cancelled
Infra CI / mc (push) Has been cancelled
Infra CI / notebooks (push) Has started running
feat(lead): stack lead release build/verify — BagIt + RO-Crate review releases
Each LEAD living-review update becomes a BagIt 1.0 bag (sha512
manifests, Payload-Oxum, bag-info with the OSF registration DOI and git
commit) holding the protocol (OSF registration 8wq3m, verbatim and
rendered), the evidence ledger tables, human screening decisions and
PRISMA flow counts, included/screened bibliography as CSL-JSON, the
theme vocabulary, and a source-only code archive (src, config,
dev/scripts, specs — no third-party documents). RO-Crate 1.2 metadata
describes every payload file with its sha256 and records the build.
Reviewer email addresses are replaced with reviewer-N. Builds are
deterministic for the same inputs; verify checks manifests, Oxum and
the crate. Stdlib only, no new dependencies.
2026-11-11 15:09:46 -05:00

453 lines
16 KiB
Python

"""Read the review's state for a release. Every source is opened read-only.
- ledger: every table in the ``lead_evidence`` schema of the DuckDB ledger
- screening: the human labels (``lead.review``) and PRISMA flow counts
(``prisma.flow`` over the Zotero tags)
- bibliography: CSL-JSON for the screened and included Zotero records
- tags: the theme vocabulary file
- code: ``git archive`` of HEAD
"""
from __future__ import annotations
import dataclasses
import gzip
import os
import re
import shutil
import sqlite3
import subprocess
from collections.abc import Iterable, Mapping
from dataclasses import dataclass
from pathlib import Path
from types import SimpleNamespace
import polars as pl
from lead.review.codes import PROJECTS
LEDGER_SCHEMA = "lead_evidence"
_EMAIL = re.compile(r"[^@\s]+@[^@\s]+\.[^@\s]+")
# ── actors ───────────────────────────────────────────────────────
def actor_aliases(actors: Iterable[str | None]) -> dict[str, str]:
"""``reviewer-N`` for each e-mail address, numbered in sorted order.
A release is public; a reviewer's address is not published with it.
Non-address actors (``stack``, a model name) are kept as they are.
"""
emails = sorted({a for a in actors if a and _EMAIL.fullmatch(a.strip())})
return {email: f"reviewer-{n}" for n, email in enumerate(emails, start=1)}
def alias_actor(frame: pl.DataFrame, aliases: Mapping[str, str]) -> pl.DataFrame:
if "actor" not in frame.columns or not aliases:
return frame
return frame.with_columns(pl.col("actor").replace(dict(aliases)))
def write_parquet(frame: pl.DataFrame, dest: Path) -> None:
dest.parent.mkdir(parents=True, exist_ok=True)
frame.write_parquet(dest, compression="zstd", statistics=True)
# ── ledger ───────────────────────────────────────────────────────
def read_ledger(path: Path) -> dict[str, pl.DataFrame]:
"""Each ``lead_evidence`` table, rows sorted on every column."""
import duckdb
con = duckdb.connect(str(path), read_only=True)
try:
tables = [
row[0]
for row in con.execute(
"""SELECT table_name FROM information_schema.tables
WHERE table_schema = ? ORDER BY table_name""",
[LEDGER_SCHEMA],
).fetchall()
]
return {
table: con.execute(
f'SELECT * FROM {LEDGER_SCHEMA}."{table}" ORDER BY ALL'
).pl()
for table in tables
}
finally:
con.close()
# ── screening ────────────────────────────────────────────────────
_LABEL_SCHEMA = {
"label_id": pl.Int64,
"zotero_key": pl.String,
"project": pl.String,
"stage": pl.String,
"pass": pl.Int64,
"decision": pl.String,
"reason": pl.String,
"rationale": pl.String,
"actor": pl.String,
"created_at": pl.String,
}
def _sqlite_ro(path: Path) -> sqlite3.Connection:
"""Read-only, immutable: no lock taken, no -wal/-shm file written."""
if not path.is_file():
raise FileNotFoundError(path)
return sqlite3.connect(f"file:{path}?mode=ro&immutable=1", uri=True)
def read_labels(path: Path) -> pl.DataFrame:
"""Every human screening label, oldest first (the label file is append-only)."""
con = _sqlite_ro(path)
try:
rows = con.execute(
"""SELECT id, zotero_key, project, stage, pass, decision, reason,
rationale, actor, created_at
FROM label ORDER BY id"""
).fetchall()
finally:
con.close()
return pl.DataFrame(rows, schema=_LABEL_SCHEMA, orient="row")
def flow_counts(zotero: sqlite3.Connection) -> dict:
"""PRISMA 2020 flow counts per LEAD strand and per review, from Zotero tags."""
from prisma import flow
db = SimpleNamespace(con=zotero)
per = {project: flow.count(db, project) for project in sorted(PROJECTS)} # type: ignore[arg-type]
scoping = flow.combined([per[p] for p in flow.SCOPING_PROJECTS])
return {
"source": "prisma.flow.count over the Zotero tags project:<strand>, "
"stage:*, screen:*, dedup:duplicate and found-in:*",
"note": "Stage and screen tags are not scoped to a project: a record "
"shared with another review in the library is counted with that "
"review's stage tags. A record on two scoping strands is counted "
"twice in the scoping total.",
"projects": {p: dataclasses.asdict(c) for p, c in per.items()},
"reviews": {
"scoping": {
"projects": list(flow.SCOPING_PROJECTS),
**dataclasses.asdict(scoping),
},
"systematic": {
"projects": [flow.SYSTEMATIC_PROJECT],
**dataclasses.asdict(per[flow.SYSTEMATIC_PROJECT]),
},
},
}
# ── bibliography ─────────────────────────────────────────────────
_SKIP_TYPES = ("attachment", "note", "annotation")
_SCREENED_STAGES = ("stage:2-title-abstract", "stage:3-full-text", "stage:4-included")
_INCLUDED = "stage:4-included"
# Zotero item type -> CSL type.
CSL_TYPES = {
"artwork": "graphic",
"audioRecording": "song",
"bill": "bill",
"blogPost": "post-weblog",
"book": "book",
"bookSection": "chapter",
"case": "legal_case",
"computerProgram": "software",
"conferencePaper": "paper-conference",
"dataset": "dataset",
"dictionaryEntry": "entry-dictionary",
"document": "document",
"email": "personal_communication",
"encyclopediaArticle": "entry-encyclopedia",
"film": "motion_picture",
"forumPost": "post",
"hearing": "hearing",
"instantMessage": "personal_communication",
"interview": "interview",
"journalArticle": "article-journal",
"letter": "personal_communication",
"magazineArticle": "article-magazine",
"manuscript": "manuscript",
"map": "map",
"newspaperArticle": "article-newspaper",
"patent": "patent",
"podcast": "song",
"preprint": "article",
"presentation": "speech",
"radioBroadcast": "broadcast",
"report": "report",
"standard": "standard",
"statute": "legislation",
"thesis": "thesis",
"tvBroadcast": "broadcast",
"videoRecording": "motion_picture",
"webpage": "webpage",
}
_FIELDS = ("title", "nameOfAct", "caseName", "date", "dateEnacted", "DOI", "url")
_SQL_DATE = re.compile(r"^(\d{4})-(\d{2})-(\d{2})")
_YEAR = re.compile(r"\b(\d{4})\b")
@dataclass(frozen=True)
class Bibliography:
included: list[dict]
screened: list[dict]
missing: tuple[str, ...]
def issued(date: str) -> dict | None:
"""CSL ``issued`` from a Zotero date (``YYYY-MM-DD original``)."""
match = _SQL_DATE.match(date.strip())
if match:
parts = [int(p) for p in match.groups()]
while parts and parts[-1] == 0:
parts.pop()
return {"date-parts": [parts]} if parts else None
year = _YEAR.search(date)
return {"date-parts": [[int(year.group(1))]]} if year else None
def _lead_items(con: sqlite3.Connection) -> dict[int, tuple[str, str]]:
"""itemID -> (key, type) for records in a LEAD strand (no anchors)."""
tags = [f"project:{p}" for p in PROJECTS]
rows = con.execute(
f"""SELECT DISTINCT i.itemID, i.key, ty.typeName
FROM items i
JOIN itemTypes ty ON ty.itemTypeID = i.itemTypeID
JOIN itemTags it ON it.itemID = i.itemID
JOIN tags t ON t.tagID = it.tagID
WHERE t.name IN ({",".join("?" * len(tags))})
AND ty.typeName NOT IN (?, ?, ?)
AND i.itemID NOT IN (
SELECT it2.itemID FROM itemTags it2
JOIN tags t2 ON t2.tagID = it2.tagID
WHERE t2.name = 'module:prisma')""",
(*tags, *_SKIP_TYPES),
).fetchall()
return {row[0]: (row[1], row[2]) for row in rows}
def _tagged(
con: sqlite3.Connection, ids: Iterable[int], names: Iterable[str]
) -> set[int]:
wanted = list(names)
ids = list(ids)
if not ids or not wanted:
return set()
found: set[int] = set()
for start in range(0, len(ids), 500):
chunk = ids[start : start + 500]
rows = con.execute(
f"""SELECT DISTINCT it.itemID FROM itemTags it
JOIN tags t ON t.tagID = it.tagID
WHERE t.name IN ({",".join("?" * len(wanted))})
AND it.itemID IN ({",".join("?" * len(chunk))})""",
(*wanted, *chunk),
).fetchall()
found.update(row[0] for row in rows)
return found
def _by_key(con: sqlite3.Connection, keys: Iterable[str]) -> dict[str, tuple[int, str]]:
keys = sorted(set(keys))
out: dict[str, tuple[int, str]] = {}
for start in range(0, len(keys), 500):
chunk = keys[start : start + 500]
rows = con.execute(
f"""SELECT i.itemID, i.key, ty.typeName FROM items i
JOIN itemTypes ty ON ty.itemTypeID = i.itemTypeID
WHERE i.key IN ({",".join("?" * len(chunk))})
AND ty.typeName NOT IN (?, ?, ?)""",
(*chunk, *_SKIP_TYPES),
).fetchall()
out.update({row[1]: (row[0], row[2]) for row in rows})
return out
def csl_item(con: sqlite3.Connection, item_id: int, key: str, type_name: str) -> dict:
"""One record as CSL-JSON: id, type, title, author, issued, DOI, URL."""
fields = dict(
con.execute(
f"""SELECT f.fieldName, v.value FROM itemData d
JOIN fields f ON f.fieldID = d.fieldID
JOIN itemDataValues v ON v.valueID = d.valueID
WHERE d.itemID = ? AND f.fieldName IN ({",".join("?" * len(_FIELDS))})""",
(item_id, *_FIELDS),
).fetchall()
)
authors = []
for first, last, mode in con.execute(
"""SELECT c.firstName, c.lastName, c.fieldMode FROM itemCreators ic
JOIN creators c ON c.creatorID = ic.creatorID
JOIN creatorTypes ct ON ct.creatorTypeID = ic.creatorTypeID
WHERE ic.itemID = ? AND ct.creatorType = 'author'
ORDER BY ic.orderIndex""",
(item_id,),
).fetchall():
if mode == 1 or not first:
if last:
authors.append({"literal": last})
elif last:
authors.append({"family": last, "given": first})
else:
authors.append({"literal": first})
entry: dict = {"id": key, "type": CSL_TYPES.get(type_name, "document")}
title = fields.get("title") or fields.get("nameOfAct") or fields.get("caseName")
if title:
entry["title"] = str(title)
if authors:
entry["author"] = authors
when = issued(str(fields.get("date") or fields.get("dateEnacted") or ""))
if when:
entry["issued"] = when
if fields.get("DOI"):
entry["DOI"] = str(fields["DOI"])
if fields.get("url"):
entry["URL"] = str(fields["url"])
return entry
def bibliography(con: sqlite3.Connection, labelled: Iterable[str]) -> Bibliography:
"""CSL-JSON for the LEAD records, sorted by Zotero key.
Screened: every record with a human screening label, plus every LEAD
record that carries a stage 2-4 tag (the PRISMA ``screened`` box).
Included: LEAD records tagged ``stage:4-included``. Labelled keys that
are not in the library come back in ``missing``.
"""
lead = _lead_items(con)
staged = _tagged(con, lead, _SCREENED_STAGES)
included_ids = _tagged(con, lead, [_INCLUDED])
labelled = set(labelled)
known = _by_key(con, labelled)
screened: dict[str, tuple[int, str]] = {lead[i][0]: (i, lead[i][1]) for i in staged}
screened.update(known)
missing = tuple(sorted(labelled - set(known)))
entries = {
key: csl_item(con, item_id, key, type_name)
for key, (item_id, type_name) in screened.items()
}
included_keys = {lead[i][0] for i in included_ids}
return Bibliography(
included=[entries[k] for k in sorted(included_keys)],
screened=[entries[k] for k in sorted(entries)],
missing=missing,
)
def zotero_connect(path: Path) -> sqlite3.Connection:
return _sqlite_ro(path)
# ── tags ─────────────────────────────────────────────────────────
def vocab_version(path: Path) -> str:
import yaml
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
version = data.get("version") if isinstance(data, dict) else None
if version in (None, ""):
raise ValueError(f"{path} has no version")
return str(version)
def copy_vocab(path: Path, dest_dir: Path) -> Path:
dest = dest_dir / f"vocab-v{vocab_version(path)}.yaml"
dest_dir.mkdir(parents=True, exist_ok=True)
shutil.copyfile(path, dest)
return dest
# ── code ─────────────────────────────────────────────────────────
@dataclass(frozen=True)
class Code:
commit: str
dirty: bool
path: Path
def _git_env() -> dict[str, str]:
# A pre-commit hook exports GIT_DIR / GIT_INDEX_FILE; drop them so the
# command reads the repo it is pointed at.
return {k: v for k, v in os.environ.items() if not k.startswith("GIT_")}
def _git(repo: Path, *args: str) -> str:
return subprocess.run(
["git", "-C", str(repo), *args],
env=_git_env(),
check=True,
capture_output=True,
text=True,
).stdout.strip()
#: What a release's code archive carries: the code and config that build
#: it. Everything else tracked in the repo (vendor manuals under hw/docs,
#: books under dev/seeds, BCDA material) is third-party and stays out of a
#: public bag.
CODE_PATHS = (
"src",
"pyproject.toml",
"uv.lock",
"stack.toml",
"dev/scripts",
"docs/superpowers",
)
def archive_code(repo: Path, dest_dir: Path) -> Code:
"""``git archive`` of HEAD's :data:`CODE_PATHS` as
``stack-<sha>.tar.gz``; gzip carries no mtime.
Uncommitted changes are not in the archive; ``dirty`` says whether
there were any to tracked files.
"""
commit = _git(repo, "rev-parse", "HEAD")
tracked = set(_git(repo, "ls-tree", "--name-only", "HEAD").split())
paths = [p for p in CODE_PATHS if p.split("/")[0] in tracked]
dirty = bool(_git(repo, "status", "--porcelain", "--untracked-files=no"))
dest_dir.mkdir(parents=True, exist_ok=True)
dest = dest_dir / f"stack-{commit}.tar.gz"
proc = subprocess.Popen(
[
"git",
"-C",
str(repo),
"archive",
"--format=tar",
f"--prefix=stack-{commit[:12]}/",
commit,
"--",
*paths,
],
env=_git_env(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
assert proc.stdout is not None
with dest.open("wb") as raw:
with gzip.GzipFile(
filename="", mode="wb", fileobj=raw, compresslevel=6, mtime=0
) as gz:
shutil.copyfileobj(proc.stdout, gz, 1 << 20)
proc.stdout.close()
err = proc.stderr.read() if proc.stderr else b""
if proc.stderr:
proc.stderr.close()
if proc.wait() != 0:
raise RuntimeError(f"git archive failed: {err.decode(errors='replace')}")
return Code(commit=commit, dirty=dirty, path=dest)