Some checks failed
CI / lint (push) Successful in 1m25s
CI / notebooks-smoke (push) Successful in 2m12s
Deploy / changes (push) Successful in 22s
CI / test (push) Has been cancelled
Deploy / report (push) Has been cancelled
Deploy / notebooks (push) Has been cancelled
Deploy / zotero (push) Has been cancelled
Deploy / docs (push) Has been cancelled
Deploy / api (push) Has been cancelled
Deploy / bcda (push) Has been cancelled
Deploy / llm (push) Has been cancelled
Deploy / mc (push) Has been cancelled
Infra CI / api (push) Has been cancelled
Infra CI / bcda (push) Has been cancelled
Infra CI / zotero (push) Has been cancelled
Infra CI / docs (push) Has been cancelled
Infra CI / llm (push) Has been cancelled
Infra CI / mc (push) Has been cancelled
Infra CI / notebooks (push) Has started running
Each LEAD living-review update becomes a BagIt 1.0 bag (sha512 manifests, Payload-Oxum, bag-info with the OSF registration DOI and git commit) holding the protocol (OSF registration 8wq3m, verbatim and rendered), the evidence ledger tables, human screening decisions and PRISMA flow counts, included/screened bibliography as CSL-JSON, the theme vocabulary, and a source-only code archive (src, config, dev/scripts, specs — no third-party documents). RO-Crate 1.2 metadata describes every payload file with its sha256 and records the build. Reviewer email addresses are replaced with reviewer-N. Builds are deterministic for the same inputs; verify checks manifests, Oxum and the crate. Stdlib only, no new dependencies.
453 lines
16 KiB
Python
453 lines
16 KiB
Python
"""Read the review's state for a release. Every source is opened read-only.
|
|
|
|
- ledger: every table in the ``lead_evidence`` schema of the DuckDB ledger
|
|
- screening: the human labels (``lead.review``) and PRISMA flow counts
|
|
(``prisma.flow`` over the Zotero tags)
|
|
- bibliography: CSL-JSON for the screened and included Zotero records
|
|
- tags: the theme vocabulary file
|
|
- code: ``git archive`` of HEAD
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import dataclasses
|
|
import gzip
|
|
import os
|
|
import re
|
|
import shutil
|
|
import sqlite3
|
|
import subprocess
|
|
from collections.abc import Iterable, Mapping
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
import polars as pl
|
|
|
|
from lead.review.codes import PROJECTS
|
|
|
|
LEDGER_SCHEMA = "lead_evidence"
|
|
_EMAIL = re.compile(r"[^@\s]+@[^@\s]+\.[^@\s]+")
|
|
|
|
|
|
# ── actors ───────────────────────────────────────────────────────
|
|
|
|
|
|
def actor_aliases(actors: Iterable[str | None]) -> dict[str, str]:
|
|
"""``reviewer-N`` for each e-mail address, numbered in sorted order.
|
|
|
|
A release is public; a reviewer's address is not published with it.
|
|
Non-address actors (``stack``, a model name) are kept as they are.
|
|
"""
|
|
emails = sorted({a for a in actors if a and _EMAIL.fullmatch(a.strip())})
|
|
return {email: f"reviewer-{n}" for n, email in enumerate(emails, start=1)}
|
|
|
|
|
|
def alias_actor(frame: pl.DataFrame, aliases: Mapping[str, str]) -> pl.DataFrame:
|
|
if "actor" not in frame.columns or not aliases:
|
|
return frame
|
|
return frame.with_columns(pl.col("actor").replace(dict(aliases)))
|
|
|
|
|
|
def write_parquet(frame: pl.DataFrame, dest: Path) -> None:
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
frame.write_parquet(dest, compression="zstd", statistics=True)
|
|
|
|
|
|
# ── ledger ───────────────────────────────────────────────────────
|
|
|
|
|
|
def read_ledger(path: Path) -> dict[str, pl.DataFrame]:
|
|
"""Each ``lead_evidence`` table, rows sorted on every column."""
|
|
import duckdb
|
|
|
|
con = duckdb.connect(str(path), read_only=True)
|
|
try:
|
|
tables = [
|
|
row[0]
|
|
for row in con.execute(
|
|
"""SELECT table_name FROM information_schema.tables
|
|
WHERE table_schema = ? ORDER BY table_name""",
|
|
[LEDGER_SCHEMA],
|
|
).fetchall()
|
|
]
|
|
return {
|
|
table: con.execute(
|
|
f'SELECT * FROM {LEDGER_SCHEMA}."{table}" ORDER BY ALL'
|
|
).pl()
|
|
for table in tables
|
|
}
|
|
finally:
|
|
con.close()
|
|
|
|
|
|
# ── screening ────────────────────────────────────────────────────
|
|
|
|
_LABEL_SCHEMA = {
|
|
"label_id": pl.Int64,
|
|
"zotero_key": pl.String,
|
|
"project": pl.String,
|
|
"stage": pl.String,
|
|
"pass": pl.Int64,
|
|
"decision": pl.String,
|
|
"reason": pl.String,
|
|
"rationale": pl.String,
|
|
"actor": pl.String,
|
|
"created_at": pl.String,
|
|
}
|
|
|
|
|
|
def _sqlite_ro(path: Path) -> sqlite3.Connection:
|
|
"""Read-only, immutable: no lock taken, no -wal/-shm file written."""
|
|
if not path.is_file():
|
|
raise FileNotFoundError(path)
|
|
return sqlite3.connect(f"file:{path}?mode=ro&immutable=1", uri=True)
|
|
|
|
|
|
def read_labels(path: Path) -> pl.DataFrame:
|
|
"""Every human screening label, oldest first (the label file is append-only)."""
|
|
con = _sqlite_ro(path)
|
|
try:
|
|
rows = con.execute(
|
|
"""SELECT id, zotero_key, project, stage, pass, decision, reason,
|
|
rationale, actor, created_at
|
|
FROM label ORDER BY id"""
|
|
).fetchall()
|
|
finally:
|
|
con.close()
|
|
return pl.DataFrame(rows, schema=_LABEL_SCHEMA, orient="row")
|
|
|
|
|
|
def flow_counts(zotero: sqlite3.Connection) -> dict:
|
|
"""PRISMA 2020 flow counts per LEAD strand and per review, from Zotero tags."""
|
|
from prisma import flow
|
|
|
|
db = SimpleNamespace(con=zotero)
|
|
per = {project: flow.count(db, project) for project in sorted(PROJECTS)} # type: ignore[arg-type]
|
|
scoping = flow.combined([per[p] for p in flow.SCOPING_PROJECTS])
|
|
return {
|
|
"source": "prisma.flow.count over the Zotero tags project:<strand>, "
|
|
"stage:*, screen:*, dedup:duplicate and found-in:*",
|
|
"note": "Stage and screen tags are not scoped to a project: a record "
|
|
"shared with another review in the library is counted with that "
|
|
"review's stage tags. A record on two scoping strands is counted "
|
|
"twice in the scoping total.",
|
|
"projects": {p: dataclasses.asdict(c) for p, c in per.items()},
|
|
"reviews": {
|
|
"scoping": {
|
|
"projects": list(flow.SCOPING_PROJECTS),
|
|
**dataclasses.asdict(scoping),
|
|
},
|
|
"systematic": {
|
|
"projects": [flow.SYSTEMATIC_PROJECT],
|
|
**dataclasses.asdict(per[flow.SYSTEMATIC_PROJECT]),
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
# ── bibliography ─────────────────────────────────────────────────
|
|
|
|
_SKIP_TYPES = ("attachment", "note", "annotation")
|
|
_SCREENED_STAGES = ("stage:2-title-abstract", "stage:3-full-text", "stage:4-included")
|
|
_INCLUDED = "stage:4-included"
|
|
|
|
# Zotero item type -> CSL type.
|
|
CSL_TYPES = {
|
|
"artwork": "graphic",
|
|
"audioRecording": "song",
|
|
"bill": "bill",
|
|
"blogPost": "post-weblog",
|
|
"book": "book",
|
|
"bookSection": "chapter",
|
|
"case": "legal_case",
|
|
"computerProgram": "software",
|
|
"conferencePaper": "paper-conference",
|
|
"dataset": "dataset",
|
|
"dictionaryEntry": "entry-dictionary",
|
|
"document": "document",
|
|
"email": "personal_communication",
|
|
"encyclopediaArticle": "entry-encyclopedia",
|
|
"film": "motion_picture",
|
|
"forumPost": "post",
|
|
"hearing": "hearing",
|
|
"instantMessage": "personal_communication",
|
|
"interview": "interview",
|
|
"journalArticle": "article-journal",
|
|
"letter": "personal_communication",
|
|
"magazineArticle": "article-magazine",
|
|
"manuscript": "manuscript",
|
|
"map": "map",
|
|
"newspaperArticle": "article-newspaper",
|
|
"patent": "patent",
|
|
"podcast": "song",
|
|
"preprint": "article",
|
|
"presentation": "speech",
|
|
"radioBroadcast": "broadcast",
|
|
"report": "report",
|
|
"standard": "standard",
|
|
"statute": "legislation",
|
|
"thesis": "thesis",
|
|
"tvBroadcast": "broadcast",
|
|
"videoRecording": "motion_picture",
|
|
"webpage": "webpage",
|
|
}
|
|
_FIELDS = ("title", "nameOfAct", "caseName", "date", "dateEnacted", "DOI", "url")
|
|
_SQL_DATE = re.compile(r"^(\d{4})-(\d{2})-(\d{2})")
|
|
_YEAR = re.compile(r"\b(\d{4})\b")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Bibliography:
|
|
included: list[dict]
|
|
screened: list[dict]
|
|
missing: tuple[str, ...]
|
|
|
|
|
|
def issued(date: str) -> dict | None:
|
|
"""CSL ``issued`` from a Zotero date (``YYYY-MM-DD original``)."""
|
|
match = _SQL_DATE.match(date.strip())
|
|
if match:
|
|
parts = [int(p) for p in match.groups()]
|
|
while parts and parts[-1] == 0:
|
|
parts.pop()
|
|
return {"date-parts": [parts]} if parts else None
|
|
year = _YEAR.search(date)
|
|
return {"date-parts": [[int(year.group(1))]]} if year else None
|
|
|
|
|
|
def _lead_items(con: sqlite3.Connection) -> dict[int, tuple[str, str]]:
|
|
"""itemID -> (key, type) for records in a LEAD strand (no anchors)."""
|
|
tags = [f"project:{p}" for p in PROJECTS]
|
|
rows = con.execute(
|
|
f"""SELECT DISTINCT i.itemID, i.key, ty.typeName
|
|
FROM items i
|
|
JOIN itemTypes ty ON ty.itemTypeID = i.itemTypeID
|
|
JOIN itemTags it ON it.itemID = i.itemID
|
|
JOIN tags t ON t.tagID = it.tagID
|
|
WHERE t.name IN ({",".join("?" * len(tags))})
|
|
AND ty.typeName NOT IN (?, ?, ?)
|
|
AND i.itemID NOT IN (
|
|
SELECT it2.itemID FROM itemTags it2
|
|
JOIN tags t2 ON t2.tagID = it2.tagID
|
|
WHERE t2.name = 'module:prisma')""",
|
|
(*tags, *_SKIP_TYPES),
|
|
).fetchall()
|
|
return {row[0]: (row[1], row[2]) for row in rows}
|
|
|
|
|
|
def _tagged(
|
|
con: sqlite3.Connection, ids: Iterable[int], names: Iterable[str]
|
|
) -> set[int]:
|
|
wanted = list(names)
|
|
ids = list(ids)
|
|
if not ids or not wanted:
|
|
return set()
|
|
found: set[int] = set()
|
|
for start in range(0, len(ids), 500):
|
|
chunk = ids[start : start + 500]
|
|
rows = con.execute(
|
|
f"""SELECT DISTINCT it.itemID FROM itemTags it
|
|
JOIN tags t ON t.tagID = it.tagID
|
|
WHERE t.name IN ({",".join("?" * len(wanted))})
|
|
AND it.itemID IN ({",".join("?" * len(chunk))})""",
|
|
(*wanted, *chunk),
|
|
).fetchall()
|
|
found.update(row[0] for row in rows)
|
|
return found
|
|
|
|
|
|
def _by_key(con: sqlite3.Connection, keys: Iterable[str]) -> dict[str, tuple[int, str]]:
|
|
keys = sorted(set(keys))
|
|
out: dict[str, tuple[int, str]] = {}
|
|
for start in range(0, len(keys), 500):
|
|
chunk = keys[start : start + 500]
|
|
rows = con.execute(
|
|
f"""SELECT i.itemID, i.key, ty.typeName FROM items i
|
|
JOIN itemTypes ty ON ty.itemTypeID = i.itemTypeID
|
|
WHERE i.key IN ({",".join("?" * len(chunk))})
|
|
AND ty.typeName NOT IN (?, ?, ?)""",
|
|
(*chunk, *_SKIP_TYPES),
|
|
).fetchall()
|
|
out.update({row[1]: (row[0], row[2]) for row in rows})
|
|
return out
|
|
|
|
|
|
def csl_item(con: sqlite3.Connection, item_id: int, key: str, type_name: str) -> dict:
|
|
"""One record as CSL-JSON: id, type, title, author, issued, DOI, URL."""
|
|
fields = dict(
|
|
con.execute(
|
|
f"""SELECT f.fieldName, v.value FROM itemData d
|
|
JOIN fields f ON f.fieldID = d.fieldID
|
|
JOIN itemDataValues v ON v.valueID = d.valueID
|
|
WHERE d.itemID = ? AND f.fieldName IN ({",".join("?" * len(_FIELDS))})""",
|
|
(item_id, *_FIELDS),
|
|
).fetchall()
|
|
)
|
|
authors = []
|
|
for first, last, mode in con.execute(
|
|
"""SELECT c.firstName, c.lastName, c.fieldMode FROM itemCreators ic
|
|
JOIN creators c ON c.creatorID = ic.creatorID
|
|
JOIN creatorTypes ct ON ct.creatorTypeID = ic.creatorTypeID
|
|
WHERE ic.itemID = ? AND ct.creatorType = 'author'
|
|
ORDER BY ic.orderIndex""",
|
|
(item_id,),
|
|
).fetchall():
|
|
if mode == 1 or not first:
|
|
if last:
|
|
authors.append({"literal": last})
|
|
elif last:
|
|
authors.append({"family": last, "given": first})
|
|
else:
|
|
authors.append({"literal": first})
|
|
entry: dict = {"id": key, "type": CSL_TYPES.get(type_name, "document")}
|
|
title = fields.get("title") or fields.get("nameOfAct") or fields.get("caseName")
|
|
if title:
|
|
entry["title"] = str(title)
|
|
if authors:
|
|
entry["author"] = authors
|
|
when = issued(str(fields.get("date") or fields.get("dateEnacted") or ""))
|
|
if when:
|
|
entry["issued"] = when
|
|
if fields.get("DOI"):
|
|
entry["DOI"] = str(fields["DOI"])
|
|
if fields.get("url"):
|
|
entry["URL"] = str(fields["url"])
|
|
return entry
|
|
|
|
|
|
def bibliography(con: sqlite3.Connection, labelled: Iterable[str]) -> Bibliography:
|
|
"""CSL-JSON for the LEAD records, sorted by Zotero key.
|
|
|
|
Screened: every record with a human screening label, plus every LEAD
|
|
record that carries a stage 2-4 tag (the PRISMA ``screened`` box).
|
|
Included: LEAD records tagged ``stage:4-included``. Labelled keys that
|
|
are not in the library come back in ``missing``.
|
|
"""
|
|
lead = _lead_items(con)
|
|
staged = _tagged(con, lead, _SCREENED_STAGES)
|
|
included_ids = _tagged(con, lead, [_INCLUDED])
|
|
labelled = set(labelled)
|
|
known = _by_key(con, labelled)
|
|
screened: dict[str, tuple[int, str]] = {lead[i][0]: (i, lead[i][1]) for i in staged}
|
|
screened.update(known)
|
|
missing = tuple(sorted(labelled - set(known)))
|
|
entries = {
|
|
key: csl_item(con, item_id, key, type_name)
|
|
for key, (item_id, type_name) in screened.items()
|
|
}
|
|
included_keys = {lead[i][0] for i in included_ids}
|
|
return Bibliography(
|
|
included=[entries[k] for k in sorted(included_keys)],
|
|
screened=[entries[k] for k in sorted(entries)],
|
|
missing=missing,
|
|
)
|
|
|
|
|
|
def zotero_connect(path: Path) -> sqlite3.Connection:
|
|
return _sqlite_ro(path)
|
|
|
|
|
|
# ── tags ─────────────────────────────────────────────────────────
|
|
|
|
|
|
def vocab_version(path: Path) -> str:
|
|
import yaml
|
|
|
|
data = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
|
version = data.get("version") if isinstance(data, dict) else None
|
|
if version in (None, ""):
|
|
raise ValueError(f"{path} has no version")
|
|
return str(version)
|
|
|
|
|
|
def copy_vocab(path: Path, dest_dir: Path) -> Path:
|
|
dest = dest_dir / f"vocab-v{vocab_version(path)}.yaml"
|
|
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
shutil.copyfile(path, dest)
|
|
return dest
|
|
|
|
|
|
# ── code ─────────────────────────────────────────────────────────
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Code:
|
|
commit: str
|
|
dirty: bool
|
|
path: Path
|
|
|
|
|
|
def _git_env() -> dict[str, str]:
|
|
# A pre-commit hook exports GIT_DIR / GIT_INDEX_FILE; drop them so the
|
|
# command reads the repo it is pointed at.
|
|
return {k: v for k, v in os.environ.items() if not k.startswith("GIT_")}
|
|
|
|
|
|
def _git(repo: Path, *args: str) -> str:
|
|
return subprocess.run(
|
|
["git", "-C", str(repo), *args],
|
|
env=_git_env(),
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
).stdout.strip()
|
|
|
|
|
|
#: What a release's code archive carries: the code and config that build
|
|
#: it. Everything else tracked in the repo (vendor manuals under hw/docs,
|
|
#: books under dev/seeds, BCDA material) is third-party and stays out of a
|
|
#: public bag.
|
|
CODE_PATHS = (
|
|
"src",
|
|
"pyproject.toml",
|
|
"uv.lock",
|
|
"stack.toml",
|
|
"dev/scripts",
|
|
"docs/superpowers",
|
|
)
|
|
|
|
|
|
def archive_code(repo: Path, dest_dir: Path) -> Code:
|
|
"""``git archive`` of HEAD's :data:`CODE_PATHS` as
|
|
``stack-<sha>.tar.gz``; gzip carries no mtime.
|
|
|
|
Uncommitted changes are not in the archive; ``dirty`` says whether
|
|
there were any to tracked files.
|
|
"""
|
|
commit = _git(repo, "rev-parse", "HEAD")
|
|
tracked = set(_git(repo, "ls-tree", "--name-only", "HEAD").split())
|
|
paths = [p for p in CODE_PATHS if p.split("/")[0] in tracked]
|
|
dirty = bool(_git(repo, "status", "--porcelain", "--untracked-files=no"))
|
|
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
dest = dest_dir / f"stack-{commit}.tar.gz"
|
|
proc = subprocess.Popen(
|
|
[
|
|
"git",
|
|
"-C",
|
|
str(repo),
|
|
"archive",
|
|
"--format=tar",
|
|
f"--prefix=stack-{commit[:12]}/",
|
|
commit,
|
|
"--",
|
|
*paths,
|
|
],
|
|
env=_git_env(),
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
)
|
|
assert proc.stdout is not None
|
|
with dest.open("wb") as raw:
|
|
with gzip.GzipFile(
|
|
filename="", mode="wb", fileobj=raw, compresslevel=6, mtime=0
|
|
) as gz:
|
|
shutil.copyfileobj(proc.stdout, gz, 1 << 20)
|
|
proc.stdout.close()
|
|
err = proc.stderr.read() if proc.stderr else b""
|
|
if proc.stderr:
|
|
proc.stderr.close()
|
|
if proc.wait() != 0:
|
|
raise RuntimeError(f"git archive failed: {err.decode(errors='replace')}")
|
|
return Code(commit=commit, dirty=dirty, path=dest)
|