diff --git a/src/cli/lead.py b/src/cli/lead.py index ae3ab69..9c8e008 100644 --- a/src/cli/lead.py +++ b/src/cli/lead.py @@ -201,6 +201,11 @@ def release_build( vocab: str = typer.Option( "", "--vocab", help="Theme vocabulary (default src/llm/vocab/themes.yaml)." ), + bib: str = typer.Option( + "", + "--bib", + help="Bib sqlite with comment llm_tags (default [db] bib).", + ), repo: str = typer.Option("", "--repo", help="Git repo to archive (default stack)."), source_org: str = typer.Option( "", @@ -233,6 +238,7 @@ def release_build( else ROOT / "data" / "lead" / "review" / "labels.sqlite", zotero=Path(zotero) if zotero else conf_path("db.zotero"), vocab=Path(vocab) if vocab else ROOT / "src" / "llm" / "vocab" / "themes.yaml", + bib=Path(bib) if bib else conf_path("db.bib"), repo=Path(repo) if repo else ROOT, registration=reg, ) diff --git a/src/lead/release/build.py b/src/lead/release/build.py index ff48977..91cd999 100644 --- a/src/lead/release/build.py +++ b/src/lead/release/build.py @@ -9,6 +9,8 @@ screening/decisions.parquet screening/prisma-flow.json bibliography/included.csl.json bibliography/screened.csl.json tags/vocab-v.yaml + tags/assignments.parquet item_key, theme, confidence, model, + vocab_version, evidence_chunk code/stack-.tar.gz Two builds from the same commit and inputs are byte-identical apart from @@ -42,6 +44,7 @@ class Inputs: labels: Path zotero: Path vocab: Path + bib: Path repo: Path registration: dict @@ -149,6 +152,14 @@ def _context( "name": "Theme vocabulary", "description": _shown(inputs.vocab, repo), }, + { + "@id": "#input-tags", + "@type": "Dataset", + "name": "Comment theme assignments", + "description": f"{_shown(inputs.bib, repo)}, read-only. One row per " + "accepted llm_tags theme: item key, theme, confidence, model, " + "vocab version and evidence chunk.", + }, ] action = { "@id": "#build", @@ -189,7 +200,13 @@ def build_release( dest = out / bag_name(version) if dest.exists() and not force: raise FileExistsError(f"{dest} exists; pass --force to replace it") - for source in (inputs.ledger, inputs.labels, inputs.zotero, inputs.vocab): + for source in ( + inputs.ledger, + inputs.labels, + inputs.zotero, + inputs.vocab, + inputs.bib, + ): if not Path(source).is_file(): raise FileNotFoundError(source) @@ -240,6 +257,10 @@ def build_release( ) content.copy_vocab(Path(inputs.vocab), data / "tags") + content.write_parquet( + content.read_assignments(Path(inputs.bib)), + data / "tags" / "assignments.parquet", + ) code = content.archive_code(Path(inputs.repo), data / "code") if code.dirty: warnings.append( @@ -260,8 +281,8 @@ def build_release( f"Release {version} of the LEAD living review registered at OSF " f"({DOI_URL}): the protocol, the evidence ledger, screening " "decisions and PRISMA flow counts, the bibliography of screened " - "and included records, the theme vocabulary, and the code that " - "built it." + "and included records, the theme vocabulary, the comment tag " + "assignments, and the code that built it." ) crate = build_crate( data, diff --git a/src/lead/release/content.py b/src/lead/release/content.py index 1a5e885..4c0749e 100644 --- a/src/lead/release/content.py +++ b/src/lead/release/content.py @@ -4,7 +4,9 @@ - screening: the human labels (``lead.review``) and PRISMA flow counts (``prisma.flow`` over the Zotero tags) - bibliography: CSL-JSON for the screened and included Zotero records -- tags: the theme vocabulary file +- tags: the theme vocabulary file, and one row per accepted comment + tag (``items.extra_json.llm_tags``) with its model, vocab version and + evidence chunk - code: ``git archive`` of HEAD """ @@ -12,6 +14,7 @@ from __future__ import annotations import dataclasses import gzip +import json import os import re import shutil @@ -368,6 +371,60 @@ def copy_vocab(path: Path, dest_dir: Path) -> Path: return dest +_ASSIGNMENT_SCHEMA = { + "item_key": pl.String, + "theme": pl.String, + "confidence": pl.Float64, + "model": pl.String, + "vocab_version": pl.Int64, + "evidence_chunk": pl.Int64, +} + + +def read_assignments(path: Path) -> pl.DataFrame: + """One row per accepted comment tag stored on a bib item. + + Rows are ``(item_key, theme)`` sorted. An empty ``tags`` object and an + item with no ``llm_tags`` contribute nothing. ``vocab_version`` and + ``model`` are the values stored with the assignment, which can be older + than the vocabulary file copied beside it. ``evidence_chunk`` is null + when the stored tag has none. + """ + con = _sqlite_ro(path) + try: + rows = con.execute( + """SELECT key, extra_json FROM items + WHERE json_extract(extra_json, '$.llm_tags') IS NOT NULL + ORDER BY key""" + ).fetchall() + finally: + con.close() + records: list[dict] = [] + for key, raw in rows: + state = json.loads(raw).get("llm_tags") or {} + tags = state.get("tags") or {} + if not isinstance(tags, dict): + raise ValueError(f"{key}: llm_tags.tags is not an object") + model = state.get("model") + version = state.get("version") + for theme, info in tags.items(): + if not isinstance(info, dict): + raise ValueError(f"{key}: tag {theme} is not an object") + chunk = info.get("evidence_chunk") + records.append( + { + "item_key": key, + "theme": str(theme), + "confidence": info.get("confidence"), + "model": None if model is None else str(model), + "vocab_version": None if version is None else int(version), + "evidence_chunk": None if chunk is None else int(chunk), + } + ) + records.sort(key=lambda row: (row["item_key"], row["theme"])) + return pl.DataFrame(records, schema=_ASSIGNMENT_SCHEMA) + + # ── code ───────────────────────────────────────────────────────── diff --git a/tests/lead/test_release_build.py b/tests/lead/test_release_build.py index 3d2e710..fcd16e0 100644 --- a/tests/lead/test_release_build.py +++ b/tests/lead/test_release_build.py @@ -20,6 +20,7 @@ from cli import app from lead.ledger.store import append, open_ledger from lead.release import Inputs, build_release, verify_release from lead.release.bagit import read_bag_info +from lead.release.content import read_assignments from lead.release.protocol import normalize from lead.review.store import Store @@ -277,6 +278,69 @@ def _vocab(root: Path) -> Path: return path +def _bib(root: Path) -> Path: + """Comment tag state as the tagger stores it on bib items.""" + path = root / "bib.sqlite" + con = sqlite3.connect(path) + try: + con.execute("CREATE TABLE items (key TEXT PRIMARY KEY, extra_json TEXT)") + con.executemany( + "INSERT INTO items (key, extra_json) VALUES (?, ?)", + [ + ( + "CCC", + json.dumps( + { + "llm_tags": { + "version": 3, + "model": "qwen2.5:14b", + "tags": { + "work-rvu": { + "confidence": 0.75, + "evidence_chunk": 2, + }, + "evaluation-management": { + "confidence": 0.5, + "evidence_chunk": 0, + }, + }, + } + } + ), + ), + ( + "AAA", + json.dumps( + { + "llm_tags": { + "version": 3, + "model": "qwen2.5:14b", + "tags": {}, + } + } + ), + ), + ("BBB", json.dumps({"other": 1})), + ( + "DDD", + json.dumps( + { + "llm_tags": { + "version": 2, + "model": "qwen2.5:14b", + "tags": {"caregiver-training": {"confidence": 0.625}}, + } + } + ), + ), + ], + ) + con.commit() + finally: + con.close() + return path + + @pytest.fixture(scope="module") def inputs(tmp_path_factory: pytest.TempPathFactory) -> Inputs: """Built once per module: the build only reads them.""" @@ -286,6 +350,7 @@ def inputs(tmp_path_factory: pytest.TempPathFactory) -> Inputs: labels=_labels(src), zotero=_zotero(src), vocab=_vocab(src), + bib=_bib(src), repo=_repo(src), registration=REGISTRATION, ) @@ -339,6 +404,7 @@ def test_build_writes_a_valid_bag(inputs, tmp_path): "data/ro-crate-metadata.json", "data/screening/decisions.parquet", "data/screening/prisma-flow.json", + "data/tags/assignments.parquet", "data/tags/vocab-v3.yaml", ] ) @@ -354,7 +420,7 @@ def test_build_writes_a_valid_bag(inputs, tmp_path): def test_build_does_not_touch_its_inputs(inputs, tmp_path): - files = [inputs.ledger, inputs.labels, inputs.zotero, inputs.vocab] + files = [inputs.ledger, inputs.labels, inputs.zotero, inputs.vocab, inputs.bib] before = {p: (_sha(p), p.stat().st_mtime_ns) for p in files} siblings = {p.parent: sorted(x.name for x in p.parent.iterdir()) for p in files} _build(inputs, tmp_path / "out") @@ -479,11 +545,96 @@ def test_bibliography_is_csl_json(inputs, tmp_path): assert any("GGGG8888" in w for w in result.warnings) +def test_read_assignments_is_empty_without_accepted_tags(tmp_path): + path = tmp_path / "bib.sqlite" + con = sqlite3.connect(path) + con.execute("CREATE TABLE items (key TEXT, extra_json TEXT)") + con.execute("INSERT INTO items VALUES ('BBB', ?)", (json.dumps({"other": 1}),)) + con.execute( + "INSERT INTO items VALUES ('AAA', ?)", + (json.dumps({"llm_tags": {"version": 3, "model": "qwen2.5:14b", "tags": {}}}),), + ) + con.commit() + con.close() + frame = read_assignments(path) + assert frame.columns == [ + "item_key", + "theme", + "confidence", + "model", + "vocab_version", + "evidence_chunk", + ] + assert frame.height == 0 + + +def test_read_assignments_rejects_a_non_object(tmp_path): + path = tmp_path / "bib.sqlite" + con = sqlite3.connect(path) + con.execute("CREATE TABLE items (key TEXT, extra_json TEXT)") + con.execute( + "INSERT INTO items VALUES ('X', ?)", + (json.dumps({"llm_tags": {"tags": ["work-rvu"]}}),), + ) + con.execute( + "INSERT INTO items VALUES ('Y', ?)", + (json.dumps({"llm_tags": {"tags": {"work-rvu": "yes"}}}),), + ) + con.commit() + con.close() + with pytest.raises(ValueError, match="X: llm_tags.tags is not an object"): + read_assignments(path) + con = sqlite3.connect(path) + con.execute("DELETE FROM items WHERE key = 'X'") + con.commit() + con.close() + with pytest.raises(ValueError, match="Y: tag work-rvu is not an object"): + read_assignments(path) + + def test_tags_and_code(inputs, tmp_path): inputs = dataclasses.replace(inputs, repo=_repo(tmp_path)) result = _build(inputs, tmp_path / "out") bag = result.path assert (bag / "data/tags/vocab-v3.yaml").read_bytes() == inputs.vocab.read_bytes() + assigned = pl.read_parquet(bag / "data/tags/assignments.parquet") + assert assigned.columns == [ + "item_key", + "theme", + "confidence", + "model", + "vocab_version", + "evidence_chunk", + ] + assert assigned.rows(named=True) == [ + { + "item_key": "CCC", + "theme": "evaluation-management", + "confidence": 0.5, + "model": "qwen2.5:14b", + "vocab_version": 3, + "evidence_chunk": 0, + }, + { + "item_key": "CCC", + "theme": "work-rvu", + "confidence": 0.75, + "model": "qwen2.5:14b", + "vocab_version": 3, + "evidence_chunk": 2, + }, + { + "item_key": "DDD", + "theme": "caregiver-training", + "confidence": 0.625, + "model": "qwen2.5:14b", + "vocab_version": 2, + "evidence_chunk": None, + }, + ] + crate = json.loads((bag / "data/ro-crate-metadata.json").read_text()) + names = [e.get("name") for e in crate["@graph"]] + assert "Comment theme assignments" in names archive = bag / f"data/code/stack-{result.commit}.tar.gz" with tarfile.open(archive) as tar: names = tar.getnames() @@ -586,6 +737,8 @@ def _cli_args(inputs: Inputs, out: Path, reg: Path) -> list[str]: str(inputs.zotero), "--vocab", str(inputs.vocab), + "--bib", + str(inputs.bib), "--repo", str(inputs.repo), ]