feat(lead): snapshot comment tag assignments in a release bag
Some checks failed
CI / lint (push) Successful in 48s
CI / notebooks-smoke (push) Successful in 1m50s
Deploy / changes (push) Successful in 17s
CI / test (push) Successful in 4m17s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Lean Pay / pay (push) Failing after 58s
Deploy / api (push) Successful in 1m32s
Deploy / bcda (push) Successful in 1m36s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 1m10s
Deploy / report (push) Successful in 15s
Infra CI / zotero (push) Successful in 15s
Infra CI / docs (push) Successful in 19s
Infra CI / notebooks (push) Successful in 54s
Infra CI / api (push) Successful in 20s
Infra CI / bcda (push) Successful in 20s
Infra CI / llm (push) Successful in 16s
Infra CI / mc (push) Successful in 19s
Package Supply Chain / pkg-supply-chain (push) Successful in 1m30s

Each accepted llm_tags theme is a row with its item, theme, confidence,
model, vocab version, and evidence chunk.
This commit is contained in:
kert
2026-11-17 13:57:37 -05:00
parent d2b205fea8
commit 754a43620f
4 changed files with 242 additions and 5 deletions

View File

@@ -201,6 +201,11 @@ def release_build(
vocab: str = typer.Option(
"", "--vocab", help="Theme vocabulary (default src/llm/vocab/themes.yaml)."
),
bib: str = typer.Option(
"",
"--bib",
help="Bib sqlite with comment llm_tags (default [db] bib).",
),
repo: str = typer.Option("", "--repo", help="Git repo to archive (default stack)."),
source_org: str = typer.Option(
"",
@@ -233,6 +238,7 @@ def release_build(
else ROOT / "data" / "lead" / "review" / "labels.sqlite",
zotero=Path(zotero) if zotero else conf_path("db.zotero"),
vocab=Path(vocab) if vocab else ROOT / "src" / "llm" / "vocab" / "themes.yaml",
bib=Path(bib) if bib else conf_path("db.bib"),
repo=Path(repo) if repo else ROOT,
registration=reg,
)

View File

@@ -9,6 +9,8 @@
screening/decisions.parquet screening/prisma-flow.json
bibliography/included.csl.json bibliography/screened.csl.json
tags/vocab-v<N>.yaml
tags/assignments.parquet item_key, theme, confidence, model,
vocab_version, evidence_chunk
code/stack-<sha>.tar.gz
Two builds from the same commit and inputs are byte-identical apart from
@@ -42,6 +44,7 @@ class Inputs:
labels: Path
zotero: Path
vocab: Path
bib: Path
repo: Path
registration: dict
@@ -149,6 +152,14 @@ def _context(
"name": "Theme vocabulary",
"description": _shown(inputs.vocab, repo),
},
{
"@id": "#input-tags",
"@type": "Dataset",
"name": "Comment theme assignments",
"description": f"{_shown(inputs.bib, repo)}, read-only. One row per "
"accepted llm_tags theme: item key, theme, confidence, model, "
"vocab version and evidence chunk.",
},
]
action = {
"@id": "#build",
@@ -189,7 +200,13 @@ def build_release(
dest = out / bag_name(version)
if dest.exists() and not force:
raise FileExistsError(f"{dest} exists; pass --force to replace it")
for source in (inputs.ledger, inputs.labels, inputs.zotero, inputs.vocab):
for source in (
inputs.ledger,
inputs.labels,
inputs.zotero,
inputs.vocab,
inputs.bib,
):
if not Path(source).is_file():
raise FileNotFoundError(source)
@@ -240,6 +257,10 @@ def build_release(
)
content.copy_vocab(Path(inputs.vocab), data / "tags")
content.write_parquet(
content.read_assignments(Path(inputs.bib)),
data / "tags" / "assignments.parquet",
)
code = content.archive_code(Path(inputs.repo), data / "code")
if code.dirty:
warnings.append(
@@ -260,8 +281,8 @@ def build_release(
f"Release {version} of the LEAD living review registered at OSF "
f"({DOI_URL}): the protocol, the evidence ledger, screening "
"decisions and PRISMA flow counts, the bibliography of screened "
"and included records, the theme vocabulary, and the code that "
"built it."
"and included records, the theme vocabulary, the comment tag "
"assignments, and the code that built it."
)
crate = build_crate(
data,

View File

@@ -4,7 +4,9 @@
- screening: the human labels (``lead.review``) and PRISMA flow counts
(``prisma.flow`` over the Zotero tags)
- bibliography: CSL-JSON for the screened and included Zotero records
- tags: the theme vocabulary file
- tags: the theme vocabulary file, and one row per accepted comment
tag (``items.extra_json.llm_tags``) with its model, vocab version and
evidence chunk
- code: ``git archive`` of HEAD
"""
@@ -12,6 +14,7 @@ from __future__ import annotations
import dataclasses
import gzip
import json
import os
import re
import shutil
@@ -368,6 +371,60 @@ def copy_vocab(path: Path, dest_dir: Path) -> Path:
return dest
_ASSIGNMENT_SCHEMA = {
"item_key": pl.String,
"theme": pl.String,
"confidence": pl.Float64,
"model": pl.String,
"vocab_version": pl.Int64,
"evidence_chunk": pl.Int64,
}
def read_assignments(path: Path) -> pl.DataFrame:
"""One row per accepted comment tag stored on a bib item.
Rows are ``(item_key, theme)`` sorted. An empty ``tags`` object and an
item with no ``llm_tags`` contribute nothing. ``vocab_version`` and
``model`` are the values stored with the assignment, which can be older
than the vocabulary file copied beside it. ``evidence_chunk`` is null
when the stored tag has none.
"""
con = _sqlite_ro(path)
try:
rows = con.execute(
"""SELECT key, extra_json FROM items
WHERE json_extract(extra_json, '$.llm_tags') IS NOT NULL
ORDER BY key"""
).fetchall()
finally:
con.close()
records: list[dict] = []
for key, raw in rows:
state = json.loads(raw).get("llm_tags") or {}
tags = state.get("tags") or {}
if not isinstance(tags, dict):
raise ValueError(f"{key}: llm_tags.tags is not an object")
model = state.get("model")
version = state.get("version")
for theme, info in tags.items():
if not isinstance(info, dict):
raise ValueError(f"{key}: tag {theme} is not an object")
chunk = info.get("evidence_chunk")
records.append(
{
"item_key": key,
"theme": str(theme),
"confidence": info.get("confidence"),
"model": None if model is None else str(model),
"vocab_version": None if version is None else int(version),
"evidence_chunk": None if chunk is None else int(chunk),
}
)
records.sort(key=lambda row: (row["item_key"], row["theme"]))
return pl.DataFrame(records, schema=_ASSIGNMENT_SCHEMA)
# ── code ─────────────────────────────────────────────────────────

View File

@@ -20,6 +20,7 @@ from cli import app
from lead.ledger.store import append, open_ledger
from lead.release import Inputs, build_release, verify_release
from lead.release.bagit import read_bag_info
from lead.release.content import read_assignments
from lead.release.protocol import normalize
from lead.review.store import Store
@@ -277,6 +278,69 @@ def _vocab(root: Path) -> Path:
return path
def _bib(root: Path) -> Path:
"""Comment tag state as the tagger stores it on bib items."""
path = root / "bib.sqlite"
con = sqlite3.connect(path)
try:
con.execute("CREATE TABLE items (key TEXT PRIMARY KEY, extra_json TEXT)")
con.executemany(
"INSERT INTO items (key, extra_json) VALUES (?, ?)",
[
(
"CCC",
json.dumps(
{
"llm_tags": {
"version": 3,
"model": "qwen2.5:14b",
"tags": {
"work-rvu": {
"confidence": 0.75,
"evidence_chunk": 2,
},
"evaluation-management": {
"confidence": 0.5,
"evidence_chunk": 0,
},
},
}
}
),
),
(
"AAA",
json.dumps(
{
"llm_tags": {
"version": 3,
"model": "qwen2.5:14b",
"tags": {},
}
}
),
),
("BBB", json.dumps({"other": 1})),
(
"DDD",
json.dumps(
{
"llm_tags": {
"version": 2,
"model": "qwen2.5:14b",
"tags": {"caregiver-training": {"confidence": 0.625}},
}
}
),
),
],
)
con.commit()
finally:
con.close()
return path
@pytest.fixture(scope="module")
def inputs(tmp_path_factory: pytest.TempPathFactory) -> Inputs:
"""Built once per module: the build only reads them."""
@@ -286,6 +350,7 @@ def inputs(tmp_path_factory: pytest.TempPathFactory) -> Inputs:
labels=_labels(src),
zotero=_zotero(src),
vocab=_vocab(src),
bib=_bib(src),
repo=_repo(src),
registration=REGISTRATION,
)
@@ -339,6 +404,7 @@ def test_build_writes_a_valid_bag(inputs, tmp_path):
"data/ro-crate-metadata.json",
"data/screening/decisions.parquet",
"data/screening/prisma-flow.json",
"data/tags/assignments.parquet",
"data/tags/vocab-v3.yaml",
]
)
@@ -354,7 +420,7 @@ def test_build_writes_a_valid_bag(inputs, tmp_path):
def test_build_does_not_touch_its_inputs(inputs, tmp_path):
files = [inputs.ledger, inputs.labels, inputs.zotero, inputs.vocab]
files = [inputs.ledger, inputs.labels, inputs.zotero, inputs.vocab, inputs.bib]
before = {p: (_sha(p), p.stat().st_mtime_ns) for p in files}
siblings = {p.parent: sorted(x.name for x in p.parent.iterdir()) for p in files}
_build(inputs, tmp_path / "out")
@@ -479,11 +545,96 @@ def test_bibliography_is_csl_json(inputs, tmp_path):
assert any("GGGG8888" in w for w in result.warnings)
def test_read_assignments_is_empty_without_accepted_tags(tmp_path):
path = tmp_path / "bib.sqlite"
con = sqlite3.connect(path)
con.execute("CREATE TABLE items (key TEXT, extra_json TEXT)")
con.execute("INSERT INTO items VALUES ('BBB', ?)", (json.dumps({"other": 1}),))
con.execute(
"INSERT INTO items VALUES ('AAA', ?)",
(json.dumps({"llm_tags": {"version": 3, "model": "qwen2.5:14b", "tags": {}}}),),
)
con.commit()
con.close()
frame = read_assignments(path)
assert frame.columns == [
"item_key",
"theme",
"confidence",
"model",
"vocab_version",
"evidence_chunk",
]
assert frame.height == 0
def test_read_assignments_rejects_a_non_object(tmp_path):
path = tmp_path / "bib.sqlite"
con = sqlite3.connect(path)
con.execute("CREATE TABLE items (key TEXT, extra_json TEXT)")
con.execute(
"INSERT INTO items VALUES ('X', ?)",
(json.dumps({"llm_tags": {"tags": ["work-rvu"]}}),),
)
con.execute(
"INSERT INTO items VALUES ('Y', ?)",
(json.dumps({"llm_tags": {"tags": {"work-rvu": "yes"}}}),),
)
con.commit()
con.close()
with pytest.raises(ValueError, match="X: llm_tags.tags is not an object"):
read_assignments(path)
con = sqlite3.connect(path)
con.execute("DELETE FROM items WHERE key = 'X'")
con.commit()
con.close()
with pytest.raises(ValueError, match="Y: tag work-rvu is not an object"):
read_assignments(path)
def test_tags_and_code(inputs, tmp_path):
inputs = dataclasses.replace(inputs, repo=_repo(tmp_path))
result = _build(inputs, tmp_path / "out")
bag = result.path
assert (bag / "data/tags/vocab-v3.yaml").read_bytes() == inputs.vocab.read_bytes()
assigned = pl.read_parquet(bag / "data/tags/assignments.parquet")
assert assigned.columns == [
"item_key",
"theme",
"confidence",
"model",
"vocab_version",
"evidence_chunk",
]
assert assigned.rows(named=True) == [
{
"item_key": "CCC",
"theme": "evaluation-management",
"confidence": 0.5,
"model": "qwen2.5:14b",
"vocab_version": 3,
"evidence_chunk": 0,
},
{
"item_key": "CCC",
"theme": "work-rvu",
"confidence": 0.75,
"model": "qwen2.5:14b",
"vocab_version": 3,
"evidence_chunk": 2,
},
{
"item_key": "DDD",
"theme": "caregiver-training",
"confidence": 0.625,
"model": "qwen2.5:14b",
"vocab_version": 2,
"evidence_chunk": None,
},
]
crate = json.loads((bag / "data/ro-crate-metadata.json").read_text())
names = [e.get("name") for e in crate["@graph"]]
assert "Comment theme assignments" in names
archive = bag / f"data/code/stack-{result.commit}.tar.gz"
with tarfile.open(archive) as tar:
names = tar.getnames()
@@ -586,6 +737,8 @@ def _cli_args(inputs: Inputs, out: Path, reg: Path) -> list[str]:
str(inputs.zotero),
"--vocab",
str(inputs.vocab),
"--bib",
str(inputs.bib),
"--repo",
str(inputs.repo),
]