feat(lead): snapshot comment tag assignments in a release bag
Some checks failed
CI / lint (push) Successful in 48s
CI / notebooks-smoke (push) Successful in 1m50s
Deploy / changes (push) Successful in 17s
CI / test (push) Successful in 4m17s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Lean Pay / pay (push) Failing after 58s
Deploy / api (push) Successful in 1m32s
Deploy / bcda (push) Successful in 1m36s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 1m10s
Deploy / report (push) Successful in 15s
Infra CI / zotero (push) Successful in 15s
Infra CI / docs (push) Successful in 19s
Infra CI / notebooks (push) Successful in 54s
Infra CI / api (push) Successful in 20s
Infra CI / bcda (push) Successful in 20s
Infra CI / llm (push) Successful in 16s
Infra CI / mc (push) Successful in 19s
Package Supply Chain / pkg-supply-chain (push) Successful in 1m30s
Some checks failed
CI / lint (push) Successful in 48s
CI / notebooks-smoke (push) Successful in 1m50s
Deploy / changes (push) Successful in 17s
CI / test (push) Successful in 4m17s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Lean Pay / pay (push) Failing after 58s
Deploy / api (push) Successful in 1m32s
Deploy / bcda (push) Successful in 1m36s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 1m10s
Deploy / report (push) Successful in 15s
Infra CI / zotero (push) Successful in 15s
Infra CI / docs (push) Successful in 19s
Infra CI / notebooks (push) Successful in 54s
Infra CI / api (push) Successful in 20s
Infra CI / bcda (push) Successful in 20s
Infra CI / llm (push) Successful in 16s
Infra CI / mc (push) Successful in 19s
Package Supply Chain / pkg-supply-chain (push) Successful in 1m30s
Each accepted llm_tags theme is a row with its item, theme, confidence, model, vocab version, and evidence chunk.
This commit is contained in:
@@ -201,6 +201,11 @@ def release_build(
|
||||
vocab: str = typer.Option(
|
||||
"", "--vocab", help="Theme vocabulary (default src/llm/vocab/themes.yaml)."
|
||||
),
|
||||
bib: str = typer.Option(
|
||||
"",
|
||||
"--bib",
|
||||
help="Bib sqlite with comment llm_tags (default [db] bib).",
|
||||
),
|
||||
repo: str = typer.Option("", "--repo", help="Git repo to archive (default stack)."),
|
||||
source_org: str = typer.Option(
|
||||
"",
|
||||
@@ -233,6 +238,7 @@ def release_build(
|
||||
else ROOT / "data" / "lead" / "review" / "labels.sqlite",
|
||||
zotero=Path(zotero) if zotero else conf_path("db.zotero"),
|
||||
vocab=Path(vocab) if vocab else ROOT / "src" / "llm" / "vocab" / "themes.yaml",
|
||||
bib=Path(bib) if bib else conf_path("db.bib"),
|
||||
repo=Path(repo) if repo else ROOT,
|
||||
registration=reg,
|
||||
)
|
||||
|
||||
@@ -9,6 +9,8 @@
|
||||
screening/decisions.parquet screening/prisma-flow.json
|
||||
bibliography/included.csl.json bibliography/screened.csl.json
|
||||
tags/vocab-v<N>.yaml
|
||||
tags/assignments.parquet item_key, theme, confidence, model,
|
||||
vocab_version, evidence_chunk
|
||||
code/stack-<sha>.tar.gz
|
||||
|
||||
Two builds from the same commit and inputs are byte-identical apart from
|
||||
@@ -42,6 +44,7 @@ class Inputs:
|
||||
labels: Path
|
||||
zotero: Path
|
||||
vocab: Path
|
||||
bib: Path
|
||||
repo: Path
|
||||
registration: dict
|
||||
|
||||
@@ -149,6 +152,14 @@ def _context(
|
||||
"name": "Theme vocabulary",
|
||||
"description": _shown(inputs.vocab, repo),
|
||||
},
|
||||
{
|
||||
"@id": "#input-tags",
|
||||
"@type": "Dataset",
|
||||
"name": "Comment theme assignments",
|
||||
"description": f"{_shown(inputs.bib, repo)}, read-only. One row per "
|
||||
"accepted llm_tags theme: item key, theme, confidence, model, "
|
||||
"vocab version and evidence chunk.",
|
||||
},
|
||||
]
|
||||
action = {
|
||||
"@id": "#build",
|
||||
@@ -189,7 +200,13 @@ def build_release(
|
||||
dest = out / bag_name(version)
|
||||
if dest.exists() and not force:
|
||||
raise FileExistsError(f"{dest} exists; pass --force to replace it")
|
||||
for source in (inputs.ledger, inputs.labels, inputs.zotero, inputs.vocab):
|
||||
for source in (
|
||||
inputs.ledger,
|
||||
inputs.labels,
|
||||
inputs.zotero,
|
||||
inputs.vocab,
|
||||
inputs.bib,
|
||||
):
|
||||
if not Path(source).is_file():
|
||||
raise FileNotFoundError(source)
|
||||
|
||||
@@ -240,6 +257,10 @@ def build_release(
|
||||
)
|
||||
|
||||
content.copy_vocab(Path(inputs.vocab), data / "tags")
|
||||
content.write_parquet(
|
||||
content.read_assignments(Path(inputs.bib)),
|
||||
data / "tags" / "assignments.parquet",
|
||||
)
|
||||
code = content.archive_code(Path(inputs.repo), data / "code")
|
||||
if code.dirty:
|
||||
warnings.append(
|
||||
@@ -260,8 +281,8 @@ def build_release(
|
||||
f"Release {version} of the LEAD living review registered at OSF "
|
||||
f"({DOI_URL}): the protocol, the evidence ledger, screening "
|
||||
"decisions and PRISMA flow counts, the bibliography of screened "
|
||||
"and included records, the theme vocabulary, and the code that "
|
||||
"built it."
|
||||
"and included records, the theme vocabulary, the comment tag "
|
||||
"assignments, and the code that built it."
|
||||
)
|
||||
crate = build_crate(
|
||||
data,
|
||||
|
||||
@@ -4,7 +4,9 @@
|
||||
- screening: the human labels (``lead.review``) and PRISMA flow counts
|
||||
(``prisma.flow`` over the Zotero tags)
|
||||
- bibliography: CSL-JSON for the screened and included Zotero records
|
||||
- tags: the theme vocabulary file
|
||||
- tags: the theme vocabulary file, and one row per accepted comment
|
||||
tag (``items.extra_json.llm_tags``) with its model, vocab version and
|
||||
evidence chunk
|
||||
- code: ``git archive`` of HEAD
|
||||
"""
|
||||
|
||||
@@ -12,6 +14,7 @@ from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import gzip
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
@@ -368,6 +371,60 @@ def copy_vocab(path: Path, dest_dir: Path) -> Path:
|
||||
return dest
|
||||
|
||||
|
||||
_ASSIGNMENT_SCHEMA = {
|
||||
"item_key": pl.String,
|
||||
"theme": pl.String,
|
||||
"confidence": pl.Float64,
|
||||
"model": pl.String,
|
||||
"vocab_version": pl.Int64,
|
||||
"evidence_chunk": pl.Int64,
|
||||
}
|
||||
|
||||
|
||||
def read_assignments(path: Path) -> pl.DataFrame:
|
||||
"""One row per accepted comment tag stored on a bib item.
|
||||
|
||||
Rows are ``(item_key, theme)`` sorted. An empty ``tags`` object and an
|
||||
item with no ``llm_tags`` contribute nothing. ``vocab_version`` and
|
||||
``model`` are the values stored with the assignment, which can be older
|
||||
than the vocabulary file copied beside it. ``evidence_chunk`` is null
|
||||
when the stored tag has none.
|
||||
"""
|
||||
con = _sqlite_ro(path)
|
||||
try:
|
||||
rows = con.execute(
|
||||
"""SELECT key, extra_json FROM items
|
||||
WHERE json_extract(extra_json, '$.llm_tags') IS NOT NULL
|
||||
ORDER BY key"""
|
||||
).fetchall()
|
||||
finally:
|
||||
con.close()
|
||||
records: list[dict] = []
|
||||
for key, raw in rows:
|
||||
state = json.loads(raw).get("llm_tags") or {}
|
||||
tags = state.get("tags") or {}
|
||||
if not isinstance(tags, dict):
|
||||
raise ValueError(f"{key}: llm_tags.tags is not an object")
|
||||
model = state.get("model")
|
||||
version = state.get("version")
|
||||
for theme, info in tags.items():
|
||||
if not isinstance(info, dict):
|
||||
raise ValueError(f"{key}: tag {theme} is not an object")
|
||||
chunk = info.get("evidence_chunk")
|
||||
records.append(
|
||||
{
|
||||
"item_key": key,
|
||||
"theme": str(theme),
|
||||
"confidence": info.get("confidence"),
|
||||
"model": None if model is None else str(model),
|
||||
"vocab_version": None if version is None else int(version),
|
||||
"evidence_chunk": None if chunk is None else int(chunk),
|
||||
}
|
||||
)
|
||||
records.sort(key=lambda row: (row["item_key"], row["theme"]))
|
||||
return pl.DataFrame(records, schema=_ASSIGNMENT_SCHEMA)
|
||||
|
||||
|
||||
# ── code ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ from cli import app
|
||||
from lead.ledger.store import append, open_ledger
|
||||
from lead.release import Inputs, build_release, verify_release
|
||||
from lead.release.bagit import read_bag_info
|
||||
from lead.release.content import read_assignments
|
||||
from lead.release.protocol import normalize
|
||||
from lead.review.store import Store
|
||||
|
||||
@@ -277,6 +278,69 @@ def _vocab(root: Path) -> Path:
|
||||
return path
|
||||
|
||||
|
||||
def _bib(root: Path) -> Path:
|
||||
"""Comment tag state as the tagger stores it on bib items."""
|
||||
path = root / "bib.sqlite"
|
||||
con = sqlite3.connect(path)
|
||||
try:
|
||||
con.execute("CREATE TABLE items (key TEXT PRIMARY KEY, extra_json TEXT)")
|
||||
con.executemany(
|
||||
"INSERT INTO items (key, extra_json) VALUES (?, ?)",
|
||||
[
|
||||
(
|
||||
"CCC",
|
||||
json.dumps(
|
||||
{
|
||||
"llm_tags": {
|
||||
"version": 3,
|
||||
"model": "qwen2.5:14b",
|
||||
"tags": {
|
||||
"work-rvu": {
|
||||
"confidence": 0.75,
|
||||
"evidence_chunk": 2,
|
||||
},
|
||||
"evaluation-management": {
|
||||
"confidence": 0.5,
|
||||
"evidence_chunk": 0,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
),
|
||||
),
|
||||
(
|
||||
"AAA",
|
||||
json.dumps(
|
||||
{
|
||||
"llm_tags": {
|
||||
"version": 3,
|
||||
"model": "qwen2.5:14b",
|
||||
"tags": {},
|
||||
}
|
||||
}
|
||||
),
|
||||
),
|
||||
("BBB", json.dumps({"other": 1})),
|
||||
(
|
||||
"DDD",
|
||||
json.dumps(
|
||||
{
|
||||
"llm_tags": {
|
||||
"version": 2,
|
||||
"model": "qwen2.5:14b",
|
||||
"tags": {"caregiver-training": {"confidence": 0.625}},
|
||||
}
|
||||
}
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
con.commit()
|
||||
finally:
|
||||
con.close()
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def inputs(tmp_path_factory: pytest.TempPathFactory) -> Inputs:
|
||||
"""Built once per module: the build only reads them."""
|
||||
@@ -286,6 +350,7 @@ def inputs(tmp_path_factory: pytest.TempPathFactory) -> Inputs:
|
||||
labels=_labels(src),
|
||||
zotero=_zotero(src),
|
||||
vocab=_vocab(src),
|
||||
bib=_bib(src),
|
||||
repo=_repo(src),
|
||||
registration=REGISTRATION,
|
||||
)
|
||||
@@ -339,6 +404,7 @@ def test_build_writes_a_valid_bag(inputs, tmp_path):
|
||||
"data/ro-crate-metadata.json",
|
||||
"data/screening/decisions.parquet",
|
||||
"data/screening/prisma-flow.json",
|
||||
"data/tags/assignments.parquet",
|
||||
"data/tags/vocab-v3.yaml",
|
||||
]
|
||||
)
|
||||
@@ -354,7 +420,7 @@ def test_build_writes_a_valid_bag(inputs, tmp_path):
|
||||
|
||||
|
||||
def test_build_does_not_touch_its_inputs(inputs, tmp_path):
|
||||
files = [inputs.ledger, inputs.labels, inputs.zotero, inputs.vocab]
|
||||
files = [inputs.ledger, inputs.labels, inputs.zotero, inputs.vocab, inputs.bib]
|
||||
before = {p: (_sha(p), p.stat().st_mtime_ns) for p in files}
|
||||
siblings = {p.parent: sorted(x.name for x in p.parent.iterdir()) for p in files}
|
||||
_build(inputs, tmp_path / "out")
|
||||
@@ -479,11 +545,96 @@ def test_bibliography_is_csl_json(inputs, tmp_path):
|
||||
assert any("GGGG8888" in w for w in result.warnings)
|
||||
|
||||
|
||||
def test_read_assignments_is_empty_without_accepted_tags(tmp_path):
|
||||
path = tmp_path / "bib.sqlite"
|
||||
con = sqlite3.connect(path)
|
||||
con.execute("CREATE TABLE items (key TEXT, extra_json TEXT)")
|
||||
con.execute("INSERT INTO items VALUES ('BBB', ?)", (json.dumps({"other": 1}),))
|
||||
con.execute(
|
||||
"INSERT INTO items VALUES ('AAA', ?)",
|
||||
(json.dumps({"llm_tags": {"version": 3, "model": "qwen2.5:14b", "tags": {}}}),),
|
||||
)
|
||||
con.commit()
|
||||
con.close()
|
||||
frame = read_assignments(path)
|
||||
assert frame.columns == [
|
||||
"item_key",
|
||||
"theme",
|
||||
"confidence",
|
||||
"model",
|
||||
"vocab_version",
|
||||
"evidence_chunk",
|
||||
]
|
||||
assert frame.height == 0
|
||||
|
||||
|
||||
def test_read_assignments_rejects_a_non_object(tmp_path):
|
||||
path = tmp_path / "bib.sqlite"
|
||||
con = sqlite3.connect(path)
|
||||
con.execute("CREATE TABLE items (key TEXT, extra_json TEXT)")
|
||||
con.execute(
|
||||
"INSERT INTO items VALUES ('X', ?)",
|
||||
(json.dumps({"llm_tags": {"tags": ["work-rvu"]}}),),
|
||||
)
|
||||
con.execute(
|
||||
"INSERT INTO items VALUES ('Y', ?)",
|
||||
(json.dumps({"llm_tags": {"tags": {"work-rvu": "yes"}}}),),
|
||||
)
|
||||
con.commit()
|
||||
con.close()
|
||||
with pytest.raises(ValueError, match="X: llm_tags.tags is not an object"):
|
||||
read_assignments(path)
|
||||
con = sqlite3.connect(path)
|
||||
con.execute("DELETE FROM items WHERE key = 'X'")
|
||||
con.commit()
|
||||
con.close()
|
||||
with pytest.raises(ValueError, match="Y: tag work-rvu is not an object"):
|
||||
read_assignments(path)
|
||||
|
||||
|
||||
def test_tags_and_code(inputs, tmp_path):
|
||||
inputs = dataclasses.replace(inputs, repo=_repo(tmp_path))
|
||||
result = _build(inputs, tmp_path / "out")
|
||||
bag = result.path
|
||||
assert (bag / "data/tags/vocab-v3.yaml").read_bytes() == inputs.vocab.read_bytes()
|
||||
assigned = pl.read_parquet(bag / "data/tags/assignments.parquet")
|
||||
assert assigned.columns == [
|
||||
"item_key",
|
||||
"theme",
|
||||
"confidence",
|
||||
"model",
|
||||
"vocab_version",
|
||||
"evidence_chunk",
|
||||
]
|
||||
assert assigned.rows(named=True) == [
|
||||
{
|
||||
"item_key": "CCC",
|
||||
"theme": "evaluation-management",
|
||||
"confidence": 0.5,
|
||||
"model": "qwen2.5:14b",
|
||||
"vocab_version": 3,
|
||||
"evidence_chunk": 0,
|
||||
},
|
||||
{
|
||||
"item_key": "CCC",
|
||||
"theme": "work-rvu",
|
||||
"confidence": 0.75,
|
||||
"model": "qwen2.5:14b",
|
||||
"vocab_version": 3,
|
||||
"evidence_chunk": 2,
|
||||
},
|
||||
{
|
||||
"item_key": "DDD",
|
||||
"theme": "caregiver-training",
|
||||
"confidence": 0.625,
|
||||
"model": "qwen2.5:14b",
|
||||
"vocab_version": 2,
|
||||
"evidence_chunk": None,
|
||||
},
|
||||
]
|
||||
crate = json.loads((bag / "data/ro-crate-metadata.json").read_text())
|
||||
names = [e.get("name") for e in crate["@graph"]]
|
||||
assert "Comment theme assignments" in names
|
||||
archive = bag / f"data/code/stack-{result.commit}.tar.gz"
|
||||
with tarfile.open(archive) as tar:
|
||||
names = tar.getnames()
|
||||
@@ -586,6 +737,8 @@ def _cli_args(inputs: Inputs, out: Path, reg: Path) -> list[str]:
|
||||
str(inputs.zotero),
|
||||
"--vocab",
|
||||
str(inputs.vocab),
|
||||
"--bib",
|
||||
str(inputs.bib),
|
||||
"--repo",
|
||||
str(inputs.repo),
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user