diff --git a/src/cli/llm.py b/src/cli/llm.py index 459663d..fd5ebfa 100644 --- a/src/cli/llm.py +++ b/src/cli/llm.py @@ -361,6 +361,11 @@ def eval_tags( top: int = typer.Option(8, "--top"), max_tags: int = typer.Option(4, "--max-tags"), min_confidence: float = typer.Option(0.0, "--min-confidence"), + dump: str = typer.Option( + "", + "--dump", + help="With --live: write every comment's shortlist, verdicts and confidences to this JSON file.", + ), ) -> None: """Score the theme tagger against the golden set (P35 gate, #577): per-theme precision/recall, micro-F1 and the abstain rate, then the @@ -392,6 +397,7 @@ def eval_tags( from llm.tagger import tag_comment rt = _tagging_runtime(llm_config.load()) + dumped: dict[str, Any] = {} def tag_one(key: str): rows = sorted(rt["load_chunks"](key), key=lambda r: r[0]) @@ -409,9 +415,22 @@ def eval_tags( min_confidence=min_confidence, ) typer.echo(f"{key} {', '.join(r.slugs) or '(none)'}") + dumped[key] = { + "tags": {slug: info.confidence for slug, info in r.tags.items()}, + "shortlist": [ + {"slug": c.slug, "score": c.score, "verdict": r.judged.get(c.slug)} + for c in r.shortlisted + ], + } return r.slugs preds = live_predictions(rt["store"], ids, tag=tag_one) + if dump: + import json as _json + from pathlib import Path as _Path + + _Path(dump).write_text(_json.dumps(dumped, indent=1)) + typer.echo(f"dumped {len(dumped)} comments → {dump}") else: preds = stored_predictions(bib(), ids) from llm.tageval import Golden diff --git a/tests/cli/test_llm_tag_cli.py b/tests/cli/test_llm_tag_cli.py index 202b357..014a636 100644 --- a/tests/cli/test_llm_tag_cli.py +++ b/tests/cli/test_llm_tag_cli.py @@ -137,9 +137,20 @@ class TestEvalTags: def test_live_scoring_uses_the_runtime(self, rt, tmp_path): store, key, judged = rt g = self._golden(tmp_path, ["CMS-2026-2377-1"]) - res = runner.invoke(app, ["llm", "eval-tags", "--golden", str(g), "--live"]) + dump = tmp_path / "dump.json" + res = runner.invoke( + app, ["llm", "eval-tags", "--golden", str(g), "--live", "--dump", str(dump)] + ) assert res.exit_code == 0, res.output assert f"{key} telehealth" in res.output and "gate: PASS" in res.output + import json as _json + + d = _json.loads(dump.read_text()) + assert d[key]["tags"] == {"telehealth": 1.0} + assert {x["slug"]: x["verdict"] for x in d[key]["shortlist"]} == { + "telehealth": True, + "drugs": False, + } assert judged and not any( t.startswith("llm:") for t in store.get(key).tags ) # live never writes