feat(llm): eval-tags --dump writes every golden comment's shortlist scores and verdicts for offline threshold analysis (refs #577)

src/cli/llm.py

@@ -361,6 +361,11 @@ def eval_tags(
     top: int = typer.Option(8, "--top"),
     max_tags: int = typer.Option(4, "--max-tags"),
     min_confidence: float = typer.Option(0.0, "--min-confidence"),
+    dump: str = typer.Option(
+        "",
+        "--dump",
+        help="With --live: write every comment's shortlist, verdicts and confidences to this JSON file.",
+    ),
 ) -> None:
     """Score the theme tagger against the golden set (P35 gate, #577):
     per-theme precision/recall, micro-F1 and the abstain rate, then the
@@ -392,6 +397,7 @@ def eval_tags(
         from llm.tagger import tag_comment
 
         rt = _tagging_runtime(llm_config.load())
+        dumped: dict[str, Any] = {}
 
         def tag_one(key: str):
             rows = sorted(rt["load_chunks"](key), key=lambda r: r[0])
@@ -409,9 +415,22 @@ def eval_tags(
                 min_confidence=min_confidence,
             )
             typer.echo(f"{key}  {', '.join(r.slugs) or '(none)'}")
+            dumped[key] = {
+                "tags": {slug: info.confidence for slug, info in r.tags.items()},
+                "shortlist": [
+                    {"slug": c.slug, "score": c.score, "verdict": r.judged.get(c.slug)}
+                    for c in r.shortlisted
+                ],
+            }
             return r.slugs
 
         preds = live_predictions(rt["store"], ids, tag=tag_one)
+        if dump:
+            import json as _json
+            from pathlib import Path as _Path
+
+            _Path(dump).write_text(_json.dumps(dumped, indent=1))
+            typer.echo(f"dumped {len(dumped)} comments → {dump}")
     else:
         preds = stored_predictions(bib(), ids)
     from llm.tageval import Golden

tests/cli/test_llm_tag_cli.py

@@ -137,9 +137,20 @@ class TestEvalTags:
     def test_live_scoring_uses_the_runtime(self, rt, tmp_path):
         store, key, judged = rt
         g = self._golden(tmp_path, ["CMS-2026-2377-1"])
-        res = runner.invoke(app, ["llm", "eval-tags", "--golden", str(g), "--live"])
+        dump = tmp_path / "dump.json"
+        res = runner.invoke(
+            app, ["llm", "eval-tags", "--golden", str(g), "--live", "--dump", str(dump)]
+        )
         assert res.exit_code == 0, res.output
         assert f"{key}  telehealth" in res.output and "gate: PASS" in res.output
+        import json as _json
+
+        d = _json.loads(dump.read_text())
+        assert d[key]["tags"] == {"telehealth": 1.0}
+        assert {x["slug"]: x["verdict"] for x in d[key]["shortlist"]} == {
+            "telehealth": True,
+            "drugs": False,
+        }
         assert judged and not any(
             t.startswith("llm:") for t in store.get(key).tags
         )  # live never writes