From 33cd44519da42e226577d1b3a2cce4a624023f87 Mon Sep 17 00:00:00 2001 From: kert Date: Wed, 11 Nov 2026 15:29:43 -0500 Subject: [PATCH] feat(llm): retune tags from stored verdicts when only parameters change Tag state now records the parameters it ran with (top, max_tags, min_confidence). On unchanged content with other parameters, run() re-derives the tags reusing every stored judge verdict: a threshold change costs no model calls, a wider shortlist pays only for its new candidates. Same content and same parameters still skip. The 315k stored states carry their full judged shortlists, so a new threshold applies corpus-wide without the GPU. --- src/cli/llm.py | 2 +- src/llm/tagger.py | 49 ++++++++++++++++++++++++----- tests/cli/test_llm_tag_cli.py | 5 ++- tests/llm/test_tagger.py | 59 +++++++++++++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 9 deletions(-) diff --git a/src/cli/llm.py b/src/cli/llm.py index d8c53de..846724a 100644 --- a/src/cli/llm.py +++ b/src/cli/llm.py @@ -341,7 +341,7 @@ def tag( ) typer.echo( f"tag {docket} (vocab v{rt['vocab'].version}, {rt['model']}{', dry run' if dry_run else ''}): " - f"seen={stats['seen']} tagged={stats['tagged']} unchanged={stats['skipped_state']} " + f"seen={stats['seen']} tagged={stats['tagged']} retuned={stats.get('retuned', 0)} unchanged={stats['skipped_state']} " f"no_chunks={stats['no_chunks']} tags_written={stats['tags_written']}" ) diff --git a/src/llm/tagger.py b/src/llm/tagger.py index fabc1c9..18a6af0 100644 --- a/src/llm/tagger.py +++ b/src/llm/tagger.py @@ -151,14 +151,23 @@ def tag_comment( top: int = 8, max_tags: int = 3, min_confidence: float = 0.60, + prior: Mapping[str, bool | None] | None = None, ) -> TagResult: + """Shortlist, judge, keep up to *max_tags* judged-yes themes at or + above *min_confidence*. *prior* holds verdicts already given on this + exact content (same model and vocab); those candidates are not judged + again, so a threshold change costs no model calls and a wider + shortlist pays only for its new candidates.""" cands = shortlist(chunk_vecs, card_vecs, top=top) judged: dict[str, bool | None] = {} tags: dict[str, TagInfo] = {} for c in cands: if c.slug not in vocab: continue - verdict = judge(vocab.get(c.slug), chunks[c.chunk_idx]) + if prior is not None and c.slug in prior: + verdict = prior[c.slug] + else: + verdict = judge(vocab.get(c.slug), chunks[c.chunk_idx]) judged[c.slug] = verdict if verdict and c.score >= min_confidence and len(tags) < max_tags: tags[c.slug] = TagInfo(confidence=c.score, evidence_chunk=c.chunk_idx) @@ -176,10 +185,17 @@ def state_matches( ) -def state_payload(result: TagResult, *, version: int, model: str) -> dict: +def state_payload( + result: TagResult, + *, + version: int, + model: str, + params: Mapping[str, Any] | None = None, +) -> dict: return { "version": version, "model": model, + "params": dict(params or {}), "content_hash": result.content_hash, "tagged_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "tags": {s: asdict(i) for s, i in result.tags.items()}, @@ -260,9 +276,11 @@ def run( """Tag every comment in *docket* (newest first); resumable via the item's stored state; ``dry_run`` computes and reports but writes nothing.""" + params = {"top": top, "max_tags": max_tags, "min_confidence": min_confidence} stats = { "seen": 0, "tagged": 0, + "retuned": 0, "skipped_state": 0, "no_chunks": 0, "tags_written": 0, @@ -279,13 +297,25 @@ def run( rows = sorted(rows, key=lambda r: r[0]) texts = [t for _, t, _ in rows] digest = content_hash(texts) - if not force and state_matches( + same_content = not force and state_matches( extra, version=vocab.version, model=model, digest=digest - ): + ) + stored = extra.get(STATE_KEY) or {} + if same_content and stored.get("params") == params: stats["skipped_state"] += 1 if progress: progress(key, None, "unchanged") continue + # Same content, other parameters: reuse every verdict already given. + prior = ( + { + x["slug"]: x.get("verdict") + for x in stored.get("shortlist") or [] + if x.get("verdict") is not None + } + if same_content + else None + ) result = tag_comment( key, texts, @@ -296,17 +326,22 @@ def run( top=top, max_tags=max_tags, min_confidence=min_confidence, + prior=prior, ) - stats["tagged"] += 1 + stats["retuned" if prior is not None else "tagged"] += 1 if not dry_run: store.merge_extra( key, - {STATE_KEY: state_payload(result, version=vocab.version, model=model)}, + { + STATE_KEY: state_payload( + result, version=vocab.version, model=model, params=params + ) + }, ) added, _removed = apply_tags(store, key, result.slugs, vocab=vocab) stats["tags_written"] += added if progress: - progress(key, result, "tagged") + progress(key, result, "retuned" if prior is not None else "tagged") return stats diff --git a/tests/cli/test_llm_tag_cli.py b/tests/cli/test_llm_tag_cli.py index 3e76c86..60f7f14 100644 --- a/tests/cli/test_llm_tag_cli.py +++ b/tests/cli/test_llm_tag_cli.py @@ -67,7 +67,10 @@ class TestTag: ) assert res.exit_code == 0, res.output assert f"{key} telehealth" in res.output - assert "seen=1 tagged=1 unchanged=0 no_chunks=0 tags_written=1" in res.output + assert ( + "seen=1 tagged=1 retuned=0 unchanged=0 no_chunks=0 tags_written=1" + in res.output + ) assert ( "llm:telehealth" in store.get(key).tags and "llm:drugs" not in store.get(key).tags diff --git a/tests/llm/test_tagger.py b/tests/llm/test_tagger.py index 466f5c6..2cd932b 100644 --- a/tests/llm/test_tagger.py +++ b/tests/llm/test_tagger.py @@ -370,6 +370,7 @@ class TestRun: assert stats == { "seen": 2, "tagged": 1, + "retuned": 0, "skipped_state": 0, "no_chunks": 1, "tags_written": 2, @@ -395,6 +396,7 @@ class TestRun: judge=judge, model="m", top=2, + max_tags=2, ) assert ( stats2["skipped_state"] == 1 and stats2["tagged"] == 0 and judge_calls == [] @@ -445,3 +447,60 @@ class TestRun: k = keys["CMS-2026-2377-1"] assert not any(t.startswith("llm:") for t in s.get(k).tags) s.close() + + +class TestRetune: + """A parameter change on unchanged content reuses stored verdicts.""" + + def _run(self, s, keys, judge, **kw): + return run( + s, + vocab=VOCAB, + docket="CMS-2026-2377", + load_chunks=TestRun()._chunks(keys), + card_vecs=CARDS, + judge=judge, + model="m", + **kw, + ) + + def test_threshold_change_rederives_tags_without_judging(self, tmp_path): + s, keys = _store(tmp_path) + calls = [] + + def judge(theme, evidence): + calls.append(theme.slug) + return True + + self._run(s, keys, judge, top=2, max_tags=2, min_confidence=1.01) + k = keys["CMS-2026-2377-1"] + assert not [t for t in s.get(k).tags if t.startswith("llm:")] + calls.clear() + stats = self._run(s, keys, judge, top=2, max_tags=2, min_confidence=0.0) + assert calls == [] and stats["retuned"] == 1 and stats["tagged"] == 0 + assert {"llm:telehealth", "llm:care-management"} <= set(s.get(k).tags) + extra = json.loads( + s._con() + .execute("SELECT extra_json FROM items WHERE key=?", (k,)) + .fetchone()[0] # noqa: SLF001 + ) + assert extra["llm_tags"]["params"] == { + "top": 2, + "max_tags": 2, + "min_confidence": 0.0, + } + + def test_wider_shortlist_judges_only_new_candidates(self, tmp_path): + s, keys = _store(tmp_path) + calls = [] + + def judge(theme, evidence): + calls.append(theme.slug) + return True + + self._run(s, keys, judge, top=1, max_tags=3, min_confidence=0.0) + first = list(calls) + calls.clear() + stats = self._run(s, keys, judge, top=3, max_tags=3, min_confidence=0.0) + assert stats["retuned"] == 1 + assert not set(calls) & set(first) and len(calls) == 2