fix(llm): judge theme tags on instruct_model only

The judge stepped up to instruct_model_large on a host that serves it,
and the run recorded that model in every item's tag state. Pulling
qwen2.5:32b on the rig would have re-tagged all 315k comments with a
model the golden gate never scored. Chat keeps the large tier.
This commit is contained in:
kert
2026-11-11 13:12:52 -05:00
parent b614c1706f
commit 735f0dca89
3 changed files with 31 additions and 7 deletions

View File

@@ -248,7 +248,7 @@ def _tagging_runtime(cfg: Any) -> dict[str, Any]:
in each item's state."""
from conf.connect import bib
from llm.index import _engine
from llm.pool import HostPool, PoolEmbeddings, pick_model
from llm.pool import HostPool, PoolEmbeddings
from llm.tagger import card_vectors, make_judge, pg_chunks
from llm.vocab import load as load_vocab
@@ -257,8 +257,7 @@ def _tagging_runtime(cfg: Any) -> dict[str, Any]:
pool.check(cfg.embed_model)
cards = card_vectors(vocab, PoolEmbeddings(pool, cfg.embed_model).embed_documents)
judge = make_judge(cfg, pool)
with pool.acquire_generation() as host:
model = pick_model(cfg, pool, host)
model = cfg.instruct_model # what make_judge sends; recorded in tag state
return {
"store": bib(),
"vocab": vocab,

View File

@@ -322,10 +322,11 @@ def card_vectors(vocab: Vocab, embed: Embed) -> dict[str, list[float]]:
def make_judge(
cfg: Any, pool: Any, *, post: Callable[..., dict] | None = None
) -> Judge:
"""A yes/no judge on the largest live host, ``temperature 0`` — the
same route ``llm.classify.closed_vocab_classifier`` takes."""
"""A yes/no judge on the largest live host, ``temperature 0``, always
on ``cfg.instruct_model`` — the model the golden gate scored. Unlike
chat it never steps up to ``instruct_model_large``: tag state records
the model, so a bigger host appearing must not re-tag the corpus."""
from llm.classify import _default_post
from llm.pool import pick_model
send = post or _default_post
pool.check(cfg.instruct_model)
@@ -338,7 +339,7 @@ def make_judge(
# live host, the same discipline embed_texts follows (#796).
while True:
with pool.acquire_generation() as host:
model = pick_model(cfg, pool, host)
model = cfg.instruct_model
try:
data = send(
f"{host}/api/chat",

View File

@@ -200,6 +200,30 @@ class TestJudge:
)
assert sent["json"]["model"] == "m"
def test_judge_stays_on_the_gated_model_when_the_big_host_has_the_large_one(self):
# The golden gate scored instruct_model; tag state records the
# model, so judging with the large tier would re-tag the corpus.
from llm.pool import HostPool
pool = HostPool(["http://rig"], vram_gb={"http://rig": 24})
pool.check = lambda model: ["http://rig"] # type: ignore[method-assign]
pool._models = {"http://rig": {"m", "big"}}
sent = {}
def post(url, json=None):
sent["model"] = json["model"]
return {"message": {"content": "yes"}}
class Cfg:
instruct_model = "m"
instruct_model_large = "big"
large_min_vram_gb = 20.0
chat_num_ctx = 4096
judge = make_judge(Cfg(), pool, post=post)
assert judge(VOCAB.get("drugs"), "x") is True
assert sent["model"] == "m"
def test_judge_drops_a_dying_host_and_retries_on_the_survivor(self):
import httpx