fix(llm): judge theme tags on instruct_model only
The judge stepped up to instruct_model_large on a host that serves it, and the run recorded that model in every item's tag state. Pulling qwen2.5:32b on the rig would have re-tagged all 315k comments with a model the golden gate never scored. Chat keeps the large tier.
This commit is contained in:
@@ -248,7 +248,7 @@ def _tagging_runtime(cfg: Any) -> dict[str, Any]:
|
||||
in each item's state."""
|
||||
from conf.connect import bib
|
||||
from llm.index import _engine
|
||||
from llm.pool import HostPool, PoolEmbeddings, pick_model
|
||||
from llm.pool import HostPool, PoolEmbeddings
|
||||
from llm.tagger import card_vectors, make_judge, pg_chunks
|
||||
from llm.vocab import load as load_vocab
|
||||
|
||||
@@ -257,8 +257,7 @@ def _tagging_runtime(cfg: Any) -> dict[str, Any]:
|
||||
pool.check(cfg.embed_model)
|
||||
cards = card_vectors(vocab, PoolEmbeddings(pool, cfg.embed_model).embed_documents)
|
||||
judge = make_judge(cfg, pool)
|
||||
with pool.acquire_generation() as host:
|
||||
model = pick_model(cfg, pool, host)
|
||||
model = cfg.instruct_model # what make_judge sends; recorded in tag state
|
||||
return {
|
||||
"store": bib(),
|
||||
"vocab": vocab,
|
||||
|
||||
@@ -322,10 +322,11 @@ def card_vectors(vocab: Vocab, embed: Embed) -> dict[str, list[float]]:
|
||||
def make_judge(
|
||||
cfg: Any, pool: Any, *, post: Callable[..., dict] | None = None
|
||||
) -> Judge:
|
||||
"""A yes/no judge on the largest live host, ``temperature 0`` — the
|
||||
same route ``llm.classify.closed_vocab_classifier`` takes."""
|
||||
"""A yes/no judge on the largest live host, ``temperature 0``, always
|
||||
on ``cfg.instruct_model`` — the model the golden gate scored. Unlike
|
||||
chat it never steps up to ``instruct_model_large``: tag state records
|
||||
the model, so a bigger host appearing must not re-tag the corpus."""
|
||||
from llm.classify import _default_post
|
||||
from llm.pool import pick_model
|
||||
|
||||
send = post or _default_post
|
||||
pool.check(cfg.instruct_model)
|
||||
@@ -338,7 +339,7 @@ def make_judge(
|
||||
# live host, the same discipline embed_texts follows (#796).
|
||||
while True:
|
||||
with pool.acquire_generation() as host:
|
||||
model = pick_model(cfg, pool, host)
|
||||
model = cfg.instruct_model
|
||||
try:
|
||||
data = send(
|
||||
f"{host}/api/chat",
|
||||
|
||||
@@ -200,6 +200,30 @@ class TestJudge:
|
||||
)
|
||||
assert sent["json"]["model"] == "m"
|
||||
|
||||
def test_judge_stays_on_the_gated_model_when_the_big_host_has_the_large_one(self):
|
||||
# The golden gate scored instruct_model; tag state records the
|
||||
# model, so judging with the large tier would re-tag the corpus.
|
||||
from llm.pool import HostPool
|
||||
|
||||
pool = HostPool(["http://rig"], vram_gb={"http://rig": 24})
|
||||
pool.check = lambda model: ["http://rig"] # type: ignore[method-assign]
|
||||
pool._models = {"http://rig": {"m", "big"}}
|
||||
sent = {}
|
||||
|
||||
def post(url, json=None):
|
||||
sent["model"] = json["model"]
|
||||
return {"message": {"content": "yes"}}
|
||||
|
||||
class Cfg:
|
||||
instruct_model = "m"
|
||||
instruct_model_large = "big"
|
||||
large_min_vram_gb = 20.0
|
||||
chat_num_ctx = 4096
|
||||
|
||||
judge = make_judge(Cfg(), pool, post=post)
|
||||
assert judge(VOCAB.get("drugs"), "x") is True
|
||||
assert sent["model"] == "m"
|
||||
|
||||
def test_judge_drops_a_dying_host_and_retries_on_the_survivor(self):
|
||||
import httpx
|
||||
|
||||
|
||||
Reference in New Issue
Block a user