7baa789bc0
I5: `closed_vocab_classifier` called `pool.check(cfg.instruct_model)` on every `classify()` call — a host-liveness probe, not per-request work. A code with hundreds of element lines re-probed the pool hundreds of times. Move the check to the closure's construction. M1: `fr_events` matched event patterns (crosswalk, replaces, …) against the raw FR paragraph text, so a citation like "91 FR 99490" inside a "crosswalk ... 91 FR 99490" sentence read as a real crosswalk event for 99490 — the citation's page number is not a mention of the code. Strip FR citations with the now-public `pfs.families.FR_CITE_RE` before both the presence check and every event pattern. M2: `extract_text`'s `classify` parameter was unused; no call site passed it. Drop it. Claude-Session: https://claude.ai/code/session_01Aum3pEMAM3yQVdFSdVe6Gc
107 lines
3.0 KiB
Python
107 lines
3.0 KiB
Python
"""llm.classify — one-of-N choice from the local model, never free text."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from llm.classify import build_prompt, closed_vocab_classifier, parse_choice
|
|
|
|
|
|
class TestParseChoice:
|
|
def test_exact(self):
|
|
assert parse_choice("consent", ["consent", "24-7-access"]) == "consent"
|
|
|
|
def test_quoted_and_cased(self):
|
|
assert (
|
|
parse_choice(' "24-7-Access".\n', ["consent", "24-7-access"])
|
|
== "24-7-access"
|
|
)
|
|
|
|
def test_none_and_garbage(self):
|
|
assert parse_choice("none", ["consent"]) is None
|
|
assert (
|
|
parse_choice(
|
|
"I think it is about consent and access", ["consent", "24-7-access"]
|
|
)
|
|
is None
|
|
)
|
|
|
|
|
|
class TestPrompt:
|
|
def test_prompt_lists_choices_and_none(self):
|
|
msgs = build_prompt(
|
|
"Provide 24/7 access for urgent needs", ["consent", "24-7-access"]
|
|
)
|
|
assert msgs[0]["role"] == "system"
|
|
user = msgs[1]["content"]
|
|
assert "consent" in user and "24-7-access" in user and "none" in user
|
|
assert "Provide 24/7 access" in user
|
|
|
|
|
|
class _Pool:
|
|
def __init__(self):
|
|
self.checked = []
|
|
|
|
def check(self, model):
|
|
self.checked.append(model)
|
|
return ["http://h"]
|
|
|
|
def vram(self, host):
|
|
return 0.0
|
|
|
|
def serves(self, host, model):
|
|
return True
|
|
|
|
class _Ctx:
|
|
def __enter__(self):
|
|
return "http://h"
|
|
|
|
def __exit__(self, *a):
|
|
return False
|
|
|
|
def acquire_generation(self):
|
|
return self._Ctx()
|
|
|
|
|
|
class _Cfg:
|
|
instruct_model = "qwen2.5:14b"
|
|
instruct_model_large = ""
|
|
large_min_vram_gb = 20.0
|
|
chat_num_ctx = 8192
|
|
host_vram = {}
|
|
|
|
|
|
class TestClassifier:
|
|
def test_returns_choice_from_model_reply(self):
|
|
calls = []
|
|
|
|
def post(url, json):
|
|
calls.append((url, json))
|
|
return {"message": {"content": "24-7-access"}}
|
|
|
|
classify = closed_vocab_classifier(_Cfg(), _Pool(), post=post)
|
|
assert (
|
|
classify("Provide 24/7 access for urgent needs", ["consent", "24-7-access"])
|
|
== "24-7-access"
|
|
)
|
|
url, body = calls[0]
|
|
assert url == "http://h/api/chat" and body["stream"] is False
|
|
assert body["options"]["temperature"] == 0
|
|
|
|
def test_unparseable_reply_is_none(self):
|
|
classify = closed_vocab_classifier(
|
|
_Cfg(),
|
|
_Pool(),
|
|
post=lambda url, json: {"message": {"content": "maybe consent?"}},
|
|
)
|
|
assert classify("x", ["consent"]) is None
|
|
|
|
def test_pool_check_runs_once_not_per_call(self):
|
|
# I5: pool.check is a host-liveness probe, done once when the
|
|
# classifier closure is built — not per classify() call.
|
|
pool = _Pool()
|
|
classify = closed_vocab_classifier(
|
|
_Cfg(), pool, post=lambda url, json: {"message": {"content": "consent"}}
|
|
)
|
|
classify("a", ["consent"])
|
|
classify("b", ["consent"])
|
|
assert pool.checked == ["qwen2.5:14b"]
|