perf(llm): judge a comment's new themes together
A retune only asks for the themes vocab v3 added, and those yes/no calls do not depend on each other. They ran one after another, so four workers left the rig's four slots waiting on the tunnel. Up to four now run at once.
This commit is contained in:
@@ -27,6 +27,7 @@ import hashlib
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import asdict, dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Callable, Iterable, Mapping, Sequence
|
||||
@@ -38,6 +39,10 @@ log = logging.getLogger(__name__)
|
||||
Vector = Sequence[float]
|
||||
Embed = Callable[[Sequence[str]], list[list[float]]]
|
||||
Judge = Callable[[Theme, str], bool | None]
|
||||
# New themes on one comment are independent. The rig serves four judges
|
||||
# at once; overlapping a comment's calls hides the tunnel wait so four
|
||||
# workers fill those slots. Wider than that just queues.
|
||||
_JUDGE_WIDTH = 4
|
||||
ChunkLoader = Callable[[str], list[tuple[int, str, list[float]]]]
|
||||
|
||||
STATE_KEY = "llm_tags"
|
||||
@@ -160,20 +165,44 @@ def tag_comment(
|
||||
shortlist pays only for its new candidates."""
|
||||
cands = shortlist(chunk_vecs, card_vecs, top=top)
|
||||
judged: dict[str, bool | None] = {}
|
||||
tags: dict[str, TagInfo] = {}
|
||||
pending: list[Candidate] = []
|
||||
for c in cands:
|
||||
if c.slug not in vocab:
|
||||
continue
|
||||
if prior is not None and c.slug in prior:
|
||||
verdict = prior[c.slug]
|
||||
judged[c.slug] = prior[c.slug]
|
||||
else:
|
||||
verdict = judge(vocab.get(c.slug), chunks[c.chunk_idx])
|
||||
judged[c.slug] = verdict
|
||||
pending.append(c)
|
||||
for slug, verdict in _judge_pending(pending, chunks, vocab, judge):
|
||||
judged[slug] = verdict
|
||||
tags: dict[str, TagInfo] = {}
|
||||
for c in cands:
|
||||
verdict = judged.get(c.slug)
|
||||
if verdict and c.score >= min_confidence and len(tags) < max_tags:
|
||||
tags[c.slug] = TagInfo(confidence=c.score, evidence_chunk=c.chunk_idx)
|
||||
return TagResult(key, tags, tuple(cands), judged, content_hash(chunks))
|
||||
|
||||
|
||||
def _judge_pending(
|
||||
pending: Sequence[Candidate],
|
||||
chunks: Sequence[str],
|
||||
vocab: Vocab,
|
||||
judge: Judge,
|
||||
) -> list[tuple[str, bool | None]]:
|
||||
"""Judge candidates that have no stored verdict. One call stays on
|
||||
this thread; several overlap. Order matches *pending*."""
|
||||
if not pending:
|
||||
return []
|
||||
|
||||
def call(c: Candidate) -> tuple[str, bool | None]:
|
||||
return c.slug, judge(vocab.get(c.slug), chunks[c.chunk_idx])
|
||||
|
||||
if len(pending) == 1:
|
||||
return [call(pending[0])]
|
||||
with ThreadPoolExecutor(max_workers=min(_JUDGE_WIDTH, len(pending))) as ex:
|
||||
return list(ex.map(call, pending))
|
||||
|
||||
|
||||
def state_matches(
|
||||
extra: Mapping[str, Any], *, version: int, model: str, digest: str
|
||||
) -> bool:
|
||||
|
||||
@@ -109,6 +109,46 @@ class TestPure:
|
||||
) in seen # evidence is the scoring chunk
|
||||
assert r.content_hash == content_hash(chunks)
|
||||
|
||||
def test_new_themes_are_judged_together(self):
|
||||
import threading
|
||||
import time
|
||||
|
||||
chunks = ["shared"]
|
||||
vecs = [[1.0, 1.0, 1.0]]
|
||||
current = threading.Semaphore(0)
|
||||
release = threading.Semaphore(0)
|
||||
|
||||
def judge(theme, evidence):
|
||||
current.release()
|
||||
assert release.acquire(timeout=2), theme.slug
|
||||
return True
|
||||
|
||||
started = time.perf_counter()
|
||||
box: dict = {}
|
||||
|
||||
def run():
|
||||
box["result"] = tag_comment(
|
||||
"K",
|
||||
chunks,
|
||||
vecs,
|
||||
vocab=VOCAB,
|
||||
card_vecs=CARDS,
|
||||
judge=judge,
|
||||
top=3,
|
||||
max_tags=3,
|
||||
min_confidence=0.0,
|
||||
)
|
||||
|
||||
worker = threading.Thread(target=run)
|
||||
worker.start()
|
||||
for _ in range(3):
|
||||
assert current.acquire(timeout=2)
|
||||
release.release(3)
|
||||
worker.join(timeout=2)
|
||||
assert not worker.is_alive()
|
||||
assert time.perf_counter() - started < 1.5
|
||||
assert box["result"].slugs == [c.slug for c in shortlist(vecs, CARDS, top=3)]
|
||||
|
||||
def test_min_confidence_filters_weak_matches(self):
|
||||
chunks = ["x"]
|
||||
vecs = [[0.6, 0.8, 0.0]] # cosine 0.6 with telehealth, 0.8 with care-management
|
||||
|
||||
Reference in New Issue
Block a user