perf(llm): judge a comment's new themes together

A retune only asks for the themes vocab v3 added, and those yes/no calls
do not depend on each other. They ran one after another, so four workers
left the rig's four slots waiting on the tunnel. Up to four now run at once.
This commit is contained in:
kert
2026-11-16 13:29:13 -05:00
parent 6be08a3834
commit 900d03ccfa
2 changed files with 73 additions and 4 deletions

View File

@@ -27,6 +27,7 @@ import hashlib
import json
import logging
import math
from concurrent.futures import ThreadPoolExecutor
from dataclasses import asdict, dataclass
from datetime import datetime, timezone
from typing import Any, Callable, Iterable, Mapping, Sequence
@@ -38,6 +39,10 @@ log = logging.getLogger(__name__)
Vector = Sequence[float]
Embed = Callable[[Sequence[str]], list[list[float]]]
Judge = Callable[[Theme, str], bool | None]
# New themes on one comment are independent. The rig serves four judges
# at once; overlapping a comment's calls hides the tunnel wait so four
# workers fill those slots. Wider than that just queues.
_JUDGE_WIDTH = 4
ChunkLoader = Callable[[str], list[tuple[int, str, list[float]]]]
STATE_KEY = "llm_tags"
@@ -160,20 +165,44 @@ def tag_comment(
shortlist pays only for its new candidates."""
cands = shortlist(chunk_vecs, card_vecs, top=top)
judged: dict[str, bool | None] = {}
tags: dict[str, TagInfo] = {}
pending: list[Candidate] = []
for c in cands:
if c.slug not in vocab:
continue
if prior is not None and c.slug in prior:
verdict = prior[c.slug]
judged[c.slug] = prior[c.slug]
else:
verdict = judge(vocab.get(c.slug), chunks[c.chunk_idx])
judged[c.slug] = verdict
pending.append(c)
for slug, verdict in _judge_pending(pending, chunks, vocab, judge):
judged[slug] = verdict
tags: dict[str, TagInfo] = {}
for c in cands:
verdict = judged.get(c.slug)
if verdict and c.score >= min_confidence and len(tags) < max_tags:
tags[c.slug] = TagInfo(confidence=c.score, evidence_chunk=c.chunk_idx)
return TagResult(key, tags, tuple(cands), judged, content_hash(chunks))
def _judge_pending(
pending: Sequence[Candidate],
chunks: Sequence[str],
vocab: Vocab,
judge: Judge,
) -> list[tuple[str, bool | None]]:
"""Judge candidates that have no stored verdict. One call stays on
this thread; several overlap. Order matches *pending*."""
if not pending:
return []
def call(c: Candidate) -> tuple[str, bool | None]:
return c.slug, judge(vocab.get(c.slug), chunks[c.chunk_idx])
if len(pending) == 1:
return [call(pending[0])]
with ThreadPoolExecutor(max_workers=min(_JUDGE_WIDTH, len(pending))) as ex:
return list(ex.map(call, pending))
def state_matches(
extra: Mapping[str, Any], *, version: int, model: str, digest: str
) -> bool:

View File

@@ -109,6 +109,46 @@ class TestPure:
) in seen # evidence is the scoring chunk
assert r.content_hash == content_hash(chunks)
def test_new_themes_are_judged_together(self):
import threading
import time
chunks = ["shared"]
vecs = [[1.0, 1.0, 1.0]]
current = threading.Semaphore(0)
release = threading.Semaphore(0)
def judge(theme, evidence):
current.release()
assert release.acquire(timeout=2), theme.slug
return True
started = time.perf_counter()
box: dict = {}
def run():
box["result"] = tag_comment(
"K",
chunks,
vecs,
vocab=VOCAB,
card_vecs=CARDS,
judge=judge,
top=3,
max_tags=3,
min_confidence=0.0,
)
worker = threading.Thread(target=run)
worker.start()
for _ in range(3):
assert current.acquire(timeout=2)
release.release(3)
worker.join(timeout=2)
assert not worker.is_alive()
assert time.perf_counter() - started < 1.5
assert box["result"].slugs == [c.slug for c in shortlist(vecs, CARDS, top=3)]
def test_min_confidence_filters_weak_matches(self):
chunks = ["x"]
vecs = [[0.6, 0.8, 0.0]] # cosine 0.6 with telehealth, 0.8 with care-management