From 900d03ccfac056d38457605429c9bf45f5f4efb1 Mon Sep 17 00:00:00 2001 From: kert Date: Mon, 16 Nov 2026 13:29:13 -0500 Subject: [PATCH] perf(llm): judge a comment's new themes together A retune only asks for the themes vocab v3 added, and those yes/no calls do not depend on each other. They ran one after another, so four workers left the rig's four slots waiting on the tunnel. Up to four now run at once. --- src/llm/tagger.py | 37 +++++++++++++++++++++++++++++++++---- tests/llm/test_tagger.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/src/llm/tagger.py b/src/llm/tagger.py index 6fd4a77..8f27c32 100644 --- a/src/llm/tagger.py +++ b/src/llm/tagger.py @@ -27,6 +27,7 @@ import hashlib import json import logging import math +from concurrent.futures import ThreadPoolExecutor from dataclasses import asdict, dataclass from datetime import datetime, timezone from typing import Any, Callable, Iterable, Mapping, Sequence @@ -38,6 +39,10 @@ log = logging.getLogger(__name__) Vector = Sequence[float] Embed = Callable[[Sequence[str]], list[list[float]]] Judge = Callable[[Theme, str], bool | None] +# New themes on one comment are independent. The rig serves four judges +# at once; overlapping a comment's calls hides the tunnel wait so four +# workers fill those slots. Wider than that just queues. +_JUDGE_WIDTH = 4 ChunkLoader = Callable[[str], list[tuple[int, str, list[float]]]] STATE_KEY = "llm_tags" @@ -160,20 +165,44 @@ def tag_comment( shortlist pays only for its new candidates.""" cands = shortlist(chunk_vecs, card_vecs, top=top) judged: dict[str, bool | None] = {} - tags: dict[str, TagInfo] = {} + pending: list[Candidate] = [] for c in cands: if c.slug not in vocab: continue if prior is not None and c.slug in prior: - verdict = prior[c.slug] + judged[c.slug] = prior[c.slug] else: - verdict = judge(vocab.get(c.slug), chunks[c.chunk_idx]) - judged[c.slug] = verdict + pending.append(c) + for slug, verdict in _judge_pending(pending, chunks, vocab, judge): + judged[slug] = verdict + tags: dict[str, TagInfo] = {} + for c in cands: + verdict = judged.get(c.slug) if verdict and c.score >= min_confidence and len(tags) < max_tags: tags[c.slug] = TagInfo(confidence=c.score, evidence_chunk=c.chunk_idx) return TagResult(key, tags, tuple(cands), judged, content_hash(chunks)) +def _judge_pending( + pending: Sequence[Candidate], + chunks: Sequence[str], + vocab: Vocab, + judge: Judge, +) -> list[tuple[str, bool | None]]: + """Judge candidates that have no stored verdict. One call stays on + this thread; several overlap. Order matches *pending*.""" + if not pending: + return [] + + def call(c: Candidate) -> tuple[str, bool | None]: + return c.slug, judge(vocab.get(c.slug), chunks[c.chunk_idx]) + + if len(pending) == 1: + return [call(pending[0])] + with ThreadPoolExecutor(max_workers=min(_JUDGE_WIDTH, len(pending))) as ex: + return list(ex.map(call, pending)) + + def state_matches( extra: Mapping[str, Any], *, version: int, model: str, digest: str ) -> bool: diff --git a/tests/llm/test_tagger.py b/tests/llm/test_tagger.py index 79e4a4e..76f5e2a 100644 --- a/tests/llm/test_tagger.py +++ b/tests/llm/test_tagger.py @@ -109,6 +109,46 @@ class TestPure: ) in seen # evidence is the scoring chunk assert r.content_hash == content_hash(chunks) + def test_new_themes_are_judged_together(self): + import threading + import time + + chunks = ["shared"] + vecs = [[1.0, 1.0, 1.0]] + current = threading.Semaphore(0) + release = threading.Semaphore(0) + + def judge(theme, evidence): + current.release() + assert release.acquire(timeout=2), theme.slug + return True + + started = time.perf_counter() + box: dict = {} + + def run(): + box["result"] = tag_comment( + "K", + chunks, + vecs, + vocab=VOCAB, + card_vecs=CARDS, + judge=judge, + top=3, + max_tags=3, + min_confidence=0.0, + ) + + worker = threading.Thread(target=run) + worker.start() + for _ in range(3): + assert current.acquire(timeout=2) + release.release(3) + worker.join(timeout=2) + assert not worker.is_alive() + assert time.perf_counter() - started < 1.5 + assert box["result"].slugs == [c.slug for c in shortlist(vecs, CARDS, top=3)] + def test_min_confidence_filters_weak_matches(self): chunks = ["x"] vecs = [[0.6, 0.8, 0.0]] # cosine 0.6 with telehealth, 0.8 with care-management