fix(llm): tag and eval-tags read one set of tagger parameters
Some checks failed
CI / lint (push) Successful in 1m9s
CI / notebooks-smoke (push) Has started running
CI / test (push) Has been cancelled
Deploy / changes (push) Has been cancelled
Deploy / notebooks (push) Has been cancelled
Deploy / zotero (push) Has been cancelled
Deploy / docs (push) Has been cancelled
Deploy / api (push) Has been cancelled
Deploy / bcda (push) Has been cancelled
Deploy / llm (push) Has been cancelled
Deploy / mc (push) Has been cancelled
Deploy / report (push) Has been cancelled
Infra CI / mc (push) Has been cancelled
Infra CI / notebooks (push) Has been cancelled
Infra CI / zotero (push) Has been cancelled
Infra CI / docs (push) Has been cancelled
Infra CI / api (push) Has been cancelled
Infra CI / bcda (push) Has been cancelled
Infra CI / llm (push) Has been cancelled

stack llm tag defaulted to --min-confidence 0.0 while eval-tags scored
at 0.60, so a run without flags was not the configuration that was
gated. [llm].tag_top / tag_max_tags / tag_min_confidence now default
both commands (unchanged values: 8 / 3 / 0.60).
This commit is contained in:
kert
2026-11-11 15:22:31 -05:00
parent 86b2ef5240
commit bef31d3856
5 changed files with 52 additions and 9 deletions

View File

@@ -283,15 +283,17 @@ def tag(
False, "--dry-run", help="Compute and print, write nothing."
),
top: int = typer.Option(
8, "--top", help="Themes shortlisted per comment before judging."
None,
"--top",
help="Themes shortlisted per comment before judging [llm].tag_top.",
),
max_tags: int = typer.Option(
3, "--max-tags", help="Most themes written per comment."
None, "--max-tags", help="Most themes written per comment [llm].tag_max_tags."
),
min_confidence: float = typer.Option(
0.0,
None,
"--min-confidence",
help="Shortlist similarity below which a judged yes is not written.",
help="Shortlist similarity a judged yes must reach [llm].tag_min_confidence.",
),
verbose: bool = typer.Option(
False, "--verbose", help="Print every comment's tags."
@@ -306,6 +308,11 @@ def tag(
from llm.tagger import run as run_tagging
cfg = llm_config.load()
top = cfg.tag_top if top is None else top
max_tags = cfg.tag_max_tags if max_tags is None else max_tags
min_confidence = (
cfg.tag_min_confidence if min_confidence is None else min_confidence
)
rt = _tagging_runtime(cfg)
def progress(key: str, result: Any, why: str) -> None:
@@ -357,9 +364,11 @@ def eval_tags(
limit: int = typer.Option(
0, "--limit", help="Score only the first N golden entries."
),
top: int = typer.Option(8, "--top"),
max_tags: int = typer.Option(3, "--max-tags"),
min_confidence: float = typer.Option(0.60, "--min-confidence"),
top: int = typer.Option(None, "--top", help="[llm].tag_top"),
max_tags: int = typer.Option(None, "--max-tags", help="[llm].tag_max_tags"),
min_confidence: float = typer.Option(
None, "--min-confidence", help="[llm].tag_min_confidence"
),
dump: str = typer.Option(
"",
"--dump",
@@ -406,7 +415,13 @@ def eval_tags(
from llm import config as llm_config
from llm.tagger import tag_comment
rt = _tagging_runtime(llm_config.load())
cfg = llm_config.load()
top = cfg.tag_top if top is None else top
max_tags = cfg.tag_max_tags if max_tags is None else max_tags
min_confidence = (
cfg.tag_min_confidence if min_confidence is None else min_confidence
)
rt = _tagging_runtime(cfg)
dumped: dict[str, Any] = {}
def tag_one(key: str):

View File

@@ -39,6 +39,12 @@ class LlmConfig:
instruct_model_large: str = ""
large_min_vram_gb: float = 20.0
chat_num_ctx: int = 8192
#: Theme tagger (P35): candidates shortlisted per comment, most tags
#: written, and the shortlist similarity a judged yes must reach.
#: `stack llm tag` and `eval-tags` both default to these.
tag_top: int = 8
tag_max_tags: int = 3
tag_min_confidence: float = 0.60
chat_temperature: float = 0.2
chat_max_tokens: int = 1200
#: Characters per token the instruct model's tokenizer yields on the
@@ -127,6 +133,9 @@ def load() -> LlmConfig:
instruct_model_large=str(_opt(section, "instruct_model_large", "")),
large_min_vram_gb=float(_opt(section, "large_min_vram_gb", 20.0)),
chat_num_ctx=int(_opt(section, "chat_num_ctx", 8192)),
tag_top=int(_opt(section, "tag_top", 8)),
tag_max_tags=int(_opt(section, "tag_max_tags", 3)),
tag_min_confidence=float(_opt(section, "tag_min_confidence", 0.60)),
chat_temperature=float(_opt(section, "chat_temperature", 0.2)),
chat_max_tokens=int(_opt(section, "chat_max_tokens", 1200)),
chat_chars_per_token=float(_opt(section, "chat_chars_per_token", 2.4)),

View File

@@ -114,6 +114,9 @@ instruct_model = "qwen2.5:14b" # baseline, pulled on every fleet host
instruct_model_large = "qwen2.5:32b" # used when the chosen host declares >= large_min_vram_gb
large_min_vram_gb = 20
chat_num_ctx = 8192 # passed per request; no Modelfile ctx variants
tag_top = 8 # theme tagger: candidates judged per comment (P35)
tag_max_tags = 3 # most themes written per comment
tag_min_confidence = 0.60 # shortlist similarity a judged yes must reach; gated with eval-tags
chat_temperature = 0.2 # grounded answers cite labels verbatim; low temperature keeps them literal
chat_max_tokens = 1200 # num_predict: a history answer ran to 5,000 tokens (3 min) without a cap
chat_chars_per_token = 2.4 # measured 2.5–2.9 on FR/regulatory text; sizes the excerpt budget so prompt + reply fit chat_num_ctx (overflow is caught and rebuilt)

View File

@@ -2,6 +2,8 @@
from __future__ import annotations
from types import SimpleNamespace
import pytest
from typer.testing import CliRunner
@@ -47,7 +49,10 @@ def rt(monkeypatch, tmp_path):
"model": "m",
}
monkeypatch.setattr(llm_cli, "_tagging_runtime", lambda cfg: runtime)
monkeypatch.setattr("llm.config.load", lambda: object())
monkeypatch.setattr(
"llm.config.load",
lambda: SimpleNamespace(tag_top=8, tag_max_tags=3, tag_min_confidence=0.0),
)
try:
yield store, key, judged
finally:

View File

@@ -153,3 +153,14 @@ def test_instruct_model_env_override(monkeypatch):
assert load().instruct_model == "qwen2.5:32b"
monkeypatch.delenv("LLM_INSTRUCT_MODEL")
assert load().instruct_model == "qwen2.5:14b"
def test_tagging_parameters_load_from_config():
# `stack llm tag` defaulted to min_confidence 0.0 while eval-tags used
# 0.60: the run and its gate disagreed. Both now read these.
from llm.config import load
cfg = load()
assert cfg.tag_top == 8
assert cfg.tag_max_tags == 3
assert cfg.tag_min_confidence == 0.60