From bef31d38569f87a8d25f8a922f72f2459f817f0f Mon Sep 17 00:00:00 2001 From: kert Date: Wed, 11 Nov 2026 15:22:31 -0500 Subject: [PATCH] fix(llm): tag and eval-tags read one set of tagger parameters stack llm tag defaulted to --min-confidence 0.0 while eval-tags scored at 0.60, so a run without flags was not the configuration that was gated. [llm].tag_top / tag_max_tags / tag_min_confidence now default both commands (unchanged values: 8 / 3 / 0.60). --- src/cli/llm.py | 31 +++++++++++++++++++++++-------- src/llm/config.py | 9 +++++++++ stack.toml | 3 +++ tests/cli/test_llm_tag_cli.py | 7 ++++++- tests/llm/test_config.py | 11 +++++++++++ 5 files changed, 52 insertions(+), 9 deletions(-) diff --git a/src/cli/llm.py b/src/cli/llm.py index 076e70e..d8c53de 100644 --- a/src/cli/llm.py +++ b/src/cli/llm.py @@ -283,15 +283,17 @@ def tag( False, "--dry-run", help="Compute and print, write nothing." ), top: int = typer.Option( - 8, "--top", help="Themes shortlisted per comment before judging." + None, + "--top", + help="Themes shortlisted per comment before judging [llm].tag_top.", ), max_tags: int = typer.Option( - 3, "--max-tags", help="Most themes written per comment." + None, "--max-tags", help="Most themes written per comment [llm].tag_max_tags." ), min_confidence: float = typer.Option( - 0.0, + None, "--min-confidence", - help="Shortlist similarity below which a judged yes is not written.", + help="Shortlist similarity a judged yes must reach [llm].tag_min_confidence.", ), verbose: bool = typer.Option( False, "--verbose", help="Print every comment's tags." @@ -306,6 +308,11 @@ def tag( from llm.tagger import run as run_tagging cfg = llm_config.load() + top = cfg.tag_top if top is None else top + max_tags = cfg.tag_max_tags if max_tags is None else max_tags + min_confidence = ( + cfg.tag_min_confidence if min_confidence is None else min_confidence + ) rt = _tagging_runtime(cfg) def progress(key: str, result: Any, why: str) -> None: @@ -357,9 +364,11 @@ def eval_tags( limit: int = typer.Option( 0, "--limit", help="Score only the first N golden entries." ), - top: int = typer.Option(8, "--top"), - max_tags: int = typer.Option(3, "--max-tags"), - min_confidence: float = typer.Option(0.60, "--min-confidence"), + top: int = typer.Option(None, "--top", help="[llm].tag_top"), + max_tags: int = typer.Option(None, "--max-tags", help="[llm].tag_max_tags"), + min_confidence: float = typer.Option( + None, "--min-confidence", help="[llm].tag_min_confidence" + ), dump: str = typer.Option( "", "--dump", @@ -406,7 +415,13 @@ def eval_tags( from llm import config as llm_config from llm.tagger import tag_comment - rt = _tagging_runtime(llm_config.load()) + cfg = llm_config.load() + top = cfg.tag_top if top is None else top + max_tags = cfg.tag_max_tags if max_tags is None else max_tags + min_confidence = ( + cfg.tag_min_confidence if min_confidence is None else min_confidence + ) + rt = _tagging_runtime(cfg) dumped: dict[str, Any] = {} def tag_one(key: str): diff --git a/src/llm/config.py b/src/llm/config.py index 7e1cf1f..bb3a26a 100644 --- a/src/llm/config.py +++ b/src/llm/config.py @@ -39,6 +39,12 @@ class LlmConfig: instruct_model_large: str = "" large_min_vram_gb: float = 20.0 chat_num_ctx: int = 8192 + #: Theme tagger (P35): candidates shortlisted per comment, most tags + #: written, and the shortlist similarity a judged yes must reach. + #: `stack llm tag` and `eval-tags` both default to these. + tag_top: int = 8 + tag_max_tags: int = 3 + tag_min_confidence: float = 0.60 chat_temperature: float = 0.2 chat_max_tokens: int = 1200 #: Characters per token the instruct model's tokenizer yields on the @@ -127,6 +133,9 @@ def load() -> LlmConfig: instruct_model_large=str(_opt(section, "instruct_model_large", "")), large_min_vram_gb=float(_opt(section, "large_min_vram_gb", 20.0)), chat_num_ctx=int(_opt(section, "chat_num_ctx", 8192)), + tag_top=int(_opt(section, "tag_top", 8)), + tag_max_tags=int(_opt(section, "tag_max_tags", 3)), + tag_min_confidence=float(_opt(section, "tag_min_confidence", 0.60)), chat_temperature=float(_opt(section, "chat_temperature", 0.2)), chat_max_tokens=int(_opt(section, "chat_max_tokens", 1200)), chat_chars_per_token=float(_opt(section, "chat_chars_per_token", 2.4)), diff --git a/stack.toml b/stack.toml index 40fa93f..6bf51aa 100644 --- a/stack.toml +++ b/stack.toml @@ -114,6 +114,9 @@ instruct_model = "qwen2.5:14b" # baseline, pulled on every fleet host instruct_model_large = "qwen2.5:32b" # used when the chosen host declares >= large_min_vram_gb large_min_vram_gb = 20 chat_num_ctx = 8192 # passed per request; no Modelfile ctx variants +tag_top = 8 # theme tagger: candidates judged per comment (P35) +tag_max_tags = 3 # most themes written per comment +tag_min_confidence = 0.60 # shortlist similarity a judged yes must reach; gated with eval-tags chat_temperature = 0.2 # grounded answers cite labels verbatim; low temperature keeps them literal chat_max_tokens = 1200 # num_predict: a history answer ran to 5,000 tokens (3 min) without a cap chat_chars_per_token = 2.4 # measured 2.5–2.9 on FR/regulatory text; sizes the excerpt budget so prompt + reply fit chat_num_ctx (overflow is caught and rebuilt) diff --git a/tests/cli/test_llm_tag_cli.py b/tests/cli/test_llm_tag_cli.py index d847df0..3e76c86 100644 --- a/tests/cli/test_llm_tag_cli.py +++ b/tests/cli/test_llm_tag_cli.py @@ -2,6 +2,8 @@ from __future__ import annotations +from types import SimpleNamespace + import pytest from typer.testing import CliRunner @@ -47,7 +49,10 @@ def rt(monkeypatch, tmp_path): "model": "m", } monkeypatch.setattr(llm_cli, "_tagging_runtime", lambda cfg: runtime) - monkeypatch.setattr("llm.config.load", lambda: object()) + monkeypatch.setattr( + "llm.config.load", + lambda: SimpleNamespace(tag_top=8, tag_max_tags=3, tag_min_confidence=0.0), + ) try: yield store, key, judged finally: diff --git a/tests/llm/test_config.py b/tests/llm/test_config.py index 4283d86..ebf401b 100644 --- a/tests/llm/test_config.py +++ b/tests/llm/test_config.py @@ -153,3 +153,14 @@ def test_instruct_model_env_override(monkeypatch): assert load().instruct_model == "qwen2.5:32b" monkeypatch.delenv("LLM_INSTRUCT_MODEL") assert load().instruct_model == "qwen2.5:14b" + + +def test_tagging_parameters_load_from_config(): + # `stack llm tag` defaulted to min_confidence 0.0 while eval-tags used + # 0.60: the run and its gate disagreed. Both now read these. + from llm.config import load + + cfg = load() + assert cfg.tag_top == 8 + assert cfg.tag_max_tags == 3 + assert cfg.tag_min_confidence == 0.60