From d98f6105eb3e0d7608df6383b96198b5a8836c3b Mon Sep 17 00:00:00 2001
From: kert <kert@rack.localdomain>
Date: Thu, 10 Sep 2026 18:49:45 -0400
Subject: [PATCH] fix(llm): cap generation (num_predict 1200) and ask for
 concise answers; take years from the Lineage line (refs #691)

---
 src/llm/config.py     | 2 ++
 src/llm/rag.py        | 6 +++++-
 stack.toml            | 1 +
 tests/llm/test_rag.py | 6 +++++-
 4 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/llm/config.py b/src/llm/config.py
index 1eba437..180ee10 100644
--- a/src/llm/config.py
+++ b/src/llm/config.py
@@ -39,6 +39,7 @@ class LlmConfig:
     large_min_vram_gb: float = 20.0
     chat_num_ctx: int = 8192
     chat_temperature: float = 0.2
+    chat_max_tokens: int = 1200
     recency_half_life_days: float = 365.0
     recency_weight: float = 0.3
     k_per_kind: dict[str, int] = field(
@@ -117,6 +118,7 @@ def load() -> LlmConfig:
         large_min_vram_gb=float(_opt(section, "large_min_vram_gb", 20.0)),
         chat_num_ctx=int(_opt(section, "chat_num_ctx", 8192)),
         chat_temperature=float(_opt(section, "chat_temperature", 0.2)),
+        chat_max_tokens=int(_opt(section, "chat_max_tokens", 1200)),
         recency_half_life_days=float(_opt(section, "recency_half_life_days", 365.0)),
         recency_weight=float(_opt(section, "recency_weight", 0.3)),
         k_per_kind=k_per_kind,
diff --git a/src/llm/rag.py b/src/llm/rag.py
index fedb280..ea64108 100644
--- a/src/llm/rag.py
+++ b/src/llm/rag.py
@@ -237,7 +237,10 @@ _CITE_REMINDER = (
     "or a commenter said MUST end with the bracketed label of the Lineage "
     "line, Valuation row or excerpt it comes from, copied exactly — for "
     'example: "G2058 was replaced by CPT 99439 for CY2021 [CY2021 PFS final '
-    '85 FR 84547 ¶686]." Do not write such a sentence without its label.'
+    '85 FR 84547 ¶686]." Do not write such a sentence without its label. '
+    "Take the year of each fact from its Lineage line, never from a "
+    "neighbouring excerpt. Be concise: answer in at most about 400 words "
+    "unless the question asks for a full listing."
 )
 
 _SYSTEM = (
@@ -736,6 +739,7 @@ def stream_answer(
                 "options": {
                     "num_ctx": cfg.chat_num_ctx,
                     "temperature": cfg.chat_temperature,
+                    "num_predict": cfg.chat_max_tokens,
                 },
             },
         ) as resp:
diff --git a/stack.toml b/stack.toml
index b9046fb..44196bf 100644
--- a/stack.toml
+++ b/stack.toml
@@ -112,6 +112,7 @@ instruct_model_large = "qwen2.5:32b"   # used when the chosen host declares >= l
 large_min_vram_gb = 20
 chat_num_ctx = 8192                    # passed per request; no Modelfile ctx variants
 chat_temperature = 0.2                 # grounded answers cite labels verbatim; low temperature keeps them literal
+chat_max_tokens = 1200                 # num_predict: a history answer ran to 5,000 tokens (3 min) without a cap
 embed_dim = 768
 # HNSW index build. Safe since the postgres image ships an AVX-512-free
 # pgvector rebuild (infra/images/postgresql.Dockerfile, #580); before that it
diff --git a/tests/llm/test_rag.py b/tests/llm/test_rag.py
index 8fcb4d0..b52df0f 100644
--- a/tests/llm/test_rag.py
+++ b/tests/llm/test_rag.py
@@ -778,7 +778,11 @@ class TestStreamAnswer:
         assert events[-1] == {"type": "done"}
         body = client.stream.call_args.kwargs["json"]
         assert body["model"] == "big"
-        assert body["options"] == {"num_ctx": 8192, "temperature": CFG.chat_temperature}
+        assert body["options"] == {
+            "num_ctx": 8192,
+            "temperature": CFG.chat_temperature,
+            "num_predict": CFG.chat_max_tokens,
+        }
         assert client.stream.call_args.args[1] == "http://h1:11434/api/chat"
 
     @patch("llm.rag.valuation_evidence", return_value=None)