fix(llm): the chat cites what it is told to — lineage lines as sentences ending with their label, junk page numbers filtered from event code lists, an end-of-prompt citation rule, temperature 0.2; valuation labels link in the page (refs #691 #692)

Measured on qwen2.5:32b: with the rule only in the system prompt the CCM
history answers cited nothing; with the lineage block rendered as sentences
("2021: G2058 was replaced by 99439 [CY2021 PFS final 85 FR 84547 ¶686]")
and the rule restated before the question they cite, and the audio-only
answer cites every dated sentence. from/to code lists are filtered against
pfs.rvu ∪ pfs.cpt_code (15% of to-tokens were FR page numbers).

src/llm/config.py

@@ -38,6 +38,7 @@ class LlmConfig:
     instruct_model_large: str = ""
     large_min_vram_gb: float = 20.0
     chat_num_ctx: int = 8192
+    chat_temperature: float = 0.2
     recency_half_life_days: float = 365.0
     recency_weight: float = 0.3
     k_per_kind: dict[str, int] = field(
@@ -115,6 +116,7 @@ def load() -> LlmConfig:
         instruct_model_large=str(_opt(section, "instruct_model_large", "")),
         large_min_vram_gb=float(_opt(section, "large_min_vram_gb", 20.0)),
         chat_num_ctx=int(_opt(section, "chat_num_ctx", 8192)),
+        chat_temperature=float(_opt(section, "chat_temperature", 0.2)),
         recency_half_life_days=float(_opt(section, "recency_half_life_days", 365.0)),
         recency_weight=float(_opt(section, "recency_weight", 0.3)),
         k_per_kind=k_per_kind,

src/llm/lineage.py

@@ -51,7 +51,9 @@ from pfs.families import detect_codes
 log = logging.getLogger(__name__)
 
 _HEADER = (
-    "Lineage (dated events with anchors; cite the bracketed label for any dated claim):"
+    "Lineage — dated facts about these codes from the Federal Register rules and "
+    "the PFS files. Each line ends with the bracketed label to copy into any "
+    "sentence that uses it:"
 )
 
 #: Event kinds a prompt block never drops for the row cap (ruling 17) —
@@ -149,13 +151,47 @@ class GuidanceRef:
     p_id_src: int
 
 
+#: kind -> plain-English predicate. The prompt line is a sentence the
+#: model can restate almost verbatim, ending with the label it must
+#: copy — "2021: G2058 was replaced by 99439 [CY2021 PFS final …]" —
+#: not the earlier "[label] 2021 replaced_by G2058 ( → 99439)", which
+#: qwen2.5 read as a table it did not have to cite.
+_KIND_TEXT = {
+    "created": "was created",
+    "adopted_cpt": "was adopted from CPT",
+    "appeared": "first appeared in the PFS RVU file",
+    "replaces": "replaced {from_}",
+    "replaced_by": "was replaced by {to}",
+    "deleted": "was deleted",
+    "disappeared": "dropped out of the PFS RVU file",
+    "cpt_deleted": "was deleted from CPT",
+    "cpt_changed": "was changed in CPT",
+    "crosswalk": "was crosswalked to {to}",
+    "bundled": "was bundled",
+    "revalued": "was revalued",
+    "status_change": "changed payment status",
+    "descriptor_change": "changed its descriptor",
+    "telehealth_list": "was added to the Medicare telehealth list",
+}
+
+
 def _event_line(e: LineageEvent) -> str:
-    line = f"[{e.label}] {e.year} {e.kind} {e.code}"
+    to = _codes_str(e.to_codes)
-    if e.from_codes or e.to_codes:
+    from_ = _codes_str(e.from_codes)
-        line += f" ({', '.join(e.from_codes)} → {', '.join(e.to_codes)})"
+    text = _KIND_TEXT.get(e.kind, e.kind.replace("_", " "))
+    if "{to}" in text and not to:
+        text = text.replace(" {to}", "").replace("{to}", "").rstrip()
+    if "{from_}" in text and not from_:
+        text = text.replace(" {from_}", "").replace("{from_}", "").rstrip()
+    text = text.format(to=to, from_=from_)
+    line = f"{e.year}: {e.code} {text}"
+    if e.from_codes and "{from_}" not in _KIND_TEXT.get(e.kind, ""):
+        line += f" (from {from_})"
+    if e.to_codes and "{to}" not in _KIND_TEXT.get(e.kind, ""):
+        line += f" (to {to})"
     if e.note:
         line += f" — {e.note}"
-    return line
+    return f"{line} [{e.label}]"
 
 
 #: Ruling B11: a priority-kind event (created/replaces/…) is only
@@ -177,13 +213,13 @@ def _codes_str(codes: Sequence[str]) -> str:
 
 def _diff_line(d: ElementDiff) -> str:
     return (
-        f"[{d.label}] {d.type}={d.value}: "
+        f"{d.type}={d.value}: required for {_codes_str(d.in_codes)}; "
-        f"in {_codes_str(d.in_codes)}; not in {_codes_str(d.not_in_codes)}"
+        f"not for {_codes_str(d.not_in_codes)} [{d.label}]"
     )
 
 
 def _guidance_line(g: GuidanceRef) -> str:
-    return f"[{g.label}] {g.locator} — {g.kind.upper()}"
+    return f"{g.locator} ({g.kind.upper()}) is cited for these codes [{g.label}]"
 
 
 def _select_for_prompt(
@@ -500,12 +536,55 @@ def _fr_url(store: Any, item_key: str, p_id: int, mtime: int) -> str:
     return url
 
 
-def _to_lineage_event(store: Any, r: EventRow, mtime: int) -> LineageEvent:
+#: Event kinds whose whole meaning is the other code(s) — a row of one
+#: of these with no real code left after junk filtering says nothing.
+_DIRECTIONAL_KINDS = frozenset({"replaces", "replaced_by", "crosswalk"})
+
+#: "known" -> frozenset of every HCPCS/CPT code the replica knows
+#: (pfs.rvu ∪ pfs.cpt_code), keyed on the replica mtime.
+_KNOWN_CACHE = _ReplicaCache()
+
+
+def _known_codes(cur: Any, mtime: int) -> frozenset[str]:
+    """Every code in ``pfs.rvu`` or ``pfs.cpt_code`` (18.6k + 11.4k
+    live). The FR event extractor's from/to lists carry FR page numbers
+    that look like CPT codes ("(84 FR 62694 through 62695)" leaves
+    ``62695``); 15% of the to-tokens in ``pfs.code_event`` are such
+    junk. Empty (no filtering) when neither table can be read."""
+    cached = _KNOWN_CACHE.get("known", mtime)
+    if cached is not _MISSING:
+        return cached
+    known: set[str] = set()
+    for sql in (
+        "SELECT DISTINCT hcpcs FROM pfs.rvu",
+        "SELECT DISTINCT code FROM pfs.cpt_code",
+    ):
+        try:
+            known.update(str(row[0]) for row in cur.execute(sql).fetchall())
+        except Exception as e:  # noqa: BLE001
+            if not is_missing_table_error(e):
+                log.warning("known-code lookup failed: %s", e)
+    out = frozenset(known)
+    _KNOWN_CACHE.put("known", mtime, out)
+    return out
+
+
+def _to_lineage_event(
+    store: Any, r: EventRow, mtime: int, known: frozenset[str] = frozenset()
+) -> LineageEvent | None:
     """*r* as a ``LineageEvent`` with its label resolved but ``url=""``
     — FR urls are resolved afterwards, only for the rows selected for
-    the prompt block (``_resolve_prompt_urls``)."""
+    the prompt block (``_resolve_prompt_urls``). With a non-empty
+    *known* set, from/to tokens that are not real codes are dropped,
+    and a directional row (replaces/replaced_by/crosswalk) left with no
+    code is dropped altogether (``None``)."""
     from_codes = tuple(r.from_codes.split()) if r.from_codes else ()
     to_codes = tuple(r.to_codes.split()) if r.to_codes else ()
+    if known:
+        from_codes = tuple(c for c in from_codes if c in known)
+        to_codes = tuple(c for c in to_codes if c in known)
+        if r.kind in _DIRECTIONAL_KINDS and not from_codes and not to_codes:
+            return None
     if r.source == "fr":
         label = _source_label(store, r.item_key, r.p_id, mtime)
     elif r.source == "cpt":
@@ -1008,9 +1087,11 @@ def lineage_evidence(question: str, cfg: LlmConfig) -> LineageEvidence | None:
         raw_events = _collect_events(
             cur, store, mtime, codes, cfg.lineage_on_demand_max
         )
+        known = _known_codes(cur, mtime)
         events_list = [
-            _to_lineage_event(store, r, bib_mtime)
+            ev
             for r in _collapse(raw_events, store, bib_mtime)
+            if (ev := _to_lineage_event(store, r, bib_mtime, known)) is not None
         ]
         events = _resolve_prompt_urls(
             store, events_list, bib_mtime, cfg.lineage_max_rows

src/llm/rag.py

@@ -230,6 +230,16 @@ def _resolve_mode(question: str, mode: str) -> str:
     return "recent"
 
 
+_CITE_REMINDER = (
+    "Answer format: markdown, with headings or lists where they help. Every "
+    "sentence that states a date, a code being created, adopted, replaced or "
+    "deleted, an RVU, a conversion factor, a payment amount, or what a rule "
+    "or a commenter said MUST end with the bracketed label of the Lineage "
+    "line, Valuation row or excerpt it comes from, copied exactly — for "
+    'example: "G2058 was replaced by CPT 99439 for CY2021 [CY2021 PFS final '
+    '85 FR 84547 ¶686]." Do not write such a sentence without its label.'
+)
+
 _SYSTEM = (
     "You answer questions about CMS rulemaking using ONLY the material "
     "provided below — the excerpts, and the Lineage and Valuation sections "
@@ -255,6 +265,14 @@ _SYSTEM = (
     "those numbers, cite the row's bracketed label exactly, e.g. [PFS "
     "CY2026 Addendum B]. Do not compute, extrapolate or convert numbers "
     "beyond what the rows show; if a code is listed as not priced, say so."
+    " CITATION FORMAT (required): every sentence that states a date, a "
+    "code's creation, adoption, replacement or deletion, an RVU, a "
+    "conversion factor, a payment amount, or what a commenter or a rule "
+    "said MUST end with the bracketed label of the row or excerpt it comes "
+    'from, copied verbatim, e.g. "CPT 99490 was adopted for CY2015 '
+    '[CY2015 PFS final 79 FR 67716 ¶1251]." Cite one or more labels per '
+    "such sentence; a paragraph without a citation is not acceptable. "
+    "Never invent a label. Use markdown headings and lists when they help."
 )
 
 
@@ -365,6 +383,14 @@ def _user_message(
         parts.append(lineage.prompt_block(lineage_rows_cap))
     if evidence is not None:
         parts.append(evidence.prompt_block(valuation_rows_cap))
+    if lineage is not None or evidence is not None:
+        # Restated right before the question: a long grounded prompt
+        # (dozens of lineage/valuation rows plus excerpts) is where the
+        # local model stops following the system prompt's citation rule —
+        # measured on qwen2.5:32b, the CCM history answers cited nothing
+        # with the rule in the system prompt alone. A codeless question
+        # gets no instruction, so its prompt stays byte-identical.
+        parts.append(_CITE_REMINDER)
     parts.append(f"Question: {question}")
     return "\n\n".join(parts)
 
@@ -643,7 +669,10 @@ def stream_answer(
                 "model": model,
                 "messages": messages,
                 "stream": True,
-                "options": {"num_ctx": cfg.chat_num_ctx},
+                "options": {
+                    "num_ctx": cfg.chat_num_ctx,
+                    "temperature": cfg.chat_temperature,
+                },
             },
         ) as resp:
             resp.raise_for_status()

src/llm/web/chat.html

@@ -298,7 +298,13 @@
     // label → url for this turn, filled by the lineage/valuation/sources
     // events; the answer is repainted whenever the map grows.
     function collectLinks(links, ev) {
-      const add = (label, url) => { if (label && url && !links[label]) links[label] = url; };
+      // Valuation provenance labels arrive with their brackets
+      // ("[PFS CY2017 Addendum B]"); the model cites the inside.
+      const add = (label, url) => {
+        if (!label || !url) return;
+        const key = String(label).replace(/^\[|\]$/g, '');
+        if (!links[key]) links[key] = url;
+      };
       if (ev.type === 'lineage') {
         for (const e of ev.events || []) add(e.label, e.url);
         for (const g of ev.guidance || []) { add(g.label, g.url); add(g.locator, g.url); }

stack.toml

@@ -111,6 +111,7 @@ instruct_model = "qwen2.5:14b"         # baseline, pulled on every fleet host
 instruct_model_large = "qwen2.5:32b"   # used when the chosen host declares >= large_min_vram_gb
 large_min_vram_gb = 20
 chat_num_ctx = 8192                    # passed per request; no Modelfile ctx variants
+chat_temperature = 0.2                 # grounded answers cite labels verbatim; low temperature keeps them literal
 embed_dim = 768
 # HNSW index build. Safe since the postgres image ships an AVX-512-free
 # pgvector rebuild (infra/images/postgresql.Dockerfile, #580); before that it

tests/llm/test_lineage.py

@@ -696,60 +696,61 @@ class TestPromptBlock:
             guidance=guidance,
         )
         assert ev.prompt_block() == (
-            "Lineage (dated events with anchors; cite the bracketed label "
+            lineage._HEADER + "\n"
-            "for any dated claim):\n"
+            "2021: G2058 was replaced by 99439 [CY2021 PFS final ¶686]\n"
-            "[CY2021 PFS final ¶686] 2021 replaced_by G2058 ( → 99439)\n"
+            "2021: 99439 was created — from CPT Editorial Panel "
-            "[CY2021 PFS final ¶686] 2021 created 99439 — "
+            "[CY2021 PFS final ¶686]\n"
-            "from CPT Editorial Panel\n"
             "Element differences:\n"
-            "[CY2021 PFS final ¶1578] consent=required: "
+            "consent=required: required for 99490, 99491; not for G0556 "
-            "in 99490, 99491; not in G0556\n"
+            "[CY2021 PFS final ¶1578]\n"
             "Guidance:\n"
-            "[CY2021 PFS final ¶1578] 42 CFR 410.78(a)(3) — CFR"
+            "42 CFR 410.78(a)(3) (CFR) is cited for these codes "
+            "[CY2021 PFS final ¶1578]"
         )
 
     def test_header_only_with_no_events(self):
         ev = LineageEvidence(
             codes=("99490",), families=(), events=(), element_diffs=(), guidance=()
         )
-        assert ev.prompt_block() == (
+        assert ev.prompt_block() == lineage._HEADER
-            "Lineage (dated events with anchors; cite the bracketed label "
-            "for any dated claim):"
-        )
 
-    def test_priority_kinds_always_kept_non_priority_trimmed(self):
+    def test_event_lines_are_sentences_ending_with_the_label(self):
-        priority = LineageEvent(
+        """qwen2.5 read the old "[label] 2021 replaced_by G2058 ( → 99439)"
-            "99490", 2021, "created", (), (), "[L]", "K", 1, 0, "", "fr", True, ""
+        rows as a table it did not have to cite; a sentence ending in the
-        )
+        label is the shape the system prompt asks the answer to copy."""
-        noisy = [
+
-            LineageEvent(
+        def line(kind, frm=(), to=(), note=""):
-                "99490",
+            return lineage._event_line(
-                2022 + i,
+                LineageEvent(
-                "revalued",
+                    code="G2058",
-                (),
+                    year=2021,
-                (),
+                    kind=kind,
-                f"[L{i}]",
+                    from_codes=frm,
-                "K",
+                    to_codes=to,
-                1,
+                    label="L",
-                0,
+                    item_key="K",
-                "",
+                    p_id=1,
-                "rvu",
+                    page=1,
-                True,
+                    url="",
-                "",
+                    source="fr",
+                    anchored=True,
+                    note=note,
+                )
             )
-            for i in range(5)
+
-        ]
+        assert (
-        ev = LineageEvidence(
+            line("replaced_by", to=("99439",))
-            codes=("99490",),
+            == "2021: G2058 was replaced by 99439 [L]"
-            families=(),
-            events=(priority, *noisy),
-            element_diffs=(),
-            guidance=(),
-            max_prompt_rows=3,
         )
-        rendered = ev.prompt_block()
+        assert line("replaces", frm=("G0506",)) == "2021: G2058 replaced G0506 [L]"
-        assert "created" in rendered
+        assert line("created") == "2021: G2058 was created [L]"
-        assert rendered.count("revalued") == 2  # budget: 3 - 1 priority = 2 kept
+        assert (
+            line("crosswalk", to=("99439",))
+            == "2021: G2058 was crosswalked to 99439 [L]"
+        )
+        assert line("disappeared") == "2021: G2058 dropped out of the PFS RVU file [L]"
+        # a directional kind whose codes were filtered away still reads
+        assert line("replaced_by") == "2021: G2058 was replaced by [L]"
 
     def test_priority_kinds_exceed_cap_but_are_never_dropped(self):
         priorities = [
@@ -767,8 +768,8 @@ class TestPromptBlock:
             max_prompt_rows=2,
         )
         rendered = ev.prompt_block()
-        for kind in ("created", "replaces", "replaced_by", "deleted"):
+        for i in range(4):  # every priority row rendered, each ending with its label
-            assert kind in rendered
+            assert f"[[L{i}]]" in rendered
 
     def test_under_budget_every_priority_row_for_a_code_survives(self):
         """Ruling B15: the per-code priority cap is a budget safety net,
@@ -1099,7 +1100,8 @@ class TestCptLabel:
         assert event.note == "absent from CPT 2024"
         line = lineage._event_line(event)
         assert (
-            line == "[CPT Changes 2024] 2024 cpt_deleted 99213 — absent from CPT 2024"
+            line
+            == "2024: 99213 was deleted from CPT — absent from CPT 2024 [CPT Changes 2024]"
         )
         assert (
             line.count("absent from CPT 2024") == 1
@@ -1751,3 +1753,69 @@ class TestLineageEvidenceLive:
         median = times_ms[len(times_ms) // 2]
         print(f"lineage_evidence median: {median:.2f} ms (all: {times_ms})")
         assert median < 300, f"{median:.2f} ms over {times_ms}"
+
+
+class TestKnownCodeFilter:
+    """FR page numbers leak into from/to lists ("(84 FR 62694 through
+    62695)" → 62695); with the replica's known-code set they are dropped
+    and an empty directional row disappears."""
+
+    def test_junk_tokens_dropped_and_empty_directional_row_removed(self):
+        known = frozenset({"G2058", "99439", "99487"})
+        row = _ev("G2058", 2021, "replaced_by", to="84750 99439 62694", source="fr")
+        ev = lineage._to_lineage_event(None, row, 1, known)
+        assert ev is not None and ev.to_codes == ("99439",)
+        junk = _ev("G2058", 2022, "replaced_by", to="84755 84756", source="fr")
+        assert lineage._to_lineage_event(None, junk, 1, known) is None
+        # non-directional rows keep going even with nothing left
+        created = _ev("G2058", 2021, "created", to="84750", source="fr")
+        assert lineage._to_lineage_event(None, created, 1, known).to_codes == ()
+
+    def test_no_known_set_means_no_filtering(self):
+        row = _ev("G2058", 2021, "replaced_by", to="84750 99439", source="fr")
+        assert lineage._to_lineage_event(None, row, 1).to_codes == ("84750", "99439")
+
+    def test_known_codes_reads_both_tables_and_tolerates_missing(self, con):
+        c, _ = con
+        c.execute("CREATE TABLE pfs.rvu (hcpcs VARCHAR, year INTEGER)")
+        c.execute("INSERT INTO pfs.rvu VALUES ('99490', 2026)")
+        known = lineage._known_codes(c, 1)
+        assert "99490" in known
+        import duckdb
+
+        bare = duckdb.connect(":memory:")
+        assert lineage._known_codes(bare, 2) == frozenset()
+
+    def test_priority_kinds_always_kept_non_priority_trimmed(self):
+        priority = LineageEvent(
+            "99490", 2021, "created", (), (), "[L]", "K", 1, 0, "", "fr", True, ""
+        )
+        noisy = [
+            LineageEvent(
+                "99490",
+                2022 + i,
+                "revalued",
+                (),
+                (),
+                f"[L{i}]",
+                "K",
+                1,
+                0,
+                "",
+                "rvu",
+                True,
+                "",
+            )
+            for i in range(5)
+        ]
+        ev = LineageEvidence(
+            codes=("99490",),
+            families=(),
+            events=(priority, *noisy),
+            element_diffs=(),
+            guidance=(),
+            max_prompt_rows=3,
+        )
+        rendered = ev.prompt_block()
+        assert "created" in rendered
+        assert rendered.count("revalued") == 2  # budget: 3 - 1 priority = 2 kept

tests/llm/test_rag.py

@@ -569,11 +569,11 @@ class TestBuildMessages:
         user = msgs[1]["content"]
         assert (
             user.index("Excerpts:")
-            < user.index("Lineage (dated events")
+            < user.index("Lineage — dated facts")
             < user.index("Valuation (authoritative")
             < user.index("Question: history?")
         )
-        assert "[CY2021 PFS final ¶686] 2021 replaced_by G2058" in user
+        assert "2021: G2058 was replaced by 99439 [CY2021 PFS final ¶686]" in user
         assert "lineage" in msgs[0]["content"].lower()
         assert "cite its bracketed label for every dated claim" in msgs[0]["content"]
 
@@ -583,7 +583,7 @@ class TestBuildMessages:
         )
         msgs = build_messages("q", [], lineage=le)
         user = msgs[1]["content"]
-        assert "Lineage (dated events" in user
+        assert "Lineage — dated facts" in user
         assert "Valuation (authoritative" not in user
 
     def test_no_lineage_prompt_unchanged(self):
@@ -778,7 +778,7 @@ class TestStreamAnswer:
         assert events[-1] == {"type": "done"}
         body = client.stream.call_args.kwargs["json"]
         assert body["model"] == "big"
-        assert body["options"] == {"num_ctx": 8192}
+        assert body["options"] == {"num_ctx": 8192, "temperature": CFG.chat_temperature}
         assert client.stream.call_args.args[1] == "http://h1:11434/api/chat"
 
     @patch("llm.rag.valuation_evidence", return_value=None)
@@ -966,7 +966,7 @@ class TestStreamAnswer:
         assert events[0]["codes"] == ["G2058"]
         body = client.stream.call_args.kwargs["json"]
         content = body["messages"][1]["content"]
-        assert content.index("Lineage (dated events") < content.index(
+        assert content.index("Lineage — dated facts") < content.index(
             "Valuation (authoritative"
         )