homelab/stack
1
0
Fork 0
Code Issues 262 Pull Requests 2 Actions Packages Projects 1 Releases Wiki Activity

feat(llm): chat cites code valuations — Valuation prompt block, merged code-cited sources, valuation SSE event (refs P48)

Browse Source
This commit is contained in:
kert
2026-09-08 22:50:47 -04:00
parent 086d35445c
commit 82541cf24c
2 changed files with 133 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
src/llm/rag.py
View File

@@ -7,7 +7,11 @@ its own. Retrieval embeds the question once and searches the
merged and re-ranked by similarity × recency (``llm.rerank``), then each
source gets a deep link (``llm.links``). Generation streams from the
largest live Ollama host (``HostPool.acquire_generation``) with the
model tier that host can hold (``pick_model``).
model tier that host can hold (``pick_model``). When the question
cites HCPCS/CPT codes, ``llm.evidence`` supplies authoritative
valuation rows and rule chunks that cite those codes; the SSE event
order is ``valuation`` (only when evidence is found) → ``token``* →
``sources`` → ``done``.
"""
from __future__ import annotations
@@ -19,6 +23,13 @@ from typing import Iterator
import httpx
from llm.config import LlmConfig
from llm.evidence import (
ValuationEvidence,
code_cited_sources,
merge_sources,
valuation_evidence,
)
from llm.index import _engine
from llm.links import as_source
from llm.pool import HostPool, PoolEmbeddings, pick_model
from llm.rerank import Hit, blend, filter_since
@@ -40,6 +51,12 @@ _SYSTEM = (
"statement comes from when it matters. If the excerpts do not contain "
"the answer, say you don't have information on that in the indexed "
"library — do not invent facts."
" A Valuation section may follow the excerpts. Its rows are "
"authoritative for RVUs, conversion factors and payment amounts; when "
"you state any of those numbers, cite the row's bracketed label exactly, "
"e.g. [PFS CY2026 Addendum B]. Do not compute, extrapolate or convert "
"numbers beyond what the rows show; if a code is listed as not priced, "
"say so."
)
@@ -91,8 +108,12 @@ def retrieve(
return [_source(h) for h in ranked]
def build_messages(question: str, sources: list[dict]) -> list[dict]:
"""Grounded chat messages: system rules + question with excerpts."""
def build_messages(
question: str, sources: list[dict], evidence: ValuationEvidence | None = None
) -> list[dict]:
"""Grounded chat messages: system rules + question with excerpts and,
when available, a Valuation block between the excerpts and the
question."""
if sources:
context = "\n\n".join(
f"[{s['label']}] ({s.get('kind', '')}, {s.get('date', '') or 'undated'}) "
@@ -101,7 +122,11 @@ def build_messages(question: str, sources: list[dict]) -> list[dict]:
)
else:
context = "(no relevant excerpts found)"
user = f"Excerpts:\n\n{context}\n\nQuestion: {question}"
parts = [f"Excerpts:\n\n{context}"]
if evidence is not None:
parts.append(evidence.prompt_block())
parts.append(f"Question: {question}")
user = "\n\n".join(parts)
return [
{"role": "system", "content": _SYSTEM},
{"role": "user", "content": user},
@@ -113,13 +138,23 @@ def stream_answer(
) -> Iterator[dict]:
"""Retrieve, then stream a grounded answer from the largest live host.
Yields ``{"type":"token","text":…}`` events as the model generates,
then one ``{"type":"sources", "sources": […], "model": …, "host": …}``
and a final ``{"type":"done"}``.
When the question cites HCPCS/CPT codes, yields a ``valuation`` event
first (evidence's ``payload()``) and merges rule chunks that cite
those codes into ``sources``. Then yields ``{"type":"token","text":…}``
events as the model generates, then one
``{"type":"sources", "sources": […], "model": …, "host": …}`` and a
final ``{"type":"done"}``.
"""
sources = retrieve(question, cfg=cfg, pool=pool, since=since)
evidence = valuation_evidence(question, cfg)
if evidence is not None:
cited = code_cited_sources(
_engine(cfg), evidence.codes, per_code=cfg.code_cited_per_code
)
sources = merge_sources(sources, cited)
yield evidence.payload()
pool.check(cfg.instruct_model)
messages = build_messages(question, sources)
messages = build_messages(question, sources, evidence)
with pool.acquire_generation() as host, httpx.Client(timeout=_TIMEOUT) as client:
model = pick_model(cfg, pool, host)
with client.stream(

94
tests/llm/test_rag.py
View File

@@ -247,6 +247,46 @@ class TestBuildMessages:
msgs = build_messages("q", [])
assert "no relevant excerpts" in msgs[1]["content"].lower()
def test_evidence_block_between_excerpts_and_question(self):
from llm.evidence import ValuationEvidence
from pfs.valuation import ValuationRow
row = ValuationRow(
code="G0556",
description="d",
vintage="CY2026 final",
year=2026,
proposed=False,
status="A",
work=0.25,
pe_nf=0.22,
pe_f=0.06,
mp=0.02,
total_nf=0.49,
total_f=0.33,
cf=33.4009,
pay_nf=16.37,
pay_f=11.02,
label="[PFS CY2026 Addendum B]",
citation="90 FR 49266",
url="u",
)
ev = ValuationEvidence(("G0556",), ("APCM",), (row,), ())
msgs = build_messages("how much?", [], evidence=ev)
user = msgs[1]["content"]
assert (
user.index("Excerpts:")
< user.index("Valuation (authoritative")
< user.index("Question: how much?")
)
assert "[PFS CY2026 Addendum B] G0556" in user
assert "valuation" in msgs[0]["content"].lower()
assert "do not compute" in msgs[0]["content"].lower()
def test_no_evidence_prompt_unchanged(self):
assert build_messages("q", []) == build_messages("q", [], evidence=None)
assert "Valuation" not in build_messages("q", [])[1]["content"]
class TestStreamAnswer:
def _pool(self, vram=24.0, serves_big=True):
@@ -256,9 +296,10 @@ class TestStreamAnswer:
pool.serves.return_value = serves_big
return pool
@patch("llm.rag.valuation_evidence", return_value=None)
@patch("llm.rag.httpx.Client")
@patch("llm.rag.retrieve")
def test_yields_tokens_then_sources_then_done(self, mock_retrieve, MockClient):
def test_yields_tokens_then_sources_then_done(self, mock_retrieve, MockClient, _ev):
src = {
"id": "C1",
"label": "C1",
@@ -296,9 +337,10 @@ class TestStreamAnswer:
assert body["options"] == {"num_ctx": 8192}
assert client.stream.call_args.args[1] == "http://h1:11434/api/chat"
@patch("llm.rag.valuation_evidence", return_value=None)
@patch("llm.rag.httpx.Client")
@patch("llm.rag.retrieve")
def test_small_host_uses_baseline_model(self, mock_retrieve, MockClient):
def test_small_host_uses_baseline_model(self, mock_retrieve, MockClient, _ev):
mock_retrieve.return_value = []
client = MockClient.return_value.__enter__.return_value
resp = client.stream.return_value.__enter__.return_value
@@ -307,9 +349,10 @@ class TestStreamAnswer:
assert client.stream.call_args.kwargs["json"]["model"] == "chat"
assert events[-2]["model"] == "chat"
@patch("llm.rag.valuation_evidence", return_value=None)
@patch("llm.rag.httpx.Client")
@patch("llm.rag.retrieve")
def test_since_forwarded(self, mock_retrieve, MockClient):
def test_since_forwarded(self, mock_retrieve, MockClient, _ev):
mock_retrieve.return_value = []
client = MockClient.return_value.__enter__.return_value
resp = client.stream.return_value.__enter__.return_value
@@ -317,12 +360,55 @@ class TestStreamAnswer:
list(stream_answer("q", cfg=CFG, pool=self._pool(), since="2025-09-01"))
assert mock_retrieve.call_args.kwargs["since"] == "2025-09-01"
@patch("llm.rag.valuation_evidence", return_value=None)
@patch("llm.rag.httpx.Client")
@patch("llm.rag.retrieve")
def test_http_error_propagates(self, mock_retrieve, MockClient):
def test_http_error_propagates(self, mock_retrieve, MockClient, _ev):
mock_retrieve.return_value = []
client = MockClient.return_value.__enter__.return_value
resp = client.stream.return_value.__enter__.return_value
resp.raise_for_status.side_effect = RuntimeError("ollama down")
with pytest.raises(RuntimeError, match="ollama down"):
list(stream_answer("q", cfg=CFG, pool=self._pool()))
@patch("llm.rag._engine")
@patch("llm.rag.code_cited_sources")
@patch("llm.rag.valuation_evidence")
@patch("llm.rag.httpx.Client")
@patch("llm.rag.retrieve")
def test_valuation_event_first_and_sources_merged(
self, mock_retrieve, MockClient, mock_ev, mock_cited, mock_engine
):
from llm.evidence import ValuationEvidence
src = {
"id": "C1",
"label": "C1",
"kind": "comment",
"snippet": "s",
"score": 0.1,
}
cited = {
"id": "89 FR 97710 ¶3",
"label": "89 FR 97710 ¶3",
"kind": "rule",
"snippet": "G0556",
"score": 0.0,
}
mock_retrieve.return_value = [src]
mock_cited.return_value = [cited]
mock_ev.return_value = ValuationEvidence(("G0556",), ("APCM",), (), ())
client = MockClient.return_value.__enter__.return_value
resp = client.stream.return_value.__enter__.return_value
resp.iter_lines.return_value = iter(['{"message":{"content":"x"},"done":true}'])
events = list(stream_answer("APCM?", cfg=CFG, pool=self._pool()))
assert events[0]["type"] == "valuation" and events[0]["codes"] == ["G0556"]
assert events[1] == {"type": "token", "text": "x"}
assert events[-2]["sources"] == [src, cited]
mock_cited.assert_called_once_with(
mock_engine.return_value, ("G0556",), per_code=CFG.code_cited_per_code
)
body = client.stream.call_args.kwargs["json"]
assert "Valuation (authoritative" in body["messages"][1]["content"]