feat(llm): the CPT manual's heading guideline as a chat source; item_key/p_id on sources (refs #691 #705)
This commit is contained in:
@@ -25,6 +25,7 @@ from sqlalchemy import text
|
|||||||
|
|
||||||
from llm.config import LlmConfig
|
from llm.config import LlmConfig
|
||||||
from llm.links import as_source
|
from llm.links import as_source
|
||||||
|
from pfs.codetables import is_missing_table_error
|
||||||
from pfs.families import detect_codes, refresh_from
|
from pfs.families import detect_codes, refresh_from
|
||||||
from pfs.valuation import ValuationRow, valuation
|
from pfs.valuation import ValuationRow, valuation
|
||||||
|
|
||||||
@@ -390,6 +391,113 @@ def code_cited_sources(
|
|||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _family_notes(con: Any, family: str) -> list[str]:
|
||||||
|
"""Every distinct non-empty ``pfs.code_family.note`` (a CPT heading
|
||||||
|
``path_key``, C7) for *family* — a family may be classified under
|
||||||
|
more than one heading (e.g. CCM also has a "… > Complex Chronic
|
||||||
|
Care Management Services" note)."""
|
||||||
|
rows = con.execute(
|
||||||
|
"SELECT DISTINCT note FROM pfs.code_family "
|
||||||
|
"WHERE key = ? AND note IS NOT NULL AND note != '' ORDER BY note",
|
||||||
|
[family],
|
||||||
|
).fetchall()
|
||||||
|
return [r[0] for r in rows]
|
||||||
|
|
||||||
|
|
||||||
|
def _guideline_row(con: Any, path_key: str) -> tuple[int, str, str, str, str] | None:
|
||||||
|
"""``(edition_year, item_key, sec_id, title, guideline)`` for the
|
||||||
|
newest CPT edition of *path_key* with a non-empty guideline —
|
||||||
|
falling back once to the parent path (drop the last ``" > "``
|
||||||
|
segment) when the leaf heading has no guideline text of its own.
|
||||||
|
``None`` when neither has one."""
|
||||||
|
candidates = [path_key]
|
||||||
|
if " > " in path_key:
|
||||||
|
candidates.append(path_key.rsplit(" > ", 1)[0])
|
||||||
|
for key in candidates:
|
||||||
|
row = con.execute(
|
||||||
|
"SELECT edition_year, item_key, sec_id, title, guideline "
|
||||||
|
"FROM pfs.cpt_section WHERE path_key = ? "
|
||||||
|
"AND guideline IS NOT NULL AND guideline != '' "
|
||||||
|
"ORDER BY edition_year DESC, sec_id LIMIT 1",
|
||||||
|
[key],
|
||||||
|
).fetchone()
|
||||||
|
if row:
|
||||||
|
return row
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def manual_sources(
|
||||||
|
con: Any,
|
||||||
|
families: Sequence[str],
|
||||||
|
*,
|
||||||
|
per_family: int = 1,
|
||||||
|
snippet_chars: int = 900,
|
||||||
|
) -> list[dict]:
|
||||||
|
"""The AMA CPT manual's own guideline text for each detected
|
||||||
|
*family*'s CPT heading(s) — built straight from the DuckDB replica's
|
||||||
|
``pfs.code_family``/``pfs.cpt_section`` tables, not a pgvector query
|
||||||
|
(the CPT editions' chunk ``section`` metadata is the EPUB's raw
|
||||||
|
markdown headings, junk for this purpose — Ruling B4).
|
||||||
|
|
||||||
|
For each of *family*'s distinct ``pfs.code_family.note`` values (a
|
||||||
|
CPT heading ``path_key``), the newest edition with a non-empty
|
||||||
|
``guideline`` for that heading (one-level parent fallback,
|
||||||
|
``_guideline_row``) becomes one source via ``as_source``, labelled
|
||||||
|
``"CPT <year> — <heading title>"``; the source's ``date``/``url``
|
||||||
|
come from the bib item at the edition's ``item_key`` when it
|
||||||
|
resolves, else ``f"{year}-01-01"``/``""``. At most *per_family*
|
||||||
|
sources per family (newest-edition notes are not otherwise
|
||||||
|
ordered — the first *per_family* distinct notes, alphabetically,
|
||||||
|
are used). A family with no note, or whose heading(s) have no
|
||||||
|
guideline text anywhere, contributes nothing. *con* is a cursor on
|
||||||
|
the cached replica connection (the same per-call-cursor pattern as
|
||||||
|
``valuation_evidence``). Never raises — a missing
|
||||||
|
``pfs.code_family``/``pfs.cpt_section`` table degrades to ``[]``,
|
||||||
|
same as every other replica read in this module."""
|
||||||
|
out: list[dict] = []
|
||||||
|
for family in families:
|
||||||
|
try:
|
||||||
|
notes = _family_notes(con, family)
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
if not is_missing_table_error(e):
|
||||||
|
log.warning("manual sources skipped (%s notes): %s", family, e)
|
||||||
|
continue
|
||||||
|
kept = 0
|
||||||
|
for note in notes:
|
||||||
|
if kept >= per_family:
|
||||||
|
break
|
||||||
|
try:
|
||||||
|
row = _guideline_row(con, note)
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
if not is_missing_table_error(e):
|
||||||
|
log.warning("manual sources skipped (%s guideline): %s", family, e)
|
||||||
|
break # a broken cpt_section table won't clear up for the next note
|
||||||
|
if row is None:
|
||||||
|
continue
|
||||||
|
year, item_key, sec_id, title, guideline = row
|
||||||
|
date, url = f"{year}-01-01", ""
|
||||||
|
try:
|
||||||
|
item = _store().get(item_key)
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
log.warning("manual source item unresolved (%s): %s", item_key, e)
|
||||||
|
item = None
|
||||||
|
if item is not None:
|
||||||
|
date = item.date_published or date
|
||||||
|
url = item.url or ""
|
||||||
|
md = {
|
||||||
|
"kind": "corpus",
|
||||||
|
"item_key": item_key,
|
||||||
|
"seq": str(sec_id),
|
||||||
|
"section": title,
|
||||||
|
"title": f"CPT {year} — {title}",
|
||||||
|
"date": date,
|
||||||
|
"url": url,
|
||||||
|
}
|
||||||
|
out.append(as_source(md, guideline[:snippet_chars], 0.0))
|
||||||
|
kept += 1
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
def merge_sources(retrieved: list[dict], extra: list[dict]) -> list[dict]:
|
def merge_sources(retrieved: list[dict], extra: list[dict]) -> list[dict]:
|
||||||
labels = {s["label"] for s in retrieved}
|
labels = {s["label"] for s in retrieved}
|
||||||
return retrieved + [s for s in extra if s["label"] not in labels]
|
return retrieved + [s for s in extra if s["label"] not in labels]
|
||||||
@@ -399,4 +507,6 @@ def merge_sources(retrieved: list[dict], extra: list[dict]) -> list[dict]:
|
|||||||
# like valuation_evidence — llm.lineage lazily imports this module's
|
# like valuation_evidence — llm.lineage lazily imports this module's
|
||||||
# _connect/_replica_path/_mtime/warm inside lineage_evidence() rather
|
# _connect/_replica_path/_mtime/warm inside lineage_evidence() rather
|
||||||
# than at module scope, so this import carries no circularity risk.
|
# than at module scope, so this import carries no circularity risk.
|
||||||
from llm.lineage import lineage_evidence # noqa: E402,F401
|
# _store is used by manual_sources, above, to resolve a CPT edition's
|
||||||
|
# bib item (date_published/url) — imported here for the same reason.
|
||||||
|
from llm.lineage import _store, lineage_evidence # noqa: E402,F401
|
||||||
|
|||||||
@@ -88,13 +88,21 @@ def for_source(md: dict[str, str], snippet: str) -> tuple[str, str]:
|
|||||||
|
|
||||||
|
|
||||||
def as_source(md: dict[str, str], text: str, score: float) -> dict:
|
def as_source(md: dict[str, str], text: str, score: float) -> dict:
|
||||||
"""The source dict the chat sends to the prompt and the UI."""
|
"""The source dict the chat sends to the prompt and the UI.
|
||||||
|
|
||||||
|
``item_key``, ``p_id`` (rules only; ``""`` otherwise), ``seq``
|
||||||
|
(non-rule kinds only; ``""`` for rules) and ``section`` are carried
|
||||||
|
straight from *md* so the golden evaluation can match a source back
|
||||||
|
to its chunk anchor: ``(item_key, p_id)`` for a rule paragraph,
|
||||||
|
``(item_key, seq)`` for a comment/corpus chunk.
|
||||||
|
"""
|
||||||
snippet = text[:_SNIPPET_CHARS].strip()
|
snippet = text[:_SNIPPET_CHARS].strip()
|
||||||
url, label = for_source(md, snippet)
|
url, label = for_source(md, snippet)
|
||||||
|
kind = md.get("kind", "")
|
||||||
return {
|
return {
|
||||||
"id": label,
|
"id": label,
|
||||||
"label": label,
|
"label": label,
|
||||||
"kind": md.get("kind", ""),
|
"kind": kind,
|
||||||
"url": url,
|
"url": url,
|
||||||
"title": md.get("title", ""),
|
"title": md.get("title", ""),
|
||||||
"date": md.get("date", ""),
|
"date": md.get("date", ""),
|
||||||
@@ -102,4 +110,8 @@ def as_source(md: dict[str, str], text: str, score: float) -> dict:
|
|||||||
"comment_id": md.get("comment_id", ""),
|
"comment_id": md.get("comment_id", ""),
|
||||||
"snippet": snippet,
|
"snippet": snippet,
|
||||||
"score": round(score, 4),
|
"score": round(score, 4),
|
||||||
|
"item_key": md.get("item_key", ""),
|
||||||
|
"p_id": md.get("p_id", "") if kind == "rule" else "",
|
||||||
|
"seq": md.get("seq", "") if kind != "rule" else "",
|
||||||
|
"section": md.get("section", ""),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,20 +23,24 @@ SSE event order is ``lineage`` (only when detected) → ``valuation``
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import threading
|
import threading
|
||||||
from dataclasses import replace
|
from dataclasses import replace
|
||||||
from datetime import date
|
from datetime import date
|
||||||
from typing import Iterator
|
from typing import Iterator, Sequence
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
from llm.config import LlmConfig
|
from llm.config import LlmConfig
|
||||||
from llm.evidence import (
|
from llm.evidence import (
|
||||||
ValuationEvidence,
|
ValuationEvidence,
|
||||||
|
_connect,
|
||||||
|
_replica_path,
|
||||||
code_cited_sources,
|
code_cited_sources,
|
||||||
lineage_evidence,
|
lineage_evidence,
|
||||||
|
manual_sources,
|
||||||
merge_sources,
|
merge_sources,
|
||||||
valuation_evidence,
|
valuation_evidence,
|
||||||
)
|
)
|
||||||
@@ -47,6 +51,9 @@ from llm.links import as_source
|
|||||||
from llm.pool import HostPool, PoolEmbeddings, pick_model
|
from llm.pool import HostPool, PoolEmbeddings, pick_model
|
||||||
from llm.rerank import Hit, _similarity, blend, filter_since
|
from llm.rerank import Hit, _similarity, blend, filter_since
|
||||||
from pfs.descriptors import rule_year_of
|
from pfs.descriptors import rule_year_of
|
||||||
|
from pfs.families import detect_codes
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
_TIMEOUT = httpx.Timeout(300.0, connect=5.0)
|
_TIMEOUT = httpx.Timeout(300.0, connect=5.0)
|
||||||
_COLLECTIONS = {"comment": "comments", "rule": "rules", "corpus": "corpus"}
|
_COLLECTIONS = {"comment": "comments", "rule": "rules", "corpus": "corpus"}
|
||||||
@@ -328,6 +335,27 @@ def build_messages(
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _manual_sources(families: Sequence[str], cfg: LlmConfig) -> list[dict]:
|
||||||
|
"""Cursor-per-call wrapper around ``evidence.manual_sources`` — the
|
||||||
|
same cached-replica-cursor pattern ``valuation_evidence`` uses
|
||||||
|
(``llm.evidence._connect``/``_replica_path``), so a failed or
|
||||||
|
missing replica just yields no manual sources rather than breaking
|
||||||
|
the chat. Skipped entirely (no replica touch) when *families* is
|
||||||
|
empty."""
|
||||||
|
if not families:
|
||||||
|
return []
|
||||||
|
path = _replica_path(cfg)
|
||||||
|
try:
|
||||||
|
cur = _connect(path).cursor()
|
||||||
|
except Exception as e: # noqa: BLE001 — duckdb raises several types
|
||||||
|
log.warning("manual sources skipped (%s): %s", path, e)
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
return manual_sources(cur, families)
|
||||||
|
finally:
|
||||||
|
cur.close() # the cached parent connection stays open
|
||||||
|
|
||||||
|
|
||||||
def stream_answer(
|
def stream_answer(
|
||||||
question: str,
|
question: str,
|
||||||
*,
|
*,
|
||||||
@@ -346,8 +374,12 @@ def stream_answer(
|
|||||||
(evidence's ``payload()``, the dated timeline) then a ``valuation``
|
(evidence's ``payload()``, the dated timeline) then a ``valuation``
|
||||||
event (RVUs and payment) — both before any tokens — and merges
|
event (RVUs and payment) — both before any tokens — and merges
|
||||||
chunks from rules, comments and the corpus that cite those codes —
|
chunks from rules, comments and the corpus that cite those codes —
|
||||||
or, for a detected family, its family key — into ``sources``. Then
|
or, for a detected family, its family key — into ``sources``, then
|
||||||
yields ``{"type":"token","text":…}`` events as the model generates,
|
(for any non-wide detected family) the AMA CPT manual's own
|
||||||
|
guideline text for that family's heading (``llm.evidence.
|
||||||
|
manual_sources`` — deterministic, not a retrieval hit, so it never
|
||||||
|
counts toward ``code_cited_max``). Then yields
|
||||||
|
``{"type":"token","text":…}`` events as the model generates,
|
||||||
then one ``{"type":"sources", "sources": […], "model": …, "host":
|
then one ``{"type":"sources", "sources": […], "model": …, "host":
|
||||||
…, "mode": "timeline"|"recent"}`` (the resolved mode) and a final
|
…, "mode": "timeline"|"recent"}`` (the resolved mode) and a final
|
||||||
``{"type":"done"}``.
|
``{"type":"done"}``.
|
||||||
@@ -371,6 +403,12 @@ def stream_answer(
|
|||||||
max_total=cfg.code_cited_max,
|
max_total=cfg.code_cited_max,
|
||||||
)
|
)
|
||||||
sources = merge_sources(sources, cited)
|
sources = merge_sources(sources, cited)
|
||||||
|
# The CPT manual's own heading text for a detected family — a
|
||||||
|
# wide family (its code list too big to expand, MAX_FAMILY_EXPAND)
|
||||||
|
# is excluded, and these rows never count toward code_cited_max.
|
||||||
|
wide = set(detect_codes(question).wide)
|
||||||
|
narrow_families = tuple(f for f in families if f not in wide)
|
||||||
|
sources = merge_sources(sources, _manual_sources(narrow_families, cfg))
|
||||||
if lineage is not None:
|
if lineage is not None:
|
||||||
yield lineage.payload()
|
yield lineage.payload()
|
||||||
if evidence is not None:
|
if evidence is not None:
|
||||||
|
|||||||
@@ -15,10 +15,12 @@ from llm.config import LlmConfig
|
|||||||
from llm.evidence import (
|
from llm.evidence import (
|
||||||
ValuationEvidence,
|
ValuationEvidence,
|
||||||
code_cited_sources,
|
code_cited_sources,
|
||||||
|
manual_sources,
|
||||||
merge_sources,
|
merge_sources,
|
||||||
valuation_evidence,
|
valuation_evidence,
|
||||||
warm,
|
warm,
|
||||||
)
|
)
|
||||||
|
from pfs.codetables import ensure_tables
|
||||||
from pfs.valuation import ValuationRow
|
from pfs.valuation import ValuationRow
|
||||||
|
|
||||||
CFG = LlmConfig(
|
CFG = LlmConfig(
|
||||||
@@ -757,3 +759,166 @@ class TestMergeSources:
|
|||||||
b = [{"label": "Y", "score": 0.0}, {"label": "Z", "score": 0.0}]
|
b = [{"label": "Y", "score": 0.0}, {"label": "Z", "score": 0.0}]
|
||||||
assert [s["label"] for s in merge_sources(a, b)] == ["X", "Y", "Z"]
|
assert [s["label"] for s in merge_sources(a, b)] == ["X", "Y", "Z"]
|
||||||
assert merge_sources(a, b)[1]["score"] == 0.8
|
assert merge_sources(a, b)[1]["score"] == 0.8
|
||||||
|
|
||||||
|
|
||||||
|
def _cpt_section_row(con, year, item_key, sec_id, title, path_key, guideline):
|
||||||
|
con.execute(
|
||||||
|
"INSERT INTO pfs.cpt_section VALUES (?,?,?,?,?,?,?,?,?,?)",
|
||||||
|
[
|
||||||
|
year,
|
||||||
|
item_key,
|
||||||
|
sec_id,
|
||||||
|
3,
|
||||||
|
title,
|
||||||
|
path_key.split(" > "),
|
||||||
|
path_key,
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
guideline,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _family_row(con, key, note, code="99490"):
|
||||||
|
con.execute(
|
||||||
|
"INSERT INTO pfs.code_family VALUES (?,?,?,?,?,?,?,?,?)",
|
||||||
|
[key, key, code, "member", None, None, "", 0, note],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
CCM_PATH = "Evaluation and Management > Care Management Services > Chronic Care Management Services"
|
||||||
|
PARENT_PATH = "Surgery > General"
|
||||||
|
LEAF_PATH = "Surgery > General > Leaf With No Guideline"
|
||||||
|
|
||||||
|
|
||||||
|
class TestManualSources:
|
||||||
|
"""``manual_sources`` — the CPT manual's own guideline text for a
|
||||||
|
detected family's heading, straight from the DuckDB replica (#691
|
||||||
|
Ruling B4: never a pgvector query — the CPT chunks' ``section``
|
||||||
|
metadata is junk EPUB headings, not the family heading)."""
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def cpt_replica(self):
|
||||||
|
con = duckdb.connect(":memory:")
|
||||||
|
ensure_tables(con)
|
||||||
|
# CCM: two editions have the heading's guideline text; the newer
|
||||||
|
# (2024) one must win over the older (2022) one.
|
||||||
|
_family_row(con, "CCM", CCM_PATH)
|
||||||
|
_cpt_section_row(
|
||||||
|
con,
|
||||||
|
2022,
|
||||||
|
"OLDED001",
|
||||||
|
"S0",
|
||||||
|
"Chronic Care Management Services",
|
||||||
|
CCM_PATH,
|
||||||
|
"2022 text: non-face-to-face care management services.",
|
||||||
|
)
|
||||||
|
_cpt_section_row(
|
||||||
|
con,
|
||||||
|
2024,
|
||||||
|
"GQGTPGYV",
|
||||||
|
"S1",
|
||||||
|
"Chronic Care Management Services",
|
||||||
|
CCM_PATH,
|
||||||
|
"Chronic care management services are non-face-to-face "
|
||||||
|
"services provided to a patient with two or more chronic conditions.",
|
||||||
|
)
|
||||||
|
# PARENTONLY: the leaf heading has no guideline of its own, but
|
||||||
|
# its one-level parent does — the fallback must pick it up.
|
||||||
|
_family_row(con, "PARENTONLY", LEAF_PATH)
|
||||||
|
_cpt_section_row(
|
||||||
|
con, 2024, "LEAFED001", "S2", "Leaf With No Guideline", LEAF_PATH, ""
|
||||||
|
)
|
||||||
|
_cpt_section_row(
|
||||||
|
con,
|
||||||
|
2023,
|
||||||
|
"PARED0001",
|
||||||
|
"S3",
|
||||||
|
"General",
|
||||||
|
PARENT_PATH,
|
||||||
|
"General guidance text for the parent heading.",
|
||||||
|
)
|
||||||
|
# NONOTE: a real family row, but its note is empty.
|
||||||
|
_family_row(con, "NONOTE", "", code="99999")
|
||||||
|
yield con
|
||||||
|
con.close()
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _fake_store(self):
|
||||||
|
items = {
|
||||||
|
"GQGTPGYV": SimpleNamespace(
|
||||||
|
date_published="2023-11-16",
|
||||||
|
url="https://ebooks.ama-assn.org/cpt2024",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
def _get(key):
|
||||||
|
if key in items:
|
||||||
|
return items[key]
|
||||||
|
raise KeyError(key)
|
||||||
|
|
||||||
|
store = MagicMock()
|
||||||
|
store.get.side_effect = _get
|
||||||
|
with patch("llm.evidence._store", return_value=store):
|
||||||
|
yield store
|
||||||
|
|
||||||
|
def test_newest_edition_with_guideline_wins(self, cpt_replica):
|
||||||
|
out = manual_sources(cpt_replica, ["CCM"])
|
||||||
|
assert len(out) == 1
|
||||||
|
s = out[0]
|
||||||
|
assert s["kind"] == "corpus"
|
||||||
|
assert s["title"] == "CPT 2024 — Chronic Care Management Services"
|
||||||
|
assert s["item_key"] == "GQGTPGYV" and s["seq"] == "S1"
|
||||||
|
assert s["section"] == "Chronic Care Management Services"
|
||||||
|
assert s["date"] == "2023-11-16"
|
||||||
|
assert s["url"] == "https://ebooks.ama-assn.org/cpt2024"
|
||||||
|
assert s["snippet"].startswith("Chronic care management services")
|
||||||
|
|
||||||
|
def test_parent_fallback_when_leaf_has_no_guideline(self, cpt_replica):
|
||||||
|
out = manual_sources(cpt_replica, ["PARENTONLY"])
|
||||||
|
assert len(out) == 1
|
||||||
|
s = out[0]
|
||||||
|
assert s["title"] == "CPT 2023 — General"
|
||||||
|
assert s["item_key"] == "PARED0001" and s["seq"] == "S3"
|
||||||
|
assert "General guidance text" in s["snippet"]
|
||||||
|
# unresolved bib item (not in the fake store) — falls back to
|
||||||
|
# the plain edition year and an empty url.
|
||||||
|
assert s["date"] == "2023-01-01" and s["url"] == ""
|
||||||
|
|
||||||
|
def test_family_with_empty_note_yields_nothing(self, cpt_replica):
|
||||||
|
assert manual_sources(cpt_replica, ["NONOTE"]) == []
|
||||||
|
|
||||||
|
def test_family_with_no_family_row_yields_nothing(self, cpt_replica):
|
||||||
|
assert manual_sources(cpt_replica, ["GHOST"]) == []
|
||||||
|
|
||||||
|
def test_no_families_yields_nothing_without_querying(self, cpt_replica):
|
||||||
|
assert manual_sources(cpt_replica, []) == []
|
||||||
|
|
||||||
|
def test_per_family_cap(self, cpt_replica):
|
||||||
|
complex_path = (
|
||||||
|
"Evaluation and Management > Care Management Services > "
|
||||||
|
"Complex Chronic Care Management Services"
|
||||||
|
)
|
||||||
|
_family_row(cpt_replica, "CCM", complex_path, code="99487")
|
||||||
|
_cpt_section_row(
|
||||||
|
cpt_replica,
|
||||||
|
2024,
|
||||||
|
"GQGTPGYV",
|
||||||
|
"S4",
|
||||||
|
"Complex Chronic Care Management Services",
|
||||||
|
complex_path,
|
||||||
|
"Complex chronic care management guideline text.",
|
||||||
|
)
|
||||||
|
assert len(manual_sources(cpt_replica, ["CCM"], per_family=1)) == 1
|
||||||
|
assert len(manual_sources(cpt_replica, ["CCM"], per_family=2)) == 2
|
||||||
|
|
||||||
|
def test_snippet_chars_bounds_the_text_passed_to_as_source(self, cpt_replica):
|
||||||
|
out = manual_sources(cpt_replica, ["CCM"], snippet_chars=10)
|
||||||
|
assert len(out[0]["snippet"]) <= 10
|
||||||
|
|
||||||
|
def test_missing_tables_yield_empty(self):
|
||||||
|
con = duckdb.connect(":memory:")
|
||||||
|
try:
|
||||||
|
assert manual_sources(con, ["CCM"]) == []
|
||||||
|
finally:
|
||||||
|
con.close()
|
||||||
|
|||||||
@@ -140,6 +140,9 @@ class TestAsSource:
|
|||||||
"date": "2026-08-19",
|
"date": "2026-08-19",
|
||||||
"title": "t",
|
"title": "t",
|
||||||
"docket": "CMS-2026-2377",
|
"docket": "CMS-2026-2377",
|
||||||
|
"item_key": "ABCD1234",
|
||||||
|
"seq": "3",
|
||||||
|
"section": "Some Heading",
|
||||||
},
|
},
|
||||||
"body " * 200,
|
"body " * 200,
|
||||||
0.123456,
|
0.123456,
|
||||||
@@ -155,6 +158,27 @@ class TestAsSource:
|
|||||||
"comment_id",
|
"comment_id",
|
||||||
"snippet",
|
"snippet",
|
||||||
"score",
|
"score",
|
||||||
|
"item_key",
|
||||||
|
"p_id",
|
||||||
|
"seq",
|
||||||
|
"section",
|
||||||
}
|
}
|
||||||
assert s["label"] == "CMS-2026-2377-1" and s["kind"] == "comment"
|
assert s["label"] == "CMS-2026-2377-1" and s["kind"] == "comment"
|
||||||
assert len(s["snippet"]) <= 500 and s["score"] == 0.1235
|
assert len(s["snippet"]) <= 500 and s["score"] == 0.1235
|
||||||
|
# non-rule kind: seq/section carried through, p_id blanked
|
||||||
|
assert s["item_key"] == "ABCD1234" and s["seq"] == "3"
|
||||||
|
assert s["section"] == "Some Heading" and s["p_id"] == ""
|
||||||
|
|
||||||
|
def test_rule_kind_carries_p_id_not_seq(self):
|
||||||
|
from llm.links import as_source
|
||||||
|
|
||||||
|
s = as_source({**TestRule.MD, "item_key": "WXYZ5678", "seq": "9"}, "text", 0.0)
|
||||||
|
assert s["item_key"] == "WXYZ5678"
|
||||||
|
assert s["p_id"] == "938" and s["seq"] == ""
|
||||||
|
|
||||||
|
def test_missing_keys_default_to_empty(self):
|
||||||
|
from llm.links import as_source
|
||||||
|
|
||||||
|
s = as_source({"kind": "corpus"}, "text", 0.0)
|
||||||
|
assert s["item_key"] == "" and s["p_id"] == ""
|
||||||
|
assert s["seq"] == "" and s["section"] == ""
|
||||||
|
|||||||
@@ -699,6 +699,7 @@ class TestStreamAnswer:
|
|||||||
with pytest.raises(RuntimeError, match="ollama down"):
|
with pytest.raises(RuntimeError, match="ollama down"):
|
||||||
list(stream_answer("q", cfg=CFG, pool=self._pool()))
|
list(stream_answer("q", cfg=CFG, pool=self._pool()))
|
||||||
|
|
||||||
|
@patch("llm.rag._manual_sources", return_value=[])
|
||||||
@patch("llm.rag._engine")
|
@patch("llm.rag._engine")
|
||||||
@patch("llm.rag.code_cited_sources")
|
@patch("llm.rag.code_cited_sources")
|
||||||
@patch("llm.rag.valuation_evidence")
|
@patch("llm.rag.valuation_evidence")
|
||||||
@@ -706,7 +707,14 @@ class TestStreamAnswer:
|
|||||||
@patch("llm.rag.httpx.Client")
|
@patch("llm.rag.httpx.Client")
|
||||||
@patch("llm.rag.retrieve")
|
@patch("llm.rag.retrieve")
|
||||||
def test_valuation_event_first_and_sources_merged(
|
def test_valuation_event_first_and_sources_merged(
|
||||||
self, mock_retrieve, MockClient, _lin, mock_ev, mock_cited, mock_engine
|
self,
|
||||||
|
mock_retrieve,
|
||||||
|
MockClient,
|
||||||
|
_lin,
|
||||||
|
mock_ev,
|
||||||
|
mock_cited,
|
||||||
|
mock_engine,
|
||||||
|
_manual,
|
||||||
):
|
):
|
||||||
from llm.evidence import ValuationEvidence
|
from llm.evidence import ValuationEvidence
|
||||||
|
|
||||||
@@ -747,6 +755,7 @@ class TestStreamAnswer:
|
|||||||
body = client.stream.call_args.kwargs["json"]
|
body = client.stream.call_args.kwargs["json"]
|
||||||
assert "Valuation (authoritative" in body["messages"][1]["content"]
|
assert "Valuation (authoritative" in body["messages"][1]["content"]
|
||||||
|
|
||||||
|
@patch("llm.rag._manual_sources", return_value=[])
|
||||||
@patch("llm.rag._engine")
|
@patch("llm.rag._engine")
|
||||||
@patch("llm.rag.code_cited_sources")
|
@patch("llm.rag.code_cited_sources")
|
||||||
@patch("llm.rag.valuation_evidence")
|
@patch("llm.rag.valuation_evidence")
|
||||||
@@ -754,7 +763,14 @@ class TestStreamAnswer:
|
|||||||
@patch("llm.rag.httpx.Client")
|
@patch("llm.rag.httpx.Client")
|
||||||
@patch("llm.rag.retrieve")
|
@patch("llm.rag.retrieve")
|
||||||
def test_lineage_event_before_valuation_and_tokens(
|
def test_lineage_event_before_valuation_and_tokens(
|
||||||
self, mock_retrieve, MockClient, mock_lin, mock_ev, mock_cited, mock_engine
|
self,
|
||||||
|
mock_retrieve,
|
||||||
|
MockClient,
|
||||||
|
mock_lin,
|
||||||
|
mock_ev,
|
||||||
|
mock_cited,
|
||||||
|
mock_engine,
|
||||||
|
_manual,
|
||||||
):
|
):
|
||||||
from llm.evidence import ValuationEvidence
|
from llm.evidence import ValuationEvidence
|
||||||
|
|
||||||
@@ -804,6 +820,7 @@ class TestStreamAnswer:
|
|||||||
"Valuation (authoritative"
|
"Valuation (authoritative"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@patch("llm.rag._manual_sources", return_value=[])
|
||||||
@patch("llm.rag._engine")
|
@patch("llm.rag._engine")
|
||||||
@patch("llm.rag.code_cited_sources")
|
@patch("llm.rag.code_cited_sources")
|
||||||
@patch("llm.rag.valuation_evidence", return_value=None)
|
@patch("llm.rag.valuation_evidence", return_value=None)
|
||||||
@@ -811,7 +828,7 @@ class TestStreamAnswer:
|
|||||||
@patch("llm.rag.httpx.Client")
|
@patch("llm.rag.httpx.Client")
|
||||||
@patch("llm.rag.retrieve")
|
@patch("llm.rag.retrieve")
|
||||||
def test_lineage_only_calls_cited_sources_with_lineage_codes(
|
def test_lineage_only_calls_cited_sources_with_lineage_codes(
|
||||||
self, mock_retrieve, MockClient, mock_lin, _ev, mock_cited, mock_engine
|
self, mock_retrieve, MockClient, mock_lin, _ev, mock_cited, mock_engine, _manual
|
||||||
):
|
):
|
||||||
mock_retrieve.return_value = []
|
mock_retrieve.return_value = []
|
||||||
mock_cited.return_value = []
|
mock_cited.return_value = []
|
||||||
@@ -838,6 +855,7 @@ class TestStreamAnswer:
|
|||||||
max_total=CFG.code_cited_max,
|
max_total=CFG.code_cited_max,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@patch("llm.rag._manual_sources", return_value=[])
|
||||||
@patch("llm.rag._engine")
|
@patch("llm.rag._engine")
|
||||||
@patch("llm.rag.code_cited_sources")
|
@patch("llm.rag.code_cited_sources")
|
||||||
@patch("llm.rag.valuation_evidence")
|
@patch("llm.rag.valuation_evidence")
|
||||||
@@ -845,7 +863,14 @@ class TestStreamAnswer:
|
|||||||
@patch("llm.rag.httpx.Client")
|
@patch("llm.rag.httpx.Client")
|
||||||
@patch("llm.rag.retrieve")
|
@patch("llm.rag.retrieve")
|
||||||
def test_codes_and_families_unioned_when_both_present(
|
def test_codes_and_families_unioned_when_both_present(
|
||||||
self, mock_retrieve, MockClient, mock_lin, mock_ev, mock_cited, mock_engine
|
self,
|
||||||
|
mock_retrieve,
|
||||||
|
MockClient,
|
||||||
|
mock_lin,
|
||||||
|
mock_ev,
|
||||||
|
mock_cited,
|
||||||
|
mock_engine,
|
||||||
|
_manual,
|
||||||
):
|
):
|
||||||
from llm.evidence import ValuationEvidence
|
from llm.evidence import ValuationEvidence
|
||||||
|
|
||||||
@@ -875,6 +900,118 @@ class TestStreamAnswer:
|
|||||||
max_total=CFG.code_cited_max,
|
max_total=CFG.code_cited_max,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@patch("llm.rag._manual_sources")
|
||||||
|
@patch("llm.rag._engine")
|
||||||
|
@patch("llm.rag.code_cited_sources")
|
||||||
|
@patch("llm.rag.valuation_evidence")
|
||||||
|
@patch("llm.rag.lineage_evidence", return_value=None)
|
||||||
|
@patch("llm.rag.httpx.Client")
|
||||||
|
@patch("llm.rag.retrieve")
|
||||||
|
def test_manual_sources_merged_after_cited_sources(
|
||||||
|
self,
|
||||||
|
mock_retrieve,
|
||||||
|
MockClient,
|
||||||
|
_lin,
|
||||||
|
mock_ev,
|
||||||
|
mock_cited,
|
||||||
|
mock_engine,
|
||||||
|
mock_manual,
|
||||||
|
):
|
||||||
|
"""#691/#705: the CPT manual's guideline source is a third,
|
||||||
|
deterministic layer — merged after the retrieved and code-cited
|
||||||
|
sources, never before them."""
|
||||||
|
from llm.evidence import ValuationEvidence
|
||||||
|
|
||||||
|
src = {
|
||||||
|
"id": "C1",
|
||||||
|
"label": "C1",
|
||||||
|
"kind": "comment",
|
||||||
|
"snippet": "s",
|
||||||
|
"score": 0.1,
|
||||||
|
}
|
||||||
|
cited = {
|
||||||
|
"id": "89 FR 97710 ¶3",
|
||||||
|
"label": "89 FR 97710 ¶3",
|
||||||
|
"kind": "rule",
|
||||||
|
"snippet": "G0556",
|
||||||
|
"score": 0.0,
|
||||||
|
}
|
||||||
|
manual = {
|
||||||
|
"id": "CPT 2024 — Advanced Primary Care Management",
|
||||||
|
"label": "CPT 2024 — Advanced Primary Care Management",
|
||||||
|
"kind": "corpus",
|
||||||
|
"snippet": "guideline text",
|
||||||
|
"score": 0.0,
|
||||||
|
}
|
||||||
|
mock_retrieve.return_value = [src]
|
||||||
|
mock_cited.return_value = [cited]
|
||||||
|
mock_manual.return_value = [manual]
|
||||||
|
mock_ev.return_value = ValuationEvidence(("G0556",), ("APCM",), (), ())
|
||||||
|
client = MockClient.return_value.__enter__.return_value
|
||||||
|
resp = client.stream.return_value.__enter__.return_value
|
||||||
|
resp.iter_lines.return_value = iter(['{"message":{"content":"x"},"done":true}'])
|
||||||
|
|
||||||
|
events = list(stream_answer("APCM?", cfg=CFG, pool=self._pool()))
|
||||||
|
|
||||||
|
assert events[-2]["sources"] == [src, cited, manual]
|
||||||
|
mock_manual.assert_called_once_with(("APCM",), CFG)
|
||||||
|
|
||||||
|
@patch("llm.rag.detect_codes")
|
||||||
|
@patch("llm.rag._manual_sources", return_value=[])
|
||||||
|
@patch("llm.rag._engine")
|
||||||
|
@patch("llm.rag.code_cited_sources")
|
||||||
|
@patch("llm.rag.valuation_evidence")
|
||||||
|
@patch("llm.rag.lineage_evidence", return_value=None)
|
||||||
|
@patch("llm.rag.httpx.Client")
|
||||||
|
@patch("llm.rag.retrieve")
|
||||||
|
def test_manual_sources_skips_wide_families(
|
||||||
|
self,
|
||||||
|
mock_retrieve,
|
||||||
|
MockClient,
|
||||||
|
_lin,
|
||||||
|
mock_ev,
|
||||||
|
mock_cited,
|
||||||
|
mock_engine,
|
||||||
|
mock_manual,
|
||||||
|
mock_detect,
|
||||||
|
):
|
||||||
|
from llm.evidence import ValuationEvidence
|
||||||
|
from pfs.families import Detection
|
||||||
|
|
||||||
|
mock_retrieve.return_value = []
|
||||||
|
mock_cited.return_value = []
|
||||||
|
mock_detect.return_value = Detection(
|
||||||
|
codes=(), families=("BIGFAM", "APCM"), explicit=(), wide=("BIGFAM",)
|
||||||
|
)
|
||||||
|
mock_ev.return_value = ValuationEvidence((), ("BIGFAM", "APCM"), (), ())
|
||||||
|
client = MockClient.return_value.__enter__.return_value
|
||||||
|
resp = client.stream.return_value.__enter__.return_value
|
||||||
|
resp.iter_lines.return_value = iter(['{"message":{"content":"x"},"done":true}'])
|
||||||
|
|
||||||
|
list(stream_answer("q", cfg=CFG, pool=self._pool()))
|
||||||
|
|
||||||
|
mock_manual.assert_called_once_with(("APCM",), CFG)
|
||||||
|
|
||||||
|
@patch("llm.rag._manual_sources")
|
||||||
|
@patch("llm.rag._engine")
|
||||||
|
@patch("llm.rag.code_cited_sources")
|
||||||
|
@patch("llm.rag.valuation_evidence", return_value=None)
|
||||||
|
@patch("llm.rag.lineage_evidence", return_value=None)
|
||||||
|
@patch("llm.rag.httpx.Client")
|
||||||
|
@patch("llm.rag.retrieve")
|
||||||
|
def test_no_evidence_no_lineage_never_calls_manual_sources(
|
||||||
|
self, mock_retrieve, MockClient, _lin, _ev, mock_cited, mock_engine, mock_manual
|
||||||
|
):
|
||||||
|
mock_retrieve.return_value = []
|
||||||
|
client = MockClient.return_value.__enter__.return_value
|
||||||
|
resp = client.stream.return_value.__enter__.return_value
|
||||||
|
resp.iter_lines.return_value = iter(['{"message":{"content":"x"},"done":true}'])
|
||||||
|
|
||||||
|
list(stream_answer("q", cfg=CFG, pool=self._pool()))
|
||||||
|
|
||||||
|
mock_cited.assert_not_called()
|
||||||
|
mock_manual.assert_not_called()
|
||||||
|
|
||||||
@patch("llm.rag.valuation_evidence", return_value=None)
|
@patch("llm.rag.valuation_evidence", return_value=None)
|
||||||
@patch("llm.rag.httpx.Client")
|
@patch("llm.rag.httpx.Client")
|
||||||
@patch("llm.rag.retrieve")
|
@patch("llm.rag.retrieve")
|
||||||
@@ -904,3 +1041,36 @@ class TestStreamAnswer:
|
|||||||
with patch("llm.rag.lineage_evidence", return_value=None):
|
with patch("llm.rag.lineage_evidence", return_value=None):
|
||||||
with_lineage_off = json.dumps(_run())
|
with_lineage_off = json.dumps(_run())
|
||||||
assert with_real_lineage == with_lineage_off
|
assert with_real_lineage == with_lineage_off
|
||||||
|
|
||||||
|
|
||||||
|
class TestManualSourcesWrapper:
|
||||||
|
"""``_manual_sources`` — the cursor-per-call wrapper around
|
||||||
|
``evidence.manual_sources`` (mirrors ``valuation_evidence``'s own
|
||||||
|
cached-replica-cursor pattern)."""
|
||||||
|
|
||||||
|
def test_empty_families_short_circuits_without_touching_the_replica(self):
|
||||||
|
with patch("llm.rag._connect") as mock_connect:
|
||||||
|
assert rag._manual_sources((), CFG) == []
|
||||||
|
mock_connect.assert_not_called()
|
||||||
|
|
||||||
|
@patch("llm.rag.manual_sources")
|
||||||
|
@patch("llm.rag._connect")
|
||||||
|
@patch("llm.rag._replica_path", return_value="/x/aco.ro.duckdb")
|
||||||
|
def test_opens_a_cursor_calls_through_and_closes_it(
|
||||||
|
self, _path, mock_connect, mock_manual
|
||||||
|
):
|
||||||
|
cur = MagicMock()
|
||||||
|
mock_connect.return_value.cursor.return_value = cur
|
||||||
|
mock_manual.return_value = [{"label": "CPT 2024 — X"}]
|
||||||
|
|
||||||
|
out = rag._manual_sources(("CCM",), CFG)
|
||||||
|
|
||||||
|
assert out == [{"label": "CPT 2024 — X"}]
|
||||||
|
mock_connect.assert_called_once_with("/x/aco.ro.duckdb")
|
||||||
|
mock_manual.assert_called_once_with(cur, ("CCM",))
|
||||||
|
cur.close.assert_called_once() # the cached parent connection stays open
|
||||||
|
|
||||||
|
@patch("llm.rag._connect", side_effect=OSError("no replica"))
|
||||||
|
def test_unopenable_replica_yields_empty(self, _connect, caplog):
|
||||||
|
assert rag._manual_sources(("CCM",), CFG) == []
|
||||||
|
assert "manual sources skipped" in caplog.text
|
||||||
|
|||||||
Reference in New Issue
Block a user