Files
stack/tests/llm/test_dossier.py

630 lines
22 KiB
Python

"""llm.dossier — the family dossier as chat evidence (Task 15)."""
from __future__ import annotations
import dataclasses
import logging
import os
from dataclasses import replace
from pathlib import Path
import duckdb
import pytest
from llm import evidence
from llm.config import LlmConfig
from llm.dossier import (
TIMELINE_FLOOR,
DossierEvidence,
dossier_evidence,
dossier_url,
match_facts,
question_years,
)
from pfs.cite import Cite
from pfs.codetables import ensure_tables
from pfs.dossier import Dossier, Fact, TimelineEvent, write_dossier
CFG = LlmConfig(
ollama_hosts=("http://h1:11434",),
embed_model="e",
instruct_model="c",
embed_dim=768,
build_ann_index=False,
pg_host="x",
pg_port=5432,
pg_db="llm",
pg_user="llm",
duckdb_replica="/nonexistent/aco.ro.duckdb",
)
ACC = (
"Preliminary model classification, measured on a held-out set the prompt "
"was never tuned on (20 hand-coded comments, 24 positions; CMS-2025-0304): "
"it finds 79% of hand-coded positions, and the direction is right for 74% "
"of those it finds."
)
def _fact(i, kind, text, year=2023, label=None, note="", codes=("99341",), values=None):
return Fact(
id=f"F{i}",
kind=kind,
year=year,
codes=codes,
text=text,
values=values or {},
cite=Cite("fr", label or f"L{i}", f"https://x/{i}"),
note=note,
)
def _dossier(extra=(), accuracy=ACC):
old = ("99324", "99325")
facts = [
_fact(1, "payment", "payment row"),
_fact(2, "round", "CMS finalized work RVU 1.0 for 99341 for CY2023."),
_fact(
3,
"event",
"99324 was deleted for CY2023.",
label="DEL23",
codes=old,
),
_fact(
4,
"stance_split",
"Of 3 CY2023 commenters we classified, 2 supported and 1 opposed. " + ACC,
values={
"docket": "CMS-2022-0113",
"recall": 0.792,
"direction": 0.737,
"measured_on": "held-out",
},
),
_fact(5, "cms_summary", "CMS reported that commenters supported it."),
_fact(
6,
"seat",
"Jane Doe, MD (Acme Clinic) commented on the CY2023 rule.",
label="CMS-2022-0113-0001",
),
_fact(7, "round", "flagged paraphrase", note="model paraphrase, unreviewed"),
_fact(8, "stance_split", "Of 3 CY2018 commenters we classified, 3 supported."),
_fact(9, "finding", "a finding"),
# live-shaped: far more CY2008 rounds than any cap
*(
_fact(
100 + i,
"round",
f"For CY2008, CMS finalized work RVU {i} for 99326.",
year=2008,
codes=("99326",),
)
for i in range(40)
),
*extra,
]
tl = [
TimelineEvent(
2008,
"valuation",
("99326",),
"For CY2008, CMS finalized RVUs [F100].",
("F100",),
(),
(),
),
TimelineEvent(
2023,
"deletion",
old,
"99324 and 99325 were deleted for CY2023 [F3].",
("F3",),
(),
(),
),
]
return Dossier(
family="HOME-VISITS",
name="Home or Residence Services",
codes=("99341",),
status="active",
years_active="2008-",
summary="CMS finalized it [F2][F2] and deleted codes [F3].",
timeline=tl,
rounds=[],
payment=[],
utilization=[],
stakeholders=[],
governance=[],
sources={},
gaps=["RVU files before 2015 not yet loaded."],
facts={f.id: f for f in facts},
built_at="2026-11-12T00:00:00",
check_passed=True,
extractor="template",
review=[{"fact_id": "F9", "reason": "model extract, unreviewed"}],
stance_accuracy=accuracy,
)
@pytest.fixture
def replica(tmp_path, monkeypatch):
evidence._REPLICA = None
path = tmp_path / "aco.ro.duckdb"
con = duckdb.connect(str(path))
ensure_tables(con)
write_dossier(con, _dossier())
con.close()
monkeypatch.setenv("DOMAIN", "")
yield replace(CFG, duckdb_replica=str(path))
evidence._REPLICA = None
def test_dossier_evidence_for_home_visit_question(replica):
ev = dossier_evidence("how were home visits valued?", replica)
assert ev is not None
assert ev.family == "HOME-VISITS"
assert ev.name == "Home or Residence Services"
assert ev.url == "https://lex.fhirworx.io/?family=HOME-VISITS#dossier"
assert ev.max_prompt_rows == 30
assert len(ev.facts) == 49
assert len(ev.timeline) == 2
assert ev.gaps == ("RVU files before 2015 not yet loaded.",)
def test_none_without_dossier_row(tmp_path):
evidence._REPLICA = None
path = tmp_path / "empty.duckdb"
con = duckdb.connect(str(path))
ensure_tables(con)
con.close()
cfg = replace(CFG, duckdb_replica=str(path))
assert dossier_evidence("how were home visits valued?", cfg) is None
evidence._REPLICA = None
def test_none_without_family_or_replica(replica):
assert (
dossier_evidence("what did commenters say about telehealth?", replica) is None
)
assert dossier_evidence("home visits?", CFG) is None # replica file missing
def test_unreadable_replica_is_logged_not_raised(replica, monkeypatch, caplog):
def boom(_p):
raise OSError("locked")
monkeypatch.setattr(evidence, "_connect", boom)
with caplog.at_level(logging.WARNING, logger="llm.dossier"):
assert dossier_evidence("home visits?", replica) is None
assert "dossier evidence skipped" in caplog.text
def test_first_family_curated_before_derived(replica, monkeypatch):
from types import SimpleNamespace
import llm.dossier as mod
monkeypatch.setattr(
mod,
"detect_codes",
lambda q: SimpleNamespace(
families=("AAA-DERIVED", "HOME-VISITS", "ZZZ"), explicit=(), codes=()
),
)
monkeypatch.setattr(mod, "CURATED_FAMILIES", {"HOME-VISITS": 1})
seen = []
real = mod.read_dossier
def spy(cur, fam):
seen.append(fam)
return real(cur, fam)
monkeypatch.setattr(mod, "read_dossier", spy)
ev = dossier_evidence("q", replica)
assert ev.family == "HOME-VISITS" and seen[0] == "HOME-VISITS"
def test_dossier_payload_shape_and_summary_labels(replica):
p = dossier_evidence("home visits?", replica).payload()
assert p["type"] == "dossier"
assert p["family"] == "HOME-VISITS"
assert p["url"] == "https://lex.fhirworx.io/?family=HOME-VISITS#dossier"
# [F#] became the facts' labels; the doubled cite collapsed
assert p["summary"] == "CMS finalized it [L2] and deleted codes [DEL23]."
assert {"label", "text", "url", "year", "kind", "review"} == set(p["facts"][0])
assert p["gaps"] == ["RVU files before 2015 not yet loaded."]
def test_payload_carries_only_cited_and_selected_facts(replica):
ev = dossier_evidence("home visits?", replica)
p = ev.payload()
labels = {f["label"] for f in p["facts"]}
assert {"L2", "DEL23", "L100"} <= labels # summary + timeline cites
assert "L1" not in labels # payment rows stay on the page
assert len(p["facts"]) < len(ev.facts)
assert all(f["review"] is False for f in p["facts"])
# a flagged fact cited by the timeline travels flagged
ev2 = replace(ev, flagged=("F3",))
assert [f["review"] for f in ev2.payload()["facts"] if f["label"] == "DEL23"] == [
True
]
def test_payload_unknown_fact_tag_kept(replica):
ev = dossier_evidence("home visits?", replica)
ev = replace(ev, summary="see [F999].")
assert ev.payload()["summary"] == "see [F999]."
def test_payload_names_come_from_own_records(replica):
"""Commenters are named as the public record names them (user ruling)."""
p = dossier_evidence("home visits?", replica).payload()
seat = [f for f in p["facts"] if f["kind"] == "seat"][0]
assert "Jane Doe, MD" in seat["text"]
assert seat["label"] == "CMS-2022-0113-0001"
def _ev(replica_cfg, question):
return dossier_evidence(question, replica_cfg)
def test_prompt_block_order_and_labels(replica):
ev = _ev(replica, "When were domiciliary visit codes 99324 and 99325 deleted?")
block = ev.prompt_block()
lines = block.splitlines()
assert lines[0] == (
"Dossier — cited facts about the Home or Residence Services family "
"(authoritative for its history, valuation rounds and who commented). "
"Every claim carries its label in square brackets inside the line; copy "
"labels exactly:"
)
assert lines[1] == ("Summary: CMS finalized it [L2] and deleted codes [DEL23].")
# timeline, newest first, in the dossier's own prose with labels
assert lines[2] == "CY2023: 99324 and 99325 were deleted for CY2023 [DEL23]."
assert lines[3] == "CY2008: For CY2008, CMS finalized RVUs [L100]."
# then the question's matched facts, then the stance unit
assert block.index("99324 was deleted for CY2023. [DEL23]") > block.index("CY2008:")
assert block.index("Our own classification") > block.index("[DEL23]")
assert "flagged paraphrase" not in block and "a finding" not in block
assert "payment row" not in block
# every line but the stance header ends with a label
for ln in lines:
assert ln.rstrip(".").endswith("]") or ln.startswith(
("Dossier —", "Our own classification")
)
def test_deletion_question_surfaces_cy2023_over_cy2008_rounds(replica):
"""The live bug: 30 CY2008 round lines crowded out every other fact."""
block = _ev(replica, "When were 99324-99325 deleted?").prompt_block()
assert "99324 was deleted for CY2023. [DEL23]" in block
assert block.count("For CY2008, CMS finalized work RVU") < 10
def test_matching_is_question_aware(replica):
ev = _ev(replica, "How did CMS value 99326 in 2008?")
first = [f for f in ev.facts if f.id == ev.matched[0]][0]
assert first.year == 2008 and "99326" in first.codes
def test_match_facts_round_robin_newest_first_and_limit():
facts = [
_fact(1, "round", "r-old", year=2008),
_fact(2, "round", "r-new", year=2023),
_fact(3, "event", "e", year=2015),
_fact(4, "round", "r-mid", year=2015),
_fact(5, "stance_split", "s"),
_fact(6, "payment", "p"),
_fact(7, "round", "flag", note="x"),
_fact(8, "round", "elsewhere", codes=("A1",), year=2000),
]
assert match_facts(facts, (), ("99341",), ()) == ("F2", "F3", "F4", "F1")
# a named year also matches; the limit applies after interleaving
assert match_facts(facts, (), ("zzz",), {2000}, limit=1) == ("F8",)
# nothing matches -> every candidate
assert match_facts(facts, ("F2",), ("zzz",), ()) == ("F4", "F3", "F1", "F8")
def test_question_words_that_tell_facts_apart_come_first():
facts = [
_fact(1, "round", "CMS finalized the home visit RVU.", year=2027),
_fact(2, "round", "CMS finalized another home visit RVU.", year=2026),
_fact(3, "event", "Codes were deleted for CY2023.", year=2023),
_fact(4, "cms_summary", "A commenter disliked the home visit rule.", year=2025),
_fact(5, "event", "A home visit code was created.", year=2024),
]
plain = match_facts(facts, (), ("99341",), ())
assert plain[0] == "F1"
asked = match_facts(facts, (), ("99341",), (), question="When were codes deleted?")
assert asked[0] == "F3" and set(asked) == set(plain)
# the family's own name words never rank, however rare in the facts
named = [
_fact(1, "round", "a", year=2027),
_fact(2, "event", "Residence rule, 2019.", year=2019),
_fact(3, "event", "b", year=2020),
_fact(4, "event", "c", year=2021),
_fact(5, "event", "d", year=2022),
_fact(6, "event", "e", year=2023),
]
q = "Which residence societies supported it?"
assert match_facts(named, (), ("99341",), (), question=q)[:2] == ("F2", "F1")
assert match_facts(
named, (), ("99341",), (), question=q, family_name="Home or Residence Services"
) == match_facts(named, (), ("99341",), ())
# a word found in most candidates ("visit") tells nothing
assert match_facts(facts, (), ("99341",), (), question="home visit?") == plain
def test_question_years():
assert question_years("CY2023 vs 2008, and cy 2015?") == {2023, 2008, 2015}
def test_stance_unit_has_one_header_and_no_repeated_accuracy(replica):
block = _ev(replica, "home visits?").prompt_block()
assert block.count("Preliminary model classification") == 1
head = [ln for ln in block.splitlines() if ln.startswith("Our own")][0]
assert ACC in head
for ln in block.splitlines():
if "commenters we classified" in ln:
assert "Preliminary" not in ln and ln.endswith("]")
assert block.index(head) < block.index("Of 3 CY2023 commenters")
def test_stance_header_says_unmeasured_without_accuracy():
d = _dossier(accuracy="")
ev = DossierEvidence(
d.family, d.name, "u", d.summary, tuple(d.facts.values()), (), accuracy=""
)
assert "accuracy is not yet measured" in ev.prompt_block()
def test_no_stance_facts_no_unit():
ev = DossierEvidence("F", "N", "u", "s", (), ())
assert ev.prompt_block().splitlines()[1] == "Summary: s"
assert "Our own" not in ev.prompt_block()
def test_max_rows_caps_matched_facts(replica):
ev = _ev(replica, "home visits?")
n = len(ev.prompt_block(max_rows=3).splitlines())
assert n == len(ev.prompt_block(max_rows=0).splitlines()) + 3
def _tl(n):
return tuple(
TimelineEvent(2000 + i, "k", (), f"event {i} [F1]", ("F1",), (), ())
for i in range(n)
)
def _many(intent=False, n_tl=12, matched=6):
d = _dossier()
facts = tuple(d.facts.values())
return DossierEvidence(
d.family,
d.name,
"u",
d.summary,
facts,
(),
accuracy=d.stance_accuracy,
timeline=_tl(n_tl),
matched=tuple(f.id for f in facts if f.year == 2008)[:matched],
stance_intent=intent,
)
def _state(block):
return (
block.count("\nCY"),
block.count("For CY2008"),
"Our own" in block,
)
def test_shed_order_old_timeline_then_matched_then_timeline_then_stance():
ev = _many()
full = ev.prompt_block()
assert _state(full) == (12, 6, True)
states = {_state(ev.prompt_block(max_chars=c)) for c in range(len(full), 0, -1)}
for n_tl, n_m, stance in states:
if n_tl > 8: # the oldest timeline lines go first
assert (n_m, stance) == (6, True)
if 2 < n_m < 6: # then matched facts, down to 2
assert (n_tl, stance) == (8, True)
if 4 < n_tl < 8: # then timeline down to the floor
assert (n_m, stance) == (2, True)
if not stance: # and only then the stance unit, all of it
assert n_tl == 4 and n_m <= 2
assert (11, 6, True) in states and (8, 2, True) in states
one = ev.prompt_block(max_chars=len(full) - 1)
assert "CY2011:" in one and "CY2000:" not in one # oldest went first
tiny = ev.prompt_block(max_chars=1)
assert _state(tiny) == (TIMELINE_FLOOR, 0, False)
assert "commenters we classified" not in tiny
def test_stance_intent_keeps_the_stance_unit_through_the_cap():
for intent, keeps in ((True, True), (False, False)):
ev = _many(intent)
block = ev.prompt_block(max_chars=1)
assert ("Our own" in block) is keeps
assert block.count("\nCY") == TIMELINE_FLOOR
assert "For CY2008" not in block # matched facts went first
# and just over the cap: the unit survives a small overage either way
ev = _many(True)
full = ev.prompt_block()
assert "Our own" in ev.prompt_block(max_chars=len(full) - 200)
assert "Our own" in _many(False).prompt_block(max_chars=len(full) - 200)
def test_stance_intent_detection():
from llm.dossier import stance_intent
for q in (
"What did commenters say?",
"who supported it",
"Which societies opposed it?",
"any stakeholder or lobby groups",
"the AMA association",
):
assert stance_intent(q)
assert not stance_intent("When were 99324-99337 deleted?")
def test_stance_question_block_carries_the_unit(replica):
q = "What did commenters say about home visits in CY2023?"
ev = _ev(replica, q)
assert ev.stance_intent
assert "Our own" in ev.prompt_block()
assert "Our own" in ev.prompt_block(max_chars=len(ev.prompt_block()) - 200)
assert "Of 3 CY2023 commenters" in ev.prompt_block(max_chars=1)
def test_deletion_question_block_carries_matched_deletion_facts(replica):
ev = _ev(replica, "When were codes 99324-99325 deleted?")
assert not ev.stance_intent
block = ev.prompt_block()
assert "99324 was deleted for CY2023. [DEL23]" in block
# still there when the cap bites a little
assert "[DEL23]" in ev.prompt_block(max_chars=len(block) - 100)
def test_range_matches_the_middle_codes_of_the_range(replica, monkeypatch):
from llm.dossier import expand_ranges
fam = ("99324", "99325", "99326", "99327", "99334", "99337", "99341")
for dash in ("-", "\u2013", "\u2014", " - "):
assert expand_ranges(f"deleted 99324{dash}99334?", fam) == (
"99324",
"99325",
"99326",
"99327",
"99334",
)
assert expand_ranges("no range 99324 and 99337", fam) == ()
assert expand_ranges("9932-99334", fam) == ()
# end to end: a fact on a middle code (99325, never named) is matched
d = _dossier(
extra=[
_fact(
300, "event", "99325 middle code event.", codes=("99325",), label="MID"
)
]
)
d.codes = ("99324", "99325", "99341")
evidence._REPLICA = None
con = duckdb.connect(replica.duckdb_replica)
write_dossier(con, d)
con.close()
ev = _ev(replica, "When were domiciliary codes 99324-99337 deleted?")
assert "MID" in {f.cite.label for f in ev.facts if f.id in ev.matched}
def test_dossier_url_uses_env_domain(monkeypatch):
monkeypatch.setenv("DOMAIN", "example.org")
assert dossier_url("A-B", CFG) == "https://lex.example.org/?family=A-B#dossier"
monkeypatch.delenv("DOMAIN")
assert dossier_url("A-B", replace(CFG, domain="cfg.test")) == (
"https://lex.cfg.test/?family=A-B#dossier"
)
def test_failed_dossier_query_is_logged_not_raised(replica, monkeypatch, caplog):
import llm.dossier as mod
def boom(_cur):
raise RuntimeError("bad table")
monkeypatch.setattr(mod, "dossier_families", boom)
with caplog.at_level(logging.WARNING, logger="llm.dossier"):
assert dossier_evidence("home visits?", replica) is None
assert "dossier evidence skipped (query)" in caplog.text
STANCE_LABEL = (
"Our classification of docket CMS-2022-0113 comments — preliminary: "
"recall 79%, direction 74% on held-out letters"
)
def test_stance_label_carries_the_accuracy_caveat(replica):
ev = _ev(replica, "What did commenters say about home visits in CY2023?")
block = ev.prompt_block()
assert f"2 supported and 1 opposed. [{STANCE_LABEL}]" in block
p = ev.payload()
stance = [f for f in p["facts"] if f["kind"] == "stance_split"]
assert STANCE_LABEL in {f["label"] for f in stance} # collectLinks key
# a fact without a docket keeps its own label
assert "L8" in {f["label"] for f in stance}
def test_stance_label_variants():
"""M3: the label reads the fact's own measured values, never the wording
of the accuracy sentence."""
base = _dossier()
facts = tuple(base.facts.values())
mk = lambda acc: DossierEvidence( # noqa: E731
"F", "N", "u", "[F4]", facts, (), accuracy=acc
)
f4 = base.facts["F4"]
def label(values, acc=ACC):
return mk(acc)._label(dataclasses.replace(f4, values=values))
assert label({"docket": "D"}).endswith("preliminary: accuracy not yet measured")
tuned = {
"docket": "D",
"recall": 0.407,
"direction": 0.818,
"measured_on": "tuning-set",
}
assert label(tuned).endswith("recall 41%, direction 82% on tuning-set letters")
# a reworded accuracy sentence changes nothing
assert label(f4.values, acc="Reworded.") == STANCE_LABEL
# the summary tag resolves to the same label
assert f"[{STANCE_LABEL}]" in mk(ACC).payload()["summary"]
def test_a_failed_check_is_never_served(tmp_path, caplog):
"""M8: a stored dossier whose check failed reaches no chat answer."""
evidence._REPLICA = None
path = tmp_path / "failed.duckdb"
con = duckdb.connect(str(path))
ensure_tables(con)
write_dossier(con, dataclasses.replace(_dossier(), check_passed=False))
con.close()
cfg = replace(CFG, duckdb_replica=str(path))
with caplog.at_level(logging.WARNING, logger="llm.dossier"):
assert dossier_evidence("how were home visits valued?", cfg) is None
assert "check failed" in caplog.text
evidence._REPLICA = None
def test_the_parsed_dossier_is_cached_per_replica_file(replica, monkeypatch):
"""M14: a chat turn re-parses the dossier JSON only when the replica
was republished (path and mtime)."""
import llm.dossier as mod
calls = []
real = mod.read_dossier
def counting(cur, family):
calls.append(family)
return real(cur, family)
monkeypatch.setattr(mod, "read_dossier", counting)
mod._PARSED.clear()
assert dossier_evidence("how were home visits valued?", replica) is not None
assert dossier_evidence("home visit history?", replica) is not None
assert calls == ["HOME-VISITS"]
path = Path(replica.duckdb_replica)
st = path.stat()
os.utime(path, ns=(st.st_atime_ns, st.st_mtime_ns + 1_000_000))
evidence._REPLICA = None
assert dossier_evidence("home visit history?", replica) is not None
assert calls == ["HOME-VISITS", "HOME-VISITS"]