630 lines
22 KiB
Python
630 lines
22 KiB
Python
"""llm.dossier — the family dossier as chat evidence (Task 15)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import dataclasses
|
|
import logging
|
|
import os
|
|
from dataclasses import replace
|
|
from pathlib import Path
|
|
|
|
import duckdb
|
|
import pytest
|
|
|
|
from llm import evidence
|
|
from llm.config import LlmConfig
|
|
from llm.dossier import (
|
|
TIMELINE_FLOOR,
|
|
DossierEvidence,
|
|
dossier_evidence,
|
|
dossier_url,
|
|
match_facts,
|
|
question_years,
|
|
)
|
|
from pfs.cite import Cite
|
|
from pfs.codetables import ensure_tables
|
|
from pfs.dossier import Dossier, Fact, TimelineEvent, write_dossier
|
|
|
|
CFG = LlmConfig(
|
|
ollama_hosts=("http://h1:11434",),
|
|
embed_model="e",
|
|
instruct_model="c",
|
|
embed_dim=768,
|
|
build_ann_index=False,
|
|
pg_host="x",
|
|
pg_port=5432,
|
|
pg_db="llm",
|
|
pg_user="llm",
|
|
duckdb_replica="/nonexistent/aco.ro.duckdb",
|
|
)
|
|
|
|
ACC = (
|
|
"Preliminary model classification, measured on a held-out set the prompt "
|
|
"was never tuned on (20 hand-coded comments, 24 positions; CMS-2025-0304): "
|
|
"it finds 79% of hand-coded positions, and the direction is right for 74% "
|
|
"of those it finds."
|
|
)
|
|
|
|
|
|
def _fact(i, kind, text, year=2023, label=None, note="", codes=("99341",), values=None):
|
|
return Fact(
|
|
id=f"F{i}",
|
|
kind=kind,
|
|
year=year,
|
|
codes=codes,
|
|
text=text,
|
|
values=values or {},
|
|
cite=Cite("fr", label or f"L{i}", f"https://x/{i}"),
|
|
note=note,
|
|
)
|
|
|
|
|
|
def _dossier(extra=(), accuracy=ACC):
|
|
old = ("99324", "99325")
|
|
facts = [
|
|
_fact(1, "payment", "payment row"),
|
|
_fact(2, "round", "CMS finalized work RVU 1.0 for 99341 for CY2023."),
|
|
_fact(
|
|
3,
|
|
"event",
|
|
"99324 was deleted for CY2023.",
|
|
label="DEL23",
|
|
codes=old,
|
|
),
|
|
_fact(
|
|
4,
|
|
"stance_split",
|
|
"Of 3 CY2023 commenters we classified, 2 supported and 1 opposed. " + ACC,
|
|
values={
|
|
"docket": "CMS-2022-0113",
|
|
"recall": 0.792,
|
|
"direction": 0.737,
|
|
"measured_on": "held-out",
|
|
},
|
|
),
|
|
_fact(5, "cms_summary", "CMS reported that commenters supported it."),
|
|
_fact(
|
|
6,
|
|
"seat",
|
|
"Jane Doe, MD (Acme Clinic) commented on the CY2023 rule.",
|
|
label="CMS-2022-0113-0001",
|
|
),
|
|
_fact(7, "round", "flagged paraphrase", note="model paraphrase, unreviewed"),
|
|
_fact(8, "stance_split", "Of 3 CY2018 commenters we classified, 3 supported."),
|
|
_fact(9, "finding", "a finding"),
|
|
# live-shaped: far more CY2008 rounds than any cap
|
|
*(
|
|
_fact(
|
|
100 + i,
|
|
"round",
|
|
f"For CY2008, CMS finalized work RVU {i} for 99326.",
|
|
year=2008,
|
|
codes=("99326",),
|
|
)
|
|
for i in range(40)
|
|
),
|
|
*extra,
|
|
]
|
|
tl = [
|
|
TimelineEvent(
|
|
2008,
|
|
"valuation",
|
|
("99326",),
|
|
"For CY2008, CMS finalized RVUs [F100].",
|
|
("F100",),
|
|
(),
|
|
(),
|
|
),
|
|
TimelineEvent(
|
|
2023,
|
|
"deletion",
|
|
old,
|
|
"99324 and 99325 were deleted for CY2023 [F3].",
|
|
("F3",),
|
|
(),
|
|
(),
|
|
),
|
|
]
|
|
return Dossier(
|
|
family="HOME-VISITS",
|
|
name="Home or Residence Services",
|
|
codes=("99341",),
|
|
status="active",
|
|
years_active="2008-",
|
|
summary="CMS finalized it [F2][F2] and deleted codes [F3].",
|
|
timeline=tl,
|
|
rounds=[],
|
|
payment=[],
|
|
utilization=[],
|
|
stakeholders=[],
|
|
governance=[],
|
|
sources={},
|
|
gaps=["RVU files before 2015 not yet loaded."],
|
|
facts={f.id: f for f in facts},
|
|
built_at="2026-11-12T00:00:00",
|
|
check_passed=True,
|
|
extractor="template",
|
|
review=[{"fact_id": "F9", "reason": "model extract, unreviewed"}],
|
|
stance_accuracy=accuracy,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def replica(tmp_path, monkeypatch):
|
|
evidence._REPLICA = None
|
|
path = tmp_path / "aco.ro.duckdb"
|
|
con = duckdb.connect(str(path))
|
|
ensure_tables(con)
|
|
write_dossier(con, _dossier())
|
|
con.close()
|
|
monkeypatch.setenv("DOMAIN", "")
|
|
yield replace(CFG, duckdb_replica=str(path))
|
|
evidence._REPLICA = None
|
|
|
|
|
|
def test_dossier_evidence_for_home_visit_question(replica):
|
|
ev = dossier_evidence("how were home visits valued?", replica)
|
|
assert ev is not None
|
|
assert ev.family == "HOME-VISITS"
|
|
assert ev.name == "Home or Residence Services"
|
|
assert ev.url == "https://lex.fhirworx.io/?family=HOME-VISITS#dossier"
|
|
assert ev.max_prompt_rows == 30
|
|
assert len(ev.facts) == 49
|
|
assert len(ev.timeline) == 2
|
|
assert ev.gaps == ("RVU files before 2015 not yet loaded.",)
|
|
|
|
|
|
def test_none_without_dossier_row(tmp_path):
|
|
evidence._REPLICA = None
|
|
path = tmp_path / "empty.duckdb"
|
|
con = duckdb.connect(str(path))
|
|
ensure_tables(con)
|
|
con.close()
|
|
cfg = replace(CFG, duckdb_replica=str(path))
|
|
assert dossier_evidence("how were home visits valued?", cfg) is None
|
|
evidence._REPLICA = None
|
|
|
|
|
|
def test_none_without_family_or_replica(replica):
|
|
assert (
|
|
dossier_evidence("what did commenters say about telehealth?", replica) is None
|
|
)
|
|
assert dossier_evidence("home visits?", CFG) is None # replica file missing
|
|
|
|
|
|
def test_unreadable_replica_is_logged_not_raised(replica, monkeypatch, caplog):
|
|
def boom(_p):
|
|
raise OSError("locked")
|
|
|
|
monkeypatch.setattr(evidence, "_connect", boom)
|
|
with caplog.at_level(logging.WARNING, logger="llm.dossier"):
|
|
assert dossier_evidence("home visits?", replica) is None
|
|
assert "dossier evidence skipped" in caplog.text
|
|
|
|
|
|
def test_first_family_curated_before_derived(replica, monkeypatch):
|
|
from types import SimpleNamespace
|
|
|
|
import llm.dossier as mod
|
|
|
|
monkeypatch.setattr(
|
|
mod,
|
|
"detect_codes",
|
|
lambda q: SimpleNamespace(
|
|
families=("AAA-DERIVED", "HOME-VISITS", "ZZZ"), explicit=(), codes=()
|
|
),
|
|
)
|
|
monkeypatch.setattr(mod, "CURATED_FAMILIES", {"HOME-VISITS": 1})
|
|
seen = []
|
|
real = mod.read_dossier
|
|
|
|
def spy(cur, fam):
|
|
seen.append(fam)
|
|
return real(cur, fam)
|
|
|
|
monkeypatch.setattr(mod, "read_dossier", spy)
|
|
ev = dossier_evidence("q", replica)
|
|
assert ev.family == "HOME-VISITS" and seen[0] == "HOME-VISITS"
|
|
|
|
|
|
def test_dossier_payload_shape_and_summary_labels(replica):
|
|
p = dossier_evidence("home visits?", replica).payload()
|
|
assert p["type"] == "dossier"
|
|
assert p["family"] == "HOME-VISITS"
|
|
assert p["url"] == "https://lex.fhirworx.io/?family=HOME-VISITS#dossier"
|
|
# [F#] became the facts' labels; the doubled cite collapsed
|
|
assert p["summary"] == "CMS finalized it [L2] and deleted codes [DEL23]."
|
|
assert {"label", "text", "url", "year", "kind", "review"} == set(p["facts"][0])
|
|
assert p["gaps"] == ["RVU files before 2015 not yet loaded."]
|
|
|
|
|
|
def test_payload_carries_only_cited_and_selected_facts(replica):
|
|
ev = dossier_evidence("home visits?", replica)
|
|
p = ev.payload()
|
|
labels = {f["label"] for f in p["facts"]}
|
|
assert {"L2", "DEL23", "L100"} <= labels # summary + timeline cites
|
|
assert "L1" not in labels # payment rows stay on the page
|
|
assert len(p["facts"]) < len(ev.facts)
|
|
assert all(f["review"] is False for f in p["facts"])
|
|
# a flagged fact cited by the timeline travels flagged
|
|
ev2 = replace(ev, flagged=("F3",))
|
|
assert [f["review"] for f in ev2.payload()["facts"] if f["label"] == "DEL23"] == [
|
|
True
|
|
]
|
|
|
|
|
|
def test_payload_unknown_fact_tag_kept(replica):
|
|
ev = dossier_evidence("home visits?", replica)
|
|
ev = replace(ev, summary="see [F999].")
|
|
assert ev.payload()["summary"] == "see [F999]."
|
|
|
|
|
|
def test_payload_names_come_from_own_records(replica):
|
|
"""Commenters are named as the public record names them (user ruling)."""
|
|
p = dossier_evidence("home visits?", replica).payload()
|
|
seat = [f for f in p["facts"] if f["kind"] == "seat"][0]
|
|
assert "Jane Doe, MD" in seat["text"]
|
|
assert seat["label"] == "CMS-2022-0113-0001"
|
|
|
|
|
|
def _ev(replica_cfg, question):
|
|
return dossier_evidence(question, replica_cfg)
|
|
|
|
|
|
def test_prompt_block_order_and_labels(replica):
|
|
ev = _ev(replica, "When were domiciliary visit codes 99324 and 99325 deleted?")
|
|
block = ev.prompt_block()
|
|
lines = block.splitlines()
|
|
assert lines[0] == (
|
|
"Dossier — cited facts about the Home or Residence Services family "
|
|
"(authoritative for its history, valuation rounds and who commented). "
|
|
"Every claim carries its label in square brackets inside the line; copy "
|
|
"labels exactly:"
|
|
)
|
|
assert lines[1] == ("Summary: CMS finalized it [L2] and deleted codes [DEL23].")
|
|
# timeline, newest first, in the dossier's own prose with labels
|
|
assert lines[2] == "CY2023: 99324 and 99325 were deleted for CY2023 [DEL23]."
|
|
assert lines[3] == "CY2008: For CY2008, CMS finalized RVUs [L100]."
|
|
# then the question's matched facts, then the stance unit
|
|
assert block.index("99324 was deleted for CY2023. [DEL23]") > block.index("CY2008:")
|
|
assert block.index("Our own classification") > block.index("[DEL23]")
|
|
assert "flagged paraphrase" not in block and "a finding" not in block
|
|
assert "payment row" not in block
|
|
# every line but the stance header ends with a label
|
|
for ln in lines:
|
|
assert ln.rstrip(".").endswith("]") or ln.startswith(
|
|
("Dossier —", "Our own classification")
|
|
)
|
|
|
|
|
|
def test_deletion_question_surfaces_cy2023_over_cy2008_rounds(replica):
|
|
"""The live bug: 30 CY2008 round lines crowded out every other fact."""
|
|
block = _ev(replica, "When were 99324-99325 deleted?").prompt_block()
|
|
assert "99324 was deleted for CY2023. [DEL23]" in block
|
|
assert block.count("For CY2008, CMS finalized work RVU") < 10
|
|
|
|
|
|
def test_matching_is_question_aware(replica):
|
|
ev = _ev(replica, "How did CMS value 99326 in 2008?")
|
|
first = [f for f in ev.facts if f.id == ev.matched[0]][0]
|
|
assert first.year == 2008 and "99326" in first.codes
|
|
|
|
|
|
def test_match_facts_round_robin_newest_first_and_limit():
|
|
facts = [
|
|
_fact(1, "round", "r-old", year=2008),
|
|
_fact(2, "round", "r-new", year=2023),
|
|
_fact(3, "event", "e", year=2015),
|
|
_fact(4, "round", "r-mid", year=2015),
|
|
_fact(5, "stance_split", "s"),
|
|
_fact(6, "payment", "p"),
|
|
_fact(7, "round", "flag", note="x"),
|
|
_fact(8, "round", "elsewhere", codes=("A1",), year=2000),
|
|
]
|
|
assert match_facts(facts, (), ("99341",), ()) == ("F2", "F3", "F4", "F1")
|
|
# a named year also matches; the limit applies after interleaving
|
|
assert match_facts(facts, (), ("zzz",), {2000}, limit=1) == ("F8",)
|
|
# nothing matches -> every candidate
|
|
assert match_facts(facts, ("F2",), ("zzz",), ()) == ("F4", "F3", "F1", "F8")
|
|
|
|
|
|
def test_question_words_that_tell_facts_apart_come_first():
|
|
facts = [
|
|
_fact(1, "round", "CMS finalized the home visit RVU.", year=2027),
|
|
_fact(2, "round", "CMS finalized another home visit RVU.", year=2026),
|
|
_fact(3, "event", "Codes were deleted for CY2023.", year=2023),
|
|
_fact(4, "cms_summary", "A commenter disliked the home visit rule.", year=2025),
|
|
_fact(5, "event", "A home visit code was created.", year=2024),
|
|
]
|
|
plain = match_facts(facts, (), ("99341",), ())
|
|
assert plain[0] == "F1"
|
|
asked = match_facts(facts, (), ("99341",), (), question="When were codes deleted?")
|
|
assert asked[0] == "F3" and set(asked) == set(plain)
|
|
# the family's own name words never rank, however rare in the facts
|
|
named = [
|
|
_fact(1, "round", "a", year=2027),
|
|
_fact(2, "event", "Residence rule, 2019.", year=2019),
|
|
_fact(3, "event", "b", year=2020),
|
|
_fact(4, "event", "c", year=2021),
|
|
_fact(5, "event", "d", year=2022),
|
|
_fact(6, "event", "e", year=2023),
|
|
]
|
|
q = "Which residence societies supported it?"
|
|
assert match_facts(named, (), ("99341",), (), question=q)[:2] == ("F2", "F1")
|
|
assert match_facts(
|
|
named, (), ("99341",), (), question=q, family_name="Home or Residence Services"
|
|
) == match_facts(named, (), ("99341",), ())
|
|
# a word found in most candidates ("visit") tells nothing
|
|
assert match_facts(facts, (), ("99341",), (), question="home visit?") == plain
|
|
|
|
|
|
def test_question_years():
|
|
assert question_years("CY2023 vs 2008, and cy 2015?") == {2023, 2008, 2015}
|
|
|
|
|
|
def test_stance_unit_has_one_header_and_no_repeated_accuracy(replica):
|
|
block = _ev(replica, "home visits?").prompt_block()
|
|
assert block.count("Preliminary model classification") == 1
|
|
head = [ln for ln in block.splitlines() if ln.startswith("Our own")][0]
|
|
assert ACC in head
|
|
for ln in block.splitlines():
|
|
if "commenters we classified" in ln:
|
|
assert "Preliminary" not in ln and ln.endswith("]")
|
|
assert block.index(head) < block.index("Of 3 CY2023 commenters")
|
|
|
|
|
|
def test_stance_header_says_unmeasured_without_accuracy():
|
|
d = _dossier(accuracy="")
|
|
ev = DossierEvidence(
|
|
d.family, d.name, "u", d.summary, tuple(d.facts.values()), (), accuracy=""
|
|
)
|
|
assert "accuracy is not yet measured" in ev.prompt_block()
|
|
|
|
|
|
def test_no_stance_facts_no_unit():
|
|
ev = DossierEvidence("F", "N", "u", "s", (), ())
|
|
assert ev.prompt_block().splitlines()[1] == "Summary: s"
|
|
assert "Our own" not in ev.prompt_block()
|
|
|
|
|
|
def test_max_rows_caps_matched_facts(replica):
|
|
ev = _ev(replica, "home visits?")
|
|
n = len(ev.prompt_block(max_rows=3).splitlines())
|
|
assert n == len(ev.prompt_block(max_rows=0).splitlines()) + 3
|
|
|
|
|
|
def _tl(n):
|
|
return tuple(
|
|
TimelineEvent(2000 + i, "k", (), f"event {i} [F1]", ("F1",), (), ())
|
|
for i in range(n)
|
|
)
|
|
|
|
|
|
def _many(intent=False, n_tl=12, matched=6):
|
|
d = _dossier()
|
|
facts = tuple(d.facts.values())
|
|
return DossierEvidence(
|
|
d.family,
|
|
d.name,
|
|
"u",
|
|
d.summary,
|
|
facts,
|
|
(),
|
|
accuracy=d.stance_accuracy,
|
|
timeline=_tl(n_tl),
|
|
matched=tuple(f.id for f in facts if f.year == 2008)[:matched],
|
|
stance_intent=intent,
|
|
)
|
|
|
|
|
|
def _state(block):
|
|
return (
|
|
block.count("\nCY"),
|
|
block.count("For CY2008"),
|
|
"Our own" in block,
|
|
)
|
|
|
|
|
|
def test_shed_order_old_timeline_then_matched_then_timeline_then_stance():
|
|
ev = _many()
|
|
full = ev.prompt_block()
|
|
assert _state(full) == (12, 6, True)
|
|
states = {_state(ev.prompt_block(max_chars=c)) for c in range(len(full), 0, -1)}
|
|
for n_tl, n_m, stance in states:
|
|
if n_tl > 8: # the oldest timeline lines go first
|
|
assert (n_m, stance) == (6, True)
|
|
if 2 < n_m < 6: # then matched facts, down to 2
|
|
assert (n_tl, stance) == (8, True)
|
|
if 4 < n_tl < 8: # then timeline down to the floor
|
|
assert (n_m, stance) == (2, True)
|
|
if not stance: # and only then the stance unit, all of it
|
|
assert n_tl == 4 and n_m <= 2
|
|
assert (11, 6, True) in states and (8, 2, True) in states
|
|
one = ev.prompt_block(max_chars=len(full) - 1)
|
|
assert "CY2011:" in one and "CY2000:" not in one # oldest went first
|
|
tiny = ev.prompt_block(max_chars=1)
|
|
assert _state(tiny) == (TIMELINE_FLOOR, 0, False)
|
|
assert "commenters we classified" not in tiny
|
|
|
|
|
|
def test_stance_intent_keeps_the_stance_unit_through_the_cap():
|
|
for intent, keeps in ((True, True), (False, False)):
|
|
ev = _many(intent)
|
|
block = ev.prompt_block(max_chars=1)
|
|
assert ("Our own" in block) is keeps
|
|
assert block.count("\nCY") == TIMELINE_FLOOR
|
|
assert "For CY2008" not in block # matched facts went first
|
|
# and just over the cap: the unit survives a small overage either way
|
|
ev = _many(True)
|
|
full = ev.prompt_block()
|
|
assert "Our own" in ev.prompt_block(max_chars=len(full) - 200)
|
|
assert "Our own" in _many(False).prompt_block(max_chars=len(full) - 200)
|
|
|
|
|
|
def test_stance_intent_detection():
|
|
from llm.dossier import stance_intent
|
|
|
|
for q in (
|
|
"What did commenters say?",
|
|
"who supported it",
|
|
"Which societies opposed it?",
|
|
"any stakeholder or lobby groups",
|
|
"the AMA association",
|
|
):
|
|
assert stance_intent(q)
|
|
assert not stance_intent("When were 99324-99337 deleted?")
|
|
|
|
|
|
def test_stance_question_block_carries_the_unit(replica):
|
|
q = "What did commenters say about home visits in CY2023?"
|
|
ev = _ev(replica, q)
|
|
assert ev.stance_intent
|
|
assert "Our own" in ev.prompt_block()
|
|
assert "Our own" in ev.prompt_block(max_chars=len(ev.prompt_block()) - 200)
|
|
assert "Of 3 CY2023 commenters" in ev.prompt_block(max_chars=1)
|
|
|
|
|
|
def test_deletion_question_block_carries_matched_deletion_facts(replica):
|
|
ev = _ev(replica, "When were codes 99324-99325 deleted?")
|
|
assert not ev.stance_intent
|
|
block = ev.prompt_block()
|
|
assert "99324 was deleted for CY2023. [DEL23]" in block
|
|
# still there when the cap bites a little
|
|
assert "[DEL23]" in ev.prompt_block(max_chars=len(block) - 100)
|
|
|
|
|
|
def test_range_matches_the_middle_codes_of_the_range(replica, monkeypatch):
|
|
from llm.dossier import expand_ranges
|
|
|
|
fam = ("99324", "99325", "99326", "99327", "99334", "99337", "99341")
|
|
for dash in ("-", "\u2013", "\u2014", " - "):
|
|
assert expand_ranges(f"deleted 99324{dash}99334?", fam) == (
|
|
"99324",
|
|
"99325",
|
|
"99326",
|
|
"99327",
|
|
"99334",
|
|
)
|
|
assert expand_ranges("no range 99324 and 99337", fam) == ()
|
|
assert expand_ranges("9932-99334", fam) == ()
|
|
# end to end: a fact on a middle code (99325, never named) is matched
|
|
d = _dossier(
|
|
extra=[
|
|
_fact(
|
|
300, "event", "99325 middle code event.", codes=("99325",), label="MID"
|
|
)
|
|
]
|
|
)
|
|
d.codes = ("99324", "99325", "99341")
|
|
evidence._REPLICA = None
|
|
con = duckdb.connect(replica.duckdb_replica)
|
|
write_dossier(con, d)
|
|
con.close()
|
|
ev = _ev(replica, "When were domiciliary codes 99324-99337 deleted?")
|
|
assert "MID" in {f.cite.label for f in ev.facts if f.id in ev.matched}
|
|
|
|
|
|
def test_dossier_url_uses_env_domain(monkeypatch):
|
|
monkeypatch.setenv("DOMAIN", "example.org")
|
|
assert dossier_url("A-B", CFG) == "https://lex.example.org/?family=A-B#dossier"
|
|
monkeypatch.delenv("DOMAIN")
|
|
assert dossier_url("A-B", replace(CFG, domain="cfg.test")) == (
|
|
"https://lex.cfg.test/?family=A-B#dossier"
|
|
)
|
|
|
|
|
|
def test_failed_dossier_query_is_logged_not_raised(replica, monkeypatch, caplog):
|
|
import llm.dossier as mod
|
|
|
|
def boom(_cur):
|
|
raise RuntimeError("bad table")
|
|
|
|
monkeypatch.setattr(mod, "dossier_families", boom)
|
|
with caplog.at_level(logging.WARNING, logger="llm.dossier"):
|
|
assert dossier_evidence("home visits?", replica) is None
|
|
assert "dossier evidence skipped (query)" in caplog.text
|
|
|
|
|
|
STANCE_LABEL = (
|
|
"Our classification of docket CMS-2022-0113 comments — preliminary: "
|
|
"recall 79%, direction 74% on held-out letters"
|
|
)
|
|
|
|
|
|
def test_stance_label_carries_the_accuracy_caveat(replica):
|
|
ev = _ev(replica, "What did commenters say about home visits in CY2023?")
|
|
block = ev.prompt_block()
|
|
assert f"2 supported and 1 opposed. [{STANCE_LABEL}]" in block
|
|
p = ev.payload()
|
|
stance = [f for f in p["facts"] if f["kind"] == "stance_split"]
|
|
assert STANCE_LABEL in {f["label"] for f in stance} # collectLinks key
|
|
# a fact without a docket keeps its own label
|
|
assert "L8" in {f["label"] for f in stance}
|
|
|
|
|
|
def test_stance_label_variants():
|
|
"""M3: the label reads the fact's own measured values, never the wording
|
|
of the accuracy sentence."""
|
|
base = _dossier()
|
|
facts = tuple(base.facts.values())
|
|
mk = lambda acc: DossierEvidence( # noqa: E731
|
|
"F", "N", "u", "[F4]", facts, (), accuracy=acc
|
|
)
|
|
f4 = base.facts["F4"]
|
|
|
|
def label(values, acc=ACC):
|
|
return mk(acc)._label(dataclasses.replace(f4, values=values))
|
|
|
|
assert label({"docket": "D"}).endswith("preliminary: accuracy not yet measured")
|
|
tuned = {
|
|
"docket": "D",
|
|
"recall": 0.407,
|
|
"direction": 0.818,
|
|
"measured_on": "tuning-set",
|
|
}
|
|
assert label(tuned).endswith("recall 41%, direction 82% on tuning-set letters")
|
|
# a reworded accuracy sentence changes nothing
|
|
assert label(f4.values, acc="Reworded.") == STANCE_LABEL
|
|
# the summary tag resolves to the same label
|
|
assert f"[{STANCE_LABEL}]" in mk(ACC).payload()["summary"]
|
|
|
|
|
|
def test_a_failed_check_is_never_served(tmp_path, caplog):
|
|
"""M8: a stored dossier whose check failed reaches no chat answer."""
|
|
evidence._REPLICA = None
|
|
path = tmp_path / "failed.duckdb"
|
|
con = duckdb.connect(str(path))
|
|
ensure_tables(con)
|
|
write_dossier(con, dataclasses.replace(_dossier(), check_passed=False))
|
|
con.close()
|
|
cfg = replace(CFG, duckdb_replica=str(path))
|
|
with caplog.at_level(logging.WARNING, logger="llm.dossier"):
|
|
assert dossier_evidence("how were home visits valued?", cfg) is None
|
|
assert "check failed" in caplog.text
|
|
evidence._REPLICA = None
|
|
|
|
|
|
def test_the_parsed_dossier_is_cached_per_replica_file(replica, monkeypatch):
|
|
"""M14: a chat turn re-parses the dossier JSON only when the replica
|
|
was republished (path and mtime)."""
|
|
import llm.dossier as mod
|
|
|
|
calls = []
|
|
real = mod.read_dossier
|
|
|
|
def counting(cur, family):
|
|
calls.append(family)
|
|
return real(cur, family)
|
|
|
|
monkeypatch.setattr(mod, "read_dossier", counting)
|
|
mod._PARSED.clear()
|
|
assert dossier_evidence("how were home visits valued?", replica) is not None
|
|
assert dossier_evidence("home visit history?", replica) is not None
|
|
assert calls == ["HOME-VISITS"]
|
|
path = Path(replica.duckdb_replica)
|
|
st = path.stat()
|
|
os.utime(path, ns=(st.st_atime_ns, st.st_mtime_ns + 1_000_000))
|
|
evidence._REPLICA = None
|
|
assert dossier_evidence("home visit history?", replica) is not None
|
|
assert calls == ["HOME-VISITS", "HOME-VISITS"]
|