diff --git a/src/llm/dossier.py b/src/llm/dossier.py index d0123f6..8855bae 100644 --- a/src/llm/dossier.py +++ b/src/llm/dossier.py @@ -15,24 +15,29 @@ import os import re from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, Collection, Sequence import llm.evidence as evidence from llm.config import LlmConfig -from pfs.dossier import Fact, dossier_families, read_dossier, stance_accuracy_line +from pfs.dossier import ( + Fact, + TimelineEvent, + dossier_families, + read_dossier, + stance_accuracy_line, +) from pfs.families import CURATED_FAMILIES, detect_codes __all__ = ["DossierEvidence", "dossier_evidence", "dossier_url"] log = logging.getLogger(__name__) -#: prompt priority: what the model is most likely asked about first. Payment -#: facts come last; the Valuation block is the authority on payment. -_PRIORITY = ( +#: round-robin order of the question-matched facts. Payment facts are left +#: to the Valuation block; stance counts form their own unit. +_KIND_ORDER = ( "round", "event", "cms_summary", - "stance_split", "link", "origin", "formation", @@ -41,17 +46,59 @@ _PRIORITY = ( "impact", "seat", "finding", - "payment", ) -_RANK = {k: i for i, k in enumerate(_PRIORITY)} _ACCURACY_MARK = "Preliminary model classification" _TAG = re.compile(r"\[(F\d+)\]") +_YEAR = re.compile(r"\b(?:CY\s?)?(20\d\d|19\d\d)\b", re.I) +#: summary + this many timeline lines survive the hardest trim +TIMELINE_FLOOR = 8 _HEADER = ( "Dossier — cited facts about the {name} family (authoritative for its " "history, valuation rounds and who commented). Each line ends with the " "label to copy:" ) +_STANCE_HEADER = ( + "Our own classification of commenters (never CMS's counts). Accuracy of " + "every count below: {acc}" +) + + +def question_years(question: str) -> frozenset[int]: + """Years the question names ("2008", "CY2023").""" + return frozenset(int(m) for m in _YEAR.findall(question)) + + +def match_facts( + facts: Sequence[Fact], + flagged: Sequence[str], + codes: Sequence[str], + years: Collection[int], + limit: int = 0, +) -> tuple[str, ...]: + """Ids of the facts that bear on the question, interleaved round-robin + by kind with the newest year first. A fact matches when its codes overlap + *codes* or its year is one of *years*; when nothing matches every + unflagged fact is a candidate. Flagged, stance and payment facts never + appear (stance counts are their own unit, payment rows are the Valuation + block's).""" + bad = set(flagged) + pool = [ + f for f in facts if f.kind in _KIND_ORDER and not f.note and f.id not in bad + ] + want = set(codes) + hit = [f for f in pool if (want & set(f.codes)) or (f.year in years)] + pool = hit or pool + by_kind: dict[str, list[Fact]] = {k: [] for k in _KIND_ORDER} + for f in pool: + by_kind[f.kind].append(f) + queues = [sorted(v, key=lambda f: -(f.year or 0)) for v in by_kind.values() if v] + out: list[str] = [] + while any(queues): + for q in queues: + if q: + out.append(q.pop(0).id) + return tuple(out[:limit] if limit else out) def dossier_url(family: str, cfg: LlmConfig) -> str: @@ -68,29 +115,61 @@ class DossierEvidence: summary: str facts: tuple[Fact, ...] gaps: tuple[str, ...] + #: matched facts kept in the prompt block (the timeline and the stance + #: unit are sized by characters and the budget trim instead) max_prompt_rows: int = 30 - #: the dossier's own stance-accuracy sentence, appended to any stance - #: count whose text does not carry one (a count never stands alone) + #: the dossier's own stance-accuracy sentence: the header of the stance + #: unit (a count never stands without it) accuracy: str = "" #: ids of facts shown flagged on the page (unreviewed model paraphrases); #: never given to the model flagged: tuple[str, ...] = () + timeline: tuple[TimelineEvent, ...] = () + #: ``match_facts`` result for the question, in prompt order + matched: tuple[str, ...] = () - def payload(self) -> dict[str, Any]: + def _labelled(self, text: str) -> str: label = {f.id: f.cite.label for f in self.facts} def tag(m: re.Match[str]) -> str: return f"[{label[m.group(1)]}]" if m.group(1) in label else m.group(0) - summary = _TAG.sub(tag, self.summary) + out = _TAG.sub(tag, text) # a run of the same cite ("[L2][L2]") reads once - summary = re.sub(r"(\[[^\[\]]+\])(?:\1)+", r"\1", summary) + return re.sub(r"(\[[^\[\]]+\])(?:\1)+", r"\1", out) + + def _timeline_lines(self) -> list[str]: + return [ + f"CY{e.year}: {self._labelled(e.what_changed)}" + for e in sorted(self.timeline, key=lambda e: -e.year) + ] + + def _stance(self) -> list[Fact]: + return [f for f in self.facts if f.kind == "stance_split"] + + def _prompt_fact_ids(self, rows: int | None = None) -> list[str]: + n = self.max_prompt_rows if rows is None else rows + return list(self.matched[: max(n, 0)]) + + def payload(self) -> dict[str, Any]: + """Facts cited by the summary or the timeline plus those the prompt + block carries (same shape as the brief, plus ``review``).""" + by_id = {f.id: f for f in self.facts} + wanted: set[str] = set() + for text in [self.summary, *(e.what_changed for e in self.timeline)]: + wanted.update(_TAG.findall(text)) + for e in self.timeline: + wanted.update(e.stance_fact_ids) + wanted.update(e.cms_summary_fact_ids) + wanted.update(self._prompt_fact_ids()) + wanted.update(f.id for f in self._stance()) + flagged = set(self.flagged) return { "type": "dossier", "family": self.family, "name": self.name, "url": self.url, - "summary": summary, + "summary": self._labelled(self.summary), "facts": [ { "label": f.cite.label, @@ -98,41 +177,67 @@ class DossierEvidence: "url": f.cite.url, "year": f.year, "kind": f.kind, + "review": bool(f.note) or f.id in flagged, } for f in self.facts + if f.id in wanted and f.id in by_id ], "gaps": list(self.gaps), } - def _line(self, f: Fact) -> str: - text = f.text - if f.kind == "stance_split" and _ACCURACY_MARK not in text: - text = f"{text} {self.accuracy or stance_accuracy_line(None)}" - return f"{text} [{f.cite.label}]" + def _stance_unit(self) -> list[str]: + stance = self._stance() + if not stance: + return [] + acc = self.accuracy or stance_accuracy_line(None) + lines = [_STANCE_HEADER.format(acc=acc)] + for f in stance: + text = f.text.split(f" {_ACCURACY_MARK}")[0] + lines.append(f"{text} [{f.cite.label}]") + return lines - def prompt_block(self, max_rows: int | None = None) -> str: - """*max_rows* (default ``max_prompt_rows``) caps the rows other than - stance counts; every stance count stays, each with its accuracy - sentence.""" - cap = self.max_prompt_rows if max_rows is None else max_rows - flagged = set(self.flagged) - usable = [ - (i, f) - for i, f in enumerate(self.facts) - if not f.note and f.id not in flagged + def prompt_block( + self, + max_rows: int | None = None, + max_chars: int | None = None, + floor: bool = False, + ) -> str: + """Summary, timeline (newest first), the question's matched facts and + one stance unit. *max_rows* caps the matched facts; *max_chars* sheds + matched facts, then the stance unit (all or none), then the oldest + timeline lines down to ``TIMELINE_FLOOR``; *floor* is the budget + trim's last step: the summary and the newest ``TIMELINE_FLOOR`` + timeline lines only.""" + by_id = {f.id: f for f in self.facts} + summary = f"Summary: {self._labelled(self.summary)}" + tl = self._timeline_lines() + matched = [ + f"{f.text} [{f.cite.label}]" + for f in (by_id[i] for i in self._prompt_fact_ids(max_rows) if i in by_id) ] - usable.sort(key=lambda p: (_RANK.get(p[1].kind, len(_RANK)), p[0])) - kept: list[Fact] = [] - others = 0 - for _i, f in usable: - if f.kind == "stance_split": - kept.append(f) - elif others < cap: - kept.append(f) - others += 1 - lines = [_HEADER.format(name=self.name)] - lines.extend(self._line(f) for f in kept) - return "\n".join(lines) + stance = self._stance_unit() + + def build(n_tl: int, n_m: int, with_stance: bool) -> str: + lines = [_HEADER.format(name=self.name), summary] + lines.extend(tl[:n_tl]) + lines.extend(matched[:n_m]) + if with_stance: + lines.extend(stance) + return "\n".join(lines) + + if floor: + return build(TIMELINE_FLOOR, 0, False) + n_tl, n_m, with_stance = len(tl), len(matched), bool(stance) + while max_chars is not None and len(build(n_tl, n_m, with_stance)) > max_chars: + if n_m: + n_m -= 1 + elif with_stance: + with_stance = False + elif n_tl > TIMELINE_FLOOR: + n_tl -= 1 + else: + break + return build(n_tl, n_m, with_stance) def dossier_evidence(question: str, cfg: LlmConfig) -> DossierEvidence | None: @@ -165,14 +270,23 @@ def dossier_evidence(question: str, cfg: LlmConfig) -> DossierEvidence | None: cur.close() if d is None: return None + facts = tuple(d.facts.values()) + flagged = tuple(r["fact_id"] for r in d.review) return DossierEvidence( family=d.family, name=d.name, url=dossier_url(d.family, cfg), summary=d.summary, - facts=tuple(d.facts.values()), + facts=facts, gaps=tuple(d.gaps), max_prompt_rows=cfg.dossier_max_rows, accuracy=d.stance_accuracy, - flagged=tuple(r["fact_id"] for r in d.review), + flagged=flagged, + timeline=tuple(d.timeline), + matched=match_facts( + facts, + flagged, + det.explicit or det.codes, + question_years(question), + ), ) diff --git a/src/llm/evidence.py b/src/llm/evidence.py index 24f4ac6..ff3d9c2 100644 --- a/src/llm/evidence.py +++ b/src/llm/evidence.py @@ -122,6 +122,9 @@ class ValuationEvidence: #: ``codes``) — carried through so ``rag.stream_answer`` doesn't #: need a third ``detect_codes`` call just to find them. wide: tuple[str, ...] = () + #: the default-years rows the prompt block selects from when the payload + #: carries a dossier family's whole history (``None``: ``rows`` itself) + prompt_rows: tuple[ValuationRow, ...] | None = None def prompt_block(self, max_rows: int | None = None) -> str: """*max_rows* overrides ``self.max_prompt_rows`` for one call — @@ -129,7 +132,8 @@ class ValuationEvidence: smaller block on demand without needing a new ``ValuationEvidence``.""" cap = self.max_prompt_rows if max_rows is None else max_rows lines = [_HEADER] - for r in _cap_valuation_rows(self.rows, self.explicit, cap): + source = self.rows if self.prompt_rows is None else self.prompt_rows + for r in _cap_valuation_rows(source, self.explicit, cap): # The status note says why an unpaid code shows no dollars, and # the CF note says which of the year's CFs the money came from. status = f"{r.status or '?'}" + ( @@ -265,14 +269,19 @@ def valuation_evidence(question: str, cfg: LlmConfig) -> ValuationEvidence | Non # A family with a dossier gets its whole payment history (every final # year, every proposed round) in the payload; the prompt block still # keeps only ``valuation_rows_max`` rows. - kwargs: dict[str, Any] = {"years": years} + full = False try: - if set(det.families) & set(dossier_families(cur)): - kwargs = {"years": 0, "all_proposed": True} + full = bool(set(det.families) & set(dossier_families(cur))) except Exception as e: # noqa: BLE001 — the valuation must still answer log.warning("dossier families unavailable for valuation: %s", e) try: - rows, unpriced = valuation(cur, list(codes), **kwargs) + rows, unpriced = valuation(cur, list(codes), years=years) + prompt_rows = None + if full: + # the prompt picks from the default window, so the whole history + # never pushes the newest final rows out of its row cap + prompt_rows = tuple(rows) + rows, unpriced = valuation(cur, list(codes), years=0, all_proposed=True) except Exception as e: # noqa: BLE001 log.warning("valuation evidence skipped (query): %s", e) return None @@ -286,6 +295,7 @@ def valuation_evidence(question: str, cfg: LlmConfig) -> ValuationEvidence | Non explicit=det.explicit, max_prompt_rows=cfg.valuation_rows_max, wide=det.wide, + prompt_rows=prompt_rows, ) diff --git a/src/llm/rag.py b/src/llm/rag.py index ee28ebc..c96585b 100644 --- a/src/llm/rag.py +++ b/src/llm/rag.py @@ -235,7 +235,7 @@ _CITE_REMINDER = ( "sentence that states a date, a code being created, adopted, replaced or " "deleted, an RVU, a conversion factor, a payment amount, or what a rule " "or a commenter said MUST end with the bracketed label of the Lineage " - "line, Valuation row or excerpt it comes from, copied exactly — for " + "line, Dossier line, Valuation row or excerpt it comes from, copied exactly — for " 'example: "G2058 was replaced by CPT 99439 for CY2021 [CY2021 PFS final ' '85 FR 84547 ¶686]." Do not write such a sentence without its label. ' "Take the year of each fact from its Lineage line, never from a " @@ -454,7 +454,8 @@ _BUDGET_RETRIEVED_KEEP = 6 _BUDGET_CITED_KEEP = 8 _BUDGET_MANUAL_KEEP = 1 _BUDGET_VALUATION_ROWS_KEEP = 12 -_BUDGET_DOSSIER_ROWS_FLOOR = 8 +#: share of the prompt budget the dossier block may take +_DOSSIER_BUDGET_SHARE = 0.35 # The model counts its context in tokens; the excerpts are trimmed in # characters before any model sees them. ``prompt_budget_chars`` converts @@ -540,7 +541,8 @@ def _user_message( valuation_rows_cap: int | None, lineage_rows_cap: int | None, dossier: DossierEvidence | None = None, - dossier_rows_cap: int | None = None, + dossier_chars: int | None = None, + dossier_floor: bool = False, ) -> str: sources = retrieved + cited + lineage_src + manual + rest if sources: @@ -558,7 +560,7 @@ def _user_message( context = "(no relevant excerpts found)" parts = [f"Excerpts:\n\n{context}"] if dossier is not None: - parts.append(dossier.prompt_block(dossier_rows_cap)) + parts.append(dossier.prompt_block(None, dossier_chars, dossier_floor)) if lineage is not None: parts.append(lineage.prompt_block(lineage_rows_cap)) if evidence is not None: @@ -601,8 +603,8 @@ def build_messages( budget_chars: int | None = None, ) -> list[dict]: """Grounded chat messages: system rules + question with excerpts and, - when available, a Lineage block and a Valuation block — in that - order — between the excerpts and the question. + when available, a Dossier block, a Lineage block and a Valuation + block — in that order — between the excerpts and the question. *sources* is assumed laid out contiguously in the order ``stream_answer`` appends them — retrieved, then cited @@ -635,7 +637,13 @@ def build_messages( valuation_cap = evidence.max_prompt_rows if evidence is not None else None lineage_cap = lineage.max_prompt_rows if lineage is not None else None - dossier_cap = dossier.max_prompt_rows if dossier is not None else None + # the dossier block never takes more than a third of the budget + dossier_chars = ( + int(budget_chars * _DOSSIER_BUDGET_SHARE) + if dossier is not None and budget_chars is not None + else None + ) + dossier_floor = False def content() -> str: return _user_message( @@ -650,7 +658,8 @@ def build_messages( valuation_cap, lineage_cap, dossier, - dossier_cap, + dossier_chars, + dossier_floor, ) if budget_chars is not None and len(content()) > budget_chars: @@ -711,6 +720,16 @@ def build_messages( before - budget_chars, budget_chars, ) + # The dossier block, before the lineage rows: down to the summary + # and the newest TIMELINE_FLOOR timeline lines. + if len(content()) > budget_chars and dossier is not None and not dossier_floor: + before = len(content()) + dossier_floor = True + log.info( + "prompt budget: cut the dossier block to its floor (%d chars over %d)", + before - budget_chars, + budget_chars, + ) if len(content()) > budget_chars and lineage is not None: target = lineage.max_prompt_rows half = max(target // 2, 1) @@ -723,22 +742,6 @@ def build_messages( before - budget_chars, budget_chars, ) - # Last of all, the dossier's rows (stance counts always stay), - # down to a floor of 8. - if ( - len(content()) > budget_chars - and dossier is not None - and (dossier_cap is None or dossier_cap > _BUDGET_DOSSIER_ROWS_FLOOR) - ): - before = len(content()) - dossier_cap = _BUDGET_DOSSIER_ROWS_FLOOR - log.info( - "prompt budget: capped dossier rows to %d (%d chars over %d)", - dossier_cap, - before - budget_chars, - budget_chars, - ) - return [ {"role": "system", "content": _SYSTEM}, {"role": "user", "content": content()}, @@ -794,6 +797,12 @@ def stream_answer( ) -> Iterator[dict]: """Retrieve, then stream a grounded answer from the largest live host. + When the detected family has a stored dossier, a ``dossier`` event + (``llm.dossier``) comes first and a Dossier block (summary, timeline, + question-matched facts, one stance unit) leads the prompt's evidence + blocks, capped at a third of the prompt budget; ``sources`` then starts + with a link entry for the dossier page. + ``mode`` ("auto"/"timeline"/"recent") is forwarded to ``retrieve``; "auto" (the default) switches to era-balanced retrieval on its own for a history-shaped question (``is_history_question``). diff --git a/src/llm/web/chat.html b/src/llm/web/chat.html index 07559b6..491ec04 100644 --- a/src/llm/web/chat.html +++ b/src/llm/web/chat.html @@ -486,7 +486,7 @@ if ((ev.gaps || []).length) { const g = document.createElement('div'); g.className = 'provenance'; - g.textContent = 'Not yet captured: ' + ev.gaps.join(' '); + g.textContent = 'Gaps and notes: ' + ev.gaps.join(' '); box.appendChild(g); } if (ev.url) { diff --git a/tests/llm/test_api.py b/tests/llm/test_api.py index ab3bea0..5a91562 100644 --- a/tests/llm/test_api.py +++ b/tests/llm/test_api.py @@ -60,6 +60,7 @@ class TestIndex: assert "function renderDossier" in html assert "ev.type === 'dossier'" in html assert "Open the full dossier" in html + assert "Gaps and notes" in html and "Not yet captured" not in html assert "tr.proposed" in html assert "rows.length > 40" in html assert " block.index("CY2008:") + assert block.index("Our own classification") > block.index("[DEL23]") + assert "flagged paraphrase" not in block and "a finding" not in block + assert "payment row" not in block + # every line but the stance header ends with a label + for ln in lines: + assert ln.rstrip(".").endswith("]") or ln.startswith( + ("Dossier —", "Our own classification") + ) + + +def test_deletion_question_surfaces_cy2023_over_cy2008_rounds(replica): + """The live bug: 30 CY2008 round lines crowded out every other fact.""" + block = _ev(replica, "When were 99324-99325 deleted?").prompt_block() + assert "99324 was deleted for CY2023. [DEL23]" in block + assert block.count("For CY2008, CMS finalized work RVU") < 10 + + +def test_matching_is_question_aware(replica): + ev = _ev(replica, "How did CMS value 99326 in 2008?") + first = [f for f in ev.facts if f.id == ev.matched[0]][0] + assert first.year == 2008 and "99326" in first.codes + + +def test_match_facts_round_robin_newest_first_and_limit(): + facts = [ + _fact(1, "round", "r-old", year=2008), + _fact(2, "round", "r-new", year=2023), + _fact(3, "event", "e", year=2015), + _fact(4, "round", "r-mid", year=2015), + _fact(5, "stance_split", "s"), + _fact(6, "payment", "p"), + _fact(7, "round", "flag", note="x"), + _fact(8, "round", "elsewhere", codes=("A1",), year=2000), ] - assert kinds_at == sorted(kinds_at) - # flagged / reviewed facts are never given to the model; payment rows - # come through the Valuation block - assert "flagged paraphrase" not in block - assert "a finding" not in block - assert "payment row" in block # lowest priority, still within the cap + assert match_facts(facts, (), ("99341",), ()) == ("F2", "F3", "F4", "F1") + # a named year also matches; the limit applies after interleaving + assert match_facts(facts, (), ("zzz",), {2000}, limit=1) == ("F8",) + # nothing matches -> every candidate + assert match_facts(facts, ("F2",), ("zzz",), ()) == ("F4", "F3", "F1", "F8") -def test_prompt_block_cap_keeps_stance_with_accuracy(replica): - ev = dossier_evidence("home visits?", replica) - block = ev.prompt_block(max_rows=1) - assert "[L2]" in block and "[L3]" not in block # capped - # every stance count stays, each with its accuracy sentence - assert block.count("Preliminary model classification") == 2 - assert "Of 3 CY2018 commenters we classified, 3 supported." in block +def test_question_years(): + assert question_years("CY2023 vs 2008, and cy 2015?") == {2023, 2008, 2015} + + +def test_stance_unit_has_one_header_and_no_repeated_accuracy(replica): + block = _ev(replica, "home visits?").prompt_block() + assert block.count("Preliminary model classification") == 1 + head = [ln for ln in block.splitlines() if ln.startswith("Our own")][0] + assert ACC in head for ln in block.splitlines(): if "commenters we classified" in ln: - assert "Preliminary model classification" in ln + assert "Preliminary" not in ln and ln.endswith("]") + assert block.index(head) < block.index("Of 3 CY2023 commenters") -def test_stance_count_without_accuracy_gets_dossier_accuracy_line(replica): - ev = dossier_evidence("home visits?", replica) - block = ev.prompt_block() - line = [ln for ln in block.splitlines() if "CY2018 commenters" in ln][0] - assert ACC in line and line.endswith("[L8]") - - -def test_stance_count_without_any_accuracy_says_unmeasured(): +def test_stance_header_says_unmeasured_without_accuracy(): d = _dossier(accuracy="") ev = DossierEvidence( - d.family, - d.name, - "u", - d.summary, - tuple(d.facts.values()), - (), - accuracy="", + d.family, d.name, "u", d.summary, tuple(d.facts.values()), (), accuracy="" ) - line = [ln for ln in ev.prompt_block().splitlines() if "CY2018" in ln][0] - assert "accuracy is not yet measured" in line + assert "accuracy is not yet measured" in ev.prompt_block() + + +def test_no_stance_facts_no_unit(): + ev = DossierEvidence("F", "N", "u", "s", (), ()) + assert ev.prompt_block().splitlines()[1] == "Summary: s" + assert "Our own" not in ev.prompt_block() + + +def test_max_rows_caps_matched_facts(replica): + ev = _ev(replica, "home visits?") + n = len(ev.prompt_block(max_rows=3).splitlines()) + assert n == len(ev.prompt_block(max_rows=0).splitlines()) + 3 + + +def test_max_chars_sheds_matched_then_stance_then_old_timeline(replica): + ev = _ev(replica, "home visits?") + full = ev.prompt_block() + no_matched = ev.prompt_block(max_rows=0) + assert "Our own" in no_matched + # a limit that only the matched facts must give way to + cut = ev.prompt_block(max_chars=len(no_matched) + 5) + assert len(cut) <= len(no_matched) + 5 and "Our own" in cut + # smaller: the whole stance unit goes (never counts without the header) + tiny = ev.prompt_block(max_chars=1) + assert "Our own" not in tiny and "commenters we classified" not in tiny + assert tiny.count("\nCY") == 2 and len(tiny) < len(full) + + +def test_max_chars_sheds_timeline_down_to_the_floor(): + tl = tuple( + TimelineEvent(2000 + i, "k", (), f"event {i} [F1]", ("F1",), (), ()) + for i in range(12) + ) + ev = DossierEvidence("F", "N", "u", "s", (_fact(1, "round", "r"),), (), timeline=tl) + block = ev.prompt_block(max_chars=1) + assert block.count("\nCY") == TIMELINE_FLOOR + assert "CY2011:" in block and "CY2003:" not in block # oldest went first + assert ev.prompt_block(floor=True) == block + assert ev.prompt_block().count("\nCY") == 12 def test_dossier_url_uses_env_domain(monkeypatch): diff --git a/tests/llm/test_evidence.py b/tests/llm/test_evidence.py index cef0267..daa5278 100644 --- a/tests/llm/test_evidence.py +++ b/tests/llm/test_evidence.py @@ -267,13 +267,17 @@ class TestValuationForDossierFamilies: _row(code="99341", year=2023 + i, proposed=True, label=f"[P{i}]") for i in range(2) ] - mock_val.return_value = (rows, []) + # the default window: only the newest final rows + recent = [_row(code="99341", year=2025 + i, label=f"[R{i}]") for i in range(2)] + mock_val.side_effect = [(recent, []), (rows, [])] cfg = replace(CFG, valuation_rows_max=10) ev = valuation_evidence("How were home visits (99341) valued?", cfg) assert mock_val.call_args.kwargs == {"years": 0, "all_proposed": True} assert len(ev.payload()["rows"]) == 15 - block_rows = ev.prompt_block().count("99341 ") - assert block_rows == 10 <= cfg.valuation_rows_max + block = ev.prompt_block() + # the prompt selects from the default-years rows, never the history + assert block.count("99341 ") == 2 + assert "[R0]" in block and "[R1]" in block and "[Y0]" not in block @patch("llm.evidence.dossier_families", return_value=("HOME-VISITS",)) @patch("llm.evidence.valuation", return_value=([], [])) diff --git a/tests/llm/test_rag.py b/tests/llm/test_rag.py index da9a8b6..a62cf7a 100644 --- a/tests/llm/test_rag.py +++ b/tests/llm/test_rag.py @@ -1380,6 +1380,7 @@ def _dossier_ev(n_facts=3): "Summary [DL0].", facts, ("a gap",), + matched=tuple(f.id for f in facts), ) @@ -1467,8 +1468,26 @@ class TestDossierInChat: @patch("llm.rag.valuation_evidence", return_value=None) @patch("llm.rag.httpx.Client") @patch("llm.rag.retrieve") - def test_control_question_byte_identical(self, mock_retrieve, MockClient, _ev): - cfg = replace(CFG, duckdb_replica="/nonexistent/aco.ro.duckdb") + def test_control_question_byte_identical( + self, mock_retrieve, MockClient, _ev, tmp_path + ): + """A temp replica really holds the HOME-VISITS dossier; a question + with no dossier family must send the model and the client the same + bytes as a run with the dossier lookup switched off.""" + import duckdb + + from llm import evidence + from pfs.codetables import ensure_tables + from pfs.dossier import write_dossier + from tests.llm.test_dossier import _dossier + + path = tmp_path / "aco.ro.duckdb" + con = duckdb.connect(str(path)) + ensure_tables(con) + write_dossier(con, _dossier()) + con.close() + evidence._REPLICA = None + cfg = replace(CFG, duckdb_replica=str(path)) mock_retrieve.return_value = [] client = MockClient.return_value.__enter__.return_value resp = client.stream.return_value.__enter__.return_value @@ -1477,18 +1496,28 @@ class TestDossierInChat: resp.iter_lines.return_value = iter( ['{"message":{"content":"x"},"done":true}'] ) - return json.dumps( + events = json.dumps( list( stream_answer( "why did CMS finalize this?", cfg=cfg, pool=self._pool() ) ) ) + sent = json.dumps(client.stream.call_args.kwargs["json"]["messages"]) + return events, sent - real = run() - with patch("llm.rag.dossier_evidence", return_value=None): - off = run() + try: + from llm.dossier import dossier_evidence + + # the family really is there: a home-visit question finds it + assert dossier_evidence("how were home visits valued?", cfg) is not None + real = run() + with patch("llm.rag.dossier_evidence", return_value=None): + off = run() + finally: + evidence._REPLICA = None assert real == off + assert "Dossier" not in real[1].split("Question:")[0].split("Excerpts:")[-1] class TestDossierPromptBlock: @@ -1510,18 +1539,78 @@ class TestDossierPromptBlock: def test_no_dossier_prompt_unchanged(self): assert build_messages("q", []) == build_messages("q", [], dossier=None) - def test_dossier_rows_trimmed_to_floor_8_as_a_last_resort(self): - d = replace(_dossier_ev(40), max_prompt_rows=30) + def test_block_takes_at_most_35_percent_of_the_budget(self): + d = _dossier_ev(40) + d = replace(d, matched=tuple(f.id for f in d.facts)) big = build_messages("q", [], dossier=d)[1]["content"] - assert big.count("[DL") == 30 - msgs = build_messages("q", [], dossier=d, budget_chars=len(big) - 1) - assert msgs[1]["content"].count("[DL") == 8 + assert big.count("[DL") > 30 + budget = len(big) + 1000 # roomy: only the 35% share bites + msgs = build_messages("q", [], dossier=d, budget_chars=budget) + block = msgs[1]["content"].split("\n\n")[1] + assert len(block) <= int(budget * 0.35) - def test_dossier_cap_untouched_when_budget_met_earlier(self): - d = _dossier_ev(10) - big = build_messages("q", [], dossier=d)[1]["content"] - msgs = build_messages("q", [], dossier=d, budget_chars=len(big) + 1) - assert msgs[1]["content"].count("[DL") == 10 + def test_dossier_floor_is_summary_plus_eight_timeline_lines(self): + from pfs.dossier import TimelineEvent + + d = replace( + _dossier_ev(10), + matched=tuple(f"F{i}" for i in range(10)), + timeline=tuple( + TimelineEvent(2000 + i, "k", (), f"ev {i} [F0]", ("F0",), (), ()) + for i in range(12) + ), + ) + msgs = build_messages("q", [], dossier=d, budget_chars=1) + block = msgs[1]["content"] + assert block.count("\nCY") == 8 and "fact 0. [DL0]" not in block + assert "Summary:" in block + + def test_dossier_shed_before_lineage_rows(self): + from pfs.dossier import TimelineEvent + + events = tuple( + LineageEvent( + f"C{i}", + 2020, + "revalued", + (), + (), + f"[L{i}]", + "K", + i, + 0, + "", + "fr", + True, + "", + ) + for i in range(20) + ) + le = LineageEvidence( + codes=("C",), + families=(), + events=events, + element_diffs=(), + guidance=(), + max_prompt_rows=20, + ) + d = replace( + _dossier_ev(10), + matched=tuple(f"F{i}" for i in range(10)), + timeline=tuple( + TimelineEvent(2000 + i, "k", (), f"ev {i} [F0]", ("F0",), (), ()) + for i in range(12) + ), + ) + full = build_messages("q", [], lineage=le, dossier=d)[1]["content"] + # a budget that the dossier floor alone satisfies keeps every lineage row + floor = build_messages("q", [], lineage=le, dossier=d, budget_chars=1) + assert floor[1]["content"].count("[L") < full.count("[L") # last resort bites + mid = build_messages( + "q", [], lineage=le, dossier=d, budget_chars=len(full) - 200 + )[1]["content"] + assert mid.count("[L") == full.count("[L") + assert mid.count("\nCY") == 8 def test_system_prompt_names_the_dossier_rules(self): s = build_messages("q", [])[0]["content"]