fix(llm): dossier prompt block is question-aware and compact — summary, timeline, matched facts, one stance unit; 35% budget share; default-years valuation prompt rows
Some checks failed
CI / lint (push) Successful in 1m16s
CI / notebooks-smoke (push) Successful in 2m0s
Deploy / changes (push) Successful in 15s
CI / test (push) Failing after 4m21s
Infra CI / zotero (push) Successful in 17s
Infra CI / notebooks (push) Successful in 1m4s
Infra CI / docs (push) Successful in 28s
Infra CI / api (push) Successful in 1m53s
Infra CI / bcda (push) Successful in 2m55s
Infra CI / llm (push) Successful in 1m42s
Infra CI / mc (push) Successful in 26s
Package Supply Chain / pkg-supply-chain (push) Successful in 2m42s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Successful in 1m49s
Deploy / bcda (push) Successful in 2m9s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 1m24s
Deploy / report (push) Successful in 17s

This commit is contained in:
kert
2026-11-13 00:55:29 -05:00
parent e250c516ed
commit a60baf2a84
8 changed files with 495 additions and 141 deletions

View File

@@ -15,24 +15,29 @@ import os
import re
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from typing import Any, Collection, Sequence
import llm.evidence as evidence
from llm.config import LlmConfig
from pfs.dossier import Fact, dossier_families, read_dossier, stance_accuracy_line
from pfs.dossier import (
Fact,
TimelineEvent,
dossier_families,
read_dossier,
stance_accuracy_line,
)
from pfs.families import CURATED_FAMILIES, detect_codes
__all__ = ["DossierEvidence", "dossier_evidence", "dossier_url"]
log = logging.getLogger(__name__)
#: prompt priority: what the model is most likely asked about first. Payment
#: facts come last; the Valuation block is the authority on payment.
_PRIORITY = (
#: round-robin order of the question-matched facts. Payment facts are left
#: to the Valuation block; stance counts form their own unit.
_KIND_ORDER = (
"round",
"event",
"cms_summary",
"stance_split",
"link",
"origin",
"formation",
@@ -41,17 +46,59 @@ _PRIORITY = (
"impact",
"seat",
"finding",
"payment",
)
_RANK = {k: i for i, k in enumerate(_PRIORITY)}
_ACCURACY_MARK = "Preliminary model classification"
_TAG = re.compile(r"\[(F\d+)\]")
_YEAR = re.compile(r"\b(?:CY\s?)?(20\d\d|19\d\d)\b", re.I)
#: summary + this many timeline lines survive the hardest trim
TIMELINE_FLOOR = 8
_HEADER = (
"Dossier — cited facts about the {name} family (authoritative for its "
"history, valuation rounds and who commented). Each line ends with the "
"label to copy:"
)
_STANCE_HEADER = (
"Our own classification of commenters (never CMS's counts). Accuracy of "
"every count below: {acc}"
)
def question_years(question: str) -> frozenset[int]:
"""Years the question names ("2008", "CY2023")."""
return frozenset(int(m) for m in _YEAR.findall(question))
def match_facts(
facts: Sequence[Fact],
flagged: Sequence[str],
codes: Sequence[str],
years: Collection[int],
limit: int = 0,
) -> tuple[str, ...]:
"""Ids of the facts that bear on the question, interleaved round-robin
by kind with the newest year first. A fact matches when its codes overlap
*codes* or its year is one of *years*; when nothing matches every
unflagged fact is a candidate. Flagged, stance and payment facts never
appear (stance counts are their own unit, payment rows are the Valuation
block's)."""
bad = set(flagged)
pool = [
f for f in facts if f.kind in _KIND_ORDER and not f.note and f.id not in bad
]
want = set(codes)
hit = [f for f in pool if (want & set(f.codes)) or (f.year in years)]
pool = hit or pool
by_kind: dict[str, list[Fact]] = {k: [] for k in _KIND_ORDER}
for f in pool:
by_kind[f.kind].append(f)
queues = [sorted(v, key=lambda f: -(f.year or 0)) for v in by_kind.values() if v]
out: list[str] = []
while any(queues):
for q in queues:
if q:
out.append(q.pop(0).id)
return tuple(out[:limit] if limit else out)
def dossier_url(family: str, cfg: LlmConfig) -> str:
@@ -68,29 +115,61 @@ class DossierEvidence:
summary: str
facts: tuple[Fact, ...]
gaps: tuple[str, ...]
#: matched facts kept in the prompt block (the timeline and the stance
#: unit are sized by characters and the budget trim instead)
max_prompt_rows: int = 30
#: the dossier's own stance-accuracy sentence, appended to any stance
#: count whose text does not carry one (a count never stands alone)
#: the dossier's own stance-accuracy sentence: the header of the stance
#: unit (a count never stands without it)
accuracy: str = ""
#: ids of facts shown flagged on the page (unreviewed model paraphrases);
#: never given to the model
flagged: tuple[str, ...] = ()
timeline: tuple[TimelineEvent, ...] = ()
#: ``match_facts`` result for the question, in prompt order
matched: tuple[str, ...] = ()
def payload(self) -> dict[str, Any]:
def _labelled(self, text: str) -> str:
label = {f.id: f.cite.label for f in self.facts}
def tag(m: re.Match[str]) -> str:
return f"[{label[m.group(1)]}]" if m.group(1) in label else m.group(0)
summary = _TAG.sub(tag, self.summary)
out = _TAG.sub(tag, text)
# a run of the same cite ("[L2][L2]") reads once
summary = re.sub(r"(\[[^\[\]]+\])(?:\1)+", r"\1", summary)
return re.sub(r"(\[[^\[\]]+\])(?:\1)+", r"\1", out)
def _timeline_lines(self) -> list[str]:
return [
f"CY{e.year}: {self._labelled(e.what_changed)}"
for e in sorted(self.timeline, key=lambda e: -e.year)
]
def _stance(self) -> list[Fact]:
return [f for f in self.facts if f.kind == "stance_split"]
def _prompt_fact_ids(self, rows: int | None = None) -> list[str]:
n = self.max_prompt_rows if rows is None else rows
return list(self.matched[: max(n, 0)])
def payload(self) -> dict[str, Any]:
"""Facts cited by the summary or the timeline plus those the prompt
block carries (same shape as the brief, plus ``review``)."""
by_id = {f.id: f for f in self.facts}
wanted: set[str] = set()
for text in [self.summary, *(e.what_changed for e in self.timeline)]:
wanted.update(_TAG.findall(text))
for e in self.timeline:
wanted.update(e.stance_fact_ids)
wanted.update(e.cms_summary_fact_ids)
wanted.update(self._prompt_fact_ids())
wanted.update(f.id for f in self._stance())
flagged = set(self.flagged)
return {
"type": "dossier",
"family": self.family,
"name": self.name,
"url": self.url,
"summary": summary,
"summary": self._labelled(self.summary),
"facts": [
{
"label": f.cite.label,
@@ -98,41 +177,67 @@ class DossierEvidence:
"url": f.cite.url,
"year": f.year,
"kind": f.kind,
"review": bool(f.note) or f.id in flagged,
}
for f in self.facts
if f.id in wanted and f.id in by_id
],
"gaps": list(self.gaps),
}
def _line(self, f: Fact) -> str:
text = f.text
if f.kind == "stance_split" and _ACCURACY_MARK not in text:
text = f"{text} {self.accuracy or stance_accuracy_line(None)}"
return f"{text} [{f.cite.label}]"
def _stance_unit(self) -> list[str]:
stance = self._stance()
if not stance:
return []
acc = self.accuracy or stance_accuracy_line(None)
lines = [_STANCE_HEADER.format(acc=acc)]
for f in stance:
text = f.text.split(f" {_ACCURACY_MARK}")[0]
lines.append(f"{text} [{f.cite.label}]")
return lines
def prompt_block(self, max_rows: int | None = None) -> str:
"""*max_rows* (default ``max_prompt_rows``) caps the rows other than
stance counts; every stance count stays, each with its accuracy
sentence."""
cap = self.max_prompt_rows if max_rows is None else max_rows
flagged = set(self.flagged)
usable = [
(i, f)
for i, f in enumerate(self.facts)
if not f.note and f.id not in flagged
def prompt_block(
self,
max_rows: int | None = None,
max_chars: int | None = None,
floor: bool = False,
) -> str:
"""Summary, timeline (newest first), the question's matched facts and
one stance unit. *max_rows* caps the matched facts; *max_chars* sheds
matched facts, then the stance unit (all or none), then the oldest
timeline lines down to ``TIMELINE_FLOOR``; *floor* is the budget
trim's last step: the summary and the newest ``TIMELINE_FLOOR``
timeline lines only."""
by_id = {f.id: f for f in self.facts}
summary = f"Summary: {self._labelled(self.summary)}"
tl = self._timeline_lines()
matched = [
f"{f.text} [{f.cite.label}]"
for f in (by_id[i] for i in self._prompt_fact_ids(max_rows) if i in by_id)
]
usable.sort(key=lambda p: (_RANK.get(p[1].kind, len(_RANK)), p[0]))
kept: list[Fact] = []
others = 0
for _i, f in usable:
if f.kind == "stance_split":
kept.append(f)
elif others < cap:
kept.append(f)
others += 1
lines = [_HEADER.format(name=self.name)]
lines.extend(self._line(f) for f in kept)
return "\n".join(lines)
stance = self._stance_unit()
def build(n_tl: int, n_m: int, with_stance: bool) -> str:
lines = [_HEADER.format(name=self.name), summary]
lines.extend(tl[:n_tl])
lines.extend(matched[:n_m])
if with_stance:
lines.extend(stance)
return "\n".join(lines)
if floor:
return build(TIMELINE_FLOOR, 0, False)
n_tl, n_m, with_stance = len(tl), len(matched), bool(stance)
while max_chars is not None and len(build(n_tl, n_m, with_stance)) > max_chars:
if n_m:
n_m -= 1
elif with_stance:
with_stance = False
elif n_tl > TIMELINE_FLOOR:
n_tl -= 1
else:
break
return build(n_tl, n_m, with_stance)
def dossier_evidence(question: str, cfg: LlmConfig) -> DossierEvidence | None:
@@ -165,14 +270,23 @@ def dossier_evidence(question: str, cfg: LlmConfig) -> DossierEvidence | None:
cur.close()
if d is None:
return None
facts = tuple(d.facts.values())
flagged = tuple(r["fact_id"] for r in d.review)
return DossierEvidence(
family=d.family,
name=d.name,
url=dossier_url(d.family, cfg),
summary=d.summary,
facts=tuple(d.facts.values()),
facts=facts,
gaps=tuple(d.gaps),
max_prompt_rows=cfg.dossier_max_rows,
accuracy=d.stance_accuracy,
flagged=tuple(r["fact_id"] for r in d.review),
flagged=flagged,
timeline=tuple(d.timeline),
matched=match_facts(
facts,
flagged,
det.explicit or det.codes,
question_years(question),
),
)

View File

@@ -122,6 +122,9 @@ class ValuationEvidence:
#: ``codes``) — carried through so ``rag.stream_answer`` doesn't
#: need a third ``detect_codes`` call just to find them.
wide: tuple[str, ...] = ()
#: the default-years rows the prompt block selects from when the payload
#: carries a dossier family's whole history (``None``: ``rows`` itself)
prompt_rows: tuple[ValuationRow, ...] | None = None
def prompt_block(self, max_rows: int | None = None) -> str:
"""*max_rows* overrides ``self.max_prompt_rows`` for one call —
@@ -129,7 +132,8 @@ class ValuationEvidence:
smaller block on demand without needing a new ``ValuationEvidence``."""
cap = self.max_prompt_rows if max_rows is None else max_rows
lines = [_HEADER]
for r in _cap_valuation_rows(self.rows, self.explicit, cap):
source = self.rows if self.prompt_rows is None else self.prompt_rows
for r in _cap_valuation_rows(source, self.explicit, cap):
# The status note says why an unpaid code shows no dollars, and
# the CF note says which of the year's CFs the money came from.
status = f"{r.status or '?'}" + (
@@ -265,14 +269,19 @@ def valuation_evidence(question: str, cfg: LlmConfig) -> ValuationEvidence | Non
# A family with a dossier gets its whole payment history (every final
# year, every proposed round) in the payload; the prompt block still
# keeps only ``valuation_rows_max`` rows.
kwargs: dict[str, Any] = {"years": years}
full = False
try:
if set(det.families) & set(dossier_families(cur)):
kwargs = {"years": 0, "all_proposed": True}
full = bool(set(det.families) & set(dossier_families(cur)))
except Exception as e: # noqa: BLE001 — the valuation must still answer
log.warning("dossier families unavailable for valuation: %s", e)
try:
rows, unpriced = valuation(cur, list(codes), **kwargs)
rows, unpriced = valuation(cur, list(codes), years=years)
prompt_rows = None
if full:
# the prompt picks from the default window, so the whole history
# never pushes the newest final rows out of its row cap
prompt_rows = tuple(rows)
rows, unpriced = valuation(cur, list(codes), years=0, all_proposed=True)
except Exception as e: # noqa: BLE001
log.warning("valuation evidence skipped (query): %s", e)
return None
@@ -286,6 +295,7 @@ def valuation_evidence(question: str, cfg: LlmConfig) -> ValuationEvidence | Non
explicit=det.explicit,
max_prompt_rows=cfg.valuation_rows_max,
wide=det.wide,
prompt_rows=prompt_rows,
)

View File

@@ -235,7 +235,7 @@ _CITE_REMINDER = (
"sentence that states a date, a code being created, adopted, replaced or "
"deleted, an RVU, a conversion factor, a payment amount, or what a rule "
"or a commenter said MUST end with the bracketed label of the Lineage "
"line, Valuation row or excerpt it comes from, copied exactly — for "
"line, Dossier line, Valuation row or excerpt it comes from, copied exactly — for "
'example: "G2058 was replaced by CPT 99439 for CY2021 [CY2021 PFS final '
'85 FR 84547 ¶686]." Do not write such a sentence without its label. '
"Take the year of each fact from its Lineage line, never from a "
@@ -454,7 +454,8 @@ _BUDGET_RETRIEVED_KEEP = 6
_BUDGET_CITED_KEEP = 8
_BUDGET_MANUAL_KEEP = 1
_BUDGET_VALUATION_ROWS_KEEP = 12
_BUDGET_DOSSIER_ROWS_FLOOR = 8
#: share of the prompt budget the dossier block may take
_DOSSIER_BUDGET_SHARE = 0.35
# The model counts its context in tokens; the excerpts are trimmed in
# characters before any model sees them. ``prompt_budget_chars`` converts
@@ -540,7 +541,8 @@ def _user_message(
valuation_rows_cap: int | None,
lineage_rows_cap: int | None,
dossier: DossierEvidence | None = None,
dossier_rows_cap: int | None = None,
dossier_chars: int | None = None,
dossier_floor: bool = False,
) -> str:
sources = retrieved + cited + lineage_src + manual + rest
if sources:
@@ -558,7 +560,7 @@ def _user_message(
context = "(no relevant excerpts found)"
parts = [f"Excerpts:\n\n{context}"]
if dossier is not None:
parts.append(dossier.prompt_block(dossier_rows_cap))
parts.append(dossier.prompt_block(None, dossier_chars, dossier_floor))
if lineage is not None:
parts.append(lineage.prompt_block(lineage_rows_cap))
if evidence is not None:
@@ -601,8 +603,8 @@ def build_messages(
budget_chars: int | None = None,
) -> list[dict]:
"""Grounded chat messages: system rules + question with excerpts and,
when available, a Lineage block and a Valuation block — in that
order — between the excerpts and the question.
when available, a Dossier block, a Lineage block and a Valuation
block — in that order — between the excerpts and the question.
*sources* is assumed laid out contiguously in the order
``stream_answer`` appends them — retrieved, then cited
@@ -635,7 +637,13 @@ def build_messages(
valuation_cap = evidence.max_prompt_rows if evidence is not None else None
lineage_cap = lineage.max_prompt_rows if lineage is not None else None
dossier_cap = dossier.max_prompt_rows if dossier is not None else None
# the dossier block never takes more than a third of the budget
dossier_chars = (
int(budget_chars * _DOSSIER_BUDGET_SHARE)
if dossier is not None and budget_chars is not None
else None
)
dossier_floor = False
def content() -> str:
return _user_message(
@@ -650,7 +658,8 @@ def build_messages(
valuation_cap,
lineage_cap,
dossier,
dossier_cap,
dossier_chars,
dossier_floor,
)
if budget_chars is not None and len(content()) > budget_chars:
@@ -711,6 +720,16 @@ def build_messages(
before - budget_chars,
budget_chars,
)
# The dossier block, before the lineage rows: down to the summary
# and the newest TIMELINE_FLOOR timeline lines.
if len(content()) > budget_chars and dossier is not None and not dossier_floor:
before = len(content())
dossier_floor = True
log.info(
"prompt budget: cut the dossier block to its floor (%d chars over %d)",
before - budget_chars,
budget_chars,
)
if len(content()) > budget_chars and lineage is not None:
target = lineage.max_prompt_rows
half = max(target // 2, 1)
@@ -723,22 +742,6 @@ def build_messages(
before - budget_chars,
budget_chars,
)
# Last of all, the dossier's rows (stance counts always stay),
# down to a floor of 8.
if (
len(content()) > budget_chars
and dossier is not None
and (dossier_cap is None or dossier_cap > _BUDGET_DOSSIER_ROWS_FLOOR)
):
before = len(content())
dossier_cap = _BUDGET_DOSSIER_ROWS_FLOOR
log.info(
"prompt budget: capped dossier rows to %d (%d chars over %d)",
dossier_cap,
before - budget_chars,
budget_chars,
)
return [
{"role": "system", "content": _SYSTEM},
{"role": "user", "content": content()},
@@ -794,6 +797,12 @@ def stream_answer(
) -> Iterator[dict]:
"""Retrieve, then stream a grounded answer from the largest live host.
When the detected family has a stored dossier, a ``dossier`` event
(``llm.dossier``) comes first and a Dossier block (summary, timeline,
question-matched facts, one stance unit) leads the prompt's evidence
blocks, capped at a third of the prompt budget; ``sources`` then starts
with a link entry for the dossier page.
``mode`` ("auto"/"timeline"/"recent") is forwarded to ``retrieve``;
"auto" (the default) switches to era-balanced retrieval on its own
for a history-shaped question (``is_history_question``).

View File

@@ -486,7 +486,7 @@
if ((ev.gaps || []).length) {
const g = document.createElement('div');
g.className = 'provenance';
g.textContent = 'Not yet captured: ' + ev.gaps.join(' ');
g.textContent = 'Gaps and notes: ' + ev.gaps.join(' ');
box.appendChild(g);
}
if (ev.url) {

View File

@@ -60,6 +60,7 @@ class TestIndex:
assert "function renderDossier" in html
assert "ev.type === 'dossier'" in html
assert "Open the full dossier" in html
assert "Gaps and notes" in html and "Not yet captured" not in html
assert "tr.proposed" in html
assert "rows.length > 40" in html
assert "<details" in html or "createElement('details')" in html

View File

@@ -10,10 +10,17 @@ import pytest
from llm import evidence
from llm.config import LlmConfig
from llm.dossier import DossierEvidence, dossier_evidence, dossier_url
from llm.dossier import (
TIMELINE_FLOOR,
DossierEvidence,
dossier_evidence,
dossier_url,
match_facts,
question_years,
)
from pfs.cite import Cite
from pfs.codetables import ensure_tables
from pfs.dossier import Dossier, Fact, write_dossier
from pfs.dossier import Dossier, Fact, TimelineEvent, write_dossier
CFG = LlmConfig(
ollama_hosts=("http://h1:11434",),
@@ -36,12 +43,12 @@ ACC = (
)
def _fact(i, kind, text, year=2023, label=None, note=""):
def _fact(i, kind, text, year=2023, label=None, note="", codes=("99341",)):
return Fact(
id=f"F{i}",
kind=kind,
year=year,
codes=("99341",),
codes=codes,
text=text,
values={},
cite=Cite("fr", label or f"L{i}", f"https://x/{i}"),
@@ -50,10 +57,17 @@ def _fact(i, kind, text, year=2023, label=None, note=""):
def _dossier(extra=(), accuracy=ACC):
old = ("99324", "99325")
facts = [
_fact(1, "payment", "payment row"),
_fact(2, "round", "CMS finalized work RVU 1.0 for 99341 for CY2023."),
_fact(3, "event", "99324 was deleted for CY2023."),
_fact(
3,
"event",
"99324 was deleted for CY2023.",
label="DEL23",
codes=old,
),
_fact(
4,
"stance_split",
@@ -69,8 +83,39 @@ def _dossier(extra=(), accuracy=ACC):
_fact(7, "round", "flagged paraphrase", note="model paraphrase, unreviewed"),
_fact(8, "stance_split", "Of 3 CY2018 commenters we classified, 3 supported."),
_fact(9, "finding", "a finding"),
# live-shaped: far more CY2008 rounds than any cap
*(
_fact(
100 + i,
"round",
f"For CY2008, CMS finalized work RVU {i} for 99326.",
year=2008,
codes=("99326",),
)
for i in range(40)
),
*extra,
]
tl = [
TimelineEvent(
2008,
"valuation",
("99326",),
"For CY2008, CMS finalized RVUs [F100].",
("F100",),
(),
(),
),
TimelineEvent(
2023,
"deletion",
old,
"99324 and 99325 were deleted for CY2023 [F3].",
("F3",),
(),
(),
),
]
return Dossier(
family="HOME-VISITS",
name="Home or Residence Services",
@@ -78,7 +123,7 @@ def _dossier(extra=(), accuracy=ACC):
status="active",
years_active="2008-",
summary="CMS finalized it [F2][F2] and deleted codes [F3].",
timeline=[],
timeline=tl,
rounds=[],
payment=[],
utilization=[],
@@ -115,7 +160,8 @@ def test_dossier_evidence_for_home_visit_question(replica):
assert ev.name == "Home or Residence Services"
assert ev.url == "https://lex.fhirworx.io/?family=HOME-VISITS#dossier"
assert ev.max_prompt_rows == 30
assert len(ev.facts) == 9
assert len(ev.facts) == 49
assert len(ev.timeline) == 2
assert ev.gaps == ("RVU files before 2015 not yet loaded.",)
@@ -155,7 +201,9 @@ def test_first_family_curated_before_derived(replica, monkeypatch):
monkeypatch.setattr(
mod,
"detect_codes",
lambda q: SimpleNamespace(families=("AAA-DERIVED", "HOME-VISITS", "ZZZ")),
lambda q: SimpleNamespace(
families=("AAA-DERIVED", "HOME-VISITS", "ZZZ"), explicit=(), codes=()
),
)
monkeypatch.setattr(mod, "CURATED_FAMILIES", {"HOME-VISITS": 1})
seen = []
@@ -176,11 +224,26 @@ def test_dossier_payload_shape_and_summary_labels(replica):
assert p["family"] == "HOME-VISITS"
assert p["url"] == "https://lex.fhirworx.io/?family=HOME-VISITS#dossier"
# [F#] became the facts' labels; the doubled cite collapsed
assert p["summary"] == "CMS finalized it [L2] and deleted codes [L3]."
assert {"label", "text", "url", "year", "kind"} == set(p["facts"][0])
assert p["summary"] == "CMS finalized it [L2] and deleted codes [DEL23]."
assert {"label", "text", "url", "year", "kind", "review"} == set(p["facts"][0])
assert p["gaps"] == ["RVU files before 2015 not yet loaded."]
def test_payload_carries_only_cited_and_selected_facts(replica):
ev = dossier_evidence("home visits?", replica)
p = ev.payload()
labels = {f["label"] for f in p["facts"]}
assert {"L2", "DEL23", "L100"} <= labels # summary + timeline cites
assert "L1" not in labels # payment rows stay on the page
assert len(p["facts"]) < len(ev.facts)
assert all(f["review"] is False for f in p["facts"])
# a flagged fact cited by the timeline travels flagged
ev2 = replace(ev, flagged=("F3",))
assert [f["review"] for f in ev2.payload()["facts"] if f["label"] == "DEL23"] == [
True
]
def test_payload_unknown_fact_tag_kept(replica):
ev = dossier_evidence("home visits?", replica)
ev = replace(ev, summary="see [F999].")
@@ -195,8 +258,12 @@ def test_payload_names_come_from_own_records(replica):
assert seat["label"] == "CMS-2022-0113-0001"
def test_prompt_block_lines_end_with_labels(replica):
ev = dossier_evidence("home visits?", replica)
def _ev(replica_cfg, question):
return dossier_evidence(question, replica_cfg)
def test_prompt_block_order_and_labels(replica):
ev = _ev(replica, "When were domiciliary visit codes 99324 and 99325 deleted?")
block = ev.prompt_block()
lines = block.splitlines()
assert lines[0] == (
@@ -204,53 +271,113 @@ def test_prompt_block_lines_end_with_labels(replica):
"(authoritative for its history, valuation rounds and who commented). "
"Each line ends with the label to copy:"
)
body = lines[1:]
assert body and all(ln.endswith("]") for ln in body)
assert "CMS finalized work RVU 1.0 for 99341 for CY2023. [L2]" in body
# priority order: round, event, cms_summary, stance_split, ... seat, finding
kinds_at = [
block.index(t) for t in ("[L2]", "[L3]", "[L5]", "[L4]", "[CMS-2022-0113-0001]")
assert lines[1] == ("Summary: CMS finalized it [L2] and deleted codes [DEL23].")
# timeline, newest first, in the dossier's own prose with labels
assert lines[2] == "CY2023: 99324 and 99325 were deleted for CY2023 [DEL23]."
assert lines[3] == "CY2008: For CY2008, CMS finalized RVUs [L100]."
# then the question's matched facts, then the stance unit
assert block.index("99324 was deleted for CY2023. [DEL23]") > block.index("CY2008:")
assert block.index("Our own classification") > block.index("[DEL23]")
assert "flagged paraphrase" not in block and "a finding" not in block
assert "payment row" not in block
# every line but the stance header ends with a label
for ln in lines:
assert ln.rstrip(".").endswith("]") or ln.startswith(
("Dossier —", "Our own classification")
)
def test_deletion_question_surfaces_cy2023_over_cy2008_rounds(replica):
"""The live bug: 30 CY2008 round lines crowded out every other fact."""
block = _ev(replica, "When were 99324-99325 deleted?").prompt_block()
assert "99324 was deleted for CY2023. [DEL23]" in block
assert block.count("For CY2008, CMS finalized work RVU") < 10
def test_matching_is_question_aware(replica):
ev = _ev(replica, "How did CMS value 99326 in 2008?")
first = [f for f in ev.facts if f.id == ev.matched[0]][0]
assert first.year == 2008 and "99326" in first.codes
def test_match_facts_round_robin_newest_first_and_limit():
facts = [
_fact(1, "round", "r-old", year=2008),
_fact(2, "round", "r-new", year=2023),
_fact(3, "event", "e", year=2015),
_fact(4, "round", "r-mid", year=2015),
_fact(5, "stance_split", "s"),
_fact(6, "payment", "p"),
_fact(7, "round", "flag", note="x"),
_fact(8, "round", "elsewhere", codes=("A1",), year=2000),
]
assert kinds_at == sorted(kinds_at)
# flagged / reviewed facts are never given to the model; payment rows
# come through the Valuation block
assert "flagged paraphrase" not in block
assert "a finding" not in block
assert "payment row" in block # lowest priority, still within the cap
assert match_facts(facts, (), ("99341",), ()) == ("F2", "F3", "F4", "F1")
# a named year also matches; the limit applies after interleaving
assert match_facts(facts, (), ("zzz",), {2000}, limit=1) == ("F8",)
# nothing matches -> every candidate
assert match_facts(facts, ("F2",), ("zzz",), ()) == ("F4", "F3", "F1", "F8")
def test_prompt_block_cap_keeps_stance_with_accuracy(replica):
ev = dossier_evidence("home visits?", replica)
block = ev.prompt_block(max_rows=1)
assert "[L2]" in block and "[L3]" not in block # capped
# every stance count stays, each with its accuracy sentence
assert block.count("Preliminary model classification") == 2
assert "Of 3 CY2018 commenters we classified, 3 supported." in block
def test_question_years():
assert question_years("CY2023 vs 2008, and cy 2015?") == {2023, 2008, 2015}
def test_stance_unit_has_one_header_and_no_repeated_accuracy(replica):
block = _ev(replica, "home visits?").prompt_block()
assert block.count("Preliminary model classification") == 1
head = [ln for ln in block.splitlines() if ln.startswith("Our own")][0]
assert ACC in head
for ln in block.splitlines():
if "commenters we classified" in ln:
assert "Preliminary model classification" in ln
assert "Preliminary" not in ln and ln.endswith("]")
assert block.index(head) < block.index("Of 3 CY2023 commenters")
def test_stance_count_without_accuracy_gets_dossier_accuracy_line(replica):
ev = dossier_evidence("home visits?", replica)
block = ev.prompt_block()
line = [ln for ln in block.splitlines() if "CY2018 commenters" in ln][0]
assert ACC in line and line.endswith("[L8]")
def test_stance_count_without_any_accuracy_says_unmeasured():
def test_stance_header_says_unmeasured_without_accuracy():
d = _dossier(accuracy="")
ev = DossierEvidence(
d.family,
d.name,
"u",
d.summary,
tuple(d.facts.values()),
(),
accuracy="",
d.family, d.name, "u", d.summary, tuple(d.facts.values()), (), accuracy=""
)
line = [ln for ln in ev.prompt_block().splitlines() if "CY2018" in ln][0]
assert "accuracy is not yet measured" in line
assert "accuracy is not yet measured" in ev.prompt_block()
def test_no_stance_facts_no_unit():
ev = DossierEvidence("F", "N", "u", "s", (), ())
assert ev.prompt_block().splitlines()[1] == "Summary: s"
assert "Our own" not in ev.prompt_block()
def test_max_rows_caps_matched_facts(replica):
ev = _ev(replica, "home visits?")
n = len(ev.prompt_block(max_rows=3).splitlines())
assert n == len(ev.prompt_block(max_rows=0).splitlines()) + 3
def test_max_chars_sheds_matched_then_stance_then_old_timeline(replica):
ev = _ev(replica, "home visits?")
full = ev.prompt_block()
no_matched = ev.prompt_block(max_rows=0)
assert "Our own" in no_matched
# a limit that only the matched facts must give way to
cut = ev.prompt_block(max_chars=len(no_matched) + 5)
assert len(cut) <= len(no_matched) + 5 and "Our own" in cut
# smaller: the whole stance unit goes (never counts without the header)
tiny = ev.prompt_block(max_chars=1)
assert "Our own" not in tiny and "commenters we classified" not in tiny
assert tiny.count("\nCY") == 2 and len(tiny) < len(full)
def test_max_chars_sheds_timeline_down_to_the_floor():
tl = tuple(
TimelineEvent(2000 + i, "k", (), f"event {i} [F1]", ("F1",), (), ())
for i in range(12)
)
ev = DossierEvidence("F", "N", "u", "s", (_fact(1, "round", "r"),), (), timeline=tl)
block = ev.prompt_block(max_chars=1)
assert block.count("\nCY") == TIMELINE_FLOOR
assert "CY2011:" in block and "CY2003:" not in block # oldest went first
assert ev.prompt_block(floor=True) == block
assert ev.prompt_block().count("\nCY") == 12
def test_dossier_url_uses_env_domain(monkeypatch):

View File

@@ -267,13 +267,17 @@ class TestValuationForDossierFamilies:
_row(code="99341", year=2023 + i, proposed=True, label=f"[P{i}]")
for i in range(2)
]
mock_val.return_value = (rows, [])
# the default window: only the newest final rows
recent = [_row(code="99341", year=2025 + i, label=f"[R{i}]") for i in range(2)]
mock_val.side_effect = [(recent, []), (rows, [])]
cfg = replace(CFG, valuation_rows_max=10)
ev = valuation_evidence("How were home visits (99341) valued?", cfg)
assert mock_val.call_args.kwargs == {"years": 0, "all_proposed": True}
assert len(ev.payload()["rows"]) == 15
block_rows = ev.prompt_block().count("99341 ")
assert block_rows == 10 <= cfg.valuation_rows_max
block = ev.prompt_block()
# the prompt selects from the default-years rows, never the history
assert block.count("99341 ") == 2
assert "[R0]" in block and "[R1]" in block and "[Y0]" not in block
@patch("llm.evidence.dossier_families", return_value=("HOME-VISITS",))
@patch("llm.evidence.valuation", return_value=([], []))

View File

@@ -1380,6 +1380,7 @@ def _dossier_ev(n_facts=3):
"Summary [DL0].",
facts,
("a gap",),
matched=tuple(f.id for f in facts),
)
@@ -1467,8 +1468,26 @@ class TestDossierInChat:
@patch("llm.rag.valuation_evidence", return_value=None)
@patch("llm.rag.httpx.Client")
@patch("llm.rag.retrieve")
def test_control_question_byte_identical(self, mock_retrieve, MockClient, _ev):
cfg = replace(CFG, duckdb_replica="/nonexistent/aco.ro.duckdb")
def test_control_question_byte_identical(
self, mock_retrieve, MockClient, _ev, tmp_path
):
"""A temp replica really holds the HOME-VISITS dossier; a question
with no dossier family must send the model and the client the same
bytes as a run with the dossier lookup switched off."""
import duckdb
from llm import evidence
from pfs.codetables import ensure_tables
from pfs.dossier import write_dossier
from tests.llm.test_dossier import _dossier
path = tmp_path / "aco.ro.duckdb"
con = duckdb.connect(str(path))
ensure_tables(con)
write_dossier(con, _dossier())
con.close()
evidence._REPLICA = None
cfg = replace(CFG, duckdb_replica=str(path))
mock_retrieve.return_value = []
client = MockClient.return_value.__enter__.return_value
resp = client.stream.return_value.__enter__.return_value
@@ -1477,18 +1496,28 @@ class TestDossierInChat:
resp.iter_lines.return_value = iter(
['{"message":{"content":"x"},"done":true}']
)
return json.dumps(
events = json.dumps(
list(
stream_answer(
"why did CMS finalize this?", cfg=cfg, pool=self._pool()
)
)
)
sent = json.dumps(client.stream.call_args.kwargs["json"]["messages"])
return events, sent
real = run()
with patch("llm.rag.dossier_evidence", return_value=None):
off = run()
try:
from llm.dossier import dossier_evidence
# the family really is there: a home-visit question finds it
assert dossier_evidence("how were home visits valued?", cfg) is not None
real = run()
with patch("llm.rag.dossier_evidence", return_value=None):
off = run()
finally:
evidence._REPLICA = None
assert real == off
assert "Dossier" not in real[1].split("Question:")[0].split("Excerpts:")[-1]
class TestDossierPromptBlock:
@@ -1510,18 +1539,78 @@ class TestDossierPromptBlock:
def test_no_dossier_prompt_unchanged(self):
assert build_messages("q", []) == build_messages("q", [], dossier=None)
def test_dossier_rows_trimmed_to_floor_8_as_a_last_resort(self):
d = replace(_dossier_ev(40), max_prompt_rows=30)
def test_block_takes_at_most_35_percent_of_the_budget(self):
d = _dossier_ev(40)
d = replace(d, matched=tuple(f.id for f in d.facts))
big = build_messages("q", [], dossier=d)[1]["content"]
assert big.count("[DL") == 30
msgs = build_messages("q", [], dossier=d, budget_chars=len(big) - 1)
assert msgs[1]["content"].count("[DL") == 8
assert big.count("[DL") > 30
budget = len(big) + 1000 # roomy: only the 35% share bites
msgs = build_messages("q", [], dossier=d, budget_chars=budget)
block = msgs[1]["content"].split("\n\n")[1]
assert len(block) <= int(budget * 0.35)
def test_dossier_cap_untouched_when_budget_met_earlier(self):
d = _dossier_ev(10)
big = build_messages("q", [], dossier=d)[1]["content"]
msgs = build_messages("q", [], dossier=d, budget_chars=len(big) + 1)
assert msgs[1]["content"].count("[DL") == 10
def test_dossier_floor_is_summary_plus_eight_timeline_lines(self):
from pfs.dossier import TimelineEvent
d = replace(
_dossier_ev(10),
matched=tuple(f"F{i}" for i in range(10)),
timeline=tuple(
TimelineEvent(2000 + i, "k", (), f"ev {i} [F0]", ("F0",), (), ())
for i in range(12)
),
)
msgs = build_messages("q", [], dossier=d, budget_chars=1)
block = msgs[1]["content"]
assert block.count("\nCY") == 8 and "fact 0. [DL0]" not in block
assert "Summary:" in block
def test_dossier_shed_before_lineage_rows(self):
from pfs.dossier import TimelineEvent
events = tuple(
LineageEvent(
f"C{i}",
2020,
"revalued",
(),
(),
f"[L{i}]",
"K",
i,
0,
"",
"fr",
True,
"",
)
for i in range(20)
)
le = LineageEvidence(
codes=("C",),
families=(),
events=events,
element_diffs=(),
guidance=(),
max_prompt_rows=20,
)
d = replace(
_dossier_ev(10),
matched=tuple(f"F{i}" for i in range(10)),
timeline=tuple(
TimelineEvent(2000 + i, "k", (), f"ev {i} [F0]", ("F0",), (), ())
for i in range(12)
),
)
full = build_messages("q", [], lineage=le, dossier=d)[1]["content"]
# a budget that the dossier floor alone satisfies keeps every lineage row
floor = build_messages("q", [], lineage=le, dossier=d, budget_chars=1)
assert floor[1]["content"].count("[L") < full.count("[L") # last resort bites
mid = build_messages(
"q", [], lineage=le, dossier=d, budget_chars=len(full) - 200
)[1]["content"]
assert mid.count("[L") == full.count("[L")
assert mid.count("\nCY") == 8
def test_system_prompt_names_the_dossier_rules(self):
s = build_messages("q", [])[0]["content"]