fix(pfs): dossier fix round 1 — CMS's own words, template-only counts, link ends (R94-R96)

CMS's summaries render the Comment's first sentence verbatim with CMS's
quantifier and the rule-read response action; the model's stance reading is
labelled unreviewed; a summary enters only when its Comment opens on the
family (or names no code and CMS's Response opens on it) and sits beside the
event its Comment is about, else its year's comments event. The summary and
valuation/comments lines are template copy; model event lines also pass a
negation/quantifier guard and a number-role guard. CPT edition facts read
"earliest CPT edition on file" and changes are dated to the CPT Changes code
set and cited to the coding paragraph; a curated link replaced by a later
final rule ends there (G2211 -> B1/B2 in CY2027), its payment rows stop and
the status says so; deletions cite the curated coding paragraph. Minors:
RUC-recommended time wording, enum words, roster notes on CPT/Primary Care
seats, every sentence tagged, verb agreement, the docket's tuning record.
This commit is contained in:
kert
2026-11-13 00:09:10 -05:00
parent 0d8c9740b1
commit 2401b5cb38
7 changed files with 1182 additions and 366 deletions

View File

@@ -67,6 +67,7 @@ from pfs.commenters import family_comment_items, identify, regs_for
from pfs.cpt_load import ingest as cpt_ingest
from pfs.curated import apply_curated, load_curated
from pfs.dossier import (
TEMPLATE_ONLY_EVENTS,
build_dossier,
collect_facts,
governance_excerpts,
@@ -2434,10 +2435,12 @@ def dossier_cmd(
if dry_run:
facts = collect_facts(con, store, fam)
events = dossier_timeline(facts)
drafted = [e for e in events if e.kind not in TEMPLATE_ONLY_EVENTS]
typer.echo(
f"{fam.key}: {len(facts)} facts, {len(events)} timeline events; "
f"model calls a run would make: {len(events) + 1} "
"(each retried once at most)"
f"model calls a run would make: {len(drafted)} event lines "
"(each retried once at most; the summary and the valuation and "
"comments lines are template copy)"
)
continue
d = build_dossier(

View File

@@ -12,14 +12,21 @@ from __future__ import annotations
import re
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Iterable, Mapping
from typing import TYPE_CHECKING, Any, Iterable, Mapping, Sequence
if TYPE_CHECKING: # pragma: no cover
from pfs.dossier import Fact
PROBLEM_KINDS = ("uncited", "unknown_fact", "number_mismatch")
#: what ``grounding_problems`` adds for model prose
GROUNDING_KINDS = ("ungrounded", "code_mismatch", "merged", "unattributed")
GROUNDING_KINDS = (
"ungrounded",
"code_mismatch",
"negation",
"number_role",
"merged",
"unattributed",
)
@dataclass(frozen=True)
@@ -82,7 +89,6 @@ def split_sentences(text: str) -> list[str]:
out: list[str] = []
start = 0
for m in _END.finditer(text):
chunk = text[start : m.start()].strip()
# the end match sits after the punctuation; a lone closing char is
# part of the sentence
chunk = text[start : m.end()].strip()
@@ -98,7 +104,9 @@ def split_sentences(text: str) -> list[str]:
if lead and merged:
merged[-1] = f"{merged[-1]} {lead.group(1).strip()}"
s = s[lead.end() :].strip()
if s:
if s and merged and not re.search(r"\w", s):
merged[-1] = f"{merged[-1]}{s}" # a stray full stop after the tags
elif s:
merged.append(s)
return merged
@@ -207,15 +215,112 @@ _PROSE_GENERIC = frozenset(
"following together respectively through value values both also rvu rvus".split()
)
_CODE5 = re.compile(r"\b\d{5}\b")
#: R95: words that turn or scale a statement; one may appear only when a
#: cited fact says it too
_GUARDED = ("not", "no", "never", "all", "most", "many", "few", "majority", "none")
_GUARDED_RE = re.compile(r"\b(" + "|".join(_GUARDED) + r")\b", re.I)
_CONTRACTION = re.compile(r"n't\b", re.I)
_TOKENS = re.compile(r"[A-Za-z]+|\d[\d,.]*\d|\d")
_CODE_TOKEN = re.compile(r"^(?:\d{5}|[A-Z]\d{4})$")
_ROLES = (
("ruc", re.compile(r"^(?:recommend\w*|RUC)$", re.I)),
("proposed", re.compile(r"^propos\w*$", re.I)),
("final", re.compile(r"^finaliz\w*$", re.I)),
("requested", re.compile(r"^request\w*$", re.I)),
("support", re.compile(r"^support\w*$", re.I)),
("oppose", re.compile(r"^oppos\w*$", re.I)),
("modify", re.compile(r"^modif\w*$", re.I)),
("unclear", re.compile(r"^unclear$", re.I)),
)
def _role(word: str) -> str | None:
return next((r for r, pat in _ROLES if pat.match(word)), None)
def number_roles(text: str) -> dict[str, set[tuple[str | None, str | None]]]:
"""``value -> {(role, code)}`` for the numbers *text* writes (codes and
years aside): the role word right after a number (within three words:
"14 supported") or else the nearest before it (within six: "finalized
2.63"), and the code nearest to it (within six words)."""
toks = [m.group(0) for m in _TOKENS.finditer(_BRACKETS.sub(" ", text or ""))]
toks = [t.rstrip(".,") for t in toks]
out: dict[str, set[tuple[str | None, str | None]]] = {}
for i, t in enumerate(toks):
if not t[:1].isdigit() or _CODE_TOKEN.match(t):
continue
if re.fullmatch(r"(?:19|20)\d\d", t):
continue
role = next((r for w in toks[i + 1 : i + 4] if (r := _role(w))), None)
if role is None:
role = next(
(r for w in reversed(toks[max(0, i - 6) : i]) if (r := _role(w))), None
)
near = [
(abs(j - i), j > i, w)
for j, w in enumerate(toks)
if _CODE_TOKEN.match(w) and abs(j - i) <= 6
]
code = min(near)[2] if near else None
out.setdefault(_norm_number(t), set()).add((role, code))
return out
def _guard_problems(sentence: str, body: str, cited: Sequence[Fact]) -> list[Problem]:
texts = " ".join(f.text for f in cited)
texts = _CONTRACTION.sub(" not", texts)
for m in _GUARDED_RE.finditer(body):
word = m.group(1).lower()
if not re.search(rf"\b{word}\b", texts, re.I):
return [
Problem(
sentence,
"negation",
f"'{m.group(0)}' is not said by the cited facts",
)
]
facts_roles: dict[str, set[tuple[str | None, str | None]]] = {}
for f in cited:
for value, pairs in number_roles(f.text).items():
facts_roles.setdefault(value, set()).update(pairs)
for value, pairs in number_roles(body).items():
known = facts_roles.get(value)
if not known:
continue
roles = {r for r, _c in known if r}
codes = {c for _r, c in known if c}
for role, code in pairs:
if role and roles and role not in roles:
return [
Problem(
sentence,
"number_role",
f"{value} is {role} here but {'/'.join(sorted(roles))} in the facts",
)
]
if code and codes and code not in codes:
return [
Problem(
sentence,
"number_role",
f"{value} sits beside {code} here but "
f"{'/'.join(sorted(codes))} in the facts",
)
]
return []
def grounding_problems(prose: str, facts: Mapping[str, Fact]) -> list[Problem]:
"""What model prose must also respect beyond :func:`check_prose` (the
spec's anti-goals: no invented history, CMS's summary never merged with
our counts): every content word of a sentence occurs (stemmed) in the text
of the facts it cites, generic connective words aside (``ungrounded``);
every code it names, ranges expanded, is one those facts name
(``code_mismatch``); a sentence citing CMS's summary says "CMS" and one
our counts; R95): every content word of a sentence occurs (stemmed) in
the text of the facts it cites, generic connective words aside
(``ungrounded``); every code it names, ranges expanded, is one those
facts name (``code_mismatch``); a negation or quantifier (not, no, never,
all, most, many, few, majority, none; contractions read as "not") is one
a cited fact says too (``negation``); a number keeps the role word and
the code its fact gives it ("CMS finalized 2.63", "14 supported";
``number_role``); a sentence citing CMS's summary says "CMS" and one
citing our classification says "classified" (``unattributed``), and no
sentence cites both (``merged``). Sentences without a known tag are left
to ``check_prose``."""
@@ -228,7 +333,7 @@ def grounding_problems(prose: str, facts: Mapping[str, Fact]) -> list[Problem]:
cited = [facts[t] for t in tags_of(sentence) if t in facts]
if not cited:
continue
body = _BRACKETS.sub(" ", sentence)
body = _CONTRACTION.sub(" not", _BRACKETS.sub(" ", sentence))
known = set().union(*(_content(f.text) for f in cited)) | generic
missing = sorted({w for w in _content(body) if w not in known})
if missing:
@@ -254,6 +359,10 @@ def grounding_problems(prose: str, facts: Mapping[str, Fact]) -> list[Problem]:
)
)
continue
guarded = _guard_problems(sentence, body, cited)
if guarded:
out += guarded
continue
kinds = {f.kind for f in cited}
if {"cms_summary", "stance_split"} <= kinds:
out.append(

File diff suppressed because it is too large Load Diff

View File

@@ -3708,6 +3708,7 @@ class TestDossier:
)
assert res.exit_code == 0, res.output
assert "model calls a run would make:" in res.output
assert "event lines" in res.output
assert env.published == []

View File

@@ -31,3 +31,5 @@
{"item_key": "78F558TF", "p_id": 2820, "page": 73431, "ordinal": 12, "text": "99304 through 99340 for initial, subsequent, discharge, and other nursing facility E/M services; new and ( printed page 73432) established patient domiciliary, rest home (eg, boarding home), or custodial care E/M services; and domiciliary, rest home (eg, assisted living facility), or home care plan oversight services; and", "title": "Medicare Program; CY 2011 PFS Final Rule", "date_published": "2010-11-29", "document_number": "2010-27969", "html_url": "https://www.federalregister.gov/documents/2010/11/29/2010-27969/medicare-program-payment-policies-under-the-physician-fee-schedule-and-other-revisions-to-part-b-for", "start_page": 73170, "end_page": 73860, "fr_volume": 75}
{"item_key": "78F558TF", "p_id": 2821, "page": 73432, "ordinal": 1, "text": "99341 through 99350 for new and established patient home E/M visits. These codes are displayed in Table 66. All of these codes remain active in CY 2011 and there are no other codes used to describe these services.", "title": "Medicare Program; CY 2011 PFS Final Rule", "date_published": "2010-11-29", "document_number": "2010-27969", "html_url": "https://www.federalregister.gov/documents/2010/11/29/2010-27969/medicare-program-payment-policies-under-the-physician-fee-schedule-and-other-revisions-to-part-b-for", "start_page": 73170, "end_page": 73860, "fr_volume": 75}
{"item_key": "GD84NJ2Y", "p_id": 8, "page": 55246, "ordinal": 8, "text": "Mail written comments (1 original and 2 copies) to the following address: Centers for Medicare & Medicaid Services, Department of Health and Human Services, Attention: CMS-1169-FC, P.O. Box 8013, Baltimore, MD 21244-8013.", "title": "Medicare Program; CY 2002 PFS Final Rule", "date_published": "2001-11-01", "document_number": "01-27275", "html_url": "https://www.federalregister.gov/documents/2001/11/01/01-27275/medicare-program-revisions-to-payment-policies-and-five-year-review-of-and-adjustments-to-the", "start_page": 55246, "end_page": 55503, "fr_volume": 66}
{"item_key": "JCWARTJD", "p_id": 3526, "page": 71320, "ordinal": 35, "text": "Additionally, in section II.D. of the final rule we are finalizing a policy to replace office/outpatient evaluation and management (O/O E/M) visit complexity add-on code, HCPCS code G2211 with modifiers that would allow for differential payment for ACO participants.", "title": "Medicare and Medicaid Programs; CY 2027 Payment Policies Under the Physician Fee Schedule and Other Changes to Part B Payment and Coverage Policies; Medicare Shared Savings Program Requirements; and Medicare Prescription Drug Inflation Rebate Program", "date_published": "2026-11-09", "document_number": "2026-22743", "html_url": "https://www.federalregister.gov/documents/2026/11/09/2026-22743/medicare-and-medicaid-programs-cy-2027-payment-policies-under-the-physician-fee-schedule-and-other", "start_page": 70992, "end_page": 72020, "fr_volume": 91}
{"item_key": "JCWARTJD", "p_id": 6666, "page": 71655, "ordinal": 1, "text": "We are finalizing the proposal to replace HCPCS code G2211 with a modifier (MOD1) that is set to 16 percent of the total RVUs for the E/M service on the claim. This policy is expected to have minimal impact given that the volume previously reported using HCPCS code G2211 would now be reported with the modifier, and that was accounted for using the weighted average of the percent increase for utilization of HCPCS code G2211 and adjusted for budget neutrality. The proposal to pay for services furnished in an ACO using a modifier (MOD2) that pays 32 percent of the total RVUs for the E/M service on the claim is expected to increase payment for practitioners that furnish a higher percentage of their services in ACOs. We note that MOD1 and MOD2 were placeholder codes. We are finalizing that MOD1 is now modifier B1, and MOD2 is now modifier B2. Our utilization estimates for MOD1 and MOD2 are reflected in the file titled “CY 2026 PFS Proposed Rule 2025 Utilization Data Crosswalked to 2027” available under “Downloads” for the CY 2027 PFS proposed rule on the CMS website at https://www.cms.gov/​medicare/​payment/​fee-schedules/​physician/​federal-regulation-notices.", "title": "Medicare and Medicaid Programs; CY 2027 Payment Policies Under the Physician Fee Schedule and Other Changes to Part B Payment and Coverage Policies; Medicare Shared Savings Program Requirements; and Medicare Prescription Drug Inflation Rebate Program", "date_published": "2026-11-09", "document_number": "2026-22743", "html_url": "https://www.federalregister.gov/documents/2026/11/09/2026-22743/medicare-and-medicaid-programs-cy-2027-payment-policies-under-the-physician-fee-schedule-and-other", "start_page": 70992, "end_page": 72020, "fr_volume": 91}

View File

@@ -212,3 +212,122 @@ def test_cms_summary_stays_attributed_and_apart():
def test_a_range_the_fact_writes_may_be_restated():
facts = {"F1": _kind("The Panel deleted codes 99324-99328.", "formation")}
assert grounding_problems("The Panel deleted codes 99324-99328 [F1].", facts) == []
# ── R95: negation/quantifier and number-role guards (reviewer's probes) ──
P = {
"F64": _kind(
'In the CY2013 final rule, CMS summarized: "A few commenters were divided."',
"cms_summary",
"F64",
),
"F325": _kind(
"For CY2023, the RUC recommended a work RVU of 1.00 for 99341 and CMS "
"finalized 1.00 (accept).",
fid="F325",
codes=("99341",),
),
"F20": _kind(
"For CY2008, the RUC recommended a work RVU of 2.27 for 99326 and CMS "
"finalized 2.63 (refine).",
fid="F20",
codes=("99326",),
),
"F369": _kind(
"Of 52 CY2023 commenters on this family in docket CMS-2022-0113, we "
"classified 28 with a firm position: 14 supported, 8 asked for "
"modifications and 6 opposed; 1 more was unclear.",
"stance_split",
"F369",
),
"F2": _kind(
"For CY2023, the RUC recommended a work RVU of 1.65 for 99342 and CMS "
"finalized 1.65 (accept).",
fid="F2",
codes=("99342",),
),
}
def test_probe_added_quantifier_is_rejected():
[p] = grounding_problems("CMS reported that all commenters were divided [F64].", P)
assert p.kind == "negation" and "'all'" in p.detail
def test_probe_added_negation_is_rejected():
[p] = grounding_problems(
"CMS did not finalize a work RVU of 1.00 for 99341 [F325].", P
)
assert p.kind == "negation"
[q] = grounding_problems("CMS didn't finalize 1.00 for 99341 [F325].", P)
assert q.kind == "negation"
def test_probe_swapped_ruc_and_final_values_are_rejected():
[p] = grounding_problems(
"For CY2008, the RUC recommended 2.63 for 99326 and CMS finalized 2.27 [F20].",
P,
)
assert p.kind == "number_role" and "2.63" in p.detail
def test_probe_swapped_stance_counts_are_rejected():
[p] = grounding_problems(
"Of 52 CY2023 commenters we classified, 14 opposed and 6 supported [F369].", P
)
assert p.kind == "number_role"
def test_number_beside_another_code_is_rejected():
[p] = grounding_problems("The RUC recommended 1.65 for 99341 [F325][F2].", P)
assert p.kind == "number_role" and "99341" in p.detail
def test_faithful_restatements_pass_the_guards():
assert (
grounding_problems(
"For CY2008, the RUC recommended 2.27 for 99326 and CMS finalized 2.63 [F20].",
P,
)
== []
)
assert (
grounding_problems(
"We classified 28 of 52 CY2023 commenters: 14 supported and 6 opposed [F369].",
P,
)
== []
)
assert (
grounding_problems("A few commenters were divided, CMS summarized [F64].", P)
== []
)
facts = {"F1": _kind("CMS did not finalize the change.", fid="F1")}
assert grounding_problems("CMS didn't finalize the change [F1].", facts) == []
def test_number_roles():
from pfs.citecheck import number_roles
roles = number_roles(P["F20"].text)
assert roles["2.27"] == {("ruc", "99326")} and roles["2.63"] == {("final", "99326")}
assert number_roles("In 2023, the 99341 code [F1].") == {}
def test_a_number_only_in_values_has_no_role_to_check():
facts = {
"F1": _kind(
"CMS finalized the value for 99341 (accept).", fid="F1", codes=("99341",)
)
}
facts["F1"] = Fact(
"F1",
"round",
2023,
("99341",),
facts["F1"].text,
{"final": 1.0},
Cite("fr_para", "x", "u"),
)
assert grounding_problems("CMS finalized 1.00 for 99341 [F1].", facts) == []

View File

@@ -93,65 +93,57 @@ def test_round_template_copy():
assert round_text(2008, "99326") == ""
@pytest.mark.parametrize(
"q,stance,want",
[
(
"majority",
"support",
"CMS reported that a majority of commenters supported the proposal (CY2023 final rule).",
),
(
"many",
"modify",
"CMS reported that many commenters asked for changes (CY2023 final rule).",
),
(
"several",
"oppose",
"CMS reported that several commenters opposed the proposal (CY2023 final rule).",
),
(
"some",
"mixed",
"CMS reported that some commenters were divided (CY2023 final rule).",
),
(
"few",
"unclear",
"CMS reported that a few commenters commented (CY2023 final rule).",
),
(
"all",
"support",
"CMS reported that all commenters supported the proposal (CY2023 final rule).",
),
(
"unspecified",
"support",
"CMS reported that commenters supported the proposal (CY2023 final rule).",
),
(
"one",
"support",
"CMS reported that one commenter supported the proposal (CY2023 final rule).",
),
(
"one",
"mixed",
"CMS reported that one commenter gave a mixed view (CY2023 final rule).",
),
],
)
def test_cms_summary_template(q, stance, want):
assert cms_summary_text(q, stance, 2023, "final") == want
def test_cms_summary_template():
text = cms_summary_text(
"The majority of commenters supported our proposal.",
2023,
"final",
"majority",
"acknowledged",
"support",
)
assert text == (
'In the CY2023 final rule, CMS summarized: "The majority of commenters '
"supported our proposal.\" CMS's quantifier: a majority of commenters. "
"CMS's response: acknowledged the comments. Stance of CMS's summary "
"(model reading, unreviewed): support."
)
assert "no quantifier" in cms_summary_text(
"x", 2005, "final", "unspecified", "unclear", "mixed"
)
assert "not determinable from its wording" in cms_summary_text(
"x", 2005, "final", "one", "unclear", "unclear"
)
def test_comment_first_sentence_is_cms_words_without_cpt_text():
para = (
"Comment: One commenter requested clarification on CPT codes 99347 and "
"99348 ( Home visit for the evaluation and management of an established "
"patient (two of three)) and ( printed page 84506) more. Second sentence."
)
assert dos.comment_first_sentence(para) == (
"One commenter requested clarification on CPT codes 99347 and 99348 "
"[CPT descriptor omitted] and more."
)
long = "Comment: " + "word " * 100 + "end."
first = dos.comment_first_sentence(long)
assert len(first) <= 300 and first.endswith("\u2026")
assert dos.comment_first_sentence("") == ""
assert (
dos.strip_descriptors("codes (99341-99350) stay") == "codes (99341-99350) stay"
)
def test_stance_split_template():
assert (
stance_split_text(12, 2023, "the work RVU valuation", 6, 2, 3, 1)
== "Of 12 CY2023 commenters we classified on the work RVU valuation, 6 "
"supported, 2 asked for modifications, 3 opposed and 1 were unclear."
"supported, 2 asked for modifications, 3 opposed and 1 was unclear."
)
assert stance_split_text(1, 2023, "x", 1, 0, 0, 0) == (
"Of 1 CY2023 commenter we classified on x, 1 supported, 0 asked for "
"modifications, 0 opposed and 0 were unclear."
)
@@ -586,18 +578,31 @@ def test_guideline_facts(home_fixture):
con, store, family, _ask = home_fixture
facts = collect_facts(con, store, family)
texts = [f.text for f in facts]
assert (
"In the CPT 2022 codebook, 99341 is selected on history, examination and "
"medical decision making, with straightforward medical decision making and a "
"typical time of 20 minutes." in texts
first = next(
f
for f in facts
if f.text.startswith("In the earliest CPT edition on file (2022), 99341")
)
assert first.text == (
"In the earliest CPT edition on file (2022), 99341 is selected on history, "
"examination and medical decision making, with straightforward medical "
"decision making and a typical time of 20 minutes."
)
assert first.kind == "event" # never formation (R96)
change = next(
f for f in facts if f.text.startswith("In the CPT 2024 codebook, 99341")
f for f in facts if f.text.startswith("From the CY2023 code set, 99341")
)
assert change.kind == "event"
assert change.year == 2023 and change.kind == "event"
assert "medical decision making or total time" in change.text
assert "a time threshold of 15 minutes" in change.text
assert change.cite.label == "CPT 2024 p. 29"
assert "a time threshold of 15 minutes (CPT 2024 codebook)" in change.text
# dated to the code set that made it: the coding paragraph, CPT Changes,
# and the 2024 book as evidence
assert (change.cite.item_key, change.cite.p_id) == ("PETM225D", 1672)
assert [c.label for c in change.also] == [
"CPT Changes 2023 p. 48",
"CPT 2024 p. 29",
]
assert not [t for t in texts if t.startswith("In the CPT 2024 codebook")]
deleted = next(f for f in facts if f.text.startswith("CPT deleted"))
assert deleted.text == (
"CPT deleted 99324 and 99343 in its 2023 code set (last listed in the "
@@ -618,12 +623,19 @@ def test_cms_summary_and_stance_split_are_separate_facts(home_fixture):
facts = collect_facts(con, store, family)
cms = [f for f in facts if f.kind == "cms_summary"]
ours = [f for f in facts if f.kind == "stance_split"]
assert (
"CMS reported that a majority of commenters supported the proposal (CY2023 final rule)."
in [f.text for f in cms]
majority = next(
f for f in cms if (f.cite.item_key, f.cite.p_id) == ("PETM225D", 1679)
)
majority = next(f for f in cms if "majority" in f.text)
assert (majority.cite.item_key, majority.cite.p_id) == ("PETM225D", 1679)
# CMS's own words, CMS's quantifier and the response action read by rule;
# the model's stance only as an unreviewed reading (R94)
assert majority.text == (
'In the CY2023 final rule, CMS summarized: "The majority of commenters '
"supported our proposal to accept the RUC-recommended RVUs for all of the "
"Home or Residence Services codes.\" CMS's quantifier: a majority of "
"commenters. CMS's response: acknowledged the comments. Stance of CMS's "
"summary (model reading, unreviewed): support."
)
assert majority.values["model_reading_unreviewed"] == "support"
val = next(f for f in ours if "work RVU valuation" in f.text)
assert val.text.startswith(
"Of 3 CY2023 commenters we classified on the work RVU valuation, 1 supported, "
@@ -633,7 +645,7 @@ def test_cms_summary_and_stance_split_are_separate_facts(home_fixture):
assert not {f.id for f in cms} & {f.id for f in ours}
assert all(f.cite.kind == "fr_para" for f in cms)
assert all(f.cite.kind == "dataset" for f in ours)
assert all("CMS reported" not in f.text for f in ours)
assert all("CMS summarized" not in f.text for f in ours)
assert all("we classified" not in f.text for f in cms)
@@ -707,14 +719,12 @@ def test_seat_facts_use_the_prior_year_roster(home_fixture):
"PRACTICE held 1 permanent and 0 rotating seats." in seats
)
assert (
"On the 2022 RUC roster, the Primary Care Rotating Seat is a rotating seat."
in seats
"On the 2022 RUC roster (the RUC that valued CY2023 met in 2022), the "
"Primary Care Rotating Seat is a rotating seat." in seats
)
assert any(
t.startswith(
"The 2026 CPT Editorial Panel roster lists 12 seats for national medical specialty societies"
)
for t in seats
assert (
"The 2026 CPT Editorial Panel roster (the Panel of 2026, for CY2027) lists "
"12 seats for national medical specialty societies." in seats
)
@@ -727,6 +737,10 @@ def test_roster_year_falls_back_to_the_rule_year():
2022,
" (the RUC that valued CY2023 met in 2022)",
)
assert dos.roster_year({2026}, 2027, "CPT_PANEL") == (
2026,
" (the Panel of 2026, for CY2027)",
)
assert dos.roster_year(set(), 2023) == (None, "")
@@ -899,20 +913,30 @@ def _echo(system, user):
return {"text": f"{m.group(2).rstrip('.')} [{m.group(1)}]."}
def _eligible(events):
return [e for e in events if e.kind not in dos.TEMPLATE_ONLY_EVENTS]
def test_model_prose_accepted(home_fixture):
con, store, family, _ask = home_fixture
facts = collect_facts(con, store, family)
events = timeline(facts)
ask = FakeAsk(default=_echo)
summary, lines, extractor, ok = write_prose(facts, events, ask)
assert ok is True and extractor == "model:fake-14b"
assert ok is True and extractor == "model:fake-14b+template"
assert len(lines) == len(events)
ids = {f.id: f for f in facts}
assert check_prose(summary, ids) == []
# the summary and the valuation and comments lines are template copy (R95)
assert summary == template_summary(facts, events)
for i, e in enumerate(events):
if e.kind in dos.TEMPLATE_ONLY_EVENTS:
assert lines[i] == template_event_line(e, ids)
assert all(check_prose(line, ids) == [] for line in lines.values())
assert len(ask.calls) == len(events) + 1
# unreviewed paraphrases never reach the model
assert "Fixture paraphrase" not in "".join(u for _s, u in ask.calls)
assert len(ask.calls) == len(_eligible(events))
prompts = "".join(u for _s, u in ask.calls)
# unreviewed paraphrases, our counts and CMS's summaries never reach it
assert "Fixture paraphrase" not in prompts
assert "we classified" not in prompts and "CMS summarized" not in prompts
def test_prose_rejected_twice_falls_back_to_templates(home_fixture):
@@ -925,7 +949,7 @@ def test_prose_rejected_twice_falls_back_to_templates(home_fixture):
assert summary == template_summary(facts, events)
ids = {f.id: f for f in facts}
assert lines == {i: template_event_line(e, ids) for i, e in enumerate(events)}
assert len(ask.calls) == 2 * (len(events) + 1) # one retry each
assert len(ask.calls) == 2 * len(_eligible(events)) # one retry each
retry = ask.calls[1][1]
assert "uncited" in retry
@@ -933,7 +957,7 @@ def test_prose_rejected_twice_falls_back_to_templates(home_fixture):
def test_retry_with_problems_listed_can_succeed(home_fixture):
con, store, family, _ask = home_fixture
facts = collect_facts(con, store, family)
events = timeline(facts)[:1]
events = _eligible(timeline(facts))[:1]
answers = iter([{"text": "No tag here."}, None])
def first_bad(system, user):
@@ -942,22 +966,23 @@ def test_retry_with_problems_listed_can_succeed(home_fixture):
ask = FakeAsk(default=first_bad)
summary, lines, extractor, ok = write_prose(facts, events, ask)
assert extractor.startswith("model:fake-14b")
assert ok
assert extractor == "model:fake-14b+template"
assert ok and len(ask.calls) == 2
def test_mixed_prose_extractor(home_fixture):
con, store, family, _ask = home_fixture
facts = collect_facts(con, store, family)
events = timeline(facts)[:2]
events = _eligible(timeline(facts))[:2]
seen = []
def only_summary(system, user):
return _echo(system, user) if "summary" in system.lower() else {"text": "x"}
def only_first(system, user):
seen.append(user)
return _echo(system, user) if len(seen) == 1 else {"text": "x"}
summary, lines, extractor, ok = write_prose(
facts, events, FakeAsk(default=only_summary)
)
assert extractor == "model:fake-14b+template"
fb: list[str] = []
write_prose(facts, events, FakeAsk(default=only_first), fallbacks=fb)
assert fb == [f"CY{events[1].year} {events[1].kind}"]
def test_no_model_means_templates(home_fixture):
@@ -971,14 +996,14 @@ def test_no_model_means_templates(home_fixture):
def test_prose_ledger_resumes_and_replays(home_fixture, tmp_path):
con, store, family, _ask = home_fixture
facts = collect_facts(con, store, family)
events = timeline(facts)[:3]
events = _eligible(timeline(facts))[:3]
log = RunLog(tmp_path / "prose.jsonl")
ask = FakeAsk(default=_echo)
first = write_prose(facts, events, ask, log=log, limit=2)
assert len(ask.calls) == 2
assert first[2] == "model:fake-14b+template"
assert first[1][2] == template_event_line(events[2], {f.id: f for f in facts})
second = write_prose(facts, events, ask, log=RunLog(tmp_path / "prose.jsonl"))
assert len(ask.calls) == 4 # the two not yet logged
assert len(ask.calls) == 3 # the one not yet logged
replay = write_prose(
facts,
events,
@@ -987,17 +1012,16 @@ def test_prose_ledger_resumes_and_replays(home_fixture, tmp_path):
replay_model="fake-14b",
)
assert replay == second
assert len(ask.calls) == 4
assert len(ask.calls) == 3
def test_unchecked_model_text_is_never_returned(home_fixture):
con, store, family, _ask = home_fixture
facts = collect_facts(con, store, family)
events = timeline(facts)[:1]
events = _eligible(timeline(facts))[:1]
# numbers no cited fact carries
ask = FakeAsk(
default=lambda s, u: {"text": f"99341 was paid $999.99 [{facts[0].id}]."}
)
fid = events[0].fact_ids[0]
ask = FakeAsk(default=lambda s, u: {"text": f"99341 was paid $999.99 [{fid}]."})
summary, lines, extractor, ok = write_prose(facts, events, ask)
assert "999.99" not in summary and "999.99" not in lines[0]
assert extractor == "template"
@@ -1235,9 +1259,11 @@ def test_round_edges(home_fixture):
texts = [f.text for f in facts]
assert "For CY2020, CMS proposed a work RVU of 1.60 for 99342." in texts
assert "For CY2020, the RUC recommended a work RVU of 1.65 for 99342." in texts
assert "For CY2020, the RUC survey total time for 99342 was 97 minutes." in texts
assert (
"For CY2020, the RUC survey intra-service time for 99342 was 40 minutes."
"For CY2020, the RUC-recommended total time for 99342 was 97 minutes." in texts
)
assert (
"For CY2020, the RUC-recommended intra-service time for 99342 was 40 minutes."
in texts
)
assert "For CY2020, the comparator codes named for 99342 were 99215." in texts
@@ -1324,10 +1350,13 @@ def test_guideline_edges(home_fixture):
facts = collect_facts(con, store, family)
texts = [f.text for f in facts]
assert (
"In the CPT 2021 codebook, 99341 carries with low medical decision making and add-on status."
"In the earliest CPT edition on file (2021), 99341 has low medical decision making and add-on status."
in texts
)
assert (
"In the earliest CPT edition on file (2021), 99350 is selected on other basis."
in texts
)
assert "In the CPT 2021 codebook, 99350 is selected on other basis." in texts
# identical paraphrases of one year collapse to one fact
assert texts.count("A single-code point.") == 1
off = next(f for f in facts if f.text == "Off-family.")
@@ -1784,32 +1813,60 @@ def _f(fid, kind, year, text, codes=(), **values):
return Fact(fid, kind, year, tuple(codes), text, values, Cite("fr_para", "l", "u"))
def test_timeline_comment_only_years_and_fallbacks():
def _cms(fid, year, quote, codes=(), links=""):
return Fact(
fid,
"cms_summary",
year,
tuple(codes),
f'In the CY{year} final rule, CMS summarized: "{quote}"',
{"comment_first_sentence": quote, "link_codes": links},
Cite("fr_para", "l", "u"),
)
def test_timeline_routes_cms_summaries_by_what_the_comment_is_about():
facts = [
_f(
"F1",
"cms_summary",
2005,
"CMS reported that one commenter commented (CY2005 final rule).",
("99341",),
),
_cms("F1", 2005, "One commenter asked about house calls.", ("99341",)),
_f(
"F2",
"event",
"round",
2010,
"99341 was created for CY2010.",
"For CY2010, CMS finalized a work RVU of 1.00 for 99341.",
("99341",),
event_kind="created",
),
_f(
"F3",
"cms_summary",
2010,
"CMS reported that commenters commented (CY2010 final rule).",
("99350",),
),
_f(
_cms("F3", 2010, "Commenters said the RVUs were too low.", ("99350",)),
_cms(
"F4",
2010,
"Commenters asked about place of service 12 for 99341.",
("99341",),
),
_f(
"F5",
"link",
2023,
"From CY2023, G0318 can be reported with this family (prolonged-add-on).",
("G0318",),
link_kind="prolonged-add-on",
),
_cms("F6", 2023, "A few commenters asked about GXXX3.", (), "G0318"),
_f(
"F7",
"event",
2023,
"99341 is named in a telehealth-list paragraph of the CY2023 rule.",
("99341",),
event_kind="telehealth_list",
),
_cms(
"F8",
2023,
"Commenters supported adding the codes to the telehealth list.",
("99341",),
),
_f(
"F9",
"stance_split",
2011,
"Of 2 CY2011 commenters we classified on the coding changes, 1 supported, 1 asked for modifications, 0 opposed and 0 were unclear. Preliminary.",
@@ -1819,14 +1876,18 @@ def test_timeline_comment_only_years_and_fallbacks():
events = timeline(facts)
by = {(e.year, e.kind): e for e in events}
assert by[(2005, "comments")].cms_summary_fact_ids == ("F1",)
assert (
by[(2005, "comments")].what_changed
== "CMS reported that one commenter commented (CY2005 final rule) [F1]."
assert by[(2005, "comments")].what_changed.startswith(
'In the CY2005 final rule, CMS summarized: "One commenter'
)
assert by[(2010, "valuation")].cms_summary_fact_ids == ("F3",)
# not about the valuation: its own comments event, never filed under it
assert by[(2010, "comments")].cms_summary_fact_ids == ("F4",)
assert by[(2023, "link:G0318")].cms_summary_fact_ids == ("F6",)
assert by[(2023, "telehealth_list")].cms_summary_fact_ids == ("F8",)
line = by[(2011, "comments")].what_changed
assert line.startswith("Of 2 CY2011 commenters") and line.endswith(
"Preliminary [F9]."
)
assert by[(2010, "created")].cms_summary_fact_ids == ("F3",)
assert by[(2011, "comments")].what_changed.startswith(
"Of 2 CY2011 commenters"
) and by[(2011, "comments")].what_changed.endswith("[F4].")
ids = {f.id: f for f in facts}
assert check_prose(template_summary(facts, events), ids) == []
@@ -1834,30 +1895,34 @@ def test_timeline_comment_only_years_and_fallbacks():
def test_prose_too_long_is_a_problem(home_fixture):
con, store, family, _ask = home_fixture
facts = collect_facts(con, store, family)
events = timeline(facts)[:1]
six = " ".join(f"{facts[0].text.rstrip('.')} [{facts[0].id}]." for _ in range(6))
ask = FakeAsk(default={"text": six})
events = _eligible(timeline(facts))[:1]
lead = next(f for f in facts if f.id == events[0].fact_ids[0])
three = " ".join(f"{lead.text.rstrip('.')} [{lead.id}]." for _ in range(3))
ask = FakeAsk(default={"text": three})
summary, lines, extractor, ok = write_prose(facts, events, ask)
assert extractor == "template"
assert "too long: 6 sentences, at most 5" in ask.calls[1][1]
assert "too long: 3 sentences, at most 2" in ask.calls[1][1]
def test_in_sample_line():
assert dos._in_sample_line({"recall": 0.5}) == ""
line = dos._in_sample_line(
{
"in_sample_record": {
"recall": 0.407,
"direction": 0.818,
"n_items": 20,
"n_positions": 27,
"docket": "CMS-2022-0113",
}
ev = {
"in_sample_record": {
"recall": 0.407,
"direction": 0.818,
"n_items": 20,
"n_positions": 27,
"docket": "CMS-2022-0113",
}
}
line = dos._in_sample_line(ev)
assert line == (
" Tuning-set record on CMS-2022-0113 (20 hand-coded comments, 27 "
"positions): it finds 41% of hand-coded positions, and the direction is "
"right for 82% of those it finds."
)
assert (
line.startswith(" Tuning-set record: (20 hand-coded comments") and "41%" in line
)
assert dos._in_sample_line(ev, "CMS-2022-0113") == line
assert dos._in_sample_line(ev, "CMS-2025-0304") == ""
def test_expand_range():
@@ -1872,11 +1937,243 @@ def test_expand_range():
def test_fallbacks_are_reported_per_unit(home_fixture):
con, store, family, _ask = home_fixture
facts = collect_facts(con, store, family)
events = timeline(facts)[:2]
events = _eligible(timeline(facts))[:2]
fb: list[str] = []
write_prose(facts, events, FakeAsk(default=_echo), fallbacks=fb)
assert fb == []
write_prose(facts, events, None, fallbacks=fb)
assert fb == ["summary", *(f"CY{e.year} {e.kind}" for e in events)]
assert fb == [f"CY{e.year} {e.kind}" for e in events]
d = build_dossier(con, store, family)
assert d.fallbacks[0] == "summary" and len(d.fallbacks) == len(d.timeline) + 1
assert len(d.fallbacks) == len(_eligible(d.timeline))
assert "summary" not in d.fallbacks # template by rule, not a fallback
# ── fix round 1: R94 family filter, R96 link end, deletion cites ─────
from pfs.codetables import CmsSummaryRow, write_cms_summaries # noqa: E402
def test_cms_summaries_must_be_about_the_family(home_fixture):
con, store, family, _ask = home_fixture
paras = {
99101: "Comment: Commenters disagreed that CPT codes 99091 and 99457 can be billed together.",
99102: "Response: The codebook lists 99339 and 99340 among others.",
99103: "Comment: Commenters asked us to revisit travel costs.",
99104: "Response: The home visit codes 99347 and 99348 reflect travel. More.",
99105: "Comment: Commenters thanked us.",
99106: "Response: We appreciate the support.",
99107: "Comment: Commenters supported the e-prescribing measure. They noted domiciliary care visit codes.",
99108: "Response: We appreciate the support. The codes include 99324.",
99109: "Comment: Commenters asked about home or residence services.",
}
for p_id, text in paras.items():
_add_paragraph(store, p_id, text)
rows = [
CmsSummaryRow(
family.key,
"PETM225D",
2023,
"final",
c,
r,
"",
"unspecified",
"support",
"unclear",
"regex",
)
for c, r in (
(99101, 99102),
(99103, 99104),
(99105, 99106),
(99107, 99108),
(99109, 99999),
)
]
write_cms_summaries(con, family.key, rows)
cms = [f for f in collect_facts(con, store, family) if f.kind == "cms_summary"]
# about other codes (the family only listed in the response): dropped;
# nothing about the family: dropped; the family only in a later sentence:
# dropped; no code in the Comment and the Response opening with the
# family: kept; the Comment's first sentence naming the family: kept
assert sorted((f.cite.p_id, f.codes) for f in cms) == [
(99103, ("99347", "99348")),
(99109, ()),
]
assert any(
'CMS summarized: "Commenters asked us to revisit travel costs."' in f.text
for f in cms
)
def test_link_end_status_payment_and_summary(home_fixture):
con, store, family, _ask = home_fixture
d = build_dossier(con, store, family)
end = next(f for f in d.facts.values() if f.values.get("link_end") == "G2211")
assert end.text == "From CY2027, G2211 is replaced by modifiers B1 and B2."
assert (end.cite.item_key, end.cite.p_id) == ("JCWARTJD", 6666)
assert [(c.item_key, c.p_id) for c in end.also] == [("JCWARTJD", 3526)]
assert "G2211 CY2026 only; replaced by modifiers B1/B2 from CY2027" in d.status
assert any(e.kind == "link:G2211" and e.year == 2027 for e in d.timeline)
# the summary spans the earliest finalized valuation to the newest link,
# each statement once
sents = dos.split_sentences(d.summary)
assert len(sents) <= 5
assert sents[0].startswith("For CY2008, CMS finalized work RVUs")
assert "G2211 (from CY2026)" in sents[-1] and "B2 (from CY2027)" in sents[-1]
assert "from CY2027, G2211 is replaced by modifiers B1 and B2" in sents[-1]
assert sum("deleted" in s for s in sents) == 1
ids = d.facts
assert (
check_prose(d.summary, ids) == []
and dos.grounding_problems(d.summary, ids) == []
)
def test_payment_stops_at_a_link_end(home_fixture):
con, _store, family, _ask = home_fixture
rows = dos._payment_rows(con, family, {"99341": 2027})
assert rows and max(r.year for r in rows if r.code == "99341") == 2026
def test_link_end_edges(home_fixture):
con, store, family, _ask = home_fixture
from pfs.curated import CuratedLink
_add_paragraph(
store, 99201, "We are finalizing our proposal to replace code G9001."
)
_add_paragraph(store, 99202, "We proposed to replace code G9002 with G9003.")
_add_paragraph(
store,
99203,
"We are finalizing a policy to replace code G9004 with code G9005.",
)
fam = dataclasses.replace(
family,
links=(
CuratedLink("add-on", "G9000", None, None, "", 0, 0, 0),
CuratedLink("add-on", "G9001", 2020, None, "", 0, 0, 2020),
CuratedLink("add-on", "G9002", 2020, None, "", 0, 0, 2020),
CuratedLink("add-on", "G9004", 2023, None, "", 0, 0, 2023),
CuratedLink("add-on", "G9006", 2019, None, "", 0, 0, 2019),
CuratedLink("add-on", "G9005", 2023, None, "", 0, 0, 2023),
),
)
ends = dos._link_ends(dos._Bib(store), fam)
# G9001 has no successor named; G9002's sentence is a proposal; G9004's
# rule year is not after its since
assert [f.text for f in ends] == ["From CY2023, G9001 is replaced."]
assert dos._status(fam, ends).endswith(
"G9001 CY2020–CY2022; replaced from CY2023, G9002 from CY2020, G9004 from CY2023, G9006 from CY2019, G9005 from CY2023"
)
fam2 = dataclasses.replace(
family,
links=(
CuratedLink("add-on", "G9004", 2022, None, "", 0, 0, 2022),
CuratedLink("add-on", "G9005", 2023, None, "", 0, 0, 2023),
),
)
[e] = dos._link_ends(dos._Bib(store), fam2)
assert e.text == "From CY2023, G9004 is replaced by codes G9005."
assert dos._status(fam2, [e]).endswith(
"G9004 CY2022 only; replaced by codes G9005 from CY2023, G9005 from CY2023"
)
def test_deletions_cite_the_coding_paragraph(home_fixture):
con, store, family, _ask = home_fixture
facts = collect_facts(con, store, family)
for code in ("99324", "99328", "99343"):
f = next(f for f in facts if f.text == f"{code} was deleted for CY2023.")
assert (f.cite.item_key, f.cite.p_id) == ("PETM225D", 1672)
# 99339 has no curated paragraph: the paragraph that states it
f = next(f for f in facts if f.text == "99339 was deleted for CY2023.")
assert (f.cite.item_key, f.cite.p_id) == ("PETM225D", 390)
def test_edition_change_without_a_coding_paragraph(home_fixture):
con, store, family, _ask = home_fixture
write_guideline_facts(
con,
family.key,
[
GuidelineFactRow(
family.key, "99339", 2022, "mdm_level", "low", "K22", "p. 5", "cpt-book"
),
GuidelineFactRow(
family.key,
"99339",
2024,
"mdm_level",
"high",
"K24",
"p. 6",
"cpt-book",
),
GuidelineFactRow(
family.key,
"99343",
2023,
"deleted_in_edition",
"2022",
"CH23",
"p. 48",
"cpt-book",
),
],
)
facts = collect_facts(con, store, family)
f = next(f for f in facts if f.text.startswith("From the CY2023 code set, 99339"))
assert f.cite.label == "CPT Changes 2023 p. 48"
assert [c.label for c in f.also] == ["CPT 2024 p. 6"]
def test_nested_descriptor_is_stripped_once():
assert dos.strip_descriptors(
"99341 ( Home visit, see 99342 ( Nested words)) end"
) == ("99341 [CPT descriptor omitted] end")
def test_empty_model_answer_is_a_problem(home_fixture):
con, store, family, _ask = home_fixture
facts = collect_facts(con, store, family)
events = _eligible(timeline(facts))[:1]
ask = FakeAsk(default={"text": " "})
write_prose(facts, events, ask)
assert "empty: no text" in ask.calls[1][1]
def test_template_summary_without_valuations_or_links():
facts = [_cms("F1", 2005, "One commenter asked about house calls.", ("99341",))]
events = timeline(facts)
assert template_summary(facts, events).startswith(
'In the CY2005 final rule, CMS summarized: "One commenter'
)
def test_a_quote_keeps_its_closing_mark_and_every_sentence_is_tagged():
line = dos._with_tags(
'CMS summarized: "Codes were not valued." CMS said more.', ["F1"]
)
assert line == 'CMS summarized: "Codes were not valued." [F1] CMS said more [F1].'
assert dos.split_sentences(line) == [
'CMS summarized: "Codes were not valued." [F1]',
"CMS said more [F1].",
]
facts = {
"F1": _f(
"F1",
"cms_summary",
2005,
'CMS summarized: "Codes were not valued." CMS said more.',
)
}
assert check_prose(line, facts) == [] and dos.grounding_problems(line, facts) == []
def test_a_stray_full_stop_stays_with_its_sentence():
assert dos.split_sentences('He said "x." [F1]. Next [F2].') == [
'He said "x." [F1].',
"Next [F2].",
]