From 07e134f570bc49ce297fd090d1315f77264f24e2 Mon Sep 17 00:00:00 2001 From: kert Date: Thu, 12 Nov 2026 01:37:59 -0500 Subject: [PATCH] =?UTF-8?q?fix(pfs):=20rounds=20=E2=80=94=20per-field=20ci?= =?UTF-8?q?tations,=20model=20fields=20only=20from=20plain=20sentences=20c?= =?UTF-8?q?overing=20the=20code,=20maintain/recap/hypothetical=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R27: RoundRow.field_cites (JSON {field: p_id | "page:N"}, list for a joined PE note); stamped per source, carried by merge with the winning value, set by the paragraph rules; a derived decision cites the acted value's source. R28: model numbers survive only in a non-recap, non-hypothetical sentence covering the code; comparators/societies/quote only from such sentences; a model label needs the RUC value and the acted value. R29: maintain only as CMS's own 'are maintaining'/'will maintain', never after 'recommend(ed) (that)'; Response:/Comment: lead-ins and 'CY 20XX final rule' recaps; 'would be the/consistent/...' no longer hypothetical; stated interim/reject labels are not overridden by the numbers. --- src/pfs/codetables.py | 11 +- src/pfs/rounds.py | 315 ++++++++++++++++++++++------- tests/cli/test_pfs_cli.py | 6 +- tests/pfs/test_rounds.py | 405 +++++++++++++++++++++++++++++++++++++- 4 files changed, 655 insertions(+), 82 deletions(-) diff --git a/src/pfs/codetables.py b/src/pfs/codetables.py index fc49e89..d91b565 100644 --- a/src/pfs/codetables.py +++ b/src/pfs/codetables.py @@ -313,6 +313,10 @@ class RoundRow: final_p_id: int = 0 #: the paragraph stating the RUC value / percentile / survey data (R26) ruc_p_id: int = 0 + #: R27: JSON object citing each non-empty value field to its source: + #: ``{field: p_id}`` (a paragraph of ``item_key``) or ``"page:N"`` (an FR + #: table cell); ``pe_note`` joined from several sources cites a list + field_cites: str = "" @dataclass(frozen=True) @@ -379,7 +383,8 @@ CREATE TABLE IF NOT EXISTS pfs.valuation_round ( survey_total_min DOUBLE, comparators VARCHAR, surveying_societies VARCHAR, requested_work_rvu DOUBLE, proposed_work_rvu DOUBLE, final_work_rvu DOUBLE, decision VARCHAR, rationale_p_id INTEGER, rationale_quote VARCHAR, - pe_note VARCHAR, extractor VARCHAR, final_p_id INTEGER, ruc_p_id INTEGER); + pe_note VARCHAR, extractor VARCHAR, final_p_id INTEGER, ruc_p_id INTEGER, + field_cites VARCHAR); """ @@ -422,6 +427,10 @@ def ensure_tables(con: Any) -> None: con.execute( "ALTER TABLE pfs.valuation_round ADD COLUMN IF NOT EXISTS ruc_p_id INTEGER" ) + # R27: per-field citations. + con.execute( + "ALTER TABLE pfs.valuation_round ADD COLUMN IF NOT EXISTS field_cites VARCHAR" + ) def _insert(con: Any, table: str, rows: Sequence[Any]) -> int: diff --git a/src/pfs/rounds.py b/src/pfs/rounds.py index d1af6b2..e27ac23 100644 --- a/src/pfs/rounds.py +++ b/src/pfs/rounds.py @@ -12,6 +12,7 @@ model; first non-empty value wins) and ``family_rounds`` runs the lot. from __future__ import annotations +import json import re from collections.abc import Sequence from dataclasses import dataclass, fields, replace @@ -210,6 +211,68 @@ def decision_of(ruc: float | None, final: float | None, basis_text: str) -> str: return "" +# ── per-field citations (R27) ───────────────────────────────────────── + +#: value fields ``RoundRow.field_cites`` cites to the paragraph (``p_id``) +#: or FR table page (``"page:N"``) that states them +CITE_FIELDS = ( + "ruc_work_rvu", + "ruc_percentile", + "survey_intra_min", + "survey_total_min", + "comparators", + "surveying_societies", + "requested_work_rvu", + "proposed_work_rvu", + "final_work_rvu", + "decision", + "rationale_quote", + "pe_note", +) + + +def _empty(v: Any) -> bool: + return v is None or v == "" or (type(v) is int and v == 0) + + +def field_cites(row: RoundRow) -> dict[str, Any]: + """``row.field_cites`` decoded: ``{field: p_id | "page:N"}`` (a list of + those for a PE note joined from several sources).""" + return json.loads(row.field_cites) if row.field_cites else {} + + +def _encode(cites: dict[str, Any]) -> str: + return json.dumps(cites, sort_keys=True) if cites else "" + + +def _stamp(row: RoundRow, cite: int | str) -> RoundRow: + """A single-source row: every non-empty value field cites *cite*.""" + return replace( + row, + field_cites=_encode( + {f: cite for f in CITE_FIELDS if not _empty(getattr(row, f))} + ), + ) + + +def _put(row: RoundRow, cite: Any, **changes: Any) -> RoundRow: + """*row* with *changes*; each changed value field now cites *cite* + (an emptied field, or a ``None`` cite, drops its citation).""" + cites = field_cites(row) + for f, v in changes.items(): + if f in CITE_FIELDS: + cites.pop(f, None) + if not _empty(v) and cite is not None: + cites[f] = cite + return replace(row, **changes, field_cites=_encode(cites)) + + +def _prune(row: RoundRow) -> RoundRow: + """Drop the citations of fields that ended up empty.""" + cites = {k: v for k, v in field_cites(row).items() if not _empty(getattr(row, k))} + return replace(row, field_cites=_encode(cites)) + + def table_rounds( table: FrTable, *, @@ -245,32 +308,35 @@ def table_rounds( and re.search(r"refine|disagree", basis, re.I) ): note = f"basis says {basis}; work RVU equals RUC" - pages = table.row_pages + page = table.row_pages[idx] if table.row_pages else table.page out.append( - RoundRow( - family=family, - code=row[0], - rule_year=rule_year, - item_key=item_key, - rule_kind=rule_kind, - p_id=0, - page=pages[idx] if pages else table.page, - source="fr-table", - table_title=table.title, - ruc_work_rvu=ruc, - ruc_percentile="", - survey_intra_min=None, - survey_total_min=None, - comparators="", - surveying_societies="", - requested_work_rvu=_num(cell(row, "requested")), - proposed_work_rvu=_num(cell(row, "proposed")), - final_work_rvu=final, - decision=decision, - rationale_p_id=0, - rationale_quote=basis[:300], - pe_note=note, - extractor="fr-table", + _stamp( + RoundRow( + family=family, + code=row[0], + rule_year=rule_year, + item_key=item_key, + rule_kind=rule_kind, + p_id=0, + page=page, + source="fr-table", + table_title=table.title, + ruc_work_rvu=ruc, + ruc_percentile="", + survey_intra_min=None, + survey_total_min=None, + comparators="", + surveying_societies="", + requested_work_rvu=_num(cell(row, "requested")), + proposed_work_rvu=_num(cell(row, "proposed")), + final_work_rvu=final, + decision=decision, + rationale_p_id=0, + rationale_quote=basis[:300], + pe_note=note, + extractor="fr-table", + ), + f"page:{page}", ) ) return out @@ -311,7 +377,19 @@ _NEGATED = re.compile( re.I, ) _RUC_VERB = re.compile(r"RUC[- ]recommended|RUC recommends", re.I) -_MAINTAIN = re.compile(r"\b(?:we|are|is)\s+(?:(?:are|is)\s+)?maintain\w*", re.I) +#: CMS's own "are maintaining"/"will maintain" (a proposal in a proposed +#: rule, a final in a final one); a bare "we maintain that ..." argues, and +#: "recommended that we maintain ..." is someone else's request (N3-a) +_MAINTAIN = re.compile( + "".join( + f"(? list[str]: return _ANY_CODE.findall(text) -def _percentiles(text: str) -> dict[str, str]: +def _percentiles(text: str, rule_year: int | None = None) -> dict[str, str]: """``{code: "25th"|"median"|"75th"}`` for the codes the paragraph states a survey value for, and only those: (P1) a value followed by ``for [CPT] code(s) `` in one clause, (P2) `` … at the `` before the next code, (P3) `` for all … codes, except … ``. A code the paragraph names as deleted - never gets one; a code with two different statements gets none.""" + never gets one; a code with two different statements gets none; a + hypothetical sentence, or (given *rule_year*) one recapping another + year's rule, states none.""" text = _PRINTED_PAGE.sub(" ", _expand_ranges(text)) found: dict[str, set[str]] = {} deleted: set[str] = set() @@ -562,7 +642,7 @@ def _percentiles(text: str) -> dict[str, str]: deleted.update(_list_codes(sent)) for sent in _sentences(text): terms = [(m.start(), m.end(), _label(m)) for m in _TERM.finditer(sent)] - if not terms: + if not terms or _recap(sent, rule_year) or _HYPOTHETICAL.search(sent): continue coll = _COLLECTIVE_FOR.search(sent) if coll and terms[0][0] < coll.start(): @@ -700,7 +780,7 @@ def guard(row: RoundRow, text: str) -> RoundRow: changes: dict[str, Any] = { f: None for f in _NUM_FIELDS if getattr(row, f) not in present } - if row.ruc_percentile != _percentiles(text).get(row.code): + if row.ruc_percentile != _percentiles(text, row.rule_year).get(row.code): changes["ruc_percentile"] = "" if row.rationale_quote and row.rationale_quote not in text: changes["rationale_quote"] = "" @@ -755,11 +835,11 @@ def regex_rounds(para: dict, codes: set[str], *, family: str) -> list[RoundRow]: if field == "ruc_work_rvu": extra = {"ruc_p_id": para["p_id"]} rows[code] = replace(row, **{field: value}, **extra) - for code, pct in _percentiles(para["text"]).items(): + for code, pct in _percentiles(para["text"], year).items(): if code in codes: row = rows.get(code) or _blank(family, code, para, "regex") rows[code] = replace(row, ruc_percentile=pct, ruc_p_id=para["p_id"]) - return list(rows.values()) + return [_stamp(r, para["p_id"]) for r in rows.values()] def _range_note(text: str) -> str: @@ -787,7 +867,8 @@ def _supported( ) -> bool: """Is *value* stated as a proposal/finalization for *code*: a sentence naming the code and the value carries a proposing (or finalizing, in a - final rule) verb and no "would need to"/"if we were to" hedge.""" + final rule) verb and no "would need to"/"if we were to" hedge, and + neither recaps another year's rule nor is hypothetical.""" verb = _PROPOSE if field == "proposed_work_rvu" else _FINALIZE if field == "final_work_rvu" and kind != "final": return False @@ -798,6 +879,7 @@ def _supported( and not _recap(sent, year) and verb.search(sent) and not _CONDITIONAL.search(sent) + and not _HYPOTHETICAL.search(sent) and not (field == "final_work_rvu" and _NEGATED.search(sent)) and value in _numbers(sent) ): @@ -830,15 +912,31 @@ def _pe_note(note: str | None) -> str: return "" if note.lower().startswith("none") else note +#: labels the numbers never override: the model keeps them only when the +#: code's sentence says the word (N3-d) +_STATED_LABELS = ("interim", "reject") + + +def _acted_field(row: RoundRow) -> str: + """The value CMS acted on: the final in a final rule, else the proposal.""" + return "final_work_rvu" if row.rule_kind == "final" else "proposed_work_rvu" + + def _numbers_decide(row: RoundRow) -> RoundRow: - """When the RUC value and the value CMS acted on (the final in a final - rule, the proposal in a proposed one) are both present, the numbers - decide the label ("accept" next to 2.27 -> 2.50 is a refinement); a - model label survives only where they cannot.""" - acted = row.final_work_rvu if row.rule_kind == "final" else row.proposed_work_rvu - if row.ruc_work_rvu is None or acted is None: + """When the RUC value and the value CMS acted on are both present, the + numbers decide the label ("accept" next to 2.27 -> 2.50 is a + refinement) and it cites the acted value's source; a model label + survives only where they cannot, or when it is a stated + "interim"/"reject".""" + acted_f = _acted_field(row) + acted = getattr(row, acted_f) + if row.ruc_work_rvu is None or acted is None or row.decision in _STATED_LABELS: return row - return replace(row, decision=decision_of(row.ruc_work_rvu, acted, "")) + return _put( + row, + field_cites(row).get(acted_f), + decision=decision_of(row.ruc_work_rvu, acted, ""), + ) def has_value(row: RoundRow) -> bool: @@ -851,17 +949,60 @@ def has_content(row: RoundRow) -> bool: return has_value(row) or bool(row.decision) +def _plain_sentences(text: str, year: int | None) -> list[str]: + """The sentences that state something: neither a recap of another + year's rule nor a hypothetical.""" + return [ + s + for s in _sentences(text) + if not _recap(s, year) and not _HYPOTHETICAL.search(s) + ] + + +def _covers(sent: str, code: str) -> bool: + """*sent* names *code* (ranges expanded), or speaks collectively ("for + all eight CPT codes ...") without naming any code.""" + expanded = _expand_ranges(sent) + if re.search(rf"\b{re.escape(code)}\b", expanded): + return True + return bool(_COLLECTIVE_FOR.search(sent)) and not _ANY_CODE.search(expanded) + + +def _stated(code: str, value: Any, plain: Sequence[str]) -> Any: + """*value* when a plain sentence covering *code* carries it, else None.""" + if value is None: + return None + return ( + value if any(_covers(s, code) and value in _numbers(s) for s in plain) else None + ) + + +def _in_plain(phrase: str, plain: Sequence[str]) -> bool: + return any(phrase.lower() in s.lower() for s in plain) + + def rows_from_output( out: dict, para: dict, codes: set[str], *, family: str, model: str ) -> list[RoundRow]: - """Build guarded rows from one model reply (a ledger output).""" + """Build guarded rows from one model reply (a ledger output). R28: a + model field counts only when a plain sentence of the paragraph (no + recap of another year's rule, no hypothetical) states it: a number in + one that covers the code, a comparator/society/quote within one; a + label needs the RUC value and the value CMS acted on, and the numbers + decide it unless the code's sentence says "interim"/"reject". Every + surviving field cites the paragraph.""" year, kind = _para_meta(para) text = para["text"] extractor = f"model:{model}@{ROUND_PROMPT_VERSION}" regex = para_values(text, codes, rule_kind=kind, rule_year=year) - pcts = _percentiles(text) + pcts = _percentiles(text, year) + plain = _plain_sentences(text, year) quote = out.get("rationale_quote") or "" - societies = "; ".join(out.get("surveying_societies") or []) + if any(_recap(s, year) or _HYPOTHETICAL.search(s) for s in _sentences(quote)): + quote = "" + societies = "; ".join( + x for x in out.get("surveying_societies") or [] if _in_plain(x, plain) + ) rows: list[RoundRow] = [] for d in out.get("codes") or []: code = d.get("code") @@ -874,11 +1015,15 @@ def rows_from_output( } row = replace( _blank(family, code, para, extractor), - ruc_work_rvu=d.get("ruc_work_rvu"), + ruc_work_rvu=_stated(code, d.get("ruc_work_rvu"), plain), ruc_percentile=pcts.get(code, ""), - survey_intra_min=d.get("survey_intra_min"), - survey_total_min=d.get("survey_total_min"), - comparators="; ".join(c for c in d.get("comparators") or [] if c != code), + survey_intra_min=_stated(code, d.get("survey_intra_min"), plain), + survey_total_min=_stated(code, d.get("survey_total_min"), plain), + comparators="; ".join( + c + for c in d.get("comparators") or [] + if c != code and _in_plain(c, plain) + ), surveying_societies=societies, proposed_work_rvu=vals["proposed_work_rvu"], final_work_rvu=vals["final_work_rvu"], @@ -890,11 +1035,6 @@ def rows_from_output( row = guard(row, text) if row.final_work_rvu is not None: row = replace(row, final_p_id=para["p_id"]) - if not has_value(row) or row.ruc_work_rvu is None: - # a bare or unsupported label ("accept" with no RUC value to - # accept) is dropped; the numbers decide later - row = replace(row, decision="") - row = _numbers_decide(row) if ( row.ruc_work_rvu is not None or row.ruc_percentile @@ -902,7 +1042,19 @@ def rows_from_output( or row.survey_total_min is not None ): row = replace(row, ruc_p_id=para["p_id"]) - rows.append(row) + row = _stamp(row, para["p_id"]) + unstated = row.decision in _STATED_LABELS and not any( + _covers(s, code) and re.search(rf"\b{row.decision}", s, re.I) for s in plain + ) + if ( + row.ruc_work_rvu is None + or getattr(row, _acted_field(row)) is None + or unstated + ): + # no RUC value to judge, no value CMS acted on, or an + # unstated "interim"/"reject": the label goes, numbers decide + row = _put(row, None, decision="") + rows.append(_numbers_decide(row)) return rows @@ -931,21 +1083,24 @@ _CONFLICT_FIELDS = ( _KEYS = ("family", "code", "item_key") -def _empty(v: Any) -> bool: - return v is None or v == "" or (type(v) is int and v == 0) - - def _differs(a: Any, b: Any) -> bool: if isinstance(a, float) and isinstance(b, float): return abs(a - b) > 1e-9 return a != b +def _cite_list(cite: Any) -> list[Any]: + if cite is None: + return [] + return list(cite) if isinstance(cite, list) else [cite] + + def merge_rounds(*sources: Sequence[RoundRow]) -> list[RoundRow]: """One row per ``(code, item_key)``. Per field the first non-empty value in source order wins (``0``/``""``/``None`` are empty, so a - paragraph can fill a table row's missing anchor); a later value that - disagrees on a valuation field is noted in ``pe_note``.""" + paragraph can fill a table row's missing anchor) and keeps its source's + citation; a later value that disagrees on a valuation field is noted in + ``pe_note``, which then cites every source its notes came from.""" merged: dict[tuple[str, str], RoundRow] = {} for rows in sources: for r in rows: @@ -958,6 +1113,8 @@ def merge_rounds(*sources: Sequence[RoundRow]) -> list[RoundRow]: continue changes: dict[str, Any] = {} notes = [cur.pe_note] if cur.pe_note else [] + cites, theirs = field_cites(cur), field_cites(r) + note_cites = _cite_list(cites.get("pe_note")) if cur.proposed_work_rvu is None and r.proposed_work_rvu is not None: # the proposal's paragraph anchor travels with the proposal # (a percentile-only row from another paragraph must not @@ -966,7 +1123,7 @@ def merge_rounds(*sources: Sequence[RoundRow]) -> list[RoundRow]: if cur.source != "fr-table": changes["page"] = r.page for f in fields(RoundRow): - if f.name in _KEYS or f.name in ("pe_note", "extractor"): + if f.name in _KEYS or f.name in ("pe_note", "extractor", "field_cites"): continue old, new = getattr(cur, f.name), getattr(r, f.name) if f.name in changes or _empty(new): @@ -983,14 +1140,24 @@ def merge_rounds(*sources: Sequence[RoundRow]) -> list[RoundRow]: continue # a table's RUC value is cited by its page if _empty(old): changes[f.name] = new + cites.pop(f.name, None) + if f.name in theirs: # the cite travels with the value + cites[f.name] = theirs[f.name] elif f.name in _CONFLICT_FIELDS and _differs(old, new): notes.append(f"conflict: {f.name} {new} ({r.extractor})") + note_cites += _cite_list(theirs.get(f.name)) if r.pe_note: notes.append(r.pe_note) + note_cites += _cite_list(theirs.get("pe_note")) + uniq = list(dict.fromkeys(note_cites)) + if uniq: + cites["pe_note"] = uniq[0] if len(uniq) == 1 else uniq if r.extractor not in cur.extractor.split("+"): changes["extractor"] = f"{cur.extractor}+{r.extractor}" - merged[key] = replace(cur, pe_note="; ".join(notes), **changes) - return list(merged.values()) + merged[key] = replace( + cur, pe_note="; ".join(notes), field_cites=_encode(cites), **changes + ) + return [_prune(r) for r in merged.values()] def apply_paragraph_rules( @@ -1015,8 +1182,9 @@ def apply_paragraph_rules( and r.final_work_rvu is None and r.proposed_work_rvu is not None ): - out[i] = replace( + out[i] = _put( r, + para["p_id"], final_work_rvu=r.proposed_work_rvu, final_p_id=para["p_id"], rationale_p_id=para["p_id"], @@ -1033,8 +1201,9 @@ def apply_paragraph_rules( and r.proposed_work_rvu is not None and r.code in present ): - out[i] = replace( + out[i] = _put( r, + para["p_id"], ruc_work_rvu=r.proposed_work_rvu, ruc_p_id=r.ruc_p_id or para["p_id"], ) diff --git a/tests/cli/test_pfs_cli.py b/tests/cli/test_pfs_cli.py index 1c1479b..029fe34 100644 --- a/tests/cli/test_pfs_cli.py +++ b/tests/cli/test_pfs_cli.py @@ -1344,7 +1344,7 @@ class TestRounds: rows = [ self._row(), self._row(rule_year=2008, rule_kind="final"), - self._row(code="99342", rule_year=2023), + self._row(code="99342", rule_year=2023, field_cites='{"decision": 1}'), ] seen = {} @@ -1406,7 +1406,9 @@ class TestRounds: assert "CY2023 final: 2 rounds" in res.output assert "done 2, skipped 1, failed 0, failed_pending 3" in res.output assert "wrote 3 rounds" in res.output - assert len(read_rounds(con._real, "HOME-VISITS")) == 3 + got = read_rounds(con._real, "HOME-VISITS") + assert len(got) == 3 + assert {r.code: r.field_cites for r in got}["99342"] == '{"decision": 1}' def test_rounds_without_write_writes_nothing(self, con, monkeypatch): order = [] diff --git a/tests/pfs/test_rounds.py b/tests/pfs/test_rounds.py index 73274d1..7a55700 100644 --- a/tests/pfs/test_rounds.py +++ b/tests/pfs/test_rounds.py @@ -13,6 +13,7 @@ from bib.item import Rule from bib.store import Store from pfs.codetables import RoundRow, ensure_tables, read_rounds, write_rounds from pfs.rounds import ( + CITE_FIELDS, ROUND_PROMPT_VERSION, ROUND_SCHEMA, FrTable, @@ -22,6 +23,7 @@ from pfs.rounds import ( decision_of, family_as_proposed, family_rounds, + field_cites, find_tables, fr_text_path, guard, @@ -1139,10 +1141,13 @@ def test_final_p_id_roundtrips_and_old_tables_gain_the_column(): " rationale_quote VARCHAR, pe_note VARCHAR, extractor VARCHAR)" ) ensure_tables(con) - row = replace(_row(code="99341"), final_p_id=1685, ruc_p_id=877) + cites = '{"final_work_rvu": 1685, "ruc_work_rvu": "page:66361"}' + row = replace(_row(code="99341"), final_p_id=1685, ruc_p_id=877, field_cites=cites) write_rounds(con, "F", [row]) got = read_rounds(con, "F")[0] - assert (got.final_p_id, got.ruc_p_id) == (1685, 877) + assert (got.final_p_id, got.ruc_p_id, got.field_cites) == (1685, 877, cites) + assert field_cites(got) == {"final_work_rvu": 1685, "ruc_work_rvu": "page:66361"} + assert got == row # ── fix round 3 (N1, N2, R26) ──────────────────────────────────────── @@ -1346,13 +1351,401 @@ def test_hypothetical_sentences_yield_no_regex_values(): "proposed", "Were we to propose a work RVU of 2.00 for CPT code 99348, it would be consistent.", ), - ( - "proposed", - "Similarly, the RUC-recommended work RVU of 1.40 for CPT code 99348 would be the second highest.", - ), ): assert para_values(t, {"99348"}, rule_kind=k) == [], t real = "We agree that it would be more accurate to finalize the RUC-recommended work RVU of 1.25 for CPT code 99348." assert ("99348", "final_work_rvu", 1.25) in para_values( real, {"99348"}, rule_kind="final" ) + # N3-c: "would be the/consistent/..." states the value (5ITGVDJV ¶429, JCWARTJD ¶833) + genuine = "Similarly, the RUC-recommended work RVU of 1.40 for CPT code 92979 would be the second highest work RVU for add-on codes." + for k in ("proposed", "final"): + assert para_values(genuine, {"92979"}, rule_kind=k) == [ + ("92979", "ruc_work_rvu", 1.4) + ] + t = "We are proposing a work RVU of 1.50 for CPT code 99348, which would be consistent with the RUC survey." + assert para_values(t, {"99348"}, rule_kind="proposed") == [ + ("99348", "proposed_work_rvu", 1.5) + ] + + +# ── fix round 4 (R27 field cites, R28 model gates, R29 minors) ─────── + +P878 = ( + "We are proposing the RUC-recommended work RVU for all eight CPT codes in the " + "Home or Residence Services CPT code family. We are proposing a work RVU of " + f"3.88 for CPT code 99345 {DESC}, and a work RVU of 3.60 for CPT code 99350 {DESC}." +) +P880 = ( + "This means that for CPT code 99345, assuming we finalize the RUC-recommended " + "total time of 126 minutes, prolonged services would be reportable once 141 or " + "more minutes are spent by a physician or NPP providing home or residence " + "services. Likewise, for CPT code 99350, assuming we finalize the " + "RUC-recommended total time of 97 minutes, prolonged services would be " + "reportable once 112 or more minutes are spent by a physician or NPP providing " + "home or residence services." +) + + +def _p23(p_id, text): + return para( + p_id, text, item="MZ24MX5S", kind="proposed", title=TITLE23P, date="2022-07-29" + ) + + +def _cited_fields(r): + return {f for f in CITE_FIELDS if getattr(r, f) not in (None, "")} + + +def test_field_cites_cite_each_value_to_the_paragraph_stating_it(): + """R26b/R27: CY2023 proposed 99350 — percentile ¶877, proposal and RUC + value ¶878, total time ¶880 (only ¶880 states the minutes).""" + codes = {"99345", "99350"} + paras = [_p23(877, P877), _p23(878, P878), _p23(880, P880)] + regex = [r for p in paras for r in regex_rounds(p, codes, family="F")] + out = _reply( + _d("99345", ruc_work_rvu=3.88, proposed_work_rvu=3.88, survey_total_min=126), + _d( + "99350", + ruc_work_rvu=3.6, + proposed_work_rvu=3.6, + survey_total_min=97, + decision="accept", + ), + ) + model = rows_from_output(out, paras[2], codes, family="F", model="m") + rows = { + r.code: r for r in apply_paragraph_rules(merge_rounds([], regex, model), paras) + } + r = rows["99350"] + assert ( + r.ruc_work_rvu, + r.proposed_work_rvu, + r.survey_total_min, + r.ruc_percentile, + r.decision, + ) == (3.6, 3.6, 97, "median", "accept") + assert field_cites(r) == { + "ruc_percentile": 877, + "ruc_work_rvu": 878, + "proposed_work_rvu": 878, + "survey_total_min": 880, + "decision": 878, + } + assert (r.p_id, r.ruc_p_id) == (878, 877) # row-level anchors unchanged + assert field_cites(rows["99345"])["survey_total_min"] == 880 + for row in rows.values(): + assert set(field_cites(row)) == _cited_fields(row) + + +def test_field_cites_final_rule_family_as_proposed(): + """CY2023 final 99341: the final, the decision and the rationale cite ¶1685.""" + r = _applied()["99341"] + assert field_cites(r) == { + "ruc_percentile": 1676, + "proposed_work_rvu": 1676, + "ruc_work_rvu": 1676, + "final_work_rvu": 1685, + "decision": 1685, + "rationale_quote": 1685, + } + + +def test_field_cites_for_table_cells_and_model_rows(): + rows = {r.code: r for r in _rounds(codes={"99343", "92557"})} + page = "page:66361" + assert field_cites(rows["99343"]) == { + f: page + for f in ( + "ruc_work_rvu", + "requested_work_rvu", + "proposed_work_rvu", + "final_work_rvu", + "decision", + "rationale_quote", + ) + } + assert field_cites(rows["92557"])["pe_note"] == page + out = _reply( + _d( + "99341", + proposed_work_rvu=1.0, + comparators=["99350"], + pe_note="removed SK062", + ), + quote="We proposed the RUC-recommended work RVU for all eight CPT codes in the Home or Residence Services CPT code family.", + societies=["American Academy of Family Physicians"], + ) + text = P1676 + " The American Academy of Family Physicians surveyed." + [r] = rows_from_output(out, para(1676, text), {"99341"}, family="F", model="m") + assert field_cites(r) == { + f: 1676 + for f in ( + "ruc_percentile", + "proposed_work_rvu", + "comparators", + "surveying_societies", + "rationale_quote", + "pe_note", + ) + } + + +def test_merge_carries_the_winning_sources_cite_and_lists_pe_note_sources(): + [table] = _rounds(codes={"99343"}) + table = replace(table, item_key="K", proposed_work_rvu=None) + model = rounds_mod._stamp( + replace( + table, + source="fr-para", + extractor="model:m@rounds-v1", + p_id=77, + proposed_work_rvu=2.27, + final_work_rvu=2.6, + ruc_percentile="25th", + survey_intra_min=40.0, + rationale_quote="", + pe_note="pe says", + decision="", + field_cites="", + ), + 77, + ) + later = rounds_mod._stamp( + replace(model, p_id=78, pe_note="more pe", final_work_rvu=2.7), 78 + ) + [m] = merge_rounds([table], [], [model, later]) + c = field_cites(m) + assert c["final_work_rvu"] == c["ruc_work_rvu"] == c["decision"] == "page:66361" + assert c["proposed_work_rvu"] == c["ruc_percentile"] == c["survey_intra_min"] == 77 + assert c["rationale_quote"] == "page:66361" + # pe_note joins three notes: the model's two and the conflicts they raised + assert m.pe_note.startswith("conflict: final_work_rvu 2.6") + assert c["pe_note"] == [77, 78] + assert set(c) == _cited_fields(m) + # one source -> a single cite; rows without cites merge without any + [one] = merge_rounds([model]) + assert field_cites(one)["pe_note"] == 77 + bare = replace(model, field_cites="") + [m] = merge_rounds([bare], [replace(bare, pe_note="x")]) + assert field_cites(m) == {} + + +def test_model_values_from_another_years_rule_are_dropped_in_both_kinds(): + """R28 (R26c): a recap sentence yields no model value and no label.""" + t = ( + "In the CY 2020 PFS final rule (84 FR 62796), we finalized the " + "RUC-recommended work RVU of 0.48 for CPT code 99281." + ) + out = _reply( + _d( + "99281", + ruc_work_rvu=0.48, + final_work_rvu=0.48, + decision="accept", + comparators=["99281", "99282"], + survey_total_min=0.48, + ), + quote=t, + ) + for kind, title, date in ( + ("proposed", "CY 2027 PFS Proposed Rule", "2026-07-16"), + ("final", "CY 2027 PFS Final Rule", "2026-11-09"), + ): + # the only other sentence names the code but states no number + p = para( + 1, t + " CPT code 99281 is related.", title=title, date=date, kind=kind + ) + [r] = rows_from_output(out, p, {"99281"}, family="F", model="m") + assert not has_content(r), kind + assert (r.ruc_work_rvu, r.decision, r.comparators, r.rationale_quote) == ( + None, + "", + "", + "", + ) + assert not any( + has_content(x) for x in apply_paragraph_rules(merge_rounds([r]), [p]) + ) + + +def test_model_numbers_need_a_plain_sentence_covering_the_code(): + t = ( + "The RUC recommended a work RVU of 2.00 for CPT code 99348. " + "If we were to accept a survey intra-service time of 30 minutes for CPT code " + "99348, the value would rise. CPT code 99349 is similar. " + "The RUC recommended a total time of 60 minutes for all eight CPT codes in the family. " + "For all codes except CPT code 99350 the RUC recommended 45 minutes of post-service time." + ) + out = _reply( + _d("99348", ruc_work_rvu=2.0, survey_intra_min=30, survey_total_min=60), + _d("99349", ruc_work_rvu=2.0, survey_intra_min=45), + ) + rows = { + r.code: r + for r in rows_from_output(out, pp(t), {"99348", "99349"}, family="F", model="m") + } + assert (rows["99348"].ruc_work_rvu, rows["99348"].survey_intra_min) == (2.0, None) + assert rows["99348"].survey_total_min == 60 # a collective sentence covers it + # 2.00 is 99348's; 45 sits in a sentence naming another code + assert (rows["99349"].ruc_work_rvu, rows["99349"].survey_intra_min) == (None, None) + # a hypothetical rationale quote is dropped too + q = "If we were to accept a survey intra-service time of 30 minutes for CPT code 99348, the value would rise." + [r] = rows_from_output( + _reply(_d("99348", ruc_work_rvu=2.0), quote=q), + pp(t), + {"99348"}, + family="F", + model="m", + ) + assert (r.ruc_work_rvu, r.rationale_quote, r.rationale_p_id) == (2.0, "", 0) + + +def test_model_label_needs_ruc_and_the_acted_value(): + """R28: a final rule's label needs a final; a proposed rule's a proposal.""" + t = "We proposed the RUC-recommended work RVU of 2.00 for CPT code 99348." + out = _reply( + _d("99348", ruc_work_rvu=2.0, proposed_work_rvu=2.0, decision="refine") + ) + [r] = rows_from_output(out, para(1, t), {"99348"}, family="F", model="m") + assert (r.ruc_work_rvu, r.proposed_work_rvu, r.final_work_rvu) == (2.0, 2.0, None) + assert r.decision == "" and "decision" not in field_cites(r) + t = "We are proposing the RUC-recommended work RVU of 2.00 for CPT code 99348." + [r] = rows_from_output(out, pp(t), {"99348"}, family="F", model="m") + assert r.decision == "accept" and field_cites(r)["decision"] == 1 + + +def test_stated_interim_or_reject_labels_are_not_overridden(): + """N3-d: numbers do not override "interim"/"reject" the sentence says.""" + t = "On an interim basis, we are finalizing the RUC-recommended work RVU of 2.00 for CPT code 99348." + out = _reply(_d("99348", ruc_work_rvu=2.0, final_work_rvu=2.0, decision="interim")) + [r] = rows_from_output(out, para(1, t), {"99348"}, family="F", model="m") + assert r.decision == "interim" + assert apply_paragraph_rules([r], [])[0].decision == "interim" + t = ( + "We reject the RUC-recommended work RVU of 2.00 for CPT code 99348 and are " + "finalizing a work RVU of 1.80 for CPT code 99348." + ) + out = _reply(_d("99348", ruc_work_rvu=2.0, final_work_rvu=1.8, decision="reject")) + [r] = rows_from_output(out, para(1, t), {"99348"}, family="F", model="m") + assert r.decision == "reject" + # the word is not in the sentence: the numbers decide + t = "We are finalizing the RUC-recommended work RVU of 2.00 for CPT code 99348." + for label in ("interim", "reject"): + out = _reply(_d("99348", ruc_work_rvu=2.0, final_work_rvu=2.0, decision=label)) + [r] = rows_from_output(out, para(1, t), {"99348"}, family="F", model="m") + assert r.decision == "accept", label + + +def test_maintain_only_as_cms_own_verb(): + """N3-a: NV3Q43B7 ¶1598/¶1602 are recommendations, not CMS finals.""" + for t in ( + "The AMA RUC referred CPT code 24160 to the CPT Editorial Panel for revision of " + "the descriptor and recommended that we maintain the current (CY 2012) work RVU " + "of 8.00 for CPT code 24160 for CY 2013.", + "Commenters recommended we maintain the 2009 valued AMA RUC work RVU of 0.60 for CPT code 24160.", + "Commenters recommended that we will maintain a work RVU of 0.60 for CPT code 24160.", + "Commenters recommend we are maintaining a work RVU of 0.60 for CPT code 24160.", + ): + assert para_values(t, {"24160"}, rule_kind="final") == [], t + # bare "we maintain that" is an argument: no proposal or final (the RUC's + # own figure is still the RUC's) + t = "We maintain that the RUC-recommended work RVU of 2.00 for CPT code 99348 overstates the work." + assert para_values(t, {"99348"}, rule_kind="final") == [ + ("99348", "ruc_work_rvu", 2.0) + ] + t = "We will maintain a work RVU of 1.56 for CPT code 99348." + assert para_values(t, {"99348"}, rule_kind="final") == [ + ("99348", "final_work_rvu", 1.56) + ] + assert para_values(t, {"99348"}, rule_kind="proposed") == [ + ("99348", "proposed_work_rvu", 1.56) + ] + t = "Therefore we are maintaining 2.8 work RVUs for CPT code 43219." + assert para_values(t, {"43219"}, rule_kind="final") == [ + ("43219", "final_work_rvu", 2.8) + ] + + +def test_recap_lead_ins_and_prefixes(): + """N3-b: "Response:"/"Comment:" lead-ins and "CY 20XX final rule" without "PFS".""" + for t in ( + "Response: In the CY 2023 PFS final rule, we finalized a work RVU of 1.00 for CPT code 99341.", + "Comment: In the CY 2023 final rule, we finalized a work RVU of 1.00 for CPT code 99341.", + "In the CY 2023 final rule, we finalized a work RVU of 1.00 for CPT code 99341.", + "In the CY 2023 proposed rule, we proposed a work RVU of 1.00 for CPT code 99341.", + ): + for kind in ("proposed", "final"): + assert para_values(t, {"99341"}, rule_kind=kind, rule_year=2027) == [], t + # mid-sentence references and "Comment on the ..." headings still state values + kept = ( + ( + "The AMA RUC recommended 3.61 work RVUs for CPT code 17106, which we " + "accepted in the CY 2009 PFS final rule with comment.", + 2010, + ("17106", "ruc_work_rvu", 3.61), + ), + ( + "Comment on the CY 2016 PFS final rule with comment period: Commenters " + "urged CMS to accept the RUC-recommended work RVU of 9.00 for CPT code 43210.", + 2017, + ("43210", "ruc_work_rvu", 9.0), + ), + ( + "Similarly, we are finalizing a work RVU of 8.94 for CPT code 43276 based " + "upon the formula described in the CY 2014 final rule with comment period.", + 2015, + ("43276", "final_work_rvu", 8.94), + ), + ) + for t, year, triple in kept: + assert para_values(t, {triple[0]}, rule_kind="final", rule_year=year) == [ + triple + ], t + + +def test_percentiles_skip_recaps_and_hypotheticals(): + recap = "In the CY 2020 PFS final rule, the RUC recommended the 25th percentile for CPT code 99341." + assert _percentiles(recap, rule_year=2027) == {} + assert _percentiles(recap) == {"99341": "25th"} # no rule year, no recap test + hypo = "If the RUC recommended the median for CPT code 99341, the value would rise." + assert _percentiles(hypo) == {} + [r] = regex_rounds( + para( + 1, + recap + " We proposed a work RVU of 1.00 for CPT code 99341.", + title="CY 2027 PFS Final Rule", + date="2026-11-09", + ), + {"99341"}, + family="F", + ) + assert (r.ruc_percentile, r.proposed_work_rvu) == ("", 1.0) + assert guard(replace(r, ruc_percentile="25th"), recap).ruc_percentile == "" + + +def test_family_rounds_cite_every_value(tmp_path): + s, _ = _store(tmp_path) + ask = FakeAsk(_good_reply()) + rows, _ = family_rounds(None, s, FAM, ask=ask, log=RunLog(tmp_path / "l.jsonl")) + assert rows + for r in rows: + assert set(field_cites(r)) == _cited_fields(r), r.code + + +def test_rule_items_keep_an_item_without_an_fr_document_row(tmp_path): + s = Store(":memory:", storage_dir=tmp_path / "st") + key = s.create(Rule(title=TITLE23, date_published="2022-11-18")) + s._con().execute( + "INSERT INTO fr_anchors (item_key, p_id, page, ordinal, text) VALUES (?,?,?,?,?)", + (key, 1, 1, 1, P1685), + ) + assert [i["item_key"] for i in rule_items(s, [])] == [key] + + +def test_percentiles_code_mentions_without_a_survey_value(): + t = ( + "The RUC recommended the 25th percentile work RVU after CPT code 99342 was " + "surveyed, with CPT code 99341 at the median intra-service time." + ) + assert _percentiles(t) == {}