411 lines
13 KiB
Python
411 lines
13 KiB
Python
"""pfs.citecheck — the sentence-level citation check (dossier task 13)."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import pytest
|
||
|
||
from pfs.cite import Cite
|
||
from pfs.citecheck import (
|
||
Problem,
|
||
check_prose,
|
||
grounding_problems,
|
||
number_tokens,
|
||
split_sentences,
|
||
)
|
||
from pfs.dossier import Fact
|
||
|
||
|
||
def fact(text, values=None, fid="F1"):
|
||
return Fact(
|
||
id=fid,
|
||
kind="round",
|
||
year=2023,
|
||
codes=(),
|
||
text=text,
|
||
values=values or {},
|
||
cite=Cite(kind="fr_para", label="x", url="u"),
|
||
)
|
||
|
||
|
||
F = {
|
||
"F1": fact(
|
||
"For CY2023, the RUC recommended a work RVU of 1.00 for 99341 and CMS "
|
||
"finalized 1.00 (accept).",
|
||
{"final": 1.0},
|
||
),
|
||
"F2": fact(
|
||
"For CY2023, the RUC recommended a work RVU of 1.65 for 99342 and CMS "
|
||
"finalized 1.65 (accept).",
|
||
{"final": 1.65},
|
||
"F2",
|
||
),
|
||
"F3": fact(
|
||
"In CY2023 the payment for 99350 was $131.17 and the total changed −16.5%.",
|
||
{},
|
||
"F3",
|
||
),
|
||
}
|
||
|
||
|
||
def test_uncited_sentence_rejected():
|
||
assert check_prose("CMS accepted the RUC values.", F)[0].kind == "uncited"
|
||
|
||
|
||
def test_number_from_uncited_fact_rejected():
|
||
[p] = check_prose("99341 was finalized at 1.65 [F1].", F)
|
||
assert p.kind == "number_mismatch"
|
||
assert "1.65" in p.detail
|
||
|
||
|
||
def test_equivalent_number_formats_accepted():
|
||
assert check_prose("In CY2023 99341 was set at 1.0 work RVUs [F1].", F) == []
|
||
|
||
|
||
def test_unknown_fact_rejected():
|
||
[p] = check_prose("CMS finalized 1.00 for 99341 [F9].", F)
|
||
assert p.kind == "unknown_fact"
|
||
assert "F9" in p.detail
|
||
|
||
|
||
def test_abbreviations_do_not_split():
|
||
text = (
|
||
"The U.S. statute, e.g. section 1833, i.e. the Act, No. 5 and p. 49 "
|
||
"set 1.00 for 99341 [F1]."
|
||
)
|
||
assert split_sentences(text) == [text]
|
||
assert len(split_sentences("A one [F1]. B two [F2].")) == 2
|
||
|
||
|
||
def test_tags_after_the_period_stay_with_their_sentence():
|
||
assert split_sentences("CMS finalized 1.00 for 99341. [F1] Next [F2].") == [
|
||
"CMS finalized 1.00 for 99341. [F1]",
|
||
"Next [F2].",
|
||
]
|
||
assert check_prose("CMS finalized 1.00 for 99341 in CY2023. [F1]", F) == []
|
||
|
||
|
||
def test_money_and_percent_normalized():
|
||
assert number_tokens("$131.17") == {"131.17"}
|
||
assert number_tokens("−16.5%") == number_tokens("-16.5%") == {"-16.5%"}
|
||
assert number_tokens("–16.5%") == {"-16.5%"}
|
||
assert check_prose("99350 paid 131.17 in CY2023 and moved -16.5% [F3].", F) == []
|
||
|
||
|
||
def test_number_tokens_normalization():
|
||
assert number_tokens("1.00 3.60 +2 1,234 25th 2.0") == {
|
||
"1",
|
||
"3.6",
|
||
"2",
|
||
"1234",
|
||
"25th",
|
||
}
|
||
# bracketed segments (fact tags, labels) carry no numbers
|
||
assert number_tokens("x [F12] [87 FR 69608 ¶1672] y") == set()
|
||
# a range is two codes, never a negative number
|
||
assert number_tokens("99324–99328") == {"99324", "99328"}
|
||
# CY years and docket ids
|
||
assert number_tokens("CY2023") == {"2023"}
|
||
assert number_tokens("CMS-2022-0113") == {"2022", "0113"}
|
||
# HCPCS letters codes are whole tokens; modifiers carry no number
|
||
assert number_tokens("G0318 and B1") == {"G0318"}
|
||
|
||
|
||
def test_values_count_as_supported_numbers():
|
||
facts = {"F1": fact("CMS finalized the value for 99341 (accept).", {"final": 1.0})}
|
||
assert check_prose("99341 was finalized at 1.00 [F1].", facts) == []
|
||
facts = {"F1": fact("x", {"share": "12.5%", "label": "n/a"})}
|
||
assert check_prose("It was 12.5% [F1].", facts) == []
|
||
|
||
|
||
def test_several_tags_in_one_bracket():
|
||
assert check_prose("99341 and 99342 got 1.00 and 1.65 [F1, F2].", F) == []
|
||
|
||
|
||
def test_problem_carries_the_sentence():
|
||
p = check_prose("Uncited one. Fine 99341 1.00 [F1].", F)
|
||
assert p == [Problem("Uncited one.", "uncited", "no [F#] tag")]
|
||
|
||
|
||
@pytest.mark.parametrize("text", ["", " "])
|
||
def test_empty_prose_has_no_problems(text):
|
||
assert check_prose(text, F) == []
|
||
|
||
|
||
def test_zero_and_empty_values():
|
||
assert number_tokens("-0.00 and +0") == {"0"}
|
||
facts = {"F1": fact("x", {"none": None, "flag": True, "n": 0})}
|
||
assert check_prose("It was 0 [F1].", facts) == []
|
||
|
||
|
||
def _kind(text, kind="round", fid="F1", codes=()):
|
||
return Fact(fid, kind, 2023, codes, text, {}, Cite("fr_para", "x", "u"))
|
||
|
||
|
||
G = {
|
||
"F1": _kind(
|
||
"For CY2008, the RUC recommended a work RVU of 2.27 for 99326 and CMS finalized 2.63 (refine).",
|
||
codes=("99326",),
|
||
),
|
||
"F2": _kind(
|
||
"For CY2008, CMS finalized a work RVU of 3.28 for 99350.",
|
||
fid="F2",
|
||
codes=("99350",),
|
||
),
|
||
"F3": _kind(
|
||
"CMS reported that commenters supported the proposal (CY2005 final rule).",
|
||
"cms_summary",
|
||
"F3",
|
||
),
|
||
"F4": _kind(
|
||
"Of 3 CY2023 commenters we classified on the work RVU valuation, 1 supported.",
|
||
"stance_split",
|
||
"F4",
|
||
),
|
||
}
|
||
|
||
|
||
def test_grounded_restatement_passes():
|
||
assert (
|
||
grounding_problems(
|
||
"For CY2008, CMS finalized work RVUs for 99326 and 99350 [F1][F2].", G
|
||
)
|
||
== []
|
||
)
|
||
assert grounding_problems("No tag here.", G) == [] # left to check_prose
|
||
|
||
|
||
def test_invented_words_are_ungrounded():
|
||
[p] = grounding_problems(
|
||
"The CPT 2019 codebook introduced new codes for office visits [F1].", G
|
||
)
|
||
assert p.kind == "ungrounded" and "introduc" in p.detail
|
||
|
||
|
||
def test_a_range_spanning_uncited_codes_is_rejected():
|
||
[p] = grounding_problems(
|
||
"CMS finalized work RVUs for 99326 through 99350 [F1][F2].", G
|
||
)
|
||
assert p.kind == "code_mismatch" and "99341" in p.detail
|
||
|
||
|
||
def test_cms_summary_stays_attributed_and_apart():
|
||
[p] = grounding_problems(
|
||
"Commenters supported the proposal (CY2005 final rule) [F3].", G
|
||
)
|
||
assert p.kind == "unattributed"
|
||
assert (
|
||
grounding_problems(
|
||
"CMS reported that commenters supported the proposal [F3].", G
|
||
)
|
||
== []
|
||
)
|
||
[m] = grounding_problems(
|
||
"CMS reported that commenters supported; we classified 3 [F3][F4].", G
|
||
)
|
||
assert m.kind == "merged"
|
||
[u] = grounding_problems(
|
||
"Of 3 CY2023 commenters, 1 supported the work RVU valuation [F4].", G
|
||
)
|
||
assert u.kind == "unattributed" and "classified" in u.detail
|
||
|
||
|
||
def test_a_range_the_fact_writes_may_be_restated():
|
||
facts = {"F1": _kind("The Panel deleted codes 99324-99328.", "formation")}
|
||
assert grounding_problems("The Panel deleted codes 99324-99328 [F1].", facts) == []
|
||
|
||
|
||
# ── R95: negation/quantifier and number-role guards (reviewer's probes) ──
|
||
|
||
P = {
|
||
"F64": _kind(
|
||
'In the CY2013 final rule, CMS summarized: "A few commenters were divided."',
|
||
"cms_summary",
|
||
"F64",
|
||
),
|
||
"F325": _kind(
|
||
"For CY2023, the RUC recommended a work RVU of 1.00 for 99341 and CMS "
|
||
"finalized 1.00 (accept).",
|
||
fid="F325",
|
||
codes=("99341",),
|
||
),
|
||
"F20": _kind(
|
||
"For CY2008, the RUC recommended a work RVU of 2.27 for 99326 and CMS "
|
||
"finalized 2.63 (refine).",
|
||
fid="F20",
|
||
codes=("99326",),
|
||
),
|
||
"F369": _kind(
|
||
"Of 52 CY2023 commenters on this family in docket CMS-2022-0113, we "
|
||
"classified 28 with a firm position: 14 supported, 8 asked for "
|
||
"modifications and 6 opposed; 1 more was unclear.",
|
||
"stance_split",
|
||
"F369",
|
||
),
|
||
"F2": _kind(
|
||
"For CY2023, the RUC recommended a work RVU of 1.65 for 99342 and CMS "
|
||
"finalized 1.65 (accept).",
|
||
fid="F2",
|
||
codes=("99342",),
|
||
),
|
||
}
|
||
|
||
|
||
def test_probe_added_quantifier_is_rejected():
|
||
[p] = grounding_problems("CMS reported that all commenters were divided [F64].", P)
|
||
assert p.kind == "negation" and "'all'" in p.detail
|
||
|
||
|
||
def test_probe_added_negation_is_rejected():
|
||
[p] = grounding_problems(
|
||
"CMS did not finalize a work RVU of 1.00 for 99341 [F325].", P
|
||
)
|
||
assert p.kind == "negation"
|
||
[q] = grounding_problems("CMS didn't finalize 1.00 for 99341 [F325].", P)
|
||
assert q.kind == "negation"
|
||
|
||
|
||
def test_probe_swapped_ruc_and_final_values_are_rejected():
|
||
[p] = grounding_problems(
|
||
"For CY2008, the RUC recommended 2.63 for 99326 and CMS finalized 2.27 [F20].",
|
||
P,
|
||
)
|
||
assert p.kind == "number_role" and "2.63" in p.detail
|
||
|
||
|
||
def test_probe_swapped_stance_counts_are_rejected():
|
||
[p] = grounding_problems(
|
||
"Of 52 CY2023 commenters we classified, 14 opposed and 6 supported [F369].", P
|
||
)
|
||
assert p.kind == "number_role"
|
||
|
||
|
||
def test_number_beside_another_code_is_rejected():
|
||
[p] = grounding_problems("The RUC recommended 1.65 for 99341 [F325][F2].", P)
|
||
assert p.kind == "number_role" and "99341" in p.detail
|
||
|
||
|
||
def test_faithful_restatements_pass_the_guards():
|
||
assert (
|
||
grounding_problems(
|
||
"For CY2008, the RUC recommended 2.27 for 99326 and CMS finalized 2.63 [F20].",
|
||
P,
|
||
)
|
||
== []
|
||
)
|
||
assert (
|
||
grounding_problems(
|
||
"We classified 28 of 52 CY2023 commenters: 14 supported and 6 opposed [F369].",
|
||
P,
|
||
)
|
||
== []
|
||
)
|
||
assert (
|
||
grounding_problems("A few commenters were divided, CMS summarized [F64].", P)
|
||
== []
|
||
)
|
||
facts = {"F1": _kind("CMS did not finalize the change.", fid="F1")}
|
||
assert grounding_problems("CMS didn't finalize the change [F1].", facts) == []
|
||
|
||
|
||
def test_number_roles():
|
||
from pfs.citecheck import number_roles
|
||
|
||
roles = number_roles(P["F20"].text)
|
||
assert roles["2.27"] == {("ruc", "99326")} and roles["2.63"] == {("final", "99326")}
|
||
assert number_roles("In 2023, the 99341 code [F1].") == {}
|
||
|
||
|
||
def test_a_number_only_in_values_has_no_role_to_check():
|
||
facts = {
|
||
"F1": _kind(
|
||
"CMS finalized the value for 99341 (accept).", fid="F1", codes=("99341",)
|
||
)
|
||
}
|
||
facts["F1"] = Fact(
|
||
"F1",
|
||
"round",
|
||
2023,
|
||
("99341",),
|
||
facts["F1"].text,
|
||
{"final": 1.0},
|
||
Cite("fr_para", "x", "u"),
|
||
)
|
||
assert grounding_problems("CMS finalized 1.00 for 99341 [F1].", facts) == []
|
||
|
||
|
||
# ── I8b: a negation a cited fact says must stay in the sentence ──────
|
||
|
||
N = {
|
||
"F303": _kind(
|
||
"99343 no longer appears in the PFS RVU file from CY2023.",
|
||
"event",
|
||
"F303",
|
||
codes=("99343",),
|
||
),
|
||
"F7": _kind("CMS did not finalize the change for 99341.", fid="F7"),
|
||
"F8": _kind(
|
||
'In the CY2023 final rule, CMS summarized: "Commenters supported '
|
||
"finalizing the values.\" CMS's response: not finalized.",
|
||
"cms_summary",
|
||
"F8",
|
||
),
|
||
}
|
||
|
||
|
||
def test_probe_dropped_negation_is_rejected_by_the_citation_check():
|
||
[p] = check_prose("99343 appears in the PFS RVU file from CY2023 [F303].", N)
|
||
assert p.kind == "negation" and "appear" in p.detail
|
||
[q] = check_prose("CMS finalized the change for 99341 [F7].", N)
|
||
assert q.kind == "negation"
|
||
|
||
|
||
def test_kept_negations_and_template_copies_pass():
|
||
for text in (
|
||
"99343 no longer appears in the PFS RVU file from CY2023 [F303].",
|
||
"From CY2023, 99343 is not in the PFS RVU file [F303].",
|
||
"CMS didn't finalize the change for 99341 [F7].",
|
||
# a fact's other sentence is not the one the negation belongs to
|
||
'In the CY2023 final rule, CMS summarized: "Commenters supported '
|
||
"finalizing the values.\" [F8] CMS's response: not finalized [F8].",
|
||
):
|
||
assert check_prose(text, N) == [], text
|
||
|
||
|
||
def test_probe_first_written_over_a_disappearance_is_ungrounded():
|
||
[p] = grounding_problems(
|
||
"99343 no longer appears in the PFS RVU file first from CY2023 [F303].", N
|
||
)
|
||
assert p.kind == "ungrounded" and "first" in p.detail
|
||
for word in ("later", "before", "since"):
|
||
[q] = grounding_problems(
|
||
f"99343 no longer appears in the PFS RVU file {word} CY2023 [F303].", N
|
||
)
|
||
assert q.kind == "ungrounded" and word[:4] in q.detail
|
||
|
||
|
||
def test_negation_flips():
|
||
from pfs.citecheck import negation_flips
|
||
|
||
assert negation_flips(
|
||
"Travel time is counted.", ["Travel time is not counted."]
|
||
) == ["count"]
|
||
assert (
|
||
negation_flips("Travel time is excluded.", ["Travel time is not counted."])
|
||
== []
|
||
)
|
||
# the other way only when asked for
|
||
assert (
|
||
negation_flips("Travel time is not counted.", ["Travel time is counted."]) == []
|
||
)
|
||
assert negation_flips(
|
||
"Travel time is not counted.", ["Travel time is counted."], both_ways=True
|
||
) == ["count"]
|
||
# a word the source says both ways is not a flip
|
||
assert (
|
||
negation_flips(
|
||
"Y is counted.", ["X is not counted, but Y is counted."], both_ways=True
|
||
)
|
||
== []
|
||
)
|
||
assert negation_flips("", ["x"]) == [] and negation_flips("x is y.", []) == []
|