Files
stack/tests/pfs/test_citecheck.py

411 lines
13 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""pfs.citecheck — the sentence-level citation check (dossier task 13)."""
from __future__ import annotations
import pytest
from pfs.cite import Cite
from pfs.citecheck import (
Problem,
check_prose,
grounding_problems,
number_tokens,
split_sentences,
)
from pfs.dossier import Fact
def fact(text, values=None, fid="F1"):
return Fact(
id=fid,
kind="round",
year=2023,
codes=(),
text=text,
values=values or {},
cite=Cite(kind="fr_para", label="x", url="u"),
)
F = {
"F1": fact(
"For CY2023, the RUC recommended a work RVU of 1.00 for 99341 and CMS "
"finalized 1.00 (accept).",
{"final": 1.0},
),
"F2": fact(
"For CY2023, the RUC recommended a work RVU of 1.65 for 99342 and CMS "
"finalized 1.65 (accept).",
{"final": 1.65},
"F2",
),
"F3": fact(
"In CY2023 the payment for 99350 was $131.17 and the total changed −16.5%.",
{},
"F3",
),
}
def test_uncited_sentence_rejected():
assert check_prose("CMS accepted the RUC values.", F)[0].kind == "uncited"
def test_number_from_uncited_fact_rejected():
[p] = check_prose("99341 was finalized at 1.65 [F1].", F)
assert p.kind == "number_mismatch"
assert "1.65" in p.detail
def test_equivalent_number_formats_accepted():
assert check_prose("In CY2023 99341 was set at 1.0 work RVUs [F1].", F) == []
def test_unknown_fact_rejected():
[p] = check_prose("CMS finalized 1.00 for 99341 [F9].", F)
assert p.kind == "unknown_fact"
assert "F9" in p.detail
def test_abbreviations_do_not_split():
text = (
"The U.S. statute, e.g. section 1833, i.e. the Act, No. 5 and p. 49 "
"set 1.00 for 99341 [F1]."
)
assert split_sentences(text) == [text]
assert len(split_sentences("A one [F1]. B two [F2].")) == 2
def test_tags_after_the_period_stay_with_their_sentence():
assert split_sentences("CMS finalized 1.00 for 99341. [F1] Next [F2].") == [
"CMS finalized 1.00 for 99341. [F1]",
"Next [F2].",
]
assert check_prose("CMS finalized 1.00 for 99341 in CY2023. [F1]", F) == []
def test_money_and_percent_normalized():
assert number_tokens("$131.17") == {"131.17"}
assert number_tokens("−16.5%") == number_tokens("-16.5%") == {"-16.5%"}
assert number_tokens("–16.5%") == {"-16.5%"}
assert check_prose("99350 paid 131.17 in CY2023 and moved -16.5% [F3].", F) == []
def test_number_tokens_normalization():
assert number_tokens("1.00 3.60 +2 1,234 25th 2.0") == {
"1",
"3.6",
"2",
"1234",
"25th",
}
# bracketed segments (fact tags, labels) carry no numbers
assert number_tokens("x [F12] [87 FR 69608 ¶1672] y") == set()
# a range is two codes, never a negative number
assert number_tokens("99324–99328") == {"99324", "99328"}
# CY years and docket ids
assert number_tokens("CY2023") == {"2023"}
assert number_tokens("CMS-2022-0113") == {"2022", "0113"}
# HCPCS letters codes are whole tokens; modifiers carry no number
assert number_tokens("G0318 and B1") == {"G0318"}
def test_values_count_as_supported_numbers():
facts = {"F1": fact("CMS finalized the value for 99341 (accept).", {"final": 1.0})}
assert check_prose("99341 was finalized at 1.00 [F1].", facts) == []
facts = {"F1": fact("x", {"share": "12.5%", "label": "n/a"})}
assert check_prose("It was 12.5% [F1].", facts) == []
def test_several_tags_in_one_bracket():
assert check_prose("99341 and 99342 got 1.00 and 1.65 [F1, F2].", F) == []
def test_problem_carries_the_sentence():
p = check_prose("Uncited one. Fine 99341 1.00 [F1].", F)
assert p == [Problem("Uncited one.", "uncited", "no [F#] tag")]
@pytest.mark.parametrize("text", ["", " "])
def test_empty_prose_has_no_problems(text):
assert check_prose(text, F) == []
def test_zero_and_empty_values():
assert number_tokens("-0.00 and +0") == {"0"}
facts = {"F1": fact("x", {"none": None, "flag": True, "n": 0})}
assert check_prose("It was 0 [F1].", facts) == []
def _kind(text, kind="round", fid="F1", codes=()):
return Fact(fid, kind, 2023, codes, text, {}, Cite("fr_para", "x", "u"))
G = {
"F1": _kind(
"For CY2008, the RUC recommended a work RVU of 2.27 for 99326 and CMS finalized 2.63 (refine).",
codes=("99326",),
),
"F2": _kind(
"For CY2008, CMS finalized a work RVU of 3.28 for 99350.",
fid="F2",
codes=("99350",),
),
"F3": _kind(
"CMS reported that commenters supported the proposal (CY2005 final rule).",
"cms_summary",
"F3",
),
"F4": _kind(
"Of 3 CY2023 commenters we classified on the work RVU valuation, 1 supported.",
"stance_split",
"F4",
),
}
def test_grounded_restatement_passes():
assert (
grounding_problems(
"For CY2008, CMS finalized work RVUs for 99326 and 99350 [F1][F2].", G
)
== []
)
assert grounding_problems("No tag here.", G) == [] # left to check_prose
def test_invented_words_are_ungrounded():
[p] = grounding_problems(
"The CPT 2019 codebook introduced new codes for office visits [F1].", G
)
assert p.kind == "ungrounded" and "introduc" in p.detail
def test_a_range_spanning_uncited_codes_is_rejected():
[p] = grounding_problems(
"CMS finalized work RVUs for 99326 through 99350 [F1][F2].", G
)
assert p.kind == "code_mismatch" and "99341" in p.detail
def test_cms_summary_stays_attributed_and_apart():
[p] = grounding_problems(
"Commenters supported the proposal (CY2005 final rule) [F3].", G
)
assert p.kind == "unattributed"
assert (
grounding_problems(
"CMS reported that commenters supported the proposal [F3].", G
)
== []
)
[m] = grounding_problems(
"CMS reported that commenters supported; we classified 3 [F3][F4].", G
)
assert m.kind == "merged"
[u] = grounding_problems(
"Of 3 CY2023 commenters, 1 supported the work RVU valuation [F4].", G
)
assert u.kind == "unattributed" and "classified" in u.detail
def test_a_range_the_fact_writes_may_be_restated():
facts = {"F1": _kind("The Panel deleted codes 99324-99328.", "formation")}
assert grounding_problems("The Panel deleted codes 99324-99328 [F1].", facts) == []
# ── R95: negation/quantifier and number-role guards (reviewer's probes) ──
P = {
"F64": _kind(
'In the CY2013 final rule, CMS summarized: "A few commenters were divided."',
"cms_summary",
"F64",
),
"F325": _kind(
"For CY2023, the RUC recommended a work RVU of 1.00 for 99341 and CMS "
"finalized 1.00 (accept).",
fid="F325",
codes=("99341",),
),
"F20": _kind(
"For CY2008, the RUC recommended a work RVU of 2.27 for 99326 and CMS "
"finalized 2.63 (refine).",
fid="F20",
codes=("99326",),
),
"F369": _kind(
"Of 52 CY2023 commenters on this family in docket CMS-2022-0113, we "
"classified 28 with a firm position: 14 supported, 8 asked for "
"modifications and 6 opposed; 1 more was unclear.",
"stance_split",
"F369",
),
"F2": _kind(
"For CY2023, the RUC recommended a work RVU of 1.65 for 99342 and CMS "
"finalized 1.65 (accept).",
fid="F2",
codes=("99342",),
),
}
def test_probe_added_quantifier_is_rejected():
[p] = grounding_problems("CMS reported that all commenters were divided [F64].", P)
assert p.kind == "negation" and "'all'" in p.detail
def test_probe_added_negation_is_rejected():
[p] = grounding_problems(
"CMS did not finalize a work RVU of 1.00 for 99341 [F325].", P
)
assert p.kind == "negation"
[q] = grounding_problems("CMS didn't finalize 1.00 for 99341 [F325].", P)
assert q.kind == "negation"
def test_probe_swapped_ruc_and_final_values_are_rejected():
[p] = grounding_problems(
"For CY2008, the RUC recommended 2.63 for 99326 and CMS finalized 2.27 [F20].",
P,
)
assert p.kind == "number_role" and "2.63" in p.detail
def test_probe_swapped_stance_counts_are_rejected():
[p] = grounding_problems(
"Of 52 CY2023 commenters we classified, 14 opposed and 6 supported [F369].", P
)
assert p.kind == "number_role"
def test_number_beside_another_code_is_rejected():
[p] = grounding_problems("The RUC recommended 1.65 for 99341 [F325][F2].", P)
assert p.kind == "number_role" and "99341" in p.detail
def test_faithful_restatements_pass_the_guards():
assert (
grounding_problems(
"For CY2008, the RUC recommended 2.27 for 99326 and CMS finalized 2.63 [F20].",
P,
)
== []
)
assert (
grounding_problems(
"We classified 28 of 52 CY2023 commenters: 14 supported and 6 opposed [F369].",
P,
)
== []
)
assert (
grounding_problems("A few commenters were divided, CMS summarized [F64].", P)
== []
)
facts = {"F1": _kind("CMS did not finalize the change.", fid="F1")}
assert grounding_problems("CMS didn't finalize the change [F1].", facts) == []
def test_number_roles():
from pfs.citecheck import number_roles
roles = number_roles(P["F20"].text)
assert roles["2.27"] == {("ruc", "99326")} and roles["2.63"] == {("final", "99326")}
assert number_roles("In 2023, the 99341 code [F1].") == {}
def test_a_number_only_in_values_has_no_role_to_check():
facts = {
"F1": _kind(
"CMS finalized the value for 99341 (accept).", fid="F1", codes=("99341",)
)
}
facts["F1"] = Fact(
"F1",
"round",
2023,
("99341",),
facts["F1"].text,
{"final": 1.0},
Cite("fr_para", "x", "u"),
)
assert grounding_problems("CMS finalized 1.00 for 99341 [F1].", facts) == []
# ── I8b: a negation a cited fact says must stay in the sentence ──────
N = {
"F303": _kind(
"99343 no longer appears in the PFS RVU file from CY2023.",
"event",
"F303",
codes=("99343",),
),
"F7": _kind("CMS did not finalize the change for 99341.", fid="F7"),
"F8": _kind(
'In the CY2023 final rule, CMS summarized: "Commenters supported '
"finalizing the values.\" CMS's response: not finalized.",
"cms_summary",
"F8",
),
}
def test_probe_dropped_negation_is_rejected_by_the_citation_check():
[p] = check_prose("99343 appears in the PFS RVU file from CY2023 [F303].", N)
assert p.kind == "negation" and "appear" in p.detail
[q] = check_prose("CMS finalized the change for 99341 [F7].", N)
assert q.kind == "negation"
def test_kept_negations_and_template_copies_pass():
for text in (
"99343 no longer appears in the PFS RVU file from CY2023 [F303].",
"From CY2023, 99343 is not in the PFS RVU file [F303].",
"CMS didn't finalize the change for 99341 [F7].",
# a fact's other sentence is not the one the negation belongs to
'In the CY2023 final rule, CMS summarized: "Commenters supported '
"finalizing the values.\" [F8] CMS's response: not finalized [F8].",
):
assert check_prose(text, N) == [], text
def test_probe_first_written_over_a_disappearance_is_ungrounded():
[p] = grounding_problems(
"99343 no longer appears in the PFS RVU file first from CY2023 [F303].", N
)
assert p.kind == "ungrounded" and "first" in p.detail
for word in ("later", "before", "since"):
[q] = grounding_problems(
f"99343 no longer appears in the PFS RVU file {word} CY2023 [F303].", N
)
assert q.kind == "ungrounded" and word[:4] in q.detail
def test_negation_flips():
from pfs.citecheck import negation_flips
assert negation_flips(
"Travel time is counted.", ["Travel time is not counted."]
) == ["count"]
assert (
negation_flips("Travel time is excluded.", ["Travel time is not counted."])
== []
)
# the other way only when asked for
assert (
negation_flips("Travel time is not counted.", ["Travel time is counted."]) == []
)
assert negation_flips(
"Travel time is not counted.", ["Travel time is counted."], both_ways=True
) == ["count"]
# a word the source says both ways is not a flip
assert (
negation_flips(
"Y is counted.", ["X is not counted, but Y is counted."], both_ways=True
)
== []
)
assert negation_flips("", ["x"]) == [] and negation_flips("x is y.", []) == []