fix(pfs): C1 — descriptor-stem regex and element-paragraph heuristic swallow FR cross-reference lines
`_stem_pattern`'s `re.I` made `[A-Z]` match any letter, so an enumeration item like "( 9) 99439 (code for non-complex chronic care management)." read as the opening of 99439's own descriptor; `descriptor_runs` then consumed the following list item as an element paragraph, and `extract_run` minted rows anchored to a paragraph naming a different code entirely. Live table already held wrong rows from this (an APCM paragraph under 99439, a TCM enumeration line under 99490). Keep the stem match case-insensitive (some rules print descriptors lowercase) but reject the cross-reference form with a negative lookahead on "code(s)". `is_element_paragraph` also rejects lines that open with an enumeration marker or contain "code(s) for" — the same enumeration items were passing its ")."-ending heuristic too. Claude-Session: https://claude.ai/code/session_01Aum3pEMAM3yQVdFSdVe6Gc
This commit is contained in:
@@ -61,7 +61,12 @@ def rule_year_of(title: str, date_published: str) -> int:
|
||||
|
||||
def _stem_pattern(code: str) -> re.Pattern[str]:
|
||||
# "code 99490 (Chronic…", "CPT code 99490 (", "HCPCS code G0556 ( Advanced…"
|
||||
return re.compile(rf"\b{re.escape(code)}\s*\(\s*[A-Z]", re.I)
|
||||
# Case-insensitive on purpose (some rules print descriptors lowercase:
|
||||
# "CPT code 99490 (chronic care management services…"), but the
|
||||
# negative lookahead rejects the cross-reference/enumeration form
|
||||
# "( 9) 99439 (code for non-complex chronic care management)." — that
|
||||
# opening "(code..." is never a descriptor stem.
|
||||
return re.compile(rf"\b{re.escape(code)}\s*\(\s*(?!codes?\b)[A-Za-z]", re.I)
|
||||
|
||||
|
||||
def descriptor_runs(
|
||||
|
||||
@@ -334,6 +334,11 @@ def parse_descriptor(text: str) -> tuple[Element, ...]:
|
||||
_RE_PROSE_START = re.compile(
|
||||
r"^(?:Comment|Response|We|In the|After|For|As|The|CMS|Section)\b"
|
||||
)
|
||||
# An enumeration list item ("( 9) 99439 (code for …)") or a cross-reference
|
||||
# sentence ("… codes for remote physiologic monitoring …") is prose about a
|
||||
# code, not a fragment of that code's own descriptor — even though it can
|
||||
# end in ");" and otherwise look like a real element line (C1).
|
||||
_RE_ENUM_OR_CODE_FOR = re.compile(r"^\(\s*\d+\)|\b(?:code|codes)\s+for\b", re.I)
|
||||
|
||||
|
||||
def is_element_paragraph(text: str) -> bool:
|
||||
@@ -344,7 +349,7 @@ def is_element_paragraph(text: str) -> bool:
|
||||
return False
|
||||
if t.startswith("++"):
|
||||
return True
|
||||
if _RE_PROSE_START.match(t):
|
||||
if _RE_PROSE_START.match(t) or _RE_ENUM_OR_CODE_FOR.search(t):
|
||||
return False
|
||||
return t.endswith((";", ")", ").", ":")) or (
|
||||
t[0].islower() is False and t.count(". ") <= 1 and len(t) < 300
|
||||
|
||||
@@ -154,6 +154,61 @@ class TestDescriptorRuns:
|
||||
run = descriptor_runs(store, "99490", max_elements=2)[0]
|
||||
assert [p.p_id for p in run.elements] == [1245, 1246]
|
||||
|
||||
def test_enumeration_list_item_is_not_a_descriptor_stem(self, store):
|
||||
# C1 live-corpus incident: "( 9) 99439 (code for non-complex
|
||||
# chronic care management)." reads like a stem ("99439 (") but is
|
||||
# an enumeration/cross-reference line, not the opening of 99439's
|
||||
# own descriptor.
|
||||
store.con.executemany(
|
||||
"INSERT INTO fr_anchors VALUES (?,?,?,?,?)",
|
||||
[
|
||||
(
|
||||
"DE2VH9PD",
|
||||
1300,
|
||||
67800,
|
||||
1300,
|
||||
"( 9) 99439 (code for non-complex chronic care management).",
|
||||
),
|
||||
(
|
||||
"DE2VH9PD",
|
||||
1301,
|
||||
67800,
|
||||
1301,
|
||||
"( 10) 99457 and 99458 (codes for remote physiologic monitoring).",
|
||||
),
|
||||
],
|
||||
)
|
||||
assert descriptor_runs(store, "99439") == []
|
||||
|
||||
def test_lowercase_stem_still_matches(self):
|
||||
# `_stem_pattern` stays case-insensitive: some rules print the
|
||||
# descriptor lowercase.
|
||||
s = _Store()
|
||||
s.con.execute(
|
||||
"INSERT INTO items VALUES ('AAAAAAAA', 'Medicare Program; CY 2021 PFS Final Rule', '2020-11-13')"
|
||||
)
|
||||
s.con.executemany(
|
||||
"INSERT INTO fr_anchors VALUES (?,?,?,?,?)",
|
||||
[
|
||||
(
|
||||
"AAAAAAAA",
|
||||
1,
|
||||
100,
|
||||
1,
|
||||
"CPT code 99490 (chronic care management services, at least 20 "
|
||||
"minutes of clinical staff time directed by a physician or other "
|
||||
"qualified health care professional, per calendar month, with the "
|
||||
"following required elements:",
|
||||
),
|
||||
("AAAAAAAA", 2, 100, 2, "Consent;"),
|
||||
],
|
||||
)
|
||||
try:
|
||||
runs = descriptor_runs(s, "99490")
|
||||
assert len(runs) == 1
|
||||
finally:
|
||||
s.close()
|
||||
|
||||
|
||||
class TestDuckDBSources:
|
||||
def test_long_description(self, con):
|
||||
|
||||
@@ -152,3 +152,22 @@ class TestHelpers:
|
||||
def test_slug(self):
|
||||
assert slug(Element(ElementType.TIME, "first", "20")) == "time=first:20"
|
||||
assert slug(Element(ElementType.ACTIVITY, "consent")) == "activity=consent"
|
||||
|
||||
def test_enumeration_list_items_are_not_element_paragraphs(self):
|
||||
# C1: "( 9) 99439 (code for non-complex chronic care management)."
|
||||
# is a cross-reference enumeration item, not a fragment of 99439's
|
||||
# own descriptor, even though it ends in ")." like a real element
|
||||
# line. The still-accepted lines below must keep passing.
|
||||
assert not is_element_paragraph(
|
||||
"( 9) 99439 (code for non-complex chronic care management)."
|
||||
)
|
||||
assert not is_element_paragraph(
|
||||
"( 10) 99457 and 99458 (codes for remote physiologic monitoring)."
|
||||
)
|
||||
assert is_element_paragraph("Consent;")
|
||||
assert is_element_paragraph(
|
||||
"++ Document in patient's medical record that consent was obtained."
|
||||
)
|
||||
assert is_element_paragraph(
|
||||
"Comprehensive care plan established, implemented, revised, or monitored)."
|
||||
)
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pfs.descriptors import DescriptorRun, Para
|
||||
import sqlite3
|
||||
|
||||
from pfs.descriptors import DescriptorRun, Para, descriptor_runs
|
||||
from pfs.extract import Extraction, extract_run, extract_text
|
||||
|
||||
STEM = Para(
|
||||
@@ -91,6 +93,64 @@ class TestClassifier:
|
||||
assert calls == ["Something the vocabulary does not know about at all;"]
|
||||
|
||||
|
||||
class TestAnchorIntegrity:
|
||||
def test_no_row_anchored_to_a_foreign_code_paragraph(self):
|
||||
# C1 live-corpus incident: an enumeration/cross-reference line for
|
||||
# a *different* code ("( 9) 99457 and 99458 (codes for remote
|
||||
# physiologic monitoring, each additional 20 minutes).") must
|
||||
# never contribute an ElementRow to a 99439 run. The fix lives in
|
||||
# `descriptor_runs`/`is_element_paragraph` (it stops the run
|
||||
# before absorbing such a line) — this test drives the real
|
||||
# `descriptor_runs` to build the run, then checks `extract_run`'s
|
||||
# output end to end: no row may be anchored to a paragraph whose
|
||||
# text contains a different 5-char code and none of 99439.
|
||||
con = sqlite3.connect(":memory:")
|
||||
con.row_factory = sqlite3.Row
|
||||
con.executescript(
|
||||
"CREATE TABLE items (key TEXT PRIMARY KEY, title TEXT, date_published TEXT);"
|
||||
"CREATE TABLE fr_anchors (item_key TEXT, p_id INTEGER, page INTEGER, ordinal INTEGER, text TEXT);"
|
||||
)
|
||||
con.execute(
|
||||
"INSERT INTO items VALUES ('XFGGRBDH', "
|
||||
"'Medicare and Medicaid Programs; CY 2025 Payment Policies', '2024-07-31')"
|
||||
)
|
||||
rows = [
|
||||
(
|
||||
832,
|
||||
61652,
|
||||
"CPT code 99439 (Chronic care management services, each additional 20 "
|
||||
"minutes of clinical staff time directed by a physician or other "
|
||||
"qualified health care professional, per calendar month, with the "
|
||||
"following required elements:",
|
||||
),
|
||||
(833, 61652, "Consent;"),
|
||||
(
|
||||
834,
|
||||
61652,
|
||||
"( 9) 99457 and 99458 (codes for remote physiologic monitoring, "
|
||||
"each additional 20 minutes).",
|
||||
),
|
||||
]
|
||||
con.executemany(
|
||||
"INSERT INTO fr_anchors VALUES (?,?,?,?,?)",
|
||||
[("XFGGRBDH", p, pg, p, t) for p, pg, t in rows],
|
||||
)
|
||||
|
||||
class _S:
|
||||
def _con(self):
|
||||
return con
|
||||
|
||||
try:
|
||||
run = descriptor_runs(_S(), "99439")[0]
|
||||
# the enumeration line for 99457/99458 never joins the run
|
||||
assert [p.p_id for p in run.elements] == [833]
|
||||
x = extract_run(run)
|
||||
for r in x.rows:
|
||||
assert "99457" not in r.text and "99458" not in r.text
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
class TestText:
|
||||
def test_hcpcs_long_description(self):
|
||||
x = extract_text(
|
||||
|
||||
Reference in New Issue
Block a user