fix(pfs): C1 — descriptor-stem regex and element-paragraph heuristic swallow FR cross-reference lines
`_stem_pattern`'s `re.I` made `[A-Z]` match any letter, so an enumeration item like "( 9) 99439 (code for non-complex chronic care management)." read as the opening of 99439's own descriptor; `descriptor_runs` then consumed the following list item as an element paragraph, and `extract_run` minted rows anchored to a paragraph naming a different code entirely. Live table already held wrong rows from this (an APCM paragraph under 99439, a TCM enumeration line under 99490). Keep the stem match case-insensitive (some rules print descriptors lowercase) but reject the cross-reference form with a negative lookahead on "code(s)". `is_element_paragraph` also rejects lines that open with an enumeration marker or contain "code(s) for" — the same enumeration items were passing its ")."-ending heuristic too. Claude-Session: https://claude.ai/code/session_01Aum3pEMAM3yQVdFSdVe6Gc
This commit is contained in:
@@ -61,7 +61,12 @@ def rule_year_of(title: str, date_published: str) -> int:
|
|||||||
|
|
||||||
def _stem_pattern(code: str) -> re.Pattern[str]:
|
def _stem_pattern(code: str) -> re.Pattern[str]:
|
||||||
# "code 99490 (Chronic…", "CPT code 99490 (", "HCPCS code G0556 ( Advanced…"
|
# "code 99490 (Chronic…", "CPT code 99490 (", "HCPCS code G0556 ( Advanced…"
|
||||||
return re.compile(rf"\b{re.escape(code)}\s*\(\s*[A-Z]", re.I)
|
# Case-insensitive on purpose (some rules print descriptors lowercase:
|
||||||
|
# "CPT code 99490 (chronic care management services…"), but the
|
||||||
|
# negative lookahead rejects the cross-reference/enumeration form
|
||||||
|
# "( 9) 99439 (code for non-complex chronic care management)." — that
|
||||||
|
# opening "(code..." is never a descriptor stem.
|
||||||
|
return re.compile(rf"\b{re.escape(code)}\s*\(\s*(?!codes?\b)[A-Za-z]", re.I)
|
||||||
|
|
||||||
|
|
||||||
def descriptor_runs(
|
def descriptor_runs(
|
||||||
|
|||||||
@@ -334,6 +334,11 @@ def parse_descriptor(text: str) -> tuple[Element, ...]:
|
|||||||
_RE_PROSE_START = re.compile(
|
_RE_PROSE_START = re.compile(
|
||||||
r"^(?:Comment|Response|We|In the|After|For|As|The|CMS|Section)\b"
|
r"^(?:Comment|Response|We|In the|After|For|As|The|CMS|Section)\b"
|
||||||
)
|
)
|
||||||
|
# An enumeration list item ("( 9) 99439 (code for …)") or a cross-reference
|
||||||
|
# sentence ("… codes for remote physiologic monitoring …") is prose about a
|
||||||
|
# code, not a fragment of that code's own descriptor — even though it can
|
||||||
|
# end in ");" and otherwise look like a real element line (C1).
|
||||||
|
_RE_ENUM_OR_CODE_FOR = re.compile(r"^\(\s*\d+\)|\b(?:code|codes)\s+for\b", re.I)
|
||||||
|
|
||||||
|
|
||||||
def is_element_paragraph(text: str) -> bool:
|
def is_element_paragraph(text: str) -> bool:
|
||||||
@@ -344,7 +349,7 @@ def is_element_paragraph(text: str) -> bool:
|
|||||||
return False
|
return False
|
||||||
if t.startswith("++"):
|
if t.startswith("++"):
|
||||||
return True
|
return True
|
||||||
if _RE_PROSE_START.match(t):
|
if _RE_PROSE_START.match(t) or _RE_ENUM_OR_CODE_FOR.search(t):
|
||||||
return False
|
return False
|
||||||
return t.endswith((";", ")", ").", ":")) or (
|
return t.endswith((";", ")", ").", ":")) or (
|
||||||
t[0].islower() is False and t.count(". ") <= 1 and len(t) < 300
|
t[0].islower() is False and t.count(". ") <= 1 and len(t) < 300
|
||||||
|
|||||||
@@ -154,6 +154,61 @@ class TestDescriptorRuns:
|
|||||||
run = descriptor_runs(store, "99490", max_elements=2)[0]
|
run = descriptor_runs(store, "99490", max_elements=2)[0]
|
||||||
assert [p.p_id for p in run.elements] == [1245, 1246]
|
assert [p.p_id for p in run.elements] == [1245, 1246]
|
||||||
|
|
||||||
|
def test_enumeration_list_item_is_not_a_descriptor_stem(self, store):
|
||||||
|
# C1 live-corpus incident: "( 9) 99439 (code for non-complex
|
||||||
|
# chronic care management)." reads like a stem ("99439 (") but is
|
||||||
|
# an enumeration/cross-reference line, not the opening of 99439's
|
||||||
|
# own descriptor.
|
||||||
|
store.con.executemany(
|
||||||
|
"INSERT INTO fr_anchors VALUES (?,?,?,?,?)",
|
||||||
|
[
|
||||||
|
(
|
||||||
|
"DE2VH9PD",
|
||||||
|
1300,
|
||||||
|
67800,
|
||||||
|
1300,
|
||||||
|
"( 9) 99439 (code for non-complex chronic care management).",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"DE2VH9PD",
|
||||||
|
1301,
|
||||||
|
67800,
|
||||||
|
1301,
|
||||||
|
"( 10) 99457 and 99458 (codes for remote physiologic monitoring).",
|
||||||
|
),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
assert descriptor_runs(store, "99439") == []
|
||||||
|
|
||||||
|
def test_lowercase_stem_still_matches(self):
|
||||||
|
# `_stem_pattern` stays case-insensitive: some rules print the
|
||||||
|
# descriptor lowercase.
|
||||||
|
s = _Store()
|
||||||
|
s.con.execute(
|
||||||
|
"INSERT INTO items VALUES ('AAAAAAAA', 'Medicare Program; CY 2021 PFS Final Rule', '2020-11-13')"
|
||||||
|
)
|
||||||
|
s.con.executemany(
|
||||||
|
"INSERT INTO fr_anchors VALUES (?,?,?,?,?)",
|
||||||
|
[
|
||||||
|
(
|
||||||
|
"AAAAAAAA",
|
||||||
|
1,
|
||||||
|
100,
|
||||||
|
1,
|
||||||
|
"CPT code 99490 (chronic care management services, at least 20 "
|
||||||
|
"minutes of clinical staff time directed by a physician or other "
|
||||||
|
"qualified health care professional, per calendar month, with the "
|
||||||
|
"following required elements:",
|
||||||
|
),
|
||||||
|
("AAAAAAAA", 2, 100, 2, "Consent;"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
runs = descriptor_runs(s, "99490")
|
||||||
|
assert len(runs) == 1
|
||||||
|
finally:
|
||||||
|
s.close()
|
||||||
|
|
||||||
|
|
||||||
class TestDuckDBSources:
|
class TestDuckDBSources:
|
||||||
def test_long_description(self, con):
|
def test_long_description(self, con):
|
||||||
|
|||||||
@@ -152,3 +152,22 @@ class TestHelpers:
|
|||||||
def test_slug(self):
|
def test_slug(self):
|
||||||
assert slug(Element(ElementType.TIME, "first", "20")) == "time=first:20"
|
assert slug(Element(ElementType.TIME, "first", "20")) == "time=first:20"
|
||||||
assert slug(Element(ElementType.ACTIVITY, "consent")) == "activity=consent"
|
assert slug(Element(ElementType.ACTIVITY, "consent")) == "activity=consent"
|
||||||
|
|
||||||
|
def test_enumeration_list_items_are_not_element_paragraphs(self):
|
||||||
|
# C1: "( 9) 99439 (code for non-complex chronic care management)."
|
||||||
|
# is a cross-reference enumeration item, not a fragment of 99439's
|
||||||
|
# own descriptor, even though it ends in ")." like a real element
|
||||||
|
# line. The still-accepted lines below must keep passing.
|
||||||
|
assert not is_element_paragraph(
|
||||||
|
"( 9) 99439 (code for non-complex chronic care management)."
|
||||||
|
)
|
||||||
|
assert not is_element_paragraph(
|
||||||
|
"( 10) 99457 and 99458 (codes for remote physiologic monitoring)."
|
||||||
|
)
|
||||||
|
assert is_element_paragraph("Consent;")
|
||||||
|
assert is_element_paragraph(
|
||||||
|
"++ Document in patient's medical record that consent was obtained."
|
||||||
|
)
|
||||||
|
assert is_element_paragraph(
|
||||||
|
"Comprehensive care plan established, implemented, revised, or monitored)."
|
||||||
|
)
|
||||||
|
|||||||
@@ -2,7 +2,9 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from pfs.descriptors import DescriptorRun, Para
|
import sqlite3
|
||||||
|
|
||||||
|
from pfs.descriptors import DescriptorRun, Para, descriptor_runs
|
||||||
from pfs.extract import Extraction, extract_run, extract_text
|
from pfs.extract import Extraction, extract_run, extract_text
|
||||||
|
|
||||||
STEM = Para(
|
STEM = Para(
|
||||||
@@ -91,6 +93,64 @@ class TestClassifier:
|
|||||||
assert calls == ["Something the vocabulary does not know about at all;"]
|
assert calls == ["Something the vocabulary does not know about at all;"]
|
||||||
|
|
||||||
|
|
||||||
|
class TestAnchorIntegrity:
|
||||||
|
def test_no_row_anchored_to_a_foreign_code_paragraph(self):
|
||||||
|
# C1 live-corpus incident: an enumeration/cross-reference line for
|
||||||
|
# a *different* code ("( 9) 99457 and 99458 (codes for remote
|
||||||
|
# physiologic monitoring, each additional 20 minutes).") must
|
||||||
|
# never contribute an ElementRow to a 99439 run. The fix lives in
|
||||||
|
# `descriptor_runs`/`is_element_paragraph` (it stops the run
|
||||||
|
# before absorbing such a line) — this test drives the real
|
||||||
|
# `descriptor_runs` to build the run, then checks `extract_run`'s
|
||||||
|
# output end to end: no row may be anchored to a paragraph whose
|
||||||
|
# text contains a different 5-char code and none of 99439.
|
||||||
|
con = sqlite3.connect(":memory:")
|
||||||
|
con.row_factory = sqlite3.Row
|
||||||
|
con.executescript(
|
||||||
|
"CREATE TABLE items (key TEXT PRIMARY KEY, title TEXT, date_published TEXT);"
|
||||||
|
"CREATE TABLE fr_anchors (item_key TEXT, p_id INTEGER, page INTEGER, ordinal INTEGER, text TEXT);"
|
||||||
|
)
|
||||||
|
con.execute(
|
||||||
|
"INSERT INTO items VALUES ('XFGGRBDH', "
|
||||||
|
"'Medicare and Medicaid Programs; CY 2025 Payment Policies', '2024-07-31')"
|
||||||
|
)
|
||||||
|
rows = [
|
||||||
|
(
|
||||||
|
832,
|
||||||
|
61652,
|
||||||
|
"CPT code 99439 (Chronic care management services, each additional 20 "
|
||||||
|
"minutes of clinical staff time directed by a physician or other "
|
||||||
|
"qualified health care professional, per calendar month, with the "
|
||||||
|
"following required elements:",
|
||||||
|
),
|
||||||
|
(833, 61652, "Consent;"),
|
||||||
|
(
|
||||||
|
834,
|
||||||
|
61652,
|
||||||
|
"( 9) 99457 and 99458 (codes for remote physiologic monitoring, "
|
||||||
|
"each additional 20 minutes).",
|
||||||
|
),
|
||||||
|
]
|
||||||
|
con.executemany(
|
||||||
|
"INSERT INTO fr_anchors VALUES (?,?,?,?,?)",
|
||||||
|
[("XFGGRBDH", p, pg, p, t) for p, pg, t in rows],
|
||||||
|
)
|
||||||
|
|
||||||
|
class _S:
|
||||||
|
def _con(self):
|
||||||
|
return con
|
||||||
|
|
||||||
|
try:
|
||||||
|
run = descriptor_runs(_S(), "99439")[0]
|
||||||
|
# the enumeration line for 99457/99458 never joins the run
|
||||||
|
assert [p.p_id for p in run.elements] == [833]
|
||||||
|
x = extract_run(run)
|
||||||
|
for r in x.rows:
|
||||||
|
assert "99457" not in r.text and "99458" not in r.text
|
||||||
|
finally:
|
||||||
|
con.close()
|
||||||
|
|
||||||
|
|
||||||
class TestText:
|
class TestText:
|
||||||
def test_hcpcs_long_description(self):
|
def test_hcpcs_long_description(self):
|
||||||
x = extract_text(
|
x = extract_text(
|
||||||
|
|||||||
Reference in New Issue
Block a user