feat(pfs): lineage events from FR paragraphs (created/adopted/replaced/deleted/crosswalk/bundled/telehealth) + RVU cross-check (refs #686)

src/pfs/lineage.py

@@ -13,9 +13,13 @@ event for the same code lies within ±1 rule year; the rest are
 
 from __future__ import annotations
 
+import dataclasses
+import re
 from typing import Any
 
 from pfs.codetables import EventRow
+from pfs.descriptors import rule_year_of
+from pfs.families import find_codes
 
 RVU_KINDS = (
     "appeared",
@@ -71,3 +75,163 @@ def rvu_events(
             if abs(delta) > revalue_threshold:
                 out.append(_ev(code, y, "revalued", f"{delta:+.1%}"))
     return sorted(out, key=lambda e: (e.year, RVU_KINDS.index(e.kind)))
+
+
+FR_KINDS = (
+    "created",
+    "adopted_cpt",
+    "replaces",
+    "replaced_by",
+    "deleted",
+    "crosswalk",
+    "bundled",
+    "telehealth_list",
+)
+_CODE = r"(?:[A-Z]\d{4}|\d{5})"
+
+
+def _patterns(code: str) -> list[tuple[str, re.Pattern[str], str]]:
+    c = re.escape(code)
+    return [
+        (
+            "created",
+            re.compile(
+                rf"(?:\b(?:creat|establish)\w*\b.{{0,80}}\b(?:HCPCS|CPT|G-?)\s*codes?\b[^.]{{0,80}}{c})"
+                rf"|(?:new (?:HCPCS|CPT) code\s+{c})",
+                re.I,
+            ),
+            "",
+        ),
+        (
+            "adopted_cpt",
+            re.compile(rf"adopt(?:ed|ing)?\s+CPT code\s+{c}", re.I),
+            "",
+        ),
+        (
+            "replaces",
+            re.compile(
+                rf"{c}[^.]{{0,120}}\b(?:replace|replacing|replacement for|instead of|with a descriptor identical to)\b",
+                re.I,
+            ),
+            "from",
+        ),
+        (
+            "replaced_by",
+            re.compile(
+                rf"(?:\b(?:replace\w*|replacement)\b[^.]{{0,120}}{c})"
+                rf"|(?:{c}[^.]{{0,80}}\b(?:as|is being|will be|which we are) (?:finaliz|replac)\w+ (?:in this rule )?as new (?:CPT|HCPCS) code\s+{_CODE})",
+                re.I,
+            ),
+            "to",
+        ),
+        (
+            "deleted",
+            re.compile(
+                rf"(?:\bdelet(?:ed|ion|es)\b[^.]{{0,120}}{c})|(?:{c}[^.]{{0,60}}\bdeleted\b)",
+                re.I,
+            ),
+            "",
+        ),
+        (
+            "crosswalk",
+            re.compile(
+                rf"(?:\bcrosswalk\w*\b[^.]{{0,160}}{c})|(?:{c}[^.]{{0,160}}\bcrosswalk)",
+                re.I,
+            ),
+            "",
+        ),
+        (
+            "bundled",
+            re.compile(rf"{c}[^.]{{0,120}}\b(?:bundled|bundled status)\b", re.I),
+            "",
+        ),
+        (
+            "telehealth_list",
+            re.compile(
+                rf"(?:Medicare telehealth services list[^.]{{0,200}}{c})"
+                rf"|(?:{c}[^.]{{0,200}}telehealth services list)",
+                re.I,
+            ),
+            "",
+        ),
+    ]
+
+
+_RE_RANGE = re.compile(r"\b(\d{5})-(\d{5})\b")
+
+
+def _expand_ranges(text: str) -> str:
+    """ "99441-99443" -> "99441 99442 99443" so a range mentions each code."""
+
+    def rep(m: re.Match[str]) -> str:
+        a, b = int(m.group(1)), int(m.group(2))
+        return (
+            " ".join(str(n) for n in range(a, b + 1)) if 0 < b - a <= 20 else m.group(0)
+        )
+
+    return _RE_RANGE.sub(rep, text)
+
+
+def fr_events(store: Any, code: str) -> list[EventRow]:
+    code = code.upper()
+    con = store._con()
+    rows = con.execute(
+        "SELECT a.item_key, a.p_id, a.page, a.text, i.title, i.date_published "
+        "FROM fr_anchors a JOIN items i ON i.key = a.item_key WHERE a.text LIKE ? OR a.text LIKE ?",
+        (f"%{code}%", f"%{code[:5]}-%"),
+    ).fetchall()
+    seen: set[tuple[int, str, str, int]] = set()
+    out: list[EventRow] = []
+    for r in rows:
+        text = _expand_ranges(r["text"])
+        if code not in text.upper():
+            continue
+        year = rule_year_of(r["title"], r["date_published"])
+        others = tuple(c for c in find_codes(text) if c != code)
+        for kind, pat, direction in _patterns(code):
+            m = pat.search(text)
+            if not m:
+                continue
+            frm = " ".join(others) if direction == "from" else ""
+            to = ""
+            if direction == "to":
+                tail = find_codes(text[m.start() : m.end()])
+                to = " ".join(c for c in tail if c != code) or " ".join(others)
+            # A directional kind ("replaces"/"replaced_by") with no other
+            # code found nearby is not a real relationship — e.g. "...
+            # instead of the G code" names no actual code, so the sentence
+            # is prose, not a replacement event.
+            if direction == "from" and not frm:
+                continue
+            if direction == "to" and not to:
+                continue
+            key = (year, kind, r["item_key"], r["p_id"])
+            if key in seen:
+                continue
+            seen.add(key)
+            out.append(
+                EventRow(
+                    code,
+                    year,
+                    kind,
+                    frm,
+                    to,
+                    r["item_key"],
+                    r["p_id"],
+                    r["page"],
+                    "fr",
+                    True,
+                    "",
+                )
+            )
+    return sorted(out, key=lambda e: (e.year, FR_KINDS.index(e.kind), e.p_id))
+
+
+def lineage(con: Any, store: Any, code: str) -> list[EventRow]:
+    fr = fr_events(store, code)
+    fr_years = {e.year for e in fr}
+    rvu = [
+        dataclasses.replace(ev, anchored=any(abs(ev.year - y) <= 1 for y in fr_years))
+        for ev in rvu_events(con, code)
+    ]
+    return sorted([*rvu, *fr], key=lambda e: (e.year, e.source != "fr", e.kind))

tests/pfs/test_lineage.py

@@ -2,10 +2,12 @@
 
 from __future__ import annotations
 
+import sqlite3
+
 import duckdb
 import pytest
 
-from pfs.lineage import rvu_events, rvu_year_span
+from pfs.lineage import fr_events, lineage, rvu_events, rvu_year_span
 
 RVU_COLS = "hcpcs VARCHAR, mod VARCHAR, description VARCHAR, status_code VARCHAR, non_fac_total DOUBLE, year INTEGER"
 
@@ -81,3 +83,119 @@ class TestRvuEvents:
 
     def test_unknown_code(self, con):
         assert rvu_events(con, "99999") == []
+
+
+class _Store:
+    def __init__(self):
+        self.con = sqlite3.connect(":memory:")
+        self.con.row_factory = sqlite3.Row
+        self.con.executescript(
+            "CREATE TABLE items (key TEXT PRIMARY KEY, title TEXT, date_published TEXT);"
+            "CREATE TABLE fr_anchors (item_key TEXT, p_id INTEGER, page INTEGER, ordinal INTEGER, text TEXT);"
+        )
+
+    def _con(self):
+        return self.con
+
+    def close(self):
+        self.con.close()
+
+
+@pytest.fixture
+def store():
+    s = _Store()
+    s.con.executemany(
+        "INSERT INTO items VALUES (?,?,?)",
+        [
+            ("DE2VH9PD", "Medicare Program; CY 2015 PFS Final Rule", "2014-11-13"),
+            (
+                "YBM4IZUS",
+                "Medicare Program; CY 2021 Payment Policies Under the PFS",
+                "2020-12-28",
+            ),
+            (
+                "XFGGRBDH",
+                "Medicare and Medicaid Programs; CY 2025 Payment Policies",
+                "2024-07-31",
+            ),
+        ],
+    )
+    paras = [
+        (
+            "DE2VH9PD",
+            1251,
+            67716,
+            "Accordingly, we will adopt CPT code 99490 for Medicare CCM services, effective January 1, 2015 instead of the G code.",
+        ),
+        (
+            "YBM4IZUS",
+            686,
+            84547,
+            "We also are finalizing our proposal to allow HCPCS code G2058 (which we are finalizing in this rule as new CPT code 99439, see the codes in section II.H.) to be billed concurrently with TCM.",
+        ),
+        (
+            "YBM4IZUS",
+            1578,
+            84639,
+            "At the January 2020 RUC meeting, specialty societies requested a temporary crosswalk through CY 2021 between the value established by CMS for HCPCS code G2058 and the value of new CPT code 99439 (with a descriptor identical to G2058).",
+        ),
+        (
+            "XFGGRBDH",
+            489,
+            61652,
+            "The CPT Editorial Panel also deleted three codes (99441-99443) for reporting telephone E/M services. We note that CPT codes 99441, 99442, and 99443, each are assigned provisional status on the Medicare telehealth services list, and would return to bundled status when the telehealth flexibilities expire.",
+        ),
+    ]
+    s.con.executemany(
+        "INSERT INTO fr_anchors VALUES (?,?,?,?,?)",
+        [(k, p, pg, p, t) for k, p, pg, t in paras],
+    )
+    yield s
+    s.close()
+
+
+class TestFrEvents:
+    def test_adopted_cpt(self, store):
+        ev = fr_events(store, "99490")
+        assert [(e.year, e.kind, e.item_key, e.p_id) for e in ev] == [
+            (2015, "adopted_cpt", "DE2VH9PD", 1251)
+        ]
+        assert ev[0].anchored is True and ev[0].source == "fr" and ev[0].page == 67716
+
+    def test_replaced_by_and_crosswalk(self, store):
+        ev = fr_events(store, "G2058")
+        kinds = {(e.year, e.kind, e.to_codes) for e in ev}
+        assert (2021, "replaced_by", "99439") in kinds
+        assert (2021, "crosswalk", "") in kinds or any(
+            e.kind == "crosswalk" for e in ev
+        )
+
+    def test_replaces_from_the_successor_side(self, store):
+        ev = fr_events(store, "99439")
+        rep = [e for e in ev if e.kind == "replaces"]
+        assert rep and rep[0].from_codes == "G2058"
+
+    def test_deleted_bundled_and_telehealth(self, store):
+        kinds = {e.kind for e in fr_events(store, "99441")}
+        assert {"deleted", "bundled", "telehealth_list"} <= kinds
+
+
+class TestCrossCheck:
+    def test_rvu_events_anchored_by_nearby_fr_event(self, con, store):
+        ev = lineage(con, store, "G2058")
+        by_kind = {e.kind: e for e in ev}
+        assert (
+            by_kind["appeared"].year == 2020 and by_kind["appeared"].anchored is True
+        )  # 2021 is within ±1
+        assert (
+            by_kind["disappeared"].year == 2021
+            and by_kind["disappeared"].anchored is True
+        )  # 2021 replaced_by
+        assert "replaced_by" in by_kind
+        assert all(
+            not e.anchored for e in lineage(con, store, "99487") if e.source == "rvu"
+        )
+
+    def test_sorted_by_year_then_source(self, con, store):
+        ev = lineage(con, store, "99439")
+        assert [e.year for e in ev] == sorted(e.year for e in ev)