9e374e8c29
I2 (Ruling 17): `refresh_from` used to `FAMILIES.clear()` before loading the derived table, so any hand family the derivation table doesn't reach (e.g. only CCM got re-derived) vanished from the live registry. Seed from HAND_FAMILIES first, then let derived rows extend/override per key — a hand key present in the table gets the derived (superset) codes, HAND_FAMILIES' own name/synonyms carry through via `load_families`; a hand key absent from the table survives untouched. I3: the live table has 627 stem-derived keys shared by >= 2 unrelated components (`GENE` x13, `ADM` x12) — `load_families` silently merged them under one key. `derive_families` now appends the group's representative code to a colliding non-hand key (`GENE-81105`) and rejects members that aren't code-shaped (`CODE_RE.fullmatch`) before deriving anything, so a corpus artifact like a bare `\x1a` can't become a family. I7: `FAMILIES`'s docstring claimed the chat/notebooks see the derived registry too; only the `stack pfs` CLI calls `refresh_from` today (the spec's import-time-load ask is blocked by the no-DuckDB-at-import constraint — controller to file a follow-up). Also adds a public `FR_CITE_RE` alias for `pfs.lineage` to reuse (M1, next commit). Claude-Session: https://claude.ai/code/session_01Aum3pEMAM3yQVdFSdVe6Gc
496 lines
19 KiB
Python
496 lines
19 KiB
Python
"""pfs.families — code-family registry + deterministic code detection."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import time
|
|
|
|
import duckdb
|
|
import pytest
|
|
|
|
from pfs.codetables import (
|
|
ElementRow,
|
|
EventRow,
|
|
FamilyRow,
|
|
ensure_tables,
|
|
write_families,
|
|
)
|
|
from pfs.families import (
|
|
FAMILIES,
|
|
HAND_FAMILIES,
|
|
Detection,
|
|
derive_families,
|
|
detect_codes,
|
|
family_of,
|
|
find_codes,
|
|
load_families,
|
|
refresh_from,
|
|
stem_tokens,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def restore_families():
|
|
"""Snapshot ``pfs.families.FAMILIES`` and restore it in teardown, even
|
|
if the test body raises — a test that calls ``refresh_from`` must not
|
|
leave the live registry clobbered for the rest of the session."""
|
|
from pfs import families as mod
|
|
|
|
before = dict(mod.FAMILIES)
|
|
try:
|
|
yield mod
|
|
finally:
|
|
mod.FAMILIES.clear()
|
|
mod.FAMILIES.update(before)
|
|
|
|
|
|
class TestFindCodes:
|
|
def test_hcpcs_and_cpt(self):
|
|
assert find_codes("Codes G0556 and 99490 apply; see g0557.") == (
|
|
"99490",
|
|
"G0556",
|
|
"G0557",
|
|
)
|
|
|
|
def test_lower_case_codes_are_normalised(self):
|
|
assert find_codes("what is g0556?") == ("G0556",)
|
|
|
|
def test_no_false_hits_on_years_and_fr_pages(self):
|
|
# A Federal Register citation is stripped before scanning, so its page
|
|
# number (43842 — a well-formed CPT code) never reads as a code.
|
|
assert find_codes("91 FR 43842, CY2026") == ()
|
|
|
|
def test_fr_citation_alone_yields_nothing(self):
|
|
assert find_codes("What does 91 FR 43842 say?") == ()
|
|
|
|
def test_dedupes_and_sorts(self):
|
|
assert find_codes("G0558 G0556 G0558") == ("G0556", "G0558")
|
|
|
|
def test_empty(self):
|
|
assert find_codes("") == ()
|
|
|
|
|
|
class TestRegistry:
|
|
def test_apcm_family(self):
|
|
f = FAMILIES["APCM"]
|
|
assert f.codes == ("G0556", "G0557", "G0558")
|
|
assert "advanced primary care management" in f.synonyms
|
|
assert family_of("G0557") is f
|
|
assert family_of("00000") is None
|
|
|
|
def test_all_families_present(self):
|
|
assert set(FAMILIES) == {"ACP", "CCM", "PCM", "TCM", "APCM"}
|
|
|
|
|
|
class TestDetectCodes:
|
|
def test_family_name_expands_to_codes(self):
|
|
d = detect_codes("What is APCM and how is it valued?")
|
|
assert d == Detection(
|
|
codes=("G0556", "G0557", "G0558"), families=("APCM",), explicit=()
|
|
)
|
|
|
|
def test_synonym_case_insensitive_word_boundary(self):
|
|
d = detect_codes("Tell me about Advanced Primary Care Management.")
|
|
assert d.families == ("APCM",)
|
|
assert detect_codes("the apcmx code").families == ()
|
|
|
|
def test_single_member_code_expands_family(self):
|
|
d = detect_codes("How much does G0557 pay?")
|
|
assert d.explicit == ("G0557",)
|
|
assert d.families == ("APCM",)
|
|
assert d.codes == ("G0556", "G0557", "G0558")
|
|
|
|
def test_unregistered_code_stays_alone(self):
|
|
d = detect_codes("value of 99213")
|
|
assert d == Detection(codes=("99213",), families=(), explicit=("99213",))
|
|
|
|
def test_multiple_families(self):
|
|
d = detect_codes("compare CCM and TCM")
|
|
assert d.families == ("CCM", "TCM")
|
|
assert d.codes == tuple(sorted(FAMILIES["CCM"].codes + FAMILIES["TCM"].codes))
|
|
|
|
def test_nothing(self):
|
|
assert detect_codes("what did commenters say about telehealth?") == Detection(
|
|
(), (), ()
|
|
)
|
|
|
|
|
|
def _el(code, type_, value, detail=""):
|
|
return ElementRow(code, 2025, type_, value, detail, "", "", 0, 0, "fr")
|
|
|
|
|
|
def _ev(code, year, kind, frm="", to=""):
|
|
return EventRow(code, year, kind, frm, to, "", 0, 0, "fr", True, "")
|
|
|
|
|
|
class TestStemTokens:
|
|
def test_strips_time_and_actor_noise(self):
|
|
assert stem_tokens("Chrnc care mgmt staff 1st 20") == frozenset(
|
|
{"chrnc", "care", "mgmt"}
|
|
)
|
|
assert stem_tokens("Chrnc care mgmt phys ea addl") == frozenset(
|
|
{"chrnc", "care", "mgmt"}
|
|
)
|
|
assert stem_tokens("Adv prim care mgmt lvl 1") == frozenset(
|
|
{"adv", "prim", "care", "mgmt"}
|
|
)
|
|
|
|
def test_none_description_yields_empty_set(self):
|
|
# A real pfs.rvu row can carry a NULL description (corpus parsing
|
|
# artifact, e.g. a garbage hcpcs value) — stem_tokens must not
|
|
# crash on it (#687 live-run regression: AttributeError on
|
|
# `None.lower()` in derive_families).
|
|
assert stem_tokens(None) == frozenset()
|
|
|
|
|
|
class TestDerive:
|
|
def test_ccm_reproduced_with_predecessor_and_addon_roles(self):
|
|
# #687 ruling 16: `not-with` is an exclusion, not a membership
|
|
# signal, and was dropped from _LINK_RELATIONS. The `not-with`
|
|
# element below is now inert as a merge edge — 99487 joins CCM
|
|
# through the stem-Jaccard + shared "comprehensive-care-plan"
|
|
# activity rule instead (its stem overlaps 99490's >= 0.5).
|
|
elements = {
|
|
"99490": [
|
|
_el("99490", "actor", "clinical-staff-directed"),
|
|
_el("99490", "activity", "comprehensive-care-plan"),
|
|
],
|
|
"99439": [
|
|
_el("99439", "relation", "addon-of", "99490"),
|
|
_el("99439", "relation", "not-with", "99487"),
|
|
],
|
|
"99487": [_el("99487", "activity", "comprehensive-care-plan")],
|
|
"99489": [_el("99489", "relation", "addon-of", "99487")],
|
|
"99491": [
|
|
_el("99491", "actor", "physician-or-qhp-personally"),
|
|
_el("99491", "activity", "comprehensive-care-plan"),
|
|
],
|
|
"99437": [_el("99437", "relation", "addon-of", "99491")],
|
|
"G2058": [_el("G2058", "relation", "addon-of", "99490")],
|
|
"99497": [_el("99497", "activity", "advance-directive-discussion")],
|
|
"99498": [_el("99498", "relation", "addon-of", "99497")],
|
|
}
|
|
events = {
|
|
"G2058": [
|
|
_ev("G2058", 2020, "appeared"),
|
|
_ev("G2058", 2021, "disappeared"),
|
|
_ev("G2058", 2021, "replaced_by", to="99439"),
|
|
],
|
|
"99439": [
|
|
_ev("99439", 2021, "appeared"),
|
|
_ev("99439", 2021, "replaces", frm="G2058"),
|
|
],
|
|
}
|
|
descriptions = {
|
|
"99490": "Chrnc care mgmt staff 1st 20",
|
|
"99439": "Chrnc care mgmt staf ea addl",
|
|
"99487": "Cplx chrnc care 1st 60 min",
|
|
"99489": "Cplx chrnc care ea addl 30",
|
|
"99491": "Chrnc care mgmt phys 1st 30",
|
|
"99437": "Chrnc care mgmt phys ea addl",
|
|
"G2058": "Ccm add 20min",
|
|
"99497": "Advncd care plan 30 min",
|
|
"99498": "Advncd care plan addl 30 min",
|
|
}
|
|
rows = derive_families(elements, events, descriptions)
|
|
ccm = {r.code: r for r in rows if r.key == "CCM"}
|
|
assert set(ccm) == {
|
|
"99490",
|
|
"99439",
|
|
"99487",
|
|
"99489",
|
|
"99491",
|
|
"99437",
|
|
"G2058",
|
|
}
|
|
assert ccm["99439"].role == "add-on" and ccm["G2058"].role == "predecessor"
|
|
assert ccm["G2058"].since == 2020 and ccm["G2058"].until == 2021
|
|
assert ccm["99490"].role == "base"
|
|
acp = {r.code for r in rows if r.key == "ACP"}
|
|
assert acp == {"99497", "99498"}
|
|
assert all(
|
|
r.name == HAND_FAMILIES[r.key].name for r in rows if r.key in HAND_FAMILIES
|
|
)
|
|
|
|
def test_multi_code_replaced_by_does_not_merge(self):
|
|
# #687 ruling 16: a `replaced_by` event naming more than one
|
|
# to_codes is ambiguous (a blanket "these codes are replaced by
|
|
# ..." sentence spanning several families in the real corpus) and
|
|
# must not become a merge edge.
|
|
elements = {"11111": [], "22222": []}
|
|
events = {"11111": [_ev("11111", 2021, "replaced_by", to="22222 33333")]}
|
|
descriptions = {"11111": "Alpha widget", "22222": "Zeta gadget"}
|
|
rows = derive_families(elements, events, descriptions)
|
|
keys = {r.code: r.key for r in rows}
|
|
assert keys["11111"] != keys["22222"]
|
|
|
|
def test_not_with_relation_does_not_merge(self):
|
|
# #687 ruling 16: `not-with` is an explicit exclusion (cannot be
|
|
# billed together), the opposite of family membership.
|
|
elements = {
|
|
"99490": [_el("99490", "relation", "not-with", "G0556")],
|
|
"G0556": [],
|
|
}
|
|
descriptions = {
|
|
"99490": "Chrnc care mgmt staff 1st 20",
|
|
"G0556": "Adv prim care mgmt lvl 1",
|
|
}
|
|
rows = derive_families(elements, {}, descriptions)
|
|
keys = {r.code: r.key for r in rows}
|
|
assert keys["99490"] != keys["G0556"]
|
|
|
|
def test_bridging_addon_chain_splits_by_hand_family_and_warns(self, caplog):
|
|
# #687 ruling 16 guard: a component that reaches into two or more
|
|
# hand families is split by a BFS seeded from each hand family's
|
|
# own codes, run in HAND_FAMILIES order (ACP, CCM, PCM, TCM,
|
|
# APCM). Here "77777" is addon-of both a CCM seed (99490) and an
|
|
# ACP seed (99497), joining all three into one component; ACP
|
|
# comes first in HAND_FAMILIES, so its BFS reaches — and claims —
|
|
# the bridging code before CCM's BFS runs.
|
|
elements = {
|
|
"99490": [],
|
|
"99497": [],
|
|
"77777": [
|
|
_el("77777", "relation", "addon-of", "99490"),
|
|
_el("77777", "relation", "addon-of", "99497"),
|
|
],
|
|
}
|
|
with caplog.at_level(logging.WARNING, logger="pfs.families"):
|
|
rows = derive_families(elements, {}, {})
|
|
by_code = {r.code: r for r in rows}
|
|
assert by_code["99490"].key == "CCM"
|
|
assert by_code["99497"].key == "ACP"
|
|
assert by_code["77777"].key == "ACP"
|
|
assert any("spans hand families" in rec.message for rec in caplog.records), (
|
|
caplog.text
|
|
)
|
|
|
|
def test_unknown_family_gets_stem_key(self):
|
|
elements = {
|
|
"99453": [_el("99453", "activity", "device-data-review")],
|
|
"99454": [_el("99454", "activity", "device-data-review")],
|
|
}
|
|
rows = derive_families(
|
|
elements,
|
|
{},
|
|
{
|
|
"99453": "Rem mntr physiol param setup",
|
|
"99454": "Rem mntr physiol param dev supl",
|
|
},
|
|
)
|
|
keys = {r.key for r in rows}
|
|
assert len(keys) == 1 and next(iter(keys)).startswith("REM-MNTR")
|
|
|
|
def test_no_activity_elements_never_merges_on_stem_alone(self):
|
|
# Both codes have identical stem tokens (jaccard == 1.0, well past
|
|
# the 0.5 threshold) but neither has an activity element — an empty
|
|
# activity set must not compare equal-and-qualifying, or every
|
|
# activity-less code with a common description would merge with
|
|
# every other one. Word order differs so a stem-based merge would
|
|
# be visible as a shared key even though the codes never touch.
|
|
elements = {"11111": [], "22222": []}
|
|
descriptions = {"11111": "Foo bar widget", "22222": "Bar widget foo"}
|
|
rows = derive_families(elements, {}, descriptions)
|
|
keys = {r.code: r.key for r in rows}
|
|
assert keys["11111"] != keys["22222"]
|
|
|
|
def test_disjoint_stem_never_merges(self):
|
|
# 10000/10001 share enough stem tokens (and a matching activity) to
|
|
# merge; 20000's stem shares no token with either, so it must never
|
|
# even be compared, let alone merged.
|
|
elements = {
|
|
"10000": [_el("10000", "activity", "act-a")],
|
|
"10001": [_el("10001", "activity", "act-a")],
|
|
"20000": [_el("20000", "activity", "act-a")],
|
|
}
|
|
descriptions = {
|
|
"10000": "Alpha beta gamma",
|
|
"10001": "Alpha beta delta",
|
|
"20000": "Zeta eta theta",
|
|
}
|
|
rows = derive_families(elements, {}, descriptions)
|
|
keys = {r.code: r.key for r in rows}
|
|
assert keys["10000"] == keys["10001"]
|
|
assert keys["20000"] != keys["10000"]
|
|
|
|
def test_non_code_member_is_dropped(self):
|
|
# I3: a corpus parsing artifact like '\x1a' must never become a
|
|
# family of its own — reject non-code-shaped members before
|
|
# deriving anything.
|
|
elements = {
|
|
"99453": [_el("99453", "activity", "device-data-review")],
|
|
"\x1a": [_el("\x1a", "activity", "device-data-review")],
|
|
}
|
|
descriptions = {
|
|
"99453": "Rem mntr physiol param setup",
|
|
"\x1a": "corpus artifact",
|
|
}
|
|
rows = derive_families(elements, {}, descriptions)
|
|
assert {r.code for r in rows} == {"99453"}
|
|
|
|
def test_duplicate_derived_keys_get_representative_suffix(self):
|
|
# I3: 627 stem keys in the live table are shared by >= 2 unrelated
|
|
# components (e.g. "GENE" x13); a collision must not silently
|
|
# merge them under `load_families`' one-row-per-key model.
|
|
elements = {
|
|
"10001": [_el("10001", "activity", "act-a")],
|
|
"10002": [_el("10002", "activity", "act-a")],
|
|
"20001": [_el("20001", "activity", "act-b")],
|
|
"20002": [_el("20002", "activity", "act-b")],
|
|
}
|
|
descriptions = {
|
|
"10001": "Rare widget",
|
|
"10002": "Rare widget",
|
|
"20001": "Rare widget",
|
|
"20002": "Rare widget",
|
|
}
|
|
rows = derive_families(elements, {}, descriptions)
|
|
keys = {r.code: r.key for r in rows}
|
|
assert keys["10001"] == keys["10002"]
|
|
assert keys["20001"] == keys["20002"]
|
|
assert keys["10001"] != keys["20001"]
|
|
both = {keys["10001"], keys["20001"]}
|
|
assert "RARE-WIDGET" in both
|
|
other = (both - {"RARE-WIDGET"}).pop()
|
|
assert other.startswith("RARE-WIDGET-")
|
|
|
|
def test_none_description_does_not_raise_and_stands_alone(self):
|
|
# #687 live-run regression: a code whose `descriptions` mapping
|
|
# holds an explicit None (a real NULL pfs.rvu description, not a
|
|
# missing key — dict.get(c, "") only substitutes its default when
|
|
# c is absent) must not crash derive_families, and with nothing
|
|
# else linking it to another code it lands in its own family.
|
|
elements = {"99999": [_el("99999", "activity", "act-a")]}
|
|
descriptions = {"99999": None}
|
|
rows = derive_families(elements, {}, descriptions)
|
|
assert {r.code for r in rows} == {"99999"}
|
|
assert rows[0].key == "99999"
|
|
assert rows[0].name == "99999"
|
|
|
|
def test_many_distinct_stems_is_fast(self):
|
|
# ~2,000 codes whose descriptions share no stem token with any
|
|
# other code's. The token-bucketed merge must not degrade to the
|
|
# O(n^2) all-pairs scan this guards against; kept generous (< 2s)
|
|
# so CI timing noise doesn't make it flaky.
|
|
def _word(i: int) -> str:
|
|
letters = []
|
|
n = i + 1
|
|
while n:
|
|
n, r = divmod(n - 1, 26)
|
|
letters.append(chr(97 + r))
|
|
return "".join(reversed(letters))
|
|
|
|
n = 2000
|
|
elements = {}
|
|
descriptions = {}
|
|
for i in range(n):
|
|
code = f"{10000 + i}"
|
|
elements[code] = [_el(code, "activity", "act")]
|
|
descriptions[code] = f"stem{_word(i)} term{_word(i + n)}"
|
|
|
|
start = time.perf_counter()
|
|
rows = derive_families(elements, {}, descriptions)
|
|
elapsed = time.perf_counter() - start
|
|
|
|
assert len(rows) == n
|
|
assert elapsed < 2.0
|
|
|
|
|
|
class TestLoadAndRefresh:
|
|
def test_load_and_refresh_in_place(self, restore_families):
|
|
mod = restore_families
|
|
con = duckdb.connect(":memory:")
|
|
try:
|
|
ensure_tables(con)
|
|
assert load_families(con) == {}
|
|
before = dict(mod.FAMILIES)
|
|
assert refresh_from(con) == 0 and mod.FAMILIES == before
|
|
write_families(
|
|
con,
|
|
[
|
|
FamilyRow(
|
|
"CCM",
|
|
"Chronic Care Management",
|
|
"99490",
|
|
"base",
|
|
None,
|
|
None,
|
|
"",
|
|
0,
|
|
),
|
|
FamilyRow(
|
|
"CCM",
|
|
"Chronic Care Management",
|
|
"G2058",
|
|
"predecessor",
|
|
2020,
|
|
2021,
|
|
"",
|
|
0,
|
|
),
|
|
],
|
|
)
|
|
n = refresh_from(con)
|
|
# Ruling 17: a hand key the table reaches (CCM) is replaced by
|
|
# the derived (superset) codes; every other hand key survives
|
|
# untouched since `refresh_from` seeds from HAND_FAMILIES
|
|
# first instead of clearing the registry outright.
|
|
assert n == len(HAND_FAMILIES)
|
|
assert set(mod.FAMILIES) >= set(HAND_FAMILIES)
|
|
assert set(mod.FAMILIES["CCM"].codes) == {"99490", "G2058"}
|
|
assert mod.FAMILIES["CCM"].name == HAND_FAMILIES["CCM"].name
|
|
finally:
|
|
con.close()
|
|
|
|
def test_hand_key_absent_from_table_survives(self, restore_families):
|
|
# Ruling 17: only CCM appears in the derived table; the other four
|
|
# hand families (ACP, PCM, TCM, APCM) must come through unchanged.
|
|
mod = restore_families
|
|
con = duckdb.connect(":memory:")
|
|
try:
|
|
ensure_tables(con)
|
|
write_families(
|
|
con,
|
|
[
|
|
FamilyRow(
|
|
"CCM",
|
|
"Chronic Care Management",
|
|
"99490",
|
|
"base",
|
|
None,
|
|
None,
|
|
"",
|
|
0,
|
|
)
|
|
],
|
|
)
|
|
refresh_from(con)
|
|
for key in ("ACP", "PCM", "TCM", "APCM"):
|
|
assert mod.FAMILIES[key] == HAND_FAMILIES[key]
|
|
finally:
|
|
con.close()
|
|
|
|
def test_schema_present_table_absent_returns_empty(self):
|
|
# pfs schema created (e.g. by an earlier ensure_tables call for a
|
|
# sibling table) but pfs.code_family itself never materialized —
|
|
# an old replica shape, not a real error.
|
|
con = duckdb.connect(":memory:")
|
|
try:
|
|
con.execute("CREATE SCHEMA IF NOT EXISTS pfs;")
|
|
assert load_families(con) == {}
|
|
finally:
|
|
con.close()
|
|
|
|
def test_wrong_shape_table_raises(self):
|
|
# A pfs.code_family table exists but not in the expected shape —
|
|
# this is a real bug, not an absent table, and must not be
|
|
# swallowed.
|
|
con = duckdb.connect(":memory:")
|
|
try:
|
|
con.execute("CREATE SCHEMA IF NOT EXISTS pfs;")
|
|
con.execute("CREATE TABLE pfs.code_family (only_col VARCHAR);")
|
|
with pytest.raises(Exception):
|
|
load_families(con)
|
|
finally:
|
|
con.close()
|