feat(pfs): derived code families (relation/lineage/stem components) with the hand registry as fixture; live refresh_from (refs #687)

Claude-Session: https://claude.ai/code/session_01Aum3pEMAM3yQVdFSdVe6Gc

src/pfs/families.py

@@ -10,6 +10,7 @@ from __future__ import annotations
 
 import re
 from dataclasses import dataclass
+from typing import Any, Mapping, Sequence
 
 # HCPCS level II (letter + 4 digits) or CPT (5 digits). Validation against
 # the fee schedule happens at lookup time (pfs.valuation), not here.
@@ -39,7 +40,7 @@ class Family:
     synonyms: tuple[str, ...]  # lower-case; matched on word boundaries
 
 
-FAMILIES: dict[str, Family] = {
+HAND_FAMILIES: dict[str, Family] = {
     "ACP": Family(
         "ACP",
         "Advance Care Planning",
@@ -72,11 +73,18 @@ FAMILIES: dict[str, Family] = {
     ),
 }
 
-_CODE_TO_FAMILY: dict[str, Family] = {c: f for f in FAMILIES.values() for c in f.codes}
+#: Live registry. Starts as the hand list; ``refresh_from(con)`` swaps in
+#: the derived families (``pfs.code_family``) in place so every importer
+#: (chat, notebooks) sees the same dict object.
+FAMILIES: dict[str, Family] = dict(HAND_FAMILIES)
 
 
 def family_of(code: str) -> Family | None:
-    return _CODE_TO_FAMILY.get(code.upper())
+    code = code.upper()
+    for fam in FAMILIES.values():
+        if code in fam.codes:
+            return fam
+    return None
 
 
 @dataclass(frozen=True)
@@ -107,3 +115,195 @@ def detect_codes(text: str) -> Detection:
         families=tuple(sorted(families)),
         explicit=explicit,
     )
+
+
+_NOISE = frozenset(
+    {
+        "1st",
+        "ea",
+        "addl",
+        "add",
+        "min",
+        "mo",
+        "staff",
+        "staf",
+        "phys",
+        "qhp",
+        "lvl",
+        "svc",
+        "srvc",
+        "svcs",
+        "w",
+        "w/o",
+        "pt",
+        "per",
+        "the",
+        "a",
+        "of",
+        "&",
+        "each",
+        "first",
+        "initial",
+        "additional",
+        "level",
+    }
+)
+_RE_TOKEN = re.compile(r"(?<![a-z0-9])[a-z]+(?:/[a-z]+)?(?![a-z0-9])")
+
+
+def stem_tokens(description: str) -> frozenset[str]:
+    """Short-descriptor tokens minus time/actor/number noise — the part
+    of a description that names the service."""
+    return frozenset(
+        t
+        for t in _RE_TOKEN.findall(description.lower())
+        if t not in _NOISE and len(t) > 1
+    )
+
+
+_LINK_RELATIONS = {
+    "addon-of",
+    "not-with",
+    "defined-by-reference-to",
+    "replaces",
+    "replaced-by",
+    "crosswalk-valued-to",
+}
+_LINK_EVENTS = {"replaces", "replaced_by", "crosswalk"}
+
+
+def _jaccard(a: frozenset[str], b: frozenset[str]) -> float:
+    return len(a & b) / len(a | b) if a or b else 0.0
+
+
+def derive_families(
+    elements: Mapping[str, Sequence[Any]],
+    events: Mapping[str, Sequence[Any]],
+    descriptions: Mapping[str, str],
+) -> list[Any]:
+    """Connected components over codes linked by relation elements,
+    lineage events, or a shared service stem + activity set.
+
+    Role precedence when a code qualifies for more than one:
+    predecessor (has a ``replaced_by`` event) > add-on (has an
+    ``addon-of`` relation element) > successor (has a ``replaces``
+    event) > base.
+    """
+    from pfs.codetables import FamilyRow
+
+    codes = sorted(set(elements) | set(events) | set(descriptions))
+    parent = {c: c for c in codes}
+
+    def find(c: str) -> str:
+        while parent[c] != c:
+            parent[c] = parent[parent[c]]
+            c = parent[c]
+        return c
+
+    def union(a: str, b: str) -> None:
+        if a in parent and b in parent:
+            parent[find(a)] = find(b)
+
+    for code, els in elements.items():
+        for e in els:
+            if e.type == "relation" and e.value in _LINK_RELATIONS and e.detail:
+                union(code, e.detail.upper())
+    for code, evs in events.items():
+        for ev in evs:
+            if ev.kind in _LINK_EVENTS:
+                for other in (ev.from_codes + " " + ev.to_codes).split():
+                    union(code, other.upper())
+    stems = {c: stem_tokens(descriptions.get(c, "")) for c in codes}
+    acts = {
+        c: frozenset(e.value for e in elements.get(c, ()) if e.type == "activity")
+        for c in codes
+    }
+    for i, a in enumerate(codes):
+        for b in codes[i + 1 :]:
+            if (
+                stems[a]
+                and stems[b]
+                and _jaccard(stems[a], stems[b]) >= 0.5
+                and acts[a] == acts[b]
+            ):
+                union(a, b)
+
+    groups: dict[str, list[str]] = {}
+    for c in codes:
+        groups.setdefault(find(c), []).append(c)
+
+    def tenure(c: str) -> int:
+        since = next((e.year for e in events.get(c, ()) if e.kind == "appeared"), 0)
+        until = next(
+            (e.year for e in events.get(c, ()) if e.kind == "disappeared"), 9999
+        )
+        return until - since
+
+    rows: list[FamilyRow] = []
+    for members in groups.values():
+        hand = next(
+            (k for k, f in HAND_FAMILIES.items() if set(f.codes) & set(members)), None
+        )
+        if hand:
+            key, name = hand, HAND_FAMILIES[hand].name
+        else:
+            rep = sorted(members, key=lambda c: (-tenure(c), c))[0]
+            key = (
+                "-".join(
+                    t
+                    for t in _RE_TOKEN.findall(descriptions.get(rep, "").lower())
+                    if t in stems[rep]
+                ).upper()
+                or rep
+            )
+            name = descriptions.get(rep, rep)
+        for c in sorted(members):
+            els = elements.get(c, ())
+            evs = events.get(c, ())
+            kinds = {ev.kind for ev in evs}
+            if "replaced_by" in kinds:
+                role = "predecessor"
+            elif any(e.type == "relation" and e.value == "addon-of" for e in els):
+                role = "add-on"
+            elif "replaces" in kinds:
+                role = "successor"
+            else:
+                role = "base"
+            since = next((ev.year for ev in evs if ev.kind == "appeared"), None)
+            until = next((ev.year for ev in evs if ev.kind == "disappeared"), None)
+            anchor = next(
+                ((ev.item_key, ev.p_id) for ev in evs if ev.item_key), ("", 0)
+            )
+            rows.append(
+                FamilyRow(key, name, c, role, since, until, anchor[0], anchor[1])
+            )
+    return sorted(rows, key=lambda r: (r.key, r.code))
+
+
+def load_families(con: Any) -> dict[str, Family]:
+    from pfs.codetables import read_families
+
+    try:
+        rows = read_families(con)
+    except Exception:  # table missing on an old replica
+        return {}
+    out: dict[str, Family] = {}
+    for r in rows:
+        fam = out.get(r.key)
+        codes = (*(fam.codes if fam else ()), r.code)
+        syn = (
+            HAND_FAMILIES[r.key].synonyms
+            if r.key in HAND_FAMILIES
+            else (r.key.lower(), r.name.lower())
+        )
+        out[r.key] = Family(r.key, r.name, tuple(sorted(set(codes))), syn)
+    return out
+
+
+def refresh_from(con: Any) -> int:
+    derived = load_families(con)
+    if not derived:
+        return 0
+    FAMILIES.clear()
+    FAMILIES.update(derived)
+    return len(FAMILIES)

tests/pfs/test_families.py

@@ -2,7 +2,27 @@
 
 from __future__ import annotations
 
-from pfs.families import FAMILIES, Detection, detect_codes, family_of, find_codes
+import duckdb
+
+from pfs.codetables import (
+    ElementRow,
+    EventRow,
+    FamilyRow,
+    ensure_tables,
+    write_families,
+)
+from pfs.families import (
+    FAMILIES,
+    HAND_FAMILIES,
+    Detection,
+    derive_families,
+    detect_codes,
+    family_of,
+    find_codes,
+    load_families,
+    refresh_from,
+    stem_tokens,
+)
 
 
 class TestFindCodes:
@@ -74,3 +94,153 @@ class TestDetectCodes:
         assert detect_codes("what did commenters say about telehealth?") == Detection(
             (), (), ()
         )
+
+
+def _el(code, type_, value, detail=""):
+    return ElementRow(code, 2025, type_, value, detail, "", "", 0, 0, "fr")
+
+
+def _ev(code, year, kind, frm="", to=""):
+    return EventRow(code, year, kind, frm, to, "", 0, 0, "fr", True, "")
+
+
+class TestStemTokens:
+    def test_strips_time_and_actor_noise(self):
+        assert stem_tokens("Chrnc care mgmt staff 1st 20") == frozenset(
+            {"chrnc", "care", "mgmt"}
+        )
+        assert stem_tokens("Chrnc care mgmt phys ea addl") == frozenset(
+            {"chrnc", "care", "mgmt"}
+        )
+        assert stem_tokens("Adv prim care mgmt lvl 1") == frozenset(
+            {"adv", "prim", "care", "mgmt"}
+        )
+
+
+class TestDerive:
+    def test_ccm_reproduced_with_predecessor_and_addon_roles(self):
+        elements = {
+            "99490": [
+                _el("99490", "actor", "clinical-staff-directed"),
+                _el("99490", "activity", "comprehensive-care-plan"),
+            ],
+            "99439": [
+                _el("99439", "relation", "addon-of", "99490"),
+                _el("99439", "relation", "not-with", "99487"),
+            ],
+            "99487": [_el("99487", "activity", "comprehensive-care-plan")],
+            "99489": [_el("99489", "relation", "addon-of", "99487")],
+            "99491": [
+                _el("99491", "actor", "physician-or-qhp-personally"),
+                _el("99491", "activity", "comprehensive-care-plan"),
+            ],
+            "99437": [_el("99437", "relation", "addon-of", "99491")],
+            "G2058": [_el("G2058", "relation", "addon-of", "99490")],
+            "99497": [_el("99497", "activity", "advance-directive-discussion")],
+            "99498": [_el("99498", "relation", "addon-of", "99497")],
+        }
+        events = {
+            "G2058": [
+                _ev("G2058", 2020, "appeared"),
+                _ev("G2058", 2021, "disappeared"),
+                _ev("G2058", 2021, "replaced_by", to="99439"),
+            ],
+            "99439": [
+                _ev("99439", 2021, "appeared"),
+                _ev("99439", 2021, "replaces", frm="G2058"),
+            ],
+        }
+        descriptions = {
+            "99490": "Chrnc care mgmt staff 1st 20",
+            "99439": "Chrnc care mgmt staf ea addl",
+            "99487": "Cplx chrnc care 1st 60 min",
+            "99489": "Cplx chrnc care ea addl 30",
+            "99491": "Chrnc care mgmt phys 1st 30",
+            "99437": "Chrnc care mgmt phys ea addl",
+            "G2058": "Ccm add 20min",
+            "99497": "Advncd care plan 30 min",
+            "99498": "Advncd care plan addl 30 min",
+        }
+        rows = derive_families(elements, events, descriptions)
+        ccm = {r.code: r for r in rows if r.key == "CCM"}
+        assert set(ccm) == {
+            "99490",
+            "99439",
+            "99487",
+            "99489",
+            "99491",
+            "99437",
+            "G2058",
+        }
+        assert ccm["99439"].role == "add-on" and ccm["G2058"].role == "predecessor"
+        assert ccm["G2058"].since == 2020 and ccm["G2058"].until == 2021
+        assert ccm["99490"].role == "base"
+        acp = {r.code for r in rows if r.key == "ACP"}
+        assert acp == {"99497", "99498"}
+        assert all(
+            r.name == HAND_FAMILIES[r.key].name for r in rows if r.key in HAND_FAMILIES
+        )
+
+    def test_unknown_family_gets_stem_key(self):
+        elements = {
+            "99453": [_el("99453", "activity", "device-data-review")],
+            "99454": [_el("99454", "activity", "device-data-review")],
+        }
+        rows = derive_families(
+            elements,
+            {},
+            {
+                "99453": "Rem mntr physiol param setup",
+                "99454": "Rem mntr physiol param dev supl",
+            },
+        )
+        keys = {r.key for r in rows}
+        assert len(keys) == 1 and next(iter(keys)).startswith("REM-MNTR")
+
+
+class TestLoadAndRefresh:
+    def test_load_and_refresh_in_place(self):
+        from pfs import families as mod
+
+        con = duckdb.connect(":memory:")
+        try:
+            ensure_tables(con)
+            assert load_families(con) == {}
+            before = dict(mod.FAMILIES)
+            assert refresh_from(con) == 0 and mod.FAMILIES == before
+            write_families(
+                con,
+                [
+                    FamilyRow(
+                        "CCM",
+                        "Chronic Care Management",
+                        "99490",
+                        "base",
+                        None,
+                        None,
+                        "",
+                        0,
+                    ),
+                    FamilyRow(
+                        "CCM",
+                        "Chronic Care Management",
+                        "G2058",
+                        "predecessor",
+                        2020,
+                        2021,
+                        "",
+                        0,
+                    ),
+                ],
+            )
+            n = refresh_from(con)
+            assert (
+                n == 1
+                and set(mod.FAMILIES) == {"CCM"}
+                and "G2058" in mod.FAMILIES["CCM"].codes
+            )
+            # restore for other tests
+            mod.FAMILIES.clear()
+            mod.FAMILIES.update(before)
+        finally:
+            con.close()