feat(notebooks): code_families — CPT hierarchy, instructions and family headings (refs #687)

Section 0 states the two organizing principles in order (CPT hierarchy
first, FR second) with the replica's CPT edition years. Section 1 gains
"Where the manual files this code": breadcrumb from cpt_section.path_key,
symbols, the manual's own stem/elements/tail, and its parenthetical
instructions — a HCPCS pick (G-codes) renders a "not in the CPT book"
note instead of erroring. Section 3 explains CPT Changes years (AMA's
own revision years) versus FR mentioning-rule dating. Section 4 shows a
family's CPT heading, sibling headings under the same parent, and
summary counts (total/multi-code/CPT-named families by top-level
chapter). Section 8 adds pfs.cpt_* row counts per edition and the
edition bib items with titles.

Claude-Session: https://claude.ai/code/session_01Aum3pEMAM3yQVdFSdVe6Gc

notebooks/code_families.py

@@ -127,6 +127,31 @@ def _(fr_md, mo, store):
     return
 
 
+@app.cell(hide_code=True)
+def _(mo, not_built, q):
+    # Two organizing principles behind every table on this page
+    _editions = q(
+        "SELECT DISTINCT edition_year FROM pfs.cpt_code ORDER BY edition_year"
+    )
+    if _editions.is_empty():
+        _view = mo.md(not_built("stack pfs cpt-ingest --all"))
+    else:
+        _years = ", ".join(str(y) for y in _editions["edition_year"].to_list())
+        _view = mo.md(
+            "**Two organizing principles, in that order.** The CPT codebook is the **first**: "
+            "the American Medical Association's own hierarchy — Section → subsection → category "
+            "→ subcategory → codes, with guideline text at every section, symbols marking new, "
+            "revised, add-on and telemedicine codes, and parenthetical instructions "
+            "cross-referencing related codes. That hierarchy is what groups codes into clinical "
+            "families below, and it is what a descriptor's elements are drawn from. The Federal "
+            "Register is the **second**: it is where CMS decides whether Medicare pays for a code "
+            "the AMA has already defined, and how much — the FR reprices codes, it does not "
+            f"reorganize the code set. CPT editions on this replica: {_years}."
+        )
+    _view
+    return
+
+
 @app.cell(hide_code=True)
 def _(mo):
     # ── 1. Reading a descriptor ──
@@ -219,6 +244,92 @@ def _(code_picker, fr_md, mo, pl, store):
     return code, elements
 
 
+@app.cell(hide_code=True)
+def _(code, mo, not_built, pl, q):
+    # Where the manual files this code
+    _ed = q("SELECT max(edition_year) y FROM pfs.cpt_code")
+    if _ed.is_empty():
+        _view = mo.md(
+            "**Where the manual files this code.**\n\n"
+            + not_built("stack pfs cpt-ingest --all")
+        )
+    else:
+        _year = _ed.item(0, "y")
+        _cpt = q(
+            "SELECT c.stem, c.elements, c.tail, c.addon, c.resequenced, c.new, c.revised, "
+            "c.telemedicine, c.mod51_exempt, c.audio_only, c.fda_pending, c.pla, s.path_key "
+            "FROM pfs.cpt_code c JOIN pfs.cpt_section s USING (edition_year, item_key, sec_id) "
+            "WHERE c.edition_year = ? AND c.code = ?",
+            (_year, code),
+        )
+        if _cpt.is_empty():
+            _view = mo.md(
+                f"**Where the manual files this code.** `{code}` is a HCPCS Level II code — "
+                "CMS's own coding system, used when Medicare has a programmatic need CPT does "
+                f"not cover — and is **not in the CPT {_year} book**; the manual's hierarchy "
+                "below does not apply to it."
+            )
+        else:
+            _row = _cpt.row(0, named=True)
+            _crumb = " → ".join(_row["path_key"].split(" > "))
+            _flags = [
+                label
+                for label, on in (
+                    ("add-on ✚", _row["addon"]),
+                    ("resequenced #", _row["resequenced"]),
+                    ("new ●", _row["new"]),
+                    ("revised ▲", _row["revised"]),
+                    ("telemedicine ★", _row["telemedicine"]),
+                    ("modifier-51 exempt ⦸", _row["mod51_exempt"]),
+                    ("audio-only", _row["audio_only"]),
+                    ("FDA-pending ⚡", _row["fda_pending"]),
+                    ("PLA", _row["pla"]),
+                )
+                if on
+            ]
+            _instr = q(
+                "SELECT kind, text, targets FROM pfs.cpt_instruction WHERE edition_year = ? "
+                "AND code = ? ORDER BY kind, text",
+                (_year, code),
+            )
+            _view = mo.vstack(
+                [
+                    mo.md(
+                        "**Where the manual files this code.** The CPT hierarchy is the first "
+                        f"organizing principle: in the CY{_year} book `{code}` sits under "
+                        f"**{_crumb}**."
+                        + (
+                            f" Symbols: {', '.join(_flags)}."
+                            if _flags
+                            else " No symbols set."
+                        )
+                    ),
+                    mo.ui.table(
+                        pl.DataFrame(
+                            {
+                                "part": [
+                                    "stem",
+                                    *["element"] * len(_row["elements"]),
+                                    "tail",
+                                ],
+                                "text": [_row["stem"], *_row["elements"], _row["tail"]],
+                            }
+                        ),
+                        label="The manual's own stem / elements / tail",
+                    ),
+                    (
+                        mo.ui.table(_instr, label="Parenthetical instructions")
+                        if not _instr.is_empty()
+                        else mo.md(
+                            "_No parenthetical instructions for this code in this edition._"
+                        )
+                    ),
+                ]
+            )
+    _view
+    return
+
+
 @app.cell(hide_code=True)
 def _(code, mo, not_built, q):
     # ── 2. What the extractor wrote ──
@@ -323,7 +434,11 @@ def _(alt, code, fr_md, mo, not_built, pl, plain_years, q):
                     "earliest anchored event is the origin, not the latest one. An RVU event is "
                     "`anchored` when a Federal Register event for the same code lies within one "
                     "rule year of it; an unanchored RVU event is evidence CMS never wrote a "
-                    "sentence about, and is weaker to cite."
+                    "sentence about, and is weaker to cite. CPT-source events (`source=cpt`) are "
+                    "dated a third way: `cpt_changed` rows use the AMA's own CPT Changes edition "
+                    "year — the year the AMA revised the code, independent of whether or when any "
+                    "FR rule mentions it — and both an FR event and a CPT event for the same code "
+                    "can anchor the same RVU change from two independent directions."
                 ),
                 mo.ui.altair_chart(_chart),
                 mo.ui.table(
@@ -336,7 +451,7 @@ def _(alt, code, fr_md, mo, not_built, pl, plain_years, q):
 
 
 @app.cell(hide_code=True)
-def _(code, mo, not_built, plain_years, q):
+def _(code, mo, not_built, pl, plain_years, q):
     # ── 4. Families ──
     from pfs.families import family_of as _family_of
 
@@ -344,8 +459,8 @@ def _(code, mo, not_built, plain_years, q):
     _key = _hand.key if _hand else ""
     _rows = (
         q(
-            "SELECT key, name, code, role, since, until, item_key, p_id FROM pfs.code_family "
+            "SELECT key, name, code, role, since, until, item_key, p_id, note "
-            "WHERE key = ? ORDER BY code",
+            "FROM pfs.code_family WHERE key = ? ORDER BY code",
             (_key,),
         )
         if _key
@@ -354,6 +469,44 @@ def _(code, mo, not_built, plain_years, q):
     if _rows is None or _rows.is_empty():
         _view = mo.md("## 4. Families\n\n" + not_built("stack pfs families --write"))
     else:
+        # The family's CPT heading — the organizing principle that classified its members
+        # (Task 6 context: "note" is per-code, so different members can carry different
+        # headings when the hand list or a derived edge pulls in a neighboring heading).
+        _notes = [n for n in _rows["note"].to_list() if n]
+        _note = _notes[0] if _notes else ""
+        _siblings = pl.DataFrame()
+        if _note:
+            _parts = _note.split(" > ")
+            _parent_prefix = " > ".join(_parts[:-1])
+            _yr = q("SELECT max(edition_year) y FROM pfs.cpt_section")
+            _year = _yr.item(0, "y") if not _yr.is_empty() else None
+            if _year is not None:
+                _lvl_row = q(
+                    "SELECT min(level) lvl FROM pfs.cpt_section WHERE edition_year = ? "
+                    "AND path_key = ?",
+                    (_year, _note),
+                )
+                _lvl = (
+                    _lvl_row.item(0, "lvl")
+                    if not _lvl_row.is_empty() and _lvl_row.item(0, "lvl") is not None
+                    else None
+                )
+                if _lvl is not None:
+                    _siblings = q(
+                        "SELECT DISTINCT title FROM pfs.cpt_section WHERE edition_year = ? "
+                        "AND path_key LIKE ? AND level = ? ORDER BY title",
+                        (_year, _parent_prefix + " > %", _lvl),
+                    )
+        _summary = q(
+            "WITH fam AS (SELECT key, count(*) n, bool_or(note <> '') has_note "
+            "FROM pfs.code_family GROUP BY key) "
+            "SELECT count(*) total, count(*) FILTER (WHERE n > 1) multi, "
+            "count(*) FILTER (WHERE has_note) cpt_named FROM fam"
+        )
+        _by_chapter = q(
+            "SELECT split_part(note, ' > ', 1) AS chapter, count(DISTINCT key) AS families "
+            "FROM pfs.code_family WHERE note <> '' GROUP BY 1 ORDER BY families DESC"
+        )
         _view = mo.vstack(
             [
                 mo.md(
@@ -369,7 +522,19 @@ def _(code, mo, not_built, plain_years, q):
                     f"shows {_rows.height}, because the connected-component search also reaches "
                     "codes the hand list never named."
                 ),
+                mo.md(
+                    f"**Where this family sits in the manual.** {_note}"
+                    if _note
+                    else "_No member of this family carries a CPT heading (HCPCS-only family)._"
+                ),
                 mo.ui.table(plain_years(_rows), label=f"pfs.code_family — {_key}"),
+                (
+                    mo.ui.table(
+                        _siblings, label="Sibling headings under the same parent"
+                    )
+                    if not _siblings.is_empty()
+                    else mo.md("_No sibling headings found._")
+                ),
                 mo.md(
                     "**Example (single-target replacement):** HCPCS G2058, billable only in "
                     "2020, was replaced the following year by CPT 99439 — an identical "
@@ -377,6 +542,22 @@ def _(code, mo, not_built, plain_years, q):
                     "exactly one target code, so it is unambiguous evidence, and G2058 joins CCM "
                     "as a *predecessor* rather than becoming a one-code family of its own."
                 ),
+                (
+                    mo.md(
+                        f"**Across all families:** {_summary.item(0, 'total')} total, "
+                        f"{_summary.item(0, 'multi')} multi-code, "
+                        f"{_summary.item(0, 'cpt_named')} carry a CPT heading (`note <> ''`)."
+                    )
+                    if not _summary.is_empty()
+                    else mo.md("")
+                ),
+                (
+                    mo.ui.table(
+                        _by_chapter, label="CPT-named families by top-level chapter"
+                    )
+                    if not _by_chapter.is_empty()
+                    else mo.md("")
+                ),
             ]
         )
     _view
@@ -422,7 +603,7 @@ def _(mo, not_built):
 
 
 @app.cell(hide_code=True)
-def _(NOTES, REPLICA_PATH, con, mo, q):
+def _(NOTES, REPLICA_PATH, con, mo, pl, q, store):
     # ── 8. Provenance ──
     import datetime as _dt
 
@@ -444,6 +625,40 @@ def _(NOTES, REPLICA_PATH, con, mo, q):
         "fr_citation, pincite_key FROM cms.ingest_log WHERE table_name LIKE 'pfs.%' "
         "ORDER BY ingested_at DESC LIMIT 20"
     )
+    # pfs.cpt_* comes from a separate ingest path (stack pfs cpt-ingest) with its own
+    # per-edition grain, so it gets its own counts table and its own bib provenance —
+    # one row per CPT edition on the replica, with the edition's bib title.
+    _cpt_counts = q(
+        "SELECT 'cpt_section' t, edition_year, count(*) n FROM pfs.cpt_section GROUP BY 1, 2 "
+        "UNION ALL SELECT 'cpt_code', edition_year, count(*) FROM pfs.cpt_code GROUP BY 1, 2 "
+        "UNION ALL SELECT 'cpt_instruction', edition_year, count(*) "
+        "FROM pfs.cpt_instruction GROUP BY 1, 2 "
+        "UNION ALL SELECT 'cpt_reference', edition_year, count(*) "
+        "FROM pfs.cpt_reference GROUP BY 1, 2 "
+        "UNION ALL SELECT 'cpt_crosswalk', edition_year, count(*) "
+        "FROM pfs.cpt_crosswalk GROUP BY 1, 2 "
+        "UNION ALL SELECT 'cpt_list', edition_year, count(*) FROM pfs.cpt_list GROUP BY 1, 2 "
+        "UNION ALL SELECT 'cpt_code_alt', edition_year, count(*) "
+        "FROM pfs.cpt_code_alt GROUP BY 1, 2 "
+        "ORDER BY edition_year, t"
+    )
+    _cpt_editions = q(
+        "SELECT DISTINCT edition_year, item_key FROM pfs.cpt_code ORDER BY edition_year"
+    )
+    _bib_rows = []
+    for _yr, _key in zip(
+        [] if _cpt_editions.is_empty() else _cpt_editions["edition_year"].to_list(),
+        [] if _cpt_editions.is_empty() else _cpt_editions["item_key"].to_list(),
+    ):
+        if store is None:
+            _title = "(bibliography unavailable)"
+        else:
+            try:
+                _title = store.get(_key).title
+            except KeyError:
+                _title = "(item not found)"
+        _bib_rows.append({"edition_year": _yr, "item_key": _key, "title": _title})
+    _cpt_bib = pl.DataFrame(_bib_rows) if _bib_rows else pl.DataFrame()
     mo.vstack(
         [
             mo.md(
@@ -456,7 +671,9 @@ def _(NOTES, REPLICA_PATH, con, mo, q):
                 )
                 + ". Row counts and the most recent ingest-log entries for every `pfs.code_*` "
                 "table this notebook reads follow, so a reader can tell how fresh the family "
-                "data is without leaving the page."
+                "data is without leaving the page. The CPT codebook tables (`pfs.cpt_*`) are "
+                "ingested separately, one edition at a time, so their counts and bib "
+                "provenance are broken out by edition below."
             ),
             mo.ui.table(_counts, label="pfs.code_* row counts")
             if not _counts.is_empty()
@@ -464,6 +681,14 @@ def _(NOTES, REPLICA_PATH, con, mo, q):
             mo.ui.table(_log, label="Ingest log — pfs.*")
             if not _log.is_empty()
             else mo.md("_no ingest log rows_"),
+            mo.ui.table(_cpt_counts, label="pfs.cpt_* row counts by edition")
+            if not _cpt_counts.is_empty()
+            else mo.md(
+                "_no CPT codebook tables built yet — run `stack pfs cpt-ingest --all`_"
+            ),
+            mo.ui.table(_cpt_bib, label="CPT edition bib items")
+            if not _cpt_bib.is_empty()
+            else mo.md("_no CPT editions ingested_"),
             mo.md(
                 "\n".join(f"- {k}: {v}" for k, v in NOTES.items())
                 if NOTES