pfs.cite (Cite + builders; rule_label moved from llm.lineage and re-exported), pfs.citecheck (sentence split, number normalization, check_prose; plus grounding_problems: words, codes and CMS-vs-ours attribution of model prose), pfs.dossier (facts with resolved cites from every Task 1-12 table, timeline, stakeholder panels with names from pfs.commenter, stance accuracy beside every count, computed coverage gaps, ledgered model prose with template fallback, pfs.dossier table), llm.search item_keys filter, `stack pfs dossier`. Lineage rows reach the dossier only when a final rule's own plain sentence states them. HOME-VISITS live: 558 facts, 28 timeline events, check passed, prose model:qwen2.5:14b+template (8 of 29 units on templates).
2482 lines
90 KiB
Python
2482 lines
90 KiB
Python
"""stack pfs — code elements, lineage events and derived families.
|
||
|
||
uv run stack pfs elements --code G0556 --code 99490
|
||
uv run stack pfs elements --family CCM
|
||
uv run stack pfs elements --all-payable --no-llm --dry-run
|
||
uv run stack pfs lineage --code G2058 [--write]
|
||
uv run stack pfs families [--write]
|
||
uv run stack pfs guidance --family CCM [--family …] [--write]
|
||
uv run stack pfs reaction --family CCM [--code G2211] [--stance-sample 30] [--write]
|
||
uv run stack pfs review [--code G0556]
|
||
uv run stack pfs cpt-ingest --edition 2024 [--edition 2022 …] [--dry-run]
|
||
uv run stack pfs cpt-ingest --all
|
||
uv run stack pfs dossier --family HOME-VISITS [--no-llm] [--limit N] [--write]
|
||
|
||
Writes go through ``duckdb_batch`` (single-writer rule) and republish
|
||
the read-only replica so the chat and notebooks see them.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import logging
|
||
from collections import Counter
|
||
from collections.abc import Sequence
|
||
from dataclasses import replace
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
from typing import Any
|
||
|
||
import httpx
|
||
import typer
|
||
|
||
from pfs import seats as seat_lib
|
||
from pfs.cms_summaries import summaries as cms_summaries_rows
|
||
from pfs.codetables import (
|
||
STANCE_METHOD_FIELDS,
|
||
ReactionRow,
|
||
cpt_years,
|
||
ensure_tables,
|
||
is_missing_table_error,
|
||
read_all_elements,
|
||
read_all_events,
|
||
read_all_reviews,
|
||
read_commenters,
|
||
read_cpt_codes,
|
||
read_cpt_instructions,
|
||
read_cpt_sections,
|
||
read_exposures,
|
||
read_guidance,
|
||
read_reaction,
|
||
read_reviews,
|
||
read_stances,
|
||
write_cms_summaries,
|
||
write_commenters,
|
||
write_elements,
|
||
write_events,
|
||
write_exposures,
|
||
write_families,
|
||
write_guidance,
|
||
write_guideline_facts,
|
||
write_origin,
|
||
write_reaction,
|
||
write_rounds,
|
||
write_stances,
|
||
)
|
||
from pfs.commenters import family_comment_items, identify, regs_for
|
||
from pfs.cpt_load import ingest as cpt_ingest
|
||
from pfs.curated import apply_curated, load_curated
|
||
from pfs.dossier import (
|
||
build_dossier,
|
||
collect_facts,
|
||
governance_excerpts,
|
||
stakeholder_lines,
|
||
write_dossier,
|
||
)
|
||
from pfs.dossier import timeline as dossier_timeline
|
||
from pfs.exposure import build as build_exposures
|
||
from pfs.exposure import control_codes
|
||
from pfs.extract import extract_codes
|
||
from pfs.families import (
|
||
CURATED_FAMILIES,
|
||
FAMILIES,
|
||
HAND_FAMILIES,
|
||
curated_family,
|
||
derive_families,
|
||
normalize_key,
|
||
refresh_from,
|
||
)
|
||
from pfs.guidance import build as build_guidance
|
||
from pfs.guideline_facts import guideline_facts
|
||
from pfs.lineage import lineage, lineage_all, seen_codes
|
||
from pfs.origin import DEFAULT_BOOK, family_origin, origin_inventory, resolve_book
|
||
from pfs.psps import DATASETS as PSPS_YEARS
|
||
from pfs.psps import USER_AGENT as PSPS_USER_AGENT
|
||
from pfs.psps import cache_path as psps_cache_path
|
||
from pfs.psps import cite as cite_psps
|
||
from pfs.psps import fetch_code_year as fetch_psps
|
||
from pfs.psps import load_cache as load_psps_cache
|
||
from pfs.psps import newest_cache as newest_psps_cache
|
||
from pfs.psps import read_by_specialty as read_psps_rows
|
||
from pfs.psps import save_cache as save_psps_cache
|
||
from pfs.psps import to_rows as psps_rows
|
||
from pfs.psps import write as write_psps_rows
|
||
from pfs.reaction import series as reaction_series
|
||
from pfs.rounds import family_rounds, rounds_inventory
|
||
from pfs.runlog import RunLog, input_hash, ledger_path, run_units
|
||
from pfs.stakes import (
|
||
discover_508,
|
||
discover_fr,
|
||
parse_impact_508,
|
||
parse_impact_fr_text,
|
||
write_impacts,
|
||
)
|
||
from pfs.stance import (
|
||
DEFAULT_GENERATION,
|
||
DEFAULT_METHOD,
|
||
DEFAULT_MODEL,
|
||
GENERATIONS,
|
||
GOLDEN_PATH,
|
||
HELDOUT_PATHS,
|
||
METHODS,
|
||
TEXT_CHARS,
|
||
TEXT_CHUNKS,
|
||
docket_events,
|
||
docket_rule_year,
|
||
eval_record,
|
||
event_subjects,
|
||
fixed_text,
|
||
golden_dockets,
|
||
guideline_text,
|
||
item_dockets,
|
||
prompt_version,
|
||
rollup,
|
||
score,
|
||
stances_for,
|
||
)
|
||
from pfs.utilization import DATASETS as UTILIZATION_YEARS
|
||
from pfs.utilization import cache_path as utilization_cache_path
|
||
from pfs.utilization import cite as cite_utilization
|
||
from pfs.utilization import fetch_year as fetch_utilization_year
|
||
from pfs.utilization import load_cache as load_utilization_cache
|
||
from pfs.utilization import read_series as read_utilization_series
|
||
from pfs.utilization import save_cache as save_utilization_cache
|
||
from pfs.utilization import to_rows as utilization_rows
|
||
from pfs.utilization import write as write_utilization
|
||
|
||
log = logging.getLogger(__name__)
|
||
|
||
app = typer.Typer(no_args_is_help=True)
|
||
|
||
|
||
#: A replica that hasn't had `elements`/`lineage --write` (or an initial
|
||
#: `families --write`) run yet is missing `pfs.code_element`/
|
||
#: `pfs.code_event`/`pfs.code_family` — not a bug, just nothing derived
|
||
#: yet (I4). DuckDB's "table does not exist" errors are Catalog errors —
|
||
#: the predicate itself lives in `pfs.codetables` so `pfs.lineage`/
|
||
#: `pfs.extract` can reuse it for the same replica shape (a CPT-table-
|
||
#: free replica is not a bug either).
|
||
def _missing_table(exc: Exception) -> bool:
|
||
return is_missing_table_error(exc)
|
||
|
||
|
||
# ── indirections the tests monkeypatch ───────────────────────────────
|
||
def _batch() -> Any:
|
||
from conf.connect import duckdb_batch
|
||
|
||
return duckdb_batch("aco")
|
||
|
||
|
||
def _read() -> Any:
|
||
from conf.connect import duckdb
|
||
|
||
return duckdb("aco", read_only=True)
|
||
|
||
|
||
def _store() -> Any:
|
||
from conf.connect import bib
|
||
|
||
return bib()
|
||
|
||
|
||
def _publish() -> None:
|
||
from conf.connect import publish_replica
|
||
|
||
typer.echo(f"replica → {publish_replica('aco')}")
|
||
|
||
|
||
def _classifier(*, pin: bool = False) -> Any:
|
||
from llm import config as llm_config
|
||
from llm.classify import closed_vocab_classifier
|
||
from llm.pool import HostPool
|
||
|
||
cfg = llm_config.load()
|
||
return closed_vocab_classifier(
|
||
cfg, HostPool.from_config(cfg), model=cfg.instruct_model if pin else None
|
||
)
|
||
|
||
|
||
def _asker(large: bool = False) -> Any:
|
||
"""The JSON extractor on ``cfg.instruct_model``, or with *large* on
|
||
``cfg.instruct_model_large`` (still one pinned model, never a fallback)."""
|
||
from llm import config as llm_config
|
||
from llm.pool import HostPool
|
||
from llm.structured import json_extractor
|
||
|
||
cfg = llm_config.load()
|
||
if large:
|
||
if not cfg.instruct_model_large:
|
||
raise typer.BadParameter("no instruct_model_large configured")
|
||
cfg = replace(cfg, instruct_model=cfg.instruct_model_large)
|
||
return json_extractor(
|
||
cfg, HostPool.from_config(cfg), post=_slow_post if large else None
|
||
)
|
||
|
||
|
||
def _slow_post(url: str, json: dict) -> dict:
|
||
"""``llm.classify._default_post`` with a 10-minute read timeout: the large
|
||
model reloads on every call and a whole-comment prompt outlasts the usual
|
||
120 s."""
|
||
import httpx
|
||
|
||
with httpx.Client(timeout=httpx.Timeout(600.0, connect=10.0)) as client:
|
||
resp = client.post(url, json=json)
|
||
resp.raise_for_status()
|
||
return resp.json()
|
||
|
||
|
||
def _model_name(large: bool = False) -> str:
|
||
"""The instruct model's name (or the large one), without touching the GPU
|
||
pool: what a ledger replay (``--no-llm``) keys the recorded outputs by."""
|
||
from llm import config as llm_config
|
||
|
||
cfg = llm_config.load()
|
||
return cfg.instruct_model_large if large else cfg.instruct_model
|
||
|
||
|
||
def _mirror() -> Any:
|
||
from bib.regulations_gov import Mirror
|
||
|
||
return Mirror()
|
||
|
||
|
||
def _engine() -> Any:
|
||
# llm.index/llm.evidence expose no public "give me a bare SQLAlchemy
|
||
# engine over the pgvector DB" helper — cli/bib.py and cli/llm.py
|
||
# both already reach into llm.index._engine for the same reason, so
|
||
# this is the established reuse, not a new private import.
|
||
from llm import config as llm_config
|
||
from llm.index import _engine as pgvector_engine
|
||
|
||
return pgvector_engine(llm_config.load())
|
||
|
||
|
||
def _run_elements(
|
||
con: Any, store: Any, targets: list[str], classify: Any, *, write: bool
|
||
) -> None:
|
||
# #698: one inverted pass over fr_anchors (plus one query each over
|
||
# pfs.cpt_code / terminology.hcpcs_level_2 / pfs.rvu) for the whole
|
||
# target list — the per-code form re-scanned the 193k-row, unindexed
|
||
# fr_anchors table twice per code, ~15-25 min for --all-payable.
|
||
by_code = extract_codes(store, con, targets, classify=classify)
|
||
for c in targets:
|
||
x = by_code[c]
|
||
if write:
|
||
write_elements(con, c, x.rows, x.reviews)
|
||
confirmed = sum(1 for r in x.rows if r.confirmed_by)
|
||
note = f", {confirmed} confirmed by a second source" if confirmed else ""
|
||
typer.echo(f"{c}: {len(x.rows)} elements, {len(x.reviews)} for review{note}")
|
||
|
||
|
||
def _codes_for(
|
||
con: Any,
|
||
codes: list[str],
|
||
families: list[str],
|
||
all_payable: bool,
|
||
*,
|
||
all_seen: bool = False,
|
||
warn_slow: bool = True,
|
||
) -> list[str]:
|
||
"""*warn_slow*: log how many codes ``--all-payable`` resolved to, so a
|
||
run that is about to make one model call per unplaced line for
|
||
thousands of codes says so up front. ``lineage --all-payable`` is
|
||
pure SQL/regex and needs no such note, so it passes
|
||
``warn_slow=False``. (Both commands now sweep ``fr_anchors`` in one
|
||
inverted pass — #698 — so neither is the ~20 min of per-code SQL the
|
||
warning used to describe.)"""
|
||
out: list[str] = [c.upper() for c in codes]
|
||
if families:
|
||
refresh_from(con)
|
||
for family in families:
|
||
fam = curated_family(family) or FAMILIES.get(normalize_key(family))
|
||
if fam is None:
|
||
# List only the hand families here (M3) — FAMILIES can hold
|
||
# ~15k derived keys once `refresh_from` has run, and that
|
||
# dump is useless in an error message.
|
||
raise typer.BadParameter(
|
||
f"unknown family {family!r}; known: {', '.join(HAND_FAMILIES)}"
|
||
)
|
||
out.extend(fam.codes)
|
||
if all_payable:
|
||
rows = con.execute(
|
||
"SELECT DISTINCT hcpcs FROM pfs.rvu WHERE status_code IN ('A','R','T') "
|
||
"AND year = (SELECT max(year) FROM pfs.rvu) ORDER BY hcpcs"
|
||
).fetchall()
|
||
if warn_slow:
|
||
log.warning("--all-payable: targeting %d codes", len(rows))
|
||
out.extend(r[0] for r in rows)
|
||
if all_seen:
|
||
seen = seen_codes(con)
|
||
if warn_slow:
|
||
log.warning("--all-seen: targeting %d codes", len(seen))
|
||
out.extend(seen)
|
||
if not out:
|
||
raise typer.BadParameter("pass --code, --family, --all-payable or --all-seen")
|
||
return sorted(set(out))
|
||
|
||
|
||
@app.command()
|
||
def elements(
|
||
code: list[str] = typer.Option([], "--code", help="HCPCS/CPT code (repeatable)."),
|
||
family: list[str] = typer.Option(
|
||
[], "--family", help="Expand a registered family (CCM, APCM, …); repeatable."
|
||
),
|
||
all_payable: bool = typer.Option(
|
||
False,
|
||
"--all-payable",
|
||
help=(
|
||
"Every A/R/T code in the newest RVU year, one inverted pass over "
|
||
"fr_anchors."
|
||
),
|
||
),
|
||
no_llm: bool = typer.Option(
|
||
False,
|
||
"--no-llm",
|
||
help="Skip the local-model classifier (unknown lines go to review).",
|
||
),
|
||
dry_run: bool = typer.Option(
|
||
False, "--dry-run", help="Extract and report; write nothing."
|
||
),
|
||
) -> None:
|
||
"""Extract typed elements for codes into pfs.code_element (+ review queue)."""
|
||
store = _store()
|
||
classify = None if no_llm else _classifier()
|
||
if dry_run:
|
||
# A preview never contends for the DuckDB single-writer lock a
|
||
# notebook may be holding (#508-#514) — read the replica, don't
|
||
# open a batch writer.
|
||
con = _read()
|
||
try:
|
||
targets = _codes_for(con, code, family, all_payable)
|
||
_run_elements(con, store, targets, classify, write=False)
|
||
finally:
|
||
con.close()
|
||
else:
|
||
with _batch() as con:
|
||
ensure_tables(con)
|
||
targets = _codes_for(con, code, family, all_payable)
|
||
_run_elements(con, store, targets, classify, write=True)
|
||
_publish()
|
||
|
||
|
||
def _print_events(events: list[Any]) -> None:
|
||
for e in events:
|
||
arrow = f"{e.from_codes or '·'}→{e.to_codes or '·'}"
|
||
if e.source == "cpt":
|
||
anchor = f"cpt {e.year} {e.item_key}"
|
||
elif e.item_key:
|
||
anchor = f"{e.item_key} ¶{e.p_id}"
|
||
else:
|
||
anchor = f"rvu {e.note}".strip()
|
||
flag = "" if e.anchored else " UNANCHORED"
|
||
typer.echo(f"{e.year} {e.kind:<18}{arrow:<16}{anchor}{flag}")
|
||
|
||
|
||
def _print_lineage_all(by_code: dict[str, list[Any]]) -> None:
|
||
per_source: dict[str, int] = {}
|
||
with_events = 0
|
||
unanchored = 0
|
||
for events in by_code.values():
|
||
if events:
|
||
with_events += 1
|
||
for e in events:
|
||
per_source[e.source] = per_source.get(e.source, 0) + 1
|
||
if e.source == "rvu" and not e.anchored:
|
||
unanchored += 1
|
||
detail = ", ".join(f"{k}={v}" for k, v in sorted(per_source.items())) or "none"
|
||
typer.echo(
|
||
f"lineage: {len(by_code)} codes targeted, {with_events} with >=1 event, "
|
||
f"events per source: {detail}, unanchored rvu events: {unanchored}"
|
||
)
|
||
|
||
|
||
@app.command("lineage")
|
||
def lineage_cmd(
|
||
code: str = typer.Option("", "--code"),
|
||
family: str = typer.Option(
|
||
"", "--family", help="A curated or registered family's codes."
|
||
),
|
||
all_payable: bool = typer.Option(
|
||
False,
|
||
"--all-payable",
|
||
help="Every A/R/T code in the newest RVU year, one inverted pass over fr_anchors.",
|
||
),
|
||
all_seen: bool = typer.Option(
|
||
False,
|
||
"--all-seen",
|
||
help="Every code ever seen in any RVU year or CPT edition (incl. deleted codes).",
|
||
),
|
||
limit: int = typer.Option(
|
||
0,
|
||
"--limit",
|
||
help="Only the first N target codes (sorted) — for smoke runs with --all-payable.",
|
||
),
|
||
write: bool = typer.Option(
|
||
False, "--write", help="Persist to pfs.code_event and republish."
|
||
),
|
||
) -> None:
|
||
"""Timeline of a code: RVU-file diffs and FR paragraphs, cross-checked.
|
||
|
||
--all-payable computes lineage for every payable code in ONE pass over
|
||
fr_anchors instead of one query per code.
|
||
"""
|
||
if not code and not all_payable and not all_seen and not family:
|
||
raise typer.BadParameter("pass --code, --family, --all-payable or --all-seen")
|
||
store = _store()
|
||
|
||
if all_payable or all_seen or family:
|
||
# Ruling A13: resolve targets and build every code's lineage
|
||
# against a plain DuckDB read (I4) — never the write lock a
|
||
# notebook may be holding (#508-#514). The batch below only ever
|
||
# calls write_events and publishes.
|
||
con = _read()
|
||
try:
|
||
targets = _codes_for(
|
||
con,
|
||
[],
|
||
[family] if family else [],
|
||
all_payable,
|
||
all_seen=all_seen,
|
||
warn_slow=False,
|
||
)
|
||
if limit:
|
||
targets = targets[:limit]
|
||
if code:
|
||
targets = sorted({*targets, code.upper()})
|
||
by_code = lineage_all(con, store, targets)
|
||
finally:
|
||
con.close()
|
||
if write:
|
||
with _batch() as con:
|
||
ensure_tables(con)
|
||
for c in targets:
|
||
write_events(con, c, by_code.get(c, []))
|
||
_publish()
|
||
_print_lineage_all(by_code)
|
||
return
|
||
|
||
if write:
|
||
# Same A13 discipline for the single-code path: build the events
|
||
# against a plain read, then open the batch only to write them.
|
||
con = _read()
|
||
try:
|
||
events = lineage(con, store, code.upper())
|
||
finally:
|
||
con.close()
|
||
_print_events(events)
|
||
with _batch() as con:
|
||
ensure_tables(con)
|
||
write_events(con, code.upper(), events)
|
||
_publish()
|
||
return
|
||
# No --write: a plain lookup never needs the DuckDB single-writer lock
|
||
# a notebook may be holding (#508-#514) — read the replica (I4).
|
||
con = _read()
|
||
try:
|
||
try:
|
||
events = lineage(con, store, code.upper())
|
||
except Exception as exc:
|
||
if _missing_table(exc):
|
||
typer.echo("no derived tables yet — run with --write")
|
||
return
|
||
raise
|
||
_print_events(events)
|
||
finally:
|
||
con.close()
|
||
|
||
|
||
def _cpt_inputs(
|
||
con: Any,
|
||
) -> tuple[list[Any], list[Any], list[Any], dict[str, tuple[int, ...]]]:
|
||
"""The newest CPT edition's rows for ``derive_families`` (grouping/
|
||
naming stays newest-edition-scoped, Ruling C12), plus ``cpt_presence``
|
||
— every ingested edition's ``code -> years present`` map, one extra
|
||
full-table pass, used only for ``since``/``until``: a code an earlier
|
||
edition ingested but the newest one dropped still gets its true
|
||
``since`` (not clamped to the newest edition) and a real ``until``.
|
||
All empty on any replica shape that doesn't have them yet (I4: a
|
||
plain read of an old replica, pre pfs.cpt_*, is not a bug)."""
|
||
try:
|
||
years = cpt_years(con)
|
||
except Exception as exc:
|
||
if _missing_table(exc):
|
||
return [], [], [], {}
|
||
raise
|
||
if not years:
|
||
return [], [], [], {}
|
||
newest = max(years)
|
||
presence: dict[str, list[int]] = {}
|
||
for code, year in con.execute(
|
||
"SELECT code, edition_year FROM pfs.cpt_code"
|
||
).fetchall():
|
||
presence.setdefault(code, []).append(year)
|
||
cpt_presence = {code: tuple(sorted(set(yrs))) for code, yrs in presence.items()}
|
||
return (
|
||
read_cpt_codes(con, newest),
|
||
read_cpt_sections(con, newest),
|
||
read_cpt_instructions(con, newest),
|
||
cpt_presence,
|
||
)
|
||
|
||
|
||
def _derive_rows(con: Any) -> list[Any]:
|
||
codes = [
|
||
r[0]
|
||
for r in con.execute(
|
||
"SELECT DISTINCT code FROM pfs.code_element UNION SELECT DISTINCT code FROM pfs.code_event "
|
||
"UNION SELECT DISTINCT hcpcs FROM pfs.rvu WHERE year = (SELECT max(year) FROM pfs.rvu) AND status_code IN ('A','R','T')"
|
||
).fetchall()
|
||
]
|
||
# Two full-table reads grouped in Python instead of ~2xN per-code
|
||
# queries run inside the write lock (I9).
|
||
all_elements = read_all_elements(con)
|
||
all_events = read_all_events(con)
|
||
elements_by_code = {c: all_elements.get(c, []) for c in codes}
|
||
events_by_code = {c: all_events.get(c, []) for c in codes}
|
||
descriptions = {
|
||
r[0]: r[1] or ""
|
||
for r in con.execute(
|
||
"SELECT hcpcs, arg_max(description, year) FROM pfs.rvu WHERE mod IS NULL OR mod = '' GROUP BY hcpcs"
|
||
).fetchall()
|
||
}
|
||
cpt_codes, cpt_sections, cpt_instructions, cpt_presence = _cpt_inputs(con)
|
||
return derive_families(
|
||
elements_by_code,
|
||
events_by_code,
|
||
descriptions,
|
||
cpt_codes=cpt_codes,
|
||
cpt_sections=cpt_sections,
|
||
cpt_instructions=cpt_instructions,
|
||
cpt_presence=cpt_presence,
|
||
)
|
||
|
||
|
||
def _print_families(rows: list[Any], *, verbose: bool = False) -> None:
|
||
by_key: dict[str, list[Any]] = {}
|
||
for r in rows:
|
||
by_key.setdefault(r.key, []).append(r)
|
||
if verbose:
|
||
for key, members in sorted(by_key.items()):
|
||
typer.echo(f"{key}: {' '.join(f'{r.code}({r.role})' for r in members)}")
|
||
total = len(by_key)
|
||
multi = sum(1 for members in by_key.values() if len(members) >= 2)
|
||
hand = sum(1 for key in by_key if key in HAND_FAMILIES)
|
||
curated = sum(1 for key in by_key if key in CURATED_FAMILIES)
|
||
cpt_named = sum(
|
||
1
|
||
for key, members in by_key.items()
|
||
if key not in HAND_FAMILIES
|
||
and key not in CURATED_FAMILIES
|
||
and any(r.note for r in members)
|
||
)
|
||
other = total - hand - curated - cpt_named
|
||
typer.echo(
|
||
f"families: total {total}, multi-code {multi}, cpt-named {cpt_named}, "
|
||
f"hand {hand}, curated {curated}, other {other}"
|
||
)
|
||
|
||
|
||
@app.command()
|
||
def families(
|
||
write: bool = typer.Option(False, "--write"),
|
||
verbose: bool = typer.Option(
|
||
False, "--verbose", help="Print every family's members, not just the summary."
|
||
),
|
||
) -> None:
|
||
"""Derive families from pfs.code_element / pfs.code_event / pfs.rvu."""
|
||
if write:
|
||
with _batch() as con:
|
||
ensure_tables(con)
|
||
rows = apply_curated(_derive_rows(con), load_curated())
|
||
write_families(con, rows)
|
||
_publish()
|
||
# refresh_from and printing don't need the write lock open — do
|
||
# them after it's released, from the freshly published replica
|
||
# (the same pattern `stack llm restamp` uses).
|
||
refresh_from(_read())
|
||
_print_families(rows, verbose=verbose)
|
||
return
|
||
# No --write: derive-and-print never needs the write lock (I4).
|
||
con = _read()
|
||
try:
|
||
try:
|
||
rows = _derive_rows(con)
|
||
except Exception as exc:
|
||
if _missing_table(exc):
|
||
typer.echo("no derived tables yet — run with --write")
|
||
return
|
||
raise
|
||
_print_families(rows, verbose=verbose)
|
||
finally:
|
||
con.close()
|
||
|
||
|
||
def _print_guidance(rows: list[Any]) -> None:
|
||
for r in sorted(rows, key=lambda r: (r.family, r.code, r.kind, r.locator)):
|
||
item = r.item_key or "—"
|
||
typer.echo(
|
||
f"{r.family} {r.kind} {r.locator} {item} ← {r.item_key_src} ¶{r.p_id_src}"
|
||
)
|
||
|
||
|
||
def _resolve_guidance_families(con: Any, keys: list[str]) -> list[tuple[str, Any]]:
|
||
"""(key, Family) per requested key, refreshed from *con* — raises
|
||
``typer.BadParameter`` on the first unknown key, before any family's
|
||
rows are built or any write happens (mirrors ``_reaction_targets``'s
|
||
resolve-everything-up-front discipline)."""
|
||
refresh_from(con)
|
||
out: list[tuple[str, Any]] = []
|
||
for key in keys:
|
||
fam = FAMILIES.get(key)
|
||
if fam is None:
|
||
raise typer.BadParameter(
|
||
f"unknown family {key!r}; known: {', '.join(HAND_FAMILIES)}"
|
||
)
|
||
out.append((key, fam))
|
||
return out
|
||
|
||
|
||
@app.command()
|
||
def guidance(
|
||
family: list[str] = typer.Option(
|
||
[], "--family", help="Family key (CCM, APCM, …); repeatable."
|
||
),
|
||
write: bool = typer.Option(False, "--write"),
|
||
) -> None:
|
||
"""CFR/IOM/MLN sub-regulatory crosswalk per code family
|
||
(pfs.code_guidance): which CFR sections and IOM manual chapters the
|
||
family's FR paragraphs and the CPT manual cite, with FR provenance."""
|
||
if not family:
|
||
raise typer.BadParameter("pass --family (repeatable)")
|
||
keys = [normalize_key(f) for f in family]
|
||
store = _store()
|
||
if write:
|
||
# Ruling A13 (mirrors `stack pfs reaction`): resolve every family
|
||
# key up front and build every family's rows against a plain
|
||
# DuckDB read (I4) — build_guidance only reads pfs.cpt_* — never
|
||
# the write lock a notebook may be holding (#508-#514). The batch
|
||
# below only ever calls write_guidance and publishes.
|
||
con = _read()
|
||
try:
|
||
targets = _resolve_guidance_families(con, keys)
|
||
by_family: dict[str, list[Any]] = {}
|
||
rows: list[Any] = []
|
||
for key, fam in targets:
|
||
fam_rows = build_guidance(con, store, fam.codes, family=key)
|
||
by_family[key] = fam_rows
|
||
rows.extend(fam_rows)
|
||
finally:
|
||
con.close()
|
||
|
||
with _batch() as con:
|
||
ensure_tables(con)
|
||
for key, fam_rows in by_family.items():
|
||
write_guidance(con, key, fam_rows)
|
||
_publish()
|
||
_print_guidance(rows)
|
||
return
|
||
# No --write: read-only never needs the DuckDB single-writer lock a
|
||
# notebook may be holding (#508-#514) — read the replica (I4).
|
||
con = _read()
|
||
try:
|
||
refresh_from(con)
|
||
rows = []
|
||
for key in keys:
|
||
if key not in FAMILIES:
|
||
raise typer.BadParameter(
|
||
f"unknown family {key!r}; known: {', '.join(HAND_FAMILIES)}"
|
||
)
|
||
try:
|
||
rows.extend(read_guidance(con, key))
|
||
except Exception as exc:
|
||
if _missing_table(exc):
|
||
typer.echo(f"{key}: no derived guidance yet — run with --write")
|
||
continue
|
||
raise
|
||
_print_guidance(rows)
|
||
finally:
|
||
con.close()
|
||
|
||
|
||
def _print_reaction(rows: list[Any]) -> None:
|
||
for r in sorted(rows, key=lambda r: (r.family, r.period_kind, r.year, r.period)):
|
||
if r.period_kind == "docket":
|
||
stance = ""
|
||
if (
|
||
r.stance_support
|
||
or r.stance_oppose
|
||
or r.stance_modify
|
||
or r.stance_unclear
|
||
):
|
||
stance = (
|
||
f" stance support={r.stance_support} oppose={r.stance_oppose} "
|
||
f"modify={r.stance_modify} unclear={r.stance_unclear}"
|
||
)
|
||
typer.echo(
|
||
f"{r.family} {r.period} ({r.year}) {r.n_items}/{r.n_total} commenters{stance}"
|
||
)
|
||
else:
|
||
typer.echo(
|
||
f"{r.family} {r.period} CY{r.year} "
|
||
f"{r.n_items} comment/response pairs"
|
||
)
|
||
|
||
|
||
def _reaction_targets(
|
||
con: Any, family: list[str], code: list[str]
|
||
) -> list[tuple[str, tuple[str, ...], str | None]]:
|
||
"""(key, codes, display name) per target — *name* is the registered
|
||
family's own name for a ``--family`` target, ``None`` for a
|
||
``--code`` pseudo-family (``pfs.reaction.series`` falls back to the
|
||
key itself, which is already the right label for a bare code)."""
|
||
refresh_from(con)
|
||
targets: list[tuple[str, tuple[str, ...], str | None]] = []
|
||
for f in family:
|
||
key = normalize_key(f)
|
||
fam = FAMILIES.get(key)
|
||
if fam is None:
|
||
raise typer.BadParameter(
|
||
f"unknown family {key!r}; known: {', '.join(HAND_FAMILIES)}"
|
||
)
|
||
targets.append((key, fam.codes, fam.name))
|
||
for c in code:
|
||
key = c.upper()
|
||
targets.append((key, (key,), None))
|
||
return targets
|
||
|
||
|
||
def _stored(read: Any, con: Any, family: str) -> list[Any]:
|
||
"""``read(con, family)``; none when the table does not exist yet."""
|
||
try:
|
||
return read(con, family)
|
||
except Exception as exc: # noqa: BLE001 - only a missing table is tolerated
|
||
if is_missing_table_error(exc):
|
||
return []
|
||
raise
|
||
|
||
|
||
def _overlay_stances(
|
||
rows: list[Any], stances: list[Any], stored: list[Any], commenters: Counter
|
||
) -> list[Any]:
|
||
"""Put the classified counts of each docket's stored stances on its row,
|
||
so a reaction rerun never zeroes them. The event order and the stance
|
||
method record (``STANCE_METHOD_FIELDS``, R66a) are the ones the docket's
|
||
last ``stance`` run stored (*stored* are the family's stored reaction
|
||
rows); *commenters* counts ``pfs.commenter`` rows per docket (R62)."""
|
||
old = {r.period: r for r in stored}
|
||
out = []
|
||
for r in rows:
|
||
mine = [s for s in stances if s.docket == r.period]
|
||
if r.period_kind != "docket" or not mine:
|
||
out.append(r)
|
||
continue
|
||
prev = old.get(r.period)
|
||
order = list(json.loads(prev.events_json)) if prev else []
|
||
rolled = rollup(mine, base=r, order=order, n_commenters=commenters[r.period])
|
||
if prev:
|
||
rolled = replace(
|
||
rolled, **{f: getattr(prev, f) for f in STANCE_METHOD_FIELDS}
|
||
)
|
||
out.append(rolled)
|
||
return out
|
||
|
||
|
||
def _commenters_of(con: Any, family: str) -> list[Any]:
|
||
return read_commenters(con, family=family)
|
||
|
||
|
||
@app.command()
|
||
def reaction(
|
||
family: list[str] = typer.Option(
|
||
[], "--family", help="Family key (CCM, APCM, …); repeatable."
|
||
),
|
||
code: list[str] = typer.Option(
|
||
[],
|
||
"--code",
|
||
help="A single code as its own pseudo-family, keyed by the code; repeatable.",
|
||
),
|
||
stance_sample: int = typer.Option(
|
||
0,
|
||
"--stance-sample",
|
||
help="Classify the newest N commenters per docket into support/oppose/"
|
||
"modify/unclear via the self-hosted classifier (0 = skip; no stance "
|
||
"sample means no LLM call at all).",
|
||
),
|
||
write: bool = typer.Option(False, "--write"),
|
||
) -> None:
|
||
"""Public-reaction series per code family (pfs.code_reaction): comment
|
||
counts per docket (share of the docket's commenters mentioning the
|
||
family), an optional stance sample, and the FR Comment:/Response:
|
||
pair counts per rule as the pre-2017 proxy. Always recomputed live
|
||
from pgvector/bib — there is no persisted-table fallback to read
|
||
without --write, so --write only adds persisting the result."""
|
||
if not family and not code:
|
||
raise typer.BadParameter("pass --family or --code (repeatable)")
|
||
store = _store()
|
||
|
||
# Resolving family keys is a plain DuckDB read (I4) — never the
|
||
# write lock a notebook may be holding (#508-#514) — and it's the
|
||
# only DuckDB touch this command needs before writing.
|
||
con = _read()
|
||
try:
|
||
targets = _reaction_targets(con, family, code)
|
||
stored = {key: _stored(read_stances, con, key) for key, _, _ in targets}
|
||
old_rows = {key: _stored(read_reaction, con, key) for key in stored}
|
||
n_comm = {
|
||
key: Counter(c.docket for c in _stored(_commenters_of, con, key))
|
||
for key, v in stored.items()
|
||
if v
|
||
}
|
||
finally:
|
||
con.close()
|
||
if stance_sample > 0:
|
||
held = [k for k, v in stored.items() if v]
|
||
if held:
|
||
raise typer.BadParameter(
|
||
f"{', '.join(held)} already has stored stances (stack pfs stance); "
|
||
"a stance sample would be a second, weaker count"
|
||
)
|
||
|
||
# Ruling A13: the classifier and every family's series (docket
|
||
# counts, the stance sample — ~1-2s/sample — and FR pairs) all run
|
||
# against pgvector/bib, not DuckDB, so they run BEFORE any DuckDB
|
||
# batch is opened. The batch below only ever calls write_reaction
|
||
# and publishes — it never holds the single-writer lock (#508-#514)
|
||
# while the slow work happens.
|
||
classify = _classifier() if stance_sample > 0 else None
|
||
engine = _engine()
|
||
by_family: dict[str, list[Any]] = {}
|
||
rows: list[Any] = []
|
||
for key, codes, name in targets:
|
||
fam_rows = reaction_series(
|
||
engine,
|
||
store,
|
||
key,
|
||
codes,
|
||
classify=classify,
|
||
stance_sample_n=stance_sample,
|
||
name=name,
|
||
)
|
||
fam_rows = _overlay_stances(
|
||
fam_rows, stored[key], old_rows[key], n_comm.get(key, Counter())
|
||
)
|
||
by_family[key] = fam_rows
|
||
rows.extend(fam_rows)
|
||
|
||
if write:
|
||
with _batch() as con:
|
||
ensure_tables(con)
|
||
for key, fam_rows in by_family.items():
|
||
write_reaction(con, key, fam_rows)
|
||
_publish()
|
||
_print_reaction(rows)
|
||
|
||
|
||
def _print_exposures(rows: list[Any]) -> None:
|
||
for r in rows:
|
||
anchor = f"{r.item_key} ¶{r.p_id}" if r.item_key else "unanchored"
|
||
if r.correction:
|
||
anchor += " (correction)"
|
||
extra = ""
|
||
if r.kind == "revalued":
|
||
extra = f" {r.delta_pct:+.1%}"
|
||
elif r.kind in ("becomes-payable", "ends", "status-change"):
|
||
extra = f" {r.prior_status or '—'}→{r.status or '—'}"
|
||
fam = f" [{r.family}]" if r.family else ""
|
||
typer.echo(
|
||
f"{r.code} {r.effective_date.isoformat()} {r.kind:<17}"
|
||
f"{extra} band {r.rvu_band}{fam} ← {anchor}"
|
||
)
|
||
per_kind: dict[str, int] = {}
|
||
for r in rows:
|
||
per_kind[r.kind] = per_kind.get(r.kind, 0) + 1
|
||
detail = ", ".join(f"{k}={v}" for k, v in sorted(per_kind.items())) or "none"
|
||
anchored = sum(1 for r in rows if r.item_key)
|
||
typer.echo(
|
||
f"exposures: {len(rows)} rows, {len({r.code for r in rows})} codes, "
|
||
f"anchored {anchored}, per kind: {detail}"
|
||
)
|
||
|
||
|
||
@app.command()
|
||
def exposure(
|
||
code: list[str] = typer.Option([], "--code", help="Filter to a code; repeatable."),
|
||
family: list[str] = typer.Option(
|
||
[], "--family", help="Filter to a family key (CCM, APCM, …); repeatable."
|
||
),
|
||
controls: int = typer.Option(
|
||
0,
|
||
"--controls",
|
||
help="With one --code and --year: list up to N never-treated control codes "
|
||
"(same status + RVU band, no exposure within --window years).",
|
||
),
|
||
year: int = typer.Option(
|
||
0, "--year", help="The exposure year --controls refers to."
|
||
),
|
||
window: int = typer.Option(
|
||
2, "--window", help="±years a control must be quiet for."
|
||
),
|
||
write: bool = typer.Option(
|
||
False,
|
||
"--write",
|
||
help="Rebuild the whole calendar into pfs.code_exposure and republish.",
|
||
),
|
||
) -> None:
|
||
"""The exposure calendar (pfs.code_exposure, #693): when each code
|
||
became payable, was revalued, changed payable status, ended, or was
|
||
listed for telehealth — dated 1 January of the rule year (or the
|
||
correction notice's date), anchored to the FR paragraph from its
|
||
lineage, with the status and RVU-decile band a control set matches on.
|
||
|
||
--write always rebuilds every code (control sets need the whole
|
||
calendar); --code/--family only filter what is printed."""
|
||
codes = [c.upper() for c in code]
|
||
keys = [normalize_key(f) for f in family]
|
||
|
||
def _filter(rows: list[Any]) -> list[Any]:
|
||
if not codes and not keys:
|
||
return rows
|
||
return [r for r in rows if r.code in codes or r.family in keys]
|
||
|
||
if write:
|
||
# Ruling A13: build against a plain read (I4) — never the write
|
||
# lock a notebook may be holding (#508-#514); the batch below only
|
||
# writes and publishes.
|
||
store = _store()
|
||
con = _read()
|
||
try:
|
||
rows = build_exposures(con, store)
|
||
finally:
|
||
con.close()
|
||
with _batch() as con:
|
||
ensure_tables(con)
|
||
write_exposures(con, rows)
|
||
_publish()
|
||
_print_exposures(_filter(rows))
|
||
return
|
||
|
||
con = _read()
|
||
try:
|
||
if controls:
|
||
if len(codes) != 1 or not year:
|
||
raise typer.BadParameter(
|
||
"--controls needs exactly one --code and --year"
|
||
)
|
||
try:
|
||
ctrl = control_codes(con, codes[0], year, window=window, limit=controls)
|
||
except Exception as exc:
|
||
if _missing_table(exc):
|
||
typer.echo("no exposure calendar yet — run with --write")
|
||
return
|
||
raise
|
||
for c, fam, status, band in ctrl:
|
||
typer.echo(f"{c} {status} band {band} {fam}")
|
||
typer.echo(
|
||
f"controls for {codes[0]} @ {year} (±{window}y): {len(ctrl)} codes"
|
||
)
|
||
return
|
||
try:
|
||
rows = read_exposures(con)
|
||
except Exception as exc:
|
||
if _missing_table(exc):
|
||
typer.echo("no exposure calendar yet — run with --write")
|
||
return
|
||
raise
|
||
_print_exposures(_filter(rows))
|
||
finally:
|
||
con.close()
|
||
|
||
|
||
def _http() -> Any:
|
||
import httpx
|
||
|
||
return httpx.Client(headers={"User-Agent": "stack-pfs-utilization/1.0"})
|
||
|
||
|
||
def _print_series(rows: list[Any]) -> None:
|
||
for r in rows:
|
||
svc = f"{r.n_services:,.0f}" if r.n_services is not None else "—"
|
||
ben = f"{r.n_beneficiaries:,}" if r.n_beneficiaries is not None else "—"
|
||
prv = f"{r.n_providers:,}" if r.n_providers is not None else "—"
|
||
allowed = f"${r.avg_allowed:,.2f}" if r.avg_allowed is not None else "—"
|
||
typer.echo(
|
||
f"{r.hcpcs} {r.year} services {svc:>12} benes {ben:>10} "
|
||
f"providers {prv:>8} avg allowed {allowed}"
|
||
)
|
||
typer.echo(f"utilization: {len(rows)} (code, year) rows")
|
||
|
||
|
||
@app.command()
|
||
def utilization(
|
||
year: list[int] = typer.Option(
|
||
[], "--year", help="Calendar year(s) to ingest with --write; default all 2013–."
|
||
),
|
||
code: list[str] = typer.Option(
|
||
[], "--code", help="Print the per-year series for these codes (repeatable)."
|
||
),
|
||
write: bool = typer.Option(
|
||
False,
|
||
"--write",
|
||
help="Fetch from data.cms.gov, load pfs.utilization, cite, republish.",
|
||
),
|
||
offline: bool = typer.Option(
|
||
False,
|
||
"--offline",
|
||
help="With --write: load from the cached JSON instead of the API.",
|
||
),
|
||
) -> None:
|
||
"""Part B utilization per HCPCS (pfs.utilization, #694): the CMS
|
||
"Medicare Physician & Other Practitioners — by Geography and Service"
|
||
national totals, one row per (year, code, place of service), with
|
||
each year's dataset cited in bib and logged in cms.ingest_log.
|
||
|
||
--write pulls every year (or --year) through the Data API, caches the
|
||
raw pages under data/cms/utilization/, replaces that year's rows and
|
||
republishes the replica; --code prints series from the replica."""
|
||
codes = [c.upper() for c in code]
|
||
if write:
|
||
years = sorted(year) if year else sorted(UTILIZATION_YEARS)
|
||
unknown = [y for y in years if y not in UTILIZATION_YEARS]
|
||
if unknown:
|
||
raise typer.BadParameter(
|
||
f"no dataset for {unknown}; known years "
|
||
f"{min(UTILIZATION_YEARS)}–{max(UTILIZATION_YEARS)}"
|
||
)
|
||
# Ruling A13: all network + bib work first, then one short batch
|
||
# that only writes and publishes — never the write lock a
|
||
# notebook may be holding (#508-#514) while the API is paged.
|
||
store = _store()
|
||
staged: list[tuple[int, list[Any], str, str]] = []
|
||
client = None if offline else _http()
|
||
try:
|
||
for y in years:
|
||
cache = utilization_cache_path(y)
|
||
if offline:
|
||
records = load_utilization_cache(cache)
|
||
else:
|
||
records = fetch_utilization_year(client, y)
|
||
save_utilization_cache(cache, records)
|
||
key = cite_utilization(store, y)
|
||
rows = utilization_rows(
|
||
y, records, dataset_id=UTILIZATION_YEARS[y], item_key=key
|
||
)
|
||
staged.append((y, rows, key, str(cache)))
|
||
typer.echo(
|
||
f"{y}: {len(records)} records → {len(rows)} national rows ← {key}"
|
||
)
|
||
finally:
|
||
if client is not None:
|
||
client.close()
|
||
from cms.ingest_log import log_ingest
|
||
|
||
with _batch() as con:
|
||
for y, rows, key, cache in staged:
|
||
n = write_utilization(con, y, rows)
|
||
log_ingest(
|
||
con,
|
||
module="pfs.utilization",
|
||
table_name="pfs.utilization",
|
||
rows=n,
|
||
source_file=cache,
|
||
pincite_key=key,
|
||
)
|
||
_publish()
|
||
if not codes:
|
||
return
|
||
con = _read()
|
||
try:
|
||
if not codes:
|
||
raise typer.BadParameter("pass --code (repeatable) or --write")
|
||
try:
|
||
rows = read_utilization_series(con, codes)
|
||
except Exception as exc:
|
||
if _missing_table(exc):
|
||
typer.echo("no utilization table yet — run with --write")
|
||
return
|
||
raise
|
||
_print_series(rows)
|
||
finally:
|
||
con.close()
|
||
|
||
|
||
def _psps_names(con: Any) -> dict[str, str]:
|
||
"""Medicare provider specialty code -> description."""
|
||
try:
|
||
rows = con.execute(
|
||
"SELECT specialty_code, specialty_description "
|
||
"FROM terminology.cms_acceptable_provider_specialty_codes"
|
||
).fetchall()
|
||
except Exception as exc:
|
||
if _missing_table(exc):
|
||
return {}
|
||
raise
|
||
return {str(c): str(d or "") for c, d in rows}
|
||
|
||
|
||
def _family_codes(con: Any, key: str) -> list[str]:
|
||
refresh_from(con)
|
||
fam = FAMILIES.get(normalize_key(key))
|
||
if fam is None:
|
||
raise typer.BadParameter(
|
||
f"unknown family {key!r}; known: {', '.join(HAND_FAMILIES)}"
|
||
)
|
||
return sorted(fam.codes)
|
||
|
||
|
||
@app.command()
|
||
def psps(
|
||
family: str = typer.Option(..., "--family", help="Family key, e.g. HOME-VISITS."),
|
||
year: list[int] = typer.Option(
|
||
[], "--year", help="Calendar year(s); default every PSPS year."
|
||
),
|
||
write: bool = typer.Option(
|
||
False, "--write", help="Fetch data.cms.gov, load, cite, republish."
|
||
),
|
||
offline: bool = typer.Option(
|
||
False, "--offline", help="With --write: use the cached JSON, no network."
|
||
),
|
||
) -> None:
|
||
"""Medicare utilization by provider specialty for a family's codes
|
||
(pfs.utilization_specialty): the PSPS national totals per code, year
|
||
and specialty. --write pulls each (year, code) through the Data API,
|
||
caches it under data/cms/psps/ (never overwriting a cached file) and
|
||
republishes; without --write it prints what the replica holds."""
|
||
import time
|
||
|
||
years = sorted(year) if year else sorted(PSPS_YEARS)
|
||
unknown = [y for y in years if y not in PSPS_YEARS]
|
||
if unknown:
|
||
raise typer.BadParameter(
|
||
f"no PSPS dataset for {unknown}; known {min(PSPS_YEARS)}-{max(PSPS_YEARS)}"
|
||
)
|
||
con = _read()
|
||
try:
|
||
codes = _family_codes(con, family)
|
||
names = _psps_names(con)
|
||
if not write:
|
||
try:
|
||
rows = read_psps_rows(con, codes, years=years)
|
||
except Exception as exc:
|
||
if _missing_table(exc):
|
||
typer.echo("no PSPS table yet - run with --write")
|
||
return
|
||
raise
|
||
finally:
|
||
con.close()
|
||
if not write:
|
||
for r in rows:
|
||
if r.services is not None:
|
||
svc = f"{r.services:,.0f}"
|
||
elif r.services_reported is not None:
|
||
svc = f">={r.services_reported:,.0f}" # suppressed lines omitted
|
||
else:
|
||
svc = "-"
|
||
typer.echo(
|
||
f"{r.year} {r.hcpcs} {r.specialty_cd} {r.specialty:<32} {svc:>12}"
|
||
)
|
||
typer.echo(f"psps: {len(rows)} rows")
|
||
return
|
||
store = _store()
|
||
staged: list[tuple[int, str, list[Any], str, str]] = []
|
||
client = None if offline else _http_dossier()
|
||
gap = ""
|
||
try:
|
||
for y in years:
|
||
key = cite_psps(store, y)
|
||
for code in codes:
|
||
cache = psps_cache_path(y, code)
|
||
if offline:
|
||
cache = newest_psps_cache(cache)
|
||
if not cache.is_file():
|
||
typer.echo(f"{y} {code}: no cached file, skipped")
|
||
continue
|
||
records = load_psps_cache(cache)
|
||
else:
|
||
try:
|
||
records = fetch_psps(client, y, code)
|
||
except httpx.HTTPStatusError as exc:
|
||
# a 403/429/bot page stops this source: keep what is
|
||
# staged, note the gap, never work around it
|
||
gap = (
|
||
f"{y} {code}: HTTP {exc.response.status_code}; "
|
||
"stopped fetching PSPS"
|
||
)
|
||
break
|
||
if records:
|
||
cache = save_psps_cache(cache, records)
|
||
time.sleep(0.2)
|
||
if not records:
|
||
# an empty answer must not delete rows nor log a source file
|
||
typer.echo(f"{y} {code}: no lines, kept existing rows")
|
||
continue
|
||
rows = psps_rows(
|
||
y,
|
||
code,
|
||
records,
|
||
dataset_id=PSPS_YEARS[y],
|
||
item_key=key,
|
||
names=names,
|
||
)
|
||
staged.append((y, code, rows, key, str(cache)))
|
||
typer.echo(
|
||
f"{y} {code}: {len(records)} lines -> {len(rows)} specialties"
|
||
)
|
||
if gap:
|
||
break
|
||
finally:
|
||
if client is not None:
|
||
client.close()
|
||
if gap:
|
||
typer.echo(f"gap: {gap}")
|
||
from cms.ingest_log import log_ingest
|
||
|
||
with _batch() as con:
|
||
for y, code, rows, key, cache in staged:
|
||
n = write_psps_rows(con, y, code, rows)
|
||
log_ingest(
|
||
con,
|
||
module="pfs.psps",
|
||
table_name="pfs.utilization_specialty",
|
||
rows=n,
|
||
source_file=cache,
|
||
pincite_key=key,
|
||
)
|
||
_publish()
|
||
typer.echo(f"psps: {sum(len(r) for _, _, r, _, _ in staged)} rows written")
|
||
|
||
|
||
def _http_dossier() -> Any:
|
||
return httpx.Client(headers={"User-Agent": PSPS_USER_AGENT})
|
||
|
||
|
||
def _seat_root() -> Path:
|
||
"""``data/ama/rosters``: the saved archive captures of AMA rosters."""
|
||
from conf import path
|
||
|
||
return path("db.aco").parent / "ama" / "rosters"
|
||
|
||
|
||
def _gap_status(exc: Exception) -> str:
|
||
if isinstance(exc, httpx.HTTPStatusError):
|
||
return str(exc.response.status_code)
|
||
return type(exc).__name__
|
||
|
||
|
||
@app.command("seats")
|
||
def seats(
|
||
body: str = typer.Option(..., "--body", help="RUC or CPT_PANEL."),
|
||
url: str = typer.Option(..., "--url", help="The AMA roster page (live URL)."),
|
||
year_from: int = typer.Option(2005, "--from", help="First year to capture."),
|
||
year_to: int = typer.Option(2026, "--to", help="Last year to capture."),
|
||
limit: int = typer.Option(0, "--limit", help="Cap the snapshots read (0 = all)."),
|
||
dry_run: bool = typer.Option(
|
||
False, "--dry-run", help="List the archive's captures; fetch and parse nothing."
|
||
),
|
||
cross_check: bool = typer.Option(
|
||
False,
|
||
"--cross-check",
|
||
help="Also ask the pinned local model for the organizations and log "
|
||
"where it disagrees with the parse (resumable ledger).",
|
||
),
|
||
write: bool = typer.Option(False, "--write"),
|
||
) -> None:
|
||
"""Seats on the RUC or the CPT Editorial Panel by year
|
||
(pfs.governance_seat), parsed deterministically from the Internet
|
||
Archive's copies of the AMA roster page (R88). Each capture is saved
|
||
under data/ama/rosters and cited as a bib Download. A year with no
|
||
capture, a refusal (403/429, bot page) or an unrecognised layout is a
|
||
gap line (and, with --write, an ingest_log row with rows=0)."""
|
||
import time
|
||
|
||
if body not in seat_lib.BODIES:
|
||
raise typer.BadParameter(f"--body must be one of {', '.join(seat_lib.BODIES)}")
|
||
if year_from > year_to:
|
||
raise typer.BadParameter("--from is after --to")
|
||
years = list(range(year_from, year_to + 1))
|
||
gaps: dict[int, str] = {}
|
||
refusal = ""
|
||
staged: list[tuple[int, str, str, Path, list[Any]]] = []
|
||
texts: dict[int, str] = {}
|
||
year_notes: dict[int, list[str]] = {}
|
||
client = _http_dossier()
|
||
try:
|
||
snaps = seat_lib.cdx_snapshots(client, url, years, gaps=gaps)
|
||
if gaps:
|
||
refusal = next(iter(gaps.values()))
|
||
found = {y for y, _ in snaps}
|
||
for y in years:
|
||
if y not in found and y not in gaps:
|
||
gaps[y] = "no archive capture"
|
||
if limit:
|
||
snaps = snaps[:limit]
|
||
if dry_run:
|
||
for y, ts in snaps:
|
||
typer.echo(f"{y} {ts} {seat_lib.snapshot_url(ts, url)}")
|
||
else:
|
||
store = _store()
|
||
root = _seat_root()
|
||
for n, (y, ts) in enumerate(snaps):
|
||
path = seat_lib.snapshot_path(root, body, y, ts)
|
||
if path.is_file(): # resumable: a saved capture is never refetched
|
||
raw = path.read_bytes()
|
||
else:
|
||
try:
|
||
raw = seat_lib.fetch_snapshot(
|
||
client, seat_lib.snapshot_url(ts, url)
|
||
)
|
||
except httpx.HTTPError as exc:
|
||
refusal = _gap_status(exc)
|
||
gaps[y] = refusal
|
||
for later, _ts in snaps[n + 1 :]:
|
||
gaps[later] = f"not tried after {refusal}"
|
||
break
|
||
time.sleep(1.0)
|
||
text = seat_lib.html_text(raw)
|
||
if seat_lib.is_refusal(text):
|
||
refusal = "bot page"
|
||
gaps[y] = refusal
|
||
for later, _ts in snaps[n + 1 :]:
|
||
gaps[later] = f"not tried after {refusal}"
|
||
break
|
||
path = seat_lib.save_snapshot(path, raw)
|
||
key = seat_lib.cite(store, body, y, url, ts, path)
|
||
try:
|
||
parsed, notes = (
|
||
seat_lib.parse_ruc(text)
|
||
if body == "RUC"
|
||
else seat_lib.parse_cpt(text)
|
||
)
|
||
except seat_lib.ParseError:
|
||
gaps[y] = "layout not recognised"
|
||
continue
|
||
for note in notes:
|
||
typer.echo(f"note: {body} {y}: {note}")
|
||
year_notes[y] = notes
|
||
texts[y] = text
|
||
staged.append(
|
||
(
|
||
y,
|
||
ts,
|
||
key,
|
||
path,
|
||
seat_lib.to_rows(
|
||
body, y, parsed, source_url=url, item_key=key, snapshot=ts
|
||
),
|
||
)
|
||
)
|
||
finally:
|
||
client.close()
|
||
if refusal:
|
||
typer.echo(f"gap: {body} roster not captured ({refusal})")
|
||
for y in sorted(gaps):
|
||
typer.echo(f"gap: {body} {y} not captured ({gaps[y]})")
|
||
if dry_run:
|
||
typer.echo(f"seats: {len(snaps)} snapshots; dry run")
|
||
return
|
||
rows: list[Any] = []
|
||
for y, ts, _key, _path, got in staged:
|
||
rows += got
|
||
kinds = Counter(r.seat_type for r in got)
|
||
typer.echo(
|
||
f"{y} {ts}: {sum(r.n_seats for r in got)} seats "
|
||
+ ", ".join(f"{k} {v}" for k, v in sorted(kinds.items()))
|
||
)
|
||
typer.echo(f"seats: {len(rows)} rows from {len(staged)} snapshots")
|
||
if cross_check and staged:
|
||
_seats_cross_check(body, staged, texts)
|
||
if write and (rows or gaps):
|
||
from cms.ingest_log import ensure_table, log_ingest
|
||
|
||
def _log(bcon: Any, *, source: str, rule: str = "", **kw: Any) -> None:
|
||
"""One ingest_log row, replacing any earlier row for the same
|
||
source and kind so a rerun does not duplicate it."""
|
||
bcon.execute(
|
||
"DELETE FROM cms.ingest_log WHERE module = 'pfs.seats' "
|
||
"AND source_file = ? AND rule_id = ?",
|
||
[source, rule],
|
||
)
|
||
log_ingest(
|
||
bcon,
|
||
module="pfs.seats",
|
||
table_name=seat_lib.TABLE,
|
||
source_file=source,
|
||
rule_id=rule,
|
||
**kw,
|
||
)
|
||
|
||
with _batch() as bcon:
|
||
ensure_table(bcon)
|
||
for y in sorted({*gaps, *(st[0] for st in staged)}):
|
||
# a rerun replaces this year's gap and note rows (also the
|
||
# earlier unkinded gap rows) instead of adding to them
|
||
bcon.execute(
|
||
"DELETE FROM cms.ingest_log WHERE module = 'pfs.seats' "
|
||
"AND starts_with(source_file, ?)",
|
||
[f"{url}#{y} "],
|
||
)
|
||
if rows:
|
||
seat_lib.write_seats(bcon, body, rows)
|
||
for y, _ts, ikey, src, got in staged:
|
||
_log(bcon, source=str(src), rows=len(got), pincite_key=ikey)
|
||
for note in year_notes.get(y, []):
|
||
# the page disagrees with itself: kept for the dossier
|
||
_log(
|
||
bcon,
|
||
source=f"{url}#{y} note: {note}",
|
||
rule="note",
|
||
rows=len(got),
|
||
pincite_key=ikey,
|
||
)
|
||
for y in sorted(gaps):
|
||
_log(bcon, source=f"{url}#{y} ({gaps[y]})", rule="gap", rows=0)
|
||
_publish()
|
||
typer.echo(f"wrote {len(rows)} seats")
|
||
|
||
|
||
def _seats_cross_check(body: str, staged: Sequence[Any], texts: dict[int, str]) -> None:
|
||
"""Ask the pinned model for each page's organizations and log where it
|
||
disagrees with the parsed rows. The parse is the record; this only flags
|
||
pages worth a look. Resumable through the run ledger."""
|
||
from pfs.commenters import specialty_vocab
|
||
from pfs.seats import _key # the crosswalk's text normaliser
|
||
|
||
con = _read()
|
||
try:
|
||
specialties = specialty_vocab(con)
|
||
finally:
|
||
con.close()
|
||
ask = _asker()
|
||
version = f"{seat_lib.SEAT_PROMPT_VERSION}-xcheck"
|
||
log = RunLog(ledger_path("seats", body))
|
||
parsed = {
|
||
y: sorted({_key(r.organization) for r in got}) for y, _t, _k, _p, got in staged
|
||
}
|
||
|
||
def unit(year: int) -> Any:
|
||
def run() -> dict | None:
|
||
try:
|
||
found = seat_lib.extract_seats(
|
||
texts[year], ask, specialties=specialties
|
||
)
|
||
except seat_lib.ExtractionFailed:
|
||
return None
|
||
model = sorted({_key(it["organization"]) for it in found})
|
||
return {
|
||
"model_only": [m for m in model if m not in parsed[year]],
|
||
"parsed_only": [p for p in parsed[year] if p not in model],
|
||
}
|
||
|
||
return run
|
||
|
||
units = [
|
||
(f"{body}:{y}:{ts}:xcheck", input_hash(texts[y]), unit(y))
|
||
for y, ts, *_ in staged
|
||
]
|
||
run_units(
|
||
units,
|
||
log,
|
||
model=ask.model,
|
||
version=version,
|
||
retry_failed=True,
|
||
progress=lambda u, status: typer.echo(f"{u} {status}"),
|
||
)
|
||
for y, ts, *_ in staged:
|
||
entry = log.get(
|
||
f"{body}:{y}:{ts}:xcheck", input_hash(texts[y]), ask.model, version
|
||
)
|
||
if entry is None or entry.output is None:
|
||
typer.echo(f"cross-check {y}: model gave no valid reply")
|
||
continue
|
||
out = entry.output
|
||
if not out["model_only"] and not out["parsed_only"]:
|
||
typer.echo(f"cross-check {y}: model and parse agree")
|
||
continue
|
||
typer.echo(
|
||
f"cross-check {y}: model-only {len(out['model_only'])}: "
|
||
f"{'; '.join(out['model_only'])} | parsed-only {len(out['parsed_only'])}: "
|
||
f"{'; '.join(out['parsed_only'])}"
|
||
)
|
||
|
||
|
||
@app.command()
|
||
def impacts(
|
||
scan: bool = typer.Option(
|
||
False,
|
||
"--scan",
|
||
help="Every CMS 508 impact file in bib / data/cms/pfs_final and every "
|
||
"PFS rule whose FR text prints the table.",
|
||
),
|
||
path: Path = typer.Option(None, "--path", help="One impact file or FR text."),
|
||
year: int = typer.Option(0, "--year", help="Rule year, with --path."),
|
||
vintage: str = typer.Option("final", "--vintage", help="final | proposed"),
|
||
write: bool = typer.Option(False, "--write"),
|
||
) -> None:
|
||
"""CMS's estimated impact on total allowed charges by specialty
|
||
(pfs.specialty_impact): the 508 text files CMS posts with a rule and
|
||
the table printed in the 2012-2019 Federal Register texts."""
|
||
from conf import ROOT
|
||
|
||
if scan == (path is not None):
|
||
raise typer.BadParameter("pass exactly one of --scan or --path")
|
||
if vintage not in ("final", "proposed"):
|
||
raise typer.BadParameter("--vintage is final or proposed")
|
||
groups: list[tuple[int, str, str, list[Any], str, str]] = []
|
||
if scan:
|
||
store = _store()
|
||
sources = discover_508(store, ROOT / "data/cms/pfs_final")
|
||
sources += discover_fr(store)
|
||
else:
|
||
if not year:
|
||
raise typer.BadParameter("--path needs --year")
|
||
kind = "508" if path.name.endswith("_508.txt") else "fr"
|
||
from pfs.stakes import ImpactSource
|
||
|
||
sources = [ImpactSource(path, year, vintage, "", kind)]
|
||
for src in sources:
|
||
if src.kind == "508":
|
||
rows = parse_impact_508(
|
||
src.path,
|
||
rule_year=src.rule_year,
|
||
vintage=src.vintage,
|
||
item_key=src.item_key,
|
||
)
|
||
else:
|
||
rows = parse_impact_fr_text(
|
||
src.path.read_text(errors="replace"),
|
||
rule_year=src.rule_year,
|
||
vintage=src.vintage,
|
||
item_key=src.item_key,
|
||
)
|
||
groups.append(
|
||
(src.rule_year, src.vintage, src.kind, rows, src.item_key, str(src.path))
|
||
)
|
||
typer.echo(
|
||
f"CY{src.rule_year} {src.vintage} ({src.kind}): {len(rows)} rows "
|
||
f"<- {src.path.name}"
|
||
)
|
||
if not write:
|
||
return
|
||
from cms.ingest_log import log_ingest
|
||
|
||
with _batch() as con:
|
||
for ry, vin, kind, rows, key, file in groups:
|
||
n = write_impacts(con, ry, vin, rows, source=kind)
|
||
log_ingest(
|
||
con,
|
||
module="pfs.stakes",
|
||
table_name="pfs.specialty_impact",
|
||
rows=n,
|
||
source_file=file,
|
||
pincite_key=key,
|
||
)
|
||
_publish()
|
||
|
||
|
||
@app.command()
|
||
def review(code: str = typer.Option("", "--code")) -> None:
|
||
"""Element lines the classifier could not place."""
|
||
con = _read()
|
||
rows = read_reviews(con, code.upper()) if code else read_all_reviews(con)
|
||
for r in rows:
|
||
src = f" ({r.source})" if r.source else ""
|
||
typer.echo(
|
||
f"{r.code} {r.item_key} ¶{r.p_id}{src} [{r.proposed_value or '?'}] "
|
||
f"{r.text[:120]}"
|
||
)
|
||
|
||
|
||
def _print_cpt_counts(counts: dict[int, dict[str, int]]) -> None:
|
||
for year in sorted(counts):
|
||
c = counts[year]
|
||
detail = ", ".join(f"{k}={v}" for k, v in c.items())
|
||
typer.echo(f"{year}: {detail}")
|
||
|
||
|
||
@app.command("cpt-ingest")
|
||
def cpt_ingest_cmd(
|
||
edition: list[int] = typer.Option(
|
||
[], "--edition", help="CPT edition year to ingest (repeatable)."
|
||
),
|
||
all_editions: bool = typer.Option(
|
||
False, "--all", help="Every CPT EPUB edition found in bib (source:ama)."
|
||
),
|
||
dry_run: bool = typer.Option(
|
||
False, "--dry-run", help="Parse and report counts; write nothing."
|
||
),
|
||
) -> None:
|
||
"""Parse CPT EPUB editions into pfs.cpt_section/cpt_code/cpt_instruction/
|
||
cpt_reference/cpt_crosswalk/cpt_list (docs/superpowers/specs/
|
||
2026-09-09-cpt-canonical-schema-design.md §3)."""
|
||
if not edition and not all_editions:
|
||
raise typer.BadParameter("pass --edition YEAR (repeatable) or --all")
|
||
years = None if all_editions else sorted(set(edition))
|
||
store = _store()
|
||
if dry_run:
|
||
# A preview never needs the DuckDB single-writer lock a notebook
|
||
# may be holding (#508-#514) — parsing alone answers a dry run,
|
||
# so no connection is opened at all (I4).
|
||
counts = cpt_ingest(store, None, years=years, dry_run=True)
|
||
_print_cpt_counts(counts)
|
||
return
|
||
with _batch() as con:
|
||
# write_cpt_edition (via cpt_ingest) already calls ensure_tables
|
||
# itself — no need to duplicate that here.
|
||
counts = cpt_ingest(store, con, years=years, dry_run=False)
|
||
_print_cpt_counts(counts)
|
||
_publish()
|
||
|
||
|
||
@app.command()
|
||
def rounds(
|
||
family: str = typer.Option(..., "--family", help="Curated family key."),
|
||
tables_only: bool = typer.Option(
|
||
False,
|
||
"--tables-only",
|
||
help="No model calls (as --no-llm); the finished ledger is replayed.",
|
||
),
|
||
no_llm: bool = typer.Option(
|
||
False, "--no-llm", help="Make no model calls; the finished ledger is replayed."
|
||
),
|
||
limit: int = typer.Option(0, "--limit", help="Cap new model calls (0 = all)."),
|
||
dry_run: bool = typer.Option(
|
||
False, "--dry-run", help="Count candidates; no model calls, no writes."
|
||
),
|
||
retry_failed: bool = typer.Option(
|
||
False, "--retry-failed", help="Re-run ledger entries that failed."
|
||
),
|
||
write: bool = typer.Option(False, "--write"),
|
||
) -> None:
|
||
"""Valuation rounds per code and rule (pfs.valuation_round): RUC
|
||
recommendation vs CMS proposal/final, from FR tables, regex over FR
|
||
paragraphs, and the local model through a resumable ledger."""
|
||
fam = curated_family(normalize_key(family))
|
||
if fam is None:
|
||
raise typer.BadParameter(f"unknown curated family {family!r}")
|
||
store = _store()
|
||
if dry_run:
|
||
paras, table = rounds_inventory(store, fam)
|
||
typer.echo(
|
||
f"{fam.key}: {len(paras)} candidate paragraphs, "
|
||
f"{len(table)} table rows; no model calls made"
|
||
)
|
||
return
|
||
replay = tables_only or no_llm
|
||
ask = None if replay else _asker()
|
||
log = RunLog(ledger_path("rounds", fam.key))
|
||
con = _read()
|
||
try:
|
||
rows, counts = family_rounds(
|
||
con,
|
||
store,
|
||
fam,
|
||
ask=ask,
|
||
log=log,
|
||
limit=limit,
|
||
tables_only=tables_only,
|
||
retry_failed=retry_failed,
|
||
replay_model=_model_name() if replay else None,
|
||
)
|
||
finally:
|
||
con.close()
|
||
by_year: dict[tuple[int, str], int] = {}
|
||
for r in rows:
|
||
by_year[(r.rule_year, r.rule_kind)] = (
|
||
by_year.get((r.rule_year, r.rule_kind), 0) + 1
|
||
)
|
||
for (year, kind), n in sorted(by_year.items()):
|
||
typer.echo(f"CY{year} {kind}: {n} rounds")
|
||
typer.echo(
|
||
f"rounds: {len(rows)}; done {counts['done']}, skipped {counts['skipped']}, "
|
||
f"failed {counts['failed']}, failed_pending {counts['failed_pending']}"
|
||
)
|
||
if write:
|
||
with _batch() as con:
|
||
ensure_tables(con)
|
||
n = write_rounds(con, fam.key, rows)
|
||
typer.echo(f"wrote {n} rounds")
|
||
_publish()
|
||
|
||
|
||
def _origin_books(store: Any, keys: list[str]) -> list[tuple[str, str, Path]]:
|
||
"""The EPUBs to read. A book asked for with ``--book`` must exist; the
|
||
default one that is missing is skipped with a warning (an FR-only run)."""
|
||
books = []
|
||
for key in keys or [DEFAULT_BOOK]:
|
||
found = resolve_book(store, key)
|
||
if found is not None:
|
||
books.append((key, found[0], found[1]))
|
||
elif keys:
|
||
raise typer.BadParameter(f"no EPUB attachment for bib item {key}")
|
||
else:
|
||
typer.echo(f"warning: no EPUB attachment for {key}; skipping the book")
|
||
return books
|
||
|
||
|
||
@app.command()
|
||
def origin(
|
||
family: str = typer.Option(..., "--family", help="Curated family key."),
|
||
book: list[str] = typer.Option(
|
||
[],
|
||
"--book",
|
||
help=f"Bib item of a licensed EPUB to read (default {DEFAULT_BOOK}, "
|
||
"CPT Changes 2023); repeatable.",
|
||
),
|
||
limit: int = typer.Option(0, "--limit", help="Cap new model calls (0 = all)."),
|
||
dry_run: bool = typer.Option(
|
||
False, "--dry-run", help="Count candidates; no model calls, no writes."
|
||
),
|
||
no_llm: bool = typer.Option(
|
||
False, "--no-llm", help="Make no model calls; the finished ledger is replayed."
|
||
),
|
||
retry_failed: bool = typer.Option(
|
||
False, "--retry-failed", help="Re-run ledger entries that failed."
|
||
),
|
||
write: bool = typer.Option(False, "--write"),
|
||
) -> None:
|
||
"""Origin and formation rows per code (pfs.code_origin): the statute,
|
||
regulation, program, demonstration, CPT Editorial Panel action, G-code
|
||
creation or coverage rule behind them, from Federal Register
|
||
paragraphs, the licensed CPT Changes book (paraphrase + page only) and
|
||
the guidance rows, through a resumable local-model ledger."""
|
||
fam = curated_family(normalize_key(family))
|
||
if fam is None:
|
||
raise typer.BadParameter(f"unknown curated family {family!r}")
|
||
store = _store()
|
||
books = _origin_books(store, book)
|
||
if dry_run:
|
||
paras, pages = origin_inventory(store, fam, books)
|
||
typer.echo(f"{fam.key}: {len(paras)} candidate Federal Register paragraphs")
|
||
for key, labels in pages.items():
|
||
typer.echo(f"{key}: {len(labels)} pages naming a code")
|
||
typer.echo("no model calls made")
|
||
return
|
||
ask = None
|
||
if not no_llm:
|
||
try:
|
||
ask = _asker()
|
||
except RuntimeError as exc:
|
||
typer.echo(f"no model host available ({exc}); replaying the ledger")
|
||
log = RunLog(ledger_path("origin", fam.key))
|
||
con = _read()
|
||
try:
|
||
rows, counts = family_origin(
|
||
con,
|
||
store,
|
||
fam,
|
||
ask=ask,
|
||
log=log,
|
||
books=books,
|
||
limit=limit,
|
||
retry_failed=retry_failed,
|
||
replay_model=_model_name() if ask is None else None,
|
||
)
|
||
finally:
|
||
con.close()
|
||
for field in ("kind", "cite_kind"):
|
||
tally: dict[str, int] = {}
|
||
for r in rows:
|
||
tally[getattr(r, field)] = tally.get(getattr(r, field), 0) + 1
|
||
for name, n in sorted(tally.items()):
|
||
typer.echo(f"{field} {name}: {n}")
|
||
typer.echo(
|
||
f"origins: {len(rows)}; done {counts['done']}, skipped {counts['skipped']}, "
|
||
f"failed {counts['failed']}, failed_pending {counts['failed_pending']}"
|
||
)
|
||
if write:
|
||
with _batch() as con:
|
||
ensure_tables(con)
|
||
n = write_origin(con, fam.key, rows)
|
||
typer.echo(f"wrote {n} origins")
|
||
_publish()
|
||
|
||
|
||
@app.command("guideline-facts")
|
||
def guideline_facts_cmd(
|
||
family: str = typer.Option(..., "--family", help="Curated family key."),
|
||
limit: int = typer.Option(0, "--limit", help="Cap new model calls (0 = all)."),
|
||
no_llm: bool = typer.Option(
|
||
False, "--no-llm", help="Make no model calls; the finished ledger is replayed."
|
||
),
|
||
retry_failed: bool = typer.Option(
|
||
False, "--retry-failed", help="Re-run ledger entries that failed."
|
||
),
|
||
write: bool = typer.Option(False, "--write"),
|
||
) -> None:
|
||
"""Guideline facts from the licensed CPT books (pfs.code_guideline_fact):
|
||
selection basis, MDM level, time threshold, add-on and deletion per code
|
||
and edition, scope points and the Panel action as paraphrases, each cited
|
||
by (edition, page). Book text is never stored."""
|
||
fam = curated_family(normalize_key(family))
|
||
if fam is None:
|
||
raise typer.BadParameter(f"unknown curated family {family!r}")
|
||
store = _store()
|
||
ask = None
|
||
if not no_llm:
|
||
try:
|
||
ask = _asker()
|
||
except RuntimeError as exc:
|
||
typer.echo(f"no model host available ({exc}); replaying the ledger")
|
||
counts: dict[str, int] = {}
|
||
con = _read()
|
||
try:
|
||
rows = guideline_facts(
|
||
con,
|
||
store,
|
||
fam,
|
||
ask=ask,
|
||
log=RunLog(ledger_path("guideline", fam.key)),
|
||
limit=limit,
|
||
retry_failed=retry_failed,
|
||
replay_model=_model_name() if ask is None else None,
|
||
counts=counts,
|
||
)
|
||
finally:
|
||
con.close()
|
||
tally: dict[tuple[int, str], int] = {}
|
||
for r in rows:
|
||
tally[(r.edition_year, r.fact)] = tally.get((r.edition_year, r.fact), 0) + 1
|
||
for (year, fact), n in sorted(tally.items()):
|
||
typer.echo(f"{year} {fact}: {n}")
|
||
typer.echo(
|
||
f"facts: {len(rows)}; done {counts.get('done', 0)}, skipped "
|
||
f"{counts.get('skipped', 0)}, failed {counts.get('failed', 0)}, "
|
||
f"failed_pending {counts.get('failed_pending', 0)}, "
|
||
f"cited by section (no page) {counts.get('unpaged', 0)}"
|
||
)
|
||
if write:
|
||
with _batch() as bcon:
|
||
ensure_tables(bcon)
|
||
n = write_guideline_facts(bcon, fam.key, rows)
|
||
typer.echo(f"wrote {n} facts")
|
||
_publish()
|
||
|
||
|
||
@app.command("cms-summaries")
|
||
def cms_summaries(
|
||
family: str = typer.Option(..., "--family", help="Curated family key."),
|
||
no_llm: bool = typer.Option(
|
||
False, "--no-llm", help="Make no model calls; the finished ledger is replayed."
|
||
),
|
||
limit: int = typer.Option(0, "--limit", help="Cap new model calls (0 = all)."),
|
||
retry_failed: bool = typer.Option(
|
||
False, "--retry-failed", help="Re-run ledger entries that failed."
|
||
),
|
||
write: bool = typer.Option(False, "--write"),
|
||
) -> None:
|
||
"""CMS's own Comment:/Response: summaries (pfs.cms_comment_summary):
|
||
CMS's quantifier, the stance of the summarized comments and CMS's
|
||
response action, per pair a rule prints. Kept apart from our own
|
||
commenter counts."""
|
||
fam = curated_family(normalize_key(family))
|
||
if fam is None:
|
||
raise typer.BadParameter(f"unknown curated family {family!r}")
|
||
store = _store()
|
||
model = _model_name()
|
||
counts: dict[str, int] = {}
|
||
rows = cms_summaries_rows(
|
||
store,
|
||
fam,
|
||
classify=None if no_llm else _classifier(pin=True),
|
||
model=model,
|
||
log=RunLog(ledger_path("cms-summaries", fam.key)),
|
||
limit=limit,
|
||
retry_failed=retry_failed,
|
||
replay_model=model if no_llm else None,
|
||
counts=counts,
|
||
)
|
||
by_year: dict[tuple[int, str], int] = {}
|
||
for r in rows:
|
||
by_year[(r.rule_year, r.rule_kind)] = (
|
||
by_year.get((r.rule_year, r.rule_kind), 0) + 1
|
||
)
|
||
for (year, kind), n in sorted(by_year.items()):
|
||
typer.echo(f"CY{year} {kind}: {n} summaries")
|
||
typer.echo(
|
||
f"summaries: {len(rows)}; done {counts.get('done', 0)}, "
|
||
f"skipped {counts.get('skipped', 0)}, failed {counts.get('failed', 0)}, "
|
||
f"failed_pending {counts.get('failed_pending', 0)}"
|
||
)
|
||
if write:
|
||
with _batch() as con:
|
||
ensure_tables(con)
|
||
n = write_cms_summaries(con, fam.key, rows)
|
||
typer.echo(f"wrote {n} summaries")
|
||
_publish()
|
||
|
||
|
||
@app.command("commenters")
|
||
def commenters(
|
||
family: str = typer.Option(..., "--family", help="Curated family key."),
|
||
docket: str = typer.Option(..., "--docket", help="regulations.gov docket id."),
|
||
limit: int = typer.Option(0, "--limit", help="Cap new model calls (0 = all)."),
|
||
retry_failed: bool = typer.Option(
|
||
False, "--retry-failed", help="Re-run ledger entries that failed."
|
||
),
|
||
dry_run: bool = typer.Option(
|
||
False,
|
||
"--dry-run",
|
||
help="Count the family's comments and regulations.gov categories; no model.",
|
||
),
|
||
write: bool = typer.Option(False, "--write"),
|
||
) -> None:
|
||
"""Who wrote each family comment (pfs.commenter): organization, type,
|
||
specialty, form-letter campaign and the submitter name as the
|
||
regulations.gov record posts it (official public record)."""
|
||
fam = curated_family(normalize_key(family))
|
||
if fam is None:
|
||
raise typer.BadParameter(f"unknown curated family {family!r}")
|
||
engine = _engine()
|
||
if dry_run:
|
||
items = family_comment_items(
|
||
engine, fam.codes, fam.key, fam.pair_phrases, docket=docket
|
||
)
|
||
mirror = _mirror()
|
||
_, regs = regs_for(_store(), mirror, docket, [k for k, _ in items])
|
||
cats = Counter(r.category for r in regs.values())
|
||
if len(regs) < len(items):
|
||
typer.echo(f" unreadable mirror JSON: {len(items) - len(regs)}")
|
||
typer.echo(f"{docket}: {len(items)} items; no model calls made")
|
||
for cat, n in sorted(cats.items(), key=lambda kv: (-kv[1], kv[0])):
|
||
typer.echo(f" category {cat or '(none)'}: {n}")
|
||
return
|
||
con = _read()
|
||
try:
|
||
rows, counts = identify(
|
||
engine,
|
||
_store(),
|
||
con,
|
||
fam,
|
||
docket=docket,
|
||
mirror=_mirror(),
|
||
ask=_asker(),
|
||
log=RunLog(ledger_path("commenters", fam.key)),
|
||
limit=limit,
|
||
retry_failed=retry_failed,
|
||
progress=lambda unit, status: typer.echo(f"{unit} {status}"),
|
||
)
|
||
finally:
|
||
con.close()
|
||
types = Counter(r.org_type for r in rows)
|
||
for t, n in sorted(types.items()):
|
||
typer.echo(f"{t}: {n}")
|
||
typer.echo(
|
||
f"commenters: {len(rows)}; done {counts['done']}, skipped {counts['skipped']}, "
|
||
f"failed {counts['failed']}, failed_pending {counts['failed_pending']}, "
|
||
f"pending {counts['pending']}, blocked {counts['blocked']}"
|
||
)
|
||
if write:
|
||
if counts["pending"]:
|
||
raise typer.BadParameter(
|
||
f"{counts['pending']} comments not yet identified; rerun to resume"
|
||
)
|
||
with _batch() as bcon:
|
||
n = write_commenters(bcon, rows, docket=docket, family=fam.key)
|
||
typer.echo(f"wrote {n} commenters")
|
||
_publish()
|
||
|
||
|
||
@app.command("stance")
|
||
def stance(
|
||
family: str = typer.Option(..., "--family", help="Curated family key."),
|
||
docket: str = typer.Option(..., "--docket", help="regulations.gov docket id."),
|
||
limit: int = typer.Option(0, "--limit", help="Cap new model calls (0 = all)."),
|
||
retry_failed: bool = typer.Option(
|
||
False, "--retry-failed", help="Re-run ledger entries that failed."
|
||
),
|
||
dry_run: bool = typer.Option(
|
||
False, "--dry-run", help="Count commenters and events; no model."
|
||
),
|
||
no_llm: bool = typer.Option(
|
||
False, "--no-llm", help="Make no model calls; the finished ledger is replayed."
|
||
),
|
||
method: str = typer.Option(
|
||
DEFAULT_METHOD, "--method", help="multi (one call per comment) or per-event."
|
||
),
|
||
model: str = typer.Option(
|
||
DEFAULT_MODEL, "--model", help="instruct or large (the pinned model)."
|
||
),
|
||
golden: Path = typer.Option(
|
||
GOLDEN_PATH,
|
||
"--golden",
|
||
help="Tuning (in-sample) golden set --write records as in_sample_record.",
|
||
),
|
||
heldout: list[Path] = typer.Option(
|
||
list(HELDOUT_PATHS),
|
||
"--heldout",
|
||
help="Held-out golden sets; the primary one is the stamped measurement.",
|
||
),
|
||
generation: str = typer.Option(
|
||
DEFAULT_GENERATION,
|
||
"--generation",
|
||
help="Prompt generation: stance-v4 or stance-v5 (CPT guideline facts).",
|
||
),
|
||
write: bool = typer.Option(False, "--write"),
|
||
) -> None:
|
||
"""Each commenter's stance (support / modify / oppose / unclear) on each
|
||
family event the docket's rule touched, with a reason and a verbatim
|
||
quote (pfs.comment_stance); --write also rolls the counts up into the
|
||
docket's pfs.code_reaction row. CMS's own summaries are never used."""
|
||
if method not in METHODS:
|
||
raise typer.BadParameter(f"--method must be one of {', '.join(METHODS)}")
|
||
if model not in ("instruct", "large"):
|
||
raise typer.BadParameter("--model must be instruct or large")
|
||
if generation not in GENERATIONS:
|
||
raise typer.BadParameter(
|
||
f"--generation must be one of {', '.join(GENERATIONS)}"
|
||
)
|
||
fam = curated_family(normalize_key(family))
|
||
if fam is None:
|
||
raise typer.BadParameter(f"unknown curated family {family!r}")
|
||
store = _store()
|
||
found = [d for d in store.dockets() if d.id == docket]
|
||
if not found:
|
||
raise typer.BadParameter(f"unknown docket {docket!r}")
|
||
year = docket_rule_year(found[0].comment_end_date)
|
||
con = _read()
|
||
try:
|
||
events = docket_events(
|
||
con,
|
||
fam,
|
||
year,
|
||
store,
|
||
guidelines=guideline_text(generation),
|
||
fixed=fixed_text(generation),
|
||
)
|
||
commenter_rows = [
|
||
c for c in read_commenters(con, family=fam.key) if c.docket == docket
|
||
]
|
||
stored = _stored(read_reaction, con, fam.key)
|
||
finally:
|
||
con.close()
|
||
if not commenter_rows:
|
||
typer.echo(f"{docket}: no commenter rows (run stack pfs commenters first)")
|
||
return
|
||
# the item set is the commenter rows, joined to the family's comment text
|
||
wanted = {c.item_key for c in commenter_rows}
|
||
texts = {
|
||
k: t
|
||
for k, t in family_comment_items(
|
||
_engine(),
|
||
fam.codes,
|
||
fam.key,
|
||
fam.pair_phrases,
|
||
docket=docket,
|
||
max_chars=TEXT_CHARS,
|
||
max_chunks=TEXT_CHUNKS,
|
||
)
|
||
if k in wanted
|
||
}
|
||
with_text = sum(1 for c in commenter_rows if c.item_key in texts)
|
||
typer.echo(
|
||
f"{docket}: {len(commenter_rows)} commenters, {with_text} with comment "
|
||
f"text, {len(events)} events"
|
||
)
|
||
for e in events:
|
||
typer.echo(f" {e.event_key}: {e.text}")
|
||
if dry_run:
|
||
typer.echo("no model calls made")
|
||
return
|
||
large = model == "large"
|
||
ask = None if no_llm else (_asker(large=True) if large else _asker())
|
||
name = _model_of(ask, large)
|
||
rows, counts = stances_for(
|
||
commenter_rows,
|
||
texts,
|
||
events,
|
||
ask,
|
||
RunLog(ledger_path("stance", fam.key)),
|
||
family=fam.key,
|
||
docket=docket,
|
||
method=method,
|
||
generation=generation,
|
||
model=name if no_llm else None,
|
||
subjects=event_subjects(fam, events, by_kind=guideline_text(generation)),
|
||
limit=limit,
|
||
retry_failed=retry_failed,
|
||
progress=lambda unit, status: typer.echo(f"{unit} {status}"),
|
||
)
|
||
blank = ReactionRow(fam.key, docket, "docket", year, 0, 0, 0, 0, 0, 0, "[]")
|
||
order = [e.event_key for e in events]
|
||
rolled = rollup(rows, base=blank, order=order, n_commenters=len(commenter_rows))
|
||
for key, n in json.loads(rolled.events_json).items():
|
||
typer.echo(
|
||
f"{key}: support {n['support']}, modify {n['modify']}, "
|
||
f"oppose {n['oppose']}, unclear {n['unclear']}"
|
||
)
|
||
typer.echo(
|
||
f"commenters classified: {rolled.n_classified} of {len(commenter_rows)}; "
|
||
f"support {rolled.stance_support}, modify {rolled.stance_modify}, "
|
||
f"oppose {rolled.stance_oppose}, unclear {rolled.stance_unclear}"
|
||
)
|
||
typer.echo(
|
||
f"stances: {len(rows)}; done {counts['done']}, skipped {counts['skipped']}, "
|
||
f"failed {counts['failed']}, failed_pending {counts['failed_pending']}, "
|
||
f"pending {counts['pending']}, no_text {counts['no_text']}"
|
||
)
|
||
if not write:
|
||
return
|
||
if counts["pending"]:
|
||
raise typer.BadParameter(
|
||
f"{counts['pending']} comments not yet classified; rerun to resume"
|
||
)
|
||
series_rows = reaction_series(_engine(), store, fam.key, fam.codes, name=fam.name)
|
||
base = next(
|
||
(r for r in series_rows if r.period_kind == "docket" and r.period == docket),
|
||
blank,
|
||
)
|
||
new = replace(
|
||
rollup(rows, base=base, order=order, n_commenters=len(commenter_rows)),
|
||
stance_method=f"{prompt_version(method, generation)}:{name}",
|
||
stance_agreement=None,
|
||
stance_eval=_method_eval(
|
||
golden, method, name, fam.key, generation, tuple(heldout)
|
||
),
|
||
)
|
||
keep = [
|
||
r
|
||
for r in (stored or series_rows)
|
||
if r.period != docket or r.period_kind != "docket"
|
||
]
|
||
with _batch() as bcon:
|
||
ensure_tables(bcon)
|
||
n = write_stances(bcon, fam.key, docket, rows)
|
||
write_reaction(bcon, fam.key, [*keep, new])
|
||
typer.echo(f"wrote {n} stances")
|
||
_publish()
|
||
|
||
|
||
def _method_eval(
|
||
golden: Path,
|
||
method: str,
|
||
name: str,
|
||
family: str,
|
||
generation: str | None = None,
|
||
heldout: Sequence[Path] = HELDOUT_PATHS,
|
||
) -> str | None:
|
||
"""The method's published measurement as ``stance_eval`` JSON (R82):
|
||
the family's PRIMARY held-out set for *generation*, replayed from the
|
||
ledger (no model call), with the in-sample tuning set's scores beside it
|
||
as ``in_sample_record``. ``None`` when no primary set matches the family
|
||
and generation; a set the ledger does not fully cover is an error, never
|
||
a partial measurement."""
|
||
gen = generation or DEFAULT_GENERATION
|
||
primary = None
|
||
for path in heldout:
|
||
if not path.exists():
|
||
continue
|
||
data = json.loads(path.read_text(encoding="utf-8"))
|
||
if (
|
||
data.get("role") == "primary"
|
||
and data.get("family") == family
|
||
and data.get("generation", gen) == gen
|
||
):
|
||
primary = data
|
||
break
|
||
if primary is None:
|
||
return None
|
||
sc, _ = _golden_run(primary, method, ask=None, name=name, generation=generation)
|
||
if sc is None:
|
||
raise typer.BadParameter(
|
||
f"held-out set not fully classified for {method} on {name}; "
|
||
"run stack pfs stance-eval on it first"
|
||
)
|
||
rec = eval_record(primary, sc)
|
||
rec.pop("docket")
|
||
rec["generation"] = gen
|
||
if golden.exists():
|
||
tuning = json.loads(golden.read_text(encoding="utf-8"))
|
||
if tuning.get("family") == family:
|
||
tsc, _ = _golden_run(
|
||
tuning, method, ask=None, name=name, generation=generation
|
||
)
|
||
if tsc is None:
|
||
raise typer.BadParameter(
|
||
f"tuning set not fully classified for {method} on {name}"
|
||
)
|
||
t = eval_record(tuning, tsc)
|
||
rec["in_sample_record"] = {
|
||
k: t[k]
|
||
for k in (
|
||
"cell",
|
||
"recall",
|
||
"direction",
|
||
"n_items",
|
||
"n_positions",
|
||
"docket",
|
||
)
|
||
}
|
||
return json.dumps(rec)
|
||
|
||
|
||
def _golden_run(
|
||
data: dict,
|
||
method: str,
|
||
*,
|
||
ask: Any,
|
||
name: str,
|
||
generation: str | None = None,
|
||
relevance: bool = True,
|
||
limit: int = 0,
|
||
retry_failed: bool = False,
|
||
progress: Any = None,
|
||
) -> tuple[Any, dict[str, int]]:
|
||
"""Classify the golden items with *method* through the stance ledger (a
|
||
``None`` *ask* only replays model *name*'s entries) and score them. The
|
||
scores are ``None`` while any item is still pending."""
|
||
fam = curated_family(normalize_key(data["family"]))
|
||
if fam is None:
|
||
raise typer.BadParameter(f"unknown curated family {data['family']!r}")
|
||
store = _store()
|
||
of_docket = item_dockets(data)
|
||
all_rows: list[Any] = []
|
||
counts = {
|
||
"done": 0, "skipped": 0, "failed": 0, "failed_pending": 0, "pending": 0,
|
||
"no_text": 0,
|
||
} # fmt: skip
|
||
remaining = limit
|
||
for docket in golden_dockets(data):
|
||
found = [d for d in store.dockets() if d.id == docket]
|
||
if not found:
|
||
raise typer.BadParameter(f"unknown docket {docket!r}")
|
||
con = _read()
|
||
try:
|
||
events = docket_events(
|
||
con,
|
||
fam,
|
||
docket_rule_year(found[0].comment_end_date),
|
||
store,
|
||
guidelines=guideline_text(generation),
|
||
fixed=fixed_text(generation),
|
||
)
|
||
finally:
|
||
con.close()
|
||
keys = [k for k, d in of_docket.items() if d == docket]
|
||
texts = {
|
||
k: t
|
||
for k, t in family_comment_items(
|
||
_engine(),
|
||
fam.codes,
|
||
fam.key,
|
||
fam.pair_phrases,
|
||
docket=docket,
|
||
max_chars=TEXT_CHARS,
|
||
max_chunks=TEXT_CHUNKS,
|
||
)
|
||
if k in keys
|
||
}
|
||
# the call limit is shared across dockets; once spent, only replay
|
||
live = ask if (ask is not None and (not limit or remaining > 0)) else None
|
||
rows, got = stances_for(
|
||
[SimpleNamespace(item_key=k) for k in keys],
|
||
texts,
|
||
events,
|
||
live,
|
||
RunLog(ledger_path("stance", fam.key)),
|
||
family=fam.key,
|
||
docket=docket,
|
||
method=method,
|
||
generation=generation,
|
||
model=name if live is None else None,
|
||
subjects=(
|
||
event_subjects(fam, events, by_kind=guideline_text(generation))
|
||
if relevance
|
||
else None
|
||
),
|
||
limit=remaining,
|
||
retry_failed=retry_failed,
|
||
progress=progress,
|
||
)
|
||
remaining -= got["done"] + got["failed"]
|
||
all_rows += rows
|
||
counts = {k: counts[k] + got[k] for k in counts}
|
||
rows, counts = all_rows, counts
|
||
incomplete = counts["pending"] or counts["no_text"]
|
||
return (None if incomplete else score(data, rows)), counts
|
||
|
||
|
||
@app.command("stance-eval")
|
||
def stance_eval(
|
||
golden: Path = typer.Option(..., "--golden", help="Golden stance JSON."),
|
||
method: str = typer.Option(DEFAULT_METHOD, "--method", help="multi or per-event."),
|
||
model: str = typer.Option("instruct", "--model", help="instruct or large."),
|
||
limit: int = typer.Option(0, "--limit", help="Cap new model calls (0 = all)."),
|
||
retry_failed: bool = typer.Option(False, "--retry-failed"),
|
||
no_llm: bool = typer.Option(
|
||
False, "--no-llm", help="Make no model calls; the finished ledger is replayed."
|
||
),
|
||
no_relevance: bool = typer.Option(
|
||
False, "--no-relevance", help="Skip the event-subject filter (R65)."
|
||
),
|
||
generation: str = typer.Option(
|
||
DEFAULT_GENERATION, "--generation", help="stance-v4 or stance-v5."
|
||
),
|
||
) -> None:
|
||
"""Score a stance method against a hand-coded golden set: cell agreement,
|
||
presence agreement, firm-stance accuracy and a confusion matrix. The
|
||
golden items are classified through the stance ledger (resumable); nothing
|
||
is written to pfs.comment_stance."""
|
||
import json
|
||
import time
|
||
|
||
if method not in METHODS:
|
||
raise typer.BadParameter(f"--method must be one of {', '.join(METHODS)}")
|
||
if model not in ("instruct", "large"):
|
||
raise typer.BadParameter("--model must be instruct or large")
|
||
if generation not in GENERATIONS:
|
||
raise typer.BadParameter(
|
||
f"--generation must be one of {', '.join(GENERATIONS)}"
|
||
)
|
||
data = json.loads(golden.read_text(encoding="utf-8"))
|
||
large = model == "large"
|
||
ask = None if no_llm else _asker(large=large)
|
||
name = _model_of(ask, large)
|
||
began = time.monotonic()
|
||
sc, counts = _golden_run(
|
||
data,
|
||
method,
|
||
ask=ask,
|
||
name=name,
|
||
generation=generation,
|
||
relevance=not no_relevance,
|
||
limit=limit,
|
||
retry_failed=retry_failed,
|
||
progress=lambda unit, status: typer.echo(f"{unit} {status}"),
|
||
)
|
||
calls = counts["done"] + counts["failed"]
|
||
typer.echo(
|
||
f"{prompt_version(method, generation)} {method} on {name}"
|
||
f"{' (no relevance filter)' if no_relevance else ''}: "
|
||
f"{calls} new calls in {time.monotonic() - began:.0f}s; done {counts['done']}, "
|
||
f"skipped {counts['skipped']}, failed {counts['failed']}, failed_pending "
|
||
f"{counts['failed_pending']}, pending {counts['pending']}, "
|
||
f"no_text {counts['no_text']}"
|
||
)
|
||
if sc is None:
|
||
typer.echo("incomplete: rerun to resume; no scores yet")
|
||
return
|
||
typer.echo(
|
||
f"cells {sc.cells}: agreement {sc.cell_agreement:.3f} ({sc.agree}); "
|
||
f"presence {sc.presence_agreement:.3f} ({sc.presence_agree}); "
|
||
f"firm accuracy {sc.firm_accuracy:.3f} ({sc.firm_correct}/{sc.firm_cells}); "
|
||
f"gate {'PASS' if sc.passes() else 'FAIL'}"
|
||
)
|
||
typer.echo(
|
||
f"recall {sc.recall:.3f} ({sc.found}/{sc.positions}); "
|
||
f"direction {sc.direction:.3f} ({sc.right}/{sc.found})"
|
||
)
|
||
labels = list(sc.confusion)
|
||
typer.echo(
|
||
"confusion (rows golden, columns model): " + " ".join(f"{x:>8}" for x in labels)
|
||
)
|
||
for g in labels:
|
||
typer.echo(f"{g:>8} " + " ".join(f"{sc.confusion[g][m]:>8}" for m in labels))
|
||
for d in sc.disagreements[:10]:
|
||
typer.echo(
|
||
f"disagree {d.item_key} {d.event_key.rsplit(':', 1)[-1]}: "
|
||
f"golden {d.golden}, model {d.model}"
|
||
)
|
||
|
||
|
||
def _model_of(ask: Any, large: bool) -> str:
|
||
"""The pinned model's name: the asker's, or (no asker) the configured one."""
|
||
if ask is not None:
|
||
return ask.model
|
||
return _model_name(large=True) if large else _model_name()
|
||
|
||
|
||
def _excerpts(store: Any, name: str) -> list[Any]:
|
||
"""Governance excerpts for a family (embeddings on the llm pool; no
|
||
generation)."""
|
||
from llm import config as llm_config
|
||
from llm.pool import HostPool
|
||
|
||
cfg = llm_config.load()
|
||
return governance_excerpts(cfg, HostPool.from_config(cfg), store, name)
|
||
|
||
|
||
@app.command("dossier")
|
||
def dossier_cmd(
|
||
family: str = typer.Option("", "--family", help="Curated family key."),
|
||
all_families: bool = typer.Option(False, "--all", help="Every curated family."),
|
||
no_llm: bool = typer.Option(
|
||
False,
|
||
"--no-llm",
|
||
help="Make no model calls; the finished prose ledger is replayed.",
|
||
),
|
||
limit: int = typer.Option(0, "--limit", help="Cap new model calls (0 = all)."),
|
||
dry_run: bool = typer.Option(
|
||
False,
|
||
"--dry-run",
|
||
help="Count facts and timeline events; no model calls, no writes.",
|
||
),
|
||
write: bool = typer.Option(False, "--write"),
|
||
) -> None:
|
||
"""Assemble a family's dossier (pfs.dossier): every stored fact with its
|
||
resolved citation, the computed gaps, and model prose that passes the
|
||
sentence-level citation check (template lines otherwise), through a
|
||
resumable ledger. Prints the fact count, the gaps, the stakeholder table
|
||
and the check result; --write stores it."""
|
||
if all_families:
|
||
registry = load_curated()
|
||
fams = [registry[k] for k in sorted(registry)]
|
||
elif family:
|
||
fam = curated_family(normalize_key(family))
|
||
if fam is None:
|
||
raise typer.BadParameter(f"unknown curated family {family!r}")
|
||
fams = [fam]
|
||
else:
|
||
raise typer.BadParameter("give --family KEY or --all")
|
||
store = _store()
|
||
built = []
|
||
for fam in fams:
|
||
con = _read()
|
||
try:
|
||
if dry_run:
|
||
facts = collect_facts(con, store, fam)
|
||
events = dossier_timeline(facts)
|
||
typer.echo(
|
||
f"{fam.key}: {len(facts)} facts, {len(events)} timeline events; "
|
||
f"model calls a run would make: {len(events) + 1} "
|
||
"(each retried once at most)"
|
||
)
|
||
continue
|
||
d = build_dossier(
|
||
con,
|
||
store,
|
||
fam,
|
||
ask=None if no_llm else _asker(),
|
||
excerpts=[] if no_llm else _excerpts(store, fam.name),
|
||
log=RunLog(ledger_path("prose", fam.key)),
|
||
replay_model=_model_name() if no_llm else None,
|
||
limit=limit,
|
||
)
|
||
finally:
|
||
con.close()
|
||
kinds = Counter(f.kind for f in d.facts.values())
|
||
typer.echo(
|
||
f"{fam.key}: {len(d.facts)} facts ("
|
||
+ ", ".join(f"{k} {n}" for k, n in sorted(kinds.items()))
|
||
+ ")"
|
||
)
|
||
typer.echo(
|
||
"timeline years: "
|
||
+ ", ".join(str(y) for y in sorted({e.year for e in d.timeline}))
|
||
)
|
||
for gap in d.gaps:
|
||
typer.echo(f"gap: {gap}")
|
||
for line in stakeholder_lines(d):
|
||
typer.echo(line)
|
||
typer.echo(
|
||
f"check: {'passed' if d.check_passed else 'FAILED'}; prose {d.extractor}"
|
||
)
|
||
if d.extractor != "template" and d.fallbacks:
|
||
typer.echo("template fallback: " + "; ".join(d.fallbacks))
|
||
built.append(d)
|
||
if write and built:
|
||
with _batch() as bcon:
|
||
ensure_tables(bcon)
|
||
for d in built:
|
||
write_dossier(bcon, d)
|
||
typer.echo(f"wrote dossier {d.family}")
|
||
_publish()
|