Files
stack/src/cli/pfs.py
kert 481cd3f2e5 feat(pfs): dossier assembly — cited facts, computed gaps, model prose behind a sentence-level citation check
pfs.cite (Cite + builders; rule_label moved from llm.lineage and re-exported),
pfs.citecheck (sentence split, number normalization, check_prose; plus
grounding_problems: words, codes and CMS-vs-ours attribution of model prose),
pfs.dossier (facts with resolved cites from every Task 1-12 table, timeline,
stakeholder panels with names from pfs.commenter, stance accuracy beside every
count, computed coverage gaps, ledgered model prose with template fallback,
pfs.dossier table), llm.search item_keys filter, `stack pfs dossier`.
Lineage rows reach the dossier only when a final rule's own plain sentence
states them. HOME-VISITS live: 558 facts, 28 timeline events, check passed,
prose model:qwen2.5:14b+template (8 of 29 units on templates).
2026-11-12 23:07:03 -05:00

2482 lines
90 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""stack pfs — code elements, lineage events and derived families.
uv run stack pfs elements --code G0556 --code 99490
uv run stack pfs elements --family CCM
uv run stack pfs elements --all-payable --no-llm --dry-run
uv run stack pfs lineage --code G2058 [--write]
uv run stack pfs families [--write]
uv run stack pfs guidance --family CCM [--family …] [--write]
uv run stack pfs reaction --family CCM [--code G2211] [--stance-sample 30] [--write]
uv run stack pfs review [--code G0556]
uv run stack pfs cpt-ingest --edition 2024 [--edition 2022 …] [--dry-run]
uv run stack pfs cpt-ingest --all
uv run stack pfs dossier --family HOME-VISITS [--no-llm] [--limit N] [--write]
Writes go through ``duckdb_batch`` (single-writer rule) and republish
the read-only replica so the chat and notebooks see them.
"""
from __future__ import annotations
import json
import logging
from collections import Counter
from collections.abc import Sequence
from dataclasses import replace
from pathlib import Path
from types import SimpleNamespace
from typing import Any
import httpx
import typer
from pfs import seats as seat_lib
from pfs.cms_summaries import summaries as cms_summaries_rows
from pfs.codetables import (
STANCE_METHOD_FIELDS,
ReactionRow,
cpt_years,
ensure_tables,
is_missing_table_error,
read_all_elements,
read_all_events,
read_all_reviews,
read_commenters,
read_cpt_codes,
read_cpt_instructions,
read_cpt_sections,
read_exposures,
read_guidance,
read_reaction,
read_reviews,
read_stances,
write_cms_summaries,
write_commenters,
write_elements,
write_events,
write_exposures,
write_families,
write_guidance,
write_guideline_facts,
write_origin,
write_reaction,
write_rounds,
write_stances,
)
from pfs.commenters import family_comment_items, identify, regs_for
from pfs.cpt_load import ingest as cpt_ingest
from pfs.curated import apply_curated, load_curated
from pfs.dossier import (
build_dossier,
collect_facts,
governance_excerpts,
stakeholder_lines,
write_dossier,
)
from pfs.dossier import timeline as dossier_timeline
from pfs.exposure import build as build_exposures
from pfs.exposure import control_codes
from pfs.extract import extract_codes
from pfs.families import (
CURATED_FAMILIES,
FAMILIES,
HAND_FAMILIES,
curated_family,
derive_families,
normalize_key,
refresh_from,
)
from pfs.guidance import build as build_guidance
from pfs.guideline_facts import guideline_facts
from pfs.lineage import lineage, lineage_all, seen_codes
from pfs.origin import DEFAULT_BOOK, family_origin, origin_inventory, resolve_book
from pfs.psps import DATASETS as PSPS_YEARS
from pfs.psps import USER_AGENT as PSPS_USER_AGENT
from pfs.psps import cache_path as psps_cache_path
from pfs.psps import cite as cite_psps
from pfs.psps import fetch_code_year as fetch_psps
from pfs.psps import load_cache as load_psps_cache
from pfs.psps import newest_cache as newest_psps_cache
from pfs.psps import read_by_specialty as read_psps_rows
from pfs.psps import save_cache as save_psps_cache
from pfs.psps import to_rows as psps_rows
from pfs.psps import write as write_psps_rows
from pfs.reaction import series as reaction_series
from pfs.rounds import family_rounds, rounds_inventory
from pfs.runlog import RunLog, input_hash, ledger_path, run_units
from pfs.stakes import (
discover_508,
discover_fr,
parse_impact_508,
parse_impact_fr_text,
write_impacts,
)
from pfs.stance import (
DEFAULT_GENERATION,
DEFAULT_METHOD,
DEFAULT_MODEL,
GENERATIONS,
GOLDEN_PATH,
HELDOUT_PATHS,
METHODS,
TEXT_CHARS,
TEXT_CHUNKS,
docket_events,
docket_rule_year,
eval_record,
event_subjects,
fixed_text,
golden_dockets,
guideline_text,
item_dockets,
prompt_version,
rollup,
score,
stances_for,
)
from pfs.utilization import DATASETS as UTILIZATION_YEARS
from pfs.utilization import cache_path as utilization_cache_path
from pfs.utilization import cite as cite_utilization
from pfs.utilization import fetch_year as fetch_utilization_year
from pfs.utilization import load_cache as load_utilization_cache
from pfs.utilization import read_series as read_utilization_series
from pfs.utilization import save_cache as save_utilization_cache
from pfs.utilization import to_rows as utilization_rows
from pfs.utilization import write as write_utilization
log = logging.getLogger(__name__)
app = typer.Typer(no_args_is_help=True)
#: A replica that hasn't had `elements`/`lineage --write` (or an initial
#: `families --write`) run yet is missing `pfs.code_element`/
#: `pfs.code_event`/`pfs.code_family` — not a bug, just nothing derived
#: yet (I4). DuckDB's "table does not exist" errors are Catalog errors —
#: the predicate itself lives in `pfs.codetables` so `pfs.lineage`/
#: `pfs.extract` can reuse it for the same replica shape (a CPT-table-
#: free replica is not a bug either).
def _missing_table(exc: Exception) -> bool:
return is_missing_table_error(exc)
# ── indirections the tests monkeypatch ───────────────────────────────
def _batch() -> Any:
from conf.connect import duckdb_batch
return duckdb_batch("aco")
def _read() -> Any:
from conf.connect import duckdb
return duckdb("aco", read_only=True)
def _store() -> Any:
from conf.connect import bib
return bib()
def _publish() -> None:
from conf.connect import publish_replica
typer.echo(f"replica → {publish_replica('aco')}")
def _classifier(*, pin: bool = False) -> Any:
from llm import config as llm_config
from llm.classify import closed_vocab_classifier
from llm.pool import HostPool
cfg = llm_config.load()
return closed_vocab_classifier(
cfg, HostPool.from_config(cfg), model=cfg.instruct_model if pin else None
)
def _asker(large: bool = False) -> Any:
"""The JSON extractor on ``cfg.instruct_model``, or with *large* on
``cfg.instruct_model_large`` (still one pinned model, never a fallback)."""
from llm import config as llm_config
from llm.pool import HostPool
from llm.structured import json_extractor
cfg = llm_config.load()
if large:
if not cfg.instruct_model_large:
raise typer.BadParameter("no instruct_model_large configured")
cfg = replace(cfg, instruct_model=cfg.instruct_model_large)
return json_extractor(
cfg, HostPool.from_config(cfg), post=_slow_post if large else None
)
def _slow_post(url: str, json: dict) -> dict:
"""``llm.classify._default_post`` with a 10-minute read timeout: the large
model reloads on every call and a whole-comment prompt outlasts the usual
120 s."""
import httpx
with httpx.Client(timeout=httpx.Timeout(600.0, connect=10.0)) as client:
resp = client.post(url, json=json)
resp.raise_for_status()
return resp.json()
def _model_name(large: bool = False) -> str:
"""The instruct model's name (or the large one), without touching the GPU
pool: what a ledger replay (``--no-llm``) keys the recorded outputs by."""
from llm import config as llm_config
cfg = llm_config.load()
return cfg.instruct_model_large if large else cfg.instruct_model
def _mirror() -> Any:
from bib.regulations_gov import Mirror
return Mirror()
def _engine() -> Any:
# llm.index/llm.evidence expose no public "give me a bare SQLAlchemy
# engine over the pgvector DB" helper — cli/bib.py and cli/llm.py
# both already reach into llm.index._engine for the same reason, so
# this is the established reuse, not a new private import.
from llm import config as llm_config
from llm.index import _engine as pgvector_engine
return pgvector_engine(llm_config.load())
def _run_elements(
con: Any, store: Any, targets: list[str], classify: Any, *, write: bool
) -> None:
# #698: one inverted pass over fr_anchors (plus one query each over
# pfs.cpt_code / terminology.hcpcs_level_2 / pfs.rvu) for the whole
# target list — the per-code form re-scanned the 193k-row, unindexed
# fr_anchors table twice per code, ~15-25 min for --all-payable.
by_code = extract_codes(store, con, targets, classify=classify)
for c in targets:
x = by_code[c]
if write:
write_elements(con, c, x.rows, x.reviews)
confirmed = sum(1 for r in x.rows if r.confirmed_by)
note = f", {confirmed} confirmed by a second source" if confirmed else ""
typer.echo(f"{c}: {len(x.rows)} elements, {len(x.reviews)} for review{note}")
def _codes_for(
con: Any,
codes: list[str],
families: list[str],
all_payable: bool,
*,
all_seen: bool = False,
warn_slow: bool = True,
) -> list[str]:
"""*warn_slow*: log how many codes ``--all-payable`` resolved to, so a
run that is about to make one model call per unplaced line for
thousands of codes says so up front. ``lineage --all-payable`` is
pure SQL/regex and needs no such note, so it passes
``warn_slow=False``. (Both commands now sweep ``fr_anchors`` in one
inverted pass — #698 — so neither is the ~20 min of per-code SQL the
warning used to describe.)"""
out: list[str] = [c.upper() for c in codes]
if families:
refresh_from(con)
for family in families:
fam = curated_family(family) or FAMILIES.get(normalize_key(family))
if fam is None:
# List only the hand families here (M3) — FAMILIES can hold
# ~15k derived keys once `refresh_from` has run, and that
# dump is useless in an error message.
raise typer.BadParameter(
f"unknown family {family!r}; known: {', '.join(HAND_FAMILIES)}"
)
out.extend(fam.codes)
if all_payable:
rows = con.execute(
"SELECT DISTINCT hcpcs FROM pfs.rvu WHERE status_code IN ('A','R','T') "
"AND year = (SELECT max(year) FROM pfs.rvu) ORDER BY hcpcs"
).fetchall()
if warn_slow:
log.warning("--all-payable: targeting %d codes", len(rows))
out.extend(r[0] for r in rows)
if all_seen:
seen = seen_codes(con)
if warn_slow:
log.warning("--all-seen: targeting %d codes", len(seen))
out.extend(seen)
if not out:
raise typer.BadParameter("pass --code, --family, --all-payable or --all-seen")
return sorted(set(out))
@app.command()
def elements(
code: list[str] = typer.Option([], "--code", help="HCPCS/CPT code (repeatable)."),
family: list[str] = typer.Option(
[], "--family", help="Expand a registered family (CCM, APCM, …); repeatable."
),
all_payable: bool = typer.Option(
False,
"--all-payable",
help=(
"Every A/R/T code in the newest RVU year, one inverted pass over "
"fr_anchors."
),
),
no_llm: bool = typer.Option(
False,
"--no-llm",
help="Skip the local-model classifier (unknown lines go to review).",
),
dry_run: bool = typer.Option(
False, "--dry-run", help="Extract and report; write nothing."
),
) -> None:
"""Extract typed elements for codes into pfs.code_element (+ review queue)."""
store = _store()
classify = None if no_llm else _classifier()
if dry_run:
# A preview never contends for the DuckDB single-writer lock a
# notebook may be holding (#508-#514) — read the replica, don't
# open a batch writer.
con = _read()
try:
targets = _codes_for(con, code, family, all_payable)
_run_elements(con, store, targets, classify, write=False)
finally:
con.close()
else:
with _batch() as con:
ensure_tables(con)
targets = _codes_for(con, code, family, all_payable)
_run_elements(con, store, targets, classify, write=True)
_publish()
def _print_events(events: list[Any]) -> None:
for e in events:
arrow = f"{e.from_codes or '·'}→{e.to_codes or '·'}"
if e.source == "cpt":
anchor = f"cpt {e.year} {e.item_key}"
elif e.item_key:
anchor = f"{e.item_key}{e.p_id}"
else:
anchor = f"rvu {e.note}".strip()
flag = "" if e.anchored else " UNANCHORED"
typer.echo(f"{e.year} {e.kind:<18}{arrow:<16}{anchor}{flag}")
def _print_lineage_all(by_code: dict[str, list[Any]]) -> None:
per_source: dict[str, int] = {}
with_events = 0
unanchored = 0
for events in by_code.values():
if events:
with_events += 1
for e in events:
per_source[e.source] = per_source.get(e.source, 0) + 1
if e.source == "rvu" and not e.anchored:
unanchored += 1
detail = ", ".join(f"{k}={v}" for k, v in sorted(per_source.items())) or "none"
typer.echo(
f"lineage: {len(by_code)} codes targeted, {with_events} with >=1 event, "
f"events per source: {detail}, unanchored rvu events: {unanchored}"
)
@app.command("lineage")
def lineage_cmd(
code: str = typer.Option("", "--code"),
family: str = typer.Option(
"", "--family", help="A curated or registered family's codes."
),
all_payable: bool = typer.Option(
False,
"--all-payable",
help="Every A/R/T code in the newest RVU year, one inverted pass over fr_anchors.",
),
all_seen: bool = typer.Option(
False,
"--all-seen",
help="Every code ever seen in any RVU year or CPT edition (incl. deleted codes).",
),
limit: int = typer.Option(
0,
"--limit",
help="Only the first N target codes (sorted) — for smoke runs with --all-payable.",
),
write: bool = typer.Option(
False, "--write", help="Persist to pfs.code_event and republish."
),
) -> None:
"""Timeline of a code: RVU-file diffs and FR paragraphs, cross-checked.
--all-payable computes lineage for every payable code in ONE pass over
fr_anchors instead of one query per code.
"""
if not code and not all_payable and not all_seen and not family:
raise typer.BadParameter("pass --code, --family, --all-payable or --all-seen")
store = _store()
if all_payable or all_seen or family:
# Ruling A13: resolve targets and build every code's lineage
# against a plain DuckDB read (I4) — never the write lock a
# notebook may be holding (#508-#514). The batch below only ever
# calls write_events and publishes.
con = _read()
try:
targets = _codes_for(
con,
[],
[family] if family else [],
all_payable,
all_seen=all_seen,
warn_slow=False,
)
if limit:
targets = targets[:limit]
if code:
targets = sorted({*targets, code.upper()})
by_code = lineage_all(con, store, targets)
finally:
con.close()
if write:
with _batch() as con:
ensure_tables(con)
for c in targets:
write_events(con, c, by_code.get(c, []))
_publish()
_print_lineage_all(by_code)
return
if write:
# Same A13 discipline for the single-code path: build the events
# against a plain read, then open the batch only to write them.
con = _read()
try:
events = lineage(con, store, code.upper())
finally:
con.close()
_print_events(events)
with _batch() as con:
ensure_tables(con)
write_events(con, code.upper(), events)
_publish()
return
# No --write: a plain lookup never needs the DuckDB single-writer lock
# a notebook may be holding (#508-#514) — read the replica (I4).
con = _read()
try:
try:
events = lineage(con, store, code.upper())
except Exception as exc:
if _missing_table(exc):
typer.echo("no derived tables yet — run with --write")
return
raise
_print_events(events)
finally:
con.close()
def _cpt_inputs(
con: Any,
) -> tuple[list[Any], list[Any], list[Any], dict[str, tuple[int, ...]]]:
"""The newest CPT edition's rows for ``derive_families`` (grouping/
naming stays newest-edition-scoped, Ruling C12), plus ``cpt_presence``
— every ingested edition's ``code -> years present`` map, one extra
full-table pass, used only for ``since``/``until``: a code an earlier
edition ingested but the newest one dropped still gets its true
``since`` (not clamped to the newest edition) and a real ``until``.
All empty on any replica shape that doesn't have them yet (I4: a
plain read of an old replica, pre pfs.cpt_*, is not a bug)."""
try:
years = cpt_years(con)
except Exception as exc:
if _missing_table(exc):
return [], [], [], {}
raise
if not years:
return [], [], [], {}
newest = max(years)
presence: dict[str, list[int]] = {}
for code, year in con.execute(
"SELECT code, edition_year FROM pfs.cpt_code"
).fetchall():
presence.setdefault(code, []).append(year)
cpt_presence = {code: tuple(sorted(set(yrs))) for code, yrs in presence.items()}
return (
read_cpt_codes(con, newest),
read_cpt_sections(con, newest),
read_cpt_instructions(con, newest),
cpt_presence,
)
def _derive_rows(con: Any) -> list[Any]:
codes = [
r[0]
for r in con.execute(
"SELECT DISTINCT code FROM pfs.code_element UNION SELECT DISTINCT code FROM pfs.code_event "
"UNION SELECT DISTINCT hcpcs FROM pfs.rvu WHERE year = (SELECT max(year) FROM pfs.rvu) AND status_code IN ('A','R','T')"
).fetchall()
]
# Two full-table reads grouped in Python instead of ~2xN per-code
# queries run inside the write lock (I9).
all_elements = read_all_elements(con)
all_events = read_all_events(con)
elements_by_code = {c: all_elements.get(c, []) for c in codes}
events_by_code = {c: all_events.get(c, []) for c in codes}
descriptions = {
r[0]: r[1] or ""
for r in con.execute(
"SELECT hcpcs, arg_max(description, year) FROM pfs.rvu WHERE mod IS NULL OR mod = '' GROUP BY hcpcs"
).fetchall()
}
cpt_codes, cpt_sections, cpt_instructions, cpt_presence = _cpt_inputs(con)
return derive_families(
elements_by_code,
events_by_code,
descriptions,
cpt_codes=cpt_codes,
cpt_sections=cpt_sections,
cpt_instructions=cpt_instructions,
cpt_presence=cpt_presence,
)
def _print_families(rows: list[Any], *, verbose: bool = False) -> None:
by_key: dict[str, list[Any]] = {}
for r in rows:
by_key.setdefault(r.key, []).append(r)
if verbose:
for key, members in sorted(by_key.items()):
typer.echo(f"{key}: {' '.join(f'{r.code}({r.role})' for r in members)}")
total = len(by_key)
multi = sum(1 for members in by_key.values() if len(members) >= 2)
hand = sum(1 for key in by_key if key in HAND_FAMILIES)
curated = sum(1 for key in by_key if key in CURATED_FAMILIES)
cpt_named = sum(
1
for key, members in by_key.items()
if key not in HAND_FAMILIES
and key not in CURATED_FAMILIES
and any(r.note for r in members)
)
other = total - hand - curated - cpt_named
typer.echo(
f"families: total {total}, multi-code {multi}, cpt-named {cpt_named}, "
f"hand {hand}, curated {curated}, other {other}"
)
@app.command()
def families(
write: bool = typer.Option(False, "--write"),
verbose: bool = typer.Option(
False, "--verbose", help="Print every family's members, not just the summary."
),
) -> None:
"""Derive families from pfs.code_element / pfs.code_event / pfs.rvu."""
if write:
with _batch() as con:
ensure_tables(con)
rows = apply_curated(_derive_rows(con), load_curated())
write_families(con, rows)
_publish()
# refresh_from and printing don't need the write lock open — do
# them after it's released, from the freshly published replica
# (the same pattern `stack llm restamp` uses).
refresh_from(_read())
_print_families(rows, verbose=verbose)
return
# No --write: derive-and-print never needs the write lock (I4).
con = _read()
try:
try:
rows = _derive_rows(con)
except Exception as exc:
if _missing_table(exc):
typer.echo("no derived tables yet — run with --write")
return
raise
_print_families(rows, verbose=verbose)
finally:
con.close()
def _print_guidance(rows: list[Any]) -> None:
for r in sorted(rows, key=lambda r: (r.family, r.code, r.kind, r.locator)):
item = r.item_key or "—"
typer.echo(
f"{r.family} {r.kind} {r.locator} {item} ← {r.item_key_src}{r.p_id_src}"
)
def _resolve_guidance_families(con: Any, keys: list[str]) -> list[tuple[str, Any]]:
"""(key, Family) per requested key, refreshed from *con* — raises
``typer.BadParameter`` on the first unknown key, before any family's
rows are built or any write happens (mirrors ``_reaction_targets``'s
resolve-everything-up-front discipline)."""
refresh_from(con)
out: list[tuple[str, Any]] = []
for key in keys:
fam = FAMILIES.get(key)
if fam is None:
raise typer.BadParameter(
f"unknown family {key!r}; known: {', '.join(HAND_FAMILIES)}"
)
out.append((key, fam))
return out
@app.command()
def guidance(
family: list[str] = typer.Option(
[], "--family", help="Family key (CCM, APCM, …); repeatable."
),
write: bool = typer.Option(False, "--write"),
) -> None:
"""CFR/IOM/MLN sub-regulatory crosswalk per code family
(pfs.code_guidance): which CFR sections and IOM manual chapters the
family's FR paragraphs and the CPT manual cite, with FR provenance."""
if not family:
raise typer.BadParameter("pass --family (repeatable)")
keys = [normalize_key(f) for f in family]
store = _store()
if write:
# Ruling A13 (mirrors `stack pfs reaction`): resolve every family
# key up front and build every family's rows against a plain
# DuckDB read (I4) — build_guidance only reads pfs.cpt_* — never
# the write lock a notebook may be holding (#508-#514). The batch
# below only ever calls write_guidance and publishes.
con = _read()
try:
targets = _resolve_guidance_families(con, keys)
by_family: dict[str, list[Any]] = {}
rows: list[Any] = []
for key, fam in targets:
fam_rows = build_guidance(con, store, fam.codes, family=key)
by_family[key] = fam_rows
rows.extend(fam_rows)
finally:
con.close()
with _batch() as con:
ensure_tables(con)
for key, fam_rows in by_family.items():
write_guidance(con, key, fam_rows)
_publish()
_print_guidance(rows)
return
# No --write: read-only never needs the DuckDB single-writer lock a
# notebook may be holding (#508-#514) — read the replica (I4).
con = _read()
try:
refresh_from(con)
rows = []
for key in keys:
if key not in FAMILIES:
raise typer.BadParameter(
f"unknown family {key!r}; known: {', '.join(HAND_FAMILIES)}"
)
try:
rows.extend(read_guidance(con, key))
except Exception as exc:
if _missing_table(exc):
typer.echo(f"{key}: no derived guidance yet — run with --write")
continue
raise
_print_guidance(rows)
finally:
con.close()
def _print_reaction(rows: list[Any]) -> None:
for r in sorted(rows, key=lambda r: (r.family, r.period_kind, r.year, r.period)):
if r.period_kind == "docket":
stance = ""
if (
r.stance_support
or r.stance_oppose
or r.stance_modify
or r.stance_unclear
):
stance = (
f" stance support={r.stance_support} oppose={r.stance_oppose} "
f"modify={r.stance_modify} unclear={r.stance_unclear}"
)
typer.echo(
f"{r.family} {r.period} ({r.year}) {r.n_items}/{r.n_total} commenters{stance}"
)
else:
typer.echo(
f"{r.family} {r.period} CY{r.year} "
f"{r.n_items} comment/response pairs"
)
def _reaction_targets(
con: Any, family: list[str], code: list[str]
) -> list[tuple[str, tuple[str, ...], str | None]]:
"""(key, codes, display name) per target — *name* is the registered
family's own name for a ``--family`` target, ``None`` for a
``--code`` pseudo-family (``pfs.reaction.series`` falls back to the
key itself, which is already the right label for a bare code)."""
refresh_from(con)
targets: list[tuple[str, tuple[str, ...], str | None]] = []
for f in family:
key = normalize_key(f)
fam = FAMILIES.get(key)
if fam is None:
raise typer.BadParameter(
f"unknown family {key!r}; known: {', '.join(HAND_FAMILIES)}"
)
targets.append((key, fam.codes, fam.name))
for c in code:
key = c.upper()
targets.append((key, (key,), None))
return targets
def _stored(read: Any, con: Any, family: str) -> list[Any]:
"""``read(con, family)``; none when the table does not exist yet."""
try:
return read(con, family)
except Exception as exc: # noqa: BLE001 - only a missing table is tolerated
if is_missing_table_error(exc):
return []
raise
def _overlay_stances(
rows: list[Any], stances: list[Any], stored: list[Any], commenters: Counter
) -> list[Any]:
"""Put the classified counts of each docket's stored stances on its row,
so a reaction rerun never zeroes them. The event order and the stance
method record (``STANCE_METHOD_FIELDS``, R66a) are the ones the docket's
last ``stance`` run stored (*stored* are the family's stored reaction
rows); *commenters* counts ``pfs.commenter`` rows per docket (R62)."""
old = {r.period: r for r in stored}
out = []
for r in rows:
mine = [s for s in stances if s.docket == r.period]
if r.period_kind != "docket" or not mine:
out.append(r)
continue
prev = old.get(r.period)
order = list(json.loads(prev.events_json)) if prev else []
rolled = rollup(mine, base=r, order=order, n_commenters=commenters[r.period])
if prev:
rolled = replace(
rolled, **{f: getattr(prev, f) for f in STANCE_METHOD_FIELDS}
)
out.append(rolled)
return out
def _commenters_of(con: Any, family: str) -> list[Any]:
return read_commenters(con, family=family)
@app.command()
def reaction(
family: list[str] = typer.Option(
[], "--family", help="Family key (CCM, APCM, …); repeatable."
),
code: list[str] = typer.Option(
[],
"--code",
help="A single code as its own pseudo-family, keyed by the code; repeatable.",
),
stance_sample: int = typer.Option(
0,
"--stance-sample",
help="Classify the newest N commenters per docket into support/oppose/"
"modify/unclear via the self-hosted classifier (0 = skip; no stance "
"sample means no LLM call at all).",
),
write: bool = typer.Option(False, "--write"),
) -> None:
"""Public-reaction series per code family (pfs.code_reaction): comment
counts per docket (share of the docket's commenters mentioning the
family), an optional stance sample, and the FR Comment:/Response:
pair counts per rule as the pre-2017 proxy. Always recomputed live
from pgvector/bib — there is no persisted-table fallback to read
without --write, so --write only adds persisting the result."""
if not family and not code:
raise typer.BadParameter("pass --family or --code (repeatable)")
store = _store()
# Resolving family keys is a plain DuckDB read (I4) — never the
# write lock a notebook may be holding (#508-#514) — and it's the
# only DuckDB touch this command needs before writing.
con = _read()
try:
targets = _reaction_targets(con, family, code)
stored = {key: _stored(read_stances, con, key) for key, _, _ in targets}
old_rows = {key: _stored(read_reaction, con, key) for key in stored}
n_comm = {
key: Counter(c.docket for c in _stored(_commenters_of, con, key))
for key, v in stored.items()
if v
}
finally:
con.close()
if stance_sample > 0:
held = [k for k, v in stored.items() if v]
if held:
raise typer.BadParameter(
f"{', '.join(held)} already has stored stances (stack pfs stance); "
"a stance sample would be a second, weaker count"
)
# Ruling A13: the classifier and every family's series (docket
# counts, the stance sample — ~1-2s/sample — and FR pairs) all run
# against pgvector/bib, not DuckDB, so they run BEFORE any DuckDB
# batch is opened. The batch below only ever calls write_reaction
# and publishes — it never holds the single-writer lock (#508-#514)
# while the slow work happens.
classify = _classifier() if stance_sample > 0 else None
engine = _engine()
by_family: dict[str, list[Any]] = {}
rows: list[Any] = []
for key, codes, name in targets:
fam_rows = reaction_series(
engine,
store,
key,
codes,
classify=classify,
stance_sample_n=stance_sample,
name=name,
)
fam_rows = _overlay_stances(
fam_rows, stored[key], old_rows[key], n_comm.get(key, Counter())
)
by_family[key] = fam_rows
rows.extend(fam_rows)
if write:
with _batch() as con:
ensure_tables(con)
for key, fam_rows in by_family.items():
write_reaction(con, key, fam_rows)
_publish()
_print_reaction(rows)
def _print_exposures(rows: list[Any]) -> None:
for r in rows:
anchor = f"{r.item_key}{r.p_id}" if r.item_key else "unanchored"
if r.correction:
anchor += " (correction)"
extra = ""
if r.kind == "revalued":
extra = f" {r.delta_pct:+.1%}"
elif r.kind in ("becomes-payable", "ends", "status-change"):
extra = f" {r.prior_status or '—'}→{r.status or '—'}"
fam = f" [{r.family}]" if r.family else ""
typer.echo(
f"{r.code} {r.effective_date.isoformat()} {r.kind:<17}"
f"{extra} band {r.rvu_band}{fam} ← {anchor}"
)
per_kind: dict[str, int] = {}
for r in rows:
per_kind[r.kind] = per_kind.get(r.kind, 0) + 1
detail = ", ".join(f"{k}={v}" for k, v in sorted(per_kind.items())) or "none"
anchored = sum(1 for r in rows if r.item_key)
typer.echo(
f"exposures: {len(rows)} rows, {len({r.code for r in rows})} codes, "
f"anchored {anchored}, per kind: {detail}"
)
@app.command()
def exposure(
code: list[str] = typer.Option([], "--code", help="Filter to a code; repeatable."),
family: list[str] = typer.Option(
[], "--family", help="Filter to a family key (CCM, APCM, …); repeatable."
),
controls: int = typer.Option(
0,
"--controls",
help="With one --code and --year: list up to N never-treated control codes "
"(same status + RVU band, no exposure within --window years).",
),
year: int = typer.Option(
0, "--year", help="The exposure year --controls refers to."
),
window: int = typer.Option(
2, "--window", help="±years a control must be quiet for."
),
write: bool = typer.Option(
False,
"--write",
help="Rebuild the whole calendar into pfs.code_exposure and republish.",
),
) -> None:
"""The exposure calendar (pfs.code_exposure, #693): when each code
became payable, was revalued, changed payable status, ended, or was
listed for telehealth — dated 1 January of the rule year (or the
correction notice's date), anchored to the FR paragraph from its
lineage, with the status and RVU-decile band a control set matches on.
--write always rebuilds every code (control sets need the whole
calendar); --code/--family only filter what is printed."""
codes = [c.upper() for c in code]
keys = [normalize_key(f) for f in family]
def _filter(rows: list[Any]) -> list[Any]:
if not codes and not keys:
return rows
return [r for r in rows if r.code in codes or r.family in keys]
if write:
# Ruling A13: build against a plain read (I4) — never the write
# lock a notebook may be holding (#508-#514); the batch below only
# writes and publishes.
store = _store()
con = _read()
try:
rows = build_exposures(con, store)
finally:
con.close()
with _batch() as con:
ensure_tables(con)
write_exposures(con, rows)
_publish()
_print_exposures(_filter(rows))
return
con = _read()
try:
if controls:
if len(codes) != 1 or not year:
raise typer.BadParameter(
"--controls needs exactly one --code and --year"
)
try:
ctrl = control_codes(con, codes[0], year, window=window, limit=controls)
except Exception as exc:
if _missing_table(exc):
typer.echo("no exposure calendar yet — run with --write")
return
raise
for c, fam, status, band in ctrl:
typer.echo(f"{c} {status} band {band} {fam}")
typer.echo(
f"controls for {codes[0]} @ {year} (±{window}y): {len(ctrl)} codes"
)
return
try:
rows = read_exposures(con)
except Exception as exc:
if _missing_table(exc):
typer.echo("no exposure calendar yet — run with --write")
return
raise
_print_exposures(_filter(rows))
finally:
con.close()
def _http() -> Any:
import httpx
return httpx.Client(headers={"User-Agent": "stack-pfs-utilization/1.0"})
def _print_series(rows: list[Any]) -> None:
for r in rows:
svc = f"{r.n_services:,.0f}" if r.n_services is not None else "—"
ben = f"{r.n_beneficiaries:,}" if r.n_beneficiaries is not None else "—"
prv = f"{r.n_providers:,}" if r.n_providers is not None else "—"
allowed = f"${r.avg_allowed:,.2f}" if r.avg_allowed is not None else "—"
typer.echo(
f"{r.hcpcs} {r.year} services {svc:>12} benes {ben:>10} "
f"providers {prv:>8} avg allowed {allowed}"
)
typer.echo(f"utilization: {len(rows)} (code, year) rows")
@app.command()
def utilization(
year: list[int] = typer.Option(
[], "--year", help="Calendar year(s) to ingest with --write; default all 2013–."
),
code: list[str] = typer.Option(
[], "--code", help="Print the per-year series for these codes (repeatable)."
),
write: bool = typer.Option(
False,
"--write",
help="Fetch from data.cms.gov, load pfs.utilization, cite, republish.",
),
offline: bool = typer.Option(
False,
"--offline",
help="With --write: load from the cached JSON instead of the API.",
),
) -> None:
"""Part B utilization per HCPCS (pfs.utilization, #694): the CMS
"Medicare Physician & Other Practitioners — by Geography and Service"
national totals, one row per (year, code, place of service), with
each year's dataset cited in bib and logged in cms.ingest_log.
--write pulls every year (or --year) through the Data API, caches the
raw pages under data/cms/utilization/, replaces that year's rows and
republishes the replica; --code prints series from the replica."""
codes = [c.upper() for c in code]
if write:
years = sorted(year) if year else sorted(UTILIZATION_YEARS)
unknown = [y for y in years if y not in UTILIZATION_YEARS]
if unknown:
raise typer.BadParameter(
f"no dataset for {unknown}; known years "
f"{min(UTILIZATION_YEARS)}–{max(UTILIZATION_YEARS)}"
)
# Ruling A13: all network + bib work first, then one short batch
# that only writes and publishes — never the write lock a
# notebook may be holding (#508-#514) while the API is paged.
store = _store()
staged: list[tuple[int, list[Any], str, str]] = []
client = None if offline else _http()
try:
for y in years:
cache = utilization_cache_path(y)
if offline:
records = load_utilization_cache(cache)
else:
records = fetch_utilization_year(client, y)
save_utilization_cache(cache, records)
key = cite_utilization(store, y)
rows = utilization_rows(
y, records, dataset_id=UTILIZATION_YEARS[y], item_key=key
)
staged.append((y, rows, key, str(cache)))
typer.echo(
f"{y}: {len(records)} records → {len(rows)} national rows ← {key}"
)
finally:
if client is not None:
client.close()
from cms.ingest_log import log_ingest
with _batch() as con:
for y, rows, key, cache in staged:
n = write_utilization(con, y, rows)
log_ingest(
con,
module="pfs.utilization",
table_name="pfs.utilization",
rows=n,
source_file=cache,
pincite_key=key,
)
_publish()
if not codes:
return
con = _read()
try:
if not codes:
raise typer.BadParameter("pass --code (repeatable) or --write")
try:
rows = read_utilization_series(con, codes)
except Exception as exc:
if _missing_table(exc):
typer.echo("no utilization table yet — run with --write")
return
raise
_print_series(rows)
finally:
con.close()
def _psps_names(con: Any) -> dict[str, str]:
"""Medicare provider specialty code -> description."""
try:
rows = con.execute(
"SELECT specialty_code, specialty_description "
"FROM terminology.cms_acceptable_provider_specialty_codes"
).fetchall()
except Exception as exc:
if _missing_table(exc):
return {}
raise
return {str(c): str(d or "") for c, d in rows}
def _family_codes(con: Any, key: str) -> list[str]:
refresh_from(con)
fam = FAMILIES.get(normalize_key(key))
if fam is None:
raise typer.BadParameter(
f"unknown family {key!r}; known: {', '.join(HAND_FAMILIES)}"
)
return sorted(fam.codes)
@app.command()
def psps(
family: str = typer.Option(..., "--family", help="Family key, e.g. HOME-VISITS."),
year: list[int] = typer.Option(
[], "--year", help="Calendar year(s); default every PSPS year."
),
write: bool = typer.Option(
False, "--write", help="Fetch data.cms.gov, load, cite, republish."
),
offline: bool = typer.Option(
False, "--offline", help="With --write: use the cached JSON, no network."
),
) -> None:
"""Medicare utilization by provider specialty for a family's codes
(pfs.utilization_specialty): the PSPS national totals per code, year
and specialty. --write pulls each (year, code) through the Data API,
caches it under data/cms/psps/ (never overwriting a cached file) and
republishes; without --write it prints what the replica holds."""
import time
years = sorted(year) if year else sorted(PSPS_YEARS)
unknown = [y for y in years if y not in PSPS_YEARS]
if unknown:
raise typer.BadParameter(
f"no PSPS dataset for {unknown}; known {min(PSPS_YEARS)}-{max(PSPS_YEARS)}"
)
con = _read()
try:
codes = _family_codes(con, family)
names = _psps_names(con)
if not write:
try:
rows = read_psps_rows(con, codes, years=years)
except Exception as exc:
if _missing_table(exc):
typer.echo("no PSPS table yet - run with --write")
return
raise
finally:
con.close()
if not write:
for r in rows:
if r.services is not None:
svc = f"{r.services:,.0f}"
elif r.services_reported is not None:
svc = f">={r.services_reported:,.0f}" # suppressed lines omitted
else:
svc = "-"
typer.echo(
f"{r.year} {r.hcpcs} {r.specialty_cd} {r.specialty:<32} {svc:>12}"
)
typer.echo(f"psps: {len(rows)} rows")
return
store = _store()
staged: list[tuple[int, str, list[Any], str, str]] = []
client = None if offline else _http_dossier()
gap = ""
try:
for y in years:
key = cite_psps(store, y)
for code in codes:
cache = psps_cache_path(y, code)
if offline:
cache = newest_psps_cache(cache)
if not cache.is_file():
typer.echo(f"{y} {code}: no cached file, skipped")
continue
records = load_psps_cache(cache)
else:
try:
records = fetch_psps(client, y, code)
except httpx.HTTPStatusError as exc:
# a 403/429/bot page stops this source: keep what is
# staged, note the gap, never work around it
gap = (
f"{y} {code}: HTTP {exc.response.status_code}; "
"stopped fetching PSPS"
)
break
if records:
cache = save_psps_cache(cache, records)
time.sleep(0.2)
if not records:
# an empty answer must not delete rows nor log a source file
typer.echo(f"{y} {code}: no lines, kept existing rows")
continue
rows = psps_rows(
y,
code,
records,
dataset_id=PSPS_YEARS[y],
item_key=key,
names=names,
)
staged.append((y, code, rows, key, str(cache)))
typer.echo(
f"{y} {code}: {len(records)} lines -> {len(rows)} specialties"
)
if gap:
break
finally:
if client is not None:
client.close()
if gap:
typer.echo(f"gap: {gap}")
from cms.ingest_log import log_ingest
with _batch() as con:
for y, code, rows, key, cache in staged:
n = write_psps_rows(con, y, code, rows)
log_ingest(
con,
module="pfs.psps",
table_name="pfs.utilization_specialty",
rows=n,
source_file=cache,
pincite_key=key,
)
_publish()
typer.echo(f"psps: {sum(len(r) for _, _, r, _, _ in staged)} rows written")
def _http_dossier() -> Any:
return httpx.Client(headers={"User-Agent": PSPS_USER_AGENT})
def _seat_root() -> Path:
"""``data/ama/rosters``: the saved archive captures of AMA rosters."""
from conf import path
return path("db.aco").parent / "ama" / "rosters"
def _gap_status(exc: Exception) -> str:
if isinstance(exc, httpx.HTTPStatusError):
return str(exc.response.status_code)
return type(exc).__name__
@app.command("seats")
def seats(
body: str = typer.Option(..., "--body", help="RUC or CPT_PANEL."),
url: str = typer.Option(..., "--url", help="The AMA roster page (live URL)."),
year_from: int = typer.Option(2005, "--from", help="First year to capture."),
year_to: int = typer.Option(2026, "--to", help="Last year to capture."),
limit: int = typer.Option(0, "--limit", help="Cap the snapshots read (0 = all)."),
dry_run: bool = typer.Option(
False, "--dry-run", help="List the archive's captures; fetch and parse nothing."
),
cross_check: bool = typer.Option(
False,
"--cross-check",
help="Also ask the pinned local model for the organizations and log "
"where it disagrees with the parse (resumable ledger).",
),
write: bool = typer.Option(False, "--write"),
) -> None:
"""Seats on the RUC or the CPT Editorial Panel by year
(pfs.governance_seat), parsed deterministically from the Internet
Archive's copies of the AMA roster page (R88). Each capture is saved
under data/ama/rosters and cited as a bib Download. A year with no
capture, a refusal (403/429, bot page) or an unrecognised layout is a
gap line (and, with --write, an ingest_log row with rows=0)."""
import time
if body not in seat_lib.BODIES:
raise typer.BadParameter(f"--body must be one of {', '.join(seat_lib.BODIES)}")
if year_from > year_to:
raise typer.BadParameter("--from is after --to")
years = list(range(year_from, year_to + 1))
gaps: dict[int, str] = {}
refusal = ""
staged: list[tuple[int, str, str, Path, list[Any]]] = []
texts: dict[int, str] = {}
year_notes: dict[int, list[str]] = {}
client = _http_dossier()
try:
snaps = seat_lib.cdx_snapshots(client, url, years, gaps=gaps)
if gaps:
refusal = next(iter(gaps.values()))
found = {y for y, _ in snaps}
for y in years:
if y not in found and y not in gaps:
gaps[y] = "no archive capture"
if limit:
snaps = snaps[:limit]
if dry_run:
for y, ts in snaps:
typer.echo(f"{y} {ts} {seat_lib.snapshot_url(ts, url)}")
else:
store = _store()
root = _seat_root()
for n, (y, ts) in enumerate(snaps):
path = seat_lib.snapshot_path(root, body, y, ts)
if path.is_file(): # resumable: a saved capture is never refetched
raw = path.read_bytes()
else:
try:
raw = seat_lib.fetch_snapshot(
client, seat_lib.snapshot_url(ts, url)
)
except httpx.HTTPError as exc:
refusal = _gap_status(exc)
gaps[y] = refusal
for later, _ts in snaps[n + 1 :]:
gaps[later] = f"not tried after {refusal}"
break
time.sleep(1.0)
text = seat_lib.html_text(raw)
if seat_lib.is_refusal(text):
refusal = "bot page"
gaps[y] = refusal
for later, _ts in snaps[n + 1 :]:
gaps[later] = f"not tried after {refusal}"
break
path = seat_lib.save_snapshot(path, raw)
key = seat_lib.cite(store, body, y, url, ts, path)
try:
parsed, notes = (
seat_lib.parse_ruc(text)
if body == "RUC"
else seat_lib.parse_cpt(text)
)
except seat_lib.ParseError:
gaps[y] = "layout not recognised"
continue
for note in notes:
typer.echo(f"note: {body} {y}: {note}")
year_notes[y] = notes
texts[y] = text
staged.append(
(
y,
ts,
key,
path,
seat_lib.to_rows(
body, y, parsed, source_url=url, item_key=key, snapshot=ts
),
)
)
finally:
client.close()
if refusal:
typer.echo(f"gap: {body} roster not captured ({refusal})")
for y in sorted(gaps):
typer.echo(f"gap: {body} {y} not captured ({gaps[y]})")
if dry_run:
typer.echo(f"seats: {len(snaps)} snapshots; dry run")
return
rows: list[Any] = []
for y, ts, _key, _path, got in staged:
rows += got
kinds = Counter(r.seat_type for r in got)
typer.echo(
f"{y} {ts}: {sum(r.n_seats for r in got)} seats "
+ ", ".join(f"{k} {v}" for k, v in sorted(kinds.items()))
)
typer.echo(f"seats: {len(rows)} rows from {len(staged)} snapshots")
if cross_check and staged:
_seats_cross_check(body, staged, texts)
if write and (rows or gaps):
from cms.ingest_log import ensure_table, log_ingest
def _log(bcon: Any, *, source: str, rule: str = "", **kw: Any) -> None:
"""One ingest_log row, replacing any earlier row for the same
source and kind so a rerun does not duplicate it."""
bcon.execute(
"DELETE FROM cms.ingest_log WHERE module = 'pfs.seats' "
"AND source_file = ? AND rule_id = ?",
[source, rule],
)
log_ingest(
bcon,
module="pfs.seats",
table_name=seat_lib.TABLE,
source_file=source,
rule_id=rule,
**kw,
)
with _batch() as bcon:
ensure_table(bcon)
for y in sorted({*gaps, *(st[0] for st in staged)}):
# a rerun replaces this year's gap and note rows (also the
# earlier unkinded gap rows) instead of adding to them
bcon.execute(
"DELETE FROM cms.ingest_log WHERE module = 'pfs.seats' "
"AND starts_with(source_file, ?)",
[f"{url}#{y} "],
)
if rows:
seat_lib.write_seats(bcon, body, rows)
for y, _ts, ikey, src, got in staged:
_log(bcon, source=str(src), rows=len(got), pincite_key=ikey)
for note in year_notes.get(y, []):
# the page disagrees with itself: kept for the dossier
_log(
bcon,
source=f"{url}#{y} note: {note}",
rule="note",
rows=len(got),
pincite_key=ikey,
)
for y in sorted(gaps):
_log(bcon, source=f"{url}#{y} ({gaps[y]})", rule="gap", rows=0)
_publish()
typer.echo(f"wrote {len(rows)} seats")
def _seats_cross_check(body: str, staged: Sequence[Any], texts: dict[int, str]) -> None:
"""Ask the pinned model for each page's organizations and log where it
disagrees with the parsed rows. The parse is the record; this only flags
pages worth a look. Resumable through the run ledger."""
from pfs.commenters import specialty_vocab
from pfs.seats import _key # the crosswalk's text normaliser
con = _read()
try:
specialties = specialty_vocab(con)
finally:
con.close()
ask = _asker()
version = f"{seat_lib.SEAT_PROMPT_VERSION}-xcheck"
log = RunLog(ledger_path("seats", body))
parsed = {
y: sorted({_key(r.organization) for r in got}) for y, _t, _k, _p, got in staged
}
def unit(year: int) -> Any:
def run() -> dict | None:
try:
found = seat_lib.extract_seats(
texts[year], ask, specialties=specialties
)
except seat_lib.ExtractionFailed:
return None
model = sorted({_key(it["organization"]) for it in found})
return {
"model_only": [m for m in model if m not in parsed[year]],
"parsed_only": [p for p in parsed[year] if p not in model],
}
return run
units = [
(f"{body}:{y}:{ts}:xcheck", input_hash(texts[y]), unit(y))
for y, ts, *_ in staged
]
run_units(
units,
log,
model=ask.model,
version=version,
retry_failed=True,
progress=lambda u, status: typer.echo(f"{u} {status}"),
)
for y, ts, *_ in staged:
entry = log.get(
f"{body}:{y}:{ts}:xcheck", input_hash(texts[y]), ask.model, version
)
if entry is None or entry.output is None:
typer.echo(f"cross-check {y}: model gave no valid reply")
continue
out = entry.output
if not out["model_only"] and not out["parsed_only"]:
typer.echo(f"cross-check {y}: model and parse agree")
continue
typer.echo(
f"cross-check {y}: model-only {len(out['model_only'])}: "
f"{'; '.join(out['model_only'])} | parsed-only {len(out['parsed_only'])}: "
f"{'; '.join(out['parsed_only'])}"
)
@app.command()
def impacts(
scan: bool = typer.Option(
False,
"--scan",
help="Every CMS 508 impact file in bib / data/cms/pfs_final and every "
"PFS rule whose FR text prints the table.",
),
path: Path = typer.Option(None, "--path", help="One impact file or FR text."),
year: int = typer.Option(0, "--year", help="Rule year, with --path."),
vintage: str = typer.Option("final", "--vintage", help="final | proposed"),
write: bool = typer.Option(False, "--write"),
) -> None:
"""CMS's estimated impact on total allowed charges by specialty
(pfs.specialty_impact): the 508 text files CMS posts with a rule and
the table printed in the 2012-2019 Federal Register texts."""
from conf import ROOT
if scan == (path is not None):
raise typer.BadParameter("pass exactly one of --scan or --path")
if vintage not in ("final", "proposed"):
raise typer.BadParameter("--vintage is final or proposed")
groups: list[tuple[int, str, str, list[Any], str, str]] = []
if scan:
store = _store()
sources = discover_508(store, ROOT / "data/cms/pfs_final")
sources += discover_fr(store)
else:
if not year:
raise typer.BadParameter("--path needs --year")
kind = "508" if path.name.endswith("_508.txt") else "fr"
from pfs.stakes import ImpactSource
sources = [ImpactSource(path, year, vintage, "", kind)]
for src in sources:
if src.kind == "508":
rows = parse_impact_508(
src.path,
rule_year=src.rule_year,
vintage=src.vintage,
item_key=src.item_key,
)
else:
rows = parse_impact_fr_text(
src.path.read_text(errors="replace"),
rule_year=src.rule_year,
vintage=src.vintage,
item_key=src.item_key,
)
groups.append(
(src.rule_year, src.vintage, src.kind, rows, src.item_key, str(src.path))
)
typer.echo(
f"CY{src.rule_year} {src.vintage} ({src.kind}): {len(rows)} rows "
f"<- {src.path.name}"
)
if not write:
return
from cms.ingest_log import log_ingest
with _batch() as con:
for ry, vin, kind, rows, key, file in groups:
n = write_impacts(con, ry, vin, rows, source=kind)
log_ingest(
con,
module="pfs.stakes",
table_name="pfs.specialty_impact",
rows=n,
source_file=file,
pincite_key=key,
)
_publish()
@app.command()
def review(code: str = typer.Option("", "--code")) -> None:
"""Element lines the classifier could not place."""
con = _read()
rows = read_reviews(con, code.upper()) if code else read_all_reviews(con)
for r in rows:
src = f" ({r.source})" if r.source else ""
typer.echo(
f"{r.code} {r.item_key}{r.p_id}{src} [{r.proposed_value or '?'}] "
f"{r.text[:120]}"
)
def _print_cpt_counts(counts: dict[int, dict[str, int]]) -> None:
for year in sorted(counts):
c = counts[year]
detail = ", ".join(f"{k}={v}" for k, v in c.items())
typer.echo(f"{year}: {detail}")
@app.command("cpt-ingest")
def cpt_ingest_cmd(
edition: list[int] = typer.Option(
[], "--edition", help="CPT edition year to ingest (repeatable)."
),
all_editions: bool = typer.Option(
False, "--all", help="Every CPT EPUB edition found in bib (source:ama)."
),
dry_run: bool = typer.Option(
False, "--dry-run", help="Parse and report counts; write nothing."
),
) -> None:
"""Parse CPT EPUB editions into pfs.cpt_section/cpt_code/cpt_instruction/
cpt_reference/cpt_crosswalk/cpt_list (docs/superpowers/specs/
2026-09-09-cpt-canonical-schema-design.md §3)."""
if not edition and not all_editions:
raise typer.BadParameter("pass --edition YEAR (repeatable) or --all")
years = None if all_editions else sorted(set(edition))
store = _store()
if dry_run:
# A preview never needs the DuckDB single-writer lock a notebook
# may be holding (#508-#514) — parsing alone answers a dry run,
# so no connection is opened at all (I4).
counts = cpt_ingest(store, None, years=years, dry_run=True)
_print_cpt_counts(counts)
return
with _batch() as con:
# write_cpt_edition (via cpt_ingest) already calls ensure_tables
# itself — no need to duplicate that here.
counts = cpt_ingest(store, con, years=years, dry_run=False)
_print_cpt_counts(counts)
_publish()
@app.command()
def rounds(
family: str = typer.Option(..., "--family", help="Curated family key."),
tables_only: bool = typer.Option(
False,
"--tables-only",
help="No model calls (as --no-llm); the finished ledger is replayed.",
),
no_llm: bool = typer.Option(
False, "--no-llm", help="Make no model calls; the finished ledger is replayed."
),
limit: int = typer.Option(0, "--limit", help="Cap new model calls (0 = all)."),
dry_run: bool = typer.Option(
False, "--dry-run", help="Count candidates; no model calls, no writes."
),
retry_failed: bool = typer.Option(
False, "--retry-failed", help="Re-run ledger entries that failed."
),
write: bool = typer.Option(False, "--write"),
) -> None:
"""Valuation rounds per code and rule (pfs.valuation_round): RUC
recommendation vs CMS proposal/final, from FR tables, regex over FR
paragraphs, and the local model through a resumable ledger."""
fam = curated_family(normalize_key(family))
if fam is None:
raise typer.BadParameter(f"unknown curated family {family!r}")
store = _store()
if dry_run:
paras, table = rounds_inventory(store, fam)
typer.echo(
f"{fam.key}: {len(paras)} candidate paragraphs, "
f"{len(table)} table rows; no model calls made"
)
return
replay = tables_only or no_llm
ask = None if replay else _asker()
log = RunLog(ledger_path("rounds", fam.key))
con = _read()
try:
rows, counts = family_rounds(
con,
store,
fam,
ask=ask,
log=log,
limit=limit,
tables_only=tables_only,
retry_failed=retry_failed,
replay_model=_model_name() if replay else None,
)
finally:
con.close()
by_year: dict[tuple[int, str], int] = {}
for r in rows:
by_year[(r.rule_year, r.rule_kind)] = (
by_year.get((r.rule_year, r.rule_kind), 0) + 1
)
for (year, kind), n in sorted(by_year.items()):
typer.echo(f"CY{year} {kind}: {n} rounds")
typer.echo(
f"rounds: {len(rows)}; done {counts['done']}, skipped {counts['skipped']}, "
f"failed {counts['failed']}, failed_pending {counts['failed_pending']}"
)
if write:
with _batch() as con:
ensure_tables(con)
n = write_rounds(con, fam.key, rows)
typer.echo(f"wrote {n} rounds")
_publish()
def _origin_books(store: Any, keys: list[str]) -> list[tuple[str, str, Path]]:
"""The EPUBs to read. A book asked for with ``--book`` must exist; the
default one that is missing is skipped with a warning (an FR-only run)."""
books = []
for key in keys or [DEFAULT_BOOK]:
found = resolve_book(store, key)
if found is not None:
books.append((key, found[0], found[1]))
elif keys:
raise typer.BadParameter(f"no EPUB attachment for bib item {key}")
else:
typer.echo(f"warning: no EPUB attachment for {key}; skipping the book")
return books
@app.command()
def origin(
family: str = typer.Option(..., "--family", help="Curated family key."),
book: list[str] = typer.Option(
[],
"--book",
help=f"Bib item of a licensed EPUB to read (default {DEFAULT_BOOK}, "
"CPT Changes 2023); repeatable.",
),
limit: int = typer.Option(0, "--limit", help="Cap new model calls (0 = all)."),
dry_run: bool = typer.Option(
False, "--dry-run", help="Count candidates; no model calls, no writes."
),
no_llm: bool = typer.Option(
False, "--no-llm", help="Make no model calls; the finished ledger is replayed."
),
retry_failed: bool = typer.Option(
False, "--retry-failed", help="Re-run ledger entries that failed."
),
write: bool = typer.Option(False, "--write"),
) -> None:
"""Origin and formation rows per code (pfs.code_origin): the statute,
regulation, program, demonstration, CPT Editorial Panel action, G-code
creation or coverage rule behind them, from Federal Register
paragraphs, the licensed CPT Changes book (paraphrase + page only) and
the guidance rows, through a resumable local-model ledger."""
fam = curated_family(normalize_key(family))
if fam is None:
raise typer.BadParameter(f"unknown curated family {family!r}")
store = _store()
books = _origin_books(store, book)
if dry_run:
paras, pages = origin_inventory(store, fam, books)
typer.echo(f"{fam.key}: {len(paras)} candidate Federal Register paragraphs")
for key, labels in pages.items():
typer.echo(f"{key}: {len(labels)} pages naming a code")
typer.echo("no model calls made")
return
ask = None
if not no_llm:
try:
ask = _asker()
except RuntimeError as exc:
typer.echo(f"no model host available ({exc}); replaying the ledger")
log = RunLog(ledger_path("origin", fam.key))
con = _read()
try:
rows, counts = family_origin(
con,
store,
fam,
ask=ask,
log=log,
books=books,
limit=limit,
retry_failed=retry_failed,
replay_model=_model_name() if ask is None else None,
)
finally:
con.close()
for field in ("kind", "cite_kind"):
tally: dict[str, int] = {}
for r in rows:
tally[getattr(r, field)] = tally.get(getattr(r, field), 0) + 1
for name, n in sorted(tally.items()):
typer.echo(f"{field} {name}: {n}")
typer.echo(
f"origins: {len(rows)}; done {counts['done']}, skipped {counts['skipped']}, "
f"failed {counts['failed']}, failed_pending {counts['failed_pending']}"
)
if write:
with _batch() as con:
ensure_tables(con)
n = write_origin(con, fam.key, rows)
typer.echo(f"wrote {n} origins")
_publish()
@app.command("guideline-facts")
def guideline_facts_cmd(
family: str = typer.Option(..., "--family", help="Curated family key."),
limit: int = typer.Option(0, "--limit", help="Cap new model calls (0 = all)."),
no_llm: bool = typer.Option(
False, "--no-llm", help="Make no model calls; the finished ledger is replayed."
),
retry_failed: bool = typer.Option(
False, "--retry-failed", help="Re-run ledger entries that failed."
),
write: bool = typer.Option(False, "--write"),
) -> None:
"""Guideline facts from the licensed CPT books (pfs.code_guideline_fact):
selection basis, MDM level, time threshold, add-on and deletion per code
and edition, scope points and the Panel action as paraphrases, each cited
by (edition, page). Book text is never stored."""
fam = curated_family(normalize_key(family))
if fam is None:
raise typer.BadParameter(f"unknown curated family {family!r}")
store = _store()
ask = None
if not no_llm:
try:
ask = _asker()
except RuntimeError as exc:
typer.echo(f"no model host available ({exc}); replaying the ledger")
counts: dict[str, int] = {}
con = _read()
try:
rows = guideline_facts(
con,
store,
fam,
ask=ask,
log=RunLog(ledger_path("guideline", fam.key)),
limit=limit,
retry_failed=retry_failed,
replay_model=_model_name() if ask is None else None,
counts=counts,
)
finally:
con.close()
tally: dict[tuple[int, str], int] = {}
for r in rows:
tally[(r.edition_year, r.fact)] = tally.get((r.edition_year, r.fact), 0) + 1
for (year, fact), n in sorted(tally.items()):
typer.echo(f"{year} {fact}: {n}")
typer.echo(
f"facts: {len(rows)}; done {counts.get('done', 0)}, skipped "
f"{counts.get('skipped', 0)}, failed {counts.get('failed', 0)}, "
f"failed_pending {counts.get('failed_pending', 0)}, "
f"cited by section (no page) {counts.get('unpaged', 0)}"
)
if write:
with _batch() as bcon:
ensure_tables(bcon)
n = write_guideline_facts(bcon, fam.key, rows)
typer.echo(f"wrote {n} facts")
_publish()
@app.command("cms-summaries")
def cms_summaries(
family: str = typer.Option(..., "--family", help="Curated family key."),
no_llm: bool = typer.Option(
False, "--no-llm", help="Make no model calls; the finished ledger is replayed."
),
limit: int = typer.Option(0, "--limit", help="Cap new model calls (0 = all)."),
retry_failed: bool = typer.Option(
False, "--retry-failed", help="Re-run ledger entries that failed."
),
write: bool = typer.Option(False, "--write"),
) -> None:
"""CMS's own Comment:/Response: summaries (pfs.cms_comment_summary):
CMS's quantifier, the stance of the summarized comments and CMS's
response action, per pair a rule prints. Kept apart from our own
commenter counts."""
fam = curated_family(normalize_key(family))
if fam is None:
raise typer.BadParameter(f"unknown curated family {family!r}")
store = _store()
model = _model_name()
counts: dict[str, int] = {}
rows = cms_summaries_rows(
store,
fam,
classify=None if no_llm else _classifier(pin=True),
model=model,
log=RunLog(ledger_path("cms-summaries", fam.key)),
limit=limit,
retry_failed=retry_failed,
replay_model=model if no_llm else None,
counts=counts,
)
by_year: dict[tuple[int, str], int] = {}
for r in rows:
by_year[(r.rule_year, r.rule_kind)] = (
by_year.get((r.rule_year, r.rule_kind), 0) + 1
)
for (year, kind), n in sorted(by_year.items()):
typer.echo(f"CY{year} {kind}: {n} summaries")
typer.echo(
f"summaries: {len(rows)}; done {counts.get('done', 0)}, "
f"skipped {counts.get('skipped', 0)}, failed {counts.get('failed', 0)}, "
f"failed_pending {counts.get('failed_pending', 0)}"
)
if write:
with _batch() as con:
ensure_tables(con)
n = write_cms_summaries(con, fam.key, rows)
typer.echo(f"wrote {n} summaries")
_publish()
@app.command("commenters")
def commenters(
family: str = typer.Option(..., "--family", help="Curated family key."),
docket: str = typer.Option(..., "--docket", help="regulations.gov docket id."),
limit: int = typer.Option(0, "--limit", help="Cap new model calls (0 = all)."),
retry_failed: bool = typer.Option(
False, "--retry-failed", help="Re-run ledger entries that failed."
),
dry_run: bool = typer.Option(
False,
"--dry-run",
help="Count the family's comments and regulations.gov categories; no model.",
),
write: bool = typer.Option(False, "--write"),
) -> None:
"""Who wrote each family comment (pfs.commenter): organization, type,
specialty, form-letter campaign and the submitter name as the
regulations.gov record posts it (official public record)."""
fam = curated_family(normalize_key(family))
if fam is None:
raise typer.BadParameter(f"unknown curated family {family!r}")
engine = _engine()
if dry_run:
items = family_comment_items(
engine, fam.codes, fam.key, fam.pair_phrases, docket=docket
)
mirror = _mirror()
_, regs = regs_for(_store(), mirror, docket, [k for k, _ in items])
cats = Counter(r.category for r in regs.values())
if len(regs) < len(items):
typer.echo(f" unreadable mirror JSON: {len(items) - len(regs)}")
typer.echo(f"{docket}: {len(items)} items; no model calls made")
for cat, n in sorted(cats.items(), key=lambda kv: (-kv[1], kv[0])):
typer.echo(f" category {cat or '(none)'}: {n}")
return
con = _read()
try:
rows, counts = identify(
engine,
_store(),
con,
fam,
docket=docket,
mirror=_mirror(),
ask=_asker(),
log=RunLog(ledger_path("commenters", fam.key)),
limit=limit,
retry_failed=retry_failed,
progress=lambda unit, status: typer.echo(f"{unit} {status}"),
)
finally:
con.close()
types = Counter(r.org_type for r in rows)
for t, n in sorted(types.items()):
typer.echo(f"{t}: {n}")
typer.echo(
f"commenters: {len(rows)}; done {counts['done']}, skipped {counts['skipped']}, "
f"failed {counts['failed']}, failed_pending {counts['failed_pending']}, "
f"pending {counts['pending']}, blocked {counts['blocked']}"
)
if write:
if counts["pending"]:
raise typer.BadParameter(
f"{counts['pending']} comments not yet identified; rerun to resume"
)
with _batch() as bcon:
n = write_commenters(bcon, rows, docket=docket, family=fam.key)
typer.echo(f"wrote {n} commenters")
_publish()
@app.command("stance")
def stance(
family: str = typer.Option(..., "--family", help="Curated family key."),
docket: str = typer.Option(..., "--docket", help="regulations.gov docket id."),
limit: int = typer.Option(0, "--limit", help="Cap new model calls (0 = all)."),
retry_failed: bool = typer.Option(
False, "--retry-failed", help="Re-run ledger entries that failed."
),
dry_run: bool = typer.Option(
False, "--dry-run", help="Count commenters and events; no model."
),
no_llm: bool = typer.Option(
False, "--no-llm", help="Make no model calls; the finished ledger is replayed."
),
method: str = typer.Option(
DEFAULT_METHOD, "--method", help="multi (one call per comment) or per-event."
),
model: str = typer.Option(
DEFAULT_MODEL, "--model", help="instruct or large (the pinned model)."
),
golden: Path = typer.Option(
GOLDEN_PATH,
"--golden",
help="Tuning (in-sample) golden set --write records as in_sample_record.",
),
heldout: list[Path] = typer.Option(
list(HELDOUT_PATHS),
"--heldout",
help="Held-out golden sets; the primary one is the stamped measurement.",
),
generation: str = typer.Option(
DEFAULT_GENERATION,
"--generation",
help="Prompt generation: stance-v4 or stance-v5 (CPT guideline facts).",
),
write: bool = typer.Option(False, "--write"),
) -> None:
"""Each commenter's stance (support / modify / oppose / unclear) on each
family event the docket's rule touched, with a reason and a verbatim
quote (pfs.comment_stance); --write also rolls the counts up into the
docket's pfs.code_reaction row. CMS's own summaries are never used."""
if method not in METHODS:
raise typer.BadParameter(f"--method must be one of {', '.join(METHODS)}")
if model not in ("instruct", "large"):
raise typer.BadParameter("--model must be instruct or large")
if generation not in GENERATIONS:
raise typer.BadParameter(
f"--generation must be one of {', '.join(GENERATIONS)}"
)
fam = curated_family(normalize_key(family))
if fam is None:
raise typer.BadParameter(f"unknown curated family {family!r}")
store = _store()
found = [d for d in store.dockets() if d.id == docket]
if not found:
raise typer.BadParameter(f"unknown docket {docket!r}")
year = docket_rule_year(found[0].comment_end_date)
con = _read()
try:
events = docket_events(
con,
fam,
year,
store,
guidelines=guideline_text(generation),
fixed=fixed_text(generation),
)
commenter_rows = [
c for c in read_commenters(con, family=fam.key) if c.docket == docket
]
stored = _stored(read_reaction, con, fam.key)
finally:
con.close()
if not commenter_rows:
typer.echo(f"{docket}: no commenter rows (run stack pfs commenters first)")
return
# the item set is the commenter rows, joined to the family's comment text
wanted = {c.item_key for c in commenter_rows}
texts = {
k: t
for k, t in family_comment_items(
_engine(),
fam.codes,
fam.key,
fam.pair_phrases,
docket=docket,
max_chars=TEXT_CHARS,
max_chunks=TEXT_CHUNKS,
)
if k in wanted
}
with_text = sum(1 for c in commenter_rows if c.item_key in texts)
typer.echo(
f"{docket}: {len(commenter_rows)} commenters, {with_text} with comment "
f"text, {len(events)} events"
)
for e in events:
typer.echo(f" {e.event_key}: {e.text}")
if dry_run:
typer.echo("no model calls made")
return
large = model == "large"
ask = None if no_llm else (_asker(large=True) if large else _asker())
name = _model_of(ask, large)
rows, counts = stances_for(
commenter_rows,
texts,
events,
ask,
RunLog(ledger_path("stance", fam.key)),
family=fam.key,
docket=docket,
method=method,
generation=generation,
model=name if no_llm else None,
subjects=event_subjects(fam, events, by_kind=guideline_text(generation)),
limit=limit,
retry_failed=retry_failed,
progress=lambda unit, status: typer.echo(f"{unit} {status}"),
)
blank = ReactionRow(fam.key, docket, "docket", year, 0, 0, 0, 0, 0, 0, "[]")
order = [e.event_key for e in events]
rolled = rollup(rows, base=blank, order=order, n_commenters=len(commenter_rows))
for key, n in json.loads(rolled.events_json).items():
typer.echo(
f"{key}: support {n['support']}, modify {n['modify']}, "
f"oppose {n['oppose']}, unclear {n['unclear']}"
)
typer.echo(
f"commenters classified: {rolled.n_classified} of {len(commenter_rows)}; "
f"support {rolled.stance_support}, modify {rolled.stance_modify}, "
f"oppose {rolled.stance_oppose}, unclear {rolled.stance_unclear}"
)
typer.echo(
f"stances: {len(rows)}; done {counts['done']}, skipped {counts['skipped']}, "
f"failed {counts['failed']}, failed_pending {counts['failed_pending']}, "
f"pending {counts['pending']}, no_text {counts['no_text']}"
)
if not write:
return
if counts["pending"]:
raise typer.BadParameter(
f"{counts['pending']} comments not yet classified; rerun to resume"
)
series_rows = reaction_series(_engine(), store, fam.key, fam.codes, name=fam.name)
base = next(
(r for r in series_rows if r.period_kind == "docket" and r.period == docket),
blank,
)
new = replace(
rollup(rows, base=base, order=order, n_commenters=len(commenter_rows)),
stance_method=f"{prompt_version(method, generation)}:{name}",
stance_agreement=None,
stance_eval=_method_eval(
golden, method, name, fam.key, generation, tuple(heldout)
),
)
keep = [
r
for r in (stored or series_rows)
if r.period != docket or r.period_kind != "docket"
]
with _batch() as bcon:
ensure_tables(bcon)
n = write_stances(bcon, fam.key, docket, rows)
write_reaction(bcon, fam.key, [*keep, new])
typer.echo(f"wrote {n} stances")
_publish()
def _method_eval(
golden: Path,
method: str,
name: str,
family: str,
generation: str | None = None,
heldout: Sequence[Path] = HELDOUT_PATHS,
) -> str | None:
"""The method's published measurement as ``stance_eval`` JSON (R82):
the family's PRIMARY held-out set for *generation*, replayed from the
ledger (no model call), with the in-sample tuning set's scores beside it
as ``in_sample_record``. ``None`` when no primary set matches the family
and generation; a set the ledger does not fully cover is an error, never
a partial measurement."""
gen = generation or DEFAULT_GENERATION
primary = None
for path in heldout:
if not path.exists():
continue
data = json.loads(path.read_text(encoding="utf-8"))
if (
data.get("role") == "primary"
and data.get("family") == family
and data.get("generation", gen) == gen
):
primary = data
break
if primary is None:
return None
sc, _ = _golden_run(primary, method, ask=None, name=name, generation=generation)
if sc is None:
raise typer.BadParameter(
f"held-out set not fully classified for {method} on {name}; "
"run stack pfs stance-eval on it first"
)
rec = eval_record(primary, sc)
rec.pop("docket")
rec["generation"] = gen
if golden.exists():
tuning = json.loads(golden.read_text(encoding="utf-8"))
if tuning.get("family") == family:
tsc, _ = _golden_run(
tuning, method, ask=None, name=name, generation=generation
)
if tsc is None:
raise typer.BadParameter(
f"tuning set not fully classified for {method} on {name}"
)
t = eval_record(tuning, tsc)
rec["in_sample_record"] = {
k: t[k]
for k in (
"cell",
"recall",
"direction",
"n_items",
"n_positions",
"docket",
)
}
return json.dumps(rec)
def _golden_run(
data: dict,
method: str,
*,
ask: Any,
name: str,
generation: str | None = None,
relevance: bool = True,
limit: int = 0,
retry_failed: bool = False,
progress: Any = None,
) -> tuple[Any, dict[str, int]]:
"""Classify the golden items with *method* through the stance ledger (a
``None`` *ask* only replays model *name*'s entries) and score them. The
scores are ``None`` while any item is still pending."""
fam = curated_family(normalize_key(data["family"]))
if fam is None:
raise typer.BadParameter(f"unknown curated family {data['family']!r}")
store = _store()
of_docket = item_dockets(data)
all_rows: list[Any] = []
counts = {
"done": 0, "skipped": 0, "failed": 0, "failed_pending": 0, "pending": 0,
"no_text": 0,
} # fmt: skip
remaining = limit
for docket in golden_dockets(data):
found = [d for d in store.dockets() if d.id == docket]
if not found:
raise typer.BadParameter(f"unknown docket {docket!r}")
con = _read()
try:
events = docket_events(
con,
fam,
docket_rule_year(found[0].comment_end_date),
store,
guidelines=guideline_text(generation),
fixed=fixed_text(generation),
)
finally:
con.close()
keys = [k for k, d in of_docket.items() if d == docket]
texts = {
k: t
for k, t in family_comment_items(
_engine(),
fam.codes,
fam.key,
fam.pair_phrases,
docket=docket,
max_chars=TEXT_CHARS,
max_chunks=TEXT_CHUNKS,
)
if k in keys
}
# the call limit is shared across dockets; once spent, only replay
live = ask if (ask is not None and (not limit or remaining > 0)) else None
rows, got = stances_for(
[SimpleNamespace(item_key=k) for k in keys],
texts,
events,
live,
RunLog(ledger_path("stance", fam.key)),
family=fam.key,
docket=docket,
method=method,
generation=generation,
model=name if live is None else None,
subjects=(
event_subjects(fam, events, by_kind=guideline_text(generation))
if relevance
else None
),
limit=remaining,
retry_failed=retry_failed,
progress=progress,
)
remaining -= got["done"] + got["failed"]
all_rows += rows
counts = {k: counts[k] + got[k] for k in counts}
rows, counts = all_rows, counts
incomplete = counts["pending"] or counts["no_text"]
return (None if incomplete else score(data, rows)), counts
@app.command("stance-eval")
def stance_eval(
golden: Path = typer.Option(..., "--golden", help="Golden stance JSON."),
method: str = typer.Option(DEFAULT_METHOD, "--method", help="multi or per-event."),
model: str = typer.Option("instruct", "--model", help="instruct or large."),
limit: int = typer.Option(0, "--limit", help="Cap new model calls (0 = all)."),
retry_failed: bool = typer.Option(False, "--retry-failed"),
no_llm: bool = typer.Option(
False, "--no-llm", help="Make no model calls; the finished ledger is replayed."
),
no_relevance: bool = typer.Option(
False, "--no-relevance", help="Skip the event-subject filter (R65)."
),
generation: str = typer.Option(
DEFAULT_GENERATION, "--generation", help="stance-v4 or stance-v5."
),
) -> None:
"""Score a stance method against a hand-coded golden set: cell agreement,
presence agreement, firm-stance accuracy and a confusion matrix. The
golden items are classified through the stance ledger (resumable); nothing
is written to pfs.comment_stance."""
import json
import time
if method not in METHODS:
raise typer.BadParameter(f"--method must be one of {', '.join(METHODS)}")
if model not in ("instruct", "large"):
raise typer.BadParameter("--model must be instruct or large")
if generation not in GENERATIONS:
raise typer.BadParameter(
f"--generation must be one of {', '.join(GENERATIONS)}"
)
data = json.loads(golden.read_text(encoding="utf-8"))
large = model == "large"
ask = None if no_llm else _asker(large=large)
name = _model_of(ask, large)
began = time.monotonic()
sc, counts = _golden_run(
data,
method,
ask=ask,
name=name,
generation=generation,
relevance=not no_relevance,
limit=limit,
retry_failed=retry_failed,
progress=lambda unit, status: typer.echo(f"{unit} {status}"),
)
calls = counts["done"] + counts["failed"]
typer.echo(
f"{prompt_version(method, generation)} {method} on {name}"
f"{' (no relevance filter)' if no_relevance else ''}: "
f"{calls} new calls in {time.monotonic() - began:.0f}s; done {counts['done']}, "
f"skipped {counts['skipped']}, failed {counts['failed']}, failed_pending "
f"{counts['failed_pending']}, pending {counts['pending']}, "
f"no_text {counts['no_text']}"
)
if sc is None:
typer.echo("incomplete: rerun to resume; no scores yet")
return
typer.echo(
f"cells {sc.cells}: agreement {sc.cell_agreement:.3f} ({sc.agree}); "
f"presence {sc.presence_agreement:.3f} ({sc.presence_agree}); "
f"firm accuracy {sc.firm_accuracy:.3f} ({sc.firm_correct}/{sc.firm_cells}); "
f"gate {'PASS' if sc.passes() else 'FAIL'}"
)
typer.echo(
f"recall {sc.recall:.3f} ({sc.found}/{sc.positions}); "
f"direction {sc.direction:.3f} ({sc.right}/{sc.found})"
)
labels = list(sc.confusion)
typer.echo(
"confusion (rows golden, columns model): " + " ".join(f"{x:>8}" for x in labels)
)
for g in labels:
typer.echo(f"{g:>8} " + " ".join(f"{sc.confusion[g][m]:>8}" for m in labels))
for d in sc.disagreements[:10]:
typer.echo(
f"disagree {d.item_key} {d.event_key.rsplit(':', 1)[-1]}: "
f"golden {d.golden}, model {d.model}"
)
def _model_of(ask: Any, large: bool) -> str:
"""The pinned model's name: the asker's, or (no asker) the configured one."""
if ask is not None:
return ask.model
return _model_name(large=True) if large else _model_name()
def _excerpts(store: Any, name: str) -> list[Any]:
"""Governance excerpts for a family (embeddings on the llm pool; no
generation)."""
from llm import config as llm_config
from llm.pool import HostPool
cfg = llm_config.load()
return governance_excerpts(cfg, HostPool.from_config(cfg), store, name)
@app.command("dossier")
def dossier_cmd(
family: str = typer.Option("", "--family", help="Curated family key."),
all_families: bool = typer.Option(False, "--all", help="Every curated family."),
no_llm: bool = typer.Option(
False,
"--no-llm",
help="Make no model calls; the finished prose ledger is replayed.",
),
limit: int = typer.Option(0, "--limit", help="Cap new model calls (0 = all)."),
dry_run: bool = typer.Option(
False,
"--dry-run",
help="Count facts and timeline events; no model calls, no writes.",
),
write: bool = typer.Option(False, "--write"),
) -> None:
"""Assemble a family's dossier (pfs.dossier): every stored fact with its
resolved citation, the computed gaps, and model prose that passes the
sentence-level citation check (template lines otherwise), through a
resumable ledger. Prints the fact count, the gaps, the stakeholder table
and the check result; --write stores it."""
if all_families:
registry = load_curated()
fams = [registry[k] for k in sorted(registry)]
elif family:
fam = curated_family(normalize_key(family))
if fam is None:
raise typer.BadParameter(f"unknown curated family {family!r}")
fams = [fam]
else:
raise typer.BadParameter("give --family KEY or --all")
store = _store()
built = []
for fam in fams:
con = _read()
try:
if dry_run:
facts = collect_facts(con, store, fam)
events = dossier_timeline(facts)
typer.echo(
f"{fam.key}: {len(facts)} facts, {len(events)} timeline events; "
f"model calls a run would make: {len(events) + 1} "
"(each retried once at most)"
)
continue
d = build_dossier(
con,
store,
fam,
ask=None if no_llm else _asker(),
excerpts=[] if no_llm else _excerpts(store, fam.name),
log=RunLog(ledger_path("prose", fam.key)),
replay_model=_model_name() if no_llm else None,
limit=limit,
)
finally:
con.close()
kinds = Counter(f.kind for f in d.facts.values())
typer.echo(
f"{fam.key}: {len(d.facts)} facts ("
+ ", ".join(f"{k} {n}" for k, n in sorted(kinds.items()))
+ ")"
)
typer.echo(
"timeline years: "
+ ", ".join(str(y) for y in sorted({e.year for e in d.timeline}))
)
for gap in d.gaps:
typer.echo(f"gap: {gap}")
for line in stakeholder_lines(d):
typer.echo(line)
typer.echo(
f"check: {'passed' if d.check_passed else 'FAILED'}; prose {d.extractor}"
)
if d.extractor != "template" and d.fallbacks:
typer.echo("template fallback: " + "; ".join(d.fallbacks))
built.append(d)
if write and built:
with _batch() as bcon:
ensure_tables(bcon)
for d in built:
write_dossier(bcon, d)
typer.echo(f"wrote dossier {d.family}")
_publish()