61fc26cfbe
Some checks failed
CI / lint (push) Successful in 1m2s
CI / notebooks-smoke (push) Successful in 1m54s
Deploy / changes (push) Successful in 18s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Has been skipped
Deploy / bcda (push) Has been skipped
Deploy / llm (push) Has been skipped
Deploy / mc (push) Has been skipped
CI / test (push) Failing after 3m44s
Deploy / report (push) Successful in 18s
build.py inlines assets/css/fhirworx.css (palette, fonts, base classes), maps the Gitea fhirworx-dark palette onto the same token names (opt-in, as in Gitea; light stays light under an OS dark setting), and inlines the logo as an SVG symbol plus the favicon set from assets/icons. The template drops its own palette and uses only fhirworx tokens: sidebar navy navbar with the logo, the dashboard masthead (rule lines, eyebrow, title, tagline) carrying the lex name text and the preview status as a health box, dashboard tiles (Tabler icon plate, eyebrow, title, description, health box, address, stripe) for the Contents and Dossier sections, the dashboard footer, inject.css tables/buttons/inputs, and the chart palette (indigo final, amber proposed; stances teal/amber/red/grey).
953 lines
32 KiB
Python
953 lines
32 KiB
Python
"""Build the lex dossier page (HOME-VISITS): a static snapshot, data inlined.
|
|
|
|
lex (Latin for law) is the legal-epidemiology workbench. The page is served
|
|
by Gitea from its bind-mounted custom directory at ``lex.<domain>/``
|
|
(Traefik routers ``lex*``, behind ``git-sso``; the old address
|
|
``git.<domain>/dossier`` redirects there; see
|
|
infra/traefik/dynamic/services.yml). Run from the primary checkout,
|
|
never a worktree (the replica and bib.sqlite live in its data/):
|
|
|
|
bash -c 'cd /home/kert/stack && set -a && . ./.env && set +a && \
|
|
uv run --no-sync python -I dev/scripts/dossier_page/build.py'
|
|
|
|
Reads the DuckDB replica and bib.sqlite read-only, and the public
|
|
regulations.gov mirror (Mirrulations, AWS Open Data) for submitter names
|
|
(public record). Writes nothing to any database. Writes one complete HTML
|
|
document (template.html with the data inlined) to --out, by default
|
|
infra/gitea/custom/public/assets/dossier/index.html (gitignored: the data
|
|
is rebuilt, never committed). --json also writes the raw data.
|
|
|
|
Design: lex is a fhirworx (HTI-5) page built from the system's sources, so
|
|
it follows them on every rebuild. The page inlines assets/css/fhirworx.css
|
|
(palette, fonts, base classes), the fhirworx dark palette from the Gitea
|
|
theme (infra/gitea/theme/theme-fhirworx-dark.css) mapped onto the same token
|
|
names (opt-in, as in Gitea), the logo (assets/icons/logo.svg, as an SVG
|
|
symbol) and the favicon set (assets/icons). The template uses only those
|
|
tokens.
|
|
|
|
AMA license: CPT descriptors never ship. The RVU-file short descriptors
|
|
are not read at all, and descriptor_change events carry a fixed note in
|
|
place of the RVU-file text.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import html
|
|
import json
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import sys
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
import duckdb
|
|
|
|
from bib import cfrlink, frlink
|
|
from bib.regulations_gov import Mirror
|
|
from pfs.commenters import _COMMENT_ID
|
|
from pfs.descriptors import rule_year_of
|
|
from pfs.families import curated_family
|
|
from pfs.rules import RULES, proposed_for
|
|
from pfs.valuation import (
|
|
_PROPOSED_SQL,
|
|
_pay,
|
|
_sum,
|
|
final_cf_note,
|
|
proposed_cf_note,
|
|
valuation,
|
|
)
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
ROOT = HERE.parents[2]
|
|
DOMAIN = os.environ.get("DOMAIN", "fhirworx.io")
|
|
FAMILY = "HOME-VISITS"
|
|
DOCKET = "CMS-2022-0113"
|
|
|
|
ap = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
|
|
ap.add_argument(
|
|
"--out",
|
|
type=Path,
|
|
default=ROOT / "infra/gitea/custom/public/assets/dossier/index.html",
|
|
help="page to write (default: %(default)s)",
|
|
)
|
|
ap.add_argument("--json", type=Path, default=None, help="also write the raw data here")
|
|
args = ap.parse_args()
|
|
|
|
BUILT = datetime.now().astimezone()
|
|
|
|
duck = duckdb.connect(str(ROOT / "data/replica/aco.ro.duckdb"), read_only=True)
|
|
bibc = sqlite3.connect(f"file:{ROOT / 'data/bib.sqlite'}?mode=ro", uri=True)
|
|
bibc.row_factory = sqlite3.Row
|
|
|
|
|
|
class ROStore:
|
|
"""The slice of bib.store.Store that frlink needs, over a read-only
|
|
connection (frlink only calls ``_con()``)."""
|
|
|
|
def __init__(self, con: sqlite3.Connection) -> None:
|
|
self._c = con
|
|
|
|
def _con(self) -> sqlite3.Connection:
|
|
return self._c
|
|
|
|
|
|
store = ROStore(bibc)
|
|
|
|
fam = curated_family(FAMILY)
|
|
assert fam is not None
|
|
BASE = sorted(m.code for m in fam.members if m.role == "base")
|
|
PRED = sorted(m.code for m in fam.members if m.role == "predecessor")
|
|
LINKED = ["G0318"]
|
|
CODES = BASE + PRED
|
|
ALL_CODES = CODES + LINKED
|
|
|
|
# ── sources and citations ───────────────────────────────────────────
|
|
|
|
SOURCES: dict[str, dict] = {}
|
|
PARAS: dict[str, str] = {}
|
|
|
|
|
|
def item(key: str) -> dict | None:
|
|
if not key:
|
|
return None
|
|
if key in SOURCES:
|
|
return SOURCES[key]
|
|
r = bibc.execute(
|
|
"SELECT key, item_type, title, url, date_published, institution FROM items WHERE key = ?",
|
|
(key,),
|
|
).fetchone()
|
|
if r is None:
|
|
return None
|
|
src = {
|
|
"key": r["key"],
|
|
"type": r["item_type"],
|
|
"title": r["title"],
|
|
"url": r["url"] or "",
|
|
"date": r["date_published"] or "",
|
|
"publisher": r["institution"] or "",
|
|
}
|
|
doc = bibc.execute(
|
|
"SELECT document_number, html_url, fr_volume, start_page, end_page FROM fr_anchor_docs WHERE item_key = ?",
|
|
(key,),
|
|
).fetchone()
|
|
if doc is not None:
|
|
src["fr"] = {
|
|
"doc": doc["document_number"],
|
|
"html": doc["html_url"],
|
|
"vol": doc["fr_volume"],
|
|
"start": doc["start_page"],
|
|
"end": doc["end_page"],
|
|
}
|
|
src["kind"] = frlink.rule_kind(store, key, title=r["title"])
|
|
src["cy"] = rule_year_of(r["title"], r["date_published"] or "")
|
|
src["label"] = f"CY{src['cy']} PFS {src['kind']} rule"
|
|
src["cite"] = f"{doc['fr_volume']} FR {doc['start_page']}"
|
|
SOURCES[key] = src
|
|
return src
|
|
|
|
|
|
def fr_cite(key: str, p_id: int | None = None, page: int | None = None) -> dict | None:
|
|
"""A Federal Register jump link: paragraph anchor via frlink.resolve,
|
|
else a printed-page link. Label "87 FR 69608 ¶1676" (¶ = paragraph
|
|
anchor id, as the dossier spec cites it)."""
|
|
src = item(key)
|
|
if src is None or "fr" not in src:
|
|
return None
|
|
vol = src["fr"]["vol"]
|
|
if p_id:
|
|
try:
|
|
link = frlink.resolve(f"p-{p_id}", store=store, item_key=key)
|
|
except ValueError:
|
|
link = None
|
|
if link is not None:
|
|
PARAS.setdefault(f"{key}:{p_id}", excerpt(para_text(key, p_id), "", 520))
|
|
return {
|
|
"k": key,
|
|
"p": p_id,
|
|
"pg": link.page,
|
|
"label": f"{vol} FR {link.page} ¶{p_id}",
|
|
"url": link.url,
|
|
}
|
|
if page:
|
|
return {
|
|
"k": key,
|
|
"p": 0,
|
|
"pg": page,
|
|
"label": f"{vol} FR {page}",
|
|
"url": f"{src['fr']['html']}#page-{page}",
|
|
}
|
|
return {
|
|
"k": key,
|
|
"p": 0,
|
|
"pg": src["fr"]["start"],
|
|
"label": src["cite"],
|
|
"url": src["fr"]["html"],
|
|
}
|
|
|
|
|
|
def para_text(key: str, p_id: int) -> str:
|
|
r = bibc.execute(
|
|
"SELECT text FROM fr_anchors WHERE item_key = ? AND p_id = ?", (key, p_id)
|
|
).fetchone()
|
|
return " ".join((r["text"] or "").split()) if r else ""
|
|
|
|
|
|
_SENT = re.compile(r"(?<=[.;:!?])\s+(?=[A-Z(“\"])")
|
|
|
|
|
|
def excerpt(text: str, kw: str = "", maxlen: int = 420) -> str:
|
|
"""Verbatim: from the sentence containing *kw* (or the start), cut at a
|
|
word boundary near *maxlen*."""
|
|
if not text:
|
|
return ""
|
|
if kw:
|
|
i = text.lower().find(kw.lower())
|
|
if i > 0:
|
|
starts = [0] + [m.end() for m in _SENT.finditer(text)]
|
|
s = max(x for x in starts if x <= i)
|
|
text = ("… " if s else "") + text[s:]
|
|
if len(text) <= maxlen:
|
|
return text
|
|
cut = text[:maxlen].rsplit(" ", 1)[0].rstrip(",;:")
|
|
return cut + " …"
|
|
|
|
|
|
def keep_para(key: str, p_id: int, kw: str = "", maxlen: int = 520) -> str:
|
|
t = excerpt(para_text(key, p_id), kw, maxlen)
|
|
if t:
|
|
PARAS[f"{key}:{p_id}"] = t
|
|
return t
|
|
|
|
|
|
# ── membership and links ────────────────────────────────────────────
|
|
|
|
members = []
|
|
for m in fam.members:
|
|
c = fr_cite(m.item_key, m.p_id) if m.item_key else None
|
|
members.append(
|
|
{
|
|
"code": m.code,
|
|
"role": m.role,
|
|
"since": m.since,
|
|
"until": m.until,
|
|
"cite": c,
|
|
"rvu_year": m.rvu_year or None,
|
|
}
|
|
)
|
|
links = []
|
|
for ln in fam.links:
|
|
links.append(
|
|
{
|
|
"kind": ln.kind,
|
|
"target": ln.target,
|
|
"since": ln.since,
|
|
"cite": fr_cite(ln.item_key, ln.p_id),
|
|
"aliases": list(ln.aliases),
|
|
"alias_cite": fr_cite(ln.alias_item_key, ln.alias_p_id)
|
|
if ln.alias_item_key
|
|
else None,
|
|
}
|
|
)
|
|
keep_para(ln.item_key, ln.p_id, maxlen=360)
|
|
if ln.alias_item_key:
|
|
keep_para(ln.alias_item_key, ln.alias_p_id, maxlen=300)
|
|
keep_para("PETM225D", 1672, maxlen=520)
|
|
|
|
# ── valuation rounds ────────────────────────────────────────────────
|
|
|
|
rounds = []
|
|
for r in duck.execute(
|
|
"""SELECT code, rule_year, item_key, rule_kind, p_id, page, source, table_title,
|
|
ruc_work_rvu, ruc_percentile, survey_intra_min, survey_total_min,
|
|
requested_work_rvu, proposed_work_rvu, final_work_rvu, decision,
|
|
rationale_p_id, rationale_quote, pe_note, extractor, final_p_id, ruc_p_id, field_cites
|
|
FROM pfs.valuation_round WHERE family = ? ORDER BY rule_year, code, rule_kind""",
|
|
[FAMILY],
|
|
).fetchall():
|
|
(
|
|
code,
|
|
ry,
|
|
key,
|
|
kind,
|
|
p_id,
|
|
page,
|
|
source,
|
|
ttitle,
|
|
ruc,
|
|
pct,
|
|
intra,
|
|
total,
|
|
req,
|
|
prop,
|
|
fin,
|
|
dec,
|
|
rat_p,
|
|
rat_q,
|
|
pe_note,
|
|
extractor,
|
|
fin_p,
|
|
ruc_p,
|
|
field_cites,
|
|
) = r
|
|
fc = json.loads(field_cites or "{}")
|
|
cites = {}
|
|
for field, v in fc.items():
|
|
if isinstance(v, list):
|
|
cites[field] = [fr_cite(key, int(x)) for x in v]
|
|
elif isinstance(v, str) and v.startswith("page:"):
|
|
cites[field] = [fr_cite(key, None, int(v.split(":")[1]))]
|
|
elif v:
|
|
cites[field] = [fr_cite(key, int(v))]
|
|
rounds.append(
|
|
{
|
|
"code": code,
|
|
"cy": ry,
|
|
"key": key,
|
|
"kind": kind,
|
|
"source": source,
|
|
"table": ttitle or "",
|
|
"ruc": ruc,
|
|
"pct": pct or "",
|
|
"intra": intra,
|
|
"total": total,
|
|
"req": req,
|
|
"prop": prop,
|
|
"fin": fin,
|
|
"decision": dec,
|
|
"quote": rat_q or "",
|
|
"pe_note": pe_note or "",
|
|
"extractor": extractor,
|
|
"cite": fr_cite(key, p_id or None, page or None),
|
|
"cites": cites,
|
|
}
|
|
)
|
|
for k, p, kw, n in [
|
|
("PETM225D", 1676, "The RUC recommended the survey 25th", 420),
|
|
("PETM225D", 1685, "", 300),
|
|
("MZ24MX5S", 878, "", 160),
|
|
("MZ24MX5S", 877, "", 420),
|
|
("54GTKR5G", 2241, "", 520),
|
|
("54GTKR5G", 2242, "", 260),
|
|
]:
|
|
keep_para(k, p, kw, n)
|
|
|
|
# ── origins ─────────────────────────────────────────────────────────
|
|
|
|
origins = []
|
|
groups: dict[tuple, dict] = {}
|
|
for r in duck.execute(
|
|
"""SELECT kind, reference, year, cite_kind, item_key, p_id, page_label, locator, extractor, ref_p_id,
|
|
summary, code
|
|
FROM pfs.code_origin WHERE family = ?
|
|
ORDER BY kind, year NULLS LAST, reference, code""",
|
|
[FAMILY],
|
|
).fetchall():
|
|
(
|
|
kind,
|
|
ref,
|
|
year,
|
|
ck,
|
|
key,
|
|
p_id,
|
|
page_label,
|
|
locator,
|
|
extractor,
|
|
ref_p,
|
|
summary,
|
|
code,
|
|
) = r
|
|
gk = (kind, ref, year, ck, key, p_id, page_label)
|
|
g = groups.get(gk)
|
|
if g is None:
|
|
cite = (
|
|
fr_cite(key, p_id)
|
|
if (ck in ("fr_para", "iom", "cfr") and key and p_id)
|
|
else None
|
|
)
|
|
book = None
|
|
if ck == "cpt_book":
|
|
src = item(key)
|
|
book = {
|
|
"key": key,
|
|
"title": src["title"] if src else key,
|
|
"page": page_label,
|
|
}
|
|
target = None
|
|
if ck == "cfr":
|
|
try:
|
|
c = cfrlink.parse_cite(ref)
|
|
target = {"label": cfrlink.canonical(c), "url": cfrlink.url(c)}
|
|
except ValueError:
|
|
target = None
|
|
g = groups[gk] = {
|
|
"kind": kind,
|
|
"ref": ref,
|
|
"year": year,
|
|
"cite_kind": ck,
|
|
"cite": cite,
|
|
"book": book,
|
|
"target": target,
|
|
"extractors": [],
|
|
"codes": [],
|
|
"summaries": {},
|
|
"ref_cite": fr_cite(key, ref_p) if (ref_p and ck != "cpt_book") else None,
|
|
}
|
|
g["codes"].append(code)
|
|
if extractor not in g["extractors"]:
|
|
g["extractors"].append(extractor)
|
|
g["summaries"].setdefault(summary or "", []).append(code)
|
|
for g in groups.values():
|
|
sums = g.pop("summaries")
|
|
# per-code templated summaries ("... for 99324 points to ...") say nothing
|
|
# the reference does not; keep a summary only when few distinct ones exist
|
|
g["summaries"] = (
|
|
[{"text": t, "codes": c} for t, c in sums.items() if t]
|
|
if len(sums) <= 2
|
|
else []
|
|
)
|
|
g["n_rows"] = len(g["codes"])
|
|
g["codes"] = sorted(set(g["codes"]))
|
|
origins.append(g)
|
|
|
|
# ── lineage events ──────────────────────────────────────────────────
|
|
|
|
events = []
|
|
review_rows = []
|
|
for r in duck.execute(
|
|
"""SELECT year, kind, source, item_key, p_id, page, from_codes, to_codes, note,
|
|
string_agg(code, ',' ORDER BY code) AS codes
|
|
FROM pfs.code_event WHERE code IN (SELECT unnest(?))
|
|
GROUP BY ALL ORDER BY year, kind""",
|
|
[ALL_CODES],
|
|
).fetchall():
|
|
year, kind, source, key, p_id, page, fcodes, tcodes, note, codes = r
|
|
ev = {
|
|
"year": year,
|
|
"kind": kind,
|
|
"source": source,
|
|
"codes": codes.split(","),
|
|
"from": fcodes or "",
|
|
"to": tcodes or "",
|
|
# AMA license: RVU-file short descriptors are CPT text; never ship them.
|
|
"note": (
|
|
"Short descriptor revised in the RVU file (AMA text not reproduced)"
|
|
if kind == "descriptor_change"
|
|
else (note or "")
|
|
),
|
|
}
|
|
if source == "fr":
|
|
ev["cite"] = fr_cite(key, p_id, page)
|
|
text = para_text(key, p_id)
|
|
if ev["codes"] == ["G0318"] and year < 2023 and "ESRD" in text:
|
|
# G0318 named a different (ESRD) service before CY2023: keep it off
|
|
# the family timeline; it is a real row for the review queue sketch.
|
|
review_rows.append(
|
|
{
|
|
"what": f"lineage event: {kind} {codes} CY{year}",
|
|
"cite": ev["cite"],
|
|
"quote": excerpt(text, "G0318", 260),
|
|
"why": "The cited paragraph uses G0318 for an ESRD service, before the code numbered the CY2023 prolonged home visit add-on.",
|
|
}
|
|
)
|
|
continue
|
|
elif source == "cpt":
|
|
src = item(key)
|
|
ev["book"] = {"key": key, "title": src["title"] if src else key}
|
|
elif source == "rvu":
|
|
rule = RULES.get(int(year))
|
|
ev["rvu"] = {
|
|
"label": f"PFS CY{year} Addendum B (RVU file)",
|
|
"cite": rule.federal_register_citation if rule else "",
|
|
"url": f"https://www.federalregister.gov/citation/{rule.federal_register_citation.replace(' ', '-')}"
|
|
if rule
|
|
else "",
|
|
}
|
|
events.append(ev)
|
|
for k, p in [
|
|
("2AZVKTCE", 2404),
|
|
("PETM225D", 1694),
|
|
("PETM225D", 390),
|
|
("PETM225D", 497),
|
|
]:
|
|
keep_para(k, p, maxlen=300)
|
|
|
|
# ── CMS's own comment summaries ─────────────────────────────────────
|
|
|
|
summaries = []
|
|
for r in duck.execute(
|
|
"""SELECT item_key, rule_year, rule_kind, comment_p_id, response_p_id, codes, quantifier, stance,
|
|
response_action, extractor
|
|
FROM pfs.cms_comment_summary WHERE family = ? ORDER BY rule_year, comment_p_id""",
|
|
[FAMILY],
|
|
).fetchall():
|
|
key, ry, rk, cp, rp, codes, q, st, act, extractor = r
|
|
summaries.append(
|
|
{
|
|
"key": key,
|
|
"cy": ry,
|
|
"kind": rk,
|
|
"q": q,
|
|
"stance": st,
|
|
"action": act,
|
|
"extractor": extractor,
|
|
"codes": [c for c in (codes or "").split(",") if c],
|
|
"comment": fr_cite(key, cp),
|
|
"response": fr_cite(key, rp),
|
|
"ctext": excerpt(para_text(key, cp), "", 360),
|
|
"rtext": excerpt(para_text(key, rp), "", 260),
|
|
}
|
|
)
|
|
|
|
# ── commenters (CY2023 docket) + submitter names from the mirror ────
|
|
|
|
commenters = []
|
|
rows = duck.execute(
|
|
"""SELECT item_key, docket, comment_id, org_name, org_type, specialty, form_group, form_size, basis,
|
|
reg_category, extractor
|
|
FROM pfs.commenter WHERE family = ? AND docket = ? ORDER BY org_type, specialty, org_name""",
|
|
[FAMILY, DOCKET],
|
|
).fetchall()
|
|
mirror = Mirror()
|
|
|
|
|
|
ITEM_URL = (
|
|
{
|
|
r["key"]: r["url"] or ""
|
|
for r in bibc.execute(
|
|
"SELECT key, url FROM items WHERE key IN (%s)" % ",".join("?" * len(rows)),
|
|
[r[0] for r in rows],
|
|
).fetchall()
|
|
}
|
|
if rows
|
|
else {}
|
|
)
|
|
|
|
|
|
def submitter(row) -> dict:
|
|
url = ITEM_URL.get(row[0], "")
|
|
m = _COMMENT_ID.search(url)
|
|
cid = m.group(1) if m else (row[2] or "")
|
|
out = {
|
|
"cid": cid,
|
|
"url": f"https://www.regulations.gov/comment/{cid}" if cid else url,
|
|
"first": "",
|
|
"last": "",
|
|
"org": "",
|
|
"posted": "",
|
|
"ok": False,
|
|
}
|
|
if not cid:
|
|
return out
|
|
for _ in range(3):
|
|
try:
|
|
a = mirror.get_json(row[1], cid)["data"]["attributes"]
|
|
out.update(
|
|
first=(a.get("firstName") or "").strip(),
|
|
last=(a.get("lastName") or "").strip(),
|
|
org=(a.get("organization") or "").strip(),
|
|
posted=(a.get("postedDate") or "")[:10],
|
|
ok=True,
|
|
)
|
|
return out
|
|
except Exception as exc: # noqa: BLE001
|
|
out["err"] = type(exc).__name__
|
|
return out
|
|
|
|
|
|
with ThreadPoolExecutor(8) as pool:
|
|
subs = list(pool.map(submitter, rows))
|
|
for row, sub in zip(rows, subs, strict=True):
|
|
key, docket, cid, org, otype, spec, fg, fs, basis, regcat, extractor = row
|
|
commenters.append(
|
|
{
|
|
"key": key,
|
|
"cid": sub["cid"] or cid,
|
|
"url": sub["url"],
|
|
"org": org or "",
|
|
"type": otype or "",
|
|
"specialty": spec or "",
|
|
"form_group": fg or "",
|
|
"form_size": fs or 0,
|
|
"basis": basis or "",
|
|
"reg_category": regcat or "",
|
|
"extractor": extractor,
|
|
"name": " ".join(x for x in (sub["first"], sub["last"]) if x),
|
|
"reg_org": sub["org"],
|
|
"posted": sub["posted"],
|
|
"mirror_ok": sub["ok"],
|
|
}
|
|
)
|
|
|
|
stances = [
|
|
{
|
|
"key": k,
|
|
"event": e,
|
|
"stance": s,
|
|
"reason": r or "",
|
|
"evidence": ev or "",
|
|
"extractor": x,
|
|
}
|
|
for k, e, s, r, ev, x in duck.execute(
|
|
"""SELECT item_key, event_key, stance, reason, evidence, extractor FROM pfs.comment_stance
|
|
WHERE family = ? AND docket = ? ORDER BY event_key, stance, item_key""",
|
|
[FAMILY, DOCKET],
|
|
).fetchall()
|
|
]
|
|
|
|
reaction = []
|
|
for r in duck.execute(
|
|
"""SELECT period, period_kind, year, n_items, n_total, stance_support, stance_oppose, stance_modify,
|
|
stance_unclear, n_classified, n_commenters, events_json, stance_method, stance_eval
|
|
FROM pfs.code_reaction WHERE family = ? ORDER BY year, period""",
|
|
[FAMILY],
|
|
).fetchall():
|
|
period, pk, year, ni, nt, s_sup, s_opp, s_mod, s_unc, ncl, ncm, ej, smeth, seval = r
|
|
d = {"period": period, "pk": pk, "year": year, "n": ni, "total": nt}
|
|
if pk == "docket":
|
|
dk = bibc.execute(
|
|
"SELECT rule_cms_id, comment_end_date FROM dockets WHERE id = ?", (period,)
|
|
).fetchone()
|
|
if dk:
|
|
d["rule"] = dk["rule_cms_id"] or ""
|
|
d["close"] = dk["comment_end_date"] or ""
|
|
d["url"] = f"https://www.regulations.gov/docket/{period}"
|
|
else:
|
|
d["cite"] = fr_cite(period)
|
|
if ncm is not None:
|
|
d.update(
|
|
support=s_sup,
|
|
oppose=s_opp,
|
|
modify=s_mod,
|
|
unclear=s_unc,
|
|
classified=ncl,
|
|
commenters=ncm,
|
|
events=json.loads(ej or "{}"),
|
|
method=smeth,
|
|
eval=json.loads(seval) if seval else None,
|
|
)
|
|
reaction.append(d)
|
|
|
|
# ── guidance pointers ───────────────────────────────────────────────
|
|
|
|
guidance = []
|
|
for r in duck.execute(
|
|
"""SELECT kind, locator, item_key, item_key_src, p_id_src, page_src, page,
|
|
string_agg(code, ',' ORDER BY code) AS codes
|
|
FROM pfs.code_guidance WHERE family = ? GROUP BY ALL ORDER BY kind, locator, item_key_src""",
|
|
[FAMILY],
|
|
).fetchall():
|
|
kind, loc, key, ksrc, psrc, pgsrc, page, codes = r
|
|
tgt = item(key) if key else None
|
|
turl = ""
|
|
if kind == "cfr" and key:
|
|
try:
|
|
turl = cfrlink.url(cfrlink.parse_cite(loc))
|
|
except ValueError:
|
|
turl = tgt["url"] if tgt else ""
|
|
elif tgt:
|
|
turl = tgt["url"] + (f"#page={page}" if page else "")
|
|
g = {
|
|
"kind": kind,
|
|
"locator": loc,
|
|
"target": {"title": tgt["title"], "url": turl} if tgt else None,
|
|
"src": fr_cite(ksrc, psrc),
|
|
"page": page or None,
|
|
"codes": codes.split(","),
|
|
}
|
|
if not key:
|
|
review_rows.append(
|
|
{
|
|
"what": f"guidance pointer: {loc}",
|
|
"cite": g["src"],
|
|
"quote": "",
|
|
"why": "The locator did not resolve to a regulation in the library. The same paragraph is also cited for 42 CFR 425.400(c)(1)(vi).",
|
|
}
|
|
)
|
|
g["unresolved"] = True
|
|
guidance.append(g)
|
|
|
|
# ── payment: final (pfs.rvu) and proposed (pfs.rvu_proposed), never blended ──
|
|
|
|
pay_rows, missing = valuation(duck, ALL_CODES, years=40)
|
|
payment = [
|
|
{
|
|
"code": p.code,
|
|
"cy": p.year,
|
|
"proposed": p.proposed,
|
|
"status": p.status,
|
|
"work": p.work,
|
|
"pe_nf": p.pe_nf,
|
|
"mp": p.mp,
|
|
"total_nf": p.total_nf,
|
|
"cf": p.cf,
|
|
"pay_nf": p.pay_nf,
|
|
"cite": p.citation,
|
|
"url": p.url,
|
|
"label": p.label,
|
|
"status_note": p.status_note,
|
|
}
|
|
for p in pay_rows
|
|
]
|
|
have = {(p["code"], p["cy"], p["proposed"]) for p in payment}
|
|
for cy in sorted(RULES):
|
|
prop = proposed_for(cy)
|
|
if prop is None:
|
|
continue
|
|
for hcpcs, desc, status, work, pe_nf, pe_f, mp in duck.execute(
|
|
_PROPOSED_SQL, [prop.cms_rule_id, ALL_CODES]
|
|
).fetchall():
|
|
if (hcpcs, cy, True) in have:
|
|
continue
|
|
tot = _sum(work, pe_nf, mp)
|
|
payment.append(
|
|
{
|
|
"code": hcpcs,
|
|
"cy": cy,
|
|
"proposed": True,
|
|
"status": status or "",
|
|
"work": work,
|
|
"pe_nf": pe_nf,
|
|
"mp": mp,
|
|
"total_nf": tot,
|
|
"cf": prop.conversion_factor,
|
|
"pay_nf": _pay(tot, prop.conversion_factor, status or ""),
|
|
"cite": prop.federal_register_citation,
|
|
"url": f"https://www.federalregister.gov/citation/{prop.federal_register_citation.replace(' ', '-')}",
|
|
"label": f"[CY{cy} NPRM Addendum B]",
|
|
"status_note": "",
|
|
}
|
|
)
|
|
payment.sort(key=lambda p: (p["code"], p["cy"], p["proposed"]))
|
|
cfs = []
|
|
for cy in sorted(RULES):
|
|
rule = RULES[cy]
|
|
cfs.append(
|
|
{
|
|
"cy": cy,
|
|
"proposed": False,
|
|
"cf": rule.conversion_factor,
|
|
"note": final_cf_note(rule),
|
|
"cite": rule.federal_register_citation,
|
|
}
|
|
)
|
|
prop = proposed_for(cy)
|
|
if prop is not None:
|
|
cfs.append(
|
|
{
|
|
"cy": cy,
|
|
"proposed": True,
|
|
"cf": prop.conversion_factor,
|
|
"note": proposed_cf_note(prop),
|
|
"cite": prop.federal_register_citation,
|
|
}
|
|
)
|
|
|
|
# ── utilization (national services, all places of service) ─────────
|
|
|
|
util = []
|
|
for year, hcpcs, svc, benes, key in duck.execute(
|
|
"""SELECT year, hcpcs, sum(n_services), sum(n_beneficiaries), any_value(item_key)
|
|
FROM pfs.utilization WHERE hcpcs IN (SELECT unnest(?)) GROUP BY 1, 2 ORDER BY 2, 1""",
|
|
[ALL_CODES],
|
|
).fetchall():
|
|
item(key)
|
|
util.append({"cy": year, "code": hcpcs, "services": svc, "key": key})
|
|
|
|
# AMA license: no RVU-file short descriptors (CPT text) are read or shipped.
|
|
|
|
for k in ["3ANRT854", "GQGTPGYV", "T6H4ZPEQ"]:
|
|
item(k)
|
|
|
|
n_fam, n_single = duck.execute(
|
|
"SELECT count(*), count(*) FILTER (WHERE n = 1) FROM (SELECT key, count(DISTINCT code) AS n FROM pfs.code_family GROUP BY key)"
|
|
).fetchone()
|
|
|
|
data = {
|
|
"meta": {
|
|
"derived_families": n_fam,
|
|
"derived_single": n_single,
|
|
"built": BUILT.isoformat(timespec="seconds"),
|
|
"family": FAMILY,
|
|
"name": fam.name,
|
|
"docket": DOCKET,
|
|
"base": BASE,
|
|
"pred": PRED,
|
|
"linked": LINKED,
|
|
"rvu_years": [min(p["cy"] for p in payment), max(p["cy"] for p in payment)],
|
|
"util_years": [min(u["cy"] for u in util), max(u["cy"] for u in util)]
|
|
if util
|
|
else None,
|
|
"missing_payment": missing,
|
|
},
|
|
"members": members,
|
|
"links": links,
|
|
"rounds": rounds,
|
|
"origins": origins,
|
|
"events": events,
|
|
"summaries": summaries,
|
|
"commenters": commenters,
|
|
"stances": stances,
|
|
"reaction": reaction,
|
|
"guidance": guidance,
|
|
"payment": payment,
|
|
"cfs": cfs,
|
|
"util": util,
|
|
"review": review_rows,
|
|
"paras": PARAS,
|
|
"sources": SOURCES,
|
|
}
|
|
blob = json.dumps(data, ensure_ascii=False, separators=(",", ":"))
|
|
if args.json:
|
|
args.json.write_text(blob, encoding="utf-8")
|
|
print(f"wrote {args.json} ({args.json.stat().st_size:,} bytes)", file=sys.stderr)
|
|
|
|
|
|
def _fill(tpl: str, marker: str, value: str, count: int = 1) -> str:
|
|
if tpl.count(marker) != count:
|
|
raise SystemExit(
|
|
f"template.html: expected {count} {marker!r}, found {tpl.count(marker)}"
|
|
)
|
|
return tpl.replace(marker, value)
|
|
|
|
|
|
# ── the fhirworx design system, from its sources ──────────────────────────
|
|
FHIRWORX_CSS = (
|
|
ROOT / "assets/css/fhirworx.css"
|
|
) # single source of truth for the palette
|
|
GITEA_DARK = (
|
|
ROOT / "infra/gitea/theme/theme-fhirworx-dark.css"
|
|
) # the system's dark palette
|
|
ICONS = ROOT / "assets/icons"
|
|
|
|
# fhirworx.css token -> the Gitea fhirworx-dark variable that plays the same
|
|
# role. chart-1..3 (the indigo scale) have no dark counterpart in the system,
|
|
# so they keep their fhirworx.css values; the template picks the step that
|
|
# reads on the dark card.
|
|
DARK_ROLES = {
|
|
"background": "--fw-background",
|
|
"foreground": "--fw-foreground",
|
|
"card": "--fw-card",
|
|
"card-foreground": "--fw-foreground",
|
|
"popover": "--fw-card",
|
|
"popover-foreground": "--fw-foreground",
|
|
"primary": "--fw-primary",
|
|
"primary-foreground": "--fw-primary-foreground",
|
|
"secondary": "--fw-secondary",
|
|
"secondary-foreground": "--color-text-light",
|
|
"muted": "--fw-secondary",
|
|
"muted-foreground": "--fw-muted-foreground",
|
|
"accent": "--fw-secondary",
|
|
"accent-foreground": "--fw-foreground",
|
|
"destructive": "--fw-destructive",
|
|
"destructive-foreground": "--fw-primary-foreground",
|
|
"border": "--fw-border",
|
|
"input": "--fw-border",
|
|
"ring": "--fw-primary",
|
|
"sidebar": "--fw-nav-bg",
|
|
"sidebar-foreground": "--fw-nav-text",
|
|
"sidebar-primary": "--fw-link",
|
|
"sidebar-primary-foreground": "--fw-primary-foreground",
|
|
"sidebar-accent": "--fw-nav-hover-bg",
|
|
"sidebar-accent-foreground": "--fw-nav-text",
|
|
"sidebar-border": "--fw-nav-hover-bg",
|
|
"sidebar-ring": "--fw-link",
|
|
"chart-4": "--color-green",
|
|
"chart-5": "--fw-chart-5",
|
|
}
|
|
|
|
|
|
def _root_vars(path: Path) -> dict[str, str]:
|
|
"""Custom properties of the stylesheet's first :root block (raw values)."""
|
|
m = re.search(r"^:root\s*\{(.*?)^\}", path.read_text(encoding="utf-8"), re.S | re.M)
|
|
if not m:
|
|
raise SystemExit(f"{path}: no :root block")
|
|
body = re.sub(r"/\*.*?\*/", "", m.group(1), flags=re.S)
|
|
return {k: v.strip() for k, v in re.findall(r"(--[\w-]+)\s*:\s*([^;]+);", body)}
|
|
|
|
|
|
def _resolve(raw: dict[str, str], name: str, where: Path, depth: int = 0) -> str:
|
|
"""The value of one custom property, following var(--x) references."""
|
|
if depth > 8 or name not in raw:
|
|
raise SystemExit(f"{where}: cannot resolve {name}")
|
|
return re.sub(
|
|
r"var\((--[\w-]+)\)",
|
|
lambda r: _resolve(raw, r.group(1), where, depth + 1),
|
|
raw[name],
|
|
)
|
|
|
|
|
|
def _dark_palette() -> str:
|
|
light = _root_vars(FHIRWORX_CSS)
|
|
dark = _root_vars(GITEA_DARK)
|
|
lines = []
|
|
for token, src in DARK_ROLES.items():
|
|
if f"--{token}" not in light:
|
|
raise SystemExit(
|
|
f"{FHIRWORX_CSS}: token --{token} is gone; update DARK_ROLES"
|
|
)
|
|
value = _resolve(dark, src, GITEA_DARK)
|
|
if not re.fullmatch(r"#[0-9A-Fa-f]{6}", value):
|
|
raise SystemExit(f"{GITEA_DARK}: {src} is {value!r}, expected a hex colour")
|
|
lines.append(f" --{token}: {value}; /* {src} */")
|
|
return ':root[data-theme="dark"] {\n' + "\n".join(lines) + "\n}"
|
|
|
|
|
|
def _data_uri(path: Path, mime: str) -> str:
|
|
return f"data:{mime};base64," + base64.b64encode(path.read_bytes()).decode("ascii")
|
|
|
|
|
|
def _logo_symbol() -> str:
|
|
"""assets/icons/logo.svg as an SVG <symbol id="fw-logo">, used by <use>."""
|
|
svg = (ICONS / "logo.svg").read_text(encoding="utf-8")
|
|
m = re.fullmatch(r"\s*<svg\b([^>]*)>(.*)</svg>\s*", svg, re.S)
|
|
vb = re.search(r'viewBox="([^"]+)"', m.group(1)) if m else None
|
|
if not (m and vb):
|
|
raise SystemExit(
|
|
f"{ICONS / 'logo.svg'}: expected one <svg viewBox=...> element"
|
|
)
|
|
return f'<symbol id="fw-logo" viewBox="{vb.group(1)}">{m.group(2).strip()}</symbol>'
|
|
|
|
|
|
fw_css = FHIRWORX_CSS.read_text(encoding="utf-8")
|
|
if "</" in fw_css:
|
|
raise SystemExit(f"{FHIRWORX_CSS}: '</' would end the inline <style>")
|
|
|
|
# Inline the data into the page. Every "<" in the JSON is escaped so no
|
|
# string in it can close or confuse the <script> element.
|
|
page = (HERE / "template.html").read_text(encoding="utf-8")
|
|
page = _fill(page, "/*__FHIRWORX_CSS__*/", fw_css)
|
|
page = _fill(page, "/*__FHIRWORX_DARK__*/", _dark_palette())
|
|
page = _fill(page, "<!--__FW_LOGO__-->", _logo_symbol())
|
|
page = _fill(
|
|
page, "/*__FAVICON_SVG__*/", _data_uri(ICONS / "favicon.svg", "image/svg+xml")
|
|
)
|
|
page = _fill(page, "/*__FAVICON_PNG__*/", _data_uri(ICONS / "fav32.png", "image/png"))
|
|
page = _fill(
|
|
page,
|
|
"/*__APPLE_TOUCH_ICON__*/",
|
|
_data_uri(ICONS / "apple-touch-icon.png", "image/png"),
|
|
)
|
|
# status line and footer
|
|
page = _fill(
|
|
page, "/*__BUILT__*/", html.escape(BUILT.strftime("%Y-%m-%d %H:%M %Z")), count=2
|
|
)
|
|
# navbar links back to Gitea (on lex.<domain>, "/" is this page), the
|
|
# masthead and footer address, the dashboard link, the tiles' deep links
|
|
page = _fill(page, "/*__DOMAIN__*/", html.escape(DOMAIN), count=6)
|
|
page = _fill(page, "/*__DATA__*/{}", blob.replace("<", "\\u003c"))
|
|
dst = args.out
|
|
dst.parent.mkdir(parents=True, exist_ok=True)
|
|
tmp = dst.with_name(f".{dst.name}.tmp")
|
|
tmp.write_text(page, encoding="utf-8")
|
|
os.replace(tmp, dst) # Gitea serves dst live: never expose a half-written page
|
|
print(f"wrote {dst} ({dst.stat().st_size:,} bytes)", file=sys.stderr)
|
|
named = sum(1 for c in commenters if c["name"])
|
|
print(
|
|
f"commenters {len(commenters)}; mirror ok {sum(c['mirror_ok'] for c in commenters)}; with a posted name {named}",
|
|
file=sys.stderr,
|
|
)
|