feat(bib): discover IPPS, OPPS and DMEPOS rules and pull their comments
The PFS register-and-walk path now serves the annual IPPS/LTCH and OPPS/ASC rules and DMEPOS payment rules: title-filtered FR discovery (discover-*-rules) and docket registration + comment walks (fetch-*-comments). Correction documents (C1-...) parse, a null FR abstract no longer becomes 'None', and the store waits out the mail poller's write bursts (busy_timeout) instead of failing SQLITE_BUSY.
This commit is contained in:
kert
@@ -35,6 +35,44 @@ _BASE = "https://www.federalregister.gov/api/v1"
|
||||
# rules that only mention the phrase "physician fee schedule" in passing.
|
||||
PFS_TITLE_PATTERN = "Payment Policies Under the Physician Fee Schedule"
|
||||
|
||||
# The annual IPPS/LTCH rule. Requiring the long-term-care clause keeps
|
||||
# out the GME/organ-acquisition rule, which shares only the acute-care
|
||||
# half of the title. "Long Term" and "Long-Term" both occur.
|
||||
IPPS_TITLE_PATTERN = (
|
||||
"Hospital Inpatient Prospective Payment Systems for Acute Care Hospitals "
|
||||
"and the Long-Term Care Hospital Prospective Payment System"
|
||||
)
|
||||
|
||||
# The annual OPPS/ASC rule. The 340B remedy rule says "Outpatient
|
||||
# Prospective Payment System" and does not name the ASC payment system.
|
||||
OPPS_TITLE_PATTERN = (
|
||||
"Hospital Outpatient Prospective Payment and Ambulatory Surgical Center "
|
||||
"Payment Systems"
|
||||
)
|
||||
|
||||
# DMEPOS payment policy has no stable annual title. It rode inside the
|
||||
# ESRD rule, then a standalone HCPCS rule, then the home health rule.
|
||||
# Keep a document when the title names the benefit (the acronym, or the
|
||||
# three-word "durable medical equipment" + prosthetics + orthotics), and
|
||||
# drop a Physician Fee Schedule title that only mentions DMEPOS in a
|
||||
# conditions-of-payment clause. The prosthetics-supplier rule (CMS-6012)
|
||||
# never says DMEPOS; its title is matched as its own phrase.
|
||||
_DMEPOS_SUPPLIER_PHRASE = (
|
||||
"qualified suppliers of prosthetics and custom-fabricated orthotics"
|
||||
)
|
||||
# The 2018 rural blend (CMS-1687-IFC) says "fee schedule" and never
|
||||
# "DMEPOS". Later master-list notices (CMS-6095-N and after) dropped
|
||||
# the acronym; "face to face" and "face-to-face" both occur.
|
||||
_DMEPOS_FEE_SCHEDULE = "durable medical equipment fee schedule"
|
||||
_DMEPOS_MASTER_LIST = "master list of items potentially subject to face-to-face"
|
||||
_DMEPOS_SEARCH_TERMS = (
|
||||
"dmepos",
|
||||
"durable medical equipment",
|
||||
"custom-fabricated orthotics",
|
||||
"durable medical equipment fee schedule",
|
||||
"master list of items potentially subject",
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class FRDocument:
|
||||
@@ -130,6 +168,43 @@ def split_docket_ids(raw: list[str]) -> list[str]:
|
||||
return out
|
||||
|
||||
|
||||
def _titled_rules(
|
||||
*,
|
||||
term: str,
|
||||
pattern: str,
|
||||
since: str,
|
||||
until: str | None,
|
||||
client: httpx.Client | None,
|
||||
) -> list[FRDocument]:
|
||||
"""CMS rules whose title contains *pattern*.
|
||||
|
||||
``long term`` and ``long-term`` compare equal so a hyphen drift in
|
||||
an annual-rule title does not drop a year.
|
||||
"""
|
||||
needle = pattern.lower().replace("long term", "long-term")
|
||||
found: list[FRDocument] = []
|
||||
seen: set[str] = set()
|
||||
for doc in search(
|
||||
term=term,
|
||||
agencies=["centers-for-medicare-medicaid-services"],
|
||||
types=["RULE", "PRORULE"],
|
||||
since=since,
|
||||
until=until,
|
||||
client=client,
|
||||
):
|
||||
# FY2026+ inserts "(IPPS)" after "Acute Care Hospitals".
|
||||
title = (
|
||||
doc.title.lower().replace("long term", "long-term").replace(" (ipps)", "")
|
||||
)
|
||||
if needle not in title:
|
||||
continue
|
||||
if doc.document_number in seen:
|
||||
continue
|
||||
seen.add(doc.document_number)
|
||||
found.append(doc)
|
||||
return found
|
||||
|
||||
|
||||
def pfs_rules(
|
||||
*,
|
||||
since: str = "2017-01-01",
|
||||
@@ -141,22 +216,121 @@ def pfs_rules(
|
||||
Title-filtered so we get the true annual PFS rules, not every CMS
|
||||
rule that mentions PFS in passing.
|
||||
"""
|
||||
found: list[FRDocument] = []
|
||||
seen: set[str] = set()
|
||||
for doc in search(
|
||||
return _titled_rules(
|
||||
term="physician fee schedule",
|
||||
agencies=["centers-for-medicare-medicaid-services"],
|
||||
types=["RULE", "PRORULE"],
|
||||
pattern=PFS_TITLE_PATTERN,
|
||||
since=since,
|
||||
until=until,
|
||||
client=client,
|
||||
):
|
||||
if PFS_TITLE_PATTERN.lower() not in doc.title.lower():
|
||||
continue
|
||||
if doc.document_number in seen:
|
||||
continue
|
||||
seen.add(doc.document_number)
|
||||
found.append(doc)
|
||||
)
|
||||
|
||||
|
||||
def ipps_rules(
|
||||
*,
|
||||
since: str = "2017-01-01",
|
||||
until: str | None = None,
|
||||
client: httpx.Client | None = None,
|
||||
) -> list[FRDocument]:
|
||||
"""Every annual IPPS/LTCH proposed/final rule since *since*."""
|
||||
return _titled_rules(
|
||||
term="inpatient prospective payment",
|
||||
pattern=IPPS_TITLE_PATTERN,
|
||||
since=since,
|
||||
until=until,
|
||||
client=client,
|
||||
)
|
||||
|
||||
|
||||
def opps_rules(
|
||||
*,
|
||||
since: str = "2017-01-01",
|
||||
until: str | None = None,
|
||||
client: httpx.Client | None = None,
|
||||
) -> list[FRDocument]:
|
||||
"""Every annual OPPS/ASC proposed/final rule since *since*."""
|
||||
return _titled_rules(
|
||||
term="outpatient prospective payment",
|
||||
pattern=OPPS_TITLE_PATTERN,
|
||||
since=since,
|
||||
until=until,
|
||||
client=client,
|
||||
)
|
||||
|
||||
|
||||
def _norm_rule_title(title: str) -> str:
|
||||
t = " ".join(title.lower().split())
|
||||
return t.replace("face to face", "face-to-face")
|
||||
|
||||
|
||||
def is_dmepos_title(title: str) -> bool:
|
||||
"""True when *title* is a DMEPOS payment rule, not a drive-by mention.
|
||||
|
||||
A Physician Fee Schedule title is excluded even when it contains
|
||||
"(DMEPOS)": that clause is a conditions-of-payment rider on the PFS
|
||||
rule, and the PFS docket is already in the corpus.
|
||||
"""
|
||||
t = _norm_rule_title(title)
|
||||
if "physician fee schedule" in t:
|
||||
return False
|
||||
if "dmepos" in t:
|
||||
return True
|
||||
if "durable medical equipment" in t and "prosthetics" in t and "orthotics" in t:
|
||||
return True
|
||||
if _DMEPOS_SUPPLIER_PHRASE in t:
|
||||
return True
|
||||
return _DMEPOS_FEE_SCHEDULE in t or _DMEPOS_MASTER_LIST in t
|
||||
|
||||
|
||||
def dmepos_title_needle(title: str) -> str:
|
||||
"""Substring a reg.gov title search can require for this document.
|
||||
|
||||
Empty when the title is not a DMEPOS rule. Callers pass the result
|
||||
as ``title_contains`` only when it is non-empty.
|
||||
"""
|
||||
t = _norm_rule_title(title)
|
||||
if "physician fee schedule" in t:
|
||||
return ""
|
||||
if "dmepos" in t:
|
||||
return "dmepos"
|
||||
if "durable medical equipment" in t and "prosthetics" in t and "orthotics" in t:
|
||||
return "durable medical equipment"
|
||||
if _DMEPOS_SUPPLIER_PHRASE in t:
|
||||
return "custom-fabricated orthotics"
|
||||
if _DMEPOS_FEE_SCHEDULE in t:
|
||||
return _DMEPOS_FEE_SCHEDULE
|
||||
if _DMEPOS_MASTER_LIST in t:
|
||||
return "master list of items potentially subject"
|
||||
return ""
|
||||
|
||||
|
||||
def dmepos_rules(
|
||||
*,
|
||||
since: str = "2017-01-01",
|
||||
until: str | None = None,
|
||||
client: httpx.Client | None = None,
|
||||
) -> list[FRDocument]:
|
||||
"""DMEPOS proposed and final rules, notices included, since *since*.
|
||||
|
||||
Three Federal Register searches are unioned, then
|
||||
:func:`is_dmepos_title` drops body-only hits (PFS, IPPS, OPPS,
|
||||
hospice, home health years that do not name the benefit).
|
||||
"""
|
||||
found: list[FRDocument] = []
|
||||
seen: set[str] = set()
|
||||
for term in _DMEPOS_SEARCH_TERMS:
|
||||
for doc in search(
|
||||
term=term,
|
||||
agencies=["centers-for-medicare-medicaid-services"],
|
||||
types=["RULE", "PRORULE"],
|
||||
since=since,
|
||||
until=until,
|
||||
client=client,
|
||||
):
|
||||
if doc.document_number in seen or not is_dmepos_title(doc.title):
|
||||
continue
|
||||
seen.add(doc.document_number)
|
||||
found.append(doc)
|
||||
found.sort(key=lambda d: (d.publication_date, d.document_number))
|
||||
return found
|
||||
|
||||
|
||||
|
||||
@@ -144,7 +144,9 @@ class Client:
|
||||
|
||||
# ── Discovery ──────────────────────────────────────────────
|
||||
|
||||
def resolve_docket(self, cms_rule_id: str) -> str | None:
|
||||
def resolve_docket(
|
||||
self, cms_rule_id: str, *, title_contains: str = ""
|
||||
) -> str | None:
|
||||
"""Map a CMS rule ID (e.g. ``CMS-1676-P``) to its regulations.gov
|
||||
docket (e.g. ``CMS-2017-0092``).
|
||||
|
||||
@@ -154,21 +156,36 @@ class Client:
|
||||
surfaces the matching document, whose ``attributes.docketId``
|
||||
is the real docket we need for comment pulls.
|
||||
|
||||
The search also returns unrelated notices that merely mention
|
||||
the CMS id. Pass ``title_contains`` (for example ``"inpatient
|
||||
prospective payment"``) to skip those. Without it, the first
|
||||
hit wins, which is what the PFS rules have always used.
|
||||
|
||||
Returns None when no document matches — happens for rules that
|
||||
were never opened for public comment (corrections, IFCs).
|
||||
"""
|
||||
data = self._get(
|
||||
"/documents",
|
||||
**{
|
||||
"filter[searchTerm]": cms_rule_id,
|
||||
"filter[agencyId]": "CMS",
|
||||
"page[size]": 5,
|
||||
},
|
||||
)
|
||||
for row in data.get("data", []):
|
||||
docket = (row.get("attributes") or {}).get("docketId")
|
||||
if docket:
|
||||
return docket
|
||||
needle = _norm_title(title_contains)
|
||||
page = 1
|
||||
while page <= 4:
|
||||
data = self._get(
|
||||
"/documents",
|
||||
**{
|
||||
"filter[searchTerm]": cms_rule_id,
|
||||
"filter[agencyId]": "CMS",
|
||||
"page[size]": 20 if needle else 5,
|
||||
"page[number]": page,
|
||||
},
|
||||
)
|
||||
for row in data.get("data", []):
|
||||
attrs = row.get("attributes") or {}
|
||||
docket = attrs.get("docketId")
|
||||
title = _norm_title(attrs.get("title") or "")
|
||||
if docket and (not needle or needle in title):
|
||||
return docket
|
||||
total_pages = (data.get("meta") or {}).get("totalPages", 1) or 1
|
||||
if not needle or page >= total_pages:
|
||||
return None
|
||||
page += 1
|
||||
return None
|
||||
|
||||
def find_documents_in_docket(self, docket_id: str) -> list[dict]:
|
||||
@@ -754,6 +771,11 @@ def backfill_from_mirror(
|
||||
if limit:
|
||||
todo = todo[: int(limit)]
|
||||
cms_id = _rule_tag_for_docket(store, docket)
|
||||
if not cms_id:
|
||||
# A first fill has no comment to copy the rule tag from. The
|
||||
# docket row is the other place the CMS id is stored.
|
||||
registered = store.docket_get(docket)
|
||||
cms_id = registered.rule_cms_id if registered else ""
|
||||
|
||||
stats = {"created": 0, "enriched": 0, "attached": 0, "errors": 0, "seen": len(todo)}
|
||||
|
||||
@@ -895,13 +917,35 @@ def upsert_comment_status(
|
||||
# ── Docket walks ───────────────────────────────────────────────
|
||||
|
||||
|
||||
def discover_docket(client: Client, docket_id: str, *, rule_cms_id: str = "") -> Docket:
|
||||
def _norm_title(title: str) -> str:
|
||||
"""Lowercase and collapse whitespace.
|
||||
|
||||
regulations.gov wraps some annual-rule titles across lines
|
||||
(``Inpatient\\nProspective``). A raw substring check then misses
|
||||
the document the letters were filed on.
|
||||
"""
|
||||
return " ".join(title.lower().split())
|
||||
|
||||
|
||||
def discover_docket(
|
||||
client: Client,
|
||||
docket_id: str,
|
||||
*,
|
||||
rule_cms_id: str = "",
|
||||
title_contains: str = "",
|
||||
) -> Docket:
|
||||
"""One ``/documents`` listing → the docket's commentable document.
|
||||
|
||||
Picks the Proposed Rule that has an ``objectId`` and a
|
||||
``commentEndDate`` — latest close date wins among proposed rules;
|
||||
falls back to any commentable document. Called once per docket; the
|
||||
result is persisted so later runs make no API call.
|
||||
|
||||
``title_contains`` restricts the choice to documents whose title
|
||||
carries that phrase (the annual IPPS or OPPS rule, not a PRA notice
|
||||
filed on the same docket). When several still match, the one
|
||||
comments were actually filed on wins — an extension posted later
|
||||
can have the later close date and almost no letters.
|
||||
"""
|
||||
qualifying: list[dict] = []
|
||||
for fr_doc in client.find_documents_in_docket(docket_id):
|
||||
@@ -916,16 +960,34 @@ def discover_docket(client: Client, docket_id: str, *, rule_cms_id: str = "") ->
|
||||
if (fr_doc.get("attributes") or {}).get("documentType") == "Proposed Rule"
|
||||
]
|
||||
candidates = proposed_rules or qualifying
|
||||
needle = _norm_title(title_contains)
|
||||
if needle:
|
||||
titled = [
|
||||
fr_doc
|
||||
for fr_doc in candidates
|
||||
if needle
|
||||
in _norm_title((fr_doc.get("attributes") or {}).get("title") or "")
|
||||
]
|
||||
if titled:
|
||||
candidates = titled
|
||||
|
||||
best: dict | None = None
|
||||
for fr_doc in candidates:
|
||||
attrs = fr_doc.get("attributes") or {}
|
||||
if (
|
||||
best is None
|
||||
or attrs["commentEndDate"]
|
||||
> (best.get("attributes") or {})["commentEndDate"]
|
||||
):
|
||||
best = fr_doc
|
||||
if needle and len(candidates) > 1:
|
||||
best = max(candidates, key=lambda fr: _comment_total(client, fr))
|
||||
else:
|
||||
for fr_doc in candidates:
|
||||
attrs = fr_doc.get("attributes") or {}
|
||||
end = attrs["commentEndDate"]
|
||||
best_attrs = (best or {}).get("attributes") or {}
|
||||
best_end = best_attrs.get("commentEndDate") or ""
|
||||
if best is None or end > best_end:
|
||||
best = fr_doc
|
||||
elif end == best_end and _comment_total(client, fr_doc) > _comment_total(
|
||||
client, best or {}
|
||||
):
|
||||
# Two copies of the same NPRM often share a close date.
|
||||
# The display posting is the thin one.
|
||||
best = fr_doc
|
||||
attrs = (best or {}).get("attributes") or {}
|
||||
return Docket(
|
||||
id=docket_id,
|
||||
@@ -936,6 +998,61 @@ def discover_docket(client: Client, docket_id: str, *, rule_cms_id: str = "") ->
|
||||
)
|
||||
|
||||
|
||||
def _comment_total(client: Client, fr_doc: dict) -> int:
|
||||
obj = (fr_doc.get("attributes") or {}).get("objectId") or ""
|
||||
if not obj:
|
||||
return -1
|
||||
try:
|
||||
meta = (
|
||||
client._get( # noqa: SLF001
|
||||
"/comments",
|
||||
**{"filter[commentOnId]": obj, "page[size]": "5"},
|
||||
).get("meta")
|
||||
or {}
|
||||
)
|
||||
return int(meta.get("totalElements") or 0)
|
||||
except (httpx.HTTPError, TypeError, ValueError):
|
||||
return -1
|
||||
|
||||
|
||||
def pull_titled_siblings(
|
||||
store: Store,
|
||||
client: Client,
|
||||
docket: Docket,
|
||||
*,
|
||||
title_contains: str,
|
||||
) -> int:
|
||||
"""Upsert comments filed on this docket's other matching proposed rules.
|
||||
|
||||
The docket row tracks one document. A display copy or a correction
|
||||
posted as its own proposed rule carries a separate comment pile.
|
||||
Those letters get the same ``reg-docket`` and ``rule`` tags. The
|
||||
docket row is left alone, so the primary watermark stays put.
|
||||
"""
|
||||
needle = _norm_title(title_contains)
|
||||
if not needle:
|
||||
return 0
|
||||
n = 0
|
||||
for fr_doc in client.find_documents_in_docket(docket.id):
|
||||
attrs = fr_doc.get("attributes") or {}
|
||||
if attrs.get("documentType") != "Proposed Rule":
|
||||
continue
|
||||
if needle not in _norm_title(attrs.get("title") or ""):
|
||||
continue
|
||||
obj = attrs.get("objectId") or ""
|
||||
if not obj or obj == docket.fr_object_id:
|
||||
continue
|
||||
for comment in client.iter_comments(obj):
|
||||
upsert_comment_status(
|
||||
store,
|
||||
comment,
|
||||
cms_id=docket.rule_cms_id,
|
||||
extra_tags=[f"reg-docket:{docket.id}"],
|
||||
)
|
||||
n += 1
|
||||
return n
|
||||
|
||||
|
||||
@dataclass
|
||||
class WalkResult:
|
||||
created: int = 0
|
||||
|
||||
@@ -78,6 +78,9 @@ class Store:
|
||||
self._connection.row_factory = sqlite3.Row
|
||||
self._connection.execute("PRAGMA journal_mode=WAL")
|
||||
self._connection.execute("PRAGMA foreign_keys=ON")
|
||||
# The mail poller writes this file on its own cadence. A comment
|
||||
# walk waits through that burst instead of dying on SQLITE_BUSY.
|
||||
self._connection.execute("PRAGMA busy_timeout=60000")
|
||||
return self._connection
|
||||
|
||||
def _init_schema(self) -> None:
|
||||
|
||||
@@ -109,12 +109,13 @@ def federal_register(url: str) -> Rule:
|
||||
Populated rule model with FR citation, CMS ID, dates.
|
||||
"""
|
||||
# Extract document number from URL path
|
||||
doc_match = re.search(r"/documents/\d{4}/\d{2}/\d{2}/([\d-]+)/", url)
|
||||
# Correction documents are numbered C1-2021-27763, not just digits.
|
||||
doc_match = re.search(r"/documents/\d{4}/\d{2}/\d{2}/([^/]+)/", url)
|
||||
if doc_match:
|
||||
doc_number = doc_match.group(1)
|
||||
else:
|
||||
# Try API URL pattern
|
||||
api_match = re.search(r"/documents/([\d-]+)\.json", url)
|
||||
api_match = re.search(r"/documents/([^/]+)\.json", url)
|
||||
doc_number = api_match.group(1) if api_match else ""
|
||||
|
||||
api_url = f"https://www.federalregister.gov/api/v1/documents/{doc_number}.json"
|
||||
@@ -148,7 +149,7 @@ def federal_register(url: str) -> Rule:
|
||||
date_published=data.get("publication_date", ""),
|
||||
effective_date=data.get("effective_on") or "",
|
||||
url=data.get("html_url", url),
|
||||
abstract=data.get("abstract", ""),
|
||||
abstract=data.get("abstract") or "",
|
||||
)
|
||||
|
||||
rule.add_tag(Tag.source("federal-register").label)
|
||||
|
||||
336
src/cli/bib.py
336
src/cli/bib.py
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
import typer
|
||||
@@ -84,27 +85,14 @@ def _http_client():
|
||||
return httpx.Client(headers={"User-Agent": _UA})
|
||||
|
||||
|
||||
@app.command(name="discover-pfs-rules")
|
||||
def discover_pfs_rules(
|
||||
since: str = typer.Option("2017-01-01", "--since"),
|
||||
until: str = typer.Option("", "--until"),
|
||||
dry_run: bool = typer.Option(False, "--dry-run"),
|
||||
) -> None:
|
||||
"""Walk the Federal Register API for every PFS rule since a date.
|
||||
|
||||
Upserts each as a bib ``Rule`` record tagged ``module:pfs``,
|
||||
``source:federal-register``, ``rule:<cms-id>``. Cheap — the FR API
|
||||
has no auth and no rate charge we can trigger at this scale
|
||||
(~20 rules since 2017).
|
||||
"""
|
||||
def _discover_rules(docs, *, module: str, dry_run: bool) -> None:
|
||||
"""Upsert Federal Register rule documents under ``module:<module>``."""
|
||||
from bib import connect
|
||||
from bib.federalregister import pfs_rules
|
||||
from bib.tag import Tag
|
||||
from bib.translate import federal_register
|
||||
|
||||
store = connect() if not dry_run else None
|
||||
|
||||
for doc in pfs_rules(since=since, until=until or None):
|
||||
for doc in docs:
|
||||
typer.echo(
|
||||
f" {doc.publication_date} {doc.type:14s} "
|
||||
f"{doc.document_number:14s} {','.join(doc.dockets) or '(no docket)'}"
|
||||
@@ -120,12 +108,81 @@ def discover_pfs_rules(
|
||||
typer.echo(f" skipped: {e.__class__.__name__}: {e}")
|
||||
continue
|
||||
rule.add_tag(Tag.source("federal-register").label)
|
||||
rule.add_tag("module:pfs")
|
||||
rule.add_tag(f"module:{module}")
|
||||
for d in doc.dockets:
|
||||
rule.add_tag(f"docket:{d}")
|
||||
store.upsert(rule)
|
||||
|
||||
|
||||
@app.command(name="discover-pfs-rules")
|
||||
def discover_pfs_rules(
|
||||
since: str = typer.Option("2017-01-01", "--since"),
|
||||
until: str = typer.Option("", "--until"),
|
||||
dry_run: bool = typer.Option(False, "--dry-run"),
|
||||
) -> None:
|
||||
"""Walk the Federal Register API for every PFS rule since a date.
|
||||
|
||||
Upserts each as a bib ``Rule`` record tagged ``module:pfs``,
|
||||
``source:federal-register``, ``docket:<cms-id>``. Cheap — the FR API
|
||||
has no auth and no rate charge we can trigger at this scale
|
||||
(~20 rules since 2017).
|
||||
"""
|
||||
from bib.federalregister import pfs_rules
|
||||
|
||||
_discover_rules(
|
||||
pfs_rules(since=since, until=until or None), module="pfs", dry_run=dry_run
|
||||
)
|
||||
|
||||
|
||||
@app.command(name="discover-ipps-rules")
|
||||
def discover_ipps_rules(
|
||||
since: str = typer.Option("2017-01-01", "--since"),
|
||||
until: str = typer.Option("", "--until"),
|
||||
dry_run: bool = typer.Option(False, "--dry-run"),
|
||||
) -> None:
|
||||
"""Walk the Federal Register API for every annual IPPS/LTCH rule."""
|
||||
from bib.federalregister import ipps_rules
|
||||
|
||||
_discover_rules(
|
||||
ipps_rules(since=since, until=until or None), module="ipps", dry_run=dry_run
|
||||
)
|
||||
|
||||
|
||||
@app.command(name="discover-opps-rules")
|
||||
def discover_opps_rules(
|
||||
since: str = typer.Option("2017-01-01", "--since"),
|
||||
until: str = typer.Option("", "--until"),
|
||||
dry_run: bool = typer.Option(False, "--dry-run"),
|
||||
) -> None:
|
||||
"""Walk the Federal Register API for every annual OPPS/ASC rule."""
|
||||
from bib.federalregister import opps_rules
|
||||
|
||||
_discover_rules(
|
||||
opps_rules(since=since, until=until or None), module="opps", dry_run=dry_run
|
||||
)
|
||||
|
||||
|
||||
@app.command(name="discover-dmepos-rules")
|
||||
def discover_dmepos_rules(
|
||||
since: str = typer.Option("2017-01-01", "--since"),
|
||||
until: str = typer.Option("", "--until"),
|
||||
dry_run: bool = typer.Option(False, "--dry-run"),
|
||||
) -> None:
|
||||
"""Walk the Federal Register API for every DMEPOS rule since a date.
|
||||
|
||||
The benefit moved between the ESRD rule, a standalone HCPCS rule,
|
||||
and the home health rule. A title filter keeps those vehicles and
|
||||
drops a Physician Fee Schedule title that only mentions DMEPOS.
|
||||
"""
|
||||
from bib.federalregister import dmepos_rules
|
||||
|
||||
_discover_rules(
|
||||
dmepos_rules(since=since, until=until or None),
|
||||
module="dmepos",
|
||||
dry_run=dry_run,
|
||||
)
|
||||
|
||||
|
||||
def _walk_and_report(store, api, docket, *, attachments, limit, force, echo) -> None:
|
||||
from bib.dockets import quiet_days
|
||||
from bib.regulations_gov import walk_docket
|
||||
@@ -276,51 +333,33 @@ def pull_open_dockets(
|
||||
)
|
||||
|
||||
|
||||
@app.command(name="fetch-pfs-comments")
|
||||
def fetch_pfs_comments(
|
||||
since: str = typer.Option("2017-01-01", "--since"),
|
||||
until: str = typer.Option("", "--until"),
|
||||
attachments: bool = typer.Option(False, "--attachments"),
|
||||
per_docket_limit: int = typer.Option(
|
||||
0, "--per-docket-limit", help="Cap comments fetched per docket. 0 = unlimited."
|
||||
),
|
||||
sleep: float = typer.Option(1.3, "--sleep"),
|
||||
docket: str = typer.Option(
|
||||
"",
|
||||
"--docket",
|
||||
help=(
|
||||
"Only pull this reg.gov docket (e.g. CMS-2026-2377). Filtered "
|
||||
"before any API call ONLY once the docket is known (a `dockets` "
|
||||
"row already exists for its CMS id) — a CMS id with no `dockets` "
|
||||
"row yet still costs one Federal Register rule-metadata fetch "
|
||||
"plus one resolve_docket call, because the reg.gov docket id "
|
||||
"isn't known until those calls resolve it."
|
||||
),
|
||||
),
|
||||
force: bool = typer.Option(
|
||||
False,
|
||||
"--force",
|
||||
help="Walk every docket from page 1, sealed or not. Never unseals.",
|
||||
),
|
||||
def _fetch_rule_comments(
|
||||
rules,
|
||||
*,
|
||||
module: str,
|
||||
label: str,
|
||||
since: str,
|
||||
attachments: bool,
|
||||
per_docket_limit: int,
|
||||
sleep: float,
|
||||
docket: str,
|
||||
force: bool,
|
||||
title_contains: str | Callable[[object], str] = "",
|
||||
) -> None:
|
||||
"""Discover every PFS proposed rule since *since* and pull new comments
|
||||
on each docket from its stored watermark.
|
||||
"""Register each proposed rule's reg.gov docket and walk its comments.
|
||||
|
||||
Sealed dockets (comment period closed + quiet period + an empty pull)
|
||||
cost nothing: no rule-metadata fetch, no resolve call, no walk. The
|
||||
same is true for a docket already known via `--docket` that resolves
|
||||
to a different id. A CMS id with no `dockets` row at all is not free
|
||||
even under `--docket`: see that option's help.
|
||||
``title_contains`` is a substring every resolved document must
|
||||
carry, or a callable from the rule document to that substring.
|
||||
An empty result keeps the PFS first-hit resolve.
|
||||
"""
|
||||
from bib import connect
|
||||
from bib.federalregister import pfs_rules, split_docket_ids
|
||||
from bib.regulations_gov import Client, discover_docket
|
||||
from bib.federalregister import split_docket_ids
|
||||
from bib.regulations_gov import Client, discover_docket, pull_titled_siblings
|
||||
from bib.tag import Tag
|
||||
from bib.translate import federal_register
|
||||
|
||||
store = connect()
|
||||
rules = pfs_rules(since=since, until=until or None)
|
||||
typer.echo(f"==> {len(rules)} PFS rules since {since}")
|
||||
typer.echo(f"==> {len(rules)} {label} rules since {since}")
|
||||
proposed = [r for r in rules if r.type == "Proposed Rule"]
|
||||
typer.echo(f" {len(proposed)} proposed rules (the ones with comments)")
|
||||
|
||||
@@ -354,7 +393,7 @@ def fetch_pfs_comments(
|
||||
typer.echo(f" skipped rule meta: {e}")
|
||||
continue
|
||||
rule.add_tag(Tag.source("federal-register").label)
|
||||
rule.add_tag("module:pfs")
|
||||
rule.add_tag(f"module:{module}")
|
||||
for cid in cms_ids:
|
||||
rule.add_tag(f"cms-rule:{cid}")
|
||||
# Upsert now, before any docket resolution: a CMS id whose
|
||||
@@ -364,17 +403,36 @@ def fetch_pfs_comments(
|
||||
# the extra call a no-op when nothing changed.
|
||||
store.upsert(rule)
|
||||
|
||||
needle = title_contains(doc) if callable(title_contains) else title_contains
|
||||
for cms_id in todo:
|
||||
d = known[cms_id]
|
||||
if d is None:
|
||||
reg_docket = api.resolve_docket(cms_id)
|
||||
# An empty needle keeps the PFS first-hit call. Passing
|
||||
# title_contains="" would change that signature for every
|
||||
# existing caller.
|
||||
resolve_kw = {"title_contains": needle} if needle else {}
|
||||
reg_docket = api.resolve_docket(cms_id, **resolve_kw)
|
||||
if not reg_docket:
|
||||
typer.echo(f" skip {cms_id}: no reg.gov docket found")
|
||||
continue
|
||||
if docket and reg_docket != docket:
|
||||
continue
|
||||
d = discover_docket(api, reg_docket, rule_cms_id=cms_id)
|
||||
store.docket_upsert(d)
|
||||
# A correction (CMS-XXXX-CN) often lives on the same
|
||||
# reg.gov docket as the proposed rule. Re-upserting a
|
||||
# fresh row would wipe the watermark and the seal.
|
||||
d = store.docket_get(reg_docket)
|
||||
if d is None:
|
||||
d = discover_docket(
|
||||
api, reg_docket, rule_cms_id=cms_id, **resolve_kw
|
||||
)
|
||||
store.docket_upsert(d)
|
||||
if d.sealed and not force:
|
||||
typer.echo(
|
||||
f" {d.id}: sealed {d.sealed_at[:10]} ({d.seal_reason}); skipping"
|
||||
)
|
||||
rule.add_tag(f"reg-docket:{d.id}")
|
||||
store.upsert(rule)
|
||||
continue
|
||||
typer.echo(f" {cms_id} → {d.id} ({doc.publication_date})")
|
||||
rule.add_tag(f"reg-docket:{d.id}")
|
||||
store.upsert(rule)
|
||||
@@ -390,6 +448,168 @@ def fetch_pfs_comments(
|
||||
force=force,
|
||||
echo=typer.echo,
|
||||
)
|
||||
if needle and not per_docket_limit:
|
||||
extra = pull_titled_siblings(store, api, d, title_contains=needle)
|
||||
if extra:
|
||||
typer.echo(f" {d.id}: {extra} comments on sibling documents")
|
||||
|
||||
|
||||
_DOCKET_HELP = (
|
||||
"Only pull this reg.gov docket (e.g. CMS-2026-2377). Filtered "
|
||||
"before any API call ONLY once the docket is known (a `dockets` "
|
||||
"row already exists for its CMS id) — a CMS id with no `dockets` "
|
||||
"row yet still costs one Federal Register rule-metadata fetch "
|
||||
"plus one resolve_docket call, because the reg.gov docket id "
|
||||
"isn't known until those calls resolve it."
|
||||
)
|
||||
|
||||
|
||||
@app.command(name="fetch-pfs-comments")
|
||||
def fetch_pfs_comments(
|
||||
since: str = typer.Option("2017-01-01", "--since"),
|
||||
until: str = typer.Option("", "--until"),
|
||||
attachments: bool = typer.Option(False, "--attachments"),
|
||||
per_docket_limit: int = typer.Option(
|
||||
0, "--per-docket-limit", help="Cap comments fetched per docket. 0 = unlimited."
|
||||
),
|
||||
sleep: float = typer.Option(1.3, "--sleep"),
|
||||
docket: str = typer.Option("", "--docket", help=_DOCKET_HELP),
|
||||
force: bool = typer.Option(
|
||||
False,
|
||||
"--force",
|
||||
help="Walk every docket from page 1, sealed or not. Never unseals.",
|
||||
),
|
||||
) -> None:
|
||||
"""Discover every PFS proposed rule since *since* and pull new comments
|
||||
on each docket from its stored watermark.
|
||||
|
||||
Sealed dockets (comment period closed + quiet period + an empty pull)
|
||||
cost nothing: no rule-metadata fetch, no resolve call, no walk. The
|
||||
same is true for a docket already known via `--docket` that resolves
|
||||
to a different id. A CMS id with no `dockets` row at all is not free
|
||||
even under `--docket`: see that option's help.
|
||||
"""
|
||||
from bib.federalregister import pfs_rules
|
||||
|
||||
_fetch_rule_comments(
|
||||
pfs_rules(since=since, until=until or None),
|
||||
module="pfs",
|
||||
label="PFS",
|
||||
since=since,
|
||||
attachments=attachments,
|
||||
per_docket_limit=per_docket_limit,
|
||||
sleep=sleep,
|
||||
docket=docket,
|
||||
force=force,
|
||||
)
|
||||
|
||||
|
||||
@app.command(name="fetch-ipps-comments")
|
||||
def fetch_ipps_comments(
|
||||
since: str = typer.Option("2017-01-01", "--since"),
|
||||
until: str = typer.Option("", "--until"),
|
||||
attachments: bool = typer.Option(False, "--attachments"),
|
||||
per_docket_limit: int = typer.Option(
|
||||
0, "--per-docket-limit", help="Cap comments fetched per docket. 0 = unlimited."
|
||||
),
|
||||
sleep: float = typer.Option(1.3, "--sleep"),
|
||||
docket: str = typer.Option("", "--docket", help=_DOCKET_HELP),
|
||||
force: bool = typer.Option(
|
||||
False,
|
||||
"--force",
|
||||
help="Walk every docket from page 1, sealed or not. Never unseals.",
|
||||
),
|
||||
) -> None:
|
||||
"""Discover every annual IPPS/LTCH proposed rule and pull its comments."""
|
||||
from bib.federalregister import ipps_rules
|
||||
|
||||
_fetch_rule_comments(
|
||||
ipps_rules(since=since, until=until or None),
|
||||
module="ipps",
|
||||
label="IPPS",
|
||||
since=since,
|
||||
attachments=attachments,
|
||||
per_docket_limit=per_docket_limit,
|
||||
sleep=sleep,
|
||||
docket=docket,
|
||||
force=force,
|
||||
title_contains="inpatient prospective payment",
|
||||
)
|
||||
|
||||
|
||||
@app.command(name="fetch-opps-comments")
|
||||
def fetch_opps_comments(
|
||||
since: str = typer.Option("2017-01-01", "--since"),
|
||||
until: str = typer.Option("", "--until"),
|
||||
attachments: bool = typer.Option(False, "--attachments"),
|
||||
per_docket_limit: int = typer.Option(
|
||||
0, "--per-docket-limit", help="Cap comments fetched per docket. 0 = unlimited."
|
||||
),
|
||||
sleep: float = typer.Option(1.3, "--sleep"),
|
||||
docket: str = typer.Option("", "--docket", help=_DOCKET_HELP),
|
||||
force: bool = typer.Option(
|
||||
False,
|
||||
"--force",
|
||||
help="Walk every docket from page 1, sealed or not. Never unseals.",
|
||||
),
|
||||
) -> None:
|
||||
"""Discover every annual OPPS/ASC proposed rule and pull its comments."""
|
||||
from bib.federalregister import opps_rules
|
||||
|
||||
_fetch_rule_comments(
|
||||
opps_rules(since=since, until=until or None),
|
||||
module="opps",
|
||||
label="OPPS",
|
||||
since=since,
|
||||
attachments=attachments,
|
||||
per_docket_limit=per_docket_limit,
|
||||
sleep=sleep,
|
||||
docket=docket,
|
||||
force=force,
|
||||
title_contains="outpatient prospective payment",
|
||||
)
|
||||
|
||||
|
||||
@app.command(name="fetch-dmepos-comments")
|
||||
def fetch_dmepos_comments(
|
||||
since: str = typer.Option("2017-01-01", "--since"),
|
||||
until: str = typer.Option("", "--until"),
|
||||
attachments: bool = typer.Option(False, "--attachments"),
|
||||
per_docket_limit: int = typer.Option(
|
||||
0, "--per-docket-limit", help="Cap comments fetched per docket. 0 = unlimited."
|
||||
),
|
||||
sleep: float = typer.Option(1.3, "--sleep"),
|
||||
docket: str = typer.Option("", "--docket", help=_DOCKET_HELP),
|
||||
force: bool = typer.Option(
|
||||
False,
|
||||
"--force",
|
||||
help="Walk every docket from page 1, sealed or not. Never unseals.",
|
||||
),
|
||||
) -> None:
|
||||
"""Discover every DMEPOS proposed rule since *since* and pull its comments.
|
||||
|
||||
Combined ESRD and home-health vehicles come in whole: the comment
|
||||
file is the docket's file, not a DMEPOS-only slice. Notices and final
|
||||
rules are registered by ``discover-dmepos-rules`` and are not walked.
|
||||
|
||||
No title needle is passed to regulations.gov. The agency's copy of a
|
||||
long title is often truncated (``etc.``) or hyphenated differently
|
||||
from the Federal Register, and the short display posting is then
|
||||
the only document that still contains "DMEPOS".
|
||||
"""
|
||||
from bib.federalregister import dmepos_rules
|
||||
|
||||
_fetch_rule_comments(
|
||||
dmepos_rules(since=since, until=until or None),
|
||||
module="dmepos",
|
||||
label="DMEPOS",
|
||||
since=since,
|
||||
attachments=attachments,
|
||||
per_docket_limit=per_docket_limit,
|
||||
sleep=sleep,
|
||||
docket=docket,
|
||||
force=force,
|
||||
)
|
||||
|
||||
|
||||
@app.command(name="ingest-mail")
|
||||
|
||||
@@ -5,9 +5,16 @@ from __future__ import annotations
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from bib.federalregister import (
|
||||
IPPS_TITLE_PATTERN,
|
||||
OPPS_TITLE_PATTERN,
|
||||
PFS_TITLE_PATTERN,
|
||||
FRDocument,
|
||||
_to_doc,
|
||||
dmepos_rules,
|
||||
dmepos_title_needle,
|
||||
ipps_rules,
|
||||
is_dmepos_title,
|
||||
opps_rules,
|
||||
pfs_rules,
|
||||
search,
|
||||
split_docket_ids,
|
||||
@@ -176,3 +183,176 @@ class TestPfsRules:
|
||||
client.get.return_value = _mock_fr_response([doc, doc])
|
||||
found = pfs_rules(since="2023-01-01")
|
||||
assert len(found) == 1
|
||||
|
||||
|
||||
class TestIppsOppsRules:
|
||||
@patch("bib.federalregister.httpx.Client")
|
||||
def test_ipps_keeps_the_annual_rule_only(self, MockClient):
|
||||
client = MockClient.return_value
|
||||
client.close = MagicMock()
|
||||
client.get.return_value = _mock_fr_response(
|
||||
[
|
||||
{
|
||||
"document_number": "A",
|
||||
"title": f"Medicare Program; {IPPS_TITLE_PATTERN} and FY 2019 Rates",
|
||||
"type": "Proposed Rule",
|
||||
"publication_date": "2018-05-07",
|
||||
},
|
||||
{
|
||||
"document_number": "B",
|
||||
"title": (
|
||||
"Hospital Inpatient Prospective Payment Systems for "
|
||||
"Acute Care Hospitals; Graduate Medical Education"
|
||||
),
|
||||
"type": "Rule",
|
||||
"publication_date": "2021-12-27",
|
||||
},
|
||||
{
|
||||
"document_number": "C",
|
||||
"title": IPPS_TITLE_PATTERN.replace("Long-Term", "Long Term"),
|
||||
"type": "Rule",
|
||||
"publication_date": "2018-08-17",
|
||||
},
|
||||
{
|
||||
"document_number": "D",
|
||||
"title": IPPS_TITLE_PATTERN.replace(
|
||||
"Acute Care Hospitals", "Acute Care Hospitals (IPPS)"
|
||||
),
|
||||
"type": "Proposed Rule",
|
||||
"publication_date": "2026-04-14",
|
||||
},
|
||||
]
|
||||
)
|
||||
found = ipps_rules(since="2017-01-01")
|
||||
assert [d.document_number for d in found] == ["A", "C", "D"]
|
||||
|
||||
@patch("bib.federalregister.httpx.Client")
|
||||
def test_opps_keeps_the_annual_rule_only(self, MockClient):
|
||||
client = MockClient.return_value
|
||||
client.close = MagicMock()
|
||||
client.get.return_value = _mock_fr_response(
|
||||
[
|
||||
{
|
||||
"document_number": "A",
|
||||
"title": f"Medicare Program: {OPPS_TITLE_PATTERN}",
|
||||
"type": "Proposed Rule",
|
||||
"publication_date": "2024-07-22",
|
||||
},
|
||||
{
|
||||
"document_number": "B",
|
||||
"title": (
|
||||
"Hospital Outpatient Prospective Payment System: "
|
||||
"Remedy for the 340B-Acquired Drug Payment Policy"
|
||||
),
|
||||
"type": "Proposed Rule",
|
||||
"publication_date": "2023-07-11",
|
||||
},
|
||||
]
|
||||
)
|
||||
found = opps_rules(since="2017-01-01")
|
||||
assert [d.document_number for d in found] == ["A"]
|
||||
|
||||
|
||||
class TestDmeposRules:
|
||||
def test_title_keeps_the_benefit_and_drops_pfs(self):
|
||||
esrd = (
|
||||
"End-Stage Renal Disease Prospective Payment System, "
|
||||
"Durable Medical Equipment, Prosthetics, Orthotics and "
|
||||
"Supplies (DMEPOS) Competitive Bidding Program"
|
||||
)
|
||||
home_health = (
|
||||
"Certain Requirements for Durable Medical Equipment "
|
||||
"Prosthetics and Orthotics Supplies"
|
||||
)
|
||||
suppliers = (
|
||||
"Requirements for Qualified Practitioners and Qualified "
|
||||
"Suppliers of Prosthetics and Custom-Fabricated Orthotics"
|
||||
)
|
||||
pfs = (
|
||||
f"{PFS_TITLE_PATTERN} and Conditions of Payment for "
|
||||
"Suppliers of Durable Medicaid Equipment, Prosthetics, "
|
||||
"Orthotics, and Supplies (DMEPOS)"
|
||||
)
|
||||
opps_prior_auth = f"{OPPS_TITLE_PATTERN} and Prior Authorization Process"
|
||||
hospice = "Hospice Wage Index and Durable Medical Equipment mention"
|
||||
other_hh = "Home Health Prospective Payment System; Other Medicare Policies"
|
||||
|
||||
assert is_dmepos_title(esrd)
|
||||
assert is_dmepos_title(home_health)
|
||||
assert is_dmepos_title(suppliers)
|
||||
assert not is_dmepos_title(pfs)
|
||||
assert not is_dmepos_title(opps_prior_auth)
|
||||
assert not is_dmepos_title(hospice)
|
||||
assert not is_dmepos_title(other_hh)
|
||||
fee = (
|
||||
"Durable Medical Equipment Fee Schedule Adjustments To Resume "
|
||||
"the Transitional 50/50 Blended Rates"
|
||||
)
|
||||
master = (
|
||||
"Updates to the Master List of Items Potentially Subject to "
|
||||
"Face to Face Encounter and Written Order Prior to Delivery"
|
||||
)
|
||||
assert is_dmepos_title(fee)
|
||||
assert is_dmepos_title(master)
|
||||
assert dmepos_title_needle(esrd) == "dmepos"
|
||||
assert dmepos_title_needle(home_health) == "durable medical equipment"
|
||||
assert dmepos_title_needle(suppliers) == "custom-fabricated orthotics"
|
||||
assert dmepos_title_needle(fee) == "durable medical equipment fee schedule"
|
||||
assert dmepos_title_needle(master) == "master list of items potentially subject"
|
||||
assert dmepos_title_needle(pfs) == ""
|
||||
|
||||
@patch("bib.federalregister.httpx.Client")
|
||||
def test_search_union_keeps_annual_vehicles_only(self, MockClient):
|
||||
client = MockClient.return_value
|
||||
client.close = MagicMock()
|
||||
client.get.return_value = _mock_fr_response(
|
||||
[
|
||||
{
|
||||
"document_number": "2018-14986",
|
||||
"title": (
|
||||
"Durable Medical Equipment, Prosthetics, Orthotics "
|
||||
"and Supplies (DMEPOS) Competitive Bidding Program"
|
||||
),
|
||||
"type": "Proposed Rule",
|
||||
"publication_date": "2018-07-19",
|
||||
},
|
||||
{
|
||||
"document_number": "2022-14562",
|
||||
"title": (
|
||||
f"Revisions to {PFS_TITLE_PATTERN} and Supplies (DMEPOS)"
|
||||
),
|
||||
"type": "Proposed Rule",
|
||||
"publication_date": "2022-07-29",
|
||||
},
|
||||
{
|
||||
"document_number": "2023-14044",
|
||||
"title": (
|
||||
"Certain Requirements for Durable Medical Equipment "
|
||||
"Prosthetics and Orthotics Supplies"
|
||||
),
|
||||
"type": "Proposed Rule",
|
||||
"publication_date": "2023-07-10",
|
||||
},
|
||||
{
|
||||
"document_number": "2017-00425",
|
||||
"title": (
|
||||
"Qualified Suppliers of Prosthetics and "
|
||||
"Custom-Fabricated Orthotics"
|
||||
),
|
||||
"type": "Proposed Rule",
|
||||
"publication_date": "2017-01-12",
|
||||
},
|
||||
{
|
||||
"document_number": "2024-15087",
|
||||
"title": f"{OPPS_TITLE_PATTERN} and Prior Authorization",
|
||||
"type": "Proposed Rule",
|
||||
"publication_date": "2024-07-22",
|
||||
},
|
||||
]
|
||||
)
|
||||
found = dmepos_rules(since="2017-01-01")
|
||||
assert [d.document_number for d in found] == [
|
||||
"2017-00425",
|
||||
"2018-14986",
|
||||
"2023-14044",
|
||||
]
|
||||
|
||||
@@ -221,6 +221,40 @@ class TestClient:
|
||||
result = c.resolve_docket("CMS-1676-P")
|
||||
assert result == "CMS-2017-0092"
|
||||
|
||||
@patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test-key"})
|
||||
def test_resolve_docket_skips_titles_that_do_not_match(self):
|
||||
mock_client = MagicMock()
|
||||
resp = MagicMock()
|
||||
resp.status_code = 200
|
||||
resp.json.return_value = {
|
||||
"meta": {"totalPages": 1},
|
||||
"data": [
|
||||
{
|
||||
"attributes": {
|
||||
"docketId": "CMS-2019-0132",
|
||||
"title": "Agency Information Collection Activities",
|
||||
}
|
||||
},
|
||||
{
|
||||
"attributes": {
|
||||
"docketId": "CMS-2019-0073",
|
||||
"title": "Hospital Inpatient Prospective Payment Systems",
|
||||
}
|
||||
},
|
||||
],
|
||||
}
|
||||
resp.raise_for_status = MagicMock()
|
||||
mock_client.get.return_value = resp
|
||||
|
||||
with patch("bib.regulations_gov.time.sleep"):
|
||||
c = Client(sleep=0, client=mock_client)
|
||||
assert (
|
||||
c.resolve_docket(
|
||||
"CMS-1716-P", title_contains="inpatient prospective payment"
|
||||
)
|
||||
== "CMS-2019-0073"
|
||||
)
|
||||
|
||||
@patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test-key"})
|
||||
def test_resolve_docket_not_found(self):
|
||||
mock_client = MagicMock()
|
||||
|
||||
@@ -6,6 +6,7 @@ import json
|
||||
|
||||
import httpx
|
||||
|
||||
from bib.dockets import Docket
|
||||
from bib.regulations_gov import (
|
||||
Comment,
|
||||
Mirror,
|
||||
@@ -166,6 +167,28 @@ class TestBackfillFromMirror:
|
||||
att = tmp_path / "scratch" / _DOCKET / f"{_DOCKET}-0002" / "attachment_1.pdf"
|
||||
assert att.is_file() and att.read_bytes().startswith(b"%PDF")
|
||||
|
||||
def test_rule_tag_from_docket_row_when_no_comment_has_one(self, tmp_path):
|
||||
store = Store(str(tmp_path / "bib.sqlite"))
|
||||
store.docket_upsert(Docket(id=_DOCKET, rule_cms_id="CMS-1694-P"))
|
||||
stats = backfill_from_mirror(
|
||||
store,
|
||||
_mirror(),
|
||||
docket=_DOCKET,
|
||||
scratch_root=tmp_path / "scratch",
|
||||
workers=2,
|
||||
)
|
||||
assert stats["created"] == 2
|
||||
row = store._con().execute("SELECT id FROM items LIMIT 1").fetchone()
|
||||
tags = {
|
||||
r[0]
|
||||
for r in store._con().execute(
|
||||
"SELECT t.name FROM tags t JOIN item_tags it ON it.tag_id=t.id "
|
||||
"WHERE it.item_id=?",
|
||||
(row[0],),
|
||||
)
|
||||
}
|
||||
assert "rule:CMS-1694-P" in tags
|
||||
|
||||
def test_idempotent_rerun(self, tmp_path):
|
||||
store = self._store(tmp_path)
|
||||
m = _mirror()
|
||||
|
||||
@@ -85,6 +85,27 @@ class TestFederalRegister:
|
||||
assert "year:2025" in rule.tags
|
||||
assert "rule:cms-1832-f" in rule.tags
|
||||
|
||||
def test_correction_document_number(self) -> None:
|
||||
url = (
|
||||
"https://www.federalregister.gov/documents/2022/01/04/"
|
||||
"C1-2021-27763/medicare-program"
|
||||
)
|
||||
data = self._mock_fr_data()
|
||||
with patch("bib.translate._fetch_json", return_value=data) as fetch:
|
||||
rule = federal_register(url)
|
||||
assert fetch.call_args.args[0].endswith("/C1-2021-27763.json")
|
||||
assert rule.document_number == "C1-2021-27763"
|
||||
|
||||
def test_missing_abstract_is_empty(self) -> None:
|
||||
url = (
|
||||
"https://www.federalregister.gov/documents/2022/01/04/"
|
||||
"C1-2021-27763/medicare-program"
|
||||
)
|
||||
data = self._mock_fr_data(abstract=None)
|
||||
with patch("bib.translate._fetch_json", return_value=data):
|
||||
rule = federal_register(url)
|
||||
assert rule.abstract == ""
|
||||
|
||||
def test_from_api_url(self) -> None:
|
||||
url = "https://www.federalregister.gov/api/v1/documents/2025-19787.json"
|
||||
data = self._mock_fr_data()
|
||||
|
||||
@@ -104,6 +104,159 @@ def test_discover_docket_prefers_proposed_rule():
|
||||
)
|
||||
|
||||
|
||||
def test_discover_title_picks_the_document_comments_were_filed_on():
|
||||
api = MagicMock()
|
||||
api.find_documents_in_docket.return_value = [
|
||||
{
|
||||
"id": "late",
|
||||
"attributes": {
|
||||
"objectId": "small",
|
||||
"commentEndDate": "2024-11-28T04:59:59Z",
|
||||
"documentType": "Proposed Rule",
|
||||
"title": "Hospital Outpatient Prospective Payment extension",
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "main",
|
||||
"attributes": {
|
||||
"objectId": "big",
|
||||
"commentEndDate": "2024-09-10T03:59:59Z",
|
||||
"documentType": "Proposed Rule",
|
||||
"title": "Hospital Outpatient Prospective Payment and ASC",
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
def _get(_path, **params):
|
||||
obj = params["filter[commentOnId]"]
|
||||
return {"meta": {"totalElements": 7 if obj == "small" else 2483}}
|
||||
|
||||
api._get.side_effect = _get
|
||||
d = discover_docket(
|
||||
api,
|
||||
"CMS-2024-0199",
|
||||
rule_cms_id="CMS-1809-P",
|
||||
title_contains="outpatient prospective payment",
|
||||
)
|
||||
assert d.fr_object_id == "big"
|
||||
assert d.comment_end_date == "2024-09-10"
|
||||
|
||||
|
||||
def test_discover_title_joins_a_line_wrapped_nprm():
|
||||
api = MagicMock()
|
||||
api.find_documents_in_docket.return_value = [
|
||||
{
|
||||
"id": "display",
|
||||
"attributes": {
|
||||
"objectId": "small",
|
||||
"commentEndDate": "2020-05-30T03:59:59Z",
|
||||
"documentType": "Proposed Rule",
|
||||
"title": "Hospital Inpatient Prospective Payment Systems display",
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "wrapped",
|
||||
"attributes": {
|
||||
"objectId": "big",
|
||||
"commentEndDate": "2020-07-11T03:59:59Z",
|
||||
"documentType": "Proposed Rule",
|
||||
"title": "Hospital Inpatient\nProspective Payment Systems for Acute Care",
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
def _get(_path, **params):
|
||||
obj = params["filter[commentOnId]"]
|
||||
return {"meta": {"totalElements": 25 if obj == "small" else 832}}
|
||||
|
||||
api._get.side_effect = _get
|
||||
d = discover_docket(
|
||||
api,
|
||||
"CMS-2020-0052",
|
||||
rule_cms_id="CMS-1735-P",
|
||||
title_contains="inpatient prospective payment",
|
||||
)
|
||||
assert d.fr_object_id == "big"
|
||||
assert d.comment_end_date == "2020-07-11"
|
||||
|
||||
|
||||
def test_discover_same_close_date_picks_the_comment_pile():
|
||||
api = MagicMock()
|
||||
api.find_documents_in_docket.return_value = [
|
||||
{
|
||||
"id": "display",
|
||||
"attributes": {
|
||||
"objectId": "small",
|
||||
"commentEndDate": "2023-08-30T03:59:59Z",
|
||||
"documentType": "Proposed Rule",
|
||||
"title": "Home Health display",
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "main",
|
||||
"attributes": {
|
||||
"objectId": "big",
|
||||
"commentEndDate": "2023-08-30T03:59:59Z",
|
||||
"documentType": "Proposed Rule",
|
||||
"title": "Home Health Prospective Payment System",
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
def _get(_path, **params):
|
||||
obj = params["filter[commentOnId]"]
|
||||
return {"meta": {"totalElements": 150 if obj == "small" else 750}}
|
||||
|
||||
api._get.side_effect = _get
|
||||
d = discover_docket(api, "CMS-2023-0113", rule_cms_id="CMS-1780-P")
|
||||
assert d.fr_object_id == "big"
|
||||
assert d.fr_document_id == "main"
|
||||
|
||||
|
||||
def test_pull_titled_siblings_leaves_the_docket_row():
|
||||
from bib.regulations_gov import Comment, pull_titled_siblings
|
||||
|
||||
s = _store()
|
||||
s.docket_upsert(_docket(fr_object_id="primary"))
|
||||
api = MagicMock()
|
||||
api.find_documents_in_docket.return_value = [
|
||||
{
|
||||
"id": "main",
|
||||
"attributes": {
|
||||
"objectId": "primary",
|
||||
"documentType": "Proposed Rule",
|
||||
"title": "Inpatient Prospective Payment Systems",
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": "display",
|
||||
"attributes": {
|
||||
"objectId": "other",
|
||||
"documentType": "Proposed Rule",
|
||||
"title": "Inpatient Prospective Payment display",
|
||||
},
|
||||
},
|
||||
]
|
||||
api.iter_comments.return_value = [
|
||||
Comment(
|
||||
id=f"{D}-9",
|
||||
title="",
|
||||
posted_date="2018-06-01",
|
||||
received_date="2018-06-01",
|
||||
docket_id=D,
|
||||
comment_on_id="other",
|
||||
comment_text="display copy",
|
||||
)
|
||||
]
|
||||
n = pull_titled_siblings(
|
||||
s, api, s.docket_get(D), title_contains="inpatient prospective payment"
|
||||
)
|
||||
assert n == 1
|
||||
saved = s.docket_get(D)
|
||||
assert saved is not None and saved.fr_object_id == "primary"
|
||||
assert s._con().execute("SELECT count(*) FROM items").fetchone()[0] == 1
|
||||
|
||||
|
||||
def test_walk_creates_counts_and_advances_watermark():
|
||||
s = _store()
|
||||
s.docket_upsert(_docket())
|
||||
|
||||
209
tests/cli/test_bib_payment_comments.py
Normal file
209
tests/cli/test_bib_payment_comments.py
Normal file
@@ -0,0 +1,209 @@
|
||||
"""IPPS and OPPS comment discovery uses the PFS register-and-walk path."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from bib.dockets import Docket
|
||||
from bib.item import Rule
|
||||
from bib.regulations_gov import WalkResult
|
||||
from bib.store import Store
|
||||
from cli.bib import app
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
def _api():
|
||||
api = MagicMock()
|
||||
api.__enter__ = MagicMock(return_value=api)
|
||||
api.__exit__ = MagicMock(return_value=False)
|
||||
api.find_documents_in_docket.return_value = []
|
||||
api.iter_comments.return_value = []
|
||||
return api
|
||||
|
||||
|
||||
def _doc(number: str, cms_id: str):
|
||||
doc = MagicMock()
|
||||
doc.type = "Proposed Rule"
|
||||
doc.publication_date = "2018-05-07"
|
||||
doc.dockets = [cms_id]
|
||||
doc.html_url = f"https://example.com/{number}"
|
||||
doc.document_number = number
|
||||
return doc
|
||||
|
||||
|
||||
@patch("bib.federalregister.ipps_rules", return_value=[])
|
||||
@patch("bib.connect")
|
||||
def test_fetch_ipps_comments_empty(mc_connect, _rules):
|
||||
mc_connect.return_value = MagicMock()
|
||||
api = _api()
|
||||
with patch("bib.regulations_gov.Client", return_value=api):
|
||||
result = runner.invoke(app, ["fetch-ipps-comments"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "IPPS rules" in result.output
|
||||
|
||||
|
||||
@patch("bib.federalregister.opps_rules", return_value=[])
|
||||
@patch("bib.connect")
|
||||
def test_fetch_opps_comments_empty(mc_connect, _rules):
|
||||
mc_connect.return_value = MagicMock()
|
||||
api = _api()
|
||||
with patch("bib.regulations_gov.Client", return_value=api):
|
||||
result = runner.invoke(app, ["fetch-opps-comments"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "OPPS rules" in result.output
|
||||
|
||||
|
||||
@patch("bib.federalregister.ipps_rules", return_value=[])
|
||||
def test_discover_ipps_dry_run(_rules):
|
||||
result = runner.invoke(app, ["discover-ipps-rules", "--dry-run"])
|
||||
assert result.exit_code == 0
|
||||
|
||||
|
||||
@patch("bib.federalregister.opps_rules", return_value=[])
|
||||
def test_discover_opps_dry_run(_rules):
|
||||
result = runner.invoke(app, ["discover-opps-rules", "--dry-run"])
|
||||
assert result.exit_code == 0
|
||||
|
||||
|
||||
@patch("bib.regulations_gov.walk_docket", return_value=WalkResult())
|
||||
@patch("bib.regulations_gov.discover_docket")
|
||||
@patch("bib.regulations_gov.Client")
|
||||
@patch("bib.translate.federal_register")
|
||||
@patch("bib.federalregister.ipps_rules")
|
||||
@patch("bib.connect")
|
||||
def test_correction_reuses_the_proposed_rule_docket(
|
||||
mc_connect, mc_rules, mc_fr, mc_client, mc_disc, mc_walk
|
||||
):
|
||||
store = Store(":memory:", storage_dir="/tmp/nope")
|
||||
mc_connect.return_value = store
|
||||
mc_rules.return_value = [
|
||||
_doc("2018-1", "CMS-1694-P"),
|
||||
_doc("2018-2", "CMS-1694-CN"),
|
||||
]
|
||||
mc_fr.return_value = Rule(title="FY2019 IPPS", url="https://example.com/2018-1")
|
||||
api = _api()
|
||||
api.resolve_docket.return_value = "CMS-2018-0040"
|
||||
mc_client.return_value = api
|
||||
|
||||
def _discover(_api, docket_id, rule_cms_id="", title_contains=""):
|
||||
assert title_contains == "inpatient prospective payment"
|
||||
return Docket(
|
||||
id=docket_id,
|
||||
rule_cms_id=rule_cms_id,
|
||||
fr_object_id="obj",
|
||||
comment_end_date="2018-06-25",
|
||||
)
|
||||
|
||||
mc_disc.side_effect = _discover
|
||||
|
||||
result = runner.invoke(app, ["fetch-ipps-comments"])
|
||||
assert result.exit_code == 0, result.output
|
||||
mc_disc.assert_called_once()
|
||||
saved = store.docket_get("CMS-2018-0040")
|
||||
assert saved is not None
|
||||
assert saved.rule_cms_id == "CMS-1694-P"
|
||||
assert mc_walk.call_count == 2
|
||||
|
||||
|
||||
@patch("bib.regulations_gov.walk_docket", return_value=WalkResult())
|
||||
@patch("bib.regulations_gov.discover_docket")
|
||||
@patch("bib.regulations_gov.Client")
|
||||
@patch("bib.translate.federal_register")
|
||||
@patch("bib.federalregister.opps_rules")
|
||||
@patch("bib.connect")
|
||||
def test_reused_sealed_docket_is_not_walked(
|
||||
mc_connect, mc_rules, mc_fr, mc_client, mc_disc, mc_walk
|
||||
):
|
||||
store = Store(":memory:", storage_dir="/tmp/nope")
|
||||
store.docket_upsert(
|
||||
Docket(
|
||||
id="CMS-2018-0042",
|
||||
rule_cms_id="CMS-1695-P",
|
||||
fr_object_id="obj",
|
||||
sealed_at="2020-01-01T00:00:00Z",
|
||||
seal_reason="auto",
|
||||
)
|
||||
)
|
||||
mc_connect.return_value = store
|
||||
mc_rules.return_value = [_doc("2018-3", "CMS-1695-CN")]
|
||||
mc_fr.return_value = Rule(title="CY2019 OPPS", url="https://example.com/2018-3")
|
||||
api = _api()
|
||||
api.resolve_docket.return_value = "CMS-2018-0042"
|
||||
mc_client.return_value = api
|
||||
|
||||
result = runner.invoke(app, ["fetch-opps-comments"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "sealed" in result.output
|
||||
mc_disc.assert_not_called()
|
||||
mc_walk.assert_not_called()
|
||||
|
||||
|
||||
@patch("bib.federalregister.dmepos_rules", return_value=[])
|
||||
@patch("bib.connect")
|
||||
def test_fetch_dmepos_comments_empty(mc_connect, _rules):
|
||||
mc_connect.return_value = MagicMock()
|
||||
api = _api()
|
||||
with patch("bib.regulations_gov.Client", return_value=api):
|
||||
result = runner.invoke(app, ["fetch-dmepos-comments"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "DMEPOS rules" in result.output
|
||||
|
||||
|
||||
@patch("bib.federalregister.dmepos_rules", return_value=[])
|
||||
def test_discover_dmepos_dry_run(_rules):
|
||||
result = runner.invoke(app, ["discover-dmepos-rules", "--dry-run"])
|
||||
assert result.exit_code == 0
|
||||
|
||||
|
||||
@patch("bib.regulations_gov.walk_docket", return_value=WalkResult())
|
||||
@patch("bib.regulations_gov.pull_titled_siblings", return_value=0)
|
||||
@patch("bib.regulations_gov.discover_docket")
|
||||
@patch("bib.regulations_gov.Client")
|
||||
@patch("bib.translate.federal_register")
|
||||
@patch("bib.federalregister.dmepos_rules")
|
||||
@patch("bib.connect")
|
||||
def test_dmepos_resolve_omits_the_title_needle(
|
||||
mc_connect, mc_rules, mc_fr, mc_client, mc_disc, _siblings, _walk
|
||||
):
|
||||
store = Store(":memory:", storage_dir="/tmp/nope")
|
||||
mc_connect.return_value = store
|
||||
acronym = _doc("2018-14986", "CMS-1691-P")
|
||||
acronym.title = (
|
||||
"Durable Medical Equipment, Prosthetics, Orthotics and "
|
||||
"Supplies (DMEPOS) Competitive Bidding Program"
|
||||
)
|
||||
spelled = _doc("2023-14044", "CMS-1780-P")
|
||||
spelled.title = (
|
||||
"Certain Requirements for Durable Medical Equipment "
|
||||
"Prosthetics and Orthotics Supplies"
|
||||
)
|
||||
mc_rules.return_value = [acronym, spelled]
|
||||
mc_fr.return_value = Rule(title="DMEPOS", url="https://example.com/dmepos")
|
||||
api = _api()
|
||||
needles: list[tuple[str, str]] = []
|
||||
|
||||
def _resolve(cms_id, title_contains=""):
|
||||
needles.append((cms_id, title_contains))
|
||||
return "CMS-2018-0099" if cms_id == "CMS-1691-P" else "CMS-2023-0100"
|
||||
|
||||
api.resolve_docket.side_effect = _resolve
|
||||
mc_client.return_value = api
|
||||
|
||||
def _discover(_api, docket_id, rule_cms_id="", title_contains=""):
|
||||
return Docket(
|
||||
id=docket_id,
|
||||
rule_cms_id=rule_cms_id,
|
||||
fr_object_id="obj",
|
||||
comment_end_date="2018-09-10",
|
||||
)
|
||||
|
||||
mc_disc.side_effect = _discover
|
||||
|
||||
result = runner.invoke(app, ["fetch-dmepos-comments"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert needles == [("CMS-1691-P", ""), ("CMS-1780-P", "")]
|
||||
for call in api.resolve_docket.call_args_list:
|
||||
assert "title_contains" not in call.kwargs
|
||||
Reference in New Issue
Block a user