diff --git a/src/bib/federalregister.py b/src/bib/federalregister.py index cf38ea6..a24cafd 100644 --- a/src/bib/federalregister.py +++ b/src/bib/federalregister.py @@ -35,6 +35,44 @@ _BASE = "https://www.federalregister.gov/api/v1" # rules that only mention the phrase "physician fee schedule" in passing. PFS_TITLE_PATTERN = "Payment Policies Under the Physician Fee Schedule" +# The annual IPPS/LTCH rule. Requiring the long-term-care clause keeps +# out the GME/organ-acquisition rule, which shares only the acute-care +# half of the title. "Long Term" and "Long-Term" both occur. +IPPS_TITLE_PATTERN = ( + "Hospital Inpatient Prospective Payment Systems for Acute Care Hospitals " + "and the Long-Term Care Hospital Prospective Payment System" +) + +# The annual OPPS/ASC rule. The 340B remedy rule says "Outpatient +# Prospective Payment System" and does not name the ASC payment system. +OPPS_TITLE_PATTERN = ( + "Hospital Outpatient Prospective Payment and Ambulatory Surgical Center " + "Payment Systems" +) + +# DMEPOS payment policy has no stable annual title. It rode inside the +# ESRD rule, then a standalone HCPCS rule, then the home health rule. +# Keep a document when the title names the benefit (the acronym, or the +# three-word "durable medical equipment" + prosthetics + orthotics), and +# drop a Physician Fee Schedule title that only mentions DMEPOS in a +# conditions-of-payment clause. The prosthetics-supplier rule (CMS-6012) +# never says DMEPOS; its title is matched as its own phrase. +_DMEPOS_SUPPLIER_PHRASE = ( + "qualified suppliers of prosthetics and custom-fabricated orthotics" +) +# The 2018 rural blend (CMS-1687-IFC) says "fee schedule" and never +# "DMEPOS". Later master-list notices (CMS-6095-N and after) dropped +# the acronym; "face to face" and "face-to-face" both occur. +_DMEPOS_FEE_SCHEDULE = "durable medical equipment fee schedule" +_DMEPOS_MASTER_LIST = "master list of items potentially subject to face-to-face" +_DMEPOS_SEARCH_TERMS = ( + "dmepos", + "durable medical equipment", + "custom-fabricated orthotics", + "durable medical equipment fee schedule", + "master list of items potentially subject", +) + @dataclass class FRDocument: @@ -130,6 +168,43 @@ def split_docket_ids(raw: list[str]) -> list[str]: return out +def _titled_rules( + *, + term: str, + pattern: str, + since: str, + until: str | None, + client: httpx.Client | None, +) -> list[FRDocument]: + """CMS rules whose title contains *pattern*. + + ``long term`` and ``long-term`` compare equal so a hyphen drift in + an annual-rule title does not drop a year. + """ + needle = pattern.lower().replace("long term", "long-term") + found: list[FRDocument] = [] + seen: set[str] = set() + for doc in search( + term=term, + agencies=["centers-for-medicare-medicaid-services"], + types=["RULE", "PRORULE"], + since=since, + until=until, + client=client, + ): + # FY2026+ inserts "(IPPS)" after "Acute Care Hospitals". + title = ( + doc.title.lower().replace("long term", "long-term").replace(" (ipps)", "") + ) + if needle not in title: + continue + if doc.document_number in seen: + continue + seen.add(doc.document_number) + found.append(doc) + return found + + def pfs_rules( *, since: str = "2017-01-01", @@ -141,22 +216,121 @@ def pfs_rules( Title-filtered so we get the true annual PFS rules, not every CMS rule that mentions PFS in passing. """ - found: list[FRDocument] = [] - seen: set[str] = set() - for doc in search( + return _titled_rules( term="physician fee schedule", - agencies=["centers-for-medicare-medicaid-services"], - types=["RULE", "PRORULE"], + pattern=PFS_TITLE_PATTERN, since=since, until=until, client=client, - ): - if PFS_TITLE_PATTERN.lower() not in doc.title.lower(): - continue - if doc.document_number in seen: - continue - seen.add(doc.document_number) - found.append(doc) + ) + + +def ipps_rules( + *, + since: str = "2017-01-01", + until: str | None = None, + client: httpx.Client | None = None, +) -> list[FRDocument]: + """Every annual IPPS/LTCH proposed/final rule since *since*.""" + return _titled_rules( + term="inpatient prospective payment", + pattern=IPPS_TITLE_PATTERN, + since=since, + until=until, + client=client, + ) + + +def opps_rules( + *, + since: str = "2017-01-01", + until: str | None = None, + client: httpx.Client | None = None, +) -> list[FRDocument]: + """Every annual OPPS/ASC proposed/final rule since *since*.""" + return _titled_rules( + term="outpatient prospective payment", + pattern=OPPS_TITLE_PATTERN, + since=since, + until=until, + client=client, + ) + + +def _norm_rule_title(title: str) -> str: + t = " ".join(title.lower().split()) + return t.replace("face to face", "face-to-face") + + +def is_dmepos_title(title: str) -> bool: + """True when *title* is a DMEPOS payment rule, not a drive-by mention. + + A Physician Fee Schedule title is excluded even when it contains + "(DMEPOS)": that clause is a conditions-of-payment rider on the PFS + rule, and the PFS docket is already in the corpus. + """ + t = _norm_rule_title(title) + if "physician fee schedule" in t: + return False + if "dmepos" in t: + return True + if "durable medical equipment" in t and "prosthetics" in t and "orthotics" in t: + return True + if _DMEPOS_SUPPLIER_PHRASE in t: + return True + return _DMEPOS_FEE_SCHEDULE in t or _DMEPOS_MASTER_LIST in t + + +def dmepos_title_needle(title: str) -> str: + """Substring a reg.gov title search can require for this document. + + Empty when the title is not a DMEPOS rule. Callers pass the result + as ``title_contains`` only when it is non-empty. + """ + t = _norm_rule_title(title) + if "physician fee schedule" in t: + return "" + if "dmepos" in t: + return "dmepos" + if "durable medical equipment" in t and "prosthetics" in t and "orthotics" in t: + return "durable medical equipment" + if _DMEPOS_SUPPLIER_PHRASE in t: + return "custom-fabricated orthotics" + if _DMEPOS_FEE_SCHEDULE in t: + return _DMEPOS_FEE_SCHEDULE + if _DMEPOS_MASTER_LIST in t: + return "master list of items potentially subject" + return "" + + +def dmepos_rules( + *, + since: str = "2017-01-01", + until: str | None = None, + client: httpx.Client | None = None, +) -> list[FRDocument]: + """DMEPOS proposed and final rules, notices included, since *since*. + + Three Federal Register searches are unioned, then + :func:`is_dmepos_title` drops body-only hits (PFS, IPPS, OPPS, + hospice, home health years that do not name the benefit). + """ + found: list[FRDocument] = [] + seen: set[str] = set() + for term in _DMEPOS_SEARCH_TERMS: + for doc in search( + term=term, + agencies=["centers-for-medicare-medicaid-services"], + types=["RULE", "PRORULE"], + since=since, + until=until, + client=client, + ): + if doc.document_number in seen or not is_dmepos_title(doc.title): + continue + seen.add(doc.document_number) + found.append(doc) + found.sort(key=lambda d: (d.publication_date, d.document_number)) return found diff --git a/src/bib/regulations_gov.py b/src/bib/regulations_gov.py index d6c9347..3b2893c 100644 --- a/src/bib/regulations_gov.py +++ b/src/bib/regulations_gov.py @@ -144,7 +144,9 @@ class Client: # ── Discovery ────────────────────────────────────────────── - def resolve_docket(self, cms_rule_id: str) -> str | None: + def resolve_docket( + self, cms_rule_id: str, *, title_contains: str = "" + ) -> str | None: """Map a CMS rule ID (e.g. ``CMS-1676-P``) to its regulations.gov docket (e.g. ``CMS-2017-0092``). @@ -154,21 +156,36 @@ class Client: surfaces the matching document, whose ``attributes.docketId`` is the real docket we need for comment pulls. + The search also returns unrelated notices that merely mention + the CMS id. Pass ``title_contains`` (for example ``"inpatient + prospective payment"``) to skip those. Without it, the first + hit wins, which is what the PFS rules have always used. + Returns None when no document matches — happens for rules that were never opened for public comment (corrections, IFCs). """ - data = self._get( - "/documents", - **{ - "filter[searchTerm]": cms_rule_id, - "filter[agencyId]": "CMS", - "page[size]": 5, - }, - ) - for row in data.get("data", []): - docket = (row.get("attributes") or {}).get("docketId") - if docket: - return docket + needle = _norm_title(title_contains) + page = 1 + while page <= 4: + data = self._get( + "/documents", + **{ + "filter[searchTerm]": cms_rule_id, + "filter[agencyId]": "CMS", + "page[size]": 20 if needle else 5, + "page[number]": page, + }, + ) + for row in data.get("data", []): + attrs = row.get("attributes") or {} + docket = attrs.get("docketId") + title = _norm_title(attrs.get("title") or "") + if docket and (not needle or needle in title): + return docket + total_pages = (data.get("meta") or {}).get("totalPages", 1) or 1 + if not needle or page >= total_pages: + return None + page += 1 return None def find_documents_in_docket(self, docket_id: str) -> list[dict]: @@ -754,6 +771,11 @@ def backfill_from_mirror( if limit: todo = todo[: int(limit)] cms_id = _rule_tag_for_docket(store, docket) + if not cms_id: + # A first fill has no comment to copy the rule tag from. The + # docket row is the other place the CMS id is stored. + registered = store.docket_get(docket) + cms_id = registered.rule_cms_id if registered else "" stats = {"created": 0, "enriched": 0, "attached": 0, "errors": 0, "seen": len(todo)} @@ -895,13 +917,35 @@ def upsert_comment_status( # ── Docket walks ─────────────────────────────────────────────── -def discover_docket(client: Client, docket_id: str, *, rule_cms_id: str = "") -> Docket: +def _norm_title(title: str) -> str: + """Lowercase and collapse whitespace. + + regulations.gov wraps some annual-rule titles across lines + (``Inpatient\\nProspective``). A raw substring check then misses + the document the letters were filed on. + """ + return " ".join(title.lower().split()) + + +def discover_docket( + client: Client, + docket_id: str, + *, + rule_cms_id: str = "", + title_contains: str = "", +) -> Docket: """One ``/documents`` listing → the docket's commentable document. Picks the Proposed Rule that has an ``objectId`` and a ``commentEndDate`` — latest close date wins among proposed rules; falls back to any commentable document. Called once per docket; the result is persisted so later runs make no API call. + + ``title_contains`` restricts the choice to documents whose title + carries that phrase (the annual IPPS or OPPS rule, not a PRA notice + filed on the same docket). When several still match, the one + comments were actually filed on wins — an extension posted later + can have the later close date and almost no letters. """ qualifying: list[dict] = [] for fr_doc in client.find_documents_in_docket(docket_id): @@ -916,16 +960,34 @@ def discover_docket(client: Client, docket_id: str, *, rule_cms_id: str = "") -> if (fr_doc.get("attributes") or {}).get("documentType") == "Proposed Rule" ] candidates = proposed_rules or qualifying + needle = _norm_title(title_contains) + if needle: + titled = [ + fr_doc + for fr_doc in candidates + if needle + in _norm_title((fr_doc.get("attributes") or {}).get("title") or "") + ] + if titled: + candidates = titled best: dict | None = None - for fr_doc in candidates: - attrs = fr_doc.get("attributes") or {} - if ( - best is None - or attrs["commentEndDate"] - > (best.get("attributes") or {})["commentEndDate"] - ): - best = fr_doc + if needle and len(candidates) > 1: + best = max(candidates, key=lambda fr: _comment_total(client, fr)) + else: + for fr_doc in candidates: + attrs = fr_doc.get("attributes") or {} + end = attrs["commentEndDate"] + best_attrs = (best or {}).get("attributes") or {} + best_end = best_attrs.get("commentEndDate") or "" + if best is None or end > best_end: + best = fr_doc + elif end == best_end and _comment_total(client, fr_doc) > _comment_total( + client, best or {} + ): + # Two copies of the same NPRM often share a close date. + # The display posting is the thin one. + best = fr_doc attrs = (best or {}).get("attributes") or {} return Docket( id=docket_id, @@ -936,6 +998,61 @@ def discover_docket(client: Client, docket_id: str, *, rule_cms_id: str = "") -> ) +def _comment_total(client: Client, fr_doc: dict) -> int: + obj = (fr_doc.get("attributes") or {}).get("objectId") or "" + if not obj: + return -1 + try: + meta = ( + client._get( # noqa: SLF001 + "/comments", + **{"filter[commentOnId]": obj, "page[size]": "5"}, + ).get("meta") + or {} + ) + return int(meta.get("totalElements") or 0) + except (httpx.HTTPError, TypeError, ValueError): + return -1 + + +def pull_titled_siblings( + store: Store, + client: Client, + docket: Docket, + *, + title_contains: str, +) -> int: + """Upsert comments filed on this docket's other matching proposed rules. + + The docket row tracks one document. A display copy or a correction + posted as its own proposed rule carries a separate comment pile. + Those letters get the same ``reg-docket`` and ``rule`` tags. The + docket row is left alone, so the primary watermark stays put. + """ + needle = _norm_title(title_contains) + if not needle: + return 0 + n = 0 + for fr_doc in client.find_documents_in_docket(docket.id): + attrs = fr_doc.get("attributes") or {} + if attrs.get("documentType") != "Proposed Rule": + continue + if needle not in _norm_title(attrs.get("title") or ""): + continue + obj = attrs.get("objectId") or "" + if not obj or obj == docket.fr_object_id: + continue + for comment in client.iter_comments(obj): + upsert_comment_status( + store, + comment, + cms_id=docket.rule_cms_id, + extra_tags=[f"reg-docket:{docket.id}"], + ) + n += 1 + return n + + @dataclass class WalkResult: created: int = 0 diff --git a/src/bib/store.py b/src/bib/store.py index ee527db..4981e98 100644 --- a/src/bib/store.py +++ b/src/bib/store.py @@ -78,6 +78,9 @@ class Store: self._connection.row_factory = sqlite3.Row self._connection.execute("PRAGMA journal_mode=WAL") self._connection.execute("PRAGMA foreign_keys=ON") + # The mail poller writes this file on its own cadence. A comment + # walk waits through that burst instead of dying on SQLITE_BUSY. + self._connection.execute("PRAGMA busy_timeout=60000") return self._connection def _init_schema(self) -> None: diff --git a/src/bib/translate.py b/src/bib/translate.py index e15448c..8e38e27 100644 --- a/src/bib/translate.py +++ b/src/bib/translate.py @@ -109,12 +109,13 @@ def federal_register(url: str) -> Rule: Populated rule model with FR citation, CMS ID, dates. """ # Extract document number from URL path - doc_match = re.search(r"/documents/\d{4}/\d{2}/\d{2}/([\d-]+)/", url) + # Correction documents are numbered C1-2021-27763, not just digits. + doc_match = re.search(r"/documents/\d{4}/\d{2}/\d{2}/([^/]+)/", url) if doc_match: doc_number = doc_match.group(1) else: # Try API URL pattern - api_match = re.search(r"/documents/([\d-]+)\.json", url) + api_match = re.search(r"/documents/([^/]+)\.json", url) doc_number = api_match.group(1) if api_match else "" api_url = f"https://www.federalregister.gov/api/v1/documents/{doc_number}.json" @@ -148,7 +149,7 @@ def federal_register(url: str) -> Rule: date_published=data.get("publication_date", ""), effective_date=data.get("effective_on") or "", url=data.get("html_url", url), - abstract=data.get("abstract", ""), + abstract=data.get("abstract") or "", ) rule.add_tag(Tag.source("federal-register").label) diff --git a/src/cli/bib.py b/src/cli/bib.py index 62cb166..75e6e2b 100644 --- a/src/cli/bib.py +++ b/src/cli/bib.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Callable from pathlib import Path import typer @@ -84,27 +85,14 @@ def _http_client(): return httpx.Client(headers={"User-Agent": _UA}) -@app.command(name="discover-pfs-rules") -def discover_pfs_rules( - since: str = typer.Option("2017-01-01", "--since"), - until: str = typer.Option("", "--until"), - dry_run: bool = typer.Option(False, "--dry-run"), -) -> None: - """Walk the Federal Register API for every PFS rule since a date. - - Upserts each as a bib ``Rule`` record tagged ``module:pfs``, - ``source:federal-register``, ``rule:``. Cheap — the FR API - has no auth and no rate charge we can trigger at this scale - (~20 rules since 2017). - """ +def _discover_rules(docs, *, module: str, dry_run: bool) -> None: + """Upsert Federal Register rule documents under ``module:``.""" from bib import connect - from bib.federalregister import pfs_rules from bib.tag import Tag from bib.translate import federal_register store = connect() if not dry_run else None - - for doc in pfs_rules(since=since, until=until or None): + for doc in docs: typer.echo( f" {doc.publication_date} {doc.type:14s} " f"{doc.document_number:14s} {','.join(doc.dockets) or '(no docket)'}" @@ -120,12 +108,81 @@ def discover_pfs_rules( typer.echo(f" skipped: {e.__class__.__name__}: {e}") continue rule.add_tag(Tag.source("federal-register").label) - rule.add_tag("module:pfs") + rule.add_tag(f"module:{module}") for d in doc.dockets: rule.add_tag(f"docket:{d}") store.upsert(rule) +@app.command(name="discover-pfs-rules") +def discover_pfs_rules( + since: str = typer.Option("2017-01-01", "--since"), + until: str = typer.Option("", "--until"), + dry_run: bool = typer.Option(False, "--dry-run"), +) -> None: + """Walk the Federal Register API for every PFS rule since a date. + + Upserts each as a bib ``Rule`` record tagged ``module:pfs``, + ``source:federal-register``, ``docket:``. Cheap — the FR API + has no auth and no rate charge we can trigger at this scale + (~20 rules since 2017). + """ + from bib.federalregister import pfs_rules + + _discover_rules( + pfs_rules(since=since, until=until or None), module="pfs", dry_run=dry_run + ) + + +@app.command(name="discover-ipps-rules") +def discover_ipps_rules( + since: str = typer.Option("2017-01-01", "--since"), + until: str = typer.Option("", "--until"), + dry_run: bool = typer.Option(False, "--dry-run"), +) -> None: + """Walk the Federal Register API for every annual IPPS/LTCH rule.""" + from bib.federalregister import ipps_rules + + _discover_rules( + ipps_rules(since=since, until=until or None), module="ipps", dry_run=dry_run + ) + + +@app.command(name="discover-opps-rules") +def discover_opps_rules( + since: str = typer.Option("2017-01-01", "--since"), + until: str = typer.Option("", "--until"), + dry_run: bool = typer.Option(False, "--dry-run"), +) -> None: + """Walk the Federal Register API for every annual OPPS/ASC rule.""" + from bib.federalregister import opps_rules + + _discover_rules( + opps_rules(since=since, until=until or None), module="opps", dry_run=dry_run + ) + + +@app.command(name="discover-dmepos-rules") +def discover_dmepos_rules( + since: str = typer.Option("2017-01-01", "--since"), + until: str = typer.Option("", "--until"), + dry_run: bool = typer.Option(False, "--dry-run"), +) -> None: + """Walk the Federal Register API for every DMEPOS rule since a date. + + The benefit moved between the ESRD rule, a standalone HCPCS rule, + and the home health rule. A title filter keeps those vehicles and + drops a Physician Fee Schedule title that only mentions DMEPOS. + """ + from bib.federalregister import dmepos_rules + + _discover_rules( + dmepos_rules(since=since, until=until or None), + module="dmepos", + dry_run=dry_run, + ) + + def _walk_and_report(store, api, docket, *, attachments, limit, force, echo) -> None: from bib.dockets import quiet_days from bib.regulations_gov import walk_docket @@ -276,51 +333,33 @@ def pull_open_dockets( ) -@app.command(name="fetch-pfs-comments") -def fetch_pfs_comments( - since: str = typer.Option("2017-01-01", "--since"), - until: str = typer.Option("", "--until"), - attachments: bool = typer.Option(False, "--attachments"), - per_docket_limit: int = typer.Option( - 0, "--per-docket-limit", help="Cap comments fetched per docket. 0 = unlimited." - ), - sleep: float = typer.Option(1.3, "--sleep"), - docket: str = typer.Option( - "", - "--docket", - help=( - "Only pull this reg.gov docket (e.g. CMS-2026-2377). Filtered " - "before any API call ONLY once the docket is known (a `dockets` " - "row already exists for its CMS id) — a CMS id with no `dockets` " - "row yet still costs one Federal Register rule-metadata fetch " - "plus one resolve_docket call, because the reg.gov docket id " - "isn't known until those calls resolve it." - ), - ), - force: bool = typer.Option( - False, - "--force", - help="Walk every docket from page 1, sealed or not. Never unseals.", - ), +def _fetch_rule_comments( + rules, + *, + module: str, + label: str, + since: str, + attachments: bool, + per_docket_limit: int, + sleep: float, + docket: str, + force: bool, + title_contains: str | Callable[[object], str] = "", ) -> None: - """Discover every PFS proposed rule since *since* and pull new comments - on each docket from its stored watermark. + """Register each proposed rule's reg.gov docket and walk its comments. - Sealed dockets (comment period closed + quiet period + an empty pull) - cost nothing: no rule-metadata fetch, no resolve call, no walk. The - same is true for a docket already known via `--docket` that resolves - to a different id. A CMS id with no `dockets` row at all is not free - even under `--docket`: see that option's help. + ``title_contains`` is a substring every resolved document must + carry, or a callable from the rule document to that substring. + An empty result keeps the PFS first-hit resolve. """ from bib import connect - from bib.federalregister import pfs_rules, split_docket_ids - from bib.regulations_gov import Client, discover_docket + from bib.federalregister import split_docket_ids + from bib.regulations_gov import Client, discover_docket, pull_titled_siblings from bib.tag import Tag from bib.translate import federal_register store = connect() - rules = pfs_rules(since=since, until=until or None) - typer.echo(f"==> {len(rules)} PFS rules since {since}") + typer.echo(f"==> {len(rules)} {label} rules since {since}") proposed = [r for r in rules if r.type == "Proposed Rule"] typer.echo(f" {len(proposed)} proposed rules (the ones with comments)") @@ -354,7 +393,7 @@ def fetch_pfs_comments( typer.echo(f" skipped rule meta: {e}") continue rule.add_tag(Tag.source("federal-register").label) - rule.add_tag("module:pfs") + rule.add_tag(f"module:{module}") for cid in cms_ids: rule.add_tag(f"cms-rule:{cid}") # Upsert now, before any docket resolution: a CMS id whose @@ -364,17 +403,36 @@ def fetch_pfs_comments( # the extra call a no-op when nothing changed. store.upsert(rule) + needle = title_contains(doc) if callable(title_contains) else title_contains for cms_id in todo: d = known[cms_id] if d is None: - reg_docket = api.resolve_docket(cms_id) + # An empty needle keeps the PFS first-hit call. Passing + # title_contains="" would change that signature for every + # existing caller. + resolve_kw = {"title_contains": needle} if needle else {} + reg_docket = api.resolve_docket(cms_id, **resolve_kw) if not reg_docket: typer.echo(f" skip {cms_id}: no reg.gov docket found") continue if docket and reg_docket != docket: continue - d = discover_docket(api, reg_docket, rule_cms_id=cms_id) - store.docket_upsert(d) + # A correction (CMS-XXXX-CN) often lives on the same + # reg.gov docket as the proposed rule. Re-upserting a + # fresh row would wipe the watermark and the seal. + d = store.docket_get(reg_docket) + if d is None: + d = discover_docket( + api, reg_docket, rule_cms_id=cms_id, **resolve_kw + ) + store.docket_upsert(d) + if d.sealed and not force: + typer.echo( + f" {d.id}: sealed {d.sealed_at[:10]} ({d.seal_reason}); skipping" + ) + rule.add_tag(f"reg-docket:{d.id}") + store.upsert(rule) + continue typer.echo(f" {cms_id} → {d.id} ({doc.publication_date})") rule.add_tag(f"reg-docket:{d.id}") store.upsert(rule) @@ -390,6 +448,168 @@ def fetch_pfs_comments( force=force, echo=typer.echo, ) + if needle and not per_docket_limit: + extra = pull_titled_siblings(store, api, d, title_contains=needle) + if extra: + typer.echo(f" {d.id}: {extra} comments on sibling documents") + + +_DOCKET_HELP = ( + "Only pull this reg.gov docket (e.g. CMS-2026-2377). Filtered " + "before any API call ONLY once the docket is known (a `dockets` " + "row already exists for its CMS id) — a CMS id with no `dockets` " + "row yet still costs one Federal Register rule-metadata fetch " + "plus one resolve_docket call, because the reg.gov docket id " + "isn't known until those calls resolve it." +) + + +@app.command(name="fetch-pfs-comments") +def fetch_pfs_comments( + since: str = typer.Option("2017-01-01", "--since"), + until: str = typer.Option("", "--until"), + attachments: bool = typer.Option(False, "--attachments"), + per_docket_limit: int = typer.Option( + 0, "--per-docket-limit", help="Cap comments fetched per docket. 0 = unlimited." + ), + sleep: float = typer.Option(1.3, "--sleep"), + docket: str = typer.Option("", "--docket", help=_DOCKET_HELP), + force: bool = typer.Option( + False, + "--force", + help="Walk every docket from page 1, sealed or not. Never unseals.", + ), +) -> None: + """Discover every PFS proposed rule since *since* and pull new comments + on each docket from its stored watermark. + + Sealed dockets (comment period closed + quiet period + an empty pull) + cost nothing: no rule-metadata fetch, no resolve call, no walk. The + same is true for a docket already known via `--docket` that resolves + to a different id. A CMS id with no `dockets` row at all is not free + even under `--docket`: see that option's help. + """ + from bib.federalregister import pfs_rules + + _fetch_rule_comments( + pfs_rules(since=since, until=until or None), + module="pfs", + label="PFS", + since=since, + attachments=attachments, + per_docket_limit=per_docket_limit, + sleep=sleep, + docket=docket, + force=force, + ) + + +@app.command(name="fetch-ipps-comments") +def fetch_ipps_comments( + since: str = typer.Option("2017-01-01", "--since"), + until: str = typer.Option("", "--until"), + attachments: bool = typer.Option(False, "--attachments"), + per_docket_limit: int = typer.Option( + 0, "--per-docket-limit", help="Cap comments fetched per docket. 0 = unlimited." + ), + sleep: float = typer.Option(1.3, "--sleep"), + docket: str = typer.Option("", "--docket", help=_DOCKET_HELP), + force: bool = typer.Option( + False, + "--force", + help="Walk every docket from page 1, sealed or not. Never unseals.", + ), +) -> None: + """Discover every annual IPPS/LTCH proposed rule and pull its comments.""" + from bib.federalregister import ipps_rules + + _fetch_rule_comments( + ipps_rules(since=since, until=until or None), + module="ipps", + label="IPPS", + since=since, + attachments=attachments, + per_docket_limit=per_docket_limit, + sleep=sleep, + docket=docket, + force=force, + title_contains="inpatient prospective payment", + ) + + +@app.command(name="fetch-opps-comments") +def fetch_opps_comments( + since: str = typer.Option("2017-01-01", "--since"), + until: str = typer.Option("", "--until"), + attachments: bool = typer.Option(False, "--attachments"), + per_docket_limit: int = typer.Option( + 0, "--per-docket-limit", help="Cap comments fetched per docket. 0 = unlimited." + ), + sleep: float = typer.Option(1.3, "--sleep"), + docket: str = typer.Option("", "--docket", help=_DOCKET_HELP), + force: bool = typer.Option( + False, + "--force", + help="Walk every docket from page 1, sealed or not. Never unseals.", + ), +) -> None: + """Discover every annual OPPS/ASC proposed rule and pull its comments.""" + from bib.federalregister import opps_rules + + _fetch_rule_comments( + opps_rules(since=since, until=until or None), + module="opps", + label="OPPS", + since=since, + attachments=attachments, + per_docket_limit=per_docket_limit, + sleep=sleep, + docket=docket, + force=force, + title_contains="outpatient prospective payment", + ) + + +@app.command(name="fetch-dmepos-comments") +def fetch_dmepos_comments( + since: str = typer.Option("2017-01-01", "--since"), + until: str = typer.Option("", "--until"), + attachments: bool = typer.Option(False, "--attachments"), + per_docket_limit: int = typer.Option( + 0, "--per-docket-limit", help="Cap comments fetched per docket. 0 = unlimited." + ), + sleep: float = typer.Option(1.3, "--sleep"), + docket: str = typer.Option("", "--docket", help=_DOCKET_HELP), + force: bool = typer.Option( + False, + "--force", + help="Walk every docket from page 1, sealed or not. Never unseals.", + ), +) -> None: + """Discover every DMEPOS proposed rule since *since* and pull its comments. + + Combined ESRD and home-health vehicles come in whole: the comment + file is the docket's file, not a DMEPOS-only slice. Notices and final + rules are registered by ``discover-dmepos-rules`` and are not walked. + + No title needle is passed to regulations.gov. The agency's copy of a + long title is often truncated (``etc.``) or hyphenated differently + from the Federal Register, and the short display posting is then + the only document that still contains "DMEPOS". + """ + from bib.federalregister import dmepos_rules + + _fetch_rule_comments( + dmepos_rules(since=since, until=until or None), + module="dmepos", + label="DMEPOS", + since=since, + attachments=attachments, + per_docket_limit=per_docket_limit, + sleep=sleep, + docket=docket, + force=force, + ) @app.command(name="ingest-mail") diff --git a/tests/bib/test_federalregister.py b/tests/bib/test_federalregister.py index 20edb97..35608c4 100644 --- a/tests/bib/test_federalregister.py +++ b/tests/bib/test_federalregister.py @@ -5,9 +5,16 @@ from __future__ import annotations from unittest.mock import MagicMock, patch from bib.federalregister import ( + IPPS_TITLE_PATTERN, + OPPS_TITLE_PATTERN, PFS_TITLE_PATTERN, FRDocument, _to_doc, + dmepos_rules, + dmepos_title_needle, + ipps_rules, + is_dmepos_title, + opps_rules, pfs_rules, search, split_docket_ids, @@ -176,3 +183,176 @@ class TestPfsRules: client.get.return_value = _mock_fr_response([doc, doc]) found = pfs_rules(since="2023-01-01") assert len(found) == 1 + + +class TestIppsOppsRules: + @patch("bib.federalregister.httpx.Client") + def test_ipps_keeps_the_annual_rule_only(self, MockClient): + client = MockClient.return_value + client.close = MagicMock() + client.get.return_value = _mock_fr_response( + [ + { + "document_number": "A", + "title": f"Medicare Program; {IPPS_TITLE_PATTERN} and FY 2019 Rates", + "type": "Proposed Rule", + "publication_date": "2018-05-07", + }, + { + "document_number": "B", + "title": ( + "Hospital Inpatient Prospective Payment Systems for " + "Acute Care Hospitals; Graduate Medical Education" + ), + "type": "Rule", + "publication_date": "2021-12-27", + }, + { + "document_number": "C", + "title": IPPS_TITLE_PATTERN.replace("Long-Term", "Long Term"), + "type": "Rule", + "publication_date": "2018-08-17", + }, + { + "document_number": "D", + "title": IPPS_TITLE_PATTERN.replace( + "Acute Care Hospitals", "Acute Care Hospitals (IPPS)" + ), + "type": "Proposed Rule", + "publication_date": "2026-04-14", + }, + ] + ) + found = ipps_rules(since="2017-01-01") + assert [d.document_number for d in found] == ["A", "C", "D"] + + @patch("bib.federalregister.httpx.Client") + def test_opps_keeps_the_annual_rule_only(self, MockClient): + client = MockClient.return_value + client.close = MagicMock() + client.get.return_value = _mock_fr_response( + [ + { + "document_number": "A", + "title": f"Medicare Program: {OPPS_TITLE_PATTERN}", + "type": "Proposed Rule", + "publication_date": "2024-07-22", + }, + { + "document_number": "B", + "title": ( + "Hospital Outpatient Prospective Payment System: " + "Remedy for the 340B-Acquired Drug Payment Policy" + ), + "type": "Proposed Rule", + "publication_date": "2023-07-11", + }, + ] + ) + found = opps_rules(since="2017-01-01") + assert [d.document_number for d in found] == ["A"] + + +class TestDmeposRules: + def test_title_keeps_the_benefit_and_drops_pfs(self): + esrd = ( + "End-Stage Renal Disease Prospective Payment System, " + "Durable Medical Equipment, Prosthetics, Orthotics and " + "Supplies (DMEPOS) Competitive Bidding Program" + ) + home_health = ( + "Certain Requirements for Durable Medical Equipment " + "Prosthetics and Orthotics Supplies" + ) + suppliers = ( + "Requirements for Qualified Practitioners and Qualified " + "Suppliers of Prosthetics and Custom-Fabricated Orthotics" + ) + pfs = ( + f"{PFS_TITLE_PATTERN} and Conditions of Payment for " + "Suppliers of Durable Medicaid Equipment, Prosthetics, " + "Orthotics, and Supplies (DMEPOS)" + ) + opps_prior_auth = f"{OPPS_TITLE_PATTERN} and Prior Authorization Process" + hospice = "Hospice Wage Index and Durable Medical Equipment mention" + other_hh = "Home Health Prospective Payment System; Other Medicare Policies" + + assert is_dmepos_title(esrd) + assert is_dmepos_title(home_health) + assert is_dmepos_title(suppliers) + assert not is_dmepos_title(pfs) + assert not is_dmepos_title(opps_prior_auth) + assert not is_dmepos_title(hospice) + assert not is_dmepos_title(other_hh) + fee = ( + "Durable Medical Equipment Fee Schedule Adjustments To Resume " + "the Transitional 50/50 Blended Rates" + ) + master = ( + "Updates to the Master List of Items Potentially Subject to " + "Face to Face Encounter and Written Order Prior to Delivery" + ) + assert is_dmepos_title(fee) + assert is_dmepos_title(master) + assert dmepos_title_needle(esrd) == "dmepos" + assert dmepos_title_needle(home_health) == "durable medical equipment" + assert dmepos_title_needle(suppliers) == "custom-fabricated orthotics" + assert dmepos_title_needle(fee) == "durable medical equipment fee schedule" + assert dmepos_title_needle(master) == "master list of items potentially subject" + assert dmepos_title_needle(pfs) == "" + + @patch("bib.federalregister.httpx.Client") + def test_search_union_keeps_annual_vehicles_only(self, MockClient): + client = MockClient.return_value + client.close = MagicMock() + client.get.return_value = _mock_fr_response( + [ + { + "document_number": "2018-14986", + "title": ( + "Durable Medical Equipment, Prosthetics, Orthotics " + "and Supplies (DMEPOS) Competitive Bidding Program" + ), + "type": "Proposed Rule", + "publication_date": "2018-07-19", + }, + { + "document_number": "2022-14562", + "title": ( + f"Revisions to {PFS_TITLE_PATTERN} and Supplies (DMEPOS)" + ), + "type": "Proposed Rule", + "publication_date": "2022-07-29", + }, + { + "document_number": "2023-14044", + "title": ( + "Certain Requirements for Durable Medical Equipment " + "Prosthetics and Orthotics Supplies" + ), + "type": "Proposed Rule", + "publication_date": "2023-07-10", + }, + { + "document_number": "2017-00425", + "title": ( + "Qualified Suppliers of Prosthetics and " + "Custom-Fabricated Orthotics" + ), + "type": "Proposed Rule", + "publication_date": "2017-01-12", + }, + { + "document_number": "2024-15087", + "title": f"{OPPS_TITLE_PATTERN} and Prior Authorization", + "type": "Proposed Rule", + "publication_date": "2024-07-22", + }, + ] + ) + found = dmepos_rules(since="2017-01-01") + assert [d.document_number for d in found] == [ + "2017-00425", + "2018-14986", + "2023-14044", + ] diff --git a/tests/bib/test_regulations_gov.py b/tests/bib/test_regulations_gov.py index 1bcdd20..af468a6 100644 --- a/tests/bib/test_regulations_gov.py +++ b/tests/bib/test_regulations_gov.py @@ -221,6 +221,40 @@ class TestClient: result = c.resolve_docket("CMS-1676-P") assert result == "CMS-2017-0092" + @patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test-key"}) + def test_resolve_docket_skips_titles_that_do_not_match(self): + mock_client = MagicMock() + resp = MagicMock() + resp.status_code = 200 + resp.json.return_value = { + "meta": {"totalPages": 1}, + "data": [ + { + "attributes": { + "docketId": "CMS-2019-0132", + "title": "Agency Information Collection Activities", + } + }, + { + "attributes": { + "docketId": "CMS-2019-0073", + "title": "Hospital Inpatient Prospective Payment Systems", + } + }, + ], + } + resp.raise_for_status = MagicMock() + mock_client.get.return_value = resp + + with patch("bib.regulations_gov.time.sleep"): + c = Client(sleep=0, client=mock_client) + assert ( + c.resolve_docket( + "CMS-1716-P", title_contains="inpatient prospective payment" + ) + == "CMS-2019-0073" + ) + @patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test-key"}) def test_resolve_docket_not_found(self): mock_client = MagicMock() diff --git a/tests/bib/test_regulations_gov_mirror.py b/tests/bib/test_regulations_gov_mirror.py index 3ebd62d..e4e5bdc 100644 --- a/tests/bib/test_regulations_gov_mirror.py +++ b/tests/bib/test_regulations_gov_mirror.py @@ -6,6 +6,7 @@ import json import httpx +from bib.dockets import Docket from bib.regulations_gov import ( Comment, Mirror, @@ -166,6 +167,28 @@ class TestBackfillFromMirror: att = tmp_path / "scratch" / _DOCKET / f"{_DOCKET}-0002" / "attachment_1.pdf" assert att.is_file() and att.read_bytes().startswith(b"%PDF") + def test_rule_tag_from_docket_row_when_no_comment_has_one(self, tmp_path): + store = Store(str(tmp_path / "bib.sqlite")) + store.docket_upsert(Docket(id=_DOCKET, rule_cms_id="CMS-1694-P")) + stats = backfill_from_mirror( + store, + _mirror(), + docket=_DOCKET, + scratch_root=tmp_path / "scratch", + workers=2, + ) + assert stats["created"] == 2 + row = store._con().execute("SELECT id FROM items LIMIT 1").fetchone() + tags = { + r[0] + for r in store._con().execute( + "SELECT t.name FROM tags t JOIN item_tags it ON it.tag_id=t.id " + "WHERE it.item_id=?", + (row[0],), + ) + } + assert "rule:CMS-1694-P" in tags + def test_idempotent_rerun(self, tmp_path): store = self._store(tmp_path) m = _mirror() diff --git a/tests/bib/test_translate.py b/tests/bib/test_translate.py index 7f9dcd3..568ad12 100644 --- a/tests/bib/test_translate.py +++ b/tests/bib/test_translate.py @@ -85,6 +85,27 @@ class TestFederalRegister: assert "year:2025" in rule.tags assert "rule:cms-1832-f" in rule.tags + def test_correction_document_number(self) -> None: + url = ( + "https://www.federalregister.gov/documents/2022/01/04/" + "C1-2021-27763/medicare-program" + ) + data = self._mock_fr_data() + with patch("bib.translate._fetch_json", return_value=data) as fetch: + rule = federal_register(url) + assert fetch.call_args.args[0].endswith("/C1-2021-27763.json") + assert rule.document_number == "C1-2021-27763" + + def test_missing_abstract_is_empty(self) -> None: + url = ( + "https://www.federalregister.gov/documents/2022/01/04/" + "C1-2021-27763/medicare-program" + ) + data = self._mock_fr_data(abstract=None) + with patch("bib.translate._fetch_json", return_value=data): + rule = federal_register(url) + assert rule.abstract == "" + def test_from_api_url(self) -> None: url = "https://www.federalregister.gov/api/v1/documents/2025-19787.json" data = self._mock_fr_data() diff --git a/tests/bib/test_walk_docket.py b/tests/bib/test_walk_docket.py index d8a389a..61388ad 100644 --- a/tests/bib/test_walk_docket.py +++ b/tests/bib/test_walk_docket.py @@ -104,6 +104,159 @@ def test_discover_docket_prefers_proposed_rule(): ) +def test_discover_title_picks_the_document_comments_were_filed_on(): + api = MagicMock() + api.find_documents_in_docket.return_value = [ + { + "id": "late", + "attributes": { + "objectId": "small", + "commentEndDate": "2024-11-28T04:59:59Z", + "documentType": "Proposed Rule", + "title": "Hospital Outpatient Prospective Payment extension", + }, + }, + { + "id": "main", + "attributes": { + "objectId": "big", + "commentEndDate": "2024-09-10T03:59:59Z", + "documentType": "Proposed Rule", + "title": "Hospital Outpatient Prospective Payment and ASC", + }, + }, + ] + + def _get(_path, **params): + obj = params["filter[commentOnId]"] + return {"meta": {"totalElements": 7 if obj == "small" else 2483}} + + api._get.side_effect = _get + d = discover_docket( + api, + "CMS-2024-0199", + rule_cms_id="CMS-1809-P", + title_contains="outpatient prospective payment", + ) + assert d.fr_object_id == "big" + assert d.comment_end_date == "2024-09-10" + + +def test_discover_title_joins_a_line_wrapped_nprm(): + api = MagicMock() + api.find_documents_in_docket.return_value = [ + { + "id": "display", + "attributes": { + "objectId": "small", + "commentEndDate": "2020-05-30T03:59:59Z", + "documentType": "Proposed Rule", + "title": "Hospital Inpatient Prospective Payment Systems display", + }, + }, + { + "id": "wrapped", + "attributes": { + "objectId": "big", + "commentEndDate": "2020-07-11T03:59:59Z", + "documentType": "Proposed Rule", + "title": "Hospital Inpatient\nProspective Payment Systems for Acute Care", + }, + }, + ] + + def _get(_path, **params): + obj = params["filter[commentOnId]"] + return {"meta": {"totalElements": 25 if obj == "small" else 832}} + + api._get.side_effect = _get + d = discover_docket( + api, + "CMS-2020-0052", + rule_cms_id="CMS-1735-P", + title_contains="inpatient prospective payment", + ) + assert d.fr_object_id == "big" + assert d.comment_end_date == "2020-07-11" + + +def test_discover_same_close_date_picks_the_comment_pile(): + api = MagicMock() + api.find_documents_in_docket.return_value = [ + { + "id": "display", + "attributes": { + "objectId": "small", + "commentEndDate": "2023-08-30T03:59:59Z", + "documentType": "Proposed Rule", + "title": "Home Health display", + }, + }, + { + "id": "main", + "attributes": { + "objectId": "big", + "commentEndDate": "2023-08-30T03:59:59Z", + "documentType": "Proposed Rule", + "title": "Home Health Prospective Payment System", + }, + }, + ] + + def _get(_path, **params): + obj = params["filter[commentOnId]"] + return {"meta": {"totalElements": 150 if obj == "small" else 750}} + + api._get.side_effect = _get + d = discover_docket(api, "CMS-2023-0113", rule_cms_id="CMS-1780-P") + assert d.fr_object_id == "big" + assert d.fr_document_id == "main" + + +def test_pull_titled_siblings_leaves_the_docket_row(): + from bib.regulations_gov import Comment, pull_titled_siblings + + s = _store() + s.docket_upsert(_docket(fr_object_id="primary")) + api = MagicMock() + api.find_documents_in_docket.return_value = [ + { + "id": "main", + "attributes": { + "objectId": "primary", + "documentType": "Proposed Rule", + "title": "Inpatient Prospective Payment Systems", + }, + }, + { + "id": "display", + "attributes": { + "objectId": "other", + "documentType": "Proposed Rule", + "title": "Inpatient Prospective Payment display", + }, + }, + ] + api.iter_comments.return_value = [ + Comment( + id=f"{D}-9", + title="", + posted_date="2018-06-01", + received_date="2018-06-01", + docket_id=D, + comment_on_id="other", + comment_text="display copy", + ) + ] + n = pull_titled_siblings( + s, api, s.docket_get(D), title_contains="inpatient prospective payment" + ) + assert n == 1 + saved = s.docket_get(D) + assert saved is not None and saved.fr_object_id == "primary" + assert s._con().execute("SELECT count(*) FROM items").fetchone()[0] == 1 + + def test_walk_creates_counts_and_advances_watermark(): s = _store() s.docket_upsert(_docket()) diff --git a/tests/cli/test_bib_payment_comments.py b/tests/cli/test_bib_payment_comments.py new file mode 100644 index 0000000..0330af2 --- /dev/null +++ b/tests/cli/test_bib_payment_comments.py @@ -0,0 +1,209 @@ +"""IPPS and OPPS comment discovery uses the PFS register-and-walk path.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +from typer.testing import CliRunner + +from bib.dockets import Docket +from bib.item import Rule +from bib.regulations_gov import WalkResult +from bib.store import Store +from cli.bib import app + +runner = CliRunner() + + +def _api(): + api = MagicMock() + api.__enter__ = MagicMock(return_value=api) + api.__exit__ = MagicMock(return_value=False) + api.find_documents_in_docket.return_value = [] + api.iter_comments.return_value = [] + return api + + +def _doc(number: str, cms_id: str): + doc = MagicMock() + doc.type = "Proposed Rule" + doc.publication_date = "2018-05-07" + doc.dockets = [cms_id] + doc.html_url = f"https://example.com/{number}" + doc.document_number = number + return doc + + +@patch("bib.federalregister.ipps_rules", return_value=[]) +@patch("bib.connect") +def test_fetch_ipps_comments_empty(mc_connect, _rules): + mc_connect.return_value = MagicMock() + api = _api() + with patch("bib.regulations_gov.Client", return_value=api): + result = runner.invoke(app, ["fetch-ipps-comments"]) + assert result.exit_code == 0, result.output + assert "IPPS rules" in result.output + + +@patch("bib.federalregister.opps_rules", return_value=[]) +@patch("bib.connect") +def test_fetch_opps_comments_empty(mc_connect, _rules): + mc_connect.return_value = MagicMock() + api = _api() + with patch("bib.regulations_gov.Client", return_value=api): + result = runner.invoke(app, ["fetch-opps-comments"]) + assert result.exit_code == 0, result.output + assert "OPPS rules" in result.output + + +@patch("bib.federalregister.ipps_rules", return_value=[]) +def test_discover_ipps_dry_run(_rules): + result = runner.invoke(app, ["discover-ipps-rules", "--dry-run"]) + assert result.exit_code == 0 + + +@patch("bib.federalregister.opps_rules", return_value=[]) +def test_discover_opps_dry_run(_rules): + result = runner.invoke(app, ["discover-opps-rules", "--dry-run"]) + assert result.exit_code == 0 + + +@patch("bib.regulations_gov.walk_docket", return_value=WalkResult()) +@patch("bib.regulations_gov.discover_docket") +@patch("bib.regulations_gov.Client") +@patch("bib.translate.federal_register") +@patch("bib.federalregister.ipps_rules") +@patch("bib.connect") +def test_correction_reuses_the_proposed_rule_docket( + mc_connect, mc_rules, mc_fr, mc_client, mc_disc, mc_walk +): + store = Store(":memory:", storage_dir="/tmp/nope") + mc_connect.return_value = store + mc_rules.return_value = [ + _doc("2018-1", "CMS-1694-P"), + _doc("2018-2", "CMS-1694-CN"), + ] + mc_fr.return_value = Rule(title="FY2019 IPPS", url="https://example.com/2018-1") + api = _api() + api.resolve_docket.return_value = "CMS-2018-0040" + mc_client.return_value = api + + def _discover(_api, docket_id, rule_cms_id="", title_contains=""): + assert title_contains == "inpatient prospective payment" + return Docket( + id=docket_id, + rule_cms_id=rule_cms_id, + fr_object_id="obj", + comment_end_date="2018-06-25", + ) + + mc_disc.side_effect = _discover + + result = runner.invoke(app, ["fetch-ipps-comments"]) + assert result.exit_code == 0, result.output + mc_disc.assert_called_once() + saved = store.docket_get("CMS-2018-0040") + assert saved is not None + assert saved.rule_cms_id == "CMS-1694-P" + assert mc_walk.call_count == 2 + + +@patch("bib.regulations_gov.walk_docket", return_value=WalkResult()) +@patch("bib.regulations_gov.discover_docket") +@patch("bib.regulations_gov.Client") +@patch("bib.translate.federal_register") +@patch("bib.federalregister.opps_rules") +@patch("bib.connect") +def test_reused_sealed_docket_is_not_walked( + mc_connect, mc_rules, mc_fr, mc_client, mc_disc, mc_walk +): + store = Store(":memory:", storage_dir="/tmp/nope") + store.docket_upsert( + Docket( + id="CMS-2018-0042", + rule_cms_id="CMS-1695-P", + fr_object_id="obj", + sealed_at="2020-01-01T00:00:00Z", + seal_reason="auto", + ) + ) + mc_connect.return_value = store + mc_rules.return_value = [_doc("2018-3", "CMS-1695-CN")] + mc_fr.return_value = Rule(title="CY2019 OPPS", url="https://example.com/2018-3") + api = _api() + api.resolve_docket.return_value = "CMS-2018-0042" + mc_client.return_value = api + + result = runner.invoke(app, ["fetch-opps-comments"]) + assert result.exit_code == 0, result.output + assert "sealed" in result.output + mc_disc.assert_not_called() + mc_walk.assert_not_called() + + +@patch("bib.federalregister.dmepos_rules", return_value=[]) +@patch("bib.connect") +def test_fetch_dmepos_comments_empty(mc_connect, _rules): + mc_connect.return_value = MagicMock() + api = _api() + with patch("bib.regulations_gov.Client", return_value=api): + result = runner.invoke(app, ["fetch-dmepos-comments"]) + assert result.exit_code == 0, result.output + assert "DMEPOS rules" in result.output + + +@patch("bib.federalregister.dmepos_rules", return_value=[]) +def test_discover_dmepos_dry_run(_rules): + result = runner.invoke(app, ["discover-dmepos-rules", "--dry-run"]) + assert result.exit_code == 0 + + +@patch("bib.regulations_gov.walk_docket", return_value=WalkResult()) +@patch("bib.regulations_gov.pull_titled_siblings", return_value=0) +@patch("bib.regulations_gov.discover_docket") +@patch("bib.regulations_gov.Client") +@patch("bib.translate.federal_register") +@patch("bib.federalregister.dmepos_rules") +@patch("bib.connect") +def test_dmepos_resolve_omits_the_title_needle( + mc_connect, mc_rules, mc_fr, mc_client, mc_disc, _siblings, _walk +): + store = Store(":memory:", storage_dir="/tmp/nope") + mc_connect.return_value = store + acronym = _doc("2018-14986", "CMS-1691-P") + acronym.title = ( + "Durable Medical Equipment, Prosthetics, Orthotics and " + "Supplies (DMEPOS) Competitive Bidding Program" + ) + spelled = _doc("2023-14044", "CMS-1780-P") + spelled.title = ( + "Certain Requirements for Durable Medical Equipment " + "Prosthetics and Orthotics Supplies" + ) + mc_rules.return_value = [acronym, spelled] + mc_fr.return_value = Rule(title="DMEPOS", url="https://example.com/dmepos") + api = _api() + needles: list[tuple[str, str]] = [] + + def _resolve(cms_id, title_contains=""): + needles.append((cms_id, title_contains)) + return "CMS-2018-0099" if cms_id == "CMS-1691-P" else "CMS-2023-0100" + + api.resolve_docket.side_effect = _resolve + mc_client.return_value = api + + def _discover(_api, docket_id, rule_cms_id="", title_contains=""): + return Docket( + id=docket_id, + rule_cms_id=rule_cms_id, + fr_object_id="obj", + comment_end_date="2018-09-10", + ) + + mc_disc.side_effect = _discover + + result = runner.invoke(app, ["fetch-dmepos-comments"]) + assert result.exit_code == 0, result.output + assert needles == [("CMS-1691-P", ""), ("CMS-1780-P", "")] + for call in api.resolve_docket.call_args_list: + assert "title_contains" not in call.kwargs