feat(cli): fetch commands walk from watermark, skip sealed dockets before any API call (refs #615)
This commit is contained in:
216
src/cli/bib.py
216
src/cli/bib.py
@@ -126,9 +126,29 @@ def discover_pfs_rules(
|
||||
store.upsert(rule)
|
||||
|
||||
|
||||
def _walk_and_report(store, api, docket, *, attachments, limit, force, echo) -> None:
|
||||
from bib.dockets import quiet_days
|
||||
from bib.regulations_gov import walk_docket
|
||||
|
||||
r = walk_docket(
|
||||
store,
|
||||
api,
|
||||
docket,
|
||||
attachments=attachments,
|
||||
limit=limit,
|
||||
force=force,
|
||||
quiet_days=quiet_days(),
|
||||
echo=echo,
|
||||
)
|
||||
echo(
|
||||
f" {docket.id}: created={r.created} updated={r.updated} "
|
||||
f"unchanged={r.unchanged} clean={r.clean}" + (" — sealed" if r.sealed else "")
|
||||
)
|
||||
|
||||
|
||||
@app.command(name="fetch-docket-comments")
|
||||
def fetch_docket_comments(
|
||||
docket: str = typer.Argument(..., help="e.g. CMS-1676-P"),
|
||||
docket: str = typer.Argument(..., help="reg.gov docket id, e.g. CMS-2026-2377"),
|
||||
cms_id: str = typer.Option(
|
||||
"", "--cms-id", help="Associate comments with a specific CMS rule tag."
|
||||
),
|
||||
@@ -138,52 +158,54 @@ def fetch_docket_comments(
|
||||
attachments: bool = typer.Option(
|
||||
False,
|
||||
"--attachments",
|
||||
help="Also download each comment's PDF/DOCX attachments "
|
||||
"(expensive — skim first without).",
|
||||
help="Also download each comment's PDF/DOCX attachments.",
|
||||
),
|
||||
sleep: float = typer.Option(
|
||||
1.3, "--sleep", help="Seconds between API calls (rate budget)."
|
||||
),
|
||||
force: bool = typer.Option(
|
||||
False,
|
||||
"--force",
|
||||
help="Walk from page 1 even if sealed / watermarked. Never unseals.",
|
||||
),
|
||||
) -> None:
|
||||
"""Walk every comment on one docket; upsert as Source items.
|
||||
"""Walk one docket's comments from its stored watermark; upsert as Source items.
|
||||
|
||||
Policy: we iterate every FR document indexed under the docket and
|
||||
pull their comments. For one PFS rule that usually means one parent
|
||||
document with a few thousand to tens of thousands of comments.
|
||||
The first run discovers the docket's commentable FR document (one
|
||||
listing call) and stores it; later runs make no discovery calls.
|
||||
Sealed dockets are skipped unless --force.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
from bib import connect
|
||||
from bib.regulations_gov import Client, upsert_comment
|
||||
from bib.regulations_gov import Client, discover_docket
|
||||
|
||||
store = connect()
|
||||
scratch = Path(f".state/comments/{docket}")
|
||||
|
||||
with Client(sleep=sleep) as api:
|
||||
docs = api.find_documents_in_docket(docket)
|
||||
typer.echo(f" docket {docket}: {len(docs)} FR documents indexed")
|
||||
total = 0
|
||||
for fr_doc in docs:
|
||||
attrs = fr_doc.get("attributes") or {}
|
||||
fr_id = fr_doc["id"]
|
||||
object_id = attrs.get("objectId")
|
||||
if not object_id or not attrs.get("commentEndDate"):
|
||||
continue
|
||||
typer.echo(f" ↓ comments on {fr_id} (objectId={object_id})")
|
||||
for c in api.iter_comments(object_id):
|
||||
if limit and total >= limit:
|
||||
return
|
||||
key = upsert_comment(store, c, cms_id=cms_id)
|
||||
if attachments and c.attachment_count:
|
||||
for att in api.attachments_for(c.id):
|
||||
path = api.download_attachment(att.url, scratch / c.id)
|
||||
if path:
|
||||
store.attach_file(key, path, title=att.filename)
|
||||
total += 1
|
||||
if total % 50 == 0:
|
||||
typer.echo(f" processed {total} comments")
|
||||
store._con().commit() # noqa: SLF001
|
||||
typer.echo(f" total: {total} comments upserted")
|
||||
d = store.docket_get(docket)
|
||||
if d is None:
|
||||
d = discover_docket(api, docket, rule_cms_id=cms_id)
|
||||
store.docket_upsert(d)
|
||||
elif cms_id and not d.rule_cms_id:
|
||||
from dataclasses import replace
|
||||
|
||||
d = replace(d, rule_cms_id=cms_id)
|
||||
store.docket_upsert(d)
|
||||
if d.sealed and not force:
|
||||
typer.echo(
|
||||
f" {docket}: sealed {d.sealed_at[:10]} ({d.seal_reason}); skipping — use --force to re-walk"
|
||||
)
|
||||
return
|
||||
if not d.fr_object_id:
|
||||
typer.echo(f" {docket}: no commentable FR document found")
|
||||
return
|
||||
_walk_and_report(
|
||||
store,
|
||||
api,
|
||||
d,
|
||||
attachments=attachments,
|
||||
limit=limit,
|
||||
force=force,
|
||||
echo=typer.echo,
|
||||
)
|
||||
|
||||
|
||||
@app.command(name="fetch-pfs-comments")
|
||||
@@ -192,46 +214,59 @@ def fetch_pfs_comments(
|
||||
until: str = typer.Option("", "--until"),
|
||||
attachments: bool = typer.Option(False, "--attachments"),
|
||||
per_docket_limit: int = typer.Option(
|
||||
0,
|
||||
"--per-docket-limit",
|
||||
help="Cap comments fetched per docket. 0 = unlimited.",
|
||||
0, "--per-docket-limit", help="Cap comments fetched per docket. 0 = unlimited."
|
||||
),
|
||||
sleep: float = typer.Option(1.3, "--sleep"),
|
||||
docket: str = typer.Option(
|
||||
"",
|
||||
"--docket",
|
||||
help="Only pull this reg.gov docket (e.g. CMS-2026-2377); "
|
||||
"the other PFS dockets are skipped without any API calls.",
|
||||
help="Only pull this reg.gov docket (e.g. CMS-2026-2377); other dockets are skipped before any API call once known.",
|
||||
),
|
||||
force: bool = typer.Option(
|
||||
False,
|
||||
"--force",
|
||||
help="Walk every docket from page 1, sealed or not. Never unseals.",
|
||||
),
|
||||
) -> None:
|
||||
"""One-shot: discover every PFS rule since *since* and pull every
|
||||
comment on each of their dockets.
|
||||
"""Discover every PFS proposed rule since *since* and pull new comments
|
||||
on each docket from its stored watermark.
|
||||
|
||||
Heavy run. A single PFS proposed rule can hold 5K–25K comments;
|
||||
times ~20 rules and with attachments this runs for many hours
|
||||
under the reg.gov rate budget. Use ``--per-docket-limit`` to
|
||||
smoke-test first.
|
||||
Sealed dockets (comment period closed + quiet period + an empty pull)
|
||||
cost nothing: no rule-metadata fetch, no resolve call, no walk.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
from bib import connect
|
||||
from bib.federalregister import pfs_rules, split_docket_ids
|
||||
from bib.regulations_gov import Client, upsert_comment
|
||||
from bib.regulations_gov import Client, discover_docket
|
||||
from bib.tag import Tag
|
||||
from bib.translate import federal_register
|
||||
|
||||
store = connect()
|
||||
rules = pfs_rules(since=since, until=until or None)
|
||||
typer.echo(f"==> {len(rules)} PFS rules since {since}")
|
||||
|
||||
# Only proposed rules have public comment periods — skip finals +
|
||||
# corrections to keep the work focused.
|
||||
proposed = [r for r in rules if r.type == "Proposed Rule"]
|
||||
typer.echo(f" {len(proposed)} proposed rules (the ones with comments)")
|
||||
|
||||
with Client(sleep=sleep) as api:
|
||||
for doc in proposed:
|
||||
cms_ids = split_docket_ids(doc.dockets)
|
||||
known = {cid: store.docket_for_rule(cid) for cid in cms_ids}
|
||||
|
||||
# Decide what this rule needs BEFORE touching the network.
|
||||
def _wanted(cid: str) -> bool:
|
||||
d = known[cid]
|
||||
if docket and d is not None and d.id != docket:
|
||||
return False
|
||||
if d is not None and d.sealed and not force:
|
||||
typer.echo(
|
||||
f" {d.id}: sealed {d.sealed_at[:10]} ({d.seal_reason}); skipping"
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
todo = [cid for cid in cms_ids if _wanted(cid)]
|
||||
if not todo:
|
||||
continue
|
||||
|
||||
try:
|
||||
rule = federal_register(
|
||||
doc.html_url
|
||||
@@ -244,58 +279,33 @@ def fetch_pfs_comments(
|
||||
rule.add_tag("module:pfs")
|
||||
for cid in cms_ids:
|
||||
rule.add_tag(f"cms-rule:{cid}")
|
||||
store.upsert(rule)
|
||||
|
||||
# Resolve each CMS-XXXX-P to its reg.gov docket id.
|
||||
for cms_id in cms_ids:
|
||||
reg_docket = api.resolve_docket(cms_id)
|
||||
if not reg_docket:
|
||||
typer.echo(f" skip {cms_id}: no reg.gov docket found")
|
||||
continue
|
||||
if docket and reg_docket != docket:
|
||||
continue
|
||||
typer.echo(f" {cms_id} → {reg_docket} ({doc.publication_date})")
|
||||
rule.add_tag(f"reg-docket:{reg_docket}")
|
||||
for cms_id in todo:
|
||||
d = known[cms_id]
|
||||
if d is None:
|
||||
reg_docket = api.resolve_docket(cms_id)
|
||||
if not reg_docket:
|
||||
typer.echo(f" skip {cms_id}: no reg.gov docket found")
|
||||
continue
|
||||
if docket and reg_docket != docket:
|
||||
continue
|
||||
d = discover_docket(api, reg_docket, rule_cms_id=cms_id)
|
||||
store.docket_upsert(d)
|
||||
typer.echo(f" {cms_id} → {d.id} ({doc.publication_date})")
|
||||
rule.add_tag(f"reg-docket:{d.id}")
|
||||
store.upsert(rule)
|
||||
|
||||
fr_docs = api.find_documents_in_docket(reg_docket)
|
||||
per_docket_count = 0
|
||||
scratch = Path(f".state/comments/{reg_docket}")
|
||||
for fr_doc in fr_docs:
|
||||
attrs = fr_doc.get("attributes") or {}
|
||||
object_id = attrs.get("objectId")
|
||||
# Skip docs with no objectId or no real comment
|
||||
# window — final rules and internal display versions
|
||||
# don't carry meaningful comment traffic.
|
||||
if not object_id:
|
||||
continue
|
||||
if not attrs.get("commentEndDate"):
|
||||
continue
|
||||
for c in api.iter_comments(object_id):
|
||||
if per_docket_limit and per_docket_count >= per_docket_limit:
|
||||
break
|
||||
key = upsert_comment(
|
||||
store,
|
||||
c,
|
||||
cms_id=cms_id,
|
||||
extra_tags=[f"reg-docket:{reg_docket}"],
|
||||
)
|
||||
if attachments and c.attachment_count:
|
||||
for att in api.attachments_for(c.id):
|
||||
path = api.download_attachment(
|
||||
att.url,
|
||||
scratch / c.id,
|
||||
)
|
||||
if path:
|
||||
store.attach_file(key, path, title=att.filename)
|
||||
per_docket_count += 1
|
||||
if per_docket_count % 50 == 0:
|
||||
typer.echo(f" {per_docket_count} comments")
|
||||
store._con().commit() # noqa: SLF001
|
||||
if per_docket_limit and per_docket_count >= per_docket_limit:
|
||||
break
|
||||
store._con().commit() # noqa: SLF001
|
||||
typer.echo(f" docket total: {per_docket_count}")
|
||||
if not d.fr_object_id:
|
||||
typer.echo(f" {d.id}: no commentable FR document; skipping")
|
||||
continue
|
||||
_walk_and_report(
|
||||
store,
|
||||
api,
|
||||
d,
|
||||
attachments=attachments,
|
||||
limit=per_docket_limit,
|
||||
force=force,
|
||||
echo=typer.echo,
|
||||
)
|
||||
|
||||
|
||||
@app.command(name="ingest-mail")
|
||||
|
||||
@@ -106,10 +106,14 @@ class TestDiscoverPfsRulesExercise:
|
||||
class TestFetchDocketComments:
|
||||
@patch("bib.connect")
|
||||
@patch("bib.regulations_gov.Client")
|
||||
@patch("bib.regulations_gov.upsert_comment", return_value="KEY1")
|
||||
def test_basic(self, mc_upsert, mc_client_cls, mc_connect):
|
||||
store = MagicMock()
|
||||
store._con.return_value = MagicMock()
|
||||
@patch("bib.regulations_gov.discover_docket")
|
||||
@patch("bib.regulations_gov.walk_docket")
|
||||
def test_basic(self, mc_walk, mc_disc, mc_client_cls, mc_connect):
|
||||
from bib.dockets import Docket
|
||||
from bib.regulations_gov import WalkResult
|
||||
from bib.store import Store
|
||||
|
||||
store = Store(":memory:", storage_dir="/tmp/nope")
|
||||
mc_connect.return_value = store
|
||||
|
||||
api = MagicMock()
|
||||
@@ -117,26 +121,26 @@ class TestFetchDocketComments:
|
||||
api.__exit__ = MagicMock(return_value=False)
|
||||
mc_client_cls.return_value = api
|
||||
|
||||
fr_doc = {
|
||||
"id": "DOC1",
|
||||
"attributes": {"objectId": "09000001", "commentEndDate": "2023-12-31"},
|
||||
}
|
||||
api.find_documents_in_docket.return_value = [fr_doc]
|
||||
|
||||
comment = MagicMock()
|
||||
comment.id = "C1"
|
||||
comment.attachment_count = 0
|
||||
api.iter_comments.return_value = [comment]
|
||||
mc_disc.return_value = Docket(
|
||||
id="CMS-1676-P", fr_object_id="09000001", comment_end_date="2023-12-31"
|
||||
)
|
||||
mc_walk.return_value = WalkResult(created=1)
|
||||
|
||||
result = runner.invoke(app, ["fetch-docket-comments", "CMS-1676-P", "-n", "5"])
|
||||
assert result.exit_code == 0
|
||||
mc_disc.assert_called_once()
|
||||
mc_walk.assert_called_once()
|
||||
|
||||
@patch("bib.connect")
|
||||
@patch("bib.regulations_gov.Client")
|
||||
@patch("bib.regulations_gov.upsert_comment", return_value="KEY1")
|
||||
def test_with_attachments(self, mc_upsert, mc_client_cls, mc_connect):
|
||||
store = MagicMock()
|
||||
store._con.return_value = MagicMock()
|
||||
@patch("bib.regulations_gov.discover_docket")
|
||||
@patch("bib.regulations_gov.walk_docket")
|
||||
def test_with_attachments(self, mc_walk, mc_disc, mc_client_cls, mc_connect):
|
||||
from bib.dockets import Docket
|
||||
from bib.regulations_gov import WalkResult
|
||||
from bib.store import Store
|
||||
|
||||
store = Store(":memory:", storage_dir="/tmp/nope")
|
||||
mc_connect.return_value = store
|
||||
|
||||
api = MagicMock()
|
||||
@@ -144,33 +148,27 @@ class TestFetchDocketComments:
|
||||
api.__exit__ = MagicMock(return_value=False)
|
||||
mc_client_cls.return_value = api
|
||||
|
||||
fr_doc = {
|
||||
"id": "DOC1",
|
||||
"attributes": {"objectId": "09000001", "commentEndDate": "2023-12-31"},
|
||||
}
|
||||
api.find_documents_in_docket.return_value = [fr_doc]
|
||||
|
||||
comment = MagicMock()
|
||||
comment.id = "C1"
|
||||
comment.attachment_count = 1
|
||||
api.iter_comments.return_value = [comment]
|
||||
|
||||
att = MagicMock()
|
||||
att.url = "https://example.com/att.pdf"
|
||||
att.filename = "att.pdf"
|
||||
api.attachments_for.return_value = [att]
|
||||
api.download_attachment.return_value = Path("/tmp/att.pdf")
|
||||
mc_disc.return_value = Docket(
|
||||
id="CMS-1676-P", fr_object_id="09000001", comment_end_date="2023-12-31"
|
||||
)
|
||||
mc_walk.return_value = WalkResult(created=1)
|
||||
|
||||
result = runner.invoke(
|
||||
app,
|
||||
["fetch-docket-comments", "CMS-1676-P", "--attachments", "-n", "5"],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert mc_walk.call_args.kwargs["attachments"] is True
|
||||
|
||||
@patch("bib.connect")
|
||||
@patch("bib.regulations_gov.Client")
|
||||
def test_skip_no_object_id(self, mc_client_cls, mc_connect):
|
||||
store = MagicMock()
|
||||
@patch("bib.regulations_gov.discover_docket")
|
||||
@patch("bib.regulations_gov.walk_docket")
|
||||
def test_skip_no_object_id(self, mc_walk, mc_disc, mc_client_cls, mc_connect):
|
||||
from bib.dockets import Docket
|
||||
from bib.store import Store
|
||||
|
||||
store = Store(":memory:", storage_dir="/tmp/nope")
|
||||
mc_connect.return_value = store
|
||||
|
||||
api = MagicMock()
|
||||
@@ -178,11 +176,12 @@ class TestFetchDocketComments:
|
||||
api.__exit__ = MagicMock(return_value=False)
|
||||
mc_client_cls.return_value = api
|
||||
|
||||
fr_doc = {"id": "DOC1", "attributes": {"objectId": None}}
|
||||
api.find_documents_in_docket.return_value = [fr_doc]
|
||||
mc_disc.return_value = Docket(id="CMS-1676-P")
|
||||
|
||||
result = runner.invoke(app, ["fetch-docket-comments", "CMS-1676-P"])
|
||||
assert result.exit_code == 0
|
||||
assert "no commentable FR document" in result.output
|
||||
mc_walk.assert_not_called()
|
||||
|
||||
|
||||
class TestFetchPfsComments:
|
||||
@@ -191,12 +190,24 @@ class TestFetchPfsComments:
|
||||
@patch("bib.federalregister.split_docket_ids", return_value=["CMS-1676-P"])
|
||||
@patch("bib.translate.federal_register")
|
||||
@patch("bib.regulations_gov.Client")
|
||||
@patch("bib.regulations_gov.upsert_comment", return_value="KEY1")
|
||||
@patch("bib.regulations_gov.discover_docket")
|
||||
@patch("bib.regulations_gov.walk_docket")
|
||||
def test_full_loop(
|
||||
self, mc_upsert, mc_client_cls, mc_translate, mc_split, mc_pfs, mc_connect
|
||||
self,
|
||||
mc_walk,
|
||||
mc_disc,
|
||||
mc_client_cls,
|
||||
mc_translate,
|
||||
mc_split,
|
||||
mc_pfs,
|
||||
mc_connect,
|
||||
):
|
||||
store = MagicMock()
|
||||
store._con.return_value = MagicMock()
|
||||
from bib.dockets import Docket
|
||||
from bib.item import Rule
|
||||
from bib.regulations_gov import WalkResult
|
||||
from bib.store import Store
|
||||
|
||||
store = Store(":memory:", storage_dir="/tmp/nope")
|
||||
mc_connect.return_value = store
|
||||
|
||||
doc = MagicMock()
|
||||
@@ -207,7 +218,7 @@ class TestFetchPfsComments:
|
||||
doc.document_number = "2023-12345"
|
||||
mc_pfs.return_value = [doc]
|
||||
|
||||
rule = MagicMock()
|
||||
rule = Rule(title="CY2024 PFS NPRM", url="https://example.com")
|
||||
mc_translate.return_value = rule
|
||||
|
||||
api = MagicMock()
|
||||
@@ -216,22 +227,18 @@ class TestFetchPfsComments:
|
||||
mc_client_cls.return_value = api
|
||||
api.resolve_docket.return_value = "CMS-2023-0001"
|
||||
|
||||
fr_doc = {
|
||||
"id": "DOC1",
|
||||
"attributes": {
|
||||
"objectId": "09000001",
|
||||
"commentEndDate": "2023-12-31",
|
||||
},
|
||||
}
|
||||
api.find_documents_in_docket.return_value = [fr_doc]
|
||||
|
||||
comment = MagicMock()
|
||||
comment.id = "C1"
|
||||
comment.attachment_count = 0
|
||||
api.iter_comments.return_value = [comment]
|
||||
mc_disc.return_value = Docket(
|
||||
id="CMS-2023-0001",
|
||||
rule_cms_id="CMS-1676-P",
|
||||
fr_object_id="09000001",
|
||||
comment_end_date="2023-12-31",
|
||||
)
|
||||
mc_walk.return_value = WalkResult(created=1)
|
||||
|
||||
result = runner.invoke(app, ["fetch-pfs-comments", "--per-docket-limit", "1"])
|
||||
assert result.exit_code == 0
|
||||
mc_disc.assert_called_once()
|
||||
mc_walk.assert_called_once()
|
||||
|
||||
@patch("bib.connect")
|
||||
@patch("bib.federalregister.pfs_rules", return_value=[])
|
||||
@@ -276,7 +283,9 @@ class TestFetchPfsComments:
|
||||
@patch("bib.translate.federal_register")
|
||||
@patch("bib.regulations_gov.Client")
|
||||
def test_no_docket(self, mc_client_cls, mc_translate, mc_split, mc_pfs, mc_connect):
|
||||
store = MagicMock()
|
||||
from bib.store import Store
|
||||
|
||||
store = Store(":memory:", storage_dir="/tmp/nope")
|
||||
mc_connect.return_value = store
|
||||
|
||||
doc = MagicMock()
|
||||
|
||||
244
tests/cli/test_bib_fetch_sealed.py
Normal file
244
tests/cli/test_bib_fetch_sealed.py
Normal file
@@ -0,0 +1,244 @@
|
||||
"""fetch-pfs-comments / fetch-docket-comments must not call reg.gov for
|
||||
sealed dockets and must reuse stored document ids for known ones."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from bib.dockets import Docket
|
||||
from bib.item import Rule
|
||||
from bib.regulations_gov import WalkResult
|
||||
from bib.store import Store
|
||||
from cli.bib import app
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
def _store() -> Store:
|
||||
return Store(":memory:", storage_dir="/tmp/nope")
|
||||
|
||||
|
||||
def _rule_doc():
|
||||
doc = MagicMock()
|
||||
doc.type = "Proposed Rule"
|
||||
doc.publication_date = "2026-07-16"
|
||||
doc.dockets = ["CMS-1848-P"]
|
||||
doc.html_url = "https://example.com"
|
||||
doc.document_number = "2026-1"
|
||||
return doc
|
||||
|
||||
|
||||
def _api():
|
||||
api = MagicMock()
|
||||
api.__enter__ = MagicMock(return_value=api)
|
||||
api.__exit__ = MagicMock(return_value=False)
|
||||
return api
|
||||
|
||||
|
||||
@patch("bib.regulations_gov.walk_docket", return_value=WalkResult())
|
||||
@patch("bib.regulations_gov.Client")
|
||||
@patch("bib.translate.federal_register")
|
||||
@patch("bib.federalregister.split_docket_ids", return_value=["CMS-1848-P"])
|
||||
@patch("bib.federalregister.pfs_rules")
|
||||
@patch("bib.connect")
|
||||
def test_sealed_docket_skipped_before_any_call(
|
||||
mc_connect, mc_pfs, _split, mc_fr, mc_client, mc_walk
|
||||
):
|
||||
s = _store()
|
||||
s.docket_upsert(
|
||||
Docket(
|
||||
id="CMS-2026-2377",
|
||||
rule_cms_id="CMS-1848-P",
|
||||
fr_object_id="o",
|
||||
sealed_at="2026-10-20T00:00:00Z",
|
||||
seal_reason="auto",
|
||||
)
|
||||
)
|
||||
mc_connect.return_value = s
|
||||
mc_pfs.return_value = [_rule_doc()]
|
||||
api = _api()
|
||||
mc_client.return_value = api
|
||||
|
||||
result = runner.invoke(app, ["fetch-pfs-comments"])
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "sealed" in result.output
|
||||
mc_fr.assert_not_called() # no rule-metadata fetch
|
||||
api.resolve_docket.assert_not_called()
|
||||
api.find_documents_in_docket.assert_not_called()
|
||||
mc_walk.assert_not_called()
|
||||
|
||||
|
||||
@patch("bib.regulations_gov.walk_docket", return_value=WalkResult(created=2))
|
||||
@patch("bib.regulations_gov.Client")
|
||||
@patch("bib.translate.federal_register")
|
||||
@patch("bib.federalregister.split_docket_ids", return_value=["CMS-1848-P"])
|
||||
@patch("bib.federalregister.pfs_rules")
|
||||
@patch("bib.connect")
|
||||
def test_known_open_docket_walks_without_resolve(
|
||||
mc_connect, mc_pfs, _split, mc_fr, mc_client, mc_walk
|
||||
):
|
||||
s = _store()
|
||||
s.docket_upsert(
|
||||
Docket(
|
||||
id="CMS-2026-2377",
|
||||
rule_cms_id="CMS-1848-P",
|
||||
fr_object_id="o",
|
||||
comment_end_date="2026-09-14",
|
||||
)
|
||||
)
|
||||
mc_connect.return_value = s
|
||||
mc_pfs.return_value = [_rule_doc()]
|
||||
mc_fr.return_value = Rule(
|
||||
title="CY2027 PFS NPRM", url="https://www.federalregister.gov/d/2026-1"
|
||||
)
|
||||
api = _api()
|
||||
mc_client.return_value = api
|
||||
|
||||
result = runner.invoke(app, ["fetch-pfs-comments"])
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
api.resolve_docket.assert_not_called()
|
||||
api.find_documents_in_docket.assert_not_called()
|
||||
mc_walk.assert_called_once()
|
||||
assert mc_walk.call_args.args[2].id == "CMS-2026-2377"
|
||||
assert "created=2" in result.output
|
||||
|
||||
|
||||
@patch("bib.regulations_gov.walk_docket", return_value=WalkResult())
|
||||
@patch("bib.regulations_gov.discover_docket")
|
||||
@patch("bib.regulations_gov.Client")
|
||||
@patch("bib.translate.federal_register")
|
||||
@patch("bib.federalregister.split_docket_ids", return_value=["CMS-1848-P"])
|
||||
@patch("bib.federalregister.pfs_rules")
|
||||
@patch("bib.connect")
|
||||
def test_unknown_docket_is_resolved_once_and_stored(
|
||||
mc_connect, mc_pfs, _split, mc_fr, mc_client, mc_disc, mc_walk
|
||||
):
|
||||
s = _store()
|
||||
mc_connect.return_value = s
|
||||
mc_pfs.return_value = [_rule_doc()]
|
||||
mc_fr.return_value = Rule(
|
||||
title="CY2027 PFS NPRM", url="https://www.federalregister.gov/d/2026-1"
|
||||
)
|
||||
api = _api()
|
||||
api.resolve_docket.return_value = "CMS-2026-2377"
|
||||
mc_client.return_value = api
|
||||
mc_disc.return_value = Docket(
|
||||
id="CMS-2026-2377",
|
||||
rule_cms_id="CMS-1848-P",
|
||||
fr_object_id="o",
|
||||
comment_end_date="2026-09-14",
|
||||
)
|
||||
|
||||
result = runner.invoke(app, ["fetch-pfs-comments"])
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
api.resolve_docket.assert_called_once_with("CMS-1848-P")
|
||||
mc_disc.assert_called_once()
|
||||
assert s.docket_get("CMS-2026-2377").fr_object_id == "o"
|
||||
|
||||
|
||||
@patch("bib.regulations_gov.walk_docket", return_value=WalkResult())
|
||||
@patch("bib.regulations_gov.Client")
|
||||
@patch("bib.translate.federal_register")
|
||||
@patch("bib.federalregister.split_docket_ids", return_value=["CMS-1848-P"])
|
||||
@patch("bib.federalregister.pfs_rules")
|
||||
@patch("bib.connect")
|
||||
def test_docket_filter_skips_other_known_dockets_without_calls(
|
||||
mc_connect, mc_pfs, _split, mc_fr, mc_client, mc_walk
|
||||
):
|
||||
s = _store()
|
||||
s.docket_upsert(
|
||||
Docket(id="CMS-2026-2377", rule_cms_id="CMS-1848-P", fr_object_id="o")
|
||||
)
|
||||
mc_connect.return_value = s
|
||||
mc_pfs.return_value = [_rule_doc()]
|
||||
api = _api()
|
||||
mc_client.return_value = api
|
||||
|
||||
result = runner.invoke(app, ["fetch-pfs-comments", "--docket", "CMS-2019-0111"])
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
mc_fr.assert_not_called()
|
||||
api.resolve_docket.assert_not_called()
|
||||
mc_walk.assert_not_called()
|
||||
|
||||
|
||||
@patch("bib.regulations_gov.walk_docket", return_value=WalkResult())
|
||||
@patch("bib.regulations_gov.Client")
|
||||
@patch("bib.translate.federal_register")
|
||||
@patch("bib.federalregister.split_docket_ids", return_value=["CMS-1848-P"])
|
||||
@patch("bib.federalregister.pfs_rules")
|
||||
@patch("bib.connect")
|
||||
def test_force_walks_sealed_docket(
|
||||
mc_connect, mc_pfs, _split, mc_fr, mc_client, mc_walk
|
||||
):
|
||||
s = _store()
|
||||
s.docket_upsert(
|
||||
Docket(
|
||||
id="CMS-2026-2377",
|
||||
rule_cms_id="CMS-1848-P",
|
||||
fr_object_id="o",
|
||||
sealed_at="2026-10-20T00:00:00Z",
|
||||
)
|
||||
)
|
||||
mc_connect.return_value = s
|
||||
mc_pfs.return_value = [_rule_doc()]
|
||||
mc_fr.return_value = Rule(
|
||||
title="CY2027 PFS NPRM", url="https://www.federalregister.gov/d/2026-1"
|
||||
)
|
||||
mc_client.return_value = _api()
|
||||
|
||||
result = runner.invoke(app, ["fetch-pfs-comments", "--force"])
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
mc_walk.assert_called_once()
|
||||
assert mc_walk.call_args.kwargs["force"] is True
|
||||
|
||||
|
||||
@patch("bib.regulations_gov.walk_docket", return_value=WalkResult(created=1))
|
||||
@patch("bib.regulations_gov.discover_docket")
|
||||
@patch("bib.regulations_gov.Client")
|
||||
@patch("bib.connect")
|
||||
def test_fetch_docket_comments_discovers_then_walks(
|
||||
mc_connect, mc_client, mc_disc, mc_walk
|
||||
):
|
||||
s = _store()
|
||||
mc_connect.return_value = s
|
||||
mc_client.return_value = _api()
|
||||
mc_disc.return_value = Docket(
|
||||
id="CMS-2026-2377",
|
||||
rule_cms_id="CMS-1848-P",
|
||||
fr_object_id="o",
|
||||
comment_end_date="2026-09-14",
|
||||
)
|
||||
|
||||
result = runner.invoke(
|
||||
app, ["fetch-docket-comments", "CMS-2026-2377", "--cms-id", "CMS-1848-P"]
|
||||
)
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
mc_disc.assert_called_once()
|
||||
mc_walk.assert_called_once()
|
||||
assert s.docket_get("CMS-2026-2377") is not None
|
||||
|
||||
|
||||
@patch("bib.regulations_gov.walk_docket")
|
||||
@patch("bib.regulations_gov.Client")
|
||||
@patch("bib.connect")
|
||||
def test_fetch_docket_comments_sealed_notice(mc_connect, mc_client, mc_walk):
|
||||
s = _store()
|
||||
s.docket_upsert(
|
||||
Docket(id="CMS-2019-0111", fr_object_id="o", sealed_at="2020-01-01T00:00:00Z")
|
||||
)
|
||||
mc_connect.return_value = s
|
||||
mc_client.return_value = _api()
|
||||
|
||||
result = runner.invoke(app, ["fetch-docket-comments", "CMS-2019-0111"])
|
||||
|
||||
assert result.exit_code == 0
|
||||
assert "sealed" in result.output
|
||||
mc_walk.assert_not_called()
|
||||
Reference in New Issue
Block a user