feat(cli): fetch commands walk from watermark, skip sealed dockets before any API call (refs #615)

This commit is contained in:
kert
2026-09-08 12:53:54 -04:00
parent 5936cb8d42
commit fb335605d5
3 changed files with 423 additions and 160 deletions

View File

@@ -126,9 +126,29 @@ def discover_pfs_rules(
store.upsert(rule)
def _walk_and_report(store, api, docket, *, attachments, limit, force, echo) -> None:
from bib.dockets import quiet_days
from bib.regulations_gov import walk_docket
r = walk_docket(
store,
api,
docket,
attachments=attachments,
limit=limit,
force=force,
quiet_days=quiet_days(),
echo=echo,
)
echo(
f" {docket.id}: created={r.created} updated={r.updated} "
f"unchanged={r.unchanged} clean={r.clean}" + (" — sealed" if r.sealed else "")
)
@app.command(name="fetch-docket-comments")
def fetch_docket_comments(
docket: str = typer.Argument(..., help="e.g. CMS-1676-P"),
docket: str = typer.Argument(..., help="reg.gov docket id, e.g. CMS-2026-2377"),
cms_id: str = typer.Option(
"", "--cms-id", help="Associate comments with a specific CMS rule tag."
),
@@ -138,52 +158,54 @@ def fetch_docket_comments(
attachments: bool = typer.Option(
False,
"--attachments",
help="Also download each comment's PDF/DOCX attachments "
"(expensive — skim first without).",
help="Also download each comment's PDF/DOCX attachments.",
),
sleep: float = typer.Option(
1.3, "--sleep", help="Seconds between API calls (rate budget)."
),
force: bool = typer.Option(
False,
"--force",
help="Walk from page 1 even if sealed / watermarked. Never unseals.",
),
) -> None:
"""Walk every comment on one docket; upsert as Source items.
"""Walk one docket's comments from its stored watermark; upsert as Source items.
Policy: we iterate every FR document indexed under the docket and
pull their comments. For one PFS rule that usually means one parent
document with a few thousand to tens of thousands of comments.
The first run discovers the docket's commentable FR document (one
listing call) and stores it; later runs make no discovery calls.
Sealed dockets are skipped unless --force.
"""
from pathlib import Path
from bib import connect
from bib.regulations_gov import Client, upsert_comment
from bib.regulations_gov import Client, discover_docket
store = connect()
scratch = Path(f".state/comments/{docket}")
with Client(sleep=sleep) as api:
docs = api.find_documents_in_docket(docket)
typer.echo(f" docket {docket}: {len(docs)} FR documents indexed")
total = 0
for fr_doc in docs:
attrs = fr_doc.get("attributes") or {}
fr_id = fr_doc["id"]
object_id = attrs.get("objectId")
if not object_id or not attrs.get("commentEndDate"):
continue
typer.echo(f" ↓ comments on {fr_id} (objectId={object_id})")
for c in api.iter_comments(object_id):
if limit and total >= limit:
return
key = upsert_comment(store, c, cms_id=cms_id)
if attachments and c.attachment_count:
for att in api.attachments_for(c.id):
path = api.download_attachment(att.url, scratch / c.id)
if path:
store.attach_file(key, path, title=att.filename)
total += 1
if total % 50 == 0:
typer.echo(f" processed {total} comments")
store._con().commit() # noqa: SLF001
typer.echo(f" total: {total} comments upserted")
d = store.docket_get(docket)
if d is None:
d = discover_docket(api, docket, rule_cms_id=cms_id)
store.docket_upsert(d)
elif cms_id and not d.rule_cms_id:
from dataclasses import replace
d = replace(d, rule_cms_id=cms_id)
store.docket_upsert(d)
if d.sealed and not force:
typer.echo(
f" {docket}: sealed {d.sealed_at[:10]} ({d.seal_reason}); skipping — use --force to re-walk"
)
return
if not d.fr_object_id:
typer.echo(f" {docket}: no commentable FR document found")
return
_walk_and_report(
store,
api,
d,
attachments=attachments,
limit=limit,
force=force,
echo=typer.echo,
)
@app.command(name="fetch-pfs-comments")
@@ -192,46 +214,59 @@ def fetch_pfs_comments(
until: str = typer.Option("", "--until"),
attachments: bool = typer.Option(False, "--attachments"),
per_docket_limit: int = typer.Option(
0,
"--per-docket-limit",
help="Cap comments fetched per docket. 0 = unlimited.",
0, "--per-docket-limit", help="Cap comments fetched per docket. 0 = unlimited."
),
sleep: float = typer.Option(1.3, "--sleep"),
docket: str = typer.Option(
"",
"--docket",
help="Only pull this reg.gov docket (e.g. CMS-2026-2377); "
"the other PFS dockets are skipped without any API calls.",
help="Only pull this reg.gov docket (e.g. CMS-2026-2377); other dockets are skipped before any API call once known.",
),
force: bool = typer.Option(
False,
"--force",
help="Walk every docket from page 1, sealed or not. Never unseals.",
),
) -> None:
"""One-shot: discover every PFS rule since *since* and pull every
comment on each of their dockets.
"""Discover every PFS proposed rule since *since* and pull new comments
on each docket from its stored watermark.
Heavy run. A single PFS proposed rule can hold 5K–25K comments;
times ~20 rules and with attachments this runs for many hours
under the reg.gov rate budget. Use ``--per-docket-limit`` to
smoke-test first.
Sealed dockets (comment period closed + quiet period + an empty pull)
cost nothing: no rule-metadata fetch, no resolve call, no walk.
"""
from pathlib import Path
from bib import connect
from bib.federalregister import pfs_rules, split_docket_ids
from bib.regulations_gov import Client, upsert_comment
from bib.regulations_gov import Client, discover_docket
from bib.tag import Tag
from bib.translate import federal_register
store = connect()
rules = pfs_rules(since=since, until=until or None)
typer.echo(f"==> {len(rules)} PFS rules since {since}")
# Only proposed rules have public comment periods — skip finals +
# corrections to keep the work focused.
proposed = [r for r in rules if r.type == "Proposed Rule"]
typer.echo(f" {len(proposed)} proposed rules (the ones with comments)")
with Client(sleep=sleep) as api:
for doc in proposed:
cms_ids = split_docket_ids(doc.dockets)
known = {cid: store.docket_for_rule(cid) for cid in cms_ids}
# Decide what this rule needs BEFORE touching the network.
def _wanted(cid: str) -> bool:
d = known[cid]
if docket and d is not None and d.id != docket:
return False
if d is not None and d.sealed and not force:
typer.echo(
f" {d.id}: sealed {d.sealed_at[:10]} ({d.seal_reason}); skipping"
)
return False
return True
todo = [cid for cid in cms_ids if _wanted(cid)]
if not todo:
continue
try:
rule = federal_register(
doc.html_url
@@ -244,58 +279,33 @@ def fetch_pfs_comments(
rule.add_tag("module:pfs")
for cid in cms_ids:
rule.add_tag(f"cms-rule:{cid}")
store.upsert(rule)
# Resolve each CMS-XXXX-P to its reg.gov docket id.
for cms_id in cms_ids:
reg_docket = api.resolve_docket(cms_id)
if not reg_docket:
typer.echo(f" skip {cms_id}: no reg.gov docket found")
continue
if docket and reg_docket != docket:
continue
typer.echo(f" {cms_id} → {reg_docket} ({doc.publication_date})")
rule.add_tag(f"reg-docket:{reg_docket}")
for cms_id in todo:
d = known[cms_id]
if d is None:
reg_docket = api.resolve_docket(cms_id)
if not reg_docket:
typer.echo(f" skip {cms_id}: no reg.gov docket found")
continue
if docket and reg_docket != docket:
continue
d = discover_docket(api, reg_docket, rule_cms_id=cms_id)
store.docket_upsert(d)
typer.echo(f" {cms_id} → {d.id} ({doc.publication_date})")
rule.add_tag(f"reg-docket:{d.id}")
store.upsert(rule)
fr_docs = api.find_documents_in_docket(reg_docket)
per_docket_count = 0
scratch = Path(f".state/comments/{reg_docket}")
for fr_doc in fr_docs:
attrs = fr_doc.get("attributes") or {}
object_id = attrs.get("objectId")
# Skip docs with no objectId or no real comment
# window — final rules and internal display versions
# don't carry meaningful comment traffic.
if not object_id:
continue
if not attrs.get("commentEndDate"):
continue
for c in api.iter_comments(object_id):
if per_docket_limit and per_docket_count >= per_docket_limit:
break
key = upsert_comment(
store,
c,
cms_id=cms_id,
extra_tags=[f"reg-docket:{reg_docket}"],
)
if attachments and c.attachment_count:
for att in api.attachments_for(c.id):
path = api.download_attachment(
att.url,
scratch / c.id,
)
if path:
store.attach_file(key, path, title=att.filename)
per_docket_count += 1
if per_docket_count % 50 == 0:
typer.echo(f" {per_docket_count} comments")
store._con().commit() # noqa: SLF001
if per_docket_limit and per_docket_count >= per_docket_limit:
break
store._con().commit() # noqa: SLF001
typer.echo(f" docket total: {per_docket_count}")
if not d.fr_object_id:
typer.echo(f" {d.id}: no commentable FR document; skipping")
continue
_walk_and_report(
store,
api,
d,
attachments=attachments,
limit=per_docket_limit,
force=force,
echo=typer.echo,
)
@app.command(name="ingest-mail")

View File

@@ -106,10 +106,14 @@ class TestDiscoverPfsRulesExercise:
class TestFetchDocketComments:
@patch("bib.connect")
@patch("bib.regulations_gov.Client")
@patch("bib.regulations_gov.upsert_comment", return_value="KEY1")
def test_basic(self, mc_upsert, mc_client_cls, mc_connect):
store = MagicMock()
store._con.return_value = MagicMock()
@patch("bib.regulations_gov.discover_docket")
@patch("bib.regulations_gov.walk_docket")
def test_basic(self, mc_walk, mc_disc, mc_client_cls, mc_connect):
from bib.dockets import Docket
from bib.regulations_gov import WalkResult
from bib.store import Store
store = Store(":memory:", storage_dir="/tmp/nope")
mc_connect.return_value = store
api = MagicMock()
@@ -117,26 +121,26 @@ class TestFetchDocketComments:
api.__exit__ = MagicMock(return_value=False)
mc_client_cls.return_value = api
fr_doc = {
"id": "DOC1",
"attributes": {"objectId": "09000001", "commentEndDate": "2023-12-31"},
}
api.find_documents_in_docket.return_value = [fr_doc]
comment = MagicMock()
comment.id = "C1"
comment.attachment_count = 0
api.iter_comments.return_value = [comment]
mc_disc.return_value = Docket(
id="CMS-1676-P", fr_object_id="09000001", comment_end_date="2023-12-31"
)
mc_walk.return_value = WalkResult(created=1)
result = runner.invoke(app, ["fetch-docket-comments", "CMS-1676-P", "-n", "5"])
assert result.exit_code == 0
mc_disc.assert_called_once()
mc_walk.assert_called_once()
@patch("bib.connect")
@patch("bib.regulations_gov.Client")
@patch("bib.regulations_gov.upsert_comment", return_value="KEY1")
def test_with_attachments(self, mc_upsert, mc_client_cls, mc_connect):
store = MagicMock()
store._con.return_value = MagicMock()
@patch("bib.regulations_gov.discover_docket")
@patch("bib.regulations_gov.walk_docket")
def test_with_attachments(self, mc_walk, mc_disc, mc_client_cls, mc_connect):
from bib.dockets import Docket
from bib.regulations_gov import WalkResult
from bib.store import Store
store = Store(":memory:", storage_dir="/tmp/nope")
mc_connect.return_value = store
api = MagicMock()
@@ -144,33 +148,27 @@ class TestFetchDocketComments:
api.__exit__ = MagicMock(return_value=False)
mc_client_cls.return_value = api
fr_doc = {
"id": "DOC1",
"attributes": {"objectId": "09000001", "commentEndDate": "2023-12-31"},
}
api.find_documents_in_docket.return_value = [fr_doc]
comment = MagicMock()
comment.id = "C1"
comment.attachment_count = 1
api.iter_comments.return_value = [comment]
att = MagicMock()
att.url = "https://example.com/att.pdf"
att.filename = "att.pdf"
api.attachments_for.return_value = [att]
api.download_attachment.return_value = Path("/tmp/att.pdf")
mc_disc.return_value = Docket(
id="CMS-1676-P", fr_object_id="09000001", comment_end_date="2023-12-31"
)
mc_walk.return_value = WalkResult(created=1)
result = runner.invoke(
app,
["fetch-docket-comments", "CMS-1676-P", "--attachments", "-n", "5"],
)
assert result.exit_code == 0
assert mc_walk.call_args.kwargs["attachments"] is True
@patch("bib.connect")
@patch("bib.regulations_gov.Client")
def test_skip_no_object_id(self, mc_client_cls, mc_connect):
store = MagicMock()
@patch("bib.regulations_gov.discover_docket")
@patch("bib.regulations_gov.walk_docket")
def test_skip_no_object_id(self, mc_walk, mc_disc, mc_client_cls, mc_connect):
from bib.dockets import Docket
from bib.store import Store
store = Store(":memory:", storage_dir="/tmp/nope")
mc_connect.return_value = store
api = MagicMock()
@@ -178,11 +176,12 @@ class TestFetchDocketComments:
api.__exit__ = MagicMock(return_value=False)
mc_client_cls.return_value = api
fr_doc = {"id": "DOC1", "attributes": {"objectId": None}}
api.find_documents_in_docket.return_value = [fr_doc]
mc_disc.return_value = Docket(id="CMS-1676-P")
result = runner.invoke(app, ["fetch-docket-comments", "CMS-1676-P"])
assert result.exit_code == 0
assert "no commentable FR document" in result.output
mc_walk.assert_not_called()
class TestFetchPfsComments:
@@ -191,12 +190,24 @@ class TestFetchPfsComments:
@patch("bib.federalregister.split_docket_ids", return_value=["CMS-1676-P"])
@patch("bib.translate.federal_register")
@patch("bib.regulations_gov.Client")
@patch("bib.regulations_gov.upsert_comment", return_value="KEY1")
@patch("bib.regulations_gov.discover_docket")
@patch("bib.regulations_gov.walk_docket")
def test_full_loop(
self, mc_upsert, mc_client_cls, mc_translate, mc_split, mc_pfs, mc_connect
self,
mc_walk,
mc_disc,
mc_client_cls,
mc_translate,
mc_split,
mc_pfs,
mc_connect,
):
store = MagicMock()
store._con.return_value = MagicMock()
from bib.dockets import Docket
from bib.item import Rule
from bib.regulations_gov import WalkResult
from bib.store import Store
store = Store(":memory:", storage_dir="/tmp/nope")
mc_connect.return_value = store
doc = MagicMock()
@@ -207,7 +218,7 @@ class TestFetchPfsComments:
doc.document_number = "2023-12345"
mc_pfs.return_value = [doc]
rule = MagicMock()
rule = Rule(title="CY2024 PFS NPRM", url="https://example.com")
mc_translate.return_value = rule
api = MagicMock()
@@ -216,22 +227,18 @@ class TestFetchPfsComments:
mc_client_cls.return_value = api
api.resolve_docket.return_value = "CMS-2023-0001"
fr_doc = {
"id": "DOC1",
"attributes": {
"objectId": "09000001",
"commentEndDate": "2023-12-31",
},
}
api.find_documents_in_docket.return_value = [fr_doc]
comment = MagicMock()
comment.id = "C1"
comment.attachment_count = 0
api.iter_comments.return_value = [comment]
mc_disc.return_value = Docket(
id="CMS-2023-0001",
rule_cms_id="CMS-1676-P",
fr_object_id="09000001",
comment_end_date="2023-12-31",
)
mc_walk.return_value = WalkResult(created=1)
result = runner.invoke(app, ["fetch-pfs-comments", "--per-docket-limit", "1"])
assert result.exit_code == 0
mc_disc.assert_called_once()
mc_walk.assert_called_once()
@patch("bib.connect")
@patch("bib.federalregister.pfs_rules", return_value=[])
@@ -276,7 +283,9 @@ class TestFetchPfsComments:
@patch("bib.translate.federal_register")
@patch("bib.regulations_gov.Client")
def test_no_docket(self, mc_client_cls, mc_translate, mc_split, mc_pfs, mc_connect):
store = MagicMock()
from bib.store import Store
store = Store(":memory:", storage_dir="/tmp/nope")
mc_connect.return_value = store
doc = MagicMock()

View File

@@ -0,0 +1,244 @@
"""fetch-pfs-comments / fetch-docket-comments must not call reg.gov for
sealed dockets and must reuse stored document ids for known ones."""
from __future__ import annotations
from unittest.mock import MagicMock, patch
from typer.testing import CliRunner
from bib.dockets import Docket
from bib.item import Rule
from bib.regulations_gov import WalkResult
from bib.store import Store
from cli.bib import app
runner = CliRunner()
def _store() -> Store:
return Store(":memory:", storage_dir="/tmp/nope")
def _rule_doc():
doc = MagicMock()
doc.type = "Proposed Rule"
doc.publication_date = "2026-07-16"
doc.dockets = ["CMS-1848-P"]
doc.html_url = "https://example.com"
doc.document_number = "2026-1"
return doc
def _api():
api = MagicMock()
api.__enter__ = MagicMock(return_value=api)
api.__exit__ = MagicMock(return_value=False)
return api
@patch("bib.regulations_gov.walk_docket", return_value=WalkResult())
@patch("bib.regulations_gov.Client")
@patch("bib.translate.federal_register")
@patch("bib.federalregister.split_docket_ids", return_value=["CMS-1848-P"])
@patch("bib.federalregister.pfs_rules")
@patch("bib.connect")
def test_sealed_docket_skipped_before_any_call(
mc_connect, mc_pfs, _split, mc_fr, mc_client, mc_walk
):
s = _store()
s.docket_upsert(
Docket(
id="CMS-2026-2377",
rule_cms_id="CMS-1848-P",
fr_object_id="o",
sealed_at="2026-10-20T00:00:00Z",
seal_reason="auto",
)
)
mc_connect.return_value = s
mc_pfs.return_value = [_rule_doc()]
api = _api()
mc_client.return_value = api
result = runner.invoke(app, ["fetch-pfs-comments"])
assert result.exit_code == 0, result.output
assert "sealed" in result.output
mc_fr.assert_not_called() # no rule-metadata fetch
api.resolve_docket.assert_not_called()
api.find_documents_in_docket.assert_not_called()
mc_walk.assert_not_called()
@patch("bib.regulations_gov.walk_docket", return_value=WalkResult(created=2))
@patch("bib.regulations_gov.Client")
@patch("bib.translate.federal_register")
@patch("bib.federalregister.split_docket_ids", return_value=["CMS-1848-P"])
@patch("bib.federalregister.pfs_rules")
@patch("bib.connect")
def test_known_open_docket_walks_without_resolve(
mc_connect, mc_pfs, _split, mc_fr, mc_client, mc_walk
):
s = _store()
s.docket_upsert(
Docket(
id="CMS-2026-2377",
rule_cms_id="CMS-1848-P",
fr_object_id="o",
comment_end_date="2026-09-14",
)
)
mc_connect.return_value = s
mc_pfs.return_value = [_rule_doc()]
mc_fr.return_value = Rule(
title="CY2027 PFS NPRM", url="https://www.federalregister.gov/d/2026-1"
)
api = _api()
mc_client.return_value = api
result = runner.invoke(app, ["fetch-pfs-comments"])
assert result.exit_code == 0, result.output
api.resolve_docket.assert_not_called()
api.find_documents_in_docket.assert_not_called()
mc_walk.assert_called_once()
assert mc_walk.call_args.args[2].id == "CMS-2026-2377"
assert "created=2" in result.output
@patch("bib.regulations_gov.walk_docket", return_value=WalkResult())
@patch("bib.regulations_gov.discover_docket")
@patch("bib.regulations_gov.Client")
@patch("bib.translate.federal_register")
@patch("bib.federalregister.split_docket_ids", return_value=["CMS-1848-P"])
@patch("bib.federalregister.pfs_rules")
@patch("bib.connect")
def test_unknown_docket_is_resolved_once_and_stored(
mc_connect, mc_pfs, _split, mc_fr, mc_client, mc_disc, mc_walk
):
s = _store()
mc_connect.return_value = s
mc_pfs.return_value = [_rule_doc()]
mc_fr.return_value = Rule(
title="CY2027 PFS NPRM", url="https://www.federalregister.gov/d/2026-1"
)
api = _api()
api.resolve_docket.return_value = "CMS-2026-2377"
mc_client.return_value = api
mc_disc.return_value = Docket(
id="CMS-2026-2377",
rule_cms_id="CMS-1848-P",
fr_object_id="o",
comment_end_date="2026-09-14",
)
result = runner.invoke(app, ["fetch-pfs-comments"])
assert result.exit_code == 0, result.output
api.resolve_docket.assert_called_once_with("CMS-1848-P")
mc_disc.assert_called_once()
assert s.docket_get("CMS-2026-2377").fr_object_id == "o"
@patch("bib.regulations_gov.walk_docket", return_value=WalkResult())
@patch("bib.regulations_gov.Client")
@patch("bib.translate.federal_register")
@patch("bib.federalregister.split_docket_ids", return_value=["CMS-1848-P"])
@patch("bib.federalregister.pfs_rules")
@patch("bib.connect")
def test_docket_filter_skips_other_known_dockets_without_calls(
mc_connect, mc_pfs, _split, mc_fr, mc_client, mc_walk
):
s = _store()
s.docket_upsert(
Docket(id="CMS-2026-2377", rule_cms_id="CMS-1848-P", fr_object_id="o")
)
mc_connect.return_value = s
mc_pfs.return_value = [_rule_doc()]
api = _api()
mc_client.return_value = api
result = runner.invoke(app, ["fetch-pfs-comments", "--docket", "CMS-2019-0111"])
assert result.exit_code == 0, result.output
mc_fr.assert_not_called()
api.resolve_docket.assert_not_called()
mc_walk.assert_not_called()
@patch("bib.regulations_gov.walk_docket", return_value=WalkResult())
@patch("bib.regulations_gov.Client")
@patch("bib.translate.federal_register")
@patch("bib.federalregister.split_docket_ids", return_value=["CMS-1848-P"])
@patch("bib.federalregister.pfs_rules")
@patch("bib.connect")
def test_force_walks_sealed_docket(
mc_connect, mc_pfs, _split, mc_fr, mc_client, mc_walk
):
s = _store()
s.docket_upsert(
Docket(
id="CMS-2026-2377",
rule_cms_id="CMS-1848-P",
fr_object_id="o",
sealed_at="2026-10-20T00:00:00Z",
)
)
mc_connect.return_value = s
mc_pfs.return_value = [_rule_doc()]
mc_fr.return_value = Rule(
title="CY2027 PFS NPRM", url="https://www.federalregister.gov/d/2026-1"
)
mc_client.return_value = _api()
result = runner.invoke(app, ["fetch-pfs-comments", "--force"])
assert result.exit_code == 0, result.output
mc_walk.assert_called_once()
assert mc_walk.call_args.kwargs["force"] is True
@patch("bib.regulations_gov.walk_docket", return_value=WalkResult(created=1))
@patch("bib.regulations_gov.discover_docket")
@patch("bib.regulations_gov.Client")
@patch("bib.connect")
def test_fetch_docket_comments_discovers_then_walks(
mc_connect, mc_client, mc_disc, mc_walk
):
s = _store()
mc_connect.return_value = s
mc_client.return_value = _api()
mc_disc.return_value = Docket(
id="CMS-2026-2377",
rule_cms_id="CMS-1848-P",
fr_object_id="o",
comment_end_date="2026-09-14",
)
result = runner.invoke(
app, ["fetch-docket-comments", "CMS-2026-2377", "--cms-id", "CMS-1848-P"]
)
assert result.exit_code == 0, result.output
mc_disc.assert_called_once()
mc_walk.assert_called_once()
assert s.docket_get("CMS-2026-2377") is not None
@patch("bib.regulations_gov.walk_docket")
@patch("bib.regulations_gov.Client")
@patch("bib.connect")
def test_fetch_docket_comments_sealed_notice(mc_connect, mc_client, mc_walk):
s = _store()
s.docket_upsert(
Docket(id="CMS-2019-0111", fr_object_id="o", sealed_at="2020-01-01T00:00:00Z")
)
mc_connect.return_value = s
mc_client.return_value = _api()
result = runner.invoke(app, ["fetch-docket-comments", "CMS-2019-0111"])
assert result.exit_code == 0
assert "sealed" in result.output
mc_walk.assert_not_called()