diff --git a/src/bib/regulations_gov.py b/src/bib/regulations_gov.py index 5d0b9c7..b2c2c44 100644 --- a/src/bib/regulations_gov.py +++ b/src/bib/regulations_gov.py @@ -950,6 +950,11 @@ def walk_docket( advance the watermark on a clean walk, and auto-seal when :func:`should_seal` says so. ``force`` walks from page 1 and never seals. Returns per-status counts. + + Attachments are fetched for created/updated comments, and for every + comment under ``force`` (an unchanged row can still be missing its + files). A docket is only auto-sealed when bib actually holds at + least one of its comments. """ from bib.dockets import should_seal @@ -970,7 +975,7 @@ def walk_docket( store, c, cms_id=docket.rule_cms_id, extra_tags=[f"reg-docket:{docket.id}"] ) setattr(res, status, getattr(res, status) + 1) - if attachments and c.attachment_count and status != "unchanged": + if attachments and c.attachment_count and (force or status != "unchanged"): for att in client.attachments_for(c.id): path = client.download_attachment(att.url, scratch / c.id) if path: @@ -1000,10 +1005,13 @@ def walk_docket( docket, pull_watermark=max_lm, last_pull_at=now, last_pull_new=res.created ) store.docket_upsert(updated) - if should_seal(updated, today or date.today(), quiet_days): - store.docket_seal( - docket.id, reason="auto", counts=docket_counts(store, docket.id) - ) + counts = docket_counts(store, docket.id) + # A docket with nothing in bib is a farm that never landed, not a + # finished one: an empty walk over it must not seal it shut. + if counts["comments"] and should_seal( + updated, today or date.today(), quiet_days + ): + store.docket_seal(docket.id, reason="auto", counts=counts) res.sealed = True return res diff --git a/tests/bib/test_walk_docket.py b/tests/bib/test_walk_docket.py index 23cdb2b..7ac4aad 100644 --- a/tests/bib/test_walk_docket.py +++ b/tests/bib/test_walk_docket.py @@ -1,12 +1,19 @@ from __future__ import annotations +import json from datetime import date from unittest.mock import MagicMock import httpx from bib.dockets import Docket -from bib.regulations_gov import Comment, WalkResult, discover_docket, walk_docket +from bib.regulations_gov import ( + Attachment, + Comment, + WalkResult, + discover_docket, + walk_docket, +) from bib.store import Store D = "CMS-2026-2377" @@ -129,11 +136,54 @@ def test_auto_seal_after_quiet_empty_pull(): s = _store() s.docket_upsert(_docket()) api = MagicMock() + api.iter_comments.return_value = [_c(1, "2026-09-01T00:00:00Z")] + walk_docket(s, api, s.docket_get(D), today=date(2026, 9, 8)) api.iter_comments.return_value = [] r = walk_docket(s, api, s.docket_get(D), today=date(2026, 10, 20), quiet_days=30) assert r.sealed is True d = s.docket_get(D) assert d.sealed and d.seal_reason == "auto" + assert json.loads(d.counts_json)["comments"] == 1 + + +def test_no_auto_seal_when_the_docket_has_no_stored_comments(): + """An empty walk over a docket bib never ingested is a failed farm, + not a finished one — sealing it would freeze zero comments forever.""" + s = _store() + s.docket_upsert(_docket()) + api = MagicMock() + api.iter_comments.return_value = [] + r = walk_docket(s, api, s.docket_get(D), today=date(2026, 10, 20), quiet_days=30) + assert r.sealed is False + assert not s.docket_get(D).sealed + + +def test_attachments_fetched_for_new_comments_only_unless_forced(tmp_path): + s = _store() + s.docket_upsert(_docket()) + att = Attachment(url="https://cdn/att.pdf", filename="att.pdf") + blob = tmp_path / "att.pdf" + blob.write_bytes(b"%PDF-1.4") + api = MagicMock() + api.attachments_for.return_value = [att] + api.download_attachment.return_value = blob + c = _c(1, "2026-09-01T00:00:00Z") + c.attachment_count = 1 + + api.iter_comments.return_value = [c] + walk_docket(s, api, s.docket_get(D), attachments=True, today=date(2026, 9, 8)) + assert api.download_attachment.call_count == 1 # created → fetched + + api.iter_comments.return_value = [c] + r = walk_docket(s, api, s.docket_get(D), attachments=True, today=date(2026, 9, 8)) + assert r.unchanged == 1 + assert api.download_attachment.call_count == 1 # unchanged → not re-fetched + + api.iter_comments.return_value = [c] + walk_docket( + s, api, s.docket_get(D), attachments=True, force=True, today=date(2026, 9, 8) + ) + assert api.download_attachment.call_count == 2 # --force re-fetches def test_progress_echo_and_commit_every_50():