fix(bib): walk_docket never seals an empty docket; --force re-fetches attachments (refs #615)

An empty walk over a docket bib never ingested (a farm that never
landed) satisfied should_seal and sealed it shut with zero comments.
Read docket_counts before sealing and require at least one stored
comment; the counts read is reused for the seal row.

--force also skipped attachments for every comment whose row came back
"unchanged", so the one flag meant to repair a docket could not repair
its missing files.
This commit is contained in:
kert
2026-09-08 15:36:46 -04:00
parent 066b304111
commit 32c06ad204
2 changed files with 64 additions and 6 deletions

View File

@@ -950,6 +950,11 @@ def walk_docket(
advance the watermark on a clean walk, and auto-seal when advance the watermark on a clean walk, and auto-seal when
:func:`should_seal` says so. ``force`` walks from page 1 and never :func:`should_seal` says so. ``force`` walks from page 1 and never
seals. Returns per-status counts. seals. Returns per-status counts.
Attachments are fetched for created/updated comments, and for every
comment under ``force`` (an unchanged row can still be missing its
files). A docket is only auto-sealed when bib actually holds at
least one of its comments.
""" """
from bib.dockets import should_seal from bib.dockets import should_seal
@@ -970,7 +975,7 @@ def walk_docket(
store, c, cms_id=docket.rule_cms_id, extra_tags=[f"reg-docket:{docket.id}"] store, c, cms_id=docket.rule_cms_id, extra_tags=[f"reg-docket:{docket.id}"]
) )
setattr(res, status, getattr(res, status) + 1) setattr(res, status, getattr(res, status) + 1)
if attachments and c.attachment_count and status != "unchanged": if attachments and c.attachment_count and (force or status != "unchanged"):
for att in client.attachments_for(c.id): for att in client.attachments_for(c.id):
path = client.download_attachment(att.url, scratch / c.id) path = client.download_attachment(att.url, scratch / c.id)
if path: if path:
@@ -1000,10 +1005,13 @@ def walk_docket(
docket, pull_watermark=max_lm, last_pull_at=now, last_pull_new=res.created docket, pull_watermark=max_lm, last_pull_at=now, last_pull_new=res.created
) )
store.docket_upsert(updated) store.docket_upsert(updated)
if should_seal(updated, today or date.today(), quiet_days): counts = docket_counts(store, docket.id)
store.docket_seal( # A docket with nothing in bib is a farm that never landed, not a
docket.id, reason="auto", counts=docket_counts(store, docket.id) # finished one: an empty walk over it must not seal it shut.
) if counts["comments"] and should_seal(
updated, today or date.today(), quiet_days
):
store.docket_seal(docket.id, reason="auto", counts=counts)
res.sealed = True res.sealed = True
return res return res

View File

@@ -1,12 +1,19 @@
from __future__ import annotations from __future__ import annotations
import json
from datetime import date from datetime import date
from unittest.mock import MagicMock from unittest.mock import MagicMock
import httpx import httpx
from bib.dockets import Docket from bib.dockets import Docket
from bib.regulations_gov import Comment, WalkResult, discover_docket, walk_docket from bib.regulations_gov import (
Attachment,
Comment,
WalkResult,
discover_docket,
walk_docket,
)
from bib.store import Store from bib.store import Store
D = "CMS-2026-2377" D = "CMS-2026-2377"
@@ -129,11 +136,54 @@ def test_auto_seal_after_quiet_empty_pull():
s = _store() s = _store()
s.docket_upsert(_docket()) s.docket_upsert(_docket())
api = MagicMock() api = MagicMock()
api.iter_comments.return_value = [_c(1, "2026-09-01T00:00:00Z")]
walk_docket(s, api, s.docket_get(D), today=date(2026, 9, 8))
api.iter_comments.return_value = [] api.iter_comments.return_value = []
r = walk_docket(s, api, s.docket_get(D), today=date(2026, 10, 20), quiet_days=30) r = walk_docket(s, api, s.docket_get(D), today=date(2026, 10, 20), quiet_days=30)
assert r.sealed is True assert r.sealed is True
d = s.docket_get(D) d = s.docket_get(D)
assert d.sealed and d.seal_reason == "auto" assert d.sealed and d.seal_reason == "auto"
assert json.loads(d.counts_json)["comments"] == 1
def test_no_auto_seal_when_the_docket_has_no_stored_comments():
"""An empty walk over a docket bib never ingested is a failed farm,
not a finished one — sealing it would freeze zero comments forever."""
s = _store()
s.docket_upsert(_docket())
api = MagicMock()
api.iter_comments.return_value = []
r = walk_docket(s, api, s.docket_get(D), today=date(2026, 10, 20), quiet_days=30)
assert r.sealed is False
assert not s.docket_get(D).sealed
def test_attachments_fetched_for_new_comments_only_unless_forced(tmp_path):
s = _store()
s.docket_upsert(_docket())
att = Attachment(url="https://cdn/att.pdf", filename="att.pdf")
blob = tmp_path / "att.pdf"
blob.write_bytes(b"%PDF-1.4")
api = MagicMock()
api.attachments_for.return_value = [att]
api.download_attachment.return_value = blob
c = _c(1, "2026-09-01T00:00:00Z")
c.attachment_count = 1
api.iter_comments.return_value = [c]
walk_docket(s, api, s.docket_get(D), attachments=True, today=date(2026, 9, 8))
assert api.download_attachment.call_count == 1 # created → fetched
api.iter_comments.return_value = [c]
r = walk_docket(s, api, s.docket_get(D), attachments=True, today=date(2026, 9, 8))
assert r.unchanged == 1
assert api.download_attachment.call_count == 1 # unchanged → not re-fetched
api.iter_comments.return_value = [c]
walk_docket(
s, api, s.docket_get(D), attachments=True, force=True, today=date(2026, 9, 8)
)
assert api.download_attachment.call_count == 2 # --force re-fetches
def test_progress_echo_and_commit_every_50(): def test_progress_echo_and_commit_every_50():