fix(bib): walk_docket never seals an empty docket; --force re-fetches attachments (refs #615)

An empty walk over a docket bib never ingested (a farm that never
landed) satisfied should_seal and sealed it shut with zero comments.
Read docket_counts before sealing and require at least one stored
comment; the counts read is reused for the seal row.

--force also skipped attachments for every comment whose row came back
"unchanged", so the one flag meant to repair a docket could not repair
its missing files.
This commit is contained in:
kert
2026-09-08 15:36:46 -04:00
parent 066b304111
commit 32c06ad204
2 changed files with 64 additions and 6 deletions

View File

@@ -950,6 +950,11 @@ def walk_docket(
advance the watermark on a clean walk, and auto-seal when
:func:`should_seal` says so. ``force`` walks from page 1 and never
seals. Returns per-status counts.
Attachments are fetched for created/updated comments, and for every
comment under ``force`` (an unchanged row can still be missing its
files). A docket is only auto-sealed when bib actually holds at
least one of its comments.
"""
from bib.dockets import should_seal
@@ -970,7 +975,7 @@ def walk_docket(
store, c, cms_id=docket.rule_cms_id, extra_tags=[f"reg-docket:{docket.id}"]
)
setattr(res, status, getattr(res, status) + 1)
if attachments and c.attachment_count and status != "unchanged":
if attachments and c.attachment_count and (force or status != "unchanged"):
for att in client.attachments_for(c.id):
path = client.download_attachment(att.url, scratch / c.id)
if path:
@@ -1000,10 +1005,13 @@ def walk_docket(
docket, pull_watermark=max_lm, last_pull_at=now, last_pull_new=res.created
)
store.docket_upsert(updated)
if should_seal(updated, today or date.today(), quiet_days):
store.docket_seal(
docket.id, reason="auto", counts=docket_counts(store, docket.id)
)
counts = docket_counts(store, docket.id)
# A docket with nothing in bib is a farm that never landed, not a
# finished one: an empty walk over it must not seal it shut.
if counts["comments"] and should_seal(
updated, today or date.today(), quiet_days
):
store.docket_seal(docket.id, reason="auto", counts=counts)
res.sealed = True
return res

View File

@@ -1,12 +1,19 @@
from __future__ import annotations
import json
from datetime import date
from unittest.mock import MagicMock
import httpx
from bib.dockets import Docket
from bib.regulations_gov import Comment, WalkResult, discover_docket, walk_docket
from bib.regulations_gov import (
Attachment,
Comment,
WalkResult,
discover_docket,
walk_docket,
)
from bib.store import Store
D = "CMS-2026-2377"
@@ -129,11 +136,54 @@ def test_auto_seal_after_quiet_empty_pull():
s = _store()
s.docket_upsert(_docket())
api = MagicMock()
api.iter_comments.return_value = [_c(1, "2026-09-01T00:00:00Z")]
walk_docket(s, api, s.docket_get(D), today=date(2026, 9, 8))
api.iter_comments.return_value = []
r = walk_docket(s, api, s.docket_get(D), today=date(2026, 10, 20), quiet_days=30)
assert r.sealed is True
d = s.docket_get(D)
assert d.sealed and d.seal_reason == "auto"
assert json.loads(d.counts_json)["comments"] == 1
def test_no_auto_seal_when_the_docket_has_no_stored_comments():
"""An empty walk over a docket bib never ingested is a failed farm,
not a finished one — sealing it would freeze zero comments forever."""
s = _store()
s.docket_upsert(_docket())
api = MagicMock()
api.iter_comments.return_value = []
r = walk_docket(s, api, s.docket_get(D), today=date(2026, 10, 20), quiet_days=30)
assert r.sealed is False
assert not s.docket_get(D).sealed
def test_attachments_fetched_for_new_comments_only_unless_forced(tmp_path):
s = _store()
s.docket_upsert(_docket())
att = Attachment(url="https://cdn/att.pdf", filename="att.pdf")
blob = tmp_path / "att.pdf"
blob.write_bytes(b"%PDF-1.4")
api = MagicMock()
api.attachments_for.return_value = [att]
api.download_attachment.return_value = blob
c = _c(1, "2026-09-01T00:00:00Z")
c.attachment_count = 1
api.iter_comments.return_value = [c]
walk_docket(s, api, s.docket_get(D), attachments=True, today=date(2026, 9, 8))
assert api.download_attachment.call_count == 1 # created → fetched
api.iter_comments.return_value = [c]
r = walk_docket(s, api, s.docket_get(D), attachments=True, today=date(2026, 9, 8))
assert r.unchanged == 1
assert api.download_attachment.call_count == 1 # unchanged → not re-fetched
api.iter_comments.return_value = [c]
walk_docket(
s, api, s.docket_get(D), attachments=True, force=True, today=date(2026, 9, 8)
)
assert api.download_attachment.call_count == 2 # --force re-fetches
def test_progress_echo_and_commit_every_50():