fix(bib): walk_docket never seals an empty docket; --force re-fetches attachments (refs #615)
An empty walk over a docket bib never ingested (a farm that never landed) satisfied should_seal and sealed it shut with zero comments. Read docket_counts before sealing and require at least one stored comment; the counts read is reused for the seal row. --force also skipped attachments for every comment whose row came back "unchanged", so the one flag meant to repair a docket could not repair its missing files.
This commit is contained in:
@@ -950,6 +950,11 @@ def walk_docket(
|
||||
advance the watermark on a clean walk, and auto-seal when
|
||||
:func:`should_seal` says so. ``force`` walks from page 1 and never
|
||||
seals. Returns per-status counts.
|
||||
|
||||
Attachments are fetched for created/updated comments, and for every
|
||||
comment under ``force`` (an unchanged row can still be missing its
|
||||
files). A docket is only auto-sealed when bib actually holds at
|
||||
least one of its comments.
|
||||
"""
|
||||
from bib.dockets import should_seal
|
||||
|
||||
@@ -970,7 +975,7 @@ def walk_docket(
|
||||
store, c, cms_id=docket.rule_cms_id, extra_tags=[f"reg-docket:{docket.id}"]
|
||||
)
|
||||
setattr(res, status, getattr(res, status) + 1)
|
||||
if attachments and c.attachment_count and status != "unchanged":
|
||||
if attachments and c.attachment_count and (force or status != "unchanged"):
|
||||
for att in client.attachments_for(c.id):
|
||||
path = client.download_attachment(att.url, scratch / c.id)
|
||||
if path:
|
||||
@@ -1000,10 +1005,13 @@ def walk_docket(
|
||||
docket, pull_watermark=max_lm, last_pull_at=now, last_pull_new=res.created
|
||||
)
|
||||
store.docket_upsert(updated)
|
||||
if should_seal(updated, today or date.today(), quiet_days):
|
||||
store.docket_seal(
|
||||
docket.id, reason="auto", counts=docket_counts(store, docket.id)
|
||||
)
|
||||
counts = docket_counts(store, docket.id)
|
||||
# A docket with nothing in bib is a farm that never landed, not a
|
||||
# finished one: an empty walk over it must not seal it shut.
|
||||
if counts["comments"] and should_seal(
|
||||
updated, today or date.today(), quiet_days
|
||||
):
|
||||
store.docket_seal(docket.id, reason="auto", counts=counts)
|
||||
res.sealed = True
|
||||
return res
|
||||
|
||||
|
||||
@@ -1,12 +1,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import date
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import httpx
|
||||
|
||||
from bib.dockets import Docket
|
||||
from bib.regulations_gov import Comment, WalkResult, discover_docket, walk_docket
|
||||
from bib.regulations_gov import (
|
||||
Attachment,
|
||||
Comment,
|
||||
WalkResult,
|
||||
discover_docket,
|
||||
walk_docket,
|
||||
)
|
||||
from bib.store import Store
|
||||
|
||||
D = "CMS-2026-2377"
|
||||
@@ -129,11 +136,54 @@ def test_auto_seal_after_quiet_empty_pull():
|
||||
s = _store()
|
||||
s.docket_upsert(_docket())
|
||||
api = MagicMock()
|
||||
api.iter_comments.return_value = [_c(1, "2026-09-01T00:00:00Z")]
|
||||
walk_docket(s, api, s.docket_get(D), today=date(2026, 9, 8))
|
||||
api.iter_comments.return_value = []
|
||||
r = walk_docket(s, api, s.docket_get(D), today=date(2026, 10, 20), quiet_days=30)
|
||||
assert r.sealed is True
|
||||
d = s.docket_get(D)
|
||||
assert d.sealed and d.seal_reason == "auto"
|
||||
assert json.loads(d.counts_json)["comments"] == 1
|
||||
|
||||
|
||||
def test_no_auto_seal_when_the_docket_has_no_stored_comments():
|
||||
"""An empty walk over a docket bib never ingested is a failed farm,
|
||||
not a finished one — sealing it would freeze zero comments forever."""
|
||||
s = _store()
|
||||
s.docket_upsert(_docket())
|
||||
api = MagicMock()
|
||||
api.iter_comments.return_value = []
|
||||
r = walk_docket(s, api, s.docket_get(D), today=date(2026, 10, 20), quiet_days=30)
|
||||
assert r.sealed is False
|
||||
assert not s.docket_get(D).sealed
|
||||
|
||||
|
||||
def test_attachments_fetched_for_new_comments_only_unless_forced(tmp_path):
|
||||
s = _store()
|
||||
s.docket_upsert(_docket())
|
||||
att = Attachment(url="https://cdn/att.pdf", filename="att.pdf")
|
||||
blob = tmp_path / "att.pdf"
|
||||
blob.write_bytes(b"%PDF-1.4")
|
||||
api = MagicMock()
|
||||
api.attachments_for.return_value = [att]
|
||||
api.download_attachment.return_value = blob
|
||||
c = _c(1, "2026-09-01T00:00:00Z")
|
||||
c.attachment_count = 1
|
||||
|
||||
api.iter_comments.return_value = [c]
|
||||
walk_docket(s, api, s.docket_get(D), attachments=True, today=date(2026, 9, 8))
|
||||
assert api.download_attachment.call_count == 1 # created → fetched
|
||||
|
||||
api.iter_comments.return_value = [c]
|
||||
r = walk_docket(s, api, s.docket_get(D), attachments=True, today=date(2026, 9, 8))
|
||||
assert r.unchanged == 1
|
||||
assert api.download_attachment.call_count == 1 # unchanged → not re-fetched
|
||||
|
||||
api.iter_comments.return_value = [c]
|
||||
walk_docket(
|
||||
s, api, s.docket_get(D), attachments=True, force=True, today=date(2026, 9, 8)
|
||||
)
|
||||
assert api.download_attachment.call_count == 2 # --force re-fetches
|
||||
|
||||
|
||||
def test_progress_echo_and_commit_every_50():
|
||||
|
||||
Reference in New Issue
Block a user