fix(bib): walk_docket never seals an empty docket; --force re-fetches attachments (refs #615)
An empty walk over a docket bib never ingested (a farm that never landed) satisfied should_seal and sealed it shut with zero comments. Read docket_counts before sealing and require at least one stored comment; the counts read is reused for the seal row. --force also skipped attachments for every comment whose row came back "unchanged", so the one flag meant to repair a docket could not repair its missing files.
This commit is contained in:
@@ -950,6 +950,11 @@ def walk_docket(
|
|||||||
advance the watermark on a clean walk, and auto-seal when
|
advance the watermark on a clean walk, and auto-seal when
|
||||||
:func:`should_seal` says so. ``force`` walks from page 1 and never
|
:func:`should_seal` says so. ``force`` walks from page 1 and never
|
||||||
seals. Returns per-status counts.
|
seals. Returns per-status counts.
|
||||||
|
|
||||||
|
Attachments are fetched for created/updated comments, and for every
|
||||||
|
comment under ``force`` (an unchanged row can still be missing its
|
||||||
|
files). A docket is only auto-sealed when bib actually holds at
|
||||||
|
least one of its comments.
|
||||||
"""
|
"""
|
||||||
from bib.dockets import should_seal
|
from bib.dockets import should_seal
|
||||||
|
|
||||||
@@ -970,7 +975,7 @@ def walk_docket(
|
|||||||
store, c, cms_id=docket.rule_cms_id, extra_tags=[f"reg-docket:{docket.id}"]
|
store, c, cms_id=docket.rule_cms_id, extra_tags=[f"reg-docket:{docket.id}"]
|
||||||
)
|
)
|
||||||
setattr(res, status, getattr(res, status) + 1)
|
setattr(res, status, getattr(res, status) + 1)
|
||||||
if attachments and c.attachment_count and status != "unchanged":
|
if attachments and c.attachment_count and (force or status != "unchanged"):
|
||||||
for att in client.attachments_for(c.id):
|
for att in client.attachments_for(c.id):
|
||||||
path = client.download_attachment(att.url, scratch / c.id)
|
path = client.download_attachment(att.url, scratch / c.id)
|
||||||
if path:
|
if path:
|
||||||
@@ -1000,10 +1005,13 @@ def walk_docket(
|
|||||||
docket, pull_watermark=max_lm, last_pull_at=now, last_pull_new=res.created
|
docket, pull_watermark=max_lm, last_pull_at=now, last_pull_new=res.created
|
||||||
)
|
)
|
||||||
store.docket_upsert(updated)
|
store.docket_upsert(updated)
|
||||||
if should_seal(updated, today or date.today(), quiet_days):
|
counts = docket_counts(store, docket.id)
|
||||||
store.docket_seal(
|
# A docket with nothing in bib is a farm that never landed, not a
|
||||||
docket.id, reason="auto", counts=docket_counts(store, docket.id)
|
# finished one: an empty walk over it must not seal it shut.
|
||||||
)
|
if counts["comments"] and should_seal(
|
||||||
|
updated, today or date.today(), quiet_days
|
||||||
|
):
|
||||||
|
store.docket_seal(docket.id, reason="auto", counts=counts)
|
||||||
res.sealed = True
|
res.sealed = True
|
||||||
return res
|
return res
|
||||||
|
|
||||||
|
|||||||
@@ -1,12 +1,19 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
from datetime import date
|
from datetime import date
|
||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
from bib.dockets import Docket
|
from bib.dockets import Docket
|
||||||
from bib.regulations_gov import Comment, WalkResult, discover_docket, walk_docket
|
from bib.regulations_gov import (
|
||||||
|
Attachment,
|
||||||
|
Comment,
|
||||||
|
WalkResult,
|
||||||
|
discover_docket,
|
||||||
|
walk_docket,
|
||||||
|
)
|
||||||
from bib.store import Store
|
from bib.store import Store
|
||||||
|
|
||||||
D = "CMS-2026-2377"
|
D = "CMS-2026-2377"
|
||||||
@@ -129,11 +136,54 @@ def test_auto_seal_after_quiet_empty_pull():
|
|||||||
s = _store()
|
s = _store()
|
||||||
s.docket_upsert(_docket())
|
s.docket_upsert(_docket())
|
||||||
api = MagicMock()
|
api = MagicMock()
|
||||||
|
api.iter_comments.return_value = [_c(1, "2026-09-01T00:00:00Z")]
|
||||||
|
walk_docket(s, api, s.docket_get(D), today=date(2026, 9, 8))
|
||||||
api.iter_comments.return_value = []
|
api.iter_comments.return_value = []
|
||||||
r = walk_docket(s, api, s.docket_get(D), today=date(2026, 10, 20), quiet_days=30)
|
r = walk_docket(s, api, s.docket_get(D), today=date(2026, 10, 20), quiet_days=30)
|
||||||
assert r.sealed is True
|
assert r.sealed is True
|
||||||
d = s.docket_get(D)
|
d = s.docket_get(D)
|
||||||
assert d.sealed and d.seal_reason == "auto"
|
assert d.sealed and d.seal_reason == "auto"
|
||||||
|
assert json.loads(d.counts_json)["comments"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_auto_seal_when_the_docket_has_no_stored_comments():
|
||||||
|
"""An empty walk over a docket bib never ingested is a failed farm,
|
||||||
|
not a finished one — sealing it would freeze zero comments forever."""
|
||||||
|
s = _store()
|
||||||
|
s.docket_upsert(_docket())
|
||||||
|
api = MagicMock()
|
||||||
|
api.iter_comments.return_value = []
|
||||||
|
r = walk_docket(s, api, s.docket_get(D), today=date(2026, 10, 20), quiet_days=30)
|
||||||
|
assert r.sealed is False
|
||||||
|
assert not s.docket_get(D).sealed
|
||||||
|
|
||||||
|
|
||||||
|
def test_attachments_fetched_for_new_comments_only_unless_forced(tmp_path):
|
||||||
|
s = _store()
|
||||||
|
s.docket_upsert(_docket())
|
||||||
|
att = Attachment(url="https://cdn/att.pdf", filename="att.pdf")
|
||||||
|
blob = tmp_path / "att.pdf"
|
||||||
|
blob.write_bytes(b"%PDF-1.4")
|
||||||
|
api = MagicMock()
|
||||||
|
api.attachments_for.return_value = [att]
|
||||||
|
api.download_attachment.return_value = blob
|
||||||
|
c = _c(1, "2026-09-01T00:00:00Z")
|
||||||
|
c.attachment_count = 1
|
||||||
|
|
||||||
|
api.iter_comments.return_value = [c]
|
||||||
|
walk_docket(s, api, s.docket_get(D), attachments=True, today=date(2026, 9, 8))
|
||||||
|
assert api.download_attachment.call_count == 1 # created → fetched
|
||||||
|
|
||||||
|
api.iter_comments.return_value = [c]
|
||||||
|
r = walk_docket(s, api, s.docket_get(D), attachments=True, today=date(2026, 9, 8))
|
||||||
|
assert r.unchanged == 1
|
||||||
|
assert api.download_attachment.call_count == 1 # unchanged → not re-fetched
|
||||||
|
|
||||||
|
api.iter_comments.return_value = [c]
|
||||||
|
walk_docket(
|
||||||
|
s, api, s.docket_get(D), attachments=True, force=True, today=date(2026, 9, 8)
|
||||||
|
)
|
||||||
|
assert api.download_attachment.call_count == 2 # --force re-fetches
|
||||||
|
|
||||||
|
|
||||||
def test_progress_echo_and_commit_every_50():
|
def test_progress_echo_and_commit_every_50():
|
||||||
|
|||||||
Reference in New Issue
Block a user