feat(bib): Docket model, should_seal, fingerprint_files, seal_quiet_days knob (refs #615)

src/bib/dockets.py

@@ -0,0 +1,86 @@
+"""Docket-level state for the regulations.gov comment pipeline.
+
+A *docket* row (``dockets`` table in bib.sqlite) remembers what every
+stage would otherwise re-derive from the network: the reg.gov document
+that carries the comments, the comment close date, the pull watermark,
+and — once the docket is known complete — a *seal*. Sealed dockets are
+skipped by fetch, backfill, extract and index unless ``--force``.
+
+Pure helpers only; the store owns persistence (``Store.docket_*``).
+"""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+from datetime import date, timedelta
+from pathlib import Path
+from typing import Iterable
+
+DEFAULT_QUIET_DAYS = 30
+
+
+@dataclass(frozen=True)
+class Docket:
+    id: str
+    rule_cms_id: str = ""
+    fr_document_id: str = ""
+    fr_object_id: str = ""
+    comment_end_date: str = ""  # YYYY-MM-DD
+    pull_watermark: str = ""  # max lastModifiedDate seen on a clean walk
+    last_pull_at: str = ""  # ISO-8601 UTC
+    last_pull_new: int | None = None
+    sealed_at: str = ""
+    seal_reason: str = ""
+    counts_json: str = "{}"
+
+    @property
+    def sealed(self) -> bool:
+        return bool(self.sealed_at)
+
+
+def should_seal(docket: Docket, today: date, quiet_days: int) -> bool:
+    """True when the comment period closed ≥ *quiet_days* ago and the
+    most recent completed pull, itself after that quiet boundary, found
+    nothing new."""
+    if docket.sealed or not docket.comment_end_date or not docket.last_pull_at:
+        return False
+    if docket.last_pull_new is None or docket.last_pull_new > 0:
+        return False
+    try:
+        end = date.fromisoformat(docket.comment_end_date[:10])
+        pulled = date.fromisoformat(docket.last_pull_at[:10])
+    except ValueError:
+        return False
+    boundary = end + timedelta(days=quiet_days)
+    return today >= boundary and pulled >= boundary
+
+
+def fingerprint_files(paths: Iterable[Path]) -> str:
+    """sha256 over sorted ``(name, size, mtime_ns)`` — cheap change
+    detection without reading contents. Missing files contribute their
+    name with size -1 so a deletion changes the fingerprint too."""
+    rows: list[str] = []
+    for p in paths:
+        p = Path(p)
+        try:
+            st = p.stat()
+            rows.append(f"{p.name}\x00{st.st_size}\x00{st.st_mtime_ns}")
+        except FileNotFoundError:
+            rows.append(f"{p.name}\x00-1\x000")
+    rows.sort()
+    return hashlib.sha256("\n".join(rows).encode()).hexdigest()
+
+
+def _cfg():
+    from conf import cfg
+
+    return cfg
+
+
+def quiet_days() -> int:
+    """``[comments] seal_quiet_days`` from stack.toml, default 30."""
+    c = _cfg()
+    if "comments" in c and "seal_quiet_days" in c.comments:
+        return int(c.comments.seal_quiet_days)
+    return DEFAULT_QUIET_DAYS

stack.toml

@@ -79,6 +79,9 @@ aco = "data/aco.duckdb"
 bib = "data/bib.sqlite"
 zotero = "data/zotero/data/zotero.sqlite"
 
+[comments]
+seal_quiet_days = 30   # days after a docket's comment close date before an empty pull seals it
+
 [storage]
 bib = "data/bib/storage"
 zotero = "data/zotero/data/storage"

tests/bib/test_dockets.py

@@ -0,0 +1,115 @@
+"""bib.dockets — pure docket state helpers."""
+
+from __future__ import annotations
+
+import os
+from datetime import date
+from pathlib import Path
+
+from bib.dockets import Docket, fingerprint_files, quiet_days, should_seal
+
+
+def _d(**kw) -> Docket:
+    base = dict(
+        id="CMS-2026-2377",
+        rule_cms_id="CMS-1848-P",
+        fr_document_id="CMS-2026-2377-0001",
+        fr_object_id="0900006482921ba1",
+        comment_end_date="2026-09-14",
+        pull_watermark="",
+        last_pull_at="",
+        last_pull_new=None,
+        sealed_at="",
+        seal_reason="",
+        counts_json="{}",
+    )
+    base.update(kw)
+    return Docket(**base)
+
+
+class TestShouldSeal:
+    def test_seals_after_quiet_period_with_no_new(self):
+        d = _d(last_pull_at="2026-10-15T03:00:00Z", last_pull_new=0)
+        assert should_seal(d, date(2026, 10, 15), 30) is True
+
+    def test_not_before_quiet_period(self):
+        d = _d(last_pull_at="2026-10-13T03:00:00Z", last_pull_new=0)
+        assert should_seal(d, date(2026, 10, 13), 30) is False
+
+    def test_not_when_last_pull_found_new(self):
+        d = _d(last_pull_at="2026-10-20T03:00:00Z", last_pull_new=3)
+        assert should_seal(d, date(2026, 10, 20), 30) is False
+
+    def test_not_when_last_pull_predates_quiet_period(self):
+        # Pull happened before end+quiet even though today is well past it.
+        d = _d(last_pull_at="2026-09-20T03:00:00Z", last_pull_new=0)
+        assert should_seal(d, date(2026, 12, 1), 30) is False
+
+    def test_not_without_close_date(self):
+        d = _d(
+            comment_end_date="", last_pull_at="2027-01-01T00:00:00Z", last_pull_new=0
+        )
+        assert should_seal(d, date(2027, 1, 1), 30) is False
+
+    def test_not_when_already_sealed(self):
+        d = _d(
+            sealed_at="2026-10-15T00:00:00Z",
+            last_pull_at="2026-10-15T03:00:00Z",
+            last_pull_new=0,
+        )
+        assert should_seal(d, date(2026, 10, 15), 30) is False
+
+    def test_not_when_never_pulled(self):
+        d = _d(last_pull_at="", last_pull_new=None)
+        assert should_seal(d, date(2027, 1, 1), 30) is False
+
+
+class TestFingerprintFiles:
+    def test_order_independent(self, tmp_path: Path):
+        a = tmp_path / "a.pdf"
+        b = tmp_path / "b.pdf"
+        a.write_bytes(b"aa")
+        b.write_bytes(b"bbb")
+        assert fingerprint_files([a, b]) == fingerprint_files([b, a])
+
+    def test_changes_when_size_changes(self, tmp_path: Path):
+        a = tmp_path / "a.pdf"
+        a.write_bytes(b"aa")
+        f1 = fingerprint_files([a])
+        a.write_bytes(b"aaaa")
+        assert fingerprint_files([a]) != f1
+
+    def test_changes_when_mtime_changes(self, tmp_path: Path):
+        a = tmp_path / "a.pdf"
+        a.write_bytes(b"aa")
+        f1 = fingerprint_files([a])
+        os.utime(a, ns=(1_000_000_000_000_000_000, 1_000_000_000_000_000_000))
+        assert fingerprint_files([a]) != f1
+
+    def test_missing_file_is_recorded_not_fatal(self, tmp_path: Path):
+        assert fingerprint_files([tmp_path / "nope.pdf"]) == fingerprint_files(
+            [tmp_path / "nope.pdf"]
+        )
+        assert fingerprint_files([tmp_path / "nope.pdf"]) != fingerprint_files([])
+
+    def test_empty_is_stable(self):
+        assert fingerprint_files([]) == fingerprint_files([])
+        assert len(fingerprint_files([])) == 64
+
+
+class TestQuietDays:
+    def test_default_when_section_missing(self, monkeypatch):
+        import bib.dockets as mod
+        from conf import _Cfg
+
+        monkeypatch.setattr(mod, "_cfg", lambda: _Cfg({}))
+        assert quiet_days() == 30
+
+    def test_reads_section(self, monkeypatch):
+        import bib.dockets as mod
+        from conf import _Cfg
+
+        monkeypatch.setattr(
+            mod, "_cfg", lambda: _Cfg({"comments": {"seal_quiet_days": 7}})
+        )
+        assert quiet_days() == 7