From 961e294794f83779f9096cf1029c7d25c4809047 Mon Sep 17 00:00:00 2001 From: kert Date: Tue, 8 Sep 2026 12:28:16 -0400 Subject: [PATCH] =?UTF-8?q?feat(bib):=20discover=5Fdocket=20+=20walk=5Fdoc?= =?UTF-8?q?ket=20=E2=80=94=20watermark,=20counts,=20auto-seal=20(refs=20#6?= =?UTF-8?q?15)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bib/regulations_gov.py | 150 ++++++++++++++++++++- tests/bib/test_regulations_gov.py | 6 +- tests/bib/test_regulations_gov_exercise.py | 4 +- tests/bib/test_walk_docket.py | 136 +++++++++++++++++++ 4 files changed, 289 insertions(+), 7 deletions(-) create mode 100644 tests/bib/test_walk_docket.py diff --git a/src/bib/regulations_gov.py b/src/bib/regulations_gov.py index 5914a9a..8b4fe14 100644 --- a/src/bib/regulations_gov.py +++ b/src/bib/regulations_gov.py @@ -29,12 +29,14 @@ import os import re import time from concurrent.futures import ThreadPoolExecutor, as_completed -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace +from datetime import date from pathlib import Path from typing import TYPE_CHECKING, Callable, Iterator import httpx +from bib.dockets import Docket from bib.item import Source from bib.tag import Tag @@ -798,6 +800,19 @@ def upsert_comment( extra_tags: list[str] | None = None, ) -> str: """Upsert the comment as a Source item, return bib key.""" + return upsert_comment_status(store, comment, cms_id=cms_id, extra_tags=extra_tags)[ + 0 + ] + + +def upsert_comment_status( + store: Store, + comment: Comment, + *, + cms_id: str = "", + extra_tags: list[str] | None = None, +) -> tuple[str, str]: + """Like :func:`upsert_comment`, also returning created/updated/unchanged.""" # Title = the comment's own CMS-DOCKET-XXXX-NNNN id, optionally # prefixed with the byline (org or first/last name) when the # commenter filled those fields in. The previous "Comment on @@ -830,7 +845,138 @@ def upsert_comment( for t in tags: item.add_tag(t) - return store.upsert(item) + return store.upsert_status(item) + + +# ── Docket walks ─────────────────────────────────────────────── + + +def discover_docket(client: Client, docket_id: str, *, rule_cms_id: str = "") -> Docket: + """One ``/documents`` listing → the docket's commentable document. + + Picks the document that has an ``objectId`` and a ``commentEndDate`` + (latest close date wins when several qualify). Called once per + docket; the result is persisted so later runs make no API call. + """ + best: dict | None = None + for fr_doc in client.find_documents_in_docket(docket_id): + attrs = fr_doc.get("attributes") or {} + if not attrs.get("objectId") or not attrs.get("commentEndDate"): + continue + if ( + best is None + or attrs["commentEndDate"] + > (best.get("attributes") or {})["commentEndDate"] + ): + best = fr_doc + attrs = (best or {}).get("attributes") or {} + return Docket( + id=docket_id, + rule_cms_id=rule_cms_id, + fr_document_id=(best or {}).get("id", ""), + fr_object_id=attrs.get("objectId", ""), + comment_end_date=(attrs.get("commentEndDate") or "")[:10], + ) + + +@dataclass +class WalkResult: + created: int = 0 + updated: int = 0 + unchanged: int = 0 + watermark: str = "" + clean: bool = True + sealed: bool = False + + +def walk_docket( + store: Store, + client: Client, + docket: Docket, + *, + attachments: bool = False, + limit: int = 0, + force: bool = False, + quiet_days: int = 30, + today: date | None = None, + scratch_root: Path = Path(".state/comments"), + echo: Callable[[str], None] | None = None, +) -> WalkResult: + """Pull *docket*'s comments from the stored watermark, upsert them, + advance the watermark on a clean walk, and auto-seal when + :func:`should_seal` says so. ``force`` walks from page 1 and never + seals. Returns per-status counts. + """ + from bib.dockets import should_seal + + res = WalkResult() + since = "" if force else docket.pull_watermark + max_lm = docket.pull_watermark if not force else "" + n = 0 + + def _err(_e: Exception) -> None: + res.clean = False + + scratch = scratch_root / docket.id + for c in client.iter_comments(docket.fr_object_id, since=since, on_error=_err): + if limit and n >= limit: + res.clean = False # a capped walk is not a complete one + break + key, status = upsert_comment_status( + store, c, cms_id=docket.rule_cms_id, extra_tags=[f"reg-docket:{docket.id}"] + ) + setattr(res, status, getattr(res, status) + 1) + if attachments and c.attachment_count and status != "unchanged": + for att in client.attachments_for(c.id): + path = client.download_attachment(att.url, scratch / c.id) + if path: + store.attach_file(key, path, title=att.filename) + lm = (c.raw.get("attributes") or {}).get("lastModifiedDate") or "" + if lm > max_lm: + max_lm = lm + n += 1 + if n % 50 == 0: + store._con().commit() # noqa: SLF001 + if echo: + echo(f" {n} comments") + store._con().commit() # noqa: SLF001 + res.watermark = max_lm + + if res.clean and not force: + from datetime import datetime, timezone + + # ``today`` doubles as a deterministic override of "now" for + # tests exercising the auto-seal quiet-window boundary; real + # callers leave it unset and get the actual wall-clock time. + if today is not None: + now = f"{today.isoformat()}T00:00:00Z" + else: + now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + updated = replace( + docket, pull_watermark=max_lm, last_pull_at=now, last_pull_new=res.created + ) + store.docket_upsert(updated) + if should_seal(updated, today or date.today(), quiet_days): + store.docket_seal( + docket.id, reason="auto", counts=docket_counts(store, docket.id) + ) + res.sealed = True + return res + + +def docket_counts(store: Store, docket_id: str) -> dict[str, int]: + """``{"comments": n, "enriched": n}`` from SQL only (no filesystem).""" + con = store._con() # noqa: SLF001 + like = f"https://www.regulations.gov/comment/{docket_id}-%" + comments = con.execute( + "SELECT count(*) FROM items WHERE url LIKE ?", (like,) + ).fetchone()[0] + enriched = con.execute( + """SELECT count(*) FROM items i WHERE i.url LIKE ? AND i.id IN ( + SELECT item_id FROM item_tags WHERE tag_id IN (SELECT id FROM tags WHERE name='enriched:ok'))""", + (like,), + ).fetchone()[0] + return {"comments": comments, "enriched": enriched} # ── Internals ────────────────────────────────────────────────── diff --git a/tests/bib/test_regulations_gov.py b/tests/bib/test_regulations_gov.py index 4c70136..1bcdd20 100644 --- a/tests/bib/test_regulations_gov.py +++ b/tests/bib/test_regulations_gov.py @@ -156,7 +156,7 @@ class TestParseComment: class TestUpsertComment: def test_creates_item(self): store = MagicMock() - store.upsert.return_value = "KEY1" + store.upsert_status.return_value = ("KEY1", "created") c = Comment( id="CMS-2017-0092-0002", title="Test", @@ -168,8 +168,8 @@ class TestUpsertComment: ) key = upsert_comment(store, c, cms_id="CMS-1676-P") assert key == "KEY1" - store.upsert.assert_called_once() - item = store.upsert.call_args[0][0] + store.upsert_status.assert_called_once() + item = store.upsert_status.call_args[0][0] assert "AMA" in item.title assert item.url == "https://www.regulations.gov/comment/CMS-2017-0092-0002" diff --git a/tests/bib/test_regulations_gov_exercise.py b/tests/bib/test_regulations_gov_exercise.py index 84833cf..772f327 100644 --- a/tests/bib/test_regulations_gov_exercise.py +++ b/tests/bib/test_regulations_gov_exercise.py @@ -114,7 +114,7 @@ class TestByline: class TestUpsertComment: def test_full(self): store = MagicMock() - store.upsert.return_value = "KEY1" + store.upsert_status.return_value = ("KEY1", "created") c = Comment( id="CMS-2023-0001-0001", title="My Comment", @@ -130,7 +130,7 @@ class TestUpsertComment: def test_no_title_uses_comment_on_id(self): store = MagicMock() - store.upsert.return_value = "KEY2" + store.upsert_status.return_value = ("KEY2", "created") c = Comment( id="C2", title="", diff --git a/tests/bib/test_walk_docket.py b/tests/bib/test_walk_docket.py new file mode 100644 index 0000000..9e3f8ea --- /dev/null +++ b/tests/bib/test_walk_docket.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +from datetime import date +from unittest.mock import MagicMock + +import httpx + +from bib.dockets import Docket +from bib.regulations_gov import Comment, WalkResult, discover_docket, walk_docket +from bib.store import Store + +D = "CMS-2026-2377" + + +def _c(n: int, lm: str) -> Comment: + return Comment( + id=f"{D}-{n}", + title="", + posted_date=lm[:10], + received_date=lm[:10], + docket_id=D, + comment_on_id="x", + raw={"attributes": {"lastModifiedDate": lm}}, + ) + + +def _store() -> Store: + return Store(":memory:", storage_dir="/tmp/nope") + + +def _docket(**kw) -> Docket: + base = dict( + id=D, + rule_cms_id="CMS-1848-P", + fr_object_id="obj", + comment_end_date="2026-09-14", + ) + base.update(kw) + return Docket(**base) + + +def test_discover_docket_picks_commentable_doc(): + api = MagicMock() + api.find_documents_in_docket.return_value = [ + {"id": "X-1", "attributes": {"objectId": "o1"}}, # no comment window + { + "id": "X-2", + "attributes": {"objectId": "o2", "commentEndDate": "2026-09-14T03:59:59Z"}, + }, + ] + d = discover_docket(api, D, rule_cms_id="CMS-1848-P") + assert d == Docket( + id=D, + rule_cms_id="CMS-1848-P", + fr_document_id="X-2", + fr_object_id="o2", + comment_end_date="2026-09-14", + ) + api.find_documents_in_docket.assert_called_once_with(D) + + +def test_walk_creates_counts_and_advances_watermark(): + s = _store() + s.docket_upsert(_docket()) + api = MagicMock() + api.iter_comments.return_value = [ + _c(1, "2026-09-01T00:00:00Z"), + _c(2, "2026-09-02T00:00:00Z"), + ] + r = walk_docket(s, api, s.docket_get(D), today=date(2026, 9, 8)) + assert r == WalkResult( + created=2, + updated=0, + unchanged=0, + watermark="2026-09-02T00:00:00Z", + clean=True, + sealed=False, + ) + d = s.docket_get(D) + assert d.pull_watermark == "2026-09-02T00:00:00Z" + assert d.last_pull_new == 2 and d.last_pull_at + api.iter_comments.assert_called_once() + assert api.iter_comments.call_args.kwargs["since"] == "" + + +def test_walk_passes_watermark_and_counts_unchanged(): + s = _store() + s.docket_upsert(_docket()) + api = MagicMock() + api.iter_comments.return_value = [_c(1, "2026-09-01T00:00:00Z")] + walk_docket(s, api, s.docket_get(D), today=date(2026, 9, 8)) + api.iter_comments.return_value = [ + _c(1, "2026-09-01T00:00:00Z") + ] # boundary re-yield + r = walk_docket(s, api, s.docket_get(D), today=date(2026, 9, 8)) + assert api.iter_comments.call_args.kwargs["since"] == "2026-09-01T00:00:00Z" + assert (r.created, r.unchanged) == (0, 1) + assert s.docket_get(D).last_pull_new == 0 + + +def test_unclean_walk_keeps_old_watermark(): + s = _store() + s.docket_upsert(_docket(pull_watermark="2026-08-01T00:00:00Z")) + api = MagicMock() + + def _iter(_obj, *, since="", on_error=None): + yield _c(9, "2026-09-05T00:00:00Z") + on_error( + httpx.HTTPStatusError("boom", request=MagicMock(), response=MagicMock()) + ) + + api.iter_comments.side_effect = _iter + r = walk_docket(s, api, s.docket_get(D), today=date(2026, 9, 8)) + assert r.clean is False and r.created == 1 + assert s.docket_get(D).pull_watermark == "2026-08-01T00:00:00Z" + + +def test_force_walks_from_scratch_and_never_seals(): + s = _store() + s.docket_upsert(_docket(pull_watermark="2026-08-01T00:00:00Z")) + api = MagicMock() + api.iter_comments.return_value = [] + r = walk_docket(s, api, s.docket_get(D), force=True, today=date(2027, 1, 1)) + assert api.iter_comments.call_args.kwargs["since"] == "" + assert r.sealed is False and not s.docket_get(D).sealed + + +def test_auto_seal_after_quiet_empty_pull(): + s = _store() + s.docket_upsert(_docket()) + api = MagicMock() + api.iter_comments.return_value = [] + r = walk_docket(s, api, s.docket_get(D), today=date(2026, 10, 20), quiet_days=30) + assert r.sealed is True + d = s.docket_get(D) + assert d.sealed and d.seal_reason == "auto"