b5e07e67b2
The PFS register-and-walk path now serves the annual IPPS/LTCH and OPPS/ASC rules and DMEPOS payment rules: title-filtered FR discovery (discover-*-rules) and docket registration + comment walks (fetch-*-comments). Correction documents (C1-...) parse, a null FR abstract no longer becomes 'None', and the store waits out the mail poller's write bursts (busy_timeout) instead of failing SQLITE_BUSY.
404 lines
12 KiB
Python
404 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import date
|
|
from unittest.mock import MagicMock
|
|
|
|
import httpx
|
|
|
|
from bib.dockets import Docket
|
|
from bib.regulations_gov import (
|
|
Attachment,
|
|
Comment,
|
|
WalkResult,
|
|
discover_docket,
|
|
walk_docket,
|
|
)
|
|
from bib.store import Store
|
|
|
|
D = "CMS-2026-2377"
|
|
|
|
|
|
def _c(n: int, lm: str) -> Comment:
|
|
return Comment(
|
|
id=f"{D}-{n}",
|
|
title="",
|
|
posted_date=lm[:10],
|
|
received_date=lm[:10],
|
|
docket_id=D,
|
|
comment_on_id="x",
|
|
raw={"attributes": {"lastModifiedDate": lm}},
|
|
)
|
|
|
|
|
|
def _store() -> Store:
|
|
return Store(":memory:", storage_dir="/tmp/nope")
|
|
|
|
|
|
def _docket(**kw) -> Docket:
|
|
base = dict(
|
|
id=D,
|
|
rule_cms_id="CMS-1848-P",
|
|
fr_object_id="obj",
|
|
comment_end_date="2026-09-14",
|
|
)
|
|
base.update(kw)
|
|
return Docket(**base)
|
|
|
|
|
|
def test_discover_docket_picks_commentable_doc():
|
|
api = MagicMock()
|
|
api.find_documents_in_docket.return_value = [
|
|
{"id": "X-1", "attributes": {"objectId": "o1"}}, # no comment window
|
|
{
|
|
"id": "X-2",
|
|
"attributes": {"objectId": "o2", "commentEndDate": "2026-09-14T03:59:59Z"},
|
|
},
|
|
]
|
|
d = discover_docket(api, D, rule_cms_id="CMS-1848-P")
|
|
assert d == Docket(
|
|
id=D,
|
|
rule_cms_id="CMS-1848-P",
|
|
fr_document_id="X-2",
|
|
fr_object_id="o2",
|
|
comment_end_date="2026-09-14",
|
|
)
|
|
api.find_documents_in_docket.assert_called_once_with(D)
|
|
|
|
|
|
def test_discover_docket_prefers_proposed_rule():
|
|
api = MagicMock()
|
|
api.find_documents_in_docket.return_value = [
|
|
{
|
|
"id": "CMS-2019-0111-41991",
|
|
"attributes": {
|
|
"objectId": "o-rule",
|
|
"commentEndDate": "2020-01-03T03:59:59Z",
|
|
"documentType": "Rule",
|
|
},
|
|
},
|
|
{
|
|
"id": "CMS-2019-0111-0092",
|
|
"attributes": {
|
|
"objectId": "o-nprm",
|
|
"commentEndDate": "2019-09-28T03:59:59Z",
|
|
"documentType": "Proposed Rule",
|
|
},
|
|
},
|
|
{
|
|
"id": "CMS-2019-0111-0001",
|
|
"attributes": {
|
|
"objectId": "o-display",
|
|
"commentEndDate": "2019-08-14T03:59:59Z",
|
|
"documentType": "Proposed Rule",
|
|
},
|
|
},
|
|
]
|
|
d = discover_docket(api, "CMS-2019-0111", rule_cms_id="CMS-1738-P")
|
|
assert d == Docket(
|
|
id="CMS-2019-0111",
|
|
rule_cms_id="CMS-1738-P",
|
|
fr_document_id="CMS-2019-0111-0092",
|
|
fr_object_id="o-nprm",
|
|
comment_end_date="2019-09-28",
|
|
)
|
|
|
|
|
|
def test_discover_title_picks_the_document_comments_were_filed_on():
|
|
api = MagicMock()
|
|
api.find_documents_in_docket.return_value = [
|
|
{
|
|
"id": "late",
|
|
"attributes": {
|
|
"objectId": "small",
|
|
"commentEndDate": "2024-11-28T04:59:59Z",
|
|
"documentType": "Proposed Rule",
|
|
"title": "Hospital Outpatient Prospective Payment extension",
|
|
},
|
|
},
|
|
{
|
|
"id": "main",
|
|
"attributes": {
|
|
"objectId": "big",
|
|
"commentEndDate": "2024-09-10T03:59:59Z",
|
|
"documentType": "Proposed Rule",
|
|
"title": "Hospital Outpatient Prospective Payment and ASC",
|
|
},
|
|
},
|
|
]
|
|
|
|
def _get(_path, **params):
|
|
obj = params["filter[commentOnId]"]
|
|
return {"meta": {"totalElements": 7 if obj == "small" else 2483}}
|
|
|
|
api._get.side_effect = _get
|
|
d = discover_docket(
|
|
api,
|
|
"CMS-2024-0199",
|
|
rule_cms_id="CMS-1809-P",
|
|
title_contains="outpatient prospective payment",
|
|
)
|
|
assert d.fr_object_id == "big"
|
|
assert d.comment_end_date == "2024-09-10"
|
|
|
|
|
|
def test_discover_title_joins_a_line_wrapped_nprm():
|
|
api = MagicMock()
|
|
api.find_documents_in_docket.return_value = [
|
|
{
|
|
"id": "display",
|
|
"attributes": {
|
|
"objectId": "small",
|
|
"commentEndDate": "2020-05-30T03:59:59Z",
|
|
"documentType": "Proposed Rule",
|
|
"title": "Hospital Inpatient Prospective Payment Systems display",
|
|
},
|
|
},
|
|
{
|
|
"id": "wrapped",
|
|
"attributes": {
|
|
"objectId": "big",
|
|
"commentEndDate": "2020-07-11T03:59:59Z",
|
|
"documentType": "Proposed Rule",
|
|
"title": "Hospital Inpatient\nProspective Payment Systems for Acute Care",
|
|
},
|
|
},
|
|
]
|
|
|
|
def _get(_path, **params):
|
|
obj = params["filter[commentOnId]"]
|
|
return {"meta": {"totalElements": 25 if obj == "small" else 832}}
|
|
|
|
api._get.side_effect = _get
|
|
d = discover_docket(
|
|
api,
|
|
"CMS-2020-0052",
|
|
rule_cms_id="CMS-1735-P",
|
|
title_contains="inpatient prospective payment",
|
|
)
|
|
assert d.fr_object_id == "big"
|
|
assert d.comment_end_date == "2020-07-11"
|
|
|
|
|
|
def test_discover_same_close_date_picks_the_comment_pile():
|
|
api = MagicMock()
|
|
api.find_documents_in_docket.return_value = [
|
|
{
|
|
"id": "display",
|
|
"attributes": {
|
|
"objectId": "small",
|
|
"commentEndDate": "2023-08-30T03:59:59Z",
|
|
"documentType": "Proposed Rule",
|
|
"title": "Home Health display",
|
|
},
|
|
},
|
|
{
|
|
"id": "main",
|
|
"attributes": {
|
|
"objectId": "big",
|
|
"commentEndDate": "2023-08-30T03:59:59Z",
|
|
"documentType": "Proposed Rule",
|
|
"title": "Home Health Prospective Payment System",
|
|
},
|
|
},
|
|
]
|
|
|
|
def _get(_path, **params):
|
|
obj = params["filter[commentOnId]"]
|
|
return {"meta": {"totalElements": 150 if obj == "small" else 750}}
|
|
|
|
api._get.side_effect = _get
|
|
d = discover_docket(api, "CMS-2023-0113", rule_cms_id="CMS-1780-P")
|
|
assert d.fr_object_id == "big"
|
|
assert d.fr_document_id == "main"
|
|
|
|
|
|
def test_pull_titled_siblings_leaves_the_docket_row():
|
|
from bib.regulations_gov import Comment, pull_titled_siblings
|
|
|
|
s = _store()
|
|
s.docket_upsert(_docket(fr_object_id="primary"))
|
|
api = MagicMock()
|
|
api.find_documents_in_docket.return_value = [
|
|
{
|
|
"id": "main",
|
|
"attributes": {
|
|
"objectId": "primary",
|
|
"documentType": "Proposed Rule",
|
|
"title": "Inpatient Prospective Payment Systems",
|
|
},
|
|
},
|
|
{
|
|
"id": "display",
|
|
"attributes": {
|
|
"objectId": "other",
|
|
"documentType": "Proposed Rule",
|
|
"title": "Inpatient Prospective Payment display",
|
|
},
|
|
},
|
|
]
|
|
api.iter_comments.return_value = [
|
|
Comment(
|
|
id=f"{D}-9",
|
|
title="",
|
|
posted_date="2018-06-01",
|
|
received_date="2018-06-01",
|
|
docket_id=D,
|
|
comment_on_id="other",
|
|
comment_text="display copy",
|
|
)
|
|
]
|
|
n = pull_titled_siblings(
|
|
s, api, s.docket_get(D), title_contains="inpatient prospective payment"
|
|
)
|
|
assert n == 1
|
|
saved = s.docket_get(D)
|
|
assert saved is not None and saved.fr_object_id == "primary"
|
|
assert s._con().execute("SELECT count(*) FROM items").fetchone()[0] == 1
|
|
|
|
|
|
def test_walk_creates_counts_and_advances_watermark():
|
|
s = _store()
|
|
s.docket_upsert(_docket())
|
|
api = MagicMock()
|
|
api.iter_comments.return_value = [
|
|
_c(1, "2026-09-01T00:00:00Z"),
|
|
_c(2, "2026-09-02T00:00:00Z"),
|
|
]
|
|
r = walk_docket(s, api, s.docket_get(D), today=date(2026, 9, 8))
|
|
assert r == WalkResult(
|
|
created=2,
|
|
updated=0,
|
|
unchanged=0,
|
|
watermark="2026-09-02T00:00:00Z",
|
|
clean=True,
|
|
sealed=False,
|
|
)
|
|
d = s.docket_get(D)
|
|
assert d.pull_watermark == "2026-09-02T00:00:00Z"
|
|
assert d.last_pull_new == 2 and d.last_pull_at
|
|
api.iter_comments.assert_called_once()
|
|
assert api.iter_comments.call_args.kwargs["since"] == ""
|
|
|
|
|
|
def test_walk_passes_watermark_and_counts_unchanged():
|
|
s = _store()
|
|
s.docket_upsert(_docket())
|
|
api = MagicMock()
|
|
api.iter_comments.return_value = [_c(1, "2026-09-01T00:00:00Z")]
|
|
walk_docket(s, api, s.docket_get(D), today=date(2026, 9, 8))
|
|
api.iter_comments.return_value = [
|
|
_c(1, "2026-09-01T00:00:00Z")
|
|
] # boundary re-yield
|
|
r = walk_docket(s, api, s.docket_get(D), today=date(2026, 9, 8))
|
|
assert api.iter_comments.call_args.kwargs["since"] == "2026-09-01T00:00:00Z"
|
|
assert (r.created, r.unchanged) == (0, 1)
|
|
assert s.docket_get(D).last_pull_new == 0
|
|
|
|
|
|
def test_unclean_walk_keeps_old_watermark():
|
|
s = _store()
|
|
s.docket_upsert(_docket(pull_watermark="2026-08-01T00:00:00Z"))
|
|
api = MagicMock()
|
|
|
|
def _iter(_obj, *, since="", on_error=None):
|
|
yield _c(9, "2026-09-05T00:00:00Z")
|
|
on_error(
|
|
httpx.HTTPStatusError("boom", request=MagicMock(), response=MagicMock())
|
|
)
|
|
|
|
api.iter_comments.side_effect = _iter
|
|
r = walk_docket(s, api, s.docket_get(D), today=date(2026, 9, 8))
|
|
assert r.clean is False and r.created == 1
|
|
assert s.docket_get(D).pull_watermark == "2026-08-01T00:00:00Z"
|
|
|
|
|
|
def test_force_walks_from_scratch_and_never_seals():
|
|
s = _store()
|
|
s.docket_upsert(_docket(pull_watermark="2026-08-01T00:00:00Z"))
|
|
api = MagicMock()
|
|
api.iter_comments.return_value = []
|
|
r = walk_docket(s, api, s.docket_get(D), force=True, today=date(2027, 1, 1))
|
|
assert api.iter_comments.call_args.kwargs["since"] == ""
|
|
assert r.sealed is False and not s.docket_get(D).sealed
|
|
|
|
|
|
def test_auto_seal_after_quiet_empty_pull():
|
|
s = _store()
|
|
s.docket_upsert(_docket())
|
|
api = MagicMock()
|
|
api.iter_comments.return_value = [_c(1, "2026-09-01T00:00:00Z")]
|
|
walk_docket(s, api, s.docket_get(D), today=date(2026, 9, 8))
|
|
api.iter_comments.return_value = []
|
|
r = walk_docket(s, api, s.docket_get(D), today=date(2026, 10, 20), quiet_days=30)
|
|
assert r.sealed is True
|
|
d = s.docket_get(D)
|
|
assert d.sealed and d.seal_reason == "auto"
|
|
assert json.loads(d.counts_json)["comments"] == 1
|
|
|
|
|
|
def test_no_auto_seal_when_the_docket_has_no_stored_comments():
|
|
"""An empty walk over a docket bib never ingested is a failed farm,
|
|
not a finished one — sealing it would freeze zero comments forever."""
|
|
s = _store()
|
|
s.docket_upsert(_docket())
|
|
api = MagicMock()
|
|
api.iter_comments.return_value = []
|
|
r = walk_docket(s, api, s.docket_get(D), today=date(2026, 10, 20), quiet_days=30)
|
|
assert r.sealed is False
|
|
assert not s.docket_get(D).sealed
|
|
|
|
|
|
def test_attachments_fetched_for_new_comments_only_unless_forced(tmp_path):
|
|
s = _store()
|
|
s.docket_upsert(_docket())
|
|
att = Attachment(url="https://cdn/att.pdf", filename="att.pdf")
|
|
blob = tmp_path / "att.pdf"
|
|
blob.write_bytes(b"%PDF-1.4")
|
|
api = MagicMock()
|
|
api.attachments_for.return_value = [att]
|
|
api.download_attachment.return_value = blob
|
|
c = _c(1, "2026-09-01T00:00:00Z")
|
|
c.attachment_count = 1
|
|
|
|
api.iter_comments.return_value = [c]
|
|
walk_docket(s, api, s.docket_get(D), attachments=True, today=date(2026, 9, 8))
|
|
assert api.download_attachment.call_count == 1 # created → fetched
|
|
|
|
api.iter_comments.return_value = [c]
|
|
r = walk_docket(s, api, s.docket_get(D), attachments=True, today=date(2026, 9, 8))
|
|
assert r.unchanged == 1
|
|
assert api.download_attachment.call_count == 1 # unchanged → not re-fetched
|
|
|
|
api.iter_comments.return_value = [c]
|
|
walk_docket(
|
|
s, api, s.docket_get(D), attachments=True, force=True, today=date(2026, 9, 8)
|
|
)
|
|
assert api.download_attachment.call_count == 2 # --force re-fetches
|
|
|
|
|
|
def test_progress_echo_and_commit_every_50():
|
|
s = _store()
|
|
s.docket_upsert(_docket())
|
|
api = MagicMock()
|
|
api.iter_comments.return_value = [_c(i, "2026-09-01T00:00:00Z") for i in range(101)]
|
|
echoed: list[str] = []
|
|
r = walk_docket(s, api, s.docket_get(D), echo=echoed.append, today=date(2026, 9, 8))
|
|
assert r.created == 101
|
|
assert " 50 comments" in echoed
|
|
assert " 100 comments" in echoed
|
|
|
|
|
|
def test_limit_stops_early_and_leaves_watermark_unadvanced():
|
|
s = _store()
|
|
s.docket_upsert(_docket())
|
|
api = MagicMock()
|
|
api.iter_comments.return_value = [
|
|
_c(1, "2026-09-01T00:00:00Z"),
|
|
_c(2, "2026-09-02T00:00:00Z"),
|
|
]
|
|
r = walk_docket(s, api, s.docket_get(D), limit=1, today=date(2026, 9, 8))
|
|
assert r.created == 1
|
|
assert r.clean is False
|
|
assert s.docket_get(D).pull_watermark == ""
|