b5e07e67b2
The PFS register-and-walk path now serves the annual IPPS/LTCH and OPPS/ASC rules and DMEPOS payment rules: title-filtered FR discovery (discover-*-rules) and docket registration + comment walks (fetch-*-comments). Correction documents (C1-...) parse, a null FR abstract no longer becomes 'None', and the store waits out the mail poller's write bursts (busy_timeout) instead of failing SQLITE_BUSY.
576 lines
22 KiB
Python
576 lines
22 KiB
Python
"""Tests for bib.translate — CMS documentation translators."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from bib.translate import (
|
|
_extract_4i_field,
|
|
_find_part_title,
|
|
_infer_page_type,
|
|
_parse_4i_date,
|
|
_parse_iom_url,
|
|
cms_manual,
|
|
cms_website,
|
|
ecfr,
|
|
federal_register,
|
|
four_i,
|
|
)
|
|
|
|
# ── _fetch / _fetch_json helpers ────────────────────────────────────
|
|
|
|
|
|
class TestFetchHelpers:
|
|
def test_fetch_uses_urlopen(self) -> None:
|
|
"""Lines 67-74: _fetch makes urllib request."""
|
|
mock_resp = MagicMock()
|
|
mock_resp.read.return_value = b"<html>hello</html>"
|
|
mock_resp.__enter__ = lambda s: s
|
|
mock_resp.__exit__ = MagicMock(return_value=False)
|
|
|
|
with patch("urllib.request.urlopen", return_value=mock_resp):
|
|
from bib.translate import _fetch
|
|
|
|
result = _fetch("https://example.com")
|
|
assert result == "<html>hello</html>"
|
|
|
|
def test_fetch_json_parses(self) -> None:
|
|
mock_resp = MagicMock()
|
|
mock_resp.read.return_value = b'{"key": "value"}'
|
|
mock_resp.__enter__ = lambda s: s
|
|
mock_resp.__exit__ = MagicMock(return_value=False)
|
|
|
|
with patch("urllib.request.urlopen", return_value=mock_resp):
|
|
from bib.translate import _fetch_json
|
|
|
|
result = _fetch_json("https://example.com")
|
|
assert result == {"key": "value"}
|
|
|
|
|
|
# ── federal_register ────────────────────────────────────────────────
|
|
|
|
|
|
class TestFederalRegister:
|
|
def _mock_fr_data(self, **overrides: object) -> dict:
|
|
base = {
|
|
"title": "Medicare Program; CY 2026 PFS Final Rule (CMS-1832-F)",
|
|
"volume": 90,
|
|
"start_page": 98452,
|
|
"publication_date": "2025-11-01",
|
|
"effective_on": "2026-01-01",
|
|
"html_url": "https://www.federalregister.gov/d/2025-19787",
|
|
"abstract": "This rule finalizes...",
|
|
"type": "Rule - Final",
|
|
"regulation_id_numbers": [],
|
|
}
|
|
base.update(overrides)
|
|
return base
|
|
|
|
def test_from_html_url(self) -> None:
|
|
url = (
|
|
"https://www.federalregister.gov/documents/2025/11/01/"
|
|
"2025-19787/medicare-program"
|
|
)
|
|
data = self._mock_fr_data()
|
|
with patch("bib.translate._fetch_json", return_value=data):
|
|
rule = federal_register(url)
|
|
assert rule.item_type == "rule"
|
|
assert rule.document_number == "2025-19787"
|
|
assert rule.fr_volume == "90"
|
|
assert rule.fr_page == "98452"
|
|
assert rule.cms_id == "CMS-1832-F"
|
|
assert rule.rule_type == "final"
|
|
assert "source:federal-register" in rule.tags
|
|
assert "module:pfs" in rule.tags
|
|
assert "year:2025" in rule.tags
|
|
assert "rule:cms-1832-f" in rule.tags
|
|
|
|
def test_correction_document_number(self) -> None:
|
|
url = (
|
|
"https://www.federalregister.gov/documents/2022/01/04/"
|
|
"C1-2021-27763/medicare-program"
|
|
)
|
|
data = self._mock_fr_data()
|
|
with patch("bib.translate._fetch_json", return_value=data) as fetch:
|
|
rule = federal_register(url)
|
|
assert fetch.call_args.args[0].endswith("/C1-2021-27763.json")
|
|
assert rule.document_number == "C1-2021-27763"
|
|
|
|
def test_missing_abstract_is_empty(self) -> None:
|
|
url = (
|
|
"https://www.federalregister.gov/documents/2022/01/04/"
|
|
"C1-2021-27763/medicare-program"
|
|
)
|
|
data = self._mock_fr_data(abstract=None)
|
|
with patch("bib.translate._fetch_json", return_value=data):
|
|
rule = federal_register(url)
|
|
assert rule.abstract == ""
|
|
|
|
def test_from_api_url(self) -> None:
|
|
url = "https://www.federalregister.gov/api/v1/documents/2025-19787.json"
|
|
data = self._mock_fr_data()
|
|
with patch("bib.translate._fetch_json", return_value=data):
|
|
rule = federal_register(url)
|
|
assert rule.document_number == "2025-19787"
|
|
|
|
def test_proposed_rule_type(self) -> None:
|
|
url = (
|
|
"https://www.federalregister.gov/documents/2025/07/01/"
|
|
"2025-12345/proposed-rule"
|
|
)
|
|
data = self._mock_fr_data(type="Rule - Proposed", title="Proposed Rule")
|
|
with patch("bib.translate._fetch_json", return_value=data):
|
|
rule = federal_register(url)
|
|
assert rule.rule_type == "proposed"
|
|
|
|
def test_regulation_id_fallback(self) -> None:
|
|
url = (
|
|
"https://www.federalregister.gov/documents/2025/07/01/2025-12345/some-rule"
|
|
)
|
|
data = self._mock_fr_data(
|
|
title="Some Rule Without CMS ID",
|
|
regulation_id_numbers=["RIN-0938-AU99"],
|
|
)
|
|
with patch("bib.translate._fetch_json", return_value=data):
|
|
rule = federal_register(url)
|
|
assert rule.cms_id == "RIN-0938-AU99"
|
|
|
|
def test_no_pub_year_no_year_tag(self) -> None:
|
|
url = (
|
|
"https://www.federalregister.gov/documents/2025/07/01/2025-12345/some-rule"
|
|
)
|
|
data = self._mock_fr_data(publication_date="", title="Test")
|
|
with patch("bib.translate._fetch_json", return_value=data):
|
|
rule = federal_register(url)
|
|
assert not any(t.startswith("year:") for t in rule.tags)
|
|
|
|
def test_no_doc_number(self) -> None:
|
|
url = "https://www.federalregister.gov/other/path"
|
|
data = self._mock_fr_data(title="Test")
|
|
with patch("bib.translate._fetch_json", return_value=data):
|
|
rule = federal_register(url)
|
|
assert rule.document_number == ""
|
|
|
|
def test_other_doc_type(self) -> None:
|
|
url = "https://www.federalregister.gov/documents/2025/07/01/2025-12345/notice"
|
|
data = self._mock_fr_data(type="Notice", title="A Notice")
|
|
with patch("bib.translate._fetch_json", return_value=data):
|
|
rule = federal_register(url)
|
|
assert rule.rule_type == "notice"
|
|
|
|
|
|
# ── cms_website ─────────────────────────────────────────────────────
|
|
|
|
|
|
class TestCmsWebsite:
|
|
def test_basic_download_page(self) -> None:
|
|
html = """
|
|
<html>
|
|
<title>RVU26A | CMS</title>
|
|
<body>
|
|
<a href="/files/rvu26a.zip">Download</a>
|
|
<a href="/files/data.csv">CSV</a>
|
|
</body>
|
|
</html>
|
|
"""
|
|
url = "https://www.cms.gov/medicare/payment/fee-schedules/physician/pfs-relative-value-files/rvu26a"
|
|
with patch("bib.translate._fetch", return_value=html):
|
|
dl = cms_website(url, module="pfs")
|
|
assert dl.item_type == "download"
|
|
assert dl.title == "RVU26A"
|
|
assert len(dl.file_urls) == 2
|
|
assert any("rvu26a.zip" in u for u in dl.file_urls)
|
|
assert "source:cms-website" in dl.tags
|
|
assert "module:pfs" in dl.tags
|
|
assert dl.page_type == "rvu"
|
|
|
|
def test_year_from_title(self) -> None:
|
|
html = "<html><title>CY 2026 RVU Files</title><body></body></html>"
|
|
url = "https://www.cms.gov/medicare/payment/fee-schedules/physician"
|
|
with patch("bib.translate._fetch", return_value=html):
|
|
dl = cms_website(url)
|
|
assert dl.year == 2026
|
|
assert "year:2026" in dl.tags
|
|
|
|
def test_year_from_url(self) -> None:
|
|
html = "<html><title>Files</title><body></body></html>"
|
|
url = "https://www.cms.gov/rvu26a"
|
|
with patch("bib.translate._fetch", return_value=html):
|
|
dl = cms_website(url)
|
|
assert dl.year == 2026
|
|
|
|
def test_quarter_from_url(self) -> None:
|
|
html = "<html><title>RVU Files</title><body></body></html>"
|
|
url = "https://www.cms.gov/rvu26b"
|
|
with patch("bib.translate._fetch", return_value=html):
|
|
dl = cms_website(url)
|
|
assert dl.quarter == "B"
|
|
|
|
def test_absolute_file_urls(self) -> None:
|
|
html = '<html><title>Test</title><body><a href="https://cdn.cms.gov/data.xlsx">dl</a></body></html>'
|
|
url = "https://www.cms.gov/medicare/physician"
|
|
with patch("bib.translate._fetch", return_value=html):
|
|
dl = cms_website(url)
|
|
assert "https://cdn.cms.gov/data.xlsx" in dl.file_urls
|
|
|
|
def test_no_file_urls(self) -> None:
|
|
html = "<html><title>Info Page</title><body><a href='/about'>About</a></body></html>"
|
|
url = "https://www.cms.gov/medicare/physician"
|
|
with patch("bib.translate._fetch", return_value=html):
|
|
dl = cms_website(url)
|
|
assert dl.file_urls == []
|
|
|
|
|
|
# ── _infer_page_type ────────────────────────────────────────────────
|
|
|
|
|
|
class TestInferPageType:
|
|
def test_rvu(self) -> None:
|
|
assert _infer_page_type("https://cms.gov/rvu-files") == "rvu"
|
|
|
|
def test_relative_value(self) -> None:
|
|
assert _infer_page_type("https://cms.gov/relative-value") == "rvu"
|
|
|
|
def test_carrier(self) -> None:
|
|
assert _infer_page_type("https://cms.gov/carrier-files") == "carrier"
|
|
|
|
def test_pe_inputs(self) -> None:
|
|
assert _infer_page_type("https://cms.gov/practice-expense") == "pe_inputs"
|
|
|
|
def test_pe_dash(self) -> None:
|
|
assert _infer_page_type("https://cms.gov/pe-inputs") == "pe_inputs"
|
|
|
|
def test_gpci(self) -> None:
|
|
assert _infer_page_type("https://cms.gov/gpci-files") == "gpci"
|
|
|
|
def test_addendum(self) -> None:
|
|
assert _infer_page_type("https://cms.gov/addendum-a") == "addendum"
|
|
|
|
def test_other(self) -> None:
|
|
assert _infer_page_type("https://cms.gov/unknown-page") == "other"
|
|
|
|
|
|
# ── cms_manual ──────────────────────────────────────────────────────
|
|
|
|
|
|
class TestCmsManual:
|
|
def test_explicit_params(self) -> None:
|
|
item = cms_manual(
|
|
"https://www.cms.gov/manuals/downloads/clm104c12.pdf",
|
|
manual="Claims Processing",
|
|
chapter=12,
|
|
transmittal="R100",
|
|
)
|
|
assert item.item_type == "manual"
|
|
assert item.manual_name == "Claims Processing"
|
|
assert item.chapter == "12"
|
|
assert item.pub_number == "100-04"
|
|
assert item.transmittal == "R100"
|
|
assert "source:iom" in item.tags
|
|
assert "module:aco" in item.tags
|
|
|
|
def test_inferred_from_url(self) -> None:
|
|
item = cms_manual("https://www.cms.gov/manuals/downloads/clm104c12.pdf")
|
|
assert item.manual_name == "Claims Processing"
|
|
assert item.chapter == "12"
|
|
assert item.pub_number == "100-04"
|
|
assert "Claims Processing" in item.title
|
|
assert "Chapter 12" in item.title
|
|
|
|
def test_benefit_policy(self) -> None:
|
|
item = cms_manual("https://www.cms.gov/manuals/downloads/bp102c05.pdf")
|
|
assert item.manual_name == "Benefit Policy"
|
|
assert item.chapter == "5"
|
|
assert item.pub_number == "100-02"
|
|
|
|
def test_program_integrity(self) -> None:
|
|
item = cms_manual("https://www.cms.gov/manuals/downloads/pi108c01.pdf")
|
|
assert item.manual_name == "Program Integrity"
|
|
assert item.chapter == "1"
|
|
assert item.pub_number == "100-08"
|
|
|
|
def test_general_info(self) -> None:
|
|
item = cms_manual("https://www.cms.gov/manuals/downloads/ge101c03.pdf")
|
|
assert item.manual_name == "General Information"
|
|
assert item.chapter == "3"
|
|
|
|
def test_manual_only_title(self) -> None:
|
|
item = cms_manual(
|
|
"https://example.com/some-manual.pdf",
|
|
manual="Claims Processing",
|
|
)
|
|
assert item.title == "Claims Processing"
|
|
|
|
def test_chapter_only_title(self) -> None:
|
|
item = cms_manual("https://example.com/some.pdf", chapter=5)
|
|
assert item.title == "Chapter 5"
|
|
|
|
|
|
# ── _parse_iom_url ─────────────────────────────────────────────────
|
|
|
|
|
|
class TestParseIomUrl:
|
|
def test_claims_processing(self) -> None:
|
|
result = _parse_iom_url("https://www.cms.gov/manuals/downloads/clm104c12.pdf")
|
|
assert result["manual"] == "Claims Processing"
|
|
assert result["chapter"] == "12"
|
|
|
|
def test_benefit_policy(self) -> None:
|
|
result = _parse_iom_url("https://www.cms.gov/manuals/downloads/bp102c05.pdf")
|
|
assert result["manual"] == "Benefit Policy"
|
|
assert result["chapter"] == "5"
|
|
|
|
def test_no_match(self) -> None:
|
|
result = _parse_iom_url("https://example.com/unknown.pdf")
|
|
assert result == {}
|
|
|
|
def test_chapter_leading_zeros(self) -> None:
|
|
result = _parse_iom_url("https://www.cms.gov/manuals/downloads/clm104c001.pdf")
|
|
assert result["chapter"] == "1"
|
|
|
|
|
|
# ── ecfr ────────────────────────────────────────────────────────────
|
|
|
|
|
|
class TestEcfr:
|
|
def test_part_url(self) -> None:
|
|
url = "https://www.ecfr.gov/current/title-42/chapter-IV/subchapter-B/part-414"
|
|
api_data = {
|
|
"type": "title",
|
|
"children": [
|
|
{
|
|
"type": "part",
|
|
"identifier": "414",
|
|
"label": "Medicare Part B Payment",
|
|
"children": [],
|
|
}
|
|
],
|
|
}
|
|
with patch("bib.translate._fetch_json", return_value=api_data):
|
|
reg = ecfr(url)
|
|
assert reg.item_type == "regulation"
|
|
assert reg.cfr_title == "42"
|
|
assert reg.cfr_part == "414"
|
|
assert reg.cfr_section == ""
|
|
assert "Medicare Part B Payment" in reg.title
|
|
assert "source:ecfr" in reg.tags
|
|
assert "module:aco" in reg.tags
|
|
|
|
def test_section_url(self) -> None:
|
|
url = "https://www.ecfr.gov/current/title-42/part-414/section-414.22"
|
|
with patch("bib.translate._fetch_json", side_effect=Exception("API error")):
|
|
reg = ecfr(url)
|
|
assert reg.cfr_title == "42"
|
|
assert reg.cfr_part == "414"
|
|
assert reg.cfr_section == "414.22"
|
|
assert "414.22" in reg.title
|
|
|
|
def test_non_42_title(self) -> None:
|
|
url = "https://www.ecfr.gov/current/title-45/part-164"
|
|
with patch("bib.translate._fetch_json", side_effect=Exception):
|
|
reg = ecfr(url)
|
|
assert reg.cfr_title == "45"
|
|
assert "module:aco" not in reg.tags
|
|
|
|
def test_api_no_part_title(self) -> None:
|
|
url = "https://www.ecfr.gov/current/title-42/part-414"
|
|
api_data = {"type": "title", "children": []}
|
|
with patch("bib.translate._fetch_json", return_value=api_data):
|
|
reg = ecfr(url)
|
|
# No API title found, should still have basic title
|
|
assert "42 CFR" in reg.title
|
|
assert "Part 414" in reg.title
|
|
|
|
def test_no_title_or_part(self) -> None:
|
|
url = "https://www.ecfr.gov/current/some-weird-url"
|
|
with patch("bib.translate._fetch_json", side_effect=Exception):
|
|
reg = ecfr(url)
|
|
assert reg.cfr_title == ""
|
|
assert reg.cfr_part == ""
|
|
assert reg.title == url
|
|
|
|
|
|
# ── _find_part_title ────────────────────────────────────────────────
|
|
|
|
|
|
class TestFindPartTitle:
|
|
def test_direct_match(self) -> None:
|
|
structure = {
|
|
"type": "part",
|
|
"identifier": "414",
|
|
"label": "Medicare Part B",
|
|
"children": [],
|
|
}
|
|
assert _find_part_title(structure, "414") == "Medicare Part B"
|
|
|
|
def test_nested_match(self) -> None:
|
|
structure = {
|
|
"type": "title",
|
|
"children": [
|
|
{
|
|
"type": "chapter",
|
|
"children": [
|
|
{
|
|
"type": "part",
|
|
"identifier": "414",
|
|
"label": "Nested Part",
|
|
"children": [],
|
|
}
|
|
],
|
|
}
|
|
],
|
|
}
|
|
assert _find_part_title(structure, "414") == "Nested Part"
|
|
|
|
def test_no_match(self) -> None:
|
|
structure = {
|
|
"type": "title",
|
|
"children": [
|
|
{
|
|
"type": "part",
|
|
"identifier": "410",
|
|
"label": "Wrong Part",
|
|
"children": [],
|
|
}
|
|
],
|
|
}
|
|
assert _find_part_title(structure, "414") == ""
|
|
|
|
|
|
# ── four_i ──────────────────────────────────────────────────────────
|
|
|
|
|
|
class TestFourI:
|
|
def _sample_html(self) -> str:
|
|
return """
|
|
<html>
|
|
<title>4i Knowledge Base</title>
|
|
<body>
|
|
<span id="title-1">PY 2026 Financial Guarantees</span>
|
|
<span id="subcategory-1">ACO REACH</span>
|
|
<span id="description-1">Guidance on financial guarantees.</span>
|
|
<span id="state-last-updated-date">06/15/2025</span>
|
|
<span id="state-creatdBy">John Doe</span>
|
|
<span id="view-count-1">42</span>
|
|
<span id="file-1">attachment1.pdf</span>
|
|
<span id="file-2">attachment2.xlsx</span>
|
|
<a href="https://4innovation.cms.gov/secure/knowledge-management/view/2190">link</a>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
def test_from_raw_html(self) -> None:
|
|
html = self._sample_html()
|
|
item = four_i(html)
|
|
assert item.item_type == "source"
|
|
assert item.title == "PY 2026 Financial Guarantees"
|
|
assert item.institution == "CMS Innovation Center"
|
|
assert item.doc_type == "Knowledge Base Article"
|
|
assert item.date_published == "2025-06-15"
|
|
assert "Guidance on financial guarantees." in item.abstract
|
|
assert "source:4i" in item.tags
|
|
assert "module:aco" in item.tags
|
|
assert "category:aco-reach" in item.tags
|
|
assert "author:John Doe" in item.tags
|
|
assert "year:2025" in item.tags
|
|
assert "KM-ID: 2190" in item.extra
|
|
assert "Views: 42" in item.extra
|
|
assert "attachment1.pdf" in item.extra
|
|
|
|
def test_from_file(self, tmp_path) -> None:
|
|
html_file = tmp_path / "2190.html"
|
|
html_file.write_text(self._sample_html())
|
|
item = four_i(str(html_file))
|
|
assert item.title == "PY 2026 Financial Guarantees"
|
|
|
|
def test_explicit_url_and_km_id(self) -> None:
|
|
html = '<span id="title-1">Test Article</span>'
|
|
item = four_i(
|
|
html,
|
|
url="https://4innovation.cms.gov/secure/knowledge-management/view/9999",
|
|
km_id="9999",
|
|
)
|
|
assert "KM-ID: 9999" in item.extra
|
|
assert (
|
|
item.url
|
|
== "https://4innovation.cms.gov/secure/knowledge-management/view/9999"
|
|
)
|
|
|
|
def test_km_id_from_url(self) -> None:
|
|
html = '<span id="title-1">Test</span>'
|
|
item = four_i(
|
|
html,
|
|
url="https://4innovation.cms.gov/secure/knowledge-management/view/1234",
|
|
)
|
|
assert "KM-ID: 1234" in item.extra
|
|
|
|
def test_title_fallback_to_title_tag(self) -> None:
|
|
html = "<html><title>Fallback Title</title><body></body></html>"
|
|
item = four_i(html)
|
|
assert item.title == "Fallback Title"
|
|
|
|
def test_title_fallback_to_km_id(self) -> None:
|
|
html = "<html><body></body></html>"
|
|
item = four_i(html, km_id="5678")
|
|
assert "5678" in item.title
|
|
|
|
def test_no_date(self) -> None:
|
|
html = '<span id="title-1">No Date</span>'
|
|
item = four_i(html)
|
|
assert item.date_published == ""
|
|
|
|
def test_no_subcategory(self) -> None:
|
|
html = '<span id="title-1">No Sub</span>'
|
|
item = four_i(html)
|
|
assert not any(t.startswith("category:") for t in item.tags)
|
|
|
|
def test_no_author(self) -> None:
|
|
html = '<span id="title-1">No Author</span>'
|
|
item = four_i(html)
|
|
assert not any(t.startswith("author:") for t in item.tags)
|
|
|
|
|
|
# ── _parse_4i_date ──────────────────────────────────────────────────
|
|
|
|
|
|
class TestParse4iDate:
|
|
def test_valid_date(self) -> None:
|
|
assert _parse_4i_date("06/15/2025") == "2025-06-15"
|
|
|
|
def test_with_whitespace(self) -> None:
|
|
assert _parse_4i_date(" 01/01/2026 ") == "2026-01-01"
|
|
|
|
def test_invalid_format(self) -> None:
|
|
assert _parse_4i_date("2025-06-15") == "2025-06-15"
|
|
|
|
def test_empty(self) -> None:
|
|
assert _parse_4i_date("") == ""
|
|
|
|
|
|
# ── _extract_4i_field ───────────────────────────────────────────────
|
|
|
|
|
|
class TestExtract4iField:
|
|
def test_quoted_id(self) -> None:
|
|
html = '<span id="title-1">My Title</span>'
|
|
assert _extract_4i_field(html, "title-1") == "My Title"
|
|
|
|
def test_unquoted_id(self) -> None:
|
|
html = "<span id=title-1>My Title</span>"
|
|
assert _extract_4i_field(html, "title-1") == "My Title"
|
|
|
|
def test_nested_html(self) -> None:
|
|
html = '<span id="desc-1"><b>Bold</b> text <em>here</em></span>'
|
|
result = _extract_4i_field(html, "desc-1")
|
|
assert "Bold" in result
|
|
assert "text" in result
|
|
assert "<b>" not in result
|
|
|
|
def test_missing_field(self) -> None:
|
|
html = "<span id=other>Text</span>"
|
|
assert _extract_4i_field(html, "title-1") == ""
|
|
|
|
def test_p_tag(self) -> None:
|
|
html = '<p id="description-1">Paragraph content</p>'
|
|
assert _extract_4i_field(html, "description-1") == "Paragraph content"
|