5261032a9b
Cover remaining uncovered lines in bcda (client, store, log, pipe/cclf, flatten), bib (sync, spider, translate, ingest, item, store, format, ui), cms (express wrappers, log edge cases), api (base, gitea, rustfs, woodpecker, zotero), bls (table import), and pfs (pragma on race guard). 37,687 statements, 0 missed — 11,061 tests passing.
555 lines
21 KiB
Python
555 lines
21 KiB
Python
"""Tests for bib.translate — CMS documentation translators."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from bib.translate import (
|
|
_extract_4i_field,
|
|
_find_part_title,
|
|
_infer_page_type,
|
|
_parse_4i_date,
|
|
_parse_iom_url,
|
|
cms_manual,
|
|
cms_website,
|
|
ecfr,
|
|
federal_register,
|
|
four_i,
|
|
)
|
|
|
|
# ── _fetch / _fetch_json helpers ────────────────────────────────────
|
|
|
|
|
|
class TestFetchHelpers:
|
|
def test_fetch_uses_urlopen(self) -> None:
|
|
"""Lines 67-74: _fetch makes urllib request."""
|
|
mock_resp = MagicMock()
|
|
mock_resp.read.return_value = b"<html>hello</html>"
|
|
mock_resp.__enter__ = lambda s: s
|
|
mock_resp.__exit__ = MagicMock(return_value=False)
|
|
|
|
with patch("urllib.request.urlopen", return_value=mock_resp):
|
|
from bib.translate import _fetch
|
|
|
|
result = _fetch("https://example.com")
|
|
assert result == "<html>hello</html>"
|
|
|
|
def test_fetch_json_parses(self) -> None:
|
|
mock_resp = MagicMock()
|
|
mock_resp.read.return_value = b'{"key": "value"}'
|
|
mock_resp.__enter__ = lambda s: s
|
|
mock_resp.__exit__ = MagicMock(return_value=False)
|
|
|
|
with patch("urllib.request.urlopen", return_value=mock_resp):
|
|
from bib.translate import _fetch_json
|
|
|
|
result = _fetch_json("https://example.com")
|
|
assert result == {"key": "value"}
|
|
|
|
|
|
# ── federal_register ────────────────────────────────────────────────
|
|
|
|
|
|
class TestFederalRegister:
|
|
def _mock_fr_data(self, **overrides: object) -> dict:
|
|
base = {
|
|
"title": "Medicare Program; CY 2026 PFS Final Rule (CMS-1832-F)",
|
|
"volume": 90,
|
|
"start_page": 98452,
|
|
"publication_date": "2025-11-01",
|
|
"effective_on": "2026-01-01",
|
|
"html_url": "https://www.federalregister.gov/d/2025-19787",
|
|
"abstract": "This rule finalizes...",
|
|
"type": "Rule - Final",
|
|
"regulation_id_numbers": [],
|
|
}
|
|
base.update(overrides)
|
|
return base
|
|
|
|
def test_from_html_url(self) -> None:
|
|
url = (
|
|
"https://www.federalregister.gov/documents/2025/11/01/"
|
|
"2025-19787/medicare-program"
|
|
)
|
|
data = self._mock_fr_data()
|
|
with patch("bib.translate._fetch_json", return_value=data):
|
|
rule = federal_register(url)
|
|
assert rule.item_type == "rule"
|
|
assert rule.document_number == "2025-19787"
|
|
assert rule.fr_volume == "90"
|
|
assert rule.fr_page == "98452"
|
|
assert rule.cms_id == "CMS-1832-F"
|
|
assert rule.rule_type == "final"
|
|
assert "source:federal-register" in rule.tags
|
|
assert "module:pfs" in rule.tags
|
|
assert "year:2025" in rule.tags
|
|
assert "rule:cms-1832-f" in rule.tags
|
|
|
|
def test_from_api_url(self) -> None:
|
|
url = "https://www.federalregister.gov/api/v1/documents/2025-19787.json"
|
|
data = self._mock_fr_data()
|
|
with patch("bib.translate._fetch_json", return_value=data):
|
|
rule = federal_register(url)
|
|
assert rule.document_number == "2025-19787"
|
|
|
|
def test_proposed_rule_type(self) -> None:
|
|
url = (
|
|
"https://www.federalregister.gov/documents/2025/07/01/"
|
|
"2025-12345/proposed-rule"
|
|
)
|
|
data = self._mock_fr_data(type="Rule - Proposed", title="Proposed Rule")
|
|
with patch("bib.translate._fetch_json", return_value=data):
|
|
rule = federal_register(url)
|
|
assert rule.rule_type == "proposed"
|
|
|
|
def test_regulation_id_fallback(self) -> None:
|
|
url = (
|
|
"https://www.federalregister.gov/documents/2025/07/01/2025-12345/some-rule"
|
|
)
|
|
data = self._mock_fr_data(
|
|
title="Some Rule Without CMS ID",
|
|
regulation_id_numbers=["RIN-0938-AU99"],
|
|
)
|
|
with patch("bib.translate._fetch_json", return_value=data):
|
|
rule = federal_register(url)
|
|
assert rule.cms_id == "RIN-0938-AU99"
|
|
|
|
def test_no_pub_year_no_year_tag(self) -> None:
|
|
url = (
|
|
"https://www.federalregister.gov/documents/2025/07/01/2025-12345/some-rule"
|
|
)
|
|
data = self._mock_fr_data(publication_date="", title="Test")
|
|
with patch("bib.translate._fetch_json", return_value=data):
|
|
rule = federal_register(url)
|
|
assert not any(t.startswith("year:") for t in rule.tags)
|
|
|
|
def test_no_doc_number(self) -> None:
|
|
url = "https://www.federalregister.gov/other/path"
|
|
data = self._mock_fr_data(title="Test")
|
|
with patch("bib.translate._fetch_json", return_value=data):
|
|
rule = federal_register(url)
|
|
assert rule.document_number == ""
|
|
|
|
def test_other_doc_type(self) -> None:
|
|
url = "https://www.federalregister.gov/documents/2025/07/01/2025-12345/notice"
|
|
data = self._mock_fr_data(type="Notice", title="A Notice")
|
|
with patch("bib.translate._fetch_json", return_value=data):
|
|
rule = federal_register(url)
|
|
assert rule.rule_type == "notice"
|
|
|
|
|
|
# ── cms_website ─────────────────────────────────────────────────────
|
|
|
|
|
|
class TestCmsWebsite:
|
|
def test_basic_download_page(self) -> None:
|
|
html = """
|
|
<html>
|
|
<title>RVU26A | CMS</title>
|
|
<body>
|
|
<a href="/files/rvu26a.zip">Download</a>
|
|
<a href="/files/data.csv">CSV</a>
|
|
</body>
|
|
</html>
|
|
"""
|
|
url = "https://www.cms.gov/medicare/payment/fee-schedules/physician/pfs-relative-value-files/rvu26a"
|
|
with patch("bib.translate._fetch", return_value=html):
|
|
dl = cms_website(url, module="pfs")
|
|
assert dl.item_type == "download"
|
|
assert dl.title == "RVU26A"
|
|
assert len(dl.file_urls) == 2
|
|
assert any("rvu26a.zip" in u for u in dl.file_urls)
|
|
assert "source:cms-website" in dl.tags
|
|
assert "module:pfs" in dl.tags
|
|
assert dl.page_type == "rvu"
|
|
|
|
def test_year_from_title(self) -> None:
|
|
html = "<html><title>CY 2026 RVU Files</title><body></body></html>"
|
|
url = "https://www.cms.gov/medicare/payment/fee-schedules/physician"
|
|
with patch("bib.translate._fetch", return_value=html):
|
|
dl = cms_website(url)
|
|
assert dl.year == 2026
|
|
assert "year:2026" in dl.tags
|
|
|
|
def test_year_from_url(self) -> None:
|
|
html = "<html><title>Files</title><body></body></html>"
|
|
url = "https://www.cms.gov/rvu26a"
|
|
with patch("bib.translate._fetch", return_value=html):
|
|
dl = cms_website(url)
|
|
assert dl.year == 2026
|
|
|
|
def test_quarter_from_url(self) -> None:
|
|
html = "<html><title>RVU Files</title><body></body></html>"
|
|
url = "https://www.cms.gov/rvu26b"
|
|
with patch("bib.translate._fetch", return_value=html):
|
|
dl = cms_website(url)
|
|
assert dl.quarter == "B"
|
|
|
|
def test_absolute_file_urls(self) -> None:
|
|
html = '<html><title>Test</title><body><a href="https://cdn.cms.gov/data.xlsx">dl</a></body></html>'
|
|
url = "https://www.cms.gov/medicare/physician"
|
|
with patch("bib.translate._fetch", return_value=html):
|
|
dl = cms_website(url)
|
|
assert "https://cdn.cms.gov/data.xlsx" in dl.file_urls
|
|
|
|
def test_no_file_urls(self) -> None:
|
|
html = "<html><title>Info Page</title><body><a href='/about'>About</a></body></html>"
|
|
url = "https://www.cms.gov/medicare/physician"
|
|
with patch("bib.translate._fetch", return_value=html):
|
|
dl = cms_website(url)
|
|
assert dl.file_urls == []
|
|
|
|
|
|
# ── _infer_page_type ────────────────────────────────────────────────
|
|
|
|
|
|
class TestInferPageType:
|
|
def test_rvu(self) -> None:
|
|
assert _infer_page_type("https://cms.gov/rvu-files") == "rvu"
|
|
|
|
def test_relative_value(self) -> None:
|
|
assert _infer_page_type("https://cms.gov/relative-value") == "rvu"
|
|
|
|
def test_carrier(self) -> None:
|
|
assert _infer_page_type("https://cms.gov/carrier-files") == "carrier"
|
|
|
|
def test_pe_inputs(self) -> None:
|
|
assert _infer_page_type("https://cms.gov/practice-expense") == "pe_inputs"
|
|
|
|
def test_pe_dash(self) -> None:
|
|
assert _infer_page_type("https://cms.gov/pe-inputs") == "pe_inputs"
|
|
|
|
def test_gpci(self) -> None:
|
|
assert _infer_page_type("https://cms.gov/gpci-files") == "gpci"
|
|
|
|
def test_addendum(self) -> None:
|
|
assert _infer_page_type("https://cms.gov/addendum-a") == "addendum"
|
|
|
|
def test_other(self) -> None:
|
|
assert _infer_page_type("https://cms.gov/unknown-page") == "other"
|
|
|
|
|
|
# ── cms_manual ──────────────────────────────────────────────────────
|
|
|
|
|
|
class TestCmsManual:
|
|
def test_explicit_params(self) -> None:
|
|
item = cms_manual(
|
|
"https://www.cms.gov/manuals/downloads/clm104c12.pdf",
|
|
manual="Claims Processing",
|
|
chapter=12,
|
|
transmittal="R100",
|
|
)
|
|
assert item.item_type == "manual"
|
|
assert item.manual_name == "Claims Processing"
|
|
assert item.chapter == "12"
|
|
assert item.pub_number == "100-04"
|
|
assert item.transmittal == "R100"
|
|
assert "source:iom" in item.tags
|
|
assert "module:aco" in item.tags
|
|
|
|
def test_inferred_from_url(self) -> None:
|
|
item = cms_manual("https://www.cms.gov/manuals/downloads/clm104c12.pdf")
|
|
assert item.manual_name == "Claims Processing"
|
|
assert item.chapter == "12"
|
|
assert item.pub_number == "100-04"
|
|
assert "Claims Processing" in item.title
|
|
assert "Chapter 12" in item.title
|
|
|
|
def test_benefit_policy(self) -> None:
|
|
item = cms_manual("https://www.cms.gov/manuals/downloads/bp102c05.pdf")
|
|
assert item.manual_name == "Benefit Policy"
|
|
assert item.chapter == "5"
|
|
assert item.pub_number == "100-02"
|
|
|
|
def test_program_integrity(self) -> None:
|
|
item = cms_manual("https://www.cms.gov/manuals/downloads/pi108c01.pdf")
|
|
assert item.manual_name == "Program Integrity"
|
|
assert item.chapter == "1"
|
|
assert item.pub_number == "100-08"
|
|
|
|
def test_general_info(self) -> None:
|
|
item = cms_manual("https://www.cms.gov/manuals/downloads/ge101c03.pdf")
|
|
assert item.manual_name == "General Information"
|
|
assert item.chapter == "3"
|
|
|
|
def test_manual_only_title(self) -> None:
|
|
item = cms_manual(
|
|
"https://example.com/some-manual.pdf",
|
|
manual="Claims Processing",
|
|
)
|
|
assert item.title == "Claims Processing"
|
|
|
|
def test_chapter_only_title(self) -> None:
|
|
item = cms_manual("https://example.com/some.pdf", chapter=5)
|
|
assert item.title == "Chapter 5"
|
|
|
|
|
|
# ── _parse_iom_url ─────────────────────────────────────────────────
|
|
|
|
|
|
class TestParseIomUrl:
|
|
def test_claims_processing(self) -> None:
|
|
result = _parse_iom_url("https://www.cms.gov/manuals/downloads/clm104c12.pdf")
|
|
assert result["manual"] == "Claims Processing"
|
|
assert result["chapter"] == "12"
|
|
|
|
def test_benefit_policy(self) -> None:
|
|
result = _parse_iom_url("https://www.cms.gov/manuals/downloads/bp102c05.pdf")
|
|
assert result["manual"] == "Benefit Policy"
|
|
assert result["chapter"] == "5"
|
|
|
|
def test_no_match(self) -> None:
|
|
result = _parse_iom_url("https://example.com/unknown.pdf")
|
|
assert result == {}
|
|
|
|
def test_chapter_leading_zeros(self) -> None:
|
|
result = _parse_iom_url("https://www.cms.gov/manuals/downloads/clm104c001.pdf")
|
|
assert result["chapter"] == "1"
|
|
|
|
|
|
# ── ecfr ────────────────────────────────────────────────────────────
|
|
|
|
|
|
class TestEcfr:
|
|
def test_part_url(self) -> None:
|
|
url = "https://www.ecfr.gov/current/title-42/chapter-IV/subchapter-B/part-414"
|
|
api_data = {
|
|
"type": "title",
|
|
"children": [
|
|
{
|
|
"type": "part",
|
|
"identifier": "414",
|
|
"label": "Medicare Part B Payment",
|
|
"children": [],
|
|
}
|
|
],
|
|
}
|
|
with patch("bib.translate._fetch_json", return_value=api_data):
|
|
reg = ecfr(url)
|
|
assert reg.item_type == "regulation"
|
|
assert reg.cfr_title == "42"
|
|
assert reg.cfr_part == "414"
|
|
assert reg.cfr_section == ""
|
|
assert "Medicare Part B Payment" in reg.title
|
|
assert "source:ecfr" in reg.tags
|
|
assert "module:aco" in reg.tags
|
|
|
|
def test_section_url(self) -> None:
|
|
url = "https://www.ecfr.gov/current/title-42/part-414/section-414.22"
|
|
with patch("bib.translate._fetch_json", side_effect=Exception("API error")):
|
|
reg = ecfr(url)
|
|
assert reg.cfr_title == "42"
|
|
assert reg.cfr_part == "414"
|
|
assert reg.cfr_section == "414.22"
|
|
assert "414.22" in reg.title
|
|
|
|
def test_non_42_title(self) -> None:
|
|
url = "https://www.ecfr.gov/current/title-45/part-164"
|
|
with patch("bib.translate._fetch_json", side_effect=Exception):
|
|
reg = ecfr(url)
|
|
assert reg.cfr_title == "45"
|
|
assert "module:aco" not in reg.tags
|
|
|
|
def test_api_no_part_title(self) -> None:
|
|
url = "https://www.ecfr.gov/current/title-42/part-414"
|
|
api_data = {"type": "title", "children": []}
|
|
with patch("bib.translate._fetch_json", return_value=api_data):
|
|
reg = ecfr(url)
|
|
# No API title found, should still have basic title
|
|
assert "42 CFR" in reg.title
|
|
assert "Part 414" in reg.title
|
|
|
|
def test_no_title_or_part(self) -> None:
|
|
url = "https://www.ecfr.gov/current/some-weird-url"
|
|
with patch("bib.translate._fetch_json", side_effect=Exception):
|
|
reg = ecfr(url)
|
|
assert reg.cfr_title == ""
|
|
assert reg.cfr_part == ""
|
|
assert reg.title == url
|
|
|
|
|
|
# ── _find_part_title ────────────────────────────────────────────────
|
|
|
|
|
|
class TestFindPartTitle:
|
|
def test_direct_match(self) -> None:
|
|
structure = {
|
|
"type": "part",
|
|
"identifier": "414",
|
|
"label": "Medicare Part B",
|
|
"children": [],
|
|
}
|
|
assert _find_part_title(structure, "414") == "Medicare Part B"
|
|
|
|
def test_nested_match(self) -> None:
|
|
structure = {
|
|
"type": "title",
|
|
"children": [
|
|
{
|
|
"type": "chapter",
|
|
"children": [
|
|
{
|
|
"type": "part",
|
|
"identifier": "414",
|
|
"label": "Nested Part",
|
|
"children": [],
|
|
}
|
|
],
|
|
}
|
|
],
|
|
}
|
|
assert _find_part_title(structure, "414") == "Nested Part"
|
|
|
|
def test_no_match(self) -> None:
|
|
structure = {
|
|
"type": "title",
|
|
"children": [
|
|
{
|
|
"type": "part",
|
|
"identifier": "410",
|
|
"label": "Wrong Part",
|
|
"children": [],
|
|
}
|
|
],
|
|
}
|
|
assert _find_part_title(structure, "414") == ""
|
|
|
|
|
|
# ── four_i ──────────────────────────────────────────────────────────
|
|
|
|
|
|
class TestFourI:
|
|
def _sample_html(self) -> str:
|
|
return """
|
|
<html>
|
|
<title>4i Knowledge Base</title>
|
|
<body>
|
|
<span id="title-1">PY 2026 Financial Guarantees</span>
|
|
<span id="subcategory-1">ACO REACH</span>
|
|
<span id="description-1">Guidance on financial guarantees.</span>
|
|
<span id="state-last-updated-date">06/15/2025</span>
|
|
<span id="state-creatdBy">John Doe</span>
|
|
<span id="view-count-1">42</span>
|
|
<span id="file-1">attachment1.pdf</span>
|
|
<span id="file-2">attachment2.xlsx</span>
|
|
<a href="https://4innovation.cms.gov/secure/knowledge-management/view/2190">link</a>
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
def test_from_raw_html(self) -> None:
|
|
html = self._sample_html()
|
|
item = four_i(html)
|
|
assert item.item_type == "source"
|
|
assert item.title == "PY 2026 Financial Guarantees"
|
|
assert item.institution == "CMS Innovation Center"
|
|
assert item.doc_type == "Knowledge Base Article"
|
|
assert item.date_published == "2025-06-15"
|
|
assert "Guidance on financial guarantees." in item.abstract
|
|
assert "source:4i" in item.tags
|
|
assert "module:aco" in item.tags
|
|
assert "category:aco-reach" in item.tags
|
|
assert "author:John Doe" in item.tags
|
|
assert "year:2025" in item.tags
|
|
assert "KM-ID: 2190" in item.extra
|
|
assert "Views: 42" in item.extra
|
|
assert "attachment1.pdf" in item.extra
|
|
|
|
def test_from_file(self, tmp_path) -> None:
|
|
html_file = tmp_path / "2190.html"
|
|
html_file.write_text(self._sample_html())
|
|
item = four_i(str(html_file))
|
|
assert item.title == "PY 2026 Financial Guarantees"
|
|
|
|
def test_explicit_url_and_km_id(self) -> None:
|
|
html = '<span id="title-1">Test Article</span>'
|
|
item = four_i(
|
|
html,
|
|
url="https://4innovation.cms.gov/secure/knowledge-management/view/9999",
|
|
km_id="9999",
|
|
)
|
|
assert "KM-ID: 9999" in item.extra
|
|
assert (
|
|
item.url
|
|
== "https://4innovation.cms.gov/secure/knowledge-management/view/9999"
|
|
)
|
|
|
|
def test_km_id_from_url(self) -> None:
|
|
html = '<span id="title-1">Test</span>'
|
|
item = four_i(
|
|
html,
|
|
url="https://4innovation.cms.gov/secure/knowledge-management/view/1234",
|
|
)
|
|
assert "KM-ID: 1234" in item.extra
|
|
|
|
def test_title_fallback_to_title_tag(self) -> None:
|
|
html = "<html><title>Fallback Title</title><body></body></html>"
|
|
item = four_i(html)
|
|
assert item.title == "Fallback Title"
|
|
|
|
def test_title_fallback_to_km_id(self) -> None:
|
|
html = "<html><body></body></html>"
|
|
item = four_i(html, km_id="5678")
|
|
assert "5678" in item.title
|
|
|
|
def test_no_date(self) -> None:
|
|
html = '<span id="title-1">No Date</span>'
|
|
item = four_i(html)
|
|
assert item.date_published == ""
|
|
|
|
def test_no_subcategory(self) -> None:
|
|
html = '<span id="title-1">No Sub</span>'
|
|
item = four_i(html)
|
|
assert not any(t.startswith("category:") for t in item.tags)
|
|
|
|
def test_no_author(self) -> None:
|
|
html = '<span id="title-1">No Author</span>'
|
|
item = four_i(html)
|
|
assert not any(t.startswith("author:") for t in item.tags)
|
|
|
|
|
|
# ── _parse_4i_date ──────────────────────────────────────────────────
|
|
|
|
|
|
class TestParse4iDate:
|
|
def test_valid_date(self) -> None:
|
|
assert _parse_4i_date("06/15/2025") == "2025-06-15"
|
|
|
|
def test_with_whitespace(self) -> None:
|
|
assert _parse_4i_date(" 01/01/2026 ") == "2026-01-01"
|
|
|
|
def test_invalid_format(self) -> None:
|
|
assert _parse_4i_date("2025-06-15") == "2025-06-15"
|
|
|
|
def test_empty(self) -> None:
|
|
assert _parse_4i_date("") == ""
|
|
|
|
|
|
# ── _extract_4i_field ───────────────────────────────────────────────
|
|
|
|
|
|
class TestExtract4iField:
|
|
def test_quoted_id(self) -> None:
|
|
html = '<span id="title-1">My Title</span>'
|
|
assert _extract_4i_field(html, "title-1") == "My Title"
|
|
|
|
def test_unquoted_id(self) -> None:
|
|
html = "<span id=title-1>My Title</span>"
|
|
assert _extract_4i_field(html, "title-1") == "My Title"
|
|
|
|
def test_nested_html(self) -> None:
|
|
html = '<span id="desc-1"><b>Bold</b> text <em>here</em></span>'
|
|
result = _extract_4i_field(html, "desc-1")
|
|
assert "Bold" in result
|
|
assert "text" in result
|
|
assert "<b>" not in result
|
|
|
|
def test_missing_field(self) -> None:
|
|
html = "<span id=other>Text</span>"
|
|
assert _extract_4i_field(html, "title-1") == ""
|
|
|
|
def test_p_tag(self) -> None:
|
|
html = '<p id="description-1">Paragraph content</p>'
|
|
assert _extract_4i_field(html, "description-1") == "Paragraph content"
|