Some checks failed
CI / lint (push) Successful in 1m10s
CI / notebooks-smoke (push) Successful in 2m53s
Deploy / changes (push) Successful in 2m7s
CI / test (push) Has been cancelled
Deploy / report (push) Has been cancelled
Deploy / bcda (push) Has been cancelled
Deploy / notebooks (push) Has been cancelled
Deploy / zotero (push) Has been cancelled
Deploy / docs (push) Has been cancelled
Deploy / api (push) Has been cancelled
Deploy / llm (push) Has been cancelled
Deploy / mc (push) Has been cancelled
Infra CI / zotero (push) Has been cancelled
Infra CI / docs (push) Has been cancelled
Infra CI / api (push) Has been cancelled
Infra CI / bcda (push) Has been cancelled
Infra CI / llm (push) Has been cancelled
Infra CI / mc (push) Has been cancelled
Infra CI / notebooks (push) Has started running
attach-iom compared each download with an item's *first* attachment (a dead Zotero-relative storage: row on 29 manuals), then called attach_file with the same title — which returns the existing attachment without copying. Every revised chapter since 2026-04-15 was downloaded nightly and thrown away; bib and Zotero both still hold the first copy. Revisions are now kept, never replaced: compared against the newest copy on disk and attached as <stem>.<YYYY-MM-DD><ext> titled '<title> (<date>)'. Zotero sync likewise never overwrites: a changed same-name file is added beside the old one under a dated name, and a missing Zotero file is restored. The corpus index reads only the newest member of each version family, so hoarded revisions are not indexed twice.
300 lines
9.6 KiB
Python
300 lines
9.6 KiB
Python
"""Exercise bib.iom — covers fetch_chapters regex parsing, download, check_future."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import MagicMock
|
|
|
|
from bib.iom import (
|
|
IOMEntry,
|
|
_existing_attachment_hash,
|
|
_futurepdf_anchor,
|
|
_prune_stale,
|
|
_sha256,
|
|
check_future_updates,
|
|
download_attachments,
|
|
fetch_chapters,
|
|
fetch_index,
|
|
ingest_all,
|
|
ingest_entry,
|
|
)
|
|
|
|
_ENTRY = IOMEntry(
|
|
pub="100-04",
|
|
title="Medicare Claims Processing Manual",
|
|
landing_url="https://www.cms.gov/regulations-and-guidance/guidance/manuals/internet-only-manuals-ioms-items/cms018912",
|
|
)
|
|
|
|
|
|
class TestFetchChaptersRegex:
|
|
def _client(self, html):
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.text = html
|
|
resp.status_code = 200
|
|
client.get.return_value = resp
|
|
return client
|
|
|
|
def test_matches_chapters(self):
|
|
html = """
|
|
<a href="/regulations-and-guidance/guidance/manuals/downloads/clm104c01.pdf">
|
|
Chapter 1 - General Billing Requirements
|
|
</a>
|
|
<a href="/regulations-and-guidance/guidance/manuals/downloads/clm104c02.pdf">
|
|
Chapter 2 - Submission of Claims
|
|
</a>
|
|
"""
|
|
chapters = fetch_chapters(self._client(html), _ENTRY)
|
|
assert len(chapters) == 2
|
|
assert chapters[0].chapter == "1"
|
|
assert "General Billing" in chapters[0].title
|
|
|
|
def test_skips_supplements(self):
|
|
html = """
|
|
<a href="/regulations-and-guidance/guidance/manuals/downloads/clm104c01.pdf">
|
|
Chapter 1 - General
|
|
</a>
|
|
<a href="/regulations-and-guidance/guidance/manuals/downloads/crosswalk.pdf">
|
|
Crosswalk of Changes
|
|
</a>
|
|
"""
|
|
chapters = fetch_chapters(self._client(html), _ENTRY)
|
|
assert len(chapters) == 1
|
|
|
|
def test_whole_pub_fallback(self):
|
|
html = """
|
|
<a href="/regulations-and-guidance/guidance/manuals/downloads/pub100_18.pdf">
|
|
Pub 100-18 - Medicare Prescription Drug Benefit Manual
|
|
</a>
|
|
"""
|
|
chapters = fetch_chapters(self._client(html), _ENTRY)
|
|
assert len(chapters) == 1
|
|
assert chapters[0].chapter == ""
|
|
|
|
def test_dedup_chapters(self):
|
|
html = """
|
|
<a href="/regulations-and-guidance/guidance/manuals/downloads/clm104c01.pdf">
|
|
Chapter 1 - General
|
|
</a>
|
|
<a href="/regulations-and-guidance/guidance/manuals/downloads/clm104c01v2.pdf">
|
|
Chapter 1 - General (Updated)
|
|
</a>
|
|
"""
|
|
chapters = fetch_chapters(self._client(html), _ENTRY)
|
|
assert len(chapters) == 1
|
|
|
|
def test_part_chapters(self):
|
|
html = """
|
|
<a href="/regulations-and-guidance/guidance/manuals/downloads/ncd103c03p1.pdf">
|
|
Chapter 3, Part 1 — Section A
|
|
</a>
|
|
<a href="/regulations-and-guidance/guidance/manuals/downloads/ncd103c03p2.pdf">
|
|
Chapter 3, Part 2 — Section B
|
|
</a>
|
|
"""
|
|
chapters = fetch_chapters(self._client(html), _ENTRY)
|
|
assert len(chapters) == 2
|
|
assert chapters[0].chapter in ("3P1", "3p1")
|
|
|
|
|
|
class TestFetchIndex:
|
|
def test_real_html_pattern(self):
|
|
html = """
|
|
<a href="/regulations-and-guidance/guidance/manuals/internet-only-manuals-ioms-items/cms018912"
|
|
class="title-link">
|
|
100-04
|
|
</a>
|
|
<div><label>Title</label>Medicare Claims Processing Manual</div>
|
|
"""
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.text = html
|
|
resp.status_code = 200
|
|
client.get.return_value = resp
|
|
entries = fetch_index(client)
|
|
assert len(entries) == 1
|
|
assert entries[0].pub == "100-04"
|
|
|
|
|
|
class TestPruneStale:
|
|
def test_prunes(self):
|
|
store = MagicMock()
|
|
con = MagicMock()
|
|
store._con.return_value = con
|
|
con.execute.return_value.fetchall.return_value = [
|
|
{"key": "K1", "url": "https://old.pdf"},
|
|
{"key": "K2", "url": "https://live.pdf"},
|
|
]
|
|
n = _prune_stale(store, "100-04", {"https://live.pdf"})
|
|
assert n == 1
|
|
store.delete.assert_called_once_with("K1")
|
|
|
|
|
|
class TestIngestEntryDeep:
|
|
def test_with_chapters(self):
|
|
store = MagicMock()
|
|
store.upsert.return_value = "KEY1"
|
|
store._con.return_value = MagicMock(
|
|
execute=MagicMock(
|
|
return_value=MagicMock(fetchall=MagicMock(return_value=[]))
|
|
)
|
|
)
|
|
|
|
html = """
|
|
<a href="/regulations-and-guidance/guidance/manuals/downloads/clm104c01.pdf">
|
|
Chapter 1 - General
|
|
</a>
|
|
"""
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.text = html
|
|
resp.status_code = 200
|
|
client.get.return_value = resp
|
|
|
|
keys = ingest_entry(store, client, _ENTRY)
|
|
assert len(keys) == 1
|
|
store.upsert.assert_called_once()
|
|
|
|
|
|
class TestIngestAllFiltered:
|
|
def test_with_pub_filter(self):
|
|
store = MagicMock()
|
|
store.upsert.return_value = "KEY1"
|
|
store._con.return_value = MagicMock(
|
|
execute=MagicMock(
|
|
return_value=MagicMock(fetchall=MagicMock(return_value=[]))
|
|
)
|
|
)
|
|
|
|
index_html = """
|
|
<a href="/regulations-and-guidance/guidance/manuals/internet-only-manuals-ioms-items/cms018912">
|
|
100-04
|
|
</a>
|
|
<div><label>Title</label>Claims Processing Manual</div>
|
|
"""
|
|
chapter_html = """
|
|
<a href="/regulations-and-guidance/guidance/manuals/downloads/clm104c01.pdf">
|
|
Chapter 1 - General
|
|
</a>
|
|
"""
|
|
client = MagicMock()
|
|
resp1 = MagicMock(text=index_html, status_code=200)
|
|
resp2 = MagicMock(text=chapter_html, status_code=200)
|
|
client.get.side_effect = [resp1, resp2]
|
|
|
|
result = ingest_all(store, client, pubs=["100-04"])
|
|
assert isinstance(result, dict)
|
|
|
|
def test_skips_unmatched_pubs(self):
|
|
store = MagicMock()
|
|
index_html = """
|
|
<a href="/regulations-and-guidance/guidance/manuals/internet-only-manuals-ioms-items/cms018912">
|
|
100-04
|
|
</a>
|
|
<div><label>Title</label>Claims Processing Manual</div>
|
|
"""
|
|
client = MagicMock()
|
|
resp = MagicMock(text=index_html, status_code=200)
|
|
client.get.return_value = resp
|
|
|
|
result = ingest_all(store, client, pubs=["100-99"])
|
|
assert result == {}
|
|
|
|
def test_handles_exception(self):
|
|
store = MagicMock()
|
|
index_html = """
|
|
<a href="/regulations-and-guidance/guidance/manuals/internet-only-manuals-ioms-items/cms018912">
|
|
100-04
|
|
</a>
|
|
<div><label>Title</label>Claims Processing Manual</div>
|
|
"""
|
|
client = MagicMock()
|
|
resp1 = MagicMock(text=index_html, status_code=200)
|
|
resp2 = MagicMock()
|
|
resp2.raise_for_status.side_effect = Exception("fail")
|
|
client.get.side_effect = [resp1, resp2]
|
|
|
|
result = ingest_all(store, client)
|
|
assert result.get("100-04") == 0
|
|
|
|
|
|
class TestExistingAttachmentHash:
|
|
def test_no_attachment(self):
|
|
store = MagicMock()
|
|
con = MagicMock()
|
|
store._con.return_value = con
|
|
con.execute.return_value.fetchone.return_value = None
|
|
assert _existing_attachment_hash(store, "K1") is None
|
|
|
|
def test_with_file(self, tmp_path):
|
|
f = tmp_path / "test.pdf"
|
|
f.write_bytes(b"pdf content")
|
|
store = MagicMock()
|
|
con = MagicMock()
|
|
store._con.return_value = con
|
|
# newest first; a dead Zotero-relative row is skipped
|
|
con.execute.return_value.fetchall.return_value = [
|
|
{"storage_path": "storage:gone.pdf"},
|
|
{"storage_path": str(f)},
|
|
]
|
|
result = _existing_attachment_hash(store, "K1")
|
|
assert result == _sha256(f)
|
|
|
|
|
|
class TestDownloadAttachments:
|
|
def test_downloads_and_attaches(self, tmp_path):
|
|
store = MagicMock()
|
|
store._db_path = str(tmp_path / "bib.sqlite")
|
|
item = MagicMock()
|
|
item.url = "https://cms.gov/manuals/downloads/ch1.pdf"
|
|
item.key = "K1"
|
|
item.title = "Chapter 1"
|
|
row = {"extra_json": '{"pub_number": "100-04"}'}
|
|
item.to_row.return_value = row
|
|
store.list_items.return_value = [item]
|
|
store._con.return_value.execute.return_value.fetchone.return_value = None
|
|
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.content = b"pdf bytes"
|
|
resp.status_code = 200
|
|
client.get.return_value = resp
|
|
|
|
result = download_attachments(store, client, tmp_dir=tmp_path / "dl")
|
|
assert result.get("100-04", 0) >= 1
|
|
|
|
|
|
class TestFuturepdfAnchor:
|
|
def test_creates_new(self):
|
|
store = MagicMock()
|
|
store.list_items.return_value = []
|
|
store.upsert.return_value = "FKEY"
|
|
key = _futurepdf_anchor(store)
|
|
assert key == "FKEY"
|
|
store.upsert.assert_called_once()
|
|
|
|
def test_returns_existing(self):
|
|
store = MagicMock()
|
|
item = MagicMock()
|
|
item.key = "EXISTING"
|
|
store.list_items.return_value = [item]
|
|
assert _futurepdf_anchor(store) == "EXISTING"
|
|
|
|
|
|
class TestCheckFutureUpdates:
|
|
def test_changed(self, tmp_path):
|
|
store = MagicMock()
|
|
store._db_path = str(tmp_path / "bib.sqlite")
|
|
store.list_items.return_value = []
|
|
store.upsert.return_value = "FKEY"
|
|
store._con.return_value.execute.return_value.fetchone.return_value = None
|
|
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.content = b"new pdf content"
|
|
resp.status_code = 200
|
|
client.get.return_value = resp
|
|
|
|
changed, digest = check_future_updates(store, client, tmp_dir=tmp_path / "dl")
|
|
assert changed is True
|
|
assert len(digest) == 64
|