Files
stack/tests/bib/test_iom_exercise.py
kert 2f889ff9fc
Some checks failed
CI / lint (push) Successful in 1m10s
CI / notebooks-smoke (push) Successful in 2m53s
Deploy / changes (push) Successful in 2m7s
CI / test (push) Has been cancelled
Deploy / report (push) Has been cancelled
Deploy / bcda (push) Has been cancelled
Deploy / notebooks (push) Has been cancelled
Deploy / zotero (push) Has been cancelled
Deploy / docs (push) Has been cancelled
Deploy / api (push) Has been cancelled
Deploy / llm (push) Has been cancelled
Deploy / mc (push) Has been cancelled
Infra CI / zotero (push) Has been cancelled
Infra CI / docs (push) Has been cancelled
Infra CI / api (push) Has been cancelled
Infra CI / bcda (push) Has been cancelled
Infra CI / llm (push) Has been cancelled
Infra CI / mc (push) Has been cancelled
Infra CI / notebooks (push) Has started running
fix(bib): hoard IOM revisions instead of discarding them
attach-iom compared each download with an item's *first* attachment
(a dead Zotero-relative storage: row on 29 manuals), then called
attach_file with the same title — which returns the existing attachment
without copying. Every revised chapter since 2026-04-15 was downloaded
nightly and thrown away; bib and Zotero both still hold the first copy.

Revisions are now kept, never replaced: compared against the newest
copy on disk and attached as <stem>.<YYYY-MM-DD><ext> titled
'<title> (<date>)'. Zotero sync likewise never overwrites: a changed
same-name file is added beside the old one under a dated name, and a
missing Zotero file is restored. The corpus index reads only the newest
member of each version family, so hoarded revisions are not indexed
twice.
2026-11-11 14:20:35 -05:00

300 lines
9.6 KiB
Python

"""Exercise bib.iom — covers fetch_chapters regex parsing, download, check_future."""
from __future__ import annotations
from unittest.mock import MagicMock
from bib.iom import (
IOMEntry,
_existing_attachment_hash,
_futurepdf_anchor,
_prune_stale,
_sha256,
check_future_updates,
download_attachments,
fetch_chapters,
fetch_index,
ingest_all,
ingest_entry,
)
_ENTRY = IOMEntry(
pub="100-04",
title="Medicare Claims Processing Manual",
landing_url="https://www.cms.gov/regulations-and-guidance/guidance/manuals/internet-only-manuals-ioms-items/cms018912",
)
class TestFetchChaptersRegex:
def _client(self, html):
client = MagicMock()
resp = MagicMock()
resp.text = html
resp.status_code = 200
client.get.return_value = resp
return client
def test_matches_chapters(self):
html = """
<a href="/regulations-and-guidance/guidance/manuals/downloads/clm104c01.pdf">
Chapter 1 - General Billing Requirements
</a>
<a href="/regulations-and-guidance/guidance/manuals/downloads/clm104c02.pdf">
Chapter 2 - Submission of Claims
</a>
"""
chapters = fetch_chapters(self._client(html), _ENTRY)
assert len(chapters) == 2
assert chapters[0].chapter == "1"
assert "General Billing" in chapters[0].title
def test_skips_supplements(self):
html = """
<a href="/regulations-and-guidance/guidance/manuals/downloads/clm104c01.pdf">
Chapter 1 - General
</a>
<a href="/regulations-and-guidance/guidance/manuals/downloads/crosswalk.pdf">
Crosswalk of Changes
</a>
"""
chapters = fetch_chapters(self._client(html), _ENTRY)
assert len(chapters) == 1
def test_whole_pub_fallback(self):
html = """
<a href="/regulations-and-guidance/guidance/manuals/downloads/pub100_18.pdf">
Pub 100-18 - Medicare Prescription Drug Benefit Manual
</a>
"""
chapters = fetch_chapters(self._client(html), _ENTRY)
assert len(chapters) == 1
assert chapters[0].chapter == ""
def test_dedup_chapters(self):
html = """
<a href="/regulations-and-guidance/guidance/manuals/downloads/clm104c01.pdf">
Chapter 1 - General
</a>
<a href="/regulations-and-guidance/guidance/manuals/downloads/clm104c01v2.pdf">
Chapter 1 - General (Updated)
</a>
"""
chapters = fetch_chapters(self._client(html), _ENTRY)
assert len(chapters) == 1
def test_part_chapters(self):
html = """
<a href="/regulations-and-guidance/guidance/manuals/downloads/ncd103c03p1.pdf">
Chapter 3, Part 1 — Section A
</a>
<a href="/regulations-and-guidance/guidance/manuals/downloads/ncd103c03p2.pdf">
Chapter 3, Part 2 — Section B
</a>
"""
chapters = fetch_chapters(self._client(html), _ENTRY)
assert len(chapters) == 2
assert chapters[0].chapter in ("3P1", "3p1")
class TestFetchIndex:
def test_real_html_pattern(self):
html = """
<a href="/regulations-and-guidance/guidance/manuals/internet-only-manuals-ioms-items/cms018912"
class="title-link">
100-04
</a>
<div><label>Title</label>Medicare Claims Processing Manual</div>
"""
client = MagicMock()
resp = MagicMock()
resp.text = html
resp.status_code = 200
client.get.return_value = resp
entries = fetch_index(client)
assert len(entries) == 1
assert entries[0].pub == "100-04"
class TestPruneStale:
def test_prunes(self):
store = MagicMock()
con = MagicMock()
store._con.return_value = con
con.execute.return_value.fetchall.return_value = [
{"key": "K1", "url": "https://old.pdf"},
{"key": "K2", "url": "https://live.pdf"},
]
n = _prune_stale(store, "100-04", {"https://live.pdf"})
assert n == 1
store.delete.assert_called_once_with("K1")
class TestIngestEntryDeep:
def test_with_chapters(self):
store = MagicMock()
store.upsert.return_value = "KEY1"
store._con.return_value = MagicMock(
execute=MagicMock(
return_value=MagicMock(fetchall=MagicMock(return_value=[]))
)
)
html = """
<a href="/regulations-and-guidance/guidance/manuals/downloads/clm104c01.pdf">
Chapter 1 - General
</a>
"""
client = MagicMock()
resp = MagicMock()
resp.text = html
resp.status_code = 200
client.get.return_value = resp
keys = ingest_entry(store, client, _ENTRY)
assert len(keys) == 1
store.upsert.assert_called_once()
class TestIngestAllFiltered:
def test_with_pub_filter(self):
store = MagicMock()
store.upsert.return_value = "KEY1"
store._con.return_value = MagicMock(
execute=MagicMock(
return_value=MagicMock(fetchall=MagicMock(return_value=[]))
)
)
index_html = """
<a href="/regulations-and-guidance/guidance/manuals/internet-only-manuals-ioms-items/cms018912">
100-04
</a>
<div><label>Title</label>Claims Processing Manual</div>
"""
chapter_html = """
<a href="/regulations-and-guidance/guidance/manuals/downloads/clm104c01.pdf">
Chapter 1 - General
</a>
"""
client = MagicMock()
resp1 = MagicMock(text=index_html, status_code=200)
resp2 = MagicMock(text=chapter_html, status_code=200)
client.get.side_effect = [resp1, resp2]
result = ingest_all(store, client, pubs=["100-04"])
assert isinstance(result, dict)
def test_skips_unmatched_pubs(self):
store = MagicMock()
index_html = """
<a href="/regulations-and-guidance/guidance/manuals/internet-only-manuals-ioms-items/cms018912">
100-04
</a>
<div><label>Title</label>Claims Processing Manual</div>
"""
client = MagicMock()
resp = MagicMock(text=index_html, status_code=200)
client.get.return_value = resp
result = ingest_all(store, client, pubs=["100-99"])
assert result == {}
def test_handles_exception(self):
store = MagicMock()
index_html = """
<a href="/regulations-and-guidance/guidance/manuals/internet-only-manuals-ioms-items/cms018912">
100-04
</a>
<div><label>Title</label>Claims Processing Manual</div>
"""
client = MagicMock()
resp1 = MagicMock(text=index_html, status_code=200)
resp2 = MagicMock()
resp2.raise_for_status.side_effect = Exception("fail")
client.get.side_effect = [resp1, resp2]
result = ingest_all(store, client)
assert result.get("100-04") == 0
class TestExistingAttachmentHash:
def test_no_attachment(self):
store = MagicMock()
con = MagicMock()
store._con.return_value = con
con.execute.return_value.fetchone.return_value = None
assert _existing_attachment_hash(store, "K1") is None
def test_with_file(self, tmp_path):
f = tmp_path / "test.pdf"
f.write_bytes(b"pdf content")
store = MagicMock()
con = MagicMock()
store._con.return_value = con
# newest first; a dead Zotero-relative row is skipped
con.execute.return_value.fetchall.return_value = [
{"storage_path": "storage:gone.pdf"},
{"storage_path": str(f)},
]
result = _existing_attachment_hash(store, "K1")
assert result == _sha256(f)
class TestDownloadAttachments:
def test_downloads_and_attaches(self, tmp_path):
store = MagicMock()
store._db_path = str(tmp_path / "bib.sqlite")
item = MagicMock()
item.url = "https://cms.gov/manuals/downloads/ch1.pdf"
item.key = "K1"
item.title = "Chapter 1"
row = {"extra_json": '{"pub_number": "100-04"}'}
item.to_row.return_value = row
store.list_items.return_value = [item]
store._con.return_value.execute.return_value.fetchone.return_value = None
client = MagicMock()
resp = MagicMock()
resp.content = b"pdf bytes"
resp.status_code = 200
client.get.return_value = resp
result = download_attachments(store, client, tmp_dir=tmp_path / "dl")
assert result.get("100-04", 0) >= 1
class TestFuturepdfAnchor:
def test_creates_new(self):
store = MagicMock()
store.list_items.return_value = []
store.upsert.return_value = "FKEY"
key = _futurepdf_anchor(store)
assert key == "FKEY"
store.upsert.assert_called_once()
def test_returns_existing(self):
store = MagicMock()
item = MagicMock()
item.key = "EXISTING"
store.list_items.return_value = [item]
assert _futurepdf_anchor(store) == "EXISTING"
class TestCheckFutureUpdates:
def test_changed(self, tmp_path):
store = MagicMock()
store._db_path = str(tmp_path / "bib.sqlite")
store.list_items.return_value = []
store.upsert.return_value = "FKEY"
store._con.return_value.execute.return_value.fetchone.return_value = None
client = MagicMock()
resp = MagicMock()
resp.content = b"new pdf content"
resp.status_code = 200
client.get.return_value = resp
changed, digest = check_future_updates(store, client, tmp_dir=tmp_path / "dl")
assert changed is True
assert len(digest) == 64