dbf71a6594
Fix Zotero table models for Zotero 9:
- Remove stale Annotations/Highlights/Transaction* models
- Add ItemAnnotations, RetractedItems, DeletedCollections,
DeletedSearches, DbDebug1
- Fix ItemAttachments, Libraries, Users column mismatches
New test files covering all major modules:
- cli/{bib,prisma,rec,zot,mail,run} deep exercising tests
- mail/{droplet,postmark,resend,cloudflare} lifecycle tests
- bib/{iom,oig,pincite,sync,regulations_gov,email_ingest,format,store}
- prisma/{vpn,fetch,export,llm,screen,eligibility,extract,project,ingest,flow}
- aco/lake/{unity,quality,deploy} + api/aco coverage gaps
- zot/{ops,db,extract,duck} + rec/{report,engine,base,pricers}
- pfs/{pipe,rules,eq,files}
Add pytest-xdist for parallel test execution.
Tracks #353
272 lines
8.2 KiB
Python
272 lines
8.2 KiB
Python
"""Full exercising tests for bib.iom — CMS IOM crawler."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import MagicMock
|
|
|
|
from bib.iom import (
|
|
IOMEntry,
|
|
_sha256,
|
|
_short_manual_name,
|
|
fetch_chapters,
|
|
fetch_index,
|
|
ingest_all,
|
|
ingest_entry,
|
|
)
|
|
|
|
|
|
class TestShortManualName:
|
|
def test_strips_medicare(self):
|
|
assert "Claims Processing" in _short_manual_name(
|
|
"Medicare Claims Processing Manual"
|
|
)
|
|
|
|
def test_preserves_short(self):
|
|
result = _short_manual_name("Short Title")
|
|
assert result == "Short Title"
|
|
|
|
def test_empty(self):
|
|
assert _short_manual_name("") == ""
|
|
|
|
|
|
class TestSha256:
|
|
def test_deterministic(self, tmp_path):
|
|
f = tmp_path / "test.txt"
|
|
f.write_text("hello")
|
|
assert _sha256(f) == _sha256(f)
|
|
assert len(_sha256(f)) == 64
|
|
|
|
|
|
class TestFetchIndex:
|
|
def test_parses_index_page(self):
|
|
html = """
|
|
<table>
|
|
<tr><td><a href="/iom/100-01">Pub 100-01</a></td>
|
|
<td>Medicare General Information</td></tr>
|
|
<tr><td><a href="/iom/100-02">Pub 100-02</a></td>
|
|
<td>Medicare Benefit Policy Manual</td></tr>
|
|
</table>
|
|
"""
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 200
|
|
resp.text = html
|
|
client.get.return_value = resp
|
|
|
|
entries = fetch_index(client)
|
|
assert isinstance(entries, list)
|
|
|
|
|
|
class TestFetchChapters:
|
|
def test_parses_chapter_page(self):
|
|
html = """
|
|
<table>
|
|
<tr><td><a href="/ch1.pdf">Chapter 1</a></td><td>General</td></tr>
|
|
<tr><td><a href="/ch2.pdf">Chapter 2</a></td><td>Specific</td></tr>
|
|
</table>
|
|
"""
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 200
|
|
resp.text = html
|
|
client.get.return_value = resp
|
|
|
|
entry = IOMEntry(
|
|
pub="100-01", title="Test Manual", landing_url="https://cms.gov/iom/100-01"
|
|
)
|
|
chapters = fetch_chapters(client, entry)
|
|
assert isinstance(chapters, list)
|
|
|
|
|
|
class TestIngestEntry:
|
|
def test_upserts_chapters(self):
|
|
store = MagicMock()
|
|
store.upsert.return_value = "KEY1"
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 200
|
|
resp.text = "<table></table>"
|
|
client.get.return_value = resp
|
|
|
|
entry = IOMEntry(
|
|
pub="100-01", title="Test Manual", landing_url="https://cms.gov/iom"
|
|
)
|
|
result = ingest_entry(store, client, entry)
|
|
assert isinstance(result, (int, list))
|
|
|
|
|
|
class TestIngestAll:
|
|
def test_ingests_all_pubs(self):
|
|
store = MagicMock()
|
|
store.upsert.return_value = "KEY1"
|
|
store.list_items.return_value = []
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 200
|
|
resp.text = "<table></table>"
|
|
client.get.return_value = resp
|
|
|
|
result = ingest_all(store, client)
|
|
assert isinstance(result, dict)
|
|
|
|
|
|
class TestFetchChaptersWholePub:
|
|
def test_whole_pub_match(self):
|
|
"""Line 118: fetch_chapters picks up whole-pub entries when no chapter links."""
|
|
html = """
|
|
<div>
|
|
<a href="/regulations-and-guidance/guidance/manuals/downloads/pub100-18.pdf">
|
|
Pub 100-18 - Medicare Prescription Drug Benefit Manual
|
|
</a>
|
|
</div>
|
|
"""
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 200
|
|
resp.text = html
|
|
client.get.return_value = resp
|
|
|
|
entry = IOMEntry(
|
|
pub="100-18",
|
|
title="Medicare Prescription Drug Benefit Manual",
|
|
landing_url="https://cms.gov/iom/100-18",
|
|
)
|
|
chapters = fetch_chapters(client, entry)
|
|
assert isinstance(chapters, list)
|
|
# Should find the whole-pub entry since no chapter links exist
|
|
if chapters:
|
|
assert chapters[0].chapter == ""
|
|
|
|
|
|
class TestDownloadAttachments:
|
|
def test_downloads_and_attaches(self, tmp_path):
|
|
"""Lines 367, 370: download_attachments end-to-end."""
|
|
from bib.iom import download_attachments
|
|
from bib.item import Manual
|
|
from bib.store import Store
|
|
|
|
db = tmp_path / "bib.sqlite"
|
|
store = Store(db, storage_dir=tmp_path / "storage")
|
|
item = Manual(
|
|
title="IOM Ch1",
|
|
url="https://cms.gov/manuals/downloads/clm104c01.pdf",
|
|
pub_number="100-04",
|
|
)
|
|
item.add_tag("source:iom")
|
|
store.create(item)
|
|
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 200
|
|
resp.content = b"PDF content"
|
|
client.get.return_value = resp
|
|
|
|
results = download_attachments(store, client, tmp_dir=tmp_path / "tmp")
|
|
assert isinstance(results, dict)
|
|
assert results.get("100-04", 0) >= 1
|
|
store.close()
|
|
|
|
def test_no_url_skips(self, tmp_path):
|
|
"""Line 370: items without URL are skipped."""
|
|
from bib.iom import download_attachments
|
|
from bib.item import Manual
|
|
from bib.store import Store
|
|
|
|
db = tmp_path / "bib.sqlite"
|
|
store = Store(db, storage_dir=tmp_path / "storage")
|
|
item = Manual(title="No URL", pub_number="100-04")
|
|
item.add_tag("source:iom")
|
|
store.create(item)
|
|
|
|
client = MagicMock()
|
|
results = download_attachments(store, client, tmp_dir=tmp_path / "tmp")
|
|
assert results == {}
|
|
client.get.assert_not_called()
|
|
store.close()
|
|
|
|
def test_download_failure(self, tmp_path):
|
|
"""Lines 377-379: download exception is caught."""
|
|
from bib.iom import download_attachments
|
|
from bib.item import Manual
|
|
from bib.store import Store
|
|
|
|
db = tmp_path / "bib.sqlite"
|
|
store = Store(db, storage_dir=tmp_path / "storage")
|
|
item = Manual(
|
|
title="Fail",
|
|
url="https://cms.gov/manuals/downloads/fail.pdf",
|
|
pub_number="100-04",
|
|
)
|
|
item.add_tag("source:iom")
|
|
store.create(item)
|
|
|
|
client = MagicMock()
|
|
client.get.side_effect = Exception("network fail")
|
|
|
|
results = download_attachments(store, client, tmp_dir=tmp_path / "tmp")
|
|
assert results == {}
|
|
store.close()
|
|
|
|
def test_hash_match_skips(self, tmp_path):
|
|
"""Lines 383, 384: matching SHA-256 skips re-attach."""
|
|
from bib.iom import download_attachments
|
|
from bib.item import Manual
|
|
from bib.store import Store
|
|
|
|
db = tmp_path / "bib.sqlite"
|
|
store = Store(db, storage_dir=tmp_path / "storage")
|
|
item = Manual(
|
|
title="Same",
|
|
url="https://cms.gov/manuals/downloads/same.pdf",
|
|
pub_number="100-04",
|
|
)
|
|
item.add_tag("source:iom")
|
|
store.create(item)
|
|
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 200
|
|
resp.content = b"Same PDF content"
|
|
client.get.return_value = resp
|
|
|
|
# First download
|
|
download_attachments(store, client, tmp_dir=tmp_path / "tmp")
|
|
# Second pass: hash matches → no new attach
|
|
results = download_attachments(store, client, tmp_dir=tmp_path / "tmp")
|
|
assert results.get("100-04", 0) == 0
|
|
store.close()
|
|
|
|
def test_pub_filter(self, tmp_path):
|
|
"""Line 367: pubs filter limits which items are processed."""
|
|
from bib.iom import download_attachments
|
|
from bib.item import Manual
|
|
from bib.store import Store
|
|
|
|
db = tmp_path / "bib.sqlite"
|
|
store = Store(db, storage_dir=tmp_path / "storage")
|
|
item1 = Manual(
|
|
title="A",
|
|
url="https://cms.gov/manuals/downloads/a.pdf",
|
|
pub_number="100-04",
|
|
)
|
|
item2 = Manual(
|
|
title="B",
|
|
url="https://cms.gov/manuals/downloads/b.pdf",
|
|
pub_number="100-02",
|
|
)
|
|
store.create(item1)
|
|
store.create(item2)
|
|
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 200
|
|
resp.content = b"PDF"
|
|
client.get.return_value = resp
|
|
|
|
results = download_attachments(
|
|
store, client, pubs=["100-04"], tmp_dir=tmp_path / "tmp"
|
|
)
|
|
# Only 100-04 should be processed
|
|
assert "100-04" in results or results == {}
|
|
store.close()
|