dbf71a6594
Fix Zotero table models for Zotero 9:
- Remove stale Annotations/Highlights/Transaction* models
- Add ItemAnnotations, RetractedItems, DeletedCollections,
DeletedSearches, DbDebug1
- Fix ItemAttachments, Libraries, Users column mismatches
New test files covering all major modules:
- cli/{bib,prisma,rec,zot,mail,run} deep exercising tests
- mail/{droplet,postmark,resend,cloudflare} lifecycle tests
- bib/{iom,oig,pincite,sync,regulations_gov,email_ingest,format,store}
- prisma/{vpn,fetch,export,llm,screen,eligibility,extract,project,ingest,flow}
- aco/lake/{unity,quality,deploy} + api/aco coverage gaps
- zot/{ops,db,extract,duck} + rec/{report,engine,base,pricers}
- pfs/{pipe,rules,eq,files}
Add pytest-xdist for parallel test execution.
Tracks #353
204 lines
6.0 KiB
Python
204 lines
6.0 KiB
Python
"""Full exercising tests for bib.oig — OIG scraper."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import MagicMock
|
|
|
|
from bib.oig import (
|
|
OIGDoc,
|
|
_build_item,
|
|
_classify_sector,
|
|
_classify_type,
|
|
_extract_docs,
|
|
_fetch,
|
|
fetch_alerts,
|
|
fetch_cpgs,
|
|
ingest_all,
|
|
)
|
|
|
|
|
|
class TestClassifyType:
|
|
def test_cpg(self):
|
|
result = _classify_type(
|
|
"Compliance Program Guidance for Hospitals", "/cpg/hospital"
|
|
)
|
|
assert isinstance(result, str)
|
|
|
|
def test_alert(self):
|
|
result = _classify_type("Special Fraud Alert", "/fraud/alerts")
|
|
assert isinstance(result, str)
|
|
|
|
def test_unknown(self):
|
|
result = _classify_type("Random Document", "/random")
|
|
assert isinstance(result, str)
|
|
|
|
|
|
class TestClassifySector:
|
|
def test_hospital(self):
|
|
result = _classify_sector("Hospital Compliance")
|
|
assert isinstance(result, str)
|
|
|
|
def test_empty(self):
|
|
result = _classify_sector("")
|
|
assert isinstance(result, str)
|
|
|
|
|
|
class TestExtractDocs:
|
|
def test_extracts_links(self):
|
|
html = """
|
|
<div class="field-items">
|
|
<a href="/doc1.pdf">Document One</a>
|
|
<a href="/doc2.pdf">Document Two</a>
|
|
</div>
|
|
"""
|
|
result = _extract_docs(html)
|
|
assert isinstance(result, list)
|
|
|
|
def test_empty_page(self):
|
|
result = _extract_docs("<html></html>")
|
|
assert isinstance(result, list)
|
|
|
|
|
|
class TestFetch:
|
|
def test_returns_text(self):
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 200
|
|
resp.text = "<html>content</html>"
|
|
client.get.return_value = resp
|
|
result = _fetch(client, "https://oig.hhs.gov/test")
|
|
assert "content" in result
|
|
|
|
|
|
class TestBuildItem:
|
|
def test_creates_source(self):
|
|
doc = OIGDoc(
|
|
title="Test CPG",
|
|
url="https://oig.hhs.gov/cpg/test",
|
|
guidance_type="cpg",
|
|
sector="hospitals",
|
|
)
|
|
item = _build_item(doc)
|
|
assert item.title == "Test CPG"
|
|
assert "agency:oig" in item.tags
|
|
|
|
|
|
class TestFetchCpgs:
|
|
def test_returns_docs(self):
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 200
|
|
resp.text = "<html></html>"
|
|
client.get.return_value = resp
|
|
result = fetch_cpgs(client)
|
|
assert isinstance(result, list)
|
|
|
|
|
|
class TestFetchAlerts:
|
|
def test_returns_docs(self):
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 200
|
|
resp.text = "<html></html>"
|
|
client.get.return_value = resp
|
|
result = fetch_alerts(client)
|
|
assert isinstance(result, list)
|
|
|
|
|
|
class TestIngestAll:
|
|
def test_runs(self):
|
|
store = MagicMock()
|
|
store.upsert.return_value = "KEY1"
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 200
|
|
resp.text = "<html></html>"
|
|
client.get.return_value = resp
|
|
result = ingest_all(store, client)
|
|
assert isinstance(result, dict)
|
|
|
|
def test_alerts_only(self):
|
|
"""Lines 240, 241: ingest_all with kinds=['alerts'] only processes alerts."""
|
|
store = MagicMock()
|
|
store.upsert.return_value = "KEY1"
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 200
|
|
resp.text = '<a href="/documents/special-fraud-alerts/test.pdf">SFA Test</a>'
|
|
client.get.return_value = resp
|
|
result = ingest_all(store, client, kinds=["alerts"])
|
|
assert isinstance(result, dict)
|
|
assert "alerts" in result
|
|
|
|
|
|
class TestDownloadAttachments:
|
|
def test_downloads_and_attaches(self, tmp_path):
|
|
"""Lines 268, 269, 289, 302, 303: download_attachments full path."""
|
|
from bib.item import Source
|
|
from bib.oig import download_attachments
|
|
from bib.store import Store
|
|
|
|
db = tmp_path / "bib.sqlite"
|
|
store = Store(db, storage_dir=tmp_path / "storage")
|
|
# Create an OIG item
|
|
item = Source(title="Test CPG", url="https://oig.hhs.gov/documents/test.pdf")
|
|
item.add_tag("agency:oig")
|
|
item.add_tag("source:oig")
|
|
store.create(item, tags=["agency:oig"])
|
|
|
|
# Mock the client
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 200
|
|
resp.content = b"PDF content for test"
|
|
client.get.return_value = resp
|
|
|
|
changed = download_attachments(store, client)
|
|
assert changed >= 1
|
|
store.close()
|
|
|
|
def test_download_failure_skips(self, tmp_path):
|
|
"""Lines 296-298: download failure is caught and skipped."""
|
|
from bib.item import Source
|
|
from bib.oig import download_attachments
|
|
from bib.store import Store
|
|
|
|
db = tmp_path / "bib.sqlite"
|
|
store = Store(db, storage_dir=tmp_path / "storage")
|
|
item = Source(title="Test", url="https://oig.hhs.gov/documents/fail.pdf")
|
|
item.add_tag("agency:oig")
|
|
store.create(item, tags=["agency:oig"])
|
|
|
|
client = MagicMock()
|
|
client.get.side_effect = Exception("network fail")
|
|
|
|
changed = download_attachments(store, client)
|
|
assert changed == 0
|
|
store.close()
|
|
|
|
def test_hash_match_skips(self, tmp_path):
|
|
"""Lines 302, 303: matching hash means no re-download."""
|
|
from bib.item import Source
|
|
from bib.oig import download_attachments
|
|
from bib.store import Store
|
|
|
|
db = tmp_path / "bib.sqlite"
|
|
store = Store(db, storage_dir=tmp_path / "storage")
|
|
item = Source(title="Test", url="https://oig.hhs.gov/documents/same.pdf")
|
|
item.add_tag("agency:oig")
|
|
store.create(item, tags=["agency:oig"])
|
|
|
|
# First download and attach
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 200
|
|
resp.content = b"Same PDF content"
|
|
client.get.return_value = resp
|
|
|
|
download_attachments(store, client)
|
|
|
|
# Second pass — same content → hash match → skip
|
|
changed = download_attachments(store, client)
|
|
assert changed == 0
|
|
store.close()
|