test: mocked integration tests for bib, prisma, zot — tracks #353
Some checks failed
CI / skinny-install (aco) (push) Successful in 1m7s
CI / skinny-install (api) (push) Successful in 33s
CI / skinny-install (bcda) (push) Successful in 39s
CI / skinny-install (bib) (push) Successful in 38s
CI / skinny-install (bls) (push) Successful in 34s
CI / skinny-install (ccw) (push) Successful in 38s
CI / skinny-install (cli) (push) Successful in 34s
CI / skinny-install (cms) (push) Successful in 38s
CI / skinny-install (conf) (push) Successful in 35s
CI / skinny-install (opps) (push) Successful in 33s
CI / skinny-install (perf) (push) Successful in 35s
CI / skinny-install (pfs) (push) Successful in 42s
CI / skinny-install (rex) (push) Successful in 40s
CI / lint-test (push) Failing after 11m44s
Deploy / build-scan-report (push) Failing after 6m13s
Some checks failed
CI / skinny-install (aco) (push) Successful in 1m7s
CI / skinny-install (api) (push) Successful in 33s
CI / skinny-install (bcda) (push) Successful in 39s
CI / skinny-install (bib) (push) Successful in 38s
CI / skinny-install (bls) (push) Successful in 34s
CI / skinny-install (ccw) (push) Successful in 38s
CI / skinny-install (cli) (push) Successful in 34s
CI / skinny-install (cms) (push) Successful in 38s
CI / skinny-install (conf) (push) Successful in 35s
CI / skinny-install (opps) (push) Successful in 33s
CI / skinny-install (perf) (push) Successful in 35s
CI / skinny-install (pfs) (push) Successful in 42s
CI / skinny-install (rex) (push) Successful in 40s
CI / lint-test (push) Failing after 11m44s
Deploy / build-scan-report (push) Failing after 6m13s
80 new tests covering federalregister, email_ingest, regulations_gov, prisma/fetch (incl. deterministic altcha PoW vector), and zot/ops. All external deps mocked (httpx, imaplib, subprocess, pydo). No network calls in CI.
This commit is contained in:
166
tests/bib/test_email_ingest.py
Normal file
166
tests/bib/test_email_ingest.py
Normal file
@@ -0,0 +1,166 @@
|
||||
"""Tests for bib.email_ingest — IMAP → bib ingest."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import email.message
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from bib.email_ingest import (
|
||||
Mailbox,
|
||||
_date_iso,
|
||||
_extract_body,
|
||||
_safe_filename,
|
||||
_sender_domain,
|
||||
_slug,
|
||||
ingest,
|
||||
)
|
||||
|
||||
# ── Pure helpers ────────────────────<E29480><E29480>─────────────────────────────
|
||||
|
||||
|
||||
class TestSlug:
|
||||
def test_basic(self):
|
||||
assert _slug("Hello World!") == "hello-world"
|
||||
|
||||
def test_truncate(self):
|
||||
assert len(_slug("x" * 100)) <= 60
|
||||
|
||||
def test_empty(self):
|
||||
assert _slug("") == ""
|
||||
|
||||
|
||||
class TestSafeFilename:
|
||||
def test_special_chars(self):
|
||||
assert _safe_filename("file (1).pdf") == "file_1_.pdf"
|
||||
|
||||
def test_empty(self):
|
||||
assert _safe_filename("") == "attachment"
|
||||
|
||||
|
||||
class TestSenderDomain:
|
||||
def test_angle_bracket(self):
|
||||
assert _sender_domain("John <john@example.com>") == "example.com"
|
||||
|
||||
def test_bare(self):
|
||||
assert _sender_domain("user@domain.org") == "domain.org"
|
||||
|
||||
def test_no_domain(self):
|
||||
assert _sender_domain("no email here") == ""
|
||||
|
||||
|
||||
class TestDateIso:
|
||||
def test_rfc2822(self):
|
||||
assert _date_iso("Wed, 15 Apr 2026 22:00:35 +0000") == "2026-04-15"
|
||||
|
||||
def test_empty(self):
|
||||
assert _date_iso("") == ""
|
||||
|
||||
def test_garbage(self):
|
||||
assert _date_iso("not a date") == ""
|
||||
|
||||
|
||||
class TestExtractBody:
|
||||
def test_plain_text(self):
|
||||
msg = email.message.EmailMessage()
|
||||
msg.set_content("Hello world")
|
||||
assert _extract_body(msg) == "Hello world\n"
|
||||
|
||||
def test_html_fallback(self):
|
||||
msg = email.message.EmailMessage()
|
||||
msg["Content-Type"] = "multipart/alternative"
|
||||
msg.add_alternative("<p>Hello</p>", subtype="html")
|
||||
body = _extract_body(msg)
|
||||
assert "Hello" in body
|
||||
|
||||
|
||||
# ── ingest (mocked IMAP + Store) ─────────────────────────────────
|
||||
|
||||
|
||||
def _make_raw_email(
|
||||
subject="Test", sender="cms@example.gov", body="Hello", msg_id="<abc@test>"
|
||||
):
|
||||
msg = email.message.EmailMessage()
|
||||
msg["Subject"] = subject
|
||||
msg["From"] = sender
|
||||
msg["Date"] = "Wed, 15 Apr 2026 10:00:00 +0000"
|
||||
msg["Message-ID"] = msg_id
|
||||
msg.set_content(body)
|
||||
return msg.as_bytes()
|
||||
|
||||
|
||||
class TestIngest:
|
||||
@patch("bib.email_ingest.imaplib.IMAP4_SSL")
|
||||
def test_processes_unseen(self, MockIMAP):
|
||||
conn = MockIMAP.return_value
|
||||
conn.login.return_value = ("OK", [])
|
||||
conn.select.return_value = ("OK", [])
|
||||
conn.uid.side_effect = [
|
||||
("OK", [b"1 2"]), # SEARCH
|
||||
(
|
||||
"OK",
|
||||
[(b"1 (RFC822 {100}", _make_raw_email(msg_id="<m1@test>"))],
|
||||
), # FETCH 1
|
||||
("OK", []), # STORE 1
|
||||
(
|
||||
"OK",
|
||||
[
|
||||
(
|
||||
b"2 (RFC822 {100}",
|
||||
_make_raw_email(subject="Second", msg_id="<m2@test>"),
|
||||
)
|
||||
],
|
||||
), # FETCH 2
|
||||
("OK", []), # STORE 2
|
||||
]
|
||||
conn.close.return_value = ("OK", [])
|
||||
conn.logout.return_value = ("OK", [])
|
||||
|
||||
store = MagicMock()
|
||||
store.upsert.return_value = "KEY1"
|
||||
|
||||
mb = Mailbox(
|
||||
host="mail.test", port=993, username="test@test.com", password="pw"
|
||||
)
|
||||
stats = ingest(store, mb, scratch_root=Path("/tmp/test-ingest"))
|
||||
assert stats["seen"] == 2
|
||||
assert stats["ingested"] == 2
|
||||
assert stats["errors"] == 0
|
||||
assert store.upsert.call_count == 2
|
||||
|
||||
@patch("bib.email_ingest.imaplib.IMAP4_SSL")
|
||||
def test_empty_inbox(self, MockIMAP):
|
||||
conn = MockIMAP.return_value
|
||||
conn.login.return_value = ("OK", [])
|
||||
conn.select.return_value = ("OK", [])
|
||||
conn.uid.return_value = ("OK", [b""])
|
||||
conn.close.return_value = ("OK", [])
|
||||
conn.logout.return_value = ("OK", [])
|
||||
|
||||
store = MagicMock()
|
||||
mb = Mailbox(
|
||||
host="mail.test", port=993, username="test@test.com", password="pw"
|
||||
)
|
||||
stats = ingest(store, mb)
|
||||
assert stats["seen"] == 0
|
||||
assert stats["ingested"] == 0
|
||||
|
||||
@patch("bib.email_ingest.imaplib.IMAP4_SSL")
|
||||
def test_limit(self, MockIMAP):
|
||||
conn = MockIMAP.return_value
|
||||
conn.login.return_value = ("OK", [])
|
||||
conn.select.return_value = ("OK", [])
|
||||
conn.uid.side_effect = [
|
||||
("OK", [b"1 2 3 4 5"]), # SEARCH returns 5
|
||||
("OK", [(b"1 (RFC822 {100}", _make_raw_email(msg_id="<l1@test>"))]),
|
||||
("OK", []),
|
||||
]
|
||||
conn.close.return_value = ("OK", [])
|
||||
conn.logout.return_value = ("OK", [])
|
||||
|
||||
store = MagicMock()
|
||||
store.upsert.return_value = "K"
|
||||
mb = Mailbox(host="h", port=993, username="u@d", password="p")
|
||||
stats = ingest(store, mb, limit=1)
|
||||
assert stats["seen"] == 1
|
||||
assert stats["ingested"] == 1
|
||||
178
tests/bib/test_federalregister.py
Normal file
178
tests/bib/test_federalregister.py
Normal file
@@ -0,0 +1,178 @@
|
||||
"""Tests for bib.federalregister — Federal Register API client."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from bib.federalregister import (
|
||||
PFS_TITLE_PATTERN,
|
||||
FRDocument,
|
||||
_to_doc,
|
||||
pfs_rules,
|
||||
search,
|
||||
split_docket_ids,
|
||||
)
|
||||
|
||||
# ── split_docket_ids (pure function) ──────────────────────────────
|
||||
|
||||
|
||||
class TestSplitDocketIds:
|
||||
def test_single_clean(self):
|
||||
assert split_docket_ids(["CMS-1676-P"]) == ["CMS-1676-P"]
|
||||
|
||||
def test_comma_separated(self):
|
||||
result = split_docket_ids(["CMS-1693-F, CMS-1693-IFC, CMS-5522-F3"])
|
||||
assert result == ["CMS-1693-F", "CMS-1693-IFC", "CMS-5522-F3"]
|
||||
|
||||
def test_and_separated(self):
|
||||
result = split_docket_ids(["CMS-1807-F and CMS-4201-F5"])
|
||||
assert result == ["CMS-1807-F", "CMS-4201-F5"]
|
||||
|
||||
def test_mixed(self):
|
||||
result = split_docket_ids(
|
||||
["CMS-1734-F, CMS-1734-IFC, CMS-5531-F and CMS-1744-F"]
|
||||
)
|
||||
assert "CMS-1734-F" in result
|
||||
assert "CMS-1734-IFC" in result
|
||||
assert "CMS-1744-F" in result
|
||||
|
||||
def test_dedup(self):
|
||||
result = split_docket_ids(["CMS-1676-P", "CMS-1676-P"])
|
||||
assert result == ["CMS-1676-P"]
|
||||
|
||||
def test_empty(self):
|
||||
assert split_docket_ids([]) == []
|
||||
|
||||
def test_whitespace(self):
|
||||
assert split_docket_ids([" CMS-1676-P "]) == ["CMS-1676-P"]
|
||||
|
||||
|
||||
# ── _to_doc ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestToDoc:
|
||||
def test_basic(self):
|
||||
raw = {
|
||||
"document_number": "2017-13010",
|
||||
"title": "Revisions to Payment Policies Under the Physician Fee Schedule",
|
||||
"type": "Proposed Rule",
|
||||
"publication_date": "2017-07-21",
|
||||
"docket_ids": ["CMS-1676-P"],
|
||||
"volume": "82",
|
||||
"start_page": "33950",
|
||||
"regulation_id_numbers": ["0938-AS81"],
|
||||
"html_url": "https://www.federalregister.gov/documents/2017-13010",
|
||||
}
|
||||
doc = _to_doc(raw)
|
||||
assert isinstance(doc, FRDocument)
|
||||
assert doc.document_number == "2017-13010"
|
||||
assert doc.type == "Proposed Rule"
|
||||
assert doc.dockets == ["CMS-1676-P"]
|
||||
assert doc.volume == "82"
|
||||
|
||||
def test_missing_fields(self):
|
||||
doc = _to_doc({})
|
||||
assert doc.document_number == ""
|
||||
assert doc.dockets == []
|
||||
assert doc.volume == ""
|
||||
|
||||
|
||||
# ── search (mocked HTTP) ─────────────────────────────────────────
|
||||
|
||||
|
||||
def _mock_fr_response(results, total_pages=1):
|
||||
resp = MagicMock()
|
||||
resp.status_code = 200
|
||||
resp.json.return_value = {
|
||||
"results": results,
|
||||
"total_pages": total_pages,
|
||||
}
|
||||
resp.raise_for_status = MagicMock()
|
||||
return resp
|
||||
|
||||
|
||||
class TestSearch:
|
||||
@patch("bib.federalregister.httpx.Client")
|
||||
def test_yields_documents(self, MockClient):
|
||||
client = MockClient.return_value
|
||||
client.get.return_value = _mock_fr_response(
|
||||
[
|
||||
{
|
||||
"document_number": "2017-13010",
|
||||
"title": "PFS Rule",
|
||||
"type": "Proposed Rule",
|
||||
"publication_date": "2017-07-21",
|
||||
}
|
||||
]
|
||||
)
|
||||
client.close = MagicMock()
|
||||
docs = list(search(term="physician fee schedule"))
|
||||
assert len(docs) == 1
|
||||
assert docs[0].document_number == "2017-13010"
|
||||
|
||||
@patch("bib.federalregister.httpx.Client")
|
||||
def test_pagination(self, MockClient):
|
||||
client = MockClient.return_value
|
||||
page1 = _mock_fr_response(
|
||||
[{"document_number": "A", "title": "", "type": "", "publication_date": ""}],
|
||||
total_pages=2,
|
||||
)
|
||||
page2 = _mock_fr_response(
|
||||
[{"document_number": "B", "title": "", "type": "", "publication_date": ""}],
|
||||
total_pages=2,
|
||||
)
|
||||
client.get.side_effect = [page1, page2]
|
||||
client.close = MagicMock()
|
||||
docs = list(search())
|
||||
assert len(docs) == 2
|
||||
|
||||
@patch("bib.federalregister.httpx.Client")
|
||||
def test_empty_results(self, MockClient):
|
||||
client = MockClient.return_value
|
||||
client.get.return_value = _mock_fr_response([])
|
||||
client.close = MagicMock()
|
||||
docs = list(search(term="nonexistent"))
|
||||
assert docs == []
|
||||
|
||||
|
||||
# ── pfs_rules (mocked HTTP) ──────────────────────────────────────
|
||||
|
||||
|
||||
class TestPfsRules:
|
||||
@patch("bib.federalregister.httpx.Client")
|
||||
def test_filters_by_title(self, MockClient):
|
||||
client = MockClient.return_value
|
||||
client.close = MagicMock()
|
||||
client.get.return_value = _mock_fr_response(
|
||||
[
|
||||
{
|
||||
"document_number": "A",
|
||||
"title": "CY 2024 Payment Policies Under the Physician Fee Schedule",
|
||||
"type": "Proposed Rule",
|
||||
"publication_date": "2023-07-01",
|
||||
},
|
||||
{
|
||||
"document_number": "B",
|
||||
"title": "Unrelated Medicare Rule",
|
||||
"type": "Rule",
|
||||
"publication_date": "2023-07-01",
|
||||
},
|
||||
]
|
||||
)
|
||||
found = pfs_rules(since="2023-01-01")
|
||||
assert len(found) == 1
|
||||
assert found[0].document_number == "A"
|
||||
|
||||
@patch("bib.federalregister.httpx.Client")
|
||||
def test_deduplicates(self, MockClient):
|
||||
client = MockClient.return_value
|
||||
client.close = MagicMock()
|
||||
doc = {
|
||||
"document_number": "A",
|
||||
"title": f"Revisions to {PFS_TITLE_PATTERN} and More",
|
||||
"type": "Rule",
|
||||
"publication_date": "2023-07-01",
|
||||
}
|
||||
client.get.return_value = _mock_fr_response([doc, doc])
|
||||
found = pfs_rules(since="2023-01-01")
|
||||
assert len(found) == 1
|
||||
235
tests/bib/test_regulations_gov.py
Normal file
235
tests/bib/test_regulations_gov.py
Normal file
@@ -0,0 +1,235 @@
|
||||
"""Tests for bib.regulations_gov — reg.gov v4 API client."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from bib.regulations_gov import (
|
||||
Client,
|
||||
Comment,
|
||||
_byline,
|
||||
_docket_from_comment_id,
|
||||
_filename_from,
|
||||
_hash,
|
||||
_parse_comment,
|
||||
_reg_date,
|
||||
_slug,
|
||||
upsert_comment,
|
||||
)
|
||||
|
||||
# ── Pure helpers ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestRegDate:
|
||||
def test_iso_to_space(self):
|
||||
assert _reg_date("2017-08-30T18:35:48Z") == "2017-08-30 18:35:48"
|
||||
|
||||
def test_already_space(self):
|
||||
assert _reg_date("2017-08-30 18:35:48") == "2017-08-30 18:35:48"
|
||||
|
||||
def test_empty(self):
|
||||
assert _reg_date("") == ""
|
||||
|
||||
def test_none(self):
|
||||
assert _reg_date("") == ""
|
||||
|
||||
|
||||
class TestSlug:
|
||||
def test_basic(self):
|
||||
assert _slug("American Medical Association") == "american-medical-association"
|
||||
|
||||
def test_special(self):
|
||||
assert _slug("Test & Co., Inc.") == "test-co-inc"
|
||||
|
||||
def test_truncate(self):
|
||||
assert len(_slug("x" * 100)) <= 40
|
||||
|
||||
|
||||
class TestFilenameFrom:
|
||||
def test_url(self):
|
||||
assert (
|
||||
_filename_from(
|
||||
"https://downloads.regulations.gov/CMS-2021-0119-26001/attachment_1.pdf"
|
||||
)
|
||||
== "attachment_1.pdf"
|
||||
)
|
||||
|
||||
def test_with_query(self):
|
||||
assert _filename_from("https://x.com/file.pdf?token=abc") == "file.pdf"
|
||||
|
||||
def test_empty(self):
|
||||
assert _filename_from("") == ""
|
||||
|
||||
|
||||
class TestHash:
|
||||
def test_deterministic(self):
|
||||
assert _hash("test") == _hash("test")
|
||||
assert len(_hash("test")) == 10
|
||||
|
||||
|
||||
class TestByline:
|
||||
def test_org(self):
|
||||
c = Comment(
|
||||
id="1",
|
||||
title="",
|
||||
posted_date="",
|
||||
received_date="",
|
||||
docket_id="",
|
||||
comment_on_id="",
|
||||
organization="AMA",
|
||||
)
|
||||
assert _byline(c) == "AMA"
|
||||
|
||||
def test_name(self):
|
||||
c = Comment(
|
||||
id="1",
|
||||
title="",
|
||||
posted_date="",
|
||||
received_date="",
|
||||
docket_id="",
|
||||
comment_on_id="",
|
||||
first_name="John",
|
||||
last_name="Doe",
|
||||
)
|
||||
assert _byline(c) == "John Doe"
|
||||
|
||||
def test_empty(self):
|
||||
c = Comment(
|
||||
id="1",
|
||||
title="",
|
||||
posted_date="",
|
||||
received_date="",
|
||||
docket_id="",
|
||||
comment_on_id="",
|
||||
)
|
||||
assert _byline(c) == ""
|
||||
|
||||
|
||||
class TestDocketFromCommentId:
|
||||
def test_standard(self):
|
||||
assert _docket_from_comment_id("CMS-2025-0304-14107") == "CMS-2025-0304"
|
||||
|
||||
def test_short(self):
|
||||
assert _docket_from_comment_id("AB") == "AB"
|
||||
|
||||
|
||||
# ── _parse_comment ────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestParseComment:
|
||||
def test_basic(self):
|
||||
row = {
|
||||
"id": "CMS-2017-0092-0002",
|
||||
"attributes": {
|
||||
"title": "Test Comment",
|
||||
"postedDate": "2017-07-18T00:00:00Z",
|
||||
"receivedDate": "2017-07-15T00:00:00Z",
|
||||
"docketId": "CMS-2017-0092",
|
||||
"commentOnId": "0900006482921ba1",
|
||||
"firstName": "Jane",
|
||||
"lastName": "Doe",
|
||||
"organization": "AMA",
|
||||
"comment": "This is the body.",
|
||||
"attachmentCount": 2,
|
||||
},
|
||||
}
|
||||
c = _parse_comment(row)
|
||||
assert c.id == "CMS-2017-0092-0002"
|
||||
assert c.title == "Test Comment"
|
||||
assert c.posted_date == "2017-07-18"
|
||||
assert c.organization == "AMA"
|
||||
assert c.comment_text == "This is the body."
|
||||
assert c.attachment_count == 2
|
||||
|
||||
def test_missing_attrs(self):
|
||||
c = _parse_comment({"id": "X", "attributes": {}})
|
||||
assert c.id == "X"
|
||||
assert c.title == ""
|
||||
assert c.attachment_count == 0
|
||||
|
||||
|
||||
# ── upsert_comment ────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestUpsertComment:
|
||||
def test_creates_item(self):
|
||||
store = MagicMock()
|
||||
store.upsert.return_value = "KEY1"
|
||||
c = Comment(
|
||||
id="CMS-2017-0092-0002",
|
||||
title="Test",
|
||||
posted_date="2017-07-18",
|
||||
received_date="2017-07-15",
|
||||
docket_id="CMS-2017-0092",
|
||||
comment_on_id="obj123",
|
||||
organization="AMA",
|
||||
)
|
||||
key = upsert_comment(store, c, cms_id="CMS-1676-P")
|
||||
assert key == "KEY1"
|
||||
store.upsert.assert_called_once()
|
||||
item = store.upsert.call_args[0][0]
|
||||
assert "AMA" in item.title
|
||||
assert item.url == "https://www.regulations.gov/comment/CMS-2017-0092-0002"
|
||||
|
||||
|
||||
# ── Client (mocked httpx) ────────────────────────────────────────
|
||||
|
||||
|
||||
class TestClient:
|
||||
@patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test-key"})
|
||||
def test_init(self):
|
||||
c = Client(sleep=0)
|
||||
assert c._key == "test-key"
|
||||
c.close()
|
||||
|
||||
@patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": ""})
|
||||
def test_init_no_key(self):
|
||||
with pytest.raises(RuntimeError, match="REGULATIONS_GOV_API_KEY"):
|
||||
Client()
|
||||
|
||||
@patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test-key"})
|
||||
def test_get_retry_429(self):
|
||||
mock_client = MagicMock()
|
||||
resp_429 = MagicMock()
|
||||
resp_429.status_code = 429
|
||||
resp_ok = MagicMock()
|
||||
resp_ok.status_code = 200
|
||||
resp_ok.json.return_value = {"data": []}
|
||||
resp_ok.raise_for_status = MagicMock()
|
||||
mock_client.get.side_effect = [resp_429, resp_ok]
|
||||
|
||||
with patch("bib.regulations_gov.time.sleep"):
|
||||
c = Client(sleep=0, client=mock_client)
|
||||
result = c._get("/test")
|
||||
assert result == {"data": []}
|
||||
|
||||
@patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test-key"})
|
||||
def test_resolve_docket(self):
|
||||
mock_client = MagicMock()
|
||||
resp = MagicMock()
|
||||
resp.status_code = 200
|
||||
resp.json.return_value = {
|
||||
"data": [{"attributes": {"docketId": "CMS-2017-0092"}}]
|
||||
}
|
||||
resp.raise_for_status = MagicMock()
|
||||
mock_client.get.return_value = resp
|
||||
|
||||
with patch("bib.regulations_gov.time.sleep"):
|
||||
c = Client(sleep=0, client=mock_client)
|
||||
result = c.resolve_docket("CMS-1676-P")
|
||||
assert result == "CMS-2017-0092"
|
||||
|
||||
@patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test-key"})
|
||||
def test_resolve_docket_not_found(self):
|
||||
mock_client = MagicMock()
|
||||
resp = MagicMock()
|
||||
resp.status_code = 200
|
||||
resp.json.return_value = {"data": []}
|
||||
resp.raise_for_status = MagicMock()
|
||||
mock_client.get.return_value = resp
|
||||
|
||||
with patch("bib.regulations_gov.time.sleep"):
|
||||
c = Client(sleep=0, client=mock_client)
|
||||
assert c.resolve_docket("NONEXISTENT") is None
|
||||
0
tests/mail/__init__.py
Normal file
0
tests/mail/__init__.py
Normal file
0
tests/prisma/__init__.py
Normal file
0
tests/prisma/__init__.py
Normal file
284
tests/prisma/test_fetch.py
Normal file
284
tests/prisma/test_fetch.py
Normal file
@@ -0,0 +1,284 @@
|
||||
"""Tests for prisma.fetch — PDF retrieval with altcha solver."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from prisma.fetch import (
|
||||
PendingItem,
|
||||
_altcha_bootstrap,
|
||||
_extract_pdf_url,
|
||||
_solve_altcha_pow,
|
||||
download,
|
||||
fetch_one,
|
||||
fetch_pmc,
|
||||
fetch_unpaywall,
|
||||
resolve_doi_from_title,
|
||||
)
|
||||
|
||||
# ── Altcha solver (pure, deterministic) ───────────────────────────
|
||||
|
||||
|
||||
class TestAltchaPow:
|
||||
def test_known_vector(self):
|
||||
salt = "testsalt"
|
||||
target_n = 42
|
||||
challenge = hashlib.sha256(f"{salt}{target_n}".encode()).hexdigest()
|
||||
result = _solve_altcha_pow(salt, challenge, 1000)
|
||||
assert result is not None
|
||||
nonce, elapsed = result
|
||||
assert nonce == 42
|
||||
assert elapsed >= 0
|
||||
|
||||
def test_not_found(self):
|
||||
result = _solve_altcha_pow("salt", "impossible" * 4, 10)
|
||||
assert result is None
|
||||
|
||||
def test_zero_nonce(self):
|
||||
salt = "zero"
|
||||
challenge = hashlib.sha256(f"{salt}0".encode()).hexdigest()
|
||||
result = _solve_altcha_pow(salt, challenge, 100)
|
||||
assert result is not None
|
||||
assert result[0] == 0
|
||||
|
||||
|
||||
# ── _extract_pdf_url ──────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestExtractPdfUrl:
|
||||
def test_citation_meta(self):
|
||||
html = '<meta name="citation_pdf_url" content="/storage/test.pdf">'
|
||||
assert (
|
||||
_extract_pdf_url(html, "https://example.com")
|
||||
== "https://example.com/storage/test.pdf"
|
||||
)
|
||||
|
||||
def test_embed_src(self):
|
||||
html = '<embed src="//cdn.example.com/paper.pdf" type="application/pdf">'
|
||||
assert (
|
||||
_extract_pdf_url(html, "https://x.com")
|
||||
== "https://cdn.example.com/paper.pdf"
|
||||
)
|
||||
|
||||
def test_no_match(self):
|
||||
assert _extract_pdf_url("<html>no pdf here</html>", "https://x.com") is None
|
||||
|
||||
def test_absolute_url(self):
|
||||
html = '<meta name="citation_pdf_url" content="https://full.url/doc.pdf">'
|
||||
assert _extract_pdf_url(html, "https://x.com") == "https://full.url/doc.pdf"
|
||||
|
||||
|
||||
# ── _altcha_bootstrap ─────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestAltchaBootstrap:
|
||||
def test_solves_challenge(self):
|
||||
salt = "bootsalt"
|
||||
nonce = 77
|
||||
challenge = hashlib.sha256(f"{salt}{nonce}".encode()).hexdigest()
|
||||
|
||||
client = MagicMock()
|
||||
challenge_resp = MagicMock()
|
||||
challenge_resp.json.return_value = {
|
||||
"algorithm": "SHA-256",
|
||||
"salt": salt,
|
||||
"challenge": challenge,
|
||||
"maxNumber": 1000,
|
||||
"signature": "sig",
|
||||
}
|
||||
solve_resp = MagicMock()
|
||||
solve_resp.status_code = 200
|
||||
solve_resp.json.return_value = {"success": True}
|
||||
|
||||
client.get.return_value = challenge_resp
|
||||
client.post.return_value = solve_resp
|
||||
|
||||
html = (
|
||||
'<altcha-widget challengeurl="/captcha/challenge/123">'
|
||||
"fetch('/captcha/solution/123'"
|
||||
)
|
||||
result = _altcha_bootstrap(client, "https://mirror.test/doi", html)
|
||||
assert result is True
|
||||
|
||||
def test_no_challenge(self):
|
||||
assert (
|
||||
_altcha_bootstrap(MagicMock(), "https://x.com/doi", "<html></html>")
|
||||
is False
|
||||
)
|
||||
|
||||
|
||||
# ── fetch_unpaywall ───────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestFetchUnpaywall:
|
||||
def test_returns_pdf_url(self):
|
||||
client = MagicMock()
|
||||
resp = MagicMock()
|
||||
resp.status_code = 200
|
||||
resp.json.return_value = {
|
||||
"best_oa_location": {"url_for_pdf": "https://oa.org/paper.pdf"}
|
||||
}
|
||||
client.get.return_value = resp
|
||||
assert (
|
||||
fetch_unpaywall(client, "10.1234/test", "e@x.com")
|
||||
== "https://oa.org/paper.pdf"
|
||||
)
|
||||
|
||||
def test_no_doi(self):
|
||||
assert fetch_unpaywall(MagicMock(), "", "e@x.com") is None
|
||||
|
||||
def test_404(self):
|
||||
client = MagicMock()
|
||||
resp = MagicMock()
|
||||
resp.status_code = 404
|
||||
client.get.return_value = resp
|
||||
assert fetch_unpaywall(client, "10.1234/x", "e@x.com") is None
|
||||
|
||||
|
||||
# ── fetch_pmc ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestFetchPmc:
|
||||
def test_with_prefix(self):
|
||||
assert (
|
||||
fetch_pmc(MagicMock(), "PMC12345")
|
||||
== "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC12345/pdf/"
|
||||
)
|
||||
|
||||
def test_without_prefix(self):
|
||||
assert (
|
||||
fetch_pmc(MagicMock(), "12345")
|
||||
== "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC12345/pdf/"
|
||||
)
|
||||
|
||||
def test_empty(self):
|
||||
assert fetch_pmc(MagicMock(), "") is None
|
||||
|
||||
|
||||
# ── resolve_doi_from_title ────────────────────────────────────────
|
||||
|
||||
|
||||
class TestResolveDoi:
|
||||
def test_match(self):
|
||||
client = MagicMock()
|
||||
resp = MagicMock()
|
||||
resp.status_code = 200
|
||||
resp.json.return_value = {
|
||||
"message": {
|
||||
"items": [
|
||||
{
|
||||
"DOI": "10.1234/found",
|
||||
"title": ["Skin substitutes in wound care"],
|
||||
"score": 80,
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
client.get.return_value = resp
|
||||
assert (
|
||||
resolve_doi_from_title(client, "Skin substitutes in wound care")
|
||||
== "10.1234/found"
|
||||
)
|
||||
|
||||
def test_low_score(self):
|
||||
client = MagicMock()
|
||||
resp = MagicMock()
|
||||
resp.status_code = 200
|
||||
resp.json.return_value = {
|
||||
"message": {
|
||||
"items": [{"DOI": "10.1234/x", "title": ["Unrelated"], "score": 10}]
|
||||
}
|
||||
}
|
||||
client.get.return_value = resp
|
||||
assert resolve_doi_from_title(client, "Skin substitutes in wound care") is None
|
||||
|
||||
def test_short_title(self):
|
||||
assert resolve_doi_from_title(MagicMock(), "Short") is None
|
||||
|
||||
|
||||
# ── download ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestDownload:
|
||||
def test_valid_pdf(self, tmp_path):
|
||||
client = MagicMock()
|
||||
stream_ctx = MagicMock()
|
||||
stream_ctx.__enter__ = MagicMock(return_value=stream_ctx)
|
||||
stream_ctx.__exit__ = MagicMock(return_value=False)
|
||||
stream_ctx.status_code = 200
|
||||
stream_ctx.headers = {"content-type": "application/pdf"}
|
||||
stream_ctx.iter_bytes.return_value = [b"%PDF-1.7\n" + b"x" * 2000]
|
||||
client.stream.return_value = stream_ctx
|
||||
|
||||
dest = tmp_path / "test.pdf"
|
||||
result = download(client, "https://x.com/paper.pdf", dest)
|
||||
assert result == dest
|
||||
assert dest.read_bytes().startswith(b"%PDF")
|
||||
|
||||
def test_not_pdf(self, tmp_path):
|
||||
client = MagicMock()
|
||||
stream_ctx = MagicMock()
|
||||
stream_ctx.__enter__ = MagicMock(return_value=stream_ctx)
|
||||
stream_ctx.__exit__ = MagicMock(return_value=False)
|
||||
stream_ctx.status_code = 200
|
||||
stream_ctx.headers = {"content-type": "text/html"}
|
||||
stream_ctx.iter_bytes.return_value = [b"<html>Not a PDF</html>" + b"x" * 2000]
|
||||
client.stream.return_value = stream_ctx
|
||||
|
||||
dest = tmp_path / "bad.pdf"
|
||||
result = download(client, "https://x.com/bad", dest)
|
||||
assert result is None
|
||||
|
||||
def test_too_small(self, tmp_path):
|
||||
client = MagicMock()
|
||||
stream_ctx = MagicMock()
|
||||
stream_ctx.__enter__ = MagicMock(return_value=stream_ctx)
|
||||
stream_ctx.__exit__ = MagicMock(return_value=False)
|
||||
stream_ctx.status_code = 200
|
||||
stream_ctx.headers = {}
|
||||
stream_ctx.iter_bytes.return_value = [b"%PDF"]
|
||||
client.stream.return_value = stream_ctx
|
||||
|
||||
dest = tmp_path / "tiny.pdf"
|
||||
result = download(client, "https://x.com/tiny", dest)
|
||||
assert result is None
|
||||
|
||||
|
||||
# ── fetch_one cascade ─────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestFetchOne:
|
||||
def test_unpaywall_hit(self, tmp_path):
|
||||
item = PendingItem(
|
||||
zot_id=1, doi="10.1234/test", pmid="", pmcid="", title="Test"
|
||||
)
|
||||
client = MagicMock()
|
||||
resp = MagicMock()
|
||||
resp.status_code = 200
|
||||
resp.json.return_value = {
|
||||
"best_oa_location": {"url_for_pdf": "https://oa.org/p.pdf"}
|
||||
}
|
||||
client.get.return_value = resp
|
||||
|
||||
stream_ctx = MagicMock()
|
||||
stream_ctx.__enter__ = MagicMock(return_value=stream_ctx)
|
||||
stream_ctx.__exit__ = MagicMock(return_value=False)
|
||||
stream_ctx.status_code = 200
|
||||
stream_ctx.headers = {"content-type": "application/pdf"}
|
||||
stream_ctx.iter_bytes.return_value = [b"%PDF-1.7\n" + b"x" * 2000]
|
||||
client.stream.return_value = stream_ctx
|
||||
|
||||
result = fetch_one(item, email="e@x.com", scratch=tmp_path, client=client)
|
||||
assert result is not None
|
||||
source, path = result
|
||||
assert source == "unpaywall"
|
||||
|
||||
def test_all_miss(self, tmp_path):
|
||||
item = PendingItem(zot_id=1, doi="", pmid="", pmcid="", title="X")
|
||||
client = MagicMock()
|
||||
resp = MagicMock()
|
||||
resp.status_code = 404
|
||||
client.get.return_value = resp
|
||||
result = fetch_one(item, email="e@x.com", scratch=tmp_path, client=client)
|
||||
assert result is None
|
||||
62
tests/zot/test_ops.py
Normal file
62
tests/zot/test_ops.py
Normal file
@@ -0,0 +1,62 @@
|
||||
"""Tests for zot.ops — Zotero maintenance operations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.zot.test_db import ZOTERO_SCHEMA, _seed_schema_maps
|
||||
from zot.db import TYPE_MAP, Db
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def zot_path(tmp_path):
|
||||
path = str(tmp_path / "zotero.sqlite")
|
||||
con = sqlite3.connect(path)
|
||||
con.executescript(ZOTERO_SCHEMA)
|
||||
_seed_schema_maps(con)
|
||||
con.commit()
|
||||
con.close()
|
||||
return path
|
||||
|
||||
|
||||
class TestDumpSchema:
|
||||
def test_dumps_types(self, zot_path):
|
||||
from zot.ops import dump_schema
|
||||
|
||||
result = dump_schema(zot_path)
|
||||
assert "TYPE_MAP" in result
|
||||
assert "FIELD_MAP" in result
|
||||
assert "CREATOR_TYPES" in result
|
||||
assert "attachment" in result["TYPE_MAP"]
|
||||
|
||||
|
||||
class TestFixDates:
|
||||
def test_runs(self, zot_path):
|
||||
from zot.ops import fix_dates
|
||||
|
||||
with Db(zot_path) as db:
|
||||
item_id = db.create_item(TYPE_MAP["document"])
|
||||
db.set_fields(item_id, {"date": "2023-07-15"})
|
||||
result = fix_dates(zot_path)
|
||||
assert isinstance(result, dict)
|
||||
|
||||
|
||||
class TestFixKeys:
|
||||
def test_runs(self, zot_path):
|
||||
from zot.ops import fix_keys
|
||||
|
||||
result = fix_keys(zot_path)
|
||||
assert isinstance(result, dict)
|
||||
|
||||
|
||||
class TestFixFields:
|
||||
@pytest.mark.skip(
|
||||
reason="requires itemTypeFieldsCombined view from full Zotero schema"
|
||||
)
|
||||
def test_runs(self, zot_path):
|
||||
from zot.ops import fix_fields
|
||||
|
||||
result = fix_fields(zot_path)
|
||||
assert isinstance(result, dict)
|
||||
Reference in New Issue
Block a user