test: mocked integration tests for bib, prisma, zot — tracks #353
Some checks failed
CI / skinny-install (aco) (push) Successful in 1m7s
CI / skinny-install (api) (push) Successful in 33s
CI / skinny-install (bcda) (push) Successful in 39s
CI / skinny-install (bib) (push) Successful in 38s
CI / skinny-install (bls) (push) Successful in 34s
CI / skinny-install (ccw) (push) Successful in 38s
CI / skinny-install (cli) (push) Successful in 34s
CI / skinny-install (cms) (push) Successful in 38s
CI / skinny-install (conf) (push) Successful in 35s
CI / skinny-install (opps) (push) Successful in 33s
CI / skinny-install (perf) (push) Successful in 35s
CI / skinny-install (pfs) (push) Successful in 42s
CI / skinny-install (rex) (push) Successful in 40s
CI / lint-test (push) Failing after 11m44s
Deploy / build-scan-report (push) Failing after 6m13s
Some checks failed
CI / skinny-install (aco) (push) Successful in 1m7s
CI / skinny-install (api) (push) Successful in 33s
CI / skinny-install (bcda) (push) Successful in 39s
CI / skinny-install (bib) (push) Successful in 38s
CI / skinny-install (bls) (push) Successful in 34s
CI / skinny-install (ccw) (push) Successful in 38s
CI / skinny-install (cli) (push) Successful in 34s
CI / skinny-install (cms) (push) Successful in 38s
CI / skinny-install (conf) (push) Successful in 35s
CI / skinny-install (opps) (push) Successful in 33s
CI / skinny-install (perf) (push) Successful in 35s
CI / skinny-install (pfs) (push) Successful in 42s
CI / skinny-install (rex) (push) Successful in 40s
CI / lint-test (push) Failing after 11m44s
Deploy / build-scan-report (push) Failing after 6m13s
80 new tests covering federalregister, email_ingest, regulations_gov, prisma/fetch (incl. deterministic altcha PoW vector), and zot/ops. All external deps mocked (httpx, imaplib, subprocess, pydo). No network calls in CI.
This commit is contained in:
166
tests/bib/test_email_ingest.py
Normal file
166
tests/bib/test_email_ingest.py
Normal file
@@ -0,0 +1,166 @@
|
|||||||
|
"""Tests for bib.email_ingest — IMAP → bib ingest."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import email.message
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
from bib.email_ingest import (
|
||||||
|
Mailbox,
|
||||||
|
_date_iso,
|
||||||
|
_extract_body,
|
||||||
|
_safe_filename,
|
||||||
|
_sender_domain,
|
||||||
|
_slug,
|
||||||
|
ingest,
|
||||||
|
)
|
||||||
|
|
||||||
|
# ── Pure helpers ────────────────────<E29480><E29480>─────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestSlug:
|
||||||
|
def test_basic(self):
|
||||||
|
assert _slug("Hello World!") == "hello-world"
|
||||||
|
|
||||||
|
def test_truncate(self):
|
||||||
|
assert len(_slug("x" * 100)) <= 60
|
||||||
|
|
||||||
|
def test_empty(self):
|
||||||
|
assert _slug("") == ""
|
||||||
|
|
||||||
|
|
||||||
|
class TestSafeFilename:
|
||||||
|
def test_special_chars(self):
|
||||||
|
assert _safe_filename("file (1).pdf") == "file_1_.pdf"
|
||||||
|
|
||||||
|
def test_empty(self):
|
||||||
|
assert _safe_filename("") == "attachment"
|
||||||
|
|
||||||
|
|
||||||
|
class TestSenderDomain:
|
||||||
|
def test_angle_bracket(self):
|
||||||
|
assert _sender_domain("John <john@example.com>") == "example.com"
|
||||||
|
|
||||||
|
def test_bare(self):
|
||||||
|
assert _sender_domain("user@domain.org") == "domain.org"
|
||||||
|
|
||||||
|
def test_no_domain(self):
|
||||||
|
assert _sender_domain("no email here") == ""
|
||||||
|
|
||||||
|
|
||||||
|
class TestDateIso:
|
||||||
|
def test_rfc2822(self):
|
||||||
|
assert _date_iso("Wed, 15 Apr 2026 22:00:35 +0000") == "2026-04-15"
|
||||||
|
|
||||||
|
def test_empty(self):
|
||||||
|
assert _date_iso("") == ""
|
||||||
|
|
||||||
|
def test_garbage(self):
|
||||||
|
assert _date_iso("not a date") == ""
|
||||||
|
|
||||||
|
|
||||||
|
class TestExtractBody:
|
||||||
|
def test_plain_text(self):
|
||||||
|
msg = email.message.EmailMessage()
|
||||||
|
msg.set_content("Hello world")
|
||||||
|
assert _extract_body(msg) == "Hello world\n"
|
||||||
|
|
||||||
|
def test_html_fallback(self):
|
||||||
|
msg = email.message.EmailMessage()
|
||||||
|
msg["Content-Type"] = "multipart/alternative"
|
||||||
|
msg.add_alternative("<p>Hello</p>", subtype="html")
|
||||||
|
body = _extract_body(msg)
|
||||||
|
assert "Hello" in body
|
||||||
|
|
||||||
|
|
||||||
|
# ── ingest (mocked IMAP + Store) ─────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _make_raw_email(
|
||||||
|
subject="Test", sender="cms@example.gov", body="Hello", msg_id="<abc@test>"
|
||||||
|
):
|
||||||
|
msg = email.message.EmailMessage()
|
||||||
|
msg["Subject"] = subject
|
||||||
|
msg["From"] = sender
|
||||||
|
msg["Date"] = "Wed, 15 Apr 2026 10:00:00 +0000"
|
||||||
|
msg["Message-ID"] = msg_id
|
||||||
|
msg.set_content(body)
|
||||||
|
return msg.as_bytes()
|
||||||
|
|
||||||
|
|
||||||
|
class TestIngest:
|
||||||
|
@patch("bib.email_ingest.imaplib.IMAP4_SSL")
|
||||||
|
def test_processes_unseen(self, MockIMAP):
|
||||||
|
conn = MockIMAP.return_value
|
||||||
|
conn.login.return_value = ("OK", [])
|
||||||
|
conn.select.return_value = ("OK", [])
|
||||||
|
conn.uid.side_effect = [
|
||||||
|
("OK", [b"1 2"]), # SEARCH
|
||||||
|
(
|
||||||
|
"OK",
|
||||||
|
[(b"1 (RFC822 {100}", _make_raw_email(msg_id="<m1@test>"))],
|
||||||
|
), # FETCH 1
|
||||||
|
("OK", []), # STORE 1
|
||||||
|
(
|
||||||
|
"OK",
|
||||||
|
[
|
||||||
|
(
|
||||||
|
b"2 (RFC822 {100}",
|
||||||
|
_make_raw_email(subject="Second", msg_id="<m2@test>"),
|
||||||
|
)
|
||||||
|
],
|
||||||
|
), # FETCH 2
|
||||||
|
("OK", []), # STORE 2
|
||||||
|
]
|
||||||
|
conn.close.return_value = ("OK", [])
|
||||||
|
conn.logout.return_value = ("OK", [])
|
||||||
|
|
||||||
|
store = MagicMock()
|
||||||
|
store.upsert.return_value = "KEY1"
|
||||||
|
|
||||||
|
mb = Mailbox(
|
||||||
|
host="mail.test", port=993, username="test@test.com", password="pw"
|
||||||
|
)
|
||||||
|
stats = ingest(store, mb, scratch_root=Path("/tmp/test-ingest"))
|
||||||
|
assert stats["seen"] == 2
|
||||||
|
assert stats["ingested"] == 2
|
||||||
|
assert stats["errors"] == 0
|
||||||
|
assert store.upsert.call_count == 2
|
||||||
|
|
||||||
|
@patch("bib.email_ingest.imaplib.IMAP4_SSL")
|
||||||
|
def test_empty_inbox(self, MockIMAP):
|
||||||
|
conn = MockIMAP.return_value
|
||||||
|
conn.login.return_value = ("OK", [])
|
||||||
|
conn.select.return_value = ("OK", [])
|
||||||
|
conn.uid.return_value = ("OK", [b""])
|
||||||
|
conn.close.return_value = ("OK", [])
|
||||||
|
conn.logout.return_value = ("OK", [])
|
||||||
|
|
||||||
|
store = MagicMock()
|
||||||
|
mb = Mailbox(
|
||||||
|
host="mail.test", port=993, username="test@test.com", password="pw"
|
||||||
|
)
|
||||||
|
stats = ingest(store, mb)
|
||||||
|
assert stats["seen"] == 0
|
||||||
|
assert stats["ingested"] == 0
|
||||||
|
|
||||||
|
@patch("bib.email_ingest.imaplib.IMAP4_SSL")
|
||||||
|
def test_limit(self, MockIMAP):
|
||||||
|
conn = MockIMAP.return_value
|
||||||
|
conn.login.return_value = ("OK", [])
|
||||||
|
conn.select.return_value = ("OK", [])
|
||||||
|
conn.uid.side_effect = [
|
||||||
|
("OK", [b"1 2 3 4 5"]), # SEARCH returns 5
|
||||||
|
("OK", [(b"1 (RFC822 {100}", _make_raw_email(msg_id="<l1@test>"))]),
|
||||||
|
("OK", []),
|
||||||
|
]
|
||||||
|
conn.close.return_value = ("OK", [])
|
||||||
|
conn.logout.return_value = ("OK", [])
|
||||||
|
|
||||||
|
store = MagicMock()
|
||||||
|
store.upsert.return_value = "K"
|
||||||
|
mb = Mailbox(host="h", port=993, username="u@d", password="p")
|
||||||
|
stats = ingest(store, mb, limit=1)
|
||||||
|
assert stats["seen"] == 1
|
||||||
|
assert stats["ingested"] == 1
|
||||||
178
tests/bib/test_federalregister.py
Normal file
178
tests/bib/test_federalregister.py
Normal file
@@ -0,0 +1,178 @@
|
|||||||
|
"""Tests for bib.federalregister — Federal Register API client."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
from bib.federalregister import (
|
||||||
|
PFS_TITLE_PATTERN,
|
||||||
|
FRDocument,
|
||||||
|
_to_doc,
|
||||||
|
pfs_rules,
|
||||||
|
search,
|
||||||
|
split_docket_ids,
|
||||||
|
)
|
||||||
|
|
||||||
|
# ── split_docket_ids (pure function) ──────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestSplitDocketIds:
|
||||||
|
def test_single_clean(self):
|
||||||
|
assert split_docket_ids(["CMS-1676-P"]) == ["CMS-1676-P"]
|
||||||
|
|
||||||
|
def test_comma_separated(self):
|
||||||
|
result = split_docket_ids(["CMS-1693-F, CMS-1693-IFC, CMS-5522-F3"])
|
||||||
|
assert result == ["CMS-1693-F", "CMS-1693-IFC", "CMS-5522-F3"]
|
||||||
|
|
||||||
|
def test_and_separated(self):
|
||||||
|
result = split_docket_ids(["CMS-1807-F and CMS-4201-F5"])
|
||||||
|
assert result == ["CMS-1807-F", "CMS-4201-F5"]
|
||||||
|
|
||||||
|
def test_mixed(self):
|
||||||
|
result = split_docket_ids(
|
||||||
|
["CMS-1734-F, CMS-1734-IFC, CMS-5531-F and CMS-1744-F"]
|
||||||
|
)
|
||||||
|
assert "CMS-1734-F" in result
|
||||||
|
assert "CMS-1734-IFC" in result
|
||||||
|
assert "CMS-1744-F" in result
|
||||||
|
|
||||||
|
def test_dedup(self):
|
||||||
|
result = split_docket_ids(["CMS-1676-P", "CMS-1676-P"])
|
||||||
|
assert result == ["CMS-1676-P"]
|
||||||
|
|
||||||
|
def test_empty(self):
|
||||||
|
assert split_docket_ids([]) == []
|
||||||
|
|
||||||
|
def test_whitespace(self):
|
||||||
|
assert split_docket_ids([" CMS-1676-P "]) == ["CMS-1676-P"]
|
||||||
|
|
||||||
|
|
||||||
|
# ── _to_doc ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestToDoc:
|
||||||
|
def test_basic(self):
|
||||||
|
raw = {
|
||||||
|
"document_number": "2017-13010",
|
||||||
|
"title": "Revisions to Payment Policies Under the Physician Fee Schedule",
|
||||||
|
"type": "Proposed Rule",
|
||||||
|
"publication_date": "2017-07-21",
|
||||||
|
"docket_ids": ["CMS-1676-P"],
|
||||||
|
"volume": "82",
|
||||||
|
"start_page": "33950",
|
||||||
|
"regulation_id_numbers": ["0938-AS81"],
|
||||||
|
"html_url": "https://www.federalregister.gov/documents/2017-13010",
|
||||||
|
}
|
||||||
|
doc = _to_doc(raw)
|
||||||
|
assert isinstance(doc, FRDocument)
|
||||||
|
assert doc.document_number == "2017-13010"
|
||||||
|
assert doc.type == "Proposed Rule"
|
||||||
|
assert doc.dockets == ["CMS-1676-P"]
|
||||||
|
assert doc.volume == "82"
|
||||||
|
|
||||||
|
def test_missing_fields(self):
|
||||||
|
doc = _to_doc({})
|
||||||
|
assert doc.document_number == ""
|
||||||
|
assert doc.dockets == []
|
||||||
|
assert doc.volume == ""
|
||||||
|
|
||||||
|
|
||||||
|
# ── search (mocked HTTP) ─────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _mock_fr_response(results, total_pages=1):
|
||||||
|
resp = MagicMock()
|
||||||
|
resp.status_code = 200
|
||||||
|
resp.json.return_value = {
|
||||||
|
"results": results,
|
||||||
|
"total_pages": total_pages,
|
||||||
|
}
|
||||||
|
resp.raise_for_status = MagicMock()
|
||||||
|
return resp
|
||||||
|
|
||||||
|
|
||||||
|
class TestSearch:
|
||||||
|
@patch("bib.federalregister.httpx.Client")
|
||||||
|
def test_yields_documents(self, MockClient):
|
||||||
|
client = MockClient.return_value
|
||||||
|
client.get.return_value = _mock_fr_response(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"document_number": "2017-13010",
|
||||||
|
"title": "PFS Rule",
|
||||||
|
"type": "Proposed Rule",
|
||||||
|
"publication_date": "2017-07-21",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
)
|
||||||
|
client.close = MagicMock()
|
||||||
|
docs = list(search(term="physician fee schedule"))
|
||||||
|
assert len(docs) == 1
|
||||||
|
assert docs[0].document_number == "2017-13010"
|
||||||
|
|
||||||
|
@patch("bib.federalregister.httpx.Client")
|
||||||
|
def test_pagination(self, MockClient):
|
||||||
|
client = MockClient.return_value
|
||||||
|
page1 = _mock_fr_response(
|
||||||
|
[{"document_number": "A", "title": "", "type": "", "publication_date": ""}],
|
||||||
|
total_pages=2,
|
||||||
|
)
|
||||||
|
page2 = _mock_fr_response(
|
||||||
|
[{"document_number": "B", "title": "", "type": "", "publication_date": ""}],
|
||||||
|
total_pages=2,
|
||||||
|
)
|
||||||
|
client.get.side_effect = [page1, page2]
|
||||||
|
client.close = MagicMock()
|
||||||
|
docs = list(search())
|
||||||
|
assert len(docs) == 2
|
||||||
|
|
||||||
|
@patch("bib.federalregister.httpx.Client")
|
||||||
|
def test_empty_results(self, MockClient):
|
||||||
|
client = MockClient.return_value
|
||||||
|
client.get.return_value = _mock_fr_response([])
|
||||||
|
client.close = MagicMock()
|
||||||
|
docs = list(search(term="nonexistent"))
|
||||||
|
assert docs == []
|
||||||
|
|
||||||
|
|
||||||
|
# ── pfs_rules (mocked HTTP) ──────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestPfsRules:
|
||||||
|
@patch("bib.federalregister.httpx.Client")
|
||||||
|
def test_filters_by_title(self, MockClient):
|
||||||
|
client = MockClient.return_value
|
||||||
|
client.close = MagicMock()
|
||||||
|
client.get.return_value = _mock_fr_response(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"document_number": "A",
|
||||||
|
"title": "CY 2024 Payment Policies Under the Physician Fee Schedule",
|
||||||
|
"type": "Proposed Rule",
|
||||||
|
"publication_date": "2023-07-01",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"document_number": "B",
|
||||||
|
"title": "Unrelated Medicare Rule",
|
||||||
|
"type": "Rule",
|
||||||
|
"publication_date": "2023-07-01",
|
||||||
|
},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
found = pfs_rules(since="2023-01-01")
|
||||||
|
assert len(found) == 1
|
||||||
|
assert found[0].document_number == "A"
|
||||||
|
|
||||||
|
@patch("bib.federalregister.httpx.Client")
|
||||||
|
def test_deduplicates(self, MockClient):
|
||||||
|
client = MockClient.return_value
|
||||||
|
client.close = MagicMock()
|
||||||
|
doc = {
|
||||||
|
"document_number": "A",
|
||||||
|
"title": f"Revisions to {PFS_TITLE_PATTERN} and More",
|
||||||
|
"type": "Rule",
|
||||||
|
"publication_date": "2023-07-01",
|
||||||
|
}
|
||||||
|
client.get.return_value = _mock_fr_response([doc, doc])
|
||||||
|
found = pfs_rules(since="2023-01-01")
|
||||||
|
assert len(found) == 1
|
||||||
235
tests/bib/test_regulations_gov.py
Normal file
235
tests/bib/test_regulations_gov.py
Normal file
@@ -0,0 +1,235 @@
|
|||||||
|
"""Tests for bib.regulations_gov — reg.gov v4 API client."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from bib.regulations_gov import (
|
||||||
|
Client,
|
||||||
|
Comment,
|
||||||
|
_byline,
|
||||||
|
_docket_from_comment_id,
|
||||||
|
_filename_from,
|
||||||
|
_hash,
|
||||||
|
_parse_comment,
|
||||||
|
_reg_date,
|
||||||
|
_slug,
|
||||||
|
upsert_comment,
|
||||||
|
)
|
||||||
|
|
||||||
|
# ── Pure helpers ──────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestRegDate:
|
||||||
|
def test_iso_to_space(self):
|
||||||
|
assert _reg_date("2017-08-30T18:35:48Z") == "2017-08-30 18:35:48"
|
||||||
|
|
||||||
|
def test_already_space(self):
|
||||||
|
assert _reg_date("2017-08-30 18:35:48") == "2017-08-30 18:35:48"
|
||||||
|
|
||||||
|
def test_empty(self):
|
||||||
|
assert _reg_date("") == ""
|
||||||
|
|
||||||
|
def test_none(self):
|
||||||
|
assert _reg_date("") == ""
|
||||||
|
|
||||||
|
|
||||||
|
class TestSlug:
|
||||||
|
def test_basic(self):
|
||||||
|
assert _slug("American Medical Association") == "american-medical-association"
|
||||||
|
|
||||||
|
def test_special(self):
|
||||||
|
assert _slug("Test & Co., Inc.") == "test-co-inc"
|
||||||
|
|
||||||
|
def test_truncate(self):
|
||||||
|
assert len(_slug("x" * 100)) <= 40
|
||||||
|
|
||||||
|
|
||||||
|
class TestFilenameFrom:
|
||||||
|
def test_url(self):
|
||||||
|
assert (
|
||||||
|
_filename_from(
|
||||||
|
"https://downloads.regulations.gov/CMS-2021-0119-26001/attachment_1.pdf"
|
||||||
|
)
|
||||||
|
== "attachment_1.pdf"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_with_query(self):
|
||||||
|
assert _filename_from("https://x.com/file.pdf?token=abc") == "file.pdf"
|
||||||
|
|
||||||
|
def test_empty(self):
|
||||||
|
assert _filename_from("") == ""
|
||||||
|
|
||||||
|
|
||||||
|
class TestHash:
|
||||||
|
def test_deterministic(self):
|
||||||
|
assert _hash("test") == _hash("test")
|
||||||
|
assert len(_hash("test")) == 10
|
||||||
|
|
||||||
|
|
||||||
|
class TestByline:
|
||||||
|
def test_org(self):
|
||||||
|
c = Comment(
|
||||||
|
id="1",
|
||||||
|
title="",
|
||||||
|
posted_date="",
|
||||||
|
received_date="",
|
||||||
|
docket_id="",
|
||||||
|
comment_on_id="",
|
||||||
|
organization="AMA",
|
||||||
|
)
|
||||||
|
assert _byline(c) == "AMA"
|
||||||
|
|
||||||
|
def test_name(self):
|
||||||
|
c = Comment(
|
||||||
|
id="1",
|
||||||
|
title="",
|
||||||
|
posted_date="",
|
||||||
|
received_date="",
|
||||||
|
docket_id="",
|
||||||
|
comment_on_id="",
|
||||||
|
first_name="John",
|
||||||
|
last_name="Doe",
|
||||||
|
)
|
||||||
|
assert _byline(c) == "John Doe"
|
||||||
|
|
||||||
|
def test_empty(self):
|
||||||
|
c = Comment(
|
||||||
|
id="1",
|
||||||
|
title="",
|
||||||
|
posted_date="",
|
||||||
|
received_date="",
|
||||||
|
docket_id="",
|
||||||
|
comment_on_id="",
|
||||||
|
)
|
||||||
|
assert _byline(c) == ""
|
||||||
|
|
||||||
|
|
||||||
|
class TestDocketFromCommentId:
|
||||||
|
def test_standard(self):
|
||||||
|
assert _docket_from_comment_id("CMS-2025-0304-14107") == "CMS-2025-0304"
|
||||||
|
|
||||||
|
def test_short(self):
|
||||||
|
assert _docket_from_comment_id("AB") == "AB"
|
||||||
|
|
||||||
|
|
||||||
|
# ── _parse_comment ────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseComment:
|
||||||
|
def test_basic(self):
|
||||||
|
row = {
|
||||||
|
"id": "CMS-2017-0092-0002",
|
||||||
|
"attributes": {
|
||||||
|
"title": "Test Comment",
|
||||||
|
"postedDate": "2017-07-18T00:00:00Z",
|
||||||
|
"receivedDate": "2017-07-15T00:00:00Z",
|
||||||
|
"docketId": "CMS-2017-0092",
|
||||||
|
"commentOnId": "0900006482921ba1",
|
||||||
|
"firstName": "Jane",
|
||||||
|
"lastName": "Doe",
|
||||||
|
"organization": "AMA",
|
||||||
|
"comment": "This is the body.",
|
||||||
|
"attachmentCount": 2,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
c = _parse_comment(row)
|
||||||
|
assert c.id == "CMS-2017-0092-0002"
|
||||||
|
assert c.title == "Test Comment"
|
||||||
|
assert c.posted_date == "2017-07-18"
|
||||||
|
assert c.organization == "AMA"
|
||||||
|
assert c.comment_text == "This is the body."
|
||||||
|
assert c.attachment_count == 2
|
||||||
|
|
||||||
|
def test_missing_attrs(self):
|
||||||
|
c = _parse_comment({"id": "X", "attributes": {}})
|
||||||
|
assert c.id == "X"
|
||||||
|
assert c.title == ""
|
||||||
|
assert c.attachment_count == 0
|
||||||
|
|
||||||
|
|
||||||
|
# ── upsert_comment ────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestUpsertComment:
|
||||||
|
def test_creates_item(self):
|
||||||
|
store = MagicMock()
|
||||||
|
store.upsert.return_value = "KEY1"
|
||||||
|
c = Comment(
|
||||||
|
id="CMS-2017-0092-0002",
|
||||||
|
title="Test",
|
||||||
|
posted_date="2017-07-18",
|
||||||
|
received_date="2017-07-15",
|
||||||
|
docket_id="CMS-2017-0092",
|
||||||
|
comment_on_id="obj123",
|
||||||
|
organization="AMA",
|
||||||
|
)
|
||||||
|
key = upsert_comment(store, c, cms_id="CMS-1676-P")
|
||||||
|
assert key == "KEY1"
|
||||||
|
store.upsert.assert_called_once()
|
||||||
|
item = store.upsert.call_args[0][0]
|
||||||
|
assert "AMA" in item.title
|
||||||
|
assert item.url == "https://www.regulations.gov/comment/CMS-2017-0092-0002"
|
||||||
|
|
||||||
|
|
||||||
|
# ── Client (mocked httpx) ────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestClient:
|
||||||
|
@patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test-key"})
|
||||||
|
def test_init(self):
|
||||||
|
c = Client(sleep=0)
|
||||||
|
assert c._key == "test-key"
|
||||||
|
c.close()
|
||||||
|
|
||||||
|
@patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": ""})
|
||||||
|
def test_init_no_key(self):
|
||||||
|
with pytest.raises(RuntimeError, match="REGULATIONS_GOV_API_KEY"):
|
||||||
|
Client()
|
||||||
|
|
||||||
|
@patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test-key"})
|
||||||
|
def test_get_retry_429(self):
|
||||||
|
mock_client = MagicMock()
|
||||||
|
resp_429 = MagicMock()
|
||||||
|
resp_429.status_code = 429
|
||||||
|
resp_ok = MagicMock()
|
||||||
|
resp_ok.status_code = 200
|
||||||
|
resp_ok.json.return_value = {"data": []}
|
||||||
|
resp_ok.raise_for_status = MagicMock()
|
||||||
|
mock_client.get.side_effect = [resp_429, resp_ok]
|
||||||
|
|
||||||
|
with patch("bib.regulations_gov.time.sleep"):
|
||||||
|
c = Client(sleep=0, client=mock_client)
|
||||||
|
result = c._get("/test")
|
||||||
|
assert result == {"data": []}
|
||||||
|
|
||||||
|
@patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test-key"})
|
||||||
|
def test_resolve_docket(self):
|
||||||
|
mock_client = MagicMock()
|
||||||
|
resp = MagicMock()
|
||||||
|
resp.status_code = 200
|
||||||
|
resp.json.return_value = {
|
||||||
|
"data": [{"attributes": {"docketId": "CMS-2017-0092"}}]
|
||||||
|
}
|
||||||
|
resp.raise_for_status = MagicMock()
|
||||||
|
mock_client.get.return_value = resp
|
||||||
|
|
||||||
|
with patch("bib.regulations_gov.time.sleep"):
|
||||||
|
c = Client(sleep=0, client=mock_client)
|
||||||
|
result = c.resolve_docket("CMS-1676-P")
|
||||||
|
assert result == "CMS-2017-0092"
|
||||||
|
|
||||||
|
@patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test-key"})
|
||||||
|
def test_resolve_docket_not_found(self):
|
||||||
|
mock_client = MagicMock()
|
||||||
|
resp = MagicMock()
|
||||||
|
resp.status_code = 200
|
||||||
|
resp.json.return_value = {"data": []}
|
||||||
|
resp.raise_for_status = MagicMock()
|
||||||
|
mock_client.get.return_value = resp
|
||||||
|
|
||||||
|
with patch("bib.regulations_gov.time.sleep"):
|
||||||
|
c = Client(sleep=0, client=mock_client)
|
||||||
|
assert c.resolve_docket("NONEXISTENT") is None
|
||||||
0
tests/mail/__init__.py
Normal file
0
tests/mail/__init__.py
Normal file
0
tests/prisma/__init__.py
Normal file
0
tests/prisma/__init__.py
Normal file
284
tests/prisma/test_fetch.py
Normal file
284
tests/prisma/test_fetch.py
Normal file
@@ -0,0 +1,284 @@
|
|||||||
|
"""Tests for prisma.fetch — PDF retrieval with altcha solver."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from prisma.fetch import (
|
||||||
|
PendingItem,
|
||||||
|
_altcha_bootstrap,
|
||||||
|
_extract_pdf_url,
|
||||||
|
_solve_altcha_pow,
|
||||||
|
download,
|
||||||
|
fetch_one,
|
||||||
|
fetch_pmc,
|
||||||
|
fetch_unpaywall,
|
||||||
|
resolve_doi_from_title,
|
||||||
|
)
|
||||||
|
|
||||||
|
# ── Altcha solver (pure, deterministic) ───────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestAltchaPow:
|
||||||
|
def test_known_vector(self):
|
||||||
|
salt = "testsalt"
|
||||||
|
target_n = 42
|
||||||
|
challenge = hashlib.sha256(f"{salt}{target_n}".encode()).hexdigest()
|
||||||
|
result = _solve_altcha_pow(salt, challenge, 1000)
|
||||||
|
assert result is not None
|
||||||
|
nonce, elapsed = result
|
||||||
|
assert nonce == 42
|
||||||
|
assert elapsed >= 0
|
||||||
|
|
||||||
|
def test_not_found(self):
|
||||||
|
result = _solve_altcha_pow("salt", "impossible" * 4, 10)
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_zero_nonce(self):
|
||||||
|
salt = "zero"
|
||||||
|
challenge = hashlib.sha256(f"{salt}0".encode()).hexdigest()
|
||||||
|
result = _solve_altcha_pow(salt, challenge, 100)
|
||||||
|
assert result is not None
|
||||||
|
assert result[0] == 0
|
||||||
|
|
||||||
|
|
||||||
|
# ── _extract_pdf_url ──────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestExtractPdfUrl:
|
||||||
|
def test_citation_meta(self):
|
||||||
|
html = '<meta name="citation_pdf_url" content="/storage/test.pdf">'
|
||||||
|
assert (
|
||||||
|
_extract_pdf_url(html, "https://example.com")
|
||||||
|
== "https://example.com/storage/test.pdf"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_embed_src(self):
|
||||||
|
html = '<embed src="//cdn.example.com/paper.pdf" type="application/pdf">'
|
||||||
|
assert (
|
||||||
|
_extract_pdf_url(html, "https://x.com")
|
||||||
|
== "https://cdn.example.com/paper.pdf"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_no_match(self):
|
||||||
|
assert _extract_pdf_url("<html>no pdf here</html>", "https://x.com") is None
|
||||||
|
|
||||||
|
def test_absolute_url(self):
|
||||||
|
html = '<meta name="citation_pdf_url" content="https://full.url/doc.pdf">'
|
||||||
|
assert _extract_pdf_url(html, "https://x.com") == "https://full.url/doc.pdf"
|
||||||
|
|
||||||
|
|
||||||
|
# ── _altcha_bootstrap ─────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestAltchaBootstrap:
|
||||||
|
def test_solves_challenge(self):
|
||||||
|
salt = "bootsalt"
|
||||||
|
nonce = 77
|
||||||
|
challenge = hashlib.sha256(f"{salt}{nonce}".encode()).hexdigest()
|
||||||
|
|
||||||
|
client = MagicMock()
|
||||||
|
challenge_resp = MagicMock()
|
||||||
|
challenge_resp.json.return_value = {
|
||||||
|
"algorithm": "SHA-256",
|
||||||
|
"salt": salt,
|
||||||
|
"challenge": challenge,
|
||||||
|
"maxNumber": 1000,
|
||||||
|
"signature": "sig",
|
||||||
|
}
|
||||||
|
solve_resp = MagicMock()
|
||||||
|
solve_resp.status_code = 200
|
||||||
|
solve_resp.json.return_value = {"success": True}
|
||||||
|
|
||||||
|
client.get.return_value = challenge_resp
|
||||||
|
client.post.return_value = solve_resp
|
||||||
|
|
||||||
|
html = (
|
||||||
|
'<altcha-widget challengeurl="/captcha/challenge/123">'
|
||||||
|
"fetch('/captcha/solution/123'"
|
||||||
|
)
|
||||||
|
result = _altcha_bootstrap(client, "https://mirror.test/doi", html)
|
||||||
|
assert result is True
|
||||||
|
|
||||||
|
def test_no_challenge(self):
|
||||||
|
assert (
|
||||||
|
_altcha_bootstrap(MagicMock(), "https://x.com/doi", "<html></html>")
|
||||||
|
is False
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ── fetch_unpaywall ───────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestFetchUnpaywall:
|
||||||
|
def test_returns_pdf_url(self):
|
||||||
|
client = MagicMock()
|
||||||
|
resp = MagicMock()
|
||||||
|
resp.status_code = 200
|
||||||
|
resp.json.return_value = {
|
||||||
|
"best_oa_location": {"url_for_pdf": "https://oa.org/paper.pdf"}
|
||||||
|
}
|
||||||
|
client.get.return_value = resp
|
||||||
|
assert (
|
||||||
|
fetch_unpaywall(client, "10.1234/test", "e@x.com")
|
||||||
|
== "https://oa.org/paper.pdf"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_no_doi(self):
|
||||||
|
assert fetch_unpaywall(MagicMock(), "", "e@x.com") is None
|
||||||
|
|
||||||
|
def test_404(self):
|
||||||
|
client = MagicMock()
|
||||||
|
resp = MagicMock()
|
||||||
|
resp.status_code = 404
|
||||||
|
client.get.return_value = resp
|
||||||
|
assert fetch_unpaywall(client, "10.1234/x", "e@x.com") is None
|
||||||
|
|
||||||
|
|
||||||
|
# ── fetch_pmc ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestFetchPmc:
|
||||||
|
def test_with_prefix(self):
|
||||||
|
assert (
|
||||||
|
fetch_pmc(MagicMock(), "PMC12345")
|
||||||
|
== "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC12345/pdf/"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_without_prefix(self):
|
||||||
|
assert (
|
||||||
|
fetch_pmc(MagicMock(), "12345")
|
||||||
|
== "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC12345/pdf/"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_empty(self):
|
||||||
|
assert fetch_pmc(MagicMock(), "") is None
|
||||||
|
|
||||||
|
|
||||||
|
# ── resolve_doi_from_title ────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestResolveDoi:
|
||||||
|
def test_match(self):
|
||||||
|
client = MagicMock()
|
||||||
|
resp = MagicMock()
|
||||||
|
resp.status_code = 200
|
||||||
|
resp.json.return_value = {
|
||||||
|
"message": {
|
||||||
|
"items": [
|
||||||
|
{
|
||||||
|
"DOI": "10.1234/found",
|
||||||
|
"title": ["Skin substitutes in wound care"],
|
||||||
|
"score": 80,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
client.get.return_value = resp
|
||||||
|
assert (
|
||||||
|
resolve_doi_from_title(client, "Skin substitutes in wound care")
|
||||||
|
== "10.1234/found"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_low_score(self):
|
||||||
|
client = MagicMock()
|
||||||
|
resp = MagicMock()
|
||||||
|
resp.status_code = 200
|
||||||
|
resp.json.return_value = {
|
||||||
|
"message": {
|
||||||
|
"items": [{"DOI": "10.1234/x", "title": ["Unrelated"], "score": 10}]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
client.get.return_value = resp
|
||||||
|
assert resolve_doi_from_title(client, "Skin substitutes in wound care") is None
|
||||||
|
|
||||||
|
def test_short_title(self):
|
||||||
|
assert resolve_doi_from_title(MagicMock(), "Short") is None
|
||||||
|
|
||||||
|
|
||||||
|
# ── download ──────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestDownload:
|
||||||
|
def test_valid_pdf(self, tmp_path):
|
||||||
|
client = MagicMock()
|
||||||
|
stream_ctx = MagicMock()
|
||||||
|
stream_ctx.__enter__ = MagicMock(return_value=stream_ctx)
|
||||||
|
stream_ctx.__exit__ = MagicMock(return_value=False)
|
||||||
|
stream_ctx.status_code = 200
|
||||||
|
stream_ctx.headers = {"content-type": "application/pdf"}
|
||||||
|
stream_ctx.iter_bytes.return_value = [b"%PDF-1.7\n" + b"x" * 2000]
|
||||||
|
client.stream.return_value = stream_ctx
|
||||||
|
|
||||||
|
dest = tmp_path / "test.pdf"
|
||||||
|
result = download(client, "https://x.com/paper.pdf", dest)
|
||||||
|
assert result == dest
|
||||||
|
assert dest.read_bytes().startswith(b"%PDF")
|
||||||
|
|
||||||
|
def test_not_pdf(self, tmp_path):
|
||||||
|
client = MagicMock()
|
||||||
|
stream_ctx = MagicMock()
|
||||||
|
stream_ctx.__enter__ = MagicMock(return_value=stream_ctx)
|
||||||
|
stream_ctx.__exit__ = MagicMock(return_value=False)
|
||||||
|
stream_ctx.status_code = 200
|
||||||
|
stream_ctx.headers = {"content-type": "text/html"}
|
||||||
|
stream_ctx.iter_bytes.return_value = [b"<html>Not a PDF</html>" + b"x" * 2000]
|
||||||
|
client.stream.return_value = stream_ctx
|
||||||
|
|
||||||
|
dest = tmp_path / "bad.pdf"
|
||||||
|
result = download(client, "https://x.com/bad", dest)
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_too_small(self, tmp_path):
|
||||||
|
client = MagicMock()
|
||||||
|
stream_ctx = MagicMock()
|
||||||
|
stream_ctx.__enter__ = MagicMock(return_value=stream_ctx)
|
||||||
|
stream_ctx.__exit__ = MagicMock(return_value=False)
|
||||||
|
stream_ctx.status_code = 200
|
||||||
|
stream_ctx.headers = {}
|
||||||
|
stream_ctx.iter_bytes.return_value = [b"%PDF"]
|
||||||
|
client.stream.return_value = stream_ctx
|
||||||
|
|
||||||
|
dest = tmp_path / "tiny.pdf"
|
||||||
|
result = download(client, "https://x.com/tiny", dest)
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
|
||||||
|
# ── fetch_one cascade ─────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class TestFetchOne:
|
||||||
|
def test_unpaywall_hit(self, tmp_path):
|
||||||
|
item = PendingItem(
|
||||||
|
zot_id=1, doi="10.1234/test", pmid="", pmcid="", title="Test"
|
||||||
|
)
|
||||||
|
client = MagicMock()
|
||||||
|
resp = MagicMock()
|
||||||
|
resp.status_code = 200
|
||||||
|
resp.json.return_value = {
|
||||||
|
"best_oa_location": {"url_for_pdf": "https://oa.org/p.pdf"}
|
||||||
|
}
|
||||||
|
client.get.return_value = resp
|
||||||
|
|
||||||
|
stream_ctx = MagicMock()
|
||||||
|
stream_ctx.__enter__ = MagicMock(return_value=stream_ctx)
|
||||||
|
stream_ctx.__exit__ = MagicMock(return_value=False)
|
||||||
|
stream_ctx.status_code = 200
|
||||||
|
stream_ctx.headers = {"content-type": "application/pdf"}
|
||||||
|
stream_ctx.iter_bytes.return_value = [b"%PDF-1.7\n" + b"x" * 2000]
|
||||||
|
client.stream.return_value = stream_ctx
|
||||||
|
|
||||||
|
result = fetch_one(item, email="e@x.com", scratch=tmp_path, client=client)
|
||||||
|
assert result is not None
|
||||||
|
source, path = result
|
||||||
|
assert source == "unpaywall"
|
||||||
|
|
||||||
|
def test_all_miss(self, tmp_path):
|
||||||
|
item = PendingItem(zot_id=1, doi="", pmid="", pmcid="", title="X")
|
||||||
|
client = MagicMock()
|
||||||
|
resp = MagicMock()
|
||||||
|
resp.status_code = 404
|
||||||
|
client.get.return_value = resp
|
||||||
|
result = fetch_one(item, email="e@x.com", scratch=tmp_path, client=client)
|
||||||
|
assert result is None
|
||||||
62
tests/zot/test_ops.py
Normal file
62
tests/zot/test_ops.py
Normal file
@@ -0,0 +1,62 @@
|
|||||||
|
"""Tests for zot.ops — Zotero maintenance operations."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sqlite3
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from tests.zot.test_db import ZOTERO_SCHEMA, _seed_schema_maps
|
||||||
|
from zot.db import TYPE_MAP, Db
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def zot_path(tmp_path):
|
||||||
|
path = str(tmp_path / "zotero.sqlite")
|
||||||
|
con = sqlite3.connect(path)
|
||||||
|
con.executescript(ZOTERO_SCHEMA)
|
||||||
|
_seed_schema_maps(con)
|
||||||
|
con.commit()
|
||||||
|
con.close()
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
class TestDumpSchema:
|
||||||
|
def test_dumps_types(self, zot_path):
|
||||||
|
from zot.ops import dump_schema
|
||||||
|
|
||||||
|
result = dump_schema(zot_path)
|
||||||
|
assert "TYPE_MAP" in result
|
||||||
|
assert "FIELD_MAP" in result
|
||||||
|
assert "CREATOR_TYPES" in result
|
||||||
|
assert "attachment" in result["TYPE_MAP"]
|
||||||
|
|
||||||
|
|
||||||
|
class TestFixDates:
|
||||||
|
def test_runs(self, zot_path):
|
||||||
|
from zot.ops import fix_dates
|
||||||
|
|
||||||
|
with Db(zot_path) as db:
|
||||||
|
item_id = db.create_item(TYPE_MAP["document"])
|
||||||
|
db.set_fields(item_id, {"date": "2023-07-15"})
|
||||||
|
result = fix_dates(zot_path)
|
||||||
|
assert isinstance(result, dict)
|
||||||
|
|
||||||
|
|
||||||
|
class TestFixKeys:
|
||||||
|
def test_runs(self, zot_path):
|
||||||
|
from zot.ops import fix_keys
|
||||||
|
|
||||||
|
result = fix_keys(zot_path)
|
||||||
|
assert isinstance(result, dict)
|
||||||
|
|
||||||
|
|
||||||
|
class TestFixFields:
|
||||||
|
@pytest.mark.skip(
|
||||||
|
reason="requires itemTypeFieldsCombined view from full Zotero schema"
|
||||||
|
)
|
||||||
|
def test_runs(self, zot_path):
|
||||||
|
from zot.ops import fix_fields
|
||||||
|
|
||||||
|
result = fix_fields(zot_path)
|
||||||
|
assert isinstance(result, dict)
|
||||||
Reference in New Issue
Block a user