Fix Zotero table models for Zotero 9:
- Remove stale Annotations/Highlights/Transaction* models
- Add ItemAnnotations, RetractedItems, DeletedCollections,
DeletedSearches, DbDebug1
- Fix ItemAttachments, Libraries, Users column mismatches
New test files covering all major modules:
- cli/{bib,prisma,rec,zot,mail,run} deep exercising tests
- mail/{droplet,postmark,resend,cloudflare} lifecycle tests
- bib/{iom,oig,pincite,sync,regulations_gov,email_ingest,format,store}
- prisma/{vpn,fetch,export,llm,screen,eligibility,extract,project,ingest,flow}
- aco/lake/{unity,quality,deploy} + api/aco coverage gaps
- zot/{ops,db,extract,duck} + rec/{report,engine,base,pricers}
- pfs/{pipe,rules,eq,files}
Add pytest-xdist for parallel test execution.
Tracks #353
759 lines
26 KiB
Python
759 lines
26 KiB
Python
"""Exercise prisma.fetch — fetch cascade, download, attach, run orchestrator."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import httpx
|
|
|
|
from prisma.fetch import (
|
|
PendingItem,
|
|
_altcha_bootstrap,
|
|
_extra_value,
|
|
_extract_pdf_url,
|
|
_field,
|
|
_from_url_pmcid,
|
|
_hash,
|
|
attach_pdf,
|
|
download,
|
|
fetch_fallback,
|
|
fetch_one,
|
|
fetch_pmc,
|
|
fetch_unpaywall,
|
|
pending_queue,
|
|
resolve_doi_from_title,
|
|
run,
|
|
)
|
|
from zot.db import TYPE_MAP, Db
|
|
from zot.schema import create_db
|
|
|
|
|
|
def _setup(tmp_path):
|
|
path = str(tmp_path / "z.sqlite")
|
|
con = create_db(path)
|
|
con.close()
|
|
return path
|
|
|
|
|
|
def _make_item_with_tags(db, project, screen_tag):
|
|
iid = db.create_item(TYPE_MAP["document"])
|
|
db.set_fields(iid, {"title": "Test Item", "DOI": "10.1234/test"})
|
|
db.sync_tags(iid, [f"project:{project}", screen_tag])
|
|
db.commit()
|
|
return iid
|
|
|
|
|
|
def _pdf_stream_ctx(content=b"%PDF-1.7\n" + b"x" * 2000):
|
|
ctx = MagicMock()
|
|
ctx.__enter__ = MagicMock(return_value=ctx)
|
|
ctx.__exit__ = MagicMock(return_value=False)
|
|
ctx.status_code = 200
|
|
ctx.headers = {"content-type": "application/pdf"}
|
|
ctx.iter_bytes.return_value = [content]
|
|
return ctx
|
|
|
|
|
|
class TestHelpers:
|
|
def test_hash(self):
|
|
assert len(_hash("test")) == 10
|
|
|
|
|
|
class TestExtractPdfUrl:
|
|
def test_iframe_src(self):
|
|
html = '<iframe src="/pdf/10.1234/test.pdf" id="pdf"></iframe>'
|
|
result = _extract_pdf_url(html, "https://mirror.example.com")
|
|
assert result is None or isinstance(result, str)
|
|
|
|
def test_no_match(self):
|
|
assert _extract_pdf_url("<html>no pdf</html>", "https://x.com") is None
|
|
|
|
|
|
class TestFetchUnpaywall:
|
|
def test_found(self):
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 200
|
|
resp.json.return_value = {
|
|
"best_oa_location": {"url_for_pdf": "https://example.com/paper.pdf"}
|
|
}
|
|
client.get.return_value = resp
|
|
url = fetch_unpaywall(client, "10.1234/test", "test@example.com")
|
|
assert url is not None
|
|
|
|
def test_no_doi(self):
|
|
assert fetch_unpaywall(MagicMock(), "", "e@e.com") is None
|
|
|
|
def test_no_location(self):
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 200
|
|
resp.json.return_value = {"best_oa_location": None}
|
|
client.get.return_value = resp
|
|
assert fetch_unpaywall(client, "10.1234/test", "e@e.com") is None
|
|
|
|
def test_http_error(self):
|
|
import httpx
|
|
|
|
client = MagicMock()
|
|
client.get.side_effect = httpx.ConnectError("fail")
|
|
assert fetch_unpaywall(client, "10.1234/test", "e@e.com") is None
|
|
|
|
|
|
class TestFetchPmc:
|
|
def test_no_pmcid(self):
|
|
assert fetch_pmc(MagicMock(), "") is None
|
|
|
|
|
|
class TestResolveDoi:
|
|
def test_no_title(self):
|
|
assert resolve_doi_from_title(MagicMock(), "") is None
|
|
|
|
|
|
class TestFetchFallback:
|
|
def test_no_doi(self):
|
|
assert fetch_fallback(MagicMock(), "") is None
|
|
|
|
def test_altcha_page_fails(self):
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 200
|
|
resp.text = '<div class="altcha-widget">challenge</div>'
|
|
client.get.return_value = resp
|
|
with patch("prisma.fetch._altcha_bootstrap", return_value=False):
|
|
assert fetch_fallback(client, "10.1234/test") is None
|
|
|
|
|
|
class TestDownload:
|
|
def test_success(self, tmp_path):
|
|
client = MagicMock()
|
|
stream_ctx = MagicMock()
|
|
stream_ctx.__enter__ = MagicMock(return_value=stream_ctx)
|
|
stream_ctx.__exit__ = MagicMock(return_value=False)
|
|
stream_ctx.status_code = 200
|
|
stream_ctx.headers = {"content-type": "application/pdf"}
|
|
stream_ctx.iter_bytes.return_value = [b"%PDF-1.4 content here" + b"\0" * 2000]
|
|
client.stream.return_value = stream_ctx
|
|
|
|
dest = tmp_path / "test.pdf"
|
|
result = download(client, "https://example.com/paper.pdf", dest)
|
|
assert result == dest
|
|
|
|
def test_non_pdf_header(self, tmp_path):
|
|
client = MagicMock()
|
|
stream_ctx = MagicMock()
|
|
stream_ctx.__enter__ = MagicMock(return_value=stream_ctx)
|
|
stream_ctx.__exit__ = MagicMock(return_value=False)
|
|
stream_ctx.status_code = 200
|
|
stream_ctx.headers = {"content-type": "text/html"}
|
|
stream_ctx.iter_bytes.return_value = [b"<html>not a pdf</html>"]
|
|
client.stream.return_value = stream_ctx
|
|
|
|
dest = tmp_path / "test.pdf"
|
|
result = download(client, "https://example.com/paper.pdf", dest)
|
|
assert result is None
|
|
|
|
def test_too_small(self, tmp_path):
|
|
client = MagicMock()
|
|
stream_ctx = MagicMock()
|
|
stream_ctx.__enter__ = MagicMock(return_value=stream_ctx)
|
|
stream_ctx.__exit__ = MagicMock(return_value=False)
|
|
stream_ctx.status_code = 200
|
|
stream_ctx.headers = {"content-type": "application/pdf"}
|
|
stream_ctx.iter_bytes.return_value = [b"%PDF tiny"]
|
|
client.stream.return_value = stream_ctx
|
|
|
|
dest = tmp_path / "test.pdf"
|
|
result = download(client, "https://example.com/paper.pdf", dest)
|
|
assert result is None
|
|
|
|
def test_http_error(self, tmp_path):
|
|
import httpx
|
|
|
|
client = MagicMock()
|
|
client.stream.side_effect = httpx.ConnectError("fail")
|
|
dest = tmp_path / "test.pdf"
|
|
assert download(client, "https://example.com/paper.pdf", dest) is None
|
|
|
|
|
|
class TestFetchOne:
|
|
def test_unpaywall_hit(self, tmp_path):
|
|
item = PendingItem(
|
|
zot_id=1, doi="10.1234/test", title="Paper", pmcid="", pmid=""
|
|
)
|
|
with (
|
|
patch(
|
|
"prisma.fetch.fetch_unpaywall", return_value="https://ex.com/paper.pdf"
|
|
),
|
|
patch("prisma.fetch.download", return_value=tmp_path / "paper.pdf"),
|
|
):
|
|
result = fetch_one(
|
|
item, email="e@e.com", scratch=tmp_path, client=MagicMock()
|
|
)
|
|
assert result[0] == "unpaywall"
|
|
|
|
def test_pmc_hit(self, tmp_path):
|
|
item = PendingItem(zot_id=1, doi="", title="Paper", pmcid="PMC123", pmid="")
|
|
with (
|
|
patch("prisma.fetch.fetch_unpaywall", return_value=None),
|
|
patch("prisma.fetch.fetch_pmc", return_value="https://pmc/pdf.pdf"),
|
|
patch("prisma.fetch.download", return_value=tmp_path / "paper.pdf"),
|
|
):
|
|
result = fetch_one(
|
|
item, email="e@e.com", scratch=tmp_path, client=MagicMock()
|
|
)
|
|
assert result[0] == "pmc"
|
|
|
|
def test_fallback_hit(self, tmp_path):
|
|
item = PendingItem(
|
|
zot_id=1, doi="10.1234/test", title="Paper", pmcid="", pmid=""
|
|
)
|
|
proxied = MagicMock()
|
|
with (
|
|
patch("prisma.fetch.fetch_unpaywall", return_value=None),
|
|
patch("prisma.fetch.fetch_pmc", return_value=None),
|
|
patch(
|
|
"prisma.fetch.fetch_fallback", return_value="https://mirror/paper.pdf"
|
|
),
|
|
patch("prisma.fetch.download", return_value=tmp_path / "paper.pdf"),
|
|
):
|
|
result = fetch_one(
|
|
item,
|
|
email="e@e.com",
|
|
scratch=tmp_path,
|
|
client=MagicMock(),
|
|
client_proxied=proxied,
|
|
)
|
|
assert result[0] == "fallback"
|
|
|
|
def test_all_miss(self, tmp_path):
|
|
item = PendingItem(
|
|
zot_id=1, doi="10.1234/test", title="Paper", pmcid="", pmid=""
|
|
)
|
|
with (
|
|
patch("prisma.fetch.fetch_unpaywall", return_value=None),
|
|
patch("prisma.fetch.fetch_pmc", return_value=None),
|
|
):
|
|
result = fetch_one(
|
|
item, email="e@e.com", scratch=tmp_path, client=MagicMock()
|
|
)
|
|
assert result is None
|
|
|
|
def test_resolves_doi_from_title(self, tmp_path):
|
|
item = PendingItem(zot_id=1, doi="", title="Paper Title", pmcid="", pmid="")
|
|
with (
|
|
patch(
|
|
"prisma.fetch.resolve_doi_from_title", return_value="10.1234/resolved"
|
|
),
|
|
patch("prisma.fetch.fetch_unpaywall", return_value="https://ex.com/p.pdf"),
|
|
patch("prisma.fetch.download", return_value=tmp_path / "p.pdf"),
|
|
):
|
|
result = fetch_one(
|
|
item, email="e@e.com", scratch=tmp_path, client=MagicMock()
|
|
)
|
|
assert result is not None
|
|
|
|
|
|
class TestAttachPdf:
|
|
def test_attaches(self, tmp_path):
|
|
path = _setup(tmp_path)
|
|
pdf = tmp_path / "paper.pdf"
|
|
pdf.write_bytes(b"%PDF-1.4 content")
|
|
storage = tmp_path / "storage"
|
|
storage.mkdir()
|
|
|
|
with Db(path) as db:
|
|
parent_id = db.create_item(TYPE_MAP["journalArticle"])
|
|
db.commit()
|
|
att_id = attach_pdf(db, parent_id, pdf, storage, title="My Paper")
|
|
db.commit()
|
|
assert att_id > 0
|
|
|
|
|
|
class TestPendingQueue:
|
|
def test_empty(self, tmp_path):
|
|
path = _setup(tmp_path)
|
|
with Db(path) as db:
|
|
items = pending_queue(db, "test-proj")
|
|
assert items == []
|
|
|
|
|
|
class TestRun:
|
|
@patch("prisma.fetch.pending_queue", return_value=[])
|
|
@patch("httpx.Client")
|
|
def test_empty_queue(self, mc_client, mc_queue, tmp_path):
|
|
path = _setup(tmp_path)
|
|
mc_client.return_value = MagicMock()
|
|
|
|
with Db(path) as db:
|
|
stats = run(
|
|
db,
|
|
project="test-proj",
|
|
storage_dir=tmp_path / "storage",
|
|
scratch_dir=tmp_path / "scratch",
|
|
email="e@e.com",
|
|
fetch_proxy=None,
|
|
)
|
|
assert stats["missed"] == 0
|
|
|
|
@patch("prisma.fetch.pending_queue")
|
|
@patch("prisma.fetch.fetch_one")
|
|
@patch("prisma.fetch.attach_pdf")
|
|
@patch("httpx.Client")
|
|
def test_with_proxy(
|
|
self, mc_client_cls, mc_attach, mc_fetch_one, mc_queue, tmp_path
|
|
):
|
|
path = _setup(tmp_path)
|
|
item = PendingItem(
|
|
zot_id=1, doi="10.1234/test", title="Paper", pmcid="", pmid=""
|
|
)
|
|
mc_queue.return_value = [item]
|
|
|
|
pdf = tmp_path / "paper.pdf"
|
|
pdf.write_bytes(b"%PDF content")
|
|
mc_fetch_one.return_value = ("unpaywall", pdf)
|
|
mc_client_cls.return_value = MagicMock()
|
|
|
|
with Db(path) as db:
|
|
db.create_item(TYPE_MAP["journalArticle"])
|
|
db.commit()
|
|
stats = run(
|
|
db,
|
|
project="test-proj",
|
|
storage_dir=tmp_path / "storage",
|
|
scratch_dir=tmp_path / "scratch",
|
|
email="e@e.com",
|
|
fetch_proxy="socks5://localhost:1080",
|
|
)
|
|
assert stats["unpaywall"] == 1
|
|
|
|
@patch("prisma.fetch.pending_queue")
|
|
@patch("prisma.fetch.fetch_one", side_effect=Exception("boom"))
|
|
@patch("httpx.Client")
|
|
def test_fetch_error(self, mc_client_cls, mc_fetch_one, mc_queue, tmp_path):
|
|
path = _setup(tmp_path)
|
|
item = PendingItem(
|
|
zot_id=1, doi="10.1234/test", title="Paper", pmcid="", pmid=""
|
|
)
|
|
mc_queue.return_value = [item]
|
|
mc_client_cls.return_value = MagicMock()
|
|
|
|
with Db(path) as db:
|
|
stats = run(
|
|
db,
|
|
project="test-proj",
|
|
storage_dir=tmp_path / "storage",
|
|
scratch_dir=tmp_path / "scratch",
|
|
email="e@e.com",
|
|
fetch_proxy=None,
|
|
)
|
|
assert stats["errors"] == 1
|
|
|
|
@patch("prisma.fetch.pending_queue")
|
|
@patch("prisma.fetch.fetch_one")
|
|
@patch("prisma.fetch.attach_pdf", side_effect=RuntimeError("db error"))
|
|
@patch("httpx.Client")
|
|
def test_attach_error(self, mc_cls, mc_attach, mc_fetch, mc_queue, tmp_path):
|
|
path = _setup(tmp_path)
|
|
item = PendingItem(zot_id=1, doi="10.1/x", pmid="", pmcid="", title="P")
|
|
mc_queue.return_value = [item]
|
|
pdf = tmp_path / "paper.pdf"
|
|
pdf.write_bytes(b"%PDF content")
|
|
mc_fetch.return_value = ("unpaywall", pdf)
|
|
mc_cls.return_value = MagicMock()
|
|
with Db(path) as db:
|
|
stats = run(
|
|
db,
|
|
project="t",
|
|
storage_dir=tmp_path / "storage",
|
|
scratch_dir=tmp_path / "scratch",
|
|
email="e@e.com",
|
|
fetch_proxy=None,
|
|
)
|
|
assert stats["errors"] == 1
|
|
|
|
@patch("prisma.fetch.pending_queue")
|
|
@patch("prisma.fetch.fetch_one", return_value=None)
|
|
@patch("httpx.Client")
|
|
def test_missed(self, mc_cls, mc_fetch, mc_queue, tmp_path):
|
|
path = _setup(tmp_path)
|
|
item = PendingItem(zot_id=1, doi="10.1/x", pmid="", pmcid="", title="P")
|
|
mc_queue.return_value = [item]
|
|
mc_cls.return_value = MagicMock()
|
|
with Db(path) as db:
|
|
stats = run(
|
|
db,
|
|
project="t",
|
|
storage_dir=tmp_path / "storage",
|
|
scratch_dir=tmp_path / "scratch",
|
|
email="e@e.com",
|
|
fetch_proxy=None,
|
|
)
|
|
assert stats["missed"] == 1
|
|
|
|
@patch("prisma.fetch.pending_queue")
|
|
@patch("prisma.fetch.fetch_one", return_value=None)
|
|
@patch("httpx.Client")
|
|
def test_progress_print(self, mc_cls, mc_fetch, mc_queue, tmp_path, capsys):
|
|
path = _setup(tmp_path)
|
|
items = [
|
|
PendingItem(zot_id=i, doi="", pmid="", pmcid="", title="P")
|
|
for i in range(1, 26)
|
|
]
|
|
mc_queue.return_value = items
|
|
mc_cls.return_value = MagicMock()
|
|
with Db(path) as db:
|
|
run(
|
|
db,
|
|
project="t",
|
|
storage_dir=tmp_path / "storage",
|
|
scratch_dir=tmp_path / "scratch",
|
|
email="e@e.com",
|
|
fetch_proxy=None,
|
|
progress=True,
|
|
)
|
|
assert "fetched 25/" in capsys.readouterr().out
|
|
|
|
|
|
# ── field / extra helpers (lines 511-535) ────────────────────────
|
|
|
|
|
|
class TestFieldHelpers:
|
|
def test_field_missing(self, tmp_path):
|
|
path = _setup(tmp_path)
|
|
with Db(path) as db:
|
|
iid = db.create_item(TYPE_MAP["document"])
|
|
db.commit()
|
|
assert _field(db, iid, "DOI") == ""
|
|
|
|
def test_field_unknown(self, tmp_path):
|
|
path = _setup(tmp_path)
|
|
with Db(path) as db:
|
|
iid = db.create_item(TYPE_MAP["document"])
|
|
db.commit()
|
|
assert _field(db, iid, "nonexistent_xyz") == ""
|
|
|
|
def test_field_present(self, tmp_path):
|
|
path = _setup(tmp_path)
|
|
with Db(path) as db:
|
|
iid = db.create_item(TYPE_MAP["document"])
|
|
db.set_fields(iid, {"DOI": "10.1/abc"})
|
|
db.commit()
|
|
assert _field(db, iid, "DOI") == "10.1/abc"
|
|
|
|
def test_extra_value_found(self, tmp_path):
|
|
path = _setup(tmp_path)
|
|
with Db(path) as db:
|
|
iid = db.create_item(TYPE_MAP["document"])
|
|
db.set_fields(iid, {"extra": "PMID: 99999\nPMCID: PMC11111"})
|
|
db.commit()
|
|
assert _extra_value(db, iid, "PMID") == "99999"
|
|
assert _extra_value(db, iid, "PMCID") == "PMC11111"
|
|
|
|
def test_extra_value_empty(self, tmp_path):
|
|
path = _setup(tmp_path)
|
|
with Db(path) as db:
|
|
iid = db.create_item(TYPE_MAP["document"])
|
|
db.commit()
|
|
assert _extra_value(db, iid, "PMID") == ""
|
|
|
|
def test_extra_value_no_match(self, tmp_path):
|
|
path = _setup(tmp_path)
|
|
with Db(path) as db:
|
|
iid = db.create_item(TYPE_MAP["document"])
|
|
db.set_fields(iid, {"extra": "Other: stuff"})
|
|
db.commit()
|
|
assert _extra_value(db, iid, "PMID") == ""
|
|
|
|
def test_from_url_pmcid(self, tmp_path):
|
|
path = _setup(tmp_path)
|
|
with Db(path) as db:
|
|
iid = db.create_item(TYPE_MAP["document"])
|
|
db.set_fields(
|
|
iid,
|
|
{"url": "https://ncbi.nlm.nih.gov/pmc/articles/PMC12345/"},
|
|
)
|
|
db.commit()
|
|
assert _from_url_pmcid(db, iid) == "PMC12345"
|
|
|
|
def test_from_url_pmcid_no_match(self, tmp_path):
|
|
path = _setup(tmp_path)
|
|
with Db(path) as db:
|
|
iid = db.create_item(TYPE_MAP["document"])
|
|
db.set_fields(iid, {"url": "https://example.com/paper"})
|
|
db.commit()
|
|
assert _from_url_pmcid(db, iid) == ""
|
|
|
|
|
|
# ── _altcha_bootstrap edge cases (lines 221-258) ────────────────
|
|
|
|
|
|
class TestAltchaBootstrapEdges:
|
|
def test_no_origin(self):
|
|
html = 'challengeurl="/captcha/challenge/1" /captcha/solution/1'
|
|
assert _altcha_bootstrap(MagicMock(), "noproto/path", html) is False
|
|
|
|
def test_challenge_get_http_error(self):
|
|
client = MagicMock()
|
|
client.get.side_effect = httpx.HTTPError("timeout")
|
|
html = 'challengeurl="/captcha/challenge/1" /captcha/solution/1'
|
|
assert _altcha_bootstrap(client, "https://mirror.test/doi", html) is False
|
|
|
|
def test_solve_fails(self):
|
|
client = MagicMock()
|
|
chall_resp = MagicMock()
|
|
chall_resp.json.return_value = {
|
|
"salt": "s",
|
|
"challenge": "impossible" * 4,
|
|
"maxNumber": 5,
|
|
"signature": "sig",
|
|
}
|
|
client.get.return_value = chall_resp
|
|
html = 'challengeurl="/captcha/challenge/1" /captcha/solution/1'
|
|
assert _altcha_bootstrap(client, "https://mirror.test/doi", html) is False
|
|
|
|
def test_post_http_error(self):
|
|
salt, nonce = "postsalt", 3
|
|
challenge = hashlib.sha256(f"{salt}{nonce}".encode()).hexdigest()
|
|
client = MagicMock()
|
|
chall_resp = MagicMock()
|
|
chall_resp.json.return_value = {
|
|
"salt": salt,
|
|
"challenge": challenge,
|
|
"maxNumber": 100,
|
|
"signature": "sig",
|
|
}
|
|
client.get.return_value = chall_resp
|
|
client.post.side_effect = httpx.HTTPError("network")
|
|
html = 'challengeurl="/captcha/challenge/1" /captcha/solution/1'
|
|
assert _altcha_bootstrap(client, "https://mirror.test/doi", html) is False
|
|
|
|
def test_post_non_200(self):
|
|
salt, nonce = "nonsalt", 2
|
|
challenge = hashlib.sha256(f"{salt}{nonce}".encode()).hexdigest()
|
|
client = MagicMock()
|
|
chall_resp = MagicMock()
|
|
chall_resp.json.return_value = {
|
|
"salt": salt,
|
|
"challenge": challenge,
|
|
"maxNumber": 100,
|
|
"signature": "sig",
|
|
}
|
|
client.get.return_value = chall_resp
|
|
solve_resp = MagicMock()
|
|
solve_resp.status_code = 403
|
|
client.post.return_value = solve_resp
|
|
html = 'challengeurl="/captcha/challenge/1" /captcha/solution/1'
|
|
assert _altcha_bootstrap(client, "https://mirror.test/doi", html) is False
|
|
|
|
def test_post_json_parse_error(self):
|
|
salt, nonce = "jsonsalt", 1
|
|
challenge = hashlib.sha256(f"{salt}{nonce}".encode()).hexdigest()
|
|
client = MagicMock()
|
|
chall_resp = MagicMock()
|
|
chall_resp.json.return_value = {
|
|
"salt": salt,
|
|
"challenge": challenge,
|
|
"maxNumber": 100,
|
|
"signature": "sig",
|
|
}
|
|
client.get.return_value = chall_resp
|
|
solve_resp = MagicMock()
|
|
solve_resp.status_code = 200
|
|
solve_resp.json.side_effect = ValueError("bad json")
|
|
client.post.return_value = solve_resp
|
|
html = 'challengeurl="/captcha/challenge/1" /captcha/solution/1'
|
|
assert _altcha_bootstrap(client, "https://mirror.test/doi", html) is False
|
|
|
|
|
|
# ── fetch_unpaywall extra edges (lines 136-138) ─────────────────
|
|
|
|
|
|
class TestFetchUnpaywallEdges:
|
|
def test_fallback_to_url(self):
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 200
|
|
resp.json.return_value = {
|
|
"best_oa_location": {"url": "https://oa.org/landing"},
|
|
}
|
|
client.get.return_value = resp
|
|
assert fetch_unpaywall(client, "10.1/x", "e@x") == "https://oa.org/landing"
|
|
|
|
|
|
# ── resolve_doi_from_title edges (lines 165-176) ────────────────
|
|
|
|
|
|
class TestResolveDoiEdges:
|
|
def test_non_200(self):
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 500
|
|
client.get.return_value = resp
|
|
assert resolve_doi_from_title(client, "Long enough title here") is None
|
|
|
|
def test_http_error(self):
|
|
client = MagicMock()
|
|
client.get.side_effect = httpx.HTTPError("timeout")
|
|
assert resolve_doi_from_title(client, "Long enough title here") is None
|
|
|
|
def test_skip_empty_doi_or_title(self):
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 200
|
|
resp.json.return_value = {
|
|
"message": {
|
|
"items": [
|
|
{"DOI": "", "title": ["Something"], "score": 90},
|
|
{"DOI": "10.1/x", "title": [], "score": 90},
|
|
]
|
|
},
|
|
}
|
|
client.get.return_value = resp
|
|
assert resolve_doi_from_title(client, "Long enough title here") is None
|
|
|
|
|
|
# ── fetch_fallback extra edges (lines 294-318) ──────────────────
|
|
|
|
|
|
class TestFetchFallbackEdges:
|
|
def test_http_error_on_get(self):
|
|
client = MagicMock()
|
|
client.get.side_effect = httpx.HTTPError("fail")
|
|
assert fetch_fallback(client, "10.1234/test") is None
|
|
|
|
def test_non_200(self):
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 403
|
|
client.get.return_value = resp
|
|
assert fetch_fallback(client, "10.1234/test") is None
|
|
|
|
def test_altcha_success_retry_http_error(self):
|
|
client = MagicMock()
|
|
first_resp = MagicMock()
|
|
first_resp.status_code = 200
|
|
first_resp.text = "<html>altcha-widget captcha/challenge</html>"
|
|
client.get.side_effect = [
|
|
first_resp,
|
|
httpx.HTTPError("retry fail"),
|
|
httpx.HTTPError("f"),
|
|
httpx.HTTPError("f"),
|
|
httpx.HTTPError("f"),
|
|
httpx.HTTPError("f"),
|
|
]
|
|
with patch("prisma.fetch._altcha_bootstrap", return_value=True):
|
|
assert fetch_fallback(client, "10.1234/test") is None
|
|
|
|
def test_altcha_success_retry_non_200(self):
|
|
client = MagicMock()
|
|
first_resp = MagicMock()
|
|
first_resp.status_code = 200
|
|
first_resp.text = "<html>altcha-widget captcha/challenge</html>"
|
|
retry_resp = MagicMock()
|
|
retry_resp.status_code = 503
|
|
retry_resp.text = ""
|
|
client.get.side_effect = [
|
|
first_resp,
|
|
retry_resp,
|
|
first_resp,
|
|
retry_resp,
|
|
first_resp,
|
|
retry_resp,
|
|
first_resp,
|
|
retry_resp,
|
|
]
|
|
with patch("prisma.fetch._altcha_bootstrap", return_value=True):
|
|
assert fetch_fallback(client, "10.1234/test") is None
|
|
|
|
def test_success_pdf_extracted(self):
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 200
|
|
resp.text = '<embed src="/downloads/paper.pdf" type="application/pdf">'
|
|
client.get.return_value = resp
|
|
result = fetch_fallback(client, "10.1234/test")
|
|
assert result is not None and "paper.pdf" in result
|
|
|
|
def test_no_pdf_in_html(self):
|
|
client = MagicMock()
|
|
resp = MagicMock()
|
|
resp.status_code = 200
|
|
resp.text = "<html><body>No links here</body></html>"
|
|
client.get.return_value = resp
|
|
assert fetch_fallback(client, "10.1234/test") is None
|
|
|
|
|
|
# ── download extra edges (lines 329, 347-348) ───────────────────
|
|
|
|
|
|
class TestDownloadEdges:
|
|
def test_non_200(self, tmp_path):
|
|
client = MagicMock()
|
|
ctx = MagicMock()
|
|
ctx.__enter__ = MagicMock(return_value=ctx)
|
|
ctx.__exit__ = MagicMock(return_value=False)
|
|
ctx.status_code = 404
|
|
client.stream.return_value = ctx
|
|
assert download(client, "https://x.com/p.pdf", tmp_path / "t.pdf") is None
|
|
|
|
def test_os_error_on_stat(self, tmp_path):
|
|
client = MagicMock()
|
|
client.stream.return_value = _pdf_stream_ctx()
|
|
dest = tmp_path / "test.pdf"
|
|
with patch.object(Path, "stat", side_effect=OSError("no stat")):
|
|
assert download(client, "https://x.com/p.pdf", dest) is None
|
|
|
|
|
|
# ── pending_queue with real DB (line 108) ────────────────────────
|
|
|
|
|
|
class TestPendingQueueReal:
|
|
def test_returns_matching(self, tmp_path):
|
|
path = _setup(tmp_path)
|
|
with Db(path) as db:
|
|
iid = _make_item_with_tags(db, "skin-subs", "screen:include")
|
|
db.set_fields(iid, {"extra": "PMID: 12345\nPMCID: PMC67890"})
|
|
db.commit()
|
|
result = pending_queue(db, "skin-subs")
|
|
assert len(result) == 1
|
|
assert result[0].zot_id == iid
|
|
assert result[0].pmid == "12345"
|
|
assert result[0].pmcid == "PMC67890"
|
|
|
|
def test_excludes_module_prisma(self, tmp_path):
|
|
path = _setup(tmp_path)
|
|
with Db(path) as db:
|
|
iid = _make_item_with_tags(db, "proj", "screen:include")
|
|
db.sync_tags(iid, ["module:prisma"])
|
|
db.commit()
|
|
assert pending_queue(db, "proj") == []
|
|
|
|
def test_excludes_items_with_pdf(self, tmp_path):
|
|
path = _setup(tmp_path)
|
|
with Db(path) as db:
|
|
iid = _make_item_with_tags(db, "proj", "screen:include")
|
|
db.add_attachment(
|
|
iid,
|
|
content_type="application/pdf",
|
|
path="storage:t.pdf",
|
|
)
|
|
db.commit()
|
|
assert pending_queue(db, "proj") == []
|
|
|
|
def test_uncertain_qualifies(self, tmp_path):
|
|
path = _setup(tmp_path)
|
|
with Db(path) as db:
|
|
_make_item_with_tags(db, "proj", "screen:uncertain")
|
|
db.commit()
|
|
assert len(pending_queue(db, "proj")) == 1
|
|
|
|
def test_limit(self, tmp_path):
|
|
path = _setup(tmp_path)
|
|
with Db(path) as db:
|
|
_make_item_with_tags(db, "proj", "screen:include")
|
|
_make_item_with_tags(db, "proj", "screen:include")
|
|
db.commit()
|
|
assert len(pending_queue(db, "proj", limit=1)) == 1
|