diff --git a/src/zot/fetch/resolve.py b/src/zot/fetch/resolve.py new file mode 100644 index 0000000..2b050a1 --- /dev/null +++ b/src/zot/fetch/resolve.py @@ -0,0 +1,226 @@ +"""Identifiers → download candidates. + +DOI: Unpaywall (direct, legal) first, then the libgen JSON index (through +the VPN proxy). ISBN: libgen JSON. A book with no ISBN: libgen title +search → edition ids → JSON. libgen serves a bare nginx page to +non-browser user agents, so the proxied client must send ``BROWSER_UA``. +""" + +from __future__ import annotations + +import re +import time +from collections.abc import Callable, Iterable +from dataclasses import dataclass + +import httpx + +from prisma.fetch import unpaywall_urls +from zot.fetch.select import ItemRef + +LIBGEN = "https://libgen.li" +BROWSER_UA = ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/128.0 Safari/537.36" +) +_EDITION_RE = re.compile(r'edition\.php\?id=(\d+)"[^>]*>([^<]*)<') +_YEAR_RE = re.compile(r"\b(19|20)\d\d\b") +_EXT_ORDER = {"book": ("epub", "pdf"), "paper": ("pdf",)} + + +@dataclass(frozen=True) +class Candidate: + source: str # unpaywall | libgen + md5: str | None + url: str | None + ext: str + size_bytes: int | None + title: str + + +class Pacer: + """Keep calls at least *min_interval* seconds apart.""" + + def __init__( + self, + min_interval: float, + *, + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, + ) -> None: + self.min_interval = min_interval + self._clock = clock + self._sleep = sleep + self._last: float | None = None + + def wait(self) -> None: + now = self._clock() + if self._last is not None: + remaining = self.min_interval - (now - self._last) + if remaining > 0: + self._sleep(remaining) + self._last = self._clock() + + +def _get_json( + client: httpx.Client, url: str, params: dict[str, str], pacer: Pacer +) -> dict: + pacer.wait() + try: + r = client.get(url, params=params, timeout=30) + if r.status_code != 200: + return {} + data = r.json() + except (httpx.HTTPError, ValueError): + return {} + return data if isinstance(data, dict) else {} + + +def libgen_editions( + client: httpx.Client, + *, + doi: str = "", + isbn: str = "", + ids: Iterable[str] = (), + pacer: Pacer, +) -> dict: + params = {"object": "e", "addkeys": "*"} + if doi: + params["doi"] = doi + elif isbn: + params["isbn"] = isbn + else: + params["ids"] = ",".join(ids) + return _get_json(client, f"{LIBGEN}/json.php", params, pacer) + + +def libgen_files(client: httpx.Client, f_ids: list[str], *, pacer: Pacer) -> dict: + if not f_ids: + return {} + params = {"object": "f", "addkeys": "*", "ids": ",".join(f_ids)} + return _get_json(client, f"{LIBGEN}/json.php", params, pacer) + + +def _candidates_from_editions( + client: httpx.Client, editions: dict, *, kind: str, pacer: Pacer +) -> list[Candidate]: + files: list[tuple[str, str, str]] = [] # (f_id, md5, title) + for ed in editions.values(): + if not isinstance(ed, dict): + continue + title = ed.get("title") or "" + raw = ed.get("files") + entries = raw.values() if isinstance(raw, dict) else (raw or []) + for f in entries: + md5 = (f.get("md5") or "").lower() + if md5: + files.append((str(f.get("f_id") or ""), md5, title)) + info = libgen_files(client, [fid for fid, _, _ in files if fid], pacer=pacer) + out: list[Candidate] = [] + for fid, md5, title in files: + meta = info.get(fid) or {} + ext = (meta.get("extension") or "").lower() + if not ext and kind == "paper": + ext = "pdf" + if ext not in _EXT_ORDER[kind]: + continue + size = meta.get("filesize") + out.append( + Candidate("libgen", md5, None, ext, int(size) if size else None, title) + ) + order = _EXT_ORDER[kind] + out.sort(key=lambda c: order.index(c.ext)) + return out + + +def libgen_candidates( + client: httpx.Client, *, kind: str, doi: str = "", isbn: str = "", pacer: Pacer +) -> list[Candidate]: + editions = libgen_editions(client, doi=doi, isbn=isbn, pacer=pacer) + if not editions: + return [] + return _candidates_from_editions(client, editions, kind=kind, pacer=pacer) + + +def _norm(s: str) -> str: + return re.sub(r"[^a-z0-9]+", " ", s.lower()).strip() + + +def _year(s: str) -> str: + m = _YEAR_RE.search(s or "") + return m.group(0) if m else "" + + +def title_matches(target: str, year: str, entries: list[dict]) -> list[dict]: + """Exact normalised title, else containment with agreeing years + (the AMA script's folder-matching rule).""" + t = _norm(target) + exact = [e for e in entries if _norm(e.get("title", "")) == t] + if exact: + return exact + tight = t.replace(" ", "") + out: list[dict] = [] + for e in entries: + ey = _year(e.get("title", "")) or str(e.get("year") or "") + if year and ey and year != ey: + continue + cand = _norm(e.get("title", "")).replace(" ", "") + if cand and (cand in tight or tight in cand): + out.append(e) + return out + + +def title_candidates( + client: httpx.Client, *, title: str, year: str, pacer: Pacer +) -> list[Candidate]: + if not title: + return [] + pacer.wait() + params = { + "req": title, + "columns[]": "t", + "objects[]": "e", + "topics[]": "l", + "res": "25", + } + try: + r = client.get(f"{LIBGEN}/index.php", params=params, timeout=30) + except httpx.HTTPError: + return [] + if r.status_code != 200: + return [] + entries = [{"id": i, "title": t.strip()} for i, t in _EDITION_RE.findall(r.text)] + hits = title_matches(title, year, entries) + if not hits: + return [] + editions = libgen_editions(client, ids=[h["id"] for h in hits], pacer=pacer) + return _candidates_from_editions(client, editions, kind="book", pacer=pacer) + + +def candidates( + ref: ItemRef, + *, + direct: httpx.Client, + proxied: httpx.Client, + email: str, + up_pacer: Pacer, + lg_pacer: Pacer, +) -> list[Candidate]: + """Ordered candidates for one item: Unpaywall URLs, then libgen md5s.""" + out: list[Candidate] = [] + if ref.doi: + up_pacer.wait() + for url in unpaywall_urls(direct, ref.doi, email): + out.append(Candidate("unpaywall", None, url, "pdf", None, ref.title)) + out.extend( + libgen_candidates(proxied, kind=ref.kind, doi=ref.doi, pacer=lg_pacer) + ) + elif ref.isbn: + out.extend( + libgen_candidates(proxied, kind=ref.kind, isbn=ref.isbn, pacer=lg_pacer) + ) + elif ref.kind == "book" and ref.title: + out.extend( + title_candidates(proxied, title=ref.title, year=ref.year, pacer=lg_pacer) + ) + return out diff --git a/tests/zot/fetch/fixtures/libgen_index.html b/tests/zot/fetch/fixtures/libgen_index.html new file mode 100644 index 0000000..9ab462c --- /dev/null +++ b/tests/zot/fetch/fixtures/libgen_index.html @@ -0,0 +1,4 @@ + + + +
HCPCS Level II Professional 20232023
HCPCS Level II Professional 20192019
diff --git a/tests/zot/fetch/fixtures/libgen_json_doi.json b/tests/zot/fetch/fixtures/libgen_json_doi.json new file mode 100644 index 0000000..476dad1 --- /dev/null +++ b/tests/zot/fetch/fixtures/libgen_json_doi.json @@ -0,0 +1 @@ +{"27703795": {"title": "Early Palliative Care for Patients with Metastatic Non–Small-Cell Lung Cancer", "libgen_topic": "a", "type": "a", "author": "Temel, Jennifer S.; Greer, Joseph A.", "year": "2010", "doi": "10.1056/NEJMoa1000678", "files": {"26575841": {"f_id": "27858481", "md5": "a78add1bd62047ead5945134488f470f", "time_added": "2021-07-17 13:59:28"}, "26575842": {"f_id": "27858482", "md5": "b78add1bd62047ead5945134488f470f", "time_added": "2021-07-17 13:59:28"}}, "add": {"68149362": {"key": "904", "value": "20818875", "name_en": "PMID"}}}} diff --git a/tests/zot/fetch/fixtures/libgen_json_file.json b/tests/zot/fetch/fixtures/libgen_json_file.json new file mode 100644 index 0000000..866f443 --- /dev/null +++ b/tests/zot/fetch/fixtures/libgen_json_file.json @@ -0,0 +1 @@ +{"27858481": {"md5": "a78add1bd62047ead5945134488f470f", "filesize": "401662", "extension": "pdf"}, "104009164": {"md5": "120ddd118103ea2ea927aaa489310dc8", "filesize": "9000000", "extension": "pdf"}, "104009165": {"md5": "220ddd118103ea2ea927aaa489310dc8", "filesize": "4000000", "extension": "epub"}} diff --git a/tests/zot/fetch/fixtures/libgen_json_isbn.json b/tests/zot/fetch/fixtures/libgen_json_isbn.json new file mode 100644 index 0000000..9ba93ab --- /dev/null +++ b/tests/zot/fetch/fixtures/libgen_json_isbn.json @@ -0,0 +1 @@ +{"146962484": {"title": "HCPCS Level II Professional 2023", "libgen_topic": "l", "type": "b", "year": "2023", "edition": "1", "files": {"103379671": {"f_id": "104009164", "md5": "120ddd118103ea2ea927aaa489310dc8"}, "103379672": {"f_id": "104009165", "md5": "220ddd118103ea2ea927aaa489310dc8"}}}} diff --git a/tests/zot/fetch/fixtures/unpaywall_doi.json b/tests/zot/fetch/fixtures/unpaywall_doi.json new file mode 100644 index 0000000..e087a26 --- /dev/null +++ b/tests/zot/fetch/fixtures/unpaywall_doi.json @@ -0,0 +1 @@ +{"doi": "10.1056/nejmoa1000678", "is_oa": true, "best_oa_location": {"url_for_pdf": "https://example.org/best.pdf", "url": "https://example.org/best"}, "oa_locations": [{"url_for_pdf": "https://example.org/best.pdf", "url": "https://example.org/best"}, {"url_for_pdf": null, "url": "https://example.org/landing"}]} diff --git a/tests/zot/fetch/test_resolve.py b/tests/zot/fetch/test_resolve.py new file mode 100644 index 0000000..b253ef3 --- /dev/null +++ b/tests/zot/fetch/test_resolve.py @@ -0,0 +1,263 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import httpx +import pytest + +from zot.fetch.resolve import ( + BROWSER_UA, + LIBGEN, + Candidate, + Pacer, + candidates, + libgen_candidates, + libgen_editions, + libgen_files, + title_candidates, + title_matches, +) +from zot.fetch.select import ItemRef + +FX = Path(__file__).parent / "fixtures" +TEMEL = "Early Palliative Care for Patients with Metastatic Non–Small-Cell Lung Cancer" + + +def _fx(name: str) -> str: + return (FX / name).read_text() + + +def _pacer() -> Pacer: + return Pacer(3.0, clock=lambda: 0.0, sleep=lambda s: None) + + +def _client(handler) -> httpx.Client: + return httpx.Client( + transport=httpx.MockTransport(handler), headers={"User-Agent": BROWSER_UA} + ) + + +def _libgen_handler(seen: list[str]): + def handler(req: httpx.Request) -> httpx.Response: + seen.append(str(req.url)) + assert req.headers["user-agent"] == BROWSER_UA + q = req.url.params + if req.url.path == "/json.php" and q.get("object") == "e": + if q.get("doi") == "10.1056/NEJMoa1000678": + return httpx.Response(200, text=_fx("libgen_json_doi.json")) + if q.get("isbn") == "9781640162280" or q.get("ids") == "146962484": + return httpx.Response(200, text=_fx("libgen_json_isbn.json")) + return httpx.Response(200, text="[]") + if req.url.path == "/json.php" and q.get("object") == "f": + want = set(q.get("ids", "").split(",")) + allf = json.loads(_fx("libgen_json_file.json")) + return httpx.Response( + 200, json={k: v for k, v in allf.items() if k in want} + ) + if req.url.path == "/index.php": + return httpx.Response(200, text=_fx("libgen_index.html")) + return httpx.Response(404) + + return handler + + +def test_pacer_sleeps_only_the_remaining_interval(): + now = [100.0] + slept: list[float] = [] + p = Pacer(3.0, clock=lambda: now[0], sleep=slept.append) + p.wait() # first call never sleeps + now[0] += 1.0 + p.wait() + assert slept == [2.0] + now[0] += 10.0 + p.wait() + assert slept == [2.0] + + +def test_libgen_editions_doi_and_isbn_and_ids(): + seen: list[str] = [] + c = _client(_libgen_handler(seen)) + e = libgen_editions(c, doi="10.1056/NEJMoa1000678", pacer=_pacer()) + assert list(e) == ["27703795"] + assert list(libgen_editions(c, isbn="9781640162280", pacer=_pacer())) == [ + "146962484" + ] + assert list(libgen_editions(c, ids=("146962484",), pacer=_pacer())) == ["146962484"] + assert seen[0].startswith(f"{LIBGEN}/json.php?object=e&addkeys=%2A&doi=") + + +@pytest.mark.parametrize( + "status, body", + [(500, "x"), (200, "nginx"), (200, "[]"), (200, "null")], +) +def test_libgen_editions_bad_answers_are_empty(status, body): + c = _client(lambda req: httpx.Response(status, text=body)) + assert libgen_editions(c, doi="10.1/x", pacer=_pacer()) == {} + + +def test_libgen_editions_transport_error_is_empty(): + def boom(req): + raise httpx.ConnectError("down") + + assert libgen_editions(_client(boom), doi="10.1/x", pacer=_pacer()) == {} + + +def test_libgen_files_batches_ids_in_one_call(): + seen: list[str] = [] + c = _client(_libgen_handler(seen)) + f = libgen_files(c, ["27858481", "104009164"], pacer=_pacer()) + assert set(f) == {"27858481", "104009164"} + assert len(seen) == 1 and "ids=27858481%2C104009164" in seen[0] + assert libgen_files(c, [], pacer=_pacer()) == {} + + +def test_libgen_candidates_paper_pdf_only_with_size(): + c = _client(_libgen_handler([])) + got = libgen_candidates( + c, kind="paper", doi="10.1056/NEJMoa1000678", pacer=_pacer() + ) + assert got == [ + Candidate( + "libgen", "a78add1bd62047ead5945134488f470f", None, "pdf", 401662, TEMEL + ), + Candidate( + "libgen", "b78add1bd62047ead5945134488f470f", None, "pdf", None, TEMEL + ), + ] + + +def test_libgen_candidates_book_epub_before_pdf(): + c = _client(_libgen_handler([])) + got = libgen_candidates(c, kind="book", isbn="9781640162280", pacer=_pacer()) + assert [(g.ext, g.md5[:3]) for g in got] == [("epub", "220"), ("pdf", "120")] + + +def test_libgen_file_lookup_missing_defaults_paper_to_pdf_drops_book(): + def handler(req: httpx.Request) -> httpx.Response: + q = req.url.params + if q.get("object") == "f": + return httpx.Response(503) + if q.get("doi"): + return httpx.Response(200, text=_fx("libgen_json_doi.json")) + return httpx.Response(200, text=_fx("libgen_json_isbn.json")) + + c = _client(handler) + paper = libgen_candidates( + c, kind="paper", doi="10.1056/NEJMoa1000678", pacer=_pacer() + ) + assert [p.ext for p in paper] == ["pdf", "pdf"] and paper[0].size_bytes is None + book = libgen_candidates(c, kind="book", isbn="9781640162280", pacer=_pacer()) + assert book == [] + + +def test_libgen_candidates_skips_files_without_md5(): + body = json.dumps( + {"1": {"title": "T", "files": {"9": {"f_id": "9"}}, "year": "2000"}} + ) + c = _client(lambda req: httpx.Response(200, text=body)) + assert libgen_candidates(c, kind="paper", doi="10.1/x", pacer=_pacer()) == [] + + +def test_title_matches_exact_then_containment_with_year(): + entries = [ + {"id": "1", "title": "HCPCS Level II Professional 2023", "year": "2023"}, + {"id": "2", "title": "HCPCS Level II Professional 2019", "year": "2019"}, + {"id": "3", "title": "CPT 2023 Professional", "year": "2023"}, + ] + exact = title_matches("HCPCS Level II Professional 2023", "", entries) + assert [e["id"] for e in exact] == ["1"] + by_year = title_matches("HCPCS Level II Professional", "2019", entries) + assert [e["id"] for e in by_year] == ["2"] + both = title_matches("HCPCS Level II Professional", "", entries) + assert [e["id"] for e in both] == ["1", "2"] + assert title_matches("Netter's Atlas", "", entries) == [] + + +def test_title_candidates_parses_index_then_fetches_editions(): + seen: list[str] = [] + c = _client(_libgen_handler(seen)) + got = title_candidates( + c, title="HCPCS Level II Professional", year="2023", pacer=_pacer() + ) + assert [g.ext for g in got] == ["epub", "pdf"] + assert any( + "/index.php?" in s and "req=HCPCS+Level+II+Professional" in s for s in seen + ) + assert any("object=e" in s and "ids=146962484" in s for s in seen) + + +def test_title_candidates_no_rows_or_error_is_empty(): + empty = _client(lambda r: httpx.Response(200, text="")) + assert title_candidates(empty, title="X", year="", pacer=_pacer()) == [] + err = _client(lambda r: httpx.Response(500)) + assert title_candidates(err, title="X", year="", pacer=_pacer()) == [] + ok = _client(lambda r: httpx.Response(200)) + assert title_candidates(ok, title="", year="", pacer=_pacer()) == [] + + +def test_title_candidates_transport_error_is_empty(): + def boom(req): + raise httpx.ConnectError("down") + + assert title_candidates(_client(boom), title="X", year="", pacer=_pacer()) == [] + + +def _direct(oa: bool): + def handler(req: httpx.Request) -> httpx.Response: + assert req.url.host == "api.unpaywall.org" + assert req.url.params["email"] == "dev@fhirworx.io" + if not oa: + return httpx.Response(404) + return httpx.Response(200, text=_fx("unpaywall_doi.json")) + + return httpx.Client(transport=httpx.MockTransport(handler)) + + +def _ref(**kw) -> ItemRef: + base = dict( + item_id=1, + key="ABCD2345", + item_type="journalArticle", + title="T", + doi="", + isbn="", + year="", + ) + base.update(kw) + return ItemRef(**base) + + +def test_candidates_doi_unpaywall_first_then_libgen(): + got = candidates( + _ref(doi="10.1056/NEJMoa1000678"), + direct=_direct(True), + proxied=_client(_libgen_handler([])), + email="dev@fhirworx.io", + up_pacer=_pacer(), + lg_pacer=_pacer(), + ) + assert [g.source for g in got] == ["unpaywall"] * 3 + ["libgen"] * 2 + assert got[0] == Candidate( + "unpaywall", None, "https://example.org/best.pdf", "pdf", None, "T" + ) + assert got[3].md5 == "a78add1bd62047ead5945134488f470f" + + +def test_candidates_isbn_book_and_title_fallback_and_nothing(): + proxied = _client(_libgen_handler([])) + kw = dict( + direct=_direct(False), + proxied=proxied, + email="dev@fhirworx.io", + up_pacer=_pacer(), + lg_pacer=_pacer(), + ) + book = candidates(_ref(item_type="book", isbn="9781640162280"), **kw) + assert [b.source for b in book] == ["libgen", "libgen"] + by_title = candidates( + _ref(item_type="book", title="HCPCS Level II Professional", year="2023"), **kw + ) + assert [b.ext for b in by_title] == ["epub", "pdf"] + assert candidates(_ref(title="paper without doi"), **kw) == [] + assert candidates(_ref(item_type="book", title=""), **kw) == []