feat(zot): fetch.resolve — Unpaywall + libgen JSON (DOI/ISBN/title) → ordered candidates (refs Z1)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

src/zot/fetch/resolve.py

@@ -0,0 +1,226 @@
+"""Identifiers → download candidates.
+
+DOI: Unpaywall (direct, legal) first, then the libgen JSON index (through
+the VPN proxy). ISBN: libgen JSON. A book with no ISBN: libgen title
+search → edition ids → JSON. libgen serves a bare nginx page to
+non-browser user agents, so the proxied client must send ``BROWSER_UA``.
+"""
+
+from __future__ import annotations
+
+import re
+import time
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+
+import httpx
+
+from prisma.fetch import unpaywall_urls
+from zot.fetch.select import ItemRef
+
+LIBGEN = "https://libgen.li"
+BROWSER_UA = (
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/128.0 Safari/537.36"
+)
+_EDITION_RE = re.compile(r'edition\.php\?id=(\d+)"[^>]*>([^<]*)<')
+_YEAR_RE = re.compile(r"\b(19|20)\d\d\b")
+_EXT_ORDER = {"book": ("epub", "pdf"), "paper": ("pdf",)}
+
+
+@dataclass(frozen=True)
+class Candidate:
+    source: str  # unpaywall | libgen
+    md5: str | None
+    url: str | None
+    ext: str
+    size_bytes: int | None
+    title: str
+
+
+class Pacer:
+    """Keep calls at least *min_interval* seconds apart."""
+
+    def __init__(
+        self,
+        min_interval: float,
+        *,
+        clock: Callable[[], float] = time.monotonic,
+        sleep: Callable[[float], None] = time.sleep,
+    ) -> None:
+        self.min_interval = min_interval
+        self._clock = clock
+        self._sleep = sleep
+        self._last: float | None = None
+
+    def wait(self) -> None:
+        now = self._clock()
+        if self._last is not None:
+            remaining = self.min_interval - (now - self._last)
+            if remaining > 0:
+                self._sleep(remaining)
+        self._last = self._clock()
+
+
+def _get_json(
+    client: httpx.Client, url: str, params: dict[str, str], pacer: Pacer
+) -> dict:
+    pacer.wait()
+    try:
+        r = client.get(url, params=params, timeout=30)
+        if r.status_code != 200:
+            return {}
+        data = r.json()
+    except (httpx.HTTPError, ValueError):
+        return {}
+    return data if isinstance(data, dict) else {}
+
+
+def libgen_editions(
+    client: httpx.Client,
+    *,
+    doi: str = "",
+    isbn: str = "",
+    ids: Iterable[str] = (),
+    pacer: Pacer,
+) -> dict:
+    params = {"object": "e", "addkeys": "*"}
+    if doi:
+        params["doi"] = doi
+    elif isbn:
+        params["isbn"] = isbn
+    else:
+        params["ids"] = ",".join(ids)
+    return _get_json(client, f"{LIBGEN}/json.php", params, pacer)
+
+
+def libgen_files(client: httpx.Client, f_ids: list[str], *, pacer: Pacer) -> dict:
+    if not f_ids:
+        return {}
+    params = {"object": "f", "addkeys": "*", "ids": ",".join(f_ids)}
+    return _get_json(client, f"{LIBGEN}/json.php", params, pacer)
+
+
+def _candidates_from_editions(
+    client: httpx.Client, editions: dict, *, kind: str, pacer: Pacer
+) -> list[Candidate]:
+    files: list[tuple[str, str, str]] = []  # (f_id, md5, title)
+    for ed in editions.values():
+        if not isinstance(ed, dict):
+            continue
+        title = ed.get("title") or ""
+        raw = ed.get("files")
+        entries = raw.values() if isinstance(raw, dict) else (raw or [])
+        for f in entries:
+            md5 = (f.get("md5") or "").lower()
+            if md5:
+                files.append((str(f.get("f_id") or ""), md5, title))
+    info = libgen_files(client, [fid for fid, _, _ in files if fid], pacer=pacer)
+    out: list[Candidate] = []
+    for fid, md5, title in files:
+        meta = info.get(fid) or {}
+        ext = (meta.get("extension") or "").lower()
+        if not ext and kind == "paper":
+            ext = "pdf"
+        if ext not in _EXT_ORDER[kind]:
+            continue
+        size = meta.get("filesize")
+        out.append(
+            Candidate("libgen", md5, None, ext, int(size) if size else None, title)
+        )
+    order = _EXT_ORDER[kind]
+    out.sort(key=lambda c: order.index(c.ext))
+    return out
+
+
+def libgen_candidates(
+    client: httpx.Client, *, kind: str, doi: str = "", isbn: str = "", pacer: Pacer
+) -> list[Candidate]:
+    editions = libgen_editions(client, doi=doi, isbn=isbn, pacer=pacer)
+    if not editions:
+        return []
+    return _candidates_from_editions(client, editions, kind=kind, pacer=pacer)
+
+
+def _norm(s: str) -> str:
+    return re.sub(r"[^a-z0-9]+", " ", s.lower()).strip()
+
+
+def _year(s: str) -> str:
+    m = _YEAR_RE.search(s or "")
+    return m.group(0) if m else ""
+
+
+def title_matches(target: str, year: str, entries: list[dict]) -> list[dict]:
+    """Exact normalised title, else containment with agreeing years
+    (the AMA script's folder-matching rule)."""
+    t = _norm(target)
+    exact = [e for e in entries if _norm(e.get("title", "")) == t]
+    if exact:
+        return exact
+    tight = t.replace(" ", "")
+    out: list[dict] = []
+    for e in entries:
+        ey = _year(e.get("title", "")) or str(e.get("year") or "")
+        if year and ey and year != ey:
+            continue
+        cand = _norm(e.get("title", "")).replace(" ", "")
+        if cand and (cand in tight or tight in cand):
+            out.append(e)
+    return out
+
+
+def title_candidates(
+    client: httpx.Client, *, title: str, year: str, pacer: Pacer
+) -> list[Candidate]:
+    if not title:
+        return []
+    pacer.wait()
+    params = {
+        "req": title,
+        "columns[]": "t",
+        "objects[]": "e",
+        "topics[]": "l",
+        "res": "25",
+    }
+    try:
+        r = client.get(f"{LIBGEN}/index.php", params=params, timeout=30)
+    except httpx.HTTPError:
+        return []
+    if r.status_code != 200:
+        return []
+    entries = [{"id": i, "title": t.strip()} for i, t in _EDITION_RE.findall(r.text)]
+    hits = title_matches(title, year, entries)
+    if not hits:
+        return []
+    editions = libgen_editions(client, ids=[h["id"] for h in hits], pacer=pacer)
+    return _candidates_from_editions(client, editions, kind="book", pacer=pacer)
+
+
+def candidates(
+    ref: ItemRef,
+    *,
+    direct: httpx.Client,
+    proxied: httpx.Client,
+    email: str,
+    up_pacer: Pacer,
+    lg_pacer: Pacer,
+) -> list[Candidate]:
+    """Ordered candidates for one item: Unpaywall URLs, then libgen md5s."""
+    out: list[Candidate] = []
+    if ref.doi:
+        up_pacer.wait()
+        for url in unpaywall_urls(direct, ref.doi, email):
+            out.append(Candidate("unpaywall", None, url, "pdf", None, ref.title))
+        out.extend(
+            libgen_candidates(proxied, kind=ref.kind, doi=ref.doi, pacer=lg_pacer)
+        )
+    elif ref.isbn:
+        out.extend(
+            libgen_candidates(proxied, kind=ref.kind, isbn=ref.isbn, pacer=lg_pacer)
+        )
+    elif ref.kind == "book" and ref.title:
+        out.extend(
+            title_candidates(proxied, title=ref.title, year=ref.year, pacer=lg_pacer)
+        )
+    return out

tests/zot/fetch/fixtures/libgen_index.html

@@ -0,0 +1,4 @@
+<html><body><table id="tablelibgen"><tbody>
+<tr><td><a href="edition.php?id=146962484">HCPCS Level II Professional 2023</a></td><td>2023</td></tr>
+<tr><td><a href="edition.php?id=146962485">HCPCS Level II Professional 2019</a></td><td>2019</td></tr>
+</tbody></table></body></html>

tests/zot/fetch/fixtures/libgen_json_doi.json

@@ -0,0 +1 @@
+{"27703795": {"title": "Early Palliative Care for Patients with Metastatic Non–Small-Cell Lung Cancer", "libgen_topic": "a", "type": "a", "author": "Temel, Jennifer S.; Greer, Joseph A.", "year": "2010", "doi": "10.1056/NEJMoa1000678", "files": {"26575841": {"f_id": "27858481", "md5": "a78add1bd62047ead5945134488f470f", "time_added": "2021-07-17 13:59:28"}, "26575842": {"f_id": "27858482", "md5": "b78add1bd62047ead5945134488f470f", "time_added": "2021-07-17 13:59:28"}}, "add": {"68149362": {"key": "904", "value": "20818875", "name_en": "PMID"}}}}

tests/zot/fetch/fixtures/libgen_json_file.json

@@ -0,0 +1 @@
+{"27858481": {"md5": "a78add1bd62047ead5945134488f470f", "filesize": "401662", "extension": "pdf"}, "104009164": {"md5": "120ddd118103ea2ea927aaa489310dc8", "filesize": "9000000", "extension": "pdf"}, "104009165": {"md5": "220ddd118103ea2ea927aaa489310dc8", "filesize": "4000000", "extension": "epub"}}

tests/zot/fetch/fixtures/libgen_json_isbn.json

@@ -0,0 +1 @@
+{"146962484": {"title": "HCPCS Level II Professional 2023", "libgen_topic": "l", "type": "b", "year": "2023", "edition": "1", "files": {"103379671": {"f_id": "104009164", "md5": "120ddd118103ea2ea927aaa489310dc8"}, "103379672": {"f_id": "104009165", "md5": "220ddd118103ea2ea927aaa489310dc8"}}}}

tests/zot/fetch/fixtures/unpaywall_doi.json

@@ -0,0 +1 @@
+{"doi": "10.1056/nejmoa1000678", "is_oa": true, "best_oa_location": {"url_for_pdf": "https://example.org/best.pdf", "url": "https://example.org/best"}, "oa_locations": [{"url_for_pdf": "https://example.org/best.pdf", "url": "https://example.org/best"}, {"url_for_pdf": null, "url": "https://example.org/landing"}]}

tests/zot/fetch/test_resolve.py

@@ -0,0 +1,263 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import httpx
+import pytest
+
+from zot.fetch.resolve import (
+    BROWSER_UA,
+    LIBGEN,
+    Candidate,
+    Pacer,
+    candidates,
+    libgen_candidates,
+    libgen_editions,
+    libgen_files,
+    title_candidates,
+    title_matches,
+)
+from zot.fetch.select import ItemRef
+
+FX = Path(__file__).parent / "fixtures"
+TEMEL = "Early Palliative Care for Patients with Metastatic Non–Small-Cell Lung Cancer"
+
+
+def _fx(name: str) -> str:
+    return (FX / name).read_text()
+
+
+def _pacer() -> Pacer:
+    return Pacer(3.0, clock=lambda: 0.0, sleep=lambda s: None)
+
+
+def _client(handler) -> httpx.Client:
+    return httpx.Client(
+        transport=httpx.MockTransport(handler), headers={"User-Agent": BROWSER_UA}
+    )
+
+
+def _libgen_handler(seen: list[str]):
+    def handler(req: httpx.Request) -> httpx.Response:
+        seen.append(str(req.url))
+        assert req.headers["user-agent"] == BROWSER_UA
+        q = req.url.params
+        if req.url.path == "/json.php" and q.get("object") == "e":
+            if q.get("doi") == "10.1056/NEJMoa1000678":
+                return httpx.Response(200, text=_fx("libgen_json_doi.json"))
+            if q.get("isbn") == "9781640162280" or q.get("ids") == "146962484":
+                return httpx.Response(200, text=_fx("libgen_json_isbn.json"))
+            return httpx.Response(200, text="[]")
+        if req.url.path == "/json.php" and q.get("object") == "f":
+            want = set(q.get("ids", "").split(","))
+            allf = json.loads(_fx("libgen_json_file.json"))
+            return httpx.Response(
+                200, json={k: v for k, v in allf.items() if k in want}
+            )
+        if req.url.path == "/index.php":
+            return httpx.Response(200, text=_fx("libgen_index.html"))
+        return httpx.Response(404)
+
+    return handler
+
+
+def test_pacer_sleeps_only_the_remaining_interval():
+    now = [100.0]
+    slept: list[float] = []
+    p = Pacer(3.0, clock=lambda: now[0], sleep=slept.append)
+    p.wait()  # first call never sleeps
+    now[0] += 1.0
+    p.wait()
+    assert slept == [2.0]
+    now[0] += 10.0
+    p.wait()
+    assert slept == [2.0]
+
+
+def test_libgen_editions_doi_and_isbn_and_ids():
+    seen: list[str] = []
+    c = _client(_libgen_handler(seen))
+    e = libgen_editions(c, doi="10.1056/NEJMoa1000678", pacer=_pacer())
+    assert list(e) == ["27703795"]
+    assert list(libgen_editions(c, isbn="9781640162280", pacer=_pacer())) == [
+        "146962484"
+    ]
+    assert list(libgen_editions(c, ids=("146962484",), pacer=_pacer())) == ["146962484"]
+    assert seen[0].startswith(f"{LIBGEN}/json.php?object=e&addkeys=%2A&doi=")
+
+
+@pytest.mark.parametrize(
+    "status, body",
+    [(500, "x"), (200, "<html>nginx</html>"), (200, "[]"), (200, "null")],
+)
+def test_libgen_editions_bad_answers_are_empty(status, body):
+    c = _client(lambda req: httpx.Response(status, text=body))
+    assert libgen_editions(c, doi="10.1/x", pacer=_pacer()) == {}
+
+
+def test_libgen_editions_transport_error_is_empty():
+    def boom(req):
+        raise httpx.ConnectError("down")
+
+    assert libgen_editions(_client(boom), doi="10.1/x", pacer=_pacer()) == {}
+
+
+def test_libgen_files_batches_ids_in_one_call():
+    seen: list[str] = []
+    c = _client(_libgen_handler(seen))
+    f = libgen_files(c, ["27858481", "104009164"], pacer=_pacer())
+    assert set(f) == {"27858481", "104009164"}
+    assert len(seen) == 1 and "ids=27858481%2C104009164" in seen[0]
+    assert libgen_files(c, [], pacer=_pacer()) == {}
+
+
+def test_libgen_candidates_paper_pdf_only_with_size():
+    c = _client(_libgen_handler([]))
+    got = libgen_candidates(
+        c, kind="paper", doi="10.1056/NEJMoa1000678", pacer=_pacer()
+    )
+    assert got == [
+        Candidate(
+            "libgen", "a78add1bd62047ead5945134488f470f", None, "pdf", 401662, TEMEL
+        ),
+        Candidate(
+            "libgen", "b78add1bd62047ead5945134488f470f", None, "pdf", None, TEMEL
+        ),
+    ]
+
+
+def test_libgen_candidates_book_epub_before_pdf():
+    c = _client(_libgen_handler([]))
+    got = libgen_candidates(c, kind="book", isbn="9781640162280", pacer=_pacer())
+    assert [(g.ext, g.md5[:3]) for g in got] == [("epub", "220"), ("pdf", "120")]
+
+
+def test_libgen_file_lookup_missing_defaults_paper_to_pdf_drops_book():
+    def handler(req: httpx.Request) -> httpx.Response:
+        q = req.url.params
+        if q.get("object") == "f":
+            return httpx.Response(503)
+        if q.get("doi"):
+            return httpx.Response(200, text=_fx("libgen_json_doi.json"))
+        return httpx.Response(200, text=_fx("libgen_json_isbn.json"))
+
+    c = _client(handler)
+    paper = libgen_candidates(
+        c, kind="paper", doi="10.1056/NEJMoa1000678", pacer=_pacer()
+    )
+    assert [p.ext for p in paper] == ["pdf", "pdf"] and paper[0].size_bytes is None
+    book = libgen_candidates(c, kind="book", isbn="9781640162280", pacer=_pacer())
+    assert book == []
+
+
+def test_libgen_candidates_skips_files_without_md5():
+    body = json.dumps(
+        {"1": {"title": "T", "files": {"9": {"f_id": "9"}}, "year": "2000"}}
+    )
+    c = _client(lambda req: httpx.Response(200, text=body))
+    assert libgen_candidates(c, kind="paper", doi="10.1/x", pacer=_pacer()) == []
+
+
+def test_title_matches_exact_then_containment_with_year():
+    entries = [
+        {"id": "1", "title": "HCPCS Level II Professional 2023", "year": "2023"},
+        {"id": "2", "title": "HCPCS Level II Professional 2019", "year": "2019"},
+        {"id": "3", "title": "CPT 2023 Professional", "year": "2023"},
+    ]
+    exact = title_matches("HCPCS Level II Professional 2023", "", entries)
+    assert [e["id"] for e in exact] == ["1"]
+    by_year = title_matches("HCPCS Level II Professional", "2019", entries)
+    assert [e["id"] for e in by_year] == ["2"]
+    both = title_matches("HCPCS Level II Professional", "", entries)
+    assert [e["id"] for e in both] == ["1", "2"]
+    assert title_matches("Netter's Atlas", "", entries) == []
+
+
+def test_title_candidates_parses_index_then_fetches_editions():
+    seen: list[str] = []
+    c = _client(_libgen_handler(seen))
+    got = title_candidates(
+        c, title="HCPCS Level II Professional", year="2023", pacer=_pacer()
+    )
+    assert [g.ext for g in got] == ["epub", "pdf"]
+    assert any(
+        "/index.php?" in s and "req=HCPCS+Level+II+Professional" in s for s in seen
+    )
+    assert any("object=e" in s and "ids=146962484" in s for s in seen)
+
+
+def test_title_candidates_no_rows_or_error_is_empty():
+    empty = _client(lambda r: httpx.Response(200, text="<html></html>"))
+    assert title_candidates(empty, title="X", year="", pacer=_pacer()) == []
+    err = _client(lambda r: httpx.Response(500))
+    assert title_candidates(err, title="X", year="", pacer=_pacer()) == []
+    ok = _client(lambda r: httpx.Response(200))
+    assert title_candidates(ok, title="", year="", pacer=_pacer()) == []
+
+
+def test_title_candidates_transport_error_is_empty():
+    def boom(req):
+        raise httpx.ConnectError("down")
+
+    assert title_candidates(_client(boom), title="X", year="", pacer=_pacer()) == []
+
+
+def _direct(oa: bool):
+    def handler(req: httpx.Request) -> httpx.Response:
+        assert req.url.host == "api.unpaywall.org"
+        assert req.url.params["email"] == "dev@fhirworx.io"
+        if not oa:
+            return httpx.Response(404)
+        return httpx.Response(200, text=_fx("unpaywall_doi.json"))
+
+    return httpx.Client(transport=httpx.MockTransport(handler))
+
+
+def _ref(**kw) -> ItemRef:
+    base = dict(
+        item_id=1,
+        key="ABCD2345",
+        item_type="journalArticle",
+        title="T",
+        doi="",
+        isbn="",
+        year="",
+    )
+    base.update(kw)
+    return ItemRef(**base)
+
+
+def test_candidates_doi_unpaywall_first_then_libgen():
+    got = candidates(
+        _ref(doi="10.1056/NEJMoa1000678"),
+        direct=_direct(True),
+        proxied=_client(_libgen_handler([])),
+        email="dev@fhirworx.io",
+        up_pacer=_pacer(),
+        lg_pacer=_pacer(),
+    )
+    assert [g.source for g in got] == ["unpaywall"] * 3 + ["libgen"] * 2
+    assert got[0] == Candidate(
+        "unpaywall", None, "https://example.org/best.pdf", "pdf", None, "T"
+    )
+    assert got[3].md5 == "a78add1bd62047ead5945134488f470f"
+
+
+def test_candidates_isbn_book_and_title_fallback_and_nothing():
+    proxied = _client(_libgen_handler([]))
+    kw = dict(
+        direct=_direct(False),
+        proxied=proxied,
+        email="dev@fhirworx.io",
+        up_pacer=_pacer(),
+        lg_pacer=_pacer(),
+    )
+    book = candidates(_ref(item_type="book", isbn="9781640162280"), **kw)
+    assert [b.source for b in book] == ["libgen", "libgen"]
+    by_title = candidates(
+        _ref(item_type="book", title="HCPCS Level II Professional", year="2023"), **kw
+    )
+    assert [b.ext for b in by_title] == ["epub", "pdf"]
+    assert candidates(_ref(title="paper without doi"), **kw) == []
+    assert candidates(_ref(item_type="book", title=""), **kw) == []