feat(zot): fetch.deliver — libgen ads→get with throttle retry, Anna's fast download under a daily reserve (refs Z1)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

src/zot/fetch/deliver.py

@@ -0,0 +1,175 @@
+"""Candidate → file on disk.
+
+libgen: ``ads.php?md5=`` (browser UA + Referer; a 200 with an error div
+and no ``get.php`` link is a throttle → one retry after 3 s) then the
+keyed ``get.php``. Anna's: the member fast-download JSON API by md5,
+guarded by a daily reserve. Unpaywall URLs download directly.
+"""
+
+from __future__ import annotations
+
+import re
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from pathlib import Path
+
+import httpx
+
+from zot.fetch.resolve import LIBGEN, Candidate, Pacer
+
+ANNAS = "https://annas-archive.gl"
+MIN_BYTES = 10_240
+MAGIC = {"pdf": b"%PDF", "epub": b"PK"}
+_GET_LINK = re.compile(r"get\.php\?md5=([0-9a-f]{32})&(?:amp;)?key=([A-Za-z0-9]+)")
+_ADS_RETRY_SECONDS = 3.0
+
+
+class AnnasAuthError(RuntimeError):
+    """The Anna's key is wrong or the account is not a member — stop the run."""
+
+
+@dataclass
+class Budget:
+    annas_key: str = ""
+    reserve: int = 5
+    annas_left: int | None = None
+    annas_disabled: bool = False
+    quota_hit: bool = False
+
+    def can_use_annas(self) -> bool:
+        if not self.annas_key or self.annas_disabled:
+            return False
+        return self.annas_left is None or self.annas_left > self.reserve
+
+
+def download(
+    client: httpx.Client,
+    url: str,
+    dest: Path,
+    *,
+    ext: str,
+    headers: dict[str, str] | None = None,
+) -> Path | None:
+    """Stream *url* to *dest*; None unless it is a real file of type *ext*."""
+    try:
+        with client.stream(
+            "GET", url, headers=headers, timeout=120, follow_redirects=True
+        ) as r:
+            if r.status_code != 200:
+                return None
+            if "text/html" in r.headers.get("content-type", "").lower():
+                return None
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            with dest.open("wb") as fh:
+                for chunk in r.iter_bytes(1 << 16):
+                    fh.write(chunk)
+    except (httpx.HTTPError, OSError):
+        dest.unlink(missing_ok=True)
+        return None
+    size = dest.stat().st_size
+    with dest.open("rb") as fh:
+        head = fh.read(4)
+    if size < MIN_BYTES or not head.startswith(MAGIC[ext]):
+        dest.unlink(missing_ok=True)
+        return None
+    return dest
+
+
+def _ads_key(client: httpx.Client, md5: str, pacer: Pacer) -> str | None:
+    pacer.wait()
+    try:
+        r = client.get(
+            f"{LIBGEN}/ads.php",
+            params={"md5": md5},
+            headers={"Referer": f"{LIBGEN}/"},
+            timeout=30,
+        )
+    except httpx.HTTPError:
+        return None
+    if r.status_code != 200:
+        return None
+    m = _GET_LINK.search(r.text)
+    return m.group(2) if m and m.group(1) == md5 else None
+
+
+def fetch_libgen(
+    client: httpx.Client,
+    cand: Candidate,
+    dest: Path,
+    *,
+    pacer: Pacer,
+    sleep: Callable[[float], None],
+) -> Path | None:
+    md5 = cand.md5 or ""
+    key = _ads_key(client, md5, pacer)
+    if key is None:
+        sleep(_ADS_RETRY_SECONDS)
+        key = _ads_key(client, md5, pacer)
+    if key is None:
+        return None
+    return download(
+        client,
+        f"{LIBGEN}/get.php?md5={md5}&key={key}",
+        dest,
+        ext=cand.ext,
+        headers={"Referer": f"{LIBGEN}/ads.php?md5={md5}"},
+    )
+
+
+def fetch_annas(
+    client: httpx.Client, cand: Candidate, dest: Path, *, budget: Budget
+) -> Path | None:
+    try:
+        r = client.get(
+            f"{ANNAS}/dyn/api/fast_download.json",
+            params={"md5": cand.md5 or "", "key": budget.annas_key},
+            timeout=30,
+        )
+    except httpx.HTTPError:
+        return None
+    if r.status_code in (401, 403):
+        raise AnnasAuthError(
+            f"Anna's Archive refused the key ({r.status_code}): {r.text[:200]}"
+        )
+    if r.status_code not in (200, 204):
+        budget.annas_disabled = True
+        return None
+    try:
+        data = r.json()
+    except ValueError:
+        return None
+    info = data.get("account_fast_download_info") or {}
+    if isinstance(info.get("downloads_left"), int):
+        budget.annas_left = info["downloads_left"]
+    url = data.get("download_url")
+    if not url:
+        return None
+    return download(client, url, dest, ext=cand.ext)
+
+
+def fetch(
+    cand: Candidate,
+    dest: Path,
+    *,
+    direct: httpx.Client,
+    proxied: httpx.Client,
+    budget: Budget,
+    pacer: Pacer,
+    sleep: Callable[[float], None] = time.sleep,
+) -> str | None:
+    """Deliver *cand* to *dest*; returns the source that worked, else None.
+
+    Sets ``budget.quota_hit`` when Anna's would have been tried but the
+    reserve (or an earlier API failure) blocked it.
+    """
+    if cand.source == "unpaywall":
+        got = download(direct, cand.url or "", dest, ext=cand.ext)
+        return "unpaywall" if got else None
+    if fetch_libgen(proxied, cand, dest, pacer=pacer, sleep=sleep):
+        return "libgen"
+    if budget.can_use_annas():
+        return "annas" if fetch_annas(proxied, cand, dest, budget=budget) else None
+    if budget.annas_key:
+        budget.quota_hit = True
+    return None

tests/zot/fetch/fixtures/annas_fast_download.json

@@ -0,0 +1 @@
+{"download_url": "https://fast.example.org/dl/a78add1bd62047ead5945134488f470f.pdf", "account_fast_download_info": {"downloads_left": 42, "downloads_per_day": 50, "recently_downloaded_md5s": []}}

tests/zot/fetch/fixtures/libgen_ads.html

@@ -0,0 +1 @@
+<html><body><table><tr><td><a href="get.php?md5=a78add1bd62047ead5945134488f470f&key=AJPW7IO3P1LGRGWA">GET</a></td></tr></table></body></html>

tests/zot/fetch/test_deliver.py

@@ -0,0 +1,341 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import httpx
+import pytest
+
+from zot.fetch.deliver import (
+    MIN_BYTES,
+    AnnasAuthError,
+    Budget,
+    download,
+    fetch,
+    fetch_annas,
+    fetch_libgen,
+)
+from zot.fetch.resolve import LIBGEN, Candidate, Pacer
+
+FX = Path(__file__).parent / "fixtures"
+PDF = b"%PDF-1.4\n" + b"0" * MIN_BYTES
+EPUB = b"PK\x03\x04" + b"0" * MIN_BYTES
+MD5 = "a78add1bd62047ead5945134488f470f"
+ADS = (FX / "libgen_ads.html").read_text()
+ANNAS_OK = (FX / "annas_fast_download.json").read_text()
+PDF_HDR = {"content-type": "application/pdf"}
+
+
+def _pacer() -> Pacer:
+    return Pacer(3.0, clock=lambda: 0.0, sleep=lambda s: None)
+
+
+def _client(handler) -> httpx.Client:
+    return httpx.Client(transport=httpx.MockTransport(handler))
+
+
+def _lg(md5=MD5) -> Candidate:
+    return Candidate("libgen", md5, None, "pdf", None, "T")
+
+
+def _up() -> Candidate:
+    return Candidate("unpaywall", None, "https://example.org/x.pdf", "pdf", None, "T")
+
+
+def _nosleep(s: float) -> None:
+    return None
+
+
+# ── download ──────────────────────────────────────────────────────
+
+
+@pytest.mark.parametrize(
+    "status, ctype, body, ext, ok",
+    [
+        (200, "application/pdf", PDF, "pdf", True),
+        (200, "application/octet-stream", PDF, "pdf", True),
+        (200, "application/epub+zip", EPUB, "epub", True),
+        (200, "text/html; charset=utf-8", PDF, "pdf", False),
+        (200, "application/pdf", b"%PDF tiny", "pdf", False),
+        (200, "application/pdf", b"<html>" + b"0" * MIN_BYTES, "pdf", False),
+        (503, "application/pdf", PDF, "pdf", False),
+    ],
+)
+def test_download_validates_status_type_size_and_magic(
+    tmp_path, status, ctype, body, ext, ok
+):
+    c = _client(
+        lambda r: httpx.Response(status, content=body, headers={"content-type": ctype})
+    )
+    dest = tmp_path / f"f.{ext}"
+    got = download(c, "https://x/y", dest, ext=ext)
+    assert (got == dest) is ok
+    assert dest.exists() is ok
+
+
+def test_download_transport_error_is_none(tmp_path):
+    def boom(r):
+        raise httpx.ReadTimeout("t")
+
+    assert download(_client(boom), "https://x", tmp_path / "f.pdf", ext="pdf") is None
+
+
+def test_download_passes_headers(tmp_path):
+    seen = {}
+
+    def h(r):
+        seen.update(r.headers)
+        return httpx.Response(200, content=PDF, headers=PDF_HDR)
+
+    download(
+        _client(h),
+        "https://x",
+        tmp_path / "f.pdf",
+        ext="pdf",
+        headers={"Referer": "https://r/"},
+    )
+    assert seen["referer"] == "https://r/"
+
+
+# ── libgen ────────────────────────────────────────────────────────
+
+
+def _libgen_handler(ads_pages: list[httpx.Response], get_status=200):
+    calls: list[str] = []
+
+    def h(r: httpx.Request) -> httpx.Response:
+        calls.append(r.url.path)
+        if r.url.path == "/ads.php":
+            assert r.url.params["md5"] == MD5
+            assert r.headers["referer"] == f"{LIBGEN}/"
+            return ads_pages.pop(0)
+        if r.url.path == "/get.php":
+            assert r.url.params["key"] == "AJPW7IO3P1LGRGWA"
+            return httpx.Response(get_status, content=PDF, headers=PDF_HDR)
+        raise AssertionError(r.url)
+
+    return h, calls
+
+
+def test_fetch_libgen_ads_then_get(tmp_path):
+    h, calls = _libgen_handler([httpx.Response(200, text=ADS)])
+    got = fetch_libgen(
+        _client(h), _lg(), tmp_path / "f.pdf", pacer=_pacer(), sleep=_nosleep
+    )
+    assert got == tmp_path / "f.pdf" and calls == ["/ads.php", "/get.php"]
+
+
+def test_libgen_ads_error_div_retries_once_then_gives_up(tmp_path):
+    err = httpx.Response(
+        200,
+        text='<div class="alert alert-danger">Could not connect to the database 3306</div>',
+    )
+    h, calls = _libgen_handler([err, err])
+    slept: list[float] = []
+    got = fetch_libgen(
+        _client(h), _lg(), tmp_path / "f.pdf", pacer=_pacer(), sleep=slept.append
+    )
+    assert got is None and calls == ["/ads.php", "/ads.php"] and slept == [3.0]
+
+
+def test_libgen_ads_error_then_success_on_retry(tmp_path):
+    h, calls = _libgen_handler([httpx.Response(500), httpx.Response(200, text=ADS)])
+    got = fetch_libgen(
+        _client(h), _lg(), tmp_path / "f.pdf", pacer=_pacer(), sleep=_nosleep
+    )
+    assert got is not None and calls == ["/ads.php", "/ads.php", "/get.php"]
+
+
+def test_libgen_ads_link_for_other_md5_is_ignored(tmp_path):
+    other = httpx.Response(200, text=ADS.replace(MD5, "f" * 32))
+    h, calls = _libgen_handler([other, other])
+    got = fetch_libgen(
+        _client(h), _lg(), tmp_path / "f.pdf", pacer=_pacer(), sleep=_nosleep
+    )
+    assert got is None and calls == ["/ads.php", "/ads.php"]
+
+
+def test_libgen_get_503_is_none(tmp_path):
+    h, _ = _libgen_handler([httpx.Response(200, text=ADS)], get_status=503)
+    got = fetch_libgen(
+        _client(h), _lg(), tmp_path / "f.pdf", pacer=_pacer(), sleep=_nosleep
+    )
+    assert got is None
+
+
+def test_libgen_transport_error_is_none(tmp_path):
+    def boom(r):
+        raise httpx.ConnectError("x")
+
+    got = fetch_libgen(
+        _client(boom), _lg(), tmp_path / "f.pdf", pacer=_pacer(), sleep=_nosleep
+    )
+    assert got is None
+
+
+# ── Anna's ────────────────────────────────────────────────────────
+
+
+def _annas_handler(api: httpx.Response, file_status=200):
+    def h(r: httpx.Request) -> httpx.Response:
+        if r.url.host == "annas-archive.gl":
+            assert r.url.path == "/dyn/api/fast_download.json"
+            assert r.url.params["md5"] == MD5 and r.url.params["key"] == "SECRET"
+            return api
+        return httpx.Response(file_status, content=PDF, headers=PDF_HDR)
+
+    return h
+
+
+def test_fetch_annas_downloads_and_updates_budget(tmp_path):
+    b = Budget(annas_key="SECRET")
+    c = _client(_annas_handler(httpx.Response(200, text=ANNAS_OK)))
+    got = fetch_annas(c, _lg(), tmp_path / "f.pdf", budget=b)
+    assert got == tmp_path / "f.pdf" and b.annas_left == 42
+
+
+@pytest.mark.parametrize("status", [401, 403])
+def test_fetch_annas_auth_errors_stop_the_run(tmp_path, status):
+    b = Budget(annas_key="SECRET")
+    c = _client(_annas_handler(httpx.Response(status, json={"error": "Not a member"})))
+    with pytest.raises(AnnasAuthError):
+        fetch_annas(c, _lg(), tmp_path / "f.pdf", budget=b)
+
+
+def test_fetch_annas_other_error_disables_annas(tmp_path):
+    b = Budget(annas_key="SECRET")
+    c = _client(_annas_handler(httpx.Response(500)))
+    assert fetch_annas(c, _lg(), tmp_path / "f.pdf", budget=b) is None
+    assert b.annas_disabled is True and b.can_use_annas() is False
+
+
+def test_fetch_annas_null_download_url_is_none(tmp_path):
+    b = Budget(annas_key="SECRET")
+    api = httpx.Response(200, json={"download_url": None, "error": "Invalid md5"})
+    assert (
+        fetch_annas(_client(_annas_handler(api)), _lg(), tmp_path / "f.pdf", budget=b)
+        is None
+    )
+
+
+def test_fetch_annas_bad_json_or_transport_is_none(tmp_path):
+    b = Budget(annas_key="SECRET")
+    c = _client(_annas_handler(httpx.Response(200, text="nope")))
+    assert fetch_annas(c, _lg(), tmp_path / "f.pdf", budget=b) is None
+
+    def boom(r):
+        raise httpx.ConnectError("x")
+
+    assert fetch_annas(_client(boom), _lg(), tmp_path / "f.pdf", budget=b) is None
+
+
+def test_fetch_annas_file_failure_still_reports_budget(tmp_path):
+    b = Budget(annas_key="SECRET")
+    c = _client(_annas_handler(httpx.Response(200, text=ANNAS_OK), file_status=503))
+    got = fetch_annas(c, _lg(), tmp_path / "f.pdf", budget=b)
+    assert got is None and b.annas_left == 42  # the API call itself consumed one
+
+
+def test_budget_reserve_blocks_annas():
+    assert Budget(annas_key="").can_use_annas() is False
+    assert Budget(annas_key="k").can_use_annas() is True  # unknown budget → allowed
+    assert Budget(annas_key="k", annas_left=6, reserve=5).can_use_annas() is True
+    assert Budget(annas_key="k", annas_left=5, reserve=5).can_use_annas() is False
+
+
+# ── fetch (the cascade) ───────────────────────────────────────────
+
+
+def test_fetch_unpaywall_uses_direct_client(tmp_path):
+    direct = _client(lambda r: httpx.Response(200, content=PDF, headers=PDF_HDR))
+
+    def never(r):
+        raise AssertionError("proxied client used for Unpaywall")
+
+    src = fetch(
+        _up(),
+        tmp_path / "f.pdf",
+        direct=direct,
+        proxied=_client(never),
+        budget=Budget(),
+        pacer=_pacer(),
+    )
+    assert src == "unpaywall"
+
+
+def test_fetch_unpaywall_failure_is_none(tmp_path):
+    direct = _client(lambda r: httpx.Response(404))
+    src = fetch(
+        _up(),
+        tmp_path / "f.pdf",
+        direct=direct,
+        proxied=direct,
+        budget=Budget(),
+        pacer=_pacer(),
+    )
+    assert src is None
+
+
+def test_fetch_libgen_then_annas_on_failure(tmp_path):
+    def h(r: httpx.Request) -> httpx.Response:
+        if r.url.path == "/ads.php":
+            return httpx.Response(500)
+        if r.url.host == "annas-archive.gl":
+            return httpx.Response(200, text=ANNAS_OK)
+        return httpx.Response(200, content=PDF, headers=PDF_HDR)
+
+    b = Budget(annas_key="SECRET")
+    src = fetch(
+        _lg(),
+        tmp_path / "f.pdf",
+        direct=_client(h),
+        proxied=_client(h),
+        budget=b,
+        pacer=_pacer(),
+        sleep=_nosleep,
+    )
+    assert src == "annas" and b.annas_left == 42
+
+
+def test_fetch_libgen_success_skips_annas(tmp_path):
+    h, calls = _libgen_handler([httpx.Response(200, text=ADS)])
+    b = Budget(annas_key="SECRET")
+    src = fetch(
+        _lg(),
+        tmp_path / "f.pdf",
+        direct=_client(h),
+        proxied=_client(h),
+        budget=b,
+        pacer=_pacer(),
+        sleep=_nosleep,
+    )
+    assert src == "libgen" and calls == ["/ads.php", "/get.php"]
+
+
+def test_fetch_marks_quota_when_annas_blocked_by_reserve(tmp_path):
+    proxied = _client(lambda r: httpx.Response(500))
+    b = Budget(annas_key="SECRET", annas_left=2, reserve=5)
+    src = fetch(
+        _lg(),
+        tmp_path / "f.pdf",
+        direct=proxied,
+        proxied=proxied,
+        budget=b,
+        pacer=_pacer(),
+        sleep=_nosleep,
+    )
+    assert src is None and b.quota_hit is True
+
+
+def test_fetch_all_fail_without_annas_is_not_quota(tmp_path):
+    proxied = _client(lambda r: httpx.Response(500))
+    b = Budget()
+    src = fetch(
+        _lg(),
+        tmp_path / "f.pdf",
+        direct=proxied,
+        proxied=proxied,
+        budget=b,
+        pacer=_pacer(),
+        sleep=_nosleep,
+    )
+    assert src is None and b.quota_hit is False