From f69a3c34309e9f9aaf167f274b6917f98fb3e10c Mon Sep 17 00:00:00 2001 From: kert Date: Wed, 23 Sep 2026 21:07:59 -0400 Subject: [PATCH] =?UTF-8?q?feat(zot):=20fetch.deliver=20=E2=80=94=20libgen?= =?UTF-8?q?=20ads=E2=86=92get=20with=20throttle=20retry,=20Anna's=20fast?= =?UTF-8?q?=20download=20under=20a=20daily=20reserve=20(refs=20Z1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 --- src/zot/fetch/deliver.py | 175 +++++++++ .../fetch/fixtures/annas_fast_download.json | 1 + tests/zot/fetch/fixtures/libgen_ads.html | 1 + tests/zot/fetch/test_deliver.py | 341 ++++++++++++++++++ 4 files changed, 518 insertions(+) create mode 100644 src/zot/fetch/deliver.py create mode 100644 tests/zot/fetch/fixtures/annas_fast_download.json create mode 100644 tests/zot/fetch/fixtures/libgen_ads.html create mode 100644 tests/zot/fetch/test_deliver.py diff --git a/src/zot/fetch/deliver.py b/src/zot/fetch/deliver.py new file mode 100644 index 0000000..c6b31ba --- /dev/null +++ b/src/zot/fetch/deliver.py @@ -0,0 +1,175 @@ +"""Candidate → file on disk. + +libgen: ``ads.php?md5=`` (browser UA + Referer; a 200 with an error div +and no ``get.php`` link is a throttle → one retry after 3 s) then the +keyed ``get.php``. Anna's: the member fast-download JSON API by md5, +guarded by a daily reserve. Unpaywall URLs download directly. +""" + +from __future__ import annotations + +import re +import time +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +import httpx + +from zot.fetch.resolve import LIBGEN, Candidate, Pacer + +ANNAS = "https://annas-archive.gl" +MIN_BYTES = 10_240 +MAGIC = {"pdf": b"%PDF", "epub": b"PK"} +_GET_LINK = re.compile(r"get\.php\?md5=([0-9a-f]{32})&(?:amp;)?key=([A-Za-z0-9]+)") +_ADS_RETRY_SECONDS = 3.0 + + +class AnnasAuthError(RuntimeError): + """The Anna's key is wrong or the account is not a member — stop the run.""" + + +@dataclass +class Budget: + annas_key: str = "" + reserve: int = 5 + annas_left: int | None = None + annas_disabled: bool = False + quota_hit: bool = False + + def can_use_annas(self) -> bool: + if not self.annas_key or self.annas_disabled: + return False + return self.annas_left is None or self.annas_left > self.reserve + + +def download( + client: httpx.Client, + url: str, + dest: Path, + *, + ext: str, + headers: dict[str, str] | None = None, +) -> Path | None: + """Stream *url* to *dest*; None unless it is a real file of type *ext*.""" + try: + with client.stream( + "GET", url, headers=headers, timeout=120, follow_redirects=True + ) as r: + if r.status_code != 200: + return None + if "text/html" in r.headers.get("content-type", "").lower(): + return None + dest.parent.mkdir(parents=True, exist_ok=True) + with dest.open("wb") as fh: + for chunk in r.iter_bytes(1 << 16): + fh.write(chunk) + except (httpx.HTTPError, OSError): + dest.unlink(missing_ok=True) + return None + size = dest.stat().st_size + with dest.open("rb") as fh: + head = fh.read(4) + if size < MIN_BYTES or not head.startswith(MAGIC[ext]): + dest.unlink(missing_ok=True) + return None + return dest + + +def _ads_key(client: httpx.Client, md5: str, pacer: Pacer) -> str | None: + pacer.wait() + try: + r = client.get( + f"{LIBGEN}/ads.php", + params={"md5": md5}, + headers={"Referer": f"{LIBGEN}/"}, + timeout=30, + ) + except httpx.HTTPError: + return None + if r.status_code != 200: + return None + m = _GET_LINK.search(r.text) + return m.group(2) if m and m.group(1) == md5 else None + + +def fetch_libgen( + client: httpx.Client, + cand: Candidate, + dest: Path, + *, + pacer: Pacer, + sleep: Callable[[float], None], +) -> Path | None: + md5 = cand.md5 or "" + key = _ads_key(client, md5, pacer) + if key is None: + sleep(_ADS_RETRY_SECONDS) + key = _ads_key(client, md5, pacer) + if key is None: + return None + return download( + client, + f"{LIBGEN}/get.php?md5={md5}&key={key}", + dest, + ext=cand.ext, + headers={"Referer": f"{LIBGEN}/ads.php?md5={md5}"}, + ) + + +def fetch_annas( + client: httpx.Client, cand: Candidate, dest: Path, *, budget: Budget +) -> Path | None: + try: + r = client.get( + f"{ANNAS}/dyn/api/fast_download.json", + params={"md5": cand.md5 or "", "key": budget.annas_key}, + timeout=30, + ) + except httpx.HTTPError: + return None + if r.status_code in (401, 403): + raise AnnasAuthError( + f"Anna's Archive refused the key ({r.status_code}): {r.text[:200]}" + ) + if r.status_code not in (200, 204): + budget.annas_disabled = True + return None + try: + data = r.json() + except ValueError: + return None + info = data.get("account_fast_download_info") or {} + if isinstance(info.get("downloads_left"), int): + budget.annas_left = info["downloads_left"] + url = data.get("download_url") + if not url: + return None + return download(client, url, dest, ext=cand.ext) + + +def fetch( + cand: Candidate, + dest: Path, + *, + direct: httpx.Client, + proxied: httpx.Client, + budget: Budget, + pacer: Pacer, + sleep: Callable[[float], None] = time.sleep, +) -> str | None: + """Deliver *cand* to *dest*; returns the source that worked, else None. + + Sets ``budget.quota_hit`` when Anna's would have been tried but the + reserve (or an earlier API failure) blocked it. + """ + if cand.source == "unpaywall": + got = download(direct, cand.url or "", dest, ext=cand.ext) + return "unpaywall" if got else None + if fetch_libgen(proxied, cand, dest, pacer=pacer, sleep=sleep): + return "libgen" + if budget.can_use_annas(): + return "annas" if fetch_annas(proxied, cand, dest, budget=budget) else None + if budget.annas_key: + budget.quota_hit = True + return None diff --git a/tests/zot/fetch/fixtures/annas_fast_download.json b/tests/zot/fetch/fixtures/annas_fast_download.json new file mode 100644 index 0000000..a619a4d --- /dev/null +++ b/tests/zot/fetch/fixtures/annas_fast_download.json @@ -0,0 +1 @@ +{"download_url": "https://fast.example.org/dl/a78add1bd62047ead5945134488f470f.pdf", "account_fast_download_info": {"downloads_left": 42, "downloads_per_day": 50, "recently_downloaded_md5s": []}} diff --git a/tests/zot/fetch/fixtures/libgen_ads.html b/tests/zot/fetch/fixtures/libgen_ads.html new file mode 100644 index 0000000..d76b9dc --- /dev/null +++ b/tests/zot/fetch/fixtures/libgen_ads.html @@ -0,0 +1 @@ +
GET
diff --git a/tests/zot/fetch/test_deliver.py b/tests/zot/fetch/test_deliver.py new file mode 100644 index 0000000..2491895 --- /dev/null +++ b/tests/zot/fetch/test_deliver.py @@ -0,0 +1,341 @@ +from __future__ import annotations + +from pathlib import Path + +import httpx +import pytest + +from zot.fetch.deliver import ( + MIN_BYTES, + AnnasAuthError, + Budget, + download, + fetch, + fetch_annas, + fetch_libgen, +) +from zot.fetch.resolve import LIBGEN, Candidate, Pacer + +FX = Path(__file__).parent / "fixtures" +PDF = b"%PDF-1.4\n" + b"0" * MIN_BYTES +EPUB = b"PK\x03\x04" + b"0" * MIN_BYTES +MD5 = "a78add1bd62047ead5945134488f470f" +ADS = (FX / "libgen_ads.html").read_text() +ANNAS_OK = (FX / "annas_fast_download.json").read_text() +PDF_HDR = {"content-type": "application/pdf"} + + +def _pacer() -> Pacer: + return Pacer(3.0, clock=lambda: 0.0, sleep=lambda s: None) + + +def _client(handler) -> httpx.Client: + return httpx.Client(transport=httpx.MockTransport(handler)) + + +def _lg(md5=MD5) -> Candidate: + return Candidate("libgen", md5, None, "pdf", None, "T") + + +def _up() -> Candidate: + return Candidate("unpaywall", None, "https://example.org/x.pdf", "pdf", None, "T") + + +def _nosleep(s: float) -> None: + return None + + +# ── download ────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + "status, ctype, body, ext, ok", + [ + (200, "application/pdf", PDF, "pdf", True), + (200, "application/octet-stream", PDF, "pdf", True), + (200, "application/epub+zip", EPUB, "epub", True), + (200, "text/html; charset=utf-8", PDF, "pdf", False), + (200, "application/pdf", b"%PDF tiny", "pdf", False), + (200, "application/pdf", b"" + b"0" * MIN_BYTES, "pdf", False), + (503, "application/pdf", PDF, "pdf", False), + ], +) +def test_download_validates_status_type_size_and_magic( + tmp_path, status, ctype, body, ext, ok +): + c = _client( + lambda r: httpx.Response(status, content=body, headers={"content-type": ctype}) + ) + dest = tmp_path / f"f.{ext}" + got = download(c, "https://x/y", dest, ext=ext) + assert (got == dest) is ok + assert dest.exists() is ok + + +def test_download_transport_error_is_none(tmp_path): + def boom(r): + raise httpx.ReadTimeout("t") + + assert download(_client(boom), "https://x", tmp_path / "f.pdf", ext="pdf") is None + + +def test_download_passes_headers(tmp_path): + seen = {} + + def h(r): + seen.update(r.headers) + return httpx.Response(200, content=PDF, headers=PDF_HDR) + + download( + _client(h), + "https://x", + tmp_path / "f.pdf", + ext="pdf", + headers={"Referer": "https://r/"}, + ) + assert seen["referer"] == "https://r/" + + +# ── libgen ──────────────────────────────────────────────────────── + + +def _libgen_handler(ads_pages: list[httpx.Response], get_status=200): + calls: list[str] = [] + + def h(r: httpx.Request) -> httpx.Response: + calls.append(r.url.path) + if r.url.path == "/ads.php": + assert r.url.params["md5"] == MD5 + assert r.headers["referer"] == f"{LIBGEN}/" + return ads_pages.pop(0) + if r.url.path == "/get.php": + assert r.url.params["key"] == "AJPW7IO3P1LGRGWA" + return httpx.Response(get_status, content=PDF, headers=PDF_HDR) + raise AssertionError(r.url) + + return h, calls + + +def test_fetch_libgen_ads_then_get(tmp_path): + h, calls = _libgen_handler([httpx.Response(200, text=ADS)]) + got = fetch_libgen( + _client(h), _lg(), tmp_path / "f.pdf", pacer=_pacer(), sleep=_nosleep + ) + assert got == tmp_path / "f.pdf" and calls == ["/ads.php", "/get.php"] + + +def test_libgen_ads_error_div_retries_once_then_gives_up(tmp_path): + err = httpx.Response( + 200, + text='
Could not connect to the database 3306
', + ) + h, calls = _libgen_handler([err, err]) + slept: list[float] = [] + got = fetch_libgen( + _client(h), _lg(), tmp_path / "f.pdf", pacer=_pacer(), sleep=slept.append + ) + assert got is None and calls == ["/ads.php", "/ads.php"] and slept == [3.0] + + +def test_libgen_ads_error_then_success_on_retry(tmp_path): + h, calls = _libgen_handler([httpx.Response(500), httpx.Response(200, text=ADS)]) + got = fetch_libgen( + _client(h), _lg(), tmp_path / "f.pdf", pacer=_pacer(), sleep=_nosleep + ) + assert got is not None and calls == ["/ads.php", "/ads.php", "/get.php"] + + +def test_libgen_ads_link_for_other_md5_is_ignored(tmp_path): + other = httpx.Response(200, text=ADS.replace(MD5, "f" * 32)) + h, calls = _libgen_handler([other, other]) + got = fetch_libgen( + _client(h), _lg(), tmp_path / "f.pdf", pacer=_pacer(), sleep=_nosleep + ) + assert got is None and calls == ["/ads.php", "/ads.php"] + + +def test_libgen_get_503_is_none(tmp_path): + h, _ = _libgen_handler([httpx.Response(200, text=ADS)], get_status=503) + got = fetch_libgen( + _client(h), _lg(), tmp_path / "f.pdf", pacer=_pacer(), sleep=_nosleep + ) + assert got is None + + +def test_libgen_transport_error_is_none(tmp_path): + def boom(r): + raise httpx.ConnectError("x") + + got = fetch_libgen( + _client(boom), _lg(), tmp_path / "f.pdf", pacer=_pacer(), sleep=_nosleep + ) + assert got is None + + +# ── Anna's ──────────────────────────────────────────────────────── + + +def _annas_handler(api: httpx.Response, file_status=200): + def h(r: httpx.Request) -> httpx.Response: + if r.url.host == "annas-archive.gl": + assert r.url.path == "/dyn/api/fast_download.json" + assert r.url.params["md5"] == MD5 and r.url.params["key"] == "SECRET" + return api + return httpx.Response(file_status, content=PDF, headers=PDF_HDR) + + return h + + +def test_fetch_annas_downloads_and_updates_budget(tmp_path): + b = Budget(annas_key="SECRET") + c = _client(_annas_handler(httpx.Response(200, text=ANNAS_OK))) + got = fetch_annas(c, _lg(), tmp_path / "f.pdf", budget=b) + assert got == tmp_path / "f.pdf" and b.annas_left == 42 + + +@pytest.mark.parametrize("status", [401, 403]) +def test_fetch_annas_auth_errors_stop_the_run(tmp_path, status): + b = Budget(annas_key="SECRET") + c = _client(_annas_handler(httpx.Response(status, json={"error": "Not a member"}))) + with pytest.raises(AnnasAuthError): + fetch_annas(c, _lg(), tmp_path / "f.pdf", budget=b) + + +def test_fetch_annas_other_error_disables_annas(tmp_path): + b = Budget(annas_key="SECRET") + c = _client(_annas_handler(httpx.Response(500))) + assert fetch_annas(c, _lg(), tmp_path / "f.pdf", budget=b) is None + assert b.annas_disabled is True and b.can_use_annas() is False + + +def test_fetch_annas_null_download_url_is_none(tmp_path): + b = Budget(annas_key="SECRET") + api = httpx.Response(200, json={"download_url": None, "error": "Invalid md5"}) + assert ( + fetch_annas(_client(_annas_handler(api)), _lg(), tmp_path / "f.pdf", budget=b) + is None + ) + + +def test_fetch_annas_bad_json_or_transport_is_none(tmp_path): + b = Budget(annas_key="SECRET") + c = _client(_annas_handler(httpx.Response(200, text="nope"))) + assert fetch_annas(c, _lg(), tmp_path / "f.pdf", budget=b) is None + + def boom(r): + raise httpx.ConnectError("x") + + assert fetch_annas(_client(boom), _lg(), tmp_path / "f.pdf", budget=b) is None + + +def test_fetch_annas_file_failure_still_reports_budget(tmp_path): + b = Budget(annas_key="SECRET") + c = _client(_annas_handler(httpx.Response(200, text=ANNAS_OK), file_status=503)) + got = fetch_annas(c, _lg(), tmp_path / "f.pdf", budget=b) + assert got is None and b.annas_left == 42 # the API call itself consumed one + + +def test_budget_reserve_blocks_annas(): + assert Budget(annas_key="").can_use_annas() is False + assert Budget(annas_key="k").can_use_annas() is True # unknown budget → allowed + assert Budget(annas_key="k", annas_left=6, reserve=5).can_use_annas() is True + assert Budget(annas_key="k", annas_left=5, reserve=5).can_use_annas() is False + + +# ── fetch (the cascade) ─────────────────────────────────────────── + + +def test_fetch_unpaywall_uses_direct_client(tmp_path): + direct = _client(lambda r: httpx.Response(200, content=PDF, headers=PDF_HDR)) + + def never(r): + raise AssertionError("proxied client used for Unpaywall") + + src = fetch( + _up(), + tmp_path / "f.pdf", + direct=direct, + proxied=_client(never), + budget=Budget(), + pacer=_pacer(), + ) + assert src == "unpaywall" + + +def test_fetch_unpaywall_failure_is_none(tmp_path): + direct = _client(lambda r: httpx.Response(404)) + src = fetch( + _up(), + tmp_path / "f.pdf", + direct=direct, + proxied=direct, + budget=Budget(), + pacer=_pacer(), + ) + assert src is None + + +def test_fetch_libgen_then_annas_on_failure(tmp_path): + def h(r: httpx.Request) -> httpx.Response: + if r.url.path == "/ads.php": + return httpx.Response(500) + if r.url.host == "annas-archive.gl": + return httpx.Response(200, text=ANNAS_OK) + return httpx.Response(200, content=PDF, headers=PDF_HDR) + + b = Budget(annas_key="SECRET") + src = fetch( + _lg(), + tmp_path / "f.pdf", + direct=_client(h), + proxied=_client(h), + budget=b, + pacer=_pacer(), + sleep=_nosleep, + ) + assert src == "annas" and b.annas_left == 42 + + +def test_fetch_libgen_success_skips_annas(tmp_path): + h, calls = _libgen_handler([httpx.Response(200, text=ADS)]) + b = Budget(annas_key="SECRET") + src = fetch( + _lg(), + tmp_path / "f.pdf", + direct=_client(h), + proxied=_client(h), + budget=b, + pacer=_pacer(), + sleep=_nosleep, + ) + assert src == "libgen" and calls == ["/ads.php", "/get.php"] + + +def test_fetch_marks_quota_when_annas_blocked_by_reserve(tmp_path): + proxied = _client(lambda r: httpx.Response(500)) + b = Budget(annas_key="SECRET", annas_left=2, reserve=5) + src = fetch( + _lg(), + tmp_path / "f.pdf", + direct=proxied, + proxied=proxied, + budget=b, + pacer=_pacer(), + sleep=_nosleep, + ) + assert src is None and b.quota_hit is True + + +def test_fetch_all_fail_without_annas_is_not_quota(tmp_path): + proxied = _client(lambda r: httpx.Response(500)) + b = Budget() + src = fetch( + _lg(), + tmp_path / "f.pdf", + direct=proxied, + proxied=proxied, + budget=b, + pacer=_pacer(), + sleep=_nosleep, + ) + assert src is None and b.quota_hit is False