stack/src/prisma/fetch.py

"""PDF retrieval for PRISMA screening — Unpaywall → Semantic Scholar → PMC
→ fallback proxy.

Call graph::

    pending_queue(db, project, client=...)
        → list of (zot_id, doi, pmid, pmcid) needing a PDF; with a client,
          PMIDs lacking a PMCID are resolved through NCBI idconv (#723)
    fetch_one(item, *, proxy=None)
        → tries each source in order, returns (source, path_on_disk) | None
          (every Unpaywall oa_location, then Semantic Scholar's
          openAccessPdf — which also lends a PMCID to the PMC tiers)
    attach_pdf(db, zot_parent_id, path)
        → creates a Zotero attachment row and copies the file into
          the Zotero storage directory

The last tier respects a SOCKS5/HTTPS proxy via ``PRISMA_FETCH_PROXY``
env var; set it to the tunnel endpoint after ``stack prisma vpn up``.
Unpaywall and PMC are legal/open and hit direct — no proxy needed.

Queue policy: pull PDFs only for items tagged ``screen:include`` or
``screen:uncertain``. Stage-2 excludes are terminal; we never fetch
for them. The filter lives in :func:`pending_queue`.
"""

from __future__ import annotations

import base64
import hashlib
import json as _json
import logging
import re
import shutil
import threading
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING

import httpx

from zot.db import FIELD_MAP, TYPE_MAP, generate_key, now_iso

if TYPE_CHECKING:
    from zot.db import Db

log = logging.getLogger(__name__)


USER_AGENT = "stack-prisma-fetch/1.0 (mailto:{})"
_FALLBACK_MIRRORS = (
    "https://sci-hub.ru",
    "https://sci-hub.st",
    "https://sci-hub.cat",
    "https://sci-hub.box",
)

# Projects that must not use the mirror tier. Empty: the five LEAD
# reviews leave the fallback on, and any other project can be named here.
SCIHUB_OFF: frozenset[str] = frozenset()


def fallback_allowed(project: str) -> bool:
    """True when this project's fetch may use the mirror tier."""
    return project not in SCIHUB_OFF


# The fallback tier shares one proxied client (altcha solves mutate its
# cookie jar) and one VPN exit — hold this across the whole tier.
_fallback_lock = threading.Lock()


@dataclass
class PendingItem:
    """One queue row needing a PDF."""

    zot_id: int
    doi: str
    pmid: str
    pmcid: str
    title: str


# ── Queue ────────────────────────────────────────────────────────


def pending_queue(
    db: Db,
    project: str,
    limit: int | None = None,
    *,
    client: httpx.Client | None = None,
    email: str = "",
) -> list[PendingItem]:
    """Items in *project* that passed or deferred stage-2 screening and
    don't yet have a PDF attached.

    Definition:
      - tagged ``project:<name>``
      - tagged ``screen:include`` OR ``screen:uncertain``
      - NOT tagged ``module:prisma`` (anchor guard)
      - no child attachment with an existing PDF path

    With *client*, items that carry a PMID but no PMCID are resolved in
    one NCBI idconv batch and the answer is written back to ``extra`` as
    a ``PMCID:`` line, so the PMC tiers get their chance and the next
    run asks no question twice (#723). Without a client the queue is
    built from the stored fields only — no network.
    """
    rows = db.con.execute(
        """
        SELECT DISTINCT i.itemID
        FROM items i
        JOIN itemTags itp ON i.itemID = itp.itemID
        JOIN tags tp ON itp.tagID = tp.tagID AND tp.name = ?
        JOIN itemTags its ON i.itemID = its.itemID
        JOIN tags ts ON its.tagID = ts.tagID
            AND ts.name IN ('screen:include', 'screen:uncertain')
        WHERE i.itemID NOT IN (
            SELECT itx.itemID FROM itemTags itx
            JOIN tags tx ON itx.tagID = tx.tagID
            WHERE tx.name = 'module:prisma'
        )
          AND NOT EXISTS (
              SELECT 1
              FROM itemAttachments ia
              JOIN items att ON ia.itemID = att.itemID
              WHERE ia.parentItemID = i.itemID
                AND ia.path IS NOT NULL
                AND ia.path LIKE 'storage:%.pdf'
          )
        ORDER BY i.itemID
        """
        + (f" LIMIT {int(limit)}" if limit else ""),
        (f"project:{project}",),
    ).fetchall()

    out: list[PendingItem] = []
    for (zot_id,) in rows:
        out.append(
            PendingItem(
                zot_id=zot_id,
                doi=_field(db, zot_id, "DOI"),
                pmid=_extra_value(db, zot_id, "PMID"),
                pmcid=_extra_value(db, zot_id, "PMCID") or _from_url_pmcid(db, zot_id),
                title=_field(db, zot_id, "title"),
            )
        )
    if client is not None:
        _resolve_queue_pmcids(db, out, client, email)
    return out


def _resolve_queue_pmcids(
    db: Db, items: list[PendingItem], client: httpx.Client, email: str
) -> None:
    missing = [it for it in items if it.pmid and not it.pmcid]
    if not missing:
        return
    found = resolve_pmcids(client, [it.pmid for it in missing], email)
    if not found:
        return
    for it in missing:
        pmcid = found.get(it.pmid.strip())
        if not pmcid:
            continue
        it.pmcid = pmcid
        extra = _field(db, it.zot_id, "extra")
        db.set_fields(
            it.zot_id,
            {"extra": f"{extra}\nPMCID: {pmcid}" if extra else f"PMCID: {pmcid}"},
        )
    db.commit()


_IDCONV = "https://pmc.ncbi.nlm.nih.gov/tools/idconv/api/v1/articles/"
_IDCONV_BATCH = 200


def resolve_pmcids(
    client: httpx.Client, pmids: list[str], email: str
) -> dict[str, str]:
    """PMID → PMCID via NCBI's idconv service, 200 ids per call (#723).

    Only records idconv actually found are returned; ``status: error``
    rows ("Identifier not found in PMC") and transport failures drop out.
    """
    ids = [p for p in dict.fromkeys((p or "").strip() for p in pmids) if p.isdigit()]
    out: dict[str, str] = {}
    for i in range(0, len(ids), _IDCONV_BATCH):
        chunk = ids[i : i + _IDCONV_BATCH]
        try:
            r = client.get(
                _IDCONV,
                params={
                    "ids": ",".join(chunk),
                    "idtype": "pmid",
                    "format": "json",
                    "tool": "stack-prisma",
                    "email": email,
                },
                timeout=30,
            )
            if r.status_code != 200:
                continue
            records = r.json().get("records") or []
        except (httpx.HTTPError, ValueError):
            continue
        for rec in records:
            pmcid = rec.get("pmcid")
            if not pmcid or rec.get("status") == "error":
                continue
            out[str(rec.get("requested-id") or rec.get("pmid"))] = pmcid
    return out


# ── Source fetchers ─────────────────────────────────────────────


def unpaywall_urls(client: httpx.Client, doi: str, email: str) -> list[str]:
    """Every OA URL Unpaywall knows for *doi*, best location first.

    ``best_oa_location`` is a ranking, not a guarantee — on the #650
    pass it pointed at dead publisher links while a later
    ``oa_locations`` entry served the PDF (#723). Direct PDF URLs come
    before landing pages across all locations; duplicates dropped.
    """
    if not doi:
        return []
    try:
        r = client.get(
            f"https://api.unpaywall.org/v2/{doi}",
            params={"email": email},
            timeout=15,
        )
        if r.status_code != 200:
            return []
        data = r.json()
    except (httpx.HTTPError, ValueError):
        return []
    locs = [data.get("best_oa_location") or {}, *(data.get("oa_locations") or [])]
    out: list[str] = []
    for key in ("url_for_pdf", "url"):
        for loc in locs:
            u = loc.get(key)
            if u and u not in out:
                out.append(u)
    return out


def fetch_unpaywall(client: httpx.Client, doi: str, email: str) -> str | None:
    """Unpaywall → best PDF URL (or None). See :func:`unpaywall_urls`."""
    urls = unpaywall_urls(client, doi, email)
    return urls[0] if urls else None


_S2_BASE = "https://api.semanticscholar.org/graph/v1/paper"
_S2_MIN_INTERVAL = 1.0  # unauthenticated graph API: ~1 req/s shared pool
_S2_LAST = [0.0]
_s2_lock = threading.Lock()


@dataclass
class S2Hit:
    """What Semantic Scholar knows that helps the cascade."""

    pdf_url: str
    pmcid: str


def fetch_semantic_scholar(client: httpx.Client, doi: str) -> S2Hit | None:
    """Semantic Scholar graph API → ``openAccessPdf`` URL and/or a PMCID.

    Found 4 of 30 items the rest of the cascade missed on the #650 pass
    (#723). Its ``externalIds.PubMedCentral`` also fills in a PMCID the
    Zotero record lacks, so the PMC tiers run for items Unpaywall only
    knows by landing page. Rate-limited to ``_S2_MIN_INTERVAL`` across
    threads; no API key needed.
    """
    if not doi:
        return None
    with _s2_lock:
        wait = _S2_MIN_INTERVAL - (time.monotonic() - _S2_LAST[0])
        if wait > 0:
            time.sleep(wait)
        try:
            r = client.get(
                f"{_S2_BASE}/DOI:{doi}",
                params={"fields": "openAccessPdf,externalIds"},
                timeout=15,
            )
        except httpx.HTTPError:
            return None
        finally:
            _S2_LAST[0] = time.monotonic()
    if r.status_code != 200:
        return None
    try:
        data = r.json()
    except ValueError:
        return None
    pdf = (data.get("openAccessPdf") or {}).get("url") or ""
    pmc = str((data.get("externalIds") or {}).get("PubMedCentral") or "")
    if pmc and not pmc.startswith("PMC"):
        pmc = f"PMC{pmc}"
    if not pdf and not pmc:
        return None
    return S2Hit(pdf_url=pdf, pmcid=pmc)


def fetch_pmc(client: httpx.Client, pmcid: str) -> str | None:
    """PMC PDF URL via the OA Web Service.

    The old ``/pmc/articles/<id>/pdf/`` guess URL now serves an HTML
    bot-challenge to scripted clients (200, text/html — #657), so ask
    the sanctioned OA service instead. Only articles in the OA subset
    resolve; free-to-read author manuscripts return
    ``idIsNotOpenAccess`` and yield None. FTP hrefs are rewritten to
    the HTTPS mirror.
    """
    if not pmcid:
        return None
    pmc = pmcid if pmcid.startswith("PMC") else f"PMC{pmcid}"
    try:
        r = client.get(
            "https://www.ncbi.nlm.nih.gov/pmc/utils/oa/oa.fcgi",
            params={"id": pmc, "format": "pdf"},
            follow_redirects=True,
            timeout=15,
        )
        r.raise_for_status()
    except httpx.HTTPError:
        return None
    m = re.search(r'<link[^>]*href="([^"]+\.pdf)"', r.text)
    if not m:
        return None
    # NCBI moved /pub/pmc/ content under deprecated/ — the plain
    # rewrite 404s on every article (#657).
    return m.group(1).replace(
        "ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/",
        "https://ftp.ncbi.nlm.nih.gov/pub/pmc/deprecated/",
    )


_S3_BASE = "https://pmc-oa-opendata.s3.amazonaws.com"


def fetch_pmc_s3(client: httpx.Client, pmcid: str) -> str | None:
    """PMC OA PDF via the pmc-oa-opendata S3 bucket.

    The current canonical OA route — the FTP tree moved to
    ``deprecated/`` (#657). Keys are ``PMC{n}.{ver}/PMC{n}.{ver}.pdf``;
    pick the newest version. Uncapped and challenge-free, so it runs
    before the OA Web Service.
    """
    if not pmcid:
        return None
    pmc = pmcid if pmcid.startswith("PMC") else f"PMC{pmcid}"
    try:
        r = client.get(
            f"{_S3_BASE}/",
            params={"list-type": "2", "prefix": f"{pmc}.", "max-keys": "100"},
            timeout=15,
        )
        if r.status_code != 200:
            return None
    except httpx.HTTPError:
        return None
    versions = [
        int(m.group(1))
        for m in re.finditer(rf"<Key>{pmc}\.(\d+)/{pmc}\.\1\.pdf</Key>", r.text)
    ]
    if not versions:
        return None
    v = max(versions)
    return f"{_S3_BASE}/{pmc}.{v}/{pmc}.{v}.pdf"


_EPMC_BREAKER = {"fails": 0}
_EPMC_MAX_FAILS = 3


def fetch_europepmc_download(
    client: httpx.Client, pmcid: str, dest: Path
) -> Path | None:
    """Europe PMC render endpoint — the lawful full-text route for the
    PMC author manuscripts the OA subset excludes (#657).

    EBI outages happen, so three consecutive connection failures open a
    circuit breaker that skips the tier for the rest of the run; any
    success (even a non-PDF response) closes it.
    """
    if not pmcid or _EPMC_BREAKER["fails"] >= _EPMC_MAX_FAILS:
        return None
    pmc = pmcid if pmcid.startswith("PMC") else f"PMC{pmcid}"
    url = f"https://europepmc.org/backend/ptpmcrender.fcgi?accid={pmc}&blobtype=pdf"
    try:
        with client.stream(
            "GET", url, timeout=httpx.Timeout(30, connect=5), follow_redirects=True
        ) as r:
            _EPMC_BREAKER["fails"] = 0  # host reachable
            if r.status_code != 200:
                return None
            with open(dest, "wb") as f:
                for chunk in r.iter_bytes(1 << 16):
                    f.write(chunk)
    except (httpx.ConnectError, httpx.ConnectTimeout):
        _EPMC_BREAKER["fails"] += 1
        return None
    except (httpx.HTTPError, OSError):
        return None
    try:
        with open(dest, "rb") as f:
            head = f.read(4)
        if dest.stat().st_size < 1024 or head != b"%PDF":
            dest.unlink(missing_ok=True)
            return None
    except OSError:
        return None
    return dest


def resolve_doi_from_title(client: httpx.Client, title: str) -> str | None:
    """CrossRef bibliographic search → best-match DOI.

    Only accept a hit when CrossRef's relevance score is strong enough
    (>50) and the returned title meaningfully matches the query — the
    API happily returns low-confidence fuzzy matches that would poison
    the downstream cascade.
    """
    if not title or len(title) < 10:
        return None
    try:
        r = client.get(
            "https://api.crossref.org/works",
            params={"query.bibliographic": title, "rows": 3},
            timeout=15,
        )
        if r.status_code != 200:
            return None
        items = (r.json().get("message") or {}).get("items") or []
    except (httpx.HTTPError, ValueError):
        return None
    qnorm = _simplify(title)
    for it in items:
        doi = (it.get("DOI") or "").lower()
        cand_title = " ".join(it.get("title") or []) or ""
        score = it.get("score") or 0
        if not doi or not cand_title:
            continue
        # Require high CrossRef relevance AND meaningful title overlap.
        if score < 50:
            continue
        if _simplify(cand_title)[:80] == qnorm[:80]:
            return doi
    return None


def _simplify(s: str) -> str:
    """Lowercase + strip punctuation for crude title equality checks."""
    return re.sub(r"[^a-z0-9]+", " ", (s or "").lower()).strip()


_ALTCHA_MAX_ITERATIONS = 2_000_000  # hashes; >1M is rare, stops runaway


def _solve_altcha_pow(
    salt: str, challenge: str, maxnumber: int
) -> tuple[int, int] | None:
    """Brute-force the SHA-256 PoW. Returns (nonce, elapsed_ms) or None.

    The challenge is solved when ``sha256(salt+str(n)).hexdigest() == challenge``.
    Typical difficulty on the mirror is ~100k-500k iterations → <1s.
    """
    target = challenge.lower()
    t0 = time.monotonic()
    cap = min(maxnumber, _ALTCHA_MAX_ITERATIONS)
    for n in range(cap + 1):
        if hashlib.sha256(f"{salt}{n}".encode()).hexdigest() == target:
            return n, int((time.monotonic() - t0) * 1000)
    return None


def _altcha_bootstrap(client: httpx.Client, page_url: str, html: str) -> bool:
    """Detect the altcha robot-check embedded in *html*, solve it, persist
    the session cookie. Returns True on success, False if no challenge or
    the solution was rejected.
    """
    ch_match = re.search(r'challengeurl\s*=\s*["\']([^"\']+)["\']', html)
    cid_match = re.search(r"/captcha/solution/(\d+)", html)
    if not ch_match or not cid_match:
        return False
    origin = re.match(r"^(https?://[^/]+)", page_url)
    if not origin:
        return False
    base = origin.group(1)
    try:
        chall = client.get(f"{base}{ch_match.group(1)}", timeout=20).json()
    except (httpx.HTTPError, ValueError):
        return False

    maxnum = chall.get("maxNumber") or chall.get("maxnumber") or 1_000_000
    solved = _solve_altcha_pow(chall["salt"], chall["challenge"], maxnum)
    if not solved:
        return False
    nonce, elapsed = solved

    payload = {
        "algorithm": chall.get("algorithm", "SHA-256"),
        "challenge": chall["challenge"],
        "number": nonce,
        "salt": chall["salt"],
        "signature": chall["signature"],
        "test": False,
        "took": elapsed,
    }
    b64 = base64.b64encode(_json.dumps(payload).encode()).decode()
    try:
        r = client.post(
            f"{base}/captcha/solution/{cid_match.group(1)}",
            json={"captcha": b64},
            headers={"Content-Type": "application/json"},
            timeout=20,
        )
    except httpx.HTTPError:
        return False
    if r.status_code != 200:
        return False
    try:
        return bool(r.json().get("success"))
    except ValueError:
        return False


def _extract_pdf_url(html: str, base: str) -> str | None:
    """Pull a PDF URL out of a mirror's citation page.

    Mirror renders include ``<meta name="citation_pdf_url" content=…>``,
    an ``<embed src=…pdf…>``, or a legacy ``location = "…pdf"`` script.
    """
    patterns = (
        r'<meta\s+name\s*=\s*["\']citation_pdf_url["\']\s+content\s*=\s*["\']([^"\']+)["\']',
        r'<embed[^>]+src\s*=\s*["\']([^"\']+\.pdf[^"\']*)',
        r'<iframe[^>]+src\s*=\s*["\']([^"\']+\.pdf[^"\']*)',
        r'(?:src|location|href)\s*=\s*["\']([^"\']+\.pdf[^"\']*)',
    )
    for pat in patterns:
        m = re.search(pat, html, re.IGNORECASE)
        if not m:
            continue
        url = m.group(1)
        if url.startswith("//"):
            return "https:" + url
        if url.startswith("/"):
            return base + url
        return url
    return None


def fetch_fallback(client: httpx.Client, doi: str) -> str | None:
    """Last-resort fallback mirror lookup for a DOI.

    Client should be configured with ``proxy=`` pointing at the VPN
    tunnel — direct hits from residential/commercial US endpoints are
    unreliable. The session is expected to be long-lived so altcha
    solves persist across calls.
    """
    if not doi:
        return None
    for mirror in _FALLBACK_MIRRORS:
        url = f"{mirror}/{doi}"
        try:
            r = client.get(url, timeout=30)
        except httpx.HTTPError:
            continue
        if r.status_code != 200:
            continue
        html = r.text
        if "altcha-widget" in html or "captcha/challenge" in html:
            if not _altcha_bootstrap(client, url, html):
                continue
            try:
                r = client.get(url, timeout=30)
            except httpx.HTTPError:
                continue
            if r.status_code != 200:
                continue
            html = r.text
        pdf = _extract_pdf_url(html, mirror)
        if pdf:
            return pdf
    return None


# ── Download + attach ────────────────────────────────────────────


def download(client: httpx.Client, url: str, dest: Path) -> Path | None:
    """Stream a PDF to disk; validate it actually is a PDF."""
    try:
        with client.stream("GET", url, timeout=60, follow_redirects=True) as r:
            if r.status_code != 200:
                return None
            ctype = r.headers.get("content-type", "")
            if "pdf" not in ctype.lower() and not url.lower().endswith(".pdf"):
                pass
            with open(dest, "wb") as f:
                for chunk in r.iter_bytes(1 << 16):
                    f.write(chunk)
    except (httpx.HTTPError, OSError):
        return None
    try:
        if dest.stat().st_size < 1024:
            dest.unlink(missing_ok=True)
            return None
        with open(dest, "rb") as f:
            head = f.read(4)
        if head != b"%PDF":
            dest.unlink(missing_ok=True)
            return None
    except OSError:
        return None
    return dest


def fetch_one(
    item: PendingItem,
    *,
    email: str,
    scratch: Path,
    client: httpx.Client,
    client_proxied: httpx.Client | None = None,
    allow_fallback: bool = True,
) -> tuple[str, Path] | None:
    """Run one item through the source cascade.

    ``client`` is the unproxied httpx client (Unpaywall, Semantic
    Scholar, PMC). ``client_proxied`` (optional) routes the fallback
    tier through the VPN; if None, the fallback is skipped.

    Order: every Unpaywall location → Semantic Scholar openAccessPdf →
    PMC S3 → PMC OA service → Europe PMC → fallback. A PMCID the item
    lacks may arrive from Semantic Scholar before the PMC tiers run.
    """
    filename = scratch / f"{item.zot_id}-{_hash(item.doi or item.title)}.pdf"
    doi = item.doi
    if not doi and item.title:
        doi = resolve_doi_from_title(client, item.title) or ""

    for url in unpaywall_urls(client, doi, email):
        path = download(client, url, filename)
        if path:
            return "unpaywall", path

    pmcid = item.pmcid
    s2 = fetch_semantic_scholar(client, doi)
    if s2:
        if s2.pdf_url:
            path = download(client, s2.pdf_url, filename)
            if path:
                return "s2", path
        if not pmcid:
            pmcid = s2.pmcid

    url = fetch_pmc_s3(client, pmcid)
    if url:
        path = download(client, url, filename)
        if path:
            return "pmc", path

    url = fetch_pmc(client, pmcid)
    if url:
        path = download(client, url, filename)
        if path:
            return "pmc", path

    path = fetch_europepmc_download(client, pmcid, filename)
    if path:
        return "pmc", path

    if allow_fallback and client_proxied is not None:
        with _fallback_lock:
            url = fetch_fallback(client_proxied, doi)
            if url:
                path = download(client_proxied, url, filename)
                if path:
                    return "fallback", path

    return None


def attach_pdf(
    db: Db,
    zot_parent_id: int,
    path: Path,
    storage_dir: Path,
    *,
    title: str = "",
) -> int:
    """Copy PDF into Zotero storage and register it as a child attachment.

    Returns the attachment's itemID.
    """
    att_key = generate_key()
    target_dir = storage_dir / att_key
    target_dir.mkdir(parents=True, exist_ok=True)
    dest = target_dir / path.name
    shutil.copy2(path, dest)

    att_id = db.create_item(TYPE_MAP["attachment"], key=att_key, now=now_iso())
    db.con.execute(
        """INSERT INTO itemAttachments
           (itemID, parentItemID, linkMode, contentType, path)
           VALUES (?, ?, ?, ?, ?)""",
        (att_id, zot_parent_id, 1, "application/pdf", f"storage:{path.name}"),
    )
    if title:
        db.set_fields(att_id, {"title": title})
    return att_id


# ── Orchestrator ────────────────────────────────────────────────


def run(
    db: Db,
    *,
    project: str,
    storage_dir: Path,
    scratch_dir: Path,
    email: str,
    fetch_proxy: str | None,
    limit: int | None = None,
    progress: bool = True,
    workers: int = 8,
    allow_fallback: bool = True,
) -> dict[str, int]:
    """Fetch PDFs for every non-excluded item in the project.

    Attaches each successful download to its Zotero parent. Leaves
    un-retrievable items untouched — stage 3 eligibility will flag them
    as `screen:reason:unavailable-fulltext`.
    """
    scratch_dir.mkdir(parents=True, exist_ok=True)
    stats = {"unpaywall": 0, "s2": 0, "pmc": 0, "fallback": 0, "missed": 0, "errors": 0}

    client = httpx.Client(
        headers={"User-Agent": USER_AGENT.format(email)},
        follow_redirects=True,
    )
    pending = pending_queue(db, project, limit=limit, client=client, email=email)
    client_proxied: httpx.Client | None = None
    if fetch_proxy:
        # Mirrors behind altcha treat our generic UA as a bot; a real
        # browser UA passes through the challenge reliably.
        browser_ua = (
            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
            "(KHTML, like Gecko) Firefox/122.0"
        )
        client_proxied = httpx.Client(
            headers={"User-Agent": browser_ua},
            follow_redirects=True,
            proxy=fetch_proxy,
        )

    done = 0
    try:
        with ThreadPoolExecutor(max_workers=max(1, workers)) as pool:
            futures = {
                pool.submit(
                    fetch_one,
                    item,
                    email=email,
                    scratch=scratch_dir,
                    client=client,
                    client_proxied=client_proxied,
                    allow_fallback=allow_fallback,
                ): item
                for item in pending
            }
            # All DB work stays on this thread — the sqlite connection
            # must never be shared with the pool.
            for fut in as_completed(futures):
                item = futures[fut]
                done += 1
                try:
                    hit = fut.result()
                except Exception as e:  # noqa: BLE001
                    log.warning("fetch failed for item %s: %s", item.zot_id, e)
                    stats["errors"] += 1
                    continue

                if not hit:
                    stats["missed"] += 1
                else:
                    source, path = hit
                    try:
                        attach_pdf(db, item.zot_id, path, storage_dir, title=item.title)
                        db.commit()
                        path.unlink(missing_ok=True)
                        stats[source] += 1
                    except Exception as e:  # noqa: BLE001
                        log.warning("attach failed for item %s: %s", item.zot_id, e)
                        stats["errors"] += 1

                if progress and done % 25 == 0:
                    print(
                        f"  fetched {done}/{len(pending)}  "
                        f"(up={stats['unpaywall']} s2={stats['s2']} "
                        f"pmc={stats['pmc']} fb={stats['fallback']} "
                        f"miss={stats['missed']})"
                    )
    finally:
        client.close()
        if client_proxied is not None:
            client_proxied.close()

    return stats


# ── Helpers ─────────────────────────────────────────────────────


def _field(db: Db, zot_id: int, field_name: str) -> str:
    field_id = FIELD_MAP.get(field_name)
    if field_id is None:
        return ""
    row = db.con.execute(
        """SELECT idv.value FROM itemData id
           JOIN itemDataValues idv ON id.valueID = idv.valueID
           WHERE id.itemID = ? AND id.fieldID = ?""",
        (zot_id, field_id),
    ).fetchone()
    return (row[0] if row else "") or ""


def _extra_value(db: Db, zot_id: int, key: str) -> str:
    """Scrape ``KEY: VALUE`` pairs out of the free-form ``extra`` field."""
    extra = _field(db, zot_id, "extra")
    if not extra:
        return ""
    m = re.search(rf"^{re.escape(key)}\s*:\s*(.+)$", extra, re.M | re.I)
    return m.group(1).strip() if m else ""


def _from_url_pmcid(db: Db, zot_id: int) -> str:
    url = _field(db, zot_id, "url")
    m = re.search(r"PMC\d+", url or "")
    return m.group(0) if m else ""


def _hash(s: str) -> str:
    return hashlib.sha1((s or "").encode()).hexdigest()[:10]  # noqa: S324 — not security