feat(prisma): per-project Sci-Hub switch, on for LEAD

src/cli/prisma.py

@@ -398,6 +398,7 @@ def fetch(
                 fetch_proxy=fetch_proxy,
                 limit=limit or None,
                 workers=workers,
+                allow_fallback=_fetch.fallback_allowed(name),
             )
 
     stats = _hold(_go, hold=hold)
@@ -599,6 +600,7 @@ def run_all(
                         email=os.environ.get("UNPAYWALL_EMAIL", "dev@fhirworx.io"),
                         fetch_proxy=fetch_proxy,
                         limit=limit or None,
+                        allow_fallback=_fetch.fallback_allowed(name),
                     )
                 typer.echo(f"   {fetch_stats}")
 

src/prisma/fetch.py

@@ -56,6 +56,16 @@ _FALLBACK_MIRRORS = (
     "https://sci-hub.box",
 )
 
+# Projects that must not use the mirror tier. Empty: the five LEAD
+# reviews leave the fallback on, and any other project can be named here.
+SCIHUB_OFF: frozenset[str] = frozenset()
+
+
+def fallback_allowed(project: str) -> bool:
+    """True when this project's fetch may use the mirror tier."""
+    return project not in SCIHUB_OFF
+
+
 # The fallback tier shares one proxied client (altcha solves mutate its
 # cookie jar) and one VPN exit — hold this across the whole tier.
 _fallback_lock = threading.Lock()
@@ -621,6 +631,7 @@ def fetch_one(
     scratch: Path,
     client: httpx.Client,
     client_proxied: httpx.Client | None = None,
+    allow_fallback: bool = True,
 ) -> tuple[str, Path] | None:
     """Run one item through the source cascade.
 
@@ -668,7 +679,7 @@ def fetch_one(
     if path:
         return "pmc", path
 
-    if client_proxied is not None:
+    if allow_fallback and client_proxied is not None:
         with _fallback_lock:
             url = fetch_fallback(client_proxied, doi)
             if url:
@@ -723,6 +734,7 @@ def run(
     limit: int | None = None,
     progress: bool = True,
     workers: int = 8,
+    allow_fallback: bool = True,
 ) -> dict[str, int]:
     """Fetch PDFs for every non-excluded item in the project.
 
@@ -763,6 +775,7 @@ def run(
                     scratch=scratch_dir,
                     client=client,
                     client_proxied=client_proxied,
+                    allow_fallback=allow_fallback,
                 ): item
                 for item in pending
             }

tests/prisma/test_fetch_deep.py

@@ -105,7 +105,9 @@ class TestRunParallel:
     def test_all_items_attached_and_counted(self, zotero_db, tmp_path, monkeypatch):
         _seed_project(zotero_db, 6)
 
-        def fake_fetch_one(item, *, email, scratch, client, client_proxied=None):
+        def fake_fetch_one(
+            item, *, email, scratch, client, client_proxied=None, allow_fallback=True
+        ):
             time.sleep(0.05)
             p = scratch / f"{item.zot_id}.pdf"
             p.write_bytes(b"%PDF-1.4 fake")
@@ -133,7 +135,9 @@ class TestRunParallel:
         state = {"active": 0, "max_active": 0}
         gauge = threading.Lock()
 
-        def slow_fetch_one(item, *, email, scratch, client, client_proxied=None):
+        def slow_fetch_one(
+            item, *, email, scratch, client, client_proxied=None, allow_fallback=True
+        ):
             with gauge:
                 state["active"] += 1
                 state["max_active"] = max(state["max_active"], state["active"])
@@ -159,7 +163,9 @@ class TestRunParallel:
     def test_worker_exception_counts_as_error(self, zotero_db, tmp_path, monkeypatch):
         _seed_project(zotero_db, 3)
 
-        def boom(item, *, email, scratch, client, client_proxied=None):
+        def boom(
+            item, *, email, scratch, client, client_proxied=None, allow_fallback=True
+        ):
             raise RuntimeError("kaput")
 
         monkeypatch.setattr(fetch_mod, "fetch_one", boom)

tests/prisma/test_fetch_fallback_switch.py

@@ -0,0 +1,134 @@
+"""Per-project switch for the Sci-Hub fallback. LEAD projects leave it on."""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+from typer.testing import CliRunner
+
+import prisma.fetch as fetch
+from cli.prisma import app
+from prisma.fetch import PendingItem, fallback_allowed
+
+runner = CliRunner()
+
+LEAD = (
+    "lead-history",
+    "lead-limits",
+    "lead-scan",
+    "lead-precursors",
+    "lead-external",
+)
+
+
+def _mock_db():
+    db = MagicMock()
+    db.__enter__ = MagicMock(return_value=db)
+    db.__exit__ = MagicMock(return_value=False)
+    return db
+
+
+def test_lead_projects_keep_the_fallback():
+    for name in LEAD:
+        assert fallback_allowed(name) is True
+
+
+def test_named_project_skips_the_mirror(monkeypatch):
+    monkeypatch.setattr(fetch, "SCIHUB_OFF", frozenset({"skin-subs"}))
+    assert fallback_allowed("skin-subs") is False
+    assert fallback_allowed("lead-history") is True
+
+
+def test_allow_fallback_false_skips_the_mirror(tmp_path):
+    item = PendingItem(zot_id=1, doi="10.1/x", title="T", pmcid="", pmid="")
+    path = tmp_path / "paper.pdf"
+    client = MagicMock()
+    with (
+        patch("prisma.fetch.fetch_fallback", return_value="https://mirror/p.pdf") as fb,
+        patch("prisma.fetch.download", return_value=path),
+    ):
+        missed = fetch.fetch_one(
+            item,
+            email="e@x",
+            scratch=tmp_path,
+            client=client,
+            client_proxied=client,
+            allow_fallback=False,
+        )
+        assert missed is None
+        fb.assert_not_called()
+        hit = fetch.fetch_one(
+            item,
+            email="e@x",
+            scratch=tmp_path,
+            client=client,
+            client_proxied=client,
+            allow_fallback=True,
+        )
+    assert hit == ("fallback", path)
+
+
+def test_run_threads_allow_fallback(tmp_path, monkeypatch):
+    item = PendingItem(zot_id=1, doi="10.1/x", title="T", pmcid="", pmid="")
+    seen: dict[str, object] = {}
+
+    def spy(one, **kwargs):
+        seen.update(kwargs)
+        return None
+
+    monkeypatch.setattr(fetch, "pending_queue", lambda *a, **k: [item])
+    monkeypatch.setattr(fetch, "fetch_one", spy)
+    stats = fetch.run(
+        MagicMock(),
+        project="skin-subs",
+        storage_dir=tmp_path,
+        scratch_dir=tmp_path / "scratch",
+        email="e@x",
+        fetch_proxy=None,
+        allow_fallback=False,
+        progress=False,
+    )
+    assert seen["allow_fallback"] is False
+    assert stats["missed"] == 1
+
+
+def test_fetch_command_passes_the_project_switch(monkeypatch):
+    with (
+        patch("prisma.fetch.run", return_value={"missed": 0}) as mc_run,
+        patch("zot.db.Db", return_value=_mock_db()),
+        patch("prisma.vpn.status", return_value={"status": "down"}),
+    ):
+        result = runner.invoke(
+            app, ["fetch", "lead-precursors", "--no-hold", "--no-vpn"]
+        )
+    assert result.exit_code == 0
+    assert mc_run.call_args.kwargs["allow_fallback"] is True
+
+    monkeypatch.setattr(fetch, "SCIHUB_OFF", frozenset({"skin-subs"}))
+    with (
+        patch("prisma.fetch.run", return_value={"missed": 0}) as mc_run,
+        patch("zot.db.Db", return_value=_mock_db()),
+        patch("prisma.vpn.status", return_value={"status": "down"}),
+    ):
+        result = runner.invoke(app, ["fetch", "skin-subs", "--no-hold", "--no-vpn"])
+    assert result.exit_code == 0
+    assert mc_run.call_args.kwargs["allow_fallback"] is False
+
+
+def test_orchestrated_run_uses_the_same_switch(monkeypatch):
+    monkeypatch.setattr(fetch, "SCIHUB_OFF", frozenset({"skin-subs"}))
+    with (
+        patch("prisma.screen.run", return_value={"screened": 0, "errors": 0}),
+        patch("prisma.fetch.run", return_value={"missed": 0}) as mc_fetch,
+        patch("prisma.eligibility.run", return_value={"assessed": 0}),
+        patch("prisma.extract.run", return_value={"extracted": 0}),
+        patch("prisma.flow.count", return_value=MagicMock()),
+        patch("prisma.flow.text_summary", return_value="ok"),
+        patch("prisma.vpn.status", return_value={"status": "down"}),
+        patch("prisma.llm.make_provider", return_value=MagicMock()),
+        patch("prisma.project.load", return_value=MagicMock()),
+        patch("zot.db.Db", return_value=_mock_db()),
+    ):
+        result = runner.invoke(app, ["run", "skin-subs", "--no-hold"])
+    assert result.exit_code == 0
+    assert mc_fetch.call_args.kwargs["allow_fallback"] is False