diff --git a/src/cli/prisma.py b/src/cli/prisma.py index 0a52f8e..74bc6c8 100644 --- a/src/cli/prisma.py +++ b/src/cli/prisma.py @@ -398,6 +398,7 @@ def fetch( fetch_proxy=fetch_proxy, limit=limit or None, workers=workers, + allow_fallback=_fetch.fallback_allowed(name), ) stats = _hold(_go, hold=hold) @@ -599,6 +600,7 @@ def run_all( email=os.environ.get("UNPAYWALL_EMAIL", "dev@fhirworx.io"), fetch_proxy=fetch_proxy, limit=limit or None, + allow_fallback=_fetch.fallback_allowed(name), ) typer.echo(f" {fetch_stats}") diff --git a/src/prisma/fetch.py b/src/prisma/fetch.py index f521cf9..5497f64 100644 --- a/src/prisma/fetch.py +++ b/src/prisma/fetch.py @@ -56,6 +56,16 @@ _FALLBACK_MIRRORS = ( "https://sci-hub.box", ) +# Projects that must not use the mirror tier. Empty: the five LEAD +# reviews leave the fallback on, and any other project can be named here. +SCIHUB_OFF: frozenset[str] = frozenset() + + +def fallback_allowed(project: str) -> bool: + """True when this project's fetch may use the mirror tier.""" + return project not in SCIHUB_OFF + + # The fallback tier shares one proxied client (altcha solves mutate its # cookie jar) and one VPN exit — hold this across the whole tier. _fallback_lock = threading.Lock() @@ -621,6 +631,7 @@ def fetch_one( scratch: Path, client: httpx.Client, client_proxied: httpx.Client | None = None, + allow_fallback: bool = True, ) -> tuple[str, Path] | None: """Run one item through the source cascade. @@ -668,7 +679,7 @@ def fetch_one( if path: return "pmc", path - if client_proxied is not None: + if allow_fallback and client_proxied is not None: with _fallback_lock: url = fetch_fallback(client_proxied, doi) if url: @@ -723,6 +734,7 @@ def run( limit: int | None = None, progress: bool = True, workers: int = 8, + allow_fallback: bool = True, ) -> dict[str, int]: """Fetch PDFs for every non-excluded item in the project. @@ -763,6 +775,7 @@ def run( scratch=scratch_dir, client=client, client_proxied=client_proxied, + allow_fallback=allow_fallback, ): item for item in pending } diff --git a/tests/prisma/test_fetch_deep.py b/tests/prisma/test_fetch_deep.py index 6587e65..0aa7956 100644 --- a/tests/prisma/test_fetch_deep.py +++ b/tests/prisma/test_fetch_deep.py @@ -105,7 +105,9 @@ class TestRunParallel: def test_all_items_attached_and_counted(self, zotero_db, tmp_path, monkeypatch): _seed_project(zotero_db, 6) - def fake_fetch_one(item, *, email, scratch, client, client_proxied=None): + def fake_fetch_one( + item, *, email, scratch, client, client_proxied=None, allow_fallback=True + ): time.sleep(0.05) p = scratch / f"{item.zot_id}.pdf" p.write_bytes(b"%PDF-1.4 fake") @@ -133,7 +135,9 @@ class TestRunParallel: state = {"active": 0, "max_active": 0} gauge = threading.Lock() - def slow_fetch_one(item, *, email, scratch, client, client_proxied=None): + def slow_fetch_one( + item, *, email, scratch, client, client_proxied=None, allow_fallback=True + ): with gauge: state["active"] += 1 state["max_active"] = max(state["max_active"], state["active"]) @@ -159,7 +163,9 @@ class TestRunParallel: def test_worker_exception_counts_as_error(self, zotero_db, tmp_path, monkeypatch): _seed_project(zotero_db, 3) - def boom(item, *, email, scratch, client, client_proxied=None): + def boom( + item, *, email, scratch, client, client_proxied=None, allow_fallback=True + ): raise RuntimeError("kaput") monkeypatch.setattr(fetch_mod, "fetch_one", boom) diff --git a/tests/prisma/test_fetch_fallback_switch.py b/tests/prisma/test_fetch_fallback_switch.py new file mode 100644 index 0000000..091f632 --- /dev/null +++ b/tests/prisma/test_fetch_fallback_switch.py @@ -0,0 +1,134 @@ +"""Per-project switch for the Sci-Hub fallback. LEAD projects leave it on.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +from typer.testing import CliRunner + +import prisma.fetch as fetch +from cli.prisma import app +from prisma.fetch import PendingItem, fallback_allowed + +runner = CliRunner() + +LEAD = ( + "lead-history", + "lead-limits", + "lead-scan", + "lead-precursors", + "lead-external", +) + + +def _mock_db(): + db = MagicMock() + db.__enter__ = MagicMock(return_value=db) + db.__exit__ = MagicMock(return_value=False) + return db + + +def test_lead_projects_keep_the_fallback(): + for name in LEAD: + assert fallback_allowed(name) is True + + +def test_named_project_skips_the_mirror(monkeypatch): + monkeypatch.setattr(fetch, "SCIHUB_OFF", frozenset({"skin-subs"})) + assert fallback_allowed("skin-subs") is False + assert fallback_allowed("lead-history") is True + + +def test_allow_fallback_false_skips_the_mirror(tmp_path): + item = PendingItem(zot_id=1, doi="10.1/x", title="T", pmcid="", pmid="") + path = tmp_path / "paper.pdf" + client = MagicMock() + with ( + patch("prisma.fetch.fetch_fallback", return_value="https://mirror/p.pdf") as fb, + patch("prisma.fetch.download", return_value=path), + ): + missed = fetch.fetch_one( + item, + email="e@x", + scratch=tmp_path, + client=client, + client_proxied=client, + allow_fallback=False, + ) + assert missed is None + fb.assert_not_called() + hit = fetch.fetch_one( + item, + email="e@x", + scratch=tmp_path, + client=client, + client_proxied=client, + allow_fallback=True, + ) + assert hit == ("fallback", path) + + +def test_run_threads_allow_fallback(tmp_path, monkeypatch): + item = PendingItem(zot_id=1, doi="10.1/x", title="T", pmcid="", pmid="") + seen: dict[str, object] = {} + + def spy(one, **kwargs): + seen.update(kwargs) + return None + + monkeypatch.setattr(fetch, "pending_queue", lambda *a, **k: [item]) + monkeypatch.setattr(fetch, "fetch_one", spy) + stats = fetch.run( + MagicMock(), + project="skin-subs", + storage_dir=tmp_path, + scratch_dir=tmp_path / "scratch", + email="e@x", + fetch_proxy=None, + allow_fallback=False, + progress=False, + ) + assert seen["allow_fallback"] is False + assert stats["missed"] == 1 + + +def test_fetch_command_passes_the_project_switch(monkeypatch): + with ( + patch("prisma.fetch.run", return_value={"missed": 0}) as mc_run, + patch("zot.db.Db", return_value=_mock_db()), + patch("prisma.vpn.status", return_value={"status": "down"}), + ): + result = runner.invoke( + app, ["fetch", "lead-precursors", "--no-hold", "--no-vpn"] + ) + assert result.exit_code == 0 + assert mc_run.call_args.kwargs["allow_fallback"] is True + + monkeypatch.setattr(fetch, "SCIHUB_OFF", frozenset({"skin-subs"})) + with ( + patch("prisma.fetch.run", return_value={"missed": 0}) as mc_run, + patch("zot.db.Db", return_value=_mock_db()), + patch("prisma.vpn.status", return_value={"status": "down"}), + ): + result = runner.invoke(app, ["fetch", "skin-subs", "--no-hold", "--no-vpn"]) + assert result.exit_code == 0 + assert mc_run.call_args.kwargs["allow_fallback"] is False + + +def test_orchestrated_run_uses_the_same_switch(monkeypatch): + monkeypatch.setattr(fetch, "SCIHUB_OFF", frozenset({"skin-subs"})) + with ( + patch("prisma.screen.run", return_value={"screened": 0, "errors": 0}), + patch("prisma.fetch.run", return_value={"missed": 0}) as mc_fetch, + patch("prisma.eligibility.run", return_value={"assessed": 0}), + patch("prisma.extract.run", return_value={"extracted": 0}), + patch("prisma.flow.count", return_value=MagicMock()), + patch("prisma.flow.text_summary", return_value="ok"), + patch("prisma.vpn.status", return_value={"status": "down"}), + patch("prisma.llm.make_provider", return_value=MagicMock()), + patch("prisma.project.load", return_value=MagicMock()), + patch("zot.db.Db", return_value=_mock_db()), + ): + result = runner.invoke(app, ["run", "skin-subs", "--no-hold"]) + assert result.exit_code == 0 + assert mc_fetch.call_args.kwargs["allow_fallback"] is False