From 0b29cd7acce79b7b49504db95de6abc17c0eca2b Mon Sep 17 00:00:00 2001 From: kert Date: Wed, 23 Sep 2026 21:11:53 -0400 Subject: [PATCH] =?UTF-8?q?feat(zot):=20stack=20zot=20fetch=20=E2=80=94=20?= =?UTF-8?q?full-text=20connector=20CLI=20under=20the=20--hold=20guard,=20w?= =?UTF-8?q?ith=20docs=20(refs=20Z1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 --- docs/docs/cli/zot-fetch.md | 35 +++++++ docs/docs/cli/zot.md | 1 + ...026-09-09-zotero-annas-connector-design.md | 2 +- src/cli/zot.py | 98 +++++++++++++++++++ tests/cli/conftest.py | 3 + tests/cli/test_zot_fetch.py | 97 ++++++++++++++++++ 6 files changed, 235 insertions(+), 1 deletion(-) create mode 100644 docs/docs/cli/zot-fetch.md create mode 100644 tests/cli/conftest.py create mode 100644 tests/cli/test_zot_fetch.py diff --git a/docs/docs/cli/zot-fetch.md b/docs/docs/cli/zot-fetch.md new file mode 100644 index 0000000..ec57d5f --- /dev/null +++ b/docs/docs/cli/zot-fetch.md @@ -0,0 +1,35 @@ +--- +title: stack zot fetch +sidebar_position: 87 +--- + +# `stack zot fetch` + +``` +Usage: stack zot fetch [OPTIONS] + + Attach full text (PDF/EPUB) to items that lack it. + + DOI → Unpaywall, then the libgen index; ISBN → libgen; delivery by + libgen or Anna's fast download (member key). Items with a PDF/EPUB, + deleted items and ``document`` items are always skipped. Every + attempt is a line in the ledger; misses wait 30 days unless --retry. + +╭─ Options ────────────────────────────────────────────────────────────────────╮ +│ --collection TEXT Collection name. │ +│ --tag TEXT Tag name. │ +│ --type TEXT Item type, e.g. journalArticle, book. │ +│ --item TEXT One item key. │ +│ --all Every eligible item. │ +│ --limit INTEGER Stop after N attempts (0 = all). │ +│ [default: 0] │ +│ --dry-run List candidates, write nothing. │ +│ --retry Retry items the ledger marked as │ +│ misses. │ +│ --db PATH │ +│ --ledger PATH [default: │ +│ data/zotero/fetch_ledger.jsonl] │ +│ --hold --no-hold [default: hold] │ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────╯ +``` diff --git a/docs/docs/cli/zot.md b/docs/docs/cli/zot.md index 454d657..3cc990a 100644 --- a/docs/docs/cli/zot.md +++ b/docs/docs/cli/zot.md @@ -21,5 +21,6 @@ Usage: stack zot [OPTIONS] COMMAND [ARGS]... │ fix-fields Remap base-field rows to type-specific fieldIDs; drop │ │ orphans. │ │ verify-parity Check that ``zot.db`` constants match the live Zotero schema. │ +│ fetch Attach full text (PDF/EPUB) to items that lack it. │ ╰──────────────────────────────────────────────────────────────────────────────╯ ``` diff --git a/docs/superpowers/specs/2026-09-09-zotero-annas-connector-design.md b/docs/superpowers/specs/2026-09-09-zotero-annas-connector-design.md index 4390e65..453db49 100644 --- a/docs/superpowers/specs/2026-09-09-zotero-annas-connector-design.md +++ b/docs/superpowers/specs/2026-09-09-zotero-annas-connector-design.md @@ -1,6 +1,6 @@ # Zotero full-text connector (Anna's Archive, libgen, Unpaywall) — design -**Status:** draft for operator approval, 2026-09-09. +**Status:** Z1 approved and built 2026-09-22/23 (plan `docs/superpowers/plans/2026-09-22-zotero-fetch-z1.md`); Z2–Z4 pending. **Related:** `dev/scripts/add_ama_coding_to_zotero.py` (PR #697), corwins.media issue #707 (Anna's key), corwins spec `2026-09-09-direct-source-tier-design.md` (egress proxy). ## Problem diff --git a/src/cli/zot.py b/src/cli/zot.py index 2a3f6ff..62f2181 100644 --- a/src/cli/zot.py +++ b/src/cli/zot.py @@ -7,11 +7,18 @@ they run (SQLite file is WAL-locked by the desktop app). ``--hold`` from __future__ import annotations +import os import subprocess from pathlib import Path import typer +from zot.db import Db +from zot.fetch.deliver import AnnasAuthError, Budget +from zot.fetch.ledger import Ledger +from zot.fetch.run import make_clients, run +from zot.fetch.select import select_items + app = typer.Typer(no_args_is_help=True) _DEFAULT_DB = Path("data/zotero/data/zotero.sqlite") @@ -126,3 +133,94 @@ def verify_parity( with Db(str(_resolve(db_path))): typer.echo("parity OK") + + +_DEFAULT_LEDGER = Path("data/zotero/fetch_ledger.jsonl") +_PROXY_HELP = ( + "VPN_HTTP_PROXY is not set. libgen and Anna's must go through the gluetun " + "HTTP proxy published by corwins.media (docker compose up gluetun → " + "127.0.0.1:19035): add VPN_HTTP_PROXY=http://127.0.0.1:19035 to .env and " + "source it (set -a; . ./.env; set +a) before running." +) + + +@app.command(name="fetch") +def fetch( + collection: str = typer.Option("", "--collection", help="Collection name."), + tag: str = typer.Option("", "--tag", help="Tag name."), + type_: str = typer.Option( + "", "--type", help="Item type, e.g. journalArticle, book." + ), + item: str = typer.Option("", "--item", help="One item key."), + all_: bool = typer.Option(False, "--all", help="Every eligible item."), + limit: int = typer.Option(0, "--limit", help="Stop after N attempts (0 = all)."), + dry_run: bool = typer.Option( + False, "--dry-run", help="List candidates, write nothing." + ), + retry: bool = typer.Option( + False, "--retry", help="Retry items the ledger marked as misses." + ), + db_path: Path | None = typer.Option(None, "--db"), + ledger_path: Path = typer.Option(_DEFAULT_LEDGER, "--ledger"), + hold: bool = typer.Option(True, "--hold/--no-hold"), +) -> None: + """Attach full text (PDF/EPUB) to items that lack it. + + DOI → Unpaywall, then the libgen index; ISBN → libgen; delivery by + libgen or Anna's fast download (member key). Items with a PDF/EPUB, + deleted items and ``document`` items are always skipped. Every + attempt is a line in the ledger; misses wait 30 days unless --retry. + """ + from conf import path as conf_path + + proxy = os.environ.get("VPN_HTTP_PROXY", "") + if not proxy: + raise typer.BadParameter(_PROXY_HELP) + annas_key = os.environ.get("ANNAS_ARCHIVE_KEY", "") + if not annas_key: + typer.echo( + "ANNAS_ARCHIVE_KEY not set — Anna's fast download disabled for this run." + ) + email = os.environ.get("UNPAYWALL_EMAIL", "dev@fhirworx.io") + reserve = int(os.environ.get("ANNAS_DAILY_RESERVE", "5")) + path = _resolve(db_path) + + def _go() -> dict[str, int]: + with Db(str(path)) as db: + try: + refs = select_items( + db, + collection=collection, + tag=tag, + type_=type_, + item=item, + all_=all_, + ) + except ValueError as exc: + raise typer.BadParameter(str(exc)) from exc + typer.echo(f"{len(refs)} eligible item(s)") + clients = make_clients(proxy, email=email) + try: + return run( + db, + refs, + ledger=Ledger(ledger_path), + budget=Budget(annas_key=annas_key, reserve=reserve), + clients=clients, + email=email, + storage_dir=conf_path("storage.zotero"), + scratch_dir=Path(".state/zot/fetch"), + limit=limit, + dry_run=dry_run, + retry=retry, + out=typer.echo, + ) + except AnnasAuthError as exc: + typer.echo(f"error: {exc}", err=True) + raise typer.Exit(2) from exc + finally: + clients.direct.close() + clients.proxied.close() + + stats = _hold_zotero(_go, hold) + typer.echo("fetch: " + " ".join(f"{k}={v}" for k, v in stats.items())) diff --git a/tests/cli/conftest.py b/tests/cli/conftest.py new file mode 100644 index 0000000..97d26fa --- /dev/null +++ b/tests/cli/conftest.py @@ -0,0 +1,3 @@ +"""Shared fixtures for CLI tests.""" + +from tests.zot.fetch.conftest import zdb # noqa: F401 — scratch Zotero DB diff --git a/tests/cli/test_zot_fetch.py b/tests/cli/test_zot_fetch.py new file mode 100644 index 0000000..8cd66cf --- /dev/null +++ b/tests/cli/test_zot_fetch.py @@ -0,0 +1,97 @@ +"""stack zot fetch — CLI wiring (no network, no docker).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from cli.zot import app +from tests.zot.fetch.conftest import mk_item +from zot.db import Db +from zot.fetch.deliver import AnnasAuthError +from zot.fetch.run import STATS + +runner = CliRunner() + + +@pytest.fixture() +def env(monkeypatch, tmp_path: Path): + monkeypatch.setenv("VPN_HTTP_PROXY", "http://127.0.0.1:19035") + monkeypatch.setenv("ANNAS_ARCHIVE_KEY", "k") + monkeypatch.setenv("ANNAS_DAILY_RESERVE", "7") + monkeypatch.setenv("UNPAYWALL_EMAIL", "dev@fhirworx.io") + monkeypatch.setattr("cli.zot._hold_zotero", lambda fn, hold: fn()) + return tmp_path + + +def _db_with_item(zdb: Db) -> str: + mk_item(zdb, title="T", doi="10.1/x") + zdb.commit() + return zdb.path + + +def test_fetch_requires_proxy(monkeypatch, zdb: Db): + monkeypatch.delenv("VPN_HTTP_PROXY", raising=False) + monkeypatch.setattr("cli.zot._hold_zotero", lambda fn, hold: fn()) + r = runner.invoke(app, ["fetch", "--all", "--db", _db_with_item(zdb), "--no-hold"]) + assert r.exit_code != 0 + assert "VPN_HTTP_PROXY" in r.output and "gluetun" in r.output + + +def test_fetch_dry_run_calls_run_with_selection(env, zdb: Db, monkeypatch): + captured: dict = {} + + def fake_run(db, refs, **kw): + captured["refs"] = refs + captured["kw"] = kw + return dict.fromkeys(STATS, 0) | {"seen": 1, "attempted": 1} + + monkeypatch.setattr("cli.zot.run", fake_run) + r = runner.invoke( + app, + [ + "fetch", + "--all", + "--dry-run", + "--limit", + "3", + "--db", + _db_with_item(zdb), + "--ledger", + str(env / "ledger.jsonl"), + "--no-hold", + ], + ) + assert r.exit_code == 0, r.output + assert [x.doi for x in captured["refs"]] == ["10.1/x"] + kw = captured["kw"] + assert kw["dry_run"] is True and kw["limit"] == 3 and kw["retry"] is False + assert kw["budget"].annas_key == "k" and kw["budget"].reserve == 7 + assert kw["email"] == "dev@fhirworx.io" + assert kw["ledger"].path == env / "ledger.jsonl" + assert kw["storage_dir"].name == "storage" + assert "1 eligible" in r.output + assert "seen=1" in r.output and "attempted=1" in r.output + + +def test_fetch_selection_error_is_a_usage_error(env, zdb: Db): + r = runner.invoke(app, ["fetch", "--db", _db_with_item(zdb), "--no-hold"]) + assert r.exit_code != 0 and "exactly one" in r.output + + +def test_fetch_annas_auth_error_exits_nonzero_with_message(env, zdb: Db, monkeypatch): + def fake_run(db, refs, **kw): + raise AnnasAuthError("Anna's Archive refused the key (403)") + + monkeypatch.setattr("cli.zot.run", fake_run) + r = runner.invoke(app, ["fetch", "--all", "--db", _db_with_item(zdb), "--no-hold"]) + assert r.exit_code != 0 and "refused the key" in r.output + + +def test_fetch_without_annas_key_prints_note(env, zdb: Db, monkeypatch): + monkeypatch.delenv("ANNAS_ARCHIVE_KEY") + monkeypatch.setattr("cli.zot.run", lambda db, refs, **kw: dict.fromkeys(STATS, 0)) + r = runner.invoke(app, ["fetch", "--all", "--db", _db_with_item(zdb), "--no-hold"]) + assert r.exit_code == 0 and "ANNAS_ARCHIVE_KEY not set" in r.output