feat(zot): stack zot fetch — full-text connector CLI under the --hold guard, with docs (refs Z1)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

docs/docs/cli/zot-fetch.md

@@ -0,0 +1,35 @@
+---
+title: stack zot fetch
+sidebar_position: 87
+---
+
+# `stack zot fetch`
+
+```
+Usage: stack zot fetch [OPTIONS]                                               
+                                                                                
+ Attach full text (PDF/EPUB) to items that lack it.                             
+                                                                                
+ DOI → Unpaywall, then the libgen index; ISBN → libgen; delivery by             
+ libgen or Anna's fast download (member key). Items with a PDF/EPUB,            
+ deleted items and ``document`` items are always skipped. Every                 
+ attempt is a line in the ledger; misses wait 30 days unless --retry.           
+                                                                                
+╭─ Options ────────────────────────────────────────────────────────────────────╮
+│ --collection                 TEXT     Collection name.                       │
+│ --tag                        TEXT     Tag name.                              │
+│ --type                       TEXT     Item type, e.g. journalArticle, book.  │
+│ --item                       TEXT     One item key.                          │
+│ --all                                 Every eligible item.                   │
+│ --limit                      INTEGER  Stop after N attempts (0 = all).       │
+│                                       [default: 0]                           │
+│ --dry-run                             List candidates, write nothing.        │
+│ --retry                               Retry items the ledger marked as       │
+│                                       misses.                                │
+│ --db                         PATH                                            │
+│ --ledger                     PATH     [default:                              │
+│                                       data/zotero/fetch_ledger.jsonl]        │
+│ --hold          --no-hold             [default: hold]                        │
+│ --help                                Show this message and exit.            │
+╰──────────────────────────────────────────────────────────────────────────────╯
+```

docs/docs/cli/zot.md

@@ -21,5 +21,6 @@ Usage: stack zot [OPTIONS] COMMAND [ARGS]...
 │ fix-fields     Remap base-field rows to type-specific fieldIDs; drop         │
 │                orphans.                                                      │
 │ verify-parity  Check that ``zot.db`` constants match the live Zotero schema. │
+│ fetch          Attach full text (PDF/EPUB) to items that lack it.            │
 ╰──────────────────────────────────────────────────────────────────────────────╯
 ```

docs/superpowers/specs/2026-09-09-zotero-annas-connector-design.md

@@ -1,6 +1,6 @@
 # Zotero full-text connector (Anna's Archive, libgen, Unpaywall) — design
 
-**Status:** draft for operator approval, 2026-09-09.
+**Status:** Z1 approved and built 2026-09-22/23 (plan `docs/superpowers/plans/2026-09-22-zotero-fetch-z1.md`); Z2–Z4 pending.
 **Related:** `dev/scripts/add_ama_coding_to_zotero.py` (PR #697), corwins.media issue #707 (Anna's key), corwins spec `2026-09-09-direct-source-tier-design.md` (egress proxy).
 
 ## Problem

src/cli/zot.py

@@ -7,11 +7,18 @@ they run (SQLite file is WAL-locked by the desktop app). ``--hold``
 
 from __future__ import annotations
 
+import os
 import subprocess
 from pathlib import Path
 
 import typer
 
+from zot.db import Db
+from zot.fetch.deliver import AnnasAuthError, Budget
+from zot.fetch.ledger import Ledger
+from zot.fetch.run import make_clients, run
+from zot.fetch.select import select_items
+
 app = typer.Typer(no_args_is_help=True)
 
 _DEFAULT_DB = Path("data/zotero/data/zotero.sqlite")
@@ -126,3 +133,94 @@ def verify_parity(
 
     with Db(str(_resolve(db_path))):
         typer.echo("parity OK")
+
+
+_DEFAULT_LEDGER = Path("data/zotero/fetch_ledger.jsonl")
+_PROXY_HELP = (
+    "VPN_HTTP_PROXY is not set. libgen and Anna's must go through the gluetun "
+    "HTTP proxy published by corwins.media (docker compose up gluetun → "
+    "127.0.0.1:19035): add VPN_HTTP_PROXY=http://127.0.0.1:19035 to .env and "
+    "source it (set -a; . ./.env; set +a) before running."
+)
+
+
+@app.command(name="fetch")
+def fetch(
+    collection: str = typer.Option("", "--collection", help="Collection name."),
+    tag: str = typer.Option("", "--tag", help="Tag name."),
+    type_: str = typer.Option(
+        "", "--type", help="Item type, e.g. journalArticle, book."
+    ),
+    item: str = typer.Option("", "--item", help="One item key."),
+    all_: bool = typer.Option(False, "--all", help="Every eligible item."),
+    limit: int = typer.Option(0, "--limit", help="Stop after N attempts (0 = all)."),
+    dry_run: bool = typer.Option(
+        False, "--dry-run", help="List candidates, write nothing."
+    ),
+    retry: bool = typer.Option(
+        False, "--retry", help="Retry items the ledger marked as misses."
+    ),
+    db_path: Path | None = typer.Option(None, "--db"),
+    ledger_path: Path = typer.Option(_DEFAULT_LEDGER, "--ledger"),
+    hold: bool = typer.Option(True, "--hold/--no-hold"),
+) -> None:
+    """Attach full text (PDF/EPUB) to items that lack it.
+
+    DOI → Unpaywall, then the libgen index; ISBN → libgen; delivery by
+    libgen or Anna's fast download (member key). Items with a PDF/EPUB,
+    deleted items and ``document`` items are always skipped. Every
+    attempt is a line in the ledger; misses wait 30 days unless --retry.
+    """
+    from conf import path as conf_path
+
+    proxy = os.environ.get("VPN_HTTP_PROXY", "")
+    if not proxy:
+        raise typer.BadParameter(_PROXY_HELP)
+    annas_key = os.environ.get("ANNAS_ARCHIVE_KEY", "")
+    if not annas_key:
+        typer.echo(
+            "ANNAS_ARCHIVE_KEY not set — Anna's fast download disabled for this run."
+        )
+    email = os.environ.get("UNPAYWALL_EMAIL", "dev@fhirworx.io")
+    reserve = int(os.environ.get("ANNAS_DAILY_RESERVE", "5"))
+    path = _resolve(db_path)
+
+    def _go() -> dict[str, int]:
+        with Db(str(path)) as db:
+            try:
+                refs = select_items(
+                    db,
+                    collection=collection,
+                    tag=tag,
+                    type_=type_,
+                    item=item,
+                    all_=all_,
+                )
+            except ValueError as exc:
+                raise typer.BadParameter(str(exc)) from exc
+            typer.echo(f"{len(refs)} eligible item(s)")
+            clients = make_clients(proxy, email=email)
+            try:
+                return run(
+                    db,
+                    refs,
+                    ledger=Ledger(ledger_path),
+                    budget=Budget(annas_key=annas_key, reserve=reserve),
+                    clients=clients,
+                    email=email,
+                    storage_dir=conf_path("storage.zotero"),
+                    scratch_dir=Path(".state/zot/fetch"),
+                    limit=limit,
+                    dry_run=dry_run,
+                    retry=retry,
+                    out=typer.echo,
+                )
+            except AnnasAuthError as exc:
+                typer.echo(f"error: {exc}", err=True)
+                raise typer.Exit(2) from exc
+            finally:
+                clients.direct.close()
+                clients.proxied.close()
+
+    stats = _hold_zotero(_go, hold)
+    typer.echo("fetch: " + " ".join(f"{k}={v}" for k, v in stats.items()))

tests/cli/conftest.py

@@ -0,0 +1,3 @@
+"""Shared fixtures for CLI tests."""
+
+from tests.zot.fetch.conftest import zdb  # noqa: F401 — scratch Zotero DB

tests/cli/test_zot_fetch.py

@@ -0,0 +1,97 @@
+"""stack zot fetch — CLI wiring (no network, no docker)."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+from typer.testing import CliRunner
+
+from cli.zot import app
+from tests.zot.fetch.conftest import mk_item
+from zot.db import Db
+from zot.fetch.deliver import AnnasAuthError
+from zot.fetch.run import STATS
+
+runner = CliRunner()
+
+
+@pytest.fixture()
+def env(monkeypatch, tmp_path: Path):
+    monkeypatch.setenv("VPN_HTTP_PROXY", "http://127.0.0.1:19035")
+    monkeypatch.setenv("ANNAS_ARCHIVE_KEY", "k")
+    monkeypatch.setenv("ANNAS_DAILY_RESERVE", "7")
+    monkeypatch.setenv("UNPAYWALL_EMAIL", "dev@fhirworx.io")
+    monkeypatch.setattr("cli.zot._hold_zotero", lambda fn, hold: fn())
+    return tmp_path
+
+
+def _db_with_item(zdb: Db) -> str:
+    mk_item(zdb, title="T", doi="10.1/x")
+    zdb.commit()
+    return zdb.path
+
+
+def test_fetch_requires_proxy(monkeypatch, zdb: Db):
+    monkeypatch.delenv("VPN_HTTP_PROXY", raising=False)
+    monkeypatch.setattr("cli.zot._hold_zotero", lambda fn, hold: fn())
+    r = runner.invoke(app, ["fetch", "--all", "--db", _db_with_item(zdb), "--no-hold"])
+    assert r.exit_code != 0
+    assert "VPN_HTTP_PROXY" in r.output and "gluetun" in r.output
+
+
+def test_fetch_dry_run_calls_run_with_selection(env, zdb: Db, monkeypatch):
+    captured: dict = {}
+
+    def fake_run(db, refs, **kw):
+        captured["refs"] = refs
+        captured["kw"] = kw
+        return dict.fromkeys(STATS, 0) | {"seen": 1, "attempted": 1}
+
+    monkeypatch.setattr("cli.zot.run", fake_run)
+    r = runner.invoke(
+        app,
+        [
+            "fetch",
+            "--all",
+            "--dry-run",
+            "--limit",
+            "3",
+            "--db",
+            _db_with_item(zdb),
+            "--ledger",
+            str(env / "ledger.jsonl"),
+            "--no-hold",
+        ],
+    )
+    assert r.exit_code == 0, r.output
+    assert [x.doi for x in captured["refs"]] == ["10.1/x"]
+    kw = captured["kw"]
+    assert kw["dry_run"] is True and kw["limit"] == 3 and kw["retry"] is False
+    assert kw["budget"].annas_key == "k" and kw["budget"].reserve == 7
+    assert kw["email"] == "dev@fhirworx.io"
+    assert kw["ledger"].path == env / "ledger.jsonl"
+    assert kw["storage_dir"].name == "storage"
+    assert "1 eligible" in r.output
+    assert "seen=1" in r.output and "attempted=1" in r.output
+
+
+def test_fetch_selection_error_is_a_usage_error(env, zdb: Db):
+    r = runner.invoke(app, ["fetch", "--db", _db_with_item(zdb), "--no-hold"])
+    assert r.exit_code != 0 and "exactly one" in r.output
+
+
+def test_fetch_annas_auth_error_exits_nonzero_with_message(env, zdb: Db, monkeypatch):
+    def fake_run(db, refs, **kw):
+        raise AnnasAuthError("Anna's Archive refused the key (403)")
+
+    monkeypatch.setattr("cli.zot.run", fake_run)
+    r = runner.invoke(app, ["fetch", "--all", "--db", _db_with_item(zdb), "--no-hold"])
+    assert r.exit_code != 0 and "refused the key" in r.output
+
+
+def test_fetch_without_annas_key_prints_note(env, zdb: Db, monkeypatch):
+    monkeypatch.delenv("ANNAS_ARCHIVE_KEY")
+    monkeypatch.setattr("cli.zot.run", lambda db, refs, **kw: dict.fromkeys(STATS, 0))
+    r = runner.invoke(app, ["fetch", "--all", "--db", _db_with_item(zdb), "--no-hold"])
+    assert r.exit_code == 0 and "ANNAS_ARCHIVE_KEY not set" in r.output