feat(zot): fetch ledger — JSONL attempts with the 30-day retry rule (refs spec 2026-09-09 Z1)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

src/zot/fetch/__init__.py

@@ -0,0 +1,4 @@
+"""Full-text connector for Zotero items (spec 2026-09-09, slice Z1).
+
+select → resolve → deliver → attach, with a JSONL ledger of attempts.
+"""

src/zot/fetch/ledger.py

@@ -0,0 +1,95 @@
+"""JSONL ledger of fetch attempts — the report the operator reads.
+
+One line per attempt. ``should_try`` keeps a terminal miss
+(``no_candidate`` / ``all_failed``) from being retried for 30 days,
+never retries an ``attached`` item, and always retries ``quota``.
+"""
+
+from __future__ import annotations
+
+import json
+from collections import Counter
+from dataclasses import asdict, dataclass, field
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+TERMINAL = {"no_candidate", "all_failed"}
+RETRY_AFTER = timedelta(days=30)
+
+
+@dataclass(frozen=True)
+class Attempt:
+    item_key: str
+    outcome: str
+    source: str = ""
+    md5: str = ""
+    ids: dict[str, str] = field(default_factory=dict)
+    note: str = ""
+    ts: str = ""
+
+    @property
+    def when(self) -> datetime:
+        return datetime.fromisoformat(self.ts)
+
+
+def _now_iso(now: datetime | None) -> str:
+    return (now or datetime.now(timezone.utc)).isoformat()
+
+
+class Ledger:
+    def __init__(self, path: Path) -> None:
+        self.path = Path(path)
+        self._last: dict[str, Attempt] = {}
+        self._session: list[Attempt] = []
+        if self.path.is_file():
+            for line in self.path.read_text().splitlines():
+                try:
+                    att = Attempt(**json.loads(line))
+                except (ValueError, TypeError):
+                    continue
+                self._last[att.item_key] = att
+
+    def record(
+        self,
+        item_key: str,
+        *,
+        outcome: str,
+        source: str = "",
+        md5: str = "",
+        ids: dict[str, str] | None = None,
+        note: str = "",
+        now: datetime | None = None,
+    ) -> Attempt:
+        att = Attempt(
+            item_key=item_key,
+            outcome=outcome,
+            source=source,
+            md5=md5,
+            ids=dict(ids or {}),
+            note=note,
+            ts=_now_iso(now),
+        )
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        with self.path.open("a", encoding="utf-8") as fh:
+            fh.write(json.dumps(asdict(att), ensure_ascii=False) + "\n")
+        self._last[item_key] = att
+        self._session.append(att)
+        return att
+
+    def should_try(
+        self, item_key: str, *, retry: bool = False, now: datetime | None = None
+    ) -> bool:
+        if retry:
+            return True
+        last = self._last.get(item_key)
+        if last is None:
+            return True
+        if last.outcome == "attached":
+            return False
+        if last.outcome in TERMINAL:
+            current = now or datetime.now(timezone.utc)
+            return current - last.when >= RETRY_AFTER
+        return True
+
+    def summary(self) -> dict[str, int]:
+        return dict(Counter(a.outcome for a in self._session))

tests/zot/fetch/conftest.py

@@ -0,0 +1,50 @@
+"""Scratch Zotero DB for the fetch tests (no live DB, no network)."""
+
+from __future__ import annotations
+
+import sqlite3
+from pathlib import Path
+
+import pytest
+
+from tests.zot.test_db import ZOTERO_SCHEMA, _seed_schema_maps
+from zot.db import TYPE_MAP, Db
+
+EXTRA_SCHEMA = """
+CREATE TABLE IF NOT EXISTS deletedItems (
+    itemID INTEGER PRIMARY KEY,
+    dateDeleted TEXT DEFAULT CURRENT_TIMESTAMP NOT NULL
+);
+"""
+
+
+@pytest.fixture()
+def zdb(tmp_path: Path) -> Db:
+    path = str(tmp_path / "zotero.sqlite")
+    con = sqlite3.connect(path)
+    con.executescript(ZOTERO_SCHEMA)
+    con.executescript(EXTRA_SCHEMA)
+    _seed_schema_maps(con)
+    con.commit()
+    con.close()
+    d = Db(path)
+    yield d
+    d.close()
+
+
+def mk_item(
+    db: Db,
+    *,
+    type_name: str = "journalArticle",
+    title: str = "",
+    doi: str = "",
+    isbn: str = "",
+    date: str = "",
+    extra: str = "",
+) -> int:
+    item_id = db.create_item(TYPE_MAP[type_name])
+    db.set_fields(
+        item_id,
+        {"title": title, "DOI": doi, "ISBN": isbn, "date": date, "extra": extra},
+    )
+    return item_id

tests/zot/fetch/test_ledger.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+from zot.fetch.ledger import RETRY_AFTER, TERMINAL, Attempt, Ledger
+
+T0 = datetime(2026, 9, 22, 12, 0, tzinfo=timezone.utc)
+
+
+def test_record_appends_one_json_line(tmp_path: Path):
+    led = Ledger(tmp_path / "ledger.jsonl")
+    att = led.record(
+        "ABCD2345",
+        outcome="attached",
+        source="libgen",
+        md5="a" * 32,
+        ids={"doi": "10.1/x"},
+        now=T0,
+    )
+    assert isinstance(att, Attempt)
+    lines = (tmp_path / "ledger.jsonl").read_text().splitlines()
+    assert len(lines) == 1
+    row = json.loads(lines[0])
+    assert row == {
+        "item_key": "ABCD2345",
+        "outcome": "attached",
+        "source": "libgen",
+        "md5": "a" * 32,
+        "ids": {"doi": "10.1/x"},
+        "note": "",
+        "ts": "2026-09-22T12:00:00+00:00",
+    }
+
+
+def test_unknown_item_should_try(tmp_path: Path):
+    led = Ledger(tmp_path / "ledger.jsonl")
+    assert led.should_try("NEW00000") is True
+
+
+def test_attached_item_never_retried(tmp_path: Path):
+    led = Ledger(tmp_path / "ledger.jsonl")
+    led.record("K1", outcome="attached", source="unpaywall", now=T0)
+    assert led.should_try("K1", now=T0 + timedelta(days=400)) is False
+
+
+def test_terminal_miss_waits_thirty_days(tmp_path: Path):
+    led = Ledger(tmp_path / "ledger.jsonl")
+    for outcome in TERMINAL:
+        led.record("K1", outcome=outcome, now=T0)
+        early = T0 + RETRY_AFTER - timedelta(seconds=1)
+        assert led.should_try("K1", now=early) is False
+        assert led.should_try("K1", now=T0 + RETRY_AFTER) is True
+
+
+def test_retry_flag_overrides_terminal_miss(tmp_path: Path):
+    led = Ledger(tmp_path / "ledger.jsonl")
+    led.record("K1", outcome="no_candidate", now=T0)
+    assert led.should_try("K1", retry=True, now=T0) is True
+
+
+def test_quota_miss_retries_immediately(tmp_path: Path):
+    led = Ledger(tmp_path / "ledger.jsonl")
+    led.record("K1", outcome="quota", now=T0)
+    assert led.should_try("K1", now=T0) is True
+
+
+def test_ledger_reloads_last_attempt_per_key(tmp_path: Path):
+    p = tmp_path / "ledger.jsonl"
+    Ledger(p).record("K1", outcome="no_candidate", now=T0)
+    Ledger(p).record("K1", outcome="attached", now=T0)
+    led = Ledger(p)
+    assert led.should_try("K1", now=T0 + timedelta(days=60)) is False
+
+
+def test_ledger_skips_corrupt_lines(tmp_path: Path):
+    p = tmp_path / "ledger.jsonl"
+    good = (
+        '{"item_key": "K1", "outcome": "attached", "ts": "2026-09-22T12:00:00+00:00"}'
+    )
+    p.write_text(good + "\nnot json\n")
+    led = Ledger(p)
+    assert led.should_try("K1") is False
+    assert led.should_try("K2") is True
+
+
+def test_summary_counts_this_session_only(tmp_path: Path):
+    p = tmp_path / "ledger.jsonl"
+    Ledger(p).record("K0", outcome="attached", now=T0)
+    led = Ledger(p)
+    led.record("K1", outcome="attached", now=T0)
+    led.record("K2", outcome="no_candidate", now=T0)
+    led.record("K3", outcome="no_candidate", now=T0)
+    assert led.summary() == {"attached": 1, "no_candidate": 2}
+
+
+def test_default_now_is_utc_iso(tmp_path: Path):
+    led = Ledger(tmp_path / "ledger.jsonl")
+    att = led.record("K1", outcome="attached")
+    assert att.ts.endswith("+00:00")