From f149c6e984353d9d1fb0ce950e77dbe93927555e Mon Sep 17 00:00:00 2001 From: kert Date: Wed, 23 Sep 2026 21:00:36 -0400 Subject: [PATCH] =?UTF-8?q?feat(zot):=20fetch=20ledger=20=E2=80=94=20JSONL?= =?UTF-8?q?=20attempts=20with=20the=2030-day=20retry=20rule=20(refs=20spec?= =?UTF-8?q?=202026-09-09=20Z1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5.1 --- src/zot/fetch/__init__.py | 4 ++ src/zot/fetch/ledger.py | 95 +++++++++++++++++++++++++++++++ tests/zot/fetch/__init__.py | 0 tests/zot/fetch/conftest.py | 50 ++++++++++++++++ tests/zot/fetch/test_ledger.py | 101 +++++++++++++++++++++++++++++++++ 5 files changed, 250 insertions(+) create mode 100644 src/zot/fetch/__init__.py create mode 100644 src/zot/fetch/ledger.py create mode 100644 tests/zot/fetch/__init__.py create mode 100644 tests/zot/fetch/conftest.py create mode 100644 tests/zot/fetch/test_ledger.py diff --git a/src/zot/fetch/__init__.py b/src/zot/fetch/__init__.py new file mode 100644 index 0000000..b258baa --- /dev/null +++ b/src/zot/fetch/__init__.py @@ -0,0 +1,4 @@ +"""Full-text connector for Zotero items (spec 2026-09-09, slice Z1). + +select → resolve → deliver → attach, with a JSONL ledger of attempts. +""" diff --git a/src/zot/fetch/ledger.py b/src/zot/fetch/ledger.py new file mode 100644 index 0000000..e5b9b00 --- /dev/null +++ b/src/zot/fetch/ledger.py @@ -0,0 +1,95 @@ +"""JSONL ledger of fetch attempts — the report the operator reads. + +One line per attempt. ``should_try`` keeps a terminal miss +(``no_candidate`` / ``all_failed``) from being retried for 30 days, +never retries an ``attached`` item, and always retries ``quota``. +""" + +from __future__ import annotations + +import json +from collections import Counter +from dataclasses import asdict, dataclass, field +from datetime import datetime, timedelta, timezone +from pathlib import Path + +TERMINAL = {"no_candidate", "all_failed"} +RETRY_AFTER = timedelta(days=30) + + +@dataclass(frozen=True) +class Attempt: + item_key: str + outcome: str + source: str = "" + md5: str = "" + ids: dict[str, str] = field(default_factory=dict) + note: str = "" + ts: str = "" + + @property + def when(self) -> datetime: + return datetime.fromisoformat(self.ts) + + +def _now_iso(now: datetime | None) -> str: + return (now or datetime.now(timezone.utc)).isoformat() + + +class Ledger: + def __init__(self, path: Path) -> None: + self.path = Path(path) + self._last: dict[str, Attempt] = {} + self._session: list[Attempt] = [] + if self.path.is_file(): + for line in self.path.read_text().splitlines(): + try: + att = Attempt(**json.loads(line)) + except (ValueError, TypeError): + continue + self._last[att.item_key] = att + + def record( + self, + item_key: str, + *, + outcome: str, + source: str = "", + md5: str = "", + ids: dict[str, str] | None = None, + note: str = "", + now: datetime | None = None, + ) -> Attempt: + att = Attempt( + item_key=item_key, + outcome=outcome, + source=source, + md5=md5, + ids=dict(ids or {}), + note=note, + ts=_now_iso(now), + ) + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(asdict(att), ensure_ascii=False) + "\n") + self._last[item_key] = att + self._session.append(att) + return att + + def should_try( + self, item_key: str, *, retry: bool = False, now: datetime | None = None + ) -> bool: + if retry: + return True + last = self._last.get(item_key) + if last is None: + return True + if last.outcome == "attached": + return False + if last.outcome in TERMINAL: + current = now or datetime.now(timezone.utc) + return current - last.when >= RETRY_AFTER + return True + + def summary(self) -> dict[str, int]: + return dict(Counter(a.outcome for a in self._session)) diff --git a/tests/zot/fetch/__init__.py b/tests/zot/fetch/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/zot/fetch/conftest.py b/tests/zot/fetch/conftest.py new file mode 100644 index 0000000..612a220 --- /dev/null +++ b/tests/zot/fetch/conftest.py @@ -0,0 +1,50 @@ +"""Scratch Zotero DB for the fetch tests (no live DB, no network).""" + +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest + +from tests.zot.test_db import ZOTERO_SCHEMA, _seed_schema_maps +from zot.db import TYPE_MAP, Db + +EXTRA_SCHEMA = """ +CREATE TABLE IF NOT EXISTS deletedItems ( + itemID INTEGER PRIMARY KEY, + dateDeleted TEXT DEFAULT CURRENT_TIMESTAMP NOT NULL +); +""" + + +@pytest.fixture() +def zdb(tmp_path: Path) -> Db: + path = str(tmp_path / "zotero.sqlite") + con = sqlite3.connect(path) + con.executescript(ZOTERO_SCHEMA) + con.executescript(EXTRA_SCHEMA) + _seed_schema_maps(con) + con.commit() + con.close() + d = Db(path) + yield d + d.close() + + +def mk_item( + db: Db, + *, + type_name: str = "journalArticle", + title: str = "", + doi: str = "", + isbn: str = "", + date: str = "", + extra: str = "", +) -> int: + item_id = db.create_item(TYPE_MAP[type_name]) + db.set_fields( + item_id, + {"title": title, "DOI": doi, "ISBN": isbn, "date": date, "extra": extra}, + ) + return item_id diff --git a/tests/zot/fetch/test_ledger.py b/tests/zot/fetch/test_ledger.py new file mode 100644 index 0000000..18ed58b --- /dev/null +++ b/tests/zot/fetch/test_ledger.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import json +from datetime import datetime, timedelta, timezone +from pathlib import Path + +from zot.fetch.ledger import RETRY_AFTER, TERMINAL, Attempt, Ledger + +T0 = datetime(2026, 9, 22, 12, 0, tzinfo=timezone.utc) + + +def test_record_appends_one_json_line(tmp_path: Path): + led = Ledger(tmp_path / "ledger.jsonl") + att = led.record( + "ABCD2345", + outcome="attached", + source="libgen", + md5="a" * 32, + ids={"doi": "10.1/x"}, + now=T0, + ) + assert isinstance(att, Attempt) + lines = (tmp_path / "ledger.jsonl").read_text().splitlines() + assert len(lines) == 1 + row = json.loads(lines[0]) + assert row == { + "item_key": "ABCD2345", + "outcome": "attached", + "source": "libgen", + "md5": "a" * 32, + "ids": {"doi": "10.1/x"}, + "note": "", + "ts": "2026-09-22T12:00:00+00:00", + } + + +def test_unknown_item_should_try(tmp_path: Path): + led = Ledger(tmp_path / "ledger.jsonl") + assert led.should_try("NEW00000") is True + + +def test_attached_item_never_retried(tmp_path: Path): + led = Ledger(tmp_path / "ledger.jsonl") + led.record("K1", outcome="attached", source="unpaywall", now=T0) + assert led.should_try("K1", now=T0 + timedelta(days=400)) is False + + +def test_terminal_miss_waits_thirty_days(tmp_path: Path): + led = Ledger(tmp_path / "ledger.jsonl") + for outcome in TERMINAL: + led.record("K1", outcome=outcome, now=T0) + early = T0 + RETRY_AFTER - timedelta(seconds=1) + assert led.should_try("K1", now=early) is False + assert led.should_try("K1", now=T0 + RETRY_AFTER) is True + + +def test_retry_flag_overrides_terminal_miss(tmp_path: Path): + led = Ledger(tmp_path / "ledger.jsonl") + led.record("K1", outcome="no_candidate", now=T0) + assert led.should_try("K1", retry=True, now=T0) is True + + +def test_quota_miss_retries_immediately(tmp_path: Path): + led = Ledger(tmp_path / "ledger.jsonl") + led.record("K1", outcome="quota", now=T0) + assert led.should_try("K1", now=T0) is True + + +def test_ledger_reloads_last_attempt_per_key(tmp_path: Path): + p = tmp_path / "ledger.jsonl" + Ledger(p).record("K1", outcome="no_candidate", now=T0) + Ledger(p).record("K1", outcome="attached", now=T0) + led = Ledger(p) + assert led.should_try("K1", now=T0 + timedelta(days=60)) is False + + +def test_ledger_skips_corrupt_lines(tmp_path: Path): + p = tmp_path / "ledger.jsonl" + good = ( + '{"item_key": "K1", "outcome": "attached", "ts": "2026-09-22T12:00:00+00:00"}' + ) + p.write_text(good + "\nnot json\n") + led = Ledger(p) + assert led.should_try("K1") is False + assert led.should_try("K2") is True + + +def test_summary_counts_this_session_only(tmp_path: Path): + p = tmp_path / "ledger.jsonl" + Ledger(p).record("K0", outcome="attached", now=T0) + led = Ledger(p) + led.record("K1", outcome="attached", now=T0) + led.record("K2", outcome="no_candidate", now=T0) + led.record("K3", outcome="no_candidate", now=T0) + assert led.summary() == {"attached": 1, "no_candidate": 2} + + +def test_default_now_is_utc_iso(tmp_path: Path): + led = Ledger(tmp_path / "ledger.jsonl") + att = led.record("K1", outcome="attached") + assert att.ts.endswith("+00:00")