Files
stack/tests/zot/fetch/test_attach.py
kert d821ccc964
Some checks failed
CI / lint (pull_request) Successful in 45s
CI / notebooks-smoke (pull_request) Successful in 1m58s
CI / test (pull_request) Successful in 2m55s
CI / lint (push) Successful in 45s
CI / notebooks-smoke (push) Successful in 1m43s
CI / test (push) Successful in 2m38s
Infra CI / zotero (pull_request) Successful in 15s
Infra CI / notebooks (pull_request) Successful in 1m5s
Infra CI / api (pull_request) Successful in 1m12s
Infra CI / docs (pull_request) Successful in 2m1s
Infra CI / mc (pull_request) Failing after 44s
Infra CI / llm (pull_request) Successful in 1m5s
Infra CI / zotero (push) Successful in 22s
Infra CI / notebooks (push) Successful in 1m14s
Infra CI / llm (push) Successful in 19s
Infra CI / docs (push) Successful in 15s
Infra CI / api (push) Successful in 24s
Infra CI / mc (push) Failing after 55s
fix(zot): fetch review pass — quota 403 degrades instead of aborting, per-item isolation with rollback and scratch cleanup, ledger notes with an 'error' outcome that retries next run (refs Z1)
- Anna's 403 whose body is not a key/membership error disables Anna's for
  the run (annas_left=0) instead of raising AnnasAuthError.
- run() isolates each item: any exception rolls the transaction back,
  unlinks scratch files, ledgers outcome=error with the exception text,
  and continues; AnnasAuthError still aborts.
- attach_file writes rows before copying so a failed copy rolls back
  cleanly instead of orphaning a storage dir.
- Notes (miss vs error) from resolve/deliver land in every ledger line;
  an item whose lookups/deliveries errored (transport, 5xx, throttle
  page) is 'error' and retried next run rather than a 30-day miss.
- resolve tolerates non-dict file entries and non-numeric filesize.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-23 21:27:19 -04:00

153 lines
4.7 KiB
Python

from __future__ import annotations
from pathlib import Path
import pytest
from tests.zot.fetch.conftest import mk_item
from zot.db import Db
from zot.fetch.attach import (
CONTENT_TYPES,
append_extra,
attach_file,
has_full_text,
sudo_copier,
)
def _py_copier(src: Path, dest_dir: Path) -> None:
dest_dir.mkdir(parents=True, exist_ok=True)
(dest_dir / src.name).write_bytes(src.read_bytes())
def _att(db: Db, parent: int, ctype: str) -> None:
db.add_attachment(parent, content_type=ctype, path="storage:x")
@pytest.mark.parametrize(
"ctypes, expected",
[
([], False),
(["text/html"], False),
(["application/pdf"], True),
(["application/epub+zip"], True),
(["text/html", "application/pdf"], True),
],
)
def test_has_full_text_truth_table(zdb: Db, ctypes, expected):
parent = mk_item(zdb, title="P")
for c in ctypes:
_att(zdb, parent, c)
assert has_full_text(zdb, parent) is expected
def test_has_full_text_ignores_other_parents(zdb: Db):
a = mk_item(zdb, title="A")
b = mk_item(zdb, title="B")
_att(zdb, a, "application/pdf")
assert has_full_text(zdb, b) is False
def test_attach_file_writes_row_tags_and_extra(zdb: Db, tmp_path: Path):
parent = mk_item(zdb, title="Paper", extra="PMID: 1")
src = tmp_path / "Paper.pdf"
src.write_bytes(b"%PDF-1.4 x")
storage = tmp_path / "storage"
att_id = attach_file(
zdb,
parent,
src,
source="libgen",
md5="a" * 32,
storage_dir=storage,
copier=_py_copier,
)
row = zdb.con.execute(
"SELECT parentItemID, linkMode, contentType, path FROM itemAttachments "
"WHERE itemID = ?",
(att_id,),
).fetchone()
assert tuple(row) == (parent, 0, "application/pdf", "storage:Paper.pdf")
assert zdb.get_field(att_id, "title") == "Paper.pdf"
att_key = zdb.get_item(att_id)["key"]
assert (storage / att_key / "Paper.pdf").read_bytes() == b"%PDF-1.4 x"
assert "source:libgen" in zdb.get_tags(parent)
assert zdb.get_field(parent, "extra") == "PMID: 1\nmd5: " + "a" * 32
assert has_full_text(zdb, parent) is True
def test_attach_file_epub_content_type(zdb: Db, tmp_path: Path):
parent = mk_item(zdb, type_name="book", title="Book")
src = tmp_path / "Book.epub"
src.write_bytes(b"PK\x03\x04")
att_id = attach_file(
zdb, parent, src, source="annas", storage_dir=tmp_path / "s", copier=_py_copier
)
ctype = zdb.con.execute(
"SELECT contentType FROM itemAttachments WHERE itemID = ?", (att_id,)
).fetchone()[0]
assert ctype == CONTENT_TYPES[".epub"]
assert zdb.get_field(parent, "extra") is None # no md5 → extra untouched
def test_attach_file_rejects_unknown_extension(zdb: Db, tmp_path: Path):
parent = mk_item(zdb, title="P")
src = tmp_path / "x.mobi"
src.write_bytes(b"x")
with pytest.raises(ValueError, match="mobi"):
attach_file(
zdb, parent, src, source="libgen", storage_dir=tmp_path, copier=_py_copier
)
def test_append_extra_is_idempotent_and_keeps_lines(zdb: Db):
item = mk_item(zdb, title="P", extra="PMID: 1")
append_extra(zdb, item, "md5: abc")
append_extra(zdb, item, "md5: abc")
assert zdb.get_field(item, "extra") == "PMID: 1\nmd5: abc"
def test_append_extra_on_empty_extra(zdb: Db):
item = mk_item(zdb, title="P")
append_extra(zdb, item, "md5: abc")
assert zdb.get_field(item, "extra") == "md5: abc"
def test_sudo_copier_runs_mkdir_cp_chown(monkeypatch, tmp_path: Path):
calls: list[list[str]] = []
def fake_run(cmd, check):
calls.append(cmd)
monkeypatch.setattr("zot.fetch.attach.subprocess.run", fake_run)
sudo_copier(tmp_path / "a.pdf", tmp_path / "storage" / "KEY")
dest = tmp_path / "storage" / "KEY"
assert calls == [
["sudo", "mkdir", "-p", str(dest)],
["sudo", "cp", str(tmp_path / "a.pdf"), str(dest / "a.pdf")],
["sudo", "chown", "-R", "100999:100999", str(dest)],
]
def test_attach_file_writes_rows_before_copying(zdb: Db, tmp_path: Path):
"""A failed copy must leave rows the caller can roll back, never an
orphan storage dir with no row."""
parent = mk_item(zdb, title="P")
src = tmp_path / "P.pdf"
src.write_bytes(b"%PDF-1.4 x")
seen = {}
def spy_copier(s: Path, d: Path) -> None:
seen["rows"] = zdb.con.execute(
"SELECT COUNT(*) FROM itemAttachments WHERE parentItemID = ?", (parent,)
).fetchone()[0]
raise OSError("disk full")
with pytest.raises(OSError):
attach_file(
zdb, parent, src, source="libgen", storage_dir=tmp_path, copier=spy_copier
)
assert seen["rows"] == 1
zdb.con.rollback()
assert has_full_text(zdb, parent) is False