5261032a9b
Cover remaining uncovered lines in bcda (client, store, log, pipe/cclf, flatten), bib (sync, spider, translate, ingest, item, store, format, ui), cms (express wrappers, log edge cases), api (base, gitea, rustfs, woodpecker, zotero), bls (table import), and pfs (pragma on race guard). 37,687 statements, 0 missed — 11,061 tests passing.
412 lines
14 KiB
Python
412 lines
14 KiB
Python
"""Tests for bib.ingest — download, extract, attach."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import io
|
|
import json
|
|
import zipfile
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from bib.ingest import (
|
|
attach_file,
|
|
attach_note,
|
|
download_files,
|
|
extract_zip,
|
|
ingest,
|
|
)
|
|
from bib.item import Download
|
|
from bib.store import Store
|
|
|
|
# ── download_files ──────────────────────────────────────────────────
|
|
|
|
|
|
class TestDownloadFiles:
|
|
def test_downloads_to_dest(self, tmp_path: Path) -> None:
|
|
content = b"file content here"
|
|
mock_resp = MagicMock()
|
|
mock_resp.read.return_value = content
|
|
mock_resp.__enter__ = lambda s: s
|
|
mock_resp.__exit__ = MagicMock(return_value=False)
|
|
|
|
with patch("urllib.request.urlopen", return_value=mock_resp):
|
|
paths = download_files(
|
|
["https://example.com/data.csv"],
|
|
tmp_path / "downloads",
|
|
)
|
|
|
|
assert len(paths) == 1
|
|
assert paths[0].name == "data.csv"
|
|
assert paths[0].read_bytes() == content
|
|
|
|
def test_skip_existing(self, tmp_path: Path) -> None:
|
|
dest = tmp_path / "downloads"
|
|
dest.mkdir()
|
|
existing = dest / "data.csv"
|
|
existing.write_bytes(b"old content")
|
|
|
|
paths = download_files(
|
|
["https://example.com/data.csv"],
|
|
dest,
|
|
overwrite=False,
|
|
)
|
|
assert len(paths) == 1
|
|
assert paths[0].read_bytes() == b"old content"
|
|
|
|
def test_overwrite_existing(self, tmp_path: Path) -> None:
|
|
dest = tmp_path / "downloads"
|
|
dest.mkdir()
|
|
existing = dest / "data.csv"
|
|
existing.write_bytes(b"old")
|
|
|
|
new_content = b"new content"
|
|
mock_resp = MagicMock()
|
|
mock_resp.read.return_value = new_content
|
|
mock_resp.__enter__ = lambda s: s
|
|
mock_resp.__exit__ = MagicMock(return_value=False)
|
|
|
|
with patch("urllib.request.urlopen", return_value=mock_resp):
|
|
paths = download_files(
|
|
["https://example.com/data.csv"],
|
|
dest,
|
|
overwrite=True,
|
|
)
|
|
|
|
assert paths[0].read_bytes() == new_content
|
|
|
|
def test_multiple_files(self, tmp_path: Path) -> None:
|
|
mock_resp = MagicMock()
|
|
mock_resp.read.return_value = b"data"
|
|
mock_resp.__enter__ = lambda s: s
|
|
mock_resp.__exit__ = MagicMock(return_value=False)
|
|
|
|
with patch("urllib.request.urlopen", return_value=mock_resp):
|
|
paths = download_files(
|
|
[
|
|
"https://example.com/a.csv",
|
|
"https://example.com/b.csv",
|
|
],
|
|
tmp_path / "dl",
|
|
)
|
|
assert len(paths) == 2
|
|
|
|
def test_creates_dest_dir(self, tmp_path: Path) -> None:
|
|
dest = tmp_path / "nested" / "dir"
|
|
mock_resp = MagicMock()
|
|
mock_resp.read.return_value = b"x"
|
|
mock_resp.__enter__ = lambda s: s
|
|
mock_resp.__exit__ = MagicMock(return_value=False)
|
|
|
|
with patch("urllib.request.urlopen", return_value=mock_resp):
|
|
download_files(["https://example.com/x.csv"], dest)
|
|
assert dest.exists()
|
|
|
|
|
|
# ── extract_zip ─────────────────────────────────────────────────────
|
|
|
|
|
|
class TestExtractZip:
|
|
def test_extracts_files(self, tmp_path: Path) -> None:
|
|
zip_path = tmp_path / "archive.zip"
|
|
buf = io.BytesIO()
|
|
with zipfile.ZipFile(buf, "w") as zf:
|
|
zf.writestr("file1.csv", "a,b,c\n1,2,3")
|
|
zf.writestr("file2.txt", "hello")
|
|
zip_path.write_bytes(buf.getvalue())
|
|
|
|
extracted = extract_zip(zip_path)
|
|
assert len(extracted) == 2
|
|
names = {p.name for p in extracted}
|
|
assert "file1.csv" in names
|
|
assert "file2.txt" in names
|
|
|
|
def test_default_dest(self, tmp_path: Path) -> None:
|
|
zip_path = tmp_path / "data.zip"
|
|
buf = io.BytesIO()
|
|
with zipfile.ZipFile(buf, "w") as zf:
|
|
zf.writestr("f.txt", "x")
|
|
zip_path.write_bytes(buf.getvalue())
|
|
|
|
extracted = extract_zip(zip_path)
|
|
assert extracted[0].parent == tmp_path / "data"
|
|
|
|
def test_custom_dest(self, tmp_path: Path) -> None:
|
|
zip_path = tmp_path / "data.zip"
|
|
buf = io.BytesIO()
|
|
with zipfile.ZipFile(buf, "w") as zf:
|
|
zf.writestr("f.txt", "x")
|
|
zip_path.write_bytes(buf.getvalue())
|
|
|
|
custom = tmp_path / "custom_dir"
|
|
extracted = extract_zip(zip_path, dest=custom)
|
|
assert extracted[0].parent == custom
|
|
|
|
def test_skips_directories(self, tmp_path: Path) -> None:
|
|
zip_path = tmp_path / "withdir.zip"
|
|
buf = io.BytesIO()
|
|
with zipfile.ZipFile(buf, "w") as zf:
|
|
zf.writestr("subdir/", "")
|
|
zf.writestr("subdir/file.csv", "data")
|
|
zip_path.write_bytes(buf.getvalue())
|
|
|
|
extracted = extract_zip(zip_path)
|
|
names = [p.name for p in extracted]
|
|
assert "file.csv" in names
|
|
assert "" not in names
|
|
|
|
|
|
# ── attach_file / attach_note ───────────────────────────────────────
|
|
|
|
|
|
class TestAttachFile:
|
|
def test_delegates_to_store(self) -> None:
|
|
mock_store = MagicMock()
|
|
mock_store.attach_file.return_value = "ATT12345"
|
|
key = attach_file(mock_store, "ITEM1234", Path("/tmp/doc.pdf"), title="My Doc")
|
|
assert key == "ATT12345"
|
|
mock_store.attach_file.assert_called_once_with(
|
|
"ITEM1234", Path("/tmp/doc.pdf"), title="My Doc"
|
|
)
|
|
|
|
|
|
class TestAttachNote:
|
|
def test_delegates_to_store(self) -> None:
|
|
mock_store = MagicMock()
|
|
mock_store.attach_note.return_value = 42
|
|
note_id = attach_note(mock_store, "ITEM1234", "<p>Note</p>", title="Summary")
|
|
assert note_id == 42
|
|
mock_store.attach_note.assert_called_once_with(
|
|
"ITEM1234", "<p>Note</p>", title="Summary"
|
|
)
|
|
|
|
|
|
# ── ingest ──────────────────────────────────────────────────────────
|
|
|
|
|
|
class TestIngest:
|
|
def _make_store_with_download(
|
|
self, tmp_path: Path, file_urls: list[str]
|
|
) -> tuple[Store, str]:
|
|
db = tmp_path / "bib.sqlite"
|
|
s = Store(db, storage_dir=tmp_path / "storage")
|
|
dl = Download(
|
|
title="Test Download",
|
|
file_urls=file_urls,
|
|
url="https://example.com/test-page",
|
|
)
|
|
key = s.create(dl)
|
|
return s, key
|
|
|
|
def test_no_file_urls(self, tmp_path: Path) -> None:
|
|
s, key = self._make_store_with_download(tmp_path, [])
|
|
result = ingest(s, item_key=key, download_dir=tmp_path / "dl")
|
|
assert result == {"files": [], "extracted": [], "attachments": []}
|
|
s.close()
|
|
|
|
def test_download_and_attach(self, tmp_path: Path) -> None:
|
|
s, key = self._make_store_with_download(
|
|
tmp_path, ["https://example.com/file.csv"]
|
|
)
|
|
dl_dir = tmp_path / "dl"
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.read.return_value = b"a,b\n1,2"
|
|
mock_resp.__enter__ = lambda self: self
|
|
mock_resp.__exit__ = MagicMock(return_value=False)
|
|
|
|
with patch("urllib.request.urlopen", return_value=mock_resp):
|
|
result = ingest(
|
|
s,
|
|
item_key=key,
|
|
download_dir=dl_dir,
|
|
attach=True,
|
|
)
|
|
|
|
assert len(result["files"]) == 1
|
|
assert result["files"][0].name == "file.csv"
|
|
assert len(result["attachments"]) == 1
|
|
assert result["extracted"] == []
|
|
s.close()
|
|
|
|
def test_download_with_zip(self, tmp_path: Path) -> None:
|
|
s, key = self._make_store_with_download(
|
|
tmp_path, ["https://example.com/data.zip"]
|
|
)
|
|
dl_dir = tmp_path / "dl"
|
|
|
|
# Create a real zip in memory
|
|
buf = io.BytesIO()
|
|
with zipfile.ZipFile(buf, "w") as zf:
|
|
zf.writestr("inner.csv", "x,y\n1,2")
|
|
zip_bytes = buf.getvalue()
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.read.return_value = zip_bytes
|
|
mock_resp.__enter__ = lambda self: self
|
|
mock_resp.__exit__ = MagicMock(return_value=False)
|
|
|
|
with patch("urllib.request.urlopen", return_value=mock_resp):
|
|
result = ingest(
|
|
s,
|
|
item_key=key,
|
|
download_dir=dl_dir,
|
|
attach=True,
|
|
)
|
|
|
|
assert len(result["files"]) == 1
|
|
assert result["files"][0].name == "inner.csv"
|
|
s.close()
|
|
|
|
def test_no_attach(self, tmp_path: Path) -> None:
|
|
s, key = self._make_store_with_download(
|
|
tmp_path, ["https://example.com/file.csv"]
|
|
)
|
|
dl_dir = tmp_path / "dl"
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.read.return_value = b"data"
|
|
mock_resp.__enter__ = lambda self: self
|
|
mock_resp.__exit__ = MagicMock(return_value=False)
|
|
|
|
with patch("urllib.request.urlopen", return_value=mock_resp):
|
|
result = ingest(
|
|
s,
|
|
item_key=key,
|
|
download_dir=dl_dir,
|
|
attach=False,
|
|
)
|
|
|
|
assert result["attachments"] == []
|
|
s.close()
|
|
|
|
def test_item_without_file_urls_attr(self, tmp_path: Path) -> None:
|
|
"""Test ingest when item is fetched from store (no file_urls attr)
|
|
— falls back to extra_json."""
|
|
db = tmp_path / "bib.sqlite"
|
|
s = Store(db, storage_dir=tmp_path / "storage")
|
|
# Insert a generic Item (not Download) with file_urls in extra_json
|
|
from bib.item import Source
|
|
|
|
src = Source(title="Test", url="https://example.com/src")
|
|
key = s.create(src)
|
|
# Manually update extra_json to include file_urls
|
|
con = s._con()
|
|
ej = json.dumps({"doc_type": "", "file_urls": []})
|
|
con.execute(
|
|
"UPDATE items SET extra_json = ? WHERE key = ?",
|
|
(ej, key),
|
|
)
|
|
con.commit()
|
|
|
|
result = ingest(s, item_key=key, download_dir=tmp_path / "dl")
|
|
assert result == {"files": [], "extracted": [], "attachments": []}
|
|
s.close()
|
|
|
|
def test_ingest_with_format_skips_non_parseable(self, tmp_path: Path) -> None:
|
|
"""format= branch: mock rex import and verify it handles exceptions."""
|
|
s, key = self._make_store_with_download(
|
|
tmp_path, ["https://example.com/file.pdf"]
|
|
)
|
|
dl_dir = tmp_path / "dl"
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.read.return_value = b"PDF bytes"
|
|
mock_resp.__enter__ = lambda self: self
|
|
mock_resp.__exit__ = MagicMock(return_value=False)
|
|
|
|
with patch("urllib.request.urlopen", return_value=mock_resp):
|
|
result = ingest(
|
|
s,
|
|
item_key=key,
|
|
download_dir=dl_dir,
|
|
attach=True,
|
|
)
|
|
# PDF is not in the parseable extensions, so extracted is empty
|
|
assert result["extracted"] == []
|
|
s.close()
|
|
|
|
def test_ingest_with_format_rex_exception(self, tmp_path: Path) -> None:
|
|
"""Lines 240-241: rex extraction raises exception, silently caught."""
|
|
s, key = self._make_store_with_download(
|
|
tmp_path, ["https://example.com/file.csv"]
|
|
)
|
|
dl_dir = tmp_path / "dl"
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.read.return_value = b"a,b\n1,2"
|
|
mock_resp.__enter__ = lambda self: self
|
|
mock_resp.__exit__ = MagicMock(return_value=False)
|
|
|
|
mock_rex_extract = MagicMock(side_effect=ValueError("parse error"))
|
|
mock_pfs_files = MagicMock()
|
|
|
|
import sys
|
|
|
|
with (
|
|
patch("urllib.request.urlopen", return_value=mock_resp),
|
|
patch.dict(
|
|
sys.modules,
|
|
{
|
|
"pfs.files": mock_pfs_files,
|
|
"rex": MagicMock(extract=mock_rex_extract),
|
|
},
|
|
),
|
|
):
|
|
result = ingest(
|
|
s,
|
|
item_key=key,
|
|
download_dir=dl_dir,
|
|
format="pfs_rvu",
|
|
attach=True,
|
|
)
|
|
|
|
# Exception was caught, so extracted is empty
|
|
assert result["extracted"] == []
|
|
s.close()
|
|
|
|
def test_ingest_with_format_and_csv(self, tmp_path: Path) -> None:
|
|
"""Test format= branch with CSV file and mocked rex."""
|
|
s, key = self._make_store_with_download(
|
|
tmp_path, ["https://example.com/file.csv"]
|
|
)
|
|
dl_dir = tmp_path / "dl"
|
|
|
|
mock_resp = MagicMock()
|
|
mock_resp.read.return_value = b"a,b\n1,2\n3,4"
|
|
mock_resp.__enter__ = lambda self: self
|
|
mock_resp.__exit__ = MagicMock(return_value=False)
|
|
|
|
# Create mock DataFrame with .shape attribute
|
|
mock_df = MagicMock()
|
|
mock_df.shape = (2, 2)
|
|
|
|
mock_rex_extract = MagicMock(return_value=mock_df)
|
|
mock_pfs_files = MagicMock()
|
|
|
|
import sys
|
|
|
|
with (
|
|
patch("urllib.request.urlopen", return_value=mock_resp),
|
|
patch.dict(sys.modules, {"pfs.files": mock_pfs_files, "rex": MagicMock()}),
|
|
patch("rex.extract", mock_rex_extract, create=True),
|
|
):
|
|
# We need to patch at the point of import
|
|
with patch.dict(
|
|
sys.modules,
|
|
{
|
|
"pfs.files": mock_pfs_files,
|
|
"rex": MagicMock(extract=mock_rex_extract),
|
|
},
|
|
):
|
|
result = ingest(
|
|
s,
|
|
item_key=key,
|
|
download_dir=dl_dir,
|
|
format="pfs_rvu",
|
|
attach=True,
|
|
)
|
|
|
|
assert len(result["extracted"]) == 1
|
|
assert result["extracted"][0]["file"] == "file.csv"
|
|
s.close()
|