118 lines
4.2 KiB
Python
118 lines
4.2 KiB
Python
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import sqlite3
|
|
import sys
|
|
from importlib import resources
|
|
from pathlib import Path
|
|
|
|
from bib.item import Source
|
|
from bib.store import Store
|
|
|
|
_SCRIPT = (
|
|
Path(__file__).resolve().parents[2] / "dev" / "scripts" / "dedupe_attachments.py"
|
|
)
|
|
spec = importlib.util.spec_from_file_location("dedupe_attachments", _SCRIPT)
|
|
mod = importlib.util.module_from_spec(spec)
|
|
sys.modules["dedupe_attachments"] = mod
|
|
spec.loader.exec_module(mod)
|
|
|
|
|
|
def _seed(tmp_path: Path) -> tuple[Store, str]:
|
|
# Seed the item + duplicate attachment rows via a raw connection,
|
|
# *before* any Store ever opens this file. Store's schema guard adds
|
|
# the (item_id, filename) unique index the moment it sees zero
|
|
# duplicate groups — true the instant the attachments table exists —
|
|
# so seeding duplicates through a Store-opened connection can never
|
|
# succeed. This mirrors the real migration scenario: the duplicates
|
|
# were written by pre-Task-4 code that predates this guard.
|
|
db_path = tmp_path / "bib.sqlite"
|
|
storage = tmp_path / "storage"
|
|
ddl = resources.files("bib").joinpath("schema.sql").read_text()
|
|
raw = sqlite3.connect(str(db_path))
|
|
raw.executescript(ddl)
|
|
item = Source(title="T", url="https://x/1")
|
|
item.stamp_access()
|
|
row = item.to_row()
|
|
key = row["key"] or "SEEDKEY1"
|
|
row["key"] = key
|
|
raw.execute(
|
|
"""INSERT INTO items
|
|
(key, item_type, title, url, date_published,
|
|
access_date, abstract, institution, extra, extra_json)
|
|
VALUES (:key, :item_type, :title, :url,
|
|
:date_published, :access_date, :abstract,
|
|
:institution, :extra, :extra_json)""",
|
|
row,
|
|
)
|
|
item_id = raw.execute("SELECT id FROM items WHERE key=?", (key,)).fetchone()[0]
|
|
# Simulate the old non-idempotent attach: three rows, three copies.
|
|
for k in ("AAAAAAAA", "BBBBBBBB", "CCCCCCCC"):
|
|
d = storage / k
|
|
d.mkdir(parents=True)
|
|
(d / "attachment_1.pdf").write_bytes(b"%PDF-dup")
|
|
raw.execute(
|
|
"INSERT INTO attachments (item_id, key, filename, content_type, storage_path) VALUES (?,?,?,?,?)",
|
|
(
|
|
item_id,
|
|
k,
|
|
"attachment_1.pdf",
|
|
"application/pdf",
|
|
str(d / "attachment_1.pdf"),
|
|
),
|
|
)
|
|
raw.commit()
|
|
raw.close()
|
|
s = Store(str(db_path), storage_dir=storage)
|
|
return s, key
|
|
|
|
|
|
def test_plan_finds_group_and_keeps_oldest(tmp_path: Path):
|
|
s, _ = _seed(tmp_path)
|
|
groups = mod.plan(s._con())
|
|
assert len(groups) == 1
|
|
g = groups[0]
|
|
assert g.keep == "AAAAAAAA"
|
|
assert sorted(g.remove) == ["BBBBBBBB", "CCCCCCCC"]
|
|
|
|
|
|
def test_apply_removes_rows_and_files(tmp_path: Path):
|
|
s, _ = _seed(tmp_path)
|
|
rep = mod.apply(s._con(), mod.plan(s._con()))
|
|
assert rep.rows_removed == 2
|
|
assert rep.files_removed == 2
|
|
assert s._con().execute("SELECT count(*) FROM attachments").fetchone()[0] == 1
|
|
assert (tmp_path / "storage" / "AAAAAAAA" / "attachment_1.pdf").is_file()
|
|
assert not (tmp_path / "storage" / "BBBBBBBB").exists()
|
|
assert mod.plan(s._con()) == []
|
|
|
|
|
|
def test_refuses_group_with_differing_sizes(tmp_path: Path):
|
|
s, _ = _seed(tmp_path)
|
|
(tmp_path / "storage" / "CCCCCCCC" / "attachment_1.pdf").write_bytes(
|
|
b"different-longer"
|
|
)
|
|
groups = mod.plan(s._con())
|
|
assert groups[0].conflict is True
|
|
rep = mod.apply(s._con(), groups)
|
|
assert rep.rows_removed == 0 and rep.skipped_conflicts == 1
|
|
|
|
|
|
def test_unique_index_created_only_when_clean(tmp_path: Path):
|
|
s, _ = _seed(tmp_path)
|
|
s.close()
|
|
s2 = Store(str(tmp_path / "bib.sqlite"), storage_dir=tmp_path / "storage")
|
|
idx = {
|
|
r[0]
|
|
for r in s2._con().execute("SELECT name FROM sqlite_master WHERE type='index'")
|
|
}
|
|
assert "idx_attachments_item_filename" not in idx
|
|
mod.apply(s2._con(), mod.plan(s2._con()))
|
|
s2.close()
|
|
s3 = Store(str(tmp_path / "bib.sqlite"), storage_dir=tmp_path / "storage")
|
|
idx = {
|
|
r[0]
|
|
for r in s3._con().execute("SELECT name FROM sqlite_master WHERE type='index'")
|
|
}
|
|
assert "idx_attachments_item_filename" in idx
|