stack/tests/dev/test_dedupe_attachments.py

from __future__ import annotations

import importlib.util
import sqlite3
import sys
from importlib import resources
from pathlib import Path

from bib.item import Source
from bib.store import Store

_SCRIPT = (
    Path(__file__).resolve().parents[2] / "dev" / "scripts" / "dedupe_attachments.py"
)
spec = importlib.util.spec_from_file_location("dedupe_attachments", _SCRIPT)
mod = importlib.util.module_from_spec(spec)
sys.modules["dedupe_attachments"] = mod
spec.loader.exec_module(mod)


def _seed(tmp_path: Path) -> tuple[Store, str]:
    # Seed the item + duplicate attachment rows via a raw connection,
    # *before* any Store ever opens this file. Store's schema guard adds
    # the (item_id, filename) unique index the moment it sees zero
    # duplicate groups — true the instant the attachments table exists —
    # so seeding duplicates through a Store-opened connection can never
    # succeed. This mirrors the real migration scenario: the duplicates
    # were written by pre-Task-4 code that predates this guard.
    db_path = tmp_path / "bib.sqlite"
    storage = tmp_path / "storage"
    ddl = resources.files("bib").joinpath("schema.sql").read_text()
    raw = sqlite3.connect(str(db_path))
    raw.executescript(ddl)
    item = Source(title="T", url="https://x/1")
    item.stamp_access()
    row = item.to_row()
    key = row["key"] or "SEEDKEY1"
    row["key"] = key
    raw.execute(
        """INSERT INTO items
           (key, item_type, title, url, date_published,
            access_date, abstract, institution, extra, extra_json)
           VALUES (:key, :item_type, :title, :url,
                   :date_published, :access_date, :abstract,
                   :institution, :extra, :extra_json)""",
        row,
    )
    item_id = raw.execute("SELECT id FROM items WHERE key=?", (key,)).fetchone()[0]
    # Simulate the old non-idempotent attach: three rows, three copies.
    for k in ("AAAAAAAA", "BBBBBBBB", "CCCCCCCC"):
        d = storage / k
        d.mkdir(parents=True)
        (d / "attachment_1.pdf").write_bytes(b"%PDF-dup")
        raw.execute(
            "INSERT INTO attachments (item_id, key, filename, content_type, storage_path) VALUES (?,?,?,?,?)",
            (
                item_id,
                k,
                "attachment_1.pdf",
                "application/pdf",
                str(d / "attachment_1.pdf"),
            ),
        )
    raw.commit()
    raw.close()
    s = Store(str(db_path), storage_dir=storage)
    return s, key


def test_plan_finds_group_and_keeps_oldest(tmp_path: Path):
    s, _ = _seed(tmp_path)
    groups = mod.plan(s._con())
    assert len(groups) == 1
    g = groups[0]
    assert g.keep == "AAAAAAAA"
    assert sorted(g.remove) == ["BBBBBBBB", "CCCCCCCC"]


def test_apply_removes_rows_and_files(tmp_path: Path):
    s, _ = _seed(tmp_path)
    rep = mod.apply(s._con(), mod.plan(s._con()))
    assert rep.rows_removed == 2
    assert rep.files_removed == 2
    assert s._con().execute("SELECT count(*) FROM attachments").fetchone()[0] == 1
    assert (tmp_path / "storage" / "AAAAAAAA" / "attachment_1.pdf").is_file()
    assert not (tmp_path / "storage" / "BBBBBBBB").exists()
    assert mod.plan(s._con()) == []


def test_refuses_group_with_differing_sizes(tmp_path: Path):
    s, _ = _seed(tmp_path)
    (tmp_path / "storage" / "CCCCCCCC" / "attachment_1.pdf").write_bytes(
        b"different-longer"
    )
    groups = mod.plan(s._con())
    assert groups[0].conflict is True
    rep = mod.apply(s._con(), groups)
    assert rep.rows_removed == 0 and rep.skipped_conflicts == 1


def test_conflict_when_kept_file_is_missing(tmp_path: Path):
    """refs #680: the kept row's file being gone (e.g. an earlier
    partial cleanup, or a manual delete) must not be treated as "no
    conflict" just because a missing file's _size() sentinel (-1)
    happens to also mean "ignore mismatches" for individual dupes — a
    duplicate with real bytes on disk while the kept copy has none is
    exactly the size-mismatch case this guard exists for."""
    s, _ = _seed(tmp_path)
    (tmp_path / "storage" / "AAAAAAAA" / "attachment_1.pdf").unlink()
    groups = mod.plan(s._con())
    assert len(groups) == 1
    assert groups[0].conflict is True
    rep = mod.apply(s._con(), groups)
    assert rep.rows_removed == 0
    assert rep.skipped_conflicts == 1
    # nothing touched: all three rows and both surviving files remain
    assert s._con().execute("SELECT count(*) FROM attachments").fetchone()[0] == 3
    assert (tmp_path / "storage" / "BBBBBBBB" / "attachment_1.pdf").is_file()
    assert (tmp_path / "storage" / "CCCCCCCC" / "attachment_1.pdf").is_file()


def test_unique_index_created_only_when_clean(tmp_path: Path):
    s, _ = _seed(tmp_path)
    s.close()
    s2 = Store(str(tmp_path / "bib.sqlite"), storage_dir=tmp_path / "storage")
    idx = {
        r[0]
        for r in s2._con().execute("SELECT name FROM sqlite_master WHERE type='index'")
    }
    assert "idx_attachments_item_filename" not in idx
    mod.apply(s2._con(), mod.plan(s2._con()))
    s2.close()
    s3 = Store(str(tmp_path / "bib.sqlite"), storage_dir=tmp_path / "storage")
    idx = {
        r[0]
        for r in s3._con().execute("SELECT name FROM sqlite_master WHERE type='index'")
    }
    assert "idx_attachments_item_filename" in idx


class _ConnSpy:
    """Wraps a real sqlite3.Connection to record GROUP BY calls.

    ``sqlite3.Connection`` is a C type and can't be monkeypatched
    in-place, so this proxies everything through ``__getattr__`` except
    ``execute`` (recorded) and attribute assignment (forwarded — needed
    for ``row_factory``, which ``Store._con`` sets directly)."""

    def __init__(self, real: sqlite3.Connection) -> None:
        object.__setattr__(self, "_real", real)
        object.__setattr__(self, "group_by_calls", [])

    def execute(self, sql, *args, **kwargs):
        if "GROUP BY item_id, filename" in sql:
            self.group_by_calls.append(sql)
        return self._real.execute(sql, *args, **kwargs)

    def __getattr__(self, name):
        return getattr(self._real, name)

    def __setattr__(self, name, value):
        setattr(self._real, name, value)


def test_reopen_skips_group_by_scan_once_indexed(tmp_path: Path, monkeypatch):
    """refs #680: Store._init_schema short-circuits via sqlite_master
    once the unique index exists — a re-open must not re-run the
    duplicate-attachments GROUP BY scan at all."""
    s, _ = _seed(tmp_path)
    mod.apply(s._con(), mod.plan(s._con()))
    s.close()

    # First reopen creates the unique index (no duplicates left).
    s_setup = Store(str(tmp_path / "bib.sqlite"), storage_dir=tmp_path / "storage")
    s_setup.close()

    import bib.store as store_mod

    real_connect = store_mod.sqlite3.connect
    spies: list[_ConnSpy] = []

    def fake_connect(path, *args, **kwargs):
        spy = _ConnSpy(real_connect(path, *args, **kwargs))
        spies.append(spy)
        return spy

    monkeypatch.setattr(store_mod.sqlite3, "connect", fake_connect)

    # A further reopen must skip the GROUP BY scan now that the index
    # already exists.
    s2 = Store(str(tmp_path / "bib.sqlite"), storage_dir=tmp_path / "storage")
    s2.close()

    assert len(spies) == 1
    assert spies[0].group_by_calls == []


class _FailAfterFirstDelete:
    """Connection proxy that dies partway through the transaction."""

    def __init__(self, con: sqlite3.Connection) -> None:
        self._con = con
        self.deletes = 0

    def execute(self, sql: str, *args):
        if sql.lstrip().upper().startswith("DELETE"):
            self.deletes += 1
            if self.deletes == 2:
                raise sqlite3.OperationalError("disk I/O error")
        return self._con.execute(sql, *args)


def test_apply_removes_no_file_when_the_transaction_fails(tmp_path: Path):
    """A rollback restores the rows — so the files they point at must
    still be there. Unlinking inside the transaction loses data."""
    import pytest

    s, _ = _seed(tmp_path)
    groups = mod.plan(s._con())
    flaky = _FailAfterFirstDelete(s._con())
    with pytest.raises(sqlite3.OperationalError):
        mod.apply(flaky, groups)
    assert s._con().execute("SELECT count(*) FROM attachments").fetchone()[0] == 3
    for k in ("AAAAAAAA", "BBBBBBBB", "CCCCCCCC"):
        assert (tmp_path / "storage" / k / "attachment_1.pdf").is_file()


def test_main_apply_reports_removed_keys(tmp_path: Path, capsys):
    s, _ = _seed(tmp_path)
    s.close()
    rc = mod.main(["--db", str(tmp_path / "bib.sqlite"), "--apply"])
    out = capsys.readouterr().out
    assert rc == 0
    assert "removed rows=2" in out
    assert "removed keys (2)" in out
    assert "BBBBBBBB" in out and "CCCCCCCC" in out