feat(bib): legacy dedupe script — merge same-url and re-sent-email rows (refs #665)

dev/scripts/dedupe_bib_legacy.py

@@ -0,0 +1,136 @@
+"""One-off: merge legacy duplicate rows inside bib.sqlite.
+
+Class C — same-url rows created before Store.upsert deduped by URL
+(#624/91c3306). Class B — listserv re-sends: same (title, body-hash)
+among source:email items, each under a distinct email:<Message-ID>
+url. Both classes predate Task 3's ingest-side dedup, so legacy email
+rows have no stored ``body_sha1`` in ``extra_json`` — Class B grouping
+is instead computed in Python from title + whitespace-normalized
+abstract, matching bib/email_ingest.py's hash recipe.
+
+Keep-earliest (by created_at, then id); union tags/collections into
+the keeper; alias surplus Message-IDs into the keeper's
+``extra_json["alias_mids"]`` (idempotent — matches the
+email_ingest.py convention: read/write extra_json via raw SQL, never
+through Item.to_row(), since to_row() round-trips through the
+item's declared pydantic fields and would silently drop
+alias_mids/body_sha1 for item types that don't declare them, e.g.
+Source). Delete surplus rows. Prints stats; --dry-run is the default.
+
+Returns keeper/loser key pairs so the Zotero-side merge (zot.merge,
+Task 5) can be driven off the same decisions.
+
+Usage: uv run python dev/scripts/dedupe_bib_legacy.py [--live]
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import sys
+from collections import defaultdict
+from typing import Any
+
+
+def _body_hash(abstract: str) -> str:
+    return hashlib.sha1(" ".join((abstract or "").split()).encode()).hexdigest()
+
+
+def dedupe_bib(store: Any, *, dry_run: bool = True) -> dict:
+    """Merge legacy duplicate rows in ``store``.
+
+    Returns ``{url_groups, email_groups, removed, aliased, pairs}``
+    where ``pairs`` is a list of ``(keeper_key, loser_key)`` tuples
+    describing every merge decision made (or, under ``dry_run``, that
+    would be made).
+    """
+    con = store._con()
+    out: dict[str, Any] = {
+        "url_groups": 0,
+        "email_groups": 0,
+        "removed": 0,
+        "aliased": 0,
+        "pairs": [],
+    }
+
+    groups: list[list[str]] = []  # each: [keeper_key, loser_key, ...]
+
+    # Class C: exact-url groups (http only; email: urls are unique per
+    # Message-ID and are handled by the content-based Class B pass below).
+    by_url: dict[str, list[str]] = defaultdict(list)
+    for key, url in con.execute(
+        "SELECT key, url FROM items WHERE url LIKE 'http%' ORDER BY created_at, id"
+    ):
+        by_url[url].append(key)
+    for keys in by_url.values():
+        if len(keys) > 1:
+            out["url_groups"] += 1
+            groups.append(keys)
+
+    # Class B: (title, body-hash) among source:email items.
+    by_content: dict[tuple[str, str], list[str]] = defaultdict(list)
+    for key, title, abstract in con.execute(
+        "SELECT i.key, i.title, COALESCE(i.abstract,'') FROM items i"
+        " JOIN item_tags it ON it.item_id = i.id"
+        " JOIN tags t ON t.id = it.tag_id AND t.name = 'source:email'"
+        " ORDER BY i.created_at, i.id"
+    ):
+        by_content[(title, _body_hash(abstract))].append(key)
+    for keys in by_content.values():
+        if len(keys) > 1:
+            out["email_groups"] += 1
+            groups.append(keys)
+
+    for keys in groups:
+        keeper_key, losers = keys[0], keys[1:]
+        if dry_run:
+            continue
+
+        keeper = store.get(keeper_key)
+        # Read extra_json via raw SQL, not keeper.to_row() — to_row()
+        # rebuilds extra_json from the item's declared subclass fields
+        # only (e.g. Source only knows doc_type) and would silently
+        # drop any alias_mids/body_sha1 already stored on the row.
+        row = con.execute(
+            "SELECT extra_json FROM items WHERE key = ?", (keeper_key,)
+        ).fetchone()
+        ej = json.loads((row["extra_json"] if row else "") or "{}")
+        aliases = list(ej.get("alias_mids", []))
+        tags = list(keeper.tags)
+        collections = list(keeper.collections)
+
+        for lk in losers:
+            loser = store.get(lk)
+            tags += [t for t in loser.tags if t not in tags]
+            collections += [c for c in loser.collections if c not in collections]
+            if loser.url.startswith("email:"):
+                mid = loser.url[len("email:") :]
+                if mid not in aliases:
+                    aliases.append(mid)
+                    out["aliased"] += 1
+            out["pairs"].append((keeper_key, lk))
+            store.delete(lk)
+            out["removed"] += 1
+
+        if aliases:
+            ej["alias_mids"] = aliases
+        store.update(
+            keeper_key,
+            tags=tags,
+            collections=collections,
+            extra_json=json.dumps(ej),
+        )
+
+    return out
+
+
+if __name__ == "__main__":
+    from bib import Store
+
+    live = "--live" in sys.argv
+    _store = Store()
+    result = dedupe_bib(_store, dry_run=not live)
+    _pairs = result.pop("pairs")
+    print(json.dumps(result, indent=1))
+    for keeper, loser in _pairs:
+        print(f"pair {keeper} <- {loser}")

tests/bib/test_dedupe_legacy.py

@@ -0,0 +1,125 @@
+"""Tests for dev/scripts/dedupe_bib_legacy.py — merging legacy duplicate
+rows inside bib.sqlite: Class C (same-url rows predating URL-dedup in
+Store.upsert, #624/91c3306) and Class B (listserv re-sends stored as
+separate items under distinct email:<Message-ID> urls, refs #665).
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+from bib.item import Source
+from bib.store import Store
+
+# Load the script as a module — it lives outside the src/ tree, mirroring
+# the pattern in tests/dev/test_migrate_fr_txt.py (dev/ isn't an
+# importable package).
+_SCRIPT = (
+    Path(__file__).resolve().parents[2] / "dev" / "scripts" / "dedupe_bib_legacy.py"
+)
+_spec = importlib.util.spec_from_file_location("_dedupe_bib_legacy", _SCRIPT)
+assert _spec and _spec.loader
+_dedupe_bib_legacy = importlib.util.module_from_spec(_spec)
+sys.modules["_dedupe_bib_legacy"] = _dedupe_bib_legacy
+_spec.loader.exec_module(_dedupe_bib_legacy)
+dedupe_bib = _dedupe_bib_legacy.dedupe_bib
+
+
+@pytest.fixture
+def store(tmp_path):
+    s = Store(str(tmp_path / "bib.sqlite"))
+    yield s
+    s.close()
+
+
+def test_url_group_keeps_earliest_and_unions_tags(store):
+    a = store.create(
+        Source(title="ACO REACH", url="https://x.test/reach"),
+        tags=["module:aco", "keep-me"],
+    )
+    # Second row with the same url — simulate a pre-#624 duplicate insert
+    # that predates Store.upsert's URL-based dedup.
+    con = store._con()
+    con.execute(
+        "INSERT INTO items (key, item_type, title, url, extra_json)"
+        " VALUES ('ZZZZZZZ2','source','ACO REACH','https://x.test/reach','{}')"
+    )
+    con.commit()
+    store.add_tag("ZZZZZZZ2", "loser-tag")
+
+    out = dedupe_bib(store, dry_run=False)
+
+    assert out["url_groups"] == 1
+    assert out["removed"] == 1
+    kept = store.get(a)
+    assert "loser-tag" in kept.tags
+    assert "keep-me" in kept.tags
+    assert (a, "ZZZZZZZ2") in out["pairs"]
+    with pytest.raises(KeyError):
+        store.get("ZZZZZZZ2")
+
+
+def test_email_group_aliases_mids(store):
+    s1 = Source(title="iQIES Hold Times", url="email:aaa@x")
+    s1.abstract = "Same body."
+    s2 = Source(title="iQIES Hold Times", url="email:bbb@x")
+    s2.abstract = "Same body."
+    k1 = store.create(s1, tags=["source:email"])
+    k2 = store.create(s2, tags=["source:email"])
+
+    out = dedupe_bib(store, dry_run=False)
+
+    assert out["email_groups"] == 1
+    assert out["removed"] == 1
+    assert out["aliased"] == 1
+    assert (k1, k2) in out["pairs"]
+    # Read extra_json via raw SQL, not Item.to_row() — Source.to_row()
+    # only serializes its own declared fields (doc_type) and would
+    # silently drop alias_mids on a round-trip through the pydantic
+    # model (see dedupe_bib_legacy.py's docstring / email_ingest.py:180).
+    con = store._con()
+    row = con.execute("SELECT extra_json FROM items WHERE key = ?", (k1,)).fetchone()
+    ej = json.loads(row["extra_json"])
+    assert "bbb@x" in ej["alias_mids"]
+    with pytest.raises(KeyError):
+        store.get(k2)
+
+
+def test_email_group_idempotent_second_run_adds_nothing(store):
+    s1 = Source(title="iQIES Hold Times", url="email:aaa@x")
+    s1.abstract = "Same body."
+    s2 = Source(title="iQIES Hold Times", url="email:bbb@x")
+    s2.abstract = "Same body."
+    k1 = store.create(s1, tags=["source:email"])
+    store.create(s2, tags=["source:email"])
+
+    dedupe_bib(store, dry_run=False)
+    out2 = dedupe_bib(store, dry_run=False)
+
+    assert out2["removed"] == 0
+    assert out2["aliased"] == 0
+    con = store._con()
+    row = con.execute("SELECT extra_json FROM items WHERE key = ?", (k1,)).fetchone()
+    ej = json.loads(row["extra_json"])
+    assert ej["alias_mids"] == ["bbb@x"]
+
+
+def test_dry_run_touches_nothing(store):
+    s1 = Source(title="T", url="email:a@x")
+    s1.abstract = "B"
+    s2 = Source(title="T", url="email:b@x")
+    s2.abstract = "B"
+    store.create(s1, tags=["source:email"])
+    store.create(s2, tags=["source:email"])
+
+    out = dedupe_bib(store, dry_run=True)
+
+    assert out["removed"] == 0
+    assert out["email_groups"] == 1
+    assert out["pairs"] == []
+    assert len(store.list_items(query="T")) == 2