feat(bib): dedupe pass in nightly zotero-sync — keep-earliest on title+date

CMS listservs re-send identical mail with a fresh Message-ID, so the
ingest correctly stores each send as a distinct Source and every sync
pushes both — 10 such twin pairs had accumulated in Cmsupdates
(removed by hand today). New bib.sync.dedupe_collection() groups a
Zotero collection by (title, date), keeps the earliest copy, and
deletes later ones from Zotero AND bib so the next sync can't
resurrect them. Wired as --dedupe-collection on sync-zotero and passed
by the nightly zotero-sync workflow. Runs under the same
zotero-stopped lock window as the push.

.gitea/workflows/zotero-sync.yml

@@ -24,9 +24,14 @@ jobs:
         run: docker stop zotero
 
       - name: Sync mail-ingested bib items into Zotero
+        # --dedupe-collection: CMS listservs re-send identical mail with
+        # fresh Message-IDs; the ingest correctly stores each send, so
+        # title+date duplicates accumulate unless swept here (keeps the
+        # earliest copy, removes later ones from Zotero AND bib).
         run: |
           docker exec api uv run --no-sync \
-            stack bib sync-zotero --tag source:email --no-hold
+            stack bib sync-zotero --tag source:email --no-hold \
+              --dedupe-collection Cmsupdates
 
       - name: Restart zotero
         if: always()

dev/scripts/backends/gitea.py

@@ -615,9 +615,14 @@ jobs:
         run: docker stop zotero
 
       - name: Sync mail-ingested bib items into Zotero
+        # --dedupe-collection: CMS listservs re-send identical mail with
+        # fresh Message-IDs; the ingest correctly stores each send, so
+        # title+date duplicates accumulate unless swept here (keeps the
+        # earliest copy, removes later ones from Zotero AND bib).
         run: |
           docker exec api uv run --no-sync \\
-            stack bib sync-zotero --tag source:email --no-hold
+            stack bib sync-zotero --tag source:email --no-hold \\
+              --dedupe-collection Cmsupdates
 
       - name: Restart zotero
         if: always()

src/bib/sync.py

@@ -549,3 +549,66 @@ def _sync_notes(
         existing_titles.add(title)
         count += 1
     return count
+
+
+# ── Dedupe ──────────────────────────────────────────────────────────
+
+
+def dedupe_collection(
+    collection_name: str,
+    *,
+    store: "Store | None" = None,
+    zotero_db: str | None = None,
+) -> dict[str, int]:
+    """Remove duplicate items from a Zotero collection (and bib).
+
+    Two items are duplicates when they share both *title* and *date* —
+    CMS listservs re-send the same message with a fresh Message-ID, so
+    the mail ingest correctly stores each send as a distinct Source and
+    every sync pushes both. The earliest copy (lowest itemID) is kept;
+    later copies are deleted from Zotero and, when *store* is given,
+    from bib as well so the next sync cannot resurrect them.
+
+    Zotero must not be running (exclusive SQLite lock) — callers hold
+    the same stop/start window used for the push.
+    """
+    if zotero_db is None:
+        from conf import path
+
+        zotero_db = str(path("db.zotero"))
+
+    stats = {"groups": 0, "removed": 0}
+    with Db(zotero_db) as db:
+        rows = db.con.execute(
+            """
+            SELECT i.itemID, i.key,
+              MAX(CASE WHEN f.fieldName = 'title' THEN idv.value END) AS title,
+              MAX(CASE WHEN f.fieldName = 'date' THEN idv.value END) AS date
+            FROM collections c
+            JOIN collectionItems ci ON c.collectionID = ci.collectionID
+            JOIN items i ON ci.itemID = i.itemID
+            LEFT JOIN itemData id ON i.itemID = id.itemID
+            LEFT JOIN fields f ON id.fieldID = f.fieldID
+            LEFT JOIN itemDataValues idv ON id.valueID = idv.valueID
+            WHERE c.collectionName = ?
+            GROUP BY i.itemID
+            """,
+            (collection_name,),
+        ).fetchall()
+
+        groups: dict[tuple, list] = {}
+        for item_id, key, title, date in rows:
+            groups.setdefault((title, date), []).append((item_id, key))
+
+        for members in groups.values():
+            if len(members) < 2:
+                continue
+            stats["groups"] += 1
+            members.sort()  # lowest itemID = earliest insert
+            for item_id, key in members[1:]:
+                db.delete_item(item_id)
+                if store is not None:
+                    store.delete(key)
+                stats["removed"] += 1
+        db.commit()
+    return stats

src/cli/bib.py

@@ -471,6 +471,14 @@ def sync_zotero(
         help="Stop the 'zotero' container during sync so Zotero releases "
         "its exclusive SQLite lock; restart afterwards.",
     ),
+    dedupe_collection: str = typer.Option(
+        "",
+        "--dedupe-collection",
+        help="After the push, remove title+date duplicates from this Zotero "
+        "collection (and from bib, so the next sync can't resurrect them). "
+        "CMS listservs re-send identical mail with fresh Message-IDs, so "
+        "duplicates otherwise accumulate — e.g. 'Cmsupdates'.",
+    ),
 ) -> None:
     """Push bib items (and their attachments) to the Zotero database.
 
@@ -481,6 +489,7 @@ def sync_zotero(
     import subprocess
 
     from bib import connect
+    from bib.sync import dedupe_collection as run_dedupe
     from bib.sync import push_to_zotero
 
     store = connect()
@@ -498,6 +507,8 @@ def sync_zotero(
         if hold_zotero:
             _docker("stop", "zotero")
         stats = push_to_zotero(items, store=store)
+        if dedupe_collection:
+            stats |= run_dedupe(dedupe_collection, store=store)
     finally:
         if hold_zotero:
             _docker("start", "zotero")

tests/bib/test_sync.py

@@ -9,6 +9,8 @@ from __future__ import annotations
 
 import sqlite3
 
+import pytest
+
 from bib.item import Download, Manual, Regulation, Rule, Source
 from bib.sync import _item_to_zotero_fields, push_to_zotero
 
@@ -486,3 +488,70 @@ class TestSyncNotes:
         n = con.execute("SELECT COUNT(*) FROM itemNotes").fetchone()[0]
         con.close()
         assert n == 1
+
+
+# ── dedupe_collection ───────────────────────────────────────────────
+
+
+class TestDedupeCollection:
+    # Two distinct sends of the same message (CMS re-sends with a fresh
+    # Message-ID → distinct bib keys, same title+date) plus one genuinely
+    # different email. Store.create generates keys, so tests capture the
+    # returned key per item.
+    _TAGS = ["source:email", "mailbox:cmsupdates"]
+
+    def test_removes_title_date_twins_keeps_earliest(self, tmp_path) -> None:
+        from bib.store import Store
+        from bib.sync import dedupe_collection
+
+        zot_db = str(tmp_path / "zotero.sqlite")
+        _make_zotero_db(zot_db).close()
+
+        store = Store(str(tmp_path / "bib.sqlite"))
+        first = store.create(
+            Source(title="Weekly Update", date_published="2026-07-01"),
+            tags=self._TAGS,
+        )
+        twin = store.create(
+            Source(title="Weekly Update", date_published="2026-07-01"),
+            tags=self._TAGS,
+        )
+        distinct = store.create(
+            Source(title="Weekly Update", date_published="2026-07-08"),
+            tags=self._TAGS,
+        )
+        push_to_zotero(store.list_items(), store=store, zotero_db=zot_db)
+
+        stats = dedupe_collection("Cmsupdates", store=store, zotero_db=zot_db)
+        assert stats == {"groups": 1, "removed": 1}
+
+        con = sqlite3.connect(zot_db)
+        keys = {r[0] for r in con.execute("SELECT key FROM items").fetchall()}
+        con.close()
+        assert first in keys  # earliest kept
+        assert twin not in keys
+        assert distinct in keys
+        # bib twin removed too, so the next sync can't resurrect it
+        with pytest.raises(KeyError):
+            store.get(twin)
+        assert store.get(first) is not None
+        store.close()
+
+    def test_noop_when_no_dupes(self, tmp_path) -> None:
+        from bib.store import Store
+        from bib.sync import dedupe_collection
+
+        zot_db = str(tmp_path / "zotero.sqlite")
+        _make_zotero_db(zot_db).close()
+        store = Store(str(tmp_path / "bib.sqlite"))
+        store.create(
+            Source(title="Weekly Update", date_published="2026-07-08"),
+            tags=self._TAGS,
+        )
+        push_to_zotero(store.list_items(), store=store, zotero_db=zot_db)
+
+        assert dedupe_collection("Cmsupdates", store=store, zotero_db=zot_db) == {
+            "groups": 0,
+            "removed": 0,
+        }
+        store.close()