feat(bib,zot): deterministic URL lookup + nightly library-wide merge-dupes (refs #665)

find_item_by_url now ORDER BY i.itemID so bib.sync consistently keeps
updating the same "earliest" item when duplicates exist — the one
zot.merge treats as the merge keeper. Wires --merge-dupes/--no-merge-dupes
(default off) into `stack bib sync-zotero`, running merge_url_duplicates
library-wide after the push, inside the same stop window.
--dedupe-collection stays but is documented as deprecated (deletes
without merging). Regenerates .gitea/workflows/zotero-sync.yml (source:
dev/scripts/backends/gitea.py) to swap --dedupe-collection Cmsupdates
for --merge-dupes, since listserv re-sends are now aliased at ingest
(Task 3) and any same-URL twins are merged, not deleted.

.gitea/workflows/zotero-sync.yml

@@ -24,14 +24,14 @@ jobs:
         run: docker stop zotero
 
       - name: Sync mail-ingested bib items into Zotero
-        # --dedupe-collection: CMS listservs re-send identical mail with
-        # fresh Message-IDs; the ingest correctly stores each send, so
-        # title+date duplicates accumulate unless swept here (keeps the
-        # earliest copy, removes later ones from Zotero AND bib).
+        # --merge-dupes: listserv re-sends are now aliased at ingest
+        # (Task 3), and any same-URL twins that still land in Zotero are
+        # merged (union tags/collections/notes/attachments, keep-earliest)
+        # rather than deleted.
         run: |
           docker exec api uv run --no-sync \
             stack bib sync-zotero --tag source:email --no-hold \
-              --dedupe-collection Cmsupdates
+              --merge-dupes
 
       - name: Sync Federal Register rule items into Zotero
         # Rule items only ever reached Zotero via one-off manual syncs

dev/scripts/backends/gitea.py

@@ -616,14 +616,14 @@ jobs:
         run: docker stop zotero
 
       - name: Sync mail-ingested bib items into Zotero
-        # --dedupe-collection: CMS listservs re-send identical mail with
-        # fresh Message-IDs; the ingest correctly stores each send, so
-        # title+date duplicates accumulate unless swept here (keeps the
-        # earliest copy, removes later ones from Zotero AND bib).
+        # --merge-dupes: listserv re-sends are now aliased at ingest
+        # (Task 3), and any same-URL twins that still land in Zotero are
+        # merged (union tags/collections/notes/attachments, keep-earliest)
+        # rather than deleted.
         run: |
           docker exec api uv run --no-sync \\
             stack bib sync-zotero --tag source:email --no-hold \\
-              --dedupe-collection Cmsupdates
+              --merge-dupes
 
       - name: Sync Federal Register rule items into Zotero
         # Rule items only ever reached Zotero via one-off manual syncs

src/cli/bib.py

@@ -506,10 +506,19 @@ def sync_zotero(
     dedupe_collection: str = typer.Option(
         "",
         "--dedupe-collection",
-        help="After the push, remove title+date duplicates from this Zotero "
-        "collection (and from bib, so the next sync can't resurrect them). "
-        "CMS listservs re-send identical mail with fresh Message-IDs, so "
-        "duplicates otherwise accumulate — e.g. 'Cmsupdates'.",
+        help="DEPRECATED — use --merge-dupes instead, which merges rather "
+        "than deletes. After the push, remove title+date duplicates from "
+        "this Zotero collection (and from bib, so the next sync can't "
+        "resurrect them). CMS listservs re-send identical mail with fresh "
+        "Message-IDs, so duplicates otherwise accumulate — e.g. "
+        "'Cmsupdates'.",
+    ),
+    merge_dupes: bool = typer.Option(
+        False,
+        "--merge-dupes/--no-merge-dupes",
+        help="After the push, merge same-URL duplicate items library-wide "
+        "(union tags/collections/notes/attachments; keep-earliest). "
+        "Supersedes --dedupe-collection, which deletes without merging.",
     ),
 ) -> None:
     """Push bib items (and their attachments) to the Zotero database.
@@ -523,9 +532,11 @@ def sync_zotero(
     from bib import connect
     from bib.sync import dedupe_collection as run_dedupe
     from bib.sync import push_to_zotero
+    from conf import path
 
     store = connect()
     items = store.list_items(tag=tag, item_type=item_type)
+    zotero_db_path = str(path("db.zotero"))
 
     def _docker(*args: str) -> None:
         subprocess.run(
@@ -538,9 +549,16 @@ def sync_zotero(
     try:
         if hold_zotero:
             _docker("stop", "zotero")
-        stats = push_to_zotero(items, store=store)
+        stats = push_to_zotero(items, store=store, zotero_db=zotero_db_path)
         if dedupe_collection:
             stats |= run_dedupe(dedupe_collection, store=store)
+        if merge_dupes:
+            from zot.merge import merge_url_duplicates
+
+            stats |= {
+                f"merge_{k}": v
+                for k, v in merge_url_duplicates(zotero_db_path, dry_run=False).items()
+            }
     finally:
         if hold_zotero:
             _docker("start", "zotero")

src/zot/db.py

@@ -389,12 +389,19 @@ class Db:
         return row[0] if row else None
 
     def find_item_by_url(self, url: str) -> int | None:
-        """Return ``itemID`` of the first item whose ``url`` field matches."""
+        """Return the lowest ``itemID`` of items whose ``url`` field matches.
+
+        Deterministic (``ORDER BY i.itemID``) so that when duplicates
+        exist, callers (e.g. ``bib.sync``) consistently keep updating
+        the same "earliest" item — the one ``zot.merge`` also treats
+        as the merge keeper.
+        """
         row = self.con.execute(
             """SELECT i.itemID FROM items i
                JOIN itemData id ON i.itemID = id.itemID
                JOIN itemDataValues idv ON id.valueID = idv.valueID
-               WHERE id.fieldID = ? AND idv.value = ?""",
+               WHERE id.fieldID = ? AND idv.value = ?
+               ORDER BY i.itemID""",
             (FIELD_MAP["url"], url),
         ).fetchone()
         return row[0] if row else None

tests/zot/test_db.py

@@ -836,3 +836,21 @@ class TestValidation:
         # Should have at least author
         assert len(result) >= 1
         con.close()
+
+
+class TestFindItemByUrlDeterministic:
+    def test_returns_lowest_item_id(self, tmp_path):
+        from zot.db import TYPE_MAP, Db
+        from zot.schema import create_db
+
+        p = tmp_path / "z.sqlite"
+        create_db(str(p)).close()
+        with Db(str(p)) as db:
+            a = db.create_item(TYPE_MAP["document"], key="AAAAAAAA")
+            b = db.create_item(TYPE_MAP["document"], key="BBBBBBBB")
+            # insert the URL row for the HIGHER itemID first so a
+            # scan-order query would return it
+            db.set_field(b, "url", "https://example.org/det")
+            db.set_field(a, "url", "https://example.org/det")
+            db.commit()
+            assert db.find_item_by_url("https://example.org/det") == a