feat(bib): alias listserv re-sends by body hash — one item per content (refs #665)

src/bib/email_ingest.py

@@ -20,7 +20,9 @@ from __future__ import annotations
 
 import email
 import email.utils
+import hashlib
 import imaplib
+import json
 import logging
 import re
 import ssl
@@ -124,8 +126,32 @@ def _ingest_message(
     subject = (msg.get("Subject") or "(no subject)").strip()
     sender_domain = _sender_domain(sender)
     body = _extract_body(msg)
+    title = subject[:255]
+    body_sha1 = hashlib.sha1(" ".join((body or "").split()).encode()).hexdigest()
 
-    item = Source(title=subject[:255], url=url)
+    # Listservs re-send identical content under a fresh Message-ID
+    # (Class B in docs/superpowers/plans/2026-08-26-zotero-dedupe.md).
+    # If an item with this exact title+body already exists, treat this
+    # message as an alias of it rather than creating a new Source.
+    # json_extract is more robust than a LIKE-on-serialized-JSON match
+    # since it doesn't depend on json.dumps's separator spacing.
+    con = store._con()
+    dup = con.execute(
+        "SELECT key, extra_json FROM items"
+        " WHERE item_type = 'source' AND title = ?"
+        " AND json_extract(extra_json, '$.body_sha1') = ?",
+        (title, body_sha1),
+    ).fetchone()
+    if dup:
+        ej = json.loads(dup["extra_json"] or "{}")
+        aliases = ej.get("alias_mids", [])
+        if raw_mid not in aliases:
+            aliases.append(raw_mid)
+            ej["alias_mids"] = aliases
+            store.update(dup["key"], extra_json=json.dumps(ej))
+        return 0
+
+    item = Source(title=title, url=url)
     item.doc_type = "Email"
     item.institution = sender_domain or "Email"
     item.date_published = _date_iso(msg.get("Date", ""))
@@ -144,8 +170,24 @@ def _ingest_message(
     if item.date_published:
         item.add_tag(f"year:{item.date_published[:4]}")
 
+    # Preserve any extra_json already on this exact url (e.g. alias_mids
+    # accumulated from prior re-sends of this same Message-ID) — upsert()
+    # doesn't merge extra_json, it overwrites with item.to_row()'s output.
+    prior = con.execute("SELECT extra_json FROM items WHERE url = ?", (url,)).fetchone()
+    prior_ej = json.loads(prior["extra_json"] or "{}") if prior else {}
+
     key = store.upsert(item)
 
+    # Record the body hash so future re-sends under a fresh Message-ID
+    # can be matched to this item instead of creating a duplicate. Merge
+    # order: prior_ej first (carries forward alias_mids), then this
+    # item's own to_row() extra_json (carries the current doc_type),
+    # then body_sha1 last so it always reflects this ingest's body.
+    fresh_ej = json.loads(item.to_row()["extra_json"] or "{}")
+    store.update(
+        key, extra_json=json.dumps({**prior_ej, **fresh_ej, "body_sha1": body_sha1})
+    )
+
     # Attachments → temp dir → attach_file
     attached = 0
     msg_scratch = scratch_root / _slug(raw_mid)[:60]

tests/bib/test_email_ingest.py

@@ -3,18 +3,23 @@
 from __future__ import annotations
 
 import email.message
+import json
 from pathlib import Path
 from unittest.mock import MagicMock, patch
 
+import pytest
+
 from bib.email_ingest import (
     Mailbox,
     _date_iso,
     _extract_body,
+    _ingest_message,
     _safe_filename,
     _sender_domain,
     _slug,
     ingest,
 )
+from bib.store import Store
 
 # ── Pure helpers ────────────────────<E29480><E29480>─────────────────────────────
 
@@ -118,6 +123,9 @@ class TestIngest:
 
         store = MagicMock()
         store.upsert.return_value = "KEY1"
+        # No pre-existing item to alias against — the resend-dedup lookup
+        # in _ingest_message must find nothing for these mocked messages.
+        store._con.return_value.execute.return_value.fetchone.return_value = None
 
         mb = Mailbox(
             host="mail.test", port=993, username="test@test.com", password="pw"
@@ -160,7 +168,130 @@ class TestIngest:
 
         store = MagicMock()
         store.upsert.return_value = "K"
+        store._con.return_value.execute.return_value.fetchone.return_value = None
         mb = Mailbox(host="h", port=993, username="u@d", password="p")
         stats = ingest(store, mb, limit=1)
         assert stats["seen"] == 1
         assert stats["ingested"] == 1
+
+
+# ── _ingest_message: content-hash aliasing of listserv re-sends ──
+
+
+@pytest.fixture
+def store():
+    return Store(":memory:")
+
+
+@pytest.fixture
+def mailbox():
+    return Mailbox(
+        host="mail.test", port=993, username="cmslists@test.com", password="pw"
+    )
+
+
+def make_msg(subject: str, body: str, mid: str, date: str):
+    m = email.message.EmailMessage()
+    m["Subject"] = subject
+    m["From"] = "CMS Updates <cmslists@subscriptions.cms.hhs.gov>"
+    m["Message-ID"] = f"<{mid}>"
+    m["Date"] = date
+    m.set_content(body)
+    return m
+
+
+def _raw_extra_json(store: Store, key: str) -> dict:
+    """Read extra_json straight off the row.
+
+    ``Source`` only declares ``doc_type`` — fields packed into
+    ``extra_json`` that aren't declared on the model (``body_sha1``,
+    ``alias_mids``) get silently dropped by pydantic's ``extra="ignore"``
+    default when the row round-trips through ``Item.from_row()`` /
+    ``to_row()``. Reading the column directly is the only way to see
+    what's actually persisted.
+    """
+    row = (
+        store._con()
+        .execute(  # noqa: SLF001
+            "SELECT extra_json FROM items WHERE key = ?", (key,)
+        )
+        .fetchone()
+    )
+    return json.loads(row["extra_json"] or "{}")
+
+
+class TestResendAliasing:
+    def test_same_body_new_mid_aliases_instead_of_new_item(
+        self, store, mailbox, tmp_path
+    ):
+        m1 = make_msg(
+            "Upcoming iQIES Hold Times",
+            "Same body.",
+            "aaa@x.example",
+            "Mon, 27 Apr 2026 10:00:00 -0400",
+        )
+        m2 = make_msg(
+            "Upcoming iQIES Hold Times",
+            "Same body.",
+            "bbb@x.example",
+            "Fri, 10 Jul 2026 10:00:00 -0400",
+        )
+        _ingest_message(store, mailbox, m1, tmp_path)
+        _ingest_message(store, mailbox, m2, tmp_path)
+        items = store.list_items(query="iQIES")
+        assert len(items) == 1
+        ej = _raw_extra_json(store, items[0].key)
+        assert "bbb@x.example" in ej.get("alias_mids", [])
+
+    def test_resend_is_idempotent(self, store, mailbox, tmp_path):
+        """Ingesting the identical alias message twice doesn't duplicate
+        its Message-ID in alias_mids."""
+        m1 = make_msg(
+            "Idempotency Check",
+            "Same body.",
+            "eee@x.example",
+            "Mon, 27 Apr 2026 10:00:00 -0400",
+        )
+        m2 = make_msg(
+            "Idempotency Check",
+            "Same body.",
+            "fff@x.example",
+            "Fri, 10 Jul 2026 10:00:00 -0400",
+        )
+        _ingest_message(store, mailbox, m1, tmp_path)
+        _ingest_message(store, mailbox, m2, tmp_path)
+        _ingest_message(store, mailbox, m2, tmp_path)
+        items = store.list_items(query="Idempotency")
+        assert len(items) == 1
+        ej = _raw_extra_json(store, items[0].key)
+        assert ej.get("alias_mids", []).count("fff@x.example") == 1
+
+    def test_different_body_same_subject_stays_separate(self, store, mailbox, tmp_path):
+        m1 = make_msg(
+            "Weekly Digest",
+            "Body one.",
+            "ccc@x.example",
+            "Mon, 27 Apr 2026 10:00:00 -0400",
+        )
+        m2 = make_msg(
+            "Weekly Digest",
+            "Body two.",
+            "ddd@x.example",
+            "Fri, 10 Jul 2026 10:00:00 -0400",
+        )
+        _ingest_message(store, mailbox, m1, tmp_path)
+        _ingest_message(store, mailbox, m2, tmp_path)
+        assert len(store.list_items(query="Weekly Digest")) == 2
+
+    def test_new_item_records_body_sha1(self, store, mailbox, tmp_path):
+        m1 = make_msg(
+            "First Send",
+            "Only body.",
+            "ggg@x.example",
+            "Mon, 27 Apr 2026 10:00:00 -0400",
+        )
+        _ingest_message(store, mailbox, m1, tmp_path)
+        items = store.list_items(query="First Send")
+        assert len(items) == 1
+        ej = _raw_extra_json(store, items[0].key)
+        assert len(ej.get("body_sha1", "")) == 40

tests/bib/test_email_ingest_deep.py

@@ -8,6 +8,21 @@ from unittest.mock import MagicMock
 from bib.email_ingest import Mailbox, _ingest_message, _sender_domain
 
 
+def _mock_store() -> MagicMock:
+    """MagicMock store safe to pass through _ingest_message.
+
+    _ingest_message's resend-dedup lookup calls
+    ``store._con().execute(...).fetchone()`` (refs #665) — on a bare
+    MagicMock that returns a truthy Mock, which _ingest_message
+    misreads as an existing duplicate and short-circuits before ever
+    calling store.upsert(). Configuring fetchone() to return None
+    makes the mock behave like an empty store.
+    """
+    s = MagicMock()
+    s._con.return_value.execute.return_value.fetchone.return_value = None
+    return s
+
+
 class TestIngestMessageWithAttachment:
     def test_extracts_attachment(self, tmp_path):
         msg = email.message.EmailMessage()
@@ -22,7 +37,7 @@ class TestIngestMessageWithAttachment:
             subtype="pdf",
             filename="doc.pdf",
         )
-        store = MagicMock()
+        store = _mock_store()
         store.upsert.return_value = "KEY1"
         mb = Mailbox(host="h", port=993, username="u@d", password="p")
         attached = _ingest_message(store, mb, msg, tmp_path / "scratch")
@@ -45,7 +60,7 @@ class TestIngestFinallyBlock:
 
         from bib.email_ingest import ingest
 
-        store = MagicMock()
+        store = _mock_store()
         mb = Mailbox(host="h", port=993, username="u@d", password="p")
 
         mock_conn = MagicMock()
@@ -65,7 +80,7 @@ class TestIngestFinallyBlock:
 
         from bib.email_ingest import ingest
 
-        store = MagicMock()
+        store = _mock_store()
         mb = Mailbox(host="h", port=993, username="u@d", password="p")
 
         mock_conn = MagicMock()
@@ -96,7 +111,7 @@ class TestIngestMessageNoFilename:
             # Note: no filename parameter
         )
 
-        store = MagicMock()
+        store = _mock_store()
         store.upsert.return_value = "KEY1"
         mb = Mailbox(host="h", port=993, username="u@d", password="p")
         attached = _ingest_message(store, mb, msg, tmp_path / "scratch")
@@ -117,7 +132,7 @@ class TestIngestMessageNoFilename:
             filename="doc.pdf",
         )
 
-        store = MagicMock()
+        store = _mock_store()
         store.upsert.return_value = "KEY1"
         store.attach_file.side_effect = Exception("attach fail")
         mb = Mailbox(host="h", port=993, username="u@d", password="p")

tests/bib/test_email_ingest_exercise.py

@@ -18,6 +18,21 @@ from bib.email_ingest import (
 )
 
 
+def _mock_store() -> MagicMock:
+    """MagicMock store safe to pass through _ingest_message.
+
+    _ingest_message's resend-dedup lookup calls
+    ``store._con().execute(...).fetchone()`` (refs #665) — on a bare
+    MagicMock that returns a truthy Mock, which _ingest_message
+    misreads as an existing duplicate and short-circuits before ever
+    calling store.upsert(). Configuring fetchone() to return None
+    makes the mock behave like an empty store.
+    """
+    s = MagicMock()
+    s._con.return_value.execute.return_value.fetchone.return_value = None
+    return s
+
+
 def _make_mailbox():
     return Mailbox(
         host="mail.example.com",
@@ -73,7 +88,7 @@ class TestIngest:
             ("OK", [(b"2", _make_msg(subject="Second").as_bytes())]),  # FETCH uid 2
             ("OK", None),  # STORE uid 2
         ]
-        store = MagicMock()
+        store = _mock_store()
         store.upsert.return_value = "KEY1"
         stats = ingest(store, _make_mailbox())
         assert stats["seen"] == 2
@@ -84,7 +99,7 @@ class TestIngest:
         conn = MagicMock()
         mc_imap_cls.return_value = conn
         conn.uid.return_value = ("NO", [b""])
-        store = MagicMock()
+        store = _mock_store()
         stats = ingest(store, _make_mailbox())
         assert stats["ingested"] == 0
 
@@ -96,7 +111,7 @@ class TestIngest:
             ("OK", [b"1"]),  # SEARCH
             ("OK", [None]),  # FETCH returns None
         ]
-        store = MagicMock()
+        store = _mock_store()
         stats = ingest(store, _make_mailbox())
         assert stats["errors"] == 1
 
@@ -109,7 +124,7 @@ class TestIngest:
             ("OK", [(b"1", _make_msg().as_bytes())]),  # FETCH uid 1
             ("OK", None),  # STORE uid 1
         ]
-        store = MagicMock()
+        store = _mock_store()
         store.upsert.return_value = "KEY1"
         stats = ingest(store, _make_mailbox(), limit=1)
         assert stats["seen"] == 1
@@ -122,14 +137,14 @@ class TestIngest:
             ("OK", [b"1"]),
             Exception("boom"),
         ]
-        store = MagicMock()
+        store = _mock_store()
         stats = ingest(store, _make_mailbox())
         assert stats["errors"] == 1
 
 
 class TestIngestMessage:
     def test_basic(self, tmp_path):
-        store = MagicMock()
+        store = _mock_store()
         store.upsert.return_value = "KEY1"
         msg = _make_msg()
         parsed = email.message_from_bytes(msg.as_bytes(), policy=email.policy.default)
@@ -138,7 +153,7 @@ class TestIngestMessage:
         store.upsert.assert_called_once()
 
     def test_with_attachment(self, tmp_path):
-        store = MagicMock()
+        store = _mock_store()
         store.upsert.return_value = "KEY1"
         msg = _make_msg(attachment_name="doc.pdf", attachment_data=b"PDF content")
         parsed = email.message_from_bytes(msg.as_bytes(), policy=email.policy.default)
@@ -146,7 +161,7 @@ class TestIngestMessage:
         assert n == 1
 
     def test_no_message_id(self, tmp_path):
-        store = MagicMock()
+        store = _mock_store()
         store.upsert.return_value = "KEY1"
         msg = _make_msg(message_id="")
         del msg["Message-ID"]
@@ -155,7 +170,7 @@ class TestIngestMessage:
         assert n == 0
 
     def test_with_list_id(self, tmp_path):
-        store = MagicMock()
+        store = _mock_store()
         store.upsert.return_value = "KEY1"
         msg = _make_msg(list_id="<cms-updates.listserv.cms.gov>")
         parsed = email.message_from_bytes(msg.as_bytes(), policy=email.policy.default)
@@ -163,7 +178,7 @@ class TestIngestMessage:
         assert n == 0
 
     def test_unnamed_attachment(self, tmp_path):
-        store = MagicMock()
+        store = _mock_store()
         store.upsert.return_value = "KEY1"
         msg = MIMEMultipart()
         msg["Subject"] = "Test"