homelab/stack
1
0
Fork 0
Code Issues 70 Pull Requests 1 Actions Packages Projects 1 Releases Wiki Activity

feat(bib): alias listserv re-sends by body hash — one item per content (refs #665)

Browse Source
This commit is contained in:
kert
2026-08-26 13:41:41 -04:00
parent a632908911
commit b0a168fc64
4 changed files with 219 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
src/bib/email_ingest.py
View File

@@ -20,7 +20,9 @@ from __future__ import annotations
import email
import email.utils
import hashlib
import imaplib
import json
import logging
import re
import ssl
@@ -124,8 +126,32 @@ def _ingest_message(
subject = (msg.get("Subject") or "(no subject)").strip()
sender_domain = _sender_domain(sender)
body = _extract_body(msg)
title = subject[:255]
body_sha1 = hashlib.sha1(" ".join((body or "").split()).encode()).hexdigest()
item = Source(title=subject[:255], url=url)
# Listservs re-send identical content under a fresh Message-ID
# (Class B in docs/superpowers/plans/2026-08-26-zotero-dedupe.md).
# If an item with this exact title+body already exists, treat this
# message as an alias of it rather than creating a new Source.
# json_extract is more robust than a LIKE-on-serialized-JSON match
# since it doesn't depend on json.dumps's separator spacing.
con = store._con()
dup = con.execute(
"SELECT key, extra_json FROM items"
" WHERE item_type = 'source' AND title = ?"
" AND json_extract(extra_json, '$.body_sha1') = ?",
(title, body_sha1),
).fetchone()
if dup:
ej = json.loads(dup["extra_json"] or "{}")
aliases = ej.get("alias_mids", [])
if raw_mid not in aliases:
aliases.append(raw_mid)
ej["alias_mids"] = aliases
store.update(dup["key"], extra_json=json.dumps(ej))
return 0
item = Source(title=title, url=url)
item.doc_type = "Email"
item.institution = sender_domain or "Email"
item.date_published = _date_iso(msg.get("Date", ""))
@@ -144,8 +170,24 @@ def _ingest_message(
if item.date_published:
item.add_tag(f"year:{item.date_published[:4]}")
# Preserve any extra_json already on this exact url (e.g. alias_mids
# accumulated from prior re-sends of this same Message-ID) — upsert()
# doesn't merge extra_json, it overwrites with item.to_row()'s output.
prior = con.execute("SELECT extra_json FROM items WHERE url = ?", (url,)).fetchone()
prior_ej = json.loads(prior["extra_json"] or "{}") if prior else {}
key = store.upsert(item)
# Record the body hash so future re-sends under a fresh Message-ID
# can be matched to this item instead of creating a duplicate. Merge
# order: prior_ej first (carries forward alias_mids), then this
# item's own to_row() extra_json (carries the current doc_type),
# then body_sha1 last so it always reflects this ingest's body.
fresh_ej = json.loads(item.to_row()["extra_json"] or "{}")
store.update(
key, extra_json=json.dumps({**prior_ej, **fresh_ej, "body_sha1": body_sha1})
)
# Attachments → temp dir → attach_file
attached = 0
msg_scratch = scratch_root / _slug(raw_mid)[:60]

131
tests/bib/test_email_ingest.py
View File

@@ -3,18 +3,23 @@
from __future__ import annotations
import email.message
import json
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
from bib.email_ingest import (
Mailbox,
_date_iso,
_extract_body,
_ingest_message,
_safe_filename,
_sender_domain,
_slug,
ingest,
)
from bib.store import Store
# ── Pure helpers ────────────────────<E29480><E29480>─────────────────────────────
@@ -118,6 +123,9 @@ class TestIngest:
store = MagicMock()
store.upsert.return_value = "KEY1"
# No pre-existing item to alias against — the resend-dedup lookup
# in _ingest_message must find nothing for these mocked messages.
store._con.return_value.execute.return_value.fetchone.return_value = None
mb = Mailbox(
host="mail.test", port=993, username="test@test.com", password="pw"
@@ -160,7 +168,130 @@ class TestIngest:
store = MagicMock()
store.upsert.return_value = "K"
store._con.return_value.execute.return_value.fetchone.return_value = None
mb = Mailbox(host="h", port=993, username="u@d", password="p")
stats = ingest(store, mb, limit=1)
assert stats["seen"] == 1
assert stats["ingested"] == 1
# ── _ingest_message: content-hash aliasing of listserv re-sends ──
@pytest.fixture
def store():
return Store(":memory:")
@pytest.fixture
def mailbox():
return Mailbox(
host="mail.test", port=993, username="cmslists@test.com", password="pw"
)
def make_msg(subject: str, body: str, mid: str, date: str):
m = email.message.EmailMessage()
m["Subject"] = subject
m["From"] = "CMS Updates <cmslists@subscriptions.cms.hhs.gov>"
m["Message-ID"] = f"<{mid}>"
m["Date"] = date
m.set_content(body)
return m
def _raw_extra_json(store: Store, key: str) -> dict:
"""Read extra_json straight off the row.
``Source`` only declares ``doc_type`` — fields packed into
``extra_json`` that aren't declared on the model (``body_sha1``,
``alias_mids``) get silently dropped by pydantic's ``extra="ignore"``
default when the row round-trips through ``Item.from_row()`` /
``to_row()``. Reading the column directly is the only way to see
what's actually persisted.
"""
row = (
store._con()
.execute( # noqa: SLF001
"SELECT extra_json FROM items WHERE key = ?", (key,)
)
.fetchone()
)
return json.loads(row["extra_json"] or "{}")
class TestResendAliasing:
def test_same_body_new_mid_aliases_instead_of_new_item(
self, store, mailbox, tmp_path
):
m1 = make_msg(
"Upcoming iQIES Hold Times",
"Same body.",
"aaa@x.example",
"Mon, 27 Apr 2026 10:00:00 -0400",
)
m2 = make_msg(
"Upcoming iQIES Hold Times",
"Same body.",
"bbb@x.example",
"Fri, 10 Jul 2026 10:00:00 -0400",
)
_ingest_message(store, mailbox, m1, tmp_path)
_ingest_message(store, mailbox, m2, tmp_path)
items = store.list_items(query="iQIES")
assert len(items) == 1
ej = _raw_extra_json(store, items[0].key)
assert "bbb@x.example" in ej.get("alias_mids", [])
def test_resend_is_idempotent(self, store, mailbox, tmp_path):
"""Ingesting the identical alias message twice doesn't duplicate
its Message-ID in alias_mids."""
m1 = make_msg(
"Idempotency Check",
"Same body.",
"eee@x.example",
"Mon, 27 Apr 2026 10:00:00 -0400",
)
m2 = make_msg(
"Idempotency Check",
"Same body.",
"fff@x.example",
"Fri, 10 Jul 2026 10:00:00 -0400",
)
_ingest_message(store, mailbox, m1, tmp_path)
_ingest_message(store, mailbox, m2, tmp_path)
_ingest_message(store, mailbox, m2, tmp_path)
items = store.list_items(query="Idempotency")
assert len(items) == 1
ej = _raw_extra_json(store, items[0].key)
assert ej.get("alias_mids", []).count("fff@x.example") == 1
def test_different_body_same_subject_stays_separate(self, store, mailbox, tmp_path):
m1 = make_msg(
"Weekly Digest",
"Body one.",
"ccc@x.example",
"Mon, 27 Apr 2026 10:00:00 -0400",
)
m2 = make_msg(
"Weekly Digest",
"Body two.",
"ddd@x.example",
"Fri, 10 Jul 2026 10:00:00 -0400",
)
_ingest_message(store, mailbox, m1, tmp_path)
_ingest_message(store, mailbox, m2, tmp_path)
assert len(store.list_items(query="Weekly Digest")) == 2
def test_new_item_records_body_sha1(self, store, mailbox, tmp_path):
m1 = make_msg(
"First Send",
"Only body.",
"ggg@x.example",
"Mon, 27 Apr 2026 10:00:00 -0400",
)
_ingest_message(store, mailbox, m1, tmp_path)
items = store.list_items(query="First Send")
assert len(items) == 1
ej = _raw_extra_json(store, items[0].key)
assert len(ej.get("body_sha1", "")) == 40

25
tests/bib/test_email_ingest_deep.py
View File

@@ -8,6 +8,21 @@ from unittest.mock import MagicMock
from bib.email_ingest import Mailbox, _ingest_message, _sender_domain
def _mock_store() -> MagicMock:
"""MagicMock store safe to pass through _ingest_message.
_ingest_message's resend-dedup lookup calls
``store._con().execute(...).fetchone()`` (refs #665) — on a bare
MagicMock that returns a truthy Mock, which _ingest_message
misreads as an existing duplicate and short-circuits before ever
calling store.upsert(). Configuring fetchone() to return None
makes the mock behave like an empty store.
"""
s = MagicMock()
s._con.return_value.execute.return_value.fetchone.return_value = None
return s
class TestIngestMessageWithAttachment:
def test_extracts_attachment(self, tmp_path):
msg = email.message.EmailMessage()
@@ -22,7 +37,7 @@ class TestIngestMessageWithAttachment:
subtype="pdf",
filename="doc.pdf",
)
store = MagicMock()
store = _mock_store()
store.upsert.return_value = "KEY1"
mb = Mailbox(host="h", port=993, username="u@d", password="p")
attached = _ingest_message(store, mb, msg, tmp_path / "scratch")
@@ -45,7 +60,7 @@ class TestIngestFinallyBlock:
from bib.email_ingest import ingest
store = MagicMock()
store = _mock_store()
mb = Mailbox(host="h", port=993, username="u@d", password="p")
mock_conn = MagicMock()
@@ -65,7 +80,7 @@ class TestIngestFinallyBlock:
from bib.email_ingest import ingest
store = MagicMock()
store = _mock_store()
mb = Mailbox(host="h", port=993, username="u@d", password="p")
mock_conn = MagicMock()
@@ -96,7 +111,7 @@ class TestIngestMessageNoFilename:
# Note: no filename parameter
)
store = MagicMock()
store = _mock_store()
store.upsert.return_value = "KEY1"
mb = Mailbox(host="h", port=993, username="u@d", password="p")
attached = _ingest_message(store, mb, msg, tmp_path / "scratch")
@@ -117,7 +132,7 @@ class TestIngestMessageNoFilename:
filename="doc.pdf",
)
store = MagicMock()
store = _mock_store()
store.upsert.return_value = "KEY1"
store.attach_file.side_effect = Exception("attach fail")
mb = Mailbox(host="h", port=993, username="u@d", password="p")

35
tests/bib/test_email_ingest_exercise.py
View File

@@ -18,6 +18,21 @@ from bib.email_ingest import (
)
def _mock_store() -> MagicMock:
"""MagicMock store safe to pass through _ingest_message.
_ingest_message's resend-dedup lookup calls
``store._con().execute(...).fetchone()`` (refs #665) — on a bare
MagicMock that returns a truthy Mock, which _ingest_message
misreads as an existing duplicate and short-circuits before ever
calling store.upsert(). Configuring fetchone() to return None
makes the mock behave like an empty store.
"""
s = MagicMock()
s._con.return_value.execute.return_value.fetchone.return_value = None
return s
def _make_mailbox():
return Mailbox(
host="mail.example.com",
@@ -73,7 +88,7 @@ class TestIngest:
("OK", [(b"2", _make_msg(subject="Second").as_bytes())]), # FETCH uid 2
("OK", None), # STORE uid 2
]
store = MagicMock()
store = _mock_store()
store.upsert.return_value = "KEY1"
stats = ingest(store, _make_mailbox())
assert stats["seen"] == 2
@@ -84,7 +99,7 @@ class TestIngest:
conn = MagicMock()
mc_imap_cls.return_value = conn
conn.uid.return_value = ("NO", [b""])
store = MagicMock()
store = _mock_store()
stats = ingest(store, _make_mailbox())
assert stats["ingested"] == 0
@@ -96,7 +111,7 @@ class TestIngest:
("OK", [b"1"]), # SEARCH
("OK", [None]), # FETCH returns None
]
store = MagicMock()
store = _mock_store()
stats = ingest(store, _make_mailbox())
assert stats["errors"] == 1
@@ -109,7 +124,7 @@ class TestIngest:
("OK", [(b"1", _make_msg().as_bytes())]), # FETCH uid 1
("OK", None), # STORE uid 1
]
store = MagicMock()
store = _mock_store()
store.upsert.return_value = "KEY1"
stats = ingest(store, _make_mailbox(), limit=1)
assert stats["seen"] == 1
@@ -122,14 +137,14 @@ class TestIngest:
("OK", [b"1"]),
Exception("boom"),
]
store = MagicMock()
store = _mock_store()
stats = ingest(store, _make_mailbox())
assert stats["errors"] == 1
class TestIngestMessage:
def test_basic(self, tmp_path):
store = MagicMock()
store = _mock_store()
store.upsert.return_value = "KEY1"
msg = _make_msg()
parsed = email.message_from_bytes(msg.as_bytes(), policy=email.policy.default)
@@ -138,7 +153,7 @@ class TestIngestMessage:
store.upsert.assert_called_once()
def test_with_attachment(self, tmp_path):
store = MagicMock()
store = _mock_store()
store.upsert.return_value = "KEY1"
msg = _make_msg(attachment_name="doc.pdf", attachment_data=b"PDF content")
parsed = email.message_from_bytes(msg.as_bytes(), policy=email.policy.default)
@@ -146,7 +161,7 @@ class TestIngestMessage:
assert n == 1
def test_no_message_id(self, tmp_path):
store = MagicMock()
store = _mock_store()
store.upsert.return_value = "KEY1"
msg = _make_msg(message_id="")
del msg["Message-ID"]
@@ -155,7 +170,7 @@ class TestIngestMessage:
assert n == 0
def test_with_list_id(self, tmp_path):
store = MagicMock()
store = _mock_store()
store.upsert.return_value = "KEY1"
msg = _make_msg(list_id="<cms-updates.listserv.cms.gov>")
parsed = email.message_from_bytes(msg.as_bytes(), policy=email.policy.default)
@@ -163,7 +178,7 @@ class TestIngestMessage:
assert n == 0
def test_unnamed_attachment(self, tmp_path):
store = MagicMock()
store = _mock_store()
store.upsert.return_value = "KEY1"
msg = MIMEMultipart()
msg["Subject"] = "Test"