375 lines
12 KiB
Python
375 lines
12 KiB
Python
"""Tests for bib.email_ingest — IMAP → bib ingest."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import email.message
|
||
import json
|
||
from pathlib import Path
|
||
from unittest.mock import MagicMock, patch
|
||
|
||
import pytest
|
||
|
||
from bib.email_ingest import (
|
||
Mailbox,
|
||
_date_iso,
|
||
_extract_body,
|
||
_ingest_message,
|
||
_safe_filename,
|
||
_sender_domain,
|
||
_slug,
|
||
ingest,
|
||
)
|
||
from bib.store import Store
|
||
|
||
# ── Pure helpers ────────────────────<E29480><E29480>─────────────────────────────
|
||
|
||
|
||
class TestSlug:
|
||
def test_basic(self):
|
||
assert _slug("Hello World!") == "hello-world"
|
||
|
||
def test_truncate(self):
|
||
assert len(_slug("x" * 100)) <= 60
|
||
|
||
def test_empty(self):
|
||
assert _slug("") == ""
|
||
|
||
|
||
class TestSafeFilename:
|
||
def test_special_chars(self):
|
||
assert _safe_filename("file (1).pdf") == "file_1_.pdf"
|
||
|
||
def test_empty(self):
|
||
assert _safe_filename("") == "attachment"
|
||
|
||
|
||
class TestSenderDomain:
|
||
def test_angle_bracket(self):
|
||
assert _sender_domain("John <john@example.com>") == "example.com"
|
||
|
||
def test_bare(self):
|
||
assert _sender_domain("user@domain.org") == "domain.org"
|
||
|
||
def test_no_domain(self):
|
||
assert _sender_domain("no email here") == ""
|
||
|
||
|
||
class TestDateIso:
|
||
def test_rfc2822(self):
|
||
assert _date_iso("Wed, 15 Apr 2026 22:00:35 +0000") == "2026-04-15"
|
||
|
||
def test_empty(self):
|
||
assert _date_iso("") == ""
|
||
|
||
def test_garbage(self):
|
||
assert _date_iso("not a date") == ""
|
||
|
||
|
||
class TestExtractBody:
|
||
def test_plain_text(self):
|
||
msg = email.message.EmailMessage()
|
||
msg.set_content("Hello world")
|
||
assert _extract_body(msg) == "Hello world\n"
|
||
|
||
def test_html_fallback(self):
|
||
msg = email.message.EmailMessage()
|
||
msg["Content-Type"] = "multipart/alternative"
|
||
msg.add_alternative("<p>Hello</p>", subtype="html")
|
||
body = _extract_body(msg)
|
||
assert "Hello" in body
|
||
|
||
|
||
# ── ingest (mocked IMAP + Store) ─────────────────────────────────
|
||
|
||
|
||
def _make_raw_email(
|
||
subject="Test", sender="cms@example.gov", body="Hello", msg_id="<abc@test>"
|
||
):
|
||
msg = email.message.EmailMessage()
|
||
msg["Subject"] = subject
|
||
msg["From"] = sender
|
||
msg["Date"] = "Wed, 15 Apr 2026 10:00:00 +0000"
|
||
msg["Message-ID"] = msg_id
|
||
msg.set_content(body)
|
||
return msg.as_bytes()
|
||
|
||
|
||
class TestIngest:
|
||
@patch("bib.email_ingest.imaplib.IMAP4_SSL")
|
||
def test_processes_unseen(self, MockIMAP):
|
||
conn = MockIMAP.return_value
|
||
conn.login.return_value = ("OK", [])
|
||
conn.select.return_value = ("OK", [])
|
||
conn.uid.side_effect = [
|
||
("OK", [b"1 2"]), # SEARCH
|
||
(
|
||
"OK",
|
||
[(b"1 (RFC822 {100}", _make_raw_email(msg_id="<m1@test>"))],
|
||
), # FETCH 1
|
||
("OK", []), # STORE 1
|
||
(
|
||
"OK",
|
||
[
|
||
(
|
||
b"2 (RFC822 {100}",
|
||
_make_raw_email(subject="Second", msg_id="<m2@test>"),
|
||
)
|
||
],
|
||
), # FETCH 2
|
||
("OK", []), # STORE 2
|
||
]
|
||
conn.close.return_value = ("OK", [])
|
||
conn.logout.return_value = ("OK", [])
|
||
|
||
store = MagicMock()
|
||
store.upsert.return_value = "KEY1"
|
||
# No pre-existing item to alias against — the resend-dedup lookup
|
||
# in _ingest_message must find nothing for these mocked messages.
|
||
store._con.return_value.execute.return_value.fetchone.return_value = None
|
||
|
||
mb = Mailbox(
|
||
host="mail.test", port=993, username="test@test.com", password="pw"
|
||
)
|
||
stats = ingest(store, mb, scratch_root=Path("/tmp/test-ingest"))
|
||
assert stats["seen"] == 2
|
||
assert stats["ingested"] == 2
|
||
assert stats["errors"] == 0
|
||
assert store.upsert.call_count == 2
|
||
|
||
@patch("bib.email_ingest.imaplib.IMAP4_SSL")
|
||
def test_empty_inbox(self, MockIMAP):
|
||
conn = MockIMAP.return_value
|
||
conn.login.return_value = ("OK", [])
|
||
conn.select.return_value = ("OK", [])
|
||
conn.uid.return_value = ("OK", [b""])
|
||
conn.close.return_value = ("OK", [])
|
||
conn.logout.return_value = ("OK", [])
|
||
|
||
store = MagicMock()
|
||
mb = Mailbox(
|
||
host="mail.test", port=993, username="test@test.com", password="pw"
|
||
)
|
||
stats = ingest(store, mb)
|
||
assert stats["seen"] == 0
|
||
assert stats["ingested"] == 0
|
||
|
||
@patch("bib.email_ingest.imaplib.IMAP4_SSL")
|
||
def test_limit(self, MockIMAP):
|
||
conn = MockIMAP.return_value
|
||
conn.login.return_value = ("OK", [])
|
||
conn.select.return_value = ("OK", [])
|
||
conn.uid.side_effect = [
|
||
("OK", [b"1 2 3 4 5"]), # SEARCH returns 5
|
||
("OK", [(b"1 (RFC822 {100}", _make_raw_email(msg_id="<l1@test>"))]),
|
||
("OK", []),
|
||
]
|
||
conn.close.return_value = ("OK", [])
|
||
conn.logout.return_value = ("OK", [])
|
||
|
||
store = MagicMock()
|
||
store.upsert.return_value = "K"
|
||
store._con.return_value.execute.return_value.fetchone.return_value = None
|
||
mb = Mailbox(host="h", port=993, username="u@d", password="p")
|
||
stats = ingest(store, mb, limit=1)
|
||
assert stats["seen"] == 1
|
||
assert stats["ingested"] == 1
|
||
|
||
|
||
# ── _ingest_message: content-hash aliasing of listserv re-sends ──
|
||
|
||
|
||
@pytest.fixture
|
||
def store():
|
||
return Store(":memory:")
|
||
|
||
|
||
@pytest.fixture
|
||
def mailbox():
|
||
return Mailbox(
|
||
host="mail.test", port=993, username="cmslists@test.com", password="pw"
|
||
)
|
||
|
||
|
||
def make_msg(subject: str, body: str, mid: str, date: str):
|
||
m = email.message.EmailMessage()
|
||
m["Subject"] = subject
|
||
m["From"] = "CMS Updates <cmslists@subscriptions.cms.hhs.gov>"
|
||
m["Message-ID"] = f"<{mid}>"
|
||
m["Date"] = date
|
||
m.set_content(body)
|
||
return m
|
||
|
||
|
||
def _raw_extra_json(store: Store, key: str) -> dict:
|
||
"""Read extra_json straight off the row.
|
||
|
||
``Source`` only declares ``doc_type`` — fields packed into
|
||
``extra_json`` that aren't declared on the model (``body_sha1``,
|
||
``alias_mids``) get silently dropped by pydantic's ``extra="ignore"``
|
||
default when the row round-trips through ``Item.from_row()`` /
|
||
``to_row()``. Reading the column directly is the only way to see
|
||
what's actually persisted.
|
||
"""
|
||
row = (
|
||
store._con()
|
||
.execute( # noqa: SLF001
|
||
"SELECT extra_json FROM items WHERE key = ?", (key,)
|
||
)
|
||
.fetchone()
|
||
)
|
||
return json.loads(row["extra_json"] or "{}")
|
||
|
||
|
||
class TestResendAliasing:
|
||
def test_same_body_new_mid_aliases_instead_of_new_item(
|
||
self, store, mailbox, tmp_path
|
||
):
|
||
m1 = make_msg(
|
||
"Upcoming iQIES Hold Times",
|
||
"Same body.",
|
||
"aaa@x.example",
|
||
"Mon, 27 Apr 2026 10:00:00 -0400",
|
||
)
|
||
m2 = make_msg(
|
||
"Upcoming iQIES Hold Times",
|
||
"Same body.",
|
||
"bbb@x.example",
|
||
"Fri, 10 Jul 2026 10:00:00 -0400",
|
||
)
|
||
_ingest_message(store, mailbox, m1, tmp_path)
|
||
_ingest_message(store, mailbox, m2, tmp_path)
|
||
items = store.list_items(query="iQIES")
|
||
assert len(items) == 1
|
||
ej = _raw_extra_json(store, items[0].key)
|
||
assert "bbb@x.example" in ej.get("alias_mids", [])
|
||
|
||
def test_resend_is_idempotent(self, store, mailbox, tmp_path):
|
||
"""Ingesting the identical alias message twice doesn't duplicate
|
||
its Message-ID in alias_mids."""
|
||
m1 = make_msg(
|
||
"Idempotency Check",
|
||
"Same body.",
|
||
"eee@x.example",
|
||
"Mon, 27 Apr 2026 10:00:00 -0400",
|
||
)
|
||
m2 = make_msg(
|
||
"Idempotency Check",
|
||
"Same body.",
|
||
"fff@x.example",
|
||
"Fri, 10 Jul 2026 10:00:00 -0400",
|
||
)
|
||
_ingest_message(store, mailbox, m1, tmp_path)
|
||
_ingest_message(store, mailbox, m2, tmp_path)
|
||
_ingest_message(store, mailbox, m2, tmp_path)
|
||
items = store.list_items(query="Idempotency")
|
||
assert len(items) == 1
|
||
ej = _raw_extra_json(store, items[0].key)
|
||
assert ej.get("alias_mids", []).count("fff@x.example") == 1
|
||
|
||
def test_different_body_same_subject_stays_separate(self, store, mailbox, tmp_path):
|
||
m1 = make_msg(
|
||
"Weekly Digest",
|
||
"Body one.",
|
||
"ccc@x.example",
|
||
"Mon, 27 Apr 2026 10:00:00 -0400",
|
||
)
|
||
m2 = make_msg(
|
||
"Weekly Digest",
|
||
"Body two.",
|
||
"ddd@x.example",
|
||
"Fri, 10 Jul 2026 10:00:00 -0400",
|
||
)
|
||
_ingest_message(store, mailbox, m1, tmp_path)
|
||
_ingest_message(store, mailbox, m2, tmp_path)
|
||
assert len(store.list_items(query="Weekly Digest")) == 2
|
||
|
||
def test_new_item_records_body_sha1(self, store, mailbox, tmp_path):
|
||
m1 = make_msg(
|
||
"First Send",
|
||
"Only body.",
|
||
"ggg@x.example",
|
||
"Mon, 27 Apr 2026 10:00:00 -0400",
|
||
)
|
||
_ingest_message(store, mailbox, m1, tmp_path)
|
||
items = store.list_items(query="First Send")
|
||
assert len(items) == 1
|
||
ej = _raw_extra_json(store, items[0].key)
|
||
assert len(ej.get("body_sha1", "")) == 40
|
||
|
||
def test_same_mid_reingest_does_not_self_alias(self, store, mailbox, tmp_path):
|
||
"""Re-ingesting the identical message (same Message-ID — e.g. IMAP
|
||
Seen state was lost) must fall through to the normal upsert-by-url
|
||
update path, not match itself in the resend-dedup lookup and
|
||
record its own mid as an alias of itself."""
|
||
m = make_msg(
|
||
"Same Message Twice",
|
||
"Body content.",
|
||
"same-mid@x.example",
|
||
"Mon, 27 Apr 2026 10:00:00 -0400",
|
||
)
|
||
_ingest_message(store, mailbox, m, tmp_path)
|
||
with patch.object(store, "upsert", wraps=store.upsert) as spy_upsert:
|
||
_ingest_message(store, mailbox, m, tmp_path)
|
||
assert spy_upsert.called, (
|
||
"second ingest must go through the normal upsert-by-url path,"
|
||
" not the alias short-circuit"
|
||
)
|
||
items = store.list_items(query="Same Message Twice")
|
||
assert len(items) == 1
|
||
ej = _raw_extra_json(store, items[0].key)
|
||
assert "same-mid@x.example" not in ej.get("alias_mids", [])
|
||
|
||
def test_empty_body_attachment_only_resends_stay_separate(
|
||
self, store, mailbox, tmp_path
|
||
):
|
||
"""Attachment-only emails have an empty extracted body, so
|
||
body_sha1 is the constant sha1(""). Two such messages under the
|
||
same subject must NOT alias together — aliasing would hit the
|
||
early return in _ingest_message and silently drop the second
|
||
message's attachments."""
|
||
m1 = make_msg(
|
||
"Attachment Only",
|
||
"",
|
||
"att1@x.example",
|
||
"Mon, 27 Apr 2026 10:00:00 -0400",
|
||
)
|
||
m2 = make_msg(
|
||
"Attachment Only",
|
||
" \n ",
|
||
"att2@x.example",
|
||
"Fri, 10 Jul 2026 10:00:00 -0400",
|
||
)
|
||
_ingest_message(store, mailbox, m1, tmp_path)
|
||
_ingest_message(store, mailbox, m2, tmp_path)
|
||
items = store.list_items(query="Attachment Only")
|
||
assert len(items) == 2
|
||
|
||
def test_alias_resend_tags_merge_into_keeper(self, store, tmp_path):
|
||
"""A re-send arriving via a different mailbox must contribute its
|
||
mailbox:/sender:/list:/year: tags to the keeper item, not just
|
||
record its Message-ID as an alias."""
|
||
mb1 = Mailbox(
|
||
host="mail.test", port=993, username="cmslists@test.com", password="pw"
|
||
)
|
||
mb2 = Mailbox(
|
||
host="mail.test", port=993, username="otherlist@test.com", password="pw"
|
||
)
|
||
m1 = make_msg(
|
||
"Cross Mailbox Resend",
|
||
"Same body.",
|
||
"hhh@x.example",
|
||
"Mon, 27 Apr 2026 10:00:00 -0400",
|
||
)
|
||
m2 = make_msg(
|
||
"Cross Mailbox Resend",
|
||
"Same body.",
|
||
"iii@x.example",
|
||
"Fri, 10 Jul 2026 10:00:00 -0400",
|
||
)
|
||
_ingest_message(store, mb1, m1, tmp_path)
|
||
_ingest_message(store, mb2, m2, tmp_path)
|
||
items = store.list_items(query="Cross Mailbox Resend")
|
||
assert len(items) == 1
|
||
assert "mailbox:cmslists" in items[0].tags
|
||
assert "mailbox:otherlist" in items[0].tags
|