148 lines
4.7 KiB
Python
148 lines
4.7 KiB
Python
"""Tests for dev/scripts/backfill_body_sha1.py.
|
|
|
|
Legacy ``source:email`` rows created before Task 3's ingest-side dedup
|
|
have no ``body_sha1`` in ``extra_json``, so the first post-deploy
|
|
re-send of a legacy notice creates a fresh item instead of aliasing.
|
|
This script backfills ``body_sha1`` for legacy rows whose ``abstract``
|
|
wasn't truncated by email_ingest's ``body[:4000]`` cap, using the exact
|
|
same normalization + hash recipe as ``bib.email_ingest``.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from bib.item import Source
|
|
from bib.store import Store
|
|
|
|
# ``dev/scripts`` isn't a package on sys.path — load the module by path,
|
|
# matching the convention other one-off dev scripts use in this repo.
|
|
_SCRIPT_PATH = (
|
|
Path(__file__).resolve().parents[2] / "dev" / "scripts" / "backfill_body_sha1.py"
|
|
)
|
|
_spec = importlib.util.spec_from_file_location("backfill_body_sha1", _SCRIPT_PATH)
|
|
_mod = importlib.util.module_from_spec(_spec)
|
|
sys.modules[_spec.name] = _mod
|
|
_spec.loader.exec_module(_mod)
|
|
backfill_body_sha1 = _mod.backfill_body_sha1
|
|
|
|
|
|
def _body_hash(text: str) -> str:
|
|
return hashlib.sha1(" ".join((text or "").split()).encode()).hexdigest()
|
|
|
|
|
|
def _raw_extra_json(store: Store, key: str) -> dict:
|
|
row = (
|
|
store._con() # noqa: SLF001
|
|
.execute("SELECT extra_json FROM items WHERE key = ?", (key,))
|
|
.fetchone()
|
|
)
|
|
return json.loads(row["extra_json"] or "{}")
|
|
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path):
|
|
db_path = tmp_path / "test-backfill.sqlite"
|
|
s = Store(db_path)
|
|
yield s
|
|
s.close()
|
|
|
|
|
|
def _make_email_item(
|
|
store: Store, *, title: str, abstract: str, url: str, with_body_sha1: bool = False
|
|
) -> str:
|
|
item = Source(title=title, url=url, abstract=abstract)
|
|
item.doc_type = "Email"
|
|
item.add_tag("source:email")
|
|
key = store.create(item)
|
|
if with_body_sha1:
|
|
ej = _raw_extra_json(store, key)
|
|
ej["body_sha1"] = _body_hash(abstract)
|
|
store.update(key, extra_json=json.dumps(ej))
|
|
return key
|
|
|
|
|
|
class TestBackfillBodySha1:
|
|
def test_backfills_untruncated_item(self, store):
|
|
key = _make_email_item(
|
|
store,
|
|
title="Legacy Notice",
|
|
abstract="Some legacy body content.",
|
|
url="email:legacy1@x.example",
|
|
)
|
|
result = backfill_body_sha1(store, dry_run=False)
|
|
assert result["candidates"] == 1
|
|
assert result["backfilled"] == 1
|
|
assert result["skipped_truncated"] == 0
|
|
|
|
ej = _raw_extra_json(store, key)
|
|
assert ej["body_sha1"] == _body_hash("Some legacy body content.")
|
|
|
|
def test_skips_truncated_item(self, store):
|
|
long_abstract = "x" * 4000
|
|
key = _make_email_item(
|
|
store,
|
|
title="Long Legacy Notice",
|
|
abstract=long_abstract,
|
|
url="email:legacy2@x.example",
|
|
)
|
|
result = backfill_body_sha1(store, dry_run=False)
|
|
assert result["candidates"] == 1
|
|
assert result["backfilled"] == 0
|
|
assert result["skipped_truncated"] == 1
|
|
|
|
ej = _raw_extra_json(store, key)
|
|
assert "body_sha1" not in ej
|
|
|
|
def test_skips_item_with_existing_body_sha1(self, store):
|
|
_make_email_item(
|
|
store,
|
|
title="Already Hashed",
|
|
abstract="Already has a hash.",
|
|
url="email:legacy3@x.example",
|
|
with_body_sha1=True,
|
|
)
|
|
result = backfill_body_sha1(store, dry_run=False)
|
|
assert result["candidates"] == 0
|
|
assert result["backfilled"] == 0
|
|
|
|
def test_dry_run_makes_no_changes(self, store):
|
|
key = _make_email_item(
|
|
store,
|
|
title="Dry Run Notice",
|
|
abstract="Body content.",
|
|
url="email:legacy4@x.example",
|
|
)
|
|
result = backfill_body_sha1(store, dry_run=True)
|
|
assert result["candidates"] == 1
|
|
assert result["backfilled"] == 0
|
|
|
|
ej = _raw_extra_json(store, key)
|
|
assert "body_sha1" not in ej
|
|
|
|
def test_second_run_is_idempotent(self, store):
|
|
_make_email_item(
|
|
store,
|
|
title="Idempotent Notice",
|
|
abstract="Body content.",
|
|
url="email:legacy5@x.example",
|
|
)
|
|
first = backfill_body_sha1(store, dry_run=False)
|
|
assert first["backfilled"] == 1
|
|
|
|
second = backfill_body_sha1(store, dry_run=False)
|
|
assert second["candidates"] == 0
|
|
assert second["backfilled"] == 0
|
|
|
|
def test_ignores_non_email_items(self, store):
|
|
item = Source(title="Not an email", url="https://example.com/x")
|
|
store.create(item)
|
|
result = backfill_body_sha1(store, dry_run=False)
|
|
assert result["candidates"] == 0
|