104 lines
3.9 KiB
Python
104 lines
3.9 KiB
Python
"""One-off: backfill ``body_sha1`` onto legacy ``source:email`` rows.
|
|
|
|
Ingest dedup (``bib/email_ingest.py``) matches candidate re-sends only
|
|
on ``json_extract(extra_json,'$.body_sha1')``. Legacy ``source:email``
|
|
rows created before Task 3's ingest-side dedup landed have no stored
|
|
``body_sha1``, so the first post-deploy re-send of a legacy notice
|
|
misses the match and creates a fresh item instead of aliasing onto the
|
|
existing one.
|
|
|
|
This backfills ``extra_json.body_sha1`` for every ``source:email`` item
|
|
that's missing it, computed the same way ``email_ingest.py`` does:
|
|
``sha1(" ".join(body.split()).encode()).hexdigest()``. The catch is
|
|
that ``abstract`` on the stored row is ``body[:4000]`` (see
|
|
``_ingest_message``) — for a message whose body was actually cut at
|
|
4000 chars, hashing the truncated abstract does NOT reproduce the hash
|
|
that would be computed from the full original body, so any such item
|
|
would get a *wrong* body_sha1 that can never match a genuine re-send.
|
|
Those items are skipped (``skipped_truncated``) rather than backfilled
|
|
with a hash that would silently misbehave forever.
|
|
|
|
Truncation heuristic: ``len(abstract) < 3999`` (not ``< 4000``) —
|
|
leaves a 1-char margin since the exact boundary between "the body was
|
|
precisely 4000 chars and happened to not get cut" and "the body was
|
|
longer and got cut to 4000" is not distinguishable from the stored
|
|
abstract alone; treating anything at-or-near the cap as possibly
|
|
truncated is the safe default (a false skip just means the row keeps
|
|
missing body_sha1 and gets caught by the next dedupe pass some other
|
|
way; a false backfill would poison dedup with a bad hash forever).
|
|
|
|
Reuses the raw-SQL extra_json read/merge/write convention from
|
|
``dedupe_bib_legacy.py`` and ``email_ingest.py`` — ``Item.to_row()``
|
|
only serializes the fields a subclass declares (``Source`` only knows
|
|
``doc_type``) and would silently drop ``body_sha1`` if a merge went
|
|
through it instead of raw SQL.
|
|
|
|
``--dry-run`` is the default; pass ``--live`` to actually write.
|
|
|
|
Usage: uv run python dev/scripts/backfill_body_sha1.py [--live]
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import sys
|
|
from typing import Any
|
|
|
|
TRUNCATION_THRESHOLD = 3999
|
|
|
|
|
|
def _body_hash(text: str) -> str:
|
|
return hashlib.sha1(" ".join((text or "").split()).encode()).hexdigest()
|
|
|
|
|
|
def backfill_body_sha1(store: Any, *, dry_run: bool = True) -> dict:
|
|
"""Backfill ``body_sha1`` onto legacy ``source:email`` rows.
|
|
|
|
Returns ``{candidates, backfilled, skipped_truncated}``.
|
|
``candidates`` counts every ``source:email`` row missing
|
|
``body_sha1`` (whether or not it ends up backfilled);
|
|
``skipped_truncated`` is the subset whose ``abstract`` looks like
|
|
it was cut by email_ingest's ``body[:4000]`` cap and is therefore
|
|
excluded from ``backfilled``.
|
|
"""
|
|
con = store._con() # noqa: SLF001
|
|
out = {"candidates": 0, "backfilled": 0, "skipped_truncated": 0}
|
|
|
|
rows = con.execute(
|
|
"SELECT i.key, i.abstract, i.extra_json FROM items i"
|
|
" JOIN item_tags it ON it.item_id = i.id"
|
|
" JOIN tags t ON t.id = it.tag_id AND t.name = 'source:email'"
|
|
" WHERE i.item_type = 'source'"
|
|
" AND json_extract(i.extra_json, '$.body_sha1') IS NULL"
|
|
" ORDER BY i.id"
|
|
).fetchall()
|
|
|
|
for key, abstract, extra_json in rows:
|
|
out["candidates"] += 1
|
|
abstract = abstract or ""
|
|
if len(abstract) >= TRUNCATION_THRESHOLD:
|
|
out["skipped_truncated"] += 1
|
|
continue
|
|
|
|
if dry_run:
|
|
continue
|
|
|
|
ej = json.loads(extra_json or "{}")
|
|
ej["body_sha1"] = _body_hash(abstract)
|
|
store.update(key, extra_json=json.dumps(ej))
|
|
out["backfilled"] += 1
|
|
|
|
return out
|
|
|
|
|
|
if __name__ == "__main__":
|
|
from bib import Store
|
|
|
|
live = "--live" in sys.argv
|
|
_store = Store()
|
|
_result = backfill_body_sha1(_store, dry_run=not live)
|
|
print(json.dumps(_result, indent=1))
|
|
if not live:
|
|
print("(dry run — pass --live to write)")
|