137 lines
4.9 KiB
Python
137 lines
4.9 KiB
Python
"""One-off: merge legacy duplicate rows inside bib.sqlite.
|
|
|
|
Class C — same-url rows created before Store.upsert deduped by URL
|
|
(#624/91c3306). Class B — listserv re-sends: same (title, body-hash)
|
|
among source:email items, each under a distinct email:<Message-ID>
|
|
url. Both classes predate Task 3's ingest-side dedup, so legacy email
|
|
rows have no stored ``body_sha1`` in ``extra_json`` — Class B grouping
|
|
is instead computed in Python from title + whitespace-normalized
|
|
abstract, matching bib/email_ingest.py's hash recipe.
|
|
|
|
Keep-earliest (by created_at, then id); union tags/collections into
|
|
the keeper; alias surplus Message-IDs into the keeper's
|
|
``extra_json["alias_mids"]`` (idempotent — matches the
|
|
email_ingest.py convention: read/write extra_json via raw SQL, never
|
|
through Item.to_row(), since to_row() round-trips through the
|
|
item's declared pydantic fields and would silently drop
|
|
alias_mids/body_sha1 for item types that don't declare them, e.g.
|
|
Source). Delete surplus rows. Prints stats; --dry-run is the default.
|
|
|
|
Returns keeper/loser key pairs so the Zotero-side merge (zot.merge,
|
|
Task 5) can be driven off the same decisions.
|
|
|
|
Usage: uv run python dev/scripts/dedupe_bib_legacy.py [--live]
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import sys
|
|
from collections import defaultdict
|
|
from typing import Any
|
|
|
|
|
|
def _body_hash(abstract: str) -> str:
|
|
return hashlib.sha1(" ".join((abstract or "").split()).encode()).hexdigest()
|
|
|
|
|
|
def dedupe_bib(store: Any, *, dry_run: bool = True) -> dict:
|
|
"""Merge legacy duplicate rows in ``store``.
|
|
|
|
Returns ``{url_groups, email_groups, removed, aliased, pairs}``
|
|
where ``pairs`` is a list of ``(keeper_key, loser_key)`` tuples
|
|
describing every merge decision made (or, under ``dry_run``, that
|
|
would be made).
|
|
"""
|
|
con = store._con()
|
|
out: dict[str, Any] = {
|
|
"url_groups": 0,
|
|
"email_groups": 0,
|
|
"removed": 0,
|
|
"aliased": 0,
|
|
"pairs": [],
|
|
}
|
|
|
|
groups: list[list[str]] = [] # each: [keeper_key, loser_key, ...]
|
|
|
|
# Class C: exact-url groups (http only; email: urls are unique per
|
|
# Message-ID and are handled by the content-based Class B pass below).
|
|
by_url: dict[str, list[str]] = defaultdict(list)
|
|
for key, url in con.execute(
|
|
"SELECT key, url FROM items WHERE url LIKE 'http%' ORDER BY created_at, id"
|
|
):
|
|
by_url[url].append(key)
|
|
for keys in by_url.values():
|
|
if len(keys) > 1:
|
|
out["url_groups"] += 1
|
|
groups.append(keys)
|
|
|
|
# Class B: (title, body-hash) among source:email items.
|
|
by_content: dict[tuple[str, str], list[str]] = defaultdict(list)
|
|
for key, title, abstract in con.execute(
|
|
"SELECT i.key, i.title, COALESCE(i.abstract,'') FROM items i"
|
|
" JOIN item_tags it ON it.item_id = i.id"
|
|
" JOIN tags t ON t.id = it.tag_id AND t.name = 'source:email'"
|
|
" ORDER BY i.created_at, i.id"
|
|
):
|
|
by_content[(title, _body_hash(abstract))].append(key)
|
|
for keys in by_content.values():
|
|
if len(keys) > 1:
|
|
out["email_groups"] += 1
|
|
groups.append(keys)
|
|
|
|
for keys in groups:
|
|
keeper_key, losers = keys[0], keys[1:]
|
|
if dry_run:
|
|
continue
|
|
|
|
keeper = store.get(keeper_key)
|
|
# Read extra_json via raw SQL, not keeper.to_row() — to_row()
|
|
# rebuilds extra_json from the item's declared subclass fields
|
|
# only (e.g. Source only knows doc_type) and would silently
|
|
# drop any alias_mids/body_sha1 already stored on the row.
|
|
row = con.execute(
|
|
"SELECT extra_json FROM items WHERE key = ?", (keeper_key,)
|
|
).fetchone()
|
|
ej = json.loads((row["extra_json"] if row else "") or "{}")
|
|
aliases = list(ej.get("alias_mids", []))
|
|
tags = list(keeper.tags)
|
|
collections = list(keeper.collections)
|
|
|
|
for lk in losers:
|
|
loser = store.get(lk)
|
|
tags += [t for t in loser.tags if t not in tags]
|
|
collections += [c for c in loser.collections if c not in collections]
|
|
if loser.url.startswith("email:"):
|
|
mid = loser.url[len("email:") :]
|
|
if mid not in aliases:
|
|
aliases.append(mid)
|
|
out["aliased"] += 1
|
|
out["pairs"].append((keeper_key, lk))
|
|
store.delete(lk)
|
|
out["removed"] += 1
|
|
|
|
if aliases:
|
|
ej["alias_mids"] = aliases
|
|
store.update(
|
|
keeper_key,
|
|
tags=tags,
|
|
collections=collections,
|
|
extra_json=json.dumps(ej),
|
|
)
|
|
|
|
return out
|
|
|
|
|
|
if __name__ == "__main__":
|
|
from bib import Store
|
|
|
|
live = "--live" in sys.argv
|
|
_store = Store()
|
|
result = dedupe_bib(_store, dry_run=not live)
|
|
_pairs = result.pop("pairs")
|
|
print(json.dumps(result, indent=1))
|
|
for keeper, loser in _pairs:
|
|
print(f"pair {keeper} <- {loser}")
|