stack/dev/scripts/migrate_zotero.py

"""One-time migration from Zotero SQLite (EAV) to bib SQLite (denormalized).

Reads ``zotero.sqlite`` (61-table EAV schema with 4-5 joins per item)
and writes ``bib.sqlite`` (8-table denormalized schema).

Usage::

    uv run python dev/scripts/migrate_zotero.py \\
        --src zotero/data/zotero.sqlite \\
        --dst data/bib.sqlite

Mapping:
    Zotero itemType → bib item_type:
    - statute (code="FR")    → rule
    - statute (code="C.F.R.") → regulation
    - report                  → manual
    - webpage                 → download
    - document                → source
"""

from __future__ import annotations

import argparse
import json
import sqlite3
import sys
from pathlib import Path

# Ensure project root is on sys.path for bib imports
sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "src"))

from bib.store import Store  # noqa: E402


def _read_zotero(src: str) -> sqlite3.Connection:
    """Open Zotero SQLite read-only."""
    con = sqlite3.connect(f"file:{src}?mode=ro", uri=True)
    con.row_factory = sqlite3.Row
    return con


def _get_item_fields(zcon: sqlite3.Connection, item_id: int) -> dict:
    """Read all fields for a Zotero item via the EAV join."""
    rows = zcon.execute(
        """SELECT f.fieldName, idv.value
           FROM itemData id
           JOIN fields f ON id.fieldID = f.fieldID
           JOIN itemDataValues idv ON id.valueID = idv.valueID
           WHERE id.itemID = ?""",
        (item_id,),
    ).fetchall()
    return {r["fieldName"]: r["value"] for r in rows}


def _get_item_tags(zcon: sqlite3.Connection, item_id: int) -> list[str]:
    """Read tags for a Zotero item."""
    rows = zcon.execute(
        """SELECT t.name FROM itemTags it
           JOIN tags t ON it.tagID = t.tagID
           WHERE it.itemID = ?""",
        (item_id,),
    ).fetchall()
    return [r["name"] for r in rows]


def _get_item_collections(zcon: sqlite3.Connection, item_id: int) -> list[str]:
    """Read collection keys for a Zotero item."""
    rows = zcon.execute(
        """SELECT c.key FROM collectionItems ci
           JOIN collections c ON ci.collectionID = c.collectionID
           WHERE ci.itemID = ?""",
        (item_id,),
    ).fetchall()
    return [r["key"] for r in rows]


def _get_item_creators(zcon: sqlite3.Connection, item_id: int) -> list[dict]:
    """Read creators for a Zotero item."""
    rows = zcon.execute(
        """SELECT c.firstName, c.lastName, ct.creatorType,
                  ic.orderIndex
           FROM itemCreators ic
           JOIN creators c ON ic.creatorID = c.creatorID
           JOIN creatorTypes ct ON ic.creatorTypeID = ct.creatorTypeID
           WHERE ic.itemID = ?
           ORDER BY ic.orderIndex""",
        (item_id,),
    ).fetchall()
    return [
        {
            "first_name": r["firstName"] or "",
            "last_name": r["lastName"] or "",
            "role": r["creatorType"] or "author",
            "sort_order": r["orderIndex"],
        }
        for r in rows
    ]


def _classify_item(zotero_type: str, fields: dict) -> str:
    """Map Zotero itemType + fields to bib item_type."""
    if zotero_type == "statute":
        code = fields.get("code", "")
        if "C.F.R." in code:
            return "regulation"
        return "rule"
    if zotero_type == "report":
        return "manual"
    if zotero_type == "webpage":
        return "download"
    if zotero_type == "document":
        return "source"
    # Fallback
    return "source"


def _build_extra_json(item_type: str, fields: dict) -> str:
    """Build the extra_json blob from Zotero fields."""
    if item_type == "rule":
        history = fields.get("history", "")
        doc_num = ""
        rule_type = ""
        eff_date = ""
        for part in history.split("; "):
            if part.startswith("Document: "):
                doc_num = part[10:]
            elif part.startswith("Type: "):
                rule_type = part[6:]
            elif part.startswith("Effective: "):
                eff_date = part[11:]
        return json.dumps(
            {
                "fr_volume": fields.get("codeNumber", ""),
                "fr_page": fields.get("pages", ""),
                "document_number": doc_num,
                "cms_id": fields.get("session", ""),
                "rule_type": rule_type,
                "effective_date": eff_date,
            }
        )

    if item_type == "regulation":
        history = fields.get("history", "")
        part = ""
        authority = ""
        for segment in history.split("; "):
            if segment.startswith("Part "):
                part = segment[5:]
            elif segment.startswith("Authority: "):
                authority = segment[11:]
        return json.dumps(
            {
                "cfr_title": fields.get("codeNumber", ""),
                "cfr_part": part,
                "cfr_section": fields.get("section", ""),
                "authority": authority,
                "effective_date": fields.get("dateEnacted", ""),
            }
        )

    if item_type == "manual":
        extra = fields.get("extra", "")
        transmittal = ""
        for line in extra.split("\n"):
            if line.startswith("Transmittal: "):
                transmittal = line[13:]
        chapter = fields.get("seriesNumber", "")
        if chapter.startswith("Chapter "):
            chapter = chapter[8:]
        return json.dumps(
            {
                "manual_name": fields.get("seriesTitle", ""),
                "pub_number": fields.get("reportNumber", ""),
                "chapter": chapter,
                "transmittal": transmittal,
            }
        )

    if item_type == "download":
        extra = fields.get("extra", "")
        file_urls: list[str] = []
        for line in extra.split("\n"):
            if line.startswith("Files: "):
                file_urls = [u.strip() for u in line[7:].split(";") if u.strip()]
        return json.dumps(
            {
                "page_type": "",
                "file_urls": file_urls,
                "year": None,
                "quarter": "",
                "website_title": fields.get("websiteTitle", ""),
            }
        )

    if item_type == "source":
        return json.dumps(
            {
                "doc_type": fields.get("type", ""),
            }
        )

    return "{}"


def _get_title(item_type: str, fields: dict) -> str:
    """Extract the title field (differs by Zotero itemType)."""
    if item_type in ("rule", "regulation"):
        return fields.get("nameOfAct", "")
    return fields.get("title", "")


def _get_date(item_type: str, fields: dict) -> str:
    """Extract the publication date."""
    if item_type in ("rule", "regulation"):
        return fields.get("dateEnacted", "")
    return fields.get("date", "")


def migrate(src: str, dst: str) -> dict[str, int]:
    """Run the migration. Returns count summary."""
    zcon = _read_zotero(src)
    store = Store(dst)
    bcon = store._con()

    counts = {
        "items": 0,
        "tags": 0,
        "collections": 0,
        "attachments": 0,
        "notes": 0,
        "creators": 0,
        "skipped": 0,
    }

    # ── 1. Collections ───────────────────────────────────────
    # Build a mapping of Zotero collectionID → bib collection id
    zot_collections = zcon.execute(
        """SELECT collectionID, collectionName, key, parentCollectionID
           FROM collections WHERE libraryID = 1
           ORDER BY parentCollectionID NULLS FIRST, collectionName"""
    ).fetchall()

    zot_col_id_to_key: dict[int, str] = {}
    for zc in zot_collections:
        zot_col_id_to_key[zc["collectionID"]] = zc["key"]

    for zc in zot_collections:
        parent_id = None
        if zc["parentCollectionID"]:
            parent_key = zot_col_id_to_key.get(zc["parentCollectionID"])
            if parent_key:
                row = bcon.execute(
                    "SELECT id FROM collections WHERE key = ?",
                    (parent_key,),
                ).fetchone()
                if row:
                    parent_id = row["id"]

        bcon.execute(
            "INSERT OR IGNORE INTO collections (key, name, parent_id) VALUES (?, ?, ?)",
            (zc["key"], zc["collectionName"], parent_id),
        )
        counts["collections"] += 1

    bcon.commit()

    # ── 2. Content items ─────────────────────────────────────
    content_items = zcon.execute(
        """SELECT i.itemID, i.key, it.typeName
           FROM items i
           JOIN itemTypes it ON i.itemTypeID = it.itemTypeID
           WHERE i.libraryID = 1
             AND i.itemID NOT IN (SELECT itemID FROM deletedItems)
             AND it.typeName NOT IN ('attachment', 'note')
           ORDER BY i.itemID"""
    ).fetchall()

    for zi in content_items:
        fields = _get_item_fields(zcon, zi["itemID"])
        zot_type = zi["typeName"]

        item_type = _classify_item(zot_type, fields)
        title = _get_title(item_type, fields)
        date_pub = _get_date(item_type, fields)
        url = fields.get("url", "")
        access_date = fields.get("accessDate", "")
        abstract = fields.get("abstractNote", "")
        institution = fields.get("institution", "") or fields.get("publisher", "")
        extra = fields.get("extra", "")
        extra_json = _build_extra_json(item_type, fields)

        bcon.execute(
            """INSERT OR IGNORE INTO items
               (key, item_type, title, url, date_published,
                access_date, abstract, institution, extra, extra_json)
               VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
            (
                zi["key"],
                item_type,
                title,
                url,
                date_pub,
                access_date,
                abstract,
                institution,
                extra,
                extra_json,
            ),
        )

        bib_item = bcon.execute(
            "SELECT id FROM items WHERE key = ?", (zi["key"],)
        ).fetchone()
        if bib_item is None:
            counts["skipped"] += 1
            continue

        bib_id = bib_item["id"]
        counts["items"] += 1

        # Tags
        ztags = _get_item_tags(zcon, zi["itemID"])
        for tag_name in ztags:
            tag_id = store._ensure_tag(tag_name)
            bcon.execute(
                "INSERT OR IGNORE INTO item_tags (item_id, tag_id) VALUES (?, ?)",
                (bib_id, tag_id),
            )
            counts["tags"] += 1

        # Collections
        zcols = _get_item_collections(zcon, zi["itemID"])
        for ckey in zcols:
            crow = bcon.execute(
                "SELECT id FROM collections WHERE key = ?", (ckey,)
            ).fetchone()
            if crow:
                bcon.execute(
                    "INSERT OR IGNORE INTO collection_items "
                    "(collection_id, item_id) VALUES (?, ?)",
                    (crow["id"], bib_id),
                )

        # Creators
        creators = _get_item_creators(zcon, zi["itemID"])
        for cr in creators:
            # Find or create creator
            crow = bcon.execute(
                "SELECT id FROM creators WHERE first_name = ? AND last_name = ?",
                (cr["first_name"], cr["last_name"]),
            ).fetchone()
            if crow:
                creator_id = crow["id"]
            else:
                cur = bcon.execute(
                    "INSERT INTO creators (first_name, last_name) VALUES (?, ?)",
                    (cr["first_name"], cr["last_name"]),
                )
                creator_id = cur.lastrowid
                counts["creators"] += 1

            bcon.execute(
                "INSERT OR IGNORE INTO item_creators "
                "(item_id, creator_id, role, sort_order) "
                "VALUES (?, ?, ?, ?)",
                (bib_id, creator_id, cr["role"], cr["sort_order"]),
            )

    bcon.commit()

    # ── 3. Attachments ───────────────────────────────────────
    zot_attachments = zcon.execute(
        """SELECT ia.itemID, ia.parentItemID, ia.path,
                  ia.contentType, i.key
           FROM itemAttachments ia
           JOIN items i ON ia.itemID = i.itemID
           WHERE i.libraryID = 1
             AND ia.parentItemID IS NOT NULL
             AND i.itemID NOT IN (SELECT itemID FROM deletedItems)"""
    ).fetchall()

    for za in zot_attachments:
        # Find parent in bib
        parent_row = zcon.execute(
            "SELECT key FROM items WHERE itemID = ?",
            (za["parentItemID"],),
        ).fetchone()
        if parent_row is None:
            continue

        bib_parent = bcon.execute(
            "SELECT id FROM items WHERE key = ?", (parent_row["key"],)
        ).fetchone()
        if bib_parent is None:
            continue

        # Extract filename from Zotero storage path
        zpath = za["path"] or ""
        filename = zpath.replace("storage:", "") if zpath else za["key"]

        bcon.execute(
            """INSERT OR IGNORE INTO attachments
               (item_id, key, filename, content_type, storage_path)
               VALUES (?, ?, ?, ?, ?)""",
            (
                bib_parent["id"],
                za["key"],
                filename,
                za["contentType"] or "",
                zpath,
            ),
        )
        counts["attachments"] += 1

    bcon.commit()

    # ── 4. Notes ─────────────────────────────────────────────
    zot_notes = zcon.execute(
        """SELECT inote.itemID, inote.parentItemID, inote.note,
                  inote.title, i.key
           FROM itemNotes inote
           JOIN items i ON inote.itemID = i.itemID
           WHERE i.libraryID = 1
             AND inote.parentItemID IS NOT NULL
             AND i.itemID NOT IN (SELECT itemID FROM deletedItems)"""
    ).fetchall()

    for zn in zot_notes:
        parent_row = zcon.execute(
            "SELECT key FROM items WHERE itemID = ?",
            (zn["parentItemID"],),
        ).fetchone()
        if parent_row is None:
            continue

        bib_parent = bcon.execute(
            "SELECT id FROM items WHERE key = ?", (parent_row["key"],)
        ).fetchone()
        if bib_parent is None:
            continue

        bcon.execute(
            "INSERT INTO notes (item_id, title, content) VALUES (?, ?, ?)",
            (bib_parent["id"], zn["title"] or "", zn["note"] or ""),
        )
        counts["notes"] += 1

    bcon.commit()

    # ── 5. Metadata ──────────────────────────────────────────
    bcon.execute(
        "INSERT OR REPLACE INTO bib_meta (key, value) VALUES (?, ?)",
        ("migrated_from", src),
    )
    bcon.commit()

    zcon.close()
    store.close()
    return counts


def main() -> None:
    parser = argparse.ArgumentParser(description="Migrate Zotero SQLite to bib SQLite")
    from conf import path as _conf_path

    parser.add_argument(
        "--src",
        default=str(_conf_path("db.zotero")),
        help="Path to Zotero SQLite database",
    )
    parser.add_argument(
        "--dst",
        default=str(_conf_path("db.bib")),
        help="Path to output bib SQLite database",
    )
    args = parser.parse_args()

    # Ensure destination directory exists
    Path(args.dst).parent.mkdir(parents=True, exist_ok=True)

    print(f"Migrating: {args.src} → {args.dst}")
    counts = migrate(args.src, args.dst)

    print("Migration complete:")
    for k, v in counts.items():
        print(f"  {k}: {v}")


if __name__ == "__main__":
    main()