stack/notebooks/zotero_tutorial.py

import marimo

__generated_with = "0.19.8"
app = marimo.App(width="medium")


@app.cell(hide_code=True)
def _():
    import marimo as mo

    mo.md("""
    # Zotero Library Explorer

    Inspect and work with the Zotero library running in the local `zotero` container.

    This notebook provides two modes of access:

    - **Local SQLite** — reads the Zotero database directly from the path configured in `stack.toml`
    - **Web API via pyzotero** — connects to `api.zotero.org` when API credentials are configured

    The local SQLite path works immediately with the container's library.
    The Web API requires a [Zotero API key](https://www.zotero.org/settings/keys/new).
    """)
    return (mo,)


@app.cell(hide_code=True)
def _(mo):
    import sqlite3

    import polars as pl
    from pyzotero import zotero

    from conf import connect
    from conf import path as _conf_path

    ZOTERO_DB = _conf_path("db.zotero")

    mo.md(f"""
    ## 1. Local Database Connection

    Database path: `{ZOTERO_DB}`
    Exists: **{ZOTERO_DB.exists()}**
    """)
    return ZOTERO_DB, connect, pl, sqlite3, zotero


@app.cell(hide_code=True)
def _(connect, sqlite3):
    db = connect.zotero()
    db.row_factory = sqlite3.Row

    # Quick health check
    tables = [
        r[0]
        for r in db.execute(
            "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
        ).fetchall()
    ]
    print(f"Connected — {len(tables)} tables found")
    print(f"Tables: {', '.join(tables[:15])}{'...' if len(tables) > 15 else ''}")
    return (db,)


@app.cell(hide_code=True)
def _(mo):
    mo.md("""
    ## 2. Library Overview

    Top-level statistics about what's in the Zotero library.
    """)
    return


@app.cell(hide_code=True)
def _(db, mo):
    stats = {}
    stats["Items"] = db.execute(
        "SELECT COUNT(*) FROM items WHERE itemID NOT IN (SELECT itemID FROM deletedItems)"
    ).fetchone()[0]
    stats["Collections"] = db.execute("SELECT COUNT(*) FROM collections").fetchone()[0]
    stats["Tags"] = db.execute("SELECT COUNT(DISTINCT name) FROM tags").fetchone()[0]
    stats["Creators"] = db.execute("SELECT COUNT(*) FROM creators").fetchone()[0]
    stats["Attachments"] = db.execute(
        "SELECT COUNT(*) FROM itemAttachments WHERE parentItemID IS NOT NULL"
    ).fetchone()[0]

    stat_table = "\n".join(f"| {k} | {v} |" for k, v in stats.items())
    mo.md(f"""
    | Metric | Count |
    |--------|-------|
    {stat_table}
    """)
    return


@app.cell(hide_code=True)
def _(mo):
    mo.md("""
    ## 3. Browse Collections

    The collection hierarchy in the library.
    """)
    return


@app.cell(hide_code=True)
def _(db, mo, pl):
    collections_df = pl.read_database(
        """
        SELECT
            c.collectionID,
            c.collectionName,
            c.parentCollectionID,
            (SELECT COUNT(*) FROM collectionItems ci WHERE ci.collectionID = c.collectionID) AS itemCount
        FROM collections c
        ORDER BY c.collectionName
        """,
        connection=db,
        infer_schema_length=None,
    )

    def _build_tree(df, parent_id=None, depth=0):
        _rows = df.filter(
            pl.col("parentCollectionID") == parent_id
            if parent_id
            else pl.col("parentCollectionID").is_null()
        )
        _lines = []
        for _row in _rows.iter_rows(named=True):
            _indent = "  " * depth
            _lines.append(
                f"{_indent}- **{_row['collectionName']}** ({_row['itemCount']} items)"
            )
            _lines.extend(_build_tree(df, _row["collectionID"], depth + 1))
        return _lines

    _tree = _build_tree(collections_df) if collections_df.height > 0 else []
    mo.md(
        "\n".join(_tree)
        if _tree
        else "No collections found. Add some in the Zotero desktop app."
    )
    return


@app.cell(hide_code=True)
def _(mo):
    mo.md("""
    ## 4. Item Types

    Breakdown of items by type (journal article, book, webpage, etc.).
    """)
    return


@app.cell(hide_code=True)
def _(db, mo, pl):
    item_types_df = pl.read_database(
        """
        SELECT
            it.typeName,
            COUNT(*) as count
        FROM items i
        JOIN itemTypes it ON i.itemTypeID = it.itemTypeID
        WHERE i.itemID NOT IN (SELECT itemID FROM deletedItems)
          AND it.typeName NOT IN ('attachment', 'note', 'annotation')
        GROUP BY it.typeName
        ORDER BY count DESC
        """,
        connection=db,
        infer_schema_length=None,
    )

    mo.ui.table(
        item_types_df, label="Item Types"
    ) if item_types_df.height > 0 else mo.md("No library items found yet.")
    return


@app.cell(hide_code=True)
def _(mo):
    mo.md("""
    ## 5. Recent Items

    The most recently added items in the library.
    """)
    return


@app.cell(hide_code=True)
def _(db, mo, pl):
    recent_df = pl.read_database(
        """
        SELECT
            i.itemID,
            it.typeName AS type,
            MAX(CASE WHEN f.fieldName = 'title' THEN idv.value END) AS title,
            MAX(CASE WHEN f.fieldName = 'date' THEN idv.value END) AS date,
            MAX(CASE WHEN f.fieldName = 'DOI' THEN idv.value END) AS doi,
            MAX(CASE WHEN f.fieldName = 'url' THEN idv.value END) AS url,
            i.dateAdded
        FROM items i
        JOIN itemTypes it ON i.itemTypeID = it.itemTypeID
        LEFT JOIN itemData id ON i.itemID = id.itemID
        LEFT JOIN fields f ON id.fieldID = f.fieldID
        LEFT JOIN itemDataValues idv ON id.valueID = idv.valueID
        WHERE i.itemID NOT IN (SELECT itemID FROM deletedItems)
          AND it.typeName NOT IN ('attachment', 'note', 'annotation')
        GROUP BY i.itemID
        ORDER BY i.dateAdded DESC
        LIMIT 25
        """,
        connection=db,
        infer_schema_length=None,
    )

    mo.ui.table(recent_df, label="Recent Items") if recent_df.height > 0 else mo.md(
        "No items found. Add references through the Zotero desktop app (VNC)."
    )
    return (recent_df,)


@app.cell(hide_code=True)
def _(mo):
    mo.md("""
    ## 6. Search Items

    Search across titles, creators, tags, and notes.
    """)
    return


@app.cell(hide_code=True)
def _(mo):
    search_input = mo.ui.text(
        placeholder="Enter search terms...", label="Search", full_width=True
    )
    search_input
    return (search_input,)


@app.cell(hide_code=True)
def _(db, mo, pl, search_input):
    _query = search_input.value.strip() if search_input.value else ""

    if not _query:
        mo.md("Type a search query above.")
        mo.stop(True)

    _search_results = pl.read_database(
        """
        SELECT DISTINCT
            i.itemID,
            it.typeName AS type,
            MAX(CASE WHEN f.fieldName = 'title' THEN idv.value END) AS title,
            MAX(CASE WHEN f.fieldName = 'date' THEN idv.value END) AS date,
            GROUP_CONCAT(DISTINCT c.lastName) AS authors
        FROM items i
        JOIN itemTypes it ON i.itemTypeID = it.itemTypeID
        LEFT JOIN itemData id ON i.itemID = id.itemID
        LEFT JOIN fields f ON id.fieldID = f.fieldID
        LEFT JOIN itemDataValues idv ON id.valueID = idv.valueID
        LEFT JOIN itemCreators ic ON i.itemID = ic.itemID
        LEFT JOIN creators c ON ic.creatorID = c.creatorID
        LEFT JOIN itemTags itag ON i.itemID = itag.itemID
        LEFT JOIN tags t ON itag.tagID = t.tagID
        WHERE i.itemID NOT IN (SELECT itemID FROM deletedItems)
          AND it.typeName NOT IN ('attachment', 'note', 'annotation')
          AND (
              idv.value LIKE '%' || :q || '%'
              OR c.lastName LIKE '%' || :q || '%'
              OR c.firstName LIKE '%' || :q || '%'
              OR t.name LIKE '%' || :q || '%'
          )
        GROUP BY i.itemID
        ORDER BY i.dateAdded DESC
        LIMIT 50
        """,
        connection=db,
        execute_options={"parameters": {"q": _query}},
    )

    mo.ui.table(
        _search_results, label=f"Results for '{_query}'"
    ) if _search_results.height > 0 else mo.md(f"No results for **{_query}**")
    return


@app.cell(hide_code=True)
def _(mo):
    mo.md("""
    ## 7. Item Detail Inspector

    Pick an item ID from the tables above to see its full metadata, creators, tags, and attachments.
    """)
    return


@app.cell(hide_code=True)
def _(mo, recent_df):
    default_id = str(recent_df["itemID"][0]) if recent_df.height > 0 else ""
    item_id_input = mo.ui.text(value=default_id, label="Item ID", full_width=False)
    item_id_input
    return (item_id_input,)


@app.cell(hide_code=True)
def _(db, item_id_input, mo):
    item_id = item_id_input.value.strip()

    if not item_id or not item_id.isdigit():
        mo.md("Enter a numeric item ID above.")
        mo.stop(True)

    iid = int(item_id)

    # Metadata fields
    fields = db.execute(
        """
        SELECT f.fieldName, idv.value
        FROM itemData id
        JOIN fields f ON id.fieldID = f.fieldID
        JOIN itemDataValues idv ON id.valueID = idv.valueID
        WHERE id.itemID = ?
        ORDER BY f.fieldName
        """,
        (iid,),
    ).fetchall()

    # Creators
    _creators = db.execute(
        """
        SELECT c.firstName, c.lastName, ct.creatorType, ic.orderIndex
        FROM itemCreators ic
        JOIN creators c ON ic.creatorID = c.creatorID
        JOIN creatorTypes ct ON ic.creatorTypeID = ct.creatorTypeID
        WHERE ic.itemID = ?
        ORDER BY ic.orderIndex
        """,
        (iid,),
    ).fetchall()

    # Tags
    _tags = db.execute(
        """
        SELECT t.name, CASE itag.type WHEN 0 THEN 'manual' ELSE 'automatic' END AS source
        FROM itemTags itag
        JOIN tags t ON itag.tagID = t.tagID
        WHERE itag.itemID = ?
        ORDER BY t.name
        """,
        (iid,),
    ).fetchall()

    # Attachments
    _attachments = db.execute(
        """
        SELECT ia.path, ia.contentType, i.key
        FROM itemAttachments ia
        JOIN items i ON ia.itemID = i.itemID
        WHERE ia.parentItemID = ?
        """,
        (iid,),
    ).fetchall()

    _sections = [f"### Item {iid}\n"]

    if fields:
        _sections.append("**Metadata**\n")
        _sections.append("| Field | Value |")
        _sections.append("|-------|-------|")
        for _f in fields:
            _val = str(_f["value"])[:120]
            _sections.append(f"| {_f['fieldName']} | {_val} |")

    if _creators:
        _sections.append("\n**Creators**\n")
        for _c in _creators:
            _sections.append(
                f"- {_c['firstName']} {_c['lastName']} ({_c['creatorType']})"
            )

    if _tags:
        _sections.append("\n**Tags**\n")
        _sections.append(", ".join(f"`{_t['name']}`" for _t in _tags))

    if _attachments:
        _sections.append("\n**Attachments**\n")
        for _a in _attachments:
            _path = _a["path"] or "(linked)"
            _sections.append(f"- `{_path}` ({_a['contentType'] or 'unknown'})")

    mo.md("\n".join(_sections))
    return


@app.cell(hide_code=True)
def _(mo):
    mo.md("""
    ## 8. Tags Overview

    All tags in the library with usage counts.
    """)
    return


@app.cell(hide_code=True)
def _(db, mo, pl):
    tags_df = pl.read_database(
        """
        SELECT t.name AS tag, COUNT(*) AS count
        FROM itemTags itag
        JOIN tags t ON itag.tagID = t.tagID
        JOIN items i ON itag.itemID = i.itemID
        WHERE i.itemID NOT IN (SELECT itemID FROM deletedItems)
        GROUP BY t.name
        ORDER BY count DESC
        """,
        connection=db,
        infer_schema_length=None,
    )

    mo.ui.table(tags_df, label="Tags") if tags_df.height > 0 else mo.md(
        "No tags found."
    )
    return


@app.cell(hide_code=True)
def _(mo):
    mo.md("""
    ## 9. Creator Network

    Most prolific authors/creators in the library.
    """)
    return


@app.cell(hide_code=True)
def _(db, mo, pl):
    creators_df = pl.read_database(
        """
        SELECT
            c.firstName || ' ' || c.lastName AS name,
            ct.creatorType,
            COUNT(DISTINCT ic.itemID) AS items
        FROM itemCreators ic
        JOIN creators c ON ic.creatorID = c.creatorID
        JOIN creatorTypes ct ON ic.creatorTypeID = ct.creatorTypeID
        JOIN items i ON ic.itemID = i.itemID
        WHERE i.itemID NOT IN (SELECT itemID FROM deletedItems)
        GROUP BY c.creatorID, ct.creatorType
        ORDER BY items DESC
        LIMIT 50
        """,
        connection=db,
        infer_schema_length=None,
    )

    mo.ui.table(creators_df, label="Top Creators") if creators_df.height > 0 else mo.md(
        "No creators found."
    )
    return


@app.cell(hide_code=True)
def _(mo):
    mo.md("""
    ## 10. Export to DataFrame

    Full library export as a Polars DataFrame for further analysis.
    """)
    return


@app.cell(hide_code=True)
def _(db, mo, pl):
    library_df = pl.read_database(
        """
        SELECT
            i.itemID,
            i.key,
            it.typeName AS itemType,
            MAX(CASE WHEN f.fieldName = 'title' THEN idv.value END) AS title,
            MAX(CASE WHEN f.fieldName = 'date' THEN idv.value END) AS date,
            MAX(CASE WHEN f.fieldName = 'DOI' THEN idv.value END) AS doi,
            MAX(CASE WHEN f.fieldName = 'url' THEN idv.value END) AS url,
            MAX(CASE WHEN f.fieldName = 'abstractNote' THEN idv.value END) AS abstract,
            MAX(CASE WHEN f.fieldName = 'publicationTitle' THEN idv.value END) AS publication,
            GROUP_CONCAT(DISTINCT c.lastName) AS authors,
            GROUP_CONCAT(DISTINCT t.name) AS tags,
            i.dateAdded,
            i.dateModified
        FROM items i
        JOIN itemTypes it ON i.itemTypeID = it.itemTypeID
        LEFT JOIN itemData id ON i.itemID = id.itemID
        LEFT JOIN fields f ON id.fieldID = f.fieldID
        LEFT JOIN itemDataValues idv ON id.valueID = idv.valueID
        LEFT JOIN itemCreators ic ON i.itemID = ic.itemID
        LEFT JOIN creators c ON ic.creatorID = c.creatorID
        LEFT JOIN itemTags itag ON i.itemID = itag.itemID
        LEFT JOIN tags t ON itag.tagID = t.tagID
        WHERE i.itemID NOT IN (SELECT itemID FROM deletedItems)
          AND it.typeName NOT IN ('attachment', 'note', 'annotation')
        GROUP BY i.itemID
        ORDER BY i.dateAdded DESC
        """,
        connection=db,
        infer_schema_length=None,
    )

    mo.md(f"Exported **{library_df.height}** items to `library_df`")
    return (library_df,)


@app.cell(hide_code=True)
def _(library_df, mo):
    mo.ui.dataframe(library_df)
    return


@app.cell(hide_code=True)
def _(mo):
    mo.md("""
    ## 11. Pyzotero Web API (Optional)

    If your Zotero library syncs to zotero.org, you can use pyzotero for read/write operations.

    1. Go to [zotero.org/settings/keys/new](https://www.zotero.org/settings/keys/new) and create an API key
    2. Find your library ID at [zotero.org/settings/keys](https://www.zotero.org/settings/keys) (it's the number in the "Your userID" section)
    3. Enter them below
    """)
    return


@app.cell(hide_code=True)
def _(mo):
    api_key_input = mo.ui.text(
        placeholder="Zotero API key", label="API Key", kind="password", full_width=True
    )
    library_id_input = mo.ui.text(
        placeholder="Library ID (numeric)", label="Library ID", full_width=True
    )
    lib_type_input = mo.ui.dropdown(
        options=["user", "group"], value="user", label="Library Type"
    )
    mo.hstack([library_id_input, lib_type_input, api_key_input], widths=[1, 1, 2])
    return api_key_input, lib_type_input, library_id_input


@app.cell(hide_code=True)
def _(api_key_input, lib_type_input, library_id_input, mo, zotero):
    api_key = api_key_input.value.strip() if api_key_input.value else ""
    lib_id = library_id_input.value.strip() if library_id_input.value else ""
    lib_type = lib_type_input.value

    if not api_key or not lib_id:
        mo.md(
            "Enter your Zotero API key and library ID above to enable Web API access."
        )
        mo.stop(True)

    zot = zotero.Zotero(lib_id, lib_type, api_key)

    # Test the connection
    _conn_error = None
    try:
        _key_info = zot.key_info()
    except Exception as e:
        _conn_error = str(e)

    if _conn_error:
        mo.md(f"Connection failed: `{_conn_error}`")
        mo.stop(True)

    mo.md(f"""
    Connected to Zotero Web API

    - **User**: {_key_info.get("userID", "N/A")}
    - **Key name**: {_key_info.get("key", "N/A")[:8]}...
    - **Access**: {"read/write" if _key_info.get("access", {}).get("user", {}).get("library") else "read-only"}
    """)
    return (zot,)


@app.cell(hide_code=True)
def _(mo, zot):
    mo.md("### Web API — Top 10 Recent Items")

    _top_items = zot.top(limit=10)
    if _top_items:
        for _item in _top_items:
            _data = _item["data"]
            _title = _data.get("title", "(untitled)")
            _item_type = _data.get("itemType", "?")
            _authors = ", ".join(
                _cr.get("lastName", _cr.get("name", "?"))
                for _cr in _data.get("creators", [])
            )
            print(f"[{_item_type}] {_title}")
            if _authors:
                print(f"  by {_authors}")
    else:
        print("No items in library.")
    return


@app.cell(hide_code=True)
def _(mo, zot):
    mo.md("### Web API — Collections")

    _collections = zot.collections()
    for _col in _collections:
        _name = _col["data"]["name"]
        _count = _col["meta"].get("numItems", 0)
        print(f"  {_name} ({_count} items)")
    if not _collections:
        print("No collections.")
    return


@app.cell(hide_code=True)
def _(mo):
    mo.md("""
    ## Summary

    | Approach | When to use |
    |----------|-------------|
    | Local SQLite | Always available, read-only, fast, no credentials needed |
    | Pyzotero Web API | Read/write, sync support, requires zotero.org API key |

    The local database is mounted read-only from the `zotero` container at the path configured in `stack.toml` (`db.zotero`).
    Changes made through the VNC desktop app are reflected here on next Zotero save.
    """)
    return


if __name__ == "__main__":
    app.run()