962dc48509
- bcda/client.py: output_dir default → conf.path("storage.bcda")
- bcda/express/flatten.py: store_path default → conf.path("storage.bcda")
- notebooks: eliminate all /home/kert/ absolute paths from
bib_explorer.py and zotero_tutorial.py
- generate_quality_measure_docs.py: complete truncated f-string at
line 561 (file was committed incomplete), add return statement
- test_client.py: update default assertion to match conf-resolved path
599 lines
17 KiB
Python
599 lines
17 KiB
Python
import marimo
|
|
|
|
__generated_with = "0.19.8"
|
|
app = marimo.App(width="medium")
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _():
|
|
import marimo as mo
|
|
mo.md("""
|
|
# Zotero Library Explorer
|
|
|
|
Inspect and work with the Zotero library running in the local `zotero` container.
|
|
|
|
This notebook provides two modes of access:
|
|
|
|
- **Local SQLite** — reads the Zotero database directly from the path configured in `stack.toml`
|
|
- **Web API via pyzotero** — connects to `api.zotero.org` when API credentials are configured
|
|
|
|
The local SQLite path works immediately with the container's library.
|
|
The Web API requires a [Zotero API key](https://www.zotero.org/settings/keys/new).
|
|
""")
|
|
return (mo,)
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
import sqlite3
|
|
import polars as pl
|
|
from pathlib import Path
|
|
from pyzotero import zotero
|
|
|
|
from conf import path as _conf_path
|
|
|
|
ZOTERO_DB = _conf_path("db.zotero")
|
|
|
|
mo.md(f"""
|
|
## 1. Local Database Connection
|
|
|
|
Database path: `{ZOTERO_DB}`
|
|
Exists: **{ZOTERO_DB.exists()}**
|
|
""")
|
|
return ZOTERO_DB, pl, sqlite3, zotero
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(ZOTERO_DB, sqlite3):
|
|
db = sqlite3.connect(f"file:{ZOTERO_DB}?mode=ro", uri=True)
|
|
db.row_factory = sqlite3.Row
|
|
|
|
# Quick health check
|
|
tables = [r[0] for r in db.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
|
|
).fetchall()]
|
|
print(f"Connected — {len(tables)} tables found")
|
|
print(f"Tables: {', '.join(tables[:15])}{'...' if len(tables) > 15 else ''}")
|
|
return (db,)
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
mo.md("""
|
|
## 2. Library Overview
|
|
|
|
Top-level statistics about what's in the Zotero library.
|
|
""")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(db, mo):
|
|
stats = {}
|
|
stats["Items"] = db.execute(
|
|
"SELECT COUNT(*) FROM items WHERE itemID NOT IN (SELECT itemID FROM deletedItems)"
|
|
).fetchone()[0]
|
|
stats["Collections"] = db.execute("SELECT COUNT(*) FROM collections").fetchone()[0]
|
|
stats["Tags"] = db.execute("SELECT COUNT(DISTINCT name) FROM tags").fetchone()[0]
|
|
stats["Creators"] = db.execute("SELECT COUNT(*) FROM creators").fetchone()[0]
|
|
stats["Attachments"] = db.execute(
|
|
"SELECT COUNT(*) FROM itemAttachments WHERE parentItemID IS NOT NULL"
|
|
).fetchone()[0]
|
|
|
|
stat_table = "\n".join(f"| {k} | {v} |" for k, v in stats.items())
|
|
mo.md(f"""
|
|
| Metric | Count |
|
|
|--------|-------|
|
|
{stat_table}
|
|
""")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
mo.md("""
|
|
## 3. Browse Collections
|
|
|
|
The collection hierarchy in the library.
|
|
""")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(db, mo, pl):
|
|
collections_df = pl.read_database(
|
|
"""
|
|
SELECT
|
|
c.collectionID,
|
|
c.collectionName,
|
|
c.parentCollectionID,
|
|
(SELECT COUNT(*) FROM collectionItems ci WHERE ci.collectionID = c.collectionID) AS itemCount
|
|
FROM collections c
|
|
ORDER BY c.collectionName
|
|
""",
|
|
connection=db,
|
|
)
|
|
|
|
def _build_tree(df, parent_id=None, depth=0):
|
|
_rows = df.filter(
|
|
pl.col("parentCollectionID") == parent_id if parent_id else pl.col("parentCollectionID").is_null()
|
|
)
|
|
_lines = []
|
|
for _row in _rows.iter_rows(named=True):
|
|
_indent = " " * depth
|
|
_lines.append(f"{_indent}- **{_row['collectionName']}** ({_row['itemCount']} items)")
|
|
_lines.extend(_build_tree(df, _row["collectionID"], depth + 1))
|
|
return _lines
|
|
|
|
_tree = _build_tree(collections_df) if collections_df.height > 0 else []
|
|
mo.md("\n".join(_tree) if _tree else "No collections found. Add some in the Zotero desktop app.")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
mo.md("""
|
|
## 4. Item Types
|
|
|
|
Breakdown of items by type (journal article, book, webpage, etc.).
|
|
""")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(db, mo, pl):
|
|
item_types_df = pl.read_database(
|
|
"""
|
|
SELECT
|
|
it.typeName,
|
|
COUNT(*) as count
|
|
FROM items i
|
|
JOIN itemTypes it ON i.itemTypeID = it.itemTypeID
|
|
WHERE i.itemID NOT IN (SELECT itemID FROM deletedItems)
|
|
AND it.typeName NOT IN ('attachment', 'note', 'annotation')
|
|
GROUP BY it.typeName
|
|
ORDER BY count DESC
|
|
""",
|
|
connection=db,
|
|
)
|
|
|
|
mo.ui.table(item_types_df, label="Item Types") if item_types_df.height > 0 else mo.md("No library items found yet.")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
mo.md("""
|
|
## 5. Recent Items
|
|
|
|
The most recently added items in the library.
|
|
""")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(db, mo, pl):
|
|
recent_df = pl.read_database(
|
|
"""
|
|
SELECT
|
|
i.itemID,
|
|
it.typeName AS type,
|
|
MAX(CASE WHEN f.fieldName = 'title' THEN idv.value END) AS title,
|
|
MAX(CASE WHEN f.fieldName = 'date' THEN idv.value END) AS date,
|
|
MAX(CASE WHEN f.fieldName = 'DOI' THEN idv.value END) AS doi,
|
|
MAX(CASE WHEN f.fieldName = 'url' THEN idv.value END) AS url,
|
|
i.dateAdded
|
|
FROM items i
|
|
JOIN itemTypes it ON i.itemTypeID = it.itemTypeID
|
|
LEFT JOIN itemData id ON i.itemID = id.itemID
|
|
LEFT JOIN fields f ON id.fieldID = f.fieldID
|
|
LEFT JOIN itemDataValues idv ON id.valueID = idv.valueID
|
|
WHERE i.itemID NOT IN (SELECT itemID FROM deletedItems)
|
|
AND it.typeName NOT IN ('attachment', 'note', 'annotation')
|
|
GROUP BY i.itemID
|
|
ORDER BY i.dateAdded DESC
|
|
LIMIT 25
|
|
""",
|
|
connection=db,
|
|
)
|
|
|
|
mo.ui.table(recent_df, label="Recent Items") if recent_df.height > 0 else mo.md("No items found. Add references through the Zotero desktop app (VNC).")
|
|
return (recent_df,)
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
mo.md("""
|
|
## 6. Search Items
|
|
|
|
Search across titles, creators, tags, and notes.
|
|
""")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
search_input = mo.ui.text(placeholder="Enter search terms...", label="Search", full_width=True)
|
|
search_input
|
|
return (search_input,)
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(db, mo, pl, search_input):
|
|
_query = search_input.value.strip() if search_input.value else ""
|
|
|
|
if not _query:
|
|
mo.md("Type a search query above.")
|
|
mo.stop(True)
|
|
|
|
_search_results = pl.read_database(
|
|
"""
|
|
SELECT DISTINCT
|
|
i.itemID,
|
|
it.typeName AS type,
|
|
MAX(CASE WHEN f.fieldName = 'title' THEN idv.value END) AS title,
|
|
MAX(CASE WHEN f.fieldName = 'date' THEN idv.value END) AS date,
|
|
GROUP_CONCAT(DISTINCT c.lastName) AS authors
|
|
FROM items i
|
|
JOIN itemTypes it ON i.itemTypeID = it.itemTypeID
|
|
LEFT JOIN itemData id ON i.itemID = id.itemID
|
|
LEFT JOIN fields f ON id.fieldID = f.fieldID
|
|
LEFT JOIN itemDataValues idv ON id.valueID = idv.valueID
|
|
LEFT JOIN itemCreators ic ON i.itemID = ic.itemID
|
|
LEFT JOIN creators c ON ic.creatorID = c.creatorID
|
|
LEFT JOIN itemTags itag ON i.itemID = itag.itemID
|
|
LEFT JOIN tags t ON itag.tagID = t.tagID
|
|
WHERE i.itemID NOT IN (SELECT itemID FROM deletedItems)
|
|
AND it.typeName NOT IN ('attachment', 'note', 'annotation')
|
|
AND (
|
|
idv.value LIKE '%' || :q || '%'
|
|
OR c.lastName LIKE '%' || :q || '%'
|
|
OR c.firstName LIKE '%' || :q || '%'
|
|
OR t.name LIKE '%' || :q || '%'
|
|
)
|
|
GROUP BY i.itemID
|
|
ORDER BY i.dateAdded DESC
|
|
LIMIT 50
|
|
""",
|
|
connection=db,
|
|
execute_options={"parameters": {"q": _query}},
|
|
)
|
|
|
|
mo.ui.table(_search_results, label=f"Results for '{_query}'") if _search_results.height > 0 else mo.md(f"No results for **{_query}**")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
mo.md("""
|
|
## 7. Item Detail Inspector
|
|
|
|
Pick an item ID from the tables above to see its full metadata, creators, tags, and attachments.
|
|
""")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo, recent_df):
|
|
default_id = str(recent_df["itemID"][0]) if recent_df.height > 0 else ""
|
|
item_id_input = mo.ui.text(value=default_id, label="Item ID", full_width=False)
|
|
item_id_input
|
|
return (item_id_input,)
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(db, item_id_input, mo):
|
|
item_id = item_id_input.value.strip()
|
|
|
|
if not item_id or not item_id.isdigit():
|
|
mo.md("Enter a numeric item ID above.")
|
|
mo.stop(True)
|
|
|
|
iid = int(item_id)
|
|
|
|
# Metadata fields
|
|
fields = db.execute(
|
|
"""
|
|
SELECT f.fieldName, idv.value
|
|
FROM itemData id
|
|
JOIN fields f ON id.fieldID = f.fieldID
|
|
JOIN itemDataValues idv ON id.valueID = idv.valueID
|
|
WHERE id.itemID = ?
|
|
ORDER BY f.fieldName
|
|
""",
|
|
(iid,),
|
|
).fetchall()
|
|
|
|
# Creators
|
|
_creators = db.execute(
|
|
"""
|
|
SELECT c.firstName, c.lastName, ct.creatorType, ic.orderIndex
|
|
FROM itemCreators ic
|
|
JOIN creators c ON ic.creatorID = c.creatorID
|
|
JOIN creatorTypes ct ON ic.creatorTypeID = ct.creatorTypeID
|
|
WHERE ic.itemID = ?
|
|
ORDER BY ic.orderIndex
|
|
""",
|
|
(iid,),
|
|
).fetchall()
|
|
|
|
# Tags
|
|
_tags = db.execute(
|
|
"""
|
|
SELECT t.name, CASE itag.type WHEN 0 THEN 'manual' ELSE 'automatic' END AS source
|
|
FROM itemTags itag
|
|
JOIN tags t ON itag.tagID = t.tagID
|
|
WHERE itag.itemID = ?
|
|
ORDER BY t.name
|
|
""",
|
|
(iid,),
|
|
).fetchall()
|
|
|
|
# Attachments
|
|
_attachments = db.execute(
|
|
"""
|
|
SELECT ia.path, ia.contentType, i.key
|
|
FROM itemAttachments ia
|
|
JOIN items i ON ia.itemID = i.itemID
|
|
WHERE ia.parentItemID = ?
|
|
""",
|
|
(iid,),
|
|
).fetchall()
|
|
|
|
_sections = [f"### Item {iid}\n"]
|
|
|
|
if fields:
|
|
_sections.append("**Metadata**\n")
|
|
_sections.append("| Field | Value |")
|
|
_sections.append("|-------|-------|")
|
|
for _f in fields:
|
|
_val = str(_f["value"])[:120]
|
|
_sections.append(f"| {_f['fieldName']} | {_val} |")
|
|
|
|
if _creators:
|
|
_sections.append("\n**Creators**\n")
|
|
for _c in _creators:
|
|
_sections.append(f"- {_c['firstName']} {_c['lastName']} ({_c['creatorType']})")
|
|
|
|
if _tags:
|
|
_sections.append("\n**Tags**\n")
|
|
_sections.append(", ".join(f"`{_t['name']}`" for _t in _tags))
|
|
|
|
if _attachments:
|
|
_sections.append("\n**Attachments**\n")
|
|
for _a in _attachments:
|
|
_path = _a["path"] or "(linked)"
|
|
_sections.append(f"- `{_path}` ({_a['contentType'] or 'unknown'})")
|
|
|
|
mo.md("\n".join(_sections))
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
mo.md("""
|
|
## 8. Tags Overview
|
|
|
|
All tags in the library with usage counts.
|
|
""")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(db, mo, pl):
|
|
tags_df = pl.read_database(
|
|
"""
|
|
SELECT t.name AS tag, COUNT(*) AS count
|
|
FROM itemTags itag
|
|
JOIN tags t ON itag.tagID = t.tagID
|
|
JOIN items i ON itag.itemID = i.itemID
|
|
WHERE i.itemID NOT IN (SELECT itemID FROM deletedItems)
|
|
GROUP BY t.name
|
|
ORDER BY count DESC
|
|
""",
|
|
connection=db,
|
|
)
|
|
|
|
mo.ui.table(tags_df, label="Tags") if tags_df.height > 0 else mo.md("No tags found.")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
mo.md("""
|
|
## 9. Creator Network
|
|
|
|
Most prolific authors/creators in the library.
|
|
""")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(db, mo, pl):
|
|
creators_df = pl.read_database(
|
|
"""
|
|
SELECT
|
|
c.firstName || ' ' || c.lastName AS name,
|
|
ct.creatorType,
|
|
COUNT(DISTINCT ic.itemID) AS items
|
|
FROM itemCreators ic
|
|
JOIN creators c ON ic.creatorID = c.creatorID
|
|
JOIN creatorTypes ct ON ic.creatorTypeID = ct.creatorTypeID
|
|
JOIN items i ON ic.itemID = i.itemID
|
|
WHERE i.itemID NOT IN (SELECT itemID FROM deletedItems)
|
|
GROUP BY c.creatorID, ct.creatorType
|
|
ORDER BY items DESC
|
|
LIMIT 50
|
|
""",
|
|
connection=db,
|
|
)
|
|
|
|
mo.ui.table(creators_df, label="Top Creators") if creators_df.height > 0 else mo.md("No creators found.")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
mo.md("""
|
|
## 10. Export to DataFrame
|
|
|
|
Full library export as a Polars DataFrame for further analysis.
|
|
""")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(db, mo, pl):
|
|
library_df = pl.read_database(
|
|
"""
|
|
SELECT
|
|
i.itemID,
|
|
i.key,
|
|
it.typeName AS itemType,
|
|
MAX(CASE WHEN f.fieldName = 'title' THEN idv.value END) AS title,
|
|
MAX(CASE WHEN f.fieldName = 'date' THEN idv.value END) AS date,
|
|
MAX(CASE WHEN f.fieldName = 'DOI' THEN idv.value END) AS doi,
|
|
MAX(CASE WHEN f.fieldName = 'url' THEN idv.value END) AS url,
|
|
MAX(CASE WHEN f.fieldName = 'abstractNote' THEN idv.value END) AS abstract,
|
|
MAX(CASE WHEN f.fieldName = 'publicationTitle' THEN idv.value END) AS publication,
|
|
GROUP_CONCAT(DISTINCT c.lastName) AS authors,
|
|
GROUP_CONCAT(DISTINCT t.name) AS tags,
|
|
i.dateAdded,
|
|
i.dateModified
|
|
FROM items i
|
|
JOIN itemTypes it ON i.itemTypeID = it.itemTypeID
|
|
LEFT JOIN itemData id ON i.itemID = id.itemID
|
|
LEFT JOIN fields f ON id.fieldID = f.fieldID
|
|
LEFT JOIN itemDataValues idv ON id.valueID = idv.valueID
|
|
LEFT JOIN itemCreators ic ON i.itemID = ic.itemID
|
|
LEFT JOIN creators c ON ic.creatorID = c.creatorID
|
|
LEFT JOIN itemTags itag ON i.itemID = itag.itemID
|
|
LEFT JOIN tags t ON itag.tagID = t.tagID
|
|
WHERE i.itemID NOT IN (SELECT itemID FROM deletedItems)
|
|
AND it.typeName NOT IN ('attachment', 'note', 'annotation')
|
|
GROUP BY i.itemID
|
|
ORDER BY i.dateAdded DESC
|
|
""",
|
|
connection=db,
|
|
)
|
|
|
|
mo.md(f"Exported **{library_df.height}** items to `library_df`")
|
|
return (library_df,)
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(library_df, mo):
|
|
mo.ui.dataframe(library_df)
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
mo.md("""
|
|
## 11. Pyzotero Web API (Optional)
|
|
|
|
If your Zotero library syncs to zotero.org, you can use pyzotero for read/write operations.
|
|
|
|
1. Go to [zotero.org/settings/keys/new](https://www.zotero.org/settings/keys/new) and create an API key
|
|
2. Find your library ID at [zotero.org/settings/keys](https://www.zotero.org/settings/keys) (it's the number in the "Your userID" section)
|
|
3. Enter them below
|
|
""")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
api_key_input = mo.ui.text(placeholder="Zotero API key", label="API Key", kind="password", full_width=True)
|
|
library_id_input = mo.ui.text(placeholder="Library ID (numeric)", label="Library ID", full_width=True)
|
|
lib_type_input = mo.ui.dropdown(options=["user", "group"], value="user", label="Library Type")
|
|
mo.hstack([library_id_input, lib_type_input, api_key_input], widths=[1, 1, 2])
|
|
return api_key_input, lib_type_input, library_id_input
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(api_key_input, lib_type_input, library_id_input, mo, zotero):
|
|
api_key = api_key_input.value.strip() if api_key_input.value else ""
|
|
lib_id = library_id_input.value.strip() if library_id_input.value else ""
|
|
lib_type = lib_type_input.value
|
|
|
|
if not api_key or not lib_id:
|
|
mo.md("Enter your Zotero API key and library ID above to enable Web API access.")
|
|
mo.stop(True)
|
|
|
|
zot = zotero.Zotero(lib_id, lib_type, api_key)
|
|
|
|
# Test the connection
|
|
_conn_error = None
|
|
try:
|
|
_key_info = zot.key_info()
|
|
except Exception as e:
|
|
_conn_error = str(e)
|
|
|
|
if _conn_error:
|
|
mo.md(f"Connection failed: `{_conn_error}`")
|
|
mo.stop(True)
|
|
|
|
mo.md(f"""
|
|
Connected to Zotero Web API
|
|
|
|
- **User**: {_key_info.get('userID', 'N/A')}
|
|
- **Key name**: {_key_info.get('key', 'N/A')[:8]}...
|
|
- **Access**: {'read/write' if _key_info.get('access', {}).get('user', {}).get('library') else 'read-only'}
|
|
""")
|
|
return (zot,)
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo, zot):
|
|
mo.md("### Web API — Top 10 Recent Items")
|
|
|
|
_top_items = zot.top(limit=10)
|
|
if _top_items:
|
|
for _item in _top_items:
|
|
_data = _item["data"]
|
|
_title = _data.get("title", "(untitled)")
|
|
_item_type = _data.get("itemType", "?")
|
|
_authors = ", ".join(
|
|
_cr.get("lastName", _cr.get("name", "?")) for _cr in _data.get("creators", [])
|
|
)
|
|
print(f"[{_item_type}] {_title}")
|
|
if _authors:
|
|
print(f" by {_authors}")
|
|
else:
|
|
print("No items in library.")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo, zot):
|
|
mo.md("### Web API — Collections")
|
|
|
|
_collections = zot.collections()
|
|
for _col in _collections:
|
|
_name = _col["data"]["name"]
|
|
_count = _col["meta"].get("numItems", 0)
|
|
print(f" {_name} ({_count} items)")
|
|
if not _collections:
|
|
print("No collections.")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
mo.md("""
|
|
## Summary
|
|
|
|
| Approach | When to use |
|
|
|----------|-------------|
|
|
| Local SQLite | Always available, read-only, fast, no credentials needed |
|
|
| Pyzotero Web API | Read/write, sync support, requires zotero.org API key |
|
|
|
|
The local database is mounted read-only from the `zotero` container at the path configured in `stack.toml` (`db.zotero`).
|
|
Changes made through the VNC desktop app are reflected here on next Zotero save.
|
|
""")
|
|
return
|
|
|
|
|
|
if __name__ == "__main__":
|
|
app.run()
|