Some checks failed
CI / skinny-install (aco) (push) Successful in 1m12s
CI / skinny-install (api) (push) Successful in 30s
CI / skinny-install (bcda) (push) Successful in 36s
CI / skinny-install (bib) (push) Successful in 35s
CI / skinny-install (bls) (push) Successful in 27s
CI / skinny-install (ccw) (push) Successful in 32s
CI / skinny-install (cli) (push) Successful in 41s
CI / skinny-install (cms) (push) Successful in 37s
CI / skinny-install (conf) (push) Successful in 38s
CI / skinny-install (opps) (push) Successful in 33s
CI / skinny-install (perf) (push) Successful in 38s
CI / skinny-install (pfs) (push) Successful in 38s
CI / skinny-install (rex) (push) Successful in 34s
Deploy / build-scan-report (push) Failing after 46s
Infra CI / notebooks (push) Failing after 25s
Infra CI / zotero (push) Successful in 12s
Infra CI / docs (push) Failing after 16s
CI / lint-test (push) Failing after 11m2s
Infra CI / mc (push) Successful in 21s
Infra CI / api (push) Successful in 29s
Package Supply Chain / pkg-supply-chain (push) Failing after 41s
Mail: Maddy on DO (corwins.media+Resend, fhirworx.io+Postmark), touchless/stateless/idempotent. Gitea SMTP via env_file. CMS inbox at cmsupdates@mail.fhirworx.io with IMAP→bib poller. Bib: regulations.gov v4 client, Federal Register discovery, 164K comment backfill (running), IMAP email ingest, Zotero sync routing. PRISMA: altcha PoW solver, CrossRef DOI resolution, 83/129 PDFs. Zotero: schema parity, ops module, CLI, fail-fast guard. CI: docs.Dockerfile COPY glob fix (tracks #341). Infra: Gitea+marimo fhirworx themes, IOM/OIG modules.
897 lines
24 KiB
Python
897 lines
24 KiB
Python
import marimo
|
|
|
|
__generated_with = "0.19.9"
|
|
app = marimo.App(width="medium")
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _():
|
|
import marimo as mo
|
|
|
|
mo.md("""
|
|
# Bibliography Explorer
|
|
|
|
Browse, search, and cite from the **bib** reference library — a denormalized SQLite store
|
|
that replaces Zotero's 61-table schema with 8 clean tables.
|
|
|
|
This notebook provides:
|
|
|
|
1. **Migration** — one-click import from the Zotero container database
|
|
2. **Library overview** — stats, collections, tags
|
|
3. **Browse & search** — filter by type, tag, collection, or free text
|
|
4. **Item inspector** — full metadata for any item
|
|
5. **Citations** — APA and Bluebook formatted references
|
|
6. **DataFrame export** — polars DataFrame for downstream analysis
|
|
""")
|
|
return (mo,)
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
from bib.client import COLLECTIONS
|
|
from bib.format import format_citation
|
|
from bib.item import Download, Manual, Regulation, Rule, Source
|
|
from bib.tag import Tag
|
|
from conf import connect
|
|
from conf import path as _conf_path
|
|
|
|
store = connect.bib()
|
|
ZOTERO_DB = _conf_path("db.zotero")
|
|
BIB_DB = _conf_path("db.bib")
|
|
|
|
mo.md(f"""
|
|
## 1. Connect
|
|
|
|
| Setting | Value |
|
|
|---------|-------|
|
|
| Database | `{BIB_DB}` |
|
|
| Exists | **{BIB_DB.exists()}** |
|
|
| Zotero source | `{ZOTERO_DB}` |
|
|
| Zotero available | **{ZOTERO_DB.exists()}** |
|
|
""")
|
|
return (
|
|
COLLECTIONS,
|
|
Download,
|
|
Manual,
|
|
Regulation,
|
|
Rule,
|
|
Source,
|
|
Tag,
|
|
ZOTERO_DB,
|
|
format_citation,
|
|
store,
|
|
)
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(ZOTERO_DB, mo, store):
|
|
_item_count = store._con().execute("SELECT COUNT(*) FROM items").fetchone()[0]
|
|
|
|
if _item_count > 0:
|
|
_msg = f"""
|
|
## 2. Migration
|
|
|
|
Database already has **{_item_count}** items — migration not needed.
|
|
|
|
To re-migrate, delete `data/bib.sqlite` and restart.
|
|
"""
|
|
elif not ZOTERO_DB.exists():
|
|
_msg = """
|
|
## 2. Migration
|
|
|
|
No Zotero database found at `{ZOTERO_DB}`.
|
|
Start the Zotero container first, or add items manually below.
|
|
"""
|
|
else:
|
|
_msg = """
|
|
## 2. Migration
|
|
|
|
Database is empty and Zotero source is available. Click **Migrate** to import.
|
|
"""
|
|
|
|
mo.md(_msg)
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(ZOTERO_DB, mo, store):
|
|
_count = store._con().execute("SELECT COUNT(*) FROM items").fetchone()[0]
|
|
_show_migrate = _count == 0 and ZOTERO_DB.exists()
|
|
|
|
migrate_btn = (
|
|
mo.ui.run_button(label="Migrate from Zotero") if _show_migrate else None
|
|
)
|
|
migrate_btn
|
|
return (migrate_btn,)
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(ZOTERO_DB, migrate_btn, mo, store):
|
|
if migrate_btn is None or not migrate_btn.value:
|
|
mo.stop(True)
|
|
|
|
# Inline migration — reads Zotero EAV, writes to the bib store
|
|
import sqlite3
|
|
|
|
zcon = sqlite3.connect(f"file:{ZOTERO_DB}?mode=ro", uri=True)
|
|
zcon.row_factory = sqlite3.Row
|
|
|
|
# Migrate collections
|
|
_zot_cols = zcon.execute(
|
|
"""SELECT collectionID, collectionName, key, parentCollectionID
|
|
FROM collections WHERE libraryID = 1
|
|
ORDER BY parentCollectionID NULLS FIRST"""
|
|
).fetchall()
|
|
_col_id_to_key = {r["collectionID"]: r["key"] for r in _zot_cols}
|
|
bcon = store._con()
|
|
for _zc in _zot_cols:
|
|
_pid = None
|
|
if _zc["parentCollectionID"]:
|
|
_pk = _col_id_to_key.get(_zc["parentCollectionID"])
|
|
if _pk:
|
|
_pr = bcon.execute(
|
|
"SELECT id FROM collections WHERE key = ?", (_pk,)
|
|
).fetchone()
|
|
if _pr:
|
|
_pid = _pr["id"]
|
|
bcon.execute(
|
|
"INSERT OR IGNORE INTO collections (key, name, parent_id) VALUES (?, ?, ?)",
|
|
(_zc["key"], _zc["collectionName"], _pid),
|
|
)
|
|
bcon.commit()
|
|
|
|
# Migrate content items
|
|
_content = zcon.execute(
|
|
"""SELECT i.itemID, i.key, it.typeName
|
|
FROM items i JOIN itemTypes it ON i.itemTypeID = it.itemTypeID
|
|
WHERE i.libraryID = 1
|
|
AND i.itemID NOT IN (SELECT itemID FROM deletedItems)
|
|
AND it.typeName NOT IN ('attachment', 'note')
|
|
ORDER BY i.itemID"""
|
|
).fetchall()
|
|
|
|
_migrated = 0
|
|
for _zi in _content:
|
|
# Read fields via EAV join
|
|
_fields = {
|
|
r["fieldName"]: r["value"]
|
|
for r in zcon.execute(
|
|
"""SELECT f.fieldName, idv.value FROM itemData id
|
|
JOIN fields f ON id.fieldID = f.fieldID
|
|
JOIN itemDataValues idv ON id.valueID = idv.valueID
|
|
WHERE id.itemID = ?""",
|
|
(_zi["itemID"],),
|
|
).fetchall()
|
|
}
|
|
|
|
# Classify
|
|
_zt = _zi["typeName"]
|
|
if _zt == "statute":
|
|
_it = "regulation" if "C.F.R." in _fields.get("code", "") else "rule"
|
|
elif _zt == "report":
|
|
_it = "manual"
|
|
elif _zt == "webpage":
|
|
_it = "download"
|
|
else:
|
|
_it = "source"
|
|
|
|
# Title
|
|
_title = (
|
|
_fields.get("nameOfAct", "")
|
|
if _it in ("rule", "regulation")
|
|
else _fields.get("title", "")
|
|
)
|
|
_date = (
|
|
_fields.get("dateEnacted", "")
|
|
if _it in ("rule", "regulation")
|
|
else _fields.get("date", "")
|
|
)
|
|
|
|
bcon.execute(
|
|
"""INSERT OR IGNORE INTO items
|
|
(key, item_type, title, url, date_published, access_date, abstract, institution, extra, extra_json)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
|
(
|
|
_zi["key"],
|
|
_it,
|
|
_title,
|
|
_fields.get("url", ""),
|
|
_date,
|
|
_fields.get("accessDate", ""),
|
|
_fields.get("abstractNote", ""),
|
|
_fields.get("institution", "") or _fields.get("publisher", ""),
|
|
_fields.get("extra", ""),
|
|
"{}",
|
|
),
|
|
)
|
|
|
|
_bib_row = bcon.execute(
|
|
"SELECT id FROM items WHERE key = ?", (_zi["key"],)
|
|
).fetchone()
|
|
if _bib_row is None:
|
|
continue
|
|
_bid = _bib_row["id"]
|
|
_migrated += 1
|
|
|
|
# Tags
|
|
for _tr in zcon.execute(
|
|
"SELECT t.name FROM itemTags it JOIN tags t ON it.tagID = t.tagID WHERE it.itemID = ?",
|
|
(_zi["itemID"],),
|
|
).fetchall():
|
|
_tid = store._ensure_tag(_tr["name"])
|
|
bcon.execute(
|
|
"INSERT OR IGNORE INTO item_tags (item_id, tag_id) VALUES (?, ?)",
|
|
(_bid, _tid),
|
|
)
|
|
|
|
# Collection membership
|
|
for _cr in zcon.execute(
|
|
"SELECT c.key FROM collectionItems ci JOIN collections c ON ci.collectionID = c.collectionID WHERE ci.itemID = ?",
|
|
(_zi["itemID"],),
|
|
).fetchall():
|
|
_cid_row = bcon.execute(
|
|
"SELECT id FROM collections WHERE key = ?", (_cr["key"],)
|
|
).fetchone()
|
|
if _cid_row:
|
|
bcon.execute(
|
|
"INSERT OR IGNORE INTO collection_items (collection_id, item_id) VALUES (?, ?)",
|
|
(_cid_row["id"], _bid),
|
|
)
|
|
|
|
bcon.commit()
|
|
zcon.close()
|
|
|
|
mo.md(f"Migrated **{_migrated}** items from Zotero.")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(COLLECTIONS, mo, store):
|
|
_cols = store.ensure_collections(COLLECTIONS)
|
|
mo.md(f"Ensured **{len(_cols)}** collections.")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
mo.md("""
|
|
## 3. Library Overview
|
|
""")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo, store):
|
|
_con = store._con()
|
|
|
|
_stats = {}
|
|
_stats["Items"] = _con.execute("SELECT COUNT(*) FROM items").fetchone()[0]
|
|
_stats["Collections"] = _con.execute("SELECT COUNT(*) FROM collections").fetchone()[
|
|
0
|
|
]
|
|
_stats["Tags"] = _con.execute("SELECT COUNT(*) FROM tags").fetchone()[0]
|
|
_stats["Creators"] = _con.execute("SELECT COUNT(*) FROM creators").fetchone()[0]
|
|
_stats["Attachments"] = _con.execute("SELECT COUNT(*) FROM attachments").fetchone()[
|
|
0
|
|
]
|
|
_stats["Notes"] = _con.execute("SELECT COUNT(*) FROM notes").fetchone()[0]
|
|
|
|
_stat_table = "\n".join(f"| {k} | {v:,} |" for k, v in _stats.items())
|
|
mo.md(f"""
|
|
| Metric | Count |
|
|
|--------|-------|
|
|
{_stat_table}
|
|
""")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
mo.md("""
|
|
### Item Types
|
|
""")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo, store):
|
|
import polars as pl
|
|
|
|
_types_rows = (
|
|
store._con()
|
|
.execute(
|
|
"""SELECT item_type, COUNT(*) as count
|
|
FROM items GROUP BY item_type ORDER BY count DESC"""
|
|
)
|
|
.fetchall()
|
|
)
|
|
_types_df = (
|
|
pl.DataFrame([dict(r) for r in _types_rows]) if _types_rows else pl.DataFrame()
|
|
)
|
|
|
|
mo.ui.table(_types_df, label="Item Types") if _types_df.height > 0 else mo.md(
|
|
"No items yet."
|
|
)
|
|
return (pl,)
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
mo.md("""
|
|
### Collections
|
|
""")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo, store):
|
|
_collections = store.list_collections()
|
|
|
|
# Build a tree from the flat list
|
|
_by_key = {c["key"]: c for c in _collections}
|
|
_children: dict[str, list] = {}
|
|
_roots = []
|
|
for _c in _collections:
|
|
pk = _c["parent_key"]
|
|
if pk:
|
|
_children.setdefault(pk, []).append(_c)
|
|
else:
|
|
_roots.append(_c)
|
|
|
|
# Build tree iteratively (recursive local fns break in marimo cells)
|
|
_tree_lines = []
|
|
_stack = [(n, 0) for n in sorted(_roots, key=lambda x: x["name"], reverse=True)]
|
|
while _stack:
|
|
_node, _depth = _stack.pop()
|
|
_tree_lines.append(
|
|
f"{' ' * _depth}- **{_node['name']}** ({_node['item_count']} items)"
|
|
)
|
|
for _ch in sorted(
|
|
_children.get(_node["key"], []), key=lambda x: x["name"], reverse=True
|
|
):
|
|
_stack.append((_ch, _depth + 1))
|
|
mo.md(
|
|
"\n".join(_tree_lines)
|
|
if _tree_lines
|
|
else "No collections. Ensure collections above."
|
|
)
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
mo.md("""
|
|
### Tags
|
|
""")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo, pl, store):
|
|
_tag_list = store.list_tags()
|
|
_tag_df = pl.DataFrame(_tag_list) if _tag_list else pl.DataFrame()
|
|
|
|
mo.ui.table(_tag_df, label="Tags") if _tag_df.height > 0 else mo.md("No tags yet.")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
mo.md("""
|
|
## 4. Browse & Search
|
|
""")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo, store):
|
|
_type_opts = [""] + sorted(
|
|
{
|
|
r[0]
|
|
for r in store._con()
|
|
.execute("SELECT DISTINCT item_type FROM items")
|
|
.fetchall()
|
|
}
|
|
)
|
|
_tag_opts = [""] + [t["name"] for t in store.list_tags()]
|
|
_col_opts = [""] + [f"{c['name']}|{c['key']}" for c in store.list_collections()]
|
|
|
|
search_input = mo.ui.text(
|
|
placeholder="Search titles and abstracts...", label="Search", full_width=True
|
|
)
|
|
type_filter = mo.ui.dropdown(options=_type_opts, value="", label="Type")
|
|
tag_filter = mo.ui.dropdown(options=_tag_opts, value="", label="Tag")
|
|
col_filter = mo.ui.dropdown(options=_col_opts, value="", label="Collection")
|
|
|
|
mo.hstack([search_input], widths=[1])
|
|
mo.hstack([type_filter, tag_filter, col_filter], widths=[1, 1, 1])
|
|
return col_filter, search_input, tag_filter, type_filter
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(col_filter, mo, pl, search_input, store, tag_filter, type_filter):
|
|
_filters = {}
|
|
if search_input.value:
|
|
_filters["query"] = search_input.value.strip()
|
|
if type_filter.value:
|
|
_filters["item_type"] = type_filter.value
|
|
if tag_filter.value:
|
|
_filters["tag"] = tag_filter.value
|
|
if col_filter.value and "|" in col_filter.value:
|
|
_filters["collection"] = col_filter.value.split("|")[1]
|
|
|
|
_items = store.list_items(**_filters)
|
|
|
|
_rows = [
|
|
{
|
|
"key": it.key,
|
|
"type": it.item_type,
|
|
"title": it.title[:100],
|
|
"date": it.date_published,
|
|
"tags": "; ".join(it.tags[:4]),
|
|
"url": it.url[:60] if it.url else "",
|
|
}
|
|
for it in _items
|
|
]
|
|
_browse_df = pl.DataFrame(_rows) if _rows else pl.DataFrame()
|
|
|
|
_label = f"Items ({len(_items)} total)"
|
|
if _browse_df.height > 0:
|
|
items_table = mo.ui.table(_browse_df, selection="single", label=_label)
|
|
else:
|
|
items_table = None
|
|
mo.output.replace(mo.md("No items match the current filters."))
|
|
return (items_table,)
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
mo.md("""
|
|
## 5. Item Inspector
|
|
|
|
Select a row above, or enter a key below.
|
|
""")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(items_table, mo):
|
|
_default_key = ""
|
|
if items_table is not None:
|
|
_val = items_table.value
|
|
import polars as _pl
|
|
|
|
if isinstance(_val, _pl.DataFrame) and _val.height > 0:
|
|
_default_key = _val["key"][0]
|
|
elif isinstance(_val, list) and len(_val) > 0:
|
|
_default_key = _val[0].get("key", "") if isinstance(_val[0], dict) else ""
|
|
|
|
key_input = mo.ui.text(value=_default_key, label="Item Key", full_width=False)
|
|
key_input
|
|
return (key_input,)
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(format_citation, key_input, mo, store):
|
|
_key = key_input.value.strip() if key_input.value else ""
|
|
|
|
if not _key:
|
|
mo.md("Enter an item key above.")
|
|
mo.stop(True)
|
|
|
|
try:
|
|
_item = store.get(_key)
|
|
except KeyError:
|
|
mo.md(f"Item **{_key}** not found.")
|
|
mo.stop(True)
|
|
|
|
_sections = [f"### {_item.title}\n"]
|
|
|
|
# Metadata table
|
|
_sections.append("| Field | Value |")
|
|
_sections.append("|-------|-------|")
|
|
_sections.append(f"| Key | `{_item.key}` |")
|
|
_sections.append(f"| Type | {_item.item_type} |")
|
|
if _item.date_published:
|
|
_sections.append(f"| Published | {_item.date_published} |")
|
|
if _item.institution:
|
|
_sections.append(f"| Institution | {_item.institution} |")
|
|
if _item.url:
|
|
_sections.append(f"| URL | {_item.url[:80]} |")
|
|
if _item.access_date:
|
|
_sections.append(f"| Accessed | {_item.access_date} |")
|
|
|
|
# Type-specific fields from extra_json
|
|
import json as _json
|
|
|
|
_row = _item.to_row()
|
|
_ej = _json.loads(_row.get("extra_json", "{}"))
|
|
for _field, _val in _ej.items():
|
|
if _val:
|
|
_sections.append(f"| {_field} | {str(_val)[:80]} |")
|
|
|
|
# Tags
|
|
if _item.tags:
|
|
_sections.append(f"\n**Tags**: {', '.join(f'`{t}`' for t in _item.tags)}")
|
|
|
|
# Abstract
|
|
if _item.abstract:
|
|
_sections.append(
|
|
f"\n**Abstract**: {_item.abstract[:300]}{'...' if len(_item.abstract) > 300 else ''}"
|
|
)
|
|
|
|
# Citation preview
|
|
_cite_apa = format_citation(_item, style="apa")
|
|
_sections.append(f"\n**APA Citation**:\n> {_cite_apa}")
|
|
|
|
if _item.item_type == "regulation":
|
|
_cite_bb = format_citation(_item, style="bluebook")
|
|
_sections.append(f"\n**Bluebook Citation**:\n> {_cite_bb}")
|
|
|
|
# Attachments
|
|
_atts = (
|
|
store._con()
|
|
.execute(
|
|
"""SELECT key, filename, content_type FROM attachments
|
|
WHERE item_id = (SELECT id FROM items WHERE key = ?)""",
|
|
(_key,),
|
|
)
|
|
.fetchall()
|
|
)
|
|
if _atts:
|
|
_sections.append("\n**Attachments**")
|
|
for _a in _atts:
|
|
_sections.append(
|
|
f"- `{_a['filename']}` ({_a['content_type'] or 'unknown'})"
|
|
)
|
|
|
|
# Notes
|
|
_notes = (
|
|
store._con()
|
|
.execute(
|
|
"""SELECT title, content FROM notes
|
|
WHERE item_id = (SELECT id FROM items WHERE key = ?)""",
|
|
(_key,),
|
|
)
|
|
.fetchall()
|
|
)
|
|
if _notes:
|
|
_sections.append("\n**Notes**")
|
|
for _n in _notes:
|
|
_title = _n["title"] or "(untitled)"
|
|
_content = _n["content"][:200] if _n["content"] else ""
|
|
_sections.append(f"- **{_title}**: {_content}")
|
|
|
|
mo.md("\n".join(_sections))
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
mo.md("""
|
|
## 6. Create an Item
|
|
|
|
Add a new citation manually using one of the five item types.
|
|
""")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
new_type = mo.ui.dropdown(
|
|
options=["rule", "manual", "regulation", "download", "source"],
|
|
value="rule",
|
|
label="Item Type",
|
|
)
|
|
new_title = mo.ui.text(placeholder="Title", label="Title", full_width=True)
|
|
new_url = mo.ui.text(placeholder="URL", label="URL", full_width=True)
|
|
new_tags_input = mo.ui.text(
|
|
placeholder="module:pfs, year:2026",
|
|
label="Tags (comma-separated)",
|
|
full_width=True,
|
|
)
|
|
create_btn = mo.ui.run_button(label="Create Item")
|
|
|
|
mo.vstack(
|
|
[
|
|
mo.hstack([new_type, new_title], widths=[1, 3]),
|
|
new_url,
|
|
mo.hstack([new_tags_input, create_btn], widths=[3, 1]),
|
|
]
|
|
)
|
|
return create_btn, new_tags_input, new_title, new_type, new_url
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(
|
|
Download,
|
|
Manual,
|
|
Regulation,
|
|
Rule,
|
|
Source,
|
|
Tag,
|
|
create_btn,
|
|
mo,
|
|
new_tags_input,
|
|
new_title,
|
|
new_type,
|
|
new_url,
|
|
store,
|
|
):
|
|
if not create_btn.value:
|
|
mo.stop(True)
|
|
|
|
_title = new_title.value.strip() if new_title.value else ""
|
|
if not _title:
|
|
mo.md("Enter a title.")
|
|
mo.stop(True)
|
|
|
|
_type_map = {
|
|
"rule": Rule,
|
|
"manual": Manual,
|
|
"regulation": Regulation,
|
|
"download": Download,
|
|
"source": Source,
|
|
}
|
|
_cls = _type_map[new_type.value]
|
|
_item = _cls(title=_title, url=new_url.value.strip() if new_url.value else "")
|
|
|
|
# Parse tags
|
|
_tags = []
|
|
if new_tags_input.value:
|
|
for _t in new_tags_input.value.split(","):
|
|
_t = _t.strip()
|
|
if ":" in _t:
|
|
_tags.append(Tag.from_label(_t))
|
|
|
|
_key = store.upsert(_item, tags=_tags)
|
|
mo.md(f"Created item `{_key}`: **{_title}**")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
mo.md("""
|
|
## 7. Use a Translator
|
|
|
|
Fetch and parse a CMS source page directly into a citation.
|
|
""")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
translator_type = mo.ui.dropdown(
|
|
options=["cms_manual", "cms_website", "ecfr"],
|
|
value="cms_manual",
|
|
label="Translator",
|
|
)
|
|
translator_url = mo.ui.text(
|
|
placeholder="https://www.cms.gov/...",
|
|
label="URL",
|
|
full_width=True,
|
|
)
|
|
translate_btn = mo.ui.run_button(label="Translate & Save")
|
|
|
|
mo.hstack([translator_type, translator_url, translate_btn], widths=[1, 3, 1])
|
|
return translate_btn, translator_type, translator_url
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(
|
|
format_citation,
|
|
mo,
|
|
store,
|
|
translate_btn,
|
|
translator_type,
|
|
translator_url,
|
|
):
|
|
if not translate_btn.value:
|
|
mo.stop(True)
|
|
|
|
_url = translator_url.value.strip() if translator_url.value else ""
|
|
if not _url:
|
|
mo.md("Enter a URL to translate.")
|
|
mo.stop(True)
|
|
|
|
from bib import translate
|
|
|
|
_translators = {
|
|
"cms_manual": translate.cms_manual,
|
|
"cms_website": translate.cms_website,
|
|
"ecfr": translate.ecfr,
|
|
}
|
|
|
|
try:
|
|
_fn = _translators[translator_type.value]
|
|
_item = _fn(_url)
|
|
_key = store.upsert(_item)
|
|
_cite = format_citation(_item, style="apa")
|
|
_msg = f"""
|
|
Saved as `{_key}`: **{_item.title}**
|
|
|
|
Tags: {", ".join(f"`{t}`" for t in _item.tags)}
|
|
|
|
> {_cite}
|
|
"""
|
|
except Exception as e:
|
|
_msg = f"Translation failed: `{e}`"
|
|
|
|
mo.md(_msg)
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
mo.md("""
|
|
## 8. Bibliography Generator
|
|
|
|
Select items by tag or type to generate a formatted reference list.
|
|
""")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo, store):
|
|
_bib_tag_opts = [""] + [t["name"] for t in store.list_tags()]
|
|
_bib_type_opts = [""] + sorted(
|
|
{
|
|
r[0]
|
|
for r in store._con()
|
|
.execute("SELECT DISTINCT item_type FROM items")
|
|
.fetchall()
|
|
}
|
|
)
|
|
|
|
bib_tag = mo.ui.dropdown(options=_bib_tag_opts, value="", label="Filter by tag")
|
|
bib_type = mo.ui.dropdown(options=_bib_type_opts, value="", label="Filter by type")
|
|
bib_style = mo.ui.dropdown(options=["apa", "bluebook"], value="apa", label="Style")
|
|
|
|
mo.hstack([bib_tag, bib_type, bib_style], widths=[1, 1, 1])
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
mo.md("""
|
|
## 9. Export to DataFrame
|
|
""")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo, store):
|
|
try:
|
|
library_df = store.to_dataframe()
|
|
_msg = f"Exported **{library_df.height}** items to `library_df`"
|
|
except Exception:
|
|
library_df = None
|
|
_msg = "No items to export."
|
|
|
|
mo.md(_msg)
|
|
return (library_df,)
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(library_df, mo):
|
|
_out = (
|
|
mo.ui.dataframe(library_df)
|
|
if library_df is not None and library_df.height > 0
|
|
else mo.md("DataFrame is empty. Add some items first.")
|
|
)
|
|
_out
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
mo.md("""
|
|
## 10. Zotero WebDAV Storage (S3)
|
|
|
|
Browse files synced from Zotero to the WebDAV-backed S3 bucket.
|
|
The `webdav` service (rclone) serves the `zotero` bucket in RustFS.
|
|
""")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
import os
|
|
|
|
import s3fs
|
|
|
|
_endpoint = os.environ.get("S3_ENDPOINT", "http://rustfs:9000")
|
|
_key = os.environ.get("RUSTFS_ACCESS_KEY", "")
|
|
_secret = os.environ.get("RUSTFS_SECRET_KEY", "")
|
|
|
|
fs = s3fs.S3FileSystem(
|
|
key=_key,
|
|
secret=_secret,
|
|
endpoint_url=_endpoint,
|
|
client_kwargs={"region_name": "us-east-1"},
|
|
)
|
|
|
|
try:
|
|
_files = fs.ls("zotero/", detail=True)
|
|
_rows = [
|
|
{
|
|
"path": f["Key"].removeprefix("zotero/"),
|
|
"size_kb": round(f.get("size", f.get("Size", 0)) / 1024, 1),
|
|
"modified": str(f.get("LastModified", "")),
|
|
}
|
|
for f in _files
|
|
if f.get("type", f.get("StorageClass", "")) != "directory"
|
|
]
|
|
except Exception as e:
|
|
_rows = []
|
|
mo.md(f"Could not list S3 bucket: `{e}`")
|
|
|
|
import polars as _pl
|
|
|
|
s3_df = _pl.DataFrame(_rows) if _rows else _pl.DataFrame()
|
|
|
|
if s3_df.height > 0:
|
|
mo.md(f"**{s3_df.height}** files in `s3://zotero/`")
|
|
else:
|
|
mo.md("No files in `s3://zotero/` yet. Sync from Zotero to populate.")
|
|
|
|
return (fs, s3_df)
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo, s3_df):
|
|
mo.ui.table(s3_df, label="Zotero S3 Files") if s3_df.height > 0 else mo.md("")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(fs, mo):
|
|
_zotero_dir = []
|
|
try:
|
|
_zotero_dir = fs.ls("zotero/zotero/", detail=True)
|
|
except Exception:
|
|
pass
|
|
|
|
_sync_rows = [
|
|
{
|
|
"file": f["Key"].removeprefix("zotero/zotero/"),
|
|
"size_kb": round(f.get("size", f.get("Size", 0)) / 1024, 1),
|
|
}
|
|
for f in _zotero_dir
|
|
if f.get("type", f.get("StorageClass", "")) != "directory"
|
|
]
|
|
|
|
import polars as _pl
|
|
|
|
_sync_df = _pl.DataFrame(_sync_rows) if _sync_rows else _pl.DataFrame()
|
|
|
|
if _sync_df.height > 0:
|
|
mo.md(f"### Zotero Sync Directory\n\n**{_sync_df.height}** synced items")
|
|
mo.ui.table(_sync_df, label="Synced Files")
|
|
else:
|
|
mo.md("No synced files in `zotero/zotero/` yet.")
|
|
return
|
|
|
|
|
|
@app.cell(hide_code=True)
|
|
def _(mo):
|
|
mo.md("""
|
|
## Summary
|
|
|
|
| Feature | Details |
|
|
|---------|---------|
|
|
| Storage | SQLite, 8 denormalized tables |
|
|
| Item types | rule, manual, regulation, download, source |
|
|
| Tags | `namespace:value` (module, source, year, rule, file, table) |
|
|
| Citations | APA (all types), Bluebook (regulations) |
|
|
| Translators | Federal Register, CMS Website, CMS Manual, eCFR, 4i |
|
|
| Source | `src/bib/` (mounted read-only via `PYTHONPATH`) |
|
|
|
|
The `bib` module is available at `from bib import connect, Store, Rule, Tag`.
|
|
""")
|
|
return
|
|
|
|
|
|
if __name__ == "__main__":
|
|
app.run()
|