From 8bd413fe44cc4a62df848add46a9207e4b0af745 Mon Sep 17 00:00:00 2001 From: kert Date: Sat, 14 Nov 2026 11:31:53 -0500 Subject: [PATCH] feat(focus): export highlights to Zotero notes --- src/cli/__init__.py | 2 + src/cli/focus.py | 90 ++++++++++++++++ src/focus/export.py | 126 ++++++++++++++++++++++ tests/focus/test_export.py | 212 +++++++++++++++++++++++++++++++++++++ 4 files changed, 430 insertions(+) create mode 100644 src/cli/focus.py create mode 100644 src/focus/export.py create mode 100644 tests/focus/test_export.py diff --git a/src/cli/__init__.py b/src/cli/__init__.py index aa4a1b6..38d140c 100644 --- a/src/cli/__init__.py +++ b/src/cli/__init__.py @@ -20,6 +20,7 @@ from cli.comments import app as comments_app from cli.db import app as db_app from cli.docs import app as docs_app from cli.drugs import app as drugs_app +from cli.focus import app as focus_app from cli.generate import app as generate_app from cli.health import health from cli.lake import app as lake_app @@ -71,6 +72,7 @@ app.add_typer( app.add_typer(llm_app, name="llm", help="Local RAG — index, search, tag.") app.add_typer(db_app, name="db", help="DuckDB utilities.") app.add_typer(docs_app, name="docs", help="Documentation generation.") +app.add_typer(focus_app, name="focus", help="Deep-focus annotations.") app.add_typer(api_app, name="api", help="API server.") app.add_typer(perf_app, name="perf", help="Pipeline telemetry utilities.") app.add_typer( diff --git a/src/cli/focus.py b/src/cli/focus.py new file mode 100644 index 0000000..f173510 --- /dev/null +++ b/src/cli/focus.py @@ -0,0 +1,90 @@ +"""stack focus — deep-focus annotations.""" + +from __future__ import annotations + +from pathlib import Path + +import typer + +app = typer.Typer(no_args_is_help=True) + +_DEFAULT_DB = Path("data/zotero/data/zotero.sqlite") + + +def _live_store(): # pragma: no cover + """The annotation store on the llm database.""" + from focus.schema import ensure + from focus.store import Store + from llm.config import load as load_config + from llm.index import _engine + + engine = _engine(load_config()) + ensure(engine) + return Store(engine), engine + + +@app.command("export-notes") +def export_notes_cmd( + protocol: str | None = typer.Option( + None, + "--protocol", + help="Only this protocol. Default: every protocol.", + ), + write: bool = typer.Option( + False, + "--write", + help="Write notes. Without this flag the command only reports.", + ), + db: Path | None = typer.Option( + None, + "--db", + help=( + f"Zotero sqlite. Default: {_DEFAULT_DB}. " + "Do not pass --write while Zotero is running." + ), + ), + focus_db: str = typer.Option( + "", + "--focus-db", + help="SQLAlchemy URL for the annotation store. Default: the llm database.", + ), +) -> None: + """Export human highlights to one Zotero note per item and protocol. + + Dry-run unless ``--write``. Does not schedule itself and does not + write labels.sqlite. Stop Zotero before ``--write`` on the live database. + """ + from sqlalchemy import create_engine + + from focus.export import export_notes + from focus.schema import ensure + from focus.store import Store + from zot.db import Db + + path = db or _DEFAULT_DB + if not path.is_file(): + raise typer.BadParameter(f"Zotero SQLite not found: {path}") + if focus_db: + engine = create_engine(focus_db) + ensure(engine) + store = Store(engine) + else: # pragma: no cover + store, engine = _live_store() + try: + with Db(str(path)) as zot: + try: + summary = export_notes( + store, + zot, + protocol=protocol or None, + dry_run=not write, + ) + except ValueError as exc: + typer.echo(str(exc), err=True) + raise typer.Exit(code=1) from exc + if write: + zot.commit() + finally: + engine.dispose() + for key in ("dry_run", "create", "replace", "skipped_no_zotero_key"): + typer.echo(f"{key}: {summary[key]}") diff --git a/src/focus/export.py b/src/focus/export.py new file mode 100644 index 0000000..ac8c0c0 --- /dev/null +++ b/src/focus/export.py @@ -0,0 +1,126 @@ +"""Export human highlights to Zotero notes. + +One note per (item, protocol). The marker ```` +makes a second run replace that note and leave every other note alone. +Dry-run is the default. This module does not write ``labels.sqlite`` +and does not schedule itself. +""" + +from __future__ import annotations + +import html +from typing import Any + +from sqlalchemy import select + +from focus.protocols import load +from focus.store import Annotation, Store, annotation + +_SUMMARY_KEYS = ("dry_run", "create", "replace", "skipped_no_zotero_key") + + +def export_notes( + store: Store, + zot_db: Any, + *, + protocol: str | None = None, + dry_run: bool = True, +) -> dict[str, Any]: + """Render live human annotations into Zotero notes. + + Items whose key is not in Zotero are counted and skipped. A model + annotation is not a person's note, so it is left out. ``dry_run`` + reports the creates and replaces and writes nothing. + """ + if protocol is not None: + load(protocol) + summary = { + "dry_run": dry_run, + "create": 0, + "replace": 0, + "skipped_no_zotero_key": 0, + } + skipped: set[str] = set() + for (item_key, proto_name), anns in _groups(store, protocol).items(): + parent = zot_db.find_item_by_key(item_key) + if parent is None: + skipped.add(item_key) + continue + action = _apply( + zot_db, + parent, + proto_name, + _render(proto_name, anns), + dry_run=dry_run, + ) + summary[action] += 1 + summary["skipped_no_zotero_key"] = len(skipped) + return {key: summary[key] for key in _SUMMARY_KEYS} + + +def _groups( + store: Store, protocol: str | None +) -> dict[tuple[str, str], list[Annotation]]: + with store.engine.connect() as conn: + keys = [ + row[0] + for row in conn.execute( + select(annotation.c.item_key).distinct().order_by(annotation.c.item_key) + ) + ] + grouped: dict[tuple[str, str], list[Annotation]] = {} + for key in keys: + for ann in store.list(key, protocol): + if ann.provenance != "human": + continue + grouped.setdefault((ann.item_key, ann.protocol), []).append(ann) + return grouped + + +def _render(protocol: str, anns: list[Annotation]) -> str: + proto = load(protocol) + lines = [] + for ann in anns: + label = proto.label(ann.label) + text = label.text if label is not None else ann.label + parts = [html.escape(text), html.escape(ann.quote)] + if ann.note: + parts.append(html.escape(ann.note)) + lines.append( + f'
  • {" — ".join(parts)}
  • ' + ) + body = "\n".join(lines) + return f"\n\n" + + +def _apply( + zot_db: Any, parent_id: int, protocol: str, body: str, *, dry_run: bool +) -> str: + marker = f"" + rows = zot_db.con.execute( + """SELECT itemID, note FROM itemNotes + WHERE parentItemID = ? ORDER BY itemID""", + (parent_id,), + ).fetchall() + matches = [row[0] for row in rows if row[1] and marker in row[1]] + if dry_run: + return "replace" if matches else "create" + title = f"Focus: {protocol}" + if matches: + from zot.db import now_iso + + ts = now_iso() + for note_id in matches: + zot_db.con.execute( + "UPDATE itemNotes SET note = ?, title = ? WHERE itemID = ?", + (body, title, note_id), + ) + zot_db.con.execute( + """UPDATE items + SET dateModified = ?, clientDateModified = ?, synced = 0 + WHERE itemID = ?""", + (ts, ts, note_id), + ) + return "replace" + zot_db.add_note(parent_id, body, title=title) + return "create" diff --git a/tests/focus/test_export.py b/tests/focus/test_export.py new file mode 100644 index 0000000..f126f55 --- /dev/null +++ b/tests/focus/test_export.py @@ -0,0 +1,212 @@ +"""Zotero note export. Dry-run writes nothing. One note per item and protocol.""" + +from __future__ import annotations + +from sqlalchemy import create_engine +from sqlalchemy.pool import StaticPool +from typer.testing import CliRunner + +from cli import app +from focus.export import export_notes +from focus.schema import ensure +from focus.store import Store +from zot.db import TYPE_MAP, Db +from zot.schema import create_db + +runner = CliRunner() + + +def _engine(): + return create_engine( + "sqlite://", + poolclass=StaticPool, + connect_args={"check_same_thread": False}, + ) + + +def _add(store: Store, **overrides): + fields = { + "item_key": "AAAAAAAA", + "attachment": "letter.pdf", + "protocol": "comment-stance", + "label": "support", + "quote": "we support the proposal", + "prefix": "", + "suffix": "", + "start_char": None, + "end_char": None, + "page": None, + "rects": None, + "note": "worth keeping", + "actor": "kert", + } + fields.update(overrides) + return store.add(**fields) + + +def _library(tmp_path): + path = tmp_path / "zotero.sqlite" + create_db(str(path)).close() + db = Db(str(path)) + kept = db.create_item(TYPE_MAP["journalArticle"], key="AAAAAAAA") + other = db.create_item(TYPE_MAP["journalArticle"], key="BBBBBBBB") + db.add_note(kept, "

    keep me

    ", title="Mine") + db.commit() + return db, path, kept, other + + +def _notes(db: Db, parent_id: int) -> list[tuple[str, str]]: + rows = db.con.execute( + """SELECT n.title, n.note FROM itemNotes n + WHERE n.parentItemID = ? ORDER BY n.itemID""", + (parent_id,), + ).fetchall() + return [(r[0], r[1]) for r in rows] + + +def test_dry_run_counts_and_writes_nothing(tmp_path): + engine = _engine() + ensure(engine) + store = Store(engine) + ann = _add(store, quote="a & c") + _add( + store, + item_key="NOZOTERO", + quote="a comment with no library row", + note="", + ) + _add( + store, + item_key="BBBBBBBB", + protocol="cms-disposition", + label="finalized-as-proposed", + quote="finalized as proposed", + note="", + ) + db, _path, kept, _other = _library(tmp_path) + try: + summary = export_notes(store, db, dry_run=True) + assert summary == { + "dry_run": True, + "create": 2, + "replace": 0, + "skipped_no_zotero_key": 1, + } + assert _notes(db, kept) == [("Mine", "

    keep me

    ")] + body = export_notes(store, db, dry_run=False) + assert body["dry_run"] is False + assert body["create"] == 2 + assert body["skipped_no_zotero_key"] == 1 + notes = _notes(db, kept) + assert notes[0] == ("Mine", "

    keep me

    ") + title, html = notes[1] + assert title == "Focus: comment-stance" + assert html.startswith("\n