feat(focus): export highlights to Zotero notes

This commit is contained in:
kert
2026-11-14 11:31:53 -05:00
parent 59fa4a5790
commit 8bd413fe44
4 changed files with 430 additions and 0 deletions

View File

@@ -20,6 +20,7 @@ from cli.comments import app as comments_app
from cli.db import app as db_app
from cli.docs import app as docs_app
from cli.drugs import app as drugs_app
from cli.focus import app as focus_app
from cli.generate import app as generate_app
from cli.health import health
from cli.lake import app as lake_app
@@ -71,6 +72,7 @@ app.add_typer(
app.add_typer(llm_app, name="llm", help="Local RAG — index, search, tag.")
app.add_typer(db_app, name="db", help="DuckDB utilities.")
app.add_typer(docs_app, name="docs", help="Documentation generation.")
app.add_typer(focus_app, name="focus", help="Deep-focus annotations.")
app.add_typer(api_app, name="api", help="API server.")
app.add_typer(perf_app, name="perf", help="Pipeline telemetry utilities.")
app.add_typer(

90
src/cli/focus.py Normal file
View File

@@ -0,0 +1,90 @@
"""stack focus — deep-focus annotations."""
from __future__ import annotations
from pathlib import Path
import typer
app = typer.Typer(no_args_is_help=True)
_DEFAULT_DB = Path("data/zotero/data/zotero.sqlite")
def _live_store(): # pragma: no cover
"""The annotation store on the llm database."""
from focus.schema import ensure
from focus.store import Store
from llm.config import load as load_config
from llm.index import _engine
engine = _engine(load_config())
ensure(engine)
return Store(engine), engine
@app.command("export-notes")
def export_notes_cmd(
protocol: str | None = typer.Option(
None,
"--protocol",
help="Only this protocol. Default: every protocol.",
),
write: bool = typer.Option(
False,
"--write",
help="Write notes. Without this flag the command only reports.",
),
db: Path | None = typer.Option(
None,
"--db",
help=(
f"Zotero sqlite. Default: {_DEFAULT_DB}. "
"Do not pass --write while Zotero is running."
),
),
focus_db: str = typer.Option(
"",
"--focus-db",
help="SQLAlchemy URL for the annotation store. Default: the llm database.",
),
) -> None:
"""Export human highlights to one Zotero note per item and protocol.
Dry-run unless ``--write``. Does not schedule itself and does not
write labels.sqlite. Stop Zotero before ``--write`` on the live database.
"""
from sqlalchemy import create_engine
from focus.export import export_notes
from focus.schema import ensure
from focus.store import Store
from zot.db import Db
path = db or _DEFAULT_DB
if not path.is_file():
raise typer.BadParameter(f"Zotero SQLite not found: {path}")
if focus_db:
engine = create_engine(focus_db)
ensure(engine)
store = Store(engine)
else: # pragma: no cover
store, engine = _live_store()
try:
with Db(str(path)) as zot:
try:
summary = export_notes(
store,
zot,
protocol=protocol or None,
dry_run=not write,
)
except ValueError as exc:
typer.echo(str(exc), err=True)
raise typer.Exit(code=1) from exc
if write:
zot.commit()
finally:
engine.dispose()
for key in ("dry_run", "create", "replace", "skipped_no_zotero_key"):
typer.echo(f"{key}: {summary[key]}")

126
src/focus/export.py Normal file
View File

@@ -0,0 +1,126 @@
"""Export human highlights to Zotero notes.
One note per (item, protocol). The marker ``<!-- focus:<protocol> -->``
makes a second run replace that note and leave every other note alone.
Dry-run is the default. This module does not write ``labels.sqlite``
and does not schedule itself.
"""
from __future__ import annotations
import html
from typing import Any
from sqlalchemy import select
from focus.protocols import load
from focus.store import Annotation, Store, annotation
_SUMMARY_KEYS = ("dry_run", "create", "replace", "skipped_no_zotero_key")
def export_notes(
store: Store,
zot_db: Any,
*,
protocol: str | None = None,
dry_run: bool = True,
) -> dict[str, Any]:
"""Render live human annotations into Zotero notes.
Items whose key is not in Zotero are counted and skipped. A model
annotation is not a person's note, so it is left out. ``dry_run``
reports the creates and replaces and writes nothing.
"""
if protocol is not None:
load(protocol)
summary = {
"dry_run": dry_run,
"create": 0,
"replace": 0,
"skipped_no_zotero_key": 0,
}
skipped: set[str] = set()
for (item_key, proto_name), anns in _groups(store, protocol).items():
parent = zot_db.find_item_by_key(item_key)
if parent is None:
skipped.add(item_key)
continue
action = _apply(
zot_db,
parent,
proto_name,
_render(proto_name, anns),
dry_run=dry_run,
)
summary[action] += 1
summary["skipped_no_zotero_key"] = len(skipped)
return {key: summary[key] for key in _SUMMARY_KEYS}
def _groups(
store: Store, protocol: str | None
) -> dict[tuple[str, str], list[Annotation]]:
with store.engine.connect() as conn:
keys = [
row[0]
for row in conn.execute(
select(annotation.c.item_key).distinct().order_by(annotation.c.item_key)
)
]
grouped: dict[tuple[str, str], list[Annotation]] = {}
for key in keys:
for ann in store.list(key, protocol):
if ann.provenance != "human":
continue
grouped.setdefault((ann.item_key, ann.protocol), []).append(ann)
return grouped
def _render(protocol: str, anns: list[Annotation]) -> str:
proto = load(protocol)
lines = []
for ann in anns:
label = proto.label(ann.label)
text = label.text if label is not None else ann.label
parts = [html.escape(text), html.escape(ann.quote)]
if ann.note:
parts.append(html.escape(ann.note))
lines.append(
f'<li data-focus-id="{html.escape(ann.id)}">{" — ".join(parts)}</li>'
)
body = "\n".join(lines)
return f"<!-- focus:{protocol} -->\n<ul>\n{body}\n</ul>\n"
def _apply(
zot_db: Any, parent_id: int, protocol: str, body: str, *, dry_run: bool
) -> str:
marker = f"<!-- focus:{protocol} -->"
rows = zot_db.con.execute(
"""SELECT itemID, note FROM itemNotes
WHERE parentItemID = ? ORDER BY itemID""",
(parent_id,),
).fetchall()
matches = [row[0] for row in rows if row[1] and marker in row[1]]
if dry_run:
return "replace" if matches else "create"
title = f"Focus: {protocol}"
if matches:
from zot.db import now_iso
ts = now_iso()
for note_id in matches:
zot_db.con.execute(
"UPDATE itemNotes SET note = ?, title = ? WHERE itemID = ?",
(body, title, note_id),
)
zot_db.con.execute(
"""UPDATE items
SET dateModified = ?, clientDateModified = ?, synced = 0
WHERE itemID = ?""",
(ts, ts, note_id),
)
return "replace"
zot_db.add_note(parent_id, body, title=title)
return "create"

212
tests/focus/test_export.py Normal file
View File

@@ -0,0 +1,212 @@
"""Zotero note export. Dry-run writes nothing. One note per item and protocol."""
from __future__ import annotations
from sqlalchemy import create_engine
from sqlalchemy.pool import StaticPool
from typer.testing import CliRunner
from cli import app
from focus.export import export_notes
from focus.schema import ensure
from focus.store import Store
from zot.db import TYPE_MAP, Db
from zot.schema import create_db
runner = CliRunner()
def _engine():
return create_engine(
"sqlite://",
poolclass=StaticPool,
connect_args={"check_same_thread": False},
)
def _add(store: Store, **overrides):
fields = {
"item_key": "AAAAAAAA",
"attachment": "letter.pdf",
"protocol": "comment-stance",
"label": "support",
"quote": "we support the proposal",
"prefix": "",
"suffix": "",
"start_char": None,
"end_char": None,
"page": None,
"rects": None,
"note": "worth keeping",
"actor": "kert",
}
fields.update(overrides)
return store.add(**fields)
def _library(tmp_path):
path = tmp_path / "zotero.sqlite"
create_db(str(path)).close()
db = Db(str(path))
kept = db.create_item(TYPE_MAP["journalArticle"], key="AAAAAAAA")
other = db.create_item(TYPE_MAP["journalArticle"], key="BBBBBBBB")
db.add_note(kept, "<p>keep me</p>", title="Mine")
db.commit()
return db, path, kept, other
def _notes(db: Db, parent_id: int) -> list[tuple[str, str]]:
rows = db.con.execute(
"""SELECT n.title, n.note FROM itemNotes n
WHERE n.parentItemID = ? ORDER BY n.itemID""",
(parent_id,),
).fetchall()
return [(r[0], r[1]) for r in rows]
def test_dry_run_counts_and_writes_nothing(tmp_path):
engine = _engine()
ensure(engine)
store = Store(engine)
ann = _add(store, quote="a <b>& c")
_add(
store,
item_key="NOZOTERO",
quote="a comment with no library row",
note="",
)
_add(
store,
item_key="BBBBBBBB",
protocol="cms-disposition",
label="finalized-as-proposed",
quote="finalized as proposed",
note="",
)
db, _path, kept, _other = _library(tmp_path)
try:
summary = export_notes(store, db, dry_run=True)
assert summary == {
"dry_run": True,
"create": 2,
"replace": 0,
"skipped_no_zotero_key": 1,
}
assert _notes(db, kept) == [("Mine", "<p>keep me</p>")]
body = export_notes(store, db, dry_run=False)
assert body["dry_run"] is False
assert body["create"] == 2
assert body["skipped_no_zotero_key"] == 1
notes = _notes(db, kept)
assert notes[0] == ("Mine", "<p>keep me</p>")
title, html = notes[1]
assert title == "Focus: comment-stance"
assert html.startswith("<!-- focus:comment-stance -->\n<ul>\n")
assert f'data-focus-id="{ann.id}"' in html
assert "Support" in html
assert "a &lt;b&gt;&amp; c" in html
assert "worth keeping" in html
assert "<b>" not in html
again = export_notes(store, db, dry_run=True)
assert again["create"] == 0
assert again["replace"] == 2
assert _notes(db, kept)[1][1] == html
finally:
db.close()
engine.dispose()
def test_replace_is_idempotent_and_skips_deleted_and_model(tmp_path):
engine = _engine()
ensure(engine)
store = Store(engine)
ann = _add(store)
gone = _add(store, quote="this was deleted", note="")
store.delete(gone.id, "kert")
_add(
store,
quote="model said so",
note="not a person",
provenance="model",
model="local",
)
db, _path, kept, _other = _library(tmp_path)
try:
export_notes(store, db, dry_run=False)
store.update_note(ann.id, "edited", "kert")
second = export_notes(store, db, protocol="comment-stance", dry_run=False)
assert second == {
"dry_run": False,
"create": 0,
"replace": 1,
"skipped_no_zotero_key": 0,
}
notes = _notes(db, kept)
assert len(notes) == 2
assert notes[0] == ("Mine", "<p>keep me</p>")
assert "edited" in notes[1][1]
assert "this was deleted" not in notes[1][1]
assert "model said so" not in notes[1][1]
assert notes[1][1].count("data-focus-id") == 1
finally:
db.close()
engine.dispose()
def test_cli_dry_run_then_write(tmp_path):
focus_path = tmp_path / "focus.sqlite"
url = "sqlite:///" + str(focus_path)
engine = create_engine(url)
ensure(engine)
store = Store(engine)
_add(store, quote="from the cli")
engine.dispose()
db, path, kept, _other = _library(tmp_path)
db.close()
dry = runner.invoke(
app,
["focus", "export-notes", "--db", str(path), "--focus-db", url],
)
assert dry.exit_code == 0, dry.output
assert "dry_run: True" in dry.stdout
assert "create: 1" in dry.stdout
with Db(str(path)) as check:
assert _notes(check, kept) == [("Mine", "<p>keep me</p>")]
wrote = runner.invoke(
app,
["focus", "export-notes", "--db", str(path), "--focus-db", url, "--write"],
)
assert wrote.exit_code == 0, wrote.output
assert "dry_run: False" in wrote.stdout
assert "create: 1" in wrote.stdout
with Db(str(path)) as check:
notes = _notes(check, kept)
assert notes[1][0] == "Focus: comment-stance"
assert "from the cli" in notes[1][1]
assert "<!-- focus:comment-stance -->" in notes[1][1]
def test_cli_rejects_an_unknown_protocol(tmp_path):
url = "sqlite:///" + str(tmp_path / "focus.sqlite")
engine = create_engine(url)
ensure(engine)
engine.dispose()
_db, path, _kept, _other = _library(tmp_path)
_db.close()
result = runner.invoke(
app,
[
"focus",
"export-notes",
"--db",
str(path),
"--focus-db",
url,
"--protocol",
"no-such",
],
)
assert result.exit_code != 0
assert "no-such" in result.output