feat(focus): export highlights to Zotero notes
This commit is contained in:
@@ -20,6 +20,7 @@ from cli.comments import app as comments_app
|
||||
from cli.db import app as db_app
|
||||
from cli.docs import app as docs_app
|
||||
from cli.drugs import app as drugs_app
|
||||
from cli.focus import app as focus_app
|
||||
from cli.generate import app as generate_app
|
||||
from cli.health import health
|
||||
from cli.lake import app as lake_app
|
||||
@@ -71,6 +72,7 @@ app.add_typer(
|
||||
app.add_typer(llm_app, name="llm", help="Local RAG — index, search, tag.")
|
||||
app.add_typer(db_app, name="db", help="DuckDB utilities.")
|
||||
app.add_typer(docs_app, name="docs", help="Documentation generation.")
|
||||
app.add_typer(focus_app, name="focus", help="Deep-focus annotations.")
|
||||
app.add_typer(api_app, name="api", help="API server.")
|
||||
app.add_typer(perf_app, name="perf", help="Pipeline telemetry utilities.")
|
||||
app.add_typer(
|
||||
|
||||
90
src/cli/focus.py
Normal file
90
src/cli/focus.py
Normal file
@@ -0,0 +1,90 @@
|
||||
"""stack focus — deep-focus annotations."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import typer
|
||||
|
||||
app = typer.Typer(no_args_is_help=True)
|
||||
|
||||
_DEFAULT_DB = Path("data/zotero/data/zotero.sqlite")
|
||||
|
||||
|
||||
def _live_store(): # pragma: no cover
|
||||
"""The annotation store on the llm database."""
|
||||
from focus.schema import ensure
|
||||
from focus.store import Store
|
||||
from llm.config import load as load_config
|
||||
from llm.index import _engine
|
||||
|
||||
engine = _engine(load_config())
|
||||
ensure(engine)
|
||||
return Store(engine), engine
|
||||
|
||||
|
||||
@app.command("export-notes")
|
||||
def export_notes_cmd(
|
||||
protocol: str | None = typer.Option(
|
||||
None,
|
||||
"--protocol",
|
||||
help="Only this protocol. Default: every protocol.",
|
||||
),
|
||||
write: bool = typer.Option(
|
||||
False,
|
||||
"--write",
|
||||
help="Write notes. Without this flag the command only reports.",
|
||||
),
|
||||
db: Path | None = typer.Option(
|
||||
None,
|
||||
"--db",
|
||||
help=(
|
||||
f"Zotero sqlite. Default: {_DEFAULT_DB}. "
|
||||
"Do not pass --write while Zotero is running."
|
||||
),
|
||||
),
|
||||
focus_db: str = typer.Option(
|
||||
"",
|
||||
"--focus-db",
|
||||
help="SQLAlchemy URL for the annotation store. Default: the llm database.",
|
||||
),
|
||||
) -> None:
|
||||
"""Export human highlights to one Zotero note per item and protocol.
|
||||
|
||||
Dry-run unless ``--write``. Does not schedule itself and does not
|
||||
write labels.sqlite. Stop Zotero before ``--write`` on the live database.
|
||||
"""
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
from focus.export import export_notes
|
||||
from focus.schema import ensure
|
||||
from focus.store import Store
|
||||
from zot.db import Db
|
||||
|
||||
path = db or _DEFAULT_DB
|
||||
if not path.is_file():
|
||||
raise typer.BadParameter(f"Zotero SQLite not found: {path}")
|
||||
if focus_db:
|
||||
engine = create_engine(focus_db)
|
||||
ensure(engine)
|
||||
store = Store(engine)
|
||||
else: # pragma: no cover
|
||||
store, engine = _live_store()
|
||||
try:
|
||||
with Db(str(path)) as zot:
|
||||
try:
|
||||
summary = export_notes(
|
||||
store,
|
||||
zot,
|
||||
protocol=protocol or None,
|
||||
dry_run=not write,
|
||||
)
|
||||
except ValueError as exc:
|
||||
typer.echo(str(exc), err=True)
|
||||
raise typer.Exit(code=1) from exc
|
||||
if write:
|
||||
zot.commit()
|
||||
finally:
|
||||
engine.dispose()
|
||||
for key in ("dry_run", "create", "replace", "skipped_no_zotero_key"):
|
||||
typer.echo(f"{key}: {summary[key]}")
|
||||
126
src/focus/export.py
Normal file
126
src/focus/export.py
Normal file
@@ -0,0 +1,126 @@
|
||||
"""Export human highlights to Zotero notes.
|
||||
|
||||
One note per (item, protocol). The marker ``<!-- focus:<protocol> -->``
|
||||
makes a second run replace that note and leave every other note alone.
|
||||
Dry-run is the default. This module does not write ``labels.sqlite``
|
||||
and does not schedule itself.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from focus.protocols import load
|
||||
from focus.store import Annotation, Store, annotation
|
||||
|
||||
_SUMMARY_KEYS = ("dry_run", "create", "replace", "skipped_no_zotero_key")
|
||||
|
||||
|
||||
def export_notes(
|
||||
store: Store,
|
||||
zot_db: Any,
|
||||
*,
|
||||
protocol: str | None = None,
|
||||
dry_run: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Render live human annotations into Zotero notes.
|
||||
|
||||
Items whose key is not in Zotero are counted and skipped. A model
|
||||
annotation is not a person's note, so it is left out. ``dry_run``
|
||||
reports the creates and replaces and writes nothing.
|
||||
"""
|
||||
if protocol is not None:
|
||||
load(protocol)
|
||||
summary = {
|
||||
"dry_run": dry_run,
|
||||
"create": 0,
|
||||
"replace": 0,
|
||||
"skipped_no_zotero_key": 0,
|
||||
}
|
||||
skipped: set[str] = set()
|
||||
for (item_key, proto_name), anns in _groups(store, protocol).items():
|
||||
parent = zot_db.find_item_by_key(item_key)
|
||||
if parent is None:
|
||||
skipped.add(item_key)
|
||||
continue
|
||||
action = _apply(
|
||||
zot_db,
|
||||
parent,
|
||||
proto_name,
|
||||
_render(proto_name, anns),
|
||||
dry_run=dry_run,
|
||||
)
|
||||
summary[action] += 1
|
||||
summary["skipped_no_zotero_key"] = len(skipped)
|
||||
return {key: summary[key] for key in _SUMMARY_KEYS}
|
||||
|
||||
|
||||
def _groups(
|
||||
store: Store, protocol: str | None
|
||||
) -> dict[tuple[str, str], list[Annotation]]:
|
||||
with store.engine.connect() as conn:
|
||||
keys = [
|
||||
row[0]
|
||||
for row in conn.execute(
|
||||
select(annotation.c.item_key).distinct().order_by(annotation.c.item_key)
|
||||
)
|
||||
]
|
||||
grouped: dict[tuple[str, str], list[Annotation]] = {}
|
||||
for key in keys:
|
||||
for ann in store.list(key, protocol):
|
||||
if ann.provenance != "human":
|
||||
continue
|
||||
grouped.setdefault((ann.item_key, ann.protocol), []).append(ann)
|
||||
return grouped
|
||||
|
||||
|
||||
def _render(protocol: str, anns: list[Annotation]) -> str:
|
||||
proto = load(protocol)
|
||||
lines = []
|
||||
for ann in anns:
|
||||
label = proto.label(ann.label)
|
||||
text = label.text if label is not None else ann.label
|
||||
parts = [html.escape(text), html.escape(ann.quote)]
|
||||
if ann.note:
|
||||
parts.append(html.escape(ann.note))
|
||||
lines.append(
|
||||
f'<li data-focus-id="{html.escape(ann.id)}">{" — ".join(parts)}</li>'
|
||||
)
|
||||
body = "\n".join(lines)
|
||||
return f"<!-- focus:{protocol} -->\n<ul>\n{body}\n</ul>\n"
|
||||
|
||||
|
||||
def _apply(
|
||||
zot_db: Any, parent_id: int, protocol: str, body: str, *, dry_run: bool
|
||||
) -> str:
|
||||
marker = f"<!-- focus:{protocol} -->"
|
||||
rows = zot_db.con.execute(
|
||||
"""SELECT itemID, note FROM itemNotes
|
||||
WHERE parentItemID = ? ORDER BY itemID""",
|
||||
(parent_id,),
|
||||
).fetchall()
|
||||
matches = [row[0] for row in rows if row[1] and marker in row[1]]
|
||||
if dry_run:
|
||||
return "replace" if matches else "create"
|
||||
title = f"Focus: {protocol}"
|
||||
if matches:
|
||||
from zot.db import now_iso
|
||||
|
||||
ts = now_iso()
|
||||
for note_id in matches:
|
||||
zot_db.con.execute(
|
||||
"UPDATE itemNotes SET note = ?, title = ? WHERE itemID = ?",
|
||||
(body, title, note_id),
|
||||
)
|
||||
zot_db.con.execute(
|
||||
"""UPDATE items
|
||||
SET dateModified = ?, clientDateModified = ?, synced = 0
|
||||
WHERE itemID = ?""",
|
||||
(ts, ts, note_id),
|
||||
)
|
||||
return "replace"
|
||||
zot_db.add_note(parent_id, body, title=title)
|
||||
return "create"
|
||||
212
tests/focus/test_export.py
Normal file
212
tests/focus/test_export.py
Normal file
@@ -0,0 +1,212 @@
|
||||
"""Zotero note export. Dry-run writes nothing. One note per item and protocol."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.pool import StaticPool
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from cli import app
|
||||
from focus.export import export_notes
|
||||
from focus.schema import ensure
|
||||
from focus.store import Store
|
||||
from zot.db import TYPE_MAP, Db
|
||||
from zot.schema import create_db
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
def _engine():
|
||||
return create_engine(
|
||||
"sqlite://",
|
||||
poolclass=StaticPool,
|
||||
connect_args={"check_same_thread": False},
|
||||
)
|
||||
|
||||
|
||||
def _add(store: Store, **overrides):
|
||||
fields = {
|
||||
"item_key": "AAAAAAAA",
|
||||
"attachment": "letter.pdf",
|
||||
"protocol": "comment-stance",
|
||||
"label": "support",
|
||||
"quote": "we support the proposal",
|
||||
"prefix": "",
|
||||
"suffix": "",
|
||||
"start_char": None,
|
||||
"end_char": None,
|
||||
"page": None,
|
||||
"rects": None,
|
||||
"note": "worth keeping",
|
||||
"actor": "kert",
|
||||
}
|
||||
fields.update(overrides)
|
||||
return store.add(**fields)
|
||||
|
||||
|
||||
def _library(tmp_path):
|
||||
path = tmp_path / "zotero.sqlite"
|
||||
create_db(str(path)).close()
|
||||
db = Db(str(path))
|
||||
kept = db.create_item(TYPE_MAP["journalArticle"], key="AAAAAAAA")
|
||||
other = db.create_item(TYPE_MAP["journalArticle"], key="BBBBBBBB")
|
||||
db.add_note(kept, "<p>keep me</p>", title="Mine")
|
||||
db.commit()
|
||||
return db, path, kept, other
|
||||
|
||||
|
||||
def _notes(db: Db, parent_id: int) -> list[tuple[str, str]]:
|
||||
rows = db.con.execute(
|
||||
"""SELECT n.title, n.note FROM itemNotes n
|
||||
WHERE n.parentItemID = ? ORDER BY n.itemID""",
|
||||
(parent_id,),
|
||||
).fetchall()
|
||||
return [(r[0], r[1]) for r in rows]
|
||||
|
||||
|
||||
def test_dry_run_counts_and_writes_nothing(tmp_path):
|
||||
engine = _engine()
|
||||
ensure(engine)
|
||||
store = Store(engine)
|
||||
ann = _add(store, quote="a <b>& c")
|
||||
_add(
|
||||
store,
|
||||
item_key="NOZOTERO",
|
||||
quote="a comment with no library row",
|
||||
note="",
|
||||
)
|
||||
_add(
|
||||
store,
|
||||
item_key="BBBBBBBB",
|
||||
protocol="cms-disposition",
|
||||
label="finalized-as-proposed",
|
||||
quote="finalized as proposed",
|
||||
note="",
|
||||
)
|
||||
db, _path, kept, _other = _library(tmp_path)
|
||||
try:
|
||||
summary = export_notes(store, db, dry_run=True)
|
||||
assert summary == {
|
||||
"dry_run": True,
|
||||
"create": 2,
|
||||
"replace": 0,
|
||||
"skipped_no_zotero_key": 1,
|
||||
}
|
||||
assert _notes(db, kept) == [("Mine", "<p>keep me</p>")]
|
||||
body = export_notes(store, db, dry_run=False)
|
||||
assert body["dry_run"] is False
|
||||
assert body["create"] == 2
|
||||
assert body["skipped_no_zotero_key"] == 1
|
||||
notes = _notes(db, kept)
|
||||
assert notes[0] == ("Mine", "<p>keep me</p>")
|
||||
title, html = notes[1]
|
||||
assert title == "Focus: comment-stance"
|
||||
assert html.startswith("<!-- focus:comment-stance -->\n<ul>\n")
|
||||
assert f'data-focus-id="{ann.id}"' in html
|
||||
assert "Support" in html
|
||||
assert "a <b>& c" in html
|
||||
assert "worth keeping" in html
|
||||
assert "<b>" not in html
|
||||
again = export_notes(store, db, dry_run=True)
|
||||
assert again["create"] == 0
|
||||
assert again["replace"] == 2
|
||||
assert _notes(db, kept)[1][1] == html
|
||||
finally:
|
||||
db.close()
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def test_replace_is_idempotent_and_skips_deleted_and_model(tmp_path):
|
||||
engine = _engine()
|
||||
ensure(engine)
|
||||
store = Store(engine)
|
||||
ann = _add(store)
|
||||
gone = _add(store, quote="this was deleted", note="")
|
||||
store.delete(gone.id, "kert")
|
||||
_add(
|
||||
store,
|
||||
quote="model said so",
|
||||
note="not a person",
|
||||
provenance="model",
|
||||
model="local",
|
||||
)
|
||||
db, _path, kept, _other = _library(tmp_path)
|
||||
try:
|
||||
export_notes(store, db, dry_run=False)
|
||||
store.update_note(ann.id, "edited", "kert")
|
||||
second = export_notes(store, db, protocol="comment-stance", dry_run=False)
|
||||
assert second == {
|
||||
"dry_run": False,
|
||||
"create": 0,
|
||||
"replace": 1,
|
||||
"skipped_no_zotero_key": 0,
|
||||
}
|
||||
notes = _notes(db, kept)
|
||||
assert len(notes) == 2
|
||||
assert notes[0] == ("Mine", "<p>keep me</p>")
|
||||
assert "edited" in notes[1][1]
|
||||
assert "this was deleted" not in notes[1][1]
|
||||
assert "model said so" not in notes[1][1]
|
||||
assert notes[1][1].count("data-focus-id") == 1
|
||||
finally:
|
||||
db.close()
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def test_cli_dry_run_then_write(tmp_path):
|
||||
focus_path = tmp_path / "focus.sqlite"
|
||||
url = "sqlite:///" + str(focus_path)
|
||||
engine = create_engine(url)
|
||||
ensure(engine)
|
||||
store = Store(engine)
|
||||
_add(store, quote="from the cli")
|
||||
engine.dispose()
|
||||
db, path, kept, _other = _library(tmp_path)
|
||||
db.close()
|
||||
|
||||
dry = runner.invoke(
|
||||
app,
|
||||
["focus", "export-notes", "--db", str(path), "--focus-db", url],
|
||||
)
|
||||
assert dry.exit_code == 0, dry.output
|
||||
assert "dry_run: True" in dry.stdout
|
||||
assert "create: 1" in dry.stdout
|
||||
with Db(str(path)) as check:
|
||||
assert _notes(check, kept) == [("Mine", "<p>keep me</p>")]
|
||||
|
||||
wrote = runner.invoke(
|
||||
app,
|
||||
["focus", "export-notes", "--db", str(path), "--focus-db", url, "--write"],
|
||||
)
|
||||
assert wrote.exit_code == 0, wrote.output
|
||||
assert "dry_run: False" in wrote.stdout
|
||||
assert "create: 1" in wrote.stdout
|
||||
with Db(str(path)) as check:
|
||||
notes = _notes(check, kept)
|
||||
assert notes[1][0] == "Focus: comment-stance"
|
||||
assert "from the cli" in notes[1][1]
|
||||
assert "<!-- focus:comment-stance -->" in notes[1][1]
|
||||
|
||||
|
||||
def test_cli_rejects_an_unknown_protocol(tmp_path):
|
||||
url = "sqlite:///" + str(tmp_path / "focus.sqlite")
|
||||
engine = create_engine(url)
|
||||
ensure(engine)
|
||||
engine.dispose()
|
||||
_db, path, _kept, _other = _library(tmp_path)
|
||||
_db.close()
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"focus",
|
||||
"export-notes",
|
||||
"--db",
|
||||
str(path),
|
||||
"--focus-db",
|
||||
url,
|
||||
"--protocol",
|
||||
"no-such",
|
||||
],
|
||||
)
|
||||
assert result.exit_code != 0
|
||||
assert "no-such" in result.output
|
||||
Reference in New Issue
Block a user