diff --git a/src/llm/source.py b/src/llm/source.py index 532c397..a149007 100644 --- a/src/llm/source.py +++ b/src/llm/source.py @@ -87,6 +87,26 @@ def _attachment_paths(store: Store, item_key: str) -> list[Path]: return [Path(r[0]) for r in rows] +def _attachment_paths_by_key(store: Store) -> dict[str, list[Path]]: + """Every item's attachment paths, grouped by key, in one scan. + + ``iter_rule_refs``/``iter_corpus_refs`` used to call + :func:`_attachment_paths` once per item just to build the + fingerprint — one query per rule/corpus item on every run, before + any change is even known. One table scan replaces that N+1.""" + rows = ( + store._con() + .execute( + "SELECT i.key, a.storage_path FROM attachments a JOIN items i ON i.id = a.item_id" + ) + .fetchall() + ) + by_key: dict[str, list[Path]] = {} + for key, path in rows: + by_key.setdefault(key, []).append(Path(path)) + return by_key + + # ── comments ── @@ -183,12 +203,12 @@ def iter_comment_refs( _, body = parse_combined(combined.read_text()) if body.strip(): return Doc(key=key, text=body, metadata=meta, files=files) - row = ( + item_row = ( store._con() .execute("SELECT abstract FROM items WHERE key = ?", (key,)) .fetchone() ) - abstract = (row["abstract"] if row else "") or "" + abstract = (item_row["abstract"] if item_row else "") or "" return ( Doc(key=key, text=abstract, metadata=meta) if abstract.strip() else None ) @@ -208,6 +228,9 @@ def iter_comment_docs( yield doc +# ── rules ── + + def _attachment_text(store: Store, item_key: str) -> str: from rex.comments.combine import extract_attachment @@ -265,9 +288,6 @@ def _anchor_doc(store: Store, item_key: str) -> tuple[str, str]: return (row[0], str(row[1])) if row else ("", "") -# ── rules ── - - def iter_rule_refs( store: Store, *, keys: tuple[str, ...] = (), tag: str = "" ) -> Iterator[DocRef]: @@ -291,6 +311,13 @@ def iter_rule_refs( ) .fetchall() ) + # Attachment paths are only needed for rows without an anchor sha; + # one batched scan replaces one _attachment_paths query per such row. + paths_by_key = ( + _attachment_paths_by_key(store) + if any(not r["anchor_sha"] for r in rows) + else {} + ) for row in rows: key = row["key"] if keys and key not in keys: @@ -300,11 +327,7 @@ def iter_rule_refs( # item's own metadata (title, cms-rule: tag, date). fp = f"anchors:{row['anchor_sha']}|{row['updated_at']}" else: - fp = ( - fingerprint_files(_attachment_paths(store, key)) - + "|" - + row["updated_at"] - ) + fp = fingerprint_files(paths_by_key.get(key, [])) + "|" + row["updated_at"] def _load(key=key) -> Doc | None: return _build_rule_doc(store, store.get(key)) @@ -524,11 +547,16 @@ def iter_corpus_refs( ) + " ORDER BY i.id" ) - for row in store._con().execute(sql, (tag,) if tag else ()).fetchall(): + rows = store._con().execute(sql, (tag,) if tag else ()).fetchall() + # One batched scan replaces one _attachment_paths query per item — + # the fingerprint step used to cost a query per corpus item before + # any change was even known. + paths_by_key = _attachment_paths_by_key(store) + for row in rows: key = row["key"] if keys and key not in keys: continue - fp = row["updated_at"] + "|" + fingerprint_files(_attachment_paths(store, key)) + fp = row["updated_at"] + "|" + fingerprint_files(paths_by_key.get(key, [])) def _load(key=key) -> Doc | None: return _build_corpus_doc(store, store.get(key), zotero)