feat(llm): corpus indexer skips items tagged llm:skip (refs #688)

This commit is contained in:
kert
2026-09-09 16:15:29 -04:00
parent c244cebba1
commit 5009e9de6f
2 changed files with 15 additions and 4 deletions

View File

@@ -495,13 +495,17 @@ def _attachment_sections(
def iter_corpus_refs( def iter_corpus_refs(
store: Store, *, tag: str = "", zotero: "ZoteroPdfIndex | None" = None store: Store, *, tag: str = "", zotero: "ZoteroPdfIndex | None" = None
) -> Iterator[DocRef]: ) -> Iterator[DocRef]:
"""One DocRef per non-comment item. Fingerprint = updated_at + the """One DocRef per non-comment, non-skipped item. Fingerprint =
bib attachment file stats; the Zotero fallback is only consulted by updated_at + the bib attachment file stats; the Zotero fallback is
``load()``.""" only consulted by ``load()``.
Items tagged ``llm:skip`` are excluded — the escape hatch for
material that must never enter the indexer (e.g. copyrighted CPT
editions imported for citation only, not for RAG text)."""
sql = ( sql = (
"SELECT i.key, COALESCE(i.updated_at,'') AS updated_at FROM items i " "SELECT i.key, COALESCE(i.updated_at,'') AS updated_at FROM items i "
"WHERE i.id NOT IN (SELECT item_id FROM item_tags WHERE tag_id IN " "WHERE i.id NOT IN (SELECT item_id FROM item_tags WHERE tag_id IN "
"(SELECT id FROM tags WHERE name = 'doctype:comment'))" "(SELECT id FROM tags WHERE name IN ('doctype:comment', 'llm:skip')))"
+ ( + (
" AND i.id IN (SELECT item_id FROM item_tags WHERE tag_id IN " " AND i.id IN (SELECT item_id FROM item_tags WHERE tag_id IN "
"(SELECT id FROM tags WHERE name = ?))" "(SELECT id FROM tags WHERE name = ?))"

View File

@@ -133,6 +133,13 @@ class TestCorpusDocs:
def test_comments_excluded_from_corpus(self, store): def test_comments_excluded_from_corpus(self, store):
assert all(d.metadata["doctype"] != "comment" for d in iter_corpus_docs(store)) assert all(d.metadata["doctype"] != "comment" for d in iter_corpus_docs(store))
def test_llm_skip_tag_excludes_item_from_corpus(self, store):
key = store.create(
Item(item_type="source", title="CPT edition", abstract="Descriptor text.")
)
store.add_tag(key, "llm:skip")
assert key not in {d.key for d in iter_corpus_docs(store)}
def test_attachment_text_sectioned_and_listed_in_files(self, store, tmp_path): def test_attachment_text_sectioned_and_listed_in_files(self, store, tmp_path):
key = store.create(Item(item_type="rule", title="Rule with attachment")) key = store.create(Item(item_type="rule", title="Rule with attachment"))
store.add_tag(key, "year:2021") store.add_tag(key, "year:2021")