feat(llm): corpus indexer skips items tagged llm:skip (refs #688)
This commit is contained in:
@@ -495,13 +495,17 @@ def _attachment_sections(
|
|||||||
def iter_corpus_refs(
|
def iter_corpus_refs(
|
||||||
store: Store, *, tag: str = "", zotero: "ZoteroPdfIndex | None" = None
|
store: Store, *, tag: str = "", zotero: "ZoteroPdfIndex | None" = None
|
||||||
) -> Iterator[DocRef]:
|
) -> Iterator[DocRef]:
|
||||||
"""One DocRef per non-comment item. Fingerprint = updated_at + the
|
"""One DocRef per non-comment, non-skipped item. Fingerprint =
|
||||||
bib attachment file stats; the Zotero fallback is only consulted by
|
updated_at + the bib attachment file stats; the Zotero fallback is
|
||||||
``load()``."""
|
only consulted by ``load()``.
|
||||||
|
|
||||||
|
Items tagged ``llm:skip`` are excluded — the escape hatch for
|
||||||
|
material that must never enter the indexer (e.g. copyrighted CPT
|
||||||
|
editions imported for citation only, not for RAG text)."""
|
||||||
sql = (
|
sql = (
|
||||||
"SELECT i.key, COALESCE(i.updated_at,'') AS updated_at FROM items i "
|
"SELECT i.key, COALESCE(i.updated_at,'') AS updated_at FROM items i "
|
||||||
"WHERE i.id NOT IN (SELECT item_id FROM item_tags WHERE tag_id IN "
|
"WHERE i.id NOT IN (SELECT item_id FROM item_tags WHERE tag_id IN "
|
||||||
"(SELECT id FROM tags WHERE name = 'doctype:comment'))"
|
"(SELECT id FROM tags WHERE name IN ('doctype:comment', 'llm:skip')))"
|
||||||
+ (
|
+ (
|
||||||
" AND i.id IN (SELECT item_id FROM item_tags WHERE tag_id IN "
|
" AND i.id IN (SELECT item_id FROM item_tags WHERE tag_id IN "
|
||||||
"(SELECT id FROM tags WHERE name = ?))"
|
"(SELECT id FROM tags WHERE name = ?))"
|
||||||
|
|||||||
@@ -133,6 +133,13 @@ class TestCorpusDocs:
|
|||||||
def test_comments_excluded_from_corpus(self, store):
|
def test_comments_excluded_from_corpus(self, store):
|
||||||
assert all(d.metadata["doctype"] != "comment" for d in iter_corpus_docs(store))
|
assert all(d.metadata["doctype"] != "comment" for d in iter_corpus_docs(store))
|
||||||
|
|
||||||
|
def test_llm_skip_tag_excludes_item_from_corpus(self, store):
|
||||||
|
key = store.create(
|
||||||
|
Item(item_type="source", title="CPT edition", abstract="Descriptor text.")
|
||||||
|
)
|
||||||
|
store.add_tag(key, "llm:skip")
|
||||||
|
assert key not in {d.key for d in iter_corpus_docs(store)}
|
||||||
|
|
||||||
def test_attachment_text_sectioned_and_listed_in_files(self, store, tmp_path):
|
def test_attachment_text_sectioned_and_listed_in_files(self, store, tmp_path):
|
||||||
key = store.create(Item(item_type="rule", title="Rule with attachment"))
|
key = store.create(Item(item_type="rule", title="Rule with attachment"))
|
||||||
store.add_tag(key, "year:2021")
|
store.add_tag(key, "year:2021")
|
||||||
|
|||||||
Reference in New Issue
Block a user