fix(llm): chat-path dossier stub moves to tests/llm/conftest; family-name words never rank dossier facts
Some checks failed
Infra CI / zotero (push) Waiting to run
CI / lint (push) Successful in 1m5s
CI / notebooks-smoke (push) Successful in 2m17s
Deploy / changes (push) Successful in 23s
CI / test (push) Failing after 5m19s
Infra CI / docs (push) Successful in 1m0s
Infra CI / notebooks (push) Successful in 3m17s
Infra CI / api (push) Successful in 1m33s
Infra CI / llm (push) Successful in 1m14s
Infra CI / bcda (push) Successful in 2m13s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Infra CI / mc (push) Successful in 27s
Deploy / report (push) Has been cancelled
Deploy / bcda (push) Has been cancelled
Deploy / llm (push) Has been cancelled
Deploy / mc (push) Has been cancelled
Deploy / api (push) Has been cancelled
Deploy / docs (push) Has been cancelled
Some checks failed
Infra CI / zotero (push) Waiting to run
CI / lint (push) Successful in 1m5s
CI / notebooks-smoke (push) Successful in 2m17s
Deploy / changes (push) Successful in 23s
CI / test (push) Failing after 5m19s
Infra CI / docs (push) Successful in 1m0s
Infra CI / notebooks (push) Successful in 3m17s
Infra CI / api (push) Successful in 1m33s
Infra CI / llm (push) Successful in 1m14s
Infra CI / bcda (push) Successful in 2m13s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Infra CI / mc (push) Successful in 27s
Deploy / report (push) Has been cancelled
Deploy / bcda (push) Has been cancelled
Deploy / llm (push) Has been cancelled
Deploy / mc (push) Has been cancelled
Deploy / api (push) Has been cancelled
Deploy / docs (push) Has been cancelled
This commit is contained in:
@@ -125,6 +125,7 @@ def match_facts(
|
||||
years: Collection[int],
|
||||
limit: int = 0,
|
||||
question: str = "",
|
||||
family_name: str = "",
|
||||
) -> tuple[str, ...]:
|
||||
"""Ids of the facts that bear on the question, interleaved round-robin
|
||||
by kind with the newest year first. A fact matches when its codes overlap
|
||||
@@ -132,7 +133,7 @@ def match_facts(
|
||||
unflagged fact is a candidate. Flagged, stance and payment facts never
|
||||
appear (stance counts are their own unit, payment rows are the Valuation
|
||||
block's). Facts sharing a telling word with *question* ("deleted") come
|
||||
first."""
|
||||
first; the family's own name ("Home or Residence Services") never ranks."""
|
||||
bad = set(flagged)
|
||||
pool = [
|
||||
f for f in facts if f.kind in _KIND_ORDER and not f.note and f.id not in bad
|
||||
@@ -145,7 +146,7 @@ def match_facts(
|
||||
own = {f.id: _stems(f.text) for f in pool}
|
||||
stems = {
|
||||
w
|
||||
for w in _stems(question)
|
||||
for w in _stems(question) - _stems(family_name)
|
||||
if sum(w in own[f.id] for f in pool) <= 0.4 * len(pool)
|
||||
}
|
||||
scored = [f for f in pool if stems & own[f.id]]
|
||||
@@ -350,6 +351,7 @@ def dossier_evidence(question: str, cfg: LlmConfig) -> DossierEvidence | None:
|
||||
codes,
|
||||
question_years(question),
|
||||
question=question,
|
||||
family_name=d.name,
|
||||
),
|
||||
stance_intent=stance_intent(question),
|
||||
)
|
||||
|
||||
21
tests/llm/conftest.py
Normal file
21
tests/llm/conftest.py
Normal file
@@ -0,0 +1,21 @@
|
||||
"""Shared fixtures for tests/llm."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_real_dossier_lookup(request):
|
||||
"""``dossier_evidence`` warms the real replica, which merges its ~8.6k
|
||||
derived families into the process-global registry and breaks unrelated
|
||||
tests (CCM conflict phrases, tests/pfs/test_families.py). Chat tests never
|
||||
ask for a dossier unless they say so; mark a test ``real_dossier`` to run
|
||||
the real lookup (against a temp replica)."""
|
||||
if request.node.get_closest_marker("real_dossier"):
|
||||
yield
|
||||
return
|
||||
with patch("llm.rag.dossier_evidence", return_value=None):
|
||||
yield
|
||||
@@ -331,6 +331,20 @@ def test_question_words_that_tell_facts_apart_come_first():
|
||||
assert plain[0] == "F1"
|
||||
asked = match_facts(facts, (), ("99341",), (), question="When were codes deleted?")
|
||||
assert asked[0] == "F3" and set(asked) == set(plain)
|
||||
# the family's own name words never rank, however rare in the facts
|
||||
named = [
|
||||
_fact(1, "round", "a", year=2027),
|
||||
_fact(2, "event", "Residence rule, 2019.", year=2019),
|
||||
_fact(3, "event", "b", year=2020),
|
||||
_fact(4, "event", "c", year=2021),
|
||||
_fact(5, "event", "d", year=2022),
|
||||
_fact(6, "event", "e", year=2023),
|
||||
]
|
||||
q = "Which residence societies supported it?"
|
||||
assert match_facts(named, (), ("99341",), (), question=q)[:2] == ("F2", "F1")
|
||||
assert match_facts(
|
||||
named, (), ("99341",), (), question=q, family_name="Home or Residence Services"
|
||||
) == match_facts(named, (), ("99341",), ())
|
||||
# a word found in most candidates ("visit") tells nothing
|
||||
assert match_facts(facts, (), ("99341",), (), question="home visit?") == plain
|
||||
|
||||
|
||||
@@ -57,18 +57,6 @@ def _reset_docket_era_cache():
|
||||
_reset()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _no_real_dossier_lookup(request):
|
||||
"""``dossier_evidence`` warms the real replica, which merges its derived
|
||||
families into the process-global registry (the leak the control test below
|
||||
documents); tests of the older events never ask for a dossier."""
|
||||
if request.node.get_closest_marker("real_dossier"):
|
||||
yield
|
||||
return
|
||||
with patch("llm.rag.dossier_evidence", return_value=None):
|
||||
yield
|
||||
|
||||
|
||||
def _doc(text, **md):
|
||||
return Document(page_content=text, metadata=md)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user