fix(llm): dossier prompt block is question-aware and compact — summary, timeline, matched facts, one stance unit; 35% budget share; default-years valuation prompt rows
Some checks failed
CI / lint (push) Successful in 1m16s
CI / notebooks-smoke (push) Successful in 2m0s
Deploy / changes (push) Successful in 15s
CI / test (push) Failing after 4m21s
Infra CI / zotero (push) Successful in 17s
Infra CI / notebooks (push) Successful in 1m4s
Infra CI / docs (push) Successful in 28s
Infra CI / api (push) Successful in 1m53s
Infra CI / bcda (push) Successful in 2m55s
Infra CI / llm (push) Successful in 1m42s
Infra CI / mc (push) Successful in 26s
Package Supply Chain / pkg-supply-chain (push) Successful in 2m42s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Successful in 1m49s
Deploy / bcda (push) Successful in 2m9s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 1m24s
Deploy / report (push) Successful in 17s
Some checks failed
CI / lint (push) Successful in 1m16s
CI / notebooks-smoke (push) Successful in 2m0s
Deploy / changes (push) Successful in 15s
CI / test (push) Failing after 4m21s
Infra CI / zotero (push) Successful in 17s
Infra CI / notebooks (push) Successful in 1m4s
Infra CI / docs (push) Successful in 28s
Infra CI / api (push) Successful in 1m53s
Infra CI / bcda (push) Successful in 2m55s
Infra CI / llm (push) Successful in 1m42s
Infra CI / mc (push) Successful in 26s
Package Supply Chain / pkg-supply-chain (push) Successful in 2m42s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Successful in 1m49s
Deploy / bcda (push) Successful in 2m9s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 1m24s
Deploy / report (push) Successful in 17s
This commit is contained in:
@@ -15,24 +15,29 @@ import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from typing import Any, Collection, Sequence
|
||||
|
||||
import llm.evidence as evidence
|
||||
from llm.config import LlmConfig
|
||||
from pfs.dossier import Fact, dossier_families, read_dossier, stance_accuracy_line
|
||||
from pfs.dossier import (
|
||||
Fact,
|
||||
TimelineEvent,
|
||||
dossier_families,
|
||||
read_dossier,
|
||||
stance_accuracy_line,
|
||||
)
|
||||
from pfs.families import CURATED_FAMILIES, detect_codes
|
||||
|
||||
__all__ = ["DossierEvidence", "dossier_evidence", "dossier_url"]
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
#: prompt priority: what the model is most likely asked about first. Payment
|
||||
#: facts come last; the Valuation block is the authority on payment.
|
||||
_PRIORITY = (
|
||||
#: round-robin order of the question-matched facts. Payment facts are left
|
||||
#: to the Valuation block; stance counts form their own unit.
|
||||
_KIND_ORDER = (
|
||||
"round",
|
||||
"event",
|
||||
"cms_summary",
|
||||
"stance_split",
|
||||
"link",
|
||||
"origin",
|
||||
"formation",
|
||||
@@ -41,17 +46,59 @@ _PRIORITY = (
|
||||
"impact",
|
||||
"seat",
|
||||
"finding",
|
||||
"payment",
|
||||
)
|
||||
_RANK = {k: i for i, k in enumerate(_PRIORITY)}
|
||||
_ACCURACY_MARK = "Preliminary model classification"
|
||||
_TAG = re.compile(r"\[(F\d+)\]")
|
||||
_YEAR = re.compile(r"\b(?:CY\s?)?(20\d\d|19\d\d)\b", re.I)
|
||||
#: summary + this many timeline lines survive the hardest trim
|
||||
TIMELINE_FLOOR = 8
|
||||
|
||||
_HEADER = (
|
||||
"Dossier — cited facts about the {name} family (authoritative for its "
|
||||
"history, valuation rounds and who commented). Each line ends with the "
|
||||
"label to copy:"
|
||||
)
|
||||
_STANCE_HEADER = (
|
||||
"Our own classification of commenters (never CMS's counts). Accuracy of "
|
||||
"every count below: {acc}"
|
||||
)
|
||||
|
||||
|
||||
def question_years(question: str) -> frozenset[int]:
|
||||
"""Years the question names ("2008", "CY2023")."""
|
||||
return frozenset(int(m) for m in _YEAR.findall(question))
|
||||
|
||||
|
||||
def match_facts(
|
||||
facts: Sequence[Fact],
|
||||
flagged: Sequence[str],
|
||||
codes: Sequence[str],
|
||||
years: Collection[int],
|
||||
limit: int = 0,
|
||||
) -> tuple[str, ...]:
|
||||
"""Ids of the facts that bear on the question, interleaved round-robin
|
||||
by kind with the newest year first. A fact matches when its codes overlap
|
||||
*codes* or its year is one of *years*; when nothing matches every
|
||||
unflagged fact is a candidate. Flagged, stance and payment facts never
|
||||
appear (stance counts are their own unit, payment rows are the Valuation
|
||||
block's)."""
|
||||
bad = set(flagged)
|
||||
pool = [
|
||||
f for f in facts if f.kind in _KIND_ORDER and not f.note and f.id not in bad
|
||||
]
|
||||
want = set(codes)
|
||||
hit = [f for f in pool if (want & set(f.codes)) or (f.year in years)]
|
||||
pool = hit or pool
|
||||
by_kind: dict[str, list[Fact]] = {k: [] for k in _KIND_ORDER}
|
||||
for f in pool:
|
||||
by_kind[f.kind].append(f)
|
||||
queues = [sorted(v, key=lambda f: -(f.year or 0)) for v in by_kind.values() if v]
|
||||
out: list[str] = []
|
||||
while any(queues):
|
||||
for q in queues:
|
||||
if q:
|
||||
out.append(q.pop(0).id)
|
||||
return tuple(out[:limit] if limit else out)
|
||||
|
||||
|
||||
def dossier_url(family: str, cfg: LlmConfig) -> str:
|
||||
@@ -68,29 +115,61 @@ class DossierEvidence:
|
||||
summary: str
|
||||
facts: tuple[Fact, ...]
|
||||
gaps: tuple[str, ...]
|
||||
#: matched facts kept in the prompt block (the timeline and the stance
|
||||
#: unit are sized by characters and the budget trim instead)
|
||||
max_prompt_rows: int = 30
|
||||
#: the dossier's own stance-accuracy sentence, appended to any stance
|
||||
#: count whose text does not carry one (a count never stands alone)
|
||||
#: the dossier's own stance-accuracy sentence: the header of the stance
|
||||
#: unit (a count never stands without it)
|
||||
accuracy: str = ""
|
||||
#: ids of facts shown flagged on the page (unreviewed model paraphrases);
|
||||
#: never given to the model
|
||||
flagged: tuple[str, ...] = ()
|
||||
timeline: tuple[TimelineEvent, ...] = ()
|
||||
#: ``match_facts`` result for the question, in prompt order
|
||||
matched: tuple[str, ...] = ()
|
||||
|
||||
def payload(self) -> dict[str, Any]:
|
||||
def _labelled(self, text: str) -> str:
|
||||
label = {f.id: f.cite.label for f in self.facts}
|
||||
|
||||
def tag(m: re.Match[str]) -> str:
|
||||
return f"[{label[m.group(1)]}]" if m.group(1) in label else m.group(0)
|
||||
|
||||
summary = _TAG.sub(tag, self.summary)
|
||||
out = _TAG.sub(tag, text)
|
||||
# a run of the same cite ("[L2][L2]") reads once
|
||||
summary = re.sub(r"(\[[^\[\]]+\])(?:\1)+", r"\1", summary)
|
||||
return re.sub(r"(\[[^\[\]]+\])(?:\1)+", r"\1", out)
|
||||
|
||||
def _timeline_lines(self) -> list[str]:
|
||||
return [
|
||||
f"CY{e.year}: {self._labelled(e.what_changed)}"
|
||||
for e in sorted(self.timeline, key=lambda e: -e.year)
|
||||
]
|
||||
|
||||
def _stance(self) -> list[Fact]:
|
||||
return [f for f in self.facts if f.kind == "stance_split"]
|
||||
|
||||
def _prompt_fact_ids(self, rows: int | None = None) -> list[str]:
|
||||
n = self.max_prompt_rows if rows is None else rows
|
||||
return list(self.matched[: max(n, 0)])
|
||||
|
||||
def payload(self) -> dict[str, Any]:
|
||||
"""Facts cited by the summary or the timeline plus those the prompt
|
||||
block carries (same shape as the brief, plus ``review``)."""
|
||||
by_id = {f.id: f for f in self.facts}
|
||||
wanted: set[str] = set()
|
||||
for text in [self.summary, *(e.what_changed for e in self.timeline)]:
|
||||
wanted.update(_TAG.findall(text))
|
||||
for e in self.timeline:
|
||||
wanted.update(e.stance_fact_ids)
|
||||
wanted.update(e.cms_summary_fact_ids)
|
||||
wanted.update(self._prompt_fact_ids())
|
||||
wanted.update(f.id for f in self._stance())
|
||||
flagged = set(self.flagged)
|
||||
return {
|
||||
"type": "dossier",
|
||||
"family": self.family,
|
||||
"name": self.name,
|
||||
"url": self.url,
|
||||
"summary": summary,
|
||||
"summary": self._labelled(self.summary),
|
||||
"facts": [
|
||||
{
|
||||
"label": f.cite.label,
|
||||
@@ -98,41 +177,67 @@ class DossierEvidence:
|
||||
"url": f.cite.url,
|
||||
"year": f.year,
|
||||
"kind": f.kind,
|
||||
"review": bool(f.note) or f.id in flagged,
|
||||
}
|
||||
for f in self.facts
|
||||
if f.id in wanted and f.id in by_id
|
||||
],
|
||||
"gaps": list(self.gaps),
|
||||
}
|
||||
|
||||
def _line(self, f: Fact) -> str:
|
||||
text = f.text
|
||||
if f.kind == "stance_split" and _ACCURACY_MARK not in text:
|
||||
text = f"{text} {self.accuracy or stance_accuracy_line(None)}"
|
||||
return f"{text} [{f.cite.label}]"
|
||||
def _stance_unit(self) -> list[str]:
|
||||
stance = self._stance()
|
||||
if not stance:
|
||||
return []
|
||||
acc = self.accuracy or stance_accuracy_line(None)
|
||||
lines = [_STANCE_HEADER.format(acc=acc)]
|
||||
for f in stance:
|
||||
text = f.text.split(f" {_ACCURACY_MARK}")[0]
|
||||
lines.append(f"{text} [{f.cite.label}]")
|
||||
return lines
|
||||
|
||||
def prompt_block(self, max_rows: int | None = None) -> str:
|
||||
"""*max_rows* (default ``max_prompt_rows``) caps the rows other than
|
||||
stance counts; every stance count stays, each with its accuracy
|
||||
sentence."""
|
||||
cap = self.max_prompt_rows if max_rows is None else max_rows
|
||||
flagged = set(self.flagged)
|
||||
usable = [
|
||||
(i, f)
|
||||
for i, f in enumerate(self.facts)
|
||||
if not f.note and f.id not in flagged
|
||||
def prompt_block(
|
||||
self,
|
||||
max_rows: int | None = None,
|
||||
max_chars: int | None = None,
|
||||
floor: bool = False,
|
||||
) -> str:
|
||||
"""Summary, timeline (newest first), the question's matched facts and
|
||||
one stance unit. *max_rows* caps the matched facts; *max_chars* sheds
|
||||
matched facts, then the stance unit (all or none), then the oldest
|
||||
timeline lines down to ``TIMELINE_FLOOR``; *floor* is the budget
|
||||
trim's last step: the summary and the newest ``TIMELINE_FLOOR``
|
||||
timeline lines only."""
|
||||
by_id = {f.id: f for f in self.facts}
|
||||
summary = f"Summary: {self._labelled(self.summary)}"
|
||||
tl = self._timeline_lines()
|
||||
matched = [
|
||||
f"{f.text} [{f.cite.label}]"
|
||||
for f in (by_id[i] for i in self._prompt_fact_ids(max_rows) if i in by_id)
|
||||
]
|
||||
usable.sort(key=lambda p: (_RANK.get(p[1].kind, len(_RANK)), p[0]))
|
||||
kept: list[Fact] = []
|
||||
others = 0
|
||||
for _i, f in usable:
|
||||
if f.kind == "stance_split":
|
||||
kept.append(f)
|
||||
elif others < cap:
|
||||
kept.append(f)
|
||||
others += 1
|
||||
lines = [_HEADER.format(name=self.name)]
|
||||
lines.extend(self._line(f) for f in kept)
|
||||
return "\n".join(lines)
|
||||
stance = self._stance_unit()
|
||||
|
||||
def build(n_tl: int, n_m: int, with_stance: bool) -> str:
|
||||
lines = [_HEADER.format(name=self.name), summary]
|
||||
lines.extend(tl[:n_tl])
|
||||
lines.extend(matched[:n_m])
|
||||
if with_stance:
|
||||
lines.extend(stance)
|
||||
return "\n".join(lines)
|
||||
|
||||
if floor:
|
||||
return build(TIMELINE_FLOOR, 0, False)
|
||||
n_tl, n_m, with_stance = len(tl), len(matched), bool(stance)
|
||||
while max_chars is not None and len(build(n_tl, n_m, with_stance)) > max_chars:
|
||||
if n_m:
|
||||
n_m -= 1
|
||||
elif with_stance:
|
||||
with_stance = False
|
||||
elif n_tl > TIMELINE_FLOOR:
|
||||
n_tl -= 1
|
||||
else:
|
||||
break
|
||||
return build(n_tl, n_m, with_stance)
|
||||
|
||||
|
||||
def dossier_evidence(question: str, cfg: LlmConfig) -> DossierEvidence | None:
|
||||
@@ -165,14 +270,23 @@ def dossier_evidence(question: str, cfg: LlmConfig) -> DossierEvidence | None:
|
||||
cur.close()
|
||||
if d is None:
|
||||
return None
|
||||
facts = tuple(d.facts.values())
|
||||
flagged = tuple(r["fact_id"] for r in d.review)
|
||||
return DossierEvidence(
|
||||
family=d.family,
|
||||
name=d.name,
|
||||
url=dossier_url(d.family, cfg),
|
||||
summary=d.summary,
|
||||
facts=tuple(d.facts.values()),
|
||||
facts=facts,
|
||||
gaps=tuple(d.gaps),
|
||||
max_prompt_rows=cfg.dossier_max_rows,
|
||||
accuracy=d.stance_accuracy,
|
||||
flagged=tuple(r["fact_id"] for r in d.review),
|
||||
flagged=flagged,
|
||||
timeline=tuple(d.timeline),
|
||||
matched=match_facts(
|
||||
facts,
|
||||
flagged,
|
||||
det.explicit or det.codes,
|
||||
question_years(question),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -122,6 +122,9 @@ class ValuationEvidence:
|
||||
#: ``codes``) — carried through so ``rag.stream_answer`` doesn't
|
||||
#: need a third ``detect_codes`` call just to find them.
|
||||
wide: tuple[str, ...] = ()
|
||||
#: the default-years rows the prompt block selects from when the payload
|
||||
#: carries a dossier family's whole history (``None``: ``rows`` itself)
|
||||
prompt_rows: tuple[ValuationRow, ...] | None = None
|
||||
|
||||
def prompt_block(self, max_rows: int | None = None) -> str:
|
||||
"""*max_rows* overrides ``self.max_prompt_rows`` for one call —
|
||||
@@ -129,7 +132,8 @@ class ValuationEvidence:
|
||||
smaller block on demand without needing a new ``ValuationEvidence``."""
|
||||
cap = self.max_prompt_rows if max_rows is None else max_rows
|
||||
lines = [_HEADER]
|
||||
for r in _cap_valuation_rows(self.rows, self.explicit, cap):
|
||||
source = self.rows if self.prompt_rows is None else self.prompt_rows
|
||||
for r in _cap_valuation_rows(source, self.explicit, cap):
|
||||
# The status note says why an unpaid code shows no dollars, and
|
||||
# the CF note says which of the year's CFs the money came from.
|
||||
status = f"{r.status or '?'}" + (
|
||||
@@ -265,14 +269,19 @@ def valuation_evidence(question: str, cfg: LlmConfig) -> ValuationEvidence | Non
|
||||
# A family with a dossier gets its whole payment history (every final
|
||||
# year, every proposed round) in the payload; the prompt block still
|
||||
# keeps only ``valuation_rows_max`` rows.
|
||||
kwargs: dict[str, Any] = {"years": years}
|
||||
full = False
|
||||
try:
|
||||
if set(det.families) & set(dossier_families(cur)):
|
||||
kwargs = {"years": 0, "all_proposed": True}
|
||||
full = bool(set(det.families) & set(dossier_families(cur)))
|
||||
except Exception as e: # noqa: BLE001 — the valuation must still answer
|
||||
log.warning("dossier families unavailable for valuation: %s", e)
|
||||
try:
|
||||
rows, unpriced = valuation(cur, list(codes), **kwargs)
|
||||
rows, unpriced = valuation(cur, list(codes), years=years)
|
||||
prompt_rows = None
|
||||
if full:
|
||||
# the prompt picks from the default window, so the whole history
|
||||
# never pushes the newest final rows out of its row cap
|
||||
prompt_rows = tuple(rows)
|
||||
rows, unpriced = valuation(cur, list(codes), years=0, all_proposed=True)
|
||||
except Exception as e: # noqa: BLE001
|
||||
log.warning("valuation evidence skipped (query): %s", e)
|
||||
return None
|
||||
@@ -286,6 +295,7 @@ def valuation_evidence(question: str, cfg: LlmConfig) -> ValuationEvidence | Non
|
||||
explicit=det.explicit,
|
||||
max_prompt_rows=cfg.valuation_rows_max,
|
||||
wide=det.wide,
|
||||
prompt_rows=prompt_rows,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -235,7 +235,7 @@ _CITE_REMINDER = (
|
||||
"sentence that states a date, a code being created, adopted, replaced or "
|
||||
"deleted, an RVU, a conversion factor, a payment amount, or what a rule "
|
||||
"or a commenter said MUST end with the bracketed label of the Lineage "
|
||||
"line, Valuation row or excerpt it comes from, copied exactly — for "
|
||||
"line, Dossier line, Valuation row or excerpt it comes from, copied exactly — for "
|
||||
'example: "G2058 was replaced by CPT 99439 for CY2021 [CY2021 PFS final '
|
||||
'85 FR 84547 ¶686]." Do not write such a sentence without its label. '
|
||||
"Take the year of each fact from its Lineage line, never from a "
|
||||
@@ -454,7 +454,8 @@ _BUDGET_RETRIEVED_KEEP = 6
|
||||
_BUDGET_CITED_KEEP = 8
|
||||
_BUDGET_MANUAL_KEEP = 1
|
||||
_BUDGET_VALUATION_ROWS_KEEP = 12
|
||||
_BUDGET_DOSSIER_ROWS_FLOOR = 8
|
||||
#: share of the prompt budget the dossier block may take
|
||||
_DOSSIER_BUDGET_SHARE = 0.35
|
||||
|
||||
# The model counts its context in tokens; the excerpts are trimmed in
|
||||
# characters before any model sees them. ``prompt_budget_chars`` converts
|
||||
@@ -540,7 +541,8 @@ def _user_message(
|
||||
valuation_rows_cap: int | None,
|
||||
lineage_rows_cap: int | None,
|
||||
dossier: DossierEvidence | None = None,
|
||||
dossier_rows_cap: int | None = None,
|
||||
dossier_chars: int | None = None,
|
||||
dossier_floor: bool = False,
|
||||
) -> str:
|
||||
sources = retrieved + cited + lineage_src + manual + rest
|
||||
if sources:
|
||||
@@ -558,7 +560,7 @@ def _user_message(
|
||||
context = "(no relevant excerpts found)"
|
||||
parts = [f"Excerpts:\n\n{context}"]
|
||||
if dossier is not None:
|
||||
parts.append(dossier.prompt_block(dossier_rows_cap))
|
||||
parts.append(dossier.prompt_block(None, dossier_chars, dossier_floor))
|
||||
if lineage is not None:
|
||||
parts.append(lineage.prompt_block(lineage_rows_cap))
|
||||
if evidence is not None:
|
||||
@@ -601,8 +603,8 @@ def build_messages(
|
||||
budget_chars: int | None = None,
|
||||
) -> list[dict]:
|
||||
"""Grounded chat messages: system rules + question with excerpts and,
|
||||
when available, a Lineage block and a Valuation block — in that
|
||||
order — between the excerpts and the question.
|
||||
when available, a Dossier block, a Lineage block and a Valuation
|
||||
block — in that order — between the excerpts and the question.
|
||||
|
||||
*sources* is assumed laid out contiguously in the order
|
||||
``stream_answer`` appends them — retrieved, then cited
|
||||
@@ -635,7 +637,13 @@ def build_messages(
|
||||
|
||||
valuation_cap = evidence.max_prompt_rows if evidence is not None else None
|
||||
lineage_cap = lineage.max_prompt_rows if lineage is not None else None
|
||||
dossier_cap = dossier.max_prompt_rows if dossier is not None else None
|
||||
# the dossier block never takes more than a third of the budget
|
||||
dossier_chars = (
|
||||
int(budget_chars * _DOSSIER_BUDGET_SHARE)
|
||||
if dossier is not None and budget_chars is not None
|
||||
else None
|
||||
)
|
||||
dossier_floor = False
|
||||
|
||||
def content() -> str:
|
||||
return _user_message(
|
||||
@@ -650,7 +658,8 @@ def build_messages(
|
||||
valuation_cap,
|
||||
lineage_cap,
|
||||
dossier,
|
||||
dossier_cap,
|
||||
dossier_chars,
|
||||
dossier_floor,
|
||||
)
|
||||
|
||||
if budget_chars is not None and len(content()) > budget_chars:
|
||||
@@ -711,6 +720,16 @@ def build_messages(
|
||||
before - budget_chars,
|
||||
budget_chars,
|
||||
)
|
||||
# The dossier block, before the lineage rows: down to the summary
|
||||
# and the newest TIMELINE_FLOOR timeline lines.
|
||||
if len(content()) > budget_chars and dossier is not None and not dossier_floor:
|
||||
before = len(content())
|
||||
dossier_floor = True
|
||||
log.info(
|
||||
"prompt budget: cut the dossier block to its floor (%d chars over %d)",
|
||||
before - budget_chars,
|
||||
budget_chars,
|
||||
)
|
||||
if len(content()) > budget_chars and lineage is not None:
|
||||
target = lineage.max_prompt_rows
|
||||
half = max(target // 2, 1)
|
||||
@@ -723,22 +742,6 @@ def build_messages(
|
||||
before - budget_chars,
|
||||
budget_chars,
|
||||
)
|
||||
# Last of all, the dossier's rows (stance counts always stay),
|
||||
# down to a floor of 8.
|
||||
if (
|
||||
len(content()) > budget_chars
|
||||
and dossier is not None
|
||||
and (dossier_cap is None or dossier_cap > _BUDGET_DOSSIER_ROWS_FLOOR)
|
||||
):
|
||||
before = len(content())
|
||||
dossier_cap = _BUDGET_DOSSIER_ROWS_FLOOR
|
||||
log.info(
|
||||
"prompt budget: capped dossier rows to %d (%d chars over %d)",
|
||||
dossier_cap,
|
||||
before - budget_chars,
|
||||
budget_chars,
|
||||
)
|
||||
|
||||
return [
|
||||
{"role": "system", "content": _SYSTEM},
|
||||
{"role": "user", "content": content()},
|
||||
@@ -794,6 +797,12 @@ def stream_answer(
|
||||
) -> Iterator[dict]:
|
||||
"""Retrieve, then stream a grounded answer from the largest live host.
|
||||
|
||||
When the detected family has a stored dossier, a ``dossier`` event
|
||||
(``llm.dossier``) comes first and a Dossier block (summary, timeline,
|
||||
question-matched facts, one stance unit) leads the prompt's evidence
|
||||
blocks, capped at a third of the prompt budget; ``sources`` then starts
|
||||
with a link entry for the dossier page.
|
||||
|
||||
``mode`` ("auto"/"timeline"/"recent") is forwarded to ``retrieve``;
|
||||
"auto" (the default) switches to era-balanced retrieval on its own
|
||||
for a history-shaped question (``is_history_question``).
|
||||
|
||||
@@ -486,7 +486,7 @@
|
||||
if ((ev.gaps || []).length) {
|
||||
const g = document.createElement('div');
|
||||
g.className = 'provenance';
|
||||
g.textContent = 'Not yet captured: ' + ev.gaps.join(' ');
|
||||
g.textContent = 'Gaps and notes: ' + ev.gaps.join(' ');
|
||||
box.appendChild(g);
|
||||
}
|
||||
if (ev.url) {
|
||||
|
||||
@@ -60,6 +60,7 @@ class TestIndex:
|
||||
assert "function renderDossier" in html
|
||||
assert "ev.type === 'dossier'" in html
|
||||
assert "Open the full dossier" in html
|
||||
assert "Gaps and notes" in html and "Not yet captured" not in html
|
||||
assert "tr.proposed" in html
|
||||
assert "rows.length > 40" in html
|
||||
assert "<details" in html or "createElement('details')" in html
|
||||
|
||||
@@ -10,10 +10,17 @@ import pytest
|
||||
|
||||
from llm import evidence
|
||||
from llm.config import LlmConfig
|
||||
from llm.dossier import DossierEvidence, dossier_evidence, dossier_url
|
||||
from llm.dossier import (
|
||||
TIMELINE_FLOOR,
|
||||
DossierEvidence,
|
||||
dossier_evidence,
|
||||
dossier_url,
|
||||
match_facts,
|
||||
question_years,
|
||||
)
|
||||
from pfs.cite import Cite
|
||||
from pfs.codetables import ensure_tables
|
||||
from pfs.dossier import Dossier, Fact, write_dossier
|
||||
from pfs.dossier import Dossier, Fact, TimelineEvent, write_dossier
|
||||
|
||||
CFG = LlmConfig(
|
||||
ollama_hosts=("http://h1:11434",),
|
||||
@@ -36,12 +43,12 @@ ACC = (
|
||||
)
|
||||
|
||||
|
||||
def _fact(i, kind, text, year=2023, label=None, note=""):
|
||||
def _fact(i, kind, text, year=2023, label=None, note="", codes=("99341",)):
|
||||
return Fact(
|
||||
id=f"F{i}",
|
||||
kind=kind,
|
||||
year=year,
|
||||
codes=("99341",),
|
||||
codes=codes,
|
||||
text=text,
|
||||
values={},
|
||||
cite=Cite("fr", label or f"L{i}", f"https://x/{i}"),
|
||||
@@ -50,10 +57,17 @@ def _fact(i, kind, text, year=2023, label=None, note=""):
|
||||
|
||||
|
||||
def _dossier(extra=(), accuracy=ACC):
|
||||
old = ("99324", "99325")
|
||||
facts = [
|
||||
_fact(1, "payment", "payment row"),
|
||||
_fact(2, "round", "CMS finalized work RVU 1.0 for 99341 for CY2023."),
|
||||
_fact(3, "event", "99324 was deleted for CY2023."),
|
||||
_fact(
|
||||
3,
|
||||
"event",
|
||||
"99324 was deleted for CY2023.",
|
||||
label="DEL23",
|
||||
codes=old,
|
||||
),
|
||||
_fact(
|
||||
4,
|
||||
"stance_split",
|
||||
@@ -69,8 +83,39 @@ def _dossier(extra=(), accuracy=ACC):
|
||||
_fact(7, "round", "flagged paraphrase", note="model paraphrase, unreviewed"),
|
||||
_fact(8, "stance_split", "Of 3 CY2018 commenters we classified, 3 supported."),
|
||||
_fact(9, "finding", "a finding"),
|
||||
# live-shaped: far more CY2008 rounds than any cap
|
||||
*(
|
||||
_fact(
|
||||
100 + i,
|
||||
"round",
|
||||
f"For CY2008, CMS finalized work RVU {i} for 99326.",
|
||||
year=2008,
|
||||
codes=("99326",),
|
||||
)
|
||||
for i in range(40)
|
||||
),
|
||||
*extra,
|
||||
]
|
||||
tl = [
|
||||
TimelineEvent(
|
||||
2008,
|
||||
"valuation",
|
||||
("99326",),
|
||||
"For CY2008, CMS finalized RVUs [F100].",
|
||||
("F100",),
|
||||
(),
|
||||
(),
|
||||
),
|
||||
TimelineEvent(
|
||||
2023,
|
||||
"deletion",
|
||||
old,
|
||||
"99324 and 99325 were deleted for CY2023 [F3].",
|
||||
("F3",),
|
||||
(),
|
||||
(),
|
||||
),
|
||||
]
|
||||
return Dossier(
|
||||
family="HOME-VISITS",
|
||||
name="Home or Residence Services",
|
||||
@@ -78,7 +123,7 @@ def _dossier(extra=(), accuracy=ACC):
|
||||
status="active",
|
||||
years_active="2008-",
|
||||
summary="CMS finalized it [F2][F2] and deleted codes [F3].",
|
||||
timeline=[],
|
||||
timeline=tl,
|
||||
rounds=[],
|
||||
payment=[],
|
||||
utilization=[],
|
||||
@@ -115,7 +160,8 @@ def test_dossier_evidence_for_home_visit_question(replica):
|
||||
assert ev.name == "Home or Residence Services"
|
||||
assert ev.url == "https://lex.fhirworx.io/?family=HOME-VISITS#dossier"
|
||||
assert ev.max_prompt_rows == 30
|
||||
assert len(ev.facts) == 9
|
||||
assert len(ev.facts) == 49
|
||||
assert len(ev.timeline) == 2
|
||||
assert ev.gaps == ("RVU files before 2015 not yet loaded.",)
|
||||
|
||||
|
||||
@@ -155,7 +201,9 @@ def test_first_family_curated_before_derived(replica, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
mod,
|
||||
"detect_codes",
|
||||
lambda q: SimpleNamespace(families=("AAA-DERIVED", "HOME-VISITS", "ZZZ")),
|
||||
lambda q: SimpleNamespace(
|
||||
families=("AAA-DERIVED", "HOME-VISITS", "ZZZ"), explicit=(), codes=()
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(mod, "CURATED_FAMILIES", {"HOME-VISITS": 1})
|
||||
seen = []
|
||||
@@ -176,11 +224,26 @@ def test_dossier_payload_shape_and_summary_labels(replica):
|
||||
assert p["family"] == "HOME-VISITS"
|
||||
assert p["url"] == "https://lex.fhirworx.io/?family=HOME-VISITS#dossier"
|
||||
# [F#] became the facts' labels; the doubled cite collapsed
|
||||
assert p["summary"] == "CMS finalized it [L2] and deleted codes [L3]."
|
||||
assert {"label", "text", "url", "year", "kind"} == set(p["facts"][0])
|
||||
assert p["summary"] == "CMS finalized it [L2] and deleted codes [DEL23]."
|
||||
assert {"label", "text", "url", "year", "kind", "review"} == set(p["facts"][0])
|
||||
assert p["gaps"] == ["RVU files before 2015 not yet loaded."]
|
||||
|
||||
|
||||
def test_payload_carries_only_cited_and_selected_facts(replica):
|
||||
ev = dossier_evidence("home visits?", replica)
|
||||
p = ev.payload()
|
||||
labels = {f["label"] for f in p["facts"]}
|
||||
assert {"L2", "DEL23", "L100"} <= labels # summary + timeline cites
|
||||
assert "L1" not in labels # payment rows stay on the page
|
||||
assert len(p["facts"]) < len(ev.facts)
|
||||
assert all(f["review"] is False for f in p["facts"])
|
||||
# a flagged fact cited by the timeline travels flagged
|
||||
ev2 = replace(ev, flagged=("F3",))
|
||||
assert [f["review"] for f in ev2.payload()["facts"] if f["label"] == "DEL23"] == [
|
||||
True
|
||||
]
|
||||
|
||||
|
||||
def test_payload_unknown_fact_tag_kept(replica):
|
||||
ev = dossier_evidence("home visits?", replica)
|
||||
ev = replace(ev, summary="see [F999].")
|
||||
@@ -195,8 +258,12 @@ def test_payload_names_come_from_own_records(replica):
|
||||
assert seat["label"] == "CMS-2022-0113-0001"
|
||||
|
||||
|
||||
def test_prompt_block_lines_end_with_labels(replica):
|
||||
ev = dossier_evidence("home visits?", replica)
|
||||
def _ev(replica_cfg, question):
|
||||
return dossier_evidence(question, replica_cfg)
|
||||
|
||||
|
||||
def test_prompt_block_order_and_labels(replica):
|
||||
ev = _ev(replica, "When were domiciliary visit codes 99324 and 99325 deleted?")
|
||||
block = ev.prompt_block()
|
||||
lines = block.splitlines()
|
||||
assert lines[0] == (
|
||||
@@ -204,53 +271,113 @@ def test_prompt_block_lines_end_with_labels(replica):
|
||||
"(authoritative for its history, valuation rounds and who commented). "
|
||||
"Each line ends with the label to copy:"
|
||||
)
|
||||
body = lines[1:]
|
||||
assert body and all(ln.endswith("]") for ln in body)
|
||||
assert "CMS finalized work RVU 1.0 for 99341 for CY2023. [L2]" in body
|
||||
# priority order: round, event, cms_summary, stance_split, ... seat, finding
|
||||
kinds_at = [
|
||||
block.index(t) for t in ("[L2]", "[L3]", "[L5]", "[L4]", "[CMS-2022-0113-0001]")
|
||||
assert lines[1] == ("Summary: CMS finalized it [L2] and deleted codes [DEL23].")
|
||||
# timeline, newest first, in the dossier's own prose with labels
|
||||
assert lines[2] == "CY2023: 99324 and 99325 were deleted for CY2023 [DEL23]."
|
||||
assert lines[3] == "CY2008: For CY2008, CMS finalized RVUs [L100]."
|
||||
# then the question's matched facts, then the stance unit
|
||||
assert block.index("99324 was deleted for CY2023. [DEL23]") > block.index("CY2008:")
|
||||
assert block.index("Our own classification") > block.index("[DEL23]")
|
||||
assert "flagged paraphrase" not in block and "a finding" not in block
|
||||
assert "payment row" not in block
|
||||
# every line but the stance header ends with a label
|
||||
for ln in lines:
|
||||
assert ln.rstrip(".").endswith("]") or ln.startswith(
|
||||
("Dossier —", "Our own classification")
|
||||
)
|
||||
|
||||
|
||||
def test_deletion_question_surfaces_cy2023_over_cy2008_rounds(replica):
|
||||
"""The live bug: 30 CY2008 round lines crowded out every other fact."""
|
||||
block = _ev(replica, "When were 99324-99325 deleted?").prompt_block()
|
||||
assert "99324 was deleted for CY2023. [DEL23]" in block
|
||||
assert block.count("For CY2008, CMS finalized work RVU") < 10
|
||||
|
||||
|
||||
def test_matching_is_question_aware(replica):
|
||||
ev = _ev(replica, "How did CMS value 99326 in 2008?")
|
||||
first = [f for f in ev.facts if f.id == ev.matched[0]][0]
|
||||
assert first.year == 2008 and "99326" in first.codes
|
||||
|
||||
|
||||
def test_match_facts_round_robin_newest_first_and_limit():
|
||||
facts = [
|
||||
_fact(1, "round", "r-old", year=2008),
|
||||
_fact(2, "round", "r-new", year=2023),
|
||||
_fact(3, "event", "e", year=2015),
|
||||
_fact(4, "round", "r-mid", year=2015),
|
||||
_fact(5, "stance_split", "s"),
|
||||
_fact(6, "payment", "p"),
|
||||
_fact(7, "round", "flag", note="x"),
|
||||
_fact(8, "round", "elsewhere", codes=("A1",), year=2000),
|
||||
]
|
||||
assert kinds_at == sorted(kinds_at)
|
||||
# flagged / reviewed facts are never given to the model; payment rows
|
||||
# come through the Valuation block
|
||||
assert "flagged paraphrase" not in block
|
||||
assert "a finding" not in block
|
||||
assert "payment row" in block # lowest priority, still within the cap
|
||||
assert match_facts(facts, (), ("99341",), ()) == ("F2", "F3", "F4", "F1")
|
||||
# a named year also matches; the limit applies after interleaving
|
||||
assert match_facts(facts, (), ("zzz",), {2000}, limit=1) == ("F8",)
|
||||
# nothing matches -> every candidate
|
||||
assert match_facts(facts, ("F2",), ("zzz",), ()) == ("F4", "F3", "F1", "F8")
|
||||
|
||||
|
||||
def test_prompt_block_cap_keeps_stance_with_accuracy(replica):
|
||||
ev = dossier_evidence("home visits?", replica)
|
||||
block = ev.prompt_block(max_rows=1)
|
||||
assert "[L2]" in block and "[L3]" not in block # capped
|
||||
# every stance count stays, each with its accuracy sentence
|
||||
assert block.count("Preliminary model classification") == 2
|
||||
assert "Of 3 CY2018 commenters we classified, 3 supported." in block
|
||||
def test_question_years():
|
||||
assert question_years("CY2023 vs 2008, and cy 2015?") == {2023, 2008, 2015}
|
||||
|
||||
|
||||
def test_stance_unit_has_one_header_and_no_repeated_accuracy(replica):
|
||||
block = _ev(replica, "home visits?").prompt_block()
|
||||
assert block.count("Preliminary model classification") == 1
|
||||
head = [ln for ln in block.splitlines() if ln.startswith("Our own")][0]
|
||||
assert ACC in head
|
||||
for ln in block.splitlines():
|
||||
if "commenters we classified" in ln:
|
||||
assert "Preliminary model classification" in ln
|
||||
assert "Preliminary" not in ln and ln.endswith("]")
|
||||
assert block.index(head) < block.index("Of 3 CY2023 commenters")
|
||||
|
||||
|
||||
def test_stance_count_without_accuracy_gets_dossier_accuracy_line(replica):
|
||||
ev = dossier_evidence("home visits?", replica)
|
||||
block = ev.prompt_block()
|
||||
line = [ln for ln in block.splitlines() if "CY2018 commenters" in ln][0]
|
||||
assert ACC in line and line.endswith("[L8]")
|
||||
|
||||
|
||||
def test_stance_count_without_any_accuracy_says_unmeasured():
|
||||
def test_stance_header_says_unmeasured_without_accuracy():
|
||||
d = _dossier(accuracy="")
|
||||
ev = DossierEvidence(
|
||||
d.family,
|
||||
d.name,
|
||||
"u",
|
||||
d.summary,
|
||||
tuple(d.facts.values()),
|
||||
(),
|
||||
accuracy="",
|
||||
d.family, d.name, "u", d.summary, tuple(d.facts.values()), (), accuracy=""
|
||||
)
|
||||
line = [ln for ln in ev.prompt_block().splitlines() if "CY2018" in ln][0]
|
||||
assert "accuracy is not yet measured" in line
|
||||
assert "accuracy is not yet measured" in ev.prompt_block()
|
||||
|
||||
|
||||
def test_no_stance_facts_no_unit():
|
||||
ev = DossierEvidence("F", "N", "u", "s", (), ())
|
||||
assert ev.prompt_block().splitlines()[1] == "Summary: s"
|
||||
assert "Our own" not in ev.prompt_block()
|
||||
|
||||
|
||||
def test_max_rows_caps_matched_facts(replica):
|
||||
ev = _ev(replica, "home visits?")
|
||||
n = len(ev.prompt_block(max_rows=3).splitlines())
|
||||
assert n == len(ev.prompt_block(max_rows=0).splitlines()) + 3
|
||||
|
||||
|
||||
def test_max_chars_sheds_matched_then_stance_then_old_timeline(replica):
|
||||
ev = _ev(replica, "home visits?")
|
||||
full = ev.prompt_block()
|
||||
no_matched = ev.prompt_block(max_rows=0)
|
||||
assert "Our own" in no_matched
|
||||
# a limit that only the matched facts must give way to
|
||||
cut = ev.prompt_block(max_chars=len(no_matched) + 5)
|
||||
assert len(cut) <= len(no_matched) + 5 and "Our own" in cut
|
||||
# smaller: the whole stance unit goes (never counts without the header)
|
||||
tiny = ev.prompt_block(max_chars=1)
|
||||
assert "Our own" not in tiny and "commenters we classified" not in tiny
|
||||
assert tiny.count("\nCY") == 2 and len(tiny) < len(full)
|
||||
|
||||
|
||||
def test_max_chars_sheds_timeline_down_to_the_floor():
|
||||
tl = tuple(
|
||||
TimelineEvent(2000 + i, "k", (), f"event {i} [F1]", ("F1",), (), ())
|
||||
for i in range(12)
|
||||
)
|
||||
ev = DossierEvidence("F", "N", "u", "s", (_fact(1, "round", "r"),), (), timeline=tl)
|
||||
block = ev.prompt_block(max_chars=1)
|
||||
assert block.count("\nCY") == TIMELINE_FLOOR
|
||||
assert "CY2011:" in block and "CY2003:" not in block # oldest went first
|
||||
assert ev.prompt_block(floor=True) == block
|
||||
assert ev.prompt_block().count("\nCY") == 12
|
||||
|
||||
|
||||
def test_dossier_url_uses_env_domain(monkeypatch):
|
||||
|
||||
@@ -267,13 +267,17 @@ class TestValuationForDossierFamilies:
|
||||
_row(code="99341", year=2023 + i, proposed=True, label=f"[P{i}]")
|
||||
for i in range(2)
|
||||
]
|
||||
mock_val.return_value = (rows, [])
|
||||
# the default window: only the newest final rows
|
||||
recent = [_row(code="99341", year=2025 + i, label=f"[R{i}]") for i in range(2)]
|
||||
mock_val.side_effect = [(recent, []), (rows, [])]
|
||||
cfg = replace(CFG, valuation_rows_max=10)
|
||||
ev = valuation_evidence("How were home visits (99341) valued?", cfg)
|
||||
assert mock_val.call_args.kwargs == {"years": 0, "all_proposed": True}
|
||||
assert len(ev.payload()["rows"]) == 15
|
||||
block_rows = ev.prompt_block().count("99341 ")
|
||||
assert block_rows == 10 <= cfg.valuation_rows_max
|
||||
block = ev.prompt_block()
|
||||
# the prompt selects from the default-years rows, never the history
|
||||
assert block.count("99341 ") == 2
|
||||
assert "[R0]" in block and "[R1]" in block and "[Y0]" not in block
|
||||
|
||||
@patch("llm.evidence.dossier_families", return_value=("HOME-VISITS",))
|
||||
@patch("llm.evidence.valuation", return_value=([], []))
|
||||
|
||||
@@ -1380,6 +1380,7 @@ def _dossier_ev(n_facts=3):
|
||||
"Summary [DL0].",
|
||||
facts,
|
||||
("a gap",),
|
||||
matched=tuple(f.id for f in facts),
|
||||
)
|
||||
|
||||
|
||||
@@ -1467,8 +1468,26 @@ class TestDossierInChat:
|
||||
@patch("llm.rag.valuation_evidence", return_value=None)
|
||||
@patch("llm.rag.httpx.Client")
|
||||
@patch("llm.rag.retrieve")
|
||||
def test_control_question_byte_identical(self, mock_retrieve, MockClient, _ev):
|
||||
cfg = replace(CFG, duckdb_replica="/nonexistent/aco.ro.duckdb")
|
||||
def test_control_question_byte_identical(
|
||||
self, mock_retrieve, MockClient, _ev, tmp_path
|
||||
):
|
||||
"""A temp replica really holds the HOME-VISITS dossier; a question
|
||||
with no dossier family must send the model and the client the same
|
||||
bytes as a run with the dossier lookup switched off."""
|
||||
import duckdb
|
||||
|
||||
from llm import evidence
|
||||
from pfs.codetables import ensure_tables
|
||||
from pfs.dossier import write_dossier
|
||||
from tests.llm.test_dossier import _dossier
|
||||
|
||||
path = tmp_path / "aco.ro.duckdb"
|
||||
con = duckdb.connect(str(path))
|
||||
ensure_tables(con)
|
||||
write_dossier(con, _dossier())
|
||||
con.close()
|
||||
evidence._REPLICA = None
|
||||
cfg = replace(CFG, duckdb_replica=str(path))
|
||||
mock_retrieve.return_value = []
|
||||
client = MockClient.return_value.__enter__.return_value
|
||||
resp = client.stream.return_value.__enter__.return_value
|
||||
@@ -1477,18 +1496,28 @@ class TestDossierInChat:
|
||||
resp.iter_lines.return_value = iter(
|
||||
['{"message":{"content":"x"},"done":true}']
|
||||
)
|
||||
return json.dumps(
|
||||
events = json.dumps(
|
||||
list(
|
||||
stream_answer(
|
||||
"why did CMS finalize this?", cfg=cfg, pool=self._pool()
|
||||
)
|
||||
)
|
||||
)
|
||||
sent = json.dumps(client.stream.call_args.kwargs["json"]["messages"])
|
||||
return events, sent
|
||||
|
||||
real = run()
|
||||
with patch("llm.rag.dossier_evidence", return_value=None):
|
||||
off = run()
|
||||
try:
|
||||
from llm.dossier import dossier_evidence
|
||||
|
||||
# the family really is there: a home-visit question finds it
|
||||
assert dossier_evidence("how were home visits valued?", cfg) is not None
|
||||
real = run()
|
||||
with patch("llm.rag.dossier_evidence", return_value=None):
|
||||
off = run()
|
||||
finally:
|
||||
evidence._REPLICA = None
|
||||
assert real == off
|
||||
assert "Dossier" not in real[1].split("Question:")[0].split("Excerpts:")[-1]
|
||||
|
||||
|
||||
class TestDossierPromptBlock:
|
||||
@@ -1510,18 +1539,78 @@ class TestDossierPromptBlock:
|
||||
def test_no_dossier_prompt_unchanged(self):
|
||||
assert build_messages("q", []) == build_messages("q", [], dossier=None)
|
||||
|
||||
def test_dossier_rows_trimmed_to_floor_8_as_a_last_resort(self):
|
||||
d = replace(_dossier_ev(40), max_prompt_rows=30)
|
||||
def test_block_takes_at_most_35_percent_of_the_budget(self):
|
||||
d = _dossier_ev(40)
|
||||
d = replace(d, matched=tuple(f.id for f in d.facts))
|
||||
big = build_messages("q", [], dossier=d)[1]["content"]
|
||||
assert big.count("[DL") == 30
|
||||
msgs = build_messages("q", [], dossier=d, budget_chars=len(big) - 1)
|
||||
assert msgs[1]["content"].count("[DL") == 8
|
||||
assert big.count("[DL") > 30
|
||||
budget = len(big) + 1000 # roomy: only the 35% share bites
|
||||
msgs = build_messages("q", [], dossier=d, budget_chars=budget)
|
||||
block = msgs[1]["content"].split("\n\n")[1]
|
||||
assert len(block) <= int(budget * 0.35)
|
||||
|
||||
def test_dossier_cap_untouched_when_budget_met_earlier(self):
|
||||
d = _dossier_ev(10)
|
||||
big = build_messages("q", [], dossier=d)[1]["content"]
|
||||
msgs = build_messages("q", [], dossier=d, budget_chars=len(big) + 1)
|
||||
assert msgs[1]["content"].count("[DL") == 10
|
||||
def test_dossier_floor_is_summary_plus_eight_timeline_lines(self):
|
||||
from pfs.dossier import TimelineEvent
|
||||
|
||||
d = replace(
|
||||
_dossier_ev(10),
|
||||
matched=tuple(f"F{i}" for i in range(10)),
|
||||
timeline=tuple(
|
||||
TimelineEvent(2000 + i, "k", (), f"ev {i} [F0]", ("F0",), (), ())
|
||||
for i in range(12)
|
||||
),
|
||||
)
|
||||
msgs = build_messages("q", [], dossier=d, budget_chars=1)
|
||||
block = msgs[1]["content"]
|
||||
assert block.count("\nCY") == 8 and "fact 0. [DL0]" not in block
|
||||
assert "Summary:" in block
|
||||
|
||||
def test_dossier_shed_before_lineage_rows(self):
|
||||
from pfs.dossier import TimelineEvent
|
||||
|
||||
events = tuple(
|
||||
LineageEvent(
|
||||
f"C{i}",
|
||||
2020,
|
||||
"revalued",
|
||||
(),
|
||||
(),
|
||||
f"[L{i}]",
|
||||
"K",
|
||||
i,
|
||||
0,
|
||||
"",
|
||||
"fr",
|
||||
True,
|
||||
"",
|
||||
)
|
||||
for i in range(20)
|
||||
)
|
||||
le = LineageEvidence(
|
||||
codes=("C",),
|
||||
families=(),
|
||||
events=events,
|
||||
element_diffs=(),
|
||||
guidance=(),
|
||||
max_prompt_rows=20,
|
||||
)
|
||||
d = replace(
|
||||
_dossier_ev(10),
|
||||
matched=tuple(f"F{i}" for i in range(10)),
|
||||
timeline=tuple(
|
||||
TimelineEvent(2000 + i, "k", (), f"ev {i} [F0]", ("F0",), (), ())
|
||||
for i in range(12)
|
||||
),
|
||||
)
|
||||
full = build_messages("q", [], lineage=le, dossier=d)[1]["content"]
|
||||
# a budget that the dossier floor alone satisfies keeps every lineage row
|
||||
floor = build_messages("q", [], lineage=le, dossier=d, budget_chars=1)
|
||||
assert floor[1]["content"].count("[L") < full.count("[L") # last resort bites
|
||||
mid = build_messages(
|
||||
"q", [], lineage=le, dossier=d, budget_chars=len(full) - 200
|
||||
)[1]["content"]
|
||||
assert mid.count("[L") == full.count("[L")
|
||||
assert mid.count("\nCY") == 8
|
||||
|
||||
def test_system_prompt_names_the_dossier_rules(self):
|
||||
s = build_messages("q", [])[0]["content"]
|
||||
|
||||
Reference in New Issue
Block a user