fix(llm): stance_split labels carry the accuracy caveat (R102)
Some checks failed
CI / lint (push) Successful in 1m19s
CI / notebooks-smoke (push) Successful in 2m5s
Deploy / changes (push) Successful in 27s
CI / test (push) Failing after 5m1s
Infra CI / zotero (push) Successful in 28s
Infra CI / notebooks (push) Successful in 2m1s
Infra CI / docs (push) Successful in 26s
Infra CI / api (push) Successful in 1m59s
Infra CI / bcda (push) Successful in 2m43s
Infra CI / mc (push) Successful in 22s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Infra CI / llm (push) Successful in 1m35s
Deploy / api (push) Successful in 2m4s
Deploy / bcda (push) Successful in 4m16s
Deploy / llm (push) Successful in 2m35s
Deploy / mc (push) Has been skipped
Deploy / report (push) Successful in 19s

This commit is contained in:
kert
2026-11-13 01:59:45 -05:00
parent a790b74179
commit c292e2d196
2 changed files with 57 additions and 6 deletions

View File

@@ -58,6 +58,7 @@ TIMELINE_FLOOR = 4
_STANCE_INTENT = re.compile(
r"comment|support|oppos|stakeholder|societ|association|lobby", re.I
)
_ACC = re.compile(r"finds (\d+)% of hand-coded.*?right for (\d+)%")
_RANGE = re.compile(r"\b(\d{5})\s*[-\u2013\u2014]\s*(\d{5})\b")
_HEADER = (
@@ -186,8 +187,22 @@ class DossierEvidence:
#: the question asks who said what: the stance unit outlasts the cap
stance_intent: bool = False
def _label(self, f: Fact) -> str:
"""The label to copy. A stance count's label carries its accuracy
caveat, so an answer that copies the label copies the caveat (R102)."""
docket = f.values.get("docket") if f.kind == "stance_split" else ""
if not docket:
return f.cite.label
m = _ACC.search(self.accuracy)
if m is None:
tail = "accuracy not yet measured"
else:
where = "held-out" if "held-out" in self.accuracy else "tuning-set"
tail = f"recall {m[1]}%, direction {m[2]}% on {where} letters"
return f"Our classification of docket {docket} comments — preliminary: {tail}"
def _labelled(self, text: str) -> str:
label = {f.id: f.cite.label for f in self.facts}
label = {f.id: self._label(f) for f in self.facts}
def tag(m: re.Match[str]) -> str:
return f"[{label[m.group(1)]}]" if m.group(1) in label else m.group(0)
@@ -230,7 +245,7 @@ class DossierEvidence:
"summary": self._labelled(self.summary),
"facts": [
{
"label": f.cite.label,
"label": self._label(f),
"text": f.text,
"url": f.cite.url,
"year": f.year,
@@ -251,7 +266,7 @@ class DossierEvidence:
lines = [_STANCE_HEADER.format(acc=acc)]
for f in stance:
text = f.text.split(f" {_ACCURACY_MARK}")[0]
lines.append(f"{text} [{f.cite.label}]")
lines.append(f"{text} [{self._label(f)}]")
return lines
def prompt_block(
@@ -270,7 +285,7 @@ class DossierEvidence:
summary = f"Summary: {self._labelled(self.summary)}"
tl = self._timeline_lines()
matched = [
f"{f.text} [{f.cite.label}]"
f"{f.text} [{self._label(f)}]"
for f in (by_id[i] for i in self._prompt_fact_ids(max_rows) if i in by_id)
]
stance = self._stance_unit()

View File

@@ -43,14 +43,14 @@ ACC = (
)
def _fact(i, kind, text, year=2023, label=None, note="", codes=("99341",)):
def _fact(i, kind, text, year=2023, label=None, note="", codes=("99341",), values=None):
return Fact(
id=f"F{i}",
kind=kind,
year=year,
codes=codes,
text=text,
values={},
values=values or {},
cite=Cite("fr", label or f"L{i}", f"https://x/{i}"),
note=note,
)
@@ -72,6 +72,7 @@ def _dossier(extra=(), accuracy=ACC):
4,
"stance_split",
"Of 3 CY2023 commenters we classified, 2 supported and 1 opposed. " + ACC,
values={"docket": "CMS-2022-0113"},
),
_fact(5, "cms_summary", "CMS reported that commenters supported it."),
_fact(
@@ -534,3 +535,38 @@ def test_failed_dossier_query_is_logged_not_raised(replica, monkeypatch, caplog)
with caplog.at_level(logging.WARNING, logger="llm.dossier"):
assert dossier_evidence("home visits?", replica) is None
assert "dossier evidence skipped (query)" in caplog.text
STANCE_LABEL = (
"Our classification of docket CMS-2022-0113 comments — preliminary: "
"recall 79%, direction 74% on held-out letters"
)
def test_stance_label_carries_the_accuracy_caveat(replica):
ev = _ev(replica, "What did commenters say about home visits in CY2023?")
block = ev.prompt_block()
assert f"2 supported and 1 opposed. [{STANCE_LABEL}]" in block
p = ev.payload()
stance = [f for f in p["facts"] if f["kind"] == "stance_split"]
assert STANCE_LABEL in {f["label"] for f in stance} # collectLinks key
# a fact without a docket keeps its own label
assert "L8" in {f["label"] for f in stance}
def test_stance_label_variants():
base = _dossier()
facts = tuple(base.facts.values())
mk = lambda acc: DossierEvidence( # noqa: E731
"F", "N", "u", "[F4]", facts, (), accuracy=acc
)
label = lambda ev: ev._label(base.facts["F4"]) # noqa: E731
assert label(mk("")).endswith("preliminary: accuracy not yet measured")
tuned = (
"Preliminary model classification, measured on the tuning set (20 "
"hand-coded comments, 27 positions; X): it finds 41% of hand-coded "
"positions, and the direction is right for 82% of those it finds."
)
assert label(mk(tuned)).endswith("recall 41%, direction 82% on tuning-set letters")
# the summary tag resolves to the same label
assert f"[{STANCE_LABEL}]" in mk(ACC).payload()["summary"]