From c292e2d1964b421514b3ff6440896d70c04e22ed Mon Sep 17 00:00:00 2001 From: kert Date: Fri, 13 Nov 2026 01:59:45 -0500 Subject: [PATCH] fix(llm): stance_split labels carry the accuracy caveat (R102) --- src/llm/dossier.py | 23 ++++++++++++++++++---- tests/llm/test_dossier.py | 40 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/src/llm/dossier.py b/src/llm/dossier.py index 253266d..3650be2 100644 --- a/src/llm/dossier.py +++ b/src/llm/dossier.py @@ -58,6 +58,7 @@ TIMELINE_FLOOR = 4 _STANCE_INTENT = re.compile( r"comment|support|oppos|stakeholder|societ|association|lobby", re.I ) +_ACC = re.compile(r"finds (\d+)% of hand-coded.*?right for (\d+)%") _RANGE = re.compile(r"\b(\d{5})\s*[-\u2013\u2014]\s*(\d{5})\b") _HEADER = ( @@ -186,8 +187,22 @@ class DossierEvidence: #: the question asks who said what: the stance unit outlasts the cap stance_intent: bool = False + def _label(self, f: Fact) -> str: + """The label to copy. A stance count's label carries its accuracy + caveat, so an answer that copies the label copies the caveat (R102).""" + docket = f.values.get("docket") if f.kind == "stance_split" else "" + if not docket: + return f.cite.label + m = _ACC.search(self.accuracy) + if m is None: + tail = "accuracy not yet measured" + else: + where = "held-out" if "held-out" in self.accuracy else "tuning-set" + tail = f"recall {m[1]}%, direction {m[2]}% on {where} letters" + return f"Our classification of docket {docket} comments — preliminary: {tail}" + def _labelled(self, text: str) -> str: - label = {f.id: f.cite.label for f in self.facts} + label = {f.id: self._label(f) for f in self.facts} def tag(m: re.Match[str]) -> str: return f"[{label[m.group(1)]}]" if m.group(1) in label else m.group(0) @@ -230,7 +245,7 @@ class DossierEvidence: "summary": self._labelled(self.summary), "facts": [ { - "label": f.cite.label, + "label": self._label(f), "text": f.text, "url": f.cite.url, "year": f.year, @@ -251,7 +266,7 @@ class DossierEvidence: lines = [_STANCE_HEADER.format(acc=acc)] for f in stance: text = f.text.split(f" {_ACCURACY_MARK}")[0] - lines.append(f"{text} [{f.cite.label}]") + lines.append(f"{text} [{self._label(f)}]") return lines def prompt_block( @@ -270,7 +285,7 @@ class DossierEvidence: summary = f"Summary: {self._labelled(self.summary)}" tl = self._timeline_lines() matched = [ - f"{f.text} [{f.cite.label}]" + f"{f.text} [{self._label(f)}]" for f in (by_id[i] for i in self._prompt_fact_ids(max_rows) if i in by_id) ] stance = self._stance_unit() diff --git a/tests/llm/test_dossier.py b/tests/llm/test_dossier.py index 6066aab..9d20f49 100644 --- a/tests/llm/test_dossier.py +++ b/tests/llm/test_dossier.py @@ -43,14 +43,14 @@ ACC = ( ) -def _fact(i, kind, text, year=2023, label=None, note="", codes=("99341",)): +def _fact(i, kind, text, year=2023, label=None, note="", codes=("99341",), values=None): return Fact( id=f"F{i}", kind=kind, year=year, codes=codes, text=text, - values={}, + values=values or {}, cite=Cite("fr", label or f"L{i}", f"https://x/{i}"), note=note, ) @@ -72,6 +72,7 @@ def _dossier(extra=(), accuracy=ACC): 4, "stance_split", "Of 3 CY2023 commenters we classified, 2 supported and 1 opposed. " + ACC, + values={"docket": "CMS-2022-0113"}, ), _fact(5, "cms_summary", "CMS reported that commenters supported it."), _fact( @@ -534,3 +535,38 @@ def test_failed_dossier_query_is_logged_not_raised(replica, monkeypatch, caplog) with caplog.at_level(logging.WARNING, logger="llm.dossier"): assert dossier_evidence("home visits?", replica) is None assert "dossier evidence skipped (query)" in caplog.text + + +STANCE_LABEL = ( + "Our classification of docket CMS-2022-0113 comments — preliminary: " + "recall 79%, direction 74% on held-out letters" +) + + +def test_stance_label_carries_the_accuracy_caveat(replica): + ev = _ev(replica, "What did commenters say about home visits in CY2023?") + block = ev.prompt_block() + assert f"2 supported and 1 opposed. [{STANCE_LABEL}]" in block + p = ev.payload() + stance = [f for f in p["facts"] if f["kind"] == "stance_split"] + assert STANCE_LABEL in {f["label"] for f in stance} # collectLinks key + # a fact without a docket keeps its own label + assert "L8" in {f["label"] for f in stance} + + +def test_stance_label_variants(): + base = _dossier() + facts = tuple(base.facts.values()) + mk = lambda acc: DossierEvidence( # noqa: E731 + "F", "N", "u", "[F4]", facts, (), accuracy=acc + ) + label = lambda ev: ev._label(base.facts["F4"]) # noqa: E731 + assert label(mk("")).endswith("preliminary: accuracy not yet measured") + tuned = ( + "Preliminary model classification, measured on the tuning set (20 " + "hand-coded comments, 27 positions; X): it finds 41% of hand-coded " + "positions, and the direction is right for 82% of those it finds." + ) + assert label(mk(tuned)).endswith("recall 41%, direction 82% on tuning-set letters") + # the summary tag resolves to the same label + assert f"[{STANCE_LABEL}]" in mk(ACC).payload()["summary"]