fix(llm): stance_split labels carry the accuracy caveat (R102)
Some checks failed
CI / lint (push) Successful in 1m19s
CI / notebooks-smoke (push) Successful in 2m5s
Deploy / changes (push) Successful in 27s
CI / test (push) Failing after 5m1s
Infra CI / zotero (push) Successful in 28s
Infra CI / notebooks (push) Successful in 2m1s
Infra CI / docs (push) Successful in 26s
Infra CI / api (push) Successful in 1m59s
Infra CI / bcda (push) Successful in 2m43s
Infra CI / mc (push) Successful in 22s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Infra CI / llm (push) Successful in 1m35s
Deploy / api (push) Successful in 2m4s
Deploy / bcda (push) Successful in 4m16s
Deploy / llm (push) Successful in 2m35s
Deploy / mc (push) Has been skipped
Deploy / report (push) Successful in 19s
Some checks failed
CI / lint (push) Successful in 1m19s
CI / notebooks-smoke (push) Successful in 2m5s
Deploy / changes (push) Successful in 27s
CI / test (push) Failing after 5m1s
Infra CI / zotero (push) Successful in 28s
Infra CI / notebooks (push) Successful in 2m1s
Infra CI / docs (push) Successful in 26s
Infra CI / api (push) Successful in 1m59s
Infra CI / bcda (push) Successful in 2m43s
Infra CI / mc (push) Successful in 22s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Infra CI / llm (push) Successful in 1m35s
Deploy / api (push) Successful in 2m4s
Deploy / bcda (push) Successful in 4m16s
Deploy / llm (push) Successful in 2m35s
Deploy / mc (push) Has been skipped
Deploy / report (push) Successful in 19s
This commit is contained in:
@@ -58,6 +58,7 @@ TIMELINE_FLOOR = 4
|
||||
_STANCE_INTENT = re.compile(
|
||||
r"comment|support|oppos|stakeholder|societ|association|lobby", re.I
|
||||
)
|
||||
_ACC = re.compile(r"finds (\d+)% of hand-coded.*?right for (\d+)%")
|
||||
_RANGE = re.compile(r"\b(\d{5})\s*[-\u2013\u2014]\s*(\d{5})\b")
|
||||
|
||||
_HEADER = (
|
||||
@@ -186,8 +187,22 @@ class DossierEvidence:
|
||||
#: the question asks who said what: the stance unit outlasts the cap
|
||||
stance_intent: bool = False
|
||||
|
||||
def _label(self, f: Fact) -> str:
|
||||
"""The label to copy. A stance count's label carries its accuracy
|
||||
caveat, so an answer that copies the label copies the caveat (R102)."""
|
||||
docket = f.values.get("docket") if f.kind == "stance_split" else ""
|
||||
if not docket:
|
||||
return f.cite.label
|
||||
m = _ACC.search(self.accuracy)
|
||||
if m is None:
|
||||
tail = "accuracy not yet measured"
|
||||
else:
|
||||
where = "held-out" if "held-out" in self.accuracy else "tuning-set"
|
||||
tail = f"recall {m[1]}%, direction {m[2]}% on {where} letters"
|
||||
return f"Our classification of docket {docket} comments — preliminary: {tail}"
|
||||
|
||||
def _labelled(self, text: str) -> str:
|
||||
label = {f.id: f.cite.label for f in self.facts}
|
||||
label = {f.id: self._label(f) for f in self.facts}
|
||||
|
||||
def tag(m: re.Match[str]) -> str:
|
||||
return f"[{label[m.group(1)]}]" if m.group(1) in label else m.group(0)
|
||||
@@ -230,7 +245,7 @@ class DossierEvidence:
|
||||
"summary": self._labelled(self.summary),
|
||||
"facts": [
|
||||
{
|
||||
"label": f.cite.label,
|
||||
"label": self._label(f),
|
||||
"text": f.text,
|
||||
"url": f.cite.url,
|
||||
"year": f.year,
|
||||
@@ -251,7 +266,7 @@ class DossierEvidence:
|
||||
lines = [_STANCE_HEADER.format(acc=acc)]
|
||||
for f in stance:
|
||||
text = f.text.split(f" {_ACCURACY_MARK}")[0]
|
||||
lines.append(f"{text} [{f.cite.label}]")
|
||||
lines.append(f"{text} [{self._label(f)}]")
|
||||
return lines
|
||||
|
||||
def prompt_block(
|
||||
@@ -270,7 +285,7 @@ class DossierEvidence:
|
||||
summary = f"Summary: {self._labelled(self.summary)}"
|
||||
tl = self._timeline_lines()
|
||||
matched = [
|
||||
f"{f.text} [{f.cite.label}]"
|
||||
f"{f.text} [{self._label(f)}]"
|
||||
for f in (by_id[i] for i in self._prompt_fact_ids(max_rows) if i in by_id)
|
||||
]
|
||||
stance = self._stance_unit()
|
||||
|
||||
@@ -43,14 +43,14 @@ ACC = (
|
||||
)
|
||||
|
||||
|
||||
def _fact(i, kind, text, year=2023, label=None, note="", codes=("99341",)):
|
||||
def _fact(i, kind, text, year=2023, label=None, note="", codes=("99341",), values=None):
|
||||
return Fact(
|
||||
id=f"F{i}",
|
||||
kind=kind,
|
||||
year=year,
|
||||
codes=codes,
|
||||
text=text,
|
||||
values={},
|
||||
values=values or {},
|
||||
cite=Cite("fr", label or f"L{i}", f"https://x/{i}"),
|
||||
note=note,
|
||||
)
|
||||
@@ -72,6 +72,7 @@ def _dossier(extra=(), accuracy=ACC):
|
||||
4,
|
||||
"stance_split",
|
||||
"Of 3 CY2023 commenters we classified, 2 supported and 1 opposed. " + ACC,
|
||||
values={"docket": "CMS-2022-0113"},
|
||||
),
|
||||
_fact(5, "cms_summary", "CMS reported that commenters supported it."),
|
||||
_fact(
|
||||
@@ -534,3 +535,38 @@ def test_failed_dossier_query_is_logged_not_raised(replica, monkeypatch, caplog)
|
||||
with caplog.at_level(logging.WARNING, logger="llm.dossier"):
|
||||
assert dossier_evidence("home visits?", replica) is None
|
||||
assert "dossier evidence skipped (query)" in caplog.text
|
||||
|
||||
|
||||
STANCE_LABEL = (
|
||||
"Our classification of docket CMS-2022-0113 comments — preliminary: "
|
||||
"recall 79%, direction 74% on held-out letters"
|
||||
)
|
||||
|
||||
|
||||
def test_stance_label_carries_the_accuracy_caveat(replica):
|
||||
ev = _ev(replica, "What did commenters say about home visits in CY2023?")
|
||||
block = ev.prompt_block()
|
||||
assert f"2 supported and 1 opposed. [{STANCE_LABEL}]" in block
|
||||
p = ev.payload()
|
||||
stance = [f for f in p["facts"] if f["kind"] == "stance_split"]
|
||||
assert STANCE_LABEL in {f["label"] for f in stance} # collectLinks key
|
||||
# a fact without a docket keeps its own label
|
||||
assert "L8" in {f["label"] for f in stance}
|
||||
|
||||
|
||||
def test_stance_label_variants():
|
||||
base = _dossier()
|
||||
facts = tuple(base.facts.values())
|
||||
mk = lambda acc: DossierEvidence( # noqa: E731
|
||||
"F", "N", "u", "[F4]", facts, (), accuracy=acc
|
||||
)
|
||||
label = lambda ev: ev._label(base.facts["F4"]) # noqa: E731
|
||||
assert label(mk("")).endswith("preliminary: accuracy not yet measured")
|
||||
tuned = (
|
||||
"Preliminary model classification, measured on the tuning set (20 "
|
||||
"hand-coded comments, 27 positions; X): it finds 41% of hand-coded "
|
||||
"positions, and the direction is right for 82% of those it finds."
|
||||
)
|
||||
assert label(mk(tuned)).endswith("recall 41%, direction 82% on tuning-set letters")
|
||||
# the summary tag resolves to the same label
|
||||
assert f"[{STANCE_LABEL}]" in mk(ACC).payload()["summary"]
|
||||
|
||||
Reference in New Issue
Block a user