homelab/stack
1
0
Fork 0
Code Issues 210 Pull Requests 2 Actions Packages Projects 1 Releases 1 Wiki Activity

fix(pfs): stance R66a/R67/R68 — stance_eval JSON, method record survives reaction reruns, family-named relevance window, packaged golden set, load_duration log

Browse Source 
- reaction --write carries stance_method/stance_agreement/stance_eval over from
  the stored docket row (it used to wipe the PRELIMINARY marker)
- pfs.code_reaction.stance_eval: {cell, recall, direction, n_items,
  n_positions, family, docket, in_sample}, stamped only for the golden set's
  family; new writes leave stance_agreement NULL, older rows stay readable
- the golden set moves to src/pfs/golden/stance_home_visits_cy2023.json
- R67: the +/-1-sentence window must name the event's subject keyword AND this
  family (family code, offered link code/alias, pair phrase); valuation needs a
  family code
- JsonExtractor logs Ollama load_duration at DEBUG
- stance-eval prints recall and direction
This commit is contained in:
kert
2026-11-12 16:04:19 -05:00
parent 12d4199247
commit dfaaf43890
9 changed files with 374 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
src/cli/pfs.py
View File

@@ -29,6 +29,7 @@ import typer
from pfs.cms_summaries import summaries as cms_summaries_rows
from pfs.codetables import (
STANCE_METHOD_FIELDS,
ReactionRow,
cpt_years,
ensure_tables,
@@ -88,6 +89,7 @@ from pfs.stance import (
TEXT_CHUNKS,
docket_events,
docket_rule_year,
eval_record,
event_subjects,
rollup,
score,
@@ -718,9 +720,10 @@ def _overlay_stances(
rows: list[Any], stances: list[Any], stored: list[Any], commenters: Counter
) -> list[Any]:
"""Put the classified counts of each docket's stored stances on its row,
so a reaction rerun never zeroes them. The event order is the one the
docket's last ``stance`` run stored (*stored* are the family's stored
reaction rows); *commenters* counts ``pfs.commenter`` rows per docket (R62)."""
so a reaction rerun never zeroes them. The event order and the stance
method record (``STANCE_METHOD_FIELDS``, R66a) are the ones the docket's
last ``stance`` run stored (*stored* are the family's stored reaction
rows); *commenters* counts ``pfs.commenter`` rows per docket (R62)."""
old = {r.period: r for r in stored}
out = []
for r in rows:
@@ -728,8 +731,14 @@ def _overlay_stances(
if r.period_kind != "docket" or not mine:
out.append(r)
continue
order = list(json.loads(old[r.period].events_json)) if r.period in old else []
out.append(rollup(mine, base=r, order=order, n_commenters=commenters[r.period]))
prev = old.get(r.period)
order = list(json.loads(prev.events_json)) if prev else []
rolled = rollup(mine, base=r, order=order, n_commenters=commenters[r.period])
if prev:
rolled = replace(
rolled, **{f: getattr(prev, f) for f in STANCE_METHOD_FIELDS}
)
out.append(rolled)
return out
@@ -1414,7 +1423,9 @@ def stance(
DEFAULT_MODEL, "--model", help="instruct or large (the pinned model)."
),
golden: Path = typer.Option(
GOLDEN_PATH, "--golden", help="Golden set whose agreement --write records."
GOLDEN_PATH,
"--golden",
help="Golden set whose measurement --write records (its family only).",
),
write: bool = typer.Option(False, "--write"),
) -> None:
@@ -1521,7 +1532,8 @@ def stance(
new = replace(
rollup(rows, base=base, order=order, n_commenters=len(commenter_rows)),
stance_method=f"{PROMPT_VERSIONS[method]}:{name}",
stance_agreement=_method_agreement(golden, method, name),
stance_agreement=None,
stance_eval=_method_eval(golden, method, name, fam.key),
)
keep = [
r
@@ -1536,21 +1548,21 @@ def stance(
_publish()
def _method_agreement(golden: Path, method: str, name: str) -> float | None:
"""The method's cell agreement on the golden set, replayed from the ledger
(no model call); ``None`` when there is no golden file or its ledger is
def _method_eval(golden: Path, method: str, name: str, family: str) -> str | None:
"""The method's measurement on the golden set as ``stance_eval`` JSON
(R66a), replayed from the ledger (no model call). ``None`` when there is
no golden file, the golden set is another family's, or its ledger is
incomplete."""
import json
if not golden.exists():
return None
sc, _ = _golden_run(
json.loads(golden.read_text(encoding="utf-8")), method, ask=None, name=name
)
data = json.loads(golden.read_text(encoding="utf-8"))
if data.get("family") != family:
return None
sc, _ = _golden_run(data, method, ask=None, name=name)
if sc is None:
typer.echo(f"golden set not fully classified for {method} on {name}")
return None
return sc.cell_agreement
return json.dumps(eval_record(data, sc))
def _golden_run(
@@ -1672,6 +1684,10 @@ def stance_eval(
f"firm accuracy {sc.firm_accuracy:.3f} ({sc.firm_correct}/{sc.firm_cells}); "
f"gate {'PASS' if sc.passes() else 'FAIL'}"
)
typer.echo(
f"recall {sc.recall:.3f} ({sc.found}/{sc.positions}); "
f"direction {sc.direction:.3f} ({sc.right}/{sc.found})"
)
labels = list(sc.confusion)
typer.echo(
"confusion (rows golden, columns model): " + " ".join(f"{x:>8}" for x in labels)

9
src/llm/structured.py
View File

@@ -116,6 +116,15 @@ class JsonExtractor:
) from exc
continue
break
if "load_duration" in data:
# R68: Ollama reports the model (re)load time in ns; a reload per
# call shows up here
log.debug(
"extract: %s on %s load_duration %.2fs",
self.model,
host,
data["load_duration"] / 1e9,
)
reply = data.get("message", {}).get("content", "")
try:
obj = json.loads(reply)

24
src/pfs/codetables.py
View File

@@ -176,9 +176,20 @@ class ReactionRow:
#: classified
n_commenters: int | None = None
#: R65: how the stances were made (``<prompt version>:<model>``) and that
#: method's cell agreement on the golden set; ``None`` = never classified
#: method's cell agreement on the golden set; ``None`` = never classified.
#: ``stance_agreement`` is superseded by ``stance_eval`` (R66a): rows
#: written since carry ``None`` there; older rows keep their figure.
stance_method: str | None = None
stance_agreement: float | None = None
#: R66a: the method's measurement on the hand-coded golden set, JSON
#: ``{cell, recall, direction, n_items, n_positions, family, docket,
#: in_sample}``; ``None`` on every family but the golden set's
stance_eval: str | None = None
#: the ``ReactionRow`` fields that record how a docket's stances were made;
#: a reaction rerun carries them over from the stored row (R66a)
STANCE_METHOD_FIELDS = ("stance_method", "stance_agreement", "stance_eval")
@dataclass(frozen=True)
@@ -480,7 +491,7 @@ CREATE TABLE IF NOT EXISTS pfs.code_reaction (
n_items INTEGER, n_total INTEGER, stance_support INTEGER, stance_oppose INTEGER,
stance_modify INTEGER, stance_unclear INTEGER, sample_json VARCHAR,
events_json VARCHAR, n_classified INTEGER, n_commenters INTEGER,
stance_method VARCHAR, stance_agreement DOUBLE);
stance_method VARCHAR, stance_agreement DOUBLE, stance_eval VARCHAR);
CREATE TABLE IF NOT EXISTS pfs.cpt_section (
edition_year INTEGER, item_key VARCHAR, sec_id VARCHAR, level INTEGER, title VARCHAR,
path VARCHAR[], path_key VARCHAR, code_lo VARCHAR, code_hi VARCHAR, guideline VARCHAR);
@@ -594,6 +605,10 @@ def ensure_tables(con: Any) -> None:
con.execute(
"ALTER TABLE pfs.code_reaction ADD COLUMN IF NOT EXISTS stance_agreement DOUBLE"
)
# R66a: the method's golden-set measurement (JSON)
con.execute(
"ALTER TABLE pfs.code_reaction ADD COLUMN IF NOT EXISTS stance_eval VARCHAR"
)
con.execute(
"ALTER TABLE pfs.comment_stance ADD COLUMN IF NOT EXISTS reviewed BOOLEAN "
"DEFAULT false"
@@ -748,8 +763,9 @@ def read_reaction(con: Any, family: str) -> list[ReactionRow]:
[family],
).fetchall()
# a replica published before ``events_json``/``n_classified``/
# ``n_commenters`` existed has shorter rows; NULL events read as ``{}``
padded = [(*r, None, None, None, None, None)[:16] for r in rows]
# ``n_commenters``/``stance_*`` existed has shorter rows; NULL events
# read as ``{}``
padded = [(*r, None, None, None, None, None, None)[:17] for r in rows]
return [ReactionRow(*r[:11], r[11] or "{}", *r[12:]) for r in padded]

0
tests/pfs/fixtures/stance_golden_cy2023.json → src/pfs/golden/stance_home_visits_cy2023.json
View File

100
src/pfs/stance.py
View File

@@ -20,9 +20,9 @@ import re
from collections import defaultdict
from collections.abc import Mapping
from dataclasses import dataclass, replace
from pathlib import Path
from typing import Any, Callable, Sequence
from conf import ROOT
from pfs.codetables import (
CommenterRow,
ReactionRow,
@@ -43,12 +43,13 @@ PROMPT_VERSIONS = {"multi": "stance-v4", "per-event": "stance-v4-per-event"}
DEFAULT_METHOD = "multi"
DEFAULT_MODEL = "large"
STANCE_PROMPT_VERSION = PROMPT_VERSIONS[DEFAULT_METHOD]
#: the hand-coded golden set that scores a method (R63), packaged with pfs
#: (R67): 20 CY2023 HOME-VISITS commenters, every (item, event) cell labelled
GOLDEN_PATH = Path(__file__).resolve().parent / "golden/stance_home_visits_cy2023.json"
#: the comment text window handed to the model: ~6,000 characters is under
#: 2,000 tokens, and with the event texts (<= ~6,000 characters) it sits well
#: inside the ``num_ctx`` 8192 that ``JsonExtractor`` sends (llm.config
#: ``chat_num_ctx``)
#: the hand-coded golden set that scores a method (R63)
GOLDEN_PATH = ROOT / "tests/pfs/fixtures/stance_golden_cy2023.json"
TEXT_CHARS = 6000
TEXT_CHUNKS = 8
STANCES = ("support", "modify", "oppose", "unclear")
@@ -478,21 +479,25 @@ def raw_positions(
@dataclass(frozen=True)
class Subject:
"""What an event is about (R65): a position counts only if its quote, or
the sentence of the comment that holds it, names one of these."""
the sentences around it, name one of ``words``/``codes``/``phrases`` --
and, when ``family`` is set, name that too (R67: the event's subject AND
this family)."""
words: re.Pattern[str]
codes: re.Pattern[str] | None = None
phrases: re.Pattern[str] | None = None
family: Subject | None = None
def named_in(self, hay: str) -> bool:
return bool(
self.words.search(hay)
or (self.codes and codes_in(_expand_ranges(hay), self.codes))
or (self.phrases and self.phrases.search(hay))
)
) and (self.family is None or self.family.named_in(hay))
_NEVER = re.compile(r"(?!x)x")
_FIXED_SLUGS = ("valuation", "coding", "telehealth", "other")
def _any(words: Sequence[str]) -> re.Pattern[str]:
@@ -500,13 +505,29 @@ def _any(words: Sequence[str]) -> re.Pattern[str]:
def event_subjects(family: Any, events: Sequence[EventChoice]) -> dict[str, Subject]:
"""The relevance ``Subject`` of each event of *family*, by its slug."""
"""The relevance ``Subject`` of each event of *family*, by its slug (R65).
R67: the window must name the event's subject keyword AND this family --
one of its codes (ranges expanded), the code or alias of a link offered
among *events*, or a pair phrase; for valuation, one of its codes."""
codes = code_pattern(family.codes)
phrases = (
_any([re.escape(x) for x in family.pair_phrases])
if family.pair_phrases
else None
)
links = {ln.target: ln for ln in family.links}
link_names = {
slug: [slug, *(links[slug].aliases if slug in links else ())]
for slug in (e.event_key.rsplit(":", 1)[-1] for e in events)
if slug not in _FIXED_SLUGS
}
linked = [n for names in link_names.values() for n in names]
named = Subject(
_any([rf"\b{re.escape(n)}\b" for n in linked]) if linked else _NEVER,
codes,
phrases,
)
by_code = Subject(_NEVER, codes)
out: dict[str, Subject] = {}
for e in events:
slug = e.event_key.rsplit(":", 1)[-1]
@@ -515,7 +536,7 @@ def event_subjects(family: Any, events: Sequence[EventChoice]) -> dict[str, Subj
_any(
["RVU", "valuation", "value", "percentile", r"\btime\b", "payment"]
),
codes,
family=by_code,
)
elif slug == "coding":
sub = Subject(
@@ -529,16 +550,20 @@ def event_subjects(family: Any, events: Sequence[EventChoice]) -> dict[str, Subj
r"\btime\b",
]
),
codes,
family=named,
)
elif slug == "telehealth":
sub = Subject(_any(["telehealth", "audio", "video", "virtual"]))
sub = Subject(
_any(["telehealth", "audio", "video", "virtual"]), family=named
)
elif slug == "other":
# its subject is this family's own codes or pair phrases
sub = Subject(_NEVER, codes, phrases)
else:
link = next((ln for ln in family.links if ln.target == slug), None)
names = [slug, *(link.aliases if link else ())]
sub = Subject(_any([*(re.escape(n) for n in names), "prolonged"]))
sub = Subject(
_any([*(re.escape(n) for n in link_names[slug]), "prolonged"]),
family=named,
)
out[e.event_key] = sub
return out
@@ -789,6 +814,21 @@ class Scores:
#: ``confusion[golden][model]`` over ``LABELS``
confusion: dict[str, dict[str, int]]
disagreements: tuple[Disagreement, ...]
#: R66a: hand-coded positions (golden cells other than ``none``), those
#: the method gives any position on, and those it gives the right one
positions: int = 0
found: int = 0
right: int = 0
@property
def recall(self) -> float:
"""Hand-coded positions found / hand-coded positions."""
return self.found / self.positions if self.positions else 0.0
@property
def direction(self) -> float:
"""Found positions with the hand-coded stance / found positions."""
return self.right / self.found if self.found else 0.0
@property
def cell_agreement(self) -> float:
@@ -830,7 +870,7 @@ def score(golden: Mapping[str, Any], rows: Sequence[StanceRow]) -> Scores:
is the row's stance, or ``none`` when the item has no row for the event."""
model = {(r.item_key, r.event_key): r.stance for r in rows}
confusion = {g: dict.fromkeys(LABELS, 0) for g in LABELS}
cells = agree = presence = firm_n = firm_ok = 0
cells = agree = presence = firm_n = firm_ok = positions = found = right = 0
bad: list[Disagreement] = []
for (item, event), g in sorted(golden_labels(golden).items()):
m = model.get((item, event), "none")
@@ -841,7 +881,39 @@ def score(golden: Mapping[str, Any], rows: Sequence[StanceRow]) -> Scores:
if g in _FIRM:
firm_n += 1
firm_ok += g == m
if g != "none":
positions += 1
found += m != "none"
right += g == m
if g != m:
bad.append(Disagreement(item, event, g, m, _severity(g, m)))
bad.sort(key=lambda d: (-d.severity, d.item_key, d.event_key))
return Scores(cells, agree, presence, firm_n, firm_ok, confusion, tuple(bad))
return Scores(
cells,
agree,
presence,
firm_n,
firm_ok,
confusion,
tuple(bad),
positions,
found,
right,
)
def eval_record(golden: Mapping[str, Any], sc: Scores) -> dict[str, Any]:
"""The method's measurement as ``pfs.code_reaction.stance_eval`` stores
it (R66a): cell agreement, recall (found / hand-coded positions) and
direction (right / found), on the golden set's items, family and docket.
``in_sample`` is true unless the golden file says it is held out."""
return {
"cell": round(sc.cell_agreement, 3),
"recall": round(sc.recall, 3),
"direction": round(sc.direction, 3),
"n_items": len(golden["items"]),
"n_positions": sc.positions,
"family": golden["family"],
"docket": golden["docket"],
"in_sample": bool(golden.get("in_sample", True)),
}

69
tests/cli/test_pfs_cli.py
View File

@@ -33,6 +33,7 @@ from pfs.codetables import (
)
from pfs.cpt_model import CptCode, CptEdition, CptSection
from pfs.extract import Extraction
from pfs.stance import Scores
runner = CliRunner()
@@ -1983,6 +1984,8 @@ class TestCommenters:
ASK = SimpleNamespace(model="qwen-x")
#: a golden-set score shaped like the CY2023 measurement (R66a)
SCORES = Scores(100, 83, 88, 24, 10, {}, (), positions=27, found=15, right=10)
class TestStance:
@@ -2009,9 +2012,7 @@ class TestStance:
monkeypatch.setattr(pfs_cli, "_engine", lambda: "ENGINE")
monkeypatch.setattr(pfs_cli, "_asker", lambda large=False: ASK)
monkeypatch.setattr(
pfs_cli,
"_golden_run",
lambda data, method, **kw: (SimpleNamespace(cell_agreement=0.8), {}),
pfs_cli, "_golden_run", lambda data, method, **kw: (SCORES, {})
)
monkeypatch.setattr(pfs_cli, "_model_name", lambda large=False: "qwen-x")
from pfs.stance import EventChoice
@@ -2219,11 +2220,13 @@ class TestStance:
for k in ("K1", "K2", "K3")
],
) # fmt: skip
ev = json.dumps({"cell": 0.83, "recall": 0.556, "in_sample": True})
write_reaction(
con._real,
"CCM",
[
ReactionRow("CCM", "D1", "docket", 2023, 5, 20, 0, 0, 0, 0, "[]", '{"CCM:2023:other": {}}')
ReactionRow("CCM", "D1", "docket", 2023, 5, 20, 0, 0, 0, 0, "[]", '{"CCM:2023:other": {}}',
stance_method="stance-v4:qwen2.5:32b", stance_agreement=0.83, stance_eval=ev),
],
) # fmt: skip
write_stances(
@@ -2243,6 +2246,14 @@ class TestStance:
assert (rows["D2"].n_classified, rows["D2"].n_commenters) == (None, None)
assert list(json.loads(rows["D1"].events_json)) == ["CCM:2023:other"]
assert rows["D2"].stance_modify == 0 and rows["D2"].events_json == "{}"
# R66a: a reaction rerun keeps the stored row's stance method record
d1 = rows["D1"]
assert (d1.stance_method, d1.stance_agreement, d1.stance_eval) == (
"stance-v4:qwen2.5:32b",
0.83,
ev,
)
assert (rows["D2"].stance_method, rows["D2"].stance_eval) == (None, None)
def test_stance_sample_refused_when_stances_stored(self, con, monkeypatch):
from pfs.codetables import StanceRow, write_stances
@@ -2386,6 +2397,7 @@ class TestStanceEval:
assert "cells 4: agreement 0.750 (3)" in res.output
assert "presence 1.000 (4)" in res.output
assert "firm accuracy 0.500 (1/2); gate FAIL" in res.output
assert "recall 1.000 (2/2); direction 0.500 (1/2)" in res.output
assert "confusion (rows golden, columns model)" in res.output
assert "disagree K2 G0318: golden oppose, model support" in res.output
@@ -2421,12 +2433,38 @@ class TestStanceMethodRecord:
got = {r.period: r for r in read_reaction(con._real, "HOME-VISITS")}
d1 = got["D1"]
assert d1.stance_method == f"{pfs_cli.PROMPT_VERSIONS['multi']}:qwen-x"
assert d1.stance_agreement == 0.8
assert got["R1"].stance_method is None and got["R1"].stance_agreement is None
# R66a: the measurement is the stance_eval JSON; the single figure is gone
assert d1.stance_agreement is None
assert json.loads(d1.stance_eval) == {
"cell": 0.83,
"recall": 0.556,
"direction": 0.667,
"n_items": 20,
"n_positions": 27,
"family": "HOME-VISITS",
"docket": "CMS-2022-0113",
"in_sample": True,
}
assert got["R1"].stance_method is None and got["R1"].stance_eval is None
assert seen["ask"] is ASK
res = runner.invoke(app, t._args("--model", "huge"))
assert res.exit_code != 0 and "--model must be instruct or large" in res.output
def test_eval_is_stamped_only_for_the_golden_family(
self, con, monkeypatch, tmp_path
):
"""A golden set of another family is never replayed nor stamped."""
t = TestStance()
t._patch(con, monkeypatch)
monkeypatch.setattr(pfs_cli, "_golden_run", lambda *a, **k: 1 / 0)
g = tmp_path / "g.json"
g.write_text(json.dumps({"family": "CCM", "docket": "D9", "items": []}))
res = runner.invoke(app, t._args("--golden", str(g), "--write"))
assert res.exit_code == 0, res.output
d1 = {r.period: r for r in read_reaction(con._real, "HOME-VISITS")}["D1"]
assert d1.stance_method and d1.stance_eval is None
assert d1.stance_agreement is None
def test_large_model_builds_the_large_asker(self, con, monkeypatch):
t = TestStance()
t._patch(con, monkeypatch)
@@ -2439,22 +2477,21 @@ class TestStanceMethodRecord:
runner.invoke(app, t._args()) # the default model is the large one
assert calls == [True, False, pfs_cli.DEFAULT_MODEL == "large"]
def test_agreement_is_none_without_golden_or_with_pending(
self, monkeypatch, tmp_path
):
assert pfs_cli._method_agreement(tmp_path / "none.json", "multi", "m") is None
def test_eval_is_none_without_golden_or_with_pending(self, monkeypatch, tmp_path):
F = "HOME-VISITS"
assert pfs_cli._method_eval(tmp_path / "none.json", "multi", "m", F) is None
g = tmp_path / "g.json"
g.write_text("{}")
g.write_text(json.dumps({"family": F, "docket": "D1", "items": [{}]}))
monkeypatch.setattr(
pfs_cli, "_golden_run", lambda data, method, **kw: (None, {})
)
assert pfs_cli._method_agreement(g, "multi", "m") is None
assert pfs_cli._method_eval(g, "multi", "m", F) is None
monkeypatch.setattr(
pfs_cli,
"_golden_run",
lambda data, method, **kw: (SimpleNamespace(cell_agreement=0.5), {}),
pfs_cli, "_golden_run", lambda data, method, **kw: (SCORES, {})
)
assert pfs_cli._method_agreement(g, "multi", "m") == 0.5
got = json.loads(pfs_cli._method_eval(g, "multi", "m", F))
assert (got["recall"], got["direction"], got["n_items"]) == (0.556, 0.667, 1)
assert pfs_cli._method_eval(g, "multi", "m", "CCM") is None
def test_model_of_and_slow_post(self, monkeypatch):
import httpx

21
tests/llm/test_structured.py
View File

@@ -2,6 +2,8 @@
from __future__ import annotations
import logging
import httpx
import pytest
@@ -72,6 +74,25 @@ def test_extractor_posts_schema_and_fixed_model():
assert [m["role"] for m in body["messages"]] == ["system", "user"]
def test_extractor_logs_ollama_load_duration_at_debug(caplog):
"""R68: Ollama's load_duration (ns) is logged at DEBUG, so model reloads
are measurable; a reply without it logs no load line."""
reply = {**_reply('{"ok": true}'), "load_duration": 2_500_000_000}
ask = json_extractor(Cfg(), Pool(["http://rig"]), post=lambda u, json=None: reply)
with caplog.at_level(logging.DEBUG, logger="llm.structured"):
assert ask("s", "u", SCHEMA) == {"ok": True}
loads = [r for r in caplog.records if "load_duration" in r.getMessage()]
assert len(loads) == 1 and loads[0].levelno == logging.DEBUG
assert loads[0].getMessage() == "extract: small on http://rig load_duration 2.50s"
caplog.clear()
bare = json_extractor(
Cfg(), Pool(["h"]), post=lambda u, json=None: _reply('{"ok": true}')
)
with caplog.at_level(logging.DEBUG, logger="llm.structured"):
assert bare("s", "u", SCHEMA) == {"ok": True}
assert not [r for r in caplog.records if "load_duration" in r.getMessage()]
def test_extractor_default_post_is_classifys():
from llm.classify import _default_post

35
tests/pfs/test_reaction.py
View File

@@ -2,6 +2,7 @@
from __future__ import annotations
import dataclasses
import json
from unittest.mock import MagicMock
@@ -459,6 +460,40 @@ class TestEventsJson:
None,
None,
)
assert got.stance_eval is None
def test_stance_eval_roundtrips_and_an_older_table_gains_it(self):
"""R66a: ``stance_eval`` (JSON) is a trailing column; a table from
round 3 (``stance_agreement`` last) keeps its rows readable and gains
the column in place."""
c = duckdb.connect(":memory:")
c.execute("CREATE SCHEMA pfs")
c.execute(
"CREATE TABLE pfs.code_reaction (family VARCHAR, period VARCHAR, "
"period_kind VARCHAR, year INTEGER, n_items INTEGER, n_total INTEGER, "
"stance_support INTEGER, stance_oppose INTEGER, stance_modify INTEGER, "
"stance_unclear INTEGER, sample_json VARCHAR, events_json VARCHAR, "
"n_classified INTEGER, n_commenters INTEGER, stance_method VARCHAR, "
"stance_agreement DOUBLE)"
)
c.execute(
"INSERT INTO pfs.code_reaction VALUES ('A','D','docket',1,1,1,0,0,0,0,"
"'[]','{}',3,4,'stance-v4:m',0.83)"
)
old = read_reaction(c, "A")[0]
assert (old.stance_method, old.stance_agreement, old.stance_eval) == (
"stance-v4:m",
0.83,
None,
)
ensure_tables(c)
assert read_reaction(c, "A")[0] == old
ev = json.dumps({"cell": 0.83, "recall": 0.556})
row = dataclasses.replace(
old, period="D2", stance_agreement=None, stance_eval=ev
)
write_reaction(c, "A", [old, row])
assert read_reaction(c, "A") == [old, row]
class TestFrPairSpansRefactor:

135
tests/pfs/test_stance.py
View File

@@ -5,6 +5,7 @@ from __future__ import annotations
import dataclasses
import json
import re
from pathlib import Path
import duckdb
import pytest
@@ -29,6 +30,7 @@ from pfs.curated import load_curated
from pfs.runlog import RunLog
from pfs.stance import (
DEFAULT_METHOD,
GOLDEN_PATH,
PROMPT_VERSIONS,
STANCE_PROMPT_VERSION,
EventChoice,
@@ -873,11 +875,16 @@ def test_per_event_limit_and_partial_item_is_pending(tmp_path, events):
# ── scoring against a golden set ─────────────────────────────────────
GOLDEN_PATH = (
__import__("pathlib").Path(__file__).parent
/ "fixtures"
/ "stance_golden_cy2023.json"
)
def test_golden_set_is_packaged_with_pfs():
"""R67: production code reads the golden set from the pfs package, never
from tests/."""
import pfs
assert GOLDEN_PATH.is_file()
assert GOLDEN_PATH.parent == Path(pfs.__file__).resolve().parent / "golden"
assert GOLDEN_PATH.name == "stance_home_visits_cy2023.json"
assert "tests" not in GOLDEN_PATH.parts
def test_golden_fixture_shape():
@@ -951,6 +958,9 @@ def test_score_counts_and_severity():
("e4", 1),
]
assert sc.confusion["oppose"]["none"] == 1
# R66a: golden positions e1 e2 e4 e5 e6; found e1 e2 e4 e5; right e2
assert (sc.positions, sc.found, sc.right) == (5, 4, 1)
assert (sc.recall, sc.direction) == (0.8, 0.25)
from pfs.stance import Scores
empty = Scores(0, 0, 0, 0, 0, {}, ())
@@ -959,6 +969,46 @@ def test_score_counts_and_severity():
0,
0,
)
assert (empty.recall, empty.direction) == (0, 0)
def test_eval_record_is_the_r66a_json():
from pfs.stance import eval_record, golden_labels, score
golden = json.loads(GOLDEN_PATH.read_text())
labels = golden_labels(golden)
firm = sorted(k for k, v in labels.items() if v != "none")
# find 3 of the hand-coded positions, 2 in the right direction, and add one
# position on a golden `none` cell
wrong = "oppose" if labels[firm[2]] != "oppose" else "support"
none_cell = next(k for k, v in sorted(labels.items()) if v == "none")
rows = [
S(*firm[0], labels[firm[0]]),
S(*firm[1], labels[firm[1]]),
S(*firm[2], wrong),
S(*none_cell, "support"),
]
sc = score(golden, rows)
rec = eval_record(golden, sc)
assert list(rec) == [
"cell",
"recall",
"direction",
"n_items",
"n_positions",
"family",
"docket",
"in_sample",
]
n = len(firm)
assert rec["recall"] == round(3 / n, 3) and rec["direction"] == round(2 / 3, 3)
assert rec["cell"] == round(sc.cell_agreement, 3)
assert (rec["n_items"], rec["n_positions"]) == (len(golden["items"]), n)
assert (rec["family"], rec["docket"]) == ("HOME-VISITS", "CMS-2022-0113")
assert rec["in_sample"] is True
# a held-out set says so in its own file
held = eval_record({**golden, "in_sample": False}, sc)
assert held["in_sample"] is False
# ── relevance filter (R65) ───────────────────────────────────────────
@@ -974,22 +1024,72 @@ def test_event_subjects_by_slug():
sub = event_subjects(FAM, SUBJ_EVENTS)
v, c, t, g, o = (sub[e.event_key] for e in SUBJ_EVENTS)
assert v.named_in("the work RVU is low") and v.named_in("see 99341-99343")
# R67: the event's subject keyword AND this family; valuation needs a code
assert v.named_in("the work RVU of 99342 is low")
assert v.named_in("payment for 99341-99343 falls") # a range names 99342
assert not v.named_in("the work RVU is low") # no family code
assert not v.named_in("see 99341-99343") # no subject keyword
assert not v.named_in("home visit codes are undervalued") # a phrase, no code
assert not v.named_in("nothing relevant at all")
assert c.named_in("codes were deleted") and c.named_in("domiciliary care")
assert c.named_in("uses medical decision making") and c.named_in("99339-99341")
assert t.named_in("audio-only visits") and not t.named_in("99341 and RVUs")
assert g.named_in("the new GXXX3 code") and g.named_in("prolonged visits")
assert g.named_in("G0318 is new") and not g.named_in("telehealth only")
assert c.named_in("codes 99339-99341 were deleted")
assert c.named_in("domiciliary care visit codes merge") # keyword and phrase
assert c.named_in("MDM for home or residence visits") and not c.named_in("MDM")
assert not c.named_in("codes were deleted") and not c.named_in("see 99342")
assert c.named_in("GXXX3 time counts") # the link alias names the family
assert t.named_in("audio-only home visit codes") and t.named_in("telehealth 99341")
assert t.named_in("telehealth for GXXX3")
assert not t.named_in("we support telehealth after the PHE")
assert not t.named_in("99341 and RVUs")
assert g.named_in("the new GXXX3 code") and g.named_in("G0318 is new")
assert g.named_in("prolonged home or residence visit time")
assert not g.named_in("prolonged visits") and not g.named_in("telehealth only")
assert not g.named_in("prolonged 99417 for office visits")
assert o.named_in("code 99342 is fine") and o.named_in("a home visit e/m change")
assert not o.named_in("telehealth and RVUs")
# no pair phrases and an unknown link: still well-formed
assert not o.named_in("telehealth and RVUs") and not o.named_in("G0318 only")
# no pair phrases and an unknown link: still well-formed; the link event's
# own code names the family
bare = dataclasses.replace(FAM, pair_phrases=())
other = event_subjects(
bare, [SUBJ_EVENTS[4], EventChoice("F:2023:G9999", "x", 2023)]
)
assert not other["HOME-VISITS:2023:other"].named_in("home visit")
assert other["F:2023:G9999"].named_in("prolonged")
assert other["F:2023:G9999"].named_in("prolonged G9999")
assert not other["F:2023:G9999"].named_in("prolonged")
# a link not offered that year does not name the family
no_link = event_subjects(FAM, SUBJ_EVENTS[2:3])
assert not no_link["HOME-VISITS:2023:telehealth"].named_in("telehealth G0318")
def test_window_must_name_this_family_and_the_subject():
"""R67, from the CY2023 live rows: a general telehealth endorsement and a
valuation complaint that names only "home visit codes" are dropped; a
telehealth ask about the family's codes one sentence away is kept."""
from pfs.stance import event_subjects
sub = event_subjects(FAM, SUBJ_EVENTS)
val, tele = SUBJ_EVENTS[0].event_key, SUBJ_EVENTS[2].event_key
general = (
"This is a significant hit to our ability to provide primary care. "
"We support CMS efforts to expand telehealth services and encourage these "
"efforts continue after the end of the PHE. Many patients can now get care."
)
q = "We support CMS efforts to expand telehealth services and encourage these"
assert _rows([pos(tele, "support", "r", q)], general, sub) == []
phrase_only = (
"We support the nomination of misvalued home visit codes. By combining these "
"two code sets and not increasing the value of the home visit codes our "
"practices would incur a seven percent decrease in revenue next year."
)
q2 = "By combining these two code sets and not increasing the value"
assert _rows([pos(val, "oppose", "r", q2)], phrase_only, sub) == []
ask = (
"Telehealth flexibilities matter to our patients. We recommend that all "
"codes listed above be retained through the end of the year. This includes "
"the home visit codes 99347-99350 for established patients."
)
q3 = "We recommend that all codes listed above be retained through the end"
kept = _rows([pos(tele, "modify", "r", q3)], ask, sub)
assert [(r.event_key, r.stance) for r in kept] == [(tele, "modify")]
def test_holding_sentence():
@@ -1025,9 +1125,10 @@ def test_position_kept_only_if_quote_or_neighbours_name_the_subject():
sub = event_subjects(FAM, SUBJ_EVENTS)
tele, val = SUBJ_EVENTS[2].event_key, SUBJ_EVENTS[0].event_key
text = (
"Telehealth flexibilities matter to our patients. We recommend that all codes "
"listed above should be retained through the end of the year. Unrelated closing "
"words about something else entirely follow in this sentence here."
"Telehealth flexibilities for home visit codes matter to our patients. We "
"recommend that all codes listed above should be retained through the end of "
"the year. Unrelated closing words about something else entirely follow in "
"this sentence here."
)
q = "We recommend that all codes listed above should be retained through the end"
kept = _rows([pos(tele, "modify", "r", q)], text, sub)