diff --git a/src/cli/pfs.py b/src/cli/pfs.py index 6d61d4f..5ea190e 100644 --- a/src/cli/pfs.py +++ b/src/cli/pfs.py @@ -29,6 +29,7 @@ import typer from pfs.cms_summaries import summaries as cms_summaries_rows from pfs.codetables import ( + STANCE_METHOD_FIELDS, ReactionRow, cpt_years, ensure_tables, @@ -88,6 +89,7 @@ from pfs.stance import ( TEXT_CHUNKS, docket_events, docket_rule_year, + eval_record, event_subjects, rollup, score, @@ -718,9 +720,10 @@ def _overlay_stances( rows: list[Any], stances: list[Any], stored: list[Any], commenters: Counter ) -> list[Any]: """Put the classified counts of each docket's stored stances on its row, - so a reaction rerun never zeroes them. The event order is the one the - docket's last ``stance`` run stored (*stored* are the family's stored - reaction rows); *commenters* counts ``pfs.commenter`` rows per docket (R62).""" + so a reaction rerun never zeroes them. The event order and the stance + method record (``STANCE_METHOD_FIELDS``, R66a) are the ones the docket's + last ``stance`` run stored (*stored* are the family's stored reaction + rows); *commenters* counts ``pfs.commenter`` rows per docket (R62).""" old = {r.period: r for r in stored} out = [] for r in rows: @@ -728,8 +731,14 @@ def _overlay_stances( if r.period_kind != "docket" or not mine: out.append(r) continue - order = list(json.loads(old[r.period].events_json)) if r.period in old else [] - out.append(rollup(mine, base=r, order=order, n_commenters=commenters[r.period])) + prev = old.get(r.period) + order = list(json.loads(prev.events_json)) if prev else [] + rolled = rollup(mine, base=r, order=order, n_commenters=commenters[r.period]) + if prev: + rolled = replace( + rolled, **{f: getattr(prev, f) for f in STANCE_METHOD_FIELDS} + ) + out.append(rolled) return out @@ -1414,7 +1423,9 @@ def stance( DEFAULT_MODEL, "--model", help="instruct or large (the pinned model)." ), golden: Path = typer.Option( - GOLDEN_PATH, "--golden", help="Golden set whose agreement --write records." + GOLDEN_PATH, + "--golden", + help="Golden set whose measurement --write records (its family only).", ), write: bool = typer.Option(False, "--write"), ) -> None: @@ -1521,7 +1532,8 @@ def stance( new = replace( rollup(rows, base=base, order=order, n_commenters=len(commenter_rows)), stance_method=f"{PROMPT_VERSIONS[method]}:{name}", - stance_agreement=_method_agreement(golden, method, name), + stance_agreement=None, + stance_eval=_method_eval(golden, method, name, fam.key), ) keep = [ r @@ -1536,21 +1548,21 @@ def stance( _publish() -def _method_agreement(golden: Path, method: str, name: str) -> float | None: - """The method's cell agreement on the golden set, replayed from the ledger - (no model call); ``None`` when there is no golden file or its ledger is +def _method_eval(golden: Path, method: str, name: str, family: str) -> str | None: + """The method's measurement on the golden set as ``stance_eval`` JSON + (R66a), replayed from the ledger (no model call). ``None`` when there is + no golden file, the golden set is another family's, or its ledger is incomplete.""" - import json - if not golden.exists(): return None - sc, _ = _golden_run( - json.loads(golden.read_text(encoding="utf-8")), method, ask=None, name=name - ) + data = json.loads(golden.read_text(encoding="utf-8")) + if data.get("family") != family: + return None + sc, _ = _golden_run(data, method, ask=None, name=name) if sc is None: typer.echo(f"golden set not fully classified for {method} on {name}") return None - return sc.cell_agreement + return json.dumps(eval_record(data, sc)) def _golden_run( @@ -1672,6 +1684,10 @@ def stance_eval( f"firm accuracy {sc.firm_accuracy:.3f} ({sc.firm_correct}/{sc.firm_cells}); " f"gate {'PASS' if sc.passes() else 'FAIL'}" ) + typer.echo( + f"recall {sc.recall:.3f} ({sc.found}/{sc.positions}); " + f"direction {sc.direction:.3f} ({sc.right}/{sc.found})" + ) labels = list(sc.confusion) typer.echo( "confusion (rows golden, columns model): " + " ".join(f"{x:>8}" for x in labels) diff --git a/src/llm/structured.py b/src/llm/structured.py index dac9c2e..b64c1a0 100644 --- a/src/llm/structured.py +++ b/src/llm/structured.py @@ -116,6 +116,15 @@ class JsonExtractor: ) from exc continue break + if "load_duration" in data: + # R68: Ollama reports the model (re)load time in ns; a reload per + # call shows up here + log.debug( + "extract: %s on %s load_duration %.2fs", + self.model, + host, + data["load_duration"] / 1e9, + ) reply = data.get("message", {}).get("content", "") try: obj = json.loads(reply) diff --git a/src/pfs/codetables.py b/src/pfs/codetables.py index f4abb9a..f95e502 100644 --- a/src/pfs/codetables.py +++ b/src/pfs/codetables.py @@ -176,9 +176,20 @@ class ReactionRow: #: classified n_commenters: int | None = None #: R65: how the stances were made (``:``) and that - #: method's cell agreement on the golden set; ``None`` = never classified + #: method's cell agreement on the golden set; ``None`` = never classified. + #: ``stance_agreement`` is superseded by ``stance_eval`` (R66a): rows + #: written since carry ``None`` there; older rows keep their figure. stance_method: str | None = None stance_agreement: float | None = None + #: R66a: the method's measurement on the hand-coded golden set, JSON + #: ``{cell, recall, direction, n_items, n_positions, family, docket, + #: in_sample}``; ``None`` on every family but the golden set's + stance_eval: str | None = None + + +#: the ``ReactionRow`` fields that record how a docket's stances were made; +#: a reaction rerun carries them over from the stored row (R66a) +STANCE_METHOD_FIELDS = ("stance_method", "stance_agreement", "stance_eval") @dataclass(frozen=True) @@ -480,7 +491,7 @@ CREATE TABLE IF NOT EXISTS pfs.code_reaction ( n_items INTEGER, n_total INTEGER, stance_support INTEGER, stance_oppose INTEGER, stance_modify INTEGER, stance_unclear INTEGER, sample_json VARCHAR, events_json VARCHAR, n_classified INTEGER, n_commenters INTEGER, - stance_method VARCHAR, stance_agreement DOUBLE); + stance_method VARCHAR, stance_agreement DOUBLE, stance_eval VARCHAR); CREATE TABLE IF NOT EXISTS pfs.cpt_section ( edition_year INTEGER, item_key VARCHAR, sec_id VARCHAR, level INTEGER, title VARCHAR, path VARCHAR[], path_key VARCHAR, code_lo VARCHAR, code_hi VARCHAR, guideline VARCHAR); @@ -594,6 +605,10 @@ def ensure_tables(con: Any) -> None: con.execute( "ALTER TABLE pfs.code_reaction ADD COLUMN IF NOT EXISTS stance_agreement DOUBLE" ) + # R66a: the method's golden-set measurement (JSON) + con.execute( + "ALTER TABLE pfs.code_reaction ADD COLUMN IF NOT EXISTS stance_eval VARCHAR" + ) con.execute( "ALTER TABLE pfs.comment_stance ADD COLUMN IF NOT EXISTS reviewed BOOLEAN " "DEFAULT false" @@ -748,8 +763,9 @@ def read_reaction(con: Any, family: str) -> list[ReactionRow]: [family], ).fetchall() # a replica published before ``events_json``/``n_classified``/ - # ``n_commenters`` existed has shorter rows; NULL events read as ``{}`` - padded = [(*r, None, None, None, None, None)[:16] for r in rows] + # ``n_commenters``/``stance_*`` existed has shorter rows; NULL events + # read as ``{}`` + padded = [(*r, None, None, None, None, None, None)[:17] for r in rows] return [ReactionRow(*r[:11], r[11] or "{}", *r[12:]) for r in padded] diff --git a/tests/pfs/fixtures/stance_golden_cy2023.json b/src/pfs/golden/stance_home_visits_cy2023.json similarity index 100% rename from tests/pfs/fixtures/stance_golden_cy2023.json rename to src/pfs/golden/stance_home_visits_cy2023.json diff --git a/src/pfs/stance.py b/src/pfs/stance.py index 198b733..73eb90d 100644 --- a/src/pfs/stance.py +++ b/src/pfs/stance.py @@ -20,9 +20,9 @@ import re from collections import defaultdict from collections.abc import Mapping from dataclasses import dataclass, replace +from pathlib import Path from typing import Any, Callable, Sequence -from conf import ROOT from pfs.codetables import ( CommenterRow, ReactionRow, @@ -43,12 +43,13 @@ PROMPT_VERSIONS = {"multi": "stance-v4", "per-event": "stance-v4-per-event"} DEFAULT_METHOD = "multi" DEFAULT_MODEL = "large" STANCE_PROMPT_VERSION = PROMPT_VERSIONS[DEFAULT_METHOD] +#: the hand-coded golden set that scores a method (R63), packaged with pfs +#: (R67): 20 CY2023 HOME-VISITS commenters, every (item, event) cell labelled +GOLDEN_PATH = Path(__file__).resolve().parent / "golden/stance_home_visits_cy2023.json" #: the comment text window handed to the model: ~6,000 characters is under #: 2,000 tokens, and with the event texts (<= ~6,000 characters) it sits well #: inside the ``num_ctx`` 8192 that ``JsonExtractor`` sends (llm.config #: ``chat_num_ctx``) -#: the hand-coded golden set that scores a method (R63) -GOLDEN_PATH = ROOT / "tests/pfs/fixtures/stance_golden_cy2023.json" TEXT_CHARS = 6000 TEXT_CHUNKS = 8 STANCES = ("support", "modify", "oppose", "unclear") @@ -478,21 +479,25 @@ def raw_positions( @dataclass(frozen=True) class Subject: """What an event is about (R65): a position counts only if its quote, or - the sentence of the comment that holds it, names one of these.""" + the sentences around it, name one of ``words``/``codes``/``phrases`` -- + and, when ``family`` is set, name that too (R67: the event's subject AND + this family).""" words: re.Pattern[str] codes: re.Pattern[str] | None = None phrases: re.Pattern[str] | None = None + family: Subject | None = None def named_in(self, hay: str) -> bool: return bool( self.words.search(hay) or (self.codes and codes_in(_expand_ranges(hay), self.codes)) or (self.phrases and self.phrases.search(hay)) - ) + ) and (self.family is None or self.family.named_in(hay)) _NEVER = re.compile(r"(?!x)x") +_FIXED_SLUGS = ("valuation", "coding", "telehealth", "other") def _any(words: Sequence[str]) -> re.Pattern[str]: @@ -500,13 +505,29 @@ def _any(words: Sequence[str]) -> re.Pattern[str]: def event_subjects(family: Any, events: Sequence[EventChoice]) -> dict[str, Subject]: - """The relevance ``Subject`` of each event of *family*, by its slug.""" + """The relevance ``Subject`` of each event of *family*, by its slug (R65). + R67: the window must name the event's subject keyword AND this family -- + one of its codes (ranges expanded), the code or alias of a link offered + among *events*, or a pair phrase; for valuation, one of its codes.""" codes = code_pattern(family.codes) phrases = ( _any([re.escape(x) for x in family.pair_phrases]) if family.pair_phrases else None ) + links = {ln.target: ln for ln in family.links} + link_names = { + slug: [slug, *(links[slug].aliases if slug in links else ())] + for slug in (e.event_key.rsplit(":", 1)[-1] for e in events) + if slug not in _FIXED_SLUGS + } + linked = [n for names in link_names.values() for n in names] + named = Subject( + _any([rf"\b{re.escape(n)}\b" for n in linked]) if linked else _NEVER, + codes, + phrases, + ) + by_code = Subject(_NEVER, codes) out: dict[str, Subject] = {} for e in events: slug = e.event_key.rsplit(":", 1)[-1] @@ -515,7 +536,7 @@ def event_subjects(family: Any, events: Sequence[EventChoice]) -> dict[str, Subj _any( ["RVU", "valuation", "value", "percentile", r"\btime\b", "payment"] ), - codes, + family=by_code, ) elif slug == "coding": sub = Subject( @@ -529,16 +550,20 @@ def event_subjects(family: Any, events: Sequence[EventChoice]) -> dict[str, Subj r"\btime\b", ] ), - codes, + family=named, ) elif slug == "telehealth": - sub = Subject(_any(["telehealth", "audio", "video", "virtual"])) + sub = Subject( + _any(["telehealth", "audio", "video", "virtual"]), family=named + ) elif slug == "other": + # its subject is this family's own codes or pair phrases sub = Subject(_NEVER, codes, phrases) else: - link = next((ln for ln in family.links if ln.target == slug), None) - names = [slug, *(link.aliases if link else ())] - sub = Subject(_any([*(re.escape(n) for n in names), "prolonged"])) + sub = Subject( + _any([*(re.escape(n) for n in link_names[slug]), "prolonged"]), + family=named, + ) out[e.event_key] = sub return out @@ -789,6 +814,21 @@ class Scores: #: ``confusion[golden][model]`` over ``LABELS`` confusion: dict[str, dict[str, int]] disagreements: tuple[Disagreement, ...] + #: R66a: hand-coded positions (golden cells other than ``none``), those + #: the method gives any position on, and those it gives the right one + positions: int = 0 + found: int = 0 + right: int = 0 + + @property + def recall(self) -> float: + """Hand-coded positions found / hand-coded positions.""" + return self.found / self.positions if self.positions else 0.0 + + @property + def direction(self) -> float: + """Found positions with the hand-coded stance / found positions.""" + return self.right / self.found if self.found else 0.0 @property def cell_agreement(self) -> float: @@ -830,7 +870,7 @@ def score(golden: Mapping[str, Any], rows: Sequence[StanceRow]) -> Scores: is the row's stance, or ``none`` when the item has no row for the event.""" model = {(r.item_key, r.event_key): r.stance for r in rows} confusion = {g: dict.fromkeys(LABELS, 0) for g in LABELS} - cells = agree = presence = firm_n = firm_ok = 0 + cells = agree = presence = firm_n = firm_ok = positions = found = right = 0 bad: list[Disagreement] = [] for (item, event), g in sorted(golden_labels(golden).items()): m = model.get((item, event), "none") @@ -841,7 +881,39 @@ def score(golden: Mapping[str, Any], rows: Sequence[StanceRow]) -> Scores: if g in _FIRM: firm_n += 1 firm_ok += g == m + if g != "none": + positions += 1 + found += m != "none" + right += g == m if g != m: bad.append(Disagreement(item, event, g, m, _severity(g, m))) bad.sort(key=lambda d: (-d.severity, d.item_key, d.event_key)) - return Scores(cells, agree, presence, firm_n, firm_ok, confusion, tuple(bad)) + return Scores( + cells, + agree, + presence, + firm_n, + firm_ok, + confusion, + tuple(bad), + positions, + found, + right, + ) + + +def eval_record(golden: Mapping[str, Any], sc: Scores) -> dict[str, Any]: + """The method's measurement as ``pfs.code_reaction.stance_eval`` stores + it (R66a): cell agreement, recall (found / hand-coded positions) and + direction (right / found), on the golden set's items, family and docket. + ``in_sample`` is true unless the golden file says it is held out.""" + return { + "cell": round(sc.cell_agreement, 3), + "recall": round(sc.recall, 3), + "direction": round(sc.direction, 3), + "n_items": len(golden["items"]), + "n_positions": sc.positions, + "family": golden["family"], + "docket": golden["docket"], + "in_sample": bool(golden.get("in_sample", True)), + } diff --git a/tests/cli/test_pfs_cli.py b/tests/cli/test_pfs_cli.py index c65e5fb..97c58db 100644 --- a/tests/cli/test_pfs_cli.py +++ b/tests/cli/test_pfs_cli.py @@ -33,6 +33,7 @@ from pfs.codetables import ( ) from pfs.cpt_model import CptCode, CptEdition, CptSection from pfs.extract import Extraction +from pfs.stance import Scores runner = CliRunner() @@ -1983,6 +1984,8 @@ class TestCommenters: ASK = SimpleNamespace(model="qwen-x") +#: a golden-set score shaped like the CY2023 measurement (R66a) +SCORES = Scores(100, 83, 88, 24, 10, {}, (), positions=27, found=15, right=10) class TestStance: @@ -2009,9 +2012,7 @@ class TestStance: monkeypatch.setattr(pfs_cli, "_engine", lambda: "ENGINE") monkeypatch.setattr(pfs_cli, "_asker", lambda large=False: ASK) monkeypatch.setattr( - pfs_cli, - "_golden_run", - lambda data, method, **kw: (SimpleNamespace(cell_agreement=0.8), {}), + pfs_cli, "_golden_run", lambda data, method, **kw: (SCORES, {}) ) monkeypatch.setattr(pfs_cli, "_model_name", lambda large=False: "qwen-x") from pfs.stance import EventChoice @@ -2219,11 +2220,13 @@ class TestStance: for k in ("K1", "K2", "K3") ], ) # fmt: skip + ev = json.dumps({"cell": 0.83, "recall": 0.556, "in_sample": True}) write_reaction( con._real, "CCM", [ - ReactionRow("CCM", "D1", "docket", 2023, 5, 20, 0, 0, 0, 0, "[]", '{"CCM:2023:other": {}}') + ReactionRow("CCM", "D1", "docket", 2023, 5, 20, 0, 0, 0, 0, "[]", '{"CCM:2023:other": {}}', + stance_method="stance-v4:qwen2.5:32b", stance_agreement=0.83, stance_eval=ev), ], ) # fmt: skip write_stances( @@ -2243,6 +2246,14 @@ class TestStance: assert (rows["D2"].n_classified, rows["D2"].n_commenters) == (None, None) assert list(json.loads(rows["D1"].events_json)) == ["CCM:2023:other"] assert rows["D2"].stance_modify == 0 and rows["D2"].events_json == "{}" + # R66a: a reaction rerun keeps the stored row's stance method record + d1 = rows["D1"] + assert (d1.stance_method, d1.stance_agreement, d1.stance_eval) == ( + "stance-v4:qwen2.5:32b", + 0.83, + ev, + ) + assert (rows["D2"].stance_method, rows["D2"].stance_eval) == (None, None) def test_stance_sample_refused_when_stances_stored(self, con, monkeypatch): from pfs.codetables import StanceRow, write_stances @@ -2386,6 +2397,7 @@ class TestStanceEval: assert "cells 4: agreement 0.750 (3)" in res.output assert "presence 1.000 (4)" in res.output assert "firm accuracy 0.500 (1/2); gate FAIL" in res.output + assert "recall 1.000 (2/2); direction 0.500 (1/2)" in res.output assert "confusion (rows golden, columns model)" in res.output assert "disagree K2 G0318: golden oppose, model support" in res.output @@ -2421,12 +2433,38 @@ class TestStanceMethodRecord: got = {r.period: r for r in read_reaction(con._real, "HOME-VISITS")} d1 = got["D1"] assert d1.stance_method == f"{pfs_cli.PROMPT_VERSIONS['multi']}:qwen-x" - assert d1.stance_agreement == 0.8 - assert got["R1"].stance_method is None and got["R1"].stance_agreement is None + # R66a: the measurement is the stance_eval JSON; the single figure is gone + assert d1.stance_agreement is None + assert json.loads(d1.stance_eval) == { + "cell": 0.83, + "recall": 0.556, + "direction": 0.667, + "n_items": 20, + "n_positions": 27, + "family": "HOME-VISITS", + "docket": "CMS-2022-0113", + "in_sample": True, + } + assert got["R1"].stance_method is None and got["R1"].stance_eval is None assert seen["ask"] is ASK res = runner.invoke(app, t._args("--model", "huge")) assert res.exit_code != 0 and "--model must be instruct or large" in res.output + def test_eval_is_stamped_only_for_the_golden_family( + self, con, monkeypatch, tmp_path + ): + """A golden set of another family is never replayed nor stamped.""" + t = TestStance() + t._patch(con, monkeypatch) + monkeypatch.setattr(pfs_cli, "_golden_run", lambda *a, **k: 1 / 0) + g = tmp_path / "g.json" + g.write_text(json.dumps({"family": "CCM", "docket": "D9", "items": []})) + res = runner.invoke(app, t._args("--golden", str(g), "--write")) + assert res.exit_code == 0, res.output + d1 = {r.period: r for r in read_reaction(con._real, "HOME-VISITS")}["D1"] + assert d1.stance_method and d1.stance_eval is None + assert d1.stance_agreement is None + def test_large_model_builds_the_large_asker(self, con, monkeypatch): t = TestStance() t._patch(con, monkeypatch) @@ -2439,22 +2477,21 @@ class TestStanceMethodRecord: runner.invoke(app, t._args()) # the default model is the large one assert calls == [True, False, pfs_cli.DEFAULT_MODEL == "large"] - def test_agreement_is_none_without_golden_or_with_pending( - self, monkeypatch, tmp_path - ): - assert pfs_cli._method_agreement(tmp_path / "none.json", "multi", "m") is None + def test_eval_is_none_without_golden_or_with_pending(self, monkeypatch, tmp_path): + F = "HOME-VISITS" + assert pfs_cli._method_eval(tmp_path / "none.json", "multi", "m", F) is None g = tmp_path / "g.json" - g.write_text("{}") + g.write_text(json.dumps({"family": F, "docket": "D1", "items": [{}]})) monkeypatch.setattr( pfs_cli, "_golden_run", lambda data, method, **kw: (None, {}) ) - assert pfs_cli._method_agreement(g, "multi", "m") is None + assert pfs_cli._method_eval(g, "multi", "m", F) is None monkeypatch.setattr( - pfs_cli, - "_golden_run", - lambda data, method, **kw: (SimpleNamespace(cell_agreement=0.5), {}), + pfs_cli, "_golden_run", lambda data, method, **kw: (SCORES, {}) ) - assert pfs_cli._method_agreement(g, "multi", "m") == 0.5 + got = json.loads(pfs_cli._method_eval(g, "multi", "m", F)) + assert (got["recall"], got["direction"], got["n_items"]) == (0.556, 0.667, 1) + assert pfs_cli._method_eval(g, "multi", "m", "CCM") is None def test_model_of_and_slow_post(self, monkeypatch): import httpx diff --git a/tests/llm/test_structured.py b/tests/llm/test_structured.py index ca7caa6..a85f387 100644 --- a/tests/llm/test_structured.py +++ b/tests/llm/test_structured.py @@ -2,6 +2,8 @@ from __future__ import annotations +import logging + import httpx import pytest @@ -72,6 +74,25 @@ def test_extractor_posts_schema_and_fixed_model(): assert [m["role"] for m in body["messages"]] == ["system", "user"] +def test_extractor_logs_ollama_load_duration_at_debug(caplog): + """R68: Ollama's load_duration (ns) is logged at DEBUG, so model reloads + are measurable; a reply without it logs no load line.""" + reply = {**_reply('{"ok": true}'), "load_duration": 2_500_000_000} + ask = json_extractor(Cfg(), Pool(["http://rig"]), post=lambda u, json=None: reply) + with caplog.at_level(logging.DEBUG, logger="llm.structured"): + assert ask("s", "u", SCHEMA) == {"ok": True} + loads = [r for r in caplog.records if "load_duration" in r.getMessage()] + assert len(loads) == 1 and loads[0].levelno == logging.DEBUG + assert loads[0].getMessage() == "extract: small on http://rig load_duration 2.50s" + caplog.clear() + bare = json_extractor( + Cfg(), Pool(["h"]), post=lambda u, json=None: _reply('{"ok": true}') + ) + with caplog.at_level(logging.DEBUG, logger="llm.structured"): + assert bare("s", "u", SCHEMA) == {"ok": True} + assert not [r for r in caplog.records if "load_duration" in r.getMessage()] + + def test_extractor_default_post_is_classifys(): from llm.classify import _default_post diff --git a/tests/pfs/test_reaction.py b/tests/pfs/test_reaction.py index 681af0b..367133e 100644 --- a/tests/pfs/test_reaction.py +++ b/tests/pfs/test_reaction.py @@ -2,6 +2,7 @@ from __future__ import annotations +import dataclasses import json from unittest.mock import MagicMock @@ -459,6 +460,40 @@ class TestEventsJson: None, None, ) + assert got.stance_eval is None + + def test_stance_eval_roundtrips_and_an_older_table_gains_it(self): + """R66a: ``stance_eval`` (JSON) is a trailing column; a table from + round 3 (``stance_agreement`` last) keeps its rows readable and gains + the column in place.""" + c = duckdb.connect(":memory:") + c.execute("CREATE SCHEMA pfs") + c.execute( + "CREATE TABLE pfs.code_reaction (family VARCHAR, period VARCHAR, " + "period_kind VARCHAR, year INTEGER, n_items INTEGER, n_total INTEGER, " + "stance_support INTEGER, stance_oppose INTEGER, stance_modify INTEGER, " + "stance_unclear INTEGER, sample_json VARCHAR, events_json VARCHAR, " + "n_classified INTEGER, n_commenters INTEGER, stance_method VARCHAR, " + "stance_agreement DOUBLE)" + ) + c.execute( + "INSERT INTO pfs.code_reaction VALUES ('A','D','docket',1,1,1,0,0,0,0," + "'[]','{}',3,4,'stance-v4:m',0.83)" + ) + old = read_reaction(c, "A")[0] + assert (old.stance_method, old.stance_agreement, old.stance_eval) == ( + "stance-v4:m", + 0.83, + None, + ) + ensure_tables(c) + assert read_reaction(c, "A")[0] == old + ev = json.dumps({"cell": 0.83, "recall": 0.556}) + row = dataclasses.replace( + old, period="D2", stance_agreement=None, stance_eval=ev + ) + write_reaction(c, "A", [old, row]) + assert read_reaction(c, "A") == [old, row] class TestFrPairSpansRefactor: diff --git a/tests/pfs/test_stance.py b/tests/pfs/test_stance.py index fec02f9..ad379c3 100644 --- a/tests/pfs/test_stance.py +++ b/tests/pfs/test_stance.py @@ -5,6 +5,7 @@ from __future__ import annotations import dataclasses import json import re +from pathlib import Path import duckdb import pytest @@ -29,6 +30,7 @@ from pfs.curated import load_curated from pfs.runlog import RunLog from pfs.stance import ( DEFAULT_METHOD, + GOLDEN_PATH, PROMPT_VERSIONS, STANCE_PROMPT_VERSION, EventChoice, @@ -873,11 +875,16 @@ def test_per_event_limit_and_partial_item_is_pending(tmp_path, events): # ── scoring against a golden set ───────────────────────────────────── -GOLDEN_PATH = ( - __import__("pathlib").Path(__file__).parent - / "fixtures" - / "stance_golden_cy2023.json" -) + +def test_golden_set_is_packaged_with_pfs(): + """R67: production code reads the golden set from the pfs package, never + from tests/.""" + import pfs + + assert GOLDEN_PATH.is_file() + assert GOLDEN_PATH.parent == Path(pfs.__file__).resolve().parent / "golden" + assert GOLDEN_PATH.name == "stance_home_visits_cy2023.json" + assert "tests" not in GOLDEN_PATH.parts def test_golden_fixture_shape(): @@ -951,6 +958,9 @@ def test_score_counts_and_severity(): ("e4", 1), ] assert sc.confusion["oppose"]["none"] == 1 + # R66a: golden positions e1 e2 e4 e5 e6; found e1 e2 e4 e5; right e2 + assert (sc.positions, sc.found, sc.right) == (5, 4, 1) + assert (sc.recall, sc.direction) == (0.8, 0.25) from pfs.stance import Scores empty = Scores(0, 0, 0, 0, 0, {}, ()) @@ -959,6 +969,46 @@ def test_score_counts_and_severity(): 0, 0, ) + assert (empty.recall, empty.direction) == (0, 0) + + +def test_eval_record_is_the_r66a_json(): + from pfs.stance import eval_record, golden_labels, score + + golden = json.loads(GOLDEN_PATH.read_text()) + labels = golden_labels(golden) + firm = sorted(k for k, v in labels.items() if v != "none") + # find 3 of the hand-coded positions, 2 in the right direction, and add one + # position on a golden `none` cell + wrong = "oppose" if labels[firm[2]] != "oppose" else "support" + none_cell = next(k for k, v in sorted(labels.items()) if v == "none") + rows = [ + S(*firm[0], labels[firm[0]]), + S(*firm[1], labels[firm[1]]), + S(*firm[2], wrong), + S(*none_cell, "support"), + ] + sc = score(golden, rows) + rec = eval_record(golden, sc) + assert list(rec) == [ + "cell", + "recall", + "direction", + "n_items", + "n_positions", + "family", + "docket", + "in_sample", + ] + n = len(firm) + assert rec["recall"] == round(3 / n, 3) and rec["direction"] == round(2 / 3, 3) + assert rec["cell"] == round(sc.cell_agreement, 3) + assert (rec["n_items"], rec["n_positions"]) == (len(golden["items"]), n) + assert (rec["family"], rec["docket"]) == ("HOME-VISITS", "CMS-2022-0113") + assert rec["in_sample"] is True + # a held-out set says so in its own file + held = eval_record({**golden, "in_sample": False}, sc) + assert held["in_sample"] is False # ── relevance filter (R65) ─────────────────────────────────────────── @@ -974,22 +1024,72 @@ def test_event_subjects_by_slug(): sub = event_subjects(FAM, SUBJ_EVENTS) v, c, t, g, o = (sub[e.event_key] for e in SUBJ_EVENTS) - assert v.named_in("the work RVU is low") and v.named_in("see 99341-99343") + # R67: the event's subject keyword AND this family; valuation needs a code + assert v.named_in("the work RVU of 99342 is low") + assert v.named_in("payment for 99341-99343 falls") # a range names 99342 + assert not v.named_in("the work RVU is low") # no family code + assert not v.named_in("see 99341-99343") # no subject keyword + assert not v.named_in("home visit codes are undervalued") # a phrase, no code assert not v.named_in("nothing relevant at all") - assert c.named_in("codes were deleted") and c.named_in("domiciliary care") - assert c.named_in("uses medical decision making") and c.named_in("99339-99341") - assert t.named_in("audio-only visits") and not t.named_in("99341 and RVUs") - assert g.named_in("the new GXXX3 code") and g.named_in("prolonged visits") - assert g.named_in("G0318 is new") and not g.named_in("telehealth only") + assert c.named_in("codes 99339-99341 were deleted") + assert c.named_in("domiciliary care visit codes merge") # keyword and phrase + assert c.named_in("MDM for home or residence visits") and not c.named_in("MDM") + assert not c.named_in("codes were deleted") and not c.named_in("see 99342") + assert c.named_in("GXXX3 time counts") # the link alias names the family + assert t.named_in("audio-only home visit codes") and t.named_in("telehealth 99341") + assert t.named_in("telehealth for GXXX3") + assert not t.named_in("we support telehealth after the PHE") + assert not t.named_in("99341 and RVUs") + assert g.named_in("the new GXXX3 code") and g.named_in("G0318 is new") + assert g.named_in("prolonged home or residence visit time") + assert not g.named_in("prolonged visits") and not g.named_in("telehealth only") + assert not g.named_in("prolonged 99417 for office visits") assert o.named_in("code 99342 is fine") and o.named_in("a home visit e/m change") - assert not o.named_in("telehealth and RVUs") - # no pair phrases and an unknown link: still well-formed + assert not o.named_in("telehealth and RVUs") and not o.named_in("G0318 only") + # no pair phrases and an unknown link: still well-formed; the link event's + # own code names the family bare = dataclasses.replace(FAM, pair_phrases=()) other = event_subjects( bare, [SUBJ_EVENTS[4], EventChoice("F:2023:G9999", "x", 2023)] ) assert not other["HOME-VISITS:2023:other"].named_in("home visit") - assert other["F:2023:G9999"].named_in("prolonged") + assert other["F:2023:G9999"].named_in("prolonged G9999") + assert not other["F:2023:G9999"].named_in("prolonged") + # a link not offered that year does not name the family + no_link = event_subjects(FAM, SUBJ_EVENTS[2:3]) + assert not no_link["HOME-VISITS:2023:telehealth"].named_in("telehealth G0318") + + +def test_window_must_name_this_family_and_the_subject(): + """R67, from the CY2023 live rows: a general telehealth endorsement and a + valuation complaint that names only "home visit codes" are dropped; a + telehealth ask about the family's codes one sentence away is kept.""" + from pfs.stance import event_subjects + + sub = event_subjects(FAM, SUBJ_EVENTS) + val, tele = SUBJ_EVENTS[0].event_key, SUBJ_EVENTS[2].event_key + general = ( + "This is a significant hit to our ability to provide primary care. " + "We support CMS efforts to expand telehealth services and encourage these " + "efforts continue after the end of the PHE. Many patients can now get care." + ) + q = "We support CMS efforts to expand telehealth services and encourage these" + assert _rows([pos(tele, "support", "r", q)], general, sub) == [] + phrase_only = ( + "We support the nomination of misvalued home visit codes. By combining these " + "two code sets and not increasing the value of the home visit codes our " + "practices would incur a seven percent decrease in revenue next year." + ) + q2 = "By combining these two code sets and not increasing the value" + assert _rows([pos(val, "oppose", "r", q2)], phrase_only, sub) == [] + ask = ( + "Telehealth flexibilities matter to our patients. We recommend that all " + "codes listed above be retained through the end of the year. This includes " + "the home visit codes 99347-99350 for established patients." + ) + q3 = "We recommend that all codes listed above be retained through the end" + kept = _rows([pos(tele, "modify", "r", q3)], ask, sub) + assert [(r.event_key, r.stance) for r in kept] == [(tele, "modify")] def test_holding_sentence(): @@ -1025,9 +1125,10 @@ def test_position_kept_only_if_quote_or_neighbours_name_the_subject(): sub = event_subjects(FAM, SUBJ_EVENTS) tele, val = SUBJ_EVENTS[2].event_key, SUBJ_EVENTS[0].event_key text = ( - "Telehealth flexibilities matter to our patients. We recommend that all codes " - "listed above should be retained through the end of the year. Unrelated closing " - "words about something else entirely follow in this sentence here." + "Telehealth flexibilities for home visit codes matter to our patients. We " + "recommend that all codes listed above should be retained through the end of " + "the year. Unrelated closing words about something else entirely follow in " + "this sentence here." ) q = "We recommend that all codes listed above should be retained through the end" kept = _rows([pos(tele, "modify", "r", q)], text, sub)