From f9cfdef0b207b823916fefabe1b5c34496860f98 Mon Sep 17 00:00:00 2001 From: kert Date: Thu, 12 Nov 2026 17:53:14 -0500 Subject: [PATCH] fix(pfs): grounded guideline paraphrases (guideline-v2), CPT Changes-dated deletions, invented test fixtures, stance-v6 text rules, MOD1/MOD2 aliases, multi-docket golden eval The grounding guard checks each paraphrase against its numbered sentence (content-word coverage, unseen words, code ranges per action clause, NFKC copy guard). Deletions cite CPT Changes 2023 pages; the Panel row is built from the entry's code lists. Pair phrase: drop the bare 'home and residence'. stance-v6 is not shipped (golden recall 11/27 vs v5 14/27); v5 stays the default. --- src/cli/pfs.py | 118 ++++++----- src/pfs/codetables.py | 2 +- src/pfs/curated.yaml | 6 +- src/pfs/guideline_facts.py | 391 +++++++++++++++++++++++++++++++------ src/pfs/stance.py | 97 +++++++-- tests/cli/test_pfs_cli.py | 79 ++++++++ tests/pfs/test_curated.py | 11 +- 7 files changed, 581 insertions(+), 123 deletions(-) diff --git a/src/cli/pfs.py b/src/cli/pfs.py index 9b13756..5e5b24c 100644 --- a/src/cli/pfs.py +++ b/src/cli/pfs.py @@ -94,7 +94,10 @@ from pfs.stance import ( docket_rule_year, eval_record, event_subjects, + fixed_text, + golden_dockets, guideline_text, + item_dockets, prompt_version, rollup, score, @@ -1523,7 +1526,12 @@ def stance( con = _read() try: events = docket_events( - con, fam, year, store, guidelines=guideline_text(generation) + con, + fam, + year, + store, + guidelines=guideline_text(generation), + fixed=fixed_text(generation), ) commenter_rows = [ c for c in read_commenters(con, family=fam.key) if c.docket == docket @@ -1663,56 +1671,70 @@ def _golden_run( fam = curated_family(normalize_key(data["family"])) if fam is None: raise typer.BadParameter(f"unknown curated family {data['family']!r}") - docket = data["docket"] store = _store() - found = [d for d in store.dockets() if d.id == docket] - if not found: - raise typer.BadParameter(f"unknown docket {docket!r}") - con = _read() - try: - events = docket_events( - con, - fam, - docket_rule_year(found[0].comment_end_date), - store, - guidelines=guideline_text(generation), - ) - finally: - con.close() - keys = [it["item_key"] for it in data["items"]] - texts = { - k: t - for k, t in family_comment_items( - _engine(), - fam.codes, - fam.key, - fam.pair_phrases, + of_docket = item_dockets(data) + all_rows: list[Any] = [] + counts = { + "done": 0, "skipped": 0, "failed": 0, "failed_pending": 0, "pending": 0, + "no_text": 0, + } # fmt: skip + remaining = limit + for docket in golden_dockets(data): + found = [d for d in store.dockets() if d.id == docket] + if not found: + raise typer.BadParameter(f"unknown docket {docket!r}") + con = _read() + try: + events = docket_events( + con, + fam, + docket_rule_year(found[0].comment_end_date), + store, + guidelines=guideline_text(generation), + fixed=fixed_text(generation), + ) + finally: + con.close() + keys = [k for k, d in of_docket.items() if d == docket] + texts = { + k: t + for k, t in family_comment_items( + _engine(), + fam.codes, + fam.key, + fam.pair_phrases, + docket=docket, + max_chars=TEXT_CHARS, + max_chunks=TEXT_CHUNKS, + ) + if k in keys + } + # the call limit is shared across dockets; once spent, only replay + live = ask if (ask is not None and (not limit or remaining > 0)) else None + rows, got = stances_for( + [SimpleNamespace(item_key=k) for k in keys], + texts, + events, + live, + RunLog(ledger_path("stance", fam.key)), + family=fam.key, docket=docket, - max_chars=TEXT_CHARS, - max_chunks=TEXT_CHUNKS, + method=method, + generation=generation, + model=name if live is None else None, + subjects=( + event_subjects(fam, events, by_kind=guideline_text(generation)) + if relevance + else None + ), + limit=remaining, + retry_failed=retry_failed, + progress=progress, ) - if k in keys - } - rows, counts = stances_for( - [SimpleNamespace(item_key=k) for k in keys], - texts, - events, - ask, - RunLog(ledger_path("stance", fam.key)), - family=fam.key, - docket=docket, - method=method, - generation=generation, - model=name if ask is None else None, - subjects=( - event_subjects(fam, events, by_kind=guideline_text(generation)) - if relevance - else None - ), - limit=limit, - retry_failed=retry_failed, - progress=progress, - ) + remaining -= got["done"] + got["failed"] + all_rows += rows + counts = {k: counts[k] + got[k] for k in counts} + rows, counts = all_rows, counts incomplete = counts["pending"] or counts["no_text"] return (None if incomplete else score(data, rows)), counts diff --git a/src/pfs/codetables.py b/src/pfs/codetables.py index 7b04f40..f8b5256 100644 --- a/src/pfs/codetables.py +++ b/src/pfs/codetables.py @@ -484,7 +484,7 @@ class GuidelineFactRow: ``scope_point`` and ``panel_action``, a paraphrase of at most 240 characters; never book text. ``cite_item`` is the bib item of the book (or the Panel summary), ``cite_locator`` its printed page ("p. 28", - "tab 10, p. 4"), or the section path where the edition carries no page + "tab 10, PDF p. 4"), or the section path where the edition carries no page markers (R72). For ``scope_point``/``panel_action`` rows ``code`` is the code range the passage covers ("99341-99350").""" diff --git a/src/pfs/curated.yaml b/src/pfs/curated.yaml index 6a6d69f..5e56e8c 100644 --- a/src/pfs/curated.yaml +++ b/src/pfs/curated.yaml @@ -5,7 +5,7 @@ HOME-VISITS: synonyms: [home visit, home visits, home or residence, residence visit, domiciliary, rest home visit, house call, house calls] # Phrases that qualify a Comment:/Response: pair besides a code (pfs.cms_summaries). # Visit-naming only; each was checked against every phrase-only row it admits. - pair_phrases: [home or residence visit, home or residence services, home or residence e/m, residence visit, rest home visit, home visit codes, home visit e/m, house call, domiciliary care visit, visit in the home, home residence e/m, domiciliary care setting, domiciliary care settings, home visit pes, home or residence setting, home and residence e/m, home and residence] + pair_phrases: [home or residence visit, home or residence services, home or residence e/m, residence visit, rest home visit, home visit codes, home visit e/m, house call, domiciliary care visit, visit in the home, home residence e/m, domiciliary care setting, domiciliary care settings, home visit pes, home or residence setting, home and residence e/m] members: - {code: '99341', role: base, cite: {item_key: PETM225D, p_id: 1672}} - {code: '99342', role: base, cite: {item_key: PETM225D, p_id: 1672}} @@ -29,6 +29,6 @@ HOME-VISITS: - {code: '99340', role: predecessor, until: 2022, cite: {rvu_last_year: 2022}} links: - {kind: add-on, target: G2211, since: 2026, cite: {item_key: NHRGIHGD, p_id: 1469}} - - {kind: modifier, target: B1, since: 2027, cite: {item_key: JCWARTJD, p_id: 1168}} - - {kind: modifier, target: B2, since: 2027, cite: {item_key: JCWARTJD, p_id: 1205}} + - {kind: modifier, target: B1, since: 2027, aliases: [MOD1], alias_cite: {item_key: 5ITGVDJV, p_id: 521}, cite: {item_key: JCWARTJD, p_id: 1168}} + - {kind: modifier, target: B2, since: 2027, aliases: [MOD2], alias_cite: {item_key: 5ITGVDJV, p_id: 537}, cite: {item_key: JCWARTJD, p_id: 1205}} - {kind: prolonged-add-on, target: G0318, since: 2023, aliases: [GXXX3], alias_cite: {item_key: MZ24MX5S, p_id: 880}, cite: {item_key: PETM225D, p_id: 1694}} diff --git a/src/pfs/guideline_facts.py b/src/pfs/guideline_facts.py index c4a45e9..1dbe550 100644 --- a/src/pfs/guideline_facts.py +++ b/src/pfs/guideline_facts.py @@ -21,15 +21,19 @@ ledger stores only kept paraphrases, never book text. from __future__ import annotations import re -from collections.abc import Callable, Mapping, Sequence +import unicodedata +from collections.abc import Callable, Iterable, Mapping, Sequence from pathlib import Path from typing import Any from pfs.codetables import GuidelineFactRow, is_missing_table_error -from pfs.origin import resolve_book, shares_run +from pfs.lineage import _expand_ranges +from pfs.origin import _DEL_BEFORE, book_deleted, book_support, resolve_book, shares_run from pfs.runlog import RunLog, input_hash, ledger_path, run_units -GUIDELINE_PROMPT_VERSION = "guideline-v1" +GUIDELINE_PROMPT_VERSION = "guideline-v2" +CHANGES_ITEM = "3ANRT854" # "CPT Changes 2023": dates the deletions +CHANGES_YEAR = 2023 PANEL_ITEM = "F7SRZJD2" # "February 2021: CPT Editorial Summary of Panel Actions" PANEL_TITLES = ("Home and Residence Services",) MAX_POINTS = 6 @@ -44,39 +48,85 @@ SCHEMA: dict = { "type": "array", "items": { "type": "object", - "required": ["summary"], - "properties": {"summary": {"type": "string", "maxLength": 240}}, + "required": ["sentence", "summary"], + "properties": { + "sentence": {"type": "integer"}, + "summary": {"type": "string", "maxLength": 240}, + }, }, } }, } SYSTEM = ( - "You read a passage of licensed coding-guideline text and state, in your own " - "words, what it says about where and for whom the listed codes are reported, " - "what counts toward their time, and any reporting rule it gives. Reply with " - 'JSON {"points": [{"summary": ...}]}: at most four points, each one sentence ' - "of at most 200 characters. Paraphrase: never copy eight or more consecutive " - "words of the passage, never quote, and never state a number the passage does " - "not contain. Say nothing the passage does not say, and never describe what " - "the passage lacks." + "You read numbered sentences of licensed coding-guideline text and state, in " + "your own words, what they say about where and for whom the listed codes are " + "reported and under what conditions. Reply with JSON " + '{"points": [{"sentence": k, "summary": ...}]}: at most four points; each ' + "paraphrases ONE numbered sentence k, in one sentence of at most 200 " + "characters, and keeps that sentence's qualifiers and exceptions. Never copy " + "eight or more consecutive words, never quote, never state a number or a code " + "the sentence does not contain, and name a code only if that sentence names it. " + "Reuse the passage's own nouns, verbs and qualifiers and change only the " + "sentence structure (split or merge clauses, start with a different word, or turn a negative into a plain verb such as 'exclude'); use " + "no word the passage does not support. Say nothing the " + "sentence does not say, and never describe what the passage lacks." ) PANEL_SYSTEM = ( - "You read one entry of a CPT Editorial Panel summary of actions and state in " - "your own words what the Panel did to the listed codes and subsections and " - 'when it takes effect. Reply with JSON {"points": [{"summary": ...}]}: one ' - "point of at most 200 characters. Paraphrase: never copy eight or more " - "consecutive words, never quote, never state a number the entry does not " - "contain." + "You read the numbered sentences of one entry of a CPT Editorial Panel summary " + "of actions and state in your own words what the Panel did to the listed codes " + "and subsections and when it takes effect. Reply with JSON " + '{"points": [{"sentence": k, "summary": ...}]}: one point of at most 200 ' + "characters. Keep each action with the codes it applies to (a deletion and a " + "revision name different codes), list codes exactly as the entry lists them and " + "write a range only if the entry does. Never copy eight or more consecutive words, " + "never quote, never state a number or code the entry does not contain." ) _MARKS = re.compile("[▶◀■➢▲★●◆►◄▼\x00]") _WS = re.compile(r"\s+") _SENTENCE = re.compile(r"(?<=[.!?])\s+") _NUMBER = re.compile(r"\d+") -#: a point about what the passage lacks says nothing +#: a point about what the passage lacks says nothing; a real point that +#: starts with "No ..." ("No travel time is counted") stays _VOID = re.compile( - r"^(?:no|none|nothing)\b|\bnot (?:given|specified|stated|provided)\b", re.I + r"\b(?:passage|text|entry|sentences?)\b[^.]{0,40}\b(?:does not|doesn't|" + r"do not|no mention|says nothing|lacks)\b|" + r"^(?:no|none|nothing)\b[^.]{0,40}\b(?:information|details|rules|locations)\b" + r"[^.]{0,40}\b(?:given|provided|stated|specified|mentioned)\b", + re.I, +) +_STOP = frozenset( + "a an the and or of to in on at by for from with without as is are was were be " + "been being it its this that these those their there than then also into over " + "under per via not no nor but if when while which who whom whose what how any " + "all each other such can may must should will would could only both either " + "has have had do does did given like during including such same between " + "same".split() +) +#: generic words a paraphrase may use that the passage need not contain +_GENERIC = frozenset( + "apply applies use used report reported include includes cover covers covered " + "describe describes state states refer refers define defines require " + "requires provide provides indicate indicates mean means specify specifies " + "allow allows exclude excludes rule rules code codes service services " + "patient patients visit visits effective effect takes determine determines " + "determining select selects selecting choose chooses count counts omit omits " + "ignore ignores consider considers move moves".split() +) +_TOKEN = re.compile(r"[a-z]{3,}") +_GROUNDED = 0.6 +#: a scope point that states a reporting rule is the more useful one +_RULE_POINT = re.compile( + r"\b(?:time|travel|report|admitted)\b|\bdo not\b|\bnot use\b", re.I +) +_ACTIONS = ( + ("deleted", re.compile(r"\b(?:delet\w*|remov\w*|retir\w*|eliminat\w*)", re.I)), + ("revised", re.compile(r"\b(?:revis\w*|rewr\w*|chang\w*|updat\w*|amend\w*)", re.I)), + ( + "created", + re.compile(r"\b(?:add\w*|creat\w*|introduc\w*|establish\w*|new)\b", re.I), + ), ) _YEAR = re.compile(r"\b(20\d{2})\b") _CODE5 = re.compile(r"(? list[str]: - """The model's paraphrases that pass the guards: a string of 1..240 - characters, no 8-word run shared with *text*, no number *text* lacks; - duplicates dropped.""" +def _fold(text: str) -> str: + return unicodedata.normalize("NFKC", text).replace("\u2019", "'") + + +def _stem(word: str) -> str: + if word.endswith("ies") and len(word) > 5: + return word[:-3] + "y" + for suf in ("ions", "ion", "ing", "ed", "es", "s", "ly"): + if word.endswith(suf) and len(word) - len(suf) >= 4: + word = word[: -len(suf)] + break + return word[:-1] if word.endswith("e") and len(word) > 4 else word + + +def _content(text: str) -> list[str]: + """Stemmed content words of *text*: stopwords and short tokens removed.""" + return [_stem(w) for w in _TOKEN.findall(_fold(text).lower()) if w not in _STOP] + + +def sentences_of(text: str) -> list[str]: + return [s for s in _SENTENCE.split(norm(text)) if s] + + +def _clauses(text: str) -> list[str]: + return [c for c in re.split(r"[;.]", text) if c.strip()] + + +def _action(clause: str) -> str | None: + return next((a for a, pat in _ACTIONS if pat.search(clause)), None) + + +def _codes_ok(summary: str, window: str) -> bool: + """Every code the summary's clause names (ranges expanded) appears in a + source clause with the same action; a clause with no action word needs the + code anywhere in the window.""" + source = [ + (_action(c), set(_CODE5.findall(_expand_ranges(c)))) for c in _clauses(window) + ] + anywhere = set().union(*(codes for _, codes in source)) if source else set() + for clause in _clauses(summary): + action = _action(clause) + for code in set(_CODE5.findall(_expand_ranges(clause))): + pool = ( + set().union(*(c for a, c in source if a == action)) + if action + else anywhere + ) + if code not in pool: + return False + return True + + +def grounded(summary: str, k: int, sentences: Sequence[str]) -> bool: + """The paraphrase of numbered sentence *k* (1-based) is grounded: at least + 60% of its content words occur in sentences k-1..k+1, none occurs nowhere + in the passage (generic paraphrase words aside), and each code it names + sits in a source clause with the same action.""" + if not 1 <= k <= len(sentences): + return False + window = " ".join(sentences[max(0, k - 2) : k + 1]) + words = _content(summary) + near = set(_content(window)) + anywhere = set(_content(" ".join(sentences))) | {_stem(g) for g in _GENERIC} + if not words or any(w not in anywhere for w in words): + return False + if sum(w in near or w in _GENERIC_STEMS for w in words) / len(words) < _GROUNDED: + return False + return _codes_ok(summary, window) + + +_GENERIC_STEMS = frozenset(_stem(g) for g in _GENERIC) + + +def guarded( + out: Any, sentences: Sequence[str], guide: str = "" +) -> list[tuple[int, str]]: + """``(sentence, paraphrase)`` of the model's paraphrases that pass the guards: a string of 1..240 + characters naming a valid sentence, no 8-word run shared with *guide* (or + the sentences; NFKC-folded), no number the sentences lack, no empty + "the passage says nothing" point, grounded in its sentence + (:func:`grounded`); duplicates dropped.""" items = out.get("points") if isinstance(out, dict) else None if not isinstance(items, list): return [] - have = set(_NUMBER.findall(text)) - kept: list[str] = [] + source = _fold(" ".join([*sentences, guide])) + have = set(_NUMBER.findall(source)) + kept: list[tuple[int, str]] = [] for it in items: - s = it.get("summary") if isinstance(it, dict) else None - if not isinstance(s, str): + if not isinstance(it, dict): + continue + s, k = it.get("summary"), it.get("sentence") + if not isinstance(s, str) or isinstance(k, bool) or not isinstance(k, int): continue s = s.strip() if ( s and len(s) <= SUMMARY_CHARS - and not shares_run(s, text) + and not shares_run(_fold(s), source) and set(_NUMBER.findall(s)) <= have and not _VOID.search(s) - and s not in kept + and grounded(s, k, sentences) + and s not in [t for _, t in kept] ): - kept.append(s) + kept.append((k, s)) return kept +def guarded_points(out: Any, sentences: Sequence[str], guide: str = "") -> list[str]: + """The paraphrases of :func:`guarded`, without their sentence numbers.""" + return [s for _, s in guarded(out, sentences, guide)] + + def panel_entry(pages: Sequence[str], title: str) -> tuple[str, int, str] | None: """``(tab, page number, entry text)`` of the Panel-summary entry named *title*: the tab number precedes the name, the entry runs to the next @@ -303,8 +439,33 @@ def _where(path: tuple) -> str: return " > ".join(path) +def _change_page( + changes: Sequence[tuple[str, str]], code: str, deleted: frozenset[str] +) -> str | None: + """First printed page of CPT Changes where it says *code* was deleted.""" + for label, text in changes: + if label and "deleted" in book_support(text, code, deleted): + return label + return None + + +def _note_page(pages: Sequence[tuple[str, str]], code: str) -> str | None: + """Printed page whose "(... have been deleted ...)" note names *code*.""" + for label, text in pages: + if label and any( + code in _CODE5.findall(_expand_ranges(m.group(1))) + for m in _DEL_BEFORE.finditer(text) + ): + return label + return None + + def _facts( - family: str, rows: list[dict], pages: Mapping[int, list], counts: dict[str, int] + family: str, + rows: list[dict], + pages: Mapping[int, list], + counts: dict[str, int], + changes: Sequence[tuple[str, str]] = (), ) -> list[GuidelineFactRow]: out: list[GuidelineFactRow] = [] @@ -338,15 +499,28 @@ def _facts( for r in rows: by_year.setdefault(r["year"], {})[r["code"]] = r years = sorted(by_year) + deleted = book_deleted(changes) for prev, year in zip(years, years[1:], strict=False): here = by_year[year] anchor = here[sorted(here)[0]] - where = locate(anchor) for code in sorted(set(by_year[prev]) - set(here)): + # CPT Changes dates the deletion; the later book's own deletion + # note is the fallback for a code Changes does not name + page = ( + _change_page(changes, code, deleted) + if prev < CHANGES_YEAR <= year + else None + ) + if page is not None: + when, item, where = CHANGES_YEAR, CHANGES_ITEM, f"p. {page}" + else: + note = _note_page(pages[year], code) + when, item = year, anchor["item"] + where = f"p. {note}" if note else locate(anchor) out.append( GuidelineFactRow( - family, code, year, "deleted_in_edition", str(prev), - anchor["item"], where, BOOK_EXTRACTOR, + family, code, when, "deleted_in_edition", str(prev), item, + where, BOOK_EXTRACTOR, ) ) # fmt: skip return out @@ -398,14 +572,14 @@ def _scope_jobs( "edition": year, "item": members[0]["item"], "code": _range(codes), - "text": " ".join(group), + "sentences": group, "guide": norm(guide), "system": SYSTEM, "locator": f"p. {label}" if label else _where(path), "user": ( f"CODES: {', '.join(codes)}\nSOURCE: CPT {year} guideline, " f"{path[-1]}{f', page {label}' if label else ''}\n\n" - f"PASSAGE:\n{' '.join(group)}" + f"SENTENCES:\n{_numbered(group)}" ), } ) @@ -432,21 +606,83 @@ def _panel_jobs( "edition": int(years[-1]) if years else 0, "item": PANEL_ITEM, "code": _range(named), - "text": text, + "codes": named, + "sentences": sentences_of(text), "guide": text, "system": PANEL_SYSTEM, - "locator": f"tab {tab}, p. {page}", - "user": f"CODES: {', '.join(named)}\nSOURCE: Panel summary, tab {tab}\n\nENTRY:\n{text}", + "locator": f"tab {tab}, PDF p. {page}", + "user": ( + f"CODES: {', '.join(named)}\nSOURCE: Panel summary, tab {tab}\n\n" + f"SENTENCES:\n{_numbered(sentences_of(text))}" + ), } ) return jobs +def _numbered(sentences: Sequence[str]) -> str: + return "\n".join(f"[{i}] {s}" for i, s in enumerate(sentences, 1)) + + +#: model replies tried per unit: the guards drop most of a reply, so a unit +#: that kept nothing (or little) is asked again, up to this many times +ATTEMPTS = 6 +ENOUGH = 4 + + def _ask_job(ask: Callable, job: dict) -> dict | None: - out = ask(job["system"], job["user"], SCHEMA) - if out is None: + """Ask for the unit's paraphrases (up to ``ATTEMPTS`` replies) and keep + only those that pass the guards, so the ledger never holds copied or + ungrounded text. ``None`` when every reply failed.""" + kept: list[tuple[int, str]] = [] + answered = False + for _ in range(ATTEMPTS): + out = ask(job["system"], job["user"], SCHEMA) + if out is None: + continue + answered = True + for k, s in guarded(out, job["sentences"], job["guide"]): + if s not in [t for _, t in kept]: + kept.append((k, s)) + if len(kept) >= ENOUGH: + break + if not answered: return None - return {"points": [{"summary": s} for s in guarded_points(out, job["guide"])]} + return {"points": [{"sentence": k, "summary": s} for k, s in kept]} + + +_LIST = r"((?:\d{5}(?:\s*[-\u2013]\s*\d{5})?|[,\s]|and)+)" +_DELETION = re.compile(rf"deletion of (?:codes? )?{_LIST}", re.I) +_REVISION = re.compile(rf"revision of (?:codes? )?{_LIST}", re.I) + + +def panel_fallback(text: str, named: Sequence[str]) -> str | None: + """A deterministic one-line statement of a Panel entry from its code lists + (values, not prose): the deleted codes, the revised codes and the + effective year, each code set to the family's codes the entry names. + ``None`` when the entry lists neither.""" + from pfs.stance import _compact # noqa: PLC0415 - stance imports this module + + parts = [] + for label, pat in (("deleted", _DELETION), ("revised", _REVISION)): + listed: set[str] = set() + for m in pat.finditer(text): + listed |= set(_CODE5.findall(_expand_ranges(m.group(1)))) + codes = sorted(listed & set(named)) + if codes: + parts.append(f"{label} codes {_compact(codes).replace(' and ', ', ')}") + if not parts: + return None + years = _YEAR.findall(text) + if re.search(r"subsections?|sections?", text, re.I): + parts.append( + "the matching subsection headings and guidelines were also deleted or revised" + ) + if years: + parts.append(f"effective {years[-1]}") + out = "; ".join(parts) + out = out[0].upper() + out[1:] + "." + return out if len(out) <= SUMMARY_CHARS else None def _interleave(groups: list[list[GuidelineFactRow]]) -> list[GuidelineFactRow]: @@ -484,7 +720,8 @@ def guideline_facts( for y in years } labelled = {y: p or [] for y, p in pages.items()} - facts = _facts(family.key, rows, labelled, run) + changes = (load_pages(store, CHANGES_ITEM) or []) if len(years) > 1 else [] + facts = _facts(family.key, rows, labelled, run, changes) model = ask.model if ask is not None else replay_model if model is None: return facts @@ -515,9 +752,16 @@ def guideline_facts( "panel_action" if j["unit"].startswith("panel:") else "scope_point", s, j["item"], j["locator"], extractor, ) - for s in guarded_points(entry.output, j["guide"]) + for s in guarded_points(entry.output, j["sentences"], j["guide"]) ] # fmt: skip if j["unit"].startswith("panel:"): + if not mine and (alt := panel_fallback(j["guide"], j["codes"])): + mine = [ + GuidelineFactRow( + family.key, j["code"], j["edition"], "panel_action", alt, + j["item"], j["locator"], "cpt-panel", + ) + ] # fmt: skip facts += mine[:1] else: scope.setdefault(j["edition"], []).append(mine) @@ -529,6 +773,16 @@ def guideline_facts( # ── the stance coding text ─────────────────────────────────────────── +def _pages(locators: Iterable[str]) -> str: + """ "p. 9", "pp. 37-40" (a range once there are three or more pages), or the + locators joined when they are not printed pages.""" + locs = sorted(set(locators), key=lambda x: (len(x), x)) + nums = [int(m.group(1)) for x in locs if (m := re.fullmatch(r"p\. (\d+)", x))] + if len(nums) != len(locs) or len(nums) < 3: + return ", ".join(locs) + return f"pp. {min(nums)}\u2013{max(nums)}" + + def cpt_change_text( facts: Sequence[GuidelineFactRow], year: int, base_codes: Sequence[str] ) -> str: @@ -559,13 +813,21 @@ def cpt_change_text( new_b, old_b = of(eff, "selection_basis"), of(prior, "selection_basis") new_t = {**of(eff, "typical_time_min"), **of(eff, "time_threshold_min")} old_t = {**of(prior, "typical_time_min"), **of(prior, "time_threshold_min")} - gone = sorted( + gone = [ r.code for r in facts if r.fact == "deleted_in_edition" and prior < r.edition_year <= eff - ) + ] if not gone and new_b == {c: old_b.get(c) for c in new_b} and new_t == old_t: return "" + + def pages_of(edition: int, fact: str) -> str: + return _pages( + r.cite_locator + for r in facts + if r.edition_year == edition and r.fact == fact + ) + parts = [] for value in sorted(set(new_b.values())): codes = sorted(c for c, v in new_b.items() if v == value) @@ -574,30 +836,43 @@ def cpt_change_text( ) text = ( f"{_compact(codes)} {'is' if len(codes) == 1 else 'are'} selected by " - f"{_BASIS_LABEL[value]}" + f"{_BASIS_LABEL[value]} (CPT {eff}, {pages_of(eff, 'selection_basis')})" ) if old: - text += f" (CPT {prior}: {' / '.join(old)})" + text += f" (CPT {prior}, {pages_of(prior, 'selection_basis')}: {' / '.join(old)})" parts.append(text) if new_t: - kind = ( - "total-time thresholds" + fact = ( + "time_threshold_min" if of(eff, "time_threshold_min") - else "typical times" + else "typical_time_min" + ) + kind = ( + "total-time thresholds" if fact == "time_threshold_min" else "typical times" ) parts.append( f"{kind} " + ", ".join(f"{c} {n}" for c, n in sorted(new_t.items())) - + " minutes" + + f" minutes (CPT {eff}, {pages_of(eff, fact)})" + ) + dropped = [ + r + for r in facts + if r.fact == "deleted_in_edition" and prior < r.edition_year <= eff + ] + for key in sorted({(r.edition_year, r.cite_item) for r in dropped}): + mine = [r for r in dropped if (r.edition_year, r.cite_item) == key] + book = f"CPT Changes {key[0]}" if key[1] == CHANGES_ITEM else f"CPT {key[0]}" + locs = _pages(r.cite_locator for r in mine) + parts.append( + f"{_compact([r.code for r in mine])} deleted in CPT {key[0]} ({book}, {locs})" ) - if gone: - parts.append(f"{_compact(gone)} not listed in the CPT {eff} book") text = f"CPT books on file, {prior} and {eff}: " + "; ".join(parts) base = set(base_codes) scope = sorted( (r for r in facts if r.fact == "scope_point" and r.edition_year == eff and r.code.split("-")[0] in base), - key=lambda r: (r.cite_locator, r.value), + key=lambda r: (not _RULE_POINT.search(r.value), r.cite_locator, r.value), )[:2] # fmt: skip if scope: text += ". Scope: " + "; ".join( diff --git a/src/pfs/stance.py b/src/pfs/stance.py index 6df04e7..6bfc4f3 100644 --- a/src/pfs/stance.py +++ b/src/pfs/stance.py @@ -43,9 +43,12 @@ TASK = "stance" METHODS = ("multi", "per-event") #: prompt generations: v5 (R69/R74) builds the coding event from the CPT #: guideline facts (never from keyword paragraphs alone) and picks a link's -#: relevance words by its kind; v4 keeps the earlier texts so its ledger +#: relevance words by its kind; v6 (R75/R78) cites facts by page and dates +#: deletions by CPT Changes, makes a telehealth event need a lineage +#: telehealth_list for the family's codes, and orders event-text paragraphs by +#: p_id and cuts them at sentences; v4 keeps the earlier texts so its ledger #: still replays -GENERATIONS = ("stance-v4", "stance-v5") +GENERATIONS = ("stance-v4", "stance-v5", "stance-v6") DEFAULT_GENERATION = "stance-v5" @@ -61,6 +64,12 @@ def guideline_text(generation: str | None = None) -> bool: return (generation or DEFAULT_GENERATION) != "stance-v4" +def fixed_text(generation: str | None = None) -> bool: + """Whether *generation* has the R78 event-text rules (v6): a telehealth + event needs lineage, paragraphs go by p_id and end at sentences.""" + return (generation or DEFAULT_GENERATION) == "stance-v6" + + PROMPT_VERSIONS = {m: prompt_version(m) for m in ("multi", "per-event")} DEFAULT_METHOD = "multi" DEFAULT_MODEL = "large" @@ -167,6 +176,7 @@ def _link_text(link: Any, rule_year: int, store: Any) -> str: _PARA_CHARS = 600 +_SENT = re.compile(r"(?<=[.!?])\s+(?=[A-Z(\[])") _PARAS = 2 _CODING_WORDS = re.compile(r"guideline|medical decision making|\bMDM\b|deleted", re.I) #: "time" is in the coding keywords but counts a quarter as much: prolonged- @@ -186,19 +196,36 @@ def _year_items(store: Any, rule_year: int) -> list[dict]: return [it for it in items if it["kind"] == "proposed"] or items -def _excerpt(text: str, words: re.Pattern[str]) -> str: +def _excerpt(text: str, words: re.Pattern[str], *, sentences: bool = False) -> str: """At most ``_PARA_CHARS`` verbatim characters of *text*: all of it, or the window with the most keyword hits (later windows win ties), cut at word - boundaries.""" + boundaries; with *sentences* (v6) the run of whole sentences with the most + hits, never cut mid-sentence (a sentence longer than the limit is cut at a + word).""" text = " ".join(text.split()) if len(text) <= _PARA_CHARS: return text + if sentences: + parts = _SENT.split(text) + best: tuple[int, int, int] | None = None + for i in range(len(parts)): + j, size = i, 0 + while j < len(parts) and size + len(parts[j]) + (j > i) <= _PARA_CHARS: + size += len(parts[j]) + (j > i) + j += 1 + if j > i: + hits = len(words.findall(" ".join(parts[i:j]))) + if best is None or (hits, i) >= (best[0], best[1]): + best = (hits, i, j) + if best is not None: + return " ".join(parts[best[1] : best[2]]) + return _excerpt(parts[0], words) hits = [m.start() for m in words.finditer(text)] or [0] - best = max( + best2 = max( (sum(1 for h in hits if lo <= h < lo + _PARA_CHARS), lo) for lo in (max(0, h - 100) for h in hits) )[1] - lo = text.find(" ", best) + 1 if best else 0 + lo = text.find(" ", best2) + 1 if best2 else 0 chunk = text[lo : lo + _PARA_CHARS] return chunk.rsplit(" ", 1)[0] if lo + _PARA_CHARS < len(text) else chunk @@ -209,12 +236,15 @@ def _rule_paragraphs( words: re.Pattern[str], phrases: re.Pattern[str] | None = None, weak: re.Pattern[str] | None = None, + *, + ordered: bool = False, ) -> str: """Up to ``_PARAS`` verbatim paragraphs of the rule that name the family (a code, ranges expanded, a link alias, or one of its pair phrases) together with one of *words*, by keyword density (then paragraph order), each cut to ``_PARA_CHARS`` and cited by its paragraph number. Hits of - *weak* count a quarter.""" + *weak* count a quarter. With *ordered* (v6) the chosen paragraphs are + shown by p_id and cut at sentence boundaries.""" scored = [] for _key, p_id, text in paras: if not ( @@ -225,8 +255,12 @@ def _rule_paragraphs( if n: scored.append((-n / max(1, len(text.split())), p_id, text)) scored.sort(key=lambda t: t[:2]) + chosen = scored[:_PARAS] + if ordered: + chosen.sort(key=lambda t: t[1]) return " ".join( - f"[\u00b6{p_id}] {_excerpt(text, words)}" for _d, p_id, text in scored[:_PARAS] + f"[\u00b6{p_id}] {_excerpt(text, words, sentences=ordered)}" + for _d, p_id, text in chosen ) @@ -258,6 +292,7 @@ def docket_events( store: Any = None, *, guidelines: bool | None = None, + fixed: bool | None = None, ) -> list[EventChoice]: """The family events a docket's rule touched, each with a deterministic text of what CMS proposed (R61), always ending in ``other``. *store* @@ -270,6 +305,8 @@ def docket_events( facts with their (edition, page) cites.""" if guidelines is None: guidelines = guideline_text() + if fixed is None: + fixed = fixed_text() key = family.key def ev(slug: str, text: str) -> EventChoice: @@ -288,7 +325,7 @@ def docket_events( def cited( text: str, words: re.Pattern[str], weak: re.Pattern[str] | None = None ) -> str: - found = _rule_paragraphs(paras, names, words, phrases, weak) + found = _rule_paragraphs(paras, names, words, phrases, weak, ordered=fixed) return f"{text}. Rule text: {found}" if found else text out: list[EventChoice] = [] @@ -358,9 +395,19 @@ def docket_events( bits.append(cpt) text = "Coding changes: " + ". ".join(bits) if bits else "Coding changes" out.append(ev("coding", cited(text, _CODING_WORDS, _TIME_WORD))) - tele = sorted({e.code for e in events if e.kind == "telehealth_list"}) + tele_events = list(events) + if fixed: + # R78b: the family's own codes, and the links offered this year + tele_events += [ + e + for ln in family.links + if ln.since == rule_year + for e in read_events(con, ln.target) + if e.year == rule_year + ] + tele = sorted({e.code for e in tele_events if e.kind == "telehealth_list"}) tele_paras = _rule_paragraphs(paras, names, _TELEHEALTH_WORDS, phrases) - if tele or tele_paras: + if tele or (tele_paras and not fixed): text = ( "Medicare Telehealth Services List status of this family's codes " f"({_compact([m.code for m in family.members if m.until is None])}) " @@ -923,6 +970,31 @@ class Scores: return self.cell_agreement >= cell and self.firm_accuracy >= firm +def golden_dockets(golden: Mapping[str, Any]) -> list[str]: + """The dockets a golden file covers: ``"dockets": [...]`` (multi-docket + format, each item names its own ``"docket"``) or the single ``"docket"``.""" + if "dockets" in golden: + return list(golden["dockets"]) + return [golden["docket"]] + + +def item_dockets(golden: Mapping[str, Any]) -> dict[str, str]: + """``item_key -> docket``. A multi-docket file's item must name a docket + the file lists; a single-docket file's items all belong to its docket.""" + if "dockets" not in golden: + return {it["item_key"]: golden["docket"] for it in golden["items"]} + listed = set(golden["dockets"]) + out = {} + for it in golden["items"]: + docket = it.get("docket") + if docket not in listed: + raise ValueError( + f"golden item {it['item_key']}: docket {docket!r} is not in dockets" + ) + out[it["item_key"]] = docket + return out + + def golden_labels(golden: Mapping[str, Any]) -> dict[tuple[str, str], str]: """``(item_key, event_key) -> label`` of a golden file.""" return { @@ -991,6 +1063,7 @@ def eval_record(golden: Mapping[str, Any], sc: Scores) -> dict[str, Any]: "n_items": len(golden["items"]), "n_positions": sc.positions, "family": golden["family"], - "docket": golden["docket"], + "docket": "; ".join(golden_dockets(golden)), + **({"dockets": golden_dockets(golden)} if "dockets" in golden else {}), "in_sample": bool(golden.get("in_sample", True)), } diff --git a/tests/cli/test_pfs_cli.py b/tests/cli/test_pfs_cli.py index 7f585c3..07bfc55 100644 --- a/tests/cli/test_pfs_cli.py +++ b/tests/cli/test_pfs_cli.py @@ -2384,6 +2384,85 @@ class TestStanceEval: def _run(self, path, *extra): return runner.invoke(app, ["pfs", "stance-eval", "--golden", str(path), *extra]) + MULTI = { + "family": "HOME-VISITS", + "dockets": ["D1", "D2"], + "items": [ + { + "item_key": "K1", + "docket": "D1", + "cells": {"HOME-VISITS:2023:G0318": {"label": "support"}}, + }, + { + "item_key": "K2", + "docket": "D2", + "cells": {"HOME-VISITS:2026:G2211": {"label": "support"}}, + }, + ], + } + + def test_multi_docket_golden_scores_each_item_against_its_docket( + self, con, monkeypatch, tmp_path + ): + from bib.dockets import Docket + from pfs.codetables import StanceRow + from pfs.stance import EventChoice + + path, _ = self._patch(monkeypatch, tmp_path, golden=self.MULTI) + store = type( + "S", + (), + { + "dockets": lambda self: [ + Docket(id="D1", comment_end_date="2022-09-06"), + Docket(id="D2", comment_end_date="2025-09-12"), + ] + }, + )() + monkeypatch.setattr(pfs_cli, "_store", lambda: store) + years = [] + monkeypatch.setattr( + pfs_cli, + "docket_events", + lambda con_, fam, year, store_, **kw: ( + years.append(year) + or [EventChoice(f"HOME-VISITS:{year}:other", "o", year)] + ), + ) + seen = [] + monkeypatch.setattr( + pfs_cli, + "family_comment_items", + lambda eng, codes, key, phrases, docket, **kw: [ + ("K1", "t1"), ("K2", "t2") + ], + ) # fmt: skip + + def fake(commenters, texts, events, ask, log, **kw): + seen.append(([c.item_key for c in commenters], kw["docket"], kw["limit"])) + k = commenters[0].item_key + y = events[0].rule_year + return ( + [StanceRow("HOME-VISITS", k, kw["docket"], f"HOME-VISITS:{y}:G0318" if k == "K1" else f"HOME-VISITS:{y}:G2211", "support", "", "", "x")], + {"done": 1, "skipped": 0, "failed": 0, "failed_pending": 0, "pending": 0, "no_text": 0}, + ) # fmt: skip + + monkeypatch.setattr(pfs_cli, "stances_for", fake) + res = self._run(path, "--limit", "1") + assert res.exit_code == 0, res.output + assert years == [2023, 2026] # each docket's own rule year + assert seen[0] == (["K1"], "D1", 1) + assert seen[1][:2] == (["K2"], "D2") + assert "2 new calls" in res.output + assert "recall 1.000 (2/2)" in res.output + # the call limit is shared: D2 only replays once D1 spent it + assert seen[1][2] == 0 + + def test_multi_docket_unknown_docket(self, con, monkeypatch, tmp_path): + path, _ = self._patch(monkeypatch, tmp_path, golden=self.MULTI) + res = self._run(path) + assert "unknown docket 'D2'" in res.output + def test_scores_and_prints_the_matrix(self, con, monkeypatch, tmp_path): path, seen = self._patch(monkeypatch, tmp_path) res = self._run( diff --git a/tests/pfs/test_curated.py b/tests/pfs/test_curated.py index b66e16e..37aa50b 100644 --- a/tests/pfs/test_curated.py +++ b/tests/pfs/test_curated.py @@ -201,7 +201,16 @@ def test_link_aliases_and_their_cite(): g = next(link for link in fam.links if link.target == "G0318") assert (g.aliases, g.alias_item_key, g.alias_p_id) == (("GXXX3",), "MZ24MX5S", 880) b = next(link for link in fam.links if link.target == "B1") - assert (b.aliases, b.alias_item_key, b.alias_p_id) == ((), "", 0) + assert (b.aliases, b.alias_item_key, b.alias_p_id) == (("MOD1",), "5ITGVDJV", 521) + b2 = next(link for link in fam.links if link.target == "B2") + assert (b2.aliases, b2.alias_item_key, b2.alias_p_id) == ( + ("MOD2",), + "5ITGVDJV", + 537, + ) + # the bare phrase admitted nursing-facility G2211 letters (R75); e/m stays + assert "home and residence e/m" in fam.pair_phrases + assert "home and residence" not in fam.pair_phrases def test_alias_parsing_and_errors(tmp_path):