fix(pfs): grounded guideline paraphrases (guideline-v2), CPT Changes-dated deletions, invented test fixtures, stance-v6 text rules, MOD1/MOD2 aliases, multi-docket golden eval

The grounding guard checks each paraphrase against its numbered sentence
(content-word coverage, unseen words, code ranges per action clause, NFKC copy
guard). Deletions cite CPT Changes 2023 pages; the Panel row is built from the
entry's code lists. Pair phrase: drop the bare 'home and residence'. stance-v6
is not shipped (golden recall 11/27 vs v5 14/27); v5 stays the default.
This commit is contained in:
kert
2026-11-12 17:53:14 -05:00
parent 31c9366b93
commit f9cfdef0b2
7 changed files with 581 additions and 123 deletions

View File

@@ -94,7 +94,10 @@ from pfs.stance import (
docket_rule_year,
eval_record,
event_subjects,
fixed_text,
golden_dockets,
guideline_text,
item_dockets,
prompt_version,
rollup,
score,
@@ -1523,7 +1526,12 @@ def stance(
con = _read()
try:
events = docket_events(
con, fam, year, store, guidelines=guideline_text(generation)
con,
fam,
year,
store,
guidelines=guideline_text(generation),
fixed=fixed_text(generation),
)
commenter_rows = [
c for c in read_commenters(con, family=fam.key) if c.docket == docket
@@ -1663,56 +1671,70 @@ def _golden_run(
fam = curated_family(normalize_key(data["family"]))
if fam is None:
raise typer.BadParameter(f"unknown curated family {data['family']!r}")
docket = data["docket"]
store = _store()
found = [d for d in store.dockets() if d.id == docket]
if not found:
raise typer.BadParameter(f"unknown docket {docket!r}")
con = _read()
try:
events = docket_events(
con,
fam,
docket_rule_year(found[0].comment_end_date),
store,
guidelines=guideline_text(generation),
)
finally:
con.close()
keys = [it["item_key"] for it in data["items"]]
texts = {
k: t
for k, t in family_comment_items(
_engine(),
fam.codes,
fam.key,
fam.pair_phrases,
of_docket = item_dockets(data)
all_rows: list[Any] = []
counts = {
"done": 0, "skipped": 0, "failed": 0, "failed_pending": 0, "pending": 0,
"no_text": 0,
} # fmt: skip
remaining = limit
for docket in golden_dockets(data):
found = [d for d in store.dockets() if d.id == docket]
if not found:
raise typer.BadParameter(f"unknown docket {docket!r}")
con = _read()
try:
events = docket_events(
con,
fam,
docket_rule_year(found[0].comment_end_date),
store,
guidelines=guideline_text(generation),
fixed=fixed_text(generation),
)
finally:
con.close()
keys = [k for k, d in of_docket.items() if d == docket]
texts = {
k: t
for k, t in family_comment_items(
_engine(),
fam.codes,
fam.key,
fam.pair_phrases,
docket=docket,
max_chars=TEXT_CHARS,
max_chunks=TEXT_CHUNKS,
)
if k in keys
}
# the call limit is shared across dockets; once spent, only replay
live = ask if (ask is not None and (not limit or remaining > 0)) else None
rows, got = stances_for(
[SimpleNamespace(item_key=k) for k in keys],
texts,
events,
live,
RunLog(ledger_path("stance", fam.key)),
family=fam.key,
docket=docket,
max_chars=TEXT_CHARS,
max_chunks=TEXT_CHUNKS,
method=method,
generation=generation,
model=name if live is None else None,
subjects=(
event_subjects(fam, events, by_kind=guideline_text(generation))
if relevance
else None
),
limit=remaining,
retry_failed=retry_failed,
progress=progress,
)
if k in keys
}
rows, counts = stances_for(
[SimpleNamespace(item_key=k) for k in keys],
texts,
events,
ask,
RunLog(ledger_path("stance", fam.key)),
family=fam.key,
docket=docket,
method=method,
generation=generation,
model=name if ask is None else None,
subjects=(
event_subjects(fam, events, by_kind=guideline_text(generation))
if relevance
else None
),
limit=limit,
retry_failed=retry_failed,
progress=progress,
)
remaining -= got["done"] + got["failed"]
all_rows += rows
counts = {k: counts[k] + got[k] for k in counts}
rows, counts = all_rows, counts
incomplete = counts["pending"] or counts["no_text"]
return (None if incomplete else score(data, rows)), counts

View File

@@ -484,7 +484,7 @@ class GuidelineFactRow:
``scope_point`` and ``panel_action``, a paraphrase of at most 240
characters; never book text. ``cite_item`` is the bib item of the book
(or the Panel summary), ``cite_locator`` its printed page ("p. 28",
"tab 10, p. 4"), or the section path where the edition carries no page
"tab 10, PDF p. 4"), or the section path where the edition carries no page
markers (R72). For ``scope_point``/``panel_action`` rows ``code`` is the
code range the passage covers ("99341-99350")."""

View File

@@ -5,7 +5,7 @@ HOME-VISITS:
synonyms: [home visit, home visits, home or residence, residence visit, domiciliary, rest home visit, house call, house calls]
# Phrases that qualify a Comment:/Response: pair besides a code (pfs.cms_summaries).
# Visit-naming only; each was checked against every phrase-only row it admits.
pair_phrases: [home or residence visit, home or residence services, home or residence e/m, residence visit, rest home visit, home visit codes, home visit e/m, house call, domiciliary care visit, visit in the home, home residence e/m, domiciliary care setting, domiciliary care settings, home visit pes, home or residence setting, home and residence e/m, home and residence]
pair_phrases: [home or residence visit, home or residence services, home or residence e/m, residence visit, rest home visit, home visit codes, home visit e/m, house call, domiciliary care visit, visit in the home, home residence e/m, domiciliary care setting, domiciliary care settings, home visit pes, home or residence setting, home and residence e/m]
members:
- {code: '99341', role: base, cite: {item_key: PETM225D, p_id: 1672}}
- {code: '99342', role: base, cite: {item_key: PETM225D, p_id: 1672}}
@@ -29,6 +29,6 @@ HOME-VISITS:
- {code: '99340', role: predecessor, until: 2022, cite: {rvu_last_year: 2022}}
links:
- {kind: add-on, target: G2211, since: 2026, cite: {item_key: NHRGIHGD, p_id: 1469}}
- {kind: modifier, target: B1, since: 2027, cite: {item_key: JCWARTJD, p_id: 1168}}
- {kind: modifier, target: B2, since: 2027, cite: {item_key: JCWARTJD, p_id: 1205}}
- {kind: modifier, target: B1, since: 2027, aliases: [MOD1], alias_cite: {item_key: 5ITGVDJV, p_id: 521}, cite: {item_key: JCWARTJD, p_id: 1168}}
- {kind: modifier, target: B2, since: 2027, aliases: [MOD2], alias_cite: {item_key: 5ITGVDJV, p_id: 537}, cite: {item_key: JCWARTJD, p_id: 1205}}
- {kind: prolonged-add-on, target: G0318, since: 2023, aliases: [GXXX3], alias_cite: {item_key: MZ24MX5S, p_id: 880}, cite: {item_key: PETM225D, p_id: 1694}}

View File

@@ -21,15 +21,19 @@ ledger stores only kept paraphrases, never book text.
from __future__ import annotations
import re
from collections.abc import Callable, Mapping, Sequence
import unicodedata
from collections.abc import Callable, Iterable, Mapping, Sequence
from pathlib import Path
from typing import Any
from pfs.codetables import GuidelineFactRow, is_missing_table_error
from pfs.origin import resolve_book, shares_run
from pfs.lineage import _expand_ranges
from pfs.origin import _DEL_BEFORE, book_deleted, book_support, resolve_book, shares_run
from pfs.runlog import RunLog, input_hash, ledger_path, run_units
GUIDELINE_PROMPT_VERSION = "guideline-v1"
GUIDELINE_PROMPT_VERSION = "guideline-v2"
CHANGES_ITEM = "3ANRT854" # "CPT Changes 2023": dates the deletions
CHANGES_YEAR = 2023
PANEL_ITEM = "F7SRZJD2" # "February 2021: CPT Editorial Summary of Panel Actions"
PANEL_TITLES = ("Home and Residence Services",)
MAX_POINTS = 6
@@ -44,39 +48,85 @@ SCHEMA: dict = {
"type": "array",
"items": {
"type": "object",
"required": ["summary"],
"properties": {"summary": {"type": "string", "maxLength": 240}},
"required": ["sentence", "summary"],
"properties": {
"sentence": {"type": "integer"},
"summary": {"type": "string", "maxLength": 240},
},
},
}
},
}
SYSTEM = (
"You read a passage of licensed coding-guideline text and state, in your own "
"words, what it says about where and for whom the listed codes are reported, "
"what counts toward their time, and any reporting rule it gives. Reply with "
'JSON {"points": [{"summary": ...}]}: at most four points, each one sentence '
"of at most 200 characters. Paraphrase: never copy eight or more consecutive "
"words of the passage, never quote, and never state a number the passage does "
"not contain. Say nothing the passage does not say, and never describe what "
"the passage lacks."
"You read numbered sentences of licensed coding-guideline text and state, in "
"your own words, what they say about where and for whom the listed codes are "
"reported and under what conditions. Reply with JSON "
'{"points": [{"sentence": k, "summary": ...}]}: at most four points; each '
"paraphrases ONE numbered sentence k, in one sentence of at most 200 "
"characters, and keeps that sentence's qualifiers and exceptions. Never copy "
"eight or more consecutive words, never quote, never state a number or a code "
"the sentence does not contain, and name a code only if that sentence names it. "
"Reuse the passage's own nouns, verbs and qualifiers and change only the "
"sentence structure (split or merge clauses, start with a different word, or turn a negative into a plain verb such as 'exclude'); use "
"no word the passage does not support. Say nothing the "
"sentence does not say, and never describe what the passage lacks."
)
PANEL_SYSTEM = (
"You read one entry of a CPT Editorial Panel summary of actions and state in "
"your own words what the Panel did to the listed codes and subsections and "
'when it takes effect. Reply with JSON {"points": [{"summary": ...}]}: one '
"point of at most 200 characters. Paraphrase: never copy eight or more "
"consecutive words, never quote, never state a number the entry does not "
"contain."
"You read the numbered sentences of one entry of a CPT Editorial Panel summary "
"of actions and state in your own words what the Panel did to the listed codes "
"and subsections and when it takes effect. Reply with JSON "
'{"points": [{"sentence": k, "summary": ...}]}: one point of at most 200 '
"characters. Keep each action with the codes it applies to (a deletion and a "
"revision name different codes), list codes exactly as the entry lists them and "
"write a range only if the entry does. Never copy eight or more consecutive words, "
"never quote, never state a number or code the entry does not contain."
)
_MARKS = re.compile("[▶◀■➢▲★●◆►◄▼\x00]")
_WS = re.compile(r"\s+")
_SENTENCE = re.compile(r"(?<=[.!?])\s+")
_NUMBER = re.compile(r"\d+")
#: a point about what the passage lacks says nothing
#: a point about what the passage lacks says nothing; a real point that
#: starts with "No ..." ("No travel time is counted") stays
_VOID = re.compile(
r"^(?:no|none|nothing)\b|\bnot (?:given|specified|stated|provided)\b", re.I
r"\b(?:passage|text|entry|sentences?)\b[^.]{0,40}\b(?:does not|doesn't|"
r"do not|no mention|says nothing|lacks)\b|"
r"^(?:no|none|nothing)\b[^.]{0,40}\b(?:information|details|rules|locations)\b"
r"[^.]{0,40}\b(?:given|provided|stated|specified|mentioned)\b",
re.I,
)
_STOP = frozenset(
"a an the and or of to in on at by for from with without as is are was were be "
"been being it its this that these those their there than then also into over "
"under per via not no nor but if when while which who whom whose what how any "
"all each other such can may must should will would could only both either "
"has have had do does did given like during including such same between "
"same".split()
)
#: generic words a paraphrase may use that the passage need not contain
_GENERIC = frozenset(
"apply applies use used report reported include includes cover covers covered "
"describe describes state states refer refers define defines require "
"requires provide provides indicate indicates mean means specify specifies "
"allow allows exclude excludes rule rules code codes service services "
"patient patients visit visits effective effect takes determine determines "
"determining select selects selecting choose chooses count counts omit omits "
"ignore ignores consider considers move moves".split()
)
_TOKEN = re.compile(r"[a-z]{3,}")
_GROUNDED = 0.6
#: a scope point that states a reporting rule is the more useful one
_RULE_POINT = re.compile(
r"\b(?:time|travel|report|admitted)\b|\bdo not\b|\bnot use\b", re.I
)
_ACTIONS = (
("deleted", re.compile(r"\b(?:delet\w*|remov\w*|retir\w*|eliminat\w*)", re.I)),
("revised", re.compile(r"\b(?:revis\w*|rewr\w*|chang\w*|updat\w*|amend\w*)", re.I)),
(
"created",
re.compile(r"\b(?:add\w*|creat\w*|introduc\w*|establish\w*|new)\b", re.I),
),
)
_YEAR = re.compile(r"\b(20\d{2})\b")
_CODE5 = re.compile(r"(?<!\d)(\d{5})(?!\d)")
@@ -89,7 +139,7 @@ _LEVEL_A = re.compile(
r"\b(straightforward|(low|moderate|high) level of) medical decision making", re.I
)
_LEVEL_B = re.compile(
r"medical decision making of (low|moderate to high|moderate|high) complexity", re.I
r"decision making of (low|moderate to high|moderate|high) complexity", re.I
)
_THRESHOLD = re.compile(r"(\d+) minutes must be met or exceeded", re.I)
_TYPICAL = re.compile(r"\btypically,? (\d+) minutes", re.I)
@@ -179,32 +229,118 @@ def sentence_page(
return max(before) if before else (hits[0] if hits else None)
def guarded_points(out: Any, text: str) -> list[str]:
"""The model's paraphrases that pass the guards: a string of 1..240
characters, no 8-word run shared with *text*, no number *text* lacks;
duplicates dropped."""
def _fold(text: str) -> str:
return unicodedata.normalize("NFKC", text).replace("\u2019", "'")
def _stem(word: str) -> str:
if word.endswith("ies") and len(word) > 5:
return word[:-3] + "y"
for suf in ("ions", "ion", "ing", "ed", "es", "s", "ly"):
if word.endswith(suf) and len(word) - len(suf) >= 4:
word = word[: -len(suf)]
break
return word[:-1] if word.endswith("e") and len(word) > 4 else word
def _content(text: str) -> list[str]:
"""Stemmed content words of *text*: stopwords and short tokens removed."""
return [_stem(w) for w in _TOKEN.findall(_fold(text).lower()) if w not in _STOP]
def sentences_of(text: str) -> list[str]:
return [s for s in _SENTENCE.split(norm(text)) if s]
def _clauses(text: str) -> list[str]:
return [c for c in re.split(r"[;.]", text) if c.strip()]
def _action(clause: str) -> str | None:
return next((a for a, pat in _ACTIONS if pat.search(clause)), None)
def _codes_ok(summary: str, window: str) -> bool:
"""Every code the summary's clause names (ranges expanded) appears in a
source clause with the same action; a clause with no action word needs the
code anywhere in the window."""
source = [
(_action(c), set(_CODE5.findall(_expand_ranges(c)))) for c in _clauses(window)
]
anywhere = set().union(*(codes for _, codes in source)) if source else set()
for clause in _clauses(summary):
action = _action(clause)
for code in set(_CODE5.findall(_expand_ranges(clause))):
pool = (
set().union(*(c for a, c in source if a == action))
if action
else anywhere
)
if code not in pool:
return False
return True
def grounded(summary: str, k: int, sentences: Sequence[str]) -> bool:
"""The paraphrase of numbered sentence *k* (1-based) is grounded: at least
60% of its content words occur in sentences k-1..k+1, none occurs nowhere
in the passage (generic paraphrase words aside), and each code it names
sits in a source clause with the same action."""
if not 1 <= k <= len(sentences):
return False
window = " ".join(sentences[max(0, k - 2) : k + 1])
words = _content(summary)
near = set(_content(window))
anywhere = set(_content(" ".join(sentences))) | {_stem(g) for g in _GENERIC}
if not words or any(w not in anywhere for w in words):
return False
if sum(w in near or w in _GENERIC_STEMS for w in words) / len(words) < _GROUNDED:
return False
return _codes_ok(summary, window)
_GENERIC_STEMS = frozenset(_stem(g) for g in _GENERIC)
def guarded(
out: Any, sentences: Sequence[str], guide: str = ""
) -> list[tuple[int, str]]:
"""``(sentence, paraphrase)`` of the model's paraphrases that pass the guards: a string of 1..240
characters naming a valid sentence, no 8-word run shared with *guide* (or
the sentences; NFKC-folded), no number the sentences lack, no empty
"the passage says nothing" point, grounded in its sentence
(:func:`grounded`); duplicates dropped."""
items = out.get("points") if isinstance(out, dict) else None
if not isinstance(items, list):
return []
have = set(_NUMBER.findall(text))
kept: list[str] = []
source = _fold(" ".join([*sentences, guide]))
have = set(_NUMBER.findall(source))
kept: list[tuple[int, str]] = []
for it in items:
s = it.get("summary") if isinstance(it, dict) else None
if not isinstance(s, str):
if not isinstance(it, dict):
continue
s, k = it.get("summary"), it.get("sentence")
if not isinstance(s, str) or isinstance(k, bool) or not isinstance(k, int):
continue
s = s.strip()
if (
s
and len(s) <= SUMMARY_CHARS
and not shares_run(s, text)
and not shares_run(_fold(s), source)
and set(_NUMBER.findall(s)) <= have
and not _VOID.search(s)
and s not in kept
and grounded(s, k, sentences)
and s not in [t for _, t in kept]
):
kept.append(s)
kept.append((k, s))
return kept
def guarded_points(out: Any, sentences: Sequence[str], guide: str = "") -> list[str]:
"""The paraphrases of :func:`guarded`, without their sentence numbers."""
return [s for _, s in guarded(out, sentences, guide)]
def panel_entry(pages: Sequence[str], title: str) -> tuple[str, int, str] | None:
"""``(tab, page number, entry text)`` of the Panel-summary entry named
*title*: the tab number precedes the name, the entry runs to the next
@@ -303,8 +439,33 @@ def _where(path: tuple) -> str:
return " > ".join(path)
def _change_page(
changes: Sequence[tuple[str, str]], code: str, deleted: frozenset[str]
) -> str | None:
"""First printed page of CPT Changes where it says *code* was deleted."""
for label, text in changes:
if label and "deleted" in book_support(text, code, deleted):
return label
return None
def _note_page(pages: Sequence[tuple[str, str]], code: str) -> str | None:
"""Printed page whose "(... have been deleted ...)" note names *code*."""
for label, text in pages:
if label and any(
code in _CODE5.findall(_expand_ranges(m.group(1)))
for m in _DEL_BEFORE.finditer(text)
):
return label
return None
def _facts(
family: str, rows: list[dict], pages: Mapping[int, list], counts: dict[str, int]
family: str,
rows: list[dict],
pages: Mapping[int, list],
counts: dict[str, int],
changes: Sequence[tuple[str, str]] = (),
) -> list[GuidelineFactRow]:
out: list[GuidelineFactRow] = []
@@ -338,15 +499,28 @@ def _facts(
for r in rows:
by_year.setdefault(r["year"], {})[r["code"]] = r
years = sorted(by_year)
deleted = book_deleted(changes)
for prev, year in zip(years, years[1:], strict=False):
here = by_year[year]
anchor = here[sorted(here)[0]]
where = locate(anchor)
for code in sorted(set(by_year[prev]) - set(here)):
# CPT Changes dates the deletion; the later book's own deletion
# note is the fallback for a code Changes does not name
page = (
_change_page(changes, code, deleted)
if prev < CHANGES_YEAR <= year
else None
)
if page is not None:
when, item, where = CHANGES_YEAR, CHANGES_ITEM, f"p. {page}"
else:
note = _note_page(pages[year], code)
when, item = year, anchor["item"]
where = f"p. {note}" if note else locate(anchor)
out.append(
GuidelineFactRow(
family, code, year, "deleted_in_edition", str(prev),
anchor["item"], where, BOOK_EXTRACTOR,
family, code, when, "deleted_in_edition", str(prev), item,
where, BOOK_EXTRACTOR,
)
) # fmt: skip
return out
@@ -398,14 +572,14 @@ def _scope_jobs(
"edition": year,
"item": members[0]["item"],
"code": _range(codes),
"text": " ".join(group),
"sentences": group,
"guide": norm(guide),
"system": SYSTEM,
"locator": f"p. {label}" if label else _where(path),
"user": (
f"CODES: {', '.join(codes)}\nSOURCE: CPT {year} guideline, "
f"{path[-1]}{f', page {label}' if label else ''}\n\n"
f"PASSAGE:\n{' '.join(group)}"
f"SENTENCES:\n{_numbered(group)}"
),
}
)
@@ -432,21 +606,83 @@ def _panel_jobs(
"edition": int(years[-1]) if years else 0,
"item": PANEL_ITEM,
"code": _range(named),
"text": text,
"codes": named,
"sentences": sentences_of(text),
"guide": text,
"system": PANEL_SYSTEM,
"locator": f"tab {tab}, p. {page}",
"user": f"CODES: {', '.join(named)}\nSOURCE: Panel summary, tab {tab}\n\nENTRY:\n{text}",
"locator": f"tab {tab}, PDF p. {page}",
"user": (
f"CODES: {', '.join(named)}\nSOURCE: Panel summary, tab {tab}\n\n"
f"SENTENCES:\n{_numbered(sentences_of(text))}"
),
}
)
return jobs
def _numbered(sentences: Sequence[str]) -> str:
return "\n".join(f"[{i}] {s}" for i, s in enumerate(sentences, 1))
#: model replies tried per unit: the guards drop most of a reply, so a unit
#: that kept nothing (or little) is asked again, up to this many times
ATTEMPTS = 6
ENOUGH = 4
def _ask_job(ask: Callable, job: dict) -> dict | None:
out = ask(job["system"], job["user"], SCHEMA)
if out is None:
"""Ask for the unit's paraphrases (up to ``ATTEMPTS`` replies) and keep
only those that pass the guards, so the ledger never holds copied or
ungrounded text. ``None`` when every reply failed."""
kept: list[tuple[int, str]] = []
answered = False
for _ in range(ATTEMPTS):
out = ask(job["system"], job["user"], SCHEMA)
if out is None:
continue
answered = True
for k, s in guarded(out, job["sentences"], job["guide"]):
if s not in [t for _, t in kept]:
kept.append((k, s))
if len(kept) >= ENOUGH:
break
if not answered:
return None
return {"points": [{"summary": s} for s in guarded_points(out, job["guide"])]}
return {"points": [{"sentence": k, "summary": s} for k, s in kept]}
_LIST = r"((?:\d{5}(?:\s*[-\u2013]\s*\d{5})?|[,\s]|and)+)"
_DELETION = re.compile(rf"deletion of (?:codes? )?{_LIST}", re.I)
_REVISION = re.compile(rf"revision of (?:codes? )?{_LIST}", re.I)
def panel_fallback(text: str, named: Sequence[str]) -> str | None:
"""A deterministic one-line statement of a Panel entry from its code lists
(values, not prose): the deleted codes, the revised codes and the
effective year, each code set to the family's codes the entry names.
``None`` when the entry lists neither."""
from pfs.stance import _compact # noqa: PLC0415 - stance imports this module
parts = []
for label, pat in (("deleted", _DELETION), ("revised", _REVISION)):
listed: set[str] = set()
for m in pat.finditer(text):
listed |= set(_CODE5.findall(_expand_ranges(m.group(1))))
codes = sorted(listed & set(named))
if codes:
parts.append(f"{label} codes {_compact(codes).replace(' and ', ', ')}")
if not parts:
return None
years = _YEAR.findall(text)
if re.search(r"subsections?|sections?", text, re.I):
parts.append(
"the matching subsection headings and guidelines were also deleted or revised"
)
if years:
parts.append(f"effective {years[-1]}")
out = "; ".join(parts)
out = out[0].upper() + out[1:] + "."
return out if len(out) <= SUMMARY_CHARS else None
def _interleave(groups: list[list[GuidelineFactRow]]) -> list[GuidelineFactRow]:
@@ -484,7 +720,8 @@ def guideline_facts(
for y in years
}
labelled = {y: p or [] for y, p in pages.items()}
facts = _facts(family.key, rows, labelled, run)
changes = (load_pages(store, CHANGES_ITEM) or []) if len(years) > 1 else []
facts = _facts(family.key, rows, labelled, run, changes)
model = ask.model if ask is not None else replay_model
if model is None:
return facts
@@ -515,9 +752,16 @@ def guideline_facts(
"panel_action" if j["unit"].startswith("panel:") else "scope_point",
s, j["item"], j["locator"], extractor,
)
for s in guarded_points(entry.output, j["guide"])
for s in guarded_points(entry.output, j["sentences"], j["guide"])
] # fmt: skip
if j["unit"].startswith("panel:"):
if not mine and (alt := panel_fallback(j["guide"], j["codes"])):
mine = [
GuidelineFactRow(
family.key, j["code"], j["edition"], "panel_action", alt,
j["item"], j["locator"], "cpt-panel",
)
] # fmt: skip
facts += mine[:1]
else:
scope.setdefault(j["edition"], []).append(mine)
@@ -529,6 +773,16 @@ def guideline_facts(
# ── the stance coding text ───────────────────────────────────────────
def _pages(locators: Iterable[str]) -> str:
""" "p. 9", "pp. 37-40" (a range once there are three or more pages), or the
locators joined when they are not printed pages."""
locs = sorted(set(locators), key=lambda x: (len(x), x))
nums = [int(m.group(1)) for x in locs if (m := re.fullmatch(r"p\. (\d+)", x))]
if len(nums) != len(locs) or len(nums) < 3:
return ", ".join(locs)
return f"pp. {min(nums)}\u2013{max(nums)}"
def cpt_change_text(
facts: Sequence[GuidelineFactRow], year: int, base_codes: Sequence[str]
) -> str:
@@ -559,13 +813,21 @@ def cpt_change_text(
new_b, old_b = of(eff, "selection_basis"), of(prior, "selection_basis")
new_t = {**of(eff, "typical_time_min"), **of(eff, "time_threshold_min")}
old_t = {**of(prior, "typical_time_min"), **of(prior, "time_threshold_min")}
gone = sorted(
gone = [
r.code
for r in facts
if r.fact == "deleted_in_edition" and prior < r.edition_year <= eff
)
]
if not gone and new_b == {c: old_b.get(c) for c in new_b} and new_t == old_t:
return ""
def pages_of(edition: int, fact: str) -> str:
return _pages(
r.cite_locator
for r in facts
if r.edition_year == edition and r.fact == fact
)
parts = []
for value in sorted(set(new_b.values())):
codes = sorted(c for c, v in new_b.items() if v == value)
@@ -574,30 +836,43 @@ def cpt_change_text(
)
text = (
f"{_compact(codes)} {'is' if len(codes) == 1 else 'are'} selected by "
f"{_BASIS_LABEL[value]}"
f"{_BASIS_LABEL[value]} (CPT {eff}, {pages_of(eff, 'selection_basis')})"
)
if old:
text += f" (CPT {prior}: {' / '.join(old)})"
text += f" (CPT {prior}, {pages_of(prior, 'selection_basis')}: {' / '.join(old)})"
parts.append(text)
if new_t:
kind = (
"total-time thresholds"
fact = (
"time_threshold_min"
if of(eff, "time_threshold_min")
else "typical times"
else "typical_time_min"
)
kind = (
"total-time thresholds" if fact == "time_threshold_min" else "typical times"
)
parts.append(
f"{kind} "
+ ", ".join(f"{c} {n}" for c, n in sorted(new_t.items()))
+ " minutes"
+ f" minutes (CPT {eff}, {pages_of(eff, fact)})"
)
dropped = [
r
for r in facts
if r.fact == "deleted_in_edition" and prior < r.edition_year <= eff
]
for key in sorted({(r.edition_year, r.cite_item) for r in dropped}):
mine = [r for r in dropped if (r.edition_year, r.cite_item) == key]
book = f"CPT Changes {key[0]}" if key[1] == CHANGES_ITEM else f"CPT {key[0]}"
locs = _pages(r.cite_locator for r in mine)
parts.append(
f"{_compact([r.code for r in mine])} deleted in CPT {key[0]} ({book}, {locs})"
)
if gone:
parts.append(f"{_compact(gone)} not listed in the CPT {eff} book")
text = f"CPT books on file, {prior} and {eff}: " + "; ".join(parts)
base = set(base_codes)
scope = sorted(
(r for r in facts if r.fact == "scope_point" and r.edition_year == eff
and r.code.split("-")[0] in base),
key=lambda r: (r.cite_locator, r.value),
key=lambda r: (not _RULE_POINT.search(r.value), r.cite_locator, r.value),
)[:2] # fmt: skip
if scope:
text += ". Scope: " + "; ".join(

View File

@@ -43,9 +43,12 @@ TASK = "stance"
METHODS = ("multi", "per-event")
#: prompt generations: v5 (R69/R74) builds the coding event from the CPT
#: guideline facts (never from keyword paragraphs alone) and picks a link's
#: relevance words by its kind; v4 keeps the earlier texts so its ledger
#: relevance words by its kind; v6 (R75/R78) cites facts by page and dates
#: deletions by CPT Changes, makes a telehealth event need a lineage
#: telehealth_list for the family's codes, and orders event-text paragraphs by
#: p_id and cuts them at sentences; v4 keeps the earlier texts so its ledger
#: still replays
GENERATIONS = ("stance-v4", "stance-v5")
GENERATIONS = ("stance-v4", "stance-v5", "stance-v6")
DEFAULT_GENERATION = "stance-v5"
@@ -61,6 +64,12 @@ def guideline_text(generation: str | None = None) -> bool:
return (generation or DEFAULT_GENERATION) != "stance-v4"
def fixed_text(generation: str | None = None) -> bool:
"""Whether *generation* has the R78 event-text rules (v6): a telehealth
event needs lineage, paragraphs go by p_id and end at sentences."""
return (generation or DEFAULT_GENERATION) == "stance-v6"
PROMPT_VERSIONS = {m: prompt_version(m) for m in ("multi", "per-event")}
DEFAULT_METHOD = "multi"
DEFAULT_MODEL = "large"
@@ -167,6 +176,7 @@ def _link_text(link: Any, rule_year: int, store: Any) -> str:
_PARA_CHARS = 600
_SENT = re.compile(r"(?<=[.!?])\s+(?=[A-Z(\[])")
_PARAS = 2
_CODING_WORDS = re.compile(r"guideline|medical decision making|\bMDM\b|deleted", re.I)
#: "time" is in the coding keywords but counts a quarter as much: prolonged-
@@ -186,19 +196,36 @@ def _year_items(store: Any, rule_year: int) -> list[dict]:
return [it for it in items if it["kind"] == "proposed"] or items
def _excerpt(text: str, words: re.Pattern[str]) -> str:
def _excerpt(text: str, words: re.Pattern[str], *, sentences: bool = False) -> str:
"""At most ``_PARA_CHARS`` verbatim characters of *text*: all of it, or the
window with the most keyword hits (later windows win ties), cut at word
boundaries."""
boundaries; with *sentences* (v6) the run of whole sentences with the most
hits, never cut mid-sentence (a sentence longer than the limit is cut at a
word)."""
text = " ".join(text.split())
if len(text) <= _PARA_CHARS:
return text
if sentences:
parts = _SENT.split(text)
best: tuple[int, int, int] | None = None
for i in range(len(parts)):
j, size = i, 0
while j < len(parts) and size + len(parts[j]) + (j > i) <= _PARA_CHARS:
size += len(parts[j]) + (j > i)
j += 1
if j > i:
hits = len(words.findall(" ".join(parts[i:j])))
if best is None or (hits, i) >= (best[0], best[1]):
best = (hits, i, j)
if best is not None:
return " ".join(parts[best[1] : best[2]])
return _excerpt(parts[0], words)
hits = [m.start() for m in words.finditer(text)] or [0]
best = max(
best2 = max(
(sum(1 for h in hits if lo <= h < lo + _PARA_CHARS), lo)
for lo in (max(0, h - 100) for h in hits)
)[1]
lo = text.find(" ", best) + 1 if best else 0
lo = text.find(" ", best2) + 1 if best2 else 0
chunk = text[lo : lo + _PARA_CHARS]
return chunk.rsplit(" ", 1)[0] if lo + _PARA_CHARS < len(text) else chunk
@@ -209,12 +236,15 @@ def _rule_paragraphs(
words: re.Pattern[str],
phrases: re.Pattern[str] | None = None,
weak: re.Pattern[str] | None = None,
*,
ordered: bool = False,
) -> str:
"""Up to ``_PARAS`` verbatim paragraphs of the rule that name the family
(a code, ranges expanded, a link alias, or one of its pair phrases)
together with one of *words*, by keyword density (then paragraph order),
each cut to ``_PARA_CHARS`` and cited by its paragraph number. Hits of
*weak* count a quarter."""
*weak* count a quarter. With *ordered* (v6) the chosen paragraphs are
shown by p_id and cut at sentence boundaries."""
scored = []
for _key, p_id, text in paras:
if not (
@@ -225,8 +255,12 @@ def _rule_paragraphs(
if n:
scored.append((-n / max(1, len(text.split())), p_id, text))
scored.sort(key=lambda t: t[:2])
chosen = scored[:_PARAS]
if ordered:
chosen.sort(key=lambda t: t[1])
return " ".join(
f"[\u00b6{p_id}] {_excerpt(text, words)}" for _d, p_id, text in scored[:_PARAS]
f"[\u00b6{p_id}] {_excerpt(text, words, sentences=ordered)}"
for _d, p_id, text in chosen
)
@@ -258,6 +292,7 @@ def docket_events(
store: Any = None,
*,
guidelines: bool | None = None,
fixed: bool | None = None,
) -> list[EventChoice]:
"""The family events a docket's rule touched, each with a deterministic
text of what CMS proposed (R61), always ending in ``other``. *store*
@@ -270,6 +305,8 @@ def docket_events(
facts with their (edition, page) cites."""
if guidelines is None:
guidelines = guideline_text()
if fixed is None:
fixed = fixed_text()
key = family.key
def ev(slug: str, text: str) -> EventChoice:
@@ -288,7 +325,7 @@ def docket_events(
def cited(
text: str, words: re.Pattern[str], weak: re.Pattern[str] | None = None
) -> str:
found = _rule_paragraphs(paras, names, words, phrases, weak)
found = _rule_paragraphs(paras, names, words, phrases, weak, ordered=fixed)
return f"{text}. Rule text: {found}" if found else text
out: list[EventChoice] = []
@@ -358,9 +395,19 @@ def docket_events(
bits.append(cpt)
text = "Coding changes: " + ". ".join(bits) if bits else "Coding changes"
out.append(ev("coding", cited(text, _CODING_WORDS, _TIME_WORD)))
tele = sorted({e.code for e in events if e.kind == "telehealth_list"})
tele_events = list(events)
if fixed:
# R78b: the family's own codes, and the links offered this year
tele_events += [
e
for ln in family.links
if ln.since == rule_year
for e in read_events(con, ln.target)
if e.year == rule_year
]
tele = sorted({e.code for e in tele_events if e.kind == "telehealth_list"})
tele_paras = _rule_paragraphs(paras, names, _TELEHEALTH_WORDS, phrases)
if tele or tele_paras:
if tele or (tele_paras and not fixed):
text = (
"Medicare Telehealth Services List status of this family's codes "
f"({_compact([m.code for m in family.members if m.until is None])}) "
@@ -923,6 +970,31 @@ class Scores:
return self.cell_agreement >= cell and self.firm_accuracy >= firm
def golden_dockets(golden: Mapping[str, Any]) -> list[str]:
"""The dockets a golden file covers: ``"dockets": [...]`` (multi-docket
format, each item names its own ``"docket"``) or the single ``"docket"``."""
if "dockets" in golden:
return list(golden["dockets"])
return [golden["docket"]]
def item_dockets(golden: Mapping[str, Any]) -> dict[str, str]:
"""``item_key -> docket``. A multi-docket file's item must name a docket
the file lists; a single-docket file's items all belong to its docket."""
if "dockets" not in golden:
return {it["item_key"]: golden["docket"] for it in golden["items"]}
listed = set(golden["dockets"])
out = {}
for it in golden["items"]:
docket = it.get("docket")
if docket not in listed:
raise ValueError(
f"golden item {it['item_key']}: docket {docket!r} is not in dockets"
)
out[it["item_key"]] = docket
return out
def golden_labels(golden: Mapping[str, Any]) -> dict[tuple[str, str], str]:
"""``(item_key, event_key) -> label`` of a golden file."""
return {
@@ -991,6 +1063,7 @@ def eval_record(golden: Mapping[str, Any], sc: Scores) -> dict[str, Any]:
"n_items": len(golden["items"]),
"n_positions": sc.positions,
"family": golden["family"],
"docket": golden["docket"],
"docket": "; ".join(golden_dockets(golden)),
**({"dockets": golden_dockets(golden)} if "dockets" in golden else {}),
"in_sample": bool(golden.get("in_sample", True)),
}

View File

@@ -2384,6 +2384,85 @@ class TestStanceEval:
def _run(self, path, *extra):
return runner.invoke(app, ["pfs", "stance-eval", "--golden", str(path), *extra])
MULTI = {
"family": "HOME-VISITS",
"dockets": ["D1", "D2"],
"items": [
{
"item_key": "K1",
"docket": "D1",
"cells": {"HOME-VISITS:2023:G0318": {"label": "support"}},
},
{
"item_key": "K2",
"docket": "D2",
"cells": {"HOME-VISITS:2026:G2211": {"label": "support"}},
},
],
}
def test_multi_docket_golden_scores_each_item_against_its_docket(
self, con, monkeypatch, tmp_path
):
from bib.dockets import Docket
from pfs.codetables import StanceRow
from pfs.stance import EventChoice
path, _ = self._patch(monkeypatch, tmp_path, golden=self.MULTI)
store = type(
"S",
(),
{
"dockets": lambda self: [
Docket(id="D1", comment_end_date="2022-09-06"),
Docket(id="D2", comment_end_date="2025-09-12"),
]
},
)()
monkeypatch.setattr(pfs_cli, "_store", lambda: store)
years = []
monkeypatch.setattr(
pfs_cli,
"docket_events",
lambda con_, fam, year, store_, **kw: (
years.append(year)
or [EventChoice(f"HOME-VISITS:{year}:other", "o", year)]
),
)
seen = []
monkeypatch.setattr(
pfs_cli,
"family_comment_items",
lambda eng, codes, key, phrases, docket, **kw: [
("K1", "t1"), ("K2", "t2")
],
) # fmt: skip
def fake(commenters, texts, events, ask, log, **kw):
seen.append(([c.item_key for c in commenters], kw["docket"], kw["limit"]))
k = commenters[0].item_key
y = events[0].rule_year
return (
[StanceRow("HOME-VISITS", k, kw["docket"], f"HOME-VISITS:{y}:G0318" if k == "K1" else f"HOME-VISITS:{y}:G2211", "support", "", "", "x")],
{"done": 1, "skipped": 0, "failed": 0, "failed_pending": 0, "pending": 0, "no_text": 0},
) # fmt: skip
monkeypatch.setattr(pfs_cli, "stances_for", fake)
res = self._run(path, "--limit", "1")
assert res.exit_code == 0, res.output
assert years == [2023, 2026] # each docket's own rule year
assert seen[0] == (["K1"], "D1", 1)
assert seen[1][:2] == (["K2"], "D2")
assert "2 new calls" in res.output
assert "recall 1.000 (2/2)" in res.output
# the call limit is shared: D2 only replays once D1 spent it
assert seen[1][2] == 0
def test_multi_docket_unknown_docket(self, con, monkeypatch, tmp_path):
path, _ = self._patch(monkeypatch, tmp_path, golden=self.MULTI)
res = self._run(path)
assert "unknown docket 'D2'" in res.output
def test_scores_and_prints_the_matrix(self, con, monkeypatch, tmp_path):
path, seen = self._patch(monkeypatch, tmp_path)
res = self._run(

View File

@@ -201,7 +201,16 @@ def test_link_aliases_and_their_cite():
g = next(link for link in fam.links if link.target == "G0318")
assert (g.aliases, g.alias_item_key, g.alias_p_id) == (("GXXX3",), "MZ24MX5S", 880)
b = next(link for link in fam.links if link.target == "B1")
assert (b.aliases, b.alias_item_key, b.alias_p_id) == ((), "", 0)
assert (b.aliases, b.alias_item_key, b.alias_p_id) == (("MOD1",), "5ITGVDJV", 521)
b2 = next(link for link in fam.links if link.target == "B2")
assert (b2.aliases, b2.alias_item_key, b2.alias_p_id) == (
("MOD2",),
"5ITGVDJV",
537,
)
# the bare phrase admitted nursing-facility G2211 letters (R75); e/m stays
assert "home and residence e/m" in fam.pair_phrases
assert "home and residence" not in fam.pair_phrases
def test_alias_parsing_and_errors(tmp_path):